@l10nmonster/helpers-lqaboss 3.0.0-alpha.1

package/README.md

@@ -0,0 +1,53 @@
+# @l10nmonster/helpers-lqaboss
+
+Helper module for L10n Monster that provides LQA Boss flow capture functionality.
+
+## Features
+
+- Captures web pages with screenshots and text metadata
+- Creates `.lqaboss` files containing flow data
+- Supports multi-page flows with interactive capture
+
+## Usage
+
+This module exports an action that can be used with L10n Monster:
+
+```javascript
+import { actions } from '@l10nmonster/helpers-lqaboss';
+
+// Use the lqaboss action
+await actions.lqaboss.action(mm, {
+    url: 'https://example.com',
+    flowName: 'my-flow'
+});
+```
+
+## Testing
+
+This module includes unit tests using Node.js built-in testing facilities.
+
+### Running Tests
+
+```bash
+npm test
+```
+
+### Test Structure
+
+The tests are located in `test/index.test.js` and cover:
+
+- Help structure validation
+- Action method interface verification  
+- Filename sanitization logic
+- Module exports validation
+
+### Test Requirements
+
+- Node.js 18+ (for built-in test runner)
+- No additional test dependencies required
+
+## Dependencies
+
+- `puppeteer` - For browser automation and page capture
+- `jszip` - For creating compressed flow files
+- `@l10nmonster/core` - Core L10n Monster functionality

package/flowCapture.js

@@ -0,0 +1,238 @@
+import JSZip from 'jszip';
+import puppeteer from 'puppeteer';
+import { logInfo, logVerbose } from '@l10nmonster/core';
+
+// --- Function to be evaluated in browser context ---
+async function extractTextAndMetadataInPageContext() {
+    function fe00RangeToUtf8_browser(encoded) {
+        const encodingOffset = 0xfe00;
+        const decoder = new TextDecoder();
+        const length = encoded.length;
+        if (length % 2 !== 0) throw new Error('Invalid fe00 encoded input length');
+        const bytes = new Uint8Array(length / 2);
+        let byteIndex = 0;
+        for (let i = 0; i < length; i += 2) {
+            const highNibble = encoded.charCodeAt(i) - encodingOffset;
+            const lowNibble = encoded.charCodeAt(i + 1) - encodingOffset;
+            if (highNibble < 0 || highNibble > 15 || lowNibble < 0 || lowNibble > 15) {
+                throw new Error('Invalid char code in fe00 encoded input');
+            }
+            bytes[byteIndex++] = (highNibble << 4) | lowNibble;
+        }
+        return decoder.decode(bytes);
+    }
+
+    const textElements = [];
+    const START_MARKER_REGEX = /(?<![''<])\u200B([\uFE00-\uFE0F]+)/g;
+    const END_MARKER = '\u200B';
+
+    if (!document.body) {
+        return { error: 'Document body not found.' };
+    }
+
+    const treeWalker = document.createTreeWalker(document.body, NodeFilter.SHOW_TEXT, null);
+    
+    let activeSegment = null;
+    let node;
+
+    while (node = treeWalker.nextNode()) {
+        const parentElement = node.parentElement;
+        if (parentElement) {
+            const styles = window.getComputedStyle(parentElement);
+            if (styles.display === 'none' || styles.visibility === 'hidden' || parseFloat(styles.opacity) === 0) continue;
+            if (['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'HEAD'].includes(parentElement.tagName)) continue;
+        } else {
+            continue;
+        }
+
+        let searchPos = 0;
+        const text = node.nodeValue;
+
+        while (searchPos < text.length) {
+            if (activeSegment) {
+                const endMarkerPos = text.indexOf(END_MARKER, searchPos);
+
+                if (endMarkerPos !== -1) {
+                    activeSegment.text += text.substring(searchPos, endMarkerPos);
+                    
+                    const range = document.createRange();
+                    range.setStart(activeSegment.startNode, activeSegment.startOffset);
+                    range.setEnd(node, endMarkerPos);
+                    
+                    const rect = range.getBoundingClientRect();
+                    if (rect.width > 0 || rect.height > 0) {
+                        let parsedMetadata = {};
+                        try {
+                            const decodedJsonMetadata = fe00RangeToUtf8_browser(activeSegment.encodedMetadata);
+                            if (decodedJsonMetadata && decodedJsonMetadata.trim() !== '') {
+                                 parsedMetadata = JSON.parse(decodedJsonMetadata);
+                            }
+                        } catch (e) { parsedMetadata.decodingError = e.message; }
+
+                        textElements.push({
+                            text: activeSegment.text,
+                            x: rect.left + window.scrollX,
+                            y: rect.top + window.scrollY,
+                            width: rect.width,
+                            height: rect.height,
+                            ...parsedMetadata
+                        });
+                    }
+                    
+                    searchPos = endMarkerPos + 1;
+                    activeSegment = null;
+                } else {
+                    activeSegment.text += text.substring(searchPos);
+                    break; 
+                }
+            } else {
+                START_MARKER_REGEX.lastIndex = searchPos;
+                const match = START_MARKER_REGEX.exec(text);
+
+                if (match) {
+                    const textAfterStart = text.substring(match.index + match[0].length);
+                    const endMarkerPosInSubstring = textAfterStart.indexOf(END_MARKER);
+
+                    if (endMarkerPosInSubstring !== -1) {
+                        const capturedText = textAfterStart.substring(0, endMarkerPosInSubstring);
+                        
+                        const range = document.createRange();
+                        range.setStart(node, match.index);
+                        const endOffset = match.index + match[0].length + endMarkerPosInSubstring;
+                        range.setEnd(node, endOffset);
+
+                        const rect = range.getBoundingClientRect();
+                        if (rect.width > 0 || rect.height > 0) {
+                            let parsedMetadata = {};
+                            try {
+                                const decodedJsonMetadata = fe00RangeToUtf8_browser(match[1]);
+                                if (decodedJsonMetadata && decodedJsonMetadata.trim() !== '') {
+                                     parsedMetadata = JSON.parse(decodedJsonMetadata);
+                                }
+                            } catch (e) { parsedMetadata.decodingError = e.message; }
+    
+                            textElements.push({
+                                text: capturedText,
+                                x: rect.left + window.scrollX,
+                                y: rect.top + window.scrollY,
+                                width: rect.width,
+                                height: rect.height,
+                                ...parsedMetadata
+                            });
+                        }
+                        searchPos = endOffset + 1;
+                    } else {
+                        activeSegment = {
+                            startNode: node,
+                            startOffset: match.index,
+                            encodedMetadata: match[1],
+                            text: textAfterStart
+                        };
+                        break;
+                    }
+                } else {
+                    break;
+                }
+            }
+        }
+    }
+    return { textElements };
+}
+
+export class FlowSnapshotter {
+    constructor(startUrl, flowNameBase, options = {}) {
+        this.browser = null;
+        this.startUrl = startUrl;
+        this.flowNameBase = flowNameBase;
+        this.headless = options.headless ?? false;
+        this.capturedPagesData = [];
+        this.pageCounter = 0;
+    }
+
+    async startFlow() {
+        this.browser = await puppeteer.launch({ headless: this.headless, defaultViewport: null });
+        this.page = await this.browser.newPage();
+        await this.page.goto(this.startUrl, { waitUntil: 'networkidle2', timeout: 60000 });
+    }
+
+    async capturePage() {
+        if (!this.browser) {
+            throw new Error('Browser not started. Call startFlow first.');
+        }
+
+        await this.page.bringToFront();
+
+        const screenshotBuffer = await this.page.screenshot({
+            fullPage: true,
+            type: 'png'
+        });
+
+        const currentPageUrl = this.page.url();
+        const textDataResult = await this.page.evaluate(extractTextAndMetadataInPageContext);
+        if (textDataResult.error) {
+            throw new Error(`Error extracting text data: ${textDataResult.error}`);
+        } else if (!textDataResult.textElements || textDataResult.textElements.length === 0) {
+            logInfo`No LQA metadata segments found on page ${currentPageUrl}`;
+        }
+        logVerbose`Page ${this.pageCounter} captured: ${currentPageUrl}`;
+        const capturedData = {
+            url: currentPageUrl,
+            timestamp: new Date().toISOString(),
+            screenshotBuffer,
+            text_content: textDataResult.textElements || [],
+            id: `page_${Date.now()}_${this.pageCounter++}`,
+        };
+        this.capturedPagesData.push(capturedData);
+        return capturedData;
+    }
+
+    async endFlow(tm) {
+        if (this.browser) {
+            await this.browser.close();
+            logVerbose`Browser closed.`;
+            this.browser = null;
+            this.page = null;
+        }
+        if (this.capturedPagesData.length === 0) {
+            return null;
+        }
+        logVerbose`Total pages in flow: ${this.capturedPagesData.length}`;
+        const zip = new JSZip();
+        const flowMetadata = {
+            flowName: this.flowNameBase,
+            createdAt: new Date().toISOString(),
+            pages: []
+        };
+        const guids = new Set();
+        this.capturedPagesData.forEach((pData, index) => {
+            const imageName = `page_${index + 1}_${pData.id}.png`;
+            zip.file(imageName, pData.screenshotBuffer);
+            flowMetadata.pages.push({
+                pageId: pData.id,
+                originalUrl: pData.url,
+                timestamp: pData.timestamp,
+                imageFile: imageName,
+                segments: pData.text_content
+            });
+            pData.text_content.forEach(segment => segment.g && guids.add(segment.g));
+        });
+        zip.file('flow_metadata.json', JSON.stringify(flowMetadata, null, 2));
+        if (tm) {
+            const job = {
+                sourceLang: tm.sourceLang,
+                targetLang: tm.targetLang,
+                tus: [],
+            };
+            guids.forEach(guid => {
+                const tu = tm.getEntryByGuid(guid);
+                tu && job.tus.push(tu);
+            });
+            if (job.tus.length > 0) {
+                zip.file('job.json', JSON.stringify(job, null, 2));
+            }
+            logVerbose`${guids.size} guids captured in flow, ${job.tus.length} found in TM`;
+        }
+        const buffer = await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE', compressionOptions: { level: 6 } });
+        return buffer;
+    }
+}

package/index.js

@@ -0,0 +1,2 @@
+export { LQABossActions } from './lqabossActions.js';
+export { LQABossProvider } from './lqabossProvider.js';

package/lqabossActions.js

@@ -0,0 +1,10 @@
+import { lqaboss_capture } from './lqabossCapture.js';
+
+export class LQABossActions {
+    static name = 'lqaboss';
+    static help = {
+        description: 'Actions to integrate with LQA Boss.',
+    };
+
+    static subActions = [ lqaboss_capture ];
+}

package/lqabossCapture.js

@@ -0,0 +1,97 @@
+import fs from 'fs';
+import readline from 'readline';
+import { consoleLog } from '@l10nmonster/core';
+import { FlowSnapshotter } from './flowCapture.js';
+
+export class lqaboss_capture {
+    static help = {
+        description: 'create an lqaboss flow.',
+        arguments: [
+            ['<url>', 'the url of the page to capture'],
+            ['<flowName>', 'the name of the flow'],
+        ],
+        options: [
+            [ '--lang <srcLang,tgtLang>', 'source and target language pair' ],
+        ],
+    };
+
+    static async action(mm, options) {
+        if (!options.url || !options.flowName) {
+            throw new Error('You must specify a url and a flowName');
+        }
+        const langPairs = options.lang ? (Array.isArray(options.lang) ? options.lang : options.lang.split(',')) : null;
+        let tm;
+        if (langPairs) {
+            const [ sourceLang, targetLang ] = langPairs;
+            tm = mm.tmm.getTM(sourceLang, targetLang);
+        }
+        // Run the capture flow
+        const lqaBossBuffer = await runCapture(options.url, options.flowName, tm);
+        if (lqaBossBuffer) {
+            const filename = `${options.flowName.replace(/[^a-z0-9_.-]/gi, '_')}.lqaboss`;
+            await fs.promises.writeFile(filename, lqaBossBuffer);
+            consoleLog`Flow successfully saved as ${filename}`;
+        } else {
+            console.log('No pages were captured. Nothing to save.');
+        }
+    }
+}
+
+async function runCapture(startUrl, flowNameBase, tm) {
+    const snapShotter = new FlowSnapshotter(startUrl, flowNameBase);
+    try {
+        consoleLog`Navigating to ${startUrl}...`;
+        await snapShotter.startFlow();
+        consoleLog`----------------------------------------------------`;
+        consoleLog` LQA Boss CLI Capture Mode`;
+        consoleLog`----------------------------------------------------`;
+        consoleLog` - Interact with the opened Chrome window to navigate.`;
+        consoleLog` - Return to this terminal to issue commands.`;
+        consoleLog` - Press ENTER to capture the current page.`;
+        consoleLog` - Type 'q' then ENTER to finish and save the flow.`;
+        consoleLog`----------------------------------------------------`;
+        const rl = readline.createInterface({
+            input: process.stdin,
+            output: process.stdout,
+            prompt: '> ' // Optional: show a prompt character
+        });
+        const question = (query) => new Promise(resolve => rl.question(query, resolve));
+        // rl.on('close', () => {
+        //     consoleLog`Input stream closed.`;
+        // });
+
+        while (true) {
+            const answer = await question('Press ENTER to capture, or type "q" then ENTER to quit: ');
+
+            if (answer.toLowerCase().trim() === 'q') {
+                consoleLog`Quit command received.`;
+                break; // Exit the loop to proceed to saving
+            }
+
+            // Any input that is just ENTER (empty string after trim) or anything not 'q'
+            // will be treated as a capture command.
+            // We can be more explicit if needed, but empty string for ENTER is common.
+            if (answer.trim() === '' || answer.toLowerCase().trim() !== 'q') {
+                if (answer.trim() !== '') {
+                    consoleLog`Input "${answer.trim()}" received, treating as CAPTURE command.`;
+                } else {
+                    consoleLog`ENTER key received, treating as CAPTURE command.`;
+                }
+
+                consoleLog`Capturing current page...`;
+                try {
+                    await snapShotter.capturePage();
+                } catch (err) {
+                    console.error("! Error during capture:", err.message);
+                }
+            }
+            // No 'else' needed here as we only break on 'q' or proceed with capture.
+        }
+
+        rl.close(); // Close readline before saving
+    } catch (error) {
+        console.error("An error occurred:", error);
+        return null;
+    }
+    return await snapShotter.endFlow(tm);
+}

package/lqabossProvider.js

@@ -0,0 +1,74 @@
+import JSZip from 'jszip';
+import { getRegressionMode, providers, logVerbose, logInfo, styleString, utils, opsManager, logError, logWarn } from '@l10nmonster/core';
+
+/**
+ * @typedef {object} LQABossProviderOptions
+ * @extends BaseTranslationProvider
+ * @property {Object} delegate - Required file store delegate implementing file operations
+ */
+
+/**
+ * Provider for LQA Boss.
+ */
+export class LQABossProvider extends providers.BaseTranslationProvider {
+    #storageDelegate;
+    #opNames = {};
+
+    /**
+     * Initializes a new instance of the LQABossProvider class.
+     * @param {LQABossProviderOptions} options - Configuration options for the provider.
+     */
+    constructor({ delegate, ...options }) {
+        super(options);
+        this.#storageDelegate = delegate;
+        this.#opNames.startReviewOp = `${this.id}.startReviewOp`;
+        this.#opNames.continueReviewOp = `${this.id}.continueReviewOp`;
+        this.#opNames.completeReviewOp = `${this.id}.completeReviewOp`;
+        opsManager.registerOp(this.startReviewOp.bind(this), { opName: this.#opNames.startReviewOp, idempotent: false });
+        opsManager.registerOp(this.continueReviewOp.bind(this), { opName: this.#opNames.continueReviewOp, idempotent: true });
+        opsManager.registerOp(this.completeReviewOp.bind(this), { opName: this.#opNames.completeReviewOp, idempotent: true });
+    }
+
+    createTask(job) {
+        logVerbose`LQABossProvider creating task for job ${job.jobGuid}`;
+        const requestTranslationsTask = opsManager.createTask(this.id, this.#opNames.completeReviewOp);
+        requestTranslationsTask.rootOp.enqueue(this.#opNames.startReviewOp, { job });
+        return requestTranslationsTask;
+    }
+
+    async startReviewOp(op) {
+        const { job } = op.args;
+        const zip = new JSZip();
+        zip.file('job.json', JSON.stringify(job, null, 2));
+        const buffer = await zip.generateAsync({ type: 'nodebuffer', compression: 'DEFLATE', compressionOptions: { level: 6 } });
+        const filename = `${job.jobGuid}.lqaboss`;
+        await this.#storageDelegate.saveFile(filename, buffer);
+        logVerbose`LQABoss file ${filename} with ${job.tus.length} guids and ${buffer.length} bytes saved`;
+        op.parentTask.rootOp.enqueue(this.#opNames.continueReviewOp, { jobGuid: job.jobGuid });
+    }
+
+    /**
+     * Fetches the completed review job. This will error out until the review is complete.
+     * @param {object} op - The operation context containing fetch parameters.
+     * @returns {Promise<*>} The job response.
+     */
+    async continueReviewOp(op) {
+        const filename = `${op.args.jobGuid}.json`;
+        logVerbose`Trying to fetch completed LQABoss file ${filename}`;
+        return JSON.parse(await this.#storageDelegate.getFile(filename));
+    }
+
+    async completeReviewOp(op) {
+        const { tus, ...jobResponse } = op.inputs[1]; // the second op should be continueReviewOp
+        jobResponse.status = 'done';
+        const ts = getRegressionMode() ? 1 : new Date().getTime();
+        jobResponse.tus = tus.map(tu => ({ ...tu, ts, q: this.quality }));
+        return jobResponse;
+    }
+
+    async info() {
+        const info = await super.info();
+        info.description.push(styleString`Storage delegate: ${this.#storageDelegate.toString()}`);
+        return info;
+    }
+}

package/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "@l10nmonster/helpers-lqaboss",
+  "version": "3.0.0-alpha.1",
+  "description": "LQA Boss helper for L10n Monster",
+  "main": "index.js",
+  "type": "module",
+  "scripts": {
+    "start": "node index.js",
+    "test": "node --test test/*.test.js"
+  },
+  "dependencies": {
+    "jszip": "^3.10.1",
+    "puppeteer": "^24"
+  },
+  "peerDependencies": {
+            "@l10nmonster/core": "file:../core"
+  }
+}