@l10nmonster/helpers-translated 1.0.0 → 3.0.0-alpha.2

package/README.md

@@ -0,0 +1,61 @@
+# @l10nmonster/helpers-translated
+
+L10n Monster helpers for integrating with Translated.com services including Modern MT and Lara.
+
+## Installation
+
+```bash
+npm install @l10nmonster/helpers-translated
+```
+
+## Providers
+
+### MMTProvider
+
+Machine translation provider using Modern MT API with both real-time and batch translation support.
+
+```javascript
+import { MMTProvider } from '@l10nmonster/helpers-translated';
+
+const mmtProvider = new MMTProvider({
+    apiKey: 'your-mmt-api-key',
+    // Additional configuration options
+});
+```
+
+### LaraProvider
+
+Human translation provider using Lara API for professional translation services.
+
+```javascript
+import { LaraProvider } from '@l10nmonster/helpers-translated';
+
+const laraProvider = new LaraProvider({
+    apiKey: 'your-lara-api-key',
+    // Additional configuration options
+});
+```
+
+## Features
+
+- **Modern MT Integration**: High-quality machine translation with customizable models
+- **Lara Integration**: Professional human translation services
+- **Batch Processing**: Efficient handling of large translation jobs
+- **Real-time Translation**: Immediate translation for smaller content
+- **Cost Tracking**: Built-in billing and usage monitoring
+
+## Dependencies
+
+- `@translated/lara`: Official Lara API client
+- `modernmt`: Modern MT API client
+- `@l10nmonster/core`: Core L10n Monster functionality (peer dependency)
+
+## Testing
+
+The package includes comprehensive tests for Modern MT functionality:
+
+```bash
+npm test
+```
+
+Test artifacts are included for validation of API responses and job processing.

package/index.js

@@ -1,2 +1,3 @@
-exports.TranslationOS = require('./translationOS');
-exports.ModernMT = require('./modernMT');
+// export { TranslationOS } from './translationOS.js';
+export { MMTProvider } from './mmtProvider.js';
+export { LaraProvider } from './laraProvider.js';

package/laraProvider.js

@@ -0,0 +1,92 @@
+import { logWarn, providers, styleString } from '@l10nmonster/core';
+import { Credentials, Translator } from '@translated/lara';
+
+/**
+ * @typedef {object} LaraProviderOptions
+ * @extends ChunkedRemoteTranslationProviderOptions
+ * @property {string} keyId - The Lara API key id. This is required.
+ * @property {string} [keySecret] - The Lara API key secret. Optional, but often needed for authentication.
+ * @property {string|Array<string>} [adaptTo] - An optional single translation memory ID or an array of IDs to adapt translations to.
+ * @property {number} [maxChunkSize=60] - Maximum number of text segments (strings) allowed in a single API request chunk. Defaults to 60 if not provided.
+ */
+
+/**
+ * Provider for Translated Lara MT.
+ */
+export class LaraProvider extends providers.ChunkedRemoteTranslationProvider {
+    #keyId;
+    #keySecret;
+    #adaptTo;
+    #lara;
+    #translateOptions;
+
+    /**
+     * Initializes a new instance of the LaraProvider class.
+     * @param {LaraProviderOptions} options - Configuration options for the provider.
+     */
+    constructor({ keyId, keySecret, adaptTo, ...options }) {
+        super({ maxChunkSize: 60, ...options }); // maximum number of strings sent to Lara is 128 including notes
+        this.#keyId = keyId;
+        this.#keySecret = keySecret;
+        this.#adaptTo = adaptTo && (Array.isArray(adaptTo) ? adaptTo : adaptTo.split(','));
+        const credentials = new Credentials(this.#keyId, this.#keySecret);
+        this.#lara = new Translator(credentials);
+        this.#translateOptions = {
+            contentType: 'text/plain',
+            instructions: [],
+        };
+        this.#adaptTo && (this.#translateOptions.adaptTo = this.#adaptTo);
+        this.defaultInstructions && this.#translateOptions.instructions.push(this.defaultInstructions);
+    }
+
+    prepareTranslateChunkArgs({ sourceLang, targetLang, xmlTus, instructions }) {
+        const payload = xmlTus.map(xmlTu => {
+            const textBlock = [];
+            textBlock.push({ text: `bundle: ${xmlTu.bundle} key: ${xmlTu.key} notes: ${xmlTu.notes ?? ''}`, translatable: false });
+            textBlock.push({ text: xmlTu.source, translatable: true });
+            return textBlock;
+        }).flat(1);
+        const translateOptions = instructions ? { ...this.#translateOptions, instructions: [...this.#translateOptions.instructions, instructions] } : this.#translateOptions;
+        return { payload, sourceLang, targetLang, translateOptions };
+    }
+
+    async startTranslateChunk(args) {
+        const { payload, sourceLang, targetLang, translateOptions } = args;
+        try {
+            return await this.#lara.translate(payload, sourceLang, targetLang, translateOptions);
+        } catch (e) {
+            throw new Error(`Lara API error ${e.statusCode}: ${e.type}: ${e.message}`);
+        }
+    }
+
+    convertTranslationResponse(chunk) {
+        return chunk.translation.filter(textBlock => textBlock.translatable).map(textBlock => ({
+            tgt: textBlock.text,
+        }));
+    }
+
+    async info() {
+        const info = await super.info();
+        if (!this.#keyId || !this.#keySecret) {
+            info.description.push(styleString`Lara API key is missing. Please add the keyId and keySecret to the provider configuration.`);
+            return info;
+        }
+        try {
+            const credentials = new Credentials(this.#keyId, this.#keySecret);
+            const lara = new Translator(credentials);
+            const languages = (await lara.getLanguages()).sort();
+            info.description.push(styleString`Vendor-supported languages: ${languages?.join(', ') ?? 'unknown'}`);
+            const memories = await lara.memories.list();
+            if (memories.length > 0) {
+                memories.forEach(m =>
+                    info.description.push(styleString`Vendor TM "${m.name}": id: ${m.id} owner: ${m.ownerId} collaborators: ${m.collaboratorsCount} created: ${m.createdAt} updated: ${m.updatedAt}`)
+                );
+            } else {
+                info.description.push(styleString`No TMs configured.`);
+            }
+        } catch (error) {
+            info.description.push(styleString`Unable to connect to Lara server: ${error.message}`);
+        }
+        return info;
+    }
+}

package/mmtProvider.js

@@ -0,0 +1,112 @@
+import { logWarn, providers, styleString } from '@l10nmonster/core';
+import { ModernMT as MMTClient } from 'modernmt';
+
+/**
+ * @typedef {object} MMTProviderOptions
+ * @extends ChunkedRemoteTranslationProviderOptions
+ * @property {string} [id] - Global identifier (by default 'MMTBatch' or 'MMTRealtime')
+ * @property {string} apiKey - The ModernMT API key.
+ * @property {string} [webhook] - The webhook URL for batch translation.
+ * @property {function(any): any} [chunkFetcher] - The chunk fetcher operation name.
+ * @property {(string | number)[]} [hints] - Hints to include in the MMT request.
+ * @property {boolean} [multiline] - Whether to use multiline mode.
+ */
+
+/**
+ * Provider for Translated Modern MT.
+ */
+export class MMTProvider extends providers.ChunkedRemoteTranslationProvider {
+    /**
+     * Initializes a new instance of the MMTProvider class.
+     * @param {MMTProviderOptions} options - Configuration options for the provider.
+     */
+    constructor({ id, apiKey, webhook, chunkFetcher, hints, multiline = true, ...options }) {
+        id ??= webhook ? 'MMTBatch' : 'MMTRealtime';
+        super({ id, ...options });
+        if (webhook) {
+            if (chunkFetcher) {
+                this.chunkFetcher = chunkFetcher;
+            } else {
+                throw new Error('If you specify a webhook you must also specify a chunkFetcher');
+            }
+        }
+        this.baseRequest = {
+            mmtConstructor: [ apiKey, 'l10n.monster/MMT', '3.0' ],
+            hints,
+            options: {
+                multiline,
+                format: 'text/xml',
+            },
+            webhook,
+        }
+    }
+
+    start(job) {
+        if (!this.baseRequest.mmtConstructor[0]) {
+            throw new Error('You must have an apiKey to start an MMT job');
+        }
+        return super.start(job);
+    }
+
+    prepareTranslateChunkArgs({ sourceLang, targetLang, xmlTus, jobGuid, chunkNumber }) {
+        return {
+            sourceLang,
+            targetLang,
+            q: xmlTus.map(xmlTu => xmlTu.source),
+            hints:this.baseRequest.hints,
+            contextVector: undefined,
+            options: this.baseRequest.options,
+            webhook: this.baseRequest.webhook,
+            batchOptions: {
+                ...this.baseRequest.options,
+                idempotencyKey: `jobGuid:${jobGuid} chunk:${chunkNumber}`,
+                metadata: { jobGuid, chunk: chunkNumber },
+            },
+        };
+    }
+
+    async startTranslateChunk(args) {
+        const { sourceLang, targetLang, q, hints, contextVector, options, webhook, batchOptions } = args;
+        const [ apiKey, platform, platformVersion ] = this.baseRequest.mmtConstructor;
+        try {
+            const mmt = new MMTClient(apiKey, platform, platformVersion);
+            if (webhook) {
+                const response = await mmt.batchTranslate(webhook, sourceLang, targetLang, q, hints, contextVector, batchOptions);
+                return { enqueued: response };
+            }
+            return await mmt.translate(sourceLang, targetLang, q, hints, contextVector, options);
+        } catch(error) {
+            throw new Error(`${error.toString()}: ${error.response?.body}`);
+        }
+    }
+
+    convertTranslationResponse(chunk) {
+        if (chunk.enqueued) {
+            return null;
+        }
+        return chunk.map(mmtTx => ({
+            tgt: mmtTx.translation,
+            cost: [ mmtTx.billedCharacters, mmtTx.billed, mmtTx.characters ],
+        }));
+    }
+
+    async continueTranslateChunk(op) {
+        return await this.chunkFetcher(op.args);
+    }
+
+    async info() {
+        const info = await super.info();
+        try {
+            const response = await fetch('https://api.modernmt.com/translate/languages');
+            if (response.ok) {
+                const supportedProviderLanguages = (await response.json()).data.sort();
+                info.description.push(styleString`Vendor-supported languages: ${supportedProviderLanguages?.join(', ') ?? 'unknown'}`);
+            } else {
+                logWarn`HTTP error: status ${response.status} ${response.statusText}`
+            }
+        } catch (error) {
+            info.description.push(styleString`Unable to connect to MMT server: ${error.message}`);
+        }
+        return info;
+    }
+}

package/package.json

@@ -1,17 +1,19 @@
 {
-  "name": "@l10nmonster/helpers-translated",
-  "version": "1.0.0",
-  "description": "Translators integrating with Translated.com",
-  "main": "index.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "author": "Diego Lagunas",
-  "license": "MIT",
-  "peerDependencies": {
-    "@l10nmonster/helpers": "^1"
-  },
-  "dependencies": {
-    "modernmt": "^1.2.3"
-  }
+    "name": "@l10nmonster/helpers-translated",
+    "version": "3.0.0-alpha.2",
+    "description": "Helpers to integrate with Translated.com",
+    "main": "index.js",
+    "type": "module",
+    "scripts": {
+        "test": "node --test"
+    },
+    "author": "Diego Lagunas",
+    "license": "MIT",
+    "peerDependencies": {
+        "@l10nmonster/core": "^3.0.0-alpha.0"
+    },
+    "dependencies": {
+        "@translated/lara": "^1.4.0",
+        "modernmt": "^1.2.3"
+    }
 }

package/translationOS.js

@@ -1,270 +1,265 @@
-/* eslint-disable camelcase */
-const { utils } = require('@l10nmonster/helpers');
+// /* eslint-disable camelcase */
+// import { L10nContext, utils } from '@l10nmonster/core';
 
-function createTUFromTOSTranslation({ tosUnit, content, tuMeta, quality, refreshMode }) {
-    const guid = tosUnit.id_content;
-    !content && (content = tosUnit.translated_content);
-    const tu = {
-        guid,
-        ts: new Date().getTime(), // actual_delivery_date is garbage as it doesn't change after a bugfix, so it's better to use the retrieval time
-        q: quality,
-        th: tosUnit.translated_content_hash, // this is vendor-specific but it's ok to generalize
-    };
-    !refreshMode && (tu.cost = [ tosUnit.total, tosUnit.currency, tosUnit.wc_raw, tosUnit.wc_weighted ]);
-    if (tosUnit.revised_words) {
-        tu.rev = [ tosUnit.revised_words, tosUnit.error_points ?? 0];
-    }
-    if (tuMeta[guid]) {
-        tuMeta[guid].src && (tu.src = tuMeta[guid].src); // TODO: remove this
-        tuMeta[guid].nsrc && (tu.nsrc = tuMeta[guid].nsrc);
-        tu.ntgt = utils.extractNormalizedPartsV1(content, tuMeta[guid].phMap);
-        if (tu.ntgt.filter(e => e === undefined).length > 0) {
-            l10nmonster.logger.warn(`Unable to extract normalized parts of TU: ${JSON.stringify(tu)}`);
-            return null;
-        }
-    } else {
-        // simple content doesn't have meta
-        tu.ntgt = [ content ];
-    }
-    return tu;
-}
+// function createTUFromTOSTranslation({ tosUnit, content, tuMeta, quality, refreshMode }) {
+//     const guid = tosUnit.id_content;
+//     !content && (content = tosUnit.translated_content);
+//     const tu = {
+//         guid,
+//         ts: new Date().getTime(), // actual_delivery_date is garbage as it doesn't change after a bugfix, so it's better to use the retrieval time
+//         q: quality,
+//         th: tosUnit.translated_content_hash, // this is vendor-specific but it's ok to generalize
+//     };
+//     !refreshMode && (tu.cost = [ tosUnit.total, tosUnit.currency, tosUnit.wc_raw, tosUnit.wc_weighted ]);
+//     if (tosUnit.revised_words) {
+//         tu.rev = [ tosUnit.revised_words, tosUnit.error_points ?? 0];
+//     }
+//     if (tuMeta[guid]) {
+//         tuMeta[guid].src && (tu.src = tuMeta[guid].src); // TODO: remove this
+//         tuMeta[guid].nsrc && (tu.nsrc = tuMeta[guid].nsrc);
+//         tu.ntgt = utils.extractNormalizedPartsV1(content, tuMeta[guid].phMap);
+//         if (tu.ntgt.filter(e => e === undefined).length > 0) {
+//             L10nContext.logger.warn(`Unable to extract normalized parts of TU: ${JSON.stringify(tu)}`);
+//             return null;
+//         }
+//     } else {
+//         // simple content doesn't have meta
+//         tu.ntgt = [ content ];
+//     }
+//     return tu;
+// }
 
-async function tosRequestTranslationOfChunkOp({ request }) {
-    const { url, json, ...options } = request;
-    options.body = JSON.stringify(json);
-    const rawResponse = await fetch(url, options);
-    if (rawResponse.ok) {
-        const response = await (rawResponse.json());
-        const submittedGuids = json.map(tu => tu.id_content);
-        const committedGuids = response.map(contentStatus => contentStatus.id_content);
-        const missingTus = submittedGuids.filter(submittedGuid => !committedGuids.includes(submittedGuid));
-        if (submittedGuids.length !== committedGuids.length || missingTus.length > 0) {
-            l10nmonster.logger.error(`sent ${submittedGuids.length} got ${committedGuids.length} missing tus: ${missingTus.map(tu => tu.id_content).join(', ')}`);
-            throw `TOS: inconsistent behavior. submitted ${submittedGuids.length}, committed ${committedGuids.length}, missing ${missingTus.length}`;
-        }
-        return committedGuids;
-    } else {
-        throw `${rawResponse.status} ${rawResponse.statusText}: ${await rawResponse.text()}`;
-    }
-}
+// async function tosRequestTranslationOfChunkOp({ request }) {
+//     const { url, json, ...options } = request;
+//     options.body = JSON.stringify(json);
+//     const rawResponse = await fetch(url, options);
+//     if (rawResponse.ok) {
+//         const response = await (rawResponse.json());
+//         const submittedGuids = json.map(tu => tu.id_content);
+//         const committedGuids = response.map(contentStatus => contentStatus.id_content);
+//         const missingTus = submittedGuids.filter(submittedGuid => !committedGuids.includes(submittedGuid));
+//         if (submittedGuids.length !== committedGuids.length || missingTus.length > 0) {
+//             L10nContext.logger.error(`sent ${submittedGuids.length} got ${committedGuids.length} missing tus: ${missingTus.map(tu => tu.id_content).join(', ')}`);
+//             throw `TOS: inconsistent behavior. submitted ${submittedGuids.length}, committed ${committedGuids.length}, missing ${missingTus.length}`;
+//         }
+//         return committedGuids;
+//     } else {
+//         throw `${rawResponse.status} ${rawResponse.statusText}: ${await rawResponse.text()}`;
+//     }
+// }
 
-async function tosCombineTranslationChunksOp(args, committedGuids) {
-    return committedGuids.flat(1);
-}
+// async function tosCombineTranslationChunksOp(args, committedGuids) {
+//     return committedGuids.flat(1);
+// }
 
-async function tosFetchContentByGuidOp({ refreshMode, tuMap, tuMeta, request, quality, parallelism }) {
-    const { url, json, ...options } = request;
-    options.body = JSON.stringify(json);
-    try {
-        let tosContent = await (await fetch(url, options)).json();
-        tosContent = tosContent.filter(tosUnit => tosUnit.translated_content_url);
-        // eslint-disable-next-line no-invalid-this
-        l10nmonster.logger.info(`Retrieved ${tosContent.length} translations from TOS`);
-        refreshMode && (tosContent = tosContent.filter(tosUnit => !(tuMap[tosUnit.id_content].th === tosUnit.translated_content_hash))); // need to consider th being undefined/null for some entries
-        // sanitize bad responses
-        const fetchedTus = [];
-        const seenGuids = {};
-        while (tosContent.length > 0) {
-            const chunk = tosContent.splice(0, parallelism);
-            const fetchedRaw = (await Promise.all(chunk.map(tosUnit => fetch(tosUnit.translated_content_url)))).map(async r => await r.text());
-            const fetchedContent = await Promise.all(fetchedRaw);
-            (await Promise.all(chunk.map(tosUnit => fetch(tosUnit.translated_content_url)))).map(async r => await r.text());
-            // eslint-disable-next-line no-invalid-this
-            l10nmonster.logger.info(`Fetched ${chunk.length} pieces of content from AWS`);
-            chunk.forEach((tosUnit, idx) => {
-                if (seenGuids[tosUnit.id_content]) {
-                    throw `TOS: Duplicate translations found for guid: ${tosUnit.id_content}`;
-                } else {
-                    seenGuids[tosUnit.id_content] = true;
-                }
-                if (fetchedContent[idx] !== null && fetchedContent[idx].indexOf('|||UNTRANSLATED_CONTENT_START|||') === -1) {
-                    // eslint-disable-next-line no-invalid-this
-                    const newTU = createTUFromTOSTranslation({ tosUnit, content: fetchedContent[idx], tuMeta, quality, refreshMode });
-                    fetchedTus.push(newTU);
-                } else {
-                    // eslint-disable-next-line no-invalid-this
-                    l10nmonster.logger.info(`TOS: for guid ${tosUnit.id_content} retrieved untranslated content ${fetchedContent[idx]}`);
-                }
-            });
-        }
-        return fetchedTus;
-    } catch(error) {
-        throw `${error.toString()}: ${error.response?.body ?? error.stack}`;
-    }
-}
+// async function tosFetchContentByGuidOp({ refreshMode, tuMap, tuMeta, request, quality, parallelism }) {
+//     const { url, json, ...options } = request;
+//     options.body = JSON.stringify(json);
+//     try {
+//         let tosContent = await (await fetch(url, options)).json();
+//         tosContent = tosContent.filter(tosUnit => tosUnit.translated_content_url);
+//         L10nContext.logger.info(`Retrieved ${tosContent.length} translations from TOS`);
+//         refreshMode && (tosContent = tosContent.filter(tosUnit => !(tuMap[tosUnit.id_content].th === tosUnit.translated_content_hash))); // need to consider th being undefined/null for some entries
+//         // sanitize bad responses
+//         const fetchedTus = [];
+//         const seenGuids = {};
+//         while (tosContent.length > 0) {
+//             const chunk = tosContent.splice(0, parallelism);
+//             const fetchedRaw = (await Promise.all(chunk.map(tosUnit => fetch(tosUnit.translated_content_url)))).map(async r => await r.text());
+//             const fetchedContent = await Promise.all(fetchedRaw);
+//             (await Promise.all(chunk.map(tosUnit => fetch(tosUnit.translated_content_url)))).map(async r => await r.text());
+//             L10nContext.logger.info(`Fetched ${chunk.length} pieces of content from AWS`);
+//             chunk.forEach((tosUnit, idx) => {
+//                 if (seenGuids[tosUnit.id_content]) {
+//                     throw `TOS: Duplicate translations found for guid: ${tosUnit.id_content}`;
+//                 } else {
+//                     seenGuids[tosUnit.id_content] = true;
+//                 }
+//                 if (fetchedContent[idx] !== null && fetchedContent[idx].indexOf('|||UNTRANSLATED_CONTENT_START|||') === -1) {
+//                     const newTU = createTUFromTOSTranslation({ tosUnit, content: fetchedContent[idx], tuMeta, quality, refreshMode });
+//                     fetchedTus.push(newTU);
+//                 } else {
+//                     L10nContext.logger.info(`TOS: for guid ${tosUnit.id_content} retrieved untranslated content ${fetchedContent[idx]}`);
+//                 }
+//             });
+//         }
+//         return fetchedTus;
+//     } catch(error) {
+//         throw `${error.toString()}: ${error.response?.body ?? error.message}`;
+//     }
+// }
 
-async function tosCombineFetchedTusOp(args, tuChunks) {
-    return tuChunks.flat(1).filter(tu => Boolean(tu));
-}
+// async function tosCombineFetchedTusOp(args, tuChunks) {
+//     return tuChunks.flat(1).filter(tu => Boolean(tu));
+// }
 
-module.exports = class TranslationOS {
-    constructor({ baseURL, apiKey, costAttributionLabel, serviceType, quality, tuDecorator, maxTranslationRequestSize, maxFetchSize, parallelism, requestOnly }) {
-        if ((apiKey && quality) === undefined) {
-            throw 'You must specify apiKey, quality for TranslationOS';
-        } else {
-            this.baseURL = baseURL ?? 'https://api.translated.com/v2';
-            this.stdHeaders = {
-                'x-api-key': apiKey,
-                'user-agent': 'l10n.monster/TOS v0.1',
-                'Content-Type': 'application/json',
-            }
-            this.serviceType = serviceType ?? 'premium';
-            this.costAttributionLabel = costAttributionLabel;
-            this.quality = quality;
-            this.tuDecorator = tuDecorator;
-            this.maxTranslationRequestSize = maxTranslationRequestSize || 100;
-            this.maxFetchSize = maxFetchSize || 512;
-            this.parallelism = parallelism || 128;
-            this.requestOnly = requestOnly;
-            l10nmonster.opsMgr.registerOp(tosRequestTranslationOfChunkOp, { idempotent: false });
-            l10nmonster.opsMgr.registerOp(tosCombineTranslationChunksOp, { idempotent: true });
-            l10nmonster.opsMgr.registerOp(tosFetchContentByGuidOp, { idempotent: true });
-            l10nmonster.opsMgr.registerOp(tosCombineFetchedTusOp, { idempotent: true });
-        }
-    }
+// export class TranslationOS {
+//     constructor({ baseURL, apiKey, costAttributionLabel, serviceType, quality, tuDecorator, maxTranslationRequestSize, maxFetchSize, parallelism, requestOnly }) {
+//         if ((apiKey && quality) === undefined) {
+//             throw 'You must specify apiKey, quality for TranslationOS';
+//         } else {
+//             this.baseURL = baseURL ?? 'https://api.translated.com/v2';
+//             this.stdHeaders = {
+//                 'x-api-key': apiKey,
+//                 'user-agent': 'l10n.monster/TOS v0.1',
+//                 'Content-Type': 'application/json',
+//             }
+//             this.serviceType = serviceType ?? 'premium';
+//             this.costAttributionLabel = costAttributionLabel;
+//             this.quality = quality;
+//             this.tuDecorator = tuDecorator;
+//             this.maxTranslationRequestSize = maxTranslationRequestSize || 100;
+//             this.maxFetchSize = maxFetchSize || 512;
+//             this.parallelism = parallelism || 128;
+//             this.requestOnly = requestOnly;
+//             L10nContext.opsMgr.registerOp(tosRequestTranslationOfChunkOp, { idempotent: false });
+//             L10nContext.opsMgr.registerOp(tosCombineTranslationChunksOp, { idempotent: true });
+//             L10nContext.opsMgr.registerOp(tosFetchContentByGuidOp, { idempotent: true });
+//             L10nContext.opsMgr.registerOp(tosCombineFetchedTusOp, { idempotent: true });
+//         }
+//     }
 
-    async requestTranslations(jobRequest) {
-        const { tus, ...jobResponse } = jobRequest;
-        const { contentMap, phNotes } = utils.getTUMaps(tus);
-        const tosPayload = tus.map(tu => {
-            const notes = typeof tu.notes === 'string' ? utils.extractStructuredNotes(tu.notes) : tu.notes;
-            let tosTU = {
-                'id_order': jobRequest.jobGuid,
-                'id_content': tu.guid,
-                content: contentMap[tu.guid],
-                metadata: 'mf=v1',
-                context: {
-                    notes: `${notes?.maxWidth ? `▶▶▶MAXIMUM WIDTH ${notes.maxWidth} chars◀◀◀\n` : ''}${notes?.desc ?? ''}${phNotes[tu.guid] ?? ''}\n rid: ${tu.rid}\n sid: ${tu.sid}\n ${tu.seq ? `seq: id_${utils.integerToLabel(tu.seq)}` : ''}`
-                },
-                'source_language': jobRequest.sourceLang,
-                'target_languages': [ jobRequest.targetLang ],
-                // 'content_type': 'text/html',
-                'service_type': this.serviceType,
-                'cost_attribution_label': this.costAttributionLabel,
-                'dashboard_query_labels': [],
-            };
-            notes?.screenshot && (tosTU.context.screenshot = notes.screenshot);
-            jobRequest.instructions && (tosTU.context.instructions = jobRequest.instructions);
-            tu.seq && tosTU.dashboard_query_labels.push(`id_${utils.integerToLabel(tu.seq)}`);
-            tu.rid && tosTU.dashboard_query_labels.push(tu.rid.slice(-50));
-            tosTU.dashboard_query_labels.push(tu.sid.replaceAll('\n', '').slice(-50));
-            if (tu.prj !== undefined) {
-                // eslint-disable-next-line camelcase
-                tosTU.id_order_group = tu.prj;
-            }
-            if (typeof this.tuDecorator === 'function') {
-                tosTU = this.tuDecorator(tosTU, tu, jobResponse);
-            }
-            return tosTU;
-        });
+//     async requestTranslations(jobRequest) {
+//         const { tus, ...jobResponse } = jobRequest;
+//         const { contentMap, phNotes } = utils.getTUMaps(tus);
+//         const tosPayload = tus.map(tu => {
+//             const notes = typeof tu.notes === 'string' ? utils.extractStructuredNotes(tu.notes) : tu.notes;
+//             let tosTU = {
+//                 'id_order': jobRequest.jobGuid,
+//                 'id_content': tu.guid,
+//                 content: contentMap[tu.guid],
+//                 metadata: 'mf=v1',
+//                 context: {
+//                     notes: `${notes?.maxWidth ? `▶▶▶MAXIMUM WIDTH ${notes.maxWidth} chars◀◀◀\n` : ''}${notes?.desc ?? ''}${phNotes[tu.guid] ?? ''}\n rid: ${tu.rid}\n sid: ${tu.sid}\n ${tu.seq ? `seq: id_${utils.integerToLabel(tu.seq)}` : ''}`
+//                 },
+//                 'source_language': jobRequest.sourceLang,
+//                 'target_languages': [ jobRequest.targetLang ],
+//                 // 'content_type': 'text/html',
+//                 'service_type': this.serviceType,
+//                 'cost_attribution_label': this.costAttributionLabel,
+//                 'dashboard_query_labels': [],
+//             };
+//             notes?.screenshot && (tosTU.context.screenshot = notes.screenshot);
+//             jobRequest.instructions && (tosTU.context.instructions = jobRequest.instructions);
+//             tu.seq && tosTU.dashboard_query_labels.push(`id_${utils.integerToLabel(tu.seq)}`);
+//             tu.rid && tosTU.dashboard_query_labels.push(tu.rid.slice(-50));
+//             tosTU.dashboard_query_labels.push(tu.sid.replaceAll('\n', '').slice(-50));
+//             if (tu.prj !== undefined) {
+//                 tosTU.id_order_group = tu.prj;
+//             }
+//             if (typeof this.tuDecorator === 'function') {
+//                 tosTU = this.tuDecorator(tosTU, tu, jobResponse);
+//             }
+//             return tosTU;
+//         });
 
-        const requestTranslationsTask = l10nmonster.opsMgr.createTask();
-        try {
-            let chunkNumber = 0;
-            const chunkOps = [];
-            while (tosPayload.length > 0) {
-                const json = tosPayload.splice(0, this.maxTranslationRequestSize);
-                chunkNumber++;
-                l10nmonster.logger.info(`Enqueueing TOS translation job ${jobResponse.jobGuid} chunk size: ${json.length}`);
-                chunkOps.push(requestTranslationsTask.enqueue(tosRequestTranslationOfChunkOp, {
-                    request: {
-                        url: `${this.baseURL}/translate`,
-                        method: 'POST',
-                        json,
-                        headers: {
-                            ...this.stdHeaders,
-                            'x-idempotency-id': `jobGuid:${jobRequest.jobGuid} chunk:${chunkNumber}`,
-                        },
-                    },
-                 }));
-            }
-            requestTranslationsTask.commit(tosCombineTranslationChunksOp, null, chunkOps);
-            jobResponse.taskName = requestTranslationsTask.taskName;
-            const committedGuids = await requestTranslationsTask.execute();
-            if (this.requestOnly) {
-                return {
-                    ...jobResponse,
-                    tus: [],
-                    status: 'done'
-                };
-            }
-            return {
-                ...jobResponse,
-                inflight: committedGuids,
-                status: 'pending'
-            };
-        } catch (error) {
-            throw `TOS call failed - ${error}`;
-        }
-    }
+//         const requestTranslationsTask = L10nContext.opsMgr.createTask();
+//         try {
+//             let chunkNumber = 0;
+//             const chunkOps = [];
+//             while (tosPayload.length > 0) {
+//                 const json = tosPayload.splice(0, this.maxTranslationRequestSize);
+//                 chunkNumber++;
+//                 L10nContext.logger.info(`Enqueueing TOS translation job ${jobResponse.jobGuid} chunk size: ${json.length}`);
+//                 chunkOps.push(requestTranslationsTask.enqueue(tosRequestTranslationOfChunkOp, {
+//                     request: {
+//                         url: `${this.baseURL}/translate`,
+//                         method: 'POST',
+//                         json,
+//                         headers: {
+//                             ...this.stdHeaders,
+//                             'x-idempotency-id': `jobGuid:${jobRequest.jobGuid} chunk:${chunkNumber}`,
+//                         },
+//                     },
+//                  }));
+//             }
+//             requestTranslationsTask.commit(tosCombineTranslationChunksOp, null, chunkOps);
+//             jobResponse.taskName = requestTranslationsTask.taskName;
+//             const committedGuids = await requestTranslationsTask.execute();
+//             if (this.requestOnly) {
+//                 return {
+//                     ...jobResponse,
+//                     tus: [],
+//                     status: 'done'
+//                 };
+//             }
+//             return {
+//                 ...jobResponse,
+//                 inflight: committedGuids,
+//                 status: 'pending'
+//             };
+//         } catch (error) {
+//             throw `TOS call failed - ${error}`;
+//         }
+//     }
 
-    async #fetchTranslatedTus({ jobGuid, targetLang, reqTus, refreshMode }) {
-        const guids = reqTus.filter(tu => tu.src ?? tu.nsrc).map(tu => tu.guid); // TODO: remove .src
-        const refreshTranslationsTask = l10nmonster.opsMgr.createTask();
-        let chunkNumber = 0;
-        const refreshOps = [];
-        while (guids.length > 0) {
-            chunkNumber++;
-            const guidsInChunk = guids.splice(0, this.maxFetchSize);
-            const tusInChunk = reqTus.filter(tu => guidsInChunk.includes(tu.guid));
-            const tuMap = tusInChunk.reduce((p,c) => (p[c.guid] = c, p), {});
-            const { tuMeta } = utils.getTUMaps(tusInChunk);
-            l10nmonster.logger.verbose(`Enqueueing refresh of TOS chunk ${chunkNumber} (${guidsInChunk.length} units)...`);
-            const json = {
-                id_content: guidsInChunk,
-                target_language: targetLang,
-                fetch_content: false,
-                limit: this.maxFetchSize,
-            };
-            if (refreshMode) {
-                json.status = ['delivered', 'invoiced'];
-                json.last_delivered_only = true;
-            } else {
-                json.id_order = jobGuid;
-            }
-            refreshOps.push(refreshTranslationsTask.enqueue(tosFetchContentByGuidOp, {
-                refreshMode,
-                tuMap,
-                tuMeta,
-                request: {
-                    url: `${this.baseURL}/status`,
-                    method: 'POST',
-                    json,
-                    headers: this.stdHeaders,
-                },
-                quality: this.quality,
-                parallelism: this.parallelism,
-            }));
-        }
-        refreshTranslationsTask.commit(tosCombineFetchedTusOp, null, refreshOps);
-        const jobResponse = await refreshTranslationsTask.execute();
-        jobResponse.taskName = refreshTranslationsTask.taskName;
-        return jobResponse;
-    }
+//     async #fetchTranslatedTus({ jobGuid, targetLang, reqTus, refreshMode }) {
+//         const guids = reqTus.filter(tu => tu.src ?? tu.nsrc).map(tu => tu.guid); // TODO: remove .src
+//         const refreshTranslationsTask = L10nContext.opsMgr.createTask();
+//         let chunkNumber = 0;
+//         const refreshOps = [];
+//         while (guids.length > 0) {
+//             chunkNumber++;
+//             const guidsInChunk = guids.splice(0, this.maxFetchSize);
+//             const tusInChunk = reqTus.filter(tu => guidsInChunk.includes(tu.guid));
+//             const tuMap = tusInChunk.reduce((p,c) => (p[c.guid] = c, p), {});
+//             const { tuMeta } = utils.getTUMaps(tusInChunk);
+//             L10nContext.logger.verbose(`Enqueueing refresh of TOS chunk ${chunkNumber} (${guidsInChunk.length} units)...`);
+//             const json = {
+//                 id_content: guidsInChunk,
+//                 target_language: targetLang,
+//                 fetch_content: false,
+//                 limit: this.maxFetchSize,
+//             };
+//             if (refreshMode) {
+//                 json.status = ['delivered', 'invoiced'];
+//                 json.last_delivered_only = true;
+//             } else {
+//                 json.id_order = jobGuid;
+//             }
+//             refreshOps.push(refreshTranslationsTask.enqueue(tosFetchContentByGuidOp, {
+//                 refreshMode,
+//                 tuMap,
+//                 tuMeta,
+//                 request: {
+//                     url: `${this.baseURL}/status`,
+//                     method: 'POST',
+//                     json,
+//                     headers: this.stdHeaders,
+//                 },
+//                 quality: this.quality,
+//                 parallelism: this.parallelism,
+//             }));
+//         }
+//         refreshTranslationsTask.commit(tosCombineFetchedTusOp, null, refreshOps);
+//         const jobResponse = await refreshTranslationsTask.execute();
+//         jobResponse.taskName = refreshTranslationsTask.taskName;
+//         return jobResponse;
+//     }
 
-    async fetchTranslations(pendingJob, jobRequest) {
-        const { inflight, ...jobResponse } = pendingJob;
-        const reqTus = jobRequest.tus.filter(tu => inflight.includes(tu.guid));
-        const tus = await this.#fetchTranslatedTus({ jobGuid: pendingJob.originalJobGuid ?? jobRequest.originalJobGuid ?? jobRequest.jobGuid, targetLang: jobRequest.targetLang, reqTus, refreshMode: false });
-        const tuMap = tus.reduce((p,c) => (p[c.guid] = c, p), {});
-        const nowInflight = inflight.filter(guid => !tuMap[guid]);
-        if (tus.length > 0) {
-            const response = {
-                ...jobResponse,
-                tus,
-                status: nowInflight.length === 0 ? 'done' : 'pending',
-            };
-            nowInflight.length > 0 && (response.inflight = nowInflight);
-            return response;
-        }
-        return null;
-    }
+//     async fetchTranslations(pendingJob) {
+//         const { inflight, ...jobResponse } = pendingJob;
+//         const reqTus = pendingJob.tus.filter(tu => inflight.includes(tu.guid));
+//         const tus = await this.#fetchTranslatedTus({ jobGuid: pendingJob.originalJobGuid ?? pendingJob.originalJobGuid ?? pendingJob.jobGuid, targetLang: pendingJob.targetLang, reqTus, refreshMode: false });
+//         const tuMap = tus.reduce((p,c) => (p[c.guid] = c, p), {});
+//         const nowInflight = inflight.filter(guid => !tuMap[guid]);
+//         if (tus.length > 0) {
+//             const response = {
+//                 ...jobResponse,
+//                 tus,
+//                 status: nowInflight.length === 0 ? 'done' : 'pending',
+//             };
+//             nowInflight.length > 0 && (response.inflight = nowInflight);
+//             return response;
+//         }
+//         return null;
+//     }
 
-    async refreshTranslations(jobRequest) {
-        return {
-            ...jobRequest,
-            tus: await this.#fetchTranslatedTus({ targetLang: jobRequest.originalJobGuid ?? jobRequest.targetLang, reqTus: jobRequest.tus, refreshMode: true }),
-            status: 'done',
-        };
-    }
-}
+//     async refreshTranslations(jobRequest) {
+//         return {
+//             ...jobRequest,
+//             tus: await this.#fetchTranslatedTus({ jobGuid: jobRequest.originalJobGuid, targetLang: jobRequest.originalJobGuid ?? jobRequest.targetLang, reqTus: jobRequest.tus, refreshMode: true }),
+//             status: 'done',
+//         };
+//     }
+// }

package/modernMT.js

@@ -1,234 +0,0 @@
-/* eslint-disable no-invalid-this */
-const { utils, normalizers } = require('@l10nmonster/helpers');
-const { ModernMT } = require('modernmt');
-
-const MAX_CHAR_LENGTH = 9900;
-const MAX_CHUNK_SIZE = 125;
-
-async function mmtTranslateChunkOp({ q, batchOptions }) {
-    const baseRequest = this.context.baseRequest;
-    try {
-        const mmt = new ModernMT(...baseRequest.mmtConstructor);
-        if (batchOptions) {
-            const response = await mmt.batchTranslate(
-                baseRequest.webhook,
-                baseRequest.sourceLang,
-                baseRequest.targetLang,
-                q,
-                baseRequest.hints,
-                undefined,
-                {
-                    ...baseRequest.options,
-                    ...batchOptions
-                }
-            );
-            return {
-                enqueued: response
-            }
-        } else {
-            const response = await mmt.translate(
-                baseRequest.sourceLang,
-                baseRequest.targetLang,
-                q,
-                baseRequest.hints,
-                undefined,
-                baseRequest.options
-            );
-            return response;
-        }
-    } catch(error) {
-        throw `${error.toString()}: ${error.response?.body}`;
-    }
-}
-
-async function mmtMergeSubmittedChunksOp(args, chunks) {
-    chunks.forEach((response, idx) => l10nmonster.logger.verbose(`MMT Chunk ${idx} enqueued=${response.enqueued}`));
-}
-
-async function mmtMergeTranslatedChunksOp({ jobRequest, tuMeta, quality, ts, chunkSizes }, chunks) {
-    chunks.forEach((chunk, idx) => {
-        if (chunk.length !== chunkSizes[idx]) {
-            throw `MMT: Expected chunk ${idx} to have ${chunkSizes[idx]} translations but got ${chunk.length}`;
-        }
-    });
-    const { tus, ...jobResponse } = jobRequest;
-    const translations = chunks.flat(1);
-    jobResponse.tus = tus.map((tu, idx) => {
-        const translation = { guid: tu.guid, ts };
-        const mmtTx = translations[idx] || {};
-        translation.ntgt = utils.extractNormalizedPartsFromXmlV1(mmtTx.translation, tuMeta[idx] || {});
-        translation.q = quality;
-        translation.cost = [ mmtTx.billedCharacters, mmtTx.billed, mmtTx.characters ];
-        return translation;
-    });
-    jobResponse.status = 'done';
-    return jobResponse;
-}
-
-function applyGlossary(glossaryEncoder, jobResponse) {
-    if (glossaryEncoder) {
-            const flags = { targetLang: jobResponse.targetLang };
-        for (const tu of jobResponse.tus) {
-            tu.ntgt.forEach((part, idx) => {
-                // not very solid, but if placeholder follows glossary conventions, then convert it back to a string
-                if (typeof part === 'object' && part.v.indexOf('glossary:') === 0) {
-                    tu.ntgt[idx] = glossaryEncoder(part.v, flags);
-                }
-            });
-            tu.ntgt = utils.consolidateDecodedParts(tu.ntgt, flags, true);
-        }
-    }
-}
-
-module.exports = class ModernMT {
-    constructor({ baseURL, apiKey, webhook, chunkFetcher, hints, multiline, quality, maxCharLength, languageMapper, glossary }) {
-        if ((apiKey && quality) === undefined) {
-            throw 'You must specify apiKey, quality for ModernMT';
-        } else {
-            if (webhook && !chunkFetcher) {
-                throw 'If you specify a webhook you must also specify a chunkFetcher';
-            }
-            this.baseURL = baseURL ?? 'https://api.modernmt.com';
-            this.mmtConstructor = [ apiKey, 'l10n.monster/MMT', '1.0' ];
-            this.webhook = webhook;
-            this.chunkFetcher = chunkFetcher;
-            chunkFetcher && l10nmonster.opsMgr.registerOp(chunkFetcher, { idempotent: true });
-            this.hints = hints;
-            this.multiline = multiline ?? true;
-            this.quality = quality;
-            this.maxCharLength = maxCharLength ?? MAX_CHAR_LENGTH;
-            this.languageMapper = languageMapper;
-            if (glossary) {
-                const [ glossaryDecoder, glossaryEncoder ] = normalizers.keywordTranslatorMaker('glossary', glossary);
-                this.glossaryDecoder = [ glossaryDecoder ];
-                this.glossaryEncoder = glossaryEncoder;
-            }
-            l10nmonster.opsMgr.registerOp(mmtTranslateChunkOp, { idempotent: false });
-            l10nmonster.opsMgr.registerOp(mmtMergeSubmittedChunksOp, { idempotent: true });
-            l10nmonster.opsMgr.registerOp(mmtMergeTranslatedChunksOp, { idempotent: true });
-        }
-    }
-
-    async requestTranslations(jobRequest) {
-        const sourceLang = (this.languageMapper && this.languageMapper(jobRequest.sourceLang)) ?? jobRequest.sourceLang;
-        const targetLang = (this.languageMapper && this.languageMapper(jobRequest.targetLang)) ?? jobRequest.targetLang;
-        const tuMeta = {};
-        const mmtPayload = jobRequest.tus.map((tu, idx) => {
-            const nsrc = utils.decodeNormalizedString(tu.nsrc, this.glossaryDecoder);
-            const [xmlSrc, phMap ] = utils.flattenNormalizedSourceToXmlV1(nsrc);
-            if (Object.keys(phMap).length > 0) {
-                tuMeta[idx] = phMap;
-            }
-            return xmlSrc;
-        });
-        const context = {
-            baseRequest: {
-                mmtConstructor: this.mmtConstructor,
-                sourceLang,
-                targetLang,
-                hints: this.hints,
-                options: {
-                    multiline: this.multiline,
-                    format: 'text/xml',
-                },
-                webhook: this.webhook,
-            }
-        };
-        const requestTranslationsTask = l10nmonster.opsMgr.createTask();
-        requestTranslationsTask.setContext(context);
-        const chunkOps = [];
-        const chunkSizes = [];
-        for (let currentIdx = 0; currentIdx < mmtPayload.length;) {
-            const q = [];
-            let currentTotalLength = 0;
-            while (currentIdx < mmtPayload.length && q.length < MAX_CHUNK_SIZE && mmtPayload[currentIdx].length + currentTotalLength < this.maxCharLength) {
-                currentTotalLength += mmtPayload[currentIdx].length;
-                q.push(mmtPayload[currentIdx]);
-                currentIdx++;
-            }
-            if (q.length === 0) {
-                throw `String at index ${currentIdx} exceeds ${this.maxCharLength} max char length`;
-            }
-            l10nmonster.logger.info(`Preparing MMT translate: chunk strings: ${q.length} chunk char length: ${currentTotalLength}`);
-            const req = { q };
-            if (this.webhook) {
-                req.batchOptions = {
-                    idempotencyKey: `jobGuid:${jobRequest.jobGuid} chunk:${chunkOps.length}`,
-                    metadata: {
-                        jobGuid: jobRequest.jobGuid,
-                        chunk: chunkOps.length
-                    }
-                };
-            }
-            chunkSizes.push(q.length);
-            chunkOps.push(requestTranslationsTask.enqueue(mmtTranslateChunkOp, req));
-        }
-        try {
-            if (this.webhook) {
-                requestTranslationsTask.commit(mmtMergeSubmittedChunksOp, null, chunkOps);
-                await requestTranslationsTask.execute();
-                const { tus, ...jobResponse } = jobRequest;
-                jobResponse.inflight = tus.map(tu => tu.guid);
-                jobResponse.envelope = { chunkSizes, tuMeta };
-                jobResponse.status = 'pending';
-                jobResponse.taskName = l10nmonster.regression ? 'x' : requestTranslationsTask.taskName;
-                return jobResponse;
-            } else {
-                requestTranslationsTask.commit(mmtMergeTranslatedChunksOp, {
-                    jobRequest,
-                    tuMeta,
-                    quality: this.quality,
-                    ts: l10nmonster.regression ? 1 : new Date().getTime(),
-                    chunkSizes,
-                }, chunkOps);
-                const jobResponse = await requestTranslationsTask.execute();
-                jobResponse.taskName = l10nmonster.regression ? 'x' : requestTranslationsTask.taskName;
-                applyGlossary(this.glossaryEncoder, jobResponse);
-                return jobResponse;
-            }
-        } catch (error) {
-            throw `MMT call failed - ${error}`;
-        }
-    }
-
-    async fetchTranslations(pendingJob, jobRequest) {
-        try {
-            // eslint-disable-next-line no-unused-vars
-            const requestTranslationsTask = l10nmonster.opsMgr.createTask();
-            const chunkOps = [];
-            pendingJob.envelope.chunkSizes.forEach(async (chunkSize, chunk) => {
-                l10nmonster.logger.info(`Enqueue chunk fetcher for job: ${jobRequest.jobGuid} chunk:${chunk} chunkSize:${chunkSize}`);
-                chunkOps.push(requestTranslationsTask.enqueue(this.chunkFetcher, {
-                    jobGuid: jobRequest.jobGuid,
-                    chunk,
-                    chunkSize,
-                }));
-            });
-            requestTranslationsTask.commit(mmtMergeTranslatedChunksOp, {
-                jobRequest,
-                tuMeta: pendingJob.envelope.tuMeta,
-                quality: this.quality,
-                ts: l10nmonster.regression ? 1 : new Date().getTime(),
-                chunkSizes: pendingJob.envelope.chunkSizes,
-            }, chunkOps);
-            const jobResponse = await requestTranslationsTask.execute();
-            jobResponse.taskName = l10nmonster.regression ? 'x' : requestTranslationsTask.taskName;
-            applyGlossary(this.glossaryEncoder, jobResponse);
-            return jobResponse;
-        } catch (error) {
-            return null; // getting errors is expected, just leave the job pending
-        }
-    }
-
-    async refreshTranslations(jobRequest) {
-        if (this.webhook) {
-            throw 'Refreshing MMT translations not supported in batch mode';
-        }
-        const fullResponse = await this.requestTranslations(jobRequest);
-        const reqTuMap = jobRequest.tus.reduce((p,c) => (p[c.guid] = c, p), {});
-        return {
-            ...fullResponse,
-            tus: fullResponse.tus.filter(tu => !utils.normalizedStringsAreEqual(reqTuMap[tu.guid].ntgt, tu.ntgt)),
-        };
-    }
-}