@qvac/harness 0.2.10 → 0.3.0

package/dist-lib/lib/tools/translate.js

@@ -0,0 +1,228 @@
+import ScopeLock from 'scope-lock';
+import { getLangName } from '@qvac/langdetect-text';
+import { detectLanguage, UNDETERMINED } from '../capabilities/detect-language.js';
+import { AbortedError, raceAbort } from '../helpers/race-abort.js';
+import { ENGLISH, NMT_FROM_ENGLISH, NMT_TO_ENGLISH, languageLabel, SLM_LANGUAGE_CODES, sameLanguage, translationPlanFor } from '../models/nmt-catalog.js';
+import { createLoadStall } from '../helpers/load-stall.js';
+import { createLoadTracker } from '../models/model-pool.js';
+import { resolveModelSrc } from '../models/model-source.js';
+export const TRANSLATION_LOAD_STALL_MS = 15 * 60_000;
+export const TRANSLATION_LOAD_STALL_ERROR = 'translation model download stalled';
+function packName(plan) {
+    const pair = plan.model.match(/model\.([a-z]+)\.intgemm/)?.[1];
+    if (pair)
+        return `Bergamot ${pair}`;
+    return plan.model.includes('/en-xx/') ? 'EuroNano en-xx' : 'EuroNano xx-en';
+}
+const fingerprintOf = (plan) => `${plan.model}|${plan.pivot ?? ''}`;
+function bergamotConfig(plan) {
+    const config = { engine: 'Bergamot', from: plan.from, to: plan.to };
+    if (plan.srcVocab)
+        config.srcVocabSrc = plan.srcVocab;
+    if (plan.dstVocab)
+        config.dstVocabSrc = plan.dstVocab;
+    return config;
+}
+export function createTranslationRuntime({ loadSdk, loads = createLoadTracker(), onProgress, stallMs = TRANSLATION_LOAD_STALL_MS }) {
+    let loading = null;
+    let residentFingerprint = null;
+    const lock = new ScopeLock();
+    const turn = new ScopeLock();
+    const loadPack = async (sdk, plan) => {
+        const modelSrc = await resolveModelSrc(sdk, plan.model, 'nmt');
+        const modelConfig = bergamotConfig(plan);
+        if (plan.pivot) {
+            modelConfig.pivotModel = { modelSrc: await resolveModelSrc(sdk, plan.pivot, 'nmt') };
+        }
+        const name = packName(plan);
+        const stall = createLoadStall(stallMs, TRANSLATION_LOAD_STALL_ERROR);
+        onProgress?.(name, {});
+        const pending = sdk.loadModel({
+            modelSrc,
+            modelType: 'nmtcpp-translation',
+            modelConfig,
+            onProgress: (progress) => {
+                stall.bump();
+                onProgress?.(name, progress);
+            }
+        });
+        try {
+            const modelId = await Promise.race([loads.track(pending), stall.promise]);
+            return { modelId };
+        }
+        catch (err) {
+            if (err instanceof Error &&
+                err.message === TRANSLATION_LOAD_STALL_ERROR &&
+                pending.requestId) {
+                void sdk.cancel?.({ requestId: pending.requestId }).catch(() => { });
+            }
+            throw err;
+        }
+        finally {
+            stall.clear();
+        }
+    };
+    const doUnload = async () => {
+        const pending = loading;
+        if (!pending) {
+            residentFingerprint = null;
+            return;
+        }
+        const loaded = await pending.catch(() => null);
+        if (!loaded) {
+            loading = null;
+            residentFingerprint = null;
+            return;
+        }
+        const sdk = await loadSdk();
+        await sdk.unloadModel({ modelId: loaded.modelId });
+        loading = null;
+        residentFingerprint = null;
+    };
+    const unload = async () => {
+        await lock.lock();
+        try {
+            await doUnload();
+        }
+        finally {
+            lock.unlock();
+        }
+    };
+    return {
+        loadSdk,
+        ensureLoaded: async (sdk, plan, signal) => {
+            const fingerprint = fingerprintOf(plan);
+            await lock.lock();
+            try {
+                if (loading && residentFingerprint !== fingerprint)
+                    await doUnload();
+                residentFingerprint = fingerprint;
+                loading ??= loadPack(sdk, plan).catch((err) => {
+                    loading = null;
+                    residentFingerprint = null;
+                    throw err;
+                });
+            }
+            finally {
+                lock.unlock();
+            }
+            return raceAbort(loading, signal);
+        },
+        unload,
+        enter: async (signal) => {
+            const pending = turn.lock();
+            try {
+                await raceAbort(pending, signal);
+            }
+            catch (err) {
+                void pending.then(() => turn.unlock());
+                throw err;
+            }
+        },
+        exit: () => turn.unlock()
+    };
+}
+export function stripLanguageLabel(text, ...codes) {
+    const at = text.indexOf(':');
+    if (at === -1)
+        return text;
+    const head = text.slice(0, at).trim();
+    const labels = codes.map((code) => languageLabel(code) ?? getLangName(code));
+    return labels.includes(head) ? text.slice(at + 1).trim() : text;
+}
+export function parseTranslateArgs(args) {
+    const text = typeof args.text === 'string' ? args.text.trim() : '';
+    if (!text)
+        return { error: 'text must be a non-empty string' };
+    const to = typeof args.to === 'string' ? args.to.trim() : '';
+    if (!to)
+        return { error: 'to must be a language code' };
+    const from = typeof args.from === 'string' ? args.from.trim() : undefined;
+    return { text, to, from };
+}
+export function sourceLanguage(text, from) {
+    if (from)
+        return from;
+    const [{ code }] = detectLanguage({ text });
+    if (code === UNDETERMINED)
+        return { error: NEEDS_SOURCE };
+    return code;
+}
+const NEEDS_SOURCE = 'could not detect the source language; pass `from` explicitly';
+export function createTranslateTool(runtime, { slm } = {}) {
+    return {
+        name: 'translate',
+        toolset: null,
+        description: 'Translate text between languages using the local neural translation models. ' +
+            `\`to\` and \`from\` are ISO 639-1 codes. Translating into ${ENGLISH} supports ` +
+            `${NMT_TO_ENGLISH.length} languages and out of it ${NMT_FROM_ENGLISH.length}; a pair ` +
+            'with English on neither side is translated through English, except between the ' +
+            'languages below. ' +
+            `${SLM_LANGUAGE_CODES.length} further languages, mostly African, may also be named by ` +
+            'their FLORES-200 code and always need an explicit `from`, which the detector cannot ' +
+            'name. Elsewhere `from` is detected from the text when omitted; pass it when known.',
+        parameters: {
+            type: 'object',
+            properties: {
+                text: { type: 'string', description: 'The text to translate' },
+                to: {
+                    type: 'string',
+                    description: 'Target language as an ISO 639-1 code, e.g. `es`, or a FLORES-200 code'
+                },
+                from: {
+                    type: 'string',
+                    description: 'Source language as an ISO 639-1 or FLORES-200 code; detected from the text when omitted'
+                }
+            },
+            required: ['text', 'to']
+        },
+        effect: () => 'read',
+        resource: (args) => ({ kind: 'translation', label: String(args.to ?? '') }),
+        execute: async (args, ctx) => {
+            const parsed = parseTranslateArgs(args);
+            if ('error' in parsed)
+                return parsed;
+            const from = sourceLanguage(parsed.text, parsed.from);
+            if (typeof from === 'object')
+                return from;
+            if (sameLanguage(from, parsed.to)) {
+                return { text: parsed.text, from, to: parsed.to, engine: 'none' };
+            }
+            const plan = translationPlanFor(from, parsed.to);
+            if ('error' in plan)
+                return plan;
+            if (plan.route === 'slm' && !slm) {
+                return { error: `this device cannot translate ${from} to ${parsed.to}` };
+            }
+            let entered = false;
+            try {
+                await runtime.enter(ctx?.signal);
+                entered = true;
+                if (plan.route === 'slm') {
+                    const text = await slm(plan.model, { text: parsed.text, from: plan.from, to: plan.to }, ctx?.signal);
+                    return {
+                        text: stripLanguageLabel(text, plan.from, plan.to),
+                        from,
+                        to: parsed.to,
+                        engine: 'slm',
+                        model: plan.model
+                    };
+                }
+                const sdk = await runtime.loadSdk();
+                const { modelId } = await runtime.ensureLoaded(sdk, plan, ctx?.signal);
+                const source = plan.tag ? `${plan.tag} ${parsed.text}` : parsed.text;
+                const text = await raceAbort(sdk.translate({ modelId, text: source, stream: false, modelType: 'nmt' }).text, ctx?.signal);
+                return { text, from, to: parsed.to, engine: 'nmt', pivoted: Boolean(plan.pivot) };
+            }
+            catch (err) {
+                if (err instanceof AbortedError)
+                    return { error: 'translation canceled' };
+                throw err;
+            }
+            finally {
+                if (entered)
+                    runtime.exit();
+            }
+        }
+    };
+}

package/dist-lib/lib/tools/web-fetch.js

@@ -28,7 +28,11 @@ export function createWebFetchTool({ fetch: fetchImpl, resolveHost = lookupHost
         toolset: 'web-search',
         description: 'Fetch a public web page and return its readable text. ' +
             'Use this to read the full content behind a web_search result URL, or a page the user ' +
-            'shares, when a search snippet is not enough to answer.',
+            'shares, when a search snippet is not enough to answer. ' +
+            'When the URL serves a file rather than a page — PDF, .docx, .xlsx, .pptx, an image, any ' +
+            'binary — this is also the tool that downloads it: the bytes become a chat attachment and ' +
+            'the result carries its attachmentId. It is the only way to get a remote file into the ' +
+            'chat; exec has no network and http_request truncates binary bodies.',
         parameters: {
             type: 'object',
             properties: {
@@ -178,8 +182,14 @@ async function attachDocument(response, url, mimeType, attach) {
         raw: !mime.startsWith('image/')
     });
     return {
+        attachmentId: record.id,
+        fileName: record.fileName,
+        byteLength: record.byteLength,
         content: `Attached the file "${record.fileName}" (${Math.round(record.byteLength / 1024)}KB, ${mime}) from ${url.toString()} — ` +
-            'the user can open or save it. Its contents were NOT read: do not claim to know what is inside.'
+            'the user can open or save it. Its contents were NOT read, so do not describe what is inside ' +
+            'yet. To read it, stage it with exec as ' +
+            `inputs [{ "attachmentId": "${record.id}", "path": "${record.fileName}" }] — never ask the ` +
+            'user to attach a file this call already attached.'
     };
 }
 const MIME_EXTENSIONS = {

package/dist-lib/schema/harness.js

@@ -69,7 +69,8 @@ export function registerHarnessTypes(ns) {
             'fetch-result',
             'rag-result',
             'question-result',
-            'drive-result'
+            'drive-result',
+            'invoke'
         ],
         strings: true
     });
@@ -109,7 +110,10 @@ export function registerHarnessTypes(ns) {
             { name: 'grantedDrives', type: '@harness/granted-drive', array: true },
             { name: 'payload', type: 'json' },
             { name: 'grantedWriteDirs', type: 'string', array: true },
-            { name: 'captureInferenceCalls', type: 'bool' }
+            { name: 'captureInferenceCalls', type: 'bool' },
+            { name: 'capability', type: 'string' },
+            { name: 'arguments', type: 'json' },
+            { name: 'enhance', type: 'string' }
         ]
     });
     ns.register({
@@ -459,6 +463,25 @@ export function registerHarnessTypes(ns) {
         name: 'transcribe-response',
         fields: [{ name: 'text', type: 'string', required: true }]
     });
+    ns.register({
+        name: 'detect-language-request',
+        fields: [
+            { name: 'text', type: 'string', required: true },
+            { name: 'topK', type: 'uint' }
+        ]
+    });
+    ns.register({
+        name: 'detected-language',
+        fields: [
+            { name: 'code', type: 'string', required: true },
+            { name: 'language', type: 'string', required: true },
+            { name: 'probability', type: 'float64' }
+        ]
+    });
+    ns.register({
+        name: 'detect-language-response',
+        fields: [{ name: 'languages', type: '@harness/detected-language', array: true, required: true }]
+    });
     ns.register({
         name: 'speak-request',
         fields: [
@@ -507,7 +530,8 @@ export function registerHarnessTypes(ns) {
             { name: 'steps', type: 'string', array: true },
             { name: 'configSteps', type: 'string', array: true },
             { name: 'fields', type: '@harness/skill-setup-field', array: true },
-            { name: 'satisfied', type: 'bool' }
+            { name: 'satisfied', type: 'bool' },
+            { name: 'oauth', type: '@harness/skill-oauth-config' }
         ]
     });
     ns.register({
@@ -614,6 +638,27 @@ export function registerHarnessTypes(ns) {
         name: 'kv-cache-delete-request',
         fields: [{ name: 'kvCacheKey', type: 'string', required: true }]
     });
+    ns.register({
+        name: 'skill-oauth-param',
+        fields: [
+            { name: 'key', type: 'string', required: true },
+            { name: 'value', type: 'string', required: true }
+        ]
+    });
+    ns.register({
+        name: 'skill-oauth-config',
+        fields: [
+            { name: 'stateKey', type: 'string', required: true },
+            { name: 'authUrl', type: 'string', required: true },
+            { name: 'tokenUrl', type: 'string', required: true },
+            { name: 'tokenAuth', type: 'string', required: true },
+            { name: 'scopes', type: 'string', array: true },
+            { name: 'extraAuthParams', type: '@harness/skill-oauth-param', array: true },
+            { name: 'registrationEndpoint', type: 'string' },
+            { name: 'port', type: 'uint' },
+            { name: 'sharedIosSession', type: 'bool' }
+        ]
+    });
 }
 export function registerHarnessApi(api) {
     api.register({
@@ -756,4 +801,9 @@ export function registerHarnessApi(api) {
         request: { name: '@harness/kv-cache-delete-request', stream: false },
         response: { name: '@harness/kv-cache-deletion-response', stream: false }
     });
+    api.register({
+        name: 'detect-language',
+        request: { name: '@harness/detect-language-request', stream: false },
+        response: { name: '@harness/detect-language-response', stream: false }
+    });
 }

package/dist-lib/spec/bind.js

@@ -124,4 +124,7 @@ export function bindApi(rpc, api) {
   if (api.deleteKvCache) {
     rpc.onDeleteKvCache((req) => api.deleteKvCache(req));
   }
+  if (api.detectLanguage) {
+    rpc.onDetectLanguage((req) => api.detectLanguage(req));
+  }
 }

package/dist-lib/spec/calls.js

@@ -38,5 +38,6 @@ export function harnessClient(rpc) {
     copySkillForEdit: (input) => rpc.copySkillForEdit(input),
     deleteOverlaySkill: (input) => rpc.deleteOverlaySkill(input),
     deleteKvCache: (input) => rpc.deleteKvCache(input),
+    detectLanguage: (input) => rpc.detectLanguage(input),
   };
 }

package/dist-lib/spec/hrpc/hrpc.json

@@ -364,6 +364,19 @@
         "stream": false
       },
       "version": 1
+    },
+    {
+      "id": 28,
+      "name": "@harness/detect-language",
+      "request": {
+        "name": "@harness/detect-language-request",
+        "stream": false
+      },
+      "response": {
+        "name": "@harness/detect-language-response",
+        "stream": false
+      },
+      "version": 1
     }
   ]
 }

package/dist-lib/spec/hrpc/index.js

@@ -61,7 +61,9 @@ const methods = new Map([
   ['@harness/delete-overlay-skill', 26],
   [26, '@harness/delete-overlay-skill'],
   ['@harness/delete-kv-cache', 27],
-  [27, '@harness/delete-kv-cache']
+  [27, '@harness/delete-kv-cache'],
+  ['@harness/detect-language', 28],
+  [28, '@harness/detect-language']
 ])
 
 class HRPC {
@@ -96,7 +98,8 @@ class HRPC {
       ['@harness/write-skill-file', getEncoding('@harness/write-skill-request')],
       ['@harness/copy-skill-for-edit', getEncoding('@harness/skill-name-request')],
       ['@harness/delete-overlay-skill', getEncoding('@harness/skill-name-request')],
-      ['@harness/delete-kv-cache', getEncoding('@harness/kv-cache-delete-request')]
+      ['@harness/delete-kv-cache', getEncoding('@harness/kv-cache-delete-request')],
+      ['@harness/detect-language', getEncoding('@harness/detect-language-request')]
     ])
     this._responseEncodings = new Map([
       ['@harness/completion', getEncoding('@harness/completion-event')],
@@ -126,7 +129,8 @@ class HRPC {
       ['@harness/write-skill-file', getEncoding('@harness/ok-response')],
       ['@harness/copy-skill-for-edit', getEncoding('@harness/copy-skill-response')],
       ['@harness/delete-overlay-skill', getEncoding('@harness/delete-skill-response')],
-      ['@harness/delete-kv-cache', getEncoding('@harness/kv-cache-deletion-response')]
+      ['@harness/delete-kv-cache', getEncoding('@harness/kv-cache-deletion-response')],
+      ['@harness/detect-language', getEncoding('@harness/detect-language-response')]
     ])
     this._rpc = new RPC(stream, async (req) => {
       const command = methods.get(req.command)
@@ -336,6 +340,10 @@ class HRPC {
     return this._call('@harness/delete-kv-cache', args)
   }
 
+  async detectLanguage(args) {
+    return this._call('@harness/detect-language', args)
+  }
+
   onCompletion(responseFn) {
     this._handlers['@harness/completion'] = responseFn
   }
@@ -448,6 +456,10 @@ class HRPC {
     this._handlers['@harness/delete-kv-cache'] = responseFn
   }
 
+  onDetectLanguage(responseFn) {
+    this._handlers['@harness/detect-language'] = responseFn
+  }
+
   _requestIsStream(command) {
     return [
       '@harness/completion',