@qvac/harness 0.2.12 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +213 -0
- package/dist/harness.bundle +16 -13
- package/dist-lib/index.d.ts +12 -2
- package/dist-lib/index.js +66 -5
- package/dist-lib/lib/capabilities/completion.d.ts +5 -0
- package/dist-lib/lib/capabilities/completion.js +17 -5
- package/dist-lib/lib/capabilities/enhance.d.ts +10 -0
- package/dist-lib/lib/capabilities/enhance.js +29 -0
- package/dist-lib/lib/capabilities/invoke.d.ts +11 -2
- package/dist-lib/lib/capabilities/invoke.js +32 -8
- package/dist-lib/lib/connect.js +1 -0
- package/dist-lib/lib/contract-types.d.ts +1 -0
- package/dist-lib/lib/helpers/clean-answer.d.ts +1 -0
- package/dist-lib/lib/helpers/clean-answer.js +16 -0
- package/dist-lib/lib/helpers/load-stall.d.ts +6 -0
- package/dist-lib/lib/helpers/load-stall.js +24 -0
- package/dist-lib/lib/helpers/tool-call-markup.js +15 -7
- package/dist-lib/lib/models/model-pool.d.ts +6 -0
- package/dist-lib/lib/models/model-pool.js +12 -0
- package/dist-lib/lib/models/nmt-catalog.d.ts +34 -0
- package/dist-lib/lib/models/nmt-catalog.js +166 -0
- package/dist-lib/lib/no-redirect-fetch-impl.d.ts +3 -2
- package/dist-lib/lib/no-redirect-fetch-impl.js +4 -1
- package/dist-lib/lib/serve.js +1 -0
- package/dist-lib/lib/tools/exec/adapters/adapter.d.ts +2 -0
- package/dist-lib/lib/tools/exec/adapters/python/distributions.d.ts +1 -0
- package/dist-lib/lib/tools/exec/adapters/python/distributions.js +4 -0
- package/dist-lib/lib/tools/exec/adapters/python/guarded-fetch.d.ts +67 -0
- package/dist-lib/lib/tools/exec/adapters/python/guarded-fetch.js +196 -0
- package/dist-lib/lib/tools/exec/adapters/python/loader.js +2 -1
- package/dist-lib/lib/tools/exec/adapters/python/module-distributions.d.ts +1 -0
- package/dist-lib/lib/tools/exec/adapters/python/module-distributions.js +52 -0
- package/dist-lib/lib/tools/exec/adapters/python/net-shim.d.ts +2 -0
- package/dist-lib/lib/tools/exec/adapters/python/net-shim.js +148 -0
- package/dist-lib/lib/tools/exec/adapters/python/output-policy.js +13 -1
- package/dist-lib/lib/tools/exec/adapters/python/pyodide.d.ts +1 -0
- package/dist-lib/lib/tools/exec/adapters/python/pyodide.js +1 -0
- package/dist-lib/lib/tools/exec/adapters/python/python-adapter.d.ts +9 -3
- package/dist-lib/lib/tools/exec/adapters/python/python-adapter.js +130 -20
- package/dist-lib/lib/tools/exec/adapters/python/run-session.d.ts +3 -0
- package/dist-lib/lib/tools/exec/adapters/python/run-session.js +25 -8
- package/dist-lib/lib/tools/exec/adapters/python/wheels.d.ts +6 -0
- package/dist-lib/lib/tools/exec/adapters/python/wheels.js +17 -3
- package/dist-lib/lib/tools/exec/adapters/shell/shell-adapter.js +7 -1
- package/dist-lib/lib/tools/exec/shell-guidance.d.ts +10 -0
- package/dist-lib/lib/tools/exec/shell-guidance.js +123 -0
- package/dist-lib/lib/tools/exec/video-tools.d.ts +5 -0
- package/dist-lib/lib/tools/exec/video-tools.js +40 -0
- package/dist-lib/lib/tools/exec.d.ts +2 -1
- package/dist-lib/lib/tools/exec.js +59 -32
- package/dist-lib/lib/tools/generate-image.js +2 -25
- package/dist-lib/lib/tools/http-request/truncate-response.d.ts +1 -0
- package/dist-lib/lib/tools/http-request/truncate-response.js +1 -1
- package/dist-lib/lib/tools/http-request.js +4 -1
- package/dist-lib/lib/tools/operations/builders.d.ts +1 -1
- package/dist-lib/lib/tools/operations/builders.js +9 -0
- package/dist-lib/lib/tools/operations/compile.js +41 -18
- package/dist-lib/lib/tools/operations/decoders.d.ts +6 -0
- package/dist-lib/lib/tools/operations/decoders.js +82 -0
- package/dist-lib/lib/tools/operations/manifest.d.ts +16 -2
- package/dist-lib/lib/tools/operations/manifest.js +6 -1
- package/dist-lib/lib/tools/skill.d.ts +3 -0
- package/dist-lib/lib/tools/skill.js +38 -5
- package/dist-lib/lib/tools/skills/manifest-parse.js +43 -1
- package/dist-lib/lib/tools/skills/manifest.d.ts +13 -0
- package/dist-lib/lib/tools/translate.d.ts +78 -0
- package/dist-lib/lib/tools/translate.js +228 -0
- package/dist-lib/schema/harness.js +22 -2
- package/dist-lib/spec/hrpc/messages.js +101 -8
- package/dist-lib/spec/hyperschema/index.js +101 -8
- package/dist-lib/spec/hyperschema/schema.json +71 -1
- package/dist-lib/spec/hyperschema/types.d.ts +19 -1
- package/package.json +35 -2
- package/wheels/manifest.json +15 -0
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
import b4a from 'b4a';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { extractPageText } from '../../web-fetch/extract.js';
|
|
4
|
+
import { decodeHtmlEntities } from '../../web-search/html-helpers.js';
|
|
5
|
+
import { ELISION } from '../http-request/truncate-response.js';
|
|
6
|
+
export const DECODER_NAMES = ['gmail-message'];
|
|
7
|
+
const KEPT_HEADERS = ['From', 'To', 'Subject', 'Date', 'Message-ID', 'References'];
|
|
8
|
+
const MAX_TEXT_CHARS = 4000;
|
|
9
|
+
const PartSchema = z.lazy(() => z.object({
|
|
10
|
+
mimeType: z.string().optional(),
|
|
11
|
+
filename: z.string().optional(),
|
|
12
|
+
headers: z.array(z.object({ name: z.string(), value: z.string() })).optional(),
|
|
13
|
+
body: z.object({ data: z.string().optional() }).optional(),
|
|
14
|
+
parts: z.array(PartSchema).optional()
|
|
15
|
+
}));
|
|
16
|
+
const MessageSchema = z.object({
|
|
17
|
+
id: z.string().optional(),
|
|
18
|
+
threadId: z.string().optional(),
|
|
19
|
+
labelIds: z.array(z.string()).optional(),
|
|
20
|
+
snippet: z.string().optional(),
|
|
21
|
+
payload: PartSchema.optional()
|
|
22
|
+
});
|
|
23
|
+
const gmailMessage = {
|
|
24
|
+
decode(body) {
|
|
25
|
+
const parsed = MessageSchema.safeParse(body);
|
|
26
|
+
if (!parsed.success)
|
|
27
|
+
return body;
|
|
28
|
+
const { id, threadId, labelIds, snippet, payload } = parsed.data;
|
|
29
|
+
return {
|
|
30
|
+
id,
|
|
31
|
+
threadId,
|
|
32
|
+
labelIds,
|
|
33
|
+
headers: payload && keptHeaders(payload),
|
|
34
|
+
snippet: snippet && decodeHtmlEntities(snippet),
|
|
35
|
+
text: payload && capped(textOf(payload))
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
};
|
|
39
|
+
function keptHeaders(part) {
|
|
40
|
+
const kept = {};
|
|
41
|
+
for (const header of part.headers ?? []) {
|
|
42
|
+
const name = KEPT_HEADERS.find((known) => known.toLowerCase() === header.name.toLowerCase());
|
|
43
|
+
if (name && !(name in kept))
|
|
44
|
+
kept[name] = header.value;
|
|
45
|
+
}
|
|
46
|
+
return kept;
|
|
47
|
+
}
|
|
48
|
+
function textOf(part) {
|
|
49
|
+
const plain = leaves(part, 'text/plain');
|
|
50
|
+
if (plain.length)
|
|
51
|
+
return plain.join('\n\n');
|
|
52
|
+
const html = leaves(part, 'text/html')
|
|
53
|
+
.map((leaf) => extractPageText(leaf).text)
|
|
54
|
+
.filter((text) => text.length > 0);
|
|
55
|
+
if (html.length)
|
|
56
|
+
return html.join('\n\n');
|
|
57
|
+
return undefined;
|
|
58
|
+
}
|
|
59
|
+
function leaves(part, mimeType) {
|
|
60
|
+
if (part.parts?.length)
|
|
61
|
+
return part.parts.flatMap((child) => leaves(child, mimeType));
|
|
62
|
+
if (part.mimeType?.toLowerCase() !== mimeType || part.filename || !part.body?.data)
|
|
63
|
+
return [];
|
|
64
|
+
const text = decodeData(part.body.data);
|
|
65
|
+
return text ? [text] : [];
|
|
66
|
+
}
|
|
67
|
+
function decodeData(data) {
|
|
68
|
+
try {
|
|
69
|
+
return b4a.toString(b4a.from(data, 'base64url'), 'utf8').replace(/\r\n/g, '\n').trim();
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
return '';
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
function capped(text) {
|
|
76
|
+
if (text === undefined || text.length <= MAX_TEXT_CHARS)
|
|
77
|
+
return text;
|
|
78
|
+
return `${text.slice(0, MAX_TEXT_CHARS)}${ELISION}`;
|
|
79
|
+
}
|
|
80
|
+
export const DECODERS = {
|
|
81
|
+
'gmail-message': gmailMessage
|
|
82
|
+
};
|
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import type { ToolParameters } from '../../capabilities/completion.ts';
|
|
3
|
+
declare const ResponseSchema: z.ZodUnion<readonly [z.ZodObject<{
|
|
4
|
+
pick: z.ZodArray<z.ZodString>;
|
|
5
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
6
|
+
decode: z.ZodEnum<{
|
|
7
|
+
"gmail-message": "gmail-message";
|
|
8
|
+
}>;
|
|
9
|
+
}, z.core.$strict>]>;
|
|
3
10
|
declare const OperationSchema: z.ZodObject<{
|
|
4
11
|
tool: z.ZodString;
|
|
5
12
|
description: z.ZodString;
|
|
@@ -16,6 +23,7 @@ declare const OperationSchema: z.ZodObject<{
|
|
|
16
23
|
"drive-create-folder": "drive-create-folder";
|
|
17
24
|
"drive-trash": "drive-trash";
|
|
18
25
|
"gmail-draft": "gmail-draft";
|
|
26
|
+
"gmail-get": "gmail-get";
|
|
19
27
|
"gmail-send": "gmail-send";
|
|
20
28
|
"notion-create-page": "notion-create-page";
|
|
21
29
|
"notion-insert-content": "notion-insert-content";
|
|
@@ -43,6 +51,7 @@ declare const OperationSchema: z.ZodObject<{
|
|
|
43
51
|
"drive-create-folder": "drive-create-folder";
|
|
44
52
|
"drive-trash": "drive-trash";
|
|
45
53
|
"gmail-draft": "gmail-draft";
|
|
54
|
+
"gmail-get": "gmail-get";
|
|
46
55
|
"gmail-send": "gmail-send";
|
|
47
56
|
"notion-create-page": "notion-create-page";
|
|
48
57
|
"notion-insert-content": "notion-insert-content";
|
|
@@ -51,10 +60,15 @@ declare const OperationSchema: z.ZodObject<{
|
|
|
51
60
|
"sheets-write-values": "sheets-write-values";
|
|
52
61
|
}>;
|
|
53
62
|
}, z.core.$strip>]>;
|
|
54
|
-
response: z.ZodOptional<z.ZodObject<{
|
|
63
|
+
response: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
|
|
55
64
|
pick: z.ZodArray<z.ZodString>;
|
|
56
|
-
}, z.core.$
|
|
65
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
66
|
+
decode: z.ZodEnum<{
|
|
67
|
+
"gmail-message": "gmail-message";
|
|
68
|
+
}>;
|
|
69
|
+
}, z.core.$strict>]>>;
|
|
57
70
|
}, z.core.$strip>;
|
|
58
71
|
export type OperationSpec = z.infer<typeof OperationSchema>;
|
|
72
|
+
export type OperationResponse = z.infer<typeof ResponseSchema>;
|
|
59
73
|
export declare function parseOperationsManifest(json: string): OperationSpec[] | null;
|
|
60
74
|
export {};
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { BUILDER_NAMES } from './builders.js';
|
|
3
|
+
import { DECODER_NAMES } from './decoders.js';
|
|
3
4
|
const ParameterType = z.enum(['string', 'number', 'boolean', 'object', 'integer', 'array', 'null']);
|
|
4
5
|
const ParameterSchema = z.lazy(() => z.object({
|
|
5
6
|
type: z.union([ParameterType, z.array(ParameterType).min(1)]),
|
|
@@ -29,12 +30,16 @@ const McpRequestSchema = z.object({
|
|
|
29
30
|
builder: z.enum(BUILDER_NAMES)
|
|
30
31
|
});
|
|
31
32
|
const RequestSchema = z.union([McpRequestSchema, HttpRequestSchema]);
|
|
33
|
+
const ResponseSchema = z.union([
|
|
34
|
+
z.strictObject({ pick: z.array(z.string().min(1)).min(1) }),
|
|
35
|
+
z.strictObject({ decode: z.enum(DECODER_NAMES) })
|
|
36
|
+
]);
|
|
32
37
|
const OperationSchema = z.object({
|
|
33
38
|
tool: z.string().min(1),
|
|
34
39
|
description: z.string().min(1),
|
|
35
40
|
parameters: ParametersSchema,
|
|
36
41
|
request: RequestSchema,
|
|
37
|
-
response:
|
|
42
|
+
response: ResponseSchema.optional()
|
|
38
43
|
});
|
|
39
44
|
const ManifestSchema = z.object({ operations: z.array(OperationSchema) });
|
|
40
45
|
export function parseOperationsManifest(json) {
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import type { ToolDefinition } from '../capabilities/completion.ts';
|
|
2
2
|
import { type CredentialStoreFn } from './skills/auth-resolver.ts';
|
|
3
3
|
export type ReadSkillFn = (name: string, file: string) => Promise<string> | string;
|
|
4
|
+
export declare function mainFileRequested(name: string, file: unknown): boolean;
|
|
5
|
+
export declare function skillFileArg(args: Record<string, unknown> | null | undefined): unknown;
|
|
6
|
+
export declare function skillFileStem(file: string): string;
|
|
4
7
|
export interface SkillToolOptions {
|
|
5
8
|
skillsDir?: string;
|
|
6
9
|
overlayDir?: string | null;
|
|
@@ -10,6 +10,25 @@ import { parseToolGrant } from './skills/tool-grants.js';
|
|
|
10
10
|
import { shellEscapeArg } from './exec/shell-argv.js';
|
|
11
11
|
import { isObsidianVaultPath, OBSIDIAN_SKILL_NAME, OBSIDIAN_VAULT_PATH_CREDENTIAL_KEY } from './skills/obsidian-readiness.js';
|
|
12
12
|
const MAIN_FILE = 'SKILL.md';
|
|
13
|
+
const NO_FILE = /^(?:none|null|undefined|main)$/i;
|
|
14
|
+
export function mainFileRequested(name, file) {
|
|
15
|
+
if (file === undefined || file === null)
|
|
16
|
+
return true;
|
|
17
|
+
const asked = unquoteArg(file).toLowerCase();
|
|
18
|
+
return (asked === '' ||
|
|
19
|
+
NO_FILE.test(asked) ||
|
|
20
|
+
asked === name.toLowerCase() ||
|
|
21
|
+
asked === MAIN_FILE.toLowerCase());
|
|
22
|
+
}
|
|
23
|
+
export function skillFileArg(args) {
|
|
24
|
+
return args?.file ?? args?.path ?? args?.filename;
|
|
25
|
+
}
|
|
26
|
+
export function skillFileStem(file) {
|
|
27
|
+
return file
|
|
28
|
+
.toLowerCase()
|
|
29
|
+
.replace(/^references\//, '')
|
|
30
|
+
.replace(/\.md$/, '');
|
|
31
|
+
}
|
|
13
32
|
function resolveSkillDir(root, name) {
|
|
14
33
|
if (!root)
|
|
15
34
|
return null;
|
|
@@ -47,6 +66,10 @@ async function listSkillFiles(drives, name) {
|
|
|
47
66
|
}
|
|
48
67
|
return [...files].sort();
|
|
49
68
|
}
|
|
69
|
+
function nearestFile(wanted, available) {
|
|
70
|
+
const matches = available.filter((file) => skillFileStem(file) === skillFileStem(wanted));
|
|
71
|
+
return matches.length === 1 ? matches[0] : null;
|
|
72
|
+
}
|
|
50
73
|
async function vaultModeNote(name, credentialStore, ctx) {
|
|
51
74
|
if (name !== OBSIDIAN_SKILL_NAME || !credentialStore)
|
|
52
75
|
return null;
|
|
@@ -107,13 +130,13 @@ export function createSkillTool({ skillsDir, overlayDir = resolveSkillsOverlayDi
|
|
|
107
130
|
autoApprove: true,
|
|
108
131
|
async execute(args, ctx) {
|
|
109
132
|
const name = unquoteArg(args.name ?? args.skill_name ?? args.skill);
|
|
110
|
-
const fileArg = args
|
|
111
|
-
const file =
|
|
133
|
+
const fileArg = skillFileArg(args);
|
|
134
|
+
const file = mainFileRequested(name, fileArg) ? MAIN_FILE : unquoteArg(fileArg);
|
|
112
135
|
if (file.startsWith('/') || file.includes('\\') || file.split('/').includes('..')) {
|
|
113
136
|
return { error: `file not found in skill ${name}: ${file}` };
|
|
114
137
|
}
|
|
115
|
-
|
|
116
|
-
const { raw, root } = await resolveSkill(name,
|
|
138
|
+
const load = async (target) => {
|
|
139
|
+
const { raw, root } = await resolveSkill(name, target);
|
|
117
140
|
const { body, meta } = parseSkillFrontMatter(raw);
|
|
118
141
|
const dir = resolveSkillDir(root, name);
|
|
119
142
|
const content = dir ? body.replaceAll('{{SKILL_DIR}}', shellEscapeArg(dir)) : body;
|
|
@@ -123,14 +146,24 @@ export function createSkillTool({ skillsDir, overlayDir = resolveSkillsOverlayDi
|
|
|
123
146
|
].filter(Boolean);
|
|
124
147
|
return {
|
|
125
148
|
name,
|
|
126
|
-
file:
|
|
149
|
+
file: target === MAIN_FILE ? undefined : target,
|
|
127
150
|
content: notes.length ? [content.trim(), ...notes].join('\n\n') : content
|
|
128
151
|
};
|
|
152
|
+
};
|
|
153
|
+
try {
|
|
154
|
+
return await load(file);
|
|
129
155
|
}
|
|
130
156
|
catch {
|
|
131
157
|
if (file === MAIN_FILE)
|
|
132
158
|
return { error: `skill not found: ${name}` };
|
|
133
159
|
const available = await listSkillFiles([overlayDrive, baseDrive], name);
|
|
160
|
+
const near = nearestFile(file, available);
|
|
161
|
+
if (near !== null) {
|
|
162
|
+
try {
|
|
163
|
+
return await load(near);
|
|
164
|
+
}
|
|
165
|
+
catch { }
|
|
166
|
+
}
|
|
134
167
|
const rest = available.length
|
|
135
168
|
? ` — files this skill has: ${available.join(', ')}`
|
|
136
169
|
: '. This skill has no other files; you already have its full instructions — act now.';
|
|
@@ -223,6 +223,46 @@ function readAuthParams(v) {
|
|
|
223
223
|
const out = Object.fromEntries(Object.entries(record).filter(([key]) => !OAUTH_RESERVED_PARAMS.has(key)));
|
|
224
224
|
return Object.keys(out).length > 0 ? out : undefined;
|
|
225
225
|
}
|
|
226
|
+
const OAUTH_ERROR_CODE = /^[a-z0-9_]+$/;
|
|
227
|
+
function siteOf(url) {
|
|
228
|
+
return new URL(url).hostname.split('.').slice(-2).join('.');
|
|
229
|
+
}
|
|
230
|
+
function readVerifyRule(v) {
|
|
231
|
+
if (!isObj(v) || typeof v.error !== 'string' || !OAUTH_ERROR_CODE.test(v.error))
|
|
232
|
+
return undefined;
|
|
233
|
+
const status = Array.isArray(v.status)
|
|
234
|
+
? v.status.filter((s) => Number.isInteger(s) && s >= 100 && s < 600)
|
|
235
|
+
: [];
|
|
236
|
+
const field = typeof v.field === 'string' && v.field.length > 0 ? v.field : undefined;
|
|
237
|
+
if (status.length === 0 && !field)
|
|
238
|
+
return undefined;
|
|
239
|
+
return {
|
|
240
|
+
error: v.error,
|
|
241
|
+
...(status.length > 0 ? { status } : {}),
|
|
242
|
+
...(field ? { field } : {}),
|
|
243
|
+
...(field && typeof v.notEqual === 'string' ? { notEqual: v.notEqual } : {})
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
function readVerify(v, tokenUrl) {
|
|
247
|
+
if (!isObj(v))
|
|
248
|
+
return undefined;
|
|
249
|
+
const url = httpsUrl(v.url);
|
|
250
|
+
if (!url || siteOf(url) !== siteOf(tokenUrl))
|
|
251
|
+
return undefined;
|
|
252
|
+
if (v.method !== undefined && v.method !== null && v.method !== 'GET' && v.method !== 'POST') {
|
|
253
|
+
return undefined;
|
|
254
|
+
}
|
|
255
|
+
const reject = Array.isArray(v.reject) ? v.reject.flatMap((r) => readVerifyRule(r) ?? []) : [];
|
|
256
|
+
if (reject.length === 0)
|
|
257
|
+
return undefined;
|
|
258
|
+
const body = typeof v.body === 'string' ? v.body : isObj(v.body) ? JSON.stringify(v.body) : undefined;
|
|
259
|
+
return {
|
|
260
|
+
url,
|
|
261
|
+
...(v.method === 'GET' || v.method === 'POST' ? { method: v.method } : {}),
|
|
262
|
+
...(body !== undefined ? { body } : {}),
|
|
263
|
+
reject
|
|
264
|
+
};
|
|
265
|
+
}
|
|
226
266
|
function readOAuth(v) {
|
|
227
267
|
if (!isObj(v))
|
|
228
268
|
return undefined;
|
|
@@ -240,6 +280,7 @@ function readOAuth(v) {
|
|
|
240
280
|
return undefined;
|
|
241
281
|
const extraAuthParams = readAuthParams(v.extraAuthParams);
|
|
242
282
|
const registrationEndpoint = httpsUrl(v.registrationEndpoint);
|
|
283
|
+
const verify = readVerify(v.verify, tokenUrl);
|
|
243
284
|
return {
|
|
244
285
|
stateKey: v.stateKey,
|
|
245
286
|
authUrl,
|
|
@@ -249,7 +290,8 @@ function readOAuth(v) {
|
|
|
249
290
|
...(extraAuthParams ? { extraAuthParams } : {}),
|
|
250
291
|
...(registrationEndpoint ? { registrationEndpoint } : {}),
|
|
251
292
|
...(isPort(v.port) ? { port: v.port } : {}),
|
|
252
|
-
...(typeof v.sharedIosSession === 'boolean' ? { sharedIosSession: v.sharedIosSession } : {})
|
|
293
|
+
...(typeof v.sharedIosSession === 'boolean' ? { sharedIosSession: v.sharedIosSession } : {}),
|
|
294
|
+
...(verify ? { verify } : {})
|
|
253
295
|
};
|
|
254
296
|
}
|
|
255
297
|
export function toWireSetup(setup) {
|
|
@@ -39,6 +39,19 @@ export interface SkillOAuthConfig {
|
|
|
39
39
|
readonly registrationEndpoint?: string;
|
|
40
40
|
readonly port?: number;
|
|
41
41
|
readonly sharedIosSession?: boolean;
|
|
42
|
+
readonly verify?: SkillOAuthVerify;
|
|
43
|
+
}
|
|
44
|
+
export interface SkillOAuthVerify {
|
|
45
|
+
readonly url: string;
|
|
46
|
+
readonly method?: 'GET' | 'POST';
|
|
47
|
+
readonly body?: string;
|
|
48
|
+
readonly reject: readonly SkillOAuthVerifyRule[];
|
|
49
|
+
}
|
|
50
|
+
export interface SkillOAuthVerifyRule {
|
|
51
|
+
readonly error: string;
|
|
52
|
+
readonly status?: readonly number[];
|
|
53
|
+
readonly field?: string;
|
|
54
|
+
readonly notEqual?: string;
|
|
42
55
|
}
|
|
43
56
|
export interface SkillSetupRoute {
|
|
44
57
|
readonly kind: 'oauth' | 'token' | 'install' | 'picker' | 'instructions';
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import type { AbortSignal as BareAbortSignal } from 'bare-abort-controller';
|
|
2
|
+
import type { ToolDefinition } from '../capabilities/completion.ts';
|
|
3
|
+
import type { ModelSrc } from '../models/context.ts';
|
|
4
|
+
import { type NmtPlan } from '../models/nmt-catalog.ts';
|
|
5
|
+
import { type LoadTracker, type TrackedLoad } from '../models/model-pool.ts';
|
|
6
|
+
import { type ModelRegistrySdk } from '../models/model-source.ts';
|
|
7
|
+
interface BergamotConfig {
|
|
8
|
+
engine: 'Bergamot';
|
|
9
|
+
from: string;
|
|
10
|
+
to: string;
|
|
11
|
+
pivotModel?: {
|
|
12
|
+
modelSrc: ModelSrc;
|
|
13
|
+
};
|
|
14
|
+
srcVocabSrc?: string;
|
|
15
|
+
dstVocabSrc?: string;
|
|
16
|
+
}
|
|
17
|
+
export type TranslationSdk = ModelRegistrySdk & {
|
|
18
|
+
loadModel(params: {
|
|
19
|
+
modelSrc: ModelSrc;
|
|
20
|
+
modelType: 'nmtcpp-translation';
|
|
21
|
+
modelConfig: BergamotConfig;
|
|
22
|
+
onProgress?: (progress: Record<string, unknown>) => void;
|
|
23
|
+
}): TrackedLoad;
|
|
24
|
+
unloadModel(params: {
|
|
25
|
+
modelId: string;
|
|
26
|
+
}): Promise<unknown>;
|
|
27
|
+
cancel?(params: {
|
|
28
|
+
requestId: string;
|
|
29
|
+
}): Promise<void>;
|
|
30
|
+
translate(params: {
|
|
31
|
+
modelId: string;
|
|
32
|
+
text: string;
|
|
33
|
+
stream: false;
|
|
34
|
+
modelType: 'nmt';
|
|
35
|
+
}): {
|
|
36
|
+
text: Promise<string>;
|
|
37
|
+
};
|
|
38
|
+
};
|
|
39
|
+
export declare const TRANSLATION_LOAD_STALL_MS: number;
|
|
40
|
+
export declare const TRANSLATION_LOAD_STALL_ERROR = "translation model download stalled";
|
|
41
|
+
export interface TranslationRuntimeOptions {
|
|
42
|
+
loadSdk: () => Promise<TranslationSdk>;
|
|
43
|
+
loads?: LoadTracker;
|
|
44
|
+
onProgress?: (model: string, progress: Record<string, unknown>) => void;
|
|
45
|
+
stallMs?: number;
|
|
46
|
+
}
|
|
47
|
+
export interface TranslationRuntime {
|
|
48
|
+
loadSdk: () => Promise<TranslationSdk>;
|
|
49
|
+
ensureLoaded: (sdk: TranslationSdk, plan: NmtPlan, signal?: BareAbortSignal) => Promise<{
|
|
50
|
+
modelId: string;
|
|
51
|
+
}>;
|
|
52
|
+
unload: () => Promise<void>;
|
|
53
|
+
enter: (signal?: BareAbortSignal) => Promise<void>;
|
|
54
|
+
exit: () => void;
|
|
55
|
+
}
|
|
56
|
+
export declare function createTranslationRuntime({ loadSdk, loads, onProgress, stallMs }: TranslationRuntimeOptions): TranslationRuntime;
|
|
57
|
+
export type SlmTranslateFn = (model: string, params: {
|
|
58
|
+
text: string;
|
|
59
|
+
from: string;
|
|
60
|
+
to: string;
|
|
61
|
+
}, signal?: BareAbortSignal) => Promise<string>;
|
|
62
|
+
export declare function stripLanguageLabel(text: string, ...codes: string[]): string;
|
|
63
|
+
export interface TranslateArgs {
|
|
64
|
+
text: string;
|
|
65
|
+
to: string;
|
|
66
|
+
from?: string;
|
|
67
|
+
}
|
|
68
|
+
export declare function parseTranslateArgs(args: Record<string, unknown>): TranslateArgs | {
|
|
69
|
+
error: string;
|
|
70
|
+
};
|
|
71
|
+
export declare function sourceLanguage(text: string, from?: string): string | {
|
|
72
|
+
error: string;
|
|
73
|
+
};
|
|
74
|
+
export interface TranslateToolOptions {
|
|
75
|
+
slm?: SlmTranslateFn;
|
|
76
|
+
}
|
|
77
|
+
export declare function createTranslateTool(runtime: TranslationRuntime, { slm }?: TranslateToolOptions): ToolDefinition;
|
|
78
|
+
export {};
|
|
@@ -0,0 +1,228 @@
|
|
|
1
|
+
import ScopeLock from 'scope-lock';
|
|
2
|
+
import { getLangName } from '@qvac/langdetect-text';
|
|
3
|
+
import { detectLanguage, UNDETERMINED } from '../capabilities/detect-language.js';
|
|
4
|
+
import { AbortedError, raceAbort } from '../helpers/race-abort.js';
|
|
5
|
+
import { ENGLISH, NMT_FROM_ENGLISH, NMT_TO_ENGLISH, languageLabel, SLM_LANGUAGE_CODES, sameLanguage, translationPlanFor } from '../models/nmt-catalog.js';
|
|
6
|
+
import { createLoadStall } from '../helpers/load-stall.js';
|
|
7
|
+
import { createLoadTracker } from '../models/model-pool.js';
|
|
8
|
+
import { resolveModelSrc } from '../models/model-source.js';
|
|
9
|
+
export const TRANSLATION_LOAD_STALL_MS = 15 * 60_000;
|
|
10
|
+
export const TRANSLATION_LOAD_STALL_ERROR = 'translation model download stalled';
|
|
11
|
+
function packName(plan) {
|
|
12
|
+
const pair = plan.model.match(/model\.([a-z]+)\.intgemm/)?.[1];
|
|
13
|
+
if (pair)
|
|
14
|
+
return `Bergamot ${pair}`;
|
|
15
|
+
return plan.model.includes('/en-xx/') ? 'EuroNano en-xx' : 'EuroNano xx-en';
|
|
16
|
+
}
|
|
17
|
+
const fingerprintOf = (plan) => `${plan.model}|${plan.pivot ?? ''}`;
|
|
18
|
+
function bergamotConfig(plan) {
|
|
19
|
+
const config = { engine: 'Bergamot', from: plan.from, to: plan.to };
|
|
20
|
+
if (plan.srcVocab)
|
|
21
|
+
config.srcVocabSrc = plan.srcVocab;
|
|
22
|
+
if (plan.dstVocab)
|
|
23
|
+
config.dstVocabSrc = plan.dstVocab;
|
|
24
|
+
return config;
|
|
25
|
+
}
|
|
26
|
+
export function createTranslationRuntime({ loadSdk, loads = createLoadTracker(), onProgress, stallMs = TRANSLATION_LOAD_STALL_MS }) {
|
|
27
|
+
let loading = null;
|
|
28
|
+
let residentFingerprint = null;
|
|
29
|
+
const lock = new ScopeLock();
|
|
30
|
+
const turn = new ScopeLock();
|
|
31
|
+
const loadPack = async (sdk, plan) => {
|
|
32
|
+
const modelSrc = await resolveModelSrc(sdk, plan.model, 'nmt');
|
|
33
|
+
const modelConfig = bergamotConfig(plan);
|
|
34
|
+
if (plan.pivot) {
|
|
35
|
+
modelConfig.pivotModel = { modelSrc: await resolveModelSrc(sdk, plan.pivot, 'nmt') };
|
|
36
|
+
}
|
|
37
|
+
const name = packName(plan);
|
|
38
|
+
const stall = createLoadStall(stallMs, TRANSLATION_LOAD_STALL_ERROR);
|
|
39
|
+
onProgress?.(name, {});
|
|
40
|
+
const pending = sdk.loadModel({
|
|
41
|
+
modelSrc,
|
|
42
|
+
modelType: 'nmtcpp-translation',
|
|
43
|
+
modelConfig,
|
|
44
|
+
onProgress: (progress) => {
|
|
45
|
+
stall.bump();
|
|
46
|
+
onProgress?.(name, progress);
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
try {
|
|
50
|
+
const modelId = await Promise.race([loads.track(pending), stall.promise]);
|
|
51
|
+
return { modelId };
|
|
52
|
+
}
|
|
53
|
+
catch (err) {
|
|
54
|
+
if (err instanceof Error &&
|
|
55
|
+
err.message === TRANSLATION_LOAD_STALL_ERROR &&
|
|
56
|
+
pending.requestId) {
|
|
57
|
+
void sdk.cancel?.({ requestId: pending.requestId }).catch(() => { });
|
|
58
|
+
}
|
|
59
|
+
throw err;
|
|
60
|
+
}
|
|
61
|
+
finally {
|
|
62
|
+
stall.clear();
|
|
63
|
+
}
|
|
64
|
+
};
|
|
65
|
+
const doUnload = async () => {
|
|
66
|
+
const pending = loading;
|
|
67
|
+
if (!pending) {
|
|
68
|
+
residentFingerprint = null;
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
const loaded = await pending.catch(() => null);
|
|
72
|
+
if (!loaded) {
|
|
73
|
+
loading = null;
|
|
74
|
+
residentFingerprint = null;
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
const sdk = await loadSdk();
|
|
78
|
+
await sdk.unloadModel({ modelId: loaded.modelId });
|
|
79
|
+
loading = null;
|
|
80
|
+
residentFingerprint = null;
|
|
81
|
+
};
|
|
82
|
+
const unload = async () => {
|
|
83
|
+
await lock.lock();
|
|
84
|
+
try {
|
|
85
|
+
await doUnload();
|
|
86
|
+
}
|
|
87
|
+
finally {
|
|
88
|
+
lock.unlock();
|
|
89
|
+
}
|
|
90
|
+
};
|
|
91
|
+
return {
|
|
92
|
+
loadSdk,
|
|
93
|
+
ensureLoaded: async (sdk, plan, signal) => {
|
|
94
|
+
const fingerprint = fingerprintOf(plan);
|
|
95
|
+
await lock.lock();
|
|
96
|
+
try {
|
|
97
|
+
if (loading && residentFingerprint !== fingerprint)
|
|
98
|
+
await doUnload();
|
|
99
|
+
residentFingerprint = fingerprint;
|
|
100
|
+
loading ??= loadPack(sdk, plan).catch((err) => {
|
|
101
|
+
loading = null;
|
|
102
|
+
residentFingerprint = null;
|
|
103
|
+
throw err;
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
finally {
|
|
107
|
+
lock.unlock();
|
|
108
|
+
}
|
|
109
|
+
return raceAbort(loading, signal);
|
|
110
|
+
},
|
|
111
|
+
unload,
|
|
112
|
+
enter: async (signal) => {
|
|
113
|
+
const pending = turn.lock();
|
|
114
|
+
try {
|
|
115
|
+
await raceAbort(pending, signal);
|
|
116
|
+
}
|
|
117
|
+
catch (err) {
|
|
118
|
+
void pending.then(() => turn.unlock());
|
|
119
|
+
throw err;
|
|
120
|
+
}
|
|
121
|
+
},
|
|
122
|
+
exit: () => turn.unlock()
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
export function stripLanguageLabel(text, ...codes) {
|
|
126
|
+
const at = text.indexOf(':');
|
|
127
|
+
if (at === -1)
|
|
128
|
+
return text;
|
|
129
|
+
const head = text.slice(0, at).trim();
|
|
130
|
+
const labels = codes.map((code) => languageLabel(code) ?? getLangName(code));
|
|
131
|
+
return labels.includes(head) ? text.slice(at + 1).trim() : text;
|
|
132
|
+
}
|
|
133
|
+
export function parseTranslateArgs(args) {
|
|
134
|
+
const text = typeof args.text === 'string' ? args.text.trim() : '';
|
|
135
|
+
if (!text)
|
|
136
|
+
return { error: 'text must be a non-empty string' };
|
|
137
|
+
const to = typeof args.to === 'string' ? args.to.trim() : '';
|
|
138
|
+
if (!to)
|
|
139
|
+
return { error: 'to must be a language code' };
|
|
140
|
+
const from = typeof args.from === 'string' ? args.from.trim() : undefined;
|
|
141
|
+
return { text, to, from };
|
|
142
|
+
}
|
|
143
|
+
export function sourceLanguage(text, from) {
|
|
144
|
+
if (from)
|
|
145
|
+
return from;
|
|
146
|
+
const [{ code }] = detectLanguage({ text });
|
|
147
|
+
if (code === UNDETERMINED)
|
|
148
|
+
return { error: NEEDS_SOURCE };
|
|
149
|
+
return code;
|
|
150
|
+
}
|
|
151
|
+
const NEEDS_SOURCE = 'could not detect the source language; pass `from` explicitly';
|
|
152
|
+
export function createTranslateTool(runtime, { slm } = {}) {
|
|
153
|
+
return {
|
|
154
|
+
name: 'translate',
|
|
155
|
+
toolset: null,
|
|
156
|
+
description: 'Translate text between languages using the local neural translation models. ' +
|
|
157
|
+
`\`to\` and \`from\` are ISO 639-1 codes. Translating into ${ENGLISH} supports ` +
|
|
158
|
+
`${NMT_TO_ENGLISH.length} languages and out of it ${NMT_FROM_ENGLISH.length}; a pair ` +
|
|
159
|
+
'with English on neither side is translated through English, except between the ' +
|
|
160
|
+
'languages below. ' +
|
|
161
|
+
`${SLM_LANGUAGE_CODES.length} further languages, mostly African, may also be named by ` +
|
|
162
|
+
'their FLORES-200 code and always need an explicit `from`, which the detector cannot ' +
|
|
163
|
+
'name. Elsewhere `from` is detected from the text when omitted; pass it when known.',
|
|
164
|
+
parameters: {
|
|
165
|
+
type: 'object',
|
|
166
|
+
properties: {
|
|
167
|
+
text: { type: 'string', description: 'The text to translate' },
|
|
168
|
+
to: {
|
|
169
|
+
type: 'string',
|
|
170
|
+
description: 'Target language as an ISO 639-1 code, e.g. `es`, or a FLORES-200 code'
|
|
171
|
+
},
|
|
172
|
+
from: {
|
|
173
|
+
type: 'string',
|
|
174
|
+
description: 'Source language as an ISO 639-1 or FLORES-200 code; detected from the text when omitted'
|
|
175
|
+
}
|
|
176
|
+
},
|
|
177
|
+
required: ['text', 'to']
|
|
178
|
+
},
|
|
179
|
+
effect: () => 'read',
|
|
180
|
+
resource: (args) => ({ kind: 'translation', label: String(args.to ?? '') }),
|
|
181
|
+
execute: async (args, ctx) => {
|
|
182
|
+
const parsed = parseTranslateArgs(args);
|
|
183
|
+
if ('error' in parsed)
|
|
184
|
+
return parsed;
|
|
185
|
+
const from = sourceLanguage(parsed.text, parsed.from);
|
|
186
|
+
if (typeof from === 'object')
|
|
187
|
+
return from;
|
|
188
|
+
if (sameLanguage(from, parsed.to)) {
|
|
189
|
+
return { text: parsed.text, from, to: parsed.to, engine: 'none' };
|
|
190
|
+
}
|
|
191
|
+
const plan = translationPlanFor(from, parsed.to);
|
|
192
|
+
if ('error' in plan)
|
|
193
|
+
return plan;
|
|
194
|
+
if (plan.route === 'slm' && !slm) {
|
|
195
|
+
return { error: `this device cannot translate ${from} to ${parsed.to}` };
|
|
196
|
+
}
|
|
197
|
+
let entered = false;
|
|
198
|
+
try {
|
|
199
|
+
await runtime.enter(ctx?.signal);
|
|
200
|
+
entered = true;
|
|
201
|
+
if (plan.route === 'slm') {
|
|
202
|
+
const text = await slm(plan.model, { text: parsed.text, from: plan.from, to: plan.to }, ctx?.signal);
|
|
203
|
+
return {
|
|
204
|
+
text: stripLanguageLabel(text, plan.from, plan.to),
|
|
205
|
+
from,
|
|
206
|
+
to: parsed.to,
|
|
207
|
+
engine: 'slm',
|
|
208
|
+
model: plan.model
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
const sdk = await runtime.loadSdk();
|
|
212
|
+
const { modelId } = await runtime.ensureLoaded(sdk, plan, ctx?.signal);
|
|
213
|
+
const source = plan.tag ? `${plan.tag} ${parsed.text}` : parsed.text;
|
|
214
|
+
const text = await raceAbort(sdk.translate({ modelId, text: source, stream: false, modelType: 'nmt' }).text, ctx?.signal);
|
|
215
|
+
return { text, from, to: parsed.to, engine: 'nmt', pivoted: Boolean(plan.pivot) };
|
|
216
|
+
}
|
|
217
|
+
catch (err) {
|
|
218
|
+
if (err instanceof AbortedError)
|
|
219
|
+
return { error: 'translation canceled' };
|
|
220
|
+
throw err;
|
|
221
|
+
}
|
|
222
|
+
finally {
|
|
223
|
+
if (entered)
|
|
224
|
+
runtime.exit();
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
};
|
|
228
|
+
}
|
|
@@ -112,7 +112,8 @@ export function registerHarnessTypes(ns) {
|
|
|
112
112
|
{ name: 'grantedWriteDirs', type: 'string', array: true },
|
|
113
113
|
{ name: 'captureInferenceCalls', type: 'bool' },
|
|
114
114
|
{ name: 'capability', type: 'string' },
|
|
115
|
-
{ name: 'arguments', type: 'json' }
|
|
115
|
+
{ name: 'arguments', type: 'json' },
|
|
116
|
+
{ name: 'enhance', type: 'string' }
|
|
116
117
|
]
|
|
117
118
|
});
|
|
118
119
|
ns.register({
|
|
@@ -644,6 +645,24 @@ export function registerHarnessTypes(ns) {
|
|
|
644
645
|
{ name: 'value', type: 'string', required: true }
|
|
645
646
|
]
|
|
646
647
|
});
|
|
648
|
+
ns.register({
|
|
649
|
+
name: 'skill-oauth-verify-rule',
|
|
650
|
+
fields: [
|
|
651
|
+
{ name: 'error', type: 'string', required: true },
|
|
652
|
+
{ name: 'status', type: 'uint', array: true },
|
|
653
|
+
{ name: 'field', type: 'string' },
|
|
654
|
+
{ name: 'notEqual', type: 'string' }
|
|
655
|
+
]
|
|
656
|
+
});
|
|
657
|
+
ns.register({
|
|
658
|
+
name: 'skill-oauth-verify',
|
|
659
|
+
fields: [
|
|
660
|
+
{ name: 'url', type: 'string', required: true },
|
|
661
|
+
{ name: 'method', type: 'string' },
|
|
662
|
+
{ name: 'body', type: 'string' },
|
|
663
|
+
{ name: 'reject', type: '@harness/skill-oauth-verify-rule', array: true }
|
|
664
|
+
]
|
|
665
|
+
});
|
|
647
666
|
ns.register({
|
|
648
667
|
name: 'skill-oauth-config',
|
|
649
668
|
fields: [
|
|
@@ -655,7 +674,8 @@ export function registerHarnessTypes(ns) {
|
|
|
655
674
|
{ name: 'extraAuthParams', type: '@harness/skill-oauth-param', array: true },
|
|
656
675
|
{ name: 'registrationEndpoint', type: 'string' },
|
|
657
676
|
{ name: 'port', type: 'uint' },
|
|
658
|
-
{ name: 'sharedIosSession', type: 'bool' }
|
|
677
|
+
{ name: 'sharedIosSession', type: 'bool' },
|
|
678
|
+
{ name: 'verify', type: '@harness/skill-oauth-verify' }
|
|
659
679
|
]
|
|
660
680
|
});
|
|
661
681
|
}
|