@qvac/harness 0.2.12 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/harness.bundle +9 -8
- package/dist-lib/index.d.ts +11 -1
- package/dist-lib/index.js +66 -5
- package/dist-lib/lib/capabilities/completion.d.ts +2 -0
- package/dist-lib/lib/capabilities/enhance.d.ts +10 -0
- package/dist-lib/lib/capabilities/enhance.js +29 -0
- package/dist-lib/lib/capabilities/invoke.d.ts +11 -2
- package/dist-lib/lib/capabilities/invoke.js +32 -8
- package/dist-lib/lib/connect.js +1 -0
- package/dist-lib/lib/contract-types.d.ts +1 -0
- package/dist-lib/lib/helpers/clean-answer.d.ts +1 -0
- package/dist-lib/lib/helpers/clean-answer.js +16 -0
- package/dist-lib/lib/helpers/load-stall.d.ts +6 -0
- package/dist-lib/lib/helpers/load-stall.js +24 -0
- package/dist-lib/lib/models/model-pool.d.ts +6 -0
- package/dist-lib/lib/models/model-pool.js +12 -0
- package/dist-lib/lib/models/nmt-catalog.d.ts +34 -0
- package/dist-lib/lib/models/nmt-catalog.js +166 -0
- package/dist-lib/lib/serve.js +1 -0
- package/dist-lib/lib/tools/generate-image.js +2 -25
- package/dist-lib/lib/tools/translate.d.ts +78 -0
- package/dist-lib/lib/tools/translate.js +228 -0
- package/dist-lib/schema/harness.js +2 -1
- package/dist-lib/spec/hrpc/messages.js +10 -5
- package/dist-lib/spec/hyperschema/index.js +10 -5
- package/dist-lib/spec/hyperschema/schema.json +6 -1
- package/dist-lib/spec/hyperschema/types.d.ts +2 -1
- package/package.json +6 -1
package/dist-lib/index.d.ts
CHANGED
|
@@ -3,6 +3,7 @@ import ReadyResource from 'ready-resource';
|
|
|
3
3
|
import { type Agent, type ApproveFn, type AskUserFn, type AttachFn, type CompletionEvent, type FetchAttachmentFn, type Message, type ProgressFn, type RagSearchFn, type ResponseFormat, type ToolDefinition, type ToolExecuteContext } from './lib/capabilities/completion.ts';
|
|
4
4
|
import { type DetectLanguageContext, type DetectedLanguage } from './lib/capabilities/detect-language.ts';
|
|
5
5
|
import { type Capability } from './lib/capabilities/invoke.ts';
|
|
6
|
+
import { type EnhanceMode } from './lib/capabilities/enhance.ts';
|
|
6
7
|
import { type DownloadArgs, type DownloadEventStream } from './lib/capabilities/download.ts';
|
|
7
8
|
import { type EmbedContext } from './lib/capabilities/embed.ts';
|
|
8
9
|
import { type KvCacheDeletionResult } from './lib/capabilities/kv-cache.ts';
|
|
@@ -25,6 +26,8 @@ import { type SkillSettingEntry } from './lib/tools/skills/skill-settings.ts';
|
|
|
25
26
|
export * from './lib/capabilities/completion.ts';
|
|
26
27
|
export { attachmentTurn, loadedSkillsFrom, skillLoadedTurn } from './lib/capabilities/tool-turns.ts';
|
|
27
28
|
export { CAPABILITIES, isCapability, type Capability } from './lib/capabilities/invoke.ts';
|
|
29
|
+
export type { EnhanceMode } from './lib/capabilities/enhance.ts';
|
|
30
|
+
export { EURONANO_LANGUAGES, NMT_FROM_ENGLISH, NMT_TO_ENGLISH, SLM_LANGUAGE_CODES, SLM_LANGUAGE_NAMES, translationPlanFor, type TranslationPlan } from './lib/models/nmt-catalog.ts';
|
|
28
31
|
export type { DetectLanguageContext, DetectedLanguage } from './lib/capabilities/detect-language.ts';
|
|
29
32
|
export * from './lib/capabilities/download.ts';
|
|
30
33
|
export type { EmbedContext } from './lib/capabilities/embed.ts';
|
|
@@ -113,6 +116,7 @@ export interface CompletionContext {
|
|
|
113
116
|
export interface InvokeContext {
|
|
114
117
|
capability: Capability;
|
|
115
118
|
args: Record<string, unknown>;
|
|
119
|
+
enhance?: EnhanceMode;
|
|
116
120
|
agent?: Agent;
|
|
117
121
|
attach?: AttachFn;
|
|
118
122
|
fetchAttachment?: FetchAttachmentFn;
|
|
@@ -157,7 +161,10 @@ export declare class Harness extends ReadyResource {
|
|
|
157
161
|
private _diffusionSdk;
|
|
158
162
|
private _imageRuntime;
|
|
159
163
|
private _musicSdk;
|
|
164
|
+
private _translationSdk;
|
|
160
165
|
private _musicRuntime;
|
|
166
|
+
private _translationRuntime;
|
|
167
|
+
private _capabilityTools;
|
|
161
168
|
private _capability;
|
|
162
169
|
private _capabilityInflight;
|
|
163
170
|
private readonly _modelFaults;
|
|
@@ -210,6 +217,8 @@ export declare class Harness extends ReadyResource {
|
|
|
210
217
|
private _loadedModels;
|
|
211
218
|
private _loadPluginSdk;
|
|
212
219
|
private _loadDiffusionSdk;
|
|
220
|
+
private _slmTranslate;
|
|
221
|
+
private _loadTranslationSdk;
|
|
213
222
|
private _loadMusicSdk;
|
|
214
223
|
private _deviceConfig;
|
|
215
224
|
private _createPool;
|
|
@@ -254,8 +263,9 @@ export declare class Harness extends ReadyResource {
|
|
|
254
263
|
agent?: Agent;
|
|
255
264
|
signal?: BareAbortSignal;
|
|
256
265
|
}): Promise<void>;
|
|
257
|
-
invoke({ capability, args, agent, attach, fetchAttachment, progress, signal }: InvokeContext): AsyncGenerator<CompletionEvent>;
|
|
266
|
+
invoke({ capability, args, enhance, agent, attach, fetchAttachment, progress, signal }: InvokeContext): AsyncGenerator<CompletionEvent>;
|
|
258
267
|
completion({ model, agent, messages, cacheKey, thinkingDefault, toolConfigs, generationParams, responseFormat, captureInferenceCalls, grantedDirs, grantedDrives, driveOp, grantedWriteDirs, approve, attach, fetchAttachment, ragSearch, progress, askUser, signal }: CompletionContext): AsyncGenerator<CompletionEvent>;
|
|
268
|
+
private _residentEnhancer;
|
|
259
269
|
private _makeRoomForCompletion;
|
|
260
270
|
private _evictWedgedModel;
|
|
261
271
|
private _evictAfterLeases;
|
package/dist-lib/index.js
CHANGED
|
@@ -8,6 +8,10 @@ import { closePythonWorker } from './lib/tools/exec/adapters/python/worker/clien
|
|
|
8
8
|
import { completion, imageSettingsFrom } from './lib/capabilities/completion.js';
|
|
9
9
|
import { detectLanguage } from './lib/capabilities/detect-language.js';
|
|
10
10
|
import { invoke } from './lib/capabilities/invoke.js';
|
|
11
|
+
import { expandImagePrompt } from './lib/capabilities/enhance.js';
|
|
12
|
+
import { cleanAnswer } from './lib/helpers/clean-answer.js';
|
|
13
|
+
import { isTranslationModel } from './lib/models/nmt-catalog.js';
|
|
14
|
+
import { createTranslateTool, createTranslationRuntime } from './lib/tools/translate.js';
|
|
11
15
|
import { download } from './lib/capabilities/download.js';
|
|
12
16
|
import { createEmbedLane } from './lib/capabilities/embed.js';
|
|
13
17
|
import { deleteAllKvCaches, deleteKvCacheKey } from './lib/capabilities/kv-cache.js';
|
|
@@ -19,7 +23,7 @@ import { AbortedError, raceAbort } from './lib/helpers/race-abort.js';
|
|
|
19
23
|
import { audioGenDownloadMetadata, customProjectionMap, defaultProjectionMap, defaultTtsFor, defaultVoiceModel, diffusionDownloadMetadata, infraModelsFor, modelNameToCapabilityProfile, musicModelSet, setCustomModels } from './lib/models/model-catalog.js';
|
|
20
24
|
import { createModelStatusCache, invalidateModelStatusCache, listModels } from './lib/models/model-list.js';
|
|
21
25
|
import { deleteModelFiles, summarizeDelete } from './lib/models/model-delete.js';
|
|
22
|
-
import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, awaitModelCapacity, evictIdle, evictModelWithin, isResident, leaseCount, loadedModels, modelKey, releaseModel, resolveLoadOptions, progressState } from './lib/models/model-pool.js';
|
|
26
|
+
import { createEmbedPool, createLoadTracker, createModelPool, createVoicePool, awaitModelCapacity, evictIdle, evictModelWithin, isResident, leaseResidentModel, leaseCount, loadedModels, modelKey, releaseModel, resolveLoadOptions, progressState } from './lib/models/model-pool.js';
|
|
23
27
|
import { resolveMainGpu } from './lib/models/gpu-selection.js';
|
|
24
28
|
import { openclCacheDir } from './lib/models/opencl-cache.js';
|
|
25
29
|
import { assertEmbedDevice, createEmbedConfig } from './lib/models/embed-config.js';
|
|
@@ -48,6 +52,7 @@ import { settingsScopeKey } from './lib/tools/skills/skill-settings.js';
|
|
|
48
52
|
export * from './lib/capabilities/completion.js';
|
|
49
53
|
export { attachmentTurn, loadedSkillsFrom, skillLoadedTurn } from './lib/capabilities/tool-turns.js';
|
|
50
54
|
export { CAPABILITIES, isCapability } from './lib/capabilities/invoke.js';
|
|
55
|
+
export { EURONANO_LANGUAGES, NMT_FROM_ENGLISH, NMT_TO_ENGLISH, SLM_LANGUAGE_CODES, SLM_LANGUAGE_NAMES, translationPlanFor } from './lib/models/nmt-catalog.js';
|
|
51
56
|
export * from './lib/capabilities/download.js';
|
|
52
57
|
export * from './lib/models/model-catalog.js';
|
|
53
58
|
export * from './lib/models/model-list.js';
|
|
@@ -119,7 +124,10 @@ export class Harness extends ReadyResource {
|
|
|
119
124
|
_diffusionSdk;
|
|
120
125
|
_imageRuntime;
|
|
121
126
|
_musicSdk;
|
|
127
|
+
_translationSdk;
|
|
122
128
|
_musicRuntime;
|
|
129
|
+
_translationRuntime;
|
|
130
|
+
_capabilityTools;
|
|
123
131
|
_capability = null;
|
|
124
132
|
_capabilityInflight = null;
|
|
125
133
|
_modelFaults = new Map();
|
|
@@ -204,6 +212,16 @@ export class Harness extends ReadyResource {
|
|
|
204
212
|
}
|
|
205
213
|
})
|
|
206
214
|
: null;
|
|
215
|
+
this._translationRuntime = createTranslationRuntime({
|
|
216
|
+
loads: this._loads,
|
|
217
|
+
loadSdk: () => this._loadTranslationSdk(),
|
|
218
|
+
onProgress: (model, progress) => this.emit('model', progressState(model, progress))
|
|
219
|
+
});
|
|
220
|
+
this._capabilityTools = [
|
|
221
|
+
createTranslateTool(this._translationRuntime, {
|
|
222
|
+
slm: (...args) => this._slmTranslate(...args)
|
|
223
|
+
})
|
|
224
|
+
];
|
|
207
225
|
this._tools = [
|
|
208
226
|
...tools,
|
|
209
227
|
...(this._imageRuntime
|
|
@@ -223,6 +241,7 @@ export class Harness extends ReadyResource {
|
|
|
223
241
|
this._platform = platform;
|
|
224
242
|
this._diffusionSdk = null;
|
|
225
243
|
this._musicSdk = null;
|
|
244
|
+
this._translationSdk = null;
|
|
226
245
|
}
|
|
227
246
|
async _open() {
|
|
228
247
|
installResolutionDetector(import.meta);
|
|
@@ -516,6 +535,27 @@ export class Harness extends ReadyResource {
|
|
|
516
535
|
this._diffusionSdk = sdk;
|
|
517
536
|
}, () => import('@qvac/sdk/sdcpp-generation/plugin').then((m) => m.diffusionPlugin));
|
|
518
537
|
}
|
|
538
|
+
async _slmTranslate(model, { text, from, to }, signal) {
|
|
539
|
+
const loadOpts = resolveLoadOptions({}, this._deviceConfig());
|
|
540
|
+
const key = modelKey(model, loadOpts);
|
|
541
|
+
const pool = this._models;
|
|
542
|
+
if (!isResident(pool, key))
|
|
543
|
+
await this._makeRoomForCompletion(signal);
|
|
544
|
+
try {
|
|
545
|
+
const { modelId } = await raceAbort(pool.open(key, model, loadOpts).load(), signal);
|
|
546
|
+
const sdk = await this._sdk.load();
|
|
547
|
+
const answer = await raceAbort(sdk.translate({ modelId, text, from, to, stream: false, modelType: 'llm' }).text, signal);
|
|
548
|
+
return cleanAnswer(answer, text);
|
|
549
|
+
}
|
|
550
|
+
finally {
|
|
551
|
+
releaseModel(pool, key);
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
_loadTranslationSdk() {
|
|
555
|
+
return this._loadPluginSdk(() => this._translationSdk, (sdk) => {
|
|
556
|
+
this._translationSdk = sdk;
|
|
557
|
+
}, () => import('@qvac/sdk/nmtcpp-translation/plugin').then((m) => m.nmtPlugin));
|
|
558
|
+
}
|
|
519
559
|
_loadMusicSdk() {
|
|
520
560
|
return this._loadPluginSdk(() => this._musicSdk, (sdk) => {
|
|
521
561
|
this._musicSdk = sdk;
|
|
@@ -648,7 +688,8 @@ export class Harness extends ReadyResource {
|
|
|
648
688
|
this._ttsModels.destroy(),
|
|
649
689
|
this._transcriptionModels.destroy(),
|
|
650
690
|
this._imageRuntime?.unload(),
|
|
651
|
-
this._musicRuntime?.unload()
|
|
691
|
+
this._musicRuntime?.unload(),
|
|
692
|
+
this._translationRuntime.unload()
|
|
652
693
|
]);
|
|
653
694
|
for (const outcome of released) {
|
|
654
695
|
if (outcome.status !== 'rejected')
|
|
@@ -949,17 +990,19 @@ export class Harness extends ReadyResource {
|
|
|
949
990
|
releaseModel(pool, key);
|
|
950
991
|
}
|
|
951
992
|
}
|
|
952
|
-
async *invoke({ capability, args, agent, attach, fetchAttachment, progress, signal }) {
|
|
993
|
+
async *invoke({ capability, args, enhance, agent, attach, fetchAttachment, progress, signal }) {
|
|
953
994
|
if (!this.opened)
|
|
954
995
|
await this.ready();
|
|
955
996
|
if (this.suspended)
|
|
956
997
|
return yield { type: 'aborted' };
|
|
957
|
-
hlog('invoke start', { capability });
|
|
998
|
+
hlog('invoke start', { capability, enhance });
|
|
958
999
|
try {
|
|
959
1000
|
yield* invoke({
|
|
960
1001
|
capability,
|
|
961
1002
|
args,
|
|
962
|
-
|
|
1003
|
+
enhance,
|
|
1004
|
+
enhancer: this._residentEnhancer(),
|
|
1005
|
+
tools: [...this._tools, ...this._capabilityTools],
|
|
963
1006
|
ctx: {
|
|
964
1007
|
image: imageSettingsFrom(agent),
|
|
965
1008
|
attach,
|
|
@@ -1089,6 +1132,24 @@ export class Harness extends ReadyResource {
|
|
|
1089
1132
|
void this._musicRuntime?.unloadIdle().catch(noop);
|
|
1090
1133
|
}
|
|
1091
1134
|
}
|
|
1135
|
+
_residentEnhancer() {
|
|
1136
|
+
return async (prompt, signal) => {
|
|
1137
|
+
const lease = await leaseResidentModel(this._models, (model) => !isTranslationModel(model));
|
|
1138
|
+
if (!lease)
|
|
1139
|
+
return null;
|
|
1140
|
+
try {
|
|
1141
|
+
const sdk = await this._sdk.load();
|
|
1142
|
+
return await expandImagePrompt(sdk, lease.modelId, lease.model, prompt, signal);
|
|
1143
|
+
}
|
|
1144
|
+
catch (err) {
|
|
1145
|
+
hlog('enhance failed', { err: err instanceof Error ? err.message : String(err) });
|
|
1146
|
+
return null;
|
|
1147
|
+
}
|
|
1148
|
+
finally {
|
|
1149
|
+
lease.release();
|
|
1150
|
+
}
|
|
1151
|
+
};
|
|
1152
|
+
}
|
|
1092
1153
|
async _makeRoomForCompletion(signal, deadlineMs = MODEL_ROOM_WAIT_MS) {
|
|
1093
1154
|
const roomed = await awaitModelCapacity(this._models, this._completionModelCap, deadlineMs, 250, signal);
|
|
1094
1155
|
if (!roomed) {
|
|
@@ -222,6 +222,8 @@ export type MetricsEvent = {
|
|
|
222
222
|
ctxSize?: number | undefined;
|
|
223
223
|
backends?: RunBackendUse[] | undefined;
|
|
224
224
|
usage?: UsageBreakdown | undefined;
|
|
225
|
+
route?: 'direct' | 'enhanced' | undefined;
|
|
226
|
+
routeModel?: string | undefined;
|
|
225
227
|
stopReason?: string | undefined;
|
|
226
228
|
};
|
|
227
229
|
export interface InferenceCall {
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { AbortSignal as BareAbortSignal } from 'bare-abort-controller';
|
|
2
|
+
import type { CompletionSdk } from './completion.ts';
|
|
3
|
+
export type EnhanceMode = 'never' | 'auto';
|
|
4
|
+
export interface EnhanceResult {
|
|
5
|
+
text: string;
|
|
6
|
+
model: string;
|
|
7
|
+
totalTokens?: number;
|
|
8
|
+
}
|
|
9
|
+
export type Enhancer = (prompt: string, signal?: BareAbortSignal) => Promise<EnhanceResult | null>;
|
|
10
|
+
export declare function expandImagePrompt(sdk: CompletionSdk, modelId: string, model: string, prompt: string, signal?: BareAbortSignal): Promise<EnhanceResult | null>;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { cleanAnswer } from '../helpers/clean-answer.js';
|
|
2
|
+
import { raceAbort } from '../helpers/race-abort.js';
|
|
3
|
+
const SYSTEM = "Rewrite the user's description as a single vivid prompt for an image generator. " +
|
|
4
|
+
'Answer with the prompt alone: no preamble, no quotes, no explanation.';
|
|
5
|
+
const MAX_PROMPT_CHARS = 400;
|
|
6
|
+
export async function expandImagePrompt(sdk, modelId, model, prompt, signal) {
|
|
7
|
+
const run = sdk.completion({
|
|
8
|
+
modelId,
|
|
9
|
+
history: [
|
|
10
|
+
{ role: 'system', content: SYSTEM },
|
|
11
|
+
{ role: 'user', content: prompt }
|
|
12
|
+
],
|
|
13
|
+
stream: true,
|
|
14
|
+
generationParams: { temp: 0.7, reasoning_budget: 0 }
|
|
15
|
+
});
|
|
16
|
+
let final;
|
|
17
|
+
try {
|
|
18
|
+
final = await raceAbort(run.final, signal);
|
|
19
|
+
}
|
|
20
|
+
catch (err) {
|
|
21
|
+
if (run.requestId)
|
|
22
|
+
void sdk.cancel?.({ requestId: run.requestId }).catch(() => { });
|
|
23
|
+
throw err;
|
|
24
|
+
}
|
|
25
|
+
const text = cleanAnswer(final.contentText).slice(0, MAX_PROMPT_CHARS);
|
|
26
|
+
if (!text)
|
|
27
|
+
return null;
|
|
28
|
+
return { text, model, totalTokens: final.stats?.generatedTokens };
|
|
29
|
+
}
|
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
import type { CompletionEvent, ToolDefinition, ToolExecuteContext } from './completion.ts';
|
|
2
|
+
import type { EnhanceMode, Enhancer } from './enhance.ts';
|
|
2
3
|
declare const CAPABILITY_TOOLS: {
|
|
3
|
-
readonly image:
|
|
4
|
+
readonly image: {
|
|
5
|
+
readonly tool: 'generate_image';
|
|
6
|
+
readonly enhances: 'prompt';
|
|
7
|
+
};
|
|
8
|
+
readonly translate: {
|
|
9
|
+
readonly tool: 'translate';
|
|
10
|
+
};
|
|
4
11
|
};
|
|
5
12
|
export type Capability = keyof typeof CAPABILITY_TOOLS;
|
|
6
13
|
export declare const CAPABILITIES: Capability[];
|
|
@@ -10,6 +17,8 @@ export interface InvokeOptions {
|
|
|
10
17
|
args: Record<string, unknown>;
|
|
11
18
|
tools: readonly ToolDefinition[];
|
|
12
19
|
ctx: ToolExecuteContext;
|
|
20
|
+
enhance?: EnhanceMode;
|
|
21
|
+
enhancer?: Enhancer;
|
|
13
22
|
}
|
|
14
|
-
export declare function invoke({ capability, args, tools, ctx }: InvokeOptions): AsyncGenerator<CompletionEvent>;
|
|
23
|
+
export declare function invoke({ capability, args, tools, ctx, enhance, enhancer }: InvokeOptions): AsyncGenerator<CompletionEvent>;
|
|
15
24
|
export {};
|
|
@@ -1,25 +1,43 @@
|
|
|
1
1
|
import { AbortedError } from '../helpers/race-abort.js';
|
|
2
2
|
const CAPABILITY_TOOLS = {
|
|
3
|
-
image: 'generate_image'
|
|
3
|
+
image: { tool: 'generate_image', enhances: 'prompt' },
|
|
4
|
+
translate: { tool: 'translate' }
|
|
4
5
|
};
|
|
5
6
|
export const CAPABILITIES = Object.keys(CAPABILITY_TOOLS);
|
|
6
7
|
export function isCapability(name) {
|
|
7
8
|
return name in CAPABILITY_TOOLS;
|
|
8
9
|
}
|
|
9
|
-
export async function* invoke({ capability, args, tools, ctx }) {
|
|
10
|
-
const
|
|
11
|
-
const tool = tools.find((candidate) => candidate.name ===
|
|
10
|
+
export async function* invoke({ capability, args, tools, ctx, enhance = 'never', enhancer }) {
|
|
11
|
+
const entry = CAPABILITY_TOOLS[capability];
|
|
12
|
+
const tool = tools.find((candidate) => candidate.name === entry.tool);
|
|
12
13
|
if (!tool) {
|
|
13
14
|
yield { type: 'error', message: `${capability} is not available on this device` };
|
|
14
15
|
return;
|
|
15
16
|
}
|
|
16
17
|
const startedAt = Date.now();
|
|
17
|
-
|
|
18
|
+
const field = 'enhances' in entry ? entry.enhances : null;
|
|
19
|
+
let routeModel;
|
|
20
|
+
let totalTokens = 0;
|
|
21
|
+
let called = args;
|
|
22
|
+
if (enhance === 'auto' && field && enhancer) {
|
|
23
|
+
const source = called[field];
|
|
24
|
+
if (typeof source === 'string' && source) {
|
|
25
|
+
const enhanced = await enhancer(source, ctx.signal);
|
|
26
|
+
if (enhanced) {
|
|
27
|
+
called = { ...called, [field]: enhanced.text };
|
|
28
|
+
routeModel = enhanced.model;
|
|
29
|
+
totalTokens = enhanced.totalTokens ?? 0;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
if (ctx.signal?.aborted)
|
|
34
|
+
return yield { type: 'aborted' };
|
|
35
|
+
yield { type: 'tool-call', name: entry.tool, args: called, effect: tool.effect(called, ctx) };
|
|
18
36
|
try {
|
|
19
|
-
const result = await tool.execute(
|
|
37
|
+
const result = await tool.execute(called, ctx);
|
|
20
38
|
if (ctx.signal?.aborted)
|
|
21
39
|
return yield { type: 'aborted' };
|
|
22
|
-
yield { type: 'tool-result', name, result };
|
|
40
|
+
yield { type: 'tool-result', name: entry.tool, result };
|
|
23
41
|
}
|
|
24
42
|
catch (err) {
|
|
25
43
|
if (err instanceof AbortedError || ctx.signal?.aborted) {
|
|
@@ -29,5 +47,11 @@ export async function* invoke({ capability, args, tools, ctx }) {
|
|
|
29
47
|
yield { type: 'error', message: err instanceof Error ? err.message : String(err) };
|
|
30
48
|
return;
|
|
31
49
|
}
|
|
32
|
-
yield {
|
|
50
|
+
yield {
|
|
51
|
+
type: 'metrics',
|
|
52
|
+
totalTokens,
|
|
53
|
+
durationMs: Date.now() - startedAt,
|
|
54
|
+
route: routeModel ? 'enhanced' : 'direct',
|
|
55
|
+
routeModel
|
|
56
|
+
};
|
|
33
57
|
}
|
package/dist-lib/lib/connect.js
CHANGED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function cleanAnswer(raw: string, source?: string): string;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { unwrap } from 'asenion';
|
|
2
|
+
import { firstMarker, markers, truncateAt } from 'asenion/guards';
|
|
3
|
+
const LEAKS = [
|
|
4
|
+
...markers.controlTokens,
|
|
5
|
+
...markers.turnEnd,
|
|
6
|
+
...markers.thinking,
|
|
7
|
+
...markers.toolMarkup,
|
|
8
|
+
...markers.scaffold
|
|
9
|
+
];
|
|
10
|
+
const THINKING_CLOSE = markers.thinking.filter((marker) => marker.startsWith('</'));
|
|
11
|
+
export function cleanAnswer(raw, source = '') {
|
|
12
|
+
const close = firstMarker(raw, THINKING_CLOSE);
|
|
13
|
+
const body = close ? raw.slice(close.index + close.marker.length) : raw;
|
|
14
|
+
const text = truncateAt(body, LEAKS).trim();
|
|
15
|
+
return unwrap(source) === source ? unwrap(text) : text;
|
|
16
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
export function createLoadStall(ms, message) {
|
|
2
|
+
let timer = null;
|
|
3
|
+
let rejectStall = null;
|
|
4
|
+
const promise = new Promise((_, reject) => {
|
|
5
|
+
rejectStall = reject;
|
|
6
|
+
});
|
|
7
|
+
const arm = () => {
|
|
8
|
+
if (timer !== null)
|
|
9
|
+
clearTimeout(timer);
|
|
10
|
+
timer = setTimeout(() => {
|
|
11
|
+
rejectStall?.(new Error(message));
|
|
12
|
+
}, ms);
|
|
13
|
+
};
|
|
14
|
+
arm();
|
|
15
|
+
return {
|
|
16
|
+
bump: arm,
|
|
17
|
+
clear: () => {
|
|
18
|
+
if (timer !== null)
|
|
19
|
+
clearTimeout(timer);
|
|
20
|
+
timer = null;
|
|
21
|
+
},
|
|
22
|
+
promise
|
|
23
|
+
};
|
|
24
|
+
}
|
|
@@ -69,6 +69,12 @@ export interface LoadedModel {
|
|
|
69
69
|
type PoolLike = ReturnType<typeof createModelPool>;
|
|
70
70
|
export declare function releaseModel(pool: PoolLike, key: string): void;
|
|
71
71
|
export declare function loadedModels(pool: PoolLike): LoadedModel[];
|
|
72
|
+
export interface ResidentLease {
|
|
73
|
+
model: string;
|
|
74
|
+
modelId: string;
|
|
75
|
+
release: () => void;
|
|
76
|
+
}
|
|
77
|
+
export declare function leaseResidentModel(pool: PoolLike, wanted?: (model: string) => boolean): Promise<ResidentLease | null>;
|
|
72
78
|
export declare function isResident(pool: PoolLike, key: string): boolean;
|
|
73
79
|
export declare function leaseCount(pool: PoolLike, key: string): number;
|
|
74
80
|
export declare function evictIdle(pool: PoolLike): Promise<void>;
|
|
@@ -91,6 +91,18 @@ export function loadedModels(pool) {
|
|
|
91
91
|
}
|
|
92
92
|
return out;
|
|
93
93
|
}
|
|
94
|
+
export async function leaseResidentModel(pool, wanted = () => true) {
|
|
95
|
+
for (const entry of pool.resources.values()) {
|
|
96
|
+
const key = entry.id;
|
|
97
|
+
const pending = entry.value.peek();
|
|
98
|
+
if (!pending || entry.value.loading || !wanted(key.split('\0')[0]))
|
|
99
|
+
continue;
|
|
100
|
+
pool.open(key);
|
|
101
|
+
const loaded = await pending;
|
|
102
|
+
return { model: loaded.model, modelId: loaded.modelId, release: () => releaseModel(pool, key) };
|
|
103
|
+
}
|
|
104
|
+
return null;
|
|
105
|
+
}
|
|
94
106
|
export function isResident(pool, key) {
|
|
95
107
|
const handle = entryOf(pool, key)?.value;
|
|
96
108
|
return handle !== undefined && handle.peek() !== null;
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
export declare const ENGLISH = "en";
|
|
2
|
+
export declare const NMT_TO_ENGLISH: readonly string[];
|
|
3
|
+
export declare const NMT_FROM_ENGLISH: readonly string[];
|
|
4
|
+
export declare const SLM_MODEL = "TranslatePsy-AfriSLM-0.8B-Q4_K_M-imat";
|
|
5
|
+
export declare function isTranslationModel(model: string): boolean;
|
|
6
|
+
export declare const EURONANO_LANGUAGES: readonly string[];
|
|
7
|
+
export declare const SLM_LANGUAGE_CODES: readonly string[];
|
|
8
|
+
export declare function languageLabel(code: string): string | undefined;
|
|
9
|
+
export declare const SLM_LANGUAGE_NAMES: ReadonlyArray<{
|
|
10
|
+
code: string;
|
|
11
|
+
name: string;
|
|
12
|
+
}>;
|
|
13
|
+
export declare function slmLanguage(code: string): string | undefined;
|
|
14
|
+
export interface NmtPlan {
|
|
15
|
+
route: 'nmt';
|
|
16
|
+
from: string;
|
|
17
|
+
to: string;
|
|
18
|
+
model: string;
|
|
19
|
+
pivot?: string;
|
|
20
|
+
srcVocab?: string;
|
|
21
|
+
dstVocab?: string;
|
|
22
|
+
tag?: string;
|
|
23
|
+
}
|
|
24
|
+
export interface SlmPlan {
|
|
25
|
+
route: 'slm';
|
|
26
|
+
from: string;
|
|
27
|
+
to: string;
|
|
28
|
+
model: string;
|
|
29
|
+
}
|
|
30
|
+
export type TranslationPlan = NmtPlan | SlmPlan;
|
|
31
|
+
export declare function sameLanguage(from: string, to: string): boolean;
|
|
32
|
+
export declare function translationPlanFor(requestedFrom: string, requestedTo: string): TranslationPlan | {
|
|
33
|
+
error: string;
|
|
34
|
+
};
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
export const ENGLISH = 'en';
|
|
2
|
+
const TO_ENGLISH = [
|
|
3
|
+
'ar',
|
|
4
|
+
'az',
|
|
5
|
+
'be',
|
|
6
|
+
'bg',
|
|
7
|
+
'bn',
|
|
8
|
+
'bs',
|
|
9
|
+
'ca',
|
|
10
|
+
'cs',
|
|
11
|
+
'da',
|
|
12
|
+
'de',
|
|
13
|
+
'el',
|
|
14
|
+
'es',
|
|
15
|
+
'et',
|
|
16
|
+
'fa',
|
|
17
|
+
'fi',
|
|
18
|
+
'fr',
|
|
19
|
+
'gu',
|
|
20
|
+
'he',
|
|
21
|
+
'hi',
|
|
22
|
+
'hr',
|
|
23
|
+
'hu',
|
|
24
|
+
'id',
|
|
25
|
+
'is',
|
|
26
|
+
'it',
|
|
27
|
+
'ja',
|
|
28
|
+
'kn',
|
|
29
|
+
'ko',
|
|
30
|
+
'lt',
|
|
31
|
+
'lv',
|
|
32
|
+
'ml',
|
|
33
|
+
'ms',
|
|
34
|
+
'mt',
|
|
35
|
+
'nb',
|
|
36
|
+
'nl',
|
|
37
|
+
'nn',
|
|
38
|
+
'no',
|
|
39
|
+
'pl',
|
|
40
|
+
'pt',
|
|
41
|
+
'ro',
|
|
42
|
+
'ru',
|
|
43
|
+
'sk',
|
|
44
|
+
'sl',
|
|
45
|
+
'sq',
|
|
46
|
+
'sr',
|
|
47
|
+
'sv',
|
|
48
|
+
'ta',
|
|
49
|
+
'te',
|
|
50
|
+
'th',
|
|
51
|
+
'tr',
|
|
52
|
+
'uk',
|
|
53
|
+
'vi',
|
|
54
|
+
'zh'
|
|
55
|
+
];
|
|
56
|
+
const FROM_ENGLISH = TO_ENGLISH.filter((code) => !['be', 'mt', 'nn'].includes(code));
|
|
57
|
+
export const NMT_TO_ENGLISH = TO_ENGLISH;
|
|
58
|
+
export const NMT_FROM_ENGLISH = FROM_ENGLISH;
|
|
59
|
+
const SLM_LANGUAGES = [
|
|
60
|
+
{ iso: 'af', flores: 'afr_Latn', name: 'Afrikaans' },
|
|
61
|
+
{ iso: 'am', flores: 'amh_Ethi', name: 'Amharic' },
|
|
62
|
+
{ iso: 'ha', flores: 'hau_Latn', name: 'Hausa' },
|
|
63
|
+
{ iso: 'ig', flores: 'ibo_Latn', name: 'Igbo' },
|
|
64
|
+
{ iso: 'rw', flores: 'kin_Latn', name: 'Kinyarwanda' },
|
|
65
|
+
{ iso: 'mg', flores: 'plt_Latn', name: 'Malagasy' },
|
|
66
|
+
{ iso: 'ny', flores: 'nya_Latn', name: 'Nyanja' },
|
|
67
|
+
{ iso: 'om', flores: 'gaz_Latn', name: 'Oromo' },
|
|
68
|
+
{ iso: 'sn', flores: 'sna_Latn', name: 'Shona' },
|
|
69
|
+
{ iso: 'so', flores: 'som_Latn', name: 'Somali' },
|
|
70
|
+
{ iso: 'st', flores: 'sot_Latn', name: 'Southern Sotho' },
|
|
71
|
+
{ iso: 'sw', flores: 'swh_Latn', name: 'Swahili' },
|
|
72
|
+
{ iso: 'tn', flores: 'tsn_Latn', name: 'Tswana' },
|
|
73
|
+
{ iso: 'xh', flores: 'xho_Latn', name: 'Xhosa' },
|
|
74
|
+
{ iso: 'yo', flores: 'yor_Latn', name: 'Yoruba' },
|
|
75
|
+
{ iso: 'zu', flores: 'zul_Latn', name: 'Zulu' }
|
|
76
|
+
];
|
|
77
|
+
export const SLM_MODEL = 'TranslatePsy-AfriSLM-0.8B-Q4_K_M-imat';
|
|
78
|
+
export function isTranslationModel(model) {
|
|
79
|
+
return /(^|[/\\])TranslatePsy-/.test(model);
|
|
80
|
+
}
|
|
81
|
+
const EURONANO = 'https://huggingface.co/qvac/TranslatePsy-EuroNano/resolve/main';
|
|
82
|
+
const EURONANO_SIZE = 'Base';
|
|
83
|
+
export const EURONANO_LANGUAGES = [
|
|
84
|
+
'cs',
|
|
85
|
+
'de',
|
|
86
|
+
'es',
|
|
87
|
+
'fi',
|
|
88
|
+
'fr',
|
|
89
|
+
'it',
|
|
90
|
+
'nl',
|
|
91
|
+
'pt',
|
|
92
|
+
'sv'
|
|
93
|
+
];
|
|
94
|
+
function euroNanoPack(direction) {
|
|
95
|
+
const dir = `${EURONANO}/${direction}/${EURONANO_SIZE}/intgemm`;
|
|
96
|
+
return { model: `${dir}/model.intgemm.alphas.bin`, vocab: `${dir}/vocab.spm` };
|
|
97
|
+
}
|
|
98
|
+
export const SLM_LANGUAGE_CODES = SLM_LANGUAGES.map((entry) => entry.iso);
|
|
99
|
+
export function languageLabel(code) {
|
|
100
|
+
const wanted = code.toLowerCase();
|
|
101
|
+
return SLM_LANGUAGES.find((entry) => entry.flores.toLowerCase() === wanted || entry.iso === wanted)?.name;
|
|
102
|
+
}
|
|
103
|
+
export const SLM_LANGUAGE_NAMES = SLM_LANGUAGES.map(({ iso, name }) => ({ code: iso, name }));
|
|
104
|
+
export function slmLanguage(code) {
|
|
105
|
+
const wanted = code.toLowerCase();
|
|
106
|
+
return SLM_LANGUAGES.find((entry) => entry.flores.toLowerCase() === wanted || entry.iso === wanted)?.flores;
|
|
107
|
+
}
|
|
108
|
+
const modelFilter = (pair) => `model.${pair}.intgemm`;
|
|
109
|
+
export function sameLanguage(from, to) {
|
|
110
|
+
return canonical(from) === canonical(to);
|
|
111
|
+
}
|
|
112
|
+
function canonical(code) {
|
|
113
|
+
return slmLanguage(code) ?? code.toLowerCase();
|
|
114
|
+
}
|
|
115
|
+
export function translationPlanFor(requestedFrom, requestedTo) {
|
|
116
|
+
const from = requestedFrom.toLowerCase();
|
|
117
|
+
const to = requestedTo.toLowerCase();
|
|
118
|
+
if (sameLanguage(from, to))
|
|
119
|
+
return { error: `${from} is already the target language` };
|
|
120
|
+
const slmFrom = slmLanguage(from);
|
|
121
|
+
const slmTo = slmLanguage(to);
|
|
122
|
+
if (slmFrom || slmTo) {
|
|
123
|
+
return { route: 'slm', from: slmFrom ?? from, to: slmTo ?? to, model: SLM_MODEL };
|
|
124
|
+
}
|
|
125
|
+
if (to === ENGLISH && EURONANO_LANGUAGES.includes(from)) {
|
|
126
|
+
const pack = euroNanoPack('xx-en');
|
|
127
|
+
return {
|
|
128
|
+
route: 'nmt',
|
|
129
|
+
from,
|
|
130
|
+
to,
|
|
131
|
+
model: pack.model,
|
|
132
|
+
srcVocab: pack.vocab,
|
|
133
|
+
dstVocab: pack.vocab
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
if (from === ENGLISH && EURONANO_LANGUAGES.includes(to)) {
|
|
137
|
+
const pack = euroNanoPack('en-xx');
|
|
138
|
+
return {
|
|
139
|
+
route: 'nmt',
|
|
140
|
+
from,
|
|
141
|
+
to,
|
|
142
|
+
model: pack.model,
|
|
143
|
+
srcVocab: pack.vocab,
|
|
144
|
+
dstVocab: pack.vocab,
|
|
145
|
+
tag: `##${to.toUpperCase()}`
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
if (to === ENGLISH) {
|
|
149
|
+
if (!TO_ENGLISH.includes(from))
|
|
150
|
+
return { error: `cannot translate from ${from}` };
|
|
151
|
+
return { route: 'nmt', from, to, model: modelFilter(`${from}${ENGLISH}`) };
|
|
152
|
+
}
|
|
153
|
+
if (!FROM_ENGLISH.includes(to))
|
|
154
|
+
return { error: `cannot translate to ${to}` };
|
|
155
|
+
if (from === ENGLISH)
|
|
156
|
+
return { route: 'nmt', from, to, model: modelFilter(`${ENGLISH}${to}`) };
|
|
157
|
+
if (!TO_ENGLISH.includes(from))
|
|
158
|
+
return { error: `cannot translate from ${from}` };
|
|
159
|
+
return {
|
|
160
|
+
route: 'nmt',
|
|
161
|
+
from,
|
|
162
|
+
to,
|
|
163
|
+
model: modelFilter(`${from}${ENGLISH}`),
|
|
164
|
+
pivot: modelFilter(`${ENGLISH}${to}`)
|
|
165
|
+
};
|
|
166
|
+
}
|
package/dist-lib/lib/serve.js
CHANGED
|
@@ -296,6 +296,7 @@ function runFrame(harness, frame, context) {
|
|
|
296
296
|
return harness.invoke({
|
|
297
297
|
capability,
|
|
298
298
|
args: frame.arguments ?? {},
|
|
299
|
+
enhance: frame.enhance === 'auto' ? 'auto' : 'never',
|
|
299
300
|
agent: frame.agent ?? undefined,
|
|
300
301
|
attach: context.attach,
|
|
301
302
|
fetchAttachment: context.fetchAttachment,
|