@qvac/harness 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +740 -141
- package/README.md +75 -7
- package/dist/harness.bundle +72 -50
- package/dist-lib/index.d.ts +49 -39
- package/dist-lib/index.js +448 -161
- package/dist-lib/lib/audio-transcode-bare.d.ts +2 -2
- package/dist-lib/lib/audio-transcode-bare.js +28 -18
- package/dist-lib/lib/audio-transcode-node.d.ts +2 -2
- package/dist-lib/lib/capabilities/audio-resample.d.ts +7 -6
- package/dist-lib/lib/capabilities/audio-resample.js +30 -18
- package/dist-lib/lib/capabilities/completion.d.ts +20 -19
- package/dist-lib/lib/capabilities/completion.js +193 -132
- package/dist-lib/lib/capabilities/download.d.ts +6 -6
- package/dist-lib/lib/capabilities/download.js +15 -8
- package/dist-lib/lib/capabilities/embed.js +2 -1
- package/dist-lib/lib/capabilities/enhance.d.ts +6 -1
- package/dist-lib/lib/capabilities/enhance.js +7 -4
- package/dist-lib/lib/capabilities/invoke.js +7 -4
- package/dist-lib/lib/capabilities/kv-cache.js +2 -1
- package/dist-lib/lib/capabilities/preview-model-source.d.ts +9 -7
- package/dist-lib/lib/capabilities/preview-model-source.js +80 -28
- package/dist-lib/lib/capabilities/speech-text.js +5 -4
- package/dist-lib/lib/capabilities/tool-selection.js +10 -9
- package/dist-lib/lib/capabilities/tool-turns.js +30 -16
- package/dist-lib/lib/capabilities/usage-breakdown.js +2 -1
- package/dist-lib/lib/capabilities/voice.d.ts +5 -1
- package/dist-lib/lib/capabilities/voice.js +22 -12
- package/dist-lib/lib/connect.d.ts +13 -32
- package/dist-lib/lib/connect.js +170 -112
- package/dist-lib/lib/context-window.js +25 -21
- package/dist-lib/lib/contract-types.d.ts +35 -15
- package/dist-lib/lib/diagram-repair.d.ts +4 -4
- package/dist-lib/lib/diagram-repair.js +6 -3
- package/dist-lib/lib/docx/index.d.ts +3 -2
- package/dist-lib/lib/{error-fields.d.ts → guards.d.ts} +1 -0
- package/dist-lib/lib/{error-fields.js → guards.js} +3 -0
- package/dist-lib/lib/hardware/capability-probe.js +20 -10
- package/dist-lib/lib/hardware/gpu-classify.js +6 -3
- package/dist-lib/lib/hardware/gpu-probe-addon.js +8 -4
- package/dist-lib/lib/hardware/gpu-probe.js +4 -2
- package/dist-lib/lib/hardware/gpu-stats.native.cjs +8 -9
- package/dist-lib/lib/hardware/parsers.js +8 -4
- package/dist-lib/lib/helpers/buffer-burst.js +4 -2
- package/dist-lib/lib/helpers/control-tokens.js +4 -2
- package/dist-lib/lib/helpers/join-guard.d.ts +2 -2
- package/dist-lib/lib/helpers/join-guard.js +4 -2
- package/dist-lib/lib/helpers/load-stall.js +4 -2
- package/dist-lib/lib/helpers/markup-stream-filter.d.ts +2 -2
- package/dist-lib/lib/helpers/markup-stream-filter.js +6 -3
- package/dist-lib/lib/helpers/race-abort.js +2 -1
- package/dist-lib/lib/helpers/race-timeout.d.ts +5 -0
- package/dist-lib/lib/helpers/race-timeout.js +14 -0
- package/dist-lib/lib/helpers/run-outcome.d.ts +6 -8
- package/dist-lib/lib/helpers/run-outcome.js +36 -20
- package/dist-lib/lib/helpers/spawn-path.js +6 -3
- package/dist-lib/lib/helpers/text-coalescer.js +6 -3
- package/dist-lib/lib/helpers/thinking.js +38 -45
- package/dist-lib/lib/helpers/tool-call-markup.d.ts +9 -7
- package/dist-lib/lib/helpers/tool-call-markup.js +36 -20
- package/dist-lib/lib/helpers/tool-intent.d.ts +3 -2
- package/dist-lib/lib/helpers/tool-intent.js +4 -2
- package/dist-lib/lib/helpers/tool-label.js +18 -9
- package/dist-lib/lib/hooks/built-in/arg-repair.js +23 -13
- package/dist-lib/lib/hooks/built-in/egress.js +2 -1
- package/dist-lib/lib/hooks/built-in/skill-catalog.js +2 -1
- package/dist-lib/lib/hooks/pipeline.d.ts +5 -4
- package/dist-lib/lib/hooks/pipeline.js +6 -3
- package/dist-lib/lib/hooks/run.js +12 -8
- package/dist-lib/lib/model-download-progress.js +10 -8
- package/dist-lib/lib/models/context.d.ts +1 -1
- package/dist-lib/lib/models/context.js +5 -1
- package/dist-lib/lib/models/custom-model-cache.d.ts +10 -1
- package/dist-lib/lib/models/custom-model-cache.js +43 -15
- package/dist-lib/lib/models/gguf-header.d.ts +3 -2
- package/dist-lib/lib/models/gguf-header.js +44 -22
- package/dist-lib/lib/models/idle-unload.d.ts +14 -0
- package/dist-lib/lib/models/idle-unload.js +51 -0
- package/dist-lib/lib/models/kv-cache.d.ts +3 -2
- package/dist-lib/lib/models/kv-cache.js +4 -3
- package/dist-lib/lib/models/llm-presets.d.ts +2 -2
- package/dist-lib/lib/models/llm-presets.js +4 -2
- package/dist-lib/lib/models/model-catalog-constants.js +3 -2
- package/dist-lib/lib/models/model-catalog.d.ts +15 -20
- package/dist-lib/lib/models/model-catalog.js +66 -31
- package/dist-lib/lib/models/model-delete.js +18 -8
- package/dist-lib/lib/models/model-handle.js +2 -1
- package/dist-lib/lib/models/model-list.d.ts +8 -3
- package/dist-lib/lib/models/model-list.js +11 -6
- package/dist-lib/lib/models/model-pool.d.ts +48 -10
- package/dist-lib/lib/models/model-pool.js +106 -45
- package/dist-lib/lib/models/model-source.js +10 -5
- package/dist-lib/lib/models/music-catalog.d.ts +2 -2
- package/dist-lib/lib/models/nmt-catalog.d.ts +4 -3
- package/dist-lib/lib/models/nmt-catalog.js +10 -5
- package/dist-lib/lib/models/opencl-cache.d.ts +6 -1
- package/dist-lib/lib/models/opencl-cache.js +5 -3
- package/dist-lib/lib/models/sdk-faults.d.ts +11 -8
- package/dist-lib/lib/models/sdk-faults.js +9 -6
- package/dist-lib/lib/models/shards.d.ts +11 -0
- package/dist-lib/lib/models/shards.js +15 -0
- package/dist-lib/lib/models/tool-dialect.js +10 -5
- package/dist-lib/lib/no-redirect-fetch-bare.d.ts +3 -3
- package/dist-lib/lib/no-redirect-fetch-bare.js +1 -1
- package/dist-lib/lib/no-redirect-fetch-impl.js +6 -3
- package/dist-lib/lib/no-redirect-fetch-node.d.ts +1 -1
- package/dist-lib/lib/no-redirect-fetch-node.js +9 -9
- package/dist-lib/lib/office/index.d.ts +3 -2
- package/dist-lib/lib/office/python.d.ts +3 -2
- package/dist-lib/lib/office/python.js +2 -1
- package/dist-lib/lib/prompt.d.ts +5 -4
- package/dist-lib/lib/prompt.js +8 -4
- package/dist-lib/lib/queue.js +10 -5
- package/dist-lib/lib/resolution-detector.js +21 -11
- package/dist-lib/lib/rpc-session.js +2 -1
- package/dist-lib/lib/sdk-loader.js +2 -1
- package/dist-lib/lib/sdk-storage.js +4 -2
- package/dist-lib/lib/serve.js +122 -66
- package/dist-lib/lib/spawn.js +7 -4
- package/dist-lib/lib/tmp-path.d.ts +7 -6
- package/dist-lib/lib/tool-call-display.d.ts +2 -2
- package/dist-lib/lib/tools/create-documents.js +18 -10
- package/dist-lib/lib/tools/edit-documents.js +23 -13
- package/dist-lib/lib/tools/edit-image.d.ts +2 -2
- package/dist-lib/lib/tools/edit-image.js +60 -36
- package/dist-lib/lib/tools/exec/adapters/adapter.d.ts +10 -10
- package/dist-lib/lib/tools/exec/adapters/python/attachments.d.ts +17 -11
- package/dist-lib/lib/tools/exec/adapters/python/attachments.js +53 -40
- package/dist-lib/lib/tools/exec/adapters/python/create-effect.d.ts +2 -2
- package/dist-lib/lib/tools/exec/adapters/python/create-effect.js +60 -32
- package/dist-lib/lib/tools/exec/adapters/python/guarded-fetch.d.ts +33 -22
- package/dist-lib/lib/tools/exec/adapters/python/guarded-fetch.js +58 -29
- package/dist-lib/lib/tools/exec/adapters/python/interrupt-timer.d.ts +2 -2
- package/dist-lib/lib/tools/exec/adapters/python/loader.js +23 -12
- package/dist-lib/lib/tools/exec/adapters/python/memfs.d.ts +13 -11
- package/dist-lib/lib/tools/exec/adapters/python/memfs.js +10 -5
- package/dist-lib/lib/tools/exec/adapters/python/net-shim.js +3 -1
- package/dist-lib/lib/tools/exec/adapters/python/output-policy.d.ts +6 -4
- package/dist-lib/lib/tools/exec/adapters/python/output-policy.js +20 -10
- package/dist-lib/lib/tools/exec/adapters/python/pyodide.d.ts +2 -2
- package/dist-lib/lib/tools/exec/adapters/python/pyodide.js +14 -7
- package/dist-lib/lib/tools/exec/adapters/python/python-adapter.d.ts +41 -34
- package/dist-lib/lib/tools/exec/adapters/python/python-adapter.js +316 -166
- package/dist-lib/lib/tools/exec/adapters/python/run-session.d.ts +9 -8
- package/dist-lib/lib/tools/exec/adapters/python/run-session.js +56 -28
- package/dist-lib/lib/tools/exec/adapters/python/wheels.d.ts +8 -10
- package/dist-lib/lib/tools/exec/adapters/python/wheels.js +22 -11
- package/dist-lib/lib/tools/exec/adapters/python/worker/client.d.ts +3 -2
- package/dist-lib/lib/tools/exec/adapters/python/worker/client.js +41 -26
- package/dist-lib/lib/tools/exec/adapters/python/worker/entry.js +27 -18
- package/dist-lib/lib/tools/exec/adapters/python/worker/protocol.d.ts +2 -2
- package/dist-lib/lib/tools/exec/adapters/python/worker/protocol.js +14 -13
- package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-node.d.ts +4 -4
- package/dist-lib/lib/tools/exec/adapters/python/worker/spawn-node.js +10 -5
- package/dist-lib/lib/tools/exec/adapters/shell/shell-adapter.d.ts +4 -4
- package/dist-lib/lib/tools/exec/adapters/shell/shell-adapter.js +2 -1
- package/dist-lib/lib/tools/exec/obsidian/cli-process.d.ts +6 -1
- package/dist-lib/lib/tools/exec/obsidian/cli-process.js +8 -5
- package/dist-lib/lib/tools/exec/obsidian/cli.d.ts +6 -6
- package/dist-lib/lib/tools/exec/obsidian/cli.js +4 -4
- package/dist-lib/lib/tools/exec/obsidian/command.d.ts +3 -2
- package/dist-lib/lib/tools/exec/obsidian/command.js +6 -3
- package/dist-lib/lib/tools/exec/obsidian/output.js +2 -1
- package/dist-lib/lib/tools/exec/obsidian/route.d.ts +4 -6
- package/dist-lib/lib/tools/exec/obsidian/route.js +17 -11
- package/dist-lib/lib/tools/exec/obsidian/safe-path.js +2 -1
- package/dist-lib/lib/tools/exec/obsidian/vault-create.js +15 -7
- package/dist-lib/lib/tools/exec/obsidian/vault-paths.d.ts +2 -2
- package/dist-lib/lib/tools/exec/obsidian/vault-paths.js +20 -10
- package/dist-lib/lib/tools/exec/obsidian/vault-read.js +20 -13
- package/dist-lib/lib/tools/exec/obsidian/vault-write.js +24 -12
- package/dist-lib/lib/tools/exec/runner.d.ts +14 -14
- package/dist-lib/lib/tools/exec/runner.js +5 -3
- package/dist-lib/lib/tools/exec/scope.js +52 -28
- package/dist-lib/lib/tools/exec/shell-argv.js +31 -18
- package/dist-lib/lib/tools/exec/shell-guidance.d.ts +6 -1
- package/dist-lib/lib/tools/exec/shell-guidance.js +35 -20
- package/dist-lib/lib/tools/exec/video-tools.js +2 -2
- package/dist-lib/lib/tools/exec.d.ts +6 -8
- package/dist-lib/lib/tools/exec.js +113 -54
- package/dist-lib/lib/tools/file-glob.d.ts +4 -5
- package/dist-lib/lib/tools/file-glob.js +16 -11
- package/dist-lib/lib/tools/fs/glob.js +13 -18
- package/dist-lib/lib/tools/fs/remote-protocol.d.ts +10 -10
- package/dist-lib/lib/tools/fs/remote.d.ts +3 -2
- package/dist-lib/lib/tools/fs/remote.js +13 -7
- package/dist-lib/lib/tools/fs/scope.d.ts +8 -3
- package/dist-lib/lib/tools/fs/scope.js +20 -12
- package/dist-lib/lib/tools/fs/walk.js +6 -3
- package/dist-lib/lib/tools/fs/write.d.ts +3 -2
- package/dist-lib/lib/tools/generate-image.d.ts +11 -9
- package/dist-lib/lib/tools/generate-image.js +73 -37
- package/dist-lib/lib/tools/generate-music.d.ts +22 -11
- package/dist-lib/lib/tools/generate-music.js +79 -48
- package/dist-lib/lib/tools/grep.js +66 -41
- package/dist-lib/lib/tools/http-request/encode-fields.js +18 -9
- package/dist-lib/lib/tools/http-request/{utils.js → request.js} +20 -11
- package/dist-lib/lib/tools/http-request/truncate-response.js +57 -46
- package/dist-lib/lib/tools/http-request.d.ts +1 -1
- package/dist-lib/lib/tools/http-request.js +33 -18
- package/dist-lib/lib/tools/mcp/protocol.d.ts +53 -0
- package/dist-lib/lib/tools/mcp/{utils.js → protocol.js} +45 -23
- package/dist-lib/lib/tools/mcp-call.d.ts +1 -1
- package/dist-lib/lib/tools/mcp-call.js +37 -16
- package/dist-lib/lib/tools/operations/builders.js +10 -6
- package/dist-lib/lib/tools/operations/compile.js +20 -10
- package/dist-lib/lib/tools/operations/decoders.js +12 -6
- package/dist-lib/lib/tools/operations/gmail-message.js +3 -2
- package/dist-lib/lib/tools/outbound-guard.d.ts +4 -3
- package/dist-lib/lib/tools/outbound-guard.js +14 -8
- package/dist-lib/lib/tools/private-ip.js +8 -7
- package/dist-lib/lib/tools/rag-search.js +14 -7
- package/dist-lib/lib/tools/read-files.js +23 -20
- package/dist-lib/lib/tools/skill.js +19 -10
- package/dist-lib/lib/tools/skills/auth-resolver.d.ts +11 -18
- package/dist-lib/lib/tools/skills/auth-resolver.js +30 -18
- package/dist-lib/lib/tools/skills/bin-check.d.ts +3 -2
- package/dist-lib/lib/tools/skills/bin-check.js +10 -3
- package/dist-lib/lib/tools/skills/bundled-dir.js +20 -10
- package/dist-lib/lib/tools/skills/bundled-hash.d.ts +3 -2
- package/dist-lib/lib/tools/skills/bundled-hash.js +6 -1
- package/dist-lib/lib/tools/skills/catalog.d.ts +3 -2
- package/dist-lib/lib/tools/skills/catalog.js +33 -28
- package/dist-lib/lib/tools/skills/cli-argv.d.ts +3 -2
- package/dist-lib/lib/tools/skills/cli-argv.js +55 -52
- package/dist-lib/lib/tools/skills/cli-schema-check.js +21 -10
- package/dist-lib/lib/tools/skills/cli-schema-load.js +28 -20
- package/dist-lib/lib/tools/skills/cli-schema-types.d.ts +6 -6
- package/dist-lib/lib/tools/skills/cli-schema.js +2 -1
- package/dist-lib/lib/tools/skills/credential-check.d.ts +6 -3
- package/dist-lib/lib/tools/skills/credential-check.js +24 -12
- package/dist-lib/lib/tools/skills/credential-grants.js +1 -6
- package/dist-lib/lib/tools/skills/credential-health.d.ts +5 -1
- package/dist-lib/lib/tools/skills/credential-health.js +5 -3
- package/dist-lib/lib/tools/skills/frontmatter-parser.js +10 -5
- package/dist-lib/lib/tools/skills/manifest-parse.js +84 -64
- package/dist-lib/lib/tools/skills/mention.js +7 -6
- package/dist-lib/lib/tools/skills/obsidian-readiness.d.ts +2 -2
- package/dist-lib/lib/tools/skills/obsidian-readiness.js +11 -3
- package/dist-lib/lib/tools/skills/overlay-crud.d.ts +6 -4
- package/dist-lib/lib/tools/skills/overlay-crud.js +12 -6
- package/dist-lib/lib/tools/skills/overlay-dir.js +4 -2
- package/dist-lib/lib/tools/skills/overlay-watch.d.ts +3 -2
- package/dist-lib/lib/tools/skills/overlay-watch.js +8 -4
- package/dist-lib/lib/tools/skills/setup-routes.js +1 -3
- package/dist-lib/lib/tools/skills/skill-settings.js +4 -1
- package/dist-lib/lib/tools/skills/skill-state.d.ts +6 -11
- package/dist-lib/lib/tools/skills/skill-state.js +48 -29
- package/dist-lib/lib/tools/skills/tool-grants.js +6 -3
- package/dist-lib/lib/tools/toolsets.js +2 -1
- package/dist-lib/lib/tools/translate.d.ts +4 -5
- package/dist-lib/lib/tools/translate.js +31 -14
- package/dist-lib/lib/tools/url-guard.js +2 -1
- package/dist-lib/lib/tools/weather-lookup/place.js +8 -4
- package/dist-lib/lib/tools/weather-lookup/report.d.ts +7 -1
- package/dist-lib/lib/tools/weather-lookup/report.js +46 -9
- package/dist-lib/lib/tools/weather-lookup.js +18 -12
- package/dist-lib/lib/tools/web-fetch.js +47 -24
- package/dist-lib/lib/tools/web-search.js +8 -3
- package/dist-lib/lib/validating-lookup.d.ts +4 -4
- package/dist-lib/lib/wav-encode.d.ts +1 -1
- package/dist-lib/lib/wav-encode.js +11 -10
- package/dist-lib/lib/web-fetch/embedded-state.js +24 -12
- package/dist-lib/lib/web-fetch/extract.js +4 -2
- package/dist-lib/lib/web-fetch/images.js +8 -4
- package/dist-lib/lib/web-fetch/video.d.ts +2 -2
- package/dist-lib/lib/web-fetch/video.js +8 -4
- package/dist-lib/lib/web-search/engine-registry.d.ts +2 -2
- package/dist-lib/lib/web-search/engines/aol.js +1 -1
- package/dist-lib/lib/web-search/engines/bing.js +1 -1
- package/dist-lib/lib/web-search/engines/brave.js +4 -1
- package/dist-lib/lib/web-search/engines/duckduckgo.js +3 -2
- package/dist-lib/lib/web-search/engines/google.js +7 -3
- package/dist-lib/lib/web-search/engines/wikipedia.js +5 -4
- package/dist-lib/lib/web-search/engines/yahoo.js +1 -1
- package/dist-lib/lib/web-search/fetch-html.d.ts +5 -4
- package/dist-lib/lib/web-search/fetch-html.js +13 -7
- package/dist-lib/lib/web-search/orchestrator.d.ts +9 -5
- package/dist-lib/lib/web-search/orchestrator.js +3 -3
- package/dist-lib/lib/wire.d.ts +3 -2
- package/dist-lib/lib/wire.js +23 -15
- package/dist-lib/schema/baseline.js +10 -5
- package/dist-lib/schema/build.js +2 -1
- package/dist-lib/schema/harness.d.ts +15 -2
- package/dist-lib/schema/harness.js +39 -2
- package/dist-lib/schema/release-tags.d.ts +3 -2
- package/dist-lib/schema/release-tags.js +6 -3
- package/dist-lib/spec/bind.js +9 -0
- package/dist-lib/spec/calls.js +3 -0
- package/dist-lib/spec/hrpc/hrpc.json +39 -0
- package/dist-lib/spec/hrpc/index.js +39 -3
- package/dist-lib/spec/hrpc/messages.js +105 -8
- package/dist-lib/spec/hyperschema/index.js +105 -8
- package/dist-lib/spec/hyperschema/schema.json +77 -1
- package/dist-lib/spec/hyperschema/types.d.ts +25 -1
- package/package.json +14 -14
- package/worker/harness-sidecar.mjs +15 -22
- package/worker/stow-entry.mjs +7 -4
- package/dist-lib/lib/tools/mcp/utils.d.ts +0 -47
- /package/dist-lib/lib/tools/http-request/{utils.d.ts → request.d.ts} +0 -0
|
@@ -3,16 +3,18 @@ import { transferProgress } from '../model-download-progress.js';
|
|
|
3
3
|
import ResourcePuddle from 'resource-puddle';
|
|
4
4
|
import { ModelHandle } from './model-handle.js';
|
|
5
5
|
import { defaultContextSize } from './context.js';
|
|
6
|
+
import { customContextLength } from './model-catalog.js';
|
|
6
7
|
import { adrenoSafeKvCache } from './kv-cache.js';
|
|
7
8
|
import { DEFAULT_EMBED_BATCH_SIZE } from './embed-config.js';
|
|
8
9
|
import { hlog } from '../log.js';
|
|
9
10
|
import { RUNAWAY_STOP_SEQUENCES } from '../helpers/control-tokens.js';
|
|
11
|
+
import { raceTimeout } from '../helpers/race-timeout.js';
|
|
10
12
|
import { resolveModelSrc } from './model-source.js';
|
|
11
13
|
export function createLoadTracker() {
|
|
12
14
|
const inflight = new Set();
|
|
13
15
|
let draining = null;
|
|
14
16
|
const cancel = (sdk, requestId) => {
|
|
15
|
-
void sdk.cancel({ requestId }).catch(
|
|
17
|
+
void sdk.cancel({ requestId }).catch(noop);
|
|
16
18
|
};
|
|
17
19
|
return {
|
|
18
20
|
get draining() {
|
|
@@ -20,8 +22,9 @@ export function createLoadTracker() {
|
|
|
20
22
|
},
|
|
21
23
|
track(pending) {
|
|
22
24
|
const { requestId } = pending;
|
|
23
|
-
if (!requestId)
|
|
25
|
+
if (!requestId) {
|
|
24
26
|
return pending;
|
|
27
|
+
}
|
|
25
28
|
if (draining) {
|
|
26
29
|
cancel(draining, requestId);
|
|
27
30
|
return pending;
|
|
@@ -33,8 +36,9 @@ export function createLoadTracker() {
|
|
|
33
36
|
},
|
|
34
37
|
drain(sdk) {
|
|
35
38
|
draining = sdk;
|
|
36
|
-
for (const requestId of inflight)
|
|
39
|
+
for (const requestId of inflight) {
|
|
37
40
|
cancel(sdk, requestId);
|
|
41
|
+
}
|
|
38
42
|
inflight.clear();
|
|
39
43
|
},
|
|
40
44
|
undrain() {
|
|
@@ -60,9 +64,14 @@ export function modelKey(model, opts = {}) {
|
|
|
60
64
|
opts.projectionModel ?? ''
|
|
61
65
|
].join('\0');
|
|
62
66
|
}
|
|
67
|
+
class ModelPuddle extends ResourcePuddle {
|
|
68
|
+
}
|
|
63
69
|
function entryOf(pool, key) {
|
|
64
70
|
return pool.resources.get(key);
|
|
65
71
|
}
|
|
72
|
+
export function modelOf(key) {
|
|
73
|
+
return key.split('\0')[0];
|
|
74
|
+
}
|
|
66
75
|
export function releaseModel(pool, key) {
|
|
67
76
|
if (pool.destroyed)
|
|
68
77
|
return;
|
|
@@ -79,8 +88,9 @@ export function releaseModel(pool, key) {
|
|
|
79
88
|
void pending.then(() => releaseSettled(pool, key), () => releaseSettled(pool, key));
|
|
80
89
|
}
|
|
81
90
|
function releaseSettled(pool, key) {
|
|
82
|
-
if (!pool.destroyed)
|
|
91
|
+
if (!pool.destroyed) {
|
|
83
92
|
pool.release(key);
|
|
93
|
+
}
|
|
84
94
|
}
|
|
85
95
|
export function loadedModels(pool) {
|
|
86
96
|
const out = [];
|
|
@@ -95,8 +105,9 @@ export async function leaseResidentModel(pool, wanted = () => true) {
|
|
|
95
105
|
for (const entry of pool.resources.values()) {
|
|
96
106
|
const key = entry.id;
|
|
97
107
|
const pending = entry.value.peek();
|
|
98
|
-
if (!pending || entry.value.loading || !wanted(key.split('\0')[0]))
|
|
108
|
+
if (!pending || entry.value.loading || !wanted(key.split('\0')[0])) {
|
|
99
109
|
continue;
|
|
110
|
+
}
|
|
100
111
|
pool.open(key);
|
|
101
112
|
const loaded = await pending;
|
|
102
113
|
return { model: loaded.model, modelId: loaded.modelId, release: () => releaseModel(pool, key) };
|
|
@@ -115,8 +126,43 @@ export async function evictIdle(pool) {
|
|
|
115
126
|
await pool.gc();
|
|
116
127
|
}
|
|
117
128
|
}
|
|
118
|
-
export async function
|
|
119
|
-
|
|
129
|
+
export async function evictIdleWhere(pool, wanted) {
|
|
130
|
+
for (let tick = 0; tick <= pool.gcIdleTicks; tick++) {
|
|
131
|
+
if (pool.destroyed)
|
|
132
|
+
return;
|
|
133
|
+
const held = [];
|
|
134
|
+
for (const entry of pool.resources.values()) {
|
|
135
|
+
if (entry.refs === 0 && !wanted(entry.id)) {
|
|
136
|
+
held.push(entry.id);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
if (pool.idle === held.length) {
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
for (const key of held) {
|
|
143
|
+
pool.open(key);
|
|
144
|
+
}
|
|
145
|
+
try {
|
|
146
|
+
await pool.gc();
|
|
147
|
+
}
|
|
148
|
+
finally {
|
|
149
|
+
for (const key of held) {
|
|
150
|
+
releaseSettled(pool, key);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
function residentCount(pool, wanted) {
|
|
156
|
+
let count = 0;
|
|
157
|
+
for (const entry of pool.resources.values()) {
|
|
158
|
+
if (wanted(entry.id) && entry.value.peek() !== null) {
|
|
159
|
+
count++;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return count;
|
|
163
|
+
}
|
|
164
|
+
export async function awaitModelCapacity(pool, { cap, deadlineMs, tickMs = 250, signal, evictable = () => true }) {
|
|
165
|
+
const roomed = () => residentCount(pool, evictable) < cap;
|
|
120
166
|
let done = false;
|
|
121
167
|
let timer = null;
|
|
122
168
|
const expired = new Promise((resolve) => {
|
|
@@ -126,14 +172,15 @@ export async function awaitModelCapacity(pool, cap, deadlineMs, tickMs = 250, si
|
|
|
126
172
|
}, deadlineMs);
|
|
127
173
|
});
|
|
128
174
|
const evicting = (async () => {
|
|
129
|
-
await
|
|
175
|
+
await evictIdleWhere(pool, evictable);
|
|
130
176
|
while (!roomed()) {
|
|
131
177
|
if (done || signal?.aborted)
|
|
132
178
|
return false;
|
|
133
179
|
await new Promise((resolve) => setTimeout(resolve, tickMs));
|
|
134
|
-
if (done || signal?.aborted)
|
|
180
|
+
if (done || signal?.aborted) {
|
|
135
181
|
return false;
|
|
136
|
-
|
|
182
|
+
}
|
|
183
|
+
await evictIdleWhere(pool, evictable);
|
|
137
184
|
}
|
|
138
185
|
return true;
|
|
139
186
|
})();
|
|
@@ -142,40 +189,51 @@ export async function awaitModelCapacity(pool, cap, deadlineMs, tickMs = 250, si
|
|
|
142
189
|
}
|
|
143
190
|
finally {
|
|
144
191
|
done = true;
|
|
145
|
-
if (timer !== null)
|
|
192
|
+
if (timer !== null) {
|
|
146
193
|
clearTimeout(timer);
|
|
194
|
+
}
|
|
147
195
|
}
|
|
148
196
|
}
|
|
149
|
-
export
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
197
|
+
export function evictModelWithin(pool, key, options) {
|
|
198
|
+
return evictWhereWithin(pool, (candidate) => candidate === key, options);
|
|
199
|
+
}
|
|
200
|
+
export function evictModel(pool, model, options) {
|
|
201
|
+
return evictWhereWithin(pool, (key) => modelOf(key) === model, options);
|
|
202
|
+
}
|
|
203
|
+
async function evictWhereWithin(pool, wanted, { deadlineMs, tickMs = 250 }) {
|
|
204
|
+
let done = false;
|
|
154
205
|
const evicting = (async () => {
|
|
155
|
-
while (
|
|
156
|
-
await
|
|
157
|
-
if (
|
|
206
|
+
while (residentCount(pool, wanted) > 0) {
|
|
207
|
+
await evictIdleWhere(pool, wanted);
|
|
208
|
+
if (residentCount(pool, wanted) === 0) {
|
|
158
209
|
break;
|
|
210
|
+
}
|
|
159
211
|
await new Promise((resolve) => setTimeout(resolve, tickMs));
|
|
212
|
+
if (done) {
|
|
213
|
+
return false;
|
|
214
|
+
}
|
|
160
215
|
}
|
|
161
216
|
return true;
|
|
162
217
|
})();
|
|
163
218
|
try {
|
|
164
|
-
return await
|
|
219
|
+
return await raceTimeout(evicting, { ms: deadlineMs, fallback: false });
|
|
165
220
|
}
|
|
166
221
|
finally {
|
|
167
|
-
|
|
168
|
-
clearTimeout(timer);
|
|
222
|
+
done = true;
|
|
169
223
|
}
|
|
170
224
|
}
|
|
225
|
+
function definedFields(config) {
|
|
226
|
+
return Object.fromEntries(Object.entries(config).filter(([, value]) => value !== undefined));
|
|
227
|
+
}
|
|
171
228
|
function llmVerbosity() {
|
|
172
229
|
const raw = Number(process.env.QVAC_LLM_VERBOSITY ?? 0);
|
|
173
230
|
return raw === 1 || raw === 2 || raw === 3 ? raw : 0;
|
|
174
231
|
}
|
|
175
232
|
export function progressState(model, progress) {
|
|
176
233
|
const transfer = transferProgress(progress);
|
|
177
|
-
if (!transfer)
|
|
234
|
+
if (!transfer) {
|
|
178
235
|
return { status: 'loading', model };
|
|
236
|
+
}
|
|
179
237
|
const { percentage, downloaded, total } = transfer;
|
|
180
238
|
return percentage < 100
|
|
181
239
|
? { ...progress, status: 'downloading', model, percentage, downloaded, total }
|
|
@@ -188,14 +246,11 @@ function monotonicMs() {
|
|
|
188
246
|
}
|
|
189
247
|
export function loadClockGuard(model, sdk, { now = () => Date.now(), monoMs = monotonicMs, sampleMs = CLOCK_SAMPLE_MS, jumpMs = CLOCK_JUMP_MS } = {}) {
|
|
190
248
|
let timer = null;
|
|
191
|
-
|
|
249
|
+
const { promise: jumped, reject: fail } = Promise.withResolvers();
|
|
192
250
|
let done = false;
|
|
193
251
|
let abandoned = false;
|
|
194
252
|
let lastWall = now();
|
|
195
253
|
let lastMono = monoMs();
|
|
196
|
-
const jumped = new Promise((_resolve, reject) => {
|
|
197
|
-
fail = reject;
|
|
198
|
-
});
|
|
199
254
|
const arm = () => {
|
|
200
255
|
timer = setTimeout(sample, sampleMs);
|
|
201
256
|
timer.unref?.();
|
|
@@ -221,36 +276,40 @@ export function loadClockGuard(model, sdk, { now = () => Date.now(), monoMs = mo
|
|
|
221
276
|
}
|
|
222
277
|
catch (err) {
|
|
223
278
|
if (abandoned && pending.requestId) {
|
|
224
|
-
void sdk.cancel?.({ requestId: pending.requestId }).catch(
|
|
279
|
+
void sdk.cancel?.({ requestId: pending.requestId }).catch(noop);
|
|
225
280
|
}
|
|
226
281
|
throw err;
|
|
227
282
|
}
|
|
228
283
|
finally {
|
|
229
284
|
done = true;
|
|
230
|
-
if (timer !== null)
|
|
285
|
+
if (timer !== null) {
|
|
231
286
|
clearTimeout(timer);
|
|
287
|
+
}
|
|
232
288
|
}
|
|
233
289
|
};
|
|
234
290
|
return { race };
|
|
235
291
|
}
|
|
236
|
-
export function createModelPool(loadSdk, notify, deviceConfig = {}, loads = createLoadTracker()) {
|
|
292
|
+
export function createModelPool(loadSdk, notify, { deviceConfig = {}, loads = createLoadTracker() } = {}) {
|
|
237
293
|
const load = async (model, opts) => {
|
|
238
294
|
const sdk = await loadSdk();
|
|
239
295
|
notify({ status: 'loading', model });
|
|
240
296
|
const modelSrc = await resolveModelSrc(sdk, model);
|
|
241
297
|
const resolved = resolveLoadOptions(opts, deviceConfig);
|
|
242
|
-
const ctxSize = resolved.ctxSize || defaultContextSize(modelSrc);
|
|
298
|
+
const ctxSize = resolved.ctxSize || defaultContextSize(modelSrc, customContextLength(model));
|
|
243
299
|
const projectionModelSrc = resolved.projectionModel
|
|
244
300
|
? await resolveModelSrc(sdk, resolved.projectionModel)
|
|
245
301
|
: undefined;
|
|
246
302
|
const mainGpu = deviceConfig.device !== 'cpu' && deviceConfig.mainGpu
|
|
247
303
|
? await deviceConfig.mainGpu()
|
|
248
304
|
: undefined;
|
|
249
|
-
const kvCache = adrenoSafeKvCache(deviceConfig.device,
|
|
305
|
+
const kvCache = adrenoSafeKvCache(deviceConfig.device, {
|
|
306
|
+
k: resolved.kvCacheK,
|
|
307
|
+
v: resolved.kvCacheV
|
|
308
|
+
});
|
|
250
309
|
const openclCache = deviceConfig.device !== 'cpu' && deviceConfig.openclCacheDir
|
|
251
310
|
? deviceConfig.openclCacheDir()
|
|
252
311
|
: undefined;
|
|
253
|
-
const modelConfig = {
|
|
312
|
+
const modelConfig = definedFields({
|
|
254
313
|
verbosity: llmVerbosity(),
|
|
255
314
|
tools: true,
|
|
256
315
|
stop_sequences: RUNAWAY_STOP_SEQUENCES,
|
|
@@ -258,11 +317,11 @@ export function createModelPool(loadSdk, notify, deviceConfig = {}, loads = crea
|
|
|
258
317
|
'cache-type-k': kvCache.k,
|
|
259
318
|
'cache-type-v': kvCache.v,
|
|
260
319
|
projectionModelSrc,
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
};
|
|
320
|
+
device: deviceConfig.device,
|
|
321
|
+
gpu_layers: deviceConfig.gpuLayers,
|
|
322
|
+
'main-gpu': mainGpu,
|
|
323
|
+
openclCacheDir: openclCache || undefined
|
|
324
|
+
});
|
|
266
325
|
const { projectionModelSrc: projection, ...loggedConfig } = modelConfig;
|
|
267
326
|
hlog('llm load config', { model, projection: Boolean(projection), ...loggedConfig });
|
|
268
327
|
const loadStarted = Date.now();
|
|
@@ -276,13 +335,13 @@ export function createModelPool(loadSdk, notify, deviceConfig = {}, loads = crea
|
|
|
276
335
|
notify({ status: 'ready', model });
|
|
277
336
|
return { model, modelId, ctxSize };
|
|
278
337
|
};
|
|
279
|
-
return new
|
|
338
|
+
return new ModelPuddle({
|
|
280
339
|
open: (key, model = key, opts = {}) => new ModelHandle({ load: () => load(model, opts) }),
|
|
281
340
|
gc: unloadOnGc(loadSdk, notify),
|
|
282
341
|
...NO_IDLE_GC
|
|
283
342
|
});
|
|
284
343
|
}
|
|
285
|
-
export function createEmbedPool(loadSdk, notify, faults, batchSize = DEFAULT_EMBED_BATCH_SIZE, openclCacheDir, loads = createLoadTracker()) {
|
|
344
|
+
export function createEmbedPool(loadSdk, notify, { faults, batchSize = DEFAULT_EMBED_BATCH_SIZE, openclCacheDir, loads = createLoadTracker() }) {
|
|
286
345
|
const load = async (model) => {
|
|
287
346
|
const sdk = await loadSdk();
|
|
288
347
|
notify({ status: 'loading', model });
|
|
@@ -295,20 +354,20 @@ export function createEmbedPool(loadSdk, notify, faults, batchSize = DEFAULT_EMB
|
|
|
295
354
|
batchSize,
|
|
296
355
|
gpuLayers: faults.gpuLayers(),
|
|
297
356
|
device: faults.device,
|
|
298
|
-
|
|
357
|
+
openclCacheDir: openclCache || undefined
|
|
299
358
|
},
|
|
300
359
|
onProgress: (progress) => notify(progressState(model, progress))
|
|
301
360
|
})));
|
|
302
361
|
notify({ status: 'ready', model });
|
|
303
362
|
return { model, modelId };
|
|
304
363
|
};
|
|
305
|
-
return new
|
|
364
|
+
return new ModelPuddle({
|
|
306
365
|
open: (key, model = key) => new ModelHandle({ load: () => load(model) }),
|
|
307
366
|
gc: unloadOnGc(loadSdk, notify),
|
|
308
367
|
...NO_IDLE_GC
|
|
309
368
|
});
|
|
310
369
|
}
|
|
311
|
-
export function createVoicePool(loadSdk, notify, modelType, addon, modelConfig, openclCacheDir, loads = createLoadTracker()) {
|
|
370
|
+
export function createVoicePool(loadSdk, notify, { modelType, addon, modelConfig, openclCacheDir, loads = createLoadTracker() }) {
|
|
312
371
|
const wantsGpu = modelConfig?.['useGPU'] === true;
|
|
313
372
|
let gpuRefused = false;
|
|
314
373
|
const load = async (model) => {
|
|
@@ -335,7 +394,7 @@ export function createVoicePool(loadSdk, notify, modelType, addon, modelConfig,
|
|
|
335
394
|
notify({ status: 'ready', model });
|
|
336
395
|
return { model, modelId };
|
|
337
396
|
};
|
|
338
|
-
return new
|
|
397
|
+
return new ModelPuddle({
|
|
339
398
|
open: (key, model = key) => new ModelHandle({ load: () => load(model) }),
|
|
340
399
|
gc: unloadOnGc(loadSdk, notify),
|
|
341
400
|
...NO_IDLE_GC
|
|
@@ -346,8 +405,9 @@ function unloadOnGc(loadSdk, notify) {
|
|
|
346
405
|
const sdk = await loadSdk();
|
|
347
406
|
for (const handle of handles) {
|
|
348
407
|
const pending = handle.peek();
|
|
349
|
-
if (pending === null)
|
|
408
|
+
if (pending === null) {
|
|
350
409
|
continue;
|
|
410
|
+
}
|
|
351
411
|
let loaded;
|
|
352
412
|
try {
|
|
353
413
|
loaded = await pending;
|
|
@@ -367,3 +427,4 @@ function unloadOnGc(loadSdk, notify) {
|
|
|
367
427
|
}
|
|
368
428
|
};
|
|
369
429
|
}
|
|
430
|
+
function noop() { }
|
|
@@ -1,21 +1,26 @@
|
|
|
1
1
|
import { audioGenModelByName, curatedModelAsset, customModelSrc } from './model-catalog.js';
|
|
2
2
|
export async function resolveModelSrc(sdk, model, addon = 'llm') {
|
|
3
|
-
if (/^(\/|~|\.|file:|https?:|registry:|pear:|hyper:)/.test(model))
|
|
3
|
+
if (/^(\/|~|\.|file:|https?:|registry:|pear:|hyper:)/.test(model)) {
|
|
4
4
|
return model;
|
|
5
|
+
}
|
|
5
6
|
const custom = customModelSrc(model, addon);
|
|
6
|
-
if (custom)
|
|
7
|
+
if (custom) {
|
|
7
8
|
return custom;
|
|
9
|
+
}
|
|
8
10
|
if (addon === 'audiogen') {
|
|
9
11
|
const audio = audioGenModelByName(model);
|
|
10
|
-
if (!audio)
|
|
12
|
+
if (!audio) {
|
|
11
13
|
throw new Error(`audiogen model not found: ${model}`);
|
|
14
|
+
}
|
|
12
15
|
return audio.src;
|
|
13
16
|
}
|
|
14
17
|
const asset = curatedModelAsset(model, addon);
|
|
15
|
-
if (asset)
|
|
18
|
+
if (asset) {
|
|
16
19
|
return asset;
|
|
20
|
+
}
|
|
17
21
|
const [entry] = await sdk.modelRegistrySearch({ filter: model, addon });
|
|
18
|
-
if (!entry)
|
|
22
|
+
if (!entry) {
|
|
19
23
|
throw new Error(`model not found in registry: ${model} (addon=${addon})`);
|
|
24
|
+
}
|
|
20
25
|
return { ...entry, src: `registry://${entry.registrySource}/${entry.registryPath}` };
|
|
21
26
|
}
|
|
@@ -2,11 +2,11 @@ export declare const MUSIC_MODEL_CHOICES: readonly ['turbo', 'turbo-hq', 'sft'];
|
|
|
2
2
|
export type MusicModelChoice = (typeof MUSIC_MODEL_CHOICES)[number];
|
|
3
3
|
export declare const GENERATE_MUSIC_TOOL = "generate_music";
|
|
4
4
|
export declare const DEFAULT_MUSIC_MODEL_CHOICE: MusicModelChoice;
|
|
5
|
-
export
|
|
5
|
+
export interface MusicChoiceEntry {
|
|
6
6
|
readonly choice: MusicModelChoice;
|
|
7
7
|
readonly label: string;
|
|
8
8
|
readonly sizeBytesApprox: number;
|
|
9
9
|
readonly requiresGpu: boolean;
|
|
10
10
|
readonly notes: string;
|
|
11
|
-
}
|
|
11
|
+
}
|
|
12
12
|
export declare const MUSIC_CHOICE_CATALOG: ReadonlyArray<MusicChoiceEntry>;
|
|
@@ -28,7 +28,8 @@ export interface SlmPlan {
|
|
|
28
28
|
model: string;
|
|
29
29
|
}
|
|
30
30
|
export type TranslationPlan = NmtPlan | SlmPlan;
|
|
31
|
-
export
|
|
32
|
-
export declare function translationPlanFor(requestedFrom: string, requestedTo: string): TranslationPlan | {
|
|
31
|
+
export interface TranslationRefusal {
|
|
33
32
|
error: string;
|
|
34
|
-
}
|
|
33
|
+
}
|
|
34
|
+
export declare function sameLanguage(from: string, to: string): boolean;
|
|
35
|
+
export declare function translationPlanFor(requestedFrom: string, requestedTo: string): TranslationPlan | TranslationRefusal;
|
|
@@ -115,8 +115,9 @@ function canonical(code) {
|
|
|
115
115
|
export function translationPlanFor(requestedFrom, requestedTo) {
|
|
116
116
|
const from = requestedFrom.toLowerCase();
|
|
117
117
|
const to = requestedTo.toLowerCase();
|
|
118
|
-
if (sameLanguage(from, to))
|
|
118
|
+
if (sameLanguage(from, to)) {
|
|
119
119
|
return { error: `${from} is already the target language` };
|
|
120
|
+
}
|
|
120
121
|
const slmFrom = slmLanguage(from);
|
|
121
122
|
const slmTo = slmLanguage(to);
|
|
122
123
|
if (slmFrom || slmTo) {
|
|
@@ -146,16 +147,20 @@ export function translationPlanFor(requestedFrom, requestedTo) {
|
|
|
146
147
|
};
|
|
147
148
|
}
|
|
148
149
|
if (to === ENGLISH) {
|
|
149
|
-
if (!TO_ENGLISH.includes(from))
|
|
150
|
+
if (!TO_ENGLISH.includes(from)) {
|
|
150
151
|
return { error: `cannot translate from ${from}` };
|
|
152
|
+
}
|
|
151
153
|
return { route: 'nmt', from, to, model: modelFilter(`${from}${ENGLISH}`) };
|
|
152
154
|
}
|
|
153
|
-
if (!FROM_ENGLISH.includes(to))
|
|
155
|
+
if (!FROM_ENGLISH.includes(to)) {
|
|
154
156
|
return { error: `cannot translate to ${to}` };
|
|
155
|
-
|
|
157
|
+
}
|
|
158
|
+
if (from === ENGLISH) {
|
|
156
159
|
return { route: 'nmt', from, to, model: modelFilter(`${ENGLISH}${to}`) };
|
|
157
|
-
|
|
160
|
+
}
|
|
161
|
+
if (!TO_ENGLISH.includes(from)) {
|
|
158
162
|
return { error: `cannot translate from ${from}` };
|
|
163
|
+
}
|
|
159
164
|
return {
|
|
160
165
|
route: 'nmt',
|
|
161
166
|
from,
|
|
@@ -2,4 +2,9 @@ export declare const OPENCL_CACHE_MAX_AGE_MS: number;
|
|
|
2
2
|
export declare function openclEngineKey(sdkVersion?: string): string;
|
|
3
3
|
export declare function openclCacheRoot(env?: Record<string, string | undefined>, hostPlatform?: string): string | undefined;
|
|
4
4
|
export declare function openclCacheDir(env?: Record<string, string | undefined>, hostPlatform?: string, sdkVersion?: string): string | undefined;
|
|
5
|
-
export
|
|
5
|
+
export interface PrepareOpenclCacheOptions {
|
|
6
|
+
hostPlatform?: string;
|
|
7
|
+
sdkVersion?: string | undefined;
|
|
8
|
+
now?: number;
|
|
9
|
+
}
|
|
10
|
+
export declare function prepareOpenclCache(env?: Record<string, string | undefined>, { hostPlatform, sdkVersion, now }?: PrepareOpenclCacheOptions): void;
|
|
@@ -13,18 +13,20 @@ export function openclCacheRoot(env = process.env, hostPlatform = platform()) {
|
|
|
13
13
|
if (hostPlatform !== 'android')
|
|
14
14
|
return undefined;
|
|
15
15
|
const dir = env[OPENCL_CACHE_DIR_ENV];
|
|
16
|
-
if (!dir?.startsWith('/'))
|
|
16
|
+
if (!dir?.startsWith('/')) {
|
|
17
17
|
return undefined;
|
|
18
|
+
}
|
|
18
19
|
return dir.split('/').includes('..') ? undefined : dir;
|
|
19
20
|
}
|
|
20
21
|
export function openclCacheDir(env = process.env, hostPlatform = platform(), sdkVersion) {
|
|
21
22
|
const root = openclCacheRoot(env, hostPlatform);
|
|
22
23
|
return root === undefined ? undefined : path.join(root, openclEngineKey(sdkVersion));
|
|
23
24
|
}
|
|
24
|
-
export function prepareOpenclCache(env = process.env, hostPlatform = platform(), sdkVersion, now = Date.now()) {
|
|
25
|
+
export function prepareOpenclCache(env = process.env, { hostPlatform = platform(), sdkVersion, now = Date.now() } = {}) {
|
|
25
26
|
const root = openclCacheRoot(env, hostPlatform);
|
|
26
|
-
if (root === undefined)
|
|
27
|
+
if (root === undefined) {
|
|
27
28
|
return;
|
|
29
|
+
}
|
|
28
30
|
const key = openclEngineKey(sdkVersion);
|
|
29
31
|
try {
|
|
30
32
|
fs.mkdirSync(path.join(root, key), { recursive: true });
|
|
@@ -3,19 +3,22 @@ export declare function isSdkGpuDeviceLostError(err: unknown): boolean;
|
|
|
3
3
|
export declare function isSdkModelWedgedError(err: unknown): boolean;
|
|
4
4
|
export declare function isContextOverflowError(err: unknown): boolean;
|
|
5
5
|
export type GpuFaultDecision = 'reload' | 'fallback';
|
|
6
|
-
|
|
6
|
+
interface GpuFaultEscalation {
|
|
7
7
|
recordFault(onGpu: boolean): GpuFaultDecision;
|
|
8
8
|
reset(): void;
|
|
9
|
-
}
|
|
10
|
-
export
|
|
11
|
-
export
|
|
12
|
-
device?: EmbedDevice;
|
|
13
|
-
gpuLayers?: number;
|
|
14
|
-
}): {
|
|
9
|
+
}
|
|
10
|
+
export declare function createGpuFaultEscalation(): GpuFaultEscalation;
|
|
11
|
+
export interface GpuFaultState {
|
|
15
12
|
readonly device: EmbedDevice;
|
|
16
13
|
readonly generation: number;
|
|
17
14
|
gpuLayers(): number;
|
|
18
15
|
key(model: string): string;
|
|
19
16
|
recordFault(bornAt: number): void;
|
|
20
17
|
reset(): void;
|
|
21
|
-
}
|
|
18
|
+
}
|
|
19
|
+
export interface GpuFaultStateOptions {
|
|
20
|
+
device?: EmbedDevice;
|
|
21
|
+
gpuLayers?: number;
|
|
22
|
+
}
|
|
23
|
+
export declare function createGpuFaultState({ device, gpuLayers }?: GpuFaultStateOptions): GpuFaultState;
|
|
24
|
+
export {};
|
|
@@ -1,23 +1,25 @@
|
|
|
1
1
|
export function isSdkGpuDeviceLostError(err) {
|
|
2
2
|
if (typeof err !== 'object' || err === null)
|
|
3
3
|
return false;
|
|
4
|
-
const message = err.message;
|
|
5
|
-
if (typeof message !== 'string')
|
|
4
|
+
const message = 'message' in err ? err.message : undefined;
|
|
5
|
+
if (typeof message !== 'string') {
|
|
6
6
|
return false;
|
|
7
|
+
}
|
|
7
8
|
return /ErrorDeviceLost/i.test(message) || /VK_ERROR_DEVICE_LOST/i.test(message);
|
|
8
9
|
}
|
|
9
10
|
export function isSdkModelWedgedError(err) {
|
|
10
11
|
if (typeof err !== 'object' || err === null)
|
|
11
12
|
return false;
|
|
12
|
-
const message = err.message;
|
|
13
|
-
if (typeof message !== 'string')
|
|
13
|
+
const message = 'message' in err ? err.message : undefined;
|
|
14
|
+
if (typeof message !== 'string') {
|
|
14
15
|
return false;
|
|
16
|
+
}
|
|
15
17
|
return /failed to (decode next token|eval chunk)/i.test(message);
|
|
16
18
|
}
|
|
17
19
|
export function isContextOverflowError(err) {
|
|
18
20
|
if (typeof err !== 'object' || err === null)
|
|
19
21
|
return false;
|
|
20
|
-
return err.name === 'CONTEXT_OVERFLOW';
|
|
22
|
+
return 'name' in err && err.name === 'CONTEXT_OVERFLOW';
|
|
21
23
|
}
|
|
22
24
|
export function createGpuFaultEscalation() {
|
|
23
25
|
let count = 0;
|
|
@@ -56,8 +58,9 @@ export function createGpuFaultState({ device, gpuLayers } = {}) {
|
|
|
56
58
|
return;
|
|
57
59
|
lastFaulted = bornAt;
|
|
58
60
|
generation += 1;
|
|
59
|
-
if (escalation.recordFault(current === 'gpu') === 'fallback')
|
|
61
|
+
if (escalation.recordFault(current === 'gpu') === 'fallback') {
|
|
60
62
|
current = 'cpu';
|
|
63
|
+
}
|
|
61
64
|
},
|
|
62
65
|
reset() {
|
|
63
66
|
escalation.reset();
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface Shard {
|
|
2
|
+
readonly url: string;
|
|
3
|
+
readonly filename: string;
|
|
4
|
+
}
|
|
5
|
+
export interface ShardSet {
|
|
6
|
+
readonly base: string;
|
|
7
|
+
readonly baseUrl: string;
|
|
8
|
+
readonly extension: string;
|
|
9
|
+
readonly shards: ReadonlyArray<Shard>;
|
|
10
|
+
}
|
|
11
|
+
export declare function shardSet(src: string): ShardSet | null;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
const SHARD = /^(.+)-(\d{5})-of-(\d{5})(\.\w+)$/;
|
|
2
|
+
export function shardSet(src) {
|
|
3
|
+
const parts = src.split('/');
|
|
4
|
+
const match = SHARD.exec(parts[parts.length - 1]?.split('?')[0] ?? '');
|
|
5
|
+
if (!match) {
|
|
6
|
+
return null;
|
|
7
|
+
}
|
|
8
|
+
const [, base, , total, extension] = match;
|
|
9
|
+
const baseUrl = parts.slice(0, -1).join('/');
|
|
10
|
+
const shards = Array.from({ length: Number(total) }, (_, i) => {
|
|
11
|
+
const filename = `${base}-${String(i + 1).padStart(5, '0')}-of-${total}${extension}`;
|
|
12
|
+
return { url: `${baseUrl}/${filename}`, filename };
|
|
13
|
+
});
|
|
14
|
+
return { base, baseUrl, extension, shards };
|
|
15
|
+
}
|
|
@@ -5,15 +5,20 @@ export function detectToolDialect(src, name) {
|
|
|
5
5
|
.filter(Boolean)
|
|
6
6
|
.pop() ?? '';
|
|
7
7
|
const tag = `${(name ?? '').toLowerCase()}|${basename}`;
|
|
8
|
-
if (/qwen3[._-]?[56](?![a-z0-9])/.test(tag))
|
|
8
|
+
if (/qwen3[._-]?[56](?![a-z0-9])/.test(tag)) {
|
|
9
9
|
return 'qwen35';
|
|
10
|
-
|
|
10
|
+
}
|
|
11
|
+
if (/gemma[-_]?4(?=[^a-z0-9]|$)/.test(tag)) {
|
|
11
12
|
return 'gemma4';
|
|
12
|
-
|
|
13
|
+
}
|
|
14
|
+
if (/gpt[_-]?oss/.test(tag)) {
|
|
13
15
|
return 'harmony';
|
|
14
|
-
|
|
16
|
+
}
|
|
17
|
+
if (/deepseek[-_. ]?v(?:4|3[._-]?2)(?![0-9])/.test(tag)) {
|
|
15
18
|
return 'dsml';
|
|
16
|
-
|
|
19
|
+
}
|
|
20
|
+
if (/lfm[_-]?\d/.test(tag)) {
|
|
17
21
|
return 'pythonic';
|
|
22
|
+
}
|
|
18
23
|
return undefined;
|
|
19
24
|
}
|
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
import { type BareLookupAll } from './validating-lookup.ts';
|
|
2
2
|
import { type NoRedirectFetchInit, type NoRedirectFetchResponse } from './no-redirect-fetch-impl.ts';
|
|
3
3
|
export type { NoRedirectFetchResponse };
|
|
4
|
-
export
|
|
4
|
+
export interface ConnectionAgent {
|
|
5
5
|
createConnection(opts: {
|
|
6
6
|
readonly host?: string;
|
|
7
7
|
}): unknown;
|
|
8
|
-
}
|
|
8
|
+
}
|
|
9
9
|
export type AgentCtor<A extends ConnectionAgent> = new (opts: object) => A;
|
|
10
10
|
export declare class BlockedHostError extends Error {
|
|
11
11
|
readonly code = "EBLOCKEDHOST";
|
|
12
12
|
constructor(host: string);
|
|
13
13
|
}
|
|
14
14
|
export declare function makeValidatingAgent<A extends ConnectionAgent>(Agent: AgentCtor<A>, lookup: BareLookupAll): A;
|
|
15
|
-
export
|
|
15
|
+
export declare function noRedirectFetch(url: URL, init?: NoRedirectFetchInit): Promise<NoRedirectFetchResponse>;
|
|
@@ -35,7 +35,7 @@ const rawAllLookup = (hostname, opts, cb) => {
|
|
|
35
35
|
const validatingLookup = makeValidatingLookup(rawAllLookup);
|
|
36
36
|
const httpAgent = makeValidatingAgent(HttpAgent, validatingLookup);
|
|
37
37
|
const httpsAgent = makeValidatingAgent(HttpsAgent, validatingLookup);
|
|
38
|
-
export
|
|
38
|
+
export function noRedirectFetch(url, init = {}) {
|
|
39
39
|
const request = (target, opts, onresponse) => {
|
|
40
40
|
const common = { method: readMethod(opts.method), headers: opts.headers };
|
|
41
41
|
const forward = (res) => onresponse(res);
|