@qvac/inference 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +107 -33
- package/README.md +8 -5
- package/dist/api/assess-model-fit.d.ts +9 -4
- package/dist/api/assess-model-fit.js +9 -4
- package/dist/api/bci-transcribe.d.ts +4 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/rpc-server.d.ts +19 -0
- package/dist/api/rpc-server.js +44 -0
- package/dist/api/text-to-speech.d.ts +9 -0
- package/dist/api/text-to-speech.js +9 -0
- package/dist/api/transcribe.d.ts +11 -7
- package/dist/api/transcribe.js +47 -15
- package/dist/api/video-request.d.ts +5 -5
- package/dist/dispatch.js +39 -7
- package/dist/errors/index.d.ts +7 -0
- package/dist/errors/index.js +15 -0
- package/dist/handlers/assess-model-fit.d.ts +18 -6
- package/dist/handlers/assess-model-fit.js +215 -35
- package/dist/handlers/completion-orchestrate.js +17 -0
- package/dist/handlers/get-loaded-model-info.js +4 -1
- package/dist/handlers/get-model-info.js +29 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
- package/dist/handlers/load-model/coreml-sidecars.js +274 -0
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
- package/dist/handlers/load-model/registry-download-utils.js +2 -1
- package/dist/handlers/load-model/registry.js +59 -0
- package/dist/handlers/rpc-server.d.ts +23 -0
- package/dist/handlers/rpc-server.js +141 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/models/registry/models.d.ts +7731 -5672
- package/dist/models/registry/models.js +4652 -2526
- package/dist/models/registry/resource-profiles.js +481 -94
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +80 -8
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
- package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
- package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
- package/dist/plugins/ops/bci-transcribe.js +18 -13
- package/dist/plugins/ops/load-model.js +2 -2
- package/dist/plugins/ops/transcribe.d.ts +4 -2
- package/dist/plugins/ops/transcribe.js +39 -2
- package/dist/plugins/ops/translate-stats.d.ts +2 -1
- package/dist/plugins/ops/translate-stats.js +64 -7
- package/dist/plugins/ops/translate.js +20 -4
- package/dist/profiling/operation-metrics.js +56 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +46 -2
- package/dist/resources/model-fit/assess.js +67 -3
- package/dist/resources/model-fit/estimators/llm.js +2 -2
- package/dist/resources/model-fit/estimators/whisper.js +3 -4
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +102 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +212 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
- package/dist/resources/model-fit/floor.js +2 -2
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
- package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
- package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
- package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
- package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
- package/dist/resources/model-fit/platform.d.ts +9 -0
- package/dist/resources/model-fit/platform.js +28 -0
- package/dist/rpc/device-map.d.ts +13 -0
- package/dist/rpc/device-map.js +22 -0
- package/dist/rpc/discovery.d.ts +12 -0
- package/dist/rpc/discovery.js +225 -0
- package/dist/rpc/ggml-provider.d.ts +4 -0
- package/dist/rpc/ggml-provider.js +25 -0
- package/dist/rpc/index.d.ts +2 -0
- package/dist/rpc/index.js +1 -0
- package/dist/rpc/instance.d.ts +6 -0
- package/dist/rpc/instance.js +21 -0
- package/dist/rpc/inventory.d.ts +5 -0
- package/dist/rpc/inventory.js +144 -0
- package/dist/rpc/manager.d.ts +14 -0
- package/dist/rpc/manager.js +174 -0
- package/dist/rpc/network.d.ts +11 -0
- package/dist/rpc/network.js +85 -0
- package/dist/rpc/provider.d.ts +6 -0
- package/dist/rpc/provider.js +24 -0
- package/dist/runtime/lifecycle.js +31 -13
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +284 -77
- package/dist/schemas/assess-model-fit.js +93 -55
- package/dist/schemas/audio-gen.d.ts +6 -6
- package/dist/schemas/audio-gen.js +2 -2
- package/dist/schemas/batch-completion-stream.d.ts +13 -5
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci-config.js +1 -1
- package/dist/schemas/bci.d.ts +86 -1
- package/dist/schemas/bci.js +21 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +765 -56
- package/dist/schemas/common.js +7 -0
- package/dist/schemas/completion-event.d.ts +6 -6
- package/dist/schemas/completion-stream.d.ts +27 -8
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +236 -30
- package/dist/schemas/config.js +10 -0
- package/dist/schemas/embed.d.ts +2 -2
- package/dist/schemas/error.js +13 -0
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +105 -14
- package/dist/schemas/llamacpp-config.js +113 -3
- package/dist/schemas/load-model.d.ts +3102 -627
- package/dist/schemas/load-model.js +2 -2
- package/dist/schemas/mcp-adapter.d.ts +4 -0
- package/dist/schemas/model-config-schemas.d.ts +205 -5
- package/dist/schemas/model-info.d.ts +56 -0
- package/dist/schemas/model-info.js +5 -1
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/reload-config.d.ts +30 -0
- package/dist/schemas/rpc-server.d.ts +101 -0
- package/dist/schemas/rpc-server.js +87 -0
- package/dist/schemas/sdcpp-config.d.ts +7 -3
- package/dist/schemas/sdcpp-config.js +37 -2
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +529 -1
- package/dist/schemas/text-to-speech.js +122 -12
- package/dist/schemas/tools.d.ts +2 -0
- package/dist/schemas/tools.js +8 -0
- package/dist/schemas/transcription-config.d.ts +5 -0
- package/dist/schemas/transcription-config.js +20 -2
- package/dist/schemas/transcription.d.ts +164 -0
- package/dist/schemas/transcription.js +71 -4
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/surface.d.ts +4 -1
- package/dist/surface.js +3 -1
- package/dist/utils/addon-responses.d.ts +5 -1
- package/dist/utils/archive.js +5 -1
- package/dist/utils/asr-diagnostics.d.ts +12 -0
- package/dist/utils/asr-diagnostics.js +42 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +3 -2
- package/dist/utils/cache/paths.js +3 -2
- package/dist/utils/mcp-adapter.js +7 -1
- package/dist/utils/multi-gpu-mobile.d.ts +1 -1
- package/dist/utils/multi-gpu-mobile.js +8 -6
- package/dist/utils/path-security.d.ts +5 -0
- package/dist/utils/path-security.js +14 -0
- package/dist/utils/tool-helpers.d.ts +4 -0
- package/dist/utils/tool-helpers.js +2 -0
- package/dist/utils/tools/defer.d.ts +93 -0
- package/dist/utils/tools/defer.js +290 -0
- package/dist/utils/tools/index.d.ts +1 -0
- package/dist/utils/tools/index.js +1 -0
- package/dist/utils/transcribe-metadata.d.ts +21 -7
- package/dist/utils/transcribe-metadata.js +24 -8
- package/dist/utils/tts-stats.d.ts +6 -0
- package/dist/utils/tts-stats.js +14 -1
- package/package.json +44 -30
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -171
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { type AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
import type { QVACBlobBinding } from '@qvac/registry-client';
|
|
3
|
+
import type { Logger } from '../../../logging/types.js';
|
|
4
|
+
/**
|
|
5
|
+
* A weightless description of an artifact, served by the registry beside the
|
|
6
|
+
* weights: for a GGUF, the tensor list and the settings with the tokenizer
|
|
7
|
+
* tables stripped and no data section — tens of KB against gigabytes. The
|
|
8
|
+
* fitter reads it exactly as it reads the artifact, so a projection can be made
|
|
9
|
+
* before anything is downloaded.
|
|
10
|
+
*/
|
|
11
|
+
export interface FitStubRef {
|
|
12
|
+
name: string;
|
|
13
|
+
sha256Checksum: string;
|
|
14
|
+
/** Registry coordinates. A constant that carries neither cannot be resolved. */
|
|
15
|
+
registryPath?: string | undefined;
|
|
16
|
+
registrySource?: string | undefined;
|
|
17
|
+
}
|
|
18
|
+
export interface FitStubSetMember extends FitStubRef {
|
|
19
|
+
targetName: string;
|
|
20
|
+
}
|
|
21
|
+
export interface FitStubSet {
|
|
22
|
+
primary: FitStubSetMember;
|
|
23
|
+
others: readonly FitStubSetMember[];
|
|
24
|
+
}
|
|
25
|
+
export type FitStubUnavailableReason =
|
|
26
|
+
/** The ref carries no registry coordinates to look up. */
|
|
27
|
+
'unresolvable-ref'
|
|
28
|
+
/** The registry has no entry at those coordinates. */
|
|
29
|
+
| 'not-in-registry'
|
|
30
|
+
/** The entry predates fit blobs, or its description could not be built. */
|
|
31
|
+
| 'no-fit-blob' | 'download-failed'
|
|
32
|
+
/** The lookup and the fetch together did not finish within the budget. */
|
|
33
|
+
| 'timed-out';
|
|
34
|
+
export type FitStubOutcome =
|
|
35
|
+
/** `path` is the caller's to remove once the fitter has read it. */
|
|
36
|
+
{
|
|
37
|
+
status: 'ready';
|
|
38
|
+
path: string;
|
|
39
|
+
bytes: number;
|
|
40
|
+
} | {
|
|
41
|
+
status: 'unavailable';
|
|
42
|
+
reason: FitStubUnavailableReason;
|
|
43
|
+
message?: string;
|
|
44
|
+
};
|
|
45
|
+
export type FitBlobBinding = QVACBlobBinding;
|
|
46
|
+
export interface FitStubEntry {
|
|
47
|
+
fitBlobBinding?: FitBlobBinding | undefined;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Injection seams. Every field defaults to the real runtime dependency; tests
|
|
51
|
+
* substitute them rather than mocking modules.
|
|
52
|
+
*/
|
|
53
|
+
export interface FitStubOptions {
|
|
54
|
+
signal?: AbortSignal;
|
|
55
|
+
/** Where stubs are staged. Defaults to `fit-stubs` under the QVAC cache root. */
|
|
56
|
+
cacheDir?: string;
|
|
57
|
+
/** Overall budget for the lookup and the fetch. Defaults to `FIT_STUB_BUDGET_MS`. */
|
|
58
|
+
budgetMs?: number;
|
|
59
|
+
logger?: Logger;
|
|
60
|
+
getEntry?: (registryPath: string, registrySource: string) => Promise<FitStubEntry | null>;
|
|
61
|
+
downloadBlob?: (binding: FitBlobBinding, outputFile: string, signal?: AbortSignal) => Promise<unknown>;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Bounds the registry lookup and the blob fetch together. The client's own
|
|
65
|
+
* defaults — 30s per attempt, three attempts, a peer wait between them, and an
|
|
66
|
+
* uncapped wait for the registry view on first use — suit a weights download.
|
|
67
|
+
* Here an unreachable registry is the whole latency of the call, so it is
|
|
68
|
+
* capped. Work still in flight at expiry is abandoned: the download is
|
|
69
|
+
* aborted, and a lookup that cannot be cancelled settles on its own with
|
|
70
|
+
* nothing waiting for it.
|
|
71
|
+
*
|
|
72
|
+
* One budget covers a whole companion set, so it holds the slowest case: every
|
|
73
|
+
* member of a multi-stage voice model staged over a mobile network. It sits
|
|
74
|
+
* above the client's per-attempt timeout rather than on it, so an attempt that
|
|
75
|
+
* is about to answer is not cut at the moment it would have.
|
|
76
|
+
*/
|
|
77
|
+
export declare const FIT_STUB_BUDGET_MS = 40000;
|
|
78
|
+
/** Removes a fetched stub and the directory it was staged in. Never throws. */
|
|
79
|
+
export declare function removeStub(stubPath: string): Promise<void>;
|
|
80
|
+
/**
|
|
81
|
+
* Fetches the fit stub for one artifact into its own staging directory and
|
|
82
|
+
* returns its path. The stub is a per-call payload, not a cache: the caller
|
|
83
|
+
* removes it once the fitter has read it, so nothing accumulates under the
|
|
84
|
+
* QVAC root and no cleanup path has to know about it.
|
|
85
|
+
*
|
|
86
|
+
* Never throws: a missing entry, an entry without a description, a failed or
|
|
87
|
+
* corrupt download and an expired budget are all `unavailable`, because a
|
|
88
|
+
* pre-download assessment has to survive an offline caller and an older
|
|
89
|
+
* registry record.
|
|
90
|
+
*
|
|
91
|
+
* Only the artifact named by `ref` is fetched. A split model's shards each
|
|
92
|
+
* carry their own binding, and the fitter needs the whole set laid out under
|
|
93
|
+
* llama's shard naming to read any of it — assembling that is not done here.
|
|
94
|
+
*/
|
|
95
|
+
export declare function fetchFitStub(ref: FitStubRef, options?: FitStubOptions): Promise<FitStubOutcome>;
|
|
96
|
+
/**
|
|
97
|
+
* Stages a companion set under its target names in one directory, the layout a
|
|
98
|
+
* downloaded set has, for engines that find their companions beside the
|
|
99
|
+
* checkpoint. The primary decides the outcome; a member the registry holds no
|
|
100
|
+
* description for is skipped.
|
|
101
|
+
*/
|
|
102
|
+
export declare function fetchFitStubSet(set: FitStubSet, options?: FitStubOptions): Promise<FitStubOutcome>;
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
import fs, { promises as fsPromises } from 'bare-fs';
|
|
2
|
+
import path from 'bare-path';
|
|
3
|
+
import { AbortController } from 'bare-abort-controller';
|
|
4
|
+
import { getCacheDir } from '../../../utils/cache/paths.js';
|
|
5
|
+
import { calculateFileChecksum } from '../../../utils/checksum.js';
|
|
6
|
+
import { getEngineLogger } from '../../../logging/index.js';
|
|
7
|
+
/**
|
|
8
|
+
* Bounds the registry lookup and the blob fetch together. The client's own
|
|
9
|
+
* defaults — 30s per attempt, three attempts, a peer wait between them, and an
|
|
10
|
+
* uncapped wait for the registry view on first use — suit a weights download.
|
|
11
|
+
* Here an unreachable registry is the whole latency of the call, so it is
|
|
12
|
+
* capped. Work still in flight at expiry is abandoned: the download is
|
|
13
|
+
* aborted, and a lookup that cannot be cancelled settles on its own with
|
|
14
|
+
* nothing waiting for it.
|
|
15
|
+
*
|
|
16
|
+
* One budget covers a whole companion set, so it holds the slowest case: every
|
|
17
|
+
* member of a multi-stage voice model staged over a mobile network. It sits
|
|
18
|
+
* above the client's per-attempt timeout rather than on it, so an attempt that
|
|
19
|
+
* is about to answer is not cut at the moment it would have.
|
|
20
|
+
*/
|
|
21
|
+
export const FIT_STUB_BUDGET_MS = 40_000;
|
|
22
|
+
async function defaultGetEntry(registryPath, registrySource) {
|
|
23
|
+
const { getRegistryClient } = await import('../../../runtime/registry-client.js');
|
|
24
|
+
const client = await getRegistryClient();
|
|
25
|
+
return client.getModel(registryPath, registrySource);
|
|
26
|
+
}
|
|
27
|
+
async function defaultDownloadBlob(binding, outputFile, signal) {
|
|
28
|
+
const { getRegistryClient } = await import('../../../runtime/registry-client.js');
|
|
29
|
+
const client = await getRegistryClient();
|
|
30
|
+
return client.downloadBlob(binding, {
|
|
31
|
+
outputFile,
|
|
32
|
+
maxRetries: 1,
|
|
33
|
+
...(signal !== undefined && { signal })
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
/** Removes a fetched stub and the directory it was staged in. Never throws. */
|
|
37
|
+
export async function removeStub(stubPath) {
|
|
38
|
+
await fsPromises.rm(path.dirname(stubPath), { recursive: true, force: true }).catch(() => { });
|
|
39
|
+
}
|
|
40
|
+
function unavailable(reason, message) {
|
|
41
|
+
return message === undefined
|
|
42
|
+
? { status: 'unavailable', reason }
|
|
43
|
+
: { status: 'unavailable', reason, message };
|
|
44
|
+
}
|
|
45
|
+
function describe(error) {
|
|
46
|
+
return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Fetches the fit stub for one artifact into its own staging directory and
|
|
50
|
+
* returns its path. The stub is a per-call payload, not a cache: the caller
|
|
51
|
+
* removes it once the fitter has read it, so nothing accumulates under the
|
|
52
|
+
* QVAC root and no cleanup path has to know about it.
|
|
53
|
+
*
|
|
54
|
+
* Never throws: a missing entry, an entry without a description, a failed or
|
|
55
|
+
* corrupt download and an expired budget are all `unavailable`, because a
|
|
56
|
+
* pre-download assessment has to survive an offline caller and an older
|
|
57
|
+
* registry record.
|
|
58
|
+
*
|
|
59
|
+
* Only the artifact named by `ref` is fetched. A split model's shards each
|
|
60
|
+
* carry their own binding, and the fitter needs the whole set laid out under
|
|
61
|
+
* llama's shard naming to read any of it — assembling that is not done here.
|
|
62
|
+
*/
|
|
63
|
+
export async function fetchFitStub(ref, options = {}) {
|
|
64
|
+
return withinBudget(ref.name, options, (signal) => fetchWithin(ref, options, signal));
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Stages a companion set under its target names in one directory, the layout a
|
|
68
|
+
* downloaded set has, for engines that find their companions beside the
|
|
69
|
+
* checkpoint. The primary decides the outcome; a member the registry holds no
|
|
70
|
+
* description for is skipped.
|
|
71
|
+
*/
|
|
72
|
+
export async function fetchFitStubSet(set, options = {}) {
|
|
73
|
+
return withinBudget(set.primary.name, options, (signal) => fetchSetWithin(set, options, signal));
|
|
74
|
+
}
|
|
75
|
+
function withinBudget(name, options, work) {
|
|
76
|
+
const budgetMs = options.budgetMs ?? FIT_STUB_BUDGET_MS;
|
|
77
|
+
const controller = new AbortController();
|
|
78
|
+
const onAbort = () => controller.abort(new Error('fit stub fetch aborted by the caller'));
|
|
79
|
+
options.signal?.addEventListener('abort', onAbort, { once: true });
|
|
80
|
+
let timer;
|
|
81
|
+
const expiry = new Promise((resolve) => {
|
|
82
|
+
timer = setTimeout(() => {
|
|
83
|
+
controller.abort(new Error('fit stub budget expired'));
|
|
84
|
+
resolve(unavailable('timed-out', `no fit stub for ${name} from the registry within ${budgetMs}ms`));
|
|
85
|
+
}, budgetMs);
|
|
86
|
+
});
|
|
87
|
+
return Promise.race([work(controller.signal), expiry]).finally(() => {
|
|
88
|
+
clearTimeout(timer);
|
|
89
|
+
options.signal?.removeEventListener('abort', onAbort);
|
|
90
|
+
});
|
|
91
|
+
}
|
|
92
|
+
// One directory per fetch, so two assessments of the same model never share a
|
|
93
|
+
// file that one of them is about to remove.
|
|
94
|
+
async function makeStagingDir(options) {
|
|
95
|
+
const root = options.cacheDir ?? getCacheDir('fit-stubs');
|
|
96
|
+
await fsPromises.mkdir(root, { recursive: true });
|
|
97
|
+
return fsPromises.mkdtemp(path.join(root, 'stub-'));
|
|
98
|
+
}
|
|
99
|
+
async function bindingFor(ref, options) {
|
|
100
|
+
const { registryPath, registrySource } = ref;
|
|
101
|
+
if (!registryPath || !registrySource)
|
|
102
|
+
return 'unresolvable-ref';
|
|
103
|
+
const getEntry = options.getEntry ?? defaultGetEntry;
|
|
104
|
+
const entry = await getEntry(registryPath, registrySource);
|
|
105
|
+
if (entry === null || entry === undefined)
|
|
106
|
+
return 'not-in-registry';
|
|
107
|
+
const binding = entry.fitBlobBinding;
|
|
108
|
+
if (binding === undefined || binding === null)
|
|
109
|
+
return 'no-fit-blob';
|
|
110
|
+
return binding;
|
|
111
|
+
}
|
|
112
|
+
async function stageOne(ref, binding, dest, options, signal) {
|
|
113
|
+
const downloadBlob = options.downloadBlob ?? defaultDownloadBlob;
|
|
114
|
+
let verified = false;
|
|
115
|
+
try {
|
|
116
|
+
await downloadBlob(binding, dest, signal);
|
|
117
|
+
const bytes = fs.statSync(dest).size;
|
|
118
|
+
if (bytes !== binding.byteLength) {
|
|
119
|
+
return unavailable('download-failed', `fit stub for ${ref.name} is ${bytes} bytes, the record says ${binding.byteLength}`);
|
|
120
|
+
}
|
|
121
|
+
// The record binds the description by digest; a stub that reads back
|
|
122
|
+
// differently is not the description the fitter should answer for.
|
|
123
|
+
const digest = await calculateFileChecksum(dest);
|
|
124
|
+
if (digest.toLowerCase() !== binding.sha256.toLowerCase()) {
|
|
125
|
+
return unavailable('download-failed', `fit stub for ${ref.name} hashes to ${digest}, the record says ${binding.sha256}`);
|
|
126
|
+
}
|
|
127
|
+
// Past the budget nobody is waiting for this result, so the stub must
|
|
128
|
+
// not be marked as kept.
|
|
129
|
+
if (signal.aborted)
|
|
130
|
+
return unavailable('timed-out');
|
|
131
|
+
verified = true;
|
|
132
|
+
return { status: 'staged', bytes };
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
return unavailable('download-failed', describe(error));
|
|
136
|
+
}
|
|
137
|
+
finally {
|
|
138
|
+
// A short, corrupt, half-written or late stub must not outlive the call.
|
|
139
|
+
if (!verified)
|
|
140
|
+
await fsPromises.rm(dest, { force: true }).catch(() => { });
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
async function fetchWithin(ref, options, signal) {
|
|
144
|
+
const logger = options.logger ?? getEngineLogger();
|
|
145
|
+
try {
|
|
146
|
+
const binding = await bindingFor(ref, options);
|
|
147
|
+
if (typeof binding === 'string')
|
|
148
|
+
return unavailable(binding);
|
|
149
|
+
if (signal.aborted)
|
|
150
|
+
return unavailable('timed-out');
|
|
151
|
+
const dir = await makeStagingDir(options);
|
|
152
|
+
const dest = path.join(dir, `${binding.sha256}.gguf`);
|
|
153
|
+
const staged = await stageOne(ref, binding, dest, options, signal);
|
|
154
|
+
if (staged.status !== 'staged') {
|
|
155
|
+
await removeStub(dest);
|
|
156
|
+
return staged;
|
|
157
|
+
}
|
|
158
|
+
return { status: 'ready', path: dest, bytes: staged.bytes };
|
|
159
|
+
}
|
|
160
|
+
catch (error) {
|
|
161
|
+
logger.debug(`fit stub for ${ref.name} unavailable: ${describe(error)}`);
|
|
162
|
+
return unavailable('download-failed', describe(error));
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
async function fetchSetWithin(set, options, signal) {
|
|
166
|
+
const logger = options.logger ?? getEngineLogger();
|
|
167
|
+
let dir;
|
|
168
|
+
let ready = false;
|
|
169
|
+
try {
|
|
170
|
+
const binding = await bindingFor(set.primary, options);
|
|
171
|
+
if (typeof binding === 'string')
|
|
172
|
+
return unavailable(binding);
|
|
173
|
+
if (signal.aborted)
|
|
174
|
+
return unavailable('timed-out');
|
|
175
|
+
dir = await makeStagingDir(options);
|
|
176
|
+
const dest = path.join(dir, set.primary.targetName);
|
|
177
|
+
const staged = await stageOne(set.primary, binding, dest, options, signal);
|
|
178
|
+
if (staged.status !== 'staged')
|
|
179
|
+
return staged;
|
|
180
|
+
for (const member of set.others) {
|
|
181
|
+
if (signal.aborted)
|
|
182
|
+
break;
|
|
183
|
+
const memberBinding = await bindingFor(member, options);
|
|
184
|
+
if (typeof memberBinding === 'string') {
|
|
185
|
+
logger.debug(`companion ${member.targetName} has no fit stub: ${memberBinding}`);
|
|
186
|
+
continue;
|
|
187
|
+
}
|
|
188
|
+
const memberDest = path.join(dir, member.targetName);
|
|
189
|
+
const memberStaged = await stageOne(member, memberBinding, memberDest, options, signal);
|
|
190
|
+
if (memberStaged.status !== 'staged') {
|
|
191
|
+
logger.debug(`companion ${member.targetName} was not staged`);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
// The budget races this work and has already answered the caller once it
|
|
195
|
+
// expires, so a set finished after that point is staged for nobody.
|
|
196
|
+
// The budget answers the caller on its own, so a set finished after it
|
|
197
|
+
// expired is staged for nobody.
|
|
198
|
+
if (signal.aborted)
|
|
199
|
+
return unavailable('timed-out');
|
|
200
|
+
ready = true;
|
|
201
|
+
return { status: 'ready', path: dest, bytes: staged.bytes };
|
|
202
|
+
}
|
|
203
|
+
catch (error) {
|
|
204
|
+
logger.debug(`fit stub set for ${set.primary.name} unavailable: ${describe(error)}`);
|
|
205
|
+
return unavailable('download-failed', describe(error));
|
|
206
|
+
}
|
|
207
|
+
finally {
|
|
208
|
+
if (!ready && dir !== undefined) {
|
|
209
|
+
await fsPromises.rm(dir, { recursive: true, force: true }).catch(() => { });
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { type LoadDescription } from './resolve-load-from-stubs.js';
|
|
2
|
+
import type { FitStubOptions, FitStubUnavailableReason } from './fetch-fit-stub.js';
|
|
3
|
+
import { type AdvisoryFitOptions, type AdvisoryFitOutcome } from '../native-probe/advisory-fit.js';
|
|
4
|
+
export type LoadFitOutcome = {
|
|
5
|
+
status: 'projected';
|
|
6
|
+
fit: AdvisoryFitOutcome;
|
|
7
|
+
} | {
|
|
8
|
+
status: 'no-stub';
|
|
9
|
+
reason: FitStubUnavailableReason;
|
|
10
|
+
message?: string;
|
|
11
|
+
} | {
|
|
12
|
+
status: 'unsupported-load';
|
|
13
|
+
detail: string;
|
|
14
|
+
};
|
|
15
|
+
export interface LoadFitOptions {
|
|
16
|
+
stub?: FitStubOptions;
|
|
17
|
+
fit?: AdvisoryFitOptions;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Projects the load a caller describes, before its weights exist locally.
|
|
21
|
+
*
|
|
22
|
+
* The same probe `loadModel` runs, over the same config and artifacts that
|
|
23
|
+
* load's own plugin resolves, pointed at files that are tens of KB instead of
|
|
24
|
+
* gigabytes. It inherits the probe's fail-open contract: every failure is an
|
|
25
|
+
* `unknown` verdict or a non-projected outcome, never a throw.
|
|
26
|
+
*/
|
|
27
|
+
export declare function projectFitFromLoad(load: LoadDescription, modelId: string, options?: LoadFitOptions): Promise<LoadFitOutcome>;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { resolveLoadFromStubs } from './resolve-load-from-stubs.js';
|
|
2
|
+
import { runAdvisoryFitCheck } from '../native-probe/advisory-fit.js';
|
|
3
|
+
/**
|
|
4
|
+
* Projects the load a caller describes, before its weights exist locally.
|
|
5
|
+
*
|
|
6
|
+
* The same probe `loadModel` runs, over the same config and artifacts that
|
|
7
|
+
* load's own plugin resolves, pointed at files that are tens of KB instead of
|
|
8
|
+
* gigabytes. It inherits the probe's fail-open contract: every failure is an
|
|
9
|
+
* `unknown` verdict or a non-projected outcome, never a throw.
|
|
10
|
+
*/
|
|
11
|
+
export async function projectFitFromLoad(load, modelId, options = {}) {
|
|
12
|
+
const resolved = await resolveLoadFromStubs(load, options.stub);
|
|
13
|
+
if (resolved.status !== 'resolved')
|
|
14
|
+
return resolved;
|
|
15
|
+
try {
|
|
16
|
+
const fit = await runAdvisoryFitCheck({
|
|
17
|
+
modelId,
|
|
18
|
+
modelType: load.modelType,
|
|
19
|
+
modelPath: resolved.modelPath,
|
|
20
|
+
modelConfig: resolved.modelConfig,
|
|
21
|
+
artifacts: resolved.artifacts,
|
|
22
|
+
// Nothing here assembles a shard set, so no load may claim the fitter
|
|
23
|
+
// is looking at a split model.
|
|
24
|
+
isShardedModel: false
|
|
25
|
+
}, options.fit);
|
|
26
|
+
return { status: 'projected', fit };
|
|
27
|
+
}
|
|
28
|
+
finally {
|
|
29
|
+
await resolved.release();
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { ModelSrcInput } from '../../../schemas/index.js';
|
|
2
|
+
import type { CanonicalModelType } from '../../../schemas/index.js';
|
|
3
|
+
import { type FitStubOptions, type FitStubUnavailableReason } from './fetch-fit-stub.js';
|
|
4
|
+
export interface LoadDescription {
|
|
5
|
+
/** Absent for the loads `loadModel` also takes without one. */
|
|
6
|
+
modelSrc?: ModelSrcInput | undefined;
|
|
7
|
+
modelType: CanonicalModelType;
|
|
8
|
+
modelConfig?: Record<string, unknown> | undefined;
|
|
9
|
+
}
|
|
10
|
+
export type ResolvedLoad = {
|
|
11
|
+
status: 'resolved';
|
|
12
|
+
modelPath: string;
|
|
13
|
+
modelConfig: unknown;
|
|
14
|
+
artifacts: Record<string, string>;
|
|
15
|
+
/** Removes every stub staged for this load. Never throws. */
|
|
16
|
+
release: () => Promise<void>;
|
|
17
|
+
};
|
|
18
|
+
export type LoadResolution = ResolvedLoad | {
|
|
19
|
+
status: 'no-stub';
|
|
20
|
+
reason: FitStubUnavailableReason;
|
|
21
|
+
message?: string;
|
|
22
|
+
} | {
|
|
23
|
+
status: 'unsupported-load';
|
|
24
|
+
detail: string;
|
|
25
|
+
};
|
|
26
|
+
/**
|
|
27
|
+
* Resolves a load the way `loadModel` would, against the registry's weightless
|
|
28
|
+
* descriptions instead of the weights.
|
|
29
|
+
*
|
|
30
|
+
* The plugin's own `resolveConfig` does the work: it strips the source fields
|
|
31
|
+
* from the config and returns the artifact paths keyed the way its engine
|
|
32
|
+
* expects. Only what a source resolves to differs here, so the config and the
|
|
33
|
+
* keys are the real load's.
|
|
34
|
+
*
|
|
35
|
+
* A source already on disk is read directly, since it describes the load more
|
|
36
|
+
* exactly than a stub does.
|
|
37
|
+
*/
|
|
38
|
+
export declare function resolveLoadFromStubs(load: LoadDescription, options?: FitStubOptions): Promise<LoadResolution>;
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import fs from 'bare-fs';
|
|
2
|
+
import { getPlugin } from '../../../plugins/registry.js';
|
|
3
|
+
import { getRuntimeContext } from '../../../runtime/state.js';
|
|
4
|
+
import { formatZodError } from '../../../utils/zod-error.js';
|
|
5
|
+
import { getModelByPath } from '../../../models/registry/index.js';
|
|
6
|
+
import { fetchFitStub, fetchFitStubSet, removeStub } from './fetch-fit-stub.js';
|
|
7
|
+
function srcString(modelSrc) {
|
|
8
|
+
return typeof modelSrc === 'string' ? modelSrc : modelSrc.src;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* The stub ref a source carries, or `undefined` where it names no registry
|
|
12
|
+
* artifact. A catalog constant carries the checksum and coordinates; a bare
|
|
13
|
+
* string names a location and nothing the registry can be asked about.
|
|
14
|
+
*/
|
|
15
|
+
function stubRefFor(modelSrc) {
|
|
16
|
+
if (typeof modelSrc === 'string')
|
|
17
|
+
return undefined;
|
|
18
|
+
const { sha256Checksum, registryPath, registrySource } = modelSrc;
|
|
19
|
+
if (sha256Checksum === undefined || registryPath === undefined)
|
|
20
|
+
return undefined;
|
|
21
|
+
return {
|
|
22
|
+
name: modelSrc.name ?? modelSrc.modelId ?? registryPath,
|
|
23
|
+
sha256Checksum,
|
|
24
|
+
registryPath,
|
|
25
|
+
...(registrySource !== undefined && { registrySource })
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
function stubSetFor(registryPath) {
|
|
29
|
+
const companionSet = getModelByPath(registryPath)?.companionSet;
|
|
30
|
+
if (companionSet === undefined)
|
|
31
|
+
return undefined;
|
|
32
|
+
let primary;
|
|
33
|
+
const others = [];
|
|
34
|
+
for (const file of companionSet.files) {
|
|
35
|
+
const member = {
|
|
36
|
+
name: file.targetName,
|
|
37
|
+
sha256Checksum: file.sha256Checksum,
|
|
38
|
+
registryPath: file.registryPath,
|
|
39
|
+
registrySource: file.registrySource,
|
|
40
|
+
targetName: file.targetName
|
|
41
|
+
};
|
|
42
|
+
if (file.key === companionSet.primaryKey)
|
|
43
|
+
primary = member;
|
|
44
|
+
else
|
|
45
|
+
others.push(member);
|
|
46
|
+
}
|
|
47
|
+
return primary === undefined ? undefined : { primary, others };
|
|
48
|
+
}
|
|
49
|
+
class NoStubError extends Error {
|
|
50
|
+
reason;
|
|
51
|
+
constructor(reason, message) {
|
|
52
|
+
super(message ?? reason);
|
|
53
|
+
this.reason = reason;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Resolves a load the way `loadModel` would, against the registry's weightless
|
|
58
|
+
* descriptions instead of the weights.
|
|
59
|
+
*
|
|
60
|
+
* The plugin's own `resolveConfig` does the work: it strips the source fields
|
|
61
|
+
* from the config and returns the artifact paths keyed the way its engine
|
|
62
|
+
* expects. Only what a source resolves to differs here, so the config and the
|
|
63
|
+
* keys are the real load's.
|
|
64
|
+
*
|
|
65
|
+
* A source already on disk is read directly, since it describes the load more
|
|
66
|
+
* exactly than a stub does.
|
|
67
|
+
*/
|
|
68
|
+
export async function resolveLoadFromStubs(load, options = {}) {
|
|
69
|
+
const plugin = getPlugin(load.modelType);
|
|
70
|
+
if (!plugin) {
|
|
71
|
+
return { status: 'unsupported-load', detail: `no plugin registered for ${load.modelType}` };
|
|
72
|
+
}
|
|
73
|
+
const parsed = plugin.loadConfigSchema.safeParse(load.modelConfig ?? {});
|
|
74
|
+
if (!parsed.success) {
|
|
75
|
+
return { status: 'unsupported-load', detail: formatZodError(parsed.error) };
|
|
76
|
+
}
|
|
77
|
+
const modelConfig = parsed.data;
|
|
78
|
+
const staged = [];
|
|
79
|
+
const release = async () => {
|
|
80
|
+
for (const path of staged)
|
|
81
|
+
await removeStub(path);
|
|
82
|
+
};
|
|
83
|
+
const resolveModelPath = async (src) => {
|
|
84
|
+
const location = srcString(src);
|
|
85
|
+
if (location !== '' && fs.existsSync(location))
|
|
86
|
+
return location;
|
|
87
|
+
const ref = stubRefFor(src);
|
|
88
|
+
if (ref === undefined)
|
|
89
|
+
throw new NoStubError('unresolvable-ref', location);
|
|
90
|
+
const set = stubSetFor(ref.registryPath);
|
|
91
|
+
const stub = set === undefined ? await fetchFitStub(ref, options) : await fetchFitStubSet(set, options);
|
|
92
|
+
if (stub.status !== 'ready')
|
|
93
|
+
throw new NoStubError(stub.reason, stub.message);
|
|
94
|
+
staged.push(stub.path);
|
|
95
|
+
return stub.path;
|
|
96
|
+
};
|
|
97
|
+
try {
|
|
98
|
+
const modelPath = load.modelSrc === undefined ? '' : await resolveModelPath(load.modelSrc);
|
|
99
|
+
if (plugin.resolveConfig === undefined) {
|
|
100
|
+
return {
|
|
101
|
+
status: 'resolved',
|
|
102
|
+
modelPath,
|
|
103
|
+
modelConfig,
|
|
104
|
+
artifacts: {},
|
|
105
|
+
release
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
const context = {
|
|
109
|
+
resolveModelPath,
|
|
110
|
+
modelSrc: load.modelSrc === undefined ? '' : srcString(load.modelSrc),
|
|
111
|
+
modelType: load.modelType,
|
|
112
|
+
...(getRuntimeContext().platform !== undefined && {
|
|
113
|
+
platform: getRuntimeContext().platform
|
|
114
|
+
})
|
|
115
|
+
};
|
|
116
|
+
const resolved = await plugin.resolveConfig(modelConfig, context);
|
|
117
|
+
return {
|
|
118
|
+
status: 'resolved',
|
|
119
|
+
modelPath,
|
|
120
|
+
modelConfig: resolved.config,
|
|
121
|
+
artifacts: (resolved.artifacts ?? {}),
|
|
122
|
+
release
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
catch (error) {
|
|
126
|
+
await release();
|
|
127
|
+
if (error instanceof NoStubError) {
|
|
128
|
+
return error.message === error.reason
|
|
129
|
+
? { status: 'no-stub', reason: error.reason }
|
|
130
|
+
: { status: 'no-stub', reason: error.reason, message: error.message };
|
|
131
|
+
}
|
|
132
|
+
// A config the plugin refuses describes a load that would not start either.
|
|
133
|
+
return {
|
|
134
|
+
status: 'unsupported-load',
|
|
135
|
+
detail: error instanceof Error ? `${error.name}: ${error.message}` : String(error)
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
}
|
|
@@ -47,7 +47,7 @@ export function computeFloor(input) {
|
|
|
47
47
|
reasons.push('no GGUF metadata for this model in the catalog, so the floor counts weights only');
|
|
48
48
|
return { bytes: artifactBytes, reasons, assumptions };
|
|
49
49
|
}
|
|
50
|
-
let contextTokens = workload.contextTokens;
|
|
50
|
+
let contextTokens = workload.contextTokens ?? facts.contextLength;
|
|
51
51
|
if (contextTokens > facts.contextLength) {
|
|
52
52
|
assumptions.push(`requested ${workload.contextTokens} tokens exceeds the trained context of ${facts.contextLength}; clamped to the trained context`);
|
|
53
53
|
contextTokens = facts.contextLength;
|
|
@@ -59,7 +59,7 @@ export function computeFloor(input) {
|
|
|
59
59
|
const kv = kvCacheBytesForWidth(facts, contextTokens, element).lower;
|
|
60
60
|
assumptions.push(element < 2
|
|
61
61
|
? 'the KV cache is floored at the q8_0 element width, the narrowest default the engine picks on any backend; a CPU load allocates an f16 cache above it'
|
|
62
|
-
: `${facts.architecture} loads with flash attention off, so the KV cache is floored at f16 on every backend`, 'default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not
|
|
62
|
+
: `${facts.architecture} loads with flash attention off, so the KV cache is floored at f16 on every backend`, 'default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not read here and would change these numbers');
|
|
63
63
|
reasons.push(`KV cache floored for ${contextTokens} tokens at the narrowest default element width`);
|
|
64
64
|
return { bytes: Math.ceil(artifactBytes + kv), reasons, assumptions };
|
|
65
65
|
}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
2
|
import type { Logger } from '../../../logging/types.js';
|
|
3
3
|
import type { CanonicalModelType, NativeProbeFit, NativeProbeVerdict } from '../../../schemas/index.js';
|
|
4
|
-
import
|
|
4
|
+
import { runFit as runFitDefault } from './run-fit.js';
|
|
5
5
|
/**
|
|
6
6
|
* `fit` and `does-not-fit` are projections of the load the SDK is about to run,
|
|
7
|
-
* not admission decisions.
|
|
8
|
-
*
|
|
9
|
-
*
|
|
7
|
+
* not admission decisions. The real loader neither consumes nor verifies the
|
|
8
|
+
* fitted plan, so neither verdict is denial-grade and no verdict changes the
|
|
9
|
+
* load.
|
|
10
10
|
*
|
|
11
11
|
* The outcome is the wire shape: `loadModel` hands it to the model registry and
|
|
12
12
|
* `getLoadedModelInfo` returns it, so no caller has to read a verdict out of a
|
|
@@ -31,12 +31,15 @@ export interface AdvisoryFitOptions {
|
|
|
31
31
|
enabled?: boolean;
|
|
32
32
|
mobile?: boolean;
|
|
33
33
|
timeoutMs?: number;
|
|
34
|
-
runFit?: typeof
|
|
34
|
+
runFit?: typeof runFitDefault;
|
|
35
35
|
logger?: Logger;
|
|
36
|
+
availableSystemBytes?: () => Promise<number | undefined>;
|
|
37
|
+
countsDeviceRows?: () => Promise<boolean>;
|
|
36
38
|
residentModelBytes?: () => Promise<number>;
|
|
37
39
|
}
|
|
38
40
|
/**
|
|
39
|
-
* Runs the advisory
|
|
41
|
+
* Runs the advisory fit check for a load that is about to start, against the
|
|
42
|
+
* fitter belonging to the engine that will run it.
|
|
40
43
|
*
|
|
41
44
|
* Fail-open by construction: an unsupported shape, a crashed or wedged child, a
|
|
42
45
|
* malformed response, and an unexpected internal error all resolve to `unknown`
|