@qvac/inference 0.19.1 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +43 -11
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +8 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +44 -0
- package/dist/schemas/assess-model-fit.js +40 -0
- package/dist/schemas/audio-gen.d.ts +479 -4
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +7 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/common.d.ts +749 -15
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +11 -1
- package/dist/schemas/completion-event.js +3 -0
- package/dist/schemas/completion-stream.d.ts +20 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +48 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +4 -8
- package/dist/schemas/llamacpp-config.js +4 -4
- package/dist/schemas/load-model.d.ts +2503 -139
- package/dist/schemas/model-config-schemas.d.ts +228 -5
- package/dist/schemas/plugin.d.ts +10 -0
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +29 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +28 -25
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import { ModelType } from '../../../schemas/index.js';
|
|
2
|
+
import { transformLlmConfig } from '../../../plugins/builtin/llamacpp-completion/transform.js';
|
|
3
|
+
import { transformEmbedConfig } from '../../../plugins/builtin/llamacpp-embedding/transform.js';
|
|
4
|
+
/**
|
|
5
|
+
* Keys `@qvac/model-fit` reads as load evidence, in the spelling the SDK's own
|
|
6
|
+
* completion/embedding transforms emit. The package canonicalizes `_` to `-`,
|
|
7
|
+
* so `ctx_size` and `ctx-size` reach the same native setting.
|
|
8
|
+
*
|
|
9
|
+
* This mirrors `SUPPORTED_LOAD_KEYS` in the package's `LlamaLoadConfig.cpp`
|
|
10
|
+
* intersected with what the two transforms can produce. It is a duplicated
|
|
11
|
+
* policy for the duration of the experiment: a key added to a load config
|
|
12
|
+
* without being classified here must not silently change the question the
|
|
13
|
+
* fitter answers, so `partitionParams` refuses the request instead.
|
|
14
|
+
*/
|
|
15
|
+
const FIT_LOAD_KEYS = {
|
|
16
|
+
completion: [
|
|
17
|
+
'device',
|
|
18
|
+
'ctx_size',
|
|
19
|
+
'gpu_layers',
|
|
20
|
+
'load_mode',
|
|
21
|
+
'parallel',
|
|
22
|
+
'cache-type-k',
|
|
23
|
+
'cache-type-v',
|
|
24
|
+
// Alters KV/compute memory; `model-fit` reads it as evidence.
|
|
25
|
+
'flash-attn',
|
|
26
|
+
'main-gpu',
|
|
27
|
+
'split-mode',
|
|
28
|
+
'tensor-split'
|
|
29
|
+
],
|
|
30
|
+
embedding: [
|
|
31
|
+
'device',
|
|
32
|
+
'gpu_layers',
|
|
33
|
+
'batch_size',
|
|
34
|
+
'flash_attn',
|
|
35
|
+
'main-gpu',
|
|
36
|
+
'split-mode',
|
|
37
|
+
'tensor-split'
|
|
38
|
+
]
|
|
39
|
+
};
|
|
40
|
+
/**
|
|
41
|
+
* Keys that reach the addon but cannot move a load's memory footprint —
|
|
42
|
+
* sampling, generation, logging, and JS-side presentation settings. They are
|
|
43
|
+
* dropped rather than forwarded: `@qvac/model-fit` rejects everything outside
|
|
44
|
+
* its own allowlist, and a key it deliberately ignores must not turn a
|
|
45
|
+
* supported load into `unsupported-config`.
|
|
46
|
+
*/
|
|
47
|
+
const NON_FIT_KEYS = {
|
|
48
|
+
completion: [
|
|
49
|
+
'temp',
|
|
50
|
+
'top_p',
|
|
51
|
+
'top_k',
|
|
52
|
+
'seed',
|
|
53
|
+
'predict',
|
|
54
|
+
'presence_penalty',
|
|
55
|
+
'frequency_penalty',
|
|
56
|
+
'repeat_penalty',
|
|
57
|
+
'reverse_prompt',
|
|
58
|
+
'n_discarded',
|
|
59
|
+
'tools',
|
|
60
|
+
'verbosity',
|
|
61
|
+
'reasoning_budget',
|
|
62
|
+
'image_tile_mode',
|
|
63
|
+
'image_no_upscale',
|
|
64
|
+
'mmproj-use-gpu',
|
|
65
|
+
'openclCacheDir'
|
|
66
|
+
],
|
|
67
|
+
embedding: ['pooling', 'attention', 'embd_normalize', 'verbosity', 'openclCacheDir']
|
|
68
|
+
};
|
|
69
|
+
/**
|
|
70
|
+
* Load-config keys that describe a shape the fitter cannot answer for.
|
|
71
|
+
* `projection_model_src` reaches here when the config still carries the
|
|
72
|
+
* unresolved source, ahead of the resolved-artifact refusal below.
|
|
73
|
+
*/
|
|
74
|
+
const UNSUPPORTED_KEYS = ['lora', 'projection_model_src'];
|
|
75
|
+
/**
|
|
76
|
+
* A CPU load's weights stay file-backed and evictable, so the fitter projects
|
|
77
|
+
* `fits` for nearly any model the OS could page through. Measured on a 24 GiB
|
|
78
|
+
* M4 Pro against `@qvac/model-fit@0.8.0`: an 18.3 GiB model at 32k context
|
|
79
|
+
* still reports `fits` on `device: 'cpu'` (and it does decode — at 0.1 tok/s).
|
|
80
|
+
* That answer carries no admission information, so it is refused here rather
|
|
81
|
+
* than spending a fit call to produce it.
|
|
82
|
+
*/
|
|
83
|
+
function isCpuLoad(params) {
|
|
84
|
+
return params['device']?.toLowerCase() === 'cpu';
|
|
85
|
+
}
|
|
86
|
+
function loadKindFor(modelType) {
|
|
87
|
+
if (modelType === ModelType.llamacppCompletion)
|
|
88
|
+
return 'completion';
|
|
89
|
+
if (modelType === ModelType.llamacppEmbedding)
|
|
90
|
+
return 'embedding';
|
|
91
|
+
return undefined;
|
|
92
|
+
}
|
|
93
|
+
function unsupported(detail) {
|
|
94
|
+
return { supported: false, detail };
|
|
95
|
+
}
|
|
96
|
+
function partitionParams(loadKind, transformed) {
|
|
97
|
+
const forwarded = new Set(FIT_LOAD_KEYS[loadKind]);
|
|
98
|
+
const dropped = new Set(NON_FIT_KEYS[loadKind]);
|
|
99
|
+
const params = {};
|
|
100
|
+
for (const [key, value] of Object.entries(transformed)) {
|
|
101
|
+
if (UNSUPPORTED_KEYS.includes(key)) {
|
|
102
|
+
return { detail: `unsupported load setting: ${key}` };
|
|
103
|
+
}
|
|
104
|
+
if (forwarded.has(key)) {
|
|
105
|
+
params[key] = value;
|
|
106
|
+
continue;
|
|
107
|
+
}
|
|
108
|
+
if (dropped.has(key))
|
|
109
|
+
continue;
|
|
110
|
+
// Neither fit evidence nor a known non-memory setting: this load carries a
|
|
111
|
+
// setting the SDK cannot classify, so it must not be answered for.
|
|
112
|
+
return { detail: `unclassified load setting: ${key}` };
|
|
113
|
+
}
|
|
114
|
+
return { params };
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Pins the requested context so the fitter reports on the exact load the SDK
|
|
118
|
+
* is about to run. Left unset for an auto context (`0`, "use the model's
|
|
119
|
+
* trained context"), where the package's own floor applies and the fitter
|
|
120
|
+
* stays free to reduce.
|
|
121
|
+
*/
|
|
122
|
+
function contextFloor(params) {
|
|
123
|
+
const ctxSize = params['ctx_size'];
|
|
124
|
+
if (ctxSize === undefined)
|
|
125
|
+
return undefined;
|
|
126
|
+
const parsed = Number(ctxSize);
|
|
127
|
+
if (!Number.isSafeInteger(parsed) || parsed <= 0)
|
|
128
|
+
return undefined;
|
|
129
|
+
return parsed;
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Builds a protocol-v2 fit request from the same resolved model config the real
|
|
133
|
+
* load is about to use, or explains why this load cannot be answered for.
|
|
134
|
+
*
|
|
135
|
+
* Structural shapes the SDK owns are refused here so the fitter is not asked
|
|
136
|
+
* a question it cannot answer. Value-level policy (device names, symbolic GPU
|
|
137
|
+
* selection, context bounds) stays inside `@qvac/model-fit`.
|
|
138
|
+
*/
|
|
139
|
+
export function createLlamaFitRequest(params) {
|
|
140
|
+
const loadKind = loadKindFor(params.modelType);
|
|
141
|
+
if (loadKind === undefined) {
|
|
142
|
+
return unsupported(`model type is not a llama.cpp load: ${params.modelType}`);
|
|
143
|
+
}
|
|
144
|
+
if (params.isShardedModel) {
|
|
145
|
+
return unsupported('sharded models are not representable');
|
|
146
|
+
}
|
|
147
|
+
if (loadKind === 'completion' && params.artifacts?.['projectionModelPath'] !== undefined) {
|
|
148
|
+
return unsupported('multimodal projection loads are not representable');
|
|
149
|
+
}
|
|
150
|
+
const modelConfig = (params.modelConfig ?? {});
|
|
151
|
+
const transformed = loadKind === 'completion'
|
|
152
|
+
? transformLlmConfig(modelConfig)
|
|
153
|
+
: transformEmbedConfig(modelConfig);
|
|
154
|
+
const partitioned = partitionParams(loadKind, transformed);
|
|
155
|
+
if ('detail' in partitioned)
|
|
156
|
+
return unsupported(partitioned.detail);
|
|
157
|
+
if (isCpuLoad(partitioned.params)) {
|
|
158
|
+
return unsupported('cpu loads carry no device-memory evidence');
|
|
159
|
+
}
|
|
160
|
+
const nCtxMin = contextFloor(partitioned.params);
|
|
161
|
+
return {
|
|
162
|
+
supported: true,
|
|
163
|
+
loadKind,
|
|
164
|
+
config: {
|
|
165
|
+
// Sharded loads are refused above, so the resolved path is the whole model.
|
|
166
|
+
modelPath: params.modelPath,
|
|
167
|
+
params: partitioned.params,
|
|
168
|
+
...(nCtxMin !== undefined && { nCtxMin })
|
|
169
|
+
}
|
|
170
|
+
};
|
|
171
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { FitConfig, FitResult } from '@qvac/model-fit';
|
|
2
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
3
|
+
import type { FitLlamaProcessConfig, LlamaLoadKind } from '@qvac/model-fit/process';
|
|
4
|
+
import type { IsolatedFitUnknownReason } from './run-isolated-fit.js';
|
|
5
|
+
export interface RunInProcessFitOptions {
|
|
6
|
+
signal?: AbortSignal;
|
|
7
|
+
timeoutMs?: number;
|
|
8
|
+
fit?: (config: FitConfig) => FitResult | Promise<FitResult>;
|
|
9
|
+
stateDir?: string;
|
|
10
|
+
}
|
|
11
|
+
export declare function toFitConfig(config: FitLlamaProcessConfig): {
|
|
12
|
+
splitMode?: number;
|
|
13
|
+
flashAttnType?: number;
|
|
14
|
+
typeV?: number;
|
|
15
|
+
typeK?: number;
|
|
16
|
+
mainGpu?: number;
|
|
17
|
+
nBatch?: number;
|
|
18
|
+
nGpuLayers?: number;
|
|
19
|
+
nCtx?: number;
|
|
20
|
+
nCtxMin?: number;
|
|
21
|
+
marginMiB?: number;
|
|
22
|
+
backendsDir?: string;
|
|
23
|
+
modelPath: string;
|
|
24
|
+
};
|
|
25
|
+
export declare function crashMarkerPath(stateDir: string, config: FitLlamaProcessConfig): string;
|
|
26
|
+
export declare function crashedMarkerPath(stateDir: string, config: FitLlamaProcessConfig): string;
|
|
27
|
+
/**
|
|
28
|
+
* In-process llama.cpp fit for hosts with no disposable child (Android/iOS).
|
|
29
|
+
*
|
|
30
|
+
* The fit runs on a model-fit worker thread, so the JS loop stays free, but a
|
|
31
|
+
* thread is not a process boundary: a leftover `.running` marker means a
|
|
32
|
+
* previous call aborted the process. That is converted to `.crashed` so later
|
|
33
|
+
* launches skip native for the same path+config instead of retrying the abort.
|
|
34
|
+
*/
|
|
35
|
+
export declare function runInProcessFit(_loadKind: LlamaLoadKind, config: FitLlamaProcessConfig, options?: RunInProcessFitOptions): Promise<{
|
|
36
|
+
status: string;
|
|
37
|
+
reason: IsolatedFitUnknownReason;
|
|
38
|
+
message: string;
|
|
39
|
+
} | {
|
|
40
|
+
status: "completed";
|
|
41
|
+
result: FitResult;
|
|
42
|
+
}>;
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import fs from 'bare-fs';
|
|
2
|
+
import path from 'bare-path';
|
|
3
|
+
import { generateShortHash } from '../../../utils/formatting.js';
|
|
4
|
+
/**
|
|
5
|
+
* Stable `ggml_type` ordinals from `ggml.h` (the header `fitParams` is compiled
|
|
6
|
+
* against). Names llama.cpp has kept numbered since the enum was frozen:
|
|
7
|
+
* F32=0, F16=1, Q4_0=2, Q4_1=3, (4/5 removed Q4_2/Q4_3), Q5_0=6, Q5_1=7,
|
|
8
|
+
* Q8_0=8, Q8_1=9, IQ4_NL=20, BF16=30. Unmapped names (TurboQuant/PolarQuant)
|
|
9
|
+
* fail the conversion instead of silently projecting against llama defaults.
|
|
10
|
+
*/
|
|
11
|
+
const GGML_TYPE = {
|
|
12
|
+
f32: 0,
|
|
13
|
+
f16: 1,
|
|
14
|
+
q4_0: 2,
|
|
15
|
+
q4_1: 3,
|
|
16
|
+
q5_0: 6,
|
|
17
|
+
q5_1: 7,
|
|
18
|
+
q8_0: 8,
|
|
19
|
+
q8_1: 9,
|
|
20
|
+
iq4_nl: 20,
|
|
21
|
+
bf16: 30
|
|
22
|
+
};
|
|
23
|
+
/** `enum llama_flash_attn_type` — domain checked in `@qvac/model-fit` index.js. */
|
|
24
|
+
const FLASH_ATTN_TYPE = {
|
|
25
|
+
auto: -1,
|
|
26
|
+
off: 0,
|
|
27
|
+
on: 1
|
|
28
|
+
};
|
|
29
|
+
/** `enum llama_split_mode`. */
|
|
30
|
+
const SPLIT_MODE = {
|
|
31
|
+
none: 0,
|
|
32
|
+
layer: 1,
|
|
33
|
+
tensor: 3
|
|
34
|
+
};
|
|
35
|
+
function parseOptionalInt(value) {
|
|
36
|
+
if (value === undefined)
|
|
37
|
+
return undefined;
|
|
38
|
+
const parsed = Number(value);
|
|
39
|
+
if (!Number.isSafeInteger(parsed))
|
|
40
|
+
return undefined;
|
|
41
|
+
return parsed;
|
|
42
|
+
}
|
|
43
|
+
function lookup(table, value, label) {
|
|
44
|
+
if (value === undefined)
|
|
45
|
+
return undefined;
|
|
46
|
+
const mapped = table[value.toLowerCase()];
|
|
47
|
+
if (mapped === undefined) {
|
|
48
|
+
throw new TypeError(`Unsupported ${label} for in-process fit: ${value}`);
|
|
49
|
+
}
|
|
50
|
+
return mapped;
|
|
51
|
+
}
|
|
52
|
+
function param(config, ...keys) {
|
|
53
|
+
for (const key of keys) {
|
|
54
|
+
const value = config.params[key];
|
|
55
|
+
if (value !== undefined)
|
|
56
|
+
return value;
|
|
57
|
+
}
|
|
58
|
+
return undefined;
|
|
59
|
+
}
|
|
60
|
+
export function toFitConfig(config) {
|
|
61
|
+
const nCtx = parseOptionalInt(param(config, 'ctx_size', 'ctx-size'));
|
|
62
|
+
const nGpuLayers = parseOptionalInt(param(config, 'gpu_layers', 'gpu-layers', 'n-gpu-layers', 'n_gpu_layers'));
|
|
63
|
+
const nBatch = parseOptionalInt(param(config, 'batch_size', 'batch-size'));
|
|
64
|
+
const mainGpu = parseOptionalInt(param(config, 'main-gpu', 'main_gpu'));
|
|
65
|
+
const typeK = lookup(GGML_TYPE, param(config, 'cache-type-k'), 'KV cache type');
|
|
66
|
+
const typeV = lookup(GGML_TYPE, param(config, 'cache-type-v'), 'KV cache type');
|
|
67
|
+
const flashAttnType = lookup(FLASH_ATTN_TYPE, param(config, 'flash-attn', 'flash_attn'), 'flash-attn');
|
|
68
|
+
const splitMode = lookup(SPLIT_MODE, param(config, 'split-mode', 'split_mode'), 'split-mode');
|
|
69
|
+
// device, load_mode, parallel, and tensor-split are fit evidence on the
|
|
70
|
+
// process path but are not fields on public FitConfig — omitting them here
|
|
71
|
+
// is the API limit, not a silent default.
|
|
72
|
+
return {
|
|
73
|
+
modelPath: config.modelPath,
|
|
74
|
+
...(config.backendsDir !== undefined && { backendsDir: config.backendsDir }),
|
|
75
|
+
...(config.marginMiB !== undefined && { marginMiB: config.marginMiB }),
|
|
76
|
+
...(config.nCtxMin !== undefined && { nCtxMin: config.nCtxMin }),
|
|
77
|
+
...(nCtx !== undefined && nCtx > 0 && { nCtx }),
|
|
78
|
+
...(nGpuLayers !== undefined && { nGpuLayers }),
|
|
79
|
+
...(nBatch !== undefined && { nBatch }),
|
|
80
|
+
...(mainGpu !== undefined && { mainGpu }),
|
|
81
|
+
...(typeK !== undefined && { typeK }),
|
|
82
|
+
...(typeV !== undefined && { typeV }),
|
|
83
|
+
...(flashAttnType !== undefined && { flashAttnType }),
|
|
84
|
+
...(splitMode !== undefined && { splitMode })
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
function markerKey(config) {
|
|
88
|
+
return generateShortHash(JSON.stringify(config));
|
|
89
|
+
}
|
|
90
|
+
function runningMarkerPath(stateDir, config) {
|
|
91
|
+
return path.join(stateDir, `${markerKey(config)}.running`);
|
|
92
|
+
}
|
|
93
|
+
function crashedMarkerPathForConfig(stateDir, config) {
|
|
94
|
+
return path.join(stateDir, `${markerKey(config)}.crashed`);
|
|
95
|
+
}
|
|
96
|
+
export function crashMarkerPath(stateDir, config) {
|
|
97
|
+
return runningMarkerPath(stateDir, toFitConfig(config));
|
|
98
|
+
}
|
|
99
|
+
export function crashedMarkerPath(stateDir, config) {
|
|
100
|
+
return crashedMarkerPathForConfig(stateDir, toFitConfig(config));
|
|
101
|
+
}
|
|
102
|
+
function exists(file) {
|
|
103
|
+
try {
|
|
104
|
+
fs.accessSync(file);
|
|
105
|
+
return true;
|
|
106
|
+
}
|
|
107
|
+
catch {
|
|
108
|
+
return false;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
function writeMarker(file) {
|
|
112
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
113
|
+
fs.writeFileSync(file, '');
|
|
114
|
+
}
|
|
115
|
+
function clearMarker(file) {
|
|
116
|
+
try {
|
|
117
|
+
fs.unlinkSync(file);
|
|
118
|
+
}
|
|
119
|
+
catch {
|
|
120
|
+
// Marker cleanup must not turn an advisory fit into a load failure.
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
function persistCrash(running, crashed) {
|
|
124
|
+
try {
|
|
125
|
+
writeMarker(crashed);
|
|
126
|
+
clearMarker(running);
|
|
127
|
+
}
|
|
128
|
+
catch {
|
|
129
|
+
// Keep `.running` if `.crashed` could not be written so the next launch
|
|
130
|
+
// still skips native instead of retrying the abort.
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
function unknown(reason, message) {
|
|
134
|
+
return { status: 'unknown', reason, message };
|
|
135
|
+
}
|
|
136
|
+
function formatError(error) {
|
|
137
|
+
return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
|
138
|
+
}
|
|
139
|
+
async function defaultFit(config) {
|
|
140
|
+
const { fitParamsAsync } = await import('@qvac/model-fit');
|
|
141
|
+
return fitParamsAsync(config);
|
|
142
|
+
}
|
|
143
|
+
async function resolveStateDir(explicit) {
|
|
144
|
+
if (explicit !== undefined)
|
|
145
|
+
return explicit;
|
|
146
|
+
const { getCacheDir } = await import('../../../utils/cache/paths.js');
|
|
147
|
+
return getCacheDir('model-fit');
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* In-process llama.cpp fit for hosts with no disposable child (Android/iOS).
|
|
151
|
+
*
|
|
152
|
+
* The fit runs on a model-fit worker thread, so the JS loop stays free, but a
|
|
153
|
+
* thread is not a process boundary: a leftover `.running` marker means a
|
|
154
|
+
* previous call aborted the process. That is converted to `.crashed` so later
|
|
155
|
+
* launches skip native for the same path+config instead of retrying the abort.
|
|
156
|
+
*/
|
|
157
|
+
export async function runInProcessFit(_loadKind, config, options = {}) {
|
|
158
|
+
if (options.signal?.aborted === true) {
|
|
159
|
+
return unknown('cancelled', 'In-process fit was cancelled');
|
|
160
|
+
}
|
|
161
|
+
let fitConfig;
|
|
162
|
+
try {
|
|
163
|
+
fitConfig = toFitConfig(config);
|
|
164
|
+
}
|
|
165
|
+
catch (error) {
|
|
166
|
+
return unknown('invocation-error', formatError(error));
|
|
167
|
+
}
|
|
168
|
+
const stateDir = await resolveStateDir(options.stateDir);
|
|
169
|
+
const running = runningMarkerPath(stateDir, fitConfig);
|
|
170
|
+
const crashed = crashedMarkerPathForConfig(stateDir, fitConfig);
|
|
171
|
+
if (exists(crashed) || exists(running)) {
|
|
172
|
+
persistCrash(running, crashed);
|
|
173
|
+
return unknown('crashed', 'Previous in-process fit did not finish; treating this model as unknown');
|
|
174
|
+
}
|
|
175
|
+
try {
|
|
176
|
+
writeMarker(running);
|
|
177
|
+
}
|
|
178
|
+
catch (error) {
|
|
179
|
+
return unknown('invocation-error', `Fit crash marker could not be written: ${formatError(error)}`);
|
|
180
|
+
}
|
|
181
|
+
try {
|
|
182
|
+
const fit = options.fit ?? defaultFit;
|
|
183
|
+
const result = await fit(fitConfig);
|
|
184
|
+
return { status: 'completed', result };
|
|
185
|
+
}
|
|
186
|
+
catch (error) {
|
|
187
|
+
return unknown('invocation-error', formatError(error));
|
|
188
|
+
}
|
|
189
|
+
finally {
|
|
190
|
+
clearMarker(running);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { type FitLlamaProcessConfig, type FitLlamaResult, type LlamaLoadKind } from '@qvac/model-fit/process';
|
|
2
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
3
|
+
export type IsolatedFitUnknownReason = 'unsupported-platform' | 'spawn-failed' | 'timeout' | 'cancelled' | 'crashed' | 'invalid-response' | 'invocation-error';
|
|
4
|
+
export type IsolatedFitResult = {
|
|
5
|
+
status: 'completed';
|
|
6
|
+
result: FitLlamaResult;
|
|
7
|
+
} | {
|
|
8
|
+
status: 'unknown';
|
|
9
|
+
reason: IsolatedFitUnknownReason;
|
|
10
|
+
message: string;
|
|
11
|
+
stderrTail?: string;
|
|
12
|
+
};
|
|
13
|
+
export interface SpawnContext {
|
|
14
|
+
command: string;
|
|
15
|
+
options: {
|
|
16
|
+
args: string[];
|
|
17
|
+
platform: string;
|
|
18
|
+
arch: string;
|
|
19
|
+
stdio: string[];
|
|
20
|
+
env: Record<string, string>;
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
export interface RuntimeContext {
|
|
24
|
+
platform: string;
|
|
25
|
+
arch: string;
|
|
26
|
+
isAndroid: boolean;
|
|
27
|
+
isBrowser: boolean;
|
|
28
|
+
isIOS: boolean;
|
|
29
|
+
}
|
|
30
|
+
export interface RunIsolatedFitOptions {
|
|
31
|
+
timeoutMs?: number;
|
|
32
|
+
signal?: AbortSignal;
|
|
33
|
+
spawnProcess?: (context: SpawnContext) => ChildProcess;
|
|
34
|
+
runtime?: RuntimeContext;
|
|
35
|
+
environment?: Record<string, string | undefined>;
|
|
36
|
+
runnerPath?: string;
|
|
37
|
+
runnerArgs?: string[];
|
|
38
|
+
terminationGraceMs?: number;
|
|
39
|
+
finalKillGraceMs?: number;
|
|
40
|
+
drainGraceMs?: number;
|
|
41
|
+
}
|
|
42
|
+
export interface ErrorEmitter {
|
|
43
|
+
on(event: 'error', listener: (error: Error) => void): unknown;
|
|
44
|
+
off(event: 'error', listener: (error: Error) => void): unknown;
|
|
45
|
+
}
|
|
46
|
+
export interface ReadableChildStream {
|
|
47
|
+
setEncoding(encoding: 'utf8'): void;
|
|
48
|
+
destroy(): void;
|
|
49
|
+
on(event: 'error', listener: (error: Error) => void): unknown;
|
|
50
|
+
on(event: 'data', listener: (chunk: string) => void): unknown;
|
|
51
|
+
off(event: 'error', listener: (error: Error) => void): unknown;
|
|
52
|
+
off(event: 'data', listener: (chunk: string) => void): unknown;
|
|
53
|
+
}
|
|
54
|
+
export interface WritableChildStream {
|
|
55
|
+
end(data?: string): void;
|
|
56
|
+
destroy(): void;
|
|
57
|
+
on(event: 'error', listener: (error: Error) => void): unknown;
|
|
58
|
+
off(event: 'error', listener: (error: Error) => void): unknown;
|
|
59
|
+
}
|
|
60
|
+
export interface ChildProcess {
|
|
61
|
+
stdin: WritableChildStream | null;
|
|
62
|
+
stdout: ReadableChildStream | null;
|
|
63
|
+
stderr: ReadableChildStream | null;
|
|
64
|
+
kill(signal?: string): boolean;
|
|
65
|
+
on(event: 'error', listener: (error: Error) => void): unknown;
|
|
66
|
+
on(event: 'exit' | 'close', listener: (code: number | null, signal: string | number | null) => void): unknown;
|
|
67
|
+
off(event: 'error', listener: (error: Error) => void): unknown;
|
|
68
|
+
off(event: 'exit' | 'close', listener: (code: number | null, signal: string | number | null) => void): unknown;
|
|
69
|
+
}
|
|
70
|
+
export declare function runIsolatedFit(loadKind: LlamaLoadKind, config: FitLlamaProcessConfig, options?: RunIsolatedFitOptions): Promise<IsolatedFitResult>;
|