@qvac/inference 0.20.1 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +90 -24
- package/README.md +8 -5
- package/dist/api/bci-transcribe.d.ts +4 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/rpc-server.d.ts +19 -0
- package/dist/api/rpc-server.js +44 -0
- package/dist/api/text-to-speech.d.ts +9 -0
- package/dist/api/text-to-speech.js +9 -0
- package/dist/api/transcribe.d.ts +11 -7
- package/dist/api/transcribe.js +47 -15
- package/dist/api/video-request.d.ts +5 -5
- package/dist/dispatch.js +39 -7
- package/dist/errors/index.d.ts +7 -0
- package/dist/errors/index.js +15 -0
- package/dist/handlers/assess-model-fit.d.ts +8 -20
- package/dist/handlers/assess-model-fit.js +195 -77
- package/dist/handlers/completion-orchestrate.js +17 -0
- package/dist/handlers/get-loaded-model-info.js +4 -1
- package/dist/handlers/get-model-info.js +29 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
- package/dist/handlers/load-model/coreml-sidecars.js +274 -0
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/registry.js +59 -0
- package/dist/handlers/rpc-server.d.ts +23 -0
- package/dist/handlers/rpc-server.js +141 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/models/registry/models.d.ts +7731 -5672
- package/dist/models/registry/models.js +4652 -2526
- package/dist/models/registry/resource-profiles.js +481 -94
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +22 -11
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
- package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
- package/dist/plugins/ops/bci-transcribe.js +18 -13
- package/dist/plugins/ops/load-model.js +2 -2
- package/dist/plugins/ops/transcribe.d.ts +4 -2
- package/dist/plugins/ops/transcribe.js +39 -2
- package/dist/plugins/ops/translate-stats.d.ts +2 -1
- package/dist/plugins/ops/translate-stats.js +64 -7
- package/dist/plugins/ops/translate.js +20 -4
- package/dist/profiling/operation-metrics.js +56 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +37 -2
- package/dist/resources/model-fit/assess.js +24 -3
- package/dist/resources/model-fit/estimators/llm.js +2 -2
- package/dist/resources/model-fit/estimators/whisper.js +3 -4
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +20 -1
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +126 -47
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
- package/dist/resources/model-fit/floor.js +2 -2
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
- package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
- package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
- package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
- package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
- package/dist/resources/model-fit/platform.d.ts +9 -0
- package/dist/resources/model-fit/platform.js +28 -0
- package/dist/rpc/device-map.d.ts +13 -0
- package/dist/rpc/device-map.js +22 -0
- package/dist/rpc/discovery.d.ts +12 -0
- package/dist/rpc/discovery.js +225 -0
- package/dist/rpc/ggml-provider.d.ts +4 -0
- package/dist/rpc/ggml-provider.js +25 -0
- package/dist/rpc/index.d.ts +2 -0
- package/dist/rpc/index.js +1 -0
- package/dist/rpc/instance.d.ts +6 -0
- package/dist/rpc/instance.js +21 -0
- package/dist/rpc/inventory.d.ts +5 -0
- package/dist/rpc/inventory.js +144 -0
- package/dist/rpc/manager.d.ts +14 -0
- package/dist/rpc/manager.js +174 -0
- package/dist/rpc/network.d.ts +11 -0
- package/dist/rpc/network.js +85 -0
- package/dist/rpc/provider.d.ts +6 -0
- package/dist/rpc/provider.js +24 -0
- package/dist/runtime/lifecycle.js +31 -13
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +269 -85
- package/dist/schemas/assess-model-fit.js +78 -54
- package/dist/schemas/audio-gen.d.ts +6 -6
- package/dist/schemas/audio-gen.js +2 -2
- package/dist/schemas/batch-completion-stream.d.ts +13 -5
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci-config.js +1 -1
- package/dist/schemas/bci.d.ts +86 -1
- package/dist/schemas/bci.js +21 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +708 -66
- package/dist/schemas/common.js +7 -0
- package/dist/schemas/completion-event.d.ts +6 -6
- package/dist/schemas/completion-stream.d.ts +27 -8
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +104 -36
- package/dist/schemas/config.js +10 -0
- package/dist/schemas/embed.d.ts +2 -2
- package/dist/schemas/error.js +13 -0
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +25 -18
- package/dist/schemas/llamacpp-config.js +26 -16
- package/dist/schemas/load-model.d.ts +2608 -469
- package/dist/schemas/load-model.js +2 -2
- package/dist/schemas/mcp-adapter.d.ts +4 -0
- package/dist/schemas/model-config-schemas.d.ts +183 -6
- package/dist/schemas/model-info.d.ts +56 -0
- package/dist/schemas/model-info.js +5 -1
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/reload-config.d.ts +30 -0
- package/dist/schemas/rpc-server.d.ts +101 -0
- package/dist/schemas/rpc-server.js +87 -0
- package/dist/schemas/sdcpp-config.d.ts +7 -3
- package/dist/schemas/sdcpp-config.js +37 -2
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +529 -1
- package/dist/schemas/text-to-speech.js +122 -12
- package/dist/schemas/tools.d.ts +2 -0
- package/dist/schemas/tools.js +8 -0
- package/dist/schemas/transcription-config.d.ts +5 -0
- package/dist/schemas/transcription-config.js +20 -2
- package/dist/schemas/transcription.d.ts +164 -0
- package/dist/schemas/transcription.js +71 -4
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/surface.d.ts +4 -1
- package/dist/surface.js +3 -1
- package/dist/utils/addon-responses.d.ts +5 -1
- package/dist/utils/archive.js +5 -1
- package/dist/utils/asr-diagnostics.d.ts +12 -0
- package/dist/utils/asr-diagnostics.js +42 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +3 -2
- package/dist/utils/cache/paths.js +3 -2
- package/dist/utils/mcp-adapter.js +7 -1
- package/dist/utils/multi-gpu-mobile.d.ts +1 -1
- package/dist/utils/multi-gpu-mobile.js +8 -6
- package/dist/utils/path-security.d.ts +5 -0
- package/dist/utils/path-security.js +14 -0
- package/dist/utils/tool-helpers.d.ts +4 -0
- package/dist/utils/tool-helpers.js +2 -0
- package/dist/utils/tools/defer.d.ts +93 -0
- package/dist/utils/tools/defer.js +290 -0
- package/dist/utils/tools/index.d.ts +1 -0
- package/dist/utils/tools/index.js +1 -0
- package/dist/utils/transcribe-metadata.d.ts +21 -7
- package/dist/utils/transcribe-metadata.js +24 -8
- package/dist/utils/tts-stats.d.ts +6 -0
- package/dist/utils/tts-stats.js +14 -1
- package/package.json +43 -29
- package/dist/resources/model-fit/fit-stub/index.d.ts +0 -2
- package/dist/resources/model-fit/fit-stub/index.js +0 -2
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +0 -31
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +0 -36
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -191
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import processModule from 'bare-process';
|
|
2
|
+
import { callChildEngineFit } from './child-engine-fit.js';
|
|
3
|
+
import { FIT_PROCESS_MAX_REQUEST_BYTES, parseFitProcessRequest } from './fit-process.js';
|
|
4
|
+
/**
|
|
5
|
+
* Entry point of the disposable fit child. It loads one engine addon, asks it
|
|
6
|
+
* for a projection and exits, so a fitter that aborts the process takes nothing
|
|
7
|
+
* with it — which is the whole reason the projection does not run in the host.
|
|
8
|
+
*
|
|
9
|
+
* Writes exactly one JSON line to stdout and nothing else. Whatever an addon
|
|
10
|
+
* prints goes to stderr, where the supervisor keeps the tail.
|
|
11
|
+
*/
|
|
12
|
+
const process = processModule;
|
|
13
|
+
let finished = false;
|
|
14
|
+
function respond(line) {
|
|
15
|
+
if (finished)
|
|
16
|
+
return;
|
|
17
|
+
finished = true;
|
|
18
|
+
// One shot: stop reading before replying, so a still-open stdin cannot hold
|
|
19
|
+
// the child open once the response has been flushed.
|
|
20
|
+
process.stdin.pause();
|
|
21
|
+
process.stdout.write(line, (error) => {
|
|
22
|
+
if (error !== null && error !== undefined) {
|
|
23
|
+
process.stderr.write(`fit runner failed to write its response: ${error.message}\n`, () => {
|
|
24
|
+
process.exit(2);
|
|
25
|
+
});
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
process.exit(0);
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
function fail(error) {
|
|
32
|
+
const name = error instanceof Error ? error.name : 'Error';
|
|
33
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
34
|
+
respond(`${JSON.stringify({ status: 'invocation-error', error: { name, message } })}\n`);
|
|
35
|
+
}
|
|
36
|
+
async function answer(input) {
|
|
37
|
+
const { probe } = parseFitProcessRequest(JSON.parse(input));
|
|
38
|
+
const result = await callChildEngineFit(probe);
|
|
39
|
+
respond(`${JSON.stringify({ status: 'completed', probe: result })}\n`);
|
|
40
|
+
}
|
|
41
|
+
let input = '';
|
|
42
|
+
process.stdin.setEncoding('utf8');
|
|
43
|
+
process.stdin.on('data', (chunk) => {
|
|
44
|
+
if (finished)
|
|
45
|
+
return;
|
|
46
|
+
input += chunk;
|
|
47
|
+
if (Buffer.byteLength(input) > FIT_PROCESS_MAX_REQUEST_BYTES) {
|
|
48
|
+
fail(new RangeError(`Fit request exceeds ${FIT_PROCESS_MAX_REQUEST_BYTES} bytes`));
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
process.stdin.on('error', (error) => {
|
|
52
|
+
fail(error ?? new Error('fit runner stdin failed'));
|
|
53
|
+
});
|
|
54
|
+
process.stdin.on('end', () => {
|
|
55
|
+
if (finished)
|
|
56
|
+
return;
|
|
57
|
+
answer(input).catch(fail);
|
|
58
|
+
});
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { AbortSignal } from 'bare-abort-controller';
|
|
2
|
+
import type { FitProbeRequest } from './engine-fit.js';
|
|
3
|
+
import type { FitRunResult } from './fit-outcome.js';
|
|
4
|
+
export interface RunFitOptions {
|
|
5
|
+
signal?: AbortSignal;
|
|
6
|
+
timeoutMs?: number;
|
|
7
|
+
/**
|
|
8
|
+
* Whether this host can spawn a disposable child. Resolved by the caller,
|
|
9
|
+
* which already knows the runtime.
|
|
10
|
+
*/
|
|
11
|
+
mobile?: boolean;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Runs one engine's fitter and returns what it projected.
|
|
15
|
+
*
|
|
16
|
+
* A disposable child is the stronger boundary — a fitter that aborts takes only
|
|
17
|
+
* the child with it — so it is used wherever one can be spawned. Mobile has no
|
|
18
|
+
* such child and runs the fitter in process, carrying crash markers to bound
|
|
19
|
+
* the abort it cannot contain. `bare-subprocess` stays deferred from the mobile
|
|
20
|
+
* pack that way.
|
|
21
|
+
*
|
|
22
|
+
* Both strategies are imported lazily, so a host only loads the one it uses.
|
|
23
|
+
*/
|
|
24
|
+
export declare function runFit(probe: FitProbeRequest, options?: RunFitOptions): Promise<FitRunResult>;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runs one engine's fitter and returns what it projected.
|
|
3
|
+
*
|
|
4
|
+
* A disposable child is the stronger boundary — a fitter that aborts takes only
|
|
5
|
+
* the child with it — so it is used wherever one can be spawned. Mobile has no
|
|
6
|
+
* such child and runs the fitter in process, carrying crash markers to bound
|
|
7
|
+
* the abort it cannot contain. `bare-subprocess` stays deferred from the mobile
|
|
8
|
+
* pack that way.
|
|
9
|
+
*
|
|
10
|
+
* Both strategies are imported lazily, so a host only loads the one it uses.
|
|
11
|
+
*/
|
|
12
|
+
export async function runFit(probe, options = {}) {
|
|
13
|
+
const { mobile, timeoutMs, ...runOptions } = options;
|
|
14
|
+
if (mobile === true) {
|
|
15
|
+
const { runInProcessFit } = await import('./run-in-process-fit.js');
|
|
16
|
+
return runInProcessFit(probe, runOptions);
|
|
17
|
+
}
|
|
18
|
+
const { runIsolatedFit } = await import('./run-isolated-fit.js');
|
|
19
|
+
return runIsolatedFit(probe, { ...runOptions, ...(timeoutMs !== undefined && { timeoutMs }) });
|
|
20
|
+
}
|
|
@@ -1,42 +1,22 @@
|
|
|
1
|
-
import type { FitConfig, FitResult } from '@qvac/model-fit';
|
|
2
1
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
3
|
-
import
|
|
4
|
-
import type {
|
|
2
|
+
import { callEngineFit } from './engine-fit.js';
|
|
3
|
+
import type { FitProbeRequest } from './engine-fit.js';
|
|
4
|
+
import type { FitRunResult } from './fit-outcome.js';
|
|
5
5
|
export interface RunInProcessFitOptions {
|
|
6
6
|
signal?: AbortSignal;
|
|
7
|
-
timeoutMs?: number;
|
|
8
|
-
fit?: (config: FitConfig) => FitResult | Promise<FitResult>;
|
|
9
7
|
stateDir?: string;
|
|
8
|
+
callFit?: typeof callEngineFit;
|
|
10
9
|
}
|
|
11
|
-
export declare function
|
|
12
|
-
|
|
13
|
-
flashAttnType?: number;
|
|
14
|
-
typeV?: number;
|
|
15
|
-
typeK?: number;
|
|
16
|
-
mainGpu?: number;
|
|
17
|
-
nBatch?: number;
|
|
18
|
-
nGpuLayers?: number;
|
|
19
|
-
nCtx?: number;
|
|
20
|
-
nCtxMin?: number;
|
|
21
|
-
marginMiB?: number;
|
|
22
|
-
backendsDir?: string;
|
|
23
|
-
modelPath: string;
|
|
24
|
-
};
|
|
25
|
-
export declare function crashMarkerPath(stateDir: string, config: FitLlamaProcessConfig): string;
|
|
26
|
-
export declare function crashedMarkerPath(stateDir: string, config: FitLlamaProcessConfig): string;
|
|
10
|
+
export declare function crashMarkerPath(stateDir: string, probe: FitProbeRequest): string;
|
|
11
|
+
export declare function crashedMarkerPath(stateDir: string, probe: FitProbeRequest): string;
|
|
27
12
|
/**
|
|
28
|
-
*
|
|
13
|
+
* Runs one engine's fitter on the calling thread, for hosts that cannot spawn a
|
|
14
|
+
* disposable child.
|
|
29
15
|
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
16
|
+
* There is no boundary here to contain a fitter that aborts: it takes the host
|
|
17
|
+
* with it. A leftover `.running` marker is that abort, observed on the next
|
|
18
|
+
* launch, and it is promoted to `.crashed` so the same request skips native
|
|
19
|
+
* rather than repeating the abort. `timeoutMs` has no meaning on this path,
|
|
20
|
+
* since the native call blocks the thread until it returns.
|
|
34
21
|
*/
|
|
35
|
-
export declare function runInProcessFit(
|
|
36
|
-
status: string;
|
|
37
|
-
reason: IsolatedFitUnknownReason;
|
|
38
|
-
message: string;
|
|
39
|
-
} | {
|
|
40
|
-
status: "completed";
|
|
41
|
-
result: FitResult;
|
|
42
|
-
}>;
|
|
22
|
+
export declare function runInProcessFit(probe: FitProbeRequest, options?: RunInProcessFitOptions): Promise<FitRunResult>;
|
|
@@ -1,103 +1,28 @@
|
|
|
1
1
|
import fs from 'bare-fs';
|
|
2
2
|
import path from 'bare-path';
|
|
3
3
|
import { generateShortHash } from '../../../utils/formatting.js';
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
* F32=0, F16=1, Q4_0=2, Q4_1=3, (4/5 removed Q4_2/Q4_3), Q5_0=6, Q5_1=7,
|
|
8
|
-
* Q8_0=8, Q8_1=9, IQ4_NL=20, BF16=30. Unmapped names (TurboQuant/PolarQuant)
|
|
9
|
-
* fail the conversion instead of silently projecting against llama defaults.
|
|
10
|
-
*/
|
|
11
|
-
const GGML_TYPE = {
|
|
12
|
-
f32: 0,
|
|
13
|
-
f16: 1,
|
|
14
|
-
q4_0: 2,
|
|
15
|
-
q4_1: 3,
|
|
16
|
-
q5_0: 6,
|
|
17
|
-
q5_1: 7,
|
|
18
|
-
q8_0: 8,
|
|
19
|
-
q8_1: 9,
|
|
20
|
-
iq4_nl: 20,
|
|
21
|
-
bf16: 30
|
|
22
|
-
};
|
|
23
|
-
/** `enum llama_flash_attn_type` — domain checked in `@qvac/model-fit` index.js. */
|
|
24
|
-
const FLASH_ATTN_TYPE = {
|
|
25
|
-
auto: -1,
|
|
26
|
-
off: 0,
|
|
27
|
-
on: 1
|
|
28
|
-
};
|
|
29
|
-
/** `enum llama_split_mode`. */
|
|
30
|
-
const SPLIT_MODE = {
|
|
31
|
-
none: 0,
|
|
32
|
-
layer: 1,
|
|
33
|
-
tensor: 3
|
|
34
|
-
};
|
|
35
|
-
function parseOptionalInt(value) {
|
|
36
|
-
if (value === undefined)
|
|
37
|
-
return undefined;
|
|
38
|
-
const parsed = Number(value);
|
|
39
|
-
if (!Number.isSafeInteger(parsed))
|
|
40
|
-
return undefined;
|
|
41
|
-
return parsed;
|
|
42
|
-
}
|
|
43
|
-
function lookup(table, value, label) {
|
|
44
|
-
if (value === undefined)
|
|
45
|
-
return undefined;
|
|
46
|
-
const mapped = table[value.toLowerCase()];
|
|
47
|
-
if (mapped === undefined) {
|
|
48
|
-
throw new TypeError(`Unsupported ${label} for in-process fit: ${value}`);
|
|
49
|
-
}
|
|
50
|
-
return mapped;
|
|
51
|
-
}
|
|
52
|
-
function param(config, ...keys) {
|
|
53
|
-
for (const key of keys) {
|
|
54
|
-
const value = config.params[key];
|
|
55
|
-
if (value !== undefined)
|
|
56
|
-
return value;
|
|
57
|
-
}
|
|
58
|
-
return undefined;
|
|
59
|
-
}
|
|
60
|
-
export function toFitConfig(config) {
|
|
61
|
-
const nCtx = parseOptionalInt(param(config, 'ctx_size', 'ctx-size'));
|
|
62
|
-
const nGpuLayers = parseOptionalInt(param(config, 'gpu_layers', 'gpu-layers', 'n-gpu-layers', 'n_gpu_layers'));
|
|
63
|
-
const nBatch = parseOptionalInt(param(config, 'batch_size', 'batch-size'));
|
|
64
|
-
const mainGpu = parseOptionalInt(param(config, 'main-gpu', 'main_gpu'));
|
|
65
|
-
const typeK = lookup(GGML_TYPE, param(config, 'cache-type-k'), 'KV cache type');
|
|
66
|
-
const typeV = lookup(GGML_TYPE, param(config, 'cache-type-v'), 'KV cache type');
|
|
67
|
-
const flashAttnType = lookup(FLASH_ATTN_TYPE, param(config, 'flash-attn', 'flash_attn'), 'flash-attn');
|
|
68
|
-
const splitMode = lookup(SPLIT_MODE, param(config, 'split-mode', 'split_mode'), 'split-mode');
|
|
69
|
-
// device, load_mode, parallel, and tensor-split are fit evidence on the
|
|
70
|
-
// process path but are not fields on public FitConfig — omitting them here
|
|
71
|
-
// is the API limit, not a silent default.
|
|
72
|
-
return {
|
|
73
|
-
modelPath: config.modelPath,
|
|
74
|
-
...(config.backendsDir !== undefined && { backendsDir: config.backendsDir }),
|
|
75
|
-
...(config.marginMiB !== undefined && { marginMiB: config.marginMiB }),
|
|
76
|
-
...(config.nCtxMin !== undefined && { nCtxMin: config.nCtxMin }),
|
|
77
|
-
...(nCtx !== undefined && nCtx > 0 && { nCtx }),
|
|
78
|
-
...(nGpuLayers !== undefined && { nGpuLayers }),
|
|
79
|
-
...(nBatch !== undefined && { nBatch }),
|
|
80
|
-
...(mainGpu !== undefined && { mainGpu }),
|
|
81
|
-
...(typeK !== undefined && { typeK }),
|
|
82
|
-
...(typeV !== undefined && { typeV }),
|
|
83
|
-
...(flashAttnType !== undefined && { flashAttnType }),
|
|
84
|
-
...(splitMode !== undefined && { splitMode })
|
|
85
|
-
};
|
|
86
|
-
}
|
|
87
|
-
function markerKey(config) {
|
|
88
|
-
return generateShortHash(JSON.stringify(config));
|
|
4
|
+
import { callEngineFit } from './engine-fit.js';
|
|
5
|
+
function unknown(reason, message) {
|
|
6
|
+
return { status: 'unknown', reason, message };
|
|
89
7
|
}
|
|
90
|
-
function
|
|
91
|
-
return
|
|
8
|
+
function describe(error) {
|
|
9
|
+
return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
|
92
10
|
}
|
|
93
|
-
|
|
94
|
-
|
|
11
|
+
/** How long a marker keeps a model off the native path. */
|
|
12
|
+
const MARKER_TTL_MS = 7 * 24 * 60 * 60 * 1000;
|
|
13
|
+
/**
|
|
14
|
+
* `marginBytes` is excluded: it carries the bytes of every model resident at
|
|
15
|
+
* the time, so it differs between two attempts at the same load.
|
|
16
|
+
*/
|
|
17
|
+
function markerKey(probe) {
|
|
18
|
+
const { marginBytes: _margin, ...request } = probe.request;
|
|
19
|
+
return generateShortHash(JSON.stringify({ engine: probe.engine, request }));
|
|
95
20
|
}
|
|
96
|
-
export function crashMarkerPath(stateDir,
|
|
97
|
-
return
|
|
21
|
+
export function crashMarkerPath(stateDir, probe) {
|
|
22
|
+
return path.join(stateDir, `${markerKey(probe)}.running`);
|
|
98
23
|
}
|
|
99
|
-
export function crashedMarkerPath(stateDir,
|
|
100
|
-
return
|
|
24
|
+
export function crashedMarkerPath(stateDir, probe) {
|
|
25
|
+
return path.join(stateDir, `${markerKey(probe)}.crashed`);
|
|
101
26
|
}
|
|
102
27
|
function exists(file) {
|
|
103
28
|
try {
|
|
@@ -108,38 +33,22 @@ function exists(file) {
|
|
|
108
33
|
return false;
|
|
109
34
|
}
|
|
110
35
|
}
|
|
111
|
-
function
|
|
112
|
-
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
113
|
-
fs.writeFileSync(file, '');
|
|
114
|
-
}
|
|
115
|
-
function clearMarker(file) {
|
|
36
|
+
function expired(file) {
|
|
116
37
|
try {
|
|
117
|
-
fs.
|
|
38
|
+
return Date.now() - fs.statSync(file).mtimeMs > MARKER_TTL_MS;
|
|
118
39
|
}
|
|
119
40
|
catch {
|
|
120
|
-
|
|
41
|
+
return false;
|
|
121
42
|
}
|
|
122
43
|
}
|
|
123
|
-
function
|
|
44
|
+
function clearRunning(file) {
|
|
124
45
|
try {
|
|
125
|
-
|
|
126
|
-
clearMarker(running);
|
|
46
|
+
fs.unlinkSync(file);
|
|
127
47
|
}
|
|
128
48
|
catch {
|
|
129
|
-
//
|
|
130
|
-
// still skips native instead of retrying the abort.
|
|
49
|
+
// Marker cleanup must not turn an advisory fit into a load failure.
|
|
131
50
|
}
|
|
132
51
|
}
|
|
133
|
-
function unknown(reason, message) {
|
|
134
|
-
return { status: 'unknown', reason, message };
|
|
135
|
-
}
|
|
136
|
-
function formatError(error) {
|
|
137
|
-
return error instanceof Error ? `${error.name}: ${error.message}` : String(error);
|
|
138
|
-
}
|
|
139
|
-
async function defaultFit(config) {
|
|
140
|
-
const { fitParamsAsync } = await import('@qvac/model-fit');
|
|
141
|
-
return fitParamsAsync(config);
|
|
142
|
-
}
|
|
143
52
|
async function resolveStateDir(explicit) {
|
|
144
53
|
if (explicit !== undefined)
|
|
145
54
|
return explicit;
|
|
@@ -147,46 +56,53 @@ async function resolveStateDir(explicit) {
|
|
|
147
56
|
return getCacheDir('model-fit');
|
|
148
57
|
}
|
|
149
58
|
/**
|
|
150
|
-
*
|
|
59
|
+
* Runs one engine's fitter on the calling thread, for hosts that cannot spawn a
|
|
60
|
+
* disposable child.
|
|
151
61
|
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
*
|
|
155
|
-
*
|
|
62
|
+
* There is no boundary here to contain a fitter that aborts: it takes the host
|
|
63
|
+
* with it. A leftover `.running` marker is that abort, observed on the next
|
|
64
|
+
* launch, and it is promoted to `.crashed` so the same request skips native
|
|
65
|
+
* rather than repeating the abort. `timeoutMs` has no meaning on this path,
|
|
66
|
+
* since the native call blocks the thread until it returns.
|
|
156
67
|
*/
|
|
157
|
-
export async function runInProcessFit(
|
|
68
|
+
export async function runInProcessFit(probe, options = {}) {
|
|
158
69
|
if (options.signal?.aborted === true) {
|
|
159
70
|
return unknown('cancelled', 'In-process fit was cancelled');
|
|
160
71
|
}
|
|
161
|
-
let fitConfig;
|
|
162
|
-
try {
|
|
163
|
-
fitConfig = toFitConfig(config);
|
|
164
|
-
}
|
|
165
|
-
catch (error) {
|
|
166
|
-
return unknown('invocation-error', formatError(error));
|
|
167
|
-
}
|
|
168
72
|
const stateDir = await resolveStateDir(options.stateDir);
|
|
169
|
-
const running =
|
|
170
|
-
const crashed =
|
|
73
|
+
const running = crashMarkerPath(stateDir, probe);
|
|
74
|
+
const crashed = crashedMarkerPath(stateDir, probe);
|
|
75
|
+
for (const marker of [crashed, running]) {
|
|
76
|
+
if (exists(marker) && expired(marker))
|
|
77
|
+
clearRunning(marker);
|
|
78
|
+
}
|
|
171
79
|
if (exists(crashed) || exists(running)) {
|
|
172
|
-
|
|
173
|
-
|
|
80
|
+
try {
|
|
81
|
+
fs.mkdirSync(path.dirname(crashed), { recursive: true });
|
|
82
|
+
fs.writeFileSync(crashed, '');
|
|
83
|
+
clearRunning(running);
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
// Keep `.running` if `.crashed` could not be written so the next launch
|
|
87
|
+
// still skips native.
|
|
88
|
+
}
|
|
89
|
+
return unknown('crashed', 'Previous fit did not finish; treating this model as unknown');
|
|
174
90
|
}
|
|
175
91
|
try {
|
|
176
|
-
|
|
92
|
+
fs.mkdirSync(path.dirname(running), { recursive: true });
|
|
93
|
+
fs.writeFileSync(running, '');
|
|
177
94
|
}
|
|
178
95
|
catch (error) {
|
|
179
|
-
return unknown('invocation-error', `Fit crash marker could not be written: ${
|
|
96
|
+
return unknown('invocation-error', `Fit crash marker could not be written: ${describe(error)}`);
|
|
180
97
|
}
|
|
181
98
|
try {
|
|
182
|
-
const
|
|
183
|
-
|
|
184
|
-
return { status: 'completed', result };
|
|
99
|
+
const callFit = options.callFit ?? callEngineFit;
|
|
100
|
+
return { status: 'completed', probe: await callFit(probe) };
|
|
185
101
|
}
|
|
186
102
|
catch (error) {
|
|
187
|
-
return unknown('invocation-error',
|
|
103
|
+
return unknown('invocation-error', describe(error));
|
|
188
104
|
}
|
|
189
105
|
finally {
|
|
190
|
-
|
|
106
|
+
clearRunning(running);
|
|
191
107
|
}
|
|
192
108
|
}
|
|
@@ -1,15 +1,8 @@
|
|
|
1
|
-
import { type FitLlamaProcessConfig, type FitLlamaResult, type LlamaLoadKind } from '@qvac/model-fit/process';
|
|
2
1
|
import type { AbortSignal } from 'bare-abort-controller';
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
} | {
|
|
8
|
-
status: 'unknown';
|
|
9
|
-
reason: IsolatedFitUnknownReason;
|
|
10
|
-
message: string;
|
|
11
|
-
stderrTail?: string;
|
|
12
|
-
};
|
|
2
|
+
import type { FitProbeRequest } from './engine-fit.js';
|
|
3
|
+
import type { FitRunResult, FitRunUnknownReason } from './fit-outcome.js';
|
|
4
|
+
export type IsolatedFitUnknownReason = FitRunUnknownReason;
|
|
5
|
+
export type IsolatedFitResult = FitRunResult;
|
|
13
6
|
export interface SpawnContext {
|
|
14
7
|
command: string;
|
|
15
8
|
options: {
|
|
@@ -67,4 +60,4 @@ export interface ChildProcess {
|
|
|
67
60
|
off(event: 'error', listener: (error: Error) => void): unknown;
|
|
68
61
|
off(event: 'exit' | 'close', listener: (code: number | null, signal: string | number | null) => void): unknown;
|
|
69
62
|
}
|
|
70
|
-
export declare function runIsolatedFit(
|
|
63
|
+
export declare function runIsolatedFit(probe: FitProbeRequest, options?: RunIsolatedFitOptions): Promise<IsolatedFitResult>;
|
|
@@ -1,7 +1,15 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { fileURLToPath } from 'bare-url';
|
|
2
2
|
import env from 'bare-env';
|
|
3
3
|
import spawnBare from 'bare-runtime/spawn';
|
|
4
4
|
import { arch, isAndroid, isBrowser, isIOS, platform } from 'which-runtime';
|
|
5
|
+
import { encodeFitProcessRequest, FIT_PROCESS_MAX_RESPONSE_BYTES, parseFitProcessResponse } from './fit-process.js';
|
|
6
|
+
/**
|
|
7
|
+
* The child is this package's own runner, built beside this module, so the
|
|
8
|
+
* engine dispatch it performs is the same code the in-process path runs.
|
|
9
|
+
*/
|
|
10
|
+
function resolveFitRunnerPath() {
|
|
11
|
+
return fileURLToPath(new URL('./fit-runner.js', import.meta.url));
|
|
12
|
+
}
|
|
5
13
|
const DEFAULT_TIMEOUT_MS = 60_000;
|
|
6
14
|
const TERMINATION_GRACE_MS = 1_000;
|
|
7
15
|
const FINAL_KILL_GRACE_MS = 1_000;
|
|
@@ -61,13 +69,13 @@ function parseResponse(line, stderrTail) {
|
|
|
61
69
|
if (response.status === 'invocation-error') {
|
|
62
70
|
return unknown('invocation-error', `${response.error.name}: ${response.error.message}`, stderrTail);
|
|
63
71
|
}
|
|
64
|
-
return { status: 'completed',
|
|
72
|
+
return { status: 'completed', probe: response.probe };
|
|
65
73
|
}
|
|
66
74
|
catch (error) {
|
|
67
75
|
return unknown('invalid-response', formatError(error), stderrTail);
|
|
68
76
|
}
|
|
69
77
|
}
|
|
70
|
-
export function runIsolatedFit(
|
|
78
|
+
export function runIsolatedFit(probe, options = {}) {
|
|
71
79
|
const runtime = options.runtime ?? {
|
|
72
80
|
platform,
|
|
73
81
|
arch,
|
|
@@ -303,10 +311,7 @@ export function runIsolatedFit(loadKind, config, options = {}) {
|
|
|
303
311
|
child = spawnProcess({
|
|
304
312
|
command: 'bare',
|
|
305
313
|
options: {
|
|
306
|
-
args: [
|
|
307
|
-
options.runnerPath ?? resolveFitProcessRunnerPath(),
|
|
308
|
-
...(options.runnerArgs ?? [])
|
|
309
|
-
],
|
|
314
|
+
args: [options.runnerPath ?? resolveFitRunnerPath(), ...(options.runnerArgs ?? [])],
|
|
310
315
|
platform: runtime.platform,
|
|
311
316
|
arch: runtime.arch,
|
|
312
317
|
stdio,
|
|
@@ -362,7 +367,7 @@ export function runIsolatedFit(loadKind, config, options = {}) {
|
|
|
362
367
|
return;
|
|
363
368
|
}
|
|
364
369
|
try {
|
|
365
|
-
stdin.end(
|
|
370
|
+
stdin.end(encodeFitProcessRequest(probe));
|
|
366
371
|
}
|
|
367
372
|
catch (error) {
|
|
368
373
|
requestTermination('invalid-response', `Fit subprocess request write failed: ${formatError(error)}`);
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { ModelFitPlatform } from './types.js';
|
|
2
|
+
/**
|
|
3
|
+
* Maps the runtime's platform and architecture onto a calibration target.
|
|
4
|
+
*
|
|
5
|
+
* @returns `undefined` for any pair this feature has no calibration target for,
|
|
6
|
+
* which assesses as `unknown` rather than borrowing another platform's
|
|
7
|
+
* coefficients.
|
|
8
|
+
*/
|
|
9
|
+
export declare function detectPlatform(): ModelFitPlatform | undefined;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import os from 'bare-os';
|
|
2
|
+
/**
|
|
3
|
+
* Maps the runtime's platform and architecture onto a calibration target.
|
|
4
|
+
*
|
|
5
|
+
* @returns `undefined` for any pair this feature has no calibration target for,
|
|
6
|
+
* which assesses as `unknown` rather than borrowing another platform's
|
|
7
|
+
* coefficients.
|
|
8
|
+
*/
|
|
9
|
+
export function detectPlatform() {
|
|
10
|
+
const key = `${os.platform()}-${os.arch()}`;
|
|
11
|
+
switch (key) {
|
|
12
|
+
case 'darwin-arm64':
|
|
13
|
+
case 'darwin-x64':
|
|
14
|
+
case 'linux-arm64':
|
|
15
|
+
case 'linux-x64':
|
|
16
|
+
case 'win32-x64':
|
|
17
|
+
// Listed for completeness rather than reach: no engine addon is built for
|
|
18
|
+
// win32-arm64 (`@qvac/llm-llamacpp/prebuilds` has no such target, and no
|
|
19
|
+
// windows-arm runner exists to build one), so nothing on that platform can
|
|
20
|
+
// load a model today. Calibrating it needs that build first, not a fixture.
|
|
21
|
+
case 'win32-arm64':
|
|
22
|
+
case 'android-arm64':
|
|
23
|
+
case 'ios-arm64':
|
|
24
|
+
return key;
|
|
25
|
+
default:
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { RpcDevice, RpcServerCandidate } from '../schemas/rpc-server.js';
|
|
2
|
+
export type RpcDeviceMapping = RpcDevice & {
|
|
3
|
+
url: string;
|
|
4
|
+
alias: string;
|
|
5
|
+
};
|
|
6
|
+
/**
|
|
7
|
+
* Pass exactly this load's rpc-servers endpoints in the same order, even on
|
|
8
|
+
* reused workers. Aliases enumerate every device in that order, starting at RPC0
|
|
9
|
+
* for each load. Use the aliases to select devices; tensor-split weights follow
|
|
10
|
+
* the selected devices order. Repeated endpoints are skipped by this helper,
|
|
11
|
+
* but rpc-servers must contain unique endpoints. Discovery does not register devices.
|
|
12
|
+
*/
|
|
13
|
+
export declare function getRpcDeviceMap(servers: readonly RpcServerCandidate[]): RpcDeviceMapping[];
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pass exactly this load's rpc-servers endpoints in the same order, even on
|
|
3
|
+
* reused workers. Aliases enumerate every device in that order, starting at RPC0
|
|
4
|
+
* for each load. Use the aliases to select devices; tensor-split weights follow
|
|
5
|
+
* the selected devices order. Repeated endpoints are skipped by this helper,
|
|
6
|
+
* but rpc-servers must contain unique endpoints. Discovery does not register devices.
|
|
7
|
+
*/
|
|
8
|
+
export function getRpcDeviceMap(servers) {
|
|
9
|
+
const seen = new Set();
|
|
10
|
+
const result = [];
|
|
11
|
+
for (const server of servers) {
|
|
12
|
+
if (seen.has(server.url))
|
|
13
|
+
continue;
|
|
14
|
+
seen.add(server.url);
|
|
15
|
+
for (const [index, device] of server.devices.entries()) {
|
|
16
|
+
if (device.index !== index)
|
|
17
|
+
throw new Error('RPC inventory must be in native device order');
|
|
18
|
+
result.push({ ...device, url: server.url, alias: `RPC${result.length}` });
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
return result;
|
|
22
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import Hyperswarm from 'hyperswarm';
|
|
2
|
+
import type { RequestContext } from '../runtime/request-context.js';
|
|
3
|
+
import { queryRpcInventory } from './inventory.js';
|
|
4
|
+
import type { RpcServerCandidate } from '../schemas/rpc-server.js';
|
|
5
|
+
export declare function createDiscoverySwarm(): Hyperswarm;
|
|
6
|
+
export declare function destroyDiscoverySwarm(swarm: Hyperswarm): Promise<void>;
|
|
7
|
+
export declare function parseAnnouncement(line: string): string | undefined;
|
|
8
|
+
export declare function advertiseRpcServer(topic: string, url: string, createSwarm?: typeof createDiscoverySwarm): Promise<() => Promise<void>>;
|
|
9
|
+
export declare function discoverRpcEndpoints(topic: string, timeoutMs: number, ctx: RequestContext, deps?: {
|
|
10
|
+
createSwarm?: typeof createDiscoverySwarm;
|
|
11
|
+
probe?: typeof queryRpcInventory;
|
|
12
|
+
}): Promise<RpcServerCandidate[]>;
|