@qvac/inference 0.20.1 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +90 -24
- package/README.md +8 -5
- package/dist/api/bci-transcribe.d.ts +4 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/rpc-server.d.ts +19 -0
- package/dist/api/rpc-server.js +44 -0
- package/dist/api/text-to-speech.d.ts +9 -0
- package/dist/api/text-to-speech.js +9 -0
- package/dist/api/transcribe.d.ts +11 -7
- package/dist/api/transcribe.js +47 -15
- package/dist/api/video-request.d.ts +5 -5
- package/dist/dispatch.js +39 -7
- package/dist/errors/index.d.ts +7 -0
- package/dist/errors/index.js +15 -0
- package/dist/handlers/assess-model-fit.d.ts +8 -20
- package/dist/handlers/assess-model-fit.js +195 -77
- package/dist/handlers/completion-orchestrate.js +17 -0
- package/dist/handlers/get-loaded-model-info.js +4 -1
- package/dist/handlers/get-model-info.js +29 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
- package/dist/handlers/load-model/coreml-sidecars.js +274 -0
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/registry.js +59 -0
- package/dist/handlers/rpc-server.d.ts +23 -0
- package/dist/handlers/rpc-server.js +141 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/models/registry/models.d.ts +7731 -5672
- package/dist/models/registry/models.js +4652 -2526
- package/dist/models/registry/resource-profiles.js +481 -94
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +22 -11
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
- package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
- package/dist/plugins/ops/bci-transcribe.js +18 -13
- package/dist/plugins/ops/load-model.js +2 -2
- package/dist/plugins/ops/transcribe.d.ts +4 -2
- package/dist/plugins/ops/transcribe.js +39 -2
- package/dist/plugins/ops/translate-stats.d.ts +2 -1
- package/dist/plugins/ops/translate-stats.js +64 -7
- package/dist/plugins/ops/translate.js +20 -4
- package/dist/profiling/operation-metrics.js +56 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +37 -2
- package/dist/resources/model-fit/assess.js +24 -3
- package/dist/resources/model-fit/estimators/llm.js +2 -2
- package/dist/resources/model-fit/estimators/whisper.js +3 -4
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +20 -1
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +126 -47
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
- package/dist/resources/model-fit/floor.js +2 -2
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
- package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
- package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
- package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
- package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
- package/dist/resources/model-fit/platform.d.ts +9 -0
- package/dist/resources/model-fit/platform.js +28 -0
- package/dist/rpc/device-map.d.ts +13 -0
- package/dist/rpc/device-map.js +22 -0
- package/dist/rpc/discovery.d.ts +12 -0
- package/dist/rpc/discovery.js +225 -0
- package/dist/rpc/ggml-provider.d.ts +4 -0
- package/dist/rpc/ggml-provider.js +25 -0
- package/dist/rpc/index.d.ts +2 -0
- package/dist/rpc/index.js +1 -0
- package/dist/rpc/instance.d.ts +6 -0
- package/dist/rpc/instance.js +21 -0
- package/dist/rpc/inventory.d.ts +5 -0
- package/dist/rpc/inventory.js +144 -0
- package/dist/rpc/manager.d.ts +14 -0
- package/dist/rpc/manager.js +174 -0
- package/dist/rpc/network.d.ts +11 -0
- package/dist/rpc/network.js +85 -0
- package/dist/rpc/provider.d.ts +6 -0
- package/dist/rpc/provider.js +24 -0
- package/dist/runtime/lifecycle.js +31 -13
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +269 -85
- package/dist/schemas/assess-model-fit.js +78 -54
- package/dist/schemas/audio-gen.d.ts +6 -6
- package/dist/schemas/audio-gen.js +2 -2
- package/dist/schemas/batch-completion-stream.d.ts +13 -5
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci-config.js +1 -1
- package/dist/schemas/bci.d.ts +86 -1
- package/dist/schemas/bci.js +21 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +708 -66
- package/dist/schemas/common.js +7 -0
- package/dist/schemas/completion-event.d.ts +6 -6
- package/dist/schemas/completion-stream.d.ts +27 -8
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +104 -36
- package/dist/schemas/config.js +10 -0
- package/dist/schemas/embed.d.ts +2 -2
- package/dist/schemas/error.js +13 -0
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +25 -18
- package/dist/schemas/llamacpp-config.js +26 -16
- package/dist/schemas/load-model.d.ts +2608 -469
- package/dist/schemas/load-model.js +2 -2
- package/dist/schemas/mcp-adapter.d.ts +4 -0
- package/dist/schemas/model-config-schemas.d.ts +183 -6
- package/dist/schemas/model-info.d.ts +56 -0
- package/dist/schemas/model-info.js +5 -1
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/reload-config.d.ts +30 -0
- package/dist/schemas/rpc-server.d.ts +101 -0
- package/dist/schemas/rpc-server.js +87 -0
- package/dist/schemas/sdcpp-config.d.ts +7 -3
- package/dist/schemas/sdcpp-config.js +37 -2
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +529 -1
- package/dist/schemas/text-to-speech.js +122 -12
- package/dist/schemas/tools.d.ts +2 -0
- package/dist/schemas/tools.js +8 -0
- package/dist/schemas/transcription-config.d.ts +5 -0
- package/dist/schemas/transcription-config.js +20 -2
- package/dist/schemas/transcription.d.ts +164 -0
- package/dist/schemas/transcription.js +71 -4
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/surface.d.ts +4 -1
- package/dist/surface.js +3 -1
- package/dist/utils/addon-responses.d.ts +5 -1
- package/dist/utils/archive.js +5 -1
- package/dist/utils/asr-diagnostics.d.ts +12 -0
- package/dist/utils/asr-diagnostics.js +42 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +3 -2
- package/dist/utils/cache/paths.js +3 -2
- package/dist/utils/mcp-adapter.js +7 -1
- package/dist/utils/multi-gpu-mobile.d.ts +1 -1
- package/dist/utils/multi-gpu-mobile.js +8 -6
- package/dist/utils/path-security.d.ts +5 -0
- package/dist/utils/path-security.js +14 -0
- package/dist/utils/tool-helpers.d.ts +4 -0
- package/dist/utils/tool-helpers.js +2 -0
- package/dist/utils/tools/defer.d.ts +93 -0
- package/dist/utils/tools/defer.js +290 -0
- package/dist/utils/tools/index.d.ts +1 -0
- package/dist/utils/tools/index.js +1 -0
- package/dist/utils/transcribe-metadata.d.ts +21 -7
- package/dist/utils/transcribe-metadata.js +24 -8
- package/dist/utils/tts-stats.d.ts +6 -0
- package/dist/utils/tts-stats.js +14 -1
- package/package.json +43 -29
- package/dist/resources/model-fit/fit-stub/index.d.ts +0 -2
- package/dist/resources/model-fit/fit-stub/index.js +0 -2
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.d.ts +0 -31
- package/dist/resources/model-fit/fit-stub/project-fit-from-stub.js +0 -36
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -191
|
@@ -56,6 +56,7 @@ import { getEngineLogger } from '../../logging/index.js';
|
|
|
56
56
|
import { TranscriptionFailedError } from '../../errors/index.js';
|
|
57
57
|
import { nowMs } from '../../profiling/index.js';
|
|
58
58
|
import { buildStreamResult } from '../../profiling/model-execution.js';
|
|
59
|
+
import { buildAsrBackendDiagnostics } from '../../utils/asr-diagnostics.js';
|
|
59
60
|
import { toTranscribeSegment } from '../../utils/transcribe-metadata.js';
|
|
60
61
|
import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
|
|
61
62
|
import { generateRandomRequestId } from '../../runtime/request-id.js';
|
|
@@ -123,12 +124,6 @@ export async function* bciTranscribe(params, requestId) {
|
|
|
123
124
|
}
|
|
124
125
|
const modelExecutionMs = nowMs() - modelStart;
|
|
125
126
|
const stats = {
|
|
126
|
-
...(response.stats?.audioDurationMs !== undefined && {
|
|
127
|
-
audioDuration: response.stats.audioDurationMs
|
|
128
|
-
}),
|
|
129
|
-
...(response.stats?.realTimeFactor !== undefined && {
|
|
130
|
-
realTimeFactor: response.stats.realTimeFactor
|
|
131
|
-
}),
|
|
132
127
|
...(response.stats?.tokensPerSecond !== undefined && {
|
|
133
128
|
tokensPerSecond: response.stats.tokensPerSecond
|
|
134
129
|
}),
|
|
@@ -144,14 +139,18 @@ export async function* bciTranscribe(params, requestId) {
|
|
|
144
139
|
...(response.stats?.whisperDecodeMs !== undefined && {
|
|
145
140
|
whisperDecodeTime: response.stats.whisperDecodeMs
|
|
146
141
|
}),
|
|
147
|
-
...(response.stats?.
|
|
148
|
-
|
|
142
|
+
...(response.stats?.whisperSampleMs !== undefined && {
|
|
143
|
+
whisperSampleMs: response.stats.whisperSampleMs
|
|
144
|
+
}),
|
|
145
|
+
...(response.stats?.whisperBatchdMs !== undefined && {
|
|
146
|
+
whisperBatchdMs: response.stats.whisperBatchdMs
|
|
149
147
|
}),
|
|
150
|
-
...(response.stats?.
|
|
151
|
-
|
|
148
|
+
...(response.stats?.whisperPromptMs !== undefined && {
|
|
149
|
+
whisperPromptMs: response.stats.whisperPromptMs
|
|
152
150
|
}),
|
|
153
|
-
...(response.stats?.
|
|
154
|
-
|
|
151
|
+
...(response.stats?.totalWallMs !== undefined && { totalWallMs: response.stats.totalWallMs }),
|
|
152
|
+
...(response.stats?.processCalls !== undefined && {
|
|
153
|
+
processCalls: response.stats.processCalls
|
|
155
154
|
}),
|
|
156
155
|
...(response.stats?.backendDevice !== undefined && {
|
|
157
156
|
backendDevice: response.stats.backendDevice
|
|
@@ -166,7 +165,11 @@ export async function* bciTranscribe(params, requestId) {
|
|
|
166
165
|
gpuMemFreeMb: response.stats.gpuMemFreeMb
|
|
167
166
|
})
|
|
168
167
|
};
|
|
169
|
-
|
|
168
|
+
const diagnostics = buildAsrBackendDiagnostics(stats);
|
|
169
|
+
return {
|
|
170
|
+
...buildStreamResult(modelExecutionMs, stats),
|
|
171
|
+
...(diagnostics && { diagnostics })
|
|
172
|
+
};
|
|
170
173
|
}
|
|
171
174
|
catch (e_1) {
|
|
172
175
|
env_1.error = e_1;
|
|
@@ -216,6 +219,7 @@ export async function* bciTranscribeStream(modelId, neuralStream, metadata, opts
|
|
|
216
219
|
...(opts?.hopTimesteps !== undefined && { hopTimesteps: opts.hopTimesteps }),
|
|
217
220
|
...(opts?.emit !== undefined && { emit: opts.emit })
|
|
218
221
|
};
|
|
222
|
+
const modelStart = nowMs();
|
|
219
223
|
const response = await model.transcribeStream(neuralStream, streamOpts);
|
|
220
224
|
for await (const output of response.iterate()) {
|
|
221
225
|
if (ctx.signal.aborted)
|
|
@@ -235,6 +239,7 @@ export async function* bciTranscribeStream(modelId, neuralStream, metadata, opts
|
|
|
235
239
|
yield text;
|
|
236
240
|
}
|
|
237
241
|
}
|
|
242
|
+
return buildStreamResult(nowMs() - modelStart);
|
|
238
243
|
}
|
|
239
244
|
catch (e_2) {
|
|
240
245
|
env_2.error = e_2;
|
|
@@ -57,8 +57,8 @@ export async function loadModel(params, options) {
|
|
|
57
57
|
// to the ordinary load below. Runs after config resolution and path
|
|
58
58
|
// validation so it sees the same state the real load uses, and before
|
|
59
59
|
// `createModel()` so it never competes with the native load for device
|
|
60
|
-
// memory. The outcome is stored on the registry entry
|
|
61
|
-
//
|
|
60
|
+
// memory. The outcome is stored on the registry entry and returned by
|
|
61
|
+
// `getLoadedModelInfo` as `fitProbe`.
|
|
62
62
|
const fitProbe = await runAdvisoryFitCheck({
|
|
63
63
|
modelId,
|
|
64
64
|
modelType: modelType,
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { type TranscribeParams, type TranscribeSegment, type TranscribeStats, type TranscribeStreamEvent, type ParakeetStreamingRunConfig } from '../../schemas/index.js';
|
|
2
2
|
import type Buffer from 'bare-buffer';
|
|
3
|
-
import {
|
|
4
|
-
|
|
3
|
+
import type { InferenceBackendDiagnostics } from '../../schemas/index.js';
|
|
4
|
+
import { assertMetadataSupported, toTranscribeSegment, type AsrAddonSegment } from '../../utils/transcribe-metadata.js';
|
|
5
|
+
export { assertMetadataSupported, toTranscribeSegment, type AsrAddonSegment };
|
|
5
6
|
type TranscribeReturn = {
|
|
6
7
|
modelExecutionMs: number;
|
|
7
8
|
stats?: TranscribeStats;
|
|
9
|
+
diagnostics?: InferenceBackendDiagnostics;
|
|
8
10
|
};
|
|
9
11
|
export declare function transcribe(params: TranscribeParams & {
|
|
10
12
|
metadata: true;
|
|
@@ -57,6 +57,7 @@ import { getEngineLogger } from '../../logging/index.js';
|
|
|
57
57
|
import { TranscriptionFailedError } from '../../errors/index.js';
|
|
58
58
|
import { nowMs } from '../../profiling/index.js';
|
|
59
59
|
import { buildStreamResult } from '../../profiling/model-execution.js';
|
|
60
|
+
import { buildAsrBackendDiagnostics } from '../../utils/asr-diagnostics.js';
|
|
60
61
|
import { assertMetadataSupported, toTranscribeSegment } from '../../utils/transcribe-metadata.js';
|
|
61
62
|
import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
|
|
62
63
|
import { generateRandomRequestId } from '../../runtime/request-id.js';
|
|
@@ -146,6 +147,32 @@ function mapTranscribeStats(stats) {
|
|
|
146
147
|
}),
|
|
147
148
|
...(stats?.gpuMemFreeMb !== undefined && {
|
|
148
149
|
gpuMemFreeMb: stats.gpuMemFreeMb
|
|
150
|
+
}),
|
|
151
|
+
...(stats?.totalTime !== undefined && { totalTime: stats.totalTime }),
|
|
152
|
+
...(stats?.totalWallMs !== undefined && { totalWallMs: stats.totalWallMs }),
|
|
153
|
+
...(stats?.totalSamples !== undefined && { totalSamples: stats.totalSamples }),
|
|
154
|
+
...(stats?.processCalls !== undefined && { processCalls: stats.processCalls }),
|
|
155
|
+
...(stats?.whisperSampleMs !== undefined && {
|
|
156
|
+
whisperSampleMs: stats.whisperSampleMs
|
|
157
|
+
}),
|
|
158
|
+
...(stats?.whisperBatchdMs !== undefined && {
|
|
159
|
+
whisperBatchdMs: stats.whisperBatchdMs
|
|
160
|
+
}),
|
|
161
|
+
...(stats?.whisperPromptMs !== undefined && {
|
|
162
|
+
whisperPromptMs: stats.whisperPromptMs
|
|
163
|
+
}),
|
|
164
|
+
...(stats?.totalTranscriptions !== undefined && {
|
|
165
|
+
totalTranscriptions: stats.totalTranscriptions
|
|
166
|
+
}),
|
|
167
|
+
...(stats?.modelLoadMs !== undefined && { modelLoadMs: stats.modelLoadMs }),
|
|
168
|
+
...(stats?.totalEncodedFrames !== undefined && {
|
|
169
|
+
totalEncodedFrames: stats.totalEncodedFrames
|
|
170
|
+
}),
|
|
171
|
+
...(stats?.encoderOnCoreml !== undefined && {
|
|
172
|
+
encoderOnCoreml: stats.encoderOnCoreml
|
|
173
|
+
}),
|
|
174
|
+
...(stats?.encoderUsedCoreml !== undefined && {
|
|
175
|
+
encoderUsedCoreml: stats.encoderUsedCoreml
|
|
149
176
|
})
|
|
150
177
|
};
|
|
151
178
|
}
|
|
@@ -222,7 +249,12 @@ export async function* transcribe(params, requestId) {
|
|
|
222
249
|
}
|
|
223
250
|
}
|
|
224
251
|
const modelExecutionMs = nowMs() - modelStart;
|
|
225
|
-
|
|
252
|
+
const mapped = mapTranscribeStats(response.stats);
|
|
253
|
+
const diagnostics = buildAsrBackendDiagnostics(mapped);
|
|
254
|
+
return {
|
|
255
|
+
...buildStreamResult(modelExecutionMs, mapped),
|
|
256
|
+
...(diagnostics && { diagnostics })
|
|
257
|
+
};
|
|
226
258
|
}
|
|
227
259
|
catch (e_1) {
|
|
228
260
|
env_1.error = e_1;
|
|
@@ -321,7 +353,12 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
|
|
|
321
353
|
}
|
|
322
354
|
await response.await();
|
|
323
355
|
const modelExecutionMs = nowMs() - modelStart;
|
|
324
|
-
|
|
356
|
+
const mapped = mapTranscribeStats(response.stats);
|
|
357
|
+
const diagnostics = buildAsrBackendDiagnostics(mapped);
|
|
358
|
+
return {
|
|
359
|
+
...buildStreamResult(modelExecutionMs, mapped),
|
|
360
|
+
...(diagnostics && { diagnostics })
|
|
361
|
+
};
|
|
325
362
|
}
|
|
326
363
|
catch (e_2) {
|
|
327
364
|
env_2.error = e_2;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { TranslationStats } from '../../schemas/index.js';
|
|
2
2
|
import type { NmtStats } from '../../utils/addon-responses.js';
|
|
3
3
|
export declare const NMT_SECONDS_TO_MS = 1000;
|
|
4
|
-
export declare function
|
|
4
|
+
export declare function markNmtCountersStale(model: object): void;
|
|
5
|
+
export declare function buildNmtTranslationStats(stats: Partial<NmtStats> | undefined, model: object, config?: unknown): TranslationStats;
|
|
@@ -1,11 +1,68 @@
|
|
|
1
1
|
export const NMT_SECONDS_TO_MS = 1000;
|
|
2
|
-
|
|
2
|
+
const NMT_CUMULATIVE_KEYS = [
|
|
3
|
+
'totalTime',
|
|
4
|
+
'totalTokens',
|
|
5
|
+
'decodeTime',
|
|
6
|
+
'encodeTime',
|
|
7
|
+
'TTFT'
|
|
8
|
+
];
|
|
9
|
+
// The addon's counters accumulate over a loaded model's life and its jobs are
|
|
10
|
+
// serialized per model, so the previous job's reading is this request's
|
|
11
|
+
// baseline. Keyed on the model instance: a reload gets fresh counters and a
|
|
12
|
+
// fresh baseline. `null` marks a baseline a batch has advanced past.
|
|
13
|
+
const lastCounters = new WeakMap();
|
|
14
|
+
// `runBatch` advances the counters without reporting stats, so the next single
|
|
15
|
+
// request cannot separate its own work from the batch's and reports none.
|
|
16
|
+
export function markNmtCountersStale(model) {
|
|
17
|
+
lastCounters.set(model, null);
|
|
18
|
+
}
|
|
19
|
+
// GGML beam search overwrites the token count with the job's own figure rather
|
|
20
|
+
// than adding to it; Bergamot and GGML greedy decoding accumulate it.
|
|
21
|
+
function tokensAccumulate(config) {
|
|
22
|
+
const { engine, beamsize } = (config ?? {});
|
|
23
|
+
return engine !== 'IndicTrans' || (beamsize ?? 4) <= 1;
|
|
24
|
+
}
|
|
25
|
+
function takeDeltas(stats, model, config) {
|
|
26
|
+
const previous = lastCounters.get(model);
|
|
27
|
+
const current = { ...previous };
|
|
28
|
+
const deltas = {};
|
|
29
|
+
const differenceTokens = tokensAccumulate(config);
|
|
30
|
+
for (const key of NMT_CUMULATIVE_KEYS) {
|
|
31
|
+
const value = stats[key];
|
|
32
|
+
if (typeof value !== 'number')
|
|
33
|
+
continue;
|
|
34
|
+
if (key === 'totalTokens' && !differenceTokens) {
|
|
35
|
+
deltas[key] = value;
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
current[key] = value;
|
|
39
|
+
const delta = value - (previous?.[key] ?? 0);
|
|
40
|
+
// A counter that went backwards was reset under us (e.g. by a cancel); the
|
|
41
|
+
// reading is then the request's own figure.
|
|
42
|
+
deltas[key] = delta < 0 ? value : delta;
|
|
43
|
+
}
|
|
44
|
+
lastCounters.set(model, current);
|
|
45
|
+
return previous === null ? undefined : deltas;
|
|
46
|
+
}
|
|
47
|
+
export function buildNmtTranslationStats(stats, model, config) {
|
|
48
|
+
if (!stats)
|
|
49
|
+
return {};
|
|
50
|
+
const deltas = takeDeltas(stats, model, config);
|
|
51
|
+
if (!deltas)
|
|
52
|
+
return {};
|
|
53
|
+
const { totalTime, totalTokens, decodeTime, encodeTime, TTFT } = deltas;
|
|
54
|
+
// The addon's own TPS divides the lifetime totals, which reads lowest right
|
|
55
|
+
// after load and climbs as the load cost amortises. Derive it from this
|
|
56
|
+
// request's figures instead, and omit it when they cannot carry it.
|
|
57
|
+
const tokensPerSecond = totalTokens !== undefined && totalTime !== undefined && totalTime > 0
|
|
58
|
+
? totalTokens / totalTime
|
|
59
|
+
: undefined;
|
|
3
60
|
return {
|
|
4
|
-
...(
|
|
5
|
-
...(
|
|
6
|
-
...(
|
|
7
|
-
...(
|
|
8
|
-
...(
|
|
9
|
-
...(
|
|
61
|
+
...(totalTime !== undefined && { totalTime: totalTime * NMT_SECONDS_TO_MS }),
|
|
62
|
+
...(totalTokens !== undefined && { totalTokens }),
|
|
63
|
+
...(decodeTime !== undefined && { decodeTime: decodeTime * NMT_SECONDS_TO_MS }),
|
|
64
|
+
...(encodeTime !== undefined && { encodeTime: encodeTime * NMT_SECONDS_TO_MS }),
|
|
65
|
+
...(tokensPerSecond !== undefined && { tokensPerSecond }),
|
|
66
|
+
...(TTFT !== undefined && { timeToFirstToken: TTFT })
|
|
10
67
|
};
|
|
11
68
|
}
|
|
@@ -55,7 +55,7 @@ import { translateServerParamsSchema, normalizeModelType, ModelType, AFRICAN_LAN
|
|
|
55
55
|
import { getLangName, detectOne } from '@qvac/langdetect-text';
|
|
56
56
|
import { nowMs } from '../../profiling/index.js';
|
|
57
57
|
import { buildStreamResult } from '../../profiling/model-execution.js';
|
|
58
|
-
import { buildNmtTranslationStats } from './translate-stats.js';
|
|
58
|
+
import { buildNmtTranslationStats, markNmtCountersStale } from './translate-stats.js';
|
|
59
59
|
import { ModelNotFoundError, ModelTypeMismatchError, TranslationFailedError } from '../../errors/index.js';
|
|
60
60
|
import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
|
|
61
61
|
import { isAddonContextOverflowError } from '../builtin/llamacpp-completion/ops/context-overflow.js';
|
|
@@ -203,7 +203,15 @@ export async function* translate(params, requestId) {
|
|
|
203
203
|
}
|
|
204
204
|
// Use runBatch for batch processing
|
|
205
205
|
const modelStart = nowMs();
|
|
206
|
-
|
|
206
|
+
let translations;
|
|
207
|
+
try {
|
|
208
|
+
translations = await model.runBatch(text);
|
|
209
|
+
}
|
|
210
|
+
finally {
|
|
211
|
+
// The batch advanced the addon's counters and reported no stats, so the
|
|
212
|
+
// next single request has no baseline to difference against.
|
|
213
|
+
markNmtCountersStale(model);
|
|
214
|
+
}
|
|
207
215
|
const modelExecutionMs = nowMs() - modelStart;
|
|
208
216
|
// Soft-cancel boundary: if `cancel({ requestId })` landed while
|
|
209
217
|
// the addon was running the batch, bail before yielding anything.
|
|
@@ -301,14 +309,22 @@ export async function* translate(params, requestId) {
|
|
|
301
309
|
return buildStreamResult(modelExecutionMs, stats);
|
|
302
310
|
}
|
|
303
311
|
const nmtResponse = response;
|
|
312
|
+
// Difference the counters the moment the job ends rather than after the
|
|
313
|
+
// consumer has drained the tokens, so a peer request on the same model that
|
|
314
|
+
// ends in between cannot move the baseline first.
|
|
315
|
+
const statsAtEnd = nmtResponse.await().then(() => buildNmtTranslationStats(nmtResponse.stats, model, entry.local.config), () => undefined);
|
|
304
316
|
for await (const token of nmtResponse.iterate()) {
|
|
305
317
|
if (ctx.signal.aborted)
|
|
306
318
|
break;
|
|
307
319
|
yield token;
|
|
308
320
|
}
|
|
309
321
|
const modelExecutionMs = nowMs() - modelStart;
|
|
310
|
-
|
|
311
|
-
|
|
322
|
+
// A soft-cancelled request drops its result; do not hold the generator open
|
|
323
|
+
// for a job that may still be running.
|
|
324
|
+
if (ctx.signal.aborted) {
|
|
325
|
+
return { modelExecutionMs };
|
|
326
|
+
}
|
|
327
|
+
return buildStreamResult(modelExecutionMs, await statsAtEnd);
|
|
312
328
|
}
|
|
313
329
|
catch (e_1) {
|
|
314
330
|
env_1.error = e_1;
|
|
@@ -165,6 +165,62 @@ registerOperationMetrics({
|
|
|
165
165
|
return Object.keys(gauges).length > 0 ? gauges : undefined;
|
|
166
166
|
}
|
|
167
167
|
});
|
|
168
|
+
// BCI runs on the same whisper.cpp decoder as the whisper ASR engine, so it
|
|
169
|
+
// reports that engine's stage timings plus its own window counters. The
|
|
170
|
+
// streaming variant gets no stats from the addon — only the timing.
|
|
171
|
+
function bciGauges(res) {
|
|
172
|
+
const gauges = {};
|
|
173
|
+
const modelExecMs = readModelExecutionMs(res);
|
|
174
|
+
if (modelExecMs !== undefined)
|
|
175
|
+
gauges['modelExecutionTime'] = modelExecMs;
|
|
176
|
+
if (res.stats?.tokensPerSecond !== undefined) {
|
|
177
|
+
gauges['tokensPerSecond'] = res.stats.tokensPerSecond;
|
|
178
|
+
}
|
|
179
|
+
if (res.stats?.totalTokens !== undefined)
|
|
180
|
+
gauges['totalTokens'] = res.stats.totalTokens;
|
|
181
|
+
if (res.stats?.totalSegments !== undefined)
|
|
182
|
+
gauges['totalSegments'] = res.stats.totalSegments;
|
|
183
|
+
// No totalTime gauge: the addon reports it in seconds while every other
|
|
184
|
+
// producer of this gauge uses ms. totalWallMs is the same measurement in ms.
|
|
185
|
+
if (res.stats?.totalWallMs !== undefined)
|
|
186
|
+
gauges['totalWallMs'] = res.stats.totalWallMs;
|
|
187
|
+
if (res.stats?.processCalls !== undefined)
|
|
188
|
+
gauges['processCalls'] = res.stats.processCalls;
|
|
189
|
+
if (res.stats?.whisperEncodeTime !== undefined) {
|
|
190
|
+
gauges['whisperEncodeTime'] = res.stats.whisperEncodeTime;
|
|
191
|
+
}
|
|
192
|
+
if (res.stats?.whisperDecodeTime !== undefined) {
|
|
193
|
+
gauges['whisperDecodeTime'] = res.stats.whisperDecodeTime;
|
|
194
|
+
}
|
|
195
|
+
if (res.stats?.whisperSampleMs !== undefined) {
|
|
196
|
+
gauges['whisperSampleMs'] = res.stats.whisperSampleMs;
|
|
197
|
+
}
|
|
198
|
+
if (res.stats?.whisperBatchdMs !== undefined) {
|
|
199
|
+
gauges['whisperBatchdMs'] = res.stats.whisperBatchdMs;
|
|
200
|
+
}
|
|
201
|
+
if (res.stats?.whisperPromptMs !== undefined) {
|
|
202
|
+
gauges['whisperPromptMs'] = res.stats.whisperPromptMs;
|
|
203
|
+
}
|
|
204
|
+
return gauges;
|
|
205
|
+
}
|
|
206
|
+
registerOperationMetrics({
|
|
207
|
+
op: 'bciTranscribe',
|
|
208
|
+
kind: 'handler',
|
|
209
|
+
getTags: (req) => (req.modelId ? { modelId: req.modelId } : {}),
|
|
210
|
+
fromFinalChunk: (res) => {
|
|
211
|
+
const gauges = bciGauges(res);
|
|
212
|
+
return Object.keys(gauges).length > 0 ? gauges : undefined;
|
|
213
|
+
}
|
|
214
|
+
});
|
|
215
|
+
registerOperationMetrics({
|
|
216
|
+
op: 'bciTranscribeStream',
|
|
217
|
+
kind: 'handler',
|
|
218
|
+
getTags: (req) => (req.modelId ? { modelId: req.modelId } : {}),
|
|
219
|
+
fromFinalChunk: (res) => {
|
|
220
|
+
const gauges = bciGauges(res);
|
|
221
|
+
return Object.keys(gauges).length > 0 ? gauges : undefined;
|
|
222
|
+
}
|
|
223
|
+
});
|
|
168
224
|
registerOperationMetrics({
|
|
169
225
|
op: 'textToSpeech',
|
|
170
226
|
kind: 'handler',
|
package/dist/registry.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { handleStartRpcServer, handleStopRpcServer, handleDiscoverRpcServers } from './handlers/rpc-server.js';
|
|
1
2
|
import { dispatchPluginReply, dispatchPluginStream } from './handlers/plugin-dispatch.js';
|
|
2
3
|
import { handleLoadModel } from './handlers/load-model/index.js';
|
|
3
4
|
import { handleUnloadModel } from './handlers/unload-model.js';
|
|
@@ -42,6 +43,9 @@ function finetuneSupportsProgress(request) {
|
|
|
42
43
|
return ['start', 'resume', undefined].includes(request.operation);
|
|
43
44
|
}
|
|
44
45
|
export const registry = {
|
|
46
|
+
startRpcServer: { type: 'reply', handler: handleStartRpcServer },
|
|
47
|
+
stopRpcServer: { type: 'reply', handler: handleStopRpcServer },
|
|
48
|
+
discoverRpcServers: { type: 'reply', handler: handleDiscoverRpcServers },
|
|
45
49
|
// Reply
|
|
46
50
|
heartbeat: {
|
|
47
51
|
type: 'reply',
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import type { AssessModelFitResult,
|
|
1
|
+
import type { AssessModelFitResult, ModelFitBasis, ModelFitEstimateTarget, ModelFitExecution, NativeProbeFit } from '../../schemas/assess-model-fit.js';
|
|
2
2
|
import type { SystemResources } from '../../schemas/system-resources.js';
|
|
3
3
|
import type { ModelResourceProfile } from '../../schemas/model-resource-profile.js';
|
|
4
4
|
import type { ModelFitPlatform, PlatformCalibration } from './types.js';
|
|
5
5
|
/** Resolves a checksum to its catalog resource profile. */
|
|
6
6
|
export type ProfileResolver = (sha256Checksum: string) => ModelResourceProfile | undefined;
|
|
7
7
|
export interface AssessModelFitOptions {
|
|
8
|
-
models: readonly
|
|
8
|
+
models: readonly ModelFitEstimateTarget[];
|
|
9
9
|
execution: ModelFitExecution;
|
|
10
10
|
resources: SystemResources;
|
|
11
11
|
/**
|
|
@@ -33,6 +33,8 @@ export interface AssessModelFitOptions {
|
|
|
33
33
|
* a combined budget the way two byte estimates can.
|
|
34
34
|
*/
|
|
35
35
|
nativeFit?: NativeProbeFit | undefined;
|
|
36
|
+
/** Why the fitter produced no verdict, when one was sought. */
|
|
37
|
+
nativeFitUnavailable?: string | undefined;
|
|
36
38
|
}
|
|
37
39
|
/**
|
|
38
40
|
* Turns catalog profiles plus a fresh system-memory sample into an advisory
|
|
@@ -43,3 +45,36 @@ export interface AssessModelFitOptions {
|
|
|
43
45
|
* happen in the handler.
|
|
44
46
|
*/
|
|
45
47
|
export declare function assessModelFitFromResources(options: AssessModelFitOptions): AssessModelFitResult;
|
|
48
|
+
/**
|
|
49
|
+
* Whether every allocation a load makes comes out of the memory the system (or
|
|
50
|
+
* process) budget measures — the precondition for the computed floor to be
|
|
51
|
+
* compared against that budget.
|
|
52
|
+
*
|
|
53
|
+
* True on unified-memory platforms whatever the collector reports, and
|
|
54
|
+
* elsewhere when no usable GPU is present or every usable GPU allocates out of
|
|
55
|
+
* system RAM. A card with its own memory, or a device whose class cannot be
|
|
56
|
+
* told (an AMD APU on linux), fails it: the weights may live where the budget
|
|
57
|
+
* cannot see them.
|
|
58
|
+
*/
|
|
59
|
+
export declare function boundBySystemMemory(resources: SystemResources, platform: ModelFitPlatform | undefined): boolean;
|
|
60
|
+
/**
|
|
61
|
+
* Picks the budget basis for a platform.
|
|
62
|
+
*
|
|
63
|
+
* iOS is the exception: jetsam terminates an app on its own footprint against
|
|
64
|
+
* a per-process limit well below device RAM, so a system-memory budget there
|
|
65
|
+
* would defend `likely-fits` verdicts the OS does not honor. Android's
|
|
66
|
+
* low-memory killer acts system-wide and native allocations carry no
|
|
67
|
+
* per-process cap, so it deliberately keeps the system basis with the mobile
|
|
68
|
+
* reserve.
|
|
69
|
+
*/
|
|
70
|
+
export declare function resolveBasis(platform: ModelFitPlatform | undefined): ModelFitBasis;
|
|
71
|
+
/**
|
|
72
|
+
* Derives the memory budget from the sample, under the platform's basis.
|
|
73
|
+
*
|
|
74
|
+
* Only `sample.memory` is used: capabilities-only totals say nothing about what
|
|
75
|
+
* is free right now, and a verdict without that is not worth giving. Under the
|
|
76
|
+
* process basis the ceiling is reconstructed as allowance + footprint — the
|
|
77
|
+
* relation the OS enforces — so every budget field keeps the same meaning
|
|
78
|
+
* under either basis.
|
|
79
|
+
*/
|
|
80
|
+
export declare function resolveBudget(resources: SystemResources, platform: ModelFitPlatform | undefined, basis: ModelFitBasis, reasons: string[]): AssessModelFitResult['budget'];
|
|
@@ -185,6 +185,10 @@ export function assessModelFitFromResources(options) {
|
|
|
185
185
|
assumptions: []
|
|
186
186
|
};
|
|
187
187
|
}
|
|
188
|
+
const declined = nativeFitDecline(options.nativeFit, options.nativeFitUnavailable);
|
|
189
|
+
if (declined !== undefined) {
|
|
190
|
+
reasons.push(declined);
|
|
191
|
+
}
|
|
188
192
|
return {
|
|
189
193
|
verdict,
|
|
190
194
|
basis,
|
|
@@ -200,6 +204,19 @@ export function assessModelFitFromResources(options) {
|
|
|
200
204
|
assumptions
|
|
201
205
|
};
|
|
202
206
|
}
|
|
207
|
+
/**
|
|
208
|
+
* Why no engine verdict backs this assessment, for the cases where one was
|
|
209
|
+
* sought: the fitter never ran, or it ran and declined. `undefined` where none
|
|
210
|
+
* was sought, which is every multi-candidate assessment.
|
|
211
|
+
*/
|
|
212
|
+
function nativeFitDecline(nativeFit, unavailable) {
|
|
213
|
+
if (unavailable !== undefined)
|
|
214
|
+
return `no engine fit: ${unavailable}`;
|
|
215
|
+
if (nativeFit?.verdict !== 'unknown')
|
|
216
|
+
return undefined;
|
|
217
|
+
const detail = nativeFit.message === undefined ? nativeFit.reason : nativeFit.message;
|
|
218
|
+
return `the engine fitter reached no verdict: ${detail}`;
|
|
219
|
+
}
|
|
203
220
|
/**
|
|
204
221
|
* The per-model result a native fit produces, or `undefined` when the modelled
|
|
205
222
|
* one stands.
|
|
@@ -222,6 +239,7 @@ function nativeModelResult(nativeFit, modelResults) {
|
|
|
222
239
|
name: modelled.name,
|
|
223
240
|
verdict: nativeFit.verdict === 'fit' ? 'likely-fits' : 'likely-too-large',
|
|
224
241
|
evidence: 'native-fit',
|
|
242
|
+
...(modelled.device !== undefined && { device: modelled.device }),
|
|
225
243
|
estimatorVersion: nativeFit.estimatorVersion,
|
|
226
244
|
reasons: nativeFit.message === undefined ? [nativeFit.reason] : [nativeFit.reason, nativeFit.message]
|
|
227
245
|
};
|
|
@@ -237,7 +255,7 @@ function nativeModelResult(nativeFit, modelResults) {
|
|
|
237
255
|
* told (an AMD APU on linux), fails it: the weights may live where the budget
|
|
238
256
|
* cannot see them.
|
|
239
257
|
*/
|
|
240
|
-
function boundBySystemMemory(resources, platform) {
|
|
258
|
+
export function boundBySystemMemory(resources, platform) {
|
|
241
259
|
if (platform !== undefined && UNIFIED_MEMORY_PLATFORMS.includes(platform))
|
|
242
260
|
return true;
|
|
243
261
|
const gpus = usableGpus(resources);
|
|
@@ -530,7 +548,7 @@ function worst(a, b) {
|
|
|
530
548
|
* per-process cap, so it deliberately keeps the system basis with the mobile
|
|
531
549
|
* reserve.
|
|
532
550
|
*/
|
|
533
|
-
function resolveBasis(platform) {
|
|
551
|
+
export function resolveBasis(platform) {
|
|
534
552
|
return platform === 'ios-arm64' ? 'process-memory' : 'system-memory';
|
|
535
553
|
}
|
|
536
554
|
function basisEvidence(basis) {
|
|
@@ -551,7 +569,7 @@ function basisEvidence(basis) {
|
|
|
551
569
|
* relation the OS enforces — so every budget field keeps the same meaning
|
|
552
570
|
* under either basis.
|
|
553
571
|
*/
|
|
554
|
-
function resolveBudget(resources, platform, basis, reasons) {
|
|
572
|
+
export function resolveBudget(resources, platform, basis, reasons) {
|
|
555
573
|
const sample = resources.sample;
|
|
556
574
|
if (!sample) {
|
|
557
575
|
reasons.push('no memory sample was available');
|
|
@@ -684,6 +702,7 @@ deviceBudgets, alsoBoundBy) {
|
|
|
684
702
|
return {
|
|
685
703
|
name: candidate.model.name,
|
|
686
704
|
verdict: 'unknown',
|
|
705
|
+
...(candidate.device !== undefined && { device: candidate.device }),
|
|
687
706
|
reasons: [...result.reasons]
|
|
688
707
|
};
|
|
689
708
|
}
|
|
@@ -693,6 +712,7 @@ deviceBudgets, alsoBoundBy) {
|
|
|
693
712
|
name: candidate.model.name,
|
|
694
713
|
verdict,
|
|
695
714
|
evidence: 'computed-only',
|
|
715
|
+
...(candidate.device !== undefined && { device: candidate.device }),
|
|
696
716
|
floorBytes: result.bytes,
|
|
697
717
|
estimatorVersion: FLOOR_VERSION,
|
|
698
718
|
reasons: [
|
|
@@ -713,6 +733,7 @@ deviceBudgets, alsoBoundBy) {
|
|
|
713
733
|
name: candidate.model.name,
|
|
714
734
|
verdict: budget ? verdictAgainst(total, deviceBudgets ?? [budget], alsoBoundBy) : 'unknown',
|
|
715
735
|
evidence: 'calibration',
|
|
736
|
+
...(candidate.device !== undefined && { device: candidate.device }),
|
|
716
737
|
estimate: { lowerBoundBytes: total.lower, upperBoundBytes: total.upper },
|
|
717
738
|
estimatorVersion: result.estimatorVersion,
|
|
718
739
|
reasons: budget
|
|
@@ -59,7 +59,7 @@ export function estimateLlm(input) {
|
|
|
59
59
|
assumptions.push('companion artifacts passed in `artifacts` are counted at full size');
|
|
60
60
|
}
|
|
61
61
|
// Context: never more than the model was trained for — llama.cpp clamps.
|
|
62
|
-
let contextTokens = workload.contextTokens;
|
|
62
|
+
let contextTokens = workload.contextTokens ?? facts.contextLength;
|
|
63
63
|
if (contextTokens > facts.contextLength) {
|
|
64
64
|
assumptions.push(`requested ${workload.contextTokens} tokens exceeds the trained context of ${facts.contextLength}; clamped to the trained context`);
|
|
65
65
|
contextTokens = facts.contextLength;
|
|
@@ -85,7 +85,7 @@ export function estimateLlm(input) {
|
|
|
85
85
|
if (calibration.workingPeakBytes && calibration.workingPeakBytes.upper > 0) {
|
|
86
86
|
assumptions.push('one operation is assumed in flight per model; the working peak is what a single completion was measured to add on top of the resident cost');
|
|
87
87
|
}
|
|
88
|
-
assumptions.push('default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not
|
|
88
|
+
assumptions.push('default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not read here and would change these numbers');
|
|
89
89
|
return {
|
|
90
90
|
kind: 'estimate',
|
|
91
91
|
estimatorVersion: LLM_ESTIMATOR_VERSION,
|
|
@@ -50,18 +50,17 @@ export function estimateWhisper(input) {
|
|
|
50
50
|
assumptions.push('companion artifacts passed in `artifacts` (e.g. a VAD model) are counted at full size');
|
|
51
51
|
}
|
|
52
52
|
const windows = workload.windowMs / CALIBRATION_WINDOW_MS;
|
|
53
|
-
const batch = workload.batch ?? 1;
|
|
54
53
|
const streamingLower = workload.streaming ? calibration.audioStreamingBytes.lower : 0;
|
|
55
54
|
const streamingUpper = workload.streaming ? calibration.audioStreamingBytes.upper : 0;
|
|
56
55
|
const working = {
|
|
57
56
|
lower: calibration.fixedOverheadBytes.lower +
|
|
58
|
-
calibration.audioWindowBytes.lower * windows
|
|
57
|
+
calibration.audioWindowBytes.lower * windows +
|
|
59
58
|
streamingLower,
|
|
60
59
|
upper: calibration.fixedOverheadBytes.upper +
|
|
61
|
-
calibration.audioWindowBytes.upper * windows
|
|
60
|
+
calibration.audioWindowBytes.upper * windows +
|
|
62
61
|
streamingUpper
|
|
63
62
|
};
|
|
64
|
-
reasons.push(`working memory scaled from the calibrated 30 s window to ${Math.round(workload.windowMs)} ms
|
|
63
|
+
reasons.push(`working memory scaled from the calibrated 30 s window to ${Math.round(workload.windowMs)} ms`);
|
|
65
64
|
assumptions.push('the audio window scales working memory linearly from the engine’s 30 s default; longer windows are chunked by the engine rather than held whole');
|
|
66
65
|
if (workload.streaming) {
|
|
67
66
|
assumptions.push('a streaming session is assumed to keep one calibrated stream buffer resident');
|
|
@@ -15,6 +15,13 @@ export interface FitStubRef {
|
|
|
15
15
|
registryPath?: string | undefined;
|
|
16
16
|
registrySource?: string | undefined;
|
|
17
17
|
}
|
|
18
|
+
export interface FitStubSetMember extends FitStubRef {
|
|
19
|
+
targetName: string;
|
|
20
|
+
}
|
|
21
|
+
export interface FitStubSet {
|
|
22
|
+
primary: FitStubSetMember;
|
|
23
|
+
others: readonly FitStubSetMember[];
|
|
24
|
+
}
|
|
18
25
|
export type FitStubUnavailableReason =
|
|
19
26
|
/** The ref carries no registry coordinates to look up. */
|
|
20
27
|
'unresolvable-ref'
|
|
@@ -61,8 +68,13 @@ export interface FitStubOptions {
|
|
|
61
68
|
* capped. Work still in flight at expiry is abandoned: the download is
|
|
62
69
|
* aborted, and a lookup that cannot be cancelled settles on its own with
|
|
63
70
|
* nothing waiting for it.
|
|
71
|
+
*
|
|
72
|
+
* One budget covers a whole companion set, so it holds the slowest case: every
|
|
73
|
+
* member of a multi-stage voice model staged over a mobile network. It sits
|
|
74
|
+
* above the client's per-attempt timeout rather than on it, so an attempt that
|
|
75
|
+
* is about to answer is not cut at the moment it would have.
|
|
64
76
|
*/
|
|
65
|
-
export declare const FIT_STUB_BUDGET_MS =
|
|
77
|
+
export declare const FIT_STUB_BUDGET_MS = 40000;
|
|
66
78
|
/** Removes a fetched stub and the directory it was staged in. Never throws. */
|
|
67
79
|
export declare function removeStub(stubPath: string): Promise<void>;
|
|
68
80
|
/**
|
|
@@ -81,3 +93,10 @@ export declare function removeStub(stubPath: string): Promise<void>;
|
|
|
81
93
|
* llama's shard naming to read any of it — assembling that is not done here.
|
|
82
94
|
*/
|
|
83
95
|
export declare function fetchFitStub(ref: FitStubRef, options?: FitStubOptions): Promise<FitStubOutcome>;
|
|
96
|
+
/**
|
|
97
|
+
* Stages a companion set under its target names in one directory, the layout a
|
|
98
|
+
* downloaded set has, for engines that find their companions beside the
|
|
99
|
+
* checkpoint. The primary decides the outcome; a member the registry holds no
|
|
100
|
+
* description for is skipped.
|
|
101
|
+
*/
|
|
102
|
+
export declare function fetchFitStubSet(set: FitStubSet, options?: FitStubOptions): Promise<FitStubOutcome>;
|