@qvac/inference 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +107 -33
- package/README.md +8 -5
- package/dist/api/assess-model-fit.d.ts +9 -4
- package/dist/api/assess-model-fit.js +9 -4
- package/dist/api/bci-transcribe.d.ts +4 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/rpc-server.d.ts +19 -0
- package/dist/api/rpc-server.js +44 -0
- package/dist/api/text-to-speech.d.ts +9 -0
- package/dist/api/text-to-speech.js +9 -0
- package/dist/api/transcribe.d.ts +11 -7
- package/dist/api/transcribe.js +47 -15
- package/dist/api/video-request.d.ts +5 -5
- package/dist/dispatch.js +39 -7
- package/dist/errors/index.d.ts +7 -0
- package/dist/errors/index.js +15 -0
- package/dist/handlers/assess-model-fit.d.ts +18 -6
- package/dist/handlers/assess-model-fit.js +215 -35
- package/dist/handlers/completion-orchestrate.js +17 -0
- package/dist/handlers/get-loaded-model-info.js +4 -1
- package/dist/handlers/get-model-info.js +29 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
- package/dist/handlers/load-model/coreml-sidecars.js +274 -0
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
- package/dist/handlers/load-model/registry-download-utils.js +2 -1
- package/dist/handlers/load-model/registry.js +59 -0
- package/dist/handlers/rpc-server.d.ts +23 -0
- package/dist/handlers/rpc-server.js +141 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/models/registry/models.d.ts +7731 -5672
- package/dist/models/registry/models.js +4652 -2526
- package/dist/models/registry/resource-profiles.js +481 -94
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +80 -8
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
- package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
- package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
- package/dist/plugins/ops/bci-transcribe.js +18 -13
- package/dist/plugins/ops/load-model.js +2 -2
- package/dist/plugins/ops/transcribe.d.ts +4 -2
- package/dist/plugins/ops/transcribe.js +39 -2
- package/dist/plugins/ops/translate-stats.d.ts +2 -1
- package/dist/plugins/ops/translate-stats.js +64 -7
- package/dist/plugins/ops/translate.js +20 -4
- package/dist/profiling/operation-metrics.js +56 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +46 -2
- package/dist/resources/model-fit/assess.js +67 -3
- package/dist/resources/model-fit/estimators/llm.js +2 -2
- package/dist/resources/model-fit/estimators/whisper.js +3 -4
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +102 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +212 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
- package/dist/resources/model-fit/floor.js +2 -2
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
- package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
- package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
- package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
- package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
- package/dist/resources/model-fit/platform.d.ts +9 -0
- package/dist/resources/model-fit/platform.js +28 -0
- package/dist/rpc/device-map.d.ts +13 -0
- package/dist/rpc/device-map.js +22 -0
- package/dist/rpc/discovery.d.ts +12 -0
- package/dist/rpc/discovery.js +225 -0
- package/dist/rpc/ggml-provider.d.ts +4 -0
- package/dist/rpc/ggml-provider.js +25 -0
- package/dist/rpc/index.d.ts +2 -0
- package/dist/rpc/index.js +1 -0
- package/dist/rpc/instance.d.ts +6 -0
- package/dist/rpc/instance.js +21 -0
- package/dist/rpc/inventory.d.ts +5 -0
- package/dist/rpc/inventory.js +144 -0
- package/dist/rpc/manager.d.ts +14 -0
- package/dist/rpc/manager.js +174 -0
- package/dist/rpc/network.d.ts +11 -0
- package/dist/rpc/network.js +85 -0
- package/dist/rpc/provider.d.ts +6 -0
- package/dist/rpc/provider.js +24 -0
- package/dist/runtime/lifecycle.js +31 -13
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +284 -77
- package/dist/schemas/assess-model-fit.js +93 -55
- package/dist/schemas/audio-gen.d.ts +6 -6
- package/dist/schemas/audio-gen.js +2 -2
- package/dist/schemas/batch-completion-stream.d.ts +13 -5
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci-config.js +1 -1
- package/dist/schemas/bci.d.ts +86 -1
- package/dist/schemas/bci.js +21 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +765 -56
- package/dist/schemas/common.js +7 -0
- package/dist/schemas/completion-event.d.ts +6 -6
- package/dist/schemas/completion-stream.d.ts +27 -8
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +236 -30
- package/dist/schemas/config.js +10 -0
- package/dist/schemas/embed.d.ts +2 -2
- package/dist/schemas/error.js +13 -0
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +105 -14
- package/dist/schemas/llamacpp-config.js +113 -3
- package/dist/schemas/load-model.d.ts +3102 -627
- package/dist/schemas/load-model.js +2 -2
- package/dist/schemas/mcp-adapter.d.ts +4 -0
- package/dist/schemas/model-config-schemas.d.ts +205 -5
- package/dist/schemas/model-info.d.ts +56 -0
- package/dist/schemas/model-info.js +5 -1
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/reload-config.d.ts +30 -0
- package/dist/schemas/rpc-server.d.ts +101 -0
- package/dist/schemas/rpc-server.js +87 -0
- package/dist/schemas/sdcpp-config.d.ts +7 -3
- package/dist/schemas/sdcpp-config.js +37 -2
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +529 -1
- package/dist/schemas/text-to-speech.js +122 -12
- package/dist/schemas/tools.d.ts +2 -0
- package/dist/schemas/tools.js +8 -0
- package/dist/schemas/transcription-config.d.ts +5 -0
- package/dist/schemas/transcription-config.js +20 -2
- package/dist/schemas/transcription.d.ts +164 -0
- package/dist/schemas/transcription.js +71 -4
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/surface.d.ts +4 -1
- package/dist/surface.js +3 -1
- package/dist/utils/addon-responses.d.ts +5 -1
- package/dist/utils/archive.js +5 -1
- package/dist/utils/asr-diagnostics.d.ts +12 -0
- package/dist/utils/asr-diagnostics.js +42 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +3 -2
- package/dist/utils/cache/paths.js +3 -2
- package/dist/utils/mcp-adapter.js +7 -1
- package/dist/utils/multi-gpu-mobile.d.ts +1 -1
- package/dist/utils/multi-gpu-mobile.js +8 -6
- package/dist/utils/path-security.d.ts +5 -0
- package/dist/utils/path-security.js +14 -0
- package/dist/utils/tool-helpers.d.ts +4 -0
- package/dist/utils/tool-helpers.js +2 -0
- package/dist/utils/tools/defer.d.ts +93 -0
- package/dist/utils/tools/defer.js +290 -0
- package/dist/utils/tools/index.d.ts +1 -0
- package/dist/utils/tools/index.js +1 -0
- package/dist/utils/transcribe-metadata.d.ts +21 -7
- package/dist/utils/transcribe-metadata.js +24 -8
- package/dist/utils/tts-stats.d.ts +6 -0
- package/dist/utils/tts-stats.js +14 -1
- package/package.json +44 -30
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -171
|
@@ -56,6 +56,7 @@ import { getEngineLogger } from '../../logging/index.js';
|
|
|
56
56
|
import { TranscriptionFailedError } from '../../errors/index.js';
|
|
57
57
|
import { nowMs } from '../../profiling/index.js';
|
|
58
58
|
import { buildStreamResult } from '../../profiling/model-execution.js';
|
|
59
|
+
import { buildAsrBackendDiagnostics } from '../../utils/asr-diagnostics.js';
|
|
59
60
|
import { toTranscribeSegment } from '../../utils/transcribe-metadata.js';
|
|
60
61
|
import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
|
|
61
62
|
import { generateRandomRequestId } from '../../runtime/request-id.js';
|
|
@@ -123,12 +124,6 @@ export async function* bciTranscribe(params, requestId) {
|
|
|
123
124
|
}
|
|
124
125
|
const modelExecutionMs = nowMs() - modelStart;
|
|
125
126
|
const stats = {
|
|
126
|
-
...(response.stats?.audioDurationMs !== undefined && {
|
|
127
|
-
audioDuration: response.stats.audioDurationMs
|
|
128
|
-
}),
|
|
129
|
-
...(response.stats?.realTimeFactor !== undefined && {
|
|
130
|
-
realTimeFactor: response.stats.realTimeFactor
|
|
131
|
-
}),
|
|
132
127
|
...(response.stats?.tokensPerSecond !== undefined && {
|
|
133
128
|
tokensPerSecond: response.stats.tokensPerSecond
|
|
134
129
|
}),
|
|
@@ -144,14 +139,18 @@ export async function* bciTranscribe(params, requestId) {
|
|
|
144
139
|
...(response.stats?.whisperDecodeMs !== undefined && {
|
|
145
140
|
whisperDecodeTime: response.stats.whisperDecodeMs
|
|
146
141
|
}),
|
|
147
|
-
...(response.stats?.
|
|
148
|
-
|
|
142
|
+
...(response.stats?.whisperSampleMs !== undefined && {
|
|
143
|
+
whisperSampleMs: response.stats.whisperSampleMs
|
|
144
|
+
}),
|
|
145
|
+
...(response.stats?.whisperBatchdMs !== undefined && {
|
|
146
|
+
whisperBatchdMs: response.stats.whisperBatchdMs
|
|
149
147
|
}),
|
|
150
|
-
...(response.stats?.
|
|
151
|
-
|
|
148
|
+
...(response.stats?.whisperPromptMs !== undefined && {
|
|
149
|
+
whisperPromptMs: response.stats.whisperPromptMs
|
|
152
150
|
}),
|
|
153
|
-
...(response.stats?.
|
|
154
|
-
|
|
151
|
+
...(response.stats?.totalWallMs !== undefined && { totalWallMs: response.stats.totalWallMs }),
|
|
152
|
+
...(response.stats?.processCalls !== undefined && {
|
|
153
|
+
processCalls: response.stats.processCalls
|
|
155
154
|
}),
|
|
156
155
|
...(response.stats?.backendDevice !== undefined && {
|
|
157
156
|
backendDevice: response.stats.backendDevice
|
|
@@ -166,7 +165,11 @@ export async function* bciTranscribe(params, requestId) {
|
|
|
166
165
|
gpuMemFreeMb: response.stats.gpuMemFreeMb
|
|
167
166
|
})
|
|
168
167
|
};
|
|
169
|
-
|
|
168
|
+
const diagnostics = buildAsrBackendDiagnostics(stats);
|
|
169
|
+
return {
|
|
170
|
+
...buildStreamResult(modelExecutionMs, stats),
|
|
171
|
+
...(diagnostics && { diagnostics })
|
|
172
|
+
};
|
|
170
173
|
}
|
|
171
174
|
catch (e_1) {
|
|
172
175
|
env_1.error = e_1;
|
|
@@ -216,6 +219,7 @@ export async function* bciTranscribeStream(modelId, neuralStream, metadata, opts
|
|
|
216
219
|
...(opts?.hopTimesteps !== undefined && { hopTimesteps: opts.hopTimesteps }),
|
|
217
220
|
...(opts?.emit !== undefined && { emit: opts.emit })
|
|
218
221
|
};
|
|
222
|
+
const modelStart = nowMs();
|
|
219
223
|
const response = await model.transcribeStream(neuralStream, streamOpts);
|
|
220
224
|
for await (const output of response.iterate()) {
|
|
221
225
|
if (ctx.signal.aborted)
|
|
@@ -235,6 +239,7 @@ export async function* bciTranscribeStream(modelId, neuralStream, metadata, opts
|
|
|
235
239
|
yield text;
|
|
236
240
|
}
|
|
237
241
|
}
|
|
242
|
+
return buildStreamResult(nowMs() - modelStart);
|
|
238
243
|
}
|
|
239
244
|
catch (e_2) {
|
|
240
245
|
env_2.error = e_2;
|
|
@@ -57,8 +57,8 @@ export async function loadModel(params, options) {
|
|
|
57
57
|
// to the ordinary load below. Runs after config resolution and path
|
|
58
58
|
// validation so it sees the same state the real load uses, and before
|
|
59
59
|
// `createModel()` so it never competes with the native load for device
|
|
60
|
-
// memory. The outcome is stored on the registry entry
|
|
61
|
-
//
|
|
60
|
+
// memory. The outcome is stored on the registry entry and returned by
|
|
61
|
+
// `getLoadedModelInfo` as `fitProbe`.
|
|
62
62
|
const fitProbe = await runAdvisoryFitCheck({
|
|
63
63
|
modelId,
|
|
64
64
|
modelType: modelType,
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { type TranscribeParams, type TranscribeSegment, type TranscribeStats, type TranscribeStreamEvent, type ParakeetStreamingRunConfig } from '../../schemas/index.js';
|
|
2
2
|
import type Buffer from 'bare-buffer';
|
|
3
|
-
import {
|
|
4
|
-
|
|
3
|
+
import type { InferenceBackendDiagnostics } from '../../schemas/index.js';
|
|
4
|
+
import { assertMetadataSupported, toTranscribeSegment, type AsrAddonSegment } from '../../utils/transcribe-metadata.js';
|
|
5
|
+
export { assertMetadataSupported, toTranscribeSegment, type AsrAddonSegment };
|
|
5
6
|
type TranscribeReturn = {
|
|
6
7
|
modelExecutionMs: number;
|
|
7
8
|
stats?: TranscribeStats;
|
|
9
|
+
diagnostics?: InferenceBackendDiagnostics;
|
|
8
10
|
};
|
|
9
11
|
export declare function transcribe(params: TranscribeParams & {
|
|
10
12
|
metadata: true;
|
|
@@ -57,6 +57,7 @@ import { getEngineLogger } from '../../logging/index.js';
|
|
|
57
57
|
import { TranscriptionFailedError } from '../../errors/index.js';
|
|
58
58
|
import { nowMs } from '../../profiling/index.js';
|
|
59
59
|
import { buildStreamResult } from '../../profiling/model-execution.js';
|
|
60
|
+
import { buildAsrBackendDiagnostics } from '../../utils/asr-diagnostics.js';
|
|
60
61
|
import { assertMetadataSupported, toTranscribeSegment } from '../../utils/transcribe-metadata.js';
|
|
61
62
|
import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
|
|
62
63
|
import { generateRandomRequestId } from '../../runtime/request-id.js';
|
|
@@ -146,6 +147,32 @@ function mapTranscribeStats(stats) {
|
|
|
146
147
|
}),
|
|
147
148
|
...(stats?.gpuMemFreeMb !== undefined && {
|
|
148
149
|
gpuMemFreeMb: stats.gpuMemFreeMb
|
|
150
|
+
}),
|
|
151
|
+
...(stats?.totalTime !== undefined && { totalTime: stats.totalTime }),
|
|
152
|
+
...(stats?.totalWallMs !== undefined && { totalWallMs: stats.totalWallMs }),
|
|
153
|
+
...(stats?.totalSamples !== undefined && { totalSamples: stats.totalSamples }),
|
|
154
|
+
...(stats?.processCalls !== undefined && { processCalls: stats.processCalls }),
|
|
155
|
+
...(stats?.whisperSampleMs !== undefined && {
|
|
156
|
+
whisperSampleMs: stats.whisperSampleMs
|
|
157
|
+
}),
|
|
158
|
+
...(stats?.whisperBatchdMs !== undefined && {
|
|
159
|
+
whisperBatchdMs: stats.whisperBatchdMs
|
|
160
|
+
}),
|
|
161
|
+
...(stats?.whisperPromptMs !== undefined && {
|
|
162
|
+
whisperPromptMs: stats.whisperPromptMs
|
|
163
|
+
}),
|
|
164
|
+
...(stats?.totalTranscriptions !== undefined && {
|
|
165
|
+
totalTranscriptions: stats.totalTranscriptions
|
|
166
|
+
}),
|
|
167
|
+
...(stats?.modelLoadMs !== undefined && { modelLoadMs: stats.modelLoadMs }),
|
|
168
|
+
...(stats?.totalEncodedFrames !== undefined && {
|
|
169
|
+
totalEncodedFrames: stats.totalEncodedFrames
|
|
170
|
+
}),
|
|
171
|
+
...(stats?.encoderOnCoreml !== undefined && {
|
|
172
|
+
encoderOnCoreml: stats.encoderOnCoreml
|
|
173
|
+
}),
|
|
174
|
+
...(stats?.encoderUsedCoreml !== undefined && {
|
|
175
|
+
encoderUsedCoreml: stats.encoderUsedCoreml
|
|
149
176
|
})
|
|
150
177
|
};
|
|
151
178
|
}
|
|
@@ -222,7 +249,12 @@ export async function* transcribe(params, requestId) {
|
|
|
222
249
|
}
|
|
223
250
|
}
|
|
224
251
|
const modelExecutionMs = nowMs() - modelStart;
|
|
225
|
-
|
|
252
|
+
const mapped = mapTranscribeStats(response.stats);
|
|
253
|
+
const diagnostics = buildAsrBackendDiagnostics(mapped);
|
|
254
|
+
return {
|
|
255
|
+
...buildStreamResult(modelExecutionMs, mapped),
|
|
256
|
+
...(diagnostics && { diagnostics })
|
|
257
|
+
};
|
|
226
258
|
}
|
|
227
259
|
catch (e_1) {
|
|
228
260
|
env_1.error = e_1;
|
|
@@ -321,7 +353,12 @@ export async function* transcribeStream(modelId, audioInputStream, prompt, metad
|
|
|
321
353
|
}
|
|
322
354
|
await response.await();
|
|
323
355
|
const modelExecutionMs = nowMs() - modelStart;
|
|
324
|
-
|
|
356
|
+
const mapped = mapTranscribeStats(response.stats);
|
|
357
|
+
const diagnostics = buildAsrBackendDiagnostics(mapped);
|
|
358
|
+
return {
|
|
359
|
+
...buildStreamResult(modelExecutionMs, mapped),
|
|
360
|
+
...(diagnostics && { diagnostics })
|
|
361
|
+
};
|
|
325
362
|
}
|
|
326
363
|
catch (e_2) {
|
|
327
364
|
env_2.error = e_2;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { TranslationStats } from '../../schemas/index.js';
|
|
2
2
|
import type { NmtStats } from '../../utils/addon-responses.js';
|
|
3
3
|
export declare const NMT_SECONDS_TO_MS = 1000;
|
|
4
|
-
export declare function
|
|
4
|
+
export declare function markNmtCountersStale(model: object): void;
|
|
5
|
+
export declare function buildNmtTranslationStats(stats: Partial<NmtStats> | undefined, model: object, config?: unknown): TranslationStats;
|
|
@@ -1,11 +1,68 @@
|
|
|
1
1
|
export const NMT_SECONDS_TO_MS = 1000;
|
|
2
|
-
|
|
2
|
+
const NMT_CUMULATIVE_KEYS = [
|
|
3
|
+
'totalTime',
|
|
4
|
+
'totalTokens',
|
|
5
|
+
'decodeTime',
|
|
6
|
+
'encodeTime',
|
|
7
|
+
'TTFT'
|
|
8
|
+
];
|
|
9
|
+
// The addon's counters accumulate over a loaded model's life and its jobs are
|
|
10
|
+
// serialized per model, so the previous job's reading is this request's
|
|
11
|
+
// baseline. Keyed on the model instance: a reload gets fresh counters and a
|
|
12
|
+
// fresh baseline. `null` marks a baseline a batch has advanced past.
|
|
13
|
+
const lastCounters = new WeakMap();
|
|
14
|
+
// `runBatch` advances the counters without reporting stats, so the next single
|
|
15
|
+
// request cannot separate its own work from the batch's and reports none.
|
|
16
|
+
export function markNmtCountersStale(model) {
|
|
17
|
+
lastCounters.set(model, null);
|
|
18
|
+
}
|
|
19
|
+
// GGML beam search overwrites the token count with the job's own figure rather
|
|
20
|
+
// than adding to it; Bergamot and GGML greedy decoding accumulate it.
|
|
21
|
+
function tokensAccumulate(config) {
|
|
22
|
+
const { engine, beamsize } = (config ?? {});
|
|
23
|
+
return engine !== 'IndicTrans' || (beamsize ?? 4) <= 1;
|
|
24
|
+
}
|
|
25
|
+
function takeDeltas(stats, model, config) {
|
|
26
|
+
const previous = lastCounters.get(model);
|
|
27
|
+
const current = { ...previous };
|
|
28
|
+
const deltas = {};
|
|
29
|
+
const differenceTokens = tokensAccumulate(config);
|
|
30
|
+
for (const key of NMT_CUMULATIVE_KEYS) {
|
|
31
|
+
const value = stats[key];
|
|
32
|
+
if (typeof value !== 'number')
|
|
33
|
+
continue;
|
|
34
|
+
if (key === 'totalTokens' && !differenceTokens) {
|
|
35
|
+
deltas[key] = value;
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
current[key] = value;
|
|
39
|
+
const delta = value - (previous?.[key] ?? 0);
|
|
40
|
+
// A counter that went backwards was reset under us (e.g. by a cancel); the
|
|
41
|
+
// reading is then the request's own figure.
|
|
42
|
+
deltas[key] = delta < 0 ? value : delta;
|
|
43
|
+
}
|
|
44
|
+
lastCounters.set(model, current);
|
|
45
|
+
return previous === null ? undefined : deltas;
|
|
46
|
+
}
|
|
47
|
+
export function buildNmtTranslationStats(stats, model, config) {
|
|
48
|
+
if (!stats)
|
|
49
|
+
return {};
|
|
50
|
+
const deltas = takeDeltas(stats, model, config);
|
|
51
|
+
if (!deltas)
|
|
52
|
+
return {};
|
|
53
|
+
const { totalTime, totalTokens, decodeTime, encodeTime, TTFT } = deltas;
|
|
54
|
+
// The addon's own TPS divides the lifetime totals, which reads lowest right
|
|
55
|
+
// after load and climbs as the load cost amortises. Derive it from this
|
|
56
|
+
// request's figures instead, and omit it when they cannot carry it.
|
|
57
|
+
const tokensPerSecond = totalTokens !== undefined && totalTime !== undefined && totalTime > 0
|
|
58
|
+
? totalTokens / totalTime
|
|
59
|
+
: undefined;
|
|
3
60
|
return {
|
|
4
|
-
...(
|
|
5
|
-
...(
|
|
6
|
-
...(
|
|
7
|
-
...(
|
|
8
|
-
...(
|
|
9
|
-
...(
|
|
61
|
+
...(totalTime !== undefined && { totalTime: totalTime * NMT_SECONDS_TO_MS }),
|
|
62
|
+
...(totalTokens !== undefined && { totalTokens }),
|
|
63
|
+
...(decodeTime !== undefined && { decodeTime: decodeTime * NMT_SECONDS_TO_MS }),
|
|
64
|
+
...(encodeTime !== undefined && { encodeTime: encodeTime * NMT_SECONDS_TO_MS }),
|
|
65
|
+
...(tokensPerSecond !== undefined && { tokensPerSecond }),
|
|
66
|
+
...(TTFT !== undefined && { timeToFirstToken: TTFT })
|
|
10
67
|
};
|
|
11
68
|
}
|
|
@@ -55,7 +55,7 @@ import { translateServerParamsSchema, normalizeModelType, ModelType, AFRICAN_LAN
|
|
|
55
55
|
import { getLangName, detectOne } from '@qvac/langdetect-text';
|
|
56
56
|
import { nowMs } from '../../profiling/index.js';
|
|
57
57
|
import { buildStreamResult } from '../../profiling/model-execution.js';
|
|
58
|
-
import { buildNmtTranslationStats } from './translate-stats.js';
|
|
58
|
+
import { buildNmtTranslationStats, markNmtCountersStale } from './translate-stats.js';
|
|
59
59
|
import { ModelNotFoundError, ModelTypeMismatchError, TranslationFailedError } from '../../errors/index.js';
|
|
60
60
|
import { getRequestRegistry, withRequestContext } from '../../runtime/index.js';
|
|
61
61
|
import { isAddonContextOverflowError } from '../builtin/llamacpp-completion/ops/context-overflow.js';
|
|
@@ -203,7 +203,15 @@ export async function* translate(params, requestId) {
|
|
|
203
203
|
}
|
|
204
204
|
// Use runBatch for batch processing
|
|
205
205
|
const modelStart = nowMs();
|
|
206
|
-
|
|
206
|
+
let translations;
|
|
207
|
+
try {
|
|
208
|
+
translations = await model.runBatch(text);
|
|
209
|
+
}
|
|
210
|
+
finally {
|
|
211
|
+
// The batch advanced the addon's counters and reported no stats, so the
|
|
212
|
+
// next single request has no baseline to difference against.
|
|
213
|
+
markNmtCountersStale(model);
|
|
214
|
+
}
|
|
207
215
|
const modelExecutionMs = nowMs() - modelStart;
|
|
208
216
|
// Soft-cancel boundary: if `cancel({ requestId })` landed while
|
|
209
217
|
// the addon was running the batch, bail before yielding anything.
|
|
@@ -301,14 +309,22 @@ export async function* translate(params, requestId) {
|
|
|
301
309
|
return buildStreamResult(modelExecutionMs, stats);
|
|
302
310
|
}
|
|
303
311
|
const nmtResponse = response;
|
|
312
|
+
// Difference the counters the moment the job ends rather than after the
|
|
313
|
+
// consumer has drained the tokens, so a peer request on the same model that
|
|
314
|
+
// ends in between cannot move the baseline first.
|
|
315
|
+
const statsAtEnd = nmtResponse.await().then(() => buildNmtTranslationStats(nmtResponse.stats, model, entry.local.config), () => undefined);
|
|
304
316
|
for await (const token of nmtResponse.iterate()) {
|
|
305
317
|
if (ctx.signal.aborted)
|
|
306
318
|
break;
|
|
307
319
|
yield token;
|
|
308
320
|
}
|
|
309
321
|
const modelExecutionMs = nowMs() - modelStart;
|
|
310
|
-
|
|
311
|
-
|
|
322
|
+
// A soft-cancelled request drops its result; do not hold the generator open
|
|
323
|
+
// for a job that may still be running.
|
|
324
|
+
if (ctx.signal.aborted) {
|
|
325
|
+
return { modelExecutionMs };
|
|
326
|
+
}
|
|
327
|
+
return buildStreamResult(modelExecutionMs, await statsAtEnd);
|
|
312
328
|
}
|
|
313
329
|
catch (e_1) {
|
|
314
330
|
env_1.error = e_1;
|
|
@@ -165,6 +165,62 @@ registerOperationMetrics({
|
|
|
165
165
|
return Object.keys(gauges).length > 0 ? gauges : undefined;
|
|
166
166
|
}
|
|
167
167
|
});
|
|
168
|
+
// BCI runs on the same whisper.cpp decoder as the whisper ASR engine, so it
|
|
169
|
+
// reports that engine's stage timings plus its own window counters. The
|
|
170
|
+
// streaming variant gets no stats from the addon — only the timing.
|
|
171
|
+
function bciGauges(res) {
|
|
172
|
+
const gauges = {};
|
|
173
|
+
const modelExecMs = readModelExecutionMs(res);
|
|
174
|
+
if (modelExecMs !== undefined)
|
|
175
|
+
gauges['modelExecutionTime'] = modelExecMs;
|
|
176
|
+
if (res.stats?.tokensPerSecond !== undefined) {
|
|
177
|
+
gauges['tokensPerSecond'] = res.stats.tokensPerSecond;
|
|
178
|
+
}
|
|
179
|
+
if (res.stats?.totalTokens !== undefined)
|
|
180
|
+
gauges['totalTokens'] = res.stats.totalTokens;
|
|
181
|
+
if (res.stats?.totalSegments !== undefined)
|
|
182
|
+
gauges['totalSegments'] = res.stats.totalSegments;
|
|
183
|
+
// No totalTime gauge: the addon reports it in seconds while every other
|
|
184
|
+
// producer of this gauge uses ms. totalWallMs is the same measurement in ms.
|
|
185
|
+
if (res.stats?.totalWallMs !== undefined)
|
|
186
|
+
gauges['totalWallMs'] = res.stats.totalWallMs;
|
|
187
|
+
if (res.stats?.processCalls !== undefined)
|
|
188
|
+
gauges['processCalls'] = res.stats.processCalls;
|
|
189
|
+
if (res.stats?.whisperEncodeTime !== undefined) {
|
|
190
|
+
gauges['whisperEncodeTime'] = res.stats.whisperEncodeTime;
|
|
191
|
+
}
|
|
192
|
+
if (res.stats?.whisperDecodeTime !== undefined) {
|
|
193
|
+
gauges['whisperDecodeTime'] = res.stats.whisperDecodeTime;
|
|
194
|
+
}
|
|
195
|
+
if (res.stats?.whisperSampleMs !== undefined) {
|
|
196
|
+
gauges['whisperSampleMs'] = res.stats.whisperSampleMs;
|
|
197
|
+
}
|
|
198
|
+
if (res.stats?.whisperBatchdMs !== undefined) {
|
|
199
|
+
gauges['whisperBatchdMs'] = res.stats.whisperBatchdMs;
|
|
200
|
+
}
|
|
201
|
+
if (res.stats?.whisperPromptMs !== undefined) {
|
|
202
|
+
gauges['whisperPromptMs'] = res.stats.whisperPromptMs;
|
|
203
|
+
}
|
|
204
|
+
return gauges;
|
|
205
|
+
}
|
|
206
|
+
registerOperationMetrics({
|
|
207
|
+
op: 'bciTranscribe',
|
|
208
|
+
kind: 'handler',
|
|
209
|
+
getTags: (req) => (req.modelId ? { modelId: req.modelId } : {}),
|
|
210
|
+
fromFinalChunk: (res) => {
|
|
211
|
+
const gauges = bciGauges(res);
|
|
212
|
+
return Object.keys(gauges).length > 0 ? gauges : undefined;
|
|
213
|
+
}
|
|
214
|
+
});
|
|
215
|
+
registerOperationMetrics({
|
|
216
|
+
op: 'bciTranscribeStream',
|
|
217
|
+
kind: 'handler',
|
|
218
|
+
getTags: (req) => (req.modelId ? { modelId: req.modelId } : {}),
|
|
219
|
+
fromFinalChunk: (res) => {
|
|
220
|
+
const gauges = bciGauges(res);
|
|
221
|
+
return Object.keys(gauges).length > 0 ? gauges : undefined;
|
|
222
|
+
}
|
|
223
|
+
});
|
|
168
224
|
registerOperationMetrics({
|
|
169
225
|
op: 'textToSpeech',
|
|
170
226
|
kind: 'handler',
|
package/dist/registry.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { handleStartRpcServer, handleStopRpcServer, handleDiscoverRpcServers } from './handlers/rpc-server.js';
|
|
1
2
|
import { dispatchPluginReply, dispatchPluginStream } from './handlers/plugin-dispatch.js';
|
|
2
3
|
import { handleLoadModel } from './handlers/load-model/index.js';
|
|
3
4
|
import { handleUnloadModel } from './handlers/unload-model.js';
|
|
@@ -42,6 +43,9 @@ function finetuneSupportsProgress(request) {
|
|
|
42
43
|
return ['start', 'resume', undefined].includes(request.operation);
|
|
43
44
|
}
|
|
44
45
|
export const registry = {
|
|
46
|
+
startRpcServer: { type: 'reply', handler: handleStartRpcServer },
|
|
47
|
+
stopRpcServer: { type: 'reply', handler: handleStopRpcServer },
|
|
48
|
+
discoverRpcServers: { type: 'reply', handler: handleDiscoverRpcServers },
|
|
45
49
|
// Reply
|
|
46
50
|
heartbeat: {
|
|
47
51
|
type: 'reply',
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
import type { AssessModelFitResult,
|
|
1
|
+
import type { AssessModelFitResult, ModelFitBasis, ModelFitEstimateTarget, ModelFitExecution, NativeProbeFit } from '../../schemas/assess-model-fit.js';
|
|
2
2
|
import type { SystemResources } from '../../schemas/system-resources.js';
|
|
3
3
|
import type { ModelResourceProfile } from '../../schemas/model-resource-profile.js';
|
|
4
4
|
import type { ModelFitPlatform, PlatformCalibration } from './types.js';
|
|
5
5
|
/** Resolves a checksum to its catalog resource profile. */
|
|
6
6
|
export type ProfileResolver = (sha256Checksum: string) => ModelResourceProfile | undefined;
|
|
7
7
|
export interface AssessModelFitOptions {
|
|
8
|
-
models: readonly
|
|
8
|
+
models: readonly ModelFitEstimateTarget[];
|
|
9
9
|
execution: ModelFitExecution;
|
|
10
10
|
resources: SystemResources;
|
|
11
11
|
/**
|
|
@@ -24,6 +24,17 @@ export interface AssessModelFitOptions {
|
|
|
24
24
|
resolveSharedGpuCalibration?: (platform: ModelFitPlatform, backend: string) => PlatformCalibration | undefined;
|
|
25
25
|
/** Defaults to the generated catalog table; injected in tests. */
|
|
26
26
|
resolveProfile?: ProfileResolver;
|
|
27
|
+
/**
|
|
28
|
+
* The engine's own verdict for the single candidate, from the registry's fit
|
|
29
|
+
* stub. Resolved in the handler, which is where the network is.
|
|
30
|
+
*
|
|
31
|
+
* Only ever set for a one-candidate assessment: the fitter answers about one
|
|
32
|
+
* model against the whole machine, and two such answers cannot be summed into
|
|
33
|
+
* a combined budget the way two byte estimates can.
|
|
34
|
+
*/
|
|
35
|
+
nativeFit?: NativeProbeFit | undefined;
|
|
36
|
+
/** Why the fitter produced no verdict, when one was sought. */
|
|
37
|
+
nativeFitUnavailable?: string | undefined;
|
|
27
38
|
}
|
|
28
39
|
/**
|
|
29
40
|
* Turns catalog profiles plus a fresh system-memory sample into an advisory
|
|
@@ -34,3 +45,36 @@ export interface AssessModelFitOptions {
|
|
|
34
45
|
* happen in the handler.
|
|
35
46
|
*/
|
|
36
47
|
export declare function assessModelFitFromResources(options: AssessModelFitOptions): AssessModelFitResult;
|
|
48
|
+
/**
|
|
49
|
+
* Whether every allocation a load makes comes out of the memory the system (or
|
|
50
|
+
* process) budget measures — the precondition for the computed floor to be
|
|
51
|
+
* compared against that budget.
|
|
52
|
+
*
|
|
53
|
+
* True on unified-memory platforms whatever the collector reports, and
|
|
54
|
+
* elsewhere when no usable GPU is present or every usable GPU allocates out of
|
|
55
|
+
* system RAM. A card with its own memory, or a device whose class cannot be
|
|
56
|
+
* told (an AMD APU on linux), fails it: the weights may live where the budget
|
|
57
|
+
* cannot see them.
|
|
58
|
+
*/
|
|
59
|
+
export declare function boundBySystemMemory(resources: SystemResources, platform: ModelFitPlatform | undefined): boolean;
|
|
60
|
+
/**
|
|
61
|
+
* Picks the budget basis for a platform.
|
|
62
|
+
*
|
|
63
|
+
* iOS is the exception: jetsam terminates an app on its own footprint against
|
|
64
|
+
* a per-process limit well below device RAM, so a system-memory budget there
|
|
65
|
+
* would defend `likely-fits` verdicts the OS does not honor. Android's
|
|
66
|
+
* low-memory killer acts system-wide and native allocations carry no
|
|
67
|
+
* per-process cap, so it deliberately keeps the system basis with the mobile
|
|
68
|
+
* reserve.
|
|
69
|
+
*/
|
|
70
|
+
export declare function resolveBasis(platform: ModelFitPlatform | undefined): ModelFitBasis;
|
|
71
|
+
/**
|
|
72
|
+
* Derives the memory budget from the sample, under the platform's basis.
|
|
73
|
+
*
|
|
74
|
+
* Only `sample.memory` is used: capabilities-only totals say nothing about what
|
|
75
|
+
* is free right now, and a verdict without that is not worth giving. Under the
|
|
76
|
+
* process basis the ceiling is reconstructed as allowance + footprint — the
|
|
77
|
+
* relation the OS enforces — so every budget field keeps the same meaning
|
|
78
|
+
* under either basis.
|
|
79
|
+
*/
|
|
80
|
+
export declare function resolveBudget(resources: SystemResources, platform: ModelFitPlatform | undefined, basis: ModelFitBasis, reasons: string[]): AssessModelFitResult['budget'];
|
|
@@ -168,6 +168,27 @@ export function assessModelFitFromResources(options) {
|
|
|
168
168
|
if (anyFloor) {
|
|
169
169
|
reasons.push('at least one model is bounded only by its computed floor (weights, plus the KV cache for llama.cpp), which can refuse the set but never confirm a fit');
|
|
170
170
|
}
|
|
171
|
+
// The engine's own fitter, where it reached a verdict, replaces the modelled
|
|
172
|
+
// one: it read this model's tensor list and measured this machine, which is
|
|
173
|
+
// what the coefficients above approximate. The estimator's reasons and
|
|
174
|
+
// assumptions describe the path not taken, so none of them are reported.
|
|
175
|
+
const native = nativeModelResult(options.nativeFit, modelResults);
|
|
176
|
+
if (native) {
|
|
177
|
+
return {
|
|
178
|
+
verdict: native.verdict,
|
|
179
|
+
basis,
|
|
180
|
+
execution,
|
|
181
|
+
evidence: 'native-fit',
|
|
182
|
+
...(budget && { budget }),
|
|
183
|
+
models: [native],
|
|
184
|
+
reasons: ['the engine fitter read the registry description of this model'],
|
|
185
|
+
assumptions: []
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
const declined = nativeFitDecline(options.nativeFit, options.nativeFitUnavailable);
|
|
189
|
+
if (declined !== undefined) {
|
|
190
|
+
reasons.push(declined);
|
|
191
|
+
}
|
|
171
192
|
return {
|
|
172
193
|
verdict,
|
|
173
194
|
basis,
|
|
@@ -183,6 +204,46 @@ export function assessModelFitFromResources(options) {
|
|
|
183
204
|
assumptions
|
|
184
205
|
};
|
|
185
206
|
}
|
|
207
|
+
/**
|
|
208
|
+
* Why no engine verdict backs this assessment, for the cases where one was
|
|
209
|
+
* sought: the fitter never ran, or it ran and declined. `undefined` where none
|
|
210
|
+
* was sought, which is every multi-candidate assessment.
|
|
211
|
+
*/
|
|
212
|
+
function nativeFitDecline(nativeFit, unavailable) {
|
|
213
|
+
if (unavailable !== undefined)
|
|
214
|
+
return `no engine fit: ${unavailable}`;
|
|
215
|
+
if (nativeFit?.verdict !== 'unknown')
|
|
216
|
+
return undefined;
|
|
217
|
+
const detail = nativeFit.message === undefined ? nativeFit.reason : nativeFit.message;
|
|
218
|
+
return `the engine fitter reached no verdict: ${detail}`;
|
|
219
|
+
}
|
|
220
|
+
/**
|
|
221
|
+
* The per-model result a native fit produces, or `undefined` when the modelled
|
|
222
|
+
* one stands.
|
|
223
|
+
*
|
|
224
|
+
* The probe answers about one model against the whole machine. Two such answers
|
|
225
|
+
* carry no byte demand that could be summed under one budget, so a multi-model
|
|
226
|
+
* assessment keeps the estimator that can aggregate. `unknown` — the probe
|
|
227
|
+
* disabled, the load shape unsupported, the child unusable — is not a verdict
|
|
228
|
+
* and falls through the same way.
|
|
229
|
+
*/
|
|
230
|
+
function nativeModelResult(nativeFit, modelResults) {
|
|
231
|
+
if (!nativeFit || modelResults.length !== 1)
|
|
232
|
+
return undefined;
|
|
233
|
+
if (nativeFit.verdict === 'unknown')
|
|
234
|
+
return undefined;
|
|
235
|
+
const modelled = modelResults[0];
|
|
236
|
+
if (!modelled)
|
|
237
|
+
return undefined;
|
|
238
|
+
return {
|
|
239
|
+
name: modelled.name,
|
|
240
|
+
verdict: nativeFit.verdict === 'fit' ? 'likely-fits' : 'likely-too-large',
|
|
241
|
+
evidence: 'native-fit',
|
|
242
|
+
...(modelled.device !== undefined && { device: modelled.device }),
|
|
243
|
+
estimatorVersion: nativeFit.estimatorVersion,
|
|
244
|
+
reasons: nativeFit.message === undefined ? [nativeFit.reason] : [nativeFit.reason, nativeFit.message]
|
|
245
|
+
};
|
|
246
|
+
}
|
|
186
247
|
/**
|
|
187
248
|
* Whether every allocation a load makes comes out of the memory the system (or
|
|
188
249
|
* process) budget measures — the precondition for the computed floor to be
|
|
@@ -194,7 +255,7 @@ export function assessModelFitFromResources(options) {
|
|
|
194
255
|
* told (an AMD APU on linux), fails it: the weights may live where the budget
|
|
195
256
|
* cannot see them.
|
|
196
257
|
*/
|
|
197
|
-
function boundBySystemMemory(resources, platform) {
|
|
258
|
+
export function boundBySystemMemory(resources, platform) {
|
|
198
259
|
if (platform !== undefined && UNIFIED_MEMORY_PLATFORMS.includes(platform))
|
|
199
260
|
return true;
|
|
200
261
|
const gpus = usableGpus(resources);
|
|
@@ -487,7 +548,7 @@ function worst(a, b) {
|
|
|
487
548
|
* per-process cap, so it deliberately keeps the system basis with the mobile
|
|
488
549
|
* reserve.
|
|
489
550
|
*/
|
|
490
|
-
function resolveBasis(platform) {
|
|
551
|
+
export function resolveBasis(platform) {
|
|
491
552
|
return platform === 'ios-arm64' ? 'process-memory' : 'system-memory';
|
|
492
553
|
}
|
|
493
554
|
function basisEvidence(basis) {
|
|
@@ -508,7 +569,7 @@ function basisEvidence(basis) {
|
|
|
508
569
|
* relation the OS enforces — so every budget field keeps the same meaning
|
|
509
570
|
* under either basis.
|
|
510
571
|
*/
|
|
511
|
-
function resolveBudget(resources, platform, basis, reasons) {
|
|
572
|
+
export function resolveBudget(resources, platform, basis, reasons) {
|
|
512
573
|
const sample = resources.sample;
|
|
513
574
|
if (!sample) {
|
|
514
575
|
reasons.push('no memory sample was available');
|
|
@@ -641,6 +702,7 @@ deviceBudgets, alsoBoundBy) {
|
|
|
641
702
|
return {
|
|
642
703
|
name: candidate.model.name,
|
|
643
704
|
verdict: 'unknown',
|
|
705
|
+
...(candidate.device !== undefined && { device: candidate.device }),
|
|
644
706
|
reasons: [...result.reasons]
|
|
645
707
|
};
|
|
646
708
|
}
|
|
@@ -650,6 +712,7 @@ deviceBudgets, alsoBoundBy) {
|
|
|
650
712
|
name: candidate.model.name,
|
|
651
713
|
verdict,
|
|
652
714
|
evidence: 'computed-only',
|
|
715
|
+
...(candidate.device !== undefined && { device: candidate.device }),
|
|
653
716
|
floorBytes: result.bytes,
|
|
654
717
|
estimatorVersion: FLOOR_VERSION,
|
|
655
718
|
reasons: [
|
|
@@ -670,6 +733,7 @@ deviceBudgets, alsoBoundBy) {
|
|
|
670
733
|
name: candidate.model.name,
|
|
671
734
|
verdict: budget ? verdictAgainst(total, deviceBudgets ?? [budget], alsoBoundBy) : 'unknown',
|
|
672
735
|
evidence: 'calibration',
|
|
736
|
+
...(candidate.device !== undefined && { device: candidate.device }),
|
|
673
737
|
estimate: { lowerBoundBytes: total.lower, upperBoundBytes: total.upper },
|
|
674
738
|
estimatorVersion: result.estimatorVersion,
|
|
675
739
|
reasons: budget
|
|
@@ -59,7 +59,7 @@ export function estimateLlm(input) {
|
|
|
59
59
|
assumptions.push('companion artifacts passed in `artifacts` are counted at full size');
|
|
60
60
|
}
|
|
61
61
|
// Context: never more than the model was trained for — llama.cpp clamps.
|
|
62
|
-
let contextTokens = workload.contextTokens;
|
|
62
|
+
let contextTokens = workload.contextTokens ?? facts.contextLength;
|
|
63
63
|
if (contextTokens > facts.contextLength) {
|
|
64
64
|
assumptions.push(`requested ${workload.contextTokens} tokens exceeds the trained context of ${facts.contextLength}; clamped to the trained context`);
|
|
65
65
|
contextTokens = facts.contextLength;
|
|
@@ -85,7 +85,7 @@ export function estimateLlm(input) {
|
|
|
85
85
|
if (calibration.workingPeakBytes && calibration.workingPeakBytes.upper > 0) {
|
|
86
86
|
assumptions.push('one operation is assumed in flight per model; the working peak is what a single completion was measured to add on top of the resident cost');
|
|
87
87
|
}
|
|
88
|
-
assumptions.push('default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not
|
|
88
|
+
assumptions.push('default KV-cache types are assumed; an explicit `cache-type-k`/`cache-type-v` in `modelConfig` is not read here and would change these numbers');
|
|
89
89
|
return {
|
|
90
90
|
kind: 'estimate',
|
|
91
91
|
estimatorVersion: LLM_ESTIMATOR_VERSION,
|
|
@@ -50,18 +50,17 @@ export function estimateWhisper(input) {
|
|
|
50
50
|
assumptions.push('companion artifacts passed in `artifacts` (e.g. a VAD model) are counted at full size');
|
|
51
51
|
}
|
|
52
52
|
const windows = workload.windowMs / CALIBRATION_WINDOW_MS;
|
|
53
|
-
const batch = workload.batch ?? 1;
|
|
54
53
|
const streamingLower = workload.streaming ? calibration.audioStreamingBytes.lower : 0;
|
|
55
54
|
const streamingUpper = workload.streaming ? calibration.audioStreamingBytes.upper : 0;
|
|
56
55
|
const working = {
|
|
57
56
|
lower: calibration.fixedOverheadBytes.lower +
|
|
58
|
-
calibration.audioWindowBytes.lower * windows
|
|
57
|
+
calibration.audioWindowBytes.lower * windows +
|
|
59
58
|
streamingLower,
|
|
60
59
|
upper: calibration.fixedOverheadBytes.upper +
|
|
61
|
-
calibration.audioWindowBytes.upper * windows
|
|
60
|
+
calibration.audioWindowBytes.upper * windows +
|
|
62
61
|
streamingUpper
|
|
63
62
|
};
|
|
64
|
-
reasons.push(`working memory scaled from the calibrated 30 s window to ${Math.round(workload.windowMs)} ms
|
|
63
|
+
reasons.push(`working memory scaled from the calibrated 30 s window to ${Math.round(workload.windowMs)} ms`);
|
|
65
64
|
assumptions.push('the audio window scales working memory linearly from the engine’s 30 s default; longer windows are chunked by the engine rather than held whole');
|
|
66
65
|
if (workload.streaming) {
|
|
67
66
|
assumptions.push('a streaming session is assumed to keep one calibrated stream buffer resident');
|