@qvac/inference 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +95 -39
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/delete-cache.js +3 -0
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +48 -27
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +239 -135
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.js +169 -68
- package/dist/resources/model-fit/estimators/llm.d.ts +1 -0
- package/dist/resources/model-fit/estimators/llm.js +3 -1
- package/dist/resources/model-fit/floor.d.ts +31 -0
- package/dist/resources/model-fit/floor.js +65 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/resources/native.js +5 -4
- package/dist/resources/types.d.ts +1 -3
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +84 -0
- package/dist/schemas/assess-model-fit.js +72 -3
- package/dist/schemas/audio-gen.d.ts +482 -7
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +10 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci.d.ts +1 -1
- package/dist/schemas/common.d.ts +782 -19
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +14 -1
- package/dist/schemas/completion-event.js +4 -0
- package/dist/schemas/completion-stream.d.ts +22 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +48 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +4 -8
- package/dist/schemas/llamacpp-config.js +4 -4
- package/dist/schemas/load-model.d.ts +2604 -180
- package/dist/schemas/model-config-schemas.d.ts +228 -5
- package/dist/schemas/model-ops.d.ts +3 -0
- package/dist/schemas/model-ops.js +6 -0
- package/dist/schemas/plugin.d.ts +21 -11
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +30 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +29 -26
package/dist/api/delete-cache.js
CHANGED
|
@@ -4,7 +4,8 @@ import { InvalidDeleteCacheParamsError, DeleteCacheFailedError } from '../errors
|
|
|
4
4
|
* Deletes KV cache files.
|
|
5
5
|
*
|
|
6
6
|
* @param params - The delete cache parameters
|
|
7
|
-
* @param params.all - If true, deletes all cache files
|
|
7
|
+
* @param params.all - If true, deletes all cache files, including caller-owned named caches
|
|
8
|
+
* @param params.auto - If true, reclaims only automatic caches (`completion({ kvCache: true })`) that no request is using. Named caches are untouched. Covers the whole cache directory, which is shared by every QVAC process using the same home directory
|
|
8
9
|
* @param params.kvCacheKey - The cache key to delete
|
|
9
10
|
* @param params.modelId - Optional: specific model ID to delete within the cache key. If not provided, deletes entire cache key.
|
|
10
11
|
* @returns Promise resolving to success status
|
|
@@ -14,6 +15,9 @@ import { InvalidDeleteCacheParamsError, DeleteCacheFailedError } from '../errors
|
|
|
14
15
|
* // Delete all caches
|
|
15
16
|
* await deleteCache({ all: true });
|
|
16
17
|
*
|
|
18
|
+
* // Reclaim disk from automatic caches, leaving named caches alone
|
|
19
|
+
* await deleteCache({ auto: true });
|
|
20
|
+
*
|
|
17
21
|
* // Delete entire cache key (all models)
|
|
18
22
|
* await deleteCache({ kvCacheKey: "my-session" });
|
|
19
23
|
*
|
|
@@ -29,6 +33,12 @@ export async function deleteCache(params) {
|
|
|
29
33
|
all: true
|
|
30
34
|
};
|
|
31
35
|
}
|
|
36
|
+
else if ('auto' in params && params.auto) {
|
|
37
|
+
req = {
|
|
38
|
+
type: 'deleteCache',
|
|
39
|
+
auto: true
|
|
40
|
+
};
|
|
41
|
+
}
|
|
32
42
|
else if ('kvCacheKey' in params) {
|
|
33
43
|
req = {
|
|
34
44
|
type: 'deleteCache',
|
package/dist/api/index.d.ts
CHANGED
|
@@ -14,6 +14,7 @@ export { finetune, type FinetuneHandle } from './finetune.js';
|
|
|
14
14
|
export { translate } from './translate.js';
|
|
15
15
|
export { cancel } from './cancel.js';
|
|
16
16
|
export { ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace } from './rag.js';
|
|
17
|
+
export { createVectorIndex, loadVectorIndex, type VectorIndex } from './vector-index.js';
|
|
17
18
|
export { textToSpeech, textToSpeechStream } from './text-to-speech.js';
|
|
18
19
|
export { getModelInfo } from './get-model-info.js';
|
|
19
20
|
export { getLoadedModelInfo } from './get-loaded-model-info.js';
|
|
@@ -22,7 +23,7 @@ export { assessModelFit } from './assess-model-fit.js';
|
|
|
22
23
|
export { ocr } from './ocr.js';
|
|
23
24
|
export { invokePlugin, invokePluginStream } from './invoke-plugin.js';
|
|
24
25
|
export { diffusion, type DiffusionProgressTick } from './diffusion.js';
|
|
25
|
-
export { audioGen } from './audio-gen.js';
|
|
26
|
+
export { audioGen, audioEdit, audioUnderstand } from './audio-gen.js';
|
|
26
27
|
export { classify } from './classify.js';
|
|
27
28
|
export { video, type VideoProgressTick } from './video.js';
|
|
28
29
|
export { upscale } from './upscale.js';
|
package/dist/api/index.js
CHANGED
|
@@ -14,6 +14,7 @@ export { finetune } from './finetune.js';
|
|
|
14
14
|
export { translate } from './translate.js';
|
|
15
15
|
export { cancel } from './cancel.js';
|
|
16
16
|
export { ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace } from './rag.js';
|
|
17
|
+
export { createVectorIndex, loadVectorIndex } from './vector-index.js';
|
|
17
18
|
export { textToSpeech, textToSpeechStream } from './text-to-speech.js';
|
|
18
19
|
export { getModelInfo } from './get-model-info.js';
|
|
19
20
|
export { getLoadedModelInfo } from './get-loaded-model-info.js';
|
|
@@ -22,7 +23,7 @@ export { assessModelFit } from './assess-model-fit.js';
|
|
|
22
23
|
export { ocr } from './ocr.js';
|
|
23
24
|
export { invokePlugin, invokePluginStream } from './invoke-plugin.js';
|
|
24
25
|
export { diffusion } from './diffusion.js';
|
|
25
|
-
export { audioGen } from './audio-gen.js';
|
|
26
|
+
export { audioGen, audioEdit, audioUnderstand } from './audio-gen.js';
|
|
26
27
|
export { classify } from './classify.js';
|
|
27
28
|
export { video } from './video.js';
|
|
28
29
|
export { upscale } from './upscale.js';
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type TtsClientParamsInput, type RPCOptions, type TtsResponse, type TextToSpeechStreamClientParams, type TextToSpeechStreamSession, type TextToSpeechStreamResult } from '../schemas/index.js';
|
|
1
|
+
import { type TtsClientParamsInput, type RPCOptions, type TtsResponse, type TextToSpeechStreamClientParams, type TextToSpeechStreamSession, type TextToSpeechStreamResult, type TtsStats } from '../schemas/index.js';
|
|
2
2
|
/**
|
|
3
3
|
* Fan-out queue that lets multiple consumers iterate the same TTS response
|
|
4
4
|
* stream independently. Items are retained only until every active subscriber
|
|
@@ -33,9 +33,9 @@ export declare class TtsMulticast {
|
|
|
33
33
|
*
|
|
34
34
|
* Three modes selected by `params.stream` and `params.sentenceStream`:
|
|
35
35
|
*
|
|
36
|
-
* - `stream: false`
|
|
36
|
+
* - `stream: false` — collect all PCM samples and resolve once via
|
|
37
37
|
* `result.buffer` (`Promise<number[]>`). `bufferStream` is empty.
|
|
38
|
-
* - `stream: true` — yield PCM samples through `result.bufferStream`
|
|
38
|
+
* - `stream: true` (default) — yield PCM samples through `result.bufferStream`
|
|
39
39
|
* (`AsyncGenerator<number>`) as they arrive. `buffer` resolves to an empty
|
|
40
40
|
* array.
|
|
41
41
|
* - `stream: true, sentenceStream: true` — also exposes `result.chunkUpdates`
|
|
@@ -47,14 +47,41 @@ export declare class TtsMulticast {
|
|
|
47
47
|
* if the consumer breaks out before the terminal frame, or rejects on a
|
|
48
48
|
* pipeline error. Awaiting `done` is safe even when no stream is iterated.
|
|
49
49
|
*
|
|
50
|
+
* `result.sampleRate` resolves from the first audio frame (so it can be awaited
|
|
51
|
+
* before draining the audio); `result.stats` and `result.stopReason` resolve
|
|
52
|
+
* from the terminal frame. All three resolve — with `undefined` when the run
|
|
53
|
+
* produced none — rather than reject. In `stream: true` mode (the default) the
|
|
54
|
+
* run starts lazily: nothing is sent until `bufferStream` is first iterated, so
|
|
55
|
+
* `done`, `sampleRate`, `stats` and `stopReason` stay pending until then. In
|
|
56
|
+
* `stream: false` and `sentenceStream: true` modes the run starts immediately.
|
|
57
|
+
* Read `sampleRate` rather than assuming the engine default: `outputSampleRate`
|
|
58
|
+
* and the LavaSR enhancer both change it.
|
|
59
|
+
*
|
|
50
60
|
* @param params - TTS request parameters (see `TtsClientParamsInput`).
|
|
51
61
|
* @param options - Optional RPC options (timeout, profiling, force new connection).
|
|
52
62
|
* @returns A `TextToSpeechStreamResult` with `bufferStream`, `buffer`, `done`,
|
|
53
|
-
* and (when `sentenceStream: true`)
|
|
63
|
+
* `sampleRate`, `stats`, and (when `sentenceStream: true`)
|
|
64
|
+
* `chunkUpdates`.
|
|
54
65
|
* @throws {TextToSpeechStreamFailedError} When `sentenceStream: true` is paired
|
|
55
66
|
* with `stream: false`, or when the underlying RPC stream errors.
|
|
56
67
|
*/
|
|
57
68
|
export declare function textToSpeech(params: TtsClientParamsInput, options?: RPCOptions): TextToSpeechStreamResult;
|
|
69
|
+
/**
|
|
70
|
+
* The output rate and the runtime stats ride the response frames, but the
|
|
71
|
+
* three result shapes below expose PCM through generators a caller may abandon
|
|
72
|
+
* early. This collects both off the frames as they pass and hands them back as
|
|
73
|
+
* promises that always RESOLVE — with `undefined` when the run produced none —
|
|
74
|
+
* so `await result.sampleRate` can never hang or reject on its own.
|
|
75
|
+
*/
|
|
76
|
+
export type TtsSideChannel = {
|
|
77
|
+
sampleRate: Promise<number | undefined>;
|
|
78
|
+
stats: Promise<TtsStats | undefined>;
|
|
79
|
+
stopReason: Promise<'completed' | 'cancelled' | undefined>;
|
|
80
|
+
observe(response: TtsResponse): void;
|
|
81
|
+
settle(): void;
|
|
82
|
+
};
|
|
83
|
+
/** Exported for test use; not part of the public API surface. */
|
|
84
|
+
export declare function createTtsSideChannel(): TtsSideChannel;
|
|
58
85
|
/**
|
|
59
86
|
* Duplex session: write UTF-8 text fragments (e.g. LLM token deltas) via `write`. Each string or
|
|
60
87
|
* Buffer should be a complete UTF-8 fragment. The engine forwards them to ONNX TTS `runStreaming`
|
|
@@ -3,6 +3,7 @@ import Buffer from 'bare-buffer';
|
|
|
3
3
|
import { stream as streamRpc, duplex } from '../dispatch.js';
|
|
4
4
|
import { getAppLogger } from '../logging/index.js';
|
|
5
5
|
import { TextToSpeechStreamFailedError } from '../errors/index.js';
|
|
6
|
+
import { generateRequestId } from '../runtime/request-id.js';
|
|
6
7
|
const logger = getAppLogger();
|
|
7
8
|
/**
|
|
8
9
|
* Fan-out queue that lets multiple consumers iterate the same TTS response
|
|
@@ -129,10 +130,11 @@ export class TtsMulticast {
|
|
|
129
130
|
}
|
|
130
131
|
}
|
|
131
132
|
}
|
|
132
|
-
function buildTtsRequest(params) {
|
|
133
|
+
function buildTtsRequest(params, requestId) {
|
|
133
134
|
return {
|
|
134
135
|
type: 'textToSpeech',
|
|
135
136
|
modelId: params.modelId,
|
|
137
|
+
requestId,
|
|
136
138
|
inputType: params.inputType ?? 'text',
|
|
137
139
|
text: params.text,
|
|
138
140
|
stream: params.stream ?? true,
|
|
@@ -155,10 +157,11 @@ function buildTtsRequest(params) {
|
|
|
155
157
|
...(params.quality !== undefined && { quality: params.quality })
|
|
156
158
|
};
|
|
157
159
|
}
|
|
158
|
-
function buildTextToSpeechStreamRequest(params) {
|
|
160
|
+
function buildTextToSpeechStreamRequest(params, requestId) {
|
|
159
161
|
return {
|
|
160
162
|
type: 'textToSpeechStream',
|
|
161
163
|
modelId: params.modelId,
|
|
164
|
+
requestId,
|
|
162
165
|
inputType: params.inputType ?? 'text',
|
|
163
166
|
...(params.accumulateSentences !== undefined && {
|
|
164
167
|
accumulateSentences: params.accumulateSentences
|
|
@@ -189,9 +192,9 @@ function buildTextToSpeechStreamRequest(params) {
|
|
|
189
192
|
*
|
|
190
193
|
* Three modes selected by `params.stream` and `params.sentenceStream`:
|
|
191
194
|
*
|
|
192
|
-
* - `stream: false`
|
|
195
|
+
* - `stream: false` — collect all PCM samples and resolve once via
|
|
193
196
|
* `result.buffer` (`Promise<number[]>`). `bufferStream` is empty.
|
|
194
|
-
* - `stream: true` — yield PCM samples through `result.bufferStream`
|
|
197
|
+
* - `stream: true` (default) — yield PCM samples through `result.bufferStream`
|
|
195
198
|
* (`AsyncGenerator<number>`) as they arrive. `buffer` resolves to an empty
|
|
196
199
|
* array.
|
|
197
200
|
* - `stream: true, sentenceStream: true` — also exposes `result.chunkUpdates`
|
|
@@ -203,10 +206,21 @@ function buildTextToSpeechStreamRequest(params) {
|
|
|
203
206
|
* if the consumer breaks out before the terminal frame, or rejects on a
|
|
204
207
|
* pipeline error. Awaiting `done` is safe even when no stream is iterated.
|
|
205
208
|
*
|
|
209
|
+
* `result.sampleRate` resolves from the first audio frame (so it can be awaited
|
|
210
|
+
* before draining the audio); `result.stats` and `result.stopReason` resolve
|
|
211
|
+
* from the terminal frame. All three resolve — with `undefined` when the run
|
|
212
|
+
* produced none — rather than reject. In `stream: true` mode (the default) the
|
|
213
|
+
* run starts lazily: nothing is sent until `bufferStream` is first iterated, so
|
|
214
|
+
* `done`, `sampleRate`, `stats` and `stopReason` stay pending until then. In
|
|
215
|
+
* `stream: false` and `sentenceStream: true` modes the run starts immediately.
|
|
216
|
+
* Read `sampleRate` rather than assuming the engine default: `outputSampleRate`
|
|
217
|
+
* and the LavaSR enhancer both change it.
|
|
218
|
+
*
|
|
206
219
|
* @param params - TTS request parameters (see `TtsClientParamsInput`).
|
|
207
220
|
* @param options - Optional RPC options (timeout, profiling, force new connection).
|
|
208
221
|
* @returns A `TextToSpeechStreamResult` with `bufferStream`, `buffer`, `done`,
|
|
209
|
-
* and (when `sentenceStream: true`)
|
|
222
|
+
* `sampleRate`, `stats`, and (when `sentenceStream: true`)
|
|
223
|
+
* `chunkUpdates`.
|
|
210
224
|
* @throws {TextToSpeechStreamFailedError} When `sentenceStream: true` is paired
|
|
211
225
|
* with `stream: false`, or when the underlying RPC stream errors.
|
|
212
226
|
*/
|
|
@@ -216,7 +230,10 @@ export function textToSpeech(params, options) {
|
|
|
216
230
|
if (sentenceStream && !stream) {
|
|
217
231
|
throw new TextToSpeechStreamFailedError('textToSpeech: `sentenceStream: true` requires `stream: true`');
|
|
218
232
|
}
|
|
219
|
-
|
|
233
|
+
// Minted here and surfaced synchronously on the result so a caller can
|
|
234
|
+
// cancel a run it has only just started.
|
|
235
|
+
const requestId = params.requestId ?? generateRequestId();
|
|
236
|
+
const request = buildTtsRequest(params, requestId);
|
|
220
237
|
if (stream && sentenceStream) {
|
|
221
238
|
return sentenceStreamTts(request, options);
|
|
222
239
|
}
|
|
@@ -225,18 +242,73 @@ export function textToSpeech(params, options) {
|
|
|
225
242
|
}
|
|
226
243
|
return collectTts(request, options);
|
|
227
244
|
}
|
|
245
|
+
/** Exported for test use; not part of the public API surface. */
|
|
246
|
+
export function createTtsSideChannel() {
|
|
247
|
+
let resolveSampleRate;
|
|
248
|
+
let resolveStats;
|
|
249
|
+
let resolveStopReason;
|
|
250
|
+
const sampleRate = new Promise((resolve) => {
|
|
251
|
+
resolveSampleRate = resolve;
|
|
252
|
+
});
|
|
253
|
+
const stats = new Promise((resolve) => {
|
|
254
|
+
resolveStats = resolve;
|
|
255
|
+
});
|
|
256
|
+
const stopReason = new Promise((resolve) => {
|
|
257
|
+
resolveStopReason = resolve;
|
|
258
|
+
});
|
|
259
|
+
let seenSampleRate;
|
|
260
|
+
let latestStats;
|
|
261
|
+
let seenStopReason;
|
|
262
|
+
let settled = false;
|
|
263
|
+
return {
|
|
264
|
+
sampleRate,
|
|
265
|
+
stats,
|
|
266
|
+
stopReason,
|
|
267
|
+
observe(response) {
|
|
268
|
+
// Resolve on the FIRST frame that carries a rate, not at end of stream:
|
|
269
|
+
// a caller opening an audio device wants it before draining the audio.
|
|
270
|
+
// Promise resolution is idempotent, so settle() can still resolve the
|
|
271
|
+
// undefined fallback for a run that never reported one.
|
|
272
|
+
if (seenSampleRate === undefined && response.sampleRate !== undefined) {
|
|
273
|
+
seenSampleRate = response.sampleRate;
|
|
274
|
+
resolveSampleRate(seenSampleRate);
|
|
275
|
+
}
|
|
276
|
+
if (response.stats !== undefined)
|
|
277
|
+
latestStats = response.stats;
|
|
278
|
+
if (response.stopReason !== undefined)
|
|
279
|
+
seenStopReason = response.stopReason;
|
|
280
|
+
},
|
|
281
|
+
settle() {
|
|
282
|
+
if (settled)
|
|
283
|
+
return;
|
|
284
|
+
settled = true;
|
|
285
|
+
resolveSampleRate(seenSampleRate);
|
|
286
|
+
resolveStats(latestStats);
|
|
287
|
+
resolveStopReason(seenStopReason);
|
|
288
|
+
}
|
|
289
|
+
};
|
|
290
|
+
}
|
|
228
291
|
// Adapts the raw RPC stream into the filtered `TtsResponse` source the
|
|
229
292
|
// multicast expects. Kept here (not inlined) so the multicast can be
|
|
230
293
|
// constructed in tests with a hand-rolled source instead.
|
|
231
|
-
async function* ttsResponseSource(request, options) {
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
294
|
+
async function* ttsResponseSource(request, options, side) {
|
|
295
|
+
try {
|
|
296
|
+
for await (const response of streamRpc(request, options)) {
|
|
297
|
+
if (response.type !== 'textToSpeech')
|
|
298
|
+
continue;
|
|
299
|
+
side.observe(response);
|
|
300
|
+
yield response;
|
|
301
|
+
}
|
|
302
|
+
}
|
|
303
|
+
finally {
|
|
304
|
+
// Reached on normal end, on error, and when the multicast's pump breaks
|
|
305
|
+
// out on the terminal frame.
|
|
306
|
+
side.settle();
|
|
236
307
|
}
|
|
237
308
|
}
|
|
238
309
|
function sentenceStreamTts(request, options) {
|
|
239
|
-
const
|
|
310
|
+
const side = createTtsSideChannel();
|
|
311
|
+
const multicast = new TtsMulticast(ttsResponseSource(request, options, side));
|
|
240
312
|
// Subscribe eagerly, synchronously — before pump() can push its first
|
|
241
313
|
// item — so both subscribers see the full queue from index 0. If we
|
|
242
314
|
// deferred subscribing until the generators were iterated, the first
|
|
@@ -248,7 +320,11 @@ function sentenceStreamTts(request, options) {
|
|
|
248
320
|
bufferStream: sentenceBufferStream(bufferSubscription),
|
|
249
321
|
chunkUpdates: sentenceChunkUpdates(chunkSubscription),
|
|
250
322
|
buffer: Promise.resolve([]),
|
|
251
|
-
done: multicast.done
|
|
323
|
+
done: multicast.done,
|
|
324
|
+
requestId: request.requestId,
|
|
325
|
+
sampleRate: side.sampleRate,
|
|
326
|
+
stats: side.stats,
|
|
327
|
+
stopReason: side.stopReason
|
|
252
328
|
};
|
|
253
329
|
}
|
|
254
330
|
async function* sentenceBufferStream(source) {
|
|
@@ -268,10 +344,12 @@ async function* sentenceChunkUpdates(source) {
|
|
|
268
344
|
if (hasAudio || hasMeta) {
|
|
269
345
|
yield {
|
|
270
346
|
buffer: hasAudio ? [...m.buffer] : [],
|
|
347
|
+
...(m.sampleRate !== undefined ? { sampleRate: m.sampleRate } : {}),
|
|
271
348
|
...(m.chunkIndex !== undefined ? { chunkIndex: m.chunkIndex } : {}),
|
|
272
349
|
...(typeof m.sentenceChunk === 'string' && m.sentenceChunk.length > 0
|
|
273
350
|
? { sentenceChunk: m.sentenceChunk }
|
|
274
|
-
: {})
|
|
351
|
+
: {}),
|
|
352
|
+
...(m.isLast !== undefined ? { isLast: m.isLast } : {})
|
|
275
353
|
};
|
|
276
354
|
}
|
|
277
355
|
if (m.done)
|
|
@@ -286,13 +364,18 @@ function plainStreamTts(request, options) {
|
|
|
286
364
|
rejectDone = reject;
|
|
287
365
|
});
|
|
288
366
|
done.catch(() => { });
|
|
367
|
+
const side = createTtsSideChannel();
|
|
289
368
|
return {
|
|
290
|
-
bufferStream: plainTtsBufferStream(request, options, resolveDone, rejectDone),
|
|
369
|
+
bufferStream: plainTtsBufferStream(request, options, resolveDone, rejectDone, side),
|
|
291
370
|
buffer: Promise.resolve([]),
|
|
292
|
-
done
|
|
371
|
+
done,
|
|
372
|
+
requestId: request.requestId,
|
|
373
|
+
sampleRate: side.sampleRate,
|
|
374
|
+
stats: side.stats,
|
|
375
|
+
stopReason: side.stopReason
|
|
293
376
|
};
|
|
294
377
|
}
|
|
295
|
-
async function* plainTtsBufferStream(request, options, resolveDone, rejectDone) {
|
|
378
|
+
async function* plainTtsBufferStream(request, options, resolveDone, rejectDone, side) {
|
|
296
379
|
let settled = false;
|
|
297
380
|
try {
|
|
298
381
|
for await (const response of streamRpc(request, options)) {
|
|
@@ -300,6 +383,7 @@ async function* plainTtsBufferStream(request, options, resolveDone, rejectDone)
|
|
|
300
383
|
continue;
|
|
301
384
|
// See TtsMulticast.pump — skip per-frame Zod validation; the engine is
|
|
302
385
|
// the source of truth for this response shape.
|
|
386
|
+
side.observe(response);
|
|
303
387
|
if (response.buffer.length > 0) {
|
|
304
388
|
yield* response.buffer;
|
|
305
389
|
}
|
|
@@ -321,6 +405,7 @@ async function* plainTtsBufferStream(request, options, resolveDone, rejectDone)
|
|
|
321
405
|
// with `false` so `await result.done` never hangs.
|
|
322
406
|
if (!settled)
|
|
323
407
|
resolveDone(false);
|
|
408
|
+
side.settle();
|
|
324
409
|
}
|
|
325
410
|
}
|
|
326
411
|
function collectTts(request, options) {
|
|
@@ -331,21 +416,27 @@ function collectTts(request, options) {
|
|
|
331
416
|
rejectDone = reject;
|
|
332
417
|
});
|
|
333
418
|
done.catch(() => { });
|
|
419
|
+
const side = createTtsSideChannel();
|
|
334
420
|
return {
|
|
335
421
|
bufferStream: emptyBufferStream(),
|
|
336
|
-
buffer: collectTtsBuffer(request, options, resolveDone, rejectDone),
|
|
337
|
-
done
|
|
422
|
+
buffer: collectTtsBuffer(request, options, resolveDone, rejectDone, side),
|
|
423
|
+
done,
|
|
424
|
+
requestId: request.requestId,
|
|
425
|
+
sampleRate: side.sampleRate,
|
|
426
|
+
stats: side.stats,
|
|
427
|
+
stopReason: side.stopReason
|
|
338
428
|
};
|
|
339
429
|
}
|
|
340
430
|
async function* emptyBufferStream() {
|
|
341
431
|
// Non-streaming mode exposes no incremental buffer stream.
|
|
342
432
|
}
|
|
343
|
-
async function collectTtsBuffer(request, options, resolveDone, rejectDone) {
|
|
433
|
+
async function collectTtsBuffer(request, options, resolveDone, rejectDone, side) {
|
|
344
434
|
let buffer = [];
|
|
345
435
|
try {
|
|
346
436
|
for await (const response of streamRpc(request, options)) {
|
|
347
437
|
if (response.type !== 'textToSpeech')
|
|
348
438
|
continue;
|
|
439
|
+
side.observe(response);
|
|
349
440
|
buffer = buffer.concat(response.buffer);
|
|
350
441
|
if (response.done) {
|
|
351
442
|
resolveDone(true);
|
|
@@ -357,6 +448,9 @@ async function collectTtsBuffer(request, options, resolveDone, rejectDone) {
|
|
|
357
448
|
rejectDone(e);
|
|
358
449
|
throw e;
|
|
359
450
|
}
|
|
451
|
+
finally {
|
|
452
|
+
side.settle();
|
|
453
|
+
}
|
|
360
454
|
}
|
|
361
455
|
/**
|
|
362
456
|
* Duplex session: write UTF-8 text fragments (e.g. LLM token deltas) via `write`. Each string or
|
|
@@ -365,7 +459,8 @@ async function collectTtsBuffer(request, options, resolveDone, rejectDone) {
|
|
|
365
459
|
* lines (PCM in `buffer`, optional `chunkIndex` / `sentenceChunk`) until `done`.
|
|
366
460
|
*/
|
|
367
461
|
export async function textToSpeechStream(params, options) {
|
|
368
|
-
const
|
|
462
|
+
const requestId = params.requestId ?? generateRequestId();
|
|
463
|
+
const request = buildTextToSpeechStreamRequest(params, requestId);
|
|
369
464
|
const { requestStream, responseStream } = await duplex(request, options);
|
|
370
465
|
const responses = parseTextToSpeechStreamLines(responseStream);
|
|
371
466
|
let consumed = false;
|
|
@@ -375,6 +470,7 @@ export async function textToSpeechStream(params, options) {
|
|
|
375
470
|
// surface predictable.
|
|
376
471
|
let closed = false;
|
|
377
472
|
return {
|
|
473
|
+
requestId,
|
|
378
474
|
write(textFragment) {
|
|
379
475
|
if (closed) {
|
|
380
476
|
throw new TextToSpeechStreamFailedError('TextToSpeechStreamSession.write() called after end()/destroy()');
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import { send } from '../dispatch.js';
|
|
2
|
+
import { type CreateVectorIndexParams, type LoadVectorIndexParams, type RPCOptions, type VectorIndexAddParams, type VectorIndexHit, type VectorIndexIdsParams, type VectorIndexStorage, type VectorIndexWriteParams } from '../schemas/index.js';
|
|
3
|
+
/**
|
|
4
|
+
* A handle to a vector index that lives in worker memory.
|
|
5
|
+
*
|
|
6
|
+
* Vectors are indexed under stable unsigned 64-bit ids and searched by
|
|
7
|
+
* dot-product similarity, so L2-normalize vectors before adding and queries
|
|
8
|
+
* before searching when cosine similarity is wanted. The index holds vectors
|
|
9
|
+
* only; keep the documents they describe wherever suits the app and map
|
|
10
|
+
* result ids back to them.
|
|
11
|
+
*
|
|
12
|
+
* The index is released by `dispose()` (or `await using`) and on worker
|
|
13
|
+
* shutdown. After a worker crash or restart every method rejects with
|
|
14
|
+
* `VECTOR_INDEX_NOT_FOUND`; recover by creating a new index or loading a
|
|
15
|
+
* snapshot written earlier with `write()`.
|
|
16
|
+
*/
|
|
17
|
+
export interface VectorIndex {
|
|
18
|
+
/** Opaque id of the worker-side index; useful in logs, not needed for any call. */
|
|
19
|
+
readonly indexId: string;
|
|
20
|
+
/** Vector dimensionality every added vector and query must match. */
|
|
21
|
+
readonly dim: number;
|
|
22
|
+
/**
|
|
23
|
+
* Storage mode. Known after `createVectorIndex`. After `loadVectorIndex` it
|
|
24
|
+
* is derived from the snapshot's bit width, so 4-bit snapshots (`q4` or
|
|
25
|
+
* `turbovec-q4`) report `undefined`.
|
|
26
|
+
*/
|
|
27
|
+
readonly storage: VectorIndexStorage | undefined;
|
|
28
|
+
/** Number of live entries as of the last call that changed or reported it. */
|
|
29
|
+
readonly length: number;
|
|
30
|
+
/**
|
|
31
|
+
* Adds vectors under the given ids in one atomic batch. Rejects duplicate
|
|
32
|
+
* ids, ids equal to 2^64 - 1, and rows whose length differs from `dim`.
|
|
33
|
+
* @returns The number of live entries after the add.
|
|
34
|
+
*/
|
|
35
|
+
add(params: VectorIndexAddParams, options?: RPCOptions): Promise<{
|
|
36
|
+
length: number;
|
|
37
|
+
}>;
|
|
38
|
+
/**
|
|
39
|
+
* Finds the `k` nearest entries to one query. Hits are ordered by
|
|
40
|
+
* descending score, then ascending id, and fewer than `k` are returned
|
|
41
|
+
* when the index holds fewer entries.
|
|
42
|
+
* @overloadLabel "Single query"
|
|
43
|
+
*/
|
|
44
|
+
search(params: {
|
|
45
|
+
query: number[];
|
|
46
|
+
k: number;
|
|
47
|
+
}, options?: RPCOptions): Promise<VectorIndexHit[]>;
|
|
48
|
+
/**
|
|
49
|
+
* Finds the `k` nearest entries to each query row; one hit list per row.
|
|
50
|
+
* @overloadLabel "Multiple queries"
|
|
51
|
+
*/
|
|
52
|
+
search(params: {
|
|
53
|
+
queries: number[][];
|
|
54
|
+
k: number;
|
|
55
|
+
}, options?: RPCOptions): Promise<VectorIndexHit[][]>;
|
|
56
|
+
/**
|
|
57
|
+
* Removes ids; each boolean tells whether that id was present. Ids are
|
|
58
|
+
* applied in order, so a rejection leaves the ids before the failure
|
|
59
|
+
* removed and `length` reporting the last successful call.
|
|
60
|
+
*/
|
|
61
|
+
remove(params: VectorIndexIdsParams, options?: RPCOptions): Promise<boolean[]>;
|
|
62
|
+
/** Reports, per id, whether the index holds it. */
|
|
63
|
+
contains(params: VectorIndexIdsParams, options?: RPCOptions): Promise<boolean[]>;
|
|
64
|
+
/**
|
|
65
|
+
* Writes a checksummed snapshot to `path`, replacing any existing file.
|
|
66
|
+
* An absolute path is used as given; a relative path resolves under the
|
|
67
|
+
* QVAC data directory. Load it later with `loadVectorIndex`.
|
|
68
|
+
* @returns The resolved absolute path.
|
|
69
|
+
*/
|
|
70
|
+
write(params: VectorIndexWriteParams, options?: RPCOptions): Promise<{
|
|
71
|
+
path: string;
|
|
72
|
+
}>;
|
|
73
|
+
/** Releases worker memory. Safe to call more than once. */
|
|
74
|
+
dispose(options?: RPCOptions): Promise<void>;
|
|
75
|
+
[Symbol.asyncDispose](): Promise<void>;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Builds the two public functions over a transport. The default export pair
|
|
79
|
+
* below binds them to the engine dispatcher; tests bind a fake instead.
|
|
80
|
+
*/
|
|
81
|
+
export declare function createVectorIndexClient(transport: typeof send): {
|
|
82
|
+
createVectorIndex(params: CreateVectorIndexParams, options?: RPCOptions): Promise<VectorIndex>;
|
|
83
|
+
loadVectorIndex(params: LoadVectorIndexParams, options?: RPCOptions): Promise<VectorIndex>;
|
|
84
|
+
};
|
|
85
|
+
/**
|
|
86
|
+
* Creates an empty vector index in worker memory and returns a handle to it.
|
|
87
|
+
*
|
|
88
|
+
* The index is backed by the native TurboVec engine of the registered
|
|
89
|
+
* embedding plugin (the built-in llama.cpp embedding plugin provides it).
|
|
90
|
+
* Pair it with `embed()` to build retrieval over documents kept in any store:
|
|
91
|
+
* add each document's embedding under an id of your choosing, then search
|
|
92
|
+
* with the embedding of a query and map the returned ids back to your
|
|
93
|
+
* documents.
|
|
94
|
+
*
|
|
95
|
+
* @param params - Index parameters
|
|
96
|
+
* @param params.dim - Vector dimensionality, for example 1024 for GTE Large. TurboVec storage needs a multiple of 8 no greater than 1024.
|
|
97
|
+
* @param params.storage - Storage mode from `VectorIndexStorage`; defaults to `TURBOVEC_Q4`. `TURBOVEC_Q2` halves memory again at lower recall; `F32`, `Q8`, and `Q4` are exact or generically quantised stores for dimensions TurboVec cannot hold.
|
|
98
|
+
* @param options - Optional RPC options (timeout, profiling, force new connection, etc.).
|
|
99
|
+
* @returns A handle whose methods add, search, remove, snapshot, and dispose the index.
|
|
100
|
+
* @throws {VectorIndexProviderUnavailableError} When no registered plugin exposes a vector index provider
|
|
101
|
+
* @throws {VectorIndexFailedError} When the native index rejects the dimension or storage mode
|
|
102
|
+
*
|
|
103
|
+
* @example
|
|
104
|
+
* ```typescript
|
|
105
|
+
* const documents = new Map([
|
|
106
|
+
* ['1', 'Saturn moon Titan has lakes of liquid methane.'],
|
|
107
|
+
* ['2', 'Solar panels turn sunlight into electricity.']
|
|
108
|
+
* ])
|
|
109
|
+
* const { embedding } = await embed({ modelId, text: [...documents.values()] })
|
|
110
|
+
*
|
|
111
|
+
* await using index = await createVectorIndex({ dim: embedding[0].length })
|
|
112
|
+
* await index.add({ ids: [...documents.keys()], vectors: embedding })
|
|
113
|
+
*
|
|
114
|
+
* const { embedding: query } = await embed({ modelId, text: 'Which moon has methane lakes?' })
|
|
115
|
+
* const hits = await index.search({ query, k: 1 })
|
|
116
|
+
* console.log(documents.get(hits[0].id))
|
|
117
|
+
* ```
|
|
118
|
+
*/
|
|
119
|
+
export declare function createVectorIndex(params: CreateVectorIndexParams, options?: RPCOptions): Promise<VectorIndex>;
|
|
120
|
+
/**
|
|
121
|
+
* Loads a vector index snapshot written by `VectorIndex.write()` into worker
|
|
122
|
+
* memory and returns a handle to it.
|
|
123
|
+
*
|
|
124
|
+
* The snapshot records its dimension, storage mode, and entries, so nothing
|
|
125
|
+
* else needs to be supplied. Loading copies the file into memory; later
|
|
126
|
+
* changes are not written back until `write()` is called again.
|
|
127
|
+
*
|
|
128
|
+
* @param params - Load parameters
|
|
129
|
+
* @param params.path - Snapshot path. An absolute path is used as given; a relative path resolves under the QVAC data directory, the same rule `write()` applies.
|
|
130
|
+
* @param options - Optional RPC options (timeout, profiling, force new connection, etc.).
|
|
131
|
+
* @returns A handle to the loaded index.
|
|
132
|
+
* @throws {VectorIndexProviderUnavailableError} When no registered plugin exposes a vector index provider
|
|
133
|
+
* @throws {VectorIndexFailedError} When the file is missing, corrupt, or not a vector index snapshot
|
|
134
|
+
*
|
|
135
|
+
* @example
|
|
136
|
+
* ```typescript
|
|
137
|
+
* const index = await loadVectorIndex({ path: 'indexes/articles.qvi' })
|
|
138
|
+
* const hits = await index.search({ query, k: 5 })
|
|
139
|
+
* await index.dispose()
|
|
140
|
+
* ```
|
|
141
|
+
*/
|
|
142
|
+
export declare function loadVectorIndex(params: LoadVectorIndexParams, options?: RPCOptions): Promise<VectorIndex>;
|