@qvac/inference 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +95 -39
- package/dist/api/audio-gen.d.ts +42 -1
- package/dist/api/audio-gen.js +137 -51
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/text-to-speech.d.ts +31 -4
- package/dist/api/text-to-speech.js +117 -21
- package/dist/api/vector-index.d.ts +142 -0
- package/dist/api/vector-index.js +168 -0
- package/dist/api/world.d.ts +2 -2
- package/dist/errors/index.d.ts +12 -0
- package/dist/errors/index.js +21 -0
- package/dist/handlers/delete-cache.js +3 -0
- package/dist/handlers/vector-index.d.ts +2 -0
- package/dist/handlers/vector-index.js +40 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +351 -12
- package/dist/models/registry/models.js +2271 -1957
- package/dist/models/registry/resource-profiles.js +135 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +48 -27
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +239 -135
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +11 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
- package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins/ops/load-model.js +22 -2
- package/dist/plugins/ops/translate.js +58 -12
- package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
- package/dist/plugins/ops/translatepsy-utils.js +32 -0
- package/dist/plugins/registry.d.ts +15 -0
- package/dist/plugins/registry.js +133 -15
- package/dist/plugins/turbovec-backend.d.ts +17 -0
- package/dist/plugins/turbovec-backend.js +99 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.js +169 -68
- package/dist/resources/model-fit/estimators/llm.d.ts +1 -0
- package/dist/resources/model-fit/estimators/llm.js +3 -1
- package/dist/resources/model-fit/floor.d.ts +31 -0
- package/dist/resources/model-fit/floor.js +65 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
- package/dist/resources/native.js +5 -4
- package/dist/resources/types.d.ts +1 -3
- package/dist/runtime/env.d.ts +2 -0
- package/dist/runtime/env.js +9 -2
- package/dist/runtime/lifecycle.js +2 -0
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/request-context.js +17 -0
- package/dist/runtime/vector-index-backend.d.ts +25 -0
- package/dist/runtime/vector-index-backend.js +1 -0
- package/dist/runtime/vector-index-registry.d.ts +76 -0
- package/dist/runtime/vector-index-registry.js +161 -0
- package/dist/schemas/assess-model-fit.d.ts +84 -0
- package/dist/schemas/assess-model-fit.js +72 -3
- package/dist/schemas/audio-gen.d.ts +482 -7
- package/dist/schemas/audio-gen.js +376 -17
- package/dist/schemas/batch-completion-stream.d.ts +10 -0
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci.d.ts +1 -1
- package/dist/schemas/common.d.ts +782 -19
- package/dist/schemas/common.js +8 -1
- package/dist/schemas/completion-event.d.ts +14 -1
- package/dist/schemas/completion-event.js +4 -0
- package/dist/schemas/completion-stream.d.ts +22 -0
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +48 -30
- package/dist/schemas/errors.d.ts +4 -0
- package/dist/schemas/errors.js +22 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +4 -8
- package/dist/schemas/llamacpp-config.js +4 -4
- package/dist/schemas/load-model.d.ts +2604 -180
- package/dist/schemas/model-config-schemas.d.ts +228 -5
- package/dist/schemas/model-ops.d.ts +3 -0
- package/dist/schemas/model-ops.js +6 -0
- package/dist/schemas/plugin.d.ts +21 -11
- package/dist/schemas/sdcpp-config.d.ts +221 -4
- package/dist/schemas/sdcpp-config.js +128 -22
- package/dist/schemas/text-to-speech.d.ts +780 -19
- package/dist/schemas/text-to-speech.js +298 -44
- package/dist/schemas/transcription-config.js +7 -4
- package/dist/schemas/vector-index.d.ts +194 -0
- package/dist/schemas/vector-index.js +185 -0
- package/dist/utils/addon-responses.d.ts +30 -3
- package/dist/utils/aggregate-events.d.ts +3 -1
- package/dist/utils/aggregate-events.js +7 -1
- package/dist/utils/tts-stats.d.ts +15 -11
- package/dist/utils/tts-stats.js +33 -11
- package/package.json +29 -26
|
@@ -2,18 +2,42 @@ import { generateConfigHash } from '../../../ops/kv-cache-utils.js';
|
|
|
2
2
|
import { type CacheMessage } from '../../../../utils/index.js';
|
|
3
3
|
import type { Logger } from '../../../../logging/types.js';
|
|
4
4
|
import { type AbortSignal } from 'bare-abort-controller';
|
|
5
|
+
import { z } from 'zod';
|
|
6
|
+
/**
|
|
7
|
+
* Coordinates six KV-cache state layers:
|
|
8
|
+
*
|
|
9
|
+
* 1. `cachedPrefixes` — saved message boundaries, and whether a tool
|
|
10
|
+
* block was rendered into them.
|
|
11
|
+
* 2. `initializedCaches` — caches this process has established.
|
|
12
|
+
* 3. On-disk `.bin` files written by the addon.
|
|
13
|
+
* 4. `<bin>.meta.json` sidecars — layer 1 persisted, so a named cache's
|
|
14
|
+
* boundary outlives the process; see `readPrefixSidecar`.
|
|
15
|
+
* 5. `activeCachePaths` — per-path refs that block in-flight eviction.
|
|
16
|
+
* 6. `.auto-cache-<key>` markers — engine-generated cache ownership.
|
|
17
|
+
*
|
|
18
|
+
* Every turn must finish through `commitTurn`, `rollback`, or the
|
|
19
|
+
* non-destructive `releaseTurn` so all inference state stays aligned,
|
|
20
|
+
* the active-path ref is released, and marker metadata follows the
|
|
21
|
+
* cache directory lifecycle.
|
|
22
|
+
*/
|
|
23
|
+
/** What the kv-cache file at a given path is known to hold. */
|
|
24
|
+
declare const cachedPrefixSchema: z.ZodObject<{
|
|
25
|
+
messages: z.ZodNumber;
|
|
26
|
+
toolBlock: z.ZodBoolean;
|
|
27
|
+
}, z.core.$strip>;
|
|
28
|
+
type CachedPrefix = z.infer<typeof cachedPrefixSchema>;
|
|
5
29
|
export interface TurnHandle {
|
|
6
30
|
/** Resolved on-disk cache file path the addon will read from / write to. */
|
|
7
31
|
readonly cachePath: string;
|
|
8
32
|
/**
|
|
9
33
|
* Snapshot of the on-disk saved-message count at `beginTurn` time
|
|
10
|
-
* (0
|
|
11
|
-
*
|
|
34
|
+
* (0 on a cold turn). Consumed by `decideCachedHistorySlice` to pick the
|
|
35
|
+
* message tail for the next addon call.
|
|
12
36
|
*/
|
|
13
37
|
readonly savedCount: number;
|
|
14
38
|
/**
|
|
15
39
|
* Whether the cached prefix already holds a rendered static tool block, so
|
|
16
|
-
* this turn can leave it out of its payload. False on a
|
|
40
|
+
* this turn can leave it out of its payload. False on a cold turn and
|
|
17
41
|
* whenever the previous turn couldn't confirm the block reached the model.
|
|
18
42
|
*/
|
|
19
43
|
readonly toolBlockCached: boolean;
|
|
@@ -24,14 +48,6 @@ export interface BeginCustomTurnInput {
|
|
|
24
48
|
customKey: string;
|
|
25
49
|
/** Hash of system prompt + complete tool definitions. */
|
|
26
50
|
configHash: string;
|
|
27
|
-
/**
|
|
28
|
-
* Prime the cache by sending the system prompt to the addon. Tools are not
|
|
29
|
-
* primed — a prefix with no user turn is not a renderable conversation for
|
|
30
|
-
* every template — so they travel with a turn instead. Called when the cache
|
|
31
|
-
* doesn't exist in-memory OR on disk. Kept as an injected closure so this
|
|
32
|
-
* module has no dependency on the model registry / addon.
|
|
33
|
-
*/
|
|
34
|
-
primeIfMissing: (cachePath: string) => Promise<void>;
|
|
35
51
|
/**
|
|
36
52
|
* Request abort signal. When it aborts while this turn is queued behind a
|
|
37
53
|
* same-file peer's write lock, the wait is abandoned so the request's scope
|
|
@@ -45,8 +61,6 @@ export interface BeginAutoTurnInput {
|
|
|
45
61
|
configHash: string;
|
|
46
62
|
/** Conversation history used to compute the pre-response cache key. */
|
|
47
63
|
history: CacheMessage[];
|
|
48
|
-
/** See `BeginCustomTurnInput.primeIfMissing`. */
|
|
49
|
-
primeIfMissing: (cachePath: string) => Promise<void>;
|
|
50
64
|
/** See `BeginCustomTurnInput.signal`. */
|
|
51
65
|
signal?: AbortSignal;
|
|
52
66
|
}
|
|
@@ -80,10 +94,10 @@ export type CommitResult = StaticCommitResult | AutoRenameCommitResult;
|
|
|
80
94
|
export interface KvCacheSession {
|
|
81
95
|
/**
|
|
82
96
|
* Open a new turn against the cache. Resolves the cache file path,
|
|
83
|
-
*
|
|
84
|
-
* `
|
|
85
|
-
* `
|
|
86
|
-
*
|
|
97
|
+
* ensures its parent directory exists, and returns a `TurnHandle` the
|
|
98
|
+
* handler attaches to `ctx.scope.defer(...)` for the rollback hook. The
|
|
99
|
+
* addon's own `saveCacheToDisk` is what writes the file, so a cold turn
|
|
100
|
+
* opens without one. Auto-cache path resolution is serialized with
|
|
87
101
|
* retention deletion before the handle is returned.
|
|
88
102
|
*/
|
|
89
103
|
beginTurn(input: BeginTurnInput): Promise<TurnHandle>;
|
|
@@ -109,17 +123,17 @@ export interface KvCacheSession {
|
|
|
109
123
|
/**
|
|
110
124
|
* Non-destructive counterpart of `rollback`: releases locks and refs but
|
|
111
125
|
* keeps the committed disk cache and its recorded prefix valid for a retry.
|
|
112
|
-
* A cache
|
|
126
|
+
* A cache this same turn created rolls back instead.
|
|
113
127
|
*/
|
|
114
128
|
releaseTurn(turn: TurnHandle): Promise<void>;
|
|
115
129
|
/**
|
|
116
|
-
* Forget the
|
|
117
|
-
* without unlinking the file or clearing the init flag. Used
|
|
118
|
-
* `decideCachedHistorySlice` detects a stale boundary
|
|
130
|
+
* Forget the saved-message count for the turn's path, in memory and in the
|
|
131
|
+
* sidecar, without unlinking the cache file or clearing the init flag. Used
|
|
132
|
+
* when `decideCachedHistorySlice` detects a stale boundary
|
|
119
133
|
* (`clearStaleCount: true`) — the next turn re-sends the full history
|
|
120
134
|
* but the cache itself is still usable.
|
|
121
135
|
*/
|
|
122
|
-
dropStaleSavedCount(turn: TurnHandle): void
|
|
136
|
+
dropStaleSavedCount(turn: TurnHandle): Promise<void>;
|
|
123
137
|
}
|
|
124
138
|
/**
|
|
125
139
|
* Construct a session bound to one `(modelId, turn-owning request)`
|
|
@@ -132,9 +146,10 @@ export declare function createKvCacheSession(modelId: string, options?: {
|
|
|
132
146
|
}): KvCacheSession;
|
|
133
147
|
/**
|
|
134
148
|
* Atomically delete every layer of KV-cache state for a
|
|
135
|
-
* `(kvCacheKey, modelId)` pair,
|
|
136
|
-
*
|
|
137
|
-
* turn-scoped
|
|
149
|
+
* `(kvCacheKey, modelId)` pair, wipe everything, or reclaim the auto
|
|
150
|
+
* caches no turn is holding. Single entry point — the only mutation
|
|
151
|
+
* point for cross-model state outside of turn-scoped
|
|
152
|
+
* `commitTurn`/`rollback`.
|
|
138
153
|
*
|
|
139
154
|
* Why this isn't a method on `KvCacheSession`: deletes are
|
|
140
155
|
* cross-model (`all: true` has no model; the keyed form has
|
|
@@ -159,14 +174,18 @@ export declare function createKvCacheSession(modelId: string, options?: {
|
|
|
159
174
|
* cleared without a lock, so an interleaving that lands the delete's
|
|
160
175
|
* in-memory cleanup after a concurrent turn has already renamed/committed
|
|
161
176
|
* its file splits state: the file stays on disk while its saved-count and
|
|
162
|
-
* init flag are cleared, so the next turn
|
|
163
|
-
*
|
|
177
|
+
* init flag are cleared, so the next turn loads the file and reports
|
|
178
|
+
* `savedCount=0`. Callers must not delete a key that is in active use.
|
|
179
|
+
* The `auto` target is the exception: it skips keys an in-flight turn holds
|
|
180
|
+
* and clears each one under the cache-state lock.
|
|
164
181
|
*/
|
|
165
182
|
export declare function deleteKvCacheState(target: {
|
|
166
183
|
kvCacheKey: string;
|
|
167
184
|
modelId?: string;
|
|
168
185
|
} | {
|
|
169
186
|
all: true;
|
|
187
|
+
} | {
|
|
188
|
+
auto: true;
|
|
170
189
|
}): Promise<void>;
|
|
171
190
|
/**
|
|
172
191
|
* Test-only access to the module-scoped state. Production code reaches
|
|
@@ -181,6 +200,8 @@ export declare const __kvCacheSessionTestHooks: {
|
|
|
181
200
|
getSavedCount(cachePath: string): number | undefined;
|
|
182
201
|
setSavedCountForTest(cachePath: string, count: number): void;
|
|
183
202
|
getToolBlockCachedForTest(cachePath: string): boolean;
|
|
203
|
+
getPrefixSidecarPathForTest(cachePath: string): string;
|
|
204
|
+
readPrefixSidecarForTest(cachePath: string): Promise<CachedPrefix | null>;
|
|
184
205
|
hasInitializedPath(cachePath: string): boolean;
|
|
185
206
|
getActivePathCountForTest(cachePath: string): number;
|
|
186
207
|
getLastAutoCacheSweepMsForTest(): number;
|