@qvac/inference 0.19.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/NOTICE +95 -39
  2. package/dist/api/audio-gen.d.ts +42 -1
  3. package/dist/api/audio-gen.js +137 -51
  4. package/dist/api/delete-cache.d.ts +7 -1
  5. package/dist/api/delete-cache.js +11 -1
  6. package/dist/api/index.d.ts +2 -1
  7. package/dist/api/index.js +2 -1
  8. package/dist/api/text-to-speech.d.ts +31 -4
  9. package/dist/api/text-to-speech.js +117 -21
  10. package/dist/api/vector-index.d.ts +142 -0
  11. package/dist/api/vector-index.js +168 -0
  12. package/dist/api/world.d.ts +2 -2
  13. package/dist/errors/index.d.ts +12 -0
  14. package/dist/errors/index.js +21 -0
  15. package/dist/handlers/delete-cache.js +3 -0
  16. package/dist/handlers/vector-index.d.ts +2 -0
  17. package/dist/handlers/vector-index.js +40 -0
  18. package/dist/index.d.ts +1 -1
  19. package/dist/index.js +1 -1
  20. package/dist/models/registry/models.d.ts +351 -12
  21. package/dist/models/registry/models.js +2271 -1957
  22. package/dist/models/registry/resource-profiles.js +135 -0
  23. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
  24. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
  25. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
  26. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
  27. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
  28. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
  29. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
  30. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
  31. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
  32. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
  33. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
  34. package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
  35. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
  36. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
  37. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
  38. package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
  39. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
  40. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
  41. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
  42. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
  43. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
  44. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
  45. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
  46. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
  47. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
  48. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +48 -27
  49. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +239 -135
  50. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
  51. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
  52. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +11 -4
  53. package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
  54. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
  55. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
  56. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
  57. package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
  58. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
  59. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
  60. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
  61. package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
  62. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
  63. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
  64. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
  65. package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
  66. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
  67. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
  68. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
  69. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
  70. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
  71. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
  72. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
  73. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
  74. package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
  75. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
  76. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
  77. package/dist/plugins/index.d.ts +1 -1
  78. package/dist/plugins/index.js +1 -1
  79. package/dist/plugins/ops/load-model.js +22 -2
  80. package/dist/plugins/ops/translate.js +58 -12
  81. package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
  82. package/dist/plugins/ops/translatepsy-utils.js +32 -0
  83. package/dist/plugins/registry.d.ts +15 -0
  84. package/dist/plugins/registry.js +133 -15
  85. package/dist/plugins/turbovec-backend.d.ts +17 -0
  86. package/dist/plugins/turbovec-backend.js +99 -0
  87. package/dist/registry.js +4 -0
  88. package/dist/resources/model-fit/assess.js +169 -68
  89. package/dist/resources/model-fit/estimators/llm.d.ts +1 -0
  90. package/dist/resources/model-fit/estimators/llm.js +3 -1
  91. package/dist/resources/model-fit/floor.d.ts +31 -0
  92. package/dist/resources/model-fit/floor.js +65 -0
  93. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
  94. package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
  95. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
  96. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
  97. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
  98. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
  99. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
  100. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
  101. package/dist/resources/native.js +5 -4
  102. package/dist/resources/types.d.ts +1 -3
  103. package/dist/runtime/env.d.ts +2 -0
  104. package/dist/runtime/env.js +9 -2
  105. package/dist/runtime/lifecycle.js +2 -0
  106. package/dist/runtime/model-registry.d.ts +3 -1
  107. package/dist/runtime/model-registry.js +2 -1
  108. package/dist/runtime/request-context.js +17 -0
  109. package/dist/runtime/vector-index-backend.d.ts +25 -0
  110. package/dist/runtime/vector-index-backend.js +1 -0
  111. package/dist/runtime/vector-index-registry.d.ts +76 -0
  112. package/dist/runtime/vector-index-registry.js +161 -0
  113. package/dist/schemas/assess-model-fit.d.ts +84 -0
  114. package/dist/schemas/assess-model-fit.js +72 -3
  115. package/dist/schemas/audio-gen.d.ts +482 -7
  116. package/dist/schemas/audio-gen.js +376 -17
  117. package/dist/schemas/batch-completion-stream.d.ts +10 -0
  118. package/dist/schemas/batch-completion-stream.js +5 -3
  119. package/dist/schemas/bci.d.ts +1 -1
  120. package/dist/schemas/common.d.ts +782 -19
  121. package/dist/schemas/common.js +8 -1
  122. package/dist/schemas/completion-event.d.ts +14 -1
  123. package/dist/schemas/completion-event.js +4 -0
  124. package/dist/schemas/completion-stream.d.ts +22 -0
  125. package/dist/schemas/completion-stream.js +41 -4
  126. package/dist/schemas/config.d.ts +48 -30
  127. package/dist/schemas/errors.d.ts +4 -0
  128. package/dist/schemas/errors.js +22 -0
  129. package/dist/schemas/index.d.ts +1 -0
  130. package/dist/schemas/index.js +1 -0
  131. package/dist/schemas/llamacpp-config.d.ts +4 -8
  132. package/dist/schemas/llamacpp-config.js +4 -4
  133. package/dist/schemas/load-model.d.ts +2604 -180
  134. package/dist/schemas/model-config-schemas.d.ts +228 -5
  135. package/dist/schemas/model-ops.d.ts +3 -0
  136. package/dist/schemas/model-ops.js +6 -0
  137. package/dist/schemas/plugin.d.ts +21 -11
  138. package/dist/schemas/sdcpp-config.d.ts +221 -4
  139. package/dist/schemas/sdcpp-config.js +128 -22
  140. package/dist/schemas/text-to-speech.d.ts +780 -19
  141. package/dist/schemas/text-to-speech.js +298 -44
  142. package/dist/schemas/transcription-config.js +7 -4
  143. package/dist/schemas/vector-index.d.ts +194 -0
  144. package/dist/schemas/vector-index.js +185 -0
  145. package/dist/utils/addon-responses.d.ts +30 -3
  146. package/dist/utils/aggregate-events.d.ts +3 -1
  147. package/dist/utils/aggregate-events.js +7 -1
  148. package/dist/utils/tts-stats.d.ts +15 -11
  149. package/dist/utils/tts-stats.js +33 -11
  150. package/package.json +29 -26
@@ -2,18 +2,42 @@ import { generateConfigHash } from '../../../ops/kv-cache-utils.js';
2
2
  import { type CacheMessage } from '../../../../utils/index.js';
3
3
  import type { Logger } from '../../../../logging/types.js';
4
4
  import { type AbortSignal } from 'bare-abort-controller';
5
+ import { z } from 'zod';
6
+ /**
7
+ * Coordinates six KV-cache state layers:
8
+ *
9
+ * 1. `cachedPrefixes` — saved message boundaries, and whether a tool
10
+ * block was rendered into them.
11
+ * 2. `initializedCaches` — caches this process has established.
12
+ * 3. On-disk `.bin` files written by the addon.
13
+ * 4. `<bin>.meta.json` sidecars — layer 1 persisted, so a named cache's
14
+ * boundary outlives the process; see `readPrefixSidecar`.
15
+ * 5. `activeCachePaths` — per-path refs that block in-flight eviction.
16
+ * 6. `.auto-cache-<key>` markers — engine-generated cache ownership.
17
+ *
18
+ * Every turn must finish through `commitTurn`, `rollback`, or the
19
+ * non-destructive `releaseTurn` so all inference state stays aligned,
20
+ * the active-path ref is released, and marker metadata follows the
21
+ * cache directory lifecycle.
22
+ */
23
+ /** What the kv-cache file at a given path is known to hold. */
24
+ declare const cachedPrefixSchema: z.ZodObject<{
25
+ messages: z.ZodNumber;
26
+ toolBlock: z.ZodBoolean;
27
+ }, z.core.$strip>;
28
+ type CachedPrefix = z.infer<typeof cachedPrefixSchema>;
5
29
  export interface TurnHandle {
6
30
  /** Resolved on-disk cache file path the addon will read from / write to. */
7
31
  readonly cachePath: string;
8
32
  /**
9
33
  * Snapshot of the on-disk saved-message count at `beginTurn` time
10
- * (0 if the cache was just primed). Consumed by `decideCachedHistorySlice`
11
- * to pick the message tail for the next addon call.
34
+ * (0 on a cold turn). Consumed by `decideCachedHistorySlice` to pick the
35
+ * message tail for the next addon call.
12
36
  */
13
37
  readonly savedCount: number;
14
38
  /**
15
39
  * Whether the cached prefix already holds a rendered static tool block, so
16
- * this turn can leave it out of its payload. False on a fresh prime and
40
+ * this turn can leave it out of its payload. False on a cold turn and
17
41
  * whenever the previous turn couldn't confirm the block reached the model.
18
42
  */
19
43
  readonly toolBlockCached: boolean;
@@ -24,14 +48,6 @@ export interface BeginCustomTurnInput {
24
48
  customKey: string;
25
49
  /** Hash of system prompt + complete tool definitions. */
26
50
  configHash: string;
27
- /**
28
- * Prime the cache by sending the system prompt to the addon. Tools are not
29
- * primed — a prefix with no user turn is not a renderable conversation for
30
- * every template — so they travel with a turn instead. Called when the cache
31
- * doesn't exist in-memory OR on disk. Kept as an injected closure so this
32
- * module has no dependency on the model registry / addon.
33
- */
34
- primeIfMissing: (cachePath: string) => Promise<void>;
35
51
  /**
36
52
  * Request abort signal. When it aborts while this turn is queued behind a
37
53
  * same-file peer's write lock, the wait is abandoned so the request's scope
@@ -45,8 +61,6 @@ export interface BeginAutoTurnInput {
45
61
  configHash: string;
46
62
  /** Conversation history used to compute the pre-response cache key. */
47
63
  history: CacheMessage[];
48
- /** See `BeginCustomTurnInput.primeIfMissing`. */
49
- primeIfMissing: (cachePath: string) => Promise<void>;
50
64
  /** See `BeginCustomTurnInput.signal`. */
51
65
  signal?: AbortSignal;
52
66
  }
@@ -80,10 +94,10 @@ export type CommitResult = StaticCommitResult | AutoRenameCommitResult;
80
94
  export interface KvCacheSession {
81
95
  /**
82
96
  * Open a new turn against the cache. Resolves the cache file path,
83
- * primes the system-prompt cache if needed (delegated to
84
- * `input.primeIfMissing`), marks the cache initialized, and returns a
85
- * `TurnHandle` the handler attaches to `ctx.scope.defer(...)` for the
86
- * rollback hook. Auto-cache path resolution is serialized with
97
+ * ensures its parent directory exists, and returns a `TurnHandle` the
98
+ * handler attaches to `ctx.scope.defer(...)` for the rollback hook. The
99
+ * addon's own `saveCacheToDisk` is what writes the file, so a cold turn
100
+ * opens without one. Auto-cache path resolution is serialized with
87
101
  * retention deletion before the handle is returned.
88
102
  */
89
103
  beginTurn(input: BeginTurnInput): Promise<TurnHandle>;
@@ -109,17 +123,17 @@ export interface KvCacheSession {
109
123
  /**
110
124
  * Non-destructive counterpart of `rollback`: releases locks and refs but
111
125
  * keeps the committed disk cache and its recorded prefix valid for a retry.
112
- * A cache freshly primed by this same turn rolls back instead.
126
+ * A cache this same turn created rolls back instead.
113
127
  */
114
128
  releaseTurn(turn: TurnHandle): Promise<void>;
115
129
  /**
116
- * Forget the in-memory saved-message count for the turn's path
117
- * without unlinking the file or clearing the init flag. Used when
118
- * `decideCachedHistorySlice` detects a stale boundary
130
+ * Forget the saved-message count for the turn's path, in memory and in the
131
+ * sidecar, without unlinking the cache file or clearing the init flag. Used
132
+ * when `decideCachedHistorySlice` detects a stale boundary
119
133
  * (`clearStaleCount: true`) — the next turn re-sends the full history
120
134
  * but the cache itself is still usable.
121
135
  */
122
- dropStaleSavedCount(turn: TurnHandle): void;
136
+ dropStaleSavedCount(turn: TurnHandle): Promise<void>;
123
137
  }
124
138
  /**
125
139
  * Construct a session bound to one `(modelId, turn-owning request)`
@@ -132,9 +146,10 @@ export declare function createKvCacheSession(modelId: string, options?: {
132
146
  }): KvCacheSession;
133
147
  /**
134
148
  * Atomically delete every layer of KV-cache state for a
135
- * `(kvCacheKey, modelId)` pair, or wipe everything. Single entry point
136
- * — the only mutation point for cross-model state outside of
137
- * turn-scoped `commitTurn`/`rollback`.
149
+ * `(kvCacheKey, modelId)` pair, wipe everything, or reclaim the auto
150
+ * caches no turn is holding. Single entry point — the only mutation
151
+ * point for cross-model state outside of turn-scoped
152
+ * `commitTurn`/`rollback`.
138
153
  *
139
154
  * Why this isn't a method on `KvCacheSession`: deletes are
140
155
  * cross-model (`all: true` has no model; the keyed form has
@@ -159,14 +174,18 @@ export declare function createKvCacheSession(modelId: string, options?: {
159
174
  * cleared without a lock, so an interleaving that lands the delete's
160
175
  * in-memory cleanup after a concurrent turn has already renamed/committed
161
176
  * its file splits state: the file stays on disk while its saved-count and
162
- * init flag are cleared, so the next turn sees the file, skips priming, and
163
- * reports `savedCount=0`. Callers must not delete a key that is in active use.
177
+ * init flag are cleared, so the next turn loads the file and reports
178
+ * `savedCount=0`. Callers must not delete a key that is in active use.
179
+ * The `auto` target is the exception: it skips keys an in-flight turn holds
180
+ * and clears each one under the cache-state lock.
164
181
  */
165
182
  export declare function deleteKvCacheState(target: {
166
183
  kvCacheKey: string;
167
184
  modelId?: string;
168
185
  } | {
169
186
  all: true;
187
+ } | {
188
+ auto: true;
170
189
  }): Promise<void>;
171
190
  /**
172
191
  * Test-only access to the module-scoped state. Production code reaches
@@ -181,6 +200,8 @@ export declare const __kvCacheSessionTestHooks: {
181
200
  getSavedCount(cachePath: string): number | undefined;
182
201
  setSavedCountForTest(cachePath: string, count: number): void;
183
202
  getToolBlockCachedForTest(cachePath: string): boolean;
203
+ getPrefixSidecarPathForTest(cachePath: string): string;
204
+ readPrefixSidecarForTest(cachePath: string): Promise<CachedPrefix | null>;
184
205
  hasInitializedPath(cachePath: string): boolean;
185
206
  getActivePathCountForTest(cachePath: string): number;
186
207
  getLastAutoCacheSweepMsForTest(): number;