@qvac/inference 0.19.1 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/NOTICE +43 -11
  2. package/dist/api/audio-gen.d.ts +42 -1
  3. package/dist/api/audio-gen.js +137 -51
  4. package/dist/api/index.d.ts +2 -1
  5. package/dist/api/index.js +2 -1
  6. package/dist/api/text-to-speech.d.ts +31 -4
  7. package/dist/api/text-to-speech.js +117 -21
  8. package/dist/api/vector-index.d.ts +142 -0
  9. package/dist/api/vector-index.js +168 -0
  10. package/dist/api/world.d.ts +2 -2
  11. package/dist/errors/index.d.ts +12 -0
  12. package/dist/errors/index.js +21 -0
  13. package/dist/handlers/vector-index.d.ts +2 -0
  14. package/dist/handlers/vector-index.js +40 -0
  15. package/dist/index.d.ts +1 -1
  16. package/dist/index.js +1 -1
  17. package/dist/models/registry/models.d.ts +351 -12
  18. package/dist/models/registry/models.js +2271 -1957
  19. package/dist/models/registry/resource-profiles.js +135 -0
  20. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
  21. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
  22. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
  23. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
  24. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
  25. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
  26. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
  27. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
  28. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
  29. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
  30. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
  31. package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
  32. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
  33. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
  34. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
  35. package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
  36. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
  37. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
  38. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
  39. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
  40. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
  41. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
  42. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
  43. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
  44. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
  45. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
  46. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
  47. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
  48. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
  49. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +8 -4
  50. package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
  51. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
  52. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
  53. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
  54. package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
  55. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
  56. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
  57. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
  58. package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
  59. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
  60. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
  61. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
  62. package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
  63. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
  64. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
  65. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
  66. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
  67. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
  68. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
  69. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
  70. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
  71. package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
  72. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
  73. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
  74. package/dist/plugins/index.d.ts +1 -1
  75. package/dist/plugins/index.js +1 -1
  76. package/dist/plugins/ops/load-model.js +22 -2
  77. package/dist/plugins/ops/translate.js +58 -12
  78. package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
  79. package/dist/plugins/ops/translatepsy-utils.js +32 -0
  80. package/dist/plugins/registry.d.ts +15 -0
  81. package/dist/plugins/registry.js +133 -15
  82. package/dist/plugins/turbovec-backend.d.ts +17 -0
  83. package/dist/plugins/turbovec-backend.js +99 -0
  84. package/dist/registry.js +4 -0
  85. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
  86. package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
  87. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
  88. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
  89. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
  90. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
  91. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
  92. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
  93. package/dist/runtime/env.d.ts +2 -0
  94. package/dist/runtime/env.js +9 -2
  95. package/dist/runtime/lifecycle.js +2 -0
  96. package/dist/runtime/model-registry.d.ts +3 -1
  97. package/dist/runtime/model-registry.js +2 -1
  98. package/dist/runtime/request-context.js +17 -0
  99. package/dist/runtime/vector-index-backend.d.ts +25 -0
  100. package/dist/runtime/vector-index-backend.js +1 -0
  101. package/dist/runtime/vector-index-registry.d.ts +76 -0
  102. package/dist/runtime/vector-index-registry.js +161 -0
  103. package/dist/schemas/assess-model-fit.d.ts +44 -0
  104. package/dist/schemas/assess-model-fit.js +40 -0
  105. package/dist/schemas/audio-gen.d.ts +479 -4
  106. package/dist/schemas/audio-gen.js +376 -17
  107. package/dist/schemas/batch-completion-stream.d.ts +7 -0
  108. package/dist/schemas/batch-completion-stream.js +5 -3
  109. package/dist/schemas/common.d.ts +749 -15
  110. package/dist/schemas/common.js +8 -1
  111. package/dist/schemas/completion-event.d.ts +11 -1
  112. package/dist/schemas/completion-event.js +3 -0
  113. package/dist/schemas/completion-stream.d.ts +20 -0
  114. package/dist/schemas/completion-stream.js +41 -4
  115. package/dist/schemas/config.d.ts +48 -30
  116. package/dist/schemas/errors.d.ts +4 -0
  117. package/dist/schemas/errors.js +22 -0
  118. package/dist/schemas/index.d.ts +1 -0
  119. package/dist/schemas/index.js +1 -0
  120. package/dist/schemas/llamacpp-config.d.ts +4 -8
  121. package/dist/schemas/llamacpp-config.js +4 -4
  122. package/dist/schemas/load-model.d.ts +2503 -139
  123. package/dist/schemas/model-config-schemas.d.ts +228 -5
  124. package/dist/schemas/plugin.d.ts +10 -0
  125. package/dist/schemas/sdcpp-config.d.ts +221 -4
  126. package/dist/schemas/sdcpp-config.js +128 -22
  127. package/dist/schemas/text-to-speech.d.ts +780 -19
  128. package/dist/schemas/text-to-speech.js +298 -44
  129. package/dist/schemas/transcription-config.js +7 -4
  130. package/dist/schemas/vector-index.d.ts +194 -0
  131. package/dist/schemas/vector-index.js +185 -0
  132. package/dist/utils/addon-responses.d.ts +29 -3
  133. package/dist/utils/aggregate-events.d.ts +3 -1
  134. package/dist/utils/aggregate-events.js +7 -1
  135. package/dist/utils/tts-stats.d.ts +15 -11
  136. package/dist/utils/tts-stats.js +33 -11
  137. package/package.json +28 -25
@@ -0,0 +1,161 @@
1
+ import { promises as fsPromises } from 'bare-fs';
2
+ import path from 'bare-path';
3
+ import { QvacErrorBase } from '@qvac/error';
4
+ import { getTurboVecBackend } from '../plugins/turbovec-backend.js';
5
+ import { getConfiguredCacheDir } from './state.js';
6
+ import { generateRandomRequestId } from './request-id.js';
7
+ import { getEngineLogger } from '../logging/index.js';
8
+ import { VectorIndexFailedError, VectorIndexInvalidVectorsError, VectorIndexNotFoundError, VectorIndexProviderUnavailableError } from '../errors/index.js';
9
+ // Worker-side bookkeeping for open indexes: ids, wire-to-native conversion,
10
+ // row validation, snapshot paths, and lifetime. Engine behaviour lives in
11
+ // the backend.
12
+ const indexes = new Map();
13
+ function requireBackend() {
14
+ const backend = getTurboVecBackend();
15
+ if (!backend)
16
+ throw new VectorIndexProviderUnavailableError();
17
+ return backend;
18
+ }
19
+ function getIndex(indexId) {
20
+ const index = indexes.get(indexId);
21
+ if (!index)
22
+ throw new VectorIndexNotFoundError(indexId);
23
+ return index;
24
+ }
25
+ // Backends throw plain TypeError/RangeError/Error values from the native
26
+ // layer; wrap them so the caller receives a coded error with the original
27
+ // message preserved as cause. `describe` adds caller context to the message
28
+ // and is evaluated at throw time, so it can report progress made so far.
29
+ function runBackend(action, describe) {
30
+ try {
31
+ return action();
32
+ }
33
+ catch (error) {
34
+ if (error instanceof QvacErrorBase)
35
+ throw error;
36
+ const detail = error instanceof Error ? error.message : String(error);
37
+ throw new VectorIndexFailedError(describe ? describe(detail) : detail, error);
38
+ }
39
+ }
40
+ /**
41
+ * Snapshot paths are used as given when absolute. A relative path resolves
42
+ * against the QVAC data directory (the parent of the configured cache
43
+ * directory), the same root that holds the RAG workspace stores, so the
44
+ * same relative path works on every platform.
45
+ */
46
+ export function resolveVectorIndexPath(snapshotPath) {
47
+ if (path.isAbsolute(snapshotPath))
48
+ return snapshotPath;
49
+ return path.join(path.dirname(getConfiguredCacheDir()), snapshotPath);
50
+ }
51
+ function flattenRows(rows, dim, label) {
52
+ const flat = new Float32Array(rows.length * dim);
53
+ rows.forEach((row, rowIndex) => {
54
+ if (row.length !== dim) {
55
+ throw new VectorIndexInvalidVectorsError(`${label}[${rowIndex}] has ${row.length} components, expected ${dim}`);
56
+ }
57
+ flat.set(row, rowIndex * dim);
58
+ });
59
+ return flat;
60
+ }
61
+ function toNativeIds(ids) {
62
+ return new BigUint64Array(ids.map((id) => BigInt(id)));
63
+ }
64
+ function track(index) {
65
+ const indexId = generateRandomRequestId();
66
+ indexes.set(indexId, index);
67
+ return indexId;
68
+ }
69
+ export function createVectorIndex(params) {
70
+ const backend = requireBackend();
71
+ const index = runBackend(() => backend.create(params.dim, params.storage));
72
+ const indexId = track(index);
73
+ return { indexId, dim: index.dim, storage: params.storage, length: index.length };
74
+ }
75
+ export function loadVectorIndex(params) {
76
+ const backend = requireBackend();
77
+ const index = runBackend(() => backend.load(resolveVectorIndexPath(params.path)));
78
+ const indexId = track(index);
79
+ return {
80
+ indexId,
81
+ dim: index.dim,
82
+ length: index.length,
83
+ ...(index.storage !== undefined && { storage: index.storage })
84
+ };
85
+ }
86
+ export function addVectors(params) {
87
+ const index = getIndex(params.indexId);
88
+ if (params.ids.length !== params.vectors.length) {
89
+ throw new VectorIndexInvalidVectorsError(`ids has ${params.ids.length} entries but vectors has ${params.vectors.length} rows`);
90
+ }
91
+ const vectors = flattenRows(params.vectors, index.dim, 'vectors');
92
+ const ids = toNativeIds(params.ids);
93
+ runBackend(() => index.add(vectors, ids));
94
+ return { length: index.length };
95
+ }
96
+ export function searchVectors(params) {
97
+ const index = getIndex(params.indexId);
98
+ const queries = flattenRows(params.queries, index.dim, 'queries');
99
+ const results = runBackend(() => index.search(queries, params.k));
100
+ return { results };
101
+ }
102
+ /**
103
+ * Removes ids one at a time, which is the only shape the backend offers. A
104
+ * backend throw here means the handle itself is unusable, so the batch stops
105
+ * rather than trying the rest — but ids before it are already gone, so the
106
+ * error names how many were applied. `length` is only reported on success,
107
+ * which is why the handle documents its own `length` as of the last call that
108
+ * reported one.
109
+ */
110
+ export function removeVectors(params) {
111
+ const index = getIndex(params.indexId);
112
+ const removed = [];
113
+ for (const id of params.ids) {
114
+ removed.push(runBackend(() => index.remove(BigInt(id)), (detail) => `remove applied ${removed.length} of ${params.ids.length} ids before failing: ${detail}`));
115
+ }
116
+ return { removed, length: index.length };
117
+ }
118
+ export function containsVectors(params) {
119
+ const index = getIndex(params.indexId);
120
+ const present = params.ids.map((id) => runBackend(() => index.contains(BigInt(id))));
121
+ return { present };
122
+ }
123
+ export async function writeVectorIndex(params) {
124
+ const index = getIndex(params.indexId);
125
+ const snapshotPath = resolveVectorIndexPath(params.path);
126
+ try {
127
+ await fsPromises.mkdir(path.dirname(snapshotPath), { recursive: true });
128
+ }
129
+ catch (error) {
130
+ throw new VectorIndexFailedError(`cannot create the snapshot directory for ${snapshotPath}: ${error instanceof Error ? error.message : String(error)}`, error);
131
+ }
132
+ runBackend(() => index.write(snapshotPath));
133
+ return { path: snapshotPath };
134
+ }
135
+ // Disposing an unknown id is not an error: the caller may retry after a
136
+ // worker restart, and the memory it refers to is already gone.
137
+ export function disposeVectorIndex(params) {
138
+ const index = indexes.get(params.indexId);
139
+ if (!index)
140
+ return { disposed: false };
141
+ // Drop the entry only once the backend confirms. Deleting first would strand
142
+ // a failed dispose: the retry would find nothing and report success, and
143
+ // `disposeAllVectorIndexes` could not reach it at shutdown either.
144
+ runBackend(() => index.dispose());
145
+ indexes.delete(params.indexId);
146
+ return { disposed: true };
147
+ }
148
+ export function getOpenVectorIndexCount() {
149
+ return indexes.size;
150
+ }
151
+ export function disposeAllVectorIndexes() {
152
+ for (const [indexId, index] of indexes) {
153
+ try {
154
+ index.dispose();
155
+ }
156
+ catch (error) {
157
+ getEngineLogger().warn(`Failed to dispose vector index '${indexId}' during cleanup:`, error);
158
+ }
159
+ }
160
+ indexes.clear();
161
+ }
@@ -214,6 +214,47 @@ export declare const assessModelFitResultSchema: z.ZodObject<{
214
214
  reasons: z.ZodArray<z.ZodString>;
215
215
  assumptions: z.ZodArray<z.ZodString>;
216
216
  }, z.core.$strip>;
217
+ /**
218
+ * The other fit question, kept in this file on purpose: `assessModelFit` above
219
+ * answers "should I download this?" from calibrated coefficients and a checksum,
220
+ * while the native probe answers "will the load I am about to run fit?" by
221
+ * reading the GGUF on disk in a disposable child. Two evidence classes, so two
222
+ * verdict vocabularies:
223
+ *
224
+ * - `likely-fits` / `likely-too-large` hedge because a formula is an estimate.
225
+ * - `fit` / `does-not-fit` do not, because the probe measured this build of this
226
+ * file against this device.
227
+ *
228
+ * Precedence when both have spoken about the same model: the probe wins, because
229
+ * it saw the artifact and the resolved load settings. `assessModelFit` is the
230
+ * answer available before the bytes are on disk. Neither denies a load today.
231
+ */
232
+ export declare const nativeProbeVerdictSchema: z.ZodEnum<{
233
+ unknown: "unknown";
234
+ fit: "fit";
235
+ "does-not-fit": "does-not-fit";
236
+ }>;
237
+ export declare const nativeProbePlanSchema: z.ZodObject<{
238
+ nCtx: z.ZodNumber;
239
+ nGpuLayers: z.ZodNumber;
240
+ nGpuDevices: z.ZodNumber;
241
+ }, z.core.$strip>;
242
+ export declare const nativeProbeFitSchema: z.ZodObject<{
243
+ verdict: z.ZodEnum<{
244
+ unknown: "unknown";
245
+ fit: "fit";
246
+ "does-not-fit": "does-not-fit";
247
+ }>;
248
+ basis: z.ZodLiteral<"native-probe">;
249
+ estimatorVersion: z.ZodString;
250
+ reason: z.ZodString;
251
+ message: z.ZodOptional<z.ZodString>;
252
+ plan: z.ZodOptional<z.ZodObject<{
253
+ nCtx: z.ZodNumber;
254
+ nGpuLayers: z.ZodNumber;
255
+ nGpuDevices: z.ZodNumber;
256
+ }, z.core.$strip>>;
257
+ }, z.core.$strip>;
217
258
  export declare const assessModelFitRequestSchema: z.ZodObject<{
218
259
  models: z.ZodArray<z.ZodObject<{
219
260
  model: z.ZodObject<{
@@ -296,6 +337,9 @@ export declare const assessModelFitResponseSchema: z.ZodObject<{
296
337
  assumptions: z.ZodArray<z.ZodString>;
297
338
  type: z.ZodLiteral<"assessModelFit">;
298
339
  }, z.core.$strip>;
340
+ export type NativeProbeVerdict = z.infer<typeof nativeProbeVerdictSchema>;
341
+ export type NativeProbePlan = z.infer<typeof nativeProbePlanSchema>;
342
+ export type NativeProbeFit = z.infer<typeof nativeProbeFitSchema>;
299
343
  export type ModelFitVerdict = z.infer<typeof modelFitVerdictSchema>;
300
344
  export type ModelFitModelRef = z.infer<typeof modelFitModelRefSchema>;
301
345
  export type ModelFitWorkload = z.infer<typeof modelFitWorkloadSchema>;
@@ -174,6 +174,46 @@ export const assessModelFitResultSchema = z.object({
174
174
  .array(z.string())
175
175
  .describe('Everything the result took for granted, including estimator defaults.')
176
176
  });
177
+ // ============== Native probe (pre-load) ==============
178
+ /**
179
+ * The other fit question, kept in this file on purpose: `assessModelFit` above
180
+ * answers "should I download this?" from calibrated coefficients and a checksum,
181
+ * while the native probe answers "will the load I am about to run fit?" by
182
+ * reading the GGUF on disk in a disposable child. Two evidence classes, so two
183
+ * verdict vocabularies:
184
+ *
185
+ * - `likely-fits` / `likely-too-large` hedge because a formula is an estimate.
186
+ * - `fit` / `does-not-fit` do not, because the probe measured this build of this
187
+ * file against this device.
188
+ *
189
+ * Precedence when both have spoken about the same model: the probe wins, because
190
+ * it saw the artifact and the resolved load settings. `assessModelFit` is the
191
+ * answer available before the bytes are on disk. Neither denies a load today.
192
+ */
193
+ export const nativeProbeVerdictSchema = z.enum(['fit', 'does-not-fit', 'unknown']);
194
+ export const nativeProbePlanSchema = z.object({
195
+ nCtx: z.number().int().describe('Context the probe resolved for this load, in tokens.'),
196
+ nGpuLayers: z.number().int().describe('Layers the probe would offload.'),
197
+ nGpuDevices: z.number().int().describe('GPU devices the offload would span.')
198
+ });
199
+ export const nativeProbeFitSchema = z
200
+ .object({
201
+ verdict: nativeProbeVerdictSchema.describe('Advisory outcome. `unknown` means no verdict was obtainable — the check was disabled, the load shape is unsupported, or the child produced no usable answer.'),
202
+ basis: z
203
+ .literal('native-probe')
204
+ .describe('Evidence class: a disposable llama.cpp child that read the model file and the resolved load settings.'),
205
+ estimatorVersion: z
206
+ .string()
207
+ .describe('Version of the probe integration that produced this outcome, covering the load-setting partitioning and the headroom policy — `native-probe-v1` withholds 1024 MiB plus the on-disk bytes of every model already resident in this worker.'),
208
+ reason: z
209
+ .string()
210
+ .describe('Machine-readable reason for the verdict. Never parsed out of log text.'),
211
+ message: z.string().optional().describe('Human-readable detail, when the reason has any.'),
212
+ plan: nativeProbePlanSchema
213
+ .optional()
214
+ .describe('Placement the probe projected. Present only on a `fit` verdict.')
215
+ })
216
+ .meta({ title: 'NativeProbeFit' });
177
217
  export const assessModelFitRequestSchema = assessModelFitInputSchema.extend({
178
218
  type: z.literal('assessModelFit')
179
219
  });