@qvac/inference 0.19.1 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (137) hide show
  1. package/NOTICE +43 -11
  2. package/dist/api/audio-gen.d.ts +42 -1
  3. package/dist/api/audio-gen.js +137 -51
  4. package/dist/api/index.d.ts +2 -1
  5. package/dist/api/index.js +2 -1
  6. package/dist/api/text-to-speech.d.ts +31 -4
  7. package/dist/api/text-to-speech.js +117 -21
  8. package/dist/api/vector-index.d.ts +142 -0
  9. package/dist/api/vector-index.js +168 -0
  10. package/dist/api/world.d.ts +2 -2
  11. package/dist/errors/index.d.ts +12 -0
  12. package/dist/errors/index.js +21 -0
  13. package/dist/handlers/vector-index.d.ts +2 -0
  14. package/dist/handlers/vector-index.js +40 -0
  15. package/dist/index.d.ts +1 -1
  16. package/dist/index.js +1 -1
  17. package/dist/models/registry/models.d.ts +351 -12
  18. package/dist/models/registry/models.js +2271 -1957
  19. package/dist/models/registry/resource-profiles.js +135 -0
  20. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
  21. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
  22. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
  23. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
  24. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
  25. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
  26. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
  27. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
  28. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
  29. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
  30. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
  31. package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
  32. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
  33. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
  34. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
  35. package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
  36. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
  37. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
  38. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
  39. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -0
  40. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +3 -0
  41. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
  42. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
  43. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
  44. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
  45. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +40 -24
  46. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +187 -123
  47. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
  48. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
  49. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +8 -4
  50. package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
  51. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
  52. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
  53. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
  54. package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
  55. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
  56. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
  57. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
  58. package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
  59. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
  60. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
  61. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
  62. package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
  63. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
  64. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
  65. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
  66. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
  67. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
  68. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
  69. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
  70. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
  71. package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
  72. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
  73. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
  74. package/dist/plugins/index.d.ts +1 -1
  75. package/dist/plugins/index.js +1 -1
  76. package/dist/plugins/ops/load-model.js +22 -2
  77. package/dist/plugins/ops/translate.js +58 -12
  78. package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
  79. package/dist/plugins/ops/translatepsy-utils.js +32 -0
  80. package/dist/plugins/registry.d.ts +15 -0
  81. package/dist/plugins/registry.js +133 -15
  82. package/dist/plugins/turbovec-backend.d.ts +17 -0
  83. package/dist/plugins/turbovec-backend.js +99 -0
  84. package/dist/registry.js +4 -0
  85. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
  86. package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
  87. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
  88. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
  89. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
  90. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
  91. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
  92. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
  93. package/dist/runtime/env.d.ts +2 -0
  94. package/dist/runtime/env.js +9 -2
  95. package/dist/runtime/lifecycle.js +2 -0
  96. package/dist/runtime/model-registry.d.ts +3 -1
  97. package/dist/runtime/model-registry.js +2 -1
  98. package/dist/runtime/request-context.js +17 -0
  99. package/dist/runtime/vector-index-backend.d.ts +25 -0
  100. package/dist/runtime/vector-index-backend.js +1 -0
  101. package/dist/runtime/vector-index-registry.d.ts +76 -0
  102. package/dist/runtime/vector-index-registry.js +161 -0
  103. package/dist/schemas/assess-model-fit.d.ts +44 -0
  104. package/dist/schemas/assess-model-fit.js +40 -0
  105. package/dist/schemas/audio-gen.d.ts +479 -4
  106. package/dist/schemas/audio-gen.js +376 -17
  107. package/dist/schemas/batch-completion-stream.d.ts +7 -0
  108. package/dist/schemas/batch-completion-stream.js +5 -3
  109. package/dist/schemas/common.d.ts +749 -15
  110. package/dist/schemas/common.js +8 -1
  111. package/dist/schemas/completion-event.d.ts +11 -1
  112. package/dist/schemas/completion-event.js +3 -0
  113. package/dist/schemas/completion-stream.d.ts +20 -0
  114. package/dist/schemas/completion-stream.js +41 -4
  115. package/dist/schemas/config.d.ts +48 -30
  116. package/dist/schemas/errors.d.ts +4 -0
  117. package/dist/schemas/errors.js +22 -0
  118. package/dist/schemas/index.d.ts +1 -0
  119. package/dist/schemas/index.js +1 -0
  120. package/dist/schemas/llamacpp-config.d.ts +4 -8
  121. package/dist/schemas/llamacpp-config.js +4 -4
  122. package/dist/schemas/load-model.d.ts +2503 -139
  123. package/dist/schemas/model-config-schemas.d.ts +228 -5
  124. package/dist/schemas/plugin.d.ts +10 -0
  125. package/dist/schemas/sdcpp-config.d.ts +221 -4
  126. package/dist/schemas/sdcpp-config.js +128 -22
  127. package/dist/schemas/text-to-speech.d.ts +780 -19
  128. package/dist/schemas/text-to-speech.js +298 -44
  129. package/dist/schemas/transcription-config.js +7 -4
  130. package/dist/schemas/vector-index.d.ts +194 -0
  131. package/dist/schemas/vector-index.js +185 -0
  132. package/dist/utils/addon-responses.d.ts +29 -3
  133. package/dist/utils/aggregate-events.d.ts +3 -1
  134. package/dist/utils/aggregate-events.js +7 -1
  135. package/dist/utils/tts-stats.d.ts +15 -11
  136. package/dist/utils/tts-stats.js +33 -11
  137. package/package.json +28 -25
@@ -5,9 +5,41 @@ import { createAddonLoggerCallback } from '../logging/addon.js';
5
5
  import { getEngineLogger } from '../logging/index.js';
6
6
  import { formatZodError } from '../utils/zod-error.js';
7
7
  const plugins = new Map();
8
+ /** Addon logging modules wired through a resolver, keyed by namespace. */
9
+ const lazyAddonLoggers = new Map();
10
+ /** In-flight resolutions, so concurrent loads wire a namespace exactly once. */
11
+ const pendingAddonLoggers = new Map();
12
+ function getLoggingResolver(plugin) {
13
+ const declared = plugin.logging?.module;
14
+ return typeof declared === 'function' ? declared : undefined;
15
+ }
16
+ /**
17
+ * The module a plugin supplied directly, or `undefined` when it supplied a
18
+ * resolver. Resolver-based plugins are therefore invisible to
19
+ * registration-time shape validation, to the shared-module dedupe, and to the
20
+ * eager release sweeps; `ensureAddonLoggerReady` and `lazyAddonLoggers` cover
21
+ * them instead, keyed by namespace rather than held on the plugin.
22
+ */
8
23
  function getLoggingModule(plugin) {
24
+ if (getLoggingResolver(plugin))
25
+ return undefined;
9
26
  return plugin.logging?.module;
10
27
  }
28
+ /** CommonJS addons resolve through `import()` as a namespace with `default`. */
29
+ function unwrapLoggingModule(resolved) {
30
+ if (resolved && typeof resolved === 'object' && 'default' in resolved) {
31
+ const inner = resolved.default;
32
+ if (inner && typeof inner.setLogger === 'function')
33
+ return inner;
34
+ }
35
+ return resolved;
36
+ }
37
+ function assertLoggingModuleShape(modelType, candidate) {
38
+ if (!candidate || typeof candidate['setLogger'] !== 'function') {
39
+ throw new PluginLoggingInvalidError(modelType, 'logging.module must have a setLogger(callback) function');
40
+ }
41
+ return candidate;
42
+ }
11
43
  function findPluginUsingLoggingModule(loggingModule) {
12
44
  return Array.from(plugins.values()).find((plugin) => plugin.logging?.module === loggingModule);
13
45
  }
@@ -42,12 +74,11 @@ export function registerPlugin(plugin) {
42
74
  if (pluginProvidingTurboVecIndex) {
43
75
  throw new PluginDefinitionInvalidError(plugin.modelType, `plugin "${plugin.modelType}" cannot provide turbovecIndexProvider because plugin "${pluginProvidingTurboVecIndex.modelType}" already provides it`);
44
76
  }
45
- // Validate logging module shape if provided
46
- if (plugin.logging?.module) {
47
- const loggingModule = plugin.logging.module;
48
- if (typeof loggingModule['setLogger'] !== 'function') {
49
- throw new PluginLoggingInvalidError(plugin.modelType, 'logging.module must have a setLogger(callback) function');
50
- }
77
+ // Validate logging module shape if provided. A resolver is checked when it
78
+ // runs instead: calling it here to inspect its result would load the addon,
79
+ // which is the whole thing registration is meant to avoid.
80
+ if (plugin.logging?.module && !getLoggingResolver(plugin)) {
81
+ assertLoggingModuleShape(plugin.modelType, plugin.logging.module);
51
82
  }
52
83
  const loggingModule = getLoggingModule(plugin);
53
84
  const pluginUsingLoggingModule = loggingModule
@@ -67,6 +98,91 @@ export function registerPlugins(pluginList) {
67
98
  registerPlugin(plugin);
68
99
  }
69
100
  }
101
+ /**
102
+ * Wires the addon logger of a plugin that supplied a resolver, loading the
103
+ * addon on the way. Call it once a model of this type is actually being
104
+ * created: it is the point where the addon is needed anyway, so a host
105
+ * missing that addon's platform package fails the load it asked for instead
106
+ * of failing every plugin registration at startup.
107
+ *
108
+ * A no-op for plugins that passed their module directly — those are already
109
+ * wired by `registerPlugin` — and for a namespace that is already wired,
110
+ * which is how two plugins over one addon (whisper and Parakeet over ASR)
111
+ * share a single `setLogger` call. Also a no-op when the registry was cleared
112
+ * while the addon was loading, so a shutdown that races a load leaves no
113
+ * logger behind.
114
+ */
115
+ export async function ensureAddonLoggerReady(plugin) {
116
+ const resolver = getLoggingResolver(plugin);
117
+ const namespace = plugin.logging?.namespace;
118
+ if (!resolver || !namespace)
119
+ return;
120
+ if (lazyAddonLoggers.has(namespace))
121
+ return;
122
+ const pending = pendingAddonLoggers.get(namespace);
123
+ if (pending)
124
+ return pending;
125
+ const wiring = (async () => {
126
+ const resolved = unwrapLoggingModule(await resolver());
127
+ // Loading the addon takes real time, and the registry can be torn down
128
+ // while this is parked: `runCleanup` calls `clearRegistries()`
129
+ // synchronously and only then awaits `unloadAllModels()`, so a sweep can
130
+ // pass between the call above and this line. Wiring now would hand the
131
+ // addon a callback nothing will release — on Expo, a `js_ref_t` leaked
132
+ // into a dying isolate, which the next worklet's first `setLogger` trips
133
+ // over. Nothing has been wired yet, so dropping the resolved module is
134
+ // the whole cleanup.
135
+ if (!isNamespaceClaimed(namespace))
136
+ return;
137
+ const loggingModule = assertLoggingModuleShape(plugin.modelType, resolved);
138
+ for (const [wiredNamespace, wiredModule] of lazyAddonLoggers) {
139
+ if (wiredModule === loggingModule && wiredNamespace !== namespace) {
140
+ throw new PluginLoggingInvalidError(plugin.modelType, 'plugins sharing logging.module must use the same namespace');
141
+ }
142
+ }
143
+ loggingModule.setLogger(createAddonLoggerCallback(namespace));
144
+ lazyAddonLoggers.set(namespace, loggingModule);
145
+ })();
146
+ pendingAddonLoggers.set(namespace, wiring);
147
+ try {
148
+ await wiring;
149
+ }
150
+ finally {
151
+ pendingAddonLoggers.delete(namespace);
152
+ }
153
+ }
154
+ /** Whether a registered resolver-based plugin still claims this namespace. */
155
+ function isNamespaceClaimed(namespace) {
156
+ return Array.from(plugins.values()).some((candidate) => getLoggingResolver(candidate) && candidate.logging?.namespace === namespace);
157
+ }
158
+ /** Releases a lazily-wired addon logger once no registered plugin claims it. */
159
+ function releaseLazyAddonLogger(namespace, modelType) {
160
+ if (namespace === undefined)
161
+ return;
162
+ const loggingModule = lazyAddonLoggers.get(namespace);
163
+ if (!loggingModule)
164
+ return;
165
+ if (isNamespaceClaimed(namespace))
166
+ return;
167
+ lazyAddonLoggers.delete(namespace);
168
+ releaseLoggerSafely(loggingModule, modelType);
169
+ }
170
+ /**
171
+ * Runs an addon's `releaseLogger`. A failure must not abort a sweep or leave
172
+ * the registry half-cleared for the next caller, but it is reported rather
173
+ * than swallowed: a leaked reference or a live async handle must not pass for
174
+ * a clean teardown. `subject` names what is being released — a model type, or
175
+ * a namespace for a logger that is only keyed by one — and `during` names the
176
+ * sweep, when the release is part of one.
177
+ */
178
+ function releaseLoggerSafely(loggingModule, subject, during) {
179
+ try {
180
+ loggingModule.releaseLogger?.();
181
+ }
182
+ catch (error) {
183
+ getEngineLogger().warn(`[${subject}] releaseLogger failed${during ? ` during ${during}` : ''}: ${error instanceof Error ? error.message : String(error)}`);
184
+ }
185
+ }
70
186
  export function getPlugin(modelType) {
71
187
  return plugins.get(modelType);
72
188
  }
@@ -84,10 +200,12 @@ export function unregisterPlugin(modelType) {
84
200
  if (!plugin)
85
201
  return false;
86
202
  const loggingModule = getLoggingModule(plugin);
203
+ const resolverNamespace = getLoggingResolver(plugin) ? plugin.logging?.namespace : undefined;
87
204
  plugins.delete(modelType);
88
205
  if (loggingModule && !findPluginUsingLoggingModule(loggingModule)) {
89
206
  loggingModule.releaseLogger?.();
90
207
  }
208
+ releaseLazyAddonLogger(resolverNamespace, modelType);
91
209
  return true;
92
210
  }
93
211
  export function getAllPlugins() {
@@ -104,16 +222,16 @@ export function clearPlugins() {
104
222
  loggingModules.set(loggingModule, plugin.modelType);
105
223
  }
106
224
  }
225
+ const lazyModules = new Map(lazyAddonLoggers);
107
226
  plugins.clear();
227
+ lazyAddonLoggers.clear();
108
228
  for (const [loggingModule, modelType] of loggingModules) {
109
- try {
110
- loggingModule.releaseLogger?.();
111
- }
112
- catch (error) {
113
- // A plugin's logger teardown must not abort the sweep or leave the
114
- // registry half-cleared for the next caller — but surface it, so a leaked
115
- // reference or async handle is not masked as a clean teardown.
116
- getEngineLogger().warn(`[${modelType}] releaseLogger failed during clearPlugins: ${error instanceof Error ? error.message : String(error)}`);
117
- }
229
+ releaseLoggerSafely(loggingModule, modelType, 'clearPlugins');
230
+ }
231
+ // Loggers wired through a resolver are keyed by namespace rather than held
232
+ // on the plugin, so they need the same sweep. They are named by that
233
+ // namespace, which is not a model type — several plugins can share one.
234
+ for (const [namespace, loggingModule] of lazyModules) {
235
+ releaseLoggerSafely(loggingModule, `namespace ${namespace}`, 'clearPlugins');
118
236
  }
119
237
  }
@@ -0,0 +1,17 @@
1
+ import { VectorIndexStorage } from '../schemas/index.js';
2
+ import type { VectorIndexBackend } from '../runtime/vector-index-backend.js';
3
+ /**
4
+ * The native index reports only its effective bit width after a load. Three
5
+ * widths map to one storage mode each; 4 bits is either `q4` or
6
+ * `turbovec-q4`, which the snapshot header distinguishes but the addon does
7
+ * not expose yet, so those loads report no storage mode.
8
+ */
9
+ export declare function storageFromBitWidth(bitWidth: unknown): VectorIndexStorage | undefined;
10
+ /**
11
+ * The vector index feature's only backend: the fabric vector index that the
12
+ * embedding plugin exposes through the `turbovecIndexProvider` capability.
13
+ * It serves every `VectorIndexStorage` mode; the two TurboVec modes add the
14
+ * dimension rule (multiple of 8, at most 1024), which the native index
15
+ * enforces itself.
16
+ */
17
+ export declare function getTurboVecBackend(): VectorIndexBackend | undefined;
@@ -0,0 +1,99 @@
1
+ import { getTurboVecIndexProvider } from './registry.js';
2
+ import { VECTOR_ID_RESERVED, VectorIndexStorage } from '../schemas/index.js';
3
+ // The native index pads short result rows with UINT64_MAX.
4
+ const PADDING_ID = BigInt(VECTOR_ID_RESERVED);
5
+ /**
6
+ * The native index reports only its effective bit width after a load. Three
7
+ * widths map to one storage mode each; 4 bits is either `q4` or
8
+ * `turbovec-q4`, which the snapshot header distinguishes but the addon does
9
+ * not expose yet, so those loads report no storage mode.
10
+ */
11
+ export function storageFromBitWidth(bitWidth) {
12
+ switch (bitWidth) {
13
+ case 32:
14
+ return VectorIndexStorage.F32;
15
+ case 8:
16
+ return VectorIndexStorage.Q8;
17
+ case 2:
18
+ return VectorIndexStorage.TURBOVEC_Q2;
19
+ default:
20
+ return undefined;
21
+ }
22
+ }
23
+ function toHitRows(result) {
24
+ const rows = [];
25
+ for (let row = 0; row < result.m; row++) {
26
+ const hits = [];
27
+ for (let slot = 0; slot < result.k; slot++) {
28
+ const offset = row * result.k + slot;
29
+ const id = result.ids[offset];
30
+ if (id === undefined || id === PADDING_ID)
31
+ break;
32
+ hits.push({ id: id.toString(), score: result.scores[offset] ?? 0 });
33
+ }
34
+ rows.push(hits);
35
+ }
36
+ return rows;
37
+ }
38
+ function openIndex(index, storage) {
39
+ // TurboVec precomputes rotation and codebook state on the first search
40
+ // after a change; running it eagerly keeps that cost out of the first
41
+ // query's latency.
42
+ let needsPrepare = true;
43
+ return {
44
+ get dim() {
45
+ return index.dim;
46
+ },
47
+ get length() {
48
+ return index.length;
49
+ },
50
+ storage,
51
+ add(vectors, ids) {
52
+ index.addWithIds(vectors, ids);
53
+ needsPrepare = true;
54
+ },
55
+ search(queries, k) {
56
+ if (needsPrepare) {
57
+ index.prepare();
58
+ needsPrepare = false;
59
+ }
60
+ return toHitRows(index.search(queries, k));
61
+ },
62
+ contains(id) {
63
+ return index.contains(id);
64
+ },
65
+ remove(id) {
66
+ const removed = index.remove(id);
67
+ if (removed)
68
+ needsPrepare = true;
69
+ return removed;
70
+ },
71
+ write(path) {
72
+ index.write(path);
73
+ },
74
+ dispose() {
75
+ index.dispose();
76
+ }
77
+ };
78
+ }
79
+ /**
80
+ * The vector index feature's only backend: the fabric vector index that the
81
+ * embedding plugin exposes through the `turbovecIndexProvider` capability.
82
+ * It serves every `VectorIndexStorage` mode; the two TurboVec modes add the
83
+ * dimension rule (multiple of 8, at most 1024), which the native index
84
+ * enforces itself.
85
+ */
86
+ export function getTurboVecBackend() {
87
+ const provider = getTurboVecIndexProvider();
88
+ if (!provider)
89
+ return undefined;
90
+ return {
91
+ create(dim, storage) {
92
+ return openIndex(provider.create({ dim, storage }), storage);
93
+ },
94
+ load(path) {
95
+ const index = provider.load(path);
96
+ return openIndex(index, storageFromBitWidth(index.bitWidth));
97
+ }
98
+ };
99
+ }
package/dist/registry.js CHANGED
@@ -4,6 +4,7 @@ import { handleUnloadModel } from './handlers/unload-model.js';
4
4
  import { handleLoggingStream } from './handlers/logging-stream.js';
5
5
  import { cancelHandler } from './handlers/cancelHandler.js';
6
6
  import { handleRag } from './rag/handler.js';
7
+ import { handleVectorIndex } from './handlers/vector-index.js';
7
8
  import { handleDeleteCache } from './handlers/delete-cache.js';
8
9
  import { handleDownloadAsset } from './handlers/download-asset.js';
9
10
  import { handleGetModelInfo } from './handlers/get-model-info.js';
@@ -74,6 +75,7 @@ export const registry = {
74
75
  },
75
76
  downloadAsset: { type: 'reply', handler: handleDownloadAsset, supportsProgress: true },
76
77
  rag: { type: 'reply', handler: handleRag, supportsProgress: ragSupportsProgress },
78
+ vectorIndex: { type: 'reply', handler: handleVectorIndex },
77
79
  finetune: { type: 'reply', handler: handleFinetune, supportsProgress: finetuneSupportsProgress },
78
80
  // Stream
79
81
  completionStream: {
@@ -94,6 +96,8 @@ export const registry = {
94
96
  ocrStream: { type: 'stream', pluginOp: true, handler: pluginStream('ocrStream') },
95
97
  diffusionStream: { type: 'stream', pluginOp: true, handler: pluginStream('diffusionStream') },
96
98
  audioGenStream: { type: 'stream', pluginOp: true, handler: pluginStream('audioGenStream') },
99
+ audioEditStream: { type: 'stream', pluginOp: true, handler: pluginStream('audioEditStream') },
100
+ audioUnderstand: { type: 'stream', pluginOp: true, handler: pluginStream('audioUnderstand') },
97
101
  videoStream: { type: 'stream', pluginOp: true, handler: pluginStream('videoStream') },
98
102
  upscaleStream: { type: 'stream', pluginOp: true, handler: pluginStream('upscaleStream') },
99
103
  worldStepStream: { type: 'stream', pluginOp: true, handler: pluginStream('worldStepStream') },
@@ -0,0 +1,46 @@
1
+ import type { AbortSignal } from 'bare-abort-controller';
2
+ import type { Logger } from '../../../logging/types.js';
3
+ import type { CanonicalModelType, NativeProbeFit, NativeProbeVerdict } from '../../../schemas/index.js';
4
+ import type { runIsolatedFit } from './run-isolated-fit.js';
5
+ /**
6
+ * `fit` and `does-not-fit` are projections of the load the SDK is about to run,
7
+ * not admission decisions. `@qvac/model-fit` duplicates the loader's policy for
8
+ * this experiment and the real loader neither consumes nor verifies the fitted
9
+ * plan, so neither verdict is denial-grade and no verdict changes the load.
10
+ *
11
+ * The outcome is the wire shape: `loadModel` hands it to the model registry and
12
+ * `getLoadedModelInfo` returns it, so no caller has to read a verdict out of a
13
+ * log line.
14
+ */
15
+ export type AdvisoryFitVerdict = NativeProbeVerdict;
16
+ export type AdvisoryFitOutcome = NativeProbeFit;
17
+ export interface AdvisoryFitInput {
18
+ modelId: string;
19
+ modelType: CanonicalModelType;
20
+ modelPath: string;
21
+ modelConfig: unknown;
22
+ artifacts?: Record<string, string> | undefined;
23
+ isShardedModel: boolean;
24
+ }
25
+ /**
26
+ * Injection seams. Every field defaults to the real runtime dependency; tests
27
+ * substitute them rather than mocking modules.
28
+ */
29
+ export interface AdvisoryFitOptions {
30
+ signal?: AbortSignal;
31
+ enabled?: boolean;
32
+ mobile?: boolean;
33
+ timeoutMs?: number;
34
+ runFit?: typeof runIsolatedFit;
35
+ logger?: Logger;
36
+ residentModelBytes?: () => Promise<number>;
37
+ }
38
+ /**
39
+ * Runs the advisory llama.cpp fit check for a load that is about to start.
40
+ *
41
+ * Fail-open by construction: an unsupported shape, a crashed or wedged child, a
42
+ * malformed response, and an unexpected internal error all resolve to `unknown`
43
+ * and the caller proceeds with the ordinary load path. This function never
44
+ * throws and never rejects.
45
+ */
46
+ export declare function runAdvisoryFitCheck(input: AdvisoryFitInput, options?: AdvisoryFitOptions): Promise<AdvisoryFitOutcome>;
@@ -0,0 +1,214 @@
1
+ import { getEngineLogger } from '../../../logging/index.js';
2
+ import { createLlamaFitRequest } from './create-llama-fit-request.js';
3
+ /**
4
+ * Shorter than the supervisor's own 60s default: this check sits in front of a
5
+ * real load, so a wedged child must not hold the load for a full minute. Every
6
+ * expiry is `unknown` and the load continues.
7
+ */
8
+ const ADVISORY_FIT_TIMEOUT_MS = 30_000;
9
+ /**
10
+ * `@qvac/model-fit`'s own default margin. Made explicit here because the
11
+ * resident-model reserve below is *added* to it: setting `marginMiB` at all
12
+ * replaces the package default, so the base has to travel with the reserve.
13
+ */
14
+ const ADVISORY_FIT_BASE_MARGIN_MIB = 1024;
15
+ /**
16
+ * Names the headroom policy these two numbers add up to: withhold
17
+ * `ADVISORY_FIT_BASE_MARGIN_MIB`, plus the on-disk bytes of every model already
18
+ * resident in this worker.
19
+ *
20
+ * Deliberately *not* `interactive-v1` (withhold 20%, cap 2 GiB desktop / 1 GiB
21
+ * mobile), which `assessModelFit` applies. That policy is applied by the SDK to
22
+ * a budget the SDK computed; here the child owns the budget (total − wired −
23
+ * compressor, measured inside the disposable process) and `marginMiB` is the
24
+ * only lever this side has. Reconciling the two means one entry point owning
25
+ * both bases — see the precedence note on `nativeProbeFitSchema`.
26
+ */
27
+ const NATIVE_PROBE_ESTIMATOR_VERSION = 'native-probe-v1';
28
+ const BYTES_PER_MIB = 1024 * 1024;
29
+ const DISABLED_VALUES = new Set(['0', 'false', 'off', 'no']);
30
+ /**
31
+ * On by default; `QVAC_ADVISORY_MODEL_FIT=0` (or `false`/`off`/`no`) is the
32
+ * operator opt-out — the escape hatch for a load-heavy startup path or a
33
+ * runtime where the disposable child cannot spawn. Any other value, including
34
+ * unset, leaves the check on.
35
+ *
36
+ * The worker environment and the mobile runtime flag are imported lazily. Both
37
+ * modules reach Bare-only bindings, and resolving them eagerly would make this
38
+ * orchestration untestable outside a Bare runtime.
39
+ */
40
+ async function resolveEnabled(explicit) {
41
+ if (explicit !== undefined)
42
+ return explicit;
43
+ const { getValidatedEnv } = await import('../../../runtime/env.js');
44
+ const value = getValidatedEnv().QVAC_ADVISORY_MODEL_FIT;
45
+ return value === undefined || !DISABLED_VALUES.has(value.toLowerCase());
46
+ }
47
+ async function resolveMobile(explicit) {
48
+ if (explicit !== undefined)
49
+ return explicit;
50
+ const { isMobile } = await import('../../../runtime/state.js');
51
+ return isMobile();
52
+ }
53
+ /**
54
+ * Sums the on-disk weight sizes of every model currently registered in this
55
+ * worker. Since `@qvac/model-fit@0.8.0` (qvac-fabric#214) the fit child budgets
56
+ * against system-wide availability (total − wired − compressor), so weights the
57
+ * worker holds *wired* on the GPU are already visible to it. What the child
58
+ * still cannot see is the unwired remainder — mmap'd host-side layers and KV of
59
+ * resident models — which the OS reports as evictable right up until a decode
60
+ * needs it resident. Measured consequence on 0.7.0 (where the child saw a fully
61
+ * idle device): a verdict correct on an idle machine admitted a load that could
62
+ * not decode with another model loaded.
63
+ *
64
+ * Reserving the resident weight bytes through `marginMiB` keeps that footprint
65
+ * in the child's budget. Where those weights are wired this now double-counts
66
+ * and the verdict turns conservative — deliberately so: the measured failure
67
+ * modes punish optimism (loads that cannot decode), not caution.
68
+ *
69
+ * Advisory and fail-open like everything else here: any failure to stat a file
70
+ * contributes zero rather than an error.
71
+ */
72
+ async function defaultResidentModelBytes() {
73
+ const [{ getAllModelIds, getModelInfo }, { promises: fsPromises }] = await Promise.all([
74
+ import('../../../runtime/model-registry.js'),
75
+ import('bare-fs')
76
+ ]);
77
+ let bytes = 0;
78
+ for (const id of getAllModelIds()) {
79
+ const info = getModelInfo(id);
80
+ if (info === null)
81
+ continue;
82
+ try {
83
+ const stats = (await fsPromises.stat(info.path));
84
+ bytes += stats.size;
85
+ }
86
+ catch {
87
+ // Unreadable path: contribute nothing rather than fail the check.
88
+ }
89
+ }
90
+ return bytes;
91
+ }
92
+ /**
93
+ * Also lazy: the desktop supervisor pulls the Bare process launcher, and a
94
+ * disabled check must not load it. Mobile uses in-process `fitParamsAsync` instead
95
+ * of spawn (`bare-runtime/spawn` stays deferred from the mobile pack).
96
+ */
97
+ async function resolveRunFit(explicit, mobile) {
98
+ if (explicit !== undefined)
99
+ return explicit;
100
+ if (mobile) {
101
+ const { runInProcessFit } = await import('./run-in-process-fit.js');
102
+ return runInProcessFit;
103
+ }
104
+ return (await import('./run-isolated-fit.js')).runIsolatedFit;
105
+ }
106
+ const PROVENANCE = {
107
+ basis: 'native-probe',
108
+ estimatorVersion: NATIVE_PROBE_ESTIMATOR_VERSION
109
+ };
110
+ function unknown(reason, message) {
111
+ return message === undefined
112
+ ? { ...PROVENANCE, verdict: 'unknown', reason }
113
+ : { ...PROVENANCE, verdict: 'unknown', reason, message };
114
+ }
115
+ function classify(result) {
116
+ if (result.status === 0) {
117
+ return {
118
+ ...PROVENANCE,
119
+ verdict: 'fit',
120
+ reason: result.reason,
121
+ plan: {
122
+ nCtx: result.nCtx,
123
+ nGpuLayers: result.nGpuLayers,
124
+ nGpuDevices: result.nGpuDevices
125
+ }
126
+ };
127
+ }
128
+ if (result.status === 1) {
129
+ return { ...PROVENANCE, verdict: 'does-not-fit', reason: result.reason };
130
+ }
131
+ // `model-unreadable`, `no-backend-device`, and `unsupported-config` are all
132
+ // absence of evidence, not evidence of insufficiency.
133
+ return unknown(result.reason);
134
+ }
135
+ function report(logger, input, outcome) {
136
+ const prefix = `[advisory-fit:${input.modelType}:${input.modelId}]`;
137
+ if (outcome.verdict === 'fit') {
138
+ const plan = outcome.plan;
139
+ logger.info(`${prefix} projected to fit (advisory only)${plan === undefined
140
+ ? ''
141
+ : ` — nCtx ${plan.nCtx}, nGpuLayers ${plan.nGpuLayers} across ${plan.nGpuDevices} GPU device(s)`}`);
142
+ return;
143
+ }
144
+ if (outcome.verdict === 'does-not-fit') {
145
+ logger.warn(`${prefix} projected not to fit (advisory only — the load continues unchanged)`);
146
+ return;
147
+ }
148
+ // `unsupported-load` covers every load this check refuses up front — all
149
+ // non-llama.cpp model types — so at `info` it would tag every whisper/tts/ocr
150
+ // load on every start. `debug` for those; `info` where a fitter ran (or
151
+ // should have) and produced no verdict, because there "why was there no
152
+ // evidence" is the most useful thing this check can report.
153
+ const line = `${prefix} no fit evidence: ${outcome.reason}${outcome.message === undefined ? '' : ` (${outcome.message})`}`;
154
+ if (outcome.reason === 'unsupported-load') {
155
+ logger.debug(line);
156
+ return;
157
+ }
158
+ logger.info(line);
159
+ }
160
+ /**
161
+ * Runs the advisory llama.cpp fit check for a load that is about to start.
162
+ *
163
+ * Fail-open by construction: an unsupported shape, a crashed or wedged child, a
164
+ * malformed response, and an unexpected internal error all resolve to `unknown`
165
+ * and the caller proceeds with the ordinary load path. This function never
166
+ * throws and never rejects.
167
+ */
168
+ export async function runAdvisoryFitCheck(input, options = {}) {
169
+ let logger = options.logger;
170
+ try {
171
+ logger ??= getEngineLogger();
172
+ if (!(await resolveEnabled(options.enabled)))
173
+ return unknown('disabled');
174
+ const mobile = await resolveMobile(options.mobile);
175
+ const plan = createLlamaFitRequest({
176
+ modelType: input.modelType,
177
+ modelPath: input.modelPath,
178
+ modelConfig: input.modelConfig,
179
+ artifacts: input.artifacts,
180
+ isShardedModel: input.isShardedModel
181
+ });
182
+ if (!plan.supported) {
183
+ const outcome = unknown('unsupported-load', plan.detail);
184
+ report(logger, input, outcome);
185
+ return outcome;
186
+ }
187
+ const residentBytes = await (options.residentModelBytes ?? defaultResidentModelBytes)();
188
+ const residentReserveMiB = Math.ceil(residentBytes / BYTES_PER_MIB);
189
+ const runFit = await resolveRunFit(options.runFit, mobile);
190
+ // Always sent, even with a zero reserve: relying on the addon default for
191
+ // the base margin would leave two sources of truth that match today and
192
+ // diverge silently if the addon default moves.
193
+ const result = await runFit(plan.loadKind, { ...plan.config, marginMiB: ADVISORY_FIT_BASE_MARGIN_MIB + residentReserveMiB }, {
194
+ timeoutMs: options.timeoutMs ?? ADVISORY_FIT_TIMEOUT_MS,
195
+ ...(options.signal !== undefined && { signal: options.signal })
196
+ });
197
+ const outcome = result.status === 'completed'
198
+ ? classify(result.result)
199
+ : unknown(result.reason, result.message);
200
+ report(logger, input, outcome);
201
+ return outcome;
202
+ }
203
+ catch (error) {
204
+ const outcome = unknown('internal-error', error instanceof Error ? `${error.name}: ${error.message}` : String(error));
205
+ try {
206
+ if (logger !== undefined)
207
+ report(logger, input, outcome);
208
+ }
209
+ catch {
210
+ // A failing logger must not turn an advisory check into a load failure.
211
+ }
212
+ return outcome;
213
+ }
214
+ }
@@ -0,0 +1,26 @@
1
+ import type { FitLlamaProcessConfig, LlamaLoadKind } from '@qvac/model-fit/process';
2
+ import { type CanonicalModelType } from '../../../schemas/index.js';
3
+ export type LlamaFitRequestPlan = {
4
+ supported: true;
5
+ loadKind: LlamaLoadKind;
6
+ config: FitLlamaProcessConfig;
7
+ } | {
8
+ supported: false;
9
+ detail: string;
10
+ };
11
+ export interface CreateLlamaFitRequestParams {
12
+ modelType: CanonicalModelType;
13
+ modelPath: string;
14
+ modelConfig: unknown;
15
+ artifacts?: Record<string, string> | undefined;
16
+ isShardedModel: boolean;
17
+ }
18
+ /**
19
+ * Builds a protocol-v2 fit request from the same resolved model config the real
20
+ * load is about to use, or explains why this load cannot be answered for.
21
+ *
22
+ * Structural shapes the SDK owns are refused here so the fitter is not asked
23
+ * a question it cannot answer. Value-level policy (device names, symbolic GPU
24
+ * selection, context bounds) stays inside `@qvac/model-fit`.
25
+ */
26
+ export declare function createLlamaFitRequest(params: CreateLlamaFitRequestParams): LlamaFitRequestPlan;