@qvac/inference 0.19.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/NOTICE +95 -39
  2. package/dist/api/audio-gen.d.ts +42 -1
  3. package/dist/api/audio-gen.js +137 -51
  4. package/dist/api/delete-cache.d.ts +7 -1
  5. package/dist/api/delete-cache.js +11 -1
  6. package/dist/api/index.d.ts +2 -1
  7. package/dist/api/index.js +2 -1
  8. package/dist/api/text-to-speech.d.ts +31 -4
  9. package/dist/api/text-to-speech.js +117 -21
  10. package/dist/api/vector-index.d.ts +142 -0
  11. package/dist/api/vector-index.js +168 -0
  12. package/dist/api/world.d.ts +2 -2
  13. package/dist/errors/index.d.ts +12 -0
  14. package/dist/errors/index.js +21 -0
  15. package/dist/handlers/delete-cache.js +3 -0
  16. package/dist/handlers/vector-index.d.ts +2 -0
  17. package/dist/handlers/vector-index.js +40 -0
  18. package/dist/index.d.ts +1 -1
  19. package/dist/index.js +1 -1
  20. package/dist/models/registry/models.d.ts +351 -12
  21. package/dist/models/registry/models.js +2271 -1957
  22. package/dist/models/registry/resource-profiles.js +135 -0
  23. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +2 -0
  24. package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +65 -0
  25. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +9 -1
  26. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +24 -7
  27. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +46 -0
  28. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +218 -0
  29. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +1 -1
  30. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +56 -225
  31. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +8 -0
  32. package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +28 -0
  33. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +248 -0
  34. package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -1
  35. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +5 -2
  36. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -2
  37. package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +1 -1
  38. package/dist/plugins/builtin/ggml-ocr/plugin.js +1 -2
  39. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -4
  40. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -2
  41. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +0 -4
  42. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
  43. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
  44. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -0
  45. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +96 -89
  46. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +0 -1
  47. package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +0 -31
  48. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +48 -27
  49. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +239 -135
  50. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +11 -4
  51. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +18 -16
  52. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +11 -4
  53. package/dist/plugins/builtin/llamacpp-completion/plugin.js +1 -2
  54. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +1 -2
  55. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -38
  56. package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +9 -0
  57. package/dist/plugins/builtin/llamacpp-embedding/transform.js +42 -0
  58. package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +1 -1
  59. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +1 -2
  60. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +1 -1
  61. package/dist/plugins/builtin/parakeet-transcription/plugin.js +1 -2
  62. package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +1 -0
  63. package/dist/plugins/builtin/sdcpp-generation/ops/video.js +8 -2
  64. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +9 -4
  65. package/dist/plugins/builtin/sdcpp-generation/plugin.js +51 -19
  66. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +29 -0
  67. package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +66 -0
  68. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +10 -0
  69. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -5
  70. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +143 -28
  71. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +5 -2
  72. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +182 -66
  73. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +267 -22
  74. package/dist/plugins/builtin/tts-ggml/plugin.js +94 -35
  75. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +1 -1
  76. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +1 -2
  77. package/dist/plugins/index.d.ts +1 -1
  78. package/dist/plugins/index.js +1 -1
  79. package/dist/plugins/ops/load-model.js +22 -2
  80. package/dist/plugins/ops/translate.js +58 -12
  81. package/dist/plugins/ops/translatepsy-utils.d.ts +17 -0
  82. package/dist/plugins/ops/translatepsy-utils.js +32 -0
  83. package/dist/plugins/registry.d.ts +15 -0
  84. package/dist/plugins/registry.js +133 -15
  85. package/dist/plugins/turbovec-backend.d.ts +17 -0
  86. package/dist/plugins/turbovec-backend.js +99 -0
  87. package/dist/registry.js +4 -0
  88. package/dist/resources/model-fit/assess.js +169 -68
  89. package/dist/resources/model-fit/estimators/llm.d.ts +1 -0
  90. package/dist/resources/model-fit/estimators/llm.js +3 -1
  91. package/dist/resources/model-fit/floor.d.ts +31 -0
  92. package/dist/resources/model-fit/floor.js +65 -0
  93. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +46 -0
  94. package/dist/resources/model-fit/native-probe/advisory-fit.js +214 -0
  95. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +26 -0
  96. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +171 -0
  97. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +42 -0
  98. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +192 -0
  99. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +70 -0
  100. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +372 -0
  101. package/dist/resources/native.js +5 -4
  102. package/dist/resources/types.d.ts +1 -3
  103. package/dist/runtime/env.d.ts +2 -0
  104. package/dist/runtime/env.js +9 -2
  105. package/dist/runtime/lifecycle.js +2 -0
  106. package/dist/runtime/model-registry.d.ts +3 -1
  107. package/dist/runtime/model-registry.js +2 -1
  108. package/dist/runtime/request-context.js +17 -0
  109. package/dist/runtime/vector-index-backend.d.ts +25 -0
  110. package/dist/runtime/vector-index-backend.js +1 -0
  111. package/dist/runtime/vector-index-registry.d.ts +76 -0
  112. package/dist/runtime/vector-index-registry.js +161 -0
  113. package/dist/schemas/assess-model-fit.d.ts +84 -0
  114. package/dist/schemas/assess-model-fit.js +72 -3
  115. package/dist/schemas/audio-gen.d.ts +482 -7
  116. package/dist/schemas/audio-gen.js +376 -17
  117. package/dist/schemas/batch-completion-stream.d.ts +10 -0
  118. package/dist/schemas/batch-completion-stream.js +5 -3
  119. package/dist/schemas/bci.d.ts +1 -1
  120. package/dist/schemas/common.d.ts +782 -19
  121. package/dist/schemas/common.js +8 -1
  122. package/dist/schemas/completion-event.d.ts +14 -1
  123. package/dist/schemas/completion-event.js +4 -0
  124. package/dist/schemas/completion-stream.d.ts +22 -0
  125. package/dist/schemas/completion-stream.js +41 -4
  126. package/dist/schemas/config.d.ts +48 -30
  127. package/dist/schemas/errors.d.ts +4 -0
  128. package/dist/schemas/errors.js +22 -0
  129. package/dist/schemas/index.d.ts +1 -0
  130. package/dist/schemas/index.js +1 -0
  131. package/dist/schemas/llamacpp-config.d.ts +4 -8
  132. package/dist/schemas/llamacpp-config.js +4 -4
  133. package/dist/schemas/load-model.d.ts +2604 -180
  134. package/dist/schemas/model-config-schemas.d.ts +228 -5
  135. package/dist/schemas/model-ops.d.ts +3 -0
  136. package/dist/schemas/model-ops.js +6 -0
  137. package/dist/schemas/plugin.d.ts +21 -11
  138. package/dist/schemas/sdcpp-config.d.ts +221 -4
  139. package/dist/schemas/sdcpp-config.js +128 -22
  140. package/dist/schemas/text-to-speech.d.ts +780 -19
  141. package/dist/schemas/text-to-speech.js +298 -44
  142. package/dist/schemas/transcription-config.js +7 -4
  143. package/dist/schemas/vector-index.d.ts +194 -0
  144. package/dist/schemas/vector-index.js +185 -0
  145. package/dist/utils/addon-responses.d.ts +30 -3
  146. package/dist/utils/aggregate-events.d.ts +3 -1
  147. package/dist/utils/aggregate-events.js +7 -1
  148. package/dist/utils/tts-stats.d.ts +15 -11
  149. package/dist/utils/tts-stats.js +33 -11
  150. package/package.json +29 -26
@@ -5,9 +5,41 @@ import { createAddonLoggerCallback } from '../logging/addon.js';
5
5
  import { getEngineLogger } from '../logging/index.js';
6
6
  import { formatZodError } from '../utils/zod-error.js';
7
7
  const plugins = new Map();
8
+ /** Addon logging modules wired through a resolver, keyed by namespace. */
9
+ const lazyAddonLoggers = new Map();
10
+ /** In-flight resolutions, so concurrent loads wire a namespace exactly once. */
11
+ const pendingAddonLoggers = new Map();
12
+ function getLoggingResolver(plugin) {
13
+ const declared = plugin.logging?.module;
14
+ return typeof declared === 'function' ? declared : undefined;
15
+ }
16
+ /**
17
+ * The module a plugin supplied directly, or `undefined` when it supplied a
18
+ * resolver. Resolver-based plugins are therefore invisible to
19
+ * registration-time shape validation, to the shared-module dedupe, and to the
20
+ * eager release sweeps; `ensureAddonLoggerReady` and `lazyAddonLoggers` cover
21
+ * them instead, keyed by namespace rather than held on the plugin.
22
+ */
8
23
  function getLoggingModule(plugin) {
24
+ if (getLoggingResolver(plugin))
25
+ return undefined;
9
26
  return plugin.logging?.module;
10
27
  }
28
+ /** CommonJS addons resolve through `import()` as a namespace with `default`. */
29
+ function unwrapLoggingModule(resolved) {
30
+ if (resolved && typeof resolved === 'object' && 'default' in resolved) {
31
+ const inner = resolved.default;
32
+ if (inner && typeof inner.setLogger === 'function')
33
+ return inner;
34
+ }
35
+ return resolved;
36
+ }
37
+ function assertLoggingModuleShape(modelType, candidate) {
38
+ if (!candidate || typeof candidate['setLogger'] !== 'function') {
39
+ throw new PluginLoggingInvalidError(modelType, 'logging.module must have a setLogger(callback) function');
40
+ }
41
+ return candidate;
42
+ }
11
43
  function findPluginUsingLoggingModule(loggingModule) {
12
44
  return Array.from(plugins.values()).find((plugin) => plugin.logging?.module === loggingModule);
13
45
  }
@@ -42,12 +74,11 @@ export function registerPlugin(plugin) {
42
74
  if (pluginProvidingTurboVecIndex) {
43
75
  throw new PluginDefinitionInvalidError(plugin.modelType, `plugin "${plugin.modelType}" cannot provide turbovecIndexProvider because plugin "${pluginProvidingTurboVecIndex.modelType}" already provides it`);
44
76
  }
45
- // Validate logging module shape if provided
46
- if (plugin.logging?.module) {
47
- const loggingModule = plugin.logging.module;
48
- if (typeof loggingModule['setLogger'] !== 'function') {
49
- throw new PluginLoggingInvalidError(plugin.modelType, 'logging.module must have a setLogger(callback) function');
50
- }
77
+ // Validate logging module shape if provided. A resolver is checked when it
78
+ // runs instead: calling it here to inspect its result would load the addon,
79
+ // which is the whole thing registration is meant to avoid.
80
+ if (plugin.logging?.module && !getLoggingResolver(plugin)) {
81
+ assertLoggingModuleShape(plugin.modelType, plugin.logging.module);
51
82
  }
52
83
  const loggingModule = getLoggingModule(plugin);
53
84
  const pluginUsingLoggingModule = loggingModule
@@ -67,6 +98,91 @@ export function registerPlugins(pluginList) {
67
98
  registerPlugin(plugin);
68
99
  }
69
100
  }
101
+ /**
102
+ * Wires the addon logger of a plugin that supplied a resolver, loading the
103
+ * addon on the way. Call it once a model of this type is actually being
104
+ * created: it is the point where the addon is needed anyway, so a host
105
+ * missing that addon's platform package fails the load it asked for instead
106
+ * of failing every plugin registration at startup.
107
+ *
108
+ * A no-op for plugins that passed their module directly — those are already
109
+ * wired by `registerPlugin` — and for a namespace that is already wired,
110
+ * which is how two plugins over one addon (whisper and Parakeet over ASR)
111
+ * share a single `setLogger` call. Also a no-op when the registry was cleared
112
+ * while the addon was loading, so a shutdown that races a load leaves no
113
+ * logger behind.
114
+ */
115
+ export async function ensureAddonLoggerReady(plugin) {
116
+ const resolver = getLoggingResolver(plugin);
117
+ const namespace = plugin.logging?.namespace;
118
+ if (!resolver || !namespace)
119
+ return;
120
+ if (lazyAddonLoggers.has(namespace))
121
+ return;
122
+ const pending = pendingAddonLoggers.get(namespace);
123
+ if (pending)
124
+ return pending;
125
+ const wiring = (async () => {
126
+ const resolved = unwrapLoggingModule(await resolver());
127
+ // Loading the addon takes real time, and the registry can be torn down
128
+ // while this is parked: `runCleanup` calls `clearRegistries()`
129
+ // synchronously and only then awaits `unloadAllModels()`, so a sweep can
130
+ // pass between the call above and this line. Wiring now would hand the
131
+ // addon a callback nothing will release — on Expo, a `js_ref_t` leaked
132
+ // into a dying isolate, which the next worklet's first `setLogger` trips
133
+ // over. Nothing has been wired yet, so dropping the resolved module is
134
+ // the whole cleanup.
135
+ if (!isNamespaceClaimed(namespace))
136
+ return;
137
+ const loggingModule = assertLoggingModuleShape(plugin.modelType, resolved);
138
+ for (const [wiredNamespace, wiredModule] of lazyAddonLoggers) {
139
+ if (wiredModule === loggingModule && wiredNamespace !== namespace) {
140
+ throw new PluginLoggingInvalidError(plugin.modelType, 'plugins sharing logging.module must use the same namespace');
141
+ }
142
+ }
143
+ loggingModule.setLogger(createAddonLoggerCallback(namespace));
144
+ lazyAddonLoggers.set(namespace, loggingModule);
145
+ })();
146
+ pendingAddonLoggers.set(namespace, wiring);
147
+ try {
148
+ await wiring;
149
+ }
150
+ finally {
151
+ pendingAddonLoggers.delete(namespace);
152
+ }
153
+ }
154
+ /** Whether a registered resolver-based plugin still claims this namespace. */
155
+ function isNamespaceClaimed(namespace) {
156
+ return Array.from(plugins.values()).some((candidate) => getLoggingResolver(candidate) && candidate.logging?.namespace === namespace);
157
+ }
158
+ /** Releases a lazily-wired addon logger once no registered plugin claims it. */
159
+ function releaseLazyAddonLogger(namespace, modelType) {
160
+ if (namespace === undefined)
161
+ return;
162
+ const loggingModule = lazyAddonLoggers.get(namespace);
163
+ if (!loggingModule)
164
+ return;
165
+ if (isNamespaceClaimed(namespace))
166
+ return;
167
+ lazyAddonLoggers.delete(namespace);
168
+ releaseLoggerSafely(loggingModule, modelType);
169
+ }
170
+ /**
171
+ * Runs an addon's `releaseLogger`. A failure must not abort a sweep or leave
172
+ * the registry half-cleared for the next caller, but it is reported rather
173
+ * than swallowed: a leaked reference or a live async handle must not pass for
174
+ * a clean teardown. `subject` names what is being released — a model type, or
175
+ * a namespace for a logger that is only keyed by one — and `during` names the
176
+ * sweep, when the release is part of one.
177
+ */
178
+ function releaseLoggerSafely(loggingModule, subject, during) {
179
+ try {
180
+ loggingModule.releaseLogger?.();
181
+ }
182
+ catch (error) {
183
+ getEngineLogger().warn(`[${subject}] releaseLogger failed${during ? ` during ${during}` : ''}: ${error instanceof Error ? error.message : String(error)}`);
184
+ }
185
+ }
70
186
  export function getPlugin(modelType) {
71
187
  return plugins.get(modelType);
72
188
  }
@@ -84,10 +200,12 @@ export function unregisterPlugin(modelType) {
84
200
  if (!plugin)
85
201
  return false;
86
202
  const loggingModule = getLoggingModule(plugin);
203
+ const resolverNamespace = getLoggingResolver(plugin) ? plugin.logging?.namespace : undefined;
87
204
  plugins.delete(modelType);
88
205
  if (loggingModule && !findPluginUsingLoggingModule(loggingModule)) {
89
206
  loggingModule.releaseLogger?.();
90
207
  }
208
+ releaseLazyAddonLogger(resolverNamespace, modelType);
91
209
  return true;
92
210
  }
93
211
  export function getAllPlugins() {
@@ -104,16 +222,16 @@ export function clearPlugins() {
104
222
  loggingModules.set(loggingModule, plugin.modelType);
105
223
  }
106
224
  }
225
+ const lazyModules = new Map(lazyAddonLoggers);
107
226
  plugins.clear();
227
+ lazyAddonLoggers.clear();
108
228
  for (const [loggingModule, modelType] of loggingModules) {
109
- try {
110
- loggingModule.releaseLogger?.();
111
- }
112
- catch (error) {
113
- // A plugin's logger teardown must not abort the sweep or leave the
114
- // registry half-cleared for the next caller — but surface it, so a leaked
115
- // reference or async handle is not masked as a clean teardown.
116
- getEngineLogger().warn(`[${modelType}] releaseLogger failed during clearPlugins: ${error instanceof Error ? error.message : String(error)}`);
117
- }
229
+ releaseLoggerSafely(loggingModule, modelType, 'clearPlugins');
230
+ }
231
+ // Loggers wired through a resolver are keyed by namespace rather than held
232
+ // on the plugin, so they need the same sweep. They are named by that
233
+ // namespace, which is not a model type — several plugins can share one.
234
+ for (const [namespace, loggingModule] of lazyModules) {
235
+ releaseLoggerSafely(loggingModule, `namespace ${namespace}`, 'clearPlugins');
118
236
  }
119
237
  }
@@ -0,0 +1,17 @@
1
+ import { VectorIndexStorage } from '../schemas/index.js';
2
+ import type { VectorIndexBackend } from '../runtime/vector-index-backend.js';
3
+ /**
4
+ * The native index reports only its effective bit width after a load. Three
5
+ * widths map to one storage mode each; 4 bits is either `q4` or
6
+ * `turbovec-q4`, which the snapshot header distinguishes but the addon does
7
+ * not expose yet, so those loads report no storage mode.
8
+ */
9
+ export declare function storageFromBitWidth(bitWidth: unknown): VectorIndexStorage | undefined;
10
+ /**
11
+ * The vector index feature's only backend: the fabric vector index that the
12
+ * embedding plugin exposes through the `turbovecIndexProvider` capability.
13
+ * It serves every `VectorIndexStorage` mode; the two TurboVec modes add the
14
+ * dimension rule (multiple of 8, at most 1024), which the native index
15
+ * enforces itself.
16
+ */
17
+ export declare function getTurboVecBackend(): VectorIndexBackend | undefined;
@@ -0,0 +1,99 @@
1
+ import { getTurboVecIndexProvider } from './registry.js';
2
+ import { VECTOR_ID_RESERVED, VectorIndexStorage } from '../schemas/index.js';
3
+ // The native index pads short result rows with UINT64_MAX.
4
+ const PADDING_ID = BigInt(VECTOR_ID_RESERVED);
5
+ /**
6
+ * The native index reports only its effective bit width after a load. Three
7
+ * widths map to one storage mode each; 4 bits is either `q4` or
8
+ * `turbovec-q4`, which the snapshot header distinguishes but the addon does
9
+ * not expose yet, so those loads report no storage mode.
10
+ */
11
+ export function storageFromBitWidth(bitWidth) {
12
+ switch (bitWidth) {
13
+ case 32:
14
+ return VectorIndexStorage.F32;
15
+ case 8:
16
+ return VectorIndexStorage.Q8;
17
+ case 2:
18
+ return VectorIndexStorage.TURBOVEC_Q2;
19
+ default:
20
+ return undefined;
21
+ }
22
+ }
23
+ function toHitRows(result) {
24
+ const rows = [];
25
+ for (let row = 0; row < result.m; row++) {
26
+ const hits = [];
27
+ for (let slot = 0; slot < result.k; slot++) {
28
+ const offset = row * result.k + slot;
29
+ const id = result.ids[offset];
30
+ if (id === undefined || id === PADDING_ID)
31
+ break;
32
+ hits.push({ id: id.toString(), score: result.scores[offset] ?? 0 });
33
+ }
34
+ rows.push(hits);
35
+ }
36
+ return rows;
37
+ }
38
+ function openIndex(index, storage) {
39
+ // TurboVec precomputes rotation and codebook state on the first search
40
+ // after a change; running it eagerly keeps that cost out of the first
41
+ // query's latency.
42
+ let needsPrepare = true;
43
+ return {
44
+ get dim() {
45
+ return index.dim;
46
+ },
47
+ get length() {
48
+ return index.length;
49
+ },
50
+ storage,
51
+ add(vectors, ids) {
52
+ index.addWithIds(vectors, ids);
53
+ needsPrepare = true;
54
+ },
55
+ search(queries, k) {
56
+ if (needsPrepare) {
57
+ index.prepare();
58
+ needsPrepare = false;
59
+ }
60
+ return toHitRows(index.search(queries, k));
61
+ },
62
+ contains(id) {
63
+ return index.contains(id);
64
+ },
65
+ remove(id) {
66
+ const removed = index.remove(id);
67
+ if (removed)
68
+ needsPrepare = true;
69
+ return removed;
70
+ },
71
+ write(path) {
72
+ index.write(path);
73
+ },
74
+ dispose() {
75
+ index.dispose();
76
+ }
77
+ };
78
+ }
79
+ /**
80
+ * The vector index feature's only backend: the fabric vector index that the
81
+ * embedding plugin exposes through the `turbovecIndexProvider` capability.
82
+ * It serves every `VectorIndexStorage` mode; the two TurboVec modes add the
83
+ * dimension rule (multiple of 8, at most 1024), which the native index
84
+ * enforces itself.
85
+ */
86
+ export function getTurboVecBackend() {
87
+ const provider = getTurboVecIndexProvider();
88
+ if (!provider)
89
+ return undefined;
90
+ return {
91
+ create(dim, storage) {
92
+ return openIndex(provider.create({ dim, storage }), storage);
93
+ },
94
+ load(path) {
95
+ const index = provider.load(path);
96
+ return openIndex(index, storageFromBitWidth(index.bitWidth));
97
+ }
98
+ };
99
+ }
package/dist/registry.js CHANGED
@@ -4,6 +4,7 @@ import { handleUnloadModel } from './handlers/unload-model.js';
4
4
  import { handleLoggingStream } from './handlers/logging-stream.js';
5
5
  import { cancelHandler } from './handlers/cancelHandler.js';
6
6
  import { handleRag } from './rag/handler.js';
7
+ import { handleVectorIndex } from './handlers/vector-index.js';
7
8
  import { handleDeleteCache } from './handlers/delete-cache.js';
8
9
  import { handleDownloadAsset } from './handlers/download-asset.js';
9
10
  import { handleGetModelInfo } from './handlers/get-model-info.js';
@@ -74,6 +75,7 @@ export const registry = {
74
75
  },
75
76
  downloadAsset: { type: 'reply', handler: handleDownloadAsset, supportsProgress: true },
76
77
  rag: { type: 'reply', handler: handleRag, supportsProgress: ragSupportsProgress },
78
+ vectorIndex: { type: 'reply', handler: handleVectorIndex },
77
79
  finetune: { type: 'reply', handler: handleFinetune, supportsProgress: finetuneSupportsProgress },
78
80
  // Stream
79
81
  completionStream: {
@@ -94,6 +96,8 @@ export const registry = {
94
96
  ocrStream: { type: 'stream', pluginOp: true, handler: pluginStream('ocrStream') },
95
97
  diffusionStream: { type: 'stream', pluginOp: true, handler: pluginStream('diffusionStream') },
96
98
  audioGenStream: { type: 'stream', pluginOp: true, handler: pluginStream('audioGenStream') },
99
+ audioEditStream: { type: 'stream', pluginOp: true, handler: pluginStream('audioEditStream') },
100
+ audioUnderstand: { type: 'stream', pluginOp: true, handler: pluginStream('audioUnderstand') },
97
101
  videoStream: { type: 'stream', pluginOp: true, handler: pluginStream('videoStream') },
98
102
  upscaleStream: { type: 'stream', pluginOp: true, handler: pluginStream('upscaleStream') },
99
103
  worldStepStream: { type: 'stream', pluginOp: true, handler: pluginStream('worldStepStream') },
@@ -2,6 +2,7 @@ import { getModelResourceProfile } from '../../models/registry/resource-profiles
2
2
  import { getGpuCalibration, getSharedGpuCalibration } from './calibration/index.js';
3
3
  import { estimateLlm } from './estimators/llm.js';
4
4
  import { estimateWhisper } from './estimators/whisper.js';
5
+ import { computeFloor, FLOOR_VERSION } from './floor.js';
5
6
  const GIB = 1024 * 1024 * 1024;
6
7
  /** Engines phase 1 can estimate, and which estimator owns each. */
7
8
  const ESTIMATORS = {
@@ -10,6 +11,12 @@ const ESTIMATORS = {
10
11
  'whispercpp-transcription': estimateWhisper
11
12
  };
12
13
  const MOBILE_PLATFORMS = ['android-arm64', 'ios-arm64'];
14
+ // Every allocation, GPU included, is system RAM: the system budget always bounds the load.
15
+ const UNIFIED_MEMORY_PLATFORMS = [
16
+ 'darwin-arm64',
17
+ 'android-arm64',
18
+ 'ios-arm64'
19
+ ];
13
20
  /**
14
21
  * Platforms whose fixture describes CPU-resident execution (`device: 'cpu'`),
15
22
  * so a reported GPU sends the host through `resolveGpuPlacement` instead.
@@ -98,7 +105,12 @@ export function assessModelFitFromResources(options) {
98
105
  else if (effectiveCalibration.measuredAt) {
99
106
  assumptions.push(calibrationAssumption(platform, effectiveCalibration));
100
107
  }
101
- const evaluated = models.map((candidate) => evaluate(candidate, platform, effectiveCalibration, resources, resolveProfile, gpuMode));
108
+ // Without coefficients, the computed floor still says something — but only
109
+ // where the model executes out of the memory the budget measures. A discrete
110
+ // card holds the weights in its own memory, so the system budget bounds
111
+ // nothing there and the floor has no budget to be compared against.
112
+ const floorApplies = !onDevice && boundBySystemMemory(resources, platform);
113
+ const evaluated = models.map((candidate) => evaluate(candidate, platform, effectiveCalibration, resources, resolveProfile, gpuMode, floorApplies));
102
114
  for (const { result } of evaluated) {
103
115
  for (const assumption of result.assumptions ?? []) {
104
116
  if (!assumptions.includes(assumption))
@@ -106,19 +118,37 @@ export function assessModelFitFromResources(options) {
106
118
  }
107
119
  }
108
120
  const modelResults = evaluated.map(({ candidate, result }) => toModelResult(candidate, result, budget, deviceBudgets, alsoBoundBy));
109
- const estimates = evaluated
110
- .map(({ result }) => result)
111
- .filter((result) => {
121
+ const results = evaluated.map(({ result }) => result);
122
+ const estimates = results.filter((result) => {
112
123
  return result.kind === 'estimate';
113
124
  });
114
- const anyUnknown = estimates.length !== evaluated.length;
115
- const combined = anyUnknown ? undefined : aggregate(estimates, execution);
125
+ const anyUnknown = results.some((result) => result.kind === 'unknown');
126
+ const anyFloor = results.some((result) => result.kind === 'floor');
127
+ // A two-sided bound needs every candidate estimated; a floor needs every
128
+ // candidate to have at least a floor. One model with neither leaves the set
129
+ // with no evidence at all.
130
+ const combined = anyUnknown || anyFloor ? undefined : aggregate(estimates, execution);
131
+ const combinedFloor = anyUnknown ? undefined : aggregateFloor(results, execution);
132
+ // Absent when a candidate rests on nothing: the set then has no single kind
133
+ // of evidence to name, and an `unknown` with `evidence` present stays a
134
+ // near-miss or an uncalibrated floor rather than a missing model.
135
+ const evidence = anyUnknown
136
+ ? undefined
137
+ : anyFloor
138
+ ? 'computed-only'
139
+ : estimates.length > 0
140
+ ? 'calibration'
141
+ : undefined;
116
142
  if (anyUnknown) {
117
143
  reasons.push('at least one model could not be estimated, so the combined verdict is unknown');
118
144
  }
119
- const verdict = !budget || !combined
145
+ const verdict = !budget
120
146
  ? 'unknown'
121
- : verdictAgainst(combined, deviceBudgets ?? [budget], alsoBoundBy);
147
+ : combined
148
+ ? verdictAgainst(combined, deviceBudgets ?? [budget], alsoBoundBy)
149
+ : combinedFloor !== undefined
150
+ ? floorVerdict(combinedFloor, deviceBudgets ?? [budget])
151
+ : 'unknown';
122
152
  if (onDevice && combined && alsoBoundBy) {
123
153
  reasons.push('a GPU load is also paid for in system RAM, so every verdict is the more pessimistic of the GPU and system budgets');
124
154
  }
@@ -130,92 +160,110 @@ export function assessModelFitFromResources(options) {
130
160
  ? 'all models counted as resident with every working peak added'
131
161
  : 'all models counted as resident with only the largest working peak added');
132
162
  }
163
+ else if (budget && combinedFloor !== undefined) {
164
+ reasons.push(execution === 'concurrent'
165
+ ? 'all models counted as resident with their floors summed and every calibrated working peak added'
166
+ : 'all models counted as resident with their floors summed and only the largest calibrated working peak added');
167
+ }
168
+ if (anyFloor) {
169
+ reasons.push('at least one model is bounded only by its computed floor (weights, plus the KV cache for llama.cpp), which can refuse the set but never confirm a fit');
170
+ }
133
171
  return {
134
172
  verdict,
135
173
  basis,
136
174
  execution,
175
+ ...(evidence && { evidence }),
137
176
  ...(budget && { budget }),
138
177
  ...(combined && {
139
178
  estimate: { lowerBoundBytes: combined.lower, upperBoundBytes: combined.upper }
140
179
  }),
180
+ ...(anyFloor && combinedFloor !== undefined && { floorBytes: combinedFloor }),
141
181
  models: modelResults,
142
182
  reasons,
143
183
  assumptions
144
184
  };
145
185
  }
146
- function evaluate(candidate, platform, calibration, resources, resolveProfile, gpuMode) {
147
- if (!calibration) {
148
- return {
149
- candidate,
150
- result: {
151
- kind: 'unknown',
152
- estimatorVersion: 'none',
153
- reasons: [
154
- platform
155
- ? `no validated calibration for ${platform}`
156
- : 'the runtime platform is not one this assessment covers'
157
- ]
158
- }
159
- };
160
- }
161
- // A GPU is present and neither GPU mode engaged, so the only coefficients
162
- // left were measured with the offload disabled — not how this host runs.
163
- if (!gpuMode && platform && CPU_CALIBRATED_PLATFORMS.includes(platform) && hasGpu(resources)) {
164
- return {
165
- candidate,
166
- result: {
167
- kind: 'unknown',
168
- estimatorVersion: 'none',
169
- reasons: [
170
- 'a GPU is present, so the model executes on it rather than on the CPU this platform’s coefficients were measured against'
171
- ]
172
- }
173
- };
174
- }
186
+ /**
187
+ * Whether every allocation a load makes comes out of the memory the system (or
188
+ * process) budget measures — the precondition for the computed floor to be
189
+ * compared against that budget.
190
+ *
191
+ * True on unified-memory platforms whatever the collector reports, and
192
+ * elsewhere when no usable GPU is present or every usable GPU allocates out of
193
+ * system RAM. A card with its own memory, or a device whose class cannot be
194
+ * told (an AMD APU on linux), fails it: the weights may live where the budget
195
+ * cannot see them.
196
+ */
197
+ function boundBySystemMemory(resources, platform) {
198
+ if (platform !== undefined && UNIFIED_MEMORY_PLATFORMS.includes(platform))
199
+ return true;
200
+ const gpus = usableGpus(resources);
201
+ if (gpus.length === 0)
202
+ return true;
203
+ if (platform && gpus.some((gpu) => integratedIsIndistinguishable(gpu, platform)))
204
+ return false;
205
+ return gpus.every(allocatesFromSystemMemory);
206
+ }
207
+ /**
208
+ * One candidate's evaluation: the calibrated estimate when the coefficients
209
+ * describe this load, otherwise the computed floor where a floor can be
210
+ * compared at all, otherwise `unknown`. Whatever kept the estimate from
211
+ * forming stays in the reasons, so a floor verdict still says why it is only
212
+ * a floor.
213
+ */
214
+ function evaluate(candidate, platform, calibration, resources, resolveProfile, gpuMode, floorApplies) {
175
215
  const profile = resolveProfile(candidate.model.sha256Checksum);
176
216
  if (!profile) {
177
- return {
178
- candidate,
179
- result: {
180
- kind: 'unknown',
181
- estimatorVersion: 'none',
182
- reasons: ['no resource profile in the catalog for this checksum']
183
- }
184
- };
185
- }
186
- const estimator = ESTIMATORS[profile.engine];
187
- if (!estimator) {
188
- return {
189
- candidate,
190
- result: {
191
- kind: 'unknown',
192
- estimatorVersion: 'none',
193
- reasons: [`engine '${profile.engine}' has no estimator in this phase`]
194
- }
195
- };
217
+ return { candidate, result: unknown('no resource profile in the catalog for this checksum') };
196
218
  }
197
219
  const extra = extraArtifactBytes(candidate, resolveProfile);
198
220
  if (extra === undefined) {
199
221
  return {
200
222
  candidate,
201
- result: {
202
- kind: 'unknown',
203
- estimatorVersion: 'none',
204
- reasons: ['an entry in `artifacts` has no resource profile in the catalog']
205
- }
223
+ result: unknown('an entry in `artifacts` has no resource profile in the catalog')
206
224
  };
207
225
  }
226
+ const estimated = estimate(candidate, profile, extra, platform, calibration, resources, gpuMode);
227
+ if (estimated.kind === 'estimate' || !floorApplies)
228
+ return { candidate, result: estimated };
229
+ const floor = computeFloor({ profile, workload: candidate.workload, extraArtifactBytes: extra });
208
230
  return {
209
231
  candidate,
210
- result: estimator({
211
- profile,
212
- workload: candidate.workload,
213
- extraArtifactBytes: extra,
214
- calibration,
215
- hasGpu: hasGpu(resources)
216
- })
232
+ result: {
233
+ kind: 'floor',
234
+ bytes: floor.bytes,
235
+ reasons: [...estimated.reasons, ...floor.reasons],
236
+ assumptions: [...(estimated.assumptions ?? []), ...floor.assumptions]
237
+ }
217
238
  };
218
239
  }
240
+ function unknown(reason) {
241
+ return { kind: 'unknown', estimatorVersion: 'none', reasons: [reason] };
242
+ }
243
+ /** Runs the calibrated estimator for a candidate, or says why it cannot. */
244
+ function estimate(candidate, profile, extraArtifactBytes, platform, calibration, resources, gpuMode) {
245
+ if (!calibration) {
246
+ return unknown(platform
247
+ ? `no validated calibration for ${platform}`
248
+ : 'the runtime platform is not one this assessment covers');
249
+ }
250
+ // A GPU is present and neither GPU mode engaged, so the only coefficients
251
+ // left were measured with the offload disabled — not how this host runs.
252
+ if (!gpuMode && platform && CPU_CALIBRATED_PLATFORMS.includes(platform) && hasGpu(resources)) {
253
+ return unknown('a GPU is present, so the model executes on it rather than on the CPU this platform’s coefficients were measured against');
254
+ }
255
+ const estimator = ESTIMATORS[profile.engine];
256
+ if (!estimator) {
257
+ return unknown(`engine '${profile.engine}' has no estimator in this phase`);
258
+ }
259
+ return estimator({
260
+ profile,
261
+ workload: candidate.workload,
262
+ extraArtifactBytes,
263
+ calibration,
264
+ hasGpu: hasGpu(resources)
265
+ });
266
+ }
219
267
  /**
220
268
  * Sums the artifact bytes of companion constants.
221
269
  *
@@ -545,6 +593,40 @@ function aggregate(estimates, execution) {
545
593
  }
546
594
  return { lower: persistentLower + workingLower, upper: persistentUpper + workingUpper };
547
595
  }
596
+ /**
597
+ * The combined lower bound when at least one candidate has only a floor: every
598
+ * estimate contributes its own lower bound, every floor its bytes, aggregated
599
+ * the same way as `aggregate` — a floor has no working peak to add.
600
+ *
601
+ * @returns `undefined` when any candidate has no evidence at all.
602
+ */
603
+ function aggregateFloor(results, execution) {
604
+ let persistent = 0;
605
+ let working = 0;
606
+ for (const result of results) {
607
+ if (result.kind === 'unknown')
608
+ return undefined;
609
+ if (result.kind === 'floor') {
610
+ persistent += result.bytes;
611
+ continue;
612
+ }
613
+ persistent += result.persistent.lower;
614
+ working =
615
+ execution === 'concurrent'
616
+ ? working + result.working.lower
617
+ : Math.max(working, result.working.lower);
618
+ }
619
+ return persistent + working;
620
+ }
621
+ /**
622
+ * A floor can only refuse: over the largest candidate budget it is
623
+ * `likely-too-large`, and anything else is `unknown`, because the cost above
624
+ * the floor is unmeasured and could be anything.
625
+ */
626
+ function floorVerdict(floorBytes, candidates) {
627
+ const room = candidates.map((budget) => budget.availableAfterReserveBytes);
628
+ return floorBytes > Math.max(...room) ? 'likely-too-large' : 'unknown';
629
+ }
548
630
  function compare(estimate, budget) {
549
631
  if (estimate.lower > budget)
550
632
  return 'likely-too-large';
@@ -562,6 +644,24 @@ deviceBudgets, alsoBoundBy) {
562
644
  reasons: [...result.reasons]
563
645
  };
564
646
  }
647
+ if (result.kind === 'floor') {
648
+ const verdict = budget ? floorVerdict(result.bytes, deviceBudgets ?? [budget]) : 'unknown';
649
+ return {
650
+ name: candidate.model.name,
651
+ verdict,
652
+ evidence: 'computed-only',
653
+ floorBytes: result.bytes,
654
+ estimatorVersion: FLOOR_VERSION,
655
+ reasons: [
656
+ ...result.reasons,
657
+ !budget
658
+ ? 'no usable system-memory sample, so this model has no verdict'
659
+ : verdict === 'likely-too-large'
660
+ ? 'the computed floor alone exceeds the budget'
661
+ : 'the computed floor is within the budget, but without calibration the cost above it is unbounded, so a fit cannot be claimed'
662
+ ]
663
+ };
664
+ }
565
665
  const total = {
566
666
  lower: result.persistent.lower + result.working.lower,
567
667
  upper: result.persistent.upper + result.working.upper
@@ -569,6 +669,7 @@ deviceBudgets, alsoBoundBy) {
569
669
  return {
570
670
  name: candidate.model.name,
571
671
  verdict: budget ? verdictAgainst(total, deviceBudgets ?? [budget], alsoBoundBy) : 'unknown',
672
+ evidence: 'calibration',
572
673
  estimate: { lowerBoundBytes: total.lower, upperBoundBytes: total.upper },
573
674
  estimatorVersion: result.estimatorVersion,
574
675
  reasons: budget
@@ -1,6 +1,7 @@
1
1
  import type { GgufFacts } from '../../../schemas/model-resource-profile.js';
2
2
  import type { ByteRange, EstimatorInput, EstimatorResult } from '../types.js';
3
3
  export declare const LLM_ESTIMATOR_VERSION = "llm-v1";
4
+ export declare const LLAMA_WEIGHTS_ASSUMPTION = "weights are counted at full artifact size; llama.cpp maps them by default, so those pages are file-backed and evictable rather than anonymous RAM";
4
5
  /**
5
6
  * Estimates memory for a llama.cpp completion or embedding model from catalog
6
7
  * metadata alone.