@qvac/inference 0.17.1 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/dist/api/completion-stream.d.ts +1 -1
  2. package/dist/api/completion-stream.js +1 -1
  3. package/dist/api/index.d.ts +1 -1
  4. package/dist/api/index.js +1 -1
  5. package/dist/api/text-to-speech.js +22 -2
  6. package/dist/api/transcribe.d.ts +19 -2
  7. package/dist/api/transcribe.js +66 -27
  8. package/dist/api/translate.js +6 -17
  9. package/dist/api/vla.d.ts +42 -1
  10. package/dist/api/vla.js +50 -1
  11. package/dist/constants/audio.d.ts +2 -1
  12. package/dist/constants/audio.js +17 -1
  13. package/dist/errors/index.d.ts +3 -0
  14. package/dist/errors/index.js +5 -0
  15. package/dist/handlers/load-model/handler.js +2 -2
  16. package/dist/handlers/load-model/resolve-session.d.ts +1 -1
  17. package/dist/handlers/load-model/resolve-session.js +3 -3
  18. package/dist/handlers/load-model/resolve.d.ts +12 -2
  19. package/dist/handlers/load-model/resolve.js +90 -12
  20. package/dist/index.d.ts +1 -1
  21. package/dist/index.js +1 -1
  22. package/dist/models/registry/models.d.ts +782 -6
  23. package/dist/models/registry/models.js +2664 -1859
  24. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +19 -0
  25. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +153 -0
  26. package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +54 -4
  27. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +28 -0
  28. package/dist/plugins/builtin/ggml-ocr/plugin.js +3 -1
  29. package/dist/plugins/builtin/ggml-vla/ops/vla-set-embodiment.d.ts +2 -0
  30. package/dist/plugins/builtin/ggml-vla/ops/vla-set-embodiment.js +9 -0
  31. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +37 -0
  32. package/dist/plugins/builtin/ggml-vla/plugin.js +21 -4
  33. package/dist/plugins/builtin/index.d.ts +1 -0
  34. package/dist/plugins/builtin/index.js +1 -0
  35. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +16 -15
  36. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -1
  37. package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +2 -2
  38. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +6 -0
  39. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +10 -0
  40. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +1 -0
  41. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +131 -123
  42. package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +110 -20
  43. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +50 -23
  44. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +308 -143
  45. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +1 -0
  46. package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -2
  47. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +6 -6
  48. package/dist/plugins/builtin/llamacpp-completion/plugin.js +56 -36
  49. package/dist/plugins/builtin/llamacpp-completion/transform.js +0 -7
  50. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -3
  51. package/dist/plugins/builtin/nmtcpp-translation/plugin.js +6 -5
  52. package/dist/plugins/builtin/parakeet-transcription/plugin.js +27 -16
  53. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +23 -3
  54. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +302 -4
  55. package/dist/plugins/builtin/tts-ggml/plugin.js +131 -0
  56. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +48 -39
  57. package/dist/plugins/ops/cancel.d.ts +1 -1
  58. package/dist/plugins/ops/cancel.js +1 -1
  59. package/dist/plugins/ops/kv-cache-retention.js +11 -0
  60. package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
  61. package/dist/plugins/ops/kv-cache-utils.js +40 -12
  62. package/dist/plugins/ops/transcribe.d.ts +2 -2
  63. package/dist/plugins/ops/transcribe.js +54 -48
  64. package/dist/plugins/ops/translate-stats.d.ts +4 -0
  65. package/dist/plugins/ops/translate-stats.js +11 -0
  66. package/dist/plugins/ops/translate.js +66 -38
  67. package/dist/plugins/ops/unload-model.js +30 -24
  68. package/dist/runtime/model-registry.d.ts +1 -0
  69. package/dist/runtime/request-context.js +29 -14
  70. package/dist/runtime/request-registry.d.ts +27 -0
  71. package/dist/runtime/request-registry.js +246 -76
  72. package/dist/schemas/audio-gen.d.ts +103 -0
  73. package/dist/schemas/audio-gen.js +135 -4
  74. package/dist/schemas/common.d.ts +550 -11
  75. package/dist/schemas/config.d.ts +60 -24
  76. package/dist/schemas/errors.d.ts +1 -0
  77. package/dist/schemas/errors.js +5 -0
  78. package/dist/schemas/index.d.ts +1 -1
  79. package/dist/schemas/index.js +1 -1
  80. package/dist/schemas/llamacpp-config.d.ts +10 -10
  81. package/dist/schemas/llamacpp-config.js +12 -5
  82. package/dist/schemas/load-model.d.ts +3691 -336
  83. package/dist/schemas/load-model.js +37 -19
  84. package/dist/schemas/text-to-speech.d.ts +1086 -110
  85. package/dist/schemas/text-to-speech.js +197 -7
  86. package/dist/schemas/tools.d.ts +0 -16
  87. package/dist/schemas/tools.js +0 -15
  88. package/dist/schemas/transcription-config.d.ts +2 -0
  89. package/dist/schemas/transcription-config.js +6 -0
  90. package/dist/schemas/transcription.d.ts +3 -0
  91. package/dist/schemas/vla.d.ts +83 -0
  92. package/dist/schemas/vla.js +105 -2
  93. package/dist/surface.d.ts +1 -1
  94. package/dist/surface.js +1 -1
  95. package/dist/utils/addon-responses.d.ts +4 -12
  96. package/dist/utils/audio/decoder.d.ts +37 -1
  97. package/dist/utils/audio/decoder.js +82 -52
  98. package/dist/utils/config-transform.d.ts +3 -0
  99. package/dist/utils/config-transform.js +6 -0
  100. package/dist/utils/index.d.ts +0 -1
  101. package/dist/utils/index.js +0 -1
  102. package/dist/utils/shard-utils.d.ts +1 -0
  103. package/dist/utils/shard-utils.js +14 -0
  104. package/dist/utils/tool-integration.d.ts +3 -10
  105. package/dist/utils/tool-integration.js +3 -12
  106. package/package.json +26 -22
  107. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +0 -22
  108. package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +0 -30
  109. package/dist/utils/expand-gguf-shards.d.ts +0 -2
  110. package/dist/utils/expand-gguf-shards.js +0 -22
@@ -1,6 +1,7 @@
1
1
  import { generateConfigHash } from '../../../../plugins/ops/kv-cache-utils.js';
2
2
  import { type CacheMessage } from '../../../../utils/index.js';
3
3
  import type { Logger } from '../../../../logging/types.js';
4
+ import { type AbortSignal } from 'bare-abort-controller';
4
5
  export interface TurnHandle {
5
6
  /** Resolved on-disk cache file path the addon will read from / write to. */
6
7
  readonly cachePath: string;
@@ -10,42 +11,65 @@ export interface TurnHandle {
10
11
  * to pick the message tail for the next addon call.
11
12
  */
12
13
  readonly savedCount: number;
14
+ /**
15
+ * Whether the cached prefix already holds a rendered static tool block, so
16
+ * this turn can leave it out of its payload. False on a fresh prime and
17
+ * whenever the previous turn couldn't confirm the block reached the model.
18
+ */
19
+ readonly toolBlockCached: boolean;
13
20
  }
14
21
  export interface BeginCustomTurnInput {
15
22
  kind: 'custom';
16
23
  /** User-provided session key (`completion({ kvCache: "session-a" })`). */
17
24
  customKey: string;
18
- /** Hash of system prompt + (static) tool names. */
25
+ /** Hash of system prompt + complete tool definitions. */
19
26
  configHash: string;
20
27
  /**
21
- * Prime the cache by sending system prompt + (static) tools to the
22
- * addon. Called when the cache doesn't exist in-memory OR on disk.
23
- * Kept as an injected closure so this module has no dependency on the
24
- * model registry / addon — the handler closes over `model` and tools.
28
+ * Prime the cache by sending the system prompt to the addon. Tools are not
29
+ * primed — a prefix with no user turn is not a renderable conversation for
30
+ * every template — so they travel with a turn instead. Called when the cache
31
+ * doesn't exist in-memory OR on disk. Kept as an injected closure so this
32
+ * module has no dependency on the model registry / addon.
25
33
  */
26
34
  primeIfMissing: (cachePath: string) => Promise<void>;
35
+ /**
36
+ * Request abort signal. When it aborts while this turn is queued behind a
37
+ * same-file peer's write lock, the wait is abandoned so the request's scope
38
+ * unwinds and releases its admission slot instead of blocking on the holder.
39
+ */
40
+ signal?: AbortSignal;
27
41
  }
28
42
  export interface BeginAutoTurnInput {
29
43
  kind: 'auto';
30
- /** Hash of system prompt + (static) tool names. */
44
+ /** Hash of system prompt + complete tool definitions. */
31
45
  configHash: string;
32
46
  /** Conversation history used to compute the pre-response cache key. */
33
47
  history: CacheMessage[];
34
48
  /** See `BeginCustomTurnInput.primeIfMissing`. */
35
49
  primeIfMissing: (cachePath: string) => Promise<void>;
50
+ /** See `BeginCustomTurnInput.signal`. */
51
+ signal?: AbortSignal;
36
52
  }
37
53
  export type BeginTurnInput = BeginCustomTurnInput | BeginAutoTurnInput;
38
- export interface StaticCommitResult {
54
+ /**
55
+ * Whether the committed prefix holds a rendered static tool block. The
56
+ * handler owns this because only it knows whether the payload carried the
57
+ * block and whether the message list was one the template renders tools for.
58
+ */
59
+ interface ToolBlockCommit {
60
+ toolBlockCached: boolean;
61
+ }
62
+ export interface StaticCommitResult extends ToolBlockCommit {
39
63
  kind: 'static';
40
64
  /** `history.length + 1` — recorded at the turn's current `cachePath`. */
41
65
  messageCount: number;
42
66
  }
43
- export interface AutoRenameCommitResult {
67
+ export interface AutoRenameCommitResult extends ToolBlockCommit {
44
68
  kind: 'autoRename';
45
69
  /**
46
70
  * Destination path the addon's pre-response cache file should be
47
71
  * renamed to (computed from `cacheMessages + responseText`). The
48
- * stale entry at the source path is dropped from `cachedMessageCounts`
72
+ * stale entry at the source path is dropped from `cachedPrefixes`
49
73
  * and the new count is recorded at this target path.
50
74
  */
51
75
  targetCachePath: string;
@@ -74,7 +98,7 @@ export interface KvCacheSession {
74
98
  /**
75
99
  * Roll back an in-flight turn — atomically deletes the on-disk cache
76
100
  * file, clears the in-memory `initializedCaches` entry, forgets the
77
- * `cachedMessageCounts` entry, releases the active-path ref, and
101
+ * `cachedPrefixes` entry, releases the active-path ref, and
78
102
  * removes orphaned marker metadata. Idempotent: a turn that has
79
103
  * already been committed or rolled back is a no-op on subsequent
80
104
  * calls. Handlers register this via `ctx.scope.defer(...)` so it
@@ -116,19 +140,21 @@ export declare function createKvCacheSession(modelId: string, options?: {
116
140
  * Layers cleared, in order:
117
141
  * 1. On-disk: `deleteCache(...)` removes the matching directory
118
142
  * tree (or wipes and recreates the root for `all: true`).
119
- * 2. `cachedMessageCounts`: prefix-cleanup by the removed directory
143
+ * 2. `cachedPrefixes`: prefix-cleanup by the removed directory
120
144
  * so any per-cache count under the deleted tree is forgotten.
121
- * 3. `initializedCaches`: scope clear by `(kvCacheKey[, modelId])`,
122
- * matching the on-disk scope.
145
+ * 3. `initializedCaches`: prefix-cleanup by the same removed
146
+ * directory, since it is keyed by the resolved cache path.
123
147
  *
124
- * Concurrency with in-flight turns: this delete is wire-async with
125
- * respect to any turn currently holding a `TurnHandle` for the same
126
- * cache key. Worst case the on-disk `.bin` is removed while a turn is
127
- * mid-write; the turn's eventual `commitTurn(...)` then fails the
128
- * `verifySaveAndRecord` probe (file gone) and rolls back idempotently.
129
- * No coordination primitive is needed because every layer's mutation
130
- * is idempotent (`unlink` no-ops if missing, `Map.delete` / `Set.delete`
131
- * no-op on absent keys).
148
+ * Concurrency with in-flight turns: this delete does not take the
149
+ * per-cache-path write locks, so it races any turn holding a `TurnHandle`
150
+ * for the same key. Deleting a key that is in active use is unsupported.
151
+ * Each individual mutation is idempotent (`unlink` no-ops if missing,
152
+ * `Map.delete` / `Set.delete` no-op on absent keys), but the layers are
153
+ * cleared without a lock, so an interleaving that lands the delete's
154
+ * in-memory cleanup after a concurrent turn has already renamed/committed
155
+ * its file splits state: the file stays on disk while its saved-count and
156
+ * init flag are cleared, so the next turn sees the file, skips priming, and
157
+ * reports `savedCount=0`. Callers must not delete a key that is in active use.
132
158
  */
133
159
  export declare function deleteKvCacheState(target: {
134
160
  kvCacheKey: string;
@@ -148,8 +174,9 @@ export declare function deleteKvCacheState(target: {
148
174
  export declare const __kvCacheSessionTestHooks: {
149
175
  getSavedCount(cachePath: string): number | undefined;
150
176
  setSavedCountForTest(cachePath: string, count: number): void;
151
- hasInitializedKey(modelId: string, configHash: string, cacheKey: string): boolean;
152
- markInitializedForTest(modelId: string, configHash: string, cacheKey: string): void;
177
+ getToolBlockCachedForTest(cachePath: string): boolean;
178
+ hasInitializedPath(cachePath: string): boolean;
179
+ getActivePathCountForTest(cachePath: string): number;
153
180
  getLastAutoCacheSweepMsForTest(): number;
154
181
  setLastAutoCacheSweepMsForTest(value: number): void;
155
182
  resetForTest(): void;