@qvac/inference 0.17.1 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/index.d.ts +1 -1
- package/dist/api/index.js +1 -1
- package/dist/api/text-to-speech.js +22 -2
- package/dist/api/transcribe.d.ts +19 -2
- package/dist/api/transcribe.js +66 -27
- package/dist/api/translate.js +6 -17
- package/dist/api/vla.d.ts +42 -1
- package/dist/api/vla.js +50 -1
- package/dist/constants/audio.d.ts +2 -1
- package/dist/constants/audio.js +17 -1
- package/dist/errors/index.d.ts +3 -0
- package/dist/errors/index.js +5 -0
- package/dist/handlers/load-model/handler.js +2 -2
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +3 -3
- package/dist/handlers/load-model/resolve.d.ts +12 -2
- package/dist/handlers/load-model/resolve.js +90 -12
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +782 -6
- package/dist/models/registry/models.js +2664 -1859
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +19 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +153 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +54 -4
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +28 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +3 -1
- package/dist/plugins/builtin/ggml-vla/ops/vla-set-embodiment.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-set-embodiment.js +9 -0
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +37 -0
- package/dist/plugins/builtin/ggml-vla/plugin.js +21 -4
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +16 -15
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +2 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +10 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +131 -123
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +110 -20
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +50 -23
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +308 -143
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -2
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +6 -6
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +56 -36
- package/dist/plugins/builtin/llamacpp-completion/transform.js +0 -7
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -3
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +6 -5
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +27 -16
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +23 -3
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +302 -4
- package/dist/plugins/builtin/tts-ggml/plugin.js +131 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +48 -39
- package/dist/plugins/ops/cancel.d.ts +1 -1
- package/dist/plugins/ops/cancel.js +1 -1
- package/dist/plugins/ops/kv-cache-retention.js +11 -0
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
- package/dist/plugins/ops/kv-cache-utils.js +40 -12
- package/dist/plugins/ops/transcribe.d.ts +2 -2
- package/dist/plugins/ops/transcribe.js +54 -48
- package/dist/plugins/ops/translate-stats.d.ts +4 -0
- package/dist/plugins/ops/translate-stats.js +11 -0
- package/dist/plugins/ops/translate.js +66 -38
- package/dist/plugins/ops/unload-model.js +30 -24
- package/dist/runtime/model-registry.d.ts +1 -0
- package/dist/runtime/request-context.js +29 -14
- package/dist/runtime/request-registry.d.ts +27 -0
- package/dist/runtime/request-registry.js +246 -76
- package/dist/schemas/audio-gen.d.ts +103 -0
- package/dist/schemas/audio-gen.js +135 -4
- package/dist/schemas/common.d.ts +550 -11
- package/dist/schemas/config.d.ts +60 -24
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -1
- package/dist/schemas/index.js +1 -1
- package/dist/schemas/llamacpp-config.d.ts +10 -10
- package/dist/schemas/llamacpp-config.js +12 -5
- package/dist/schemas/load-model.d.ts +3691 -336
- package/dist/schemas/load-model.js +37 -19
- package/dist/schemas/text-to-speech.d.ts +1086 -110
- package/dist/schemas/text-to-speech.js +197 -7
- package/dist/schemas/tools.d.ts +0 -16
- package/dist/schemas/tools.js +0 -15
- package/dist/schemas/transcription-config.d.ts +2 -0
- package/dist/schemas/transcription-config.js +6 -0
- package/dist/schemas/transcription.d.ts +3 -0
- package/dist/schemas/vla.d.ts +83 -0
- package/dist/schemas/vla.js +105 -2
- package/dist/surface.d.ts +1 -1
- package/dist/surface.js +1 -1
- package/dist/utils/addon-responses.d.ts +4 -12
- package/dist/utils/audio/decoder.d.ts +37 -1
- package/dist/utils/audio/decoder.js +82 -52
- package/dist/utils/config-transform.d.ts +3 -0
- package/dist/utils/config-transform.js +6 -0
- package/dist/utils/index.d.ts +0 -1
- package/dist/utils/index.js +0 -1
- package/dist/utils/shard-utils.d.ts +1 -0
- package/dist/utils/shard-utils.js +14 -0
- package/dist/utils/tool-integration.d.ts +3 -10
- package/dist/utils/tool-integration.js +3 -12
- package/package.json +26 -22
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +0 -22
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +0 -30
- package/dist/utils/expand-gguf-shards.d.ts +0 -2
- package/dist/utils/expand-gguf-shards.js +0 -22
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { models, getModelByPath } from '../../models/registry/models.js';
|
|
2
2
|
import { hyperdriveUrlSchema, registryUrlSchema, SUPPORTED_ARCHIVE_EXTENSIONS, modelInputToSrcSchema } from '../../schemas/index.js';
|
|
3
|
-
import { getModelsCacheDir, getShardedModelCacheDir, generateShortHash, extractAndValidateShardedArchive } from '../../utils/index.js';
|
|
3
|
+
import { getModelsCacheDir, getShardedModelCacheDir, generateShortHash, extractAndValidateShardedArchive, measureChecksum } from '../../utils/index.js';
|
|
4
4
|
import { promises as fsPromises } from 'bare-fs';
|
|
5
5
|
import path from 'bare-path';
|
|
6
6
|
import { downloadModelFromHttp } from '../../handlers/load-model/http.js';
|
|
@@ -8,7 +8,7 @@ import { downloadModelFromHyperdrive } from '../../handlers/load-model/hyperdriv
|
|
|
8
8
|
import { downloadModelFromRegistry } from '../../handlers/load-model/registry.js';
|
|
9
9
|
import { getExplicitRegistryMetadata } from '../../handlers/load-model/registry-metadata.js';
|
|
10
10
|
import { downloadModelFromHttpWithStats, downloadModelFromHyperdriveWithStats, downloadModelFromRegistryWithStats } from '../../handlers/load-model/download-stats.js';
|
|
11
|
-
import { InferenceCancelledError, ModelLoadFailedError, ModelNotFoundError, SeedingNotSupportedError } from '../../errors/index.js';
|
|
11
|
+
import { ChecksumValidationFailedError, DownloadCancelledError, InferenceCancelledError, ModelFileNotFoundError, ModelLoadFailedError, ModelNotFoundError, SeedingNotSupportedError } from '../../errors/index.js';
|
|
12
12
|
import { validateAndJoinPath } from '../../utils/path-security.js';
|
|
13
13
|
import { getEngineLogger } from '../../logging/index.js';
|
|
14
14
|
const logger = getEngineLogger();
|
|
@@ -74,11 +74,66 @@ function buildResult(pathOrResult, sourceType) {
|
|
|
74
74
|
}
|
|
75
75
|
return { path: pathOrResult, sourceType };
|
|
76
76
|
}
|
|
77
|
-
|
|
77
|
+
// A fallback is a non-P2P source: only HTTP URLs and file paths are valid.
|
|
78
|
+
// registry:// and pear:// are rejected.
|
|
79
|
+
export async function resolveFallbackModel(fallbackSrc, expectedChecksum, progressCallback, mode, signal, hooks) {
|
|
80
|
+
if (fallbackSrc.startsWith('registry://') || fallbackSrc.startsWith('pear://')) {
|
|
81
|
+
throw new ModelLoadFailedError('fallbackSrc must be an HTTP URL or a local file path.');
|
|
82
|
+
}
|
|
83
|
+
// seed applies only to hyperdrive sources, which a fallback can never be.
|
|
84
|
+
const result = await resolveModelPathInner(fallbackSrc, progressCallback, false, mode, signal, hooks);
|
|
85
|
+
// Report a missing fallback file as a not-found error before checksumming.
|
|
86
|
+
try {
|
|
87
|
+
await fsPromises.access(result.path);
|
|
88
|
+
}
|
|
89
|
+
catch {
|
|
90
|
+
throw new ModelFileNotFoundError(result.path);
|
|
91
|
+
}
|
|
92
|
+
if (expectedChecksum && expectedChecksum.length === 64) {
|
|
93
|
+
const actualChecksum = await measureChecksum(result.path, hooks);
|
|
94
|
+
if (actualChecksum !== expectedChecksum) {
|
|
95
|
+
// http downloads are cached under the models dir; drop a corrupt one.
|
|
96
|
+
if (result.sourceType === 'http') {
|
|
97
|
+
await fsPromises.unlink(result.path).catch(() => { });
|
|
98
|
+
}
|
|
99
|
+
throw new ChecksumValidationFailedError(fallbackSrc);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
return result;
|
|
103
|
+
}
|
|
104
|
+
// Registry download with optional fallback. A user-initiated cancel always
|
|
105
|
+
// propagates. On any other primary failure: a fallback is taken when present;
|
|
106
|
+
// otherwise a known model surfaces a neutral error and an unknown one rethrows.
|
|
107
|
+
export async function resolveRegistryModel(params) {
|
|
108
|
+
try {
|
|
109
|
+
return await params.downloadPrimary();
|
|
110
|
+
}
|
|
111
|
+
catch (error) {
|
|
112
|
+
if (error instanceof InferenceCancelledError || error instanceof DownloadCancelledError) {
|
|
113
|
+
throw error;
|
|
114
|
+
}
|
|
115
|
+
if (params.hasFallback) {
|
|
116
|
+
return params.resolveFallback();
|
|
117
|
+
}
|
|
118
|
+
const isCorrectnessError = error instanceof ChecksumValidationFailedError ||
|
|
119
|
+
error instanceof ModelNotFoundError ||
|
|
120
|
+
error instanceof ModelFileNotFoundError;
|
|
121
|
+
if (params.isKnownModel && !isCorrectnessError) {
|
|
122
|
+
throw params.buildUnreachableError(error);
|
|
123
|
+
}
|
|
124
|
+
throw error;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
async function resolveModelPathInner(modelSrc, progressCallback, seed, mode, signal, hooks, fallbackSrc) {
|
|
78
128
|
if (signal?.aborted) {
|
|
79
129
|
throw new InferenceCancelledError(hooks?.requestBinding?.requestId ?? 'unknown');
|
|
80
130
|
}
|
|
81
131
|
const srcString = modelInputToSrcSchema.parse(modelSrc);
|
|
132
|
+
// A fallback is validated against the catalog checksum, which exists only for
|
|
133
|
+
// built-in registry models. Other sources have no checksum to validate against.
|
|
134
|
+
if (fallbackSrc !== undefined && !srcString.startsWith('registry://')) {
|
|
135
|
+
throw new ModelLoadFailedError('fallbackSrc is only supported when modelSrc is a built-in registry model.');
|
|
136
|
+
}
|
|
82
137
|
// Empty modelSrc is reserved for plugins that ship bundled weights
|
|
83
138
|
// (e.g. `@qvac/classification-ggml`). The handler skips this resolver
|
|
84
139
|
// for them when `skipPrimaryModelPathValidation` is set, but if it
|
|
@@ -104,11 +159,34 @@ async function resolveModelPathInner(modelSrc, progressCallback, seed, mode, sig
|
|
|
104
159
|
const modelMetadata = getModelByPath(registryPath);
|
|
105
160
|
const explicitMetadata = getExplicitRegistryMetadata(modelSrc);
|
|
106
161
|
const expectedChecksum = modelMetadata?.sha256Checksum ?? explicitMetadata?.sha256Checksum;
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
162
|
+
// An uncatalogued registry path carries no checksum to validate a fallback against.
|
|
163
|
+
if (fallbackSrc !== undefined && !modelMetadata) {
|
|
164
|
+
throw new ModelLoadFailedError('fallbackSrc is only supported when modelSrc is a built-in registry model.');
|
|
165
|
+
}
|
|
166
|
+
// A single-file fallback cannot satisfy a sharded or companion-set model.
|
|
167
|
+
if (fallbackSrc !== undefined &&
|
|
168
|
+
(modelMetadata?.shardMetadata || modelMetadata?.companionSet)) {
|
|
169
|
+
throw new ModelLoadFailedError('fallbackSrc is not supported for sharded or multi-file models.');
|
|
170
|
+
}
|
|
171
|
+
return resolveRegistryModel({
|
|
172
|
+
hasFallback: fallbackSrc !== undefined,
|
|
173
|
+
isKnownModel: modelMetadata !== undefined,
|
|
174
|
+
downloadPrimary: async () => {
|
|
175
|
+
const result = mode === 'stats'
|
|
176
|
+
? await downloadModelFromRegistryWithStats(registryPath, registrySource, progressCallback, expectedChecksum, hooks, explicitMetadata)
|
|
177
|
+
: await downloadModelFromRegistry(registryPath, registrySource, progressCallback, expectedChecksum, hooks, explicitMetadata);
|
|
178
|
+
logger.info(`Loaded Model to ${isDownloadResult(result) ? result.path : result}`);
|
|
179
|
+
return buildResult(result, 'registry');
|
|
180
|
+
},
|
|
181
|
+
resolveFallback: () => {
|
|
182
|
+
logger.info(`Registry download failed for ${registryPath}; trying fallbackSrc: ${fallbackSrc}`);
|
|
183
|
+
return resolveFallbackModel(fallbackSrc, expectedChecksum, progressCallback, mode, signal, hooks);
|
|
184
|
+
},
|
|
185
|
+
buildUnreachableError: (cause) => new ModelLoadFailedError(`Could not download "${modelMetadata?.modelId}" from the registry on this network ` +
|
|
186
|
+
`(${cause instanceof Error ? cause.message : String(cause)}). ` +
|
|
187
|
+
'If the registry is not reliably reachable here, pass a fallbackSrc ' +
|
|
188
|
+
'(an HTTP URL or local file path) to load this model from an alternate source.', cause)
|
|
189
|
+
});
|
|
112
190
|
}
|
|
113
191
|
// Validate seeding is only used with hyperdrive models
|
|
114
192
|
if (seed && !hyperdriveKey) {
|
|
@@ -149,10 +227,10 @@ async function resolveModelPathInner(modelSrc, progressCallback, seed, mode, sig
|
|
|
149
227
|
}
|
|
150
228
|
return { path: resolvedPath, sourceType: 'filesystem' };
|
|
151
229
|
}
|
|
152
|
-
export async function resolveModelPath(modelSrc, progressCallback, seed, signal, hooks) {
|
|
153
|
-
const result = await resolveModelPathInner(modelSrc, progressCallback, seed, 'base', signal, hooks);
|
|
230
|
+
export async function resolveModelPath(modelSrc, progressCallback, seed, signal, hooks, fallbackSrc) {
|
|
231
|
+
const result = await resolveModelPathInner(modelSrc, progressCallback, seed, 'base', signal, hooks, fallbackSrc);
|
|
154
232
|
return result.path;
|
|
155
233
|
}
|
|
156
|
-
export async function resolveModelPathWithStats(modelSrc, progressCallback, seed, signal, hooks) {
|
|
157
|
-
return resolveModelPathInner(modelSrc, progressCallback, seed, 'stats', signal, hooks);
|
|
234
|
+
export async function resolveModelPathWithStats(modelSrc, progressCallback, seed, signal, hooks, fallbackSrc) {
|
|
235
|
+
return resolveModelPathInner(modelSrc, progressCallback, seed, 'stats', signal, hooks, fallbackSrc);
|
|
158
236
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { batchCompletion, completion, deleteCache, loadModel, downloadAsset, heartbeat, startQVACProvider, stopQVACProvider, unloadModel, transcribe, transcribeStream, bciTranscribe, bciTranscribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, getSystemResources, loggingStream, subscribeServerLogs, type ServerLogHandler, ocr, invokePlugin, invokePluginStream, diffusion, type DiffusionProgressTick, audioGen, classify, video, type VideoProgressTick, upscale, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, type ModelRegistrySearchParams, suspend, resume, state, vla, vlaHparams, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE, type FinetuneHandle } from './api/index.js';
|
|
1
|
+
export { batchCompletion, completion, deleteCache, loadModel, downloadAsset, heartbeat, startQVACProvider, stopQVACProvider, unloadModel, transcribe, transcribeStream, bciTranscribe, bciTranscribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, getSystemResources, loggingStream, subscribeServerLogs, type ServerLogHandler, ocr, invokePlugin, invokePluginStream, diffusion, type DiffusionProgressTick, audioGen, classify, video, type VideoProgressTick, upscale, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, type ModelRegistrySearchParams, suspend, resume, state, vla, vlaHparams, vlaSetEmbodiment, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE, type FinetuneHandle } from './api/index.js';
|
|
2
2
|
export { close } from './dispatch.js';
|
|
3
3
|
export { plugins, registerPlugin, registerPlugins, getPlugin, hasPlugin, getAllPlugins, unregisterPlugin } from './plugins/index.js';
|
|
4
4
|
export * from './surface.js';
|
package/dist/index.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Public API: the operations, engine lifecycle and explicit plugin assembly, and
|
|
3
3
|
// the value-clean surface (schemas / types / consts / errors / model registry)
|
|
4
4
|
// re-exported from `./surface`.
|
|
5
|
-
export { batchCompletion, completion, deleteCache, loadModel, downloadAsset, heartbeat, startQVACProvider, stopQVACProvider, unloadModel, transcribe, transcribeStream, bciTranscribe, bciTranscribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, getSystemResources, loggingStream, subscribeServerLogs, ocr, invokePlugin, invokePluginStream, diffusion, audioGen, classify, video, upscale, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, suspend, resume, state, vla, vlaHparams, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE } from './api/index.js';
|
|
5
|
+
export { batchCompletion, completion, deleteCache, loadModel, downloadAsset, heartbeat, startQVACProvider, stopQVACProvider, unloadModel, transcribe, transcribeStream, bciTranscribe, bciTranscribeStream, embed, finetune, translate, cancel, ragChunk, ragIngest, ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings, ragReindex, ragListWorkspaces, ragCloseWorkspace, ragDeleteWorkspace, textToSpeech, textToSpeechStream, getModelInfo, getLoadedModelInfo, getSystemResources, loggingStream, subscribeServerLogs, ocr, invokePlugin, invokePluginStream, diffusion, audioGen, classify, video, upscale, modelRegistryList, modelRegistrySearch, modelRegistryGetModel, suspend, resume, state, vla, vlaHparams, vlaSetEmbodiment, vlaPreprocessImage, vlaPadState, VLA_DEFAULT_IMAGE_SIZE } from './api/index.js';
|
|
6
6
|
// Engine lifecycle and explicit plugin assembly. No plugins are registered by
|
|
7
7
|
// default: an app assembles the engines it needs via `plugins([...])` or the
|
|
8
8
|
// `registerPlugin`/`registerPlugins` primitives, then calls the operations.
|