@qvac/inference 0.17.1 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/index.d.ts +1 -1
- package/dist/api/index.js +1 -1
- package/dist/api/text-to-speech.js +22 -2
- package/dist/api/transcribe.d.ts +19 -2
- package/dist/api/transcribe.js +66 -27
- package/dist/api/translate.js +6 -17
- package/dist/api/vla.d.ts +42 -1
- package/dist/api/vla.js +50 -1
- package/dist/constants/audio.d.ts +2 -1
- package/dist/constants/audio.js +17 -1
- package/dist/errors/index.d.ts +3 -0
- package/dist/errors/index.js +5 -0
- package/dist/handlers/load-model/handler.js +2 -2
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +3 -3
- package/dist/handlers/load-model/resolve.d.ts +12 -2
- package/dist/handlers/load-model/resolve.js +90 -12
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/models/registry/models.d.ts +782 -6
- package/dist/models/registry/models.js +2664 -1859
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +19 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +153 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +54 -4
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +28 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +3 -1
- package/dist/plugins/builtin/ggml-vla/ops/vla-set-embodiment.d.ts +2 -0
- package/dist/plugins/builtin/ggml-vla/ops/vla-set-embodiment.js +9 -0
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +37 -0
- package/dist/plugins/builtin/ggml-vla/plugin.js +21 -4
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +16 -15
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +2 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +10 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +131 -123
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +110 -20
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +50 -23
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +308 -143
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +1 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -2
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +6 -6
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +56 -36
- package/dist/plugins/builtin/llamacpp-completion/transform.js +0 -7
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -3
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +6 -5
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +27 -16
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +23 -3
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +302 -4
- package/dist/plugins/builtin/tts-ggml/plugin.js +131 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +48 -39
- package/dist/plugins/ops/cancel.d.ts +1 -1
- package/dist/plugins/ops/cancel.js +1 -1
- package/dist/plugins/ops/kv-cache-retention.js +11 -0
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
- package/dist/plugins/ops/kv-cache-utils.js +40 -12
- package/dist/plugins/ops/transcribe.d.ts +2 -2
- package/dist/plugins/ops/transcribe.js +54 -48
- package/dist/plugins/ops/translate-stats.d.ts +4 -0
- package/dist/plugins/ops/translate-stats.js +11 -0
- package/dist/plugins/ops/translate.js +66 -38
- package/dist/plugins/ops/unload-model.js +30 -24
- package/dist/runtime/model-registry.d.ts +1 -0
- package/dist/runtime/request-context.js +29 -14
- package/dist/runtime/request-registry.d.ts +27 -0
- package/dist/runtime/request-registry.js +246 -76
- package/dist/schemas/audio-gen.d.ts +103 -0
- package/dist/schemas/audio-gen.js +135 -4
- package/dist/schemas/common.d.ts +550 -11
- package/dist/schemas/config.d.ts +60 -24
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -1
- package/dist/schemas/index.js +1 -1
- package/dist/schemas/llamacpp-config.d.ts +10 -10
- package/dist/schemas/llamacpp-config.js +12 -5
- package/dist/schemas/load-model.d.ts +3691 -336
- package/dist/schemas/load-model.js +37 -19
- package/dist/schemas/text-to-speech.d.ts +1086 -110
- package/dist/schemas/text-to-speech.js +197 -7
- package/dist/schemas/tools.d.ts +0 -16
- package/dist/schemas/tools.js +0 -15
- package/dist/schemas/transcription-config.d.ts +2 -0
- package/dist/schemas/transcription-config.js +6 -0
- package/dist/schemas/transcription.d.ts +3 -0
- package/dist/schemas/vla.d.ts +83 -0
- package/dist/schemas/vla.js +105 -2
- package/dist/surface.d.ts +1 -1
- package/dist/surface.js +1 -1
- package/dist/utils/addon-responses.d.ts +4 -12
- package/dist/utils/audio/decoder.d.ts +37 -1
- package/dist/utils/audio/decoder.js +82 -52
- package/dist/utils/config-transform.d.ts +3 -0
- package/dist/utils/config-transform.js +6 -0
- package/dist/utils/index.d.ts +0 -1
- package/dist/utils/index.js +0 -1
- package/dist/utils/shard-utils.d.ts +1 -0
- package/dist/utils/shard-utils.js +14 -0
- package/dist/utils/tool-integration.d.ts +3 -10
- package/dist/utils/tool-integration.js +3 -12
- package/package.json +26 -22
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.d.ts +0 -22
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-cancelled.js +0 -30
- package/dist/utils/expand-gguf-shards.d.ts +0 -2
- package/dist/utils/expand-gguf-shards.js +0 -22
|
@@ -1,4 +1,40 @@
|
|
|
1
|
+
import { type DecoderOutput } from '@qvac/decoder-audio';
|
|
1
2
|
import { Readable } from 'bare-stream';
|
|
2
3
|
import { type AudioFormat } from '../../schemas/index.js';
|
|
3
4
|
export declare function needsDecoding(filePath: string): boolean;
|
|
4
|
-
|
|
5
|
+
/**
|
|
6
|
+
* Longest silence the decoder may go without emitting a PCM chunk before the
|
|
7
|
+
* decode is abandoned. Measured between outputs, not against the whole file, so
|
|
8
|
+
* long inputs decode fine as long as FFmpeg keeps producing data.
|
|
9
|
+
*/
|
|
10
|
+
export declare const DECODER_INACTIVITY_TIMEOUT_MS = 10000;
|
|
11
|
+
export interface DecodeAudioOptions {
|
|
12
|
+
/** Output sample rate; the decoder's default is 16 kHz (what every transcription engine expects). */
|
|
13
|
+
sampleRate?: number;
|
|
14
|
+
/** Inactivity timeout between decoded chunks (default `DECODER_INACTIVITY_TIMEOUT_MS`). */
|
|
15
|
+
inactivityTimeoutMs?: number;
|
|
16
|
+
}
|
|
17
|
+
/** The subset of the decoder's `QvacResponse` that the stream bridge consumes. */
|
|
18
|
+
export interface DecoderResponseLike {
|
|
19
|
+
onUpdate(callback: (output: DecoderOutput) => void): DecoderResponseLike;
|
|
20
|
+
onFinish(callback: () => void): DecoderResponseLike;
|
|
21
|
+
onError(callback: (error: Error) => void): DecoderResponseLike;
|
|
22
|
+
await(): Promise<unknown>;
|
|
23
|
+
}
|
|
24
|
+
export interface DecoderStreamOptions {
|
|
25
|
+
/** Used in error messages only. */
|
|
26
|
+
inputPath: string;
|
|
27
|
+
inactivityTimeoutMs?: number;
|
|
28
|
+
/** Runs once, on whichever terminal path (finish, error, timeout) settles the stream. */
|
|
29
|
+
onSettled?: () => void;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Decode an audio file into a raw mono PCM stream.
|
|
33
|
+
*/
|
|
34
|
+
export declare function decodeAudioToStream(inputPath: string, audioFormat?: AudioFormat, options?: DecodeAudioOptions): Promise<Readable>;
|
|
35
|
+
/**
|
|
36
|
+
* Bridge a decoder response into a `Readable` of PCM chunks. The stream fails
|
|
37
|
+
* if the decoder emits nothing for `inactivityTimeoutMs`; the timer is re-armed
|
|
38
|
+
* on every chunk, so it bounds inactivity, not the total decode time.
|
|
39
|
+
*/
|
|
40
|
+
export declare function decoderResponseToStream(response: DecoderResponseLike, options: DecoderStreamOptions): Readable;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import fs from 'bare-fs';
|
|
2
2
|
import path from 'bare-path';
|
|
3
3
|
import { FFmpegDecoder } from '@qvac/decoder-audio';
|
|
4
|
-
import { FORMATS_NEEDING_DECODE } from '
|
|
4
|
+
import { FORMATS_NEEDING_DECODE } from '../../constants/audio.js';
|
|
5
5
|
import { Readable } from 'bare-stream';
|
|
6
6
|
import Buffer from 'bare-buffer';
|
|
7
7
|
import { getEngineLogger } from '../../logging/index.js';
|
|
@@ -10,10 +10,19 @@ export function needsDecoding(filePath) {
|
|
|
10
10
|
const ext = path.extname(filePath).toLowerCase();
|
|
11
11
|
return FORMATS_NEEDING_DECODE.includes(ext);
|
|
12
12
|
}
|
|
13
|
-
|
|
14
|
-
|
|
13
|
+
/**
|
|
14
|
+
* Longest silence the decoder may go without emitting a PCM chunk before the
|
|
15
|
+
* decode is abandoned. Measured between outputs, not against the whole file, so
|
|
16
|
+
* long inputs decode fine as long as FFmpeg keeps producing data.
|
|
17
|
+
*/
|
|
18
|
+
export const DECODER_INACTIVITY_TIMEOUT_MS = 10000;
|
|
19
|
+
/**
|
|
20
|
+
* Decode an audio file into a raw mono PCM stream.
|
|
21
|
+
*/
|
|
22
|
+
export async function decodeAudioToStream(inputPath, audioFormat = 's16le', options = {}) {
|
|
23
|
+
const { sampleRate, inactivityTimeoutMs = DECODER_INACTIVITY_TIMEOUT_MS } = options;
|
|
15
24
|
const decoder = new FFmpegDecoder({
|
|
16
|
-
config: { audioFormat },
|
|
25
|
+
config: { audioFormat, ...(sampleRate !== undefined && { sampleRate }) },
|
|
17
26
|
logger
|
|
18
27
|
});
|
|
19
28
|
try {
|
|
@@ -22,55 +31,11 @@ export async function decodeAudioToStream(inputPath, audioFormat = 's16le') {
|
|
|
22
31
|
// bare-fs read streams are async-iterable over Buffer at runtime; the
|
|
23
32
|
// decoder types its input as AsyncIterable<Buffer>.
|
|
24
33
|
const response = decoder.run(audioStream);
|
|
25
|
-
|
|
26
|
-
|
|
34
|
+
return decoderResponseToStream(response, {
|
|
35
|
+
inputPath,
|
|
36
|
+
inactivityTimeoutMs,
|
|
37
|
+
onSettled: () => setImmediate(() => void decoder.unload())
|
|
27
38
|
});
|
|
28
|
-
let hasReceivedData = false;
|
|
29
|
-
let hasEnded = false;
|
|
30
|
-
// Fallback timeout in case the decoder hangs without emitting any events
|
|
31
|
-
const timeoutId = setTimeout(() => {
|
|
32
|
-
if (!hasEnded) {
|
|
33
|
-
hasEnded = true;
|
|
34
|
-
const timeoutError = new Error(`Audio decoding timed out after ${DECODER_TIMEOUT_MS}ms for file: ${inputPath}`);
|
|
35
|
-
outputStream.destroy(timeoutError);
|
|
36
|
-
}
|
|
37
|
-
}, DECODER_TIMEOUT_MS);
|
|
38
|
-
response
|
|
39
|
-
.onUpdate((output) => {
|
|
40
|
-
hasReceivedData = true;
|
|
41
|
-
const bytes = new Uint8Array(output.outputArray);
|
|
42
|
-
outputStream.push(Buffer.from(bytes));
|
|
43
|
-
})
|
|
44
|
-
.onFinish(() => {
|
|
45
|
-
if (!hasEnded) {
|
|
46
|
-
hasEnded = true;
|
|
47
|
-
clearTimeout(timeoutId);
|
|
48
|
-
setImmediate(() => void decoder.unload());
|
|
49
|
-
if (!hasReceivedData) {
|
|
50
|
-
outputStream.destroy(new Error(`No audio data decoded from file: ${inputPath}`));
|
|
51
|
-
}
|
|
52
|
-
else {
|
|
53
|
-
outputStream.push(null);
|
|
54
|
-
}
|
|
55
|
-
}
|
|
56
|
-
})
|
|
57
|
-
.onError((error) => {
|
|
58
|
-
if (!hasEnded) {
|
|
59
|
-
hasEnded = true;
|
|
60
|
-
clearTimeout(timeoutId);
|
|
61
|
-
setImmediate(() => void decoder.unload());
|
|
62
|
-
outputStream.destroy(error);
|
|
63
|
-
}
|
|
64
|
-
});
|
|
65
|
-
response.await().catch((error) => {
|
|
66
|
-
if (!hasEnded) {
|
|
67
|
-
hasEnded = true;
|
|
68
|
-
clearTimeout(timeoutId);
|
|
69
|
-
setImmediate(() => void decoder.unload());
|
|
70
|
-
outputStream.destroy(error instanceof Error ? error : new Error(String(error)));
|
|
71
|
-
}
|
|
72
|
-
});
|
|
73
|
-
return outputStream;
|
|
74
39
|
}
|
|
75
40
|
catch (error) {
|
|
76
41
|
await decoder.unload();
|
|
@@ -78,3 +43,68 @@ export async function decodeAudioToStream(inputPath, audioFormat = 's16le') {
|
|
|
78
43
|
throw error;
|
|
79
44
|
}
|
|
80
45
|
}
|
|
46
|
+
/**
|
|
47
|
+
* Bridge a decoder response into a `Readable` of PCM chunks. The stream fails
|
|
48
|
+
* if the decoder emits nothing for `inactivityTimeoutMs`; the timer is re-armed
|
|
49
|
+
* on every chunk, so it bounds inactivity, not the total decode time.
|
|
50
|
+
*/
|
|
51
|
+
export function decoderResponseToStream(response, options) {
|
|
52
|
+
const { inputPath, inactivityTimeoutMs = DECODER_INACTIVITY_TIMEOUT_MS, onSettled } = options;
|
|
53
|
+
const outputStream = new Readable({
|
|
54
|
+
read() { }
|
|
55
|
+
});
|
|
56
|
+
let hasReceivedData = false;
|
|
57
|
+
let hasEnded = false;
|
|
58
|
+
let timeoutId;
|
|
59
|
+
const settle = () => {
|
|
60
|
+
if (hasEnded)
|
|
61
|
+
return false;
|
|
62
|
+
hasEnded = true;
|
|
63
|
+
if (timeoutId !== undefined)
|
|
64
|
+
clearTimeout(timeoutId);
|
|
65
|
+
onSettled?.();
|
|
66
|
+
return true;
|
|
67
|
+
};
|
|
68
|
+
const armTimeout = () => {
|
|
69
|
+
if (timeoutId !== undefined)
|
|
70
|
+
clearTimeout(timeoutId);
|
|
71
|
+
timeoutId = setTimeout(() => {
|
|
72
|
+
if (!settle())
|
|
73
|
+
return;
|
|
74
|
+
outputStream.destroy(new Error(`Audio decoding produced no output for ${inactivityTimeoutMs}ms for file: ${inputPath}`));
|
|
75
|
+
}, inactivityTimeoutMs);
|
|
76
|
+
};
|
|
77
|
+
armTimeout();
|
|
78
|
+
outputStream.on('close', () => {
|
|
79
|
+
settle();
|
|
80
|
+
});
|
|
81
|
+
response
|
|
82
|
+
.onUpdate((output) => {
|
|
83
|
+
hasReceivedData = true;
|
|
84
|
+
if (!hasEnded)
|
|
85
|
+
armTimeout();
|
|
86
|
+
const bytes = new Uint8Array(output.outputArray);
|
|
87
|
+
outputStream.push(Buffer.from(bytes));
|
|
88
|
+
})
|
|
89
|
+
.onFinish(() => {
|
|
90
|
+
if (!settle())
|
|
91
|
+
return;
|
|
92
|
+
if (!hasReceivedData) {
|
|
93
|
+
outputStream.destroy(new Error(`No audio data decoded from file: ${inputPath}`));
|
|
94
|
+
}
|
|
95
|
+
else {
|
|
96
|
+
outputStream.push(null);
|
|
97
|
+
}
|
|
98
|
+
})
|
|
99
|
+
.onError((error) => {
|
|
100
|
+
if (!settle())
|
|
101
|
+
return;
|
|
102
|
+
outputStream.destroy(error);
|
|
103
|
+
});
|
|
104
|
+
response.await().catch((error) => {
|
|
105
|
+
if (!settle())
|
|
106
|
+
return;
|
|
107
|
+
outputStream.destroy(error instanceof Error ? error : new Error(String(error)));
|
|
108
|
+
});
|
|
109
|
+
return outputStream;
|
|
110
|
+
}
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { normalizeModelType, ModelType } from '../schemas/index.js';
|
|
2
2
|
import { buildParakeetReloadConfig, buildWhisperReloadConfig } from '../plugins/builtin/asr-ggml/config.js';
|
|
3
|
+
// The model's concurrent sequence slots. Missing / 0 / NaN all mean single-slot;
|
|
4
|
+
// floor a fractional value so it can't over-admit.
|
|
5
|
+
export function getModelParallel(config) {
|
|
6
|
+
const n = Math.floor(Number(config.parallel));
|
|
7
|
+
return Number.isFinite(n) && n >= 1 ? n : 1;
|
|
8
|
+
}
|
|
3
9
|
export function transformConfigForReload(modelType, config) {
|
|
4
10
|
const canonicalType = normalizeModelType(modelType);
|
|
5
11
|
switch (canonicalType) {
|
package/dist/utils/index.d.ts
CHANGED
|
@@ -4,7 +4,6 @@ export * from '../utils/cache/auto-history.js';
|
|
|
4
4
|
export * from '../utils/cache/paths.js';
|
|
5
5
|
export * from '../utils/checksum.js';
|
|
6
6
|
export * from '../utils/config-transform.js';
|
|
7
|
-
export * from '../utils/expand-gguf-shards.js';
|
|
8
7
|
export * from '../utils/formatting.js';
|
|
9
8
|
export * from '../utils/gguf-tensor-extractor.js';
|
|
10
9
|
export * from '../utils/path-security.js';
|
package/dist/utils/index.js
CHANGED
|
@@ -4,7 +4,6 @@ export * from '../utils/cache/auto-history.js';
|
|
|
4
4
|
export * from '../utils/cache/paths.js';
|
|
5
5
|
export * from '../utils/checksum.js';
|
|
6
6
|
export * from '../utils/config-transform.js';
|
|
7
|
-
export * from '../utils/expand-gguf-shards.js';
|
|
8
7
|
export * from '../utils/formatting.js';
|
|
9
8
|
export * from '../utils/gguf-tensor-extractor.js';
|
|
10
9
|
export * from '../utils/path-security.js';
|
|
@@ -3,6 +3,7 @@ import type { ShardPatternInfo, ShardUrl } from '../schemas/index.js';
|
|
|
3
3
|
* Detect if model filename follows shard pattern (00001-of-0000x)
|
|
4
4
|
*/
|
|
5
5
|
export declare function detectShardedModel(filename: string): ShardPatternInfo;
|
|
6
|
+
export declare function getFirstShardPath(modelPath: string): string;
|
|
6
7
|
/**
|
|
7
8
|
* Generate list of shard filenames for a sharded model
|
|
8
9
|
* Accepts any shard in the group
|
|
@@ -20,6 +20,20 @@ export function detectShardedModel(filename) {
|
|
|
20
20
|
}
|
|
21
21
|
return { isSharded: false };
|
|
22
22
|
}
|
|
23
|
+
export function getFirstShardPath(modelPath) {
|
|
24
|
+
const lastSeparator = Math.max(modelPath.lastIndexOf('/'), modelPath.lastIndexOf('\\'));
|
|
25
|
+
const directory = lastSeparator >= 0 ? modelPath.slice(0, lastSeparator + 1) : '';
|
|
26
|
+
const filename = lastSeparator >= 0 ? modelPath.slice(lastSeparator + 1) : modelPath;
|
|
27
|
+
const shardInfo = detectShardedModel(filename);
|
|
28
|
+
if (!shardInfo.isSharded ||
|
|
29
|
+
!shardInfo.baseFilename ||
|
|
30
|
+
!shardInfo.totalShards ||
|
|
31
|
+
!shardInfo.extension) {
|
|
32
|
+
return modelPath;
|
|
33
|
+
}
|
|
34
|
+
const totalShards = shardInfo.totalShards.toString().padStart(5, '0');
|
|
35
|
+
return `${directory}${shardInfo.baseFilename}-00001-of-${totalShards}${shardInfo.extension}`;
|
|
36
|
+
}
|
|
23
37
|
/**
|
|
24
38
|
* Generate list of shard filenames for a sharded model
|
|
25
39
|
* Accepts any shard in the group
|
|
@@ -7,17 +7,10 @@ interface HistoryMessage {
|
|
|
7
7
|
}[] | undefined;
|
|
8
8
|
}
|
|
9
9
|
/**
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
10
|
+
* Prepend tools right after the system message (or at the very start when no
|
|
11
|
+
* system message is present). The tool block stays in the kv-cache for the
|
|
12
|
+
* whole chat session.
|
|
13
13
|
*/
|
|
14
14
|
export declare function prependToolsToHistory(history: HistoryMessage[], tools: Tool[]): Array<HistoryMessage | Tool>;
|
|
15
|
-
/**
|
|
16
|
-
* Dynamic tools mode: append tools after the last history message. The
|
|
17
|
-
* addon's compact-tools mode anchors the block after the last user message
|
|
18
|
-
* and trims it from the kv-cache once the tool-call chain resolves, so a
|
|
19
|
-
* subsequent turn can ship a different tool set without poisoning the cache.
|
|
20
|
-
*/
|
|
21
|
-
export declare function appendToolsToHistory(history: HistoryMessage[], tools: Tool[]): Array<HistoryMessage | Tool>;
|
|
22
15
|
export declare function detectToolDialect(modelId: string): ToolDialect;
|
|
23
16
|
export {};
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { detectToolDialectFromName } from '../utils/tools/index.js';
|
|
2
2
|
import { getModelInfo } from '../runtime/model-registry.js';
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
4
|
+
* Prepend tools right after the system message (or at the very start when no
|
|
5
|
+
* system message is present). The tool block stays in the kv-cache for the
|
|
6
|
+
* whole chat session.
|
|
7
7
|
*/
|
|
8
8
|
export function prependToolsToHistory(history, tools) {
|
|
9
9
|
const systemMsgIndex = history.findIndex((msg) => msg.role === 'system');
|
|
@@ -12,15 +12,6 @@ export function prependToolsToHistory(history, tools) {
|
|
|
12
12
|
}
|
|
13
13
|
return [...tools, ...history];
|
|
14
14
|
}
|
|
15
|
-
/**
|
|
16
|
-
* Dynamic tools mode: append tools after the last history message. The
|
|
17
|
-
* addon's compact-tools mode anchors the block after the last user message
|
|
18
|
-
* and trims it from the kv-cache once the tool-call chain resolves, so a
|
|
19
|
-
* subsequent turn can ship a different tool set without poisoning the cache.
|
|
20
|
-
*/
|
|
21
|
-
export function appendToolsToHistory(history, tools) {
|
|
22
|
-
return [...history, ...tools];
|
|
23
|
-
}
|
|
24
15
|
export function detectToolDialect(modelId) {
|
|
25
16
|
const info = getModelInfo(modelId);
|
|
26
17
|
if (!info)
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qvac/inference",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.18.0",
|
|
4
4
|
"description": "Bare-only in-process core of the QVAC SDK. Runs inference directly on the Bare runtime with explicit plugin assembly and no RPC, worker, or subprocess layer.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -133,6 +133,7 @@
|
|
|
133
133
|
},
|
|
134
134
|
"files": [
|
|
135
135
|
"dist/**/*",
|
|
136
|
+
"!dist/models/update-models/**",
|
|
136
137
|
"LICENSE",
|
|
137
138
|
"NOTICE"
|
|
138
139
|
],
|
|
@@ -148,7 +149,9 @@
|
|
|
148
149
|
"prepare": "npm run build",
|
|
149
150
|
"build:test": "tsc -p test/tsconfig.build.json && tsc-alias -p test/tsconfig.alias.json",
|
|
150
151
|
"test": "npm run build:test && brittle-bare 'test/dist/test/**/*.test.js'",
|
|
151
|
-
"test:bare": "npm run test"
|
|
152
|
+
"test:bare": "npm run test",
|
|
153
|
+
"check-models": "bare ./dist/models/update-models/index.js --check",
|
|
154
|
+
"update-models": "bare ./dist/models/update-models/index.js && prettier --write src/models/registry/models.ts"
|
|
152
155
|
},
|
|
153
156
|
"engines": {
|
|
154
157
|
"bare": "^1.30.3"
|
|
@@ -182,19 +185,19 @@
|
|
|
182
185
|
"zod": "^4.4.3"
|
|
183
186
|
},
|
|
184
187
|
"peerDependencies": {
|
|
185
|
-
"@qvac/asr-ggml": "^0.
|
|
186
|
-
"@qvac/audiogen-ggml": "^0.1
|
|
187
|
-
"@qvac/bci-whispercpp": "^0.
|
|
188
|
-
"@qvac/classification-ggml": "^0.
|
|
188
|
+
"@qvac/asr-ggml": "^0.3.0",
|
|
189
|
+
"@qvac/audiogen-ggml": "^0.2.1",
|
|
190
|
+
"@qvac/bci-whispercpp": "^0.7.1",
|
|
191
|
+
"@qvac/classification-ggml": "^0.20.0",
|
|
189
192
|
"@qvac/decoder-audio": "^0.5.0",
|
|
190
193
|
"@qvac/diffusion-cpp": "^0.17.0",
|
|
191
|
-
"@qvac/embed-llamacpp": "^0.
|
|
194
|
+
"@qvac/embed-llamacpp": "^0.34.0",
|
|
192
195
|
"@qvac/langdetect-text": "^0.1.2",
|
|
193
|
-
"@qvac/llm-llamacpp": "^0.
|
|
194
|
-
"@qvac/ocr-ggml": "^0.
|
|
195
|
-
"@qvac/translation-nmtcpp": "^
|
|
196
|
-
"@qvac/tts-ggml": "^0.
|
|
197
|
-
"@qvac/vla-ggml": "^0.
|
|
196
|
+
"@qvac/llm-llamacpp": "^0.45.0",
|
|
197
|
+
"@qvac/ocr-ggml": "^0.18.0",
|
|
198
|
+
"@qvac/translation-nmtcpp": "^0.10.0",
|
|
199
|
+
"@qvac/tts-ggml": "^0.7.4",
|
|
200
|
+
"@qvac/vla-ggml": "^0.21.1"
|
|
198
201
|
},
|
|
199
202
|
"peerDependenciesMeta": {
|
|
200
203
|
"@qvac/asr-ggml": {
|
|
@@ -238,23 +241,24 @@
|
|
|
238
241
|
}
|
|
239
242
|
},
|
|
240
243
|
"devDependencies": {
|
|
241
|
-
"@qvac/asr-ggml": "^0.
|
|
242
|
-
"@qvac/audiogen-ggml": "^0.1
|
|
243
|
-
"@qvac/bci-whispercpp": "^0.
|
|
244
|
-
"@qvac/classification-ggml": "^0.
|
|
244
|
+
"@qvac/asr-ggml": "^0.3.0",
|
|
245
|
+
"@qvac/audiogen-ggml": "^0.2.1",
|
|
246
|
+
"@qvac/bci-whispercpp": "^0.7.1",
|
|
247
|
+
"@qvac/classification-ggml": "^0.20.0",
|
|
245
248
|
"@qvac/decoder-audio": "^0.5.0",
|
|
246
249
|
"@qvac/diffusion-cpp": "^0.17.0",
|
|
247
|
-
"@qvac/embed-llamacpp": "^0.
|
|
250
|
+
"@qvac/embed-llamacpp": "^0.34.0",
|
|
248
251
|
"@qvac/langdetect-text": "^0.1.2",
|
|
249
|
-
"@qvac/llm-llamacpp": "^0.
|
|
250
|
-
"@qvac/ocr-ggml": "^0.
|
|
251
|
-
"@qvac/translation-nmtcpp": "^
|
|
252
|
-
"@qvac/tts-ggml": "^0.
|
|
253
|
-
"@qvac/vla-ggml": "^0.
|
|
252
|
+
"@qvac/llm-llamacpp": "^0.45.0",
|
|
253
|
+
"@qvac/ocr-ggml": "^0.18.0",
|
|
254
|
+
"@qvac/translation-nmtcpp": "^0.10.0",
|
|
255
|
+
"@qvac/tts-ggml": "^0.7.4",
|
|
256
|
+
"@qvac/vla-ggml": "^0.21.1",
|
|
254
257
|
"@types/brittle": "^3.5.0",
|
|
255
258
|
"bare": "*",
|
|
256
259
|
"bare-console": "*",
|
|
257
260
|
"bare-stdio": "^1.0.3",
|
|
261
|
+
"bare-subprocess": "^6.1.0",
|
|
258
262
|
"bare-timers": "*",
|
|
259
263
|
"brittle": "^4.1.0",
|
|
260
264
|
"lunte": "^1.8.4",
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Helper for detecting the llama.cpp addon's `Cancelled` status error
|
|
3
|
-
* (`LlmErrors.hpp::Cancelled = 27`) on the continuous-batching path.
|
|
4
|
-
*
|
|
5
|
-
* When a batch is cancelled while it still has prompts queued behind the
|
|
6
|
-
* `parallel` slot limit, `ContinuousBatchScheduler::cancelPendingLocked`
|
|
7
|
-
* fails the whole batch group with a `Cancelled` `StatusError`, which
|
|
8
|
-
* `processBatch` rethrows — so `model.run(prompts)` rejects rather than
|
|
9
|
-
* resolving with partial outputs (see
|
|
10
|
-
* `packages/llm-llamacpp/docs/continuous-batching.md` "Cancellation
|
|
11
|
-
* semantics"). In-flight slots are still cancelled gracefully, but the
|
|
12
|
-
* shared group carries the error.
|
|
13
|
-
*
|
|
14
|
-
* The addon's JS-facing throw goes through `js_throw_error(env, code,
|
|
15
|
-
* msg)` (see `JsUtils.hpp::JSCATCH`) where `code` comes from
|
|
16
|
-
* `qvac_errors::StatusError::codeString()` and formats as
|
|
17
|
-
* `"[ <addonId> :: Cancelled ]"`. This helper lets the batch handler
|
|
18
|
-
* recognise that rejection so it can complete the run with cancelled
|
|
19
|
-
* terminal events (matching the graceful cancel semantics) instead
|
|
20
|
-
* of surfacing a generic `CompletionFailedError`.
|
|
21
|
-
*/
|
|
22
|
-
export declare function isAddonCancelledError(err: unknown): boolean;
|
|
@@ -1,30 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Helper for detecting the llama.cpp addon's `Cancelled` status error
|
|
3
|
-
* (`LlmErrors.hpp::Cancelled = 27`) on the continuous-batching path.
|
|
4
|
-
*
|
|
5
|
-
* When a batch is cancelled while it still has prompts queued behind the
|
|
6
|
-
* `parallel` slot limit, `ContinuousBatchScheduler::cancelPendingLocked`
|
|
7
|
-
* fails the whole batch group with a `Cancelled` `StatusError`, which
|
|
8
|
-
* `processBatch` rethrows — so `model.run(prompts)` rejects rather than
|
|
9
|
-
* resolving with partial outputs (see
|
|
10
|
-
* `packages/llm-llamacpp/docs/continuous-batching.md` "Cancellation
|
|
11
|
-
* semantics"). In-flight slots are still cancelled gracefully, but the
|
|
12
|
-
* shared group carries the error.
|
|
13
|
-
*
|
|
14
|
-
* The addon's JS-facing throw goes through `js_throw_error(env, code,
|
|
15
|
-
* msg)` (see `JsUtils.hpp::JSCATCH`) where `code` comes from
|
|
16
|
-
* `qvac_errors::StatusError::codeString()` and formats as
|
|
17
|
-
* `"[ <addonId> :: Cancelled ]"`. This helper lets the batch handler
|
|
18
|
-
* recognise that rejection so it can complete the run with cancelled
|
|
19
|
-
* terminal events (matching the graceful cancel semantics) instead
|
|
20
|
-
* of surfacing a generic `CompletionFailedError`.
|
|
21
|
-
*/
|
|
22
|
-
export function isAddonCancelledError(err) {
|
|
23
|
-
if (typeof err !== 'object' || err === null)
|
|
24
|
-
return false;
|
|
25
|
-
const code = err.code;
|
|
26
|
-
// Anchor to the codeString tail (`[ <addonId> :: Cancelled ]`) so a
|
|
27
|
-
// hypothetical sibling like `CancelledByPolicy` after an addon-side
|
|
28
|
-
// rename doesn't false-positive here.
|
|
29
|
-
return typeof code === 'string' && /::\s*Cancelled\s*\]/.test(code);
|
|
30
|
-
}
|
|
@@ -1,22 +0,0 @@
|
|
|
1
|
-
const SHARD_PATTERN = /^(.+)-(\d{5})-of-(\d{5})\.gguf$/;
|
|
2
|
-
/** Expand a sharded GGUF path into `.tensors.txt` plus all shards. */
|
|
3
|
-
export function expandGGUFIntoShards(modelPath) {
|
|
4
|
-
const lastSep = Math.max(modelPath.lastIndexOf('/'), modelPath.lastIndexOf('\\'));
|
|
5
|
-
const dir = lastSep >= 0 ? modelPath.slice(0, lastSep) : '';
|
|
6
|
-
const sep = lastSep >= 0 ? modelPath.charAt(lastSep) : '/';
|
|
7
|
-
const filename = lastSep >= 0 ? modelPath.slice(lastSep + 1) : modelPath;
|
|
8
|
-
const match = filename.match(SHARD_PATTERN);
|
|
9
|
-
if (!match || !match[1] || !match[3])
|
|
10
|
-
return [modelPath];
|
|
11
|
-
const baseFilename = match[1];
|
|
12
|
-
const totalDigits = match[3];
|
|
13
|
-
const totalShards = Number.parseInt(totalDigits, 10);
|
|
14
|
-
if (totalShards <= 0)
|
|
15
|
-
return [modelPath];
|
|
16
|
-
const join = (name) => (dir ? `${dir}${sep}${name}` : name);
|
|
17
|
-
const shards = [join(`${baseFilename}.tensors.txt`)];
|
|
18
|
-
for (let i = 1; i <= totalShards; i++) {
|
|
19
|
-
shards.push(join(`${baseFilename}-${String(i).padStart(5, '0')}-of-${totalDigits}.gguf`));
|
|
20
|
-
}
|
|
21
|
-
return shards;
|
|
22
|
-
}
|