@qvac/sdk 0.13.4 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +95 -1151
- package/README.md +1 -2
- package/bare-imports.json +0 -5
- package/dist/client/api/bci-transcribe.js +2 -1
- package/dist/client/api/download-asset.js +2 -1
- package/dist/client/api/finetune.js +6 -5
- package/dist/client/api/index.d.ts +1 -0
- package/dist/client/api/index.js +1 -0
- package/dist/client/api/load-model.d.ts +11 -8
- package/dist/client/api/load-model.js +6 -6
- package/dist/client/api/subscribe-logs.d.ts +28 -0
- package/dist/client/api/subscribe-logs.js +43 -0
- package/dist/client/api/text-to-speech.js +2 -1
- package/dist/client/config-loader/config-utils.js +2 -4
- package/dist/client/config-loader/resolve-config.bare.js +4 -3
- package/dist/client/parse-input.d.ts +3 -0
- package/dist/client/parse-input.js +14 -0
- package/dist/client/rpc/node-rpc-client.js +8 -2
- package/dist/client/rpc/rpc-client.js +39 -7
- package/dist/commands/bundle/constants.js +2 -1
- package/dist/examples/bci/bci-filesystem-streaming.js +14 -14
- package/dist/examples/bci/bci-filesystem.js +17 -13
- package/dist/examples/cache-management.js +38 -38
- package/dist/examples/cancel-by-request-id.js +9 -9
- package/dist/examples/completion-events.js +20 -14
- package/dist/examples/completion-stop-reason.js +13 -7
- package/dist/examples/concurrent-completion-collision.js +7 -7
- package/dist/examples/config-reload.js +7 -8
- package/dist/examples/default-config-usage.js +4 -4
- package/dist/examples/delegated-inference/composite.js +14 -10
- package/dist/examples/delegated-inference/consumer-profiled.js +13 -13
- package/dist/examples/delegated-inference/consumer.js +16 -12
- package/dist/examples/delegated-inference/provider.js +12 -12
- package/dist/examples/diffusion-esrgan-upscale.js +19 -15
- package/dist/examples/diffusion-flux2-klein-img2img.js +16 -11
- package/dist/examples/diffusion-flux2-klein.js +14 -9
- package/dist/examples/diffusion-img2img.js +9 -10
- package/dist/examples/diffusion-img2vid.js +17 -12
- package/dist/examples/diffusion-simple.js +1 -1
- package/dist/examples/diffusion-txt2img.js +14 -9
- package/dist/examples/diffusion-txt2vid.js +15 -10
- package/dist/examples/download-with-blind-relays.js +16 -21
- package/dist/examples/download-with-cancel.js +15 -16
- package/dist/examples/embed-multi-gpu.js +3 -4
- package/dist/examples/embed-p2p.js +11 -7
- package/dist/examples/finetune/llamacpp-finetune.js +7 -7
- package/dist/examples/kv-cache-cleanup.js +35 -35
- package/dist/examples/kv-cache-custom-key.js +24 -24
- package/dist/examples/kv-cache-example.js +9 -9
- package/dist/examples/llamacpp-cache.js +18 -14
- package/dist/examples/llamacpp-dynamic-tools.js +15 -9
- package/dist/examples/llamacpp-filesystem.js +15 -9
- package/dist/examples/llamacpp-http.js +12 -10
- package/dist/examples/llamacpp-multi-gpu.js +4 -4
- package/dist/examples/llamacpp-multimodal.js +13 -9
- package/dist/examples/llamacpp-p2p.js +8 -4
- package/dist/examples/llamacpp-sharded.js +5 -5
- package/dist/examples/llamacpp-structured-output.js +15 -9
- package/dist/examples/logging-file-transport.js +3 -3
- package/dist/examples/logging-global.d.ts +2 -0
- package/dist/examples/logging-global.js +34 -0
- package/dist/examples/logging-streaming.js +7 -10
- package/dist/examples/mcp-websearch.js +24 -18
- package/dist/examples/multi-model-demo.js +17 -11
- package/dist/examples/ocr-fasttext.js +14 -16
- package/dist/examples/parallel-download.js +7 -7
- package/dist/examples/plugins.js +40 -28
- package/dist/examples/profiling/basic.js +16 -10
- package/dist/examples/profiling/per-call.js +17 -11
- package/dist/examples/quickstart.bare.d.ts +2 -0
- package/dist/examples/quickstart.bare.js +18 -0
- package/dist/examples/quickstart.js +10 -3
- package/dist/examples/rag/rag-chromadb.js +18 -14
- package/dist/examples/rag/rag-hyperdb/cancellation.js +17 -14
- package/dist/examples/rag/rag-hyperdb/ingest.js +14 -12
- package/dist/examples/rag/rag-hyperdb/pipeline.js +19 -17
- package/dist/examples/rag/rag-hyperdb/workspaces.js +31 -27
- package/dist/examples/rag/rag-lancedb.js +12 -8
- package/dist/examples/rag/rag-sqlite.js +11 -7
- package/dist/examples/registry-query.js +17 -17
- package/dist/examples/seed-p2p.js +10 -6
- package/dist/examples/suspend-resume.js +20 -16
- package/dist/examples/tools/llamacpp-native-tools.js +24 -18
- package/dist/examples/tools/llamacpp-tools-gemma4.js +14 -8
- package/dist/examples/tools/llamacpp-tools-harmony-multiturn.js +20 -14
- package/dist/examples/tools/llamacpp-tools-pythonic.js +14 -8
- package/dist/examples/tools/llamacpp-tools-qwen35.js +14 -8
- package/dist/examples/transcription/parakeet-ctc-filesystem.js +12 -9
- package/dist/examples/transcription/parakeet-microphone-record.js +15 -9
- package/dist/examples/transcription/parakeet-microphone-stream.js +15 -9
- package/dist/examples/transcription/parakeet-sortformer-streaming.js +12 -6
- package/dist/examples/transcription/parakeet-sortformer.js +3 -5
- package/dist/examples/transcription/parakeet-tdt-filesystem.js +13 -10
- package/dist/examples/transcription/whispercpp-filesystem-streaming.js +16 -17
- package/dist/examples/transcription/whispercpp-filesystem.js +14 -10
- package/dist/examples/transcription/whispercpp-microphone-conversation.js +10 -10
- package/dist/examples/transcription/whispercpp-microphone-record.js +6 -6
- package/dist/examples/transcription/whispercpp-prompt.js +14 -10
- package/dist/examples/translation/translation-bergamot-batch.js +11 -7
- package/dist/examples/translation/translation-bergamot-pivot.js +13 -9
- package/dist/examples/translation/translation-bergamot.js +8 -4
- package/dist/examples/translation/translation-indic.js +8 -4
- package/dist/examples/translation/translation-llm-afriquegemma.js +7 -3
- package/dist/examples/translation/translation-llm-context.js +8 -4
- package/dist/examples/translation/translation-llm.js +7 -3
- package/dist/examples/translation/translation-stream.js +4 -4
- package/dist/examples/tts/chatterbox.js +19 -11
- package/dist/examples/tts/llm-to-tts-streaming.js +27 -14
- package/dist/examples/tts/supertonic-multilingual.js +18 -14
- package/dist/examples/tts/supertonic.js +19 -15
- package/dist/examples/tts/utils.js +1 -1
- package/dist/examples/vla-pi05.js +18 -14
- package/dist/examples/vla-smolvla.js +18 -14
- package/dist/examples/voice-assistant/voice-assistant.js +12 -12
- package/dist/index.d.ts +3 -4
- package/dist/index.js +3 -4
- package/dist/logging/client-logger.js +2 -1
- package/dist/logging/index.d.ts +1 -1
- package/dist/logging/index.js +1 -1
- package/dist/logging/namespaces.d.ts +1 -0
- package/dist/logging/namespaces.js +2 -0
- package/dist/logging/server-logger.js +1 -4
- package/dist/logging/utils.js +6 -1
- package/dist/models/registry/index.d.ts +17 -0
- package/dist/models/registry/index.js +20 -1
- package/dist/models/registry/models.d.ts +1062 -38
- package/dist/models/registry/models.js +3546 -2490
- package/dist/pear/pre.js +2 -2
- package/dist/schemas/bci-config.d.ts +37 -0
- package/dist/schemas/bci-config.js +2 -0
- package/dist/schemas/bci.d.ts +2 -0
- package/dist/schemas/common.d.ts +171 -94
- package/dist/schemas/completion-stream.d.ts +6 -3
- package/dist/schemas/completion-stream.js +10 -2
- package/dist/schemas/download-asset.d.ts +5 -5
- package/dist/schemas/engine-addon-map.d.ts +1 -1
- package/dist/schemas/engine-addon-map.js +6 -3
- package/dist/schemas/error.js +6 -1
- package/dist/schemas/llamacpp-config.d.ts +27 -8
- package/dist/schemas/llamacpp-config.js +19 -2
- package/dist/schemas/load-model.d.ts +4170 -570
- package/dist/schemas/load-model.js +85 -26
- package/dist/schemas/logging-stream.js +7 -1
- package/dist/schemas/model-src-utils.d.ts +6 -6
- package/dist/schemas/model-types.d.ts +14 -15
- package/dist/schemas/model-types.js +10 -11
- package/dist/schemas/ocr.d.ts +10 -8
- package/dist/schemas/ocr.js +8 -6
- package/dist/schemas/plugin.d.ts +6 -5
- package/dist/schemas/plugin.js +4 -4
- package/dist/schemas/rag.d.ts +4 -4
- package/dist/schemas/registry.d.ts +8 -8
- package/dist/schemas/registry.js +1 -1
- package/dist/schemas/reload-config.d.ts +6 -6
- package/dist/schemas/sdcpp-config.d.ts +12 -12
- package/dist/schemas/sdk-config.d.ts +171 -117
- package/dist/schemas/sdk-config.js +5 -5
- package/dist/schemas/sdk-errors-client.d.ts +1 -0
- package/dist/schemas/sdk-errors-client.js +7 -2
- package/dist/schemas/text-to-speech.d.ts +157 -13
- package/dist/schemas/text-to-speech.js +70 -7
- package/dist/schemas/transcription-config.d.ts +5 -1
- package/dist/schemas/transcription-config.js +2 -0
- package/dist/schemas/transcription.d.ts +3 -0
- package/dist/schemas/transcription.js +2 -1
- package/dist/schemas/translate.d.ts +6 -6
- package/dist/schemas/translate.js +2 -3
- package/dist/schemas/translation-config.d.ts +20 -20
- package/dist/server/bare/hyperswarm.js +3 -3
- package/dist/server/bare/ops/transcribe.js +46 -15
- package/dist/server/bare/plugins/bci-whispercpp-transcription/args.d.ts +3 -0
- package/dist/server/bare/plugins/bci-whispercpp-transcription/args.js +13 -0
- package/dist/server/bare/plugins/bci-whispercpp-transcription/plugin.d.ts +41 -0
- package/dist/server/bare/plugins/bci-whispercpp-transcription/plugin.js +6 -11
- package/dist/server/bare/plugins/bci-whispercpp-transcription/resolve-config.d.ts +75 -0
- package/dist/server/bare/plugins/bci-whispercpp-transcription/resolve-config.js +11 -0
- package/dist/server/bare/plugins/{onnx-ocr → ggml-ocr}/plugin.d.ts +16 -10
- package/dist/server/bare/plugins/{onnx-ocr → ggml-ocr}/plugin.js +25 -18
- package/dist/server/bare/plugins/index.d.ts +1 -1
- package/dist/server/bare/plugins/index.js +1 -1
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stats.d.ts +10 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stats.js +29 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.js +4 -22
- package/dist/server/bare/plugins/llamacpp-completion/plugin.d.ts +15 -7
- package/dist/server/bare/plugins/nmtcpp-translation/plugin.d.ts +7 -7
- package/dist/server/bare/plugins/parakeet-transcription/plugin.d.ts +2 -0
- package/dist/server/bare/plugins/sdcpp-generation/plugin.d.ts +10 -10
- package/dist/server/bare/plugins/tts-ggml/plugin.d.ts +14 -8
- package/dist/server/bare/plugins/tts-ggml/plugin.js +12 -0
- package/dist/server/bare/plugins/whispercpp-transcription/plugin.d.ts +3 -1
- package/dist/server/bare/registry/logging-stream-registry.d.ts +4 -2
- package/dist/server/bare/registry/logging-stream-registry.js +12 -7
- package/dist/server/bare/registry/model-config-utils.js +2 -2
- package/dist/server/bare/registry/registry-client.js +1 -1
- package/dist/server/bare/runtime-lifecycle.d.ts +9 -0
- package/dist/server/bare/runtime-lifecycle.js +24 -0
- package/dist/server/bare/types/addon-responses.d.ts +1 -0
- package/dist/server/env.js +7 -10
- package/dist/server/plugins/registry.js +2 -4
- package/dist/server/rpc/handlers/load-model/handler.js +2 -4
- package/dist/server/rpc/handlers/load-model/http.js +253 -51
- package/dist/server/rpc/handlers/logging-stream.js +4 -2
- package/dist/server/rpc/handlers/plugin-invoke.js +5 -16
- package/dist/server/utils/completion-normalizer.js +13 -4
- package/dist/server/utils/worker-lock.js +5 -5
- package/dist/server/worker-core.d.ts +1 -1
- package/dist/server/worker-core.js +28 -18
- package/dist/utils/errors-client.d.ts +3 -0
- package/dist/utils/errors-client.js +5 -0
- package/dist/utils/errors-server.d.ts +2 -0
- package/dist/utils/errors-server.js +3 -0
- package/dist/utils/zod-error.d.ts +3 -0
- package/dist/utils/zod-error.js +4 -0
- package/package.json +21 -24
- package/dist/polyfills/bare-globals.d.ts +0 -2
- package/dist/polyfills/bare-globals.js +0 -4
- package/dist/polyfills/bare-globals.noop.d.ts +0 -2
- package/dist/polyfills/bare-globals.noop.js +0 -1
- /package/dist/server/bare/plugins/{onnx-ocr → ggml-ocr}/ops/ocr-stream.d.ts +0 -0
- /package/dist/server/bare/plugins/{onnx-ocr → ggml-ocr}/ops/ocr-stream.js +0 -0
|
@@ -2,11 +2,11 @@ import { completion, loadModel, unloadModel } from "@qvac/sdk";
|
|
|
2
2
|
// Get GGUF file path from command line arguments
|
|
3
3
|
const ggufPath = process.argv[2];
|
|
4
4
|
if (!ggufPath) {
|
|
5
|
-
console.error("
|
|
6
|
-
console.error("Usage: bun run examples/llamacpp-filesystem.ts <path-to-gguf-file>");
|
|
5
|
+
console.error("✖ Please provide the path to a GGUF file as the first argument");
|
|
6
|
+
console.error("✖ Usage: bun run examples/llamacpp-filesystem.ts <path-to-gguf-file>");
|
|
7
7
|
process.exit(1);
|
|
8
8
|
}
|
|
9
|
-
console.log(
|
|
9
|
+
console.log(`▸ Loading GGUF model from: ${ggufPath}`);
|
|
10
10
|
try {
|
|
11
11
|
// Load model from provided file path
|
|
12
12
|
const modelId = await loadModel({
|
|
@@ -15,14 +15,20 @@ try {
|
|
|
15
15
|
modelConfig: {
|
|
16
16
|
ctx_size: 4096,
|
|
17
17
|
},
|
|
18
|
-
onProgress: (
|
|
18
|
+
onProgress: (p) => {
|
|
19
|
+
const mb = (n) => (n / 1e6).toFixed(1);
|
|
20
|
+
const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
|
|
21
|
+
process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
|
|
22
|
+
if (p.percentage >= 100)
|
|
23
|
+
process.stderr.write("\n");
|
|
24
|
+
},
|
|
19
25
|
});
|
|
20
|
-
console.log(
|
|
26
|
+
console.log(`▸ Model loaded successfully! Model ID: ${modelId}`);
|
|
21
27
|
// Create conversation history
|
|
22
28
|
const history = [
|
|
23
29
|
{ role: "user", content: "Explain Bitcoin in 3 key points" },
|
|
24
30
|
];
|
|
25
|
-
console.log("\n
|
|
31
|
+
console.log("\n▸ AI Response:");
|
|
26
32
|
process.stdout.write(""); // Start response on new line
|
|
27
33
|
// Stream completion
|
|
28
34
|
const result = completion({ modelId, history, stream: true });
|
|
@@ -30,11 +36,11 @@ try {
|
|
|
30
36
|
process.stdout.write(token);
|
|
31
37
|
}
|
|
32
38
|
const stats = await result.stats;
|
|
33
|
-
console.log("\n
|
|
34
|
-
console.log("\n\n
|
|
39
|
+
console.log("\n▸ Performance Stats:", stats);
|
|
40
|
+
console.log("\n\n▸ Completed!");
|
|
35
41
|
await unloadModel({ modelId, clearStorage: false });
|
|
36
42
|
}
|
|
37
43
|
catch (error) {
|
|
38
|
-
console.error("
|
|
44
|
+
console.error("✖", error);
|
|
39
45
|
process.exit(1);
|
|
40
46
|
}
|
|
@@ -2,7 +2,7 @@ import { completion, loadModel, unloadModel } from "@qvac/sdk";
|
|
|
2
2
|
// Get HTTP URL from command line arguments or use default HuggingFace URL
|
|
3
3
|
const httpUrl = process.argv[2] ||
|
|
4
4
|
"https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf";
|
|
5
|
-
console.log(
|
|
5
|
+
console.log(`▸ Loading GGUF model from HTTP: ${httpUrl}`);
|
|
6
6
|
try {
|
|
7
7
|
// Load model from HTTP URL
|
|
8
8
|
const modelId = await loadModel({
|
|
@@ -11,18 +11,20 @@ try {
|
|
|
11
11
|
modelConfig: {
|
|
12
12
|
ctx_size: 4096,
|
|
13
13
|
},
|
|
14
|
-
onProgress: (
|
|
15
|
-
const
|
|
16
|
-
const
|
|
17
|
-
|
|
14
|
+
onProgress: (p) => {
|
|
15
|
+
const mb = (n) => (n / 1e6).toFixed(1);
|
|
16
|
+
const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
|
|
17
|
+
process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
|
|
18
|
+
if (p.percentage >= 100)
|
|
19
|
+
process.stderr.write("\n");
|
|
18
20
|
},
|
|
19
21
|
});
|
|
20
|
-
console.log(
|
|
22
|
+
console.log(`▸ Model loaded successfully! Model ID: ${modelId}`);
|
|
21
23
|
// Create conversation history
|
|
22
24
|
const history = [
|
|
23
25
|
{ role: "user", content: "Explain quantum computing in 3 key points" },
|
|
24
26
|
];
|
|
25
|
-
console.log("\n
|
|
27
|
+
console.log("\n▸ AI Response:");
|
|
26
28
|
process.stdout.write(""); // Start response on new line
|
|
27
29
|
// Stream completion
|
|
28
30
|
const result = completion({ modelId, history, stream: true });
|
|
@@ -30,11 +32,11 @@ try {
|
|
|
30
32
|
process.stdout.write(token);
|
|
31
33
|
}
|
|
32
34
|
const stats = await result.stats;
|
|
33
|
-
console.log("\n
|
|
34
|
-
console.log("\n\n
|
|
35
|
+
console.log("\n▸ Performance Stats:", stats);
|
|
36
|
+
console.log("\n\n▸ Completed!");
|
|
35
37
|
await unloadModel({ modelId, clearStorage: false });
|
|
36
38
|
}
|
|
37
39
|
catch (error) {
|
|
38
|
-
console.error("
|
|
40
|
+
console.error("✖", error);
|
|
39
41
|
process.exit(1);
|
|
40
42
|
}
|
|
@@ -28,11 +28,11 @@ try {
|
|
|
28
28
|
onProgress: (progress) => {
|
|
29
29
|
if (progress.shardInfo) {
|
|
30
30
|
const { shardInfo } = progress;
|
|
31
|
-
console.log(
|
|
31
|
+
console.log(`▸ Downloading ${shardInfo.shardName} (${shardInfo.currentShard}/${shardInfo.totalShards}) ` +
|
|
32
32
|
`— overall: ${shardInfo.overallPercentage.toFixed(1)}%`);
|
|
33
33
|
}
|
|
34
34
|
else {
|
|
35
|
-
console.log(
|
|
35
|
+
console.log(`▸ Downloading: ${progress.percentage.toFixed(1)}%`);
|
|
36
36
|
}
|
|
37
37
|
},
|
|
38
38
|
});
|
|
@@ -47,11 +47,11 @@ try {
|
|
|
47
47
|
process.stdout.write(token);
|
|
48
48
|
}
|
|
49
49
|
const stats = await result.stats;
|
|
50
|
-
console.log("\n\
|
|
50
|
+
console.log("\n\n▸ Stats:", stats);
|
|
51
51
|
await unloadModel({ modelId, clearStorage: false });
|
|
52
52
|
process.exit(0);
|
|
53
53
|
}
|
|
54
54
|
catch (error) {
|
|
55
|
-
console.error("
|
|
55
|
+
console.error("✖", error);
|
|
56
56
|
process.exit(1);
|
|
57
57
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { completion, loadModel, SMOLVLM2_500M_MULTIMODAL_Q8_0, MMPROJ_SMOLVLM2_500M_MULTIMODAL_Q8_0, unloadModel, } from "@qvac/sdk";
|
|
2
2
|
if (process.argv.length < 3) {
|
|
3
|
-
console.error(
|
|
3
|
+
console.error(`▸ Specify an image file path as the first argument and a second image file path as the second (optional) argument`);
|
|
4
4
|
process.exit(1);
|
|
5
5
|
}
|
|
6
6
|
try {
|
|
@@ -13,8 +13,12 @@ try {
|
|
|
13
13
|
ctx_size: 1024,
|
|
14
14
|
projectionModelSrc: MMPROJ_SMOLVLM2_500M_MULTIMODAL_Q8_0,
|
|
15
15
|
},
|
|
16
|
-
onProgress: (
|
|
17
|
-
|
|
16
|
+
onProgress: (p) => {
|
|
17
|
+
const mb = (n) => (n / 1e6).toFixed(1);
|
|
18
|
+
const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
|
|
19
|
+
process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
|
|
20
|
+
if (p.percentage >= 100)
|
|
21
|
+
process.stderr.write("\n");
|
|
18
22
|
},
|
|
19
23
|
});
|
|
20
24
|
//Using one particular media
|
|
@@ -30,11 +34,11 @@ try {
|
|
|
30
34
|
process.stdout.write(token);
|
|
31
35
|
}
|
|
32
36
|
const stats = await result.stats;
|
|
33
|
-
console.log("\n
|
|
34
|
-
console.log("--------------------------------");
|
|
37
|
+
console.log("\n▸ Performance Stats:", stats);
|
|
38
|
+
console.log("▸ --------------------------------");
|
|
35
39
|
//Using multiple media
|
|
36
40
|
if (process.argv.length < 4) {
|
|
37
|
-
console.log(
|
|
41
|
+
console.log(`▸ Only one image provided, terminating`);
|
|
38
42
|
process.exit(0);
|
|
39
43
|
}
|
|
40
44
|
const imageFilePath2 = process.argv[3];
|
|
@@ -50,11 +54,11 @@ try {
|
|
|
50
54
|
process.stdout.write(token);
|
|
51
55
|
}
|
|
52
56
|
const stats2 = await result2.stats;
|
|
53
|
-
console.log("\n
|
|
54
|
-
console.log("--------------------------------");
|
|
57
|
+
console.log("\n▸ Performance Stats:", stats2);
|
|
58
|
+
console.log("▸ --------------------------------");
|
|
55
59
|
await unloadModel({ modelId, clearStorage: false });
|
|
56
60
|
}
|
|
57
61
|
catch (error) {
|
|
58
|
-
console.error("
|
|
62
|
+
console.error("✖", error);
|
|
59
63
|
process.exit(1);
|
|
60
64
|
}
|
|
@@ -3,8 +3,12 @@ try {
|
|
|
3
3
|
// First just cache the model
|
|
4
4
|
await downloadAsset({
|
|
5
5
|
assetSrc: LLAMA_3_2_1B_INST_Q4_0,
|
|
6
|
-
onProgress: (
|
|
7
|
-
|
|
6
|
+
onProgress: (p) => {
|
|
7
|
+
const mb = (n) => (n / 1e6).toFixed(1);
|
|
8
|
+
const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
|
|
9
|
+
process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
|
|
10
|
+
if (p.percentage >= 100)
|
|
11
|
+
process.stderr.write("\n");
|
|
8
12
|
},
|
|
9
13
|
});
|
|
10
14
|
// Then load it in memory from cache
|
|
@@ -27,11 +31,11 @@ try {
|
|
|
27
31
|
process.stdout.write(token);
|
|
28
32
|
}
|
|
29
33
|
const stats = await result.stats;
|
|
30
|
-
console.log("\n
|
|
34
|
+
console.log("\n▸ Performance Stats:", stats);
|
|
31
35
|
// Change `clearStorage: true` to delete cached model files
|
|
32
36
|
await unloadModel({ modelId, clearStorage: false });
|
|
33
37
|
}
|
|
34
38
|
catch (error) {
|
|
35
|
-
console.error("
|
|
39
|
+
console.error("✖", error);
|
|
36
40
|
process.exit(1);
|
|
37
41
|
}
|
|
@@ -20,13 +20,13 @@ try {
|
|
|
20
20
|
if (progress.shardInfo) {
|
|
21
21
|
// For pattern-based or Hyperdrive shards
|
|
22
22
|
const { shardInfo } = progress;
|
|
23
|
-
console.log(
|
|
23
|
+
console.log(`▸ Downloading ${shardInfo.shardName} (${shardInfo.currentShard}/${shardInfo.totalShards})\n` +
|
|
24
24
|
` File: ${progress.percentage.toFixed(1)}% (${(progress.downloaded / 1024 / 1024).toFixed(2)}MB / ${(progress.total / 1024 / 1024).toFixed(2)}MB)\n` +
|
|
25
25
|
` Overall: ${shardInfo.overallPercentage.toFixed(1)}% (${(shardInfo.overallDownloaded / 1024 / 1024).toFixed(2)}MB / ${(shardInfo.overallTotal / 1024 / 1024).toFixed(2)}MB)`);
|
|
26
26
|
}
|
|
27
27
|
else {
|
|
28
28
|
// For archive-based shards
|
|
29
|
-
console.log(
|
|
29
|
+
console.log(`▸ Progress: ${progress.percentage.toFixed(1)}% ` +
|
|
30
30
|
`(${(progress.downloaded / 1024 / 1024).toFixed(2)}MB / ${(progress.total / 1024 / 1024).toFixed(2)}MB)`);
|
|
31
31
|
}
|
|
32
32
|
},
|
|
@@ -38,16 +38,16 @@ try {
|
|
|
38
38
|
},
|
|
39
39
|
];
|
|
40
40
|
const result = completion({ modelId, history, stream: true });
|
|
41
|
-
console.log("\n
|
|
41
|
+
console.log("\n▸ Model response:");
|
|
42
42
|
for await (const token of result.tokenStream) {
|
|
43
43
|
process.stdout.write(token);
|
|
44
44
|
}
|
|
45
45
|
const stats = await result.stats;
|
|
46
|
-
console.log("\n\n
|
|
46
|
+
console.log("\n\n▸ Performance Stats:", stats);
|
|
47
47
|
await unloadModel({ modelId, clearStorage: false });
|
|
48
48
|
process.exit(0);
|
|
49
49
|
}
|
|
50
50
|
catch (error) {
|
|
51
|
-
console.error("
|
|
51
|
+
console.error("✖", error);
|
|
52
52
|
process.exit(1);
|
|
53
53
|
}
|
|
@@ -58,25 +58,31 @@ async function runToContent(run) {
|
|
|
58
58
|
try {
|
|
59
59
|
const modelId = await loadModel({
|
|
60
60
|
modelSrc: QWEN3_600M_INST_Q4,
|
|
61
|
-
onProgress: (p) =>
|
|
61
|
+
onProgress: (p) => {
|
|
62
|
+
const mb = (n) => (n / 1e6).toFixed(1);
|
|
63
|
+
const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
|
|
64
|
+
process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
|
|
65
|
+
if (p.percentage >= 100)
|
|
66
|
+
process.stderr.write("\n");
|
|
67
|
+
},
|
|
62
68
|
});
|
|
63
|
-
console.log(`\
|
|
64
|
-
console.log("--- 1. responseFormat: text (baseline, free-form) ---");
|
|
69
|
+
console.log(`\n▸ Model loaded: ${modelId}\n`);
|
|
70
|
+
console.log("▸ --- 1. responseFormat: text (baseline, free-form) ---");
|
|
65
71
|
await runToContent(completion({
|
|
66
72
|
modelId,
|
|
67
73
|
history: HISTORY,
|
|
68
74
|
stream: true,
|
|
69
75
|
responseFormat: { type: "text" },
|
|
70
76
|
}));
|
|
71
|
-
console.log("\n--- 2. responseFormat: json_object (any valid JSON) ---");
|
|
77
|
+
console.log("\n▸ --- 2. responseFormat: json_object (any valid JSON) ---");
|
|
72
78
|
const jsonObjectOut = await runToContent(completion({
|
|
73
79
|
modelId,
|
|
74
80
|
history: JSON_OBJECT_HISTORY,
|
|
75
81
|
stream: true,
|
|
76
82
|
responseFormat: { type: "json_object" },
|
|
77
83
|
}));
|
|
78
|
-
console.log("parsed:", JSON.parse(jsonObjectOut.trim()));
|
|
79
|
-
console.log("\n--- 3. responseFormat: json_schema (strict shape) ---");
|
|
84
|
+
console.log("▸ parsed:", JSON.parse(jsonObjectOut.trim()));
|
|
85
|
+
console.log("\n▸ --- 3. responseFormat: json_schema (strict shape) ---");
|
|
80
86
|
const jsonSchemaOut = await runToContent(completion({
|
|
81
87
|
modelId,
|
|
82
88
|
history: HISTORY,
|
|
@@ -92,14 +98,14 @@ try {
|
|
|
92
98
|
// Output is guaranteed schema-valid JSON: object with exactly
|
|
93
99
|
// {name: string, age: integer, occupation: string}, no extras.
|
|
94
100
|
const parsed = JSON.parse(jsonSchemaOut.trim());
|
|
95
|
-
console.log("parsed:", parsed);
|
|
96
|
-
console.log("schema-valid:", typeof parsed.name === "string" &&
|
|
101
|
+
console.log("▸ parsed:", parsed);
|
|
102
|
+
console.log("▸ schema-valid:", typeof parsed.name === "string" &&
|
|
97
103
|
Number.isInteger(parsed.age) &&
|
|
98
104
|
typeof parsed.occupation === "string" &&
|
|
99
105
|
Object.keys(parsed).sort().join(",") === "age,name,occupation");
|
|
100
106
|
await unloadModel({ modelId });
|
|
101
107
|
}
|
|
102
108
|
catch (error) {
|
|
103
|
-
console.error("
|
|
109
|
+
console.error("✖", error);
|
|
104
110
|
process.exit(1);
|
|
105
111
|
}
|
|
@@ -47,10 +47,10 @@ try {
|
|
|
47
47
|
await unloadModel({ modelId });
|
|
48
48
|
logger.info("Model unloaded successfully");
|
|
49
49
|
// Show user where logs were saved
|
|
50
|
-
console.log(
|
|
51
|
-
console.log("
|
|
50
|
+
console.log(`▸ Logs saved to ${logFile}`);
|
|
51
|
+
console.log("▸ Inspect it to see detailed model operation logs");
|
|
52
52
|
}
|
|
53
53
|
catch (error) {
|
|
54
|
-
console.error("
|
|
54
|
+
console.error("✖", error);
|
|
55
55
|
process.exit(1);
|
|
56
56
|
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// loggerConsoleOutput: false in this config disables the default console
|
|
2
|
+
// transport on both the client and the worker, so the only logs printed are
|
|
3
|
+
// the ones our handler below receives.
|
|
4
|
+
const configDir = import.meta.dirname ?? process.cwd();
|
|
5
|
+
process.env["QVAC_CONFIG_PATH"] = `${configDir}/config/logging/logging.config.json`;
|
|
6
|
+
const { loadModel, completion, unloadModel, subscribeServerLogs, LLAMA_3_2_1B_INST_Q4_0 } = await import("@qvac/sdk");
|
|
7
|
+
try {
|
|
8
|
+
console.log("▸ Starting global logging demo...\n");
|
|
9
|
+
// One subscription captures every server-side log (SDK, models, RAG, …)
|
|
10
|
+
// without having to know any stream IDs ahead of time.
|
|
11
|
+
const unsubscribe = subscribeServerLogs((log) => {
|
|
12
|
+
console.log(`[SDK] [${log.level.toUpperCase()}] [${log.namespace}] ${log.message}`);
|
|
13
|
+
});
|
|
14
|
+
const modelId = await loadModel({
|
|
15
|
+
modelSrc: LLAMA_3_2_1B_INST_Q4_0,
|
|
16
|
+
modelConfig: { ctx_size: 2048 },
|
|
17
|
+
});
|
|
18
|
+
const result = completion({
|
|
19
|
+
modelId,
|
|
20
|
+
history: [{ role: "user", content: "Count from 1 to 5." }],
|
|
21
|
+
stream: true,
|
|
22
|
+
});
|
|
23
|
+
console.log("▸ Response:\n");
|
|
24
|
+
for await (const token of result.tokenStream) {
|
|
25
|
+
process.stdout.write(token);
|
|
26
|
+
}
|
|
27
|
+
await unloadModel({ modelId, clearStorage: false });
|
|
28
|
+
unsubscribe();
|
|
29
|
+
}
|
|
30
|
+
catch (error) {
|
|
31
|
+
console.error("✖", error);
|
|
32
|
+
process.exit(1);
|
|
33
|
+
}
|
|
34
|
+
export {};
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { loadModel, completion, unloadModel, loggingStream, SDK_LOG_ID, LLAMA_3_2_1B_INST_Q4_0, GTE_LARGE_FP16, VERBOSITY, embed, } from "@qvac/sdk";
|
|
2
2
|
try {
|
|
3
|
-
console.log("
|
|
3
|
+
console.log("▸ Streaming SDK and model logs");
|
|
4
4
|
// Note: To configure logging (level and console output), use config file:
|
|
5
5
|
// { "loggerLevel": "debug", "loggerConsoleOutput": false } in qvac.config.json/js/ts
|
|
6
6
|
// Subscribe to SDK server logs in background
|
|
7
|
-
console.log("
|
|
7
|
+
console.log("▸ Subscribing to SDK server logs");
|
|
8
8
|
(async () => {
|
|
9
9
|
for await (const log of loggingStream({ id: SDK_LOG_ID })) {
|
|
10
10
|
console.log(`[SDK] [${log.level.toUpperCase()}] [${log.namespace}] ${log.message}`);
|
|
@@ -13,7 +13,7 @@ try {
|
|
|
13
13
|
// Stream terminated - normal on shutdown
|
|
14
14
|
});
|
|
15
15
|
// Load models
|
|
16
|
-
console.log("
|
|
16
|
+
console.log("▸ Loading models");
|
|
17
17
|
const llmModelId = await loadModel({
|
|
18
18
|
modelSrc: LLAMA_3_2_1B_INST_Q4_0,
|
|
19
19
|
modelConfig: {
|
|
@@ -25,7 +25,7 @@ try {
|
|
|
25
25
|
const embedModelId = await loadModel({
|
|
26
26
|
modelSrc: GTE_LARGE_FP16,
|
|
27
27
|
});
|
|
28
|
-
console.log("
|
|
28
|
+
console.log("▸ Subscribing to model logs");
|
|
29
29
|
(async () => {
|
|
30
30
|
for await (const log of loggingStream({ id: llmModelId })) {
|
|
31
31
|
const timestamp = new Date(log.timestamp).toISOString();
|
|
@@ -54,20 +54,17 @@ try {
|
|
|
54
54
|
modelId: embedModelId,
|
|
55
55
|
text: messages[0]?.content ?? "Hello, world!",
|
|
56
56
|
});
|
|
57
|
-
console.log("
|
|
57
|
+
console.log("▸ Response");
|
|
58
58
|
for await (const token of result.tokenStream) {
|
|
59
59
|
process.stdout.write(token);
|
|
60
60
|
}
|
|
61
61
|
console.log("Embedding (first 20 elements)", embedding.slice(0, 20));
|
|
62
62
|
console.log("Embeddings length", embedding.length);
|
|
63
|
-
console.log("
|
|
64
|
-
" - [SDK] = SDK server operations\n" +
|
|
65
|
-
" - [LLM] = LLM model inference logs\n" +
|
|
66
|
-
" - [EMBED] = Embedding model logs\n");
|
|
63
|
+
console.log("▸ Three streams running: [SDK] server, [LLM] inference, [EMBED] embedding");
|
|
67
64
|
await unloadModel({ modelId: llmModelId, clearStorage: false });
|
|
68
65
|
await unloadModel({ modelId: embedModelId, clearStorage: false });
|
|
69
66
|
}
|
|
70
67
|
catch (error) {
|
|
71
|
-
console.error("
|
|
68
|
+
console.error("✖", error);
|
|
72
69
|
process.exit(1);
|
|
73
70
|
}
|
|
@@ -45,11 +45,11 @@ function parseSearchResults(mcpResult) {
|
|
|
45
45
|
}
|
|
46
46
|
let mcpClient = null;
|
|
47
47
|
try {
|
|
48
|
-
console.log("
|
|
48
|
+
console.log("▸ MCP DuckDuckGo Search Example\n");
|
|
49
49
|
// ============================================================
|
|
50
50
|
// STEP 1: Connect to DuckDuckGo MCP server
|
|
51
51
|
// ============================================================
|
|
52
|
-
console.log("
|
|
52
|
+
console.log("▸ Starting DuckDuckGo MCP server...");
|
|
53
53
|
mcpClient = new Client({
|
|
54
54
|
name: "qvac-ddg-example",
|
|
55
55
|
version: "1.0.0",
|
|
@@ -59,20 +59,26 @@ try {
|
|
|
59
59
|
args: ["-y", "@oevortex/ddg_search"],
|
|
60
60
|
});
|
|
61
61
|
await mcpClient.connect(transport);
|
|
62
|
-
console.log("
|
|
62
|
+
console.log("▸ MCP server connected\n");
|
|
63
63
|
// ============================================================
|
|
64
64
|
// STEP 2: Load model
|
|
65
65
|
// ============================================================
|
|
66
|
-
console.log("
|
|
66
|
+
console.log("▸ Loading model...");
|
|
67
67
|
const modelId = await loadModel({
|
|
68
68
|
modelSrc: QWEN3_1_7B_INST_Q4,
|
|
69
69
|
modelConfig: {
|
|
70
70
|
ctx_size: 4096,
|
|
71
71
|
tools: true,
|
|
72
72
|
},
|
|
73
|
-
onProgress: (
|
|
73
|
+
onProgress: (p) => {
|
|
74
|
+
const mb = (n) => (n / 1e6).toFixed(1);
|
|
75
|
+
const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
|
|
76
|
+
process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
|
|
77
|
+
if (p.percentage >= 100)
|
|
78
|
+
process.stderr.write("\n");
|
|
79
|
+
},
|
|
74
80
|
});
|
|
75
|
-
console.log(`\n
|
|
81
|
+
console.log(`\n▸ Model loaded\n`);
|
|
76
82
|
// ============================================================
|
|
77
83
|
// STEP 3: Ask AI to search the web (with MCP client)
|
|
78
84
|
// ============================================================
|
|
@@ -88,8 +94,8 @@ Always cite your sources with the URL.`,
|
|
|
88
94
|
content: "What is the current weather in New York City?",
|
|
89
95
|
},
|
|
90
96
|
];
|
|
91
|
-
console.log("
|
|
92
|
-
console.log("
|
|
97
|
+
console.log("▸ Asking AI to search the web...\n");
|
|
98
|
+
console.log("▸ AI Response:");
|
|
93
99
|
// Pass MCP client directly to completion - tools are adapted internally!
|
|
94
100
|
const result = completion({
|
|
95
101
|
modelId,
|
|
@@ -106,19 +112,19 @@ Always cite your sources with the URL.`,
|
|
|
106
112
|
// STEP 4: Execute tool calls using call() - automatic MCP routing!
|
|
107
113
|
// ============================================================
|
|
108
114
|
if (toolCalls.length > 0) {
|
|
109
|
-
console.log("
|
|
115
|
+
console.log("▸ Executing search...\n");
|
|
110
116
|
const toolResults = [];
|
|
111
117
|
for (const toolCall of toolCalls) {
|
|
112
|
-
console.log(
|
|
118
|
+
console.log(`▸ ${toolCall.name}(${JSON.stringify(toolCall.arguments)})`);
|
|
113
119
|
if (!toolCall.invoke) {
|
|
114
|
-
console.log(
|
|
120
|
+
console.log(`▸ No handler found for tool "${toolCall.name}"`);
|
|
115
121
|
continue;
|
|
116
122
|
}
|
|
117
123
|
// Use invoke() - automatically routes to the correct MCP client!
|
|
118
124
|
const mcpResult = await toolCall.invoke();
|
|
119
125
|
// Parse and clean up the search results
|
|
120
126
|
const cleanResult = parseSearchResults(mcpResult);
|
|
121
|
-
console.log(
|
|
127
|
+
console.log(`▸ Got search results:`);
|
|
122
128
|
console.log(cleanResult
|
|
123
129
|
.split("\n")
|
|
124
130
|
.map((l) => ` ${l}`)
|
|
@@ -129,7 +135,7 @@ Always cite your sources with the URL.`,
|
|
|
129
135
|
// ============================================================
|
|
130
136
|
// STEP 5: Continue with search results
|
|
131
137
|
// ============================================================
|
|
132
|
-
console.log("
|
|
138
|
+
console.log("▸ Getting AI response with search results...\n");
|
|
133
139
|
history.push({
|
|
134
140
|
role: "assistant",
|
|
135
141
|
content: await result.text,
|
|
@@ -140,7 +146,7 @@ Always cite your sources with the URL.`,
|
|
|
140
146
|
content: tr.result,
|
|
141
147
|
});
|
|
142
148
|
}
|
|
143
|
-
console.log("
|
|
149
|
+
console.log("▸ Final Response:");
|
|
144
150
|
const finalResult = completion({
|
|
145
151
|
modelId,
|
|
146
152
|
history,
|
|
@@ -155,14 +161,14 @@ Always cite your sources with the URL.`,
|
|
|
155
161
|
// ============================================================
|
|
156
162
|
// Cleanup
|
|
157
163
|
// ============================================================
|
|
158
|
-
console.log("
|
|
164
|
+
console.log("▸ Cleaning up...");
|
|
159
165
|
await unloadModel({ modelId, clearStorage: false });
|
|
160
|
-
console.log("
|
|
161
|
-
console.log("
|
|
166
|
+
console.log("▸ Done\n");
|
|
167
|
+
console.log("▸ Example completed!");
|
|
162
168
|
process.exit(0);
|
|
163
169
|
}
|
|
164
170
|
catch (error) {
|
|
165
|
-
console.error("
|
|
171
|
+
console.error("✖", error);
|
|
166
172
|
process.exit(1);
|
|
167
173
|
}
|
|
168
174
|
finally {
|
|
@@ -11,7 +11,13 @@ try {
|
|
|
11
11
|
modelConfig: {
|
|
12
12
|
ctx_size: 4096,
|
|
13
13
|
},
|
|
14
|
-
onProgress: (
|
|
14
|
+
onProgress: (p) => {
|
|
15
|
+
const mb = (n) => (n / 1e6).toFixed(1);
|
|
16
|
+
const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
|
|
17
|
+
process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
|
|
18
|
+
if (p.percentage >= 100)
|
|
19
|
+
process.stderr.write("\n");
|
|
20
|
+
},
|
|
15
21
|
});
|
|
16
22
|
const aliceHistory = [
|
|
17
23
|
{
|
|
@@ -26,7 +32,7 @@ try {
|
|
|
26
32
|
content: "You ask questions about Bitcoin in one sentence.",
|
|
27
33
|
},
|
|
28
34
|
];
|
|
29
|
-
console.log("
|
|
35
|
+
console.log("Bob: ", "What is Bitcoin?");
|
|
30
36
|
const aliceResponse = completion({
|
|
31
37
|
modelId: alice,
|
|
32
38
|
history: [...aliceHistory],
|
|
@@ -34,8 +40,8 @@ try {
|
|
|
34
40
|
});
|
|
35
41
|
const aliceText = await aliceResponse.text;
|
|
36
42
|
aliceHistory.push({ role: "assistant", content: aliceText });
|
|
37
|
-
console.log("
|
|
38
|
-
console.log("
|
|
43
|
+
console.log("Alice: ", aliceText);
|
|
44
|
+
console.log("▸ Alice Stats: ", await aliceResponse.stats);
|
|
39
45
|
const bobResponse = completion({
|
|
40
46
|
modelId: bob,
|
|
41
47
|
history: [...bobHistory, { role: "user", content: aliceText }],
|
|
@@ -43,8 +49,8 @@ try {
|
|
|
43
49
|
});
|
|
44
50
|
const bobText = await bobResponse.text;
|
|
45
51
|
bobHistory.push({ role: "assistant", content: bobText });
|
|
46
|
-
console.log("
|
|
47
|
-
console.log("
|
|
52
|
+
console.log("Bob: ", bobText);
|
|
53
|
+
console.log("▸ Bob Stats: ", await bobResponse.stats);
|
|
48
54
|
const aliceResponse2 = completion({
|
|
49
55
|
modelId: alice,
|
|
50
56
|
history: [...aliceHistory, { role: "user", content: bobText }],
|
|
@@ -52,8 +58,8 @@ try {
|
|
|
52
58
|
});
|
|
53
59
|
const aliceText2 = await aliceResponse2.text;
|
|
54
60
|
aliceHistory.push({ role: "assistant", content: aliceText2 });
|
|
55
|
-
console.log("
|
|
56
|
-
console.log("
|
|
61
|
+
console.log("Alice: ", aliceText2);
|
|
62
|
+
console.log("▸ Alice Stats: ", await aliceResponse2.stats);
|
|
57
63
|
const bobResponse2 = completion({
|
|
58
64
|
modelId: bob,
|
|
59
65
|
history: [...bobHistory, { role: "user", content: aliceText2 }],
|
|
@@ -61,12 +67,12 @@ try {
|
|
|
61
67
|
});
|
|
62
68
|
const bobText2 = await bobResponse2.text;
|
|
63
69
|
bobHistory.push({ role: "assistant", content: bobText2 });
|
|
64
|
-
console.log("
|
|
65
|
-
console.log("
|
|
70
|
+
console.log("Bob: ", bobText2);
|
|
71
|
+
console.log("▸ Bob Stats: ", await bobResponse2.stats);
|
|
66
72
|
await unloadModel({ modelId: alice, clearStorage: false });
|
|
67
73
|
await unloadModel({ modelId: bob, clearStorage: false });
|
|
68
74
|
}
|
|
69
75
|
catch (error) {
|
|
70
|
-
console.error("
|
|
76
|
+
console.error("✖", error);
|
|
71
77
|
process.exit(1);
|
|
72
78
|
}
|