@qvac/sdk 0.13.5 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +95 -1151
- package/README.md +1 -2
- package/bare-imports.json +0 -5
- package/dist/client/api/bci-transcribe.js +2 -1
- package/dist/client/api/download-asset.js +2 -1
- package/dist/client/api/finetune.js +6 -5
- package/dist/client/api/index.d.ts +1 -0
- package/dist/client/api/index.js +1 -0
- package/dist/client/api/load-model.d.ts +11 -8
- package/dist/client/api/load-model.js +6 -6
- package/dist/client/api/subscribe-logs.d.ts +28 -0
- package/dist/client/api/subscribe-logs.js +43 -0
- package/dist/client/api/text-to-speech.js +2 -1
- package/dist/client/config-loader/config-utils.js +2 -4
- package/dist/client/config-loader/resolve-config.bare.js +4 -3
- package/dist/client/parse-input.d.ts +3 -0
- package/dist/client/parse-input.js +14 -0
- package/dist/client/rpc/expo-rpc-client.js +2 -11
- package/dist/client/rpc/node-rpc-client.js +8 -2
- package/dist/client/rpc/rpc-client.js +39 -7
- package/dist/commands/bundle/constants.js +2 -1
- package/dist/commands/bundle/entry-gen.d.ts +1 -1
- package/dist/commands/bundle/entry-gen.js +6 -6
- package/dist/commands/bundle/index.js +3 -1
- package/dist/commands/bundle/resolve-sdk-import.d.ts +6 -0
- package/dist/commands/bundle/resolve-sdk-import.js +60 -0
- package/dist/examples/bci/bci-filesystem-streaming.js +14 -14
- package/dist/examples/bci/bci-filesystem.js +17 -13
- package/dist/examples/cache-management.js +38 -38
- package/dist/examples/cancel-by-request-id.js +9 -9
- package/dist/examples/completion-events.js +20 -14
- package/dist/examples/completion-stop-reason.js +13 -7
- package/dist/examples/concurrent-completion-collision.js +7 -7
- package/dist/examples/config-reload.js +7 -8
- package/dist/examples/default-config-usage.js +4 -4
- package/dist/examples/delegated-inference/composite.js +14 -10
- package/dist/examples/delegated-inference/consumer-profiled.js +13 -13
- package/dist/examples/delegated-inference/consumer.js +16 -12
- package/dist/examples/delegated-inference/provider.js +12 -12
- package/dist/examples/diffusion-esrgan-upscale.js +19 -15
- package/dist/examples/diffusion-flux2-klein-img2img.js +16 -11
- package/dist/examples/diffusion-flux2-klein.js +14 -9
- package/dist/examples/diffusion-img2img.js +9 -10
- package/dist/examples/diffusion-img2vid.js +17 -12
- package/dist/examples/diffusion-simple.js +1 -1
- package/dist/examples/diffusion-txt2img.js +14 -9
- package/dist/examples/diffusion-txt2vid.js +15 -10
- package/dist/examples/download-with-blind-relays.js +16 -21
- package/dist/examples/download-with-cancel.js +15 -16
- package/dist/examples/embed-multi-gpu.js +3 -4
- package/dist/examples/embed-p2p.js +11 -7
- package/dist/examples/finetune/llamacpp-finetune.js +7 -7
- package/dist/examples/kv-cache-cleanup.js +35 -35
- package/dist/examples/kv-cache-custom-key.js +24 -24
- package/dist/examples/kv-cache-example.js +9 -9
- package/dist/examples/llamacpp-cache.js +18 -14
- package/dist/examples/llamacpp-dynamic-tools.js +15 -9
- package/dist/examples/llamacpp-filesystem.js +15 -9
- package/dist/examples/llamacpp-http.js +12 -10
- package/dist/examples/llamacpp-multi-gpu.js +4 -4
- package/dist/examples/llamacpp-multimodal.js +13 -9
- package/dist/examples/llamacpp-p2p.js +8 -4
- package/dist/examples/llamacpp-sharded.js +5 -5
- package/dist/examples/llamacpp-structured-output.js +15 -9
- package/dist/examples/logging-file-transport.js +3 -3
- package/dist/examples/logging-global.d.ts +2 -0
- package/dist/examples/logging-global.js +34 -0
- package/dist/examples/logging-streaming.js +7 -10
- package/dist/examples/mcp-websearch.js +24 -18
- package/dist/examples/multi-model-demo.js +17 -11
- package/dist/examples/ocr-fasttext.js +14 -16
- package/dist/examples/parallel-download.js +7 -7
- package/dist/examples/plugins.js +40 -28
- package/dist/examples/profiling/basic.js +16 -10
- package/dist/examples/profiling/per-call.js +17 -11
- package/dist/examples/quickstart.bare.d.ts +2 -0
- package/dist/examples/quickstart.bare.js +18 -0
- package/dist/examples/quickstart.js +10 -3
- package/dist/examples/rag/rag-chromadb.js +18 -14
- package/dist/examples/rag/rag-hyperdb/cancellation.js +17 -14
- package/dist/examples/rag/rag-hyperdb/ingest.js +14 -12
- package/dist/examples/rag/rag-hyperdb/pipeline.js +19 -17
- package/dist/examples/rag/rag-hyperdb/workspaces.js +31 -27
- package/dist/examples/rag/rag-lancedb.js +12 -8
- package/dist/examples/rag/rag-sqlite.js +11 -7
- package/dist/examples/registry-query.js +17 -17
- package/dist/examples/seed-p2p.js +10 -6
- package/dist/examples/suspend-resume.js +20 -16
- package/dist/examples/tools/llamacpp-native-tools.js +24 -18
- package/dist/examples/tools/llamacpp-tools-gemma4.js +14 -8
- package/dist/examples/tools/llamacpp-tools-harmony-multiturn.js +20 -14
- package/dist/examples/tools/llamacpp-tools-pythonic.js +14 -8
- package/dist/examples/tools/llamacpp-tools-qwen35.js +14 -8
- package/dist/examples/transcription/parakeet-ctc-filesystem.js +12 -9
- package/dist/examples/transcription/parakeet-microphone-record.js +15 -9
- package/dist/examples/transcription/parakeet-microphone-stream.js +15 -9
- package/dist/examples/transcription/parakeet-sortformer-streaming.js +12 -6
- package/dist/examples/transcription/parakeet-sortformer.js +3 -5
- package/dist/examples/transcription/parakeet-tdt-filesystem.js +13 -10
- package/dist/examples/transcription/whispercpp-filesystem-streaming.js +16 -17
- package/dist/examples/transcription/whispercpp-filesystem.js +14 -10
- package/dist/examples/transcription/whispercpp-microphone-conversation.js +10 -10
- package/dist/examples/transcription/whispercpp-microphone-record.js +6 -6
- package/dist/examples/transcription/whispercpp-prompt.js +14 -10
- package/dist/examples/translation/translation-bergamot-batch.js +11 -7
- package/dist/examples/translation/translation-bergamot-pivot.js +13 -9
- package/dist/examples/translation/translation-bergamot.js +8 -4
- package/dist/examples/translation/translation-indic.js +8 -4
- package/dist/examples/translation/translation-llm-afriquegemma.js +7 -3
- package/dist/examples/translation/translation-llm-context.js +8 -4
- package/dist/examples/translation/translation-llm.js +7 -3
- package/dist/examples/translation/translation-stream.js +4 -4
- package/dist/examples/tts/chatterbox.js +19 -11
- package/dist/examples/tts/llm-to-tts-streaming.js +27 -14
- package/dist/examples/tts/supertonic-multilingual.js +18 -14
- package/dist/examples/tts/supertonic.js +19 -15
- package/dist/examples/tts/utils.js +1 -1
- package/dist/examples/vla-pi05.js +18 -14
- package/dist/examples/vla-smolvla.js +18 -14
- package/dist/examples/voice-assistant/voice-assistant.js +12 -12
- package/dist/index.d.ts +3 -4
- package/dist/index.js +3 -4
- package/dist/logging/client-logger.js +2 -1
- package/dist/logging/index.d.ts +1 -1
- package/dist/logging/index.js +1 -1
- package/dist/logging/namespaces.d.ts +1 -0
- package/dist/logging/namespaces.js +2 -0
- package/dist/logging/server-logger.js +1 -4
- package/dist/logging/utils.js +6 -1
- package/dist/models/registry/index.d.ts +17 -0
- package/dist/models/registry/index.js +20 -1
- package/dist/models/registry/models.d.ts +1062 -38
- package/dist/models/registry/models.js +3546 -2490
- package/dist/pear/pre.js +2 -2
- package/dist/schemas/bci-config.d.ts +37 -0
- package/dist/schemas/bci-config.js +2 -0
- package/dist/schemas/bci.d.ts +2 -0
- package/dist/schemas/common.d.ts +171 -94
- package/dist/schemas/completion-stream.d.ts +6 -3
- package/dist/schemas/completion-stream.js +10 -2
- package/dist/schemas/download-asset.d.ts +5 -5
- package/dist/schemas/engine-addon-map.d.ts +1 -1
- package/dist/schemas/engine-addon-map.js +6 -3
- package/dist/schemas/error.js +6 -1
- package/dist/schemas/llamacpp-config.d.ts +27 -8
- package/dist/schemas/llamacpp-config.js +19 -2
- package/dist/schemas/load-model.d.ts +4170 -570
- package/dist/schemas/load-model.js +85 -26
- package/dist/schemas/logging-stream.js +7 -1
- package/dist/schemas/model-src-utils.d.ts +6 -6
- package/dist/schemas/model-types.d.ts +14 -15
- package/dist/schemas/model-types.js +10 -11
- package/dist/schemas/ocr.d.ts +10 -8
- package/dist/schemas/ocr.js +8 -6
- package/dist/schemas/plugin.d.ts +6 -5
- package/dist/schemas/plugin.js +4 -4
- package/dist/schemas/rag.d.ts +4 -4
- package/dist/schemas/registry.d.ts +8 -8
- package/dist/schemas/registry.js +1 -1
- package/dist/schemas/reload-config.d.ts +6 -6
- package/dist/schemas/sdcpp-config.d.ts +12 -12
- package/dist/schemas/sdk-config.d.ts +171 -117
- package/dist/schemas/sdk-config.js +5 -5
- package/dist/schemas/sdk-errors-client.d.ts +1 -0
- package/dist/schemas/sdk-errors-client.js +7 -2
- package/dist/schemas/text-to-speech.d.ts +157 -13
- package/dist/schemas/text-to-speech.js +70 -7
- package/dist/schemas/transcription-config.d.ts +5 -1
- package/dist/schemas/transcription-config.js +2 -0
- package/dist/schemas/transcription.d.ts +3 -0
- package/dist/schemas/transcription.js +2 -1
- package/dist/schemas/translate.d.ts +6 -6
- package/dist/schemas/translate.js +2 -3
- package/dist/schemas/translation-config.d.ts +20 -20
- package/dist/server/bare/hyperswarm.js +3 -3
- package/dist/server/bare/ops/transcribe.js +46 -15
- package/dist/server/bare/plugins/bci-whispercpp-transcription/args.d.ts +3 -0
- package/dist/server/bare/plugins/bci-whispercpp-transcription/args.js +13 -0
- package/dist/server/bare/plugins/bci-whispercpp-transcription/plugin.d.ts +41 -0
- package/dist/server/bare/plugins/bci-whispercpp-transcription/plugin.js +6 -11
- package/dist/server/bare/plugins/bci-whispercpp-transcription/resolve-config.d.ts +75 -0
- package/dist/server/bare/plugins/bci-whispercpp-transcription/resolve-config.js +11 -0
- package/dist/server/bare/plugins/{onnx-ocr → ggml-ocr}/plugin.d.ts +16 -10
- package/dist/server/bare/plugins/{onnx-ocr → ggml-ocr}/plugin.js +25 -18
- package/dist/server/bare/plugins/index.d.ts +1 -1
- package/dist/server/bare/plugins/index.js +1 -1
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stats.d.ts +10 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stats.js +29 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.js +4 -22
- package/dist/server/bare/plugins/llamacpp-completion/plugin.d.ts +15 -7
- package/dist/server/bare/plugins/nmtcpp-translation/plugin.d.ts +7 -7
- package/dist/server/bare/plugins/parakeet-transcription/plugin.d.ts +2 -0
- package/dist/server/bare/plugins/sdcpp-generation/plugin.d.ts +10 -10
- package/dist/server/bare/plugins/tts-ggml/plugin.d.ts +14 -8
- package/dist/server/bare/plugins/tts-ggml/plugin.js +12 -0
- package/dist/server/bare/plugins/whispercpp-transcription/plugin.d.ts +3 -1
- package/dist/server/bare/registry/logging-stream-registry.d.ts +4 -2
- package/dist/server/bare/registry/logging-stream-registry.js +12 -7
- package/dist/server/bare/registry/model-config-utils.js +2 -2
- package/dist/server/bare/registry/registry-client.js +1 -1
- package/dist/server/bare/runtime-lifecycle.d.ts +9 -0
- package/dist/server/bare/runtime-lifecycle.js +24 -0
- package/dist/server/bare/types/addon-responses.d.ts +1 -0
- package/dist/server/env.js +7 -10
- package/dist/server/plugins/registry.js +2 -4
- package/dist/server/rpc/handlers/load-model/handler.js +2 -4
- package/dist/server/rpc/handlers/load-model/http.js +253 -51
- package/dist/server/rpc/handlers/logging-stream.js +4 -2
- package/dist/server/rpc/handlers/plugin-invoke.js +5 -16
- package/dist/server/utils/completion-normalizer.js +13 -4
- package/dist/server/utils/worker-lock.js +5 -5
- package/dist/server/worker-core.d.ts +1 -1
- package/dist/server/worker-core.js +28 -18
- package/dist/utils/errors-client.d.ts +3 -0
- package/dist/utils/errors-client.js +5 -0
- package/dist/utils/errors-server.d.ts +2 -0
- package/dist/utils/errors-server.js +3 -0
- package/dist/utils/zod-error.d.ts +3 -0
- package/dist/utils/zod-error.js +4 -0
- package/package.json +21 -24
- package/dist/polyfills/bare-globals.d.ts +0 -2
- package/dist/polyfills/bare-globals.js +0 -4
- package/dist/polyfills/bare-globals.noop.d.ts +0 -2
- package/dist/polyfills/bare-globals.noop.js +0 -1
- /package/dist/server/bare/plugins/{onnx-ocr → ggml-ocr}/ops/ocr-stream.d.ts +0 -0
- /package/dist/server/bare/plugins/{onnx-ocr → ggml-ocr}/ops/ocr-stream.js +0 -0
|
@@ -6,16 +6,16 @@ const systemPrompt = {
|
|
|
6
6
|
content: "You are a concise travel assistant. Keep track of the user's preferences across turns.",
|
|
7
7
|
};
|
|
8
8
|
function printStats(stats) {
|
|
9
|
-
console.log("\
|
|
9
|
+
console.log("\n▸ Performance Stats:", {
|
|
10
10
|
timeToFirstToken: stats?.timeToFirstToken,
|
|
11
11
|
tokensPerSecond: stats?.tokensPerSecond,
|
|
12
12
|
cacheTokens: stats?.cacheTokens,
|
|
13
13
|
});
|
|
14
14
|
}
|
|
15
15
|
async function runCachedCompletion(params) {
|
|
16
|
-
console.log(`\n
|
|
17
|
-
console.log(
|
|
18
|
-
console.log("AI Response:");
|
|
16
|
+
console.log(`\n▸ ${params.label}`);
|
|
17
|
+
console.log(`▸ Cache key: ${params.kvCache || "disabled"}`);
|
|
18
|
+
console.log("▸ AI Response:");
|
|
19
19
|
const result = completion({
|
|
20
20
|
modelId: params.modelId,
|
|
21
21
|
history: params.history,
|
|
@@ -33,18 +33,22 @@ async function runCachedCompletion(params) {
|
|
|
33
33
|
}
|
|
34
34
|
let modelId;
|
|
35
35
|
try {
|
|
36
|
-
console.log("Loading llama.cpp model with cache support...");
|
|
36
|
+
console.log("▸ Loading llama.cpp model with cache support...");
|
|
37
37
|
modelId = await loadModel({
|
|
38
38
|
modelSrc: LLAMA_3_2_1B_INST_Q4_0,
|
|
39
39
|
modelConfig: {
|
|
40
40
|
ctx_size: 4096,
|
|
41
41
|
verbosity: VERBOSITY.ERROR,
|
|
42
42
|
},
|
|
43
|
-
onProgress: (
|
|
44
|
-
|
|
43
|
+
onProgress: (p) => {
|
|
44
|
+
const mb = (n) => (n / 1e6).toFixed(1);
|
|
45
|
+
const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
|
|
46
|
+
process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
|
|
47
|
+
if (p.percentage >= 100)
|
|
48
|
+
process.stderr.write("\n");
|
|
45
49
|
},
|
|
46
50
|
});
|
|
47
|
-
console.log(
|
|
51
|
+
console.log(`▸ Model loaded successfully. Model ID: ${modelId}`);
|
|
48
52
|
await deleteCache({ kvCacheKey: sharedCacheKey });
|
|
49
53
|
await deleteCache({ kvCacheKey: isolatedCacheKey });
|
|
50
54
|
const firstUserMessage = {
|
|
@@ -83,10 +87,10 @@ try {
|
|
|
83
87
|
},
|
|
84
88
|
],
|
|
85
89
|
});
|
|
86
|
-
console.log("\
|
|
87
|
-
console.log(
|
|
88
|
-
console.log(
|
|
89
|
-
console.log('- Use deleteCache({ kvCacheKey: "your-session" }) to clear a saved session.');
|
|
90
|
+
console.log("\n▸ Summary:");
|
|
91
|
+
console.log(`▸ - Reusing "${sharedCacheKey}" keeps conversation context across turns.`);
|
|
92
|
+
console.log(`▸ - Switching to "${isolatedCacheKey}" starts a separate cache session.`);
|
|
93
|
+
console.log('▸ - Use deleteCache({ kvCacheKey: "your-session" }) to clear a saved session.');
|
|
90
94
|
await deleteCache({ kvCacheKey: sharedCacheKey });
|
|
91
95
|
await deleteCache({ kvCacheKey: isolatedCacheKey });
|
|
92
96
|
await unloadModel({ modelId, clearStorage: false });
|
|
@@ -98,9 +102,9 @@ catch (error) {
|
|
|
98
102
|
await unloadModel({ modelId, clearStorage: false });
|
|
99
103
|
}
|
|
100
104
|
catch (cleanupError) {
|
|
101
|
-
console.warn("
|
|
105
|
+
console.warn("✖ cleanup failed:", cleanupError);
|
|
102
106
|
}
|
|
103
107
|
}
|
|
104
|
-
console.error("
|
|
108
|
+
console.error("✖", error);
|
|
105
109
|
process.exit(1);
|
|
106
110
|
}
|
|
@@ -76,7 +76,7 @@ async function chatTurn({ modelId, kvCache, history, tools }) {
|
|
|
76
76
|
})();
|
|
77
77
|
const toolEventsTask = (async () => {
|
|
78
78
|
for await (const evt of result.toolCallStream) {
|
|
79
|
-
console.log(`\n
|
|
79
|
+
console.log(`\n▸ tool call: ${evt.call.name}(${JSON.stringify(evt.call.arguments)})`);
|
|
80
80
|
}
|
|
81
81
|
})();
|
|
82
82
|
await Promise.all([tokensTask, toolEventsTask]);
|
|
@@ -91,7 +91,7 @@ async function chatTurn({ modelId, kvCache, history, tools }) {
|
|
|
91
91
|
if (schema) {
|
|
92
92
|
const parsed = schema.safeParse(call.arguments);
|
|
93
93
|
if (!parsed.success) {
|
|
94
|
-
console.
|
|
94
|
+
console.log(`✖ validation failed for ${call.name}:`, parsed.error);
|
|
95
95
|
}
|
|
96
96
|
}
|
|
97
97
|
}
|
|
@@ -110,9 +110,15 @@ async function main() {
|
|
|
110
110
|
tools: true,
|
|
111
111
|
toolsMode: "dynamic",
|
|
112
112
|
},
|
|
113
|
-
onProgress: (
|
|
113
|
+
onProgress: (p) => {
|
|
114
|
+
const mb = (n) => (n / 1e6).toFixed(1);
|
|
115
|
+
const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
|
|
116
|
+
process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
|
|
117
|
+
if (p.percentage >= 100)
|
|
118
|
+
process.stderr.write("\n");
|
|
119
|
+
},
|
|
114
120
|
});
|
|
115
|
-
console.log(
|
|
121
|
+
console.log(`▸ Model loaded: ${modelId}`);
|
|
116
122
|
const kvCache = `dynamic-tools-${Date.now()}`;
|
|
117
123
|
const history = [
|
|
118
124
|
{
|
|
@@ -123,7 +129,7 @@ async function main() {
|
|
|
123
129
|
];
|
|
124
130
|
// Turn 1 — only weather tools available.
|
|
125
131
|
history.push({ role: "user", content: "What's the weather in Tokyo?" });
|
|
126
|
-
console.log("\n
|
|
132
|
+
console.log("\n▸ Turn 1 (tools=weather):\n");
|
|
127
133
|
await chatTurn({ modelId, kvCache, history, tools: weatherTools });
|
|
128
134
|
// Turn 2 — same session, swap to horoscope tools. Dynamic mode lets the
|
|
129
135
|
// model see a different tool set without invalidating the kv-cache.
|
|
@@ -131,16 +137,16 @@ async function main() {
|
|
|
131
137
|
role: "user",
|
|
132
138
|
content: "Now check my horoscope for Aquarius.",
|
|
133
139
|
});
|
|
134
|
-
console.log("\n\n
|
|
140
|
+
console.log("\n\n▸ Turn 2 (tools=horoscope):\n");
|
|
135
141
|
await chatTurn({ modelId, kvCache, history, tools: horoscopeTools });
|
|
136
142
|
// Turn 3 — swap to a parameterless tool to confirm empty-arg flows work.
|
|
137
143
|
history.push({ role: "user", content: "What's today's date?" });
|
|
138
|
-
console.log("\n\n
|
|
144
|
+
console.log("\n\n▸ Turn 3 (tools=date):\n");
|
|
139
145
|
await chatTurn({ modelId, kvCache, history, tools: dateTools });
|
|
140
|
-
console.log("\n\n
|
|
146
|
+
console.log("\n\n▸ Done.");
|
|
141
147
|
await unloadModel({ modelId, clearStorage: false });
|
|
142
148
|
}
|
|
143
149
|
main().catch((err) => {
|
|
144
|
-
console.error("
|
|
150
|
+
console.error("✖", err);
|
|
145
151
|
process.exit(1);
|
|
146
152
|
});
|
|
@@ -2,11 +2,11 @@ import { completion, loadModel, unloadModel } from "@qvac/sdk";
|
|
|
2
2
|
// Get GGUF file path from command line arguments
|
|
3
3
|
const ggufPath = process.argv[2];
|
|
4
4
|
if (!ggufPath) {
|
|
5
|
-
console.error("
|
|
6
|
-
console.error("Usage: bun run examples/llamacpp-filesystem.ts <path-to-gguf-file>");
|
|
5
|
+
console.error("✖ Please provide the path to a GGUF file as the first argument");
|
|
6
|
+
console.error("✖ Usage: bun run examples/llamacpp-filesystem.ts <path-to-gguf-file>");
|
|
7
7
|
process.exit(1);
|
|
8
8
|
}
|
|
9
|
-
console.log(
|
|
9
|
+
console.log(`▸ Loading GGUF model from: ${ggufPath}`);
|
|
10
10
|
try {
|
|
11
11
|
// Load model from provided file path
|
|
12
12
|
const modelId = await loadModel({
|
|
@@ -15,14 +15,20 @@ try {
|
|
|
15
15
|
modelConfig: {
|
|
16
16
|
ctx_size: 4096,
|
|
17
17
|
},
|
|
18
|
-
onProgress: (
|
|
18
|
+
onProgress: (p) => {
|
|
19
|
+
const mb = (n) => (n / 1e6).toFixed(1);
|
|
20
|
+
const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
|
|
21
|
+
process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
|
|
22
|
+
if (p.percentage >= 100)
|
|
23
|
+
process.stderr.write("\n");
|
|
24
|
+
},
|
|
19
25
|
});
|
|
20
|
-
console.log(
|
|
26
|
+
console.log(`▸ Model loaded successfully! Model ID: ${modelId}`);
|
|
21
27
|
// Create conversation history
|
|
22
28
|
const history = [
|
|
23
29
|
{ role: "user", content: "Explain Bitcoin in 3 key points" },
|
|
24
30
|
];
|
|
25
|
-
console.log("\n
|
|
31
|
+
console.log("\n▸ AI Response:");
|
|
26
32
|
process.stdout.write(""); // Start response on new line
|
|
27
33
|
// Stream completion
|
|
28
34
|
const result = completion({ modelId, history, stream: true });
|
|
@@ -30,11 +36,11 @@ try {
|
|
|
30
36
|
process.stdout.write(token);
|
|
31
37
|
}
|
|
32
38
|
const stats = await result.stats;
|
|
33
|
-
console.log("\n
|
|
34
|
-
console.log("\n\n
|
|
39
|
+
console.log("\n▸ Performance Stats:", stats);
|
|
40
|
+
console.log("\n\n▸ Completed!");
|
|
35
41
|
await unloadModel({ modelId, clearStorage: false });
|
|
36
42
|
}
|
|
37
43
|
catch (error) {
|
|
38
|
-
console.error("
|
|
44
|
+
console.error("✖", error);
|
|
39
45
|
process.exit(1);
|
|
40
46
|
}
|
|
@@ -2,7 +2,7 @@ import { completion, loadModel, unloadModel } from "@qvac/sdk";
|
|
|
2
2
|
// Get HTTP URL from command line arguments or use default HuggingFace URL
|
|
3
3
|
const httpUrl = process.argv[2] ||
|
|
4
4
|
"https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf";
|
|
5
|
-
console.log(
|
|
5
|
+
console.log(`▸ Loading GGUF model from HTTP: ${httpUrl}`);
|
|
6
6
|
try {
|
|
7
7
|
// Load model from HTTP URL
|
|
8
8
|
const modelId = await loadModel({
|
|
@@ -11,18 +11,20 @@ try {
|
|
|
11
11
|
modelConfig: {
|
|
12
12
|
ctx_size: 4096,
|
|
13
13
|
},
|
|
14
|
-
onProgress: (
|
|
15
|
-
const
|
|
16
|
-
const
|
|
17
|
-
|
|
14
|
+
onProgress: (p) => {
|
|
15
|
+
const mb = (n) => (n / 1e6).toFixed(1);
|
|
16
|
+
const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
|
|
17
|
+
process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
|
|
18
|
+
if (p.percentage >= 100)
|
|
19
|
+
process.stderr.write("\n");
|
|
18
20
|
},
|
|
19
21
|
});
|
|
20
|
-
console.log(
|
|
22
|
+
console.log(`▸ Model loaded successfully! Model ID: ${modelId}`);
|
|
21
23
|
// Create conversation history
|
|
22
24
|
const history = [
|
|
23
25
|
{ role: "user", content: "Explain quantum computing in 3 key points" },
|
|
24
26
|
];
|
|
25
|
-
console.log("\n
|
|
27
|
+
console.log("\n▸ AI Response:");
|
|
26
28
|
process.stdout.write(""); // Start response on new line
|
|
27
29
|
// Stream completion
|
|
28
30
|
const result = completion({ modelId, history, stream: true });
|
|
@@ -30,11 +32,11 @@ try {
|
|
|
30
32
|
process.stdout.write(token);
|
|
31
33
|
}
|
|
32
34
|
const stats = await result.stats;
|
|
33
|
-
console.log("\n
|
|
34
|
-
console.log("\n\n
|
|
35
|
+
console.log("\n▸ Performance Stats:", stats);
|
|
36
|
+
console.log("\n\n▸ Completed!");
|
|
35
37
|
await unloadModel({ modelId, clearStorage: false });
|
|
36
38
|
}
|
|
37
39
|
catch (error) {
|
|
38
|
-
console.error("
|
|
40
|
+
console.error("✖", error);
|
|
39
41
|
process.exit(1);
|
|
40
42
|
}
|
|
@@ -28,11 +28,11 @@ try {
|
|
|
28
28
|
onProgress: (progress) => {
|
|
29
29
|
if (progress.shardInfo) {
|
|
30
30
|
const { shardInfo } = progress;
|
|
31
|
-
console.log(
|
|
31
|
+
console.log(`▸ Downloading ${shardInfo.shardName} (${shardInfo.currentShard}/${shardInfo.totalShards}) ` +
|
|
32
32
|
`— overall: ${shardInfo.overallPercentage.toFixed(1)}%`);
|
|
33
33
|
}
|
|
34
34
|
else {
|
|
35
|
-
console.log(
|
|
35
|
+
console.log(`▸ Downloading: ${progress.percentage.toFixed(1)}%`);
|
|
36
36
|
}
|
|
37
37
|
},
|
|
38
38
|
});
|
|
@@ -47,11 +47,11 @@ try {
|
|
|
47
47
|
process.stdout.write(token);
|
|
48
48
|
}
|
|
49
49
|
const stats = await result.stats;
|
|
50
|
-
console.log("\n\
|
|
50
|
+
console.log("\n\n▸ Stats:", stats);
|
|
51
51
|
await unloadModel({ modelId, clearStorage: false });
|
|
52
52
|
process.exit(0);
|
|
53
53
|
}
|
|
54
54
|
catch (error) {
|
|
55
|
-
console.error("
|
|
55
|
+
console.error("✖", error);
|
|
56
56
|
process.exit(1);
|
|
57
57
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { completion, loadModel, SMOLVLM2_500M_MULTIMODAL_Q8_0, MMPROJ_SMOLVLM2_500M_MULTIMODAL_Q8_0, unloadModel, } from "@qvac/sdk";
|
|
2
2
|
if (process.argv.length < 3) {
|
|
3
|
-
console.error(
|
|
3
|
+
console.error(`▸ Specify an image file path as the first argument and a second image file path as the second (optional) argument`);
|
|
4
4
|
process.exit(1);
|
|
5
5
|
}
|
|
6
6
|
try {
|
|
@@ -13,8 +13,12 @@ try {
|
|
|
13
13
|
ctx_size: 1024,
|
|
14
14
|
projectionModelSrc: MMPROJ_SMOLVLM2_500M_MULTIMODAL_Q8_0,
|
|
15
15
|
},
|
|
16
|
-
onProgress: (
|
|
17
|
-
|
|
16
|
+
onProgress: (p) => {
|
|
17
|
+
const mb = (n) => (n / 1e6).toFixed(1);
|
|
18
|
+
const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
|
|
19
|
+
process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
|
|
20
|
+
if (p.percentage >= 100)
|
|
21
|
+
process.stderr.write("\n");
|
|
18
22
|
},
|
|
19
23
|
});
|
|
20
24
|
//Using one particular media
|
|
@@ -30,11 +34,11 @@ try {
|
|
|
30
34
|
process.stdout.write(token);
|
|
31
35
|
}
|
|
32
36
|
const stats = await result.stats;
|
|
33
|
-
console.log("\n
|
|
34
|
-
console.log("--------------------------------");
|
|
37
|
+
console.log("\n▸ Performance Stats:", stats);
|
|
38
|
+
console.log("▸ --------------------------------");
|
|
35
39
|
//Using multiple media
|
|
36
40
|
if (process.argv.length < 4) {
|
|
37
|
-
console.log(
|
|
41
|
+
console.log(`▸ Only one image provided, terminating`);
|
|
38
42
|
process.exit(0);
|
|
39
43
|
}
|
|
40
44
|
const imageFilePath2 = process.argv[3];
|
|
@@ -50,11 +54,11 @@ try {
|
|
|
50
54
|
process.stdout.write(token);
|
|
51
55
|
}
|
|
52
56
|
const stats2 = await result2.stats;
|
|
53
|
-
console.log("\n
|
|
54
|
-
console.log("--------------------------------");
|
|
57
|
+
console.log("\n▸ Performance Stats:", stats2);
|
|
58
|
+
console.log("▸ --------------------------------");
|
|
55
59
|
await unloadModel({ modelId, clearStorage: false });
|
|
56
60
|
}
|
|
57
61
|
catch (error) {
|
|
58
|
-
console.error("
|
|
62
|
+
console.error("✖", error);
|
|
59
63
|
process.exit(1);
|
|
60
64
|
}
|
|
@@ -3,8 +3,12 @@ try {
|
|
|
3
3
|
// First just cache the model
|
|
4
4
|
await downloadAsset({
|
|
5
5
|
assetSrc: LLAMA_3_2_1B_INST_Q4_0,
|
|
6
|
-
onProgress: (
|
|
7
|
-
|
|
6
|
+
onProgress: (p) => {
|
|
7
|
+
const mb = (n) => (n / 1e6).toFixed(1);
|
|
8
|
+
const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
|
|
9
|
+
process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
|
|
10
|
+
if (p.percentage >= 100)
|
|
11
|
+
process.stderr.write("\n");
|
|
8
12
|
},
|
|
9
13
|
});
|
|
10
14
|
// Then load it in memory from cache
|
|
@@ -27,11 +31,11 @@ try {
|
|
|
27
31
|
process.stdout.write(token);
|
|
28
32
|
}
|
|
29
33
|
const stats = await result.stats;
|
|
30
|
-
console.log("\n
|
|
34
|
+
console.log("\n▸ Performance Stats:", stats);
|
|
31
35
|
// Change `clearStorage: true` to delete cached model files
|
|
32
36
|
await unloadModel({ modelId, clearStorage: false });
|
|
33
37
|
}
|
|
34
38
|
catch (error) {
|
|
35
|
-
console.error("
|
|
39
|
+
console.error("✖", error);
|
|
36
40
|
process.exit(1);
|
|
37
41
|
}
|
|
@@ -20,13 +20,13 @@ try {
|
|
|
20
20
|
if (progress.shardInfo) {
|
|
21
21
|
// For pattern-based or Hyperdrive shards
|
|
22
22
|
const { shardInfo } = progress;
|
|
23
|
-
console.log(
|
|
23
|
+
console.log(`▸ Downloading ${shardInfo.shardName} (${shardInfo.currentShard}/${shardInfo.totalShards})\n` +
|
|
24
24
|
` File: ${progress.percentage.toFixed(1)}% (${(progress.downloaded / 1024 / 1024).toFixed(2)}MB / ${(progress.total / 1024 / 1024).toFixed(2)}MB)\n` +
|
|
25
25
|
` Overall: ${shardInfo.overallPercentage.toFixed(1)}% (${(shardInfo.overallDownloaded / 1024 / 1024).toFixed(2)}MB / ${(shardInfo.overallTotal / 1024 / 1024).toFixed(2)}MB)`);
|
|
26
26
|
}
|
|
27
27
|
else {
|
|
28
28
|
// For archive-based shards
|
|
29
|
-
console.log(
|
|
29
|
+
console.log(`▸ Progress: ${progress.percentage.toFixed(1)}% ` +
|
|
30
30
|
`(${(progress.downloaded / 1024 / 1024).toFixed(2)}MB / ${(progress.total / 1024 / 1024).toFixed(2)}MB)`);
|
|
31
31
|
}
|
|
32
32
|
},
|
|
@@ -38,16 +38,16 @@ try {
|
|
|
38
38
|
},
|
|
39
39
|
];
|
|
40
40
|
const result = completion({ modelId, history, stream: true });
|
|
41
|
-
console.log("\n
|
|
41
|
+
console.log("\n▸ Model response:");
|
|
42
42
|
for await (const token of result.tokenStream) {
|
|
43
43
|
process.stdout.write(token);
|
|
44
44
|
}
|
|
45
45
|
const stats = await result.stats;
|
|
46
|
-
console.log("\n\n
|
|
46
|
+
console.log("\n\n▸ Performance Stats:", stats);
|
|
47
47
|
await unloadModel({ modelId, clearStorage: false });
|
|
48
48
|
process.exit(0);
|
|
49
49
|
}
|
|
50
50
|
catch (error) {
|
|
51
|
-
console.error("
|
|
51
|
+
console.error("✖", error);
|
|
52
52
|
process.exit(1);
|
|
53
53
|
}
|
|
@@ -58,25 +58,31 @@ async function runToContent(run) {
|
|
|
58
58
|
try {
|
|
59
59
|
const modelId = await loadModel({
|
|
60
60
|
modelSrc: QWEN3_600M_INST_Q4,
|
|
61
|
-
onProgress: (p) =>
|
|
61
|
+
onProgress: (p) => {
|
|
62
|
+
const mb = (n) => (n / 1e6).toFixed(1);
|
|
63
|
+
const line = `▸ Downloading ${p.percentage.toFixed(0)}% (${mb(p.downloaded)}/${mb(p.total)} MB)`;
|
|
64
|
+
process.stderr.write(process.stderr.isTTY ? `\r${line}` : `${line}\n`);
|
|
65
|
+
if (p.percentage >= 100)
|
|
66
|
+
process.stderr.write("\n");
|
|
67
|
+
},
|
|
62
68
|
});
|
|
63
|
-
console.log(`\
|
|
64
|
-
console.log("--- 1. responseFormat: text (baseline, free-form) ---");
|
|
69
|
+
console.log(`\n▸ Model loaded: ${modelId}\n`);
|
|
70
|
+
console.log("▸ --- 1. responseFormat: text (baseline, free-form) ---");
|
|
65
71
|
await runToContent(completion({
|
|
66
72
|
modelId,
|
|
67
73
|
history: HISTORY,
|
|
68
74
|
stream: true,
|
|
69
75
|
responseFormat: { type: "text" },
|
|
70
76
|
}));
|
|
71
|
-
console.log("\n--- 2. responseFormat: json_object (any valid JSON) ---");
|
|
77
|
+
console.log("\n▸ --- 2. responseFormat: json_object (any valid JSON) ---");
|
|
72
78
|
const jsonObjectOut = await runToContent(completion({
|
|
73
79
|
modelId,
|
|
74
80
|
history: JSON_OBJECT_HISTORY,
|
|
75
81
|
stream: true,
|
|
76
82
|
responseFormat: { type: "json_object" },
|
|
77
83
|
}));
|
|
78
|
-
console.log("parsed:", JSON.parse(jsonObjectOut.trim()));
|
|
79
|
-
console.log("\n--- 3. responseFormat: json_schema (strict shape) ---");
|
|
84
|
+
console.log("▸ parsed:", JSON.parse(jsonObjectOut.trim()));
|
|
85
|
+
console.log("\n▸ --- 3. responseFormat: json_schema (strict shape) ---");
|
|
80
86
|
const jsonSchemaOut = await runToContent(completion({
|
|
81
87
|
modelId,
|
|
82
88
|
history: HISTORY,
|
|
@@ -92,14 +98,14 @@ try {
|
|
|
92
98
|
// Output is guaranteed schema-valid JSON: object with exactly
|
|
93
99
|
// {name: string, age: integer, occupation: string}, no extras.
|
|
94
100
|
const parsed = JSON.parse(jsonSchemaOut.trim());
|
|
95
|
-
console.log("parsed:", parsed);
|
|
96
|
-
console.log("schema-valid:", typeof parsed.name === "string" &&
|
|
101
|
+
console.log("▸ parsed:", parsed);
|
|
102
|
+
console.log("▸ schema-valid:", typeof parsed.name === "string" &&
|
|
97
103
|
Number.isInteger(parsed.age) &&
|
|
98
104
|
typeof parsed.occupation === "string" &&
|
|
99
105
|
Object.keys(parsed).sort().join(",") === "age,name,occupation");
|
|
100
106
|
await unloadModel({ modelId });
|
|
101
107
|
}
|
|
102
108
|
catch (error) {
|
|
103
|
-
console.error("
|
|
109
|
+
console.error("✖", error);
|
|
104
110
|
process.exit(1);
|
|
105
111
|
}
|
|
@@ -47,10 +47,10 @@ try {
|
|
|
47
47
|
await unloadModel({ modelId });
|
|
48
48
|
logger.info("Model unloaded successfully");
|
|
49
49
|
// Show user where logs were saved
|
|
50
|
-
console.log(
|
|
51
|
-
console.log("
|
|
50
|
+
console.log(`▸ Logs saved to ${logFile}`);
|
|
51
|
+
console.log("▸ Inspect it to see detailed model operation logs");
|
|
52
52
|
}
|
|
53
53
|
catch (error) {
|
|
54
|
-
console.error("
|
|
54
|
+
console.error("✖", error);
|
|
55
55
|
process.exit(1);
|
|
56
56
|
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// loggerConsoleOutput: false in this config disables the default console
|
|
2
|
+
// transport on both the client and the worker, so the only logs printed are
|
|
3
|
+
// the ones our handler below receives.
|
|
4
|
+
const configDir = import.meta.dirname ?? process.cwd();
|
|
5
|
+
process.env["QVAC_CONFIG_PATH"] = `${configDir}/config/logging/logging.config.json`;
|
|
6
|
+
const { loadModel, completion, unloadModel, subscribeServerLogs, LLAMA_3_2_1B_INST_Q4_0 } = await import("@qvac/sdk");
|
|
7
|
+
try {
|
|
8
|
+
console.log("▸ Starting global logging demo...\n");
|
|
9
|
+
// One subscription captures every server-side log (SDK, models, RAG, …)
|
|
10
|
+
// without having to know any stream IDs ahead of time.
|
|
11
|
+
const unsubscribe = subscribeServerLogs((log) => {
|
|
12
|
+
console.log(`[SDK] [${log.level.toUpperCase()}] [${log.namespace}] ${log.message}`);
|
|
13
|
+
});
|
|
14
|
+
const modelId = await loadModel({
|
|
15
|
+
modelSrc: LLAMA_3_2_1B_INST_Q4_0,
|
|
16
|
+
modelConfig: { ctx_size: 2048 },
|
|
17
|
+
});
|
|
18
|
+
const result = completion({
|
|
19
|
+
modelId,
|
|
20
|
+
history: [{ role: "user", content: "Count from 1 to 5." }],
|
|
21
|
+
stream: true,
|
|
22
|
+
});
|
|
23
|
+
console.log("▸ Response:\n");
|
|
24
|
+
for await (const token of result.tokenStream) {
|
|
25
|
+
process.stdout.write(token);
|
|
26
|
+
}
|
|
27
|
+
await unloadModel({ modelId, clearStorage: false });
|
|
28
|
+
unsubscribe();
|
|
29
|
+
}
|
|
30
|
+
catch (error) {
|
|
31
|
+
console.error("✖", error);
|
|
32
|
+
process.exit(1);
|
|
33
|
+
}
|
|
34
|
+
export {};
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { loadModel, completion, unloadModel, loggingStream, SDK_LOG_ID, LLAMA_3_2_1B_INST_Q4_0, GTE_LARGE_FP16, VERBOSITY, embed, } from "@qvac/sdk";
|
|
2
2
|
try {
|
|
3
|
-
console.log("
|
|
3
|
+
console.log("▸ Streaming SDK and model logs");
|
|
4
4
|
// Note: To configure logging (level and console output), use config file:
|
|
5
5
|
// { "loggerLevel": "debug", "loggerConsoleOutput": false } in qvac.config.json/js/ts
|
|
6
6
|
// Subscribe to SDK server logs in background
|
|
7
|
-
console.log("
|
|
7
|
+
console.log("▸ Subscribing to SDK server logs");
|
|
8
8
|
(async () => {
|
|
9
9
|
for await (const log of loggingStream({ id: SDK_LOG_ID })) {
|
|
10
10
|
console.log(`[SDK] [${log.level.toUpperCase()}] [${log.namespace}] ${log.message}`);
|
|
@@ -13,7 +13,7 @@ try {
|
|
|
13
13
|
// Stream terminated - normal on shutdown
|
|
14
14
|
});
|
|
15
15
|
// Load models
|
|
16
|
-
console.log("
|
|
16
|
+
console.log("▸ Loading models");
|
|
17
17
|
const llmModelId = await loadModel({
|
|
18
18
|
modelSrc: LLAMA_3_2_1B_INST_Q4_0,
|
|
19
19
|
modelConfig: {
|
|
@@ -25,7 +25,7 @@ try {
|
|
|
25
25
|
const embedModelId = await loadModel({
|
|
26
26
|
modelSrc: GTE_LARGE_FP16,
|
|
27
27
|
});
|
|
28
|
-
console.log("
|
|
28
|
+
console.log("▸ Subscribing to model logs");
|
|
29
29
|
(async () => {
|
|
30
30
|
for await (const log of loggingStream({ id: llmModelId })) {
|
|
31
31
|
const timestamp = new Date(log.timestamp).toISOString();
|
|
@@ -54,20 +54,17 @@ try {
|
|
|
54
54
|
modelId: embedModelId,
|
|
55
55
|
text: messages[0]?.content ?? "Hello, world!",
|
|
56
56
|
});
|
|
57
|
-
console.log("
|
|
57
|
+
console.log("▸ Response");
|
|
58
58
|
for await (const token of result.tokenStream) {
|
|
59
59
|
process.stdout.write(token);
|
|
60
60
|
}
|
|
61
61
|
console.log("Embedding (first 20 elements)", embedding.slice(0, 20));
|
|
62
62
|
console.log("Embeddings length", embedding.length);
|
|
63
|
-
console.log("
|
|
64
|
-
" - [SDK] = SDK server operations\n" +
|
|
65
|
-
" - [LLM] = LLM model inference logs\n" +
|
|
66
|
-
" - [EMBED] = Embedding model logs\n");
|
|
63
|
+
console.log("▸ Three streams running: [SDK] server, [LLM] inference, [EMBED] embedding");
|
|
67
64
|
await unloadModel({ modelId: llmModelId, clearStorage: false });
|
|
68
65
|
await unloadModel({ modelId: embedModelId, clearStorage: false });
|
|
69
66
|
}
|
|
70
67
|
catch (error) {
|
|
71
|
-
console.error("
|
|
68
|
+
console.error("✖", error);
|
|
72
69
|
process.exit(1);
|
|
73
70
|
}
|