@qvac/sdk 0.9.2 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +42 -95
- package/dist/client/api/cancel.d.ts +1 -1
- package/dist/client/api/cancel.js +1 -1
- package/dist/client/api/completion-stream.d.ts +51 -12
- package/dist/client/api/completion-stream.js +129 -56
- package/dist/client/api/delete-cache.d.ts +1 -0
- package/dist/client/api/delete-cache.js +1 -0
- package/dist/client/api/diffusion.d.ts +17 -2
- package/dist/client/api/diffusion.js +21 -4
- package/dist/client/api/embed.d.ts +4 -0
- package/dist/client/api/finetune.d.ts +21 -0
- package/dist/client/api/get-loaded-model-info.d.ts +25 -0
- package/dist/client/api/get-loaded-model-info.js +36 -0
- package/dist/client/api/get-model-info.d.ts +9 -1
- package/dist/client/api/get-model-info.js +8 -0
- package/dist/client/api/heartbeat.d.ts +2 -2
- package/dist/client/api/heartbeat.js +2 -2
- package/dist/client/api/index.d.ts +3 -1
- package/dist/client/api/index.js +3 -1
- package/dist/client/api/invoke-plugin.d.ts +16 -0
- package/dist/client/api/invoke-plugin.js +16 -0
- package/dist/client/api/load-model.d.ts +36 -3
- package/dist/client/api/load-model.js +31 -10
- package/dist/client/api/provide.d.ts +11 -5
- package/dist/client/api/provide.js +11 -6
- package/dist/client/api/rag.d.ts +9 -0
- package/dist/client/api/rag.js +9 -0
- package/dist/client/api/registry.d.ts +26 -0
- package/dist/client/api/registry.js +26 -0
- package/dist/client/api/resume.d.ts +20 -4
- package/dist/client/api/resume.js +20 -4
- package/dist/client/api/state.d.ts +23 -0
- package/dist/client/api/state.js +29 -0
- package/dist/client/api/stop-provider.d.ts +11 -5
- package/dist/client/api/stop-provider.js +11 -5
- package/dist/client/api/suspend.d.ts +21 -3
- package/dist/client/api/suspend.js +21 -3
- package/dist/client/api/text-to-speech.d.ts +64 -6
- package/dist/client/api/text-to-speech.js +423 -47
- package/dist/client/api/transcribe.d.ts +29 -4
- package/dist/client/api/transcribe.js +86 -28
- package/dist/client/api/translate.d.ts +1 -0
- package/dist/client/api/translate.js +1 -0
- package/dist/client/rpc/bare-client.js +22 -1
- package/dist/client/rpc/expo-rpc-client.d.ts +1 -1
- package/dist/client/rpc/expo-rpc-client.js +96 -4
- package/dist/client/rpc/node-rpc-client.js +36 -0
- package/dist/examples/audio/mic-input.d.ts +9 -0
- package/dist/examples/audio/mic-input.js +111 -0
- package/dist/examples/completion-events.d.ts +18 -0
- package/dist/examples/completion-events.js +79 -0
- package/dist/examples/delegated-inference/composite.js +12 -18
- package/dist/examples/delegated-inference/consumer-profiled.js +4 -11
- package/dist/examples/delegated-inference/consumer.js +8 -13
- package/dist/examples/delegated-inference/provider.js +11 -16
- package/dist/examples/diffusion-flux2-klein-img2img.d.ts +2 -0
- package/dist/examples/diffusion-flux2-klein-img2img.js +60 -0
- package/dist/examples/diffusion-flux2-klein.js +1 -0
- package/dist/examples/diffusion-img2img.d.ts +2 -0
- package/dist/examples/diffusion-img2img.js +49 -0
- package/dist/examples/diffusion-txt2img.js +2 -1
- package/dist/examples/kv-cache-example.js +11 -12
- package/dist/examples/llamacpp-cache.d.ts +2 -0
- package/dist/examples/llamacpp-cache.js +107 -0
- package/dist/examples/llamacpp-dynamic-tools.d.ts +2 -0
- package/dist/examples/llamacpp-dynamic-tools.js +144 -0
- package/dist/examples/llamacpp-structured-output.d.ts +2 -0
- package/dist/examples/llamacpp-structured-output.js +106 -0
- package/dist/examples/plugins.js +4 -4
- package/dist/examples/suspend-resume.js +21 -4
- package/dist/examples/{llamacpp-native-tools.js → tools/llamacpp-native-tools.js} +6 -55
- package/dist/examples/tools/llamacpp-tools-harmony-multiturn.d.ts +2 -0
- package/dist/examples/tools/llamacpp-tools-harmony-multiturn.js +81 -0
- package/dist/examples/tools/llamacpp-tools-pythonic.d.ts +2 -0
- package/dist/examples/tools/llamacpp-tools-pythonic.js +50 -0
- package/dist/examples/tools/shared.d.ts +21 -0
- package/dist/examples/tools/shared.js +42 -0
- package/dist/examples/transcription/parakeet-microphone-record.js +6 -33
- package/dist/examples/transcription/whispercpp-filesystem-streaming.js +7 -5
- package/dist/examples/transcription/whispercpp-filesystem.js +12 -3
- package/dist/examples/transcription/whispercpp-microphone-record.js +6 -29
- package/dist/examples/translation/translation-stream.js +5 -5
- package/dist/examples/tts/llm-to-tts-streaming.d.ts +16 -0
- package/dist/examples/tts/llm-to-tts-streaming.js +143 -0
- package/dist/examples/tts/supertonic-multilingual.js +3 -3
- package/dist/examples/tts/supertonic.js +3 -3
- package/dist/examples/tts/utils.d.ts +11 -1
- package/dist/examples/tts/utils.js +70 -17
- package/dist/examples/voice-assistant/voice-assistant.d.ts +2 -0
- package/dist/examples/voice-assistant/voice-assistant.js +207 -0
- package/dist/expo/plugins/withMobileBundle.js +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.js +2 -2
- package/dist/logging/logger.d.ts +11 -0
- package/dist/logging/logger.js +11 -0
- package/dist/logging/types.d.ts +18 -0
- package/dist/models/registry/models.d.ts +6973 -7457
- package/dist/models/registry/models.js +6889 -6855
- package/dist/models/update-models/codegen.js +84 -23
- package/dist/models/update-models/companions.d.ts +18 -0
- package/dist/models/update-models/companions.js +175 -0
- package/dist/models/update-models/naming.js +0 -25
- package/dist/models/update-models/registry.d.ts +1 -0
- package/dist/models/update-models/registry.js +23 -3
- package/dist/models/update-models/types.d.ts +37 -9
- package/dist/profiling/index.d.ts +49 -0
- package/dist/profiling/index.js +49 -0
- package/dist/schemas/cancel.d.ts +6 -2
- package/dist/schemas/cancel.js +14 -7
- package/dist/schemas/common.d.ts +354 -271
- package/dist/schemas/common.js +26 -5
- package/dist/schemas/completion-event.d.ts +225 -0
- package/dist/schemas/completion-event.js +79 -0
- package/dist/schemas/completion-stream.d.ts +126 -31
- package/dist/schemas/completion-stream.js +138 -36
- package/dist/schemas/delegate.d.ts +0 -2
- package/dist/schemas/delegate.js +25 -6
- package/dist/schemas/download-asset.d.ts +5 -5
- package/dist/schemas/embed.js +21 -9
- package/dist/schemas/engine-addon-map.d.ts +11 -1
- package/dist/schemas/engine-addon-map.js +2 -2
- package/dist/schemas/get-loaded-model-info.d.ts +52 -0
- package/dist/schemas/get-loaded-model-info.js +44 -0
- package/dist/schemas/get-model-info.d.ts +2 -2
- package/dist/schemas/get-model-info.js +103 -32
- package/dist/schemas/heartbeat.d.ts +0 -1
- package/dist/schemas/index.d.ts +4 -1
- package/dist/schemas/index.js +4 -1
- package/dist/schemas/llamacpp-config.d.ts +14 -4
- package/dist/schemas/llamacpp-config.js +5 -0
- package/dist/schemas/load-model.d.ts +3233 -1260
- package/dist/schemas/load-model.js +23 -16
- package/dist/schemas/model-src-utils.d.ts +7 -6
- package/dist/schemas/model-src-utils.js +27 -1
- package/dist/schemas/model-types.d.ts +4 -4
- package/dist/schemas/ocr.d.ts +1 -1
- package/dist/schemas/plugin.d.ts +27 -3
- package/dist/schemas/plugin.js +27 -2
- package/dist/schemas/profiling.js +11 -3
- package/dist/schemas/provide.d.ts +1 -3
- package/dist/schemas/provide.js +0 -3
- package/dist/schemas/rag.d.ts +4 -4
- package/dist/schemas/registry.d.ts +10 -10
- package/dist/schemas/registry.js +36 -14
- package/dist/schemas/reload-config.d.ts +73 -7
- package/dist/schemas/sdcpp-config.d.ts +16 -8
- package/dist/schemas/sdcpp-config.js +118 -25
- package/dist/schemas/sdk-config.d.ts +116 -90
- package/dist/schemas/sdk-config.js +16 -0
- package/dist/schemas/sdk-errors-client.d.ts +3 -0
- package/dist/schemas/sdk-errors-client.js +16 -1
- package/dist/schemas/sdk-errors-server.d.ts +3 -0
- package/dist/schemas/sdk-errors-server.js +23 -0
- package/dist/schemas/state.d.ts +23 -0
- package/dist/schemas/state.js +14 -0
- package/dist/schemas/stop-provide.d.ts +0 -5
- package/dist/schemas/stop-provide.js +2 -6
- package/dist/schemas/text-to-speech.d.ts +93 -27
- package/dist/schemas/text-to-speech.js +27 -0
- package/dist/schemas/tools.d.ts +16 -0
- package/dist/schemas/tools.js +15 -0
- package/dist/schemas/transcription-config.d.ts +8 -8
- package/dist/schemas/transcription.d.ts +33 -0
- package/dist/schemas/transcription.js +9 -0
- package/dist/schemas/translate.js +59 -19
- package/dist/schemas/translation-config.d.ts +157 -133
- package/dist/schemas/translation-config.js +34 -11
- package/dist/server/bare/delegate-rpc-client.d.ts +1 -1
- package/dist/server/bare/delegate-rpc-client.js +131 -122
- package/dist/server/bare/hyperswarm.d.ts +2 -2
- package/dist/server/bare/hyperswarm.js +12 -7
- package/dist/server/bare/ops/cancel.js +7 -0
- package/dist/server/bare/ops/kv-cache-utils.d.ts +7 -10
- package/dist/server/bare/ops/kv-cache-utils.js +31 -40
- package/dist/server/bare/ops/load-model.js +0 -1
- package/dist/server/bare/ops/transcribe.d.ts +11 -4
- package/dist/server/bare/ops/transcribe.js +25 -4
- package/dist/server/bare/ops/translate.js +42 -4
- package/dist/server/bare/ops/unload-model.js +21 -19
- package/dist/server/bare/plugins/llamacpp-completion/ops/cache-logger.js +1 -2
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.d.ts +13 -8
- package/dist/server/bare/plugins/llamacpp-completion/ops/completion-stream.js +257 -85
- package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-state.d.ts +101 -0
- package/dist/server/bare/plugins/llamacpp-completion/ops/kv-cache-state.js +135 -0
- package/dist/server/bare/plugins/llamacpp-completion/plugin.d.ts +80 -22
- package/dist/server/bare/plugins/llamacpp-completion/plugin.js +57 -32
- package/dist/server/bare/plugins/llamacpp-embedding/plugin.js +10 -16
- package/dist/server/bare/plugins/nmtcpp-translation/plugin.d.ts +69 -44
- package/dist/server/bare/plugins/nmtcpp-translation/plugin.js +54 -116
- package/dist/server/bare/plugins/nmtcpp-translation/resolve-vocab.d.ts +32 -0
- package/dist/server/bare/plugins/nmtcpp-translation/resolve-vocab.js +117 -0
- package/dist/server/bare/plugins/onnx-ocr/plugin.d.ts +1 -1
- package/dist/server/bare/plugins/onnx-ocr/plugin.js +1 -2
- package/dist/server/bare/plugins/onnx-tts/ops/text-to-speech-stream.d.ts +7 -0
- package/dist/server/bare/plugins/onnx-tts/ops/text-to-speech-stream.js +116 -0
- package/dist/server/bare/plugins/onnx-tts/ops/text-to-speech.d.ts +2 -3
- package/dist/server/bare/plugins/onnx-tts/ops/text-to-speech.js +71 -9
- package/dist/server/bare/plugins/onnx-tts/plugin.d.ts +41 -13
- package/dist/server/bare/plugins/onnx-tts/plugin.js +48 -3
- package/dist/server/bare/plugins/parakeet-transcription/plugin.d.ts +15 -7
- package/dist/server/bare/plugins/parakeet-transcription/plugin.js +5 -6
- package/dist/server/bare/plugins/sdcpp-generation/ops/diffusion.js +6 -0
- package/dist/server/bare/plugins/sdcpp-generation/plugin.d.ts +8 -5
- package/dist/server/bare/plugins/sdcpp-generation/plugin.js +13 -13
- package/dist/server/bare/plugins/whispercpp-transcription/plugin.d.ts +17 -1
- package/dist/server/bare/plugins/whispercpp-transcription/plugin.js +40 -17
- package/dist/server/bare/registry/config-registry.js +14 -3
- package/dist/server/bare/registry/model-registry.d.ts +8 -10
- package/dist/server/bare/registry/model-registry.js +17 -12
- package/dist/server/bare/registry/registry-client.js +79 -19
- package/dist/server/bare/runtime-lifecycle.d.ts +2 -1
- package/dist/server/bare/runtime-lifecycle.js +13 -2
- package/dist/server/bare/types/addon-responses.d.ts +8 -3
- package/dist/server/bare/utils/transcribe-metadata.d.ts +21 -0
- package/dist/server/bare/utils/transcribe-metadata.js +28 -0
- package/dist/server/bare/utils/tts-stats.d.ts +22 -0
- package/dist/server/bare/utils/tts-stats.js +10 -0
- package/dist/server/rpc/create-server.d.ts +4 -1
- package/dist/server/rpc/create-server.js +5 -1
- package/dist/server/rpc/delegate-transport.js +36 -29
- package/dist/server/rpc/handle-request.js +28 -3
- package/dist/server/rpc/handler-registry.js +7 -1
- package/dist/server/rpc/handler-utils.d.ts +5 -0
- package/dist/server/rpc/handler-utils.js +22 -0
- package/dist/server/rpc/handlers/cancel-delegated.js +3 -5
- package/dist/server/rpc/handlers/cancelHandler.js +4 -5
- package/dist/server/rpc/handlers/completion-stream-delegated.js +14 -5
- package/dist/server/rpc/handlers/delete-cache.js +8 -3
- package/dist/server/rpc/handlers/download-asset.js +3 -24
- package/dist/server/rpc/handlers/get-loaded-model-info.d.ts +3 -0
- package/dist/server/rpc/handlers/get-loaded-model-info.js +39 -0
- package/dist/server/rpc/handlers/get-model-info.js +93 -84
- package/dist/server/rpc/handlers/heartbeat-delegated.js +2 -2
- package/dist/server/rpc/handlers/index.d.ts +6 -0
- package/dist/server/rpc/handlers/index.js +6 -0
- package/dist/server/rpc/handlers/load-model/download-manager.d.ts +35 -9
- package/dist/server/rpc/handlers/load-model/download-manager.js +142 -38
- package/dist/server/rpc/handlers/load-model/download-stats.d.ts +5 -4
- package/dist/server/rpc/handlers/load-model/download-stats.js +49 -8
- package/dist/server/rpc/handlers/load-model/handler.js +31 -50
- package/dist/server/rpc/handlers/load-model/http.d.ts +2 -2
- package/dist/server/rpc/handlers/load-model/http.js +97 -142
- package/dist/server/rpc/handlers/load-model/hyperdrive.d.ts +2 -2
- package/dist/server/rpc/handlers/load-model/hyperdrive.js +55 -74
- package/dist/server/rpc/handlers/load-model/index.d.ts +0 -1
- package/dist/server/rpc/handlers/load-model/index.js +0 -1
- package/dist/server/rpc/handlers/load-model/registry-client-options.d.ts +27 -0
- package/dist/server/rpc/handlers/load-model/registry-client-options.js +19 -0
- package/dist/server/rpc/handlers/load-model/registry-companion-set.d.ts +22 -0
- package/dist/server/rpc/handlers/load-model/registry-companion-set.js +208 -0
- package/dist/server/rpc/handlers/load-model/registry-download-utils.d.ts +24 -0
- package/dist/server/rpc/handlers/load-model/registry-download-utils.js +114 -0
- package/dist/server/rpc/handlers/load-model/registry.d.ts +2 -20
- package/dist/server/rpc/handlers/load-model/registry.js +109 -381
- package/dist/server/rpc/handlers/load-model/resolve-session.d.ts +15 -0
- package/dist/server/rpc/handlers/load-model/resolve-session.js +61 -0
- package/dist/server/rpc/handlers/load-model/resolve.d.ts +3 -3
- package/dist/server/rpc/handlers/load-model/resolve.js +14 -23
- package/dist/server/rpc/handlers/load-model/types.d.ts +11 -4
- package/dist/server/rpc/handlers/load-model/types.js +28 -1
- package/dist/server/rpc/handlers/load-model-delegated.js +2 -4
- package/dist/server/rpc/handlers/plugin-dispatch.js +9 -5
- package/dist/server/rpc/handlers/plugin-invoke.js +1 -1
- package/dist/server/rpc/handlers/provideHandler/connection.js +13 -2
- package/dist/server/rpc/handlers/provideHandler/handler.js +42 -29
- package/dist/server/rpc/handlers/state.d.ts +3 -0
- package/dist/server/rpc/handlers/state.js +7 -0
- package/dist/server/rpc/handlers/stopProvideHandler.d.ts +2 -2
- package/dist/server/rpc/handlers/stopProvideHandler.js +12 -7
- package/dist/server/rpc/handlers/text-to-speech-stream.d.ts +3 -0
- package/dist/server/rpc/handlers/text-to-speech-stream.js +4 -0
- package/dist/server/rpc/handlers/unload-model-delegated.js +3 -3
- package/dist/server/rpc/profiling/operation-metrics.js +20 -0
- package/dist/server/rpc/rpc-utils.js +1 -1
- package/dist/server/rpc/transport-selector.d.ts +3 -0
- package/dist/server/rpc/transport-selector.js +18 -0
- package/dist/server/utils/cache/auto-history.d.ts +4 -0
- package/dist/server/utils/cache/auto-history.js +22 -0
- package/dist/server/utils/{cache.d.ts → cache/paths.d.ts} +22 -5
- package/dist/server/utils/{cache.js → cache/paths.js} +36 -20
- package/dist/server/utils/cache/types.d.ts +8 -0
- package/dist/server/utils/cache/types.js +1 -0
- package/dist/server/utils/checksum.d.ts +2 -2
- package/dist/server/utils/checksum.js +1 -1
- package/dist/server/utils/clear-storage.d.ts +5 -0
- package/dist/server/utils/clear-storage.js +17 -0
- package/dist/server/utils/completion-normalizer.d.ts +17 -0
- package/dist/server/utils/completion-normalizer.js +413 -0
- package/dist/server/utils/expand-gguf-shards.d.ts +3 -0
- package/dist/server/utils/expand-gguf-shards.js +22 -0
- package/dist/server/utils/formatting.js +1 -1
- package/dist/server/utils/index.d.ts +4 -2
- package/dist/server/utils/index.js +4 -2
- package/dist/server/utils/qvac-paths.d.ts +2 -0
- package/dist/server/utils/qvac-paths.js +5 -0
- package/dist/server/utils/response-format.d.ts +3 -0
- package/dist/server/utils/response-format.js +20 -0
- package/dist/server/utils/tool-integration.d.ts +15 -4
- package/dist/server/utils/tool-integration.js +21 -41
- package/dist/server/utils/tools/dialect.d.ts +8 -0
- package/dist/server/utils/tools/dialect.js +14 -0
- package/dist/server/utils/tools/index.d.ts +3 -0
- package/dist/server/utils/tools/index.js +2 -0
- package/dist/server/utils/tools/parser.d.ts +6 -0
- package/dist/server/utils/tools/parser.js +46 -0
- package/dist/server/utils/tools/parsers/harmony.d.ts +4 -0
- package/dist/server/utils/tools/parsers/harmony.js +59 -0
- package/dist/server/utils/tools/parsers/hermes.d.ts +4 -0
- package/dist/server/utils/tools/parsers/hermes.js +100 -0
- package/dist/server/utils/tools/parsers/json.d.ts +6 -0
- package/dist/server/utils/tools/parsers/json.js +115 -0
- package/dist/server/utils/tools/parsers/pythonic.d.ts +4 -0
- package/dist/server/utils/tools/parsers/pythonic.js +267 -0
- package/dist/server/utils/tools/shared.d.ts +18 -0
- package/dist/server/utils/tools/shared.js +54 -0
- package/dist/server/utils/worker-lock.d.ts +3 -0
- package/dist/server/utils/worker-lock.js +67 -0
- package/dist/server/worker-core.d.ts +16 -1
- package/dist/server/worker-core.js +114 -14
- package/dist/utils/aggregate-events.d.ts +16 -0
- package/dist/utils/aggregate-events.js +52 -0
- package/dist/utils/cache-normalize.d.ts +10 -0
- package/dist/utils/cache-normalize.js +23 -0
- package/dist/utils/encoding.d.ts +8 -0
- package/dist/utils/encoding.js +13 -0
- package/dist/utils/errors-client.d.ts +12 -0
- package/dist/utils/errors-client.js +20 -0
- package/dist/utils/errors-server.d.ts +14 -0
- package/dist/utils/errors-server.js +31 -0
- package/dist/utils/load-model-validation.d.ts +7 -0
- package/dist/utils/load-model-validation.js +18 -0
- package/expo/plugins/patches/android-link.mjs +1 -1
- package/expo/plugins/patches/ios-link.mjs +1 -1
- package/package.json +14 -18
- package/dist/examples/translation/translation-opus.d.ts +0 -2
- package/dist/examples/translation/translation-opus.js +0 -33
- package/dist/server/bare/utils/loader-adapter.d.ts +0 -7
- package/dist/server/bare/utils/loader-adapter.js +0 -9
- package/dist/server/utils/model-path.d.ts +0 -5
- package/dist/server/utils/model-path.js +0 -7
- package/dist/server/utils/tool-parser.d.ts +0 -7
- package/dist/server/utils/tool-parser.js +0 -248
- /package/dist/examples/{llamacpp-native-tools.d.ts → tools/llamacpp-native-tools.d.ts} +0 -0
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dynamic-tools mode example.
|
|
3
|
+
*
|
|
4
|
+
* In `dynamic` mode each user prompt can carry its own tool set: the addon
|
|
5
|
+
* anchors tools after the last user message, runs the tool-call chain, then
|
|
6
|
+
* trims the tools + chain output from the kv-cache so a later turn can ship
|
|
7
|
+
* a different tool list without poisoning the cache. Compare with
|
|
8
|
+
* `llamacpp-native-tools.ts`, which uses the default `static` mode where
|
|
9
|
+
* one shared tool set lives at the top of the session.
|
|
10
|
+
*
|
|
11
|
+
* Run with:
|
|
12
|
+
* bun run build
|
|
13
|
+
* bun run bare:example dist/examples/llamacpp-dynamic-tools.js
|
|
14
|
+
*/
|
|
15
|
+
import { z } from "zod";
|
|
16
|
+
import { completion, loadModel, unloadModel, QWEN3_1_7B_INST_Q4, } from "@qvac/sdk";
|
|
17
|
+
const weatherSchema = z.object({
|
|
18
|
+
city: z.string().describe("City name"),
|
|
19
|
+
});
|
|
20
|
+
const horoscopeSchema = z.object({
|
|
21
|
+
sign: z.string().describe("An astrological sign, e.g. Taurus or Aquarius"),
|
|
22
|
+
});
|
|
23
|
+
const dateSchema = z.object({});
|
|
24
|
+
const toolSchemas = {
|
|
25
|
+
get_weather: weatherSchema,
|
|
26
|
+
get_horoscope: horoscopeSchema,
|
|
27
|
+
get_date: dateSchema,
|
|
28
|
+
};
|
|
29
|
+
const weatherTools = [
|
|
30
|
+
{
|
|
31
|
+
name: "get_weather",
|
|
32
|
+
description: "Get current weather for a city",
|
|
33
|
+
parameters: weatherSchema,
|
|
34
|
+
},
|
|
35
|
+
];
|
|
36
|
+
const horoscopeTools = [
|
|
37
|
+
{
|
|
38
|
+
name: "get_horoscope",
|
|
39
|
+
description: "Get today's horoscope for an astrological sign",
|
|
40
|
+
parameters: horoscopeSchema,
|
|
41
|
+
},
|
|
42
|
+
];
|
|
43
|
+
const dateTools = [
|
|
44
|
+
{
|
|
45
|
+
name: "get_date",
|
|
46
|
+
description: "Get today's date",
|
|
47
|
+
parameters: dateSchema,
|
|
48
|
+
},
|
|
49
|
+
];
|
|
50
|
+
function executeToolCall(call) {
|
|
51
|
+
if (call.name === "get_weather") {
|
|
52
|
+
const args = call.arguments;
|
|
53
|
+
return `The weather in ${args.city} is rainy, 8°C with heavy clouds.`;
|
|
54
|
+
}
|
|
55
|
+
if (call.name === "get_horoscope") {
|
|
56
|
+
const args = call.arguments;
|
|
57
|
+
return `Horoscope for ${args.sign}: a great day for new beginnings.`;
|
|
58
|
+
}
|
|
59
|
+
if (call.name === "get_date") {
|
|
60
|
+
return new Date().toISOString().slice(0, 10);
|
|
61
|
+
}
|
|
62
|
+
return `Unknown tool: ${call.name}`;
|
|
63
|
+
}
|
|
64
|
+
async function chatTurn({ modelId, kvCache, history, tools }) {
|
|
65
|
+
const result = completion({
|
|
66
|
+
modelId,
|
|
67
|
+
history,
|
|
68
|
+
tools,
|
|
69
|
+
kvCache,
|
|
70
|
+
stream: true,
|
|
71
|
+
});
|
|
72
|
+
const tokensTask = (async () => {
|
|
73
|
+
for await (const token of result.tokenStream) {
|
|
74
|
+
process.stdout.write(token);
|
|
75
|
+
}
|
|
76
|
+
})();
|
|
77
|
+
const toolEventsTask = (async () => {
|
|
78
|
+
for await (const evt of result.toolCallStream) {
|
|
79
|
+
console.log(`\n→ tool call: ${evt.call.name}(${JSON.stringify(evt.call.arguments)})`);
|
|
80
|
+
}
|
|
81
|
+
})();
|
|
82
|
+
await Promise.all([tokensTask, toolEventsTask]);
|
|
83
|
+
const text = await result.text;
|
|
84
|
+
const toolCalls = await result.toolCalls;
|
|
85
|
+
if (toolCalls.length === 0) {
|
|
86
|
+
history.push({ role: "assistant", content: text });
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
for (const call of toolCalls) {
|
|
90
|
+
const schema = toolSchemas[call.name];
|
|
91
|
+
if (schema) {
|
|
92
|
+
const parsed = schema.safeParse(call.arguments);
|
|
93
|
+
if (!parsed.success) {
|
|
94
|
+
console.warn(` ✗ validation failed for ${call.name}:`, parsed.error);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
history.push({ role: "assistant", content: text });
|
|
99
|
+
for (const call of toolCalls) {
|
|
100
|
+
history.push({ role: "tool", content: executeToolCall(call) });
|
|
101
|
+
}
|
|
102
|
+
// Follow-up turn so the model can incorporate the tool results.
|
|
103
|
+
await chatTurn({ modelId, kvCache, history, tools });
|
|
104
|
+
}
|
|
105
|
+
async function main() {
|
|
106
|
+
const modelId = await loadModel({
|
|
107
|
+
modelSrc: QWEN3_1_7B_INST_Q4,
|
|
108
|
+
modelType: "llm",
|
|
109
|
+
modelConfig: {
|
|
110
|
+
ctx_size: 4096,
|
|
111
|
+
tools: true,
|
|
112
|
+
toolsMode: "dynamic",
|
|
113
|
+
},
|
|
114
|
+
onProgress: (progress) => console.log(`Loading: ${progress.percentage.toFixed(1)}%`),
|
|
115
|
+
});
|
|
116
|
+
console.log(`✅ Model loaded: ${modelId}`);
|
|
117
|
+
const kvCache = `dynamic-tools-${Date.now()}`;
|
|
118
|
+
const history = [
|
|
119
|
+
{
|
|
120
|
+
role: "system",
|
|
121
|
+
content: "You are a helpful assistant that uses tools when they are available. " +
|
|
122
|
+
"User's cat is named Windy and dog is named Butch.",
|
|
123
|
+
},
|
|
124
|
+
];
|
|
125
|
+
// Turn 1 — only weather tools available.
|
|
126
|
+
history.push({ role: "user", content: "What's the weather in Tokyo?" });
|
|
127
|
+
console.log("\n🤖 Turn 1 (tools=weather):\n");
|
|
128
|
+
await chatTurn({ modelId, kvCache, history, tools: weatherTools });
|
|
129
|
+
// Turn 2 — same session, swap to horoscope tools. Dynamic mode lets the
|
|
130
|
+
// model see a different tool set without invalidating the kv-cache.
|
|
131
|
+
history.push({ role: "user", content: "Now check my horoscope for Aquarius." });
|
|
132
|
+
console.log("\n\n🤖 Turn 2 (tools=horoscope):\n");
|
|
133
|
+
await chatTurn({ modelId, kvCache, history, tools: horoscopeTools });
|
|
134
|
+
// Turn 3 — swap to a parameterless tool to confirm empty-arg flows work.
|
|
135
|
+
history.push({ role: "user", content: "What's today's date?" });
|
|
136
|
+
console.log("\n\n🤖 Turn 3 (tools=date):\n");
|
|
137
|
+
await chatTurn({ modelId, kvCache, history, tools: dateTools });
|
|
138
|
+
console.log("\n\n🎉 Done.");
|
|
139
|
+
await unloadModel({ modelId, clearStorage: false });
|
|
140
|
+
}
|
|
141
|
+
main().catch((err) => {
|
|
142
|
+
console.error("❌ Error:", err);
|
|
143
|
+
process.exit(1);
|
|
144
|
+
});
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { loadModel, completion, unloadModel, QWEN3_600M_INST_Q4, } from "@qvac/sdk";
|
|
2
|
+
// Demonstrates per-request structured output via `responseFormat`.
|
|
3
|
+
// Three modes are exercised against the same model so you can eyeball the
|
|
4
|
+
// difference between free-form text, free-form JSON, and a strict JSON Schema.
|
|
5
|
+
//
|
|
6
|
+
// Usage:
|
|
7
|
+
// bun run examples/llamacpp-structured-output.ts
|
|
8
|
+
const PERSON_SCHEMA = {
|
|
9
|
+
type: "object",
|
|
10
|
+
properties: {
|
|
11
|
+
name: { type: "string" },
|
|
12
|
+
age: { type: "integer" },
|
|
13
|
+
occupation: { type: "string" },
|
|
14
|
+
},
|
|
15
|
+
required: ["name", "age", "occupation"],
|
|
16
|
+
additionalProperties: false,
|
|
17
|
+
};
|
|
18
|
+
const HISTORY = [
|
|
19
|
+
{
|
|
20
|
+
role: "system",
|
|
21
|
+
content: "You extract structured information about people from short bios. /no_think",
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
role: "user",
|
|
25
|
+
content: "Hi, I'm Alice, I'm 30 years old and I work as a data engineer.",
|
|
26
|
+
},
|
|
27
|
+
];
|
|
28
|
+
// `json_object` mode only enforces that the output is *some* valid JSON object
|
|
29
|
+
// — it doesn't pin the keys. Small models (Qwen3-0.6B in this example) will
|
|
30
|
+
// often emit `{}` because that's the shortest valid completion under the
|
|
31
|
+
// grammar. The same gotcha is documented for OpenAI's `response_format:
|
|
32
|
+
// json_object`. To get useful keys you either need a stronger prompt + a
|
|
33
|
+
// larger model, or — better — switch to `json_schema` (mode 3 below) so the
|
|
34
|
+
// grammar itself forces the keys.
|
|
35
|
+
const JSON_OBJECT_HISTORY = [
|
|
36
|
+
{
|
|
37
|
+
role: "system",
|
|
38
|
+
content: 'You extract structured information about people from short bios and reply ONLY with a JSON object containing "name", "age", and "occupation". /no_think',
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
role: "user",
|
|
42
|
+
content: "Hi, I'm Alice, I'm 30 years old and I work as a data engineer.",
|
|
43
|
+
},
|
|
44
|
+
];
|
|
45
|
+
// Drains the run via the canonical `events` / `final` surface and returns the
|
|
46
|
+
// content text. `tokenStream` / `text` / `stats` still work but are legacy and
|
|
47
|
+
// will be deprecated, so new code should consume `events` for streaming and
|
|
48
|
+
// await `final.contentText` for the aggregated result.
|
|
49
|
+
async function runToContent(run) {
|
|
50
|
+
for await (const event of run.events) {
|
|
51
|
+
if (event.type === "contentDelta")
|
|
52
|
+
process.stdout.write(event.text);
|
|
53
|
+
}
|
|
54
|
+
const final = await run.final;
|
|
55
|
+
process.stdout.write("\n");
|
|
56
|
+
return final.contentText;
|
|
57
|
+
}
|
|
58
|
+
try {
|
|
59
|
+
const modelId = await loadModel({
|
|
60
|
+
modelSrc: QWEN3_600M_INST_Q4,
|
|
61
|
+
modelType: "llm",
|
|
62
|
+
onProgress: (p) => process.stdout.write(`\rLoading: ${p.percentage.toFixed(1)}%`),
|
|
63
|
+
});
|
|
64
|
+
console.log(`\nModel loaded: ${modelId}\n`);
|
|
65
|
+
console.log("--- 1. responseFormat: text (baseline, free-form) ---");
|
|
66
|
+
await runToContent(completion({
|
|
67
|
+
modelId,
|
|
68
|
+
history: HISTORY,
|
|
69
|
+
stream: true,
|
|
70
|
+
responseFormat: { type: "text" },
|
|
71
|
+
}));
|
|
72
|
+
console.log("\n--- 2. responseFormat: json_object (any valid JSON) ---");
|
|
73
|
+
const jsonObjectOut = await runToContent(completion({
|
|
74
|
+
modelId,
|
|
75
|
+
history: JSON_OBJECT_HISTORY,
|
|
76
|
+
stream: true,
|
|
77
|
+
responseFormat: { type: "json_object" },
|
|
78
|
+
}));
|
|
79
|
+
console.log("parsed:", JSON.parse(jsonObjectOut.trim()));
|
|
80
|
+
console.log("\n--- 3. responseFormat: json_schema (strict shape) ---");
|
|
81
|
+
const jsonSchemaOut = await runToContent(completion({
|
|
82
|
+
modelId,
|
|
83
|
+
history: HISTORY,
|
|
84
|
+
stream: true,
|
|
85
|
+
responseFormat: {
|
|
86
|
+
type: "json_schema",
|
|
87
|
+
json_schema: {
|
|
88
|
+
name: "person",
|
|
89
|
+
schema: PERSON_SCHEMA,
|
|
90
|
+
},
|
|
91
|
+
},
|
|
92
|
+
}));
|
|
93
|
+
// Output is guaranteed schema-valid JSON: object with exactly
|
|
94
|
+
// {name: string, age: integer, occupation: string}, no extras.
|
|
95
|
+
const parsed = JSON.parse(jsonSchemaOut.trim());
|
|
96
|
+
console.log("parsed:", parsed);
|
|
97
|
+
console.log("schema-valid:", typeof parsed.name === "string" &&
|
|
98
|
+
Number.isInteger(parsed.age) &&
|
|
99
|
+
typeof parsed.occupation === "string" &&
|
|
100
|
+
Object.keys(parsed).sort().join(",") === "age,name,occupation");
|
|
101
|
+
await unloadModel({ modelId });
|
|
102
|
+
}
|
|
103
|
+
catch (error) {
|
|
104
|
+
console.error("Error:", error);
|
|
105
|
+
process.exit(1);
|
|
106
|
+
}
|
package/dist/examples/plugins.js
CHANGED
|
@@ -46,7 +46,7 @@ process.on("exit", cleanup);
|
|
|
46
46
|
// Point config before importing the SDK.
|
|
47
47
|
// In a real app, place qvac.config.json (or .js/.ts) in your project root (auto-discovered).
|
|
48
48
|
process.env["QVAC_CONFIG_PATH"] = configPath;
|
|
49
|
-
const { completion, translate, embed, loadModel, unloadModel, LLAMA_3_2_1B_INST_Q4_0,
|
|
49
|
+
const { completion, translate, embed, loadModel, unloadModel, LLAMA_3_2_1B_INST_Q4_0, BERGAMOT_EN_ES, GTE_LARGE_FP16, } = await import("@qvac/sdk");
|
|
50
50
|
console.log("1. LLM Completion (llamacpp-completion plugin)");
|
|
51
51
|
try {
|
|
52
52
|
const llmModelId = await loadModel({
|
|
@@ -78,12 +78,12 @@ catch (error) {
|
|
|
78
78
|
console.log("2. Translation (nmtcpp-translation plugin)");
|
|
79
79
|
try {
|
|
80
80
|
const nmtModelId = await loadModel({
|
|
81
|
-
modelSrc:
|
|
81
|
+
modelSrc: BERGAMOT_EN_ES,
|
|
82
82
|
modelType: "nmt",
|
|
83
83
|
modelConfig: {
|
|
84
|
-
engine: "
|
|
84
|
+
engine: "Bergamot",
|
|
85
85
|
from: "en",
|
|
86
|
-
to: "
|
|
86
|
+
to: "es",
|
|
87
87
|
},
|
|
88
88
|
onProgress: (p) => console.log(` Loading NMT: ${p.percentage.toFixed(1)}%`),
|
|
89
89
|
});
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { loadModel, LLAMA_3_2_1B_INST_Q4_0, completion, unloadModel, suspend, resume, } from "@qvac/sdk";
|
|
1
|
+
import { loadModel, LLAMA_3_2_1B_INST_Q4_0, completion, unloadModel, suspend, resume, state, } from "@qvac/sdk";
|
|
2
2
|
try {
|
|
3
3
|
// Load a model
|
|
4
4
|
const modelId = await loadModel({
|
|
@@ -9,6 +9,7 @@ try {
|
|
|
9
9
|
},
|
|
10
10
|
});
|
|
11
11
|
console.log("✅ Model loaded\n");
|
|
12
|
+
console.log(`📊 Lifecycle state: ${await state()}\n`);
|
|
12
13
|
// Run a completion before suspending
|
|
13
14
|
console.log("--- Completion before suspend ---");
|
|
14
15
|
const result1 = completion({
|
|
@@ -23,14 +24,30 @@ try {
|
|
|
23
24
|
// Suspend all networking and storage (e.g. app going to background)
|
|
24
25
|
console.log("⏸️ Suspending...");
|
|
25
26
|
await suspend();
|
|
26
|
-
console.log(
|
|
27
|
+
console.log(`📊 Lifecycle state: ${await state()}\n`);
|
|
28
|
+
try {
|
|
29
|
+
await completion({
|
|
30
|
+
modelId,
|
|
31
|
+
history: [{ role: "user", content: "This should fail" }],
|
|
32
|
+
stream: false,
|
|
33
|
+
}).text;
|
|
34
|
+
}
|
|
35
|
+
catch (error) {
|
|
36
|
+
const name = error.name;
|
|
37
|
+
if (name === "LIFECYCLE_OPERATION_BLOCKED") {
|
|
38
|
+
console.log(`🚫 Operation blocked while suspended (${name})`);
|
|
39
|
+
}
|
|
40
|
+
else {
|
|
41
|
+
throw error;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
27
44
|
// Simulate time in background
|
|
28
|
-
console.log("💤 Simulating 3 seconds in background...");
|
|
45
|
+
console.log("\n💤 Simulating 3 seconds in background...");
|
|
29
46
|
await new Promise((resolve) => setTimeout(resolve, 3000));
|
|
30
47
|
// Resume when returning to foreground
|
|
31
48
|
console.log("▶️ Resuming...");
|
|
32
49
|
await resume();
|
|
33
|
-
console.log(
|
|
50
|
+
console.log(`📊 Lifecycle state: ${await state()}\n`);
|
|
34
51
|
// Run another completion after resuming
|
|
35
52
|
console.log("--- Completion after resume ---");
|
|
36
53
|
const result2 = completion({
|
|
@@ -1,44 +1,16 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
// Define Zod schemas for tool parameters
|
|
4
|
-
const weatherSchema = z.object({
|
|
5
|
-
city: z.string().describe("City name"),
|
|
6
|
-
country: z.string().describe("Country code").optional(),
|
|
7
|
-
});
|
|
8
|
-
const horoscopeSchema = z.object({
|
|
9
|
-
sign: z.string().describe("An astrological sign like Taurus or Aquarius"),
|
|
10
|
-
});
|
|
11
|
-
// Map tool names to their schemas for runtime validation
|
|
12
|
-
const toolSchemas = {
|
|
13
|
-
get_weather: weatherSchema,
|
|
14
|
-
get_horoscope: horoscopeSchema,
|
|
15
|
-
};
|
|
16
|
-
// Simple tool definitions - just name, description, and Zod schema!
|
|
17
|
-
const tools = [
|
|
18
|
-
{
|
|
19
|
-
name: "get_weather",
|
|
20
|
-
description: "Get current weather for a city",
|
|
21
|
-
parameters: weatherSchema,
|
|
22
|
-
},
|
|
23
|
-
{
|
|
24
|
-
name: "get_horoscope",
|
|
25
|
-
description: "Get today's horoscope for an astrological sign",
|
|
26
|
-
parameters: horoscopeSchema,
|
|
27
|
-
},
|
|
28
|
-
];
|
|
1
|
+
import { completion, loadModel, unloadModel, QWEN3_1_7B_INST_Q4, } from "@qvac/sdk";
|
|
2
|
+
import { tools, toolSchemas, mockExecute } from "./shared.js";
|
|
29
3
|
try {
|
|
30
|
-
// Load model from provided file path with tools support enabled
|
|
31
4
|
const modelId = await loadModel({
|
|
32
5
|
modelSrc: QWEN3_1_7B_INST_Q4,
|
|
33
6
|
modelType: "llm",
|
|
34
7
|
modelConfig: {
|
|
35
8
|
ctx_size: 4096,
|
|
36
|
-
tools: true,
|
|
9
|
+
tools: true,
|
|
37
10
|
},
|
|
38
11
|
onProgress: (progress) => console.log(`Loading: ${progress.percentage.toFixed(1)}%`),
|
|
39
12
|
});
|
|
40
13
|
console.log(`✅ Model loaded successfully! Model ID: ${modelId}`);
|
|
41
|
-
// Create conversation history
|
|
42
14
|
const history = [
|
|
43
15
|
{
|
|
44
16
|
role: "system",
|
|
@@ -52,23 +24,15 @@ try {
|
|
|
52
24
|
console.log("\n🤖 AI Response:");
|
|
53
25
|
console.log("(Streaming with tool definitions in prompt)\n");
|
|
54
26
|
const result = completion({ modelId, history, stream: true, tools });
|
|
55
|
-
// Consume token stream
|
|
56
27
|
const tokensTask = (async () => {
|
|
57
28
|
for await (const token of result.tokenStream) {
|
|
58
29
|
process.stdout.write(token);
|
|
59
30
|
}
|
|
60
31
|
})();
|
|
61
|
-
// Consume tool call events
|
|
62
32
|
const toolsTask = (async () => {
|
|
63
33
|
for await (const evt of result.toolCallStream) {
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
console.log(` ID: ${evt.call.id}`);
|
|
67
|
-
}
|
|
68
|
-
else if (evt.type === "toolCallError") {
|
|
69
|
-
console.warn(`\n⚠️ Tool Error: ${evt.error.message}`);
|
|
70
|
-
console.warn(` Code: ${evt.error.code}`);
|
|
71
|
-
}
|
|
34
|
+
console.log(`\n\n→ Tool Call Detected: ${evt.call.name}(${JSON.stringify(evt.call.arguments)})`);
|
|
35
|
+
console.log(` ID: ${evt.call.id}`);
|
|
72
36
|
}
|
|
73
37
|
})();
|
|
74
38
|
await Promise.all([tokensTask, toolsTask]);
|
|
@@ -94,36 +58,23 @@ try {
|
|
|
94
58
|
console.log(" No tool calls detected in response");
|
|
95
59
|
}
|
|
96
60
|
console.log("\n📊 Performance Stats:", stats);
|
|
97
|
-
// Execute tool calls and send results back to the model
|
|
98
61
|
if (toolCalls.length > 0) {
|
|
99
62
|
console.log("\n\n🔧 Simulating Tool Execution...");
|
|
100
|
-
// Simulate tool execution (in a real app, you'd call actual APIs)
|
|
101
63
|
const toolResults = toolCalls.map((call) => {
|
|
102
|
-
|
|
103
|
-
if (call.name === "get_weather") {
|
|
104
|
-
const args = call.arguments;
|
|
105
|
-
result = `The weather in ${args.city} is sunny, 22°C with light clouds.`;
|
|
106
|
-
}
|
|
107
|
-
else if (call.name === "get_horoscope") {
|
|
108
|
-
const args = call.arguments;
|
|
109
|
-
result = `Horoscope for ${args.sign}: Today is a great day for new beginnings and creative endeavors!`;
|
|
110
|
-
}
|
|
64
|
+
const result = mockExecute(call.name, call.arguments);
|
|
111
65
|
console.log(` ✓ ${call.name}: ${result}`);
|
|
112
66
|
return { toolCallId: call.id, result };
|
|
113
67
|
});
|
|
114
|
-
// Add tool results to conversation history
|
|
115
68
|
history.push({
|
|
116
69
|
role: "assistant",
|
|
117
70
|
content: await result.text,
|
|
118
71
|
});
|
|
119
|
-
// Add tool results as tool messages
|
|
120
72
|
for (const toolResult of toolResults) {
|
|
121
73
|
history.push({
|
|
122
74
|
role: "tool",
|
|
123
75
|
content: toolResult.result,
|
|
124
76
|
});
|
|
125
77
|
}
|
|
126
|
-
// Send follow-up question with tool results
|
|
127
78
|
console.log("\n\n🤖 Follow-up Response with Tool Results:");
|
|
128
79
|
const followUpResult = completion({
|
|
129
80
|
modelId,
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Multi-turn tool-calling example using the GPT-OSS model (Harmony dialect).
|
|
3
|
+
*
|
|
4
|
+
* GPT-OSS emits one tool call per generation, so multi-tool requests need
|
|
5
|
+
* a turn-by-turn loop: send prompt + tools → mock-execute the returned
|
|
6
|
+
* `ToolCall` → append the raw assistant frame + tool result to history →
|
|
7
|
+
* repeat until the model produces a final answer (no tool call).
|
|
8
|
+
*
|
|
9
|
+
*/
|
|
10
|
+
import { completion, loadModel, unloadModel, GPT_OSS_20B_INST_Q4_K_M, } from "@qvac/sdk";
|
|
11
|
+
import { tools, mockExecute } from "./shared.js";
|
|
12
|
+
const MAX_TURNS = 5;
|
|
13
|
+
let modelId;
|
|
14
|
+
try {
|
|
15
|
+
modelId = await loadModel({
|
|
16
|
+
modelSrc: GPT_OSS_20B_INST_Q4_K_M,
|
|
17
|
+
modelConfig: { ctx_size: 4096, tools: true },
|
|
18
|
+
onProgress: (progress) => console.log(`Loading: ${progress.percentage.toFixed(1)}%`),
|
|
19
|
+
});
|
|
20
|
+
console.log(`✅ Model loaded: ${modelId}`);
|
|
21
|
+
const history = [
|
|
22
|
+
{
|
|
23
|
+
role: "system",
|
|
24
|
+
content: "You are a helpful assistant that can use tools to get the weather and horoscope.",
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
role: "user",
|
|
28
|
+
content: "What's the weather in Tokyo and my horoscope for Aquarius?",
|
|
29
|
+
},
|
|
30
|
+
];
|
|
31
|
+
for (let turn = 1; turn <= MAX_TURNS; turn++) {
|
|
32
|
+
console.log(`\n========== TURN ${turn} ==========`);
|
|
33
|
+
const result = completion({ modelId, history, tools, stream: true });
|
|
34
|
+
for await (const token of result.tokenStream) {
|
|
35
|
+
process.stdout.write(token);
|
|
36
|
+
}
|
|
37
|
+
console.log();
|
|
38
|
+
const final = await result.final;
|
|
39
|
+
console.log(`\n--- TURN ${turn} SUMMARY ---`);
|
|
40
|
+
console.log(`[sdk toolCalls] ${final.toolCalls.length}`);
|
|
41
|
+
for (const call of final.toolCalls) {
|
|
42
|
+
console.log(` → ${call.name}(${JSON.stringify(call.arguments)})`);
|
|
43
|
+
}
|
|
44
|
+
if (final.toolCalls.length === 0) {
|
|
45
|
+
console.log(`\n🎉 Final response received — exiting loop.`);
|
|
46
|
+
break;
|
|
47
|
+
}
|
|
48
|
+
// Raw, not result.text — GPT-OSS needs its own framed output in history
|
|
49
|
+
// to anchor the next turn.
|
|
50
|
+
const assistantContent = final.raw.fullText;
|
|
51
|
+
history.push({ role: "assistant", content: assistantContent });
|
|
52
|
+
for (const call of final.toolCalls) {
|
|
53
|
+
const toolResult = mockExecute(call.name, call.arguments);
|
|
54
|
+
console.log(` ✓ mock-executed ${call.name}: ${toolResult}`);
|
|
55
|
+
history.push({ role: "tool", content: toolResult });
|
|
56
|
+
}
|
|
57
|
+
if (turn === MAX_TURNS) {
|
|
58
|
+
console.log(`\n⚠️ MAX_TURNS (${MAX_TURNS}) reached — stopping.`);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
console.log(`\n========== HISTORY ==========`);
|
|
62
|
+
for (const msg of history) {
|
|
63
|
+
const preview = msg.content.slice(0, 120).replace(/\n/g, "\\n");
|
|
64
|
+
console.log(`[${msg.role}] ${preview}${msg.content.length > 120 ? "…" : ""}`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
console.error("❌ Error:", error);
|
|
69
|
+
process.exit(1);
|
|
70
|
+
}
|
|
71
|
+
finally {
|
|
72
|
+
if (modelId) {
|
|
73
|
+
try {
|
|
74
|
+
await unloadModel({ modelId, clearStorage: false });
|
|
75
|
+
console.log(`\n[unload] done`);
|
|
76
|
+
}
|
|
77
|
+
catch (unloadError) {
|
|
78
|
+
console.error("[unload] failed:", unloadError);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import { completion, loadModel, unloadModel, } from "@qvac/sdk";
|
|
2
|
+
import { tools } from "./shared.js";
|
|
3
|
+
const LFM_TOOL_HF = "https://huggingface.co/LiquidAI/LFM2-1.2B-Tool-GGUF/resolve/main/LFM2-1.2B-Tool-Q4_K_M.gguf";
|
|
4
|
+
try {
|
|
5
|
+
const modelId = await loadModel({
|
|
6
|
+
modelSrc: LFM_TOOL_HF,
|
|
7
|
+
modelType: "llm",
|
|
8
|
+
modelConfig: { ctx_size: 4096, tools: true },
|
|
9
|
+
onProgress: (progress) => console.log(`Loading: ${progress.percentage.toFixed(1)}%`),
|
|
10
|
+
});
|
|
11
|
+
console.log(`✅ Model loaded: ${modelId}`);
|
|
12
|
+
const history = [
|
|
13
|
+
{
|
|
14
|
+
role: "system",
|
|
15
|
+
content: "You are a helpful assistant that can call tools to look up weather and horoscopes.",
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
role: "user",
|
|
19
|
+
content: "What's the weather in Tokyo and my horoscope for Aquarius?",
|
|
20
|
+
},
|
|
21
|
+
];
|
|
22
|
+
console.log("\n🤖 Streaming...\n");
|
|
23
|
+
const result = completion({ modelId, history, stream: true, tools });
|
|
24
|
+
const tokensTask = (async () => {
|
|
25
|
+
for await (const token of result.tokenStream) {
|
|
26
|
+
process.stdout.write(token);
|
|
27
|
+
}
|
|
28
|
+
})();
|
|
29
|
+
const toolsTask = (async () => {
|
|
30
|
+
for await (const evt of result.toolCallStream) {
|
|
31
|
+
console.log(`\n→ ${evt.call.name}(${JSON.stringify(evt.call.arguments)})`);
|
|
32
|
+
}
|
|
33
|
+
})();
|
|
34
|
+
await Promise.all([tokensTask, toolsTask]);
|
|
35
|
+
const toolCalls = await result.toolCalls;
|
|
36
|
+
console.log("\n\n📋 Final tool calls:");
|
|
37
|
+
if (toolCalls.length > 0) {
|
|
38
|
+
for (const call of toolCalls) {
|
|
39
|
+
console.log(` - ${call.name}(${JSON.stringify(call.arguments)})`);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
else {
|
|
43
|
+
console.log(" (none)");
|
|
44
|
+
}
|
|
45
|
+
await unloadModel({ modelId });
|
|
46
|
+
}
|
|
47
|
+
catch (error) {
|
|
48
|
+
console.error("❌ Error:", error);
|
|
49
|
+
process.exit(1);
|
|
50
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import type { ToolInput } from "@qvac/sdk";
|
|
3
|
+
export declare const weatherSchema: z.ZodObject<{
|
|
4
|
+
city: z.ZodString;
|
|
5
|
+
country: z.ZodOptional<z.ZodString>;
|
|
6
|
+
}, z.core.$strip>;
|
|
7
|
+
export declare const horoscopeSchema: z.ZodObject<{
|
|
8
|
+
sign: z.ZodString;
|
|
9
|
+
}, z.core.$strip>;
|
|
10
|
+
export declare const toolSchemas: {
|
|
11
|
+
readonly get_weather: z.ZodObject<{
|
|
12
|
+
city: z.ZodString;
|
|
13
|
+
country: z.ZodOptional<z.ZodString>;
|
|
14
|
+
}, z.core.$strip>;
|
|
15
|
+
readonly get_horoscope: z.ZodObject<{
|
|
16
|
+
sign: z.ZodString;
|
|
17
|
+
}, z.core.$strip>;
|
|
18
|
+
};
|
|
19
|
+
export declare const tools: ToolInput[];
|
|
20
|
+
export declare function mockExecute(name: string, args: Record<string, unknown>): string;
|
|
21
|
+
//# sourceMappingURL=shared.d.ts.map
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
export const weatherSchema = z.object({
|
|
3
|
+
city: z.string().describe("City name"),
|
|
4
|
+
country: z.string().describe("Country code").optional(),
|
|
5
|
+
});
|
|
6
|
+
export const horoscopeSchema = z.object({
|
|
7
|
+
sign: z.string().describe("An astrological sign like Taurus or Aquarius"),
|
|
8
|
+
});
|
|
9
|
+
export const toolSchemas = {
|
|
10
|
+
get_weather: weatherSchema,
|
|
11
|
+
get_horoscope: horoscopeSchema,
|
|
12
|
+
};
|
|
13
|
+
export const tools = [
|
|
14
|
+
{
|
|
15
|
+
name: "get_weather",
|
|
16
|
+
description: "Get current weather for a city",
|
|
17
|
+
parameters: weatherSchema,
|
|
18
|
+
},
|
|
19
|
+
{
|
|
20
|
+
name: "get_horoscope",
|
|
21
|
+
description: "Get today's horoscope for an astrological sign",
|
|
22
|
+
parameters: horoscopeSchema,
|
|
23
|
+
},
|
|
24
|
+
];
|
|
25
|
+
export function mockExecute(name, args) {
|
|
26
|
+
if (name === "get_weather") {
|
|
27
|
+
const a = args;
|
|
28
|
+
return JSON.stringify({
|
|
29
|
+
city: a.city,
|
|
30
|
+
country: a.country ?? "unknown",
|
|
31
|
+
temperature: "22°C",
|
|
32
|
+
condition: "Partly cloudy",
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
if (name === "get_horoscope") {
|
|
36
|
+
return JSON.stringify({
|
|
37
|
+
sign: args.sign,
|
|
38
|
+
horoscope: "Today is a great day for new beginnings.",
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
return JSON.stringify({ error: `Unknown tool: ${name}` });
|
|
42
|
+
}
|