@qvac/sdk 0.3.0 → 0.4.0

package/README.md

@@ -272,6 +272,12 @@ This works for all model types (LLM, Whisper, NMT, Embeddings) and provides valu
 - Indic language translation: [`examples/translation-indic.ts`](examples/translation-indic.ts)
 - LLM-based translation: [`examples/translation-llm.ts`](examples/translation-llm.ts)
 
+### Text to Speech
+
+- Text to Speech: [`examples/text-to-speech.ts`](examples/text-to-speech.ts)
+
+**Note**: To run the text-to-speech example, espeak data is required which can be obtained from [here](https://github.com/espeak-ng/espeak-ng).
+
 ### Multimodel
 
 - Load multiple models simultaneously: [`examples/multi-model-demo.ts`](examples/multi-model-demo.ts)
@@ -288,6 +294,22 @@ This works for all model types (LLM, Whisper, NMT, Embeddings) and provides valu
 
 - Pause and resume download: [`examples/download-with-cancel.ts`](examples/download-with-cancel.ts)
 
+### Sharded Models
+
+- Load sharded models from Hyperdrive: [`examples/llamacpp-sharded.ts`](examples/llamacpp-sharded.ts)
+- Load models from local filesystem: [`examples/llamacpp-filesystem.ts`](examples/llamacpp-filesystem.ts)
+
+**Note**: Sharded models automatically download all parts sequentially with detailed progress tracking per shard. For local sharded models, pass the path to the first shard file (e.g., `model-00001-of-00005.gguf`), and remaining shards will be loaded automatically.
+
+### Blind Relays
+
+Blind relays help establish peer connections through NAT/firewalls by routing traffic through relay nodes.
+
+- Model downloads via Hyperdrive: [`examples/download-with-blind-relays.ts`](./examples/download-with-blind-relays.ts)
+- Delegated inference: You can reuse the same pattern for delegated inference by calling `setConfig({ swarmRelays: [...relayPublicKeys] })` before starting your provider/consumer.
+
+**Note**: The example uses mock relay keys. In real deployments, you **must** use your own relay servers or trusted public relays.
+
 ## Known issues
 
 - **One off scripts don't terminate / hang**

package/dist/client/api/cancel.d.ts

@@ -7,7 +7,7 @@ import { type CancelParams } from "../../schemas/index";
  * @param params.modelId - The model ID (required for inference cancellation)
  * @param params.downloadKey - The download key (required for download cancellation)
  * @param params.clearCache - If true, deletes the partial download file (default: false)
- * @throws {Error} When the response type is invalid or when the cancellation fails
+ * @throws {QvacErrorBase} When the response type is invalid or when the cancellation fails
  *
  * @example
  * // Cancel inference

package/dist/client/api/cancel.js

@@ -1,5 +1,6 @@
 import { send } from "../rpc/rpc-client.js";
 import {} from "../../schemas/index.js";
+import { InvalidResponseError, CancelFailedError } from "../../utils/errors-client.js";
 /**
  * Cancels an ongoing operation.
  *
@@ -8,7 +9,7 @@ import {} from "../../schemas/index.js";
  * @param params.modelId - The model ID (required for inference cancellation)
  * @param params.downloadKey - The download key (required for download cancellation)
  * @param params.clearCache - If true, deletes the partial download file (default: false)
- * @throws {Error} When the response type is invalid or when the cancellation fails
+ * @throws {QvacErrorBase} When the response type is invalid or when the cancellation fails
  *
  * @example
  * // Cancel inference
@@ -29,9 +30,9 @@ export async function cancel(params) {
     };
     const response = await send(request);
     if (response.type !== "cancel") {
-        throw new Error("Invalid response type");
+        throw new InvalidResponseError("cancel");
     }
     if (!response.success) {
-        throw new Error(response.error ?? "Failed to cancel");
+        throw new CancelFailedError(response.error);
     }
 }

package/dist/client/api/completion-stream.d.ts

@@ -1,4 +1,8 @@
-import { type CompletionStats, type CompletionClientParams } from "../../schemas/index";
+import { type CompletionStats, type CompletionClientParams, type ToolCall, type ToolCallEvent, type Tool } from "../../schemas/index";
+import { type ToolInput } from "../../utils/tool-helpers";
+type CompletionParams = Omit<CompletionClientParams, "tools"> & {
+    tools?: Tool[] | ToolInput[];
+};
 /**
  * Generates completion from a language model based on conversation history.
  *
@@ -6,7 +10,7 @@ import { type CompletionStats, type CompletionClientParams } from "../../schemas
  * @param params.modelId - The identifier of the model to use for completion
  * @param params.history - Array of conversation messages with role, content, and optional attachments
  * @param params.stream - Whether to stream tokens (true) or return complete response (false). Defaults to true
- * @param params.tools - Optional array of tools that can be used by the model
+ * @param params.tools - Optional array of tools (can be simple ToolInput with Zod schemas or full Tool objects)
  * @param params.kvCache - Optional KV cache configuration. Cache files are organized hierarchically:
  *   - Structure: `{kvCacheKey}/{modelId}/{configHash}.bin`
  *   - The configHash includes model config + system prompt to ensure cache isolation
@@ -15,51 +19,39 @@ import { type CompletionStats, type CompletionClientParams } from "../../schemas
  *   - `false` or `undefined`: No caching
  *   - ⚡ Performance: When cache exists, only the last message is sent to the model (includes multimodal attachments)
  *   - 🗑️ Cleanup: Use `deleteCache({ kvCacheKey })` to remove cached sessions
- * @returns Object with tokenStream generator, text promise, and stats promise
+ * @returns Object with tokenStream generator, toolCallStream generator, text promise, toolCalls promise, and stats promise
  * @example
  * ```typescript
- * // Streaming mode with auto-generated cache
+ * import { z } from "zod";
+ *
+ * // Simple tool definition with Zod
  * const result = completion({
  *   modelId: "llama-2",
  *   history: [
- *     { role: "user", content: "Hello, how are you?" }
+ *     { role: "user", content: "What's the weather in Tokyo?" }
  *   ],
  *   stream: true,
- *   kvCache: true, // Auto-generate cache key
+ *   tools: [{
+ *     name: "get_weather",
+ *     description: "Get current weather",
+ *     parameters: z.object({
+ *       city: z.string().describe("City name"),
+ *       country: z.string().describe("Country code").optional(),
+ *     }),
+ *   }]
  * });
  *
  * for await (const token of result.tokenStream) {
- *   console.log(token);
+ *   process.stdout.write(token);
  * }
- *
- * // Custom session management with cache key
- * const sessionResult = completion({
- *   modelId: "llama-2",
- *   history: [
- *     { role: "user", content: "Continue our conversation" }
- *   ],
- *   stream: true,
- *   kvCache: "user-session-123", // Custom cache key
- * });
- *
- * // Non-streaming mode
- * const response = completion({
- *   modelId: "llama-2",
- *   history: [
- *     { role: "user", content: "What is the capital of France?" }
- *   ],
- *   stream: false,
- * });
- *
- * const text = await response.text;
- * const stats = await response.stats;
- * console.log(text);
- * console.log(stats);
  * ```
  */
-export declare function completion(params: CompletionClientParams): {
+export declare function completion(params: CompletionParams): {
     tokenStream: AsyncGenerator<string>;
+    toolCallStream: AsyncGenerator<ToolCallEvent>;
     stats: Promise<CompletionStats | undefined>;
     text: Promise<string>;
+    toolCalls: Promise<ToolCall[]>;
 };
+export {};
 //# sourceMappingURL=completion-stream.d.ts.map

package/dist/client/api/completion-stream.js

@@ -1,5 +1,6 @@
 import { completionStreamResponseSchema, } from "../../schemas/index.js";
 import { stream as streamRpc } from "../rpc/rpc-client.js";
+import { validateTools } from "../../utils/tool-helpers.js";
 /**
  * Generates completion from a language model based on conversation history.
  *
@@ -7,7 +8,7 @@ import { stream as streamRpc } from "../rpc/rpc-client.js";
  * @param params.modelId - The identifier of the model to use for completion
  * @param params.history - Array of conversation messages with role, content, and optional attachments
  * @param params.stream - Whether to stream tokens (true) or return complete response (false). Defaults to true
- * @param params.tools - Optional array of tools that can be used by the model
+ * @param params.tools - Optional array of tools (can be simple ToolInput with Zod schemas or full Tool objects)
  * @param params.kvCache - Optional KV cache configuration. Cache files are organized hierarchically:
  *   - Structure: `{kvCacheKey}/{modelId}/{configHash}.bin`
  *   - The configHash includes model config + system prompt to ensure cache isolation
@@ -16,103 +17,194 @@ import { stream as streamRpc } from "../rpc/rpc-client.js";
  *   - `false` or `undefined`: No caching
  *   - ⚡ Performance: When cache exists, only the last message is sent to the model (includes multimodal attachments)
  *   - 🗑️ Cleanup: Use `deleteCache({ kvCacheKey })` to remove cached sessions
- * @returns Object with tokenStream generator, text promise, and stats promise
+ * @returns Object with tokenStream generator, toolCallStream generator, text promise, toolCalls promise, and stats promise
  * @example
  * ```typescript
- * // Streaming mode with auto-generated cache
+ * import { z } from "zod";
+ *
+ * // Simple tool definition with Zod
  * const result = completion({
  *   modelId: "llama-2",
  *   history: [
- *     { role: "user", content: "Hello, how are you?" }
+ *     { role: "user", content: "What's the weather in Tokyo?" }
  *   ],
  *   stream: true,
- *   kvCache: true, // Auto-generate cache key
+ *   tools: [{
+ *     name: "get_weather",
+ *     description: "Get current weather",
+ *     parameters: z.object({
+ *       city: z.string().describe("City name"),
+ *       country: z.string().describe("Country code").optional(),
+ *     }),
+ *   }]
  * });
  *
  * for await (const token of result.tokenStream) {
- *   console.log(token);
+ *   process.stdout.write(token);
  * }
- *
- * // Custom session management with cache key
- * const sessionResult = completion({
- *   modelId: "llama-2",
- *   history: [
- *     { role: "user", content: "Continue our conversation" }
- *   ],
- *   stream: true,
- *   kvCache: "user-session-123", // Custom cache key
- * });
- *
- * // Non-streaming mode
- * const response = completion({
- *   modelId: "llama-2",
- *   history: [
- *     { role: "user", content: "What is the capital of France?" }
- *   ],
- *   stream: false,
- * });
- *
- * const text = await response.text;
- * const stats = await response.stats;
- * console.log(text);
- * console.log(stats);
  * ```
  */
 export function completion(params) {
+    const tools = params.tools ? validateTools(params.tools) : undefined;
     const request = {
         type: "completionStream",
         ...params,
+        tools,
         stream: params.stream ?? true,
     };
     let stats;
     let statsResolver = () => { };
-    const statsPromise = new Promise((resolve) => {
+    let statsRejecter = () => { };
+    const statsPromise = new Promise((resolve, reject) => {
         statsResolver = resolve;
+        statsRejecter = reject;
     });
-    if (params.stream) {
-        const tokenStream = (async function* () {
-            for await (const response of streamRpc(request)) {
-                if (response.type === "completionStream") {
+    statsPromise.catch(() => { });
+    let toolCallsArray = [];
+    let toolCallsResolver = () => { };
+    let toolCallsRejecter = () => { };
+    const toolCallsPromise = new Promise((resolve, reject) => {
+        toolCallsResolver = resolve;
+        toolCallsRejecter = reject;
+    });
+    toolCallsPromise.catch(() => { });
+    const responses = streamRpc(request);
+    const tokenQueue = [];
+    const toolEventQueue = [];
+    let tokenDone = false;
+    let toolDone = false;
+    let tokenResolve = null;
+    let toolResolve = null;
+    let textBuffer = "";
+    let streamError = null;
+    const processResponses = async () => {
+        try {
+            for await (const response of responses) {
+                if (response &&
+                    typeof response === "object" &&
+                    "type" in response &&
+                    response.type === "completionStream") {
                     const streamResponse = completionStreamResponseSchema.parse(response);
                     if (!streamResponse.done) {
-                        yield streamResponse.token;
+                        const token = streamResponse.token;
+                        if (token) {
+                            textBuffer += token;
+                            tokenQueue.push(token);
+                            if (tokenResolve) {
+                                tokenResolve();
+                                tokenResolve = null;
+                            }
+                        }
+                        if (streamResponse.toolCallEvent) {
+                            toolEventQueue.push(streamResponse.toolCallEvent);
+                            if (toolResolve) {
+                                toolResolve();
+                                toolResolve = null;
+                            }
+                        }
                     }
                     else {
+                        if (streamResponse.token) {
+                            textBuffer += streamResponse.token;
+                        }
                         stats = streamResponse.stats;
                         statsResolver(stats);
+                        toolCallsArray = streamResponse.toolCalls || [];
+                        toolCallsResolver(toolCallsArray);
+                        tokenDone = true;
+                        toolDone = true;
+                        if (tokenResolve) {
+                            tokenResolve();
+                            tokenResolve = null;
+                        }
+                        if (toolResolve) {
+                            toolResolve();
+                            toolResolve = null;
+                        }
                     }
                 }
             }
+        }
+        catch (error) {
+            streamError = error instanceof Error ? error : new Error(String(error));
+            statsRejecter(error);
+            toolCallsRejecter(error);
+            tokenDone = true;
+            toolDone = true;
+            if (tokenResolve) {
+                tokenResolve();
+                tokenResolve = null;
+            }
+            if (toolResolve) {
+                toolResolve();
+                toolResolve = null;
+            }
+        }
+    };
+    void processResponses();
+    const textPromise = (async () => {
+        await statsPromise;
+        return textBuffer;
+    })();
+    textPromise.catch(() => { });
+    if (params.stream) {
+        const tokenStream = (async function* () {
+            while (true) {
+                if (tokenQueue.length > 0) {
+                    yield tokenQueue.shift();
+                }
+                else if (tokenDone) {
+                    if (streamError !== null) {
+                        throw streamError;
+                    }
+                    break;
+                }
+                else {
+                    await new Promise((resolve) => {
+                        tokenResolve = resolve;
+                    });
+                }
+            }
+        })();
+        const toolCallStream = (async function* () {
+            while (true) {
+                if (toolEventQueue.length > 0) {
+                    yield toolEventQueue.shift();
+                }
+                else if (toolDone) {
+                    if (streamError !== null) {
+                        throw streamError;
+                    }
+                    break;
+                }
+                else {
+                    await new Promise((resolve) => {
+                        toolResolve = resolve;
+                    });
+                }
+            }
         })();
-        const textPromise = Promise.resolve("");
         return {
             tokenStream,
+            toolCallStream,
             text: textPromise,
             stats: statsPromise,
+            toolCalls: toolCallsPromise,
         };
     }
     else {
         const tokenStream = (async function* () {
             //Empty generator for non-streaming mode
         })();
-        const textPromise = (async () => {
-            let buffer = "";
-            for await (const response of streamRpc(request)) {
-                if (response.type === "completionStream") {
-                    const streamResponse = completionStreamResponseSchema.parse(response);
-                    buffer += streamResponse.token;
-                    if (streamResponse.done) {
-                        stats = streamResponse.stats;
-                        statsResolver(stats);
-                    }
-                }
-            }
-            return buffer;
+        const toolCallStream = (async function* () {
+            //Empty generator for non-streaming mode
         })();
         return {
             tokenStream,
+            toolCallStream,
             text: textPromise,
             stats: statsPromise,
+            toolCalls: toolCallsPromise,
         };
     }
 }

package/dist/client/api/delete-cache.js

@@ -1,4 +1,5 @@
 import { send } from "../rpc/rpc-client.js";
+import { InvalidDeleteCacheParamsError, DeleteCacheFailedError, } from "../../utils/errors-client.js";
 /**
  * Deletes KV cache files.
  *
@@ -35,11 +36,11 @@ export async function deleteCache(params) {
         };
     }
     else {
-        throw new Error("Invalid deleteCache parameters");
+        throw new InvalidDeleteCacheParamsError();
     }
     const response = (await send(req));
     if (!response.success && response.error) {
-        throw new Error(response.error);
+        throw new DeleteCacheFailedError(response.error);
     }
     return {
         success: response.success,

package/dist/client/api/download-asset.d.ts

@@ -14,9 +14,9 @@ export type DownloadAssetOptions = BaseDownloadAssetOptions;
  *
  * @returns Promise that resolves to the asset ID (either the provided assetSrc or a generated ID)
  *
- * @throws {Error} When asset download fails, with details in the error message
- * @throws {Error} When streaming ends unexpectedly (only when using onProgress)
- * @throws {Error} When receiving an invalid response type from the server
+ * @throws {QvacErrorBase} When asset download fails, with details in the error message
+ * @throws {QvacErrorBase} When streaming ends unexpectedly (only when using onProgress)
+ * @throws {QvacErrorBase} When receiving an invalid response type from the server
  *
  * @example
  * ```typescript

package/dist/client/api/download-asset.js

@@ -1,5 +1,6 @@
 import { send, stream } from "../rpc/rpc-client.js";
-import { downloadAssetRequestSchema, } from "../../schemas/index.js";
+import { downloadAssetOptionsToRequestSchema, } from "../../schemas/index.js";
+import { DownloadAssetFailedError, StreamEndedError, InvalidResponseError, } from "../../utils/errors-client.js";
 /**
  * Downloads an asset (model file) without loading it into memory.
  *
@@ -14,9 +15,9 @@ import { downloadAssetRequestSchema, } from "../../schemas/index.js";
  *
  * @returns Promise that resolves to the asset ID (either the provided assetSrc or a generated ID)
  *
- * @throws {Error} When asset download fails, with details in the error message
- * @throws {Error} When streaming ends unexpectedly (only when using onProgress)
- * @throws {Error} When receiving an invalid response type from the server
+ * @throws {QvacErrorBase} When asset download fails, with details in the error message
+ * @throws {QvacErrorBase} When streaming ends unexpectedly (only when using onProgress)
+ * @throws {QvacErrorBase} When receiving an invalid response type from the server
  *
  * @example
  * ```typescript
@@ -36,13 +37,7 @@ import { downloadAssetRequestSchema, } from "../../schemas/index.js";
  * ```
  */
 export async function downloadAsset(options) {
-    const { assetSrc } = options;
-    const request = downloadAssetRequestSchema.parse({
-        type: "downloadAsset",
-        assetSrc: assetSrc,
-        withProgress: !!options.onProgress,
-        seed: options.seed,
-    });
+    const request = downloadAssetOptionsToRequestSchema.parse(options);
     if (options.onProgress) {
         // Use streaming for progress updates
         for await (const response of stream(request)) {
@@ -51,22 +46,22 @@ export async function downloadAsset(options) {
             }
             else if (response.type === "downloadAsset") {
                 if (!response.success) {
-                    throw new Error(response.error ?? "Failed to download asset");
+                    throw new DownloadAssetFailedError(response.error);
                 }
-                return response.assetId ?? assetSrc;
+                return response.assetId;
             }
         }
-        throw new Error("Stream ended without final response");
+        throw new StreamEndedError();
     }
     else {
         // Use regular send for simple downloading
         const response = await send(request);
         if (response.type !== "downloadAsset") {
-            throw new Error("Invalid response type");
+            throw new InvalidResponseError("downloadAsset");
         }
         if (!response.success) {
-            throw new Error(response.error ?? "Failed to download asset");
+            throw new DownloadAssetFailedError(response.error);
         }
-        return response.assetId ?? assetSrc;
+        return response.assetId;
     }
 }

package/dist/client/api/embed.d.ts

@@ -5,7 +5,7 @@ import { type EmbedParams } from "../../schemas/index";
  * @param params - The parameters for the embedding
  * @param params.modelId - The identifier of the embedding model to use
  * @param params.text - The input text to embed
- * @throws {Error} When the response type is invalid or when the embedding fails
+ * @throws {QvacErrorBase} When the response type is invalid or when the embedding fails
  */
 export declare function embed(params: EmbedParams): Promise<number[]>;
 //# sourceMappingURL=embed.d.ts.map

package/dist/client/api/embed.js

@@ -1,12 +1,13 @@
 import { send } from "../rpc/rpc-client.js";
 import {} from "../../schemas/index.js";
+import { InvalidResponseError, EmbedFailedError } from "../../utils/errors-client.js";
 /**
  * Generates embeddings for a given text using a specified model.
  *
  * @param params - The parameters for the embedding
  * @param params.modelId - The identifier of the embedding model to use
  * @param params.text - The input text to embed
- * @throws {Error} When the response type is invalid or when the embedding fails
+ * @throws {QvacErrorBase} When the response type is invalid or when the embedding fails
  */
 export async function embed(params) {
     const request = {
@@ -15,10 +16,10 @@ export async function embed(params) {
     };
     const response = await send(request);
     if (response.type !== "embed") {
-        throw new Error("Invalid response type");
+        throw new InvalidResponseError("embed");
     }
     if (!response.success) {
-        throw new Error(response.error ?? "Failed to embed");
+        throw new EmbedFailedError(response.error);
     }
     return response.embedding;
 }

package/dist/client/api/get-model-info.d.ts

@@ -0,0 +1,29 @@
+import { type GetModelInfoParams } from "../../schemas/index";
+export declare const getModelInfo: (params: GetModelInfoParams) => Promise<{
+    name: string;
+    modelId: string;
+    hyperdriveKey: string;
+    hyperbeeKey: string;
+    expectedSize: number;
+    sha256Checksum: string;
+    addon: "llm" | "whisper" | "embeddings" | "nmt" | "vad" | "tts";
+    isCached: boolean;
+    isLoaded: boolean;
+    cacheFiles: {
+        filename: string;
+        path: string;
+        expectedSize: number;
+        sha256Checksum: string;
+        isCached: boolean;
+        actualSize?: number | undefined;
+        cachedAt?: Date | undefined;
+    }[];
+    actualSize?: number | undefined;
+    cachedAt?: Date | undefined;
+    loadedInstances?: {
+        registryId: string;
+        loadedAt: Date;
+        config?: unknown;
+    }[] | undefined;
+}>;
+//# sourceMappingURL=get-model-info.d.ts.map

package/dist/client/api/get-model-info.js

@@ -0,0 +1,14 @@
+import {} from "../../schemas/index.js";
+import { send } from "../rpc/rpc-client.js";
+import { InvalidResponseError } from "../../utils/errors-client.js";
+export const getModelInfo = async (params) => {
+    const request = {
+        type: "getModelInfo",
+        name: params.name,
+    };
+    const response = await send(request);
+    if (response.type !== "getModelInfo") {
+        throw new InvalidResponseError("getModelInfo");
+    }
+    return response.modelInfo;
+};

package/dist/client/api/index.d.ts

@@ -12,4 +12,7 @@ export { cancel } from "./cancel";
 export { startQVACProvider } from "./provide";
 export { stopQVACProvider } from "./stop-provider";
 export { ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings } from "./rag";
+export { textToSpeech } from "./text-to-speech";
+export { setConfig } from "./set-config";
+export { getModelInfo } from "./get-model-info";
 //# sourceMappingURL=index.d.ts.map

package/dist/client/api/index.js

@@ -12,3 +12,6 @@ export { cancel } from "./cancel.js";
 export { startQVACProvider } from "./provide.js";
 export { stopQVACProvider } from "./stop-provider.js";
 export { ragSaveEmbeddings, ragSearch, ragDeleteEmbeddings } from "./rag.js";
+export { textToSpeech } from "./text-to-speech.js";
+export { setConfig } from "./set-config.js";
+export { getModelInfo } from "./get-model-info.js";

package/dist/client/api/load-model.d.ts

@@ -21,9 +21,9 @@ export type LoadModelOptions = BaseLoadModelOptions & {
  *
  * @returns Promise that resolves to the model ID (either the provided modelSrc or a generated ID)
  *
- * @throws {Error} When model loading fails, with details in the error message
- * @throws {Error} When streaming ends unexpectedly (only when using onProgress)
- * @throws {Error} When receiving an invalid response type from the server
+ * @throws {QvacErrorBase} When model loading fails, with details in the error message
+ * @throws {QvacErrorBase} When streaming ends unexpectedly (only when using onProgress)
+ * @throws {QvacErrorBase} When receiving an invalid response type from the server
  *
  * @example
  * ```typescript