@xiaozhiclaw/provider-core 0.1.0

package/dist/transports/realtime-transport.js

@@ -0,0 +1,332 @@
+/**
+ * Realtime WebSocket Transport 鈥?bidirectional audio/voice streaming
+ * via the OpenAI Realtime API protocol (also compatible with GLM Realtime).
+ *
+ * ## Protocol: WebSocket JSON events
+ *
+ * Client 鈫?Server:
+ *   - session.update: configure session (model, voice, tools, etc.)
+ *   - input_audio_buffer.append: send audio chunks (base64 PCM16)
+ *   - input_audio_buffer.commit: signal end of audio input
+ *   - conversation.item.create: inject text/function_result items
+ *   - response.create: request a model response
+ *   - response.cancel: abort in-progress response
+ *
+ * Server 鈫?Client:
+ *   - session.created: session initialized
+ *   - session.updated: config acknowledged
+ *   - input_audio_buffer.speech_started: VAD detected speech
+ *   - input_audio_buffer.speech_stopped: VAD detected silence
+ *   - response.created: response generation started
+ *   - response.output_item.added: new output item (text/audio/function_call)
+ *   - response.audio.delta: audio chunk (base64 PCM16)
+ *   - response.audio_transcript.delta: transcript of generated speech
+ *   - response.text.delta: text generation delta
+ *   - response.function_call_arguments.delta: tool call args delta
+ *   - response.function_call_arguments.done: tool call complete
+ *   - response.output_item.done: output item finished
+ *   - response.done: full response complete
+ *   - error: server error
+ *
+ * ## Architecture
+ *
+ * RealtimeTransport manages a single persistent WebSocket connection per session.
+ * It exposes an event-driven API (AsyncGenerator) that the agent tool-loop
+ * can consume for voice-enabled interactions.
+ *
+ * Docs:
+ *   - OpenAI: https://platform.openai.com/docs/api-reference/realtime
+ *   - GLM:    https://docs.bigmodel.cn/cn/guide/develop/realtime-api
+ */
+// 鈹€鈹€ Transport 鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€
+/**
+ * Manages a persistent WebSocket connection for real-time audio/voice
+ * interactions with an LLM provider.
+ *
+ * Usage:
+ * ```ts
+ * const rt = new RealtimeTransport(config);
+ * rt.connect();
+ *
+ * // Send audio
+ * rt.appendAudio(base64Chunk);
+ * rt.commitAudio();
+ *
+ * // Or send text
+ * rt.sendText("Hello!");
+ *
+ * // Submit function results
+ * rt.sendFunctionResult(callId, result);
+ *
+ * // Consume events
+ * for await (const event of rt.events()) {
+ *   switch (event.type) {
+ *     case "audio_delta": playAudio(event.delta); break;
+ *     case "function_call_done": handleToolCall(event); break;
+ *   }
+ * }
+ *
+ * rt.close();
+ * ```
+ */
+export class RealtimeTransport {
+    ws = null;
+    config;
+    eventQueue = [];
+    waiters = [];
+    closed = false;
+    constructor(config) {
+        this.config = config;
+    }
+    /** Open WebSocket connection and configure session. */
+    async connect() {
+        const url = this.buildUrl();
+        // Use native WebSocket (Node 22+ has global WebSocket)
+        // For older Node, set globalThis.WebSocket from 'ws' package before calling
+        const WS = globalThis.WebSocket;
+        if (!WS)
+            throw new Error("WebSocket not available. Node 22+ required or polyfill globalThis.WebSocket.");
+        // Pass auth headers via protocol sub-protocol trick (OpenAI) or URL query (GLM)
+        const protocols = this.config.authMode === "header"
+            ? ["realtime", `openai-insecure-api-key.${this.config.apiKey}`, "openai-beta.realtime-v1"]
+            : undefined;
+        this.ws = new WS(url, protocols);
+        this.ws.onmessage = (event) => {
+            try {
+                const data = typeof event.data === "string" ? event.data : String(event.data);
+                const msg = JSON.parse(data);
+                const events = this.parseServerEvent(msg);
+                for (const ev of events) {
+                    this.push(ev);
+                }
+            }
+            catch {
+                this.push({ type: "error", code: "parse_error", message: "Failed to parse server event" });
+            }
+        };
+        this.ws.onclose = (event) => {
+            this.push({ type: "closed", code: event.code, reason: event.reason });
+            this.closed = true;
+            this.drainWaiters();
+        };
+        this.ws.onerror = () => {
+            this.push({ type: "error", code: "ws_error", message: "WebSocket connection error" });
+        };
+        // Wait for connection to be established
+        await new Promise((resolve, reject) => {
+            this.ws.onopen = () => {
+                this.sendSessionUpdate();
+                resolve();
+            };
+            this.ws.onerror = () => reject(new Error("WebSocket connection failed"));
+        });
+    }
+    /** Send audio data (base64 PCM16). */
+    appendAudio(base64Chunk) {
+        this.send({
+            type: "input_audio_buffer.append",
+            audio: base64Chunk,
+        });
+    }
+    /** Mark end of audio input and trigger response. */
+    commitAudio() {
+        this.send({ type: "input_audio_buffer.commit" });
+    }
+    /** Send a text message. */
+    sendText(text) {
+        this.send({
+            type: "conversation.item.create",
+            item: {
+                type: "message",
+                role: "user",
+                content: [{ type: "input_text", text }],
+            },
+        });
+    }
+    /** Submit a function call result back to the model. */
+    sendFunctionResult(callId, output) {
+        this.send({
+            type: "conversation.item.create",
+            item: {
+                type: "function_call_output",
+                call_id: callId,
+                output,
+            },
+        });
+        // Request a new response after submitting the result
+        this.send({ type: "response.create" });
+    }
+    /** Trigger a model response (e.g. after sending text). */
+    requestResponse() {
+        this.send({ type: "response.create" });
+    }
+    /** Cancel an in-progress response. */
+    cancelResponse() {
+        this.send({ type: "response.cancel" });
+    }
+    /** Async iterator of server events. */
+    async *events() {
+        while (!this.closed || this.eventQueue.length > 0) {
+            if (this.eventQueue.length > 0) {
+                yield this.eventQueue.shift();
+            }
+            else {
+                const event = await new Promise((resolve) => {
+                    this.waiters.push(resolve);
+                });
+                if (event.done)
+                    return;
+                yield event.value;
+            }
+        }
+    }
+    /** Close the WebSocket connection. */
+    close() {
+        this.closed = true;
+        if (this.ws && this.ws.readyState !== WebSocket.CLOSED) {
+            this.ws.close(1000, "client_close");
+        }
+        this.drainWaiters();
+    }
+    // 鈹€鈹€ Private 鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€
+    buildUrl() {
+        let url = this.config.baseUrl;
+        if (this.config.authMode === "query") {
+            // GLM style: pass model + API key as query params
+            const sep = url.includes("?") ? "&" : "?";
+            url += `${sep}model=${encodeURIComponent(this.config.model)}&token=${encodeURIComponent(this.config.apiKey)}`;
+        }
+        else {
+            // OpenAI style: model in query, auth via sub-protocol
+            const sep = url.includes("?") ? "&" : "?";
+            url += `${sep}model=${encodeURIComponent(this.config.model)}`;
+        }
+        return url;
+    }
+    sendSessionUpdate() {
+        this.send({
+            type: "session.update",
+            session: {
+                model: this.config.model,
+                voice: this.config.voice ?? "alloy",
+                modalities: this.config.outputModalities ?? ["text", "audio"],
+                instructions: "You are a helpful assistant.",
+                temperature: this.config.temperature ?? 0.8,
+                tools: this.config.tools?.map(t => ({
+                    type: t.type,
+                    name: t.name,
+                    description: t.description,
+                    parameters: t.parameters,
+                })) ?? [],
+                turn_detection: this.config.vadMode === "none"
+                    ? null
+                    : {
+                        type: "server_vad",
+                        threshold: this.config.vadThreshold ?? 0.5,
+                        prefix_padding_ms: 300,
+                        silence_duration_ms: 500,
+                    },
+            },
+        });
+    }
+    send(msg) {
+        if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+            throw new Error("WebSocket not connected");
+        }
+        this.ws.send(JSON.stringify(msg));
+    }
+    push(event) {
+        if (this.waiters.length > 0) {
+            const waiter = this.waiters.shift();
+            waiter({ value: event, done: false });
+        }
+        else {
+            this.eventQueue.push(event);
+        }
+    }
+    drainWaiters() {
+        for (const waiter of this.waiters) {
+            waiter({ value: undefined, done: true });
+        }
+        this.waiters.length = 0;
+    }
+    /**
+     * Parse a server-sent JSON event into our typed event(s).
+     */
+    parseServerEvent(msg) {
+        const type = msg.type;
+        switch (type) {
+            case "session.created":
+                return [{ type: "session_created", sessionId: String(msg.session?.id ?? "") }];
+            case "input_audio_buffer.speech_started":
+                return [{ type: "speech_started" }];
+            case "input_audio_buffer.speech_stopped":
+                return [{ type: "speech_stopped", audioEndMs: Number(msg.audio_end_ms ?? 0) }];
+            case "response.audio.delta":
+                return [{ type: "audio_delta", delta: String(msg.delta ?? "") }];
+            case "response.audio_transcript.delta":
+                return [{ type: "audio_transcript_delta", delta: String(msg.delta ?? "") }];
+            case "response.text.delta":
+                return [{ type: "text_delta", delta: String(msg.delta ?? "") }];
+            case "response.function_call_arguments.delta": {
+                return [{
+                        type: "function_call_delta",
+                        callId: String(msg.call_id ?? ""),
+                        delta: String(msg.delta ?? ""),
+                    }];
+            }
+            case "response.function_call_arguments.done": {
+                return [{
+                        type: "function_call_done",
+                        callId: String(msg.call_id ?? ""),
+                        name: String(msg.name ?? ""),
+                        arguments: String(msg.arguments ?? ""),
+                    }];
+            }
+            case "response.output_item.added": {
+                const item = msg.item;
+                if (item?.type === "function_call") {
+                    return [{
+                            type: "function_call_start",
+                            callId: String(item.call_id ?? ""),
+                            name: String(item.name ?? ""),
+                        }];
+                }
+                return [];
+            }
+            case "response.done": {
+                const response = msg.response;
+                const usage = response?.usage;
+                return [{
+                        type: "response_done",
+                        usage: usage ? {
+                            inputTokens: usage.input_tokens ?? 0,
+                            outputTokens: usage.output_tokens ?? 0,
+                            inputAudioTokens: response?.input_token_details?.audio_tokens,
+                            outputAudioTokens: response?.output_token_details?.audio_tokens,
+                        } : undefined,
+                    }];
+            }
+            case "error": {
+                const error = msg.error;
+                return [{
+                        type: "error",
+                        code: String(error?.code ?? "unknown"),
+                        message: String(error?.message ?? "Unknown error"),
+                    }];
+            }
+            // Ignored events (acknowledgements, intermediate states)
+            case "session.updated":
+            case "response.created":
+            case "response.output_item.done":
+            case "conversation.item.created":
+            case "input_audio_buffer.committed":
+            case "input_audio_buffer.cleared":
+            case "rate_limits.updated":
+                return [];
+            default:
+                // Unknown event 鈥?silently ignore for forward compatibility
+                return [];
+        }
+    }
+}

package/dist/transports/volcengine-grounding.d.ts

@@ -0,0 +1,58 @@
+/**
+ * Volcengine Grounding 鈥?spatial coordinate parser (volcengine-ProviderMax 搂14).
+ *
+ * Parses model-emitted spatial reference tags from text output:
+ *   - <bbox>x_min y_min x_max y_max</bbox>   鈫?bounding box
+ *   - <point>x y</point>                       鈫?single point
+ *   - <polygon>x1 y1 x2 y2 ...</polygon>       鈫?polygon vertices
+ *
+ * All coordinates are in normalized 1000脳1000 space, range [0, 999].
+ * Use `toPixelCoords()` to convert to actual image pixel coordinates.
+ */
+export type SpatialReference = {
+    type: "bbox";
+    x1: number;
+    y1: number;
+    x2: number;
+    y2: number;
+    space: "normalized_1000";
+} | {
+    type: "point";
+    x: number;
+    y: number;
+    space: "normalized_1000";
+} | {
+    type: "polygon";
+    points: Array<{
+        x: number;
+        y: number;
+    }>;
+    space: "normalized_1000";
+};
+export interface PixelBbox {
+    x1: number;
+    y1: number;
+    x2: number;
+    y2: number;
+}
+export interface PixelPoint {
+    x: number;
+    y: number;
+}
+/**
+ * Extract all spatial references from model output text.
+ * Returns an empty array if no grounding tags are found.
+ */
+export declare function parseGroundingTags(text: string): SpatialReference[];
+/**
+ * Convert a normalized 1000脳1000 bounding box to pixel coordinates.
+ */
+export declare function bboxToPixels(ref: Extract<SpatialReference, {
+    type: "bbox";
+}>, width: number, height: number): PixelBbox;
+/**
+ * Convert a normalized 1000脳1000 point to pixel coordinates.
+ */
+export declare function pointToPixels(ref: Extract<SpatialReference, {
+    type: "point";
+}>, width: number, height: number): PixelPoint;

package/dist/transports/volcengine-grounding.js

@@ -0,0 +1,69 @@
+/**
+ * Volcengine Grounding 鈥?spatial coordinate parser (volcengine-ProviderMax 搂14).
+ *
+ * Parses model-emitted spatial reference tags from text output:
+ *   - <bbox>x_min y_min x_max y_max</bbox>   鈫?bounding box
+ *   - <point>x y</point>                       鈫?single point
+ *   - <polygon>x1 y1 x2 y2 ...</polygon>       鈫?polygon vertices
+ *
+ * All coordinates are in normalized 1000脳1000 space, range [0, 999].
+ * Use `toPixelCoords()` to convert to actual image pixel coordinates.
+ */
+// 鈹€鈹€ Parsing 鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€
+const BBOX_RE = /<bbox>\s*(\d+)\s+(\d+)\s+(\d+)\s+(\d+)\s*<\/bbox>/g;
+const POINT_RE = /<point>\s*(\d+)\s+(\d+)\s*<\/point>/g;
+const POLYGON_RE = /<polygon>\s*([\d\s]+?)\s*<\/polygon>/g;
+/**
+ * Extract all spatial references from model output text.
+ * Returns an empty array if no grounding tags are found.
+ */
+export function parseGroundingTags(text) {
+    const results = [];
+    for (const m of text.matchAll(BBOX_RE)) {
+        const [x1, y1, x2, y2] = [+m[1], +m[2], +m[3], +m[4]];
+        if (isValidCoord(x1) && isValidCoord(y1) && isValidCoord(x2) && isValidCoord(y2)) {
+            results.push({ type: "bbox", x1, y1, x2, y2, space: "normalized_1000" });
+        }
+    }
+    for (const m of text.matchAll(POINT_RE)) {
+        const [x, y] = [+m[1], +m[2]];
+        if (isValidCoord(x) && isValidCoord(y)) {
+            results.push({ type: "point", x, y, space: "normalized_1000" });
+        }
+    }
+    for (const m of text.matchAll(POLYGON_RE)) {
+        const nums = m[1].trim().split(/\s+/).map(Number);
+        if (nums.length >= 4 && nums.length % 2 === 0 && nums.every(isValidCoord)) {
+            const points = [];
+            for (let i = 0; i < nums.length; i += 2) {
+                points.push({ x: nums[i], y: nums[i + 1] });
+            }
+            results.push({ type: "polygon", points, space: "normalized_1000" });
+        }
+    }
+    return results;
+}
+// 鈹€鈹€ Coordinate Conversion 鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€鈹€
+/**
+ * Convert a normalized 1000脳1000 bounding box to pixel coordinates.
+ */
+export function bboxToPixels(ref, width, height) {
+    return {
+        x1: Math.round(ref.x1 / 1000 * width),
+        y1: Math.round(ref.y1 / 1000 * height),
+        x2: Math.round(ref.x2 / 1000 * width),
+        y2: Math.round(ref.y2 / 1000 * height),
+    };
+}
+/**
+ * Convert a normalized 1000脳1000 point to pixel coordinates.
+ */
+export function pointToPixels(ref, width, height) {
+    return {
+        x: Math.round(ref.x / 1000 * width),
+        y: Math.round(ref.y / 1000 * height),
+    };
+}
+function isValidCoord(n) {
+    return Number.isInteger(n) && n >= 0 && n <= 999;
+}

package/dist/transports/volcengine-media.d.ts

@@ -0,0 +1,94 @@
+/**
+ * Volcengine Media Transport 鈥?Doubao Seedream (image), Seedance (video), 3D generation.
+ *
+ * API reference:
+ *   Image:  POST /v3/images/generations            (sync)
+ *   Video:  POST /v3/contents/generations/tasks     (async job)
+ *   3D:     POST /v3/contents/generations/tasks     (async job, same endpoint as video)
+ *
+ * Auth: Authorization: Bearer $ARK_API_KEY
+ * Docs: https://www.volcengine.com/docs/82379/1330310
+ *       https://www.volcengine.com/docs/82379/1874993 (3D)
+ */
+import type { AsyncMediaTransport, MediaRequest, MediaResult, MediaType } from "../media-transport.js";
+export interface VolcengineMediaConfig {
+    /** Base URL, e.g. "https://ark.cn-beijing.volces.com/api" */
+    baseUrl: string;
+    timeoutMs?: number;
+}
+export declare class VolcengineMediaTransport implements AsyncMediaTransport {
+    readonly supportedTypes: readonly MediaType[];
+    private baseUrl;
+    private timeoutMs;
+    constructor(config: VolcengineMediaConfig);
+    generate(request: MediaRequest, apiKey: string, signal?: AbortSignal): Promise<MediaResult>;
+    private generateEmbedding;
+    /**
+     * Check if this transport can handle a given operation.
+     * Video edit/merge/upscale are routed through the same video endpoint.
+     */
+    canHandle(request: MediaRequest): boolean;
+    private generateImage;
+    /**
+     * Parse streaming image SSE 鈥?yields progressive image quality upgrades.
+     * Final event contains the full-quality image URL.
+     */
+    private parseStreamingImage;
+    private generateVideo;
+    private generate3D;
+    /**
+     * Query a single video generation task by ID.
+     * GET /v3/contents/generations/tasks/{taskId}
+     */
+    getTaskStatus(taskId: string, apiKey: string, signal?: AbortSignal): Promise<{
+        status: string;
+        task: Record<string, unknown>;
+    }>;
+    /**
+     * List video generation tasks with optional filters.
+     * GET /v3/contents/generations/tasks
+     */
+    listVideoTasks(apiKey: string, options?: {
+        after?: string;
+        limit?: number;
+        status?: string;
+    }, signal?: AbortSignal): Promise<Record<string, unknown>>;
+    /**
+     * Cancel or delete a video generation task.
+     * DELETE /v3/contents/generations/tasks/{taskId}
+     */
+    deleteVideoTask(taskId: string, apiKey: string, signal?: AbortSignal): Promise<void>;
+    /**
+     * Upload a file to Volcengine Files API for reuse in multimodal requests.
+     * POST /v3/files
+     */
+    uploadFile(file: Blob | Buffer, apiKey: string, options?: {
+        purpose?: string;
+        filename?: string;
+    }, signal?: AbortSignal): Promise<{
+        id: string;
+        status: string;
+    }>;
+    /**
+     * Get file info by ID.
+     * GET /v3/files/{fileId}
+     */
+    getFile(fileId: string, apiKey: string, signal?: AbortSignal): Promise<Record<string, unknown>>;
+    /**
+     * List uploaded files.
+     * GET /v3/files
+     */
+    listFiles(apiKey: string, options?: {
+        after?: string;
+        limit?: number;
+        purpose?: string;
+        order?: "asc" | "desc";
+    }, signal?: AbortSignal): Promise<Record<string, unknown>>;
+    /**
+     * Delete a file.
+     * DELETE /v3/files/{fileId}
+     */
+    deleteFile(fileId: string, apiKey: string, signal?: AbortSignal): Promise<void>;
+    private submitTask;
+    private pollTask;
+}