@tanstack-ai-browser/apfel 0.0.0-stage → 0.0.1

package/dist/adapter.js

@@ -0,0 +1,584 @@
+/**
+ * A TanStack AI text adapter for Apple's on-device Foundation Model, served
+ * by apfel's OpenAI-compatible server (`apfel --serve --cors`). No backend
+ * involved: the browser streams straight from localhost. Hand it to `chat()`
+ * like any hosted model's adapter: `chat()` runs tools and the agent loop,
+ * and each `chatStream` call here is one completion from apfel, streamed
+ * back as AG-UI events.
+ */
+import { EventType, normalizeSystemPrompts } from "@tanstack/ai";
+import { BaseTextAdapter } from "@tanstack/ai/adapters";
+import { JSON_TOOL_CALLS_MARKER, createToolCallSplitter, parseJsonToolCalls } from "./tool-calls.js";
+/** Where `apfel --serve` listens by default. */
+export const APFEL_BASE_URL = "http://127.0.0.1:11434/v1";
+/** The model id apfel serves Apple's Foundation Model under. */
+export const APFEL_MODEL = "apple-foundationmodel";
+/**
+ * A tool result for the model. A failed tool gives its error message, so the
+ * model can see what went wrong and try again.
+ */
+function toolResultText(m) {
+    const text = contentText(m.content);
+    if (m.error)
+        return `Error: ${m.error}`;
+    return text && text !== "null" ? text : "The tool returned nothing.";
+}
+function contentText(content) {
+    if (typeof content === "string")
+        return content;
+    return (content ?? [])
+        .map((part) => (part.type === "text" ? part.content : ""))
+        .join("");
+}
+/**
+ * The conversation in OpenAI's chat format. Tool calls and results from
+ * earlier turns are left out; their answers stay. With them in the history,
+ * Apple's model writes its next tool call as broken JSON text most of the
+ * time (4 of 12 calls worked, against 12 of 12 without). The current turn's
+ * calls and results are kept: the model answers from them.
+ */
+function toOpenAIMessages(messages) {
+    const out = [];
+    const currentTurn = messages.map((m) => m.role).lastIndexOf("user");
+    for (const [i, m] of messages.entries()) {
+        const earlierToolStep = i < currentTurn &&
+            (m.role === "tool" || (m.role === "assistant" && m.toolCalls?.length));
+        const content = contentText(m.content);
+        if (earlierToolStep) {
+            if (m.role === "assistant" && content) {
+                out.push({ role: "assistant", content });
+            }
+        }
+        else if (m.role === "tool") {
+            out.push({
+                role: "tool",
+                tool_call_id: m.toolCallId ?? "",
+                content: toolResultText(m)
+            });
+        }
+        else if (m.role === "assistant" && m.toolCalls?.length) {
+            out.push({
+                role: "assistant",
+                content: content || null,
+                tool_calls: m.toolCalls.map((call) => ({
+                    id: call.id,
+                    type: "function",
+                    function: call.function
+                }))
+            });
+        }
+        else if (content) {
+            out.push({ role: m.role === "user" ? "user" : "assistant", content });
+        }
+    }
+    return out;
+}
+/** Tools as OpenAI function tools; `inputSchema` is already JSON Schema. */
+function toOpenAITools(tools) {
+    return (tools ?? []).map((tool) => ({
+        type: "function",
+        function: {
+            name: tool.name,
+            description: tool.description,
+            parameters: tool.inputSchema ?? {
+                type: "object",
+                properties: {},
+                required: []
+            }
+        }
+    }));
+}
+/**
+ * `tool_choice` for the request. "auto" is apfel's default and is left out,
+ * so a plain request is sent exactly as before.
+ */
+function toolChoice(choice) {
+    if (!choice || choice === "auto")
+        return {};
+    if (typeof choice === "string")
+        return { tool_choice: choice };
+    return {
+        tool_choice: { type: "function", function: { name: choice.name } }
+    };
+}
+/** The system prompts as one system message, if there are any. */
+function systemMessages(prompts) {
+    const system = normalizeSystemPrompts(prompts)
+        .map((p) => p.content)
+        .join("\n\n");
+    return system ? [{ role: "system", content: system }] : [];
+}
+/** Yields the `data:` payloads of an SSE response body. */
+async function* sseData(body) {
+    const reader = body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+    try {
+        for (;;) {
+            const { value, done } = await reader.read();
+            if (done)
+                break;
+            buffer += decoder.decode(value, { stream: true });
+            let newline;
+            while ((newline = buffer.indexOf("\n")) >= 0) {
+                const line = buffer.slice(0, newline).trim();
+                buffer = buffer.slice(newline + 1);
+                if (line.startsWith("data:"))
+                    yield line.slice(5).trim();
+            }
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+}
+/** A tool call's JSON arguments, or undefined if they aren't a JSON object. */
+function parseArguments(json) {
+    try {
+        const args = JSON.parse(json);
+        return args && typeof args === "object" && !Array.isArray(args)
+            ? args
+            : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+/** The message of apfel's `{"error": {"message"}}` body, or the raw body. */
+async function errorBody(response) {
+    const raw = await response.text().catch(() => "");
+    try {
+        const message = JSON.parse(raw).error?.message;
+        if (message)
+            return message;
+    }
+    catch {
+        // Not JSON: the raw body says what it says.
+    }
+    return raw.slice(0, 200);
+}
+function usageOf(usage) {
+    if (usage?.prompt_tokens === undefined ||
+        usage.completion_tokens === undefined) {
+        return undefined;
+    }
+    const cachedTokens = usage.prompt_tokens_details?.cached_tokens;
+    return {
+        promptTokens: usage.prompt_tokens,
+        completionTokens: usage.completion_tokens,
+        totalTokens: usage.prompt_tokens + usage.completion_tokens,
+        ...(cachedTokens && { promptTokensDetails: { cachedTokens } })
+    };
+}
+const errorText = (error) => error instanceof Error ? error.message : String(error);
+const unreachable = (baseUrl) => `Couldn't reach Apple's on-device model at ${baseUrl}. Start it with \`apfel --serve --cors\`.`;
+let ids = 0;
+const nextId = (prefix) => `${prefix}_${Date.now().toString(36)}_${(ids++).toString(36)}`;
+export class ApfelTextAdapter extends BaseTextAdapter {
+    name = "apfel";
+    api = "apfel";
+    baseUrl;
+    onStats;
+    constructor(options = {}) {
+        super({}, options.model ?? APFEL_MODEL);
+        this.baseUrl = (options.baseUrl ?? APFEL_BASE_URL).replace(/\/+$/, "");
+        this.onStats = options.onStats;
+    }
+    async *chatStream(options) {
+        const runId = options.runId ?? nextId("run");
+        const threadId = options.threadId;
+        const callId = nextId("call");
+        const messageId = nextId("msg");
+        const reasoningId = nextId("reasoning");
+        const model = this.model;
+        const { logger } = options;
+        const signal = options.abortController?.signal ??
+            options.request?.signal ??
+            undefined;
+        const timestamp = () => Date.now();
+        const runError = (message) => ({
+            type: EventType.RUN_ERROR,
+            model,
+            timestamp: timestamp(),
+            message,
+            error: { message }
+        });
+        yield {
+            type: EventType.RUN_STARTED,
+            runId,
+            threadId,
+            model,
+            timestamp: timestamp(),
+            ...(options.parentRunId && { parentRunId: options.parentRunId })
+        };
+        const tools = toOpenAITools(options.tools);
+        const toolNames = new Set(tools.map((tool) => tool.function.name));
+        const send = options.wrapFetch ? options.wrapFetch(fetch) : fetch;
+        const url = `${this.baseUrl}/chat/completions`;
+        logger.request(`activity=chat provider=apfel model=${model} messages=${options.messages.length} tools=${tools.length} stream=true`, { provider: "apfel", model });
+        const sentAt = performance.now();
+        let response;
+        try {
+            response = await send(url, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({
+                    ...options.modelOptions,
+                    model,
+                    stream: true,
+                    // Token counts, for the stats.
+                    stream_options: { include_usage: true },
+                    messages: [
+                        ...systemMessages(options.systemPrompts),
+                        ...toOpenAIMessages(options.messages)
+                    ],
+                    ...(tools.length > 0 && {
+                        tools,
+                        ...toolChoice(options.toolChoice)
+                    })
+                }),
+                signal
+            });
+        }
+        catch (error) {
+            if (signal?.aborted)
+                return;
+            logger.errors("apfel.chatStream fetch failed", {
+                error,
+                source: "apfel.chatStream"
+            });
+            yield runError(unreachable(this.baseUrl));
+            return;
+        }
+        if (!response.ok || !response.body) {
+            yield runError(`apfel returned HTTP ${response.status}: ${await errorBody(response)}`);
+            return;
+        }
+        // Reasoning (apfel sends none for Apple's model today) streams as
+        // reasoning events, then the answer as text up to any tool call the
+        // model wrote as text, which is held back and parsed at the end.
+        let reasoning = "none";
+        let textStarted = false;
+        let text = "";
+        const events = [];
+        const endReasoning = () => {
+            if (reasoning !== "open")
+                return;
+            reasoning = "closed";
+            events.push({
+                type: EventType.REASONING_MESSAGE_END,
+                messageId: reasoningId,
+                model,
+                timestamp: timestamp()
+            }, {
+                type: EventType.REASONING_END,
+                messageId: reasoningId,
+                model,
+                timestamp: timestamp()
+            });
+        };
+        const addReasoning = (delta) => {
+            if (reasoning === "none") {
+                events.push({
+                    type: EventType.REASONING_START,
+                    messageId: reasoningId,
+                    model,
+                    timestamp: timestamp()
+                }, {
+                    type: EventType.REASONING_MESSAGE_START,
+                    messageId: reasoningId,
+                    role: "reasoning",
+                    model,
+                    timestamp: timestamp()
+                });
+            }
+            reasoning = "open";
+            events.push({
+                type: EventType.REASONING_MESSAGE_CONTENT,
+                messageId: reasoningId,
+                delta,
+                model,
+                timestamp: timestamp()
+            });
+        };
+        const addText = (delta) => {
+            if (!delta)
+                return;
+            endReasoning();
+            if (!textStarted) {
+                textStarted = true;
+                events.push({
+                    type: EventType.TEXT_MESSAGE_START,
+                    messageId,
+                    role: "assistant",
+                    model,
+                    timestamp: timestamp()
+                });
+            }
+            text += delta;
+            events.push({
+                type: EventType.TEXT_MESSAGE_CONTENT,
+                messageId,
+                delta,
+                content: text,
+                model,
+                timestamp: timestamp()
+            });
+        };
+        const endText = () => {
+            if (!textStarted)
+                return;
+            textStarted = false;
+            events.push({
+                type: EventType.TEXT_MESSAGE_END,
+                messageId,
+                model,
+                timestamp: timestamp()
+            });
+        };
+        // When apfel doesn't recognize a tool call, it passes it on as text;
+        // that part is held back and read once the reply ends.
+        const splitter = createToolCallSplitter(JSON_TOOL_CALLS_MARKER);
+        // Tool calls stream in pieces, by index.
+        const streamed = [];
+        // apfel streams a few tokens per chunk: the first chunk's share of the
+        // characters stands in for its share of the tokens.
+        const timing = { first: 0, last: 0, firstChars: 0, chars: 0 };
+        let usage;
+        let finishReason = null;
+        try {
+            for await (const data of sseData(response.body)) {
+                if (data === "[DONE]")
+                    break;
+                const chunk = JSON.parse(data);
+                logger.provider("provider=apfel", { chunk });
+                if (chunk.error) {
+                    throw new Error(chunk.error.message ?? "apfel sent an error.");
+                }
+                usage = chunk.usage ?? usage;
+                const choice = chunk.choices?.[0];
+                const delta = choice?.delta;
+                const chars = (delta?.content?.length ?? 0) +
+                    (delta?.reasoning_content?.length ?? 0) +
+                    (delta?.tool_calls ?? []).reduce((n, piece) => n +
+                        (piece.function?.name?.length ?? 0) +
+                        (piece.function?.arguments?.length ?? 0), 0);
+                if (chars > 0) {
+                    timing.last = performance.now();
+                    if (!timing.first) {
+                        timing.first = timing.last;
+                        timing.firstChars = chars;
+                        // Token counts come only with the usage at the end.
+                        this.onStats?.({ firstTokenMs: timing.first - sentAt }, { threadId, runId, callId, done: false });
+                    }
+                    timing.chars += chars;
+                }
+                if (delta?.reasoning_content)
+                    addReasoning(delta.reasoning_content);
+                if (delta?.content)
+                    addText(splitter.push(delta.content));
+                for (const piece of delta?.tool_calls ?? []) {
+                    // apfel numbers its calls call_1, call_2… in every response; the
+                    // adapter gives them ids of its own that are unique in the thread.
+                    const call = (streamed[piece.index] ??= { name: "", arguments: "" });
+                    call.name += piece.function?.name ?? "";
+                    call.arguments += piece.function?.arguments ?? "";
+                }
+                if (choice?.finish_reason)
+                    finishReason = choice.finish_reason;
+                yield* events.splice(0);
+            }
+        }
+        catch (error) {
+            if (signal?.aborted)
+                return;
+            logger.errors("apfel.chatStream stream failed", {
+                error,
+                source: "apfel.chatStream"
+            });
+            endReasoning();
+            endText();
+            yield* events.splice(0);
+            yield runError(`apfel's stream failed: ${errorText(error)}`);
+            return;
+        }
+        const outputTokens = usage?.completion_tokens;
+        this.onStats?.({
+            inputTokens: usage?.prompt_tokens,
+            cachedTokens: usage?.prompt_tokens_details?.cached_tokens,
+            outputTokens,
+            ...(timing.first && {
+                firstTokenMs: timing.first - sentAt,
+                ...(outputTokens !== undefined && {
+                    decodeTokens: (outputTokens * (timing.chars - timing.firstChars)) /
+                        timing.chars,
+                    decodeMs: timing.last - timing.first
+                })
+            })
+        }, { threadId, runId, callId, done: true });
+        endReasoning();
+        const { text: rest, markup } = splitter.flush();
+        const fromText = markup && streamed.length === 0
+            ? parseJsonToolCalls(markup, toolNames)
+            : [];
+        // Text that wasn't a readable tool call is shown after all.
+        addText(`${rest}${fromText.length > 0 ? "" : (markup ?? "")}`);
+        yield* events.splice(0);
+        // Calls to tools that don't exist are passed on too: `chat()` answers
+        // them with an "Unknown tool" error, so the model can answer with what
+        // it has instead of stopping there.
+        const calls = [
+            ...streamed.filter((call) => call?.name),
+            ...fromText.map((call) => ({
+                name: call.name,
+                arguments: JSON.stringify(call.args)
+            }))
+        ];
+        for (const [index, call] of calls.entries()) {
+            const toolCallId = nextId("tool");
+            const args = call.arguments || "{}";
+            const input = parseArguments(args);
+            yield {
+                type: EventType.TOOL_CALL_START,
+                toolCallId,
+                toolCallName: call.name,
+                parentMessageId: messageId,
+                index,
+                model,
+                timestamp: timestamp()
+            };
+            yield {
+                type: EventType.TOOL_CALL_ARGS,
+                toolCallId,
+                delta: args,
+                args,
+                model,
+                timestamp: timestamp()
+            };
+            yield {
+                type: EventType.TOOL_CALL_END,
+                toolCallId,
+                toolCallName: call.name,
+                args,
+                // Arguments that aren't JSON are left out; `chat()` reports them.
+                ...(input && { input }),
+                model,
+                timestamp: timestamp()
+            };
+        }
+        endText();
+        yield* events.splice(0);
+        const finishUsage = usageOf(usage);
+        yield {
+            type: EventType.RUN_FINISHED,
+            runId,
+            threadId,
+            model,
+            timestamp: timestamp(),
+            finishReason: calls.length > 0
+                ? "tool_calls"
+                : finishReason === "tool_calls" || finishReason === null
+                    ? "stop"
+                    : finishReason,
+            ...(finishUsage && { usage: finishUsage })
+        };
+    }
+    /**
+     * A JSON answer for `chat({ outputSchema })`, from apfel's JSON mode:
+     * `response_format: json_schema` constrains Apple's model to the schema
+     * (guided generation). `chat()` validates the result.
+     */
+    async structuredOutput(options) {
+        const { chatOptions, outputSchema } = options;
+        const model = this.model;
+        const send = chatOptions.wrapFetch ? chatOptions.wrapFetch(fetch) : fetch;
+        chatOptions.logger.request(`activity=chat provider=apfel model=${model} messages=${chatOptions.messages.length} tools=0 stream=false`, { provider: "apfel", model });
+        let response;
+        try {
+            response = await send(`${this.baseUrl}/chat/completions`, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({
+                    ...chatOptions.modelOptions,
+                    model,
+                    messages: [
+                        ...systemMessages(chatOptions.systemPrompts),
+                        ...toOpenAIMessages(chatOptions.messages)
+                    ],
+                    response_format: {
+                        type: "json_schema",
+                        json_schema: { name: "output", schema: outputSchema }
+                    }
+                }),
+                signal: chatOptions.abortController?.signal ??
+                    chatOptions.request?.signal ??
+                    undefined
+            });
+        }
+        catch (error) {
+            chatOptions.logger.errors("apfel.structuredOutput fetch failed", {
+                error,
+                source: "apfel.structuredOutput"
+            });
+            if (chatOptions.abortController?.signal.aborted)
+                throw error;
+            throw new Error(unreachable(this.baseUrl));
+        }
+        if (!response.ok) {
+            throw new Error(`apfel returned HTTP ${response.status}: ${await errorBody(response)}`);
+        }
+        const completion = (await response.json());
+        const choice = completion.choices?.[0];
+        if (choice?.finish_reason === "length") {
+            throw new Error("apfel's answer was cut off at the token limit before its JSON ended; raise modelOptions.max_tokens.");
+        }
+        const rawText = (choice?.message?.content ??
+            choice?.message?.refusal ??
+            "").trim();
+        const json = rawText.replace(/^```(?:json)?\s*|\s*```$/g, "");
+        const usage = usageOf(completion.usage);
+        try {
+            return {
+                data: JSON.parse(json),
+                rawText,
+                model,
+                ...(usage && { usage })
+            };
+        }
+        catch {
+            throw new Error(`The model's answer isn't JSON: ${rawText.slice(0, 200)}${rawText.length > 200 ? "…" : ""}`);
+        }
+    }
+}
+/**
+ * A TanStack AI adapter for Apple's on-device Foundation Model through
+ * apfel (`apfel --serve --cors`).
+ *
+ * @example
+ * ```ts
+ * chat({ adapter: apfel(), messages, threadId })
+ * ```
+ */
+export function apfel(options) {
+    return new ApfelTextAdapter(options);
+}
+/**
+ * Whether apfel is running and serves the model: true if `GET /v1/models`
+ * lists it within 10 seconds.
+ */
+export async function isApfelAvailable(options = {}) {
+    const baseUrl = (options.baseUrl ?? APFEL_BASE_URL).replace(/\/+$/, "");
+    const model = options.model ?? APFEL_MODEL;
+    try {
+        const response = await fetch(`${baseUrl}/models`, {
+            signal: AbortSignal.timeout(10_000)
+        });
+        if (!response.ok)
+            return false;
+        const list = (await response.json());
+        return list.data?.some((m) => m.id === model) ?? false;
+    }
+    catch {
+        return false;
+    }
+}

package/dist/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * TanStack AI adapter for Apple's on-device Foundation Model, served by apfel.
+ */
+export { APFEL_BASE_URL, APFEL_MODEL, ApfelTextAdapter, apfel, isApfelAvailable } from "./adapter.js";
+export type { ApfelModelOptions, ApfelOptions, ApfelStats } from "./adapter.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EACL,cAAc,EACd,WAAW,EACX,gBAAgB,EAChB,KAAK,EACL,gBAAgB,EACjB,MAAM,cAAc,CAAA;AACrB,YAAY,EAAE,iBAAiB,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,cAAc,CAAA"}

package/dist/index.js

@@ -0,0 +1,4 @@
+/**
+ * TanStack AI adapter for Apple's on-device Foundation Model, served by apfel.
+ */
+export { APFEL_BASE_URL, APFEL_MODEL, ApfelTextAdapter, apfel, isApfelAvailable } from "./adapter.js";

package/dist/tool-calls.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Tool calls Apple's model writes as text. apfel turns the model's tool
+ * calls into OpenAI `tool_calls`, but when it doesn't recognize one it
+ * passes it on as text instead, like
+ * `{"tool_calls": [{"function": "get_weather", "arguments": {…}}]}`.
+ */
+export type ParsedToolCall = {
+    name: string;
+    args: Record<string, unknown>;
+};
+/** Where Apple's model starts a tool call it writes as text. */
+export declare const JSON_TOOL_CALLS_MARKER = "{\"tool_calls\"";
+/**
+ * Tool calls Apple's model wrote as JSON text. Its JSON varies and is often
+ * broken (`"function": "name"`, `"name: "name"`, stray strings in the
+ * arguments), so instead of parsing the whole thing, this finds each quoted
+ * name of a tool that exists and reads the `"arguments"` object after it.
+ */
+export declare function parseJsonToolCalls(text: string, toolNames: Iterable<string>): Array<ParsedToolCall>;
+/**
+ * Splits a streaming answer into the text before the first tool call written
+ * as text (passed through as it arrives) and the tool-call JSON after it
+ * (held back and parsed once the reply ends).
+ */
+export declare function createToolCallSplitter(marker?: string): {
+    push: (text: string) => string;
+    flush: () => {
+        text: string;
+        markup: string | null;
+    };
+};
+//# sourceMappingURL=tool-calls.d.ts.map

package/dist/tool-calls.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tool-calls.d.ts","sourceRoot":"","sources":["../src/tool-calls.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,MAAM,cAAc,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,CAAA;AAE5E,gEAAgE;AAChE,eAAO,MAAM,sBAAsB,oBAAkB,CAAA;AAkCrD;;;;;GAKG;AACH,wBAAgB,kBAAkB,CAChC,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,GAC1B,KAAK,CAAC,cAAc,CAAC,CAiCvB;AAUD;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,MAAM,SAAyB;iBAKhD,MAAM;;;;;EA2B3B"}