apple-llm 0.1.0 → 0.2.0

package/dist/server.js

@@ -0,0 +1,361 @@
+import {
+  TempImages
+} from "./chunk-ZB4RDEPW.js";
+import {
+  AbortError,
+  AppleLLM,
+  AppleLLMError,
+  ContextLengthError,
+  ModelBusyError,
+  ModelUnavailableError,
+  QuotaError,
+  RefusalError,
+  SchemaRejectedError,
+  SchemaValidationError,
+  SetupRequiredError,
+  TimeoutError,
+  UnsupportedError,
+  probe,
+  tool
+} from "./chunk-GM325EMJ.js";
+
+// src/server.ts
+import http from "http";
+import { randomUUID } from "crypto";
+var DEFAULT_PORT = 11436;
+var MODELS = {
+  "apple-on-device": "device",
+  "apple-private-cloud": "cloud"
+};
+function errorResponse(error) {
+  const message = error instanceof Error ? error.message : String(error);
+  const make = (status, type, code) => ({
+    status,
+    body: { error: { message, type, code: code ?? null, param: null } }
+  });
+  if (error instanceof ContextLengthError) return make(400, "invalid_request_error", "context_length_exceeded");
+  if (error instanceof SchemaRejectedError || error instanceof SchemaValidationError) {
+    return make(400, "invalid_request_error", "invalid_schema");
+  }
+  if (error instanceof UnsupportedError) return make(400, "invalid_request_error", "unsupported");
+  if (error instanceof RefusalError) return make(400, "invalid_request_error", "content_filter");
+  if (error instanceof QuotaError) return make(429, "rate_limit_error", "quota_exceeded");
+  if (error instanceof TimeoutError) return make(504, "timeout_error");
+  if (error instanceof AbortError) return make(499, "request_cancelled");
+  if (error instanceof ModelBusyError) return make(503, "service_unavailable", "model_busy");
+  if (error instanceof ModelUnavailableError || error instanceof SetupRequiredError) {
+    return make(503, "service_unavailable", "model_unavailable");
+  }
+  if (error instanceof HttpError) return make(error.status, "invalid_request_error");
+  if (error instanceof AppleLLMError) return make(500, "server_error");
+  return make(500, "server_error");
+}
+var HttpError = class extends Error {
+  constructor(status, message) {
+    super(message);
+    this.status = status;
+  }
+  status;
+};
+function textOf(content) {
+  if (typeof content === "string") return content;
+  if (content === null || content === void 0) return "";
+  if (Array.isArray(content)) {
+    return content.filter((p) => p.type === "text" || p.type === "input_text").map((p) => p.text ?? "").join("\n");
+  }
+  return String(content);
+}
+async function toMessages(raw, images, signal) {
+  if (!Array.isArray(raw) || raw.length === 0) throw new HttpError(400, "`messages` must be a non-empty array.");
+  const out = [];
+  for (const message of raw) {
+    switch (message.role) {
+      case "system":
+      case "developer":
+        out.push({ role: "system", content: textOf(message.content) });
+        break;
+      case "user": {
+        const paths = [];
+        if (Array.isArray(message.content)) {
+          for (const part of message.content) {
+            if (part.type === "image_url" && part.image_url !== void 0) {
+              const url = typeof part.image_url === "string" ? part.image_url : part.image_url.url;
+              paths.push(await images.add(url, void 0, signal));
+            } else if (part.type !== "text" && part.type !== "input_text") {
+              throw new HttpError(400, `Content part "${part.type}" is not supported.`);
+            }
+          }
+        }
+        out.push({ role: "user", content: textOf(message.content), ...paths.length > 0 ? { images: paths } : {} });
+        break;
+      }
+      case "assistant":
+        out.push({
+          role: "assistant",
+          content: textOf(message.content),
+          ...message.tool_calls !== void 0 && message.tool_calls.length > 0 ? {
+            toolCalls: message.tool_calls.map((call) => ({
+              id: call.id,
+              name: call.function.name,
+              arguments: parseArguments(call.function.arguments)
+            }))
+          } : {}
+        });
+        break;
+      case "tool":
+        out.push({ role: "tool", toolCallId: String(message.tool_call_id ?? ""), name: message.name, content: textOf(message.content) });
+        break;
+      default:
+        throw new HttpError(400, `Message role "${message.role}" is not supported.`);
+    }
+  }
+  return out;
+}
+function parseArguments(text) {
+  try {
+    return JSON.parse(text);
+  } catch {
+    return text;
+  }
+}
+function openAIToolCalls(calls) {
+  return calls.filter((call) => call.output === void 0).map((call, index) => ({
+    index,
+    id: call.id,
+    type: "function",
+    function: { name: call.name, arguments: JSON.stringify(call.arguments ?? {}) }
+  }));
+}
+function finishReason(result) {
+  return result.finishReason === "tool-calls" ? "tool_calls" : result.finishReason;
+}
+function usageOf(result) {
+  if (result.usage === void 0) return void 0;
+  return {
+    prompt_tokens: result.usage.inputTokens,
+    completion_tokens: result.usage.outputTokens,
+    total_tokens: result.usage.totalTokens
+  };
+}
+async function readBody(req, limit = 32 * 1024 * 1024) {
+  const chunks = [];
+  let size = 0;
+  for await (const chunk of req) {
+    size += chunk.length;
+    if (size > limit) throw new HttpError(413, "Request body too large.");
+    chunks.push(chunk);
+  }
+  const text = Buffer.concat(chunks).toString("utf8");
+  if (text === "") return {};
+  try {
+    return JSON.parse(text);
+  } catch {
+    throw new HttpError(400, "Request body is not valid JSON.");
+  }
+}
+function createServer(options = {}) {
+  const { port: _port, host: _host, tier: defaultTier = "device", apiKey, cors, log, ...llmOptions } = options;
+  const clients = /* @__PURE__ */ new Map();
+  const clientFor = (tier) => {
+    let client = clients.get(tier);
+    if (client === void 0) {
+      client = new AppleLLM({ ...llmOptions, tier });
+      clients.set(tier, client);
+    }
+    return client;
+  };
+  const server = http.createServer((req, res) => {
+    const started = Date.now();
+    const controller = new AbortController();
+    res.on("close", () => {
+      if (!res.writableFinished) controller.abort();
+    });
+    const send = (status, body) => {
+      if (res.headersSent) return;
+      res.writeHead(status, { "content-type": "application/json", ...corsHeaders() });
+      res.end(JSON.stringify(body));
+    };
+    const corsHeaders = () => cors === void 0 ? {} : {
+      "access-control-allow-origin": cors,
+      "access-control-allow-headers": "authorization, content-type",
+      "access-control-allow-methods": "GET, POST, OPTIONS"
+    };
+    const route = async () => {
+      const url = new URL(req.url ?? "/", "http://localhost");
+      const pathname = url.pathname.replace(/\/+$/, "") || "/";
+      if (req.method === "OPTIONS") {
+        res.writeHead(204, corsHeaders());
+        res.end();
+        return void 0;
+      }
+      if (apiKey !== void 0 && req.headers.authorization !== `Bearer ${apiKey}`) {
+        send(401, { error: { message: "Invalid API key.", type: "invalid_request_error", code: "invalid_api_key", param: null } });
+        return void 0;
+      }
+      if (req.method === "GET" && (pathname === "/health" || pathname === "/v1/health")) {
+        const state = await probe();
+        send(state.device.available || state.cloud.available ? 200 : 503, {
+          status: state.device.available || state.cloud.available ? "ok" : "unavailable",
+          device: { available: state.device.available, variant: state.device.variant, contextSize: state.device.contextSize, reason: state.device.reason },
+          cloud: { available: state.cloud.available, quota: state.cloud.quota?.status, reason: state.cloud.reason }
+        });
+        return void 0;
+      }
+      if (req.method === "GET" && pathname === "/v1/models") {
+        const created = Math.floor(Date.now() / 1e3);
+        send(200, {
+          object: "list",
+          data: Object.keys(MODELS).map((id) => ({ id, object: "model", created, owned_by: "apple" }))
+        });
+        return void 0;
+      }
+      if (req.method === "POST" && pathname === "/v1/chat/completions") {
+        const body = await readBody(req);
+        return chatCompletions(body);
+      }
+      send(404, { error: { message: `No route for ${req.method} ${pathname}.`, type: "invalid_request_error", code: "not_found", param: null } });
+      return void 0;
+    };
+    const chatCompletions = async (body) => {
+      const requested = body.model ?? "";
+      const tier = requested in MODELS ? MODELS[requested] : defaultTier;
+      const model = requested in MODELS ? requested : tier === "cloud" ? "apple-private-cloud" : "apple-on-device";
+      if (body.n !== void 0 && body.n !== 1) throw new HttpError(400, "Only n=1 is supported.");
+      const images = new TempImages();
+      try {
+        const messages = await toMessages(body.messages, images, controller.signal);
+        const functions = [];
+        if (body.tool_choice !== "none") {
+          for (const def of body.tools ?? []) {
+            if (def.type !== "function" || def.function === void 0) continue;
+            functions.push(tool({ name: def.function.name, description: def.function.description, parameters: def.function.parameters }));
+          }
+        }
+        let schema;
+        if (body.response_format?.type === "json_schema" && body.response_format.json_schema?.schema !== void 0) {
+          schema = body.response_format.json_schema.schema;
+        } else if (body.response_format?.type === "json_object") {
+          messages.unshift({ role: "system", content: "Reply with a single JSON object and nothing else." });
+        }
+        let sampling;
+        if (body.top_p !== void 0) sampling = { mode: "threshold", p: body.top_p, seed: body.seed };
+        else if (body.seed !== void 0) sampling = { mode: "topK", k: 50, seed: body.seed };
+        const call = {
+          temperature: body.temperature,
+          maxTokens: body.max_completion_tokens ?? body.max_tokens,
+          sampling,
+          tools: functions.length > 0 ? functions : void 0,
+          signal: controller.signal
+        };
+        const client = clientFor(tier);
+        const id = `chatcmpl-${randomUUID().replace(/-/g, "").slice(0, 24)}`;
+        const created = Math.floor(Date.now() / 1e3);
+        if (body.stream === true) {
+          res.writeHead(200, {
+            "content-type": "text/event-stream",
+            "cache-control": "no-cache",
+            connection: "keep-alive",
+            ...corsHeaders()
+          });
+          const event = (payload) => {
+            if (!res.destroyed) res.write(`data: ${typeof payload === "string" ? payload : JSON.stringify(payload)}
+
+`);
+          };
+          const chunk = (delta, finish = null) => {
+            event({ id, object: "chat.completion.chunk", created, model, choices: [{ index: 0, delta, finish_reason: finish, logprobs: null }] });
+          };
+          chunk({ role: "assistant", content: "" });
+          try {
+            let result2;
+            if (schema === void 0) {
+              const stream = client.stream(messages, call);
+              for await (const delta of stream) chunk({ content: delta });
+              result2 = await stream.result;
+            } else {
+              result2 = await client.generate(messages, { ...call, schema });
+              if (result2.finishReason !== "tool-calls") chunk({ content: result2.text });
+            }
+            const calls2 = openAIToolCalls(result2.toolCalls);
+            if (calls2.length > 0 && result2.finishReason === "tool-calls") chunk({ tool_calls: calls2 });
+            chunk({}, finishReason(result2));
+            if (body.stream_options?.include_usage === true) {
+              event({ id, object: "chat.completion.chunk", created, model, choices: [], usage: usageOf(result2) ?? null });
+            }
+          } catch (error) {
+            event(errorResponse(error).body);
+            if (!res.destroyed) res.end("data: [DONE]\n\n");
+            return `${model} stream failed: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}`;
+          }
+          if (!res.destroyed) res.end("data: [DONE]\n\n");
+          return `${model} stream`;
+        }
+        const result = schema === void 0 ? await client.generate(messages, call) : await client.generate(messages, { ...call, schema });
+        const calls = result.finishReason === "tool-calls" ? openAIToolCalls(result.toolCalls) : [];
+        send(200, {
+          id,
+          object: "chat.completion",
+          created,
+          model,
+          choices: [
+            {
+              index: 0,
+              message: {
+                role: "assistant",
+                content: result.finishReason === "tool-calls" ? null : result.text,
+                ...calls.length > 0 ? { tool_calls: calls.map(({ index: _i, ...rest }) => rest) } : {},
+                refusal: null
+              },
+              finish_reason: finishReason(result),
+              logprobs: null
+            }
+          ],
+          usage: usageOf(result) ?? { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }
+        });
+        return model;
+      } finally {
+        await images.dispose();
+      }
+    };
+    route().then((what) => {
+      if (what !== void 0) log?.(`${req.method} ${req.url} ${what} ${res.statusCode} ${Date.now() - started}ms`);
+    }).catch((error) => {
+      const { status, body } = errorResponse(error);
+      if (error instanceof ModelBusyError && !res.headersSent) res.setHeader("retry-after", "2");
+      send(status, body);
+      log?.(`${req.method} ${req.url} ${status} ${Date.now() - started}ms ${error instanceof Error ? error.message.split("\n")[0] : ""}`);
+    });
+  });
+  server.on("close", () => {
+    for (const client of clients.values()) client.close();
+  });
+  return server;
+}
+async function serve(options = {}) {
+  const server = createServer(options);
+  const port = options.port ?? DEFAULT_PORT;
+  const host = options.host ?? "127.0.0.1";
+  await new Promise((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(port, host, () => {
+      server.off("error", reject);
+      resolve();
+    });
+  });
+  const address = server.address();
+  const bound = typeof address === "object" && address !== null ? address.port : port;
+  return {
+    server,
+    url: `http://${host === "0.0.0.0" ? "127.0.0.1" : host}:${bound}/v1`,
+    close: () => new Promise((resolve) => {
+      server.close(() => resolve());
+      server.closeAllConnections?.();
+    })
+  };
+}
+export {
+  DEFAULT_PORT,
+  MODELS,
+  createServer,
+  serve
+};

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "apple-llm",
-  "version": "0.1.0",
-  "description": "Apple's on-device (Foundation Models) and Private Cloud Compute LLMs from Node. No API key, no account, macOS 26+ on Apple Silicon.",
+  "version": "0.2.0",
+  "description": "Apple's on-device (Foundation Models) and Private Cloud Compute LLMs from Node: tools, streaming, structured output, an OpenAI-compatible server and a Vercel AI SDK provider. No API key, no account.",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -15,13 +15,39 @@
   "type": "module",
   "main": "./dist/index.cjs",
   "module": "./dist/index.js",
-  "types": "./dist/index.d.ts",
+  "types": "./dist/index.d.cts",
   "exports": {
     ".": {
-      "types": "./dist/index.d.ts",
-      "import": "./dist/index.js",
-      "require": "./dist/index.cjs"
-    }
+      "import": {
+        "types": "./dist/index.d.ts",
+        "default": "./dist/index.js"
+      },
+      "require": {
+        "types": "./dist/index.d.cts",
+        "default": "./dist/index.cjs"
+      }
+    },
+    "./ai-sdk": {
+      "import": {
+        "types": "./dist/ai-sdk.d.ts",
+        "default": "./dist/ai-sdk.js"
+      },
+      "require": {
+        "types": "./dist/ai-sdk.d.cts",
+        "default": "./dist/ai-sdk.cjs"
+      }
+    },
+    "./server": {
+      "import": {
+        "types": "./dist/server.d.ts",
+        "default": "./dist/server.js"
+      },
+      "require": {
+        "types": "./dist/server.d.cts",
+        "default": "./dist/server.cjs"
+      }
+    },
+    "./package.json": "./package.json"
   },
   "bin": {
     "apple-llm": "dist/cli.js"
@@ -30,32 +56,58 @@
     "dist",
     "swift",
     "README.md",
+    "CHANGELOG.md",
     "LICENSE"
   ],
+  "sideEffects": false,
   "engines": {
     "node": ">=18"
   },
   "keywords": [
+    "ai-sdk",
     "apple",
-    "llm",
+    "apple-intelligence",
+    "apple-silicon",
     "foundation-models",
+    "function-calling",
+    "llm",
+    "local-llm",
+    "macos",
     "on-device",
+    "openai-compatible",
     "private-cloud-compute",
-    "apple-intelligence",
-    "macos",
-    "local-llm",
-    "structured-output"
+    "streaming",
+    "structured-output",
+    "tool-calling",
+    "vercel-ai",
+    "zod"
   ],
   "scripts": {
     "build": "node ../../scripts/embed-helper.mjs && tsup",
     "test": "vitest run",
     "test:live": "APPLE_LLM_LIVE=1 vitest run",
-    "typecheck": "tsc --noEmit"
+    "typecheck": "tsc --noEmit && tsc --noEmit -p tsconfig.examples.json",
+    "check:package": "npm run build && publint && attw --pack . --profile node16"
+  },
+  "peerDependencies": {
+    "@ai-sdk/provider": ">=3.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@ai-sdk/provider": {
+      "optional": true
+    }
   },
   "devDependencies": {
+    "@ai-sdk/provider": "^4.0.18",
+    "@arethetypeswrong/cli": "^0.18.5",
     "@types/node": "^22.10.2",
+    "ai": "^7.0.114",
+    "openai": "^7.23.0",
+    "publint": "^0.3.24",
     "tsup": "^8.3.5",
+    "tsx": "^4.23.15",
     "typescript": "^5.7.2",
-    "vitest": "^2.1.8"
+    "vitest": "^2.1.8",
+    "zod": "^4.6.5"
   }
 }