apple-llm 0.1.0 → 0.2.0

package/dist/index.js

@@ -1,4 +1,5 @@
 import {
+  AbortError,
   AppleLLM,
   AppleLLMError,
   CLOUD_CONTEXT_TOKENS,
@@ -8,14 +9,21 @@ import {
   ContextLengthError,
   Conversation,
   DEFAULT_MAX_TOKENS,
+  DEFAULT_MAX_TOOL_CALLS,
   DEFAULT_TEMPERATURE,
   DeviceClient,
+  ModelBusyError,
   ModelUnavailableError,
   QuotaError,
   RefusalError,
+  ReplayBook,
+  ResultStream,
   SchemaRejectedError,
+  SchemaValidationError,
   SetupRequiredError,
   TimeoutError,
+  ToolExecutionError,
+  UnsupportedError,
   assertTools,
   cacheDir,
   captureScreenshot,
@@ -27,18 +35,26 @@ import {
   hostTarget,
   installCloudShortcut,
   isAppleSiliconMac,
+  isStandardSchema,
+  normalizeTools,
   parseImageFlag,
   parseLlmJson,
   probe,
   probeCloud,
   probeDevice,
+  resolveSchema,
+  restoreNulls,
   shortcutDefinition,
+  splitMessages,
   stripCodeFences,
   targetTripleFrom,
   toAppleSchema,
+  tool,
+  toolOutputText,
   withDocuments
-} from "./chunk-FQTRQ3KP.js";
+} from "./chunk-GM325EMJ.js";
 export {
+  AbortError,
   AppleLLM,
   AppleLLMError,
   CLOUD_CONTEXT_TOKENS,
@@ -48,14 +64,21 @@ export {
   ContextLengthError,
   Conversation,
   DEFAULT_MAX_TOKENS,
+  DEFAULT_MAX_TOOL_CALLS,
   DEFAULT_TEMPERATURE,
   DeviceClient,
+  ModelBusyError,
   ModelUnavailableError,
   QuotaError,
   RefusalError,
+  ReplayBook,
+  ResultStream,
   SchemaRejectedError,
+  SchemaValidationError,
   SetupRequiredError,
   TimeoutError,
+  ToolExecutionError,
+  UnsupportedError,
   assertTools,
   cacheDir,
   captureScreenshot,
@@ -67,14 +90,21 @@ export {
   hostTarget,
   installCloudShortcut,
   isAppleSiliconMac,
+  isStandardSchema,
+  normalizeTools,
   parseImageFlag,
   parseLlmJson,
   probe,
   probeCloud,
   probeDevice,
+  resolveSchema,
+  restoreNulls,
   shortcutDefinition,
+  splitMessages,
   stripCodeFences,
   targetTripleFrom,
   toAppleSchema,
+  tool,
+  toolOutputText,
   withDocuments
 };

package/dist/server.cjs

@@ -0,0 +1,361 @@
+"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } function _nullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return rhsFn(); } } function _optionalChain(ops) { let lastAccessLHS = undefined; let value = ops[0]; let i = 1; while (i < ops.length) { const op = ops[i]; const fn = ops[i + 1]; i += 2; if ((op === 'optionalAccess' || op === 'optionalCall') && value == null) { return undefined; } if (op === 'access' || op === 'optionalAccess') { lastAccessLHS = value; value = fn(value); } else if (op === 'call' || op === 'optionalCall') { value = fn((...args) => value.call(lastAccessLHS, ...args)); lastAccessLHS = undefined; } } return value; }
+
+var _chunkOQATUZWFcjs = require('./chunk-OQATUZWF.cjs');
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+var _chunkNRDZIP5Gcjs = require('./chunk-NRDZIP5G.cjs');
+
+// src/server.ts
+var _http = require('http'); var _http2 = _interopRequireDefault(_http);
+var _crypto = require('crypto');
+var DEFAULT_PORT = 11436;
+var MODELS = {
+  "apple-on-device": "device",
+  "apple-private-cloud": "cloud"
+};
+function errorResponse(error) {
+  const message = error instanceof Error ? error.message : String(error);
+  const make = (status, type, code) => ({
+    status,
+    body: { error: { message, type, code: _nullishCoalesce(code, () => ( null)), param: null } }
+  });
+  if (error instanceof _chunkNRDZIP5Gcjs.ContextLengthError) return make(400, "invalid_request_error", "context_length_exceeded");
+  if (error instanceof _chunkNRDZIP5Gcjs.SchemaRejectedError || error instanceof _chunkNRDZIP5Gcjs.SchemaValidationError) {
+    return make(400, "invalid_request_error", "invalid_schema");
+  }
+  if (error instanceof _chunkNRDZIP5Gcjs.UnsupportedError) return make(400, "invalid_request_error", "unsupported");
+  if (error instanceof _chunkNRDZIP5Gcjs.RefusalError) return make(400, "invalid_request_error", "content_filter");
+  if (error instanceof _chunkNRDZIP5Gcjs.QuotaError) return make(429, "rate_limit_error", "quota_exceeded");
+  if (error instanceof _chunkNRDZIP5Gcjs.TimeoutError) return make(504, "timeout_error");
+  if (error instanceof _chunkNRDZIP5Gcjs.AbortError) return make(499, "request_cancelled");
+  if (error instanceof _chunkNRDZIP5Gcjs.ModelBusyError) return make(503, "service_unavailable", "model_busy");
+  if (error instanceof _chunkNRDZIP5Gcjs.ModelUnavailableError || error instanceof _chunkNRDZIP5Gcjs.SetupRequiredError) {
+    return make(503, "service_unavailable", "model_unavailable");
+  }
+  if (error instanceof HttpError) return make(error.status, "invalid_request_error");
+  if (error instanceof _chunkNRDZIP5Gcjs.AppleLLMError) return make(500, "server_error");
+  return make(500, "server_error");
+}
+var HttpError = class extends Error {
+  constructor(status, message) {
+    super(message);
+    this.status = status;
+  }
+  
+};
+function textOf(content) {
+  if (typeof content === "string") return content;
+  if (content === null || content === void 0) return "";
+  if (Array.isArray(content)) {
+    return content.filter((p) => p.type === "text" || p.type === "input_text").map((p) => _nullishCoalesce(p.text, () => ( ""))).join("\n");
+  }
+  return String(content);
+}
+async function toMessages(raw, images, signal) {
+  if (!Array.isArray(raw) || raw.length === 0) throw new HttpError(400, "`messages` must be a non-empty array.");
+  const out = [];
+  for (const message of raw) {
+    switch (message.role) {
+      case "system":
+      case "developer":
+        out.push({ role: "system", content: textOf(message.content) });
+        break;
+      case "user": {
+        const paths = [];
+        if (Array.isArray(message.content)) {
+          for (const part of message.content) {
+            if (part.type === "image_url" && part.image_url !== void 0) {
+              const url = typeof part.image_url === "string" ? part.image_url : part.image_url.url;
+              paths.push(await images.add(url, void 0, signal));
+            } else if (part.type !== "text" && part.type !== "input_text") {
+              throw new HttpError(400, `Content part "${part.type}" is not supported.`);
+            }
+          }
+        }
+        out.push({ role: "user", content: textOf(message.content), ...paths.length > 0 ? { images: paths } : {} });
+        break;
+      }
+      case "assistant":
+        out.push({
+          role: "assistant",
+          content: textOf(message.content),
+          ...message.tool_calls !== void 0 && message.tool_calls.length > 0 ? {
+            toolCalls: message.tool_calls.map((call) => ({
+              id: call.id,
+              name: call.function.name,
+              arguments: parseArguments(call.function.arguments)
+            }))
+          } : {}
+        });
+        break;
+      case "tool":
+        out.push({ role: "tool", toolCallId: String(_nullishCoalesce(message.tool_call_id, () => ( ""))), name: message.name, content: textOf(message.content) });
+        break;
+      default:
+        throw new HttpError(400, `Message role "${message.role}" is not supported.`);
+    }
+  }
+  return out;
+}
+function parseArguments(text) {
+  try {
+    return JSON.parse(text);
+  } catch (e) {
+    return text;
+  }
+}
+function openAIToolCalls(calls) {
+  return calls.filter((call) => call.output === void 0).map((call, index) => ({
+    index,
+    id: call.id,
+    type: "function",
+    function: { name: call.name, arguments: JSON.stringify(_nullishCoalesce(call.arguments, () => ( {}))) }
+  }));
+}
+function finishReason(result) {
+  return result.finishReason === "tool-calls" ? "tool_calls" : result.finishReason;
+}
+function usageOf(result) {
+  if (result.usage === void 0) return void 0;
+  return {
+    prompt_tokens: result.usage.inputTokens,
+    completion_tokens: result.usage.outputTokens,
+    total_tokens: result.usage.totalTokens
+  };
+}
+async function readBody(req, limit = 32 * 1024 * 1024) {
+  const chunks = [];
+  let size = 0;
+  for await (const chunk of req) {
+    size += chunk.length;
+    if (size > limit) throw new HttpError(413, "Request body too large.");
+    chunks.push(chunk);
+  }
+  const text = Buffer.concat(chunks).toString("utf8");
+  if (text === "") return {};
+  try {
+    return JSON.parse(text);
+  } catch (e2) {
+    throw new HttpError(400, "Request body is not valid JSON.");
+  }
+}
+function createServer(options = {}) {
+  const { port: _port, host: _host, tier: defaultTier = "device", apiKey, cors, log, ...llmOptions } = options;
+  const clients = /* @__PURE__ */ new Map();
+  const clientFor = (tier) => {
+    let client = clients.get(tier);
+    if (client === void 0) {
+      client = new (0, _chunkNRDZIP5Gcjs.AppleLLM)({ ...llmOptions, tier });
+      clients.set(tier, client);
+    }
+    return client;
+  };
+  const server = _http2.default.createServer((req, res) => {
+    const started = Date.now();
+    const controller = new AbortController();
+    res.on("close", () => {
+      if (!res.writableFinished) controller.abort();
+    });
+    const send = (status, body) => {
+      if (res.headersSent) return;
+      res.writeHead(status, { "content-type": "application/json", ...corsHeaders() });
+      res.end(JSON.stringify(body));
+    };
+    const corsHeaders = () => cors === void 0 ? {} : {
+      "access-control-allow-origin": cors,
+      "access-control-allow-headers": "authorization, content-type",
+      "access-control-allow-methods": "GET, POST, OPTIONS"
+    };
+    const route = async () => {
+      const url = new URL(_nullishCoalesce(req.url, () => ( "/")), "http://localhost");
+      const pathname = url.pathname.replace(/\/+$/, "") || "/";
+      if (req.method === "OPTIONS") {
+        res.writeHead(204, corsHeaders());
+        res.end();
+        return void 0;
+      }
+      if (apiKey !== void 0 && req.headers.authorization !== `Bearer ${apiKey}`) {
+        send(401, { error: { message: "Invalid API key.", type: "invalid_request_error", code: "invalid_api_key", param: null } });
+        return void 0;
+      }
+      if (req.method === "GET" && (pathname === "/health" || pathname === "/v1/health")) {
+        const state = await _chunkNRDZIP5Gcjs.probe.call(void 0, );
+        send(state.device.available || state.cloud.available ? 200 : 503, {
+          status: state.device.available || state.cloud.available ? "ok" : "unavailable",
+          device: { available: state.device.available, variant: state.device.variant, contextSize: state.device.contextSize, reason: state.device.reason },
+          cloud: { available: state.cloud.available, quota: _optionalChain([state, 'access', _ => _.cloud, 'access', _2 => _2.quota, 'optionalAccess', _3 => _3.status]), reason: state.cloud.reason }
+        });
+        return void 0;
+      }
+      if (req.method === "GET" && pathname === "/v1/models") {
+        const created = Math.floor(Date.now() / 1e3);
+        send(200, {
+          object: "list",
+          data: Object.keys(MODELS).map((id) => ({ id, object: "model", created, owned_by: "apple" }))
+        });
+        return void 0;
+      }
+      if (req.method === "POST" && pathname === "/v1/chat/completions") {
+        const body = await readBody(req);
+        return chatCompletions(body);
+      }
+      send(404, { error: { message: `No route for ${req.method} ${pathname}.`, type: "invalid_request_error", code: "not_found", param: null } });
+      return void 0;
+    };
+    const chatCompletions = async (body) => {
+      const requested = _nullishCoalesce(body.model, () => ( ""));
+      const tier = requested in MODELS ? MODELS[requested] : defaultTier;
+      const model = requested in MODELS ? requested : tier === "cloud" ? "apple-private-cloud" : "apple-on-device";
+      if (body.n !== void 0 && body.n !== 1) throw new HttpError(400, "Only n=1 is supported.");
+      const images = new (0, _chunkOQATUZWFcjs.TempImages)();
+      try {
+        const messages = await toMessages(body.messages, images, controller.signal);
+        const functions = [];
+        if (body.tool_choice !== "none") {
+          for (const def of _nullishCoalesce(body.tools, () => ( []))) {
+            if (def.type !== "function" || def.function === void 0) continue;
+            functions.push(_chunkNRDZIP5Gcjs.tool.call(void 0, { name: def.function.name, description: def.function.description, parameters: def.function.parameters }));
+          }
+        }
+        let schema;
+        if (_optionalChain([body, 'access', _4 => _4.response_format, 'optionalAccess', _5 => _5.type]) === "json_schema" && _optionalChain([body, 'access', _6 => _6.response_format, 'access', _7 => _7.json_schema, 'optionalAccess', _8 => _8.schema]) !== void 0) {
+          schema = body.response_format.json_schema.schema;
+        } else if (_optionalChain([body, 'access', _9 => _9.response_format, 'optionalAccess', _10 => _10.type]) === "json_object") {
+          messages.unshift({ role: "system", content: "Reply with a single JSON object and nothing else." });
+        }
+        let sampling;
+        if (body.top_p !== void 0) sampling = { mode: "threshold", p: body.top_p, seed: body.seed };
+        else if (body.seed !== void 0) sampling = { mode: "topK", k: 50, seed: body.seed };
+        const call = {
+          temperature: body.temperature,
+          maxTokens: _nullishCoalesce(body.max_completion_tokens, () => ( body.max_tokens)),
+          sampling,
+          tools: functions.length > 0 ? functions : void 0,
+          signal: controller.signal
+        };
+        const client = clientFor(tier);
+        const id = `chatcmpl-${_crypto.randomUUID.call(void 0, ).replace(/-/g, "").slice(0, 24)}`;
+        const created = Math.floor(Date.now() / 1e3);
+        if (body.stream === true) {
+          res.writeHead(200, {
+            "content-type": "text/event-stream",
+            "cache-control": "no-cache",
+            connection: "keep-alive",
+            ...corsHeaders()
+          });
+          const event = (payload) => {
+            if (!res.destroyed) res.write(`data: ${typeof payload === "string" ? payload : JSON.stringify(payload)}
+
+`);
+          };
+          const chunk = (delta, finish = null) => {
+            event({ id, object: "chat.completion.chunk", created, model, choices: [{ index: 0, delta, finish_reason: finish, logprobs: null }] });
+          };
+          chunk({ role: "assistant", content: "" });
+          try {
+            let result2;
+            if (schema === void 0) {
+              const stream = client.stream(messages, call);
+              for await (const delta of stream) chunk({ content: delta });
+              result2 = await stream.result;
+            } else {
+              result2 = await client.generate(messages, { ...call, schema });
+              if (result2.finishReason !== "tool-calls") chunk({ content: result2.text });
+            }
+            const calls2 = openAIToolCalls(result2.toolCalls);
+            if (calls2.length > 0 && result2.finishReason === "tool-calls") chunk({ tool_calls: calls2 });
+            chunk({}, finishReason(result2));
+            if (_optionalChain([body, 'access', _11 => _11.stream_options, 'optionalAccess', _12 => _12.include_usage]) === true) {
+              event({ id, object: "chat.completion.chunk", created, model, choices: [], usage: _nullishCoalesce(usageOf(result2), () => ( null)) });
+            }
+          } catch (error) {
+            event(errorResponse(error).body);
+            if (!res.destroyed) res.end("data: [DONE]\n\n");
+            return `${model} stream failed: ${error instanceof Error ? error.message.split("\n")[0] : String(error)}`;
+          }
+          if (!res.destroyed) res.end("data: [DONE]\n\n");
+          return `${model} stream`;
+        }
+        const result = schema === void 0 ? await client.generate(messages, call) : await client.generate(messages, { ...call, schema });
+        const calls = result.finishReason === "tool-calls" ? openAIToolCalls(result.toolCalls) : [];
+        send(200, {
+          id,
+          object: "chat.completion",
+          created,
+          model,
+          choices: [
+            {
+              index: 0,
+              message: {
+                role: "assistant",
+                content: result.finishReason === "tool-calls" ? null : result.text,
+                ...calls.length > 0 ? { tool_calls: calls.map(({ index: _i, ...rest }) => rest) } : {},
+                refusal: null
+              },
+              finish_reason: finishReason(result),
+              logprobs: null
+            }
+          ],
+          usage: _nullishCoalesce(usageOf(result), () => ( { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }))
+        });
+        return model;
+      } finally {
+        await images.dispose();
+      }
+    };
+    route().then((what) => {
+      if (what !== void 0) _optionalChain([log, 'optionalCall', _13 => _13(`${req.method} ${req.url} ${what} ${res.statusCode} ${Date.now() - started}ms`)]);
+    }).catch((error) => {
+      const { status, body } = errorResponse(error);
+      if (error instanceof _chunkNRDZIP5Gcjs.ModelBusyError && !res.headersSent) res.setHeader("retry-after", "2");
+      send(status, body);
+      _optionalChain([log, 'optionalCall', _14 => _14(`${req.method} ${req.url} ${status} ${Date.now() - started}ms ${error instanceof Error ? error.message.split("\n")[0] : ""}`)]);
+    });
+  });
+  server.on("close", () => {
+    for (const client of clients.values()) client.close();
+  });
+  return server;
+}
+async function serve(options = {}) {
+  const server = createServer(options);
+  const port = _nullishCoalesce(options.port, () => ( DEFAULT_PORT));
+  const host = _nullishCoalesce(options.host, () => ( "127.0.0.1"));
+  await new Promise((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(port, host, () => {
+      server.off("error", reject);
+      resolve();
+    });
+  });
+  const address = server.address();
+  const bound = typeof address === "object" && address !== null ? address.port : port;
+  return {
+    server,
+    url: `http://${host === "0.0.0.0" ? "127.0.0.1" : host}:${bound}/v1`,
+    close: () => new Promise((resolve) => {
+      server.close(() => resolve());
+      _optionalChain([server, 'access', _15 => _15.closeAllConnections, 'optionalCall', _16 => _16()]);
+    })
+  };
+}
+
+
+
+
+
+exports.DEFAULT_PORT = DEFAULT_PORT; exports.MODELS = MODELS; exports.createServer = createServer; exports.serve = serve;

package/dist/server.d.cts

@@ -0,0 +1,51 @@
+import http from 'node:http';
+import { A as AppleLLMOptions } from './client-CXewzZTj.cjs';
+import 'node:child_process';
+
+/**
+ * An OpenAI-compatible HTTP server for Apple's models.
+ *
+ *   apple-llm serve                        # http://127.0.0.1:11436/v1
+ *
+ * Point any OpenAI client at it — the official SDKs, LangChain, LlamaIndex,
+ * editor plugins, Open WebUI — with any API key, and it talks to the on-device
+ * model. Implements `POST /v1/chat/completions` (streaming, tools,
+ * `response_format` with JSON Schema, images), `GET /v1/models` and
+ * `GET /health`.
+ *
+ * Safe defaults, because a local model server is reachable by more than you
+ * might think: it binds to 127.0.0.1 only; CORS is off, so a web page you
+ * happen to visit cannot use your model; an API key can be required; and the
+ * default tier is the device, so nothing leaves the machine unless a client
+ * asks for the `apple-private-cloud` model by name.
+ */
+
+declare const DEFAULT_PORT = 11436;
+/** Model ids a client can ask for. Anything else gets the server's default tier. */
+declare const MODELS: {
+    readonly 'apple-on-device': "device";
+    readonly 'apple-private-cloud': "cloud";
+};
+interface ServerOptions extends Omit<AppleLLMOptions, 'tier'> {
+    port?: number;
+    /** Interface to bind. Default 127.0.0.1; use 0.0.0.0 only on a network you trust. */
+    host?: string;
+    /** Tier for requests that do not name an Apple model. Default `device`. */
+    tier?: 'device' | 'cloud' | 'auto';
+    /** Require `Authorization: Bearer <apiKey>`. */
+    apiKey?: string;
+    /** `Access-Control-Allow-Origin` value, for browser clients. Off by default. */
+    cors?: string;
+    /** One line per request. Default: none. */
+    log?: (line: string) => void;
+}
+/** Build the server without listening. `serve()` is the one-liner. */
+declare function createServer(options?: ServerOptions): http.Server;
+/** Start listening. Resolves once the port is bound. */
+declare function serve(options?: ServerOptions): Promise<{
+    server: http.Server;
+    url: string;
+    close: () => Promise<void>;
+}>;
+
+export { DEFAULT_PORT, MODELS, type ServerOptions, createServer, serve };

package/dist/server.d.ts

@@ -0,0 +1,51 @@
+import http from 'node:http';
+import { A as AppleLLMOptions } from './client-CXewzZTj.js';
+import 'node:child_process';
+
+/**
+ * An OpenAI-compatible HTTP server for Apple's models.
+ *
+ *   apple-llm serve                        # http://127.0.0.1:11436/v1
+ *
+ * Point any OpenAI client at it — the official SDKs, LangChain, LlamaIndex,
+ * editor plugins, Open WebUI — with any API key, and it talks to the on-device
+ * model. Implements `POST /v1/chat/completions` (streaming, tools,
+ * `response_format` with JSON Schema, images), `GET /v1/models` and
+ * `GET /health`.
+ *
+ * Safe defaults, because a local model server is reachable by more than you
+ * might think: it binds to 127.0.0.1 only; CORS is off, so a web page you
+ * happen to visit cannot use your model; an API key can be required; and the
+ * default tier is the device, so nothing leaves the machine unless a client
+ * asks for the `apple-private-cloud` model by name.
+ */
+
+declare const DEFAULT_PORT = 11436;
+/** Model ids a client can ask for. Anything else gets the server's default tier. */
+declare const MODELS: {
+    readonly 'apple-on-device': "device";
+    readonly 'apple-private-cloud': "cloud";
+};
+interface ServerOptions extends Omit<AppleLLMOptions, 'tier'> {
+    port?: number;
+    /** Interface to bind. Default 127.0.0.1; use 0.0.0.0 only on a network you trust. */
+    host?: string;
+    /** Tier for requests that do not name an Apple model. Default `device`. */
+    tier?: 'device' | 'cloud' | 'auto';
+    /** Require `Authorization: Bearer <apiKey>`. */
+    apiKey?: string;
+    /** `Access-Control-Allow-Origin` value, for browser clients. Off by default. */
+    cors?: string;
+    /** One line per request. Default: none. */
+    log?: (line: string) => void;
+}
+/** Build the server without listening. `serve()` is the one-liner. */
+declare function createServer(options?: ServerOptions): http.Server;
+/** Start listening. Resolves once the port is bound. */
+declare function serve(options?: ServerOptions): Promise<{
+    server: http.Server;
+    url: string;
+    close: () => Promise<void>;
+}>;
+
+export { DEFAULT_PORT, MODELS, type ServerOptions, createServer, serve };