@agentproto/llm-endpoint 0.8.0 → 0.10.0

package/README.md

@@ -150,11 +150,12 @@ configured from a JSON file instead of a pair of env vars per server:
       "defaultRequestFields": { "chat_template_kwargs": { "enable_thinking": false } },
       "timeoutMs": { "firstTokenMs": 180000 }
     },
-    { "id": "ollama", "kind": "openai", "baseUrl": "http://192.168.1.20:11434/v1" },
+    { "id": "ollama", "kind": "openai", "baseUrl": "http://192.168.1.20:11434/v1", "connector": "ollama" },
     {
       "id": "lmstudio",
       "kind": "openai",
       "baseUrl": "http://127.0.0.1:1234/v1",
+      "connector": "lmstudio",
       "defaultRequestFields": { "reasoning_effort": "none" }
     }
   ]
@@ -172,6 +173,13 @@ a 500 for the whole listing). `forge` itself keeps working unchanged — it's
 the implicit endpoint that `FORGE_BASE_URL`/`FORGE_API_KEY` configure; a file
 entry may not reuse the id `"forge"`.
 
+- **`connector`** names which local/LAN runtime `baseUrl` points at —
+  `lmstudio`, `ollama`, `vllm`, `llama-server`, or the generic
+  `openai-compatible` fallback (see `src/connectors.ts`). Optional: it never
+  affects request routing, only which connector's `listModels`/request quirks
+  `agentproto llm endpoints test`/`detect`/`doctor` use to show loaded models
+  and context size. An entry with no `connector` is treated as
+  `openai-compatible`.
 - **`apiKeyEnv`** names an env var (never the key itself). Absent, or the env
   var unset, means the endpoint is always keyless — no `Authorization` header
   sent, same as `forge` with no `FORGE_API_KEY`.

package/dist/cli.mjs

@@ -575,28 +575,28 @@ function flattenContent(content) {
   return content.map((c) => c.text).join("");
 }
 function translateInputToMessages(input, instructions) {
-  const messages = [];
-  if (instructions) {
-    messages.push({ role: "system", content: instructions });
-  }
+  const sysParts = [];
+  if (instructions) sysParts.push(instructions);
+  const rest = [];
   if (typeof input === "string") {
-    messages.push({ role: "user", content: input });
-    return messages;
-  }
-  for (const item of input) {
-    if (item.type === "message") {
-      messages.push({
-        role: item.role === "developer" ? "system" : item.role,
-        content: flattenContent(item.content)
-      });
-    } else if (item.type === "function_call_output") {
-      messages.push({
-        role: "tool",
-        tool_call_id: item.call_id,
-        content: item.output
-      });
+    rest.push({ role: "user", content: input });
+  } else {
+    for (const item of input) {
+      if (item.type === "message") {
+        if (item.role === "developer" || item.role === "system") {
+          const text = flattenContent(item.content);
+          if (text) sysParts.push(text);
+          continue;
+        }
+        rest.push({ role: item.role, content: flattenContent(item.content) });
+      } else if (item.type === "function_call_output") {
+        rest.push({ role: "tool", tool_call_id: item.call_id, content: item.output });
+      }
     }
   }
+  const messages = [];
+  if (sysParts.length) messages.push({ role: "system", content: sysParts.join("\n\n") });
+  messages.push(...rest);
   return messages;
 }
 function translateToolToChatCompletions(tool) {
@@ -1536,6 +1536,14 @@ async function resumeIncompleteLocalQueueBatches() {
     }
   }
 }
+
+// src/connectors.ts
+var CONNECTOR_IDS = ["lmstudio", "ollama", "vllm", "llama-server", "openai-compatible"];
+function isConnectorId(value) {
+  return typeof value === "string" && CONNECTOR_IDS.includes(value);
+}
+
+// src/endpoints.ts
 var FORBIDDEN_DEFAULT_REQUEST_FIELD_KEYS = /* @__PURE__ */ new Set(["model", "messages", "stream", "tools", "input"]);
 var RESERVED_ENDPOINT_IDS = /* @__PURE__ */ new Set(["forge"]);
 function validateEndpointConfig(raw, where, errors) {
@@ -1543,7 +1551,7 @@ function validateEndpointConfig(raw, where, errors) {
     errors.push(`${where}: expected an object, got ${raw === null ? "null" : typeof raw}`);
     return null;
   }
-  const { id, kind, baseUrl, apiKeyEnv, defaultRequestFields, timeoutMs } = raw;
+  const { id, kind, baseUrl, connector, apiKeyEnv, defaultRequestFields, timeoutMs } = raw;
   let ok = true;
   if (typeof id !== "string" || id.length === 0) {
     errors.push(`${where}.id: required non-empty string`);
@@ -1571,6 +1579,10 @@ function validateEndpointConfig(raw, where, errors) {
       ok = false;
     }
   }
+  if (connector !== void 0 && !isConnectorId(connector)) {
+    errors.push(`${where}.connector: must be one of ${CONNECTOR_IDS.join(", ")} when present (got ${JSON.stringify(connector)})`);
+    ok = false;
+  }
   if (apiKeyEnv !== void 0 && (typeof apiKeyEnv !== "string" || apiKeyEnv.length === 0)) {
     errors.push(`${where}.apiKeyEnv: must be a non-empty string when present`);
     ok = false;
@@ -1607,6 +1619,7 @@ function validateEndpointConfig(raw, where, errors) {
   }
   if (!ok || typeof id !== "string" || typeof baseUrl !== "string" || !validUrl) return null;
   const built = { id, kind: "openai", baseUrl };
+  if (isConnectorId(connector)) built.connector = connector;
   if (typeof apiKeyEnv === "string") built.apiKeyEnv = apiKeyEnv;
   if (builtFields) built.defaultRequestFields = builtFields;
   if (builtTimeout) built.timeoutMs = builtTimeout;
@@ -1882,6 +1895,29 @@ function resolveNebiusBaseUrl(raw = process.env.NEBIUS_BASE_URL) {
   const value = raw?.trim() || CONFIGURABLE_PROVIDERS.nebius.defaultBaseUrl;
   return parseConfigurableUpstreamUrl(value, "nebius");
 }
+var DEVICE_PROVIDER_RE = /^(.+)@([^@/]+)$/;
+function parseDeviceProvider(provider) {
+  const m = DEVICE_PROVIDER_RE.exec(provider);
+  return m ? { endpointId: m[1], device: m[2] } : null;
+}
+function resolveDeviceProviderSpec(device) {
+  const unavailableMessage = () => `"@${device}" device-inference routing requires this llm-endpoint sidecar to be started by an agentproto daemon (LLM_ENDPOINT_DAEMON_URL is unset) \u2014 run it via \`agentproto serve\` with features.llmEndpoint on, not as a bare standalone process.`;
+  return {
+    keyRequired: true,
+    apiKeyEnv: "LLM_ENDPOINT_DAEMON_TOKEN",
+    resolveUpstream: () => {
+      const daemonUrl = process.env.LLM_ENDPOINT_DAEMON_URL?.trim();
+      if (!daemonUrl) return null;
+      const base = parseUpstreamUrl(daemonUrl);
+      if (!base) return null;
+      return {
+        ...base,
+        pathPrefix: `${base.pathPrefix}/devices/${encodeURIComponent(device)}/exec-stream/device-inference/v1`
+      };
+    },
+    unavailableMessage
+  };
+}
 function getConfigurableProviderSpec(provider) {
   const staticSpec = CONFIGURABLE_PROVIDERS[provider];
   if (staticSpec) {
@@ -1903,6 +1939,8 @@ function getConfigurableProviderSpec(provider) {
       unavailableMessage: () => `"${provider}" endpoint is misconfigured (invalid baseUrl).`
     };
   }
+  const deviceRoute = parseDeviceProvider(provider);
+  if (deviceRoute) return resolveDeviceProviderSpec(deviceRoute.device);
   return void 0;
 }
 function applyDefaultRequestFields(payload, defaults, skipKeys) {
@@ -2004,7 +2042,7 @@ async function probeFileEndpointModels(endpoint) {
   return result;
 }
 function isKnownProvider(provider) {
-  return KNOWN_PROVIDERS.has(provider) || getConfiguredEndpoints().some((e) => e.id === provider);
+  return KNOWN_PROVIDERS.has(provider) || getConfiguredEndpoints().some((e) => e.id === provider) || DEVICE_PROVIDER_RE.test(provider);
 }
 function applyProviderOverride(target, providerOverride) {
   const route = { provider: target.provider, model: target.model };
@@ -2021,7 +2059,7 @@ function parseAnyTransparentModel(model) {
   const slashIdx = model.indexOf("/");
   if (slashIdx <= 0 || slashIdx === model.length - 1) return null;
   const provider = model.slice(0, slashIdx);
-  if (!getConfiguredEndpoints().some((e) => e.id === provider)) return null;
+  if (!getConfiguredEndpoints().some((e) => e.id === provider) && !DEVICE_PROVIDER_RE.test(provider)) return null;
   return { provider, model: model.slice(slashIdx + 1) };
 }
 function resolveModelRoute(payload, ctx, localPacks = getLocalPacks()) {
@@ -2493,6 +2531,8 @@ function handleChatCompletionsRequest(req, res, opts) {
         return;
       }
       payload.model = resolvedTarget.model;
+      const deviceRoute = parseDeviceProvider(resolvedTarget.provider);
+      if (deviceRoute) payload.model = `${deviceRoute.endpointId}/${resolvedTarget.model}`;
       trimTools(payload, {
         provider: resolvedTarget.provider,
         queryTools: opts.queryTools,
@@ -2532,7 +2572,11 @@ function handleChatCompletionsRequest(req, res, opts) {
         method: "POST",
         headers: {
           "Content-Type": "application/json",
-          ...targetApiKey ? { "Authorization": `Bearer ${targetApiKey}` } : {}
+          ...targetApiKey ? { "Authorization": `Bearer ${targetApiKey}` } : {},
+          // See the /v1/messages handler's identical header — the daemon's
+          // /devices/:id/exec-stream 400s without it, always POST outer verb
+          // notwithstanding.
+          ...deviceRoute ? { "x-agentproto-forward-method": "POST" } : {}
         }
       };
       const proxyReq = sendUpstreamRequest(protocol, options, (proxyRes) => {
@@ -2558,21 +2602,28 @@ function handleChatCompletionsRequest(req, res, opts) {
   });
 }
 function adaptAnthropicToOpenAI(payload) {
+  const sysParts = [];
   if (payload.system != null) {
-    let sysText = "";
     if (typeof payload.system === "string") {
-      sysText = payload.system;
+      if (payload.system) sysParts.push(payload.system);
     } else if (Array.isArray(payload.system)) {
-      sysText = payload.system.map((b) => typeof b === "string" ? b : b?.text ?? "").filter(Boolean).join("\n\n");
-    }
-    if (sysText) {
-      if (!Array.isArray(payload.messages)) payload.messages = [];
-      if (!payload.messages[0] || payload.messages[0].role !== "system") {
-        payload.messages.unshift({ role: "system", content: sysText });
-      }
+      const text = payload.system.map((b) => typeof b === "string" ? b : b?.text ?? "").filter(Boolean).join("\n\n");
+      if (text) sysParts.push(text);
     }
     delete payload.system;
   }
+  if (Array.isArray(payload.messages)) {
+    payload.messages = payload.messages.filter((m) => {
+      if (m?.role !== "system" && m?.role !== "developer") return true;
+      const text = typeof m.content === "string" ? m.content : Array.isArray(m.content) ? m.content.map((b) => typeof b === "string" ? b : b?.text ?? "").filter(Boolean).join("\n\n") : "";
+      if (text) sysParts.push(text);
+      return false;
+    });
+  }
+  if (sysParts.length) {
+    if (!Array.isArray(payload.messages)) payload.messages = [];
+    payload.messages.unshift({ role: "system", content: sysParts.join("\n\n") });
+  }
   if (payload.tool_choice && typeof payload.tool_choice === "object") {
     const tc = payload.tool_choice;
     if (tc.type === "any") {
@@ -3362,6 +3413,8 @@ var server = createServer((req, res) => {
       let cred;
       let headers = { "Content-Type": "application/json" };
       payload.model = resolvedTarget.model;
+      const deviceRoute = parseDeviceProvider(resolvedTarget.provider);
+      if (deviceRoute) payload.model = `${deviceRoute.endpointId}/${resolvedTarget.model}`;
       trimTools(payload, {
         provider: resolvedTarget.provider,
         queryTools,
@@ -3387,6 +3440,7 @@ var server = createServer((req, res) => {
         cred = await resolveUpstreamCredential(resolvedTarget.provider);
         targetApiKey = cred?.value ?? "";
         if (cred && cred.value) Object.assign(headers, buildUpstreamAuthHeaders(resolvedTarget.provider, cred));
+        if (deviceRoute) headers["x-agentproto-forward-method"] = "POST";
         const clientThinkingEnabled = isRecord(payload.thinking) && payload.thinking.type === "enabled";
         adaptAnthropicToOpenAI(payload);
         applyDefaultRequestFields(
@@ -3583,6 +3637,17 @@ var server = createServer((req, res) => {
               );
             }
           });
+          if (status < 200 || status >= 300) {
+            let errBody = "";
+            proxyRes.on("data", (c) => {
+              if (errBody.length < 300) errBody += c.toString("utf8");
+            });
+            proxyRes.on("end", () => {
+              console.error(
+                `[Proxy] upstream error: ${resolvedTarget.provider}:${resolvedTarget.model} status=${status} body=${JSON.stringify(errBody.slice(0, 300))}`
+              );
+            });
+          }
           const contentType = proxyRes.headers["content-type"] || "";
           const isStreaming = payload.stream === true && /text\/event-stream/i.test(contentType);
           const retryEmptyTurn = () => {