@agentproto/llm-endpoint 0.8.0 → 0.9.0

package/dist/index.d.ts

@@ -68,6 +68,74 @@ interface ModelPack {
     toolsAllow?: string[];
 }
 
+/**
+ * Connectors — small, declarative probes for the local/LAN inference
+ * runtimes a named endpoint (`endpoints.ts`) commonly points at: LM Studio,
+ * Ollama, vLLM, llama-server, plus a generic OpenAI-compatible fallback for
+ * anything else. A connector answers three questions about a `baseUrl`: is
+ * this runtime actually running there (`probe`), what models does it have
+ * ({@link listModels}), and does it need special request handling
+ * ({@link ConnectorQuirks}).
+ *
+ * Used by `agentproto llm endpoints add/detect/test` and `doctor`'s
+ * inference-endpoints check — never by the proxy's own request routing
+ * (`index.ts`). In particular, LM Studio's GGUF Qwen builds reject a
+ * non-leading system message once tool defs are present; the generic fix
+ * for that lives in the adapter layer (a separate PR) — `quirks` is only a
+ * hook so callers can find out which connector needs it, not a duplicate of
+ * that fix.
+ *
+ * Deliberately dependency-free and side-effect-free at import time, like
+ * `endpoints.ts` and `packs.ts`.
+ */
+type ConnectorId = 'lmstudio' | 'ollama' | 'vllm' | 'llama-server' | 'openai-compatible';
+declare const CONNECTOR_IDS: readonly ConnectorId[];
+declare function isConnectorId(value: unknown): value is ConnectorId;
+type ConnectorModelState = 'loaded' | 'not-loaded' | 'unknown';
+/** One model as reported by a connector's `listModels`. `loadedCtx`/`maxCtx`
+ *  are omitted (not `undefined`-filled) when the runtime's API doesn't
+ *  expose them — e.g. Ollama's `/api/tags`/`/api/ps` carry no context size. */
+interface ConnectorModel {
+    id: string;
+    loadedCtx?: number;
+    maxCtx?: number;
+    device?: string;
+    state: ConnectorModelState;
+}
+interface ConnectorQuirks {
+    /** This runtime's chat template can reject a system message that isn't
+     *  the first message once tool definitions are present (observed on LM
+     *  Studio GGUF Qwen builds) — callers that build the outbound message
+     *  array should keep a single leading system message for it. */
+    requiresLeadingSystemMessage?: boolean;
+}
+interface Connector {
+    id: ConnectorId;
+    label: string;
+    /** Default local port this runtime listens on; `null` for the
+     *  OpenAI-compatible fallback, which has no default port of its own and
+     *  is never probed by default-port detection. */
+    defaultPort: number | null;
+    quirks: ConnectorQuirks;
+    /** Is this runtime actually answering at `baseUrl`? Never throws — a
+     *  network error, timeout, or unexpected shape resolves `false`. */
+    probe(baseUrl: string, fetchImpl?: typeof fetch): Promise<boolean>;
+    /** Best-effort model listing. Never throws — a failed call resolves `[]`. */
+    listModels(baseUrl: string, fetchImpl?: typeof fetch): Promise<ConnectorModel[]>;
+}
+declare const CONNECTORS: Record<ConnectorId, Connector>;
+declare function connectorById(id: string): Connector | undefined;
+/** Identify which connector is serving `baseUrl`, trying the most specific
+ *  probes first and falling back to the generic OpenAI-compatible one.
+ *  Returns `null` only if even the fallback probe fails (nothing OpenAI-
+ *  compatible is reachable there at all). */
+declare function detectConnector(baseUrl: string, fetchImpl?: typeof fetch): Promise<ConnectorId | null>;
+/** Default local ports probed by `agentproto llm endpoints detect`, one per
+ *  runtime that has a real default port (`openai-compatible` doesn't — it's
+ *  only ever selected explicitly, or as the fallback identity for a
+ *  positive probe at some other port/runtime). */
+declare const DEFAULT_LOCAL_PORTS: Readonly<Partial<Record<ConnectorId, number>>>;
+
 /**
  * Named OpenAI-compatible endpoints — N local/LAN model servers (Ollama,
  * llama-server, vLLM, …) configured from a JSON file instead of one-off env
@@ -82,6 +150,7 @@ interface ModelPack {
  * (src/index.ts) and the CLI (`agentproto llm endpoints …`), which must NOT
  * pull in an HTTP server just to read/validate the config file.
  */
+
 /** A JSON value — what `JSON.parse` can ever produce. */
 type JsonValue = string | number | boolean | null | JsonValue[] | {
     [key: string]: JsonValue;
@@ -105,6 +174,13 @@ interface EndpointConfig {
     id: string;
     kind: 'openai';
     baseUrl: string;
+    /** Which local/LAN runtime this points at (`lmstudio`, `ollama`, `vllm`,
+     *  `llama-server`) or the generic `openai-compatible` fallback — see
+     *  `connectors.ts`. Absent on entries written before connectors existed;
+     *  callers that need one (`endpoints test`, `doctor`) treat a missing
+     *  value as `openai-compatible`. Never affects request routing — only
+     *  which connector's `listModels`/`quirks` apply. */
+    connector?: ConnectorId;
     /** Name of the env var holding the key — never the key itself. Absent ⇒
      *  the endpoint is always keyless (a private/LAN server with no auth). */
     apiKeyEnv?: string;
@@ -349,6 +425,7 @@ type UpstreamTestResult = {
  * (never forwarded) rather than tested.
  */
 declare function testUpstream(provider: string): Promise<UpstreamTestResult>;
+declare function adaptAnthropicToOpenAI(payload: any): void;
 declare function stripThinkingFromAnthropicJson(jsonStr: string): string;
 declare function resolveEmptyTurnRetries(): number;
 /**
@@ -424,4 +501,4 @@ declare const server: node_http.Server<typeof IncomingMessage, typeof ServerResp
 /** Démarre le proxy sur `port` (défaut : {@link PORT}). Renvoie le serveur en écoute. */
 declare function start(port?: number): node_http.Server<typeof IncomingMessage, typeof ServerResponse>;
 
-export { CANONICAL_UPSTREAMS, type ConfigurableUpstream, type EndpointConfig, type EndpointDefaultRequestFields, type EndpointsFileLoad, type ForgeUpstream, type ModelRouteContext, type ToolTrimOptions, type UpstreamCredential, type UpstreamSource, type UpstreamStatus, type UpstreamTestResult, buildUpstreamAuthHeaders, buildWafRuleExpression, collectUpstreamStatuses, describeUpstreamStatus, extractEdgeToken, extractInboundToken, getConfiguredEndpoints, isAuthorized, isCanonicalUpstream, isCredentialAllowedOnOpenAiSurface, isEdgeAuthorized, isEmptyAnthropicTurn, isPublicModelListPath, normalizeProxyPath, openaiJsonToAnthropic, parseAccessTokens, parseEndpointsConfig, readEndpointsFromDisk, resetConfiguredEndpointsCache, resolveEmptyTurnRetries, resolveEndpointsFilePath, resolveForgeBaseUrl, resolveModelRoute, resolveNebiusBaseUrl, resolveUpstreamCredential, server, start, stripThinkingFromAnthropicJson, testUpstream, trimTools };
+export { CANONICAL_UPSTREAMS, CONNECTORS, CONNECTOR_IDS, type ConfigurableUpstream, type Connector, type ConnectorId, type ConnectorModel, type ConnectorModelState, type ConnectorQuirks, DEFAULT_LOCAL_PORTS, type EndpointConfig, type EndpointDefaultRequestFields, type EndpointsFileLoad, type ForgeUpstream, type ModelRouteContext, type ToolTrimOptions, type UpstreamCredential, type UpstreamSource, type UpstreamStatus, type UpstreamTestResult, adaptAnthropicToOpenAI, buildUpstreamAuthHeaders, buildWafRuleExpression, collectUpstreamStatuses, connectorById, describeUpstreamStatus, detectConnector, extractEdgeToken, extractInboundToken, getConfiguredEndpoints, isAuthorized, isCanonicalUpstream, isConnectorId, isCredentialAllowedOnOpenAiSurface, isEdgeAuthorized, isEmptyAnthropicTurn, isPublicModelListPath, normalizeProxyPath, openaiJsonToAnthropic, parseAccessTokens, parseEndpointsConfig, readEndpointsFromDisk, resetConfiguredEndpointsCache, resolveEmptyTurnRetries, resolveEndpointsFilePath, resolveForgeBaseUrl, resolveModelRoute, resolveNebiusBaseUrl, resolveUpstreamCredential, server, start, stripThinkingFromAnthropicJson, testUpstream, trimTools };

package/dist/index.mjs

@@ -573,28 +573,28 @@ function flattenContent(content) {
   return content.map((c) => c.text).join("");
 }
 function translateInputToMessages(input, instructions) {
-  const messages = [];
-  if (instructions) {
-    messages.push({ role: "system", content: instructions });
-  }
+  const sysParts = [];
+  if (instructions) sysParts.push(instructions);
+  const rest = [];
   if (typeof input === "string") {
-    messages.push({ role: "user", content: input });
-    return messages;
-  }
-  for (const item of input) {
-    if (item.type === "message") {
-      messages.push({
-        role: item.role === "developer" ? "system" : item.role,
-        content: flattenContent(item.content)
-      });
-    } else if (item.type === "function_call_output") {
-      messages.push({
-        role: "tool",
-        tool_call_id: item.call_id,
-        content: item.output
-      });
+    rest.push({ role: "user", content: input });
+  } else {
+    for (const item of input) {
+      if (item.type === "message") {
+        if (item.role === "developer" || item.role === "system") {
+          const text = flattenContent(item.content);
+          if (text) sysParts.push(text);
+          continue;
+        }
+        rest.push({ role: item.role, content: flattenContent(item.content) });
+      } else if (item.type === "function_call_output") {
+        rest.push({ role: "tool", tool_call_id: item.call_id, content: item.output });
+      }
     }
   }
+  const messages = [];
+  if (sysParts.length) messages.push({ role: "system", content: sysParts.join("\n\n") });
+  messages.push(...rest);
   return messages;
 }
 function translateToolToChatCompletions(tool) {
@@ -1534,6 +1534,159 @@ async function resumeIncompleteLocalQueueBatches() {
     }
   }
 }
+
+// src/connectors.ts
+var CONNECTOR_IDS = ["lmstudio", "ollama", "vllm", "llama-server", "openai-compatible"];
+function isConnectorId(value) {
+  return typeof value === "string" && CONNECTOR_IDS.includes(value);
+}
+var PROBE_TIMEOUT_MS = 2500;
+async function fetchJsonSafe(url, fetchImpl, timeoutMs = PROBE_TIMEOUT_MS) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const res = await fetchImpl(url, { signal: controller.signal });
+    if (!res.ok) return { ok: false };
+    const body = await res.json().catch(() => null);
+    return { ok: true, body };
+  } catch {
+    return { ok: false };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+function runtimeRoot(baseUrl) {
+  return baseUrl.replace(/\/v1\/?$/, "");
+}
+function stripTrailingSlash(url) {
+  return url.replace(/\/+$/, "");
+}
+var lmstudio = {
+  id: "lmstudio",
+  label: "LM Studio",
+  defaultPort: 1234,
+  quirks: { requiresLeadingSystemMessage: true },
+  async probe(baseUrl, fetchImpl = fetch) {
+    const result = await fetchJsonSafe(`${runtimeRoot(baseUrl)}/api/v0/models`, fetchImpl);
+    return result.ok && isRecord(result.body) && Array.isArray(result.body.data);
+  },
+  async listModels(baseUrl, fetchImpl = fetch) {
+    const result = await fetchJsonSafe(`${runtimeRoot(baseUrl)}/api/v0/models`, fetchImpl);
+    if (!result.ok || !isRecord(result.body) || !Array.isArray(result.body.data)) return [];
+    return result.body.data.filter(isRecord).filter((m) => typeof m.id === "string").map((m) => ({
+      id: m.id,
+      ...typeof m.loaded_context_length === "number" ? { loadedCtx: m.loaded_context_length } : {},
+      ...typeof m.max_context_length === "number" ? { maxCtx: m.max_context_length } : {},
+      state: m.state === "loaded" ? "loaded" : m.state === "not-loaded" ? "not-loaded" : "unknown"
+    }));
+  }
+};
+var ollama = {
+  id: "ollama",
+  label: "Ollama",
+  defaultPort: 11434,
+  quirks: {},
+  async probe(baseUrl, fetchImpl = fetch) {
+    const result = await fetchJsonSafe(`${runtimeRoot(baseUrl)}/api/tags`, fetchImpl);
+    return result.ok && isRecord(result.body) && Array.isArray(result.body.models);
+  },
+  async listModels(baseUrl, fetchImpl = fetch) {
+    const root = runtimeRoot(baseUrl);
+    const [tags, ps] = await Promise.all([fetchJsonSafe(`${root}/api/tags`, fetchImpl), fetchJsonSafe(`${root}/api/ps`, fetchImpl)]);
+    const loadedNames = /* @__PURE__ */ new Set();
+    if (ps.ok && isRecord(ps.body) && Array.isArray(ps.body.models)) {
+      for (const m of ps.body.models) {
+        if (isRecord(m) && typeof m.name === "string") loadedNames.add(m.name);
+      }
+    }
+    if (!tags.ok || !isRecord(tags.body) || !Array.isArray(tags.body.models)) return [];
+    return tags.body.models.filter(isRecord).filter((m) => typeof m.name === "string").map((m) => ({ id: m.name, state: loadedNames.has(m.name) ? "loaded" : "not-loaded" }));
+  }
+};
+var llamaServer = {
+  id: "llama-server",
+  label: "llama-server",
+  defaultPort: 8080,
+  quirks: {},
+  async probe(baseUrl, fetchImpl = fetch) {
+    const result = await fetchJsonSafe(`${runtimeRoot(baseUrl)}/props`, fetchImpl);
+    return result.ok && isRecord(result.body) && ("default_generation_settings" in result.body || "model_path" in result.body);
+  },
+  async listModels(baseUrl, fetchImpl = fetch) {
+    const [props, models] = await Promise.all([
+      fetchJsonSafe(`${runtimeRoot(baseUrl)}/props`, fetchImpl),
+      fetchJsonSafe(`${stripTrailingSlash(baseUrl)}/models`, fetchImpl)
+    ]);
+    let ctx;
+    if (props.ok && isRecord(props.body)) {
+      const gen = props.body.default_generation_settings;
+      if (isRecord(gen) && typeof gen.n_ctx === "number") ctx = gen.n_ctx;
+      else if (typeof props.body.n_ctx === "number") ctx = props.body.n_ctx;
+    }
+    if (!models.ok || !isRecord(models.body) || !Array.isArray(models.body.data)) return [];
+    return models.body.data.filter(isRecord).filter((m) => typeof m.id === "string").map((m) => ({ id: m.id, ...ctx !== void 0 ? { loadedCtx: ctx, maxCtx: ctx } : {}, state: "loaded" }));
+  }
+};
+var vllm = {
+  id: "vllm",
+  label: "vLLM",
+  defaultPort: 8e3,
+  quirks: {},
+  async probe(baseUrl, fetchImpl = fetch) {
+    const result = await fetchJsonSafe(`${stripTrailingSlash(baseUrl)}/models`, fetchImpl);
+    if (!result.ok || !isRecord(result.body) || !Array.isArray(result.body.data)) return false;
+    return result.body.data.some((m) => isRecord(m) && typeof m.max_model_len === "number");
+  },
+  async listModels(baseUrl, fetchImpl = fetch) {
+    const result = await fetchJsonSafe(`${stripTrailingSlash(baseUrl)}/models`, fetchImpl);
+    if (!result.ok || !isRecord(result.body) || !Array.isArray(result.body.data)) return [];
+    return result.body.data.filter(isRecord).filter((m) => typeof m.id === "string").map((m) => ({
+      id: m.id,
+      ...typeof m.max_model_len === "number" ? { loadedCtx: m.max_model_len, maxCtx: m.max_model_len } : {},
+      state: "loaded"
+    }));
+  }
+};
+var openaiCompatible = {
+  id: "openai-compatible",
+  label: "OpenAI-compatible",
+  defaultPort: null,
+  quirks: {},
+  async probe(baseUrl, fetchImpl = fetch) {
+    const result = await fetchJsonSafe(`${stripTrailingSlash(baseUrl)}/models`, fetchImpl);
+    return result.ok && isRecord(result.body) && Array.isArray(result.body.data);
+  },
+  async listModels(baseUrl, fetchImpl = fetch) {
+    const result = await fetchJsonSafe(`${stripTrailingSlash(baseUrl)}/models`, fetchImpl);
+    if (!result.ok || !isRecord(result.body) || !Array.isArray(result.body.data)) return [];
+    return result.body.data.filter(isRecord).filter((m) => typeof m.id === "string").map((m) => ({ id: m.id, state: "unknown" }));
+  }
+};
+var CONNECTORS = {
+  lmstudio,
+  ollama,
+  vllm,
+  "llama-server": llamaServer,
+  "openai-compatible": openaiCompatible
+};
+function connectorById(id) {
+  return isConnectorId(id) ? CONNECTORS[id] : void 0;
+}
+var DETECTION_ORDER = ["lmstudio", "ollama", "llama-server", "vllm", "openai-compatible"];
+async function detectConnector(baseUrl, fetchImpl = fetch) {
+  for (const id of DETECTION_ORDER) {
+    if (await CONNECTORS[id].probe(baseUrl, fetchImpl)) return id;
+  }
+  return null;
+}
+var DEFAULT_LOCAL_PORTS = {
+  lmstudio: 1234,
+  ollama: 11434,
+  "llama-server": 8080,
+  vllm: 8e3
+};
+
+// src/endpoints.ts
 var FORBIDDEN_DEFAULT_REQUEST_FIELD_KEYS = /* @__PURE__ */ new Set(["model", "messages", "stream", "tools", "input"]);
 var RESERVED_ENDPOINT_IDS = /* @__PURE__ */ new Set(["forge"]);
 function validateEndpointConfig(raw, where, errors) {
@@ -1541,7 +1694,7 @@ function validateEndpointConfig(raw, where, errors) {
     errors.push(`${where}: expected an object, got ${raw === null ? "null" : typeof raw}`);
     return null;
   }
-  const { id, kind, baseUrl, apiKeyEnv, defaultRequestFields, timeoutMs } = raw;
+  const { id, kind, baseUrl, connector, apiKeyEnv, defaultRequestFields, timeoutMs } = raw;
   let ok = true;
   if (typeof id !== "string" || id.length === 0) {
     errors.push(`${where}.id: required non-empty string`);
@@ -1569,6 +1722,10 @@ function validateEndpointConfig(raw, where, errors) {
       ok = false;
     }
   }
+  if (connector !== void 0 && !isConnectorId(connector)) {
+    errors.push(`${where}.connector: must be one of ${CONNECTOR_IDS.join(", ")} when present (got ${JSON.stringify(connector)})`);
+    ok = false;
+  }
   if (apiKeyEnv !== void 0 && (typeof apiKeyEnv !== "string" || apiKeyEnv.length === 0)) {
     errors.push(`${where}.apiKeyEnv: must be a non-empty string when present`);
     ok = false;
@@ -1605,6 +1762,7 @@ function validateEndpointConfig(raw, where, errors) {
   }
   if (!ok || typeof id !== "string" || typeof baseUrl !== "string" || !validUrl) return null;
   const built = { id, kind: "openai", baseUrl };
+  if (isConnectorId(connector)) built.connector = connector;
   if (typeof apiKeyEnv === "string") built.apiKeyEnv = apiKeyEnv;
   if (builtFields) built.defaultRequestFields = builtFields;
   if (builtTimeout) built.timeoutMs = builtTimeout;
@@ -2559,21 +2717,28 @@ function handleChatCompletionsRequest(req, res, opts) {
   });
 }
 function adaptAnthropicToOpenAI(payload) {
+  const sysParts = [];
   if (payload.system != null) {
-    let sysText = "";
     if (typeof payload.system === "string") {
-      sysText = payload.system;
+      if (payload.system) sysParts.push(payload.system);
     } else if (Array.isArray(payload.system)) {
-      sysText = payload.system.map((b) => typeof b === "string" ? b : b?.text ?? "").filter(Boolean).join("\n\n");
-    }
-    if (sysText) {
-      if (!Array.isArray(payload.messages)) payload.messages = [];
-      if (!payload.messages[0] || payload.messages[0].role !== "system") {
-        payload.messages.unshift({ role: "system", content: sysText });
-      }
+      const text = payload.system.map((b) => typeof b === "string" ? b : b?.text ?? "").filter(Boolean).join("\n\n");
+      if (text) sysParts.push(text);
     }
     delete payload.system;
   }
+  if (Array.isArray(payload.messages)) {
+    payload.messages = payload.messages.filter((m) => {
+      if (m?.role !== "system" && m?.role !== "developer") return true;
+      const text = typeof m.content === "string" ? m.content : Array.isArray(m.content) ? m.content.map((b) => typeof b === "string" ? b : b?.text ?? "").filter(Boolean).join("\n\n") : "";
+      if (text) sysParts.push(text);
+      return false;
+    });
+  }
+  if (sysParts.length) {
+    if (!Array.isArray(payload.messages)) payload.messages = [];
+    payload.messages.unshift({ role: "system", content: sysParts.join("\n\n") });
+  }
   if (payload.tool_choice && typeof payload.tool_choice === "object") {
     const tc = payload.tool_choice;
     if (tc.type === "any") {
@@ -3584,6 +3749,17 @@ var server = createServer((req, res) => {
               );
             }
           });
+          if (status < 200 || status >= 300) {
+            let errBody = "";
+            proxyRes.on("data", (c) => {
+              if (errBody.length < 300) errBody += c.toString("utf8");
+            });
+            proxyRes.on("end", () => {
+              console.error(
+                `[Proxy] upstream error: ${resolvedTarget.provider}:${resolvedTarget.model} status=${status} body=${JSON.stringify(errBody.slice(0, 300))}`
+              );
+            });
+          }
           const contentType = proxyRes.headers["content-type"] || "";
           const isStreaming = payload.stream === true && /text\/event-stream/i.test(contentType);
           const retryEmptyTurn = () => {
@@ -3709,6 +3885,6 @@ function start(port = PORT) {
   });
 }
 
-export { CANONICAL_UPSTREAMS, buildUpstreamAuthHeaders, buildWafRuleExpression, collectUpstreamStatuses, describeUpstreamStatus, extractEdgeToken, extractInboundToken, getConfiguredEndpoints, isAuthorized, isCanonicalUpstream, isCredentialAllowedOnOpenAiSurface, isEdgeAuthorized, isEmptyAnthropicTurn, isPublicModelListPath, normalizeProxyPath, openaiJsonToAnthropic, parseAccessTokens, parseEndpointsConfig, readEndpointsFromDisk, resetConfiguredEndpointsCache, resolveEmptyTurnRetries, resolveEndpointsFilePath, resolveForgeBaseUrl, resolveModelRoute, resolveNebiusBaseUrl, resolveUpstreamCredential, server, start, stripThinkingFromAnthropicJson, testUpstream, trimTools };
+export { CANONICAL_UPSTREAMS, CONNECTORS, CONNECTOR_IDS, DEFAULT_LOCAL_PORTS, adaptAnthropicToOpenAI, buildUpstreamAuthHeaders, buildWafRuleExpression, collectUpstreamStatuses, connectorById, describeUpstreamStatus, detectConnector, extractEdgeToken, extractInboundToken, getConfiguredEndpoints, isAuthorized, isCanonicalUpstream, isConnectorId, isCredentialAllowedOnOpenAiSurface, isEdgeAuthorized, isEmptyAnthropicTurn, isPublicModelListPath, normalizeProxyPath, openaiJsonToAnthropic, parseAccessTokens, parseEndpointsConfig, readEndpointsFromDisk, resetConfiguredEndpointsCache, resolveEmptyTurnRetries, resolveEndpointsFilePath, resolveForgeBaseUrl, resolveModelRoute, resolveNebiusBaseUrl, resolveUpstreamCredential, server, start, stripThinkingFromAnthropicJson, testUpstream, trimTools };
 //# sourceMappingURL=index.mjs.map
 //# sourceMappingURL=index.mjs.map