@aria-framework/ai 0.1.0

package/providers/openai-compatible.js

@@ -0,0 +1,374 @@
+/**
+ * The OpenAI chat-completions shape — LM Studio, Ollama, vLLM, LiteLLM.
+ *
+ * ONE ADAPTER, MANY SERVERS. All of them expose `POST {baseUrl}/chat/completions` with the same
+ * request and response bodies, so the only thing that varies between them is the URL and whether an
+ * API key is wanted. That is why this file is not called "lmstudio": naming it after one server
+ * would invite a second copy the day somebody points it at Ollama.
+ *
+ * It is NOT the Anthropic shape, which is the whole reason the seam exists — see ../providers/
+ * anthropic.js for what differs.
+ */
+
+'use strict';
+
+const { AiError, fromFetchFailure, redact } = require('../error');
+
+/**
+ * @param {{baseUrl:string, model:string, apiKey?:string, timeoutMs:number, label?:string}} cfg
+ * @param {{system?:string, messages:Array<{role:string,content:string}>, maxTokens?:number,
+ *          temperature?:number, schema?:object, signal?:AbortSignal}} opts
+ * @returns {Promise<{text:string, json:object|null, model:string, usage:object, ms:number}>}
+ */
+async function complete(cfg, opts) {
+  const label = cfg.label || 'The model server';
+  const url = apiRoot(cfg.baseUrl) + '/chat/completions';
+
+  // The system prompt is just a message with role 'system' here. Anthropic takes it as a separate
+  // top-level parameter — one of the several small shape differences that make an adapter necessary.
+  const messages = [];
+  if (opts.system) messages.push({ role: 'system', content: opts.system });
+  for (const m of opts.messages || []) messages.push({ role: m.role, content: m.content });
+
+  const body = {
+    model: cfg.model,
+    messages,
+    max_tokens: opts.maxTokens || 1024,
+    temperature: opts.temperature == null ? 0.2 : opts.temperature,
+    stream: false
+  };
+
+  // SCHEMA-CONSTRAINED DECODING, not a prompt asking nicely for JSON. LM Studio implements OpenAI's
+  // structured-output shape, so the server refuses to emit anything that does not fit — which is
+  // what makes a 9B model dependable for the structured work (the relevance pass, the rewrite).
+  if (opts.schema) {
+    body.response_format = {
+      type: 'json_schema',
+      json_schema: { name: opts.schema.name || 'result', schema: opts.schema.schema || opts.schema, strict: true }
+    };
+  }
+
+  const started = Date.now();
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), cfg.timeoutMs || 60000);
+  if (opts.signal) opts.signal.addEventListener('abort', () => controller.abort(), { once: true });
+
+  let res;
+  try {
+    res = await fetch(url, {
+      method: 'POST',
+      headers: Object.assign(
+        { 'Content-Type': 'application/json' },
+        // LM Studio ignores it; a shared LiteLLM or a hosted vLLM will not.
+        cfg.apiKey ? { Authorization: `Bearer ${cfg.apiKey}` } : {}
+      ),
+      body: JSON.stringify(body),
+      signal: controller.signal
+    });
+  } catch (err) {
+    clearTimeout(timer);
+    throw fromFetchFailure(err, { label, url });
+  }
+
+  // THE TIMER STAYS ARMED UNTIL THE BODY IS READ. It used to be cleared in the fetch's `finally`,
+  // which disarmed the abort controller the moment HEADERS arrived — so a server that answered 200
+  // and then stalled mid-body left `res.json()` awaiting forever, with nothing left to interrupt it.
+  //
+  // That is not a hypothetical shape for this provider: LM Studio evicting or reloading a model
+  // mid-response does exactly that. And the cost is not one slow call. runner.tick() is guarded by
+  // a single-flight `running` flag cleared in its own `finally`; an await that never settles means
+  // that finally never runs, so `running` stays true and EVERY later tick returns early. One stalled
+  // response killed the entire AI job queue until the process was restarted.
+  //
+  // `httpError` reads the body too (res.text()), so it has to be inside this block as well.
+  let payload;
+  try {
+    if (!res.ok) throw await httpError(res, label, cfg.apiKey);
+    payload = await res.json();
+  } catch (err) {
+    // THE ABORT CHECK COMES FIRST, before the AiError short-circuit — the order is the fix.
+    // httpError reads the body too, and its own `.catch(() => '')` swallows an abort that fires
+    // mid-read; it then RETURNS a well-formed AiError (rate_limit, auth, …) built from a body it
+    // never received. With the instanceof check first, that bogus error won — a 429-then-stall was
+    // reported as "rate limiting this app" after the full deadline, and rate_limit being marked
+    // retryable meant withOneRetry re-sent it, reintroducing the doubled wait this file's other
+    // fix removed. An abort is the deadline; nothing constructed after it outranks it.
+    if (controller.signal.aborted || (err && err.name === 'AbortError')) {
+      // Stated as a timeout DIRECTLY, not routed through fromFetchFailure — that helper keys on
+      // err.name === 'AbortError', and the error in hand here is often httpError's AiError built
+      // from a swallowed abort, which it would misfile as 'unreachable'. The signal aborting IS
+      // the deadline; no inspection of the error object can outrank that fact.
+      throw new AiError('timeout', `${label} did not answer in time.`, { cause: err });
+    }
+    if (err instanceof AiError) throw err;
+    throw new AiError('bad_response', `${label} replied with something that is not JSON.`, { cause: err });
+  } finally {
+    clearTimeout(timer);
+  }
+
+  // LM STUDIO ANSWERS 200 FOR AN ENDPOINT IT DOES NOT HAVE, with the failure in the body:
+  //
+  //     POST /chat/completions  ->  200  {"error":"Unexpected endpoint or method. (POST /chat/completions)"}
+  //
+  // `res.ok` is therefore useless as the only gate. Without this check the body has no `choices`,
+  // the code reads an undefined message, and the operator is told the model "returned an empty
+  // completion" — which sent this investigation after a thinking-model theory that was not the
+  // cause. Verified against the real server: the same request on /v1/chat/completions answers
+  // normally.
+  if (payload && payload.error && !payload.choices) {
+    const said = typeof payload.error === 'string'
+      ? payload.error
+      : (payload.error.message || JSON.stringify(payload.error));
+    if (/unexpected endpoint/i.test(said)) {
+      throw new AiError('unconfigured',
+        `${label} does not serve ${url}. The address is probably missing its /v1 — ` +
+        'LM Studio answers 200 with an error body rather than a 404, so this looks like a working ' +
+        `server that says nothing. It reported: ${redact(said, cfg.apiKey)}`);
+    }
+    throw new AiError('bad_response', `${label} refused the request: ${redact(said, cfg.apiKey)}`);
+  }
+
+  const choice = payload && payload.choices && payload.choices[0];
+  const message = (choice && choice.message) || {};
+  const finishReason = (choice && choice.finish_reason) || null;
+
+  // REASONING MODELS ANSWER IN TWO PARTS. Qwen3.5 thinks before it speaks, and a server may hand
+  // that thinking back separately (`reasoning_content`) or inline, wrapped in <think> tags. Neither
+  // is the answer, and both have to be recognised — otherwise a model that reasoned and then ran
+  // out of room looks identical to a broken server.
+  const reasoning = message.reasoning_content || message.reasoning || '';
+  const text = stripThinking(message.content || '');
+
+  if (!text) {
+    throw emptyCompletion({ label, finishReason, reasoning, maxTokens: body.max_tokens, usage: payload.usage });
+  }
+
+  // With a schema the content is still a STRING containing JSON — the server constrains the shape,
+  // it does not parse for you. Parsing here rather than at each call site means a malformed body is
+  // one error type instead of a surprise in a page render.
+  let json = null;
+  if (opts.schema) {
+    try {
+      json = JSON.parse(text);
+    } catch (err) {
+      // THE TRUNCATION CASE GETS ITS OWN SENTENCE, because it is not the model's fault and the
+      // generic message sends the investigation the wrong way. Seen live: the model wrote a good
+      // draft and then looped inside an unbounded array until max_tokens cut the JSON mid-bracket —
+      // finish_reason 'length', unparseable by construction. "Will not parse" reads as a model
+      // problem; "ran out of room" names the actual cause and its two fixes (a schema ceiling, or
+      // a bigger reply limit).
+      if (finishReason === 'length') {
+        throw new AiError('bad_response',
+          `${label} ran out of room mid-answer (${body.max_tokens} tokens) — the structured reply ` +
+          'was cut off before it was finished. If this repeats, the schema may be letting the ' +
+          'model ramble; a Regenerate usually succeeds.', { cause: err });
+      }
+      throw new AiError('bad_response',
+        `${label} was asked for structured output and returned text that will not parse.`, { cause: err });
+    }
+  }
+
+  return {
+    text,
+    json,
+    model: payload.model || cfg.model,
+    usage: normaliseUsage(payload.usage),
+    ms: Date.now() - started,
+    finishReason,
+    // A caller that cares (the summary's coverage line) can tell a complete answer from a cut-off
+    // one without re-reading the payload.
+    truncated: finishReason === 'length',
+    reasonedFor: reasoning ? reasoning.length : 0
+  };
+}
+
+/**
+ * `<think>…</think>` removed.
+ *
+ * Served through an OpenAI-compatible endpoint, a thinking model often leaves its reasoning inline
+ * in `content` rather than in a field of its own. Rendering that to an agent as if it were the
+ * answer is worse than showing nothing.
+ */
+function stripThinking(content) {
+  return String(content || '')
+    .replace(/<think>[\s\S]*?<\/think>/gi, '')
+    .replace(/<thinking>[\s\S]*?<\/thinking>/gi, '')
+    .trim();
+}
+
+/**
+ * Why nothing came back — the message an operator can act on.
+ *
+ * "Returned an empty completion" is true and useless. The three causes are distinguishable from the
+ * payload, so they are distinguished: the budget ran out, the budget went on thinking, or the server
+ * genuinely said nothing.
+ */
+function emptyCompletion({ label, finishReason, reasoning, maxTokens, usage }) {
+  const spent = (usage && usage.completion_tokens) || 0;
+  if (finishReason === 'length' || (reasoning && !spentLeftRoom(spent, maxTokens))) {
+    return new AiError('bad_response',
+      `${label} ran out of room before it answered — it used all ${maxTokens} reply tokens` +
+      (reasoning ? ' on internal reasoning' : '') +
+      '. This model thinks before it replies, so raise the reply limit (1024 is a sensible floor).');
+  }
+  if (reasoning) {
+    return new AiError('bad_response',
+      `${label} returned only its internal reasoning and no answer. Raise the reply limit and try again.`);
+  }
+  return new AiError('bad_response',
+    `${label} returned an empty completion (finish reason: ${finishReason || 'none given'}). ` +
+    'Check the model is fully loaded on the server.');
+}
+
+/** Did the completion stop well short of the ceiling? Then the ceiling was not the problem. */
+function spentLeftRoom(spent, maxTokens) {
+  return maxTokens > 0 && spent > 0 && spent < maxTokens - 2;
+}
+
+async function httpError(res, label, apiKey) {
+  // Redacted before it is used anywhere: a server is free to quote the key back in its error body.
+  const detail = await res.text().then((t) => redact(t.slice(0, 300), apiKey)).catch(() => '');
+  if (res.status === 401 || res.status === 403) {
+    return new AiError('auth', `${label} rejected the API key.`, { status: res.status });
+  }
+  if (res.status === 429) {
+    return new AiError('rate_limit', `${label} is rate limiting this app.`, { status: res.status });
+  }
+  if (res.status === 404) {
+    // The single most common local mistake: a model name that is not the one loaded.
+    return new AiError('unconfigured',
+      `${label} does not have that model loaded (404). Check the model name matches what is running.`,
+      { status: res.status });
+  }
+  return new AiError('bad_response', `${label} returned ${res.status}. ${detail}`.trim(), { status: res.status });
+}
+
+/**
+ * The API root, with the `/v1` an operator reasonably leaves off.
+ *
+ * `http://192.168.250.249:1234` is a perfectly sensible thing to type — it is what LM Studio shows
+ * you — and it is not where the OpenAI-compatible endpoints live. A path that is already there is
+ * left alone, because other servers in this family mount elsewhere (LiteLLM behind a prefix, for
+ * one) and second-guessing an explicit path would break them.
+ */
+function apiRoot(baseUrl) {
+  const trimmed = String(baseUrl || '').replace(/\/+$/, '');
+  let path;
+  try {
+    path = new URL(trimmed).pathname.replace(/\/+$/, '');
+  } catch (err) {
+    return trimmed;   // not a URL we can parse; leave it exactly as typed
+  }
+  return path === '' ? trimmed + '/v1' : trimmed;
+}
+
+/** Both providers report tokens; they name the fields differently. One shape reaches the caller. */
+function normaliseUsage(u) {
+  if (!u) return { prompt: 0, completion: 0, total: 0 };
+  const prompt = u.prompt_tokens || 0;
+  const completion = u.completion_tokens || 0;
+  return { prompt, completion, total: u.total_tokens || prompt + completion };
+}
+
+/** What the server has loaded — used by the admin screen so nobody has to type a model name blind. */
+async function listModels(cfg) {
+  const url = apiRoot(cfg.baseUrl) + '/models';
+  try {
+    const res = await fetch(url, {
+      headers: cfg.apiKey ? { Authorization: `Bearer ${cfg.apiKey}` } : {},
+      signal: AbortSignal.timeout(cfg.timeoutMs || 10000)
+    });
+    if (!res.ok) return [];
+    const payload = await res.json();
+    // The 200-with-an-error-body case needs nothing special here: an error body has no `data`, so
+    // it already yields no models. A guard for it was written and then deleted — a control proved
+    // it could not fail, which is the definition of code that is not doing anything.
+    //
+    // The admin screen says both possibilities out loud ("nothing loaded, or the address may be
+    // wrong") because from here the two are genuinely indistinguishable.
+    return (payload.data || []).map((m) => m.id).filter(Boolean);
+  } catch (err) {
+    // A provider that cannot list models can still complete; this is a convenience, not a check.
+    return [];
+  }
+}
+
+/**
+ * Embed one or more strings.
+ *
+ * SAME ENDPOINT FAMILY, DIFFERENT PATH — `/v1/embeddings`, and the same `apiRoot` correction that
+ * `/chat/completions` needs, because a base URL missing its `/v1` fails here in exactly the way it
+ * failed there: LM Studio answers 200 with an error body rather than a 404.
+ *
+ * ORDER IS THE CONTRACT. The response carries an `index` per vector and providers are not obliged
+ * to return them in order, so they are sorted rather than assumed — a silent re-ordering would pair
+ * every document with the wrong vector, and the result would still be 768 plausible floats.
+ */
+async function embed(cfg, texts) {
+  const list = Array.isArray(texts) ? texts : [texts];
+  if (!list.length) return [];
+  const url = apiRoot(cfg.baseUrl) + '/embeddings';
+  const label = cfg.label || 'The provider';
+
+  const headers = { 'Content-Type': 'application/json' };
+  if (cfg.apiKey) headers.Authorization = `Bearer ${cfg.apiKey}`;
+
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), cfg.timeoutMs || 60000);
+  let res;
+  try {
+    res = await fetch(url, {
+      method: 'POST', headers, signal: controller.signal,
+      body: JSON.stringify({ model: cfg.embeddingModel, input: list })
+    });
+  } catch (err) {
+    clearTimeout(timer);
+    throw fromFetchFailure(err, { label, url });
+  }
+
+  // The timer stays armed until the body is read — the same fix complete() got, which this
+  // function MISSED on the first pass. Three reviews in: complete() was cured in both adapters
+  // while embed(), in the same file, kept the old shape — and the runner awaits embed() inside
+  // tick(), so one embedding-model eviction mid-response wedged the whole AI queue exactly the
+  // way the complete() comment describes. The third copy of a pattern is the one that gets
+  // missed; if a fourth ever appears, extract the scaffolding first.
+  let raw;
+  try {
+    raw = await res.text();
+  } catch (err) {
+    if (controller.signal.aborted || (err && err.name === 'AbortError')) {
+      throw fromFetchFailure(err, { label, url });
+    }
+    throw new AiError('bad_response', `${label} closed the connection mid-answer.`, { cause: err });
+  } finally {
+    clearTimeout(timer);
+  }
+  let payload;
+  try { payload = JSON.parse(raw); } catch (err) {
+    throw new AiError('bad_response', `${label} returned something that is not JSON from ${url}.`);
+  }
+  if (!res.ok || (payload && payload.error && !payload.data)) {
+    const said = payload && payload.error
+      ? (typeof payload.error === 'string' ? payload.error : payload.error.message || JSON.stringify(payload.error))
+      : `HTTP ${res.status}`;
+    if (/unexpected endpoint/i.test(String(said))) {
+      throw new AiError('unconfigured',
+        `${label} does not serve ${url}. The address is probably missing its /v1.`);
+    }
+    throw new AiError('bad_response', `${label} could not embed: ${redact(said, cfg.apiKey)}`);
+  }
+
+  const data = Array.isArray(payload.data) ? payload.data.slice() : [];
+  if (data.length !== list.length) {
+    throw new AiError('bad_response',
+      `${label} returned ${data.length} vectors for ${list.length} inputs — they cannot be paired up.`);
+  }
+  data.sort((a, b) => (a.index || 0) - (b.index || 0));
+  return data.map((d) => {
+    const v = d && d.embedding;
+    if (!Array.isArray(v) || !v.length) throw new AiError('bad_response', `${label} returned an empty vector.`);
+    return v;
+  });
+}
+
+module.exports = { complete, listModels, embed, apiRoot };