@cohortapp/agent-sdk 2.3.0

package/lib/model-router/llm-task.mjs

@@ -0,0 +1,515 @@
+/**
+ * lib/model-router/llm-task.mjs — the sanctioned cheap-model one-shot path.
+ *
+ * `llmTask(prompt, opts)` is a single, TOOL-LESS, direct-API call to the
+ * CHEAPEST direct-harness catalog model that meets `opts.minGrade` (and whose
+ * credential is present + data_class is permitted). It is the home for every
+ * classification / lookup / dedup / extraction subtask that does NOT need a full
+ * Claude Code session — the future replacement for the classifier's hard
+ * `openai` dependency (SPEC §6.3, §10 classifier seam).
+ *
+ * Two wires, both via an INJECTED `fetchImpl` (no real network here — the daemon
+ * supplies one; tests stub it):
+ *   - anthropic-messages : { model, max_tokens, system?, messages:[{role,content}] }
+ *                          → POST <endpoints.anthropic||api.anthropic.com>/v1/messages
+ *   - openai-completions : { model, max_tokens, messages:[...] }
+ *                          → POST <endpoints.openai>/chat/completions
+ * The wire is derived from the chosen row's provider (anthropic → messages, all
+ * others → openai-completions), matching resolve.wireFor.
+ *
+ * INVARIANTS (security checklist §13.6):
+ *   - tool-less by construction — no tools[] is ever sent; cheap/untrusted models
+ *     return DATA, never take actions.
+ *   - never throws — fail-open to `{ ok:false, reason, text:"" }`. A classifier
+ *     that crashes is worse than one that returns "route to the safe default".
+ *   - writes ONE ledger row per call (estimated cost from the catalog rates, or
+ *     the usage the provider reported) so budget-guard sees cheap-model spend.
+ *   - non-Anthropic payloads pass through an injectable PII-mask (opts.maskInput,
+ *     §7.4) before send.
+ *
+ * House style: ESM .mjs, Node built-ins only beyond the sibling imports,
+ * everything injectable (fetchImpl, catalog, env, clock, ledger writer).
+ *
+ * @module lib/model-router/llm-task
+ */
+
+import { loadCatalogCached, listModels, lookupModel } from "./catalog.mjs";
+import { estimateCost, writeLedgerRow } from "./ledger.mjs";
+import { keyFor, isOpen as healthIsOpen } from "./health.mjs";
+import { classifyHttp, classifyTransportError } from "./taxonomy.mjs";
+import { repairToolCalls } from "./repair.mjs";
+
+const GRADE_RANK = { A: 3, B: 2, C: 1 };
+const DATA_CLASSES = new Set(["public", "internal", "sensitive"]);
+
+function nowMs(opts) {
+  return (opts && typeof opts.now === "function" ? opts.now : Date.now)();
+}
+
+function envOf(opts) {
+  return (opts && opts.env) || process.env || {};
+}
+
+function isForceAnthropic(env) {
+  const v = env && env.MAESTRO_ROUTER_FORCE_ANTHROPIC;
+  return v === "1" || v === "true";
+}
+
+function resolveCatalog(opts) {
+  if (opts && opts.catalog) {
+    return opts.catalog.byRef ? opts.catalog : opts.catalog.catalog || opts.catalog;
+  }
+  return loadCatalogCached(opts && opts.agentRoot, opts || {});
+}
+
+/** A per-MTok "blended" sort key: input + output (cheapest direct model first). */
+function priceKey(row) {
+  const c = (row && row.cost) || {};
+  const i = Number.isFinite(c.input) ? c.input : Infinity;
+  const o = Number.isFinite(c.output) ? c.output : Infinity;
+  if (i === Infinity && o === Infinity) return Infinity;
+  return (i === Infinity ? 0 : i) + (o === Infinity ? 0 : o);
+}
+
+/**
+ * Pick the cheapest direct-harness, available catalog row that:
+ *   - meets opts.minGrade (tool_reliability >= minGrade; default "C" = any),
+ *   - has its credential present (key-absence enforcement, §7.1) — Anthropic
+ *     direct rows need ANTHROPIC_API_KEY; third-party rows need their auth_env,
+ *   - is allowed for the request's data_class (deny-default), and
+ *   - has a closed breaker.
+ * Returns { row, wire, reason? }. row=null with a reason when nothing qualifies.
+ *
+ * @returns {{row:object|null, reason?:string}}
+ */
+export function pickDirectModel(catalog, request = {}, opts = {}) {
+  const env = envOf(opts);
+  const minGrade = opts.minGrade || request.minGrade || "C";
+  const minRank = GRADE_RANK[minGrade] || GRADE_RANK.C;
+  const dataClass = DATA_CLASSES.has(request.data_class)
+    ? request.data_class
+    : (opts.config && opts.config.defaults && opts.config.defaults.data_class) || "sensitive";
+
+  const allowed = buildAllowedDataClasses(opts.config);
+  const isOpen =
+    typeof opts.isOpen === "function"
+      ? (key) => opts.isOpen(key, nowMs(opts))
+      : (key) => healthIsOpen(key, nowMs(opts), opts.breakerDeps || { agentRoot: opts.agentRoot });
+
+  const rows = listModels(catalog, { harness: "direct", available: true })
+    .slice()
+    .sort((a, b) => priceKey(a) - priceKey(b));
+
+  let firstReason = null;
+  for (const row of rows) {
+    // Grade gate.
+    const rank = row.tool_reliability ? GRADE_RANK[row.tool_reliability] : 0;
+    if (rank < minRank) {
+      firstReason = firstReason || "grade_too_low";
+      continue;
+    }
+    // Data-class gate (deny-by-default).
+    const adc = allowed[row.provider];
+    if (adc && !adc.has(dataClass)) {
+      firstReason = firstReason || "data_class_denied";
+      continue;
+    }
+    if (!adc && row.provider !== "anthropic" && dataClass !== "public") {
+      firstReason = firstReason || "data_class_denied";
+      continue;
+    }
+    // Credential gate.
+    const authEnv = credentialEnvFor(row);
+    if (authEnv) {
+      const tok = env[authEnv];
+      if (typeof tok !== "string" || tok.trim() === "") {
+        firstReason = firstReason || "missing_credential";
+        continue;
+      }
+    }
+    // Breaker gate (backend + backend:model).
+    if (isOpen(keyFor(row.provider)).open || isOpen(keyFor(row.provider, row.id)).open) {
+      firstReason = firstReason || "breaker_open";
+      continue;
+    }
+    return { row };
+  }
+  return { row: null, reason: firstReason || "no_direct_candidate" };
+}
+
+/** The env var a direct row needs: third-party auth_env, or ANTHROPIC_API_KEY. */
+function credentialEnvFor(row) {
+  if (row.provider === "anthropic") return "ANTHROPIC_API_KEY";
+  return row.auth_env || null;
+}
+
+function buildAllowedDataClasses(config) {
+  const out = {};
+  const backends = (config && config.backends) || {};
+  for (const [name, b] of Object.entries(backends)) {
+    const adc = Array.isArray(b && b.allowed_data_classes) ? b.allowed_data_classes : null;
+    if (adc) out[name] = new Set(adc.filter((c) => DATA_CLASSES.has(c)));
+  }
+  if (!out.anthropic) out.anthropic = new Set(["public", "internal", "sensitive"]);
+  return out;
+}
+
+// ---------------------------------------------------------------------------
+// Wire request construction + response parsing
+// ---------------------------------------------------------------------------
+
+function wireFor(row) {
+  return row.provider === "anthropic" ? "anthropic-messages" : "openai-completions";
+}
+
+function endpointFor(row, wire) {
+  const eps = (row && row.endpoints) || {};
+  if (wire === "anthropic-messages") {
+    const base = eps.anthropic || "https://api.anthropic.com";
+    return joinUrl(base, "/v1/messages");
+  }
+  const base = eps.openai || "https://api.openai.com";
+  return joinUrl(base, "/chat/completions");
+}
+
+function joinUrl(base, path) {
+  const b = String(base).replace(/\/+$/, "");
+  // If the base already ends in the wire suffix (e.g. /v1), don't double it.
+  if (path === "/v1/messages" && /\/v1$/.test(b)) return `${b}/messages`;
+  return `${b}${path}`;
+}
+
+/**
+ * Build the request body + headers for the chosen wire. The body is tool-less:
+ * no tools[] is ever attached.
+ */
+function buildRequest(row, wire, { prompt, system, maxTokens, env, temperature }) {
+  const max_tokens = Number.isFinite(maxTokens) ? maxTokens : 1024;
+  const messages = [{ role: "user", content: prompt }];
+  if (wire === "anthropic-messages") {
+    const headers = {
+      "content-type": "application/json",
+      "anthropic-version": "2023-06-01",
+    };
+    const key = env.ANTHROPIC_API_KEY;
+    if (typeof key === "string" && key.trim()) headers["x-api-key"] = key;
+    const body = { model: row.id, max_tokens, messages };
+    if (system) body.system = system;
+    if (temperature != null && !(row.compat && row.compat.omit_temperature)) body.temperature = temperature;
+    return { headers, body };
+  }
+  // openai-completions
+  const headers = { "content-type": "application/json" };
+  const authEnv = row.auth_env;
+  const key = authEnv ? env[authEnv] : undefined;
+  if (typeof key === "string" && key.trim()) headers["authorization"] = `Bearer ${key}`;
+  const msgs = system ? [{ role: "system", content: system }, ...messages] : messages;
+  const maxField = (row.compat && row.compat.max_tokens_field) || "max_tokens";
+  const body = { model: row.id, messages: msgs };
+  body[maxField] = max_tokens;
+  if (temperature != null && !(row.compat && row.compat.omit_temperature)) body.temperature = temperature;
+  return { headers, body };
+}
+
+/**
+ * Extract { text, usage } from a parsed provider response for the given wire.
+ * usage = { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens }.
+ */
+function parseResponse(json, wire) {
+  if (!json || typeof json !== "object") return { text: "", usage: {} };
+  if (wire === "anthropic-messages") {
+    let text = "";
+    if (Array.isArray(json.content)) {
+      for (const b of json.content) {
+        if (b && b.type === "text" && typeof b.text === "string") text += b.text;
+      }
+    }
+    const u = json.usage || {};
+    return {
+      text,
+      usage: {
+        inputTokens: numOr0(u.input_tokens),
+        outputTokens: numOr0(u.output_tokens),
+        cacheReadTokens: numOr0(u.cache_read_input_tokens),
+        cacheWriteTokens: numOr0(u.cache_creation_input_tokens),
+      },
+    };
+  }
+  // openai-completions
+  let text = "";
+  const choice = Array.isArray(json.choices) ? json.choices[0] : null;
+  if (choice && choice.message && typeof choice.message.content === "string") {
+    text = choice.message.content;
+  } else if (choice && typeof choice.text === "string") {
+    text = choice.text;
+  }
+  const u = json.usage || {};
+  return {
+    text,
+    usage: {
+      inputTokens: numOr0(u.prompt_tokens),
+      outputTokens: numOr0(u.completion_tokens),
+      cacheReadTokens: numOr0(u.prompt_tokens_details && u.prompt_tokens_details.cached_tokens),
+      cacheWriteTokens: 0,
+    },
+    raw: choice && choice.message ? choice.message : null,
+  };
+}
+
+function numOr0(v) {
+  const n = Number(v);
+  return Number.isFinite(n) ? n : 0;
+}
+
+// ---------------------------------------------------------------------------
+// llmTask — the public entry point
+// ---------------------------------------------------------------------------
+
+/**
+ * One-shot, tool-less, cheap-model direct-API call. NEVER throws.
+ *
+ * @param {string} prompt   the user prompt (already built by the caller)
+ * @param {object} [opts]
+ *   @param {(url, init) => Promise<Response>} opts.fetchImpl  REQUIRED injected fetch
+ *       (OpenAI-compatible or anthropic-messages). A Response-like with
+ *       { ok, status, json(), text() }. A throw is classified as a transport error.
+ *   @param {string}  [opts.minGrade="C"]   tool_reliability floor
+ *   @param {object}  [opts.request]        RouteRequest-ish: { data_class, task_class, source, lane, agent }
+ *   @param {string}  [opts.system]         system prompt
+ *   @param {number}  [opts.maxTokens=1024]
+ *   @param {number}  [opts.temperature]
+ *   @param {object}  [opts.catalog]        injected catalog (else loaded)
+ *   @param {object}  [opts.config]         routing config (data_class backends)
+ *   @param {object}  [opts.env]            credential + kill-switch env
+ *   @param {()=>number} [opts.now]
+ *   @param {(input:string)=>string} [opts.maskInput]  PII mask for non-Anthropic (§7.4)
+ *   @param {(catalog, entry, deps)=>any} [opts.writeLedger]  injectable ledger writer
+ *   @param {object}  [opts.ledgerDeps]     { now, ledgerDir, agentRoot }
+ *   @param {string}  [opts.decision_id]    join id for the ledger row
+ *   @param {boolean} [opts.repair=false]   run repairToolCalls over the text
+ * @returns {Promise<{ok, text, usage, model, provider, ref, estCostUSD, wire,
+ *                     reason?, status?, repaired?, toolCalls?}>}
+ */
+export async function llmTask(prompt, opts = {}) {
+  const env = envOf(opts);
+  const request = opts.request || {};
+  const fail = (reason, extra = {}) => ({
+    ok: false,
+    text: "",
+    usage: {},
+    model: null,
+    provider: null,
+    ref: null,
+    estCostUSD: null,
+    wire: null,
+    reason,
+    ...extra,
+  });
+
+  if (typeof prompt !== "string" || !prompt) return fail("empty_prompt");
+  const fetchImpl = typeof opts.fetchImpl === "function" ? opts.fetchImpl : null;
+  if (!fetchImpl) return fail("no_fetch_impl");
+
+  const catalog = safe(() => resolveCatalog(opts), null);
+  if (!catalog) return fail("no_catalog");
+
+  // Kill switch: force Anthropic — restrict selection to anthropic direct rows by
+  // requesting a public-or-actual data_class against an anthropic-only view. The
+  // simplest honest behavior is to pick the cheapest anthropic direct row.
+  let chosen;
+  if (isForceAnthropic(env)) {
+    chosen = pickAnthropicDirect(catalog, request, opts);
+  } else {
+    chosen = pickDirectModel(catalog, request, opts);
+  }
+  if (!chosen.row) return fail(chosen.reason || "no_direct_candidate");
+
+  const row = chosen.row;
+  const wire = wireFor(row);
+
+  // PII-mask non-Anthropic payloads (§7.4). Default: identity (no-op) so we never
+  // take a hard dep; the daemon injects lib/security's masker.
+  let sendPrompt = prompt;
+  let sendSystem = opts.system;
+  if (row.provider !== "anthropic" && typeof opts.maskInput === "function") {
+    try {
+      sendPrompt = String(opts.maskInput(prompt) ?? prompt);
+      if (sendSystem) sendSystem = String(opts.maskInput(sendSystem) ?? sendSystem);
+    } catch {
+      /* mask failure must not block — fall back to the original (still tool-less) */
+    }
+  }
+
+  const { headers, body } = buildRequest(row, wire, {
+    prompt: sendPrompt,
+    system: sendSystem,
+    maxTokens: opts.maxTokens,
+    temperature: opts.temperature,
+    env,
+  });
+  const url = endpointFor(row, wire);
+
+  // Execute via the injected fetch. Classify HTTP / transport faults onto the
+  // taxonomy; never throw.
+  let res;
+  try {
+    res = await fetchImpl(url, {
+      method: "POST",
+      headers,
+      body: JSON.stringify(body),
+    });
+  } catch (err) {
+    const reason = classifyTransportError(err);
+    writeRow(catalog, row, wire, {}, opts, request, reason);
+    return fail(reason, { model: row.id, provider: row.provider, ref: row.ref, wire });
+  }
+
+  // Parse the body (json() preferred, text() fallback).
+  let parsedBody = null;
+  let rawText = null;
+  try {
+    if (res && typeof res.json === "function") parsedBody = await res.json();
+  } catch {
+    /* fall through to text */
+  }
+  if (parsedBody == null) {
+    try {
+      if (res && typeof res.text === "function") {
+        rawText = await res.text();
+        parsedBody = safe(() => JSON.parse(rawText), rawText);
+      }
+    } catch {
+      /* */
+    }
+  }
+
+  const status = res && Number.isFinite(res.status) ? res.status : null;
+  const ok = res ? res.ok === true || (status != null && status >= 200 && status < 300) : false;
+
+  if (!ok) {
+    const reason = classifyHttp(status, parsedBody ?? rawText ?? null);
+    writeRow(catalog, row, wire, {}, opts, request, reason);
+    return fail(reason, { model: row.id, provider: row.provider, ref: row.ref, wire, status });
+  }
+
+  const { text, usage } = parseResponse(parsedBody, wire);
+  if (!text || !text.trim()) {
+    // A 2xx with no content is an empty_response — surface it as a soft fail so
+    // the caller can fall back, but still ledger the (likely tiny) usage.
+    const est = estimateCost(catalog, {
+      ref: row.ref,
+      inputTokens: usage.inputTokens,
+      outputTokens: usage.outputTokens,
+      cacheReadTokens: usage.cacheReadTokens,
+      cacheWriteTokens: usage.cacheWriteTokens,
+      budget: false,
+    });
+    writeRow(catalog, row, wire, usage, opts, request, "empty_response");
+    return {
+      ok: false,
+      text: "",
+      usage,
+      model: row.id,
+      provider: row.provider,
+      ref: row.ref,
+      estCostUSD: est.usd,
+      wire,
+      reason: "empty_response",
+      status,
+    };
+  }
+
+  // Cost: price the OBSERVED usage at sticker (actuals), not the budget projection.
+  const est = estimateCost(catalog, {
+    ref: row.ref,
+    inputTokens: usage.inputTokens,
+    outputTokens: usage.outputTokens,
+    cacheReadTokens: usage.cacheReadTokens,
+    cacheWriteTokens: usage.cacheWriteTokens,
+    budget: false,
+  });
+
+  writeRow(catalog, row, wire, usage, opts, request, "ok", est.usd);
+
+  const out = {
+    ok: true,
+    text,
+    usage,
+    model: row.id,
+    provider: row.provider,
+    ref: row.ref,
+    estCostUSD: est.usd,
+    wire,
+    status,
+  };
+
+  // Optional tool-call repair (cheap models leak text tool calls).
+  if (opts.repair) {
+    const repaired = repairToolCalls(text, opts.repairOpts || {});
+    out.repaired = repaired.repaired;
+    out.toolCalls = repaired.toolCalls;
+    out.content = repaired.content;
+  }
+  return out;
+}
+
+/** Pick the cheapest anthropic direct row (kill-switch path). */
+function pickAnthropicDirect(catalog, request, opts) {
+  const minRank = GRADE_RANK[opts.minGrade || request.minGrade || "C"] || GRADE_RANK.C;
+  const rows = listModels(catalog, { provider: "anthropic", harness: "direct", available: true })
+    .filter((r) => (r.tool_reliability ? GRADE_RANK[r.tool_reliability] : 0) >= minRank)
+    .sort((a, b) => priceKey(a) - priceKey(b));
+  return rows.length ? { row: rows[0] } : { row: null, reason: "no_anthropic_direct" };
+}
+
+/** Write ONE ledger row for this llmTask call. Never throws. */
+function writeRow(catalog, row, wire, usage, opts, request, exitReason, totalCostUSD) {
+  const writer = typeof opts.writeLedger === "function" ? opts.writeLedger : writeLedgerRow;
+  try {
+    writer(
+      catalog,
+      {
+        agent: request.agent ?? opts.agent ?? null,
+        decision_id: opts.decision_id ?? null,
+        task_class: request.task_class ?? "llm_task",
+        source: request.source ?? null,
+        lane: request.lane ?? "realtime",
+        harness: "direct",
+        ref: row.ref,
+        provider: row.provider,
+        model: row.id,
+        inputTokens: usage.inputTokens,
+        outputTokens: usage.outputTokens,
+        cacheReadTokens: usage.cacheReadTokens,
+        cacheWriteTokens: usage.cacheWriteTokens,
+        totalCostUSD,
+        exitReason,
+      },
+      opts.ledgerDeps || { agentRoot: opts.agentRoot }
+    );
+  } catch {
+    /* a ledger we can't write must not fail the task */
+  }
+}
+
+function safe(fn, dflt) {
+  try {
+    return fn();
+  } catch {
+    return dflt;
+  }
+}
+
+export const _internals = {
+  pickDirectModel,
+  pickAnthropicDirect,
+  priceKey,
+  wireFor,
+  endpointFor,
+  buildRequest,
+  parseResponse,
+  buildAllowedDataClasses,
+  credentialEnvFor,
+};
+
+export default { llmTask, pickDirectModel };