@tangle-network/agent-eval 0.120.2 → 0.120.3

package/dist/chunk-NJC7U437.js

@@ -0,0 +1,626 @@
+import {
+  defaultProviderRedactor,
+  providerFromBaseUrl
+} from "./chunk-PC4UYEBM.js";
+import {
+  AgentEvalError,
+  CaptureIntegrityError
+} from "./chunk-ONWEPEDO.js";
+
+// src/llm-client.ts
+function maximumChargeForLlmRequest(request, options = {}) {
+  if (request.maxTokens === void 0) return void 0;
+  if (!Number.isInteger(request.maxTokens) || request.maxTokens <= 0) {
+    throw new RangeError(`maximumChargeForLlmRequest: maxTokens must be a positive integer`);
+  }
+  if (request.messages.some(
+    (message) => Array.isArray(message.content) && message.content.some((part) => part.type === "image_url")
+  )) {
+    return void 0;
+  }
+  const attempts = resolveMaximumAttempts(options.maxRetries);
+  const requestBytes = new TextEncoder().encode(
+    JSON.stringify(buildBody(request, false))
+  ).byteLength;
+  const batches = request.jsonSchema ? 2 : 1;
+  return {
+    model: request.model,
+    inputTokens: requestBytes * attempts * batches,
+    outputTokens: request.maxTokens * attempts * batches
+  };
+}
+function costReceiptFromLlm(result) {
+  const cachedTokens = result.usage.cachedPromptTokens ?? 0;
+  return {
+    model: result.model,
+    inputTokens: Math.max(0, result.usage.promptTokens - cachedTokens),
+    outputTokens: result.usage.completionTokens,
+    cachedTokens: cachedTokens > 0 ? cachedTokens : void 0,
+    actualCostUsd: result.costUsd ?? void 0,
+    usageUnknown: result.usage.captured === false
+  };
+}
+function costReceiptFromLlmError(error) {
+  return error instanceof LlmResponseError ? costReceiptFromLlm(error.result) : void 0;
+}
+var LlmCallError = class extends AgentEvalError {
+  constructor(message, status, body, model) {
+    super("judge", message);
+    this.status = status;
+    this.body = body;
+    this.model = model;
+  }
+  status;
+  body;
+  model;
+};
+var LlmResponseError = class extends AgentEvalError {
+  constructor(message, result, options) {
+    super("judge", message, options);
+    this.result = result;
+  }
+  result;
+};
+var DEFAULT_BASE_URL = "https://router.tangle.tools/v1";
+var DEFAULT_TIMEOUT_MS = Number(process.env.TANGLE_LLM_TIMEOUT_MS) || 3e5;
+var DEFAULT_MAX_RETRIES = Number(process.env.TANGLE_LLM_MAX_RETRIES) || 3;
+function resolveMaximumAttempts(configured) {
+  const attempts = configured ?? DEFAULT_MAX_RETRIES;
+  if (!Number.isInteger(attempts) || attempts <= 0) {
+    throw new RangeError("LLM maximum attempts must be a positive integer");
+  }
+  return attempts;
+}
+function providerTokenCount(value) {
+  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : void 0;
+}
+var RETRYABLE_STATUS = /* @__PURE__ */ new Set([429, 502, 503, 504]);
+var TRANSIENT_ERROR_PATTERNS = [
+  /AbortError/i,
+  /TimeoutError/i,
+  /this operation was aborted/i,
+  /fetch failed/i,
+  /ECONNRESET/i,
+  /ETIMEDOUT/i,
+  /EAI_AGAIN/i,
+  /socket hang up/i,
+  /stream.*ended.*unexpectedly/i,
+  /terminated/i,
+  /other side closed/i,
+  /NGHTTP2/i,
+  /UND_ERR/i
+];
+function isTransientLlmError(err) {
+  return classifyTransient(err, 0);
+}
+function classifyTransient(err, depth) {
+  if (err instanceof LlmCallError) return RETRYABLE_STATUS.has(err.status);
+  if (!(err instanceof Error)) return false;
+  const status = err.status;
+  if (typeof status === "number" && RETRYABLE_STATUS.has(status)) return true;
+  const code = err.code;
+  const haystack = `${err.name}
+${err.message}
+${typeof code === "string" ? code : ""}`;
+  if (TRANSIENT_ERROR_PATTERNS.some((p) => p.test(haystack))) return true;
+  const cause = err.cause;
+  if (depth < 4 && cause instanceof Error && cause !== err) {
+    return classifyTransient(cause, depth + 1);
+  }
+  return false;
+}
+function parseRetryAfter(headers) {
+  const h = headers.get("retry-after");
+  if (!h) return null;
+  const asNumber = Number(h);
+  if (Number.isFinite(asNumber) && asNumber > 0) return asNumber * 1e3;
+  const asDate = Date.parse(h);
+  if (Number.isFinite(asDate)) return Math.max(0, asDate - Date.now());
+  return null;
+}
+function backoffMs(attempt) {
+  return Math.min(500 * 2 ** attempt, 16e3);
+}
+function buildHeaders(opts) {
+  const headers = {
+    "Content-Type": "application/json",
+    Accept: "application/json"
+  };
+  if (opts.authHeader) {
+    headers[opts.authHeader.name] = opts.authHeader.value;
+  } else if (opts.bearer || opts.apiKey) {
+    headers.Authorization = `Bearer ${opts.bearer ?? opts.apiKey}`;
+  }
+  if (opts.idempotencyKey) headers["Idempotency-Key"] = opts.idempotencyKey;
+  return headers;
+}
+function isSchemaRejection(status, body) {
+  if (status !== 400) return false;
+  const lower = body.toLowerCase();
+  return lower.includes("response_format") || lower.includes("json_schema") || lower.includes("is unavailable") || lower.includes("not supported");
+}
+function buildBody(req, forceJsonObject) {
+  const body = {
+    model: req.model,
+    messages: req.messages,
+    temperature: req.temperature ?? 0
+  };
+  if (req.maxTokens != null) {
+    if (usesMaxCompletionTokens(req.model)) body.max_completion_tokens = req.maxTokens;
+    else body.max_tokens = req.maxTokens;
+  }
+  if (req.jsonSchema && !forceJsonObject) {
+    body.response_format = {
+      type: "json_schema",
+      json_schema: { name: req.jsonSchema.name, schema: req.jsonSchema.schema, strict: true }
+    };
+  } else if (req.jsonMode || req.jsonSchema) {
+    body.response_format = { type: "json_object" };
+  }
+  return body;
+}
+function usesMaxCompletionTokens(model) {
+  return /^gpt-5(?:[.-]|$)/i.test(model);
+}
+async function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function linkSignals(timeoutController, caller) {
+  if (!caller) return timeoutController.signal;
+  if (typeof AbortSignal.any === "function") {
+    return AbortSignal.any([timeoutController.signal, caller]);
+  }
+  if (caller.aborted) {
+    timeoutController.abort();
+  } else {
+    caller.addEventListener("abort", () => timeoutController.abort(), { once: true });
+  }
+  return timeoutController.signal;
+}
+function deadlineExceeded(start, deadlineMs) {
+  return deadlineMs != null && Date.now() - start >= deadlineMs;
+}
+function stripFencedJson(raw) {
+  const trimmed = raw.trim();
+  const m = trimmed.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```\s*$/);
+  return m ? m[1].trim() : trimmed;
+}
+function extractJsonPayload(raw) {
+  const stripped = stripFencedJson(raw);
+  try {
+    JSON.parse(stripped);
+    return stripped;
+  } catch {
+    if (stripped.startsWith("{") || stripped.startsWith("[")) return stripped;
+  }
+  const starts = [...stripped.matchAll(/[[{]/g)].map((match) => match.index).filter((index) => index != null);
+  for (const start of starts) {
+    const candidate = extractBalancedJson(stripped, start);
+    if (!candidate) continue;
+    try {
+      JSON.parse(candidate);
+      return candidate;
+    } catch {
+    }
+  }
+  return stripped;
+}
+function extractBalancedJson(input, start) {
+  const opener = input[start];
+  const closer = opener === "{" ? "}" : opener === "[" ? "]" : null;
+  if (!closer) return null;
+  const stack = [closer];
+  let isInString = false;
+  let isEscaped = false;
+  for (let i = start + 1; i < input.length; i++) {
+    const char = input[i];
+    if (isEscaped) {
+      isEscaped = false;
+      continue;
+    }
+    if (char === "\\") {
+      isEscaped = isInString;
+      continue;
+    }
+    if (char === '"') {
+      isInString = !isInString;
+      continue;
+    }
+    if (isInString) continue;
+    if (char === "{") stack.push("}");
+    else if (char === "[") stack.push("]");
+    else if (char === stack[stack.length - 1]) {
+      stack.pop();
+      if (stack.length === 0) return input.slice(start, i + 1);
+    }
+  }
+  return null;
+}
+async function callLlm(req, opts = {}) {
+  const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
+  const url = `${baseUrl}/chat/completions`;
+  const endpoint = "/chat/completions";
+  const timeoutMs = req.timeoutMs ?? opts.defaultTimeoutMs ?? DEFAULT_TIMEOUT_MS;
+  const maximumAttempts = resolveMaximumAttempts(opts.maxRetries);
+  const fetchFn = opts.fetch ?? globalThis.fetch;
+  const headers = buildHeaders(opts);
+  const provider = opts.provider ?? providerFromBaseUrl(baseUrl);
+  const sink = opts.rawSink;
+  const redactor = opts.redactor ?? defaultProviderRedactor;
+  const traceContext = opts.traceContext;
+  const callerSignal = opts.signal;
+  const deadlineMs = opts.deadlineMs;
+  const deadlineStart = Date.now();
+  let lastErr;
+  for (let attempt = 0; attempt < maximumAttempts; attempt++) {
+    if (callerSignal?.aborted) {
+      throw new DOMException("callLlm aborted by caller signal", "AbortError");
+    }
+    if (attempt > 0 && deadlineExceeded(deadlineStart, deadlineMs)) {
+      throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
+    }
+    const controller = new AbortController();
+    const attemptSignal = linkSignals(controller, callerSignal);
+    const timeoutHandle = setTimeout(() => controller.abort(), timeoutMs);
+    const started = Date.now();
+    const requestBody = buildBody(req, false);
+    let attemptErrorRecorded = false;
+    if (sink) {
+      await recordRaw(sink, redactor, {
+        eventId: cryptoEventId(),
+        runId: traceContext?.runId,
+        spanId: traceContext?.spanId,
+        provider,
+        model: req.model,
+        endpoint,
+        baseUrl,
+        attemptIndex: attempt,
+        direction: "request",
+        timestamp: started,
+        requestHeaders: headers,
+        requestBody,
+        redactedFields: []
+      });
+    }
+    try {
+      const res = await fetchFn(url, {
+        method: "POST",
+        headers,
+        body: JSON.stringify(requestBody),
+        signal: attemptSignal
+      });
+      clearTimeout(timeoutHandle);
+      const responseHeaders = sink ? headersToObject(res.headers) : void 0;
+      if (!res.ok) {
+        const body = await res.text();
+        if (sink) {
+          await recordRaw(sink, redactor, {
+            eventId: cryptoEventId(),
+            runId: traceContext?.runId,
+            spanId: traceContext?.spanId,
+            provider,
+            model: req.model,
+            endpoint,
+            baseUrl,
+            attemptIndex: attempt,
+            direction: "error",
+            timestamp: Date.now(),
+            durationMs: Date.now() - started,
+            statusCode: res.status,
+            responseHeaders,
+            responseBody: body,
+            errorMessage: `HTTP ${res.status}`,
+            redactedFields: []
+          });
+          attemptErrorRecorded = true;
+        }
+        const err = new LlmCallError(
+          `LLM call ${res.status}: ${body.slice(0, 300)}`,
+          res.status,
+          body,
+          req.model
+        );
+        if (RETRYABLE_STATUS.has(res.status) && attempt < maximumAttempts - 1 && !deadlineExceeded(deadlineStart, deadlineMs)) {
+          lastErr = err;
+          const retryAfter = parseRetryAfter(res.headers);
+          await sleep(retryAfter ?? backoffMs(attempt));
+          continue;
+        }
+        throw err;
+      }
+      const text = await res.text();
+      let json;
+      try {
+        json = JSON.parse(text);
+      } catch (parseErr) {
+        if (sink) {
+          await recordRaw(sink, redactor, {
+            eventId: cryptoEventId(),
+            runId: traceContext?.runId,
+            spanId: traceContext?.spanId,
+            provider,
+            model: req.model,
+            endpoint,
+            baseUrl,
+            attemptIndex: attempt,
+            direction: "error",
+            timestamp: Date.now(),
+            durationMs: Date.now() - started,
+            statusCode: res.status,
+            responseHeaders,
+            responseBody: text,
+            errorMessage: `non-JSON response: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`,
+            redactedFields: []
+          });
+          attemptErrorRecorded = true;
+        }
+        throw parseErr;
+      }
+      if (sink) {
+        await recordRaw(sink, redactor, {
+          eventId: cryptoEventId(),
+          runId: traceContext?.runId,
+          spanId: traceContext?.spanId,
+          provider,
+          model: req.model,
+          endpoint,
+          baseUrl,
+          attemptIndex: attempt,
+          direction: "response",
+          timestamp: Date.now(),
+          durationMs: Date.now() - started,
+          statusCode: res.status,
+          responseHeaders,
+          responseBody: json,
+          redactedFields: []
+        });
+      }
+      const choice = json.choices?.[0];
+      const usageRaw = json.usage && typeof json.usage === "object" && !Array.isArray(json.usage) ? json.usage : void 0;
+      const promptTokens = providerTokenCount(usageRaw?.prompt_tokens);
+      const completionTokens = providerTokenCount(usageRaw?.completion_tokens);
+      const totalTokens = providerTokenCount(usageRaw?.total_tokens);
+      const cachedRaw = usageRaw?.prompt_tokens_details && typeof usageRaw.prompt_tokens_details === "object" && !Array.isArray(usageRaw.prompt_tokens_details) ? usageRaw.prompt_tokens_details.cached_tokens : void 0;
+      const cachedPromptTokens = cachedRaw === void 0 ? void 0 : providerTokenCount(cachedRaw);
+      const usageCaptured = promptTokens !== void 0 && completionTokens !== void 0 && (cachedRaw === void 0 || cachedPromptTokens !== void 0 && cachedPromptTokens <= promptTokens) && (totalTokens === void 0 || totalTokens === promptTokens + completionTokens);
+      const costFromProxy = json._response_cost ?? json.cost_usd;
+      const content = choice?.message?.content ?? "";
+      return {
+        content,
+        finishReason: choice?.finish_reason ?? null,
+        contentEmpty: content.trim().length === 0,
+        usage: {
+          promptTokens: promptTokens ?? 0,
+          completionTokens: completionTokens ?? 0,
+          totalTokens: totalTokens ?? (promptTokens ?? 0) + (completionTokens ?? 0),
+          captured: usageCaptured,
+          cachedPromptTokens
+        },
+        costUsd: typeof costFromProxy === "number" ? costFromProxy : null,
+        model: json.model ?? req.model,
+        durationMs: Date.now() - started,
+        raw: json
+      };
+    } catch (err) {
+      clearTimeout(timeoutHandle);
+      lastErr = err;
+      if (callerSignal?.aborted) {
+        if (sink && !attemptErrorRecorded) {
+          await recordRaw(sink, redactor, {
+            eventId: cryptoEventId(),
+            runId: traceContext?.runId,
+            spanId: traceContext?.spanId,
+            provider,
+            model: req.model,
+            endpoint,
+            baseUrl,
+            attemptIndex: attempt,
+            direction: "error",
+            timestamp: Date.now(),
+            durationMs: Date.now() - started,
+            errorMessage: err instanceof Error ? err.message : String(err),
+            redactedFields: []
+          });
+        }
+        throw err;
+      }
+      if (sink && !attemptErrorRecorded) {
+        await recordRaw(sink, redactor, {
+          eventId: cryptoEventId(),
+          runId: traceContext?.runId,
+          spanId: traceContext?.spanId,
+          provider,
+          model: req.model,
+          endpoint,
+          baseUrl,
+          attemptIndex: attempt,
+          direction: "error",
+          timestamp: Date.now(),
+          durationMs: Date.now() - started,
+          errorMessage: err instanceof Error ? err.message : String(err),
+          redactedFields: []
+        });
+      }
+      if (attempt < maximumAttempts - 1 && isTransientLlmError(err) && !deadlineExceeded(deadlineStart, deadlineMs)) {
+        await sleep(backoffMs(attempt));
+        continue;
+      }
+      throw err;
+    }
+  }
+  throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
+}
+async function recordRaw(sink, redactor, event) {
+  try {
+    await sink.record(redactor(event));
+  } catch {
+  }
+}
+function headersToObject(h) {
+  const out = {};
+  h.forEach((value, key) => {
+    out[key] = value;
+  });
+  return out;
+}
+function cryptoEventId() {
+  if (typeof globalThis.crypto?.randomUUID === "function") return globalThis.crypto.randomUUID();
+  return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
+}
+async function callLlmJson(req, opts = {}) {
+  const result = await callLlmStructured(req, opts);
+  const value = parseJsonResult(result);
+  return { value, result };
+}
+async function callLlmStructured(req, opts = {}) {
+  try {
+    return await callLlm({ ...req, jsonMode: req.jsonMode ?? !req.jsonSchema }, opts);
+  } catch (err) {
+    if (err instanceof LlmCallError && isSchemaRejection(err.status, err.body) && req.jsonSchema) {
+      const degradedReq = { ...req, jsonMode: true, jsonSchema: void 0 };
+      return await callLlm(degradedReq, opts);
+    }
+    throw err;
+  }
+}
+function parseJsonResult(result) {
+  try {
+    if (result.finishReason === "length") {
+      throw new Error(
+        `LLM returned truncated JSON content (model=${result.model}, finishReason=length)`
+      );
+    }
+    return parseJsonSafely(result.content, result.model);
+  } catch (error) {
+    if (error instanceof LlmResponseError) throw error;
+    const cause = error instanceof Error ? error : new Error(String(error));
+    throw new LlmResponseError(cause.message, result, { cause });
+  }
+}
+function parseJsonSafely(content, model) {
+  const stripped = extractJsonPayload(content);
+  try {
+    return JSON.parse(stripped);
+  } catch (err) {
+    throw new Error(
+      `LLM returned non-JSON content (model=${model}): ${err instanceof Error ? err.message : String(err)}
+--- raw content ---
+${content.slice(0, 800)}`
+    );
+  }
+}
+var LlmRouteAssertionError = class extends CaptureIntegrityError {
+  constructor(message, reason, baseUrl) {
+    super(message);
+    this.reason = reason;
+    this.baseUrl = baseUrl;
+  }
+  reason;
+  baseUrl;
+};
+function assertLlmRoute(opts, req = {}) {
+  const baseUrlExplicit = opts.baseUrl !== void 0;
+  const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
+  if (req.requireExplicitBaseUrl && !baseUrlExplicit) {
+    throw new LlmRouteAssertionError(
+      `assertLlmRoute: requireExplicitBaseUrl set but opts.baseUrl is undefined; would fall back to ${DEFAULT_BASE_URL}.`,
+      "no_explicit_base_url",
+      baseUrl
+    );
+  }
+  if (req.blockedBaseUrls?.some((p) => matchUrl(baseUrl, p))) {
+    throw new LlmRouteAssertionError(
+      `assertLlmRoute: baseUrl ${baseUrl} matches a blocked pattern.`,
+      "base_url_blocked",
+      baseUrl
+    );
+  }
+  if (req.allowedBaseUrls && req.allowedBaseUrls.length > 0) {
+    const ok = req.allowedBaseUrls.some((p) => matchUrl(baseUrl, p));
+    if (!ok) {
+      throw new LlmRouteAssertionError(
+        `assertLlmRoute: baseUrl ${baseUrl} is not in the allowed list (${req.allowedBaseUrls.map(describePattern).join(", ")}).`,
+        "base_url_not_allowed",
+        baseUrl
+      );
+    }
+  }
+  if (req.requireAuth && !opts.apiKey && !opts.bearer && !opts.authHeader) {
+    throw new LlmRouteAssertionError(
+      `assertLlmRoute: requireAuth set but no apiKey, bearer, or authHeader was supplied.`,
+      "no_auth",
+      baseUrl
+    );
+  }
+  if (req.expectedProvider) {
+    const actual = opts.provider ?? providerFromBaseUrl(baseUrl);
+    if (actual !== req.expectedProvider) {
+      throw new LlmRouteAssertionError(
+        `assertLlmRoute: expected provider ${req.expectedProvider} but baseUrl ${baseUrl} resolves to ${actual}.`,
+        "wrong_provider",
+        baseUrl
+      );
+    }
+  }
+}
+function matchUrl(url, pattern) {
+  if (pattern instanceof RegExp) return pattern.test(url);
+  return url.toLowerCase().startsWith(pattern.toLowerCase());
+}
+function describePattern(p) {
+  return p instanceof RegExp ? p.source : p;
+}
+async function probeLlm(model, opts = {}) {
+  const start = Date.now();
+  try {
+    await callLlm(
+      {
+        model,
+        messages: [{ role: "user", content: "ping" }],
+        maxTokens: 64,
+        timeoutMs: opts.timeoutMs ?? 3e4
+      },
+      opts
+    );
+    return { ok: true, latencyMs: Date.now() - start, error: null };
+  } catch (err) {
+    return {
+      ok: false,
+      latencyMs: Date.now() - start,
+      error: err instanceof Error ? err.message : String(err)
+    };
+  }
+}
+var LlmClient = class {
+  maximumAttempts;
+  opts;
+  constructor(opts = {}) {
+    this.opts = opts;
+    this.maximumAttempts = resolveMaximumAttempts(opts.maxRetries);
+  }
+  call(req, per) {
+    const options = { ...this.opts, ...per };
+    return req.jsonSchema ? callLlmStructured(req, options) : callLlm(req, options);
+  }
+  callJson(req, per) {
+    return callLlmJson(req, { ...this.opts, ...per });
+  }
+};
+
+export {
+  maximumChargeForLlmRequest,
+  costReceiptFromLlm,
+  costReceiptFromLlmError,
+  LlmCallError,
+  LlmResponseError,
+  isTransientLlmError,
+  backoffMs,
+  stripFencedJson,
+  callLlm,
+  callLlmJson,
+  LlmRouteAssertionError,
+  assertLlmRoute,
+  probeLlm,
+  LlmClient
+};
+//# sourceMappingURL=chunk-NJC7U437.js.map