@tangle-network/agent-eval 0.120.0 → 0.120.2

package/dist/chunk-NJC7U437.js

@@ -1,626 +0,0 @@
-import {
-  defaultProviderRedactor,
-  providerFromBaseUrl
-} from "./chunk-PC4UYEBM.js";
-import {
-  AgentEvalError,
-  CaptureIntegrityError
-} from "./chunk-ONWEPEDO.js";
-
-// src/llm-client.ts
-function maximumChargeForLlmRequest(request, options = {}) {
-  if (request.maxTokens === void 0) return void 0;
-  if (!Number.isInteger(request.maxTokens) || request.maxTokens <= 0) {
-    throw new RangeError(`maximumChargeForLlmRequest: maxTokens must be a positive integer`);
-  }
-  if (request.messages.some(
-    (message) => Array.isArray(message.content) && message.content.some((part) => part.type === "image_url")
-  )) {
-    return void 0;
-  }
-  const attempts = resolveMaximumAttempts(options.maxRetries);
-  const requestBytes = new TextEncoder().encode(
-    JSON.stringify(buildBody(request, false))
-  ).byteLength;
-  const batches = request.jsonSchema ? 2 : 1;
-  return {
-    model: request.model,
-    inputTokens: requestBytes * attempts * batches,
-    outputTokens: request.maxTokens * attempts * batches
-  };
-}
-function costReceiptFromLlm(result) {
-  const cachedTokens = result.usage.cachedPromptTokens ?? 0;
-  return {
-    model: result.model,
-    inputTokens: Math.max(0, result.usage.promptTokens - cachedTokens),
-    outputTokens: result.usage.completionTokens,
-    cachedTokens: cachedTokens > 0 ? cachedTokens : void 0,
-    actualCostUsd: result.costUsd ?? void 0,
-    usageUnknown: result.usage.captured === false
-  };
-}
-function costReceiptFromLlmError(error) {
-  return error instanceof LlmResponseError ? costReceiptFromLlm(error.result) : void 0;
-}
-var LlmCallError = class extends AgentEvalError {
-  constructor(message, status, body, model) {
-    super("judge", message);
-    this.status = status;
-    this.body = body;
-    this.model = model;
-  }
-  status;
-  body;
-  model;
-};
-var LlmResponseError = class extends AgentEvalError {
-  constructor(message, result, options) {
-    super("judge", message, options);
-    this.result = result;
-  }
-  result;
-};
-var DEFAULT_BASE_URL = "https://router.tangle.tools/v1";
-var DEFAULT_TIMEOUT_MS = Number(process.env.TANGLE_LLM_TIMEOUT_MS) || 3e5;
-var DEFAULT_MAX_RETRIES = Number(process.env.TANGLE_LLM_MAX_RETRIES) || 3;
-function resolveMaximumAttempts(configured) {
-  const attempts = configured ?? DEFAULT_MAX_RETRIES;
-  if (!Number.isInteger(attempts) || attempts <= 0) {
-    throw new RangeError("LLM maximum attempts must be a positive integer");
-  }
-  return attempts;
-}
-function providerTokenCount(value) {
-  return typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : void 0;
-}
-var RETRYABLE_STATUS = /* @__PURE__ */ new Set([429, 502, 503, 504]);
-var TRANSIENT_ERROR_PATTERNS = [
-  /AbortError/i,
-  /TimeoutError/i,
-  /this operation was aborted/i,
-  /fetch failed/i,
-  /ECONNRESET/i,
-  /ETIMEDOUT/i,
-  /EAI_AGAIN/i,
-  /socket hang up/i,
-  /stream.*ended.*unexpectedly/i,
-  /terminated/i,
-  /other side closed/i,
-  /NGHTTP2/i,
-  /UND_ERR/i
-];
-function isTransientLlmError(err) {
-  return classifyTransient(err, 0);
-}
-function classifyTransient(err, depth) {
-  if (err instanceof LlmCallError) return RETRYABLE_STATUS.has(err.status);
-  if (!(err instanceof Error)) return false;
-  const status = err.status;
-  if (typeof status === "number" && RETRYABLE_STATUS.has(status)) return true;
-  const code = err.code;
-  const haystack = `${err.name}
-${err.message}
-${typeof code === "string" ? code : ""}`;
-  if (TRANSIENT_ERROR_PATTERNS.some((p) => p.test(haystack))) return true;
-  const cause = err.cause;
-  if (depth < 4 && cause instanceof Error && cause !== err) {
-    return classifyTransient(cause, depth + 1);
-  }
-  return false;
-}
-function parseRetryAfter(headers) {
-  const h = headers.get("retry-after");
-  if (!h) return null;
-  const asNumber = Number(h);
-  if (Number.isFinite(asNumber) && asNumber > 0) return asNumber * 1e3;
-  const asDate = Date.parse(h);
-  if (Number.isFinite(asDate)) return Math.max(0, asDate - Date.now());
-  return null;
-}
-function backoffMs(attempt) {
-  return Math.min(500 * 2 ** attempt, 16e3);
-}
-function buildHeaders(opts) {
-  const headers = {
-    "Content-Type": "application/json",
-    Accept: "application/json"
-  };
-  if (opts.authHeader) {
-    headers[opts.authHeader.name] = opts.authHeader.value;
-  } else if (opts.bearer || opts.apiKey) {
-    headers.Authorization = `Bearer ${opts.bearer ?? opts.apiKey}`;
-  }
-  if (opts.idempotencyKey) headers["Idempotency-Key"] = opts.idempotencyKey;
-  return headers;
-}
-function isSchemaRejection(status, body) {
-  if (status !== 400) return false;
-  const lower = body.toLowerCase();
-  return lower.includes("response_format") || lower.includes("json_schema") || lower.includes("is unavailable") || lower.includes("not supported");
-}
-function buildBody(req, forceJsonObject) {
-  const body = {
-    model: req.model,
-    messages: req.messages,
-    temperature: req.temperature ?? 0
-  };
-  if (req.maxTokens != null) {
-    if (usesMaxCompletionTokens(req.model)) body.max_completion_tokens = req.maxTokens;
-    else body.max_tokens = req.maxTokens;
-  }
-  if (req.jsonSchema && !forceJsonObject) {
-    body.response_format = {
-      type: "json_schema",
-      json_schema: { name: req.jsonSchema.name, schema: req.jsonSchema.schema, strict: true }
-    };
-  } else if (req.jsonMode || req.jsonSchema) {
-    body.response_format = { type: "json_object" };
-  }
-  return body;
-}
-function usesMaxCompletionTokens(model) {
-  return /^gpt-5(?:[.-]|$)/i.test(model);
-}
-async function sleep(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-function linkSignals(timeoutController, caller) {
-  if (!caller) return timeoutController.signal;
-  if (typeof AbortSignal.any === "function") {
-    return AbortSignal.any([timeoutController.signal, caller]);
-  }
-  if (caller.aborted) {
-    timeoutController.abort();
-  } else {
-    caller.addEventListener("abort", () => timeoutController.abort(), { once: true });
-  }
-  return timeoutController.signal;
-}
-function deadlineExceeded(start, deadlineMs) {
-  return deadlineMs != null && Date.now() - start >= deadlineMs;
-}
-function stripFencedJson(raw) {
-  const trimmed = raw.trim();
-  const m = trimmed.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```\s*$/);
-  return m ? m[1].trim() : trimmed;
-}
-function extractJsonPayload(raw) {
-  const stripped = stripFencedJson(raw);
-  try {
-    JSON.parse(stripped);
-    return stripped;
-  } catch {
-    if (stripped.startsWith("{") || stripped.startsWith("[")) return stripped;
-  }
-  const starts = [...stripped.matchAll(/[[{]/g)].map((match) => match.index).filter((index) => index != null);
-  for (const start of starts) {
-    const candidate = extractBalancedJson(stripped, start);
-    if (!candidate) continue;
-    try {
-      JSON.parse(candidate);
-      return candidate;
-    } catch {
-    }
-  }
-  return stripped;
-}
-function extractBalancedJson(input, start) {
-  const opener = input[start];
-  const closer = opener === "{" ? "}" : opener === "[" ? "]" : null;
-  if (!closer) return null;
-  const stack = [closer];
-  let isInString = false;
-  let isEscaped = false;
-  for (let i = start + 1; i < input.length; i++) {
-    const char = input[i];
-    if (isEscaped) {
-      isEscaped = false;
-      continue;
-    }
-    if (char === "\\") {
-      isEscaped = isInString;
-      continue;
-    }
-    if (char === '"') {
-      isInString = !isInString;
-      continue;
-    }
-    if (isInString) continue;
-    if (char === "{") stack.push("}");
-    else if (char === "[") stack.push("]");
-    else if (char === stack[stack.length - 1]) {
-      stack.pop();
-      if (stack.length === 0) return input.slice(start, i + 1);
-    }
-  }
-  return null;
-}
-async function callLlm(req, opts = {}) {
-  const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
-  const url = `${baseUrl}/chat/completions`;
-  const endpoint = "/chat/completions";
-  const timeoutMs = req.timeoutMs ?? opts.defaultTimeoutMs ?? DEFAULT_TIMEOUT_MS;
-  const maximumAttempts = resolveMaximumAttempts(opts.maxRetries);
-  const fetchFn = opts.fetch ?? globalThis.fetch;
-  const headers = buildHeaders(opts);
-  const provider = opts.provider ?? providerFromBaseUrl(baseUrl);
-  const sink = opts.rawSink;
-  const redactor = opts.redactor ?? defaultProviderRedactor;
-  const traceContext = opts.traceContext;
-  const callerSignal = opts.signal;
-  const deadlineMs = opts.deadlineMs;
-  const deadlineStart = Date.now();
-  let lastErr;
-  for (let attempt = 0; attempt < maximumAttempts; attempt++) {
-    if (callerSignal?.aborted) {
-      throw new DOMException("callLlm aborted by caller signal", "AbortError");
-    }
-    if (attempt > 0 && deadlineExceeded(deadlineStart, deadlineMs)) {
-      throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
-    }
-    const controller = new AbortController();
-    const attemptSignal = linkSignals(controller, callerSignal);
-    const timeoutHandle = setTimeout(() => controller.abort(), timeoutMs);
-    const started = Date.now();
-    const requestBody = buildBody(req, false);
-    let attemptErrorRecorded = false;
-    if (sink) {
-      await recordRaw(sink, redactor, {
-        eventId: cryptoEventId(),
-        runId: traceContext?.runId,
-        spanId: traceContext?.spanId,
-        provider,
-        model: req.model,
-        endpoint,
-        baseUrl,
-        attemptIndex: attempt,
-        direction: "request",
-        timestamp: started,
-        requestHeaders: headers,
-        requestBody,
-        redactedFields: []
-      });
-    }
-    try {
-      const res = await fetchFn(url, {
-        method: "POST",
-        headers,
-        body: JSON.stringify(requestBody),
-        signal: attemptSignal
-      });
-      clearTimeout(timeoutHandle);
-      const responseHeaders = sink ? headersToObject(res.headers) : void 0;
-      if (!res.ok) {
-        const body = await res.text();
-        if (sink) {
-          await recordRaw(sink, redactor, {
-            eventId: cryptoEventId(),
-            runId: traceContext?.runId,
-            spanId: traceContext?.spanId,
-            provider,
-            model: req.model,
-            endpoint,
-            baseUrl,
-            attemptIndex: attempt,
-            direction: "error",
-            timestamp: Date.now(),
-            durationMs: Date.now() - started,
-            statusCode: res.status,
-            responseHeaders,
-            responseBody: body,
-            errorMessage: `HTTP ${res.status}`,
-            redactedFields: []
-          });
-          attemptErrorRecorded = true;
-        }
-        const err = new LlmCallError(
-          `LLM call ${res.status}: ${body.slice(0, 300)}`,
-          res.status,
-          body,
-          req.model
-        );
-        if (RETRYABLE_STATUS.has(res.status) && attempt < maximumAttempts - 1 && !deadlineExceeded(deadlineStart, deadlineMs)) {
-          lastErr = err;
-          const retryAfter = parseRetryAfter(res.headers);
-          await sleep(retryAfter ?? backoffMs(attempt));
-          continue;
-        }
-        throw err;
-      }
-      const text = await res.text();
-      let json;
-      try {
-        json = JSON.parse(text);
-      } catch (parseErr) {
-        if (sink) {
-          await recordRaw(sink, redactor, {
-            eventId: cryptoEventId(),
-            runId: traceContext?.runId,
-            spanId: traceContext?.spanId,
-            provider,
-            model: req.model,
-            endpoint,
-            baseUrl,
-            attemptIndex: attempt,
-            direction: "error",
-            timestamp: Date.now(),
-            durationMs: Date.now() - started,
-            statusCode: res.status,
-            responseHeaders,
-            responseBody: text,
-            errorMessage: `non-JSON response: ${parseErr instanceof Error ? parseErr.message : String(parseErr)}`,
-            redactedFields: []
-          });
-          attemptErrorRecorded = true;
-        }
-        throw parseErr;
-      }
-      if (sink) {
-        await recordRaw(sink, redactor, {
-          eventId: cryptoEventId(),
-          runId: traceContext?.runId,
-          spanId: traceContext?.spanId,
-          provider,
-          model: req.model,
-          endpoint,
-          baseUrl,
-          attemptIndex: attempt,
-          direction: "response",
-          timestamp: Date.now(),
-          durationMs: Date.now() - started,
-          statusCode: res.status,
-          responseHeaders,
-          responseBody: json,
-          redactedFields: []
-        });
-      }
-      const choice = json.choices?.[0];
-      const usageRaw = json.usage && typeof json.usage === "object" && !Array.isArray(json.usage) ? json.usage : void 0;
-      const promptTokens = providerTokenCount(usageRaw?.prompt_tokens);
-      const completionTokens = providerTokenCount(usageRaw?.completion_tokens);
-      const totalTokens = providerTokenCount(usageRaw?.total_tokens);
-      const cachedRaw = usageRaw?.prompt_tokens_details && typeof usageRaw.prompt_tokens_details === "object" && !Array.isArray(usageRaw.prompt_tokens_details) ? usageRaw.prompt_tokens_details.cached_tokens : void 0;
-      const cachedPromptTokens = cachedRaw === void 0 ? void 0 : providerTokenCount(cachedRaw);
-      const usageCaptured = promptTokens !== void 0 && completionTokens !== void 0 && (cachedRaw === void 0 || cachedPromptTokens !== void 0 && cachedPromptTokens <= promptTokens) && (totalTokens === void 0 || totalTokens === promptTokens + completionTokens);
-      const costFromProxy = json._response_cost ?? json.cost_usd;
-      const content = choice?.message?.content ?? "";
-      return {
-        content,
-        finishReason: choice?.finish_reason ?? null,
-        contentEmpty: content.trim().length === 0,
-        usage: {
-          promptTokens: promptTokens ?? 0,
-          completionTokens: completionTokens ?? 0,
-          totalTokens: totalTokens ?? (promptTokens ?? 0) + (completionTokens ?? 0),
-          captured: usageCaptured,
-          cachedPromptTokens
-        },
-        costUsd: typeof costFromProxy === "number" ? costFromProxy : null,
-        model: json.model ?? req.model,
-        durationMs: Date.now() - started,
-        raw: json
-      };
-    } catch (err) {
-      clearTimeout(timeoutHandle);
-      lastErr = err;
-      if (callerSignal?.aborted) {
-        if (sink && !attemptErrorRecorded) {
-          await recordRaw(sink, redactor, {
-            eventId: cryptoEventId(),
-            runId: traceContext?.runId,
-            spanId: traceContext?.spanId,
-            provider,
-            model: req.model,
-            endpoint,
-            baseUrl,
-            attemptIndex: attempt,
-            direction: "error",
-            timestamp: Date.now(),
-            durationMs: Date.now() - started,
-            errorMessage: err instanceof Error ? err.message : String(err),
-            redactedFields: []
-          });
-        }
-        throw err;
-      }
-      if (sink && !attemptErrorRecorded) {
-        await recordRaw(sink, redactor, {
-          eventId: cryptoEventId(),
-          runId: traceContext?.runId,
-          spanId: traceContext?.spanId,
-          provider,
-          model: req.model,
-          endpoint,
-          baseUrl,
-          attemptIndex: attempt,
-          direction: "error",
-          timestamp: Date.now(),
-          durationMs: Date.now() - started,
-          errorMessage: err instanceof Error ? err.message : String(err),
-          redactedFields: []
-        });
-      }
-      if (attempt < maximumAttempts - 1 && isTransientLlmError(err) && !deadlineExceeded(deadlineStart, deadlineMs)) {
-        await sleep(backoffMs(attempt));
-        continue;
-      }
-      throw err;
-    }
-  }
-  throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
-}
-async function recordRaw(sink, redactor, event) {
-  try {
-    await sink.record(redactor(event));
-  } catch {
-  }
-}
-function headersToObject(h) {
-  const out = {};
-  h.forEach((value, key) => {
-    out[key] = value;
-  });
-  return out;
-}
-function cryptoEventId() {
-  if (typeof globalThis.crypto?.randomUUID === "function") return globalThis.crypto.randomUUID();
-  return `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
-}
-async function callLlmJson(req, opts = {}) {
-  const result = await callLlmStructured(req, opts);
-  const value = parseJsonResult(result);
-  return { value, result };
-}
-async function callLlmStructured(req, opts = {}) {
-  try {
-    return await callLlm({ ...req, jsonMode: req.jsonMode ?? !req.jsonSchema }, opts);
-  } catch (err) {
-    if (err instanceof LlmCallError && isSchemaRejection(err.status, err.body) && req.jsonSchema) {
-      const degradedReq = { ...req, jsonMode: true, jsonSchema: void 0 };
-      return await callLlm(degradedReq, opts);
-    }
-    throw err;
-  }
-}
-function parseJsonResult(result) {
-  try {
-    if (result.finishReason === "length") {
-      throw new Error(
-        `LLM returned truncated JSON content (model=${result.model}, finishReason=length)`
-      );
-    }
-    return parseJsonSafely(result.content, result.model);
-  } catch (error) {
-    if (error instanceof LlmResponseError) throw error;
-    const cause = error instanceof Error ? error : new Error(String(error));
-    throw new LlmResponseError(cause.message, result, { cause });
-  }
-}
-function parseJsonSafely(content, model) {
-  const stripped = extractJsonPayload(content);
-  try {
-    return JSON.parse(stripped);
-  } catch (err) {
-    throw new Error(
-      `LLM returned non-JSON content (model=${model}): ${err instanceof Error ? err.message : String(err)}
---- raw content ---
-${content.slice(0, 800)}`
-    );
-  }
-}
-var LlmRouteAssertionError = class extends CaptureIntegrityError {
-  constructor(message, reason, baseUrl) {
-    super(message);
-    this.reason = reason;
-    this.baseUrl = baseUrl;
-  }
-  reason;
-  baseUrl;
-};
-function assertLlmRoute(opts, req = {}) {
-  const baseUrlExplicit = opts.baseUrl !== void 0;
-  const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
-  if (req.requireExplicitBaseUrl && !baseUrlExplicit) {
-    throw new LlmRouteAssertionError(
-      `assertLlmRoute: requireExplicitBaseUrl set but opts.baseUrl is undefined; would fall back to ${DEFAULT_BASE_URL}.`,
-      "no_explicit_base_url",
-      baseUrl
-    );
-  }
-  if (req.blockedBaseUrls?.some((p) => matchUrl(baseUrl, p))) {
-    throw new LlmRouteAssertionError(
-      `assertLlmRoute: baseUrl ${baseUrl} matches a blocked pattern.`,
-      "base_url_blocked",
-      baseUrl
-    );
-  }
-  if (req.allowedBaseUrls && req.allowedBaseUrls.length > 0) {
-    const ok = req.allowedBaseUrls.some((p) => matchUrl(baseUrl, p));
-    if (!ok) {
-      throw new LlmRouteAssertionError(
-        `assertLlmRoute: baseUrl ${baseUrl} is not in the allowed list (${req.allowedBaseUrls.map(describePattern).join(", ")}).`,
-        "base_url_not_allowed",
-        baseUrl
-      );
-    }
-  }
-  if (req.requireAuth && !opts.apiKey && !opts.bearer && !opts.authHeader) {
-    throw new LlmRouteAssertionError(
-      `assertLlmRoute: requireAuth set but no apiKey, bearer, or authHeader was supplied.`,
-      "no_auth",
-      baseUrl
-    );
-  }
-  if (req.expectedProvider) {
-    const actual = opts.provider ?? providerFromBaseUrl(baseUrl);
-    if (actual !== req.expectedProvider) {
-      throw new LlmRouteAssertionError(
-        `assertLlmRoute: expected provider ${req.expectedProvider} but baseUrl ${baseUrl} resolves to ${actual}.`,
-        "wrong_provider",
-        baseUrl
-      );
-    }
-  }
-}
-function matchUrl(url, pattern) {
-  if (pattern instanceof RegExp) return pattern.test(url);
-  return url.toLowerCase().startsWith(pattern.toLowerCase());
-}
-function describePattern(p) {
-  return p instanceof RegExp ? p.source : p;
-}
-async function probeLlm(model, opts = {}) {
-  const start = Date.now();
-  try {
-    await callLlm(
-      {
-        model,
-        messages: [{ role: "user", content: "ping" }],
-        maxTokens: 64,
-        timeoutMs: opts.timeoutMs ?? 3e4
-      },
-      opts
-    );
-    return { ok: true, latencyMs: Date.now() - start, error: null };
-  } catch (err) {
-    return {
-      ok: false,
-      latencyMs: Date.now() - start,
-      error: err instanceof Error ? err.message : String(err)
-    };
-  }
-}
-var LlmClient = class {
-  maximumAttempts;
-  opts;
-  constructor(opts = {}) {
-    this.opts = opts;
-    this.maximumAttempts = resolveMaximumAttempts(opts.maxRetries);
-  }
-  call(req, per) {
-    const options = { ...this.opts, ...per };
-    return req.jsonSchema ? callLlmStructured(req, options) : callLlm(req, options);
-  }
-  callJson(req, per) {
-    return callLlmJson(req, { ...this.opts, ...per });
-  }
-};
-
-export {
-  maximumChargeForLlmRequest,
-  costReceiptFromLlm,
-  costReceiptFromLlmError,
-  LlmCallError,
-  LlmResponseError,
-  isTransientLlmError,
-  backoffMs,
-  stripFencedJson,
-  callLlm,
-  callLlmJson,
-  LlmRouteAssertionError,
-  assertLlmRoute,
-  probeLlm,
-  LlmClient
-};
-//# sourceMappingURL=chunk-NJC7U437.js.map