@tangle-network/agent-eval 0.12.0 → 0.13.0

package/dist/index.js

@@ -1,3 +1,12 @@
+import {
+  LlmCallError,
+  LlmClient,
+  callLlm,
+  callLlmJson,
+  probeLlm,
+  stripFencedJson
+} from "./chunk-ITN4YOZY.js";
+
 // src/client.ts
 var ProductClient = class {
   baseUrl;
@@ -1926,6 +1935,244 @@ function rand(bytes) {
   return Array.from(arr).map((b) => b.toString(16).padStart(2, "0")).join("");
 }
 
+// src/experiment-tracker-fs.ts
+var FileSystemExperimentStore = class {
+  dir;
+  maxBytes;
+  index;
+  loaded = false;
+  constructor(options) {
+    this.dir = options.dir;
+    this.maxBytes = options.maxBytes ?? 32 * 1024 * 1024;
+  }
+  async saveExperiment(exp) {
+    const idx = await this.load();
+    await idx.saveExperiment(exp);
+    await this.append("experiments", exp);
+  }
+  async getExperiment(id) {
+    const idx = await this.load();
+    return idx.getExperiment(id);
+  }
+  async listExperiments() {
+    const idx = await this.load();
+    return idx.listExperiments();
+  }
+  async saveRun(run) {
+    const idx = await this.load();
+    await idx.saveRun(run);
+    await this.append("runs", run);
+  }
+  async getRun(id) {
+    const idx = await this.load();
+    return idx.getRun(id);
+  }
+  async listRuns(experimentId) {
+    const idx = await this.load();
+    return idx.listRuns(experimentId);
+  }
+  async ensureDir() {
+    const fs = await import("fs/promises");
+    await fs.mkdir(this.dir, { recursive: true });
+  }
+  async append(name, record) {
+    await this.ensureDir();
+    const fs = await import("fs/promises");
+    const path = await import("path");
+    const active = path.join(this.dir, `${name}.ndjson`);
+    try {
+      const stat = await fs.stat(active);
+      if (stat.size >= this.maxBytes) {
+        const rolled = path.join(this.dir, `${name}.${Date.now()}.ndjson`);
+        await fs.rename(active, rolled);
+      }
+    } catch {
+    }
+    await fs.appendFile(active, JSON.stringify(record) + "\n", "utf8");
+  }
+  async load() {
+    if (this.loaded && this.index) return this.index;
+    const fs = await import("fs/promises");
+    const path = await import("path");
+    const store = new InMemoryExperimentStore();
+    try {
+      const entries = await fs.readdir(this.dir);
+      const sorted = entries.filter((f) => f.endsWith(".ndjson")).sort((a, b) => a.localeCompare(b));
+      for (const file of sorted) {
+        const full = path.join(this.dir, file);
+        const content = await fs.readFile(full, "utf8");
+        const base = file.split(".")[0];
+        for (const line of content.split("\n")) {
+          if (!line.trim()) continue;
+          let record;
+          try {
+            record = JSON.parse(line);
+          } catch {
+            continue;
+          }
+          if (base === "experiments") {
+            await store.saveExperiment(record);
+          } else if (base === "runs") {
+            await store.saveRun(record);
+          }
+        }
+      }
+    } catch {
+    }
+    this.index = store;
+    this.loaded = true;
+    return store;
+  }
+};
+
+// src/experiment-tracker-d1.ts
+var SCHEMA_VERSION = 1;
+var D1ExperimentStore = class {
+  db;
+  experimentsTable;
+  runsTable;
+  metaTable;
+  schemaReady = false;
+  constructor(options) {
+    this.db = options.db;
+    const prefix = options.tablePrefix ?? "agent_eval_";
+    this.experimentsTable = `${prefix}experiments`;
+    this.runsTable = `${prefix}runs`;
+    this.metaTable = `${prefix}meta`;
+  }
+  /**
+   * Idempotent schema setup. Safe to call before every operation; the second
+   * call short-circuits via `schemaReady`. Most consumers will call it once
+   * during Worker bootstrap.
+   */
+  async ensureSchema() {
+    if (this.schemaReady) return;
+    const ddl = `
+      CREATE TABLE IF NOT EXISTS ${this.experimentsTable} (
+        id TEXT PRIMARY KEY,
+        name TEXT NOT NULL,
+        created_at TEXT NOT NULL,
+        metadata_json TEXT
+      );
+      CREATE TABLE IF NOT EXISTS ${this.runsTable} (
+        id TEXT PRIMARY KEY,
+        experiment_id TEXT NOT NULL,
+        name TEXT,
+        status TEXT NOT NULL,
+        started_at TEXT NOT NULL,
+        completed_at TEXT,
+        config_json TEXT NOT NULL,
+        report_json TEXT,
+        error TEXT
+      );
+      CREATE INDEX IF NOT EXISTS idx_${this.runsTable}_experiment ON ${this.runsTable}(experiment_id);
+      CREATE INDEX IF NOT EXISTS idx_${this.runsTable}_started ON ${this.runsTable}(started_at);
+      CREATE TABLE IF NOT EXISTS ${this.metaTable} (
+        key TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      );
+      INSERT OR REPLACE INTO ${this.metaTable}(key, value) VALUES ('schema_version', '${SCHEMA_VERSION}');
+    `;
+    await this.db.exec(ddl.trim().replace(/\s+/g, " "));
+    this.schemaReady = true;
+  }
+  async saveExperiment(exp) {
+    await this.ensureSchema();
+    await this.db.prepare(
+      `INSERT INTO ${this.experimentsTable}(id, name, created_at, metadata_json)
+         VALUES (?1, ?2, ?3, ?4)
+         ON CONFLICT(id) DO UPDATE SET
+           name = excluded.name,
+           created_at = excluded.created_at,
+           metadata_json = excluded.metadata_json`
+    ).bind(exp.id, exp.name, exp.createdAt, exp.metadata ? JSON.stringify(exp.metadata) : null).run();
+  }
+  async getExperiment(id) {
+    await this.ensureSchema();
+    const row = await this.db.prepare(
+      `SELECT id, name, created_at, metadata_json
+         FROM ${this.experimentsTable}
+         WHERE id = ?1`
+    ).bind(id).first();
+    return row ? rowToExperiment(row) : null;
+  }
+  async listExperiments() {
+    await this.ensureSchema();
+    const { results } = await this.db.prepare(
+      `SELECT id, name, created_at, metadata_json
+         FROM ${this.experimentsTable}
+         ORDER BY created_at DESC`
+    ).all();
+    return results.map(rowToExperiment);
+  }
+  async saveRun(run) {
+    await this.ensureSchema();
+    await this.db.prepare(
+      `INSERT INTO ${this.runsTable}(id, experiment_id, name, status, started_at, completed_at, config_json, report_json, error)
+         VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
+         ON CONFLICT(id) DO UPDATE SET
+           experiment_id = excluded.experiment_id,
+           name = excluded.name,
+           status = excluded.status,
+           started_at = excluded.started_at,
+           completed_at = excluded.completed_at,
+           config_json = excluded.config_json,
+           report_json = excluded.report_json,
+           error = excluded.error`
+    ).bind(
+      run.id,
+      run.experimentId,
+      run.name ?? null,
+      run.status,
+      run.startedAt,
+      run.completedAt ?? null,
+      JSON.stringify(run.config),
+      run.report ? JSON.stringify(run.report) : null,
+      run.error ?? null
+    ).run();
+  }
+  async getRun(id) {
+    await this.ensureSchema();
+    const row = await this.db.prepare(
+      `SELECT id, experiment_id, name, status, started_at, completed_at, config_json, report_json, error
+         FROM ${this.runsTable}
+         WHERE id = ?1`
+    ).bind(id).first();
+    return row ? rowToRun(row) : null;
+  }
+  async listRuns(experimentId) {
+    await this.ensureSchema();
+    const { results } = await this.db.prepare(
+      `SELECT id, experiment_id, name, status, started_at, completed_at, config_json, report_json, error
+         FROM ${this.runsTable}
+         WHERE experiment_id = ?1
+         ORDER BY started_at DESC`
+    ).bind(experimentId).all();
+    return results.map(rowToRun);
+  }
+};
+function rowToExperiment(row) {
+  return {
+    id: row.id,
+    name: row.name,
+    createdAt: row.created_at,
+    ...row.metadata_json ? { metadata: JSON.parse(row.metadata_json) } : {}
+  };
+}
+function rowToRun(row) {
+  return {
+    id: row.id,
+    experimentId: row.experiment_id,
+    ...row.name ? { name: row.name } : {},
+    status: row.status,
+    startedAt: row.started_at,
+    ...row.completed_at ? { completedAt: row.completed_at } : {},
+    config: JSON.parse(row.config_json),
+    ...row.report_json ? { report: JSON.parse(row.report_json) } : {},
+    ...row.error ? { error: row.error } : {}
+  };
+}
+
 // src/power-analysis.ts
 function requiredSampleSize(opts) {
   const effect = opts.effect;
@@ -8058,212 +8305,6 @@ async function euAiActReport(ctx, signals) {
   };
 }
 
-// src/llm-client.ts
-var LlmCallError = class extends Error {
-  constructor(message, status, body, model) {
-    super(message);
-    this.status = status;
-    this.body = body;
-    this.model = model;
-    this.name = "LlmCallError";
-  }
-  status;
-  body;
-  model;
-};
-var DEFAULT_BASE_URL = "https://router.tangle.tools/v1";
-var DEFAULT_TIMEOUT_MS = 6e4;
-var DEFAULT_MAX_RETRIES = 3;
-var RETRYABLE_STATUS = /* @__PURE__ */ new Set([429, 502, 503, 504]);
-function isRetryableError(err) {
-  if (err instanceof LlmCallError) return RETRYABLE_STATUS.has(err.status);
-  if (err instanceof Error) {
-    return err.name === "AbortError" || err.name === "TimeoutError" || /fetch failed|ECONNRESET|ETIMEDOUT|EAI_AGAIN/i.test(err.message);
-  }
-  return false;
-}
-function parseRetryAfter(headers) {
-  const h = headers.get("retry-after");
-  if (!h) return null;
-  const asNumber = Number(h);
-  if (Number.isFinite(asNumber) && asNumber > 0) return asNumber * 1e3;
-  const asDate = Date.parse(h);
-  if (Number.isFinite(asDate)) return Math.max(0, asDate - Date.now());
-  return null;
-}
-function backoffMs(attempt) {
-  return Math.min(500 * Math.pow(2, attempt), 16e3);
-}
-function buildHeaders(opts) {
-  const headers = {
-    "Content-Type": "application/json",
-    Accept: "application/json"
-  };
-  if (opts.authHeader) {
-    headers[opts.authHeader.name] = opts.authHeader.value;
-  } else if (opts.bearer || opts.apiKey) {
-    headers.Authorization = `Bearer ${opts.bearer ?? opts.apiKey}`;
-  }
-  return headers;
-}
-function isSchemaRejection(status, body) {
-  if (status !== 400) return false;
-  const lower = body.toLowerCase();
-  return lower.includes("response_format") || lower.includes("json_schema") || lower.includes("is unavailable") || lower.includes("not supported");
-}
-function buildBody(req, forceJsonObject) {
-  const body = {
-    model: req.model,
-    messages: req.messages,
-    temperature: req.temperature ?? 0
-  };
-  if (req.maxTokens != null) body.max_tokens = req.maxTokens;
-  if (req.jsonSchema && !forceJsonObject) {
-    body.response_format = {
-      type: "json_schema",
-      json_schema: { name: req.jsonSchema.name, schema: req.jsonSchema.schema, strict: true }
-    };
-  } else if (req.jsonMode || req.jsonSchema) {
-    body.response_format = { type: "json_object" };
-  }
-  return body;
-}
-async function sleep(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
-}
-function stripFencedJson(raw) {
-  const trimmed = raw.trim();
-  const m = trimmed.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?```\s*$/);
-  return m ? m[1].trim() : trimmed;
-}
-async function callLlm(req, opts = {}) {
-  const baseUrl = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
-  const url = `${baseUrl}/chat/completions`;
-  const timeoutMs = req.timeoutMs ?? opts.defaultTimeoutMs ?? DEFAULT_TIMEOUT_MS;
-  const maxRetries = opts.maxRetries ?? DEFAULT_MAX_RETRIES;
-  const fetchFn = opts.fetch ?? globalThis.fetch;
-  const headers = buildHeaders(opts);
-  let lastErr;
-  for (let attempt = 0; attempt < maxRetries; attempt++) {
-    const controller = new AbortController();
-    const timeoutHandle = setTimeout(() => controller.abort(), timeoutMs);
-    const started = Date.now();
-    try {
-      const res = await fetchFn(url, {
-        method: "POST",
-        headers,
-        body: JSON.stringify(buildBody(req, false)),
-        signal: controller.signal
-      });
-      clearTimeout(timeoutHandle);
-      if (!res.ok) {
-        const body = await res.text();
-        const err = new LlmCallError(
-          `LLM call ${res.status}: ${body.slice(0, 300)}`,
-          res.status,
-          body,
-          req.model
-        );
-        if (RETRYABLE_STATUS.has(res.status) && attempt < maxRetries - 1) {
-          lastErr = err;
-          const retryAfter = parseRetryAfter(res.headers);
-          await sleep(retryAfter ?? backoffMs(attempt));
-          continue;
-        }
-        throw err;
-      }
-      const json = await res.json();
-      const choice = json.choices?.[0];
-      const usageRaw = json.usage ?? {};
-      const costFromProxy = json._response_cost ?? json.cost_usd;
-      return {
-        content: choice?.message?.content ?? "",
-        usage: {
-          promptTokens: Number(usageRaw.prompt_tokens ?? 0),
-          completionTokens: Number(usageRaw.completion_tokens ?? 0),
-          totalTokens: Number(usageRaw.total_tokens ?? 0),
-          cachedPromptTokens: usageRaw.prompt_tokens_details && typeof usageRaw.prompt_tokens_details === "object" ? Number(
-            usageRaw.prompt_tokens_details.cached_tokens ?? 0
-          ) : void 0
-        },
-        costUsd: typeof costFromProxy === "number" ? costFromProxy : null,
-        model: json.model ?? req.model,
-        durationMs: Date.now() - started,
-        raw: json
-      };
-    } catch (err) {
-      clearTimeout(timeoutHandle);
-      lastErr = err;
-      if (attempt < maxRetries - 1 && isRetryableError(err)) {
-        await sleep(backoffMs(attempt));
-        continue;
-      }
-      throw err;
-    }
-  }
-  throw lastErr instanceof Error ? lastErr : new Error(String(lastErr));
-}
-async function callLlmJson(req, opts = {}) {
-  try {
-    const result = await callLlm({ ...req, jsonMode: req.jsonMode ?? !req.jsonSchema }, opts);
-    const value = parseJsonSafely(result.content, result.model);
-    return { value, result };
-  } catch (err) {
-    if (err instanceof LlmCallError && isSchemaRejection(err.status, err.body) && req.jsonSchema) {
-      const degradedReq = { ...req, jsonMode: true, jsonSchema: void 0 };
-      const result = await callLlm(degradedReq, opts);
-      const value = parseJsonSafely(result.content, result.model);
-      return { value, result };
-    }
-    throw err;
-  }
-}
-function parseJsonSafely(content, model) {
-  const stripped = stripFencedJson(content);
-  try {
-    return JSON.parse(stripped);
-  } catch (err) {
-    throw new Error(
-      `LLM returned non-JSON content (model=${model}): ${err instanceof Error ? err.message : String(err)}
---- raw content ---
-${content.slice(0, 800)}`
-    );
-  }
-}
-async function probeLlm(model, opts = {}) {
-  const start = Date.now();
-  try {
-    await callLlm(
-      {
-        model,
-        messages: [{ role: "user", content: "ping" }],
-        maxTokens: 64,
-        timeoutMs: opts.timeoutMs ?? 3e4
-      },
-      opts
-    );
-    return { ok: true, latencyMs: Date.now() - start, error: null };
-  } catch (err) {
-    return {
-      ok: false,
-      latencyMs: Date.now() - start,
-      error: err instanceof Error ? err.message : String(err)
-    };
-  }
-}
-var LlmClient = class {
-  constructor(opts = {}) {
-    this.opts = opts;
-  }
-  opts;
-  call(req, per) {
-    return callLlm(req, { ...this.opts, ...per });
-  }
-  callJson(req, per) {
-    return callLlmJson(req, { ...this.opts, ...per });
-  }
-};
-
 // src/multi-layer-verifier.ts
 function gradeSemanticStatus(input) {
   if (!input.available) return "error";
@@ -10594,6 +10635,7 @@ export {
   BuilderSession,
   ConvergenceTracker,
   CostTracker,
+  D1ExperimentStore,
   DEFAULT_AGENT_SLOS,
   DEFAULT_COMPLEXITY_WEIGHTS,
   DEFAULT_RULES as DEFAULT_FAILURE_RULES,
@@ -10611,6 +10653,7 @@ export {
   ERROR_COUNT_PATTERNS,
   ExperimentTracker,
   FAILURE_CLASSES,
+  FileSystemExperimentStore,
   FileSystemOutcomeStore,
   FileSystemTraceStore,
   HoldoutAuditor,