@tangle-network/agent-eval 0.23.1 → 0.24.0

package/dist/{integrity-Cr5YodSY.d.ts → integrity-DK2EBVZC.d.ts}

@@ -1,4 +1,5 @@
-import { T as TraceStore } from './store-u47QaJ9G.js';
+import { C as CaptureIntegrityError } from './errors-BZ9sTdz7.js';
+import { T as TraceStore } from './store-Db2Bv8Cf.js';
 
 /**
  * RawProviderSink — first-class persistence for the actual HTTP-level
@@ -199,7 +200,7 @@ interface RunIntegrityReport {
     };
     issues: RunIntegrityIssue[];
 }
-declare class RunIntegrityError extends Error {
+declare class RunIntegrityError extends CaptureIntegrityError {
     readonly report: RunIntegrityReport;
     constructor(report: RunIntegrityReport);
 }
@@ -207,4 +208,4 @@ declare function assertRunCaptured(store: TraceStore, runId: string, expectation
 /** Strict mode: throws `RunIntegrityError` when the report isn't ok. */
 declare function throwIfRunIncomplete(report: RunIntegrityReport): void;
 
-export { FileSystemRawProviderSink as F, InMemoryRawProviderSink as I, NoopRawProviderSink as N, type ProviderRedactor as P, type RawProviderSink as R, type RunIntegrityExpectations as a, type RunIntegrityReport as b, type FileSystemRawProviderSinkOptions as c, type InMemoryRawProviderSinkOptions as d, type RawProviderDirection as e, type RawProviderEvent as f, type RawProviderSinkFilter as g, RunIntegrityError as h, type RunIntegrityIssue as i, type RunIntegrityIssueCode as j, assertRunCaptured as k, defaultProviderRedactor as l, providerFromBaseUrl as p, throwIfRunIncomplete as t };
+export { FileSystemRawProviderSink as F, InMemoryRawProviderSink as I, NoopRawProviderSink as N, type ProviderRedactor as P, type RawProviderSink as R, type RunIntegrityExpectations as a, type RunIntegrityReport as b, type RawProviderEvent as c, type FileSystemRawProviderSinkOptions as d, type InMemoryRawProviderSinkOptions as e, type RawProviderDirection as f, type RawProviderSinkFilter as g, RunIntegrityError as h, type RunIntegrityIssue as i, type RunIntegrityIssueCode as j, assertRunCaptured as k, defaultProviderRedactor as l, providerFromBaseUrl as p, throwIfRunIncomplete as t };

package/dist/knowledge/index.d.ts

@@ -0,0 +1,102 @@
+import { j as ControlSeverity, C as ControlEvalResult } from '../control-runtime-BuJHoLg0.js';
+import { T as TraceEmitter } from '../emitter-DP_cSSiw.js';
+import '../store-Db2Bv8Cf.js';
+
+type KnowledgeRequirementCategory = 'user_specific' | 'company_specific' | 'domain_specific' | 'codebase_specific' | 'market_specific' | 'regulatory' | 'tool_api' | 'credential_or_secret' | 'runtime_environment' | 'preference' | 'historical_context';
+type KnowledgeAcquisitionMode = 'ask_user' | 'search_web' | 'query_connector' | 'inspect_repo' | 'run_command' | 'infer_low_confidence' | 'not_available';
+type KnowledgeImportance = 'blocking' | 'high' | 'medium' | 'low';
+type KnowledgeFreshness = 'static' | 'monthly' | 'weekly' | 'daily' | 'realtime';
+type KnowledgeSensitivity = 'public' | 'private' | 'secret';
+type KnowledgeFallbackPolicy = 'block' | 'ask' | 'continue_with_caveat' | 'use_default';
+interface KnowledgeRequirement {
+    id: string;
+    description: string;
+    requiredFor: string[];
+    category: KnowledgeRequirementCategory;
+    acquisitionMode: KnowledgeAcquisitionMode;
+    importance: KnowledgeImportance;
+    freshness: KnowledgeFreshness;
+    sensitivity: KnowledgeSensitivity;
+    confidenceNeeded: number;
+    currentConfidence: number;
+    evidenceIds: string[];
+    fallbackPolicy: KnowledgeFallbackPolicy;
+    /**
+     * ISO timestamp after which this requirement must be treated as stale.
+     * Stale requirements score as missing even when they still have evidence.
+     */
+    validUntil?: string;
+    /** ISO timestamp for the last source-grounding or human verification pass. */
+    lastVerifiedAt?: string;
+    metadata?: Record<string, unknown>;
+}
+interface KnowledgeBundle {
+    taskId: string;
+    requirements: KnowledgeRequirement[];
+    evidenceIds: string[];
+    claimIds: string[];
+    wikiPageIds: string[];
+    userAnswers: Record<string, string>;
+    missing: KnowledgeRequirement[];
+    readinessScore: number;
+    metadata?: Record<string, unknown>;
+}
+type KnowledgeRecommendedAction = 'run_agent' | 'ask_user' | 'collect_web_data' | 'query_connectors' | 'inspect_repo' | 'build_domain_wiki' | 'continue_with_caveat' | 'abort_or_rescope';
+interface KnowledgeReadinessReport {
+    taskId: string;
+    readinessScore: number;
+    blockingMissingRequirements: KnowledgeRequirement[];
+    nonBlockingGaps: KnowledgeRequirement[];
+    recommendedAction: KnowledgeRecommendedAction;
+    bundle: KnowledgeBundle;
+    severity: ControlSeverity;
+    reason: string;
+}
+interface UserQuestion {
+    id: string;
+    question: string;
+    reason: string;
+    requirementId: string;
+    importance: KnowledgeImportance;
+    answerType: 'free_text' | 'select_one' | 'multi_select' | 'file_upload' | 'credential' | 'url';
+    defaultIfSkipped?: string;
+    impactIfUnknown: string;
+    options?: string[];
+    metadata?: Record<string, unknown>;
+}
+interface DataAcquisitionPlan {
+    id: string;
+    requirementIds: string[];
+    mode: Exclude<KnowledgeAcquisitionMode, 'not_available' | 'infer_low_confidence'> | 'build_domain_wiki';
+    description: string;
+    priority: KnowledgeImportance;
+    expectedEvidenceIds?: string[];
+    questions?: UserQuestion[];
+    metadata?: Record<string, unknown>;
+}
+type KnowledgeResponsibleSurface = 'knowledge-requirements' | 'data-acquisition' | 'retrieval-policy' | 'user-question-policy';
+
+interface ScoreKnowledgeReadinessOptions {
+    taskId: string;
+    requirements: KnowledgeRequirement[];
+    evidenceIds?: string[];
+    claimIds?: string[];
+    wikiPageIds?: string[];
+    userAnswers?: Record<string, string>;
+    metadata?: Record<string, unknown>;
+    now?: Date;
+}
+declare function scoreKnowledgeReadiness(options: ScoreKnowledgeReadinessOptions): KnowledgeReadinessReport;
+declare function blockingKnowledgeEval(report: KnowledgeReadinessReport, options?: {
+    id?: string;
+    minimumScore?: number;
+    emitter?: TraceEmitter;
+}): ControlEvalResult;
+declare function knowledgeReadinessTracePayload(report: KnowledgeReadinessReport, options?: {
+    passed?: boolean;
+    minimumScore?: number;
+}): Record<string, unknown>;
+declare function userQuestionsForKnowledgeGaps(gaps: KnowledgeRequirement[]): UserQuestion[];
+declare function acquisitionPlansForKnowledgeGaps(gaps: KnowledgeRequirement[]): DataAcquisitionPlan[];
+
+export { type DataAcquisitionPlan, type KnowledgeAcquisitionMode, type KnowledgeBundle, type KnowledgeFallbackPolicy, type KnowledgeFreshness, type KnowledgeImportance, type KnowledgeReadinessReport, type KnowledgeRecommendedAction, type KnowledgeRequirement, type KnowledgeRequirementCategory, type KnowledgeResponsibleSurface, type KnowledgeSensitivity, type ScoreKnowledgeReadinessOptions, type UserQuestion, acquisitionPlansForKnowledgeGaps, blockingKnowledgeEval, knowledgeReadinessTracePayload, scoreKnowledgeReadiness, userQuestionsForKnowledgeGaps };

package/dist/knowledge/index.js

@@ -0,0 +1,18 @@
+import {
+  acquisitionPlansForKnowledgeGaps,
+  blockingKnowledgeEval,
+  knowledgeReadinessTracePayload,
+  scoreKnowledgeReadiness,
+  userQuestionsForKnowledgeGaps
+} from "../chunk-WWYCWKUM.js";
+import "../chunk-LSH4MMOZ.js";
+import "../chunk-TVVP3ZZQ.js";
+import "../chunk-PZ5AY32C.js";
+export {
+  acquisitionPlansForKnowledgeGaps,
+  blockingKnowledgeEval,
+  knowledgeReadinessTracePayload,
+  scoreKnowledgeReadiness,
+  userQuestionsForKnowledgeGaps
+};
+//# sourceMappingURL=index.js.map

package/dist/knowledge/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}

package/dist/meta-eval/index.d.ts

@@ -0,0 +1,99 @@
+import { R as Run, T as TraceStore } from '../store-Db2Bv8Cf.js';
+import { a as OutcomeFilter, O as OutcomeStore } from '../outcome-store-D6KWmYvj.js';
+export { D as DeploymentOutcome, F as FileSystemOutcomeStore, b as FileSystemOutcomeStoreOptions, I as InMemoryOutcomeStore } from '../outcome-store-D6KWmYvj.js';
+export { R as RubricOutcomePair, a as RubricPredictiveValidityInput, b as RubricPredictiveValidityReport, c as RubricRanking, r as rubricPredictiveValidity } from '../rubric-predictive-validity-C0uDYwG6.js';
+import '../run-record-CqzahIbx.js';
+import '../errors-BZ9sTdz7.js';
+
+/**
+ * Correlation study — "does our eval score predict real-world outcomes?"
+ *
+ * This is the load-bearing signal. Takes a TraceStore + OutcomeStore,
+ * joins on runId, computes Pearson + Spearman + bootstrap CI for every
+ * (evalMetric, outcomeMetric) pair the caller declares.
+ *
+ * Without this number the framework is ornamental. With it and r > 0.6
+ * the framework is a moat — no other agent-eval tool publishes one.
+ */
+
+interface EvalMetricSpec {
+    id: string;
+    /** Extract a scalar from a run (defaults cover score/pass/durationMs/costUsd/tokens). */
+    extract?: (run: Run, store: TraceStore) => Promise<number | null>;
+}
+interface OutcomePair {
+    evalMetric: string;
+    outcomeMetric: string;
+}
+interface CorrelationResult {
+    evalMetric: string;
+    outcomeMetric: string;
+    n: number;
+    pearson: number;
+    spearman: number;
+    /** 95% bootstrap CI for Pearson. */
+    pearsonCi95: {
+        lower: number;
+        upper: number;
+    };
+    /** Rough verdict: 'strong' ≥ 0.7, 'moderate' ≥ 0.4, else 'weak'. */
+    verdict: 'strong' | 'moderate' | 'weak';
+}
+interface CorrelationStudyResult {
+    pairs: CorrelationResult[];
+    joinedSamples: number;
+    skippedRuns: number;
+}
+interface CorrelationStudyOptions {
+    /** Only join outcomes captured within this window after run.startedAt. */
+    maxCaptureLagMs?: number;
+    /** Restrict to a subset of outcomes (cohort, region, source). */
+    outcomeFilter?: OutcomeFilter;
+    /** Which outcome per run to use when multiple exist. Default 'latest'. */
+    reduction?: 'latest' | 'mean' | 'max';
+    /** Bootstrap iterations for the CI. Default 500. */
+    bootstrapIterations?: number;
+}
+declare function correlationStudy(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetrics: EvalMetricSpec[], outcomeMetricNames: string[], options?: CorrelationStudyOptions): Promise<CorrelationStudyResult>;
+
+/**
+ * Calibration curve — binned "if eval says X, what does reality show?"
+ *
+ * Companion to correlationStudy. Raw correlation is a single number;
+ * the calibration curve shows *where* the eval is well-calibrated vs
+ * overconfident / underconfident. Buckets the eval metric, computes
+ * mean outcome per bucket, reports expected-calibration-error (ECE).
+ */
+
+interface CalibrationBin {
+    lower: number;
+    upper: number;
+    n: number;
+    evalMean: number;
+    outcomeMean: number;
+    /** |outcomeMean − evalMean|; contributes to ECE weighted by n/total. */
+    gap: number;
+}
+interface CalibrationReport {
+    evalMetric: string;
+    outcomeMetric: string;
+    n: number;
+    bins: CalibrationBin[];
+    /** Expected Calibration Error — Σ (n_i/N) × |outcomeMean_i − evalMean_i|. */
+    ece: number;
+    /** Max bin gap — upper bound on miscalibration. */
+    maxGap: number;
+}
+interface CalibrationOptions {
+    bins?: number;
+    /** Equal-width (fixed bin edges) or equal-frequency (quantile bins). */
+    binning?: 'equal-width' | 'equal-frequency';
+    /** Clip eval values to [lo, hi] before binning. */
+    range?: {
+        lo: number;
+        hi: number;
+    };
+}
+declare function calibrationCurve(traceStore: TraceStore, outcomeStore: OutcomeStore, evalMetric: EvalMetricSpec, outcomeMetric: string, options?: CalibrationOptions): Promise<CalibrationReport | null>;
+
+export { type CalibrationBin, type CalibrationOptions, type CalibrationReport, type CorrelationResult, type CorrelationStudyOptions, type CorrelationStudyResult, type EvalMetricSpec, OutcomeFilter, type OutcomePair, OutcomeStore, calibrationCurve, correlationStudy };

package/dist/meta-eval/index.js

@@ -0,0 +1,324 @@
+import {
+  rubricPredictiveValidity
+} from "../chunk-YRZ4M5GS.js";
+import {
+  aggregateLlm,
+  llmSpans
+} from "../chunk-47X6LRCE.js";
+import "../chunk-5BKGXME7.js";
+import "../chunk-PZ5AY32C.js";
+
+// src/meta-eval/calibration.ts
+async function calibrationCurve(traceStore, outcomeStore, evalMetric, outcomeMetric, options = {}) {
+  const runs = await traceStore.listRuns();
+  const outcomes = await outcomeStore.list();
+  const byRun = /* @__PURE__ */ new Map();
+  for (const o of outcomes) {
+    const arr = byRun.get(o.runId) ?? [];
+    arr.push(o);
+    byRun.set(o.runId, arr);
+  }
+  const extract = evalMetric.extract ?? defaultExtract(evalMetric.id);
+  const pairs = [];
+  for (const run of runs) {
+    const os = byRun.get(run.runId);
+    if (!os?.length) continue;
+    const x = await extract(run, traceStore);
+    if (x === null || !Number.isFinite(x)) continue;
+    const latest = [...os].sort((a, b) => b.capturedAt - a.capturedAt)[0];
+    const y = latest.metrics[outcomeMetric];
+    if (typeof y !== "number" || !Number.isFinite(y)) continue;
+    pairs.push({ x, y });
+  }
+  if (pairs.length < 2) return null;
+  const numBins = options.bins ?? 10;
+  const binning = options.binning ?? "equal-width";
+  const xs = pairs.map((p) => p.x);
+  const lo = options.range?.lo ?? Math.min(...xs);
+  const hi = options.range?.hi ?? Math.max(...xs);
+  const bins = [];
+  if (binning === "equal-frequency") {
+    const sorted = [...pairs].sort((a, b) => a.x - b.x);
+    const perBin = Math.max(1, Math.floor(sorted.length / numBins));
+    for (let i = 0; i < sorted.length; i += perBin) {
+      const chunk = sorted.slice(i, i + perBin);
+      if (chunk.length === 0) continue;
+      bins.push(toBin(chunk));
+    }
+  } else {
+    const width = (hi - lo) / numBins;
+    if (width === 0) return null;
+    for (let i = 0; i < numBins; i++) {
+      const binLo = lo + i * width;
+      const binHi = i === numBins - 1 ? hi + 1e-9 : lo + (i + 1) * width;
+      const chunk = pairs.filter((p) => p.x >= binLo && p.x < binHi);
+      if (chunk.length === 0) continue;
+      bins.push(toBin(chunk, binLo, binHi));
+    }
+  }
+  const total = bins.reduce((a, b) => a + b.n, 0);
+  const ece = bins.reduce((a, b) => a + b.n / total * b.gap, 0);
+  const maxGap = bins.reduce((a, b) => Math.max(a, b.gap), 0);
+  return { evalMetric: evalMetric.id, outcomeMetric, n: pairs.length, bins, ece, maxGap };
+}
+function toBin(chunk, lower, upper) {
+  const xs = chunk.map((c) => c.x);
+  const ys = chunk.map((c) => c.y);
+  const evalMean = mean(xs);
+  const outcomeMean = mean(ys);
+  return {
+    lower: lower ?? Math.min(...xs),
+    upper: upper ?? Math.max(...xs),
+    n: chunk.length,
+    evalMean,
+    outcomeMean,
+    gap: Math.abs(outcomeMean - evalMean)
+  };
+}
+function mean(xs) {
+  return xs.reduce((a, b) => a + b, 0) / xs.length;
+}
+function defaultExtract(metric) {
+  return async (run) => run.outcome?.score ?? (metric === "pass" ? run.outcome?.pass === true ? 1 : 0 : null);
+}
+
+// src/meta-eval/correlation-study.ts
+async function correlationStudy(traceStore, outcomeStore, evalMetrics, outcomeMetricNames, options = {}) {
+  const runs = await traceStore.listRuns();
+  const outcomes = await outcomeStore.list(options.outcomeFilter);
+  const outcomesByRun = /* @__PURE__ */ new Map();
+  for (const o of outcomes) {
+    const arr = outcomesByRun.get(o.runId) ?? [];
+    arr.push(o);
+    outcomesByRun.set(o.runId, arr);
+  }
+  const reduction = options.reduction ?? "latest";
+  const maxLag = options.maxCaptureLagMs ?? Infinity;
+  const pairs = [];
+  for (const em of evalMetrics) {
+    for (const om of outcomeMetricNames) {
+      pairs.push({ evalMetric: em.id, outcomeMetric: om, xs: [], ys: [] });
+    }
+  }
+  let joined = 0;
+  let skipped = 0;
+  for (const run of runs) {
+    const os = outcomesByRun.get(run.runId);
+    if (!os || os.length === 0) {
+      skipped++;
+      continue;
+    }
+    const eligible = os.filter((o) => o.capturedAt - run.startedAt <= maxLag);
+    if (eligible.length === 0) {
+      skipped++;
+      continue;
+    }
+    for (const em of evalMetrics) {
+      const extract = em.extract ?? defaultExtract2(em.id);
+      const x = await extract(run, traceStore);
+      if (x === null || !Number.isFinite(x)) continue;
+      for (const om of outcomeMetricNames) {
+        const values = eligible.map((o) => o.metrics[om]).filter((v) => typeof v === "number" && Number.isFinite(v));
+        if (values.length === 0) continue;
+        const y = reduce(values, reduction, eligible);
+        if (y === null) continue;
+        const pair = pairs.find((p) => p.evalMetric === em.id && p.outcomeMetric === om);
+        pair.xs.push(x);
+        pair.ys.push(y);
+      }
+    }
+    joined++;
+  }
+  const results = pairs.filter((p) => p.xs.length >= 3).map((p) => {
+    const pearson = pearsonR(p.xs, p.ys);
+    const spearman = pearsonR(ranks(p.xs), ranks(p.ys));
+    const pearsonCi95 = bootstrapPearsonCi(p.xs, p.ys, options.bootstrapIterations ?? 500);
+    const verdict = Math.abs(pearson) >= 0.7 ? "strong" : Math.abs(pearson) >= 0.4 ? "moderate" : "weak";
+    return {
+      evalMetric: p.evalMetric,
+      outcomeMetric: p.outcomeMetric,
+      n: p.xs.length,
+      pearson,
+      spearman,
+      pearsonCi95,
+      verdict
+    };
+  });
+  return { pairs: results, joinedSamples: joined, skippedRuns: skipped };
+}
+function reduce(values, kind, outcomes) {
+  if (values.length === 0) return null;
+  if (kind === "mean") return values.reduce((a, b) => a + b, 0) / values.length;
+  if (kind === "max") return Math.max(...values);
+  const latest = [...outcomes].sort((a, b) => b.capturedAt - a.capturedAt)[0];
+  if (!latest) return null;
+  const latestKey = Object.keys(latest.metrics)[0];
+  const v = latestKey !== void 0 ? latest.metrics[latestKey] : void 0;
+  const paired = outcomes.map((o) => {
+    const k = Object.keys(o.metrics)[0];
+    return {
+      at: o.capturedAt,
+      v: k !== void 0 ? values.find((x) => o.metrics[k] === x) : void 0
+    };
+  }).filter((p) => p.v !== void 0);
+  if (paired.length === 0) return v ?? null;
+  return paired.sort((a, b) => b.at - a.at)[0]?.v ?? null;
+}
+function pearsonR(a, b) {
+  if (a.length !== b.length || a.length < 2) return NaN;
+  const mA = a.reduce((s, v) => s + v, 0) / a.length;
+  const mB = b.reduce((s, v) => s + v, 0) / b.length;
+  let num = 0, dA = 0, dB = 0;
+  for (let i = 0; i < a.length; i++) {
+    const da = a[i] - mA, db = b[i] - mB;
+    num += da * db;
+    dA += da * da;
+    dB += db * db;
+  }
+  if (dA === 0 || dB === 0) return dA === 0 && dB === 0 ? 1 : 0;
+  return num / Math.sqrt(dA * dB);
+}
+function ranks(xs) {
+  const indexed = xs.map((v, i) => ({ v, i })).sort((x, y) => x.v - y.v);
+  const r = new Array(xs.length);
+  for (let i = 0; i < indexed.length; i++) {
+    let j = i;
+    while (j + 1 < indexed.length && indexed[j + 1].v === indexed[i].v) j++;
+    const avg = (i + j + 2) / 2;
+    for (let k = i; k <= j; k++) r[indexed[k].i] = avg;
+    i = j;
+  }
+  return r;
+}
+function bootstrapPearsonCi(xs, ys, iterations) {
+  const n = xs.length;
+  if (n < 3) return { lower: NaN, upper: NaN };
+  const rs = [];
+  for (let b = 0; b < iterations; b++) {
+    const rx = new Array(n);
+    const ry = new Array(n);
+    for (let i = 0; i < n; i++) {
+      const idx = Math.floor(Math.random() * n);
+      rx[i] = xs[idx];
+      ry[i] = ys[idx];
+    }
+    const r = pearsonR(rx, ry);
+    if (Number.isFinite(r)) rs.push(r);
+  }
+  rs.sort((a, b) => a - b);
+  if (rs.length === 0) return { lower: NaN, upper: NaN };
+  return {
+    lower: rs[Math.floor(0.025 * rs.length)],
+    upper: rs[Math.min(rs.length - 1, Math.floor(0.975 * rs.length))]
+  };
+}
+function defaultExtract2(metric) {
+  return async (run, store) => {
+    switch (metric) {
+      case "score":
+      case "overallScore":
+        return run.outcome?.score ?? null;
+      case "pass":
+        return run.outcome?.pass === true ? 1 : 0;
+      case "durationMs":
+        return run.endedAt && run.startedAt ? run.endedAt - run.startedAt : null;
+      case "costUsd": {
+        const llm = await llmSpans(store, run.runId);
+        return aggregateLlm(llm).costUsd;
+      }
+      case "inputTokens": {
+        const llm = await llmSpans(store, run.runId);
+        return aggregateLlm(llm).inputTokens;
+      }
+      default:
+        return null;
+    }
+  };
+}
+
+// src/meta-eval/outcome-store.ts
+var InMemoryOutcomeStore = class {
+  items = [];
+  async append(outcome) {
+    this.items.push({ ...outcome });
+  }
+  async forRun(runId) {
+    return this.items.filter((o) => o.runId === runId).map((o) => ({ ...o }));
+  }
+  async list(filter = {}) {
+    return this.items.filter((o) => matches(o, filter)).map((o) => ({ ...o }));
+  }
+};
+var FileSystemOutcomeStore = class {
+  dir;
+  maxBytes;
+  memo;
+  loaded = false;
+  constructor(options) {
+    this.dir = options.dir;
+    this.maxBytes = options.maxBytes ?? 32 * 1024 * 1024;
+  }
+  async ensureDir() {
+    const fs = await import("fs/promises");
+    await fs.mkdir(this.dir, { recursive: true });
+  }
+  async append(outcome) {
+    await this.ensureDir();
+    const fs = await import("fs/promises");
+    const path = await import("path");
+    const active = path.join(this.dir, "outcomes.ndjson");
+    try {
+      const stat = await fs.stat(active);
+      if (stat.size >= this.maxBytes) {
+        await fs.rename(active, path.join(this.dir, `outcomes.${Date.now()}.ndjson`));
+      }
+    } catch {
+    }
+    await fs.appendFile(active, `${JSON.stringify(outcome)}
+`, "utf8");
+    if (this.memo) await this.memo.append(outcome);
+  }
+  async load() {
+    if (this.loaded && this.memo) return this.memo;
+    const fs = await import("fs/promises");
+    const path = await import("path");
+    const memo = new InMemoryOutcomeStore();
+    try {
+      const entries = await fs.readdir(this.dir);
+      for (const file of entries) {
+        if (!file.endsWith(".ndjson")) continue;
+        const content = await fs.readFile(path.join(this.dir, file), "utf8");
+        for (const line of content.split("\n")) {
+          if (!line.trim()) continue;
+          await memo.append(JSON.parse(line));
+        }
+      }
+    } catch {
+    }
+    this.memo = memo;
+    this.loaded = true;
+    return memo;
+  }
+  async forRun(runId) {
+    return (await this.load()).forRun(runId);
+  }
+  async list(filter) {
+    return (await this.load()).list(filter);
+  }
+};
+function matches(o, f) {
+  if (f.runIds && !f.runIds.includes(o.runId)) return false;
+  if (f.since !== void 0 && o.capturedAt < f.since) return false;
+  if (f.until !== void 0 && o.capturedAt > f.until) return false;
+  if (f.source && o.source !== f.source) return false;
+  if (f.label && o.labels?.[f.label.key] !== f.label.value) return false;
+  return true;
+}
+export {
+  FileSystemOutcomeStore,
+  InMemoryOutcomeStore,
+  calibrationCurve,
+  correlationStudy,
+  rubricPredictiveValidity
+};
+//# sourceMappingURL=index.js.map

package/dist/meta-eval/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/meta-eval/calibration.ts","../../src/meta-eval/correlation-study.ts","../../src/meta-eval/outcome-store.ts"],"sourcesContent":["/**\n * Calibration curve — binned \"if eval says X, what does reality show?\"\n *\n * Companion to correlationStudy. Raw correlation is a single number;\n * the calibration curve shows *where* the eval is well-calibrated vs\n * overconfident / underconfident. Buckets the eval metric, computes\n * mean outcome per bucket, reports expected-calibration-error (ECE).\n */\n\nimport type { Run } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\nimport type { EvalMetricSpec } from './correlation-study'\nimport type { DeploymentOutcome, OutcomeStore } from './outcome-store'\n\nexport interface CalibrationBin {\n  lower: number\n  upper: number\n  n: number\n  evalMean: number\n  outcomeMean: number\n  /** |outcomeMean − evalMean|; contributes to ECE weighted by n/total. */\n  gap: number\n}\n\nexport interface CalibrationReport {\n  evalMetric: string\n  outcomeMetric: string\n  n: number\n  bins: CalibrationBin[]\n  /** Expected Calibration Error — Σ (n_i/N) × |outcomeMean_i − evalMean_i|. */\n  ece: number\n  /** Max bin gap — upper bound on miscalibration. */\n  maxGap: number\n}\n\nexport interface CalibrationOptions {\n  bins?: number\n  /** Equal-width (fixed bin edges) or equal-frequency (quantile bins). */\n  binning?: 'equal-width' | 'equal-frequency'\n  /** Clip eval values to [lo, hi] before binning. */\n  range?: { lo: number; hi: number }\n}\n\nexport async function calibrationCurve(\n  traceStore: TraceStore,\n  outcomeStore: OutcomeStore,\n  evalMetric: EvalMetricSpec,\n  outcomeMetric: string,\n  options: CalibrationOptions = {},\n): Promise<CalibrationReport | null> {\n  const runs = await traceStore.listRuns()\n  const outcomes = await outcomeStore.list()\n  const byRun = new Map<string, DeploymentOutcome[]>()\n  for (const o of outcomes) {\n    const arr = byRun.get(o.runId) ?? []\n    arr.push(o)\n    byRun.set(o.runId, arr)\n  }\n\n  const extract = evalMetric.extract ?? defaultExtract(evalMetric.id)\n  const pairs: Array<{ x: number; y: number }> = []\n  for (const run of runs) {\n    const os = byRun.get(run.runId)\n    if (!os?.length) continue\n    const x = await extract(run, traceStore)\n    if (x === null || !Number.isFinite(x)) continue\n    const latest = [...os].sort((a, b) => b.capturedAt - a.capturedAt)[0]!\n    const y = latest.metrics[outcomeMetric]\n    if (typeof y !== 'number' || !Number.isFinite(y)) continue\n    pairs.push({ x, y })\n  }\n  if (pairs.length < 2) return null\n\n  const numBins = options.bins ?? 10\n  const binning = options.binning ?? 'equal-width'\n  const xs = pairs.map((p) => p.x)\n  const lo = options.range?.lo ?? Math.min(...xs)\n  const hi = options.range?.hi ?? Math.max(...xs)\n\n  const bins: CalibrationBin[] = []\n  if (binning === 'equal-frequency') {\n    const sorted = [...pairs].sort((a, b) => a.x - b.x)\n    const perBin = Math.max(1, Math.floor(sorted.length / numBins))\n    for (let i = 0; i < sorted.length; i += perBin) {\n      const chunk = sorted.slice(i, i + perBin)\n      if (chunk.length === 0) continue\n      bins.push(toBin(chunk))\n    }\n  } else {\n    const width = (hi - lo) / numBins\n    if (width === 0) return null\n    for (let i = 0; i < numBins; i++) {\n      const binLo = lo + i * width\n      const binHi = i === numBins - 1 ? hi + 1e-9 : lo + (i + 1) * width\n      const chunk = pairs.filter((p) => p.x >= binLo && p.x < binHi)\n      if (chunk.length === 0) continue\n      bins.push(toBin(chunk, binLo, binHi))\n    }\n  }\n\n  const total = bins.reduce((a, b) => a + b.n, 0)\n  const ece = bins.reduce((a, b) => a + (b.n / total) * b.gap, 0)\n  const maxGap = bins.reduce((a, b) => Math.max(a, b.gap), 0)\n\n  return { evalMetric: evalMetric.id, outcomeMetric, n: pairs.length, bins, ece, maxGap }\n}\n\nfunction toBin(\n  chunk: Array<{ x: number; y: number }>,\n  lower?: number,\n  upper?: number,\n): CalibrationBin {\n  const xs = chunk.map((c) => c.x)\n  const ys = chunk.map((c) => c.y)\n  const evalMean = mean(xs)\n  const outcomeMean = mean(ys)\n  return {\n    lower: lower ?? Math.min(...xs),\n    upper: upper ?? Math.max(...xs),\n    n: chunk.length,\n    evalMean,\n    outcomeMean,\n    gap: Math.abs(outcomeMean - evalMean),\n  }\n}\n\nfunction mean(xs: number[]): number {\n  return xs.reduce((a, b) => a + b, 0) / xs.length\n}\n\nfunction defaultExtract(metric: string): (run: Run, store: TraceStore) => Promise<number | null> {\n  return async (run) =>\n    run.outcome?.score ?? (metric === 'pass' ? (run.outcome?.pass === true ? 1 : 0) : null)\n}\n","/**\n * Correlation study — \"does our eval score predict real-world outcomes?\"\n *\n * This is the load-bearing signal. Takes a TraceStore + OutcomeStore,\n * joins on runId, computes Pearson + Spearman + bootstrap CI for every\n * (evalMetric, outcomeMetric) pair the caller declares.\n *\n * Without this number the framework is ornamental. With it and r > 0.6\n * the framework is a moat — no other agent-eval tool publishes one.\n */\n\nimport { aggregateLlm, llmSpans } from '../trace/query'\nimport type { Run } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\nimport type { DeploymentOutcome, OutcomeFilter, OutcomeStore } from './outcome-store'\n\nexport interface EvalMetricSpec {\n  id: string\n  /** Extract a scalar from a run (defaults cover score/pass/durationMs/costUsd/tokens). */\n  extract?: (run: Run, store: TraceStore) => Promise<number | null>\n}\n\nexport interface OutcomePair {\n  evalMetric: string\n  outcomeMetric: string\n}\n\nexport interface CorrelationResult {\n  evalMetric: string\n  outcomeMetric: string\n  n: number\n  pearson: number\n  spearman: number\n  /** 95% bootstrap CI for Pearson. */\n  pearsonCi95: { lower: number; upper: number }\n  /** Rough verdict: 'strong' ≥ 0.7, 'moderate' ≥ 0.4, else 'weak'. */\n  verdict: 'strong' | 'moderate' | 'weak'\n}\n\nexport interface CorrelationStudyResult {\n  pairs: CorrelationResult[]\n  joinedSamples: number\n  skippedRuns: number\n}\n\nexport interface CorrelationStudyOptions {\n  /** Only join outcomes captured within this window after run.startedAt. */\n  maxCaptureLagMs?: number\n  /** Restrict to a subset of outcomes (cohort, region, source). */\n  outcomeFilter?: OutcomeFilter\n  /** Which outcome per run to use when multiple exist. Default 'latest'. */\n  reduction?: 'latest' | 'mean' | 'max'\n  /** Bootstrap iterations for the CI. Default 500. */\n  bootstrapIterations?: number\n}\n\nexport async function correlationStudy(\n  traceStore: TraceStore,\n  outcomeStore: OutcomeStore,\n  evalMetrics: EvalMetricSpec[],\n  outcomeMetricNames: string[],\n  options: CorrelationStudyOptions = {},\n): Promise<CorrelationStudyResult> {\n  const runs = await traceStore.listRuns()\n  const outcomes = await outcomeStore.list(options.outcomeFilter)\n  const outcomesByRun = new Map<string, DeploymentOutcome[]>()\n  for (const o of outcomes) {\n    const arr = outcomesByRun.get(o.runId) ?? []\n    arr.push(o)\n    outcomesByRun.set(o.runId, arr)\n  }\n\n  const reduction = options.reduction ?? 'latest'\n  const maxLag = options.maxCaptureLagMs ?? Infinity\n\n  const pairs: Array<{ evalMetric: string; outcomeMetric: string; xs: number[]; ys: number[] }> = []\n  for (const em of evalMetrics) {\n    for (const om of outcomeMetricNames) {\n      pairs.push({ evalMetric: em.id, outcomeMetric: om, xs: [], ys: [] })\n    }\n  }\n\n  let joined = 0\n  let skipped = 0\n  for (const run of runs) {\n    const os = outcomesByRun.get(run.runId)\n    if (!os || os.length === 0) {\n      skipped++\n      continue\n    }\n    const eligible = os.filter((o) => o.capturedAt - run.startedAt <= maxLag)\n    if (eligible.length === 0) {\n      skipped++\n      continue\n    }\n\n    for (const em of evalMetrics) {\n      const extract = em.extract ?? defaultExtract(em.id)\n      const x = await extract(run, traceStore)\n      if (x === null || !Number.isFinite(x)) continue\n\n      for (const om of outcomeMetricNames) {\n        const values = eligible\n          .map((o) => o.metrics[om])\n          .filter((v): v is number => typeof v === 'number' && Number.isFinite(v))\n        if (values.length === 0) continue\n        const y = reduce(values, reduction, eligible)\n        if (y === null) continue\n        const pair = pairs.find((p) => p.evalMetric === em.id && p.outcomeMetric === om)!\n        pair.xs.push(x)\n        pair.ys.push(y)\n      }\n    }\n    joined++\n  }\n\n  const results: CorrelationResult[] = pairs\n    .filter((p) => p.xs.length >= 3)\n    .map((p) => {\n      const pearson = pearsonR(p.xs, p.ys)\n      const spearman = pearsonR(ranks(p.xs), ranks(p.ys))\n      const pearsonCi95 = bootstrapPearsonCi(p.xs, p.ys, options.bootstrapIterations ?? 500)\n      const verdict: CorrelationResult['verdict'] =\n        Math.abs(pearson) >= 0.7 ? 'strong' : Math.abs(pearson) >= 0.4 ? 'moderate' : 'weak'\n      return {\n        evalMetric: p.evalMetric,\n        outcomeMetric: p.outcomeMetric,\n        n: p.xs.length,\n        pearson,\n        spearman,\n        pearsonCi95,\n        verdict,\n      }\n    })\n\n  return { pairs: results, joinedSamples: joined, skippedRuns: skipped }\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\nfunction reduce(\n  values: number[],\n  kind: 'latest' | 'mean' | 'max',\n  outcomes: DeploymentOutcome[],\n): number | null {\n  if (values.length === 0) return null\n  if (kind === 'mean') return values.reduce((a, b) => a + b, 0) / values.length\n  if (kind === 'max') return Math.max(...values)\n  // 'latest': pick the outcome captured last, then lookup its metric\n  const latest = [...outcomes].sort((a, b) => b.capturedAt - a.capturedAt)[0]\n  if (!latest) return null\n  const latestKey = Object.keys(latest.metrics)[0]\n  const v = latestKey !== undefined ? latest.metrics[latestKey] : undefined\n  // For 'latest' we already have `values` aligned; use the last-captured one\n  const paired = outcomes\n    .map((o) => {\n      const k = Object.keys(o.metrics)[0]\n      return {\n        at: o.capturedAt,\n        v: k !== undefined ? values.find((x) => o.metrics[k] === x) : undefined,\n      }\n    })\n    .filter((p) => p.v !== undefined)\n  if (paired.length === 0) return v ?? null\n  return paired.sort((a, b) => b.at - a.at)[0]?.v ?? null\n}\n\nfunction pearsonR(a: number[], b: number[]): number {\n  if (a.length !== b.length || a.length < 2) return NaN\n  const mA = a.reduce((s, v) => s + v, 0) / a.length\n  const mB = b.reduce((s, v) => s + v, 0) / b.length\n  let num = 0,\n    dA = 0,\n    dB = 0\n  for (let i = 0; i < a.length; i++) {\n    const da = a[i]! - mA,\n      db = b[i]! - mB\n    num += da * db\n    dA += da * da\n    dB += db * db\n  }\n  if (dA === 0 || dB === 0) return dA === 0 && dB === 0 ? 1 : 0\n  return num / Math.sqrt(dA * dB)\n}\n\nfunction ranks(xs: number[]): number[] {\n  const indexed = xs.map((v, i) => ({ v, i })).sort((x, y) => x.v - y.v)\n  const r = new Array<number>(xs.length)\n  for (let i = 0; i < indexed.length; i++) {\n    let j = i\n    while (j + 1 < indexed.length && indexed[j + 1]!.v === indexed[i]!.v) j++\n    const avg = (i + j + 2) / 2\n    for (let k = i; k <= j; k++) r[indexed[k]!.i] = avg\n    i = j\n  }\n  return r\n}\n\nfunction bootstrapPearsonCi(\n  xs: number[],\n  ys: number[],\n  iterations: number,\n): { lower: number; upper: number } {\n  const n = xs.length\n  if (n < 3) return { lower: NaN, upper: NaN }\n  const rs: number[] = []\n  for (let b = 0; b < iterations; b++) {\n    const rx: number[] = new Array(n)\n    const ry: number[] = new Array(n)\n    for (let i = 0; i < n; i++) {\n      const idx = Math.floor(Math.random() * n)\n      rx[i] = xs[idx]!\n      ry[i] = ys[idx]!\n    }\n    const r = pearsonR(rx, ry)\n    if (Number.isFinite(r)) rs.push(r)\n  }\n  rs.sort((a, b) => a - b)\n  if (rs.length === 0) return { lower: NaN, upper: NaN }\n  return {\n    lower: rs[Math.floor(0.025 * rs.length)]!,\n    upper: rs[Math.min(rs.length - 1, Math.floor(0.975 * rs.length))]!,\n  }\n}\n\nfunction defaultExtract(metric: string): (run: Run, store: TraceStore) => Promise<number | null> {\n  return async (run, store) => {\n    switch (metric) {\n      case 'score':\n      case 'overallScore':\n        return run.outcome?.score ?? null\n      case 'pass':\n        return run.outcome?.pass === true ? 1 : 0\n      case 'durationMs':\n        return run.endedAt && run.startedAt ? run.endedAt - run.startedAt : null\n      case 'costUsd': {\n        const llm = await llmSpans(store, run.runId)\n        return aggregateLlm(llm).costUsd\n      }\n      case 'inputTokens': {\n        const llm = await llmSpans(store, run.runId)\n        return aggregateLlm(llm).inputTokens\n      }\n      default:\n        return null\n    }\n  }\n}\n","/**\n * OutcomeStore — deployment outcomes attached to Run IDs.\n *\n * Outcomes arrive asynchronously from production telemetry after the\n * eval run completed: user ratings, retention flags, conversion events,\n * revenue, support-ticket rate, anything a product team can measure.\n * The store is a peer to TraceStore — separate lifecycle, same runId\n * foreign key.\n *\n * The whole point of this module is to make the meta-eval correlation\n * question computable: `correlate(evalMetric, outcomeMetric) → r, ρ, n, CI`.\n */\n\nexport interface DeploymentOutcome {\n  runId: string\n  capturedAt: number\n  /** Numeric outcomes keyed by name — retention_7d, csat, revenue_usd, etc. */\n  metrics: Record<string, number>\n  /** Dimensions for stratified analysis — cohort, region, user_segment. */\n  labels?: Record<string, string>\n  /** Free-form provenance (source system, pipeline version). */\n  source?: string\n}\n\nexport interface OutcomeFilter {\n  runIds?: string[]\n  since?: number\n  until?: number\n  label?: { key: string; value: string }\n  source?: string\n}\n\nexport interface OutcomeStore {\n  append(outcome: DeploymentOutcome): Promise<void>\n  /** All outcomes attached to this run (a single run can have many — multiple\n   *  capture windows over deployment time). */\n  forRun(runId: string): Promise<DeploymentOutcome[]>\n  list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]>\n}\n\nexport class InMemoryOutcomeStore implements OutcomeStore {\n  private items: DeploymentOutcome[] = []\n\n  async append(outcome: DeploymentOutcome): Promise<void> {\n    this.items.push({ ...outcome })\n  }\n\n  async forRun(runId: string): Promise<DeploymentOutcome[]> {\n    return this.items.filter((o) => o.runId === runId).map((o) => ({ ...o }))\n  }\n\n  async list(filter: OutcomeFilter = {}): Promise<DeploymentOutcome[]> {\n    return this.items.filter((o) => matches(o, filter)).map((o) => ({ ...o }))\n  }\n}\n\nexport interface FileSystemOutcomeStoreOptions {\n  dir: string\n  maxBytes?: number\n}\n\nexport class FileSystemOutcomeStore implements OutcomeStore {\n  private dir: string\n  private maxBytes: number\n  private memo?: InMemoryOutcomeStore\n  private loaded = false\n\n  constructor(options: FileSystemOutcomeStoreOptions) {\n    this.dir = options.dir\n    this.maxBytes = options.maxBytes ?? 32 * 1024 * 1024\n  }\n\n  private async ensureDir(): Promise<void> {\n    const fs = await import('node:fs/promises')\n    await fs.mkdir(this.dir, { recursive: true })\n  }\n\n  async append(outcome: DeploymentOutcome): Promise<void> {\n    await this.ensureDir()\n    const fs = await import('node:fs/promises')\n    const path = await import('node:path')\n    const active = path.join(this.dir, 'outcomes.ndjson')\n    try {\n      const stat = await fs.stat(active)\n      if (stat.size >= this.maxBytes) {\n        await fs.rename(active, path.join(this.dir, `outcomes.${Date.now()}.ndjson`))\n      }\n    } catch {\n      /* first write */\n    }\n    await fs.appendFile(active, `${JSON.stringify(outcome)}\\n`, 'utf8')\n    if (this.memo) await this.memo.append(outcome)\n  }\n\n  private async load(): Promise<InMemoryOutcomeStore> {\n    if (this.loaded && this.memo) return this.memo\n    const fs = await import('node:fs/promises')\n    const path = await import('node:path')\n    const memo = new InMemoryOutcomeStore()\n    try {\n      const entries = await fs.readdir(this.dir)\n      for (const file of entries) {\n        if (!file.endsWith('.ndjson')) continue\n        const content = await fs.readFile(path.join(this.dir, file), 'utf8')\n        for (const line of content.split('\\n')) {\n          if (!line.trim()) continue\n          await memo.append(JSON.parse(line))\n        }\n      }\n    } catch {\n      /* empty */\n    }\n    this.memo = memo\n    this.loaded = true\n    return memo\n  }\n\n  async forRun(runId: string): Promise<DeploymentOutcome[]> {\n    return (await this.load()).forRun(runId)\n  }\n\n  async list(filter?: OutcomeFilter): Promise<DeploymentOutcome[]> {\n    return (await this.load()).list(filter)\n  }\n}\n\nfunction matches(o: DeploymentOutcome, f: OutcomeFilter): boolean {\n  if (f.runIds && !f.runIds.includes(o.runId)) return false\n  if (f.since !== undefined && o.capturedAt < f.since) return false\n  if (f.until !== undefined && o.capturedAt > f.until) return false\n  if (f.source && o.source !== f.source) return false\n  if (f.label && o.labels?.[f.label.key] !== f.label.value) return false\n  return true\n}\n"],"mappings":";;;;;;;;;;;AA2CA,eAAsB,iBACpB,YACA,cACA,YACA,eACA,UAA8B,CAAC,GACI;AACnC,QAAM,OAAO,MAAM,WAAW,SAAS;AACvC,QAAM,WAAW,MAAM,aAAa,KAAK;AACzC,QAAM,QAAQ,oBAAI,IAAiC;AACnD,aAAW,KAAK,UAAU;AACxB,UAAM,MAAM,MAAM,IAAI,EAAE,KAAK,KAAK,CAAC;AACnC,QAAI,KAAK,CAAC;AACV,UAAM,IAAI,EAAE,OAAO,GAAG;AAAA,EACxB;AAEA,QAAM,UAAU,WAAW,WAAW,eAAe,WAAW,EAAE;AAClE,QAAM,QAAyC,CAAC;AAChD,aAAW,OAAO,MAAM;AACtB,UAAM,KAAK,MAAM,IAAI,IAAI,KAAK;AAC9B,QAAI,CAAC,IAAI,OAAQ;AACjB,UAAM,IAAI,MAAM,QAAQ,KAAK,UAAU;AACvC,QAAI,MAAM,QAAQ,CAAC,OAAO,SAAS,CAAC,EAAG;AACvC,UAAM,SAAS,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,CAAC;AACpE,UAAM,IAAI,OAAO,QAAQ,aAAa;AACtC,QAAI,OAAO,MAAM,YAAY,CAAC,OAAO,SAAS,CAAC,EAAG;AAClD,UAAM,KAAK,EAAE,GAAG,EAAE,CAAC;AAAA,EACrB;AACA,MAAI,MAAM,SAAS,EAAG,QAAO;AAE7B,QAAM,UAAU,QAAQ,QAAQ;AAChC,QAAM,UAAU,QAAQ,WAAW;AACnC,QAAM,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;AAC/B,QAAM,KAAK,QAAQ,OAAO,MAAM,KAAK,IAAI,GAAG,EAAE;AAC9C,QAAM,KAAK,QAAQ,OAAO,MAAM,KAAK,IAAI,GAAG,EAAE;AAE9C,QAAM,OAAyB,CAAC;AAChC,MAAI,YAAY,mBAAmB;AACjC,UAAM,SAAS,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC;AAClD,UAAM,SAAS,KAAK,IAAI,GAAG,KAAK,MAAM,OAAO,SAAS,OAAO,CAAC;AAC9D,aAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK,QAAQ;AAC9C,YAAM,QAAQ,OAAO,MAAM,GAAG,IAAI,MAAM;AACxC,UAAI,MAAM,WAAW,EAAG;AACxB,WAAK,KAAK,MAAM,KAAK,CAAC;AAAA,IACxB;AAAA,EACF,OAAO;AACL,UAAM,SAAS,KAAK,MAAM;AAC1B,QAAI,UAAU,EAAG,QAAO;AACxB,aAAS,IAAI,GAAG,IAAI,SAAS,KAAK;AAChC,YAAM,QAAQ,KAAK,IAAI;AACvB,YAAM,QAAQ,MAAM,UAAU,IAAI,KAAK,OAAO,MAAM,IAAI,KAAK;AAC7D,YAAM,QAAQ,MAAM,OAAO,CAAC,MAAM,EAAE,KAAK,SAAS,EAAE,IAAI,KAAK;AAC7D,UAAI,MAAM,WAAW,EAAG;AACxB,WAAK,KAAK,MAAM,OAAO,OAAO,KAAK,CAAC;AAAA,IACtC;AAAA,EACF;AAEA,QAAM,QAAQ,KAAK,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,GAAG,CAAC;AAC9C,QAAM,MAAM,KAAK,OAAO,CAAC,GAAG,MAAM,IAAK,EAAE,IAAI,QAAS,EAAE,KAAK,CAAC;AAC9D,QAAM,SAAS,KAAK,OAAO,CAAC,GAAG,MAAM,KAAK,IAAI,GAAG,EAAE,GAAG,GAAG,CAAC;AAE1D,SAAO,EAAE,YAAY,WAAW,IAAI,eAAe,GAAG,MAAM,QAAQ,MAAM,KAAK,OAAO;AACxF;AAEA,SAAS,MACP,OACA,OACA,OACgB;AAChB,QAAM,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;AAC/B,QAAM,KAAK,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;AAC/B,QAAM,WAAW,KAAK,EAAE;AACxB,QAAM,cAAc,KAAK,EAAE;AAC3B,SAAO;AAAA,IACL,OAAO,SAAS,KAAK,IAAI,GAAG,EAAE;AAAA,IAC9B,OAAO,SAAS,KAAK,IAAI,GAAG,EAAE;AAAA,IAC9B,GAAG,MAAM;AAAA,IACT;AAAA,IACA;AAAA,IACA,KAAK,KAAK,IAAI,cAAc,QAAQ;AAAA,EACtC;AACF;AAEA,SAAS,KAAK,IAAsB;AAClC,SAAO,GAAG,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,GAAG;AAC5C;AAEA,SAAS,eAAe,QAAyE;AAC/F,SAAO,OAAO,QACZ,IAAI,SAAS,UAAU,WAAW,SAAU,IAAI,SAAS,SAAS,OAAO,IAAI,IAAK;AACtF;;;AC7EA,eAAsB,iBACpB,YACA,cACA,aACA,oBACA,UAAmC,CAAC,GACH;AACjC,QAAM,OAAO,MAAM,WAAW,SAAS;AACvC,QAAM,WAAW,MAAM,aAAa,KAAK,QAAQ,aAAa;AAC9D,QAAM,gBAAgB,oBAAI,IAAiC;AAC3D,aAAW,KAAK,UAAU;AACxB,UAAM,MAAM,cAAc,IAAI,EAAE,KAAK,KAAK,CAAC;AAC3C,QAAI,KAAK,CAAC;AACV,kBAAc,IAAI,EAAE,OAAO,GAAG;AAAA,EAChC;AAEA,QAAM,YAAY,QAAQ,aAAa;AACvC,QAAM,SAAS,QAAQ,mBAAmB;AAE1C,QAAM,QAA0F,CAAC;AACjG,aAAW,MAAM,aAAa;AAC5B,eAAW,MAAM,oBAAoB;AACnC,YAAM,KAAK,EAAE,YAAY,GAAG,IAAI,eAAe,IAAI,IAAI,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC;AAAA,IACrE;AAAA,EACF;AAEA,MAAI,SAAS;AACb,MAAI,UAAU;AACd,aAAW,OAAO,MAAM;AACtB,UAAM,KAAK,cAAc,IAAI,IAAI,KAAK;AACtC,QAAI,CAAC,MAAM,GAAG,WAAW,GAAG;AAC1B;AACA;AAAA,IACF;AACA,UAAM,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,aAAa,IAAI,aAAa,MAAM;AACxE,QAAI,SAAS,WAAW,GAAG;AACzB;AACA;AAAA,IACF;AAEA,eAAW,MAAM,aAAa;AAC5B,YAAM,UAAU,GAAG,WAAWA,gBAAe,GAAG,EAAE;AAClD,YAAM,IAAI,MAAM,QAAQ,KAAK,UAAU;AACvC,UAAI,MAAM,QAAQ,CAAC,OAAO,SAAS,CAAC,EAAG;AAEvC,iBAAW,MAAM,oBAAoB;AACnC,cAAM,SAAS,SACZ,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,EACxB,OAAO,CAAC,MAAmB,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,CAAC;AACzE,YAAI,OAAO,WAAW,EAAG;AACzB,cAAM,IAAI,OAAO,QAAQ,WAAW,QAAQ;AAC5C,YAAI,MAAM,KAAM;AAChB,cAAM,OAAO,MAAM,KAAK,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,EAAE,kBAAkB,EAAE;AAC/E,aAAK,GAAG,KAAK,CAAC;AACd,aAAK,GAAG,KAAK,CAAC;AAAA,MAChB;AAAA,IACF;AACA;AAAA,EACF;AAEA,QAAM,UAA+B,MAClC,OAAO,CAAC,MAAM,EAAE,GAAG,UAAU,CAAC,EAC9B,IAAI,CAAC,MAAM;AACV,UAAM,UAAU,SAAS,EAAE,IAAI,EAAE,EAAE;AACnC,UAAM,WAAW,SAAS,MAAM,EAAE,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC;AAClD,UAAM,cAAc,mBAAmB,EAAE,IAAI,EAAE,IAAI,QAAQ,uBAAuB,GAAG;AACrF,UAAM,UACJ,KAAK,IAAI,OAAO,KAAK,MAAM,WAAW,KAAK,IAAI,OAAO,KAAK,MAAM,aAAa;AAChF,WAAO;AAAA,MACL,YAAY,EAAE;AAAA,MACd,eAAe,EAAE;AAAA,MACjB,GAAG,EAAE,GAAG;AAAA,MACR;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF,CAAC;AAEH,SAAO,EAAE,OAAO,SAAS,eAAe,QAAQ,aAAa,QAAQ;AACvE;AAIA,SAAS,OACP,QACA,MACA,UACe;AACf,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,MAAI,SAAS,OAAQ,QAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AACvE,MAAI,SAAS,MAAO,QAAO,KAAK,IAAI,GAAG,MAAM;AAE7C,QAAM,SAAS,CAAC,GAAG,QAAQ,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,CAAC;AAC1E,MAAI,CAAC,OAAQ,QAAO;AACpB,QAAM,YAAY,OAAO,KAAK,OAAO,OAAO,EAAE,CAAC;AAC/C,QAAM,IAAI,cAAc,SAAY,OAAO,QAAQ,SAAS,IAAI;AAEhE,QAAM,SAAS,SACZ,IAAI,CAAC,MAAM;AACV,UAAM,IAAI,OAAO,KAAK,EAAE,OAAO,EAAE,CAAC;AAClC,WAAO;AAAA,MACL,IAAI,EAAE;AAAA,MACN,GAAG,MAAM,SAAY,OAAO,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI;AAAA,IAChE;AAAA,EACF,CAAC,EACA,OAAO,CAAC,MAAM,EAAE,MAAM,MAAS;AAClC,MAAI,OAAO,WAAW,EAAG,QAAO,KAAK;AACrC,SAAO,OAAO,KAAK,CAAC,GAAG,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,GAAG,KAAK;AACrD;AAEA,SAAS,SAAS,GAAa,GAAqB;AAClD,MAAI,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAG,QAAO;AAClD,QAAM,KAAK,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE;AAC5C,QAAM,KAAK,EAAE,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,EAAE;AAC5C,MAAI,MAAM,GACR,KAAK,GACL,KAAK;AACP,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,UAAM,KAAK,EAAE,CAAC,IAAK,IACjB,KAAK,EAAE,CAAC,IAAK;AACf,WAAO,KAAK;AACZ,UAAM,KAAK;AACX,UAAM,KAAK;AAAA,EACb;AACA,MAAI,OAAO,KAAK,OAAO,EAAG,QAAO,OAAO,KAAK,OAAO,IAAI,IAAI;AAC5D,SAAO,MAAM,KAAK,KAAK,KAAK,EAAE;AAChC;AAEA,SAAS,MAAM,IAAwB;AACrC,QAAM,UAAU,GAAG,IAAI,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,IAAI,EAAE,CAAC;AACrE,QAAM,IAAI,IAAI,MAAc,GAAG,MAAM;AACrC,WAAS,IAAI,GAAG,IAAI,QAAQ,QAAQ,KAAK;AACvC,QAAI,IAAI;AACR,WAAO,IAAI,IAAI,QAAQ,UAAU,QAAQ,IAAI,CAAC,EAAG,MAAM,QAAQ,CAAC,EAAG,EAAG;AACtE,UAAM,OAAO,IAAI,IAAI,KAAK;AAC1B,aAAS,IAAI,GAAG,KAAK,GAAG,IAAK,GAAE,QAAQ,CAAC,EAAG,CAAC,IAAI;AAChD,QAAI;AAAA,EACN;AACA,SAAO;AACT;AAEA,SAAS,mBACP,IACA,IACA,YACkC;AAClC,QAAM,IAAI,GAAG;AACb,MAAI,IAAI,EAAG,QAAO,EAAE,OAAO,KAAK,OAAO,IAAI;AAC3C,QAAM,KAAe,CAAC;AACtB,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,UAAM,KAAe,IAAI,MAAM,CAAC;AAChC,UAAM,KAAe,IAAI,MAAM,CAAC;AAChC,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,YAAM,MAAM,KAAK,MAAM,KAAK,OAAO,IAAI,CAAC;AACxC,SAAG,CAAC,IAAI,GAAG,GAAG;AACd,SAAG,CAAC,IAAI,GAAG,GAAG;AAAA,IAChB;AACA,UAAM,IAAI,SAAS,IAAI,EAAE;AACzB,QAAI,OAAO,SAAS,CAAC,EAAG,IAAG,KAAK,CAAC;AAAA,EACnC;AACA,KAAG,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AACvB,MAAI,GAAG,WAAW,EAAG,QAAO,EAAE,OAAO,KAAK,OAAO,IAAI;AACrD,SAAO;AAAA,IACL,OAAO,GAAG,KAAK,MAAM,QAAQ,GAAG,MAAM,CAAC;AAAA,IACvC,OAAO,GAAG,KAAK,IAAI,GAAG,SAAS,GAAG,KAAK,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC;AAAA,EAClE;AACF;AAEA,SAASA,gBAAe,QAAyE;AAC/F,SAAO,OAAO,KAAK,UAAU;AAC3B,YAAQ,QAAQ;AAAA,MACd,KAAK;AAAA,MACL,KAAK;AACH,eAAO,IAAI,SAAS,SAAS;AAAA,MAC/B,KAAK;AACH,eAAO,IAAI,SAAS,SAAS,OAAO,IAAI;AAAA,MAC1C,KAAK;AACH,eAAO,IAAI,WAAW,IAAI,YAAY,IAAI,UAAU,IAAI,YAAY;AAAA,MACtE,KAAK,WAAW;AACd,cAAM,MAAM,MAAM,SAAS,OAAO,IAAI,KAAK;AAC3C,eAAO,aAAa,GAAG,EAAE;AAAA,MAC3B;AAAA,MACA,KAAK,eAAe;AAClB,cAAM,MAAM,MAAM,SAAS,OAAO,IAAI,KAAK;AAC3C,eAAO,aAAa,GAAG,EAAE;AAAA,MAC3B;AAAA,MACA;AACE,eAAO;AAAA,IACX;AAAA,EACF;AACF;;;AC/MO,IAAM,uBAAN,MAAmD;AAAA,EAChD,QAA6B,CAAC;AAAA,EAEtC,MAAM,OAAO,SAA2C;AACtD,SAAK,MAAM,KAAK,EAAE,GAAG,QAAQ,CAAC;AAAA,EAChC;AAAA,EAEA,MAAM,OAAO,OAA6C;AACxD,WAAO,KAAK,MAAM,OAAO,CAAC,MAAM,EAAE,UAAU,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE;AAAA,EAC1E;AAAA,EAEA,MAAM,KAAK,SAAwB,CAAC,GAAiC;AACnE,WAAO,KAAK,MAAM,OAAO,CAAC,MAAM,QAAQ,GAAG,MAAM,CAAC,EAAE,IAAI,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE;AAAA,EAC3E;AACF;AAOO,IAAM,yBAAN,MAAqD;AAAA,EAClD;AAAA,EACA;AAAA,EACA;AAAA,EACA,SAAS;AAAA,EAEjB,YAAY,SAAwC;AAClD,SAAK,MAAM,QAAQ;AACnB,SAAK,WAAW,QAAQ,YAAY,KAAK,OAAO;AAAA,EAClD;AAAA,EAEA,MAAc,YAA2B;AACvC,UAAM,KAAK,MAAM,OAAO,aAAkB;AAC1C,UAAM,GAAG,MAAM,KAAK,KAAK,EAAE,WAAW,KAAK,CAAC;AAAA,EAC9C;AAAA,EAEA,MAAM,OAAO,SAA2C;AACtD,UAAM,KAAK,UAAU;AACrB,UAAM,KAAK,MAAM,OAAO,aAAkB;AAC1C,UAAM,OAAO,MAAM,OAAO,MAAW;AACrC,UAAM,SAAS,KAAK,KAAK,KAAK,KAAK,iBAAiB;AACpD,QAAI;AACF,YAAM,OAAO,MAAM,GAAG,KAAK,MAAM;AACjC,UAAI,KAAK,QAAQ,KAAK,UAAU;AAC9B,cAAM,GAAG,OAAO,QAAQ,KAAK,KAAK,KAAK,KAAK,YAAY,KAAK,IAAI,CAAC,SAAS,CAAC;AAAA,MAC9E;AAAA,IACF,QAAQ;AAAA,IAER;AACA,UAAM,GAAG,WAAW,QAAQ,GAAG,KAAK,UAAU,OAAO,CAAC;AAAA,GAAM,MAAM;AAClE,QAAI,KAAK,KAAM,OAAM,KAAK,KAAK,OAAO,OAAO;AAAA,EAC/C;AAAA,EAEA,MAAc,OAAsC;AAClD,QAAI,KAAK,UAAU,KAAK,KAAM,QAAO,KAAK;AAC1C,UAAM,KAAK,MAAM,OAAO,aAAkB;AAC1C,UAAM,OAAO,MAAM,OAAO,MAAW;AACrC,UAAM,OAAO,IAAI,qBAAqB;AACtC,QAAI;AACF,YAAM,UAAU,MAAM,GAAG,QAAQ,KAAK,GAAG;AACzC,iBAAW,QAAQ,SAAS;AAC1B,YAAI,CAAC,KAAK,SAAS,SAAS,EAAG;AAC/B,cAAM,UAAU,MAAM,GAAG,SAAS,KAAK,KAAK,KAAK,KAAK,IAAI,GAAG,MAAM;AACnE,mBAAW,QAAQ,QAAQ,MAAM,IAAI,GAAG;AACtC,cAAI,CAAC,KAAK,KAAK,EAAG;AAClB,gBAAM,KAAK,OAAO,KAAK,MAAM,IAAI,CAAC;AAAA,QACpC;AAAA,MACF;AAAA,IACF,QAAQ;AAAA,IAER;AACA,SAAK,OAAO;AACZ,SAAK,SAAS;AACd,WAAO;AAAA,EACT;AAAA,EAEA,MAAM,OAAO,OAA6C;AACxD,YAAQ,MAAM,KAAK,KAAK,GAAG,OAAO,KAAK;AAAA,EACzC;AAAA,EAEA,MAAM,KAAK,QAAsD;AAC/D,YAAQ,MAAM,KAAK,KAAK,GAAG,KAAK,MAAM;AAAA,EACxC;AACF;AAEA,SAAS,QAAQ,GAAsB,GAA2B;AAChE,MAAI,EAAE,UAAU,CAAC,EAAE,OAAO,SAAS,EAAE,KAAK,EAAG,QAAO;AACpD,MAAI,EAAE,UAAU,UAAa,EAAE,aAAa,EAAE,MAAO,QAAO;AAC5D,MAAI,EAAE,UAAU,UAAa,EAAE,aAAa,EAAE,MAAO,QAAO;AAC5D,MAAI,EAAE,UAAU,EAAE,WAAW,EAAE,OAAQ,QAAO;AAC9C,MAAI,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,MAAM,MAAO,QAAO;AACjE,SAAO;AACT;","names":["defaultExtract"]}