@agjs/tsforge 0.1.11 → 0.1.12

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@agjs/tsforge",
   "type": "module",
-  "version": "0.1.11",
+  "version": "0.1.12",
   "license": "MIT",
   "description": "TypeScript coding harness with a deterministic gate, stack-aware guardrails, and stream-level correction.",
   "repository": {

package/src/eval/index.ts

@@ -1,3 +1,12 @@
 export * from "./eval.types";
 export { judge } from "./judge";
 export { summarize } from "./score";
+export { analyzeEvents, type IRunMetrics } from "./metrics";
+export {
+  buildSweepReport,
+  renderSweepReportMarkdown,
+  wilsonInterval,
+  twoProportionZ,
+  type ISweepReport,
+  type IVariantReport,
+} from "./report";

package/src/eval/metrics.ts

@@ -0,0 +1,87 @@
+import type { ILoopEvent } from "../loop/loop.types";
+
+/** Behavioral metrics distilled from a run's event stream — the signals the
+ *  local-model literature says predict outcomes (tokens-to-solution, repair
+ *  iterations, peak context) rather than vibes. A reusable, pure counterpart to
+ *  the cli-metrics script. */
+export interface IRunMetrics {
+  finalStatus: "done" | "stuck" | "none";
+  /** Model turns (one per `cycle` event). */
+  turns: number;
+  /** Model calls (one per `usage` event). */
+  modelCalls: number;
+  /** Total completion tokens generated. */
+  tokensOut: number;
+  /** Largest prompt-token count seen (the run's context high-water mark). */
+  peakContext: number;
+  /** File mutations (`edit` + `create`). */
+  edits: number;
+  /** Distinct files created. */
+  filesCreated: number;
+  /** Gate runs (`validated` events). */
+  gateRuns: number;
+  /** Summed turn wall-clock from `timing` events, in seconds. */
+  wallClockSeconds: number;
+  /** Mean output rate across calls that reported one (tokens/second). */
+  avgTokensPerSecond: number;
+}
+
+function emptyMetrics(): IRunMetrics {
+  return {
+    finalStatus: "none",
+    turns: 0,
+    modelCalls: 0,
+    tokensOut: 0,
+    peakContext: 0,
+    edits: 0,
+    filesCreated: 0,
+    gateRuns: 0,
+    wallClockSeconds: 0,
+    avgTokensPerSecond: 0,
+  };
+}
+
+/** Reduce a run's event stream to its behavioral metrics. Pure — feed it the
+ *  events from a `--log` JSONL or a captured `onEvent` stream. */
+export function analyzeEvents(events: readonly ILoopEvent[]): IRunMetrics {
+  const m = emptyMetrics();
+  const created = new Set<string>();
+  let tpsSum = 0;
+  let tpsCount = 0;
+
+  for (const event of events) {
+    if (event.kind === "cycle") {
+      m.turns += 1;
+    } else if (event.kind === "usage") {
+      m.modelCalls += 1;
+      m.tokensOut += event.completionTokens ?? 0;
+      m.peakContext = Math.max(m.peakContext, event.promptTokens ?? 0);
+
+      if (event.tokensPerSecond !== undefined && event.tokensPerSecond > 0) {
+        tpsSum += event.tokensPerSecond;
+        tpsCount += 1;
+      }
+    } else if (event.kind === "create") {
+      m.edits += 1;
+
+      if (event.file !== undefined && event.file.length > 0) {
+        created.add(event.file);
+      }
+    } else if (event.kind === "edit") {
+      m.edits += 1;
+    } else if (event.kind === "timing") {
+      m.wallClockSeconds += Math.round((event.ms ?? 0) / 1000);
+    } else if (event.kind === "validated") {
+      m.gateRuns += 1;
+    } else if (event.kind === "done") {
+      m.finalStatus = "done";
+    } else if (event.kind === "stuck") {
+      m.finalStatus = "stuck";
+    }
+  }
+
+  m.filesCreated = created.size;
+  m.avgTokensPerSecond = tpsCount > 0 ? Math.round(tpsSum / tpsCount) : 0;
+
+  return m;
+}

package/src/eval/report.ts

@@ -0,0 +1,168 @@
+import type { IRunRecord, IVariantSummary } from "./eval.types";
+import { summarize } from "./score";
+
+/** 95% normal quantile — the multiplier for Wilson intervals and the z-test. */
+const Z95 = 1.96;
+
+/** A variant summary enriched with a confidence interval and, when a baseline is
+ *  given, a significance test of its pass-rate difference from that baseline. */
+export interface IVariantReport extends IVariantSummary {
+  /** 95% Wilson score interval for the pass rate, as [low, high] in [0, 1]. */
+  readonly passRateCI: readonly [number, number];
+  /** Comparison vs the baseline variant (absent for the baseline itself or when
+   *  no baseline was supplied). */
+  readonly vsBaseline?: {
+    readonly deltaPassRate: number;
+    readonly z: number;
+    /** True when |z| > 1.96 (p < 0.05, two-sided). */
+    readonly significant: boolean;
+  };
+}
+
+export interface ISweepReport {
+  /** The baseline variant label, or null if none was matched. */
+  readonly baseline: string | null;
+  readonly variants: readonly IVariantReport[];
+}
+
+/** 95% Wilson score interval for `passed` successes out of `n` trials. */
+export function wilsonInterval(passed: number, n: number): [number, number] {
+  if (n === 0) {
+    return [0, 0];
+  }
+
+  const phat = passed / n;
+  const z2 = Z95 * Z95;
+  const denom = 1 + z2 / n;
+  const centre = phat + z2 / (2 * n);
+  const margin = Z95 * Math.sqrt((phat * (1 - phat)) / n + z2 / (4 * n * n));
+
+  return [
+    Math.max(0, (centre - margin) / denom),
+    Math.min(1, (centre + margin) / denom),
+  ];
+}
+
+/** Pooled two-proportion z-statistic comparing rate1 (x1/n1) to rate2 (x2/n2). */
+export function twoProportionZ(
+  x1: number,
+  n1: number,
+  x2: number,
+  n2: number
+): number {
+  if (n1 === 0 || n2 === 0) {
+    return 0;
+  }
+
+  const pooled = (x1 + x2) / (n1 + n2);
+  const se = Math.sqrt(pooled * (1 - pooled) * (1 / n1 + 1 / n2));
+
+  if (se === 0) {
+    return 0;
+  }
+
+  return (x1 / n1 - x2 / n2) / se;
+}
+
+function compareToBaseline(
+  variant: IVariantSummary,
+  baseline: IVariantSummary
+): IVariantReport["vsBaseline"] {
+  const z = twoProportionZ(
+    variant.passed,
+    variant.runs,
+    baseline.passed,
+    baseline.runs
+  );
+
+  return {
+    deltaPassRate: variant.passRate - baseline.passRate,
+    z,
+    significant: Math.abs(z) > Z95,
+  };
+}
+
+/**
+ * Aggregate raw run records into a statistical report: per-variant pass rate with
+ * a 95% Wilson interval, plus — when `baselineLabel` matches a variant — a
+ * two-proportion significance test of every other variant against it.
+ */
+export function buildSweepReport(
+  records: readonly IRunRecord[],
+  baselineLabel?: string
+): ISweepReport {
+  const summaries = summarize([...records]);
+  const baseline =
+    baselineLabel === undefined
+      ? undefined
+      : summaries.find((s) => s.label === baselineLabel);
+
+  const variants = summaries.map((summary) => {
+    const passRateCI = wilsonInterval(summary.passed, summary.runs);
+    const sameAsBaseline = baseline?.label === summary.label;
+
+    if (baseline === undefined || sameAsBaseline) {
+      return { ...summary, passRateCI };
+    }
+
+    return {
+      ...summary,
+      passRateCI,
+      vsBaseline: compareToBaseline(summary, baseline),
+    };
+  });
+
+  return { baseline: baseline?.label ?? null, variants };
+}
+
+function pct(value: number): string {
+  return `${Math.round(value * 100)}%`;
+}
+
+function baselineCell(report: IVariantReport, baseline: string | null): string {
+  if (baseline === null) {
+    return "—";
+  }
+
+  if (report.label === baseline) {
+    return "baseline";
+  }
+
+  const v = report.vsBaseline;
+
+  if (v === undefined) {
+    return "—";
+  }
+
+  const sign = v.deltaPassRate >= 0 ? "+" : "";
+  const mark = v.significant ? " *" : "";
+
+  return `${sign}${pct(v.deltaPassRate)} (z=${v.z.toFixed(2)})${mark}`;
+}
+
+/** Render a sweep report as a Markdown table. `*` marks a significant difference
+ *  (p < 0.05) from the baseline. */
+export function renderSweepReportMarkdown(report: ISweepReport): string {
+  const header =
+    "| Variant | Runs | Pass | 95% CI | Cycles | Ms | Quality | vs baseline |\n" +
+    "| --- | --- | --- | --- | --- | --- | --- | --- |";
+
+  const rows = report.variants.map((v) => {
+    const ci = `${pct(v.passRateCI[0])}–${pct(v.passRateCI[1])}`;
+
+    return (
+      `| ${v.label} | ${String(v.runs)} | ${pct(v.passRate)} | ${ci} | ` +
+      `${v.avgCycles.toFixed(1)} | ${String(Math.round(v.avgMs))} | ` +
+      `${v.avgQuality.toFixed(1)} | ${baselineCell(v, report.baseline)} |`
+    );
+  });
+
+  return [
+    "## A/B sweep report",
+    "",
+    header,
+    ...rows,
+    "",
+    "`*` = significant at p < 0.05 (two-proportion z-test vs baseline).",
+  ].join("\n");
+}