@dpeek/codeless 0.1.0 → 0.1.2

package/src/metrics.ts

@@ -5,17 +5,21 @@ import {
   readFileSync,
   readdirSync,
   renameSync,
+  rmdirSync,
   unlinkSync,
   writeFileSync,
 } from "node:fs";
 import { dirname, join } from "node:path";
 
+import { type Attempt, validAttempt } from "./attempt.ts";
+
 export type Metric = {
   stream: string;
   change: string;
   dispatchedAt?: string;
   landedAt?: string;
   landedCommit?: string;
+  attempts?: Record<string, Attempt>;
 };
 
 function metricPath(workspaceRoot: string, stream: string, change: string): string {
@@ -32,7 +36,16 @@ function readMetric(path: string): Metric {
     typeof metric["change"] !== "string" ||
     (metric["dispatchedAt"] !== undefined && typeof metric["dispatchedAt"] !== "string") ||
     (metric["landedAt"] !== undefined && typeof metric["landedAt"] !== "string") ||
-    (metric["landedCommit"] !== undefined && typeof metric["landedCommit"] !== "string")
+    (metric["landedCommit"] !== undefined && typeof metric["landedCommit"] !== "string") ||
+    (metric["attempts"] !== undefined &&
+      (typeof metric["attempts"] !== "object" ||
+        metric["attempts"] === null ||
+        Array.isArray(metric["attempts"]) ||
+        !Object.entries(metric["attempts"] as Record<string, unknown>).every(
+          ([id, attempt]) =>
+            validAttempt(attempt, metric["stream"] as string, metric["change"] as string) &&
+            attempt.id === id,
+        )))
   )
     throw new Error(`${path} is not a metric record`);
   return metric as Metric;
@@ -73,6 +86,50 @@ export function recordDispatch(workspaceRoot: string, stream: string, change: st
   readMetric(path);
 }
 
+function withMetricLock(path: string, action: () => void): void {
+  const lock = `${path}.lock`;
+  mkdirSync(dirname(path), { recursive: true });
+  for (let attempt = 0; attempt < 100; attempt += 1) {
+    try {
+      mkdirSync(lock);
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
+      Bun.sleepSync(10);
+      continue;
+    }
+    try {
+      action();
+    } finally {
+      rmdirSync(lock);
+    }
+    return;
+  }
+  throw new Error(`could not acquire metric lock ${lock}`);
+}
+
+export function recordAttempt(
+  workspaceRoot: string,
+  stream: string,
+  change: string,
+  attempt: Attempt,
+): void {
+  if (!validAttempt(attempt, stream, change))
+    throw new Error("attempt is not a valid metric attempt");
+  const path = metricPath(workspaceRoot, stream, change);
+  withMetricLock(path, () => {
+    const metric = existsSync(path) ? readMetric(path) : { stream, change };
+    if (metric.stream !== stream || metric.change !== change)
+      throw new Error(`${path} does not match ${stream} change ${change}`);
+    const existing = metric.attempts?.[attempt.id];
+    if (existing !== undefined) {
+      if (JSON.stringify(existing) !== JSON.stringify(attempt))
+        throw new Error(`attempt ${attempt.id} conflicts with its existing metric record`);
+      return;
+    }
+    writeMetric(path, { ...metric, attempts: { ...metric.attempts, [attempt.id]: attempt } });
+  });
+}
+
 export function recordLanding(
   workspaceRoot: string,
   stream: string,
@@ -144,12 +201,98 @@ export function metricReport(workspaceRoot: string): string[] {
     return `${name}\t${value.landed}\t${value.unlanded}\t${value.coverage}\t${value.total}\t${value.average}`;
   };
   const rows = [...byStream.entries()].map(([stream, records]) => ({ stream, records }));
+  const allRecords = rows.flatMap(({ records }) => records);
+  const formatCost = (amounts: number[]) => {
+    const parts = amounts.map((amount) => {
+      const [coefficient, exponent = "0"] = String(amount).toLowerCase().split("e");
+      const [whole, fraction = ""] = coefficient!.split(".");
+      return { digits: BigInt(`${whole}${fraction}`), scale: fraction.length - Number(exponent) };
+    });
+    const scale = Math.max(0, ...parts.map((part) => part.scale));
+    const total = parts.reduce(
+      (sum, part) => sum + part.digits * 10n ** BigInt(scale - part.scale),
+      0n,
+    );
+    const digits = total.toString().padStart(scale + 1, "0");
+    if (scale === 0) return digits;
+    const fraction = digits.slice(-scale).replace(/0+$/, "");
+    return fraction.length === 0
+      ? digits.slice(0, -scale)
+      : `${digits.slice(0, -scale)}.${fraction}`;
+  };
+  const attemptSummary = (records: Metric[]) => {
+    const attempts = records.flatMap((record) => Object.values(record.attempts ?? {}));
+    const usage = attempts.filter((attempt) => attempt.usage !== undefined);
+    const costs = attempts.filter((attempt) => attempt.cost !== undefined);
+    const outcomes = new Map<string, number>();
+    const currencies = new Map<string, number[]>();
+    for (const attempt of attempts) {
+      outcomes.set(attempt.outcome, (outcomes.get(attempt.outcome) ?? 0) + 1);
+      if (attempt.cost !== undefined)
+        currencies.set(attempt.cost.currency, [
+          ...(currencies.get(attempt.cost.currency) ?? []),
+          attempt.cost.amount,
+        ]);
+    }
+    const coverage = (measured: number) =>
+      `${measured} measured, ${attempts.length - measured} unavailable`;
+    return {
+      reworkedChanges: new Set(
+        records
+          .filter((record) =>
+            Object.values(record.attempts ?? {}).some((attempt) => attempt.kind === "rework"),
+          )
+          .map((record) => `${record.stream}\0${record.change}`),
+      ).size,
+      initial: attempts.filter((attempt) => attempt.kind === "initial").length,
+      rework: attempts.filter((attempt) => attempt.kind === "rework").length,
+      incomplete: attempts.filter((attempt) => attempt.incomplete).length,
+      outcomes:
+        outcomes.size === 0
+          ? "none"
+          : [...outcomes.entries()]
+              .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
+              .map(([outcome, count]) => `${outcome}: ${count}`)
+              .join(", "),
+      toolErrors: attempts.reduce((total, attempt) => total + attempt.errorCount, 0),
+      usageCoverage: coverage(usage.length),
+      input:
+        usage.length === 0
+          ? "unavailable"
+          : usage.reduce((total, attempt) => total + attempt.usage!.input, 0),
+      output:
+        usage.length === 0
+          ? "unavailable"
+          : usage.reduce((total, attempt) => total + attempt.usage!.output, 0),
+      cacheRead:
+        usage.length === 0
+          ? "unavailable"
+          : usage.reduce((total, attempt) => total + attempt.usage!.cacheRead, 0),
+      cacheWrite:
+        usage.length === 0
+          ? "unavailable"
+          : usage.reduce((total, attempt) => total + attempt.usage!.cacheWrite, 0),
+      costCoverage: coverage(costs.length),
+      costs:
+        currencies.size === 0
+          ? "unavailable"
+          : [...currencies.entries()]
+              .sort(([left], [right]) => (left < right ? -1 : left > right ? 1 : 0))
+              .map(([currency, amounts]) => `${currency} ${formatCost(amounts)}`)
+              .join(", "),
+    };
+  };
+  const attemptRow = (name: string, records: Metric[]) => {
+    const value = attemptSummary(records);
+    return `${name}\t${value.reworkedChanges}\t${value.initial}\t${value.rework}\t${value.incomplete}\t${value.outcomes}\t${value.toolErrors}\t${value.usageCoverage}\t${value.input}\t${value.output}\t${value.cacheRead}\t${value.cacheWrite}\t${value.costCoverage}\t${value.costs}`;
+  };
   return [
     "Stream\tLanded\tNot landed\tElapsed coverage\tDispatch-to-land wall clock total\tAverage",
     ...rows.map(({ stream, records }) => row(stream, records)),
-    row(
-      "Project total",
-      rows.flatMap(({ records }) => records),
-    ),
+    row("Project total", allRecords),
+    "",
+    "Stream\tChanges with rework\tInitial attempts\tRework attempts\tIncomplete collection\tTerminal outcomes\tTool errors\tUsage coverage\tInput tokens\tOutput tokens\tCache-read tokens\tCache-write tokens\tCost coverage\tCost totals",
+    ...rows.map(({ stream, records }) => attemptRow(stream, records)),
+    attemptRow("Project total", allRecords),
   ];
 }