ennodia 0.3.0 → 0.4.1

package/bench/bug-recall/scorer.ts

@@ -1,424 +0,0 @@
-import { createHash } from "node:crypto";
-import { readdir } from "node:fs/promises";
-import { fileURLToPath } from "node:url";
-
-export type Severity = "low" | "medium" | "high";
-
-export type RequiredFinding = {
-  id: string;
-  severity: Severity;
-  line: number;
-  description: string;
-  keywords: string[];
-};
-
-export type FalsePositiveTrap = {
-  id: string;
-  description: string;
-  terms: string[];
-};
-
-export type Oracle = {
-  id: string;
-  title: string;
-  prompt: string;
-  requiredFindings: RequiredFinding[];
-  knownFalsePositives: FalsePositiveTrap[];
-};
-
-export type BenchmarkCase = {
-  id: string;
-  title: string;
-  inputPath: string;
-  input: string;
-  oracle: Oracle;
-};
-
-export type OutputRecord = {
-  caseId: string;
-  condition: string;
-  text: string;
-  metadata?: Record<string, unknown>;
-};
-
-export type CaseScore = {
-  caseId: string;
-  condition: string;
-  matchedFindingIds: string[];
-  missedFindingIds: string[];
-  falsePositiveTrapIds: string[];
-  claimedFindingCount: number;
-  requiredFindingCount: number;
-  highFindingCount: number;
-  highMatchedCount: number;
-  recall: number;
-  precision: number;
-  f1: number;
-  highRecall: number;
-  outputChars: number;
-  metadata?: Record<string, unknown>;
-};
-
-export type ConditionSummary = {
-  condition: string;
-  cases: number;
-  matchedFindings: number;
-  requiredFindings: number;
-  falsePositiveTraps: number;
-  claimedFindings: number;
-  highMatchedFindings: number;
-  highRequiredFindings: number;
-  recall: number;
-  precision: number;
-  f1: number;
-  highRecall: number;
-};
-
-export type BenchmarkSummary = {
-  benchmark: "multi-model-bug-recall";
-  mode: "fixture" | "live";
-  createdAt: string;
-  scorerHash: string;
-  caseScores: CaseScore[];
-  conditions: ConditionSummary[];
-  wins: {
-    condition: string;
-    fixtures: number;
-  }[];
-};
-
-export const DEFAULT_FIXTURES_ROOT = new URL(
-  "../fixtures/bug-recall/",
-  import.meta.url,
-);
-
-export async function loadCases(
-  fixturesRoot = DEFAULT_FIXTURES_ROOT,
-): Promise<BenchmarkCase[]> {
-  const entries = await readdir(fixturesRoot, { withFileTypes: true });
-  const caseDirs = entries
-    .filter((entry) => entry.isDirectory())
-    .map((entry) => entry.name)
-    .sort();
-
-  const cases: BenchmarkCase[] = [];
-
-  for (const caseId of caseDirs) {
-    const caseUrl = new URL(`${caseId}/`, fixturesRoot);
-    const inputUrl = new URL("input.ts", caseUrl);
-    const oracleUrl = new URL("oracle.json", caseUrl);
-    const input = await Bun.file(inputUrl).text();
-    const oracle = await Bun.file(oracleUrl).json() as Oracle;
-    validateOracle(caseId, oracle);
-
-    cases.push({
-      id: oracle.id,
-      title: oracle.title,
-      inputPath: inputUrl.pathname,
-      input,
-      oracle,
-    });
-  }
-
-  return cases;
-}
-
-export async function scoreBenchmark(input: {
-  cases: BenchmarkCase[];
-  outputs: OutputRecord[];
-  mode: "fixture" | "live";
-}): Promise<BenchmarkSummary> {
-  const caseScores = input.outputs.map((output) => {
-    const benchmarkCase = input.cases.find((candidate) =>
-      candidate.id === output.caseId
-    );
-    if (!benchmarkCase) {
-      throw new Error(`Unknown benchmark case: ${output.caseId}`);
-    }
-
-    return scoreCase(benchmarkCase.oracle, output);
-  });
-
-  return {
-    benchmark: "multi-model-bug-recall",
-    mode: input.mode,
-    createdAt: new Date().toISOString(),
-    scorerHash: await hashScorerSource(),
-    caseScores,
-    conditions: summarizeConditions(caseScores),
-    wins: summarizeWins(input.cases, caseScores),
-  };
-}
-
-export function scoreCase(oracle: Oracle, output: OutputRecord): CaseScore {
-  const claims = extractClaims(output.text);
-  const normalizedOutput = normalize(output.text);
-  const claimTexts = claims.length > 0
-    ? claims.map((claim) => normalize(claim))
-    : splitFallbackClaims(output.text).map((claim) => normalize(claim));
-  const matchTexts = claimTexts.length > 0 ? claimTexts : [normalizedOutput];
-  const matchedFindingIds = oracle.requiredFindings
-    .filter((finding) => matchesFinding(finding, matchTexts))
-    .map((finding) => finding.id);
-  const missedFindingIds = oracle.requiredFindings
-    .filter((finding) => !matchedFindingIds.includes(finding.id))
-    .map((finding) => finding.id);
-  const falsePositiveTrapIds = oracle.knownFalsePositives
-    .filter((trap) => matchesTerms(trap.terms, [normalizedOutput]))
-    .map((trap) => trap.id);
-  const highFindings = oracle.requiredFindings.filter((finding) =>
-    finding.severity === "high"
-  );
-  const highMatchedCount = highFindings.filter((finding) =>
-    matchedFindingIds.includes(finding.id)
-  ).length;
-  const claimedFindingCount = Math.max(
-    claimTexts.length,
-    matchedFindingIds.length + falsePositiveTrapIds.length,
-  );
-  const precision = safeRatio(matchedFindingIds.length, claimedFindingCount);
-  const recall = safeRatio(
-    matchedFindingIds.length,
-    oracle.requiredFindings.length,
-  );
-
-  return {
-    caseId: oracle.id,
-    condition: output.condition,
-    matchedFindingIds,
-    missedFindingIds,
-    falsePositiveTrapIds,
-    claimedFindingCount,
-    requiredFindingCount: oracle.requiredFindings.length,
-    highFindingCount: highFindings.length,
-    highMatchedCount,
-    recall,
-    precision,
-    f1: f1(precision, recall),
-    highRecall: safeRatio(highMatchedCount, highFindings.length),
-    outputChars: output.text.length,
-    metadata: output.metadata,
-  };
-}
-
-export function formatConditionTable(summary: BenchmarkSummary): string {
-  const rows = [
-    "| Condition | Cases | Recall | Precision | F1 | High recall | FP traps |",
-    "|---|---:|---:|---:|---:|---:|---:|",
-    ...summary.conditions.map((condition) =>
-      `| ${condition.condition} | ${condition.cases} | ${
-        formatPercent(condition.recall)
-      } | ${formatPercent(condition.precision)} | ${formatPercent(condition.f1)} | ${
-        formatPercent(condition.highRecall)
-      } | ${condition.falsePositiveTraps} |`
-    ),
-  ];
-
-  return rows.join("\n");
-}
-
-export function buildPrompt(benchmarkCase: BenchmarkCase): string {
-  return [
-    "You are reviewing one TypeScript fixture for correctness bugs.",
-    "Return JSON only with this exact shape:",
-    '{"findings":[{"line":12,"summary":"short bug description"}]}',
-    "List real correctness, lifecycle, async, cancellation, schema, or boundary bugs.",
-    "Do not include style comments or refactors.",
-    "",
-    `Fixture: ${benchmarkCase.id} - ${benchmarkCase.title}`,
-    "```ts",
-    benchmarkCase.input.trim(),
-    "```",
-  ].join("\n");
-}
-
-function validateOracle(caseId: string, oracle: Oracle): void {
-  if (oracle.id !== caseId) {
-    throw new Error(`Oracle id ${oracle.id} does not match directory ${caseId}.`);
-  }
-
-  for (const finding of oracle.requiredFindings) {
-    if (finding.keywords.length === 0) {
-      throw new Error(`${caseId}.${finding.id} must declare keywords.`);
-    }
-  }
-}
-
-function summarizeConditions(scores: CaseScore[]): ConditionSummary[] {
-  const byCondition = new Map<string, CaseScore[]>();
-  for (const score of scores) {
-    const group = byCondition.get(score.condition) ?? [];
-    group.push(score);
-    byCondition.set(score.condition, group);
-  }
-
-  return [...byCondition.entries()]
-    .sort(([a], [b]) => a.localeCompare(b))
-    .map(([condition, group]) => {
-      const matchedFindings = sum(group, "matchedFindingIds");
-      const requiredFindings = sum(group, "requiredFindingCount");
-      const claimedFindings = sum(group, "claimedFindingCount");
-      const highMatchedFindings = sum(group, "highMatchedCount");
-      const highRequiredFindings = sum(group, "highFindingCount");
-      const falsePositiveTraps = sum(group, "falsePositiveTrapIds");
-      const precision = safeRatio(matchedFindings, claimedFindings);
-      const recall = safeRatio(matchedFindings, requiredFindings);
-
-      return {
-        condition,
-        cases: group.length,
-        matchedFindings,
-        requiredFindings,
-        falsePositiveTraps,
-        claimedFindings,
-        highMatchedFindings,
-        highRequiredFindings,
-        recall,
-        precision,
-        f1: f1(precision, recall),
-        highRecall: safeRatio(highMatchedFindings, highRequiredFindings),
-      };
-    });
-}
-
-function summarizeWins(cases: BenchmarkCase[], scores: CaseScore[]) {
-  const wins = new Map<string, number>();
-
-  for (const benchmarkCase of cases) {
-    const caseScores = scores.filter((score) => score.caseId === benchmarkCase.id);
-    const best = caseScores.reduce<CaseScore | undefined>((winner, score) => {
-      if (!winner) {
-        return score;
-      }
-
-      if (score.f1 > winner.f1) {
-        return score;
-      }
-
-      if (
-        score.f1 === winner.f1 &&
-        score.highRecall > winner.highRecall
-      ) {
-        return score;
-      }
-
-      if (
-        score.f1 === winner.f1 &&
-        score.highRecall === winner.highRecall &&
-        score.falsePositiveTrapIds.length < winner.falsePositiveTrapIds.length
-      ) {
-        return score;
-      }
-
-      return winner;
-    }, undefined);
-
-    if (best) {
-      wins.set(best.condition, (wins.get(best.condition) ?? 0) + 1);
-    }
-  }
-
-  return [...wins.entries()]
-    .sort(([a], [b]) => a.localeCompare(b))
-    .map(([condition, fixtures]) => ({ condition, fixtures }));
-}
-
-function extractClaims(text: string): string[] {
-  const json = parseJsonPayload(text);
-  if (!json || typeof json !== "object") {
-    return [];
-  }
-
-  const findings = (json as { findings?: unknown }).findings;
-  if (!Array.isArray(findings)) {
-    return [];
-  }
-
-  return findings
-    .map((finding) => {
-      if (!finding || typeof finding !== "object") {
-        return "";
-      }
-
-      const line = (finding as { line?: unknown }).line;
-      const summary = (finding as { summary?: unknown }).summary;
-      return `${typeof line === "number" ? `line ${line}` : ""} ${
-        typeof summary === "string" ? summary : ""
-      }`.trim();
-    })
-    .filter(Boolean);
-}
-
-function parseJsonPayload(text: string): unknown {
-  const fenced = text.match(/```(?:json)?\s*([\s\S]*?)```/i)?.[1];
-  const candidate = fenced ?? text.trim();
-  const first = candidate.indexOf("{");
-  const last = candidate.lastIndexOf("}");
-
-  if (first < 0 || last < first) {
-    return undefined;
-  }
-
-  try {
-    return JSON.parse(candidate.slice(first, last + 1));
-  } catch {
-    return undefined;
-  }
-}
-
-function splitFallbackClaims(text: string): string[] {
-  return text
-    .split(/\r?\n/)
-    .map((line) => line.trim())
-    .filter((line) =>
-      /^[-*]\s+/.test(line) ||
-      /^\d+[.)]\s+/.test(line) ||
-      /\bline\s+\d+\b/i.test(line)
-    );
-}
-
-function matchesFinding(finding: RequiredFinding, texts: string[]): boolean {
-  return matchesTerms(finding.keywords, texts);
-}
-
-function matchesTerms(terms: string[], texts: string[]): boolean {
-  return texts.some((text) =>
-    terms.every((term) => text.includes(normalize(term)))
-  );
-}
-
-function normalize(value: string): string {
-  return value.toLowerCase().replace(/[_-]/g, " ").replace(/\s+/g, " ").trim();
-}
-
-function safeRatio(numerator: number, denominator: number): number {
-  return denominator === 0 ? 0 : numerator / denominator;
-}
-
-function f1(precision: number, recall: number): number {
-  return precision + recall === 0 ? 0 : (2 * precision * recall) / (precision + recall);
-}
-
-function formatPercent(value: number): string {
-  return `${Math.round(value * 1000) / 10}%`;
-}
-
-function sum(group: CaseScore[], key: keyof CaseScore): number {
-  return group.reduce((total, score) => {
-    const value = score[key];
-    if (Array.isArray(value)) {
-      return total + value.length;
-    }
-
-    if (typeof value === "number") {
-      return total + value;
-    }
-
-    return total;
-  }, 0);
-}
-
-async function hashScorerSource(): Promise<string> {
-  const source = await Bun.file(fileURLToPath(import.meta.url)).text();
-  return createHash("sha256").update(source).digest("hex");
-}

package/bench/fixtures/bug-recall/001-missing-await/input.ts

@@ -1,33 +0,0 @@
-type User = {
-  id: string;
-  name: string;
-  email: string;
-  active: boolean;
-};
-
-type Mailer = {
-  send(to: string, body: string): Promise<void>;
-};
-
-async function fetchUser(userId: string): Promise<User> {
-  return {
-    id: userId,
-    name: "Ada",
-    email: "ada@example.com",
-    active: true,
-  };
-}
-
-export async function sendWelcomeEmail(
-  userId: string,
-  mailer: Mailer,
-): Promise<"sent" | "skipped"> {
-  const user = fetchUser(userId);
-
-  if (!user.active) {
-    return "skipped";
-  }
-
-  await mailer.send(user.email, `Welcome ${user.name}`);
-  return "sent";
-}

package/bench/fixtures/bug-recall/001-missing-await/oracle.json

@@ -1,28 +0,0 @@
-{
-  "id": "001-missing-await",
-  "title": "Missing await before user lookup",
-  "prompt": "Review the TypeScript fixture and list correctness bugs.",
-  "requiredFindings": [
-    {
-      "id": "missing-await-fetch-user",
-      "severity": "high",
-      "line": 24,
-      "description": "fetchUser returns a Promise but sendWelcomeEmail reads active, email, and name without awaiting it.",
-      "keywords": ["await", "fetchUser"]
-    },
-    {
-      "id": "promise-property-access",
-      "severity": "medium",
-      "line": 26,
-      "description": "The branch reads user.active on a Promise, so the inactive-user skip path is broken.",
-      "keywords": ["active", "Promise"]
-    }
-  ],
-  "knownFalsePositives": [
-    {
-      "id": "mailer-not-awaited",
-      "description": "Claims mailer.send is not awaited even though it is awaited.",
-      "terms": ["mailer.send", "not awaited"]
-    }
-  ]
-}

package/bench/fixtures/bug-recall/001-missing-await/responses/claude-code-solo.md

@@ -1 +0,0 @@
-{"findings":[{"line":24,"summary":"fetchUser returns a Promise and must be awaited before reading active/email/name."},{"line":26,"summary":"Because user is still a Promise, user.active never checks the loaded user's active flag."}]}

package/bench/fixtures/bug-recall/001-missing-await/responses/codex-solo.md

@@ -1 +0,0 @@
-{"findings":[{"line":24,"summary":"Missing await before fetchUser means user is a Promise."},{"line":29,"summary":"mailer.send is not awaited."}]}

package/bench/fixtures/bug-recall/001-missing-await/responses/ennodia-parallel-compare.md

@@ -1 +0,0 @@
-{"findings":[{"line":24,"summary":"Missing await before fetchUser leaves user as a Promise."},{"line":26,"summary":"The active check reads active on a Promise instead of the loaded User, so inactive users are not skipped."}]}

package/bench/fixtures/bug-recall/002-timeout-drain/input.ts

@@ -1,40 +0,0 @@
-type ChildProcess = {
-  stdout: AsyncIterable<Uint8Array>;
-  stderr: AsyncIterable<Uint8Array>;
-  exited: Promise<number>;
-  kill(): void;
-};
-
-type Result = {
-  exitCode: number;
-  stdout: string;
-  stderr: string;
-};
-
-const decoder = new TextDecoder();
-
-export async function collectChild(
-  child: ChildProcess,
-  timeoutMs: number,
-): Promise<Result> {
-  let stdout = "";
-  let stderr = "";
-
-  void (async () => {
-    for await (const chunk of child.stdout) {
-      stdout += decoder.decode(chunk);
-    }
-  })();
-
-  void (async () => {
-    for await (const chunk of child.stderr) {
-      stderr += decoder.decode(chunk);
-    }
-  })();
-
-  const timeout = setTimeout(() => child.kill(), timeoutMs);
-  const exitCode = await child.exited;
-  clearTimeout(timeout);
-
-  return { exitCode, stdout, stderr };
-}

package/bench/fixtures/bug-recall/002-timeout-drain/oracle.json

@@ -1,28 +0,0 @@
-{
-  "id": "002-timeout-drain",
-  "title": "Exit observed before output streams drain",
-  "prompt": "Review the TypeScript fixture and list correctness bugs.",
-  "requiredFindings": [
-    {
-      "id": "returns-before-stream-drain",
-      "severity": "high",
-      "line": 35,
-      "description": "collectChild returns after child.exited without awaiting the stdout and stderr reader loops, so captured output can be truncated.",
-      "keywords": ["await", "stdout", "stderr"]
-    },
-    {
-      "id": "timeout-resolves-as-normal-exit",
-      "severity": "medium",
-      "line": 31,
-      "description": "The timeout kills the child but the result does not record that the process timed out.",
-      "keywords": ["timeout", "kill", "timed out"]
-    }
-  ],
-  "knownFalsePositives": [
-    {
-      "id": "leaks-timeout-on-success",
-      "description": "Claims the timeout is never cleared even though clearTimeout is called.",
-      "terms": ["timeout", "never cleared"]
-    }
-  ]
-}

package/bench/fixtures/bug-recall/002-timeout-drain/responses/claude-code-solo.md

@@ -1 +0,0 @@
-{"findings":[{"line":35,"summary":"The function awaits child.exited but never awaits the stdout and stderr reader loops, so output can be returned before it drains."},{"line":31,"summary":"When the timeout fires and kill is called, the returned Result does not mark the process as timed out."}]}

package/bench/fixtures/bug-recall/002-timeout-drain/responses/codex-solo.md

@@ -1 +0,0 @@
-{"findings":[{"line":35,"summary":"Returns after child.exited without awaiting stdout and stderr collection."}]}

package/bench/fixtures/bug-recall/002-timeout-drain/responses/ennodia-parallel-compare.md

@@ -1 +0,0 @@
-{"findings":[{"line":35,"summary":"collectChild must await both stdout and stderr drain loops before returning, otherwise output can be truncated."},{"line":31,"summary":"A timeout only calls kill; the result does not say the process timed out."}]}

package/bench/fixtures/bug-recall/003-schema-contract/input.ts

@@ -1,22 +0,0 @@
-import { z } from "zod";
-
-const ListTasksInput = z.object({
-  includeEvents: z.boolean().default(false),
-  maxEvents: z.number().int().nonnegative().default(25),
-});
-
-type Event = {
-  at: string;
-  message: string;
-};
-
-export function selectEvents(events: Event[], rawInput: unknown): Event[] {
-  const input = ListTasksInput.parse(rawInput);
-
-  if (!input.includeEvents) {
-    return [];
-  }
-
-  const maxEvents = input.maxEvents || 25;
-  return events.slice(-maxEvents);
-}

package/bench/fixtures/bug-recall/003-schema-contract/oracle.json

@@ -1,28 +0,0 @@
-{
-  "id": "003-schema-contract",
-  "title": "Boundary value lost by defaulting",
-  "prompt": "Review the TypeScript fixture and list correctness bugs.",
-  "requiredFindings": [
-    {
-      "id": "zero-max-events-ignored",
-      "severity": "high",
-      "line": 19,
-      "description": "maxEvents: 0 is valid in the schema, but input.maxEvents || 25 turns it into 25.",
-      "keywords": ["maxEvents", "0"]
-    },
-    {
-      "id": "negative-slice-with-default",
-      "severity": "low",
-      "line": 20,
-      "description": "The code relies on slice(-maxEvents), so the zero case should be handled explicitly to return no events.",
-      "keywords": ["slice", "0"]
-    }
-  ],
-  "knownFalsePositives": [
-    {
-      "id": "schema-allows-negative",
-      "description": "Claims the schema allows negative maxEvents even though nonnegative is used.",
-      "terms": ["negative", "maxEvents", "allowed"]
-    }
-  ]
-}

package/bench/fixtures/bug-recall/003-schema-contract/responses/claude-code-solo.md

@@ -1 +0,0 @@
-{"findings":[{"line":19,"summary":"maxEvents 0 is valid but input.maxEvents || 25 replaces it with the default."},{"line":20,"summary":"The slice call needs an explicit 0 case so maxEvents 0 returns no events."}]}

package/bench/fixtures/bug-recall/003-schema-contract/responses/codex-solo.md

@@ -1 +0,0 @@
-{"findings":[{"line":19,"summary":"maxEvents 0 is ignored because || 25 treats it as false."},{"line":4,"summary":"The schema allows negative maxEvents values."}]}

package/bench/fixtures/bug-recall/003-schema-contract/responses/ennodia-parallel-compare.md

@@ -1 +0,0 @@
-{"findings":[{"line":19,"summary":"Using input.maxEvents || 25 loses the valid maxEvents 0 boundary value."},{"line":20,"summary":"slice(-maxEvents) should handle 0 explicitly and return an empty event list."}]}

package/bench/fixtures/bug-recall/004-cancel-propagation/input.ts

@@ -1,28 +0,0 @@
-type Task = {
-  id: string;
-  status: "running" | "succeeded" | "failed" | "cancelled";
-  cancel(): void;
-};
-
-type Run = {
-  id: string;
-  status: "running" | "cancelled";
-  taskIds: string[];
-};
-
-export function cancelRun(run: Run, tasks: Map<string, Task>): Run {
-  if (run.status !== "running") {
-    return run;
-  }
-
-  run.status = "cancelled";
-
-  for (const taskId of run.taskIds) {
-    const task = tasks.get(taskId);
-    if (!task || task.status !== "running") {
-      continue;
-    }
-  }
-
-  return run;
-}