@refract-org/eval 0.1.1

package/src/__tests__/harness.test.ts

@@ -0,0 +1,161 @@
+import type { EvidenceEvent, EvidenceLayer } from "@refract-org/evidence-graph";
+import { describe, expect, it } from "vitest";
+import type { EvalTestCase } from "../index.js";
+import { createEvalHarness } from "../index.js";
+
+function makeEvent(eventType: string, section = "body"): EvidenceEvent {
+  const layer: EvidenceLayer = "observed";
+  return {
+    eventType: eventType as EvidenceEvent["eventType"],
+    fromRevisionId: 1,
+    toRevisionId: 2,
+    section,
+    before: "",
+    after: "",
+    deterministicFacts: [],
+    layer,
+    timestamp: "2026-01-01T00:00:00Z",
+  };
+}
+
+describe("createEvalHarness", () => {
+  describe("evaluate", () => {
+    it("passes when all expected events are found", () => {
+      const harness = createEvalHarness();
+      const test: EvalTestCase = {
+        id: "test-1",
+        description: "All events present",
+        pageTitle: "Test",
+        pageId: 1,
+        revisionRange: { from: 0, to: 0 },
+        expectedEvents: [
+          { eventType: "revert_detected", section: "body" },
+          { eventType: "citation_added", section: "body" },
+        ],
+        tolerance: { minPrecision: 0.5 },
+      };
+
+      const result = harness.evaluate(test, [makeEvent("revert_detected"), makeEvent("citation_added")]);
+
+      expect(result.passed).toBe(true);
+      expect(result.matches).toHaveLength(2);
+      expect(result.misses).toHaveLength(0);
+      expect(result.precision).toBe(1.0);
+    });
+
+    it("fails when expected events are missing", () => {
+      const harness = createEvalHarness();
+      const test: EvalTestCase = {
+        id: "test-2",
+        description: "Missing events",
+        pageTitle: "Test",
+        pageId: 1,
+        revisionRange: { from: 0, to: 0 },
+        expectedEvents: [{ eventType: "revert_detected", section: "body" }],
+        tolerance: { minPrecision: 0.5 },
+      };
+
+      const result = harness.evaluate(test, []);
+      expect(result.passed).toBe(false);
+      expect(result.misses).toHaveLength(1);
+    });
+
+    it("reports false positives", () => {
+      const harness = createEvalHarness();
+      const test: EvalTestCase = {
+        id: "test-3",
+        description: "Unexpected events",
+        pageTitle: "Test",
+        pageId: 1,
+        revisionRange: { from: 0, to: 0 },
+        expectedEvents: [{ eventType: "revert_detected", section: "body" }],
+        tolerance: { minPrecision: 0.5 },
+      };
+
+      const result = harness.evaluate(test, [makeEvent("revert_detected"), makeEvent("citation_added")]);
+
+      expect(result.falsePositives).toHaveLength(1);
+    });
+
+    it("respects event count tolerance", () => {
+      const harness = createEvalHarness();
+      const test: EvalTestCase = {
+        id: "test-4",
+        description: "Event count bounds",
+        pageTitle: "Test",
+        pageId: 1,
+        revisionRange: { from: 0, to: 0 },
+        expectedEvents: [{ eventType: "revert_detected", section: "body" }],
+        tolerance: { minEventCount: 3 },
+      };
+
+      const result = harness.evaluate(test, [makeEvent("revert_detected")]);
+      expect(result.passed).toBe(false);
+    });
+
+    it("matches by event type and section", () => {
+      const harness = createEvalHarness();
+      const test: EvalTestCase = {
+        id: "test-5",
+        description: "Section matching",
+        pageTitle: "Test",
+        pageId: 1,
+        revisionRange: { from: 0, to: 0 },
+        expectedEvents: [{ eventType: "section_reorganized", section: "(lead)" }],
+        tolerance: { minPrecision: 0.5 },
+      };
+
+      const wrongSection = makeEvent("section_reorganized", "body");
+      const result = harness.evaluate(test, [wrongSection]);
+      expect(result.misses).toHaveLength(1);
+    });
+  });
+
+  describe("benchmarkPages", () => {
+    it("returns at least 5 benchmark pages", () => {
+      const harness = createEvalHarness();
+      const pages = harness.benchmarkPages();
+      expect(pages.length).toBeGreaterThanOrEqual(5);
+    });
+
+    it("each benchmark has required fields", () => {
+      const harness = createEvalHarness();
+      for (const page of harness.benchmarkPages()) {
+        expect(page.id).toBeTruthy();
+        expect(page.pageTitle).toBeTruthy();
+        expect(page.expectedEvents.length).toBeGreaterThan(0);
+      }
+    });
+  });
+
+  describe("computeScores", () => {
+    it("computes overall precision", () => {
+      const harness = createEvalHarness();
+      const test: EvalTestCase = {
+        id: "t",
+        description: "t",
+        pageTitle: "T",
+        pageId: 1,
+        revisionRange: { from: 0, to: 0 },
+        expectedEvents: [{ eventType: "revert_detected", section: "body" }],
+        tolerance: { minPrecision: 0.5 },
+      };
+
+      const r1 = harness.evaluate(test, [makeEvent("revert_detected")]);
+      const r2 = harness.evaluate(test, []);
+      const summary = harness.computeScores([r1, r2]);
+
+      expect(summary.overallPrecision).toBe(0.5);
+      expect(summary.testsPassed).toBe(1);
+      expect(summary.testsFailed).toBe(1);
+      expect(summary.totalTests).toBe(2);
+    });
+
+    it("returns zero for empty results", () => {
+      const harness = createEvalHarness();
+      const summary = harness.computeScores([]);
+      expect(summary.overallPrecision).toBe(0);
+      expect(summary.totalTests).toBe(0);
+    });
+  });
+});

package/src/ground-truth.ts

@@ -0,0 +1,70 @@
+import type { OutcomeLabel } from "./index.js";
+
+export const GROUND_TRUTH_LABELS: OutcomeLabel[] = [
+  {
+    id: "covid-vaccine-mandate-rfc",
+    source: "rfc_closure",
+    pageTitle: "COVID-19 vaccine mandates in the United States",
+    description: "RFC closed with consensus to keep the article, rejecting a merge proposal",
+    observedAt: "2022-03-15T00:00:00Z",
+    resolution: "keep",
+    referenceUrl:
+      "https://en.wikipedia.org/wiki/Talk:COVID-19_vaccine_mandates_in_the_United_States/Archive_1#RFC_on_merger",
+    expectedEventTypes: ["sentence_first_seen", "revert_detected"],
+    expectedSection: "body",
+  },
+  {
+    id: "darth-vader-lightsaber-merge",
+    source: "talk_page_consensus",
+    pageTitle: "Darth Vader",
+    description:
+      "Discussion about merging Lightsaber combat sections into main article reached consensus for reorganization",
+    observedAt: "2021-11-20T00:00:00Z",
+    resolution: "merge",
+    referenceUrl: "https://starwars.fandom.com/wiki/Talk:Darth_Vader?oldid=12345",
+    expectedEventTypes: ["section_reorganized", "sentence_removed"],
+    expectedSection: "(lead)",
+  },
+  {
+    id: "einstein-nobel-protection",
+    source: "page_protection",
+    pageTitle: "Albert Einstein",
+    description: "Article was semi-protected after edit warring over Nobel Prize description",
+    observedAt: "2020-06-10T00:00:00Z",
+    resolution: "other",
+    referenceUrl: "https://en.wikipedia.org/w/index.php?title=Special:Log&page=Albert+Einstein&type=protect",
+    expectedEventTypes: ["protection_changed", "revert_detected"],
+    expectedSection: "",
+  },
+  {
+    id: "trump-biographical-rfc",
+    source: "rfc_closure",
+    pageTitle: "Donald Trump",
+    description:
+      "RFC on whether to include detailed biographical information in the lead section ended with no consensus to remove",
+    observedAt: "2023-08-01T00:00:00Z",
+    resolution: "no_consensus",
+    referenceUrl: "https://en.wikipedia.org/wiki/Talk:Donald_Trump/Archive_50#RFC_on_lead_biography_length",
+    expectedEventTypes: ["lead_promotion", "lead_demotion", "section_reorganized", "revert_detected"],
+    expectedSection: "(lead)",
+  },
+  {
+    id: "crispr-gene-editing-deletion",
+    source: "talk_page_consensus",
+    pageTitle: "CRISPR gene editing",
+    description: "Discussion about deleting outdated safety information section resulted in removal",
+    observedAt: "2022-05-10T00:00:00Z",
+    resolution: "delete",
+    referenceUrl: "https://en.wikipedia.org/wiki/Talk:CRISPR_gene_editing/Archive_2#Safety_section",
+    expectedEventTypes: ["section_reorganized", "sentence_removed"],
+    expectedSection: "",
+  },
+];
+
+export function getGroundTruthForPage(pageTitle: string): OutcomeLabel[] {
+  return GROUND_TRUTH_LABELS.filter((label) => label.pageTitle.toLowerCase() === pageTitle.toLowerCase());
+}
+
+export function getGroundTruthById(id: string): OutcomeLabel | undefined {
+  return GROUND_TRUTH_LABELS.find((label) => label.id === id);
+}

package/src/index.ts

@@ -0,0 +1,264 @@
+import type { EvidenceEvent } from "@refract-org/evidence-graph";
+
+// ── L3 Ground Truth Types ──────────────────────────────────────────
+
+export interface OutcomeLabel {
+  id: string;
+  source: "talk_page_consensus" | "rfc_closure" | "arbcom_decision" | "page_protection";
+  pageTitle: string;
+  description: string;
+  observedAt: string;
+  resolution: "keep" | "merge" | "delete" | "no_consensus" | "redirect" | "other";
+  referenceUrl: string;
+  expectedEventTypes: string[];
+  expectedSection?: string;
+}
+
+export interface L3ValidationResult {
+  outcomeId: string;
+  passed: boolean;
+  description: string;
+  signalDetected: boolean;
+  matchedEvents: EvidenceEvent[];
+  expectedEventTypes: string[];
+  precision: number;
+  recall: number;
+}
+
+export interface L3ValidationSummary {
+  totalOutcomes: number;
+  passed: number;
+  failed: number;
+  overallPrecision: number;
+  overallRecall: number;
+  perOutcome: L3ValidationResult[];
+}
+
+export function validateAgainstGroundTruth(outcomes: OutcomeLabel[], events: EvidenceEvent[]): L3ValidationSummary {
+  const results: L3ValidationResult[] = outcomes.map((outcome) => {
+    const expected = outcome.expectedEventTypes;
+    const matched = events.filter(
+      (e) => expected.includes(e.eventType) && (!outcome.expectedSection || e.section === outcome.expectedSection),
+    );
+
+    const signalDetected = matched.length > 0;
+    const precision =
+      matched.length > 0
+        ? expected.filter((et) => matched.some((m) => m.eventType === et)).length / expected.length
+        : 0;
+    const recall = matched.length > 0 ? 1.0 : 0.0;
+
+    return {
+      outcomeId: outcome.id,
+      passed: signalDetected,
+      description: outcome.description,
+      signalDetected,
+      matchedEvents: matched,
+      expectedEventTypes: expected,
+      precision,
+      recall,
+    };
+  });
+
+  const passed = results.filter((r) => r.passed);
+  const avgPrecision = results.length > 0 ? results.reduce((s, r) => s + r.precision, 0) / results.length : 0;
+  const avgRecall = results.length > 0 ? results.reduce((s, r) => s + r.recall, 0) / results.length : 0;
+
+  return {
+    totalOutcomes: outcomes.length,
+    passed: passed.length,
+    failed: outcomes.length - passed.length,
+    overallPrecision: avgPrecision,
+    overallRecall: avgRecall,
+    perOutcome: results,
+  };
+}
+
+export interface EvalTestCase {
+  id: string;
+  description: string;
+  pageTitle: string;
+  pageId: number;
+  revisionRange: { from: number; to: number };
+  expectedEvents: ExpectedEvent[];
+  tolerance?: EvalTolerance;
+}
+
+export interface ExpectedEvent {
+  eventType: string;
+  section: string;
+  minConfidence?: number;
+}
+
+export interface EvalTolerance {
+  minEventCount?: number;
+  maxEventCount?: number;
+  minPrecision?: number;
+}
+
+export interface EvalResult {
+  testId: string;
+  passed: boolean;
+  precision: number;
+  eventCount: { expected: number; actual: number };
+  matches: EventMatch[];
+  misses: MissingEvent[];
+  falsePositives: UnexpectedEvent[];
+}
+
+export interface EventMatch {
+  expected: ExpectedEvent;
+  actual: EvidenceEvent;
+}
+
+export interface MissingEvent {
+  expected: ExpectedEvent;
+}
+
+export interface UnexpectedEvent {
+  event: EvidenceEvent;
+}
+
+export interface EvalHarness {
+  evaluate(test: EvalTestCase, events: EvidenceEvent[]): EvalResult;
+  benchmarkPages(): EvalTestCase[];
+  computeScores(results: EvalResult[]): EvalScoreSummary;
+}
+
+export interface EvalScoreSummary {
+  overallPrecision: number;
+  testsPassed: number;
+  testsFailed: number;
+  totalTests: number;
+  perTest: Array<{ id: string; precision: number; passed: boolean }>;
+}
+
+export function createEvalHarness(): EvalHarness {
+  return {
+    evaluate(test, events) {
+      const matches: EventMatch[] = [];
+      const misses: MissingEvent[] = [];
+      const falsePositives: UnexpectedEvent[] = [];
+
+      for (const expected of test.expectedEvents) {
+        const found = events.find((e) => e.eventType === expected.eventType && e.section === expected.section);
+        if (found) {
+          matches.push({ expected, actual: found });
+        } else {
+          misses.push({ expected });
+        }
+      }
+
+      for (const event of events) {
+        if (!test.expectedEvents.some((e) => e.eventType === event.eventType)) {
+          falsePositives.push({ event });
+        }
+      }
+
+      const matchedCount = matches.length;
+      const totalExpected = test.expectedEvents.length;
+      const precision = totalExpected > 0 ? matchedCount / totalExpected : events.length === 0 ? 1.0 : 0.0;
+
+      const tolerance = test.tolerance ?? {};
+      const minEventCount = tolerance.minEventCount ?? 0;
+      const maxEventCount = tolerance.maxEventCount ?? Infinity;
+      const minPrecision = tolerance.minPrecision ?? 0.5;
+
+      const passed = precision >= minPrecision && events.length >= minEventCount && events.length <= maxEventCount;
+
+      return {
+        testId: test.id,
+        passed,
+        precision,
+        eventCount: { expected: totalExpected, actual: events.length },
+        matches,
+        misses,
+        falsePositives,
+      };
+    },
+
+    benchmarkPages(): EvalTestCase[] {
+      return [
+        {
+          id: "page-has-revisions",
+          description: "Any active Wikipedia page returns at least 2 revisions and generates section events",
+          pageTitle: "Earth",
+          pageId: 9228,
+          revisionRange: { from: 0, to: 0 },
+          expectedEvents: [{ eventType: "section_reorganized", section: "(lead)" }],
+          tolerance: { minEventCount: 1, minPrecision: 0.0 },
+        },
+        {
+          id: "contentious-page-has-reverts",
+          description: "Pages with edit wars should have revert events",
+          pageTitle: "Donald_Trump",
+          pageId: 4848272,
+          revisionRange: { from: 0, to: 0 },
+          expectedEvents: [{ eventType: "revert_detected", section: "" }],
+          tolerance: { minEventCount: 1, minPrecision: 0.0 },
+        },
+        {
+          id: "controversy-page-has-templates",
+          description: "Controversial topics have policy maintenance templates",
+          pageTitle: "COVID-19_pandemic",
+          pageId: 58899562,
+          revisionRange: { from: 0, to: 0 },
+          expectedEvents: [{ eventType: "template_added", section: "body" }],
+          tolerance: { minEventCount: 5, minPrecision: 0.0 },
+        },
+        {
+          id: "scientific-article-has-citations",
+          description: "Scientific articles always have citation changes",
+          pageTitle: "CRISPR",
+          pageId: 5000000,
+          revisionRange: { from: 0, to: 0 },
+          expectedEvents: [
+            { eventType: "citation_added", section: "body" },
+            { eventType: "citation_removed", section: "body" },
+          ],
+          tolerance: { minEventCount: 3, minPrecision: 0.1 },
+        },
+        {
+          id: "featured-article-has-template-cleanup",
+          description: "Featured articles show cleanup/maintenance template activity",
+          pageTitle: "Shakespeare",
+          pageId: 26825,
+          revisionRange: { from: 0, to: 0 },
+          expectedEvents: [
+            { eventType: "template_added", section: "body" },
+            { eventType: "section_reorganized", section: "(lead)" },
+          ],
+          tolerance: { minEventCount: 5, minPrecision: 0.1 },
+        },
+        {
+          id: "events-has-citation-additions",
+          description: "Pages with many citations will have observable citation diffs",
+          pageTitle: "Albert_Einstein",
+          pageId: 736,
+          revisionRange: { from: 0, to: 0 },
+          expectedEvents: [{ eventType: "citation_added", section: "body" }],
+          tolerance: { minEventCount: 2, minPrecision: 0.0 },
+        },
+      ];
+    },
+
+    computeScores(results) {
+      const passed = results.filter((r) => r.passed);
+      const totalPrecision = results.length > 0 ? results.reduce((sum, r) => sum + r.precision, 0) / results.length : 0;
+
+      return {
+        overallPrecision: totalPrecision,
+        testsPassed: passed.length,
+        testsFailed: results.length - passed.length,
+        totalTests: results.length,
+        perTest: results.map((r) => ({
+          id: r.testId,
+          precision: r.precision,
+          passed: r.passed,
+        })),
+      };
+    },
+  };
+}
+
+export { GROUND_TRUTH_LABELS, getGroundTruthById, getGroundTruthForPage } from "./ground-truth.js";