keating 0.3.6

package/dist/src/core/verification.js

@@ -0,0 +1,162 @@
+import { createHash } from "node:crypto";
+import { readFile, writeFile } from "node:fs/promises";
+import { loadEnv } from "./env.js";
+import { piComplete, piCompleteJson } from "./pi-agent.js";
+loadEnv();
+export function topicContentHash(topic) {
+    const payload = JSON.stringify({
+        slug: topic.slug,
+        summary: topic.summary,
+        formalCore: topic.formalCore,
+        misconceptions: topic.misconceptions,
+        examples: topic.examples
+    });
+    return createHash("sha256").update(payload).digest("hex").slice(0, 16);
+}
+export function extractClaims(topic) {
+    const claims = [];
+    claims.push(`Summary: ${topic.summary}`);
+    for (const statement of topic.formalCore) {
+        claims.push(`Formal claim: ${statement}`);
+    }
+    for (const misconception of topic.misconceptions) {
+        claims.push(`Misconception assertion: ${misconception}`);
+    }
+    for (const example of topic.examples) {
+        if (example.length > 20) {
+            claims.push(`Example claim: ${example}`);
+        }
+    }
+    return claims;
+}
+export function buildVerificationChecklist(topic, result) {
+    const claims = extractClaims(topic);
+    const lines = [
+        `# Verification Checklist: ${topic.title}`,
+        "",
+        `**Domain**: ${topic.domain}`,
+        `**Content hash**: ${topicContentHash(topic)}`,
+        `**Generated**: ${new Date().toISOString()}`,
+        `**Overall Confidence**: ${result ? (result.overallConfidence * 100).toFixed(1) : 0}%`,
+        "",
+        "## Instructions",
+        "",
+        "Before teaching this topic, verify each claim below. For each claim:",
+        "1. Search for an authoritative source (textbook, peer-reviewed paper, official documentation, legal text).",
+        "2. Mark the claim as CONFIRMED, UNCONFIRMED, or CORRECTED.",
+        "3. If corrected, write the accurate version.",
+        "4. Do NOT teach unverified claims as settled facts.",
+        "",
+        "## Claims to Verify",
+        ""
+    ];
+    for (let i = 0; i < claims.length; i++) {
+        const verified = result?.claims[i];
+        lines.push(`### Claim ${i + 1}`);
+        lines.push("");
+        lines.push(`> ${claims[i]}`);
+        lines.push("");
+        lines.push(`- [${verified?.status === "confirmed" ? "x" : " "}] **Status**: _${verified?.status?.toUpperCase() || "CONFIRMED / UNCONFIRMED / CORRECTED"}_`);
+        lines.push(`- **Source**: _${verified?.source || "[cite source here]"}_`);
+        lines.push(`- **Correction** (if any): _${verified?.correction || "[write corrected claim here]"}_`);
+        lines.push("");
+    }
+    lines.push("## Verification Summary");
+    lines.push("");
+    lines.push("- **Total claims**: " + claims.length);
+    lines.push(`- **Confirmed**: ${result?.claims.filter(c => c.status === "confirmed").length || 0}`);
+    lines.push(`- **Unconfirmed**: ${result?.claims.filter(c => c.status === "unconfirmed").length || 0}`);
+    lines.push(`- **Corrected**: ${result?.claims.filter(c => c.status === "corrected").length || 0}`);
+    lines.push(`- **Overall confidence**: ${result && result.overallConfidence > 0.8 ? "high" : result && result.overallConfidence > 0.5 ? "medium" : "low"}`);
+    lines.push("");
+    lines.push("---");
+    lines.push("*This checklist was auto-generated by Keating using Chain-of-Verification (CoVe) and cross-consistency checks. The teaching agent must complete it before presenting this material.*");
+    return lines.join("\n");
+}
+export async function loadVerificationCache(cachePath) {
+    try {
+        const raw = await readFile(cachePath, "utf8");
+        return JSON.parse(raw);
+    }
+    catch {
+        return {};
+    }
+}
+export async function saveVerificationCache(cachePath, cache) {
+    await writeFile(cachePath, JSON.stringify(cache, null, 2), "utf8");
+}
+export function verificationStatus(topic, cache) {
+    const cached = cache[topic.slug];
+    if (!cached)
+        return null;
+    if (cached.contentHash !== topicContentHash(topic))
+        return null;
+    return cached;
+}
+export function buildPendingVerificationResult(topic) {
+    const claims = extractClaims(topic);
+    return {
+        topic: topic.slug,
+        contentHash: topicContentHash(topic),
+        claims: claims.map((claim) => ({
+            claim,
+            status: "unconfirmed"
+        })),
+        overallConfidence: 0,
+        checkedAt: new Date().toISOString()
+    };
+}
+/**
+ * Automated Chain-of-Verification (CoVe) with Consistency Checking.
+ * Inspired by EdinburghNLP Hallucination Detection research.
+ * 1. Generates verification questions for each claim.
+ * 2. Answers those questions independently (the "Blackbox" cross-check).
+ * 3. Checks for consistency between independent answers and the claim.
+ */
+export async function runCoveVerification(cwd, topic) {
+    const claims = extractClaims(topic);
+    const verifiedClaims = [];
+    let totalConfidence = 0;
+    for (const claim of claims) {
+        // Phase 1: Question Generation
+        const qGenPrompt = `Given this claim, generate 3 specific, granular questions that would verify its factual accuracy.
+Claim: "${claim}"
+Questions (one per line):`;
+        const questionsText = await piComplete(cwd, qGenPrompt, { thinking: "low" });
+        const questions = questionsText.trim().split("\n").filter(q => q.trim());
+        // Phase 2: Independent Answering (the "Blackbox" cross-check)
+        const answers = [];
+        for (const q of questions) {
+            const a = await piComplete(cwd, `Answer this factual question accurately and concisely: ${q}`, { thinking: "medium" });
+            answers.push(a.trim());
+        }
+        // Phase 3: Consistency Analysis (Hallucination Detection)
+        const finalPrompt = `Original Claim: "${claim}"
+Supporting Research:
+${questions.map((q, i) => `Q: ${q}\nA: ${answers[i]}`).join("\n")}
+
+Is the original claim CONFIRMED, UNCONFIRMED, or CORRECTED based ONLY on the Supporting Research?
+Provide a JSON response:
+{
+  "status": "confirmed" | "unconfirmed" | "corrected",
+  "reasoning": "string",
+  "correction": "string | null",
+  "confidence": number (0 to 1)
+}`;
+        const data = await piCompleteJson(cwd, finalPrompt, { thinking: "high" });
+        verifiedClaims.push({
+            claim,
+            status: data.status,
+            source: data.reasoning,
+            correction: data.correction || undefined
+        });
+        totalConfidence += data.confidence;
+    }
+    return {
+        topic: topic.slug,
+        contentHash: topicContentHash(topic),
+        claims: verifiedClaims,
+        overallConfidence: totalConfidence / claims.length,
+        checkedAt: new Date().toISOString()
+    };
+}