@wanshi-kg/wanshi 0.1.0

package/dist/evaluation/mine/MineDataset.js

@@ -0,0 +1,208 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MineDataset = void 0;
+const fs = __importStar(require("fs"));
+const readline = __importStar(require("readline"));
+/**
+ * Alignment guard threshold. The third-party HF mirror
+ * (josancamon/kg-gen-MINE-evaluation-dataset) ships baseline-graph columns that
+ * are DESYNCED from the essays past ~row 18 (verified against the live API: row 19
+ * is a VR essay carrying a board-games kggen graph). A misaligned baseline scores
+ * ~0 on every fact and silently poisons the four-way table. Guard: a baseline whose
+ * entities barely appear in its own essay belongs to a different article — drop it
+ * so it's excluded from scoring rather than counted as a (fake) zero. Aligned rows
+ * overlap ≥0.4; misaligned ~0.05, so 0.25 cleanly separates with margin.
+ */
+const MIN_BASELINE_ALIGNMENT = 0.25;
+class MineDataset {
+    load(dataPath, limit, logger) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            if (!fs.existsSync(dataPath)) {
+                throw new Error(`MINE data not found at: ${dataPath}\n` +
+                    `Fetch it first:  npx ts-node scripts/fetch-mine.ts\n` +
+                    `(downloads josancamon/kg-gen-MINE-evaluation-dataset → data/mine/mine.jsonl)`);
+            }
+            const samples = [];
+            const stream = fs.createReadStream(dataPath, { encoding: 'utf-8' });
+            const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+            try {
+                for (var _d = true, rl_1 = __asyncValues(rl), rl_1_1; rl_1_1 = yield rl_1.next(), _a = rl_1_1.done, !_a; _d = true) {
+                    _c = rl_1_1.value;
+                    _d = false;
+                    const line = _c;
+                    if (samples.length >= limit)
+                        break;
+                    const trimmed = line.trim();
+                    if (!trimmed)
+                        continue;
+                    let raw;
+                    try {
+                        raw = JSON.parse(trimmed);
+                    }
+                    catch (_e) {
+                        continue;
+                    }
+                    const sample = MineDataset.parseRow(raw, samples.length);
+                    if (sample)
+                        samples.push(sample);
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_d && !_a && (_b = rl_1.return)) yield _b.call(rl_1);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            MineDataset.guardBaselineAlignment(samples, logger);
+            return samples;
+        });
+    }
+    /**
+     * Drop baseline graphs that don't belong to their essay (the HF mirror's
+     * essay↔graph desync). Mutates each sample's `baselines`, removing any non-empty
+     * graph whose entities barely appear in the essay; a dropped baseline is then
+     * skipped by the runner (so it never counts as a fake zero). Logs a per-tool tally
+     * — a high drop count is the loud signal that the data source is corrupt.
+     */
+    static guardBaselineAlignment(samples, logger) {
+        var _a;
+        const dropped = {};
+        for (const s of samples) {
+            for (const tool of Object.keys(s.baselines)) {
+                const g = s.baselines[tool];
+                if (!g || g.entities.length === 0)
+                    continue; // empty = legit absence, not misalignment
+                if (MineDataset.alignmentScore(s.text, g) < MIN_BASELINE_ALIGNMENT) {
+                    delete s.baselines[tool];
+                    dropped[tool] = ((_a = dropped[tool]) !== null && _a !== void 0 ? _a : 0) + 1;
+                }
+            }
+        }
+        const total = Object.values(dropped).reduce((a, b) => a + b, 0);
+        if (total > 0 && logger) {
+            const detail = Object.entries(dropped).map(([t, n]) => `${t}:${n}`).join(' ');
+            logger.warn(`MINE alignment guard dropped ${total} misaligned baseline graph(s) [${detail}] ` +
+                `across ${samples.length} articles — the source's essay↔graph columns are desynced. ` +
+                `Those (tool,article) cells are excluded from the four-way (not scored as 0).`);
+        }
+    }
+    /** Fraction of a graph's entity names that appear (case-insensitive substring)
+     *  in the essay. ~1 when the graph is this essay's; ~0 when it's another's. */
+    static alignmentScore(text, graph) {
+        const ents = graph.entities;
+        if (ents.length === 0)
+            return 0;
+        const hay = text.toLowerCase();
+        let hit = 0;
+        for (const e of ents)
+            if (e.name && hay.includes(e.name.toLowerCase()))
+                hit++;
+        return hit / ents.length;
+    }
+    /** Pure row → MineSample mapping (static for unit testing without a file). */
+    static parseRow(raw, index) {
+        var _a, _b;
+        const text = raw.essay_content;
+        const facts = raw.generated_queries;
+        if (!text || !facts || facts.length === 0)
+            return null;
+        // Only include a baseline when it actually carries a graph. An empty/absent
+        // graph (the tool produced nothing, OR our re-align found no confident match —
+        // see scripts/realign-mine.ts) must be OMITTED, not scored as a fake 0, so the
+        // four-way aggregate covers only the (article, tool) cells we really have.
+        const baselines = {};
+        const kggen = MineDataset.toGraph(raw.kggen);
+        if (kggen.entities.length)
+            baselines.kggen = kggen;
+        const graphrag = MineDataset.toGraph(raw.graphrag_kg);
+        if (graphrag.entities.length)
+            baselines.graphrag = graphrag;
+        const openie = MineDataset.toGraph(raw.openie_kg);
+        if (openie.entities.length)
+            baselines.openie = openie;
+        return { id: String((_a = raw.id) !== null && _a !== void 0 ? _a : index), topic: (_b = raw.essay_topic) !== null && _b !== void 0 ? _b : '', text, facts, baselines };
+    }
+    /** Map a stored {entities, edges, relations} baseline graph to a wanshi
+     *  KnowledgeGraph so every tool flows through the one identical scorer. */
+    static toGraph(raw) {
+        var _a, _b;
+        if (!raw)
+            return { entities: [], relations: [] };
+        const names = new Set();
+        const entities = [];
+        const ensure = (name) => {
+            const n = name === null || name === void 0 ? void 0 : name.trim();
+            if (!n || names.has(n))
+                return;
+            names.add(n);
+            entities.push({ name: n, entityType: 'concept', observations: [], files: [] });
+        };
+        for (const e of (_a = raw.entities) !== null && _a !== void 0 ? _a : [])
+            ensure(e);
+        const relations = [];
+        for (const t of (_b = raw.relations) !== null && _b !== void 0 ? _b : []) {
+            if (!Array.isArray(t) || t.length < 3)
+                continue;
+            const [from, predicate, to] = [String(t[0]).trim(), String(t[1]).trim(), String(t[2]).trim()];
+            if (!from || !to)
+                continue;
+            ensure(from);
+            ensure(to);
+            relations.push({ from, to, relationType: [predicate || 'related to'] });
+        }
+        return { entities, relations };
+    }
+}
+exports.MineDataset = MineDataset;
+//# sourceMappingURL=MineDataset.js.map

package/dist/evaluation/mine/MineDataset.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MineDataset.js","sourceRoot":"","sources":["../../../src/evaluation/mine/MineDataset.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,mDAAqC;AAKrC;;;;;;;;;GASG;AACH,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAsBpC,MAAa,WAAW;IAChB,IAAI,CAAC,QAAgB,EAAE,KAAa,EAAE,MAAe;;;YACzD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CACb,2BAA2B,QAAQ,IAAI;oBACvC,sDAAsD;oBACtD,8EAA8E,CAC/E,CAAC;YACJ,CAAC;YAED,MAAM,OAAO,GAAiB,EAAE,CAAC;YACjC,MAAM,MAAM,GAAG,EAAE,CAAC,gBAAgB,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;YACpE,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;;gBAE5E,KAAyB,eAAA,OAAA,cAAA,EAAE,CAAA,QAAA,gEAAE,CAAC;oBAAL,kBAAE;oBAAF,WAAE;oBAAhB,MAAM,IAAI,KAAA,CAAA;oBACnB,IAAI,OAAO,CAAC,MAAM,IAAI,KAAK;wBAAE,MAAM;oBACnC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;oBAC5B,IAAI,CAAC,OAAO;wBAAE,SAAS;oBAEvB,IAAI,GAAe,CAAC;oBACpB,IAAI,CAAC;wBACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC5B,CAAC;oBAAC,WAAM,CAAC;wBACP,SAAS;oBACX,CAAC;oBAED,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;oBACzD,IAAI,MAAM;wBAAE,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACnC,CAAC;;;;;;;;;YAED,WAAW,CAAC,sBAAsB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACpD,OAAO,OAAO,CAAC;QACjB,CAAC;KAAA;IAED;;;;;;OAMG;IACH,MAAM,CAAC,sBAAsB,CAAC,OAAqB,EAAE,MAAe;;QAClE,MAAM,OAAO,GAA2B,EAAE,CAAC;QAC3C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAkC,EAAE,CAAC;gBAC7E,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBAC5B,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS,CAAC,0CAA0C;gBACvF,IAAI,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,sBAAsB,EAAE,CAAC;oBACnE,OAAO,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;oBACzB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,MAAA,OAAO,CAAC,IAAI,CAAC,mCAAI,CAAC,CAAC,GAAG,CAAC,CAAC;gBAC3C,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAChE,IAAI,KAAK,GAAG,CAAC,IAAI,MAAM,EAAE,CAAC;YACxB,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9E,MAAM,CAAC,IAAI,CACT,gCAAgC,KAAK,kCAAkC,MAAM,IAAI;gBAC/E,UAAU,OAAO,CAAC,MAAM,6DAA6D;gBACrF,8EAA8E,CACjF,CAAC;QACJ,CAAC;IACH,CAAC;IAED;mFAC+E;IAC/E,MAAM,CAAC,cAAc,CAAC,IAAY,EAAE,KAAqB;QACvD,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC;QAC5B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,CAAC,CAAC;QAChC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAC/B,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,MAAM,CAAC,IAAI,IAAI;YAAE,IAAI,CAAC,CAAC,IAAI,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;gBAAE,GAAG,EAAE,CAAC;QAC9E,OAAO,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED,8EAA8E;IAC9E,MAAM,CAAC,QAAQ,CAAC,GAAe,EAAE,KAAa;;QAC5C,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,CAAC;QAC/B,MAAM,KAAK,GAAG,GAAG,CAAC,iBAAiB,CAAC;QACpC,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAEvD,4EAA4E;QAC5E,+EAA+E;QAC/E,+EAA+E;QAC/E,2EAA2E;QAC3E,MAAM,SAAS,GAA4B,EAAE,CAAC;QAC9C,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC7C,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM;YAAE,SAAS,CAAC,KAAK,GAAG,KAAK,CAAC;QACnD,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACtD,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM;YAAE,SAAS,CAAC,QAAQ,GAAG,QAAQ,CAAC;QAC5D,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAClD,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM;YAAE,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC;QAEtD,OAAO,EAAE,EAAE,EAAE,MAAM,CAAC,MAAA,GAAG,CAAC,EAAE,mCAAI,KAAK,CAAC,EAAE,KAAK,EAAE,MAAA,GAAG,CAAC,WAAW,mCAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC;IAC/F,CAAC;IAED;+EAC2E;IAC3E,MAAM,CAAC,OAAO,CAAC,GAAqC;;QAClD,IAAI,CAAC,GAAG;YAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;QAEjD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;QAChC,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,CAAC,IAAY,EAAE,EAAE;YAC9B,MAAM,CAAC,GAAG,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,OAAO;YAC/B,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACb,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QACjF,CAAC,CAAC;QAEF,KAAK,MAAM,CAAC,IAAI,MAAA,GAAG,CAAC,QAAQ,mCAAI,EAAE;YAAE,MAAM,CAAC,CAAC,CAAC,CAAC;QAE9C,MAAM,SAAS,GAAe,EAAE,CAAC;QACjC,KAAK,MAAM,CAAC,IAAI,MAAA,GAAG,CAAC,SAAS,mCAAI,EAAE,EAAE,CAAC;YACpC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS;YAChD,MAAM,CAAC,IAAI,EAAE,SAAS,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAC9F,IAAI,CAAC,IAAI,IAAI,CAAC,EAAE;gBAAE,SAAS;YAC3B,MAAM,CAAC,IAAI,CAAC,CAAC;YACb,MAAM,CAAC,EAAE,CAAC,CAAC;YACX,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,SAAS,IAAI,YAAY,CAAC,EAAE,CAAC,CAAC;QAC1E,CAAC;QAED,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;IACjC,CAAC;CACF;AA5HD,kCA4HC"}

package/dist/evaluation/mine/MineReporter.js

@@ -0,0 +1,98 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MineReporter = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+function pct(n) {
+    return n === undefined ? '   —  ' : `${(n * 100).toFixed(2)}%`.padStart(7);
+}
+function fmtDuration(ms) {
+    const s = Math.floor(ms / 1000);
+    if (s < 60)
+        return `${s}s`;
+    return `${Math.floor(s / 60)}m ${s % 60}s`;
+}
+const TOOL_LABEL = {
+    wanshi: 'wanshi',
+    kggen: 'KGGen',
+    graphrag: 'GraphRAG',
+    openie: 'OpenIE',
+};
+/**
+ * Four-way MINE comparison table. The "Re-scored" column is the apples-to-apples
+ * number (one identical retrieve+judge over all four graphs). "Published" is the
+ * KGGen-paper headline — a reference only; it used a different judge/retrieval and
+ * must not be conflated with the re-scored column.
+ */
+class MineReporter {
+    print(result) {
+        const sep = '─'.repeat(52);
+        console.log('');
+        console.log(`MINE benchmark (n=${result.sampleCount})  Model: ${result.model}  Judge: ${result.judgeModel}`);
+        console.log('');
+        console.log(`${'Tool'.padEnd(12)}  ${'Re-scored'.padStart(9)}  ${'Published'.padStart(9)}`);
+        console.log(sep);
+        const order = ['wanshi', 'kggen', 'graphrag', 'openie'];
+        for (const tool of order) {
+            const rescored = result.byTool[tool];
+            if (rescored === undefined && tool !== 'wanshi')
+                continue;
+            const published = tool === 'wanshi' ? undefined : result.published[tool];
+            console.log(`${TOOL_LABEL[tool].padEnd(12)}  ${pct(rescored)}  ${pct(published)}`);
+        }
+        console.log(sep);
+        console.log('Re-scored = our identical retrieve+judge over all graphs (comparable).');
+        console.log('Published = KGGen-paper headline (different judge/retrieval; reference only).');
+        if (result.relatedToShare !== undefined) {
+            // Guardrail: a high wanshi `related_to` share means the closed vocab is
+            // coercing real predicates away (Bug 1) — it caps the recall ceiling on MINE.
+            console.log(`wanshi vocab fit: ${(result.relatedToShare * 100).toFixed(0)}% of relations are 'related_to' ` +
+                `(high ⇒ closed vocab is coercing predicates; hurts MINE recall)`);
+        }
+        console.log('');
+        console.log(`Duration: ${fmtDuration(result.durationMs)}`);
+        console.log('');
+    }
+    save(result, outputPath) {
+        const dir = path.dirname(outputPath);
+        if (!fs.existsSync(dir))
+            fs.mkdirSync(dir, { recursive: true });
+        fs.writeFileSync(outputPath, JSON.stringify(result, null, 2), 'utf-8');
+        console.log(`Report saved to: ${outputPath}`);
+    }
+}
+exports.MineReporter = MineReporter;
+//# sourceMappingURL=MineReporter.js.map

package/dist/evaluation/mine/MineReporter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MineReporter.js","sourceRoot":"","sources":["../../../src/evaluation/mine/MineReporter.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAG7B,SAAS,GAAG,CAAC,CAAqB;IAChC,OAAO,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AAC7E,CAAC;AAED,SAAS,WAAW,CAAC,EAAU;IAC7B,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC;IAChC,IAAI,CAAC,GAAG,EAAE;QAAE,OAAO,GAAG,CAAC,GAAG,CAAC;IAC3B,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,GAA6B;IAC3C,MAAM,EAAE,QAAQ;IAChB,KAAK,EAAE,OAAO;IACd,QAAQ,EAAE,UAAU;IACpB,MAAM,EAAE,QAAQ;CACjB,CAAC;AAEF;;;;;GAKG;AACH,MAAa,YAAY;IACvB,KAAK,CAAC,MAAkB;QACtB,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,CACT,qBAAqB,MAAM,CAAC,WAAW,aAAa,MAAM,CAAC,KAAK,YAAY,MAAM,CAAC,UAAU,EAAE,CAChG,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC5F,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEjB,MAAM,KAAK,GAAe,CAAC,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;QACpE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YACrC,IAAI,QAAQ,KAAK,SAAS,IAAI,IAAI,KAAK,QAAQ;gBAAE,SAAS;YAC1D,MAAM,SAAS,GAAG,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACzE,OAAO,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,KAAK,GAAG,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QACrF,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,wEAAwE,CAAC,CAAC;QACtF,OAAO,CAAC,GAAG,CAAC,+EAA+E,CAAC,CAAC;QAC7F,IAAI,MAAM,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;YACxC,wEAAwE;YACxE,8EAA8E;YAC9E,OAAO,CAAC,GAAG,CACT,qBAAqB,CAAC,MAAM,CAAC,cAAc,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,kCAAkC;gBAC7F,iEAAiE,CACpE,CAAC;QACJ,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,aAAa,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC3D,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC;IAED,IAAI,CAAC,MAAkB,EAAE,UAAkB;QACzC,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAChE,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;QACvE,OAAO,CAAC,GAAG,CAAC,oBAAoB,UAAU,EAAE,CAAC,CAAC;IAChD,CAAC;CACF;AAxCD,oCAwCC"}

package/dist/evaluation/mine/MineRunner.js

@@ -0,0 +1,148 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MineRunner = void 0;
+exports.relatedToShare = relatedToShare;
+const MineCheckpoint_1 = require("./MineCheckpoint");
+const types_1 = require("./types");
+const BASELINE_TOOLS = ['kggen', 'graphrag', 'openie'];
+/**
+ * Runs the MINE benchmark four ways: extracts wanshi's KG per article and scores
+ * it, then re-scores the three stored baseline graphs with the SAME MineScorer, so
+ * every tool faces one identical retrieve+judge. Aggregates mean accuracy per tool.
+ */
+class MineRunner {
+    constructor(kgBuilder, promptManager, scorer, logger) {
+        this.kgBuilder = kgBuilder;
+        this.promptManager = promptManager;
+        this.scorer = scorer;
+        this.logger = logger;
+    }
+    run(samples, opts) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
+            const start = Date.now();
+            const systemPrompt = yield this.promptManager.getSystemPrompt('mine', '**/*.txt', 'MINE benchmark', undefined, opts.glossary, opts.openPredicate);
+            if ((_b = (_a = opts.glossary) === null || _a === void 0 ? void 0 : _a.relationTypes) === null || _b === void 0 ? void 0 : _b.length) {
+                this.logger.info(`MINE glossary active: ${opts.glossary.relationTypes.length} relation types, ` +
+                    `${opts.glossary.entityTypes.length} entity types`);
+            }
+            const perArticle = [];
+            // Per-article checkpoint: resume after a crash; key folds in the run config so a
+            // config change never reuses a stale article.
+            const checkpoint = opts.checkpointPath
+                ? new MineCheckpoint_1.MineCheckpoint(opts.checkpointPath, this.logger)
+                : undefined;
+            if (checkpoint)
+                yield checkpoint.load();
+            const keyFor = (id) => { var _a; return `${opts.model}␟${opts.judgeModel}␟${(_a = opts.armKey) !== null && _a !== void 0 ? _a : ''}␟${opts.rescoreBaselines}␟${id}`; };
+            for (let i = 0; i < samples.length; i++) {
+                const s = samples[i];
+                const cached = checkpoint === null || checkpoint === void 0 ? void 0 : checkpoint.get(keyFor(s.id));
+                if (cached) {
+                    this.logger.info(`[${i + 1}/${samples.length}] MINE article ${s.id} (${s.topic}) — restored from checkpoint`);
+                    perArticle.push(cached);
+                    continue;
+                }
+                this.logger.info(`[${i + 1}/${samples.length}] MINE article ${s.id} (${s.topic}) — ${s.facts.length} facts`);
+                const wanshiKg = yield this.extract(s, systemPrompt, opts.glossary);
+                const scores = {
+                    wanshi: yield this.scorer.score('wanshi', wanshiKg, s.facts),
+                };
+                if (opts.rescoreBaselines) {
+                    for (const tool of BASELINE_TOOLS) {
+                        const g = s.baselines[tool];
+                        if (g)
+                            scores[tool] = yield this.scorer.score(tool, g, s.facts);
+                    }
+                }
+                this.logger.info(`    ${Object.keys(scores)
+                    .map((t) => `${t}=${(scores[t].accuracy * 100).toFixed(1)}%`)
+                    .join('  ')}`);
+                const article = {
+                    id: s.id,
+                    topic: s.topic,
+                    scores,
+                    wanshiGraph: wanshiKg,
+                    relatedToShare: relatedToShare(wanshiKg),
+                };
+                perArticle.push(article);
+                if (checkpoint)
+                    yield checkpoint.append(keyFor(s.id), article);
+            }
+            const meanRelatedTo = perArticle.length
+                ? perArticle.reduce((acc, a) => { var _a; return acc + ((_a = a.relatedToShare) !== null && _a !== void 0 ? _a : 0); }, 0) / perArticle.length
+                : 0;
+            return {
+                model: opts.model,
+                judgeModel: opts.judgeModel,
+                sampleCount: samples.length,
+                byTool: this.aggregate(perArticle),
+                published: types_1.MINE_PUBLISHED,
+                perArticle,
+                durationMs: Date.now() - start,
+                relatedToShare: meanRelatedTo,
+            };
+        });
+    }
+    extract(s, systemPrompt, glossary) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const processedFile = {
+                    path: `mine/${s.id}.txt`,
+                    chunks: [
+                        { content: s.text, index: 1, totalChunks: 1, startOffset: 0, endOffset: s.text.length },
+                    ],
+                    metadata: {},
+                };
+                const graphs = yield this.kgBuilder.build(processedFile, systemPrompt, undefined, glossary);
+                return graphs.length > 0 ? graphs[0] : { entities: [], relations: [] };
+            }
+            catch (err) {
+                this.logger.warn(`MINE article ${s.id} extraction failed: ${err}`);
+                return { entities: [], relations: [] };
+            }
+        });
+    }
+    /** Mean accuracy per tool across all articles that produced a score for it. */
+    aggregate(perArticle) {
+        var _a;
+        const sum = {};
+        for (const a of perArticle) {
+            for (const tool of Object.keys(a.scores)) {
+                const acc = (_a = sum[tool]) !== null && _a !== void 0 ? _a : { s: 0, n: 0 };
+                acc.s += a.scores[tool].accuracy;
+                acc.n += 1;
+                sum[tool] = acc;
+            }
+        }
+        const out = {};
+        for (const tool of Object.keys(sum)) {
+            const { s, n } = sum[tool];
+            out[tool] = n > 0 ? s / n : 0;
+        }
+        return out;
+    }
+}
+exports.MineRunner = MineRunner;
+/** Fraction of a graph's relations that are the generic `related_to` catch-all (the
+ *  vocab-fit guardrail; mirrors KnowledgeMerger.logVocabularyFit). 0 for no relations. */
+function relatedToShare(graph) {
+    const rels = graph.relations;
+    if (rels.length === 0)
+        return 0;
+    const catchAll = rels.filter((r) => {
+        const types = Array.isArray(r.relationType) ? r.relationType : [r.relationType];
+        return types.length > 0 && types.every((t) => t === 'related_to');
+    }).length;
+    return catchAll / rels.length;
+}
+//# sourceMappingURL=MineRunner.js.map

package/dist/evaluation/mine/MineRunner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MineRunner.js","sourceRoot":"","sources":["../../../src/evaluation/mine/MineRunner.ts"],"names":[],"mappings":";;;;;;;;;;;;AAoLA,wCAQC;AAtLD,qDAAkD;AAClD,mCAOiB;AAyBjB,MAAM,cAAc,GAAkC,CAAC,OAAO,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;AAEtF;;;;GAIG;AACH,MAAa,UAAU;IACrB,YACmB,SAAgC,EAChC,aAA4B,EAC5B,MAAkB,EAClB,MAAc;QAHd,cAAS,GAAT,SAAS,CAAuB;QAChC,kBAAa,GAAb,aAAa,CAAe;QAC5B,WAAM,GAAN,MAAM,CAAY;QAClB,WAAM,GAAN,MAAM,CAAQ;IAC9B,CAAC;IAEE,GAAG,CAAC,OAAqB,EAAE,IAAuB;;;YACtD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACzB,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,eAAe,CAC3D,MAAM,EACN,UAAU,EACV,gBAAgB,EAChB,SAAS,EACT,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,aAAa,CACnB,CAAC;YACF,IAAI,MAAA,MAAA,IAAI,CAAC,QAAQ,0CAAE,aAAa,0CAAE,MAAM,EAAE,CAAC;gBACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,yBAAyB,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,mBAAmB;oBAC5E,GAAG,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,eAAe,CACrD,CAAC;YACJ,CAAC;YACD,MAAM,UAAU,GAAwB,EAAE,CAAC;YAE3C,iFAAiF;YACjF,8CAA8C;YAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,cAAc;gBACpC,CAAC,CAAC,IAAI,+BAAc,CAAC,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,MAAM,CAAC;gBACtD,CAAC,CAAC,SAAS,CAAC;YACd,IAAI,UAAU;gBAAE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;YACxC,MAAM,MAAM,GAAG,CAAC,EAAU,EAAE,EAAE,WAC5B,OAAA,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,UAAU,IAAI,MAAA,IAAI,CAAC,MAAM,mCAAI,EAAE,IAAI,IAAI,CAAC,gBAAgB,IAAI,EAAE,EAAE,CAAA,EAAA,CAAC;YAEzF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;gBAErB,MAAM,MAAM,GAAG,UAAU,aAAV,UAAU,uBAAV,UAAU,CAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;gBAC7C,IAAI,MAAM,EAAE,CAAC;oBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,kBAAkB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,KAAK,8BAA8B,CAAC,CAAC;oBAC9G,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBACxB,SAAS;gBACX,CAAC;gBAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,kBAAkB,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,KAAK,OAAO,CAAC,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAC;gBAE7G,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC,EAAE,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACpE,MAAM,MAAM,GAA8C;oBACxD,MAAM,EAAE,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC;iBAC7D,CAAC;gBAEF,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBAC1B,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;wBAClC,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;wBAC5B,IAAI,CAAC;4BAAE,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;oBAClE,CAAC;gBACH,CAAC;gBAED,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,OAAQ,MAAM,CAAC,IAAI,CAAC,MAAM,CAAgB;qBACvC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;qBAC7D,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;gBACF,MAAM,OAAO,GAAsB;oBACjC,EAAE,EAAE,CAAC,CAAC,EAAE;oBACR,KAAK,EAAE,CAAC,CAAC,KAAK;oBACd,MAAM;oBACN,WAAW,EAAE,QAAQ;oBACrB,cAAc,EAAE,cAAc,CAAC,QAAQ,CAAC;iBACzC,CAAC;gBACF,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACzB,IAAI,UAAU;oBAAE,MAAM,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;YACjE,CAAC;YAED,MAAM,aAAa,GAAG,UAAU,CAAC,MAAM;gBACrC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,WAAC,OAAA,GAAG,GAAG,CAAC,MAAA,CAAC,CAAC,cAAc,mCAAI,CAAC,CAAC,CAAA,EAAA,EAAE,CAAC,CAAC,GAAG,UAAU,CAAC,MAAM;gBACrF,CAAC,CAAC,CAAC,CAAC;YAEN,OAAO;gBACL,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,WAAW,EAAE,OAAO,CAAC,MAAM;gBAC3B,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC;gBAClC,SAAS,EAAE,sBAAc;gBACzB,UAAU;gBACV,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;gBAC9B,cAAc,EAAE,aAAa;aAC9B,CAAC;QACJ,CAAC;KAAA;IAEa,OAAO,CACnB,CAAa,EACb,YAAoB,EACpB,QAAyB;;YAEzB,IAAI,CAAC;gBACH,MAAM,aAAa,GAAkB;oBACnC,IAAI,EAAE,QAAQ,CAAC,CAAC,EAAE,MAAM;oBACxB,MAAM,EAAE;wBACN,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE;qBACxF;oBACD,QAAQ,EAAE,EAAE;iBACb,CAAC;gBACF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,aAAa,EAAE,YAAY,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;gBAC5F,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;YACzE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC,EAAE,uBAAuB,GAAG,EAAE,CAAC,CAAC;gBACnE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;YACzC,CAAC;QACH,CAAC;KAAA;IAED,+EAA+E;IACvE,SAAS,CAAC,UAA+B;;QAC/C,MAAM,GAAG,GAAwD,EAAE,CAAC;QACpE,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;YAC3B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAe,EAAE,CAAC;gBACvD,MAAM,GAAG,GAAG,MAAA,GAAG,CAAC,IAAI,CAAC,mCAAI,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;gBACxC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAE,CAAC,QAAQ,CAAC;gBAClC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;gBACX,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC;YAClB,CAAC;QACH,CAAC;QACD,MAAM,GAAG,GAAsC,EAAE,CAAC;QAClD,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAe,EAAE,CAAC;YAClD,MAAM,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,GAAG,CAAC,IAAI,CAAE,CAAC;YAC5B,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChC,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AAlID,gCAkIC;AAED;0FAC0F;AAC1F,SAAgB,cAAc,CAAC,KAAqB;IAClD,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC;IAC7B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACjC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QAChF,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,YAAY,CAAC,CAAC;IACpE,CAAC,CAAC,CAAC,MAAM,CAAC;IACV,OAAO,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC;AAChC,CAAC"}

package/dist/evaluation/mine/MineScorer.js

@@ -0,0 +1,127 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MineScorer = exports.MINE_JUDGE_INSTRUCTION = void 0;
+const zod_1 = require("zod");
+const cosineSimilarity_1 = require("../../shared/utils/cosineSimilarity");
+/**
+ * MINE scorer — replicates kg-gen `experiments/MINE/_1_evaluation.py` so it can be
+ * applied IDENTICALLY to any KnowledgeGraph (wanshi's fresh extraction and the
+ * three stored baselines), which is the apples-to-apples requirement of the
+ * comparison. For each atomic fact: embed graph nodes, take the top-k nearest to
+ * the fact, gather their incident triples (serialized "from predicate to") as the
+ * retrieved context, then ask the judge whether that context contains the fact.
+ */
+// Verbatim instruction from MINE's DSPy `EvaluateResponse` signature.
+exports.MINE_JUDGE_INSTRUCTION = 'Determine whether the context contains the information stated in the correct answer. Respond with 1 if yes, 0 if no.';
+// The verbatim criterion + a firm output directive. Without it, smaller judges
+// (gemma3:4b) intermittently EXPLAIN their answer (prose), which fails the schema →
+// retries → some land as a wrong 0, biasing scores low and unequally across the
+// graphs being compared. The directive (DSPy adds the equivalent internally) drives
+// a clean bare digit; the tolerant schema accepts it (or the object form).
+const JUDGE_SYSTEM = `${exports.MINE_JUDGE_INSTRUCTION} Respond with ONLY the digit 1 or 0 — no explanation, no other text.`;
+// Output {evaluation: 0|1}. Tolerant of a BARE number/string too: gemma3:4b honors
+// the verbatim "respond with 1" prompt with a bare `1` under Ollama's soft format
+// constraint, so we coerce that into {evaluation:1} instead of 3x-retrying then
+// dropping it — which zeroed every score AND took 27min on the first dev run.
+// zod-to-json-schema emits the inner object schema, so capable judges still get an
+// object format constraint and comply directly. We threshold ≥0.5 ⇒ 1.
+const JudgeSchema = zod_1.z.preprocess((v) => (typeof v === 'number' || typeof v === 'string' ? { evaluation: v } : v), zod_1.z.object({ evaluation: zod_1.z.coerce.number() }));
+class MineScorer {
+    constructor(embeddings, judge, opts = { topK: 15 }) {
+        this.embeddings = embeddings;
+        this.judge = judge;
+        this.opts = opts;
+    }
+    score(tool, graph, facts) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const nodes = graph.entities.map((e) => e.name).filter((n) => !!n);
+            const nodeEmb = nodes.length ? yield this.embeddings.embedBatch(nodes) : [];
+            const incident = this.buildIncident(graph);
+            // Judge facts through a bounded-concurrency pool (order preserved by index).
+            const perFact = new Array(facts.length);
+            const limit = Math.max(1, Math.min((_a = this.opts.concurrency) !== null && _a !== void 0 ? _a : 8, facts.length || 1));
+            let next = 0;
+            const worker = () => __awaiter(this, void 0, void 0, function* () {
+                for (let i = next++; i < facts.length; i = next++) {
+                    const fact = facts[i];
+                    const context = nodes.length ? yield this.retrieve(fact, nodes, nodeEmb, incident) : '';
+                    const evaluation = context ? yield this.judgeFact(context, fact) : 0;
+                    perFact[i] = { fact, context, evaluation };
+                }
+            });
+            yield Promise.all(Array.from({ length: limit }, () => worker()));
+            const correct = perFact.reduce((s, r) => s + r.evaluation, 0);
+            const total = facts.length;
+            return { tool, accuracy: total ? correct / total : 0, correct, total, perFact };
+        });
+    }
+    /** node → its incident triples serialized as "from predicate to". */
+    buildIncident(graph) {
+        var _a, _b;
+        const m = new Map();
+        const add = (node, s) => {
+            const arr = m.get(node);
+            if (arr)
+                arr.push(s);
+            else
+                m.set(node, [s]);
+        };
+        for (const r of graph.relations) {
+            const pred = (_b = (_a = r.relationType) === null || _a === void 0 ? void 0 : _a[0]) !== null && _b !== void 0 ? _b : 'related to';
+            const serialized = `${r.from} ${pred} ${r.to}`;
+            add(r.from, serialized);
+            add(r.to, serialized);
+        }
+        return m;
+    }
+    retrieve(fact, nodes, nodeEmb, incident) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const q = yield this.embeddings.embed(fact);
+            const ranked = nodes
+                .map((name, i) => ({ name, sim: (0, cosineSimilarity_1.cosineSimilarity)(q, nodeEmb[i]) }))
+                .sort((a, b) => b.sim - a.sim)
+                .slice(0, this.opts.topK);
+            const seen = new Set();
+            const parts = [];
+            for (const { name } of ranked) {
+                for (const s of (_a = incident.get(name)) !== null && _a !== void 0 ? _a : []) {
+                    if (!seen.has(s)) {
+                        seen.add(s);
+                        parts.push(s);
+                    }
+                }
+            }
+            return parts.join('. ');
+        });
+    }
+    judgeFact(context, fact) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const messages = [
+                { role: 'system', content: JUDGE_SYSTEM },
+                { role: 'user', content: `Context:\n${context}\n\nCorrect answer:\n${fact}` },
+            ];
+            try {
+                // Cast: the preprocess schema's INPUT is `unknown` (it accepts bare numbers),
+                // but its OUTPUT is exactly {evaluation:number}, which is what we read.
+                const out = yield this.judge.generateStructured(messages, JudgeSchema);
+                return out.evaluation >= 0.5 ? 1 : 0;
+            }
+            catch (_a) {
+                return 0; // a judge failure is a miss, never a thrown run
+            }
+        });
+    }
+}
+exports.MineScorer = MineScorer;
+//# sourceMappingURL=MineScorer.js.map

package/dist/evaluation/mine/MineScorer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MineScorer.js","sourceRoot":"","sources":["../../../src/evaluation/mine/MineScorer.ts"],"names":[],"mappings":";;;;;;;;;;;;AAAA,6BAAwB;AAIxB,0EAAuE;AAGvE;;;;;;;GAOG;AAEH,sEAAsE;AACzD,QAAA,sBAAsB,GACjC,sHAAsH,CAAC;AAEzH,+EAA+E;AAC/E,oFAAoF;AACpF,gFAAgF;AAChF,oFAAoF;AACpF,2EAA2E;AAC3E,MAAM,YAAY,GAAG,GAAG,8BAAsB,sEAAsE,CAAC;AAErH,mFAAmF;AACnF,kFAAkF;AAClF,gFAAgF;AAChF,8EAA8E;AAC9E,mFAAmF;AACnF,uEAAuE;AACvE,MAAM,WAAW,GAAG,OAAC,CAAC,UAAU,CAC9B,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAC/E,OAAC,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,OAAC,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC,CAC5C,CAAC;AAYF,MAAa,UAAU;IACrB,YACmB,UAA8B,EAC9B,KAAmB,EACnB,OAA0B,EAAE,IAAI,EAAE,EAAE,EAAE;QAFtC,eAAU,GAAV,UAAU,CAAoB;QAC9B,UAAK,GAAL,KAAK,CAAc;QACnB,SAAI,GAAJ,IAAI,CAAkC;IACtD,CAAC;IAEE,KAAK,CAAC,IAAc,EAAE,KAAqB,EAAE,KAAe;;;YAChE,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAChF,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;YAE3C,6EAA6E;YAC7E,MAAM,OAAO,GAAqB,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAA,IAAI,CAAC,IAAI,CAAC,WAAW,mCAAI,CAAC,EAAE,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC;YACnF,IAAI,IAAI,GAAG,CAAC,CAAC;YACb,MAAM,MAAM,GAAG,GAAwB,EAAE;gBACvC,KAAK,IAAI,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC;oBAClD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;oBACtB,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBACxF,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBACrE,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;gBAC7C,CAAC;YACH,CAAC,CAAA,CAAC;YACF,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAEjE,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YAC9D,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;YAC3B,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;QAClF,CAAC;KAAA;IAED,qEAAqE;IAC7D,aAAa,CAAC,KAAqB;;QACzC,MAAM,CAAC,GAAG,IAAI,GAAG,EAAoB,CAAC;QACtC,MAAM,GAAG,GAAG,CAAC,IAAY,EAAE,CAAS,EAAE,EAAE;YACtC,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACxB,IAAI,GAAG;gBAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;;gBAChB,CAAC,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,MAAA,MAAA,CAAC,CAAC,YAAY,0CAAG,CAAC,CAAC,mCAAI,YAAY,CAAC;YACjD,MAAM,UAAU,GAAG,GAAG,CAAC,CAAC,IAAI,IAAI,IAAI,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;YAC/C,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;YACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;QACxB,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC;IAEa,QAAQ,CACpB,IAAY,EACZ,KAAe,EACf,OAAmB,EACnB,QAA+B;;;YAE/B,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5C,MAAM,MAAM,GAAG,KAAK;iBACjB,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,IAAA,mCAAgB,EAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;iBAClE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC;iBAC7B,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAE5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;YAC/B,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,KAAK,MAAM,EAAE,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC;gBAC9B,KAAK,MAAM,CAAC,IAAI,MAAA,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,mCAAI,EAAE,EAAE,CAAC;oBACzC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;wBACjB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;wBACZ,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;oBAChB,CAAC;gBACH,CAAC;YACH,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC;KAAA;IAEa,SAAS,CAAC,OAAe,EAAE,IAAY;;YACnD,MAAM,QAAQ,GAAiB;gBAC7B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,EAAE;gBACzC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,aAAa,OAAO,wBAAwB,IAAI,EAAE,EAAE;aAC9E,CAAC;YACF,IAAI,CAAC;gBACH,8EAA8E;gBAC9E,wEAAwE;gBACxE,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAC7C,QAAQ,EACR,WAA2D,CAC5D,CAAC;gBACF,OAAO,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC;YAAC,WAAM,CAAC;gBACP,OAAO,CAAC,CAAC,CAAC,gDAAgD;YAC5D,CAAC;QACH,CAAC;KAAA;CACF;AA1FD,gCA0FC"}

package/dist/evaluation/mine/types.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MINE_PUBLISHED = void 0;
+/** KGGen paper headline accuracies on MINE (their judge + retrieval). A labelled
+ *  reference row only — our re-scored numbers use one identical retrieve+judge over
+ *  all four graphs, so they may differ from these and shouldn't be conflated. */
+exports.MINE_PUBLISHED = {
+    kggen: 0.66,
+    graphrag: 0.48,
+    openie: 0.30,
+};
+//# sourceMappingURL=types.js.map

package/dist/evaluation/mine/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/evaluation/mine/types.ts"],"names":[],"mappings":";;;AAsDA;;iFAEiF;AACpE,QAAA,cAAc,GAAgD;IACzE,KAAK,EAAE,IAAI;IACX,QAAQ,EAAE,IAAI;IACd,MAAM,EAAE,IAAI;CACb,CAAC"}