@wanshi-kg/wanshi 0.1.0

package/dist/core/corpus/CorpusAnalyzer.js

@@ -0,0 +1,266 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CorpusAnalyzer = void 0;
+const crypto = __importStar(require("crypto"));
+const fs = __importStar(require("fs"));
+const zod_1 = require("zod");
+const shared_1 = require("../../shared");
+const termFrequency_1 = require("./termFrequency");
+const CorpusProfileStore_1 = require("./CorpusProfileStore");
+const normalizeGlossary_1 = require("./normalizeGlossary");
+const relPath_1 = require("./relPath");
+/** Per-file text read for the pre-pass is capped to bound frequency + classifier cost. */
+const PER_FILE_CHAR_CAP = 16000;
+/** Inline glossary system prompt, used when no versioned template is available. */
+const FALLBACK_GLOSSARY_SYSTEM = "You design a controlled vocabulary (glossary) for knowledge-graph extraction " +
+    "over a document corpus. Given the dominant content type and the most frequent " +
+    "terms, propose: (1) canonical ENTITY NAMES — the real recurring proper nouns / " +
+    "key concepts, each normalized to ONE canonical spelling so extraction stays " +
+    "consistent; (2) ENTITY TYPES appropriate to this corpus; (3) RELATION TYPES " +
+    "appropriate to this corpus. Prefer terms that actually appear. Be concise — a " +
+    "few dozen names at most. Return JSON only.";
+const GlossarySchema = zod_1.z.object({
+    entityNames: zod_1.z.array(zod_1.z.string()).describe("Canonical entity names recurring in this corpus"),
+    entityTypes: zod_1.z.array(zod_1.z.string()).describe("Entity type categories appropriate to this corpus"),
+    relationTypes: zod_1.z.array(zod_1.z.string()).describe("Relation type names appropriate to this corpus"),
+});
+/**
+ * Corpus analysis pre-pass: read + classify each file (char-capped), count term
+ * frequency, then make ONE LLM call to propose a corpus-specific glossary
+ * (canonical entity names / types / relation types). Cached to a sidecar and
+ * reused on re-run when the corpus + model are unchanged.
+ */
+class CorpusAnalyzer {
+    constructor(llm, classifier, readerFactory, logger, promptManager) {
+        this.llm = llm;
+        this.classifier = classifier;
+        this.readerFactory = readerFactory;
+        this.logger = logger;
+        this.promptManager = promptManager;
+    }
+    analyzeOrLoad(files, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const inputRoot = (_a = options.input) !== null && _a !== void 0 ? _a : "";
+            const topN = options.corpus.topTerms;
+            const profilePath = options.corpus.profilePath || `${options.output}.corpus-profile.json`;
+            const store = new CorpusProfileStore_1.CorpusProfileStore(profilePath, this.logger);
+            const key = this.computeKey(files, inputRoot, options, topN);
+            const cached = yield store.load();
+            if (cached && cached.key === key) {
+                this.logger.info(`Reusing cached corpus profile (${cached.glossary.entityNames.length} names) from ${profilePath}`);
+                return cached;
+            }
+            if (cached) {
+                this.logger.info(`Corpus profile at ${profilePath} is stale (corpus/model changed); rebuilding`);
+            }
+            if (options.corpus.clustering) {
+                this.logger.info("corpusClustering is not implemented yet (deferred to a follow-up); ignoring the flag");
+            }
+            // 1. Read (char-capped) + classify each file. The classifier call here is the
+            //    expensive bit we cache; FileProcessor reuses perFileClasses downstream.
+            const texts = [];
+            const perFileClasses = {};
+            for (const file of files) {
+                if (shared_1.shutdown.isRequested()) {
+                    this.logger.warn("Interrupted during corpus pre-pass; profiling partial corpus");
+                    break;
+                }
+                const text = yield this.readCapped(file);
+                if (!text)
+                    continue;
+                texts.push(text);
+                if (this.classifier) {
+                    try {
+                        perFileClasses[(0, relPath_1.toRelPathId)(inputRoot, file)] =
+                            yield this.classifier.classify(text, file);
+                    }
+                    catch (error) {
+                        this.logger.warn(`Corpus pre-pass could not classify ${file}: ${error}`);
+                    }
+                }
+            }
+            // 2. Frequency + corpus-level class aggregate.
+            const topTerms = (0, termFrequency_1.countTerms)(texts, { topN });
+            const corpusClasses = aggregateClasses(Object.values(perFileClasses));
+            // 3. One LLM call → glossary. A failure must NOT be cached as an empty
+            //    glossary and reused forever — run this pass without one and skip
+            //    persisting the sidecar so the next run retries (KG-02).
+            let glossary;
+            let glossaryOk = true;
+            try {
+                glossary = yield this.generateGlossary(corpusClasses, topTerms, texts);
+            }
+            catch (error) {
+                this.logger.warn(`Corpus glossary generation failed; running without it and NOT caching ` +
+                    `the profile (will rebuild next run): ${error}`);
+                glossary = { entityNames: [], entityTypes: [], relationTypes: [] };
+                glossaryOk = false;
+            }
+            const profile = {
+                generatedAt: new Date().toISOString(),
+                key,
+                fileCount: files.length,
+                corpusClasses,
+                perFileClasses,
+                topTerms,
+                glossary,
+            };
+            if (glossaryOk) {
+                yield store.save(profile);
+                this.logger.info(`Corpus profile built: ${topTerms.length} top terms, glossary ` +
+                    `${glossary.entityNames.length} names / ${glossary.entityTypes.length} types / ` +
+                    `${glossary.relationTypes.length} relations → ${profilePath}`);
+            }
+            return profile;
+        });
+    }
+    /** Read a file via its reader and concatenate chunk text, capped. Non-fatal. */
+    readCapped(file) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            try {
+                const reader = this.readerFactory.getReader(file);
+                if (!reader)
+                    return "";
+                const res = yield reader.read(file);
+                return ((_a = res.chunks) !== null && _a !== void 0 ? _a : [])
+                    .map((c) => c.content)
+                    .join("\n")
+                    .slice(0, PER_FILE_CHAR_CAP);
+            }
+            catch (error) {
+                this.logger.warn(`Corpus pre-pass could not read ${file}: ${error}`);
+                return "";
+            }
+        });
+    }
+    /**
+     * Validity key: model + topN + classifier mode + per-file (relpath, size,
+     * mtime). Folding size+mtime makes the cache **content-sensitive** (KG-06):
+     * editing a file invalidates its glossary, while a moved input tree / unchanged
+     * content stays stable (relpath is input-relative). size+mtime is a cheap
+     * `stat` proxy — deliberately *not* a byte hash, so expensive media readers
+     * (PDF/audio) aren't re-invoked on every run just to compute the key.
+     */
+    computeKey(files, inputRoot, options, topN) {
+        const entries = files
+            .map((f) => {
+            const rel = (0, relPath_1.toRelPathId)(inputRoot, f);
+            let sig = "missing";
+            try {
+                const st = fs.statSync(f);
+                sig = `${st.size}:${Math.round(st.mtimeMs)}`;
+            }
+            catch (_a) {
+                // Non-existent / unreadable file: stable sentinel, never throws.
+            }
+            return `${rel}|${sig}`;
+        })
+            .sort();
+        const hash = crypto.createHash("sha1");
+        hash.update(`${options.llm.model} ${topN} ${options.classifier.mode} ${entries.length}\n${entries.join("\n")}`);
+        return hash.digest("hex");
+    }
+    generateGlossary(corpusClasses, topTerms, texts) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c;
+            const classLine = corpusClasses.length > 0
+                ? corpusClasses
+                    .slice(0, 2)
+                    .map((c) => `${c.class} (${c.confidence.toFixed(2)})`)
+                    .join(", ")
+                : "unknown";
+            const termList = topTerms
+                .map((t) => `${t.term} (${t.count})`)
+                .join(", ");
+            const snippets = texts
+                .slice(0, 3)
+                .map((t, i) => `--- sample ${i + 1} ---\n${t.slice(0, 600)}`)
+                .join("\n\n");
+            // Prefer the versioned glossary templates (v5); fall back to inline strings
+            // when the current prompt version ships none (e.g. v4.5) or rendering fails.
+            const rendered = yield ((_a = this.promptManager) === null || _a === void 0 ? void 0 : _a.getGlossaryPrompt({
+                classLine,
+                termList,
+                snippets,
+            }));
+            const system = (_b = rendered === null || rendered === void 0 ? void 0 : rendered.system) !== null && _b !== void 0 ? _b : FALLBACK_GLOSSARY_SYSTEM;
+            const user = (_c = rendered === null || rendered === void 0 ? void 0 : rendered.user) !== null && _c !== void 0 ? _c : `Corpus content type: ${classLine}\n\n` +
+                `Most frequent terms (with counts):\n${termList}\n\n` +
+                `Representative snippets:\n${snippets}`;
+            const messages = [
+                { role: "system", content: system },
+                { role: "user", content: user },
+            ];
+            // Let failures propagate: a failed glossary must NOT be cached as empty and
+            // reused forever (KG-02). The caller (analyzeOrLoad) catches, runs this pass
+            // without a glossary, and skips persisting the sidecar so the next run retries.
+            const result = yield this.llm.generateStructured(messages, GlossarySchema);
+            // Validate + normalize before it becomes the authoritative closed vocabulary
+            // (KG-06): snake_case, dedupe, drop has_* predicates, cap to the prompt's
+            // limits — so garbage glossary output can't get enforced as the Zod enum.
+            return (0, normalizeGlossary_1.normalizeGlossary)(result);
+        });
+    }
+}
+exports.CorpusAnalyzer = CorpusAnalyzer;
+/** Average per-class confidence across files, sorted descending. */
+function aggregateClasses(perFile) {
+    var _a;
+    const sums = new Map();
+    for (const classes of perFile) {
+        for (const c of classes) {
+            const e = (_a = sums.get(c.class)) !== null && _a !== void 0 ? _a : { sum: 0, n: 0 };
+            e.sum += c.confidence;
+            e.n += 1;
+            sums.set(c.class, e);
+        }
+    }
+    return Array.from(sums.entries())
+        .map(([cls, { sum, n }]) => ({ class: cls, confidence: sum / n }))
+        .sort((a, b) => b.confidence - a.confidence);
+}
+//# sourceMappingURL=CorpusAnalyzer.js.map

package/dist/core/corpus/CorpusAnalyzer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CorpusAnalyzer.js","sourceRoot":"","sources":["../../../src/core/corpus/CorpusAnalyzer.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,+CAAiC;AACjC,uCAAyB;AACzB,6BAAwB;AAYxB,yCAAgD;AAGhD,mDAA6C;AAC7C,6DAA0D;AAC1D,2DAAwD;AACxD,uCAAwC;AAExC,0FAA0F;AAC1F,MAAM,iBAAiB,GAAG,KAAM,CAAC;AAEjC,mFAAmF;AACnF,MAAM,wBAAwB,GAC5B,+EAA+E;IAC/E,gFAAgF;IAChF,iFAAiF;IACjF,8EAA8E;IAC9E,8EAA8E;IAC9E,gFAAgF;IAChF,4CAA4C,CAAC;AAE/C,MAAM,cAAc,GAAG,OAAC,CAAC,MAAM,CAAC;IAC9B,WAAW,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,iDAAiD,CAAC;IAC5F,WAAW,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,mDAAmD,CAAC;IAC9F,aAAa,EAAE,OAAC,CAAC,KAAK,CAAC,OAAC,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,gDAAgD,CAAC;CAC9F,CAAC,CAAC;AAEH;;;;;GAKG;AACH,MAAa,cAAc;IACzB,YACmB,GAAiB,EACjB,UAA0C,EAC1C,aAAgC,EAChC,MAAc,EACd,aAA8B;QAJ9B,QAAG,GAAH,GAAG,CAAc;QACjB,eAAU,GAAV,UAAU,CAAgC;QAC1C,kBAAa,GAAb,aAAa,CAAmB;QAChC,WAAM,GAAN,MAAM,CAAQ;QACd,kBAAa,GAAb,aAAa,CAAiB;IAC9C,CAAC;IAEE,aAAa,CACjB,KAAe,EACf,OAA0B;;;YAE1B,MAAM,SAAS,GAAG,MAAA,OAAO,CAAC,KAAK,mCAAI,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC;YACrC,MAAM,WAAW,GACf,OAAO,CAAC,MAAM,CAAC,WAAW,IAAI,GAAG,OAAO,CAAC,MAAM,sBAAsB,CAAC;YACxE,MAAM,KAAK,GAAG,IAAI,uCAAkB,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC/D,MAAM,GAAG,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;YAE7D,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;YAClC,IAAI,MAAM,IAAI,MAAM,CAAC,GAAG,KAAK,GAAG,EAAE,CAAC;gBACjC,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,kCAAkC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,MAAM,gBAAgB,WAAW,EAAE,CAClG,CAAC;gBACF,OAAO,MAAM,CAAC;YAChB,CAAC;YACD,IAAI,MAAM,EAAE,CAAC;gBACX,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,qBAAqB,WAAW,8CAA8C,CAC/E,CAAC;YACJ,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC;gBAC9B,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,sFAAsF,CACvF,CAAC;YACJ,CAAC;YAED,8EAA8E;YAC9E,6EAA6E;YAC7E,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,MAAM,cAAc,GAA2C,EAAE,CAAC;YAClE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,iBAAQ,CAAC,WAAW,EAAE,EAAE,CAAC;oBAC3B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8DAA8D,CAAC,CAAC;oBACjF,MAAM;gBACR,CAAC;gBACD,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;gBACzC,IAAI,CAAC,IAAI;oBAAE,SAAS;gBACpB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACjB,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBACpB,IAAI,CAAC;wBACH,cAAc,CAAC,IAAA,qBAAW,EAAC,SAAS,EAAE,IAAI,CAAC,CAAC;4BAC1C,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;oBAC/C,CAAC;oBAAC,OAAO,KAAK,EAAE,CAAC;wBACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sCAAsC,IAAI,KAAK,KAAK,EAAE,CAAC,CAAC;oBAC3E,CAAC;gBACH,CAAC;YACH,CAAC;YAED,+CAA+C;YAC/C,MAAM,QAAQ,GAAG,IAAA,0BAAU,EAAC,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7C,MAAM,aAAa,GAAG,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,CAAC;YAEtE,uEAAuE;YACvE,sEAAsE;YACtE,6DAA6D;YAC7D,IAAI,QAAwB,CAAC;YAC7B,IAAI,UAAU,GAAG,IAAI,CAAC;YACtB,IAAI,CAAC;gBACH,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,aAAa,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;YACzE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,wEAAwE;oBACtE,wCAAwC,KAAK,EAAE,CAClD,CAAC;gBACF,QAAQ,GAAG,EAAE,WAAW,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC;gBACnE,UAAU,GAAG,KAAK,CAAC;YACrB,CAAC;YAED,MAAM,OAAO,GAAkB;gBAC7B,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBACrC,GAAG;gBACH,SAAS,EAAE,KAAK,CAAC,MAAM;gBACvB,aAAa;gBACb,cAAc;gBACd,QAAQ;gBACR,QAAQ;aACT,CAAC;YACF,IAAI,UAAU,EAAE,CAAC;gBACf,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAC1B,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,yBAAyB,QAAQ,CAAC,MAAM,uBAAuB;oBAC7D,GAAG,QAAQ,CAAC,WAAW,CAAC,MAAM,YAAY,QAAQ,CAAC,WAAW,CAAC,MAAM,WAAW;oBAChF,GAAG,QAAQ,CAAC,aAAa,CAAC,MAAM,gBAAgB,WAAW,EAAE,CAChE,CAAC;YACJ,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC;KAAA;IAED,gFAAgF;IAClE,UAAU,CAAC,IAAY;;;YACnC,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBAClD,IAAI,CAAC,MAAM;oBAAE,OAAO,EAAE,CAAC;gBACvB,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACpC,OAAO,CAAC,MAAA,GAAG,CAAC,MAAM,mCAAI,EAAE,CAAC;qBACtB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;qBACrB,IAAI,CAAC,IAAI,CAAC;qBACV,KAAK,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC;YACjC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,kCAAkC,IAAI,KAAK,KAAK,EAAE,CAAC,CAAC;gBACrE,OAAO,EAAE,CAAC;YACZ,CAAC;QACH,CAAC;KAAA;IAED;;;;;;;OAOG;IACK,UAAU,CAChB,KAAe,EACf,SAAiB,EACjB,OAA0B,EAC1B,IAAY;QAEZ,MAAM,OAAO,GAAG,KAAK;aAClB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACT,MAAM,GAAG,GAAG,IAAA,qBAAW,EAAC,SAAS,EAAE,CAAC,CAAC,CAAC;YACtC,IAAI,GAAG,GAAG,SAAS,CAAC;YACpB,IAAI,CAAC;gBACH,MAAM,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAC1B,GAAG,GAAG,GAAG,EAAE,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/C,CAAC;YAAC,WAAM,CAAC;gBACP,iEAAiE;YACnE,CAAC;YACD,OAAO,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;QACzB,CAAC,CAAC;aACD,IAAI,EAAE,CAAC;QACV,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,CACT,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,IAAI,IAAI,OAAO,CAAC,UAAU,CAAC,IAAI,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACnG,CAAC;QACF,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IAEa,gBAAgB,CAC5B,aAAqC,EACrC,QAAqB,EACrB,KAAe;;;YAEf,MAAM,SAAS,GACb,aAAa,CAAC,MAAM,GAAG,CAAC;gBACtB,CAAC,CAAC,aAAa;qBACV,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;qBACX,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;qBACrD,IAAI,CAAC,IAAI,CAAC;gBACf,CAAC,CAAC,SAAS,CAAC;YAChB,MAAM,QAAQ,GAAG,QAAQ;iBACtB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,KAAK,GAAG,CAAC;iBACpC,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,MAAM,QAAQ,GAAG,KAAK;iBACnB,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;iBACX,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;iBAC5D,IAAI,CAAC,MAAM,CAAC,CAAC;YAEhB,4EAA4E;YAC5E,6EAA6E;YAC7E,MAAM,QAAQ,GAAG,MAAM,CAAA,MAAA,IAAI,CAAC,aAAa,0CAAE,iBAAiB,CAAC;gBAC3D,SAAS;gBACT,QAAQ;gBACR,QAAQ;aACT,CAAC,CAAA,CAAC;YACH,MAAM,MAAM,GAAG,MAAA,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,MAAM,mCAAI,wBAAwB,CAAC;YAC5D,MAAM,IAAI,GACR,MAAA,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,IAAI,mCACd,wBAAwB,SAAS,MAAM;gBACrC,uCAAuC,QAAQ,MAAM;gBACrD,6BAA6B,QAAQ,EAAE,CAAC;YAE5C,MAAM,QAAQ,GAAiB;gBAC7B,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE;gBACnC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE;aAChC,CAAC;YAEF,4EAA4E;YAC5E,6EAA6E;YAC7E,gFAAgF;YAChF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;YAC3E,6EAA6E;YAC7E,0EAA0E;YAC1E,0EAA0E;YAC1E,OAAO,IAAA,qCAAiB,EAAC,MAAM,CAAC,CAAC;QACnC,CAAC;KAAA;CACF;AAtMD,wCAsMC;AAED,oEAAoE;AACpE,SAAS,gBAAgB,CACvB,OAAiC;;IAEjC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAsC,CAAC;IAC3D,KAAK,MAAM,OAAO,IAAI,OAAO,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,MAAA,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,mCAAI,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;YAChD,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,UAAU,CAAC;YACtB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YACT,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;SAC9B,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,GAAG,CAAC,EAA2B,CAAA,CAAC;SACzF,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;AACjD,CAAC"}

package/dist/core/corpus/CorpusProfileStore.js

@@ -0,0 +1,92 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CorpusProfileStore = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+/**
+ * Load/save the corpus profile sidecar (`<output>.corpus-profile.json`). Like
+ * {@link CheckpointService}, a missing or unparseable file is non-fatal — the
+ * caller just rebuilds the profile.
+ */
+class CorpusProfileStore {
+    constructor(path, logger) {
+        this.path = path;
+        this.logger = logger;
+    }
+    getPath() {
+        return this.path;
+    }
+    load() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!fs.existsSync(this.path))
+                return undefined;
+            try {
+                const parsed = JSON.parse(yield fs.promises.readFile(this.path, "utf-8"));
+                if (parsed && typeof parsed === "object" && parsed.glossary && parsed.key) {
+                    return parsed;
+                }
+                this.logger.warn(`Corpus profile at ${this.path} has an unexpected shape; ignoring`);
+                return undefined;
+            }
+            catch (error) {
+                this.logger.warn(`Could not read corpus profile at ${this.path} (ignored): ${error}`);
+                return undefined;
+            }
+        });
+    }
+    save(profile) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // The pre-pass runs before the output directory is created (that happens at
+            // export time), so ensure the sidecar's parent dir exists first.
+            const dir = path.dirname(this.path);
+            if (dir && !fs.existsSync(dir)) {
+                yield fs.promises.mkdir(dir, { recursive: true });
+            }
+            yield fs.promises.writeFile(this.path, JSON.stringify(profile, null, 2));
+        });
+    }
+}
+exports.CorpusProfileStore = CorpusProfileStore;
+//# sourceMappingURL=CorpusProfileStore.js.map

package/dist/core/corpus/CorpusProfileStore.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"CorpusProfileStore.js","sourceRoot":"","sources":["../../../src/core/corpus/CorpusProfileStore.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAI7B;;;;GAIG;AACH,MAAa,kBAAkB;IAC7B,YAA6B,IAAY,EAAmB,MAAc;QAA7C,SAAI,GAAJ,IAAI,CAAQ;QAAmB,WAAM,GAAN,MAAM,CAAQ;IAAG,CAAC;IAE9E,OAAO;QACL,OAAO,IAAI,CAAC,IAAI,CAAC;IACnB,CAAC;IAEK,IAAI;;YACR,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,OAAO,SAAS,CAAC;YAChD,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC1E,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,GAAG,EAAE,CAAC;oBAC1E,OAAO,MAAuB,CAAC;gBACjC,CAAC;gBACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,qBAAqB,IAAI,CAAC,IAAI,oCAAoC,CAAC,CAAC;gBACrF,OAAO,SAAS,CAAC;YACnB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,IAAI,CAAC,IAAI,eAAe,KAAK,EAAE,CAAC,CAAC;gBACtF,OAAO,SAAS,CAAC;YACnB,CAAC;QACH,CAAC;KAAA;IAEK,IAAI,CAAC,OAAsB;;YAC/B,4EAA4E;YAC5E,iEAAiE;YACjE,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpC,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC/B,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACpD,CAAC;YACD,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAC3E,CAAC;KAAA;CACF;AA/BD,gDA+BC"}

package/dist/core/corpus/index.js

@@ -0,0 +1,21 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./termFrequency"), exports);
+__exportStar(require("./CorpusProfileStore"), exports);
+__exportStar(require("./CorpusAnalyzer"), exports);
+__exportStar(require("./relPath"), exports);
+//# sourceMappingURL=index.js.map

package/dist/core/corpus/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/corpus/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,kDAAgC;AAChC,uDAAqC;AACrC,mDAAiC;AACjC,4CAA0B"}

package/dist/core/corpus/normalizeGlossary.js

@@ -0,0 +1,60 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeGlossary = normalizeGlossary;
+const DEFAULT_ENTITY_CAP = 20;
+const DEFAULT_RELATION_CAP = 15;
+/** Lowercase + `snake_case` a type/predicate token; '' when nothing survives. */
+function toSnakeType(raw) {
+    return raw
+        .normalize("NFKD")
+        .toLowerCase()
+        .replace(/[^a-z0-9]+/g, "_") // spaces, hyphens, slashes, unicode dashes → _
+        .replace(/^_+|_+$/g, ""); // trim leading/trailing underscores
+}
+/** Normalize a type/predicate list: snake_case, dedupe, reject has_*, cap. */
+function normalizeTypeList(raw, cap, rejectHasPrefix) {
+    const seen = new Set();
+    const out = [];
+    for (const item of raw !== null && raw !== void 0 ? raw : []) {
+        const s = toSnakeType(item);
+        if (!s)
+            continue;
+        // The banned attribute family (`has_format`, `has_length`, …). The legit
+        // base predicate `has_attribute` is supplied by the base set downstream, so
+        // dropping it from the glossary loses nothing.
+        if (rejectHasPrefix && s.startsWith("has_"))
+            continue;
+        if (seen.has(s))
+            continue;
+        seen.add(s);
+        out.push(s);
+        if (out.length >= cap)
+            break;
+    }
+    return out;
+}
+/** Trim + case-insensitive dedupe entity names, preserving first-seen casing. */
+function normalizeNames(raw) {
+    const seen = new Set();
+    const out = [];
+    for (const name of raw !== null && raw !== void 0 ? raw : []) {
+        const t = name.trim();
+        if (!t)
+            continue;
+        const key = t.toLowerCase();
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        out.push(t);
+    }
+    return out;
+}
+function normalizeGlossary(raw, caps = {}) {
+    var _a, _b;
+    return {
+        entityNames: normalizeNames(raw.entityNames),
+        entityTypes: normalizeTypeList(raw.entityTypes, (_a = caps.entityCap) !== null && _a !== void 0 ? _a : DEFAULT_ENTITY_CAP, false),
+        relationTypes: normalizeTypeList(raw.relationTypes, (_b = caps.relationCap) !== null && _b !== void 0 ? _b : DEFAULT_RELATION_CAP, true),
+    };
+}
+//# sourceMappingURL=normalizeGlossary.js.map

package/dist/core/corpus/normalizeGlossary.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"normalizeGlossary.js","sourceRoot":"","sources":["../../../src/core/corpus/normalizeGlossary.ts"],"names":[],"mappings":";;AA0EA,8CAiBC;AAnED,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEhC,iFAAiF;AACjF,SAAS,WAAW,CAAC,GAAW;IAC9B,OAAO,GAAG;SACP,SAAS,CAAC,MAAM,CAAC;SACjB,WAAW,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC,+CAA+C;SAC3E,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,oCAAoC;AAClE,CAAC;AAED,8EAA8E;AAC9E,SAAS,iBAAiB,CACxB,GAAyB,EACzB,GAAW,EACX,eAAwB;IAExB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,GAAG,aAAH,GAAG,cAAH,GAAG,GAAI,EAAE,EAAE,CAAC;QAC7B,MAAM,CAAC,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QAC5B,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,yEAAyE;QACzE,4EAA4E;QAC5E,+CAA+C;QAC/C,IAAI,eAAe,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAS;QACtD,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,SAAS;QAC1B,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACZ,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACZ,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG;YAAE,MAAM;IAC/B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,iFAAiF;AACjF,SAAS,cAAc,CAAC,GAAyB;IAC/C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,GAAG,aAAH,GAAG,cAAH,GAAG,GAAI,EAAE,EAAE,CAAC;QAC7B,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;QACtB,IAAI,CAAC,CAAC;YAAE,SAAS;QACjB,MAAM,GAAG,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;QAC5B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACd,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAgB,iBAAiB,CAC/B,GAAmB,EACnB,OAAqB,EAAE;;IAEvB,OAAO;QACL,WAAW,EAAE,cAAc,CAAC,GAAG,CAAC,WAAW,CAAC;QAC5C,WAAW,EAAE,iBAAiB,CAC5B,GAAG,CAAC,WAAW,EACf,MAAA,IAAI,CAAC,SAAS,mCAAI,kBAAkB,EACpC,KAAK,CACN;QACD,aAAa,EAAE,iBAAiB,CAC9B,GAAG,CAAC,aAAa,EACjB,MAAA,IAAI,CAAC,WAAW,mCAAI,oBAAoB,EACxC,IAAI,CACL;KACF,CAAC;AACJ,CAAC"}

package/dist/core/corpus/relPath.js

@@ -0,0 +1,52 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toRelPathId = toRelPathId;
+const path = __importStar(require("path"));
+/**
+ * Stable identity for a file: its path relative to the discovery root
+ * (`options.input`), posix-normalized — so relocating the input tree doesn't
+ * invalidate per-file caching. Mirrors `KnowledgeGraphBuilder.stablePathId`.
+ * Falls back to the raw path when the file lies outside the root.
+ */
+function toRelPathId(inputRoot, filePath) {
+    if (!inputRoot)
+        return filePath;
+    const rel = path.relative(inputRoot, filePath);
+    if (!rel || rel.startsWith("..") || path.isAbsolute(rel))
+        return filePath;
+    return rel.split(path.sep).join("/");
+}
+//# sourceMappingURL=relPath.js.map

package/dist/core/corpus/relPath.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"relPath.js","sourceRoot":"","sources":["../../../src/core/corpus/relPath.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAQA,kCAKC;AAbD,2CAA6B;AAE7B;;;;;GAKG;AACH,SAAgB,WAAW,CAAC,SAAiB,EAAE,QAAgB;IAC7D,IAAI,CAAC,SAAS;QAAE,OAAO,QAAQ,CAAC;IAChC,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAC/C,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC1E,OAAO,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACvC,CAAC"}

package/dist/core/corpus/termFrequency.js

@@ -0,0 +1,86 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_STOPWORDS = void 0;
+exports.countTerms = countTerms;
+/**
+ * Compact English stopword set. Not exhaustive — just enough to keep the top-N
+ * terms dominated by content words rather than glue. Kept inline (no dependency)
+ * to match the project's brutalist tendency.
+ */
+exports.DEFAULT_STOPWORDS = new Set([
+    "the", "and", "for", "are", "but", "not", "you", "all", "any", "can", "had",
+    "her", "was", "one", "our", "out", "his", "has", "him", "how", "its", "may",
+    "new", "now", "old", "see", "two", "way", "who", "did", "get", "let", "put",
+    "say", "she", "too", "use", "that", "this", "with", "have", "from", "they",
+    "will", "would", "there", "their", "what", "about", "which", "when", "make",
+    "like", "time", "just", "know", "into", "your", "some", "could", "them",
+    "than", "then", "look", "only", "come", "over", "also", "back", "after",
+    "work", "first", "well", "even", "want", "because", "these", "give", "most",
+    "been", "were", "such", "very", "more", "much", "many", "here", "does", "each",
+    "where", "while", "those", "being", "every", "should", "shall", "might",
+    "must", "ever", "thing", "things", "really", "actually", "going", "kind",
+    "okay", "yeah", "right", "lot", "got", "etc",
+]);
+/** Capitalized multiword runs (2–4 words) → likely proper-noun entity names. */
+const PROPER_NOUN_RE = /\b[A-Z][A-Za-z0-9'’]+(?:\s+[A-Z][A-Za-z0-9'’]+){1,3}\b/g;
+/** Word-ish single tokens (letters/digits, internal '-_'). */
+const WORD_RE = /[a-z0-9](?:[a-z0-9'’_-]*[a-z0-9])?/g;
+/**
+ * Sentence-openers / determiners that get capitalized at the start of a sentence
+ * and wrongly absorbed into a proper-noun run (e.g. "The Naive Bayes Classifier").
+ * Deliberately narrow — excludes content words that are also stopwords but can
+ * legitimately start a name (e.g. "New" in "New York").
+ */
+const LEADING_TRIM = new Set([
+    "the", "a", "an", "this", "that", "these", "those", "we", "you", "they",
+    "it", "he", "she", "but", "and", "or", "so", "then", "if", "when", "while",
+    "as", "in", "on", "at", "for", "to", "of",
+]);
+/** Drop leading determiner/opener words; keep the run only if ≥ 2 words remain. */
+function normalizeProperNoun(run) {
+    const words = run.trim().split(/\s+/);
+    while (words.length > 0 && LEADING_TRIM.has(words[0].toLowerCase())) {
+        words.shift();
+    }
+    return words.length >= 2 ? words.join(" ") : undefined;
+}
+/**
+ * Count term frequency across a set of texts and return the top-N ranked terms.
+ *
+ * Two signals are merged: lowercased single content words (stopwords / pure
+ * numbers / sub-`minLength` dropped) and original-cased capitalized multiword
+ * runs (proper-noun candidates). Deterministic: ties break alphabetically.
+ */
+function countTerms(texts, options = {}) {
+    var _a, _b, _c, _d, _e;
+    const topN = (_a = options.topN) !== null && _a !== void 0 ? _a : 100;
+    const minLength = (_b = options.minLength) !== null && _b !== void 0 ? _b : 3;
+    const stop = (_c = options.stopwords) !== null && _c !== void 0 ? _c : exports.DEFAULT_STOPWORDS;
+    const counts = new Map();
+    const bump = (term) => { var _a; return counts.set(term, ((_a = counts.get(term)) !== null && _a !== void 0 ? _a : 0) + 1); };
+    for (const text of texts) {
+        if (!text)
+            continue;
+        // Proper-noun candidates keep their original casing (canonical names).
+        for (const m of (_d = text.match(PROPER_NOUN_RE)) !== null && _d !== void 0 ? _d : []) {
+            const proper = normalizeProperNoun(m);
+            if (proper)
+                bump(proper);
+        }
+        // Single content words, lowercased.
+        for (const w of (_e = text.toLowerCase().match(WORD_RE)) !== null && _e !== void 0 ? _e : []) {
+            if (w.length < minLength)
+                continue;
+            if (/^\d+$/.test(w))
+                continue; // pure numbers carry no naming signal
+            if (stop.has(w))
+                continue;
+            bump(w);
+        }
+    }
+    return Array.from(counts.entries())
+        .map(([term, count]) => ({ term, count }))
+        .sort((a, b) => b.count - a.count || a.term.localeCompare(b.term))
+        .slice(0, topN);
+}
+//# sourceMappingURL=termFrequency.js.map

package/dist/core/corpus/termFrequency.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"termFrequency.js","sourceRoot":"","sources":["../../../src/core/corpus/termFrequency.ts"],"names":[],"mappings":";;;AA6DA,gCAiCC;AA5FD;;;;GAIG;AACU,QAAA,iBAAiB,GAAG,IAAI,GAAG,CAAS;IAC/C,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK;IAC3E,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK;IAC3E,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK;IAC3E,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IAC1E,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM;IAC3E,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;IACvE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO;IACvE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM;IAC3E,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IAC9E,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO;IACvE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM;IACxE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK;CAC7C,CAAC,CAAC;AAQH,gFAAgF;AAChF,MAAM,cAAc,GAAG,yDAAyD,CAAC;AACjF,8DAA8D;AAC9D,MAAM,OAAO,GAAG,qCAAqC,CAAC;AAEtD;;;;;GAKG;AACH,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC3B,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM;IACvE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO;IAC1E,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI;CAC1C,CAAC,CAAC;AAEH,mFAAmF;AACnF,SAAS,mBAAmB,CAAC,GAAW;IACtC,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACtC,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,EAAE,CAAC;QACpE,KAAK,CAAC,KAAK,EAAE,CAAC;IAChB,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACzD,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,UAAU,CACxB,KAAe,EACf,UAA6B,EAAE;;IAE/B,MAAM,IAAI,GAAG,MAAA,OAAO,CAAC,IAAI,mCAAI,GAAG,CAAC;IACjC,MAAM,SAAS,GAAG,MAAA,OAAO,CAAC,SAAS,mCAAI,CAAC,CAAC;IACzC,MAAM,IAAI,GAAG,MAAA,OAAO,CAAC,SAAS,mCAAI,yBAAiB,CAAC;IAEpD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,MAAM,IAAI,GAAG,CAAC,IAAY,EAAE,EAAE,WAAC,OAAA,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,MAAA,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,mCAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA,EAAA,CAAC;IAE7E,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,IAAI;YAAE,SAAS;QAEpB,uEAAuE;QACvE,KAAK,MAAM,CAAC,IAAI,MAAA,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,mCAAI,EAAE,EAAE,CAAC;YACjD,MAAM,MAAM,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;YACtC,IAAI,MAAM;gBAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3B,CAAC;QAED,oCAAoC;QACpC,KAAK,MAAM,CAAC,IAAI,MAAA,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,mCAAI,EAAE,EAAE,CAAC;YACxD,IAAI,CAAC,CAAC,MAAM,GAAG,SAAS;gBAAE,SAAS;YACnC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;gBAAE,SAAS,CAAC,sCAAsC;YACrE,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,SAAS;YAC1B,IAAI,CAAC,CAAC,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;SAChC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;SACzC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SACjE,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;AACpB,CAAC"}