@wanshi-kg/wanshi 0.1.0

package/dist/core/DirectoryProcessor.js

@@ -0,0 +1,725 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DirectoryProcessor = exports.FileDiscoveryService = void 0;
+const glob_1 = require("glob");
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const di_1 = require("./di");
+const corpus_1 = require("./corpus");
+const ReferenceResolver_1 = require("./knowledge/references/ReferenceResolver");
+const imageMetaGraph_1 = require("./knowledge/images/imageMetaGraph");
+const referenceExtraction_1 = require("./processor/readers/referenceExtraction");
+const ProcessedRegistry_1 = require("./processor/ProcessedRegistry");
+const WebReferenceProcessor_1 = require("./knowledge/references/web/WebReferenceProcessor");
+const CitationEvidenceProcessor_1 = require("./knowledge/references/citations/CitationEvidenceProcessor");
+const pipeline_1 = require("./pipeline");
+const canon_1 = require("./knowledge/canon");
+const shared_1 = require("../shared");
+const trace_1 = require("./trace");
+const cost_1 = require("./cost");
+const adapters_1 = require("./adapters");
+class FileDiscoveryService {
+    constructor(options, logger) {
+        this.logger = logger;
+        this.dir = options.input;
+        this.filter = options.filter;
+        this.exclude = options.exclude;
+    }
+    discover() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const patterns = this.filter.map(f => path.join(this.dir, f));
+            const files = yield (0, glob_1.glob)(patterns, { nodir: true, ignore: this.exclude });
+            if (files.length === 0) {
+                const message = `No files found matching pattern: ${this.filter}`;
+                this.logger.warn(message);
+                throw new Error(message);
+            }
+            this.logger.info(`Found ${files.length} files to process`);
+            return files;
+        });
+    }
+}
+exports.FileDiscoveryService = FileDiscoveryService;
+/**
+ * Refactored DirectoryProcessor using dependency injection
+ * Focuses on orchestration while delegating business logic to services
+ */
+class DirectoryProcessor {
+    constructor(container) {
+        this.container = container;
+    }
+    /**
+     * Process a directory and generate knowledge graphs
+     */
+    processDirectory(options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
+            const logger = yield this.container.resolve(di_1.TYPES.Logger);
+            const progress = yield this.container.resolve(di_1.TYPES.ProgressEmitter);
+            const fileDiscoveryService = yield this.container.resolve(di_1.TYPES.FileDiscoveryService);
+            logger.info(`Starting knowledge graph generation`);
+            logger.info(`Input: ${options.input}, Filter: ${options.filter}, Output: ${options.output}, Model: ${options.llm.model}`);
+            // Debug trace: open the run. A resumed run skips checkpointed chunks, so its
+            // trace is partial — flagged here.
+            trace_1.trace.emit({
+                stage: "run", type: "run_start",
+                output: options.output,
+                resumed: !!((_a = options.resume) === null || _a === void 0 ? void 0 : _a.enabled),
+                config: { model: options.llm.model, promptVersion: options.llm.promptVersion, grounding: (_b = options.grounding) === null || _b === void 0 ? void 0 : _b.mode },
+            });
+            // Cost meter: attach the resolved logger (configured in ContainerFactory without one).
+            if (cost_1.meter.enabled)
+                cost_1.meter.attachLogger(logger);
+            try {
+                // Orchestrate the workflow
+                const files = yield fileDiscoveryService.discover();
+                progress.emit({ type: "discovery", totalFiles: files.length });
+                // Rough pre-run cost estimate (bill-shock heads-up; the end tally is exact).
+                if (cost_1.meter.enabled)
+                    yield this.logCostEstimate(files, options, logger);
+                const knowledgeGraphs = yield this.processFiles(files, options);
+                if (shared_1.shutdown.isRequested()) {
+                    logger.warn("Run interrupted — merging and exporting the partial graph collected so far. Re-run with --resume to continue.");
+                }
+                progress.emit({ type: "merge", graphCount: knowledgeGraphs.length });
+                const mergedKG = yield this.mergeGraphs(knowledgeGraphs, logger);
+                const finalKG = yield this.applyGraphTransforms(mergedKG, options, logger);
+                const outputPath = yield this.exportKnowledgeGraph(finalKG, options);
+                progress.emit({
+                    type: "export",
+                    format: options.export.format,
+                    entities: finalKG.entities.length,
+                    relations: finalKG.relations.length,
+                    output: outputPath,
+                });
+                trace_1.trace.emit({
+                    stage: "export", type: "export",
+                    format: options.export.format,
+                    entities: finalKG.entities.length,
+                    relations: finalKG.relations.length,
+                });
+                this.logSuccess(finalKG, outputPath, logger);
+                // Cost meter: exact end-of-run tally + persist the resume-safe cumulative ledger.
+                if (cost_1.meter.enabled) {
+                    logger.info(cost_1.meter.summary());
+                    cost_1.meter.persistLedger();
+                }
+                progress.emit({
+                    type: "done",
+                    entities: finalKG.entities.length,
+                    relations: finalKG.relations.length,
+                    output: outputPath,
+                    interrupted: shared_1.shutdown.isRequested(),
+                });
+            }
+            catch (error) {
+                this.handleError(error, options.logging.debug, logger);
+                progress.emit({
+                    type: "error",
+                    message: error instanceof Error ? error.message : String(error),
+                });
+                throw error;
+            }
+        });
+    }
+    /** Rough pre-run cost projection from discovered file sizes (bytes≈chars; no double read pass). */
+    logCostEstimate(files, options, logger) {
+        return __awaiter(this, void 0, void 0, function* () {
+            let totalChars = 0;
+            for (const f of files) {
+                try {
+                    totalChars += (yield fs.promises.stat(f)).size;
+                }
+                catch (_a) {
+                    /* unreadable/removed — skip */
+                }
+            }
+            const est = cost_1.meter.estimate(totalChars, options.chunking.size, options.llm.model);
+            const tokens = est.estPromptTokens + est.estCompletionTokens;
+            const money = est.priced
+                ? `~${options.cost.currency} ${est.estCost.toFixed(est.estCost < 1 ? 4 : 2)}`
+                : `no price set (shown as ${options.cost.currency} 0)`;
+            logger.info(`Cost estimate (rough): ~${est.estChunks} chunk(s), ~${tokens.toLocaleString()} tokens for ` +
+                `model '${options.llm.model}' — ${money}. Resume-cached chunks reduce actual spend; the ` +
+                `end-of-run tally is exact.`);
+        });
+    }
+    /**
+     * Process multiple files and generate knowledge graphs
+     */
+    processFiles(files, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const knowledgeGraphs = [];
+            const logger = yield this.container.resolve(di_1.TYPES.Logger);
+            const progress = yield this.container.resolve(di_1.TYPES.ProgressEmitter);
+            // Load a prior output graph (if any) to seed retrieval CONTEXT only. It must
+            // NOT enter the merge set: re-merging already-merged output into a fresh run
+            // double-counts entities/observations on a plain (no --resume) re-run.
+            const priorGraphs = yield this.loadPriorGraphs(options.output, logger);
+            // Optional corpus analysis pre-pass: build/load a corpus-specific glossary
+            // (and cached per-file classification) once, before extraction.
+            const corpusProfile = yield this.buildCorpusProfile(files, options, logger);
+            const fileProcessor = yield this.container.resolve(di_1.TYPES.FileProcessor);
+            const kgBuilder = yield this.container.resolve(di_1.TYPES.KnowledgeGraphBuilder);
+            // Deterministic AST symbol seed (Phase 8): seed code definitions + exported
+            // members (and calls/imports edges) per file so the LLM augments the symbol
+            // set rather than originating it. Content-hash cached across the run.
+            const astSeed = options.ast.mode === "enabled"
+                ? yield this.container.resolve(di_1.TYPES.AstSeedService)
+                : undefined;
+            yield (astSeed === null || astSeed === void 0 ? void 0 : astSeed.loadCache());
+            // Structured-emit adapters (data-sink track): a graph-native source (e.g. a
+            // .db) maps directly to graph fragments, bypassing the LLM. Empty registry =
+            // every file takes the normal read→build path (default).
+            const structuredAdapters = yield this.container
+                .resolve(di_1.TYPES.StructuredAdapterRegistry)
+                .catch(() => new adapters_1.StructuredAdapterRegistry()); // empty = off-path default
+            // Reference & link resolution: the corpus-relative path set drives link
+            // resolution (resolved-flag + follow targets). In follow mode it spans the
+            // WHOLE input tree (links can point outside the glob); otherwise the glob set.
+            const follow = options.references.follow.enabled;
+            const internalLinksOn = options.references.internalLinks.enabled || follow;
+            let corpusRelPaths;
+            if (follow) {
+                const allInput = yield new FileDiscoveryService({ input: options.input, filter: ["**/*"], exclude: options.exclude }, logger)
+                    .discover()
+                    .catch(() => []);
+                corpusRelPaths = new Set(allInput.map((f) => (0, corpus_1.toRelPathId)(options.input, f)));
+            }
+            else {
+                corpusRelPaths = internalLinksOn
+                    ? new Set(files.map((f) => (0, corpus_1.toRelPathId)(options.input, f)))
+                    : new Set();
+            }
+            // Phase 1 — class-3 external web fetcher (opt-in; constructed only when
+            // references.web.enabled, so a default run never builds the network layer).
+            const webProc = options.references.web.enabled
+                ? yield this.buildWebProcessor(options, fileProcessor, kgBuilder, corpusProfile, logger)
+                : null;
+            // Phase 2 — citation span-fetch (opt-in; constructed only when
+            // references.citations.fetch.enabled). Resolves id-bearing cites to OA full
+            // text, span-selects the citing claim's evidence, and labels the edge.
+            const citeProc = options.references.citations.fetch.enabled
+                ? yield this.buildCitationProcessor(options, fileProcessor, kgBuilder, corpusProfile, logger)
+                : null;
+            // Worklist with a processed-file registry: the same file is read/extracted at
+            // most once however it's reached (overlapping globs, reference-following). The
+            // queue is seeded from follow.seeds (a crawl) or the discovered glob set, and
+            // (in follow mode) grows as internal links are resolved to existing files.
+            const registry = new ProcessedRegistry_1.ProcessedRegistry();
+            const queued = new Set();
+            const queue = [];
+            const enqueue = (file, depth) => {
+                const id = (0, corpus_1.toRelPathId)(options.input, file);
+                if (registry.has(id) || queued.has(id))
+                    return;
+                queued.add(id);
+                queue.push({ file, depth });
+            };
+            const seeds = options.references.follow.seeds;
+            if (follow && seeds.length) {
+                for (const s of seeds) {
+                    const abs = path.resolve(options.input, s);
+                    if (fs.existsSync(abs))
+                        enqueue(abs, 0);
+                    else
+                        logger.warn(`reference-follow seed not found, skipping: ${s}`);
+                }
+            }
+            else {
+                for (const f of files)
+                    enqueue(f, 0);
+            }
+            const { maxFiles, maxDepth } = options.references.follow;
+            let index = 0;
+            while (queue.length > 0) {
+                // Cooperative interrupt: stop before starting the next file so the
+                // partial graph accumulated so far can still be merged and exported.
+                if (shared_1.shutdown.isRequested()) {
+                    logger.warn(`Interrupted — flushing partial graph (${registry.size} files processed)`);
+                    break;
+                }
+                if (follow && registry.size >= maxFiles) {
+                    logger.warn(`reference-follow reached maxFiles=${maxFiles}; stopping discovery`);
+                    break;
+                }
+                const { file, depth } = queue.shift();
+                const id = (0, corpus_1.toRelPathId)(options.input, file);
+                if (registry.has(id))
+                    continue; // already processed via another path
+                index += 1;
+                const total = registry.size + queue.length + 1;
+                progress.emit({ type: "file_start", index, total, path: file });
+                try {
+                    // Retrieval sees prior output + graphs built so far this run; merge sees
+                    // only what's built this run (knowledgeGraphs).
+                    const retrievalContext = [...priorGraphs, ...knowledgeGraphs];
+                    const { graphs: fileGraphs, links: fileLinks, citations: fileCitations } = yield this.processFile(file, options, fileProcessor, kgBuilder, retrievalContext, logger, corpusProfile, astSeed, corpusRelPaths, structuredAdapters);
+                    registry.mark(id);
+                    knowledgeGraphs.push(...fileGraphs);
+                    // Reference-driven ingestion: enqueue resolved internal-link targets that
+                    // exist in the corpus and haven't been processed/queued. Network-free —
+                    // external targets are skipped (that's the web fetcher below).
+                    if (follow && (maxDepth === 0 || depth < maxDepth)) {
+                        for (const link of fileLinks) {
+                            if ((0, referenceExtraction_1.isExternalTarget)(link.target))
+                                continue;
+                            const rel = (0, ReferenceResolver_1.resolveInternalTarget)(link, id, corpusRelPaths);
+                            if (rel && !registry.has(rel) && !queued.has(rel)) {
+                                enqueue(path.resolve(options.input, rel), depth + 1);
+                            }
+                        }
+                    }
+                    // Phase 1 — class-3 external web: fetch this file's allowlisted external
+                    // links (gated), extract, emit `references` edges. Depth-1 (fetched pages
+                    // are not re-crawled). Offline unless references.web is enabled.
+                    if (webProc) {
+                        const webGraph = yield webProc.process(id, fileLinks, options.description);
+                        if (webGraph)
+                            knowledgeGraphs.push(webGraph);
+                    }
+                    // Phase 2 — citation span-fetch: resolve this file's id-bearing cites to OA
+                    // full text, fold content + label faithfulness. Offline unless enabled.
+                    if (citeProc) {
+                        const citeGraph = yield citeProc.process(id, file, fileCitations);
+                        if (citeGraph)
+                            knowledgeGraphs.push(citeGraph);
+                    }
+                    const entities = fileGraphs.reduce((n, g) => n + g.entities.length, 0);
+                    const relations = fileGraphs.reduce((n, g) => n + g.relations.length, 0);
+                    progress.emit({ type: "file_complete", index, total, path: file, entities, relations });
+                    if (options.logging.debug) {
+                        yield this.writeIntermediateResults(knowledgeGraphs, options.output);
+                    }
+                }
+                catch (error) {
+                    registry.mark(id); // don't retry a hard-failing file in this run
+                    this.handleFileError(file, error, options.logging.debug, logger);
+                    progress.emit({ type: "file_complete", index, total: registry.size + queue.length, path: file, entities: 0, relations: 0 });
+                }
+            }
+            // Persist the AST symbol cache so an unchanged file is a no-op next run.
+            yield (astSeed === null || astSeed === void 0 ? void 0 : astSeed.saveCache());
+            // Surface chunks whose extraction failed: they were left uncheckpointed (so
+            // --resume retries them) and must not pass silently as "done-and-empty". The
+            // partial graph still merges/exports; the run exits non-zero (KG-02).
+            const failedChunks = kgBuilder.getFailedChunks();
+            if (failedChunks.length > 0) {
+                logger.warn(`${failedChunks.length} chunk(s) failed extraction and were left uncheckpointed — ` +
+                    `re-run with --resume to retry them:`);
+                for (const f of failedChunks) {
+                    logger.warn(`  - ${f.filePath} [chunk ${f.chunkIndex}/${f.totalChunks}]: ${f.error}`);
+                }
+                process.exitCode = 1;
+            }
+            // Surface claims the inline grounding gate rejected (WI3 manifest trace):
+            // in `drop` mode they were removed from the graph, in `flag` mode annotated
+            // and kept — either way they must leave a visible trace, not vanish.
+            const rejections = kgBuilder.getGroundingRejections();
+            if (rejections.length > 0) {
+                const dropped = rejections.filter((r) => r.dropped).length;
+                logger.warn(`Grounding gate flagged ${rejections.length} ungrounded claim(s)` +
+                    (dropped > 0 ? ` (${dropped} dropped, ${rejections.length - dropped} flagged)` : ` (all flagged)`) +
+                    `:`);
+                for (const r of rejections) {
+                    logger.debug(`  - [${r.kind}] ${r.subject} (score ${r.score.toFixed(2)}, ` +
+                        `${r.dropped ? "dropped" : "flagged"}) in ${r.filePath} [chunk ${r.chunkIndex}]: ${r.claim}`);
+                }
+            }
+            return knowledgeGraphs;
+        });
+    }
+    /**
+     * Load a previously-written output graph for retrieval seeding. Tolerates both
+     * the current single-graph object (`{entities, relations}`) and a legacy array
+     * of per-file graphs. Returns [] (and warns) when missing/unparseable — the
+     * prior graph is a retrieval nicety, never required.
+     */
+    loadPriorGraphs(outputPath, logger) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!outputPath || !fs.existsSync(outputPath))
+                return [];
+            const raw = fs.readFileSync(outputPath, "utf-8");
+            // JSONL / mcp-jsonl outputs aren't valid JSON documents — parse them
+            // line-by-line (KG-11) instead of warning every run. Route by extension, and
+            // also fall back to the JSONL reader if a `.json` somehow fails to parse.
+            const isJsonl = /\.(jsonl|mcp-jsonl)$/i.test(outputPath);
+            if (isJsonl) {
+                const { JsonlExportStrategy } = yield Promise.resolve().then(() => __importStar(require("./export/strategies/JsonlExportStrategy")));
+                const graph = JsonlExportStrategy.fromJSONL(raw);
+                return graph.entities.length || graph.relations.length ? [graph] : [];
+            }
+            try {
+                const parsed = JSON.parse(raw);
+                if (Array.isArray(parsed))
+                    return parsed;
+                if (parsed && typeof parsed === "object" && Array.isArray(parsed.entities)) {
+                    return [parsed];
+                }
+                return [];
+            }
+            catch (_a) {
+                // Not a JSON document — try JSONL before giving up (covers a mislabeled file).
+                const { JsonlExportStrategy } = yield Promise.resolve().then(() => __importStar(require("./export/strategies/JsonlExportStrategy")));
+                const graph = JsonlExportStrategy.fromJSONL(raw);
+                if (graph.entities.length || graph.relations.length)
+                    return [graph];
+                logger.warn(`Could not load prior graph at ${outputPath} for retrieval context (ignored)`);
+                return [];
+            }
+        });
+    }
+    /**
+     * Process a single file
+     */
+    processFile(file_1, options_1, fileProcessor_1, kgBuilder_1, existingGraphs_1, logger_1, corpusProfile_1, astSeed_1) {
+        return __awaiter(this, arguments, void 0, function* (file, options, fileProcessor, kgBuilder, existingGraphs, logger, corpusProfile, astSeed, corpusRelPaths = new Set(), structuredAdapters) {
+            var _a, _b, _c, _d, _e;
+            logger.info(`Processing: ${file}`);
+            // Structured-emit path (data-sink track): if an adapter claims this file, it
+            // maps the source DIRECTLY to graph fragments (bypassing read→chunk→LLM). The
+            // fragment still enters the per-file graphs[] union → merge/canon.
+            const adapter = structuredAdapters === null || structuredAdapters === void 0 ? void 0 : structuredAdapters.match(file);
+            if (adapter) {
+                logger.info(`Structured adapter '${adapter.id}' handling ${file} (graph-native, no LLM)`);
+                const graph = yield adapter.extract(file);
+                if (trace_1.trace.enabled) {
+                    trace_1.trace.emit({
+                        stage: "ingest", type: "chunk", chunkId: `${file}#0`, file,
+                        chunkIndex: 0, totalChunks: 1, reader: `adapter:${adapter.id}`, contentLength: 0,
+                    });
+                }
+                return { graphs: graph ? [graph] : [], links: [], citations: [] };
+            }
+            // Reuse the pre-pass's cached classification for this file when available.
+            const cachedClasses = corpusProfile === null || corpusProfile === void 0 ? void 0 : corpusProfile.perFileClasses[(0, corpus_1.toRelPathId)(options.input, file)];
+            const processedFile = yield fileProcessor.processFile(file, cachedClasses);
+            // A reader can signal a graceful skip (BinaryReader for binary/unknown
+            // files) — honor it before the "no content extracted" guard turns an empty
+            // read into a per-file error.
+            if ((_a = processedFile.metadata) === null || _a === void 0 ? void 0 : _a.skip) {
+                logger.info(`Skipped ${file} (binary / no extractable text)`);
+                return { graphs: [], links: [], citations: [] };
+            }
+            this.validateProcessedFile(processedFile, file, logger);
+            const retrieve = yield this.buildRetriever(processedFile, file, existingGraphs, options);
+            const promptManager = (yield this.container.resolve(di_1.TYPES.PromptManager));
+            const systemPrompt = yield promptManager.getSystemPrompt(options.input, options.filter.join(', '), options.description, (_b = processedFile.metadata) === null || _b === void 0 ? void 0 : _b.classes, corpusProfile === null || corpusProfile === void 0 ? void 0 : corpusProfile.glossary, options.pipeline.extraction.openPredicate);
+            const graphs = yield kgBuilder.build(processedFile, systemPrompt, retrieve, corpusProfile === null || corpusProfile === void 0 ? void 0 : corpusProfile.glossary);
+            // Append the deterministic AST symbol seed (Phase 8) so it merges with the
+            // LLM's per-chunk graphs — the model augments the symbol set, not originates it.
+            const seed = astSeed ? yield astSeed.seedGraph(processedFile) : null;
+            if (seed)
+                graphs.push(seed);
+            // Deterministic image metadata (EXIF/C2PA): graph facts that AUGMENT the VLM's
+            // read of an image rather than replacing it (sourceAdapter exif/c2pa, confidence).
+            // No-op (returns null) unless a reader stashed metadata.exif/metadata.c2pa.
+            const imageGraph = (0, imageMetaGraph_1.buildImageMetaGraph)(processedFile, options.input);
+            if (imageGraph)
+                graphs.push(imageGraph);
+            // Deterministic reference edges (Phase 0, network-free): internal links +
+            // citations the document already contains, resolved against the corpus.
+            // Merges with the LLM graphs like the AST seed above. Following auto-implies
+            // internal-link resolution (you can't follow links you didn't extract).
+            const internalLinksOn = options.references.internalLinks.enabled || options.references.follow.enabled;
+            // When citation-fetch is on (Phase 2), the CitationEvidenceProcessor OWNS the
+            // `cites` edges (resolved + faithfulness) — so the network-free resolver stands
+            // down on citations to avoid emitting a competing resolved:false edge.
+            const fetchOwnsCites = options.references.citations.fetch.enabled;
+            const citationsForResolver = options.references.citations.enabled && !fetchOwnsCites;
+            if (internalLinksOn || citationsForResolver) {
+                const refGraph = (0, ReferenceResolver_1.buildReferenceGraph)(processedFile, corpusRelPaths, options.input, {
+                    internalLinks: internalLinksOn,
+                    citations: citationsForResolver,
+                });
+                if (refGraph)
+                    graphs.push(refGraph);
+            }
+            const refs = (_c = processedFile.metadata) === null || _c === void 0 ? void 0 : _c.references;
+            return { graphs, links: (_d = refs === null || refs === void 0 ? void 0 : refs.links) !== null && _d !== void 0 ? _d : [], citations: (_e = refs === null || refs === void 0 ? void 0 : refs.citations) !== null && _e !== void 0 ? _e : [] };
+        });
+    }
+    /**
+     * Build the Phase-1 web reference processor: the DI-managed gated fetcher +
+     * fetch cache, plus an extract closure that runs a fetched page through the
+     * normal reader + builder (content only — no reference-resolver/follow on
+     * fetched pages = depth-1).
+     */
+    buildWebProcessor(options, fileProcessor, kgBuilder, corpusProfile, logger) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const fetcher = yield this.container.resolve(di_1.TYPES.GatedFetcher);
+            const cache = yield this.container.resolve(di_1.TYPES.FetchCacheService);
+            const promptManager = (yield this.container.resolve(di_1.TYPES.PromptManager));
+            const extract = (tempPath) => __awaiter(this, void 0, void 0, function* () {
+                var _a, _b;
+                const pf = yield fileProcessor.processFile(tempPath);
+                if (((_a = pf.metadata) === null || _a === void 0 ? void 0 : _a.skip) || !pf.chunks.length)
+                    return [];
+                const systemPrompt = yield promptManager.getSystemPrompt(options.input, options.filter.join(", "), options.description, (_b = pf.metadata) === null || _b === void 0 ? void 0 : _b.classes, corpusProfile === null || corpusProfile === void 0 ? void 0 : corpusProfile.glossary, options.pipeline.extraction.openPredicate);
+                return kgBuilder.build(pf, systemPrompt, undefined, corpusProfile === null || corpusProfile === void 0 ? void 0 : corpusProfile.glossary);
+            });
+            return new WebReferenceProcessor_1.WebReferenceProcessor(fetcher, cache, extract, logger);
+        });
+    }
+    /**
+     * Build the Phase-2 citation evidence processor: a PDF-capable gated fetcher +
+     * its own fetch cache + the id→OA resolver, an extract closure that runs a
+     * fetched cited PDF through the normal reader (chunks for span-select) + builder
+     * (content folded onto the cited-work node), the embedding provider for
+     * span-select, and (optionally) GROBID for marker→claim linking + MiniCheck for
+     * the faithfulness label.
+     */
+    buildCitationProcessor(options, fileProcessor, kgBuilder, corpusProfile, logger) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const { CitationResolver } = yield Promise.resolve().then(() => __importStar(require("./knowledge/references/citations/CitationResolver")));
+            const fetcher = yield this.container.resolve(di_1.TYPES.CitationFetcher);
+            const cache = yield this.container.resolve(di_1.TYPES.CitationFetchCache);
+            const resolver = yield this.container.resolve(di_1.TYPES.CitationResolver);
+            const embeddings = yield this.container.resolve(di_1.TYPES.EmbeddingService);
+            const promptManager = (yield this.container.resolve(di_1.TYPES.PromptManager));
+            const cfg = options.references.citations;
+            const grobid = cfg.grobid.enabled
+                ? yield this.container.resolve(di_1.TYPES.GrobidClient)
+                : null;
+            if (grobid && !(yield grobid.isAlive())) {
+                logger.warn(`GROBID not reachable at ${cfg.grobid.url} — citation span-select/faithfulness disabled (id-bearing fetch still runs). Start it with: docker run -p 8070:8070 lfoppiano/grobid`);
+            }
+            let faithfulness = null;
+            if (cfg.fetch.minicheck) {
+                const { MiniCheckGroundingChecker } = yield Promise.resolve().then(() => __importStar(require("./knowledge/grounding")));
+                faithfulness = new MiniCheckGroundingChecker({ model: cfg.fetch.minicheckModel, host: cfg.fetch.minicheckHost, min: 0.5, escalateAbove: 1.1 }, logger);
+            }
+            const extract = (tempPath) => __awaiter(this, void 0, void 0, function* () {
+                var _a, _b;
+                const pf = yield fileProcessor.processFile(tempPath);
+                if (((_a = pf.metadata) === null || _a === void 0 ? void 0 : _a.skip) || !pf.chunks.length)
+                    return { chunks: [], graphs: [] };
+                const chunks = pf.chunks.map((ch) => ch.content);
+                const systemPrompt = yield promptManager.getSystemPrompt(options.input, options.filter.join(", "), options.description, (_b = pf.metadata) === null || _b === void 0 ? void 0 : _b.classes, corpusProfile === null || corpusProfile === void 0 ? void 0 : corpusProfile.glossary, options.pipeline.extraction.openPredicate);
+                const graphs = yield kgBuilder.build(pf, systemPrompt, undefined, corpusProfile === null || corpusProfile === void 0 ? void 0 : corpusProfile.glossary);
+                return { chunks, graphs };
+            });
+            return new CitationEvidenceProcessor_1.CitationEvidenceProcessor(fetcher, cache, resolver, extract, embeddings, logger, {
+                grobid,
+                faithfulness,
+                uncertainBand: cfg.fetch.uncertainBand,
+            });
+        });
+    }
+    /**
+     * Run the optional corpus analysis pre-pass (term frequency + cached
+     * classification + LLM glossary). Returns undefined when disabled or on
+     * failure — profiling is an enhancement, never required.
+     */
+    buildCorpusProfile(files, options, logger) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (options.corpus.profiling !== "enabled")
+                return undefined;
+            try {
+                logger.info("Corpus analysis pre-pass enabled — profiling corpus before extraction");
+                const analyzer = yield this.container.resolve(di_1.TYPES.CorpusAnalyzer);
+                return yield analyzer.analyzeOrLoad(files, options);
+            }
+            catch (error) {
+                logger.warn(`Corpus pre-pass failed (continuing without a glossary): ${error}`);
+                return undefined;
+            }
+        });
+    }
+    /**
+     * Validate processed file content
+     */
+    validateProcessedFile(processedFile, filePath, logger) {
+        var _a;
+        if (!((_a = processedFile.chunks) === null || _a === void 0 ? void 0 : _a.length)) {
+            logger.warn(`No content extracted from: ${filePath}`);
+            throw new Error(`No content extracted from file: ${filePath}`);
+        }
+    }
+    /**
+     * Build a retrieval function for a file, or undefined when retrieval is
+     * disabled / there's no existing graph to search.
+     *
+     * - `retrievalScope: "chunk"` (default) returns a function that retrieves
+     *   context per chunk using that chunk's own content.
+     * - `retrievalScope: "file"` retrieves once from the first chunk and reuses
+     *   it for every chunk (legacy behavior).
+     */
+    buildRetriever(processedFile, filePath, existingGraphs, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.shouldUseRetrieval(options) || existingGraphs.length === 0) {
+                return undefined;
+            }
+            const searchService = yield this.container.resolve(di_1.TYPES.KnowledgeGraphSearch);
+            const searchOptions = {
+                limit: options.retrieval.limit,
+                includeObservations: true,
+            };
+            const search = (content) => searchService.searchByFileContent(content, filePath, existingGraphs, searchOptions);
+            if (options.retrieval.scope === "file") {
+                // Retrieve once from the first chunk, reuse for all chunks.
+                const context = yield search(processedFile.chunks[0].content);
+                return () => __awaiter(this, void 0, void 0, function* () { return context; });
+            }
+            // Default: per-chunk retrieval.
+            return (chunkContent) => search(chunkContent);
+        });
+    }
+    /**
+     * Determine if retrieval should be used
+     */
+    shouldUseRetrieval(options) {
+        // Fix the conflicting boolean pairs issue
+        if (options.retrieval.mode === "disabled")
+            return false;
+        if (options.retrieval.mode === "enabled")
+            return true;
+        return true; // Auto to true
+    }
+    /**
+     * Merge multiple knowledge graphs
+     */
+    mergeGraphs(graphs, logger) {
+        return __awaiter(this, void 0, void 0, function* () {
+            logger.info(`Merging ${graphs.length} knowledge graphs`);
+            const merger = yield this.container.resolve(di_1.TYPES.KnowledgeGraphMerger);
+            return yield merger.merge(graphs);
+        });
+    }
+    /**
+     * Run the post-extraction graph→graph transform pipeline (grounding gate,
+     * canonicalization) over the merged graph, in the order from `pipeline.stages`.
+     * A no-op when no transform is enabled — the providers resolved here are the
+     * same singletons extraction/merge already built, so the baseline path returns
+     * the merged graph unchanged.
+     */
+    applyGraphTransforms(graph, options, logger) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const transforms = [
+                new pipeline_1.GroundingTransform(),
+                new canon_1.Canonicalizer(),
+                new pipeline_1.RelationFilterTransform(), // after canon: endpoints are canonical before pairing
+            ];
+            const ctx = {
+                options,
+                embeddings: yield this.container.resolve(di_1.TYPES.EmbeddingService),
+                llm: yield this.container.resolve(di_1.TYPES.LLMService),
+                logger,
+            };
+            const runner = new pipeline_1.PipelineRunner(transforms, ctx);
+            if (!runner.hasWork())
+                return graph;
+            return runner.run(graph);
+        });
+    }
+    /**
+     * Export knowledge graph in the requested format
+     */
+    exportKnowledgeGraph(knowledgeGraph, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.ensureOutputDirectory(options.output);
+            const exporter = yield this.container.resolve(di_1.TYPES.KnowledgeGraphExportService);
+            const exportFormat = options.export.format;
+            if (!exporter.isFormatSupported(exportFormat)) {
+                throw new Error(`Unsupported export format: ${exportFormat}. Supported: ${exporter
+                    .getSupportedFormats()
+                    .join(", ")}`);
+            }
+            const outputContent = exporter.export(knowledgeGraph, exportFormat, options);
+            const outputPath = this.getOutputPath(options.output, exportFormat);
+            yield fs.promises.writeFile(outputPath, outputContent);
+            return outputPath;
+        });
+    }
+    /**
+     * Ensure output directory exists
+     */
+    ensureOutputDirectory(outputPath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const outputDir = path.dirname(outputPath);
+            if (!fs.existsSync(outputDir)) {
+                fs.mkdirSync(outputDir, { recursive: true });
+            }
+        });
+    }
+    /**
+     * Get the final output path with correct extension
+     */
+    getOutputPath(originalPath, format) {
+        return originalPath.endsWith(`.${format}`)
+            ? originalPath
+            : originalPath.replace(/\.[^.]+$/, `.${format}`);
+    }
+    /**
+     * Write intermediate results for debugging
+     */
+    writeIntermediateResults(graphs, outputPath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const tmpPath = outputPath + ".tmp";
+            yield fs.promises.writeFile(tmpPath, JSON.stringify(graphs, null, 2));
+        });
+    }
+    /**
+     * Handle file processing errors
+     */
+    handleFileError(file, error, debug, logger) {
+        logger.error(`Failed to process file ${file}: ${error.message || error}`);
+    }
+    /**
+     * Handle general processing errors
+     */
+    handleError(error, debug, logger) {
+        logger.error(`Failed to process directory: ${error.message || error}`);
+    }
+    /**
+     * Log successful completion
+     */
+    logSuccess(knowledgeGraph, outputPath, logger) {
+        logger.info(`Knowledge graph saved to: ${outputPath}`);
+        logger.info(`Final graph: ${knowledgeGraph.entities.length} entities, ${knowledgeGraph.relations.length} relations`);
+    }
+}
+exports.DirectoryProcessor = DirectoryProcessor;
+//# sourceMappingURL=DirectoryProcessor.js.map