@wanshi-kg/wanshi 0.1.0

package/dist/core/knowledge/references/web/GatedFetcher.js

@@ -0,0 +1,228 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.GatedFetcher = void 0;
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const crypto = __importStar(require("crypto"));
+const zod_1 = require("zod");
+/**
+ * The shared Phase-1 network primitive: fetch one external URL behind layered,
+ * always-on guards. Returns a staged temp file only when EVERY guard passes;
+ * any failure yields `{ resolved:false, reason }` (never fabricated content).
+ *
+ * Guard order (cheapest/safest first): allowlist (empty ⇒ no fetch, the master
+ * switch) → rejectlist → robots.txt → per-run budget → timed fetch → content-type
+ * → size cap → LLM relevance pre-check. Offline-first: this only runs when the
+ * caller opted into `references.web`; a default run never constructs it.
+ */
+const USER_AGENT = "wanshi-reference-fetcher/1 (+https://github.com/AlexSabaka/wanshi)";
+class GatedFetcher {
+    constructor(opts, llm, logger, tempDir = "./temp", fetchFn = globalThis.fetch) {
+        this.opts = opts;
+        this.llm = llm;
+        this.logger = logger;
+        this.tempDir = tempDir;
+        this.fetchFn = fetchFn;
+        this.fetched = 0; // per-run budget counter
+        this.robotsCache = new Map(); // host → Disallow paths
+    }
+    /** True if the URL's host/prefix is allowlisted (and not rejectlisted). */
+    allowed(url) {
+        if (this.opts.rejectlist.some((p) => this.matches(url, p)))
+            return false;
+        return this.opts.allowlist.some((p) => this.matches(url, p));
+    }
+    matches(url, pattern) {
+        if (url.startsWith(pattern))
+            return true; // prefix match
+        try {
+            const host = new URL(url).hostname;
+            return host === pattern || host.endsWith(`.${pattern}`); // domain / subdomain
+        }
+        catch (_a) {
+            return false;
+        }
+    }
+    /** Minimal robots.txt check for `User-agent: *` Disallow rules (fail-open). */
+    robotsBlocked(url) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.opts.robots)
+                return false;
+            let u;
+            try {
+                u = new URL(url);
+            }
+            catch (_a) {
+                return false;
+            }
+            if (!this.robotsCache.has(u.host)) {
+                const disallow = [];
+                try {
+                    const res = yield this.timedFetch(`${u.protocol}//${u.host}/robots.txt`);
+                    if (res.ok) {
+                        let inStar = false;
+                        for (const raw of (yield res.text()).split("\n")) {
+                            const line = raw.replace(/#.*$/, "").trim();
+                            const ua = /^user-agent:\s*(.+)$/i.exec(line);
+                            if (ua)
+                                inStar = ua[1].trim() === "*";
+                            else if (inStar) {
+                                const d = /^disallow:\s*(.*)$/i.exec(line);
+                                if (d && d[1].trim())
+                                    disallow.push(d[1].trim());
+                            }
+                        }
+                    }
+                }
+                catch (_b) {
+                    /* fail-open: unreachable robots ⇒ not blocked (allowlist is the real gate) */
+                }
+                this.robotsCache.set(u.host, disallow);
+            }
+            return this.robotsCache.get(u.host).some((p) => u.pathname.startsWith(p));
+        });
+    }
+    timedFetch(url) {
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), this.opts.timeoutMs);
+        return this.fetchFn(url, {
+            signal: controller.signal,
+            redirect: "follow",
+            headers: { "User-Agent": USER_AGENT },
+        }).finally(() => clearTimeout(timer));
+    }
+    fetch(url, scope) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
+            if (!this.allowed(url))
+                return { resolved: false, reason: "not-allowlisted" };
+            if (yield this.robotsBlocked(url))
+                return { resolved: false, reason: "robots-disallow" };
+            if (this.fetched >= this.opts.maxFetches)
+                return { resolved: false, reason: "budget-exceeded" };
+            this.fetched++;
+            let res;
+            try {
+                res = yield this.timedFetch(url);
+            }
+            catch (err) {
+                const reason = (err === null || err === void 0 ? void 0 : err.name) === "AbortError" ? "timeout" : "fetch-error";
+                this.logger.warn(`Fetch ${reason} for ${url}: ${(_a = err === null || err === void 0 ? void 0 : err.message) !== null && _a !== void 0 ? _a : err}`);
+                return { resolved: false, reason, status: 0 };
+            }
+            if (!res.ok)
+                return { resolved: false, reason: `http-${res.status}`, status: res.status };
+            const contentType = (_b = res.headers.get("content-type")) !== null && _b !== void 0 ? _b : "";
+            const isHtml = /text\/html|application\/xhtml/i.test(contentType);
+            const isPdf = !!this.opts.allowPdf && /application\/pdf/i.test(contentType);
+            if (!isHtml && !isPdf) {
+                return { resolved: false, reason: `content-type:${contentType.split(";")[0] || "unknown"}`, status: res.status, contentType };
+            }
+            // PDF path (Phase 2 citation full text): stage the bytes verbatim as `.pdf`,
+            // skipping the html title/relevance gate (the allowlist of OA hosts is the gate).
+            if (isPdf) {
+                const buf = Buffer.from(yield res.arrayBuffer());
+                if (buf.byteLength > this.opts.maxBytes) {
+                    return { resolved: false, reason: "too-large", status: res.status, contentType };
+                }
+                const tempPath = yield this.stage(url, buf, "pdf");
+                return { resolved: true, status: res.status, contentType, tempPath };
+            }
+            const body = yield res.text();
+            if (Buffer.byteLength(body) > this.opts.maxBytes) {
+                return { resolved: false, reason: "too-large", status: res.status, contentType };
+            }
+            const title = this.extractTitle(body);
+            if (this.opts.relevanceCheck && !(yield this.relevant(url, title, body, scope))) {
+                return { resolved: false, reason: "irrelevant", status: res.status, contentType, title };
+            }
+            const tempPath = yield this.stage(url, body, "html");
+            return { resolved: true, status: res.status, contentType, tempPath, title };
+        });
+    }
+    extractTitle(html) {
+        var _a, _b;
+        const t = /<title[^>]*>([\s\S]*?)<\/title>/i.exec(html);
+        const d = /<meta[^>]+name=["']description["'][^>]+content=["']([^"']+)["']/i.exec(html);
+        return [(_a = t === null || t === void 0 ? void 0 : t[1]) === null || _a === void 0 ? void 0 : _a.trim(), (_b = d === null || d === void 0 ? void 0 : d[1]) === null || _b === void 0 ? void 0 : _b.trim()].filter(Boolean).join(" — ").slice(0, 400);
+    }
+    /** Cheap LLM gate before the expensive extraction (fail-open on error). */
+    relevant(url, title, html, scope) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const res = yield this.llm.generateStructured([
+                    {
+                        role: "system",
+                        content: "You decide if a web page is relevant to a knowledge-graph's scope. " +
+                            "Answer only by setting `relevant` true or false.",
+                    },
+                    {
+                        role: "user",
+                        content: `Scope: ${scope || "(general)"}\nURL: ${url}\nTitle/desc: ${title || "(none)"}\nRelevant?`,
+                    },
+                ], zod_1.z.object({ relevant: zod_1.z.boolean() }));
+                return res.relevant === true;
+            }
+            catch (err) {
+                this.logger.warn(`Relevance check failed for ${url} (treating as relevant): ${err}`);
+                return true;
+            }
+        });
+    }
+    stage(url, body, ext) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield fs.promises.mkdir(this.tempDir, { recursive: true });
+            const name = crypto.createHash("sha1").update(url).digest("hex").slice(0, 16);
+            const p = path.join(this.tempDir, `${name}.${ext}`);
+            if (typeof body === "string")
+                yield fs.promises.writeFile(p, body, "utf-8");
+            else
+                yield fs.promises.writeFile(p, body);
+            return p;
+        });
+    }
+}
+exports.GatedFetcher = GatedFetcher;
+//# sourceMappingURL=GatedFetcher.js.map

package/dist/core/knowledge/references/web/GatedFetcher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"GatedFetcher.js","sourceRoot":"","sources":["../../../../../src/core/knowledge/references/web/GatedFetcher.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AACjC,6BAAwB;AAIxB;;;;;;;;;GASG;AACH,MAAM,UAAU,GAAG,oEAAoE,CAAC;AA2BxF,MAAa,YAAY;IAIvB,YACmB,IAAyB,EACzB,GAAiB,EACjB,MAAc,EACd,UAAkB,QAAQ,EAC1B,UAAoB,UAAkB,CAAC,KAAK;QAJ5C,SAAI,GAAJ,IAAI,CAAqB;QACzB,QAAG,GAAH,GAAG,CAAc;QACjB,WAAM,GAAN,MAAM,CAAQ;QACd,YAAO,GAAP,OAAO,CAAmB;QAC1B,YAAO,GAAP,OAAO,CAAqC;QARvD,YAAO,GAAG,CAAC,CAAC,CAAC,yBAAyB;QAC7B,gBAAW,GAAG,IAAI,GAAG,EAAoB,CAAC,CAAC,wBAAwB;IAQjF,CAAC;IAEJ,2EAA2E;IACnE,OAAO,CAAC,GAAW;QACzB,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACzE,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;IAEO,OAAO,CAAC,GAAW,EAAE,OAAe;QAC1C,IAAI,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC,eAAe;QACzD,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACnC,OAAO,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,OAAO,EAAE,CAAC,CAAC,CAAC,qBAAqB;QAChF,CAAC;QAAC,WAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,+EAA+E;IACjE,aAAa,CAAC,GAAW;;YACrC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM;gBAAE,OAAO,KAAK,CAAC;YACpC,IAAI,CAAM,CAAC;YACX,IAAI,CAAC;gBACH,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;YAAC,WAAM,CAAC;gBACP,OAAO,KAAK,CAAC;YACf,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClC,MAAM,QAAQ,GAAa,EAAE,CAAC;gBAC9B,IAAI,CAAC;oBACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,IAAI,aAAa,CAAC,CAAC;oBACzE,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;wBACX,IAAI,MAAM,GAAG,KAAK,CAAC;wBACnB,KAAK,MAAM,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;4BACjD,MAAM,IAAI,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;4BAC5C,MAAM,EAAE,GAAG,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;4BAC9C,IAAI,EAAE;gCAAE,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC;iCACjC,IAAI,MAAM,EAAE,CAAC;gCAChB,MAAM,CAAC,GAAG,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gCAC3C,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE;oCAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;4BACnD,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,WAAM,CAAC;oBACP,8EAA8E;gBAChF,CAAC;gBACD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;YACzC,CAAC;YACD,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7E,CAAC;KAAA;IAEO,UAAU,CAAC,GAAW;QAC5B,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACxE,OAAO,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE;YACvB,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,QAAQ,EAAE,QAAQ;YAClB,OAAO,EAAE,EAAE,YAAY,EAAE,UAAU,EAAE;SACtC,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;IACxC,CAAC;IAEK,KAAK,CAAC,GAAW,EAAE,KAAa;;;YACpC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC;gBAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC;YAC9E,IAAI,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC;gBAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC;YACzF,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU;gBAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,iBAAiB,EAAE,CAAC;YAEhG,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,IAAI,GAAa,CAAC;YAClB,IAAI,CAAC;gBACH,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YACnC,CAAC;YAAC,OAAO,GAAQ,EAAE,CAAC;gBAClB,MAAM,MAAM,GAAG,CAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,IAAI,MAAK,YAAY,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC;gBACtE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,MAAM,QAAQ,GAAG,KAAK,MAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,OAAO,mCAAI,GAAG,EAAE,CAAC,CAAC;gBACvE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;YAChD,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,GAAG,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;YAE1F,MAAM,WAAW,GAAG,MAAA,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,mCAAI,EAAE,CAAC;YAC1D,MAAM,MAAM,GAAG,gCAAgC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAClE,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,mBAAmB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC5E,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACtB,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,gBAAgB,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,SAAS,EAAE,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC;YAChI,CAAC;YAED,6EAA6E;YAC7E,kFAAkF;YAClF,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;gBACjD,IAAI,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACxC,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC;gBACnF,CAAC;gBACD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;gBACnD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;YACvE,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;YAC9B,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACjD,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,CAAC;YACnF,CAAC;YAED,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;YACtC,IAAI,IAAI,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;gBAChF,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;YAC3F,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC;YACrD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;QAC9E,CAAC;KAAA;IAEO,YAAY,CAAC,IAAY;;QAC/B,MAAM,CAAC,GAAG,kCAAkC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxD,MAAM,CAAC,GAAG,kEAAkE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxF,OAAO,CAAC,MAAA,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAG,CAAC,CAAC,0CAAE,IAAI,EAAE,EAAE,MAAA,CAAC,aAAD,CAAC,uBAAD,CAAC,CAAG,CAAC,CAAC,0CAAE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACpF,CAAC;IAED,2EAA2E;IAC7D,QAAQ,CAAC,GAAW,EAAE,KAAa,EAAE,IAAY,EAAE,KAAa;;YAC5E,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAC3C;oBACE;wBACE,IAAI,EAAE,QAAQ;wBACd,OAAO,EACL,qEAAqE;4BACrE,kDAAkD;qBACrD;oBACD;wBACE,IAAI,EAAE,MAAM;wBACZ,OAAO,EAAE,UAAU,KAAK,IAAI,WAAW,UAAU,GAAG,iBAAiB,KAAK,IAAI,QAAQ,aAAa;qBACpG;iBACF,EACD,OAAC,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,OAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CACpC,CAAC;gBACF,OAAO,GAAG,CAAC,QAAQ,KAAK,IAAI,CAAC;YAC/B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,8BAA8B,GAAG,4BAA4B,GAAG,EAAE,CAAC,CAAC;gBACrF,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;KAAA;IAEa,KAAK,CAAC,GAAW,EAAE,IAAqB,EAAE,GAAmB;;YACzE,MAAM,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC3D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC9E,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC;YACpD,IAAI,OAAO,IAAI,KAAK,QAAQ;gBAAE,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;;gBACvE,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;YAC1C,OAAO,CAAC,CAAC;QACX,CAAC;KAAA;CACF;AA9JD,oCA8JC"}

package/dist/core/knowledge/references/web/WebReferenceProcessor.js

@@ -0,0 +1,164 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.WebReferenceProcessor = void 0;
+const fs = __importStar(require("fs"));
+const referenceExtraction_1 = require("../../../processor/readers/referenceExtraction");
+/**
+ * Phase-1 class-3 consumer: for a citing document's EXTERNAL links, fetch each
+ * (gated), extract the page's KG, and emit a `references` edge `citingDoc → url`.
+ * Depth-1 — fetched pages are not re-crawled. Every URL is cached so a re-run
+ * never refetches; blocked/gated links emit a bare `resolved:false` edge to a
+ * stub URL node (never fabricated content).
+ */
+class WebReferenceProcessor {
+    constructor(fetcher, cache, extract, logger) {
+        this.fetcher = fetcher;
+        this.cache = cache;
+        this.extract = extract;
+        this.logger = logger;
+    }
+    process(citingRel, links, scope) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
+            const urls = Array.from(new Set(links.filter((l) => (0, referenceExtraction_1.isExternalTarget)(l.target)).map((l) => l.target)));
+            if (urls.length === 0)
+                return null;
+            const entities = new Map();
+            const relations = [];
+            const seenEdge = new Set();
+            const ensureDoc = (name, observations = []) => {
+                const e = entities.get(name);
+                if (e) {
+                    if (observations.length)
+                        e.observations.push(...observations);
+                    return;
+                }
+                entities.set(name, { name, entityType: "document", files: [], observations });
+            };
+            const addEdge = (to, resolved) => {
+                if (to === citingRel || seenEdge.has(to))
+                    return;
+                seenEdge.add(to);
+                relations.push({ from: citingRel, to, relationType: ["references"], source: citingRel, resolved });
+            };
+            // The citing document node — guarantees the edge's `from` endpoint exists
+            // (idempotent with the Phase-0 resolver's path-keyed node).
+            ensureDoc(citingRel);
+            for (const url of urls) {
+                const cached = this.cache.get(url);
+                const contribution = (_a = cached === null || cached === void 0 ? void 0 : cached.graph) !== null && _a !== void 0 ? _a : (yield this.fetchAndBuild(citingRel, url, scope));
+                // Merge this URL's contribution (page entities + url node + edge) in.
+                for (const e of contribution.entities)
+                    ensureDoc(e.name, e.observations);
+                for (const r of contribution.relations) {
+                    if (r.relationType.includes("references"))
+                        addEdge(r.to, (_b = r.resolved) !== null && _b !== void 0 ? _b : false);
+                    else
+                        relations.push(r);
+                }
+            }
+            return relations.length || entities.size > 1 ? { entities: Array.from(entities.values()), relations } : null;
+        });
+    }
+    /** Fetch + extract one URL, build its contribution graph, and cache it. */
+    fetchAndBuild(citingRel, url, scope) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const fetchedAt = new Date().toISOString();
+            const r = yield this.fetcher.fetch(url, scope);
+            let graph;
+            if (r.resolved && r.tempPath) {
+                let pageGraphs = [];
+                try {
+                    pageGraphs = yield this.extract(r.tempPath, url);
+                }
+                catch (err) {
+                    this.logger.warn(`Web extraction failed for ${url}: ${err}`);
+                }
+                finally {
+                    fs.promises.unlink(r.tempPath).catch(() => undefined);
+                }
+                graph = this.buildContribution(citingRel, url, true, pageGraphs, r.title, fetchedAt);
+            }
+            else {
+                this.logger.info(`Web reference not resolved (${r.reason}): ${url}`);
+                graph = this.buildContribution(citingRel, url, false, [], undefined, fetchedAt);
+            }
+            yield this.cache.append({
+                url,
+                resolved: r.resolved,
+                reason: r.reason,
+                status: r.status,
+                contentType: r.contentType,
+                fetchedAt,
+                graph,
+            });
+            return graph;
+        });
+    }
+    /** Assemble: url document node (+ stated provenance) + page content (source
+     * re-stamped to the URL) + the `references` edge. */
+    buildContribution(citingRel, url, resolved, pageGraphs, title, fetchedAt) {
+        const entities = [];
+        const relations = [
+            { from: citingRel, to: url, relationType: ["references"], source: citingRel, resolved },
+        ];
+        const obs = [];
+        if (title)
+            obs.push({ text: `Title: ${title}`, source: url, createdAt: fetchedAt });
+        obs.push({ text: `Fetched from ${url}`, source: url, createdAt: fetchedAt });
+        entities.push({ name: url, entityType: "document", files: [], observations: obs });
+        // Fold in the fetched page's extracted KG, re-stamping observation provenance
+        // from the temp path to the URL (the page's true source).
+        for (const g of pageGraphs) {
+            for (const e of g.entities) {
+                entities.push(Object.assign(Object.assign({}, e), { observations: e.observations.map((o) => (Object.assign(Object.assign({}, o), { source: url }))) }));
+            }
+            relations.push(...g.relations);
+        }
+        return { entities, relations };
+    }
+}
+exports.WebReferenceProcessor = WebReferenceProcessor;
+//# sourceMappingURL=WebReferenceProcessor.js.map

package/dist/core/knowledge/references/web/WebReferenceProcessor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"WebReferenceProcessor.js","sourceRoot":"","sources":["../../../../../src/core/knowledge/references/web/WebReferenceProcessor.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AAGzB,wFAA2F;AAQ3F;;;;;;GAMG;AACH,MAAa,qBAAqB;IAChC,YACmB,OAAqB,EACrB,KAAwB,EACxB,OAAqB,EACrB,MAAc;QAHd,YAAO,GAAP,OAAO,CAAc;QACrB,UAAK,GAAL,KAAK,CAAmB;QACxB,YAAO,GAAP,OAAO,CAAc;QACrB,WAAM,GAAN,MAAM,CAAQ;IAC9B,CAAC;IAEE,OAAO,CACX,SAAiB,EACjB,KAAgB,EAChB,KAAa;;;YAEb,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CACrB,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAA,sCAAgB,EAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAC9E,CAAC;YACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO,IAAI,CAAC;YAEnC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;YAC3C,MAAM,SAAS,GAAe,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;YACnC,MAAM,SAAS,GAAG,CAAC,IAAY,EAAE,eAA8B,EAAE,EAAE,EAAE;gBACnE,MAAM,CAAC,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC7B,IAAI,CAAC,EAAE,CAAC;oBACN,IAAI,YAAY,CAAC,MAAM;wBAAE,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,CAAC;oBAC9D,OAAO;gBACT,CAAC;gBACD,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;YAChF,CAAC,CAAC;YACF,MAAM,OAAO,GAAG,CAAC,EAAU,EAAE,QAAiB,EAAE,EAAE;gBAChD,IAAI,EAAE,KAAK,SAAS,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;oBAAE,OAAO;gBACjD,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACjB,SAAS,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAC;YACrG,CAAC,CAAC;YAEF,0EAA0E;YAC1E,4DAA4D;YAC5D,SAAS,CAAC,SAAS,CAAC,CAAC;YAErB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACnC,MAAM,YAAY,GAAG,MAAA,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,KAAK,mCAAI,CAAC,MAAM,IAAI,CAAC,aAAa,CAAC,SAAS,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;gBACxF,sEAAsE;gBACtE,KAAK,MAAM,CAAC,IAAI,YAAY,CAAC,QAAQ;oBAAE,SAAS,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC;gBACzE,KAAK,MAAM,CAAC,IAAI,YAAY,CAAC,SAAS,EAAE,CAAC;oBACvC,IAAI,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,YAAY,CAAC;wBAAE,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,MAAA,CAAC,CAAC,QAAQ,mCAAI,KAAK,CAAC,CAAC;;wBACzE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC;YAED,OAAO,SAAS,CAAC,MAAM,IAAI,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/G,CAAC;KAAA;IAED,2EAA2E;IAC7D,aAAa,CACzB,SAAiB,EACjB,GAAW,EACX,KAAa;;YAEb,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;YAC3C,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAE/C,IAAI,KAAqB,CAAC;YAC1B,IAAI,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;gBAC7B,IAAI,UAAU,GAAqB,EAAE,CAAC;gBACtC,IAAI,CAAC;oBACH,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;gBACnD,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,GAAG,KAAK,GAAG,EAAE,CAAC,CAAC;gBAC/D,CAAC;wBAAS,CAAC;oBACT,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;gBACxD,CAAC;gBACD,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,GAAG,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;YACvF,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC,MAAM,MAAM,GAAG,EAAE,CAAC,CAAC;gBACrE,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,CAAC,CAAC;YAClF,CAAC;YAED,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;gBACtB,GAAG;gBACH,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,SAAS;gBACT,KAAK;aACN,CAAC,CAAC;YACH,OAAO,KAAK,CAAC;QACf,CAAC;KAAA;IAED;yDACqD;IAC7C,iBAAiB,CACvB,SAAiB,EACjB,GAAW,EACX,QAAiB,EACjB,UAA4B,EAC5B,KAAyB,EACzB,SAAiB;QAEjB,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAe;YAC5B,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,EAAE,YAAY,EAAE,CAAC,YAAY,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE;SACxF,CAAC;QAEF,MAAM,GAAG,GAAkB,EAAE,CAAC;QAC9B,IAAI,KAAK;YAAE,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,KAAK,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;QACpF,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,GAAG,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;QAC7E,QAAQ,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,UAAU,EAAE,UAAU,EAAE,KAAK,EAAE,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,CAAC,CAAC;QAEnF,8EAA8E;QAC9E,0DAA0D;QAC1D,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;YAC3B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;gBAC3B,QAAQ,CAAC,IAAI,iCACR,CAAC,KACJ,YAAY,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iCAAM,CAAC,KAAE,MAAM,EAAE,GAAG,IAAG,CAAC,IAChE,CAAC;YACL,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC;QACjC,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;IACjC,CAAC;CACF;AA3HD,sDA2HC"}