@astrofoundry/grimoire 1.2.2

package/dist/consumer-config.js

@@ -0,0 +1,58 @@
+import { readFile, writeFile, mkdir } from "node:fs/promises";
+import { join } from "node:path";
+import { homedir } from "node:os";
+import { createInterface } from "node:readline";
+const CONFIG_DIR = join(homedir(), ".grimoire");
+const CONFIG_FILE = join(CONFIG_DIR, "config.json");
+export async function loadConsumerConfig() {
+    const raw = await readFile(CONFIG_FILE, "utf-8").catch(() => null);
+    if (!raw)
+        return null;
+    const data = JSON.parse(raw);
+    if (typeof data.apiUrl === "string" && typeof data.apiKey === "string") {
+        return { apiUrl: data.apiUrl, apiKey: data.apiKey };
+    }
+    return null;
+}
+export async function saveConsumerConfig(config) {
+    await mkdir(CONFIG_DIR, { recursive: true });
+    await writeFile(CONFIG_FILE, JSON.stringify(config, null, 2) + "\n", "utf-8");
+}
+export async function resolveConsumerConfig() {
+    const envUrl = process.env.GRIMOIRE_API_URL;
+    const envKey = process.env.GRIMOIRE_API_KEY;
+    if (envUrl && envKey) {
+        return { apiUrl: envUrl, apiKey: envKey };
+    }
+    const fileConfig = await loadConsumerConfig();
+    if (fileConfig)
+        return fileConfig;
+    throw new Error("Grimoire is not configured. Run 'grimoire init' to set up.");
+}
+export function isConsumerMode() {
+    return !!process.env.GRIMOIRE_API_URL;
+}
+export async function detectConsumerMode() {
+    if (process.env.GRIMOIRE_API_URL)
+        return true;
+    const config = await loadConsumerConfig();
+    return config !== null;
+}
+export async function cmdInit() {
+    const rl = createInterface({ input: process.stdin, output: process.stdout });
+    const ask = (q) => new Promise((resolve) => rl.question(q, resolve));
+    const existing = await loadConsumerConfig();
+    const apiUrl = await ask(`API URL${existing ? ` [${existing.apiUrl}]` : ""}: `);
+    const apiKey = await ask(`API Key${existing ? " [****]" : ""}: `);
+    const config = {
+        apiUrl: apiUrl.trim() || existing?.apiUrl || "",
+        apiKey: apiKey.trim() || existing?.apiKey || "",
+    };
+    rl.close();
+    if (!config.apiUrl || !config.apiKey) {
+        throw new Error("Both API URL and API Key are required.");
+    }
+    await saveConsumerConfig(config);
+    console.log(`\nSaved to ${CONFIG_FILE}`);
+}
+//# sourceMappingURL=consumer-config.js.map

package/dist/consumer-config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"consumer-config.js","sourceRoot":"","sources":["../src/consumer-config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,WAAW,CAAC,CAAC;AAChD,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;AAOpD,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IACnE,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QACvE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IACtD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,MAAsB;IAC7D,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7C,MAAM,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;AAChF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB;IACzC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IAE5C,IAAI,MAAM,IAAI,MAAM,EAAE,CAAC;QACrB,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAC5C,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC9C,IAAI,UAAU;QAAE,OAAO,UAAU,CAAC;IAElC,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;AAChF,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;AACxC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB;QAAE,OAAO,IAAI,CAAC;IAC9C,MAAM,MAAM,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC1C,OAAO,MAAM,KAAK,IAAI,CAAC;AACzB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO;IAC3B,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7E,MAAM,GAAG,GAAG,CAAC,CAAS,EAAmB,EAAE,CACzC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IAEpD,MAAM,QAAQ,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAE5C,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,UAAU,QAAQ,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAChF,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,UAAU,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAElE,MAAM,MAAM,GAAmB;QAC7B,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,QAAQ,EAAE,MAAM,IAAI,EAAE;QAC/C,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,QAAQ,EAAE,MAAM,IAAI,EAAE;KAChD,CAAC;IAEF,EAAE,CAAC,KAAK,EAAE,CAAC;IAEX,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5D,CAAC;IAED,MAAM,kBAAkB,CAAC,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,cAAc,WAAW,EAAE,CAAC,CAAC;AAC3C,CAAC"}

package/dist/consumer.d.ts

@@ -0,0 +1,8 @@
+import type { ConsumerConfig } from "./consumer-config.js";
+export declare function cmdConsumerSearch(config: ConsumerConfig, query: string, options: {
+    source?: string;
+    topN?: number;
+}): Promise<void>;
+export declare function cmdConsumerList(config: ConsumerConfig): Promise<void>;
+export declare function cmdConsumerStats(config: ConsumerConfig): Promise<void>;
+//# sourceMappingURL=consumer.d.ts.map

package/dist/consumer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"consumer.d.ts","sourceRoot":"","sources":["../src/consumer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AA+B3D,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,cAAc,EACtB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAC1C,OAAO,CAAC,IAAI,CAAC,CAmBf;AAED,wBAAsB,eAAe,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAa3E;AAED,wBAAsB,gBAAgB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAiB5E"}

package/dist/consumer.js

@@ -0,0 +1,71 @@
+import { bold, cyan, yellow } from "./format.js";
+async function apiRequest(config, path, options) {
+    const url = `${config.apiUrl.replace(/\/$/, "")}${path}`;
+    let response;
+    try {
+        response = await fetch(url, {
+            ...options,
+            headers: {
+                "Content-Type": "application/json",
+                "x-api-key": config.apiKey,
+                ...options?.headers,
+            },
+        });
+    }
+    catch {
+        throw new Error(`Cannot reach Grimoire API at ${config.apiUrl}. Check your GRIMOIRE_API_URL.`);
+    }
+    if (response.status === 401 || response.status === 403) {
+        throw new Error("Invalid API key. Check your GRIMOIRE_API_KEY or run 'grimoire init'.");
+    }
+    if (!response.ok) {
+        throw new Error(`API error: ${response.status} ${response.statusText}`);
+    }
+    return response.json();
+}
+export async function cmdConsumerSearch(config, query, options) {
+    const data = await apiRequest(config, "/search", {
+        method: "POST",
+        body: JSON.stringify({ query, source: options.source, topN: options.topN }),
+    });
+    if (data.results.length === 0) {
+        console.log("No results found.");
+        return;
+    }
+    for (let i = 0; i < data.results.length; i++) {
+        const r = data.results[i];
+        const score = r.relevance_score < 0.001 ? r.relevance_score.toExponential(2) : r.relevance_score.toFixed(4);
+        console.log(`\n${bold(`[${i + 1}] ${r.title}`)} (${score})`);
+        console.log(`    ${cyan(r.url)}`);
+        console.log(`    ${yellow(r.heading_path.join(" > "))}`);
+        console.log(`    ${r.content.slice(0, 200).replace(/\n/g, " ")}...`);
+    }
+}
+export async function cmdConsumerList(config) {
+    const data = await apiRequest(config, "/list");
+    if (data.sources.length === 0) {
+        console.log("No sources available.");
+        return;
+    }
+    console.log("\nSources:\n");
+    for (const s of data.sources) {
+        console.log(`  ${bold(s.source)}`);
+        console.log(`    ${s.chunk_count} chunks, ${s.url_count} URLs, last refreshed ${s.last_refreshed}`);
+    }
+}
+export async function cmdConsumerStats(config) {
+    const data = await apiRequest(config, "/stats");
+    if (data.sources.length === 0) {
+        console.log("No sources have been refreshed yet.");
+        return;
+    }
+    console.log("\nSource Statistics:\n");
+    for (const s of data.sources) {
+        console.log(`  ${bold(s.source)}`);
+        console.log(`    Chunks: ${s.chunk_count}`);
+        console.log(`    URLs: ${s.url_count}`);
+        console.log(`    Last refreshed: ${s.last_refreshed}`);
+    }
+    console.log(`\n  Total: ${data.totalChunks} chunks across ${data.totalUrls} URLs from ${data.sources.length} sources`);
+}
+//# sourceMappingURL=consumer.js.map

package/dist/consumer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"consumer.js","sourceRoot":"","sources":["../src/consumer.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAEjD,KAAK,UAAU,UAAU,CAAI,MAAsB,EAAE,IAAY,EAAE,OAAqB;IACtF,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC;IAEzD,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC1B,GAAG,OAAO;YACV,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,WAAW,EAAE,MAAM,CAAC,MAAM;gBAC1B,GAAG,OAAO,EAAE,OAAO;aACpB;SACF,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,gCAAgC,MAAM,CAAC,MAAM,gCAAgC,CAAC,CAAC;IACjG,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;IAC1F,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,cAAc,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,OAAO,QAAQ,CAAC,IAAI,EAAgB,CAAC;AACvC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAsB,EACtB,KAAa,EACb,OAA2C;IAE3C,MAAM,IAAI,GAAG,MAAM,UAAU,CAA8B,MAAM,EAAE,SAAS,EAAE;QAC5E,MAAM,EAAE,MAAM;QACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;KAC5E,CAAC,CAAC;IAEH,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,OAAO;IACT,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,KAAK,GAAG,CAAC,CAAC,eAAe,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC5G,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,KAAK,KAAK,GAAG,CAAC,CAAC;QAC7D,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;QACzD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAsB;IAC1D,MAAM,IAAI,GAAG,MAAM,UAAU,CAA4B,MAAM,EAAE,OAAO,CAAC,CAAC;IAE1E,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO;IACT,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAC5B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,WAAW,YAAY,CAAC,CAAC,SAAS,yBAAyB,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC;IACtG,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAsB;IAC3D,MAAM,IAAI,GAAG,MAAM,UAAU,CAAoE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEnH,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;QACnD,OAAO;IACT,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAC7B,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACnC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;QACxC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,CAAC,WAAW,kBAAkB,IAAI,CAAC,SAAS,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;AACzH,CAAC"}

package/dist/converter.d.ts

@@ -0,0 +1,12 @@
+export interface ConvertedPage {
+    source: string;
+    url: string;
+    title: string;
+    markdown: string;
+}
+export declare function extractContent(html: string, contentSelector: string, removeSelectors?: string[], removeTextPatterns?: string[]): string;
+export declare function extractTitle(html: string): string;
+export declare function buildFrontmatter(source: string, url: string, title: string): string;
+export declare function convertPage(html: string, source: string, url: string, contentSelector: string, removeSelectors?: string[], removeTextPatterns?: string[]): ConvertedPage;
+export declare function convertSource(sourceName: string, urls: string[], contentSelector: string, removeSelectors: string[] | undefined, removeTextPatterns: string[] | undefined, dataDir: string, concurrency?: number, onProgress?: (current: number, total: number, url: string) => void): Promise<ConvertedPage[]>;
+//# sourceMappingURL=converter.d.ts.map

package/dist/converter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"converter.d.ts","sourceRoot":"","sources":["../src/converter.ts"],"names":[],"mappings":"AAYA,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB;AAyBD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,eAAe,EAAE,MAAM,EACvB,eAAe,CAAC,EAAE,MAAM,EAAE,EAC1B,kBAAkB,CAAC,EAAE,MAAM,EAAE,GAC5B,MAAM,CAiBR;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAKjD;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CASnF;AAED,wBAAgB,WAAW,CACzB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,EACX,eAAe,EAAE,MAAM,EACvB,eAAe,CAAC,EAAE,MAAM,EAAE,EAC1B,kBAAkB,CAAC,EAAE,MAAM,EAAE,GAC5B,aAAa,CAOf;AAID,wBAAsB,aAAa,CACjC,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EAAE,EACd,eAAe,EAAE,MAAM,EACvB,eAAe,EAAE,MAAM,EAAE,GAAG,SAAS,EACrC,kBAAkB,EAAE,MAAM,EAAE,GAAG,SAAS,EACxC,OAAO,EAAE,MAAM,EACf,WAAW,SAAsB,EACjC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,IAAI,GACjE,OAAO,CAAC,aAAa,EAAE,CAAC,CAgC1B"}

package/dist/converter.js

@@ -0,0 +1,95 @@
+import { readFile, writeFile, mkdir } from "node:fs/promises";
+import { join } from "node:path";
+import { JSDOM } from "jsdom";
+import TurndownService from "turndown";
+import { slugifyUrl } from "./scraper.js";
+const turndown = new TurndownService({
+    headingStyle: "atx",
+    codeBlockStyle: "fenced",
+    bulletListMarker: "-",
+});
+const GENERIC_REMOVE = [
+    "style",
+    "script",
+    "noscript",
+    "iframe",
+    "svg",
+];
+function cleanMarkdown(md, textPatterns) {
+    let cleaned = md
+        .replace(/^(#+)\s*$/gm, "")
+        .replace(/\n{3,}/g, "\n\n");
+    if (textPatterns) {
+        for (const pattern of textPatterns) {
+            cleaned = cleaned.replace(new RegExp(pattern, "gm"), "");
+        }
+        cleaned = cleaned.replace(/\n{3,}/g, "\n\n");
+    }
+    return cleaned.trim();
+}
+export function extractContent(html, contentSelector, removeSelectors, removeTextPatterns) {
+    const dom = new JSDOM(html);
+    const doc = dom.window.document;
+    const contentEl = doc.querySelector(contentSelector);
+    if (!contentEl) {
+        return cleanMarkdown(turndown.turndown(doc.body.innerHTML), removeTextPatterns);
+    }
+    const allSelectors = [...GENERIC_REMOVE, ...(removeSelectors ?? [])];
+    for (const selector of allSelectors) {
+        for (const el of contentEl.querySelectorAll(selector)) {
+            el.remove();
+        }
+    }
+    return cleanMarkdown(turndown.turndown(contentEl.innerHTML), removeTextPatterns);
+}
+export function extractTitle(html) {
+    const dom = new JSDOM(html);
+    const titleEl = dom.window.document.querySelector("title");
+    if (!titleEl)
+        return "Untitled";
+    return titleEl.textContent?.replace(/\s*[|–—-]\s*.+$/, "").trim() ?? "Untitled";
+}
+export function buildFrontmatter(source, url, title) {
+    return [
+        "---",
+        `source: ${source}`,
+        `url: "${url}"`,
+        `title: "${title.replace(/"/g, '\\"')}"`,
+        `fetched_at: "${new Date().toISOString()}"`,
+        "---",
+    ].join("\n");
+}
+export function convertPage(html, source, url, contentSelector, removeSelectors, removeTextPatterns) {
+    const title = extractTitle(html);
+    const content = extractContent(html, contentSelector, removeSelectors, removeTextPatterns);
+    const frontmatter = buildFrontmatter(source, url, title);
+    const markdown = `${frontmatter}\n\n${content}`;
+    return { source, url, title, markdown };
+}
+const DEFAULT_CONCURRENCY = 10;
+export async function convertSource(sourceName, urls, contentSelector, removeSelectors, removeTextPatterns, dataDir, concurrency = DEFAULT_CONCURRENCY, onProgress) {
+    const rawDir = join(dataDir, "raw", sourceName);
+    const mdDir = join(dataDir, "markdown", sourceName);
+    await mkdir(mdDir, { recursive: true });
+    const pages = new Array(urls.length);
+    let completed = 0;
+    let nextIndex = 0;
+    async function worker() {
+        while (nextIndex < urls.length) {
+            const i = nextIndex++;
+            const url = urls[i];
+            const slug = slugifyUrl(url);
+            const htmlPath = join(rawDir, `${slug}.html`);
+            const html = await readFile(htmlPath, "utf-8");
+            const page = convertPage(html, sourceName, url, contentSelector, removeSelectors, removeTextPatterns);
+            await writeFile(join(mdDir, `${slug}.md`), page.markdown, "utf-8");
+            pages[i] = page;
+            completed++;
+            onProgress?.(completed, urls.length, url);
+        }
+    }
+    const workers = Array.from({ length: Math.min(concurrency, urls.length) }, () => worker());
+    await Promise.all(workers);
+    return pages;
+}
+//# sourceMappingURL=converter.js.map

package/dist/converter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"converter.js","sourceRoot":"","sources":["../src/converter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC;IACnC,YAAY,EAAE,KAAK;IACnB,cAAc,EAAE,QAAQ;IACxB,gBAAgB,EAAE,GAAG;CACtB,CAAC,CAAC;AASH,MAAM,cAAc,GAAG;IACrB,OAAO;IACP,QAAQ;IACR,UAAU;IACV,QAAQ;IACR,KAAK;CACN,CAAC;AAEF,SAAS,aAAa,CAAC,EAAU,EAAE,YAAuB;IACxD,IAAI,OAAO,GAAG,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC;SAC1B,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE9B,IAAI,YAAY,EAAE,CAAC;QACjB,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACnC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC/C,CAAC;IAED,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,IAAY,EACZ,eAAuB,EACvB,eAA0B,EAC1B,kBAA6B;IAE7B,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;IAEhC,MAAM,SAAS,GAAG,GAAG,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC;IACrD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;IAClF,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,GAAG,cAAc,EAAE,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,CAAC;IACrE,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;QACpC,KAAK,MAAM,EAAE,IAAI,SAAS,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC;YACtD,EAAE,CAAC,MAAM,EAAE,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;AACnF,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAC3D,IAAI,CAAC,OAAO;QAAE,OAAO,UAAU,CAAC;IAChC,OAAO,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,UAAU,CAAC;AAClF,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,GAAW,EAAE,KAAa;IACzE,OAAO;QACL,KAAK;QACL,WAAW,MAAM,EAAE;QACnB,SAAS,GAAG,GAAG;QACf,WAAW,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG;QACxC,gBAAgB,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,GAAG;QAC3C,KAAK;KACN,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,IAAY,EACZ,MAAc,EACd,GAAW,EACX,eAAuB,EACvB,eAA0B,EAC1B,kBAA6B;IAE7B,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,EAAE,eAAe,EAAE,eAAe,EAAE,kBAAkB,CAAC,CAAC;IAC3F,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,GAAG,WAAW,OAAO,OAAO,EAAE,CAAC;IAEhD,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC;AAED,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAE/B,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAAkB,EAClB,IAAc,EACd,eAAuB,EACvB,eAAqC,EACrC,kBAAwC,EACxC,OAAe,EACf,WAAW,GAAG,mBAAmB,EACjC,UAAkE;IAElE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;IACpD,MAAM,KAAK,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,MAAM,KAAK,GAAoB,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACtD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,UAAU,MAAM;QACnB,OAAO,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC/B,MAAM,CAAC,GAAG,SAAS,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YACpB,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,OAAO,CAAC,CAAC;YAC9C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAE/C,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,eAAe,EAAE,eAAe,EAAE,kBAAkB,CAAC,CAAC;YACtG,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,KAAK,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACnE,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;YAChB,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CACxB,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,EAC9C,GAAG,EAAE,CAAC,MAAM,EAAE,CACf,CAAC;IACF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAE3B,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/embedder.d.ts

@@ -0,0 +1,3 @@
+export declare function embedTexts(texts: string[]): Promise<number[][]>;
+export declare function embedText(text: string): Promise<number[]>;
+//# sourceMappingURL=embedder.d.ts.map

package/dist/embedder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAmBA,wBAAsB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAqBrE;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAG/D"}

package/dist/embedder.js

@@ -0,0 +1,38 @@
+import { GoogleGenerativeAI } from "@google/generative-ai";
+const BATCH_SIZE = 100;
+const MODEL = "gemini-embedding-001";
+const OUTPUT_DIMENSIONALITY = 768;
+let genAI;
+function getClient() {
+    if (!genAI) {
+        const apiKey = process.env.GEMINI_API_KEY;
+        if (!apiKey) {
+            throw new Error("GEMINI_API_KEY environment variable is not set");
+        }
+        genAI = new GoogleGenerativeAI(apiKey);
+    }
+    return genAI;
+}
+export async function embedTexts(texts) {
+    const client = getClient();
+    const model = client.getGenerativeModel({ model: MODEL });
+    const embeddings = [];
+    for (let i = 0; i < texts.length; i += BATCH_SIZE) {
+        const batch = texts.slice(i, i + BATCH_SIZE);
+        const result = await model.batchEmbedContents({
+            requests: batch.map((text) => ({
+                content: { role: "user", parts: [{ text }] },
+                outputDimensionality: OUTPUT_DIMENSIONALITY,
+            })),
+        });
+        for (const embedding of result.embeddings) {
+            embeddings.push(embedding.values);
+        }
+    }
+    return embeddings;
+}
+export async function embedText(text) {
+    const [embedding] = await embedTexts([text]);
+    return embedding;
+}
+//# sourceMappingURL=embedder.js.map

package/dist/embedder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAE3D,MAAM,UAAU,GAAG,GAAG,CAAC;AACvB,MAAM,KAAK,GAAG,sBAAsB,CAAC;AACrC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAElC,IAAI,KAAqC,CAAC;AAE1C,SAAS,SAAS;IAChB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACpE,CAAC;QACD,KAAK,GAAG,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,KAAe;IAC9C,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,MAAM,CAAC,kBAAkB,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IAE1D,MAAM,UAAU,GAAe,EAAE,CAAC;IAElC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC;YAC5C,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBAC7B,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE;gBAC5C,oBAAoB,EAAE,qBAAqB;aAC5C,CAAC,CAAC;SACJ,CAAC,CAAC;QAEH,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAC1C,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY;IAC1C,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7C,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/dist/format.d.ts

@@ -0,0 +1,5 @@
+export declare const bold: (s: string) => string;
+export declare const cyan: (s: string) => string;
+export declare const yellow: (s: string) => string;
+export declare const red: (s: string) => string;
+//# sourceMappingURL=format.d.ts.map

package/dist/format.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../src/format.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,IAAI,GAAI,GAAG,MAAM,KAAG,MAA0C,CAAC;AAC5E,eAAO,MAAM,IAAI,GAAI,GAAG,MAAM,KAAG,MAA2C,CAAC;AAC7E,eAAO,MAAM,MAAM,GAAI,GAAG,MAAM,KAAG,MAA2C,CAAC;AAC/E,eAAO,MAAM,GAAG,GAAI,GAAG,MAAM,KAAG,MAA2C,CAAC"}

package/dist/format.js

@@ -0,0 +1,6 @@
+const isTTY = process.stdout.isTTY ?? false;
+export const bold = (s) => isTTY ? `\x1b[1m${s}\x1b[0m` : s;
+export const cyan = (s) => isTTY ? `\x1b[36m${s}\x1b[0m` : s;
+export const yellow = (s) => isTTY ? `\x1b[33m${s}\x1b[0m` : s;
+export const red = (s) => isTTY ? `\x1b[31m${s}\x1b[0m` : s;
+//# sourceMappingURL=format.js.map

package/dist/format.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"format.js","sourceRoot":"","sources":["../src/format.ts"],"names":[],"mappings":"AAAA,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC;AAE5C,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7E,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/E,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC"}

package/dist/reranker.d.ts

@@ -0,0 +1,6 @@
+export interface RerankResult {
+    index: number;
+    relevance_score: number;
+}
+export declare function rerank(query: string, documents: string[], topN?: number): Promise<RerankResult[]>;
+//# sourceMappingURL=reranker.d.ts.map

package/dist/reranker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranker.d.ts","sourceRoot":"","sources":["../src/reranker.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,MAAM,CAAC;CACzB;AAcD,wBAAsB,MAAM,CAC1B,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EAAE,EACnB,IAAI,SAAI,GACP,OAAO,CAAC,YAAY,EAAE,CAAC,CAczB"}

package/dist/reranker.js

@@ -0,0 +1,21 @@
+function getRerankerUrl() {
+    const url = process.env.RERANKER_URL;
+    if (!url) {
+        throw new Error("RERANKER_URL environment variable is not set");
+    }
+    return url;
+}
+export async function rerank(query, documents, topN = 5) {
+    const baseUrl = getRerankerUrl();
+    const response = await fetch(`${baseUrl}/v1/rerank`, {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify({ model: "zerank-2", query, documents, top_n: topN }),
+    });
+    if (!response.ok) {
+        throw new Error(`Reranker request failed: ${response.status} ${response.statusText}`);
+    }
+    const data = (await response.json());
+    return data.results;
+}
+//# sourceMappingURL=reranker.js.map

package/dist/reranker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reranker.js","sourceRoot":"","sources":["../src/reranker.ts"],"names":[],"mappings":"AASA,SAAS,cAAc;IACrB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IACrC,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,KAAa,EACb,SAAmB,EACnB,IAAI,GAAG,CAAC;IAER,MAAM,OAAO,GAAG,cAAc,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,YAAY,EAAE;QACnD,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;QAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;KAC3E,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA2B,CAAC;IAC/D,OAAO,IAAI,CAAC,OAAO,CAAC;AACtB,CAAC"}

package/dist/scraper.d.ts

@@ -0,0 +1,9 @@
+import { type Browser, type Page } from "playwright";
+import type { SourceConfig } from "./config.js";
+export declare function slugifyUrl(url: string): string;
+export declare function filterUrls(urls: string[], includePatterns?: string[], excludePatterns?: string[]): string[];
+export declare function discoverUrls(page: Page, source: SourceConfig): Promise<string[]>;
+export declare function fetchPage(page: Page, url: string): Promise<string>;
+export declare function scrapeSource(source: SourceConfig, sourceName: string, dataDir: string, onProgress?: (current: number, total: number, url: string) => void): Promise<string[]>;
+export declare function createBrowser(): Promise<Browser>;
+//# sourceMappingURL=scraper.d.ts.map

package/dist/scraper.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../src/scraper.ts"],"names":[],"mappings":"AAEA,OAAO,EAAY,KAAK,OAAO,EAAE,KAAK,IAAI,EAAE,MAAM,YAAY,CAAC;AAC/D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAO9C;AAED,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,EAAE,EACd,eAAe,CAAC,EAAE,MAAM,EAAE,EAC1B,eAAe,CAAC,EAAE,MAAM,EAAE,GACzB,MAAM,EAAE,CAgBV;AAED,wBAAsB,YAAY,CAChC,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,YAAY,GACnB,OAAO,CAAC,MAAM,EAAE,CAAC,CAanB;AAED,wBAAsB,SAAS,CAC7B,IAAI,EAAE,IAAI,EACV,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,MAAM,CAAC,CAGjB;AAsBD,wBAAsB,YAAY,CAChC,MAAM,EAAE,YAAY,EACpB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,EACf,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,IAAI,GACjE,OAAO,CAAC,MAAM,EAAE,CAAC,CAgCnB;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC,OAAO,CAAC,CAEtD"}

package/dist/scraper.js

@@ -0,0 +1,77 @@
+import { mkdir, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { chromium } from "playwright";
+export function slugifyUrl(url) {
+    const parsed = new URL(url);
+    return parsed.pathname
+        .replace(/^\//, "")
+        .replace(/\/$/, "")
+        .replace(/\//g, "-")
+        .replace(/[^a-zA-Z0-9-]/g, "");
+}
+export function filterUrls(urls, includePatterns, excludePatterns) {
+    let filtered = urls;
+    if (includePatterns && includePatterns.length > 0) {
+        filtered = filtered.filter((url) => includePatterns.some((pattern) => url.includes(pattern)));
+    }
+    if (excludePatterns && excludePatterns.length > 0) {
+        filtered = filtered.filter((url) => !excludePatterns.some((pattern) => url.includes(pattern)));
+    }
+    return [...new Set(filtered)].sort();
+}
+export async function discoverUrls(page, source) {
+    await page.goto(source.start_url, { waitUntil: "domcontentloaded" });
+    const rawUrls = await page.$$eval(`${source.nav_selector} a[href]`, (links) => links.map((a) => a.href));
+    const cleanUrls = rawUrls.filter((url) => url.startsWith("http") && !url.includes("?hl=") && !url.endsWith("#"));
+    return filterUrls(cleanUrls, source.include_patterns, source.exclude_patterns);
+}
+export async function fetchPage(page, url) {
+    await page.goto(url, { waitUntil: "domcontentloaded" });
+    return page.content();
+}
+const DEFAULT_CONCURRENCY = 10;
+async function runPool(items, concurrency, fn) {
+    let nextIndex = 0;
+    async function worker() {
+        while (nextIndex < items.length) {
+            const index = nextIndex++;
+            await fn(items[index], index);
+        }
+    }
+    const workers = Array.from({ length: Math.min(concurrency, items.length) }, () => worker());
+    await Promise.all(workers);
+}
+export async function scrapeSource(source, sourceName, dataDir, onProgress) {
+    const rawDir = join(dataDir, "raw", sourceName);
+    await mkdir(rawDir, { recursive: true });
+    const concurrency = source.concurrency ?? DEFAULT_CONCURRENCY;
+    const browser = await chromium.launch({ channel: "chrome" });
+    const context = await browser.newContext();
+    const discoveryPage = await context.newPage();
+    const urls = await discoverUrls(discoveryPage, source);
+    await discoveryPage.close();
+    let completed = 0;
+    try {
+        await runPool(urls, concurrency, async (url) => {
+            const page = await context.newPage();
+            try {
+                const html = await fetchPage(page, url);
+                const slug = slugifyUrl(url);
+                await writeFile(join(rawDir, `${slug}.html`), html, "utf-8");
+                completed++;
+                onProgress?.(completed, urls.length, url);
+            }
+            finally {
+                await page.close();
+            }
+        });
+        return urls;
+    }
+    finally {
+        await browser.close();
+    }
+}
+export async function createBrowser() {
+    return chromium.launch({ channel: "chrome" });
+}
+//# sourceMappingURL=scraper.js.map

package/dist/scraper.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scraper.js","sourceRoot":"","sources":["../src/scraper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,QAAQ,EAA2B,MAAM,YAAY,CAAC;AAG/D,MAAM,UAAU,UAAU,CAAC,GAAW;IACpC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAC5B,OAAO,MAAM,CAAC,QAAQ;SACnB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;SAClB,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;SAClB,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC;SACnB,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,UAAU,CACxB,IAAc,EACd,eAA0B,EAC1B,eAA0B;IAE1B,IAAI,QAAQ,GAAG,IAAI,CAAC;IAEpB,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CACjC,eAAe,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CACzD,CAAC;IACJ,CAAC;IAED,IAAI,eAAe,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClD,QAAQ,GAAG,QAAQ,CAAC,MAAM,CACxB,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CACnE,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;AACvC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAU,EACV,MAAoB;IAEpB,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAErE,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,MAAM,CAC/B,GAAG,MAAM,CAAC,YAAY,UAAU,EAChC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAuB,CAAC,IAAI,CAAC,CAC3D,CAAC;IAEF,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAC9B,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAC/E,CAAC;IAEF,OAAO,UAAU,CAAC,SAAS,EAAE,MAAM,CAAC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC;AACjF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAU,EACV,GAAW;IAEX,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;IACxD,OAAO,IAAI,CAAC,OAAO,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAE/B,KAAK,UAAU,OAAO,CACpB,KAAU,EACV,WAAmB,EACnB,EAA6C;IAE7C,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,UAAU,MAAM;QACnB,OAAO,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;YAC1B,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC;IAC5F,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,MAAoB,EACpB,UAAkB,EAClB,OAAe,EACf,UAAkE;IAElE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IAChD,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEzC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,mBAAmB,CAAC;IAC9D,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC7D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,EAAE,CAAC;IAE3C,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;IAC9C,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;IACvD,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;IAE5B,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;YAC7C,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAC;YACrC,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBACxC,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;gBAC7B,MAAM,SAAS,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,OAAO,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;gBAC7D,SAAS,EAAE,CAAC;gBACZ,UAAU,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;YAC5C,CAAC;oBAAS,CAAC;gBACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACrB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,OAAO,IAAI,CAAC;IACd,CAAC;YAAS,CAAC;QACT,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;IACxB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa;IACjC,OAAO,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;AAChD,CAAC"}

package/dist/search.d.ts

@@ -0,0 +1,8 @@
+import type { SearchResult } from "./types.js";
+export type { SearchResult };
+export declare function search(query: string, options?: {
+    source?: string;
+    candidates?: number;
+    topN?: number;
+}): Promise<SearchResult[]>;
+//# sourceMappingURL=search.d.ts.map

package/dist/search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C,YAAY,EAAE,YAAY,EAAE,CAAC;AAM7B,wBAAsB,MAAM,CAC1B,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO,GACpE,OAAO,CAAC,YAAY,EAAE,CAAC,CAuCzB"}

package/dist/search.js

@@ -0,0 +1,43 @@
+import { embedText } from "./embedder.js";
+import { vectorSearch } from "./store.js";
+import { rerank } from "./reranker.js";
+function hasReranker() {
+    return !!process.env.RERANKER_URL;
+}
+export async function search(query, options = {}) {
+    const { source, candidates = 20, topN = 5 } = options;
+    const queryEmbedding = await embedText(query);
+    const rawResults = await vectorSearch(queryEmbedding, candidates, source);
+    if (rawResults.length === 0)
+        return [];
+    if (hasReranker()) {
+        const documents = rawResults.map((r) => r.data.content);
+        const reranked = await rerank(query, documents, topN);
+        return reranked.map((r) => {
+            const original = rawResults[r.index];
+            const data = original.data;
+            return {
+                id: original.id,
+                source: data.source,
+                url: data.url,
+                title: data.title,
+                heading_path: data.heading_path,
+                content: data.content,
+                relevance_score: r.relevance_score,
+            };
+        });
+    }
+    return rawResults.slice(0, topN).map((r, i) => {
+        const data = r.data;
+        return {
+            id: r.id,
+            source: data.source,
+            url: data.url,
+            title: data.title,
+            heading_path: data.heading_path,
+            content: data.content,
+            relevance_score: 1 - i * 0.1,
+        };
+    });
+}
+//# sourceMappingURL=search.js.map

package/dist/search.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"search.js","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAKvC,SAAS,WAAW;IAClB,OAAO,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;AACpC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,KAAa,EACb,UAAmE,EAAE;IAErE,MAAM,EAAE,MAAM,EAAE,UAAU,GAAG,EAAE,EAAE,IAAI,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAEtD,MAAM,cAAc,GAAG,MAAM,SAAS,CAAC,KAAK,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC,cAAc,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;IAE1E,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEvC,IAAI,WAAW,EAAE,EAAE,CAAC;QAClB,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAiB,CAAC,CAAC;QAClE,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;QAEtD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACxB,MAAM,QAAQ,GAAG,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;YACrC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;YAC3B,OAAO;gBACL,EAAE,EAAE,QAAQ,CAAC,EAAE;gBACf,MAAM,EAAE,IAAI,CAAC,MAAgB;gBAC7B,GAAG,EAAE,IAAI,CAAC,GAAa;gBACvB,KAAK,EAAE,IAAI,CAAC,KAAe;gBAC3B,YAAY,EAAE,IAAI,CAAC,YAAwB;gBAC3C,OAAO,EAAE,IAAI,CAAC,OAAiB;gBAC/B,eAAe,EAAE,CAAC,CAAC,eAAe;aACnC,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC5C,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;QACpB,OAAO;YACL,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,MAAM,EAAE,IAAI,CAAC,MAAgB;YAC7B,GAAG,EAAE,IAAI,CAAC,GAAa;YACvB,KAAK,EAAE,IAAI,CAAC,KAAe;YAC3B,YAAY,EAAE,IAAI,CAAC,YAAwB;YAC3C,OAAO,EAAE,IAAI,CAAC,OAAiB;YAC/B,eAAe,EAAE,CAAC,GAAG,CAAC,GAAG,GAAG;SAC7B,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/store.d.ts

@@ -0,0 +1,11 @@
+import type { Chunk, SourceMeta } from "./types.js";
+export declare function storeChunks(chunks: Chunk[], embeddings: number[][], onProgress?: (current: number, total: number) => void): Promise<void>;
+export declare function purgeSource(sourceName: string): Promise<number>;
+export declare function updateSourceMeta(sourceName: string, chunkCount: number, urlCount: number): Promise<void>;
+export declare function getSourceMeta(sourceName: string): Promise<SourceMeta | null>;
+export declare function getAllSourcesMeta(): Promise<SourceMeta[]>;
+export declare function vectorSearch(queryEmbedding: number[], limit: number, source?: string): Promise<{
+    id: string;
+    data: Record<string, unknown>;
+}[]>;
+//# sourceMappingURL=store.d.ts.map

package/dist/store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.d.ts","sourceRoot":"","sources":["../src/store.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAwBpD,wBAAsB,WAAW,CAC/B,MAAM,EAAE,KAAK,EAAE,EACf,UAAU,EAAE,MAAM,EAAE,EAAE,EACtB,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GACpD,OAAO,CAAC,IAAI,CAAC,CA0Bf;AAED,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CA0BrE;AAED,wBAAsB,gBAAgB,CACpC,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,IAAI,CAAC,CAOf;AAED,wBAAsB,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAIlF;AAED,wBAAsB,iBAAiB,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC,CAG/D;AAED,wBAAsB,YAAY,CAChC,cAAc,EAAE,MAAM,EAAE,EACxB,KAAK,EAAE,MAAM,EACb,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAAE,EAAE,CAAC,CAmB1D"}