@astrofoundry/grimoire 3.12.3 → 3.14.0

package/dist/consumer-config.js

@@ -1,60 +0,0 @@
-import { readFile, writeFile, mkdir } from "node:fs/promises";
-import { join } from "node:path";
-import { homedir } from "node:os";
-import { createInterface } from "node:readline";
-const CONFIG_DIR = join(homedir(), ".grimoire");
-const CONFIG_FILE = join(CONFIG_DIR, "config.json");
-export async function loadConsumerConfig() {
-    const raw = await readFile(CONFIG_FILE, "utf-8").catch(() => null);
-    if (!raw)
-        return null;
-    const data = JSON.parse(raw);
-    if (typeof data.apiUrl === "string" && typeof data.apiKey === "string") {
-        return { apiUrl: data.apiUrl, apiKey: data.apiKey };
-    }
-    return null;
-}
-export async function saveConsumerConfig(config) {
-    await mkdir(CONFIG_DIR, { recursive: true });
-    await writeFile(CONFIG_FILE, JSON.stringify(config, null, 2) + "\n", "utf-8");
-}
-export async function resolveConsumerConfig() {
-    const envUrl = process.env.GRIMOIRE_API_URL;
-    const envKey = process.env.GRIMOIRE_API_KEY;
-    if (envUrl && envKey) {
-        return { apiUrl: envUrl, apiKey: envKey };
-    }
-    const fileConfig = await loadConsumerConfig();
-    if (fileConfig)
-        return fileConfig;
-    throw new Error("Grimoire is not configured. Run 'grimoire init' to set up.");
-}
-export function isConsumerMode() {
-    return !!process.env.GRIMOIRE_API_URL;
-}
-export async function detectConsumerMode() {
-    if (process.env.GOOGLE_APPLICATION_CREDENTIALS)
-        return false;
-    if (process.env.GRIMOIRE_API_URL)
-        return true;
-    const config = await loadConsumerConfig();
-    return config !== null;
-}
-export async function cmdInit() {
-    const rl = createInterface({ input: process.stdin, output: process.stdout });
-    const ask = (q) => new Promise((resolve) => rl.question(q, resolve));
-    const existing = await loadConsumerConfig();
-    const apiUrl = await ask(`API URL${existing ? ` [${existing.apiUrl}]` : ""}: `);
-    const apiKey = await ask(`API Key${existing ? " [****]" : ""}: `);
-    const config = {
-        apiUrl: apiUrl.trim() || existing?.apiUrl || "",
-        apiKey: apiKey.trim() || existing?.apiKey || "",
-    };
-    rl.close();
-    if (!config.apiUrl || !config.apiKey) {
-        throw new Error("Both API URL and API Key are required.");
-    }
-    await saveConsumerConfig(config);
-    console.log(`\nSaved to ${CONFIG_FILE}`);
-}
-//# sourceMappingURL=consumer-config.js.map

package/dist/consumer-config.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"consumer-config.js","sourceRoot":"","sources":["../src/consumer-config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,WAAW,CAAC,CAAC;AAChD,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,EAAE,aAAa,CAAC,CAAC;AAOpD,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC;IACnE,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QACvE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;IACtD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,MAAsB;IAC7D,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7C,MAAM,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;AAChF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB;IACzC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;IAE5C,IAAI,MAAM,IAAI,MAAM,EAAE,CAAC;QACrB,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;IAC5C,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC9C,IAAI,UAAU;QAAE,OAAO,UAAU,CAAC;IAElC,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAC;AAChF,CAAC;AAED,MAAM,UAAU,cAAc;IAC5B,OAAO,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC;AACxC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,IAAI,OAAO,CAAC,GAAG,CAAC,8BAA8B;QAAE,OAAO,KAAK,CAAC;IAC7D,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB;QAAE,OAAO,IAAI,CAAC;IAC9C,MAAM,MAAM,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAC1C,OAAO,MAAM,KAAK,IAAI,CAAC;AACzB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO;IAC3B,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC7E,MAAM,GAAG,GAAG,CAAC,CAAS,EAAmB,EAAE,CACzC,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IAEpD,MAAM,QAAQ,GAAG,MAAM,kBAAkB,EAAE,CAAC;IAE5C,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,UAAU,QAAQ,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAChF,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,UAAU,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAElE,MAAM,MAAM,GAAmB;QAC7B,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,QAAQ,EAAE,MAAM,IAAI,EAAE;QAC/C,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,QAAQ,EAAE,MAAM,IAAI,EAAE;KAChD,CAAC;IAEF,EAAE,CAAC,KAAK,EAAE,CAAC;IAEX,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC5D,CAAC;IAED,MAAM,kBAAkB,CAAC,MAAM,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,cAAc,WAAW,EAAE,CAAC,CAAC;AAC3C,CAAC"}

package/dist/consumer.d.ts

@@ -1,11 +0,0 @@
-import type { ConsumerConfig } from "./consumer-config.js";
-export declare function cmdConsumerSearch(config: ConsumerConfig, query: string, options: {
-    source?: string;
-    topN?: number;
-    compact?: boolean;
-}): Promise<void>;
-export declare function cmdConsumerList(config: ConsumerConfig, options?: {
-    names?: boolean;
-}): Promise<void>;
-export declare function cmdConsumerStats(config: ConsumerConfig): Promise<void>;
-//# sourceMappingURL=consumer.d.ts.map

package/dist/consumer.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"consumer.d.ts","sourceRoot":"","sources":["../src/consumer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AA+B3D,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,cAAc,EACtB,KAAK,EAAE,MAAM,EACb,OAAO,EAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAA;CAAE,GAC7D,OAAO,CAAC,IAAI,CAAC,CAyBf;AAED,wBAAsB,eAAe,CAAC,MAAM,EAAE,cAAc,EAAE,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAqB1G;AAED,wBAAsB,gBAAgB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC,CAkB5E"}

package/dist/consumer.js

@@ -1,84 +0,0 @@
-import { bold, cyan, yellow } from "./format.js";
-async function apiRequest(config, path, options) {
-    const url = `${config.apiUrl.replace(/\/$/, "")}${path}`;
-    let response;
-    try {
-        response = await fetch(url, {
-            ...options,
-            headers: {
-                "Content-Type": "application/json",
-                "x-api-key": config.apiKey,
-                ...options?.headers,
-            },
-        });
-    }
-    catch {
-        throw new Error(`Cannot reach Grimoire API at ${config.apiUrl}. Check your GRIMOIRE_API_URL.`);
-    }
-    if (response.status === 401 || response.status === 403) {
-        throw new Error("Invalid API key. Check your GRIMOIRE_API_KEY or run 'grimoire init'.");
-    }
-    if (!response.ok) {
-        throw new Error(`API error: ${response.status} ${response.statusText}`);
-    }
-    return response.json();
-}
-export async function cmdConsumerSearch(config, query, options) {
-    const data = await apiRequest(config, "/search", {
-        method: "POST",
-        body: JSON.stringify({ query, source: options.source, topN: options.topN }),
-    });
-    if (data.results.length === 0) {
-        console.log("No results found.");
-        return;
-    }
-    if (options.compact) {
-        for (const r of data.results) {
-            console.log(`${r.relevance_score.toFixed(4)} | ${r.source} | ${r.title} | ${r.heading_path.join(" > ")} | ${r.url}`);
-        }
-        return;
-    }
-    for (let i = 0; i < data.results.length; i++) {
-        const r = data.results[i];
-        console.log(`\n${bold(`[${i + 1}] ${r.title}`)} (${r.relevance_score.toFixed(4)})`);
-        console.log(`    ${cyan(r.url)}`);
-        console.log(`    ${yellow(r.heading_path.join(" > "))}`);
-        console.log(`    ${r.content.replace(/\n/g, " ")}`);
-    }
-}
-export async function cmdConsumerList(config, options) {
-    const data = await apiRequest(config, "/list");
-    if (data.sources.length === 0) {
-        console.log("No sources available.");
-        return;
-    }
-    if (options?.names) {
-        for (const s of data.sources) {
-            console.log(s.source);
-        }
-        return;
-    }
-    console.log("\nSources:\n");
-    for (const s of data.sources) {
-        const ver = s.version ? ` v${s.version}` : "";
-        console.log(`  ${bold(s.source)}${ver}`);
-        console.log(`    ${s.chunk_count} chunks, ${s.url_count} URLs, last refreshed ${s.last_refreshed}`);
-    }
-}
-export async function cmdConsumerStats(config) {
-    const data = await apiRequest(config, "/stats");
-    if (data.sources.length === 0) {
-        console.log("No sources have been refreshed yet.");
-        return;
-    }
-    console.log("\nSource Statistics:\n");
-    for (const s of data.sources) {
-        const ver = s.version ? ` v${s.version}` : "";
-        console.log(`  ${bold(s.source)}${ver}`);
-        console.log(`    Chunks: ${s.chunk_count}`);
-        console.log(`    URLs: ${s.url_count}`);
-        console.log(`    Last refreshed: ${s.last_refreshed}`);
-    }
-    console.log(`\n  Total: ${data.totalChunks} chunks across ${data.totalUrls} URLs from ${data.sources.length} sources`);
-}
-//# sourceMappingURL=consumer.js.map

package/dist/consumer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"consumer.js","sourceRoot":"","sources":["../src/consumer.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAEjD,KAAK,UAAU,UAAU,CAAI,MAAsB,EAAE,IAAY,EAAE,OAAqB;IACtF,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC;IAEzD,IAAI,QAAkB,CAAC;IACvB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC1B,GAAG,OAAO;YACV,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,WAAW,EAAE,MAAM,CAAC,MAAM;gBAC1B,GAAG,OAAO,EAAE,OAAO;aACpB;SACF,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,gCAAgC,MAAM,CAAC,MAAM,gCAAgC,CAAC,CAAC;IACjG,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CAAC,sEAAsE,CAAC,CAAC;IAC1F,CAAC;IAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,cAAc,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IAC1E,CAAC;IAED,OAAO,QAAQ,CAAC,IAAI,EAAgB,CAAC;AACvC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAsB,EACtB,KAAa,EACb,OAA8D;IAE9D,MAAM,IAAI,GAAG,MAAM,UAAU,CAA8B,MAAM,EAAE,SAAS,EAAE;QAC5E,MAAM,EAAE,MAAM;QACd,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;KAC5E,CAAC,CAAC;IAEH,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,OAAO;IACT,CAAC;IAED,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACpB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,MAAM,CAAC,CAAC,KAAK,MAAM,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QACvH,CAAC;QACD,OAAO;IACT,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC1B,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACpF,OAAO,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClC,OAAO,CAAC,GAAG,CAAC,OAAO,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;QACzD,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,MAAsB,EAAE,OAA6B;IACzF,MAAM,IAAI,GAAG,MAAM,UAAU,CAA4B,MAAM,EAAE,OAAO,CAAC,CAAC;IAE1E,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;QACrC,OAAO;IACT,CAAC;IAED,IAAI,OAAO,EAAE,KAAK,EAAE,CAAC;QACnB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACxB,CAAC;QACD,OAAO;IACT,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IAC5B,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC;QACzC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,WAAW,YAAY,CAAC,CAAC,SAAS,yBAAyB,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC;IACtG,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAAsB;IAC3D,MAAM,IAAI,GAAG,MAAM,UAAU,CAAoE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAEnH,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;QACnD,OAAO;IACT,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IACtC,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9C,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC;QACzC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;QACxC,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC,cAAc,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,CAAC,WAAW,kBAAkB,IAAI,CAAC,SAAS,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;AACzH,CAAC"}

package/dist/converter.d.ts

@@ -1,12 +0,0 @@
-export interface ConvertedPage {
-    source: string;
-    url: string;
-    title: string;
-    markdown: string;
-}
-export declare function extractContent(html: string, contentSelector: string, removeSelectors?: string[], removeTextPatterns?: string[]): string;
-export declare function extractTitle(html: string): string;
-export declare function buildFrontmatter(source: string, url: string, title: string): string;
-export declare function convertPage(html: string, source: string, url: string, contentSelector: string, removeSelectors?: string[], removeTextPatterns?: string[]): ConvertedPage;
-export declare function convertSource(sourceName: string, urls: string[], contentSelector: string, removeSelectors: string[] | undefined, removeTextPatterns: string[] | undefined, dataDir: string, concurrency?: number, onProgress?: (current: number, total: number, url: string) => void): Promise<ConvertedPage[]>;
-//# sourceMappingURL=converter.d.ts.map

package/dist/converter.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"converter.d.ts","sourceRoot":"","sources":["../src/converter.ts"],"names":[],"mappings":"AAYA,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB;AAyBD,wBAAgB,cAAc,CAC5B,IAAI,EAAE,MAAM,EACZ,eAAe,EAAE,MAAM,EACvB,eAAe,CAAC,EAAE,MAAM,EAAE,EAC1B,kBAAkB,CAAC,EAAE,MAAM,EAAE,GAC5B,MAAM,CAiBR;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAKjD;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,MAAM,CASnF;AAED,wBAAgB,WAAW,CACzB,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,MAAM,EACd,GAAG,EAAE,MAAM,EACX,eAAe,EAAE,MAAM,EACvB,eAAe,CAAC,EAAE,MAAM,EAAE,EAC1B,kBAAkB,CAAC,EAAE,MAAM,EAAE,GAC5B,aAAa,CAOf;AAID,wBAAsB,aAAa,CACjC,UAAU,EAAE,MAAM,EAClB,IAAI,EAAE,MAAM,EAAE,EACd,eAAe,EAAE,MAAM,EACvB,eAAe,EAAE,MAAM,EAAE,GAAG,SAAS,EACrC,kBAAkB,EAAE,MAAM,EAAE,GAAG,SAAS,EACxC,OAAO,EAAE,MAAM,EACf,WAAW,SAAsB,EACjC,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,IAAI,GACjE,OAAO,CAAC,aAAa,EAAE,CAAC,CAgC1B"}

package/dist/converter.js

@@ -1,95 +0,0 @@
-import { readFile, writeFile, mkdir } from "node:fs/promises";
-import { join } from "node:path";
-import { JSDOM } from "jsdom";
-import TurndownService from "turndown";
-import { slugifyUrl } from "./scraper.js";
-const turndown = new TurndownService({
-    headingStyle: "atx",
-    codeBlockStyle: "fenced",
-    bulletListMarker: "-",
-});
-const GENERIC_REMOVE = [
-    "style",
-    "script",
-    "noscript",
-    "iframe",
-    "svg",
-];
-function cleanMarkdown(md, textPatterns) {
-    let cleaned = md
-        .replace(/^(#+)\s*$/gm, "")
-        .replace(/\n{3,}/g, "\n\n");
-    if (textPatterns) {
-        for (const pattern of textPatterns) {
-            cleaned = cleaned.replace(new RegExp(pattern, "gm"), "");
-        }
-        cleaned = cleaned.replace(/\n{3,}/g, "\n\n");
-    }
-    return cleaned.trim();
-}
-export function extractContent(html, contentSelector, removeSelectors, removeTextPatterns) {
-    const dom = new JSDOM(html);
-    const doc = dom.window.document;
-    const contentEl = doc.querySelector(contentSelector);
-    if (!contentEl) {
-        return cleanMarkdown(turndown.turndown(doc.body.innerHTML), removeTextPatterns);
-    }
-    const allSelectors = [...GENERIC_REMOVE, ...(removeSelectors ?? [])];
-    for (const selector of allSelectors) {
-        for (const el of contentEl.querySelectorAll(selector)) {
-            el.remove();
-        }
-    }
-    return cleanMarkdown(turndown.turndown(contentEl.innerHTML), removeTextPatterns);
-}
-export function extractTitle(html) {
-    const dom = new JSDOM(html);
-    const titleEl = dom.window.document.querySelector("title");
-    if (!titleEl)
-        return "Untitled";
-    return titleEl.textContent?.replace(/\s*[|–—-]\s*.+$/, "").trim() ?? "Untitled";
-}
-export function buildFrontmatter(source, url, title) {
-    return [
-        "---",
-        `source: ${source}`,
-        `url: "${url}"`,
-        `title: "${title.replace(/"/g, '\\"')}"`,
-        `fetched_at: "${new Date().toISOString()}"`,
-        "---",
-    ].join("\n");
-}
-export function convertPage(html, source, url, contentSelector, removeSelectors, removeTextPatterns) {
-    const title = extractTitle(html);
-    const content = extractContent(html, contentSelector, removeSelectors, removeTextPatterns);
-    const frontmatter = buildFrontmatter(source, url, title);
-    const markdown = `${frontmatter}\n\n${content}`;
-    return { source, url, title, markdown };
-}
-const DEFAULT_CONCURRENCY = 10;
-export async function convertSource(sourceName, urls, contentSelector, removeSelectors, removeTextPatterns, dataDir, concurrency = DEFAULT_CONCURRENCY, onProgress) {
-    const rawDir = join(dataDir, "raw", sourceName);
-    const mdDir = join(dataDir, "markdown", sourceName);
-    await mkdir(mdDir, { recursive: true });
-    const pages = new Array(urls.length);
-    let completed = 0;
-    let nextIndex = 0;
-    async function worker() {
-        while (nextIndex < urls.length) {
-            const i = nextIndex++;
-            const url = urls[i];
-            const slug = slugifyUrl(url);
-            const htmlPath = join(rawDir, `${slug}.html`);
-            const html = await readFile(htmlPath, "utf-8");
-            const page = convertPage(html, sourceName, url, contentSelector, removeSelectors, removeTextPatterns);
-            await writeFile(join(mdDir, `${slug}.md`), page.markdown, "utf-8");
-            pages[i] = page;
-            completed++;
-            onProgress?.(completed, urls.length, url);
-        }
-    }
-    const workers = Array.from({ length: Math.min(concurrency, urls.length) }, () => worker());
-    await Promise.all(workers);
-    return pages;
-}
-//# sourceMappingURL=converter.js.map

package/dist/converter.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"converter.js","sourceRoot":"","sources":["../src/converter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE1C,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC;IACnC,YAAY,EAAE,KAAK;IACnB,cAAc,EAAE,QAAQ;IACxB,gBAAgB,EAAE,GAAG;CACtB,CAAC,CAAC;AASH,MAAM,cAAc,GAAG;IACrB,OAAO;IACP,QAAQ;IACR,UAAU;IACV,QAAQ;IACR,KAAK;CACN,CAAC;AAEF,SAAS,aAAa,CAAC,EAAU,EAAE,YAAuB;IACxD,IAAI,OAAO,GAAG,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC;SAC1B,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAE9B,IAAI,YAAY,EAAE,CAAC;QACjB,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACnC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3D,CAAC;QACD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC/C,CAAC;IAED,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,IAAY,EACZ,eAAuB,EACvB,eAA0B,EAC1B,kBAA6B;IAE7B,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC;IAEhC,MAAM,SAAS,GAAG,GAAG,CAAC,aAAa,CAAC,eAAe,CAAC,CAAC;IACrD,IAAI,CAAC,SAAS,EAAE,CAAC;QACf,OAAO,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;IAClF,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,GAAG,cAAc,EAAE,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC,CAAC,CAAC;IACrE,KAAK,MAAM,QAAQ,IAAI,YAAY,EAAE,CAAC;QACpC,KAAK,MAAM,EAAE,IAAI,SAAS,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC;YACtD,EAAE,CAAC,MAAM,EAAE,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,kBAAkB,CAAC,CAAC;AACnF,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAC3D,IAAI,CAAC,OAAO;QAAE,OAAO,UAAU,CAAC;IAChC,OAAO,OAAO,CAAC,WAAW,EAAE,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,UAAU,CAAC;AAClF,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,GAAW,EAAE,KAAa;IACzE,OAAO;QACL,KAAK;QACL,WAAW,MAAM,EAAE;QACnB,SAAS,GAAG,GAAG;QACf,WAAW,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG;QACxC,gBAAgB,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,GAAG;QAC3C,KAAK;KACN,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,IAAY,EACZ,MAAc,EACd,GAAW,EACX,eAAuB,EACvB,eAA0B,EAC1B,kBAA6B;IAE7B,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,EAAE,eAAe,EAAE,eAAe,EAAE,kBAAkB,CAAC,CAAC;IAC3F,MAAM,WAAW,GAAG,gBAAgB,CAAC,MAAM,EAAE,GAAG,EAAE,KAAK,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,GAAG,WAAW,OAAO,OAAO,EAAE,CAAC;IAEhD,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAC1C,CAAC;AAED,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAE/B,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAAkB,EAClB,IAAc,EACd,eAAuB,EACvB,eAAqC,EACrC,kBAAwC,EACxC,OAAe,EACf,WAAW,GAAG,mBAAmB,EACjC,UAAkE;IAElE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;IACpD,MAAM,KAAK,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,MAAM,KAAK,GAAoB,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACtD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,KAAK,UAAU,MAAM;QACnB,OAAO,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;YAC/B,MAAM,CAAC,GAAG,SAAS,EAAE,CAAC;YACtB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YACpB,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,OAAO,CAAC,CAAC;YAC9C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAE/C,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,eAAe,EAAE,eAAe,EAAE,kBAAkB,CAAC,CAAC;YACtG,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,KAAK,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACnE,KAAK,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;YAChB,SAAS,EAAE,CAAC;YACZ,UAAU,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CACxB,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,EAC9C,GAAG,EAAE,CAAC,MAAM,EAAE,CACf,CAAC;IACF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAE3B,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/embedder.d.ts

@@ -1,9 +0,0 @@
-export interface EmbedOptions {
-    onProgress?: (done: number, total: number) => void;
-    onCheckpoint?: (embeddings: number[][]) => Promise<void>;
-    checkpointEveryBatches?: number;
-    resumeFrom?: number[][];
-}
-export declare function embedTexts(texts: string[], options?: EmbedOptions): Promise<number[][]>;
-export declare function embedText(text: string): Promise<number[]>;
-//# sourceMappingURL=embedder.d.ts.map

package/dist/embedder.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AA6CA,MAAM,WAAW,YAAY;IAC3B,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACnD,YAAY,CAAC,EAAE,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;IACzD,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,UAAU,CAAC,EAAE,MAAM,EAAE,EAAE,CAAC;CACzB;AAED,wBAAsB,UAAU,CAC9B,KAAK,EAAE,MAAM,EAAE,EACf,OAAO,GAAE,YAAiB,GACzB,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CA8ErB;AAED,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAG/D"}

package/dist/embedder.js

@@ -1,108 +0,0 @@
-import { GoogleGenerativeAI } from "@google/generative-ai";
-const BATCH_SIZE = 50;
-const MODEL = "gemini-embedding-001";
-const OUTPUT_DIMENSIONALITY = 768;
-const MAX_RETRIES = 5;
-const RATE_LIMIT_BASE_DELAY_MS = 60000;
-const NETWORK_BASE_DELAY_MS = 10000;
-const BATCH_DELAY_MS = 2500;
-const DEFAULT_CHECKPOINT_EVERY_BATCHES = 20;
-const NETWORK_ERROR_PATTERNS = [
-    "fetch failed",
-    "ECONNRESET",
-    "ETIMEDOUT",
-    "ECONNREFUSED",
-    "EAI_AGAIN",
-    "ENOTFOUND",
-    "socket hang up",
-    "UND_ERR_",
-];
-let genAI;
-function getClient() {
-    if (!genAI) {
-        const apiKey = process.env.GEMINI_API_KEY;
-        if (!apiKey) {
-            throw new Error("GEMINI_API_KEY environment variable is not set");
-        }
-        genAI = new GoogleGenerativeAI(apiKey);
-    }
-    return genAI;
-}
-function classifyError(message) {
-    if (message.includes("429") || message.includes("503")) {
-        return "rate_limit";
-    }
-    if (NETWORK_ERROR_PATTERNS.some((p) => message.includes(p))) {
-        return "network";
-    }
-    return "other";
-}
-export async function embedTexts(texts, options = {}) {
-    const client = getClient();
-    const model = client.getGenerativeModel({ model: MODEL });
-    const { onProgress, onCheckpoint, resumeFrom } = options;
-    const checkpointEveryBatches = options.checkpointEveryBatches ?? DEFAULT_CHECKPOINT_EVERY_BATCHES;
-    const embeddings = resumeFrom ? [...resumeFrom] : [];
-    const startIndex = Math.floor(embeddings.length / BATCH_SIZE) * BATCH_SIZE;
-    if (embeddings.length > startIndex) {
-        embeddings.length = startIndex;
-    }
-    if (startIndex > 0) {
-        console.log(`  Resuming from chunk ${startIndex} of ${texts.length} (${embeddings.length} cached).`);
-    }
-    if (startIndex >= texts.length) {
-        return embeddings.slice(0, texts.length);
-    }
-    let batchesSinceCheckpoint = 0;
-    for (let i = startIndex; i < texts.length; i += BATCH_SIZE) {
-        const batch = texts.slice(i, i + BATCH_SIZE);
-        const batchNumber = i / BATCH_SIZE + 1;
-        let result;
-        for (let attempt = 0; attempt < MAX_RETRIES; attempt++) {
-            try {
-                result = await model.batchEmbedContents({
-                    requests: batch.map((text) => ({
-                        content: { role: "user", parts: [{ text }] },
-                        outputDimensionality: OUTPUT_DIMENSIONALITY,
-                    })),
-                });
-                break;
-            }
-            catch (err) {
-                const message = err instanceof Error ? err.message : String(err);
-                const kind = classifyError(message);
-                if (kind !== "other" && attempt < MAX_RETRIES - 1) {
-                    const baseDelay = kind === "rate_limit" ? RATE_LIMIT_BASE_DELAY_MS : NETWORK_BASE_DELAY_MS;
-                    const delay = baseDelay * Math.pow(2, attempt);
-                    const label = kind === "rate_limit" ? "Rate limited" : "Network error";
-                    console.log(`  ${label} (batch ${batchNumber}), retrying in ${delay / 1000}s...`);
-                    await new Promise((resolve) => setTimeout(resolve, delay));
-                    continue;
-                }
-                console.error(`  Embedding failed at batch ${batchNumber} (chunks ${i + 1}-${i + batch.length}): ${message}`);
-                throw err;
-            }
-        }
-        for (const embedding of result.embeddings) {
-            embeddings.push(embedding.values);
-        }
-        onProgress?.(Math.min(i + BATCH_SIZE, texts.length), texts.length);
-        batchesSinceCheckpoint++;
-        if (onCheckpoint && batchesSinceCheckpoint >= checkpointEveryBatches && i + BATCH_SIZE < texts.length) {
-            await onCheckpoint(embeddings);
-            batchesSinceCheckpoint = 0;
-        }
-        if (i + BATCH_SIZE < texts.length) {
-            await new Promise((resolve) => setTimeout(resolve, BATCH_DELAY_MS));
-        }
-    }
-    if (onCheckpoint) {
-        await onCheckpoint(embeddings);
-    }
-    return embeddings;
-}
-export async function embedText(text) {
-    const [embedding] = await embedTexts([text]);
-    return embedding;
-}
-//# sourceMappingURL=embedder.js.map

package/dist/embedder.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAC;AAE3D,MAAM,UAAU,GAAG,EAAE,CAAC;AACtB,MAAM,KAAK,GAAG,sBAAsB,CAAC;AACrC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAClC,MAAM,WAAW,GAAG,CAAC,CAAC;AACtB,MAAM,wBAAwB,GAAG,KAAK,CAAC;AACvC,MAAM,qBAAqB,GAAG,KAAK,CAAC;AACpC,MAAM,cAAc,GAAG,IAAI,CAAC;AAC5B,MAAM,gCAAgC,GAAG,EAAE,CAAC;AAE5C,MAAM,sBAAsB,GAAG;IAC7B,cAAc;IACd,YAAY;IACZ,WAAW;IACX,cAAc;IACd,WAAW;IACX,WAAW;IACX,gBAAgB;IAChB,UAAU;CACX,CAAC;AAEF,IAAI,KAAqC,CAAC;AAE1C,SAAS,SAAS;IAChB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACpE,CAAC;QACD,KAAK,GAAG,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,OAAe;IACpC,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACvD,OAAO,YAAY,CAAC;IACtB,CAAC;IACD,IAAI,sBAAsB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5D,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AASD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAe,EACf,UAAwB,EAAE;IAE1B,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAC3B,MAAM,KAAK,GAAG,MAAM,CAAC,kBAAkB,CAAC,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IAE1D,MAAM,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,GAAG,OAAO,CAAC;IACzD,MAAM,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,IAAI,gCAAgC,CAAC;IAElG,MAAM,UAAU,GAAe,UAAU,CAAC,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IACjE,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,GAAG,UAAU,CAAC;IAE3E,IAAI,UAAU,CAAC,MAAM,GAAG,UAAU,EAAE,CAAC;QACnC,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC;IACjC,CAAC;IAED,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACnB,OAAO,CAAC,GAAG,CAAC,yBAAyB,UAAU,OAAO,KAAK,CAAC,MAAM,KAAK,UAAU,CAAC,MAAM,WAAW,CAAC,CAAC;IACvG,CAAC;IAED,IAAI,UAAU,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAC/B,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3C,CAAC;IAED,IAAI,sBAAsB,GAAG,CAAC,CAAC;IAE/B,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;QAC3D,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;QAC7C,MAAM,WAAW,GAAG,CAAC,GAAG,UAAU,GAAG,CAAC,CAAC;QAEvC,IAAI,MAAM,CAAC;QACX,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;YACvD,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,KAAK,CAAC,kBAAkB,CAAC;oBACtC,QAAQ,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;wBAC7B,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE;wBAC5C,oBAAoB,EAAE,qBAAqB;qBAC5C,CAAC,CAAC;iBACJ,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gBACjE,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC;gBAEpC,IAAI,IAAI,KAAK,OAAO,IAAI,OAAO,GAAG,WAAW,GAAG,CAAC,EAAE,CAAC;oBAClD,MAAM,SAAS,GAAG,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAC,CAAC,qBAAqB,CAAC;oBAC3F,MAAM,KAAK,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;oBAC/C,MAAM,KAAK,GAAG,IAAI,KAAK,YAAY,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,eAAe,CAAC;oBACvE,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,WAAW,WAAW,kBAAkB,KAAK,GAAG,IAAI,MAAM,CAAC,CAAC;oBAClF,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;oBAC3D,SAAS;gBACX,CAAC;gBAED,OAAO,CAAC,KAAK,CAAC,+BAA+B,WAAW,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,MAAM,OAAO,EAAE,CAAC,CAAC;gBAC9G,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,MAAO,CAAC,UAAU,EAAE,CAAC;YAC3C,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QACpC,CAAC;QAED,UAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAEnE,sBAAsB,EAAE,CAAC;QACzB,IAAI,YAAY,IAAI,sBAAsB,IAAI,sBAAsB,IAAI,CAAC,GAAG,UAAU,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACtG,MAAM,YAAY,CAAC,UAAU,CAAC,CAAC;YAC/B,sBAAsB,GAAG,CAAC,CAAC;QAC7B,CAAC;QAED,IAAI,CAAC,GAAG,UAAU,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAClC,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC;QACtE,CAAC;IACH,CAAC;IAED,IAAI,YAAY,EAAE,CAAC;QACjB,MAAM,YAAY,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY;IAC1C,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7C,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/dist/format.d.ts

@@ -1,5 +0,0 @@
-export declare const bold: (s: string) => string;
-export declare const cyan: (s: string) => string;
-export declare const yellow: (s: string) => string;
-export declare const red: (s: string) => string;
-//# sourceMappingURL=format.d.ts.map

package/dist/format.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../src/format.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,IAAI,GAAI,GAAG,MAAM,KAAG,MAA0C,CAAC;AAC5E,eAAO,MAAM,IAAI,GAAI,GAAG,MAAM,KAAG,MAA2C,CAAC;AAC7E,eAAO,MAAM,MAAM,GAAI,GAAG,MAAM,KAAG,MAA2C,CAAC;AAC/E,eAAO,MAAM,GAAG,GAAI,GAAG,MAAM,KAAG,MAA2C,CAAC"}

package/dist/format.js

@@ -1,6 +0,0 @@
-const isTTY = process.stdout.isTTY ?? false;
-export const bold = (s) => isTTY ? `\x1b[1m${s}\x1b[0m` : s;
-export const cyan = (s) => isTTY ? `\x1b[36m${s}\x1b[0m` : s;
-export const yellow = (s) => isTTY ? `\x1b[33m${s}\x1b[0m` : s;
-export const red = (s) => isTTY ? `\x1b[31m${s}\x1b[0m` : s;
-//# sourceMappingURL=format.js.map

package/dist/format.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"format.js","sourceRoot":"","sources":["../src/format.ts"],"names":[],"mappings":"AAAA,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC;AAE5C,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5E,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7E,MAAM,CAAC,MAAM,MAAM,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/E,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC"}

package/dist/llms-ingest.d.ts

@@ -1,3 +0,0 @@
-import { type ConvertedPage } from "./converter.js";
-export declare function ingestLlmsFull(llmsFullUrl: string, sourceName: string, baseUrl: string, dataDir: string, onProgress?: (current: number, total: number) => void): Promise<ConvertedPage[]>;
-//# sourceMappingURL=llms-ingest.d.ts.map

package/dist/llms-ingest.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"llms-ingest.d.ts","sourceRoot":"","sources":["../src/llms-ingest.ts"],"names":[],"mappings":"AAGA,OAAO,EAAoB,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAsEtE,wBAAsB,cAAc,CAClC,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GACpD,OAAO,CAAC,aAAa,EAAE,CAAC,CAiC1B"}

package/dist/llms-ingest.js

@@ -1,85 +0,0 @@
-import { writeFile, mkdir } from "node:fs/promises";
-import { join } from "node:path";
-import { slugifyUrl } from "./scraper.js";
-import { buildFrontmatter } from "./converter.js";
-const BOILERPLATE_PATTERNS = [
-    /^\[Skip to content\]\([^)]*\)\s*$/gm,
-    /^Was this helpful\?\s*$/gm,
-    /^YesNo\s*$/gm,
-    /^\[ Edit page \]\([^)]+\) \[ Report issue \]\([^)]+\)\s*$/gm,
-    /^Copy page\s*$/gm,
-    /^```json\n\{"@context":"https:\/\/schema\.org","@type":"BreadcrumbList"[^`]*```\s*$/gm,
-];
-function splitPages(content) {
-    const pages = [];
-    const frontmatterPattern = /^---\ntitle: (.+)\n/gm;
-    const boundaries = [];
-    let match;
-    while ((match = frontmatterPattern.exec(content)) !== null) {
-        boundaries.push({ index: match.index, title: match[1] });
-    }
-    for (let i = 0; i < boundaries.length; i++) {
-        const start = boundaries[i].index;
-        const end = i + 1 < boundaries.length ? boundaries[i + 1].index : content.length;
-        const raw = content.slice(start, end).trimEnd();
-        const url = extractUrl(raw);
-        if (!url)
-            continue;
-        const bodyStart = raw.indexOf("---", 3);
-        if (bodyStart === -1)
-            continue;
-        const body = raw.slice(raw.indexOf("\n", bodyStart) + 1);
-        let cleaned = body;
-        for (const pattern of BOILERPLATE_PATTERNS) {
-            cleaned = cleaned.replace(pattern, "");
-        }
-        cleaned = cleaned.replace(/\n{3,}/g, "\n\n").trim();
-        if (!cleaned)
-            continue;
-        pages.push({
-            title: boundaries[i].title,
-            url,
-            markdown: cleaned,
-        });
-    }
-    return pages;
-}
-function extractUrl(pageContent) {
-    const match = pageContent.match(/```json\n\{"@context":"https:\/\/schema\.org","@type":"BreadcrumbList","itemListElement":\[(.+?)\]\}\n```/);
-    if (!match)
-        return null;
-    const items = JSON.parse(`[${match[1]}]`);
-    const last = items[items.length - 1];
-    if (!last?.item?.["@id"])
-        return null;
-    return `https://developers.cloudflare.com${last.item["@id"]}`;
-}
-export async function ingestLlmsFull(llmsFullUrl, sourceName, baseUrl, dataDir, onProgress) {
-    const response = await fetch(llmsFullUrl);
-    if (!response.ok) {
-        throw new Error(`Failed to fetch ${llmsFullUrl}: ${response.status} ${response.statusText}`);
-    }
-    const content = await response.text();
-    const pages = splitPages(content);
-    const mdDir = join(dataDir, "markdown", sourceName);
-    await mkdir(mdDir, { recursive: true });
-    const results = [];
-    for (let i = 0; i < pages.length; i++) {
-        const page = pages[i];
-        const frontmatter = buildFrontmatter(sourceName, page.url, page.title);
-        const fullMarkdown = `${frontmatter}\n\n${page.markdown}`;
-        const slug = slugifyUrl(page.url);
-        await writeFile(join(mdDir, `${slug}.md`), fullMarkdown, "utf-8");
-        results.push({
-            source: sourceName,
-            url: page.url,
-            title: page.title,
-            markdown: fullMarkdown,
-        });
-        if (onProgress && ((i + 1) % 100 === 0 || i + 1 === pages.length)) {
-            onProgress(i + 1, pages.length);
-        }
-    }
-    return results;
-}
-//# sourceMappingURL=llms-ingest.js.map

package/dist/llms-ingest.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"llms-ingest.js","sourceRoot":"","sources":["../src/llms-ingest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,gBAAgB,EAAsB,MAAM,gBAAgB,CAAC;AAQtE,MAAM,oBAAoB,GAAG;IAC3B,qCAAqC;IACrC,2BAA2B;IAC3B,cAAc;IACd,6DAA6D;IAC7D,kBAAkB;IAClB,uFAAuF;CACxF,CAAC;AAEF,SAAS,UAAU,CAAC,OAAe;IACjC,MAAM,KAAK,GAAe,EAAE,CAAC;IAC7B,MAAM,kBAAkB,GAAG,uBAAuB,CAAC;IACnD,MAAM,UAAU,GAAuC,EAAE,CAAC;IAE1D,IAAI,KAAK,CAAC;IACV,OAAO,CAAC,KAAK,GAAG,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QAC3D,UAAU,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAClC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC;QACjF,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC;QAEhD,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;QAC5B,IAAI,CAAC,GAAG;YAAE,SAAS;QAEnB,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QACxC,IAAI,SAAS,KAAK,CAAC,CAAC;YAAE,SAAS;QAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzD,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,KAAK,MAAM,OAAO,IAAI,oBAAoB,EAAE,CAAC;YAC3C,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QACzC,CAAC;QACD,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;QAEpD,IAAI,CAAC,OAAO;YAAE,SAAS;QAEvB,KAAK,CAAC,IAAI,CAAC;YACT,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK;YAC1B,GAAG;YACH,QAAQ,EAAE,OAAO;SAClB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,UAAU,CAAC,WAAmB;IACrC,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,CAC7B,2GAA2G,CAC5G,CAAC;IACF,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IAExB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,GAAG,CAAmC,CAAC;IAC5E,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACrC,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAEtC,OAAO,oCAAoC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;AAChE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,WAAmB,EACnB,UAAkB,EAClB,OAAe,EACf,OAAe,EACf,UAAqD;IAErD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,WAAW,CAAC,CAAC;IAC1C,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,mBAAmB,WAAW,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IAC/F,CAAC;IACD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IAEtC,MAAM,KAAK,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;IAElC,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;IACpD,MAAM,KAAK,CAAC,KAAK,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAExC,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,MAAM,WAAW,GAAG,gBAAgB,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QACvE,MAAM,YAAY,GAAG,GAAG,WAAW,OAAO,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC1D,MAAM,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClC,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,IAAI,KAAK,CAAC,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;QAElE,OAAO,CAAC,IAAI,CAAC;YACX,MAAM,EAAE,UAAU;YAClB,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE,YAAY;SACvB,CAAC,CAAC;QAEH,IAAI,UAAU,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAClE,UAAU,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/reranker.d.ts

@@ -1,6 +0,0 @@
-export interface RerankResult {
-    index: number;
-    relevance_score: number;
-}
-export declare function rerank(query: string, documents: string[], topN?: number): Promise<RerankResult[]>;
-//# sourceMappingURL=reranker.d.ts.map

package/dist/reranker.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"reranker.d.ts","sourceRoot":"","sources":["../src/reranker.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,MAAM,CAAC;CACzB;AAcD,wBAAsB,MAAM,CAC1B,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,MAAM,EAAE,EACnB,IAAI,SAAI,GACP,OAAO,CAAC,YAAY,EAAE,CAAC,CAczB"}

package/dist/reranker.js

@@ -1,21 +0,0 @@
-function getRerankerUrl() {
-    const url = process.env.RERANKER_URL;
-    if (!url) {
-        throw new Error("RERANKER_URL environment variable is not set");
-    }
-    return url;
-}
-export async function rerank(query, documents, topN = 5) {
-    const baseUrl = getRerankerUrl();
-    const response = await fetch(`${baseUrl}/v1/rerank`, {
-        method: "POST",
-        headers: { "Content-Type": "application/json" },
-        body: JSON.stringify({ query, documents, top_n: topN }),
-    });
-    if (!response.ok) {
-        throw new Error(`Reranker request failed: ${response.status} ${response.statusText}`);
-    }
-    const data = (await response.json());
-    return data.results;
-}
-//# sourceMappingURL=reranker.js.map

package/dist/reranker.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"reranker.js","sourceRoot":"","sources":["../src/reranker.ts"],"names":[],"mappings":"AASA,SAAS,cAAc;IACrB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC;IACrC,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,KAAa,EACb,SAAmB,EACnB,IAAI,GAAG,CAAC;IAER,MAAM,OAAO,GAAG,cAAc,EAAE,CAAC;IACjC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,OAAO,YAAY,EAAE;QACnD,MAAM,EAAE,MAAM;QACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;QAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;KACxD,CAAC,CAAC;IAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;IACxF,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAA2B,CAAC;IAC/D,OAAO,IAAI,CAAC,OAAO,CAAC;AACtB,CAAC"}

package/dist/scraper.d.ts

@@ -1,9 +0,0 @@
-import { type Browser, type Page } from "playwright";
-import type { SourceConfig } from "./config.js";
-export declare function slugifyUrl(url: string): string;
-export declare function filterUrls(urls: string[], includePatterns?: string[], excludePatterns?: string[]): string[];
-export declare function discoverUrls(page: Page, source: SourceConfig): Promise<string[]>;
-export declare function fetchPage(page: Page, url: string, headed?: boolean): Promise<string>;
-export declare function scrapeSource(source: SourceConfig, sourceName: string, dataDir: string, onProgress?: (current: number, total: number, url: string) => void): Promise<string[]>;
-export declare function createBrowser(): Promise<Browser>;
-//# sourceMappingURL=scraper.d.ts.map

package/dist/scraper.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"scraper.d.ts","sourceRoot":"","sources":["../src/scraper.ts"],"names":[],"mappings":"AAEA,OAAO,EAAY,KAAK,OAAO,EAAE,KAAK,IAAI,EAAE,MAAM,YAAY,CAAC;AAC/D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,wBAAgB,UAAU,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAO9C;AAED,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,EAAE,EACd,eAAe,CAAC,EAAE,MAAM,EAAE,EAC1B,eAAe,CAAC,EAAE,MAAM,EAAE,GACzB,MAAM,EAAE,CAkBV;AAoBD,wBAAsB,YAAY,CAChC,IAAI,EAAE,IAAI,EACV,MAAM,EAAE,YAAY,GACnB,OAAO,CAAC,MAAM,EAAE,CAAC,CAiBnB;AAED,wBAAsB,SAAS,CAC7B,IAAI,EAAE,IAAI,EACV,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,OAAO,GACf,OAAO,CAAC,MAAM,CAAC,CAGjB;AAsBD,wBAAsB,YAAY,CAChC,MAAM,EAAE,YAAY,EACpB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,EACf,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,KAAK,IAAI,GACjE,OAAO,CAAC,MAAM,EAAE,CAAC,CAiCnB;AAED,wBAAsB,aAAa,IAAI,OAAO,CAAC,OAAO,CAAC,CAEtD"}