@staticn0va/wigolo 0.6.6 → 1.0.0

package/dist/cli/warmup.js

@@ -3,7 +3,9 @@ import { join } from 'node:path';
 import { getConfig } from '../config.js';
 import { checkPythonAvailable, bootstrapNativeSearxng, getBootstrapState } from '../searxng/bootstrap.js';
 import { isProcessAlive } from '../searxng/process.js';
-import { resetAvailabilityCache } from '../search/flashrank.js';
+import { downloadModelAssets } from '../search/reranker/download.js';
+import { onnxRerank } from '../search/reranker/onnx.js';
+import { resolveModelId } from '../search/reranker/models.js';
 import { getPythonBin } from '../python-env.js';
 import { runCommand } from './tui/run-command.js';
 import { autoReporter } from './tui/reporter-auto.js';
@@ -53,18 +55,20 @@ async function installTrafilatura(dataDir, reporter) {
     reporter.fail('trafilatura', message);
     return 'failed';
 }
-async function installFlashRank(dataDir, reporter) {
-    reporter.start('flashrank', 'Installing ML reranker (flashrank)');
-    const py = getPythonBin(dataDir);
-    const r = await runCommand(py, ['-m', 'pip', 'install', '--quiet', 'flashrank'], { timeout: 180000 });
-    if (r.code === 0) {
-        resetAvailabilityCache();
-        reporter.success('flashrank', 'installed');
+async function downloadOnnxReranker(dataDir, reporter) {
+    const modelId = resolveModelId(getConfig().rerankerModel);
+    reporter.start('reranker', `Installing ML reranker (${modelId})`);
+    try {
+        await downloadModelAssets(modelId, dataDir, reporter);
+        await onnxRerank('warmup', [{ text: 'hello world' }], { modelId });
+        reporter.success('reranker', 'installed');
         return { reranker: 'ok' };
     }
-    const message = (r.stderr || r.stdout || `exit ${r.code}`).trim();
-    reporter.fail('flashrank', message);
-    return { reranker: 'failed', rerankerError: message };
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        reporter.fail('reranker', message);
+        return { reranker: 'failed', rerankerError: message };
+    }
 }
 async function installFirefox(reporter) {
     reporter.start('firefox', 'Installing browser engine (firefox)');
@@ -207,7 +211,7 @@ export async function runWarmup(flags = [], reporter) {
     }
     let rerankerResult = {};
     if (flagSet.has('--reranker') || flagSet.has('--all')) {
-        rerankerResult = await installFlashRank(config.dataDir, reporterImpl);
+        rerankerResult = await downloadOnnxReranker(config.dataDir, reporterImpl);
     }
     let firefoxResult = {};
     if (flagSet.has('--firefox') || flagSet.has('--all')) {

package/dist/cli/warmup.js.map

@@ -1 +1 @@
-{"version":3,"file":"warmup.js","sourceRoot":"","sources":["../../src/cli/warmup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpG,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC1G,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,SAAS,IAAI,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAoB5D,SAAS,gBAAgB,CAAC,OAAe,EAAE,QAAwB;IACjE,MAAM,iBAAiB,GAAG,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;IAC1D,IAAI,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAqB,CAAC;YACtF,IAAI,IAAI,CAAC,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzC,MAAM,IAAI,KAAK,CACb,gEAAgE,IAAI,CAAC,GAAG,KAAK;oBAC7E,uBAAuB,IAAI,CAAC,GAAG,EAAE,CAClC,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,gBAAgB,CAAC;gBAAE,MAAM,GAAG,CAAC;QAClF,CAAC;IACH,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACrD,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnE,MAAM,CAAC,iBAAiB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACvD,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACvD,QAAQ,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;AAC3E,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,QAAwB;IACvD,QAAQ,CAAC,KAAK,CAAC,YAAY,EAAE,sCAAsC,CAAC,CAAC;IACrE,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC,YAAY,EAAE,SAAS,EAAE,UAAU,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9F,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACjB,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;QAC5C,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IAC9B,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClE,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IACrC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC;AAC5D,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,OAAe,EAAE,QAAwB;IACzE,QAAQ,CAAC,KAAK,CAAC,aAAa,EAAE,4CAA4C,CAAC,CAAC;IAC5E,MAAM,EAAE,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACjC,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IACxG,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACjB,QAAQ,CAAC,OAAO,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClE,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IACtC,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,OAAe,EAAE,QAAwB;IACvE,QAAQ,CAAC,KAAK,CAAC,WAAW,EAAE,oCAAoC,CAAC,CAAC;IAClE,MAAM,EAAE,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACjC,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IACtG,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACjB,sBAAsB,EAAE,CAAC;QACzB,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QAC3C,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC5B,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClE,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACpC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;AACxD,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,QAAwB;IACpD,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,qCAAqC,CAAC,CAAC;IACjE,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC,YAAY,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAC7F,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACjB,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClE,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAClC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;AACtD,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,QAAwB;IACnD,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,oCAAoC,CAAC,CAAC;IAC/D,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC,YAAY,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5F,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACjB,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QACxC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC1B,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClE,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACjC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC;AACpD,CAAC;AAED,KAAK,UAAU,2BAA2B,CAAC,OAAe,EAAE,QAAwB;IAClF,QAAQ,CAAC,KAAK,CAAC,YAAY,EAAE,wDAAwD,CAAC,CAAC;IACvF,MAAM,EAAE,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACjC,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,uBAAuB,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAClH,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACjB,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;QAC5C,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IAC9B,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClE,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IACrC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC;AAC5D,CAAC;AAED,SAAS,gBAAgB;IACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAClC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1B,MAAM,IAAI,GAAG,oEAAoE,CAAC;IAClF,IAAI,QAAQ,KAAK,QAAQ,IAAI,IAAI,KAAK,OAAO;QAAE,OAAO,GAAG,IAAI,2BAA2B,CAAC;IACzF,IAAI,QAAQ,KAAK,OAAO,IAAI,IAAI,KAAK,KAAK;QAAE,OAAO,GAAG,IAAI,0BAA0B,CAAC;IACrF,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAC;AACtE,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,QAAwB;IACvD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAC3C,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACxB,QAAQ,CAAC,KAAK,CAAC,YAAY,EAAE,uBAAuB,CAAC,CAAC;YACtD,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,mBAAmB,CAAC,CAAC;YACpD,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QAC9B,CAAC;QACD,MAAM,GAAG,GAAG,gBAAgB,EAAE,CAAC;QAC/B,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEvC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAClD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;QAEnE,QAAQ,CAAC,KAAK,CAAC,YAAY,EAAE,wBAAwB,EAAE,EAAE,UAAU,EAAE,UAAU,IAAI,SAAS,EAAE,CAAC,CAAC;QAEhG,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACzC,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;QACrC,MAAM,EAAE,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC;YACH,OAAO,IAAI,EAAE,CAAC;gBACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,IAAI;oBAAE,MAAM;gBAChB,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;oBAAE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;gBACtF,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;gBAC/B,IAAI,UAAU,GAAG,CAAC;oBAAE,QAAQ,CAAC,QAAQ,CAAC,YAAY,EAAE,UAAU,GAAG,UAAU,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC3E,CAAC;QAED,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC1B,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;QAC5C,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IAC9B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACrC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC;IAC5D,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,OAAe,EAAE,QAAwB;IACtE,MAAM,KAAK,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,KAAK,EAAE,MAAM,KAAK,OAAO,EAAE,CAAC;QAC9B,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,kCAAkC,CAAC,CAAC;QAC9D,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;QAC9C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC9B,CAAC;IAED,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC;QAC5B,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,kCAAkC,CAAC,CAAC;QAC9D,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,kEAAkE,CAAC,CAAC;QAC7F,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;IAClC,CAAC;IAED,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,gEAAgE,CAAC,CAAC;IAC5F,IAAI,CAAC;QACH,MAAM,sBAAsB,CAAC,OAAO,CAAC,CAAC;QACtC,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;QAC5C,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC;IACrC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAClC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;IACtD,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,OAAe,EAAE,QAAwB;IAChE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAClB,QAAQ,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACpC,MAAM,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACtC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAClB,QAAQ,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IAClD,QAAQ,CAAC,IAAI,CAAC,mDAAmD,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,QAAkB,EAAE,EACpB,QAAyB;IAEzB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACrC,MAAM,YAAY,GAAG,QAAQ,IAAI,YAAY,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IAEzD,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAE3B,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,gBAAgB,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACjD,CAAC;IAED,YAAY,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAE5C,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAC;IACvD,MAAM,aAAa,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IAE1E,IAAI,UAAU,GAAgC,SAAS,CAAC;IACxD,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACzD,UAAU,GAAG,MAAM,kBAAkB,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACtE,CAAC;IAED,IAAI,cAAc,GAAqD,EAAE,CAAC;IAC1E,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACtD,cAAc,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACxE,CAAC;IAED,IAAI,aAAa,GAAmD,EAAE,CAAC;IACvE,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACrD,aAAa,GAAG,MAAM,cAAc,CAAC,YAAY,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,YAAY,GAAiD,EAAE,CAAC;IACpE,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACpD,YAAY,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,CAAC;IACnD,CAAC;IAED,IAAI,gBAAgB,GAAyD,EAAE,CAAC;IAChF,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACxD,gBAAgB,GAAG,MAAM,2BAA2B,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACrF,CAAC;IAED,IAAI,gBAAgB,GAAyD,EAAE,CAAC;IAChF,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACxD,gBAAgB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,MAAM,GAAiB;QAC3B,GAAG,QAAQ;QACX,GAAG,aAAa;QAChB,WAAW,EAAE,UAAU;QACvB,GAAG,cAAc;QACjB,GAAG,aAAa;QAChB,GAAG,YAAY;QACf,GAAG,gBAAgB;QACnB,GAAG,gBAAgB;KACpB,CAAC;IAEF,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtB,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC9B,YAAY,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC1H,YAAY,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACjH,IAAI,UAAU,KAAK,SAAS;QAAE,YAAY,CAAC,IAAI,CAAC,wBAAwB,UAAU,EAAE,CAAC,CAAC;IACtF,IAAI,MAAM,CAAC,QAAQ;QAAE,YAAY,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACzI,IAAI,MAAM,CAAC,OAAO;QAAE,YAAY,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACrI,IAAI,MAAM,CAAC,MAAM;QAAE,YAAY,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACjI,IAAI,MAAM,CAAC,UAAU;QAAE,YAAY,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACjJ,IAAI,MAAM,CAAC,UAAU;QAAE,YAAY,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEjJ,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACpD,MAAM,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IAChD,CAAC;IAED,YAAY,CAAC,MAAM,EAAE,CAAC;IACtB,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"warmup.js","sourceRoot":"","sources":["../../src/cli/warmup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpG,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC1G,OAAO,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,gCAAgC,CAAC;AACrE,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAC;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AAC9D,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAElD,OAAO,EAAE,YAAY,EAAE,MAAM,wBAAwB,CAAC;AACtD,OAAO,EAAE,SAAS,IAAI,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAoB5D,SAAS,gBAAgB,CAAC,OAAe,EAAE,QAAwB;IACjE,MAAM,iBAAiB,GAAG,IAAI,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;IAC1D,IAAI,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAqB,CAAC;YACtF,IAAI,IAAI,CAAC,GAAG,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzC,MAAM,IAAI,KAAK,CACb,gEAAgE,IAAI,CAAC,GAAG,KAAK;oBAC7E,uBAAuB,IAAI,CAAC,GAAG,EAAE,CAClC,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,KAAK,IAAI,GAAG,CAAC,OAAO,CAAC,UAAU,CAAC,gBAAgB,CAAC;gBAAE,MAAM,GAAG,CAAC;QAClF,CAAC;IACH,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACrD,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACnE,MAAM,CAAC,iBAAiB,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACvD,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACvD,QAAQ,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;AAC3E,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,QAAwB;IACvD,QAAQ,CAAC,KAAK,CAAC,YAAY,EAAE,sCAAsC,CAAC,CAAC;IACrE,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC,YAAY,EAAE,SAAS,EAAE,UAAU,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9F,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACjB,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;QAC5C,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IAC9B,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClE,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IACrC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC;AAC5D,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,OAAe,EAAE,QAAwB;IACzE,QAAQ,CAAC,KAAK,CAAC,aAAa,EAAE,4CAA4C,CAAC,CAAC;IAC5E,MAAM,EAAE,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACjC,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,aAAa,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IACxG,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACjB,QAAQ,CAAC,OAAO,CAAC,aAAa,EAAE,WAAW,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClE,QAAQ,CAAC,IAAI,CAAC,aAAa,EAAE,OAAO,CAAC,CAAC;IACtC,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,OAAe,EACf,QAAwB;IAExB,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,EAAE,CAAC,aAAa,CAAC,CAAC;IAC1D,QAAQ,CAAC,KAAK,CAAC,UAAU,EAAE,2BAA2B,OAAO,GAAG,CAAC,CAAC;IAClE,IAAI,CAAC;QACH,MAAM,mBAAmB,CAAC,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;QACtD,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QACnE,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;QAC1C,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC5B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,QAAQ,CAAC,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACnC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;IACxD,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,QAAwB;IACpD,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,qCAAqC,CAAC,CAAC;IACjE,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC,YAAY,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAC7F,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACjB,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;QACzC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClE,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAClC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;AACtD,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,QAAwB;IACnD,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,oCAAoC,CAAC,CAAC;IAC/D,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC,YAAY,EAAE,SAAS,EAAE,QAAQ,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAC5F,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACjB,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;QACxC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;IAC1B,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClE,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACjC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC;AACpD,CAAC;AAED,KAAK,UAAU,2BAA2B,CAAC,OAAe,EAAE,QAAwB;IAClF,QAAQ,CAAC,KAAK,CAAC,YAAY,EAAE,wDAAwD,CAAC,CAAC;IACvF,MAAM,EAAE,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC;IACjC,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,uBAAuB,CAAC,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAClH,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QACjB,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;QAC5C,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IAC9B,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClE,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;IACrC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC;AAC5D,CAAC;AAED,SAAS,gBAAgB;IACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;IAClC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;IAC1B,MAAM,IAAI,GAAG,oEAAoE,CAAC;IAClF,IAAI,QAAQ,KAAK,QAAQ,IAAI,IAAI,KAAK,OAAO;QAAE,OAAO,GAAG,IAAI,2BAA2B,CAAC;IACzF,IAAI,QAAQ,KAAK,OAAO,IAAI,IAAI,KAAK,KAAK;QAAE,OAAO,GAAG,IAAI,0BAA0B,CAAC;IACrF,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,IAAI,IAAI,EAAE,CAAC,CAAC;AACtE,CAAC;AAED,KAAK,UAAU,iBAAiB,CAAC,QAAwB;IACvD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QAC3C,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACxB,QAAQ,CAAC,KAAK,CAAC,YAAY,EAAE,uBAAuB,CAAC,CAAC;YACtD,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,mBAAmB,CAAC,CAAC;YACpD,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QAC9B,CAAC;QACD,MAAM,GAAG,GAAG,gBAAgB,EAAE,CAAC;QAC/B,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEvC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;QAClD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;QAEnE,QAAQ,CAAC,KAAK,CAAC,YAAY,EAAE,wBAAwB,EAAE,EAAE,UAAU,EAAE,UAAU,IAAI,SAAS,EAAE,CAAC,CAAC;QAEhG,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACzC,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;QACrC,MAAM,EAAE,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACtC,IAAI,CAAC;YACH,OAAO,IAAI,EAAE,CAAC;gBACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC5C,IAAI,IAAI;oBAAE,MAAM;gBAChB,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC;oBAAE,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;gBACtF,UAAU,IAAI,KAAK,CAAC,UAAU,CAAC;gBAC/B,IAAI,UAAU,GAAG,CAAC;oBAAE,QAAQ,CAAC,QAAQ,CAAC,YAAY,EAAE,UAAU,GAAG,UAAU,CAAC,CAAC;YAC/E,CAAC;QACH,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,GAAG,EAAE,CAAC;YACT,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QAC3E,CAAC;QAED,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAC1B,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;QAC5C,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IAC9B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACrC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC;IAC5D,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,OAAe,EAAE,QAAwB;IACtE,MAAM,KAAK,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,KAAK,EAAE,MAAM,KAAK,OAAO,EAAE,CAAC;QAC9B,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,kCAAkC,CAAC,CAAC;QAC9D,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;QAC9C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC9B,CAAC;IAED,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC;QAC5B,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,kCAAkC,CAAC,CAAC;QAC9D,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,kEAAkE,CAAC,CAAC;QAC7F,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;IAClC,CAAC;IAED,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,gEAAgE,CAAC,CAAC;IAC5F,IAAI,CAAC;QACH,MAAM,sBAAsB,CAAC,OAAO,CAAC,CAAC;QACtC,QAAQ,CAAC,OAAO,CAAC,SAAS,EAAE,cAAc,CAAC,CAAC;QAC5C,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC;IACrC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,QAAQ,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QAClC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;IACtD,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,OAAe,EAAE,QAAwB;IAChE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAClB,QAAQ,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACpC,MAAM,YAAY,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACtC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAClB,QAAQ,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IAClD,QAAQ,CAAC,IAAI,CAAC,mDAAmD,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,QAAkB,EAAE,EACpB,QAAyB;IAEzB,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;IACrC,MAAM,YAAY,GAAG,QAAQ,IAAI,YAAY,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IAEzD,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAE3B,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,gBAAgB,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACjD,CAAC;IAED,YAAY,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IAE5C,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAC;IACvD,MAAM,aAAa,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IAE1E,IAAI,UAAU,GAAgC,SAAS,CAAC;IACxD,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACzD,UAAU,GAAG,MAAM,kBAAkB,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACtE,CAAC;IAED,IAAI,cAAc,GAAqD,EAAE,CAAC;IAC1E,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACtD,cAAc,GAAG,MAAM,oBAAoB,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IAC5E,CAAC;IAED,IAAI,aAAa,GAAmD,EAAE,CAAC;IACvE,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACrD,aAAa,GAAG,MAAM,cAAc,CAAC,YAAY,CAAC,CAAC;IACrD,CAAC;IAED,IAAI,YAAY,GAAiD,EAAE,CAAC;IACpE,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACpD,YAAY,GAAG,MAAM,aAAa,CAAC,YAAY,CAAC,CAAC;IACnD,CAAC;IAED,IAAI,gBAAgB,GAAyD,EAAE,CAAC;IAChF,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACxD,gBAAgB,GAAG,MAAM,2BAA2B,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IACrF,CAAC;IAED,IAAI,gBAAgB,GAAyD,EAAE,CAAC;IAChF,IAAI,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACxD,gBAAgB,GAAG,MAAM,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAC3D,CAAC;IAED,MAAM,MAAM,GAAiB;QAC3B,GAAG,QAAQ;QACX,GAAG,aAAa;QAChB,WAAW,EAAE,UAAU;QACvB,GAAG,cAAc;QACjB,GAAG,aAAa;QAChB,GAAG,YAAY;QACf,GAAG,gBAAgB;QACnB,GAAG,gBAAgB;KACpB,CAAC;IAEF,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACtB,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC9B,YAAY,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC1H,YAAY,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACjH,IAAI,UAAU,KAAK,SAAS;QAAE,YAAY,CAAC,IAAI,CAAC,wBAAwB,UAAU,EAAE,CAAC,CAAC;IACtF,IAAI,MAAM,CAAC,QAAQ;QAAE,YAAY,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACzI,IAAI,MAAM,CAAC,OAAO;QAAE,YAAY,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACrI,IAAI,MAAM,CAAC,MAAM;QAAE,YAAY,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACjI,IAAI,MAAM,CAAC,UAAU;QAAE,YAAY,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACjJ,IAAI,MAAM,CAAC,UAAU;QAAE,YAAY,CAAC,IAAI,CAAC,oBAAoB,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEjJ,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACpD,MAAM,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IAChD,CAAC;IAED,YAAY,CAAC,MAAM,EAAE,CAAC;IACtB,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/config.d.ts

@@ -21,6 +21,8 @@ export interface Config {
     dataDir: string;
     cacheTtlSearch: number;
     cacheTtlContent: number;
+    fastStaleMaxHours: number;
+    fastTimeoutMs: number;
     crawlConcurrency: number;
     crawlDelayMs: number;
     crawlPrivateConcurrency: number;
@@ -34,7 +36,7 @@ export interface Config {
     logLevel: 'debug' | 'info' | 'warn' | 'error';
     logFormat: 'json' | 'text';
     trafilatura: 'auto' | 'always' | 'never';
-    reranker: 'flashrank' | 'none' | 'custom';
+    reranker: 'onnx' | 'none' | 'custom';
     rerankerModel: string;
     relevanceThreshold: number;
     bootstrapMaxAttempts: number;
@@ -53,6 +55,9 @@ export interface Config {
     lightpandaUrl: string | null;
     lightpandaEnabled: boolean;
     lightpandaFailureThreshold: number;
+    llmProvider: string | null;
+    llmCacheTtlDays: number;
+    llmMaxCallsPerRequest: number;
 }
 export declare function getConfig(): Config;
 export declare function resetConfig(): void;

package/dist/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,WAAW,MAAM;IACrB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,EAAE,QAAQ,GAAG,QAAQ,CAAC;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,uBAAuB,EAAE,MAAM,CAAC;IAChC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,wBAAwB,EAAE,MAAM,CAAC;IACjC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,uBAAuB,EAAE,MAAM,CAAC;IAChC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,QAAQ,EAAE,OAAO,CAAC;IAClB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,QAAQ,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAC9C,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;IACzC,QAAQ,EAAE,WAAW,GAAG,MAAM,GAAG,QAAQ,CAAC;IAC1C,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,EAAE,CAAC;IAClC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,iBAAiB,EAAE,OAAO,CAAC;IAC3B,0BAA0B,EAAE,MAAM,CAAC;CACpC;AA6BD,wBAAgB,SAAS,IAAI,MAAM,CAmElC;AAED,wBAAgB,WAAW,IAAI,IAAI,CAElC"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,MAAM,WAAW,MAAM;IACrB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,EAAE,QAAQ,GAAG,QAAQ,CAAC;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,uBAAuB,EAAE,MAAM,CAAC;IAChC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,wBAAwB,EAAE,MAAM,CAAC;IACjC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,uBAAuB,EAAE,MAAM,CAAC;IAChC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,QAAQ,EAAE,OAAO,CAAC;IAClB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,QAAQ,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAC9C,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;IACzC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,CAAC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,uBAAuB,EAAE,MAAM,EAAE,CAAC;IAClC,qBAAqB,EAAE,MAAM,CAAC;IAC9B,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,WAAW,EAAE,CAAC;IAC5B,gBAAgB,EAAE,MAAM,CAAC;IACzB,qBAAqB,EAAE,MAAM,CAAC;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,sBAAsB,EAAE,MAAM,CAAC;IAC/B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,iBAAiB,EAAE,OAAO,CAAC;IAC3B,0BAA0B,EAAE,MAAM,CAAC;IACnC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,eAAe,EAAE,MAAM,CAAC;IACxB,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AA6BD,wBAAgB,SAAS,IAAI,MAAM,CAkFlC;AAED,wBAAgB,WAAW,IAAI,IAAI,CAElC"}

package/dist/config.js

@@ -52,6 +52,8 @@ export function getConfig() {
         dataDir: envStr('WIGOLO_DATA_DIR') ?? join(homedir(), '.wigolo'),
         cacheTtlSearch: envInt('CACHE_TTL_SEARCH', 86400),
         cacheTtlContent: envInt('CACHE_TTL_CONTENT', 604800),
+        fastStaleMaxHours: envInt('WIGOLO_FAST_STALE_MAX_HOURS', 24),
+        fastTimeoutMs: envInt('WIGOLO_FAST_TIMEOUT_MS', 800),
         crawlConcurrency: envInt('CRAWL_CONCURRENCY', 2),
         crawlDelayMs: envInt('CRAWL_DELAY_MS', 500),
         crawlPrivateConcurrency: envInt('CRAWL_PRIVATE_CONCURRENCY', 10),
@@ -65,8 +67,16 @@ export function getConfig() {
         logLevel: envStr('LOG_LEVEL', 'info'),
         logFormat: envStr('LOG_FORMAT', 'json'),
         trafilatura: envStr('WIGOLO_TRAFILATURA', 'auto'),
-        reranker: (envStr('WIGOLO_RERANKER') ?? 'flashrank'),
-        rerankerModel: envStr('WIGOLO_RERANKER_MODEL') ?? 'ms-marco-MiniLM-L-12-v2',
+        reranker: (() => {
+            const raw = envStr('WIGOLO_RERANKER') ?? 'onnx';
+            if (raw === 'flashrank') {
+                throw new Error("WIGOLO_RERANKER='flashrank' is retired. Set WIGOLO_RERANKER=onnx (default) — " +
+                    'the in-process ONNX reranker replaces the flashrank Python subprocess. ' +
+                    'See CHANGELOG.md migration notes.');
+            }
+            return raw;
+        })(),
+        rerankerModel: envStr('WIGOLO_RERANKER_MODEL') ?? 'bge-reranker-v2-m3',
         relevanceThreshold: parseFloat(envStr('WIGOLO_RELEVANCE_THRESHOLD') ?? '0') || 0,
         bootstrapMaxAttempts: envInt('WIGOLO_BOOTSTRAP_MAX_ATTEMPTS', 3),
         bootstrapBackoffSeconds: envIntArray('WIGOLO_BOOTSTRAP_BACKOFF_SECONDS', [30, 3600, 86400]),
@@ -92,6 +102,9 @@ export function getConfig() {
         lightpandaUrl: envStr('WIGOLO_LIGHTPANDA_URL'),
         lightpandaEnabled: envBool('WIGOLO_LIGHTPANDA_ENABLED', false),
         lightpandaFailureThreshold: envInt('WIGOLO_LIGHTPANDA_FAILURE_THRESHOLD', 3),
+        llmProvider: envStr('WIGOLO_LLM_PROVIDER'),
+        llmCacheTtlDays: envInt('WIGOLO_LLM_CACHE_TTL_DAYS', 7),
+        llmMaxCallsPerRequest: envInt('WIGOLO_LLM_MAX_CALLS_PER_REQUEST', 1),
     };
     return cachedConfig;
 }

package/dist/config.js.map

@@ -1 +1 @@
-{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AA2D7D,SAAS,MAAM,CAAC,GAAW,EAAE,WAA0B,IAAI;IACzD,OAAO,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC;AACtC,CAAC;AAED,SAAS,MAAM,CAAC,GAAW,EAAE,QAAgB;IAC3C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACvC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACjC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC;AAC3C,CAAC;AAED,SAAS,WAAW,CAAC,GAAW,EAAE,QAAkB;IAClD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACvC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;IAC9D,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC/C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,OAAO,CAAC,GAAW,EAAE,QAAiB;IAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACvC,OAAO,GAAG,CAAC,WAAW,EAAE,KAAK,OAAO,IAAI,GAAG,KAAK,GAAG,CAAC;AACtD,CAAC;AAED,IAAI,YAAY,GAAkB,IAAI,CAAC;AAEvC,MAAM,UAAU,SAAS;IACvB,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC;IAEtC,YAAY,GAAG;QACb,UAAU,EAAE,MAAM,CAAC,aAAa,CAAC;QACjC,WAAW,EAAG,MAAM,CAAC,cAAc,EAAE,QAAQ,CAAyB;QACtE,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,IAAI,CAAC;QACzC,cAAc,EAAE,MAAM,CAAC,kBAAkB,EAAE,KAAK,CAAC;QACjD,eAAe,EAAE,MAAM,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAC/C,YAAY,EAAE,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;QACxC,uBAAuB,EAAE,MAAM,CAAC,4BAA4B,EAAE,KAAK,CAAC;QACpE,sBAAsB,EAAE,MAAM,CAAC,2BAA2B,EAAE,KAAK,CAAC;QAClE,qBAAqB,EAAE,MAAM,CAAC,0BAA0B,EAAE,IAAI,CAAC;QAC/D,oBAAoB,EAAE,MAAM,CAAC,yBAAyB,EAAE,KAAK,CAAC;QAC9D,oBAAoB,EAAE,MAAM,CAAC,yBAAyB,EAAE,KAAK,CAAC;QAC9D,iBAAiB,EAAE,MAAM,CAAC,qBAAqB,EAAE,IAAI,CAAC;QACtD,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;QACtC,oBAAoB,EAAE,MAAM,CAAC,sBAAsB,EAAE,KAAK,CAAC;QAC3D,wBAAwB,EAAE,MAAM,CAAC,4BAA4B,EAAE,CAAC,CAAC;QACjE,aAAa,EAAE,MAAM,CAAC,wBAAwB,CAAC;QAC/C,iBAAiB,EAAE,MAAM,CAAC,4BAA4B,CAAC;QACvD,MAAM,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,IAAI;QACxC,OAAO,EAAE,MAAM,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,CAAC;QAChE,cAAc,EAAE,MAAM,CAAC,kBAAkB,EAAE,KAAK,CAAC;QACjD,eAAe,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC;QACpD,gBAAgB,EAAE,MAAM,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAChD,YAAY,EAAE,MAAM,CAAC,gBAAgB,EAAE,GAAG,CAAC;QAC3C,uBAAuB,EAAE,MAAM,CAAC,2BAA2B,EAAE,EAAE,CAAC;QAChE,mBAAmB,EAAE,MAAM,CAAC,wBAAwB,EAAE,CAAC,CAAC;QACxD,QAAQ,EAAE,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC;QACrC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC;QAC7B,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC;QAC/B,aAAa,EAAE,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;QAC9C,gBAAgB,EAAE,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC;QACrD,WAAW,EAAE,MAAM,CAAC,eAAe,CAAC;QACpC,QAAQ,EAAG,MAAM,CAAC,WAAW,EAAE,MAAM,CAAwB;QAC7D,SAAS,EAAG,MAAM,CAAC,YAAY,EAAE,MAAM,CAAyB;QAChE,WAAW,EAAG,MAAM,CAAC,oBAAoB,EAAE,MAAM,CAAiC;QAClF,QAAQ,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI,WAAW,CAAuB;QAC1E,aAAa,EAAE,MAAM,CAAC,uBAAuB,CAAC,IAAI,yBAAyB;QAC3E,kBAAkB,EAAE,UAAU,CAAC,MAAM,CAAC,4BAA4B,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC;QAChF,oBAAoB,EAAE,MAAM,CAAC,+BAA+B,EAAE,CAAC,CAAC;QAChE,uBAAuB,EAAE,WAAW,CAAC,kCAAkC,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QAC3F,qBAAqB,EAAE,MAAM,CAAC,iCAAiC,EAAE,KAAK,CAAC;QACvE,UAAU,EAAE,MAAM,CAAC,oBAAoB,EAAE,IAAI,CAAC;QAC9C,UAAU,EAAE,MAAM,CAAC,oBAAoB,EAAE,WAAW,CAAC,EAAE,IAAI,EAAE,IAAI,WAAW;QAC5E,UAAU,EAAE,CAAC,GAAG,EAAE;YAChB,MAAM,GAAG,GAAG,MAAM,CAAC,oBAAoB,CAAC,CAAC;YACzC,IAAI,GAAG,EAAE,CAAC;gBACR,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC;oBAAE,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC9D,OAAO,GAAG,CAAC;YACb,CAAC;YACD,OAAO,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,CAAC,EAAE,SAAS,CAAC,CAAC;QAClF,CAAC,CAAC,EAAE;QACJ,YAAY,EAAE,iBAAiB,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QAC/D,gBAAgB,EAAE,MAAM,CAAC,2BAA2B,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,eAAe,CAAC;QACpG,qBAAqB,EAAE,MAAM,CAAC,gCAAgC,EAAE,CAAC,CAAC;QAClE,aAAa,EAAE,MAAM,CAAC,wBAAwB,EAAE,EAAE,CAAC;QACnD,cAAc,EAAE,MAAM,CAAC,wBAAwB,CAAC,IAAI,wBAAwB;QAC5E,sBAAsB,EAAE,MAAM,CAAC,+BAA+B,EAAE,MAAM,CAAC;QACvE,sBAAsB,EAAE,MAAM,CAAC,kCAAkC,EAAE,IAAI,CAAC;QACxE,aAAa,EAAE,MAAM,CAAC,uBAAuB,CAAC;QAC9C,iBAAiB,EAAE,OAAO,CAAC,2BAA2B,EAAE,KAAK,CAAC;QAC9D,0BAA0B,EAAE,MAAM,CAAC,qCAAqC,EAAE,CAAC,CAAC;KAC7E,CAAC;IAEF,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,YAAY,GAAG,IAAI,CAAC;AACtB,CAAC"}
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAgE7D,SAAS,MAAM,CAAC,GAAW,EAAE,WAA0B,IAAI;IACzD,OAAO,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC;AACtC,CAAC;AAED,SAAS,MAAM,CAAC,GAAW,EAAE,QAAgB;IAC3C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACvC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACjC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC;AAC3C,CAAC;AAED,SAAS,WAAW,CAAC,GAAW,EAAE,QAAkB;IAClD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACvC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;IAC9D,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAAE,OAAO,QAAQ,CAAC;IAC/C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,OAAO,CAAC,GAAW,EAAE,QAAiB;IAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACvC,OAAO,GAAG,CAAC,WAAW,EAAE,KAAK,OAAO,IAAI,GAAG,KAAK,GAAG,CAAC;AACtD,CAAC;AAED,IAAI,YAAY,GAAkB,IAAI,CAAC;AAEvC,MAAM,UAAU,SAAS;IACvB,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC;IAEtC,YAAY,GAAG;QACb,UAAU,EAAE,MAAM,CAAC,aAAa,CAAC;QACjC,WAAW,EAAG,MAAM,CAAC,cAAc,EAAE,QAAQ,CAAyB;QACtE,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,IAAI,CAAC;QACzC,cAAc,EAAE,MAAM,CAAC,kBAAkB,EAAE,KAAK,CAAC;QACjD,eAAe,EAAE,MAAM,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAC/C,YAAY,EAAE,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;QACxC,uBAAuB,EAAE,MAAM,CAAC,4BAA4B,EAAE,KAAK,CAAC;QACpE,sBAAsB,EAAE,MAAM,CAAC,2BAA2B,EAAE,KAAK,CAAC;QAClE,qBAAqB,EAAE,MAAM,CAAC,0BAA0B,EAAE,IAAI,CAAC;QAC/D,oBAAoB,EAAE,MAAM,CAAC,yBAAyB,EAAE,KAAK,CAAC;QAC9D,oBAAoB,EAAE,MAAM,CAAC,yBAAyB,EAAE,KAAK,CAAC;QAC9D,iBAAiB,EAAE,MAAM,CAAC,qBAAqB,EAAE,IAAI,CAAC;QACtD,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;QACtC,oBAAoB,EAAE,MAAM,CAAC,sBAAsB,EAAE,KAAK,CAAC;QAC3D,wBAAwB,EAAE,MAAM,CAAC,4BAA4B,EAAE,CAAC,CAAC;QACjE,aAAa,EAAE,MAAM,CAAC,wBAAwB,CAAC;QAC/C,iBAAiB,EAAE,MAAM,CAAC,4BAA4B,CAAC;QACvD,MAAM,EAAE,MAAM,CAAC,gBAAgB,CAAC,IAAI,IAAI;QACxC,OAAO,EAAE,MAAM,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,CAAC;QAChE,cAAc,EAAE,MAAM,CAAC,kBAAkB,EAAE,KAAK,CAAC;QACjD,eAAe,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC;QACpD,iBAAiB,EAAE,MAAM,CAAC,6BAA6B,EAAE,EAAE,CAAC;QAC5D,aAAa,EAAE,MAAM,CAAC,wBAAwB,EAAE,GAAG,CAAC;QACpD,gBAAgB,EAAE,MAAM,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAChD,YAAY,EAAE,MAAM,CAAC,gBAAgB,EAAE,GAAG,CAAC;QAC3C,uBAAuB,EAAE,MAAM,CAAC,2BAA2B,EAAE,EAAE,CAAC;QAChE,mBAAmB,EAAE,MAAM,CAAC,wBAAwB,EAAE,CAAC,CAAC;QACxD,QAAQ,EAAE,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC;QACrC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC;QAC7B,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC;QAC/B,aAAa,EAAE,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;QAC9C,gBAAgB,EAAE,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC;QACrD,WAAW,EAAE,MAAM,CAAC,eAAe,CAAC;QACpC,QAAQ,EAAG,MAAM,CAAC,WAAW,EAAE,MAAM,CAAwB;QAC7D,SAAS,EAAG,MAAM,CAAC,YAAY,EAAE,MAAM,CAAyB;QAChE,WAAW,EAAG,MAAM,CAAC,oBAAoB,EAAE,MAAM,CAAiC;QAClF,QAAQ,EAAE,CAAC,GAAG,EAAE;YACd,MAAM,GAAG,GAAG,MAAM,CAAC,iBAAiB,CAAC,IAAI,MAAM,CAAC;YAChD,IAAI,GAAG,KAAK,WAAW,EAAE,CAAC;gBACxB,MAAM,IAAI,KAAK,CACb,+EAA+E;oBAC/E,yEAAyE;oBACzE,mCAAmC,CACpC,CAAC;YACJ,CAAC;YACD,OAAO,GAAyB,CAAC;QACnC,CAAC,CAAC,EAAE;QACJ,aAAa,EAAE,MAAM,CAAC,uBAAuB,CAAC,IAAI,oBAAoB;QACtE,kBAAkB,EAAE,UAAU,CAAC,MAAM,CAAC,4BAA4B,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC;QAChF,oBAAoB,EAAE,MAAM,CAAC,+BAA+B,EAAE,CAAC,CAAC;QAChE,uBAAuB,EAAE,WAAW,CAAC,kCAAkC,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QAC3F,qBAAqB,EAAE,MAAM,CAAC,iCAAiC,EAAE,KAAK,CAAC;QACvE,UAAU,EAAE,MAAM,CAAC,oBAAoB,EAAE,IAAI,CAAC;QAC9C,UAAU,EAAE,MAAM,CAAC,oBAAoB,EAAE,WAAW,CAAC,EAAE,IAAI,EAAE,IAAI,WAAW;QAC5E,UAAU,EAAE,CAAC,GAAG,EAAE;YAChB,MAAM,GAAG,GAAG,MAAM,CAAC,oBAAoB,CAAC,CAAC;YACzC,IAAI,GAAG,EAAE,CAAC;gBACR,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC;oBAAE,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC9D,OAAO,GAAG,CAAC;YACb,CAAC;YACD,OAAO,IAAI,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,CAAC,EAAE,SAAS,CAAC,CAAC;QAClF,CAAC,CAAC,EAAE;QACJ,YAAY,EAAE,iBAAiB,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QAC/D,gBAAgB,EAAE,MAAM,CAAC,2BAA2B,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,eAAe,CAAC;QACpG,qBAAqB,EAAE,MAAM,CAAC,gCAAgC,EAAE,CAAC,CAAC;QAClE,aAAa,EAAE,MAAM,CAAC,wBAAwB,EAAE,EAAE,CAAC;QACnD,cAAc,EAAE,MAAM,CAAC,wBAAwB,CAAC,IAAI,wBAAwB;QAC5E,sBAAsB,EAAE,MAAM,CAAC,+BAA+B,EAAE,MAAM,CAAC;QACvE,sBAAsB,EAAE,MAAM,CAAC,kCAAkC,EAAE,IAAI,CAAC;QACxE,aAAa,EAAE,MAAM,CAAC,uBAAuB,CAAC;QAC9C,iBAAiB,EAAE,OAAO,CAAC,2BAA2B,EAAE,KAAK,CAAC;QAC9D,0BAA0B,EAAE,MAAM,CAAC,qCAAqC,EAAE,CAAC,CAAC;QAC5E,WAAW,EAAE,MAAM,CAAC,qBAAqB,CAAC;QAC1C,eAAe,EAAE,MAAM,CAAC,2BAA2B,EAAE,CAAC,CAAC;QACvD,qBAAqB,EAAE,MAAM,CAAC,kCAAkC,EAAE,CAAC,CAAC;KACrE,CAAC;IAEF,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,YAAY,GAAG,IAAI,CAAC;AACtB,CAAC"}

package/dist/crawl/dedup.d.ts

@@ -8,6 +8,7 @@ interface PageOutput {
     url: string;
     markdown: string;
 }
+export declare function stripRepeatedNavigationLines(pages: PageInput[]): PageInput[];
 export declare function deduplicatePages(pages: PageInput[], domain?: string): PageOutput[];
 export declare function getStoredBoilerplate(domain: string): string[];
 export declare function storeBoilerplate(domain: string, hashes: string[]): void;

package/dist/crawl/dedup.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"dedup.d.ts","sourceRoot":"","sources":["../../src/crawl/dedup.ts"],"names":[],"mappings":"AAGA,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CA2B1D;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEvD;AAMD,UAAU,SAAS;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,UAAU;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,UAAU,EAAE,CA+ClF;AAED,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAI7D;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAavE"}
+{"version":3,"file":"dedup.d.ts","sourceRoot":"","sources":["../../src/crawl/dedup.ts"],"names":[],"mappings":"AAGA,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE,CA2B1D;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEvD;AAMD,UAAU,SAAS;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,UAAU,UAAU;IAClB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;CAClB;AAWD,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,SAAS,EAAE,GAAG,SAAS,EAAE,CAqC5E;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM,GAAG,UAAU,EAAE,CAiDlF;AAED,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,CAI7D;AAED,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAavE"}

package/dist/crawl/dedup.js

@@ -30,14 +30,60 @@ export function normalizeBlockText(text) {
 function hashBlock(text) {
     return createHash('sha256').update(normalizeBlockText(text)).digest('hex');
 }
+const NAV_DEDUPE_THRESHOLD = 0.6;
+const MAX_LEADING_LINES = 30;
+const MAX_TRAILING_LINES = 20;
+const MIN_CORPUS = 4;
+function lineHash(line) {
+    return createHash('sha1').update(line.trim().toLowerCase()).digest('hex');
+}
+export function stripRepeatedNavigationLines(pages) {
+    if (pages.length < MIN_CORPUS)
+        return pages;
+    const lineSets = pages.map((p) => p.markdown.split('\n'));
+    const countLeading = new Map();
+    const countTrailing = new Map();
+    for (const lines of lineSets) {
+        const seenL = new Set();
+        for (let i = 0; i < Math.min(MAX_LEADING_LINES, lines.length); i++) {
+            const h = lineHash(lines[i]);
+            if (!seenL.has(h)) {
+                seenL.add(h);
+                countLeading.set(h, (countLeading.get(h) ?? 0) + 1);
+            }
+        }
+        const seenT = new Set();
+        for (let i = lines.length - 1; i >= Math.max(lines.length - MAX_TRAILING_LINES, 0); i--) {
+            const h = lineHash(lines[i]);
+            if (!seenT.has(h)) {
+                seenT.add(h);
+                countTrailing.set(h, (countTrailing.get(h) ?? 0) + 1);
+            }
+        }
+    }
+    const threshold = pages.length * NAV_DEDUPE_THRESHOLD;
+    const navLeading = new Set([...countLeading].filter(([, c]) => c >= threshold).map(([h]) => h));
+    const navTrailing = new Set([...countTrailing].filter(([, c]) => c >= threshold).map(([h]) => h));
+    return pages.map((page, i) => {
+        const lines = lineSets[i];
+        let head = 0;
+        while (head < lines.length && (lines[head].trim() === '' || navLeading.has(lineHash(lines[head]))))
+            head++;
+        let tail = lines.length;
+        while (tail > head && (lines[tail - 1].trim() === '' || navTrailing.has(lineHash(lines[tail - 1]))))
+            tail--;
+        return { url: page.url, markdown: lines.slice(head, tail).join('\n') };
+    });
+}
 export function deduplicatePages(pages, domain) {
     if (pages.length <= 1)
         return pages.map((p) => ({ url: p.url, markdown: p.markdown }));
+    const stripped = stripRepeatedNavigationLines(pages);
     // Pre-load stored boilerplate hashes for this domain
     const storedHashes = domain ? getStoredBoilerplate(domain) : [];
     const boilerplateHashes = new Set(storedHashes);
     // Split each page into blocks and hash them
-    const pageBlocks = pages.map((page) => ({
+    const pageBlocks = stripped.map((page) => ({
         url: page.url,
         blocks: splitIntoBlocks(page.markdown),
     }));

package/dist/crawl/dedup.js.map

@@ -1 +1 @@
-{"version":3,"file":"dedup.js","sourceRoot":"","sources":["../../src/crawl/dedup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,MAAM,UAAU,eAAe,CAAC,QAAgB;IAC9C,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAEhC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,cAAc,GAAyC,EAAE,CAAC;IAEhE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC7C,IAAI,KAAK,EAAE,CAAC;YACV,cAAc,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACtE,CAAC;IAED,kGAAkG;IAClG,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACxC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;QACzF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACxD,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC7E,CAAC;AAYD,MAAM,UAAU,gBAAgB,CAAC,KAAkB,EAAE,MAAe;IAClE,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IAEvF,qDAAqD;IACrD,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAChE,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAS,YAAY,CAAC,CAAC;IAExD,4CAA4C;IAC5C,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACtC,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,MAAM,EAAE,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC;KACvC,CAAC,CAAC,CAAC;IAEJ,kDAAkD;IAClD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAChD,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;QACrC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;YAC3B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvB,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAClB,aAAa,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;IACH,CAAC;IAED,wDAAwD;IACxD,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;QAC1C,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACtB,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,mDAAmD;IACnD,IAAI,MAAM,EAAE,CAAC;QACX,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED,0CAA0C;IAC1C,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACzF,OAAO;YACL,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;SAChC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,MAAc;IACjD,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,4DAA4D,CAAC,CAAC,GAAG,CAAC,MAAM,CAA6B,CAAC;IAC9H,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,MAAgB;IAC/D,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,iDAAiD,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CACvB,6FAA6F,CAC9F,CAAC;IACF,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,KAAe,EAAE,EAAE;QAC5C,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QACjC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,EAAE,CAAC,MAAM,CAAC,CAAC;AACb,CAAC"}
+{"version":3,"file":"dedup.js","sourceRoot":"","sources":["../../src/crawl/dedup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C,MAAM,UAAU,eAAe,CAAC,QAAgB;IAC9C,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,CAAC;IAEhC,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,cAAc,GAAyC,EAAE,CAAC;IAEhE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC7C,IAAI,KAAK,EAAE,CAAC;YACV,cAAc,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC;IAED,6DAA6D;IAC7D,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACtE,CAAC;IAED,kGAAkG;IAClG,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,cAAc,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;QACxC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;QACzF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACxD,CAAC;AAED,SAAS,SAAS,CAAC,IAAY;IAC7B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC7E,CAAC;AAYD,MAAM,oBAAoB,GAAG,GAAG,CAAC;AACjC,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,UAAU,GAAG,CAAC,CAAC;AAErB,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5E,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,KAAkB;IAC7D,IAAI,KAAK,CAAC,MAAM,GAAG,UAAU;QAAE,OAAO,KAAK,CAAC;IAC5C,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;IAE1D,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC/C,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAChD,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACnE,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACb,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;QAChC,KAAK,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACxF,MAAM,CAAC,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACb,aAAa,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,oBAAoB,CAAC;IACtD,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAChG,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAElG,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,OAAO,IAAI,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAAE,IAAI,EAAE,CAAC;QAC3G,IAAI,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC;QACxB,OAAO,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAAE,IAAI,EAAE,CAAC;QAC5G,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;IACzE,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAkB,EAAE,MAAe;IAClE,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;IAEvF,MAAM,QAAQ,GAAG,4BAA4B,CAAC,KAAK,CAAC,CAAC;IAErD,qDAAqD;IACrD,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAChE,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAS,YAAY,CAAC,CAAC;IAExD,4CAA4C;IAC5C,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACzC,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,MAAM,EAAE,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC;KACvC,CAAC,CAAC,CAAC;IAEJ,kDAAkD;IAClD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAC;IAChD,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;QACrC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChC,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;YAC3B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvB,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBAClB,aAAa,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;IACH,CAAC;IAED,wDAAwD;IACxD,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IACnC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,aAAa,EAAE,CAAC;QAC1C,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACtB,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,mDAAmD;IACnD,IAAI,MAAM,EAAE,CAAC;QACX,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED,0CAA0C;IAC1C,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACzF,OAAO;YACL,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,QAAQ,EAAE,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC;SAChC,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,MAAc;IACjD,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,4DAA4D,CAAC,CAAC,GAAG,CAAC,MAAM,CAA6B,CAAC;IAC9H,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,MAAgB;IAC/D,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,iDAAiD,CAAC,CAAC;IAC1E,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CACvB,6FAA6F,CAC9F,CAAC;IACF,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,KAAe,EAAE,EAAE;QAC5C,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC;QACjC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,EAAE,CAAC,MAAM,CAAC,CAAC;AACb,CAAC"}

package/dist/extraction/boilerplate.d.ts

@@ -0,0 +1,15 @@
+export declare const BOILERPLATE_TEXT_EQUALITY: ReadonlyArray<string>;
+export declare const BOILERPLATE_TEXT_PATTERNS: ReadonlyArray<RegExp>;
+export declare const BOILERPLATE_SELECTORS: ReadonlyArray<string>;
+export interface BoilerplateDocument {
+    querySelectorAll(selector: string): ArrayLike<BoilerplateElement>;
+}
+interface BoilerplateElement {
+    parentNode: {
+        removeChild(child: BoilerplateElement): void;
+    } | null;
+}
+export declare function stripBoilerplateMarkdown(md: string): string;
+export declare function stripBoilerplateDom(document: BoilerplateDocument): void;
+export {};
+//# sourceMappingURL=boilerplate.d.ts.map

package/dist/extraction/boilerplate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"boilerplate.d.ts","sourceRoot":"","sources":["../../src/extraction/boilerplate.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,yBAAyB,EAAE,aAAa,CAAC,MAAM,CAQ3D,CAAC;AAEF,eAAO,MAAM,yBAAyB,EAAE,aAAa,CAAC,MAAM,CAE3D,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,aAAa,CAAC,MAAM,CAUvD,CAAC;AAEF,MAAM,WAAW,mBAAmB;IAClC,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC,kBAAkB,CAAC,CAAC;CACnE;AAED,UAAU,kBAAkB;IAC1B,UAAU,EAAE;QAAE,WAAW,CAAC,KAAK,EAAE,kBAAkB,GAAG,IAAI,CAAA;KAAE,GAAG,IAAI,CAAC;CACrE;AAED,wBAAgB,wBAAwB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAU3D;AAED,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,mBAAmB,GAAG,IAAI,CASvE"}

package/dist/extraction/boilerplate.js

@@ -0,0 +1,49 @@
+export const BOILERPLATE_TEXT_EQUALITY = [
+    'was this helpful?',
+    'send',
+    'edit this page',
+    'edit on github',
+    'suggest changes',
+    'skip to main content',
+    'on this page',
+];
+export const BOILERPLATE_TEXT_PATTERNS = [
+    /^\s*last updated on .+$/i,
+];
+export const BOILERPLATE_SELECTORS = [
+    '[class*="feedback"]',
+    '[class*="edit-page"]',
+    '[aria-label*="Edit"]',
+    'footer[class*="docs"]',
+    '[class*="sticky-cta"]',
+    'main [role="banner"]',
+    '[role="navigation"]',
+    '[class*="sidebar"]',
+    '[data-collection="docs"]',
+];
+export function stripBoilerplateMarkdown(md) {
+    if (!md)
+        return md;
+    const lines = md.split('\n');
+    const kept = lines.filter((line) => {
+        const t = line.trim().toLowerCase();
+        if (!t)
+            return true;
+        if (BOILERPLATE_TEXT_EQUALITY.includes(t))
+            return false;
+        return !BOILERPLATE_TEXT_PATTERNS.some((re) => re.test(line));
+    });
+    return kept.join('\n').replace(/\n{3,}/g, '\n\n');
+}
+export function stripBoilerplateDom(document) {
+    for (const sel of BOILERPLATE_SELECTORS) {
+        const nodes = document.querySelectorAll(sel);
+        const list = [];
+        for (let i = 0; i < nodes.length; i++)
+            list.push(nodes[i]);
+        for (const el of list) {
+            el.parentNode?.removeChild(el);
+        }
+    }
+}
+//# sourceMappingURL=boilerplate.js.map

package/dist/extraction/boilerplate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"boilerplate.js","sourceRoot":"","sources":["../../src/extraction/boilerplate.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,yBAAyB,GAA0B;IAC9D,mBAAmB;IACnB,MAAM;IACN,gBAAgB;IAChB,gBAAgB;IAChB,iBAAiB;IACjB,sBAAsB;IACtB,cAAc;CACf,CAAC;AAEF,MAAM,CAAC,MAAM,yBAAyB,GAA0B;IAC9D,0BAA0B;CAC3B,CAAC;AAEF,MAAM,CAAC,MAAM,qBAAqB,GAA0B;IAC1D,qBAAqB;IACrB,sBAAsB;IACtB,sBAAsB;IACtB,uBAAuB;IACvB,uBAAuB;IACvB,sBAAsB;IACtB,qBAAqB;IACrB,oBAAoB;IACpB,0BAA0B;CAC3B,CAAC;AAUF,MAAM,UAAU,wBAAwB,CAAC,EAAU;IACjD,IAAI,CAAC,EAAE;QAAE,OAAO,EAAE,CAAC;IACnB,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;QACjC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACpC,IAAI,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;QACpB,IAAI,yBAAyB,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,OAAO,KAAK,CAAC;QACxD,OAAO,CAAC,yBAAyB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;AACpD,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,QAA6B;IAC/D,KAAK,MAAM,GAAG,IAAI,qBAAqB,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAyB,EAAE,CAAC;QACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE;YAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,KAAK,MAAM,EAAE,IAAI,IAAI,EAAE,CAAC;YACtB,EAAE,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/extraction/defuddle.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"defuddle.d.ts","sourceRoot":"","sources":["../../src/extraction/defuddle.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIpD,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAoBjG"}
+{"version":3,"file":"defuddle.d.ts","sourceRoot":"","sources":["../../src/extraction/defuddle.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAKpD,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAsBjG"}

package/dist/extraction/defuddle.js

@@ -1,13 +1,17 @@
 import { Defuddle } from 'defuddle/node';
+import { htmlToMarkdown } from './markdown.js';
 const MIN_CONTENT_THRESHOLD = 100;
 export async function defuddleExtract(html, url) {
     try {
-        const result = await Defuddle(html, url, { markdown: true });
-        if (!result.content || result.content.length < MIN_CONTENT_THRESHOLD)
+        const result = await Defuddle(html, url);
+        if (!result.content)
+            return null;
+        const markdown = htmlToMarkdown(result.content);
+        if (markdown.length < MIN_CONTENT_THRESHOLD)
             return null;
         return {
             title: result.title ?? '',
-            markdown: result.content,
+            markdown,
             metadata: {
                 description: result.description || undefined,
                 author: result.author || undefined,

package/dist/extraction/defuddle.js.map

@@ -1 +1 @@
-{"version":3,"file":"defuddle.js","sourceRoot":"","sources":["../../src/extraction/defuddle.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAGzC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAY,EAAE,GAAW;IAC7D,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,qBAAqB;YAAE,OAAO,IAAI,CAAC;QAClF,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;YACzB,QAAQ,EAAE,MAAM,CAAC,OAAO;YACxB,QAAQ,EAAE;gBACR,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,SAAS;gBAC5C,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS;gBAClC,IAAI,EAAE,MAAM,CAAC,SAAS,IAAI,SAAS;gBACnC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,SAAS;aACvC;YACD,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,SAAS,EAAE,UAAU;SACtB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
+{"version":3,"file":"defuddle.js","sourceRoot":"","sources":["../../src/extraction/defuddle.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAEzC,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAE/C,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAY,EAAE,GAAW;IAC7D,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACzC,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QACjC,MAAM,QAAQ,GAAG,cAAc,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAChD,IAAI,QAAQ,CAAC,MAAM,GAAG,qBAAqB;YAAE,OAAO,IAAI,CAAC;QACzD,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;YACzB,QAAQ;YACR,QAAQ,EAAE;gBACR,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,SAAS;gBAC5C,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS;gBAClC,IAAI,EAAE,MAAM,CAAC,SAAS,IAAI,SAAS;gBACnC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,SAAS;aACvC;YACD,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,SAAS,EAAE,UAAU;SACtB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/extraction/jsonld.js

@@ -22,7 +22,7 @@ export function extractJsonLd(html) {
             }
         }
         catch (err) {
-            log.debug('Failed to parse JSON-LD block', { error: String(err) });
+            log.warn('Failed to parse JSON-LD block', { error: String(err) });
         }
     }
     return results;

package/dist/extraction/jsonld.js.map

@@ -1 +1 @@
-{"version":3,"file":"jsonld.js","sourceRoot":"","sources":["../../src/extraction/jsonld.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAEnC,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,GAAG,CAAC,gBAAgB,CAAC,oCAAoC,CAAC,CAAC;IAC3E,MAAM,OAAO,GAA8B,EAAE,CAAC;IAE9C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;YACxC,IAAI,CAAC,IAAI;gBAAE,SAAS;YAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEhC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1B,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;YAC1B,CAAC;iBAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;gBAC/D,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,+BAA+B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,YAAuC,EACvC,MAAkB;IAElB,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAE/D,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,MAAM,SAAS,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IAE9C,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QACvD,IAAI,SAAS,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE,CAAC;YACvC,MAAM,CAAC,SAAS,CAAC,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,aAAa,CACpB,MAAiC;IAEjC,MAAM,IAAI,GAA4B,EAAE,CAAC;IAEzC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CACpB,GAA4B,EAC5B,MAA+B;IAE/B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/C,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAElC,8DAA8D;QAC9D,IAAI,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,EAAE,CAAC;YACrB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzE,aAAa,CAAC,KAAgC,EAAE,MAAM,CAAC,CAAC;YAC1D,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
+{"version":3,"file":"jsonld.js","sourceRoot":"","sources":["../../src/extraction/jsonld.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAEnC,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,GAAG,CAAC,gBAAgB,CAAC,oCAAoC,CAAC,CAAC;IAC3E,MAAM,OAAO,GAA8B,EAAE,CAAC;IAE9C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;YACxC,IAAI,CAAC,IAAI;gBAAE,SAAS;YAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEhC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1B,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;YAC1B,CAAC;iBAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;gBAC/D,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CAAC,+BAA+B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpE,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,YAAuC,EACvC,MAAkB;IAElB,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAE/D,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,MAAM,SAAS,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IAE9C,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QACvD,IAAI,SAAS,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE,CAAC;YACvC,MAAM,CAAC,SAAS,CAAC,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,aAAa,CACpB,MAAiC;IAEjC,MAAM,IAAI,GAA4B,EAAE,CAAC;IAEzC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CACpB,GAA4B,EAC5B,MAA+B;IAE/B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/C,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAElC,8DAA8D;QAC9D,IAAI,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,EAAE,CAAC;YACrB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzE,aAAa,CAAC,KAAgC,EAAE,MAAM,CAAC,CAAC;YAC1D,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/extraction/lang-hints.d.ts

@@ -0,0 +1,2 @@
+export declare function detectCodeLanguage(classAttr: string | null | undefined): string | null;
+//# sourceMappingURL=lang-hints.d.ts.map

package/dist/extraction/lang-hints.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lang-hints.d.ts","sourceRoot":"","sources":["../../src/extraction/lang-hints.ts"],"names":[],"mappings":"AAiBA,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,CAUtF"}

package/dist/extraction/lang-hints.js

@@ -0,0 +1,28 @@
+const ALIASES = {
+    typescript: 'ts',
+    javascript: 'js',
+    python: 'py',
+    rust: 'rs',
+    golang: 'go',
+    shell: 'sh',
+};
+const PATTERNS = [
+    /(?:^|\s)language-([a-z0-9+#-]+)/i,
+    /(?:^|\s)lang-([a-z0-9+#-]+)/i,
+    /(?:^|\s)hljs-([a-z0-9+#-]+)/i,
+    /(?:^|\s)prism-language-([a-z0-9+#-]+)/i,
+    /(?:^|\s)highlight-source-([a-z0-9+#-]+)/i,
+];
+export function detectCodeLanguage(classAttr) {
+    if (!classAttr)
+        return null;
+    for (const re of PATTERNS) {
+        const m = classAttr.match(re);
+        if (m) {
+            const raw = m[1].toLowerCase();
+            return ALIASES[raw] ?? raw;
+        }
+    }
+    return null;
+}
+//# sourceMappingURL=lang-hints.js.map

package/dist/extraction/lang-hints.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lang-hints.js","sourceRoot":"","sources":["../../src/extraction/lang-hints.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAA2B;IACtC,UAAU,EAAE,IAAI;IAChB,UAAU,EAAE,IAAI;IAChB,MAAM,EAAE,IAAI;IACZ,IAAI,EAAE,IAAI;IACV,MAAM,EAAE,IAAI;IACZ,KAAK,EAAE,IAAI;CACZ,CAAC;AAEF,MAAM,QAAQ,GAAG;IACf,kCAAkC;IAClC,8BAA8B;IAC9B,8BAA8B;IAC9B,wCAAwC;IACxC,0CAA0C;CAC3C,CAAC;AAEF,MAAM,UAAU,kBAAkB,CAAC,SAAoC;IACrE,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IAC5B,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC9B,IAAI,CAAC,EAAE,CAAC;YACN,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC;YAC/B,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/extraction/llm/anthropic.d.ts

@@ -0,0 +1,3 @@
+import type { LLMCallOpts, LLMExtractResult } from './types.js';
+export declare function callAnthropic(opts: LLMCallOpts, apiKey: string): Promise<LLMExtractResult>;
+//# sourceMappingURL=anthropic.d.ts.map

package/dist/extraction/llm/anthropic.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"anthropic.d.ts","sourceRoot":"","sources":["../../../src/extraction/llm/anthropic.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAKhE,wBAAsB,aAAa,CACjC,IAAI,EAAE,WAAW,EACjB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,gBAAgB,CAAC,CAqC3B"}

package/dist/extraction/llm/anthropic.js

@@ -0,0 +1,33 @@
+import Anthropic from '@anthropic-ai/sdk';
+const DEFAULT_MODEL = 'claude-haiku-4-5';
+const TOOL_NAME = 'extract';
+export async function callAnthropic(opts, apiKey) {
+    const client = new Anthropic({ apiKey });
+    const model = opts.modelOverride ?? DEFAULT_MODEL;
+    const start = Date.now();
+    const response = await client.messages.create({
+        model,
+        max_tokens: 2048,
+        tools: [
+            {
+                name: TOOL_NAME,
+                description: 'Return the extracted fields as structured JSON.',
+                input_schema: opts.jsonSchema,
+            },
+        ],
+        tool_choice: { type: 'tool', name: TOOL_NAME },
+        messages: [{ role: 'user', content: opts.prompt }],
+    }, { signal: opts.signal });
+    const block = (response.content ?? []).find((b) => b.type === 'tool_use');
+    if (!block) {
+        throw new Error('anthropic: no tool_use block in response');
+    }
+    return {
+        values: block.input,
+        provider: 'anthropic',
+        model: response.model ?? model,
+        cached: false,
+        latencyMs: Date.now() - start,
+    };
+}
+//# sourceMappingURL=anthropic.js.map

package/dist/extraction/llm/anthropic.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"anthropic.js","sourceRoot":"","sources":["../../../src/extraction/llm/anthropic.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAI1C,MAAM,aAAa,GAAG,kBAAkB,CAAC;AACzC,MAAM,SAAS,GAAG,SAAS,CAAC;AAE5B,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,IAAiB,EACjB,MAAc;IAEd,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,IAAI,aAAa,CAAC;IAClD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEzB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAC3C;QACE,KAAK;QACL,UAAU,EAAE,IAAI;QAChB,KAAK,EAAE;YACL;gBACE,IAAI,EAAE,SAAS;gBACf,WAAW,EAAE,iDAAiD;gBAC9D,YAAY,EAAE,IAAI,CAAC,UAA8B;aAClD;SACF;QACD,WAAW,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE;QAC9C,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;KACnD,EACD,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CACxB,CAAC;IAEF,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,CACzC,CAAC,CAAmB,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CACmC,CAAC;IAEpF,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;IAC9D,CAAC;IAED,OAAO;QACL,MAAM,EAAE,KAAK,CAAC,KAAK;QACnB,QAAQ,EAAE,WAAW;QACrB,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,KAAK;QAC9B,MAAM,EAAE,KAAK;QACb,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;KAC9B,CAAC;AACJ,CAAC"}

package/dist/extraction/llm/cache.d.ts

@@ -0,0 +1,5 @@
+import type { LLMCallRecord } from './types.js';
+export declare function ensureLLMCacheTable(): void;
+export declare function lookupLLMCache(modelId: string, promptHash: string, schemaHash: string): string | null;
+export declare function insertLLMCache(rec: LLMCallRecord): void;
+//# sourceMappingURL=cache.d.ts.map

package/dist/extraction/llm/cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.d.ts","sourceRoot":"","sources":["../../../src/extraction/llm/cache.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,wBAAgB,mBAAmB,IAAI,IAAI,CAc1C;AAED,wBAAgB,cAAc,CAC5B,OAAO,EAAE,MAAM,EACf,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,MAAM,GACjB,MAAM,GAAG,IAAI,CAYf;AAED,wBAAgB,cAAc,CAAC,GAAG,EAAE,aAAa,GAAG,IAAI,CAiBvD"}

package/dist/extraction/llm/cache.js

@@ -0,0 +1,35 @@
+import { getDatabase } from '../../cache/db.js';
+export function ensureLLMCacheTable() {
+    const db = getDatabase();
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS llm_cache (
+      model_id TEXT NOT NULL,
+      prompt_hash TEXT NOT NULL,
+      schema_hash TEXT NOT NULL,
+      response TEXT NOT NULL,
+      created_at INTEGER NOT NULL,
+      expires_at INTEGER NOT NULL,
+      PRIMARY KEY (model_id, prompt_hash, schema_hash)
+    );
+    CREATE INDEX IF NOT EXISTS idx_llm_cache_expires ON llm_cache(expires_at);
+  `);
+}
+export function lookupLLMCache(modelId, promptHash, schemaHash) {
+    const db = getDatabase();
+    const row = db
+        .prepare(`SELECT response FROM llm_cache
+       WHERE model_id = ? AND prompt_hash = ? AND schema_hash = ?
+         AND expires_at > ?`)
+        .get(modelId, promptHash, schemaHash, Date.now());
+    return row?.response ?? null;
+}
+export function insertLLMCache(rec) {
+    const db = getDatabase();
+    db.prepare(`INSERT INTO llm_cache (model_id, prompt_hash, schema_hash, response, created_at, expires_at)
+     VALUES (?, ?, ?, ?, ?, ?)
+     ON CONFLICT(model_id, prompt_hash, schema_hash) DO UPDATE SET
+       response = excluded.response,
+       created_at = excluded.created_at,
+       expires_at = excluded.expires_at`).run(rec.modelId, rec.promptHash, rec.schemaHash, rec.response, rec.createdAt, rec.expiresAt);
+}
+//# sourceMappingURL=cache.js.map

package/dist/extraction/llm/cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../../src/extraction/llm/cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAGhD,MAAM,UAAU,mBAAmB;IACjC,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;GAWP,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,OAAe,EACf,UAAkB,EAClB,UAAkB;IAElB,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,GAAG,GAAG,EAAE;SACX,OAAO,CACN;;4BAEsB,CACvB;SACA,GAAG,CAAC,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,CAErC,CAAC;IACd,OAAO,GAAG,EAAE,QAAQ,IAAI,IAAI,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,GAAkB;IAC/C,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,EAAE,CAAC,OAAO,CACR;;;;;wCAKoC,CACrC,CAAC,GAAG,CACH,GAAG,CAAC,OAAO,EACX,GAAG,CAAC,UAAU,EACd,GAAG,CAAC,UAAU,EACd,GAAG,CAAC,QAAQ,EACZ,GAAG,CAAC,SAAS,EACb,GAAG,CAAC,SAAS,CACd,CAAC;AACJ,CAAC"}

package/dist/extraction/llm/gemini.d.ts

@@ -0,0 +1,3 @@
+import type { LLMCallOpts, LLMExtractResult } from './types.js';
+export declare function callGemini(opts: LLMCallOpts, apiKey: string): Promise<LLMExtractResult>;
+//# sourceMappingURL=gemini.d.ts.map

package/dist/extraction/llm/gemini.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gemini.d.ts","sourceRoot":"","sources":["../../../src/extraction/llm/gemini.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAIhE,wBAAsB,UAAU,CAC9B,IAAI,EAAE,WAAW,EACjB,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,gBAAgB,CAAC,CAkC3B"}