@staticn0va/wigolo 0.1.0 → 0.1.1

package/dist/config.js

@@ -1,66 +1,116 @@
-import { homedir } from 'node:os';
-import { join } from 'node:path';
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { parseBrowserTypes } from "./fetch/browser-types.js";
 function envStr(key, fallback = null) {
-    return process.env[key] ?? fallback;
+  return process.env[key] ?? fallback;
 }
 function envInt(key, fallback) {
-    const val = process.env[key];
-    if (val === undefined)
-        return fallback;
-    const parsed = parseInt(val, 10);
-    return isNaN(parsed) ? fallback : parsed;
+  const val = process.env[key];
+  if (val === void 0) return fallback;
+  const parsed = parseInt(val, 10);
+  return isNaN(parsed) ? fallback : parsed;
+}
+function envIntArray(key, fallback) {
+  const val = process.env[key];
+  if (val === void 0) return fallback;
+  const parts = val.split(",").map((s) => parseInt(s.trim(), 10));
+  if (parts.some((n) => isNaN(n))) return fallback;
+  return parts;
 }
 function envBool(key, fallback) {
-    const val = process.env[key];
-    if (val === undefined)
-        return fallback;
-    return val.toLowerCase() !== 'false' && val !== '0';
+  const val = process.env[key];
+  if (val === void 0) return fallback;
+  return val.toLowerCase() !== "false" && val !== "0";
 }
 let cachedConfig = null;
-export function getConfig() {
-    if (cachedConfig)
-        return cachedConfig;
-    cachedConfig = {
-        searxngUrl: envStr('SEARXNG_URL'),
-        searxngMode: envStr('SEARXNG_MODE', 'native'),
-        searxngPort: envInt('SEARXNG_PORT', 8888),
-        fetchTimeoutMs: envInt('FETCH_TIMEOUT_MS', 10000),
-        fetchMaxRetries: envInt('FETCH_MAX_RETRIES', 2),
-        maxRedirects: envInt('MAX_REDIRECTS', 5),
-        playwrightLoadTimeoutMs: envInt('PLAYWRIGHT_LOAD_TIMEOUT_MS', 15000),
-        playwrightNavTimeoutMs: envInt('PLAYWRIGHT_NAV_TIMEOUT_MS', 10000),
-        searxngQueryTimeoutMs: envInt('SEARXNG_QUERY_TIMEOUT_MS', 8000),
-        searchFetchTimeoutMs: envInt('SEARCH_FETCH_TIMEOUT_MS', 15000),
-        searchTotalTimeoutMs: envInt('SEARCH_TOTAL_TIMEOUT_MS', 30000),
-        validateTimeoutMs: envInt('VALIDATE_TIMEOUT_MS', 5000),
-        maxBrowsers: envInt('MAX_BROWSERS', 3),
-        browserIdleTimeoutMs: envInt('BROWSER_IDLE_TIMEOUT', 60000),
-        browserFallbackThreshold: envInt('BROWSER_FALLBACK_THRESHOLD', 3),
-        authStatePath: envStr('WIGOLO_AUTH_STATE_PATH'),
-        chromeProfilePath: envStr('WIGOLO_CHROME_PROFILE_PATH'),
-        dataDir: envStr('WIGOLO_DATA_DIR') ?? join(homedir(), '.wigolo'),
-        cacheTtlSearch: envInt('CACHE_TTL_SEARCH', 86400),
-        cacheTtlContent: envInt('CACHE_TTL_CONTENT', 604800),
-        crawlConcurrency: envInt('CRAWL_CONCURRENCY', 2),
-        crawlDelayMs: envInt('CRAWL_DELAY_MS', 500),
-        crawlPrivateConcurrency: envInt('CRAWL_PRIVATE_CONCURRENCY', 10),
-        crawlPrivateDelayMs: envInt('CRAWL_PRIVATE_DELAY_MS', 0),
-        useProxy: envBool('USE_PROXY', false),
-        proxyUrl: envStr('PROXY_URL'),
-        userAgent: envStr('USER_AGENT'),
-        validateLinks: envBool('VALIDATE_LINKS', true),
-        respectRobotsTxt: envBool('RESPECT_ROBOTS_TXT', true),
-        braveApiKey: envStr('BRAVE_API_KEY'),
-        logLevel: envStr('LOG_LEVEL', 'info'),
-        logFormat: envStr('LOG_FORMAT', 'json'),
-        trafilatura: envStr('WIGOLO_TRAFILATURA', 'auto'),
-        reranker: (envStr('WIGOLO_RERANKER') ?? 'none'),
-        rerankerModel: envStr('WIGOLO_RERANKER_MODEL') ?? 'ms-marco-MiniLM-L-12-v2',
-        relevanceThreshold: parseFloat(envStr('WIGOLO_RELEVANCE_THRESHOLD') ?? '0') || 0,
-    };
-    return cachedConfig;
+function getConfig() {
+  if (cachedConfig) return cachedConfig;
+  cachedConfig = {
+    searxngUrl: envStr("SEARXNG_URL"),
+    searxngMode: envStr("SEARXNG_MODE", "native"),
+    searxngPort: envInt("SEARXNG_PORT", 8888),
+    fetchTimeoutMs: envInt("FETCH_TIMEOUT_MS", 1e4),
+    fetchMaxRetries: envInt("FETCH_MAX_RETRIES", 2),
+    maxRedirects: envInt("MAX_REDIRECTS", 5),
+    playwrightLoadTimeoutMs: envInt("PLAYWRIGHT_LOAD_TIMEOUT_MS", 15e3),
+    playwrightNavTimeoutMs: envInt("PLAYWRIGHT_NAV_TIMEOUT_MS", 1e4),
+    searxngQueryTimeoutMs: envInt("SEARXNG_QUERY_TIMEOUT_MS", 8e3),
+    searchFetchTimeoutMs: envInt("SEARCH_FETCH_TIMEOUT_MS", 15e3),
+    searchTotalTimeoutMs: envInt("SEARCH_TOTAL_TIMEOUT_MS", 3e4),
+    validateTimeoutMs: envInt("VALIDATE_TIMEOUT_MS", 5e3),
+    maxBrowsers: envInt("MAX_BROWSERS", 3),
+    browserIdleTimeoutMs: envInt("BROWSER_IDLE_TIMEOUT", 6e4),
+    browserFallbackThreshold: envInt("BROWSER_FALLBACK_THRESHOLD", 3),
+    authStatePath: envStr("WIGOLO_AUTH_STATE_PATH"),
+    chromeProfilePath: envStr("WIGOLO_CHROME_PROFILE_PATH"),
+    cdpUrl: envStr("WIGOLO_CDP_URL") || null,
+    dataDir: envStr("WIGOLO_DATA_DIR") ?? join(homedir(), ".wigolo"),
+    cacheTtlSearch: envInt("CACHE_TTL_SEARCH", 86400),
+    cacheTtlContent: envInt("CACHE_TTL_CONTENT", 604800),
+    fastStaleMaxHours: envInt("WIGOLO_FAST_STALE_MAX_HOURS", 24),
+    fastTimeoutMs: envInt("WIGOLO_FAST_TIMEOUT_MS", 800),
+    crawlConcurrency: envInt("CRAWL_CONCURRENCY", 2),
+    crawlDelayMs: envInt("CRAWL_DELAY_MS", 500),
+    crawlPrivateConcurrency: envInt("CRAWL_PRIVATE_CONCURRENCY", 10),
+    crawlPrivateDelayMs: envInt("CRAWL_PRIVATE_DELAY_MS", 0),
+    useProxy: envBool("USE_PROXY", false),
+    proxyUrl: envStr("PROXY_URL"),
+    userAgent: envStr("USER_AGENT"),
+    validateLinks: envBool("VALIDATE_LINKS", true),
+    respectRobotsTxt: envBool("RESPECT_ROBOTS_TXT", true),
+    braveApiKey: envStr("BRAVE_API_KEY"),
+    logLevel: envStr("LOG_LEVEL", "info"),
+    logFormat: envStr("LOG_FORMAT", "json"),
+    reranker: (() => {
+      const raw = envStr("WIGOLO_RERANKER") ?? "onnx";
+      if (raw === "flashrank") {
+        console.warn(
+          '[wigolo] WIGOLO_RERANKER=flashrank is a legacy alias; treating as onnx. The reranker runs as a Python subprocess; install via "wigolo warmup --reranker".'
+        );
+        return "onnx";
+      }
+      return raw;
+    })(),
+    rerankerModel: envStr("WIGOLO_RERANKER_MODEL") ?? "bge-reranker-v2-m3",
+    rerankerMaxLength: envInt("WIGOLO_RERANKER_MAX_LENGTH", 512),
+    rerankerReadyTimeoutMs: envInt("WIGOLO_RERANKER_READY_TIMEOUT_MS", 6e4),
+    rerankerRequestTimeoutMs: envInt("WIGOLO_RERANKER_REQUEST_TIMEOUT_MS", 3e4),
+    rerankerIdleTimeoutMs: envInt("WIGOLO_RERANKER_IDLE_TIMEOUT_MS", 3e5),
+    relevanceThreshold: parseFloat(envStr("WIGOLO_RELEVANCE_THRESHOLD") ?? "0") || 0,
+    bootstrapMaxAttempts: envInt("WIGOLO_BOOTSTRAP_MAX_ATTEMPTS", 3),
+    bootstrapBackoffSeconds: envIntArray("WIGOLO_BOOTSTRAP_BACKOFF_SECONDS", [30, 3600, 86400]),
+    healthProbeIntervalMs: envInt("WIGOLO_HEALTH_PROBE_INTERVAL_MS", 3e4),
+    daemonPort: envInt("WIGOLO_DAEMON_PORT", 3333),
+    daemonHost: envStr("WIGOLO_DAEMON_HOST", "127.0.0.1")?.trim() || "127.0.0.1",
+    pluginsDir: (() => {
+      const raw = envStr("WIGOLO_PLUGINS_DIR");
+      if (raw) {
+        if (raw.startsWith("~")) return join(homedir(), raw.slice(1));
+        return raw;
+      }
+      return join(envStr("WIGOLO_DATA_DIR") ?? join(homedir(), ".wigolo"), "plugins");
+    })(),
+    browserTypes: parseBrowserTypes(envStr("WIGOLO_BROWSER_TYPES")),
+    shellHistoryPath: envStr("WIGOLO_SHELL_HISTORY_PATH") ?? join(homedir(), ".wigolo", "shell-history"),
+    multiQueryConcurrency: envInt("WIGOLO_MULTI_QUERY_CONCURRENCY", 5),
+    multiQueryMax: envInt("WIGOLO_MULTI_QUERY_MAX", 10),
+    embeddingModel: envStr("WIGOLO_EMBEDDING_MODEL") ?? "BAAI/bge-small-en-v1.5",
+    embeddingIdleTimeoutMs: envInt("WIGOLO_EMBEDDING_IDLE_TIMEOUT", 18e5),
+    embeddingMaxTextLength: envInt("WIGOLO_EMBEDDING_MAX_TEXT_LENGTH", 8e3),
+    lightpandaUrl: envStr("WIGOLO_LIGHTPANDA_URL"),
+    lightpandaEnabled: envBool("WIGOLO_LIGHTPANDA_ENABLED", false),
+    lightpandaFailureThreshold: envInt("WIGOLO_LIGHTPANDA_FAILURE_THRESHOLD", 3),
+    llmProvider: envStr("WIGOLO_LLM_PROVIDER"),
+    llmCacheTtlDays: envInt("WIGOLO_LLM_CACHE_TTL_DAYS", 7),
+    llmMaxCallsPerRequest: envInt("WIGOLO_LLM_MAX_CALLS_PER_REQUEST", 1)
+  };
+  return cachedConfig;
 }
-export function resetConfig() {
-    cachedConfig = null;
+function resetConfig() {
+  cachedConfig = null;
 }
+export {
+  getConfig,
+  resetConfig
+};
 //# sourceMappingURL=config.js.map

package/dist/config.js.map

@@ -1 +1 @@
-{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAyCjC,SAAS,MAAM,CAAC,GAAW,EAAE,WAA0B,IAAI;IACzD,OAAO,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC;AACtC,CAAC;AAED,SAAS,MAAM,CAAC,GAAW,EAAE,QAAgB;IAC3C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACvC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACjC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC;AAC3C,CAAC;AAED,SAAS,OAAO,CAAC,GAAW,EAAE,QAAiB;IAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACvC,OAAO,GAAG,CAAC,WAAW,EAAE,KAAK,OAAO,IAAI,GAAG,KAAK,GAAG,CAAC;AACtD,CAAC;AAED,IAAI,YAAY,GAAkB,IAAI,CAAC;AAEvC,MAAM,UAAU,SAAS;IACvB,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC;IAEtC,YAAY,GAAG;QACb,UAAU,EAAE,MAAM,CAAC,aAAa,CAAC;QACjC,WAAW,EAAG,MAAM,CAAC,cAAc,EAAE,QAAQ,CAAyB;QACtE,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,IAAI,CAAC;QACzC,cAAc,EAAE,MAAM,CAAC,kBAAkB,EAAE,KAAK,CAAC;QACjD,eAAe,EAAE,MAAM,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAC/C,YAAY,EAAE,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;QACxC,uBAAuB,EAAE,MAAM,CAAC,4BAA4B,EAAE,KAAK,CAAC;QACpE,sBAAsB,EAAE,MAAM,CAAC,2BAA2B,EAAE,KAAK,CAAC;QAClE,qBAAqB,EAAE,MAAM,CAAC,0BAA0B,EAAE,IAAI,CAAC;QAC/D,oBAAoB,EAAE,MAAM,CAAC,yBAAyB,EAAE,KAAK,CAAC;QAC9D,oBAAoB,EAAE,MAAM,CAAC,yBAAyB,EAAE,KAAK,CAAC;QAC9D,iBAAiB,EAAE,MAAM,CAAC,qBAAqB,EAAE,IAAI,CAAC;QACtD,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;QACtC,oBAAoB,EAAE,MAAM,CAAC,sBAAsB,EAAE,KAAK,CAAC;QAC3D,wBAAwB,EAAE,MAAM,CAAC,4BAA4B,EAAE,CAAC,CAAC;QACjE,aAAa,EAAE,MAAM,CAAC,wBAAwB,CAAC;QAC/C,iBAAiB,EAAE,MAAM,CAAC,4BAA4B,CAAC;QACvD,OAAO,EAAE,MAAM,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,CAAC;QAChE,cAAc,EAAE,MAAM,CAAC,kBAAkB,EAAE,KAAK,CAAC;QACjD,eAAe,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC;QACpD,gBAAgB,EAAE,MAAM,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAChD,YAAY,EAAE,MAAM,CAAC,gBAAgB,EAAE,GAAG,CAAC;QAC3C,uBAAuB,EAAE,MAAM,CAAC,2BAA2B,EAAE,EAAE,CAAC;QAChE,mBAAmB,EAAE,MAAM,CAAC,wBAAwB,EAAE,CAAC,CAAC;QACxD,QAAQ,EAAE,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC;QACrC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC;QAC7B,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC;QAC/B,aAAa,EAAE,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;QAC9C,gBAAgB,EAAE,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC;QACrD,WAAW,EAAE,MAAM,CAAC,eAAe,CAAC;QACpC,QAAQ,EAAG,MAAM,CAAC,WAAW,EAAE,MAAM,CAAwB;QAC7D,SAAS,EAAG,MAAM,CAAC,YAAY,EAAE,MAAM,CAAyB;QAChE,WAAW,EAAG,MAAM,CAAC,oBAAoB,EAAE,MAAM,CAAiC;QAClF,QAAQ,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI,MAAM,CAAuB;QACrE,aAAa,EAAE,MAAM,CAAC,uBAAuB,CAAC,IAAI,yBAAyB;QAC3E,kBAAkB,EAAE,UAAU,CAAC,MAAM,CAAC,4BAA4B,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC;KACjF,CAAC;IAEF,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,YAAY,GAAG,IAAI,CAAC;AACtB,CAAC"}
+{"version":3,"sources":["../src/config.ts"],"sourcesContent":["import { homedir } from 'node:os';\nimport { join } from 'node:path';\nimport { parseBrowserTypes } from './fetch/browser-types.js';\nimport type { BrowserType } from './types.js';\n\nexport interface Config {\n  searxngUrl: string | null;\n  searxngMode: 'native' | 'docker';\n  searxngPort: number;\n  fetchTimeoutMs: number;\n  fetchMaxRetries: number;\n  maxRedirects: number;\n  playwrightLoadTimeoutMs: number;\n  playwrightNavTimeoutMs: number;\n  searxngQueryTimeoutMs: number;\n  searchFetchTimeoutMs: number;\n  searchTotalTimeoutMs: number;\n  validateTimeoutMs: number;\n  maxBrowsers: number;\n  browserIdleTimeoutMs: number;\n  browserFallbackThreshold: number;\n  authStatePath: string | null;\n  chromeProfilePath: string | null;\n  cdpUrl: string | null;\n  dataDir: string;\n  cacheTtlSearch: number;\n  cacheTtlContent: number;\n  fastStaleMaxHours: number;\n  fastTimeoutMs: number;\n  crawlConcurrency: number;\n  crawlDelayMs: number;\n  crawlPrivateConcurrency: number;\n  crawlPrivateDelayMs: number;\n  useProxy: boolean;\n  proxyUrl: string | null;\n  userAgent: string | null;\n  validateLinks: boolean;\n  respectRobotsTxt: boolean;\n  braveApiKey: string | null;\n  logLevel: 'debug' | 'info' | 'warn' | 'error';\n  logFormat: 'json' | 'text';\n  reranker: 'onnx' | 'none' | 'custom';\n  rerankerModel: string;\n  rerankerMaxLength: number;\n  rerankerReadyTimeoutMs: number;\n  rerankerRequestTimeoutMs: number;\n  rerankerIdleTimeoutMs: number;\n  relevanceThreshold: number;\n  bootstrapMaxAttempts: number;\n  bootstrapBackoffSeconds: number[];\n  healthProbeIntervalMs: number;\n  daemonPort: number;\n  daemonHost: string;\n  pluginsDir: string;\n  browserTypes: BrowserType[];\n  shellHistoryPath: string;\n  multiQueryConcurrency: number;\n  multiQueryMax: number;\n  embeddingModel: string;\n  embeddingIdleTimeoutMs: number;\n  embeddingMaxTextLength: number;\n  lightpandaUrl: string | null;\n  lightpandaEnabled: boolean;\n  lightpandaFailureThreshold: number;\n  llmProvider: string | null;\n  llmCacheTtlDays: number;\n  llmMaxCallsPerRequest: number;\n}\n\nfunction envStr(key: string, fallback: string | null = null): string | null {\n  return process.env[key] ?? fallback;\n}\n\nfunction envInt(key: string, fallback: number): number {\n  const val = process.env[key];\n  if (val === undefined) return fallback;\n  const parsed = parseInt(val, 10);\n  return isNaN(parsed) ? fallback : parsed;\n}\n\nfunction envIntArray(key: string, fallback: number[]): number[] {\n  const val = process.env[key];\n  if (val === undefined) return fallback;\n  const parts = val.split(',').map(s => parseInt(s.trim(), 10));\n  if (parts.some(n => isNaN(n))) return fallback;\n  return parts;\n}\n\nfunction envBool(key: string, fallback: boolean): boolean {\n  const val = process.env[key];\n  if (val === undefined) return fallback;\n  return val.toLowerCase() !== 'false' && val !== '0';\n}\n\nlet cachedConfig: Config | null = null;\n\nexport function getConfig(): Config {\n  if (cachedConfig) return cachedConfig;\n\n  cachedConfig = {\n    searxngUrl: envStr('SEARXNG_URL'),\n    searxngMode: (envStr('SEARXNG_MODE', 'native') as 'native' | 'docker'),\n    searxngPort: envInt('SEARXNG_PORT', 8888),\n    fetchTimeoutMs: envInt('FETCH_TIMEOUT_MS', 10000),\n    fetchMaxRetries: envInt('FETCH_MAX_RETRIES', 2),\n    maxRedirects: envInt('MAX_REDIRECTS', 5),\n    playwrightLoadTimeoutMs: envInt('PLAYWRIGHT_LOAD_TIMEOUT_MS', 15000),\n    playwrightNavTimeoutMs: envInt('PLAYWRIGHT_NAV_TIMEOUT_MS', 10000),\n    searxngQueryTimeoutMs: envInt('SEARXNG_QUERY_TIMEOUT_MS', 8000),\n    searchFetchTimeoutMs: envInt('SEARCH_FETCH_TIMEOUT_MS', 15000),\n    searchTotalTimeoutMs: envInt('SEARCH_TOTAL_TIMEOUT_MS', 30000),\n    validateTimeoutMs: envInt('VALIDATE_TIMEOUT_MS', 5000),\n    maxBrowsers: envInt('MAX_BROWSERS', 3),\n    browserIdleTimeoutMs: envInt('BROWSER_IDLE_TIMEOUT', 60000),\n    browserFallbackThreshold: envInt('BROWSER_FALLBACK_THRESHOLD', 3),\n    authStatePath: envStr('WIGOLO_AUTH_STATE_PATH'),\n    chromeProfilePath: envStr('WIGOLO_CHROME_PROFILE_PATH'),\n    cdpUrl: envStr('WIGOLO_CDP_URL') || null,\n    dataDir: envStr('WIGOLO_DATA_DIR') ?? join(homedir(), '.wigolo'),\n    cacheTtlSearch: envInt('CACHE_TTL_SEARCH', 86400),\n    cacheTtlContent: envInt('CACHE_TTL_CONTENT', 604800),\n    fastStaleMaxHours: envInt('WIGOLO_FAST_STALE_MAX_HOURS', 24),\n    fastTimeoutMs: envInt('WIGOLO_FAST_TIMEOUT_MS', 800),\n    crawlConcurrency: envInt('CRAWL_CONCURRENCY', 2),\n    crawlDelayMs: envInt('CRAWL_DELAY_MS', 500),\n    crawlPrivateConcurrency: envInt('CRAWL_PRIVATE_CONCURRENCY', 10),\n    crawlPrivateDelayMs: envInt('CRAWL_PRIVATE_DELAY_MS', 0),\n    useProxy: envBool('USE_PROXY', false),\n    proxyUrl: envStr('PROXY_URL'),\n    userAgent: envStr('USER_AGENT'),\n    validateLinks: envBool('VALIDATE_LINKS', true),\n    respectRobotsTxt: envBool('RESPECT_ROBOTS_TXT', true),\n    braveApiKey: envStr('BRAVE_API_KEY'),\n    logLevel: (envStr('LOG_LEVEL', 'info') as Config['logLevel']),\n    logFormat: (envStr('LOG_FORMAT', 'json') as Config['logFormat']),\n    reranker: (() => {\n      const raw = envStr('WIGOLO_RERANKER') ?? 'onnx';\n      if (raw === 'flashrank') {\n        console.warn(\n          '[wigolo] WIGOLO_RERANKER=flashrank is a legacy alias; treating as onnx. ' +\n          'The reranker runs as a Python subprocess; install via \"wigolo warmup --reranker\".',\n        );\n        return 'onnx';\n      }\n      return raw as Config['reranker'];\n    })(),\n    rerankerModel: envStr('WIGOLO_RERANKER_MODEL') ?? 'bge-reranker-v2-m3',\n    rerankerMaxLength: envInt('WIGOLO_RERANKER_MAX_LENGTH', 512),\n    rerankerReadyTimeoutMs: envInt('WIGOLO_RERANKER_READY_TIMEOUT_MS', 60_000),\n    rerankerRequestTimeoutMs: envInt('WIGOLO_RERANKER_REQUEST_TIMEOUT_MS', 30_000),\n    rerankerIdleTimeoutMs: envInt('WIGOLO_RERANKER_IDLE_TIMEOUT_MS', 300_000),\n    relevanceThreshold: parseFloat(envStr('WIGOLO_RELEVANCE_THRESHOLD') ?? '0') || 0,\n    bootstrapMaxAttempts: envInt('WIGOLO_BOOTSTRAP_MAX_ATTEMPTS', 3),\n    bootstrapBackoffSeconds: envIntArray('WIGOLO_BOOTSTRAP_BACKOFF_SECONDS', [30, 3600, 86400]),\n    healthProbeIntervalMs: envInt('WIGOLO_HEALTH_PROBE_INTERVAL_MS', 30000),\n    daemonPort: envInt('WIGOLO_DAEMON_PORT', 3333),\n    daemonHost: envStr('WIGOLO_DAEMON_HOST', '127.0.0.1')?.trim() || '127.0.0.1',\n    pluginsDir: (() => {\n      const raw = envStr('WIGOLO_PLUGINS_DIR');\n      if (raw) {\n        if (raw.startsWith('~')) return join(homedir(), raw.slice(1));\n        return raw;\n      }\n      return join(envStr('WIGOLO_DATA_DIR') ?? join(homedir(), '.wigolo'), 'plugins');\n    })(),\n    browserTypes: parseBrowserTypes(envStr('WIGOLO_BROWSER_TYPES')),\n    shellHistoryPath: envStr('WIGOLO_SHELL_HISTORY_PATH') ?? join(homedir(), '.wigolo', 'shell-history'),\n    multiQueryConcurrency: envInt('WIGOLO_MULTI_QUERY_CONCURRENCY', 5),\n    multiQueryMax: envInt('WIGOLO_MULTI_QUERY_MAX', 10),\n    embeddingModel: envStr('WIGOLO_EMBEDDING_MODEL') ?? 'BAAI/bge-small-en-v1.5',\n    embeddingIdleTimeoutMs: envInt('WIGOLO_EMBEDDING_IDLE_TIMEOUT', 1800000),\n    embeddingMaxTextLength: envInt('WIGOLO_EMBEDDING_MAX_TEXT_LENGTH', 8000),\n    lightpandaUrl: envStr('WIGOLO_LIGHTPANDA_URL'),\n    lightpandaEnabled: envBool('WIGOLO_LIGHTPANDA_ENABLED', false),\n    lightpandaFailureThreshold: envInt('WIGOLO_LIGHTPANDA_FAILURE_THRESHOLD', 3),\n    llmProvider: envStr('WIGOLO_LLM_PROVIDER'),\n    llmCacheTtlDays: envInt('WIGOLO_LLM_CACHE_TTL_DAYS', 7),\n    llmMaxCallsPerRequest: envInt('WIGOLO_LLM_MAX_CALLS_PER_REQUEST', 1),\n  };\n\n  return cachedConfig;\n}\n\nexport function resetConfig(): void {\n  cachedConfig = null;\n}\n"],"mappings":"AAAA,SAAS,eAAe;AACxB,SAAS,YAAY;AACrB,SAAS,yBAAyB;AAmElC,SAAS,OAAO,KAAa,WAA0B,MAAqB;AAC1E,SAAO,QAAQ,IAAI,GAAG,KAAK;AAC7B;AAEA,SAAS,OAAO,KAAa,UAA0B;AACrD,QAAM,MAAM,QAAQ,IAAI,GAAG;AAC3B,MAAI,QAAQ,OAAW,QAAO;AAC9B,QAAM,SAAS,SAAS,KAAK,EAAE;AAC/B,SAAO,MAAM,MAAM,IAAI,WAAW;AACpC;AAEA,SAAS,YAAY,KAAa,UAA8B;AAC9D,QAAM,MAAM,QAAQ,IAAI,GAAG;AAC3B,MAAI,QAAQ,OAAW,QAAO;AAC9B,QAAM,QAAQ,IAAI,MAAM,GAAG,EAAE,IAAI,OAAK,SAAS,EAAE,KAAK,GAAG,EAAE,CAAC;AAC5D,MAAI,MAAM,KAAK,OAAK,MAAM,CAAC,CAAC,EAAG,QAAO;AACtC,SAAO;AACT;AAEA,SAAS,QAAQ,KAAa,UAA4B;AACxD,QAAM,MAAM,QAAQ,IAAI,GAAG;AAC3B,MAAI,QAAQ,OAAW,QAAO;AAC9B,SAAO,IAAI,YAAY,MAAM,WAAW,QAAQ;AAClD;AAEA,IAAI,eAA8B;AAE3B,SAAS,YAAoB;AAClC,MAAI,aAAc,QAAO;AAEzB,iBAAe;AAAA,IACb,YAAY,OAAO,aAAa;AAAA,IAChC,aAAc,OAAO,gBAAgB,QAAQ;AAAA,IAC7C,aAAa,OAAO,gBAAgB,IAAI;AAAA,IACxC,gBAAgB,OAAO,oBAAoB,GAAK;AAAA,IAChD,iBAAiB,OAAO,qBAAqB,CAAC;AAAA,IAC9C,cAAc,OAAO,iBAAiB,CAAC;AAAA,IACvC,yBAAyB,OAAO,8BAA8B,IAAK;AAAA,IACnE,wBAAwB,OAAO,6BAA6B,GAAK;AAAA,IACjE,uBAAuB,OAAO,4BAA4B,GAAI;AAAA,IAC9D,sBAAsB,OAAO,2BAA2B,IAAK;AAAA,IAC7D,sBAAsB,OAAO,2BAA2B,GAAK;AAAA,IAC7D,mBAAmB,OAAO,uBAAuB,GAAI;AAAA,IACrD,aAAa,OAAO,gBAAgB,CAAC;AAAA,IACrC,sBAAsB,OAAO,wBAAwB,GAAK;AAAA,IAC1D,0BAA0B,OAAO,8BAA8B,CAAC;AAAA,IAChE,eAAe,OAAO,wBAAwB;AAAA,IAC9C,mBAAmB,OAAO,4BAA4B;AAAA,IACtD,QAAQ,OAAO,gBAAgB,KAAK;AAAA,IACpC,SAAS,OAAO,iBAAiB,KAAK,KAAK,QAAQ,GAAG,SAAS;AAAA,IAC/D,gBAAgB,OAAO,oBAAoB,KAAK;AAAA,IAChD,iBAAiB,OAAO,qBAAqB,MAAM;AAAA,IACnD,mBAAmB,OAAO,+BAA+B,EAAE;AAAA,IAC3D,eAAe,OAAO,0BAA0B,GAAG;AAAA,IACnD,kBAAkB,OAAO,qBAAqB,CAAC;AAAA,IAC/C,cAAc,OAAO,kBAAkB,GAAG;AAAA,IAC1C,yBAAyB,OAAO,6BAA6B,EAAE;AAAA,IAC/D,qBAAqB,OAAO,0BAA0B,CAAC;AAAA,IACvD,UAAU,QAAQ,aAAa,KAAK;AAAA,IACpC,UAAU,OAAO,WAAW;AAAA,IAC5B,WAAW,OAAO,YAAY;AAAA,IAC9B,eAAe,QAAQ,kBAAkB,IAAI;AAAA,IAC7C,kBAAkB,QAAQ,sBAAsB,IAAI;AAAA,IACpD,aAAa,OAAO,eAAe;AAAA,IACnC,UAAW,OAAO,aAAa,MAAM;AAAA,IACrC,WAAY,OAAO,cAAc,MAAM;AAAA,IACvC,WAAW,MAAM;AACf,YAAM,MAAM,OAAO,iBAAiB,KAAK;AACzC,UAAI,QAAQ,aAAa;AACvB,gBAAQ;AAAA,UACN;AAAA,QAEF;AACA,eAAO;AAAA,MACT;AACA,aAAO;AAAA,IACT,GAAG;AAAA,IACH,eAAe,OAAO,uBAAuB,KAAK;AAAA,IAClD,mBAAmB,OAAO,8BAA8B,GAAG;AAAA,IAC3D,wBAAwB,OAAO,oCAAoC,GAAM;AAAA,IACzE,0BAA0B,OAAO,sCAAsC,GAAM;AAAA,IAC7E,uBAAuB,OAAO,mCAAmC,GAAO;AAAA,IACxE,oBAAoB,WAAW,OAAO,4BAA4B,KAAK,GAAG,KAAK;AAAA,IAC/E,sBAAsB,OAAO,iCAAiC,CAAC;AAAA,IAC/D,yBAAyB,YAAY,oCAAoC,CAAC,IAAI,MAAM,KAAK,CAAC;AAAA,IAC1F,uBAAuB,OAAO,mCAAmC,GAAK;AAAA,IACtE,YAAY,OAAO,sBAAsB,IAAI;AAAA,IAC7C,YAAY,OAAO,sBAAsB,WAAW,GAAG,KAAK,KAAK;AAAA,IACjE,aAAa,MAAM;AACjB,YAAM,MAAM,OAAO,oBAAoB;AACvC,UAAI,KAAK;AACP,YAAI,IAAI,WAAW,GAAG,EAAG,QAAO,KAAK,QAAQ,GAAG,IAAI,MAAM,CAAC,CAAC;AAC5D,eAAO;AAAA,MACT;AACA,aAAO,KAAK,OAAO,iBAAiB,KAAK,KAAK,QAAQ,GAAG,SAAS,GAAG,SAAS;AAAA,IAChF,GAAG;AAAA,IACH,cAAc,kBAAkB,OAAO,sBAAsB,CAAC;AAAA,IAC9D,kBAAkB,OAAO,2BAA2B,KAAK,KAAK,QAAQ,GAAG,WAAW,eAAe;AAAA,IACnG,uBAAuB,OAAO,kCAAkC,CAAC;AAAA,IACjE,eAAe,OAAO,0BAA0B,EAAE;AAAA,IAClD,gBAAgB,OAAO,wBAAwB,KAAK;AAAA,IACpD,wBAAwB,OAAO,iCAAiC,IAAO;AAAA,IACvE,wBAAwB,OAAO,oCAAoC,GAAI;AAAA,IACvE,eAAe,OAAO,uBAAuB;AAAA,IAC7C,mBAAmB,QAAQ,6BAA6B,KAAK;AAAA,IAC7D,4BAA4B,OAAO,uCAAuC,CAAC;AAAA,IAC3E,aAAa,OAAO,qBAAqB;AAAA,IACzC,iBAAiB,OAAO,6BAA6B,CAAC;AAAA,IACtD,uBAAuB,OAAO,oCAAoC,CAAC;AAAA,EACrE;AAEA,SAAO;AACT;AAEO,SAAS,cAAoB;AAClC,iBAAe;AACjB;","names":[]}

package/dist/crawl/crawler.d.ts

@@ -13,6 +13,12 @@ export declare class Crawler {
     private crawlTraversal;
     private filterLinks;
     crawlSitemap(input: CrawlInput, seedOrigin: string, maxPages: number, robotsParser: RobotsParser | null): Promise<CrawlOutput>;
+    /**
+     * Crawl an explicit list of URLs (e.g. from a sitemap probe). Applies
+     * include/exclude patterns, robots.txt, max_pages, and rate limits the
+     * same way as crawlSitemap.
+     */
+    private crawlFromExplicitUrls;
     private discoverSitemapUrls;
 }
 //# sourceMappingURL=crawler.d.ts.map

package/dist/crawl/crawler.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"crawler.d.ts","sourceRoot":"","sources":["../../src/crawl/crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAA6B,cAAc,EAAE,MAAM,aAAa,CAAC;AAGnH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAO3C,MAAM,MAAM,OAAO,GAAG,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC;AAC5D,MAAM,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;AAElE,qBAAa,OAAO;IAClB,OAAO,CAAC,OAAO,CAAU;IACzB,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,WAAW,CAAqB;gBAE5B,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU;IAK9C,KAAK,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC;IAsBpD,OAAO,CAAC,gBAAgB,CAAuB;YAEjC,WAAW;YAmBX,cAAc;IA8E5B,OAAO,CAAC,WAAW;IA6Bb,YAAY,CAChB,KAAK,EAAE,UAAU,EACjB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,YAAY,GAAG,IAAI,GAChC,OAAO,CAAC,WAAW,CAAC;YAqDT,mBAAmB;CA4ClC"}
+{"version":3,"file":"crawler.d.ts","sourceRoot":"","sources":["../../src/crawl/crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAA6B,cAAc,EAAE,MAAM,aAAa,CAAC;AAGnH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAS3C,MAAM,MAAM,OAAO,GAAG,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC;AAC5D,MAAM,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;AAElE,qBAAa,OAAO;IAClB,OAAO,CAAC,OAAO,CAAU;IACzB,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,WAAW,CAAqB;gBAE5B,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU;IAK9C,KAAK,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC;IAgCpD,OAAO,CAAC,gBAAgB,CAAuB;YAEjC,WAAW;YAmBX,cAAc;IAkF5B,OAAO,CAAC,WAAW;IA6Bb,YAAY,CAChB,KAAK,EAAE,UAAU,EACjB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,YAAY,GAAG,IAAI,GAChC,OAAO,CAAC,WAAW,CAAC;IAYvB;;;;OAIG;YACW,qBAAqB;YAqDrB,mBAAmB;CA4ClC"}