@staticn0va/wigolo 0.1.0

package/dist/cache/store.js

@@ -0,0 +1,214 @@
+import { createHash } from 'node:crypto';
+import { getDatabase } from './db.js';
+import { getConfig } from '../config.js';
+const TRACKING_PARAMS = new Set([
+    'utm_source',
+    'utm_medium',
+    'utm_campaign',
+    'utm_content',
+    'utm_term',
+    'utm_id',
+    'fbclid',
+    'gclid',
+    'msclkid',
+    'mc_cid',
+    'mc_eid',
+]);
+export function normalizeUrl(url) {
+    const parsed = new URL(url);
+    parsed.protocol = parsed.protocol.toLowerCase();
+    parsed.hostname = parsed.hostname.toLowerCase().replace(/^www\./, '');
+    for (const key of [...parsed.searchParams.keys()]) {
+        if (TRACKING_PARAMS.has(key) || key.startsWith('utm_')) {
+            parsed.searchParams.delete(key);
+        }
+    }
+    parsed.searchParams.sort();
+    let result = parsed.toString();
+    // Strip trailing slash from path (but not root)
+    if (parsed.pathname !== '/' && result.endsWith('/')) {
+        result = result.slice(0, -1);
+    }
+    // Remove trailing slash from origin-only URLs too
+    if (parsed.pathname === '/' && !parsed.search && !parsed.hash) {
+        result = result.replace(/\/$/, '');
+    }
+    return result;
+}
+function toIsoSeconds(date) {
+    return date.toISOString().replace('T', ' ').replace(/\.\d+Z$/, '');
+}
+export function cacheContent(result, extraction) {
+    const db = getDatabase();
+    const config = getConfig();
+    const normalizedUrl = normalizeUrl(result.finalUrl || result.url);
+    const contentHash = createHash('sha256').update(extraction.markdown).digest('hex');
+    const now = new Date();
+    const expiresAt = new Date(now.getTime() + config.cacheTtlContent * 1000);
+    const stmt = db.prepare(`
+    INSERT OR REPLACE INTO url_cache (
+      url, normalized_url, title, markdown, raw_html,
+      metadata, links, images, fetch_method, extractor_used,
+      content_hash, fetched_at, expires_at
+    )
+    VALUES (
+      @url, @normalizedUrl, @title, @markdown, @rawHtml,
+      @metadata, @links, @images, @fetchMethod, @extractorUsed,
+      @contentHash, @fetchedAt, @expiresAt
+    )
+  `);
+    stmt.run({
+        url: result.url,
+        normalizedUrl,
+        title: extraction.title,
+        markdown: extraction.markdown,
+        rawHtml: result.html,
+        metadata: JSON.stringify(extraction.metadata),
+        links: JSON.stringify(extraction.links),
+        images: JSON.stringify(extraction.images),
+        fetchMethod: result.method,
+        extractorUsed: extraction.extractor,
+        contentHash: contentHash,
+        fetchedAt: toIsoSeconds(now),
+        expiresAt: toIsoSeconds(expiresAt),
+    });
+}
+function rowToCachedContent(row) {
+    return {
+        id: row.id,
+        url: row.url,
+        normalizedUrl: row.normalized_url,
+        title: row.title,
+        markdown: row.markdown,
+        rawHtml: row.raw_html,
+        metadata: row.metadata,
+        links: row.links,
+        images: row.images,
+        fetchMethod: row.fetch_method,
+        extractorUsed: row.extractor_used,
+        contentHash: row.content_hash,
+        fetchedAt: row.fetched_at,
+        expiresAt: row.expires_at,
+    };
+}
+export function getCachedContent(url) {
+    const db = getDatabase();
+    const normalizedUrl = normalizeUrl(url);
+    const row = db.prepare(`
+    SELECT * FROM url_cache WHERE url = ? OR normalized_url = ? LIMIT 1
+  `).get(url, normalizedUrl);
+    return row ? rowToCachedContent(row) : null;
+}
+export function isExpired(cached) {
+    if (!cached.expiresAt)
+        return false;
+    return new Date(cached.expiresAt).getTime() < Date.now();
+}
+export function searchCache(query) {
+    const db = getDatabase();
+    const rows = db.prepare(`
+    SELECT url_cache.*
+    FROM url_cache
+    JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid
+    WHERE url_cache_fts MATCH ?
+    ORDER BY rank
+  `).all(query);
+    return rows.map(rowToCachedContent);
+}
+export function cacheSearchResults(query, results, enginesUsed) {
+    const db = getDatabase();
+    const config = getConfig();
+    const queryHash = createHash('sha256').update(query.toLowerCase().trim()).digest('hex');
+    const now = new Date();
+    const expiresAt = new Date(now.getTime() + config.cacheTtlSearch * 1000);
+    const stmt = db.prepare(`
+    INSERT OR REPLACE INTO search_cache (query, query_hash, results, engines_used, searched_at, expires_at)
+    VALUES (@query, @queryHash, @results, @enginesUsed, @searchedAt, @expiresAt)
+  `);
+    stmt.run({
+        query,
+        queryHash,
+        results: JSON.stringify(results),
+        enginesUsed: JSON.stringify(enginesUsed),
+        searchedAt: toIsoSeconds(now),
+        expiresAt: toIsoSeconds(expiresAt),
+    });
+}
+export function getCachedSearchResults(query) {
+    const db = getDatabase();
+    const queryHash = createHash('sha256').update(query.toLowerCase().trim()).digest('hex');
+    const row = db.prepare(`
+    SELECT * FROM search_cache WHERE query_hash = ? AND (expires_at IS NULL OR expires_at > datetime('now'))
+  `).get(queryHash);
+    if (!row)
+        return null;
+    return {
+        query: row.query,
+        results: JSON.parse(row.results),
+        engines_used: JSON.parse(row.engines_used),
+        searched_at: row.searched_at,
+    };
+}
+export function searchCacheFiltered(options) {
+    const db = getDatabase();
+    const conditions = [];
+    const params = [];
+    let fromClause = 'url_cache';
+    if (options.query) {
+        fromClause = 'url_cache JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid';
+        conditions.push('url_cache_fts MATCH ?');
+        params.push(options.query);
+    }
+    if (options.urlPattern) {
+        conditions.push('url_cache.normalized_url GLOB ?');
+        params.push(options.urlPattern);
+    }
+    if (options.since) {
+        conditions.push('url_cache.fetched_at > datetime(?)');
+        params.push(options.since);
+    }
+    const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+    const orderClause = options.query ? 'ORDER BY rank' : 'ORDER BY url_cache.fetched_at DESC';
+    const sql = `SELECT url_cache.* FROM ${fromClause} ${whereClause} ${orderClause} LIMIT 100`;
+    const rows = db.prepare(sql).all(...params);
+    return rows.map(rowToCachedContent);
+}
+export function clearCacheEntries(options) {
+    const db = getDatabase();
+    const conditions = [];
+    const params = [];
+    if (options.query) {
+        conditions.push('id IN (SELECT url_cache.id FROM url_cache JOIN url_cache_fts ON url_cache.id = url_cache_fts.rowid WHERE url_cache_fts MATCH ?)');
+        params.push(options.query);
+    }
+    if (options.urlPattern) {
+        conditions.push('normalized_url GLOB ?');
+        params.push(options.urlPattern);
+    }
+    if (options.since) {
+        conditions.push('fetched_at > datetime(?)');
+        params.push(options.since);
+    }
+    const whereClause = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+    const sql = `DELETE FROM url_cache ${whereClause}`;
+    const result = db.prepare(sql).run(...params);
+    return result.changes;
+}
+export function getCacheStats() {
+    const db = getDatabase();
+    const row = db.prepare(`
+    SELECT
+      COUNT(*) as total_urls,
+      COALESCE(SUM(LENGTH(markdown) + LENGTH(COALESCE(raw_html, ''))), 0) as total_bytes,
+      MIN(fetched_at) as oldest,
+      MAX(fetched_at) as newest
+    FROM url_cache
+  `).get();
+    return {
+        total_urls: row.total_urls,
+        total_size_mb: Math.round((row.total_bytes / (1024 * 1024)) * 1e6) / 1e6,
+        oldest: row.oldest ?? '',
+        newest: row.newest ?? '',
+    };
+}
+//# sourceMappingURL=store.js.map

package/dist/cache/store.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"store.js","sourceRoot":"","sources":["../../src/cache/store.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAGzC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,YAAY;IACZ,YAAY;IACZ,cAAc;IACd,aAAa;IACb,UAAU;IACV,QAAQ;IACR,QAAQ;IACR,OAAO;IACP,SAAS;IACT,QAAQ;IACR,QAAQ;CACT,CAAC,CAAC;AAEH,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;IAE5B,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAChD,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;IAEtE,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;QAClD,IAAI,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACvD,MAAM,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,MAAM,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC;IAE3B,IAAI,MAAM,GAAG,MAAM,CAAC,QAAQ,EAAE,CAAC;IAE/B,gDAAgD;IAChD,IAAI,MAAM,CAAC,QAAQ,KAAK,GAAG,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACpD,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/B,CAAC;IACD,kDAAkD;IAClD,IAAI,MAAM,CAAC,QAAQ,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAC9D,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACrC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,IAAU;IAC9B,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;AACrE,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAsB,EAAE,UAA4B;IAC/E,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAE3B,MAAM,aAAa,GAAG,YAAY,CAAC,MAAM,CAAC,QAAQ,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;IAClE,MAAM,WAAW,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEnF,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC;IAE1E,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;;;;;GAWvB,CAAC,CAAC;IAEH,IAAI,CAAC,GAAG,CAAC;QACP,GAAG,EAAE,MAAM,CAAC,GAAG;QACf,aAAa;QACb,KAAK,EAAE,UAAU,CAAC,KAAK;QACvB,QAAQ,EAAE,UAAU,CAAC,QAAQ;QAC7B,OAAO,EAAE,MAAM,CAAC,IAAI;QACpB,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,QAAQ,CAAC;QAC7C,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,KAAK,CAAC;QACvC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC;QACzC,WAAW,EAAE,MAAM,CAAC,MAAM;QAC1B,aAAa,EAAE,UAAU,CAAC,SAAS;QACnC,WAAW,EAAE,WAAW;QACxB,SAAS,EAAE,YAAY,CAAC,GAAG,CAAC;QAC5B,SAAS,EAAE,YAAY,CAAC,SAAS,CAAC;KACnC,CAAC,CAAC;AACL,CAAC;AAmBD,SAAS,kBAAkB,CAAC,GAAU;IACpC,OAAO;QACL,EAAE,EAAE,GAAG,CAAC,EAAE;QACV,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,aAAa,EAAE,GAAG,CAAC,cAAc;QACjC,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,OAAO,EAAE,GAAG,CAAC,QAAQ;QACrB,QAAQ,EAAE,GAAG,CAAC,QAAQ;QACtB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,WAAW,EAAE,GAAG,CAAC,YAA4C;QAC7D,aAAa,EAAE,GAAG,CAAC,cAAgD;QACnE,WAAW,EAAE,GAAG,CAAC,YAAY;QAC7B,SAAS,EAAE,GAAG,CAAC,UAAU;QACzB,SAAS,EAAE,GAAG,CAAC,UAAU;KAC1B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC1C,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,aAAa,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC;IAExC,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC;;GAEtB,CAAC,CAAC,GAAG,CAAC,GAAG,EAAE,aAAa,CAAsB,CAAC;IAEhD,OAAO,GAAG,CAAC,CAAC,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,MAAqB;IAC7C,IAAI,CAAC,MAAM,CAAC,SAAS;QAAE,OAAO,KAAK,CAAC;IACpC,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IAEzB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;GAMvB,CAAC,CAAC,GAAG,CAAC,KAAK,CAAY,CAAC;IAEzB,OAAO,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;AACtC,CAAC;AASD,MAAM,UAAU,kBAAkB,CAChC,KAAa,EACb,OAA2B,EAC3B,WAAqB;IAErB,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAE3B,MAAM,SAAS,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACxF,MAAM,GAAG,GAAG,IAAI,IAAI,EAAE,CAAC;IACvB,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,cAAc,GAAG,IAAI,CAAC,CAAC;IAEzE,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC;;;GAGvB,CAAC,CAAC;IAEH,IAAI,CAAC,GAAG,CAAC;QACP,KAAK;QACL,SAAS;QACT,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;QAChC,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC;QACxC,UAAU,EAAE,YAAY,CAAC,GAAG,CAAC;QAC7B,SAAS,EAAE,YAAY,CAAC,SAAS,CAAC;KACnC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAAa;IAClD,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,SAAS,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAExF,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC;;GAEtB,CAAC,CAAC,GAAG,CAAC,SAAS,CAA8F,CAAC;IAE/G,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IAEtB,OAAO;QACL,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC;QAChC,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,YAAY,CAAC;QAC1C,WAAW,EAAE,GAAG,CAAC,WAAW;KAC7B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,OAInC;IACC,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,MAAM,MAAM,GAAc,EAAE,CAAC;IAC7B,IAAI,UAAU,GAAG,WAAW,CAAC;IAE7B,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,UAAU,GAAG,oEAAoE,CAAC;QAClF,UAAU,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,UAAU,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;QACnD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,UAAU,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;QACtD,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACrF,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,oCAAoC,CAAC;IAE3F,MAAM,GAAG,GAAG,2BAA2B,UAAU,IAAI,WAAW,IAAI,WAAW,YAAY,CAAC;IAC5F,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAY,CAAC;IACvD,OAAO,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;AACtC,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,OAIjC;IACC,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,MAAM,MAAM,GAAc,EAAE,CAAC;IAE7B,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,UAAU,CAAC,IAAI,CACb,iIAAiI,CAClI,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC;QACvB,UAAU,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACzC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,UAAU,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACrF,MAAM,GAAG,GAAG,yBAAyB,WAAW,EAAE,CAAC;IACnD,MAAM,MAAM,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;IAC9C,OAAO,MAAM,CAAC,OAAO,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,aAAa;IAC3B,MAAM,EAAE,GAAG,WAAW,EAAE,CAAC;IACzB,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC;;;;;;;GAOtB,CAAC,CAAC,GAAG,EAA+F,CAAC;IAEtG,OAAO;QACL,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG;QACxE,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;QACxB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;KACzB,CAAC;AACJ,CAAC"}

package/dist/cli/daemon.d.ts

@@ -0,0 +1,2 @@
+export declare function runDaemon(_args: string[]): void;
+//# sourceMappingURL=daemon.d.ts.map

package/dist/cli/daemon.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"daemon.d.ts","sourceRoot":"","sources":["../../src/cli/daemon.ts"],"names":[],"mappings":"AAAA,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAG/C"}

package/dist/cli/daemon.js

@@ -0,0 +1,5 @@
+export function runDaemon(_args) {
+    process.stderr.write('[wigolo] Daemon mode: Coming in v2\n');
+    process.stderr.write('[wigolo] For now, use stdio mode: npx wigolo\n');
+}
+//# sourceMappingURL=daemon.js.map

package/dist/cli/daemon.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"daemon.js","sourceRoot":"","sources":["../../src/cli/daemon.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,SAAS,CAAC,KAAe;IACvC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC7D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,gDAAgD,CAAC,CAAC;AACzE,CAAC"}

package/dist/cli/health.d.ts

@@ -0,0 +1,2 @@
+export declare function runHealthCheck(): void;
+//# sourceMappingURL=health.d.ts.map

package/dist/cli/health.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"health.d.ts","sourceRoot":"","sources":["../../src/cli/health.ts"],"names":[],"mappings":"AAAA,wBAAgB,cAAc,IAAI,IAAI,CAGrC"}

package/dist/cli/health.js

@@ -0,0 +1,5 @@
+export function runHealthCheck() {
+    process.stderr.write('[wigolo] Health check: Coming in v2\n');
+    process.stderr.write('[wigolo] Daemon mode is required for health checks.\n');
+}
+//# sourceMappingURL=health.js.map

package/dist/cli/health.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"health.js","sourceRoot":"","sources":["../../src/cli/health.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,cAAc;IAC5B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC9D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uDAAuD,CAAC,CAAC;AAChF,CAAC"}

package/dist/cli/index.d.ts

@@ -0,0 +1,7 @@
+export type Command = 'mcp' | 'warmup' | 'serve' | 'health';
+export interface ParsedCommand {
+    command: Command;
+    args: string[];
+}
+export declare function parseCommand(argv: string[]): ParsedCommand;
+//# sourceMappingURL=index.d.ts.map

package/dist/cli/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,OAAO,GAAG,KAAK,GAAG,QAAQ,GAAG,OAAO,GAAG,QAAQ,CAAC;AAE5D,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AAID,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,aAAa,CAQ1D"}

package/dist/cli/index.js

@@ -0,0 +1,9 @@
+const KNOWN_COMMANDS = new Set(['warmup', 'serve', 'health']);
+export function parseCommand(argv) {
+    const first = argv[0];
+    if (first && KNOWN_COMMANDS.has(first)) {
+        return { command: first, args: argv.slice(1) };
+    }
+    return { command: 'mcp', args: [] };
+}
+//# sourceMappingURL=index.js.map

package/dist/cli/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/cli/index.ts"],"names":[],"mappings":"AAOA,MAAM,cAAc,GAAwB,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;AAEnF,MAAM,UAAU,YAAY,CAAC,IAAc;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IAEtB,IAAI,KAAK,IAAI,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QACvC,OAAO,EAAE,OAAO,EAAE,KAAgB,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5D,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC;AACtC,CAAC"}

package/dist/cli/warmup.d.ts

@@ -0,0 +1,11 @@
+export interface WarmupResult {
+    playwright: 'ok' | 'failed';
+    playwrightError?: string;
+    searxng: 'ready' | 'bootstrapped' | 'failed' | 'no_python';
+    searxngError?: string;
+    trafilatura?: 'ok' | 'failed' | 'skipped';
+    reranker?: 'ok' | 'failed';
+    rerankerError?: string;
+}
+export declare function runWarmup(flags?: string[]): Promise<WarmupResult>;
+//# sourceMappingURL=warmup.d.ts.map

package/dist/cli/warmup.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"warmup.d.ts","sourceRoot":"","sources":["../../src/cli/warmup.ts"],"names":[],"mappings":"AAKA,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,IAAI,GAAG,QAAQ,CAAC;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,EAAE,OAAO,GAAG,cAAc,GAAG,QAAQ,GAAG,WAAW,CAAC;IAC3D,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,IAAI,GAAG,QAAQ,GAAG,SAAS,CAAC;IAC1C,QAAQ,CAAC,EAAE,IAAI,GAAG,QAAQ,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAqED,wBAAsB,SAAS,CAAC,KAAK,GAAE,MAAM,EAAO,GAAG,OAAO,CAAC,YAAY,CAAC,CAiD3E"}

package/dist/cli/warmup.js

@@ -0,0 +1,107 @@
+import { execSync } from 'node:child_process';
+import { getConfig } from '../config.js';
+import { checkPythonAvailable, bootstrapNativeSearxng, getBootstrapState } from '../searxng/bootstrap.js';
+import { resetAvailabilityCache } from '../search/flashrank.js';
+function log(msg) {
+    process.stderr.write(`[wigolo warmup] ${msg}\n`);
+}
+function installPlaywright() {
+    log('Installing Playwright Chromium...');
+    try {
+        execSync('npx playwright install chromium', { stdio: 'pipe', timeout: 120000 });
+        log('Playwright Chromium installed');
+        return { playwright: 'ok' };
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        log(`Playwright install failed: ${message}`);
+        return { playwright: 'failed', playwrightError: message };
+    }
+}
+function installTrafilatura() {
+    log('Installing Trafilatura...');
+    try {
+        execSync('python3 -m pip install --quiet trafilatura', {
+            stdio: 'pipe',
+            timeout: 120000,
+        });
+        log('Trafilatura installed');
+        return 'ok';
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        log(`Trafilatura install failed: ${message}`);
+        return 'failed';
+    }
+}
+function installFlashRank() {
+    log('Installing FlashRank...');
+    try {
+        execSync('python3 -m pip install --quiet flashrank', { stdio: 'pipe', timeout: 120000 });
+        resetAvailabilityCache();
+        log('FlashRank installed successfully');
+        return { reranker: 'ok' };
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        log(`FlashRank install failed: ${message}`);
+        return { reranker: 'failed', rerankerError: message };
+    }
+}
+function setupSearxng(dataDir) {
+    const state = getBootstrapState(dataDir);
+    if (state?.status === 'ready') {
+        log('SearXNG already set up');
+        return { searxng: 'ready' };
+    }
+    if (!checkPythonAvailable()) {
+        log('Python 3 not found — SearXNG requires Python. Install Python 3 or use Docker mode (SEARXNG_MODE=docker)');
+        return { searxng: 'no_python' };
+    }
+    return { needsBootstrap: true };
+}
+export async function runWarmup(flags = []) {
+    log('Starting warmup...');
+    const config = getConfig();
+    const pwResult = installPlaywright();
+    const searxngCheck = setupSearxng(config.dataDir);
+    let searxngResult;
+    if ('needsBootstrap' in searxngCheck) {
+        log('Bootstrapping SearXNG (this may take a minute)...');
+        try {
+            await bootstrapNativeSearxng(config.dataDir);
+            log('SearXNG bootstrapped successfully');
+            searxngResult = { searxng: 'bootstrapped' };
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            log(`SearXNG bootstrap failed: ${message}`);
+            searxngResult = { searxng: 'failed', searxngError: message };
+        }
+    }
+    else {
+        searxngResult = searxngCheck;
+    }
+    const flagSet = new Set(flags);
+    let trafStatus = 'skipped';
+    if (flagSet.has('--trafilatura') || flagSet.has('--all')) {
+        trafStatus = installTrafilatura();
+    }
+    let rerankerResult = {};
+    if (flagSet.has('--reranker') || flagSet.has('--all')) {
+        rerankerResult = installFlashRank();
+    }
+    const result = { ...pwResult, ...searxngResult, trafilatura: trafStatus, ...rerankerResult };
+    log('');
+    log('Summary:');
+    log(`  Playwright:    ${result.playwright}${result.playwrightError ? ` (${result.playwrightError})` : ''}`);
+    log(`  SearXNG:       ${result.searxng}${result.searxngError ? ` (${result.searxngError})` : ''}`);
+    if (trafStatus !== 'skipped') {
+        log(`  Trafilatura:   ${trafStatus}`);
+    }
+    if (result.reranker) {
+        log(`  FlashRank:     ${result.reranker}${result.rerankerError ? ` (${result.rerankerError})` : ''}`);
+    }
+    return result;
+}
+//# sourceMappingURL=warmup.js.map

package/dist/cli/warmup.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"warmup.js","sourceRoot":"","sources":["../../src/cli/warmup.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAC1G,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAYhE,SAAS,GAAG,CAAC,GAAW;IACtB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,iBAAiB;IACxB,GAAG,CAAC,mCAAmC,CAAC,CAAC;IACzC,IAAI,CAAC;QACH,QAAQ,CAAC,iCAAiC,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAChF,GAAG,CAAC,+BAA+B,CAAC,CAAC;QACrC,OAAO,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IAC9B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,GAAG,CAAC,8BAA8B,OAAO,EAAE,CAAC,CAAC;QAC7C,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,eAAe,EAAE,OAAO,EAAE,CAAC;IAC5D,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB;IACzB,GAAG,CAAC,2BAA2B,CAAC,CAAC;IACjC,IAAI,CAAC;QACH,QAAQ,CAAC,4CAA4C,EAAE;YACrD,KAAK,EAAE,MAAM;YACb,OAAO,EAAE,MAAM;SAChB,CAAC,CAAC;QACH,GAAG,CAAC,uBAAuB,CAAC,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,GAAG,CAAC,+BAA+B,OAAO,EAAE,CAAC,CAAC;QAC9C,OAAO,QAAQ,CAAC;IAClB,CAAC;AACH,CAAC;AAED,SAAS,gBAAgB;IACvB,GAAG,CAAC,yBAAyB,CAAC,CAAC;IAC/B,IAAI,CAAC;QACH,QAAQ,CAAC,0CAA0C,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QACzF,sBAAsB,EAAE,CAAC;QACzB,GAAG,CAAC,kCAAkC,CAAC,CAAC;QACxC,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;IAC5B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,GAAG,CAAC,6BAA6B,OAAO,EAAE,CAAC,CAAC;QAC5C,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,EAAE,OAAO,EAAE,CAAC;IACxD,CAAC;AACH,CAAC;AAMD,SAAS,YAAY,CAAC,OAAe;IACnC,MAAM,KAAK,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAEzC,IAAI,KAAK,EAAE,MAAM,KAAK,OAAO,EAAE,CAAC;QAC9B,GAAG,CAAC,wBAAwB,CAAC,CAAC;QAC9B,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;IAC9B,CAAC;IAED,IAAI,CAAC,oBAAoB,EAAE,EAAE,CAAC;QAC5B,GAAG,CAAC,yGAAyG,CAAC,CAAC;QAC/G,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;IAClC,CAAC;IAED,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAkB,EAAE;IAClD,GAAG,CAAC,oBAAoB,CAAC,CAAC;IAC1B,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;IAE3B,MAAM,QAAQ,GAAG,iBAAiB,EAAE,CAAC;IAErC,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAClD,IAAI,aAA6D,CAAC;IAElE,IAAI,gBAAgB,IAAI,YAAY,EAAE,CAAC;QACrC,GAAG,CAAC,mDAAmD,CAAC,CAAC;QACzD,IAAI,CAAC;YACH,MAAM,sBAAsB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC7C,GAAG,CAAC,mCAAmC,CAAC,CAAC;YACzC,aAAa,GAAG,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,GAAG,CAAC,6BAA6B,OAAO,EAAE,CAAC,CAAC;YAC5C,aAAa,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;QAC/D,CAAC;IACH,CAAC;SAAM,CAAC;QACN,aAAa,GAAG,YAAY,CAAC;IAC/B,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IAC/B,IAAI,UAAU,GAAgC,SAAS,CAAC;IACxD,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACzD,UAAU,GAAG,kBAAkB,EAAE,CAAC;IACpC,CAAC;IAED,IAAI,cAAc,GAAqD,EAAE,CAAC;IAC1E,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACtD,cAAc,GAAG,gBAAgB,EAAE,CAAC;IACtC,CAAC;IAED,MAAM,MAAM,GAAiB,EAAE,GAAG,QAAQ,EAAE,GAAG,aAAa,EAAE,WAAW,EAAE,UAAU,EAAE,GAAG,cAAc,EAAE,CAAC;IAE3G,GAAG,CAAC,EAAE,CAAC,CAAC;IACR,GAAG,CAAC,UAAU,CAAC,CAAC;IAChB,GAAG,CAAC,oBAAoB,MAAM,CAAC,UAAU,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5G,GAAG,CAAC,oBAAoB,MAAM,CAAC,OAAO,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACnG,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAC7B,GAAG,CAAC,oBAAoB,UAAU,EAAE,CAAC,CAAC;IACxC,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACpB,GAAG,CAAC,oBAAoB,MAAM,CAAC,QAAQ,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACxG,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/config.d.ts

@@ -0,0 +1,41 @@
+export interface Config {
+    searxngUrl: string | null;
+    searxngMode: 'native' | 'docker';
+    searxngPort: number;
+    fetchTimeoutMs: number;
+    fetchMaxRetries: number;
+    maxRedirects: number;
+    playwrightLoadTimeoutMs: number;
+    playwrightNavTimeoutMs: number;
+    searxngQueryTimeoutMs: number;
+    searchFetchTimeoutMs: number;
+    searchTotalTimeoutMs: number;
+    validateTimeoutMs: number;
+    maxBrowsers: number;
+    browserIdleTimeoutMs: number;
+    browserFallbackThreshold: number;
+    authStatePath: string | null;
+    chromeProfilePath: string | null;
+    dataDir: string;
+    cacheTtlSearch: number;
+    cacheTtlContent: number;
+    crawlConcurrency: number;
+    crawlDelayMs: number;
+    crawlPrivateConcurrency: number;
+    crawlPrivateDelayMs: number;
+    useProxy: boolean;
+    proxyUrl: string | null;
+    userAgent: string | null;
+    validateLinks: boolean;
+    respectRobotsTxt: boolean;
+    braveApiKey: string | null;
+    logLevel: 'debug' | 'info' | 'warn' | 'error';
+    logFormat: 'json' | 'text';
+    trafilatura: 'auto' | 'always' | 'never';
+    reranker: 'flashrank' | 'none' | 'custom';
+    rerankerModel: string;
+    relevanceThreshold: number;
+}
+export declare function getConfig(): Config;
+export declare function resetConfig(): void;
+//# sourceMappingURL=config.d.ts.map

package/dist/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,MAAM;IACrB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,WAAW,EAAE,QAAQ,GAAG,QAAQ,CAAC;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,MAAM,CAAC;IACrB,uBAAuB,EAAE,MAAM,CAAC;IAChC,sBAAsB,EAAE,MAAM,CAAC;IAC/B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;IACpB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,wBAAwB,EAAE,MAAM,CAAC;IACjC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IAC7B,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,OAAO,EAAE,MAAM,CAAC;IAChB,cAAc,EAAE,MAAM,CAAC;IACvB,eAAe,EAAE,MAAM,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;IACzB,YAAY,EAAE,MAAM,CAAC;IACrB,uBAAuB,EAAE,MAAM,CAAC;IAChC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,QAAQ,EAAE,OAAO,CAAC;IAClB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,aAAa,EAAE,OAAO,CAAC;IACvB,gBAAgB,EAAE,OAAO,CAAC;IAC1B,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,QAAQ,EAAE,OAAO,GAAG,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;IAC9C,SAAS,EAAE,MAAM,GAAG,MAAM,CAAC;IAC3B,WAAW,EAAE,MAAM,GAAG,QAAQ,GAAG,OAAO,CAAC;IACzC,QAAQ,EAAE,WAAW,GAAG,MAAM,GAAG,QAAQ,CAAC;IAC1C,aAAa,EAAE,MAAM,CAAC;IACtB,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAqBD,wBAAgB,SAAS,IAAI,MAAM,CA2ClC;AAED,wBAAgB,WAAW,IAAI,IAAI,CAElC"}

package/dist/config.js

@@ -0,0 +1,66 @@
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+function envStr(key, fallback = null) {
+    return process.env[key] ?? fallback;
+}
+function envInt(key, fallback) {
+    const val = process.env[key];
+    if (val === undefined)
+        return fallback;
+    const parsed = parseInt(val, 10);
+    return isNaN(parsed) ? fallback : parsed;
+}
+function envBool(key, fallback) {
+    const val = process.env[key];
+    if (val === undefined)
+        return fallback;
+    return val.toLowerCase() !== 'false' && val !== '0';
+}
+let cachedConfig = null;
+export function getConfig() {
+    if (cachedConfig)
+        return cachedConfig;
+    cachedConfig = {
+        searxngUrl: envStr('SEARXNG_URL'),
+        searxngMode: envStr('SEARXNG_MODE', 'native'),
+        searxngPort: envInt('SEARXNG_PORT', 8888),
+        fetchTimeoutMs: envInt('FETCH_TIMEOUT_MS', 10000),
+        fetchMaxRetries: envInt('FETCH_MAX_RETRIES', 2),
+        maxRedirects: envInt('MAX_REDIRECTS', 5),
+        playwrightLoadTimeoutMs: envInt('PLAYWRIGHT_LOAD_TIMEOUT_MS', 15000),
+        playwrightNavTimeoutMs: envInt('PLAYWRIGHT_NAV_TIMEOUT_MS', 10000),
+        searxngQueryTimeoutMs: envInt('SEARXNG_QUERY_TIMEOUT_MS', 8000),
+        searchFetchTimeoutMs: envInt('SEARCH_FETCH_TIMEOUT_MS', 15000),
+        searchTotalTimeoutMs: envInt('SEARCH_TOTAL_TIMEOUT_MS', 30000),
+        validateTimeoutMs: envInt('VALIDATE_TIMEOUT_MS', 5000),
+        maxBrowsers: envInt('MAX_BROWSERS', 3),
+        browserIdleTimeoutMs: envInt('BROWSER_IDLE_TIMEOUT', 60000),
+        browserFallbackThreshold: envInt('BROWSER_FALLBACK_THRESHOLD', 3),
+        authStatePath: envStr('WIGOLO_AUTH_STATE_PATH'),
+        chromeProfilePath: envStr('WIGOLO_CHROME_PROFILE_PATH'),
+        dataDir: envStr('WIGOLO_DATA_DIR') ?? join(homedir(), '.wigolo'),
+        cacheTtlSearch: envInt('CACHE_TTL_SEARCH', 86400),
+        cacheTtlContent: envInt('CACHE_TTL_CONTENT', 604800),
+        crawlConcurrency: envInt('CRAWL_CONCURRENCY', 2),
+        crawlDelayMs: envInt('CRAWL_DELAY_MS', 500),
+        crawlPrivateConcurrency: envInt('CRAWL_PRIVATE_CONCURRENCY', 10),
+        crawlPrivateDelayMs: envInt('CRAWL_PRIVATE_DELAY_MS', 0),
+        useProxy: envBool('USE_PROXY', false),
+        proxyUrl: envStr('PROXY_URL'),
+        userAgent: envStr('USER_AGENT'),
+        validateLinks: envBool('VALIDATE_LINKS', true),
+        respectRobotsTxt: envBool('RESPECT_ROBOTS_TXT', true),
+        braveApiKey: envStr('BRAVE_API_KEY'),
+        logLevel: envStr('LOG_LEVEL', 'info'),
+        logFormat: envStr('LOG_FORMAT', 'json'),
+        trafilatura: envStr('WIGOLO_TRAFILATURA', 'auto'),
+        reranker: (envStr('WIGOLO_RERANKER') ?? 'none'),
+        rerankerModel: envStr('WIGOLO_RERANKER_MODEL') ?? 'ms-marco-MiniLM-L-12-v2',
+        relevanceThreshold: parseFloat(envStr('WIGOLO_RELEVANCE_THRESHOLD') ?? '0') || 0,
+    };
+    return cachedConfig;
+}
+export function resetConfig() {
+    cachedConfig = null;
+}
+//# sourceMappingURL=config.js.map

package/dist/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAClC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAyCjC,SAAS,MAAM,CAAC,GAAW,EAAE,WAA0B,IAAI;IACzD,OAAO,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,QAAQ,CAAC;AACtC,CAAC;AAED,SAAS,MAAM,CAAC,GAAW,EAAE,QAAgB;IAC3C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACvC,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACjC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC;AAC3C,CAAC;AAED,SAAS,OAAO,CAAC,GAAW,EAAE,QAAiB;IAC7C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAC7B,IAAI,GAAG,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACvC,OAAO,GAAG,CAAC,WAAW,EAAE,KAAK,OAAO,IAAI,GAAG,KAAK,GAAG,CAAC;AACtD,CAAC;AAED,IAAI,YAAY,GAAkB,IAAI,CAAC;AAEvC,MAAM,UAAU,SAAS;IACvB,IAAI,YAAY;QAAE,OAAO,YAAY,CAAC;IAEtC,YAAY,GAAG;QACb,UAAU,EAAE,MAAM,CAAC,aAAa,CAAC;QACjC,WAAW,EAAG,MAAM,CAAC,cAAc,EAAE,QAAQ,CAAyB;QACtE,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,IAAI,CAAC;QACzC,cAAc,EAAE,MAAM,CAAC,kBAAkB,EAAE,KAAK,CAAC;QACjD,eAAe,EAAE,MAAM,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAC/C,YAAY,EAAE,MAAM,CAAC,eAAe,EAAE,CAAC,CAAC;QACxC,uBAAuB,EAAE,MAAM,CAAC,4BAA4B,EAAE,KAAK,CAAC;QACpE,sBAAsB,EAAE,MAAM,CAAC,2BAA2B,EAAE,KAAK,CAAC;QAClE,qBAAqB,EAAE,MAAM,CAAC,0BAA0B,EAAE,IAAI,CAAC;QAC/D,oBAAoB,EAAE,MAAM,CAAC,yBAAyB,EAAE,KAAK,CAAC;QAC9D,oBAAoB,EAAE,MAAM,CAAC,yBAAyB,EAAE,KAAK,CAAC;QAC9D,iBAAiB,EAAE,MAAM,CAAC,qBAAqB,EAAE,IAAI,CAAC;QACtD,WAAW,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;QACtC,oBAAoB,EAAE,MAAM,CAAC,sBAAsB,EAAE,KAAK,CAAC;QAC3D,wBAAwB,EAAE,MAAM,CAAC,4BAA4B,EAAE,CAAC,CAAC;QACjE,aAAa,EAAE,MAAM,CAAC,wBAAwB,CAAC;QAC/C,iBAAiB,EAAE,MAAM,CAAC,4BAA4B,CAAC;QACvD,OAAO,EAAE,MAAM,CAAC,iBAAiB,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,EAAE,SAAS,CAAC;QAChE,cAAc,EAAE,MAAM,CAAC,kBAAkB,EAAE,KAAK,CAAC;QACjD,eAAe,EAAE,MAAM,CAAC,mBAAmB,EAAE,MAAM,CAAC;QACpD,gBAAgB,EAAE,MAAM,CAAC,mBAAmB,EAAE,CAAC,CAAC;QAChD,YAAY,EAAE,MAAM,CAAC,gBAAgB,EAAE,GAAG,CAAC;QAC3C,uBAAuB,EAAE,MAAM,CAAC,2BAA2B,EAAE,EAAE,CAAC;QAChE,mBAAmB,EAAE,MAAM,CAAC,wBAAwB,EAAE,CAAC,CAAC;QACxD,QAAQ,EAAE,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC;QACrC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC;QAC7B,SAAS,EAAE,MAAM,CAAC,YAAY,CAAC;QAC/B,aAAa,EAAE,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC;QAC9C,gBAAgB,EAAE,OAAO,CAAC,oBAAoB,EAAE,IAAI,CAAC;QACrD,WAAW,EAAE,MAAM,CAAC,eAAe,CAAC;QACpC,QAAQ,EAAG,MAAM,CAAC,WAAW,EAAE,MAAM,CAAwB;QAC7D,SAAS,EAAG,MAAM,CAAC,YAAY,EAAE,MAAM,CAAyB;QAChE,WAAW,EAAG,MAAM,CAAC,oBAAoB,EAAE,MAAM,CAAiC;QAClF,QAAQ,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC,IAAI,MAAM,CAAuB;QACrE,aAAa,EAAE,MAAM,CAAC,uBAAuB,CAAC,IAAI,yBAAyB;QAC3E,kBAAkB,EAAE,UAAU,CAAC,MAAM,CAAC,4BAA4B,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC;KACjF,CAAC;IAEF,OAAO,YAAY,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,WAAW;IACzB,YAAY,GAAG,IAAI,CAAC;AACtB,CAAC"}

package/dist/crawl/crawler.d.ts

@@ -0,0 +1,18 @@
+import type { FetchOutput, CrawlInput, CrawlOutput, RawFetchResult } from '../types.js';
+import { RobotsParser } from './robots.js';
+export type FetchFn = (url: string) => Promise<FetchOutput>;
+export type RawFetchFn = (url: string) => Promise<RawFetchResult>;
+export declare class Crawler {
+    private fetchFn;
+    private rawFetchFn;
+    private rateLimiter;
+    constructor(fetchFn: FetchFn, rawFetchFn: RawFetchFn);
+    crawl(input: CrawlInput): Promise<CrawlOutput>;
+    private robotsTxtContent;
+    private fetchRobots;
+    private crawlTraversal;
+    private filterLinks;
+    crawlSitemap(input: CrawlInput, seedOrigin: string, maxPages: number, robotsParser: RobotsParser | null): Promise<CrawlOutput>;
+    private discoverSitemapUrls;
+}
+//# sourceMappingURL=crawler.d.ts.map

package/dist/crawl/crawler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"crawler.d.ts","sourceRoot":"","sources":["../../src/crawl/crawler.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,WAAW,EAA6B,cAAc,EAAE,MAAM,aAAa,CAAC;AAGnH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAO3C,MAAM,MAAM,OAAO,GAAG,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,WAAW,CAAC,CAAC;AAC5D,MAAM,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;AAElE,qBAAa,OAAO;IAClB,OAAO,CAAC,OAAO,CAAU;IACzB,OAAO,CAAC,UAAU,CAAa;IAC/B,OAAO,CAAC,WAAW,CAAqB;gBAE5B,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU;IAK9C,KAAK,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC;IAsBpD,OAAO,CAAC,gBAAgB,CAAuB;YAEjC,WAAW;YAmBX,cAAc;IA8E5B,OAAO,CAAC,WAAW;IA6Bb,YAAY,CAChB,KAAK,EAAE,UAAU,EACjB,UAAU,EAAE,MAAM,EAClB,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,YAAY,GAAG,IAAI,GAChC,OAAO,CAAC,WAAW,CAAC;YAqDT,mBAAmB;CA4ClC"}