@staticn0va/wigolo 0.1.1 → 0.1.2

package/dist/tools/cache.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/tools/cache.ts"],"sourcesContent":["import {\n  searchCacheFiltered,\n  getCacheStats,\n  clearCacheEntries,\n  ftsSearchRanked,\n  getCachedContentByNormalizedUrl,\n} from '../cache/store.js';\nimport { detectChange } from '../cache/change-detector.js';\nimport { getExtractProvider } from '../providers/extract-provider.js';\nimport { reciprocalRankFusion, sortByRRFScore, buildRankMap } from '../search/rrf.js';\nimport { getEmbedProvider } from '../providers/embed-provider.js';\nimport { getVectorStore } from '../providers/vector-store.js';\nimport { createLogger } from '../logger.js';\nimport type { CacheInput, CacheOutput, CacheResultItem, ChangeReport } from '../types.js';\nimport type { SmartRouter } from '../fetch/router.js';\n\nconst log = createLogger('cache');\n\nconst DEFAULT_HYBRID_LIMIT = 20;\nconst HYBRID_CANDIDATE_FLOOR = 50;\nconst HYBRID_CANDIDATE_FACTOR = 5;\n\nexport async function handleCache(input: CacheInput, router?: SmartRouter): Promise<CacheOutput> {\n  try {\n    if (input.check_changes) {\n      log.info('Checking for content changes', {\n        query: input.query,\n        urlPattern: input.url_pattern,\n        since: input.since,\n      });\n\n      const entries = searchCacheFiltered({\n        query: input.query,\n        urlPattern: input.url_pattern,\n        since: input.since,\n      });\n\n      const changes: ChangeReport[] = [];\n      for (const entry of entries) {\n        try {\n          if (!router) {\n            changes.push({\n              url: entry.url,\n              changed: false,\n              current_hash: entry.contentHash,\n              error: 'no router available for re-fetch',\n            });\n            continue;\n          }\n          const raw = await router.fetch(entry.url, { renderJs: 'auto' });\n          const extractor = await getExtractProvider();\n          const extraction = await extractor.extract(raw.html, raw.finalUrl, {\n            contentType: raw.contentType,\n          });\n          const changeResult = detectChange(entry.url, extraction.markdown);\n          changes.push({\n            url: entry.url,\n            changed: changeResult.changed,\n            current_hash: entry.contentHash,\n            ...(changeResult.changed ? {\n              previous_hash: changeResult.previousHash,\n              diff_summary: changeResult.diffSummary,\n            } : {}),\n          });\n        } catch (err) {\n          log.warn('change check failed for URL', {\n            url: entry.url,\n            error: err instanceof Error ? err.message : String(err),\n          });\n          changes.push({\n            url: entry.url,\n            changed: false,\n            current_hash: entry.contentHash,\n            error: err instanceof Error ? err.message : String(err),\n          });\n        }\n      }\n\n      return { changes };\n    }\n\n    if (input.stats) {\n      log.debug('Cache stats requested');\n      return { stats: getCacheStats() };\n    }\n\n    if (input.clear) {\n      if (!input.query && !input.url_pattern && !input.since) {\n        return { error: 'clear requires at least one filter (query, url_pattern, or since)' };\n      }\n      log.info('Clearing cache entries', {\n        query: input.query,\n        urlPattern: input.url_pattern,\n        since: input.since,\n      });\n      const count = clearCacheEntries({\n        query: input.query,\n        urlPattern: input.url_pattern,\n        since: input.since,\n      });\n      return { cleared: count };\n    }\n\n    if (input.mode === 'hybrid' && input.query) {\n      log.debug('Cache hybrid search', {\n        query: input.query,\n        limit: input.limit,\n      });\n      const results = await runHybridSearch(input);\n      if (results !== null) return { results };\n      // fall through to FTS-only when hybrid was unavailable\n    }\n\n    log.debug('Cache search', {\n      query: input.query,\n      urlPattern: input.url_pattern,\n      since: input.since,\n      mode: input.mode,\n    });\n    const results = searchCacheFiltered({\n      query: input.query,\n      urlPattern: input.url_pattern,\n      since: input.since,\n    });\n\n    return {\n      results: results.map((r) => ({\n        url: r.url,\n        title: r.title,\n        markdown: r.markdown,\n        fetched_at: r.fetchedAt,\n      })),\n    };\n  } catch (err) {\n    log.error('Cache tool error', { error: String(err) });\n    return { error: err instanceof Error ? err.message : String(err) };\n  }\n}\n\n/**\n * Hybrid FTS5 + vector search fused with reciprocal rank fusion.\n *\n * Pulls `max(limit*5, 50)` candidates from each ranking, fuses with RRF\n * (k=60), then hydrates the top `limit` into cache rows. Returns `null`\n * when the vector path is unavailable so the caller falls back to FTS-only.\n */\nasync function runHybridSearch(input: CacheInput): Promise<CacheResultItem[] | null> {\n  const query = input.query ?? '';\n  const limit = Math.max(1, input.limit ?? DEFAULT_HYBRID_LIMIT);\n  const candidateLimit = Math.max(HYBRID_CANDIDATE_FLOOR, limit * HYBRID_CANDIDATE_FACTOR);\n\n  let embedProvider;\n  let store;\n  try {\n    [embedProvider, store] = await Promise.all([getEmbedProvider(), getVectorStore()]);\n  } catch (err) {\n    log.warn('hybrid search unavailable — embed/vector provider failed', {\n      error: err instanceof Error ? err.message : String(err),\n    });\n    return null;\n  }\n\n  const indexSize = await store.size();\n  if (indexSize === 0) {\n    log.debug('hybrid search skipped — vector index empty');\n    return null;\n  }\n\n  let queryVectors: Float32Array[];\n  try {\n    queryVectors = await embedProvider.embed([query]);\n  } catch (err) {\n    log.warn('hybrid search aborted — query embedding failed', {\n      error: err instanceof Error ? err.message : String(err),\n    });\n    return null;\n  }\n  const queryVector = queryVectors[0];\n  if (!queryVector || queryVector.length === 0) return null;\n\n  const [ftsHits, vecHits] = await Promise.all([\n    Promise.resolve(ftsSearchRanked(query, candidateLimit)),\n    store.search(queryVector, candidateLimit),\n  ]);\n\n  const ftsRankMap = buildRankMap(ftsHits.map(h => h.url));\n  const vecRankMap = buildRankMap(vecHits.map(h => h.metadata.url));\n\n  if (ftsRankMap.size === 0 && vecRankMap.size === 0) return [];\n\n  const fused = reciprocalRankFusion([ftsRankMap, vecRankMap], 60);\n  const ordered = sortByRRFScore(fused);\n\n  const results: CacheResultItem[] = [];\n  for (const [normalizedUrl] of ordered) {\n    if (results.length >= limit) break;\n    const cached = getCachedContentByNormalizedUrl(normalizedUrl);\n    if (!cached) continue;\n    results.push({\n      url: cached.url,\n      title: cached.title,\n      markdown: cached.markdown,\n      fetched_at: cached.fetchedAt,\n    });\n  }\n\n  return results;\n}\n"],"mappings":"AAAA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,oBAAoB;AAC7B,SAAS,0BAA0B;AACnC,SAAS,sBAAsB,gBAAgB,oBAAoB;AACnE,SAAS,wBAAwB;AACjC,SAAS,sBAAsB;AAC/B,SAAS,oBAAoB;AAI7B,MAAM,MAAM,aAAa,OAAO;AAEhC,MAAM,uBAAuB;AAC7B,MAAM,yBAAyB;AAC/B,MAAM,0BAA0B;AAEhC,eAAsB,YAAY,OAAmB,QAA4C;AAC/F,MAAI;AACF,QAAI,MAAM,eAAe;AACvB,UAAI,KAAK,gCAAgC;AAAA,QACvC,OAAO,MAAM;AAAA,QACb,YAAY,MAAM;AAAA,QAClB,OAAO,MAAM;AAAA,MACf,CAAC;AAED,YAAM,UAAU,oBAAoB;AAAA,QAClC,OAAO,MAAM;AAAA,QACb,YAAY,MAAM;AAAA,QAClB,OAAO,MAAM;AAAA,MACf,CAAC;AAED,YAAM,UAA0B,CAAC;AACjC,iBAAW,SAAS,SAAS;AAC3B,YAAI;AACF,cAAI,CAAC,QAAQ;AACX,oBAAQ,KAAK;AAAA,cACX,KAAK,MAAM;AAAA,cACX,SAAS;AAAA,cACT,cAAc,MAAM;AAAA,cACpB,OAAO;AAAA,YACT,CAAC;AACD;AAAA,UACF;AACA,gBAAM,MAAM,MAAM,OAAO,MAAM,MAAM,KAAK,EAAE,UAAU,OAAO,CAAC;AAC9D,gBAAM,YAAY,MAAM,mBAAmB;AAC3C,gBAAM,aAAa,MAAM,UAAU,QAAQ,IAAI,MAAM,IAAI,UAAU;AAAA,YACjE,aAAa,IAAI;AAAA,UACnB,CAAC;AACD,gBAAM,eAAe,aAAa,MAAM,KAAK,WAAW,QAAQ;AAChE,kBAAQ,KAAK;AAAA,YACX,KAAK,MAAM;AAAA,YACX,SAAS,aAAa;AAAA,YACtB,cAAc,MAAM;AAAA,YACpB,GAAI,aAAa,UAAU;AAAA,cACzB,eAAe,aAAa;AAAA,cAC5B,cAAc,aAAa;AAAA,YAC7B,IAAI,CAAC;AAAA,UACP,CAAC;AAAA,QACH,SAAS,KAAK;AACZ,cAAI,KAAK,+BAA+B;AAAA,YACtC,KAAK,MAAM;AAAA,YACX,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,UACxD,CAAC;AACD,kBAAQ,KAAK;AAAA,YACX,KAAK,MAAM;AAAA,YACX,SAAS;AAAA,YACT,cAAc,MAAM;AAAA,YACpB,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,UACxD,CAAC;AAAA,QACH;AAAA,MACF;AAEA,aAAO,EAAE,QAAQ;AAAA,IACnB;AAEA,QAAI,MAAM,OAAO;AACf,UAAI,MAAM,uBAAuB;AACjC,aAAO,EAAE,OAAO,cAAc,EAAE;AAAA,IAClC;AAEA,QAAI,MAAM,OAAO;AACf,UAAI,CAAC,MAAM,SAAS,CAAC,MAAM,eAAe,CAAC,MAAM,OAAO;AACtD,eAAO,EAAE,OAAO,oEAAoE;AAAA,MACtF;AACA,UAAI,KAAK,0BAA0B;AAAA,QACjC,OAAO,MAAM;AAAA,QACb,YAAY,MAAM;AAAA,QAClB,OAAO,MAAM;AAAA,MACf,CAAC;AACD,YAAM,QAAQ,kBAAkB;AAAA,QAC9B,OAAO,MAAM;AAAA,QACb,YAAY,MAAM;AAAA,QAClB,OAAO,MAAM;AAAA,MACf,CAAC;AACD,aAAO,EAAE,SAAS,MAAM;AAAA,IAC1B;AAEA,QAAI,MAAM,SAAS,YAAY,MAAM,OAAO;AAC1C,UAAI,MAAM,uBAAuB;AAAA,QAC/B,OAAO,MAAM;AAAA,QACb,OAAO,MAAM;AAAA,MACf,CAAC;AACD,YAAMA,WAAU,MAAM,gBAAgB,KAAK;AAC3C,UAAIA,aAAY,KAAM,QAAO,EAAE,SAAAA,SAAQ;AAAA,IAEzC;AAEA,QAAI,MAAM,gBAAgB;AAAA,MACxB,OAAO,MAAM;AAAA,MACb,YAAY,MAAM;AAAA,MAClB,OAAO,MAAM;AAAA,MACb,MAAM,MAAM;AAAA,IACd,CAAC;AACD,UAAM,UAAU,oBAAoB;AAAA,MAClC,OAAO,MAAM;AAAA,MACb,YAAY,MAAM;AAAA,MAClB,OAAO,MAAM;AAAA,IACf,CAAC;AAED,WAAO;AAAA,MACL,SAAS,QAAQ,IAAI,CAAC,OAAO;AAAA,QAC3B,KAAK,EAAE;AAAA,QACP,OAAO,EAAE;AAAA,QACT,UAAU,EAAE;AAAA,QACZ,YAAY,EAAE;AAAA,MAChB,EAAE;AAAA,IACJ;AAAA,EACF,SAAS,KAAK;AACZ,QAAI,MAAM,oBAAoB,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AACpD,WAAO,EAAE,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAE;AAAA,EACnE;AACF;AASA,eAAe,gBAAgB,OAAsD;AACnF,QAAM,QAAQ,MAAM,SAAS;AAC7B,QAAM,QAAQ,KAAK,IAAI,GAAG,MAAM,SAAS,oBAAoB;AAC7D,QAAM,iBAAiB,KAAK,IAAI,wBAAwB,QAAQ,uBAAuB;AAEvF,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,KAAC,eAAe,KAAK,IAAI,MAAM,QAAQ,IAAI,CAAC,iBAAiB,GAAG,eAAe,CAAC,CAAC;AAAA,EACnF,SAAS,KAAK;AACZ,QAAI,KAAK,iEAA4D;AAAA,MACnE,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD,CAAC;AACD,WAAO;AAAA,EACT;AAEA,QAAM,YAAY,MAAM,MAAM,KAAK;AACnC,MAAI,cAAc,GAAG;AACnB,QAAI,MAAM,iDAA4C;AACtD,WAAO;AAAA,EACT;AAEA,MAAI;AACJ,MAAI;AACF,mBAAe,MAAM,cAAc,MAAM,CAAC,KAAK,CAAC;AAAA,EAClD,SAAS,KAAK;AACZ,QAAI,KAAK,uDAAkD;AAAA,MACzD,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD,CAAC;AACD,WAAO;AAAA,EACT;AACA,QAAM,cAAc,aAAa,CAAC;AAClC,MAAI,CAAC,eAAe,YAAY,WAAW,EAAG,QAAO;AAErD,QAAM,CAAC,SAAS,OAAO,IAAI,MAAM,QAAQ,IAAI;AAAA,IAC3C,QAAQ,QAAQ,gBAAgB,OAAO,cAAc,CAAC;AAAA,IACtD,MAAM,OAAO,aAAa,cAAc;AAAA,EAC1C,CAAC;AAED,QAAM,aAAa,aAAa,QAAQ,IAAI,OAAK,EAAE,GAAG,CAAC;AACvD,QAAM,aAAa,aAAa,QAAQ,IAAI,OAAK,EAAE,SAAS,GAAG,CAAC;AAEhE,MAAI,WAAW,SAAS,KAAK,WAAW,SAAS,EAAG,QAAO,CAAC;AAE5D,QAAM,QAAQ,qBAAqB,CAAC,YAAY,UAAU,GAAG,EAAE;AAC/D,QAAM,UAAU,eAAe,KAAK;AAEpC,QAAM,UAA6B,CAAC;AACpC,aAAW,CAAC,aAAa,KAAK,SAAS;AACrC,QAAI,QAAQ,UAAU,MAAO;AAC7B,UAAM,SAAS,gCAAgC,aAAa;AAC5D,QAAI,CAAC,OAAQ;AACb,YAAQ,KAAK;AAAA,MACX,KAAK,OAAO;AAAA,MACZ,OAAO,OAAO;AAAA,MACd,UAAU,OAAO;AAAA,MACjB,YAAY,OAAO;AAAA,IACrB,CAAC;AAAA,EACH;AAEA,SAAO;AACT;","names":["results"]}
+{"version":3,"sources":["../../src/tools/cache.ts"],"sourcesContent":["import {\n  searchCacheFiltered,\n  getCacheStats,\n  clearCacheEntries,\n  ftsSearchRanked,\n  getCachedContentByNormalizedUrl,\n} from '../cache/store.js';\nimport { detectChange } from '../cache/change-detector.js';\nimport { getExtractProvider } from '../providers/extract-provider.js';\nimport { reciprocalRankFusion, sortByRRFScore, buildRankMap } from '../search/rrf.js';\nimport { getEmbedProvider } from '../providers/embed-provider.js';\nimport { getVectorStore } from '../providers/vector-store.js';\nimport { createLogger } from '../logger.js';\nimport type { CacheInput, CacheOutput, CacheResultItem, ChangeReport } from '../types.js';\nimport type { SmartRouter } from '../fetch/router.js';\n\nconst log = createLogger('cache');\n\nconst DEFAULT_HYBRID_LIMIT = 20;\nconst HYBRID_CANDIDATE_FLOOR = 50;\nconst HYBRID_CANDIDATE_FACTOR = 5;\n\nexport async function handleCache(input: CacheInput, router?: SmartRouter): Promise<CacheOutput> {\n  try {\n    if (input.check_changes) {\n      log.info('Checking for content changes', {\n        query: input.query,\n        urlPattern: input.url_pattern,\n        since: input.since,\n      });\n\n      const entries = searchCacheFiltered({\n        query: input.query,\n        urlPattern: input.url_pattern,\n        since: input.since,\n      });\n\n      const changes: ChangeReport[] = [];\n      for (const entry of entries) {\n        try {\n          if (!router) {\n            changes.push({\n              url: entry.url,\n              changed: false,\n              current_hash: entry.contentHash,\n              error: 'no router available for re-fetch',\n            });\n            continue;\n          }\n          const raw = await router.fetch(entry.url, { renderJs: 'auto' });\n          const extractor = await getExtractProvider();\n          const extraction = await extractor.extract(raw.html, raw.finalUrl, {\n            contentType: raw.contentType,\n          });\n          const changeResult = detectChange(entry.url, extraction.markdown);\n          changes.push({\n            url: entry.url,\n            changed: changeResult.changed,\n            current_hash: entry.contentHash,\n            ...(changeResult.changed ? {\n              previous_hash: changeResult.previousHash,\n              diff_summary: changeResult.diffSummary,\n            } : {}),\n          });\n        } catch (err) {\n          log.warn('change check failed for URL', {\n            url: entry.url,\n            error: err instanceof Error ? err.message : String(err),\n          });\n          changes.push({\n            url: entry.url,\n            changed: false,\n            current_hash: entry.contentHash,\n            error: err instanceof Error ? err.message : String(err),\n          });\n        }\n      }\n\n      return { changes };\n    }\n\n    if (input.stats) {\n      log.debug('Cache stats requested');\n      return { stats: getCacheStats() };\n    }\n\n    if (input.clear) {\n      if (!input.query && !input.url_pattern && !input.since) {\n        return { error: 'clear requires at least one filter (query, url_pattern, or since)' };\n      }\n      log.info('Clearing cache entries', {\n        query: input.query,\n        urlPattern: input.url_pattern,\n        since: input.since,\n      });\n      const count = clearCacheEntries({\n        query: input.query,\n        urlPattern: input.url_pattern,\n        since: input.since,\n      });\n      return { cleared: count };\n    }\n\n    if (input.mode === 'hybrid' && input.query) {\n      log.debug('Cache hybrid search', {\n        query: input.query,\n        limit: input.limit,\n      });\n      const results = await runHybridSearch(input);\n      if (results !== null) return { results };\n      // fall through to FTS-only when hybrid was unavailable\n    }\n\n    log.debug('Cache search', {\n      query: input.query,\n      urlPattern: input.url_pattern,\n      since: input.since,\n      mode: input.mode,\n      limit: input.limit,\n    });\n    const results = searchCacheFiltered({\n      query: input.query,\n      urlPattern: input.url_pattern,\n      since: input.since,\n      limit: input.limit,\n    });\n\n    return {\n      results: results.map((r) => ({\n        url: r.url,\n        title: r.title,\n        markdown: r.markdown,\n        fetched_at: r.fetchedAt,\n      })),\n    };\n  } catch (err) {\n    log.error('Cache tool error', { error: String(err) });\n    return { error: err instanceof Error ? err.message : String(err) };\n  }\n}\n\n/**\n * Hybrid FTS5 + vector search fused with reciprocal rank fusion.\n *\n * Pulls `max(limit*5, 50)` candidates from each ranking, fuses with RRF\n * (k=60), then hydrates the top `limit` into cache rows. Returns `null`\n * when the vector path is unavailable so the caller falls back to FTS-only.\n */\nasync function runHybridSearch(input: CacheInput): Promise<CacheResultItem[] | null> {\n  const query = input.query ?? '';\n  const limit = Math.max(1, input.limit ?? DEFAULT_HYBRID_LIMIT);\n  const candidateLimit = Math.max(HYBRID_CANDIDATE_FLOOR, limit * HYBRID_CANDIDATE_FACTOR);\n\n  let embedProvider;\n  let store;\n  try {\n    [embedProvider, store] = await Promise.all([getEmbedProvider(), getVectorStore()]);\n  } catch (err) {\n    log.warn('hybrid search unavailable — embed/vector provider failed', {\n      error: err instanceof Error ? err.message : String(err),\n    });\n    return null;\n  }\n\n  const indexSize = await store.size();\n  if (indexSize === 0) {\n    log.debug('hybrid search skipped — vector index empty');\n    return null;\n  }\n\n  let queryVectors: Float32Array[];\n  try {\n    queryVectors = await embedProvider.embed([query]);\n  } catch (err) {\n    log.warn('hybrid search aborted — query embedding failed', {\n      error: err instanceof Error ? err.message : String(err),\n    });\n    return null;\n  }\n  const queryVector = queryVectors[0];\n  if (!queryVector || queryVector.length === 0) return null;\n\n  const [ftsHits, vecHits] = await Promise.all([\n    Promise.resolve(ftsSearchRanked(query, candidateLimit)),\n    store.search(queryVector, candidateLimit),\n  ]);\n\n  const ftsRankMap = buildRankMap(ftsHits.map(h => h.url));\n  const vecRankMap = buildRankMap(vecHits.map(h => h.metadata.url));\n\n  if (ftsRankMap.size === 0 && vecRankMap.size === 0) return [];\n\n  const fused = reciprocalRankFusion([ftsRankMap, vecRankMap], 60);\n  const ordered = sortByRRFScore(fused);\n\n  const results: CacheResultItem[] = [];\n  for (const [normalizedUrl] of ordered) {\n    if (results.length >= limit) break;\n    const cached = getCachedContentByNormalizedUrl(normalizedUrl);\n    if (!cached) continue;\n    results.push({\n      url: cached.url,\n      title: cached.title,\n      markdown: cached.markdown,\n      fetched_at: cached.fetchedAt,\n    });\n  }\n\n  return results;\n}\n"],"mappings":"AAAA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,oBAAoB;AAC7B,SAAS,0BAA0B;AACnC,SAAS,sBAAsB,gBAAgB,oBAAoB;AACnE,SAAS,wBAAwB;AACjC,SAAS,sBAAsB;AAC/B,SAAS,oBAAoB;AAI7B,MAAM,MAAM,aAAa,OAAO;AAEhC,MAAM,uBAAuB;AAC7B,MAAM,yBAAyB;AAC/B,MAAM,0BAA0B;AAEhC,eAAsB,YAAY,OAAmB,QAA4C;AAC/F,MAAI;AACF,QAAI,MAAM,eAAe;AACvB,UAAI,KAAK,gCAAgC;AAAA,QACvC,OAAO,MAAM;AAAA,QACb,YAAY,MAAM;AAAA,QAClB,OAAO,MAAM;AAAA,MACf,CAAC;AAED,YAAM,UAAU,oBAAoB;AAAA,QAClC,OAAO,MAAM;AAAA,QACb,YAAY,MAAM;AAAA,QAClB,OAAO,MAAM;AAAA,MACf,CAAC;AAED,YAAM,UAA0B,CAAC;AACjC,iBAAW,SAAS,SAAS;AAC3B,YAAI;AACF,cAAI,CAAC,QAAQ;AACX,oBAAQ,KAAK;AAAA,cACX,KAAK,MAAM;AAAA,cACX,SAAS;AAAA,cACT,cAAc,MAAM;AAAA,cACpB,OAAO;AAAA,YACT,CAAC;AACD;AAAA,UACF;AACA,gBAAM,MAAM,MAAM,OAAO,MAAM,MAAM,KAAK,EAAE,UAAU,OAAO,CAAC;AAC9D,gBAAM,YAAY,MAAM,mBAAmB;AAC3C,gBAAM,aAAa,MAAM,UAAU,QAAQ,IAAI,MAAM,IAAI,UAAU;AAAA,YACjE,aAAa,IAAI;AAAA,UACnB,CAAC;AACD,gBAAM,eAAe,aAAa,MAAM,KAAK,WAAW,QAAQ;AAChE,kBAAQ,KAAK;AAAA,YACX,KAAK,MAAM;AAAA,YACX,SAAS,aAAa;AAAA,YACtB,cAAc,MAAM;AAAA,YACpB,GAAI,aAAa,UAAU;AAAA,cACzB,eAAe,aAAa;AAAA,cAC5B,cAAc,aAAa;AAAA,YAC7B,IAAI,CAAC;AAAA,UACP,CAAC;AAAA,QACH,SAAS,KAAK;AACZ,cAAI,KAAK,+BAA+B;AAAA,YACtC,KAAK,MAAM;AAAA,YACX,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,UACxD,CAAC;AACD,kBAAQ,KAAK;AAAA,YACX,KAAK,MAAM;AAAA,YACX,SAAS;AAAA,YACT,cAAc,MAAM;AAAA,YACpB,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,UACxD,CAAC;AAAA,QACH;AAAA,MACF;AAEA,aAAO,EAAE,QAAQ;AAAA,IACnB;AAEA,QAAI,MAAM,OAAO;AACf,UAAI,MAAM,uBAAuB;AACjC,aAAO,EAAE,OAAO,cAAc,EAAE;AAAA,IAClC;AAEA,QAAI,MAAM,OAAO;AACf,UAAI,CAAC,MAAM,SAAS,CAAC,MAAM,eAAe,CAAC,MAAM,OAAO;AACtD,eAAO,EAAE,OAAO,oEAAoE;AAAA,MACtF;AACA,UAAI,KAAK,0BAA0B;AAAA,QACjC,OAAO,MAAM;AAAA,QACb,YAAY,MAAM;AAAA,QAClB,OAAO,MAAM;AAAA,MACf,CAAC;AACD,YAAM,QAAQ,kBAAkB;AAAA,QAC9B,OAAO,MAAM;AAAA,QACb,YAAY,MAAM;AAAA,QAClB,OAAO,MAAM;AAAA,MACf,CAAC;AACD,aAAO,EAAE,SAAS,MAAM;AAAA,IAC1B;AAEA,QAAI,MAAM,SAAS,YAAY,MAAM,OAAO;AAC1C,UAAI,MAAM,uBAAuB;AAAA,QAC/B,OAAO,MAAM;AAAA,QACb,OAAO,MAAM;AAAA,MACf,CAAC;AACD,YAAMA,WAAU,MAAM,gBAAgB,KAAK;AAC3C,UAAIA,aAAY,KAAM,QAAO,EAAE,SAAAA,SAAQ;AAAA,IAEzC;AAEA,QAAI,MAAM,gBAAgB;AAAA,MACxB,OAAO,MAAM;AAAA,MACb,YAAY,MAAM;AAAA,MAClB,OAAO,MAAM;AAAA,MACb,MAAM,MAAM;AAAA,MACZ,OAAO,MAAM;AAAA,IACf,CAAC;AACD,UAAM,UAAU,oBAAoB;AAAA,MAClC,OAAO,MAAM;AAAA,MACb,YAAY,MAAM;AAAA,MAClB,OAAO,MAAM;AAAA,MACb,OAAO,MAAM;AAAA,IACf,CAAC;AAED,WAAO;AAAA,MACL,SAAS,QAAQ,IAAI,CAAC,OAAO;AAAA,QAC3B,KAAK,EAAE;AAAA,QACP,OAAO,EAAE;AAAA,QACT,UAAU,EAAE;AAAA,QACZ,YAAY,EAAE;AAAA,MAChB,EAAE;AAAA,IACJ;AAAA,EACF,SAAS,KAAK;AACZ,QAAI,MAAM,oBAAoB,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AACpD,WAAO,EAAE,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG,EAAE;AAAA,EACnE;AACF;AASA,eAAe,gBAAgB,OAAsD;AACnF,QAAM,QAAQ,MAAM,SAAS;AAC7B,QAAM,QAAQ,KAAK,IAAI,GAAG,MAAM,SAAS,oBAAoB;AAC7D,QAAM,iBAAiB,KAAK,IAAI,wBAAwB,QAAQ,uBAAuB;AAEvF,MAAI;AACJ,MAAI;AACJ,MAAI;AACF,KAAC,eAAe,KAAK,IAAI,MAAM,QAAQ,IAAI,CAAC,iBAAiB,GAAG,eAAe,CAAC,CAAC;AAAA,EACnF,SAAS,KAAK;AACZ,QAAI,KAAK,iEAA4D;AAAA,MACnE,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD,CAAC;AACD,WAAO;AAAA,EACT;AAEA,QAAM,YAAY,MAAM,MAAM,KAAK;AACnC,MAAI,cAAc,GAAG;AACnB,QAAI,MAAM,iDAA4C;AACtD,WAAO;AAAA,EACT;AAEA,MAAI;AACJ,MAAI;AACF,mBAAe,MAAM,cAAc,MAAM,CAAC,KAAK,CAAC;AAAA,EAClD,SAAS,KAAK;AACZ,QAAI,KAAK,uDAAkD;AAAA,MACzD,OAAO,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAAA,IACxD,CAAC;AACD,WAAO;AAAA,EACT;AACA,QAAM,cAAc,aAAa,CAAC;AAClC,MAAI,CAAC,eAAe,YAAY,WAAW,EAAG,QAAO;AAErD,QAAM,CAAC,SAAS,OAAO,IAAI,MAAM,QAAQ,IAAI;AAAA,IAC3C,QAAQ,QAAQ,gBAAgB,OAAO,cAAc,CAAC;AAAA,IACtD,MAAM,OAAO,aAAa,cAAc;AAAA,EAC1C,CAAC;AAED,QAAM,aAAa,aAAa,QAAQ,IAAI,OAAK,EAAE,GAAG,CAAC;AACvD,QAAM,aAAa,aAAa,QAAQ,IAAI,OAAK,EAAE,SAAS,GAAG,CAAC;AAEhE,MAAI,WAAW,SAAS,KAAK,WAAW,SAAS,EAAG,QAAO,CAAC;AAE5D,QAAM,QAAQ,qBAAqB,CAAC,YAAY,UAAU,GAAG,EAAE;AAC/D,QAAM,UAAU,eAAe,KAAK;AAEpC,QAAM,UAA6B,CAAC;AACpC,aAAW,CAAC,aAAa,KAAK,SAAS;AACrC,QAAI,QAAQ,UAAU,MAAO;AAC7B,UAAM,SAAS,gCAAgC,aAAa;AAC5D,QAAI,CAAC,OAAQ;AACb,YAAQ,KAAK;AAAA,MACX,KAAK,OAAO;AAAA,MACZ,OAAO,OAAO;AAAA,MACd,UAAU,OAAO;AAAA,MACjB,YAAY,OAAO;AAAA,IACrB,CAAC;AAAA,EACH;AAEA,SAAO;AACT;","names":["results"]}

package/dist/tools/fetch.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../src/tools/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAiB,WAAW,EAAE,MAAM,aAAa,CAAC;AACvF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AA+EtD,wBAAsB,WAAW,CAC/B,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAoHnC"}
+{"version":3,"file":"fetch.d.ts","sourceRoot":"","sources":["../../src/tools/fetch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,WAAW,EAAiB,WAAW,EAAE,MAAM,aAAa,CAAC;AAEvF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AA6FtD,wBAAsB,WAAW,CAC/B,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,CAsHnC"}

package/dist/tools/fetch.js

@@ -1,3 +1,4 @@
+import { describeFetchError } from "../fetch/error-describe.js";
 import { getExtractProvider } from "../providers/extract-provider.js";
 import { getCachedContent, cacheContent, isCacheUsable } from "../cache/store.js";
 import { getConfig } from "../config.js";
@@ -11,6 +12,14 @@ import { createLogger } from "../logger.js";
 const log = createLogger("fetch");
 const DEFAULT_MAX_TOKENS_OUT = 4e3;
 const DEFAULT_FETCH_BODY_TOKENS = 16e3;
+const AUX_FIELD_CAP_WHEN_CHARS_BOUNDED = 50;
+const AUX_FIELD_CAP_WHEN_TIGHT = 20;
+function capAuxFields(out, maxContentChars) {
+  if (maxContentChars === void 0) return;
+  const cap = maxContentChars <= 4e3 ? AUX_FIELD_CAP_WHEN_TIGHT : AUX_FIELD_CAP_WHEN_CHARS_BOUNDED;
+  if (out.links && out.links.length > cap) out.links = out.links.slice(0, cap);
+  if (out.images && out.images.length > cap) out.images = out.images.slice(0, cap);
+}
 async function attachEvidence(output, input, markdown) {
   if (!markdown) return;
   const includeFull = input.include_full_markdown ?? true;
@@ -46,7 +55,7 @@ function formatCachedResponse(cached, input) {
   if (input.max_content_chars !== void 0) {
     markdown = truncateSmartly(markdown, input.max_content_chars);
   }
-  return {
+  const out = {
     url: cached.url,
     title: cached.title,
     markdown,
@@ -59,6 +68,8 @@ function formatCachedResponse(cached, input) {
     cached: true,
     cached_at: cached.fetchedAt
   };
+  capAuxFields(out, input.max_content_chars);
+  return out;
 }
 async function handleFetch(input, router) {
   const mode = resolveMode(input.mode);
@@ -150,16 +161,18 @@ async function handleFetch(input, router) {
         diff_summary: changeResult.diffSummary
       } : {}
     };
+    capAuxFields(out, input.max_content_chars);
     await attachEvidence(out, input, finalMarkdown);
     return { ok: true, data: out };
   } catch (err) {
     log.error("Fetch failed", { url: input.url, error: String(err) });
-    const msg = err instanceof Error ? err.message : String(err);
+    const described = describeFetchError(err);
     return {
       ok: false,
       error: "fetch_failed",
-      error_reason: msg,
-      stage: "fetch"
+      error_reason: described.reason,
+      stage: "fetch",
+      ...described.hint ? { hint: described.hint } : {}
     };
   }
 }

package/dist/tools/fetch.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/tools/fetch.ts"],"sourcesContent":["import type { FetchInput, FetchOutput, CachedContent, StageResult } from '../types.js';\nimport type { SmartRouter } from '../fetch/router.js';\nimport { getExtractProvider } from '../providers/extract-provider.js';\nimport { getCachedContent, cacheContent, isCacheUsable } from '../cache/store.js';\nimport { getConfig } from '../config.js';\nimport { extractSection } from '../extraction/markdown.js';\nimport { detectChange } from '../cache/change-detector.js';\nimport { getEmbeddingService } from '../embedding/embed.js';\nimport { truncateSmartly, applyOutputBudget } from '../search/truncate.js';\nimport { buildEvidenceFromMarkdown } from '../search/evidence.js';\nimport { resolveMode } from '../util/mode.js';\nimport { createLogger } from '../logger.js';\n\nconst log = createLogger('fetch');\n\nconst DEFAULT_MAX_TOKENS_OUT = 4000;\n// Fetch is single-URL — users explicitly want the body. Keep a generous cap\n// that fits typical MCP tool-result limits (~25k tokens) but prevents huge\n// pages (full doc sites) from blowing the cap. Override via max_tokens_out.\nconst DEFAULT_FETCH_BODY_TOKENS = 16000;\n\nasync function attachEvidence(\n  output: FetchOutput,\n  input: FetchInput,\n  markdown: string,\n): Promise<void> {\n  if (!markdown) return;\n  const includeFull = input.include_full_markdown ?? true;\n  const maxTokensOut = input.max_tokens_out ?? DEFAULT_MAX_TOKENS_OUT;\n  const evidence = await buildEvidenceFromMarkdown(\n    output.title || output.url,\n    output.title,\n    output.url,\n    markdown,\n    { maxTokensOut, maxItems: 1 },\n  );\n  if (evidence.length > 0) output.evidence = evidence;\n  if (!includeFull) {\n    output.markdown = '';\n  } else if (output.markdown) {\n    output.markdown = applyOutputBudget(output.markdown, {\n      maxTokensOut: input.max_tokens_out ?? DEFAULT_FETCH_BODY_TOKENS,\n      maxChars: input.max_chars,\n    });\n  }\n}\n\nfunction formatCachedResponse(cached: CachedContent, input: FetchInput): FetchOutput {\n  let markdown = cached.markdown;\n  let sectionMatched: boolean | undefined;\n\n  if (input.section) {\n    const result = extractSection(markdown, input.section, input.section_index);\n    markdown = result.content;\n    sectionMatched = result.matched;\n  }\n\n  if (input.max_chars && markdown.length > input.max_chars) {\n    markdown = markdown.slice(0, input.max_chars);\n  }\n\n  if (input.max_content_chars !== undefined) {\n    markdown = truncateSmartly(markdown, input.max_content_chars);\n  }\n\n  return {\n    url: cached.url,\n    title: cached.title,\n    markdown,\n    metadata: {\n      ...JSON.parse(cached.metadata || '{}'),\n      ...(sectionMatched !== undefined ? { section_matched: sectionMatched } : {}),\n    },\n    links: JSON.parse(cached.links || '[]'),\n    images: JSON.parse(cached.images || '[]'),\n    cached: true,\n    cached_at: cached.fetchedAt,\n  };\n}\n\nexport async function handleFetch(\n  input: FetchInput,\n  router: SmartRouter,\n): Promise<StageResult<FetchOutput>> {\n  const mode = resolveMode(input.mode);\n  try {\n    if (!input.force_refresh) {\n      const cached = getCachedContent(input.url);\n      if (cached && (!input.actions || input.actions.length === 0)) {\n        const staleMaxSeconds = mode === 'cache' ? getConfig().fastStaleMaxHours * 3600 : 0;\n        const { usable, stale } = isCacheUsable(cached, { staleMaxSeconds });\n        if (usable) {\n          log.info('Serving from cache', { url: input.url, stale });\n          const out = formatCachedResponse(cached, input);\n          if (stale) out.stale = true;\n          const fullMarkdown = out.markdown;\n          await attachEvidence(out, input, fullMarkdown);\n          return { ok: true, data: out };\n        }\n      }\n    }\n\n    if (mode === 'cache') {\n      return {\n        ok: false,\n        error: 'cache_miss',\n        error_reason: `URL not in cache: ${input.url}`,\n        stage: 'fetch',\n        hint: 'Use mode:default to fetch live, or run search/crawl first to populate cache',\n      };\n    }\n\n    const raw = await router.fetch(input.url, {\n      renderJs: input.render_js ?? 'auto',\n      useAuth: input.use_auth ?? false,\n      headers: input.headers,\n      screenshot: input.screenshot,\n      actions: input.actions,\n      mode,\n    });\n\n    // T11: stealth mode can return a StageError (e.g., playwright_not_installed,\n    // playwright_fetch_failed). Surface it directly.\n    if ('error' in raw && typeof (raw as { error?: unknown }).error === 'string') {\n      const stageErr = raw as unknown as { error: string; error_reason?: string; stage?: string; hint?: string };\n      return {\n        ok: false,\n        error: stageErr.error,\n        error_reason: stageErr.error_reason ?? stageErr.error,\n        stage: stageErr.stage ?? 'fetch',\n        ...(stageErr.hint ? { hint: stageErr.hint } : {}),\n      };\n    }\n\n    const extractor = await getExtractProvider();\n    const extraction = await extractor.extract(raw.html, raw.finalUrl, {\n      maxChars: input.max_chars,\n      section: input.section,\n      sectionIndex: input.section_index,\n      contentType: raw.contentType,\n      pdfBuffer: raw.rawBuffer,\n    });\n\n    let changeResult: { changed: boolean; previousHash?: string; diffSummary?: string } | undefined;\n    try {\n      changeResult = detectChange(raw.finalUrl, extraction.markdown);\n    } catch (err) {\n      log.warn('change detection failed', { url: raw.finalUrl, error: String(err) });\n    }\n\n    try {\n      cacheContent(raw, extraction);\n    } catch (err) {\n      log.warn('failed to cache fetched content', { url: raw.finalUrl, error: String(err) });\n    }\n\n    try {\n      const embeddingService = getEmbeddingService();\n      if (embeddingService.isAvailable()) {\n        embeddingService.embedAsync(raw.finalUrl, extraction.markdown);\n      }\n    } catch (err) {\n      log.debug('embedding hook skipped', { error: String(err) });\n    }\n\n    const finalMarkdown = input.max_content_chars !== undefined\n      ? truncateSmartly(extraction.markdown, input.max_content_chars)\n      : extraction.markdown;\n\n    const out: FetchOutput = {\n      url: raw.finalUrl,\n      title: extraction.title,\n      markdown: finalMarkdown,\n      metadata: extraction.metadata,\n      links: extraction.links,\n      images: extraction.images,\n      screenshot: raw.screenshot,\n      cached: false,\n      action_results: raw.actionResults,\n      ...(raw.jsRequired ? { js_required: true } : {}),\n      ...(changeResult?.changed ? {\n        changed: true,\n        previous_hash: changeResult.previousHash,\n        diff_summary: changeResult.diffSummary,\n      } : {}),\n    };\n\n    await attachEvidence(out, input, finalMarkdown);\n    return { ok: true, data: out };\n  } catch (err) {\n    log.error('Fetch failed', { url: input.url, error: String(err) });\n    const msg = err instanceof Error ? err.message : String(err);\n    return {\n      ok: false,\n      error: 'fetch_failed',\n      error_reason: msg,\n      stage: 'fetch',\n    };\n  }\n}\n"],"mappings":"AAEA,SAAS,0BAA0B;AACnC,SAAS,kBAAkB,cAAc,qBAAqB;AAC9D,SAAS,iBAAiB;AAC1B,SAAS,sBAAsB;AAC/B,SAAS,oBAAoB;AAC7B,SAAS,2BAA2B;AACpC,SAAS,iBAAiB,yBAAyB;AACnD,SAAS,iCAAiC;AAC1C,SAAS,mBAAmB;AAC5B,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,OAAO;AAEhC,MAAM,yBAAyB;AAI/B,MAAM,4BAA4B;AAElC,eAAe,eACb,QACA,OACA,UACe;AACf,MAAI,CAAC,SAAU;AACf,QAAM,cAAc,MAAM,yBAAyB;AACnD,QAAM,eAAe,MAAM,kBAAkB;AAC7C,QAAM,WAAW,MAAM;AAAA,IACrB,OAAO,SAAS,OAAO;AAAA,IACvB,OAAO;AAAA,IACP,OAAO;AAAA,IACP;AAAA,IACA,EAAE,cAAc,UAAU,EAAE;AAAA,EAC9B;AACA,MAAI,SAAS,SAAS,EAAG,QAAO,WAAW;AAC3C,MAAI,CAAC,aAAa;AAChB,WAAO,WAAW;AAAA,EACpB,WAAW,OAAO,UAAU;AAC1B,WAAO,WAAW,kBAAkB,OAAO,UAAU;AAAA,MACnD,cAAc,MAAM,kBAAkB;AAAA,MACtC,UAAU,MAAM;AAAA,IAClB,CAAC;AAAA,EACH;AACF;AAEA,SAAS,qBAAqB,QAAuB,OAAgC;AACnF,MAAI,WAAW,OAAO;AACtB,MAAI;AAEJ,MAAI,MAAM,SAAS;AACjB,UAAM,SAAS,eAAe,UAAU,MAAM,SAAS,MAAM,aAAa;AAC1E,eAAW,OAAO;AAClB,qBAAiB,OAAO;AAAA,EAC1B;AAEA,MAAI,MAAM,aAAa,SAAS,SAAS,MAAM,WAAW;AACxD,eAAW,SAAS,MAAM,GAAG,MAAM,SAAS;AAAA,EAC9C;AAEA,MAAI,MAAM,sBAAsB,QAAW;AACzC,eAAW,gBAAgB,UAAU,MAAM,iBAAiB;AAAA,EAC9D;AAEA,SAAO;AAAA,IACL,KAAK,OAAO;AAAA,IACZ,OAAO,OAAO;AAAA,IACd;AAAA,IACA,UAAU;AAAA,MACR,GAAG,KAAK,MAAM,OAAO,YAAY,IAAI;AAAA,MACrC,GAAI,mBAAmB,SAAY,EAAE,iBAAiB,eAAe,IAAI,CAAC;AAAA,IAC5E;AAAA,IACA,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI;AAAA,IACtC,QAAQ,KAAK,MAAM,OAAO,UAAU,IAAI;AAAA,IACxC,QAAQ;AAAA,IACR,WAAW,OAAO;AAAA,EACpB;AACF;AAEA,eAAsB,YACpB,OACA,QACmC;AACnC,QAAM,OAAO,YAAY,MAAM,IAAI;AACnC,MAAI;AACF,QAAI,CAAC,MAAM,eAAe;AACxB,YAAM,SAAS,iBAAiB,MAAM,GAAG;AACzC,UAAI,WAAW,CAAC,MAAM,WAAW,MAAM,QAAQ,WAAW,IAAI;AAC5D,cAAM,kBAAkB,SAAS,UAAU,UAAU,EAAE,oBAAoB,OAAO;AAClF,cAAM,EAAE,QAAQ,MAAM,IAAI,cAAc,QAAQ,EAAE,gBAAgB,CAAC;AACnE,YAAI,QAAQ;AACV,cAAI,KAAK,sBAAsB,EAAE,KAAK,MAAM,KAAK,MAAM,CAAC;AACxD,gBAAMA,OAAM,qBAAqB,QAAQ,KAAK;AAC9C,cAAI,MAAO,CAAAA,KAAI,QAAQ;AACvB,gBAAM,eAAeA,KAAI;AACzB,gBAAM,eAAeA,MAAK,OAAO,YAAY;AAC7C,iBAAO,EAAE,IAAI,MAAM,MAAMA,KAAI;AAAA,QAC/B;AAAA,MACF;AAAA,IACF;AAEA,QAAI,SAAS,SAAS;AACpB,aAAO;AAAA,QACL,IAAI;AAAA,QACJ,OAAO;AAAA,QACP,cAAc,qBAAqB,MAAM,GAAG;AAAA,QAC5C,OAAO;AAAA,QACP,MAAM;AAAA,MACR;AAAA,IACF;AAEA,UAAM,MAAM,MAAM,OAAO,MAAM,MAAM,KAAK;AAAA,MACxC,UAAU,MAAM,aAAa;AAAA,MAC7B,SAAS,MAAM,YAAY;AAAA,MAC3B,SAAS,MAAM;AAAA,MACf,YAAY,MAAM;AAAA,MAClB,SAAS,MAAM;AAAA,MACf;AAAA,IACF,CAAC;AAID,QAAI,WAAW,OAAO,OAAQ,IAA4B,UAAU,UAAU;AAC5E,YAAM,WAAW;AACjB,aAAO;AAAA,QACL,IAAI;AAAA,QACJ,OAAO,SAAS;AAAA,QAChB,cAAc,SAAS,gBAAgB,SAAS;AAAA,QAChD,OAAO,SAAS,SAAS;AAAA,QACzB,GAAI,SAAS,OAAO,EAAE,MAAM,SAAS,KAAK,IAAI,CAAC;AAAA,MACjD;AAAA,IACF;AAEA,UAAM,YAAY,MAAM,mBAAmB;AAC3C,UAAM,aAAa,MAAM,UAAU,QAAQ,IAAI,MAAM,IAAI,UAAU;AAAA,MACjE,UAAU,MAAM;AAAA,MAChB,SAAS,MAAM;AAAA,MACf,cAAc,MAAM;AAAA,MACpB,aAAa,IAAI;AAAA,MACjB,WAAW,IAAI;AAAA,IACjB,CAAC;AAED,QAAI;AACJ,QAAI;AACF,qBAAe,aAAa,IAAI,UAAU,WAAW,QAAQ;AAAA,IAC/D,SAAS,KAAK;AACZ,UAAI,KAAK,2BAA2B,EAAE,KAAK,IAAI,UAAU,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,IAC/E;AAEA,QAAI;AACF,mBAAa,KAAK,UAAU;AAAA,IAC9B,SAAS,KAAK;AACZ,UAAI,KAAK,mCAAmC,EAAE,KAAK,IAAI,UAAU,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,IACvF;AAEA,QAAI;AACF,YAAM,mBAAmB,oBAAoB;AAC7C,UAAI,iBAAiB,YAAY,GAAG;AAClC,yBAAiB,WAAW,IAAI,UAAU,WAAW,QAAQ;AAAA,MAC/D;AAAA,IACF,SAAS,KAAK;AACZ,UAAI,MAAM,0BAA0B,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,IAC5D;AAEA,UAAM,gBAAgB,MAAM,sBAAsB,SAC9C,gBAAgB,WAAW,UAAU,MAAM,iBAAiB,IAC5D,WAAW;AAEf,UAAM,MAAmB;AAAA,MACvB,KAAK,IAAI;AAAA,MACT,OAAO,WAAW;AAAA,MAClB,UAAU;AAAA,MACV,UAAU,WAAW;AAAA,MACrB,OAAO,WAAW;AAAA,MAClB,QAAQ,WAAW;AAAA,MACnB,YAAY,IAAI;AAAA,MAChB,QAAQ;AAAA,MACR,gBAAgB,IAAI;AAAA,MACpB,GAAI,IAAI,aAAa,EAAE,aAAa,KAAK,IAAI,CAAC;AAAA,MAC9C,GAAI,cAAc,UAAU;AAAA,QAC1B,SAAS;AAAA,QACT,eAAe,aAAa;AAAA,QAC5B,cAAc,aAAa;AAAA,MAC7B,IAAI,CAAC;AAAA,IACP;AAEA,UAAM,eAAe,KAAK,OAAO,aAAa;AAC9C,WAAO,EAAE,IAAI,MAAM,MAAM,IAAI;AAAA,EAC/B,SAAS,KAAK;AACZ,QAAI,MAAM,gBAAgB,EAAE,KAAK,MAAM,KAAK,OAAO,OAAO,GAAG,EAAE,CAAC;AAChE,UAAM,MAAM,eAAe,QAAQ,IAAI,UAAU,OAAO,GAAG;AAC3D,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,OAAO;AAAA,MACP,cAAc;AAAA,MACd,OAAO;AAAA,IACT;AAAA,EACF;AACF;","names":["out"]}
+{"version":3,"sources":["../../src/tools/fetch.ts"],"sourcesContent":["import type { FetchInput, FetchOutput, CachedContent, StageResult } from '../types.js';\nimport { describeFetchError } from '../fetch/error-describe.js';\nimport type { SmartRouter } from '../fetch/router.js';\nimport { getExtractProvider } from '../providers/extract-provider.js';\nimport { getCachedContent, cacheContent, isCacheUsable } from '../cache/store.js';\nimport { getConfig } from '../config.js';\nimport { extractSection } from '../extraction/markdown.js';\nimport { detectChange } from '../cache/change-detector.js';\nimport { getEmbeddingService } from '../embedding/embed.js';\nimport { truncateSmartly, applyOutputBudget } from '../search/truncate.js';\nimport { buildEvidenceFromMarkdown } from '../search/evidence.js';\nimport { resolveMode } from '../util/mode.js';\nimport { createLogger } from '../logger.js';\n\nconst log = createLogger('fetch');\n\nconst DEFAULT_MAX_TOKENS_OUT = 4000;\n// Fetch is single-URL — users explicitly want the body. Keep a generous cap\n// that fits typical MCP tool-result limits (~25k tokens) but prevents huge\n// pages (full doc sites) from blowing the cap. Override via max_tokens_out.\nconst DEFAULT_FETCH_BODY_TOKENS = 16000;\n// When the caller asks for a tight markdown budget, also clip the\n// auxiliary arrays — large doc pages emit thousands of links/images that\n// otherwise blow the user-requested response size.\nconst AUX_FIELD_CAP_WHEN_CHARS_BOUNDED = 50;\nconst AUX_FIELD_CAP_WHEN_TIGHT = 20;\n\nfunction capAuxFields(out: FetchOutput, maxContentChars?: number): void {\n  if (maxContentChars === undefined) return;\n  const cap = maxContentChars <= 4000 ? AUX_FIELD_CAP_WHEN_TIGHT : AUX_FIELD_CAP_WHEN_CHARS_BOUNDED;\n  if (out.links && out.links.length > cap) out.links = out.links.slice(0, cap);\n  if (out.images && out.images.length > cap) out.images = out.images.slice(0, cap);\n}\n\nasync function attachEvidence(\n  output: FetchOutput,\n  input: FetchInput,\n  markdown: string,\n): Promise<void> {\n  if (!markdown) return;\n  const includeFull = input.include_full_markdown ?? true;\n  const maxTokensOut = input.max_tokens_out ?? DEFAULT_MAX_TOKENS_OUT;\n  const evidence = await buildEvidenceFromMarkdown(\n    output.title || output.url,\n    output.title,\n    output.url,\n    markdown,\n    { maxTokensOut, maxItems: 1 },\n  );\n  if (evidence.length > 0) output.evidence = evidence;\n  if (!includeFull) {\n    output.markdown = '';\n  } else if (output.markdown) {\n    output.markdown = applyOutputBudget(output.markdown, {\n      maxTokensOut: input.max_tokens_out ?? DEFAULT_FETCH_BODY_TOKENS,\n      maxChars: input.max_chars,\n    });\n  }\n}\n\nfunction formatCachedResponse(cached: CachedContent, input: FetchInput): FetchOutput {\n  let markdown = cached.markdown;\n  let sectionMatched: boolean | undefined;\n\n  if (input.section) {\n    const result = extractSection(markdown, input.section, input.section_index);\n    markdown = result.content;\n    sectionMatched = result.matched;\n  }\n\n  if (input.max_chars && markdown.length > input.max_chars) {\n    markdown = markdown.slice(0, input.max_chars);\n  }\n\n  if (input.max_content_chars !== undefined) {\n    markdown = truncateSmartly(markdown, input.max_content_chars);\n  }\n\n  const out: FetchOutput = {\n    url: cached.url,\n    title: cached.title,\n    markdown,\n    metadata: {\n      ...JSON.parse(cached.metadata || '{}'),\n      ...(sectionMatched !== undefined ? { section_matched: sectionMatched } : {}),\n    },\n    links: JSON.parse(cached.links || '[]'),\n    images: JSON.parse(cached.images || '[]'),\n    cached: true,\n    cached_at: cached.fetchedAt,\n  };\n  capAuxFields(out, input.max_content_chars);\n  return out;\n}\n\nexport async function handleFetch(\n  input: FetchInput,\n  router: SmartRouter,\n): Promise<StageResult<FetchOutput>> {\n  const mode = resolveMode(input.mode);\n  try {\n    if (!input.force_refresh) {\n      const cached = getCachedContent(input.url);\n      if (cached && (!input.actions || input.actions.length === 0)) {\n        const staleMaxSeconds = mode === 'cache' ? getConfig().fastStaleMaxHours * 3600 : 0;\n        const { usable, stale } = isCacheUsable(cached, { staleMaxSeconds });\n        if (usable) {\n          log.info('Serving from cache', { url: input.url, stale });\n          const out = formatCachedResponse(cached, input);\n          if (stale) out.stale = true;\n          const fullMarkdown = out.markdown;\n          await attachEvidence(out, input, fullMarkdown);\n          return { ok: true, data: out };\n        }\n      }\n    }\n\n    if (mode === 'cache') {\n      return {\n        ok: false,\n        error: 'cache_miss',\n        error_reason: `URL not in cache: ${input.url}`,\n        stage: 'fetch',\n        hint: 'Use mode:default to fetch live, or run search/crawl first to populate cache',\n      };\n    }\n\n    const raw = await router.fetch(input.url, {\n      renderJs: input.render_js ?? 'auto',\n      useAuth: input.use_auth ?? false,\n      headers: input.headers,\n      screenshot: input.screenshot,\n      actions: input.actions,\n      mode,\n    });\n\n    // T11: stealth mode can return a StageError (e.g., playwright_not_installed,\n    // playwright_fetch_failed). Surface it directly.\n    if ('error' in raw && typeof (raw as { error?: unknown }).error === 'string') {\n      const stageErr = raw as unknown as { error: string; error_reason?: string; stage?: string; hint?: string };\n      return {\n        ok: false,\n        error: stageErr.error,\n        error_reason: stageErr.error_reason ?? stageErr.error,\n        stage: stageErr.stage ?? 'fetch',\n        ...(stageErr.hint ? { hint: stageErr.hint } : {}),\n      };\n    }\n\n    const extractor = await getExtractProvider();\n    const extraction = await extractor.extract(raw.html, raw.finalUrl, {\n      maxChars: input.max_chars,\n      section: input.section,\n      sectionIndex: input.section_index,\n      contentType: raw.contentType,\n      pdfBuffer: raw.rawBuffer,\n    });\n\n    let changeResult: { changed: boolean; previousHash?: string; diffSummary?: string } | undefined;\n    try {\n      changeResult = detectChange(raw.finalUrl, extraction.markdown);\n    } catch (err) {\n      log.warn('change detection failed', { url: raw.finalUrl, error: String(err) });\n    }\n\n    try {\n      cacheContent(raw, extraction);\n    } catch (err) {\n      log.warn('failed to cache fetched content', { url: raw.finalUrl, error: String(err) });\n    }\n\n    try {\n      const embeddingService = getEmbeddingService();\n      if (embeddingService.isAvailable()) {\n        embeddingService.embedAsync(raw.finalUrl, extraction.markdown);\n      }\n    } catch (err) {\n      log.debug('embedding hook skipped', { error: String(err) });\n    }\n\n    const finalMarkdown = input.max_content_chars !== undefined\n      ? truncateSmartly(extraction.markdown, input.max_content_chars)\n      : extraction.markdown;\n\n    const out: FetchOutput = {\n      url: raw.finalUrl,\n      title: extraction.title,\n      markdown: finalMarkdown,\n      metadata: extraction.metadata,\n      links: extraction.links,\n      images: extraction.images,\n      screenshot: raw.screenshot,\n      cached: false,\n      action_results: raw.actionResults,\n      ...(raw.jsRequired ? { js_required: true } : {}),\n      ...(changeResult?.changed ? {\n        changed: true,\n        previous_hash: changeResult.previousHash,\n        diff_summary: changeResult.diffSummary,\n      } : {}),\n    };\n\n    capAuxFields(out, input.max_content_chars);\n    await attachEvidence(out, input, finalMarkdown);\n    return { ok: true, data: out };\n  } catch (err) {\n    log.error('Fetch failed', { url: input.url, error: String(err) });\n    const described = describeFetchError(err);\n    return {\n      ok: false,\n      error: 'fetch_failed',\n      error_reason: described.reason,\n      stage: 'fetch',\n      ...(described.hint ? { hint: described.hint } : {}),\n    };\n  }\n}\n"],"mappings":"AACA,SAAS,0BAA0B;AAEnC,SAAS,0BAA0B;AACnC,SAAS,kBAAkB,cAAc,qBAAqB;AAC9D,SAAS,iBAAiB;AAC1B,SAAS,sBAAsB;AAC/B,SAAS,oBAAoB;AAC7B,SAAS,2BAA2B;AACpC,SAAS,iBAAiB,yBAAyB;AACnD,SAAS,iCAAiC;AAC1C,SAAS,mBAAmB;AAC5B,SAAS,oBAAoB;AAE7B,MAAM,MAAM,aAAa,OAAO;AAEhC,MAAM,yBAAyB;AAI/B,MAAM,4BAA4B;AAIlC,MAAM,mCAAmC;AACzC,MAAM,2BAA2B;AAEjC,SAAS,aAAa,KAAkB,iBAAgC;AACtE,MAAI,oBAAoB,OAAW;AACnC,QAAM,MAAM,mBAAmB,MAAO,2BAA2B;AACjE,MAAI,IAAI,SAAS,IAAI,MAAM,SAAS,IAAK,KAAI,QAAQ,IAAI,MAAM,MAAM,GAAG,GAAG;AAC3E,MAAI,IAAI,UAAU,IAAI,OAAO,SAAS,IAAK,KAAI,SAAS,IAAI,OAAO,MAAM,GAAG,GAAG;AACjF;AAEA,eAAe,eACb,QACA,OACA,UACe;AACf,MAAI,CAAC,SAAU;AACf,QAAM,cAAc,MAAM,yBAAyB;AACnD,QAAM,eAAe,MAAM,kBAAkB;AAC7C,QAAM,WAAW,MAAM;AAAA,IACrB,OAAO,SAAS,OAAO;AAAA,IACvB,OAAO;AAAA,IACP,OAAO;AAAA,IACP;AAAA,IACA,EAAE,cAAc,UAAU,EAAE;AAAA,EAC9B;AACA,MAAI,SAAS,SAAS,EAAG,QAAO,WAAW;AAC3C,MAAI,CAAC,aAAa;AAChB,WAAO,WAAW;AAAA,EACpB,WAAW,OAAO,UAAU;AAC1B,WAAO,WAAW,kBAAkB,OAAO,UAAU;AAAA,MACnD,cAAc,MAAM,kBAAkB;AAAA,MACtC,UAAU,MAAM;AAAA,IAClB,CAAC;AAAA,EACH;AACF;AAEA,SAAS,qBAAqB,QAAuB,OAAgC;AACnF,MAAI,WAAW,OAAO;AACtB,MAAI;AAEJ,MAAI,MAAM,SAAS;AACjB,UAAM,SAAS,eAAe,UAAU,MAAM,SAAS,MAAM,aAAa;AAC1E,eAAW,OAAO;AAClB,qBAAiB,OAAO;AAAA,EAC1B;AAEA,MAAI,MAAM,aAAa,SAAS,SAAS,MAAM,WAAW;AACxD,eAAW,SAAS,MAAM,GAAG,MAAM,SAAS;AAAA,EAC9C;AAEA,MAAI,MAAM,sBAAsB,QAAW;AACzC,eAAW,gBAAgB,UAAU,MAAM,iBAAiB;AAAA,EAC9D;AAEA,QAAM,MAAmB;AAAA,IACvB,KAAK,OAAO;AAAA,IACZ,OAAO,OAAO;AAAA,IACd;AAAA,IACA,UAAU;AAAA,MACR,GAAG,KAAK,MAAM,OAAO,YAAY,IAAI;AAAA,MACrC,GAAI,mBAAmB,SAAY,EAAE,iBAAiB,eAAe,IAAI,CAAC;AAAA,IAC5E;AAAA,IACA,OAAO,KAAK,MAAM,OAAO,SAAS,IAAI;AAAA,IACtC,QAAQ,KAAK,MAAM,OAAO,UAAU,IAAI;AAAA,IACxC,QAAQ;AAAA,IACR,WAAW,OAAO;AAAA,EACpB;AACA,eAAa,KAAK,MAAM,iBAAiB;AACzC,SAAO;AACT;AAEA,eAAsB,YACpB,OACA,QACmC;AACnC,QAAM,OAAO,YAAY,MAAM,IAAI;AACnC,MAAI;AACF,QAAI,CAAC,MAAM,eAAe;AACxB,YAAM,SAAS,iBAAiB,MAAM,GAAG;AACzC,UAAI,WAAW,CAAC,MAAM,WAAW,MAAM,QAAQ,WAAW,IAAI;AAC5D,cAAM,kBAAkB,SAAS,UAAU,UAAU,EAAE,oBAAoB,OAAO;AAClF,cAAM,EAAE,QAAQ,MAAM,IAAI,cAAc,QAAQ,EAAE,gBAAgB,CAAC;AACnE,YAAI,QAAQ;AACV,cAAI,KAAK,sBAAsB,EAAE,KAAK,MAAM,KAAK,MAAM,CAAC;AACxD,gBAAMA,OAAM,qBAAqB,QAAQ,KAAK;AAC9C,cAAI,MAAO,CAAAA,KAAI,QAAQ;AACvB,gBAAM,eAAeA,KAAI;AACzB,gBAAM,eAAeA,MAAK,OAAO,YAAY;AAC7C,iBAAO,EAAE,IAAI,MAAM,MAAMA,KAAI;AAAA,QAC/B;AAAA,MACF;AAAA,IACF;AAEA,QAAI,SAAS,SAAS;AACpB,aAAO;AAAA,QACL,IAAI;AAAA,QACJ,OAAO;AAAA,QACP,cAAc,qBAAqB,MAAM,GAAG;AAAA,QAC5C,OAAO;AAAA,QACP,MAAM;AAAA,MACR;AAAA,IACF;AAEA,UAAM,MAAM,MAAM,OAAO,MAAM,MAAM,KAAK;AAAA,MACxC,UAAU,MAAM,aAAa;AAAA,MAC7B,SAAS,MAAM,YAAY;AAAA,MAC3B,SAAS,MAAM;AAAA,MACf,YAAY,MAAM;AAAA,MAClB,SAAS,MAAM;AAAA,MACf;AAAA,IACF,CAAC;AAID,QAAI,WAAW,OAAO,OAAQ,IAA4B,UAAU,UAAU;AAC5E,YAAM,WAAW;AACjB,aAAO;AAAA,QACL,IAAI;AAAA,QACJ,OAAO,SAAS;AAAA,QAChB,cAAc,SAAS,gBAAgB,SAAS;AAAA,QAChD,OAAO,SAAS,SAAS;AAAA,QACzB,GAAI,SAAS,OAAO,EAAE,MAAM,SAAS,KAAK,IAAI,CAAC;AAAA,MACjD;AAAA,IACF;AAEA,UAAM,YAAY,MAAM,mBAAmB;AAC3C,UAAM,aAAa,MAAM,UAAU,QAAQ,IAAI,MAAM,IAAI,UAAU;AAAA,MACjE,UAAU,MAAM;AAAA,MAChB,SAAS,MAAM;AAAA,MACf,cAAc,MAAM;AAAA,MACpB,aAAa,IAAI;AAAA,MACjB,WAAW,IAAI;AAAA,IACjB,CAAC;AAED,QAAI;AACJ,QAAI;AACF,qBAAe,aAAa,IAAI,UAAU,WAAW,QAAQ;AAAA,IAC/D,SAAS,KAAK;AACZ,UAAI,KAAK,2BAA2B,EAAE,KAAK,IAAI,UAAU,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,IAC/E;AAEA,QAAI;AACF,mBAAa,KAAK,UAAU;AAAA,IAC9B,SAAS,KAAK;AACZ,UAAI,KAAK,mCAAmC,EAAE,KAAK,IAAI,UAAU,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,IACvF;AAEA,QAAI;AACF,YAAM,mBAAmB,oBAAoB;AAC7C,UAAI,iBAAiB,YAAY,GAAG;AAClC,yBAAiB,WAAW,IAAI,UAAU,WAAW,QAAQ;AAAA,MAC/D;AAAA,IACF,SAAS,KAAK;AACZ,UAAI,MAAM,0BAA0B,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,IAC5D;AAEA,UAAM,gBAAgB,MAAM,sBAAsB,SAC9C,gBAAgB,WAAW,UAAU,MAAM,iBAAiB,IAC5D,WAAW;AAEf,UAAM,MAAmB;AAAA,MACvB,KAAK,IAAI;AAAA,MACT,OAAO,WAAW;AAAA,MAClB,UAAU;AAAA,MACV,UAAU,WAAW;AAAA,MACrB,OAAO,WAAW;AAAA,MAClB,QAAQ,WAAW;AAAA,MACnB,YAAY,IAAI;AAAA,MAChB,QAAQ;AAAA,MACR,gBAAgB,IAAI;AAAA,MACpB,GAAI,IAAI,aAAa,EAAE,aAAa,KAAK,IAAI,CAAC;AAAA,MAC9C,GAAI,cAAc,UAAU;AAAA,QAC1B,SAAS;AAAA,QACT,eAAe,aAAa;AAAA,QAC5B,cAAc,aAAa;AAAA,MAC7B,IAAI,CAAC;AAAA,IACP;AAEA,iBAAa,KAAK,MAAM,iBAAiB;AACzC,UAAM,eAAe,KAAK,OAAO,aAAa;AAC9C,WAAO,EAAE,IAAI,MAAM,MAAM,IAAI;AAAA,EAC/B,SAAS,KAAK;AACZ,QAAI,MAAM,gBAAgB,EAAE,KAAK,MAAM,KAAK,OAAO,OAAO,GAAG,EAAE,CAAC;AAChE,UAAM,YAAY,mBAAmB,GAAG;AACxC,WAAO;AAAA,MACL,IAAI;AAAA,MACJ,OAAO;AAAA,MACP,cAAc,UAAU;AAAA,MACxB,OAAO;AAAA,MACP,GAAI,UAAU,OAAO,EAAE,MAAM,UAAU,KAAK,IAAI,CAAC;AAAA,IACnD;AAAA,EACF;AACF;","names":["out"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@staticn0va/wigolo",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Local-first web intelligence MCP server for AI coding agents",
   "type": "module",
   "main": "dist/index.js",