@staticn0va/wigolo 0.1.0 → 0.1.2

package/dist/extraction/llm-fallback.js

@@ -0,0 +1,130 @@
+import { getConfig } from "../config.js";
+import { callAnthropic } from "../integrations/cloud/llm/anthropic.js";
+import { callOpenAI } from "../integrations/cloud/llm/openai.js";
+import { callGemini } from "../integrations/cloud/llm/gemini.js";
+import { callGroq } from "../integrations/cloud/llm/groq.js";
+import {
+  ensureLLMCacheTable,
+  insertLLMCache,
+  lookupLLMCache
+} from "../integrations/cloud/llm/cache.js";
+import { hashPrompt, hashSchema } from "../integrations/cloud/llm/hash.js";
+import { allProviders, providerEnvVar, selectProvider } from "../integrations/cloud/llm/select.js";
+import { validateAgainstSchema } from "../integrations/cloud/llm/validate.js";
+const MAX_HTML_BYTES = 5e4;
+const ADAPTERS = {
+  anthropic: callAnthropic,
+  openai: callOpenAI,
+  gemini: callGemini,
+  groq: callGroq
+};
+async function extractWithLLM(input) {
+  if (input.missing.length === 0) {
+    return emptyResult(input.partial, []);
+  }
+  const cfg = getConfig();
+  const budget = input.budget ?? { remaining: cfg.llmMaxCallsPerRequest };
+  if (budget.remaining <= 0) {
+    return emptyResult(input.partial, [
+      `LLM fallback skipped: per-request budget exhausted (cap ${cfg.llmMaxCallsPerRequest}). Override via WIGOLO_LLM_MAX_CALLS_PER_REQUEST.`
+    ]);
+  }
+  const provider = selectProvider(process.env);
+  if (!provider) {
+    const envList = allProviders().map((p) => providerEnvVar(p)).join(", ");
+    return emptyResult(input.partial, [
+      `LLM fallback skipped: no provider key set (${envList}). ${input.missing.length} required field(s) still missing: ${input.missing.join(", ")}.`
+    ]);
+  }
+  const apiKey = process.env[providerEnvVar(provider)];
+  const prompt = buildPrompt(input);
+  const promptHash = hashPrompt(prompt);
+  const schemaHash = hashSchema(input.jsonSchema);
+  const modelId = `${provider}:default`;
+  ensureLLMCacheTable();
+  const cached = lookupLLMCache(modelId, promptHash, schemaHash);
+  if (cached) {
+    const values = JSON.parse(cached);
+    return {
+      values: mergeOnlyMissing(input.partial, values, input.missing),
+      provider,
+      model: modelId,
+      cached: true,
+      latencyMs: 0,
+      warnings: []
+    };
+  }
+  let result;
+  try {
+    result = await ADAPTERS[provider](
+      { prompt, jsonSchema: input.jsonSchema, signal: input.signal },
+      apiKey
+    );
+  } catch (e) {
+    return emptyResult(input.partial, [
+      `LLM fallback (${provider}) failed: ${e.message}`
+    ]);
+  } finally {
+    budget.remaining = Math.max(0, budget.remaining - 1);
+  }
+  const errors = validateAgainstSchema(result.values, input.jsonSchema);
+  if (errors.length > 0) {
+    return emptyResult(input.partial, [
+      `LLM fallback (${provider}) response failed schema validation: ${errors.map((e) => `${e.path} ${e.message}`).join("; ")}`
+    ]);
+  }
+  const ttlMs = cfg.llmCacheTtlDays * 24 * 60 * 60 * 1e3;
+  const now = Date.now();
+  insertLLMCache({
+    modelId,
+    promptHash,
+    schemaHash,
+    response: JSON.stringify(result.values),
+    createdAt: now,
+    expiresAt: now + ttlMs
+  });
+  return {
+    values: mergeOnlyMissing(input.partial, result.values, input.missing),
+    provider,
+    model: result.model,
+    cached: false,
+    latencyMs: result.latencyMs,
+    warnings: result.warnings ?? []
+  };
+}
+function emptyResult(partial, warnings) {
+  return {
+    values: { ...partial },
+    provider: "anthropic",
+    model: "",
+    cached: false,
+    latencyMs: 0,
+    warnings
+  };
+}
+function mergeOnlyMissing(partial, filled, missing) {
+  const out = { ...partial };
+  for (const key of missing) {
+    if (filled[key] !== void 0) out[key] = filled[key];
+  }
+  return out;
+}
+function buildPrompt(input) {
+  const html = truncate(input.html, MAX_HTML_BYTES);
+  return [
+    "Extract the following missing fields from the HTML below.",
+    `Missing fields: ${input.missing.join(", ")}.`,
+    "Return JSON matching the provided schema. Do not invent values; if a field is not present in the HTML, omit it.",
+    "",
+    "HTML:",
+    html
+  ].join("\n");
+}
+function truncate(s, maxBytes) {
+  if (s.length <= maxBytes) return s;
+  return s.slice(0, maxBytes);
+}
+export {
+  extractWithLLM
+};
+//# sourceMappingURL=llm-fallback.js.map

package/dist/extraction/llm-fallback.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/extraction/llm-fallback.ts"],"sourcesContent":["import { getConfig } from '../config.js';\nimport { callAnthropic } from '../integrations/cloud/llm/anthropic.js';\nimport { callOpenAI } from '../integrations/cloud/llm/openai.js';\nimport { callGemini } from '../integrations/cloud/llm/gemini.js';\nimport { callGroq } from '../integrations/cloud/llm/groq.js';\nimport {\n  ensureLLMCacheTable,\n  insertLLMCache,\n  lookupLLMCache,\n} from '../integrations/cloud/llm/cache.js';\nimport { hashPrompt, hashSchema } from '../integrations/cloud/llm/hash.js';\nimport { allProviders, providerEnvVar, selectProvider } from '../integrations/cloud/llm/select.js';\nimport type { LLMExtractResult, LLMProvider } from '../integrations/cloud/llm/types.js';\nimport { validateAgainstSchema } from '../integrations/cloud/llm/validate.js';\n\nconst MAX_HTML_BYTES = 50_000;\n\nexport interface LLMFallbackBudget {\n  remaining: number;\n}\n\nexport interface LLMFallbackInput {\n  html: string;\n  jsonSchema: Record<string, unknown>;\n  partial: Record<string, unknown>;\n  missing: string[];\n  signal?: AbortSignal;\n  budget?: LLMFallbackBudget;\n}\n\nexport interface LLMFallbackResult extends LLMExtractResult {\n  warnings: string[];\n}\n\nconst ADAPTERS: Record<\n  LLMProvider,\n  (\n    opts: { prompt: string; jsonSchema: Record<string, unknown>; signal?: AbortSignal },\n    apiKey: string,\n  ) => Promise<LLMExtractResult>\n> = {\n  anthropic: callAnthropic,\n  openai: callOpenAI,\n  gemini: callGemini,\n  groq: callGroq,\n};\n\nexport async function extractWithLLM(\n  input: LLMFallbackInput,\n): Promise<LLMFallbackResult> {\n  if (input.missing.length === 0) {\n    return emptyResult(input.partial, []);\n  }\n\n  const cfg = getConfig();\n  const budget = input.budget ?? { remaining: cfg.llmMaxCallsPerRequest };\n  if (budget.remaining <= 0) {\n    return emptyResult(input.partial, [\n      `LLM fallback skipped: per-request budget exhausted (cap ${cfg.llmMaxCallsPerRequest}). Override via WIGOLO_LLM_MAX_CALLS_PER_REQUEST.`,\n    ]);\n  }\n\n  const provider = selectProvider(process.env);\n  if (!provider) {\n    const envList = allProviders()\n      .map((p) => providerEnvVar(p))\n      .join(', ');\n    return emptyResult(input.partial, [\n      `LLM fallback skipped: no provider key set (${envList}). ` +\n        `${input.missing.length} required field(s) still missing: ${input.missing.join(', ')}.`,\n    ]);\n  }\n\n  const apiKey = process.env[providerEnvVar(provider)] as string;\n  const prompt = buildPrompt(input);\n  const promptHash = hashPrompt(prompt);\n  const schemaHash = hashSchema(input.jsonSchema);\n  const modelId = `${provider}:default`;\n\n  ensureLLMCacheTable();\n  const cached = lookupLLMCache(modelId, promptHash, schemaHash);\n  if (cached) {\n    const values = JSON.parse(cached) as Record<string, unknown>;\n    return {\n      values: mergeOnlyMissing(input.partial, values, input.missing),\n      provider,\n      model: modelId,\n      cached: true,\n      latencyMs: 0,\n      warnings: [],\n    };\n  }\n\n  let result: LLMExtractResult;\n  try {\n    result = await ADAPTERS[provider](\n      { prompt, jsonSchema: input.jsonSchema, signal: input.signal },\n      apiKey,\n    );\n  } catch (e) {\n    return emptyResult(input.partial, [\n      `LLM fallback (${provider}) failed: ${(e as Error).message}`,\n    ]);\n  } finally {\n    budget.remaining = Math.max(0, budget.remaining - 1);\n  }\n\n  const errors = validateAgainstSchema(result.values, input.jsonSchema);\n  if (errors.length > 0) {\n    return emptyResult(input.partial, [\n      `LLM fallback (${provider}) response failed schema validation: ${errors\n        .map((e) => `${e.path} ${e.message}`)\n        .join('; ')}`,\n    ]);\n  }\n\n  const ttlMs = cfg.llmCacheTtlDays * 24 * 60 * 60 * 1000;\n  const now = Date.now();\n  insertLLMCache({\n    modelId,\n    promptHash,\n    schemaHash,\n    response: JSON.stringify(result.values),\n    createdAt: now,\n    expiresAt: now + ttlMs,\n  });\n\n  return {\n    values: mergeOnlyMissing(input.partial, result.values, input.missing),\n    provider,\n    model: result.model,\n    cached: false,\n    latencyMs: result.latencyMs,\n    warnings: result.warnings ?? [],\n  };\n}\n\nfunction emptyResult(\n  partial: Record<string, unknown>,\n  warnings: string[],\n): LLMFallbackResult {\n  return {\n    values: { ...partial },\n    provider: 'anthropic',\n    model: '',\n    cached: false,\n    latencyMs: 0,\n    warnings,\n  };\n}\n\nfunction mergeOnlyMissing(\n  partial: Record<string, unknown>,\n  filled: Record<string, unknown>,\n  missing: string[],\n): Record<string, unknown> {\n  const out = { ...partial };\n  for (const key of missing) {\n    if (filled[key] !== undefined) out[key] = filled[key];\n  }\n  return out;\n}\n\nfunction buildPrompt(input: LLMFallbackInput): string {\n  const html = truncate(input.html, MAX_HTML_BYTES);\n  return [\n    'Extract the following missing fields from the HTML below.',\n    `Missing fields: ${input.missing.join(', ')}.`,\n    'Return JSON matching the provided schema. Do not invent values; if a field is not present in the HTML, omit it.',\n    '',\n    'HTML:',\n    html,\n  ].join('\\n');\n}\n\nfunction truncate(s: string, maxBytes: number): string {\n  if (s.length <= maxBytes) return s;\n  return s.slice(0, maxBytes);\n}\n"],"mappings":"AAAA,SAAS,iBAAiB;AAC1B,SAAS,qBAAqB;AAC9B,SAAS,kBAAkB;AAC3B,SAAS,kBAAkB;AAC3B,SAAS,gBAAgB;AACzB;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AACP,SAAS,YAAY,kBAAkB;AACvC,SAAS,cAAc,gBAAgB,sBAAsB;AAE7D,SAAS,6BAA6B;AAEtC,MAAM,iBAAiB;AAmBvB,MAAM,WAMF;AAAA,EACF,WAAW;AAAA,EACX,QAAQ;AAAA,EACR,QAAQ;AAAA,EACR,MAAM;AACR;AAEA,eAAsB,eACpB,OAC4B;AAC5B,MAAI,MAAM,QAAQ,WAAW,GAAG;AAC9B,WAAO,YAAY,MAAM,SAAS,CAAC,CAAC;AAAA,EACtC;AAEA,QAAM,MAAM,UAAU;AACtB,QAAM,SAAS,MAAM,UAAU,EAAE,WAAW,IAAI,sBAAsB;AACtE,MAAI,OAAO,aAAa,GAAG;AACzB,WAAO,YAAY,MAAM,SAAS;AAAA,MAChC,2DAA2D,IAAI,qBAAqB;AAAA,IACtF,CAAC;AAAA,EACH;AAEA,QAAM,WAAW,eAAe,QAAQ,GAAG;AAC3C,MAAI,CAAC,UAAU;AACb,UAAM,UAAU,aAAa,EAC1B,IAAI,CAAC,MAAM,eAAe,CAAC,CAAC,EAC5B,KAAK,IAAI;AACZ,WAAO,YAAY,MAAM,SAAS;AAAA,MAChC,8CAA8C,OAAO,MAChD,MAAM,QAAQ,MAAM,qCAAqC,MAAM,QAAQ,KAAK,IAAI,CAAC;AAAA,IACxF,CAAC;AAAA,EACH;AAEA,QAAM,SAAS,QAAQ,IAAI,eAAe,QAAQ,CAAC;AACnD,QAAM,SAAS,YAAY,KAAK;AAChC,QAAM,aAAa,WAAW,MAAM;AACpC,QAAM,aAAa,WAAW,MAAM,UAAU;AAC9C,QAAM,UAAU,GAAG,QAAQ;AAE3B,sBAAoB;AACpB,QAAM,SAAS,eAAe,SAAS,YAAY,UAAU;AAC7D,MAAI,QAAQ;AACV,UAAM,SAAS,KAAK,MAAM,MAAM;AAChC,WAAO;AAAA,MACL,QAAQ,iBAAiB,MAAM,SAAS,QAAQ,MAAM,OAAO;AAAA,MAC7D;AAAA,MACA,OAAO;AAAA,MACP,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,UAAU,CAAC;AAAA,IACb;AAAA,EACF;AAEA,MAAI;AACJ,MAAI;AACF,aAAS,MAAM,SAAS,QAAQ;AAAA,MAC9B,EAAE,QAAQ,YAAY,MAAM,YAAY,QAAQ,MAAM,OAAO;AAAA,MAC7D;AAAA,IACF;AAAA,EACF,SAAS,GAAG;AACV,WAAO,YAAY,MAAM,SAAS;AAAA,MAChC,iBAAiB,QAAQ,aAAc,EAAY,OAAO;AAAA,IAC5D,CAAC;AAAA,EACH,UAAE;AACA,WAAO,YAAY,KAAK,IAAI,GAAG,OAAO,YAAY,CAAC;AAAA,EACrD;AAEA,QAAM,SAAS,sBAAsB,OAAO,QAAQ,MAAM,UAAU;AACpE,MAAI,OAAO,SAAS,GAAG;AACrB,WAAO,YAAY,MAAM,SAAS;AAAA,MAChC,iBAAiB,QAAQ,wCAAwC,OAC9D,IAAI,CAAC,MAAM,GAAG,EAAE,IAAI,IAAI,EAAE,OAAO,EAAE,EACnC,KAAK,IAAI,CAAC;AAAA,IACf,CAAC;AAAA,EACH;AAEA,QAAM,QAAQ,IAAI,kBAAkB,KAAK,KAAK,KAAK;AACnD,QAAM,MAAM,KAAK,IAAI;AACrB,iBAAe;AAAA,IACb;AAAA,IACA;AAAA,IACA;AAAA,IACA,UAAU,KAAK,UAAU,OAAO,MAAM;AAAA,IACtC,WAAW;AAAA,IACX,WAAW,MAAM;AAAA,EACnB,CAAC;AAED,SAAO;AAAA,IACL,QAAQ,iBAAiB,MAAM,SAAS,OAAO,QAAQ,MAAM,OAAO;AAAA,IACpE;AAAA,IACA,OAAO,OAAO;AAAA,IACd,QAAQ;AAAA,IACR,WAAW,OAAO;AAAA,IAClB,UAAU,OAAO,YAAY,CAAC;AAAA,EAChC;AACF;AAEA,SAAS,YACP,SACA,UACmB;AACnB,SAAO;AAAA,IACL,QAAQ,EAAE,GAAG,QAAQ;AAAA,IACrB,UAAU;AAAA,IACV,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,WAAW;AAAA,IACX;AAAA,EACF;AACF;AAEA,SAAS,iBACP,SACA,QACA,SACyB;AACzB,QAAM,MAAM,EAAE,GAAG,QAAQ;AACzB,aAAW,OAAO,SAAS;AACzB,QAAI,OAAO,GAAG,MAAM,OAAW,KAAI,GAAG,IAAI,OAAO,GAAG;AAAA,EACtD;AACA,SAAO;AACT;AAEA,SAAS,YAAY,OAAiC;AACpD,QAAM,OAAO,SAAS,MAAM,MAAM,cAAc;AAChD,SAAO;AAAA,IACL;AAAA,IACA,mBAAmB,MAAM,QAAQ,KAAK,IAAI,CAAC;AAAA,IAC3C;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF,EAAE,KAAK,IAAI;AACb;AAEA,SAAS,SAAS,GAAW,UAA0B;AACrD,MAAI,EAAE,UAAU,SAAU,QAAO;AACjC,SAAO,EAAE,MAAM,GAAG,QAAQ;AAC5B;","names":[]}

package/dist/extraction/markdown-sanitize.d.ts

@@ -0,0 +1,2 @@
+export declare function sanitizeExtractedMarkdown(md: string): string;
+//# sourceMappingURL=markdown-sanitize.d.ts.map

package/dist/extraction/markdown-sanitize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"markdown-sanitize.d.ts","sourceRoot":"","sources":["../../src/extraction/markdown-sanitize.ts"],"names":[],"mappings":"AAkFA,wBAAgB,yBAAyB,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CA6D5D"}

package/dist/extraction/markdown-sanitize.js

@@ -0,0 +1,151 @@
+const STRAY_LABELS = /* @__PURE__ */ new Set([
+  "javascript",
+  "typescript",
+  "mjs",
+  "cjs",
+  "json",
+  "html",
+  "css",
+  "bash",
+  "sh",
+  "shell",
+  "python",
+  "py",
+  "go",
+  "rust",
+  "java",
+  "kotlin",
+  "swift",
+  "cpp",
+  "c++",
+  "csharp",
+  "ruby",
+  "php"
+]);
+const GLUED_LANG_PREFIXES = {
+  ts: "ts",
+  js: "js",
+  tsx: "tsx",
+  jsx: "jsx",
+  py: "python",
+  rb: "ruby",
+  go: "go",
+  rs: "rust",
+  sh: "bash",
+  json: "json",
+  html: "html",
+  css: "css",
+  yaml: "yaml",
+  yml: "yaml",
+  toml: "toml",
+  md: "markdown"
+};
+const POST_PREFIX_TOKENS = [
+  "function",
+  "const",
+  "let",
+  "var",
+  "class",
+  "interface",
+  "type",
+  "enum",
+  "import",
+  "export",
+  "async",
+  "await",
+  "return",
+  "if",
+  "for",
+  "while",
+  "def",
+  "print",
+  "echo",
+  "package",
+  "public",
+  "private",
+  "protected",
+  "struct",
+  "fn",
+  "pub",
+  "use",
+  "mod"
+];
+function unglueLangPrefix(line) {
+  for (const [prefix, lang] of Object.entries(GLUED_LANG_PREFIXES)) {
+    if (!line.startsWith(prefix)) continue;
+    const rest = line.slice(prefix.length);
+    for (const tok of POST_PREFIX_TOKENS) {
+      if (rest.startsWith(tok)) {
+        const next = rest.charAt(tok.length);
+        if (next === "" || /[\s({<\[]/.test(next)) {
+          return { lang, line: rest };
+        }
+      }
+    }
+  }
+  return null;
+}
+function isFenceLine(line) {
+  const m = line.match(/^(```+|~~~+)([a-zA-Z0-9_+-]*)\s*$/);
+  if (!m) return { open: false, close: false };
+  const lang = m[2]?.trim() || void 0;
+  return { open: !!lang, close: !lang, lang };
+}
+function sanitizeExtractedMarkdown(md) {
+  if (!md.includes("```") && !md.includes("~~~")) return md;
+  const lines = md.split("\n");
+  const out = [];
+  let inFence = false;
+  let fenceMarker = null;
+  let pendingFirstContentLine = false;
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    if (!inFence) {
+      const f = line.match(/^(```+|~~~+)([a-zA-Z0-9_+-]*)\s*$/);
+      if (f) {
+        inFence = true;
+        fenceMarker = f[1];
+        pendingFirstContentLine = true;
+        const declaredLang = f[2];
+        if (declaredLang === "markdown") {
+          out.push(fenceMarker);
+        } else {
+          out.push(line);
+        }
+        continue;
+      }
+      out.push(line);
+      continue;
+    }
+    if (fenceMarker && line.startsWith(fenceMarker) && line.replace(fenceMarker, "").trim() === "") {
+      inFence = false;
+      fenceMarker = null;
+      pendingFirstContentLine = false;
+      out.push(line);
+      continue;
+    }
+    if (pendingFirstContentLine) {
+      pendingFirstContentLine = false;
+      const unglued = unglueLangPrefix(line);
+      if (unglued) {
+        const lastIdx = out.length - 1;
+        const prev = out[lastIdx];
+        if (prev.startsWith("```") || prev.startsWith("~~~")) {
+          out[lastIdx] = `${prev.match(/^(```+|~~~+)/)[1]}${unglued.lang ?? ""}`;
+        }
+        out.push(unglued.line);
+        continue;
+      }
+    }
+    const trimmed = line.trim();
+    if (trimmed && STRAY_LABELS.has(trimmed.toLowerCase()) && !line.includes(" ")) {
+      continue;
+    }
+    out.push(line);
+  }
+  return out.join("\n");
+}
+export {
+  sanitizeExtractedMarkdown
+};
+//# sourceMappingURL=markdown-sanitize.js.map

package/dist/extraction/markdown-sanitize.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/extraction/markdown-sanitize.ts"],"sourcesContent":["// Post-extraction sanitizer for markdown returned by ensemble extractors.\n// Currently targets a Node-docs pattern where tab labels (\"javascript\",\n// \"typescript\", etc.) leak into a fenced code block as a bare line.\n\nconst STRAY_LABELS = new Set([\n  'javascript',\n  'typescript',\n  'mjs',\n  'cjs',\n  'json',\n  'html',\n  'css',\n  'bash',\n  'sh',\n  'shell',\n  'python',\n  'py',\n  'go',\n  'rust',\n  'java',\n  'kotlin',\n  'swift',\n  'cpp',\n  'c++',\n  'csharp',\n  'ruby',\n  'php',\n]);\n\n// Short aliases that can appear *glued* to the first identifier on the first\n// line of a fenced block (e.g. the TypeScript docs render `<span>ts</span>` and\n// the next token concatenates without a separator → `tsfunction`, `jsconst`).\nconst GLUED_LANG_PREFIXES: Record<string, string> = {\n  ts: 'ts',\n  js: 'js',\n  tsx: 'tsx',\n  jsx: 'jsx',\n  py: 'python',\n  rb: 'ruby',\n  go: 'go',\n  rs: 'rust',\n  sh: 'bash',\n  json: 'json',\n  html: 'html',\n  css: 'css',\n  yaml: 'yaml',\n  yml: 'yaml',\n  toml: 'toml',\n  md: 'markdown',\n};\n\n// Identifier keywords that commonly follow a stuck language prefix.\nconst POST_PREFIX_TOKENS = [\n  'function', 'const', 'let', 'var', 'class', 'interface', 'type', 'enum',\n  'import', 'export', 'async', 'await', 'return', 'if', 'for', 'while',\n  'def', 'print', 'echo', 'package', 'public', 'private', 'protected',\n  'struct', 'fn', 'pub', 'use', 'mod',\n];\n\nfunction unglueLangPrefix(line: string): { lang?: string; line: string } | null {\n  for (const [prefix, lang] of Object.entries(GLUED_LANG_PREFIXES)) {\n    if (!line.startsWith(prefix)) continue;\n    const rest = line.slice(prefix.length);\n    for (const tok of POST_PREFIX_TOKENS) {\n      if (rest.startsWith(tok)) {\n        const next = rest.charAt(tok.length);\n        if (next === '' || /[\\s({<\\[]/.test(next)) {\n          return { lang, line: rest };\n        }\n      }\n    }\n  }\n  return null;\n}\n\nfunction isFenceLine(line: string): { open: boolean; close: boolean; lang?: string } {\n  const m = line.match(/^(```+|~~~+)([a-zA-Z0-9_+-]*)\\s*$/);\n  if (!m) return { open: false, close: false };\n  const lang = m[2]?.trim() || undefined;\n  return { open: !!lang, close: !lang, lang };\n}\n\nexport function sanitizeExtractedMarkdown(md: string): string {\n  if (!md.includes('```') && !md.includes('~~~')) return md;\n  const lines = md.split('\\n');\n  const out: string[] = [];\n  let inFence = false;\n  let fenceMarker: string | null = null;\n  let pendingFirstContentLine = false;\n\n  for (let i = 0; i < lines.length; i++) {\n    const line = lines[i];\n    if (!inFence) {\n      const f = line.match(/^(```+|~~~+)([a-zA-Z0-9_+-]*)\\s*$/);\n      if (f) {\n        inFence = true;\n        fenceMarker = f[1];\n        pendingFirstContentLine = true;\n        const declaredLang = f[2];\n        // 'markdown' is a sentinel many extractors emit when the source code\n        // tag didn't carry a language class. Reset it; we'll try to recover\n        // the real lang from the first content line below.\n        if (declaredLang === 'markdown') {\n          out.push(fenceMarker);\n        } else {\n          out.push(line);\n        }\n        continue;\n      }\n      out.push(line);\n      continue;\n    }\n    // inside a fence\n    if (fenceMarker && line.startsWith(fenceMarker) && line.replace(fenceMarker, '').trim() === '') {\n      inFence = false;\n      fenceMarker = null;\n      pendingFirstContentLine = false;\n      out.push(line);\n      continue;\n    }\n    if (pendingFirstContentLine) {\n      pendingFirstContentLine = false;\n      const unglued = unglueLangPrefix(line);\n      if (unglued) {\n        // Replace the most recently pushed fence-open line with one that\n        // carries the recovered language tag.\n        const lastIdx = out.length - 1;\n        const prev = out[lastIdx];\n        if (prev.startsWith('```') || prev.startsWith('~~~')) {\n          out[lastIdx] = `${prev.match(/^(```+|~~~+)/)![1]}${unglued.lang ?? ''}`;\n        }\n        out.push(unglued.line);\n        continue;\n      }\n    }\n    const trimmed = line.trim();\n    if (trimmed && STRAY_LABELS.has(trimmed.toLowerCase()) && !line.includes(' ')) {\n      // Drop the stray language label line.\n      continue;\n    }\n    out.push(line);\n  }\n  return out.join('\\n');\n}\n"],"mappings":"AAIA,MAAM,eAAe,oBAAI,IAAI;AAAA,EAC3B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAKD,MAAM,sBAA8C;AAAA,EAClD,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,KAAK;AAAA,EACL,KAAK;AAAA,EACL,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,IAAI;AAAA,EACJ,MAAM;AAAA,EACN,MAAM;AAAA,EACN,KAAK;AAAA,EACL,MAAM;AAAA,EACN,KAAK;AAAA,EACL,MAAM;AAAA,EACN,IAAI;AACN;AAGA,MAAM,qBAAqB;AAAA,EACzB;AAAA,EAAY;AAAA,EAAS;AAAA,EAAO;AAAA,EAAO;AAAA,EAAS;AAAA,EAAa;AAAA,EAAQ;AAAA,EACjE;AAAA,EAAU;AAAA,EAAU;AAAA,EAAS;AAAA,EAAS;AAAA,EAAU;AAAA,EAAM;AAAA,EAAO;AAAA,EAC7D;AAAA,EAAO;AAAA,EAAS;AAAA,EAAQ;AAAA,EAAW;AAAA,EAAU;AAAA,EAAW;AAAA,EACxD;AAAA,EAAU;AAAA,EAAM;AAAA,EAAO;AAAA,EAAO;AAChC;AAEA,SAAS,iBAAiB,MAAsD;AAC9E,aAAW,CAAC,QAAQ,IAAI,KAAK,OAAO,QAAQ,mBAAmB,GAAG;AAChE,QAAI,CAAC,KAAK,WAAW,MAAM,EAAG;AAC9B,UAAM,OAAO,KAAK,MAAM,OAAO,MAAM;AACrC,eAAW,OAAO,oBAAoB;AACpC,UAAI,KAAK,WAAW,GAAG,GAAG;AACxB,cAAM,OAAO,KAAK,OAAO,IAAI,MAAM;AACnC,YAAI,SAAS,MAAM,YAAY,KAAK,IAAI,GAAG;AACzC,iBAAO,EAAE,MAAM,MAAM,KAAK;AAAA,QAC5B;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,YAAY,MAAgE;AACnF,QAAM,IAAI,KAAK,MAAM,mCAAmC;AACxD,MAAI,CAAC,EAAG,QAAO,EAAE,MAAM,OAAO,OAAO,MAAM;AAC3C,QAAM,OAAO,EAAE,CAAC,GAAG,KAAK,KAAK;AAC7B,SAAO,EAAE,MAAM,CAAC,CAAC,MAAM,OAAO,CAAC,MAAM,KAAK;AAC5C;AAEO,SAAS,0BAA0B,IAAoB;AAC5D,MAAI,CAAC,GAAG,SAAS,KAAK,KAAK,CAAC,GAAG,SAAS,KAAK,EAAG,QAAO;AACvD,QAAM,QAAQ,GAAG,MAAM,IAAI;AAC3B,QAAM,MAAgB,CAAC;AACvB,MAAI,UAAU;AACd,MAAI,cAA6B;AACjC,MAAI,0BAA0B;AAE9B,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,UAAM,OAAO,MAAM,CAAC;AACpB,QAAI,CAAC,SAAS;AACZ,YAAM,IAAI,KAAK,MAAM,mCAAmC;AACxD,UAAI,GAAG;AACL,kBAAU;AACV,sBAAc,EAAE,CAAC;AACjB,kCAA0B;AAC1B,cAAM,eAAe,EAAE,CAAC;AAIxB,YAAI,iBAAiB,YAAY;AAC/B,cAAI,KAAK,WAAW;AAAA,QACtB,OAAO;AACL,cAAI,KAAK,IAAI;AAAA,QACf;AACA;AAAA,MACF;AACA,UAAI,KAAK,IAAI;AACb;AAAA,IACF;AAEA,QAAI,eAAe,KAAK,WAAW,WAAW,KAAK,KAAK,QAAQ,aAAa,EAAE,EAAE,KAAK,MAAM,IAAI;AAC9F,gBAAU;AACV,oBAAc;AACd,gCAA0B;AAC1B,UAAI,KAAK,IAAI;AACb;AAAA,IACF;AACA,QAAI,yBAAyB;AAC3B,gCAA0B;AAC1B,YAAM,UAAU,iBAAiB,IAAI;AACrC,UAAI,SAAS;AAGX,cAAM,UAAU,IAAI,SAAS;AAC7B,cAAM,OAAO,IAAI,OAAO;AACxB,YAAI,KAAK,WAAW,KAAK,KAAK,KAAK,WAAW,KAAK,GAAG;AACpD,cAAI,OAAO,IAAI,GAAG,KAAK,MAAM,cAAc,EAAG,CAAC,CAAC,GAAG,QAAQ,QAAQ,EAAE;AAAA,QACvE;AACA,YAAI,KAAK,QAAQ,IAAI;AACrB;AAAA,MACF;AAAA,IACF;AACA,UAAM,UAAU,KAAK,KAAK;AAC1B,QAAI,WAAW,aAAa,IAAI,QAAQ,YAAY,CAAC,KAAK,CAAC,KAAK,SAAS,GAAG,GAAG;AAE7E;AAAA,IACF;AACA,QAAI,KAAK,IAAI;AAAA,EACf;AACA,SAAO,IAAI,KAAK,IAAI;AACtB;","names":[]}

package/dist/extraction/markdown.d.ts

@@ -1,4 +1,13 @@
+import TurndownService from 'turndown';
+export declare function buildTurndown(): TurndownService;
 export declare function htmlToMarkdown(html: string): string;
+export interface Heading {
+    level: number;
+    text: string;
+    lineIndex: number;
+}
+export declare function parseHeadings(lines: string[]): Heading[];
+export declare function lineStartCharOffsets(lines: string[]): number[];
 export declare function extractSection(markdown: string, section: string, sectionIndex?: number): {
     content: string;
     matched: boolean;
@@ -7,4 +16,6 @@ export declare function extractLinksAndImages(markdown: string): {
     links: string[];
     images: string[];
 };
+export declare function filterDecorativeImages(markdown: string): string;
+export declare function resolveRelativeUrls(markdown: string, baseUrl: string): string;
 //# sourceMappingURL=markdown.d.ts.map

package/dist/extraction/markdown.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/extraction/markdown.ts"],"names":[],"mappings":"AAkDA,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAGnD;AAmCD,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,YAAY,SAAI,GACf;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,CA2BvC;AAED,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CAoB7F"}
+{"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/extraction/markdown.ts"],"names":[],"mappings":"AAAA,OAAO,eAAe,MAAM,UAAU,CAAC;AAiBvC,wBAAgB,aAAa,IAAI,eAAe,CA2D/C;AAID,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAGnD;AAED,MAAM,WAAW,OAAO;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,EAAE,CASxD;AAID,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CAQ9D;AAkBD,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,MAAM,EACf,YAAY,SAAI,GACf;IAAE,OAAO,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,OAAO,CAAA;CAAE,CA2BvC;AAED,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,EAAE,MAAM,EAAE,CAAA;CAAE,CAoB7F;AAkBD,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAsB/D;AAID,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAyC7E"}

package/dist/extraction/markdown.js

@@ -1,107 +1,211 @@
-import TurndownService from 'turndown';
+import TurndownService from "turndown";
+import { detectCodeLanguage } from "./lang-hints.js";
+function longestBacktickRun(s) {
+  let max = 0;
+  let cur = 0;
+  for (let i = 0; i < s.length; i++) {
+    if (s.charCodeAt(i) === 96) {
+      cur++;
+      if (cur > max) max = cur;
+    } else {
+      cur = 0;
+    }
+  }
+  return max;
+}
 function buildTurndown() {
-    const td = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
-    // Remove script and style tags entirely
-    td.remove(['script', 'style']);
-    // Custom rule: convert <table> to markdown table
-    td.addRule('table', {
-        filter: 'table',
-        replacement(_content, node) {
-            const el = node;
-            const rows = Array.from(el.querySelectorAll('tr'));
-            if (rows.length === 0)
-                return '';
-            const renderRow = (row) => {
-                const cells = Array.from(row.querySelectorAll('th, td'));
-                return '| ' + cells.map(c => c.textContent?.replace(/\n/g, ' ').trim() ?? '').join(' | ') + ' |';
-            };
-            const headerRow = rows[0];
-            const isHeaderRow = headerRow.querySelectorAll('th').length > 0;
-            const headerCells = Array.from(headerRow.querySelectorAll('th, td'));
-            const separator = '| ' + headerCells.map(() => '---').join(' | ') + ' |';
-            if (isHeaderRow) {
-                const bodyRows = rows.slice(1);
-                const lines = [renderRow(headerRow), separator, ...bodyRows.map(renderRow)];
-                return '\n\n' + lines.join('\n') + '\n\n';
-            }
-            const lines = [renderRow(headerRow), separator, ...rows.slice(1).map(renderRow)];
-            return '\n\n' + lines.join('\n') + '\n\n';
-        },
-    });
-    // Suppress thead/tbody/tr/th/td individually since table rule handles the whole node
-    td.addRule('tableCell', {
-        filter: ['thead', 'tbody', 'tfoot', 'tr', 'th', 'td'],
-        replacement(content) {
-            return content;
-        },
-    });
-    return td;
+  const td = new TurndownService({ headingStyle: "atx", codeBlockStyle: "fenced" });
+  td.remove(["script", "style"]);
+  td.addRule("table", {
+    filter: "table",
+    replacement(_content, node) {
+      const el = node;
+      const rows = Array.from(el.querySelectorAll("tr"));
+      if (rows.length === 0) return "";
+      const renderRow = (row) => {
+        const cells = Array.from(row.querySelectorAll("th, td"));
+        return "| " + cells.map((c) => c.textContent?.replace(/\n/g, " ").trim() ?? "").join(" | ") + " |";
+      };
+      const headerRow = rows[0];
+      const isHeaderRow = headerRow.querySelectorAll("th").length > 0;
+      const headerCells = Array.from(headerRow.querySelectorAll("th, td"));
+      const separator = "| " + headerCells.map(() => "---").join(" | ") + " |";
+      if (isHeaderRow) {
+        const bodyRows = rows.slice(1);
+        const lines2 = [renderRow(headerRow), separator, ...bodyRows.map(renderRow)];
+        return "\n\n" + lines2.join("\n") + "\n\n";
+      }
+      const lines = [renderRow(headerRow), separator, ...rows.slice(1).map(renderRow)];
+      return "\n\n" + lines.join("\n") + "\n\n";
+    }
+  });
+  td.addRule("tableCell", {
+    filter: ["thead", "tbody", "tfoot", "tr", "th", "td"],
+    replacement(content) {
+      return content;
+    }
+  });
+  td.addRule("codeBlockLang", {
+    filter(node) {
+      return node.nodeName === "PRE" && node.querySelector("code") !== null;
+    },
+    replacement(_content, node) {
+      const pre = node;
+      const code = pre.querySelector("code");
+      const cls = code?.getAttribute("class") ?? pre.getAttribute("class") ?? "";
+      const lang = detectCodeLanguage(cls);
+      const body = code?.textContent ?? pre.textContent ?? "";
+      const fence = "`".repeat(Math.max(3, longestBacktickRun(body) + 1));
+      return `
+
+${fence}${lang ?? ""}
+${body.replace(/\n+$/, "")}
+${fence}
+
+`;
+    }
+  });
+  return td;
 }
 const turndown = buildTurndown();
-export function htmlToMarkdown(html) {
-    if (!html)
-        return '';
-    return turndown.turndown(html);
+function htmlToMarkdown(html) {
+  if (!html) return "";
+  return turndown.turndown(html);
 }
 function parseHeadings(lines) {
-    const headings = [];
-    for (let i = 0; i < lines.length; i++) {
-        const match = lines[i].match(/^(#{1,6})\s+(.+)/);
-        if (match) {
-            headings.push({ level: match[1].length, text: match[2].trim(), lineIndex: i });
-        }
+  const headings = [];
+  for (let i = 0; i < lines.length; i++) {
+    const match = lines[i].match(/^(#{1,6})\s+(.+)/);
+    if (match) {
+      headings.push({ level: match[1].length, text: match[2].trim(), lineIndex: i });
     }
-    return headings;
+  }
+  return headings;
+}
+function lineStartCharOffsets(lines) {
+  const offsets = new Array(lines.length);
+  let acc = 0;
+  for (let i = 0; i < lines.length; i++) {
+    offsets[i] = acc;
+    acc += lines[i].length + 1;
+  }
+  return offsets;
 }
 function extractFromHeading(lines, headings, headingIdx) {
-    const heading = headings[headingIdx];
-    const start = heading.lineIndex;
-    // Find the next heading of equal or higher level (lower or equal # count)
-    let end = lines.length;
-    for (let i = headingIdx + 1; i < headings.length; i++) {
-        if (headings[i].level <= heading.level) {
-            end = headings[i].lineIndex;
-            break;
-        }
+  const heading = headings[headingIdx];
+  const start = heading.lineIndex;
+  let end = lines.length;
+  for (let i = headingIdx + 1; i < headings.length; i++) {
+    if (headings[i].level <= heading.level) {
+      end = headings[i].lineIndex;
+      break;
     }
-    return lines.slice(start, end).join('\n');
+  }
+  return lines.slice(start, end).join("\n");
 }
-export function extractSection(markdown, section, sectionIndex = 0) {
-    const lines = markdown.split('\n');
-    const headings = parseHeadings(lines);
-    if (headings.length === 0)
-        return { content: markdown, matched: false };
-    const lower = section.toLowerCase();
-    const indexed = headings.map((h, i) => ({ h, i }));
-    // Collect exact matches first
-    const exactMatches = indexed.filter(({ h }) => h.text.toLowerCase() === lower);
-    // If exact matches satisfy the requested index, use them
-    if (exactMatches.length > 0 && sectionIndex < exactMatches.length) {
-        const { i } = exactMatches[sectionIndex];
-        return { content: extractFromHeading(lines, headings, i), matched: true };
-    }
-    // Fall back to substring matches (includes exact headings and partial ones)
-    const substringMatches = indexed.filter(({ h }) => h.text.toLowerCase().includes(lower));
-    if (substringMatches.length === 0 || sectionIndex >= substringMatches.length) {
-        return { content: markdown, matched: false };
+function extractSection(markdown, section, sectionIndex = 0) {
+  const lines = markdown.split("\n");
+  const headings = parseHeadings(lines);
+  if (headings.length === 0) return { content: markdown, matched: false };
+  const lower = section.toLowerCase();
+  const indexed = headings.map((h, i2) => ({ h, i: i2 }));
+  const exactMatches = indexed.filter(({ h }) => h.text.toLowerCase() === lower);
+  if (exactMatches.length > 0 && sectionIndex < exactMatches.length) {
+    const { i: i2 } = exactMatches[sectionIndex];
+    return { content: extractFromHeading(lines, headings, i2), matched: true };
+  }
+  const substringMatches = indexed.filter(({ h }) => h.text.toLowerCase().includes(lower));
+  if (substringMatches.length === 0 || sectionIndex >= substringMatches.length) {
+    return { content: markdown, matched: false };
+  }
+  const { i } = substringMatches[sectionIndex];
+  return { content: extractFromHeading(lines, headings, i), matched: true };
+}
+function extractLinksAndImages(markdown) {
+  const imagePattern = /!\[[^\]]*\]\(([^)]+)\)/g;
+  const linkPattern = /(?<!!)\[[^\]]*\]\(([^)]+)\)/g;
+  const images = /* @__PURE__ */ new Set();
+  const links = /* @__PURE__ */ new Set();
+  let match;
+  while ((match = imagePattern.exec(markdown)) !== null) {
+    images.add(match[1]);
+  }
+  while ((match = linkPattern.exec(markdown)) !== null) {
+    links.add(match[1]);
+  }
+  return { links: Array.from(links), images: Array.from(images) };
+}
+const DECORATIVE_URL_MARKERS = [
+  "avatar",
+  "icon",
+  "logo",
+  "badge",
+  "shield",
+  "tracking",
+  "pixel",
+  "sprite",
+  "emoji",
+  "favicon"
+];
+function filterDecorativeImages(markdown) {
+  if (!markdown) return markdown;
+  return markdown.replace(/!\[([^\]]*)\]\(([^)]+)\)/g, (match, alt, src) => {
+    const trimmedAlt = alt.trim();
+    const lowerSrc = src.toLowerCase();
+    if (lowerSrc.startsWith("data:image/gif;base64,")) return "";
+    if (lowerSrc.startsWith("data:image/svg+xml") && src.length < 200) return "";
+    for (const marker of DECORATIVE_URL_MARKERS) {
+      if (lowerSrc.includes(marker)) return "";
     }
-    const { i } = substringMatches[sectionIndex];
-    return { content: extractFromHeading(lines, headings, i), matched: true };
+    if (!trimmedAlt) return "";
+    return match;
+  });
 }
-export function extractLinksAndImages(markdown) {
-    const imagePattern = /!\[[^\]]*\]\(([^)]+)\)/g;
-    const linkPattern = /(?<!!)\[[^\]]*\]\(([^)]+)\)/g;
-    const images = new Set();
-    const links = new Set();
-    let match;
-    // Extract images first
-    while ((match = imagePattern.exec(markdown)) !== null) {
-        images.add(match[1]);
+function resolveRelativeUrls(markdown, baseUrl) {
+  if (!markdown || !baseUrl) return markdown;
+  const rewrite = (path) => {
+    const trimmed = path.trim();
+    if (!trimmed) return path;
+    if (/^(?:https?:|mailto:|tel:|javascript:|data:)/i.test(trimmed)) return path;
+    if (trimmed.startsWith("#")) {
+      try {
+        return new URL(trimmed, baseUrl).href;
+      } catch {
+        return path;
+      }
+    }
+    if (trimmed.startsWith("//")) {
+      try {
+        const base = new URL(baseUrl);
+        return `${base.protocol}${trimmed}`;
+      } catch {
+        return path;
+      }
     }
-    // Extract links (non-image)
-    while ((match = linkPattern.exec(markdown)) !== null) {
-        links.add(match[1]);
+    try {
+      return new URL(trimmed, baseUrl).href;
+    } catch {
+      return path;
     }
-    return { links: Array.from(links), images: Array.from(images) };
+  };
+  let result = markdown.replace(
+    /(!\[[^\]]*\]\()([^)\s]+)(\s*(?:"[^"]*")?\))/g,
+    (_m, open, path, close) => `${open}${rewrite(path)}${close}`
+  );
+  result = result.replace(
+    /(^|[^!])(\[[^\]]*\]\()([^)\s]+)(\s*(?:"[^"]*")?\))/g,
+    (_m, pre, open, path, close) => `${pre}${open}${rewrite(path)}${close}`
+  );
+  return result;
 }
+export {
+  buildTurndown,
+  extractLinksAndImages,
+  extractSection,
+  filterDecorativeImages,
+  htmlToMarkdown,
+  lineStartCharOffsets,
+  parseHeadings,
+  resolveRelativeUrls
+};
 //# sourceMappingURL=markdown.js.map