@staticn0va/wigolo 0.1.0 → 0.1.1

package/dist/extraction/boilerplate.js

@@ -0,0 +1,52 @@
+const BOILERPLATE_TEXT_EQUALITY = [
+  "was this helpful?",
+  "send",
+  "edit this page",
+  "edit on github",
+  "suggest changes",
+  "skip to main content",
+  "on this page"
+];
+const BOILERPLATE_TEXT_PATTERNS = [
+  /^\s*last updated on .+$/i
+];
+const BOILERPLATE_SELECTORS = [
+  '[class*="feedback"]',
+  '[class*="edit-page"]',
+  '[aria-label*="Edit"]',
+  'footer[class*="docs"]',
+  '[class*="sticky-cta"]',
+  'main [role="banner"]',
+  '[role="navigation"]',
+  '[class*="sidebar"]',
+  '[data-collection="docs"]'
+];
+function stripBoilerplateMarkdown(md) {
+  if (!md) return md;
+  const lines = md.split("\n");
+  const kept = lines.filter((line) => {
+    const t = line.trim().toLowerCase();
+    if (!t) return true;
+    if (BOILERPLATE_TEXT_EQUALITY.includes(t)) return false;
+    return !BOILERPLATE_TEXT_PATTERNS.some((re) => re.test(line));
+  });
+  return kept.join("\n").replace(/\n{3,}/g, "\n\n");
+}
+function stripBoilerplateDom(document) {
+  for (const sel of BOILERPLATE_SELECTORS) {
+    const nodes = document.querySelectorAll(sel);
+    const list = [];
+    for (let i = 0; i < nodes.length; i++) list.push(nodes[i]);
+    for (const el of list) {
+      el.parentNode?.removeChild(el);
+    }
+  }
+}
+export {
+  BOILERPLATE_SELECTORS,
+  BOILERPLATE_TEXT_EQUALITY,
+  BOILERPLATE_TEXT_PATTERNS,
+  stripBoilerplateDom,
+  stripBoilerplateMarkdown
+};
+//# sourceMappingURL=boilerplate.js.map

package/dist/extraction/boilerplate.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/extraction/boilerplate.ts"],"sourcesContent":["export const BOILERPLATE_TEXT_EQUALITY: ReadonlyArray<string> = [\n  'was this helpful?',\n  'send',\n  'edit this page',\n  'edit on github',\n  'suggest changes',\n  'skip to main content',\n  'on this page',\n];\n\nexport const BOILERPLATE_TEXT_PATTERNS: ReadonlyArray<RegExp> = [\n  /^\\s*last updated on .+$/i,\n];\n\nexport const BOILERPLATE_SELECTORS: ReadonlyArray<string> = [\n  '[class*=\"feedback\"]',\n  '[class*=\"edit-page\"]',\n  '[aria-label*=\"Edit\"]',\n  'footer[class*=\"docs\"]',\n  '[class*=\"sticky-cta\"]',\n  'main [role=\"banner\"]',\n  '[role=\"navigation\"]',\n  '[class*=\"sidebar\"]',\n  '[data-collection=\"docs\"]',\n];\n\nexport interface BoilerplateDocument {\n  querySelectorAll(selector: string): ArrayLike<BoilerplateElement>;\n}\n\ninterface BoilerplateElement {\n  parentNode: { removeChild(child: BoilerplateElement): void } | null;\n}\n\nexport function stripBoilerplateMarkdown(md: string): string {\n  if (!md) return md;\n  const lines = md.split('\\n');\n  const kept = lines.filter((line) => {\n    const t = line.trim().toLowerCase();\n    if (!t) return true;\n    if (BOILERPLATE_TEXT_EQUALITY.includes(t)) return false;\n    return !BOILERPLATE_TEXT_PATTERNS.some((re) => re.test(line));\n  });\n  return kept.join('\\n').replace(/\\n{3,}/g, '\\n\\n');\n}\n\nexport function stripBoilerplateDom(document: BoilerplateDocument): void {\n  for (const sel of BOILERPLATE_SELECTORS) {\n    const nodes = document.querySelectorAll(sel);\n    const list: BoilerplateElement[] = [];\n    for (let i = 0; i < nodes.length; i++) list.push(nodes[i]);\n    for (const el of list) {\n      el.parentNode?.removeChild(el);\n    }\n  }\n}\n"],"mappings":"AAAO,MAAM,4BAAmD;AAAA,EAC9D;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,MAAM,4BAAmD;AAAA,EAC9D;AACF;AAEO,MAAM,wBAA+C;AAAA,EAC1D;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAUO,SAAS,yBAAyB,IAAoB;AAC3D,MAAI,CAAC,GAAI,QAAO;AAChB,QAAM,QAAQ,GAAG,MAAM,IAAI;AAC3B,QAAM,OAAO,MAAM,OAAO,CAAC,SAAS;AAClC,UAAM,IAAI,KAAK,KAAK,EAAE,YAAY;AAClC,QAAI,CAAC,EAAG,QAAO;AACf,QAAI,0BAA0B,SAAS,CAAC,EAAG,QAAO;AAClD,WAAO,CAAC,0BAA0B,KAAK,CAAC,OAAO,GAAG,KAAK,IAAI,CAAC;AAAA,EAC9D,CAAC;AACD,SAAO,KAAK,KAAK,IAAI,EAAE,QAAQ,WAAW,MAAM;AAClD;AAEO,SAAS,oBAAoB,UAAqC;AACvE,aAAW,OAAO,uBAAuB;AACvC,UAAM,QAAQ,SAAS,iBAAiB,GAAG;AAC3C,UAAM,OAA6B,CAAC;AACpC,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,IAAK,MAAK,KAAK,MAAM,CAAC,CAAC;AACzD,eAAW,MAAM,MAAM;AACrB,SAAG,YAAY,YAAY,EAAE;AAAA,IAC/B;AAAA,EACF;AACF;","names":[]}

package/dist/extraction/defuddle.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"defuddle.d.ts","sourceRoot":"","sources":["../../src/extraction/defuddle.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIpD,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAoBjG"}
+{"version":3,"file":"defuddle.d.ts","sourceRoot":"","sources":["../../src/extraction/defuddle.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAKpD,wBAAsB,eAAe,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CAsBjG"}

package/dist/extraction/defuddle.js

@@ -1,26 +1,30 @@
-import { Defuddle } from 'defuddle/node';
+import { Defuddle } from "defuddle/node";
+import { htmlToMarkdown } from "./markdown.js";
 const MIN_CONTENT_THRESHOLD = 100;
-export async function defuddleExtract(html, url) {
-    try {
-        const result = await Defuddle(html, url, { markdown: true });
-        if (!result.content || result.content.length < MIN_CONTENT_THRESHOLD)
-            return null;
-        return {
-            title: result.title ?? '',
-            markdown: result.content,
-            metadata: {
-                description: result.description || undefined,
-                author: result.author || undefined,
-                date: result.published || undefined,
-                language: result.language || undefined,
-            },
-            links: [],
-            images: [],
-            extractor: 'defuddle',
-        };
-    }
-    catch {
-        return null;
-    }
+async function defuddleExtract(html, url) {
+  try {
+    const result = await Defuddle(html, url);
+    if (!result.content) return null;
+    const markdown = htmlToMarkdown(result.content);
+    if (markdown.length < MIN_CONTENT_THRESHOLD) return null;
+    return {
+      title: result.title ?? "",
+      markdown,
+      metadata: {
+        description: result.description || void 0,
+        author: result.author || void 0,
+        date: result.published || void 0,
+        language: result.language || void 0
+      },
+      links: [],
+      images: [],
+      extractor: "defuddle"
+    };
+  } catch {
+    return null;
+  }
 }
+export {
+  defuddleExtract
+};
 //# sourceMappingURL=defuddle.js.map

package/dist/extraction/defuddle.js.map

@@ -1 +1 @@
-{"version":3,"file":"defuddle.js","sourceRoot":"","sources":["../../src/extraction/defuddle.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAGzC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAElC,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAY,EAAE,GAAW;IAC7D,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,qBAAqB;YAAE,OAAO,IAAI,CAAC;QAClF,OAAO;YACL,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,EAAE;YACzB,QAAQ,EAAE,MAAM,CAAC,OAAO;YACxB,QAAQ,EAAE;gBACR,WAAW,EAAE,MAAM,CAAC,WAAW,IAAI,SAAS;gBAC5C,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS;gBAClC,IAAI,EAAE,MAAM,CAAC,SAAS,IAAI,SAAS;gBACnC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,SAAS;aACvC;YACD,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,SAAS,EAAE,UAAU;SACtB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}
+{"version":3,"sources":["../../src/extraction/defuddle.ts"],"sourcesContent":["import { Defuddle } from 'defuddle/node';\nimport type { ExtractionResult } from '../types.js';\nimport { htmlToMarkdown } from './markdown.js';\n\nconst MIN_CONTENT_THRESHOLD = 100;\n\nexport async function defuddleExtract(html: string, url: string): Promise<ExtractionResult | null> {\n  try {\n    const result = await Defuddle(html, url);\n    if (!result.content) return null;\n    const markdown = htmlToMarkdown(result.content);\n    if (markdown.length < MIN_CONTENT_THRESHOLD) return null;\n    return {\n      title: result.title ?? '',\n      markdown,\n      metadata: {\n        description: result.description || undefined,\n        author: result.author || undefined,\n        date: result.published || undefined,\n        language: result.language || undefined,\n      },\n      links: [],\n      images: [],\n      extractor: 'defuddle',\n    };\n  } catch {\n    return null;\n  }\n}\n"],"mappings":"AAAA,SAAS,gBAAgB;AAEzB,SAAS,sBAAsB;AAE/B,MAAM,wBAAwB;AAE9B,eAAsB,gBAAgB,MAAc,KAA+C;AACjG,MAAI;AACF,UAAM,SAAS,MAAM,SAAS,MAAM,GAAG;AACvC,QAAI,CAAC,OAAO,QAAS,QAAO;AAC5B,UAAM,WAAW,eAAe,OAAO,OAAO;AAC9C,QAAI,SAAS,SAAS,sBAAuB,QAAO;AACpD,WAAO;AAAA,MACL,OAAO,OAAO,SAAS;AAAA,MACvB;AAAA,MACA,UAAU;AAAA,QACR,aAAa,OAAO,eAAe;AAAA,QACnC,QAAQ,OAAO,UAAU;AAAA,QACzB,MAAM,OAAO,aAAa;AAAA,QAC1B,UAAU,OAAO,YAAY;AAAA,MAC/B;AAAA,MACA,OAAO,CAAC;AAAA,MACR,QAAQ,CAAC;AAAA,MACT,WAAW;AAAA,IACb;AAAA,EACF,QAAQ;AACN,WAAO;AAAA,EACT;AACF;","names":[]}

package/dist/extraction/extract.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/extraction/extract.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAS3D,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,CA2B1D;AAED,wBAAgB,eAAe,CAC7B,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,OAAO,GAChB,MAAM,GAAG,MAAM,EAAE,CAUnB;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,CA6CvD"}
+{"version":3,"file":"extract.d.ts","sourceRoot":"","sources":["../../src/extraction/extract.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAS3D,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,CAiC1D;AAED,wBAAgB,eAAe,CAC7B,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,OAAO,GAChB,MAAM,GAAG,MAAM,EAAE,CAUnB;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,CA6CvD"}

package/dist/extraction/extract.js

@@ -1,83 +1,83 @@
-import { parseHTML } from 'linkedom';
+import { parseHTML } from "linkedom";
 function getMetaContent(doc, nameOrProperty) {
-    const el = doc.querySelector(`meta[name="${nameOrProperty}"]`) ??
-        doc.querySelector(`meta[property="${nameOrProperty}"]`);
-    return el?.getAttribute('content') ?? undefined;
+  const el = doc.querySelector(`meta[name="${nameOrProperty}"]`) ?? doc.querySelector(`meta[property="${nameOrProperty}"]`);
+  return el?.getAttribute("content") ?? void 0;
 }
-export function extractMetadata(html) {
-    const { document: doc } = parseHTML(html);
-    const result = {};
-    const title = doc.querySelector('title')?.textContent?.trim();
-    if (title)
-        result.title = title;
-    const description = getMetaContent(doc, 'description') ?? getMetaContent(doc, 'og:description');
-    if (description)
-        result.description = description;
-    const author = getMetaContent(doc, 'author');
-    if (author)
-        result.author = author;
-    const date = getMetaContent(doc, 'date') ?? getMetaContent(doc, 'article:published_time');
-    if (date)
-        result.date = date;
-    const keywords = getMetaContent(doc, 'keywords');
-    if (keywords) {
-        result.keywords = keywords.split(',').map((k) => k.trim()).filter(Boolean);
-    }
-    const ogImage = getMetaContent(doc, 'og:image');
-    if (ogImage)
-        result.og_image = ogImage;
-    return result;
+function extractMetadata(html) {
+  const { document: doc } = parseHTML(html);
+  const result = {};
+  const title = doc.querySelector("title")?.textContent?.trim();
+  if (title) result.title = title;
+  const description = getMetaContent(doc, "description") ?? getMetaContent(doc, "og:description");
+  if (description) result.description = description;
+  const author = getMetaContent(doc, "author");
+  if (author) result.author = author;
+  const date = getMetaContent(doc, "date") ?? getMetaContent(doc, "article:published_time");
+  if (date) result.date = date;
+  const keywords = getMetaContent(doc, "keywords");
+  if (keywords) {
+    result.keywords = keywords.split(",").map((k) => k.trim()).filter(Boolean);
+  }
+  const ogImage = getMetaContent(doc, "og:image");
+  if (ogImage) result.og_image = ogImage;
+  const ogType = getMetaContent(doc, "og:type");
+  if (ogType) result.og_type = ogType;
+  const canonical = doc.querySelector('link[rel="canonical"]')?.getAttribute("href");
+  if (canonical) result.canonical_url = canonical;
+  return result;
 }
-export function extractSelector(html, selector, multiple) {
-    const { document: doc } = parseHTML(html);
-    if (multiple) {
-        const elements = doc.querySelectorAll(selector);
-        return Array.from(elements).map((el) => (el.textContent ?? '').trim());
-    }
-    const el = doc.querySelector(selector);
-    return el ? (el.textContent ?? '').trim() : '';
+function extractSelector(html, selector, multiple) {
+  const { document: doc } = parseHTML(html);
+  if (multiple) {
+    const elements = doc.querySelectorAll(selector);
+    return Array.from(elements).map((el2) => (el2.textContent ?? "").trim());
+  }
+  const el = doc.querySelector(selector);
+  return el ? (el.textContent ?? "").trim() : "";
 }
-export function extractTables(html) {
-    const { document: doc } = parseHTML(html);
-    const tables = doc.querySelectorAll('table');
-    if (tables.length === 0)
-        return [];
-    return Array.from(tables).map((table) => {
-        const caption = table.querySelector('caption')?.textContent?.trim() || undefined;
-        const thElements = table.querySelectorAll('thead th');
-        let headers;
-        let bodyRows;
-        if (thElements.length > 0) {
-            headers = Array.from(thElements).map((th) => (th.textContent ?? '').trim());
-            bodyRows = Array.from(table.querySelectorAll('tbody tr'));
-            if (bodyRows.length === 0) {
-                const allRows = Array.from(table.querySelectorAll('tr'));
-                bodyRows = allRows.slice(1);
-            }
-        }
-        else {
-            const allRows = Array.from(table.querySelectorAll('tr'));
-            const firstRow = allRows[0];
-            const firstRowThs = firstRow ? Array.from(firstRow.querySelectorAll('th')) : [];
-            if (firstRowThs.length > 0) {
-                headers = firstRowThs.map((th) => (th.textContent ?? '').trim());
-                bodyRows = allRows.slice(1);
-            }
-            else {
-                const cellCount = firstRow ? firstRow.querySelectorAll('td').length : 0;
-                headers = Array.from({ length: cellCount }, (_, i) => `col_${i + 1}`);
-                bodyRows = allRows;
-            }
-        }
-        const rows = bodyRows.map((row) => {
-            const cells = Array.from(row.querySelectorAll('td'));
-            const obj = {};
-            headers.forEach((header, i) => {
-                obj[header] = (cells[i]?.textContent ?? '').trim();
-            });
-            return obj;
-        });
-        return { caption, headers, rows };
+function extractTables(html) {
+  const { document: doc } = parseHTML(html);
+  const tables = doc.querySelectorAll("table");
+  if (tables.length === 0) return [];
+  return Array.from(tables).map((table) => {
+    const caption = table.querySelector("caption")?.textContent?.trim() || void 0;
+    const thElements = table.querySelectorAll("thead th");
+    let headers;
+    let bodyRows;
+    if (thElements.length > 0) {
+      headers = Array.from(thElements).map((th) => (th.textContent ?? "").trim());
+      bodyRows = Array.from(table.querySelectorAll("tbody tr"));
+      if (bodyRows.length === 0) {
+        const allRows = Array.from(table.querySelectorAll("tr"));
+        bodyRows = allRows.slice(1);
+      }
+    } else {
+      const allRows = Array.from(table.querySelectorAll("tr"));
+      const firstRow = allRows[0];
+      const firstRowThs = firstRow ? Array.from(firstRow.querySelectorAll("th")) : [];
+      if (firstRowThs.length > 0) {
+        headers = firstRowThs.map((th) => (th.textContent ?? "").trim());
+        bodyRows = allRows.slice(1);
+      } else {
+        const cellCount = firstRow ? firstRow.querySelectorAll("td").length : 0;
+        headers = Array.from({ length: cellCount }, (_, i) => `col_${i + 1}`);
+        bodyRows = allRows;
+      }
+    }
+    const rows = bodyRows.map((row) => {
+      const cells = Array.from(row.querySelectorAll("td"));
+      const obj = {};
+      headers.forEach((header, i) => {
+        obj[header] = (cells[i]?.textContent ?? "").trim();
+      });
+      return obj;
     });
+    return { caption, headers, rows };
+  });
 }
+export {
+  extractMetadata,
+  extractSelector,
+  extractTables
+};
 //# sourceMappingURL=extract.js.map

package/dist/extraction/extract.js.map

@@ -1 +1 @@
-{"version":3,"file":"extract.js","sourceRoot":"","sources":["../../src/extraction/extract.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAGrC,SAAS,cAAc,CAAC,GAAa,EAAE,cAAsB;IAC3D,MAAM,EAAE,GACN,GAAG,CAAC,aAAa,CAAC,cAAc,cAAc,IAAI,CAAC;QACnD,GAAG,CAAC,aAAa,CAAC,kBAAkB,cAAc,IAAI,CAAC,CAAC;IAC1D,OAAO,EAAE,EAAE,YAAY,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,IAAY;IAC1C,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAiB,EAAE,CAAC;IAEhC,MAAM,KAAK,GAAG,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;IAC9D,IAAI,KAAK;QAAE,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC;IAEhC,MAAM,WAAW,GACf,cAAc,CAAC,GAAG,EAAE,aAAa,CAAC,IAAI,cAAc,CAAC,GAAG,EAAE,gBAAgB,CAAC,CAAC;IAC9E,IAAI,WAAW;QAAE,MAAM,CAAC,WAAW,GAAG,WAAW,CAAC;IAElD,MAAM,MAAM,GAAG,cAAc,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC7C,IAAI,MAAM;QAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC;IAEnC,MAAM,IAAI,GACR,cAAc,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,cAAc,CAAC,GAAG,EAAE,wBAAwB,CAAC,CAAC;IAC/E,IAAI,IAAI;QAAE,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC;IAE7B,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IACjD,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC7E,CAAC;IAED,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IAChD,IAAI,OAAO;QAAE,MAAM,CAAC,QAAQ,GAAG,OAAO,CAAC;IAEvC,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,IAAY,EACZ,QAAgB,EAChB,QAAiB;IAEjB,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAE1C,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,QAAQ,GAAG,GAAG,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAChD,OAAO,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACzE,CAAC;IAED,MAAM,EAAE,GAAG,GAAG,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IACvC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AACjD,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,MAAM,GAAG,GAAG,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAEnC,OAAO,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACtC,MAAM,OAAO,GAAG,KAAK,CAAC,aAAa,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC;QAEjF,MAAM,UAAU,GAAG,KAAK,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC;QACtD,IAAI,OAAiB,CAAC;QACtB,IAAI,QAAmB,CAAC;QAExB,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAC5E,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC;YAC1D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;gBACzD,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;YACzD,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;YAC5B,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAEhF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3B,OAAO,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBACjE,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBACxE,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACtE,QAAQ,GAAG,OAAO,CAAC;YACrB,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAChC,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC;YACrD,MAAM,GAAG,GAA2B,EAAE,CAAC;YACvC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBAC5B,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YACrD,CAAC,CAAC,CAAC;YACH,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC;QAEH,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IACpC,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"sources":["../../src/extraction/extract.ts"],"sourcesContent":["import { parseHTML } from 'linkedom';\nimport type { MetadataData, TableData } from '../types.js';\n\nfunction getMetaContent(doc: Document, nameOrProperty: string): string | undefined {\n  const el =\n    doc.querySelector(`meta[name=\"${nameOrProperty}\"]`) ??\n    doc.querySelector(`meta[property=\"${nameOrProperty}\"]`);\n  return el?.getAttribute('content') ?? undefined;\n}\n\nexport function extractMetadata(html: string): MetadataData {\n  const { document: doc } = parseHTML(html);\n  const result: MetadataData = {};\n\n  const title = doc.querySelector('title')?.textContent?.trim();\n  if (title) result.title = title;\n\n  const description =\n    getMetaContent(doc, 'description') ?? getMetaContent(doc, 'og:description');\n  if (description) result.description = description;\n\n  const author = getMetaContent(doc, 'author');\n  if (author) result.author = author;\n\n  const date =\n    getMetaContent(doc, 'date') ?? getMetaContent(doc, 'article:published_time');\n  if (date) result.date = date;\n\n  const keywords = getMetaContent(doc, 'keywords');\n  if (keywords) {\n    result.keywords = keywords.split(',').map((k) => k.trim()).filter(Boolean);\n  }\n\n  const ogImage = getMetaContent(doc, 'og:image');\n  if (ogImage) result.og_image = ogImage;\n\n  const ogType = getMetaContent(doc, 'og:type');\n  if (ogType) result.og_type = ogType;\n\n  const canonical = doc.querySelector('link[rel=\"canonical\"]')?.getAttribute('href');\n  if (canonical) result.canonical_url = canonical;\n\n  return result;\n}\n\nexport function extractSelector(\n  html: string,\n  selector: string,\n  multiple: boolean,\n): string | string[] {\n  const { document: doc } = parseHTML(html);\n\n  if (multiple) {\n    const elements = doc.querySelectorAll(selector);\n    return Array.from(elements).map((el) => (el.textContent ?? '').trim());\n  }\n\n  const el = doc.querySelector(selector);\n  return el ? (el.textContent ?? '').trim() : '';\n}\n\nexport function extractTables(html: string): TableData[] {\n  const { document: doc } = parseHTML(html);\n  const tables = doc.querySelectorAll('table');\n  if (tables.length === 0) return [];\n\n  return Array.from(tables).map((table) => {\n    const caption = table.querySelector('caption')?.textContent?.trim() || undefined;\n\n    const thElements = table.querySelectorAll('thead th');\n    let headers: string[];\n    let bodyRows: Element[];\n\n    if (thElements.length > 0) {\n      headers = Array.from(thElements).map((th) => (th.textContent ?? '').trim());\n      bodyRows = Array.from(table.querySelectorAll('tbody tr'));\n      if (bodyRows.length === 0) {\n        const allRows = Array.from(table.querySelectorAll('tr'));\n        bodyRows = allRows.slice(1);\n      }\n    } else {\n      const allRows = Array.from(table.querySelectorAll('tr'));\n      const firstRow = allRows[0];\n      const firstRowThs = firstRow ? Array.from(firstRow.querySelectorAll('th')) : [];\n\n      if (firstRowThs.length > 0) {\n        headers = firstRowThs.map((th) => (th.textContent ?? '').trim());\n        bodyRows = allRows.slice(1);\n      } else {\n        const cellCount = firstRow ? firstRow.querySelectorAll('td').length : 0;\n        headers = Array.from({ length: cellCount }, (_, i) => `col_${i + 1}`);\n        bodyRows = allRows;\n      }\n    }\n\n    const rows = bodyRows.map((row) => {\n      const cells = Array.from(row.querySelectorAll('td'));\n      const obj: Record<string, string> = {};\n      headers.forEach((header, i) => {\n        obj[header] = (cells[i]?.textContent ?? '').trim();\n      });\n      return obj;\n    });\n\n    return { caption, headers, rows };\n  });\n}\n"],"mappings":"AAAA,SAAS,iBAAiB;AAG1B,SAAS,eAAe,KAAe,gBAA4C;AACjF,QAAM,KACJ,IAAI,cAAc,cAAc,cAAc,IAAI,KAClD,IAAI,cAAc,kBAAkB,cAAc,IAAI;AACxD,SAAO,IAAI,aAAa,SAAS,KAAK;AACxC;AAEO,SAAS,gBAAgB,MAA4B;AAC1D,QAAM,EAAE,UAAU,IAAI,IAAI,UAAU,IAAI;AACxC,QAAM,SAAuB,CAAC;AAE9B,QAAM,QAAQ,IAAI,cAAc,OAAO,GAAG,aAAa,KAAK;AAC5D,MAAI,MAAO,QAAO,QAAQ;AAE1B,QAAM,cACJ,eAAe,KAAK,aAAa,KAAK,eAAe,KAAK,gBAAgB;AAC5E,MAAI,YAAa,QAAO,cAAc;AAEtC,QAAM,SAAS,eAAe,KAAK,QAAQ;AAC3C,MAAI,OAAQ,QAAO,SAAS;AAE5B,QAAM,OACJ,eAAe,KAAK,MAAM,KAAK,eAAe,KAAK,wBAAwB;AAC7E,MAAI,KAAM,QAAO,OAAO;AAExB,QAAM,WAAW,eAAe,KAAK,UAAU;AAC/C,MAAI,UAAU;AACZ,WAAO,WAAW,SAAS,MAAM,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,OAAO;AAAA,EAC3E;AAEA,QAAM,UAAU,eAAe,KAAK,UAAU;AAC9C,MAAI,QAAS,QAAO,WAAW;AAE/B,QAAM,SAAS,eAAe,KAAK,SAAS;AAC5C,MAAI,OAAQ,QAAO,UAAU;AAE7B,QAAM,YAAY,IAAI,cAAc,uBAAuB,GAAG,aAAa,MAAM;AACjF,MAAI,UAAW,QAAO,gBAAgB;AAEtC,SAAO;AACT;AAEO,SAAS,gBACd,MACA,UACA,UACmB;AACnB,QAAM,EAAE,UAAU,IAAI,IAAI,UAAU,IAAI;AAExC,MAAI,UAAU;AACZ,UAAM,WAAW,IAAI,iBAAiB,QAAQ;AAC9C,WAAO,MAAM,KAAK,QAAQ,EAAE,IAAI,CAACA,SAAQA,IAAG,eAAe,IAAI,KAAK,CAAC;AAAA,EACvE;AAEA,QAAM,KAAK,IAAI,cAAc,QAAQ;AACrC,SAAO,MAAM,GAAG,eAAe,IAAI,KAAK,IAAI;AAC9C;AAEO,SAAS,cAAc,MAA2B;AACvD,QAAM,EAAE,UAAU,IAAI,IAAI,UAAU,IAAI;AACxC,QAAM,SAAS,IAAI,iBAAiB,OAAO;AAC3C,MAAI,OAAO,WAAW,EAAG,QAAO,CAAC;AAEjC,SAAO,MAAM,KAAK,MAAM,EAAE,IAAI,CAAC,UAAU;AACvC,UAAM,UAAU,MAAM,cAAc,SAAS,GAAG,aAAa,KAAK,KAAK;AAEvE,UAAM,aAAa,MAAM,iBAAiB,UAAU;AACpD,QAAI;AACJ,QAAI;AAEJ,QAAI,WAAW,SAAS,GAAG;AACzB,gBAAU,MAAM,KAAK,UAAU,EAAE,IAAI,CAAC,QAAQ,GAAG,eAAe,IAAI,KAAK,CAAC;AAC1E,iBAAW,MAAM,KAAK,MAAM,iBAAiB,UAAU,CAAC;AACxD,UAAI,SAAS,WAAW,GAAG;AACzB,cAAM,UAAU,MAAM,KAAK,MAAM,iBAAiB,IAAI,CAAC;AACvD,mBAAW,QAAQ,MAAM,CAAC;AAAA,MAC5B;AAAA,IACF,OAAO;AACL,YAAM,UAAU,MAAM,KAAK,MAAM,iBAAiB,IAAI,CAAC;AACvD,YAAM,WAAW,QAAQ,CAAC;AAC1B,YAAM,cAAc,WAAW,MAAM,KAAK,SAAS,iBAAiB,IAAI,CAAC,IAAI,CAAC;AAE9E,UAAI,YAAY,SAAS,GAAG;AAC1B,kBAAU,YAAY,IAAI,CAAC,QAAQ,GAAG,eAAe,IAAI,KAAK,CAAC;AAC/D,mBAAW,QAAQ,MAAM,CAAC;AAAA,MAC5B,OAAO;AACL,cAAM,YAAY,WAAW,SAAS,iBAAiB,IAAI,EAAE,SAAS;AACtE,kBAAU,MAAM,KAAK,EAAE,QAAQ,UAAU,GAAG,CAAC,GAAG,MAAM,OAAO,IAAI,CAAC,EAAE;AACpE,mBAAW;AAAA,MACb;AAAA,IACF;AAEA,UAAM,OAAO,SAAS,IAAI,CAAC,QAAQ;AACjC,YAAM,QAAQ,MAAM,KAAK,IAAI,iBAAiB,IAAI,CAAC;AACnD,YAAM,MAA8B,CAAC;AACrC,cAAQ,QAAQ,CAAC,QAAQ,MAAM;AAC7B,YAAI,MAAM,KAAK,MAAM,CAAC,GAAG,eAAe,IAAI,KAAK;AAAA,MACnD,CAAC;AACD,aAAO;AAAA,IACT,CAAC;AAED,WAAO,EAAE,SAAS,SAAS,KAAK;AAAA,EAClC,CAAC;AACH;","names":["el"]}

package/dist/extraction/jsonld.js

@@ -1,64 +1,60 @@
-import { parseHTML } from 'linkedom';
-import { createLogger } from '../logger.js';
-const log = createLogger('jsonld');
-export function extractJsonLd(html) {
-    const { document: doc } = parseHTML(html);
-    const scripts = doc.querySelectorAll('script[type="application/ld+json"]');
-    const results = [];
-    for (const script of scripts) {
-        try {
-            const text = script.textContent?.trim();
-            if (!text)
-                continue;
-            const parsed = JSON.parse(text);
-            if (Array.isArray(parsed)) {
-                results.push(...parsed);
-            }
-            else if (parsed['@graph'] && Array.isArray(parsed['@graph'])) {
-                results.push(...parsed['@graph']);
-            }
-            else {
-                results.push(parsed);
-            }
-        }
-        catch (err) {
-            log.debug('Failed to parse JSON-LD block', { error: String(err) });
-        }
+import { parseHTML } from "linkedom";
+import { createLogger } from "../logger.js";
+const log = createLogger("jsonld");
+function extractJsonLd(html) {
+  const { document: doc } = parseHTML(html);
+  const scripts = doc.querySelectorAll('script[type="application/ld+json"]');
+  const results = [];
+  for (const script of scripts) {
+    try {
+      const text = script.textContent?.trim();
+      if (!text) continue;
+      const parsed = JSON.parse(text);
+      if (Array.isArray(parsed)) {
+        results.push(...parsed);
+      } else if (parsed["@graph"] && Array.isArray(parsed["@graph"])) {
+        results.push(...parsed["@graph"]);
+      } else {
+        results.push(parsed);
+      }
+    } catch (err) {
+      log.warn("Failed to parse JSON-LD block", { error: String(err) });
     }
-    return results;
+  }
+  return results;
 }
-export function matchJsonLdToSchema(jsonLdBlocks, schema) {
-    if (!schema.properties || jsonLdBlocks.length === 0)
-        return {};
-    const result = {};
-    const flattened = flattenJsonLd(jsonLdBlocks);
-    for (const fieldName of Object.keys(schema.properties)) {
-        if (flattened[fieldName] !== undefined) {
-            result[fieldName] = flattened[fieldName];
-        }
+function matchJsonLdToSchema(jsonLdBlocks, schema) {
+  if (!schema.properties || jsonLdBlocks.length === 0) return {};
+  const result = {};
+  const flattened = flattenJsonLd(jsonLdBlocks);
+  for (const fieldName of Object.keys(schema.properties)) {
+    if (flattened[fieldName] !== void 0) {
+      result[fieldName] = flattened[fieldName];
     }
-    return result;
+  }
+  return result;
 }
 function flattenJsonLd(blocks) {
-    const flat = {};
-    for (const block of blocks) {
-        flattenObject(block, flat);
-    }
-    return flat;
+  const flat = {};
+  for (const block of blocks) {
+    flattenObject(block, flat);
+  }
+  return flat;
 }
 function flattenObject(obj, target) {
-    for (const [key, value] of Object.entries(obj)) {
-        if (key.startsWith('@'))
-            continue;
-        // First-wins: earlier blocks and shallower keys take priority
-        if (!(key in target)) {
-            if (typeof value === 'object' && value !== null && !Array.isArray(value)) {
-                flattenObject(value, target);
-            }
-            else {
-                target[key] = value;
-            }
-        }
+  for (const [key, value] of Object.entries(obj)) {
+    if (key.startsWith("@")) continue;
+    if (!(key in target)) {
+      if (typeof value === "object" && value !== null && !Array.isArray(value)) {
+        flattenObject(value, target);
+      } else {
+        target[key] = value;
+      }
     }
+  }
 }
+export {
+  extractJsonLd,
+  matchJsonLdToSchema
+};
 //# sourceMappingURL=jsonld.js.map

package/dist/extraction/jsonld.js.map

@@ -1 +1 @@
-{"version":3,"file":"jsonld.js","sourceRoot":"","sources":["../../src/extraction/jsonld.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,MAAM,GAAG,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;AAEnC,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,GAAG,CAAC,gBAAgB,CAAC,oCAAoC,CAAC,CAAC;IAC3E,MAAM,OAAO,GAA8B,EAAE,CAAC;IAE9C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,MAAM,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;YACxC,IAAI,CAAC,IAAI;gBAAE,SAAS;YAEpB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAEhC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1B,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;YAC1B,CAAC;iBAAM,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;gBAC/D,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,KAAK,CAAC,+BAA+B,EAAE,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,YAAuC,EACvC,MAAkB;IAElB,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAE/D,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,MAAM,SAAS,GAAG,aAAa,CAAC,YAAY,CAAC,CAAC;IAE9C,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QACvD,IAAI,SAAS,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE,CAAC;YACvC,MAAM,CAAC,SAAS,CAAC,GAAG,SAAS,CAAC,SAAS,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,aAAa,CACpB,MAAiC;IAEjC,MAAM,IAAI,GAA4B,EAAE,CAAC;IAEzC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,aAAa,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CACpB,GAA4B,EAC5B,MAA+B;IAE/B,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/C,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAElC,8DAA8D;QAC9D,IAAI,CAAC,CAAC,GAAG,IAAI,MAAM,CAAC,EAAE,CAAC;YACrB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzE,aAAa,CAAC,KAAgC,EAAE,MAAM,CAAC,CAAC;YAC1D,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;YACtB,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
+{"version":3,"sources":["../../src/extraction/jsonld.ts"],"sourcesContent":["import { parseHTML } from 'linkedom';\nimport { createLogger } from '../logger.js';\nimport type { JsonSchema } from './schema.js';\n\nconst log = createLogger('jsonld');\n\nexport function extractJsonLd(html: string): Record<string, unknown>[] {\n  const { document: doc } = parseHTML(html);\n  const scripts = doc.querySelectorAll('script[type=\"application/ld+json\"]');\n  const results: Record<string, unknown>[] = [];\n\n  for (const script of scripts) {\n    try {\n      const text = script.textContent?.trim();\n      if (!text) continue;\n\n      const parsed = JSON.parse(text);\n\n      if (Array.isArray(parsed)) {\n        results.push(...parsed);\n      } else if (parsed['@graph'] && Array.isArray(parsed['@graph'])) {\n        results.push(...parsed['@graph']);\n      } else {\n        results.push(parsed);\n      }\n    } catch (err) {\n      log.warn('Failed to parse JSON-LD block', { error: String(err) });\n    }\n  }\n\n  return results;\n}\n\nexport function matchJsonLdToSchema(\n  jsonLdBlocks: Record<string, unknown>[],\n  schema: JsonSchema,\n): Record<string, unknown> {\n  if (!schema.properties || jsonLdBlocks.length === 0) return {};\n\n  const result: Record<string, unknown> = {};\n  const flattened = flattenJsonLd(jsonLdBlocks);\n\n  for (const fieldName of Object.keys(schema.properties)) {\n    if (flattened[fieldName] !== undefined) {\n      result[fieldName] = flattened[fieldName];\n    }\n  }\n\n  return result;\n}\n\nfunction flattenJsonLd(\n  blocks: Record<string, unknown>[],\n): Record<string, unknown> {\n  const flat: Record<string, unknown> = {};\n\n  for (const block of blocks) {\n    flattenObject(block, flat);\n  }\n\n  return flat;\n}\n\nfunction flattenObject(\n  obj: Record<string, unknown>,\n  target: Record<string, unknown>,\n): void {\n  for (const [key, value] of Object.entries(obj)) {\n    if (key.startsWith('@')) continue;\n\n    // First-wins: earlier blocks and shallower keys take priority\n    if (!(key in target)) {\n      if (typeof value === 'object' && value !== null && !Array.isArray(value)) {\n        flattenObject(value as Record<string, unknown>, target);\n      } else {\n        target[key] = value;\n      }\n    }\n  }\n}\n"],"mappings":"AAAA,SAAS,iBAAiB;AAC1B,SAAS,oBAAoB;AAG7B,MAAM,MAAM,aAAa,QAAQ;AAE1B,SAAS,cAAc,MAAyC;AACrE,QAAM,EAAE,UAAU,IAAI,IAAI,UAAU,IAAI;AACxC,QAAM,UAAU,IAAI,iBAAiB,oCAAoC;AACzE,QAAM,UAAqC,CAAC;AAE5C,aAAW,UAAU,SAAS;AAC5B,QAAI;AACF,YAAM,OAAO,OAAO,aAAa,KAAK;AACtC,UAAI,CAAC,KAAM;AAEX,YAAM,SAAS,KAAK,MAAM,IAAI;AAE9B,UAAI,MAAM,QAAQ,MAAM,GAAG;AACzB,gBAAQ,KAAK,GAAG,MAAM;AAAA,MACxB,WAAW,OAAO,QAAQ,KAAK,MAAM,QAAQ,OAAO,QAAQ,CAAC,GAAG;AAC9D,gBAAQ,KAAK,GAAG,OAAO,QAAQ,CAAC;AAAA,MAClC,OAAO;AACL,gBAAQ,KAAK,MAAM;AAAA,MACrB;AAAA,IACF,SAAS,KAAK;AACZ,UAAI,KAAK,iCAAiC,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAAA,IAClE;AAAA,EACF;AAEA,SAAO;AACT;AAEO,SAAS,oBACd,cACA,QACyB;AACzB,MAAI,CAAC,OAAO,cAAc,aAAa,WAAW,EAAG,QAAO,CAAC;AAE7D,QAAM,SAAkC,CAAC;AACzC,QAAM,YAAY,cAAc,YAAY;AAE5C,aAAW,aAAa,OAAO,KAAK,OAAO,UAAU,GAAG;AACtD,QAAI,UAAU,SAAS,MAAM,QAAW;AACtC,aAAO,SAAS,IAAI,UAAU,SAAS;AAAA,IACzC;AAAA,EACF;AAEA,SAAO;AACT;AAEA,SAAS,cACP,QACyB;AACzB,QAAM,OAAgC,CAAC;AAEvC,aAAW,SAAS,QAAQ;AAC1B,kBAAc,OAAO,IAAI;AAAA,EAC3B;AAEA,SAAO;AACT;AAEA,SAAS,cACP,KACA,QACM;AACN,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,GAAG,GAAG;AAC9C,QAAI,IAAI,WAAW,GAAG,EAAG;AAGzB,QAAI,EAAE,OAAO,SAAS;AACpB,UAAI,OAAO,UAAU,YAAY,UAAU,QAAQ,CAAC,MAAM,QAAQ,KAAK,GAAG;AACxE,sBAAc,OAAkC,MAAM;AAAA,MACxD,OAAO;AACL,eAAO,GAAG,IAAI;AAAA,MAChB;AAAA,IACF;AAAA,EACF;AACF;","names":[]}

package/dist/extraction/lang-hints.d.ts

@@ -0,0 +1,2 @@
+export declare function detectCodeLanguage(classAttr: string | null | undefined): string | null;
+//# sourceMappingURL=lang-hints.d.ts.map

package/dist/extraction/lang-hints.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lang-hints.d.ts","sourceRoot":"","sources":["../../src/extraction/lang-hints.ts"],"names":[],"mappings":"AAiBA,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,CAUtF"}

package/dist/extraction/lang-hints.js

@@ -0,0 +1,30 @@
+const ALIASES = {
+  typescript: "ts",
+  javascript: "js",
+  python: "py",
+  rust: "rs",
+  golang: "go",
+  shell: "sh"
+};
+const PATTERNS = [
+  /(?:^|\s)language-([a-z0-9+#-]+)/i,
+  /(?:^|\s)lang-([a-z0-9+#-]+)/i,
+  /(?:^|\s)hljs-([a-z0-9+#-]+)/i,
+  /(?:^|\s)prism-language-([a-z0-9+#-]+)/i,
+  /(?:^|\s)highlight-source-([a-z0-9+#-]+)/i
+];
+function detectCodeLanguage(classAttr) {
+  if (!classAttr) return null;
+  for (const re of PATTERNS) {
+    const m = classAttr.match(re);
+    if (m) {
+      const raw = m[1].toLowerCase();
+      return ALIASES[raw] ?? raw;
+    }
+  }
+  return null;
+}
+export {
+  detectCodeLanguage
+};
+//# sourceMappingURL=lang-hints.js.map

package/dist/extraction/lang-hints.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/extraction/lang-hints.ts"],"sourcesContent":["const ALIASES: Record<string, string> = {\n  typescript: 'ts',\n  javascript: 'js',\n  python: 'py',\n  rust: 'rs',\n  golang: 'go',\n  shell: 'sh',\n};\n\nconst PATTERNS = [\n  /(?:^|\\s)language-([a-z0-9+#-]+)/i,\n  /(?:^|\\s)lang-([a-z0-9+#-]+)/i,\n  /(?:^|\\s)hljs-([a-z0-9+#-]+)/i,\n  /(?:^|\\s)prism-language-([a-z0-9+#-]+)/i,\n  /(?:^|\\s)highlight-source-([a-z0-9+#-]+)/i,\n];\n\nexport function detectCodeLanguage(classAttr: string | null | undefined): string | null {\n  if (!classAttr) return null;\n  for (const re of PATTERNS) {\n    const m = classAttr.match(re);\n    if (m) {\n      const raw = m[1].toLowerCase();\n      return ALIASES[raw] ?? raw;\n    }\n  }\n  return null;\n}\n"],"mappings":"AAAA,MAAM,UAAkC;AAAA,EACtC,YAAY;AAAA,EACZ,YAAY;AAAA,EACZ,QAAQ;AAAA,EACR,MAAM;AAAA,EACN,QAAQ;AAAA,EACR,OAAO;AACT;AAEA,MAAM,WAAW;AAAA,EACf;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,SAAS,mBAAmB,WAAqD;AACtF,MAAI,CAAC,UAAW,QAAO;AACvB,aAAW,MAAM,UAAU;AACzB,UAAM,IAAI,UAAU,MAAM,EAAE;AAC5B,QAAI,GAAG;AACL,YAAM,MAAM,EAAE,CAAC,EAAE,YAAY;AAC7B,aAAO,QAAQ,GAAG,KAAK;AAAA,IACzB;AAAA,EACF;AACA,SAAO;AACT;","names":[]}

package/dist/extraction/llm-fallback.d.ts

@@ -0,0 +1,17 @@
+import type { LLMExtractResult } from '../integrations/cloud/llm/types.js';
+export interface LLMFallbackBudget {
+    remaining: number;
+}
+export interface LLMFallbackInput {
+    html: string;
+    jsonSchema: Record<string, unknown>;
+    partial: Record<string, unknown>;
+    missing: string[];
+    signal?: AbortSignal;
+    budget?: LLMFallbackBudget;
+}
+export interface LLMFallbackResult extends LLMExtractResult {
+    warnings: string[];
+}
+export declare function extractWithLLM(input: LLMFallbackInput): Promise<LLMFallbackResult>;
+//# sourceMappingURL=llm-fallback.d.ts.map

package/dist/extraction/llm-fallback.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm-fallback.d.ts","sourceRoot":"","sources":["../../src/extraction/llm-fallback.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,gBAAgB,EAAe,MAAM,oCAAoC,CAAC;AAKxF,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACpC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IACjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,MAAM,CAAC,EAAE,iBAAiB,CAAC;CAC5B;AAED,MAAM,WAAW,iBAAkB,SAAQ,gBAAgB;IACzD,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAeD,wBAAsB,cAAc,CAClC,KAAK,EAAE,gBAAgB,GACtB,OAAO,CAAC,iBAAiB,CAAC,CAsF5B"}