@staticn0va/wigolo 0.1.0 → 0.1.1

package/dist/extraction/site-extractors/stackoverflow.js

@@ -1,88 +1,79 @@
-import { parseHTML } from 'linkedom';
-import TurndownService from 'turndown';
-const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+import { parseHTML } from "linkedom";
+import { htmlToMarkdown } from "../markdown.js";
 function parseVotes(el) {
-    if (!el)
-        return 0;
-    const voteEl = el.querySelector('.js-vote-count');
-    const val = voteEl?.getAttribute('data-value') ?? voteEl?.textContent?.trim() ?? '0';
-    return parseInt(val, 10) || 0;
+  if (!el) return 0;
+  const voteEl = el.querySelector(".js-vote-count");
+  const val = voteEl?.getAttribute("data-value") ?? voteEl?.textContent?.trim() ?? "0";
+  return parseInt(val, 10) || 0;
 }
 function parseAnswers(document) {
-    const answerEls = document.querySelectorAll('#answers .answer');
-    const answers = [];
-    for (const el of Array.from(answerEls)) {
-        const accepted = el.classList.contains('accepted-answer');
-        const votes = parseVotes(el);
-        const bodyEl = el.querySelector('.s-prose, .js-post-body, .post-text');
-        const bodyHtml = bodyEl ? bodyEl.innerHTML : '';
-        answers.push({ accepted, votes, bodyHtml });
-    }
-    return answers;
+  const answerEls = document.querySelectorAll("#answers .answer");
+  const answers = [];
+  for (const el of Array.from(answerEls)) {
+    const accepted = el.classList.contains("accepted-answer");
+    const votes = parseVotes(el);
+    const bodyEl = el.querySelector(".s-prose, .js-post-body, .post-text");
+    const bodyHtml = bodyEl ? bodyEl.innerHTML : "";
+    answers.push({ accepted, votes, bodyHtml });
+  }
+  return answers;
 }
 function buildMarkdown(title, tags, votes, questionHtml, answers) {
-    const tagLine = `Tags: ${tags.join(', ')} | Votes: ${votes}`;
-    const questionMd = turndown.turndown(questionHtml).trim();
-    const sections = [
-        `# ${title}`,
-        tagLine,
-        '',
-        questionMd,
-    ];
-    const accepted = answers.filter((a) => a.accepted);
-    const others = answers.filter((a) => !a.accepted).sort((a, b) => b.votes - a.votes);
-    const ordered = [...accepted, ...others];
-    for (const answer of ordered) {
-        const heading = answer.accepted
-            ? `## Accepted Answer (Votes: ${answer.votes})`
-            : `## Answer (Votes: ${answer.votes})`;
-        const bodyMd = turndown.turndown(answer.bodyHtml).trim();
-        sections.push('---', '', heading, '', bodyMd);
-    }
-    return sections.join('\n\n');
+  const tagLine = `Tags: ${tags.join(", ")} | Votes: ${votes}`;
+  const questionMd = htmlToMarkdown(questionHtml).trim();
+  const sections = [
+    `# ${title}`,
+    tagLine,
+    "",
+    questionMd
+  ];
+  const accepted = answers.filter((a) => a.accepted);
+  const others = answers.filter((a) => !a.accepted).sort((a, b) => b.votes - a.votes);
+  const ordered = [...accepted, ...others];
+  for (const answer of ordered) {
+    const heading = answer.accepted ? `## Accepted Answer (Votes: ${answer.votes})` : `## Answer (Votes: ${answer.votes})`;
+    const bodyMd = htmlToMarkdown(answer.bodyHtml).trim();
+    sections.push("---", "", heading, "", bodyMd);
+  }
+  return sections.join("\n\n");
 }
-export const stackoverflowExtractor = {
-    name: 'stackoverflow',
-    canHandle(url) {
-        try {
-            const hostname = new URL(url).hostname;
-            return hostname === 'stackoverflow.com' ||
-                hostname.endsWith('.stackoverflow.com') ||
-                hostname === 'stackexchange.com' ||
-                hostname.endsWith('.stackexchange.com');
-        }
-        catch {
-            return false;
-        }
-    },
-    extract(html, url) {
-        if (!html)
-            return null;
-        const { document } = parseHTML(html);
-        const titleEl = document.querySelector('.question-hyperlink');
-        if (!titleEl)
-            return null;
-        const title = titleEl.textContent?.trim() ?? '';
-        if (!title)
-            return null;
-        const questionBodyEl = document.querySelector('#question .s-prose, #question .js-post-body, #question .post-text');
-        if (!questionBodyEl)
-            return null;
-        const questionHtml = questionBodyEl.innerHTML;
-        const tagEls = document.querySelectorAll('.js-post-tag-list-wrapper .post-tag, .post-taglist .post-tag');
-        const tags = Array.from(tagEls).map((el) => el.textContent?.trim() ?? '').filter(Boolean);
-        const questionEl = document.querySelector('#question');
-        const votes = parseVotes(questionEl);
-        const answers = parseAnswers(document);
-        const markdown = buildMarkdown(title, tags, votes, questionHtml, answers);
-        return {
-            title,
-            markdown,
-            metadata: {},
-            links: [],
-            images: [],
-            extractor: 'site-specific',
-        };
-    },
+const stackoverflowExtractor = {
+  name: "stackoverflow",
+  canHandle(url) {
+    try {
+      const hostname = new URL(url).hostname;
+      return hostname === "stackoverflow.com" || hostname.endsWith(".stackoverflow.com") || hostname === "stackexchange.com" || hostname.endsWith(".stackexchange.com");
+    } catch {
+      return false;
+    }
+  },
+  extract(html, _url) {
+    if (!html) return null;
+    const { document } = parseHTML(html);
+    const titleEl = document.querySelector(".question-hyperlink");
+    if (!titleEl) return null;
+    const title = titleEl.textContent?.trim() ?? "";
+    if (!title) return null;
+    const questionBodyEl = document.querySelector("#question .s-prose, #question .js-post-body, #question .post-text");
+    if (!questionBodyEl) return null;
+    const questionHtml = questionBodyEl.innerHTML;
+    const tagEls = document.querySelectorAll(".js-post-tag-list-wrapper .post-tag, .post-taglist .post-tag");
+    const tags = Array.from(tagEls).map((el) => el.textContent?.trim() ?? "").filter(Boolean);
+    const questionEl = document.querySelector("#question");
+    const votes = parseVotes(questionEl);
+    const answers = parseAnswers(document);
+    const markdown = buildMarkdown(title, tags, votes, questionHtml, answers);
+    return {
+      title,
+      markdown,
+      metadata: {},
+      links: [],
+      images: [],
+      extractor: "site-specific"
+    };
+  }
+};
+export {
+  stackoverflowExtractor
 };
 //# sourceMappingURL=stackoverflow.js.map

package/dist/extraction/site-extractors/stackoverflow.js.map

@@ -1 +1 @@
-{"version":3,"file":"stackoverflow.js","sourceRoot":"","sources":["../../../src/extraction/site-extractors/stackoverflow.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,eAAe,MAAM,UAAU,CAAC;AAGvC,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;AAQxF,SAAS,UAAU,CAAC,EAAkB;IACpC,IAAI,CAAC,EAAE;QAAE,OAAO,CAAC,CAAC;IAClB,MAAM,MAAM,GAAG,EAAE,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC;IAClD,MAAM,GAAG,GAAG,MAAM,EAAE,YAAY,CAAC,YAAY,CAAC,IAAI,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,GAAG,CAAC;IACrF,OAAO,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,YAAY,CAAC,QAAkB;IACtC,MAAM,SAAS,GAAG,QAAQ,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,CAAC;IAChE,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACvC,MAAM,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAC1D,MAAM,KAAK,GAAG,UAAU,CAAC,EAAa,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,EAAE,CAAC,aAAa,CAAC,qCAAqC,CAAC,CAAC;QACvE,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAE,MAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7D,OAAO,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,aAAa,CACpB,KAAa,EACb,IAAc,EACd,KAAa,EACb,YAAoB,EACpB,OAAiB;IAEjB,MAAM,OAAO,GAAG,SAAS,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,KAAK,EAAE,CAAC;IAC7D,MAAM,UAAU,GAAG,QAAQ,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,IAAI,EAAE,CAAC;IAE1D,MAAM,QAAQ,GAAa;QACzB,KAAK,KAAK,EAAE;QACZ,OAAO;QACP,EAAE;QACF,UAAU;KACX,CAAC;IAEF,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACpF,MAAM,OAAO,GAAG,CAAC,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC,CAAC;IAEzC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ;YAC7B,CAAC,CAAC,8BAA8B,MAAM,CAAC,KAAK,GAAG;YAC/C,CAAC,CAAC,qBAAqB,MAAM,CAAC,KAAK,GAAG,CAAC;QACzC,MAAM,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;QACzD,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC;IAChD,CAAC;IAED,OAAO,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC/B,CAAC;AAED,MAAM,CAAC,MAAM,sBAAsB,GAAc;IAC/C,IAAI,EAAE,eAAe;IAErB,SAAS,CAAC,GAAW;QACnB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACvC,OAAO,QAAQ,KAAK,mBAAmB;gBACrC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB,CAAC;gBACvC,QAAQ,KAAK,mBAAmB;gBAChC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC;QAC5C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAY,EAAE,GAAW;QAC/B,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAErC,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,qBAAqB,CAAC,CAAC;QAC9D,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAE1B,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QAChD,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QAExB,MAAM,cAAc,GAAG,QAAQ,CAAC,aAAa,CAAC,mEAAmE,CAAC,CAAC;QACnH,IAAI,CAAC,cAAc;YAAE,OAAO,IAAI,CAAC;QAEjC,MAAM,YAAY,GAAI,cAA0B,CAAC,SAAS,CAAC;QAE3D,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CAAC,8DAA8D,CAAC,CAAC;QACzG,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAE1F,MAAM,UAAU,GAAG,QAAQ,CAAC,aAAa,CAAC,WAAW,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,UAAU,CAAC,UAA4B,CAAC,CAAC;QAEvD,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC;QAEvC,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;QAE1E,OAAO;YACL,KAAK;YACL,QAAQ;YACR,QAAQ,EAAE,EAAE;YACZ,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,SAAS,EAAE,eAAe;SAC3B,CAAC;IACJ,CAAC;CACF,CAAC"}
+{"version":3,"sources":["../../../src/extraction/site-extractors/stackoverflow.ts"],"sourcesContent":["import { parseHTML } from 'linkedom';\nimport { htmlToMarkdown } from '../markdown.js';\nimport type { Extractor, ExtractionResult } from '../../types.js';\n\ninterface Answer {\n  accepted: boolean;\n  votes: number;\n  bodyHtml: string;\n}\n\nfunction parseVotes(el: Element | null): number {\n  if (!el) return 0;\n  const voteEl = el.querySelector('.js-vote-count');\n  const val = voteEl?.getAttribute('data-value') ?? voteEl?.textContent?.trim() ?? '0';\n  return parseInt(val, 10) || 0;\n}\n\nfunction parseAnswers(document: Document): Answer[] {\n  const answerEls = document.querySelectorAll('#answers .answer');\n  const answers: Answer[] = [];\n\n  for (const el of Array.from(answerEls)) {\n    const accepted = el.classList.contains('accepted-answer');\n    const votes = parseVotes(el as Element);\n    const bodyEl = el.querySelector('.s-prose, .js-post-body, .post-text');\n    const bodyHtml = bodyEl ? (bodyEl as Element).innerHTML : '';\n    answers.push({ accepted, votes, bodyHtml });\n  }\n\n  return answers;\n}\n\nfunction buildMarkdown(\n  title: string,\n  tags: string[],\n  votes: number,\n  questionHtml: string,\n  answers: Answer[],\n): string {\n  const tagLine = `Tags: ${tags.join(', ')} | Votes: ${votes}`;\n  const questionMd = htmlToMarkdown(questionHtml).trim();\n\n  const sections: string[] = [\n    `# ${title}`,\n    tagLine,\n    '',\n    questionMd,\n  ];\n\n  const accepted = answers.filter((a) => a.accepted);\n  const others = answers.filter((a) => !a.accepted).sort((a, b) => b.votes - a.votes);\n  const ordered = [...accepted, ...others];\n\n  for (const answer of ordered) {\n    const heading = answer.accepted\n      ? `## Accepted Answer (Votes: ${answer.votes})`\n      : `## Answer (Votes: ${answer.votes})`;\n    const bodyMd = htmlToMarkdown(answer.bodyHtml).trim();\n    sections.push('---', '', heading, '', bodyMd);\n  }\n\n  return sections.join('\\n\\n');\n}\n\nexport const stackoverflowExtractor: Extractor = {\n  name: 'stackoverflow',\n\n  canHandle(url: string): boolean {\n    try {\n      const hostname = new URL(url).hostname;\n      return hostname === 'stackoverflow.com' ||\n        hostname.endsWith('.stackoverflow.com') ||\n        hostname === 'stackexchange.com' ||\n        hostname.endsWith('.stackexchange.com');\n    } catch {\n      return false;\n    }\n  },\n\n  extract(html: string, _url: string): ExtractionResult | null {\n    if (!html) return null;\n\n    const { document } = parseHTML(html);\n\n    const titleEl = document.querySelector('.question-hyperlink');\n    if (!titleEl) return null;\n\n    const title = titleEl.textContent?.trim() ?? '';\n    if (!title) return null;\n\n    const questionBodyEl = document.querySelector('#question .s-prose, #question .js-post-body, #question .post-text');\n    if (!questionBodyEl) return null;\n\n    const questionHtml = (questionBodyEl as Element).innerHTML;\n\n    const tagEls = document.querySelectorAll('.js-post-tag-list-wrapper .post-tag, .post-taglist .post-tag');\n    const tags = Array.from(tagEls).map((el) => el.textContent?.trim() ?? '').filter(Boolean);\n\n    const questionEl = document.querySelector('#question');\n    const votes = parseVotes(questionEl as Element | null);\n\n    const answers = parseAnswers(document);\n\n    const markdown = buildMarkdown(title, tags, votes, questionHtml, answers);\n\n    return {\n      title,\n      markdown,\n      metadata: {},\n      links: [],\n      images: [],\n      extractor: 'site-specific',\n    };\n  },\n};\n"],"mappings":"AAAA,SAAS,iBAAiB;AAC1B,SAAS,sBAAsB;AAS/B,SAAS,WAAW,IAA4B;AAC9C,MAAI,CAAC,GAAI,QAAO;AAChB,QAAM,SAAS,GAAG,cAAc,gBAAgB;AAChD,QAAM,MAAM,QAAQ,aAAa,YAAY,KAAK,QAAQ,aAAa,KAAK,KAAK;AACjF,SAAO,SAAS,KAAK,EAAE,KAAK;AAC9B;AAEA,SAAS,aAAa,UAA8B;AAClD,QAAM,YAAY,SAAS,iBAAiB,kBAAkB;AAC9D,QAAM,UAAoB,CAAC;AAE3B,aAAW,MAAM,MAAM,KAAK,SAAS,GAAG;AACtC,UAAM,WAAW,GAAG,UAAU,SAAS,iBAAiB;AACxD,UAAM,QAAQ,WAAW,EAAa;AACtC,UAAM,SAAS,GAAG,cAAc,qCAAqC;AACrE,UAAM,WAAW,SAAU,OAAmB,YAAY;AAC1D,YAAQ,KAAK,EAAE,UAAU,OAAO,SAAS,CAAC;AAAA,EAC5C;AAEA,SAAO;AACT;AAEA,SAAS,cACP,OACA,MACA,OACA,cACA,SACQ;AACR,QAAM,UAAU,SAAS,KAAK,KAAK,IAAI,CAAC,aAAa,KAAK;AAC1D,QAAM,aAAa,eAAe,YAAY,EAAE,KAAK;AAErD,QAAM,WAAqB;AAAA,IACzB,KAAK,KAAK;AAAA,IACV;AAAA,IACA;AAAA,IACA;AAAA,EACF;AAEA,QAAM,WAAW,QAAQ,OAAO,CAAC,MAAM,EAAE,QAAQ;AACjD,QAAM,SAAS,QAAQ,OAAO,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAClF,QAAM,UAAU,CAAC,GAAG,UAAU,GAAG,MAAM;AAEvC,aAAW,UAAU,SAAS;AAC5B,UAAM,UAAU,OAAO,WACnB,8BAA8B,OAAO,KAAK,MAC1C,qBAAqB,OAAO,KAAK;AACrC,UAAM,SAAS,eAAe,OAAO,QAAQ,EAAE,KAAK;AACpD,aAAS,KAAK,OAAO,IAAI,SAAS,IAAI,MAAM;AAAA,EAC9C;AAEA,SAAO,SAAS,KAAK,MAAM;AAC7B;AAEO,MAAM,yBAAoC;AAAA,EAC/C,MAAM;AAAA,EAEN,UAAU,KAAsB;AAC9B,QAAI;AACF,YAAM,WAAW,IAAI,IAAI,GAAG,EAAE;AAC9B,aAAO,aAAa,uBAClB,SAAS,SAAS,oBAAoB,KACtC,aAAa,uBACb,SAAS,SAAS,oBAAoB;AAAA,IAC1C,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,QAAQ,MAAc,MAAuC;AAC3D,QAAI,CAAC,KAAM,QAAO;AAElB,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AAEnC,UAAM,UAAU,SAAS,cAAc,qBAAqB;AAC5D,QAAI,CAAC,QAAS,QAAO;AAErB,UAAM,QAAQ,QAAQ,aAAa,KAAK,KAAK;AAC7C,QAAI,CAAC,MAAO,QAAO;AAEnB,UAAM,iBAAiB,SAAS,cAAc,mEAAmE;AACjH,QAAI,CAAC,eAAgB,QAAO;AAE5B,UAAM,eAAgB,eAA2B;AAEjD,UAAM,SAAS,SAAS,iBAAiB,8DAA8D;AACvG,UAAM,OAAO,MAAM,KAAK,MAAM,EAAE,IAAI,CAAC,OAAO,GAAG,aAAa,KAAK,KAAK,EAAE,EAAE,OAAO,OAAO;AAExF,UAAM,aAAa,SAAS,cAAc,WAAW;AACrD,UAAM,QAAQ,WAAW,UAA4B;AAErD,UAAM,UAAU,aAAa,QAAQ;AAErC,UAAM,WAAW,cAAc,OAAO,MAAM,OAAO,cAAc,OAAO;AAExE,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,UAAU,CAAC;AAAA,MACX,OAAO,CAAC;AAAA,MACR,QAAQ,CAAC;AAAA,MACT,WAAW;AAAA,IACb;AAAA,EACF;AACF;","names":[]}

package/dist/extraction/structured-data.d.ts

@@ -0,0 +1,4 @@
+import type { StructuredDataResult } from '../types.js';
+export declare function extractStructuredData(html: string): StructuredDataResult[];
+export declare const KNOWN_SCHEMA_TYPES: ReadonlySet<string>;
+//# sourceMappingURL=structured-data.d.ts.map

package/dist/extraction/structured-data.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"structured-data.d.ts","sourceRoot":"","sources":["../../src/extraction/structured-data.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAaxD,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,oBAAoB,EAAE,CAQ1E;AA8KD,eAAO,MAAM,kBAAkB,EAAE,WAAW,CAAC,MAAM,CAAe,CAAC"}

package/dist/extraction/structured-data.js

@@ -0,0 +1,173 @@
+import { parseHTML } from "linkedom";
+import { createLogger } from "../logger.js";
+const log = createLogger("structured-data");
+const KNOWN_TYPES = /* @__PURE__ */ new Set([
+  "Article",
+  "Product",
+  "Recipe",
+  "BreadcrumbList",
+  "Organization",
+  "Person"
+]);
+function extractStructuredData(html) {
+  if (!html) return [];
+  const { document: doc } = parseHTML(html);
+  const out = [];
+  out.push(...extractJsonLdBlocks(doc));
+  out.push(...extractMicrodataBlocks(doc));
+  out.push(...extractRdfaBlocks(doc));
+  return out;
+}
+function extractJsonLdBlocks(doc) {
+  const out = [];
+  const scripts = doc.querySelectorAll('script[type="application/ld+json"]');
+  for (const script of scripts) {
+    const text = script.textContent?.trim();
+    if (!text) continue;
+    let parsed;
+    try {
+      parsed = JSON.parse(text);
+    } catch (err) {
+      log.warn("Failed to parse JSON-LD block", { error: String(err) });
+      continue;
+    }
+    for (const node of flattenJsonLd(parsed)) {
+      const type = normalizeType(node["@type"]);
+      if (!type) continue;
+      const fields = {};
+      for (const [k, v] of Object.entries(node)) {
+        if (k.startsWith("@")) continue;
+        fields[k] = v;
+      }
+      out.push({ provenance: "json-ld", type, fields });
+    }
+  }
+  return out;
+}
+function flattenJsonLd(value) {
+  if (!value || typeof value !== "object") return [];
+  if (Array.isArray(value)) return value.flatMap(flattenJsonLd);
+  const obj = value;
+  if (Array.isArray(obj["@graph"])) return obj["@graph"].flatMap(flattenJsonLd);
+  return [obj];
+}
+function normalizeType(raw) {
+  if (typeof raw === "string") {
+    const tail = raw.split("/").pop();
+    return tail || null;
+  }
+  if (Array.isArray(raw)) {
+    for (const t of raw) {
+      const norm = normalizeType(t);
+      if (norm) return norm;
+    }
+  }
+  return null;
+}
+function extractMicrodataBlocks(doc) {
+  const out = [];
+  const all = Array.from(doc.querySelectorAll("[itemscope]"));
+  const tops = all.filter((el) => !hasItemscopeAncestor(el));
+  for (const el of tops) {
+    const node = readMicrodataNode(el);
+    if (!node) continue;
+    out.push(node);
+  }
+  return out;
+}
+function hasItemscopeAncestor(el) {
+  let cur = el.parentElement;
+  while (cur) {
+    if (cur.hasAttribute("itemscope")) return true;
+    cur = cur.parentElement;
+  }
+  return false;
+}
+function readMicrodataNode(el) {
+  const itemtype = el.getAttribute("itemtype") ?? "";
+  const type = itemtype ? itemtype.split("/").pop() : "";
+  if (!type) return null;
+  const fields = {};
+  collectItemprops(el, fields);
+  return { provenance: "microdata", type, fields };
+}
+function collectItemprops(root, target) {
+  const stack = Array.from(root.children);
+  while (stack.length) {
+    const el = stack.shift();
+    const prop = el.getAttribute("itemprop");
+    if (prop) {
+      let value;
+      if (el.hasAttribute("itemscope")) {
+        const nested = {};
+        collectItemprops(el, nested);
+        value = nested;
+      } else {
+        value = el.getAttribute("content") ?? el.getAttribute("href") ?? el.getAttribute("src") ?? (el.textContent ?? "").trim();
+      }
+      mergeProp(target, prop, value);
+    }
+    if (el.hasAttribute("itemscope")) continue;
+    for (const c of el.children) stack.push(c);
+  }
+}
+function mergeProp(target, prop, value) {
+  if (target[prop] === void 0) {
+    target[prop] = value;
+    return;
+  }
+  if (Array.isArray(target[prop])) {
+    target[prop].push(value);
+    return;
+  }
+  target[prop] = [target[prop], value];
+}
+function extractRdfaBlocks(doc) {
+  const out = [];
+  const all = Array.from(doc.querySelectorAll("[typeof]"));
+  const tops = all.filter((el) => !hasTypeofAncestor(el));
+  for (const el of tops) {
+    const typeAttr = el.getAttribute("typeof") ?? "";
+    const type = typeAttr.split(/\s+/)[0]?.split(/[:/]/).pop() ?? "";
+    if (!type) continue;
+    const fields = {};
+    collectRdfaProps(el, fields);
+    out.push({ provenance: "rdfa", type, fields });
+  }
+  return out;
+}
+function hasTypeofAncestor(el) {
+  let cur = el.parentElement;
+  while (cur) {
+    if (cur.hasAttribute("typeof")) return true;
+    cur = cur.parentElement;
+  }
+  return false;
+}
+function collectRdfaProps(root, target) {
+  const stack = Array.from(root.children);
+  while (stack.length) {
+    const el = stack.shift();
+    const prop = el.getAttribute("property");
+    if (prop) {
+      const propName = prop.split(/[:/]/).pop() ?? prop;
+      let value;
+      if (el.hasAttribute("typeof")) {
+        const nested = {};
+        collectRdfaProps(el, nested);
+        value = nested;
+      } else {
+        value = el.getAttribute("content") ?? el.getAttribute("href") ?? el.getAttribute("resource") ?? (el.textContent ?? "").trim();
+      }
+      mergeProp(target, propName, value);
+    }
+    if (el.hasAttribute("typeof")) continue;
+    for (const c of el.children) stack.push(c);
+  }
+}
+const KNOWN_SCHEMA_TYPES = KNOWN_TYPES;
+export {
+  KNOWN_SCHEMA_TYPES,
+  extractStructuredData
+};
+//# sourceMappingURL=structured-data.js.map

package/dist/extraction/structured-data.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/extraction/structured-data.ts"],"sourcesContent":["import { parseHTML } from 'linkedom';\nimport { createLogger } from '../logger.js';\nimport type { StructuredDataResult } from '../types.js';\n\nconst log = createLogger('structured-data');\n\nconst KNOWN_TYPES = new Set([\n  'Article',\n  'Product',\n  'Recipe',\n  'BreadcrumbList',\n  'Organization',\n  'Person',\n]);\n\nexport function extractStructuredData(html: string): StructuredDataResult[] {\n  if (!html) return [];\n  const { document: doc } = parseHTML(html);\n  const out: StructuredDataResult[] = [];\n  out.push(...extractJsonLdBlocks(doc));\n  out.push(...extractMicrodataBlocks(doc));\n  out.push(...extractRdfaBlocks(doc));\n  return out;\n}\n\nfunction extractJsonLdBlocks(doc: Document): StructuredDataResult[] {\n  const out: StructuredDataResult[] = [];\n  const scripts = doc.querySelectorAll('script[type=\"application/ld+json\"]');\n  for (const script of scripts) {\n    const text = script.textContent?.trim();\n    if (!text) continue;\n    let parsed: unknown;\n    try {\n      parsed = JSON.parse(text);\n    } catch (err) {\n      log.warn('Failed to parse JSON-LD block', { error: String(err) });\n      continue;\n    }\n    for (const node of flattenJsonLd(parsed)) {\n      const type = normalizeType(node['@type']);\n      if (!type) continue;\n      const fields: Record<string, unknown> = {};\n      for (const [k, v] of Object.entries(node)) {\n        if (k.startsWith('@')) continue;\n        fields[k] = v;\n      }\n      out.push({ provenance: 'json-ld', type, fields });\n    }\n  }\n  return out;\n}\n\nfunction flattenJsonLd(value: unknown): Record<string, unknown>[] {\n  if (!value || typeof value !== 'object') return [];\n  if (Array.isArray(value)) return value.flatMap(flattenJsonLd);\n  const obj = value as Record<string, unknown>;\n  if (Array.isArray(obj['@graph'])) return obj['@graph'].flatMap(flattenJsonLd);\n  return [obj];\n}\n\nfunction normalizeType(raw: unknown): string | null {\n  if (typeof raw === 'string') {\n    const tail = raw.split('/').pop()!;\n    return tail || null;\n  }\n  if (Array.isArray(raw)) {\n    for (const t of raw) {\n      const norm = normalizeType(t);\n      if (norm) return norm;\n    }\n  }\n  return null;\n}\n\nfunction extractMicrodataBlocks(doc: Document): StructuredDataResult[] {\n  const out: StructuredDataResult[] = [];\n  // Top-level itemscopes only — nested itemscopes are walked into as fields.\n  const all = Array.from(doc.querySelectorAll('[itemscope]'));\n  const tops = all.filter((el) => !hasItemscopeAncestor(el));\n  for (const el of tops) {\n    const node = readMicrodataNode(el);\n    if (!node) continue;\n    out.push(node);\n  }\n  return out;\n}\n\nfunction hasItemscopeAncestor(el: Element): boolean {\n  let cur = el.parentElement;\n  while (cur) {\n    if (cur.hasAttribute('itemscope')) return true;\n    cur = cur.parentElement;\n  }\n  return false;\n}\n\nfunction readMicrodataNode(el: Element): StructuredDataResult | null {\n  const itemtype = el.getAttribute('itemtype') ?? '';\n  const type = itemtype ? itemtype.split('/').pop()! : '';\n  if (!type) return null;\n  const fields: Record<string, unknown> = {};\n  // Walk descendants but stop crossing into nested itemscopes (handle them as nested objects)\n  collectItemprops(el, fields);\n  return { provenance: 'microdata', type, fields };\n}\n\nfunction collectItemprops(root: Element, target: Record<string, unknown>): void {\n  const stack: Element[] = Array.from(root.children);\n  while (stack.length) {\n    const el = stack.shift()!;\n    const prop = el.getAttribute('itemprop');\n    if (prop) {\n      let value: unknown;\n      if (el.hasAttribute('itemscope')) {\n        const nested: Record<string, unknown> = {};\n        collectItemprops(el, nested);\n        value = nested;\n      } else {\n        value =\n          el.getAttribute('content') ??\n          el.getAttribute('href') ??\n          el.getAttribute('src') ??\n          (el.textContent ?? '').trim();\n      }\n      mergeProp(target, prop, value);\n    }\n    // Always stop at any itemscope: it is an independent item, regardless of\n    // whether it carries an itemprop. Otherwise its descendants' itemprops\n    // would leak into the parent record.\n    if (el.hasAttribute('itemscope')) continue;\n    for (const c of el.children) stack.push(c);\n  }\n}\n\nfunction mergeProp(target: Record<string, unknown>, prop: string, value: unknown): void {\n  if (target[prop] === undefined) {\n    target[prop] = value;\n    return;\n  }\n  if (Array.isArray(target[prop])) {\n    (target[prop] as unknown[]).push(value);\n    return;\n  }\n  target[prop] = [target[prop], value];\n}\n\nfunction extractRdfaBlocks(doc: Document): StructuredDataResult[] {\n  const out: StructuredDataResult[] = [];\n  const all = Array.from(doc.querySelectorAll('[typeof]'));\n  const tops = all.filter((el) => !hasTypeofAncestor(el));\n  for (const el of tops) {\n    const typeAttr = el.getAttribute('typeof') ?? '';\n    const type = typeAttr.split(/\\s+/)[0]?.split(/[:/]/).pop() ?? '';\n    if (!type) continue;\n    const fields: Record<string, unknown> = {};\n    collectRdfaProps(el, fields);\n    out.push({ provenance: 'rdfa', type, fields });\n  }\n  return out;\n}\n\nfunction hasTypeofAncestor(el: Element): boolean {\n  let cur = el.parentElement;\n  while (cur) {\n    if (cur.hasAttribute('typeof')) return true;\n    cur = cur.parentElement;\n  }\n  return false;\n}\n\nfunction collectRdfaProps(root: Element, target: Record<string, unknown>): void {\n  const stack: Element[] = Array.from(root.children);\n  while (stack.length) {\n    const el = stack.shift()!;\n    const prop = el.getAttribute('property');\n    if (prop) {\n      const propName = prop.split(/[:/]/).pop() ?? prop;\n      let value: unknown;\n      if (el.hasAttribute('typeof')) {\n        const nested: Record<string, unknown> = {};\n        collectRdfaProps(el, nested);\n        value = nested;\n      } else {\n        value =\n          el.getAttribute('content') ??\n          el.getAttribute('href') ??\n          el.getAttribute('resource') ??\n          (el.textContent ?? '').trim();\n      }\n      mergeProp(target, propName, value);\n    }\n    // Always stop at any nested typeof, regardless of property — independent item.\n    if (el.hasAttribute('typeof')) continue;\n    for (const c of el.children) stack.push(c);\n  }\n}\n\nexport const KNOWN_SCHEMA_TYPES: ReadonlySet<string> = KNOWN_TYPES;\n"],"mappings":"AAAA,SAAS,iBAAiB;AAC1B,SAAS,oBAAoB;AAG7B,MAAM,MAAM,aAAa,iBAAiB;AAE1C,MAAM,cAAc,oBAAI,IAAI;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAEM,SAAS,sBAAsB,MAAsC;AAC1E,MAAI,CAAC,KAAM,QAAO,CAAC;AACnB,QAAM,EAAE,UAAU,IAAI,IAAI,UAAU,IAAI;AACxC,QAAM,MAA8B,CAAC;AACrC,MAAI,KAAK,GAAG,oBAAoB,GAAG,CAAC;AACpC,MAAI,KAAK,GAAG,uBAAuB,GAAG,CAAC;AACvC,MAAI,KAAK,GAAG,kBAAkB,GAAG,CAAC;AAClC,SAAO;AACT;AAEA,SAAS,oBAAoB,KAAuC;AAClE,QAAM,MAA8B,CAAC;AACrC,QAAM,UAAU,IAAI,iBAAiB,oCAAoC;AACzE,aAAW,UAAU,SAAS;AAC5B,UAAM,OAAO,OAAO,aAAa,KAAK;AACtC,QAAI,CAAC,KAAM;AACX,QAAI;AACJ,QAAI;AACF,eAAS,KAAK,MAAM,IAAI;AAAA,IAC1B,SAAS,KAAK;AACZ,UAAI,KAAK,iCAAiC,EAAE,OAAO,OAAO,GAAG,EAAE,CAAC;AAChE;AAAA,IACF;AACA,eAAW,QAAQ,cAAc,MAAM,GAAG;AACxC,YAAM,OAAO,cAAc,KAAK,OAAO,CAAC;AACxC,UAAI,CAAC,KAAM;AACX,YAAM,SAAkC,CAAC;AACzC,iBAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,IAAI,GAAG;AACzC,YAAI,EAAE,WAAW,GAAG,EAAG;AACvB,eAAO,CAAC,IAAI;AAAA,MACd;AACA,UAAI,KAAK,EAAE,YAAY,WAAW,MAAM,OAAO,CAAC;AAAA,IAClD;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,cAAc,OAA2C;AAChE,MAAI,CAAC,SAAS,OAAO,UAAU,SAAU,QAAO,CAAC;AACjD,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,MAAM,QAAQ,aAAa;AAC5D,QAAM,MAAM;AACZ,MAAI,MAAM,QAAQ,IAAI,QAAQ,CAAC,EAAG,QAAO,IAAI,QAAQ,EAAE,QAAQ,aAAa;AAC5E,SAAO,CAAC,GAAG;AACb;AAEA,SAAS,cAAc,KAA6B;AAClD,MAAI,OAAO,QAAQ,UAAU;AAC3B,UAAM,OAAO,IAAI,MAAM,GAAG,EAAE,IAAI;AAChC,WAAO,QAAQ;AAAA,EACjB;AACA,MAAI,MAAM,QAAQ,GAAG,GAAG;AACtB,eAAW,KAAK,KAAK;AACnB,YAAM,OAAO,cAAc,CAAC;AAC5B,UAAI,KAAM,QAAO;AAAA,IACnB;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,uBAAuB,KAAuC;AACrE,QAAM,MAA8B,CAAC;AAErC,QAAM,MAAM,MAAM,KAAK,IAAI,iBAAiB,aAAa,CAAC;AAC1D,QAAM,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,qBAAqB,EAAE,CAAC;AACzD,aAAW,MAAM,MAAM;AACrB,UAAM,OAAO,kBAAkB,EAAE;AACjC,QAAI,CAAC,KAAM;AACX,QAAI,KAAK,IAAI;AAAA,EACf;AACA,SAAO;AACT;AAEA,SAAS,qBAAqB,IAAsB;AAClD,MAAI,MAAM,GAAG;AACb,SAAO,KAAK;AACV,QAAI,IAAI,aAAa,WAAW,EAAG,QAAO;AAC1C,UAAM,IAAI;AAAA,EACZ;AACA,SAAO;AACT;AAEA,SAAS,kBAAkB,IAA0C;AACnE,QAAM,WAAW,GAAG,aAAa,UAAU,KAAK;AAChD,QAAM,OAAO,WAAW,SAAS,MAAM,GAAG,EAAE,IAAI,IAAK;AACrD,MAAI,CAAC,KAAM,QAAO;AAClB,QAAM,SAAkC,CAAC;AAEzC,mBAAiB,IAAI,MAAM;AAC3B,SAAO,EAAE,YAAY,aAAa,MAAM,OAAO;AACjD;AAEA,SAAS,iBAAiB,MAAe,QAAuC;AAC9E,QAAM,QAAmB,MAAM,KAAK,KAAK,QAAQ;AACjD,SAAO,MAAM,QAAQ;AACnB,UAAM,KAAK,MAAM,MAAM;AACvB,UAAM,OAAO,GAAG,aAAa,UAAU;AACvC,QAAI,MAAM;AACR,UAAI;AACJ,UAAI,GAAG,aAAa,WAAW,GAAG;AAChC,cAAM,SAAkC,CAAC;AACzC,yBAAiB,IAAI,MAAM;AAC3B,gBAAQ;AAAA,MACV,OAAO;AACL,gBACE,GAAG,aAAa,SAAS,KACzB,GAAG,aAAa,MAAM,KACtB,GAAG,aAAa,KAAK,MACpB,GAAG,eAAe,IAAI,KAAK;AAAA,MAChC;AACA,gBAAU,QAAQ,MAAM,KAAK;AAAA,IAC/B;AAIA,QAAI,GAAG,aAAa,WAAW,EAAG;AAClC,eAAW,KAAK,GAAG,SAAU,OAAM,KAAK,CAAC;AAAA,EAC3C;AACF;AAEA,SAAS,UAAU,QAAiC,MAAc,OAAsB;AACtF,MAAI,OAAO,IAAI,MAAM,QAAW;AAC9B,WAAO,IAAI,IAAI;AACf;AAAA,EACF;AACA,MAAI,MAAM,QAAQ,OAAO,IAAI,CAAC,GAAG;AAC/B,IAAC,OAAO,IAAI,EAAgB,KAAK,KAAK;AACtC;AAAA,EACF;AACA,SAAO,IAAI,IAAI,CAAC,OAAO,IAAI,GAAG,KAAK;AACrC;AAEA,SAAS,kBAAkB,KAAuC;AAChE,QAAM,MAA8B,CAAC;AACrC,QAAM,MAAM,MAAM,KAAK,IAAI,iBAAiB,UAAU,CAAC;AACvD,QAAM,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC;AACtD,aAAW,MAAM,MAAM;AACrB,UAAM,WAAW,GAAG,aAAa,QAAQ,KAAK;AAC9C,UAAM,OAAO,SAAS,MAAM,KAAK,EAAE,CAAC,GAAG,MAAM,MAAM,EAAE,IAAI,KAAK;AAC9D,QAAI,CAAC,KAAM;AACX,UAAM,SAAkC,CAAC;AACzC,qBAAiB,IAAI,MAAM;AAC3B,QAAI,KAAK,EAAE,YAAY,QAAQ,MAAM,OAAO,CAAC;AAAA,EAC/C;AACA,SAAO;AACT;AAEA,SAAS,kBAAkB,IAAsB;AAC/C,MAAI,MAAM,GAAG;AACb,SAAO,KAAK;AACV,QAAI,IAAI,aAAa,QAAQ,EAAG,QAAO;AACvC,UAAM,IAAI;AAAA,EACZ;AACA,SAAO;AACT;AAEA,SAAS,iBAAiB,MAAe,QAAuC;AAC9E,QAAM,QAAmB,MAAM,KAAK,KAAK,QAAQ;AACjD,SAAO,MAAM,QAAQ;AACnB,UAAM,KAAK,MAAM,MAAM;AACvB,UAAM,OAAO,GAAG,aAAa,UAAU;AACvC,QAAI,MAAM;AACR,YAAM,WAAW,KAAK,MAAM,MAAM,EAAE,IAAI,KAAK;AAC7C,UAAI;AACJ,UAAI,GAAG,aAAa,QAAQ,GAAG;AAC7B,cAAM,SAAkC,CAAC;AACzC,yBAAiB,IAAI,MAAM;AAC3B,gBAAQ;AAAA,MACV,OAAO;AACL,gBACE,GAAG,aAAa,SAAS,KACzB,GAAG,aAAa,MAAM,KACtB,GAAG,aAAa,UAAU,MACzB,GAAG,eAAe,IAAI,KAAK;AAAA,MAChC;AACA,gBAAU,QAAQ,UAAU,KAAK;AAAA,IACnC;AAEA,QAAI,GAAG,aAAa,QAAQ,EAAG;AAC/B,eAAW,KAAK,GAAG,SAAU,OAAM,KAAK,CAAC;AAAA,EAC3C;AACF;AAEO,MAAM,qBAA0C;","names":[]}

package/dist/extraction/structured.d.ts

@@ -0,0 +1,4 @@
+import type { StructuredData } from '../types.js';
+export declare function extractStructured(html: string): StructuredData;
+export type { TableData } from '../types.js';
+//# sourceMappingURL=structured.d.ts.map

package/dist/extraction/structured.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"structured.d.ts","sourceRoot":"","sources":["../../src/extraction/structured.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAA2C,MAAM,aAAa,CAAC;AAU3F,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,CAgB9D;AAuLD,YAAY,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC"}

package/dist/extraction/structured.js

@@ -0,0 +1,163 @@
+import { parseHTML } from "linkedom";
+import { extractTables } from "./extract.js";
+import { extractJsonLd } from "./jsonld.js";
+const MAX_VALUE_LEN = 400;
+const MAX_ITEMS_PER_TYPE = 200;
+function extractStructured(html) {
+  const { document: doc } = parseHTML(html);
+  const tables = extractTables(html);
+  const jsonld = extractJsonLd(html);
+  const definitions = extractDefinitions(doc);
+  const chart_hints = extractChartHints(doc);
+  const key_value_pairs = extractKeyValuePairs(doc);
+  return {
+    tables,
+    definitions,
+    jsonld,
+    chart_hints,
+    key_value_pairs
+  };
+}
+function extractDefinitions(doc) {
+  const out = [];
+  const dlists = doc.querySelectorAll("dl");
+  for (const dl of dlists) {
+    const children = Array.from(dl.children);
+    let pending = null;
+    const buffer = [];
+    const flush = () => {
+      if (pending !== null && buffer.length > 0) {
+        out.push({
+          term: pending,
+          description: truncate(buffer.join(" ").trim())
+        });
+      }
+      pending = null;
+      buffer.length = 0;
+    };
+    for (const c of children) {
+      const tag = c.tagName.toLowerCase();
+      if (tag === "dt") {
+        flush();
+        pending = truncate((c.textContent ?? "").trim());
+      } else if (tag === "dd" && pending !== null) {
+        const text = (c.textContent ?? "").trim();
+        if (text) buffer.push(text);
+      }
+      if (out.length >= MAX_ITEMS_PER_TYPE) break;
+    }
+    flush();
+    if (out.length >= MAX_ITEMS_PER_TYPE) break;
+  }
+  return out;
+}
+function extractChartHints(doc) {
+  const out = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const svg of doc.querySelectorAll("svg")) {
+    const title = (svg.querySelector("title")?.textContent ?? "").trim();
+    const aria_label = (svg.getAttribute("aria-label") ?? "").trim();
+    const role = (svg.getAttribute("role") ?? "").trim();
+    let figcaption;
+    const figParent = closestFigure(svg);
+    if (figParent) {
+      const cap = figParent.querySelector("figcaption");
+      figcaption = cap?.textContent?.trim() || void 0;
+    }
+    if (!title && !aria_label && !figcaption) continue;
+    const hint = {
+      ...title ? { title: truncate(title) } : {},
+      ...aria_label ? { aria_label: truncate(aria_label) } : {},
+      ...figcaption ? { figcaption: truncate(figcaption) } : {},
+      type_hint: inferChartType(title, aria_label, role, figcaption)
+    };
+    const key = `${hint.title ?? ""}|${hint.aria_label ?? ""}|${hint.figcaption ?? ""}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(hint);
+    if (out.length >= MAX_ITEMS_PER_TYPE) break;
+  }
+  for (const fig of doc.querySelectorAll("figure")) {
+    if (fig.querySelector("svg")) continue;
+    const cap = fig.querySelector("figcaption")?.textContent?.trim();
+    if (!cap) continue;
+    const key = `||${cap}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push({
+      figcaption: truncate(cap),
+      type_hint: inferChartType("", "", "", cap)
+    });
+    if (out.length >= MAX_ITEMS_PER_TYPE) break;
+  }
+  return out;
+}
+function closestFigure(el) {
+  let cur = el.parentElement;
+  while (cur) {
+    if (cur.tagName.toLowerCase() === "figure") return cur;
+    cur = cur.parentElement;
+  }
+  return null;
+}
+function inferChartType(title, ariaLabel, role, figcaption) {
+  const all = [title, ariaLabel, role, figcaption ?? ""].join(" ").toLowerCase();
+  if (/\b(chart|bar|line|pie|donut|scatter|area|histogram)\b/.test(all)) return "chart";
+  if (/\b(diagram|flow|architecture|topology)\b/.test(all)) return "diagram";
+  if (/\b(graph|network|tree)\b/.test(all)) return "graph";
+  if (role === "img") return "chart";
+  return void 0;
+}
+function extractKeyValuePairs(doc) {
+  const out = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const el of doc.querySelectorAll("[itemprop]")) {
+    const key = el.getAttribute("itemprop") ?? "";
+    const value = (el.getAttribute("content") ?? el.textContent ?? "").trim();
+    if (!key || !value) continue;
+    pushUnique(out, seen, { key, value: truncate(value), source: "microdata" });
+    if (out.length >= MAX_ITEMS_PER_TYPE) return out;
+  }
+  for (const el of doc.querySelectorAll("[data-label][data-value], [data-key][data-value]")) {
+    const key = (el.getAttribute("data-label") ?? el.getAttribute("data-key") ?? "").trim();
+    const value = (el.getAttribute("data-value") ?? "").trim();
+    if (!key || !value) continue;
+    pushUnique(out, seen, { key, value: truncate(value), source: "data-attr" });
+    if (out.length >= MAX_ITEMS_PER_TYPE) return out;
+  }
+  for (const row of doc.querySelectorAll('[class*="row"], [class*="spec"], [class*="field"]')) {
+    const label = row.querySelector('[class*="label"], [class*="name"], [class*="key"], dt, th');
+    const value = row.querySelector('[class*="value"], [class*="data"], dd, td');
+    if (!label || !value) continue;
+    const k = (label.textContent ?? "").trim();
+    const v = (value.textContent ?? "").trim();
+    if (!k || !v || k === v) continue;
+    if (k.length > 100 || v.length === 0) continue;
+    pushUnique(out, seen, { key: k, value: truncate(v), source: "comparison-grid" });
+    if (out.length >= MAX_ITEMS_PER_TYPE) return out;
+  }
+  for (const el of doc.querySelectorAll("li, p")) {
+    const text = (el.textContent ?? "").trim();
+    if (!text || text.length > 300) continue;
+    const m = text.match(/^([A-Z][A-Za-z0-9 _-]{1,40}):\s+(.+)$/);
+    if (!m) continue;
+    pushUnique(out, seen, { key: m[1].trim(), value: truncate(m[2].trim()), source: "text-pattern" });
+    if (out.length >= MAX_ITEMS_PER_TYPE) return out;
+  }
+  return out;
+}
+function pushUnique(list, seen, pair) {
+  const key = `${pair.key.toLowerCase()}|${pair.value.toLowerCase()}`;
+  if (seen.has(key)) return;
+  seen.add(key);
+  list.push(pair);
+}
+function truncate(text) {
+  const collapsed = text.replace(/\s+/g, " ").trim();
+  if (collapsed.length <= MAX_VALUE_LEN) return collapsed;
+  return collapsed.slice(0, MAX_VALUE_LEN - 1) + "\u2026";
+}
+export {
+  extractStructured
+};
+//# sourceMappingURL=structured.js.map

package/dist/extraction/structured.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/extraction/structured.ts"],"sourcesContent":["import { parseHTML } from 'linkedom';\nimport type { StructuredData, DefinitionPair, ChartHint, KeyValuePair } from '../types.js';\nimport { extractTables } from './extract.js';\nimport { extractJsonLd } from './jsonld.js';\n\nconst MAX_VALUE_LEN = 400;\nconst MAX_ITEMS_PER_TYPE = 200;\n\n// Entry point. Collects every structured-data pattern we can surface from\n// raw HTML so host LLMs receive a rich, schema-free brief without needing\n// to re-parse the page.\nexport function extractStructured(html: string): StructuredData {\n  const { document: doc } = parseHTML(html);\n\n  const tables = extractTables(html);\n  const jsonld = extractJsonLd(html);\n  const definitions = extractDefinitions(doc);\n  const chart_hints = extractChartHints(doc);\n  const key_value_pairs = extractKeyValuePairs(doc);\n\n  return {\n    tables,\n    definitions,\n    jsonld,\n    chart_hints,\n    key_value_pairs,\n  };\n}\n\n// <dl><dt>Term</dt><dd>Description</dd></dl> is the canonical key-value\n// structure on the web; we also handle multiple <dd> per <dt> by joining.\nfunction extractDefinitions(doc: Document): DefinitionPair[] {\n  const out: DefinitionPair[] = [];\n  const dlists = doc.querySelectorAll('dl');\n  for (const dl of dlists) {\n    const children = Array.from(dl.children);\n    let pending: string | null = null;\n    const buffer: string[] = [];\n    const flush = () => {\n      if (pending !== null && buffer.length > 0) {\n        out.push({\n          term: pending,\n          description: truncate(buffer.join(' ').trim()),\n        });\n      }\n      pending = null;\n      buffer.length = 0;\n    };\n    for (const c of children) {\n      const tag = c.tagName.toLowerCase();\n      if (tag === 'dt') {\n        flush();\n        pending = truncate((c.textContent ?? '').trim());\n      } else if (tag === 'dd' && pending !== null) {\n        const text = (c.textContent ?? '').trim();\n        if (text) buffer.push(text);\n      }\n      if (out.length >= MAX_ITEMS_PER_TYPE) break;\n    }\n    flush();\n    if (out.length >= MAX_ITEMS_PER_TYPE) break;\n  }\n  return out;\n}\n\n// SVG / figure accessibility hints are the cheapest way to surface\n// chart structure when the chart itself is rendered by JS. Host LLMs\n// use these to describe a data viz without needing the underlying data.\nfunction extractChartHints(doc: Document): ChartHint[] {\n  const out: ChartHint[] = [];\n  const seen = new Set<string>();\n\n  for (const svg of doc.querySelectorAll('svg')) {\n    const title = (svg.querySelector('title')?.textContent ?? '').trim();\n    const aria_label = (svg.getAttribute('aria-label') ?? '').trim();\n    const role = (svg.getAttribute('role') ?? '').trim();\n\n    let figcaption: string | undefined;\n    const figParent = closestFigure(svg);\n    if (figParent) {\n      const cap = figParent.querySelector('figcaption');\n      figcaption = cap?.textContent?.trim() || undefined;\n    }\n\n    if (!title && !aria_label && !figcaption) continue;\n\n    const hint: ChartHint = {\n      ...(title ? { title: truncate(title) } : {}),\n      ...(aria_label ? { aria_label: truncate(aria_label) } : {}),\n      ...(figcaption ? { figcaption: truncate(figcaption) } : {}),\n      type_hint: inferChartType(title, aria_label, role, figcaption),\n    };\n    const key = `${hint.title ?? ''}|${hint.aria_label ?? ''}|${hint.figcaption ?? ''}`;\n    if (seen.has(key)) continue;\n    seen.add(key);\n    out.push(hint);\n    if (out.length >= MAX_ITEMS_PER_TYPE) break;\n  }\n\n  // <figure><figcaption> without SVG still surfaces dataviz context for\n  // pages that render charts as images or canvas.\n  for (const fig of doc.querySelectorAll('figure')) {\n    if (fig.querySelector('svg')) continue; // already handled above\n    const cap = fig.querySelector('figcaption')?.textContent?.trim();\n    if (!cap) continue;\n    const key = `||${cap}`;\n    if (seen.has(key)) continue;\n    seen.add(key);\n    out.push({\n      figcaption: truncate(cap),\n      type_hint: inferChartType('', '', '', cap),\n    });\n    if (out.length >= MAX_ITEMS_PER_TYPE) break;\n  }\n\n  return out;\n}\n\nfunction closestFigure(el: Element): Element | null {\n  let cur: Element | null = el.parentElement;\n  while (cur) {\n    if (cur.tagName.toLowerCase() === 'figure') return cur;\n    cur = cur.parentElement;\n  }\n  return null;\n}\n\nfunction inferChartType(\n  title: string,\n  ariaLabel: string,\n  role: string,\n  figcaption?: string,\n): ChartHint['type_hint'] {\n  const all = [title, ariaLabel, role, figcaption ?? ''].join(' ').toLowerCase();\n  if (/\\b(chart|bar|line|pie|donut|scatter|area|histogram)\\b/.test(all)) return 'chart';\n  if (/\\b(diagram|flow|architecture|topology)\\b/.test(all)) return 'diagram';\n  if (/\\b(graph|network|tree)\\b/.test(all)) return 'graph';\n  if (role === 'img') return 'chart';\n  return undefined;\n}\n\n// Comparison grids, spec sheets, and product info boxes often use\n// explicit \"Label: Value\" or [data-label] patterns. We harvest those\n// plus <meta name=...> pairs not already covered by extractMetadata.\nfunction extractKeyValuePairs(doc: Document): KeyValuePair[] {\n  const out: KeyValuePair[] = [];\n  const seen = new Set<string>();\n\n  // Microdata itemprop pairs\n  for (const el of doc.querySelectorAll('[itemprop]')) {\n    const key = el.getAttribute('itemprop') ?? '';\n    const value = (el.getAttribute('content') ?? el.textContent ?? '').trim();\n    if (!key || !value) continue;\n    pushUnique(out, seen, { key, value: truncate(value), source: 'microdata' });\n    if (out.length >= MAX_ITEMS_PER_TYPE) return out;\n  }\n\n  // data-* attributes where the name looks meaningful (>= 3 chars after data-)\n  for (const el of doc.querySelectorAll('[data-label][data-value], [data-key][data-value]')) {\n    const key = (el.getAttribute('data-label') ?? el.getAttribute('data-key') ?? '').trim();\n    const value = (el.getAttribute('data-value') ?? '').trim();\n    if (!key || !value) continue;\n    pushUnique(out, seen, { key, value: truncate(value), source: 'data-attr' });\n    if (out.length >= MAX_ITEMS_PER_TYPE) return out;\n  }\n\n  // Comparison grid rows: <div class=\"row\"><div class=\"label\">X</div><div class=\"value\">Y</div></div>\n  for (const row of doc.querySelectorAll('[class*=\"row\"], [class*=\"spec\"], [class*=\"field\"]')) {\n    const label = row.querySelector('[class*=\"label\"], [class*=\"name\"], [class*=\"key\"], dt, th');\n    const value = row.querySelector('[class*=\"value\"], [class*=\"data\"], dd, td');\n    if (!label || !value) continue;\n    const k = (label.textContent ?? '').trim();\n    const v = (value.textContent ?? '').trim();\n    if (!k || !v || k === v) continue;\n    if (k.length > 100 || v.length === 0) continue;\n    pushUnique(out, seen, { key: k, value: truncate(v), source: 'comparison-grid' });\n    if (out.length >= MAX_ITEMS_PER_TYPE) return out;\n  }\n\n  // \"Key: Value\" text patterns within <li>/<p> — cheap heuristic for spec sheets\n  for (const el of doc.querySelectorAll('li, p')) {\n    const text = (el.textContent ?? '').trim();\n    if (!text || text.length > 300) continue;\n    const m = text.match(/^([A-Z][A-Za-z0-9 _-]{1,40}):\\s+(.+)$/);\n    if (!m) continue;\n    pushUnique(out, seen, { key: m[1].trim(), value: truncate(m[2].trim()), source: 'text-pattern' });\n    if (out.length >= MAX_ITEMS_PER_TYPE) return out;\n  }\n\n  return out;\n}\n\nfunction pushUnique(\n  list: KeyValuePair[],\n  seen: Set<string>,\n  pair: KeyValuePair,\n): void {\n  const key = `${pair.key.toLowerCase()}|${pair.value.toLowerCase()}`;\n  if (seen.has(key)) return;\n  seen.add(key);\n  list.push(pair);\n}\n\nfunction truncate(text: string): string {\n  const collapsed = text.replace(/\\s+/g, ' ').trim();\n  if (collapsed.length <= MAX_VALUE_LEN) return collapsed;\n  return collapsed.slice(0, MAX_VALUE_LEN - 1) + '…';\n}\n\n// Re-export types for callers that only import from this module.\nexport type { TableData } from '../types.js';\n"],"mappings":"AAAA,SAAS,iBAAiB;AAE1B,SAAS,qBAAqB;AAC9B,SAAS,qBAAqB;AAE9B,MAAM,gBAAgB;AACtB,MAAM,qBAAqB;AAKpB,SAAS,kBAAkB,MAA8B;AAC9D,QAAM,EAAE,UAAU,IAAI,IAAI,UAAU,IAAI;AAExC,QAAM,SAAS,cAAc,IAAI;AACjC,QAAM,SAAS,cAAc,IAAI;AACjC,QAAM,cAAc,mBAAmB,GAAG;AAC1C,QAAM,cAAc,kBAAkB,GAAG;AACzC,QAAM,kBAAkB,qBAAqB,GAAG;AAEhD,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAIA,SAAS,mBAAmB,KAAiC;AAC3D,QAAM,MAAwB,CAAC;AAC/B,QAAM,SAAS,IAAI,iBAAiB,IAAI;AACxC,aAAW,MAAM,QAAQ;AACvB,UAAM,WAAW,MAAM,KAAK,GAAG,QAAQ;AACvC,QAAI,UAAyB;AAC7B,UAAM,SAAmB,CAAC;AAC1B,UAAM,QAAQ,MAAM;AAClB,UAAI,YAAY,QAAQ,OAAO,SAAS,GAAG;AACzC,YAAI,KAAK;AAAA,UACP,MAAM;AAAA,UACN,aAAa,SAAS,OAAO,KAAK,GAAG,EAAE,KAAK,CAAC;AAAA,QAC/C,CAAC;AAAA,MACH;AACA,gBAAU;AACV,aAAO,SAAS;AAAA,IAClB;AACA,eAAW,KAAK,UAAU;AACxB,YAAM,MAAM,EAAE,QAAQ,YAAY;AAClC,UAAI,QAAQ,MAAM;AAChB,cAAM;AACN,kBAAU,UAAU,EAAE,eAAe,IAAI,KAAK,CAAC;AAAA,MACjD,WAAW,QAAQ,QAAQ,YAAY,MAAM;AAC3C,cAAM,QAAQ,EAAE,eAAe,IAAI,KAAK;AACxC,YAAI,KAAM,QAAO,KAAK,IAAI;AAAA,MAC5B;AACA,UAAI,IAAI,UAAU,mBAAoB;AAAA,IACxC;AACA,UAAM;AACN,QAAI,IAAI,UAAU,mBAAoB;AAAA,EACxC;AACA,SAAO;AACT;AAKA,SAAS,kBAAkB,KAA4B;AACrD,QAAM,MAAmB,CAAC;AAC1B,QAAM,OAAO,oBAAI,IAAY;AAE7B,aAAW,OAAO,IAAI,iBAAiB,KAAK,GAAG;AAC7C,UAAM,SAAS,IAAI,cAAc,OAAO,GAAG,eAAe,IAAI,KAAK;AACnE,UAAM,cAAc,IAAI,aAAa,YAAY,KAAK,IAAI,KAAK;AAC/D,UAAM,QAAQ,IAAI,aAAa,MAAM,KAAK,IAAI,KAAK;AAEnD,QAAI;AACJ,UAAM,YAAY,cAAc,GAAG;AACnC,QAAI,WAAW;AACb,YAAM,MAAM,UAAU,cAAc,YAAY;AAChD,mBAAa,KAAK,aAAa,KAAK,KAAK;AAAA,IAC3C;AAEA,QAAI,CAAC,SAAS,CAAC,cAAc,CAAC,WAAY;AAE1C,UAAM,OAAkB;AAAA,MACtB,GAAI,QAAQ,EAAE,OAAO,SAAS,KAAK,EAAE,IAAI,CAAC;AAAA,MAC1C,GAAI,aAAa,EAAE,YAAY,SAAS,UAAU,EAAE,IAAI,CAAC;AAAA,MACzD,GAAI,aAAa,EAAE,YAAY,SAAS,UAAU,EAAE,IAAI,CAAC;AAAA,MACzD,WAAW,eAAe,OAAO,YAAY,MAAM,UAAU;AAAA,IAC/D;AACA,UAAM,MAAM,GAAG,KAAK,SAAS,EAAE,IAAI,KAAK,cAAc,EAAE,IAAI,KAAK,cAAc,EAAE;AACjF,QAAI,KAAK,IAAI,GAAG,EAAG;AACnB,SAAK,IAAI,GAAG;AACZ,QAAI,KAAK,IAAI;AACb,QAAI,IAAI,UAAU,mBAAoB;AAAA,EACxC;AAIA,aAAW,OAAO,IAAI,iBAAiB,QAAQ,GAAG;AAChD,QAAI,IAAI,cAAc,KAAK,EAAG;AAC9B,UAAM,MAAM,IAAI,cAAc,YAAY,GAAG,aAAa,KAAK;AAC/D,QAAI,CAAC,IAAK;AACV,UAAM,MAAM,KAAK,GAAG;AACpB,QAAI,KAAK,IAAI,GAAG,EAAG;AACnB,SAAK,IAAI,GAAG;AACZ,QAAI,KAAK;AAAA,MACP,YAAY,SAAS,GAAG;AAAA,MACxB,WAAW,eAAe,IAAI,IAAI,IAAI,GAAG;AAAA,IAC3C,CAAC;AACD,QAAI,IAAI,UAAU,mBAAoB;AAAA,EACxC;AAEA,SAAO;AACT;AAEA,SAAS,cAAc,IAA6B;AAClD,MAAI,MAAsB,GAAG;AAC7B,SAAO,KAAK;AACV,QAAI,IAAI,QAAQ,YAAY,MAAM,SAAU,QAAO;AACnD,UAAM,IAAI;AAAA,EACZ;AACA,SAAO;AACT;AAEA,SAAS,eACP,OACA,WACA,MACA,YACwB;AACxB,QAAM,MAAM,CAAC,OAAO,WAAW,MAAM,cAAc,EAAE,EAAE,KAAK,GAAG,EAAE,YAAY;AAC7E,MAAI,wDAAwD,KAAK,GAAG,EAAG,QAAO;AAC9E,MAAI,2CAA2C,KAAK,GAAG,EAAG,QAAO;AACjE,MAAI,2BAA2B,KAAK,GAAG,EAAG,QAAO;AACjD,MAAI,SAAS,MAAO,QAAO;AAC3B,SAAO;AACT;AAKA,SAAS,qBAAqB,KAA+B;AAC3D,QAAM,MAAsB,CAAC;AAC7B,QAAM,OAAO,oBAAI,IAAY;AAG7B,aAAW,MAAM,IAAI,iBAAiB,YAAY,GAAG;AACnD,UAAM,MAAM,GAAG,aAAa,UAAU,KAAK;AAC3C,UAAM,SAAS,GAAG,aAAa,SAAS,KAAK,GAAG,eAAe,IAAI,KAAK;AACxE,QAAI,CAAC,OAAO,CAAC,MAAO;AACpB,eAAW,KAAK,MAAM,EAAE,KAAK,OAAO,SAAS,KAAK,GAAG,QAAQ,YAAY,CAAC;AAC1E,QAAI,IAAI,UAAU,mBAAoB,QAAO;AAAA,EAC/C;AAGA,aAAW,MAAM,IAAI,iBAAiB,kDAAkD,GAAG;AACzF,UAAM,OAAO,GAAG,aAAa,YAAY,KAAK,GAAG,aAAa,UAAU,KAAK,IAAI,KAAK;AACtF,UAAM,SAAS,GAAG,aAAa,YAAY,KAAK,IAAI,KAAK;AACzD,QAAI,CAAC,OAAO,CAAC,MAAO;AACpB,eAAW,KAAK,MAAM,EAAE,KAAK,OAAO,SAAS,KAAK,GAAG,QAAQ,YAAY,CAAC;AAC1E,QAAI,IAAI,UAAU,mBAAoB,QAAO;AAAA,EAC/C;AAGA,aAAW,OAAO,IAAI,iBAAiB,mDAAmD,GAAG;AAC3F,UAAM,QAAQ,IAAI,cAAc,2DAA2D;AAC3F,UAAM,QAAQ,IAAI,cAAc,2CAA2C;AAC3E,QAAI,CAAC,SAAS,CAAC,MAAO;AACtB,UAAM,KAAK,MAAM,eAAe,IAAI,KAAK;AACzC,UAAM,KAAK,MAAM,eAAe,IAAI,KAAK;AACzC,QAAI,CAAC,KAAK,CAAC,KAAK,MAAM,EAAG;AACzB,QAAI,EAAE,SAAS,OAAO,EAAE,WAAW,EAAG;AACtC,eAAW,KAAK,MAAM,EAAE,KAAK,GAAG,OAAO,SAAS,CAAC,GAAG,QAAQ,kBAAkB,CAAC;AAC/E,QAAI,IAAI,UAAU,mBAAoB,QAAO;AAAA,EAC/C;AAGA,aAAW,MAAM,IAAI,iBAAiB,OAAO,GAAG;AAC9C,UAAM,QAAQ,GAAG,eAAe,IAAI,KAAK;AACzC,QAAI,CAAC,QAAQ,KAAK,SAAS,IAAK;AAChC,UAAM,IAAI,KAAK,MAAM,uCAAuC;AAC5D,QAAI,CAAC,EAAG;AACR,eAAW,KAAK,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,GAAG,OAAO,SAAS,EAAE,CAAC,EAAE,KAAK,CAAC,GAAG,QAAQ,eAAe,CAAC;AAChG,QAAI,IAAI,UAAU,mBAAoB,QAAO;AAAA,EAC/C;AAEA,SAAO;AACT;AAEA,SAAS,WACP,MACA,MACA,MACM;AACN,QAAM,MAAM,GAAG,KAAK,IAAI,YAAY,CAAC,IAAI,KAAK,MAAM,YAAY,CAAC;AACjE,MAAI,KAAK,IAAI,GAAG,EAAG;AACnB,OAAK,IAAI,GAAG;AACZ,OAAK,KAAK,IAAI;AAChB;AAEA,SAAS,SAAS,MAAsB;AACtC,QAAM,YAAY,KAAK,QAAQ,QAAQ,GAAG,EAAE,KAAK;AACjD,MAAI,UAAU,UAAU,cAAe,QAAO;AAC9C,SAAO,UAAU,MAAM,GAAG,gBAAgB,CAAC,IAAI;AACjD;","names":[]}

package/dist/extraction/v1/classifier.d.ts

@@ -0,0 +1,3 @@
+export type ContentType = 'news' | 'recipe' | 'product' | 'code' | 'docs' | 'generic';
+export declare function classifyContent(url: string, html: string): ContentType;
+//# sourceMappingURL=classifier.d.ts.map

package/dist/extraction/v1/classifier.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"classifier.d.ts","sourceRoot":"","sources":["../../../src/extraction/v1/classifier.ts"],"names":[],"mappings":"AAGA,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;AAuBtF,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAgBtE"}