@staticn0va/wigolo 0.1.0 → 0.1.1

package/dist/extraction/site-extractors/docs-generic.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"docs-generic.d.ts","sourceRoot":"","sources":["../../../src/extraction/site-extractors/docs-generic.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAoB,MAAM,gBAAgB,CAAC;AAqFlE,eAAO,MAAM,oBAAoB,EAAE,SAqClC,CAAC"}
+{"version":3,"file":"docs-generic.d.ts","sourceRoot":"","sources":["../../../src/extraction/site-extractors/docs-generic.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAoB,MAAM,gBAAgB,CAAC;AAmFlE,eAAO,MAAM,oBAAoB,EAAE,SAqClC,CAAC"}

package/dist/extraction/site-extractors/docs-generic.js

@@ -1,104 +1,94 @@
-import { parseHTML } from 'linkedom';
-import TurndownService from 'turndown';
-const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+import { parseHTML } from "linkedom";
+import { htmlToMarkdown } from "../markdown.js";
 const STRIP_SELECTORS = [
-    'nav',
-    '.docs-sidebar',
-    '.sidebar',
-    '.toc-wrapper',
-    '.table-of-contents',
-    '.version-picker',
-    '.pagination-nav',
-    'header',
-    'footer',
+  "nav",
+  ".docs-sidebar",
+  ".sidebar",
+  ".toc-wrapper",
+  ".table-of-contents",
+  ".version-picker",
+  ".pagination-nav",
+  "header",
+  "footer"
 ];
 function detectFramework(html) {
-    if (html.includes('docs-sidebar') || html.includes('data-docusaurus-page')) {
-        return 'docusaurus';
-    }
-    if (html.includes('md-content')) {
-        return 'mkdocs';
-    }
-    if (html.includes('class="document"') || html.includes("class='document'") ||
-        (html.includes('class="body"') && html.includes('highlight'))) {
-        return 'sphinx';
-    }
-    if (html.includes('page-body')) {
-        return 'gitbook';
-    }
-    return null;
+  if (html.includes("docs-sidebar") || html.includes("data-docusaurus-page")) {
+    return "docusaurus";
+  }
+  if (html.includes("md-content")) {
+    return "mkdocs";
+  }
+  if (html.includes('class="document"') || html.includes("class='document'") || html.includes('class="body"') && html.includes("highlight")) {
+    return "sphinx";
+  }
+  if (html.includes("page-body")) {
+    return "gitbook";
+  }
+  return null;
 }
 function extractWithSelectors(document, contentSelectors) {
-    for (const selector of contentSelectors) {
-        const el = document.querySelector(selector);
-        if (el)
-            return el;
-    }
-    return null;
+  for (const selector of contentSelectors) {
+    const el = document.querySelector(selector);
+    if (el) return el;
+  }
+  return null;
 }
 function stripElements(root, selectors) {
-    for (const selector of selectors) {
-        for (const el of Array.from(root.querySelectorAll(selector))) {
-            el.parentNode?.removeChild(el);
-        }
+  for (const selector of selectors) {
+    for (const el of Array.from(root.querySelectorAll(selector))) {
+      el.parentNode?.removeChild(el);
     }
+  }
 }
 function buildResult(document, contentEl) {
-    stripElements(contentEl, STRIP_SELECTORS);
-    const titleEl = contentEl.querySelector('h1') ??
-        document.querySelector('h1') ??
-        document.querySelector('title');
-    const rawTitle = titleEl?.textContent?.trim() ?? '';
-    const title = rawTitle.includes('|')
-        ? rawTitle.split('|')[0].trim()
-        : rawTitle;
-    if (!title)
-        return null;
-    const markdown = turndown.turndown(contentEl.innerHTML).trim();
-    if (!markdown)
-        return null;
-    return {
-        title,
-        markdown,
-        metadata: {},
-        links: [],
-        images: [],
-        extractor: 'site-specific',
-    };
+  stripElements(contentEl, STRIP_SELECTORS);
+  const titleEl = contentEl.querySelector("h1") ?? document.querySelector("h1") ?? document.querySelector("title");
+  const rawTitle = titleEl?.textContent?.trim() ?? "";
+  const title = rawTitle.includes("|") ? rawTitle.split("|")[0].trim() : rawTitle;
+  if (!title) return null;
+  const markdown = htmlToMarkdown(contentEl.innerHTML).trim();
+  if (!markdown) return null;
+  return {
+    title,
+    markdown,
+    metadata: {},
+    links: [],
+    images: [],
+    extractor: "site-specific"
+  };
 }
-export const docsGenericExtractor = {
-    name: 'docs-generic',
-    canHandle(_url, html) {
-        if (!html)
-            return false;
-        return detectFramework(html) !== null;
-    },
-    extract(html, _url) {
-        if (!html)
-            return null;
-        const framework = detectFramework(html);
-        if (!framework)
-            return null;
-        const { document } = parseHTML(html);
-        let contentSelectors;
-        switch (framework) {
-            case 'docusaurus':
-                contentSelectors = ['.markdown', 'article', 'main'];
-                break;
-            case 'mkdocs':
-                contentSelectors = ['.md-content'];
-                break;
-            case 'sphinx':
-                contentSelectors = ['.document', '.body'];
-                break;
-            case 'gitbook':
-                contentSelectors = ['.page-body'];
-                break;
-        }
-        const contentEl = extractWithSelectors(document, contentSelectors);
-        if (!contentEl)
-            return null;
-        return buildResult(document, contentEl);
-    },
+const docsGenericExtractor = {
+  name: "docs-generic",
+  canHandle(_url, html) {
+    if (!html) return false;
+    return detectFramework(html) !== null;
+  },
+  extract(html, _url) {
+    if (!html) return null;
+    const framework = detectFramework(html);
+    if (!framework) return null;
+    const { document } = parseHTML(html);
+    let contentSelectors;
+    switch (framework) {
+      case "docusaurus":
+        contentSelectors = [".markdown", "article", "main"];
+        break;
+      case "mkdocs":
+        contentSelectors = [".md-content"];
+        break;
+      case "sphinx":
+        contentSelectors = [".document", ".body"];
+        break;
+      case "gitbook":
+        contentSelectors = [".page-body"];
+        break;
+    }
+    const contentEl = extractWithSelectors(document, contentSelectors);
+    if (!contentEl) return null;
+    return buildResult(document, contentEl);
+  }
+};
+export {
+  docsGenericExtractor
 };
 //# sourceMappingURL=docs-generic.js.map

package/dist/extraction/site-extractors/docs-generic.js.map

@@ -1 +1 @@
-{"version":3,"file":"docs-generic.js","sourceRoot":"","sources":["../../../src/extraction/site-extractors/docs-generic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,eAAe,MAAM,UAAU,CAAC;AAGvC,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;AAIxF,MAAM,eAAe,GAAG;IACtB,KAAK;IACL,eAAe;IACf,UAAU;IACV,cAAc;IACd,oBAAoB;IACpB,iBAAiB;IACjB,iBAAiB;IACjB,QAAQ;IACR,QAAQ;CACT,CAAC;AAEF,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,CAAC;QAC3E,OAAO,YAAY,CAAC;IACtB,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;QAChC,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC;QACtE,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC;QAClE,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAC/B,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,oBAAoB,CAC3B,QAAkB,EAClB,gBAA0B;IAE1B,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;QACxC,MAAM,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,EAAE;YAAE,OAAO,EAAa,CAAC;IAC/B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CAAC,IAAa,EAAE,SAAmB;IACvD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YAC7D,EAAE,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAClB,QAAkB,EAClB,SAAkB;IAElB,aAAa,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IAE1C,MAAM,OAAO,GACX,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC;QAC7B,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC;QAC5B,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAElC,MAAM,QAAQ,GAAG,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC;QAClC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE;QAChC,CAAC,CAAC,QAAQ,CAAC;IAEb,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IAExB,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE3B,OAAO;QACL,KAAK;QACL,QAAQ;QACR,QAAQ,EAAE,EAAE;QACZ,KAAK,EAAE,EAAE;QACT,MAAM,EAAE,EAAE;QACV,SAAS,EAAE,eAAe;KAC3B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAc;IAC7C,IAAI,EAAE,cAAc;IAEpB,SAAS,CAAC,IAAY,EAAE,IAAa;QACnC,IAAI,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QACxB,OAAO,eAAe,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;IACxC,CAAC;IAED,OAAO,CAAC,IAAY,EAAE,IAAY;QAChC,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAE5B,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAErC,IAAI,gBAA0B,CAAC;QAC/B,QAAQ,SAAS,EAAE,CAAC;YAClB,KAAK,YAAY;gBACf,gBAAgB,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;gBACpD,MAAM;YACR,KAAK,QAAQ;gBACX,gBAAgB,GAAG,CAAC,aAAa,CAAC,CAAC;gBACnC,MAAM;YACR,KAAK,QAAQ;gBACX,gBAAgB,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;gBAC1C,MAAM;YACR,KAAK,SAAS;gBACZ,gBAAgB,GAAG,CAAC,YAAY,CAAC,CAAC;gBAClC,MAAM;QACV,CAAC;QAED,MAAM,SAAS,GAAG,oBAAoB,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;QACnE,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAE5B,OAAO,WAAW,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC1C,CAAC;CACF,CAAC"}
+{"version":3,"sources":["../../../src/extraction/site-extractors/docs-generic.ts"],"sourcesContent":["import { parseHTML } from 'linkedom';\nimport { htmlToMarkdown } from '../markdown.js';\nimport type { Extractor, ExtractionResult } from '../../types.js';\n\ntype DocFramework = 'docusaurus' | 'mkdocs' | 'sphinx' | 'gitbook';\n\nconst STRIP_SELECTORS = [\n  'nav',\n  '.docs-sidebar',\n  '.sidebar',\n  '.toc-wrapper',\n  '.table-of-contents',\n  '.version-picker',\n  '.pagination-nav',\n  'header',\n  'footer',\n];\n\nfunction detectFramework(html: string): DocFramework | null {\n  if (html.includes('docs-sidebar') || html.includes('data-docusaurus-page')) {\n    return 'docusaurus';\n  }\n  if (html.includes('md-content')) {\n    return 'mkdocs';\n  }\n  if (html.includes('class=\"document\"') || html.includes(\"class='document'\") ||\n      (html.includes('class=\"body\"') && html.includes('highlight'))) {\n    return 'sphinx';\n  }\n  if (html.includes('page-body')) {\n    return 'gitbook';\n  }\n  return null;\n}\n\nfunction extractWithSelectors(\n  document: Document,\n  contentSelectors: string[],\n): Element | null {\n  for (const selector of contentSelectors) {\n    const el = document.querySelector(selector);\n    if (el) return el as Element;\n  }\n  return null;\n}\n\nfunction stripElements(root: Element, selectors: string[]): void {\n  for (const selector of selectors) {\n    for (const el of Array.from(root.querySelectorAll(selector))) {\n      el.parentNode?.removeChild(el);\n    }\n  }\n}\n\nfunction buildResult(\n  document: Document,\n  contentEl: Element,\n): ExtractionResult | null {\n  stripElements(contentEl, STRIP_SELECTORS);\n\n  const titleEl =\n    contentEl.querySelector('h1') ??\n    document.querySelector('h1') ??\n    document.querySelector('title');\n\n  const rawTitle = titleEl?.textContent?.trim() ?? '';\n  const title = rawTitle.includes('|')\n    ? rawTitle.split('|')[0]!.trim()\n    : rawTitle;\n\n  if (!title) return null;\n\n  const markdown = htmlToMarkdown(contentEl.innerHTML).trim();\n  if (!markdown) return null;\n\n  return {\n    title,\n    markdown,\n    metadata: {},\n    links: [],\n    images: [],\n    extractor: 'site-specific',\n  };\n}\n\nexport const docsGenericExtractor: Extractor = {\n  name: 'docs-generic',\n\n  canHandle(_url: string, html?: string): boolean {\n    if (!html) return false;\n    return detectFramework(html) !== null;\n  },\n\n  extract(html: string, _url: string): ExtractionResult | null {\n    if (!html) return null;\n\n    const framework = detectFramework(html);\n    if (!framework) return null;\n\n    const { document } = parseHTML(html);\n\n    let contentSelectors: string[];\n    switch (framework) {\n      case 'docusaurus':\n        contentSelectors = ['.markdown', 'article', 'main'];\n        break;\n      case 'mkdocs':\n        contentSelectors = ['.md-content'];\n        break;\n      case 'sphinx':\n        contentSelectors = ['.document', '.body'];\n        break;\n      case 'gitbook':\n        contentSelectors = ['.page-body'];\n        break;\n    }\n\n    const contentEl = extractWithSelectors(document, contentSelectors);\n    if (!contentEl) return null;\n\n    return buildResult(document, contentEl);\n  },\n};\n"],"mappings":"AAAA,SAAS,iBAAiB;AAC1B,SAAS,sBAAsB;AAK/B,MAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,SAAS,gBAAgB,MAAmC;AAC1D,MAAI,KAAK,SAAS,cAAc,KAAK,KAAK,SAAS,sBAAsB,GAAG;AAC1E,WAAO;AAAA,EACT;AACA,MAAI,KAAK,SAAS,YAAY,GAAG;AAC/B,WAAO;AAAA,EACT;AACA,MAAI,KAAK,SAAS,kBAAkB,KAAK,KAAK,SAAS,kBAAkB,KACpE,KAAK,SAAS,cAAc,KAAK,KAAK,SAAS,WAAW,GAAI;AACjE,WAAO;AAAA,EACT;AACA,MAAI,KAAK,SAAS,WAAW,GAAG;AAC9B,WAAO;AAAA,EACT;AACA,SAAO;AACT;AAEA,SAAS,qBACP,UACA,kBACgB;AAChB,aAAW,YAAY,kBAAkB;AACvC,UAAM,KAAK,SAAS,cAAc,QAAQ;AAC1C,QAAI,GAAI,QAAO;AAAA,EACjB;AACA,SAAO;AACT;AAEA,SAAS,cAAc,MAAe,WAA2B;AAC/D,aAAW,YAAY,WAAW;AAChC,eAAW,MAAM,MAAM,KAAK,KAAK,iBAAiB,QAAQ,CAAC,GAAG;AAC5D,SAAG,YAAY,YAAY,EAAE;AAAA,IAC/B;AAAA,EACF;AACF;AAEA,SAAS,YACP,UACA,WACyB;AACzB,gBAAc,WAAW,eAAe;AAExC,QAAM,UACJ,UAAU,cAAc,IAAI,KAC5B,SAAS,cAAc,IAAI,KAC3B,SAAS,cAAc,OAAO;AAEhC,QAAM,WAAW,SAAS,aAAa,KAAK,KAAK;AACjD,QAAM,QAAQ,SAAS,SAAS,GAAG,IAC/B,SAAS,MAAM,GAAG,EAAE,CAAC,EAAG,KAAK,IAC7B;AAEJ,MAAI,CAAC,MAAO,QAAO;AAEnB,QAAM,WAAW,eAAe,UAAU,SAAS,EAAE,KAAK;AAC1D,MAAI,CAAC,SAAU,QAAO;AAEtB,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,UAAU,CAAC;AAAA,IACX,OAAO,CAAC;AAAA,IACR,QAAQ,CAAC;AAAA,IACT,WAAW;AAAA,EACb;AACF;AAEO,MAAM,uBAAkC;AAAA,EAC7C,MAAM;AAAA,EAEN,UAAU,MAAc,MAAwB;AAC9C,QAAI,CAAC,KAAM,QAAO;AAClB,WAAO,gBAAgB,IAAI,MAAM;AAAA,EACnC;AAAA,EAEA,QAAQ,MAAc,MAAuC;AAC3D,QAAI,CAAC,KAAM,QAAO;AAElB,UAAM,YAAY,gBAAgB,IAAI;AACtC,QAAI,CAAC,UAAW,QAAO;AAEvB,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AAEnC,QAAI;AACJ,YAAQ,WAAW;AAAA,MACjB,KAAK;AACH,2BAAmB,CAAC,aAAa,WAAW,MAAM;AAClD;AAAA,MACF,KAAK;AACH,2BAAmB,CAAC,aAAa;AACjC;AAAA,MACF,KAAK;AACH,2BAAmB,CAAC,aAAa,OAAO;AACxC;AAAA,MACF,KAAK;AACH,2BAAmB,CAAC,YAAY;AAChC;AAAA,IACJ;AAEA,UAAM,YAAY,qBAAqB,UAAU,gBAAgB;AACjE,QAAI,CAAC,UAAW,QAAO;AAEvB,WAAO,YAAY,UAAU,SAAS;AAAA,EACxC;AACF;","names":[]}

package/dist/extraction/site-extractors/github.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../../src/extraction/site-extractors/github.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAoB,MAAM,gBAAgB,CAAC;AAoGlE,eAAO,MAAM,eAAe,EAAE,SA2B7B,CAAC"}
+{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../../src/extraction/site-extractors/github.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAoB,MAAM,gBAAgB,CAAC;AAkGlE,eAAO,MAAM,eAAe,EAAE,SA2B7B,CAAC"}

package/dist/extraction/site-extractors/github.js

@@ -1,107 +1,99 @@
-import { parseHTML } from 'linkedom';
-import TurndownService from 'turndown';
-const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+import { parseHTML } from "linkedom";
+import { htmlToMarkdown } from "../markdown.js";
 function isIssueOrPR(url) {
-    return /\/issues\/\d+|\/pull\/\d+/.test(url);
+  return /\/issues\/\d+|\/pull\/\d+/.test(url);
 }
 function isBlob(url) {
-    return /\/blob\//.test(url);
+  return /\/blob\//.test(url);
 }
-function extractIssue(document, url) {
-    const titleEl = document.querySelector('.js-issue-title') ?? document.querySelector('.gh-header-title');
-    if (!titleEl)
-        return null;
-    const title = titleEl.textContent?.trim() ?? '';
-    const labelEls = document.querySelectorAll('.IssueLabel');
-    const labels = Array.from(labelEls)
-        .map((el) => el.textContent?.trim() ?? '')
-        .filter(Boolean);
-    const commentBodies = document.querySelectorAll('.d-block.comment-body');
-    if (commentBodies.length === 0)
-        return null;
-    const sections = [];
-    if (labels.length > 0) {
-        sections.push(`**Labels:** ${labels.join(', ')}\n`);
+function extractIssue(document, _url) {
+  const titleEl = document.querySelector(".js-issue-title") ?? document.querySelector(".gh-header-title");
+  if (!titleEl) return null;
+  const title = titleEl.textContent?.trim() ?? "";
+  const labelEls = document.querySelectorAll(".IssueLabel");
+  const labels = Array.from(labelEls).map((el) => el.textContent?.trim() ?? "").filter(Boolean);
+  const commentBodies = document.querySelectorAll(".d-block.comment-body");
+  if (commentBodies.length === 0) return null;
+  const sections = [];
+  if (labels.length > 0) {
+    sections.push(`**Labels:** ${labels.join(", ")}
+`);
+  }
+  Array.from(commentBodies).forEach((body, i) => {
+    const html = body.innerHTML;
+    const md = htmlToMarkdown(html).trim();
+    if (md) {
+      sections.push(i === 0 ? md : `---
+
+${md}`);
     }
-    Array.from(commentBodies).forEach((body, i) => {
-        const html = body.innerHTML;
-        const md = turndown.turndown(html).trim();
-        if (md) {
-            sections.push(i === 0 ? md : `---\n\n${md}`);
-        }
-    });
-    const markdown = sections.join('\n\n');
-    return {
-        title,
-        markdown,
-        metadata: {},
-        links: [],
-        images: [],
-        extractor: 'site-specific',
-    };
+  });
+  const markdown = sections.join("\n\n");
+  return {
+    title,
+    markdown,
+    metadata: {},
+    links: [],
+    images: [],
+    extractor: "site-specific"
+  };
 }
 function extractReadme(document) {
-    const titleEl = document.querySelector('title');
-    const rawTitle = titleEl?.textContent?.trim() ?? '';
-    const title = rawTitle.split(':')[0]?.trim() ?? rawTitle;
-    const readmeBody = document.querySelector('#readme .markdown-body') ??
-        document.querySelector('.markdown-body');
-    if (!readmeBody)
-        return null;
-    const markdown = turndown.turndown(readmeBody.innerHTML).trim();
-    if (!markdown)
-        return null;
-    return {
-        title,
-        markdown,
-        metadata: {},
-        links: [],
-        images: [],
-        extractor: 'site-specific',
-    };
+  const titleEl = document.querySelector("title");
+  const rawTitle = titleEl?.textContent?.trim() ?? "";
+  const title = rawTitle.split(":")[0]?.trim() ?? rawTitle;
+  const readmeBody = document.querySelector("#readme .markdown-body") ?? document.querySelector(".markdown-body");
+  if (!readmeBody) return null;
+  const markdown = htmlToMarkdown(readmeBody.innerHTML).trim();
+  if (!markdown) return null;
+  return {
+    title,
+    markdown,
+    metadata: {},
+    links: [],
+    images: [],
+    extractor: "site-specific"
+  };
 }
 function extractBlob(document) {
-    const titleEl = document.querySelector('title');
-    const title = titleEl?.textContent?.trim() ?? '';
-    const codeBlock = document.querySelector('.blob-code-content') ??
-        document.querySelector('.highlight') ??
-        document.querySelector('.markdown-body');
-    if (!codeBlock)
-        return null;
-    const markdown = turndown.turndown(codeBlock.innerHTML).trim();
-    if (!markdown)
-        return null;
-    return {
-        title,
-        markdown,
-        metadata: {},
-        links: [],
-        images: [],
-        extractor: 'site-specific',
-    };
+  const titleEl = document.querySelector("title");
+  const title = titleEl?.textContent?.trim() ?? "";
+  const codeBlock = document.querySelector(".blob-code-content") ?? document.querySelector(".highlight") ?? document.querySelector(".markdown-body");
+  if (!codeBlock) return null;
+  const markdown = htmlToMarkdown(codeBlock.innerHTML).trim();
+  if (!markdown) return null;
+  return {
+    title,
+    markdown,
+    metadata: {},
+    links: [],
+    images: [],
+    extractor: "site-specific"
+  };
 }
-export const githubExtractor = {
-    name: 'github',
-    canHandle(url) {
-        try {
-            const hostname = new URL(url).hostname;
-            return hostname === 'github.com' || hostname.endsWith('.github.com');
-        }
-        catch {
-            return false;
-        }
-    },
-    extract(html, url) {
-        if (!html)
-            return null;
-        const { document } = parseHTML(html);
-        if (isIssueOrPR(url)) {
-            return extractIssue(document, url);
-        }
-        if (isBlob(url)) {
-            return extractBlob(document);
-        }
-        return extractReadme(document);
-    },
+const githubExtractor = {
+  name: "github",
+  canHandle(url) {
+    try {
+      const hostname = new URL(url).hostname;
+      return hostname === "github.com" || hostname.endsWith(".github.com");
+    } catch {
+      return false;
+    }
+  },
+  extract(html, url) {
+    if (!html) return null;
+    const { document } = parseHTML(html);
+    if (isIssueOrPR(url)) {
+      return extractIssue(document, url);
+    }
+    if (isBlob(url)) {
+      return extractBlob(document);
+    }
+    return extractReadme(document);
+  }
+};
+export {
+  githubExtractor
 };
 //# sourceMappingURL=github.js.map

package/dist/extraction/site-extractors/github.js.map

@@ -1 +1 @@
-{"version":3,"file":"github.js","sourceRoot":"","sources":["../../../src/extraction/site-extractors/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,eAAe,MAAM,UAAU,CAAC;AAGvC,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;AAExF,SAAS,WAAW,CAAC,GAAW;IAC9B,OAAO,2BAA2B,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,MAAM,CAAC,GAAW;IACzB,OAAO,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,YAAY,CAAC,QAAkB,EAAE,GAAW;IACnD,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,iBAAiB,CAAC,IAAI,QAAQ,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC;IACxG,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAE1B,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAEhD,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC;SAChC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;SACzC,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,MAAM,aAAa,GAAG,QAAQ,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC;IACzE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAE5C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QAC5C,MAAM,IAAI,GAAI,IAAgB,CAAC,SAAS,CAAC;QACzC,MAAM,EAAE,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1C,IAAI,EAAE,EAAE,CAAC;YACP,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEvC,OAAO;QACL,KAAK;QACL,QAAQ;QACR,QAAQ,EAAE,EAAE;QACZ,KAAK,EAAE,EAAE;QACT,MAAM,EAAE,EAAE;QACV,SAAS,EAAE,eAAe;KAC3B,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,QAAkB;IACvC,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,QAAQ,CAAC;IAEzD,MAAM,UAAU,GACd,QAAQ,CAAC,aAAa,CAAC,wBAAwB,CAAC;QAChD,QAAQ,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC;IAE3C,IAAI,CAAC,UAAU;QAAE,OAAO,IAAI,CAAC;IAE7B,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAE,UAAsB,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7E,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE3B,OAAO;QACL,KAAK;QACL,QAAQ;QACR,QAAQ,EAAE,EAAE;QACZ,KAAK,EAAE,EAAE;QACT,MAAM,EAAE,EAAE;QACV,SAAS,EAAE,eAAe;KAC3B,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,QAAkB;IACrC,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAEjD,MAAM,SAAS,GACb,QAAQ,CAAC,aAAa,CAAC,oBAAoB,CAAC;QAC5C,QAAQ,CAAC,aAAa,CAAC,YAAY,CAAC;QACpC,QAAQ,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC;IAE3C,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IAE5B,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAE,SAAqB,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5E,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE3B,OAAO;QACL,KAAK;QACL,QAAQ;QACR,QAAQ,EAAE,EAAE;QACZ,KAAK,EAAE,EAAE;QACT,MAAM,EAAE,EAAE;QACV,SAAS,EAAE,eAAe;KAC3B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,eAAe,GAAc;IACxC,IAAI,EAAE,QAAQ;IAEd,SAAS,CAAC,GAAW;QACnB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACvC,OAAO,QAAQ,KAAK,YAAY,IAAI,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QACvE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAY,EAAE,GAAW;QAC/B,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAErC,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YACrB,OAAO,YAAY,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAChB,OAAO,WAAW,CAAC,QAAQ,CAAC,CAAC;QAC/B,CAAC;QAED,OAAO,aAAa,CAAC,QAAQ,CAAC,CAAC;IACjC,CAAC;CACF,CAAC"}
+{"version":3,"sources":["../../../src/extraction/site-extractors/github.ts"],"sourcesContent":["import { parseHTML } from 'linkedom';\nimport { htmlToMarkdown } from '../markdown.js';\nimport type { Extractor, ExtractionResult } from '../../types.js';\n\nfunction isIssueOrPR(url: string): boolean {\n  return /\\/issues\\/\\d+|\\/pull\\/\\d+/.test(url);\n}\n\nfunction isBlob(url: string): boolean {\n  return /\\/blob\\//.test(url);\n}\n\nfunction extractIssue(document: Document, _url: string): ExtractionResult | null {\n  const titleEl = document.querySelector('.js-issue-title') ?? document.querySelector('.gh-header-title');\n  if (!titleEl) return null;\n\n  const title = titleEl.textContent?.trim() ?? '';\n\n  const labelEls = document.querySelectorAll('.IssueLabel');\n  const labels = Array.from(labelEls)\n    .map((el) => el.textContent?.trim() ?? '')\n    .filter(Boolean);\n\n  const commentBodies = document.querySelectorAll('.d-block.comment-body');\n  if (commentBodies.length === 0) return null;\n\n  const sections: string[] = [];\n\n  if (labels.length > 0) {\n    sections.push(`**Labels:** ${labels.join(', ')}\\n`);\n  }\n\n  Array.from(commentBodies).forEach((body, i) => {\n    const html = (body as Element).innerHTML;\n    const md = htmlToMarkdown(html).trim();\n    if (md) {\n      sections.push(i === 0 ? md : `---\\n\\n${md}`);\n    }\n  });\n\n  const markdown = sections.join('\\n\\n');\n\n  return {\n    title,\n    markdown,\n    metadata: {},\n    links: [],\n    images: [],\n    extractor: 'site-specific',\n  };\n}\n\nfunction extractReadme(document: Document): ExtractionResult | null {\n  const titleEl = document.querySelector('title');\n  const rawTitle = titleEl?.textContent?.trim() ?? '';\n  const title = rawTitle.split(':')[0]?.trim() ?? rawTitle;\n\n  const readmeBody =\n    document.querySelector('#readme .markdown-body') ??\n    document.querySelector('.markdown-body');\n\n  if (!readmeBody) return null;\n\n  const markdown = htmlToMarkdown((readmeBody as Element).innerHTML).trim();\n  if (!markdown) return null;\n\n  return {\n    title,\n    markdown,\n    metadata: {},\n    links: [],\n    images: [],\n    extractor: 'site-specific',\n  };\n}\n\nfunction extractBlob(document: Document): ExtractionResult | null {\n  const titleEl = document.querySelector('title');\n  const title = titleEl?.textContent?.trim() ?? '';\n\n  const codeBlock =\n    document.querySelector('.blob-code-content') ??\n    document.querySelector('.highlight') ??\n    document.querySelector('.markdown-body');\n\n  if (!codeBlock) return null;\n\n  const markdown = htmlToMarkdown((codeBlock as Element).innerHTML).trim();\n  if (!markdown) return null;\n\n  return {\n    title,\n    markdown,\n    metadata: {},\n    links: [],\n    images: [],\n    extractor: 'site-specific',\n  };\n}\n\nexport const githubExtractor: Extractor = {\n  name: 'github',\n\n  canHandle(url: string): boolean {\n    try {\n      const hostname = new URL(url).hostname;\n      return hostname === 'github.com' || hostname.endsWith('.github.com');\n    } catch {\n      return false;\n    }\n  },\n\n  extract(html: string, url: string): ExtractionResult | null {\n    if (!html) return null;\n\n    const { document } = parseHTML(html);\n\n    if (isIssueOrPR(url)) {\n      return extractIssue(document, url);\n    }\n\n    if (isBlob(url)) {\n      return extractBlob(document);\n    }\n\n    return extractReadme(document);\n  },\n};\n"],"mappings":"AAAA,SAAS,iBAAiB;AAC1B,SAAS,sBAAsB;AAG/B,SAAS,YAAY,KAAsB;AACzC,SAAO,4BAA4B,KAAK,GAAG;AAC7C;AAEA,SAAS,OAAO,KAAsB;AACpC,SAAO,WAAW,KAAK,GAAG;AAC5B;AAEA,SAAS,aAAa,UAAoB,MAAuC;AAC/E,QAAM,UAAU,SAAS,cAAc,iBAAiB,KAAK,SAAS,cAAc,kBAAkB;AACtG,MAAI,CAAC,QAAS,QAAO;AAErB,QAAM,QAAQ,QAAQ,aAAa,KAAK,KAAK;AAE7C,QAAM,WAAW,SAAS,iBAAiB,aAAa;AACxD,QAAM,SAAS,MAAM,KAAK,QAAQ,EAC/B,IAAI,CAAC,OAAO,GAAG,aAAa,KAAK,KAAK,EAAE,EACxC,OAAO,OAAO;AAEjB,QAAM,gBAAgB,SAAS,iBAAiB,uBAAuB;AACvE,MAAI,cAAc,WAAW,EAAG,QAAO;AAEvC,QAAM,WAAqB,CAAC;AAE5B,MAAI,OAAO,SAAS,GAAG;AACrB,aAAS,KAAK,eAAe,OAAO,KAAK,IAAI,CAAC;AAAA,CAAI;AAAA,EACpD;AAEA,QAAM,KAAK,aAAa,EAAE,QAAQ,CAAC,MAAM,MAAM;AAC7C,UAAM,OAAQ,KAAiB;AAC/B,UAAM,KAAK,eAAe,IAAI,EAAE,KAAK;AACrC,QAAI,IAAI;AACN,eAAS,KAAK,MAAM,IAAI,KAAK;AAAA;AAAA,EAAU,EAAE,EAAE;AAAA,IAC7C;AAAA,EACF,CAAC;AAED,QAAM,WAAW,SAAS,KAAK,MAAM;AAErC,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,UAAU,CAAC;AAAA,IACX,OAAO,CAAC;AAAA,IACR,QAAQ,CAAC;AAAA,IACT,WAAW;AAAA,EACb;AACF;AAEA,SAAS,cAAc,UAA6C;AAClE,QAAM,UAAU,SAAS,cAAc,OAAO;AAC9C,QAAM,WAAW,SAAS,aAAa,KAAK,KAAK;AACjD,QAAM,QAAQ,SAAS,MAAM,GAAG,EAAE,CAAC,GAAG,KAAK,KAAK;AAEhD,QAAM,aACJ,SAAS,cAAc,wBAAwB,KAC/C,SAAS,cAAc,gBAAgB;AAEzC,MAAI,CAAC,WAAY,QAAO;AAExB,QAAM,WAAW,eAAgB,WAAuB,SAAS,EAAE,KAAK;AACxE,MAAI,CAAC,SAAU,QAAO;AAEtB,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,UAAU,CAAC;AAAA,IACX,OAAO,CAAC;AAAA,IACR,QAAQ,CAAC;AAAA,IACT,WAAW;AAAA,EACb;AACF;AAEA,SAAS,YAAY,UAA6C;AAChE,QAAM,UAAU,SAAS,cAAc,OAAO;AAC9C,QAAM,QAAQ,SAAS,aAAa,KAAK,KAAK;AAE9C,QAAM,YACJ,SAAS,cAAc,oBAAoB,KAC3C,SAAS,cAAc,YAAY,KACnC,SAAS,cAAc,gBAAgB;AAEzC,MAAI,CAAC,UAAW,QAAO;AAEvB,QAAM,WAAW,eAAgB,UAAsB,SAAS,EAAE,KAAK;AACvE,MAAI,CAAC,SAAU,QAAO;AAEtB,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA,UAAU,CAAC;AAAA,IACX,OAAO,CAAC;AAAA,IACR,QAAQ,CAAC;AAAA,IACT,WAAW;AAAA,EACb;AACF;AAEO,MAAM,kBAA6B;AAAA,EACxC,MAAM;AAAA,EAEN,UAAU,KAAsB;AAC9B,QAAI;AACF,YAAM,WAAW,IAAI,IAAI,GAAG,EAAE;AAC9B,aAAO,aAAa,gBAAgB,SAAS,SAAS,aAAa;AAAA,IACrE,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,QAAQ,MAAc,KAAsC;AAC1D,QAAI,CAAC,KAAM,QAAO;AAElB,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AAEnC,QAAI,YAAY,GAAG,GAAG;AACpB,aAAO,aAAa,UAAU,GAAG;AAAA,IACnC;AAEA,QAAI,OAAO,GAAG,GAAG;AACf,aAAO,YAAY,QAAQ;AAAA,IAC7B;AAEA,WAAO,cAAc,QAAQ;AAAA,EAC/B;AACF;","names":[]}

package/dist/extraction/site-extractors/mdn.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mdn.d.ts","sourceRoot":"","sources":["../../../src/extraction/site-extractors/mdn.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAoB,MAAM,gBAAgB,CAAC;AAalE,eAAO,MAAM,YAAY,EAAE,SAqD1B,CAAC"}
+{"version":3,"file":"mdn.d.ts","sourceRoot":"","sources":["../../../src/extraction/site-extractors/mdn.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAoB,MAAM,gBAAgB,CAAC;AAWlE,eAAO,MAAM,YAAY,EAAE,SAqD1B,CAAC"}

package/dist/extraction/site-extractors/mdn.js

@@ -1,58 +1,50 @@
-import { parseHTML } from 'linkedom';
-import TurndownService from 'turndown';
-const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+import { parseHTML } from "linkedom";
+import { htmlToMarkdown } from "../markdown.js";
 const STRIP_SELECTORS = [
-    'nav',
-    '.sidebar',
-    'header',
-    'footer',
-    '.bc-head',
-    '.metadata',
+  "nav",
+  ".sidebar",
+  "header",
+  "footer",
+  ".bc-head",
+  ".metadata"
 ];
-export const mdnExtractor = {
-    name: 'mdn',
-    canHandle(url) {
-        try {
-            const hostname = new URL(url).hostname;
-            return hostname === 'developer.mozilla.org';
-        }
-        catch {
-            return false;
-        }
-    },
-    extract(html, url) {
-        if (!html)
-            return null;
-        const { document } = parseHTML(html);
-        const article = document.querySelector('article.main-page-content') ??
-            document.querySelector('.section-content') ??
-            document.querySelector('article');
-        if (!article)
-            return null;
-        for (const selector of STRIP_SELECTORS) {
-            for (const el of Array.from(article.querySelectorAll(selector))) {
-                el.parentNode?.removeChild(el);
-            }
-        }
-        const titleEl = article.querySelector('h1') ??
-            document.querySelector('title');
-        const rawTitle = titleEl?.textContent?.trim() ?? '';
-        const title = rawTitle.includes('|')
-            ? rawTitle.split('|')[0].trim()
-            : rawTitle;
-        if (!title)
-            return null;
-        const markdown = turndown.turndown(article.innerHTML).trim();
-        if (!markdown)
-            return null;
-        return {
-            title,
-            markdown,
-            metadata: {},
-            links: [],
-            images: [],
-            extractor: 'site-specific',
-        };
-    },
+const mdnExtractor = {
+  name: "mdn",
+  canHandle(url) {
+    try {
+      const hostname = new URL(url).hostname;
+      return hostname === "developer.mozilla.org";
+    } catch {
+      return false;
+    }
+  },
+  extract(html, _url) {
+    if (!html) return null;
+    const { document } = parseHTML(html);
+    const article = document.querySelector("article.main-page-content") ?? document.querySelector(".section-content") ?? document.querySelector("article");
+    if (!article) return null;
+    for (const selector of STRIP_SELECTORS) {
+      for (const el of Array.from(article.querySelectorAll(selector))) {
+        el.parentNode?.removeChild(el);
+      }
+    }
+    const titleEl = article.querySelector("h1") ?? document.querySelector("title");
+    const rawTitle = titleEl?.textContent?.trim() ?? "";
+    const title = rawTitle.includes("|") ? rawTitle.split("|")[0].trim() : rawTitle;
+    if (!title) return null;
+    const markdown = htmlToMarkdown(article.innerHTML).trim();
+    if (!markdown) return null;
+    return {
+      title,
+      markdown,
+      metadata: {},
+      links: [],
+      images: [],
+      extractor: "site-specific"
+    };
+  }
+};
+export {
+  mdnExtractor
 };
 //# sourceMappingURL=mdn.js.map

package/dist/extraction/site-extractors/mdn.js.map

@@ -1 +1 @@
-{"version":3,"file":"mdn.js","sourceRoot":"","sources":["../../../src/extraction/site-extractors/mdn.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,eAAe,MAAM,UAAU,CAAC;AAGvC,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;AAExF,MAAM,eAAe,GAAG;IACtB,KAAK;IACL,UAAU;IACV,QAAQ;IACR,QAAQ;IACR,UAAU;IACV,WAAW;CACZ,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAc;IACrC,IAAI,EAAE,KAAK;IAEX,SAAS,CAAC,GAAW;QACnB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACvC,OAAO,QAAQ,KAAK,uBAAuB,CAAC;QAC9C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAY,EAAE,GAAW;QAC/B,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAErC,MAAM,OAAO,GACX,QAAQ,CAAC,aAAa,CAAC,2BAA2B,CAAC;YACnD,QAAQ,CAAC,aAAa,CAAC,kBAAkB,CAAC;YAC1C,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAEpC,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAE1B,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;YACvC,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;gBAChE,EAAE,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GACX,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC;YAC3B,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAElC,MAAM,QAAQ,GAAG,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC;YAClC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE;YAChC,CAAC,CAAC,QAAQ,CAAC;QAEb,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QAExB,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAE,OAAmB,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1E,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,OAAO;YACL,KAAK;YACL,QAAQ;YACR,QAAQ,EAAE,EAAE;YACZ,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,SAAS,EAAE,eAAe;SAC3B,CAAC;IACJ,CAAC;CACF,CAAC"}
+{"version":3,"sources":["../../../src/extraction/site-extractors/mdn.ts"],"sourcesContent":["import { parseHTML } from 'linkedom';\nimport { htmlToMarkdown } from '../markdown.js';\nimport type { Extractor, ExtractionResult } from '../../types.js';\n\nconst STRIP_SELECTORS = [\n  'nav',\n  '.sidebar',\n  'header',\n  'footer',\n  '.bc-head',\n  '.metadata',\n];\n\nexport const mdnExtractor: Extractor = {\n  name: 'mdn',\n\n  canHandle(url: string): boolean {\n    try {\n      const hostname = new URL(url).hostname;\n      return hostname === 'developer.mozilla.org';\n    } catch {\n      return false;\n    }\n  },\n\n  extract(html: string, _url: string): ExtractionResult | null {\n    if (!html) return null;\n\n    const { document } = parseHTML(html);\n\n    const article =\n      document.querySelector('article.main-page-content') ??\n      document.querySelector('.section-content') ??\n      document.querySelector('article');\n\n    if (!article) return null;\n\n    for (const selector of STRIP_SELECTORS) {\n      for (const el of Array.from(article.querySelectorAll(selector))) {\n        el.parentNode?.removeChild(el);\n      }\n    }\n\n    const titleEl =\n      article.querySelector('h1') ??\n      document.querySelector('title');\n\n    const rawTitle = titleEl?.textContent?.trim() ?? '';\n    const title = rawTitle.includes('|')\n      ? rawTitle.split('|')[0]!.trim()\n      : rawTitle;\n\n    if (!title) return null;\n\n    const markdown = htmlToMarkdown((article as Element).innerHTML).trim();\n    if (!markdown) return null;\n\n    return {\n      title,\n      markdown,\n      metadata: {},\n      links: [],\n      images: [],\n      extractor: 'site-specific',\n    };\n  },\n};\n"],"mappings":"AAAA,SAAS,iBAAiB;AAC1B,SAAS,sBAAsB;AAG/B,MAAM,kBAAkB;AAAA,EACtB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,MAAM,eAA0B;AAAA,EACrC,MAAM;AAAA,EAEN,UAAU,KAAsB;AAC9B,QAAI;AACF,YAAM,WAAW,IAAI,IAAI,GAAG,EAAE;AAC9B,aAAO,aAAa;AAAA,IACtB,QAAQ;AACN,aAAO;AAAA,IACT;AAAA,EACF;AAAA,EAEA,QAAQ,MAAc,MAAuC;AAC3D,QAAI,CAAC,KAAM,QAAO;AAElB,UAAM,EAAE,SAAS,IAAI,UAAU,IAAI;AAEnC,UAAM,UACJ,SAAS,cAAc,2BAA2B,KAClD,SAAS,cAAc,kBAAkB,KACzC,SAAS,cAAc,SAAS;AAElC,QAAI,CAAC,QAAS,QAAO;AAErB,eAAW,YAAY,iBAAiB;AACtC,iBAAW,MAAM,MAAM,KAAK,QAAQ,iBAAiB,QAAQ,CAAC,GAAG;AAC/D,WAAG,YAAY,YAAY,EAAE;AAAA,MAC/B;AAAA,IACF;AAEA,UAAM,UACJ,QAAQ,cAAc,IAAI,KAC1B,SAAS,cAAc,OAAO;AAEhC,UAAM,WAAW,SAAS,aAAa,KAAK,KAAK;AACjD,UAAM,QAAQ,SAAS,SAAS,GAAG,IAC/B,SAAS,MAAM,GAAG,EAAE,CAAC,EAAG,KAAK,IAC7B;AAEJ,QAAI,CAAC,MAAO,QAAO;AAEnB,UAAM,WAAW,eAAgB,QAAoB,SAAS,EAAE,KAAK;AACrE,QAAI,CAAC,SAAU,QAAO;AAEtB,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA,UAAU,CAAC;AAAA,MACX,OAAO,CAAC;AAAA,MACR,QAAQ,CAAC;AAAA,MACT,WAAW;AAAA,IACb;AAAA,EACF;AACF;","names":[]}

package/dist/extraction/site-extractors/stackoverflow.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"stackoverflow.d.ts","sourceRoot":"","sources":["../../../src/extraction/site-extractors/stackoverflow.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAoB,MAAM,gBAAgB,CAAC;AAgElE,eAAO,MAAM,sBAAsB,EAAE,SAkDpC,CAAC"}
+{"version":3,"file":"stackoverflow.d.ts","sourceRoot":"","sources":["../../../src/extraction/site-extractors/stackoverflow.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAoB,MAAM,gBAAgB,CAAC;AA8DlE,eAAO,MAAM,sBAAsB,EAAE,SAkDpC,CAAC"}