@staticn0va/wigolo 0.1.0

package/dist/extraction/pipeline.d.ts

@@ -0,0 +1,11 @@
+import type { ExtractionResult, Extractor } from '../types.js';
+export interface ExtractionOptions {
+    maxChars?: number;
+    section?: string;
+    sectionIndex?: number;
+    contentType?: string;
+    pdfBuffer?: Buffer;
+}
+export declare function registerExtractor(extractor: Extractor): void;
+export declare function extractContent(html: string, url: string, options?: ExtractionOptions): Promise<ExtractionResult>;
+//# sourceMappingURL=pipeline.d.ts.map

package/dist/extraction/pipeline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/extraction/pipeline.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,gBAAgB,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAU/D,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AASD,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,SAAS,GAAG,IAAI,CAE5D;AAED,wBAAsB,cAAc,CAClC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,EACX,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,gBAAgB,CAAC,CAmE3B"}

package/dist/extraction/pipeline.js

@@ -0,0 +1,95 @@
+import { defuddleExtract } from './defuddle.js';
+import { readabilityExtract } from './readability.js';
+import { trafilaturaExtract, isTrafilaturaAvailable } from './trafilatura.js';
+import { htmlToMarkdown, extractSection, extractLinksAndImages } from './markdown.js';
+import { githubExtractor } from './site-extractors/github.js';
+import { stackoverflowExtractor } from './site-extractors/stackoverflow.js';
+import { mdnExtractor } from './site-extractors/mdn.js';
+import { docsGenericExtractor } from './site-extractors/docs-generic.js';
+import { createLogger } from '../logger.js';
+import { getConfig } from '../config.js';
+const log = createLogger('extract');
+const siteExtractors = [
+    githubExtractor,
+    stackoverflowExtractor,
+    mdnExtractor,
+    docsGenericExtractor,
+];
+export function registerExtractor(extractor) {
+    siteExtractors.push(extractor);
+}
+export async function extractContent(html, url, options = {}) {
+    let result = null;
+    if (options.contentType === 'application/pdf') {
+        let pdfText = '';
+        if (options.pdfBuffer) {
+            try {
+                const pdfParse = (await import('pdf-parse')).default;
+                const parsed = await pdfParse(options.pdfBuffer);
+                pdfText = parsed.text ?? '';
+            }
+            catch (err) {
+                log.warn('pdf-parse failed', { url, error: String(err) });
+            }
+        }
+        result = {
+            title: '',
+            markdown: pdfText,
+            metadata: {},
+            links: [],
+            images: [],
+            extractor: 'turndown',
+        };
+        return applyPostProcessing(result, options);
+    }
+    const siteExtractor = siteExtractors.find((e) => e.canHandle(url, html));
+    if (siteExtractor) {
+        const extracted = siteExtractor.extract(html, url);
+        if (extracted) {
+            result = extracted;
+            return applyPostProcessing(result, options);
+        }
+    }
+    result = await defuddleExtract(html, url);
+    if (!result) {
+        const config = getConfig();
+        if (config.trafilatura !== 'never') {
+            const trafAvailable = await isTrafilaturaAvailable();
+            if (trafAvailable) {
+                result = await trafilaturaExtract(html, url);
+                if (result) {
+                    log.info('Trafilatura extraction succeeded', { url, chars: result.markdown.length });
+                    return applyPostProcessing(result, options);
+                }
+            }
+        }
+    }
+    if (!result) {
+        result = readabilityExtract(html, url);
+    }
+    if (!result) {
+        const markdown = htmlToMarkdown(html);
+        result = {
+            title: '',
+            markdown,
+            metadata: {},
+            links: [],
+            images: [],
+            extractor: 'turndown',
+        };
+    }
+    return applyPostProcessing(result, options);
+}
+function applyPostProcessing(result, options) {
+    let markdown = result.markdown;
+    if (options.section) {
+        const { content } = extractSection(markdown, options.section, options.sectionIndex ?? 0);
+        markdown = content;
+    }
+    const { links, images } = extractLinksAndImages(markdown);
+    if (options.maxChars && markdown.length > options.maxChars) {
+        markdown = markdown.slice(0, options.maxChars);
+    }
+    return { ...result, markdown, links, images };
+}
+//# sourceMappingURL=pipeline.js.map

package/dist/extraction/pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/extraction/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,eAAe,CAAC;AAEtF,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,MAAM,oCAAoC,CAAC;AAC5E,OAAO,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,MAAM,GAAG,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;AAUpC,MAAM,cAAc,GAAgB;IAClC,eAAe;IACf,sBAAsB;IACtB,YAAY;IACZ,oBAAoB;CACrB,CAAC;AAEF,MAAM,UAAU,iBAAiB,CAAC,SAAoB;IACpD,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,IAAY,EACZ,GAAW,EACX,UAA6B,EAAE;IAE/B,IAAI,MAAM,GAA4B,IAAI,CAAC;IAE3C,IAAI,OAAO,CAAC,WAAW,KAAK,iBAAiB,EAAE,CAAC;QAC9C,IAAI,OAAO,GAAG,EAAE,CAAC;QACjB,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACtB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,CAAC,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC;gBACrD,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;gBACjD,OAAO,GAAG,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YAC9B,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,GAAG,CAAC,IAAI,CAAC,kBAAkB,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;QACD,MAAM,GAAG;YACP,KAAK,EAAE,EAAE;YACT,QAAQ,EAAE,OAAO;YACjB,QAAQ,EAAE,EAAE;YACZ,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,SAAS,EAAE,UAAU;SACtB,CAAC;QACF,OAAO,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,aAAa,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;IACzE,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QACnD,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,GAAG,SAAS,CAAC;YACnB,OAAO,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QAC9C,CAAC;IACH,CAAC;IAED,MAAM,GAAG,MAAM,eAAe,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAE1C,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,MAAM,GAAG,SAAS,EAAE,CAAC;QAC3B,IAAI,MAAM,CAAC,WAAW,KAAK,OAAO,EAAE,CAAC;YACnC,MAAM,aAAa,GAAG,MAAM,sBAAsB,EAAE,CAAC;YACrD,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,GAAG,MAAM,kBAAkB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;gBAC7C,IAAI,MAAM,EAAE,CAAC;oBACX,GAAG,CAAC,IAAI,CAAC,kCAAkC,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;oBACrF,OAAO,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;gBAC9C,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,GAAG,kBAAkB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACzC,CAAC;IAED,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,QAAQ,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,GAAG;YACP,KAAK,EAAE,EAAE;YACT,QAAQ;YACR,QAAQ,EAAE,EAAE;YACZ,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,SAAS,EAAE,UAAU;SACtB,CAAC;IACJ,CAAC;IAED,OAAO,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,mBAAmB,CAC1B,MAAwB,EACxB,OAA0B;IAE1B,IAAI,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IAE/B,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,EAAE,OAAO,EAAE,GAAG,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC;QACzF,QAAQ,GAAG,OAAO,CAAC;IACrB,CAAC;IAED,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;IAE1D,IAAI,OAAO,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC3D,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjD,CAAC;IAED,OAAO,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AAChD,CAAC"}

package/dist/extraction/readability.d.ts

@@ -0,0 +1,3 @@
+import type { ExtractionResult } from '../types.js';
+export declare function readabilityExtract(html: string, url: string): ExtractionResult | null;
+//# sourceMappingURL=readability.d.ts.map

package/dist/extraction/readability.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"readability.d.ts","sourceRoot":"","sources":["../../src/extraction/readability.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAIpD,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI,CA0BrF"}

package/dist/extraction/readability.js

@@ -0,0 +1,32 @@
+import { Readability } from '@mozilla/readability';
+import { parseHTML } from 'linkedom';
+import TurndownService from 'turndown';
+const MIN_CONTENT_THRESHOLD = 100;
+export function readabilityExtract(html, url) {
+    try {
+        const { document } = parseHTML(html);
+        const reader = new Readability(document);
+        const article = reader.parse();
+        if (!article || !article.content)
+            return null;
+        const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+        const markdown = turndown.turndown(article.content);
+        if (markdown.length < MIN_CONTENT_THRESHOLD)
+            return null;
+        return {
+            title: article.title ?? '',
+            markdown,
+            metadata: {
+                author: article.byline || undefined,
+                language: article.lang || undefined,
+            },
+            links: [],
+            images: [],
+            extractor: 'readability',
+        };
+    }
+    catch {
+        return null;
+    }
+}
+//# sourceMappingURL=readability.js.map

package/dist/extraction/readability.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"readability.js","sourceRoot":"","sources":["../../src/extraction/readability.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACnD,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,eAAe,MAAM,UAAU,CAAC;AAGvC,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAElC,MAAM,UAAU,kBAAkB,CAAC,IAAY,EAAE,GAAW;IAC1D,IAAI,CAAC;QACH,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,MAAM,GAAG,IAAI,WAAW,CAAC,QAAe,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAE9C,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;QACxF,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpD,IAAI,QAAQ,CAAC,MAAM,GAAG,qBAAqB;YAAE,OAAO,IAAI,CAAC;QAEzD,OAAO;YACL,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,EAAE;YAC1B,QAAQ;YACR,QAAQ,EAAE;gBACR,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,SAAS;gBACnC,QAAQ,EAAE,OAAO,CAAC,IAAI,IAAI,SAAS;aACpC;YACD,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,SAAS,EAAE,aAAa;SACzB,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC"}

package/dist/extraction/schema.d.ts

@@ -0,0 +1,7 @@
+export interface JsonSchema {
+    type?: string;
+    properties?: Record<string, JsonSchema>;
+    items?: JsonSchema;
+}
+export declare function extractWithSchema(html: string, schema: JsonSchema): Record<string, unknown>;
+//# sourceMappingURL=schema.d.ts.map

package/dist/extraction/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/extraction/schema.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,UAAU;IACzB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACxC,KAAK,CAAC,EAAE,UAAU,CAAC;CACpB;AAED,wBAAgB,iBAAiB,CAC/B,IAAI,EAAE,MAAM,EACZ,MAAM,EAAE,UAAU,GACjB,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAmBzB"}

package/dist/extraction/schema.js

@@ -0,0 +1,86 @@
+import { parseHTML } from 'linkedom';
+import { extractJsonLd, matchJsonLdToSchema } from './jsonld.js';
+export function extractWithSchema(html, schema) {
+    if (!html || !schema.properties)
+        return {};
+    const jsonLdBlocks = extractJsonLd(html);
+    const jsonLdResult = matchJsonLdToSchema(jsonLdBlocks, schema);
+    const { document: doc } = parseHTML(html);
+    const heuristicResult = {};
+    for (const [fieldName, fieldSchema] of Object.entries(schema.properties)) {
+        if (jsonLdResult[fieldName] !== undefined)
+            continue;
+        const value = findFieldValue(doc, fieldName, fieldSchema);
+        if (value !== undefined) {
+            heuristicResult[fieldName] = value;
+        }
+    }
+    return { ...jsonLdResult, ...heuristicResult };
+}
+function findFieldValue(doc, fieldName, schema) {
+    const normalizedName = fieldName.toLowerCase().replace(/_/g, '-');
+    const compactName = fieldName.replace(/_/g, '').toLowerCase();
+    const variants = [fieldName, normalizedName, compactName];
+    if (schema.type === 'array') {
+        return findArrayValues(doc, variants);
+    }
+    return findSingleValue(doc, variants);
+}
+function cssEscape(value) {
+    return value.replace(/([^\w-])/g, '\\$1');
+}
+function findSingleValue(doc, variants) {
+    for (const name of variants) {
+        const byItemprop = doc.querySelector(`[itemprop="${name}"]`);
+        if (byItemprop) {
+            const text = byItemprop.getAttribute('content') ?? byItemprop.textContent?.trim();
+            if (text)
+                return text;
+        }
+        // Substring match is intentional — heuristic best-effort for partial class names
+        const byClass = doc.querySelector(`[class*="${name}"]`);
+        if (byClass) {
+            const text = byClass.textContent?.trim();
+            if (text)
+                return text;
+        }
+        const allWithAria = doc.querySelectorAll('[aria-label]');
+        for (const el of allWithAria) {
+            const label = el.getAttribute('aria-label')?.toLowerCase().replace(/\s+/g, '-') ?? '';
+            if (label === name.toLowerCase()) {
+                const text = el.textContent?.trim();
+                if (text)
+                    return text;
+            }
+        }
+        const byId = doc.querySelector(`#${cssEscape(name)}`);
+        if (byId) {
+            const text = byId.textContent?.trim();
+            if (text)
+                return text;
+        }
+        const byData = doc.querySelector(`[data-${name}]`);
+        if (byData) {
+            return byData.getAttribute(`data-${name}`) ?? byData.textContent?.trim() ?? undefined;
+        }
+    }
+    return undefined;
+}
+function findArrayValues(doc, variants) {
+    for (const name of variants) {
+        const container = doc.querySelector(`[class*="${name}"]`);
+        if (container) {
+            const items = container.querySelectorAll('li, [class*="item"]');
+            if (items.length > 0) {
+                return Array.from(items).map((el) => (el.textContent ?? '').trim()).filter(Boolean);
+            }
+        }
+        const singular = name.replace(/s$/, '');
+        const elements = doc.querySelectorAll(`[class*="${singular}"]`);
+        if (elements.length > 1) {
+            return Array.from(elements).map((el) => (el.textContent ?? '').trim()).filter(Boolean);
+        }
+    }
+    return undefined;
+}
+//# sourceMappingURL=schema.js.map

package/dist/extraction/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/extraction/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAQjE,MAAM,UAAU,iBAAiB,CAC/B,IAAY,EACZ,MAAkB;IAElB,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU;QAAE,OAAO,EAAE,CAAC;IAE3C,MAAM,YAAY,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,YAAY,GAAG,mBAAmB,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAE/D,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,eAAe,GAA4B,EAAE,CAAC;IAEpD,KAAK,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QACzE,IAAI,YAAY,CAAC,SAAS,CAAC,KAAK,SAAS;YAAE,SAAS;QAEpD,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC;QAC1D,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YACxB,eAAe,CAAC,SAAS,CAAC,GAAG,KAAK,CAAC;QACrC,CAAC;IACH,CAAC;IAED,OAAO,EAAE,GAAG,YAAY,EAAE,GAAG,eAAe,EAAE,CAAC;AACjD,CAAC;AAED,SAAS,cAAc,CACrB,GAAa,EACb,SAAiB,EACjB,MAAkB;IAElB,MAAM,cAAc,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAClE,MAAM,WAAW,GAAG,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC9D,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,cAAc,EAAE,WAAW,CAAC,CAAC;IAE1D,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,OAAO,eAAe,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IACxC,CAAC;IAED,OAAO,eAAe,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;AACxC,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC9B,OAAO,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,eAAe,CAAC,GAAa,EAAE,QAAkB;IACxD,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,UAAU,GAAG,GAAG,CAAC,aAAa,CAAC,cAAc,IAAI,IAAI,CAAC,CAAC;QAC7D,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,UAAU,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,UAAU,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;YAClF,IAAI,IAAI;gBAAE,OAAO,IAAI,CAAC;QACxB,CAAC;QAED,iFAAiF;QACjF,MAAM,OAAO,GAAG,GAAG,CAAC,aAAa,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC;QACxD,IAAI,OAAO,EAAE,CAAC;YACZ,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;YACzC,IAAI,IAAI;gBAAE,OAAO,IAAI,CAAC;QACxB,CAAC;QAED,MAAM,WAAW,GAAG,GAAG,CAAC,gBAAgB,CAAC,cAAc,CAAC,CAAC;QACzD,KAAK,MAAM,EAAE,IAAI,WAAW,EAAE,CAAC;YAC7B,MAAM,KAAK,GAAG,EAAE,CAAC,YAAY,CAAC,YAAY,CAAC,EAAE,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC;YACtF,IAAI,KAAK,KAAK,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;gBACjC,MAAM,IAAI,GAAG,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;gBACpC,IAAI,IAAI;oBAAE,OAAO,IAAI,CAAC;YACxB,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,GAAG,CAAC,aAAa,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtD,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC;YACtC,IAAI,IAAI;gBAAE,OAAO,IAAI,CAAC;QACxB,CAAC;QAED,MAAM,MAAM,GAAG,GAAG,CAAC,aAAa,CAAC,SAAS,IAAI,GAAG,CAAC,CAAC;QACnD,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,MAAM,CAAC,YAAY,CAAC,QAAQ,IAAI,EAAE,CAAC,IAAI,MAAM,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,SAAS,CAAC;QACxF,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,eAAe,CAAC,GAAa,EAAE,QAAkB;IACxD,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,SAAS,GAAG,GAAG,CAAC,aAAa,CAAC,YAAY,IAAI,IAAI,CAAC,CAAC;QAC1D,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,KAAK,GAAG,SAAS,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,CAAC;YAChE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrB,OAAO,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACtF,CAAC;QACH,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACxC,MAAM,QAAQ,GAAG,GAAG,CAAC,gBAAgB,CAAC,YAAY,QAAQ,IAAI,CAAC,CAAC;QAChE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,OAAO,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzF,CAAC;IACH,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC"}

package/dist/extraction/site-extractors/docs-generic.d.ts

@@ -0,0 +1,3 @@
+import type { Extractor } from '../../types.js';
+export declare const docsGenericExtractor: Extractor;
+//# sourceMappingURL=docs-generic.d.ts.map

package/dist/extraction/site-extractors/docs-generic.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"docs-generic.d.ts","sourceRoot":"","sources":["../../../src/extraction/site-extractors/docs-generic.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAoB,MAAM,gBAAgB,CAAC;AAqFlE,eAAO,MAAM,oBAAoB,EAAE,SAqClC,CAAC"}

package/dist/extraction/site-extractors/docs-generic.js

@@ -0,0 +1,104 @@
+import { parseHTML } from 'linkedom';
+import TurndownService from 'turndown';
+const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+const STRIP_SELECTORS = [
+    'nav',
+    '.docs-sidebar',
+    '.sidebar',
+    '.toc-wrapper',
+    '.table-of-contents',
+    '.version-picker',
+    '.pagination-nav',
+    'header',
+    'footer',
+];
+function detectFramework(html) {
+    if (html.includes('docs-sidebar') || html.includes('data-docusaurus-page')) {
+        return 'docusaurus';
+    }
+    if (html.includes('md-content')) {
+        return 'mkdocs';
+    }
+    if (html.includes('class="document"') || html.includes("class='document'") ||
+        (html.includes('class="body"') && html.includes('highlight'))) {
+        return 'sphinx';
+    }
+    if (html.includes('page-body')) {
+        return 'gitbook';
+    }
+    return null;
+}
+function extractWithSelectors(document, contentSelectors) {
+    for (const selector of contentSelectors) {
+        const el = document.querySelector(selector);
+        if (el)
+            return el;
+    }
+    return null;
+}
+function stripElements(root, selectors) {
+    for (const selector of selectors) {
+        for (const el of Array.from(root.querySelectorAll(selector))) {
+            el.parentNode?.removeChild(el);
+        }
+    }
+}
+function buildResult(document, contentEl) {
+    stripElements(contentEl, STRIP_SELECTORS);
+    const titleEl = contentEl.querySelector('h1') ??
+        document.querySelector('h1') ??
+        document.querySelector('title');
+    const rawTitle = titleEl?.textContent?.trim() ?? '';
+    const title = rawTitle.includes('|')
+        ? rawTitle.split('|')[0].trim()
+        : rawTitle;
+    if (!title)
+        return null;
+    const markdown = turndown.turndown(contentEl.innerHTML).trim();
+    if (!markdown)
+        return null;
+    return {
+        title,
+        markdown,
+        metadata: {},
+        links: [],
+        images: [],
+        extractor: 'site-specific',
+    };
+}
+export const docsGenericExtractor = {
+    name: 'docs-generic',
+    canHandle(_url, html) {
+        if (!html)
+            return false;
+        return detectFramework(html) !== null;
+    },
+    extract(html, _url) {
+        if (!html)
+            return null;
+        const framework = detectFramework(html);
+        if (!framework)
+            return null;
+        const { document } = parseHTML(html);
+        let contentSelectors;
+        switch (framework) {
+            case 'docusaurus':
+                contentSelectors = ['.markdown', 'article', 'main'];
+                break;
+            case 'mkdocs':
+                contentSelectors = ['.md-content'];
+                break;
+            case 'sphinx':
+                contentSelectors = ['.document', '.body'];
+                break;
+            case 'gitbook':
+                contentSelectors = ['.page-body'];
+                break;
+        }
+        const contentEl = extractWithSelectors(document, contentSelectors);
+        if (!contentEl)
+            return null;
+        return buildResult(document, contentEl);
+    },
+};
+//# sourceMappingURL=docs-generic.js.map

package/dist/extraction/site-extractors/docs-generic.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"docs-generic.js","sourceRoot":"","sources":["../../../src/extraction/site-extractors/docs-generic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,eAAe,MAAM,UAAU,CAAC;AAGvC,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;AAIxF,MAAM,eAAe,GAAG;IACtB,KAAK;IACL,eAAe;IACf,UAAU;IACV,cAAc;IACd,oBAAoB;IACpB,iBAAiB;IACjB,iBAAiB;IACjB,QAAQ;IACR,QAAQ;CACT,CAAC;AAEF,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE,CAAC;QAC3E,OAAO,YAAY,CAAC;IACtB,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;QAChC,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC;QACtE,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC;QAClE,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAC/B,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,oBAAoB,CAC3B,QAAkB,EAClB,gBAA0B;IAE1B,KAAK,MAAM,QAAQ,IAAI,gBAAgB,EAAE,CAAC;QACxC,MAAM,EAAE,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QAC5C,IAAI,EAAE;YAAE,OAAO,EAAa,CAAC;IAC/B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CAAC,IAAa,EAAE,SAAmB;IACvD,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YAC7D,EAAE,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,WAAW,CAClB,QAAkB,EAClB,SAAkB;IAElB,aAAa,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IAE1C,MAAM,OAAO,GACX,SAAS,CAAC,aAAa,CAAC,IAAI,CAAC;QAC7B,QAAQ,CAAC,aAAa,CAAC,IAAI,CAAC;QAC5B,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAElC,MAAM,QAAQ,GAAG,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC;QAClC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE;QAChC,CAAC,CAAC,QAAQ,CAAC;IAEb,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IAExB,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE3B,OAAO;QACL,KAAK;QACL,QAAQ;QACR,QAAQ,EAAE,EAAE;QACZ,KAAK,EAAE,EAAE;QACT,MAAM,EAAE,EAAE;QACV,SAAS,EAAE,eAAe;KAC3B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAc;IAC7C,IAAI,EAAE,cAAc;IAEpB,SAAS,CAAC,IAAY,EAAE,IAAa;QACnC,IAAI,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QACxB,OAAO,eAAe,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC;IACxC,CAAC;IAED,OAAO,CAAC,IAAY,EAAE,IAAY;QAChC,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAE5B,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAErC,IAAI,gBAA0B,CAAC;QAC/B,QAAQ,SAAS,EAAE,CAAC;YAClB,KAAK,YAAY;gBACf,gBAAgB,GAAG,CAAC,WAAW,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;gBACpD,MAAM;YACR,KAAK,QAAQ;gBACX,gBAAgB,GAAG,CAAC,aAAa,CAAC,CAAC;gBACnC,MAAM;YACR,KAAK,QAAQ;gBACX,gBAAgB,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;gBAC1C,MAAM;YACR,KAAK,SAAS;gBACZ,gBAAgB,GAAG,CAAC,YAAY,CAAC,CAAC;gBAClC,MAAM;QACV,CAAC;QAED,MAAM,SAAS,GAAG,oBAAoB,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;QACnE,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QAE5B,OAAO,WAAW,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAC1C,CAAC;CACF,CAAC"}

package/dist/extraction/site-extractors/github.d.ts

@@ -0,0 +1,3 @@
+import type { Extractor } from '../../types.js';
+export declare const githubExtractor: Extractor;
+//# sourceMappingURL=github.d.ts.map

package/dist/extraction/site-extractors/github.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.d.ts","sourceRoot":"","sources":["../../../src/extraction/site-extractors/github.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAoB,MAAM,gBAAgB,CAAC;AAoGlE,eAAO,MAAM,eAAe,EAAE,SA2B7B,CAAC"}

package/dist/extraction/site-extractors/github.js

@@ -0,0 +1,107 @@
+import { parseHTML } from 'linkedom';
+import TurndownService from 'turndown';
+const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+function isIssueOrPR(url) {
+    return /\/issues\/\d+|\/pull\/\d+/.test(url);
+}
+function isBlob(url) {
+    return /\/blob\//.test(url);
+}
+function extractIssue(document, url) {
+    const titleEl = document.querySelector('.js-issue-title') ?? document.querySelector('.gh-header-title');
+    if (!titleEl)
+        return null;
+    const title = titleEl.textContent?.trim() ?? '';
+    const labelEls = document.querySelectorAll('.IssueLabel');
+    const labels = Array.from(labelEls)
+        .map((el) => el.textContent?.trim() ?? '')
+        .filter(Boolean);
+    const commentBodies = document.querySelectorAll('.d-block.comment-body');
+    if (commentBodies.length === 0)
+        return null;
+    const sections = [];
+    if (labels.length > 0) {
+        sections.push(`**Labels:** ${labels.join(', ')}\n`);
+    }
+    Array.from(commentBodies).forEach((body, i) => {
+        const html = body.innerHTML;
+        const md = turndown.turndown(html).trim();
+        if (md) {
+            sections.push(i === 0 ? md : `---\n\n${md}`);
+        }
+    });
+    const markdown = sections.join('\n\n');
+    return {
+        title,
+        markdown,
+        metadata: {},
+        links: [],
+        images: [],
+        extractor: 'site-specific',
+    };
+}
+function extractReadme(document) {
+    const titleEl = document.querySelector('title');
+    const rawTitle = titleEl?.textContent?.trim() ?? '';
+    const title = rawTitle.split(':')[0]?.trim() ?? rawTitle;
+    const readmeBody = document.querySelector('#readme .markdown-body') ??
+        document.querySelector('.markdown-body');
+    if (!readmeBody)
+        return null;
+    const markdown = turndown.turndown(readmeBody.innerHTML).trim();
+    if (!markdown)
+        return null;
+    return {
+        title,
+        markdown,
+        metadata: {},
+        links: [],
+        images: [],
+        extractor: 'site-specific',
+    };
+}
+function extractBlob(document) {
+    const titleEl = document.querySelector('title');
+    const title = titleEl?.textContent?.trim() ?? '';
+    const codeBlock = document.querySelector('.blob-code-content') ??
+        document.querySelector('.highlight') ??
+        document.querySelector('.markdown-body');
+    if (!codeBlock)
+        return null;
+    const markdown = turndown.turndown(codeBlock.innerHTML).trim();
+    if (!markdown)
+        return null;
+    return {
+        title,
+        markdown,
+        metadata: {},
+        links: [],
+        images: [],
+        extractor: 'site-specific',
+    };
+}
+export const githubExtractor = {
+    name: 'github',
+    canHandle(url) {
+        try {
+            const hostname = new URL(url).hostname;
+            return hostname === 'github.com' || hostname.endsWith('.github.com');
+        }
+        catch {
+            return false;
+        }
+    },
+    extract(html, url) {
+        if (!html)
+            return null;
+        const { document } = parseHTML(html);
+        if (isIssueOrPR(url)) {
+            return extractIssue(document, url);
+        }
+        if (isBlob(url)) {
+            return extractBlob(document);
+        }
+        return extractReadme(document);
+    },
+};
+//# sourceMappingURL=github.js.map

package/dist/extraction/site-extractors/github.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"github.js","sourceRoot":"","sources":["../../../src/extraction/site-extractors/github.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,eAAe,MAAM,UAAU,CAAC;AAGvC,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;AAExF,SAAS,WAAW,CAAC,GAAW;IAC9B,OAAO,2BAA2B,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,MAAM,CAAC,GAAW;IACzB,OAAO,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,YAAY,CAAC,QAAkB,EAAE,GAAW;IACnD,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,iBAAiB,CAAC,IAAI,QAAQ,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC;IACxG,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAE1B,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAEhD,MAAM,QAAQ,GAAG,QAAQ,CAAC,gBAAgB,CAAC,aAAa,CAAC,CAAC;IAC1D,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC;SAChC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;SACzC,MAAM,CAAC,OAAO,CAAC,CAAC;IAEnB,MAAM,aAAa,GAAG,QAAQ,CAAC,gBAAgB,CAAC,uBAAuB,CAAC,CAAC;IACzE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IAE5C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAE9B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,QAAQ,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QAC5C,MAAM,IAAI,GAAI,IAAgB,CAAC,SAAS,CAAC;QACzC,MAAM,EAAE,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1C,IAAI,EAAE,EAAE,CAAC;YACP,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEvC,OAAO;QACL,KAAK;QACL,QAAQ;QACR,QAAQ,EAAE,EAAE;QACZ,KAAK,EAAE,EAAE;QACT,MAAM,EAAE,EAAE;QACV,SAAS,EAAE,eAAe;KAC3B,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,QAAkB;IACvC,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,QAAQ,CAAC;IAEzD,MAAM,UAAU,GACd,QAAQ,CAAC,aAAa,CAAC,wBAAwB,CAAC;QAChD,QAAQ,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC;IAE3C,IAAI,CAAC,UAAU;QAAE,OAAO,IAAI,CAAC;IAE7B,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAE,UAAsB,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7E,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE3B,OAAO;QACL,KAAK;QACL,QAAQ;QACR,QAAQ,EAAE,EAAE;QACZ,KAAK,EAAE,EAAE;QACT,MAAM,EAAE,EAAE;QACV,SAAS,EAAE,eAAe;KAC3B,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,QAAkB;IACrC,MAAM,OAAO,GAAG,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAEjD,MAAM,SAAS,GACb,QAAQ,CAAC,aAAa,CAAC,oBAAoB,CAAC;QAC5C,QAAQ,CAAC,aAAa,CAAC,YAAY,CAAC;QACpC,QAAQ,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC;IAE3C,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IAE5B,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAE,SAAqB,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5E,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAE3B,OAAO;QACL,KAAK;QACL,QAAQ;QACR,QAAQ,EAAE,EAAE;QACZ,KAAK,EAAE,EAAE;QACT,MAAM,EAAE,EAAE;QACV,SAAS,EAAE,eAAe;KAC3B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,eAAe,GAAc;IACxC,IAAI,EAAE,QAAQ;IAEd,SAAS,CAAC,GAAW;QACnB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACvC,OAAO,QAAQ,KAAK,YAAY,IAAI,QAAQ,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC;QACvE,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAY,EAAE,GAAW;QAC/B,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAErC,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YACrB,OAAO,YAAY,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;QACrC,CAAC;QAED,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YAChB,OAAO,WAAW,CAAC,QAAQ,CAAC,CAAC;QAC/B,CAAC;QAED,OAAO,aAAa,CAAC,QAAQ,CAAC,CAAC;IACjC,CAAC;CACF,CAAC"}

package/dist/extraction/site-extractors/mdn.d.ts

@@ -0,0 +1,3 @@
+import type { Extractor } from '../../types.js';
+export declare const mdnExtractor: Extractor;
+//# sourceMappingURL=mdn.d.ts.map

package/dist/extraction/site-extractors/mdn.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mdn.d.ts","sourceRoot":"","sources":["../../../src/extraction/site-extractors/mdn.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAoB,MAAM,gBAAgB,CAAC;AAalE,eAAO,MAAM,YAAY,EAAE,SAqD1B,CAAC"}

package/dist/extraction/site-extractors/mdn.js

@@ -0,0 +1,58 @@
+import { parseHTML } from 'linkedom';
+import TurndownService from 'turndown';
+const turndown = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' });
+const STRIP_SELECTORS = [
+    'nav',
+    '.sidebar',
+    'header',
+    'footer',
+    '.bc-head',
+    '.metadata',
+];
+export const mdnExtractor = {
+    name: 'mdn',
+    canHandle(url) {
+        try {
+            const hostname = new URL(url).hostname;
+            return hostname === 'developer.mozilla.org';
+        }
+        catch {
+            return false;
+        }
+    },
+    extract(html, url) {
+        if (!html)
+            return null;
+        const { document } = parseHTML(html);
+        const article = document.querySelector('article.main-page-content') ??
+            document.querySelector('.section-content') ??
+            document.querySelector('article');
+        if (!article)
+            return null;
+        for (const selector of STRIP_SELECTORS) {
+            for (const el of Array.from(article.querySelectorAll(selector))) {
+                el.parentNode?.removeChild(el);
+            }
+        }
+        const titleEl = article.querySelector('h1') ??
+            document.querySelector('title');
+        const rawTitle = titleEl?.textContent?.trim() ?? '';
+        const title = rawTitle.includes('|')
+            ? rawTitle.split('|')[0].trim()
+            : rawTitle;
+        if (!title)
+            return null;
+        const markdown = turndown.turndown(article.innerHTML).trim();
+        if (!markdown)
+            return null;
+        return {
+            title,
+            markdown,
+            metadata: {},
+            links: [],
+            images: [],
+            extractor: 'site-specific',
+        };
+    },
+};
+//# sourceMappingURL=mdn.js.map

package/dist/extraction/site-extractors/mdn.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mdn.js","sourceRoot":"","sources":["../../../src/extraction/site-extractors/mdn.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AACrC,OAAO,eAAe,MAAM,UAAU,CAAC;AAGvC,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC,CAAC;AAExF,MAAM,eAAe,GAAG;IACtB,KAAK;IACL,UAAU;IACV,QAAQ;IACR,QAAQ;IACR,UAAU;IACV,WAAW;CACZ,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAc;IACrC,IAAI,EAAE,KAAK;IAEX,SAAS,CAAC,GAAW;QACnB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;YACvC,OAAO,QAAQ,KAAK,uBAAuB,CAAC;QAC9C,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAY,EAAE,GAAW;QAC/B,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QAEvB,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;QAErC,MAAM,OAAO,GACX,QAAQ,CAAC,aAAa,CAAC,2BAA2B,CAAC;YACnD,QAAQ,CAAC,aAAa,CAAC,kBAAkB,CAAC;YAC1C,QAAQ,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAEpC,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC;QAE1B,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE,CAAC;YACvC,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;gBAChE,EAAE,CAAC,UAAU,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC;YACjC,CAAC;QACH,CAAC;QAED,MAAM,OAAO,GACX,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC;YAC3B,QAAQ,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;QAElC,MAAM,QAAQ,GAAG,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;QACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC;YAClC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE;YAChC,CAAC,CAAC,QAAQ,CAAC;QAEb,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QAExB,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAE,OAAmB,CAAC,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1E,IAAI,CAAC,QAAQ;YAAE,OAAO,IAAI,CAAC;QAE3B,OAAO;YACL,KAAK;YACL,QAAQ;YACR,QAAQ,EAAE,EAAE;YACZ,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,SAAS,EAAE,eAAe;SAC3B,CAAC;IACJ,CAAC;CACF,CAAC"}

package/dist/extraction/site-extractors/stackoverflow.d.ts

@@ -0,0 +1,3 @@
+import type { Extractor } from '../../types.js';
+export declare const stackoverflowExtractor: Extractor;
+//# sourceMappingURL=stackoverflow.d.ts.map

package/dist/extraction/site-extractors/stackoverflow.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stackoverflow.d.ts","sourceRoot":"","sources":["../../../src/extraction/site-extractors/stackoverflow.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAoB,MAAM,gBAAgB,CAAC;AAgElE,eAAO,MAAM,sBAAsB,EAAE,SAkDpC,CAAC"}