docorbit 0.1.0 → 0.1.1

package/dist/packages/mcp/src/types.d.ts

@@ -0,0 +1,89 @@
+export interface JsonRpcRequest {
+    jsonrpc: '2.0';
+    id?: string | number | null;
+    method: string;
+    params?: Record<string, unknown>;
+}
+export interface JsonRpcResponse {
+    jsonrpc: '2.0';
+    id: string | number | null;
+    result?: unknown;
+    error?: JsonRpcErrorObject;
+}
+export interface JsonRpcNotification {
+    jsonrpc: '2.0';
+    method: string;
+    params?: Record<string, unknown>;
+}
+export interface JsonRpcErrorObject {
+    code: number;
+    message: string;
+    data?: unknown;
+}
+export declare const JSONRPC_ERRORS: {
+    readonly PARSE_ERROR: -32700;
+    readonly INVALID_REQUEST: -32600;
+    readonly METHOD_NOT_FOUND: -32601;
+    readonly INVALID_PARAMS: -32602;
+    readonly INTERNAL_ERROR: -32603;
+};
+export interface ToolInputSchema {
+    type: 'object';
+    properties: Record<string, {
+        type: string;
+        description: string;
+        items?: Record<string, unknown>;
+        enum?: string[];
+        default?: unknown;
+    }>;
+    required?: string[];
+}
+export interface McpTool {
+    name: string;
+    description: string;
+    inputSchema: ToolInputSchema;
+}
+export interface ToolContentItem {
+    type: 'text';
+    text: string;
+}
+export interface CallToolResult {
+    content: ToolContentItem[];
+    isError?: boolean;
+}
+export interface McpResource {
+    uri: string;
+    name: string;
+    description?: string;
+    mimeType?: string;
+}
+export interface ResourceContent {
+    uri: string;
+    mimeType?: string;
+    text?: string;
+    blob?: string;
+}
+export interface ReadResourceResult {
+    contents: ResourceContent[];
+}
+export interface ServerCapabilities {
+    tools?: {
+        listChanged?: boolean;
+    };
+    resources?: {
+        subscribe?: boolean;
+        listChanged?: boolean;
+    };
+    prompts?: {
+        listChanged?: boolean;
+    };
+    logging?: Record<string, unknown>;
+}
+export interface InitializeResult {
+    protocolVersion: string;
+    serverInfo: {
+        name: string;
+        version: string;
+    };
+    capabilities: ServerCapabilities;
+}

package/dist/packages/mcp/src/types.js

@@ -0,0 +1,7 @@
+export const JSONRPC_ERRORS = {
+    PARSE_ERROR: -32700,
+    INVALID_REQUEST: -32600,
+    METHOD_NOT_FOUND: -32601,
+    INVALID_PARAMS: -32602,
+    INTERNAL_ERROR: -32603,
+};

package/dist/packages/normalizer/src/example-indexer.d.ts

@@ -0,0 +1,5 @@
+import type { NormalizedPage, DocumentChunk, IndexedExample, SourceAuthority } from '../../shared/src/index.ts';
+/**
+ * Extracts and indexes first-class code examples from normalized pages and chunks.
+ */
+export declare function extractIndexedExamples(page: NormalizedPage, chunks: DocumentChunk[], authority: SourceAuthority, docVersion?: string, snapshotId?: string): IndexedExample[];

package/dist/packages/normalizer/src/example-indexer.js

@@ -0,0 +1,200 @@
+import { createHash } from 'node:crypto';
+const FRAMEWORK_PATTERNS = [
+    { framework: 'next', pattern: /(?:from\s+['"]next(?:\/.*)?['"]|require\(['"]next(?:\/.*)?['"]\)|'use client'|'use server')/i },
+    { framework: 'react', pattern: /(?:from\s+['"]react['"]|require\(['"]react['"]\)|useState|useEffect|useMemo|useCallback)/ },
+    { framework: 'express', pattern: /(?:from\s+['"]express['"]|require\(['"]express['"]\)|express\(\))/i },
+    { framework: 'fastify', pattern: /(?:from\s+['"]fastify['"]|require\(['"]fastify['"]\))/i },
+    { framework: 'hono', pattern: /(?:from\s+['"]hono['"]|require\(['"]hono['"]\))/i },
+    { framework: 'vue', pattern: /(?:from\s+['"]vue['"]|createApp\(|defineComponent)/i },
+    { framework: 'svelte', pattern: /(?:from\s+['"]svelte['"]|<script.*lang=["']ts["']>)/i },
+    { framework: 'fastapi', pattern: /(?:from\s+fastapi\s+import|FastAPI\()/i },
+    { framework: 'flask', pattern: /(?:from\s+flask\s+import|Flask\(__name__\))/i },
+    { framework: 'django', pattern: /(?:from\s+django|django\.)/i },
+    { framework: 'spring', pattern: /(?:@SpringBootApplication|@RestController|@GetMapping|@PostMapping)/i },
+    { framework: 'gin', pattern: /(?:github\.com\/gin-gonic\/gin|gin\.Default\(\))/i },
+];
+const API_METHOD_PATH_REGEX = /\b(GET|POST|PUT|DELETE|PATCH)\s+([/\w\-_{}]+)/i;
+/**
+ * Detects framework from code snippet content.
+ */
+function detectFramework(code) {
+    for (const { framework, pattern } of FRAMEWORK_PATTERNS) {
+        if (pattern.test(code)) {
+            return framework;
+        }
+    }
+    return undefined;
+}
+/**
+ * Detects an API endpoint reference from code or task context.
+ */
+function detectRelatedApi(code, context) {
+    // Check context first (e.g. "POST /v1/webhook_endpoints")
+    const contextMatch = context.match(API_METHOD_PATH_REGEX);
+    if (contextMatch) {
+        return `${contextMatch[1].toUpperCase()} ${contextMatch[2]}`;
+    }
+    // Check code
+    const codeMatch = code.match(API_METHOD_PATH_REGEX);
+    if (codeMatch) {
+        return `${codeMatch[1].toUpperCase()} ${codeMatch[2]}`;
+    }
+    // Common client library calls (e.g. fetch('/api/v1/foo', { method: 'POST' }))
+    const fetchMatch = code.match(/fetch\(\s*['"`]([/\w\-_{}]+)['"`](?:,\s*\{[^}]*method:\s*['"](\w+)['"])?/i);
+    if (fetchMatch) {
+        const method = (fetchMatch[2] || 'GET').toUpperCase();
+        return `${method} ${fetchMatch[1]}`;
+    }
+    // Framework router calls: app.post('/api/webhook', ...) or router.get('/users', ...) or @app.post('/webhook')
+    const routeMatch = code.match(/(?:app|router)\.(get|post|put|delete|patch)\(\s*['"`]([/\w\-_{}]+)['"`]/i);
+    if (routeMatch) {
+        return `${routeMatch[1].toUpperCase()} ${routeMatch[2]}`;
+    }
+    // Python decorators: @app.post("/webhook")
+    const pyRouteMatch = code.match(/@(?:app|router)\.(get|post|put|delete|patch)\(\s*['"`]([/\w\-_{}]+)['"`]/i);
+    if (pyRouteMatch) {
+        return `${pyRouteMatch[1].toUpperCase()} ${pyRouteMatch[2]}`;
+    }
+    return undefined;
+}
+/**
+ * Detects a primary symbol or function call from code.
+ */
+function detectRelatedSymbol(code) {
+    // Check function declarations
+    const funcMatch = code.match(/(?:function\s+([a-zA-Z0-9_$]+)|const\s+([a-zA-Z0-9_$]+)\s*=\s*(?:async\s*)?\()/);
+    if (funcMatch) {
+        return funcMatch[1] || funcMatch[2];
+    }
+    // Check class declarations
+    const classMatch = code.match(/class\s+([a-zA-Z0-9_$]+)/);
+    if (classMatch) {
+        return classMatch[1];
+    }
+    // Check prominent method invocations, ignoring routing boilerplate like app.post or res.json
+    const boilerplateCalls = new Set([
+        'app.get', 'app.post', 'app.put', 'app.delete', 'app.use',
+        'router.get', 'router.post', 'router.use',
+        'express.raw', 'express.json', 'express.urlencoded',
+        'res.json', 'res.send', 'res.status', 'console.log', 'console.error',
+    ]);
+    const allCalls = Array.from(code.matchAll(/([a-zA-Z0-9_$]+(?:\.[a-zA-Z0-9_$]+)+)\s*\(/g));
+    for (const match of allCalls) {
+        if (!boilerplateCalls.has(match[1])) {
+            return match[1];
+        }
+    }
+    if (allCalls.length > 0) {
+        return allCalls[0][1];
+    }
+    return undefined;
+}
+/**
+ * Extracts and indexes first-class code examples from normalized pages and chunks.
+ */
+export function extractIndexedExamples(page, chunks, authority, docVersion, snapshotId) {
+    const examples = [];
+    const seenCodeHashes = new Set();
+    const nowIso = new Date().toISOString();
+    // Map each code example in the page to its enclosing chunk
+    for (const codeExample of page.codeExamples) {
+        const rawCode = codeExample.code.trim();
+        if (!rawCode || rawCode.length < 15)
+            continue; // Skip trivial snippets
+        const codeHash = createHash('sha256').update(rawCode).digest('hex');
+        if (seenCodeHashes.has(codeHash))
+            continue;
+        seenCodeHashes.add(codeHash);
+        // Find the chunk that contains this code snippet
+        const matchingChunk = chunks.find(c => c.chunkType === 'code' && c.content.includes(rawCode)) || chunks.find(c => c.content.includes(rawCode));
+        // Determine task context from breadcrumbs or headings
+        let task = '';
+        if (matchingChunk && matchingChunk.sectionPath.length > 0) {
+            task = matchingChunk.sectionPath.join(' > ');
+        }
+        else if (matchingChunk && matchingChunk.title) {
+            task = matchingChunk.title;
+        }
+        else {
+            task = page.title;
+        }
+        const language = (codeExample.language || (matchingChunk?.language) || 'typescript').toLowerCase();
+        const framework = detectFramework(rawCode);
+        const relatedApi = detectRelatedApi(rawCode, task);
+        const relatedSymbol = detectRelatedSymbol(rawCode);
+        const exampleId = createHash('sha256')
+            .update(`${page.id}:${codeHash}`)
+            .digest('hex')
+            .substring(0, 16);
+        const provenance = {
+            sourceUrl: page.url,
+            retrievedAt: page.metadata?.extractedAt || nowIso,
+            sourceAuthority: authority,
+        };
+        const effectiveSnapshotId = snapshotId || matchingChunk?.snapshotId || page.snapshotId || 'snap_default';
+        examples.push({
+            id: exampleId,
+            chunkId: matchingChunk?.id,
+            pageId: page.id,
+            snapshotId: effectiveSnapshotId,
+            language,
+            framework,
+            task,
+            code: rawCode,
+            sourceUrl: page.url,
+            sourceAuthority: authority,
+            relatedApi,
+            relatedSymbol,
+            docVersion: docVersion || page.docVersion,
+            provenance,
+            createdAt: nowIso,
+        });
+    }
+    // Also check chunks of type 'code' that might not have been caught in page.codeExamples
+    for (const chunk of chunks) {
+        if (chunk.chunkType !== 'code')
+            continue;
+        const rawCode = chunk.content.trim();
+        if (!rawCode || rawCode.length < 15)
+            continue;
+        const codeHash = createHash('sha256').update(rawCode).digest('hex');
+        if (seenCodeHashes.has(codeHash))
+            continue;
+        seenCodeHashes.add(codeHash);
+        const task = chunk.sectionPath.length > 0
+            ? chunk.sectionPath.join(' > ')
+            : (chunk.title || page.title);
+        const language = (chunk.language || 'typescript').toLowerCase();
+        const framework = detectFramework(rawCode);
+        const relatedApi = detectRelatedApi(rawCode, task);
+        const relatedSymbol = detectRelatedSymbol(rawCode);
+        const exampleId = createHash('sha256')
+            .update(`${page.id}:${codeHash}`)
+            .digest('hex')
+            .substring(0, 16);
+        const provenance = {
+            sourceUrl: page.url,
+            retrievedAt: page.metadata?.extractedAt || nowIso,
+            sourceAuthority: authority,
+        };
+        const effectiveSnapshotId = snapshotId || chunk.snapshotId || page.snapshotId || 'snap_default';
+        examples.push({
+            id: exampleId,
+            chunkId: chunk.id,
+            pageId: page.id,
+            snapshotId: effectiveSnapshotId,
+            language,
+            framework,
+            task,
+            code: rawCode,
+            sourceUrl: page.url,
+            sourceAuthority: authority,
+            relatedApi,
+            relatedSymbol,
+            docVersion: docVersion || chunk.docVersion || page.docVersion,
+            provenance,
+            createdAt: nowIso,
+        });
+    }
+    return examples;
+}

package/dist/packages/normalizer/src/html.d.ts

@@ -0,0 +1,13 @@
+import type { Heading, Link, CodeExample } from '../../shared/src/index.ts';
+export interface ExtractedHtmlDoc {
+    title: string;
+    markdown: string;
+    headings: Heading[];
+    links: Link[];
+    codeExamples: CodeExample[];
+}
+/**
+ * Deterministically normalizes HTML documentation into clean Markdown,
+ * preserving code fences, language tags, headings, tables, callouts, and links.
+ */
+export declare function normalizeHtmlToMarkdown(html: string, baseUrl: string): ExtractedHtmlDoc;

package/dist/packages/normalizer/src/html.js

@@ -0,0 +1,221 @@
+function unescapeHtml(text) {
+    return text
+        .replace(/&/g, '&')
+        .replace(/&lt;/g, '<')
+        .replace(/&gt;/g, '>')
+        .replace(/&quot;/g, '"')
+        .replace(/&#39;/g, "'")
+        .replace(/&apos;/g, "'")
+        .replace(/&nbsp;/g, ' ')
+        .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)))
+        .replace(/&#x([0-9a-fA-F]+);/g, (_, code) => String.fromCharCode(parseInt(code, 16)));
+}
+function slugify(text) {
+    return text
+        .toLowerCase()
+        .trim()
+        .replace(/[^\w\s-]/g, '')
+        .replace(/[\s_-]+/g, '-')
+        .replace(/^-+|-+$/g, '');
+}
+/**
+ * Deterministically normalizes HTML documentation into clean Markdown,
+ * preserving code fences, language tags, headings, tables, callouts, and links.
+ */
+export function normalizeHtmlToMarkdown(html, baseUrl) {
+    const headings = [];
+    const links = [];
+    const codeExamples = [];
+    let title = '';
+    // 1. Extract title from <title> tag if present
+    const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+    if (titleMatch) {
+        title = unescapeHtml(titleMatch[1]).replace(/\s+/g, ' ').trim();
+        title = title.split(/\s+[|–—-]\s+/)[0].trim();
+    }
+    // 2. Remove non-content / boilerplate tags
+    let cleaned = html
+        .replace(/<!--[\s\S]*?-->/g, '')
+        .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, '')
+        .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, '')
+        .replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, '')
+        .replace(/<svg\b[^>]*>[\s\S]*?<\/svg>/gi, '')
+        .replace(/<iframe\b[^>]*>[\s\S]*?<\/iframe>/gi, '')
+        .replace(/<nav\b[^>]*>[\s\S]*?<\/nav>/gi, '')
+        .replace(/<header\b[^>]*>[\s\S]*?<\/header>/gi, '')
+        .replace(/<footer\b[^>]*>[\s\S]*?<\/footer>/gi, '')
+        .replace(/<div\b[^>]*(?:class|id)=["'][^"']*\b(?:sidebar|nav|menu|navbar|footer|cookie-banner|breadcrumbs)\b[^"']*["'][^>]*>[\s\S]*?<\/div>/gi, '');
+    // 3. Extract and preserve code blocks
+    const codeBlockPlaceholders = [];
+    cleaned = cleaned.replace(/<pre\b[^>]*>(?:<code\b([^>]*)>)?([\s\S]*?)(?:<\/code>)?<\/pre>/gi, (_, codeAttrs, codeBody) => {
+        let lang = '';
+        if (codeAttrs) {
+            const langMatch = codeAttrs.match(/\b(?:class|data-language)=["'][^"']*\b(?:language-|lang-)?([a-zA-Z0-9_-]+)\b[^"']*["']/i);
+            if (langMatch && !['hljs', 'code'].includes(langMatch[1].toLowerCase())) {
+                lang = langMatch[1].toLowerCase();
+            }
+        }
+        const rawCode = unescapeHtml(codeBody.replace(/<[^>]+>/g, '')).trim();
+        if (!rawCode)
+            return '';
+        const exampleId = `code_${codeExamples.length + 1}`;
+        codeExamples.push({
+            id: exampleId,
+            language: lang || 'text',
+            code: rawCode,
+        });
+        const placeholder = `__DOCORBIT_CODE_BLOCK_${codeBlockPlaceholders.length}__`;
+        codeBlockPlaceholders.push(`\n\`\`\`${lang}\n${rawCode}\n\`\`\`\n`);
+        return placeholder;
+    });
+    // 4. Extract Headings and build anchors
+    cleaned = cleaned.replace(/<h([1-6])\b([^>]*)>([\s\S]*?)<\/h\1>/gi, (_, levelStr, attrs, innerText) => {
+        const level = parseInt(levelStr, 10);
+        const plainText = unescapeHtml(innerText.replace(/<[^>]+>/g, '')).replace(/\s+/g, ' ').trim();
+        if (!plainText)
+            return '';
+        let anchor = '';
+        const idMatch = attrs.match(/\bid=["']([^"']+)["']/i);
+        if (idMatch) {
+            anchor = idMatch[1];
+        }
+        else {
+            anchor = slugify(plainText);
+        }
+        headings.push({ level, text: plainText, anchor });
+        const hashes = '#'.repeat(level);
+        return `\n\n${hashes} ${plainText}\n\n`;
+    });
+    // Prefer first H1 as title, otherwise fallback to HTML title, then first heading
+    const h1 = headings.find(h => h.level === 1);
+    if (h1) {
+        title = h1.text;
+    }
+    else if (!title && headings.length > 0) {
+        title = headings[0].text;
+    }
+    if (!title) {
+        title = 'Documentation Page';
+    }
+    // 5. Extract and normalize Links
+    cleaned = cleaned.replace(/<a\b[^>]*\bhref=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, (_, href, linkText) => {
+        const text = unescapeHtml(linkText.replace(/<[^>]+>/g, '')).replace(/\s+/g, ' ').trim();
+        if (!text || href.startsWith('#') || href.startsWith('javascript:'))
+            return text;
+        let absoluteUrl = href;
+        let isExternal = false;
+        try {
+            const resolved = new URL(href, baseUrl);
+            absoluteUrl = resolved.href;
+            const baseHost = new URL(baseUrl).hostname;
+            isExternal = resolved.hostname !== baseHost;
+        }
+        catch {
+            // Keep raw href
+        }
+        links.push({
+            text,
+            url: absoluteUrl,
+            isExternal,
+        });
+        return `[${text}](${absoluteUrl})`;
+    });
+    // 6. Handle Callouts & Blockquotes
+    cleaned = cleaned.replace(/<blockquote\b[^>]*>([\s\S]*?)<\/blockquote>/gi, (_, inner) => {
+        const blockText = inner
+            .replace(/<p\b[^>]*>/gi, '')
+            .replace(/<\/p>/gi, '\n')
+            .replace(/<br\s*\/?>/gi, '\n')
+            .replace(/<[^>]+>/g, '');
+        const cleanLines = unescapeHtml(blockText)
+            .split('\n')
+            .map(l => l.trim())
+            .filter(Boolean)
+            .map(l => `> ${l}`)
+            .join('\n');
+        return `\n\n${cleanLines}\n\n`;
+    });
+    cleaned = cleaned.replace(/<div\b[^>]*\bclass=["'][^"']*\b(?:alert|callout|admonition|tip|warning|note|info)\b[^"']*["'][^>]*>([\s\S]*?)<\/div>/gi, (_, inner) => {
+        const alertText = unescapeHtml(inner.replace(/<[^>]+>/g, ' ')).replace(/\s+/g, ' ').trim();
+        if (!alertText)
+            return '';
+        return `\n\n> [!NOTE]\n> ${alertText}\n\n`;
+    });
+    // 7. Handle Tables
+    cleaned = cleaned.replace(/<table\b[^>]*>([\s\S]*?)<\/table>/gi, (_, tableContent) => {
+        const rows = [];
+        const rowMatches = tableContent.matchAll(/<tr\b[^>]*>([\s\S]*?)<\/tr>/gi);
+        for (const rMatch of rowMatches) {
+            const rowContent = rMatch[1];
+            const cells = [];
+            const cellMatches = rowContent.matchAll(/<t[hd]\b[^>]*>([\s\S]*?)<\/t[hd]>/gi);
+            for (const cMatch of cellMatches) {
+                const cellText = unescapeHtml(cMatch[1].replace(/<[^>]+>/g, '')).replace(/\s+/g, ' ').trim();
+                cells.push(cellText);
+            }
+            if (cells.length > 0) {
+                rows.push(cells);
+            }
+        }
+        if (rows.length === 0)
+            return '';
+        const colCount = Math.max(...rows.map(r => r.length));
+        const headerRow = rows[0].map(c => c || '-');
+        while (headerRow.length < colCount)
+            headerRow.push('-');
+        const delimiterRow = Array(colCount).fill('---');
+        const tableLines = [
+            `| ${headerRow.join(' | ')} |`,
+            `| ${delimiterRow.join(' | ')} |`,
+        ];
+        for (let i = 1; i < rows.length; i++) {
+            const dataRow = rows[i].map(c => c || '');
+            while (dataRow.length < colCount)
+                dataRow.push('');
+            tableLines.push(`| ${dataRow.join(' | ')} |`);
+        }
+        return `\n\n${tableLines.join('\n')}\n\n`;
+    });
+    // 8. Handle Lists
+    cleaned = cleaned
+        .replace(/<li\b[^>]*>([\s\S]*?)<\/li>/gi, (_, inner) => {
+        const text = unescapeHtml(inner.replace(/<[^>]+>/g, '')).replace(/\s+/g, ' ').trim();
+        return `\n- ${text}`;
+    })
+        .replace(/<\/?(?:ul|ol)\b[^>]*>/gi, '\n');
+    // 9. Handle Inline elements
+    cleaned = cleaned
+        .replace(/<code\b[^>]*>([\s\S]*?)<\/code>/gi, (_, inner) => {
+        const text = unescapeHtml(inner.replace(/<[^>]+>/g, '')).trim();
+        return text ? `\`${text}\`` : '';
+    })
+        .replace(/<(?:strong|b)\b[^>]*>([\s\S]*?)<\/(?:strong|b)>/gi, (_, inner) => {
+        const text = inner.trim();
+        return text ? `**${text}**` : '';
+    })
+        .replace(/<(?:em|i)\b[^>]*>([\s\S]*?)<\/(?:em|i)>/gi, (_, inner) => {
+        const text = inner.trim();
+        return text ? `*${text}*` : '';
+    })
+        .replace(/<br\s*\/?>/gi, '\n')
+        .replace(/<\/?(?:p|div|section|article|main)\b[^>]*>/gi, '\n\n');
+    // 10. Strip all remaining HTML tags
+    cleaned = cleaned.replace(/<[^>]+>/g, '');
+    // 11. Restore preserved code blocks
+    for (let i = 0; i < codeBlockPlaceholders.length; i++) {
+        cleaned = cleaned.replace(`__DOCORBIT_CODE_BLOCK_${i}__`, codeBlockPlaceholders[i]);
+    }
+    // 12. Normalize whitespace
+    cleaned = unescapeHtml(cleaned)
+        .replace(/[ \t]+/g, ' ')
+        .replace(/\n\s+\n/g, '\n\n')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+    return {
+        title,
+        markdown: cleaned,
+        headings,
+        links,
+        codeExamples,
+    };
+}

package/{packages/normalizer/src/index.ts → dist/packages/normalizer/src/index.d.ts}

@@ -5,4 +5,3 @@ export * from './page.ts';
 export * from './slicer.ts';
 export * from './example-indexer.ts';
 export * from './pitfall-extractor.ts';
-

package/dist/packages/normalizer/src/index.js

@@ -0,0 +1,7 @@
+export * from "./html.js";
+export * from "./llms.js";
+export * from "./openapi.js";
+export * from "./page.js";
+export * from "./slicer.js";
+export * from "./example-indexer.js";
+export * from "./pitfall-extractor.js";

package/dist/packages/normalizer/src/llms.d.ts

@@ -0,0 +1,3 @@
+import type { LlmsDocument } from '../../shared/src/index.ts';
+export declare function parseLlmsTxt(content: string, baseUrl: string): LlmsDocument;
+export declare function isValidLlmsTxt(content: string): boolean;

package/dist/packages/normalizer/src/llms.js

@@ -0,0 +1,71 @@
+export function parseLlmsTxt(content, baseUrl) {
+    const lines = content.split('\n');
+    let title;
+    let summary;
+    const sections = [];
+    let currentSection = { name: 'Overview', links: [] };
+    const summaryLines = [];
+    for (let i = 0; i < lines.length; i++) {
+        const rawLine = lines[i];
+        const line = rawLine.trim();
+        if (!line)
+            continue;
+        if (line.startsWith('# ') && !title) {
+            title = line.slice(2).trim();
+            continue;
+        }
+        if (line.startsWith('>')) {
+            const text = line.replace(/^>\s*/, '').trim();
+            if (text)
+                summaryLines.push(text);
+            continue;
+        }
+        if (line.startsWith('## ')) {
+            if (currentSection.links.length > 0 || currentSection.name !== 'Overview') {
+                sections.push(currentSection);
+            }
+            currentSection = {
+                name: line.slice(3).trim(),
+                links: [],
+            };
+            continue;
+        }
+        const linkMatch = line.match(/^[-*]\s+\[([^\]]+)\]\(([^)]+)\)(?::\s*(.*))?$/);
+        if (linkMatch) {
+            const linkTitle = linkMatch[1].trim();
+            let rawHref = linkMatch[2].trim();
+            const description = linkMatch[3] ? linkMatch[3].trim() : undefined;
+            try {
+                rawHref = new URL(rawHref, baseUrl).href;
+            }
+            catch {
+                // Keep raw
+            }
+            const linkItem = {
+                title: linkTitle,
+                url: rawHref,
+                ...(description ? { description } : {}),
+            };
+            currentSection.links.push(linkItem);
+        }
+    }
+    if (currentSection.links.length > 0 || sections.length === 0) {
+        sections.push(currentSection);
+    }
+    if (summaryLines.length > 0) {
+        summary = summaryLines.join(' ');
+    }
+    return {
+        title,
+        summary,
+        sections,
+        rawContent: content,
+    };
+}
+export function isValidLlmsTxt(content) {
+    if (!content || typeof content !== 'string')
+        return false;
+    const hasH1 = /^#\s+.+/m.test(content);
+    const hasLink = /^[-*]\s+\[.+\]\(.+\)/m.test(content);
+    return hasH1 && hasLink;
+}

package/dist/packages/normalizer/src/openapi/endpoint-parser.d.ts

@@ -0,0 +1,11 @@
+import type { OpenApiSummary, ApiEndpoint } from '../../../shared/src/index.ts';
+/**
+ * Detects if a document content is an OpenAPI or Swagger specification.
+ */
+export declare function detectOpenApiSpec(content: string, sourceUrl: string): OpenApiSummary | null;
+/**
+ * Parses all API endpoints from an OpenAPI 3.x or Swagger 2.0 specification.
+ * Fully deterministic with internal $ref resolution, schema normalization,
+ * pagination heuristics, and error detection.
+ */
+export declare function parseOpenApiEndpoints(rawSpec: string | Record<string, unknown>, pageId: string, snapshotId: string, sourceUrl?: string, docVersion?: string): ApiEndpoint[];