docorbit 0.1.0

package/packages/mcp/src/types.ts

@@ -0,0 +1,102 @@
+export interface JsonRpcRequest {
+  jsonrpc: '2.0';
+  id?: string | number | null;
+  method: string;
+  params?: Record<string, unknown>;
+}
+
+export interface JsonRpcResponse {
+  jsonrpc: '2.0';
+  id: string | number | null;
+  result?: unknown;
+  error?: JsonRpcErrorObject;
+}
+
+export interface JsonRpcNotification {
+  jsonrpc: '2.0';
+  method: string;
+  params?: Record<string, unknown>;
+}
+
+export interface JsonRpcErrorObject {
+  code: number;
+  message: string;
+  data?: unknown;
+}
+
+export const JSONRPC_ERRORS = {
+  PARSE_ERROR: -32700,
+  INVALID_REQUEST: -32600,
+  METHOD_NOT_FOUND: -32601,
+  INVALID_PARAMS: -32602,
+  INTERNAL_ERROR: -32603,
+} as const;
+
+export interface ToolInputSchema {
+  type: 'object';
+  properties: Record<string, {
+    type: string;
+    description: string;
+    items?: Record<string, unknown>;
+    enum?: string[];
+    default?: unknown;
+  }>;
+  required?: string[];
+}
+
+export interface McpTool {
+  name: string;
+  description: string;
+  inputSchema: ToolInputSchema;
+}
+
+export interface ToolContentItem {
+  type: 'text';
+  text: string;
+}
+
+export interface CallToolResult {
+  content: ToolContentItem[];
+  isError?: boolean;
+}
+
+export interface McpResource {
+  uri: string;
+  name: string;
+  description?: string;
+  mimeType?: string;
+}
+
+export interface ResourceContent {
+  uri: string;
+  mimeType?: string;
+  text?: string;
+  blob?: string;
+}
+
+export interface ReadResourceResult {
+  contents: ResourceContent[];
+}
+
+export interface ServerCapabilities {
+  tools?: {
+    listChanged?: boolean;
+  };
+  resources?: {
+    subscribe?: boolean;
+    listChanged?: boolean;
+  };
+  prompts?: {
+    listChanged?: boolean;
+  };
+  logging?: Record<string, unknown>;
+}
+
+export interface InitializeResult {
+  protocolVersion: string;
+  serverInfo: {
+    name: string;
+    version: string;
+  };
+  capabilities: ServerCapabilities;
+}

package/packages/normalizer/src/example-indexer.ts

@@ -0,0 +1,240 @@
+import { createHash } from 'node:crypto';
+import type {
+  NormalizedPage,
+  DocumentChunk,
+  IndexedExample,
+  SourceAuthority,
+  Provenance,
+} from '../../shared/src/index.ts';
+
+const FRAMEWORK_PATTERNS: Array<{ framework: string; pattern: RegExp }> = [
+  { framework: 'next', pattern: /(?:from\s+['"]next(?:\/.*)?['"]|require\(['"]next(?:\/.*)?['"]\)|'use client'|'use server')/i },
+  { framework: 'react', pattern: /(?:from\s+['"]react['"]|require\(['"]react['"]\)|useState|useEffect|useMemo|useCallback)/ },
+  { framework: 'express', pattern: /(?:from\s+['"]express['"]|require\(['"]express['"]\)|express\(\))/i },
+  { framework: 'fastify', pattern: /(?:from\s+['"]fastify['"]|require\(['"]fastify['"]\))/i },
+  { framework: 'hono', pattern: /(?:from\s+['"]hono['"]|require\(['"]hono['"]\))/i },
+  { framework: 'vue', pattern: /(?:from\s+['"]vue['"]|createApp\(|defineComponent)/i },
+  { framework: 'svelte', pattern: /(?:from\s+['"]svelte['"]|<script.*lang=["']ts["']>)/i },
+  { framework: 'fastapi', pattern: /(?:from\s+fastapi\s+import|FastAPI\()/i },
+  { framework: 'flask', pattern: /(?:from\s+flask\s+import|Flask\(__name__\))/i },
+  { framework: 'django', pattern: /(?:from\s+django|django\.)/i },
+  { framework: 'spring', pattern: /(?:@SpringBootApplication|@RestController|@GetMapping|@PostMapping)/i },
+  { framework: 'gin', pattern: /(?:github\.com\/gin-gonic\/gin|gin\.Default\(\))/i },
+];
+
+const API_METHOD_PATH_REGEX = /\b(GET|POST|PUT|DELETE|PATCH)\s+([/\w\-_{}]+)/i;
+
+/**
+ * Detects framework from code snippet content.
+ */
+function detectFramework(code: string): string | undefined {
+  for (const { framework, pattern } of FRAMEWORK_PATTERNS) {
+    if (pattern.test(code)) {
+      return framework;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Detects an API endpoint reference from code or task context.
+ */
+function detectRelatedApi(code: string, context: string): string | undefined {
+  // Check context first (e.g. "POST /v1/webhook_endpoints")
+  const contextMatch = context.match(API_METHOD_PATH_REGEX);
+  if (contextMatch) {
+    return `${contextMatch[1].toUpperCase()} ${contextMatch[2]}`;
+  }
+
+  // Check code
+  const codeMatch = code.match(API_METHOD_PATH_REGEX);
+  if (codeMatch) {
+    return `${codeMatch[1].toUpperCase()} ${codeMatch[2]}`;
+  }
+
+  // Common client library calls (e.g. fetch('/api/v1/foo', { method: 'POST' }))
+  const fetchMatch = code.match(/fetch\(\s*['"`]([/\w\-_{}]+)['"`](?:,\s*\{[^}]*method:\s*['"](\w+)['"])?/i);
+  if (fetchMatch) {
+    const method = (fetchMatch[2] || 'GET').toUpperCase();
+    return `${method} ${fetchMatch[1]}`;
+  }
+
+  // Framework router calls: app.post('/api/webhook', ...) or router.get('/users', ...) or @app.post('/webhook')
+  const routeMatch = code.match(/(?:app|router)\.(get|post|put|delete|patch)\(\s*['"`]([/\w\-_{}]+)['"`]/i);
+  if (routeMatch) {
+    return `${routeMatch[1].toUpperCase()} ${routeMatch[2]}`;
+  }
+
+  // Python decorators: @app.post("/webhook")
+  const pyRouteMatch = code.match(/@(?:app|router)\.(get|post|put|delete|patch)\(\s*['"`]([/\w\-_{}]+)['"`]/i);
+  if (pyRouteMatch) {
+    return `${pyRouteMatch[1].toUpperCase()} ${pyRouteMatch[2]}`;
+  }
+
+  return undefined;
+}
+
+/**
+ * Detects a primary symbol or function call from code.
+ */
+function detectRelatedSymbol(code: string): string | undefined {
+  // Check function declarations
+  const funcMatch = code.match(/(?:function\s+([a-zA-Z0-9_$]+)|const\s+([a-zA-Z0-9_$]+)\s*=\s*(?:async\s*)?\()/);
+  if (funcMatch) {
+    return funcMatch[1] || funcMatch[2];
+  }
+
+  // Check class declarations
+  const classMatch = code.match(/class\s+([a-zA-Z0-9_$]+)/);
+  if (classMatch) {
+    return classMatch[1];
+  }
+
+  // Check prominent method invocations, ignoring routing boilerplate like app.post or res.json
+  const boilerplateCalls = new Set([
+    'app.get', 'app.post', 'app.put', 'app.delete', 'app.use',
+    'router.get', 'router.post', 'router.use',
+    'express.raw', 'express.json', 'express.urlencoded',
+    'res.json', 'res.send', 'res.status', 'console.log', 'console.error',
+  ]);
+  const allCalls = Array.from(code.matchAll(/([a-zA-Z0-9_$]+(?:\.[a-zA-Z0-9_$]+)+)\s*\(/g));
+  for (const match of allCalls) {
+    if (!boilerplateCalls.has(match[1])) {
+      return match[1];
+    }
+  }
+  if (allCalls.length > 0) {
+    return allCalls[0][1];
+  }
+
+  return undefined;
+}
+
+/**
+ * Extracts and indexes first-class code examples from normalized pages and chunks.
+ */
+export function extractIndexedExamples(
+  page: NormalizedPage,
+  chunks: DocumentChunk[],
+  authority: SourceAuthority,
+  docVersion?: string,
+  snapshotId?: string
+): IndexedExample[] {
+  const examples: IndexedExample[] = [];
+  const seenCodeHashes = new Set<string>();
+  const nowIso = new Date().toISOString();
+
+  // Map each code example in the page to its enclosing chunk
+  for (const codeExample of page.codeExamples) {
+    const rawCode = codeExample.code.trim();
+    if (!rawCode || rawCode.length < 15) continue; // Skip trivial snippets
+
+    const codeHash = createHash('sha256').update(rawCode).digest('hex');
+    if (seenCodeHashes.has(codeHash)) continue;
+    seenCodeHashes.add(codeHash);
+
+    // Find the chunk that contains this code snippet
+    const matchingChunk = chunks.find(c =>
+      c.chunkType === 'code' && c.content.includes(rawCode)
+    ) || chunks.find(c => c.content.includes(rawCode));
+
+    // Determine task context from breadcrumbs or headings
+    let task = '';
+    if (matchingChunk && matchingChunk.sectionPath.length > 0) {
+      task = matchingChunk.sectionPath.join(' > ');
+    } else if (matchingChunk && matchingChunk.title) {
+      task = matchingChunk.title;
+    } else {
+      task = page.title;
+    }
+
+    const language = (codeExample.language || (matchingChunk?.language) || 'typescript').toLowerCase();
+    const framework = detectFramework(rawCode);
+    const relatedApi = detectRelatedApi(rawCode, task);
+    const relatedSymbol = detectRelatedSymbol(rawCode);
+
+    const exampleId = createHash('sha256')
+      .update(`${page.id}:${codeHash}`)
+      .digest('hex')
+      .substring(0, 16);
+
+    const provenance: Provenance = {
+      sourceUrl: page.url,
+      retrievedAt: page.metadata?.extractedAt || nowIso,
+      sourceAuthority: authority,
+    };
+
+    const effectiveSnapshotId = snapshotId || matchingChunk?.snapshotId || (page as any).snapshotId || 'snap_default';
+
+    examples.push({
+      id: exampleId,
+      chunkId: matchingChunk?.id,
+      pageId: page.id,
+      snapshotId: effectiveSnapshotId,
+      language,
+      framework,
+      task,
+      code: rawCode,
+      sourceUrl: page.url,
+      sourceAuthority: authority,
+      relatedApi,
+      relatedSymbol,
+      docVersion: docVersion || page.docVersion,
+      provenance,
+      createdAt: nowIso,
+    });
+  }
+
+  // Also check chunks of type 'code' that might not have been caught in page.codeExamples
+  for (const chunk of chunks) {
+    if (chunk.chunkType !== 'code') continue;
+    const rawCode = chunk.content.trim();
+    if (!rawCode || rawCode.length < 15) continue;
+
+    const codeHash = createHash('sha256').update(rawCode).digest('hex');
+    if (seenCodeHashes.has(codeHash)) continue;
+    seenCodeHashes.add(codeHash);
+
+    const task = chunk.sectionPath.length > 0
+      ? chunk.sectionPath.join(' > ')
+      : (chunk.title || page.title);
+
+    const language = (chunk.language || 'typescript').toLowerCase();
+    const framework = detectFramework(rawCode);
+    const relatedApi = detectRelatedApi(rawCode, task);
+    const relatedSymbol = detectRelatedSymbol(rawCode);
+
+    const exampleId = createHash('sha256')
+      .update(`${page.id}:${codeHash}`)
+      .digest('hex')
+      .substring(0, 16);
+
+    const provenance: Provenance = {
+      sourceUrl: page.url,
+      retrievedAt: page.metadata?.extractedAt || nowIso,
+      sourceAuthority: authority,
+    };
+
+    const effectiveSnapshotId = snapshotId || chunk.snapshotId || (page as any).snapshotId || 'snap_default';
+
+    examples.push({
+      id: exampleId,
+      chunkId: chunk.id,
+      pageId: page.id,
+      snapshotId: effectiveSnapshotId,
+      language,
+      framework,
+      task,
+      code: rawCode,
+      sourceUrl: page.url,
+      sourceAuthority: authority,
+      relatedApi,
+      relatedSymbol,
+      docVersion: docVersion || chunk.docVersion || page.docVersion,
+      provenance,
+      createdAt: nowIso,
+    });
+  }
+
+  return examples;
+}

package/packages/normalizer/src/html.ts

@@ -0,0 +1,253 @@
+import type { Heading, Link, CodeExample } from '../../shared/src/index.ts';
+
+export interface ExtractedHtmlDoc {
+  title: string;
+  markdown: string;
+  headings: Heading[];
+  links: Link[];
+  codeExamples: CodeExample[];
+}
+
+function unescapeHtml(text: string): string {
+  return text
+    .replace(/&/g, '&')
+    .replace(/&lt;/g, '<')
+    .replace(/&gt;/g, '>')
+    .replace(/&quot;/g, '"')
+    .replace(/&#39;/g, "'")
+    .replace(/&apos;/g, "'")
+    .replace(/&nbsp;/g, ' ')
+    .replace(/&#(\d+);/g, (_, code) => String.fromCharCode(Number(code)))
+    .replace(/&#x([0-9a-fA-F]+);/g, (_, code) => String.fromCharCode(parseInt(code, 16)));
+}
+
+function slugify(text: string): string {
+  return text
+    .toLowerCase()
+    .trim()
+    .replace(/[^\w\s-]/g, '')
+    .replace(/[\s_-]+/g, '-')
+    .replace(/^-+|-+$/g, '');
+}
+
+/**
+ * Deterministically normalizes HTML documentation into clean Markdown,
+ * preserving code fences, language tags, headings, tables, callouts, and links.
+ */
+export function normalizeHtmlToMarkdown(html: string, baseUrl: string): ExtractedHtmlDoc {
+  const headings: Heading[] = [];
+  const links: Link[] = [];
+  const codeExamples: CodeExample[] = [];
+
+  let title = '';
+
+  // 1. Extract title from <title> tag if present
+  const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
+  if (titleMatch) {
+    title = unescapeHtml(titleMatch[1]).replace(/\s+/g, ' ').trim();
+    title = title.split(/\s+[|–—-]\s+/)[0].trim();
+  }
+
+  // 2. Remove non-content / boilerplate tags
+  let cleaned = html
+    .replace(/<!--[\s\S]*?-->/g, '')
+    .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, '')
+    .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, '')
+    .replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, '')
+    .replace(/<svg\b[^>]*>[\s\S]*?<\/svg>/gi, '')
+    .replace(/<iframe\b[^>]*>[\s\S]*?<\/iframe>/gi, '')
+    .replace(/<nav\b[^>]*>[\s\S]*?<\/nav>/gi, '')
+    .replace(/<header\b[^>]*>[\s\S]*?<\/header>/gi, '')
+    .replace(/<footer\b[^>]*>[\s\S]*?<\/footer>/gi, '')
+    .replace(/<div\b[^>]*(?:class|id)=["'][^"']*\b(?:sidebar|nav|menu|navbar|footer|cookie-banner|breadcrumbs)\b[^"']*["'][^>]*>[\s\S]*?<\/div>/gi, '');
+
+  // 3. Extract and preserve code blocks
+  const codeBlockPlaceholders: string[] = [];
+  cleaned = cleaned.replace(/<pre\b[^>]*>(?:<code\b([^>]*)>)?([\s\S]*?)(?:<\/code>)?<\/pre>/gi, (_, codeAttrs, codeBody) => {
+    let lang = '';
+    if (codeAttrs) {
+      const langMatch = codeAttrs.match(/\b(?:class|data-language)=["'][^"']*\b(?:language-|lang-)?([a-zA-Z0-9_-]+)\b[^"']*["']/i);
+      if (langMatch && !['hljs', 'code'].includes(langMatch[1].toLowerCase())) {
+        lang = langMatch[1].toLowerCase();
+      }
+    }
+
+    const rawCode = unescapeHtml(codeBody.replace(/<[^>]+>/g, '')).trim();
+    if (!rawCode) return '';
+
+    const exampleId = `code_${codeExamples.length + 1}`;
+    codeExamples.push({
+      id: exampleId,
+      language: lang || 'text',
+      code: rawCode,
+    });
+
+    const placeholder = `__DOCORBIT_CODE_BLOCK_${codeBlockPlaceholders.length}__`;
+    codeBlockPlaceholders.push(`\n\`\`\`${lang}\n${rawCode}\n\`\`\`\n`);
+    return placeholder;
+  });
+
+  // 4. Extract Headings and build anchors
+  cleaned = cleaned.replace(/<h([1-6])\b([^>]*)>([\s\S]*?)<\/h\1>/gi, (_, levelStr, attrs, innerText) => {
+    const level = parseInt(levelStr, 10);
+    const plainText = unescapeHtml(innerText.replace(/<[^>]+>/g, '')).replace(/\s+/g, ' ').trim();
+    if (!plainText) return '';
+
+    let anchor = '';
+    const idMatch = attrs.match(/\bid=["']([^"']+)["']/i);
+    if (idMatch) {
+      anchor = idMatch[1];
+    } else {
+      anchor = slugify(plainText);
+    }
+
+    headings.push({ level, text: plainText, anchor });
+    const hashes = '#'.repeat(level);
+    return `\n\n${hashes} ${plainText}\n\n`;
+  });
+
+  // Prefer first H1 as title, otherwise fallback to HTML title, then first heading
+  const h1 = headings.find(h => h.level === 1);
+  if (h1) {
+    title = h1.text;
+  } else if (!title && headings.length > 0) {
+    title = headings[0].text;
+  }
+  if (!title) {
+    title = 'Documentation Page';
+  }
+
+  // 5. Extract and normalize Links
+  cleaned = cleaned.replace(/<a\b[^>]*\bhref=["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, (_, href, linkText) => {
+    const text = unescapeHtml(linkText.replace(/<[^>]+>/g, '')).replace(/\s+/g, ' ').trim();
+    if (!text || href.startsWith('#') || href.startsWith('javascript:')) return text;
+
+    let absoluteUrl = href;
+    let isExternal = false;
+    try {
+      const resolved = new URL(href, baseUrl);
+      absoluteUrl = resolved.href;
+      const baseHost = new URL(baseUrl).hostname;
+      isExternal = resolved.hostname !== baseHost;
+    } catch {
+      // Keep raw href
+    }
+
+    links.push({
+      text,
+      url: absoluteUrl,
+      isExternal,
+    });
+
+    return `[${text}](${absoluteUrl})`;
+  });
+
+  // 6. Handle Callouts & Blockquotes
+  cleaned = cleaned.replace(/<blockquote\b[^>]*>([\s\S]*?)<\/blockquote>/gi, (_, inner) => {
+    const blockText = inner
+      .replace(/<p\b[^>]*>/gi, '')
+      .replace(/<\/p>/gi, '\n')
+      .replace(/<br\s*\/?>/gi, '\n')
+      .replace(/<[^>]+>/g, '');
+    const cleanLines = unescapeHtml(blockText)
+      .split('\n')
+      .map(l => l.trim())
+      .filter(Boolean)
+      .map(l => `> ${l}`)
+      .join('\n');
+    return `\n\n${cleanLines}\n\n`;
+  });
+
+  cleaned = cleaned.replace(/<div\b[^>]*\bclass=["'][^"']*\b(?:alert|callout|admonition|tip|warning|note|info)\b[^"']*["'][^>]*>([\s\S]*?)<\/div>/gi, (_, inner) => {
+    const alertText = unescapeHtml(inner.replace(/<[^>]+>/g, ' ')).replace(/\s+/g, ' ').trim();
+    if (!alertText) return '';
+    return `\n\n> [!NOTE]\n> ${alertText}\n\n`;
+  });
+
+  // 7. Handle Tables
+  cleaned = cleaned.replace(/<table\b[^>]*>([\s\S]*?)<\/table>/gi, (_, tableContent) => {
+    const rows: string[][] = [];
+    const rowMatches = tableContent.matchAll(/<tr\b[^>]*>([\s\S]*?)<\/tr>/gi);
+
+    for (const rMatch of rowMatches) {
+      const rowContent = rMatch[1];
+      const cells: string[] = [];
+      const cellMatches = rowContent.matchAll(/<t[hd]\b[^>]*>([\s\S]*?)<\/t[hd]>/gi);
+      for (const cMatch of cellMatches) {
+        const cellText = unescapeHtml(cMatch[1].replace(/<[^>]+>/g, '')).replace(/\s+/g, ' ').trim();
+        cells.push(cellText);
+      }
+      if (cells.length > 0) {
+        rows.push(cells);
+      }
+    }
+
+    if (rows.length === 0) return '';
+
+    const colCount = Math.max(...rows.map(r => r.length));
+    const headerRow = rows[0].map(c => c || '-');
+    while (headerRow.length < colCount) headerRow.push('-');
+
+    const delimiterRow = Array(colCount).fill('---');
+    const tableLines = [
+      `| ${headerRow.join(' | ')} |`,
+      `| ${delimiterRow.join(' | ')} |`,
+    ];
+
+    for (let i = 1; i < rows.length; i++) {
+      const dataRow = rows[i].map(c => c || '');
+      while (dataRow.length < colCount) dataRow.push('');
+      tableLines.push(`| ${dataRow.join(' | ')} |`);
+    }
+
+    return `\n\n${tableLines.join('\n')}\n\n`;
+  });
+
+  // 8. Handle Lists
+  cleaned = cleaned
+    .replace(/<li\b[^>]*>([\s\S]*?)<\/li>/gi, (_, inner) => {
+      const text = unescapeHtml(inner.replace(/<[^>]+>/g, '')).replace(/\s+/g, ' ').trim();
+      return `\n- ${text}`;
+    })
+    .replace(/<\/?(?:ul|ol)\b[^>]*>/gi, '\n');
+
+  // 9. Handle Inline elements
+  cleaned = cleaned
+    .replace(/<code\b[^>]*>([\s\S]*?)<\/code>/gi, (_, inner) => {
+      const text = unescapeHtml(inner.replace(/<[^>]+>/g, '')).trim();
+      return text ? `\`${text}\`` : '';
+    })
+    .replace(/<(?:strong|b)\b[^>]*>([\s\S]*?)<\/(?:strong|b)>/gi, (_, inner) => {
+      const text = inner.trim();
+      return text ? `**${text}**` : '';
+    })
+    .replace(/<(?:em|i)\b[^>]*>([\s\S]*?)<\/(?:em|i)>/gi, (_, inner) => {
+      const text = inner.trim();
+      return text ? `*${text}*` : '';
+    })
+    .replace(/<br\s*\/?>/gi, '\n')
+    .replace(/<\/?(?:p|div|section|article|main)\b[^>]*>/gi, '\n\n');
+
+  // 10. Strip all remaining HTML tags
+  cleaned = cleaned.replace(/<[^>]+>/g, '');
+
+  // 11. Restore preserved code blocks
+  for (let i = 0; i < codeBlockPlaceholders.length; i++) {
+    cleaned = cleaned.replace(`__DOCORBIT_CODE_BLOCK_${i}__`, codeBlockPlaceholders[i]);
+  }
+
+  // 12. Normalize whitespace
+  cleaned = unescapeHtml(cleaned)
+    .replace(/[ \t]+/g, ' ')
+    .replace(/\n\s+\n/g, '\n\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+
+  return {
+    title,
+    markdown: cleaned,
+    headings,
+    links,
+    codeExamples,
+  };
+}

package/packages/normalizer/src/index.ts

@@ -0,0 +1,8 @@
+export * from './html.ts';
+export * from './llms.ts';
+export * from './openapi.ts';
+export * from './page.ts';
+export * from './slicer.ts';
+export * from './example-indexer.ts';
+export * from './pitfall-extractor.ts';
+

package/packages/normalizer/src/llms.ts

@@ -0,0 +1,83 @@
+import type { LlmsDocument, LlmsSection, LlmsLink } from '../../shared/src/index.ts';
+
+export function parseLlmsTxt(content: string, baseUrl: string): LlmsDocument {
+  const lines = content.split('\n');
+  let title: string | undefined;
+  let summary: string | undefined;
+  const sections: LlmsSection[] = [];
+  let currentSection: LlmsSection = { name: 'Overview', links: [] };
+
+  const summaryLines: string[] = [];
+
+  for (let i = 0; i < lines.length; i++) {
+    const rawLine = lines[i];
+    const line = rawLine.trim();
+
+    if (!line) continue;
+
+    if (line.startsWith('# ') && !title) {
+      title = line.slice(2).trim();
+      continue;
+    }
+
+    if (line.startsWith('>')) {
+      const text = line.replace(/^>\s*/, '').trim();
+      if (text) summaryLines.push(text);
+      continue;
+    }
+
+    if (line.startsWith('## ')) {
+      if (currentSection.links.length > 0 || currentSection.name !== 'Overview') {
+        sections.push(currentSection);
+      }
+      currentSection = {
+        name: line.slice(3).trim(),
+        links: [],
+      };
+      continue;
+    }
+
+    const linkMatch = line.match(/^[-*]\s+\[([^\]]+)\]\(([^)]+)\)(?::\s*(.*))?$/);
+    if (linkMatch) {
+      const linkTitle = linkMatch[1].trim();
+      let rawHref = linkMatch[2].trim();
+      const description = linkMatch[3] ? linkMatch[3].trim() : undefined;
+
+      try {
+        rawHref = new URL(rawHref, baseUrl).href;
+      } catch {
+        // Keep raw
+      }
+
+      const linkItem: LlmsLink = {
+        title: linkTitle,
+        url: rawHref,
+        ...(description ? { description } : {}),
+      };
+
+      currentSection.links.push(linkItem);
+    }
+  }
+
+  if (currentSection.links.length > 0 || sections.length === 0) {
+    sections.push(currentSection);
+  }
+
+  if (summaryLines.length > 0) {
+    summary = summaryLines.join(' ');
+  }
+
+  return {
+    title,
+    summary,
+    sections,
+    rawContent: content,
+  };
+}
+
+export function isValidLlmsTxt(content: string): boolean {
+  if (!content || typeof content !== 'string') return false;
+  const hasH1 = /^#\s+.+/m.test(content);
+  const hasLink = /^[-*]\s+\[.+\]\(.+\)/m.test(content);
+  return hasH1 && hasLink;
+}