docorbit 0.1.0

package/packages/normalizer/src/page.ts

@@ -0,0 +1,184 @@
+import {
+  computeContentHash,
+  estimateTokenCount,
+} from '../../shared/src/index.ts';
+import type {
+  NormalizedPage,
+  Heading,
+  Link,
+  CodeExample,
+  Provenance,
+} from '../../shared/src/index.ts';
+import { detectSecurityAnnotations } from '../../security/src/index.ts';
+import { normalizeHtmlToMarkdown } from './html.ts';
+import { parseLlmsTxt, isValidLlmsTxt } from './llms.ts';
+import { detectOpenApiSpec } from './openapi.ts';
+
+function extractMarkdownHeadings(content: string): Heading[] {
+  const headings: Heading[] = [];
+  const lines = content.split('\n');
+  for (const line of lines) {
+    const match = line.match(/^(#{1,6})\s+(.+)$/);
+    if (match) {
+      const level = match[1].length;
+      const text = match[2].trim();
+      const anchor = text.toLowerCase().replace(/[^\w\s-]/g, '').replace(/\s+/g, '-');
+      headings.push({ level, text, anchor });
+    }
+  }
+  return headings;
+}
+
+function extractMarkdownCodeBlocks(content: string): CodeExample[] {
+  const examples: CodeExample[] = [];
+  const regex = /```([a-zA-Z0-9_-]*)\n([\s\S]*?)```/g;
+  let match: RegExpExecArray | null;
+  let idx = 1;
+  while ((match = regex.exec(content)) !== null) {
+    const lang = match[1].trim() || 'text';
+    const code = match[2].trim();
+    if (code) {
+      examples.push({
+        id: `code_${idx++}`,
+        language: lang,
+        code,
+      });
+    }
+  }
+  return examples;
+}
+
+function extractMarkdownLinks(content: string, baseUrl: string): Link[] {
+  const links: Link[] = [];
+  const regex = /\[([^\]]+)\]\(([^)]+)\)/g;
+  let match: RegExpExecArray | null;
+  while ((match = regex.exec(content)) !== null) {
+    const text = match[1].trim();
+    const href = match[2].trim();
+    if (href.startsWith('#') || href.startsWith('javascript:')) continue;
+
+    let absoluteUrl = href;
+    let isExternal = false;
+    try {
+      const resolved = new URL(href, baseUrl);
+      absoluteUrl = resolved.href;
+      const baseHost = new URL(baseUrl).hostname;
+      isExternal = resolved.hostname !== baseHost;
+    } catch {
+      // Keep raw href
+    }
+
+    links.push({ text, url: absoluteUrl, isExternal });
+  }
+  return links;
+}
+
+export interface BuildNormalizedPageInput {
+  sourceId: string;
+  url: string;
+  rawContent: string;
+  contentType?: string;
+  sourceUrl?: string;
+  targetUrl?: string;
+  discoveredBy?: string;
+  fetchedAt?: string;
+  snapshotId?: string;
+}
+
+export function buildNormalizedPage(input: BuildNormalizedPageInput): NormalizedPage {
+  const { sourceId, url, rawContent, contentType = '' } = input;
+  const rawBytes = Buffer.byteLength(rawContent, 'utf-8');
+
+  let title = 'Documentation';
+  let cleanMarkdown = '';
+  let headings: Heading[] = [];
+  let links: Link[] = [];
+  let codeExamples: CodeExample[] = [];
+
+  const isHtml = contentType.includes('text/html') || /^\s*<(!DOCTYPE|html)/i.test(rawContent);
+  const isOpenApi = contentType.includes('application/json') || url.endsWith('.json') || url.endsWith('.yaml');
+
+  if (isOpenApi) {
+    const apiSummary = detectOpenApiSpec(rawContent, url);
+    if (apiSummary) {
+      title = `${apiSummary.title} (OpenAPI ${apiSummary.specVersion})`;
+      cleanMarkdown = `# ${title}\n\n${apiSummary.description || ''}\n\n` +
+        `**Servers**: ${apiSummary.servers.join(', ') || 'None specified'}\n\n` +
+        `**Endpoints Count**: ${apiSummary.pathCount}\n\n` +
+        `\`\`\`json\n${JSON.stringify(apiSummary.rawJson, null, 2)}\n\`\`\``;
+      headings = [
+        { level: 1, text: title, anchor: 'title' },
+        { level: 2, text: 'API Overview', anchor: 'api-overview' },
+      ];
+      codeExamples = [
+        {
+          id: 'spec_1',
+          language: 'json',
+          code: JSON.stringify(apiSummary.rawJson, null, 2),
+          caption: 'OpenAPI Specification',
+        },
+      ];
+    }
+  }
+
+  if (!cleanMarkdown && isHtml) {
+    const extracted = normalizeHtmlToMarkdown(rawContent, url);
+    title = extracted.title;
+    cleanMarkdown = extracted.markdown;
+    headings = extracted.headings;
+    links = extracted.links;
+    codeExamples = extracted.codeExamples;
+  }
+
+  if (!cleanMarkdown) {
+    if (isValidLlmsTxt(rawContent)) {
+      const llmsDoc = parseLlmsTxt(rawContent, url);
+      title = llmsDoc.title || 'llms.txt Index';
+      cleanMarkdown = rawContent;
+      headings = extractMarkdownHeadings(rawContent);
+      links = extractMarkdownLinks(rawContent, url);
+      codeExamples = extractMarkdownCodeBlocks(rawContent);
+    } else {
+      cleanMarkdown = rawContent.trim();
+      headings = extractMarkdownHeadings(cleanMarkdown);
+      links = extractMarkdownLinks(cleanMarkdown, url);
+      codeExamples = extractMarkdownCodeBlocks(cleanMarkdown);
+      if (headings.length > 0 && headings[0].level === 1) {
+        title = headings[0].text;
+      }
+    }
+  }
+
+  const contentHash = computeContentHash(cleanMarkdown);
+  const estimatedTokens = estimateTokenCount(cleanMarkdown);
+  const securityAnnotations = detectSecurityAnnotations(cleanMarkdown);
+
+  const id = `page_${computeContentHash(`${sourceId}:${url}`).slice(0, 16)}`;
+  const fetchedAt = input.fetchedAt || new Date().toISOString();
+
+  const provenance: Provenance = {
+    sourceUrl: input.sourceUrl || url,
+    targetUrl: input.targetUrl || url,
+    fetchedAt,
+    discoveredBy: input.discoveredBy || 'direct',
+    contentHash,
+    snapshotId: input.snapshotId,
+  };
+
+  return {
+    id,
+    sourceId,
+    title,
+    url,
+    content: cleanMarkdown,
+    headings,
+    links,
+    codeExamples,
+    contentHash,
+    fetchedAt,
+    rawBytes,
+    estimatedTokens,
+    securityAnnotations,
+    provenance,
+  };
+}

package/packages/normalizer/src/pitfall-extractor.ts

@@ -0,0 +1,190 @@
+import { createHash } from 'node:crypto';
+import type {
+  NormalizedPage,
+  DocumentChunk,
+  Pitfall,
+  PitfallKind,
+  Provenance,
+} from '../../shared/src/index.ts';
+
+interface PitfallPattern {
+  kind: PitfallKind;
+  pattern: RegExp;
+  titlePrefix: string;
+}
+
+const PITFALL_PATTERNS: PitfallPattern[] = [
+  {
+    kind: 'removed',
+    pattern: /(?:has been removed in|was removed in|removed in version|no longer supported in)\s+([^\n.]+)/i,
+    titlePrefix: 'Removed in',
+  },
+  {
+    kind: 'deprecated',
+    pattern: /(?:@deprecated|deprecated since|deprecated in|this (?:method|function|endpoint|parameter|feature) is deprecated|is deprecated; use|deprecated:\s+use)\s+([^\n.]+)/i,
+    titlePrefix: 'Deprecated',
+  },
+  {
+    kind: 'breaking_change',
+    pattern: /(?:breaking change:?|breaking in v?\d+|migrating to v?\d+ requires|incompatible with previous)\s*([^\n.]*)/i,
+    titlePrefix: 'Breaking Change',
+  },
+  {
+    kind: 'server_only',
+    pattern: /(?:can only be called (?:on|from) the server|server-only|server components? only|cannot be imported (?:on|into) the client|never expose.*in client(?:-side)? code)/i,
+    titlePrefix: 'Server-Only Restriction',
+  },
+  {
+    kind: 'client_only',
+    pattern: /(?:client-only|can only be called in the browser|client components? only|requires a browser environment|window is not defined)/i,
+    titlePrefix: 'Client-Only Restriction',
+  },
+  {
+    kind: 'rate_limit',
+    pattern: /(?:rate limit(?:ed)?|429 too many requests|requests per (?:minute|second|hour)|burst limit|quota exceeded)/i,
+    titlePrefix: 'Rate Limit Warning',
+  },
+  {
+    kind: 'security',
+    pattern: /(?:security warning|vulnerability|signature verification failure|never commit (?:your|the) secret|timing attacks?|replay attacks?)/i,
+    titlePrefix: 'Security Requirement',
+  },
+  {
+    kind: 'permission',
+    pattern: /(?:requires permission|requires scope|insufficient privileges|403 forbidden|unauthorized access|must have admin)/i,
+    titlePrefix: 'Permission Requirement',
+  },
+  {
+    kind: 'required_config',
+    pattern: /(?:must be configured before|environment variable.*is required|missing required config|prerequisite:\s+set)\s+([^\n.]+)/i,
+    titlePrefix: 'Required Configuration',
+  },
+  {
+    kind: 'runtime_restriction',
+    pattern: /(?:not supported in (?:the )?edge runtime|node(?:\.js)? runtime only|requires webcrypto|unsupported platform)/i,
+    titlePrefix: 'Runtime Restriction',
+  },
+];
+
+const ADMONITION_REGEX = /(?:^|\n)(?:>|:::\s*)\[!(WARNING|CAUTION|DANGER|IMPORTANT|NOTE)\](?:\s+([^\n]+))?\n([\s\S]*?)(?=\n(?:>|:::|\s*$|$))/gi;
+
+/**
+ * Extracts explicit, non-inferential pitfalls, deprecations, breaking changes,
+ * and security requirements from documentation pages and chunks.
+ */
+export function extractPitfalls(
+  page: NormalizedPage,
+  chunks: DocumentChunk[],
+  docVersion?: string,
+  snapshotId?: string
+): Pitfall[] {
+  const pitfalls: Pitfall[] = [];
+  const seenKeys = new Set<string>();
+  const nowIso = new Date().toISOString();
+
+  // Helper to add pitfall with deduplication
+  const addPitfall = (
+    kind: PitfallKind,
+    title: string,
+    content: string,
+    chunk?: DocumentChunk
+  ) => {
+    const trimmedContent = content.trim();
+    if (!trimmedContent || trimmedContent.length < 15) return;
+
+    const dedupeKey = `${kind}:${title.toLowerCase()}:${trimmedContent.substring(0, 80).toLowerCase()}`;
+    if (seenKeys.has(dedupeKey)) return;
+    seenKeys.add(dedupeKey);
+
+    const id = createHash('sha256')
+      .update(`${page.id}:${kind}:${title}:${trimmedContent.substring(0, 100)}`)
+      .digest('hex')
+      .substring(0, 16);
+
+    // Extract related API or symbol from chunk if available
+    let relatedApi: string | undefined = undefined;
+    let relatedSymbol: string | undefined = undefined;
+
+    if (chunk) {
+      // Check section path or title for API
+      const apiMatch = (chunk.title || chunk.sectionPath.join(' ')).match(/\b(GET|POST|PUT|DELETE|PATCH)\s+([/\w\-_{}]+)/i);
+      if (apiMatch) {
+        relatedApi = `${apiMatch[1].toUpperCase()} ${apiMatch[2]}`;
+      }
+    }
+
+    const provenance: Provenance = {
+      sourceUrl: page.url,
+      retrievedAt: page.metadata?.extractedAt || nowIso,
+      sourceAuthority: 'official',
+    };
+
+    const effectiveSnapshotId = snapshotId || chunk?.snapshotId || (page as any).snapshotId || 'snap_default';
+
+    pitfalls.push({
+      id,
+      chunkId: chunk?.id,
+      pageId: page.id,
+      snapshotId: effectiveSnapshotId,
+      kind,
+      title: title.trim(),
+      content: trimmedContent,
+      relatedApi,
+      relatedSymbol,
+      docVersion: docVersion || chunk?.docVersion || page.docVersion,
+      provenance,
+      createdAt: nowIso,
+    });
+  };
+
+  // 1. Process chunks for admonitions (> [!WARNING], > [!CAUTION], etc.)
+  for (const chunk of chunks) {
+    const content = chunk.content;
+
+    // Match admonitions
+    let admMatch: RegExpExecArray | null;
+    const admRegex = new RegExp(ADMONITION_REGEX.source, 'gi');
+    while ((admMatch = admRegex.exec(content)) !== null) {
+      const calloutType = admMatch[1].toUpperCase();
+      const customTitle = admMatch[2]?.trim();
+      const calloutBody = admMatch[3].replace(/^>\s?/gm, '').trim();
+
+      let kind: PitfallKind = 'security';
+      if (calloutType === 'WARNING' || calloutType === 'DANGER') {
+        kind = 'breaking_change';
+      } else if (calloutType === 'CAUTION') {
+        kind = 'security';
+      } else if (calloutType === 'IMPORTANT') {
+        kind = 'required_config';
+      }
+
+      // Check if body specifically matches any pitfall pattern
+      for (const p of PITFALL_PATTERNS) {
+        if (p.pattern.test(calloutBody)) {
+          kind = p.kind;
+          break;
+        }
+      }
+
+      const defaultTitle = customTitle || `${kind.replace(/_/g, ' ').toUpperCase()} Warning: ${chunk.title || page.title}`;
+      addPitfall(kind, defaultTitle, calloutBody, chunk);
+    }
+
+    // 2. Scan chunk text for explicit pattern matches
+    for (const { kind, pattern, titlePrefix } of PITFALL_PATTERNS) {
+      const match = content.match(pattern);
+      if (match) {
+        // Extract paragraph containing the match
+        const paragraphs = content.split(/\n\s*\n/);
+        const matchingParagraph = paragraphs.find(p => pattern.test(p));
+        if (matchingParagraph) {
+          const detail = match[1]?.trim() ? `: ${match[1].trim()}` : '';
+          const title = `${titlePrefix}${detail}`;
+          addPitfall(kind, title, matchingParagraph, chunk);
+        }
+      }
+    }
+  }
+
+  return pitfalls;
+}