docorbit 0.1.0 → 0.1.2

package/dist/packages/normalizer/src/openapi/schema-resolver.d.ts

@@ -0,0 +1,10 @@
+import type { ApiSchema } from '../../../shared/src/index.ts';
+/**
+ * Resolves a JSON Pointer (e.g. `#/components/schemas/Pet` or `#/definitions/Pet`)
+ * within the root document. Handles circular references safely.
+ */
+export declare function resolveJsonPointer(root: Record<string, unknown>, pointer: string, visited?: Set<string>): unknown;
+/**
+ * Normalizes an OpenAPI / JSON schema object into an ApiSchema model.
+ */
+export declare function normalizeSchema(rawSchema: unknown, root: Record<string, unknown>, visited?: Set<string>): ApiSchema | undefined;

package/dist/packages/normalizer/src/openapi/schema-resolver.js

@@ -0,0 +1,82 @@
+/**
+ * Resolves a JSON Pointer (e.g. `#/components/schemas/Pet` or `#/definitions/Pet`)
+ * within the root document. Handles circular references safely.
+ */
+export function resolveJsonPointer(root, pointer, visited = new Set()) {
+    if (visited.has(pointer)) {
+        return { $ref: pointer, description: '[Circular Reference]' };
+    }
+    visited.add(pointer);
+    if (!pointer.startsWith('#/')) {
+        return { $ref: pointer, description: '[External Reference]' };
+    }
+    const parts = pointer.slice(2).split('/').map((part) => part.replace(/~1/g, '/').replace(/~0/g, '~'));
+    let current = root;
+    for (const part of parts) {
+        if (typeof current !== 'object' || current === null) {
+            return undefined;
+        }
+        current = current[part];
+    }
+    // If the resolved object itself is a reference, resolve further
+    if (typeof current === 'object' &&
+        current !== null &&
+        typeof current.$ref === 'string') {
+        return resolveJsonPointer(root, current.$ref, visited);
+    }
+    return current;
+}
+/**
+ * Normalizes an OpenAPI / JSON schema object into an ApiSchema model.
+ */
+export function normalizeSchema(rawSchema, root, visited = new Set()) {
+    if (!rawSchema || typeof rawSchema !== 'object')
+        return undefined;
+    let schemaObj = rawSchema;
+    if (typeof schemaObj.$ref === 'string') {
+        const resolved = resolveJsonPointer(root, schemaObj.$ref, visited);
+        if (typeof resolved === 'object' && resolved !== null) {
+            schemaObj = resolved;
+        }
+    }
+    const type = typeof schemaObj.type === 'string'
+        ? schemaObj.type
+        : (Array.isArray(schemaObj.type) ? schemaObj.type.join(' | ') : undefined);
+    const description = typeof schemaObj.description === 'string'
+        ? schemaObj.description
+        : undefined;
+    const required = Array.isArray(schemaObj.required)
+        ? schemaObj.required.filter((r) => typeof r === 'string')
+        : undefined;
+    let properties;
+    if (typeof schemaObj.properties === 'object' && schemaObj.properties !== null) {
+        properties = {};
+        for (const [k, v] of Object.entries(schemaObj.properties)) {
+            if (typeof v === 'object' && v !== null && typeof v.$ref === 'string') {
+                const refResolved = resolveJsonPointer(root, v.$ref, new Set(visited));
+                properties[k] = refResolved || v;
+            }
+            else {
+                properties[k] = v;
+            }
+        }
+    }
+    let items;
+    if (typeof schemaObj.items === 'object' && schemaObj.items !== null) {
+        if (typeof schemaObj.items.$ref === 'string') {
+            const refResolved = resolveJsonPointer(root, schemaObj.items.$ref, new Set(visited));
+            items = refResolved || schemaObj.items;
+        }
+        else {
+            items = schemaObj.items;
+        }
+    }
+    return {
+        type,
+        properties,
+        required,
+        description,
+        items,
+        raw: schemaObj,
+    };
+}

package/dist/packages/normalizer/src/openapi.js

@@ -0,0 +1,2 @@
+export * from "./openapi/schema-resolver.js";
+export * from "./openapi/endpoint-parser.js";

package/dist/packages/normalizer/src/page.d.ts

@@ -0,0 +1,13 @@
+import type { NormalizedPage } from '../../shared/src/index.ts';
+export interface BuildNormalizedPageInput {
+    sourceId: string;
+    url: string;
+    rawContent: string;
+    contentType?: string;
+    sourceUrl?: string;
+    targetUrl?: string;
+    discoveredBy?: string;
+    fetchedAt?: string;
+    snapshotId?: string;
+}
+export declare function buildNormalizedPage(input: BuildNormalizedPageInput): NormalizedPage;

package/dist/packages/normalizer/src/page.js

@@ -0,0 +1,150 @@
+import { computeContentHash, estimateTokenCount, } from "../../shared/src/index.js";
+import { detectSecurityAnnotations } from "../../security/src/index.js";
+import { normalizeHtmlToMarkdown } from "./html.js";
+import { parseLlmsTxt, isValidLlmsTxt } from "./llms.js";
+import { detectOpenApiSpec } from "./openapi.js";
+function extractMarkdownHeadings(content) {
+    const headings = [];
+    const lines = content.split('\n');
+    for (const line of lines) {
+        const match = line.match(/^(#{1,6})\s+(.+)$/);
+        if (match) {
+            const level = match[1].length;
+            const text = match[2].trim();
+            const anchor = text.toLowerCase().replace(/[^\w\s-]/g, '').replace(/\s+/g, '-');
+            headings.push({ level, text, anchor });
+        }
+    }
+    return headings;
+}
+function extractMarkdownCodeBlocks(content) {
+    const examples = [];
+    const regex = /```([a-zA-Z0-9_-]*)\n([\s\S]*?)```/g;
+    let match;
+    let idx = 1;
+    while ((match = regex.exec(content)) !== null) {
+        const lang = match[1].trim() || 'text';
+        const code = match[2].trim();
+        if (code) {
+            examples.push({
+                id: `code_${idx++}`,
+                language: lang,
+                code,
+            });
+        }
+    }
+    return examples;
+}
+function extractMarkdownLinks(content, baseUrl) {
+    const links = [];
+    const regex = /\[([^\]]+)\]\(([^)]+)\)/g;
+    let match;
+    while ((match = regex.exec(content)) !== null) {
+        const text = match[1].trim();
+        const href = match[2].trim();
+        if (href.startsWith('#') || href.startsWith('javascript:'))
+            continue;
+        let absoluteUrl = href;
+        let isExternal = false;
+        try {
+            const resolved = new URL(href, baseUrl);
+            absoluteUrl = resolved.href;
+            const baseHost = new URL(baseUrl).hostname;
+            isExternal = resolved.hostname !== baseHost;
+        }
+        catch {
+            // Keep raw href
+        }
+        links.push({ text, url: absoluteUrl, isExternal });
+    }
+    return links;
+}
+export function buildNormalizedPage(input) {
+    const { sourceId, url, rawContent, contentType = '' } = input;
+    const rawBytes = Buffer.byteLength(rawContent, 'utf-8');
+    let title = 'Documentation';
+    let cleanMarkdown = '';
+    let headings = [];
+    let links = [];
+    let codeExamples = [];
+    const isHtml = contentType.includes('text/html') || /^\s*<(!DOCTYPE|html)/i.test(rawContent);
+    const isOpenApi = contentType.includes('application/json') || url.endsWith('.json') || url.endsWith('.yaml');
+    if (isOpenApi) {
+        const apiSummary = detectOpenApiSpec(rawContent, url);
+        if (apiSummary) {
+            title = `${apiSummary.title} (OpenAPI ${apiSummary.specVersion})`;
+            cleanMarkdown = `# ${title}\n\n${apiSummary.description || ''}\n\n` +
+                `**Servers**: ${apiSummary.servers.join(', ') || 'None specified'}\n\n` +
+                `**Endpoints Count**: ${apiSummary.pathCount}\n\n` +
+                `\`\`\`json\n${JSON.stringify(apiSummary.rawJson, null, 2)}\n\`\`\``;
+            headings = [
+                { level: 1, text: title, anchor: 'title' },
+                { level: 2, text: 'API Overview', anchor: 'api-overview' },
+            ];
+            codeExamples = [
+                {
+                    id: 'spec_1',
+                    language: 'json',
+                    code: JSON.stringify(apiSummary.rawJson, null, 2),
+                    caption: 'OpenAPI Specification',
+                },
+            ];
+        }
+    }
+    if (!cleanMarkdown && isHtml) {
+        const extracted = normalizeHtmlToMarkdown(rawContent, url);
+        title = extracted.title;
+        cleanMarkdown = extracted.markdown;
+        headings = extracted.headings;
+        links = extracted.links;
+        codeExamples = extracted.codeExamples;
+    }
+    if (!cleanMarkdown) {
+        if (isValidLlmsTxt(rawContent)) {
+            const llmsDoc = parseLlmsTxt(rawContent, url);
+            title = llmsDoc.title || 'llms.txt Index';
+            cleanMarkdown = rawContent;
+            headings = extractMarkdownHeadings(rawContent);
+            links = extractMarkdownLinks(rawContent, url);
+            codeExamples = extractMarkdownCodeBlocks(rawContent);
+        }
+        else {
+            cleanMarkdown = rawContent.trim();
+            headings = extractMarkdownHeadings(cleanMarkdown);
+            links = extractMarkdownLinks(cleanMarkdown, url);
+            codeExamples = extractMarkdownCodeBlocks(cleanMarkdown);
+            if (headings.length > 0 && headings[0].level === 1) {
+                title = headings[0].text;
+            }
+        }
+    }
+    const contentHash = computeContentHash(cleanMarkdown);
+    const estimatedTokens = estimateTokenCount(cleanMarkdown);
+    const securityAnnotations = detectSecurityAnnotations(cleanMarkdown);
+    const id = `page_${computeContentHash(`${sourceId}:${url}`).slice(0, 16)}`;
+    const fetchedAt = input.fetchedAt || new Date().toISOString();
+    const provenance = {
+        sourceUrl: input.sourceUrl || url,
+        targetUrl: input.targetUrl || url,
+        fetchedAt,
+        discoveredBy: input.discoveredBy || 'direct',
+        contentHash,
+        snapshotId: input.snapshotId,
+    };
+    return {
+        id,
+        sourceId,
+        title,
+        url,
+        content: cleanMarkdown,
+        headings,
+        links,
+        codeExamples,
+        contentHash,
+        fetchedAt,
+        rawBytes,
+        estimatedTokens,
+        securityAnnotations,
+        provenance,
+    };
+}

package/dist/packages/normalizer/src/pitfall-extractor.d.ts

@@ -0,0 +1,6 @@
+import type { NormalizedPage, DocumentChunk, Pitfall } from '../../shared/src/index.ts';
+/**
+ * Extracts explicit, non-inferential pitfalls, deprecations, breaking changes,
+ * and security requirements from documentation pages and chunks.
+ */
+export declare function extractPitfalls(page: NormalizedPage, chunks: DocumentChunk[], docVersion?: string, snapshotId?: string): Pitfall[];

package/dist/packages/normalizer/src/pitfall-extractor.js

@@ -0,0 +1,153 @@
+import { createHash } from 'node:crypto';
+const PITFALL_PATTERNS = [
+    {
+        kind: 'removed',
+        pattern: /(?:has been removed in|was removed in|removed in version|no longer supported in)\s+([^\n.]+)/i,
+        titlePrefix: 'Removed in',
+    },
+    {
+        kind: 'deprecated',
+        pattern: /(?:@deprecated|deprecated since|deprecated in|this (?:method|function|endpoint|parameter|feature) is deprecated|is deprecated; use|deprecated:\s+use)\s+([^\n.]+)/i,
+        titlePrefix: 'Deprecated',
+    },
+    {
+        kind: 'breaking_change',
+        pattern: /(?:breaking change:?|breaking in v?\d+|migrating to v?\d+ requires|incompatible with previous)\s*([^\n.]*)/i,
+        titlePrefix: 'Breaking Change',
+    },
+    {
+        kind: 'server_only',
+        pattern: /(?:can only be called (?:on|from) the server|server-only|server components? only|cannot be imported (?:on|into) the client|never expose.*in client(?:-side)? code)/i,
+        titlePrefix: 'Server-Only Restriction',
+    },
+    {
+        kind: 'client_only',
+        pattern: /(?:client-only|can only be called in the browser|client components? only|requires a browser environment|window is not defined)/i,
+        titlePrefix: 'Client-Only Restriction',
+    },
+    {
+        kind: 'rate_limit',
+        pattern: /(?:rate limit(?:ed)?|429 too many requests|requests per (?:minute|second|hour)|burst limit|quota exceeded)/i,
+        titlePrefix: 'Rate Limit Warning',
+    },
+    {
+        kind: 'security',
+        pattern: /(?:security warning|vulnerability|signature verification failure|never commit (?:your|the) secret|timing attacks?|replay attacks?)/i,
+        titlePrefix: 'Security Requirement',
+    },
+    {
+        kind: 'permission',
+        pattern: /(?:requires permission|requires scope|insufficient privileges|403 forbidden|unauthorized access|must have admin)/i,
+        titlePrefix: 'Permission Requirement',
+    },
+    {
+        kind: 'required_config',
+        pattern: /(?:must be configured before|environment variable.*is required|missing required config|prerequisite:\s+set)\s+([^\n.]+)/i,
+        titlePrefix: 'Required Configuration',
+    },
+    {
+        kind: 'runtime_restriction',
+        pattern: /(?:not supported in (?:the )?edge runtime|node(?:\.js)? runtime only|requires webcrypto|unsupported platform)/i,
+        titlePrefix: 'Runtime Restriction',
+    },
+];
+const ADMONITION_REGEX = /(?:^|\n)(?:>|:::\s*)\[!(WARNING|CAUTION|DANGER|IMPORTANT|NOTE)\](?:\s+([^\n]+))?\n([\s\S]*?)(?=\n(?:>|:::|\s*$|$))/gi;
+/**
+ * Extracts explicit, non-inferential pitfalls, deprecations, breaking changes,
+ * and security requirements from documentation pages and chunks.
+ */
+export function extractPitfalls(page, chunks, docVersion, snapshotId) {
+    const pitfalls = [];
+    const seenKeys = new Set();
+    const nowIso = new Date().toISOString();
+    // Helper to add pitfall with deduplication
+    const addPitfall = (kind, title, content, chunk) => {
+        const trimmedContent = content.trim();
+        if (!trimmedContent || trimmedContent.length < 15)
+            return;
+        const dedupeKey = `${kind}:${title.toLowerCase()}:${trimmedContent.substring(0, 80).toLowerCase()}`;
+        if (seenKeys.has(dedupeKey))
+            return;
+        seenKeys.add(dedupeKey);
+        const id = createHash('sha256')
+            .update(`${page.id}:${kind}:${title}:${trimmedContent.substring(0, 100)}`)
+            .digest('hex')
+            .substring(0, 16);
+        // Extract related API or symbol from chunk if available
+        let relatedApi = undefined;
+        let relatedSymbol = undefined;
+        if (chunk) {
+            // Check section path or title for API
+            const apiMatch = (chunk.title || chunk.sectionPath.join(' ')).match(/\b(GET|POST|PUT|DELETE|PATCH)\s+([/\w\-_{}]+)/i);
+            if (apiMatch) {
+                relatedApi = `${apiMatch[1].toUpperCase()} ${apiMatch[2]}`;
+            }
+        }
+        const provenance = {
+            sourceUrl: page.url,
+            retrievedAt: page.metadata?.extractedAt || nowIso,
+            sourceAuthority: 'official',
+        };
+        const effectiveSnapshotId = snapshotId || chunk?.snapshotId || page.snapshotId || 'snap_default';
+        pitfalls.push({
+            id,
+            chunkId: chunk?.id,
+            pageId: page.id,
+            snapshotId: effectiveSnapshotId,
+            kind,
+            title: title.trim(),
+            content: trimmedContent,
+            relatedApi,
+            relatedSymbol,
+            docVersion: docVersion || chunk?.docVersion || page.docVersion,
+            provenance,
+            createdAt: nowIso,
+        });
+    };
+    // 1. Process chunks for admonitions (> [!WARNING], > [!CAUTION], etc.)
+    for (const chunk of chunks) {
+        const content = chunk.content;
+        // Match admonitions
+        let admMatch;
+        const admRegex = new RegExp(ADMONITION_REGEX.source, 'gi');
+        while ((admMatch = admRegex.exec(content)) !== null) {
+            const calloutType = admMatch[1].toUpperCase();
+            const customTitle = admMatch[2]?.trim();
+            const calloutBody = admMatch[3].replace(/^>\s?/gm, '').trim();
+            let kind = 'security';
+            if (calloutType === 'WARNING' || calloutType === 'DANGER') {
+                kind = 'breaking_change';
+            }
+            else if (calloutType === 'CAUTION') {
+                kind = 'security';
+            }
+            else if (calloutType === 'IMPORTANT') {
+                kind = 'required_config';
+            }
+            // Check if body specifically matches any pitfall pattern
+            for (const p of PITFALL_PATTERNS) {
+                if (p.pattern.test(calloutBody)) {
+                    kind = p.kind;
+                    break;
+                }
+            }
+            const defaultTitle = customTitle || `${kind.replace(/_/g, ' ').toUpperCase()} Warning: ${chunk.title || page.title}`;
+            addPitfall(kind, defaultTitle, calloutBody, chunk);
+        }
+        // 2. Scan chunk text for explicit pattern matches
+        for (const { kind, pattern, titlePrefix } of PITFALL_PATTERNS) {
+            const match = content.match(pattern);
+            if (match) {
+                // Extract paragraph containing the match
+                const paragraphs = content.split(/\n\s*\n/);
+                const matchingParagraph = paragraphs.find(p => pattern.test(p));
+                if (matchingParagraph) {
+                    const detail = match[1]?.trim() ? `: ${match[1].trim()}` : '';
+                    const title = `${titlePrefix}${detail}`;
+                    addPitfall(kind, title, matchingParagraph, chunk);
+                }
+            }
+        }
+    }
+    return pitfalls;
+}

package/dist/packages/normalizer/src/slicer.d.ts

@@ -0,0 +1,30 @@
+import type { NormalizedPage, DocumentChunk, ChunkType, ChunkCode, SymbolReference, ChunkRelationship, ChunkingConfig } from '../../shared/src/index.ts';
+export declare const DEFAULT_CHUNKING_CONFIG: ChunkingConfig;
+export interface SlicingResult {
+    chunks: DocumentChunk[];
+    relationships: ChunkRelationship[];
+    codeSnippets: ChunkCode[];
+    symbols: SymbolReference[];
+}
+interface RawBlock {
+    type: 'heading' | 'code' | 'warning' | 'prose';
+    headingLevel?: number;
+    headingText?: string;
+    language?: string;
+    content: string;
+}
+/**
+ * Shallow, deterministic symbol extractor using lightweight regex patterns.
+ * No external AST parsers or language runtimes.
+ */
+export declare function extractSymbols(content: string, chunkId: string): SymbolReference[];
+/**
+ * Detects chunk type based on block composition and content.
+ */
+export declare function detectChunkType(blocks: RawBlock[], content: string): ChunkType;
+/**
+ * Slices a NormalizedPage into standalone, context-preserving DocumentChunks,
+ * maintaining section breadcrumbs, relationship links, code snippets, and symbols.
+ */
+export declare function slicePageIntoChunks(page: NormalizedPage, snapshotId: string, config?: ChunkingConfig): SlicingResult;
+export {};