docorbit 0.1.0 → 0.1.1

package/dist/packages/normalizer/src/openapi/endpoint-parser.js

@@ -0,0 +1,357 @@
+import { createHash } from 'node:crypto';
+import { resolveJsonPointer, normalizeSchema } from "./schema-resolver.js";
+/**
+ * Detects if a document content is an OpenAPI or Swagger specification.
+ */
+export function detectOpenApiSpec(content, sourceUrl) {
+    if (!content || typeof content !== 'string')
+        return null;
+    let parsed;
+    try {
+        parsed = JSON.parse(content);
+    }
+    catch {
+        return null;
+    }
+    if (typeof parsed !== 'object' || parsed === null)
+        return null;
+    const openapiVer = typeof parsed.openapi === 'string' ? parsed.openapi : undefined;
+    const swaggerVer = typeof parsed.swagger === 'string' ? parsed.swagger : undefined;
+    const specVersion = openapiVer || swaggerVer;
+    if (!specVersion)
+        return null;
+    const info = (typeof parsed.info === 'object' && parsed.info !== null)
+        ? parsed.info
+        : {};
+    const title = typeof info.title === 'string' ? info.title : 'Untitled API';
+    const description = typeof info.description === 'string' ? info.description : undefined;
+    const servers = [];
+    if (Array.isArray(parsed.servers)) {
+        for (const server of parsed.servers) {
+            if (server && typeof server === 'object' && typeof server.url === 'string') {
+                servers.push(server.url);
+            }
+        }
+    }
+    else if (typeof parsed.host === 'string') {
+        const basePath = typeof parsed.basePath === 'string' ? parsed.basePath : '';
+        const schemes = Array.isArray(parsed.schemes) ? parsed.schemes : ['https'];
+        servers.push(`${schemes[0]}://${parsed.host}${basePath}`);
+    }
+    let pathCount = 0;
+    if (typeof parsed.paths === 'object' && parsed.paths !== null) {
+        pathCount = Object.keys(parsed.paths).length;
+    }
+    return {
+        specVersion,
+        title,
+        description,
+        servers,
+        pathCount,
+        sourceUrl,
+        rawJson: parsed,
+    };
+}
+/**
+ * Parses all API endpoints from an OpenAPI 3.x or Swagger 2.0 specification.
+ * Fully deterministic with internal $ref resolution, schema normalization,
+ * pagination heuristics, and error detection.
+ */
+export function parseOpenApiEndpoints(rawSpec, pageId, snapshotId, sourceUrl, docVersion) {
+    let root;
+    if (typeof rawSpec === 'string') {
+        try {
+            root = JSON.parse(rawSpec);
+        }
+        catch {
+            return [];
+        }
+    }
+    else {
+        root = rawSpec;
+    }
+    if (!root || typeof root !== 'object')
+        return [];
+    const isOpenApi3 = typeof root.openapi === 'string';
+    const isSwagger2 = typeof root.swagger === 'string';
+    if (!isOpenApi3 && !isSwagger2)
+        return [];
+    const paths = (typeof root.paths === 'object' && root.paths !== null)
+        ? root.paths
+        : {};
+    const endpoints = [];
+    const supportedMethods = new Set(['get', 'post', 'put', 'delete', 'patch', 'head', 'options']);
+    // Extract root-level security schemes
+    const securitySchemesMap = new Map();
+    if (isOpenApi3 && root.components && typeof root.components === 'object') {
+        const comps = root.components;
+        if (typeof comps.securitySchemes === 'object' && comps.securitySchemes !== null) {
+            for (const [name, schemeRaw] of Object.entries(comps.securitySchemes)) {
+                if (typeof schemeRaw === 'object' && schemeRaw !== null) {
+                    const s = schemeRaw;
+                    securitySchemesMap.set(name, {
+                        type: s.type || 'http',
+                        name: typeof s.name === 'string' ? s.name : name,
+                        in: s.in,
+                        scheme: typeof s.scheme === 'string' ? s.scheme : undefined,
+                        bearerFormat: typeof s.bearerFormat === 'string' ? s.bearerFormat : undefined,
+                        description: typeof s.description === 'string' ? s.description : undefined,
+                    });
+                }
+            }
+        }
+    }
+    else if (isSwagger2 && typeof root.securityDefinitions === 'object' && root.securityDefinitions !== null) {
+        for (const [name, schemeRaw] of Object.entries(root.securityDefinitions)) {
+            if (typeof schemeRaw === 'object' && schemeRaw !== null) {
+                const s = schemeRaw;
+                const rawType = typeof s.type === 'string' ? s.type : 'apiKey';
+                let mappedType = 'apiKey';
+                let mappedScheme = undefined;
+                if (rawType === 'basic') {
+                    mappedType = 'http';
+                    mappedScheme = 'basic';
+                }
+                else if (rawType === 'oauth2') {
+                    mappedType = 'oauth2';
+                }
+                securitySchemesMap.set(name, {
+                    type: mappedType,
+                    name: typeof s.name === 'string' ? s.name : name,
+                    in: s.in,
+                    scheme: mappedScheme,
+                    description: typeof s.description === 'string' ? s.description : undefined,
+                });
+            }
+        }
+    }
+    // Root security fallback
+    const rootSecurityReqs = Array.isArray(root.security)
+        ? root.security
+        : [];
+    const nowIso = new Date().toISOString();
+    for (const [pathStr, pathItemRaw] of Object.entries(paths)) {
+        if (!pathItemRaw || typeof pathItemRaw !== 'object')
+            continue;
+        const pathItem = pathItemRaw;
+        // Common path-level parameters
+        const pathCommonParams = Array.isArray(pathItem.parameters) ? pathItem.parameters : [];
+        for (const [methodKey, operationRaw] of Object.entries(pathItem)) {
+            const method = methodKey.toLowerCase();
+            if (!supportedMethods.has(method))
+                continue;
+            if (!operationRaw || typeof operationRaw !== 'object')
+                continue;
+            const op = operationRaw;
+            // Summary & Description
+            const summary = typeof op.summary === 'string' ? op.summary : undefined;
+            const description = typeof op.description === 'string' ? op.description : undefined;
+            const operationId = typeof op.operationId === 'string' ? op.operationId : undefined;
+            const deprecated = Boolean(op.deprecated || pathItem.deprecated);
+            // Parameters
+            const rawParams = [...pathCommonParams, ...(Array.isArray(op.parameters) ? op.parameters : [])];
+            const parameters = [];
+            let swaggerBodySchema = undefined;
+            for (const pRaw of rawParams) {
+                let p = pRaw;
+                if (typeof p.$ref === 'string') {
+                    const resolved = resolveJsonPointer(root, p.$ref);
+                    if (typeof resolved === 'object' && resolved !== null) {
+                        p = resolved;
+                    }
+                }
+                if (isSwagger2 && p.in === 'body') {
+                    swaggerBodySchema = p.schema;
+                    continue;
+                }
+                const name = typeof p.name === 'string' ? p.name : '';
+                const paramIn = (typeof p.in === 'string' ? p.in : 'query');
+                const required = paramIn === 'path' ? true : Boolean(p.required);
+                const pDesc = typeof p.description === 'string' ? p.description : undefined;
+                const pDefault = p.default !== undefined ? p.default : (p.schema && typeof p.schema === 'object' ? p.schema.default : undefined);
+                const pExample = p.example !== undefined ? p.example : (p.schema && typeof p.schema === 'object' ? p.schema.example : undefined);
+                let pType = typeof p.type === 'string' ? p.type : undefined;
+                let pSchema = undefined;
+                if (p.schema && typeof p.schema === 'object') {
+                    pSchema = p.schema;
+                    if (!pType && typeof pSchema.type === 'string') {
+                        pType = pSchema.type;
+                    }
+                }
+                parameters.push({
+                    name,
+                    in: paramIn,
+                    required,
+                    type: pType,
+                    description: pDesc,
+                    default: pDefault,
+                    schema: pSchema,
+                    example: pExample,
+                });
+            }
+            // Request Body
+            let requestSchema = undefined;
+            if (isOpenApi3 && op.requestBody && typeof op.requestBody === 'object') {
+                let rb = op.requestBody;
+                if (typeof rb.$ref === 'string') {
+                    const resolved = resolveJsonPointer(root, rb.$ref);
+                    if (typeof resolved === 'object' && resolved !== null) {
+                        rb = resolved;
+                    }
+                }
+                if (typeof rb.content === 'object' && rb.content !== null) {
+                    const content = rb.content;
+                    // Prefer application/json, fallback to first media type
+                    const mediaTypeObj = (content['application/json'] || Object.values(content)[0]);
+                    if (mediaTypeObj && mediaTypeObj.schema) {
+                        requestSchema = normalizeSchema(mediaTypeObj.schema, root);
+                    }
+                }
+            }
+            else if (isSwagger2 && swaggerBodySchema) {
+                requestSchema = normalizeSchema(swaggerBodySchema, root);
+            }
+            // Responses & Errors
+            const responseSchema = {};
+            const errors = [];
+            if (typeof op.responses === 'object' && op.responses !== null) {
+                for (const [statusCode, resRaw] of Object.entries(op.responses)) {
+                    let res = resRaw;
+                    if (typeof res.$ref === 'string') {
+                        const resolved = resolveJsonPointer(root, res.$ref);
+                        if (typeof resolved === 'object' && resolved !== null) {
+                            res = resolved;
+                        }
+                    }
+                    const resDesc = typeof res.description === 'string' ? res.description : '';
+                    let contentType = undefined;
+                    let resSchemaObj = undefined;
+                    let resExample = undefined;
+                    if (isOpenApi3 && typeof res.content === 'object' && res.content !== null) {
+                        const content = res.content;
+                        const ctKey = content['application/json'] ? 'application/json' : Object.keys(content)[0];
+                        contentType = ctKey;
+                        const mediaObj = content[ctKey];
+                        if (mediaObj) {
+                            resSchemaObj = mediaObj.schema;
+                            resExample = mediaObj.example || (mediaObj.examples ? Object.values(mediaObj.examples)[0] : undefined);
+                        }
+                    }
+                    else if (isSwagger2) {
+                        resSchemaObj = res.schema;
+                        contentType = 'application/json';
+                        resExample = res.examples ? Object.values(res.examples)[0] : undefined;
+                    }
+                    const normalizedResSchema = normalizeSchema(resSchemaObj, root);
+                    const apiResponse = {
+                        statusCode,
+                        description: resDesc,
+                        contentType,
+                        schema: normalizedResSchema,
+                        example: resExample,
+                    };
+                    responseSchema[statusCode] = apiResponse;
+                    // Detect errors
+                    const isErrorCode = statusCode.startsWith('4') || statusCode.startsWith('5') || statusCode === 'default';
+                    if (isErrorCode) {
+                        errors.push({
+                            statusCode,
+                            description: resDesc,
+                            schema: normalizedResSchema,
+                            example: resExample,
+                        });
+                    }
+                }
+            }
+            // Authentication Schemes
+            const auth = [];
+            const opSecurityReqs = Array.isArray(op.security)
+                ? op.security
+                : rootSecurityReqs;
+            for (const req of opSecurityReqs) {
+                if (typeof req !== 'object' || req === null)
+                    continue;
+                for (const schemeName of Object.keys(req)) {
+                    const found = securitySchemesMap.get(schemeName);
+                    if (found) {
+                        // Avoid duplicate schemes
+                        if (!auth.some(a => a.name === found.name && a.type === found.type)) {
+                            auth.push(found);
+                        }
+                    }
+                    else {
+                        auth.push({
+                            type: 'apiKey',
+                            name: schemeName,
+                        });
+                    }
+                }
+            }
+            // Pagination Heuristics
+            let pagination = undefined;
+            const paramNames = parameters.map(p => p.name.toLowerCase());
+            const cursorParams = paramNames.filter(n => ['starting_after', 'ending_before', 'cursor', 'next_cursor', 'after', 'before'].includes(n));
+            const offsetParams = paramNames.filter(n => ['offset', 'skip', 'start'].includes(n));
+            const pageParams = paramNames.filter(n => ['page', 'page_number', 'pagenumber'].includes(n));
+            const limitParams = paramNames.filter(n => ['limit', 'per_page', 'page_size', 'pagesize', 'count', 'max'].includes(n));
+            // Inspect response properties for pagination clues
+            const okResponse = responseSchema['200'] || responseSchema['201'];
+            const responseProps = okResponse?.schema?.properties ? Object.keys(okResponse.schema.properties) : [];
+            const detectedResponseFields = responseProps.filter(prop => ['has_more', 'next_cursor', 'cursor', 'next_page', 'page_info', 'total_count', 'data', 'items', 'results'].includes(prop.toLowerCase()));
+            if (cursorParams.length > 0) {
+                pagination = {
+                    type: 'cursor',
+                    parameters: [...cursorParams, ...limitParams],
+                    responseFields: detectedResponseFields,
+                };
+            }
+            else if (offsetParams.length > 0) {
+                pagination = {
+                    type: 'offset',
+                    parameters: [...offsetParams, ...limitParams],
+                    responseFields: detectedResponseFields,
+                };
+            }
+            else if (pageParams.length > 0) {
+                pagination = {
+                    type: 'page',
+                    parameters: [...pageParams, ...limitParams],
+                    responseFields: detectedResponseFields,
+                };
+            }
+            // Deterministic Endpoint ID
+            const endpointId = createHash('sha256')
+                .update(`${pageId}:${method}:${pathStr}`)
+                .digest('hex')
+                .substring(0, 16);
+            const provenance = sourceUrl
+                ? {
+                    sourceUrl,
+                    retrievedAt: nowIso,
+                    sourceAuthority: 'official',
+                }
+                : undefined;
+            endpoints.push({
+                id: endpointId,
+                pageId,
+                snapshotId,
+                method: method,
+                path: pathStr,
+                summary,
+                description,
+                operationId,
+                parameters,
+                requestSchema,
+                responseSchema,
+                auth,
+                errors,
+                pagination,
+                deprecated,
+                docVersion,
+                provenance,
+                createdAt: nowIso,
+            });
+        }
+    }
+    return endpoints;
+}

package/dist/packages/normalizer/src/openapi/schema-resolver.d.ts

@@ -0,0 +1,10 @@
+import type { ApiSchema } from '../../../shared/src/index.ts';
+/**
+ * Resolves a JSON Pointer (e.g. `#/components/schemas/Pet` or `#/definitions/Pet`)
+ * within the root document. Handles circular references safely.
+ */
+export declare function resolveJsonPointer(root: Record<string, unknown>, pointer: string, visited?: Set<string>): unknown;
+/**
+ * Normalizes an OpenAPI / JSON schema object into an ApiSchema model.
+ */
+export declare function normalizeSchema(rawSchema: unknown, root: Record<string, unknown>, visited?: Set<string>): ApiSchema | undefined;

package/dist/packages/normalizer/src/openapi/schema-resolver.js

@@ -0,0 +1,82 @@
+/**
+ * Resolves a JSON Pointer (e.g. `#/components/schemas/Pet` or `#/definitions/Pet`)
+ * within the root document. Handles circular references safely.
+ */
+export function resolveJsonPointer(root, pointer, visited = new Set()) {
+    if (visited.has(pointer)) {
+        return { $ref: pointer, description: '[Circular Reference]' };
+    }
+    visited.add(pointer);
+    if (!pointer.startsWith('#/')) {
+        return { $ref: pointer, description: '[External Reference]' };
+    }
+    const parts = pointer.slice(2).split('/').map((part) => part.replace(/~1/g, '/').replace(/~0/g, '~'));
+    let current = root;
+    for (const part of parts) {
+        if (typeof current !== 'object' || current === null) {
+            return undefined;
+        }
+        current = current[part];
+    }
+    // If the resolved object itself is a reference, resolve further
+    if (typeof current === 'object' &&
+        current !== null &&
+        typeof current.$ref === 'string') {
+        return resolveJsonPointer(root, current.$ref, visited);
+    }
+    return current;
+}
+/**
+ * Normalizes an OpenAPI / JSON schema object into an ApiSchema model.
+ */
+export function normalizeSchema(rawSchema, root, visited = new Set()) {
+    if (!rawSchema || typeof rawSchema !== 'object')
+        return undefined;
+    let schemaObj = rawSchema;
+    if (typeof schemaObj.$ref === 'string') {
+        const resolved = resolveJsonPointer(root, schemaObj.$ref, visited);
+        if (typeof resolved === 'object' && resolved !== null) {
+            schemaObj = resolved;
+        }
+    }
+    const type = typeof schemaObj.type === 'string'
+        ? schemaObj.type
+        : (Array.isArray(schemaObj.type) ? schemaObj.type.join(' | ') : undefined);
+    const description = typeof schemaObj.description === 'string'
+        ? schemaObj.description
+        : undefined;
+    const required = Array.isArray(schemaObj.required)
+        ? schemaObj.required.filter((r) => typeof r === 'string')
+        : undefined;
+    let properties;
+    if (typeof schemaObj.properties === 'object' && schemaObj.properties !== null) {
+        properties = {};
+        for (const [k, v] of Object.entries(schemaObj.properties)) {
+            if (typeof v === 'object' && v !== null && typeof v.$ref === 'string') {
+                const refResolved = resolveJsonPointer(root, v.$ref, new Set(visited));
+                properties[k] = refResolved || v;
+            }
+            else {
+                properties[k] = v;
+            }
+        }
+    }
+    let items;
+    if (typeof schemaObj.items === 'object' && schemaObj.items !== null) {
+        if (typeof schemaObj.items.$ref === 'string') {
+            const refResolved = resolveJsonPointer(root, schemaObj.items.$ref, new Set(visited));
+            items = refResolved || schemaObj.items;
+        }
+        else {
+            items = schemaObj.items;
+        }
+    }
+    return {
+        type,
+        properties,
+        required,
+        description,
+        items,
+        raw: schemaObj,
+    };
+}

package/dist/packages/normalizer/src/openapi.js

@@ -0,0 +1,2 @@
+export * from "./openapi/schema-resolver.js";
+export * from "./openapi/endpoint-parser.js";

package/dist/packages/normalizer/src/page.d.ts

@@ -0,0 +1,13 @@
+import type { NormalizedPage } from '../../shared/src/index.ts';
+export interface BuildNormalizedPageInput {
+    sourceId: string;
+    url: string;
+    rawContent: string;
+    contentType?: string;
+    sourceUrl?: string;
+    targetUrl?: string;
+    discoveredBy?: string;
+    fetchedAt?: string;
+    snapshotId?: string;
+}
+export declare function buildNormalizedPage(input: BuildNormalizedPageInput): NormalizedPage;

package/dist/packages/normalizer/src/page.js

@@ -0,0 +1,150 @@
+import { computeContentHash, estimateTokenCount, } from "../../shared/src/index.js";
+import { detectSecurityAnnotations } from "../../security/src/index.js";
+import { normalizeHtmlToMarkdown } from "./html.js";
+import { parseLlmsTxt, isValidLlmsTxt } from "./llms.js";
+import { detectOpenApiSpec } from "./openapi.js";
+function extractMarkdownHeadings(content) {
+    const headings = [];
+    const lines = content.split('\n');
+    for (const line of lines) {
+        const match = line.match(/^(#{1,6})\s+(.+)$/);
+        if (match) {
+            const level = match[1].length;
+            const text = match[2].trim();
+            const anchor = text.toLowerCase().replace(/[^\w\s-]/g, '').replace(/\s+/g, '-');
+            headings.push({ level, text, anchor });
+        }
+    }
+    return headings;
+}
+function extractMarkdownCodeBlocks(content) {
+    const examples = [];
+    const regex = /```([a-zA-Z0-9_-]*)\n([\s\S]*?)```/g;
+    let match;
+    let idx = 1;
+    while ((match = regex.exec(content)) !== null) {
+        const lang = match[1].trim() || 'text';
+        const code = match[2].trim();
+        if (code) {
+            examples.push({
+                id: `code_${idx++}`,
+                language: lang,
+                code,
+            });
+        }
+    }
+    return examples;
+}
+function extractMarkdownLinks(content, baseUrl) {
+    const links = [];
+    const regex = /\[([^\]]+)\]\(([^)]+)\)/g;
+    let match;
+    while ((match = regex.exec(content)) !== null) {
+        const text = match[1].trim();
+        const href = match[2].trim();
+        if (href.startsWith('#') || href.startsWith('javascript:'))
+            continue;
+        let absoluteUrl = href;
+        let isExternal = false;
+        try {
+            const resolved = new URL(href, baseUrl);
+            absoluteUrl = resolved.href;
+            const baseHost = new URL(baseUrl).hostname;
+            isExternal = resolved.hostname !== baseHost;
+        }
+        catch {
+            // Keep raw href
+        }
+        links.push({ text, url: absoluteUrl, isExternal });
+    }
+    return links;
+}
+export function buildNormalizedPage(input) {
+    const { sourceId, url, rawContent, contentType = '' } = input;
+    const rawBytes = Buffer.byteLength(rawContent, 'utf-8');
+    let title = 'Documentation';
+    let cleanMarkdown = '';
+    let headings = [];
+    let links = [];
+    let codeExamples = [];
+    const isHtml = contentType.includes('text/html') || /^\s*<(!DOCTYPE|html)/i.test(rawContent);
+    const isOpenApi = contentType.includes('application/json') || url.endsWith('.json') || url.endsWith('.yaml');
+    if (isOpenApi) {
+        const apiSummary = detectOpenApiSpec(rawContent, url);
+        if (apiSummary) {
+            title = `${apiSummary.title} (OpenAPI ${apiSummary.specVersion})`;
+            cleanMarkdown = `# ${title}\n\n${apiSummary.description || ''}\n\n` +
+                `**Servers**: ${apiSummary.servers.join(', ') || 'None specified'}\n\n` +
+                `**Endpoints Count**: ${apiSummary.pathCount}\n\n` +
+                `\`\`\`json\n${JSON.stringify(apiSummary.rawJson, null, 2)}\n\`\`\``;
+            headings = [
+                { level: 1, text: title, anchor: 'title' },
+                { level: 2, text: 'API Overview', anchor: 'api-overview' },
+            ];
+            codeExamples = [
+                {
+                    id: 'spec_1',
+                    language: 'json',
+                    code: JSON.stringify(apiSummary.rawJson, null, 2),
+                    caption: 'OpenAPI Specification',
+                },
+            ];
+        }
+    }
+    if (!cleanMarkdown && isHtml) {
+        const extracted = normalizeHtmlToMarkdown(rawContent, url);
+        title = extracted.title;
+        cleanMarkdown = extracted.markdown;
+        headings = extracted.headings;
+        links = extracted.links;
+        codeExamples = extracted.codeExamples;
+    }
+    if (!cleanMarkdown) {
+        if (isValidLlmsTxt(rawContent)) {
+            const llmsDoc = parseLlmsTxt(rawContent, url);
+            title = llmsDoc.title || 'llms.txt Index';
+            cleanMarkdown = rawContent;
+            headings = extractMarkdownHeadings(rawContent);
+            links = extractMarkdownLinks(rawContent, url);
+            codeExamples = extractMarkdownCodeBlocks(rawContent);
+        }
+        else {
+            cleanMarkdown = rawContent.trim();
+            headings = extractMarkdownHeadings(cleanMarkdown);
+            links = extractMarkdownLinks(cleanMarkdown, url);
+            codeExamples = extractMarkdownCodeBlocks(cleanMarkdown);
+            if (headings.length > 0 && headings[0].level === 1) {
+                title = headings[0].text;
+            }
+        }
+    }
+    const contentHash = computeContentHash(cleanMarkdown);
+    const estimatedTokens = estimateTokenCount(cleanMarkdown);
+    const securityAnnotations = detectSecurityAnnotations(cleanMarkdown);
+    const id = `page_${computeContentHash(`${sourceId}:${url}`).slice(0, 16)}`;
+    const fetchedAt = input.fetchedAt || new Date().toISOString();
+    const provenance = {
+        sourceUrl: input.sourceUrl || url,
+        targetUrl: input.targetUrl || url,
+        fetchedAt,
+        discoveredBy: input.discoveredBy || 'direct',
+        contentHash,
+        snapshotId: input.snapshotId,
+    };
+    return {
+        id,
+        sourceId,
+        title,
+        url,
+        content: cleanMarkdown,
+        headings,
+        links,
+        codeExamples,
+        contentHash,
+        fetchedAt,
+        rawBytes,
+        estimatedTokens,
+        securityAnnotations,
+        provenance,
+    };
+}

package/dist/packages/normalizer/src/pitfall-extractor.d.ts

@@ -0,0 +1,6 @@
+import type { NormalizedPage, DocumentChunk, Pitfall } from '../../shared/src/index.ts';
+/**
+ * Extracts explicit, non-inferential pitfalls, deprecations, breaking changes,
+ * and security requirements from documentation pages and chunks.
+ */
+export declare function extractPitfalls(page: NormalizedPage, chunks: DocumentChunk[], docVersion?: string, snapshotId?: string): Pitfall[];