docorbit 0.1.0

package/packages/normalizer/src/openapi/endpoint-parser.ts

@@ -0,0 +1,406 @@
+import { createHash } from 'node:crypto';
+import type {
+  OpenApiSummary,
+  ApiEndpoint,
+  ApiParameter,
+  ApiSchema,
+  ApiResponse,
+  ApiAuthScheme,
+  ApiErrorResponse,
+  ApiPaginationMetadata,
+  Provenance,
+} from '../../../shared/src/index.ts';
+import { resolveJsonPointer, normalizeSchema } from './schema-resolver.ts';
+
+/**
+ * Detects if a document content is an OpenAPI or Swagger specification.
+ */
+export function detectOpenApiSpec(content: string, sourceUrl: string): OpenApiSummary | null {
+  if (!content || typeof content !== 'string') return null;
+
+  let parsed: Record<string, unknown>;
+  try {
+    parsed = JSON.parse(content);
+  } catch {
+    return null;
+  }
+
+  if (typeof parsed !== 'object' || parsed === null) return null;
+
+  const openapiVer = typeof parsed.openapi === 'string' ? parsed.openapi : undefined;
+  const swaggerVer = typeof parsed.swagger === 'string' ? parsed.swagger : undefined;
+
+  const specVersion = openapiVer || swaggerVer;
+  if (!specVersion) return null;
+
+  const info = (typeof parsed.info === 'object' && parsed.info !== null)
+    ? (parsed.info as Record<string, unknown>)
+    : {};
+
+  const title = typeof info.title === 'string' ? info.title : 'Untitled API';
+  const description = typeof info.description === 'string' ? info.description : undefined;
+
+  const servers: string[] = [];
+  if (Array.isArray(parsed.servers)) {
+    for (const server of parsed.servers) {
+      if (server && typeof server === 'object' && typeof (server as Record<string, unknown>).url === 'string') {
+        servers.push((server as Record<string, unknown>).url as string);
+      }
+    }
+  } else if (typeof parsed.host === 'string') {
+    const basePath = typeof parsed.basePath === 'string' ? parsed.basePath : '';
+    const schemes = Array.isArray(parsed.schemes) ? parsed.schemes : ['https'];
+    servers.push(`${schemes[0]}://${parsed.host}${basePath}`);
+  }
+
+  let pathCount = 0;
+  if (typeof parsed.paths === 'object' && parsed.paths !== null) {
+    pathCount = Object.keys(parsed.paths).length;
+  }
+
+  return {
+    specVersion,
+    title,
+    description,
+    servers,
+    pathCount,
+    sourceUrl,
+    rawJson: parsed,
+  };
+}
+
+/**
+ * Parses all API endpoints from an OpenAPI 3.x or Swagger 2.0 specification.
+ * Fully deterministic with internal $ref resolution, schema normalization,
+ * pagination heuristics, and error detection.
+ */
+export function parseOpenApiEndpoints(
+  rawSpec: string | Record<string, unknown>,
+  pageId: string,
+  snapshotId: string,
+  sourceUrl?: string,
+  docVersion?: string
+): ApiEndpoint[] {
+  let root: Record<string, unknown>;
+  if (typeof rawSpec === 'string') {
+    try {
+      root = JSON.parse(rawSpec);
+    } catch {
+      return [];
+    }
+  } else {
+    root = rawSpec;
+  }
+
+  if (!root || typeof root !== 'object') return [];
+
+  const isOpenApi3 = typeof root.openapi === 'string';
+  const isSwagger2 = typeof root.swagger === 'string';
+  if (!isOpenApi3 && !isSwagger2) return [];
+
+  const paths = (typeof root.paths === 'object' && root.paths !== null)
+    ? (root.paths as Record<string, unknown>)
+    : {};
+
+  const endpoints: ApiEndpoint[] = [];
+  const supportedMethods = new Set(['get', 'post', 'put', 'delete', 'patch', 'head', 'options']);
+
+  // Extract root-level security schemes
+  const securitySchemesMap = new Map<string, ApiAuthScheme>();
+  if (isOpenApi3 && root.components && typeof root.components === 'object') {
+    const comps = root.components as Record<string, unknown>;
+    if (typeof comps.securitySchemes === 'object' && comps.securitySchemes !== null) {
+      for (const [name, schemeRaw] of Object.entries(comps.securitySchemes as Record<string, unknown>)) {
+        if (typeof schemeRaw === 'object' && schemeRaw !== null) {
+          const s = schemeRaw as Record<string, unknown>;
+          securitySchemesMap.set(name, {
+            type: (s.type as ApiAuthScheme['type']) || 'http',
+            name: typeof s.name === 'string' ? s.name : name,
+            in: s.in as ApiAuthScheme['in'],
+            scheme: typeof s.scheme === 'string' ? s.scheme : undefined,
+            bearerFormat: typeof s.bearerFormat === 'string' ? s.bearerFormat : undefined,
+            description: typeof s.description === 'string' ? s.description : undefined,
+          });
+        }
+      }
+    }
+  } else if (isSwagger2 && typeof root.securityDefinitions === 'object' && root.securityDefinitions !== null) {
+    for (const [name, schemeRaw] of Object.entries(root.securityDefinitions as Record<string, unknown>)) {
+      if (typeof schemeRaw === 'object' && schemeRaw !== null) {
+        const s = schemeRaw as Record<string, unknown>;
+        const rawType = typeof s.type === 'string' ? s.type : 'apiKey';
+        let mappedType: ApiAuthScheme['type'] = 'apiKey';
+        let mappedScheme: string | undefined = undefined;
+
+        if (rawType === 'basic') {
+          mappedType = 'http';
+          mappedScheme = 'basic';
+        } else if (rawType === 'oauth2') {
+          mappedType = 'oauth2';
+        }
+
+        securitySchemesMap.set(name, {
+          type: mappedType,
+          name: typeof s.name === 'string' ? s.name : name,
+          in: s.in as ApiAuthScheme['in'],
+          scheme: mappedScheme,
+          description: typeof s.description === 'string' ? s.description : undefined,
+        });
+      }
+    }
+  }
+
+  // Root security fallback
+  const rootSecurityReqs: Array<Record<string, unknown>> = Array.isArray(root.security)
+    ? (root.security as Array<Record<string, unknown>>)
+    : [];
+
+  const nowIso = new Date().toISOString();
+
+  for (const [pathStr, pathItemRaw] of Object.entries(paths)) {
+    if (!pathItemRaw || typeof pathItemRaw !== 'object') continue;
+    const pathItem = pathItemRaw as Record<string, unknown>;
+
+    // Common path-level parameters
+    const pathCommonParams: unknown[] = Array.isArray(pathItem.parameters) ? pathItem.parameters : [];
+
+    for (const [methodKey, operationRaw] of Object.entries(pathItem)) {
+      const method = methodKey.toLowerCase();
+      if (!supportedMethods.has(method)) continue;
+      if (!operationRaw || typeof operationRaw !== 'object') continue;
+
+      const op = operationRaw as Record<string, unknown>;
+
+      // Summary & Description
+      const summary = typeof op.summary === 'string' ? op.summary : undefined;
+      const description = typeof op.description === 'string' ? op.description : undefined;
+      const operationId = typeof op.operationId === 'string' ? op.operationId : undefined;
+      const deprecated = Boolean(op.deprecated || pathItem.deprecated);
+
+      // Parameters
+      const rawParams: unknown[] = [...pathCommonParams, ...(Array.isArray(op.parameters) ? op.parameters : [])];
+      const parameters: ApiParameter[] = [];
+      let swaggerBodySchema: unknown = undefined;
+
+      for (const pRaw of rawParams) {
+        let p = pRaw as Record<string, unknown>;
+        if (typeof p.$ref === 'string') {
+          const resolved = resolveJsonPointer(root, p.$ref);
+          if (typeof resolved === 'object' && resolved !== null) {
+            p = resolved as Record<string, unknown>;
+          }
+        }
+
+        if (isSwagger2 && p.in === 'body') {
+          swaggerBodySchema = p.schema;
+          continue;
+        }
+
+        const name = typeof p.name === 'string' ? p.name : '';
+        const paramIn = (typeof p.in === 'string' ? p.in : 'query') as ApiParameter['in'];
+        const required = paramIn === 'path' ? true : Boolean(p.required);
+        const pDesc = typeof p.description === 'string' ? p.description : undefined;
+        const pDefault = p.default !== undefined ? p.default : (p.schema && typeof p.schema === 'object' ? (p.schema as Record<string, unknown>).default : undefined);
+        const pExample = p.example !== undefined ? p.example : (p.schema && typeof p.schema === 'object' ? (p.schema as Record<string, unknown>).example : undefined);
+
+        let pType: string | undefined = typeof p.type === 'string' ? p.type : undefined;
+        let pSchema: Record<string, unknown> | undefined = undefined;
+
+        if (p.schema && typeof p.schema === 'object') {
+          pSchema = p.schema as Record<string, unknown>;
+          if (!pType && typeof pSchema.type === 'string') {
+            pType = pSchema.type;
+          }
+        }
+
+        parameters.push({
+          name,
+          in: paramIn,
+          required,
+          type: pType,
+          description: pDesc,
+          default: pDefault,
+          schema: pSchema,
+          example: pExample,
+        });
+      }
+
+      // Request Body
+      let requestSchema: ApiSchema | undefined = undefined;
+      if (isOpenApi3 && op.requestBody && typeof op.requestBody === 'object') {
+        let rb = op.requestBody as Record<string, unknown>;
+        if (typeof rb.$ref === 'string') {
+          const resolved = resolveJsonPointer(root, rb.$ref);
+          if (typeof resolved === 'object' && resolved !== null) {
+            rb = resolved as Record<string, unknown>;
+          }
+        }
+        if (typeof rb.content === 'object' && rb.content !== null) {
+          const content = rb.content as Record<string, unknown>;
+          // Prefer application/json, fallback to first media type
+          const mediaTypeObj = (content['application/json'] || Object.values(content)[0]) as Record<string, unknown> | undefined;
+          if (mediaTypeObj && mediaTypeObj.schema) {
+            requestSchema = normalizeSchema(mediaTypeObj.schema, root);
+          }
+        }
+      } else if (isSwagger2 && swaggerBodySchema) {
+        requestSchema = normalizeSchema(swaggerBodySchema, root);
+      }
+
+      // Responses & Errors
+      const responseSchema: Record<string, ApiResponse> = {};
+      const errors: ApiErrorResponse[] = [];
+
+      if (typeof op.responses === 'object' && op.responses !== null) {
+        for (const [statusCode, resRaw] of Object.entries(op.responses as Record<string, unknown>)) {
+          let res = resRaw as Record<string, unknown>;
+          if (typeof res.$ref === 'string') {
+            const resolved = resolveJsonPointer(root, res.$ref);
+            if (typeof resolved === 'object' && resolved !== null) {
+              res = resolved as Record<string, unknown>;
+            }
+          }
+
+          const resDesc = typeof res.description === 'string' ? res.description : '';
+          let contentType: string | undefined = undefined;
+          let resSchemaObj: unknown = undefined;
+          let resExample: unknown = undefined;
+
+          if (isOpenApi3 && typeof res.content === 'object' && res.content !== null) {
+            const content = res.content as Record<string, unknown>;
+            const ctKey = content['application/json'] ? 'application/json' : Object.keys(content)[0];
+            contentType = ctKey;
+            const mediaObj = content[ctKey] as Record<string, unknown> | undefined;
+            if (mediaObj) {
+              resSchemaObj = mediaObj.schema;
+              resExample = mediaObj.example || (mediaObj.examples ? Object.values(mediaObj.examples as Record<string, unknown>)[0] : undefined);
+            }
+          } else if (isSwagger2) {
+            resSchemaObj = res.schema;
+            contentType = 'application/json';
+            resExample = res.examples ? Object.values(res.examples as Record<string, unknown>)[0] : undefined;
+          }
+
+          const normalizedResSchema = normalizeSchema(resSchemaObj, root);
+
+          const apiResponse: ApiResponse = {
+            statusCode,
+            description: resDesc,
+            contentType,
+            schema: normalizedResSchema,
+            example: resExample,
+          };
+
+          responseSchema[statusCode] = apiResponse;
+
+          // Detect errors
+          const isErrorCode = statusCode.startsWith('4') || statusCode.startsWith('5') || statusCode === 'default';
+          if (isErrorCode) {
+            errors.push({
+              statusCode,
+              description: resDesc,
+              schema: normalizedResSchema,
+              example: resExample,
+            });
+          }
+        }
+      }
+
+      // Authentication Schemes
+      const auth: ApiAuthScheme[] = [];
+      const opSecurityReqs: Array<Record<string, unknown>> = Array.isArray(op.security)
+        ? (op.security as Array<Record<string, unknown>>)
+        : rootSecurityReqs;
+
+      for (const req of opSecurityReqs) {
+        if (typeof req !== 'object' || req === null) continue;
+        for (const schemeName of Object.keys(req)) {
+          const found = securitySchemesMap.get(schemeName);
+          if (found) {
+            // Avoid duplicate schemes
+            if (!auth.some(a => a.name === found.name && a.type === found.type)) {
+              auth.push(found);
+            }
+          } else {
+            auth.push({
+              type: 'apiKey',
+              name: schemeName,
+            });
+          }
+        }
+      }
+
+      // Pagination Heuristics
+      let pagination: ApiPaginationMetadata | undefined = undefined;
+      const paramNames = parameters.map(p => p.name.toLowerCase());
+      const cursorParams = paramNames.filter(n => ['starting_after', 'ending_before', 'cursor', 'next_cursor', 'after', 'before'].includes(n));
+      const offsetParams = paramNames.filter(n => ['offset', 'skip', 'start'].includes(n));
+      const pageParams = paramNames.filter(n => ['page', 'page_number', 'pagenumber'].includes(n));
+      const limitParams = paramNames.filter(n => ['limit', 'per_page', 'page_size', 'pagesize', 'count', 'max'].includes(n));
+
+      // Inspect response properties for pagination clues
+      const okResponse = responseSchema['200'] || responseSchema['201'];
+      const responseProps = okResponse?.schema?.properties ? Object.keys(okResponse.schema.properties) : [];
+      const detectedResponseFields = responseProps.filter(prop =>
+        ['has_more', 'next_cursor', 'cursor', 'next_page', 'page_info', 'total_count', 'data', 'items', 'results'].includes(prop.toLowerCase())
+      );
+
+      if (cursorParams.length > 0) {
+        pagination = {
+          type: 'cursor',
+          parameters: [...cursorParams, ...limitParams],
+          responseFields: detectedResponseFields,
+        };
+      } else if (offsetParams.length > 0) {
+        pagination = {
+          type: 'offset',
+          parameters: [...offsetParams, ...limitParams],
+          responseFields: detectedResponseFields,
+        };
+      } else if (pageParams.length > 0) {
+        pagination = {
+          type: 'page',
+          parameters: [...pageParams, ...limitParams],
+          responseFields: detectedResponseFields,
+        };
+      }
+
+      // Deterministic Endpoint ID
+      const endpointId = createHash('sha256')
+        .update(`${pageId}:${method}:${pathStr}`)
+        .digest('hex')
+        .substring(0, 16);
+
+      const provenance: Provenance | undefined = sourceUrl
+        ? {
+            sourceUrl,
+            retrievedAt: nowIso,
+            sourceAuthority: 'official',
+          }
+        : undefined;
+
+      endpoints.push({
+        id: endpointId,
+        pageId,
+        snapshotId,
+        method: method as ApiEndpoint['method'],
+        path: pathStr,
+        summary,
+        description,
+        operationId,
+        parameters,
+        requestSchema,
+        responseSchema,
+        auth,
+        errors,
+        pagination,
+        deprecated,
+        docVersion,
+        provenance,
+        createdAt: nowIso,
+      });
+    }
+  }
+
+  return endpoints;
+}

package/packages/normalizer/src/openapi/schema-resolver.ts

@@ -0,0 +1,111 @@
+import type { ApiSchema } from '../../../shared/src/index.ts';
+
+/**
+ * Resolves a JSON Pointer (e.g. `#/components/schemas/Pet` or `#/definitions/Pet`)
+ * within the root document. Handles circular references safely.
+ */
+export function resolveJsonPointer(
+  root: Record<string, unknown>,
+  pointer: string,
+  visited: Set<string> = new Set()
+): unknown {
+  if (visited.has(pointer)) {
+    return { $ref: pointer, description: '[Circular Reference]' };
+  }
+  visited.add(pointer);
+
+  if (!pointer.startsWith('#/')) {
+    return { $ref: pointer, description: '[External Reference]' };
+  }
+
+  const parts = pointer.slice(2).split('/').map((part) =>
+    part.replace(/~1/g, '/').replace(/~0/g, '~')
+  );
+
+  let current: unknown = root;
+  for (const part of parts) {
+    if (typeof current !== 'object' || current === null) {
+      return undefined;
+    }
+    current = (current as Record<string, unknown>)[part];
+  }
+
+  // If the resolved object itself is a reference, resolve further
+  if (
+    typeof current === 'object' &&
+    current !== null &&
+    typeof (current as Record<string, unknown>).$ref === 'string'
+  ) {
+    return resolveJsonPointer(
+      root,
+      (current as Record<string, unknown>).$ref as string,
+      visited
+    );
+  }
+
+  return current;
+}
+
+/**
+ * Normalizes an OpenAPI / JSON schema object into an ApiSchema model.
+ */
+export function normalizeSchema(
+  rawSchema: unknown,
+  root: Record<string, unknown>,
+  visited: Set<string> = new Set()
+): ApiSchema | undefined {
+  if (!rawSchema || typeof rawSchema !== 'object') return undefined;
+
+  let schemaObj = rawSchema as Record<string, unknown>;
+
+  if (typeof schemaObj.$ref === 'string') {
+    const resolved = resolveJsonPointer(root, schemaObj.$ref, visited);
+    if (typeof resolved === 'object' && resolved !== null) {
+      schemaObj = resolved as Record<string, unknown>;
+    }
+  }
+
+  const type = typeof schemaObj.type === 'string'
+    ? schemaObj.type
+    : (Array.isArray(schemaObj.type) ? schemaObj.type.join(' | ') : undefined);
+
+  const description = typeof schemaObj.description === 'string'
+    ? schemaObj.description
+    : undefined;
+
+  const required = Array.isArray(schemaObj.required)
+    ? schemaObj.required.filter((r): r is string => typeof r === 'string')
+    : undefined;
+
+  let properties: Record<string, unknown> | undefined;
+  if (typeof schemaObj.properties === 'object' && schemaObj.properties !== null) {
+    properties = {};
+    for (const [k, v] of Object.entries(schemaObj.properties as Record<string, unknown>)) {
+      if (typeof v === 'object' && v !== null && typeof (v as Record<string, unknown>).$ref === 'string') {
+        const refResolved = resolveJsonPointer(root, (v as Record<string, unknown>).$ref as string, new Set(visited));
+        properties[k] = refResolved || v;
+      } else {
+        properties[k] = v;
+      }
+    }
+  }
+
+  let items: Record<string, unknown> | undefined;
+  if (typeof schemaObj.items === 'object' && schemaObj.items !== null) {
+    if (typeof (schemaObj.items as Record<string, unknown>).$ref === 'string') {
+      const refResolved = resolveJsonPointer(root, (schemaObj.items as Record<string, unknown>).$ref as string, new Set(visited));
+      items = (refResolved as Record<string, unknown>) || (schemaObj.items as Record<string, unknown>);
+    } else {
+      items = schemaObj.items as Record<string, unknown>;
+    }
+  }
+
+  return {
+    type,
+    properties,
+    required,
+    description,
+    items,
+    raw: schemaObj,
+  };
+}

package/packages/normalizer/src/openapi.ts

@@ -0,0 +1,2 @@
+export * from './openapi/schema-resolver.ts';
+export * from './openapi/endpoint-parser.ts';