@apitap/core 1.6.0 → 1.6.2

package/src/replay/engine.ts

@@ -45,6 +45,15 @@ const BLOCKED_REPLAY_HEADERS = new Set([
   'sec-fetch-user',
 ]);
 
+export function safeParseJson(text: string): unknown {
+  if (text.length === 0) return text;
+  try {
+    return JSON.parse(text);
+  } catch {
+    return text;
+  }
+}
+
 export interface ReplayOptions {
   /** User-provided parameters for path, query, and body substitution */
   params?: Record<string, string>;
@@ -70,6 +79,8 @@ export interface ReplayResult {
   truncated?: boolean;
   /** Contract warnings from schema drift detection */
   contractWarnings?: ContractWarning[];
+  /** Upgrade hint: set when a low-confidence endpoint gets a 2xx response */
+  upgrade?: { confidence: 1.0; endpointProvenance: 'captured' };
 }
 
 /**
@@ -207,9 +218,15 @@ function wrapAuthError(
 /**
  * Returns a user-facing hint about confidence level, or null if confidence is high enough.
  */
-export function getConfidenceHint(confidence: number | undefined): string | null {
+export function getConfidenceHint(
+  confidence: number | undefined,
+  provenance?: string,
+): string | null {
   const c = confidence ?? 1.0;
   if (c >= 0.85) return null;
+  if (provenance === 'skeleton') {
+    return '(observed in traffic, no response shape — first successful replay will upgrade)';
+  }
   if (c >= 0.7) return '(imported from spec — params may need adjustment)';
   return '(imported from spec — provide params explicitly, no captured examples available)';
 }
@@ -548,8 +565,8 @@ export async function replayEndpoint(
       let retryData: unknown;
       const retryCt = retryResponse.headers.get('content-type') ?? '';
       const retryText = await retryResponse.text();
-      if (retryCt.includes('json') && retryText.length > 0) {
-        retryData = JSON.parse(retryText);
+      if (retryCt.includes('json')) {
+        retryData = safeParseJson(retryText);
       } else {
         retryData = retryText;
       }
@@ -558,6 +575,11 @@ export async function replayEndpoint(
         ? wrapAuthError(retryResponse.status, retryData, skill.domain)
         : retryData;
 
+      const retryUpgrade = (endpoint.confidence !== undefined && endpoint.confidence < 1.0
+        && retryResponse.status >= 200 && retryResponse.status < 300)
+        ? { confidence: 1.0 as const, endpointProvenance: 'captured' as const }
+        : undefined;
+
       if (options.maxBytes) {
         const truncated = truncateResponse(retryFinalData, { maxBytes: options.maxBytes });
         return {
@@ -566,6 +588,7 @@ export async function replayEndpoint(
           data: truncated.data,
           refreshed,
           ...(truncated.truncated ? { truncated: true } : {}),
+          ...(retryUpgrade ? { upgrade: retryUpgrade } : {}),
         };
       }
 
@@ -574,6 +597,7 @@ export async function replayEndpoint(
         headers: retryHeaders,
         data: retryFinalData,
         refreshed,
+        ...(retryUpgrade ? { upgrade: retryUpgrade } : {}),
       };
     }
   }
@@ -586,8 +610,8 @@ export async function replayEndpoint(
   let data: unknown;
   const ct = response.headers.get('content-type') ?? '';
   const text = await response.text();
-  if (ct.includes('json') && text.length > 0) {
-    data = JSON.parse(text);
+  if (ct.includes('json')) {
+    data = safeParseJson(text);
   } else {
     data = text;
   }
@@ -606,6 +630,11 @@ export async function replayEndpoint(
     }
   }
 
+  const upgrade = (endpoint.confidence !== undefined && endpoint.confidence < 1.0
+    && response.status >= 200 && response.status < 300)
+    ? { confidence: 1.0 as const, endpointProvenance: 'captured' as const }
+    : undefined;
+
   // Apply truncation if maxBytes is set
   if (options.maxBytes) {
     const truncated = truncateResponse(finalData, { maxBytes: options.maxBytes });
@@ -616,10 +645,11 @@ export async function replayEndpoint(
       ...(refreshed ? { refreshed } : {}),
       ...(truncated.truncated ? { truncated: true } : {}),
       ...(contractWarnings ? { contractWarnings } : {}),
+      ...(upgrade ? { upgrade } : {}),
     };
   }
 
-  return { status: response.status, headers: responseHeaders, data: finalData, ...(refreshed ? { refreshed } : {}), ...(contractWarnings ? { contractWarnings } : {}) };
+  return { status: response.status, headers: responseHeaders, data: finalData, ...(refreshed ? { refreshed } : {}), ...(contractWarnings ? { contractWarnings } : {}), ...(upgrade ? { upgrade } : {}) };
 }
 
 // --- Batch replay ---

package/src/skill/apis-guru.ts

@@ -1,6 +1,29 @@
 // src/skill/apis-guru.ts
 import { resolveAndValidateUrl } from '../skill/ssrf.js';
 
+const MAX_SPEC_SIZE = 10 * 1024 * 1024;   // 10 MB per spec
+const MAX_LIST_SIZE = 100 * 1024 * 1024;  // 100 MB for APIs.guru list
+
+async function fetchWithSizeLimit(url: string, maxBytes: number, options?: RequestInit): Promise<string> {
+  const response = await fetch(url, {
+    signal: AbortSignal.timeout(30_000),
+    ...options,
+    headers: { 'User-Agent': 'apitap-import/1.0', ...(options?.headers as Record<string, string> || {}) },
+  });
+  if (!response.ok) {
+    throw new Error(`HTTP ${response.status} ${response.statusText} for ${url}`);
+  }
+  const contentLength = response.headers.get('content-length');
+  if (contentLength && parseInt(contentLength, 10) > maxBytes) {
+    throw new Error(`Response too large: ${contentLength} bytes (limit: ${maxBytes})`);
+  }
+  const text = await response.text();
+  if (text.length > maxBytes) {
+    throw new Error(`Response body too large: ${text.length} bytes (limit: ${maxBytes})`);
+  }
+  return text;
+}
+
 export interface ApisGuruEntry {
   apiId: string;           // e.g., "twilio.com:api"
   providerName: string;    // e.g., "twilio.com"
@@ -121,15 +144,7 @@ export async function fetchApisGuruList(): Promise<ApisGuruEntry[]> {
   if (!ssrf.safe) {
     throw new Error(`SSRF check failed for APIs.guru list URL: ${ssrf.reason}`);
   }
-  const response = await fetch(APIS_GURU_LIST_URL, {
-    signal: AbortSignal.timeout(30_000),
-  });
-  if (!response.ok) {
-    throw new Error(
-      `Failed to fetch APIs.guru list: ${response.status} ${response.statusText}`,
-    );
-  }
-  const text = await response.text();
+  const text = await fetchWithSizeLimit(APIS_GURU_LIST_URL, MAX_LIST_SIZE);
   try {
     return parseApisGuruList(JSON.parse(text) as Record<string, any>);
   } catch {
@@ -145,16 +160,7 @@ export async function fetchSpec(specUrl: string): Promise<Record<string, any>> {
   if (!ssrf.safe) {
     throw new Error(`SSRF check failed for spec URL ${specUrl}: ${ssrf.reason}`);
   }
-  const response = await fetch(specUrl, {
-    signal: AbortSignal.timeout(30_000),
-    redirect: 'error',
-  });
-  if (!response.ok) {
-    throw new Error(
-      `Failed to fetch spec at ${specUrl}: ${response.status} ${response.statusText}`,
-    );
-  }
-  const text = await response.text();
+  const text = await fetchWithSizeLimit(specUrl, MAX_SPEC_SIZE, { redirect: 'error' });
   try {
     return JSON.parse(text) as Record<string, any>;
   } catch {

package/src/skill/generator.ts

@@ -292,6 +292,7 @@ export class SkillGenerator {
   private oauthClientSecret: string | undefined;
   private oauthRefreshToken: string | undefined;
   private totalNetworkBytes = 0; // v1.0: accumulate all response sizes
+  private _lastDedupKey: string | null = null; // side-channel for dedup key when _prepareEndpoint returns null
 
   /** Number of unique endpoints captured so far */
   get endpointCount(): number {
@@ -305,13 +306,37 @@ export class SkillGenerator {
     };
   }
 
-  /** Add a captured exchange. Returns the new endpoint if first seen, null if duplicate. */
-  addExchange(exchange: CapturedExchange): SkillEndpoint | null {
+  /**
+   * Extract shared request-side logic: URL parsing, GraphQL detection, path
+   * parameterization, dedup, OAuth, auth extraction, header/query scrubbing,
+   * and endpoint ID generation.
+   *
+   * Returns null when the endpoint is a duplicate (existing non-skeleton).
+   * When returning null, stores the dedup key in `_lastDedupKey` so callers
+   * can still perform body storage for cross-request diffing.
+   */
+  private _prepareEndpoint(request: {
+    url: string;
+    method: string;
+    headers: Record<string, string>;
+    postData?: string;
+  }): {
+    key: string;
+    method: string;
+    paramPath: string;
+    queryParams: Record<string, { type: string; example: string }>;
+    safeHeaders: Record<string, string>;
+    exampleUrl: string;
+    endpointId: string;
+    graphqlInfo: { operationName: string; query: string; variables: Record<string, unknown> | null } | null;
+    entropyDetected: Set<string>;
+    isSkeletonReplacement: boolean;
+  } | null {
     this.captureCount++;
 
-    const url = new URL(exchange.request.url);
-    const method = exchange.request.method;
-    const contentType = exchange.request.headers['content-type'] ?? '';
+    const url = new URL(request.url);
+    const method = request.method;
+    const contentType = request.headers['content-type'] ?? '';
 
     // Track baseUrl from the first captured exchange
     if (!this.baseUrl) {
@@ -319,11 +344,11 @@ export class SkillGenerator {
     }
 
     // Check for GraphQL
-    const isGraphQL = isGraphQLEndpoint(url.pathname, contentType, exchange.request.postData ?? null);
+    const isGraphQL = isGraphQLEndpoint(url.pathname, contentType, request.postData ?? null);
     let graphqlInfo: { operationName: string; query: string; variables: Record<string, unknown> | null } | null = null;
 
-    if (isGraphQL && exchange.request.postData) {
-      const parsed = parseGraphQLBody(exchange.request.postData);
+    if (isGraphQL && request.postData) {
+      const parsed = parseGraphQLBody(request.postData);
       if (parsed) {
         const opName = extractOperationName(parsed.query, parsed.operationName);
         graphqlInfo = {
@@ -344,20 +369,22 @@ export class SkillGenerator {
       ? `${method} graphql:${graphqlInfo.operationName}`
       : `${method} ${dedupPath}`;
 
-    // Track response bytes for all exchanges (for browser cost measurement)
-    this.totalNetworkBytes += exchange.response.body.length;
-
     if (this.endpoints.has(key)) {
-      // Store duplicate body for cross-request diffing (Strategy 1) — scrubbed (H3 fix)
-      if (exchange.request.postData) {
-        const bodies = this.exchangeBodies.get(key);
-        if (bodies) bodies.push(this.scrubBodyString(exchange.request.postData));
+      const existing = this.endpoints.get(key)!;
+      if (existing.endpointProvenance === 'skeleton') {
+        // Skeleton endpoint — fall through to replace it with captured data
+      } else {
+        // Non-skeleton duplicate — store key for body diffing, return null
+        this._lastDedupKey = key;
+        return null;
       }
-      return null;
     }
 
+    const isSkeletonReplacement = this.endpoints.has(key)
+      && this.endpoints.get(key)!.endpointProvenance === 'skeleton';
+
     // Detect OAuth token requests from captured traffic
-    const oauthInfo = isOAuthTokenRequest(exchange.request);
+    const oauthInfo = isOAuthTokenRequest(request);
     if (oauthInfo && !this.oauthConfig) {
       this.oauthConfig = {
         tokenEndpoint: oauthInfo.tokenEndpoint,
@@ -370,11 +397,11 @@ export class SkillGenerator {
     }
 
     // Extract auth before filtering headers (includes entropy-based detection)
-    const [auth, entropyDetected] = extractAuth(exchange.request.headers);
+    const [auth, entropyDetected] = extractAuth(request.headers);
     this.extractedAuthList.push(...auth);
 
     // Filter headers, then strip auth values (including entropy-detected tokens)
-    const filtered = filterHeaders(exchange.request.headers);
+    const filtered = filterHeaders(request.headers);
     const safeHeaders = stripAuth(filtered, entropyDetected);
 
     // Build query params, optionally scrub PII
@@ -384,11 +411,50 @@ export class SkillGenerator {
     }
 
     // Build example URL, optionally scrub PII and sensitive query params
-    let exampleUrl = exchange.request.url;
+    let exampleUrl = request.url;
     if (this.options.scrub) {
       exampleUrl = scrubUrlQueryParams(scrubPII(exampleUrl));
     }
 
+    // Generate endpoint ID - use GraphQL operation name if applicable
+    const endpointId = graphqlInfo
+      ? `${method.toLowerCase()}-graphql-${graphqlInfo.operationName}`
+      : generateEndpointId(method, paramPath);
+
+    return {
+      key,
+      method,
+      paramPath,
+      queryParams,
+      safeHeaders,
+      exampleUrl,
+      endpointId,
+      graphqlInfo,
+      entropyDetected,
+      isSkeletonReplacement,
+    };
+  }
+
+  /** Add a captured exchange. Returns the new endpoint if first seen, null if duplicate. */
+  addExchange(exchange: CapturedExchange): SkillEndpoint | null {
+    // Track response bytes for all exchanges (for browser cost measurement)
+    this.totalNetworkBytes += exchange.response.body.length;
+
+    const prep = this._prepareEndpoint(exchange.request);
+
+    if (!prep) {
+      // Dedup: store body for cross-request diffing (Strategy 1) — scrubbed (H3 fix)
+      const dedupKey = this._lastDedupKey;
+      if (dedupKey && exchange.request.postData) {
+        const bodies = this.exchangeBodies.get(dedupKey);
+        if (bodies) bodies.push(this.scrubBodyString(exchange.request.postData));
+      }
+      return null;
+    }
+
+    const { key, method, paramPath, queryParams, safeHeaders, exampleUrl,
+            endpointId, graphqlInfo, entropyDetected } = prep;
+
     // Response preview: null by default, populated with --preview
     let responsePreview: unknown = null;
     if (this.options.enablePreview) {
@@ -445,11 +511,6 @@ export class SkillGenerator {
       }
     }
 
-    // Generate endpoint ID - use GraphQL operation name if applicable
-    const endpointId = graphqlInfo
-      ? `${method.toLowerCase()}-graphql-${graphqlInfo.operationName}`
-      : generateEndpointId(method, paramPath);
-
     const endpoint: SkillEndpoint = {
       id: endpointId,
       method: exchange.request.method,
@@ -503,6 +564,52 @@ export class SkillGenerator {
     return endpoint;
   }
 
+  /** Add a skeleton endpoint (request data only, no response body). Confidence 0.8. */
+  addSkeleton(request: {
+    url: string;
+    method: string;
+    headers: Record<string, string>;
+    postData?: string;
+  }): SkillEndpoint | null {
+    const prep = this._prepareEndpoint(request);
+    if (!prep) return null;
+
+    const { key, paramPath, queryParams, safeHeaders, exampleUrl,
+            endpointId, entropyDetected, isSkeletonReplacement } = prep;
+
+    // If a skeleton already occupies this key, don't overwrite with another skeleton
+    if (isSkeletonReplacement) return null;
+
+    const endpoint: SkillEndpoint = {
+      id: endpointId,
+      method: request.method,
+      path: paramPath,
+      queryParams,
+      headers: safeHeaders,
+      responseShape: { type: 'unknown' },
+      examples: {
+        request: {
+          url: exampleUrl,
+          headers: stripAuth(filterHeaders(request.headers)),
+        },
+        responsePreview: null,
+      },
+      confidence: 0.8,
+      endpointProvenance: 'skeleton',
+      responseBytes: 0,
+    };
+
+    // Also strip entropy-detected tokens from example headers
+    if (entropyDetected.size > 0) {
+      endpoint.examples.request.headers = stripAuth(
+        filterHeaders(request.headers), entropyDetected
+      );
+    }
+
+    this.endpoints.set(key, endpoint);
+    return endpoint;
+  }
+
   /** Record a filtered-out request (for metadata tracking). */
   recordFiltered(): void {
     this.filteredCount++;

package/src/skill/merge.ts

@@ -82,6 +82,33 @@ function wouldEnrich(existing: SkillEndpoint, imported: SkillEndpoint): boolean
   return false;
 }
 
+/**
+ * Determine whether an imported endpoint would enrich an existing skeleton
+ * endpoint. Skeletons are enrichable by imports: imports can provide response
+ * shape (replacing the placeholder `{ type: 'unknown' }`), description,
+ * specSource, and query param metadata.
+ */
+function wouldEnrichSkeleton(existing: SkillEndpoint, imported: SkillEndpoint): boolean {
+  // Import can replace an unknown response shape
+  const existingShape = existing.responseShape;
+  const isUnknownShape = existingShape.type === 'unknown' && !existingShape.fields;
+  if (isUnknownShape && imported.responseShape.type !== 'unknown') return true;
+
+  if (!existing.description && imported.description) return true;
+  if (!existing.specSource && imported.specSource) return true;
+
+  // Check if any new query params would be added or enriched
+  for (const [name, specParam] of Object.entries(imported.queryParams)) {
+    if (!(name in existing.queryParams)) return true;
+    const ep = existing.queryParams[name];
+    if (!ep.enum && specParam.enum) return true;
+    if (ep.required === undefined && specParam.required !== undefined) return true;
+    if (ep.type === 'string' && specParam.type !== 'string') return true;
+  }
+
+  return false;
+}
+
 /**
  * Pure function — no I/O.
  *
@@ -182,6 +209,51 @@ export function mergeSkillFile(
       continue;
     }
 
+    // --- Skeleton branch: import can enrich skeleton endpoints ---
+    if (existingEp.endpointProvenance === 'skeleton') {
+      if (!wouldEnrichSkeleton(existingEp, importedEp)) {
+        resultEndpoints.push({
+          ...existingEp,
+          normalizedPath: normalizePath(existingEp.path),
+        });
+        const existingHasSpecData = !!(existingEp.specSource || existingEp.description);
+        const importHasSpecData = !!(importedEp.specSource || importedEp.description);
+        if (existingHasSpecData || importHasSpecData) {
+          skipped++;
+        } else {
+          preserved++;
+        }
+        continue;
+      }
+
+      const mergedQueryParams = mergeQueryParams(existingEp.queryParams, importedEp.queryParams);
+
+      // For skeletons, import can replace responseShape if existing is unknown
+      const existingShape = existingEp.responseShape;
+      const isUnknownShape = existingShape.type === 'unknown' && !existingShape.fields;
+      const responseShape = isUnknownShape && importedEp.responseShape.type !== 'unknown'
+        ? importedEp.responseShape
+        : existingShape;
+
+      const enrichedSkeleton: SkillEndpoint = {
+        ...existingEp,
+        normalizedPath: normalizePath(existingEp.path),
+        responseShape,
+        // Augment with spec fields (only if not already present)
+        ...(importedEp.description && !existingEp.description ? { description: importedEp.description } : {}),
+        ...(importedEp.specSource && !existingEp.specSource ? { specSource: importedEp.specSource } : {}),
+        // Confidence = max(skeleton, import)
+        confidence: Math.max(existingEp.confidence ?? 0, importedEp.confidence ?? 0) || existingEp.confidence,
+        // Provenance stays 'skeleton'
+        endpointProvenance: 'skeleton',
+        queryParams: mergedQueryParams,
+      };
+
+      resultEndpoints.push(enrichedSkeleton);
+      enriched++;
+      continue;
+    }
+
     // Imported endpoint matches an existing one — check if it would add anything new
     if (!wouldEnrich(existingEp, importedEp)) {
       resultEndpoints.push({

package/src/skill/openapi-converter.ts

@@ -31,6 +31,14 @@ export function resolveRef(
     return merged;
   }
 
+  // Handle oneOf/anyOf: use first option as representative
+  if (obj.oneOf && Array.isArray(obj.oneOf) && obj.oneOf.length > 0) {
+    return resolveRef(obj.oneOf[0], spec, new Set(visited), depth + 1);
+  }
+  if (obj.anyOf && Array.isArray(obj.anyOf) && obj.anyOf.length > 0) {
+    return resolveRef(obj.anyOf[0], spec, new Set(visited), depth + 1);
+  }
+
   if (!obj.$ref) return obj;
 
   const ref = obj.$ref as string;

package/src/skill/signing.ts

@@ -44,6 +44,20 @@ export function signSkillFile(skill: SkillFile, key: Buffer): SkillFile {
   };
 }
 
+/**
+ * Sign a skill file with a specific provenance value.
+ * Use 'self' for captured files, 'imported-signed' for import-only files.
+ */
+export function signSkillFileAs(
+  skill: SkillFile,
+  key: Buffer,
+  provenance: 'self' | 'imported-signed',
+): SkillFile {
+  const payload = canonicalize(skill);
+  const signature = hmacSign(payload, key);
+  return { ...skill, provenance, signature };
+}
+
 /**
  * Verify a skill file's signature.
  * Returns true if the signature is valid for the given key.

package/src/skill/store.ts

@@ -1,5 +1,5 @@
 // src/skill/store.ts
-import { readFile, writeFile, mkdir, readdir, access } from 'node:fs/promises';
+import { readFile, writeFile, mkdir, readdir, access, rename } from 'node:fs/promises';
 import { join, dirname } from 'node:path';
 import { homedir } from 'node:os';
 import type { SkillFile, SkillSummary } from '../types.js';
@@ -40,7 +40,10 @@ export async function writeSkillFile(
   await mkdir(skillsDir, { recursive: true, mode: 0o700 });
   await ensureGitignore(skillsDir);
   const filePath = skillPath(skill.domain, skillsDir);
-  await writeFile(filePath, JSON.stringify(skill, null, 2) + '\n', { mode: 0o600 });
+  const tmpPath = `${filePath}.${process.pid}.tmp`;
+  const content = JSON.stringify(skill, null, 2) + '\n';
+  await writeFile(tmpPath, content, { mode: 0o600 });
+  await rename(tmpPath, filePath);
   return filePath;
 }
 
@@ -114,8 +117,9 @@ export async function listSkillFiles(
   let files: string[];
   try {
     files = await readdir(skillsDir);
-  } catch {
-    return [];
+  } catch (err: unknown) {
+    if ((err as NodeJS.ErrnoException).code === 'ENOENT') return [];
+    throw err;
   }
 
   const summaries: SkillSummary[] = [];

package/src/types.ts

@@ -133,7 +133,7 @@ export interface SkillEndpoint {
   responseSchema?: SchemaNode; // v1.1: schema snapshot for contract validation
   normalizedPath?: string;
   confidence?: number;
-  endpointProvenance?: 'captured' | 'openapi-import';
+  endpointProvenance?: 'captured' | 'openapi-import' | 'skeleton';
   specSource?: string;
   description?: string;
 }
@@ -162,7 +162,7 @@ export interface SkillFile {
       endpointsEnriched: number;
     }>;
   };
-  provenance: 'self' | 'imported' | 'unsigned';
+  provenance: 'self' | 'imported' | 'imported-signed' | 'unsigned';
   signature?: string;
   auth?: SkillAuth; // v0.8: top-level auth config
 }
@@ -215,7 +215,7 @@ export interface SkillSummary {
   skillFile: string;
   endpointCount: number;
   capturedAt: string;
-  provenance: 'self' | 'imported' | 'unsigned';
+  provenance: 'self' | 'imported' | 'imported-signed' | 'unsigned';
 }
 
 // --- Discovery types (Milestone 2: Smart Discovery) ---