@apitap/core 1.4.1 → 1.4.3

package/src/cli.ts

@@ -17,11 +17,13 @@ import { buildInspectReport, formatInspectHuman } from './inspect/report.js';
 import { generateStatsReport, formatStatsHuman } from './stats/report.js';
 import { detectAntiBot, type AntiBotSignal } from './capture/anti-bot.js';
 import { discover } from './discovery/index.js';
+import { readIndexEntry } from './index/reader.js';
 import { peek } from './read/peek.js';
 import { read } from './read/index.js';
 import { homedir } from 'node:os';
 import { join, resolve, dirname } from 'node:path';
 import { readFileSync } from 'node:fs';
+import { stat, unlink } from 'node:fs/promises';
 import { fileURLToPath } from 'node:url';
 import { createMcpServer } from './mcp.js';
 
@@ -79,6 +81,8 @@ function printUsage(): void {
     apitap browse <url>        Browse a URL (discover + replay in one step)
     apitap peek <url>          Zero-cost triage (HEAD only)
     apitap read <url>          Extract content without a browser
+    apitap audit               Audit stored skill files and credentials
+    apitap forget <domain>     Remove skill file and credentials for a domain
     apitap stats               Show token savings report
     apitap extension install   Register native messaging host for Chrome
 
@@ -847,8 +851,17 @@ async function handleDiscover(positional: string[], flags: Record<string, string
 
   const result = await discover(url);
 
+  // Look up passive index data for this domain
+  let domain: string;
+  try {
+    domain = new URL(fullDiscoverUrl).hostname;
+  } catch {
+    domain = url;
+  }
+  const indexEntry = await readIndexEntry(domain);
+
   if (json) {
-    console.log(JSON.stringify(result, null, 2));
+    console.log(JSON.stringify({ ...result, indexEntry: indexEntry ?? undefined }, null, 2));
   } else {
     // Confidence summary
     const confidenceLabels: Record<string, string> = {
@@ -895,6 +908,24 @@ async function handleDiscover(positional: string[], flags: Record<string, string
       console.log(`\n  Skill file: ${result.skillFile.endpoints.length} endpoints predicted`);
     }
 
+    if (indexEntry) {
+      console.log(`\n  Passive Index (from browser extension):`);
+      console.log(`    Domain:     ${indexEntry.domain}`);
+      console.log(`    Total hits: ${indexEntry.totalHits}`);
+      console.log(`    Endpoints:  ${indexEntry.endpoints.length}`);
+      console.log(`    Promoted:   ${indexEntry.promoted ? 'yes' : 'no'}`);
+      if (indexEntry.endpoints.length > 0) {
+        console.log(`    Observed endpoints:`);
+        for (const ep of indexEntry.endpoints) {
+          const methods = ep.methods.join(', ');
+          const auth = ep.authType ? ` [${ep.authType}]` : '';
+          const pagination = ep.pagination ? ` (${ep.pagination})` : '';
+          const gql = ep.type === 'graphql' ? ' [GraphQL]' : '';
+          console.log(`      ${methods} ${ep.path} \u2014 ${ep.hits} hits${auth}${pagination}${gql}`);
+        }
+      }
+    }
+
     if (result.confidence === 'none') {
       console.log(`\n  Recommendation: Run \`apitap capture ${url}\` for browser-based discovery`);
     }
@@ -1050,6 +1081,112 @@ async function handleRead(positional: string[], flags: Record<string, string | b
   console.log();
 }
 
+async function handleAudit(flags: Record<string, string | boolean>): Promise<void> {
+  const skillsDir = SKILLS_DIR || join(APITAP_DIR, 'skills');
+  const summaries = await listSkillFiles(skillsDir);
+  const machineId = await getEffectiveMachineId();
+  const authManager = new AuthManager(APITAP_DIR, machineId);
+  const authDomains = new Set(await authManager.listDomains());
+
+  // Get skill file last-modified dates
+  const rows: { domain: string; endpoints: number; auth: string; modified: string }[] = [];
+  for (const s of summaries) {
+    let modified = 'unknown';
+    try {
+      const st = await stat(s.skillFile);
+      modified = st.mtime.toISOString().slice(0, 10);
+    } catch {}
+    rows.push({
+      domain: s.domain,
+      endpoints: s.endpointCount,
+      auth: authDomains.has(s.domain) ? 'yes' : 'no',
+      modified,
+    });
+  }
+
+  // Get auth.enc last-modified
+  let authModified = 'none';
+  try {
+    const authStat = await stat(join(APITAP_DIR, 'auth.enc'));
+    authModified = authStat.mtime.toISOString().slice(0, 10);
+  } catch {}
+
+  if (flags.json === true) {
+    console.log(JSON.stringify({ domains: rows, authFileModified: authModified }, null, 2));
+    return;
+  }
+
+  if (rows.length === 0) {
+    console.log('\n  No skill files found.\n');
+    return;
+  }
+
+  // Table header
+  const domW = Math.max(6, ...rows.map(r => r.domain.length));
+  console.log();
+  console.log(`  ${'DOMAIN'.padEnd(domW)}  ${'ENDPOINTS'.padStart(9)}  ${'AUTH'.padEnd(4)}  MODIFIED`);
+  console.log(`  ${''.padEnd(domW, '-')}  ${''.padEnd(9, '-')}  ${''.padEnd(4, '-')}  ${''.padEnd(10, '-')}`);
+  for (const r of rows) {
+    console.log(`  ${r.domain.padEnd(domW)}  ${String(r.endpoints).padStart(9)}  ${r.auth.padEnd(4)}  ${r.modified}`);
+  }
+  console.log();
+  console.log(`  auth.enc last modified: ${authModified}`);
+  console.log();
+}
+
+async function handleForget(positional: string[]): Promise<void> {
+  const domain = positional[0];
+  if (!domain) {
+    console.error('Error: Domain required. Usage: apitap forget <domain>');
+    process.exit(1);
+  }
+
+  if (!/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/.test(domain) || domain.includes('..')) {
+    console.error('Error: Invalid domain name');
+    process.exit(1);
+  }
+
+  const skillsDir = SKILLS_DIR || join(APITAP_DIR, 'skills');
+  let skillRemoved = false;
+  let authRemoved = false;
+  let notFound = true;
+
+  // Remove skill file
+  const skillFilePath = join(skillsDir, `${domain}.json`);
+  try {
+    await stat(skillFilePath);
+    notFound = false;
+    await unlink(skillFilePath);
+    skillRemoved = true;
+  } catch {}
+
+  // Remove stored auth
+  try {
+    const machineId = await getEffectiveMachineId();
+    const authManager = new AuthManager(APITAP_DIR, machineId);
+    const hasAuth = await authManager.has(domain);
+    if (hasAuth) {
+      notFound = false;
+      await authManager.clear(domain);
+      authRemoved = true;
+    }
+  } catch (err: any) {
+    if (skillRemoved) {
+      console.error(`Warning: Could not remove auth credentials: ${err.message}`);
+    }
+  }
+
+  if (notFound) {
+    console.log(`${domain} not found`);
+    return;
+  }
+
+  const parts: string[] = [];
+  if (skillRemoved) parts.push('skill file');
+  if (authRemoved) parts.push('credentials');
+  console.log(`Forgot ${domain} — ${parts.join(' and ')} removed`);
+}
+
 async function handleExtension(positional: string[], flags: Record<string, string | boolean>): Promise<void> {
   const subcommand = positional[0];
 
@@ -1150,6 +1287,12 @@ async function main(): Promise<void> {
     case 'read':
       await handleRead(positional, flags);
       break;
+    case 'audit':
+      await handleAudit(flags);
+      break;
+    case 'forget':
+      await handleForget(positional);
+      break;
     case 'extension':
       await handleExtension(positional, flags);
       break;

package/src/index/reader.ts

@@ -0,0 +1,65 @@
+import { promises as fs } from 'node:fs';
+import path from 'node:path';
+import os from 'node:os';
+
+// Types re-declared here (extension types can't be imported due to different tsconfig).
+// Keep in sync with extension/src/types.ts IndexFile/IndexEntry/IndexEndpoint.
+export interface IndexFile {
+  v: 1;
+  updatedAt: string;
+  entries: IndexEntry[];
+}
+
+export interface IndexEntry {
+  domain: string;
+  firstSeen: string;
+  lastSeen: string;
+  totalHits: number;
+  promoted: boolean;
+  lastPromoted?: string;
+  skillFileSource?: 'extension' | 'cli';
+  endpoints: IndexEndpoint[];
+}
+
+export interface IndexEndpoint {
+  path: string;
+  methods: string[];
+  authType?: string;
+  hasBody: boolean;
+  hits: number;
+  lastSeen: string;
+  pagination?: string;
+  type?: 'graphql';
+  queryParamNames?: string[];
+}
+
+const DEFAULT_APITAP_DIR = path.join(os.homedir(), '.apitap');
+
+/**
+ * Read the full passive index from disk.
+ * Returns null if index.json doesn't exist or is invalid.
+ */
+export async function readIndex(apitapDir: string = DEFAULT_APITAP_DIR): Promise<IndexFile | null> {
+  const indexPath = path.join(apitapDir, 'index.json');
+  try {
+    const raw = await fs.readFile(indexPath, 'utf-8');
+    const parsed = JSON.parse(raw);
+    if (parsed.v !== 1 || !Array.isArray(parsed.entries)) return null;
+    return parsed as IndexFile;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * Read a single domain's index entry.
+ * Returns null if the domain is not in the index.
+ */
+export async function readIndexEntry(
+  domain: string,
+  apitapDir: string = DEFAULT_APITAP_DIR,
+): Promise<IndexEntry | null> {
+  const index = await readIndex(apitapDir);
+  if (!index) return null;
+  return index.entries.find(e => e.domain === domain) ?? null;
+}

package/src/mcp.ts

@@ -11,6 +11,7 @@ import { deriveSigningKey } from './auth/crypto.js';
 import { requestAuth } from './auth/handoff.js';
 import { CaptureSession } from './capture/session.js';
 import { discover } from './discovery/index.js';
+import { readIndexEntry } from './index/reader.js';
 import { SessionCache } from './orchestration/cache.js';
 import { peek } from './read/peek.js';
 import { read } from './read/index.js';
@@ -139,6 +140,11 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
         }
         const result = await discover(url);
 
+        // Look up passive index data for this domain
+        let domain: string;
+        try { domain = new URL(url).hostname; } catch { domain = url; }
+        const indexEntry = await readIndexEntry(domain);
+
         // If we got a skill file, sign and save it automatically
         if (result.skillFile && (result.confidence === 'high' || result.confidence === 'medium')) {
           const { writeSkillFile } = await import('./skill/store.js');
@@ -147,10 +153,10 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
           const sigKey = deriveSigningKey(machineId);
           result.skillFile = signSkillFile(result.skillFile, sigKey);
           const path = await writeSkillFile(result.skillFile, skillsDir);
-          return wrapExternalContent({ ...result, savedTo: path }, 'apitap_discover');
+          return wrapExternalContent({ ...result, savedTo: path, indexEntry: indexEntry ?? undefined }, 'apitap_discover');
         }
 
-        return wrapExternalContent(result, 'apitap_discover');
+        return wrapExternalContent({ ...result, indexEntry: indexEntry ?? undefined }, 'apitap_discover');
       } catch (err: any) {
         return {
           content: [{ type: 'text' as const, text: `Discovery failed: ${err.message}` }],
@@ -186,7 +192,7 @@ export function createMcpServer(options: McpServerOptions = {}): McpServer {
       }
       const machineId = await getMachineId();
       const signingKey = deriveSigningKey(machineId);
-      const skill = await readSkillFile(domain, skillsDir, { verifySignature: true, signingKey });
+      const skill = await readSkillFile(domain, skillsDir, { verifySignature: true, signingKey, trustUnsigned: true });
       if (!skill) {
         return {
           content: [{ type: 'text' as const, text: `No skill file found for "${domain}". Use apitap_capture to capture it first.` }],

package/src/native-host.ts

@@ -27,10 +27,11 @@ async function signSkillJson(skillJson: string): Promise<string> {
 }
 
 export interface NativeRequest {
-  action: 'save_skill' | 'save_batch' | 'ping' | 'capture_request';
+  action: 'save_skill' | 'save_batch' | 'ping' | 'capture_request' | 'save_index';
   domain?: string;
   skillJson?: string;
   skills?: Array<{ domain: string; skillJson: string }>;
+  indexJson?: string;
 }
 
 export interface NativeResponse {
@@ -115,6 +116,32 @@ export async function handleNativeMessage(
       return { success: true, paths };
     }
 
+    if (request.action === 'save_index') {
+      if (!request.indexJson) {
+        return { success: false, error: 'Missing indexJson' };
+      }
+      if (request.indexJson.length > 5 * 1024 * 1024) {
+        return { success: false, error: 'Index too large (max 5MB)' };
+      }
+      try {
+        JSON.parse(request.indexJson);
+      } catch {
+        return { success: false, error: 'Invalid JSON in indexJson' };
+      }
+
+      // index.json lives in ~/.apitap/ (parent of skills dir)
+      const apitapDir = path.dirname(skillsDir);
+      await fs.mkdir(apitapDir, { recursive: true });
+      const indexPath = path.join(apitapDir, 'index.json');
+
+      // Atomic write: temp file + rename
+      const tmpPath = indexPath + '.tmp.' + process.pid;
+      await fs.writeFile(tmpPath, request.indexJson, { mode: 0o600 });
+      await fs.rename(tmpPath, indexPath);
+
+      return { success: true, path: indexPath };
+    }
+
     return { success: false, error: `Unknown action: ${request.action}` };
   } catch (err) {
     return { success: false, error: String(err) };
@@ -124,7 +151,7 @@ export async function handleNativeMessage(
 // --- Relay handler ---
 
 // Actions handled locally by the native host (filesystem operations)
-const LOCAL_ACTIONS = new Set(['save_skill', 'save_batch', 'ping']);
+const LOCAL_ACTIONS = new Set(['save_skill', 'save_batch', 'ping', 'save_index']);
 
 // Actions relayed to the extension (browser operations)
 const EXTENSION_ACTIONS = new Set(['capture_request']);

package/src/orchestration/browse.ts

@@ -315,6 +315,8 @@ export async function browse(
     // Check content-type: HTML responses are not usable API data
     const contentType = result.headers['content-type'] ?? '';
     if (contentType.includes('text/html')) {
+      // Invalidate stale cache so next call reads fresh skill file from disk
+      cache?.invalidate(domain);
       return {
         success: false,
         reason: 'non_api_response',

package/src/read/peek.ts

@@ -48,6 +48,22 @@ export async function peek(url: string, options: PeekOptions = {}): Promise<Peek
   const contentType = headers['content-type'] || null;
   const server = headers['server'] || null;
 
+  // JSON API responses with HTTP 200 are always accessible, regardless of CDN headers
+  if (status === 200 && contentType && contentType.includes('application/json')) {
+    const framework = detectFramework(headers, signals);
+    return {
+      url,
+      status,
+      accessible: true,
+      contentType,
+      server,
+      framework,
+      botProtection: null,
+      signals,
+      recommendation: 'read',
+    };
+  }
+
   // Detect bot protection
   const botProtection = detectBotProtection(headers, signals);
 

package/src/replay/engine.ts

@@ -628,7 +628,7 @@ export async function replayMultiple(
   const skillCache = new Map<string, SkillFile | null>();
   const uniqueDomains = [...new Set(requests.map(r => r.domain))];
   await Promise.all(uniqueDomains.map(async (domain) => {
-    const skill = await readSkillFile(domain, options.skillsDir, { verifySignature: true, signingKey });
+    const skill = await readSkillFile(domain, options.skillsDir, { verifySignature: true, signingKey, trustUnsigned: true });
     skillCache.set(domain, skill);
   }));
 

package/src/skill/store.ts

@@ -85,7 +85,17 @@ export async function readSkillFile(
         }
       } else {
         const { verifySignature } = await import('./signing.js');
-        if (!verifySignature(skill, signingKey)) {
+        let verified = verifySignature(skill, signingKey);
+        if (!verified) {
+          // Fallback: try pre-v1.4.0 legacy key (before HKDF signing key separation)
+          // Files signed with deriveKey() directly (not deriveSigningKey()) will match here
+          const { deriveKey } = await import('../auth/crypto.js');
+          const { getMachineId } = await import('../auth/manager.js');
+          const machineId = await getMachineId();
+          const legacyKey = deriveKey(machineId);
+          verified = verifySignature(skill, legacyKey);
+        }
+        if (!verified) {
           throw new Error(`Skill file signature verification failed for ${domain} — file may be tampered`);
         }
       }