@apitap/core 1.0.0

package/src/orchestration/browse.ts

@@ -0,0 +1,250 @@
+import type { SkillFile, SkillEndpoint } from '../types.js';
+import { readSkillFile } from '../skill/store.js';
+import { replayEndpoint } from '../replay/engine.js';
+import { SessionCache } from './cache.js';
+import { read } from '../read/index.js';
+
+export interface BrowseOptions {
+  skillsDir?: string;
+  cache?: SessionCache;
+  task?: string;
+  skipDiscovery?: boolean;
+  /** Maximum response size in bytes. Default: 50000 */
+  maxBytes?: number;
+  /** @internal Skip SSRF check — for testing only */
+  _skipSsrfCheck?: boolean;
+}
+
+export interface BrowseSuccess {
+  success: true;
+  data: unknown;
+  status: number;
+  domain: string;
+  endpointId: string;
+  tier: string;
+  fromCache: boolean;
+  capturedAt: string;
+  task?: string;
+  truncated?: boolean;
+}
+
+export interface BrowseGuidance {
+  success: false;
+  reason: string;
+  discoveryConfidence?: string;
+  suggestion: string;
+  domain: string;
+  url: string;
+  task?: string;
+}
+
+export type BrowseResult = BrowseSuccess | BrowseGuidance;
+
+/**
+ * High-level browse: check cache → disk → discover → replay.
+ * Auto-escalates cheap steps. Returns guidance for expensive ones.
+ */
+export async function browse(
+  url: string,
+  options: BrowseOptions = {},
+): Promise<BrowseResult> {
+  const { cache, skillsDir, task, skipDiscovery, maxBytes = 50_000 } = options;
+  const fullUrl = url.startsWith('http') ? url : `https://${url}`;
+
+  let domain: string;
+  let urlPath: string;
+  try {
+    const parsed = new URL(fullUrl);
+    domain = parsed.hostname;
+    urlPath = parsed.pathname;
+  } catch {
+    return {
+      success: false,
+      reason: 'invalid_url',
+      suggestion: 'provide_valid_url',
+      domain: '',
+      url: fullUrl,
+      task,
+    };
+  }
+
+  // Step 1: Check session cache
+  let skill: SkillFile | null = null;
+  let source: 'disk' | 'discovered' | 'captured' = 'disk';
+
+  if (cache?.has(domain)) {
+    skill = cache.get(domain)!.skillFile;
+    source = cache.get(domain)!.source;
+  }
+
+  // Step 2: Check disk
+  if (!skill) {
+    skill = await readSkillFile(domain, skillsDir);
+    if (skill) {
+      source = 'disk';
+      cache?.set(domain, skill, 'disk');
+    }
+  }
+
+  // Step 3: Try discovery
+  if (!skill && !skipDiscovery) {
+    try {
+      const { discover } = await import('../discovery/index.js');
+      const discovery = await discover(fullUrl);
+
+      if (discovery.skillFile && discovery.skillFile.endpoints.length > 0 &&
+          (discovery.confidence === 'high' || discovery.confidence === 'medium')) {
+        skill = discovery.skillFile;
+        source = 'discovered';
+
+        // Save to disk
+        const { writeSkillFile: writeSF } = await import('../skill/store.js');
+        await writeSF(skill, skillsDir);
+        cache?.set(domain, skill, 'discovered');
+      } else {
+        // Discovery didn't produce usable endpoints — try text-mode read
+        try {
+          const readResult = await read(fullUrl, { maxBytes });
+          if (readResult && readResult.content.trim().length > 0 && readResult.metadata.source !== 'spa-shell') {
+            return {
+              success: true,
+              data: readResult,
+              status: 200,
+              domain,
+              endpointId: 'read',
+              tier: 'green',
+              fromCache: false,
+              capturedAt: new Date().toISOString(),
+              task,
+            };
+          }
+        } catch {
+          // Read failed — fall through to capture_needed
+        }
+        return {
+          success: false,
+          reason: 'no_replayable_endpoints',
+          discoveryConfidence: discovery.confidence,
+          suggestion: 'capture_needed',
+          domain,
+          url: fullUrl,
+          task,
+        };
+      }
+    } catch {
+      // Discovery failed — fall through to guidance
+    }
+  }
+
+  // No skill file at all — try text-mode read before giving up
+  if (!skill) {
+    if (!skipDiscovery) {
+      try {
+        const readResult = await read(fullUrl, { maxBytes });
+        if (readResult && readResult.content.trim().length > 0 && readResult.metadata.source !== 'spa-shell') {
+          return {
+            success: true,
+            data: readResult,
+            status: 200,
+            domain,
+            endpointId: 'read',
+            tier: 'green',
+            fromCache: false,
+            capturedAt: new Date().toISOString(),
+            task,
+          };
+        }
+      } catch {
+        // Read failed — fall through to capture_needed
+      }
+    }
+    return {
+      success: false,
+      reason: 'no_skill_file',
+      suggestion: 'capture_needed',
+      domain,
+      url: fullUrl,
+      task,
+    };
+  }
+
+  // Step 4: Pick best endpoint
+  const endpoint = pickEndpoint(skill, urlPath);
+  if (!endpoint) {
+    return {
+      success: false,
+      reason: 'no_replayable_endpoints',
+      suggestion: 'capture_needed',
+      domain,
+      url: fullUrl,
+      task,
+    };
+  }
+
+  // Step 5: Replay
+  try {
+    const result = await replayEndpoint(skill, endpoint.id, { maxBytes, _skipSsrfCheck: options._skipSsrfCheck });
+    const fromCache = source === 'disk';
+
+    // Check content-type: HTML responses are not usable API data
+    const contentType = result.headers['content-type'] ?? '';
+    if (contentType.includes('text/html')) {
+      return {
+        success: false,
+        reason: 'non_api_response',
+        discoveryConfidence: source === 'discovered' ? 'medium' : undefined,
+        suggestion: 'capture_needed',
+        domain,
+        url: fullUrl,
+        task,
+      };
+    }
+
+    return {
+      success: true,
+      data: result.data,
+      status: result.status,
+      domain,
+      endpointId: endpoint.id,
+      tier: endpoint.replayability?.tier ?? 'unknown',
+      fromCache,
+      capturedAt: skill.capturedAt,
+      task,
+      ...(result.truncated ? { truncated: true } : {}),
+    };
+  } catch {
+    return {
+      success: false,
+      reason: 'replay_failed',
+      suggestion: 'capture_needed',
+      domain,
+      url: fullUrl,
+      task,
+    };
+  }
+}
+
+const REPLAYABLE_TIERS = new Set(['green', 'yellow', 'unknown']);
+
+/**
+ * Pick the best endpoint to replay. Prefers:
+ * 1. GET endpoints with green/yellow/unknown tier
+ * 2. Path overlap with the input URL
+ * 3. First match as fallback
+ */
+function pickEndpoint(skill: SkillFile, urlPath: string): SkillEndpoint | null {
+  const candidates = skill.endpoints.filter(ep =>
+    ep.method === 'GET' &&
+    REPLAYABLE_TIERS.has(ep.replayability?.tier ?? 'unknown'),
+  );
+
+  if (candidates.length === 0) return null;
+
+  // Prefer path overlap
+  if (urlPath && urlPath !== '/') {
+    const match = candidates.find(ep => urlPath.includes(ep.path) || ep.path.includes(urlPath));
+    if (match) return match;
+  }
+
+  return candidates[0];
+}

package/src/orchestration/cache.ts

@@ -0,0 +1,37 @@
+import type { SkillFile } from '../types.js';
+
+export interface CacheEntry {
+  domain: string;
+  skillFile: SkillFile;
+  discoveredAt: number;
+  source: 'disk' | 'discovered' | 'captured';
+}
+
+export class SessionCache {
+  private entries = new Map<string, CacheEntry>();
+
+  set(domain: string, skillFile: SkillFile, source: CacheEntry['source']): void {
+    this.entries.set(domain, {
+      domain,
+      skillFile,
+      discoveredAt: Date.now(),
+      source,
+    });
+  }
+
+  get(domain: string): CacheEntry | null {
+    return this.entries.get(domain) ?? null;
+  }
+
+  has(domain: string): boolean {
+    return this.entries.has(domain);
+  }
+
+  invalidate(domain: string): void {
+    this.entries.delete(domain);
+  }
+
+  domains(): string[] {
+    return [...this.entries.keys()];
+  }
+}

package/src/plugin.ts

@@ -0,0 +1,188 @@
+// src/plugin.ts
+import { searchSkills } from './skill/search.js';
+import { readSkillFile } from './skill/store.js';
+import { replayEndpoint } from './replay/engine.js';
+import { AuthManager, getMachineId } from './auth/manager.js';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+
+export interface ToolDefinition {
+  name: string;
+  description: string;
+  parameters: Record<string, unknown>;
+  execute: (args: Record<string, unknown>) => Promise<unknown>;
+}
+
+export interface Plugin {
+  name: string;
+  version: string;
+  tools: ToolDefinition[];
+}
+
+export interface PluginOptions {
+  skillsDir?: string;
+  /** @internal Skip SSRF check — for testing only */
+  _skipSsrfCheck?: boolean;
+}
+
+const APITAP_DIR = join(homedir(), '.apitap');
+
+export function createPlugin(options: PluginOptions = {}): Plugin {
+  const skillsDir = options.skillsDir;
+
+  const searchTool: ToolDefinition = {
+    name: 'apitap_search',
+    description:
+      'Search available API skill files for a domain or endpoint. ' +
+      'Use this FIRST to check if ApiTap has captured a site\'s API before trying to replay. ' +
+      'Returns matching endpoints with replayability tiers: ' +
+      'green = safe to replay directly, ' +
+      'yellow = needs auth credentials, ' +
+      'orange = fragile (CSRF/session-bound), ' +
+      'red = needs browser (anti-bot). ' +
+      'If not found, use apitap_capture to capture the site first.',
+    parameters: {
+      type: 'object',
+      properties: {
+        query: {
+          type: 'string',
+          description: 'Search query — domain name, endpoint path, or keyword (e.g. "polymarket", "events", "get-markets")',
+        },
+      },
+      required: ['query'],
+    },
+    execute: async (args) => {
+      const query = args.query as string;
+      return searchSkills(query, skillsDir);
+    },
+  };
+
+  const replayTool: ToolDefinition = {
+    name: 'apitap_replay',
+    description:
+      'Replay a captured API endpoint to get live data. ' +
+      'Check the endpoint tier first with apitap_search: ' +
+      'green = will work, yellow = needs auth, orange/red = may fail. ' +
+      'Returns { status, data } with the API response.',
+    parameters: {
+      type: 'object',
+      properties: {
+        domain: {
+          type: 'string',
+          description: 'Domain of the API (e.g. "gamma-api.polymarket.com")',
+        },
+        endpointId: {
+          type: 'string',
+          description: 'Endpoint ID from search results (e.g. "get-events")',
+        },
+        params: {
+          type: 'object',
+          description: 'Optional key-value parameters for path substitution or query params (e.g. { "id": "123", "limit": "10" })',
+          additionalProperties: { type: 'string' },
+        },
+      },
+      required: ['domain', 'endpointId'],
+    },
+    execute: async (args) => {
+      const domain = args.domain as string;
+      const endpointId = args.endpointId as string;
+      const params = args.params as Record<string, string> | undefined;
+
+      const skill = await readSkillFile(domain, skillsDir);
+      if (!skill) {
+        return {
+          error: `No skill file found for "${domain}". Use apitap_capture to capture it first.`,
+        };
+      }
+
+      // Inject stored auth if available
+      const endpoint = skill.endpoints.find(e => e.id === endpointId);
+      if (!endpoint) {
+        return {
+          error: `Endpoint "${endpointId}" not found. Available: ${skill.endpoints.map(e => e.id).join(', ')}`,
+        };
+      }
+
+      const hasStoredPlaceholder = Object.values(endpoint.headers).some(v => v === '[stored]');
+      if (hasStoredPlaceholder) {
+        try {
+          const machineId = await getMachineId();
+          const authManager = new AuthManager(APITAP_DIR, machineId);
+          const storedAuth = await authManager.retrieve(domain);
+          if (storedAuth) {
+            endpoint.headers[storedAuth.header] = storedAuth.value;
+          }
+        } catch {
+          // Auth retrieval failed — proceed without it
+        }
+      }
+
+      try {
+        const result = await replayEndpoint(skill, endpointId, {
+          params,
+          _skipSsrfCheck: options._skipSsrfCheck,
+        });
+        return { status: result.status, data: result.data };
+      } catch (err: any) {
+        return { error: err.message };
+      }
+    },
+  };
+
+  const captureTool: ToolDefinition = {
+    name: 'apitap_capture',
+    description:
+      'Capture a website\'s API traffic by browsing it with an instrumented browser. ' +
+      'Use this when apitap_search returns no results for a site. ' +
+      'Launches a browser, navigates to the URL, captures API calls for the specified duration, ' +
+      'and generates a skill file for future replay. ' +
+      'Returns { domains, endpoints, skillFiles } summary.',
+    parameters: {
+      type: 'object',
+      properties: {
+        url: {
+          type: 'string',
+          description: 'URL to capture (e.g. "https://polymarket.com")',
+        },
+        duration: {
+          type: 'number',
+          description: 'Capture duration in seconds (default: 30)',
+        },
+        allDomains: {
+          type: 'boolean',
+          description: 'Capture all domains, not just the target domain (default: false)',
+        },
+      },
+      required: ['url'],
+    },
+    execute: async (args) => {
+      const url = args.url as string;
+      const duration = (args.duration as number) ?? 30;
+      const allDomains = (args.allDomains as boolean) ?? false;
+
+      // Shell out to CLI for capture (it handles browser lifecycle, signing, etc.)
+      const { execFile } = await import('node:child_process');
+      const { promisify } = await import('node:util');
+      const execFileAsync = promisify(execFile);
+
+      const cliArgs = ['--import', 'tsx', 'src/cli.ts', 'capture', url, '--duration', String(duration), '--json', '--no-verify'];
+      if (allDomains) cliArgs.push('--all-domains');
+
+      try {
+        const { stdout } = await execFileAsync('node', cliArgs, {
+          timeout: (duration + 30) * 1000,
+          env: { ...process.env, ...(skillsDir ? { APITAP_SKILLS_DIR: skillsDir } : {}) },
+        });
+        return JSON.parse(stdout);
+      } catch (err: any) {
+        return { error: `Capture failed: ${err.message}` };
+      }
+    },
+  };
+
+  return {
+    name: 'apitap',
+    version: '0.4.0',
+    tools: [searchTool, replayTool, captureTool],
+  };
+}

package/src/read/decoders/deepwiki.ts

@@ -0,0 +1,180 @@
+// src/read/decoders/deepwiki.ts
+import type { Decoder, ReadResult } from '../types.js';
+
+function estimateTokens(text: string): number {
+  return Math.ceil(text.length / 4);
+}
+
+/**
+ * DeepWiki decoder — extracts wiki content from deepwiki.com
+ * 
+ * DeepWiki (by Devin/Cognition) auto-generates documentation wikis from GitHub repos.
+ * It's a Next.js app that serves content via React Server Components (RSC).
+ * 
+ * Trick: Send `RSC: 1` header → get full markdown content in the RSC payload
+ * instead of the JS-heavy SPA shell. No auth required.
+ * 
+ * URL patterns:
+ *   deepwiki.com/{org}/{repo}              → overview page
+ *   deepwiki.com/{org}/{repo}/{page-slug}  → specific wiki page
+ */
+
+const DEEPWIKI_PATTERN = /^https?:\/\/(www\.)?deepwiki\.com\/([^/]+)\/([^/]+)(\/.*)?$/;
+
+export const deepwikiDecoder: Decoder = {
+  name: 'deepwiki',
+  patterns: [
+    /^https?:\/\/(www\.)?deepwiki\.com\/[^/]+\/[^/]+/,
+  ],
+
+  async decode(url: string, options: { skipSsrf?: boolean; [key: string]: any } = {}): Promise<ReadResult | null> {
+    const match = url.match(DEEPWIKI_PATTERN);
+    if (!match) return null;
+
+    const org = match[2];
+    const repo = match[3];
+    const pagePath = match[4] || '';
+
+    // Construct the path for the RSC request
+    const fullPath = `/${org}/${repo}${pagePath}`;
+
+    try {
+      const response = await fetch(url, {
+        headers: {
+          'RSC': '1',
+          'Next-Url': fullPath,
+          'User-Agent': 'Mozilla/5.0 (compatible; ApiTap/1.0)',
+        },
+        signal: AbortSignal.timeout(10_000),
+      });
+
+      if (!response.ok) {
+        return null;
+      }
+
+      const rscPayload = await response.text();
+
+      // Extract markdown content from RSC text nodes
+      // Format: {id}:T{hexLength},{content}
+      const contentBlocks: string[] = [];
+      const lines = rscPayload.split('\n');
+
+      let currentBlock: string | null = null;
+      let expectedLength = 0;
+      let collectedBytes = 0;
+
+      for (const line of lines) {
+        // Start of a new text block: {id}:T{hexLength},{content...}
+        const blockMatch = line.match(/^[0-9a-f]+:T([0-9a-f]+),(.*)$/);
+
+        if (blockMatch) {
+          // Save previous block if exists
+          if (currentBlock !== null) {
+            contentBlocks.push(currentBlock);
+          }
+
+          expectedLength = parseInt(blockMatch[1], 16);
+          const content = blockMatch[2];
+          currentBlock = content;
+          collectedBytes = Buffer.byteLength(content, 'utf-8');
+          continue;
+        }
+
+        // If we're inside a block, keep collecting lines
+        if (currentBlock !== null) {
+          // Check if this line starts a new RSC record (not a continuation)
+          if (/^[0-9a-f]+:[^T]/.test(line) || /^[0-9a-f]+:T[0-9a-f]+,/.test(line)) {
+            // End of current block
+            contentBlocks.push(currentBlock);
+            currentBlock = null;
+
+            // If it's a new T block, process it
+            const newBlock = line.match(/^[0-9a-f]+:T([0-9a-f]+),(.*)$/);
+            if (newBlock) {
+              expectedLength = parseInt(newBlock[1], 16);
+              currentBlock = newBlock[2];
+              collectedBytes = Buffer.byteLength(newBlock[2], 'utf-8');
+            }
+            continue;
+          }
+
+          currentBlock += '\n' + line;
+          collectedBytes += Buffer.byteLength('\n' + line, 'utf-8');
+
+          // If we've collected enough bytes, end the block
+          if (collectedBytes >= expectedLength) {
+            contentBlocks.push(currentBlock);
+            currentBlock = null;
+          }
+        }
+      }
+
+      // Don't forget the last block
+      if (currentBlock !== null) {
+        contentBlocks.push(currentBlock);
+      }
+
+      if (contentBlocks.length === 0) {
+        return null;
+      }
+
+      // Find the largest content block — that's the main page content
+      // (smaller blocks might be TOC section titles)
+      const mainContent = contentBlocks.reduce((a, b) =>
+        a.length > b.length ? a : b
+      );
+
+      if (!mainContent || mainContent.length < 50) {
+        return null;
+      }
+
+      // Clean up the markdown
+      let content = mainContent;
+
+      // Extract title from first heading
+      const titleMatch = content.match(/^#\s+(.+)$/m);
+      const title = titleMatch
+        ? `${titleMatch[1]} — ${org}/${repo} | DeepWiki`
+        : `${org}/${repo} | DeepWiki`;
+
+      // Fix relative links to point back to correct locations
+      content = content.replace(
+        /\[([^\]]+)\]\((?!https?:\/\/)([^)]+)\)/g,
+        (full, text, href) => {
+          // Source file links (e.g., README.md, src/foo.ts)
+          if (href.match(/\.(ts|js|md|json|tsx|jsx|py|rs|go|toml|yaml|yml|css|html)$/)) {
+            return `[${text}](https://github.com/${org}/${repo}/blob/main/${href})`;
+          }
+          // Section links (#2, #3.1, etc.)
+          if (href.startsWith('#')) {
+            return `[${text}](https://deepwiki.com/${org}/${repo}/${href.slice(1)})`;
+          }
+          // Other relative links
+          return `[${text}](https://deepwiki.com/${org}/${repo}/${href})`;
+        }
+      );
+
+      const tokens = estimateTokens(content);
+
+      return {
+        url,
+        title,
+        author: null,
+        description: `DeepWiki documentation for ${org}/${repo}`,
+        content,
+        links: [],
+        images: [],
+        metadata: {
+          type: 'wiki',
+          publishedAt: null,
+          source: 'deepwiki-rsc',
+          canonical: `https://deepwiki.com/${org}/${repo}${pagePath}`,
+          siteName: 'DeepWiki',
+        },
+        cost: { tokens },
+      };
+    } catch {
+      return null;
+    }
+  },
+};