@apitap/core 1.0.0

package/src/stats/report.ts

@@ -0,0 +1,208 @@
+// src/stats/report.ts
+import { readdir, readFile, stat } from 'node:fs/promises';
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+import type { SkillFile } from '../types.js';
+
+const DEFAULT_SKILLS_DIR = join(homedir(), '.apitap', 'skills');
+
+export interface DomainStats {
+  domain: string;
+  endpoints: number;
+  replayable: number;
+  domBytes: number;
+  totalNetworkBytes: number;
+  skillFileBytes: number;
+  totalResponseBytes: number;
+  browserTokens: number;
+  replayTokens: number;
+  savingsPercent: number;
+}
+
+export interface StatsReport {
+  domains: DomainStats[];
+  totals: {
+    domains: number;
+    endpoints: number;
+    replayable: number;
+    totalDomBytes: number;
+    browserTokens: number;
+    replayTokens: number;
+    savingsPercent: number;
+  };
+}
+
+/** Default browser token estimate when no measurement available (conservative). */
+const DEFAULT_DOM_BYTES = 400_000; // ~100K tokens
+
+export async function generateStatsReport(
+  skillsDir: string = DEFAULT_SKILLS_DIR,
+): Promise<StatsReport> {
+  let files: string[];
+  try {
+    files = (await readdir(skillsDir)).filter(f => f.endsWith('.json'));
+  } catch {
+    return emptyReport();
+  }
+
+  const domains: DomainStats[] = [];
+
+  for (const file of files) {
+    const filePath = join(skillsDir, file);
+    try {
+      const content = await readFile(filePath, 'utf-8');
+      const skill = JSON.parse(content) as SkillFile;
+      const fileStats = await stat(filePath);
+      const skillFileBytes = fileStats.size;
+
+      // Browser cost: use measured if available, else default
+      const domBytes = skill.metadata.browserCost?.domBytes ?? DEFAULT_DOM_BYTES;
+      const totalNetworkBytes = skill.metadata.browserCost?.totalNetworkBytes ?? 0;
+
+      // Replay cost: skill file + response data
+      const totalResponseBytes = skill.endpoints.reduce(
+        (sum, ep) => sum + (ep.responseBytes ?? 0), 0,
+      );
+
+      const browserTokens = Math.round(domBytes / 4);
+      const replayTokens = Math.round((skillFileBytes + totalResponseBytes) / 4);
+      const savingsPercent = browserTokens > 0
+        ? Math.round((1 - replayTokens / browserTokens) * 1000) / 10
+        : 0;
+
+      const replayable = skill.endpoints.filter(ep => {
+        const tier = ep.replayability?.tier;
+        return tier === 'green' || tier === 'yellow';
+      }).length;
+
+      domains.push({
+        domain: skill.domain,
+        endpoints: skill.endpoints.length,
+        replayable,
+        domBytes,
+        totalNetworkBytes,
+        skillFileBytes,
+        totalResponseBytes,
+        browserTokens,
+        replayTokens,
+        savingsPercent,
+      });
+    } catch {
+      // Skip invalid files
+    }
+  }
+
+  const totals = {
+    domains: domains.length,
+    endpoints: domains.reduce((s, d) => s + d.endpoints, 0),
+    replayable: domains.reduce((s, d) => s + d.replayable, 0),
+    totalDomBytes: domains.reduce((s, d) => s + d.domBytes, 0),
+    browserTokens: domains.reduce((s, d) => s + d.browserTokens, 0),
+    replayTokens: domains.reduce((s, d) => s + d.replayTokens, 0),
+    savingsPercent: 0,
+  };
+  totals.savingsPercent = totals.browserTokens > 0
+    ? Math.round((1 - totals.replayTokens / totals.browserTokens) * 1000) / 10
+    : 0;
+
+  return { domains, totals };
+}
+
+export function formatStatsHuman(report: StatsReport): string {
+  if (report.domains.length === 0) {
+    return '  No skill files found. Run `apitap capture <url>` first.';
+  }
+
+  const lines: string[] = [
+    '',
+    '  ApiTap — Usage Report',
+    '  ' + '═'.repeat(22),
+    '',
+    `  Skill files:      ${report.totals.domains} domains`,
+    `  Total endpoints:  ${report.totals.endpoints}`,
+    `  Replayable:       ${report.totals.replayable} (green/yellow tier)`,
+    '',
+    '  Token savings (measured):',
+  ];
+
+  // Table
+  const domCol = 24;
+  const hdr = [
+    '  ' + pad('Domain', domCol) + pad('DOM size', 10) + pad('Browser', 10) + pad('ApiTap', 10) + 'Saved',
+  ];
+  const sep = '  ' + '─'.repeat(domCol) + '─'.repeat(10) + '─'.repeat(10) + '─'.repeat(10) + '─'.repeat(8);
+
+  lines.push(sep);
+  lines.push(...hdr);
+  lines.push(sep);
+
+  for (const d of report.domains) {
+    const domName = d.domain.length > domCol - 2
+      ? d.domain.slice(0, domCol - 3) + '…'
+      : d.domain;
+    const hasMeasured = d.domBytes !== DEFAULT_DOM_BYTES;
+    const browserLabel = hasMeasured
+      ? formatTokens(d.browserTokens)
+      : `~${formatTokens(d.browserTokens)}`;
+    lines.push(
+      '  ' +
+      pad(domName, domCol) +
+      pad(formatBytes(d.domBytes), 10) +
+      pad(browserLabel, 10) +
+      pad(formatTokens(d.replayTokens), 10) +
+      `${d.savingsPercent}%`,
+    );
+  }
+
+  lines.push(sep);
+  const hasMeasuredTotals = report.domains.some(d => d.domBytes !== DEFAULT_DOM_BYTES);
+  const totalBrowserLabel = hasMeasuredTotals
+    ? formatTokens(report.totals.browserTokens)
+    : `~${formatTokens(report.totals.browserTokens)}`;
+  lines.push(
+    '  ' +
+    pad('Total', domCol) +
+    pad(formatBytes(report.totals.totalDomBytes), 10) +
+    pad(totalBrowserLabel, 10) +
+    pad(formatTokens(report.totals.replayTokens), 10) +
+    `${report.totals.savingsPercent}%`,
+  );
+  lines.push(sep);
+  lines.push('');
+
+  if (hasMeasuredTotals) {
+    lines.push('  Browser: measured DOM size during capture (page.content().length / 4)');
+  } else {
+    lines.push('  Browser: estimated ~100K tokens/site (re-capture for measured values)');
+  }
+  lines.push('  ApiTap:  measured skill file + API response sizes');
+  lines.push('');
+
+  return lines.join('\n');
+}
+
+function pad(s: string, width: number): string {
+  return s.length >= width ? s : s + ' '.repeat(width - s.length);
+}
+
+function formatBytes(bytes: number): string {
+  if (bytes >= 1_000_000) return `${(bytes / 1_000_000).toFixed(1)} MB`;
+  if (bytes >= 1_000) return `${(bytes / 1_000).toFixed(0)} KB`;
+  return `${bytes} B`;
+}
+
+function formatTokens(tokens: number): string {
+  if (tokens >= 1_000_000) return `${(tokens / 1_000_000).toFixed(2)}M tk`;
+  if (tokens >= 1_000) return `${(tokens / 1_000).toFixed(1)}K tk`;
+  return `${tokens} tk`;
+}
+
+function emptyReport(): StatsReport {
+  return {
+    domains: [],
+    totals: {
+      domains: 0, endpoints: 0, replayable: 0,
+      totalDomBytes: 0, browserTokens: 0, replayTokens: 0, savingsPercent: 0,
+    },
+  };
+}

package/src/types.ts

@@ -0,0 +1,233 @@
+// src/types.ts
+
+/** A captured HTTP request/response pair from the browser */
+export interface CapturedExchange {
+  request: {
+    url: string;
+    method: string;
+    headers: Record<string, string>;
+    postData?: string;  // POST/PUT/PATCH body
+  };
+  response: {
+    status: number;
+    headers: Record<string, string>;
+    body: string;
+    contentType: string;
+  };
+  timestamp: string;
+}
+
+/** Stored auth credentials for a domain */
+export interface StoredAuth {
+  type: 'bearer' | 'api-key' | 'cookie' | 'custom';
+  header: string;
+  value: string;
+  // v0.8: refreshable tokens (body-based, like CSRF)
+  tokens?: Record<string, StoredToken>;
+  // v0.8: cached browser session for faster refresh
+  session?: StoredSession;
+  // v0.9: OAuth credentials (stored encrypted, never in skill file)
+  refreshToken?: string;
+  clientSecret?: string;
+}
+
+/**
+ * Extended auth storage for v0.8 token refresh.
+ */
+
+/** Stored token with refresh metadata */
+export interface StoredToken {
+  value: string;
+  refreshedAt: string; // ISO timestamp
+  expiresAt?: string; // computed from ttlHint
+}
+
+/** Cached browser session for warm restarts */
+export interface StoredSession {
+  cookies: PlaywrightCookie[];
+  localStorage?: Record<string, string>;
+  savedAt: string;
+  maxAgeMs?: number; // when to consider session stale, default 24h
+}
+
+/** Playwright cookie shape (subset of full type) */
+export interface PlaywrightCookie {
+  name: string;
+  value: string;
+  domain: string;
+  path: string;
+  expires?: number;
+  httpOnly?: boolean;
+  secure?: boolean;
+  sameSite?: 'Strict' | 'Lax' | 'None';
+}
+
+/** OAuth configuration (non-secret, lives in shareable skill file) */
+export interface OAuthConfig {
+  tokenEndpoint: string;
+  clientId: string;
+  grantType: 'refresh_token' | 'client_credentials';
+  scope?: string;
+}
+
+/** Top-level auth config on SkillFile */
+export interface SkillAuth {
+  refreshUrl?: string; // URL to navigate for refresh, defaults to baseUrl
+  browserMode: 'headless' | 'visible';
+  captchaRisk: boolean;
+  ttlHint?: number; // estimated seconds until expiry
+  oauthConfig?: OAuthConfig; // v0.9: OAuth token endpoint config
+}
+
+/** Replay difficulty classification for an endpoint */
+export interface Replayability {
+  tier: 'green' | 'yellow' | 'orange' | 'red' | 'unknown';
+  verified: boolean;
+  signals: string[];
+}
+
+/** Detected pagination pattern */
+export interface PaginationInfo {
+  type: 'offset' | 'cursor' | 'page';
+  paramName: string;
+  limitParam?: string;
+}
+
+/** Request body template for POST/PUT/PATCH endpoints */
+export interface RequestBody {
+  contentType: string;
+  template: string | Record<string, unknown>;
+  variables?: string[];  // JSON paths of substitutable fields (user-provided params)
+  refreshableTokens?: string[];  // v0.8: system-refreshed tokens
+}
+
+/** A single API endpoint in a skill file */
+export interface SkillEndpoint {
+  id: string;
+  method: string;
+  path: string;
+  queryParams: Record<string, { type: string; example: string }>;
+  headers: Record<string, string>;
+  responseShape: { type: string; fields?: string[] };
+  examples: {
+    request: { url: string; headers: Record<string, string> };
+    responsePreview: unknown;
+  };
+  replayability?: Replayability;
+  pagination?: PaginationInfo;
+  requestBody?: RequestBody;
+  responseBytes?: number; // v1.0: response body size in bytes
+}
+
+/** The full skill file written to disk */
+export interface SkillFile {
+  version: string;
+  domain: string;
+  capturedAt: string;
+  baseUrl: string;
+  endpoints: SkillEndpoint[];
+  metadata: {
+    captureCount: number;
+    filteredCount: number;
+    toolVersion: string;
+    browserCost?: { // v1.0: measured browser cost during capture
+      domBytes: number;           // page.content().length
+      totalNetworkBytes: number;  // sum of ALL response body sizes
+      totalRequests: number;
+    };
+  };
+  provenance: 'self' | 'imported' | 'unsigned';
+  signature?: string;
+  auth?: SkillAuth; // v0.8: top-level auth config
+}
+
+/** Interactive element on a page, for agent targeting */
+export interface PageElement {
+  ref: string;          // "e0", "e1", etc. — agent uses this to target clicks/types
+  tag: string;          // "button", "a", "input", "select"
+  role?: string;        // ARIA role
+  text: string;         // visible text (truncated 200 chars)
+  name?: string;        // input name
+  placeholder?: string; // input placeholder
+  href?: string;        // link href
+  type?: string;        // input type
+  disabled?: boolean;
+}
+
+/** Structured page snapshot returned after every interaction */
+export interface PageSnapshot {
+  url: string;
+  title: string;
+  elements: PageElement[];
+  endpointsCaptured: number;
+  totalRequests: number;
+  filteredRequests: number;
+  recentEndpoints: string[];  // last 5 discovered (e.g. "GET /api/search")
+}
+
+/** Result from a capture session interaction */
+export interface InteractionResult {
+  success: boolean;
+  error?: string;
+  snapshot: PageSnapshot;
+}
+
+/** Result from finishing a capture session */
+export interface FinishResult {
+  aborted: boolean;
+  domains: {
+    domain: string;
+    endpointCount: number;
+    tiers: Record<string, number>;
+    skillFile: string;
+  }[];
+}
+
+/** Summary returned by `apitap list` */
+export interface SkillSummary {
+  domain: string;
+  skillFile: string;
+  endpointCount: number;
+  capturedAt: string;
+  provenance: 'self' | 'imported' | 'unsigned';
+}
+
+// --- Discovery types (Milestone 2: Smart Discovery) ---
+
+/** Detected web framework with predicted API patterns */
+export interface DetectedFramework {
+  name: string;
+  confidence: 'high' | 'medium' | 'low';
+  apiPatterns: string[];  // predicted API paths (e.g. "/wp-json/wp/v2/posts")
+}
+
+/** Discovered API spec */
+export interface DiscoveredSpec {
+  type: 'openapi' | 'swagger' | 'graphql-introspection';
+  url: string;
+  version?: string;
+  endpointCount?: number;
+}
+
+/** Result from probing a common API path */
+export interface ProbeResult {
+  method: string;
+  path: string;
+  status: number;
+  contentType: string;
+  isApi: boolean;
+}
+
+/** Result from the discovery pipeline */
+export interface DiscoveryResult {
+  confidence: 'high' | 'medium' | 'low' | 'none';
+  skillFile?: SkillFile;
+  hints?: string[];
+  frameworks?: DetectedFramework[];
+  specs?: DiscoveredSpec[];
+  probes?: ProbeResult[];
+  duration: number; // ms
+  authRequired?: boolean;     // true if site appears to need login
+  authSignals?: string[];     // reasons auth was detected
+  loginUrl?: string;          // detected login page URL
+}