browser-flow-tracker 0.1.0

package/src/exporters/markdown.js

@@ -0,0 +1,111 @@
+// Generates a first-draft Markdown flow document from a normalized flow.
+// Claude/Cursor can read the JSON for detail; this is the human-readable draft.
+
+function shortBody(body, isJson, max = 600) {
+  if (body == null || body === '') return '';
+  let s = isJson ? JSON.stringify(body, null, 2) : String(body);
+  if (s.length > max) s = s.slice(0, max) + '\n… [truncated]';
+  return s;
+}
+
+function safeLabel(s) {
+  return String(s).replace(/[^\w./:-]/g, '_').slice(0, 40);
+}
+
+export function toMarkdown(session, options = {}) {
+  const { flow, stats } = session;
+  const apiFlow = flow.filter((e) => e.category === 'api' || e.category === 'websocket' || e.category === 'sse');
+  const title = options.title || `API Flow — ${session.target?.title || session.target?.url || 'recording'}`;
+  const lines = [];
+
+  lines.push(`# ${title}`);
+  lines.push('');
+  lines.push(`> Recorded ${session.startedAt || ''} → ${session.stoppedAt || ''}`);
+  if (session.target?.url) lines.push(`> Page: ${session.target.url}`);
+  lines.push('');
+
+  // Overview
+  lines.push('## Overview');
+  lines.push('');
+  lines.push(`- **API calls captured:** ${apiFlow.length}`);
+  lines.push(`- **Total requests seen:** ${stats.totalCaptured} (${stats.droppedCount} filtered as noise)`);
+  const hosts = Object.entries(stats.byHost).sort((a, b) => b[1] - a[1]);
+  if (hosts.length) {
+    lines.push(`- **Hosts:** ${hosts.map(([h, n]) => `\`${h}\` (${n})`).join(', ')}`);
+  }
+  const statuses = Object.entries(stats.byStatus).sort();
+  if (statuses.length) {
+    lines.push(`- **Status codes:** ${statuses.map(([s, n]) => `${s}×${n}`).join(', ')}`);
+  }
+  lines.push('');
+
+  // Sequence diagram
+  lines.push('## Sequence diagram');
+  lines.push('');
+  lines.push('```mermaid');
+  lines.push('sequenceDiagram');
+  lines.push('    participant B as Browser');
+  const participants = new Map();
+  for (const e of apiFlow) {
+    if (!participants.has(e.host)) participants.set(e.host, `S${participants.size}`);
+  }
+  for (const [host, id] of participants) lines.push(`    participant ${id} as ${host}`);
+  for (const e of apiFlow) {
+    const id = participants.get(e.host);
+    const label = `${e.method} ${safeLabel(e.path)}`;
+    lines.push(`    B->>${id}: ${label}`);
+    const st = e.failed ? `FAILED ${e.errorText || ''}` : `${e.status || '?'} ${e.statusText || ''}`.trim();
+    lines.push(`    ${id}-->>B: ${st}`);
+  }
+  lines.push('```');
+  lines.push('');
+
+  // Ordered endpoint table
+  lines.push('## Call sequence');
+  lines.push('');
+  lines.push('| # | Method | Endpoint | Status | Time |');
+  lines.push('|---|--------|----------|--------|------|');
+  apiFlow.forEach((e, i) => {
+    const status = e.failed ? `✗ ${e.errorText || 'failed'}` : e.status || '';
+    const time = e.durationMs != null ? `${e.durationMs}ms` : '';
+    lines.push(`| ${i + 1} | ${e.method} | \`${e.host}${e.path}\` | ${status} | ${time} |`);
+  });
+  lines.push('');
+
+  // Detailed breakdown
+  lines.push('## Call details');
+  lines.push('');
+  apiFlow.forEach((e, i) => {
+    lines.push(`### ${i + 1}. ${e.method} ${e.path}`);
+    lines.push('');
+    lines.push(`- **URL:** \`${e.url}\``);
+    lines.push(`- **Status:** ${e.failed ? `✗ ${e.errorText}` : `${e.status || ''} ${e.statusText || ''}`}`);
+    if (e.durationMs != null) lines.push(`- **Duration:** ${e.durationMs}ms`);
+    if (e.initiator) lines.push(`- **Initiated by:** ${e.initiator}`);
+    if (e.query && Object.keys(e.query).length) {
+      lines.push(`- **Query params:** \`${JSON.stringify(e.query)}\``);
+    }
+    const reqStr = shortBody(e.requestBody, e.requestBodyIsJson);
+    if (reqStr) {
+      lines.push('');
+      lines.push('**Request body:**');
+      lines.push('```json');
+      lines.push(reqStr);
+      lines.push('```');
+    }
+    const resStr = shortBody(e.responseBody, e.responseBodyIsJson);
+    if (resStr) {
+      lines.push('');
+      lines.push('**Response body:**');
+      lines.push('```json');
+      lines.push(resStr);
+      lines.push('```');
+    }
+    lines.push('');
+  });
+
+  lines.push('---');
+  lines.push('_Draft generated by browser-flow-tracker. Refine with Claude/Cursor using the accompanying `.flow.json`._');
+  lines.push('');
+  return lines.join('\n');
+}

package/src/filter.js

@@ -0,0 +1,89 @@
+// Classifies captured requests so we can keep the meaningful API traffic and
+// drop the noise (static assets, fonts, analytics/telemetry beacons).
+
+const STATIC_EXT = /\.(css|scss|less|js|mjs|cjs|map|png|jpe?g|gif|svg|webp|avif|ico|bmp|woff2?|ttf|otf|eot|mp4|webm|mp3|wav|ogg|pdf)(\?|$)/i;
+
+// Resource types (from CDP Network.ResourceType) that are almost never "the API".
+const NOISE_TYPES = new Set([
+  'Image', 'Font', 'Stylesheet', 'Media', 'Manifest', 'TextTrack',
+  'CSPViolationReport', 'Ping', 'Prefetch',
+]);
+
+// Third-party analytics / telemetry / ad hosts. Substring match on hostname.
+const ANALYTICS_HOSTS = [
+  'google-analytics.com', 'googletagmanager.com', 'analytics.google.com',
+  'doubleclick.net', 'googlesyndication.com', 'googleadservices.com',
+  'facebook.com/tr', 'connect.facebook.net', 'segment.io', 'segment.com',
+  'mixpanel.com', 'amplitude.com', 'hotjar.com', 'fullstory.com',
+  'sentry.io', 'bugsnag.com', 'datadoghq.com', 'newrelic.com', 'nr-data.net',
+  'intercom.io', 'clarity.ms', 'optimizely.com', 'launchdarkly.com',
+  'cloudflareinsights.com', 'doubleverify.com', 'branch.io', 'appsflyer.com',
+  'posthog.com', 'i.posthog.com', 'heapanalytics.com', 'mouseflow.com',
+  'quantserve.com', 'scorecardresearch.com', 'snowplowanalytics.com',
+  'stats.g.doubleclick.net', 'analytics.tiktok.com', 'bat.bing.com',
+];
+
+// Analytics/telemetry that hits a FIRST-PARTY host (so host matching misses it):
+// e.g. Google Analytics' /g/collect, Cloudflare RUM's /cdn-cgi/rum. Matched on path.
+const ANALYTICS_PATH_RE = /\/cdn-cgi\/(rum|beacon)|\/g\/collect|\/j\/collect|\/mp\/collect|\/gtag\/|\/gtm\.js|\/piwik\.php|\/matomo\.php|\/b\/ss\//i;
+
+// Google Analytics / Measurement Protocol requests carry a tid=G-/UA-/GT- id.
+function looksLikeGoogleAnalytics(query) {
+  const tid = query?.get?.('tid');
+  return typeof tid === 'string' && /^(G-|UA-|GT-|AW-|DC-)/.test(tid);
+}
+
+/**
+ * Decide whether a request is "API-relevant".
+ * Returns { keep: boolean, reason: string, category: string }.
+ */
+export function classify(req) {
+  const type = req.resourceType || '';
+  const url = req.url || '';
+  let host = '';
+  let path = url;
+  let query = null;
+  try {
+    const u = new URL(url);
+    host = u.hostname;
+    path = u.pathname + u.search;
+    query = u.searchParams;
+  } catch {
+    // non-URL (data:, blob:) — always noise for our purposes
+    return { keep: false, reason: 'non-http scheme', category: 'other' };
+  }
+
+  if (url.startsWith('data:') || url.startsWith('blob:')) {
+    return { keep: false, reason: 'inline resource', category: 'other' };
+  }
+  if (ANALYTICS_HOSTS.some((h) => host.includes(h.split('/')[0]) && url.includes(h))) {
+    return { keep: false, reason: 'analytics/telemetry host', category: 'analytics' };
+  }
+  if (ANALYTICS_PATH_RE.test(path)) {
+    return { keep: false, reason: 'analytics/telemetry endpoint', category: 'analytics' };
+  }
+  if (looksLikeGoogleAnalytics(query)) {
+    return { keep: false, reason: 'google analytics beacon', category: 'analytics' };
+  }
+  if (NOISE_TYPES.has(type)) {
+    return { keep: false, reason: `resource type ${type}`, category: 'asset' };
+  }
+  if (STATIC_EXT.test(path) && type !== 'XHR' && type !== 'Fetch') {
+    return { keep: false, reason: 'static asset extension', category: 'asset' };
+  }
+
+  // Keep XHR/Fetch (the classic API calls), WebSockets, EventSource, and the
+  // top-level document navigation (useful context for where a flow starts).
+  if (type === 'XHR' || type === 'Fetch') {
+    return { keep: true, reason: 'xhr/fetch', category: 'api' };
+  }
+  if (type === 'WebSocket') return { keep: true, reason: 'websocket', category: 'websocket' };
+  if (type === 'EventSource') return { keep: true, reason: 'server-sent events', category: 'sse' };
+  if (type === 'Document') return { keep: true, reason: 'navigation', category: 'document' };
+
+  // GraphQL / JSON endpoints sometimes come through as "Other".
+  if (/graphql|\/api\/|\/v\d+\//i.test(path)) {
+    return { keep: true, reason: 'looks like an api path', category: 'api' };
+  }
+  return { keep: false, reason: `type ${type || 'unknown'}`, category: 'other' };
+}

package/src/normalize.js

@@ -0,0 +1,109 @@
+// Turns raw CDP records into a clean, ordered, API-focused flow.
+
+import { classify } from './filter.js';
+
+const SENSITIVE_HEADERS = new Set([
+  'authorization', 'cookie', 'set-cookie', 'x-api-key', 'x-auth-token',
+  'proxy-authorization', 'x-csrf-token',
+]);
+
+function redactHeaders(headers, redact) {
+  if (!headers) return {};
+  const out = {};
+  for (const [k, v] of Object.entries(headers)) {
+    out[k] = redact && SENSITIVE_HEADERS.has(k.toLowerCase()) ? '[redacted]' : v;
+  }
+  return out;
+}
+
+function tryParseJson(body) {
+  if (typeof body !== 'string') return undefined;
+  const t = body.trim();
+  if (!t || (t[0] !== '{' && t[0] !== '[')) return undefined;
+  try { return JSON.parse(t); } catch { return undefined; }
+}
+
+function splitUrl(url) {
+  try {
+    const u = new URL(url);
+    return { host: u.hostname, path: u.pathname, query: u.search ? Object.fromEntries(u.searchParams) : undefined };
+  } catch {
+    return { host: '', path: url, query: undefined };
+  }
+}
+
+/**
+ * @param {Array} records raw records from CdpRecorder
+ * @param {object} opts { includeNoise, redact }
+ * @returns { flow, dropped, stats }
+ */
+export function normalize(records, { includeNoise = false, redact = true } = {}) {
+  const flow = [];
+  const dropped = [];
+  const hostCounts = {};
+  const statusCounts = {};
+
+  for (const rec of records) {
+    if (!rec.url) continue;
+    const c = classify(rec);
+    const { host, path, query } = splitUrl(rec.url);
+    const entry = {
+      index: rec.index,
+      category: c.category,
+      method: rec.method || 'GET',
+      url: rec.url,
+      host,
+      path,
+      query,
+      resourceType: rec.resourceType,
+      status: rec.status,
+      statusText: rec.statusText,
+      durationMs: rec.durationMs != null ? Math.round(rec.durationMs) : undefined,
+      sizeBytes: rec.encodedDataLength,
+      fromCache: rec.fromCache || undefined,
+      failed: rec.failed || undefined,
+      errorText: rec.errorText,
+      initiator: rec.initiator?.type,
+      redirects: rec.redirects,
+      requestHeaders: redactHeaders(rec.requestHeaders, redact),
+      responseHeaders: redactHeaders(rec.responseHeaders, redact),
+    };
+
+    const reqJson = tryParseJson(rec.requestBody);
+    entry.requestBody = reqJson !== undefined ? reqJson : rec.requestBody;
+    entry.requestBodyIsJson = reqJson !== undefined;
+
+    if (!rec.responseBodyBase64) {
+      const resJson = tryParseJson(rec.responseBody);
+      entry.responseBody = resJson !== undefined ? resJson : rec.responseBody;
+      entry.responseBodyIsJson = resJson !== undefined;
+      entry.responseBodyTruncated = rec.responseBodyTruncated || undefined;
+    } else {
+      entry.responseBody = '[binary]';
+    }
+
+    if (c.keep || includeNoise) {
+      if (!c.keep) entry.filteredReason = c.reason;
+      flow.push(entry);
+    } else {
+      dropped.push({ method: entry.method, url: rec.url, reason: c.reason });
+    }
+
+    if (c.keep) {
+      hostCounts[host] = (hostCounts[host] || 0) + 1;
+      if (rec.status) statusCounts[rec.status] = (statusCounts[rec.status] || 0) + 1;
+    }
+  }
+
+  return {
+    flow,
+    dropped,
+    stats: {
+      totalCaptured: records.length,
+      kept: flow.filter((e) => !e.filteredReason).length,
+      droppedCount: dropped.length,
+      byHost: hostCounts,
+      byStatus: statusCounts,
+    },
+  };
+}

package/src/session.js

@@ -0,0 +1,185 @@
+// Orchestrates a recording session: optionally launch a browser, attach the
+// CDP recorder, then on stop normalize + write JSON / HAR / Markdown outputs.
+
+import { mkdirSync, writeFileSync, readFileSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+import net from 'node:net';
+import CDP from 'chrome-remote-interface';
+import { CdpRecorder } from './cdpRecorder.js';
+import { launchBrowser } from './browsers.js';
+import { normalize } from './normalize.js';
+import { toHar } from './exporters/har.js';
+import { toMarkdown } from './exporters/markdown.js';
+
+function stamp() {
+  // Filesystem-safe timestamp without relying on locale.
+  return new Date().toISOString().replace(/[:.]/g, '-');
+}
+
+// Grab an OS-assigned free TCP port. Fixes the #1 launch failure: colliding
+// with something already sitting on the default 9222 debug port.
+function findFreePort() {
+  return new Promise((resolve, reject) => {
+    const srv = net.createServer();
+    srv.unref();
+    srv.on('error', reject);
+    srv.listen(0, '127.0.0.1', () => {
+      const { port } = srv.address();
+      srv.close(() => resolve(port));
+    });
+  });
+}
+
+// Is a CDP endpoint already answering on this port? (i.e. our persistent
+// profile's browser is still open from a previous recording).
+async function isCdpAlive(port, host = 'localhost') {
+  try {
+    await CDP.List({ port, host });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// A persistent, dedicated profile per browser. Separate from the user's normal
+// profile (so it never conflicts with their running browser) and persistent
+// (so a login done once carries over to the next recording).
+function profileDir(id) {
+  const dir = join(homedir(), '.browser-flow-tracker', 'profiles', id || 'default');
+  mkdirSync(dir, { recursive: true });
+  return dir;
+}
+
+export class TrackingSession {
+  constructor(opts = {}) {
+    this.opts = opts;
+    this.recorder = null;
+    this.launched = null;
+    this.active = false;
+    this.finalized = null;      // set when files have been written
+    this.closedByUser = false;  // true if finalized because the browser was closed
+  }
+
+  async start() {
+    const { launch, browser, host = 'localhost', url, headless, urlMatch } = this.opts;
+    // In attach mode, honor the given port (the user started their browser on it).
+    let port = this.opts.port || 9222;
+    this.reusedExisting = false;
+
+    if (launch) {
+      const dir = profileDir(browser);
+      const portFile = join(dir, '.bft-port');
+
+      // Reuse an already-open recording window (persistent profile still running)?
+      if (existsSync(portFile)) {
+        const prev = Number(readFileSync(portFile, 'utf8').trim());
+        if (prev && (await isCdpAlive(prev, host))) {
+          port = prev;
+          this.reusedExisting = true;
+        }
+      }
+
+      if (!this.reusedExisting) {
+        // Fresh launch: a free port + a dedicated persistent profile means this
+        // never collides with the user's normal browser or a busy 9222.
+        port = await findFreePort();
+        this.launched = launchBrowser({ id: browser, port, userDataDir: dir, headless });
+        writeFileSync(portFile, String(port));
+      }
+    }
+
+    const targetFilter = urlMatch
+      ? (t) => t.url && t.url.includes(urlMatch)
+      : undefined;
+
+    this.recorder = new CdpRecorder({ port, host, targetFilter });
+    // If the user closes the browser/tab, auto-finalize and write the files.
+    this.recorder.onDisconnect = () => { this._onBrowserClosed(); };
+    const info = await this.recorder.start();
+    this.active = true;
+    this.port = port;
+
+    // Drive the initial navigation ourselves (after we're already listening, so
+    // no early requests are missed). The user does the rest of the clicking.
+    if (launch && url) {
+      await this.recorder.navigate(url);
+    }
+
+    return {
+      attachedTo: info.target,
+      launched: Boolean(this.launched),
+      reusedExisting: this.reusedExisting,
+      port,
+    };
+  }
+
+  // Live snapshot without stopping the session.
+  snapshot() {
+    if (!this.recorder) return { flow: [], stats: {} };
+    const raw = this.recorder.getRecords();
+    return normalize(raw, {
+      includeNoise: this.opts.includeNoise,
+      redact: this.opts.redact !== false,
+    });
+  }
+
+  // Called when the CDP socket drops because the user closed the browser/tab.
+  _onBrowserClosed() {
+    if (!this.active || this.finalized) return; // already stopping/stopped
+    this.closedByUser = true;
+    // Fire-and-forget: write files with the session's default output settings.
+    this.stop({
+      outDir: this.opts.outDir,
+      name: this.opts.name,
+      closeBrowser: false,
+    }).catch(() => { /* nothing we can do here */ });
+  }
+
+  async stop({ outDir, name, write = true, closeBrowser = false } = {}) {
+    // Idempotent: if we already wrote files (e.g. the browser was closed), just
+    // return that result instead of erroring on a later explicit stop.
+    if (this.finalized) return this.finalized;
+    if (!this.recorder) throw new Error('Session not started.');
+    const raw = await this.recorder.stop();
+    this.active = false;
+
+    const norm = normalize(raw.records, {
+      includeNoise: this.opts.includeNoise,
+      redact: this.opts.redact !== false,
+    });
+
+    const session = {
+      tool: 'browser-flow-tracker',
+      version: '0.1.0',
+      startedAt: raw.startedAt,
+      stoppedAt: raw.stoppedAt,
+      target: raw.target,
+      stats: norm.stats,
+      flow: norm.flow,
+      dropped: norm.dropped,
+    };
+
+    let files = null;
+    if (write) {
+      const dir = outDir || join(process.cwd(), 'recordings');
+      mkdirSync(dir, { recursive: true });
+      const base = name || `flow-${stamp()}`;
+      const jsonPath = join(dir, `${base}.flow.json`);
+      const harPath = join(dir, `${base}.har`);
+      const mdPath = join(dir, `${base}.md`);
+      writeFileSync(jsonPath, JSON.stringify(session, null, 2));
+      writeFileSync(harPath, JSON.stringify(toHar(session.flow, session), null, 2));
+      writeFileSync(mdPath, toMarkdown({ ...session }, { title: this.opts.title }));
+      files = { json: jsonPath, har: harPath, markdown: mdPath };
+    }
+
+    if (closeBrowser && this.launched?.child) {
+      try { this.launched.child.kill(); } catch { /* ignore */ }
+    }
+
+    const result = { session, files, closedByUser: this.closedByUser };
+    this.finalized = result;
+    return result;
+  }
+}