browser-flow-tracker 0.1.0

package/mcp/server.js

@@ -0,0 +1,177 @@
+#!/usr/bin/env node
+// MCP server exposing browser-flow-tracker to Claude / Cursor.
+//
+// Tools:
+//   list_browsers   - which Chromium browsers are installed
+//   start_tracking  - launch or attach, begin recording (one session at a time)
+//   get_flow        - live snapshot of the current recording (without stopping)
+//   stop_tracking   - stop, write JSON/HAR/Markdown, return the flow
+
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import { z } from 'zod';
+import { TrackingSession } from '../src/session.js';
+import { detectInstalled } from '../src/browsers.js';
+
+let current = null; // the single active TrackingSession
+
+function json(data) {
+  return { content: [{ type: 'text', text: JSON.stringify(data, null, 2) }] };
+}
+function err(message) {
+  return { isError: true, content: [{ type: 'text', text: message }] };
+}
+
+const server = new McpServer(
+  { name: 'browser-flow-tracker', version: '0.1.0' },
+  {
+    instructions: [
+      'Use these tools whenever the user wants to record, analyze, or document the',
+      'API / network flow of a web page or app module (e.g. "what APIs does this call",',
+      '"document the checkout flow", "track the requests on this page").',
+      '',
+      'ALWAYS use these tools for that job. Do NOT hand-roll browser automation, do NOT',
+      'launch browsers or poke at debug ports via the shell, and do NOT try to analyze',
+      'network traffic yourself — start_tracking handles launching/attaching and picks a',
+      'free port automatically.',
+      '',
+      'The capture is USER-DRIVEN: call start_tracking (launch a fresh window OR attach to',
+      "the user's browser), then STOP and let the USER navigate/click through the real flow",
+      'themselves. Do not auto-navigate the flow for them. When they say they are done, call',
+      'stop_tracking and turn the returned flow into a document. Use get_flow for a live peek.',
+      'The launched window uses a persistent profile, so a login done once is remembered.',
+    ].join('\n'),
+  },
+);
+
+server.registerTool(
+  'list_browsers',
+  {
+    description: 'List Chromium-family browsers installed on this machine that can be tracked (Brave, Chrome, Arc, Edge, etc.). Safari/Firefox are not supported (no CDP).',
+    inputSchema: {},
+  },
+  async () => json({ browsers: detectInstalled() }),
+);
+
+server.registerTool(
+  'start_tracking',
+  {
+    description:
+      'Start recording a page\'s API/network flow. Either launch a fresh browser (launch=true, requires browser id) or attach to a browser already running with --remote-debugging-port (attach mode). Only one session runs at a time.',
+    inputSchema: {
+      launch: z.boolean().optional().describe('Launch a fresh browser with a throwaway profile. If false/omitted, attach to a running browser.'),
+      browser: z.string().optional().describe('Browser id for launch mode: brave | arc | chrome | edge | vivaldi | opera | chromium'),
+      port: z.number().optional().describe('CDP port (default 9222).'),
+      url: z.string().optional().describe('URL to open (launch mode).'),
+      urlMatch: z.string().optional().describe('When attaching, pick the tab whose URL contains this substring.'),
+      headless: z.boolean().optional().describe('Launch headless (launch mode).'),
+      includeNoise: z.boolean().optional().describe('Also keep filtered static/analytics requests.'),
+      redact: z.boolean().optional().describe('Redact auth/cookie headers (default true).'),
+      outDir: z.string().optional().describe('Where to write output files if the user closes the browser (default ./recordings).'),
+      name: z.string().optional().describe('Output file basename used on auto-finalize (default flow-<timestamp>).'),
+      title: z.string().optional().describe('Title for the generated Markdown document.'),
+    },
+  },
+  async (args) => {
+    if (current?.active) return err('A tracking session is already active. Call stop_tracking first.');
+    if (args.launch && !args.browser) return err('launch=true requires a browser id. Call list_browsers.');
+    current = new TrackingSession({
+      launch: Boolean(args.launch),
+      browser: args.browser,
+      port: args.port ?? 9222,
+      url: args.url,
+      urlMatch: args.urlMatch,
+      headless: Boolean(args.headless),
+      includeNoise: Boolean(args.includeNoise),
+      redact: args.redact !== false,
+      // Defaults used to write files automatically if the user closes the browser.
+      outDir: args.outDir,
+      name: args.name,
+      title: args.title,
+    });
+    try {
+      const info = await current.start();
+      const hint = info.reusedExisting
+        ? 'Reused the already-open recording window (your login is preserved). Now let the USER navigate the flow; call get_flow to peek or stop_tracking when they are done.'
+        : 'A browser window is open and recording. Do NOT navigate it yourself — let the USER click through the real flow, then call get_flow to peek or stop_tracking to finish. (Persistent profile: any login is remembered next time.)';
+      return json({ status: 'recording', ...info, hint });
+    } catch (e) {
+      current = null;
+      return err(`Failed to start tracking: ${e.message}`);
+    }
+  },
+);
+
+server.registerTool(
+  'get_flow',
+  {
+    description: 'Return a live snapshot of the API calls captured so far in the current session, without stopping it.',
+    inputSchema: {
+      full: z.boolean().optional().describe('Include full request/response bodies and headers (default: summary only).'),
+    },
+  },
+  async ({ full }) => {
+    // If the user closed the browser, the session auto-finalized and wrote files.
+    if (current && !current.active && current.finalized) {
+      const f = current.finalized;
+      return json({
+        status: 'ended',
+        reason: current.closedByUser ? 'browser closed by user' : 'stopped',
+        files: f.files,
+        stats: f.session.stats,
+        note: 'The recording is finished and files are written. Read the .flow.json to build the doc.',
+      });
+    }
+    if (!current?.active) return err('No active tracking session. Call start_tracking first.');
+    const snap = current.snapshot();
+    if (full) return json(snap);
+    const summary = snap.flow
+      .filter((e) => e.category !== 'document')
+      .map((e) => ({ i: e.index, method: e.method, url: `${e.host}${e.path}`, status: e.status, ms: e.durationMs }));
+    return json({ stats: snap.stats, calls: summary });
+  },
+);
+
+server.registerTool(
+  'stop_tracking',
+  {
+    description: 'Stop the current recording, write JSON + HAR + Markdown outputs to disk, and return the full normalized flow plus file paths.',
+    inputSchema: {
+      outDir: z.string().optional().describe('Output directory (default ./recordings).'),
+      name: z.string().optional().describe('Output file basename (default flow-<timestamp>).'),
+      title: z.string().optional().describe('Title for the generated Markdown document.'),
+      closeBrowser: z.boolean().optional().describe('Close the browser if it was launched by this tool.'),
+    },
+  },
+  async (args) => {
+    if (!current) return err('No session to stop.');
+    if (args.title) current.opts.title = args.title;
+    try {
+      const { session, files, closedByUser } = await current.stop({
+        outDir: args.outDir,
+        name: args.name,
+        closeBrowser: args.closeBrowser,
+      });
+      // Return a COMPACT summary + file paths only. Full detail (bodies/headers)
+      // can be huge, so it lives in the .flow.json file — read that to build the doc.
+      const calls = session.flow
+        .filter((e) => e.category !== 'document')
+        .map((e) => ({ i: e.index, method: e.method, url: `${e.host}${e.path}`, status: e.status, ms: e.durationMs }));
+      const result = {
+        status: closedByUser ? 'ended (browser closed)' : 'stopped',
+        files,
+        stats: session.stats,
+        target: session.target,
+        calls,
+        note: 'Full request/response detail is in the .flow.json file — read it from disk to write the doc.',
+      };
+      current = null;
+      return json(result);
+    } catch (e) {
+      return err(`Failed to stop: ${e.message}`);
+    }
+  },
+);
+
+const transport = new StdioServerTransport();
+await server.connect(transport);

package/package.json

@@ -0,0 +1,54 @@
+{
+  "name": "browser-flow-tracker",
+  "version": "0.1.0",
+  "description": "Record and document the API flow of any Chromium browser page (Brave, Chrome, Arc, Edge) via the Chrome DevTools Protocol. Ships as a local MCP server for Claude/Cursor and a CLI. Emits structured JSON + HAR + a Markdown flow-doc.",
+  "type": "module",
+  "bin": {
+    "browser-flow-tracker": "mcp/server.js",
+    "bft": "bin/bft.js"
+  },
+  "files": [
+    "bin",
+    "mcp",
+    "src",
+    "README.md",
+    "LICENSE"
+  ],
+  "scripts": {
+    "start": "node bin/bft.js",
+    "mcp": "node mcp/server.js"
+  },
+  "engines": {
+    "node": ">=18"
+  },
+  "keywords": [
+    "mcp",
+    "model-context-protocol",
+    "claude",
+    "cursor",
+    "api",
+    "network",
+    "cdp",
+    "chrome-devtools-protocol",
+    "har",
+    "network-flow",
+    "api-documentation"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/devggaurav/web-Api-scrapper.git"
+  },
+  "bugs": {
+    "url": "https://github.com/devggaurav/web-Api-scrapper/issues"
+  },
+  "homepage": "https://github.com/devggaurav/web-Api-scrapper#readme",
+  "publishConfig": {
+    "access": "public"
+  },
+  "dependencies": {
+    "chrome-remote-interface": "^0.33.3",
+    "@modelcontextprotocol/sdk": "^1.0.4",
+    "zod": "^4.4.3"
+  },
+  "license": "MIT"
+}

package/src/browsers.js

@@ -0,0 +1,104 @@
+// Browser detection + launch helpers for Chromium-family browsers on macOS,
+// Linux and Windows. All of these speak the Chrome DevTools Protocol (CDP),
+// which is what the recorder attaches to.
+
+import { spawn } from 'node:child_process';
+import { existsSync, mkdtempSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+// Per-platform candidate binary paths, keyed by a short browser id.
+const BINARIES = {
+  darwin: {
+    brave: ['/Applications/Brave Browser.app/Contents/MacOS/Brave Browser'],
+    chrome: [
+      '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome',
+      '/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary',
+    ],
+    // Arc ships a Chromium engine; the CDP endpoint works the same way.
+    arc: ['/Applications/Arc.app/Contents/MacOS/Arc'],
+    edge: ['/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge'],
+    vivaldi: ['/Applications/Vivaldi.app/Contents/MacOS/Vivaldi'],
+    opera: ['/Applications/Opera.app/Contents/MacOS/Opera'],
+    chromium: ['/Applications/Chromium.app/Contents/MacOS/Chromium'],
+  },
+  linux: {
+    brave: ['/usr/bin/brave-browser', '/usr/bin/brave'],
+    chrome: ['/usr/bin/google-chrome', '/usr/bin/google-chrome-stable'],
+    edge: ['/usr/bin/microsoft-edge'],
+    vivaldi: ['/usr/bin/vivaldi'],
+    opera: ['/usr/bin/opera'],
+    chromium: ['/usr/bin/chromium', '/usr/bin/chromium-browser'],
+  },
+  win32: {
+    brave: ['C:\\Program Files\\BraveSoftware\\Brave-Browser\\Application\\brave.exe'],
+    chrome: [
+      'C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe',
+      'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chrome.exe',
+    ],
+    edge: ['C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe'],
+  },
+};
+
+// Browsers that are Chromium-based but not covered here (Safari/WebKit,
+// Firefox/Gecko) do not expose CDP and need the proxy engine instead.
+export const UNSUPPORTED_CDP = {
+  safari: 'Safari uses WebKit and does not speak the Chrome DevTools Protocol. Use proxy mode (roadmap) or record in a Chromium browser.',
+  firefox: 'Firefox uses a different remote protocol. Use proxy mode (roadmap) or record in a Chromium browser.',
+};
+
+export function resolveBrowser(id) {
+  const platform = process.platform;
+  const table = BINARIES[platform];
+  if (!table) throw new Error(`Unsupported platform: ${platform}`);
+  const candidates = table[id];
+  if (!candidates) {
+    if (UNSUPPORTED_CDP[id]) throw new Error(UNSUPPORTED_CDP[id]);
+    throw new Error(`Unknown browser id "${id}". Known: ${Object.keys(table).join(', ')}`);
+  }
+  const bin = candidates.find((p) => existsSync(p));
+  if (!bin) {
+    throw new Error(
+      `Could not find ${id} at any known path for ${platform}:\n  ${candidates.join('\n  ')}`,
+    );
+  }
+  return bin;
+}
+
+// Which known browsers are actually installed on this machine.
+export function detectInstalled() {
+  const platform = process.platform;
+  const table = BINARIES[platform] || {};
+  const found = [];
+  for (const [id, candidates] of Object.entries(table)) {
+    const bin = candidates.find((p) => existsSync(p));
+    if (bin) found.push({ id, bin });
+  }
+  return found;
+}
+
+/**
+ * Launch a Chromium browser with remote debugging enabled and return the
+ * child process. Uses a throwaway user-data-dir by default so it never
+ * collides with the user's normal profile (which refuses --remote-debugging-port
+ * if already running).
+ */
+export function launchBrowser({ id, port = 9222, url, userDataDir, headless = false } = {}) {
+  const bin = resolveBrowser(id);
+  const dataDir = userDataDir || mkdtempSync(join(tmpdir(), `bft-${id}-`));
+  const args = [
+    `--remote-debugging-port=${port}`,
+    `--user-data-dir=${dataDir}`,
+    '--no-first-run',
+    '--no-default-browser-check',
+    '--remote-allow-origins=*',
+    // Persistent profiles otherwise nag "Chrome didn't shut down correctly".
+    '--disable-session-crashed-bubble',
+    '--hide-crash-restore-bubble',
+  ];
+  if (headless) args.push('--headless=new');
+  if (url) args.push(url);
+
+  const child = spawn(bin, args, { stdio: 'ignore', detached: false });
+  return { child, port, dataDir, bin };
+}

package/src/cdpRecorder.js

@@ -0,0 +1,200 @@
+// Core capture engine. Attaches to a running Chromium browser over the
+// Chrome DevTools Protocol and records the network flow of a page target in
+// order, with request/response metadata and (bounded) bodies.
+
+import CDP from 'chrome-remote-interface';
+
+const DEFAULT_MAX_BODY = 512 * 1024; // 512 KB per body, to stay sane
+
+// Wait until the CDP endpoint is reachable (a freshly launched browser needs
+// a moment to open its debugging port).
+async function waitForEndpoint(port, host, timeoutMs = 15000) {
+  const deadline = Date.now() + timeoutMs;
+  let lastErr;
+  while (Date.now() < deadline) {
+    try {
+      const targets = await CDP.List({ port, host });
+      if (targets.some((t) => t.type === 'page')) return targets;
+    } catch (err) {
+      lastErr = err;
+    }
+    await new Promise((r) => setTimeout(r, 250));
+  }
+  throw new Error(
+    `No CDP page target on ${host}:${port} within ${timeoutMs}ms.` +
+      (lastErr ? ` Last error: ${lastErr.message}` : ''),
+  );
+}
+
+export class CdpRecorder {
+  constructor({ port = 9222, host = 'localhost', maxBodyBytes = DEFAULT_MAX_BODY, targetFilter } = {}) {
+    this.port = port;
+    this.host = host;
+    this.maxBodyBytes = maxBodyBytes;
+    this.targetFilter = targetFilter; // optional (target) => boolean to pick a tab
+    this.client = null;
+    this.seq = 0;
+    this.startWall = null;
+    this.startedAt = null;
+    // requestId -> record
+    this.records = new Map();
+    // preserve emission order
+    this.order = [];
+    this.target = null;
+  }
+
+  async start() {
+    const targets = await waitForEndpoint(this.port, this.host);
+    const pages = targets.filter((t) => t.type === 'page' && !t.url.startsWith('devtools://'));
+    let chosen = pages;
+    if (this.targetFilter) chosen = pages.filter(this.targetFilter);
+    const target = chosen[0] || pages[0];
+    if (!target) throw new Error('No suitable page target to attach to.');
+    this.target = target;
+
+    this.client = await CDP({ port: this.port, host: this.host, target });
+    const { Network, Page } = this.client;
+
+    await Network.enable({ maxTotalBufferSize: 100_000_000, maxResourceBufferSize: 20_000_000 });
+    try { await Page.enable(); } catch { /* Page domain optional */ }
+
+    this.startedAt = new Date().toISOString();
+
+    Network.requestWillBeSent((p) => this._onRequest(p));
+    Network.responseReceived((p) => this._onResponse(p));
+    Network.loadingFinished((p) => this._onFinished(p));
+    Network.loadingFailed((p) => this._onFailed(p));
+    Network.webSocketCreated((p) => this._onWebSocket(p));
+
+    // Fires when the browser/tab is closed by the user (the CDP socket drops).
+    // Lets the session auto-finalize and write files on browser close.
+    this.client.on('disconnect', () => {
+      if (!this.stopping) this.onDisconnect?.();
+    });
+
+    return { target: { id: target.id, url: target.url, title: target.title } };
+  }
+
+  // Navigate the attached target. Used by launch mode so the recorder is
+  // listening before the page fires its first request (avoids missing early calls).
+  async navigate(url) {
+    if (!this.client) throw new Error('Recorder not started.');
+    await this.client.Page.navigate({ url });
+  }
+
+  _rec(requestId) {
+    let rec = this.records.get(requestId);
+    if (!rec) {
+      rec = { requestId, index: this.seq++ };
+      this.records.set(requestId, rec);
+      this.order.push(requestId);
+    }
+    return rec;
+  }
+
+  _onRequest(p) {
+    const rec = this._rec(p.requestId);
+    const r = p.request;
+    // A redirect reuses the requestId; keep the redirect chain instead of losing it.
+    if (rec.method && p.redirectResponse) {
+      rec.redirects = rec.redirects || [];
+      rec.redirects.push({ url: rec.url, status: p.redirectResponse.status });
+    }
+    rec.method = r.method;
+    rec.url = r.url;
+    rec.resourceType = p.type;
+    rec.requestHeaders = r.headers || {};
+    rec.requestBody = r.postData;
+    rec.hasPostData = r.hasPostData || Boolean(r.postData);
+    rec.wallTime = p.wallTime;
+    rec.timestamp = p.timestamp;
+    rec.initiator = p.initiator ? { type: p.initiator.type } : undefined;
+    if (this.startWall == null && p.wallTime) this.startWall = p.wallTime;
+  }
+
+  _onResponse(p) {
+    const rec = this._rec(p.requestId);
+    const res = p.response;
+    rec.status = res.status;
+    rec.statusText = res.statusText;
+    rec.responseHeaders = res.headers || {};
+    rec.mimeType = res.mimeType;
+    rec.remoteAddress = res.remoteIPAddress ? `${res.remoteIPAddress}:${res.remotePort}` : undefined;
+    rec.fromCache = res.fromDiskCache || res.fromServiceWorker || false;
+    rec.protocol = res.protocol;
+    if (res.timing) {
+      rec.timing = res.timing;
+      rec.responseTs = p.timestamp;
+    }
+  }
+
+  async _onFinished(p) {
+    const rec = this._rec(p.requestId);
+    rec.encodedDataLength = p.encodedDataLength;
+    rec.finishedTs = p.timestamp;
+    if (rec.timestamp != null && p.timestamp != null) {
+      rec.durationMs = Math.max(0, (p.timestamp - rec.timestamp) * 1000);
+    }
+    await this._captureBody(rec);
+  }
+
+  _onFailed(p) {
+    const rec = this._rec(p.requestId);
+    rec.failed = true;
+    rec.errorText = p.errorText;
+    rec.canceled = p.canceled;
+    rec.finishedTs = p.timestamp;
+    if (rec.timestamp != null && p.timestamp != null) {
+      rec.durationMs = Math.max(0, (p.timestamp - rec.timestamp) * 1000);
+    }
+  }
+
+  _onWebSocket(p) {
+    const rec = this._rec(p.requestId);
+    rec.method = 'WS';
+    rec.url = p.url;
+    rec.resourceType = 'WebSocket';
+  }
+
+  async _captureBody(rec) {
+    if (rec.responseBody !== undefined) return;
+    try {
+      const { body, base64Encoded } = await this.client.Network.getResponseBody({
+        requestId: rec.requestId,
+      });
+      if (base64Encoded) {
+        rec.responseBodyBase64 = true;
+        rec.responseBody = body.length > this.maxBodyBytes ? '[binary body omitted]' : body;
+      } else if (body.length > this.maxBodyBytes) {
+        rec.responseBody = body.slice(0, this.maxBodyBytes);
+        rec.responseBodyTruncated = true;
+      } else {
+        rec.responseBody = body;
+      }
+    } catch {
+      // Body may be unavailable (evicted, 204, websocket, etc.) — that's fine.
+    }
+  }
+
+  // Snapshot of everything captured so far, in emission order.
+  getRecords() {
+    return this.order.map((id) => this.records.get(id)).filter(Boolean);
+  }
+
+  async stop() {
+    this.stopping = true; // suppress the disconnect->auto-finalize path
+    // Give any in-flight loadingFinished handlers a beat to grab bodies.
+    await new Promise((r) => setTimeout(r, 200));
+    const records = this.getRecords();
+    if (this.client) {
+      try { await this.client.close(); } catch { /* ignore */ }
+      this.client = null;
+    }
+    return {
+      startedAt: this.startedAt,
+      stoppedAt: new Date().toISOString(),
+      target: this.target ? { id: this.target.id, url: this.target.url, title: this.target.title } : null,
+      records,
+    };
+  }
+}

package/src/exporters/har.js

@@ -0,0 +1,62 @@
+// Minimal HAR 1.2 exporter so recordings can be opened in Chrome DevTools,
+// Charles, Insomnia, Postman, etc.
+
+function headerArray(headers) {
+  return Object.entries(headers || {}).map(([name, value]) => ({ name, value: String(value) }));
+}
+
+function queryArray(query) {
+  return Object.entries(query || {}).map(([name, value]) => ({ name, value: String(value) }));
+}
+
+export function toHar(records, meta = {}) {
+  const entries = records
+    .filter((r) => r.url && r.method !== 'WS')
+    .map((r) => {
+      const reqBody = r.requestBody;
+      const resBody = r.responseBody;
+      return {
+        startedDateTime: meta.startedAt || new Date(0).toISOString(),
+        time: r.durationMs || 0,
+        request: {
+          method: r.method || 'GET',
+          url: r.url,
+          httpVersion: 'HTTP/1.1',
+          headers: headerArray(r.requestHeaders),
+          queryString: queryArray(r.query),
+          cookies: [],
+          headersSize: -1,
+          bodySize: reqBody ? -1 : 0,
+          ...(reqBody != null
+            ? { postData: { mimeType: 'application/json', text: typeof reqBody === 'string' ? reqBody : JSON.stringify(reqBody) } }
+            : {}),
+        },
+        response: {
+          status: r.status || 0,
+          statusText: r.statusText || '',
+          httpVersion: r.protocol || 'HTTP/1.1',
+          headers: headerArray(r.responseHeaders),
+          cookies: [],
+          content: {
+            size: r.sizeBytes || -1,
+            mimeType: (r.responseHeaders && (r.responseHeaders['content-type'] || r.responseHeaders['Content-Type'])) || '',
+            text: resBody == null ? '' : typeof resBody === 'string' ? resBody : JSON.stringify(resBody),
+          },
+          redirectURL: '',
+          headersSize: -1,
+          bodySize: -1,
+        },
+        cache: {},
+        timings: { send: 0, wait: r.durationMs || 0, receive: 0 },
+      };
+    });
+
+  return {
+    log: {
+      version: '1.2',
+      creator: { name: 'browser-flow-tracker', version: '0.1.0' },
+      pages: [],
+      entries,
+    },
+  };
+}