@taodigital/crawler 0.0.0-stage → 1.2.0

package/dist/network-action-correlator.js

@@ -0,0 +1,209 @@
+function extractUrlPattern(url) {
+  try {
+    const u = new URL(url);
+    const path = u.pathname.replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, ":uuid").replace(/\/[0-9a-f]{24}(\/|$)/g, "/:mongoId$1").replace(/\/\d{6,}(\/|$)/g, "/:id$1").replace(/\/\d{1,5}(\/|$)/g, "/:id$1");
+    return path;
+  } catch {
+    return url.slice(0, 80);
+  }
+}
+function extractBodySchema(bodyText) {
+  if (!bodyText || bodyText.length > 5e4) return null;
+  let parsed;
+  try {
+    parsed = JSON.parse(bodyText);
+  } catch {
+    return null;
+  }
+  const schema = {};
+  let keyCount = 0;
+  function traverse(value, prefix, depth) {
+    if (keyCount >= 30 || depth > 3) return;
+    if (value === null || value === void 0) return;
+    if (Array.isArray(value)) {
+      schema[prefix] = "array";
+      keyCount++;
+      return;
+    }
+    if (typeof value === "object") {
+      for (const [k, v] of Object.entries(value)) {
+        if (keyCount >= 30) break;
+        const key = prefix ? `${prefix}.${k}` : k;
+        if (v === null) {
+          schema[key] = "null";
+          keyCount++;
+        } else if (Array.isArray(v)) {
+          schema[key] = "array";
+          keyCount++;
+        } else if (typeof v === "object") {
+          traverse(v, key, depth + 1);
+        } else {
+          schema[key] = typeof v;
+          keyCount++;
+        }
+      }
+      return;
+    }
+    if (prefix) {
+      schema[prefix] = typeof value;
+      keyCount++;
+    }
+  }
+  traverse(parsed, "", 0);
+  return keyCount > 0 ? schema : null;
+}
+const ANALYTICS_DOMAINS = [
+  "google-analytics",
+  "doubleclick",
+  "hotjar",
+  "mixpanel",
+  "segment",
+  "sentry",
+  "datadog",
+  "newrelic",
+  "amplitude",
+  "fullstory"
+];
+const API_PATH_PATTERNS = [
+  "/api/",
+  "/graphql",
+  "/v1/",
+  "/v2/",
+  "/rest/",
+  "/query",
+  "/rpc",
+  "/trpc"
+];
+const EXCLUDED_RESOURCE_TYPES = /* @__PURE__ */ new Set(["image", "font", "stylesheet", "media", "websocket"]);
+function isApiRequest(url, resourceType) {
+  if (EXCLUDED_RESOURCE_TYPES.has(resourceType)) return false;
+  try {
+    const hostname = new URL(url).hostname;
+    if (ANALYTICS_DOMAINS.some((d) => hostname.includes(d))) return false;
+  } catch {
+    return false;
+  }
+  if (resourceType === "xhr" || resourceType === "fetch") return true;
+  const lower = url.toLowerCase();
+  return API_PATH_PATTERNS.some((p) => lower.includes(p));
+}
+function createCorrelator() {
+  let currentLabel = null;
+  let currentRequests = [];
+  const correlations = [];
+  return {
+    startWindow(actionLabel) {
+      currentLabel = actionLabel;
+      currentRequests = [];
+    },
+    recordRequest(rec) {
+      if (currentLabel === null) return;
+      currentRequests.push({ ...rec, correlatedActionLabel: currentLabel });
+    },
+    endWindow() {
+      const label = currentLabel ?? "(unknown)";
+      const correlation = {
+        actionLabel: label,
+        requests: currentRequests
+      };
+      correlations.push(correlation);
+      currentLabel = null;
+      currentRequests = [];
+      return correlation;
+    },
+    getResult() {
+      const uniqueEndpoints = [
+        ...new Set(
+          correlations.flatMap((c) => c.requests.map((r) => r.urlPattern))
+        )
+      ];
+      const graphqlOperations = [
+        ...new Set(
+          correlations.flatMap((c) => c.requests).filter((r) => r.isGraphQL && r.graphQLOperation).map((r) => r.graphQLOperation)
+        )
+      ];
+      const totalRequestsCaptured = correlations.reduce(
+        (sum, c) => sum + c.requests.length,
+        0
+      );
+      const coverage = correlations.length === 0 ? 0 : correlations.filter((c) => c.requests.length >= 1).length / correlations.length;
+      return {
+        correlations,
+        uniqueEndpoints,
+        graphqlOperations,
+        totalRequestsCaptured,
+        coverage
+      };
+    }
+  };
+}
+function attachNetworkInterceptor(page, correlator) {
+  const pendingRequests = /* @__PURE__ */ new Map();
+  const onRequest = (req) => {
+    const url = req.url();
+    const resourceType = req.resourceType();
+    if (!isApiRequest(url, resourceType)) return;
+    let bodyText = "";
+    try {
+      bodyText = req.postData() ?? "";
+    } catch {
+    }
+    pendingRequests.set(url, { bodyText, resourceType });
+  };
+  const onResponse = async (res) => {
+    const url = res.url();
+    const pending = pendingRequests.get(url);
+    if (!pending) return;
+    pendingRequests.delete(url);
+    const method = res.request().method();
+    let bodySchema = null;
+    try {
+      bodySchema = extractBodySchema(pending.bodyText);
+    } catch {
+    }
+    let responseSchema = null;
+    let isGraphQL = false;
+    let graphQLOperation;
+    const contentType = res.headers()["content-type"] ?? "";
+    if (contentType.includes("application/json")) {
+      try {
+        const respText = await res.text();
+        if (respText.length < 2e4) {
+          responseSchema = extractBodySchema(respText);
+        }
+      } catch {
+      }
+    }
+    try {
+      const bodyParsed = JSON.parse(pending.bodyText);
+      if (typeof bodyParsed?.operationName === "string") {
+        isGraphQL = true;
+        graphQLOperation = bodyParsed.operationName;
+      }
+    } catch {
+    }
+    const rec = {
+      url,
+      urlPattern: extractUrlPattern(url),
+      method,
+      bodySchema,
+      status: res.status(),
+      responseSchema,
+      isGraphQL,
+      graphQLOperation,
+      resourceType: pending.resourceType
+    };
+    correlator.recordRequest(rec);
+  };
+  page.on("request", onRequest);
+  page.on("response", onResponse);
+  return () => {
+    page.off("request", onRequest);
+    page.off("response", onResponse);
+  };
+}
+export {
+  attachNetworkInterceptor,
+  createCorrelator
+};
+//# sourceMappingURL=network-action-correlator.js.map

package/dist/network-action-correlator.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/network-action-correlator.ts"],
+  "sourcesContent": ["/**\n * Network-action correlator \u2014 captures XHR/fetch requests that occur during\n * a named UI action (click, form submit, navigation) and maps them to API\n * endpoint contracts.\n *\n * Usage:\n *   const correlator = createCorrelator();\n *   const detach = attachNetworkInterceptor(page, correlator);\n *   correlator.startWindow('Submit login form');\n *   await page.click('#login-btn');\n *   const correlation = correlator.endWindow();\n *   detach();\n *   const result = correlator.getResult();\n */\n\nimport type { Page, Request, Response } from 'playwright';\n\nexport interface NetworkRequestRecord {\n  url: string;\n  urlPattern: string;\n  method: string;\n  bodySchema: Record<string, string> | null;\n  status: number | null;\n  responseSchema: Record<string, string> | null;\n  isGraphQL: boolean;\n  graphQLOperation?: string;\n  resourceType: string;\n  correlatedActionLabel?: string;\n}\n\nexport interface ActionCorrelation {\n  actionLabel: string;\n  requests: NetworkRequestRecord[];\n}\n\nexport interface NetworkCorrelationResult {\n  correlations: ActionCorrelation[];\n  uniqueEndpoints: string[];\n  graphqlOperations: string[];\n  totalRequestsCaptured: number;\n  coverage: number;\n}\n\n// ---------------------------------------------------------------------------\n// Internal helpers\n// ---------------------------------------------------------------------------\n\nfunction extractUrlPattern(url: string): string {\n  try {\n    const u = new URL(url);\n    const path = u.pathname\n      .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, ':uuid')\n      .replace(/\\/[0-9a-f]{24}(\\/|$)/g, '/:mongoId$1')\n      .replace(/\\/\\d{6,}(\\/|$)/g, '/:id$1')\n      .replace(/\\/\\d{1,5}(\\/|$)/g, '/:id$1');\n    return path;\n  } catch {\n    return url.slice(0, 80);\n  }\n}\n\nfunction extractBodySchema(bodyText: string): Record<string, string> | null {\n  if (!bodyText || bodyText.length > 50_000) return null;\n  let parsed: unknown;\n  try {\n    parsed = JSON.parse(bodyText);\n  } catch {\n    return null;\n  }\n\n  const schema: Record<string, string> = {};\n  let keyCount = 0;\n\n  function traverse(value: unknown, prefix: string, depth: number): void {\n    if (keyCount >= 30 || depth > 3) return;\n    if (value === null || value === undefined) return;\n\n    if (Array.isArray(value)) {\n      schema[prefix] = 'array';\n      keyCount++;\n      return;\n    }\n\n    if (typeof value === 'object') {\n      for (const [k, v] of Object.entries(value as Record<string, unknown>)) {\n        if (keyCount >= 30) break;\n        const key = prefix ? `${prefix}.${k}` : k;\n        if (v === null) {\n          schema[key] = 'null';\n          keyCount++;\n        } else if (Array.isArray(v)) {\n          schema[key] = 'array';\n          keyCount++;\n        } else if (typeof v === 'object') {\n          traverse(v, key, depth + 1);\n        } else {\n          schema[key] = typeof v;\n          keyCount++;\n        }\n      }\n      return;\n    }\n\n    // Primitive at top level\n    if (prefix) {\n      schema[prefix] = typeof value;\n      keyCount++;\n    }\n  }\n\n  traverse(parsed, '', 0);\n  return keyCount > 0 ? schema : null;\n}\n\nconst ANALYTICS_DOMAINS = [\n  'google-analytics',\n  'doubleclick',\n  'hotjar',\n  'mixpanel',\n  'segment',\n  'sentry',\n  'datadog',\n  'newrelic',\n  'amplitude',\n  'fullstory',\n];\n\nconst API_PATH_PATTERNS = [\n  '/api/',\n  '/graphql',\n  '/v1/',\n  '/v2/',\n  '/rest/',\n  '/query',\n  '/rpc',\n  '/trpc',\n];\n\nconst EXCLUDED_RESOURCE_TYPES = new Set(['image', 'font', 'stylesheet', 'media', 'websocket']);\n\nfunction isApiRequest(url: string, resourceType: string): boolean {\n  if (EXCLUDED_RESOURCE_TYPES.has(resourceType)) return false;\n\n  try {\n    const hostname = new URL(url).hostname;\n    if (ANALYTICS_DOMAINS.some((d) => hostname.includes(d))) return false;\n  } catch {\n    return false;\n  }\n\n  if (resourceType === 'xhr' || resourceType === 'fetch') return true;\n\n  const lower = url.toLowerCase();\n  return API_PATH_PATTERNS.some((p) => lower.includes(p));\n}\n\n// ---------------------------------------------------------------------------\n// Correlator\n// ---------------------------------------------------------------------------\n\ninterface NetworkCorrelator {\n  startWindow(actionLabel: string): void;\n  recordRequest(rec: NetworkRequestRecord): void;\n  endWindow(): ActionCorrelation;\n  getResult(): NetworkCorrelationResult;\n}\n\nexport function createCorrelator(): NetworkCorrelator {\n  let currentLabel: string | null = null;\n  let currentRequests: NetworkRequestRecord[] = [];\n  const correlations: ActionCorrelation[] = [];\n\n  return {\n    startWindow(actionLabel: string): void {\n      currentLabel = actionLabel;\n      currentRequests = [];\n    },\n\n    recordRequest(rec: NetworkRequestRecord): void {\n      if (currentLabel === null) return;\n      currentRequests.push({ ...rec, correlatedActionLabel: currentLabel });\n    },\n\n    endWindow(): ActionCorrelation {\n      const label = currentLabel ?? '(unknown)';\n      const correlation: ActionCorrelation = {\n        actionLabel: label,\n        requests: currentRequests,\n      };\n      correlations.push(correlation);\n      currentLabel = null;\n      currentRequests = [];\n      return correlation;\n    },\n\n    getResult(): NetworkCorrelationResult {\n      const uniqueEndpoints = [\n        ...new Set(\n          correlations.flatMap((c) => c.requests.map((r) => r.urlPattern)),\n        ),\n      ];\n\n      const graphqlOperations = [\n        ...new Set(\n          correlations\n            .flatMap((c) => c.requests)\n            .filter((r) => r.isGraphQL && r.graphQLOperation)\n            .map((r) => r.graphQLOperation as string),\n        ),\n      ];\n\n      const totalRequestsCaptured = correlations.reduce(\n        (sum, c) => sum + c.requests.length,\n        0,\n      );\n\n      const coverage =\n        correlations.length === 0\n          ? 0\n          : correlations.filter((c) => c.requests.length >= 1).length /\n            correlations.length;\n\n      return {\n        correlations,\n        uniqueEndpoints,\n        graphqlOperations,\n        totalRequestsCaptured,\n        coverage,\n      };\n    },\n  };\n}\n\n// ---------------------------------------------------------------------------\n// Page interceptor\n// ---------------------------------------------------------------------------\n\nexport function attachNetworkInterceptor(\n  page: Page,\n  correlator: ReturnType<typeof createCorrelator>,\n): () => void {\n  const pendingRequests = new Map<string, { bodyText: string; resourceType: string }>();\n\n  const onRequest = (req: Request): void => {\n    const url = req.url();\n    const resourceType = req.resourceType();\n    if (!isApiRequest(url, resourceType)) return;\n    let bodyText = '';\n    try {\n      bodyText = req.postData() ?? '';\n    } catch {\n      // postData() may throw on non-POST requests in some Playwright versions\n    }\n    pendingRequests.set(url, { bodyText, resourceType });\n  };\n\n  const onResponse = async (res: Response): Promise<void> => {\n    const url = res.url();\n    const pending = pendingRequests.get(url);\n    if (!pending) return;\n    pendingRequests.delete(url);\n\n    const method = res.request().method();\n\n    let bodySchema: Record<string, string> | null = null;\n    try {\n      bodySchema = extractBodySchema(pending.bodyText);\n    } catch {\n      // ignore schema extraction failures\n    }\n\n    let responseSchema: Record<string, string> | null = null;\n    let isGraphQL = false;\n    let graphQLOperation: string | undefined;\n\n    const contentType = res.headers()['content-type'] ?? '';\n    if (contentType.includes('application/json')) {\n      try {\n        const respText = await res.text();\n        if (respText.length < 20_000) {\n          responseSchema = extractBodySchema(respText);\n        }\n      } catch {\n        // response body may no longer be available\n      }\n    }\n\n    // Detect GraphQL from request body\n    try {\n      const bodyParsed = JSON.parse(pending.bodyText) as Record<string, unknown>;\n      if (typeof bodyParsed?.operationName === 'string') {\n        isGraphQL = true;\n        graphQLOperation = bodyParsed.operationName;\n      }\n    } catch {\n      // not JSON or no operationName\n    }\n\n    const rec: NetworkRequestRecord = {\n      url,\n      urlPattern: extractUrlPattern(url),\n      method,\n      bodySchema,\n      status: res.status(),\n      responseSchema,\n      isGraphQL,\n      graphQLOperation,\n      resourceType: pending.resourceType,\n    };\n    correlator.recordRequest(rec);\n  };\n\n  page.on('request', onRequest);\n  page.on('response', onResponse);\n\n  return () => {\n    page.off('request', onRequest);\n    page.off('response', onResponse);\n  };\n}\n"],
+  "mappings": "AA+CA,SAAS,kBAAkB,KAAqB;AAC9C,MAAI;AACF,UAAM,IAAI,IAAI,IAAI,GAAG;AACrB,UAAM,OAAO,EAAE,SACZ,QAAQ,kEAAkE,OAAO,EACjF,QAAQ,yBAAyB,aAAa,EAC9C,QAAQ,mBAAmB,QAAQ,EACnC,QAAQ,oBAAoB,QAAQ;AACvC,WAAO;AAAA,EACT,QAAQ;AACN,WAAO,IAAI,MAAM,GAAG,EAAE;AAAA,EACxB;AACF;AAEA,SAAS,kBAAkB,UAAiD;AAC1E,MAAI,CAAC,YAAY,SAAS,SAAS,IAAQ,QAAO;AAClD,MAAI;AACJ,MAAI;AACF,aAAS,KAAK,MAAM,QAAQ;AAAA,EAC9B,QAAQ;AACN,WAAO;AAAA,EACT;AAEA,QAAM,SAAiC,CAAC;AACxC,MAAI,WAAW;AAEf,WAAS,SAAS,OAAgB,QAAgB,OAAqB;AACrE,QAAI,YAAY,MAAM,QAAQ,EAAG;AACjC,QAAI,UAAU,QAAQ,UAAU,OAAW;AAE3C,QAAI,MAAM,QAAQ,KAAK,GAAG;AACxB,aAAO,MAAM,IAAI;AACjB;AACA;AAAA,IACF;AAEA,QAAI,OAAO,UAAU,UAAU;AAC7B,iBAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,KAAgC,GAAG;AACrE,YAAI,YAAY,GAAI;AACpB,cAAM,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,KAAK;AACxC,YAAI,MAAM,MAAM;AACd,iBAAO,GAAG,IAAI;AACd;AAAA,QACF,WAAW,MAAM,QAAQ,CAAC,GAAG;AAC3B,iBAAO,GAAG,IAAI;AACd;AAAA,QACF,WAAW,OAAO,MAAM,UAAU;AAChC,mBAAS,GAAG,KAAK,QAAQ,CAAC;AAAA,QAC5B,OAAO;AACL,iBAAO,GAAG,IAAI,OAAO;AACrB;AAAA,QACF;AAAA,MACF;AACA;AAAA,IACF;AAGA,QAAI,QAAQ;AACV,aAAO,MAAM,IAAI,OAAO;AACxB;AAAA,IACF;AAAA,EACF;AAEA,WAAS,QAAQ,IAAI,CAAC;AACtB,SAAO,WAAW,IAAI,SAAS;AACjC;AAEA,MAAM,oBAAoB;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,oBAAoB;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,0BAA0B,oBAAI,IAAI,CAAC,SAAS,QAAQ,cAAc,SAAS,WAAW,CAAC;AAE7F,SAAS,aAAa,KAAa,cAA+B;AAChE,MAAI,wBAAwB,IAAI,YAAY,EAAG,QAAO;AAEtD,MAAI;AACF,UAAM,WAAW,IAAI,IAAI,GAAG,EAAE;AAC9B,QAAI,kBAAkB,KAAK,CAAC,MAAM,SAAS,SAAS,CAAC,CAAC,EAAG,QAAO;AAAA,EAClE,QAAQ;AACN,WAAO;AAAA,EACT;AAEA,MAAI,iBAAiB,SAAS,iBAAiB,QAAS,QAAO;AAE/D,QAAM,QAAQ,IAAI,YAAY;AAC9B,SAAO,kBAAkB,KAAK,CAAC,MAAM,MAAM,SAAS,CAAC,CAAC;AACxD;AAaO,SAAS,mBAAsC;AACpD,MAAI,eAA8B;AAClC,MAAI,kBAA0C,CAAC;AAC/C,QAAM,eAAoC,CAAC;AAE3C,SAAO;AAAA,IACL,YAAY,aAA2B;AACrC,qBAAe;AACf,wBAAkB,CAAC;AAAA,IACrB;AAAA,IAEA,cAAc,KAAiC;AAC7C,UAAI,iBAAiB,KAAM;AAC3B,sBAAgB,KAAK,EAAE,GAAG,KAAK,uBAAuB,aAAa,CAAC;AAAA,IACtE;AAAA,IAEA,YAA+B;AAC7B,YAAM,QAAQ,gBAAgB;AAC9B,YAAM,cAAiC;AAAA,QACrC,aAAa;AAAA,QACb,UAAU;AAAA,MACZ;AACA,mBAAa,KAAK,WAAW;AAC7B,qBAAe;AACf,wBAAkB,CAAC;AACnB,aAAO;AAAA,IACT;AAAA,IAEA,YAAsC;AACpC,YAAM,kBAAkB;AAAA,QACtB,GAAG,IAAI;AAAA,UACL,aAAa,QAAQ,CAAC,MAAM,EAAE,SAAS,IAAI,CAAC,MAAM,EAAE,UAAU,CAAC;AAAA,QACjE;AAAA,MACF;AAEA,YAAM,oBAAoB;AAAA,QACxB,GAAG,IAAI;AAAA,UACL,aACG,QAAQ,CAAC,MAAM,EAAE,QAAQ,EACzB,OAAO,CAAC,MAAM,EAAE,aAAa,EAAE,gBAAgB,EAC/C,IAAI,CAAC,MAAM,EAAE,gBAA0B;AAAA,QAC5C;AAAA,MACF;AAEA,YAAM,wBAAwB,aAAa;AAAA,QACzC,CAAC,KAAK,MAAM,MAAM,EAAE,SAAS;AAAA,QAC7B;AAAA,MACF;AAEA,YAAM,WACJ,aAAa,WAAW,IACpB,IACA,aAAa,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,CAAC,EAAE,SACnD,aAAa;AAEnB,aAAO;AAAA,QACL;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,QACA;AAAA,MACF;AAAA,IACF;AAAA,EACF;AACF;AAMO,SAAS,yBACd,MACA,YACY;AACZ,QAAM,kBAAkB,oBAAI,IAAwD;AAEpF,QAAM,YAAY,CAAC,QAAuB;AACxC,UAAM,MAAM,IAAI,IAAI;AACpB,UAAM,eAAe,IAAI,aAAa;AACtC,QAAI,CAAC,aAAa,KAAK,YAAY,EAAG;AACtC,QAAI,WAAW;AACf,QAAI;AACF,iBAAW,IAAI,SAAS,KAAK;AAAA,IAC/B,QAAQ;AAAA,IAER;AACA,oBAAgB,IAAI,KAAK,EAAE,UAAU,aAAa,CAAC;AAAA,EACrD;AAEA,QAAM,aAAa,OAAO,QAAiC;AACzD,UAAM,MAAM,IAAI,IAAI;AACpB,UAAM,UAAU,gBAAgB,IAAI,GAAG;AACvC,QAAI,CAAC,QAAS;AACd,oBAAgB,OAAO,GAAG;AAE1B,UAAM,SAAS,IAAI,QAAQ,EAAE,OAAO;AAEpC,QAAI,aAA4C;AAChD,QAAI;AACF,mBAAa,kBAAkB,QAAQ,QAAQ;AAAA,IACjD,QAAQ;AAAA,IAER;AAEA,QAAI,iBAAgD;AACpD,QAAI,YAAY;AAChB,QAAI;AAEJ,UAAM,cAAc,IAAI,QAAQ,EAAE,cAAc,KAAK;AACrD,QAAI,YAAY,SAAS,kBAAkB,GAAG;AAC5C,UAAI;AACF,cAAM,WAAW,MAAM,IAAI,KAAK;AAChC,YAAI,SAAS,SAAS,KAAQ;AAC5B,2BAAiB,kBAAkB,QAAQ;AAAA,QAC7C;AAAA,MACF,QAAQ;AAAA,MAER;AAAA,IACF;AAGA,QAAI;AACF,YAAM,aAAa,KAAK,MAAM,QAAQ,QAAQ;AAC9C,UAAI,OAAO,YAAY,kBAAkB,UAAU;AACjD,oBAAY;AACZ,2BAAmB,WAAW;AAAA,MAChC;AAAA,IACF,QAAQ;AAAA,IAER;AAEA,UAAM,MAA4B;AAAA,MAChC;AAAA,MACA,YAAY,kBAAkB,GAAG;AAAA,MACjC;AAAA,MACA;AAAA,MACA,QAAQ,IAAI,OAAO;AAAA,MACnB;AAAA,MACA;AAAA,MACA;AAAA,MACA,cAAc,QAAQ;AAAA,IACxB;AACA,eAAW,cAAc,GAAG;AAAA,EAC9B;AAEA,OAAK,GAAG,WAAW,SAAS;AAC5B,OAAK,GAAG,YAAY,UAAU;AAE9B,SAAO,MAAM;AACX,SAAK,IAAI,WAAW,SAAS;AAC7B,SAAK,IAAI,YAAY,UAAU;AAAA,EACjC;AACF;",
+  "names": []
+}

package/dist/network-observer.d.ts

@@ -0,0 +1,49 @@
+import type { BrowserContext } from 'playwright';
+export interface ApiEndpointObservationInput {
+    method: string;
+    urlPattern: string;
+    statusCode?: number;
+    requestKind?: string;
+    responseShape?: unknown;
+    durationMs?: number;
+    errorBody?: string;
+}
+export declare function attachNetworkObserver(context: BrowserContext, sink: ApiEndpointObservationInput[], onPageUrlDiscovered?: (relPath: string) => void): void;
+export interface HarEntry {
+    startedAt: string;
+    method: string;
+    url: string;
+    status: number;
+    statusText: string;
+    requestHeaders: Record<string, string>;
+    responseHeaders: Record<string, string>;
+    contentType: string;
+    bodySize: number;
+    durationMs: number;
+    timings: {
+        send: number;
+        wait: number;
+        receive: number;
+    };
+}
+export interface HarLog {
+    version: '1.2';
+    creator: {
+        name: 'ZeTa-Crawler';
+        version: '1.0';
+    };
+    pages: Array<{
+        id: string;
+        title: string;
+        startedAt: string;
+    }>;
+    entries: HarEntry[];
+}
+/**
+ * Full HAR capture via CDP — intercepts ALL network traffic, not just API endpoints.
+ * Returns a controller with getHar() and detach().
+ */
+export declare function attachHarCapture(cdpSession: any, pageTitle?: string): {
+    getHar(): HarLog;
+    detach(): void;
+};

package/dist/network-observer.js

@@ -0,0 +1,149 @@
+import { redactJson } from "./policies/crawl-safety.js";
+const API_PATTERN = /\/(api|graphql|gql|rest|v\d+)\//i;
+const ASSET_EXT = /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot|map|json\.map)(\?|$)/i;
+function normalizeApiUrl(url) {
+  const u = new URL(url);
+  return u.pathname.replace(/\/[a-f0-9-]{8,}(?=\/|$)/gi, "/{id}").replace(/\/c[a-z0-9]{20,}(?=\/|$)/g, "/{id}").replace(/\/\d+(?=\/|$)/g, "/{id}");
+}
+function inferShape(value) {
+  if (Array.isArray(value)) return value.length > 0 ? [inferShape(value[0])] : [];
+  if (value && typeof value === "object") {
+    const out = {};
+    for (const [key, item] of Object.entries(value).slice(0, 40)) out[key] = inferShape(item);
+    return out;
+  }
+  return typeof value;
+}
+function extractPageUrls(jsonStr) {
+  const PAGE_URL = /"(?:url|href|path|link|route|to|src|redirect)"\s*:\s*"(\/[\w\-\/\.~@%]+)"/g;
+  const ASSET = /\.(png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|css|js|map|json)$/i;
+  const API_PATH = /^\/(api|v\d+|graphql|gql|rest|rpc|static|_next|assets)\//i;
+  const found = /* @__PURE__ */ new Set();
+  let m;
+  PAGE_URL.lastIndex = 0;
+  while ((m = PAGE_URL.exec(jsonStr)) !== null) {
+    const u = m[1];
+    if (u && u.length > 1 && u.length < 120 && !ASSET.test(u) && !API_PATH.test(u)) found.add(u);
+  }
+  return [...found];
+}
+function attachNetworkObserver(context, sink, onPageUrlDiscovered) {
+  const seen = /* @__PURE__ */ new Map();
+  const requestTimes = /* @__PURE__ */ new Map();
+  context.on("request", (request) => {
+    try {
+      const url = request.url();
+      const rtype = request.resourceType();
+      if (rtype !== "xhr" && rtype !== "fetch" || !API_PATTERN.test(url) || ASSET_EXT.test(url)) return;
+      const method = request.method();
+      const normalized = normalizeApiUrl(url);
+      const key = `${method}:${normalized}`;
+      requestTimes.set(url, Date.now());
+      if (!seen.has(key)) {
+        const row = { method, urlPattern: normalized, requestKind: rtype };
+        seen.set(key, row);
+        sink.push(row);
+      }
+    } catch {
+    }
+  });
+  context.on("response", async (response) => {
+    try {
+      const request = response.request();
+      const url = request.url();
+      const rtype = request.resourceType();
+      if (rtype !== "xhr" && rtype !== "fetch" || !API_PATTERN.test(url) || ASSET_EXT.test(url)) return;
+      const normalized = normalizeApiUrl(url);
+      const key = `${request.method()}:${normalized}`;
+      const durationMs = Date.now() - (requestTimes.get(url) ?? Date.now());
+      let row = seen.get(key);
+      if (!row) {
+        row = { method: request.method(), urlPattern: normalized, requestKind: rtype };
+        seen.set(key, row);
+        sink.push(row);
+      }
+      row.statusCode = response.status();
+      row.durationMs = durationMs;
+      const contentType = response.headers()["content-type"] ?? "";
+      if (contentType.includes("application/json")) {
+        const body = await response.json().catch(() => void 0);
+        if (body !== void 0) {
+          row.responseShape = redactJson(inferShape(body));
+          if (onPageUrlDiscovered) {
+            for (const u of extractPageUrls(JSON.stringify(body))) onPageUrlDiscovered(u);
+          }
+        }
+      }
+      if (response.status() >= 400) {
+        try {
+          const errText = await response.text();
+          row.errorBody = errText.slice(0, 300);
+        } catch {
+        }
+      }
+    } catch {
+    }
+  });
+}
+function attachHarCapture(cdpSession, pageTitle = "Page") {
+  const entries = [];
+  const pending = /* @__PURE__ */ new Map();
+  const pageId = `page_${String(Date.now())}`;
+  const pageStarted = (/* @__PURE__ */ new Date()).toISOString();
+  const onRequest = (evt) => {
+    const { requestId, request } = evt ?? {};
+    if (!requestId || !request) return;
+    const headers = {};
+    for (const h of request.headers ?? []) if (h.name) headers[String(h.name).toLowerCase()] = String(h.value ?? "");
+    pending.set(requestId, { method: request.method ?? "GET", url: request.url ?? "", headers, startMs: Date.now() });
+  };
+  const onResponse = (evt) => {
+    const { requestId, response } = evt ?? {};
+    if (!requestId || !response) return;
+    const req = pending.get(requestId);
+    if (!req) return;
+    pending.delete(requestId);
+    const durationMs = Date.now() - req.startMs;
+    const responseHeaders = {};
+    for (const h of response.headers ?? []) if (h.name) responseHeaders[String(h.name).toLowerCase()] = String(h.value ?? "");
+    entries.push({
+      startedAt: new Date(req.startMs).toISOString(),
+      method: req.method,
+      url: req.url,
+      status: response.status ?? 0,
+      statusText: response.statusText ?? "",
+      requestHeaders: req.headers,
+      responseHeaders,
+      contentType: responseHeaders["content-type"] ?? "",
+      bodySize: response.encodedDataLength ?? -1,
+      durationMs,
+      timings: { send: 0, wait: Math.round(durationMs * 0.9), receive: Math.round(durationMs * 0.1) }
+    });
+  };
+  if (cdpSession?.on) {
+    cdpSession.on("Network.requestWillBeSent", onRequest);
+    cdpSession.on("Network.responseReceived", onResponse);
+  }
+  return {
+    getHar() {
+      return {
+        version: "1.2",
+        creator: { name: "ZeTa-Crawler", version: "1.0" },
+        pages: [{ id: pageId, title: pageTitle, startedAt: pageStarted }],
+        entries: [...entries]
+      };
+    },
+    detach() {
+      if (cdpSession?.off) {
+        cdpSession.off("Network.requestWillBeSent", onRequest);
+        cdpSession.off("Network.responseReceived", onResponse);
+      }
+      pending.clear();
+    }
+  };
+}
+export {
+  attachHarCapture,
+  attachNetworkObserver
+};
+//# sourceMappingURL=network-observer.js.map

package/dist/network-observer.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/network-observer.ts"],
+  "sourcesContent": ["import type { BrowserContext } from 'playwright';\nimport { redactJson } from './policies/crawl-safety.js';\n\nexport interface ApiEndpointObservationInput {\n  method: string;\n  urlPattern: string;\n  statusCode?: number;\n  requestKind?: string;\n  responseShape?: unknown;\n  durationMs?: number;\n  errorBody?: string;\n}\n\nconst API_PATTERN = /\\/(api|graphql|gql|rest|v\\d+)\\//i;\nconst ASSET_EXT = /\\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot|map|json\\.map)(\\?|$)/i;\n\nfunction normalizeApiUrl(url: string): string {\n  const u = new URL(url);\n  return u.pathname\n    .replace(/\\/[a-f0-9-]{8,}(?=\\/|$)/gi, '/{id}')\n    .replace(/\\/c[a-z0-9]{20,}(?=\\/|$)/g, '/{id}')\n    .replace(/\\/\\d+(?=\\/|$)/g, '/{id}');\n}\n\nfunction inferShape(value: unknown): unknown {\n  if (Array.isArray(value)) return value.length > 0 ? [inferShape(value[0])] : [];\n  if (value && typeof value === 'object') {\n    const out: Record<string, unknown> = {};\n    for (const [key, item] of Object.entries(value).slice(0, 40)) out[key] = inferShape(item);\n    return out;\n  }\n  return typeof value;\n}\n\n/** P4: Extract page-URL candidates from a JSON response body string. */\nfunction extractPageUrls(jsonStr: string): string[] {\n  const PAGE_URL = /\"(?:url|href|path|link|route|to|src|redirect)\"\\s*:\\s*\"(\\/[\\w\\-\\/\\.~@%]+)\"/g;\n  const ASSET = /\\.(png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|css|js|map|json)$/i;\n  const API_PATH = /^\\/(api|v\\d+|graphql|gql|rest|rpc|static|_next|assets)\\//i;\n  const found = new Set<string>();\n  let m: RegExpExecArray | null;\n  PAGE_URL.lastIndex = 0;\n  while ((m = PAGE_URL.exec(jsonStr)) !== null) {\n    const u = m[1];\n    if (u && u.length > 1 && u.length < 120 && !ASSET.test(u) && !API_PATH.test(u)) found.add(u);\n  }\n  return [...found];\n}\n\nexport function attachNetworkObserver(\n  context: BrowserContext,\n  sink: ApiEndpointObservationInput[],\n  onPageUrlDiscovered?: (relPath: string) => void,\n) {\n  const seen = new Map<string, ApiEndpointObservationInput>();\n  const requestTimes = new Map<string, number>();\n\n  context.on('request', (request) => {\n    try {\n      const url = request.url();\n      const rtype = request.resourceType();\n      if ((rtype !== 'xhr' && rtype !== 'fetch') || !API_PATTERN.test(url) || ASSET_EXT.test(url)) return;\n      const method = request.method();\n      const normalized = normalizeApiUrl(url);\n      const key = `${method}:${normalized}`;\n      requestTimes.set(url, Date.now());\n      if (!seen.has(key)) {\n        const row = { method, urlPattern: normalized, requestKind: rtype };\n        seen.set(key, row);\n        sink.push(row);\n      }\n    } catch {\n      // non-fatal\n    }\n  });\n\n  context.on('response', async (response) => {\n    try {\n      const request = response.request();\n      const url = request.url();\n      const rtype = request.resourceType();\n      if ((rtype !== 'xhr' && rtype !== 'fetch') || !API_PATTERN.test(url) || ASSET_EXT.test(url)) return;\n      const normalized = normalizeApiUrl(url);\n      const key = `${request.method()}:${normalized}`;\n      const durationMs = Date.now() - (requestTimes.get(url) ?? Date.now());\n      let row = seen.get(key);\n      if (!row) {\n        row = { method: request.method(), urlPattern: normalized, requestKind: rtype };\n        seen.set(key, row);\n        sink.push(row);\n      }\n      row.statusCode = response.status();\n      row.durationMs = durationMs;\n      const contentType = response.headers()['content-type'] ?? '';\n      if (contentType.includes('application/json')) {\n        const body = await response.json().catch(() => undefined);\n        if (body !== undefined) {\n          row.responseShape = redactJson(inferShape(body));\n          // P4: scan JSON body for page-URL candidates (non-API paths)\n          if (onPageUrlDiscovered) {\n            for (const u of extractPageUrls(JSON.stringify(body))) onPageUrlDiscovered(u);\n          }\n        }\n      }\n      if (response.status() >= 400) {\n        try {\n          const errText = await response.text();\n          row.errorBody = errText.slice(0, 300);\n        } catch { /* non-fatal */ }\n      }\n    } catch {\n      // non-fatal\n    }\n  });\n}\n\n// \u2500\u2500 Full HAR capture \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\nexport interface HarEntry {\n  startedAt: string;\n  method: string;\n  url: string;\n  status: number;\n  statusText: string;\n  requestHeaders: Record<string, string>;\n  responseHeaders: Record<string, string>;\n  contentType: string;\n  bodySize: number;\n  durationMs: number;\n  timings: { send: number; wait: number; receive: number };\n}\n\nexport interface HarLog {\n  version: '1.2';\n  creator: { name: 'ZeTa-Crawler'; version: '1.0' };\n  pages: Array<{ id: string; title: string; startedAt: string }>;\n  entries: HarEntry[];\n}\n\n/**\n * Full HAR capture via CDP \u2014 intercepts ALL network traffic, not just API endpoints.\n * Returns a controller with getHar() and detach().\n */\nexport function attachHarCapture(cdpSession: any, pageTitle = 'Page'): { getHar(): HarLog; detach(): void } {\n  const entries: HarEntry[] = [];\n  const pending = new Map<string, { method: string; url: string; headers: Record<string, string>; startMs: number }>();\n  const pageId = `page_${String(Date.now())}`;\n  const pageStarted = new Date().toISOString();\n\n  const onRequest = (evt: any) => {\n    const { requestId, request } = evt ?? {};\n    if (!requestId || !request) return;\n    const headers: Record<string, string> = {};\n    for (const h of (request.headers ?? [])) if (h.name) headers[String(h.name).toLowerCase()] = String(h.value ?? '');\n    pending.set(requestId, { method: request.method ?? 'GET', url: request.url ?? '', headers, startMs: Date.now() });\n  };\n\n  const onResponse = (evt: any) => {\n    const { requestId, response } = evt ?? {};\n    if (!requestId || !response) return;\n    const req = pending.get(requestId);\n    if (!req) return;\n    pending.delete(requestId);\n    const durationMs = Date.now() - req.startMs;\n    const responseHeaders: Record<string, string> = {};\n    for (const h of (response.headers ?? [])) if (h.name) responseHeaders[String(h.name).toLowerCase()] = String(h.value ?? '');\n    entries.push({\n      startedAt: new Date(req.startMs).toISOString(),\n      method: req.method,\n      url: req.url,\n      status: response.status ?? 0,\n      statusText: response.statusText ?? '',\n      requestHeaders: req.headers,\n      responseHeaders,\n      contentType: responseHeaders['content-type'] ?? '',\n      bodySize: response.encodedDataLength ?? -1,\n      durationMs,\n      timings: { send: 0, wait: Math.round(durationMs * 0.9), receive: Math.round(durationMs * 0.1) },\n    });\n  };\n\n  if (cdpSession?.on) {\n    cdpSession.on('Network.requestWillBeSent', onRequest);\n    cdpSession.on('Network.responseReceived', onResponse);\n  }\n\n  return {\n    getHar(): HarLog {\n      return {\n        version: '1.2',\n        creator: { name: 'ZeTa-Crawler', version: '1.0' },\n        pages: [{ id: pageId, title: pageTitle, startedAt: pageStarted }],\n        entries: [...entries],\n      };\n    },\n    detach() {\n      if (cdpSession?.off) {\n        cdpSession.off('Network.requestWillBeSent', onRequest);\n        cdpSession.off('Network.responseReceived', onResponse);\n      }\n      pending.clear();\n    },\n  };\n}\n"],
+  "mappings": "AACA,SAAS,kBAAkB;AAY3B,MAAM,cAAc;AACpB,MAAM,YAAY;AAElB,SAAS,gBAAgB,KAAqB;AAC5C,QAAM,IAAI,IAAI,IAAI,GAAG;AACrB,SAAO,EAAE,SACN,QAAQ,6BAA6B,OAAO,EAC5C,QAAQ,6BAA6B,OAAO,EAC5C,QAAQ,kBAAkB,OAAO;AACtC;AAEA,SAAS,WAAW,OAAyB;AAC3C,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,MAAM,SAAS,IAAI,CAAC,WAAW,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AAC9E,MAAI,SAAS,OAAO,UAAU,UAAU;AACtC,UAAM,MAA+B,CAAC;AACtC,eAAW,CAAC,KAAK,IAAI,KAAK,OAAO,QAAQ,KAAK,EAAE,MAAM,GAAG,EAAE,EAAG,KAAI,GAAG,IAAI,WAAW,IAAI;AACxF,WAAO;AAAA,EACT;AACA,SAAO,OAAO;AAChB;AAGA,SAAS,gBAAgB,SAA2B;AAClD,QAAM,WAAW;AACjB,QAAM,QAAQ;AACd,QAAM,WAAW;AACjB,QAAM,QAAQ,oBAAI,IAAY;AAC9B,MAAI;AACJ,WAAS,YAAY;AACrB,UAAQ,IAAI,SAAS,KAAK,OAAO,OAAO,MAAM;AAC5C,UAAM,IAAI,EAAE,CAAC;AACb,QAAI,KAAK,EAAE,SAAS,KAAK,EAAE,SAAS,OAAO,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,SAAS,KAAK,CAAC,EAAG,OAAM,IAAI,CAAC;AAAA,EAC7F;AACA,SAAO,CAAC,GAAG,KAAK;AAClB;AAEO,SAAS,sBACd,SACA,MACA,qBACA;AACA,QAAM,OAAO,oBAAI,IAAyC;AAC1D,QAAM,eAAe,oBAAI,IAAoB;AAE7C,UAAQ,GAAG,WAAW,CAAC,YAAY;AACjC,QAAI;AACF,YAAM,MAAM,QAAQ,IAAI;AACxB,YAAM,QAAQ,QAAQ,aAAa;AACnC,UAAK,UAAU,SAAS,UAAU,WAAY,CAAC,YAAY,KAAK,GAAG,KAAK,UAAU,KAAK,GAAG,EAAG;AAC7F,YAAM,SAAS,QAAQ,OAAO;AAC9B,YAAM,aAAa,gBAAgB,GAAG;AACtC,YAAM,MAAM,GAAG,MAAM,IAAI,UAAU;AACnC,mBAAa,IAAI,KAAK,KAAK,IAAI,CAAC;AAChC,UAAI,CAAC,KAAK,IAAI,GAAG,GAAG;AAClB,cAAM,MAAM,EAAE,QAAQ,YAAY,YAAY,aAAa,MAAM;AACjE,aAAK,IAAI,KAAK,GAAG;AACjB,aAAK,KAAK,GAAG;AAAA,MACf;AAAA,IACF,QAAQ;AAAA,IAER;AAAA,EACF,CAAC;AAED,UAAQ,GAAG,YAAY,OAAO,aAAa;AACzC,QAAI;AACF,YAAM,UAAU,SAAS,QAAQ;AACjC,YAAM,MAAM,QAAQ,IAAI;AACxB,YAAM,QAAQ,QAAQ,aAAa;AACnC,UAAK,UAAU,SAAS,UAAU,WAAY,CAAC,YAAY,KAAK,GAAG,KAAK,UAAU,KAAK,GAAG,EAAG;AAC7F,YAAM,aAAa,gBAAgB,GAAG;AACtC,YAAM,MAAM,GAAG,QAAQ,OAAO,CAAC,IAAI,UAAU;AAC7C,YAAM,aAAa,KAAK,IAAI,KAAK,aAAa,IAAI,GAAG,KAAK,KAAK,IAAI;AACnE,UAAI,MAAM,KAAK,IAAI,GAAG;AACtB,UAAI,CAAC,KAAK;AACR,cAAM,EAAE,QAAQ,QAAQ,OAAO,GAAG,YAAY,YAAY,aAAa,MAAM;AAC7E,aAAK,IAAI,KAAK,GAAG;AACjB,aAAK,KAAK,GAAG;AAAA,MACf;AACA,UAAI,aAAa,SAAS,OAAO;AACjC,UAAI,aAAa;AACjB,YAAM,cAAc,SAAS,QAAQ,EAAE,cAAc,KAAK;AAC1D,UAAI,YAAY,SAAS,kBAAkB,GAAG;AAC5C,cAAM,OAAO,MAAM,SAAS,KAAK,EAAE,MAAM,MAAM,MAAS;AACxD,YAAI,SAAS,QAAW;AACtB,cAAI,gBAAgB,WAAW,WAAW,IAAI,CAAC;AAE/C,cAAI,qBAAqB;AACvB,uBAAW,KAAK,gBAAgB,KAAK,UAAU,IAAI,CAAC,EAAG,qBAAoB,CAAC;AAAA,UAC9E;AAAA,QACF;AAAA,MACF;AACA,UAAI,SAAS,OAAO,KAAK,KAAK;AAC5B,YAAI;AACF,gBAAM,UAAU,MAAM,SAAS,KAAK;AACpC,cAAI,YAAY,QAAQ,MAAM,GAAG,GAAG;AAAA,QACtC,QAAQ;AAAA,QAAkB;AAAA,MAC5B;AAAA,IACF,QAAQ;AAAA,IAER;AAAA,EACF,CAAC;AACH;AA6BO,SAAS,iBAAiB,YAAiB,YAAY,QAA8C;AAC1G,QAAM,UAAsB,CAAC;AAC7B,QAAM,UAAU,oBAAI,IAA+F;AACnH,QAAM,SAAS,QAAQ,OAAO,KAAK,IAAI,CAAC,CAAC;AACzC,QAAM,eAAc,oBAAI,KAAK,GAAE,YAAY;AAE3C,QAAM,YAAY,CAAC,QAAa;AAC9B,UAAM,EAAE,WAAW,QAAQ,IAAI,OAAO,CAAC;AACvC,QAAI,CAAC,aAAa,CAAC,QAAS;AAC5B,UAAM,UAAkC,CAAC;AACzC,eAAW,KAAM,QAAQ,WAAW,CAAC,EAAI,KAAI,EAAE,KAAM,SAAQ,OAAO,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,OAAO,EAAE,SAAS,EAAE;AACjH,YAAQ,IAAI,WAAW,EAAE,QAAQ,QAAQ,UAAU,OAAO,KAAK,QAAQ,OAAO,IAAI,SAAS,SAAS,KAAK,IAAI,EAAE,CAAC;AAAA,EAClH;AAEA,QAAM,aAAa,CAAC,QAAa;AAC/B,UAAM,EAAE,WAAW,SAAS,IAAI,OAAO,CAAC;AACxC,QAAI,CAAC,aAAa,CAAC,SAAU;AAC7B,UAAM,MAAM,QAAQ,IAAI,SAAS;AACjC,QAAI,CAAC,IAAK;AACV,YAAQ,OAAO,SAAS;AACxB,UAAM,aAAa,KAAK,IAAI,IAAI,IAAI;AACpC,UAAM,kBAA0C,CAAC;AACjD,eAAW,KAAM,SAAS,WAAW,CAAC,EAAI,KAAI,EAAE,KAAM,iBAAgB,OAAO,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,OAAO,EAAE,SAAS,EAAE;AAC1H,YAAQ,KAAK;AAAA,MACX,WAAW,IAAI,KAAK,IAAI,OAAO,EAAE,YAAY;AAAA,MAC7C,QAAQ,IAAI;AAAA,MACZ,KAAK,IAAI;AAAA,MACT,QAAQ,SAAS,UAAU;AAAA,MAC3B,YAAY,SAAS,cAAc;AAAA,MACnC,gBAAgB,IAAI;AAAA,MACpB;AAAA,MACA,aAAa,gBAAgB,cAAc,KAAK;AAAA,MAChD,UAAU,SAAS,qBAAqB;AAAA,MACxC;AAAA,MACA,SAAS,EAAE,MAAM,GAAG,MAAM,KAAK,MAAM,aAAa,GAAG,GAAG,SAAS,KAAK,MAAM,aAAa,GAAG,EAAE;AAAA,IAChG,CAAC;AAAA,EACH;AAEA,MAAI,YAAY,IAAI;AAClB,eAAW,GAAG,6BAA6B,SAAS;AACpD,eAAW,GAAG,4BAA4B,UAAU;AAAA,EACtD;AAEA,SAAO;AAAA,IACL,SAAiB;AACf,aAAO;AAAA,QACL,SAAS;AAAA,QACT,SAAS,EAAE,MAAM,gBAAgB,SAAS,MAAM;AAAA,QAChD,OAAO,CAAC,EAAE,IAAI,QAAQ,OAAO,WAAW,WAAW,YAAY,CAAC;AAAA,QAChE,SAAS,CAAC,GAAG,OAAO;AAAA,MACtB;AAAA,IACF;AAAA,IACA,SAAS;AACP,UAAI,YAAY,KAAK;AACnB,mBAAW,IAAI,6BAA6B,SAAS;AACrD,mBAAW,IAAI,4BAA4B,UAAU;AAAA,MACvD;AACA,cAAQ,MAAM;AAAA,IAChB;AAAA,EACF;AACF;",
+  "names": []
+}

package/dist/network-throttle.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Network throttle simulation via CDP.
+ *
+ * Simulates different network conditions during crawl to test how the app
+ * behaves under slow connections. Useful for detecting loading states, error
+ * handling, and performance degradation on poor networks.
+ *
+ * Conditions mirror Chrome DevTools presets:
+ * - slow3g: ~500kbps download, high latency
+ * - fast3g: ~1.6Mbps download, medium latency
+ * - 4g: ~9Mbps download, low latency
+ * - offline: no connectivity
+ * - none: no throttle (default)
+ */
+export type NetworkConditionPreset = 'none' | 'slow3g' | 'fast3g' | '4g' | 'offline';
+export declare function applyNetworkThrottle(cdpSession: any, preset: NetworkConditionPreset): Promise<void>;
+export declare function clearNetworkThrottle(cdpSession: any): Promise<void>;
+export interface ThrottledCrawlOpts {
+    /** Network preset to apply during this URL's crawl */
+    preset: NetworkConditionPreset;
+    /** Restore to 'none' after navigation (default: true) */
+    restoreAfter?: boolean;
+}
+/** Apply throttle, run fn, restore. Returns fn's result. */
+export declare function withNetworkThrottle<T>(cdpSession: any, preset: NetworkConditionPreset, fn: () => Promise<T>, restoreAfter?: boolean): Promise<T>;

package/dist/network-throttle.js

@@ -0,0 +1,34 @@
+const PRESETS = {
+  none: { offline: false, downloadThroughput: -1, uploadThroughput: -1, latency: 0 },
+  slow3g: { offline: false, downloadThroughput: 500 * 1024 / 8, uploadThroughput: 500 * 1024 / 8, latency: 2e3 },
+  fast3g: { offline: false, downloadThroughput: 1.6 * 1024 * 1024 / 8, uploadThroughput: 750 * 1024 / 8, latency: 562.5 },
+  "4g": { offline: false, downloadThroughput: 9 * 1024 * 1024 / 8, uploadThroughput: 2.67 * 1024 * 1024 / 8, latency: 85 },
+  offline: { offline: true, downloadThroughput: 0, uploadThroughput: 0, latency: 0 }
+};
+async function applyNetworkThrottle(cdpSession, preset) {
+  if (!cdpSession?.send) return;
+  const condition = PRESETS[preset];
+  try {
+    await cdpSession.send("Network.enable");
+    await cdpSession.send("Network.emulateNetworkConditions", condition);
+  } catch {
+  }
+}
+async function clearNetworkThrottle(cdpSession) {
+  await applyNetworkThrottle(cdpSession, "none");
+}
+async function withNetworkThrottle(cdpSession, preset, fn, restoreAfter = true) {
+  await applyNetworkThrottle(cdpSession, preset);
+  try {
+    return await fn();
+  } finally {
+    if (restoreAfter) await clearNetworkThrottle(cdpSession).catch(() => {
+    });
+  }
+}
+export {
+  applyNetworkThrottle,
+  clearNetworkThrottle,
+  withNetworkThrottle
+};
+//# sourceMappingURL=network-throttle.js.map

package/dist/network-throttle.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/network-throttle.ts"],
+  "sourcesContent": ["/**\n * Network throttle simulation via CDP.\n *\n * Simulates different network conditions during crawl to test how the app\n * behaves under slow connections. Useful for detecting loading states, error\n * handling, and performance degradation on poor networks.\n *\n * Conditions mirror Chrome DevTools presets:\n * - slow3g: ~500kbps download, high latency\n * - fast3g: ~1.6Mbps download, medium latency\n * - 4g: ~9Mbps download, low latency\n * - offline: no connectivity\n * - none: no throttle (default)\n */\n\nexport type NetworkConditionPreset = 'none' | 'slow3g' | 'fast3g' | '4g' | 'offline';\n\ninterface NetworkCondition {\n  offline: boolean;\n  downloadThroughput: number; // bytes/s\n  uploadThroughput: number;   // bytes/s\n  latency: number;            // ms\n}\n\nconst PRESETS: Record<NetworkConditionPreset, NetworkCondition> = {\n  none:    { offline: false, downloadThroughput: -1, uploadThroughput: -1, latency: 0 },\n  slow3g:  { offline: false, downloadThroughput: 500 * 1024 / 8, uploadThroughput: 500 * 1024 / 8, latency: 2_000 },\n  fast3g:  { offline: false, downloadThroughput: 1.6 * 1024 * 1024 / 8, uploadThroughput: 750 * 1024 / 8, latency: 562.5 },\n  '4g':    { offline: false, downloadThroughput: 9 * 1024 * 1024 / 8, uploadThroughput: 2.67 * 1024 * 1024 / 8, latency: 85 },\n  offline: { offline: true, downloadThroughput: 0, uploadThroughput: 0, latency: 0 },\n};\n\nexport async function applyNetworkThrottle(\n  cdpSession: any,\n  preset: NetworkConditionPreset,\n): Promise<void> {\n  if (!cdpSession?.send) return;\n  const condition = PRESETS[preset];\n  try {\n    await cdpSession.send('Network.enable');\n    await cdpSession.send('Network.emulateNetworkConditions', condition);\n  } catch { /* non-fatal \u2014 CDP may not be available */ }\n}\n\nexport async function clearNetworkThrottle(cdpSession: any): Promise<void> {\n  await applyNetworkThrottle(cdpSession, 'none');\n}\n\nexport interface ThrottledCrawlOpts {\n  /** Network preset to apply during this URL's crawl */\n  preset: NetworkConditionPreset;\n  /** Restore to 'none' after navigation (default: true) */\n  restoreAfter?: boolean;\n}\n\n/** Apply throttle, run fn, restore. Returns fn's result. */\nexport async function withNetworkThrottle<T>(\n  cdpSession: any,\n  preset: NetworkConditionPreset,\n  fn: () => Promise<T>,\n  restoreAfter = true,\n): Promise<T> {\n  await applyNetworkThrottle(cdpSession, preset);\n  try {\n    return await fn();\n  } finally {\n    if (restoreAfter) await clearNetworkThrottle(cdpSession).catch(() => {});\n  }\n}\n"],
+  "mappings": "AAwBA,MAAM,UAA4D;AAAA,EAChE,MAAS,EAAE,SAAS,OAAO,oBAAoB,IAAI,kBAAkB,IAAI,SAAS,EAAE;AAAA,EACpF,QAAS,EAAE,SAAS,OAAO,oBAAoB,MAAM,OAAO,GAAG,kBAAkB,MAAM,OAAO,GAAG,SAAS,IAAM;AAAA,EAChH,QAAS,EAAE,SAAS,OAAO,oBAAoB,MAAM,OAAO,OAAO,GAAG,kBAAkB,MAAM,OAAO,GAAG,SAAS,MAAM;AAAA,EACvH,MAAS,EAAE,SAAS,OAAO,oBAAoB,IAAI,OAAO,OAAO,GAAG,kBAAkB,OAAO,OAAO,OAAO,GAAG,SAAS,GAAG;AAAA,EAC1H,SAAS,EAAE,SAAS,MAAM,oBAAoB,GAAG,kBAAkB,GAAG,SAAS,EAAE;AACnF;AAEA,eAAsB,qBACpB,YACA,QACe;AACf,MAAI,CAAC,YAAY,KAAM;AACvB,QAAM,YAAY,QAAQ,MAAM;AAChC,MAAI;AACF,UAAM,WAAW,KAAK,gBAAgB;AACtC,UAAM,WAAW,KAAK,oCAAoC,SAAS;AAAA,EACrE,QAAQ;AAAA,EAA6C;AACvD;AAEA,eAAsB,qBAAqB,YAAgC;AACzE,QAAM,qBAAqB,YAAY,MAAM;AAC/C;AAUA,eAAsB,oBACpB,YACA,QACA,IACA,eAAe,MACH;AACZ,QAAM,qBAAqB,YAAY,MAAM;AAC7C,MAAI;AACF,WAAO,MAAM,GAAG;AAAA,EAClB,UAAE;AACA,QAAI,aAAc,OAAM,qBAAqB,UAAU,EAAE,MAAM,MAAM;AAAA,IAAC,CAAC;AAAA,EACzE;AACF;",
+  "names": []
+}

package/dist/openapi-generator.d.ts

@@ -0,0 +1,49 @@
+/**
+ * OpenAPI 3.0 spec generator from observed network traffic.
+ *
+ * Takes the ApiEndpointObservationInput[] collected by the network observer
+ * during crawl and generates a minimal but valid OpenAPI 3.0 spec.
+ *
+ * The spec captures: endpoints, methods, observed response shapes, status codes.
+ * It does NOT capture request bodies (we'd need to intercept requests too) — that's
+ * marked as a TODO. But it's enough for generating smoke-test suites.
+ */
+import type { ApiEndpointObservationInput } from './network-observer.js';
+export interface OpenApiSpec {
+    openapi: '3.0.3';
+    info: {
+        title: string;
+        version: string;
+        description?: string;
+    };
+    servers: Array<{
+        url: string;
+    }>;
+    paths: Record<string, Record<string, OpenApiOperation>>;
+}
+interface OpenApiOperation {
+    summary: string;
+    parameters?: OpenApiParam[];
+    responses: Record<string, {
+        description: string;
+        content?: Record<string, {
+            schema: unknown;
+        }>;
+    }>;
+    tags?: string[];
+}
+interface OpenApiParam {
+    name: string;
+    in: 'path' | 'query';
+    required: boolean;
+    schema: {
+        type: string;
+    };
+}
+export declare function generateOpenApiSpec(observations: ApiEndpointObservationInput[], opts?: {
+    title?: string;
+    baseUrl?: string;
+    version?: string;
+}): OpenApiSpec;
+export declare function specToYaml(spec: OpenApiSpec): string;
+export {};