@detiq/crawler 1.1.8 → 1.1.9

package/dist/network-observer.js

@@ -1,149 +1,170 @@
-import { redactJson } from "./policies/crawl-safety.js";
+import { redactJson } from './policies/crawl-safety.js';
 const API_PATTERN = /\/(api|graphql|gql|rest|v\d+)\//i;
 const ASSET_EXT = /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot|map|json\.map)(\?|$)/i;
 function normalizeApiUrl(url) {
-  const u = new URL(url);
-  return u.pathname.replace(/\/[a-f0-9-]{8,}(?=\/|$)/gi, "/{id}").replace(/\/c[a-z0-9]{20,}(?=\/|$)/g, "/{id}").replace(/\/\d+(?=\/|$)/g, "/{id}");
+    const u = new URL(url);
+    return u.pathname
+        .replace(/\/[a-f0-9-]{8,}(?=\/|$)/gi, '/{id}')
+        .replace(/\/c[a-z0-9]{20,}(?=\/|$)/g, '/{id}')
+        .replace(/\/\d+(?=\/|$)/g, '/{id}');
 }
 function inferShape(value) {
-  if (Array.isArray(value)) return value.length > 0 ? [inferShape(value[0])] : [];
-  if (value && typeof value === "object") {
-    const out = {};
-    for (const [key, item] of Object.entries(value).slice(0, 40)) out[key] = inferShape(item);
-    return out;
-  }
-  return typeof value;
+    if (Array.isArray(value))
+        return value.length > 0 ? [inferShape(value[0])] : [];
+    if (value && typeof value === 'object') {
+        const out = {};
+        for (const [key, item] of Object.entries(value).slice(0, 40))
+            out[key] = inferShape(item);
+        return out;
+    }
+    return typeof value;
 }
+/** P4: Extract page-URL candidates from a JSON response body string. */
 function extractPageUrls(jsonStr) {
-  const PAGE_URL = /"(?:url|href|path|link|route|to|src|redirect)"\s*:\s*"(\/[\w\-\/\.~@%]+)"/g;
-  const ASSET = /\.(png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|css|js|map|json)$/i;
-  const API_PATH = /^\/(api|v\d+|graphql|gql|rest|rpc|static|_next|assets)\//i;
-  const found = /* @__PURE__ */ new Set();
-  let m;
-  PAGE_URL.lastIndex = 0;
-  while ((m = PAGE_URL.exec(jsonStr)) !== null) {
-    const u = m[1];
-    if (u && u.length > 1 && u.length < 120 && !ASSET.test(u) && !API_PATH.test(u)) found.add(u);
-  }
-  return [...found];
-}
-function attachNetworkObserver(context, sink, onPageUrlDiscovered) {
-  const seen = /* @__PURE__ */ new Map();
-  const requestTimes = /* @__PURE__ */ new Map();
-  context.on("request", (request) => {
-    try {
-      const url = request.url();
-      const rtype = request.resourceType();
-      if (rtype !== "xhr" && rtype !== "fetch" || !API_PATTERN.test(url) || ASSET_EXT.test(url)) return;
-      const method = request.method();
-      const normalized = normalizeApiUrl(url);
-      const key = `${method}:${normalized}`;
-      requestTimes.set(url, Date.now());
-      if (!seen.has(key)) {
-        const row = { method, urlPattern: normalized, requestKind: rtype };
-        seen.set(key, row);
-        sink.push(row);
-      }
-    } catch {
+    const PAGE_URL = /"(?:url|href|path|link|route|to|src|redirect)"\s*:\s*"(\/[\w\-\/\.~@%]+)"/g;
+    const ASSET = /\.(png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|css|js|map|json)$/i;
+    const API_PATH = /^\/(api|v\d+|graphql|gql|rest|rpc|static|_next|assets)\//i;
+    const found = new Set();
+    let m;
+    PAGE_URL.lastIndex = 0;
+    while ((m = PAGE_URL.exec(jsonStr)) !== null) {
+        const u = m[1];
+        if (u && u.length > 1 && u.length < 120 && !ASSET.test(u) && !API_PATH.test(u))
+            found.add(u);
     }
-  });
-  context.on("response", async (response) => {
-    try {
-      const request = response.request();
-      const url = request.url();
-      const rtype = request.resourceType();
-      if (rtype !== "xhr" && rtype !== "fetch" || !API_PATTERN.test(url) || ASSET_EXT.test(url)) return;
-      const normalized = normalizeApiUrl(url);
-      const key = `${request.method()}:${normalized}`;
-      const durationMs = Date.now() - (requestTimes.get(url) ?? Date.now());
-      let row = seen.get(key);
-      if (!row) {
-        row = { method: request.method(), urlPattern: normalized, requestKind: rtype };
-        seen.set(key, row);
-        sink.push(row);
-      }
-      row.statusCode = response.status();
-      row.durationMs = durationMs;
-      const contentType = response.headers()["content-type"] ?? "";
-      if (contentType.includes("application/json")) {
-        const body = await response.json().catch(() => void 0);
-        if (body !== void 0) {
-          row.responseShape = redactJson(inferShape(body));
-          if (onPageUrlDiscovered) {
-            for (const u of extractPageUrls(JSON.stringify(body))) onPageUrlDiscovered(u);
-          }
+    return [...found];
+}
+export function attachNetworkObserver(context, sink, onPageUrlDiscovered) {
+    const seen = new Map();
+    const requestTimes = new Map();
+    context.on('request', (request) => {
+        try {
+            const url = request.url();
+            const rtype = request.resourceType();
+            if ((rtype !== 'xhr' && rtype !== 'fetch') || !API_PATTERN.test(url) || ASSET_EXT.test(url))
+                return;
+            const method = request.method();
+            const normalized = normalizeApiUrl(url);
+            const key = `${method}:${normalized}`;
+            requestTimes.set(url, Date.now());
+            if (!seen.has(key)) {
+                const row = { method, urlPattern: normalized, requestKind: rtype };
+                seen.set(key, row);
+                sink.push(row);
+            }
+        }
+        catch {
+            // non-fatal
         }
-      }
-      if (response.status() >= 400) {
+    });
+    context.on('response', async (response) => {
         try {
-          const errText = await response.text();
-          row.errorBody = errText.slice(0, 300);
-        } catch {
+            const request = response.request();
+            const url = request.url();
+            const rtype = request.resourceType();
+            if ((rtype !== 'xhr' && rtype !== 'fetch') || !API_PATTERN.test(url) || ASSET_EXT.test(url))
+                return;
+            const normalized = normalizeApiUrl(url);
+            const key = `${request.method()}:${normalized}`;
+            const durationMs = Date.now() - (requestTimes.get(url) ?? Date.now());
+            let row = seen.get(key);
+            if (!row) {
+                row = { method: request.method(), urlPattern: normalized, requestKind: rtype };
+                seen.set(key, row);
+                sink.push(row);
+            }
+            row.statusCode = response.status();
+            row.durationMs = durationMs;
+            const contentType = response.headers()['content-type'] ?? '';
+            if (contentType.includes('application/json')) {
+                const body = await response.json().catch(() => undefined);
+                if (body !== undefined) {
+                    row.responseShape = redactJson(inferShape(body));
+                    // P4: scan JSON body for page-URL candidates (non-API paths)
+                    if (onPageUrlDiscovered) {
+                        for (const u of extractPageUrls(JSON.stringify(body)))
+                            onPageUrlDiscovered(u);
+                    }
+                }
+            }
+            if (response.status() >= 400) {
+                try {
+                    const errText = await response.text();
+                    row.errorBody = errText.slice(0, 300);
+                }
+                catch { /* non-fatal */ }
+            }
+        }
+        catch {
+            // non-fatal
         }
-      }
-    } catch {
-    }
-  });
-}
-function attachHarCapture(cdpSession, pageTitle = "Page") {
-  const entries = [];
-  const pending = /* @__PURE__ */ new Map();
-  const pageId = `page_${String(Date.now())}`;
-  const pageStarted = (/* @__PURE__ */ new Date()).toISOString();
-  const onRequest = (evt) => {
-    const { requestId, request } = evt ?? {};
-    if (!requestId || !request) return;
-    const headers = {};
-    for (const h of request.headers ?? []) if (h.name) headers[String(h.name).toLowerCase()] = String(h.value ?? "");
-    pending.set(requestId, { method: request.method ?? "GET", url: request.url ?? "", headers, startMs: Date.now() });
-  };
-  const onResponse = (evt) => {
-    const { requestId, response } = evt ?? {};
-    if (!requestId || !response) return;
-    const req = pending.get(requestId);
-    if (!req) return;
-    pending.delete(requestId);
-    const durationMs = Date.now() - req.startMs;
-    const responseHeaders = {};
-    for (const h of response.headers ?? []) if (h.name) responseHeaders[String(h.name).toLowerCase()] = String(h.value ?? "");
-    entries.push({
-      startedAt: new Date(req.startMs).toISOString(),
-      method: req.method,
-      url: req.url,
-      status: response.status ?? 0,
-      statusText: response.statusText ?? "",
-      requestHeaders: req.headers,
-      responseHeaders,
-      contentType: responseHeaders["content-type"] ?? "",
-      bodySize: response.encodedDataLength ?? -1,
-      durationMs,
-      timings: { send: 0, wait: Math.round(durationMs * 0.9), receive: Math.round(durationMs * 0.1) }
     });
-  };
-  if (cdpSession?.on) {
-    cdpSession.on("Network.requestWillBeSent", onRequest);
-    cdpSession.on("Network.responseReceived", onResponse);
-  }
-  return {
-    getHar() {
-      return {
-        version: "1.2",
-        creator: { name: "ZeTa-Crawler", version: "1.0" },
-        pages: [{ id: pageId, title: pageTitle, startedAt: pageStarted }],
-        entries: [...entries]
-      };
-    },
-    detach() {
-      if (cdpSession?.off) {
-        cdpSession.off("Network.requestWillBeSent", onRequest);
-        cdpSession.off("Network.responseReceived", onResponse);
-      }
-      pending.clear();
+}
+/**
+ * Full HAR capture via CDP — intercepts ALL network traffic, not just API endpoints.
+ * Returns a controller with getHar() and detach().
+ */
+export function attachHarCapture(cdpSession, pageTitle = 'Page') {
+    const entries = [];
+    const pending = new Map();
+    const pageId = `page_${String(Date.now())}`;
+    const pageStarted = new Date().toISOString();
+    const onRequest = (evt) => {
+        const { requestId, request } = evt ?? {};
+        if (!requestId || !request)
+            return;
+        const headers = {};
+        for (const h of (request.headers ?? []))
+            if (h.name)
+                headers[String(h.name).toLowerCase()] = String(h.value ?? '');
+        pending.set(requestId, { method: request.method ?? 'GET', url: request.url ?? '', headers, startMs: Date.now() });
+    };
+    const onResponse = (evt) => {
+        const { requestId, response } = evt ?? {};
+        if (!requestId || !response)
+            return;
+        const req = pending.get(requestId);
+        if (!req)
+            return;
+        pending.delete(requestId);
+        const durationMs = Date.now() - req.startMs;
+        const responseHeaders = {};
+        for (const h of (response.headers ?? []))
+            if (h.name)
+                responseHeaders[String(h.name).toLowerCase()] = String(h.value ?? '');
+        entries.push({
+            startedAt: new Date(req.startMs).toISOString(),
+            method: req.method,
+            url: req.url,
+            status: response.status ?? 0,
+            statusText: response.statusText ?? '',
+            requestHeaders: req.headers,
+            responseHeaders,
+            contentType: responseHeaders['content-type'] ?? '',
+            bodySize: response.encodedDataLength ?? -1,
+            durationMs,
+            timings: { send: 0, wait: Math.round(durationMs * 0.9), receive: Math.round(durationMs * 0.1) },
+        });
+    };
+    if (cdpSession?.on) {
+        cdpSession.on('Network.requestWillBeSent', onRequest);
+        cdpSession.on('Network.responseReceived', onResponse);
     }
-  };
+    return {
+        getHar() {
+            return {
+                version: '1.2',
+                creator: { name: 'ZeTa-Crawler', version: '1.0' },
+                pages: [{ id: pageId, title: pageTitle, startedAt: pageStarted }],
+                entries: [...entries],
+            };
+        },
+        detach() {
+            if (cdpSession?.off) {
+                cdpSession.off('Network.requestWillBeSent', onRequest);
+                cdpSession.off('Network.responseReceived', onResponse);
+            }
+            pending.clear();
+        },
+    };
 }
-export {
-  attachHarCapture,
-  attachNetworkObserver
-};
-//# sourceMappingURL=network-observer.js.map

package/dist/network-throttle.js

@@ -1,34 +1,45 @@
+/**
+ * Network throttle simulation via CDP.
+ *
+ * Simulates different network conditions during crawl to test how the app
+ * behaves under slow connections. Useful for detecting loading states, error
+ * handling, and performance degradation on poor networks.
+ *
+ * Conditions mirror Chrome DevTools presets:
+ * - slow3g: ~500kbps download, high latency
+ * - fast3g: ~1.6Mbps download, medium latency
+ * - 4g: ~9Mbps download, low latency
+ * - offline: no connectivity
+ * - none: no throttle (default)
+ */
 const PRESETS = {
-  none: { offline: false, downloadThroughput: -1, uploadThroughput: -1, latency: 0 },
-  slow3g: { offline: false, downloadThroughput: 500 * 1024 / 8, uploadThroughput: 500 * 1024 / 8, latency: 2e3 },
-  fast3g: { offline: false, downloadThroughput: 1.6 * 1024 * 1024 / 8, uploadThroughput: 750 * 1024 / 8, latency: 562.5 },
-  "4g": { offline: false, downloadThroughput: 9 * 1024 * 1024 / 8, uploadThroughput: 2.67 * 1024 * 1024 / 8, latency: 85 },
-  offline: { offline: true, downloadThroughput: 0, uploadThroughput: 0, latency: 0 }
+    none: { offline: false, downloadThroughput: -1, uploadThroughput: -1, latency: 0 },
+    slow3g: { offline: false, downloadThroughput: 500 * 1024 / 8, uploadThroughput: 500 * 1024 / 8, latency: 2_000 },
+    fast3g: { offline: false, downloadThroughput: 1.6 * 1024 * 1024 / 8, uploadThroughput: 750 * 1024 / 8, latency: 562.5 },
+    '4g': { offline: false, downloadThroughput: 9 * 1024 * 1024 / 8, uploadThroughput: 2.67 * 1024 * 1024 / 8, latency: 85 },
+    offline: { offline: true, downloadThroughput: 0, uploadThroughput: 0, latency: 0 },
 };
-async function applyNetworkThrottle(cdpSession, preset) {
-  if (!cdpSession?.send) return;
-  const condition = PRESETS[preset];
-  try {
-    await cdpSession.send("Network.enable");
-    await cdpSession.send("Network.emulateNetworkConditions", condition);
-  } catch {
-  }
+export async function applyNetworkThrottle(cdpSession, preset) {
+    if (!cdpSession?.send)
+        return;
+    const condition = PRESETS[preset];
+    try {
+        await cdpSession.send('Network.enable');
+        await cdpSession.send('Network.emulateNetworkConditions', condition);
+    }
+    catch { /* non-fatal — CDP may not be available */ }
 }
-async function clearNetworkThrottle(cdpSession) {
-  await applyNetworkThrottle(cdpSession, "none");
+export async function clearNetworkThrottle(cdpSession) {
+    await applyNetworkThrottle(cdpSession, 'none');
 }
-async function withNetworkThrottle(cdpSession, preset, fn, restoreAfter = true) {
-  await applyNetworkThrottle(cdpSession, preset);
-  try {
-    return await fn();
-  } finally {
-    if (restoreAfter) await clearNetworkThrottle(cdpSession).catch(() => {
-    });
-  }
+/** Apply throttle, run fn, restore. Returns fn's result. */
+export async function withNetworkThrottle(cdpSession, preset, fn, restoreAfter = true) {
+    await applyNetworkThrottle(cdpSession, preset);
+    try {
+        return await fn();
+    }
+    finally {
+        if (restoreAfter)
+            await clearNetworkThrottle(cdpSession).catch(() => { });
+    }
 }
-export {
-  applyNetworkThrottle,
-  clearNetworkThrottle,
-  withNetworkThrottle
-};
-//# sourceMappingURL=network-throttle.js.map

package/dist/openapi-generator.js

@@ -1,89 +1,104 @@
+/**
+ * OpenAPI 3.0 spec generator from observed network traffic.
+ *
+ * Takes the ApiEndpointObservationInput[] collected by the network observer
+ * during crawl and generates a minimal but valid OpenAPI 3.0 spec.
+ *
+ * The spec captures: endpoints, methods, observed response shapes, status codes.
+ * It does NOT capture request bodies (we'd need to intercept requests too) — that's
+ * marked as a TODO. But it's enough for generating smoke-test suites.
+ */
 function extractPathParams(urlPattern) {
-  const params = [];
-  const matches = urlPattern.matchAll(/\{(\w+)\}/g);
-  for (const m of matches) {
-    params.push({ name: m[1], in: "path", required: true, schema: { type: "string" } });
-  }
-  return params;
+    const params = [];
+    const matches = urlPattern.matchAll(/\{(\w+)\}/g);
+    for (const m of matches) {
+        params.push({ name: m[1], in: 'path', required: true, schema: { type: 'string' } });
+    }
+    return params;
 }
 function methodSummary(method, path) {
-  const resource = path.split("/").filter((s) => !s.startsWith("{") && s.length > 0).pop() ?? "resource";
-  const summaries = {
-    GET: `Get ${resource}`,
-    POST: `Create ${resource}`,
-    PUT: `Update ${resource}`,
-    PATCH: `Partially update ${resource}`,
-    DELETE: `Delete ${resource}`
-  };
-  return summaries[method.toUpperCase()] ?? `${method} ${resource}`;
+    const resource = path.split('/').filter(s => !s.startsWith('{') && s.length > 0).pop() ?? 'resource';
+    const summaries = {
+        GET: `Get ${resource}`,
+        POST: `Create ${resource}`,
+        PUT: `Update ${resource}`,
+        PATCH: `Partially update ${resource}`,
+        DELETE: `Delete ${resource}`,
+    };
+    return summaries[method.toUpperCase()] ?? `${method} ${resource}`;
 }
 function inferTag(path) {
-  const parts = path.split("/").filter((s) => s.length > 0 && !s.startsWith("{"));
-  return parts[0] ?? "default";
+    const parts = path.split('/').filter(s => s.length > 0 && !s.startsWith('{'));
+    return parts[0] ?? 'default';
 }
-function generateOpenApiSpec(observations, opts = {}) {
-  const { title = "ZeTa Discovered API", baseUrl = "/", version = "1.0.0" } = opts;
-  const spec = {
-    openapi: "3.0.3",
-    info: { title, version, description: "Auto-generated by ZeTa crawler from observed network traffic." },
-    servers: [{ url: baseUrl }],
-    paths: {}
-  };
-  for (const obs of observations) {
-    const path = obs.urlPattern;
-    const method = obs.method.toLowerCase();
-    if (!spec.paths[path]) spec.paths[path] = {};
-    const params = extractPathParams(path);
-    const statusCode = obs.statusCode ?? 200;
-    const responses = {
-      [String(statusCode)]: {
-        description: statusCode < 400 ? "Successful response" : "Error response",
-        content: obs.responseShape ? { "application/json": { schema: shapeToJsonSchema(obs.responseShape) } } : void 0
-      }
-    };
-    const op = {
-      summary: methodSummary(obs.method, path),
-      parameters: params.length > 0 ? params : void 0,
-      responses,
-      tags: [inferTag(path)]
+export function generateOpenApiSpec(observations, opts = {}) {
+    const { title = 'ZeTa Discovered API', baseUrl = '/', version = '1.0.0' } = opts;
+    const spec = {
+        openapi: '3.0.3',
+        info: { title, version, description: 'Auto-generated by ZeTa crawler from observed network traffic.' },
+        servers: [{ url: baseUrl }],
+        paths: {},
     };
-    spec.paths[path][method] = op;
-  }
-  return spec;
+    for (const obs of observations) {
+        const path = obs.urlPattern;
+        const method = obs.method.toLowerCase();
+        if (!spec.paths[path])
+            spec.paths[path] = {};
+        const params = extractPathParams(path);
+        const statusCode = obs.statusCode ?? 200;
+        const responses = {
+            [String(statusCode)]: {
+                description: statusCode < 400 ? 'Successful response' : 'Error response',
+                content: obs.responseShape
+                    ? { 'application/json': { schema: shapeToJsonSchema(obs.responseShape) } }
+                    : undefined,
+            },
+        };
+        const op = {
+            summary: methodSummary(obs.method, path),
+            parameters: params.length > 0 ? params : undefined,
+            responses,
+            tags: [inferTag(path)],
+        };
+        spec.paths[path][method] = op;
+    }
+    return spec;
 }
 function shapeToJsonSchema(shape) {
-  if (Array.isArray(shape)) return { type: "array", items: shape.length > 0 ? shapeToJsonSchema(shape[0]) : {} };
-  if (shape && typeof shape === "object") {
-    const props = {};
-    for (const [k, v] of Object.entries(shape)) props[k] = shapeToJsonSchema(v);
-    return { type: "object", properties: props };
-  }
-  return { type: String(shape) };
+    if (Array.isArray(shape))
+        return { type: 'array', items: shape.length > 0 ? shapeToJsonSchema(shape[0]) : {} };
+    if (shape && typeof shape === 'object') {
+        const props = {};
+        for (const [k, v] of Object.entries(shape))
+            props[k] = shapeToJsonSchema(v);
+        return { type: 'object', properties: props };
+    }
+    return { type: String(shape) };
 }
-function specToYaml(spec) {
-  function toYaml(val, indent = 0) {
-    const pad = "  ".repeat(indent);
-    if (val === null || val === void 0) return "null";
-    if (typeof val === "string") return val.includes("\n") || val.includes(":") ? `"${val.replace(/"/g, '\\"')}"` : val;
-    if (typeof val === "number" || typeof val === "boolean") return String(val);
-    if (Array.isArray(val)) return val.map((v) => `
-${pad}- ${toYaml(v, indent + 1).trimStart()}`).join("") || "[]";
-    if (typeof val === "object") {
-      const entries = Object.entries(val);
-      if (entries.length === 0) return "{}";
-      return entries.map(([k, v]) => {
-        const vStr = toYaml(v, indent + 1);
-        return typeof v === "object" && v !== null && !Array.isArray(v) ? `
-${pad}${k}:${vStr}` : `
-${pad}${k}: ${vStr.trimStart()}`;
-      }).join("");
+export function specToYaml(spec) {
+    // Minimal YAML serializer — avoids yaml dep
+    function toYaml(val, indent = 0) {
+        const pad = '  '.repeat(indent);
+        if (val === null || val === undefined)
+            return 'null';
+        if (typeof val === 'string')
+            return val.includes('\n') || val.includes(':') ? `"${val.replace(/"/g, '\\"')}"` : val;
+        if (typeof val === 'number' || typeof val === 'boolean')
+            return String(val);
+        if (Array.isArray(val))
+            return val.map(v => `\n${pad}- ${toYaml(v, indent + 1).trimStart()}`).join('') || '[]';
+        if (typeof val === 'object') {
+            const entries = Object.entries(val);
+            if (entries.length === 0)
+                return '{}';
+            return entries.map(([k, v]) => {
+                const vStr = toYaml(v, indent + 1);
+                return typeof v === 'object' && v !== null && !Array.isArray(v)
+                    ? `\n${pad}${k}:${vStr}`
+                    : `\n${pad}${k}: ${vStr.trimStart()}`;
+            }).join('');
+        }
+        return String(val);
     }
-    return String(val);
-  }
-  return `openapi: "${spec.openapi}"${toYaml(spec, 0)}`;
+    return `openapi: "${spec.openapi}"${toYaml(spec, 0)}`;
 }
-export {
-  generateOpenApiSpec,
-  specToYaml
-};
-//# sourceMappingURL=openapi-generator.js.map