@detiq/crawler 1.1.9 → 1.1.11

package/dist/network-observer.js

@@ -1,170 +1,149 @@
-import { redactJson } from './policies/crawl-safety.js';
+import { redactJson } from "./policies/crawl-safety.js";
 const API_PATTERN = /\/(api|graphql|gql|rest|v\d+)\//i;
 const ASSET_EXT = /\.(js|css|png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|eot|map|json\.map)(\?|$)/i;
 function normalizeApiUrl(url) {
-    const u = new URL(url);
-    return u.pathname
-        .replace(/\/[a-f0-9-]{8,}(?=\/|$)/gi, '/{id}')
-        .replace(/\/c[a-z0-9]{20,}(?=\/|$)/g, '/{id}')
-        .replace(/\/\d+(?=\/|$)/g, '/{id}');
+  const u = new URL(url);
+  return u.pathname.replace(/\/[a-f0-9-]{8,}(?=\/|$)/gi, "/{id}").replace(/\/c[a-z0-9]{20,}(?=\/|$)/g, "/{id}").replace(/\/\d+(?=\/|$)/g, "/{id}");
 }
 function inferShape(value) {
-    if (Array.isArray(value))
-        return value.length > 0 ? [inferShape(value[0])] : [];
-    if (value && typeof value === 'object') {
-        const out = {};
-        for (const [key, item] of Object.entries(value).slice(0, 40))
-            out[key] = inferShape(item);
-        return out;
-    }
-    return typeof value;
+  if (Array.isArray(value)) return value.length > 0 ? [inferShape(value[0])] : [];
+  if (value && typeof value === "object") {
+    const out = {};
+    for (const [key, item] of Object.entries(value).slice(0, 40)) out[key] = inferShape(item);
+    return out;
+  }
+  return typeof value;
 }
-/** P4: Extract page-URL candidates from a JSON response body string. */
 function extractPageUrls(jsonStr) {
-    const PAGE_URL = /"(?:url|href|path|link|route|to|src|redirect)"\s*:\s*"(\/[\w\-\/\.~@%]+)"/g;
-    const ASSET = /\.(png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|css|js|map|json)$/i;
-    const API_PATH = /^\/(api|v\d+|graphql|gql|rest|rpc|static|_next|assets)\//i;
-    const found = new Set();
-    let m;
-    PAGE_URL.lastIndex = 0;
-    while ((m = PAGE_URL.exec(jsonStr)) !== null) {
-        const u = m[1];
-        if (u && u.length > 1 && u.length < 120 && !ASSET.test(u) && !API_PATH.test(u))
-            found.add(u);
-    }
-    return [...found];
+  const PAGE_URL = /"(?:url|href|path|link|route|to|src|redirect)"\s*:\s*"(\/[\w\-\/\.~@%]+)"/g;
+  const ASSET = /\.(png|jpg|jpeg|gif|svg|ico|woff|woff2|ttf|css|js|map|json)$/i;
+  const API_PATH = /^\/(api|v\d+|graphql|gql|rest|rpc|static|_next|assets)\//i;
+  const found = /* @__PURE__ */ new Set();
+  let m;
+  PAGE_URL.lastIndex = 0;
+  while ((m = PAGE_URL.exec(jsonStr)) !== null) {
+    const u = m[1];
+    if (u && u.length > 1 && u.length < 120 && !ASSET.test(u) && !API_PATH.test(u)) found.add(u);
+  }
+  return [...found];
 }
-export function attachNetworkObserver(context, sink, onPageUrlDiscovered) {
-    const seen = new Map();
-    const requestTimes = new Map();
-    context.on('request', (request) => {
-        try {
-            const url = request.url();
-            const rtype = request.resourceType();
-            if ((rtype !== 'xhr' && rtype !== 'fetch') || !API_PATTERN.test(url) || ASSET_EXT.test(url))
-                return;
-            const method = request.method();
-            const normalized = normalizeApiUrl(url);
-            const key = `${method}:${normalized}`;
-            requestTimes.set(url, Date.now());
-            if (!seen.has(key)) {
-                const row = { method, urlPattern: normalized, requestKind: rtype };
-                seen.set(key, row);
-                sink.push(row);
-            }
-        }
-        catch {
-            // non-fatal
+function attachNetworkObserver(context, sink, onPageUrlDiscovered) {
+  const seen = /* @__PURE__ */ new Map();
+  const requestTimes = /* @__PURE__ */ new Map();
+  context.on("request", (request) => {
+    try {
+      const url = request.url();
+      const rtype = request.resourceType();
+      if (rtype !== "xhr" && rtype !== "fetch" || !API_PATTERN.test(url) || ASSET_EXT.test(url)) return;
+      const method = request.method();
+      const normalized = normalizeApiUrl(url);
+      const key = `${method}:${normalized}`;
+      requestTimes.set(url, Date.now());
+      if (!seen.has(key)) {
+        const row = { method, urlPattern: normalized, requestKind: rtype };
+        seen.set(key, row);
+        sink.push(row);
+      }
+    } catch {
+    }
+  });
+  context.on("response", async (response) => {
+    try {
+      const request = response.request();
+      const url = request.url();
+      const rtype = request.resourceType();
+      if (rtype !== "xhr" && rtype !== "fetch" || !API_PATTERN.test(url) || ASSET_EXT.test(url)) return;
+      const normalized = normalizeApiUrl(url);
+      const key = `${request.method()}:${normalized}`;
+      const durationMs = Date.now() - (requestTimes.get(url) ?? Date.now());
+      let row = seen.get(key);
+      if (!row) {
+        row = { method: request.method(), urlPattern: normalized, requestKind: rtype };
+        seen.set(key, row);
+        sink.push(row);
+      }
+      row.statusCode = response.status();
+      row.durationMs = durationMs;
+      const contentType = response.headers()["content-type"] ?? "";
+      if (contentType.includes("application/json")) {
+        const body = await response.json().catch(() => void 0);
+        if (body !== void 0) {
+          row.responseShape = redactJson(inferShape(body));
+          if (onPageUrlDiscovered) {
+            for (const u of extractPageUrls(JSON.stringify(body))) onPageUrlDiscovered(u);
+          }
         }
-    });
-    context.on('response', async (response) => {
+      }
+      if (response.status() >= 400) {
         try {
-            const request = response.request();
-            const url = request.url();
-            const rtype = request.resourceType();
-            if ((rtype !== 'xhr' && rtype !== 'fetch') || !API_PATTERN.test(url) || ASSET_EXT.test(url))
-                return;
-            const normalized = normalizeApiUrl(url);
-            const key = `${request.method()}:${normalized}`;
-            const durationMs = Date.now() - (requestTimes.get(url) ?? Date.now());
-            let row = seen.get(key);
-            if (!row) {
-                row = { method: request.method(), urlPattern: normalized, requestKind: rtype };
-                seen.set(key, row);
-                sink.push(row);
-            }
-            row.statusCode = response.status();
-            row.durationMs = durationMs;
-            const contentType = response.headers()['content-type'] ?? '';
-            if (contentType.includes('application/json')) {
-                const body = await response.json().catch(() => undefined);
-                if (body !== undefined) {
-                    row.responseShape = redactJson(inferShape(body));
-                    // P4: scan JSON body for page-URL candidates (non-API paths)
-                    if (onPageUrlDiscovered) {
-                        for (const u of extractPageUrls(JSON.stringify(body)))
-                            onPageUrlDiscovered(u);
-                    }
-                }
-            }
-            if (response.status() >= 400) {
-                try {
-                    const errText = await response.text();
-                    row.errorBody = errText.slice(0, 300);
-                }
-                catch { /* non-fatal */ }
-            }
-        }
-        catch {
-            // non-fatal
+          const errText = await response.text();
+          row.errorBody = errText.slice(0, 300);
+        } catch {
         }
-    });
+      }
+    } catch {
+    }
+  });
 }
-/**
- * Full HAR capture via CDP — intercepts ALL network traffic, not just API endpoints.
- * Returns a controller with getHar() and detach().
- */
-export function attachHarCapture(cdpSession, pageTitle = 'Page') {
-    const entries = [];
-    const pending = new Map();
-    const pageId = `page_${String(Date.now())}`;
-    const pageStarted = new Date().toISOString();
-    const onRequest = (evt) => {
-        const { requestId, request } = evt ?? {};
-        if (!requestId || !request)
-            return;
-        const headers = {};
-        for (const h of (request.headers ?? []))
-            if (h.name)
-                headers[String(h.name).toLowerCase()] = String(h.value ?? '');
-        pending.set(requestId, { method: request.method ?? 'GET', url: request.url ?? '', headers, startMs: Date.now() });
-    };
-    const onResponse = (evt) => {
-        const { requestId, response } = evt ?? {};
-        if (!requestId || !response)
-            return;
-        const req = pending.get(requestId);
-        if (!req)
-            return;
-        pending.delete(requestId);
-        const durationMs = Date.now() - req.startMs;
-        const responseHeaders = {};
-        for (const h of (response.headers ?? []))
-            if (h.name)
-                responseHeaders[String(h.name).toLowerCase()] = String(h.value ?? '');
-        entries.push({
-            startedAt: new Date(req.startMs).toISOString(),
-            method: req.method,
-            url: req.url,
-            status: response.status ?? 0,
-            statusText: response.statusText ?? '',
-            requestHeaders: req.headers,
-            responseHeaders,
-            contentType: responseHeaders['content-type'] ?? '',
-            bodySize: response.encodedDataLength ?? -1,
-            durationMs,
-            timings: { send: 0, wait: Math.round(durationMs * 0.9), receive: Math.round(durationMs * 0.1) },
-        });
-    };
-    if (cdpSession?.on) {
-        cdpSession.on('Network.requestWillBeSent', onRequest);
-        cdpSession.on('Network.responseReceived', onResponse);
+function attachHarCapture(cdpSession, pageTitle = "Page") {
+  const entries = [];
+  const pending = /* @__PURE__ */ new Map();
+  const pageId = `page_${String(Date.now())}`;
+  const pageStarted = (/* @__PURE__ */ new Date()).toISOString();
+  const onRequest = (evt) => {
+    const { requestId, request } = evt ?? {};
+    if (!requestId || !request) return;
+    const headers = {};
+    for (const h of request.headers ?? []) if (h.name) headers[String(h.name).toLowerCase()] = String(h.value ?? "");
+    pending.set(requestId, { method: request.method ?? "GET", url: request.url ?? "", headers, startMs: Date.now() });
+  };
+  const onResponse = (evt) => {
+    const { requestId, response } = evt ?? {};
+    if (!requestId || !response) return;
+    const req = pending.get(requestId);
+    if (!req) return;
+    pending.delete(requestId);
+    const durationMs = Date.now() - req.startMs;
+    const responseHeaders = {};
+    for (const h of response.headers ?? []) if (h.name) responseHeaders[String(h.name).toLowerCase()] = String(h.value ?? "");
+    entries.push({
+      startedAt: new Date(req.startMs).toISOString(),
+      method: req.method,
+      url: req.url,
+      status: response.status ?? 0,
+      statusText: response.statusText ?? "",
+      requestHeaders: req.headers,
+      responseHeaders,
+      contentType: responseHeaders["content-type"] ?? "",
+      bodySize: response.encodedDataLength ?? -1,
+      durationMs,
+      timings: { send: 0, wait: Math.round(durationMs * 0.9), receive: Math.round(durationMs * 0.1) }
+    });
+  };
+  if (cdpSession?.on) {
+    cdpSession.on("Network.requestWillBeSent", onRequest);
+    cdpSession.on("Network.responseReceived", onResponse);
+  }
+  return {
+    getHar() {
+      return {
+        version: "1.2",
+        creator: { name: "ZeTa-Crawler", version: "1.0" },
+        pages: [{ id: pageId, title: pageTitle, startedAt: pageStarted }],
+        entries: [...entries]
+      };
+    },
+    detach() {
+      if (cdpSession?.off) {
+        cdpSession.off("Network.requestWillBeSent", onRequest);
+        cdpSession.off("Network.responseReceived", onResponse);
+      }
+      pending.clear();
     }
-    return {
-        getHar() {
-            return {
-                version: '1.2',
-                creator: { name: 'ZeTa-Crawler', version: '1.0' },
-                pages: [{ id: pageId, title: pageTitle, startedAt: pageStarted }],
-                entries: [...entries],
-            };
-        },
-        detach() {
-            if (cdpSession?.off) {
-                cdpSession.off('Network.requestWillBeSent', onRequest);
-                cdpSession.off('Network.responseReceived', onResponse);
-            }
-            pending.clear();
-        },
-    };
+  };
 }
+export {
+  attachHarCapture,
+  attachNetworkObserver
+};
+//# sourceMappingURL=network-observer.js.map

package/dist/network-throttle.js

@@ -1,45 +1,34 @@
-/**
- * Network throttle simulation via CDP.
- *
- * Simulates different network conditions during crawl to test how the app
- * behaves under slow connections. Useful for detecting loading states, error
- * handling, and performance degradation on poor networks.
- *
- * Conditions mirror Chrome DevTools presets:
- * - slow3g: ~500kbps download, high latency
- * - fast3g: ~1.6Mbps download, medium latency
- * - 4g: ~9Mbps download, low latency
- * - offline: no connectivity
- * - none: no throttle (default)
- */
 const PRESETS = {
-    none: { offline: false, downloadThroughput: -1, uploadThroughput: -1, latency: 0 },
-    slow3g: { offline: false, downloadThroughput: 500 * 1024 / 8, uploadThroughput: 500 * 1024 / 8, latency: 2_000 },
-    fast3g: { offline: false, downloadThroughput: 1.6 * 1024 * 1024 / 8, uploadThroughput: 750 * 1024 / 8, latency: 562.5 },
-    '4g': { offline: false, downloadThroughput: 9 * 1024 * 1024 / 8, uploadThroughput: 2.67 * 1024 * 1024 / 8, latency: 85 },
-    offline: { offline: true, downloadThroughput: 0, uploadThroughput: 0, latency: 0 },
+  none: { offline: false, downloadThroughput: -1, uploadThroughput: -1, latency: 0 },
+  slow3g: { offline: false, downloadThroughput: 500 * 1024 / 8, uploadThroughput: 500 * 1024 / 8, latency: 2e3 },
+  fast3g: { offline: false, downloadThroughput: 1.6 * 1024 * 1024 / 8, uploadThroughput: 750 * 1024 / 8, latency: 562.5 },
+  "4g": { offline: false, downloadThroughput: 9 * 1024 * 1024 / 8, uploadThroughput: 2.67 * 1024 * 1024 / 8, latency: 85 },
+  offline: { offline: true, downloadThroughput: 0, uploadThroughput: 0, latency: 0 }
 };
-export async function applyNetworkThrottle(cdpSession, preset) {
-    if (!cdpSession?.send)
-        return;
-    const condition = PRESETS[preset];
-    try {
-        await cdpSession.send('Network.enable');
-        await cdpSession.send('Network.emulateNetworkConditions', condition);
-    }
-    catch { /* non-fatal — CDP may not be available */ }
+async function applyNetworkThrottle(cdpSession, preset) {
+  if (!cdpSession?.send) return;
+  const condition = PRESETS[preset];
+  try {
+    await cdpSession.send("Network.enable");
+    await cdpSession.send("Network.emulateNetworkConditions", condition);
+  } catch {
+  }
 }
-export async function clearNetworkThrottle(cdpSession) {
-    await applyNetworkThrottle(cdpSession, 'none');
+async function clearNetworkThrottle(cdpSession) {
+  await applyNetworkThrottle(cdpSession, "none");
 }
-/** Apply throttle, run fn, restore. Returns fn's result. */
-export async function withNetworkThrottle(cdpSession, preset, fn, restoreAfter = true) {
-    await applyNetworkThrottle(cdpSession, preset);
-    try {
-        return await fn();
-    }
-    finally {
-        if (restoreAfter)
-            await clearNetworkThrottle(cdpSession).catch(() => { });
-    }
+async function withNetworkThrottle(cdpSession, preset, fn, restoreAfter = true) {
+  await applyNetworkThrottle(cdpSession, preset);
+  try {
+    return await fn();
+  } finally {
+    if (restoreAfter) await clearNetworkThrottle(cdpSession).catch(() => {
+    });
+  }
 }
+export {
+  applyNetworkThrottle,
+  clearNetworkThrottle,
+  withNetworkThrottle
+};
+//# sourceMappingURL=network-throttle.js.map

package/dist/openapi-generator.js

@@ -1,104 +1,89 @@
-/**
- * OpenAPI 3.0 spec generator from observed network traffic.
- *
- * Takes the ApiEndpointObservationInput[] collected by the network observer
- * during crawl and generates a minimal but valid OpenAPI 3.0 spec.
- *
- * The spec captures: endpoints, methods, observed response shapes, status codes.
- * It does NOT capture request bodies (we'd need to intercept requests too) — that's
- * marked as a TODO. But it's enough for generating smoke-test suites.
- */
 function extractPathParams(urlPattern) {
-    const params = [];
-    const matches = urlPattern.matchAll(/\{(\w+)\}/g);
-    for (const m of matches) {
-        params.push({ name: m[1], in: 'path', required: true, schema: { type: 'string' } });
-    }
-    return params;
+  const params = [];
+  const matches = urlPattern.matchAll(/\{(\w+)\}/g);
+  for (const m of matches) {
+    params.push({ name: m[1], in: "path", required: true, schema: { type: "string" } });
+  }
+  return params;
 }
 function methodSummary(method, path) {
-    const resource = path.split('/').filter(s => !s.startsWith('{') && s.length > 0).pop() ?? 'resource';
-    const summaries = {
-        GET: `Get ${resource}`,
-        POST: `Create ${resource}`,
-        PUT: `Update ${resource}`,
-        PATCH: `Partially update ${resource}`,
-        DELETE: `Delete ${resource}`,
-    };
-    return summaries[method.toUpperCase()] ?? `${method} ${resource}`;
+  const resource = path.split("/").filter((s) => !s.startsWith("{") && s.length > 0).pop() ?? "resource";
+  const summaries = {
+    GET: `Get ${resource}`,
+    POST: `Create ${resource}`,
+    PUT: `Update ${resource}`,
+    PATCH: `Partially update ${resource}`,
+    DELETE: `Delete ${resource}`
+  };
+  return summaries[method.toUpperCase()] ?? `${method} ${resource}`;
 }
 function inferTag(path) {
-    const parts = path.split('/').filter(s => s.length > 0 && !s.startsWith('{'));
-    return parts[0] ?? 'default';
+  const parts = path.split("/").filter((s) => s.length > 0 && !s.startsWith("{"));
+  return parts[0] ?? "default";
 }
-export function generateOpenApiSpec(observations, opts = {}) {
-    const { title = 'ZeTa Discovered API', baseUrl = '/', version = '1.0.0' } = opts;
-    const spec = {
-        openapi: '3.0.3',
-        info: { title, version, description: 'Auto-generated by ZeTa crawler from observed network traffic.' },
-        servers: [{ url: baseUrl }],
-        paths: {},
+function generateOpenApiSpec(observations, opts = {}) {
+  const { title = "ZeTa Discovered API", baseUrl = "/", version = "1.0.0" } = opts;
+  const spec = {
+    openapi: "3.0.3",
+    info: { title, version, description: "Auto-generated by ZeTa crawler from observed network traffic." },
+    servers: [{ url: baseUrl }],
+    paths: {}
+  };
+  for (const obs of observations) {
+    const path = obs.urlPattern;
+    const method = obs.method.toLowerCase();
+    if (!spec.paths[path]) spec.paths[path] = {};
+    const params = extractPathParams(path);
+    const statusCode = obs.statusCode ?? 200;
+    const responses = {
+      [String(statusCode)]: {
+        description: statusCode < 400 ? "Successful response" : "Error response",
+        content: obs.responseShape ? { "application/json": { schema: shapeToJsonSchema(obs.responseShape) } } : void 0
+      }
     };
-    for (const obs of observations) {
-        const path = obs.urlPattern;
-        const method = obs.method.toLowerCase();
-        if (!spec.paths[path])
-            spec.paths[path] = {};
-        const params = extractPathParams(path);
-        const statusCode = obs.statusCode ?? 200;
-        const responses = {
-            [String(statusCode)]: {
-                description: statusCode < 400 ? 'Successful response' : 'Error response',
-                content: obs.responseShape
-                    ? { 'application/json': { schema: shapeToJsonSchema(obs.responseShape) } }
-                    : undefined,
-            },
-        };
-        const op = {
-            summary: methodSummary(obs.method, path),
-            parameters: params.length > 0 ? params : undefined,
-            responses,
-            tags: [inferTag(path)],
-        };
-        spec.paths[path][method] = op;
-    }
-    return spec;
+    const op = {
+      summary: methodSummary(obs.method, path),
+      parameters: params.length > 0 ? params : void 0,
+      responses,
+      tags: [inferTag(path)]
+    };
+    spec.paths[path][method] = op;
+  }
+  return spec;
 }
 function shapeToJsonSchema(shape) {
-    if (Array.isArray(shape))
-        return { type: 'array', items: shape.length > 0 ? shapeToJsonSchema(shape[0]) : {} };
-    if (shape && typeof shape === 'object') {
-        const props = {};
-        for (const [k, v] of Object.entries(shape))
-            props[k] = shapeToJsonSchema(v);
-        return { type: 'object', properties: props };
-    }
-    return { type: String(shape) };
+  if (Array.isArray(shape)) return { type: "array", items: shape.length > 0 ? shapeToJsonSchema(shape[0]) : {} };
+  if (shape && typeof shape === "object") {
+    const props = {};
+    for (const [k, v] of Object.entries(shape)) props[k] = shapeToJsonSchema(v);
+    return { type: "object", properties: props };
+  }
+  return { type: String(shape) };
 }
-export function specToYaml(spec) {
-    // Minimal YAML serializer — avoids yaml dep
-    function toYaml(val, indent = 0) {
-        const pad = '  '.repeat(indent);
-        if (val === null || val === undefined)
-            return 'null';
-        if (typeof val === 'string')
-            return val.includes('\n') || val.includes(':') ? `"${val.replace(/"/g, '\\"')}"` : val;
-        if (typeof val === 'number' || typeof val === 'boolean')
-            return String(val);
-        if (Array.isArray(val))
-            return val.map(v => `\n${pad}- ${toYaml(v, indent + 1).trimStart()}`).join('') || '[]';
-        if (typeof val === 'object') {
-            const entries = Object.entries(val);
-            if (entries.length === 0)
-                return '{}';
-            return entries.map(([k, v]) => {
-                const vStr = toYaml(v, indent + 1);
-                return typeof v === 'object' && v !== null && !Array.isArray(v)
-                    ? `\n${pad}${k}:${vStr}`
-                    : `\n${pad}${k}: ${vStr.trimStart()}`;
-            }).join('');
-        }
-        return String(val);
+function specToYaml(spec) {
+  function toYaml(val, indent = 0) {
+    const pad = "  ".repeat(indent);
+    if (val === null || val === void 0) return "null";
+    if (typeof val === "string") return val.includes("\n") || val.includes(":") ? `"${val.replace(/"/g, '\\"')}"` : val;
+    if (typeof val === "number" || typeof val === "boolean") return String(val);
+    if (Array.isArray(val)) return val.map((v) => `
+${pad}- ${toYaml(v, indent + 1).trimStart()}`).join("") || "[]";
+    if (typeof val === "object") {
+      const entries = Object.entries(val);
+      if (entries.length === 0) return "{}";
+      return entries.map(([k, v]) => {
+        const vStr = toYaml(v, indent + 1);
+        return typeof v === "object" && v !== null && !Array.isArray(v) ? `
+${pad}${k}:${vStr}` : `
+${pad}${k}: ${vStr.trimStart()}`;
+      }).join("");
     }
-    return `openapi: "${spec.openapi}"${toYaml(spec, 0)}`;
+    return String(val);
+  }
+  return `openapi: "${spec.openapi}"${toYaml(spec, 0)}`;
 }
+export {
+  generateOpenApiSpec,
+  specToYaml
+};
+//# sourceMappingURL=openapi-generator.js.map