@skrillex1224/playwright-toolkit 2.1.166 → 2.1.167

package/dist/proxy-meter.js

@@ -0,0 +1,549 @@
+import http from 'http';
+import net from 'net';
+import fs from 'fs';
+import { URL } from 'url';
+
+const HOST = '127.0.0.1';
+const PORT = Number(process.env.PROXY_METER_PORT || 8899);
+const LOG_PATH = String(process.env.PROXY_METER_LOG || '/tmp/proxy-meter.json');
+const FLUSH_INTERVAL_MS = Number(process.env.PROXY_METER_FLUSH_MS || 5000);
+const UPSTREAM_URL = String(process.env.PROXY_METER_UPSTREAM || '').trim();
+const RUN_ID = String(process.env.PROXY_METER_RUN_ID || process.env.APIFY_ACTOR_RUN_ID || '').trim();
+
+const isTruthy = (value) => /^(1|true|yes|on)$/i.test(String(value || '').trim());
+const DEBUG_ENABLED = isTruthy(process.env.PROXY_METER_DEBUG);
+const DEBUG_MAX_EVENTS = Math.max(10, Number(process.env.PROXY_METER_DEBUG_MAX_EVENTS || 400));
+const LARGE_RESPONSE_THRESHOLD_BYTES = Math.max(1024, Number(process.env.PROXY_METER_LARGE_BYTES || 1024 * 1024));
+
+const state = {
+  startedAt: new Date().toISOString(),
+  totalInBytes: 0,
+  totalOutBytes: 0,
+  hosts: {},
+  debug: DEBUG_ENABLED
+    ? {
+        enabled: true,
+        totalEvents: 0,
+        droppedEvents: 0,
+        events: [],
+        domains: {},
+        domainStatus: {},
+      }
+    : null,
+};
+
+const getHostBucket = (host) => {
+  if (!host) return null;
+  if (!state.hosts[host]) {
+    state.hosts[host] = { inBytes: 0, outBytes: 0, connections: 0, requests: 0 };
+  }
+  return state.hosts[host];
+};
+
+const addTraffic = (host, dir, bytes) => {
+  const bucket = getHostBucket(host);
+  if (!bucket) return;
+  const size = Number(bytes) || 0;
+  if (size <= 0) return;
+  if (dir === 'in') {
+    bucket.inBytes += size;
+    state.totalInBytes += size;
+  } else {
+    bucket.outBytes += size;
+    state.totalOutBytes += size;
+  }
+};
+
+const safeError = (error) => {
+  const message = String(error?.message || error || '').trim();
+  if (!message) return '';
+  return message.length > 240 ? message.slice(0, 240) : message;
+};
+
+const statusLabel = (statusCode, error) => {
+  const code = Number(statusCode) || 0;
+  if (error || code <= 0) return 'ERR';
+  return `${Math.floor(code / 100)}xx`;
+};
+
+const ensureDebugDomain = (host) => {
+  if (!state.debug || !host) return null;
+  if (!state.debug.domains[host]) {
+    state.debug.domains[host] = {
+      domain: host,
+      count: 0,
+      failedCount: 0,
+      inBytes: 0,
+      outBytes: 0,
+      totalBytes: 0,
+      durationTotalMs: 0,
+      largeResponseCount: 0,
+      reconnectCount: 0,
+      connectSuccessCount: 0,
+    };
+  }
+  return state.debug.domains[host];
+};
+
+const ensureDebugDomainStatus = (host, status) => {
+  if (!state.debug || !host || !status) return null;
+  const key = `${host}::${status}`;
+  if (!state.debug.domainStatus[key]) {
+    state.debug.domainStatus[key] = {
+      domain: host,
+      status,
+      count: 0,
+      failedCount: 0,
+      inBytes: 0,
+      outBytes: 0,
+      totalBytes: 0,
+      durationTotalMs: 0,
+      largeResponseCount: 0,
+      reconnectCount: 0,
+    };
+  }
+  return state.debug.domainStatus[key];
+};
+
+const recordDebugEvent = (event) => {
+  if (!state.debug || !event || typeof event !== 'object') return;
+  const host = String(event.host || '').trim();
+  if (!host) return;
+
+  const code = Number(event.statusCode) || 0;
+  const inBytes = Math.max(0, Number(event.inBytes) || 0);
+  const outBytes = Math.max(0, Number(event.outBytes) || 0);
+  const totalBytes = inBytes + outBytes;
+  const durationMs = Math.max(0, Number(event.durationMs) || 0);
+  const error = String(event.error || '').trim();
+  const failed = Boolean(error) || (code > 0 && code >= 400);
+  const status = statusLabel(code, error);
+  const large = inBytes >= LARGE_RESPONSE_THRESHOLD_BYTES;
+  const channel = String(event.channel || '').trim() || 'http';
+
+  const normalized = {
+    ts: event.ts || new Date().toISOString(),
+    runId: RUN_ID,
+    channel,
+    host,
+    method: String(event.method || ''),
+    path: String(event.path || ''),
+    statusCode: code,
+    durationMs,
+    inBytes,
+    outBytes,
+    totalBytes,
+    error,
+  };
+
+  state.debug.totalEvents += 1;
+  if (state.debug.events.length < DEBUG_MAX_EVENTS) {
+    state.debug.events.push(normalized);
+  } else {
+    state.debug.droppedEvents += 1;
+  }
+
+  const domain = ensureDebugDomain(host);
+  if (domain) {
+    domain.count += 1;
+    if (failed) domain.failedCount += 1;
+    domain.inBytes += inBytes;
+    domain.outBytes += outBytes;
+    domain.totalBytes += totalBytes;
+    domain.durationTotalMs += durationMs;
+    if (large) domain.largeResponseCount += 1;
+    if (channel === 'connect' && code === 200) {
+      domain.connectSuccessCount += 1;
+      if (domain.connectSuccessCount > 1) {
+        domain.reconnectCount += 1;
+      }
+    }
+  }
+
+  const byStatus = ensureDebugDomainStatus(host, status);
+  if (byStatus) {
+    byStatus.count += 1;
+    if (failed) byStatus.failedCount += 1;
+    byStatus.inBytes += inBytes;
+    byStatus.outBytes += outBytes;
+    byStatus.totalBytes += totalBytes;
+    byStatus.durationTotalMs += durationMs;
+    if (large) byStatus.largeResponseCount += 1;
+    if (channel === 'connect' && code === 200 && domain?.reconnectCount > 0) {
+      byStatus.reconnectCount = domain.reconnectCount;
+    }
+  }
+};
+
+const toDebugRow = (row) => {
+  const count = Number(row?.count) || 0;
+  const durationTotalMs = Number(row?.durationTotalMs) || 0;
+  const avgDurationMs = count > 0 ? durationTotalMs / count : 0;
+  const failedCount = Number(row?.failedCount) || 0;
+  const failureRatePct = count > 0 ? (failedCount / count) * 100 : 0;
+  return {
+    domain: String(row?.domain || ''),
+    status: String(row?.status || ''),
+    count,
+    failedCount,
+    inBytes: Math.max(0, Number(row?.inBytes) || 0),
+    outBytes: Math.max(0, Number(row?.outBytes) || 0),
+    totalBytes: Math.max(0, Number(row?.totalBytes) || 0),
+    avgDurationMs: Math.max(0, avgDurationMs),
+    reconnectCount: Math.max(0, Number(row?.reconnectCount) || 0),
+    largeResponseCount: Math.max(0, Number(row?.largeResponseCount) || 0),
+    failureRatePct,
+  };
+};
+
+const buildDebugSummary = () => {
+  if (!state.debug) return null;
+  const domainRows = Object.values(state.debug.domains || {}).map(toDebugRow);
+  const domainStatus = Object.values(state.debug.domainStatus || {}).map(toDebugRow);
+
+  const requestCount = domainRows.reduce((sum, row) => sum + row.count, 0);
+  const failedCount = domainRows.reduce((sum, row) => sum + row.failedCount, 0);
+  const largeResponseCount = domainRows.reduce((sum, row) => sum + row.largeResponseCount, 0);
+  const reconnectCount = domainRows.reduce((sum, row) => sum + row.reconnectCount, 0);
+  const failureRatePct = requestCount > 0 ? (failedCount / requestCount) * 100 : 0;
+
+  const topFailureDomains = [...domainRows]
+    .filter((row) => row.failedCount > 0)
+    .sort((a, b) => {
+      if (b.failedCount !== a.failedCount) return b.failedCount - a.failedCount;
+      return b.totalBytes - a.totalBytes;
+    })
+    .slice(0, 20);
+  const topLargeDomains = [...domainRows]
+    .filter((row) => row.largeResponseCount > 0)
+    .sort((a, b) => {
+      if (b.largeResponseCount !== a.largeResponseCount) return b.largeResponseCount - a.largeResponseCount;
+      return b.totalBytes - a.totalBytes;
+    })
+    .slice(0, 20);
+  const topReconnectDomains = [...domainRows]
+    .filter((row) => row.reconnectCount > 0)
+    .sort((a, b) => {
+      if (b.reconnectCount !== a.reconnectCount) return b.reconnectCount - a.reconnectCount;
+      return b.totalBytes - a.totalBytes;
+    })
+    .slice(0, 20);
+
+  domainStatus.sort((a, b) => {
+    if (b.totalBytes !== a.totalBytes) return b.totalBytes - a.totalBytes;
+    return b.count - a.count;
+  });
+
+  return {
+    largeResponseThresholdBytes: LARGE_RESPONSE_THRESHOLD_BYTES,
+    requestCount,
+    failedCount,
+    failureRatePct,
+    largeResponseCount,
+    reconnectCount,
+    domainStatus: domainStatus.slice(0, 200),
+    topFailureDomains,
+    topLargeDomains,
+    topReconnectDomains,
+  };
+};
+
+const flushSnapshot = () => {
+  try {
+    const snapshot = {
+      startedAt: state.startedAt,
+      totalInBytes: state.totalInBytes,
+      totalOutBytes: state.totalOutBytes,
+      hosts: state.hosts,
+      updatedAt: new Date().toISOString(),
+    };
+    if (state.debug) {
+      snapshot.debug = {
+        enabled: true,
+        totalEvents: state.debug.totalEvents,
+        sampledEvents: state.debug.events.length,
+        droppedEvents: state.debug.droppedEvents,
+        events: state.debug.events,
+        summary: buildDebugSummary(),
+      };
+    }
+    fs.writeFileSync(LOG_PATH, JSON.stringify(snapshot, null, 2));
+  } catch {}
+};
+
+const parseUpstream = () => {
+  if (!UPSTREAM_URL) return null;
+  let parsed;
+  try {
+    parsed = new URL(UPSTREAM_URL);
+  } catch {
+    throw new Error(`[proxy-meter] invalid upstream url: ${UPSTREAM_URL}`);
+  }
+  if (parsed.protocol !== 'http:') {
+    throw new Error(`[proxy-meter] upstream protocol not supported: ${parsed.protocol}`);
+  }
+  const authHeader = parsed.username
+    ? `Basic ${Buffer.from(`${decodeURIComponent(parsed.username)}:${decodeURIComponent(parsed.password || '')}`).toString('base64')}`
+    : '';
+  return {
+    host: parsed.hostname,
+    port: Number(parsed.port) || 80,
+    authHeader,
+  };
+};
+
+const upstream = parseUpstream();
+
+const resolveTarget = (req) => {
+  try {
+    if (req.url && req.url.startsWith('http')) {
+      return new URL(req.url);
+    }
+    const host = req.headers.host || '';
+    return new URL(`http://${host}${req.url || '/'}`);
+  } catch {
+    return null;
+  }
+};
+
+const createRequestTracker = (base) => {
+  const startedAt = Date.now();
+  let inBytes = 0;
+  let outBytes = 0;
+  let closed = false;
+  return {
+    addIn: (bytes) => {
+      inBytes += Math.max(0, Number(bytes) || 0);
+    },
+    addOut: (bytes) => {
+      outBytes += Math.max(0, Number(bytes) || 0);
+    },
+    close: (extra = {}) => {
+      if (closed) return;
+      closed = true;
+      recordDebugEvent({
+        ts: new Date().toISOString(),
+        runId: RUN_ID,
+        durationMs: Math.max(0, Date.now() - startedAt),
+        inBytes,
+        outBytes,
+        totalBytes: inBytes + outBytes,
+        ...base,
+        ...extra,
+      });
+    },
+  };
+};
+
+const forwardHttp = (req, res) => {
+  const target = resolveTarget(req);
+  const fallbackHost = String(req.headers?.host || '').split(':')[0].trim();
+  const hostname = target?.hostname || fallbackHost || '(unknown)';
+  const path = target ? `${target.pathname}${target.search || ''}` : String(req.url || '/');
+  const tracker = createRequestTracker({
+    channel: 'http',
+    host: hostname,
+    method: String(req.method || 'GET'),
+    path,
+  });
+
+  if (!target) {
+    tracker.close({ statusCode: 0, error: 'invalid_target_url' });
+    res.writeHead(502);
+    res.end('Bad Gateway');
+    return;
+  }
+
+  const bucket = getHostBucket(hostname);
+  if (bucket) bucket.requests += 1;
+
+  const headers = { ...req.headers };
+  if (upstream && upstream.authHeader && !headers['proxy-authorization']) {
+    headers['proxy-authorization'] = upstream.authHeader;
+  }
+
+  const requestOptions = upstream
+    ? {
+        hostname: upstream.host,
+        port: upstream.port,
+        method: req.method,
+        path: target.href,
+        headers,
+      }
+    : {
+        hostname: target.hostname,
+        port: Number(target.port) || 80,
+        method: req.method,
+        path: `${target.pathname}${target.search || ''}`,
+        headers,
+      };
+
+  let responseStatus = 0;
+  const proxyReq = http.request(requestOptions, (proxyRes) => {
+    responseStatus = Number(proxyRes.statusCode) || 0;
+    res.writeHead(responseStatus || 502, proxyRes.headers);
+    proxyRes.on('data', (chunk) => {
+      const size = chunk?.length || 0;
+      addTraffic(hostname, 'in', size);
+      tracker.addIn(size);
+    });
+    proxyRes.on('end', () => {
+      tracker.close({ statusCode: responseStatus });
+    });
+    proxyRes.on('error', (error) => {
+      tracker.close({ statusCode: responseStatus, error: safeError(error) });
+    });
+    proxyRes.pipe(res);
+  });
+
+  req.on('data', (chunk) => {
+    const size = chunk?.length || 0;
+    addTraffic(hostname, 'out', size);
+    tracker.addOut(size);
+  });
+  req.on('aborted', () => {
+    tracker.close({ statusCode: responseStatus || 499, error: 'request_aborted' });
+  });
+  req.pipe(proxyReq);
+
+  res.on('close', () => {
+    tracker.close({ statusCode: responseStatus || 499 });
+  });
+
+  proxyReq.on('error', (error) => {
+    tracker.close({ statusCode: responseStatus, error: safeError(error) });
+    res.writeHead(502);
+    res.end('Bad Gateway');
+  });
+};
+
+const forwardConnect = (req, clientSocket, head) => {
+  const [host, portText] = String(req.url || '').split(':');
+  const hostname = host || '(unknown)';
+  const port = Number(portText) || 443;
+  const tracker = createRequestTracker({
+    channel: 'connect',
+    host: hostname,
+    method: 'CONNECT',
+    path: `${hostname}:${port}`,
+  });
+
+  const bucket = getHostBucket(hostname);
+  if (bucket) bucket.connections += 1;
+
+  let established = false;
+  let responseStatus = 0;
+
+  const onError = (socketA, socketB) => (error) => {
+    tracker.close({ statusCode: responseStatus, error: safeError(error) });
+    try { socketA.destroy(); } catch {}
+    try { socketB.destroy(); } catch {}
+  };
+
+  const attachTrafficHooks = (upstreamSocket) => {
+    upstreamSocket.on('data', (chunk) => {
+      const size = chunk?.length || 0;
+      addTraffic(hostname, 'in', size);
+      tracker.addIn(size);
+    });
+    clientSocket.on('data', (chunk) => {
+      const size = chunk?.length || 0;
+      addTraffic(hostname, 'out', size);
+      tracker.addOut(size);
+    });
+    clientSocket.on('close', () => {
+      tracker.close({
+        statusCode: responseStatus || (established ? 200 : 0),
+        error: established ? '' : 'connect_closed',
+      });
+    });
+  };
+
+  if (upstream) {
+    const upstreamSocket = net.connect(upstream.port, upstream.host, () => {
+      const headers = [
+        `CONNECT ${hostname}:${port} HTTP/1.1`,
+        `Host: ${hostname}:${port}`,
+      ];
+      if (upstream.authHeader) {
+        headers.push(`Proxy-Authorization: ${upstream.authHeader}`);
+      }
+      headers.push('\r\n');
+      upstreamSocket.write(headers.join('\r\n'));
+    });
+
+    let responseBuffer = Buffer.alloc(0);
+    const onHandshake = (chunk) => {
+      responseBuffer = Buffer.concat([responseBuffer, chunk]);
+      const headerEnd = responseBuffer.indexOf('\r\n\r\n');
+      if (headerEnd === -1) return;
+      upstreamSocket.off('data', onHandshake);
+      const headerText = responseBuffer.slice(0, headerEnd).toString('utf8');
+      const statusLine = headerText.split('\r\n')[0] || '';
+      responseStatus = Number((statusLine.split(' ')[1] || '').trim());
+      if (responseStatus !== 200) {
+        tracker.close({ statusCode: responseStatus, error: `upstream_connect_${responseStatus || 0}` });
+        clientSocket.write('HTTP/1.1 502 Bad Gateway\r\n\r\n');
+        clientSocket.destroy();
+        upstreamSocket.destroy();
+        return;
+      }
+
+      established = true;
+      clientSocket.write('HTTP/1.1 200 Connection Established\r\n\r\n');
+      const rest = responseBuffer.slice(headerEnd + 4);
+      if (rest.length) {
+        addTraffic(hostname, 'in', rest.length);
+        tracker.addIn(rest.length);
+        clientSocket.write(rest);
+      }
+      if (head && head.length) {
+        addTraffic(hostname, 'out', head.length);
+        tracker.addOut(head.length);
+        upstreamSocket.write(head);
+      }
+      clientSocket.pipe(upstreamSocket);
+      upstreamSocket.pipe(clientSocket);
+    };
+
+    attachTrafficHooks(upstreamSocket);
+    upstreamSocket.on('data', onHandshake);
+    clientSocket.on('error', onError(clientSocket, upstreamSocket));
+    upstreamSocket.on('error', onError(clientSocket, upstreamSocket));
+    return;
+  }
+
+  const serverSocket = net.connect(port, hostname, () => {
+    established = true;
+    responseStatus = 200;
+    clientSocket.write('HTTP/1.1 200 Connection Established\r\n\r\n');
+    if (head && head.length) {
+      addTraffic(hostname, 'out', head.length);
+      tracker.addOut(head.length);
+      serverSocket.write(head);
+    }
+    clientSocket.pipe(serverSocket);
+    serverSocket.pipe(clientSocket);
+  });
+
+  attachTrafficHooks(serverSocket);
+  clientSocket.on('error', onError(clientSocket, serverSocket));
+  serverSocket.on('error', onError(clientSocket, serverSocket));
+};
+
+const server = http.createServer(forwardHttp);
+server.on('connect', forwardConnect);
+server.listen(PORT, HOST, () => {
+  console.log(`[proxy-meter] listening ${HOST}:${PORT} log=${LOG_PATH}`);
+});
+
+setInterval(flushSnapshot, FLUSH_INTERVAL_MS).unref();
+flushSnapshot();
+
+const shutdown = () => {
+  flushSnapshot();
+  process.exit(0);
+};
+
+process.on('SIGINT', shutdown);
+process.on('SIGTERM', shutdown);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@skrillex1224/playwright-toolkit",
-  "version": "2.1.166",
+  "version": "2.1.167",
   "description": "一个在 Apify/Crawlee Actor 中启用实时截图视图的实用工具库。",
   "main": "dist/index.cjs",
   "module": "dist/index.js",