@taodigital/crawler 0.0.0-stage → 1.2.1

package/src/playwright-recorder.ts

@@ -0,0 +1,170 @@
+/**
+ * Playwright test recorder — generate Playwright test code from crawl data.
+ *
+ * Given a list of crawled screens, generates a minimal Playwright test file that:
+ * 1. Navigates to each screen URL
+ * 2. Verifies the page title
+ * 3. Checks key elements visible on the page
+ * 4. Verifies no console errors (optional)
+ *
+ * This is a "smoke test" generator — not a full interaction recorder.
+ * Output: valid TypeScript Playwright test file ready to run with npx playwright test.
+ */
+
+export interface RecorderScreen {
+  id: string;
+  name: string;
+  url: string;
+  title?: string;
+  heading?: string;
+  markdownWordCount?: number;
+  smartLocator?: {
+    tag: string;          // e.g. 'button', 'a', 'input'
+    text?: string;        // visible text content
+    role?: string;        // ARIA role
+    placeholder?: string; // for inputs
+    nearText?: string;    // nearby label text
+  };
+}
+
+export interface RecorderOpts {
+  baseUrl?: string;
+  includeConsoleCheck?: boolean;
+  testTimeout?: number; // ms, default 30000
+  projectName?: string;
+  useSmartLocators?: boolean; // default true
+}
+
+function buildSmartLocatorHelper(): string {
+  return `
+// smartLocator — falls back to similarity search when primary selector fails.
+// Generated by ZeTa Crawler. Do not edit manually.
+async function smartLocator(page: any, primary: string, ref: { tag?: string; text?: string; role?: string; placeholder?: string }) {
+  try {
+    const el = page.locator(primary);
+    if (await el.count() > 0) return el;
+  } catch { /* fall through */ }
+  // Fallback: find by ARIA role + text
+  if (ref.role && ref.text) {
+    const byRole = page.getByRole(ref.role as any, { name: ref.text, exact: false });
+    if (await byRole.count() > 0) return byRole;
+  }
+  // Fallback: find by text content
+  if (ref.text) {
+    const byText = page.getByText(ref.text, { exact: false });
+    if (await byText.count() > 0) return byText;
+  }
+  // Fallback: find by placeholder
+  if (ref.placeholder) {
+    const byPlaceholder = page.getByPlaceholder(ref.placeholder, { exact: false });
+    if (await byPlaceholder.count() > 0) return byPlaceholder;
+  }
+  // Last resort: return the primary (will fail with original error)
+  return page.locator(primary);
+}`.trim();
+}
+
+function sanitizeTestName(name: string): string {
+  return name.replace(/[^a-zA-Z0-9 _-]/g, '').trim().slice(0, 80) || 'Screen';
+}
+
+function escapeString(s: string): string {
+  return s.replace(/\\/g, '\\\\').replace(/'/g, "\\'").replace(/`/g, '\\`');
+}
+
+export function generatePlaywrightTest(screens: RecorderScreen[], opts: RecorderOpts = {}): string {
+  const {
+    includeConsoleCheck = false,
+    testTimeout = 30_000,
+    projectName = 'ZeTa Crawl',
+    useSmartLocators = true,
+  } = opts;
+
+  const needsSmartLocator = useSmartLocators && screens.some(s => s.smartLocator != null);
+
+  const lines: string[] = [
+    `// Generated by ZeTa Crawler — ${new Date().toISOString()}`,
+    `// Project: ${escapeString(projectName)}`,
+    `// Screens: ${screens.length}`,
+    `// Run: npx playwright test`,
+    '',
+    "import { test, expect } from '@playwright/test';",
+    '',
+  ];
+
+  if (needsSmartLocator) {
+    lines.push(buildSmartLocatorHelper(), '');
+  }
+
+  lines.push(`test.setTimeout(${testTimeout});`, '');
+
+  if (includeConsoleCheck) {
+    lines.push(
+      `test.beforeEach(async ({ page }) => {`,
+      `  const errors: string[] = [];`,
+      `  page.on('console', msg => { if (msg.type() === 'error') errors.push(msg.text()); });`,
+      `  page.on('pageerror', err => errors.push(err.message));`,
+      `  (page as any).__consoleErrors = errors;`,
+      `});`,
+      '',
+    );
+  }
+
+  for (const screen of screens) {
+    const testName = sanitizeTestName(screen.name);
+    lines.push(
+      `test('${escapeString(testName)}', async ({ page }) => {`,
+      `  await page.goto('${escapeString(screen.url)}');`,
+      `  await expect(page).toHaveURL(/${escapeString(screen.url.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))}/);`,
+    );
+
+    if (screen.title) {
+      lines.push(`  await expect(page).toHaveTitle(/${escapeString(screen.title.slice(0, 40).replace(/[.*+?^${}()|[\]\\]/g, '\\$&'))}/i);`);
+    }
+
+    if (screen.heading) {
+      const h = escapeString(screen.heading.slice(0, 60));
+      if (needsSmartLocator && screen.smartLocator) {
+        const tag = escapeString(screen.smartLocator.tag);
+        const text = escapeString(screen.smartLocator.text ?? '');
+        lines.push(
+          `  const headingEl = await smartLocator(page, 'h1, h2, h3', { tag: '${tag}', text: '${text}' });`,
+          `  await expect(headingEl.first()).toContainText('${h}', { ignoreCase: true });`,
+        );
+      } else {
+        lines.push(`  await expect(page.locator('h1, h2, h3').first()).toContainText('${h}', { ignoreCase: true });`);
+      }
+    }
+
+    if (includeConsoleCheck) {
+      lines.push(`  expect((page as any).__consoleErrors ?? []).toHaveLength(0);`);
+    }
+
+    lines.push(`});`, '');
+  }
+
+  return lines.join('\n');
+}
+
+export function generatePlaywrightConfig(opts: { baseUrl?: string; reportDir?: string } = {}): string {
+  const { baseUrl = 'http://localhost:3000', reportDir = 'playwright-report' } = opts;
+  return `// playwright.config.ts — generated by ZeTa Crawler
+import { defineConfig } from '@playwright/test';
+
+export default defineConfig({
+  testDir: './tests',
+  reporter: [['html', { outputFolder: '${reportDir}' }]],
+  use: {
+    baseURL: '${baseUrl}',
+    headless: true,
+    screenshot: 'only-on-failure',
+    video: 'retain-on-failure',
+  },
+  projects: [
+    { name: 'chromium', use: { browserName: 'chromium' } },
+    { name: 'firefox', use: { browserName: 'firefox' } },
+    { name: 'webkit', use: { browserName: 'webkit' } },
+  ],
+});
+`;
+}

package/src/policies/crawl-safety.ts

@@ -0,0 +1,52 @@
+import type { DomainProfile } from '../profiles/domain-profile.js';
+
+export interface CrawlSafetyDecision {
+  allowed: boolean;
+  reason?: string;
+  severity: 'info' | 'warn' | 'block';
+}
+
+const _extraDestructive = process.env.CRAWL_EXTRA_DESTRUCTIVE_PATTERNS
+  ? process.env.CRAWL_EXTRA_DESTRUCTIVE_PATTERNS.split(',').map((p) => p.trim()).filter(Boolean)
+  : [];
+const DESTRUCTIVE_ACTION = new RegExp(
+  `\\b(delete|remove|destroy|deactivate|disable|cancel|refund|void|transfer|withdraw|pay now|place order|submit order|confirm payment|send money|publish|approve|reject|bulk update|drop${_extraDestructive.length ? `|${_extraDestructive.join('|')}` : ''})\\b`,
+  'i',
+);
+const SECRET_LIKE = /\b(api[_ -]?key|secret|token|password|authorization|cookie|set-cookie)\b/i;
+
+export function assessActionSafety(label: string, profile: DomainProfile): CrawlSafetyDecision {
+  const riskyByProfile = profile.riskyActions.some((risk) => label.toLowerCase().includes(risk.toLowerCase()));
+  if (DESTRUCTIVE_ACTION.test(label) || riskyByProfile) {
+    return {
+      allowed: false,
+      severity: 'block',
+      reason: `Blocked risky action "${label}" for ${profile.industry} profile.`,
+    };
+  }
+  return { allowed: true, severity: 'info' };
+}
+
+export function redactSensitiveValue(value: string): string {
+  let redacted = value
+    .replace(/[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi, '[email]')
+    .replace(/\b(?:\+?\d[\d -]{8,}\d)\b/g, '[phone-or-id]')
+    .replace(/\b(?:\d[ -]*?){13,19}\b/g, '[card-like-number]')
+    .replace(/\b[A-Za-z0-9_-]{24,}\b/g, '[token-like-value]');
+  if (SECRET_LIKE.test(redacted)) redacted = redacted.replace(/:\s*[^,\s}]+/g, ': [redacted]');
+  return redacted;
+}
+
+export function redactJson(value: unknown): unknown {
+  if (typeof value === 'string') return redactSensitiveValue(value);
+  if (Array.isArray(value)) return value.map(redactJson);
+  if (value && typeof value === 'object') {
+    const out: Record<string, unknown> = {};
+    for (const [key, item] of Object.entries(value)) {
+      if (SECRET_LIKE.test(key)) out[key] = '[redacted]';
+      else out[key] = redactJson(item);
+    }
+    return out;
+  }
+  return value;
+}

package/src/profiles/domain-profile.ts

@@ -0,0 +1,190 @@
+export type Industry =
+  | 'ecommerce'
+  | 'banking'
+  | 'fintech'
+  | 'healthcare'
+  | 'pharma'
+  | 'agriculture'
+  | 'legacy-enterprise'
+  | 'saas'
+  | 'unknown';
+
+export interface DomainProfile {
+  industry: Industry;
+  confidence: number;
+  importantScreenPatterns: string[];
+  workflowGoals: string[];
+  piiRules: string[];
+  riskyActions: string[];
+  safeProbeRules: string[];
+  requiredArtifacts: string[];
+  evidence: string[];
+}
+
+const COMMON_REQUIRED_ARTIFACTS = [
+  'screenshot',
+  'dom',
+  'markdown',
+  'accessibility',
+  'network',
+  'performance',
+];
+
+const PROFILES: Record<Industry, Omit<DomainProfile, 'confidence' | 'evidence'>> = {
+  ecommerce: {
+    industry: 'ecommerce',
+    importantScreenPatterns: ['catalog', 'product', 'search', 'filter', 'cart', 'checkout', 'payment', 'order', 'returns'],
+    workflowGoals: ['discover product catalogue', 'map cart and checkout', 'capture order confirmation and failure states', 'map search/filter/sort'],
+    piiRules: ['mask email', 'mask phone', 'mask address', 'mask card-like numbers'],
+    riskyActions: ['place order', 'submit payment', 'save card', 'apply real coupon'],
+    safeProbeRules: ['do not click final payment/order buttons', 'use invalid test card data only', 'stop before irreversible purchase'],
+    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
+  },
+  banking: {
+    industry: 'banking',
+    importantScreenPatterns: ['login', 'mfa', 'dashboard', 'accounts', 'transactions', 'transfer', 'beneficiary', 'statement', 'dispute'],
+    workflowGoals: ['map account overview', 'map transactions and statements', 'map transfer setup without submitting', 'map MFA/session timeout states'],
+    piiRules: ['mask account numbers', 'mask balances', 'mask names', 'mask addresses', 'mask tax identifiers'],
+    riskyActions: ['transfer money', 'add beneficiary', 'submit dispute', 'download sensitive statement'],
+    safeProbeRules: ['read-only by default', 'never submit transfer confirmation', 'never alter beneficiary data'],
+    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
+  },
+  fintech: {
+    industry: 'fintech',
+    importantScreenPatterns: ['kyc', 'wallet', 'transactions', 'cards', 'payments', 'payouts', 'risk', 'compliance', 'disputes'],
+    workflowGoals: ['map KYC onboarding', 'map wallet/payment states', 'map failed payment and dispute flows', 'capture compliance screens'],
+    piiRules: ['mask card numbers', 'mask bank accounts', 'mask identity documents', 'mask balances'],
+    riskyActions: ['send payment', 'withdraw funds', 'change card status', 'submit KYC with real data'],
+    safeProbeRules: ['sandbox data only', 'stop before money movement', 'never upload real identity documents'],
+    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
+  },
+  healthcare: {
+    industry: 'healthcare',
+    importantScreenPatterns: ['patient', 'appointment', 'claims', 'provider', 'prescription', 'records', 'lab', 'consent'],
+    workflowGoals: ['map patient lookup safely', 'map appointment booking before final submit', 'map claims and prescription screens', 'capture consent and records UX'],
+    piiRules: ['mask PHI', 'mask patient names', 'mask MRN', 'mask DOB', 'mask diagnosis', 'mask prescriptions'],
+    riskyActions: ['schedule real appointment', 'submit claim', 'change medication', 'message provider with PHI'],
+    safeProbeRules: ['use synthetic patients only', 'do not submit clinical changes', 'avoid external notifications'],
+    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
+  },
+  pharma: {
+    industry: 'pharma',
+    importantScreenPatterns: ['product', 'dosage', 'safety', 'batch', 'adverse-event', 'regulatory', 'trial', 'label'],
+    workflowGoals: ['map product and safety content', 'map adverse event intake before submit', 'capture regulatory and batch workflows'],
+    piiRules: ['mask patient reports', 'mask reporter contact', 'mask trial participant identifiers'],
+    riskyActions: ['submit adverse event', 'change safety record', 'publish regulatory content'],
+    safeProbeRules: ['stop before regulatory or safety submission', 'use synthetic reports only'],
+    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
+  },
+  agriculture: {
+    industry: 'agriculture',
+    importantScreenPatterns: ['crop', 'field', 'weather', 'inventory', 'marketplace', 'logistics', 'sensor', 'subsidy'],
+    workflowGoals: ['map crop and field planning', 'map inventory/marketplace flows', 'capture logistics and sensor dashboards'],
+    piiRules: ['mask farmer identity', 'mask location precision', 'mask subsidy identifiers'],
+    riskyActions: ['place market order', 'submit subsidy application', 'change sensor/device control'],
+    safeProbeRules: ['avoid irreversible orders', 'avoid device actuation', 'use test farm data'],
+    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
+  },
+  'legacy-enterprise': {
+    industry: 'legacy-enterprise',
+    importantScreenPatterns: ['frameset', 'iframe', 'popup', 'table', 'search', 'admin', 'report', 'export'],
+    workflowGoals: ['map menu trees', 'capture iframe and popup states', 'map table filters and reports', 'detect session timeout'],
+    piiRules: ['mask table cells that look like PII', 'mask exported identifiers'],
+    riskyActions: ['bulk update', 'delete record', 'export production data', 'submit admin change'],
+    safeProbeRules: ['read-only table probing', 'never click delete/bulk actions', 'capture popups without committing'],
+    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
+  },
+  saas: {
+    industry: 'saas',
+    importantScreenPatterns: ['dashboard', 'settings', 'users', 'billing', 'integrations', 'reports', 'admin', 'workspace'],
+    workflowGoals: ['map workspace navigation', 'map settings and integrations', 'map role/user management without changing state', 'capture dashboards and reports'],
+    piiRules: ['mask emails', 'mask tokens', 'mask API keys', 'mask billing data'],
+    riskyActions: ['delete workspace', 'invite real user', 'rotate key', 'change billing plan'],
+    safeProbeRules: ['never submit destructive settings', 'redact secrets immediately', 'use sandbox users'],
+    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
+  },
+  unknown: {
+    industry: 'unknown',
+    importantScreenPatterns: ['login', 'dashboard', 'settings', 'search', 'detail', 'report', 'form'],
+    workflowGoals: ['map top-level navigation', 'capture forms and validation states', 'discover hidden menus', 'classify domain after first pages'],
+    piiRules: ['mask emails', 'mask phone numbers', 'mask long identifiers', 'mask card-like numbers'],
+    riskyActions: ['delete', 'submit payment', 'transfer', 'publish', 'send notification'],
+    safeProbeRules: ['read-only by default', 'stop before irreversible actions', 'use synthetic data only'],
+    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
+  },
+};
+
+const BASE_INDUSTRY_HINTS: Array<{ industry: Industry; terms: RegExp[] }> = [
+  { industry: 'ecommerce', terms: [/cart/i, /checkout/i, /product/i, /catalog/i, /sku/i, /wishlist/i, /order/i] },
+  { industry: 'banking', terms: [/account/i, /statement/i, /beneficiar/i, /transfer/i, /iban/i, /routing/i, /balance/i] },
+  { industry: 'fintech', terms: [/wallet/i, /kyc/i, /payout/i, /card/i, /payment/i, /risk/i, /dispute/i] },
+  { industry: 'healthcare', terms: [/patient/i, /provider/i, /appointment/i, /claim/i, /prescription/i, /medical/i, /lab result/i] },
+  { industry: 'pharma', terms: [/pharma/i, /dosage/i, /adverse/i, /batch/i, /regulatory/i, /clinical trial/i, /label/i] },
+  { industry: 'agriculture', terms: [/crop/i, /field/i, /farm/i, /harvest/i, /sensor/i, /weather/i, /subsidy/i] },
+  { industry: 'legacy-enterprise', terms: [/frameset/i, /iframe/i, /popup/i, /legacy/i, /report export/i, /mainframe/i] },
+  { industry: 'saas', terms: [/workspace/i, /tenant/i, /integration/i, /billing/i, /api key/i, /admin/i, /dashboard/i] },
+];
+
+// Allow additional industry hint keywords via ENV without replacing the baseline.
+// Format: JSON array of { industry: string; terms: string[] } — terms are treated as case-insensitive regex.
+// Example: EXTRA_INDUSTRY_HINTS_JSON='[{"industry":"saas","terms":["platform","multitenant"]}]'
+const _extraHints: Array<{ industry: Industry; terms: RegExp[] }> = (() => {
+  const raw = process.env.EXTRA_INDUSTRY_HINTS_JSON;
+  if (!raw) return [];
+  try {
+    const parsed: Array<{ industry: string; terms: string[] }> = JSON.parse(raw);
+    return parsed.map(h => ({ industry: h.industry as Industry, terms: h.terms.map(t => new RegExp(t, 'i')) }));
+  } catch { return []; }
+})();
+
+const INDUSTRY_HINTS = [...BASE_INDUSTRY_HINTS, ..._extraHints];
+
+// Allow patching profile fields (riskyActions, piiRules, etc.) per industry via ENV.
+// Format: JSON object keyed by industry, value is partial DomainProfile fields to merge.
+// Example: DOMAIN_PROFILE_OVERRIDES_JSON='{"saas":{"riskyActions":["delete workspace","export all data"]}}'
+const _profileOverrides: Partial<Record<Industry, Partial<Omit<DomainProfile, 'confidence' | 'evidence'>>>> = (() => {
+  const raw = process.env.DOMAIN_PROFILE_OVERRIDES_JSON;
+  if (!raw) return {};
+  try { return JSON.parse(raw); } catch { return {}; }
+})();
+
+export function classifyDomainProfile(input: {
+  appUrl: string;
+  projectName?: string | null;
+  appType?: string | null;
+  seedText?: string;
+}): DomainProfile {
+  const haystack = [input.appUrl, input.projectName ?? '', input.appType ?? '', input.seedText ?? ''].join(' ');
+  const scores = new Map<Industry, { score: number; evidence: string[] }>();
+  for (const { industry, terms } of INDUSTRY_HINTS) {
+    for (const term of terms) {
+      if (term.test(haystack)) {
+        const prev = scores.get(industry) ?? { score: 0, evidence: [] };
+        prev.score += 1;
+        prev.evidence.push(term.source);
+        scores.set(industry, prev);
+      }
+    }
+  }
+  let selected: Industry = 'unknown';
+  let best = 0;
+  let evidence: string[] = [];
+  for (const [industry, result] of scores) {
+    if (result.score > best) {
+      selected = industry;
+      best = result.score;
+      evidence = result.evidence;
+    }
+  }
+  const base = PROFILES[selected];
+  return {
+    ...base,
+    confidence: best === 0 ? 0.25 : Math.min(0.95, 0.35 + best * 0.15),
+    evidence,
+  };
+}
+
+export function getProfile(industry: Industry): DomainProfile {
+  const base = PROFILES[industry] ?? PROFILES.unknown;
+  return { ...base, confidence: 1, evidence: ['explicit'] };
+}

package/src/proxy.ts

@@ -0,0 +1,238 @@
+/**
+ * Free proxy auto-selection for bot-protection bypass.
+ *
+ * Source: proxifly/free-proxy-list (MIT) via jsDelivr CDN.
+ * These are datacenter proxies — they bypass simple IP blocks but NOT
+ * full Cloudflare Bot Manager / Akamai. For Cloudflare-protected sites,
+ * users need a residential proxy (Bright Data, Oxylabs, etc.).
+ */
+
+import { createConnection } from 'net';
+
+const PROXY_LIST_URL =
+  'https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.json';
+
+interface ProxyEntry {
+  ip: string;
+  port: number;
+  protocols: string[];
+  upTime?: number;
+  speed?: number;
+}
+
+/** Fetch and filter proxy list. Returns http proxies sorted by speed (fastest first). */
+async function fetchCandidates(): Promise<ProxyEntry[]> {
+  const res = await fetch(PROXY_LIST_URL, {
+    signal: AbortSignal.timeout(10_000),
+    headers: { 'Accept': 'application/json' },
+  });
+  if (!res.ok) throw new Error(`proxy list fetch failed: ${res.status}`);
+  const raw: ProxyEntry[] = await res.json();
+  return raw
+    .filter((p) => p.protocols?.includes('http') && p.port > 0 && (p.upTime ?? 0) >= 30)
+    .sort((a, b) => (a.speed ?? 99999) - (b.speed ?? 99999))
+    .slice(0, 50); // test at most 50 candidates
+}
+
+/** TCP-level proxy liveness check via HTTP CONNECT tunnel. No extra dependencies. */
+function testProxy(ip: string, port: number, timeoutMs: number): Promise<boolean> {
+  return new Promise((resolve) => {
+    let settled = false;
+    const settle = (ok: boolean) => {
+      if (settled) return;
+      settled = true;
+      resolve(ok);
+      try { sock.destroy(); } catch { /* ignore */ }
+    };
+
+    const timer = setTimeout(() => settle(false), timeoutMs);
+    const sock = createConnection({ host: ip, port, timeout: timeoutMs });
+
+    sock.on('connect', () => {
+      sock.write('CONNECT example.com:80 HTTP/1.1\r\nHost: example.com:80\r\n\r\n');
+      sock.once('data', (data) => {
+        clearTimeout(timer);
+        const ok = /^HTTP\/1\.[01] 200/.test(data.toString());
+        settle(ok);
+      });
+    });
+    sock.on('error', () => { clearTimeout(timer); settle(false); });
+    sock.on('timeout', () => { clearTimeout(timer); settle(false); });
+  });
+}
+
+/** Test `batchSize` proxies concurrently and return first working one. */
+async function testBatch(
+  candidates: ProxyEntry[],
+  start: number,
+  batchSize: number,
+  timeoutMs: number,
+): Promise<string | null> {
+  const batch = candidates.slice(start, start + batchSize);
+  if (batch.length === 0) return null;
+
+  const results = await Promise.all(
+    batch.map(async (p) => {
+      const ok = await testProxy(p.ip, p.port, timeoutMs);
+      return ok ? `http://${p.ip}:${p.port}` : null;
+    }),
+  );
+
+  return results.find((r) => r !== null) ?? null;
+}
+
+/**
+ * Pick a working free proxy. Tests in parallel batches of 10 with a 4s timeout.
+ * Returns null if no working proxy found (not an error — crawl can proceed without proxy).
+ */
+export async function pickFreeProxy(): Promise<string | null> {
+  let candidates: ProxyEntry[];
+  try {
+    candidates = await fetchCandidates();
+  } catch (err: any) {
+    console.warn('[proxy] failed to fetch proxy list:', err?.message ?? err);
+    return null;
+  }
+
+  const BATCH = 10;
+  const TIMEOUT = 4_000;
+
+  for (let i = 0; i < candidates.length; i += BATCH) {
+    const found = await testBatch(candidates, i, BATCH, TIMEOUT);
+    if (found) {
+      console.info(`[proxy] selected free proxy: ${found} (tested ${i + BATCH} candidates)`);
+      return found;
+    }
+  }
+
+  console.warn('[proxy] no working free proxy found in list');
+  return null;
+}
+
+// ── Tiered Proxy Rotation ─────────────────────────────────────────────────────
+
+export type ProxyTier = 'free' | 'datacenter' | 'residential' | 'premium';
+
+export interface ProxyTierConfig {
+  /** List of proxy URLs for this tier. E.g. ['http://user:pass@dc1.proxy:8080'] */
+  urls: string[];
+  tier: ProxyTier;
+}
+
+interface DomainHistory {
+  tier: ProxyTier;
+  consecutiveFailures: number;
+  lastEscalatedAt: number;
+  totalRequests: number;
+  successRate: number;
+}
+
+const TIER_ORDER: ProxyTier[] = ['free', 'datacenter', 'residential', 'premium'];
+
+function nextTier(current: ProxyTier): ProxyTier {
+  const idx = TIER_ORDER.indexOf(current);
+  return TIER_ORDER[Math.min(idx + 1, TIER_ORDER.length - 1)];
+}
+
+/**
+ * Tiered proxy manager with per-domain error histograms.
+ *
+ * Tracks success/failure per domain and automatically escalates to higher proxy tiers
+ * on consecutive failures. Applies time-decay to probe returning to lower tiers.
+ *
+ * Usage:
+ *   const mgr = new TieredProxyManager([{ tier: 'datacenter', urls: [...] }]);
+ *   const proxy = await mgr.selectProxy('api.salesforce.com');
+ *   // ... make request ...
+ *   mgr.recordResult('api.salesforce.com', success);
+ */
+export class TieredProxyManager {
+  private tiers: Map<ProxyTier, string[]> = new Map();
+  private domainHistory: Map<string, DomainHistory> = new Map();
+  private readonly ESCALATE_AFTER = 3; // consecutive failures before escalation
+  private readonly DECAY_AFTER_MS = 30 * 60 * 1000; // 30 min: probe tier downgrade
+
+  constructor(configs: ProxyTierConfig[]) {
+    for (const cfg of configs) {
+      if (cfg.urls.length > 0) this.tiers.set(cfg.tier, cfg.urls);
+    }
+  }
+
+  private getHistory(domain: string): DomainHistory {
+    if (!this.domainHistory.has(domain)) {
+      this.domainHistory.set(domain, {
+        tier: this.lowestAvailableTier(),
+        consecutiveFailures: 0,
+        lastEscalatedAt: 0,
+        totalRequests: 0,
+        successRate: 1,
+      });
+    }
+    return this.domainHistory.get(domain)!;
+  }
+
+  private lowestAvailableTier(): ProxyTier {
+    for (const t of TIER_ORDER) {
+      if (this.tiers.has(t)) return t;
+    }
+    return 'free';
+  }
+
+  /** Pick a proxy URL for this domain, based on its tier history. */
+  selectProxy(domain: string): string | null {
+    const history = this.getHistory(domain);
+
+    // Time-decay: if we've been on elevated tier for DECAY_AFTER_MS, try stepping down
+    if (history.consecutiveFailures === 0 && history.lastEscalatedAt > 0) {
+      const elapsed = Date.now() - history.lastEscalatedAt;
+      if (elapsed > this.DECAY_AFTER_MS) {
+        const currentIdx = TIER_ORDER.indexOf(history.tier);
+        if (currentIdx > 0) {
+          const lowerTier = TIER_ORDER[currentIdx - 1];
+          if (this.tiers.has(lowerTier)) {
+            history.tier = lowerTier;
+            history.lastEscalatedAt = 0;
+          }
+        }
+      }
+    }
+
+    const urls = this.tiers.get(history.tier);
+    if (!urls || urls.length === 0) return null;
+    // Round-robin within tier
+    const idx = history.totalRequests % urls.length;
+    return urls[idx];
+  }
+
+  /** Record a request result for a domain. Triggers tier escalation on repeated failures. */
+  recordResult(domain: string, success: boolean): void {
+    const history = this.getHistory(domain);
+    history.totalRequests++;
+    if (success) {
+      history.consecutiveFailures = 0;
+      // Update EMA success rate
+      history.successRate = history.successRate * 0.9 + 0.1;
+    } else {
+      history.consecutiveFailures++;
+      history.successRate = history.successRate * 0.9;
+      if (history.consecutiveFailures >= this.ESCALATE_AFTER) {
+        const elevated = nextTier(history.tier);
+        if (elevated !== history.tier && this.tiers.has(elevated)) {
+          console.warn(`[proxy-tier] domain=${domain} escalating ${history.tier}→${elevated} after ${history.consecutiveFailures} failures`);
+          history.tier = elevated;
+          history.lastEscalatedAt = Date.now();
+          history.consecutiveFailures = 0;
+        }
+      }
+    }
+  }
+
+  stats(): Array<{ domain: string; tier: ProxyTier; successRate: number; totalRequests: number }> {
+    return Array.from(this.domainHistory.entries()).map(([domain, h]) => ({
+      domain,
+      tier: h.tier,
+      successRate: Math.round(h.successRate * 100) / 100,
+      totalRequests: h.totalRequests,
+    }));
+  }
+}