@taodigital/crawler 0.0.0-stage → 1.2.1

package/dist/profiles/domain-profile.js

@@ -0,0 +1,160 @@
+const COMMON_REQUIRED_ARTIFACTS = [
+  "screenshot",
+  "dom",
+  "markdown",
+  "accessibility",
+  "network",
+  "performance"
+];
+const PROFILES = {
+  ecommerce: {
+    industry: "ecommerce",
+    importantScreenPatterns: ["catalog", "product", "search", "filter", "cart", "checkout", "payment", "order", "returns"],
+    workflowGoals: ["discover product catalogue", "map cart and checkout", "capture order confirmation and failure states", "map search/filter/sort"],
+    piiRules: ["mask email", "mask phone", "mask address", "mask card-like numbers"],
+    riskyActions: ["place order", "submit payment", "save card", "apply real coupon"],
+    safeProbeRules: ["do not click final payment/order buttons", "use invalid test card data only", "stop before irreversible purchase"],
+    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
+  },
+  banking: {
+    industry: "banking",
+    importantScreenPatterns: ["login", "mfa", "dashboard", "accounts", "transactions", "transfer", "beneficiary", "statement", "dispute"],
+    workflowGoals: ["map account overview", "map transactions and statements", "map transfer setup without submitting", "map MFA/session timeout states"],
+    piiRules: ["mask account numbers", "mask balances", "mask names", "mask addresses", "mask tax identifiers"],
+    riskyActions: ["transfer money", "add beneficiary", "submit dispute", "download sensitive statement"],
+    safeProbeRules: ["read-only by default", "never submit transfer confirmation", "never alter beneficiary data"],
+    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
+  },
+  fintech: {
+    industry: "fintech",
+    importantScreenPatterns: ["kyc", "wallet", "transactions", "cards", "payments", "payouts", "risk", "compliance", "disputes"],
+    workflowGoals: ["map KYC onboarding", "map wallet/payment states", "map failed payment and dispute flows", "capture compliance screens"],
+    piiRules: ["mask card numbers", "mask bank accounts", "mask identity documents", "mask balances"],
+    riskyActions: ["send payment", "withdraw funds", "change card status", "submit KYC with real data"],
+    safeProbeRules: ["sandbox data only", "stop before money movement", "never upload real identity documents"],
+    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
+  },
+  healthcare: {
+    industry: "healthcare",
+    importantScreenPatterns: ["patient", "appointment", "claims", "provider", "prescription", "records", "lab", "consent"],
+    workflowGoals: ["map patient lookup safely", "map appointment booking before final submit", "map claims and prescription screens", "capture consent and records UX"],
+    piiRules: ["mask PHI", "mask patient names", "mask MRN", "mask DOB", "mask diagnosis", "mask prescriptions"],
+    riskyActions: ["schedule real appointment", "submit claim", "change medication", "message provider with PHI"],
+    safeProbeRules: ["use synthetic patients only", "do not submit clinical changes", "avoid external notifications"],
+    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
+  },
+  pharma: {
+    industry: "pharma",
+    importantScreenPatterns: ["product", "dosage", "safety", "batch", "adverse-event", "regulatory", "trial", "label"],
+    workflowGoals: ["map product and safety content", "map adverse event intake before submit", "capture regulatory and batch workflows"],
+    piiRules: ["mask patient reports", "mask reporter contact", "mask trial participant identifiers"],
+    riskyActions: ["submit adverse event", "change safety record", "publish regulatory content"],
+    safeProbeRules: ["stop before regulatory or safety submission", "use synthetic reports only"],
+    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
+  },
+  agriculture: {
+    industry: "agriculture",
+    importantScreenPatterns: ["crop", "field", "weather", "inventory", "marketplace", "logistics", "sensor", "subsidy"],
+    workflowGoals: ["map crop and field planning", "map inventory/marketplace flows", "capture logistics and sensor dashboards"],
+    piiRules: ["mask farmer identity", "mask location precision", "mask subsidy identifiers"],
+    riskyActions: ["place market order", "submit subsidy application", "change sensor/device control"],
+    safeProbeRules: ["avoid irreversible orders", "avoid device actuation", "use test farm data"],
+    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
+  },
+  "legacy-enterprise": {
+    industry: "legacy-enterprise",
+    importantScreenPatterns: ["frameset", "iframe", "popup", "table", "search", "admin", "report", "export"],
+    workflowGoals: ["map menu trees", "capture iframe and popup states", "map table filters and reports", "detect session timeout"],
+    piiRules: ["mask table cells that look like PII", "mask exported identifiers"],
+    riskyActions: ["bulk update", "delete record", "export production data", "submit admin change"],
+    safeProbeRules: ["read-only table probing", "never click delete/bulk actions", "capture popups without committing"],
+    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
+  },
+  saas: {
+    industry: "saas",
+    importantScreenPatterns: ["dashboard", "settings", "users", "billing", "integrations", "reports", "admin", "workspace"],
+    workflowGoals: ["map workspace navigation", "map settings and integrations", "map role/user management without changing state", "capture dashboards and reports"],
+    piiRules: ["mask emails", "mask tokens", "mask API keys", "mask billing data"],
+    riskyActions: ["delete workspace", "invite real user", "rotate key", "change billing plan"],
+    safeProbeRules: ["never submit destructive settings", "redact secrets immediately", "use sandbox users"],
+    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
+  },
+  unknown: {
+    industry: "unknown",
+    importantScreenPatterns: ["login", "dashboard", "settings", "search", "detail", "report", "form"],
+    workflowGoals: ["map top-level navigation", "capture forms and validation states", "discover hidden menus", "classify domain after first pages"],
+    piiRules: ["mask emails", "mask phone numbers", "mask long identifiers", "mask card-like numbers"],
+    riskyActions: ["delete", "submit payment", "transfer", "publish", "send notification"],
+    safeProbeRules: ["read-only by default", "stop before irreversible actions", "use synthetic data only"],
+    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
+  }
+};
+const BASE_INDUSTRY_HINTS = [
+  { industry: "ecommerce", terms: [/cart/i, /checkout/i, /product/i, /catalog/i, /sku/i, /wishlist/i, /order/i] },
+  { industry: "banking", terms: [/account/i, /statement/i, /beneficiar/i, /transfer/i, /iban/i, /routing/i, /balance/i] },
+  { industry: "fintech", terms: [/wallet/i, /kyc/i, /payout/i, /card/i, /payment/i, /risk/i, /dispute/i] },
+  { industry: "healthcare", terms: [/patient/i, /provider/i, /appointment/i, /claim/i, /prescription/i, /medical/i, /lab result/i] },
+  { industry: "pharma", terms: [/pharma/i, /dosage/i, /adverse/i, /batch/i, /regulatory/i, /clinical trial/i, /label/i] },
+  { industry: "agriculture", terms: [/crop/i, /field/i, /farm/i, /harvest/i, /sensor/i, /weather/i, /subsidy/i] },
+  { industry: "legacy-enterprise", terms: [/frameset/i, /iframe/i, /popup/i, /legacy/i, /report export/i, /mainframe/i] },
+  { industry: "saas", terms: [/workspace/i, /tenant/i, /integration/i, /billing/i, /api key/i, /admin/i, /dashboard/i] }
+];
+const _extraHints = (() => {
+  const raw = process.env.EXTRA_INDUSTRY_HINTS_JSON;
+  if (!raw) return [];
+  try {
+    const parsed = JSON.parse(raw);
+    return parsed.map((h) => ({ industry: h.industry, terms: h.terms.map((t) => new RegExp(t, "i")) }));
+  } catch {
+    return [];
+  }
+})();
+const INDUSTRY_HINTS = [...BASE_INDUSTRY_HINTS, ..._extraHints];
+const _profileOverrides = (() => {
+  const raw = process.env.DOMAIN_PROFILE_OVERRIDES_JSON;
+  if (!raw) return {};
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return {};
+  }
+})();
+function classifyDomainProfile(input) {
+  const haystack = [input.appUrl, input.projectName ?? "", input.appType ?? "", input.seedText ?? ""].join(" ");
+  const scores = /* @__PURE__ */ new Map();
+  for (const { industry, terms } of INDUSTRY_HINTS) {
+    for (const term of terms) {
+      if (term.test(haystack)) {
+        const prev = scores.get(industry) ?? { score: 0, evidence: [] };
+        prev.score += 1;
+        prev.evidence.push(term.source);
+        scores.set(industry, prev);
+      }
+    }
+  }
+  let selected = "unknown";
+  let best = 0;
+  let evidence = [];
+  for (const [industry, result] of scores) {
+    if (result.score > best) {
+      selected = industry;
+      best = result.score;
+      evidence = result.evidence;
+    }
+  }
+  const base = PROFILES[selected];
+  return {
+    ...base,
+    confidence: best === 0 ? 0.25 : Math.min(0.95, 0.35 + best * 0.15),
+    evidence
+  };
+}
+function getProfile(industry) {
+  const base = PROFILES[industry] ?? PROFILES.unknown;
+  return { ...base, confidence: 1, evidence: ["explicit"] };
+}
+export {
+  classifyDomainProfile,
+  getProfile
+};
+//# sourceMappingURL=domain-profile.js.map

package/dist/profiles/domain-profile.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../src/profiles/domain-profile.ts"],
+  "sourcesContent": ["export type Industry =\n  | 'ecommerce'\n  | 'banking'\n  | 'fintech'\n  | 'healthcare'\n  | 'pharma'\n  | 'agriculture'\n  | 'legacy-enterprise'\n  | 'saas'\n  | 'unknown';\n\nexport interface DomainProfile {\n  industry: Industry;\n  confidence: number;\n  importantScreenPatterns: string[];\n  workflowGoals: string[];\n  piiRules: string[];\n  riskyActions: string[];\n  safeProbeRules: string[];\n  requiredArtifacts: string[];\n  evidence: string[];\n}\n\nconst COMMON_REQUIRED_ARTIFACTS = [\n  'screenshot',\n  'dom',\n  'markdown',\n  'accessibility',\n  'network',\n  'performance',\n];\n\nconst PROFILES: Record<Industry, Omit<DomainProfile, 'confidence' | 'evidence'>> = {\n  ecommerce: {\n    industry: 'ecommerce',\n    importantScreenPatterns: ['catalog', 'product', 'search', 'filter', 'cart', 'checkout', 'payment', 'order', 'returns'],\n    workflowGoals: ['discover product catalogue', 'map cart and checkout', 'capture order confirmation and failure states', 'map search/filter/sort'],\n    piiRules: ['mask email', 'mask phone', 'mask address', 'mask card-like numbers'],\n    riskyActions: ['place order', 'submit payment', 'save card', 'apply real coupon'],\n    safeProbeRules: ['do not click final payment/order buttons', 'use invalid test card data only', 'stop before irreversible purchase'],\n    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,\n  },\n  banking: {\n    industry: 'banking',\n    importantScreenPatterns: ['login', 'mfa', 'dashboard', 'accounts', 'transactions', 'transfer', 'beneficiary', 'statement', 'dispute'],\n    workflowGoals: ['map account overview', 'map transactions and statements', 'map transfer setup without submitting', 'map MFA/session timeout states'],\n    piiRules: ['mask account numbers', 'mask balances', 'mask names', 'mask addresses', 'mask tax identifiers'],\n    riskyActions: ['transfer money', 'add beneficiary', 'submit dispute', 'download sensitive statement'],\n    safeProbeRules: ['read-only by default', 'never submit transfer confirmation', 'never alter beneficiary data'],\n    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,\n  },\n  fintech: {\n    industry: 'fintech',\n    importantScreenPatterns: ['kyc', 'wallet', 'transactions', 'cards', 'payments', 'payouts', 'risk', 'compliance', 'disputes'],\n    workflowGoals: ['map KYC onboarding', 'map wallet/payment states', 'map failed payment and dispute flows', 'capture compliance screens'],\n    piiRules: ['mask card numbers', 'mask bank accounts', 'mask identity documents', 'mask balances'],\n    riskyActions: ['send payment', 'withdraw funds', 'change card status', 'submit KYC with real data'],\n    safeProbeRules: ['sandbox data only', 'stop before money movement', 'never upload real identity documents'],\n    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,\n  },\n  healthcare: {\n    industry: 'healthcare',\n    importantScreenPatterns: ['patient', 'appointment', 'claims', 'provider', 'prescription', 'records', 'lab', 'consent'],\n    workflowGoals: ['map patient lookup safely', 'map appointment booking before final submit', 'map claims and prescription screens', 'capture consent and records UX'],\n    piiRules: ['mask PHI', 'mask patient names', 'mask MRN', 'mask DOB', 'mask diagnosis', 'mask prescriptions'],\n    riskyActions: ['schedule real appointment', 'submit claim', 'change medication', 'message provider with PHI'],\n    safeProbeRules: ['use synthetic patients only', 'do not submit clinical changes', 'avoid external notifications'],\n    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,\n  },\n  pharma: {\n    industry: 'pharma',\n    importantScreenPatterns: ['product', 'dosage', 'safety', 'batch', 'adverse-event', 'regulatory', 'trial', 'label'],\n    workflowGoals: ['map product and safety content', 'map adverse event intake before submit', 'capture regulatory and batch workflows'],\n    piiRules: ['mask patient reports', 'mask reporter contact', 'mask trial participant identifiers'],\n    riskyActions: ['submit adverse event', 'change safety record', 'publish regulatory content'],\n    safeProbeRules: ['stop before regulatory or safety submission', 'use synthetic reports only'],\n    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,\n  },\n  agriculture: {\n    industry: 'agriculture',\n    importantScreenPatterns: ['crop', 'field', 'weather', 'inventory', 'marketplace', 'logistics', 'sensor', 'subsidy'],\n    workflowGoals: ['map crop and field planning', 'map inventory/marketplace flows', 'capture logistics and sensor dashboards'],\n    piiRules: ['mask farmer identity', 'mask location precision', 'mask subsidy identifiers'],\n    riskyActions: ['place market order', 'submit subsidy application', 'change sensor/device control'],\n    safeProbeRules: ['avoid irreversible orders', 'avoid device actuation', 'use test farm data'],\n    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,\n  },\n  'legacy-enterprise': {\n    industry: 'legacy-enterprise',\n    importantScreenPatterns: ['frameset', 'iframe', 'popup', 'table', 'search', 'admin', 'report', 'export'],\n    workflowGoals: ['map menu trees', 'capture iframe and popup states', 'map table filters and reports', 'detect session timeout'],\n    piiRules: ['mask table cells that look like PII', 'mask exported identifiers'],\n    riskyActions: ['bulk update', 'delete record', 'export production data', 'submit admin change'],\n    safeProbeRules: ['read-only table probing', 'never click delete/bulk actions', 'capture popups without committing'],\n    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,\n  },\n  saas: {\n    industry: 'saas',\n    importantScreenPatterns: ['dashboard', 'settings', 'users', 'billing', 'integrations', 'reports', 'admin', 'workspace'],\n    workflowGoals: ['map workspace navigation', 'map settings and integrations', 'map role/user management without changing state', 'capture dashboards and reports'],\n    piiRules: ['mask emails', 'mask tokens', 'mask API keys', 'mask billing data'],\n    riskyActions: ['delete workspace', 'invite real user', 'rotate key', 'change billing plan'],\n    safeProbeRules: ['never submit destructive settings', 'redact secrets immediately', 'use sandbox users'],\n    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,\n  },\n  unknown: {\n    industry: 'unknown',\n    importantScreenPatterns: ['login', 'dashboard', 'settings', 'search', 'detail', 'report', 'form'],\n    workflowGoals: ['map top-level navigation', 'capture forms and validation states', 'discover hidden menus', 'classify domain after first pages'],\n    piiRules: ['mask emails', 'mask phone numbers', 'mask long identifiers', 'mask card-like numbers'],\n    riskyActions: ['delete', 'submit payment', 'transfer', 'publish', 'send notification'],\n    safeProbeRules: ['read-only by default', 'stop before irreversible actions', 'use synthetic data only'],\n    requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,\n  },\n};\n\nconst BASE_INDUSTRY_HINTS: Array<{ industry: Industry; terms: RegExp[] }> = [\n  { industry: 'ecommerce', terms: [/cart/i, /checkout/i, /product/i, /catalog/i, /sku/i, /wishlist/i, /order/i] },\n  { industry: 'banking', terms: [/account/i, /statement/i, /beneficiar/i, /transfer/i, /iban/i, /routing/i, /balance/i] },\n  { industry: 'fintech', terms: [/wallet/i, /kyc/i, /payout/i, /card/i, /payment/i, /risk/i, /dispute/i] },\n  { industry: 'healthcare', terms: [/patient/i, /provider/i, /appointment/i, /claim/i, /prescription/i, /medical/i, /lab result/i] },\n  { industry: 'pharma', terms: [/pharma/i, /dosage/i, /adverse/i, /batch/i, /regulatory/i, /clinical trial/i, /label/i] },\n  { industry: 'agriculture', terms: [/crop/i, /field/i, /farm/i, /harvest/i, /sensor/i, /weather/i, /subsidy/i] },\n  { industry: 'legacy-enterprise', terms: [/frameset/i, /iframe/i, /popup/i, /legacy/i, /report export/i, /mainframe/i] },\n  { industry: 'saas', terms: [/workspace/i, /tenant/i, /integration/i, /billing/i, /api key/i, /admin/i, /dashboard/i] },\n];\n\n// Allow additional industry hint keywords via ENV without replacing the baseline.\n// Format: JSON array of { industry: string; terms: string[] } \u2014 terms are treated as case-insensitive regex.\n// Example: EXTRA_INDUSTRY_HINTS_JSON='[{\"industry\":\"saas\",\"terms\":[\"platform\",\"multitenant\"]}]'\nconst _extraHints: Array<{ industry: Industry; terms: RegExp[] }> = (() => {\n  const raw = process.env.EXTRA_INDUSTRY_HINTS_JSON;\n  if (!raw) return [];\n  try {\n    const parsed: Array<{ industry: string; terms: string[] }> = JSON.parse(raw);\n    return parsed.map(h => ({ industry: h.industry as Industry, terms: h.terms.map(t => new RegExp(t, 'i')) }));\n  } catch { return []; }\n})();\n\nconst INDUSTRY_HINTS = [...BASE_INDUSTRY_HINTS, ..._extraHints];\n\n// Allow patching profile fields (riskyActions, piiRules, etc.) per industry via ENV.\n// Format: JSON object keyed by industry, value is partial DomainProfile fields to merge.\n// Example: DOMAIN_PROFILE_OVERRIDES_JSON='{\"saas\":{\"riskyActions\":[\"delete workspace\",\"export all data\"]}}'\nconst _profileOverrides: Partial<Record<Industry, Partial<Omit<DomainProfile, 'confidence' | 'evidence'>>>> = (() => {\n  const raw = process.env.DOMAIN_PROFILE_OVERRIDES_JSON;\n  if (!raw) return {};\n  try { return JSON.parse(raw); } catch { return {}; }\n})();\n\nexport function classifyDomainProfile(input: {\n  appUrl: string;\n  projectName?: string | null;\n  appType?: string | null;\n  seedText?: string;\n}): DomainProfile {\n  const haystack = [input.appUrl, input.projectName ?? '', input.appType ?? '', input.seedText ?? ''].join(' ');\n  const scores = new Map<Industry, { score: number; evidence: string[] }>();\n  for (const { industry, terms } of INDUSTRY_HINTS) {\n    for (const term of terms) {\n      if (term.test(haystack)) {\n        const prev = scores.get(industry) ?? { score: 0, evidence: [] };\n        prev.score += 1;\n        prev.evidence.push(term.source);\n        scores.set(industry, prev);\n      }\n    }\n  }\n  let selected: Industry = 'unknown';\n  let best = 0;\n  let evidence: string[] = [];\n  for (const [industry, result] of scores) {\n    if (result.score > best) {\n      selected = industry;\n      best = result.score;\n      evidence = result.evidence;\n    }\n  }\n  const base = PROFILES[selected];\n  return {\n    ...base,\n    confidence: best === 0 ? 0.25 : Math.min(0.95, 0.35 + best * 0.15),\n    evidence,\n  };\n}\n\nexport function getProfile(industry: Industry): DomainProfile {\n  const base = PROFILES[industry] ?? PROFILES.unknown;\n  return { ...base, confidence: 1, evidence: ['explicit'] };\n}\n"],
+  "mappings": "AAuBA,MAAM,4BAA4B;AAAA,EAChC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEA,MAAM,WAA6E;AAAA,EACjF,WAAW;AAAA,IACT,UAAU;AAAA,IACV,yBAAyB,CAAC,WAAW,WAAW,UAAU,UAAU,QAAQ,YAAY,WAAW,SAAS,SAAS;AAAA,IACrH,eAAe,CAAC,8BAA8B,yBAAyB,iDAAiD,wBAAwB;AAAA,IAChJ,UAAU,CAAC,cAAc,cAAc,gBAAgB,wBAAwB;AAAA,IAC/E,cAAc,CAAC,eAAe,kBAAkB,aAAa,mBAAmB;AAAA,IAChF,gBAAgB,CAAC,4CAA4C,mCAAmC,mCAAmC;AAAA,IACnI,mBAAmB;AAAA,EACrB;AAAA,EACA,SAAS;AAAA,IACP,UAAU;AAAA,IACV,yBAAyB,CAAC,SAAS,OAAO,aAAa,YAAY,gBAAgB,YAAY,eAAe,aAAa,SAAS;AAAA,IACpI,eAAe,CAAC,wBAAwB,mCAAmC,yCAAyC,gCAAgC;AAAA,IACpJ,UAAU,CAAC,wBAAwB,iBAAiB,cAAc,kBAAkB,sBAAsB;AAAA,IAC1G,cAAc,CAAC,kBAAkB,mBAAmB,kBAAkB,8BAA8B;AAAA,IACpG,gBAAgB,CAAC,wBAAwB,sCAAsC,8BAA8B;AAAA,IAC7G,mBAAmB;AAAA,EACrB;AAAA,EACA,SAAS;AAAA,IACP,UAAU;AAAA,IACV,yBAAyB,CAAC,OAAO,UAAU,gBAAgB,SAAS,YAAY,WAAW,QAAQ,cAAc,UAAU;AAAA,IAC3H,eAAe,CAAC,sBAAsB,6BAA6B,wCAAwC,4BAA4B;AAAA,IACvI,UAAU,CAAC,qBAAqB,sBAAsB,2BAA2B,eAAe;AAAA,IAChG,cAAc,CAAC,gBAAgB,kBAAkB,sBAAsB,2BAA2B;AAAA,IAClG,gBAAgB,CAAC,qBAAqB,8BAA8B,sCAAsC;AAAA,IAC1G,mBAAmB;AAAA,EACrB;AAAA,EACA,YAAY;AAAA,IACV,UAAU;AAAA,IACV,yBAAyB,CAAC,WAAW,eAAe,UAAU,YAAY,gBAAgB,WAAW,OAAO,SAAS;AAAA,IACrH,eAAe,CAAC,6BAA6B,+CAA+C,uCAAuC,gCAAgC;AAAA,IACnK,UAAU,CAAC,YAAY,sBAAsB,YAAY,YAAY,kBAAkB,oBAAoB;AAAA,IAC3G,cAAc,CAAC,6BAA6B,gBAAgB,qBAAqB,2BAA2B;AAAA,IAC5G,gBAAgB,CAAC,+BAA+B,kCAAkC,8BAA8B;AAAA,IAChH,mBAAmB;AAAA,EACrB;AAAA,EACA,QAAQ;AAAA,IACN,UAAU;AAAA,IACV,yBAAyB,CAAC,WAAW,UAAU,UAAU,SAAS,iBAAiB,cAAc,SAAS,OAAO;AAAA,IACjH,eAAe,CAAC,kCAAkC,0CAA0C,wCAAwC;AAAA,IACpI,UAAU,CAAC,wBAAwB,yBAAyB,oCAAoC;AAAA,IAChG,cAAc,CAAC,wBAAwB,wBAAwB,4BAA4B;AAAA,IAC3F,gBAAgB,CAAC,+CAA+C,4BAA4B;AAAA,IAC5F,mBAAmB;AAAA,EACrB;AAAA,EACA,aAAa;AAAA,IACX,UAAU;AAAA,IACV,yBAAyB,CAAC,QAAQ,SAAS,WAAW,aAAa,eAAe,aAAa,UAAU,SAAS;AAAA,IAClH,eAAe,CAAC,+BAA+B,mCAAmC,yCAAyC;AAAA,IAC3H,UAAU,CAAC,wBAAwB,2BAA2B,0BAA0B;AAAA,IACxF,cAAc,CAAC,sBAAsB,8BAA8B,8BAA8B;AAAA,IACjG,gBAAgB,CAAC,6BAA6B,0BAA0B,oBAAoB;AAAA,IAC5F,mBAAmB;AAAA,EACrB;AAAA,EACA,qBAAqB;AAAA,IACnB,UAAU;AAAA,IACV,yBAAyB,CAAC,YAAY,UAAU,SAAS,SAAS,UAAU,SAAS,UAAU,QAAQ;AAAA,IACvG,eAAe,CAAC,kBAAkB,mCAAmC,iCAAiC,wBAAwB;AAAA,IAC9H,UAAU,CAAC,uCAAuC,2BAA2B;AAAA,IAC7E,cAAc,CAAC,eAAe,iBAAiB,0BAA0B,qBAAqB;AAAA,IAC9F,gBAAgB,CAAC,2BAA2B,mCAAmC,mCAAmC;AAAA,IAClH,mBAAmB;AAAA,EACrB;AAAA,EACA,MAAM;AAAA,IACJ,UAAU;AAAA,IACV,yBAAyB,CAAC,aAAa,YAAY,SAAS,WAAW,gBAAgB,WAAW,SAAS,WAAW;AAAA,IACtH,eAAe,CAAC,4BAA4B,iCAAiC,mDAAmD,gCAAgC;AAAA,IAChK,UAAU,CAAC,eAAe,eAAe,iBAAiB,mBAAmB;AAAA,IAC7E,cAAc,CAAC,oBAAoB,oBAAoB,cAAc,qBAAqB;AAAA,IAC1F,gBAAgB,CAAC,qCAAqC,8BAA8B,mBAAmB;AAAA,IACvG,mBAAmB;AAAA,EACrB;AAAA,EACA,SAAS;AAAA,IACP,UAAU;AAAA,IACV,yBAAyB,CAAC,SAAS,aAAa,YAAY,UAAU,UAAU,UAAU,MAAM;AAAA,IAChG,eAAe,CAAC,4BAA4B,uCAAuC,yBAAyB,mCAAmC;AAAA,IAC/I,UAAU,CAAC,eAAe,sBAAsB,yBAAyB,wBAAwB;AAAA,IACjG,cAAc,CAAC,UAAU,kBAAkB,YAAY,WAAW,mBAAmB;AAAA,IACrF,gBAAgB,CAAC,wBAAwB,oCAAoC,yBAAyB;AAAA,IACtG,mBAAmB;AAAA,EACrB;AACF;AAEA,MAAM,sBAAsE;AAAA,EAC1E,EAAE,UAAU,aAAa,OAAO,CAAC,SAAS,aAAa,YAAY,YAAY,QAAQ,aAAa,QAAQ,EAAE;AAAA,EAC9G,EAAE,UAAU,WAAW,OAAO,CAAC,YAAY,cAAc,eAAe,aAAa,SAAS,YAAY,UAAU,EAAE;AAAA,EACtH,EAAE,UAAU,WAAW,OAAO,CAAC,WAAW,QAAQ,WAAW,SAAS,YAAY,SAAS,UAAU,EAAE;AAAA,EACvG,EAAE,UAAU,cAAc,OAAO,CAAC,YAAY,aAAa,gBAAgB,UAAU,iBAAiB,YAAY,aAAa,EAAE;AAAA,EACjI,EAAE,UAAU,UAAU,OAAO,CAAC,WAAW,WAAW,YAAY,UAAU,eAAe,mBAAmB,QAAQ,EAAE;AAAA,EACtH,EAAE,UAAU,eAAe,OAAO,CAAC,SAAS,UAAU,SAAS,YAAY,WAAW,YAAY,UAAU,EAAE;AAAA,EAC9G,EAAE,UAAU,qBAAqB,OAAO,CAAC,aAAa,WAAW,UAAU,WAAW,kBAAkB,YAAY,EAAE;AAAA,EACtH,EAAE,UAAU,QAAQ,OAAO,CAAC,cAAc,WAAW,gBAAgB,YAAY,YAAY,UAAU,YAAY,EAAE;AACvH;AAKA,MAAM,eAA+D,MAAM;AACzE,QAAM,MAAM,QAAQ,IAAI;AACxB,MAAI,CAAC,IAAK,QAAO,CAAC;AAClB,MAAI;AACF,UAAM,SAAuD,KAAK,MAAM,GAAG;AAC3E,WAAO,OAAO,IAAI,QAAM,EAAE,UAAU,EAAE,UAAsB,OAAO,EAAE,MAAM,IAAI,OAAK,IAAI,OAAO,GAAG,GAAG,CAAC,EAAE,EAAE;AAAA,EAC5G,QAAQ;AAAE,WAAO,CAAC;AAAA,EAAG;AACvB,GAAG;AAEH,MAAM,iBAAiB,CAAC,GAAG,qBAAqB,GAAG,WAAW;AAK9D,MAAM,qBAAyG,MAAM;AACnH,QAAM,MAAM,QAAQ,IAAI;AACxB,MAAI,CAAC,IAAK,QAAO,CAAC;AAClB,MAAI;AAAE,WAAO,KAAK,MAAM,GAAG;AAAA,EAAG,QAAQ;AAAE,WAAO,CAAC;AAAA,EAAG;AACrD,GAAG;AAEI,SAAS,sBAAsB,OAKpB;AAChB,QAAM,WAAW,CAAC,MAAM,QAAQ,MAAM,eAAe,IAAI,MAAM,WAAW,IAAI,MAAM,YAAY,EAAE,EAAE,KAAK,GAAG;AAC5G,QAAM,SAAS,oBAAI,IAAqD;AACxE,aAAW,EAAE,UAAU,MAAM,KAAK,gBAAgB;AAChD,eAAW,QAAQ,OAAO;AACxB,UAAI,KAAK,KAAK,QAAQ,GAAG;AACvB,cAAM,OAAO,OAAO,IAAI,QAAQ,KAAK,EAAE,OAAO,GAAG,UAAU,CAAC,EAAE;AAC9D,aAAK,SAAS;AACd,aAAK,SAAS,KAAK,KAAK,MAAM;AAC9B,eAAO,IAAI,UAAU,IAAI;AAAA,MAC3B;AAAA,IACF;AAAA,EACF;AACA,MAAI,WAAqB;AACzB,MAAI,OAAO;AACX,MAAI,WAAqB,CAAC;AAC1B,aAAW,CAAC,UAAU,MAAM,KAAK,QAAQ;AACvC,QAAI,OAAO,QAAQ,MAAM;AACvB,iBAAW;AACX,aAAO,OAAO;AACd,iBAAW,OAAO;AAAA,IACpB;AAAA,EACF;AACA,QAAM,OAAO,SAAS,QAAQ;AAC9B,SAAO;AAAA,IACL,GAAG;AAAA,IACH,YAAY,SAAS,IAAI,OAAO,KAAK,IAAI,MAAM,OAAO,OAAO,IAAI;AAAA,IACjE;AAAA,EACF;AACF;AAEO,SAAS,WAAW,UAAmC;AAC5D,QAAM,OAAO,SAAS,QAAQ,KAAK,SAAS;AAC5C,SAAO,EAAE,GAAG,MAAM,YAAY,GAAG,UAAU,CAAC,UAAU,EAAE;AAC1D;",
+  "names": []
+}

package/dist/proxy.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Free proxy auto-selection for bot-protection bypass.
+ *
+ * Source: proxifly/free-proxy-list (MIT) via jsDelivr CDN.
+ * These are datacenter proxies — they bypass simple IP blocks but NOT
+ * full Cloudflare Bot Manager / Akamai. For Cloudflare-protected sites,
+ * users need a residential proxy (Bright Data, Oxylabs, etc.).
+ */
+/**
+ * Pick a working free proxy. Tests in parallel batches of 10 with a 4s timeout.
+ * Returns null if no working proxy found (not an error — crawl can proceed without proxy).
+ */
+export declare function pickFreeProxy(): Promise<string | null>;
+export type ProxyTier = 'free' | 'datacenter' | 'residential' | 'premium';
+export interface ProxyTierConfig {
+    /** List of proxy URLs for this tier. E.g. ['http://user:pass@dc1.proxy:8080'] */
+    urls: string[];
+    tier: ProxyTier;
+}
+/**
+ * Tiered proxy manager with per-domain error histograms.
+ *
+ * Tracks success/failure per domain and automatically escalates to higher proxy tiers
+ * on consecutive failures. Applies time-decay to probe returning to lower tiers.
+ *
+ * Usage:
+ *   const mgr = new TieredProxyManager([{ tier: 'datacenter', urls: [...] }]);
+ *   const proxy = await mgr.selectProxy('api.salesforce.com');
+ *   // ... make request ...
+ *   mgr.recordResult('api.salesforce.com', success);
+ */
+export declare class TieredProxyManager {
+    private tiers;
+    private domainHistory;
+    private readonly ESCALATE_AFTER;
+    private readonly DECAY_AFTER_MS;
+    constructor(configs: ProxyTierConfig[]);
+    private getHistory;
+    private lowestAvailableTier;
+    /** Pick a proxy URL for this domain, based on its tier history. */
+    selectProxy(domain: string): string | null;
+    /** Record a request result for a domain. Triggers tier escalation on repeated failures. */
+    recordResult(domain: string, success: boolean): void;
+    stats(): Array<{
+        domain: string;
+        tier: ProxyTier;
+        successRate: number;
+        totalRequests: number;
+    }>;
+}

package/dist/proxy.js

@@ -0,0 +1,165 @@
+import { createConnection } from "net";
+const PROXY_LIST_URL = "https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.json";
+async function fetchCandidates() {
+  const res = await fetch(PROXY_LIST_URL, {
+    signal: AbortSignal.timeout(1e4),
+    headers: { "Accept": "application/json" }
+  });
+  if (!res.ok) throw new Error(`proxy list fetch failed: ${res.status}`);
+  const raw = await res.json();
+  return raw.filter((p) => p.protocols?.includes("http") && p.port > 0 && (p.upTime ?? 0) >= 30).sort((a, b) => (a.speed ?? 99999) - (b.speed ?? 99999)).slice(0, 50);
+}
+function testProxy(ip, port, timeoutMs) {
+  return new Promise((resolve) => {
+    let settled = false;
+    const settle = (ok) => {
+      if (settled) return;
+      settled = true;
+      resolve(ok);
+      try {
+        sock.destroy();
+      } catch {
+      }
+    };
+    const timer = setTimeout(() => settle(false), timeoutMs);
+    const sock = createConnection({ host: ip, port, timeout: timeoutMs });
+    sock.on("connect", () => {
+      sock.write("CONNECT example.com:80 HTTP/1.1\r\nHost: example.com:80\r\n\r\n");
+      sock.once("data", (data) => {
+        clearTimeout(timer);
+        const ok = /^HTTP\/1\.[01] 200/.test(data.toString());
+        settle(ok);
+      });
+    });
+    sock.on("error", () => {
+      clearTimeout(timer);
+      settle(false);
+    });
+    sock.on("timeout", () => {
+      clearTimeout(timer);
+      settle(false);
+    });
+  });
+}
+async function testBatch(candidates, start, batchSize, timeoutMs) {
+  const batch = candidates.slice(start, start + batchSize);
+  if (batch.length === 0) return null;
+  const results = await Promise.all(
+    batch.map(async (p) => {
+      const ok = await testProxy(p.ip, p.port, timeoutMs);
+      return ok ? `http://${p.ip}:${p.port}` : null;
+    })
+  );
+  return results.find((r) => r !== null) ?? null;
+}
+async function pickFreeProxy() {
+  let candidates;
+  try {
+    candidates = await fetchCandidates();
+  } catch (err) {
+    console.warn("[proxy] failed to fetch proxy list:", err?.message ?? err);
+    return null;
+  }
+  const BATCH = 10;
+  const TIMEOUT = 4e3;
+  for (let i = 0; i < candidates.length; i += BATCH) {
+    const found = await testBatch(candidates, i, BATCH, TIMEOUT);
+    if (found) {
+      console.info(`[proxy] selected free proxy: ${found} (tested ${i + BATCH} candidates)`);
+      return found;
+    }
+  }
+  console.warn("[proxy] no working free proxy found in list");
+  return null;
+}
+const TIER_ORDER = ["free", "datacenter", "residential", "premium"];
+function nextTier(current) {
+  const idx = TIER_ORDER.indexOf(current);
+  return TIER_ORDER[Math.min(idx + 1, TIER_ORDER.length - 1)];
+}
+class TieredProxyManager {
+  tiers = /* @__PURE__ */ new Map();
+  domainHistory = /* @__PURE__ */ new Map();
+  ESCALATE_AFTER = 3;
+  // consecutive failures before escalation
+  DECAY_AFTER_MS = 30 * 60 * 1e3;
+  // 30 min: probe tier downgrade
+  constructor(configs) {
+    for (const cfg of configs) {
+      if (cfg.urls.length > 0) this.tiers.set(cfg.tier, cfg.urls);
+    }
+  }
+  getHistory(domain) {
+    if (!this.domainHistory.has(domain)) {
+      this.domainHistory.set(domain, {
+        tier: this.lowestAvailableTier(),
+        consecutiveFailures: 0,
+        lastEscalatedAt: 0,
+        totalRequests: 0,
+        successRate: 1
+      });
+    }
+    return this.domainHistory.get(domain);
+  }
+  lowestAvailableTier() {
+    for (const t of TIER_ORDER) {
+      if (this.tiers.has(t)) return t;
+    }
+    return "free";
+  }
+  /** Pick a proxy URL for this domain, based on its tier history. */
+  selectProxy(domain) {
+    const history = this.getHistory(domain);
+    if (history.consecutiveFailures === 0 && history.lastEscalatedAt > 0) {
+      const elapsed = Date.now() - history.lastEscalatedAt;
+      if (elapsed > this.DECAY_AFTER_MS) {
+        const currentIdx = TIER_ORDER.indexOf(history.tier);
+        if (currentIdx > 0) {
+          const lowerTier = TIER_ORDER[currentIdx - 1];
+          if (this.tiers.has(lowerTier)) {
+            history.tier = lowerTier;
+            history.lastEscalatedAt = 0;
+          }
+        }
+      }
+    }
+    const urls = this.tiers.get(history.tier);
+    if (!urls || urls.length === 0) return null;
+    const idx = history.totalRequests % urls.length;
+    return urls[idx];
+  }
+  /** Record a request result for a domain. Triggers tier escalation on repeated failures. */
+  recordResult(domain, success) {
+    const history = this.getHistory(domain);
+    history.totalRequests++;
+    if (success) {
+      history.consecutiveFailures = 0;
+      history.successRate = history.successRate * 0.9 + 0.1;
+    } else {
+      history.consecutiveFailures++;
+      history.successRate = history.successRate * 0.9;
+      if (history.consecutiveFailures >= this.ESCALATE_AFTER) {
+        const elevated = nextTier(history.tier);
+        if (elevated !== history.tier && this.tiers.has(elevated)) {
+          console.warn(`[proxy-tier] domain=${domain} escalating ${history.tier}\u2192${elevated} after ${history.consecutiveFailures} failures`);
+          history.tier = elevated;
+          history.lastEscalatedAt = Date.now();
+          history.consecutiveFailures = 0;
+        }
+      }
+    }
+  }
+  stats() {
+    return Array.from(this.domainHistory.entries()).map(([domain, h]) => ({
+      domain,
+      tier: h.tier,
+      successRate: Math.round(h.successRate * 100) / 100,
+      totalRequests: h.totalRequests
+    }));
+  }
+}
+export {
+  TieredProxyManager,
+  pickFreeProxy
+};
+//# sourceMappingURL=proxy.js.map

package/dist/proxy.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/proxy.ts"],
+  "sourcesContent": ["/**\n * Free proxy auto-selection for bot-protection bypass.\n *\n * Source: proxifly/free-proxy-list (MIT) via jsDelivr CDN.\n * These are datacenter proxies \u2014 they bypass simple IP blocks but NOT\n * full Cloudflare Bot Manager / Akamai. For Cloudflare-protected sites,\n * users need a residential proxy (Bright Data, Oxylabs, etc.).\n */\n\nimport { createConnection } from 'net';\n\nconst PROXY_LIST_URL =\n  'https://cdn.jsdelivr.net/gh/proxifly/free-proxy-list@main/proxies/all/data.json';\n\ninterface ProxyEntry {\n  ip: string;\n  port: number;\n  protocols: string[];\n  upTime?: number;\n  speed?: number;\n}\n\n/** Fetch and filter proxy list. Returns http proxies sorted by speed (fastest first). */\nasync function fetchCandidates(): Promise<ProxyEntry[]> {\n  const res = await fetch(PROXY_LIST_URL, {\n    signal: AbortSignal.timeout(10_000),\n    headers: { 'Accept': 'application/json' },\n  });\n  if (!res.ok) throw new Error(`proxy list fetch failed: ${res.status}`);\n  const raw: ProxyEntry[] = await res.json();\n  return raw\n    .filter((p) => p.protocols?.includes('http') && p.port > 0 && (p.upTime ?? 0) >= 30)\n    .sort((a, b) => (a.speed ?? 99999) - (b.speed ?? 99999))\n    .slice(0, 50); // test at most 50 candidates\n}\n\n/** TCP-level proxy liveness check via HTTP CONNECT tunnel. No extra dependencies. */\nfunction testProxy(ip: string, port: number, timeoutMs: number): Promise<boolean> {\n  return new Promise((resolve) => {\n    let settled = false;\n    const settle = (ok: boolean) => {\n      if (settled) return;\n      settled = true;\n      resolve(ok);\n      try { sock.destroy(); } catch { /* ignore */ }\n    };\n\n    const timer = setTimeout(() => settle(false), timeoutMs);\n    const sock = createConnection({ host: ip, port, timeout: timeoutMs });\n\n    sock.on('connect', () => {\n      sock.write('CONNECT example.com:80 HTTP/1.1\\r\\nHost: example.com:80\\r\\n\\r\\n');\n      sock.once('data', (data) => {\n        clearTimeout(timer);\n        const ok = /^HTTP\\/1\\.[01] 200/.test(data.toString());\n        settle(ok);\n      });\n    });\n    sock.on('error', () => { clearTimeout(timer); settle(false); });\n    sock.on('timeout', () => { clearTimeout(timer); settle(false); });\n  });\n}\n\n/** Test `batchSize` proxies concurrently and return first working one. */\nasync function testBatch(\n  candidates: ProxyEntry[],\n  start: number,\n  batchSize: number,\n  timeoutMs: number,\n): Promise<string | null> {\n  const batch = candidates.slice(start, start + batchSize);\n  if (batch.length === 0) return null;\n\n  const results = await Promise.all(\n    batch.map(async (p) => {\n      const ok = await testProxy(p.ip, p.port, timeoutMs);\n      return ok ? `http://${p.ip}:${p.port}` : null;\n    }),\n  );\n\n  return results.find((r) => r !== null) ?? null;\n}\n\n/**\n * Pick a working free proxy. Tests in parallel batches of 10 with a 4s timeout.\n * Returns null if no working proxy found (not an error \u2014 crawl can proceed without proxy).\n */\nexport async function pickFreeProxy(): Promise<string | null> {\n  let candidates: ProxyEntry[];\n  try {\n    candidates = await fetchCandidates();\n  } catch (err: any) {\n    console.warn('[proxy] failed to fetch proxy list:', err?.message ?? err);\n    return null;\n  }\n\n  const BATCH = 10;\n  const TIMEOUT = 4_000;\n\n  for (let i = 0; i < candidates.length; i += BATCH) {\n    const found = await testBatch(candidates, i, BATCH, TIMEOUT);\n    if (found) {\n      console.info(`[proxy] selected free proxy: ${found} (tested ${i + BATCH} candidates)`);\n      return found;\n    }\n  }\n\n  console.warn('[proxy] no working free proxy found in list');\n  return null;\n}\n\n// \u2500\u2500 Tiered Proxy Rotation \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n\nexport type ProxyTier = 'free' | 'datacenter' | 'residential' | 'premium';\n\nexport interface ProxyTierConfig {\n  /** List of proxy URLs for this tier. E.g. ['http://user:pass@dc1.proxy:8080'] */\n  urls: string[];\n  tier: ProxyTier;\n}\n\ninterface DomainHistory {\n  tier: ProxyTier;\n  consecutiveFailures: number;\n  lastEscalatedAt: number;\n  totalRequests: number;\n  successRate: number;\n}\n\nconst TIER_ORDER: ProxyTier[] = ['free', 'datacenter', 'residential', 'premium'];\n\nfunction nextTier(current: ProxyTier): ProxyTier {\n  const idx = TIER_ORDER.indexOf(current);\n  return TIER_ORDER[Math.min(idx + 1, TIER_ORDER.length - 1)];\n}\n\n/**\n * Tiered proxy manager with per-domain error histograms.\n *\n * Tracks success/failure per domain and automatically escalates to higher proxy tiers\n * on consecutive failures. Applies time-decay to probe returning to lower tiers.\n *\n * Usage:\n *   const mgr = new TieredProxyManager([{ tier: 'datacenter', urls: [...] }]);\n *   const proxy = await mgr.selectProxy('api.salesforce.com');\n *   // ... make request ...\n *   mgr.recordResult('api.salesforce.com', success);\n */\nexport class TieredProxyManager {\n  private tiers: Map<ProxyTier, string[]> = new Map();\n  private domainHistory: Map<string, DomainHistory> = new Map();\n  private readonly ESCALATE_AFTER = 3; // consecutive failures before escalation\n  private readonly DECAY_AFTER_MS = 30 * 60 * 1000; // 30 min: probe tier downgrade\n\n  constructor(configs: ProxyTierConfig[]) {\n    for (const cfg of configs) {\n      if (cfg.urls.length > 0) this.tiers.set(cfg.tier, cfg.urls);\n    }\n  }\n\n  private getHistory(domain: string): DomainHistory {\n    if (!this.domainHistory.has(domain)) {\n      this.domainHistory.set(domain, {\n        tier: this.lowestAvailableTier(),\n        consecutiveFailures: 0,\n        lastEscalatedAt: 0,\n        totalRequests: 0,\n        successRate: 1,\n      });\n    }\n    return this.domainHistory.get(domain)!;\n  }\n\n  private lowestAvailableTier(): ProxyTier {\n    for (const t of TIER_ORDER) {\n      if (this.tiers.has(t)) return t;\n    }\n    return 'free';\n  }\n\n  /** Pick a proxy URL for this domain, based on its tier history. */\n  selectProxy(domain: string): string | null {\n    const history = this.getHistory(domain);\n\n    // Time-decay: if we've been on elevated tier for DECAY_AFTER_MS, try stepping down\n    if (history.consecutiveFailures === 0 && history.lastEscalatedAt > 0) {\n      const elapsed = Date.now() - history.lastEscalatedAt;\n      if (elapsed > this.DECAY_AFTER_MS) {\n        const currentIdx = TIER_ORDER.indexOf(history.tier);\n        if (currentIdx > 0) {\n          const lowerTier = TIER_ORDER[currentIdx - 1];\n          if (this.tiers.has(lowerTier)) {\n            history.tier = lowerTier;\n            history.lastEscalatedAt = 0;\n          }\n        }\n      }\n    }\n\n    const urls = this.tiers.get(history.tier);\n    if (!urls || urls.length === 0) return null;\n    // Round-robin within tier\n    const idx = history.totalRequests % urls.length;\n    return urls[idx];\n  }\n\n  /** Record a request result for a domain. Triggers tier escalation on repeated failures. */\n  recordResult(domain: string, success: boolean): void {\n    const history = this.getHistory(domain);\n    history.totalRequests++;\n    if (success) {\n      history.consecutiveFailures = 0;\n      // Update EMA success rate\n      history.successRate = history.successRate * 0.9 + 0.1;\n    } else {\n      history.consecutiveFailures++;\n      history.successRate = history.successRate * 0.9;\n      if (history.consecutiveFailures >= this.ESCALATE_AFTER) {\n        const elevated = nextTier(history.tier);\n        if (elevated !== history.tier && this.tiers.has(elevated)) {\n          console.warn(`[proxy-tier] domain=${domain} escalating ${history.tier}\u2192${elevated} after ${history.consecutiveFailures} failures`);\n          history.tier = elevated;\n          history.lastEscalatedAt = Date.now();\n          history.consecutiveFailures = 0;\n        }\n      }\n    }\n  }\n\n  stats(): Array<{ domain: string; tier: ProxyTier; successRate: number; totalRequests: number }> {\n    return Array.from(this.domainHistory.entries()).map(([domain, h]) => ({\n      domain,\n      tier: h.tier,\n      successRate: Math.round(h.successRate * 100) / 100,\n      totalRequests: h.totalRequests,\n    }));\n  }\n}\n"],
+  "mappings": "AASA,SAAS,wBAAwB;AAEjC,MAAM,iBACJ;AAWF,eAAe,kBAAyC;AACtD,QAAM,MAAM,MAAM,MAAM,gBAAgB;AAAA,IACtC,QAAQ,YAAY,QAAQ,GAAM;AAAA,IAClC,SAAS,EAAE,UAAU,mBAAmB;AAAA,EAC1C,CAAC;AACD,MAAI,CAAC,IAAI,GAAI,OAAM,IAAI,MAAM,4BAA4B,IAAI,MAAM,EAAE;AACrE,QAAM,MAAoB,MAAM,IAAI,KAAK;AACzC,SAAO,IACJ,OAAO,CAAC,MAAM,EAAE,WAAW,SAAS,MAAM,KAAK,EAAE,OAAO,MAAM,EAAE,UAAU,MAAM,EAAE,EAClF,KAAK,CAAC,GAAG,OAAO,EAAE,SAAS,UAAU,EAAE,SAAS,MAAM,EACtD,MAAM,GAAG,EAAE;AAChB;AAGA,SAAS,UAAU,IAAY,MAAc,WAAqC;AAChF,SAAO,IAAI,QAAQ,CAAC,YAAY;AAC9B,QAAI,UAAU;AACd,UAAM,SAAS,CAAC,OAAgB;AAC9B,UAAI,QAAS;AACb,gBAAU;AACV,cAAQ,EAAE;AACV,UAAI;AAAE,aAAK,QAAQ;AAAA,MAAG,QAAQ;AAAA,MAAe;AAAA,IAC/C;AAEA,UAAM,QAAQ,WAAW,MAAM,OAAO,KAAK,GAAG,SAAS;AACvD,UAAM,OAAO,iBAAiB,EAAE,MAAM,IAAI,MAAM,SAAS,UAAU,CAAC;AAEpE,SAAK,GAAG,WAAW,MAAM;AACvB,WAAK,MAAM,iEAAiE;AAC5E,WAAK,KAAK,QAAQ,CAAC,SAAS;AAC1B,qBAAa,KAAK;AAClB,cAAM,KAAK,qBAAqB,KAAK,KAAK,SAAS,CAAC;AACpD,eAAO,EAAE;AAAA,MACX,CAAC;AAAA,IACH,CAAC;AACD,SAAK,GAAG,SAAS,MAAM;AAAE,mBAAa,KAAK;AAAG,aAAO,KAAK;AAAA,IAAG,CAAC;AAC9D,SAAK,GAAG,WAAW,MAAM;AAAE,mBAAa,KAAK;AAAG,aAAO,KAAK;AAAA,IAAG,CAAC;AAAA,EAClE,CAAC;AACH;AAGA,eAAe,UACb,YACA,OACA,WACA,WACwB;AACxB,QAAM,QAAQ,WAAW,MAAM,OAAO,QAAQ,SAAS;AACvD,MAAI,MAAM,WAAW,EAAG,QAAO;AAE/B,QAAM,UAAU,MAAM,QAAQ;AAAA,IAC5B,MAAM,IAAI,OAAO,MAAM;AACrB,YAAM,KAAK,MAAM,UAAU,EAAE,IAAI,EAAE,MAAM,SAAS;AAClD,aAAO,KAAK,UAAU,EAAE,EAAE,IAAI,EAAE,IAAI,KAAK;AAAA,IAC3C,CAAC;AAAA,EACH;AAEA,SAAO,QAAQ,KAAK,CAAC,MAAM,MAAM,IAAI,KAAK;AAC5C;AAMA,eAAsB,gBAAwC;AAC5D,MAAI;AACJ,MAAI;AACF,iBAAa,MAAM,gBAAgB;AAAA,EACrC,SAAS,KAAU;AACjB,YAAQ,KAAK,uCAAuC,KAAK,WAAW,GAAG;AACvE,WAAO;AAAA,EACT;AAEA,QAAM,QAAQ;AACd,QAAM,UAAU;AAEhB,WAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK,OAAO;AACjD,UAAM,QAAQ,MAAM,UAAU,YAAY,GAAG,OAAO,OAAO;AAC3D,QAAI,OAAO;AACT,cAAQ,KAAK,gCAAgC,KAAK,YAAY,IAAI,KAAK,cAAc;AACrF,aAAO;AAAA,IACT;AAAA,EACF;AAEA,UAAQ,KAAK,6CAA6C;AAC1D,SAAO;AACT;AAoBA,MAAM,aAA0B,CAAC,QAAQ,cAAc,eAAe,SAAS;AAE/E,SAAS,SAAS,SAA+B;AAC/C,QAAM,MAAM,WAAW,QAAQ,OAAO;AACtC,SAAO,WAAW,KAAK,IAAI,MAAM,GAAG,WAAW,SAAS,CAAC,CAAC;AAC5D;AAcO,MAAM,mBAAmB;AAAA,EACtB,QAAkC,oBAAI,IAAI;AAAA,EAC1C,gBAA4C,oBAAI,IAAI;AAAA,EAC3C,iBAAiB;AAAA;AAAA,EACjB,iBAAiB,KAAK,KAAK;AAAA;AAAA,EAE5C,YAAY,SAA4B;AACtC,eAAW,OAAO,SAAS;AACzB,UAAI,IAAI,KAAK,SAAS,EAAG,MAAK,MAAM,IAAI,IAAI,MAAM,IAAI,IAAI;AAAA,IAC5D;AAAA,EACF;AAAA,EAEQ,WAAW,QAA+B;AAChD,QAAI,CAAC,KAAK,cAAc,IAAI,MAAM,GAAG;AACnC,WAAK,cAAc,IAAI,QAAQ;AAAA,QAC7B,MAAM,KAAK,oBAAoB;AAAA,QAC/B,qBAAqB;AAAA,QACrB,iBAAiB;AAAA,QACjB,eAAe;AAAA,QACf,aAAa;AAAA,MACf,CAAC;AAAA,IACH;AACA,WAAO,KAAK,cAAc,IAAI,MAAM;AAAA,EACtC;AAAA,EAEQ,sBAAiC;AACvC,eAAW,KAAK,YAAY;AAC1B,UAAI,KAAK,MAAM,IAAI,CAAC,EAAG,QAAO;AAAA,IAChC;AACA,WAAO;AAAA,EACT;AAAA;AAAA,EAGA,YAAY,QAA+B;AACzC,UAAM,UAAU,KAAK,WAAW,MAAM;AAGtC,QAAI,QAAQ,wBAAwB,KAAK,QAAQ,kBAAkB,GAAG;AACpE,YAAM,UAAU,KAAK,IAAI,IAAI,QAAQ;AACrC,UAAI,UAAU,KAAK,gBAAgB;AACjC,cAAM,aAAa,WAAW,QAAQ,QAAQ,IAAI;AAClD,YAAI,aAAa,GAAG;AAClB,gBAAM,YAAY,WAAW,aAAa,CAAC;AAC3C,cAAI,KAAK,MAAM,IAAI,SAAS,GAAG;AAC7B,oBAAQ,OAAO;AACf,oBAAQ,kBAAkB;AAAA,UAC5B;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAEA,UAAM,OAAO,KAAK,MAAM,IAAI,QAAQ,IAAI;AACxC,QAAI,CAAC,QAAQ,KAAK,WAAW,EAAG,QAAO;AAEvC,UAAM,MAAM,QAAQ,gBAAgB,KAAK;AACzC,WAAO,KAAK,GAAG;AAAA,EACjB;AAAA;AAAA,EAGA,aAAa,QAAgB,SAAwB;AACnD,UAAM,UAAU,KAAK,WAAW,MAAM;AACtC,YAAQ;AACR,QAAI,SAAS;AACX,cAAQ,sBAAsB;AAE9B,cAAQ,cAAc,QAAQ,cAAc,MAAM;AAAA,IACpD,OAAO;AACL,cAAQ;AACR,cAAQ,cAAc,QAAQ,cAAc;AAC5C,UAAI,QAAQ,uBAAuB,KAAK,gBAAgB;AACtD,cAAM,WAAW,SAAS,QAAQ,IAAI;AACtC,YAAI,aAAa,QAAQ,QAAQ,KAAK,MAAM,IAAI,QAAQ,GAAG;AACzD,kBAAQ,KAAK,uBAAuB,MAAM,eAAe,QAAQ,IAAI,SAAI,QAAQ,UAAU,QAAQ,mBAAmB,WAAW;AACjI,kBAAQ,OAAO;AACf,kBAAQ,kBAAkB,KAAK,IAAI;AACnC,kBAAQ,sBAAsB;AAAA,QAChC;AAAA,MACF;AAAA,IACF;AAAA,EACF;AAAA,EAEA,QAAgG;AAC9F,WAAO,MAAM,KAAK,KAAK,cAAc,QAAQ,CAAC,EAAE,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO;AAAA,MACpE;AAAA,MACA,MAAM,EAAE;AAAA,MACR,aAAa,KAAK,MAAM,EAAE,cAAc,GAAG,IAAI;AAAA,MAC/C,eAAe,EAAE;AAAA,IACnB,EAAE;AAAA,EACJ;AACF;",
+  "names": []
+}

package/dist/rate-limiter.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Domain-aware rate limiter — ensures the crawler respects per-domain rate limits.
+ *
+ * Uses a token bucket algorithm per domain. When the bucket is empty, requests
+ * wait until a token is available. Tokens refill at a configurable rate.
+ *
+ * Default: 2 requests/second per domain, burst of 5.
+ * Can be configured per-domain via the rateMap (domain → requests/second).
+ */
+export interface RateLimiterOpts {
+    defaultRps?: number;
+    burstSize?: number;
+    rateMap?: Record<string, number>;
+}
+export declare class DomainRateLimiter {
+    private buckets;
+    private readonly defaultRps;
+    private readonly burstSize;
+    private readonly rateMap;
+    constructor(opts?: RateLimiterOpts);
+    private getBucket;
+    private refill;
+    /** Wait until a token is available for the given URL's domain. */
+    throttle(url: string): Promise<void>;
+    /** Override rate for a specific domain at runtime (e.g. after seeing 429). */
+    setRate(domain: string, rps: number): void;
+    stats(): Record<string, {
+        tokens: number;
+        rps: number;
+    }>;
+}

package/dist/rate-limiter.js

@@ -0,0 +1,64 @@
+function extractDomain(url) {
+  try {
+    return new URL(url).hostname;
+  } catch {
+    return url;
+  }
+}
+class DomainRateLimiter {
+  buckets = /* @__PURE__ */ new Map();
+  defaultRps;
+  burstSize;
+  rateMap;
+  constructor(opts = {}) {
+    this.defaultRps = opts.defaultRps ?? (parseFloat(process.env.CRAWLER_RATE_LIMIT_DEFAULT_RPS ?? "") || 2);
+    this.burstSize = opts.burstSize ?? (parseInt(process.env.CRAWLER_RATE_LIMIT_BURST_SIZE ?? "") || 5);
+    this.rateMap = opts.rateMap ?? {};
+  }
+  getBucket(domain) {
+    if (!this.buckets.has(domain)) {
+      this.buckets.set(domain, { tokens: this.burstSize, lastRefill: Date.now() });
+    }
+    return this.buckets.get(domain);
+  }
+  refill(domain, bucket) {
+    const rps = this.rateMap[domain] ?? this.defaultRps;
+    const now = Date.now();
+    const elapsed = (now - bucket.lastRefill) / 1e3;
+    bucket.tokens = Math.min(this.burstSize, bucket.tokens + elapsed * rps);
+    bucket.lastRefill = now;
+  }
+  /** Wait until a token is available for the given URL's domain. */
+  async throttle(url) {
+    const domain = extractDomain(url);
+    const bucket = this.getBucket(domain);
+    this.refill(domain, bucket);
+    if (bucket.tokens >= 1) {
+      bucket.tokens -= 1;
+      return;
+    }
+    const rps = this.rateMap[domain] ?? this.defaultRps;
+    const waitMs = Math.ceil((1 - bucket.tokens) / rps * 1e3);
+    await new Promise((r) => setTimeout(r, waitMs));
+    bucket.tokens = 0;
+    bucket.lastRefill = Date.now();
+  }
+  /** Override rate for a specific domain at runtime (e.g. after seeing 429). */
+  setRate(domain, rps) {
+    this.rateMap[domain] = rps;
+    const bucket = this.getBucket(domain);
+    bucket.tokens = 0;
+  }
+  stats() {
+    const result = {};
+    for (const [domain, bucket] of this.buckets) {
+      this.refill(domain, bucket);
+      result[domain] = { tokens: Math.round(bucket.tokens * 100) / 100, rps: this.rateMap[domain] ?? this.defaultRps };
+    }
+    return result;
+  }
+}
+export {
+  DomainRateLimiter
+};
+//# sourceMappingURL=rate-limiter.js.map

package/dist/rate-limiter.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/rate-limiter.ts"],
+  "sourcesContent": ["/**\n * Domain-aware rate limiter \u2014 ensures the crawler respects per-domain rate limits.\n *\n * Uses a token bucket algorithm per domain. When the bucket is empty, requests\n * wait until a token is available. Tokens refill at a configurable rate.\n *\n * Default: 2 requests/second per domain, burst of 5.\n * Can be configured per-domain via the rateMap (domain \u2192 requests/second).\n */\n\nexport interface RateLimiterOpts {\n  defaultRps?: number;        // requests per second (default 2)\n  burstSize?: number;         // max burst (default 5)\n  rateMap?: Record<string, number>; // domain-specific overrides\n}\n\ninterface Bucket {\n  tokens: number;\n  lastRefill: number; // Date.now()\n}\n\nfunction extractDomain(url: string): string {\n  try { return new URL(url).hostname; } catch { return url; }\n}\n\nexport class DomainRateLimiter {\n  private buckets = new Map<string, Bucket>();\n  private readonly defaultRps: number;\n  private readonly burstSize: number;\n  private readonly rateMap: Record<string, number>;\n\n  constructor(opts: RateLimiterOpts = {}) {\n    this.defaultRps = opts.defaultRps ?? (parseFloat(process.env.CRAWLER_RATE_LIMIT_DEFAULT_RPS ?? '') || 2);\n    this.burstSize = opts.burstSize ?? (parseInt(process.env.CRAWLER_RATE_LIMIT_BURST_SIZE ?? '') || 5);\n    this.rateMap = opts.rateMap ?? {};\n  }\n\n  private getBucket(domain: string): Bucket {\n    if (!this.buckets.has(domain)) {\n      this.buckets.set(domain, { tokens: this.burstSize, lastRefill: Date.now() });\n    }\n    return this.buckets.get(domain)!;\n  }\n\n  private refill(domain: string, bucket: Bucket): void {\n    const rps = this.rateMap[domain] ?? this.defaultRps;\n    const now = Date.now();\n    const elapsed = (now - bucket.lastRefill) / 1000;\n    bucket.tokens = Math.min(this.burstSize, bucket.tokens + elapsed * rps);\n    bucket.lastRefill = now;\n  }\n\n  /** Wait until a token is available for the given URL's domain. */\n  async throttle(url: string): Promise<void> {\n    const domain = extractDomain(url);\n    const bucket = this.getBucket(domain);\n    this.refill(domain, bucket);\n\n    if (bucket.tokens >= 1) {\n      bucket.tokens -= 1;\n      return;\n    }\n\n    // Calculate wait time until next token\n    const rps = this.rateMap[domain] ?? this.defaultRps;\n    const waitMs = Math.ceil((1 - bucket.tokens) / rps * 1000);\n    await new Promise(r => setTimeout(r, waitMs));\n    bucket.tokens = 0;\n    bucket.lastRefill = Date.now();\n  }\n\n  /** Override rate for a specific domain at runtime (e.g. after seeing 429). */\n  setRate(domain: string, rps: number): void {\n    this.rateMap[domain] = rps;\n    const bucket = this.getBucket(domain);\n    bucket.tokens = 0; // drain immediately on rate change\n  }\n\n  stats(): Record<string, { tokens: number; rps: number }> {\n    const result: Record<string, any> = {};\n    for (const [domain, bucket] of this.buckets) {\n      this.refill(domain, bucket);\n      result[domain] = { tokens: Math.round(bucket.tokens * 100) / 100, rps: this.rateMap[domain] ?? this.defaultRps };\n    }\n    return result;\n  }\n}\n"],
+  "mappings": "AAqBA,SAAS,cAAc,KAAqB;AAC1C,MAAI;AAAE,WAAO,IAAI,IAAI,GAAG,EAAE;AAAA,EAAU,QAAQ;AAAE,WAAO;AAAA,EAAK;AAC5D;AAEO,MAAM,kBAAkB;AAAA,EACrB,UAAU,oBAAI,IAAoB;AAAA,EACzB;AAAA,EACA;AAAA,EACA;AAAA,EAEjB,YAAY,OAAwB,CAAC,GAAG;AACtC,SAAK,aAAa,KAAK,eAAe,WAAW,QAAQ,IAAI,kCAAkC,EAAE,KAAK;AACtG,SAAK,YAAY,KAAK,cAAc,SAAS,QAAQ,IAAI,iCAAiC,EAAE,KAAK;AACjG,SAAK,UAAU,KAAK,WAAW,CAAC;AAAA,EAClC;AAAA,EAEQ,UAAU,QAAwB;AACxC,QAAI,CAAC,KAAK,QAAQ,IAAI,MAAM,GAAG;AAC7B,WAAK,QAAQ,IAAI,QAAQ,EAAE,QAAQ,KAAK,WAAW,YAAY,KAAK,IAAI,EAAE,CAAC;AAAA,IAC7E;AACA,WAAO,KAAK,QAAQ,IAAI,MAAM;AAAA,EAChC;AAAA,EAEQ,OAAO,QAAgB,QAAsB;AACnD,UAAM,MAAM,KAAK,QAAQ,MAAM,KAAK,KAAK;AACzC,UAAM,MAAM,KAAK,IAAI;AACrB,UAAM,WAAW,MAAM,OAAO,cAAc;AAC5C,WAAO,SAAS,KAAK,IAAI,KAAK,WAAW,OAAO,SAAS,UAAU,GAAG;AACtE,WAAO,aAAa;AAAA,EACtB;AAAA;AAAA,EAGA,MAAM,SAAS,KAA4B;AACzC,UAAM,SAAS,cAAc,GAAG;AAChC,UAAM,SAAS,KAAK,UAAU,MAAM;AACpC,SAAK,OAAO,QAAQ,MAAM;AAE1B,QAAI,OAAO,UAAU,GAAG;AACtB,aAAO,UAAU;AACjB;AAAA,IACF;AAGA,UAAM,MAAM,KAAK,QAAQ,MAAM,KAAK,KAAK;AACzC,UAAM,SAAS,KAAK,MAAM,IAAI,OAAO,UAAU,MAAM,GAAI;AACzD,UAAM,IAAI,QAAQ,OAAK,WAAW,GAAG,MAAM,CAAC;AAC5C,WAAO,SAAS;AAChB,WAAO,aAAa,KAAK,IAAI;AAAA,EAC/B;AAAA;AAAA,EAGA,QAAQ,QAAgB,KAAmB;AACzC,SAAK,QAAQ,MAAM,IAAI;AACvB,UAAM,SAAS,KAAK,UAAU,MAAM;AACpC,WAAO,SAAS;AAAA,EAClB;AAAA,EAEA,QAAyD;AACvD,UAAM,SAA8B,CAAC;AACrC,eAAW,CAAC,QAAQ,MAAM,KAAK,KAAK,SAAS;AAC3C,WAAK,OAAO,QAAQ,MAAM;AAC1B,aAAO,MAAM,IAAI,EAAE,QAAQ,KAAK,MAAM,OAAO,SAAS,GAAG,IAAI,KAAK,KAAK,KAAK,QAAQ,MAAM,KAAK,KAAK,WAAW;AAAA,IACjH;AACA,WAAO;AAAA,EACT;AACF;",
+  "names": []
+}

package/dist/redirect-tracker.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Redirect chain tracker — records the full redirect chain for crawled URLs.
+ *
+ * Why: redirect chains (A→B→C) slow down pages; circular redirects crash crawlers;
+ * redirect loops indicate config errors. ZeTa records chains for SEO/perf audit.
+ *
+ * Integration: hook into Playwright's page.on('response') to see each redirect hop.
+ */
+export interface RedirectHop {
+    from: string;
+    to: string;
+    statusCode: number;
+    durationMs: number;
+}
+export interface RedirectChain {
+    originalUrl: string;
+    finalUrl: string;
+    hops: RedirectHop[];
+    isCircular: boolean;
+    hopCount: number;
+    totalDurationMs: number;
+}
+import type { Page } from 'playwright';
+export declare function trackRedirects(page: Page, url: string): Promise<RedirectChain>;
+export declare class RedirectRegistry {
+    private chains;
+    register(chain: RedirectChain): void;
+    get(url: string): RedirectChain | undefined;
+    getAll(): RedirectChain[];
+    getProblematic(): RedirectChain[];
+    stats(): {
+        total: number;
+        withRedirects: number;
+        circular: number;
+        deepChains: number;
+    };
+}