@detiq/crawler 1.1.9 → 1.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/dist/accessibility-auditor.js +54 -74
  2. package/dist/ai-login.js +227 -249
  3. package/dist/auth-token-rotator.js +57 -67
  4. package/dist/auth.js +299 -336
  5. package/dist/browser-extension.js +104 -142
  6. package/dist/captcha-solver.js +98 -125
  7. package/dist/color-contrast.js +77 -89
  8. package/dist/content-chunker.js +126 -126
  9. package/dist/content-dedup.js +54 -71
  10. package/dist/cookie-consent.js +68 -83
  11. package/dist/cookie-jar.js +53 -60
  12. package/dist/crawler.js +4494 -5353
  13. package/dist/custom-js-injector.js +45 -61
  14. package/dist/dark-mode-capture.js +67 -81
  15. package/dist/dead-link-detector.js +47 -55
  16. package/dist/element-hash.js +8 -12
  17. package/dist/email-otp.js +154 -195
  18. package/dist/engine-router.js +87 -98
  19. package/dist/export/bulk-export.js +70 -55
  20. package/dist/export/vector-export.js +77 -105
  21. package/dist/extractors/advanced.js +249 -266
  22. package/dist/extractors/artifact-envelope.js +59 -58
  23. package/dist/failure-analysis-agent.js +37 -33
  24. package/dist/feature-flag-detector.js +51 -66
  25. package/dist/form-detector.js +55 -75
  26. package/dist/frontier.js +76 -101
  27. package/dist/gap-pass.js +68 -89
  28. package/dist/graphql-detector.js +62 -72
  29. package/dist/http-engine.js +282 -330
  30. package/dist/incremental-diff.js +54 -87
  31. package/dist/index.js +67 -28
  32. package/dist/input-fuzzer.js +93 -97
  33. package/dist/interactive-probe.js +715 -827
  34. package/dist/job-queue.js +2151 -2169
  35. package/dist/js-coverage.js +52 -56
  36. package/dist/js-router-detector.js +86 -117
  37. package/dist/language-detector.js +51 -63
  38. package/dist/link-classifier.js +32 -50
  39. package/dist/live-view.js +91 -108
  40. package/dist/locator-extractor.js +151 -187
  41. package/dist/logger.js +9 -7
  42. package/dist/markdown-extract.js +48 -61
  43. package/dist/native-crawler.js +780 -946
  44. package/dist/network-observer.js +135 -156
  45. package/dist/network-throttle.js +29 -40
  46. package/dist/openapi-generator.js +77 -92
  47. package/dist/pdf-extractor.js +72 -90
  48. package/dist/performance-metrics.js +80 -102
  49. package/dist/phash.js +125 -144
  50. package/dist/pii-scrubber.js +65 -68
  51. package/dist/playwright-recorder.js +70 -58
  52. package/dist/policies/crawl-safety.js +31 -34
  53. package/dist/profiles/domain-profile.js +130 -125
  54. package/dist/proxy.js +146 -172
  55. package/dist/rate-limiter.js +58 -65
  56. package/dist/redirect-tracker.js +60 -64
  57. package/dist/redis-frontier.js +116 -136
  58. package/dist/redis-queue.js +157 -144
  59. package/dist/response-cache.js +127 -140
  60. package/dist/responsive-detector.js +42 -46
  61. package/dist/retry-policy.js +46 -50
  62. package/dist/robots-txt.js +69 -83
  63. package/dist/screen-reader.js +45 -70
  64. package/dist/screenshot-masker.js +59 -77
  65. package/dist/seo-checker.js +88 -107
  66. package/dist/session-capture.js +192 -212
  67. package/dist/session-export.js +107 -133
  68. package/dist/sitemap-writer.js +38 -41
  69. package/dist/sitemap.js +141 -202
  70. package/dist/skyvern.js +123 -148
  71. package/dist/sqs-queue.js +42 -42
  72. package/dist/sso-token.js +74 -90
  73. package/dist/stealth/human-behavior.js +156 -173
  74. package/dist/telemetry.js +68 -82
  75. package/dist/test-executor.js +309 -335
  76. package/dist/tls-fingerprint.js +77 -88
  77. package/dist/url-pattern-completer.js +71 -90
  78. package/dist/visual-diff-verdict.js +23 -38
  79. package/dist/webhook-notifier.js +71 -69
  80. package/dist/websocket-detector.js +56 -68
  81. package/dist/worker.js +29 -38
  82. package/package.json +1 -1
  83. package/src/job-queue.ts +136 -0
@@ -1,135 +1,140 @@
1
1
  const COMMON_REQUIRED_ARTIFACTS = [
2
- 'screenshot',
3
- 'dom',
4
- 'markdown',
5
- 'accessibility',
6
- 'network',
7
- 'performance',
2
+ "screenshot",
3
+ "dom",
4
+ "markdown",
5
+ "accessibility",
6
+ "network",
7
+ "performance"
8
8
  ];
9
9
  const PROFILES = {
10
- ecommerce: {
11
- industry: 'ecommerce',
12
- importantScreenPatterns: ['catalog', 'product', 'search', 'filter', 'cart', 'checkout', 'payment', 'order', 'returns'],
13
- workflowGoals: ['discover product catalogue', 'map cart and checkout', 'capture order confirmation and failure states', 'map search/filter/sort'],
14
- piiRules: ['mask email', 'mask phone', 'mask address', 'mask card-like numbers'],
15
- riskyActions: ['place order', 'submit payment', 'save card', 'apply real coupon'],
16
- safeProbeRules: ['do not click final payment/order buttons', 'use invalid test card data only', 'stop before irreversible purchase'],
17
- requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
18
- },
19
- banking: {
20
- industry: 'banking',
21
- importantScreenPatterns: ['login', 'mfa', 'dashboard', 'accounts', 'transactions', 'transfer', 'beneficiary', 'statement', 'dispute'],
22
- workflowGoals: ['map account overview', 'map transactions and statements', 'map transfer setup without submitting', 'map MFA/session timeout states'],
23
- piiRules: ['mask account numbers', 'mask balances', 'mask names', 'mask addresses', 'mask tax identifiers'],
24
- riskyActions: ['transfer money', 'add beneficiary', 'submit dispute', 'download sensitive statement'],
25
- safeProbeRules: ['read-only by default', 'never submit transfer confirmation', 'never alter beneficiary data'],
26
- requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
27
- },
28
- fintech: {
29
- industry: 'fintech',
30
- importantScreenPatterns: ['kyc', 'wallet', 'transactions', 'cards', 'payments', 'payouts', 'risk', 'compliance', 'disputes'],
31
- workflowGoals: ['map KYC onboarding', 'map wallet/payment states', 'map failed payment and dispute flows', 'capture compliance screens'],
32
- piiRules: ['mask card numbers', 'mask bank accounts', 'mask identity documents', 'mask balances'],
33
- riskyActions: ['send payment', 'withdraw funds', 'change card status', 'submit KYC with real data'],
34
- safeProbeRules: ['sandbox data only', 'stop before money movement', 'never upload real identity documents'],
35
- requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
36
- },
37
- healthcare: {
38
- industry: 'healthcare',
39
- importantScreenPatterns: ['patient', 'appointment', 'claims', 'provider', 'prescription', 'records', 'lab', 'consent'],
40
- workflowGoals: ['map patient lookup safely', 'map appointment booking before final submit', 'map claims and prescription screens', 'capture consent and records UX'],
41
- piiRules: ['mask PHI', 'mask patient names', 'mask MRN', 'mask DOB', 'mask diagnosis', 'mask prescriptions'],
42
- riskyActions: ['schedule real appointment', 'submit claim', 'change medication', 'message provider with PHI'],
43
- safeProbeRules: ['use synthetic patients only', 'do not submit clinical changes', 'avoid external notifications'],
44
- requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
45
- },
46
- pharma: {
47
- industry: 'pharma',
48
- importantScreenPatterns: ['product', 'dosage', 'safety', 'batch', 'adverse-event', 'regulatory', 'trial', 'label'],
49
- workflowGoals: ['map product and safety content', 'map adverse event intake before submit', 'capture regulatory and batch workflows'],
50
- piiRules: ['mask patient reports', 'mask reporter contact', 'mask trial participant identifiers'],
51
- riskyActions: ['submit adverse event', 'change safety record', 'publish regulatory content'],
52
- safeProbeRules: ['stop before regulatory or safety submission', 'use synthetic reports only'],
53
- requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
54
- },
55
- agriculture: {
56
- industry: 'agriculture',
57
- importantScreenPatterns: ['crop', 'field', 'weather', 'inventory', 'marketplace', 'logistics', 'sensor', 'subsidy'],
58
- workflowGoals: ['map crop and field planning', 'map inventory/marketplace flows', 'capture logistics and sensor dashboards'],
59
- piiRules: ['mask farmer identity', 'mask location precision', 'mask subsidy identifiers'],
60
- riskyActions: ['place market order', 'submit subsidy application', 'change sensor/device control'],
61
- safeProbeRules: ['avoid irreversible orders', 'avoid device actuation', 'use test farm data'],
62
- requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
63
- },
64
- 'legacy-enterprise': {
65
- industry: 'legacy-enterprise',
66
- importantScreenPatterns: ['frameset', 'iframe', 'popup', 'table', 'search', 'admin', 'report', 'export'],
67
- workflowGoals: ['map menu trees', 'capture iframe and popup states', 'map table filters and reports', 'detect session timeout'],
68
- piiRules: ['mask table cells that look like PII', 'mask exported identifiers'],
69
- riskyActions: ['bulk update', 'delete record', 'export production data', 'submit admin change'],
70
- safeProbeRules: ['read-only table probing', 'never click delete/bulk actions', 'capture popups without committing'],
71
- requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
72
- },
73
- saas: {
74
- industry: 'saas',
75
- importantScreenPatterns: ['dashboard', 'settings', 'users', 'billing', 'integrations', 'reports', 'admin', 'workspace'],
76
- workflowGoals: ['map workspace navigation', 'map settings and integrations', 'map role/user management without changing state', 'capture dashboards and reports'],
77
- piiRules: ['mask emails', 'mask tokens', 'mask API keys', 'mask billing data'],
78
- riskyActions: ['delete workspace', 'invite real user', 'rotate key', 'change billing plan'],
79
- safeProbeRules: ['never submit destructive settings', 'redact secrets immediately', 'use sandbox users'],
80
- requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
81
- },
82
- unknown: {
83
- industry: 'unknown',
84
- importantScreenPatterns: ['login', 'dashboard', 'settings', 'search', 'detail', 'report', 'form'],
85
- workflowGoals: ['map top-level navigation', 'capture forms and validation states', 'discover hidden menus', 'classify domain after first pages'],
86
- piiRules: ['mask emails', 'mask phone numbers', 'mask long identifiers', 'mask card-like numbers'],
87
- riskyActions: ['delete', 'submit payment', 'transfer', 'publish', 'send notification'],
88
- safeProbeRules: ['read-only by default', 'stop before irreversible actions', 'use synthetic data only'],
89
- requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
90
- },
10
+ ecommerce: {
11
+ industry: "ecommerce",
12
+ importantScreenPatterns: ["catalog", "product", "search", "filter", "cart", "checkout", "payment", "order", "returns"],
13
+ workflowGoals: ["discover product catalogue", "map cart and checkout", "capture order confirmation and failure states", "map search/filter/sort"],
14
+ piiRules: ["mask email", "mask phone", "mask address", "mask card-like numbers"],
15
+ riskyActions: ["place order", "submit payment", "save card", "apply real coupon"],
16
+ safeProbeRules: ["do not click final payment/order buttons", "use invalid test card data only", "stop before irreversible purchase"],
17
+ requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
18
+ },
19
+ banking: {
20
+ industry: "banking",
21
+ importantScreenPatterns: ["login", "mfa", "dashboard", "accounts", "transactions", "transfer", "beneficiary", "statement", "dispute"],
22
+ workflowGoals: ["map account overview", "map transactions and statements", "map transfer setup without submitting", "map MFA/session timeout states"],
23
+ piiRules: ["mask account numbers", "mask balances", "mask names", "mask addresses", "mask tax identifiers"],
24
+ riskyActions: ["transfer money", "add beneficiary", "submit dispute", "download sensitive statement"],
25
+ safeProbeRules: ["read-only by default", "never submit transfer confirmation", "never alter beneficiary data"],
26
+ requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
27
+ },
28
+ fintech: {
29
+ industry: "fintech",
30
+ importantScreenPatterns: ["kyc", "wallet", "transactions", "cards", "payments", "payouts", "risk", "compliance", "disputes"],
31
+ workflowGoals: ["map KYC onboarding", "map wallet/payment states", "map failed payment and dispute flows", "capture compliance screens"],
32
+ piiRules: ["mask card numbers", "mask bank accounts", "mask identity documents", "mask balances"],
33
+ riskyActions: ["send payment", "withdraw funds", "change card status", "submit KYC with real data"],
34
+ safeProbeRules: ["sandbox data only", "stop before money movement", "never upload real identity documents"],
35
+ requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
36
+ },
37
+ healthcare: {
38
+ industry: "healthcare",
39
+ importantScreenPatterns: ["patient", "appointment", "claims", "provider", "prescription", "records", "lab", "consent"],
40
+ workflowGoals: ["map patient lookup safely", "map appointment booking before final submit", "map claims and prescription screens", "capture consent and records UX"],
41
+ piiRules: ["mask PHI", "mask patient names", "mask MRN", "mask DOB", "mask diagnosis", "mask prescriptions"],
42
+ riskyActions: ["schedule real appointment", "submit claim", "change medication", "message provider with PHI"],
43
+ safeProbeRules: ["use synthetic patients only", "do not submit clinical changes", "avoid external notifications"],
44
+ requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
45
+ },
46
+ pharma: {
47
+ industry: "pharma",
48
+ importantScreenPatterns: ["product", "dosage", "safety", "batch", "adverse-event", "regulatory", "trial", "label"],
49
+ workflowGoals: ["map product and safety content", "map adverse event intake before submit", "capture regulatory and batch workflows"],
50
+ piiRules: ["mask patient reports", "mask reporter contact", "mask trial participant identifiers"],
51
+ riskyActions: ["submit adverse event", "change safety record", "publish regulatory content"],
52
+ safeProbeRules: ["stop before regulatory or safety submission", "use synthetic reports only"],
53
+ requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
54
+ },
55
+ agriculture: {
56
+ industry: "agriculture",
57
+ importantScreenPatterns: ["crop", "field", "weather", "inventory", "marketplace", "logistics", "sensor", "subsidy"],
58
+ workflowGoals: ["map crop and field planning", "map inventory/marketplace flows", "capture logistics and sensor dashboards"],
59
+ piiRules: ["mask farmer identity", "mask location precision", "mask subsidy identifiers"],
60
+ riskyActions: ["place market order", "submit subsidy application", "change sensor/device control"],
61
+ safeProbeRules: ["avoid irreversible orders", "avoid device actuation", "use test farm data"],
62
+ requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
63
+ },
64
+ "legacy-enterprise": {
65
+ industry: "legacy-enterprise",
66
+ importantScreenPatterns: ["frameset", "iframe", "popup", "table", "search", "admin", "report", "export"],
67
+ workflowGoals: ["map menu trees", "capture iframe and popup states", "map table filters and reports", "detect session timeout"],
68
+ piiRules: ["mask table cells that look like PII", "mask exported identifiers"],
69
+ riskyActions: ["bulk update", "delete record", "export production data", "submit admin change"],
70
+ safeProbeRules: ["read-only table probing", "never click delete/bulk actions", "capture popups without committing"],
71
+ requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
72
+ },
73
+ saas: {
74
+ industry: "saas",
75
+ importantScreenPatterns: ["dashboard", "settings", "users", "billing", "integrations", "reports", "admin", "workspace"],
76
+ workflowGoals: ["map workspace navigation", "map settings and integrations", "map role/user management without changing state", "capture dashboards and reports"],
77
+ piiRules: ["mask emails", "mask tokens", "mask API keys", "mask billing data"],
78
+ riskyActions: ["delete workspace", "invite real user", "rotate key", "change billing plan"],
79
+ safeProbeRules: ["never submit destructive settings", "redact secrets immediately", "use sandbox users"],
80
+ requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
81
+ },
82
+ unknown: {
83
+ industry: "unknown",
84
+ importantScreenPatterns: ["login", "dashboard", "settings", "search", "detail", "report", "form"],
85
+ workflowGoals: ["map top-level navigation", "capture forms and validation states", "discover hidden menus", "classify domain after first pages"],
86
+ piiRules: ["mask emails", "mask phone numbers", "mask long identifiers", "mask card-like numbers"],
87
+ riskyActions: ["delete", "submit payment", "transfer", "publish", "send notification"],
88
+ safeProbeRules: ["read-only by default", "stop before irreversible actions", "use synthetic data only"],
89
+ requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
90
+ }
91
91
  };
92
92
  const INDUSTRY_HINTS = [
93
- { industry: 'ecommerce', terms: [/cart/i, /checkout/i, /product/i, /catalog/i, /sku/i, /wishlist/i, /order/i] },
94
- { industry: 'banking', terms: [/account/i, /statement/i, /beneficiar/i, /transfer/i, /iban/i, /routing/i, /balance/i] },
95
- { industry: 'fintech', terms: [/wallet/i, /kyc/i, /payout/i, /card/i, /payment/i, /risk/i, /dispute/i] },
96
- { industry: 'healthcare', terms: [/patient/i, /provider/i, /appointment/i, /claim/i, /prescription/i, /medical/i, /lab result/i] },
97
- { industry: 'pharma', terms: [/pharma/i, /dosage/i, /adverse/i, /batch/i, /regulatory/i, /clinical trial/i, /label/i] },
98
- { industry: 'agriculture', terms: [/crop/i, /field/i, /farm/i, /harvest/i, /sensor/i, /weather/i, /subsidy/i] },
99
- { industry: 'legacy-enterprise', terms: [/frameset/i, /iframe/i, /popup/i, /legacy/i, /report export/i, /mainframe/i] },
100
- { industry: 'saas', terms: [/workspace/i, /tenant/i, /integration/i, /billing/i, /api key/i, /admin/i, /dashboard/i] },
93
+ { industry: "ecommerce", terms: [/cart/i, /checkout/i, /product/i, /catalog/i, /sku/i, /wishlist/i, /order/i] },
94
+ { industry: "banking", terms: [/account/i, /statement/i, /beneficiar/i, /transfer/i, /iban/i, /routing/i, /balance/i] },
95
+ { industry: "fintech", terms: [/wallet/i, /kyc/i, /payout/i, /card/i, /payment/i, /risk/i, /dispute/i] },
96
+ { industry: "healthcare", terms: [/patient/i, /provider/i, /appointment/i, /claim/i, /prescription/i, /medical/i, /lab result/i] },
97
+ { industry: "pharma", terms: [/pharma/i, /dosage/i, /adverse/i, /batch/i, /regulatory/i, /clinical trial/i, /label/i] },
98
+ { industry: "agriculture", terms: [/crop/i, /field/i, /farm/i, /harvest/i, /sensor/i, /weather/i, /subsidy/i] },
99
+ { industry: "legacy-enterprise", terms: [/frameset/i, /iframe/i, /popup/i, /legacy/i, /report export/i, /mainframe/i] },
100
+ { industry: "saas", terms: [/workspace/i, /tenant/i, /integration/i, /billing/i, /api key/i, /admin/i, /dashboard/i] }
101
101
  ];
102
- export function classifyDomainProfile(input) {
103
- const haystack = [input.appUrl, input.projectName ?? '', input.appType ?? '', input.seedText ?? ''].join(' ');
104
- const scores = new Map();
105
- for (const { industry, terms } of INDUSTRY_HINTS) {
106
- for (const term of terms) {
107
- if (term.test(haystack)) {
108
- const prev = scores.get(industry) ?? { score: 0, evidence: [] };
109
- prev.score += 1;
110
- prev.evidence.push(term.source);
111
- scores.set(industry, prev);
112
- }
113
- }
102
+ function classifyDomainProfile(input) {
103
+ const haystack = [input.appUrl, input.projectName ?? "", input.appType ?? "", input.seedText ?? ""].join(" ");
104
+ const scores = /* @__PURE__ */ new Map();
105
+ for (const { industry, terms } of INDUSTRY_HINTS) {
106
+ for (const term of terms) {
107
+ if (term.test(haystack)) {
108
+ const prev = scores.get(industry) ?? { score: 0, evidence: [] };
109
+ prev.score += 1;
110
+ prev.evidence.push(term.source);
111
+ scores.set(industry, prev);
112
+ }
114
113
  }
115
- let selected = 'unknown';
116
- let best = 0;
117
- let evidence = [];
118
- for (const [industry, result] of scores) {
119
- if (result.score > best) {
120
- selected = industry;
121
- best = result.score;
122
- evidence = result.evidence;
123
- }
114
+ }
115
+ let selected = "unknown";
116
+ let best = 0;
117
+ let evidence = [];
118
+ for (const [industry, result] of scores) {
119
+ if (result.score > best) {
120
+ selected = industry;
121
+ best = result.score;
122
+ evidence = result.evidence;
124
123
  }
125
- const base = PROFILES[selected];
126
- return {
127
- ...base,
128
- confidence: best === 0 ? 0.25 : Math.min(0.95, 0.35 + best * 0.15),
129
- evidence,
130
- };
124
+ }
125
+ const base = PROFILES[selected];
126
+ return {
127
+ ...base,
128
+ confidence: best === 0 ? 0.25 : Math.min(0.95, 0.35 + best * 0.15),
129
+ evidence
130
+ };
131
131
  }
132
- export function getProfile(industry) {
133
- const base = PROFILES[industry] ?? PROFILES.unknown;
134
- return { ...base, confidence: 1, evidence: ['explicit'] };
132
+ function getProfile(industry) {
133
+ const base = PROFILES[industry] ?? PROFILES.unknown;
134
+ return { ...base, confidence: 1, evidence: ["explicit"] };
135
135
  }
136
+ export {
137
+ classifyDomainProfile,
138
+ getProfile
139
+ };
140
+ //# sourceMappingURL=domain-profile.js.map