@detiq/crawler 1.1.7 → 1.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. package/dist/accessibility-auditor.js +74 -54
  2. package/dist/ai-login.js +249 -227
  3. package/dist/auth-token-rotator.js +67 -57
  4. package/dist/auth.js +336 -299
  5. package/dist/browser-extension.js +142 -104
  6. package/dist/captcha-solver.js +125 -98
  7. package/dist/color-contrast.js +89 -77
  8. package/dist/content-chunker.js +126 -126
  9. package/dist/content-dedup.js +71 -54
  10. package/dist/cookie-consent.js +83 -68
  11. package/dist/cookie-jar.js +60 -53
  12. package/dist/crawler.js +5345 -4486
  13. package/dist/custom-js-injector.js +61 -45
  14. package/dist/dark-mode-capture.js +81 -67
  15. package/dist/dead-link-detector.js +55 -47
  16. package/dist/element-hash.js +12 -8
  17. package/dist/email-otp.js +195 -154
  18. package/dist/engine-router.js +98 -87
  19. package/dist/export/bulk-export.js +55 -70
  20. package/dist/export/vector-export.js +105 -77
  21. package/dist/extractors/advanced.d.ts +10 -0
  22. package/dist/extractors/advanced.js +266 -249
  23. package/dist/extractors/artifact-envelope.js +58 -59
  24. package/dist/failure-analysis-agent.js +33 -37
  25. package/dist/feature-flag-detector.js +66 -51
  26. package/dist/form-detector.js +75 -55
  27. package/dist/frontier.js +101 -76
  28. package/dist/gap-pass.js +89 -68
  29. package/dist/graphql-detector.js +72 -62
  30. package/dist/http-engine.js +330 -282
  31. package/dist/incremental-diff.js +87 -54
  32. package/dist/index.d.ts +2 -0
  33. package/dist/index.js +28 -63
  34. package/dist/index.js.map +2 -2
  35. package/dist/input-fuzzer.js +97 -93
  36. package/dist/interactive-probe.d.ts +36 -0
  37. package/dist/interactive-probe.js +827 -715
  38. package/dist/job-queue.js +2169 -2151
  39. package/dist/js-coverage.js +56 -52
  40. package/dist/js-router-detector.js +117 -86
  41. package/dist/language-detector.js +63 -51
  42. package/dist/link-classifier.js +50 -32
  43. package/dist/live-view.js +108 -91
  44. package/dist/locator-extractor.d.ts +37 -0
  45. package/dist/locator-extractor.js +199 -0
  46. package/dist/locator-extractor.js.map +7 -0
  47. package/dist/logger.js +7 -9
  48. package/dist/markdown-extract.js +61 -48
  49. package/dist/native-crawler.js +946 -780
  50. package/dist/network-observer.js +156 -135
  51. package/dist/network-throttle.js +40 -29
  52. package/dist/openapi-generator.js +92 -77
  53. package/dist/pdf-extractor.js +90 -72
  54. package/dist/performance-metrics.js +102 -80
  55. package/dist/phash.js +144 -125
  56. package/dist/pii-scrubber.js +68 -65
  57. package/dist/playwright-recorder.js +58 -70
  58. package/dist/policies/crawl-safety.js +34 -31
  59. package/dist/profiles/domain-profile.js +125 -130
  60. package/dist/proxy.js +172 -146
  61. package/dist/rate-limiter.js +65 -58
  62. package/dist/redirect-tracker.js +64 -60
  63. package/dist/redis-frontier.js +136 -116
  64. package/dist/redis-queue.js +144 -157
  65. package/dist/response-cache.js +140 -127
  66. package/dist/responsive-detector.js +46 -42
  67. package/dist/retry-policy.js +50 -46
  68. package/dist/robots-txt.js +83 -69
  69. package/dist/screen-reader.js +70 -45
  70. package/dist/screenshot-masker.js +77 -59
  71. package/dist/seo-checker.js +107 -88
  72. package/dist/session-capture.js +212 -192
  73. package/dist/session-export.js +133 -107
  74. package/dist/sitemap-writer.js +41 -38
  75. package/dist/sitemap.js +202 -141
  76. package/dist/skyvern.js +148 -123
  77. package/dist/sqs-queue.js +42 -42
  78. package/dist/sso-token.js +90 -74
  79. package/dist/stealth/human-behavior.js +173 -156
  80. package/dist/telemetry.js +82 -68
  81. package/dist/test-executor.js +335 -309
  82. package/dist/tls-fingerprint.js +88 -77
  83. package/dist/url-pattern-completer.js +90 -71
  84. package/dist/visual-diff-verdict.js +38 -23
  85. package/dist/webhook-notifier.js +69 -71
  86. package/dist/websocket-detector.js +68 -56
  87. package/dist/worker.js +38 -29
  88. package/package.json +1 -1
  89. package/src/index.ts +2 -0
  90. package/src/locator-extractor.ts +226 -0
  91. package/types/index.d.ts +2 -0
  92. package/types/locator-extractor.d.ts +37 -0
@@ -1,140 +1,135 @@
1
1
  const COMMON_REQUIRED_ARTIFACTS = [
2
- "screenshot",
3
- "dom",
4
- "markdown",
5
- "accessibility",
6
- "network",
7
- "performance"
2
+ 'screenshot',
3
+ 'dom',
4
+ 'markdown',
5
+ 'accessibility',
6
+ 'network',
7
+ 'performance',
8
8
  ];
9
9
  const PROFILES = {
10
- ecommerce: {
11
- industry: "ecommerce",
12
- importantScreenPatterns: ["catalog", "product", "search", "filter", "cart", "checkout", "payment", "order", "returns"],
13
- workflowGoals: ["discover product catalogue", "map cart and checkout", "capture order confirmation and failure states", "map search/filter/sort"],
14
- piiRules: ["mask email", "mask phone", "mask address", "mask card-like numbers"],
15
- riskyActions: ["place order", "submit payment", "save card", "apply real coupon"],
16
- safeProbeRules: ["do not click final payment/order buttons", "use invalid test card data only", "stop before irreversible purchase"],
17
- requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
18
- },
19
- banking: {
20
- industry: "banking",
21
- importantScreenPatterns: ["login", "mfa", "dashboard", "accounts", "transactions", "transfer", "beneficiary", "statement", "dispute"],
22
- workflowGoals: ["map account overview", "map transactions and statements", "map transfer setup without submitting", "map MFA/session timeout states"],
23
- piiRules: ["mask account numbers", "mask balances", "mask names", "mask addresses", "mask tax identifiers"],
24
- riskyActions: ["transfer money", "add beneficiary", "submit dispute", "download sensitive statement"],
25
- safeProbeRules: ["read-only by default", "never submit transfer confirmation", "never alter beneficiary data"],
26
- requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
27
- },
28
- fintech: {
29
- industry: "fintech",
30
- importantScreenPatterns: ["kyc", "wallet", "transactions", "cards", "payments", "payouts", "risk", "compliance", "disputes"],
31
- workflowGoals: ["map KYC onboarding", "map wallet/payment states", "map failed payment and dispute flows", "capture compliance screens"],
32
- piiRules: ["mask card numbers", "mask bank accounts", "mask identity documents", "mask balances"],
33
- riskyActions: ["send payment", "withdraw funds", "change card status", "submit KYC with real data"],
34
- safeProbeRules: ["sandbox data only", "stop before money movement", "never upload real identity documents"],
35
- requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
36
- },
37
- healthcare: {
38
- industry: "healthcare",
39
- importantScreenPatterns: ["patient", "appointment", "claims", "provider", "prescription", "records", "lab", "consent"],
40
- workflowGoals: ["map patient lookup safely", "map appointment booking before final submit", "map claims and prescription screens", "capture consent and records UX"],
41
- piiRules: ["mask PHI", "mask patient names", "mask MRN", "mask DOB", "mask diagnosis", "mask prescriptions"],
42
- riskyActions: ["schedule real appointment", "submit claim", "change medication", "message provider with PHI"],
43
- safeProbeRules: ["use synthetic patients only", "do not submit clinical changes", "avoid external notifications"],
44
- requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
45
- },
46
- pharma: {
47
- industry: "pharma",
48
- importantScreenPatterns: ["product", "dosage", "safety", "batch", "adverse-event", "regulatory", "trial", "label"],
49
- workflowGoals: ["map product and safety content", "map adverse event intake before submit", "capture regulatory and batch workflows"],
50
- piiRules: ["mask patient reports", "mask reporter contact", "mask trial participant identifiers"],
51
- riskyActions: ["submit adverse event", "change safety record", "publish regulatory content"],
52
- safeProbeRules: ["stop before regulatory or safety submission", "use synthetic reports only"],
53
- requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
54
- },
55
- agriculture: {
56
- industry: "agriculture",
57
- importantScreenPatterns: ["crop", "field", "weather", "inventory", "marketplace", "logistics", "sensor", "subsidy"],
58
- workflowGoals: ["map crop and field planning", "map inventory/marketplace flows", "capture logistics and sensor dashboards"],
59
- piiRules: ["mask farmer identity", "mask location precision", "mask subsidy identifiers"],
60
- riskyActions: ["place market order", "submit subsidy application", "change sensor/device control"],
61
- safeProbeRules: ["avoid irreversible orders", "avoid device actuation", "use test farm data"],
62
- requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
63
- },
64
- "legacy-enterprise": {
65
- industry: "legacy-enterprise",
66
- importantScreenPatterns: ["frameset", "iframe", "popup", "table", "search", "admin", "report", "export"],
67
- workflowGoals: ["map menu trees", "capture iframe and popup states", "map table filters and reports", "detect session timeout"],
68
- piiRules: ["mask table cells that look like PII", "mask exported identifiers"],
69
- riskyActions: ["bulk update", "delete record", "export production data", "submit admin change"],
70
- safeProbeRules: ["read-only table probing", "never click delete/bulk actions", "capture popups without committing"],
71
- requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
72
- },
73
- saas: {
74
- industry: "saas",
75
- importantScreenPatterns: ["dashboard", "settings", "users", "billing", "integrations", "reports", "admin", "workspace"],
76
- workflowGoals: ["map workspace navigation", "map settings and integrations", "map role/user management without changing state", "capture dashboards and reports"],
77
- piiRules: ["mask emails", "mask tokens", "mask API keys", "mask billing data"],
78
- riskyActions: ["delete workspace", "invite real user", "rotate key", "change billing plan"],
79
- safeProbeRules: ["never submit destructive settings", "redact secrets immediately", "use sandbox users"],
80
- requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
81
- },
82
- unknown: {
83
- industry: "unknown",
84
- importantScreenPatterns: ["login", "dashboard", "settings", "search", "detail", "report", "form"],
85
- workflowGoals: ["map top-level navigation", "capture forms and validation states", "discover hidden menus", "classify domain after first pages"],
86
- piiRules: ["mask emails", "mask phone numbers", "mask long identifiers", "mask card-like numbers"],
87
- riskyActions: ["delete", "submit payment", "transfer", "publish", "send notification"],
88
- safeProbeRules: ["read-only by default", "stop before irreversible actions", "use synthetic data only"],
89
- requiredArtifacts: COMMON_REQUIRED_ARTIFACTS
90
- }
10
+ ecommerce: {
11
+ industry: 'ecommerce',
12
+ importantScreenPatterns: ['catalog', 'product', 'search', 'filter', 'cart', 'checkout', 'payment', 'order', 'returns'],
13
+ workflowGoals: ['discover product catalogue', 'map cart and checkout', 'capture order confirmation and failure states', 'map search/filter/sort'],
14
+ piiRules: ['mask email', 'mask phone', 'mask address', 'mask card-like numbers'],
15
+ riskyActions: ['place order', 'submit payment', 'save card', 'apply real coupon'],
16
+ safeProbeRules: ['do not click final payment/order buttons', 'use invalid test card data only', 'stop before irreversible purchase'],
17
+ requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
18
+ },
19
+ banking: {
20
+ industry: 'banking',
21
+ importantScreenPatterns: ['login', 'mfa', 'dashboard', 'accounts', 'transactions', 'transfer', 'beneficiary', 'statement', 'dispute'],
22
+ workflowGoals: ['map account overview', 'map transactions and statements', 'map transfer setup without submitting', 'map MFA/session timeout states'],
23
+ piiRules: ['mask account numbers', 'mask balances', 'mask names', 'mask addresses', 'mask tax identifiers'],
24
+ riskyActions: ['transfer money', 'add beneficiary', 'submit dispute', 'download sensitive statement'],
25
+ safeProbeRules: ['read-only by default', 'never submit transfer confirmation', 'never alter beneficiary data'],
26
+ requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
27
+ },
28
+ fintech: {
29
+ industry: 'fintech',
30
+ importantScreenPatterns: ['kyc', 'wallet', 'transactions', 'cards', 'payments', 'payouts', 'risk', 'compliance', 'disputes'],
31
+ workflowGoals: ['map KYC onboarding', 'map wallet/payment states', 'map failed payment and dispute flows', 'capture compliance screens'],
32
+ piiRules: ['mask card numbers', 'mask bank accounts', 'mask identity documents', 'mask balances'],
33
+ riskyActions: ['send payment', 'withdraw funds', 'change card status', 'submit KYC with real data'],
34
+ safeProbeRules: ['sandbox data only', 'stop before money movement', 'never upload real identity documents'],
35
+ requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
36
+ },
37
+ healthcare: {
38
+ industry: 'healthcare',
39
+ importantScreenPatterns: ['patient', 'appointment', 'claims', 'provider', 'prescription', 'records', 'lab', 'consent'],
40
+ workflowGoals: ['map patient lookup safely', 'map appointment booking before final submit', 'map claims and prescription screens', 'capture consent and records UX'],
41
+ piiRules: ['mask PHI', 'mask patient names', 'mask MRN', 'mask DOB', 'mask diagnosis', 'mask prescriptions'],
42
+ riskyActions: ['schedule real appointment', 'submit claim', 'change medication', 'message provider with PHI'],
43
+ safeProbeRules: ['use synthetic patients only', 'do not submit clinical changes', 'avoid external notifications'],
44
+ requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
45
+ },
46
+ pharma: {
47
+ industry: 'pharma',
48
+ importantScreenPatterns: ['product', 'dosage', 'safety', 'batch', 'adverse-event', 'regulatory', 'trial', 'label'],
49
+ workflowGoals: ['map product and safety content', 'map adverse event intake before submit', 'capture regulatory and batch workflows'],
50
+ piiRules: ['mask patient reports', 'mask reporter contact', 'mask trial participant identifiers'],
51
+ riskyActions: ['submit adverse event', 'change safety record', 'publish regulatory content'],
52
+ safeProbeRules: ['stop before regulatory or safety submission', 'use synthetic reports only'],
53
+ requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
54
+ },
55
+ agriculture: {
56
+ industry: 'agriculture',
57
+ importantScreenPatterns: ['crop', 'field', 'weather', 'inventory', 'marketplace', 'logistics', 'sensor', 'subsidy'],
58
+ workflowGoals: ['map crop and field planning', 'map inventory/marketplace flows', 'capture logistics and sensor dashboards'],
59
+ piiRules: ['mask farmer identity', 'mask location precision', 'mask subsidy identifiers'],
60
+ riskyActions: ['place market order', 'submit subsidy application', 'change sensor/device control'],
61
+ safeProbeRules: ['avoid irreversible orders', 'avoid device actuation', 'use test farm data'],
62
+ requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
63
+ },
64
+ 'legacy-enterprise': {
65
+ industry: 'legacy-enterprise',
66
+ importantScreenPatterns: ['frameset', 'iframe', 'popup', 'table', 'search', 'admin', 'report', 'export'],
67
+ workflowGoals: ['map menu trees', 'capture iframe and popup states', 'map table filters and reports', 'detect session timeout'],
68
+ piiRules: ['mask table cells that look like PII', 'mask exported identifiers'],
69
+ riskyActions: ['bulk update', 'delete record', 'export production data', 'submit admin change'],
70
+ safeProbeRules: ['read-only table probing', 'never click delete/bulk actions', 'capture popups without committing'],
71
+ requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
72
+ },
73
+ saas: {
74
+ industry: 'saas',
75
+ importantScreenPatterns: ['dashboard', 'settings', 'users', 'billing', 'integrations', 'reports', 'admin', 'workspace'],
76
+ workflowGoals: ['map workspace navigation', 'map settings and integrations', 'map role/user management without changing state', 'capture dashboards and reports'],
77
+ piiRules: ['mask emails', 'mask tokens', 'mask API keys', 'mask billing data'],
78
+ riskyActions: ['delete workspace', 'invite real user', 'rotate key', 'change billing plan'],
79
+ safeProbeRules: ['never submit destructive settings', 'redact secrets immediately', 'use sandbox users'],
80
+ requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
81
+ },
82
+ unknown: {
83
+ industry: 'unknown',
84
+ importantScreenPatterns: ['login', 'dashboard', 'settings', 'search', 'detail', 'report', 'form'],
85
+ workflowGoals: ['map top-level navigation', 'capture forms and validation states', 'discover hidden menus', 'classify domain after first pages'],
86
+ piiRules: ['mask emails', 'mask phone numbers', 'mask long identifiers', 'mask card-like numbers'],
87
+ riskyActions: ['delete', 'submit payment', 'transfer', 'publish', 'send notification'],
88
+ safeProbeRules: ['read-only by default', 'stop before irreversible actions', 'use synthetic data only'],
89
+ requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
90
+ },
91
91
  };
92
92
  const INDUSTRY_HINTS = [
93
- { industry: "ecommerce", terms: [/cart/i, /checkout/i, /product/i, /catalog/i, /sku/i, /wishlist/i, /order/i] },
94
- { industry: "banking", terms: [/account/i, /statement/i, /beneficiar/i, /transfer/i, /iban/i, /routing/i, /balance/i] },
95
- { industry: "fintech", terms: [/wallet/i, /kyc/i, /payout/i, /card/i, /payment/i, /risk/i, /dispute/i] },
96
- { industry: "healthcare", terms: [/patient/i, /provider/i, /appointment/i, /claim/i, /prescription/i, /medical/i, /lab result/i] },
97
- { industry: "pharma", terms: [/pharma/i, /dosage/i, /adverse/i, /batch/i, /regulatory/i, /clinical trial/i, /label/i] },
98
- { industry: "agriculture", terms: [/crop/i, /field/i, /farm/i, /harvest/i, /sensor/i, /weather/i, /subsidy/i] },
99
- { industry: "legacy-enterprise", terms: [/frameset/i, /iframe/i, /popup/i, /legacy/i, /report export/i, /mainframe/i] },
100
- { industry: "saas", terms: [/workspace/i, /tenant/i, /integration/i, /billing/i, /api key/i, /admin/i, /dashboard/i] }
93
+ { industry: 'ecommerce', terms: [/cart/i, /checkout/i, /product/i, /catalog/i, /sku/i, /wishlist/i, /order/i] },
94
+ { industry: 'banking', terms: [/account/i, /statement/i, /beneficiar/i, /transfer/i, /iban/i, /routing/i, /balance/i] },
95
+ { industry: 'fintech', terms: [/wallet/i, /kyc/i, /payout/i, /card/i, /payment/i, /risk/i, /dispute/i] },
96
+ { industry: 'healthcare', terms: [/patient/i, /provider/i, /appointment/i, /claim/i, /prescription/i, /medical/i, /lab result/i] },
97
+ { industry: 'pharma', terms: [/pharma/i, /dosage/i, /adverse/i, /batch/i, /regulatory/i, /clinical trial/i, /label/i] },
98
+ { industry: 'agriculture', terms: [/crop/i, /field/i, /farm/i, /harvest/i, /sensor/i, /weather/i, /subsidy/i] },
99
+ { industry: 'legacy-enterprise', terms: [/frameset/i, /iframe/i, /popup/i, /legacy/i, /report export/i, /mainframe/i] },
100
+ { industry: 'saas', terms: [/workspace/i, /tenant/i, /integration/i, /billing/i, /api key/i, /admin/i, /dashboard/i] },
101
101
  ];
102
- function classifyDomainProfile(input) {
103
- const haystack = [input.appUrl, input.projectName ?? "", input.appType ?? "", input.seedText ?? ""].join(" ");
104
- const scores = /* @__PURE__ */ new Map();
105
- for (const { industry, terms } of INDUSTRY_HINTS) {
106
- for (const term of terms) {
107
- if (term.test(haystack)) {
108
- const prev = scores.get(industry) ?? { score: 0, evidence: [] };
109
- prev.score += 1;
110
- prev.evidence.push(term.source);
111
- scores.set(industry, prev);
112
- }
102
+ export function classifyDomainProfile(input) {
103
+ const haystack = [input.appUrl, input.projectName ?? '', input.appType ?? '', input.seedText ?? ''].join(' ');
104
+ const scores = new Map();
105
+ for (const { industry, terms } of INDUSTRY_HINTS) {
106
+ for (const term of terms) {
107
+ if (term.test(haystack)) {
108
+ const prev = scores.get(industry) ?? { score: 0, evidence: [] };
109
+ prev.score += 1;
110
+ prev.evidence.push(term.source);
111
+ scores.set(industry, prev);
112
+ }
113
+ }
113
114
  }
114
- }
115
- let selected = "unknown";
116
- let best = 0;
117
- let evidence = [];
118
- for (const [industry, result] of scores) {
119
- if (result.score > best) {
120
- selected = industry;
121
- best = result.score;
122
- evidence = result.evidence;
115
+ let selected = 'unknown';
116
+ let best = 0;
117
+ let evidence = [];
118
+ for (const [industry, result] of scores) {
119
+ if (result.score > best) {
120
+ selected = industry;
121
+ best = result.score;
122
+ evidence = result.evidence;
123
+ }
123
124
  }
124
- }
125
- const base = PROFILES[selected];
126
- return {
127
- ...base,
128
- confidence: best === 0 ? 0.25 : Math.min(0.95, 0.35 + best * 0.15),
129
- evidence
130
- };
125
+ const base = PROFILES[selected];
126
+ return {
127
+ ...base,
128
+ confidence: best === 0 ? 0.25 : Math.min(0.95, 0.35 + best * 0.15),
129
+ evidence,
130
+ };
131
131
  }
132
- function getProfile(industry) {
133
- const base = PROFILES[industry] ?? PROFILES.unknown;
134
- return { ...base, confidence: 1, evidence: ["explicit"] };
132
+ export function getProfile(industry) {
133
+ const base = PROFILES[industry] ?? PROFILES.unknown;
134
+ return { ...base, confidence: 1, evidence: ['explicit'] };
135
135
  }
136
- export {
137
- classifyDomainProfile,
138
- getProfile
139
- };
140
- //# sourceMappingURL=domain-profile.js.map