@detiq/crawler 1.1.8 → 1.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/accessibility-auditor.js +74 -54
- package/dist/ai-login.js +249 -227
- package/dist/auth-token-rotator.js +67 -57
- package/dist/auth.js +336 -299
- package/dist/browser-extension.js +142 -104
- package/dist/captcha-solver.js +125 -98
- package/dist/color-contrast.js +89 -77
- package/dist/content-chunker.js +126 -126
- package/dist/content-dedup.js +71 -54
- package/dist/cookie-consent.js +83 -68
- package/dist/cookie-jar.js +60 -53
- package/dist/crawler.js +5345 -4486
- package/dist/custom-js-injector.js +61 -45
- package/dist/dark-mode-capture.js +81 -67
- package/dist/dead-link-detector.js +55 -47
- package/dist/element-hash.js +12 -8
- package/dist/email-otp.js +195 -154
- package/dist/engine-router.js +98 -87
- package/dist/export/bulk-export.js +55 -70
- package/dist/export/vector-export.js +105 -77
- package/dist/extractors/advanced.js +266 -249
- package/dist/extractors/artifact-envelope.js +58 -59
- package/dist/failure-analysis-agent.js +33 -37
- package/dist/feature-flag-detector.js +66 -51
- package/dist/form-detector.js +75 -55
- package/dist/frontier.js +101 -76
- package/dist/gap-pass.js +89 -68
- package/dist/graphql-detector.js +72 -62
- package/dist/http-engine.js +330 -282
- package/dist/incremental-diff.js +87 -54
- package/dist/index.js +28 -67
- package/dist/input-fuzzer.js +97 -93
- package/dist/interactive-probe.js +827 -715
- package/dist/job-queue.js +2169 -2151
- package/dist/js-coverage.js +56 -52
- package/dist/js-router-detector.js +117 -86
- package/dist/language-detector.js +63 -51
- package/dist/link-classifier.js +50 -32
- package/dist/live-view.js +108 -91
- package/dist/locator-extractor.js +187 -151
- package/dist/logger.js +7 -9
- package/dist/markdown-extract.js +61 -48
- package/dist/native-crawler.js +946 -780
- package/dist/network-observer.js +156 -135
- package/dist/network-throttle.js +40 -29
- package/dist/openapi-generator.js +92 -77
- package/dist/pdf-extractor.js +90 -72
- package/dist/performance-metrics.js +102 -80
- package/dist/phash.js +144 -125
- package/dist/pii-scrubber.js +68 -65
- package/dist/playwright-recorder.js +58 -70
- package/dist/policies/crawl-safety.js +34 -31
- package/dist/profiles/domain-profile.js +125 -130
- package/dist/proxy.js +172 -146
- package/dist/rate-limiter.js +65 -58
- package/dist/redirect-tracker.js +64 -60
- package/dist/redis-frontier.js +136 -116
- package/dist/redis-queue.js +144 -157
- package/dist/response-cache.js +140 -127
- package/dist/responsive-detector.js +46 -42
- package/dist/retry-policy.js +50 -46
- package/dist/robots-txt.js +83 -69
- package/dist/screen-reader.js +70 -45
- package/dist/screenshot-masker.js +77 -59
- package/dist/seo-checker.js +107 -88
- package/dist/session-capture.js +212 -192
- package/dist/session-export.js +133 -107
- package/dist/sitemap-writer.js +41 -38
- package/dist/sitemap.js +202 -141
- package/dist/skyvern.js +148 -123
- package/dist/sqs-queue.js +42 -42
- package/dist/sso-token.js +90 -74
- package/dist/stealth/human-behavior.js +173 -156
- package/dist/telemetry.js +82 -68
- package/dist/test-executor.js +335 -309
- package/dist/tls-fingerprint.js +88 -77
- package/dist/url-pattern-completer.js +90 -71
- package/dist/visual-diff-verdict.js +38 -23
- package/dist/webhook-notifier.js +69 -71
- package/dist/websocket-detector.js +68 -56
- package/dist/worker.js +38 -29
- package/package.json +1 -1
|
@@ -1,140 +1,135 @@
|
|
|
1
1
|
const COMMON_REQUIRED_ARTIFACTS = [
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
2
|
+
'screenshot',
|
|
3
|
+
'dom',
|
|
4
|
+
'markdown',
|
|
5
|
+
'accessibility',
|
|
6
|
+
'network',
|
|
7
|
+
'performance',
|
|
8
8
|
];
|
|
9
9
|
const PROFILES = {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
10
|
+
ecommerce: {
|
|
11
|
+
industry: 'ecommerce',
|
|
12
|
+
importantScreenPatterns: ['catalog', 'product', 'search', 'filter', 'cart', 'checkout', 'payment', 'order', 'returns'],
|
|
13
|
+
workflowGoals: ['discover product catalogue', 'map cart and checkout', 'capture order confirmation and failure states', 'map search/filter/sort'],
|
|
14
|
+
piiRules: ['mask email', 'mask phone', 'mask address', 'mask card-like numbers'],
|
|
15
|
+
riskyActions: ['place order', 'submit payment', 'save card', 'apply real coupon'],
|
|
16
|
+
safeProbeRules: ['do not click final payment/order buttons', 'use invalid test card data only', 'stop before irreversible purchase'],
|
|
17
|
+
requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
|
|
18
|
+
},
|
|
19
|
+
banking: {
|
|
20
|
+
industry: 'banking',
|
|
21
|
+
importantScreenPatterns: ['login', 'mfa', 'dashboard', 'accounts', 'transactions', 'transfer', 'beneficiary', 'statement', 'dispute'],
|
|
22
|
+
workflowGoals: ['map account overview', 'map transactions and statements', 'map transfer setup without submitting', 'map MFA/session timeout states'],
|
|
23
|
+
piiRules: ['mask account numbers', 'mask balances', 'mask names', 'mask addresses', 'mask tax identifiers'],
|
|
24
|
+
riskyActions: ['transfer money', 'add beneficiary', 'submit dispute', 'download sensitive statement'],
|
|
25
|
+
safeProbeRules: ['read-only by default', 'never submit transfer confirmation', 'never alter beneficiary data'],
|
|
26
|
+
requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
|
|
27
|
+
},
|
|
28
|
+
fintech: {
|
|
29
|
+
industry: 'fintech',
|
|
30
|
+
importantScreenPatterns: ['kyc', 'wallet', 'transactions', 'cards', 'payments', 'payouts', 'risk', 'compliance', 'disputes'],
|
|
31
|
+
workflowGoals: ['map KYC onboarding', 'map wallet/payment states', 'map failed payment and dispute flows', 'capture compliance screens'],
|
|
32
|
+
piiRules: ['mask card numbers', 'mask bank accounts', 'mask identity documents', 'mask balances'],
|
|
33
|
+
riskyActions: ['send payment', 'withdraw funds', 'change card status', 'submit KYC with real data'],
|
|
34
|
+
safeProbeRules: ['sandbox data only', 'stop before money movement', 'never upload real identity documents'],
|
|
35
|
+
requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
|
|
36
|
+
},
|
|
37
|
+
healthcare: {
|
|
38
|
+
industry: 'healthcare',
|
|
39
|
+
importantScreenPatterns: ['patient', 'appointment', 'claims', 'provider', 'prescription', 'records', 'lab', 'consent'],
|
|
40
|
+
workflowGoals: ['map patient lookup safely', 'map appointment booking before final submit', 'map claims and prescription screens', 'capture consent and records UX'],
|
|
41
|
+
piiRules: ['mask PHI', 'mask patient names', 'mask MRN', 'mask DOB', 'mask diagnosis', 'mask prescriptions'],
|
|
42
|
+
riskyActions: ['schedule real appointment', 'submit claim', 'change medication', 'message provider with PHI'],
|
|
43
|
+
safeProbeRules: ['use synthetic patients only', 'do not submit clinical changes', 'avoid external notifications'],
|
|
44
|
+
requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
|
|
45
|
+
},
|
|
46
|
+
pharma: {
|
|
47
|
+
industry: 'pharma',
|
|
48
|
+
importantScreenPatterns: ['product', 'dosage', 'safety', 'batch', 'adverse-event', 'regulatory', 'trial', 'label'],
|
|
49
|
+
workflowGoals: ['map product and safety content', 'map adverse event intake before submit', 'capture regulatory and batch workflows'],
|
|
50
|
+
piiRules: ['mask patient reports', 'mask reporter contact', 'mask trial participant identifiers'],
|
|
51
|
+
riskyActions: ['submit adverse event', 'change safety record', 'publish regulatory content'],
|
|
52
|
+
safeProbeRules: ['stop before regulatory or safety submission', 'use synthetic reports only'],
|
|
53
|
+
requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
|
|
54
|
+
},
|
|
55
|
+
agriculture: {
|
|
56
|
+
industry: 'agriculture',
|
|
57
|
+
importantScreenPatterns: ['crop', 'field', 'weather', 'inventory', 'marketplace', 'logistics', 'sensor', 'subsidy'],
|
|
58
|
+
workflowGoals: ['map crop and field planning', 'map inventory/marketplace flows', 'capture logistics and sensor dashboards'],
|
|
59
|
+
piiRules: ['mask farmer identity', 'mask location precision', 'mask subsidy identifiers'],
|
|
60
|
+
riskyActions: ['place market order', 'submit subsidy application', 'change sensor/device control'],
|
|
61
|
+
safeProbeRules: ['avoid irreversible orders', 'avoid device actuation', 'use test farm data'],
|
|
62
|
+
requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
|
|
63
|
+
},
|
|
64
|
+
'legacy-enterprise': {
|
|
65
|
+
industry: 'legacy-enterprise',
|
|
66
|
+
importantScreenPatterns: ['frameset', 'iframe', 'popup', 'table', 'search', 'admin', 'report', 'export'],
|
|
67
|
+
workflowGoals: ['map menu trees', 'capture iframe and popup states', 'map table filters and reports', 'detect session timeout'],
|
|
68
|
+
piiRules: ['mask table cells that look like PII', 'mask exported identifiers'],
|
|
69
|
+
riskyActions: ['bulk update', 'delete record', 'export production data', 'submit admin change'],
|
|
70
|
+
safeProbeRules: ['read-only table probing', 'never click delete/bulk actions', 'capture popups without committing'],
|
|
71
|
+
requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
|
|
72
|
+
},
|
|
73
|
+
saas: {
|
|
74
|
+
industry: 'saas',
|
|
75
|
+
importantScreenPatterns: ['dashboard', 'settings', 'users', 'billing', 'integrations', 'reports', 'admin', 'workspace'],
|
|
76
|
+
workflowGoals: ['map workspace navigation', 'map settings and integrations', 'map role/user management without changing state', 'capture dashboards and reports'],
|
|
77
|
+
piiRules: ['mask emails', 'mask tokens', 'mask API keys', 'mask billing data'],
|
|
78
|
+
riskyActions: ['delete workspace', 'invite real user', 'rotate key', 'change billing plan'],
|
|
79
|
+
safeProbeRules: ['never submit destructive settings', 'redact secrets immediately', 'use sandbox users'],
|
|
80
|
+
requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
|
|
81
|
+
},
|
|
82
|
+
unknown: {
|
|
83
|
+
industry: 'unknown',
|
|
84
|
+
importantScreenPatterns: ['login', 'dashboard', 'settings', 'search', 'detail', 'report', 'form'],
|
|
85
|
+
workflowGoals: ['map top-level navigation', 'capture forms and validation states', 'discover hidden menus', 'classify domain after first pages'],
|
|
86
|
+
piiRules: ['mask emails', 'mask phone numbers', 'mask long identifiers', 'mask card-like numbers'],
|
|
87
|
+
riskyActions: ['delete', 'submit payment', 'transfer', 'publish', 'send notification'],
|
|
88
|
+
safeProbeRules: ['read-only by default', 'stop before irreversible actions', 'use synthetic data only'],
|
|
89
|
+
requiredArtifacts: COMMON_REQUIRED_ARTIFACTS,
|
|
90
|
+
},
|
|
91
91
|
};
|
|
92
92
|
const INDUSTRY_HINTS = [
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
93
|
+
{ industry: 'ecommerce', terms: [/cart/i, /checkout/i, /product/i, /catalog/i, /sku/i, /wishlist/i, /order/i] },
|
|
94
|
+
{ industry: 'banking', terms: [/account/i, /statement/i, /beneficiar/i, /transfer/i, /iban/i, /routing/i, /balance/i] },
|
|
95
|
+
{ industry: 'fintech', terms: [/wallet/i, /kyc/i, /payout/i, /card/i, /payment/i, /risk/i, /dispute/i] },
|
|
96
|
+
{ industry: 'healthcare', terms: [/patient/i, /provider/i, /appointment/i, /claim/i, /prescription/i, /medical/i, /lab result/i] },
|
|
97
|
+
{ industry: 'pharma', terms: [/pharma/i, /dosage/i, /adverse/i, /batch/i, /regulatory/i, /clinical trial/i, /label/i] },
|
|
98
|
+
{ industry: 'agriculture', terms: [/crop/i, /field/i, /farm/i, /harvest/i, /sensor/i, /weather/i, /subsidy/i] },
|
|
99
|
+
{ industry: 'legacy-enterprise', terms: [/frameset/i, /iframe/i, /popup/i, /legacy/i, /report export/i, /mainframe/i] },
|
|
100
|
+
{ industry: 'saas', terms: [/workspace/i, /tenant/i, /integration/i, /billing/i, /api key/i, /admin/i, /dashboard/i] },
|
|
101
101
|
];
|
|
102
|
-
function classifyDomainProfile(input) {
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
102
|
+
export function classifyDomainProfile(input) {
|
|
103
|
+
const haystack = [input.appUrl, input.projectName ?? '', input.appType ?? '', input.seedText ?? ''].join(' ');
|
|
104
|
+
const scores = new Map();
|
|
105
|
+
for (const { industry, terms } of INDUSTRY_HINTS) {
|
|
106
|
+
for (const term of terms) {
|
|
107
|
+
if (term.test(haystack)) {
|
|
108
|
+
const prev = scores.get(industry) ?? { score: 0, evidence: [] };
|
|
109
|
+
prev.score += 1;
|
|
110
|
+
prev.evidence.push(term.source);
|
|
111
|
+
scores.set(industry, prev);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
113
114
|
}
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
115
|
+
let selected = 'unknown';
|
|
116
|
+
let best = 0;
|
|
117
|
+
let evidence = [];
|
|
118
|
+
for (const [industry, result] of scores) {
|
|
119
|
+
if (result.score > best) {
|
|
120
|
+
selected = industry;
|
|
121
|
+
best = result.score;
|
|
122
|
+
evidence = result.evidence;
|
|
123
|
+
}
|
|
123
124
|
}
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
};
|
|
125
|
+
const base = PROFILES[selected];
|
|
126
|
+
return {
|
|
127
|
+
...base,
|
|
128
|
+
confidence: best === 0 ? 0.25 : Math.min(0.95, 0.35 + best * 0.15),
|
|
129
|
+
evidence,
|
|
130
|
+
};
|
|
131
131
|
}
|
|
132
|
-
function getProfile(industry) {
|
|
133
|
-
|
|
134
|
-
|
|
132
|
+
export function getProfile(industry) {
|
|
133
|
+
const base = PROFILES[industry] ?? PROFILES.unknown;
|
|
134
|
+
return { ...base, confidence: 1, evidence: ['explicit'] };
|
|
135
135
|
}
|
|
136
|
-
export {
|
|
137
|
-
classifyDomainProfile,
|
|
138
|
-
getProfile
|
|
139
|
-
};
|
|
140
|
-
//# sourceMappingURL=domain-profile.js.map
|