@rester159/blacktip 0.2.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +222 -0
- package/README.md +25 -0
- package/dist/akamai-sensor.d.ts +128 -0
- package/dist/akamai-sensor.d.ts.map +1 -0
- package/dist/akamai-sensor.js +190 -0
- package/dist/akamai-sensor.js.map +1 -0
- package/dist/behavioral/parsers.d.ts +89 -0
- package/dist/behavioral/parsers.d.ts.map +1 -0
- package/dist/behavioral/parsers.js +223 -0
- package/dist/behavioral/parsers.js.map +1 -0
- package/dist/blacktip.d.ts +68 -1
- package/dist/blacktip.d.ts.map +1 -1
- package/dist/blacktip.js +140 -1
- package/dist/blacktip.js.map +1 -1
- package/dist/browser-core.d.ts +10 -0
- package/dist/browser-core.d.ts.map +1 -1
- package/dist/browser-core.js +49 -0
- package/dist/browser-core.js.map +1 -1
- package/dist/diagnostics.d.ts +31 -0
- package/dist/diagnostics.d.ts.map +1 -1
- package/dist/diagnostics.js +146 -0
- package/dist/diagnostics.js.map +1 -1
- package/dist/identity-pool.d.ts +160 -0
- package/dist/identity-pool.d.ts.map +1 -0
- package/dist/identity-pool.js +288 -0
- package/dist/identity-pool.js.map +1 -0
- package/dist/index.d.ts +11 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -1
- package/dist/index.js.map +1 -1
- package/dist/tls-rewriter.d.ts +74 -0
- package/dist/tls-rewriter.d.ts.map +1 -0
- package/dist/tls-rewriter.js +203 -0
- package/dist/tls-rewriter.js.map +1 -0
- package/dist/tls-side-channel.d.ts +91 -0
- package/dist/tls-side-channel.d.ts.map +1 -0
- package/dist/tls-side-channel.js +248 -0
- package/dist/tls-side-channel.js.map +1 -0
- package/dist/types.d.ts +46 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/docs/akamai-bypass.md +257 -0
- package/docs/akamai-sensor.md +183 -0
- package/docs/anti-bot-validation.md +84 -0
- package/docs/calibration-validation.md +93 -0
- package/docs/identity-pool.md +176 -0
- package/docs/tls-rewriting.md +121 -0
- package/docs/tls-side-channel.md +83 -0
- package/native/tls-client/go.mod +21 -0
- package/native/tls-client/go.sum +36 -0
- package/native/tls-client/main.go +216 -0
- package/package.json +8 -2
- package/scripts/fit-cmu-keystroke.mjs +186 -0
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dataset parsers for behavioral calibration.
|
|
3
|
+
*
|
|
4
|
+
* The calibration scaffold (`./calibration.ts`) defines normalized
|
|
5
|
+
* `MouseMovement` and `TypingSession` shapes plus distribution fitters.
|
|
6
|
+
* What was missing through v0.2.0 was the bridge from real public datasets
|
|
7
|
+
* into those shapes — without parsers, the scaffold required every user
|
|
8
|
+
* to write their own ETL, which kept "Tier 2 behavioral calibration" on
|
|
9
|
+
* the deferred list.
|
|
10
|
+
*
|
|
11
|
+
* This module ships parsers for the two datasets we point users at
|
|
12
|
+
* most often:
|
|
13
|
+
*
|
|
14
|
+
* 1. **CMU Keystroke Dynamics** (Killourhy & Maxion, 2009).
|
|
15
|
+
* Free for research. CSV format with columns `subject`, `sessionIndex`,
|
|
16
|
+
* `rep`, then for the fixed phrase `.tie5Roanl` a tuple of
|
|
17
|
+
* `H.<key>` (hold), `DD.<k1>.<k2>` (down-down latency), and
|
|
18
|
+
* `UD.<k1>.<k2>` (up-down = flight time). 51 subjects × 8 sessions
|
|
19
|
+
* × 50 repetitions = 20,400 phrases.
|
|
20
|
+
*
|
|
21
|
+
* 2. **Balabit Mouse Dynamics Challenge** (Antal & Egyed-Zsigmond, 2014).
|
|
22
|
+
* Free for research. Per-session CSV with columns
|
|
23
|
+
* `record_timestamp`, `client_timestamp`, `button`, `state`, `x`, `y`.
|
|
24
|
+
* Each row is a single mouse event; consecutive `Mouse` rows
|
|
25
|
+
* followed by a `Pressed` row form one movement.
|
|
26
|
+
*
|
|
27
|
+
* Plus a generic JSON loader for users with their own telemetry exported
|
|
28
|
+
* in the `MouseMovement` / `TypingSession` shapes directly.
|
|
29
|
+
*
|
|
30
|
+
* The parsers do NOT download the datasets — both have ToS that say "do
|
|
31
|
+
* not redistribute". Users acquire the data themselves and feed the file
|
|
32
|
+
* contents in as a string. We just turn raw text into normalized samples.
|
|
33
|
+
*/
|
|
34
|
+
import type { MouseMovement, TypingSession } from './calibration.js';
|
|
35
|
+
/** The fixed phrase typed by every CMU subject. Used to map column index → key. */
|
|
36
|
+
export declare const CMU_PHRASE = ".tie5Roanl";
|
|
37
|
+
/**
|
|
38
|
+
* Parse the CMU Keystroke Dynamics CSV (DSL-StrongPasswordData.csv).
|
|
39
|
+
*
|
|
40
|
+
* Each row is one repetition of the phrase `.tie5Roanl` plus Return,
|
|
41
|
+
* yielding 11 keys, 11 hold-times, 10 down-down and 10 up-down latencies.
|
|
42
|
+
* We normalize each row into one `TypingSession` of 11 keystrokes.
|
|
43
|
+
*
|
|
44
|
+
* The CSV looks like:
|
|
45
|
+
* subject,sessionIndex,rep,H.period,DD.period.t,UD.period.t,H.t,DD.t.i,UD.t.i,H.i,...
|
|
46
|
+
* s002,1,1,0.1491,0.3979,0.2488,0.1069,0.1674,0.0605,...
|
|
47
|
+
*
|
|
48
|
+
* All time values are in **seconds** in the source file; we convert to
|
|
49
|
+
* milliseconds.
|
|
50
|
+
*
|
|
51
|
+
* Returns one `TypingSession` per CSV row. Pass these straight into
|
|
52
|
+
* `fitTypingDynamics()`.
|
|
53
|
+
*/
|
|
54
|
+
export declare function parseCmuKeystrokeCsv(csvText: string): TypingSession[];
|
|
55
|
+
/**
|
|
56
|
+
* Parse a single Balabit Mouse Dynamics session CSV.
|
|
57
|
+
*
|
|
58
|
+
* Each row is one mouse event:
|
|
59
|
+
* record_timestamp,client_timestamp,button,state,x,y
|
|
60
|
+
* 1424866316.93,0.000,NoButton,Move,1192,529
|
|
61
|
+
* 1424866316.99,0.064,NoButton,Move,1183,532
|
|
62
|
+
* ...
|
|
63
|
+
* 1424866317.84,0.911,Left,Pressed,820,440
|
|
64
|
+
*
|
|
65
|
+
* We segment into movements: each contiguous run of `Move`/`Drag` rows
|
|
66
|
+
* followed by a `Pressed` row becomes one `MouseMovement`. The press row
|
|
67
|
+
* is the click target. Movements that don't end in a press are also
|
|
68
|
+
* recorded but with `endedWithClick: false` — these are navigation moves.
|
|
69
|
+
*
|
|
70
|
+
* Time is normalized to milliseconds since the first sample of the
|
|
71
|
+
* movement.
|
|
72
|
+
*/
|
|
73
|
+
export declare function parseBalabitMouseCsv(csvText: string): MouseMovement[];
|
|
74
|
+
/**
|
|
75
|
+
* Generic loader for users who already export their telemetry in the
|
|
76
|
+
* normalized shapes. Accepts JSON of the form:
|
|
77
|
+
*
|
|
78
|
+
* { "movements": MouseMovement[], "sessions": TypingSession[] }
|
|
79
|
+
*
|
|
80
|
+
* Either field may be absent. Returns the parsed object with empty
|
|
81
|
+
* defaults filled in. This is the "bring your own data" path — most
|
|
82
|
+
* production users will write a tiny exporter on their own telemetry
|
|
83
|
+
* pipeline that emits this shape, then feed it through `fitFromSamples()`.
|
|
84
|
+
*/
|
|
85
|
+
export declare function parseGenericTelemetryJson(jsonText: string): {
|
|
86
|
+
movements: MouseMovement[];
|
|
87
|
+
sessions: TypingSession[];
|
|
88
|
+
};
|
|
89
|
+
//# sourceMappingURL=parsers.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parsers.d.ts","sourceRoot":"","sources":["../../src/behavioral/parsers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,KAAK,EAAmB,aAAa,EAAe,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAInG,mFAAmF;AACnF,eAAO,MAAM,UAAU,eAAe,CAAC;AAEvC;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,aAAa,EAAE,CAoDrE;AAoBD;;;;;;;;;;;;;;;;;GAiBG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,GAAG,aAAa,EAAE,CA0DrE;AAID;;;;;;;;;;GAUG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,MAAM,GAAG;IAC3D,SAAS,EAAE,aAAa,EAAE,CAAC;IAC3B,QAAQ,EAAE,aAAa,EAAE,CAAC;CAC3B,CASA"}
|
|
@@ -0,0 +1,223 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Dataset parsers for behavioral calibration.
|
|
3
|
+
*
|
|
4
|
+
* The calibration scaffold (`./calibration.ts`) defines normalized
|
|
5
|
+
* `MouseMovement` and `TypingSession` shapes plus distribution fitters.
|
|
6
|
+
* What was missing through v0.2.0 was the bridge from real public datasets
|
|
7
|
+
* into those shapes — without parsers, the scaffold required every user
|
|
8
|
+
* to write their own ETL, which kept "Tier 2 behavioral calibration" on
|
|
9
|
+
* the deferred list.
|
|
10
|
+
*
|
|
11
|
+
* This module ships parsers for the two datasets we point users at
|
|
12
|
+
* most often:
|
|
13
|
+
*
|
|
14
|
+
* 1. **CMU Keystroke Dynamics** (Killourhy & Maxion, 2009).
|
|
15
|
+
* Free for research. CSV format with columns `subject`, `sessionIndex`,
|
|
16
|
+
* `rep`, then for the fixed phrase `.tie5Roanl` a tuple of
|
|
17
|
+
* `H.<key>` (hold), `DD.<k1>.<k2>` (down-down latency), and
|
|
18
|
+
* `UD.<k1>.<k2>` (up-down = flight time). 51 subjects × 8 sessions
|
|
19
|
+
* × 50 repetitions = 20,400 phrases.
|
|
20
|
+
*
|
|
21
|
+
* 2. **Balabit Mouse Dynamics Challenge** (Antal & Egyed-Zsigmond, 2014).
|
|
22
|
+
* Free for research. Per-session CSV with columns
|
|
23
|
+
* `record_timestamp`, `client_timestamp`, `button`, `state`, `x`, `y`.
|
|
24
|
+
* Each row is a single mouse event; consecutive `Mouse` rows
|
|
25
|
+
* followed by a `Pressed` row form one movement.
|
|
26
|
+
*
|
|
27
|
+
* Plus a generic JSON loader for users with their own telemetry exported
|
|
28
|
+
* in the `MouseMovement` / `TypingSession` shapes directly.
|
|
29
|
+
*
|
|
30
|
+
* The parsers do NOT download the datasets — both have ToS that say "do
|
|
31
|
+
* not redistribute". Users acquire the data themselves and feed the file
|
|
32
|
+
* contents in as a string. We just turn raw text into normalized samples.
|
|
33
|
+
*/
|
|
34
|
+
// ── CMU Keystroke Dynamics ──
|
|
35
|
+
/** The fixed phrase typed by every CMU subject. Used to map column index → key. */
|
|
36
|
+
export const CMU_PHRASE = '.tie5Roanl';
|
|
37
|
+
/**
|
|
38
|
+
* Parse the CMU Keystroke Dynamics CSV (DSL-StrongPasswordData.csv).
|
|
39
|
+
*
|
|
40
|
+
* Each row is one repetition of the phrase `.tie5Roanl` plus Return,
|
|
41
|
+
* yielding 11 keys, 11 hold-times, 10 down-down and 10 up-down latencies.
|
|
42
|
+
* We normalize each row into one `TypingSession` of 11 keystrokes.
|
|
43
|
+
*
|
|
44
|
+
* The CSV looks like:
|
|
45
|
+
* subject,sessionIndex,rep,H.period,DD.period.t,UD.period.t,H.t,DD.t.i,UD.t.i,H.i,...
|
|
46
|
+
* s002,1,1,0.1491,0.3979,0.2488,0.1069,0.1674,0.0605,...
|
|
47
|
+
*
|
|
48
|
+
* All time values are in **seconds** in the source file; we convert to
|
|
49
|
+
* milliseconds.
|
|
50
|
+
*
|
|
51
|
+
* Returns one `TypingSession` per CSV row. Pass these straight into
|
|
52
|
+
* `fitTypingDynamics()`.
|
|
53
|
+
*/
|
|
54
|
+
export function parseCmuKeystrokeCsv(csvText) {
|
|
55
|
+
const lines = csvText.trim().split(/\r?\n/);
|
|
56
|
+
if (lines.length < 2)
|
|
57
|
+
return [];
|
|
58
|
+
const header = lines[0].split(',').map((h) => h.trim());
|
|
59
|
+
// Build column index maps for the H.<key> (hold) and UD.<k1>.<k2> (flight) columns.
|
|
60
|
+
// We use UD (up-down) as flight time: time from previous key release to next key down.
|
|
61
|
+
const holdIdx = new Map(); // keyIndex → column
|
|
62
|
+
const flightIdx = new Map(); // keyIndex (1..n-1) → column
|
|
63
|
+
// The phrase keys, in order. The CMU dataset includes Return at the
|
|
64
|
+
// end, so the full key sequence is `.tie5Roanl` plus Enter (column
|
|
65
|
+
// labeled `Return` in the source).
|
|
66
|
+
const keys = [...CMU_PHRASE.split(''), 'Return'];
|
|
67
|
+
for (let i = 0; i < keys.length; i++) {
|
|
68
|
+
const key = cmuKeyLabel(keys[i]);
|
|
69
|
+
const colName = `H.${key}`;
|
|
70
|
+
const idx = header.indexOf(colName);
|
|
71
|
+
if (idx >= 0)
|
|
72
|
+
holdIdx.set(i, idx);
|
|
73
|
+
}
|
|
74
|
+
for (let i = 1; i < keys.length; i++) {
|
|
75
|
+
const prev = cmuKeyLabel(keys[i - 1]);
|
|
76
|
+
const cur = cmuKeyLabel(keys[i]);
|
|
77
|
+
const colName = `UD.${prev}.${cur}`;
|
|
78
|
+
const idx = header.indexOf(colName);
|
|
79
|
+
if (idx >= 0)
|
|
80
|
+
flightIdx.set(i, idx);
|
|
81
|
+
}
|
|
82
|
+
const sessions = [];
|
|
83
|
+
for (let rowIdx = 1; rowIdx < lines.length; rowIdx++) {
|
|
84
|
+
const cells = lines[rowIdx].split(',');
|
|
85
|
+
if (cells.length < 4)
|
|
86
|
+
continue;
|
|
87
|
+
const keystrokes = [];
|
|
88
|
+
for (let i = 0; i < keys.length; i++) {
|
|
89
|
+
const holdCol = holdIdx.get(i);
|
|
90
|
+
const flightCol = flightIdx.get(i);
|
|
91
|
+
const holdSec = holdCol != null ? parseFloat(cells[holdCol] ?? '') : NaN;
|
|
92
|
+
const flightSec = i === 0 ? 0 : flightCol != null ? parseFloat(cells[flightCol] ?? '') : NaN;
|
|
93
|
+
if (!Number.isFinite(holdSec))
|
|
94
|
+
continue;
|
|
95
|
+
keystrokes.push({
|
|
96
|
+
key: keys[i],
|
|
97
|
+
flightTimeMs: Number.isFinite(flightSec) ? Math.max(0, flightSec * 1000) : 0,
|
|
98
|
+
holdTimeMs: Math.max(0, holdSec * 1000),
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
if (keystrokes.length > 0) {
|
|
102
|
+
sessions.push({ keystrokes, phrase: CMU_PHRASE });
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return sessions;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* CMU's CSV uses specific labels for non-letter keys:
|
|
109
|
+
* - `.` → `period`
|
|
110
|
+
* - `5` → `five` (the dataset spells digits out)
|
|
111
|
+
* - `R` (capital, requires Shift) → `Shift.r`
|
|
112
|
+
* - Return → `Return`
|
|
113
|
+
* - lowercase letters are bare
|
|
114
|
+
*/
|
|
115
|
+
function cmuKeyLabel(ch) {
|
|
116
|
+
if (ch === '.')
|
|
117
|
+
return 'period';
|
|
118
|
+
if (ch === '5')
|
|
119
|
+
return 'five';
|
|
120
|
+
if (ch === 'Return')
|
|
121
|
+
return 'Return';
|
|
122
|
+
if (ch === 'R')
|
|
123
|
+
return 'Shift.r';
|
|
124
|
+
return ch;
|
|
125
|
+
}
|
|
126
|
+
// ── Balabit Mouse Dynamics ──
|
|
127
|
+
/**
|
|
128
|
+
* Parse a single Balabit Mouse Dynamics session CSV.
|
|
129
|
+
*
|
|
130
|
+
* Each row is one mouse event:
|
|
131
|
+
* record_timestamp,client_timestamp,button,state,x,y
|
|
132
|
+
* 1424866316.93,0.000,NoButton,Move,1192,529
|
|
133
|
+
* 1424866316.99,0.064,NoButton,Move,1183,532
|
|
134
|
+
* ...
|
|
135
|
+
* 1424866317.84,0.911,Left,Pressed,820,440
|
|
136
|
+
*
|
|
137
|
+
* We segment into movements: each contiguous run of `Move`/`Drag` rows
|
|
138
|
+
* followed by a `Pressed` row becomes one `MouseMovement`. The press row
|
|
139
|
+
* is the click target. Movements that don't end in a press are also
|
|
140
|
+
* recorded but with `endedWithClick: false` — these are navigation moves.
|
|
141
|
+
*
|
|
142
|
+
* Time is normalized to milliseconds since the first sample of the
|
|
143
|
+
* movement.
|
|
144
|
+
*/
|
|
145
|
+
export function parseBalabitMouseCsv(csvText) {
|
|
146
|
+
const lines = csvText.trim().split(/\r?\n/);
|
|
147
|
+
if (lines.length < 2)
|
|
148
|
+
return [];
|
|
149
|
+
// Header detection — Balabit ships either with or without a header row.
|
|
150
|
+
let startIdx = 0;
|
|
151
|
+
if (/[A-Za-z]/.test(lines[0].split(',')[0] ?? ''))
|
|
152
|
+
startIdx = 1;
|
|
153
|
+
const movements = [];
|
|
154
|
+
let current = [];
|
|
155
|
+
let currentStart = null;
|
|
156
|
+
const flush = (endedWithClick) => {
|
|
157
|
+
if (current.length >= 2) {
|
|
158
|
+
movements.push({ samples: current, endedWithClick });
|
|
159
|
+
}
|
|
160
|
+
current = [];
|
|
161
|
+
currentStart = null;
|
|
162
|
+
};
|
|
163
|
+
for (let i = startIdx; i < lines.length; i++) {
|
|
164
|
+
const cells = lines[i].split(',');
|
|
165
|
+
if (cells.length < 6)
|
|
166
|
+
continue;
|
|
167
|
+
const clientTs = parseFloat(cells[1] ?? '');
|
|
168
|
+
const state = (cells[3] ?? '').trim();
|
|
169
|
+
const x = parseFloat(cells[4] ?? '');
|
|
170
|
+
const y = parseFloat(cells[5] ?? '');
|
|
171
|
+
if (!Number.isFinite(clientTs) || !Number.isFinite(x) || !Number.isFinite(y))
|
|
172
|
+
continue;
|
|
173
|
+
const tMs = clientTs * 1000;
|
|
174
|
+
if (currentStart == null)
|
|
175
|
+
currentStart = tMs;
|
|
176
|
+
if (state === 'Move' || state === 'Drag') {
|
|
177
|
+
current.push({ timestampMs: tMs - currentStart, x, y });
|
|
178
|
+
}
|
|
179
|
+
else if (state === 'Pressed') {
|
|
180
|
+
// Treat the press location as the target.
|
|
181
|
+
current.push({
|
|
182
|
+
timestampMs: tMs - currentStart,
|
|
183
|
+
x,
|
|
184
|
+
y,
|
|
185
|
+
targetX: x,
|
|
186
|
+
targetY: y,
|
|
187
|
+
// Balabit doesn't ship target widths — leave undefined so the
|
|
188
|
+
// Fitts' Law fitter skips these and falls back to canonical values.
|
|
189
|
+
});
|
|
190
|
+
flush(true);
|
|
191
|
+
}
|
|
192
|
+
else if (state === 'Released') {
|
|
193
|
+
// End-of-click; ignore.
|
|
194
|
+
}
|
|
195
|
+
else {
|
|
196
|
+
// Scroll, etc. — terminate any in-flight movement.
|
|
197
|
+
flush(false);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
// Flush trailing movement if any
|
|
201
|
+
flush(false);
|
|
202
|
+
return movements;
|
|
203
|
+
}
|
|
204
|
+
// ── Generic JSON loader ──
|
|
205
|
+
/**
|
|
206
|
+
* Generic loader for users who already export their telemetry in the
|
|
207
|
+
* normalized shapes. Accepts JSON of the form:
|
|
208
|
+
*
|
|
209
|
+
* { "movements": MouseMovement[], "sessions": TypingSession[] }
|
|
210
|
+
*
|
|
211
|
+
* Either field may be absent. Returns the parsed object with empty
|
|
212
|
+
* defaults filled in. This is the "bring your own data" path — most
|
|
213
|
+
* production users will write a tiny exporter on their own telemetry
|
|
214
|
+
* pipeline that emits this shape, then feed it through `fitFromSamples()`.
|
|
215
|
+
*/
|
|
216
|
+
export function parseGenericTelemetryJson(jsonText) {
|
|
217
|
+
const parsed = JSON.parse(jsonText);
|
|
218
|
+
return {
|
|
219
|
+
movements: Array.isArray(parsed.movements) ? parsed.movements : [],
|
|
220
|
+
sessions: Array.isArray(parsed.sessions) ? parsed.sessions : [],
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
//# sourceMappingURL=parsers.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parsers.js","sourceRoot":"","sources":["../../src/behavioral/parsers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAIH,+BAA+B;AAE/B,mFAAmF;AACnF,MAAM,CAAC,MAAM,UAAU,GAAG,YAAY,CAAC;AAEvC;;;;;;;;;;;;;;;;GAgBG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAe;IAClD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC5C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IAEhC,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACzD,oFAAoF;IACpF,uFAAuF;IACvF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC,CAAC,oBAAoB;IAC/D,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC,CAAC,6BAA6B;IAE1E,oEAAoE;IACpE,mEAAmE;IACnE,mCAAmC;IACnC,MAAM,IAAI,GAAG,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC;IACjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAC;QAClC,MAAM,OAAO,GAAG,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,GAAG,IAAI,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACpC,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,CAAC;QACvC,MAAM,GAAG,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,CAAC;QAClC,MAAM,OAAO,GAAG,MAAM,IAAI,IAAI,GAAG,EAAE,CAAC;QACpC,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACpC,IAAI,GAAG,IAAI,CAAC;YAAE,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;IACtC,CAAC;IAED,MAAM,QAAQ,GAAoB,EAAE,CAAC;IACrC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;QACrD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAE/B,MAAM,UAAU,GAAsB,EAAE,CAAC;QACzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,SAAS,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACnC,MAAM,OAAO,GAAG,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;YACzE,MAAM,SAAS,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;YAC7F,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC;gBAAE,SAAS;YAExC,UAAU,CAAC,IAAI,CAAC;gBACd,GAAG,EAAE,IAAI,CAAC,CAAC,CAAE;gBACb,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC5E,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,IAAI,CAAC;aACxC,CAAC,CAAC;QACL,CAAC;QACD,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;;;GAOG;AACH,SAAS,WAAW,CAAC,EAAU;IAC7B,IAAI,EAAE,KAAK,GAAG;QAAE,OAAO,QAAQ,CAAC;IAChC,IAAI,EAAE,KAAK,GAAG;QAAE,OAAO,MAAM,CAAC;IAC9B,IAAI,EAAE,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IACrC,IAAI,EAAE,KAAK,GAAG;QAAE,OAAO,SAAS,CAAC;IACjC,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,+BAA+B;AAE/B;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAe;IAClD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC5C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IAEhC,wEAAwE;IACxE,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAAE,QAAQ,GAAG,CAAC,CAAC;IAEjE,MAAM,SAAS,GAAoB,EAAE,CAAC;IACtC,IAAI,OAAO,GAAkB,EAAE,CAAC;IAChC,IAAI,YAAY,GAAkB,IAAI,CAAC;IAEvC,MAAM,KAAK,GAAG,CAAC,cAAuB,EAAQ,EAAE;QAC9C,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACxB,SAAS,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,GAAG,EAAE,CAAC;QACb,YAAY,GAAG,IAAI,CAAC;IACtB,CAAC,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,QAAQ,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACnC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAS;QAE/B,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QAC5C,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACrC,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;YAAE,SAAS;QAEvF,MAAM,GAAG,GAAG,QAAQ,GAAG,IAAI,CAAC;QAC5B,IAAI,YAAY,IAAI,IAAI;YAAE,YAAY,GAAG,GAAG,CAAC;QAE7C,IAAI,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,MAAM,EAAE,CAAC;YACzC,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,GAAG,GAAG,YAAY,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1D,CAAC;aAAM,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;YAC/B,0CAA0C;YAC1C,OAAO,CAAC,IAAI,CAAC;gBACX,WAAW,EAAE,GAAG,GAAG,YAAY;gBAC/B,CAAC;gBACD,CAAC;gBACD,OAAO,EAAE,CAAC;gBACV,OAAO,EAAE,CAAC;gBACV,8DAA8D;gBAC9D,oEAAoE;aACrE,CAAC,CAAC;YACH,KAAK,CAAC,IAAI,CAAC,CAAC;QACd,CAAC;aAAM,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;YAChC,wBAAwB;QAC1B,CAAC;aAAM,CAAC;YACN,mDAAmD;YACnD,KAAK,CAAC,KAAK,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IACD,iCAAiC;IACjC,KAAK,CAAC,KAAK,CAAC,CAAC;IAEb,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,4BAA4B;AAE5B;;;;;;;;;;GAUG;AACH,MAAM,UAAU,yBAAyB,CAAC,QAAgB;IAIxD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAGjC,CAAC;IACF,OAAO;QACL,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;QAClE,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE;KAChE,CAAC;AACJ,CAAC"}
|
package/dist/blacktip.d.ts
CHANGED
|
@@ -4,7 +4,9 @@ import type { Frame, ElementHandle as PlaywrightElementHandle } from 'patchright
|
|
|
4
4
|
import { BehavioralEngine } from './behavioral-engine.js';
|
|
5
5
|
import { ElementFinder } from './element-finder.js';
|
|
6
6
|
import { Logger } from './logging.js';
|
|
7
|
-
import { type FingerprintSnapshot, type IpReputationResult, type AkamaiTestResult } from './diagnostics.js';
|
|
7
|
+
import { type FingerprintSnapshot, type IpReputationResult, type AkamaiTestResult, type AntiBotTestResult } from './diagnostics.js';
|
|
8
|
+
import { type TlsRequest, type TlsResponse } from './tls-side-channel.js';
|
|
9
|
+
import { type AkamaiChallengeResult } from './akamai-sensor.js';
|
|
8
10
|
import type { BlackTipConfig, ProfileConfig, ActionResult, NavigateResult, ScreenshotResult, WaitResult, TabInfo, FrameInfo, ClickOptions, TypeOptions, ScrollOptions, HoverOptions, SelectOptions, PressKeyOptions, UploadFileOptions, NavigateOptions, ScreenshotOptions, WaitForOptions, WaitForNavigationOptions, ExtractTextOptions, PageContentOptions } from './types.js';
|
|
9
11
|
/**
|
|
10
12
|
* BlackTip — Stealth browser instrument for AI agents.
|
|
@@ -54,6 +56,7 @@ export declare class BlackTip extends EventEmitter {
|
|
|
54
56
|
private core;
|
|
55
57
|
private engine;
|
|
56
58
|
private finder;
|
|
59
|
+
private tlsChannel;
|
|
57
60
|
private logger;
|
|
58
61
|
private config;
|
|
59
62
|
private customProfiles;
|
|
@@ -232,6 +235,70 @@ export declare class BlackTip extends EventEmitter {
|
|
|
232
235
|
* trying to figure out why a specific target is blocking you.
|
|
233
236
|
*/
|
|
234
237
|
testAgainstAkamai(url: string): Promise<AkamaiTestResult>;
|
|
238
|
+
/**
|
|
239
|
+
* Multi-vendor anti-bot probe. Recognises Akamai, DataDome, Cloudflare,
|
|
240
|
+
* PerimeterX/HUMAN, Imperva, Kasada, and Arkose. Reports both the
|
|
241
|
+
* vendors that served a block/challenge AND the vendor signals (cookies,
|
|
242
|
+
* scripts) present on a passing page — so you can verify the target is
|
|
243
|
+
* actually protected and BlackTip is sliding past it, not a false negative
|
|
244
|
+
* on an unprotected URL.
|
|
245
|
+
*/
|
|
246
|
+
testAgainstAntiBot(url: string): Promise<AntiBotTestResult>;
|
|
247
|
+
/**
|
|
248
|
+
* Return the TLS rewriter stats — intercepted/fulfilled/fell-through
|
|
249
|
+
* counts, WebSocket leaks, average daemon round-trip. Null when
|
|
250
|
+
* `BlackTipConfig.tlsRewriting` is `'off'` (the default).
|
|
251
|
+
*
|
|
252
|
+
* Use this to verify the rewriter is doing what you expect:
|
|
253
|
+
* - `intercepted > 0` confirms requests are being captured
|
|
254
|
+
* - `fulfilled === intercepted - webSocketLeaks` confirms no fallthroughs
|
|
255
|
+
* - `fellThrough > 0` indicates daemon failures (check daemon stderr)
|
|
256
|
+
*/
|
|
257
|
+
getTlsRewriterStats(): import("./tls-rewriter.js").TlsRewriterStats | null;
|
|
258
|
+
/**
|
|
259
|
+
* Drive this BlackTip session through Akamai's sensor challenge for
|
|
260
|
+
* `url`, waiting until the `_abck` cookie reaches a validated state.
|
|
261
|
+
* Returns the validated cookies for injection into other sessions
|
|
262
|
+
* (TLS daemon flows, IdentityPool snapshots, separate BlackTip
|
|
263
|
+
* instances).
|
|
264
|
+
*
|
|
265
|
+
* This is the v0.5.0 path for "make Akamai-protected API calls from a
|
|
266
|
+
* sessionless TLS daemon" — solve the challenge once via this method,
|
|
267
|
+
* cache the cookies, then run hundreds of `bt.fetchWithTls()` calls
|
|
268
|
+
* with the cached `Cookie` header until the session expires (~1h).
|
|
269
|
+
*
|
|
270
|
+
* NOT a pure-Go solver. Real Chrome runs the bm.js. We just centralize
|
|
271
|
+
* the browser usage to one primitive so the caller doesn't have to
|
|
272
|
+
* launch a full session per API call. See `docs/akamai-sensor.md` for
|
|
273
|
+
* the architecture rationale.
|
|
274
|
+
*/
|
|
275
|
+
solveAkamaiChallenge(url: string, options?: {
|
|
276
|
+
timeoutMs?: number;
|
|
277
|
+
pollIntervalMs?: number;
|
|
278
|
+
dwellMsBeforePolling?: number;
|
|
279
|
+
}): Promise<AkamaiChallengeResult>;
|
|
280
|
+
/**
|
|
281
|
+
* Perform an HTTP request through the bogdanfinn/tls-client Go daemon
|
|
282
|
+
* with a real Chrome TLS ClientHello, real H2 frame settings, and real
|
|
283
|
+
* H2 frame order. Lazily spawns the daemon on first call and reuses it
|
|
284
|
+
* across subsequent calls in the same BlackTip session.
|
|
285
|
+
*
|
|
286
|
+
* Use this when an edge gates the very first request before BlackTip's
|
|
287
|
+
* browser has a usable session — make the gating request via the side
|
|
288
|
+
* channel, then call `bt.injectTlsCookies(resp)` to push the resulting
|
|
289
|
+
* cookies into the browser session before navigating.
|
|
290
|
+
*
|
|
291
|
+
* Requires the Go daemon binary at `native/tls-client/blacktip-tls[.exe]`.
|
|
292
|
+
* Build it once with: `cd native/tls-client && go build .`
|
|
293
|
+
*/
|
|
294
|
+
fetchWithTls(req: TlsRequest): Promise<TlsResponse>;
|
|
295
|
+
/**
|
|
296
|
+
* Inject cookies returned by `fetchWithTls()` into the browser session
|
|
297
|
+
* so a subsequent `bt.navigate()` carries the same edge-issued tokens
|
|
298
|
+
* the gating request earned. The cookies are filtered to those whose
|
|
299
|
+
* domain matches the target URL's eTLD+1.
|
|
300
|
+
*/
|
|
301
|
+
injectTlsCookies(resp: TlsResponse, targetUrl?: string): Promise<number>;
|
|
235
302
|
/**
|
|
236
303
|
* Visit a sequence of "normal" sites with realistic dwell times before
|
|
237
304
|
* the target navigation. Accumulates cookies, populates History API,
|
package/dist/blacktip.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"blacktip.d.ts","sourceRoot":"","sources":["../src/blacktip.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAgB,KAAK,MAAM,EAAE,MAAM,UAAU,CAAC;AACrD,OAAO,KAAK,EAAE,KAAK,EAAE,aAAa,IAAI,uBAAuB,EAAE,MAAM,YAAY,CAAC;AAElF,OAAO,EAAE,gBAAgB,EAAkC,MAAM,wBAAwB,CAAC;AAE1F,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AACtC,OAAO,
|
|
1
|
+
{"version":3,"file":"blacktip.d.ts","sourceRoot":"","sources":["../src/blacktip.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAgB,KAAK,MAAM,EAAE,MAAM,UAAU,CAAC;AACrD,OAAO,KAAK,EAAE,KAAK,EAAE,aAAa,IAAI,uBAAuB,EAAE,MAAM,YAAY,CAAC;AAElF,OAAO,EAAE,gBAAgB,EAAkC,MAAM,wBAAwB,CAAC;AAE1F,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AACtC,OAAO,EAKL,KAAK,mBAAmB,EACxB,KAAK,kBAAkB,EACvB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACvB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAkB,KAAK,UAAU,EAAE,KAAK,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAC1F,OAAO,EAAoD,KAAK,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAClH,OAAO,KAAK,EACV,cAAc,EACd,aAAa,EACb,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,UAAU,EAKV,OAAO,EACP,SAAS,EAET,YAAY,EACZ,WAAW,EACX,aAAa,EACb,YAAY,EACZ,aAAa,EACb,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,iBAAiB,EACjB,cAAc,EACd,wBAAwB,EACxB,kBAAkB,EAClB,kBAAkB,EAInB,MAAM,YAAY,CAAC;AA4CpB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2CG;AACH,qBAAa,QAAS,SAAQ,YAAY;IACxC,OAAO,CAAC,IAAI,CAAc;IAC1B,OAAO,CAAC,MAAM,CAAmB;IACjC,OAAO,CAAC,MAAM,CAAgB;IAC9B,OAAO,CAAC,UAAU,CAA+B;IACjD,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,MAAM,CAAiB;IAC/B,OAAO,CAAC,cAAc,CAAoC;IAC1D,OAAO,CAAC,QAAQ,CAAS;gBAEb,MAAM,GAAE,cAAmB;IAkCvC;;;OAGG;IACG,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC;IAoCzB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAS5B,QAAQ,IAAI,OAAO;IAMb,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC;IASzE,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAyGtE,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG;QAAE,KAAK,CAAC,EAAE,OAAO,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,YAAY,CAAC;IAqE1G,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,YAAY,CAAC;IA+DxG,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC;IAmGlF,MAAM,CAAC,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IA4BtD,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAuBtE,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,YAAY,CAAC;IAqCvF,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC;IAWvE,UAAU,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,YAAY,CAAC;IAgBxG;;;;;;;OAOG;IACG,QAAQ,CACZ,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAC5C,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,iBAAiB,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IA6B5E,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;IAkBvF,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC;IAOnF;;;;;OAKG;IACG,eAAe,CAAC,WAAW,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAMtG,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;IA6BjE,cAAc,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,MAAM,CAAC;IAO7D,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC;IAqBxE,iBAAiB,CAAC,OAAO,CAAC,EAAE,wBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC;IAUpE,UAAU,CAAC,OAAO,CAAC,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAOxE;;;;;;;;OAQG;IACG,aAAa,CAAC,OAAO,GAAE;QAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;KACZ,GAAG,OAAO,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAwDxD;;;;OAIG;IACG,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,GAAE;QAAE,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAO,GAAG,OAAO,CAAC;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,CAAC;IAoBrI;;;;OAIG;IACG,OAAO,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC;QACvC,MAAM,EAAE,OAAO,CAAC;QAChB,OAAO,EAAE,OAAO,CAAC;QACjB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACpC,WAAW,CAAC,EAAE;YAAE,CAAC,EAAE,MAAM,CAAC;YAAC,CAAC,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,MAAM,CAAA;SAAE,CAAC;KACvE,CAAC;IAoCF;;;;;;;;OAQG;IACG,WAAW,CAAC,sBAAsB,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAwB1F;;;OAGG;IACG,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IA8BlI;;;;OAIG;IACG,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAKtF;;;;;OAKG;IACG,eAAe,IAAI,OAAO,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC;IAkCzE;;;;;;;;;;OAUG;IACG,kBAAkB,IAAI,OAAO,CAAC,mBAAmB,CAAC;IAKxD;;;;;;;;OAQG;IACG,iBAAiB,IAAI,OAAO,CAAC,kBAAkB,CAAC;IAKtD;;;;;;;OAOG;IACG,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAK/D;;;;;;;OAOG;IACG,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAKjE;;;;;;;;;OASG;IACH,mBAAmB;IAInB;;;;;;;;;;;;;;;;OAgBG;IACG,oBAAoB,CACxB,GAAG,EAAE,MAAM,EACX,OAAO,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,MAAM,CAAC;QAAC,oBAAoB,CAAC,EAAE,MAAM,CAAA;KAAE,GACvF,OAAO,CAAC,qBAAqB,CAAC;IAOjC;;;;;;;;;;;;;OAaG;IACG,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,WAAW,CAAC;IAOzD;;;;;OAKG;IACG,gBAAgB,CAAC,IAAI,EAAE,WAAW,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAuB9E;;;;;;;;;;;;;OAaG;IACG,WAAW,CAAC,OAAO,GAAE;QACzB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;QACjB,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;KAC5B,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAC;IAmCrD,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC;IAM/C,MAAM,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;IAO9B,OAAO,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IAI7B,MAAM,CAAC,GAAG,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAIrC,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIvC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAMvC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3B,OAAO;;;;;;IAIP,UAAU,CAAC,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE;IAIjG,YAAY;IAMZ,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAOjD,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,GAAG,IAAI;IAKjE,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa;IAIvC,YAAY,IAAI,MAAM,EAAE;IAIxB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IASjC;;;OAGG;IACH,MAAM,CAAC,UAAU,IAAI,MAAM;WAqEd,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAY9E;;;;;;;OAOG;IACG,KAAK,CAAC,IAAI,SAAO,EAAE,cAAc,SAAa,GAAG,OAAO,CAAC,MAAM,CAAC;IA6JtE;;;;;;;;;;;;OAYG;IACG,aAAa,CAAC,OAAO,EAAE;QAC3B,MAAM,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC;QAC7C,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,GAAG,OAAO,CAAC,MAAM,CAAC;YAyCL,aAAa;YA+Gb,kBAAkB;IAuChC,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,MAAM,CAAK;YAEL,gBAAgB;YAIhB,gBAAgB;IA8B9B,OAAO,CAAC,cAAc;IAetB,OAAO,CAAC,cAAc;IAMtB,OAAO,CAAC,KAAK;CAGd;AAED;;GAEG;AACH,qBAAa,aAAa;IAEtB,OAAO,CAAC,KAAK;IACb,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;gBAHN,KAAK,EAAE,KAAK,EACZ,MAAM,EAAE,gBAAgB,EACxB,MAAM,EAAE,aAAa,EACrB,MAAM,EAAE,MAAM;IAGlB,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAuBtE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC;IA2ElF,WAAW,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAK9C,KAAK,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAS9C,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAM9D,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,UAAU,CAAC;IAiB9E,OAAO,CAAC,KAAK;CAGd"}
|
package/dist/blacktip.js
CHANGED
|
@@ -4,7 +4,9 @@ import { BrowserCore } from './browser-core.js';
|
|
|
4
4
|
import { BehavioralEngine, HUMAN_PROFILE, SCRAPER_PROFILE } from './behavioral-engine.js';
|
|
5
5
|
import { ElementFinder } from './element-finder.js';
|
|
6
6
|
import { Logger } from './logging.js';
|
|
7
|
-
import { captureFingerprint as diagnosticsCaptureFingerprint, checkIpReputation as diagnosticsCheckIpReputation, testAgainstAkamai as diagnosticsTestAgainstAkamai, } from './diagnostics.js';
|
|
7
|
+
import { captureFingerprint as diagnosticsCaptureFingerprint, checkIpReputation as diagnosticsCheckIpReputation, testAgainstAkamai as diagnosticsTestAgainstAkamai, testAgainstAntiBot as diagnosticsTestAgainstAntiBot, } from './diagnostics.js';
|
|
8
|
+
import { TlsSideChannel } from './tls-side-channel.js';
|
|
9
|
+
import { solveAkamaiChallenge as solveAkamaiChallengeImpl } from './akamai-sensor.js';
|
|
8
10
|
const RETRY_STRATEGIES = ['standard', 'wait', 'reload', 'altSelector', 'scroll', 'clearOverlays'];
|
|
9
11
|
/**
|
|
10
12
|
* Text patterns that suggest an action is high-importance — submit,
|
|
@@ -91,6 +93,7 @@ export class BlackTip extends EventEmitter {
|
|
|
91
93
|
core;
|
|
92
94
|
engine;
|
|
93
95
|
finder;
|
|
96
|
+
tlsChannel = null;
|
|
94
97
|
logger;
|
|
95
98
|
config;
|
|
96
99
|
customProfiles = new Map();
|
|
@@ -128,9 +131,44 @@ export class BlackTip extends EventEmitter {
|
|
|
128
131
|
async launch() {
|
|
129
132
|
await this.core.launch();
|
|
130
133
|
this.launched = true;
|
|
134
|
+
// IP reputation gate (v0.4.0). When `requireResidentialIp` is set,
|
|
135
|
+
// we run the same check as `bt.checkIpReputation()` and either warn
|
|
136
|
+
// or throw based on the verdict. The check itself navigates to
|
|
137
|
+
// ipinfo.io, so we do it after launch is complete.
|
|
138
|
+
const gate = this.config.requireResidentialIp;
|
|
139
|
+
if (gate) {
|
|
140
|
+
try {
|
|
141
|
+
const verdict = await diagnosticsCheckIpReputation(this);
|
|
142
|
+
if (verdict.isDatacenter) {
|
|
143
|
+
const msg = `IP reputation gate: egress IP ${verdict.ip} (${verdict.org}) is on a known datacenter ASN. ${verdict.notes.join(' ')}`;
|
|
144
|
+
if (gate === 'warn') {
|
|
145
|
+
this.logger.warn(msg);
|
|
146
|
+
}
|
|
147
|
+
else {
|
|
148
|
+
// 'throw' or boolean true
|
|
149
|
+
await this.core.close();
|
|
150
|
+
this.launched = false;
|
|
151
|
+
throw new Error(msg);
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
catch (err) {
|
|
156
|
+
// If the check itself fails (offline, etc.) and the gate is
|
|
157
|
+
// 'throw', we re-raise; if 'warn', we log and continue.
|
|
158
|
+
if (err instanceof Error && err.message.startsWith('IP reputation gate:'))
|
|
159
|
+
throw err;
|
|
160
|
+
if (gate !== 'warn') {
|
|
161
|
+
this.logger.warn(`IP reputation gate check failed (allowing launch): ${err instanceof Error ? err.message : String(err)}`);
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
}
|
|
131
165
|
return BlackTip.agentGuide();
|
|
132
166
|
}
|
|
133
167
|
async close() {
|
|
168
|
+
if (this.tlsChannel) {
|
|
169
|
+
await this.tlsChannel.close().catch(() => undefined);
|
|
170
|
+
this.tlsChannel = null;
|
|
171
|
+
}
|
|
134
172
|
await this.core.close();
|
|
135
173
|
this.launched = false;
|
|
136
174
|
}
|
|
@@ -942,6 +980,107 @@ export class BlackTip extends EventEmitter {
|
|
|
942
980
|
this.ensureLaunched();
|
|
943
981
|
return diagnosticsTestAgainstAkamai(this, url);
|
|
944
982
|
}
|
|
983
|
+
/**
|
|
984
|
+
* Multi-vendor anti-bot probe. Recognises Akamai, DataDome, Cloudflare,
|
|
985
|
+
* PerimeterX/HUMAN, Imperva, Kasada, and Arkose. Reports both the
|
|
986
|
+
* vendors that served a block/challenge AND the vendor signals (cookies,
|
|
987
|
+
* scripts) present on a passing page — so you can verify the target is
|
|
988
|
+
* actually protected and BlackTip is sliding past it, not a false negative
|
|
989
|
+
* on an unprotected URL.
|
|
990
|
+
*/
|
|
991
|
+
async testAgainstAntiBot(url) {
|
|
992
|
+
this.ensureLaunched();
|
|
993
|
+
return diagnosticsTestAgainstAntiBot(this, url);
|
|
994
|
+
}
|
|
995
|
+
/**
|
|
996
|
+
* Return the TLS rewriter stats — intercepted/fulfilled/fell-through
|
|
997
|
+
* counts, WebSocket leaks, average daemon round-trip. Null when
|
|
998
|
+
* `BlackTipConfig.tlsRewriting` is `'off'` (the default).
|
|
999
|
+
*
|
|
1000
|
+
* Use this to verify the rewriter is doing what you expect:
|
|
1001
|
+
* - `intercepted > 0` confirms requests are being captured
|
|
1002
|
+
* - `fulfilled === intercepted - webSocketLeaks` confirms no fallthroughs
|
|
1003
|
+
* - `fellThrough > 0` indicates daemon failures (check daemon stderr)
|
|
1004
|
+
*/
|
|
1005
|
+
getTlsRewriterStats() {
|
|
1006
|
+
return this.core.getTlsRewriterStats();
|
|
1007
|
+
}
|
|
1008
|
+
/**
|
|
1009
|
+
* Drive this BlackTip session through Akamai's sensor challenge for
|
|
1010
|
+
* `url`, waiting until the `_abck` cookie reaches a validated state.
|
|
1011
|
+
* Returns the validated cookies for injection into other sessions
|
|
1012
|
+
* (TLS daemon flows, IdentityPool snapshots, separate BlackTip
|
|
1013
|
+
* instances).
|
|
1014
|
+
*
|
|
1015
|
+
* This is the v0.5.0 path for "make Akamai-protected API calls from a
|
|
1016
|
+
* sessionless TLS daemon" — solve the challenge once via this method,
|
|
1017
|
+
* cache the cookies, then run hundreds of `bt.fetchWithTls()` calls
|
|
1018
|
+
* with the cached `Cookie` header until the session expires (~1h).
|
|
1019
|
+
*
|
|
1020
|
+
* NOT a pure-Go solver. Real Chrome runs the bm.js. We just centralize
|
|
1021
|
+
* the browser usage to one primitive so the caller doesn't have to
|
|
1022
|
+
* launch a full session per API call. See `docs/akamai-sensor.md` for
|
|
1023
|
+
* the architecture rationale.
|
|
1024
|
+
*/
|
|
1025
|
+
async solveAkamaiChallenge(url, options) {
|
|
1026
|
+
this.ensureLaunched();
|
|
1027
|
+
return solveAkamaiChallengeImpl(this, url, options);
|
|
1028
|
+
}
|
|
1029
|
+
// ── TLS side-channel (v0.3.0) ──
|
|
1030
|
+
/**
|
|
1031
|
+
* Perform an HTTP request through the bogdanfinn/tls-client Go daemon
|
|
1032
|
+
* with a real Chrome TLS ClientHello, real H2 frame settings, and real
|
|
1033
|
+
* H2 frame order. Lazily spawns the daemon on first call and reuses it
|
|
1034
|
+
* across subsequent calls in the same BlackTip session.
|
|
1035
|
+
*
|
|
1036
|
+
* Use this when an edge gates the very first request before BlackTip's
|
|
1037
|
+
* browser has a usable session — make the gating request via the side
|
|
1038
|
+
* channel, then call `bt.injectTlsCookies(resp)` to push the resulting
|
|
1039
|
+
* cookies into the browser session before navigating.
|
|
1040
|
+
*
|
|
1041
|
+
* Requires the Go daemon binary at `native/tls-client/blacktip-tls[.exe]`.
|
|
1042
|
+
* Build it once with: `cd native/tls-client && go build .`
|
|
1043
|
+
*/
|
|
1044
|
+
async fetchWithTls(req) {
|
|
1045
|
+
if (!this.tlsChannel) {
|
|
1046
|
+
this.tlsChannel = await TlsSideChannel.spawn();
|
|
1047
|
+
}
|
|
1048
|
+
return this.tlsChannel.fetch(req);
|
|
1049
|
+
}
|
|
1050
|
+
/**
|
|
1051
|
+
* Inject cookies returned by `fetchWithTls()` into the browser session
|
|
1052
|
+
* so a subsequent `bt.navigate()` carries the same edge-issued tokens
|
|
1053
|
+
* the gating request earned. The cookies are filtered to those whose
|
|
1054
|
+
* domain matches the target URL's eTLD+1.
|
|
1055
|
+
*/
|
|
1056
|
+
async injectTlsCookies(resp, targetUrl) {
|
|
1057
|
+
this.ensureLaunched();
|
|
1058
|
+
const targetHost = (() => {
|
|
1059
|
+
if (!targetUrl)
|
|
1060
|
+
return null;
|
|
1061
|
+
try {
|
|
1062
|
+
return new URL(targetUrl).hostname;
|
|
1063
|
+
}
|
|
1064
|
+
catch {
|
|
1065
|
+
return null;
|
|
1066
|
+
}
|
|
1067
|
+
})();
|
|
1068
|
+
const filtered = resp.cookies.filter((c) => {
|
|
1069
|
+
if (!targetHost)
|
|
1070
|
+
return true;
|
|
1071
|
+
const cookieDomain = c.domain.replace(/^\./, '');
|
|
1072
|
+
return targetHost === cookieDomain || targetHost.endsWith('.' + cookieDomain);
|
|
1073
|
+
});
|
|
1074
|
+
if (filtered.length === 0)
|
|
1075
|
+
return 0;
|
|
1076
|
+
await this.setCookies(filtered.map((c) => ({
|
|
1077
|
+
name: c.name,
|
|
1078
|
+
value: c.value,
|
|
1079
|
+
domain: c.domain.startsWith('.') ? c.domain : '.' + c.domain,
|
|
1080
|
+
path: c.path || '/',
|
|
1081
|
+
})));
|
|
1082
|
+
return filtered.length;
|
|
1083
|
+
}
|
|
945
1084
|
// ── Session warming (v0.2.0) ──
|
|
946
1085
|
/**
|
|
947
1086
|
* Visit a sequence of "normal" sites with realistic dwell times before
|