captchakraken 2.8.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (77) hide show
  1. package/AGENTS.md +112 -0
  2. package/NOTICE +36 -0
  3. package/dist/cli-invocation.d.ts +9 -40
  4. package/dist/cli-invocation.js +16 -49
  5. package/dist/errors.d.ts +7 -27
  6. package/dist/errors.js +6 -25
  7. package/dist/humanize.d.ts +56 -240
  8. package/dist/humanize.js +159 -415
  9. package/dist/index.d.ts +5 -129
  10. package/dist/index.js +4 -98
  11. package/dist/kinds.d.ts +135 -0
  12. package/dist/kinds.js +70 -0
  13. package/dist/limits.d.ts +0 -48
  14. package/dist/limits.js +2 -48
  15. package/dist/model-name.d.ts +4 -19
  16. package/dist/model-name.js +32 -40
  17. package/dist/playwright-types.d.ts +25 -95
  18. package/dist/playwright-types.js +0 -23
  19. package/dist/puppeteer-adapter.d.ts +4 -43
  20. package/dist/puppeteer-adapter.js +22 -58
  21. package/dist/python-command.d.ts +0 -41
  22. package/dist/python-command.js +0 -37
  23. package/dist/selectors.d.ts +54 -0
  24. package/dist/selectors.js +117 -0
  25. package/dist/slide-geometry.d.ts +2 -29
  26. package/dist/slide-geometry.js +18 -44
  27. package/dist/solver.d.ts +112 -539
  28. package/dist/solver.js +1088 -2950
  29. package/dist/timing.d.ts +7 -40
  30. package/dist/timing.js +4 -36
  31. package/dist/token-usage.d.ts +1 -7
  32. package/dist/token-usage.js +10 -95
  33. package/dist/types.d.ts +77 -483
  34. package/dist/watcher.d.ts +5 -78
  35. package/dist/watcher.js +0 -56
  36. package/package.json +12 -4
  37. package/python/AGENTS.md +122 -0
  38. package/python/NOTICE +36 -0
  39. package/python/README.md +1 -2
  40. package/python/examples/README.md +10 -0
  41. package/python/examples/_harness.py +4 -73
  42. package/python/examples/demoHcaptcha.py +0 -13
  43. package/python/examples/demoRecaptcha.py +0 -10
  44. package/python/examples/watch_playwright.py +1 -26
  45. package/python/examples/with_local_gguf.py +28 -0
  46. package/python/examples/with_playwright.py +0 -19
  47. package/python/pyproject.toml +68 -19
  48. package/python/src/captchakraken/__init__.py +37 -45
  49. package/python/src/captchakraken/action_types.py +14 -36
  50. package/python/src/captchakraken/cli.py +262 -879
  51. package/python/src/captchakraken/config.py +60 -146
  52. package/python/src/captchakraken/errors.py +17 -95
  53. package/python/src/captchakraken/humanize.py +174 -597
  54. package/python/src/captchakraken/image_processor.py +16 -235
  55. package/python/src/captchakraken/keyframes.py +69 -434
  56. package/python/src/captchakraken/kinds.py +152 -0
  57. package/python/src/captchakraken/models.json +79 -21
  58. package/python/src/captchakraken/overlay.py +65 -498
  59. package/python/src/captchakraken/page_solver.py +1123 -2564
  60. package/python/src/captchakraken/planner.py +70 -428
  61. package/python/src/captchakraken/prompts.py +45 -261
  62. package/python/src/captchakraken/selectors.py +184 -0
  63. package/python/src/captchakraken/server_manager.py +7 -58
  64. package/python/src/captchakraken/solver.py +128 -522
  65. package/python/src/captchakraken/timing.py +20 -74
  66. package/python/src/captchakraken/tool_calls/board_painted.py +47 -0
  67. package/python/src/captchakraken/tool_calls/find_checkbox.py +20 -30
  68. package/python/src/captchakraken/tool_calls/find_grid.py +186 -1339
  69. package/python/src/captchakraken/tool_calls/track_piece.py +81 -57
  70. package/python/src/captchakraken/updater.py +7 -76
  71. package/python/src/captchakraken/watcher.py +12 -101
  72. package/scripts/copy-python.mjs +10 -10
  73. package/scripts/setup-python.js +1 -21
  74. package/dist/trajectory.d.ts +0 -58
  75. package/dist/trajectory.js +0 -266
  76. package/python/src/captchakraken/tool_calls/move_indicator.py +0 -431
  77. package/python/src/captchakraken/trajectory.py +0 -284
package/dist/solver.d.ts CHANGED
@@ -1,643 +1,216 @@
1
- import { PlaywrightPage as Page, PlaywrightElementHandle as ElementHandle } from './playwright-types';
1
+ /**
2
+ * Browser page driver. Finds the widget, photographs it, asks the bundled Python engine what to do,
3
+ * and performs the answer. Mirrors python/src/captchakraken/page_solver.py.
4
+ */
5
+ import { PlaywrightPage as Page, PlaywrightElementHandle as ElementHandle, PlaywrightLocator as Locator } from './playwright-types';
2
6
  import { CaptchaWatcher, WatchOptions } from './watcher';
3
7
  import { PhaseBudget } from './timing';
4
8
  import { CaptchaKrakenConfig, SolveResult } from './types';
5
- /**
6
- * The two numbers that bound a solve, together, because they only make sense
7
- * together: `maxSolveLoops` is the count that FITS inside the timeout at the
8
- * ~4-7s a round costs, and the timeout is the backstop for when it does not.
9
- * Named rather than inlined as `??` defaults so the relationship between them
10
- * can be pinned by a test — see no-progress.test.ts. The Python port keeps the
11
- * same pair on `PageSolverConfig`.
12
- */
9
+ import { FrameRole, Vendor } from './kinds';
10
+ /** What detection found: the handle to photograph, the locator to search inside, and which table entry named it. */
11
+ export interface Widget {
12
+ el: ElementHandle;
13
+ at: Locator;
14
+ vendor: Vendor;
15
+ role: FrameRole;
16
+ }
17
+ /** An allow-list of what needs nothing: the other direction raises instead of clicking at the origin. */
18
+ export declare function answerNeedsElementBox(actions: ReadonlyArray<{
19
+ action?: string;
20
+ }>): boolean;
21
+ /** The widget moved on under us (hCaptcha swapped rounds, GeeTest closed on accept): re-detect, do not fail. */
22
+ export declare function isStaleHandleError(message: string): boolean;
13
23
  export declare const SOLVE_DEFAULTS: {
14
24
  readonly maxSolveLoops: 6;
15
25
  readonly overallSolveTimeoutMs: 45000;
26
+ readonly keyframeWaitTimeoutMs: 9000;
16
27
  };
28
+ /** A hang detector for a burst whose screenshot never returns, sized off the ceiling. */
29
+ export declare function burstHangDeadlineMs(cfg: {
30
+ videoBurstMaxMs?: number;
31
+ }): number;
17
32
  export declare class CaptchaKrakenSolver {
18
33
  private config;
19
- /** Extra ms this solve has been granted for a recording; see recordKeyframeBurst. */
20
34
  private videoBudgetMs;
21
35
  private videoBudgetGranted;
22
- /**
23
- * Every gesture goes through here, and it owns the pointer position. See
24
- * humanize.ts — the solver names gestures, this decides what events they are
25
- * and how long they take.
26
- */
27
36
  private human;
28
- private imageCounter;
29
- private sessionDebugDir;
30
37
  private stepIndex;
31
38
  private solveStartMs;
32
- private gridDebugDir;
33
- private gridDebugSeq;
34
39
  private cvWorker;
35
40
  private cvWorkerReady;
36
41
  private cvWorkerSeq;
37
42
  private cvWorkerPending;
38
43
  private cvWorkerBuf;
44
+ /** Answers keyed by screenshot hash; a hit means the answer already ran and changed nothing. */
39
45
  private solutionCache;
40
46
  private repeatedAnswerSeen;
47
+ /** The one recording and one answer for the animated board on screen. */
48
+ private animatedPlan;
41
49
  private keyframeMode;
50
+ private keyframeSteadyScreens;
51
+ private solveDeadlineAt;
42
52
  private lastAnswerSig;
43
53
  private noProgressRounds;
44
- private state;
54
+ private resampleLevel;
55
+ /** The recording `classifyByRecording` started and left running for the branch below to finish or drop. */
56
+ private pendingBurst;
57
+ private lastBurstFps;
58
+ private actedOnBoard;
45
59
  private lastSubmitFrameHash;
46
60
  private solveSessionId;
47
- /** Interpreter + CLI root, resolved once. See resolveCli. */
48
61
  private cliCache;
49
- /** The LoRA name models.json calls `latest`, read once. See askModel. */
50
62
  private loraNameCache;
51
- /**
52
- * Phase accounting for the CURRENT or MOST RECENT solve; see timing.ts.
53
- * Null before the first solve, and readable after a failed one.
54
- */
55
63
  budget: PhaseBudget | null;
56
- /**
57
- * Attribute the enclosed wall-clock to `name` for this solve's budget.
58
- *
59
- * A no-op outside a solve, so every helper stays callable from tests and from
60
- * the watcher without a budget having been opened.
61
- */
62
64
  private ph;
63
65
  constructor(config?: CaptchaKrakenConfig);
64
- /**
65
- * Where the pointer is. Owned by the humanizer, because a mode that
66
- * dispatches no motion (mobile, between taps) still has to answer this.
67
- */
68
66
  private get lastMousePosition();
69
67
  private set lastMousePosition(value);
68
+ /** `disabled` freezes CSS animation, which once sliced GeeTest svg to a static clip; callers filming motion pass `allow`. */
69
+ private shot;
70
70
  solve(page: Page): Promise<SolveResult | void>;
71
- /**
72
- * Tell the hosted API whether the widget accepted — the one fact the server
73
- * cannot see for itself, since a wrong answer reaches it as a 200 with
74
- * well-formed JSON in it.
75
- *
76
- * THROUGH THE PYTHON CLI, not a `fetch` here. The endpoint, the credential,
77
- * the extra-header handling and the "one 404 and stop asking" rule all live
78
- * in `planner.report_outcome`; a second copy in TypeScript would agree with
79
- * it until one of them was edited. That is the drift `model-name.ts` exists
80
- * to prevent, on the same seam, and this port already crosses that subprocess
81
- * boundary for every model call.
82
- *
83
- * NOT AWAITED, and detached. The solve is over and its result is decided, so
84
- * nothing here may add latency to what the caller gets back or keep the
85
- * process alive past it. `unref()` is what makes the second true.
86
- */
71
+ /** Tell the hosted API whether the widget accepted, through the CLI, detached, never awaited. */
87
72
  private reportOutcome;
88
- /**
89
- * Install an auto-solver: probe `page` on a timer and solve any captcha that
90
- * becomes visible, until `stop()`.
91
- *
92
- * Returns immediately — the watching happens in the background, so your
93
- * automation carries on and captchas are handled underneath it.
94
- *
95
- * Nothing is injected into the page on any platform; see watcher.ts for why,
96
- * and for the note on Camoufox, where the probe's DOM reads land in its
97
- * isolated world by default.
98
- *
99
- * ```typescript
100
- * const solver = new CaptchaKrakenSolver();
101
- * const watcher = solver.watch(page);
102
- * await page.goto('https://example.com/protected'); // solved as it appears
103
- * await watcher.stop();
104
- * ```
105
- */
73
+ /** Solve captchas on `page` as they appear, until `stop()`. Injects nothing into the page. */
106
74
  watch(page: Page, options?: WatchOptions): CaptchaWatcher;
107
75
  private solveImpl;
108
- /**
109
- * Fire the optional onStep observer with a fresh screenshot of the captcha
110
- * element. No-op (beyond a cheap early return) when no callback is set, so it
111
- * stays off the critical path in normal runs. The emitted PNG is owned by the
112
- * callback — we never delete it. Best-effort: a screenshot or callback error
113
- * never fails the solve.
114
- */
76
+ /** Fire the optional onStep observer with a fresh screenshot; best-effort, never fails the solve. */
115
77
  private emitStep;
116
78
  private solveSingle;
117
79
  private getVerifyButton;
118
- private hasNonEmptyFieldValue;
119
- /**
120
- * Detect reCAPTCHA's "Please select all matching images" error banner
121
- * (and the related "Please try again" / "Please also check the new images"
122
- * variants). These appear in the bframe AFTER clicking Verify with an
123
- * incomplete selection. The tiles do NOT refresh on this error — without
124
- * special handling the LoRA sees the same image, returns "done" (because
125
- * to it everything matching IS selected), we click Verify again, and we
126
- * loop until the session times out. We use this signal to switch the next
127
- * grid call into "missed-tiles" retry mode.
128
- */
129
- private hasRecaptchaUnderselectError;
130
- private isRecaptchaAnchorChecked;
131
- private isHcaptchaAnchorChecked;
132
- /**
133
- * True the moment the vendor reports the whole captcha solved — the anchor
134
- * checkbox flipped to checked, or the response token got populated. This is a
135
- * definitive "done" signal that a lingering, animating-closed challenge frame
136
- * is not: after the final submit, hCaptcha keeps the challenge iframe VISIBLE
137
- * for a couple of seconds while it verifies, so treating that frame as a fresh
138
- * puzzle (the old behavior) burned ~18s re-running the pipeline on it.
139
- */
80
+ /** Any element at `selector` carrying a non-blank value; response fields are hidden, so no visibility filter. */
81
+ private hasValue;
82
+ /** A visible match carrying text; an empty banner or prompt is a placeholder, not a signal. */
83
+ private visibleWithText;
84
+ private checkedIn;
85
+ private bannerIsFatalAfterRetry;
86
+ /** Which verdict banner the open challenge shows; `dynamic-more` is the dynamic board's normal flow, not an error. */
87
+ private bannerKind;
88
+ /** The vendor's own done signal: a response token, a painted success state, or a checked box. */
140
89
  private isCaptchaSolved;
141
- /**
142
- * True when an image challenge is open AND has actually rendered its prompt —
143
- * i.e. a fresh round we should solve, as opposed to a frame animating closed
144
- * (whose prompt has already gone). Used to tell "next round" from "solved,
145
- * closing" after a submit without waiting out a fixed timeout.
146
- */
90
+ /** A next round has painted, as opposed to the answered frame animating closed. */
147
91
  private isChallengeFreshlyRendered;
148
92
  /**
149
- * Block until the hCaptcha challenge frame's task images have actually
150
- * painted, so we don't screenshot a blank/half-loaded grid.
151
- *
152
- * hCaptcha renders each grid tile as a `.task-image .image` div whose
153
- * `background-image` is set once the asset loads; the prompt sits in
154
- * `.prompt-text`. Image-select (click/drag) challenges use a single
155
- * `.challenge-example` / `canvas` surface instead. We wait for either family
156
- * to be present AND for the background-image URLs to be populated (not the
157
- * empty `url("")` placeholder hCaptcha ships before the asset arrives).
158
- *
159
- * Best-effort: a timeout or a missing content frame just falls through to the
160
- * screenshot rather than throwing — the existing fail-fast path still covers a
161
- * genuinely unsupported puzzle.
162
- */
163
- private waitForHcaptchaChallengeImages;
164
- /**
165
- * Stage-1 detection: is an *interactive* captcha widget present in the DOM at
166
- * all — even if its iframe hasn't finished rendering yet?
167
- *
168
- * This is deliberately broader than detectCaptcha (which only returns a
169
- * VISIBLE, not-yet-solved element). We use it to distinguish two cases that
170
- * detectCaptcha() === null cannot tell apart:
171
- *
172
- * - A reCAPTCHA-v2 / hCaptcha widget IS in the DOM but is still loading
173
- * (iframe present, glyph not painted) → we should WAIT for it.
174
- * - There is no interactive widget — reCAPTCHA v3 (score-based, invisible)
175
- * or an hCaptcha that only triggers on a user action → we must FAIL FAST.
176
- *
177
- * reCAPTCHA v3 injects only `iframe[src*="recaptcha/api2/anchor"]` with
178
- * `size=invisible` in the src, and never an `api2/bframe` challenge frame, so
179
- * we exclude the invisible variant here.
180
- */
181
- /**
182
- * Which vendors' code this page actually loaded.
183
- *
184
- * Read out of the page's own resource timing plus the `src`/`href` of
185
- * everything it linked, so it works whenever it is called — a request
186
- * listener would have to have been attached before navigation, and by the
187
- * time a solve fails that ship has sailed.
188
- *
189
- * Best-effort by construction: the resource-timing buffer is finite and a
190
- * page may have cleared it. An empty answer is "nothing seen", never "no
191
- * captcha here". Mirrors `vendors_on_the_wire` in page_solver.py.
192
- */
193
- vendorsOnTheWire(page: Page): Promise<string[]>;
194
- /**
195
- * Why nothing was found — the two cases told apart. The old text guessed
196
- * "reCAPTCHA v3 / invisible" every time, including the times a vendor had
197
- * simply moved its DOM. See VENDOR_URL_MARKERS.
93
+ * Best-effort: hold until the board's pictures have painted. Nothing to wait for is ready, and so is no prompt: waiting
94
+ * on the prompt as a selector rejects when absent and paid the whole timeout per board.
198
95
  */
96
+ private waitForBoardImages;
97
+ /** Which vendors' code the page loaded, from resource timing and linked URLs. A tripwire, not a detector. */
98
+ vendorsOnTheWire(page: Page): Promise<Vendor[]>;
199
99
  private noWidgetMessage;
200
- hasInteractiveWidgetInDom(page: Page): Promise<boolean>;
201
- detectCaptcha(page: Page): Promise<ElementHandle | null>;
202
- /**
203
- * Initialize a fresh dump directory for one reCAPTCHA 3x3 dynamic-driver
204
- * session. Frames + a state.jsonl log land here so the click/fade/wait timing
205
- * can be replayed offline. Gated on CAPTCHA_DEBUG=1 — the per-frame dumps and
206
- * extra state queries add latency, so they stay off in normal runs. Set
207
- * CAPTCHA_DEBUG=1 to capture them when diagnosing timing. Best-effort.
208
- */
209
- private initGridDebug;
210
100
  /**
211
- * Log a structured event for the grid driver: prints a one-line summary to the
212
- * console and appends a JSON record to state.jsonl. If `framePath` is given,
213
- * copies that frame into the dump dir under a sequenced, labeled name so the
214
- * record can be matched to the exact pixels the detector saw. Best-effort.
215
- */
216
- private gridDebug;
217
- private saveImageForDebug;
218
- private archiveLatestDebugRun;
219
- /**
220
- * Resolve the bundled CaptchaKraken CLI root and the python interpreter to
221
- * run it with. Prefers the packaged venv python (postinstall bootstrap),
222
- * falling back to the configured/`python` command. Throws if the CLI folder
223
- * is missing — callers that must not throw (e.g. runCliTool) wrap this.
224
- */
225
- /**
226
- * The adapter name models.json calls `latest`, resolved once per solver.
227
- *
228
- * `resolveLoraName` reads models.json (and pinned_model.json) off disk
229
- * SYNCHRONOUSLY, and it was doing that on every inference for a file that
230
- * cannot change inside one process.
101
+ * Is a widget in the DOM at all, rendered or not? Invisible reCAPTCHA is excluded by its selector. The inline vendors
102
+ * count: without them this port failed fast in under a second on every GeeTest/Yidun page and disagreed with Python in Tier 3.
231
103
  */
104
+ hasInteractiveWidgetInDom(page: Page): Promise<boolean>;
105
+ /** Open challenges first, then unsolved checkboxes, then the inline vendors; every probe is queried at once. */
106
+ detectCaptcha(page: Page): Promise<Widget | null>;
107
+ /** A checkbox the vendor has already accepted is not a captcha to solve. */
108
+ private alreadyAccepted;
232
109
  private loraName;
110
+ /** The `--model` argv, or nothing: without VLLM_BASE_URL the CLI can see a credentials file this process cannot. */
111
+ private modelName;
233
112
  private resolveCli;
234
- /**
235
- * Run an OpenCV tool subcommand of the CLI (e.g. `grid-cell-states a.png
236
- * b.png`) and return its parsed single-line JSON. These subcommands print
237
- * exactly one JSON object on stdout (timing records go to stderr), so we
238
- * parse the whole trimmed stdout. Best-effort: returns `{}` on any failure so
239
- * polling callers can treat it as "inconclusive, keep going" without throwing.
240
- */
113
+ /** One-shot CLI tool call; `{}` on any failure so polling callers keep going. */
241
114
  private runCliTool;
242
- /**
243
- * Lazily start the persistent CV worker (`python -m captchakraken.cli serve`) and resolve
244
- * once it has imported cv2/numpy and emitted its `{"ready":true}` handshake.
245
- * Returns false if it can't be started (caller then falls back to one-shot
246
- * subprocesses). Idempotent: subsequent calls await the same readiness promise.
247
- */
115
+ /** Start the persistent CV worker once; false when it cannot start, so callers fall back to one-shot spawns. */
248
116
  private ensureCvWorker;
249
- /** Send one request to the CV worker and await its JSON result. Throws on any
250
- * worker failure so callers can fall back to the one-shot path. */
251
117
  private cvWorkerRequest;
252
- /** Kill the worker and clear state. Safe to call repeatedly. */
253
118
  private teardownCvWorker;
254
- /**
255
- * Run a CV tool through the persistent worker when available, falling back to a
256
- * one-shot `runCliTool` subprocess otherwise. `cmd`/`payload` map to the
257
- * worker's protocol; `fallbackArgs` is the equivalent one-shot argv. Worker
258
- * results are wrapped to match the one-shot JSON shape:
259
- * - grid-cell-states[-fixed]: worker returns the states object directly, or
260
- * {grid:null}; the one-shot returns the same shape, so just pass through.
261
- * - find-grid: worker returns the array (or null) as `result`.
262
- * Best-effort: never throws.
263
- */
119
+ /** A CV tool through the worker when it is up, else a one-shot subprocess. Never throws. */
264
120
  private runCvTool;
121
+ /** A still board is not a loaded board: hold until the panel's centre carries structure. Returns how long it waited. */
122
+ private waitForBoardPainted;
265
123
  /**
266
- * Block until a reCAPTCHA grid's cells have settled — none blank, none
267
- * mid-fade — before we screenshot it for the model. reCAPTCHA fades new tiles
268
- * in over ~1s; capturing mid-fade feeds the LoRA a blank/partial grid.
269
- *
270
- * We poll: screenshot the challenge element, keep the last two frames, and ask
271
- * the CLI's batched `grid-cell-states` (one subprocess per poll) which cells
272
- * are empty/changing/loaded. We return as soon as every cell is loaded, or on
273
- * timeout. Best-effort, mirroring `waitForHcaptchaChallengeImages`: never
274
- * throws, and falls through on timeout so a stuck/odd grid still proceeds to
275
- * the normal screenshot path. Temp frames are always cleaned up.
124
+ * Screenshot `el` every `intervalMs` until `judge(lastTwoFrames)` answers, or the time is up.
125
+ * A screenshot that fails ends the poll: the widget is gone or going.
276
126
  */
127
+ private pollFrames;
277
128
  private waitForGridCellsLoaded;
278
- /**
279
- * Read a PNG's pixel dimensions from its IHDR chunk (bytes 16-23, big-endian).
280
- * Avoids pulling in an image-size dependency. Returns null if the file isn't a
281
- * readable PNG.
282
- */
283
- private readPngDimensions;
284
- /**
285
- * Detect the reCAPTCHA grid once for a puzzle session: screenshot the element,
286
- * run `find-grid`, and read the screenshot's pixel dimensions. Grid boxes are
287
- * pixel coords in SCREENSHOT space (not page CSS space). Returns null if no
288
- * grid is detected. The geometry is stable across the in-place dynamic refresh
289
- * (only tile images change), so callers cache the result for the session.
290
- */
291
129
  private getGridBoxes;
292
- /**
293
- * Map a model-returned normalized bbox (fractions of the element/screenshot)
294
- * to a 1-indexed grid cell. Uses the bbox center, converts to screenshot
295
- * pixels, and returns the cell whose pixel box contains it. Cell numbering is
296
- * row-major (matches the CLI's find_grid output). Returns null if the center
297
- * falls outside every cell (e.g. in a gutter) — callers click the raw bbox
298
- * anyway and skip per-tile tracking.
299
- */
300
130
  private bboxToCell;
301
- /**
302
- * Center of a 1-indexed grid cell in PAGE pixel space, for mouse moves.
303
- * Converts the cached screenshot-pixel box to page coords via the session's
304
- * scaleX/scaleY (screenshot px -> page px) and element origin.
305
- */
306
131
  private cellCenterPage;
307
- /** Smooth-move the mouse over one cell's center with intra-cell jitter. */
308
132
  private hoverCell;
309
- /**
310
- * Query per-cell grid state using the SESSION'S CACHED grid boxes via the
311
- * `grid-cell-states-fixed` CLI command. This is critical: the dynamic refresh
312
- * blanks tiles to near-white, which makes find_grid fail on that frame, so the
313
- * self-detecting `grid-cell-states` would return {grid:null} mid-fade and a
314
- * naive caller would misread that as "nothing loading / solved". Passing the
315
- * cached boxes keeps empty/changing/selected correct even while tiles are
316
- * blank. Returns null only on a genuine CLI failure. Best-effort.
317
- */
133
+ /** Per-cell state with the session's cached boxes: a blank mid-refresh frame has no lattice to detect. */
318
134
  private gridCellStates;
319
- /** Order a loading set so `priority` cells (just-clicked) come first. */
320
135
  private orderByPriority;
136
+ private hoverLoop;
321
137
  /**
322
- * Watch the just-clicked tiles until the widget says what it did with them.
323
- *
324
- * Two answers because reCAPTCHA gives a click one of exactly two replies, and
325
- * they are the two kinds of board:
326
- *
327
- * - `chipped`: the small blue chip landed in the tiles' top-left corners —
328
- * the photos were KEPT. Nothing is on its way in, the selection is the
329
- * answer, and the caller should press Verify.
330
- * - `loading`: the photos are blanking or dissolving under a large centred
331
- * check — those tiles are being SWAPPED, and what lands may match too, so
332
- * the board has to be read again.
333
- *
334
- * A widget that swaps one clicked cell swaps them all, so the two never share
335
- * a board and one look at the tiles we just clicked settles it. The chip is
336
- * what the CV layer reports as `selected` (top-left corner only, behind a
337
- * circularity and a centroid test a centred check fails), and we already read
338
- * it on every poll. `chipped` needs EVERY watched tile: a partial reading is a
339
- * misread, and calling a swapping board finished submits half an answer and
340
- * burns the attempt, where calling a chipped board unfinished costs one
341
- * inference.
342
- *
343
- * The blank/fade transition lags the click by a beat, so a single snapshot
344
- * right after clicking misses it (the tile still shows its old image — not yet
345
- * white, not yet changing). We poll consecutive frames and mark a cell loading
346
- * if it is `empty` (≥97% near-white) OR `changing` (>2% pixels differ). HOVERS
347
- * a clicked tile each poll so the mouse keeps moving (no unnatural pauses).
348
- * Returns as soon as either verdict is in, or `{loading: [], chipped: false}`
349
- * if the whole window passes with nothing happening (→ solved). Logs every
350
- * poll + frame.
138
+ * `chipped`: the photos were kept, press Verify. `loading`: those tiles are being swapped, read the board again.
139
+ * Chip is tested first: a chip zooms the photo, which reads as `changing` on the very frame that shows it.
351
140
  */
352
141
  private watchClickedTiles;
353
- /**
354
- * After loading is detected, wait until at least one of the given blank/fading
355
- * cells reaches the `loaded` state, HOVERING those cells (in order) the whole
356
- * time so the mouse never sits still. Returns true once a tile loads, false on
357
- * timeout (caller proceeds anyway). Uses the session's cached grid boxes so it
358
- * works while tiles are blank. Logs every poll + frame.
359
- */
360
142
  private waitForAnyClickedTileLoaded;
361
- /**
362
- * Multi-round driver for reCAPTCHA 3x3 dynamic puzzles ("click all X" where
363
- * tiles refresh in place). One invocation = one puzzle session.
364
- *
365
- * The CLI is authoritative about WHAT to do — it runs the blue-badge detector,
366
- * filters out already-selected and still-loading tiles, and returns one of:
367
- * - `click`: click these tiles (already filtered to fresh, ready tiles)
368
- * - `wait` : nothing to click yet, tiles are still loading — do NOT submit
369
- * - `done` : nothing matching remains — submit (click Verify)
370
- *
371
- * This driver owns the HUMAN-LIKE WAITING the CLI can't: after a click round,
372
- * and on a `wait`, it hovers the just-clicked / currently blank+fading tiles
373
- * (in click order) and waits for at least one to finish reloading before
374
- * re-screenshotting and re-solving — so we don't burn a solver call on a grid
375
- * that's still mid-fade.
376
- *
377
- * Only a board that SWAPS a clicked tile out is worth those extra rounds. One
378
- * that ticks the tile and keeps the photo has been fully answered by the round
379
- * that clicked it — same as the 4x4 — so `watchClickedTiles` reports the chip
380
- * and this submits there and then. Rounds 2..N exist for the fading board and
381
- * nothing else. It submits on `done`, and on a board that ticked our clicks.
382
- *
383
- * Returns the same shape as solveSingle so the outer solve loop — including the
384
- * under-selection retry and post-solve detectCaptcha — wraps it unchanged.
385
- */
143
+ /** Click, watch the clicked tiles, re-solve while they swap, submit on `done` or a chipped board. */
386
144
  private solveRecaptchaGrid;
387
- /**
388
- * Record the animated challenge and return the directory holding the burst.
389
- *
390
- * The driver owns the browser, so it does the recording; the CLI does the
391
- * slicing (`solve-animated`). One zero-padded PNG per frame, because the slicer
392
- * sorts by name and reads the clip's temporal structure — `frame_9.png` sorting
393
- * after `frame_10.png` would shuffle the burst and turn a detectable cycle into
394
- * noise.
395
- *
396
- * Geometry comes from config and defaults to the collector's (4s @ 10fps), so a
397
- * challenge recorded here is the same shape of artifact the model trained on.
398
- * The caller must remove the directory when the actions are done with — the
399
- * keyframes the wait gate re-reads on every poll live inside it.
400
- */
401
145
  private recordKeyframeBurst;
402
146
  /**
403
- * Slice a recorded burst into keyframes and solve them in ONE model request.
404
- *
405
- * Deliberately not routed through `solveFrameFreshnessGuarded`. That guard
406
- * re-solves when the frame changes during inference, and an animated challenge
407
- * changes by definition — every attempt would be judged stale and the whole
408
- * re-solve budget would burn without ever acting. The `frame` in the answer is
409
- * the real guard: it names the state to act in, and `waitForKeyframe` enforces it.
410
- *
411
- * Also not deduped by screenshot hash: there is no single screenshot, and two
412
- * recordings of the same widget are never byte-identical anyway.
147
+ * Start filming now and decide later what it was for: the classifier, the speculative film, or the
148
+ * recording an animated answer is sliced from. Every window is wall-clock. A cycle is a screen that
149
+ * comes back; a board that shows nothing new for a floor window has settled. There is deliberately no
150
+ * "enough screens, stop" exit: measured on number_with_highest_value_video it failed every seed either way.
413
151
  */
152
+ private startKeyframeBurst;
153
+ /** Slice a burst into keyframes and solve them in one model request. The frame number is the guard. */
414
154
  private getAnimatedSolution;
415
- /**
416
- * Hold until the widget looks like `keyframePath` around the 0–1 point (cx, cy).
417
- *
418
- * This is the reason an animated answer names a frame. The model picked the
419
- * moment its target was visible, and the coordinates are only correct at that
420
- * moment; clicking as soon as the answer arrives lands on whatever the sprite
421
- * happens to be doing, which for a cross-fade is usually background.
422
- *
423
- * Only the neighbourhood of the action point is compared, with the same box and
424
- * metric the training label's frame was chosen with. Local rather than
425
- * whole-frame because everything ELSE in these puzzles is also moving: a
426
- * whole-frame match would need every unrelated sprite to align too, and would
427
- * essentially never open.
428
- *
429
- * Never throws. Returns whether the state was reached; on timeout the caller
430
- * clicks anyway (see `keyframeWaitTimeoutMs`).
431
- *
432
- * NOT ATTEMPTED on an `even` clip. The slicer picks that mode precisely when
433
- * the clip never revisits a picture it has already shown — a rotation, a
434
- * one-way fade, a sprite crossing — so there is no state to come back to and
435
- * this can only run out its full 6s, PER CLICK, before clicking the
436
- * coordinates it already had. It is also the normal case, not a corner: all
437
- * 116 real clips under cleanSamples/test/raw are `even` and `cycle` has never
438
- * fired on real footage. Measured on hcaptcha_rotating_obj_video: 6.0s of a
439
- * 28.8s solve, closest region diff 0.0721 against a 0.05 tolerance, then the
440
- * same click, then solved. Kept for `cycle`/`static`, where the state does
441
- * come back and waiting is the difference between the sprite and background.
442
- */
155
+ /** The session id and the credential travel in the environment, never in argv. */
156
+ private solveEnvironment;
157
+ /** Exit 2 is an unsupported puzzle, exit 3 the hosted API's own refusal; both are relayed, not reworded. */
158
+ private cliError;
159
+ /** Does the chosen keyframe's answer area appear in a sibling keyframe of the same clip? */
160
+ private answerRegionRecurs;
161
+ /** Hold until the widget looks like the keyframe around the action point, or the wait is spent. */
443
162
  private waitForKeyframe;
444
- /**
445
- * Per-solve state that must not leak into the next challenge on the page.
446
- *
447
- * A repeat is a fact about ONE challenge. Carrying it forward would make the
448
- * captcha after a cycling one record a burst it does not need.
449
- */
450
163
  private resetSolveState;
451
- /**
452
- * A stable identity for "the answer this round is about to execute".
453
- *
454
- * Keyed on the retry mode too: the missed-tiles retry deliberately re-asks
455
- * about the same board and its answer legitimately overlaps the previous one,
456
- * so counting that as a repeat would abandon the one path built to recover
457
- * from an under-selection.
458
- *
459
- * Coordinates are ROUNDED rather than compared exactly — the same tile chosen
460
- * twice can differ in the last float digit after the normalise/clamp
461
- * round-trip, and a repeat that reads as "different" is a repeat that costs a
462
- * round. Returns null when the answer cannot be summarised, which counts as
463
- * "not a repeat": an unreadable answer must never be why a solve is
464
- * abandoned. Mirrors page_solver.py `_answer_signature`.
465
- */
164
+ /** Keyed on the retry mode too: the missed-tiles answer legitimately overlaps the previous one. */
466
165
  private static answerSignature;
467
- /** Count consecutive identical answers; see `maxNoProgressRounds`. */
166
+ /** True when this answer already ran and changed nothing: resample, and let the recording path have a go. */
468
167
  private noteAnswer;
469
- /**
470
- * The answer for this picture, asking the model only if we have not already.
471
- *
472
- * The cache saving is real and is kept: a byte-identical picture costs no
473
- * second inference. What changed is what a hit MEANS. It used to mean
474
- * "nothing has changed, so this answer still stands"; it actually means the
475
- * answer already ran and moved nothing, because every answer this returns is
476
- * executed. So the hit is served (it is free) and recorded, and the round
477
- * after it stops guessing at a still frame — see `shouldRetryAsAnimated`.
478
- */
168
+ /** A cache hit costs no inference, and means the answer already ran: the board cycles. */
479
169
  private answerFor;
480
- /**
481
- * Should this round be recorded and solved from keyframes instead of read as
482
- * a still?
483
- *
484
- * reCAPTCHA is excluded deliberately. Its dynamic 3x3 REPLACES tiles in place
485
- * and has its own multi-round driver with its own fade gates; its grids are
486
- * never animated, so escalating there would swap a path that works for one
487
- * that cannot read a grid.
488
- */
170
+ private discardAnimatedPlan;
171
+ /** Static or animated, decided by filming once. The film stays running for the branch that uses it. */
172
+ private classifyByRecording;
173
+ private releasePendingBurst;
174
+ private shouldSpeculate;
489
175
  private shouldRetryAsAnimated;
490
176
  private getSolution;
491
- /** One inference: build the CLI invocation, run it, parse what comes back. */
177
+ /** One inference through the CLI. */
492
178
  private askModel;
493
- /**
494
- * Query the model for `initialShot`, then guard against the captcha frame
495
- * having changed DURING inference. reCAPTCHA/hCaptcha fade fresh tiles in over
496
- * ~1s; if new imagery painted while the model was generating, its answer
497
- * targets a stale ("undeveloped") frame and its tile picks / bboxes no longer
498
- * line up with what's on screen. After the model returns we re-screenshot the
499
- * element and diff it against the frame we sent (reusing the `check-movement`
500
- * primitive); if it moved beyond the threshold we discard the answer and
501
- * re-solve on the fresh frame, up to `maxStaleFrameReSolves` times, then act on
502
- * the latest answer rather than spin.
503
- *
504
- * `runQuery` performs the actual model call for a given screenshot path (the
505
- * caller wraps it in idle-wander where appropriate); it is re-invoked with the
506
- * fresh path on each re-solve. Token usage from every query — including
507
- * discarded ones — is accumulated, since those tokens were really spent. The
508
- * caller owns `initialShot`; every fresh frame captured here is created and
509
- * cleaned up here. Best-effort: a screenshot/diff failure falls through to the
510
- * answer already in hand.
511
- */
179
+ /** Re-solve when the frame changed during inference; twice means the board cycles. */
512
180
  private solveFrameFreshnessGuarded;
513
- /**
514
- * Screenshot the captcha element to a throwaway temp frame and diff it against
515
- * `priorPath` (the frame we sent the model). Returns true when the frame
516
- * changed beyond `threshold` — i.e. tiles faded in / refreshed since. Reuses
517
- * the persistent CV worker's `check-movement` (falling back to a one-shot
518
- * subprocess), the same frame-diff the settle detectors use. Cleans up its own
519
- * temp frame. Best-effort: any screenshot/diff failure returns false so the
520
- * caller acts on the answer it already has rather than spinning.
521
- */
522
181
  private captchaFrameChangedSince;
523
- /** Record a challenge-state transition. No-op for behaviour; logs the change
524
- * (via gridDebug, so only when CAPTCHA_DEBUG=1) for offline diagnosis. */
525
- private setState;
526
- /**
527
- * Screenshot the challenge element to a throwaway file and return its sha1
528
- * content hash (or null on failure). Used to tell whether the frame has
529
- * changed (e.g. the post-submit transition to the next round). Cleans up.
530
- */
531
182
  private elementFrameHash;
532
- /**
533
- * Monitor the challenge element until its pixels stop changing (settled), or
534
- * until it's clear it never will (animated / video). Polls screenshots and
535
- * diffs consecutive frames with `check-movement` (the persistent CV worker).
536
- * Returns:
537
- * 'settled' — `settleFrames` consecutive frame-pairs showed no movement;
538
- * safe to screenshot for the model.
539
- * 'animated' — it kept moving right up to `animatedChallengeAfterMs` without
540
- * ever settling → very likely a video/animated puzzle.
541
- * 'timeout' — neither happened within `settleTimeoutMs` (proceed best-effort).
542
- *
543
- * Note this is a *pixel* settle; the caller pairs it with the DOM-level
544
- * `waitForHcaptchaChallengeImages` so a static loading frame (spinner on grey,
545
- * below the pixel threshold) isn't mistaken for painted tiles. Cleans up.
546
- */
183
+ /** Poll until the pixels stop changing. `motionStreak` exits early: rotating_obj_video changes every 133-171ms and once spent 4.5s proving it. */
547
184
  private waitForElementSettled;
548
- /**
549
- * After a submit we EXPECT the challenge frame to change (advance to the next
550
- * round, or close because it was accepted). Poll until the element's content
551
- * hash differs from `sinceHash` — the transition beginning — so we never
552
- * screenshot/solve the pre-transition frame again. Returns true once it
553
- * changed, false if it never changed within `postSubmitChangeTimeoutMs` (e.g.
554
- * it was already the final state). Best-effort.
555
- */
556
185
  private waitForChangeSince;
557
- /**
558
- * Run `fn` (typically the model query) while idly drifting the cursor over
559
- * the captcha, so the mouse behaves like a human weighing the options instead
560
- * of freezing during inference. Uses the same humanizer as real clicks, so a
561
- * mode with no cursor skips it entirely; cancelled the instant `fn` resolves.
562
- * Best-effort — any wander error is swallowed and never fails the solve.
563
- * Disable via config.idleMouseWander.
564
- */
186
+ /** Drift the cursor over the widget while `fn` (the model call) runs; cancelled the instant it resolves. */
565
187
  private withIdleWander;
566
188
  move(page: Page, selectorOrElement: string | ElementHandle, options?: {
567
189
  paddingPercentage?: number;
568
190
  }): Promise<void>;
569
191
  moveAndClick(page: Page, element: ElementHandle): Promise<void>;
570
- /**
571
- * Travel to a point. What that MEANS is the humanizer's business — on a
572
- * touchscreen with no finger down it is a bookkeeping update and emits
573
- * nothing at all.
574
- */
575
192
  private performSmoothMove;
193
+ /** Element-relative click point: a random spot inside the box, inset 10% off its border. */
194
+ private clickPointFor;
195
+ /** Park on the target first, so only a mouse-down separates the right screen from the click. */
196
+ private clickWhenFrameMatches;
576
197
  private executeClick;
577
198
  private executeDrag;
578
199
  /**
579
- * First VISIBLE match for `selectors`, tried in order.
580
- *
581
- * `scope` is the challenge frame, or — for the vendors that render into the
582
- * host page rather than an iframe — the widget element itself. Never the
583
- * page: the generic tail of both selector tables would otherwise happily
584
- * match a login form's text box or a carousel's drag handle somewhere else on
585
- * the document, and the answer would go there.
200
+ * The slider piece's box: the first visible match small enough to be a piece. Runs during detection, so it must
201
+ * not set `actedOnBoard`: marking there disabled `shouldSpeculate` on every slide solve.
586
202
  */
203
+ private measurePieceBox;
587
204
  private findControl;
588
205
  /**
589
- * Where a distorted-text captcha's answer goes.
590
- *
591
- * Inside the widget first. If the widget holds no text box at all, widen ONCE
592
- * to its enclosing <fieldset>/<form> and look again for a VENDOR-NAMED box
593
- * only.
594
- *
595
- * BotDetect is why, and nothing else needs it. BotDetect is a self-hosted
596
- * LIBRARY, so the host application owns the layout: measured on captcha.com
597
- * 2026-08-24, `.BDC_CaptchaDiv` — the element detectCaptcha returns — is
598
- * 280x50 and holds the image alone, while `#captchaCode` sits in a sibling
599
- * `<div class="validationDiv">` under the enclosing <fieldset>. Scoped to the
600
- * widget the solver found no box, textMode stayed false, and it read the code
601
- * correctly and never typed it. A clean solve thrown away, and
602
- * indistinguishable in any report from a model that cannot read warped text.
603
- *
604
- * The widened pass is restricted to TEXT_INPUT_VENDOR_SELECTORS on purpose.
605
- * The generic tail is what makes the in-widget lookup work for every other
606
- * vendor, and OUTSIDE the widget it is exactly how a captcha's answer ends up
607
- * in a login form's username box. One level, named selectors, never the page.
608
- *
609
- * Mirrors `_answer_box` in page_solver.py — CLAUDE.md 1c.
206
+ * The text box inside the widget, else a vendor-named one in its enclosing fieldset/form, never the page: the generic
207
+ * tail outside the widget is how a captcha's answer lands in a login form's username box. The widening exists for
208
+ * BotDetect, whose 280x50 `.BDC_CaptchaDiv` holds only the image while `#captchaCode` sits in a sibling div.
610
209
  */
611
210
  private answerBox;
612
- /** Put the model's reading of a distorted-text captcha into its box. */
613
211
  private executeType;
614
- /** `captchakraken track-piece` — box of what moved, handle masked out. */
615
212
  private trackPiece;
616
- /**
617
- * Drive a puzzle-piece slider until the PIECE reaches the model's slot.
618
- *
619
- * The model is asked for one thing here — the centre of the gap — because it
620
- * is the only thing the picture can tell it. What it cannot know is how far
621
- * the handle must travel to put the piece there: the handle is elsewhere on
622
- * the widget, and the ratio between the two is a vendor implementation detail
623
- * that several of them deliberately vary.
624
- *
625
- * So this is closed-loop, not a calculation. Press the handle, nudge it twice
626
- * by known amounts, and watch the screen: union(before, after) spans the
627
- * piece's ORIGINAL left edge to its CURRENT right edge, so its width is
628
- * pieceWidth + ratio x nudge. Two nudges, two widths, two unknowns — solve for
629
- * both, then steer the remaining distance and re-measure. The mouse is not
630
- * released until the piece is home, because on every one of these puzzles
631
- * releasing IS the submit; there is no Verify button to reconsider at.
632
- *
633
- * Returns false if there is nothing here to drag, leaving the caller's normal
634
- * no-op handling to deal with it.
635
- */
636
- /**
637
- * Device pixels per CSS pixel, read off the shot we are about to measure.
638
- * 1 when the image cannot be read — an unreadable shot is already the
639
- * trackPiece-returns-null path, and guessing a ratio would steer by it.
640
- */
641
213
  private shotScale;
214
+ /** Aim the handle at the slot once, then look and correct until the piece is home. Release is the submit. */
642
215
  private executeSlide;
643
216
  }