captchakraken 3.2.1 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/AGENTS.md +5 -9
  2. package/LICENSE +165 -145
  3. package/NOTICE +4 -4
  4. package/README.md +10 -11
  5. package/dist/cli-invocation.d.ts +6 -3
  6. package/dist/cli-invocation.js +26 -10
  7. package/dist/humanize.d.ts +37 -1
  8. package/dist/humanize.js +143 -9
  9. package/dist/index.d.ts +13 -12
  10. package/dist/index.js +25 -33
  11. package/dist/kinds.d.ts +15 -2
  12. package/dist/kinds.js +12 -4
  13. package/dist/playwright-types.d.ts +3 -0
  14. package/dist/selectors.d.ts +93 -11
  15. package/dist/selectors.js +226 -116
  16. package/dist/solver.d.ts +92 -12
  17. package/dist/solver.js +562 -203
  18. package/dist/types.d.ts +53 -8
  19. package/dist/verdicts.d.ts +3 -0
  20. package/dist/verdicts.js +6 -0
  21. package/package.json +4 -4
  22. package/python/AGENTS.md +7 -11
  23. package/python/LICENSE +165 -145
  24. package/python/NOTICE +4 -4
  25. package/python/README.md +10 -9
  26. package/python/pyproject.toml +27 -13
  27. package/python/src/captchakraken/__init__.py +43 -46
  28. package/python/src/captchakraken/action_types.py +32 -1
  29. package/python/src/captchakraken/captchakraken.config.json +416 -0
  30. package/python/src/captchakraken/cli.py +66 -30
  31. package/python/src/captchakraken/clipvideo.py +258 -0
  32. package/python/src/captchakraken/humanize.py +141 -10
  33. package/python/src/captchakraken/image_processor.py +3 -7
  34. package/python/src/captchakraken/keyframes.py +96 -14
  35. package/python/src/captchakraken/kinds.py +14 -2
  36. package/python/src/captchakraken/models.json +28 -3
  37. package/python/src/captchakraken/page_solver.py +595 -251
  38. package/python/src/captchakraken/planner.py +298 -47
  39. package/python/src/captchakraken/pointer/__init__.py +20 -0
  40. package/python/src/captchakraken/pointer/detector.py +333 -0
  41. package/python/src/captchakraken/pointer/elements.py +273 -0
  42. package/python/src/captchakraken/pointer/heads.py +163 -0
  43. package/python/src/captchakraken/pointer/layout.py +319 -0
  44. package/python/src/captchakraken/pointer/marks.py +184 -0
  45. package/python/src/captchakraken/pointer/server.py +146 -0
  46. package/python/src/captchakraken/pointer/service.py +553 -0
  47. package/python/src/captchakraken/pointer/vllm_plugin.py +109 -0
  48. package/python/src/captchakraken/prompts.py +111 -34
  49. package/python/src/captchakraken/selectors.py +221 -135
  50. package/python/src/captchakraken/server_manager.py +18 -0
  51. package/python/src/captchakraken/solver.py +169 -88
  52. package/python/src/captchakraken/tool_calls/board_painted.py +11 -6
  53. package/python/src/captchakraken/tool_calls/fill_tracker.py +152 -0
  54. package/python/src/captchakraken/tool_calls/find_checkbox.py +14 -59
  55. package/python/src/captchakraken/tool_calls/find_grid.py +273 -1191
  56. package/python/src/captchakraken/tool_calls/slide_tracker.py +1102 -0
  57. package/python/src/captchakraken/verdicts.py +16 -0
  58. package/dist/slide-geometry.d.ts +0 -6
  59. package/dist/slide-geometry.js +0 -38
  60. package/python/src/captchakraken/tool_calls/track_piece.py +0 -120
package/dist/solver.js CHANGED
@@ -33,7 +33,8 @@ var __importStar = (this && this.__importStar) || (function () {
33
33
  };
34
34
  })();
35
35
  Object.defineProperty(exports, "__esModule", { value: true });
36
- exports.CaptchaKrakenSolver = exports.SOLVE_DEFAULTS = exports.INPUT_SETTLE_MS = void 0;
36
+ exports.CaptchaKrakenSolver = exports.SOLVE_DEFAULTS = exports.HOLD_MAX_MS = exports.HOLD_SETTLE_MS = exports.GRID_LOAD_CAP_FACTOR = exports.IDLE_DRIFT_PX = exports.IDLE_DRIFT_EVERY_MS = exports.INPUT_SETTLE_MS = void 0;
37
+ exports.withAnswerBox = withAnswerBox;
37
38
  exports.answerNeedsElementBox = answerNeedsElementBox;
38
39
  exports.isStaleHandleError = isStaleHandleError;
39
40
  exports.isClosedTargetError = isClosedTargetError;
@@ -55,7 +56,6 @@ const errors_1 = require("./errors");
55
56
  const limits_1 = require("./limits");
56
57
  const python_command_1 = require("./python-command");
57
58
  const cli_invocation_1 = require("./cli-invocation");
58
- const slide_geometry_1 = require("./slide-geometry");
59
59
  const model_name_1 = require("./model-name");
60
60
  const selectors_1 = require("./selectors");
61
61
  const execFileAsync = (0, util_1.promisify)(child_process_1.execFile);
@@ -128,6 +128,40 @@ function readPngDimensions(filePath) {
128
128
  const VENDORS_WITH_BESPOKE_HANDLING = new Set([kinds_1.Vendor.HCAPTCHA, kinds_1.Vendor.RECAPTCHA]);
129
129
  // Long enough to resolve a locator `all()` just returned, short enough that a widget gone in between reads as stale, not hung.
130
130
  const HANDLE_TIMEOUT_MS = 1000;
131
+ /**
132
+ * A typed captcha's text box widens the shot only when it sits by the image: the union may be at most this many
133
+ * times the image's area. A vendor-named box found in the enclosing form can be a whole login form away.
134
+ */
135
+ const ANSWER_BOX_MAX_GROWTH = 4;
136
+ /**
137
+ * A typed captcha's image and its text box, photographed and pointed into as one element.
138
+ *
139
+ * BotDetect's widget element is a 280x50 strip holding only the image; its text box is a sibling. Every training
140
+ * board and every real capture of a typed captcha shows the box under the image, and on the bare strip the pointer
141
+ * read the image as a click puzzle or doubled a glyph. The answer's points are taken in this union's frame, so
142
+ * `boundingBox` is the union everywhere the round reads it; everything else is the element's own.
143
+ */
144
+ function withAnswerBox(el, box) {
145
+ const union = async () => {
146
+ const a = await el.boundingBox();
147
+ const b = a ? await box.boundingBox() : null;
148
+ if (!a || !b)
149
+ return a;
150
+ const x0 = Math.min(a.x, b.x), y0 = Math.min(a.y, b.y);
151
+ const x1 = Math.max(a.x + a.width, b.x + b.width), y1 = Math.max(a.y + a.height, b.y + b.height);
152
+ if ((x1 - x0) * (y1 - y0) > ANSWER_BOX_MAX_GROWTH * Math.max(a.width * a.height, 1))
153
+ return a;
154
+ return { x: x0, y: y0, width: x1 - x0, height: y1 - y0 };
155
+ };
156
+ return new Proxy(el, {
157
+ get(target, key) {
158
+ if (key === 'boundingBox')
159
+ return union;
160
+ const v = Reflect.get(target, key, target);
161
+ return typeof v === 'function' ? v.bind(target) : v;
162
+ },
163
+ });
164
+ }
131
165
  /** Every visible match of `selectors`, in selector order, queried at once. A selector this adapter can't parse is skipped, not fatal. */
132
166
  const visible = async (scope, selectors) => (await Promise.all(selectors.map((s) => scope.locator(s).filter({ visible: true }).all().catch(() => [])))).flat();
133
167
  const handleOf = (at) => at.elementHandle({ timeout: HANDLE_TIMEOUT_MS }).catch(() => null);
@@ -137,11 +171,14 @@ const frameOf = async (at) => (await (await handleOf(at))?.contentFrame().catch(
137
171
  * Where the host-page selectors are read: the page, then the inside of every VISIBLE vendor enclave. A hidden enclave is
138
172
  * skipped because Playwright judges an element visible within its own document, whatever its frame's visibility.
139
173
  */
140
- const hostScopes = async (page) => [page, ...(await Promise.all((await visible(page, selectors_1.ENCLAVE_SELECTORS)).map(frameOf))).filter((f) => f !== null)];
174
+ const hostScopes = async (page, enclaves) => [page, ...(await Promise.all((await visible(page, enclaves)).map(frameOf))).filter((f) => f !== null)];
141
175
  const visibleIn = async (scopes, selectors) => (await Promise.all(scopes.map((scope) => visible(scope, selectors)))).flat();
142
176
  const hasText = async (el) => !!((await el.textContent().catch(() => null)) ?? '').trim();
143
177
  const SLIDE_TOLERANCE_PX = 2;
144
178
  const SLIDE_MAX_CORRECTIONS = 3;
179
+ // The nudge that shows the tracker what the handle moves: enough to separate the part from still ground, little
180
+ // enough that a piece resting against the board's edge stays on it. Mirrors page_solver.py SLIDE_PROBE_PX.
181
+ const SLIDE_PROBE_PX = 8;
145
182
  /** An allow-list of what needs nothing: the other direction raises instead of clicking at the origin. */
146
183
  function answerNeedsElementBox(actions) {
147
184
  return actions.some((a) => a?.action !== kinds_1.ActionKind.DONE);
@@ -166,6 +203,16 @@ const NOT_THIS_BOARD_POLLS = 3;
166
203
  // How long a control we hovered, pressed or typed into takes to stop repainting in reaction. A board read inside
167
204
  // this window is wearing our own feedback, which is not a screen it ever showed by itself.
168
205
  exports.INPUT_SETTLE_MS = 400;
206
+ // While a reCAPTCHA tile reloads the hand rests on it, drifting a few pixels every second or two. Reaching for a
207
+ // different reloading tile on every 250 ms poll read as a cursor darting about the board.
208
+ exports.IDLE_DRIFT_EVERY_MS = [1200, 2800];
209
+ exports.IDLE_DRIFT_PX = 3;
210
+ // How much longer than gridLoadTimeoutMs a reCAPTCHA tile that is still painting is waited on.
211
+ exports.GRID_LOAD_CAP_FACTOR = 3;
212
+ // A press-and-hold is released once the widget's picture has stopped changing for this long after its fill
213
+ // moved, and at HOLD_MAX_MS whatever it shows. Press & Hold widgets ask for seconds, not tens of seconds.
214
+ exports.HOLD_SETTLE_MS = 900;
215
+ exports.HOLD_MAX_MS = 12000;
169
216
  /** `viewportSize()` is null on a context launched without one (camoufox), where only the window knows. */
170
217
  async function viewportOf(page) {
171
218
  const view = page.viewportSize() ?? await page.evaluate?.(() => ({ width: window.innerWidth, height: window.innerHeight }));
@@ -232,6 +279,7 @@ class CaptchaKrakenSolver {
232
279
  constructor(config = {}) {
233
280
  this.videoBudgetMs = 0;
234
281
  this.videoBudgetGranted = false;
282
+ this.sel = selectors_1.DEFAULT_TABLE;
235
283
  this.stepIndex = 0;
236
284
  this.solveStartMs = 0;
237
285
  this.cvWorker = null;
@@ -241,6 +289,15 @@ class CaptchaKrakenSolver {
241
289
  this.cvWorkerBuf = '';
242
290
  /** Answers keyed by screenshot hash; a hit means the answer already ran and changed nothing. */
243
291
  this.solutionCache = new Map();
292
+ /** The other complete plans the pointer returned for this board, best first; a refresh is last. */
293
+ this.fallbackPlans = [];
294
+ /** The screenshot those plans answered. A different board makes them the wrong answer, not an alternative. */
295
+ this.fallbackBoardHash = null;
296
+ /** True only once THIS board refused the plan at the front. Nothing is pressed speculatively. */
297
+ this.fallbackArmed = false;
298
+ this.exchangesAtAnswer = 0;
299
+ /** Earlier screens of the same paged puzzle (FunCaptcha), oldest first; the CLI sends them as `--history`. */
300
+ this.pointerHistory = [];
244
301
  this.repeatedAnswerSeen = false;
245
302
  this.knownAnimated = false;
246
303
  this.animatedProbeDone = false;
@@ -250,8 +307,9 @@ class CaptchaKrakenSolver {
250
307
  this.keyframeSteadyScreens = 0;
251
308
  this.solveDeadlineAt = 0;
252
309
  this.lastAnswerSig = null;
310
+ this.triedAnswers = new Set();
311
+ this.refreshedBoards = new Set();
253
312
  this.noProgressRounds = 0;
254
- this.resampleLevel = 0;
255
313
  /** The recording `classifyByRecording` started and left running for the branch below to finish or drop. */
256
314
  this.pendingBurst = null;
257
315
  /**
@@ -264,6 +322,7 @@ class CaptchaKrakenSolver {
264
322
  this.animatedFilm = null;
265
323
  this.lastBurstFps = null;
266
324
  this.actedOnBoard = false;
325
+ this.submittedOnBoard = false;
267
326
  this.lastSubmitFrameHash = null;
268
327
  this.solveSessionId = null;
269
328
  /** Which vendor and site this solve is for, sent to the hosted API; the site is a hostname only. */
@@ -283,6 +342,9 @@ class CaptchaKrakenSolver {
283
342
  /** When our last gesture ended; the board's reaction to it is never read as the board's own motion. */
284
343
  this.lastInputAt = -Infinity;
285
344
  this.config = config;
345
+ if (config.customSelectors || config.selectors !== undefined) {
346
+ this.sel = new selectors_1.SelectorTable(config.customSelectors, config.selectors ?? selectors_1.DEFAULT_CONFIG_PATH);
347
+ }
286
348
  this.human = (0, humanize_js_1.resolveHumanizer)({ ...config, startingMousePosition: config.startingMousePosition ?? { x: 100, y: 100 } });
287
349
  }
288
350
  get lastMousePosition() {
@@ -371,6 +433,7 @@ class CaptchaKrakenSolver {
371
433
  console.error(this.budget.report());
372
434
  // The camera outlives a round by design, so it has to be ended by the thing that outlives the solve.
373
435
  await this.stopAnimatedFilm();
436
+ this.dropPointerHistory();
374
437
  this.teardownCvWorker();
375
438
  this.cvWorkerReady = null;
376
439
  // Set by solveImpl, which the narrowing from the reset above cannot see.
@@ -390,7 +453,7 @@ class CaptchaKrakenSolver {
390
453
  return new Promise((resolve) => {
391
454
  try {
392
455
  const { cliRoot, py } = this.resolveCli();
393
- const child = (0, child_process_1.spawn)(py, ['-m', 'captchakraken.cli', 'report-outcome', sessionId, solved ? kinds_1.Outcome.SOLVED : kinds_1.Outcome.FAILED], { cwd: cliRoot, env: cliEnv(cliRoot, this.routingEnv()), stdio: ['ignore', 'pipe', 'pipe'] });
456
+ const child = (0, child_process_1.spawn)(py, ['-m', 'captchakraken.cli', 'report-outcome', sessionId, solved ? kinds_1.Outcome.SOLVED : kinds_1.Outcome.FAILED], { cwd: cliRoot, env: (0, cli_invocation_1.solveEnv)(cliEnv(cliRoot, this.routingEnv()), this.apiKey), stdio: ['ignore', 'pipe', 'pipe'] });
394
457
  let stdout = '';
395
458
  child.stdout.on('data', (chunk) => { stdout += chunk.toString(); });
396
459
  // The engine words its own warnings; a hosted refusal must be as visible here as it is from Python.
@@ -562,14 +625,18 @@ class CaptchaKrakenSolver {
562
625
  break;
563
626
  }
564
627
  if (await this.isChallengeFreshlyRendered(page)) {
565
- this.resampleLevel = 0;
566
628
  this.actedOnBoard = false;
629
+ this.submittedOnBoard = false;
567
630
  // ONE FILM, ONE BOARD — enforced where the board actually changes. The recorded answer
568
631
  // describes the board it was cut from, so replaying it here presses that board's
569
632
  // coordinates onto this one: live, a keyframe answer from an earlier board landed on
570
633
  // hCaptcha's reference photo and then pressed Skip, six rounds running.
571
634
  await this.stopAnimatedFilm();
572
635
  this.knownAnimated = false;
636
+ this.fallbackPlans = [];
637
+ this.fallbackBoardHash = null;
638
+ this.fallbackArmed = false;
639
+ this.dropPointerHistory();
573
640
  // Its own second look, but only once IT fails: what armed it was evidence about the previous board,
574
641
  // and left set it filmed every still board after a solve's first miss.
575
642
  this.repeatedAnswerSeen = false;
@@ -587,9 +654,15 @@ class CaptchaKrakenSolver {
587
654
  throw blocked('its server answered 429');
588
655
  if (await this.isBlocked(page))
589
656
  throw blocked('it is showing its try-again-later screen');
590
- if (roundSaid === kinds_1.Verdict.REJECTED)
657
+ if (roundSaid === kinds_1.Verdict.REJECTED) {
591
658
  console.log(`[verdict] the vendor rejected the answer after ${Date.now() - verdictT0}ms`);
592
- if (this.bannerIsFatalAfterRetry(await this.bannerKind(page))) {
659
+ // Only a refusal earns the server's next plan. Without it a board that merely repainted would have
660
+ // the previous board's plan pressed onto it.
661
+ this.fallbackArmed = true;
662
+ }
663
+ // Only "select all matching images" means tiles were missed; "please try again" is a wrong answer and
664
+ // deals a new board, which is asked fresh.
665
+ if ((await this.bannerKind(page)) === kinds_1.RecaptchaBanner.SELECT_MORE) {
593
666
  console.log('reCAPTCHA under-selection error; the next round asks for the missed tiles.');
594
667
  pendingRetryMode = kinds_1.RetryMode.MISSED_TILES;
595
668
  }
@@ -597,10 +670,10 @@ class CaptchaKrakenSolver {
597
670
  return done();
598
671
  if (this.noProgressRounds) {
599
672
  const stuck = `No progress: the model returned the same answer ${this.noProgressRounds + 1} times running`;
600
- // A board answered the same way after every resample is one the model cannot read: measured, no solve ever
601
- // followed a third identical answer, and each further round cost the board 3-7s.
602
- if (this.noProgressRounds >= (cfg.maxNoProgressRounds ?? 2))
673
+ // A repeat pressed nothing, so a further round is only worth its 3-7s if it can do something else.
674
+ if (this.noProgressRounds >= (cfg.maxNoProgressRounds ?? 2) || !this.anotherMoveLeft(pendingRetryMode)) {
603
675
  throw new Error(`${stuck}.`);
676
+ }
604
677
  await again(stuck);
605
678
  }
606
679
  else if (!didInteract) {
@@ -610,6 +683,9 @@ class CaptchaKrakenSolver {
610
683
  this.repeatedAnswerSeen = true; // what arms the second look
611
684
  await again('The solver performed no interactions this round');
612
685
  }
686
+ else {
687
+ lastFailure = null;
688
+ }
613
689
  }
614
690
  const last = lastFailure;
615
691
  const usage = ` Total usage: ${JSON.stringify((0, token_usage_1.aggregateTokenUsage)(cumulativeTokenUsage))}`;
@@ -638,12 +714,12 @@ class CaptchaKrakenSolver {
638
714
  /** The vendor showing its refusal screen: rate-limited or flagged, with no board to answer. */
639
715
  async isBlocked(page) {
640
716
  try {
641
- const probes = selectors_1.WIDGET_PROBES.filter((p) => p.role === kinds_1.FrameRole.CHALLENGE && selectors_1.SELECTORS[p.vendor].blocked);
642
- const scopes = await hostScopes(page);
717
+ const probes = this.sel.widgetProbes.filter((p) => p.role === kinds_1.FrameRole.CHALLENGE && this.sel.selectors[p.vendor].blocked);
718
+ const scopes = await hostScopes(page, this.sel.enclave);
643
719
  const shown = await Promise.all(probes.map(async (p) => {
644
720
  const [at] = await visibleIn(scopes, [p.selector]);
645
721
  const frame = at && await frameOf(at);
646
- return !!frame && (await Promise.all((selectors_1.SELECTORS[p.vendor].blocked ?? []).map((sel) => this.visibleWithText(frame, sel)))).some(Boolean);
722
+ return !!frame && (await Promise.all((this.sel.selectors[p.vendor].blocked ?? []).map((sel) => this.visibleWithText(frame, sel)))).some(Boolean);
647
723
  }));
648
724
  return shown.some(Boolean);
649
725
  }
@@ -652,14 +728,24 @@ class CaptchaKrakenSolver {
652
728
  }
653
729
  }
654
730
  /** The env the engine reads its routing headers from: session, and the vendor and site once they are known. */
731
+ /** This solver's endpoint and key: two solvers in one process may talk to different servers. */
732
+ get baseUrl() {
733
+ return this.config.baseUrl ?? process.env.VLLM_BASE_URL;
734
+ }
735
+ get apiKey() {
736
+ return this.config.apiKey ?? process.env.CAPTCHA_KRAKEN_API_KEY ?? process.env.VLLM_API_KEY;
737
+ }
655
738
  routingEnv() {
656
739
  return {
740
+ ...(this.config.baseUrl ? { VLLM_BASE_URL: this.config.baseUrl } : {}),
657
741
  ...(this.solveSessionId ? { CAPTCHA_KRAKEN_SESSION: this.solveSessionId } : {}),
658
742
  ...(this.solveVendor ? { CAPTCHA_KRAKEN_VENDOR: this.solveVendor } : {}),
659
743
  ...(this.solveSite ? { CAPTCHA_KRAKEN_SITE: this.solveSite } : {}),
660
744
  ...(this.solveWidgetHost ? { CAPTCHA_KRAKEN_WIDGET_HOST: this.solveWidgetHost } : {}),
661
745
  // This driver warns about a legacy model name itself, once; the engine it starts per round stays quiet.
662
746
  CAPTCHA_KRAKEN_MODEL_WARNING: '0',
747
+ // The config's detector choice; the caller's own environment variable still wins, as it does in Python.
748
+ ...(process.env.CAPTCHA_KRAKEN_DETECTOR_MODEL ? {} : { CAPTCHA_KRAKEN_DETECTOR_MODEL: this.sel.config.detector }),
663
749
  };
664
750
  }
665
751
  /** Fire the optional onStep observer with a fresh screenshot; best-effort, never fails the solve. */
@@ -685,7 +771,8 @@ class CaptchaKrakenSolver {
685
771
  }
686
772
  async solveSingle(page, widget, attempt, retryMode = null) {
687
773
  const cfg = this.config;
688
- const { el: captchaElement, vendor: puzzleSource, role: frameRole } = widget;
774
+ const { vendor: puzzleSource, role: frameRole } = widget;
775
+ let captchaElement = widget.el;
689
776
  this.solveVendor = puzzleSource;
690
777
  const frame = await captchaElement.contentFrame();
691
778
  this.solveWidgetHost = frame ? hostname(frame) : null;
@@ -694,12 +781,12 @@ class CaptchaKrakenSolver {
694
781
  await this.clickCheckbox(page, widget, frame);
695
782
  return { didInteract: true, tokenUsage: [] };
696
783
  }
697
- if (frame && frameRole === kinds_1.FrameRole.CHALLENGE && selectors_1.SELECTORS[puzzleSource].images) {
784
+ if (frame && frameRole === kinds_1.FrameRole.CHALLENGE && this.sel.selectors[puzzleSource].images) {
698
785
  if (this.lastSubmitFrameHash) {
699
786
  await this.ph(kinds_1.Phase.AWAIT_NEXT_ROUND, () => this.waitForChangeSince(captchaElement, this.lastSubmitFrameHash));
700
787
  this.lastSubmitFrameHash = null;
701
788
  }
702
- await this.ph(kinds_1.Phase.HCAPTCHA_IMAGES, () => this.waitForBoardImages(frame, selectors_1.SELECTORS[puzzleSource]));
789
+ await this.ph(kinds_1.Phase.BOARD_IMAGES, () => this.waitForBoardImages(frame, this.sel.selectors[puzzleSource]));
703
790
  }
704
791
  // THE TWO QUESTIONS BELOW ARE PUT TO A PAINTED BOARD. Which expert answers the round is read out of the
705
792
  // DOM and whether the board cycles is read off its motion, and a widget that has not drawn yet has no
@@ -711,9 +798,12 @@ class CaptchaKrakenSolver {
711
798
  const painted = puzzleSource === kinds_1.Vendor.RECAPTCHA ? null
712
799
  : await this.ph(kinds_1.Phase.BOARD_PAINT, () => this.waitForBoardPainted(captchaElement));
713
800
  // Only the DOM can tell a typed captcha from a click puzzle; hCaptcha and reCAPTCHA never type.
714
- const textMode = !VENDORS_WITH_BESPOKE_HANDLING.has(puzzleSource) && (await this.answerBox(scope, widget.at)) !== null;
715
- if (textMode)
801
+ const answerBox = VENDORS_WITH_BESPOKE_HANDLING.has(puzzleSource) ? null : await this.answerBox(scope, widget.at);
802
+ const textMode = answerBox !== null;
803
+ if (textMode) {
716
804
  console.log('Widget has a text box; solving as a distorted-text captcha.');
805
+ captchaElement = withAnswerBox(captchaElement, answerBox);
806
+ }
717
807
  // A reCAPTCHA board is read by its grid below, not filmed.
718
808
  const filmable = puzzleSource !== kinds_1.Vendor.RECAPTCHA && !textMode;
719
809
  if (filmable && !this.knownAnimated)
@@ -768,12 +858,17 @@ class CaptchaKrakenSolver {
768
858
  const screenshotPath = tmp('captcha');
769
859
  const settledFrame = (isAnimated || painted?.verdict !== kinds_1.PaintVerdict.PAINTED)
770
860
  ? null : (this.pendingBurst?.stableFrame() ?? null);
771
- if (settledFrame && fs.existsSync(settledFrame)) {
861
+ const filmed = !!settledFrame && fs.existsSync(settledFrame);
862
+ if (filmed) {
772
863
  fs.copyFileSync(settledFrame, screenshotPath);
773
864
  }
774
865
  else {
775
866
  await this.ph(kinds_1.Phase.SCREENSHOT, () => this.shot(captchaElement, screenshotPath, cfg.elementScreenshotTimeoutMs ?? 8000));
776
867
  }
868
+ const boardHash = await this.boardHash(captchaElement, screenshotPath, filmed);
869
+ // A refresh that brought the same picture back has replaced nothing, and every plan has been spent on it.
870
+ if (this.refreshedBoards.has(boardHash))
871
+ throw new Error('No progress: the board came back unchanged after a refresh.');
777
872
  if (this.stepIndex === 0) {
778
873
  await this.emitStep(captchaElement, kinds_1.SolveStage.INITIAL, 'initial (pre-action)', puzzleSource, frameRole, attempt, establishedGridSize ? { gridSize: establishedGridSize } : undefined);
779
874
  }
@@ -784,9 +879,22 @@ class CaptchaKrakenSolver {
784
879
  let burstDir = null;
785
880
  try {
786
881
  let response = null;
882
+ let fromFallback = false;
883
+ if (this.fallbackPlans.length) {
884
+ if (this.fallbackArmed && this.fallbackBoardHash === boardHash) {
885
+ response = { actions: this.fallbackPlans.shift(), token_usage: [] };
886
+ fromFallback = true;
887
+ }
888
+ else {
889
+ // The board that refused the first plan is not this one, or the last round did not refuse it.
890
+ this.fallbackPlans = [];
891
+ this.fallbackBoardHash = null;
892
+ this.fallbackArmed = false;
893
+ }
894
+ }
787
895
  const ask = (imagePath) => this.getSolution(imagePath, puzzleSource, retryMode, textMode);
788
896
  const askAnimated = () => this.ph(kinds_1.Phase.INFERENCE, () => this.withIdleWander(page, captchaElement, () => this.getAnimatedSolution(burstDir)));
789
- if (isAnimated) {
897
+ if (isAnimated && !response) {
790
898
  // The camera was paused while the last answer was performed; from here the board is its own again.
791
899
  this.animatedFilm?.resume();
792
900
  if (this.animatedPlan?.response) {
@@ -891,6 +999,11 @@ class CaptchaKrakenSolver {
891
999
  else if (!response) {
892
1000
  response = await this.ph(kinds_1.Phase.INFERENCE, () => this.solveFrameFreshnessGuarded(captchaElement, screenshotPath, (imagePath) => this.withIdleWander(page, captchaElement, () => ask(imagePath))));
893
1001
  }
1002
+ if (!fromFallback) {
1003
+ this.fallbackPlans = response.fallbacks ?? [];
1004
+ this.fallbackBoardHash = this.fallbackPlans.length ? boardHash : null;
1005
+ this.fallbackArmed = false;
1006
+ }
894
1007
  const actionList = Array.isArray(response.actions) ? response.actions : [response.actions];
895
1008
  allTokenUsage = response.token_usage;
896
1009
  // A `done` answer carries no coordinates, and several vendors close the widget the moment they accept.
@@ -905,10 +1018,27 @@ class CaptchaKrakenSolver {
905
1018
  // Tier 3 grades this line with Tier 2's grader to split driver bugs from model misses (see TRIBAL_KNOWLEDGE.md).
906
1019
  console.log('[answer] ' + JSON.stringify({ actions: actionList }));
907
1020
  // A repeated answer is not re-performed: the widget already refused it, and every extra press is
908
- // behaviour a vendor scores. Re-asking with a fresh sample or a recording is the round's only move.
1021
+ // behaviour a vendor scores. Re-asking, or asking a recording, is the round's only move.
909
1022
  // `noteAnswer` drops the refused animated answer on its way through, so the next round re-asks.
910
- if (this.noteAnswer(actionList, retryMode))
1023
+ // An animated board's screens are SUPPOSED to change; keying a repeat on them would never fire. The
1024
+ // board hash is for the still path only, where a paged puzzle's same next-arrow is a new page.
1025
+ if (this.noteAnswer(actionList, retryMode, isAnimated ? null : boardHash))
911
1026
  return { didInteract: false, tokenUsage: allTokenUsage };
1027
+ // FunCaptcha pages one option at a time and the model reads the earlier screens as history. Without
1028
+ // them it has nothing to compare against and keeps answering "next arrow", including on the match.
1029
+ if (!isAnimated && puzzleSource === kinds_1.Vendor.FUNCAPTCHA
1030
+ && actionList.length && actionList.every((a) => a?.action === kinds_1.ActionKind.CLICK)) {
1031
+ try {
1032
+ if (fs.existsSync(screenshotPath)) {
1033
+ const saved = tmp('pghist');
1034
+ fs.copyFileSync(screenshotPath, saved);
1035
+ this.pointerHistory.push(saved);
1036
+ while (this.pointerHistory.length > 8)
1037
+ unlink(this.pointerHistory.shift());
1038
+ }
1039
+ }
1040
+ catch { /* a screen we could not keep simply is not history */ }
1041
+ }
912
1042
  console.log(`Executing ${actionList.length} actions.`);
913
1043
  // Stop filming before we touch it. Everything from here to the vendor's verdict is our own answer
914
1044
  // landing, and a frame of the board wearing our clicks is not a screen the board ever showed.
@@ -937,7 +1067,7 @@ class CaptchaKrakenSolver {
937
1067
  performedAction = answered = true;
938
1068
  await this.emitStep(captchaElement, kinds_1.SolveStage.CLICK, `clicked ${bboxes.length || 1} target(s)`, puzzleSource, frameRole, attempt, { bboxes });
939
1069
  }
940
- else if (action.action === kinds_1.ActionKind.DRAG && !action.source_bounding_box) {
1070
+ else if (action.action === kinds_1.ActionKind.SLIDE || (action.action === kinds_1.ActionKind.DRAG && !action.source_bounding_box)) {
941
1071
  if (await this.executeSlide(page, captchaElement, scope, action, requireBox())) {
942
1072
  performedAction = slid = true;
943
1073
  await this.emitStep(captchaElement, kinds_1.SolveStage.DRAG, 'slid the piece into the slot', puzzleSource, frameRole, attempt, { action });
@@ -951,12 +1081,25 @@ class CaptchaKrakenSolver {
951
1081
  performedAction = answered = true;
952
1082
  await this.emitStep(captchaElement, kinds_1.SolveStage.DRAG, 'drag', puzzleSource, frameRole, attempt, { action });
953
1083
  }
1084
+ else if (action.action === kinds_1.ActionKind.HOLD) {
1085
+ // The widget submits itself when the hold completes, like a slide on release.
1086
+ const completed = await this.executeHold(page, captchaElement, action, requireBox(), this.sel.selectors[widget.vendor].holdCompletes ?? false);
1087
+ performedAction = slid = true;
1088
+ await this.emitStep(captchaElement, kinds_1.SolveStage.HOLD, completed ? 'held until the widget completed' : 'released the hold at the cap', puzzleSource, frameRole, attempt, { action });
1089
+ }
954
1090
  else if (action.action === kinds_1.ActionKind.TYPE) {
955
1091
  if (await this.executeType(page, scope, action, widget.at)) {
956
1092
  performedAction = answered = true;
957
1093
  await this.emitStep(captchaElement, kinds_1.SolveStage.TYPE, 'typed the code', puzzleSource, frameRole, attempt, { action });
958
1094
  }
959
1095
  }
1096
+ else if (action.action === kinds_1.ActionKind.REFRESH) {
1097
+ // Nothing after a refresh, Verify included, belongs to the board it replaces.
1098
+ const refreshed = await this.executeRefresh(page, captchaElement, scope, action, elementBox);
1099
+ if (refreshed)
1100
+ await this.emitStep(captchaElement, kinds_1.SolveStage.REFRESH, 'asked for a new puzzle', puzzleSource, frameRole, attempt, { action });
1101
+ return { didInteract: refreshed, tokenUsage: allTokenUsage };
1102
+ }
960
1103
  else if (action.action === kinds_1.ActionKind.WAIT && action.duration_ms > 0) {
961
1104
  await delay(action.duration_ms);
962
1105
  performedAction = true;
@@ -964,15 +1107,22 @@ class CaptchaKrakenSolver {
964
1107
  }
965
1108
  }
966
1109
  // A slide submits itself on release, and any Verify found afterwards belongs to the host page and would
967
- // submit the guarded form mid-verdict. An empty or `done` plan still presses Verify/Skip.
968
- const verifyButton = frame || !slid ? await this.getVerifyButton(scope) : null;
969
- if (!slid && (answered || !performedAction) && verifyButton) {
1110
+ // submit the guarded form mid-verdict, unless this board has already taken a Verify from us: then that button
1111
+ // is the widget's, and a slide on it submits nothing by itself. An empty or `done` plan still presses
1112
+ // Verify/Skip. FunCaptcha is the exception: its answer is the click itself — an arrow pages, Submit (which the
1113
+ // model clicks on the match) submits — so the generic fallback submits page one right after an arrow turned it.
1114
+ const submits = slid ? this.submittedOnBoard : answered || !performedAction;
1115
+ const verifyButton = submits && puzzleSource !== kinds_1.Vendor.FUNCAPTCHA ? await this.getVerifyButton(scope) : null;
1116
+ if (verifyButton) {
970
1117
  console.log(`Clicking Verify to submit (${puzzleSource}).`);
971
1118
  await this.moveAndClick(page, verifyButton);
972
- performedAction = true;
1119
+ performedAction = this.submittedOnBoard = true;
973
1120
  await this.emitStep(captchaElement, kinds_1.SolveStage.SUBMIT, 'submitted (Verify/Next)', puzzleSource, frameRole, attempt);
974
1121
  this.lastSubmitFrameHash = await this.elementFrameHash(captchaElement);
975
1122
  }
1123
+ else if (answered && puzzleSource !== kinds_1.Vendor.FUNCAPTCHA) {
1124
+ this.keepPlansOfThisManyClicks(actionList);
1125
+ }
976
1126
  }
977
1127
  finally {
978
1128
  await this.releasePendingBurst();
@@ -989,11 +1139,15 @@ class CaptchaKrakenSolver {
989
1139
  * checkbox picture with a drag.
990
1140
  */
991
1141
  async clickCheckbox(page, widget, frame) {
992
- const box = await this.checkboxBox(frame, widget.vendor);
1142
+ const box = frame ? await this.checkboxBox(frame, widget.vendor) : (this.sel.selectors[widget.vendor].box?.length ? widget.el : null);
993
1143
  if (box)
994
1144
  await this.move(page, box);
995
- else
996
- await this.performSmoothMove(page, ...await this.findCheckboxOnScreen(widget.el));
1145
+ else {
1146
+ const at = await this.findCheckboxOnScreen(page, widget.el);
1147
+ if (!at)
1148
+ return;
1149
+ await this.performSmoothMove(page, ...at);
1150
+ }
997
1151
  await this.human.pause(kinds_1.PauseKind.BETWEEN);
998
1152
  await this.gesture(() => this.human.click(page, this.human.at));
999
1153
  await this.awaitWhatTheCheckboxOpened(page);
@@ -1003,7 +1157,7 @@ class CaptchaKrakenSolver {
1003
1157
  * a closed shadow root, and a frame can refuse access; both are found on screen instead.
1004
1158
  */
1005
1159
  async checkboxBox(frame, vendor) {
1006
- const box = selectors_1.SELECTORS[vendor].box;
1160
+ const box = this.sel.selectors[vendor].box;
1007
1161
  if (!frame || !box?.length)
1008
1162
  return null;
1009
1163
  try {
@@ -1014,8 +1168,28 @@ class CaptchaKrakenSolver {
1014
1168
  return null;
1015
1169
  }
1016
1170
  }
1017
- /** One capture and OpenCV: a point inside the tick box, in page coordinates. No model is asked. */
1018
- async findCheckboxOnScreen(el) {
1171
+ /**
1172
+ * A point inside the tick box, in page coordinates, or null when the widget passed without drawing one. Turnstile
1173
+ * shows "Verifying…" before its box, so an empty capture is looked at again until `detectionTimeoutMs`.
1174
+ */
1175
+ async findCheckboxOnScreen(page, el) {
1176
+ const deadline = Date.now() + (this.config.detectionTimeoutMs ?? exports.SOLVE_DEFAULTS.detectionTimeoutMs);
1177
+ for (;;) {
1178
+ const at = await this.checkboxOnScreen(el);
1179
+ if (at)
1180
+ return at;
1181
+ if (await this.isCaptchaSolved(page))
1182
+ return null;
1183
+ if (Date.now() >= deadline) {
1184
+ const e = new Error('UNSUPPORTED_CAPTCHA: the checkbox widget shows no tick box to click');
1185
+ e.unsupported = true;
1186
+ throw e;
1187
+ }
1188
+ await delay(DETECTION_POLL_MS);
1189
+ }
1190
+ }
1191
+ /** One capture and the element detector, no model: a point inside the tick box, or null when none is drawn. */
1192
+ async checkboxOnScreen(el) {
1019
1193
  const box = await el.boundingBox();
1020
1194
  if (!box)
1021
1195
  throw new Error('Could not get bounding box of captcha element');
@@ -1030,11 +1204,8 @@ class CaptchaKrakenSolver {
1030
1204
  finally {
1031
1205
  unlink(shot);
1032
1206
  }
1033
- if (!found) {
1034
- const e = new Error('UNSUPPORTED_CAPTCHA: the checkbox widget shows no tick box to click');
1035
- e.unsupported = true;
1036
- throw e;
1037
- }
1207
+ if (!found)
1208
+ return null;
1038
1209
  const [x, y, w, h] = found.map((v) => v / scale);
1039
1210
  return [box.x + x + w * (0.3 + Math.random() * 0.4), box.y + y + h * (0.3 + Math.random() * 0.4)];
1040
1211
  }
@@ -1054,7 +1225,7 @@ class CaptchaKrakenSolver {
1054
1225
  });
1055
1226
  }
1056
1227
  getVerifyButton(scope) {
1057
- return this.findControl(scope, selectors_1.SUBMIT_SELECTORS);
1228
+ return this.findControl(scope, this.sel.submit);
1058
1229
  }
1059
1230
  /** Any element at `selector` carrying a non-blank value; response fields are hidden, so no visibility filter. */
1060
1231
  async hasValue(scopes, selector) {
@@ -1069,19 +1240,16 @@ class CaptchaKrakenSolver {
1069
1240
  const frame = await frameOf(checkbox);
1070
1241
  return !!frame && (await visible(frame, [checked])).length > 0;
1071
1242
  }
1072
- bannerIsFatalAfterRetry(kind) {
1073
- return kind === kinds_1.RecaptchaBanner.SELECT_MORE || kind === kinds_1.RecaptchaBanner.REJECTED;
1074
- }
1075
1243
  /** Which verdict banner the open challenge shows; `dynamic-more` is the dynamic board's normal flow, not an error. */
1076
1244
  async bannerKind(page) {
1077
- const probes = selectors_1.WIDGET_PROBES.filter((p) => p.role === kinds_1.FrameRole.CHALLENGE && selectors_1.SELECTORS[p.vendor].banners);
1078
- const scopes = await hostScopes(page);
1245
+ const probes = this.sel.widgetProbes.filter((p) => p.role === kinds_1.FrameRole.CHALLENGE && this.sel.selectors[p.vendor].banners);
1246
+ const scopes = await hostScopes(page, this.sel.enclave);
1079
1247
  const kinds = await Promise.all(probes.map(async (p) => {
1080
1248
  const [at] = await visibleIn(scopes, [p.selector]);
1081
1249
  const frame = at && await frameOf(at);
1082
1250
  if (!frame)
1083
1251
  return null;
1084
- const shown = await Promise.all((selectors_1.SELECTORS[p.vendor].banners ?? []).map(async ([sel, kind]) => (await this.visibleWithText(frame, sel)) ? kind : null));
1252
+ const shown = await Promise.all((this.sel.selectors[p.vendor].banners ?? []).map(async ([sel, kind]) => (await this.visibleWithText(frame, sel)) ? kind : null));
1085
1253
  return shown.find((k) => k !== null) ?? null;
1086
1254
  }));
1087
1255
  return kinds.find((k) => k !== null) ?? null;
@@ -1089,14 +1257,14 @@ class CaptchaKrakenSolver {
1089
1257
  /** The vendor's own done signal: a response token, a painted success state, or a checked box. */
1090
1258
  async isCaptchaSolved(page) {
1091
1259
  try {
1092
- const checkboxes = selectors_1.WIDGET_PROBES.filter((p) => p.role === kinds_1.FrameRole.CHECKBOX && selectors_1.SELECTORS[p.vendor].checked);
1093
- const scopes = await hostScopes(page);
1260
+ const checkboxes = this.sel.widgetProbes.filter((p) => p.role === kinds_1.FrameRole.CHECKBOX && this.sel.selectors[p.vendor].checked);
1261
+ const scopes = await hostScopes(page, this.sel.enclave);
1094
1262
  const signals = await Promise.all([
1095
- ...selectors_1.RESPONSE_SELECTORS.map((s) => this.hasValue(scopes, s)),
1096
- visibleIn(scopes, selectors_1.ACCEPTED_SELECTORS).then((found) => found.length > 0),
1263
+ ...this.sel.response.map((s) => this.hasValue(scopes, s)),
1264
+ visibleIn(scopes, this.sel.accepted).then((found) => found.length > 0),
1097
1265
  ...checkboxes.map(async (p) => {
1098
1266
  const [at] = await visibleIn(scopes, [p.selector]);
1099
- return !!at && this.checkedIn(at, selectors_1.SELECTORS[p.vendor].checked);
1267
+ return !!at && this.checkedIn(at, this.sel.selectors[p.vendor].checked);
1100
1268
  }),
1101
1269
  ]);
1102
1270
  return signals.some(Boolean);
@@ -1108,15 +1276,15 @@ class CaptchaKrakenSolver {
1108
1276
  /** A next round has painted, as opposed to the answered frame animating closed. */
1109
1277
  async isChallengeFreshlyRendered(page) {
1110
1278
  try {
1111
- const probes = selectors_1.WIDGET_PROBES.filter((p) => p.role === kinds_1.FrameRole.CHALLENGE && selectors_1.SELECTORS[p.vendor].fresh);
1112
- const scopes = await hostScopes(page);
1279
+ const probes = this.sel.widgetProbes.filter((p) => p.role === kinds_1.FrameRole.CHALLENGE && this.sel.selectors[p.vendor].fresh);
1280
+ const scopes = await hostScopes(page, this.sel.enclave);
1113
1281
  const fresh = await Promise.all(probes.map(async (p) => {
1114
1282
  const [at] = await visibleIn(scopes, [p.selector]);
1115
1283
  const el = at && await handleOf(at);
1116
1284
  if (!el || (this.lastSubmitFrameHash && (await this.elementFrameHash(el)) === this.lastSubmitFrameHash))
1117
1285
  return false;
1118
1286
  const frame = await el.contentFrame();
1119
- return !!frame && this.visibleWithText(frame, selectors_1.SELECTORS[p.vendor].fresh);
1287
+ return !!frame && this.visibleWithText(frame, this.sel.selectors[p.vendor].fresh);
1120
1288
  }));
1121
1289
  return fresh.some(Boolean);
1122
1290
  }
@@ -1141,7 +1309,7 @@ class CaptchaKrakenSolver {
1141
1309
  const bg = getComputedStyle(el).backgroundImage;
1142
1310
  return !!bg && bg !== 'none' && !/url\(["']?["']?\)/.test(bg);
1143
1311
  });
1144
- }, { prompt: s.fresh ?? '', images: (s.images ?? []).join(',') }, { timeout: this.config.hcaptchaImagesTimeoutMs ?? 3000 });
1312
+ }, { prompt: s.fresh ?? '', images: (s.images ?? []).join(',') }, { timeout: this.config.boardImagesTimeoutMs ?? this.config.hcaptchaImagesTimeoutMs ?? 3000 });
1145
1313
  }
1146
1314
  catch { /* timed out or detached mid-load; screenshot anyway */ }
1147
1315
  }
@@ -1167,12 +1335,12 @@ class CaptchaKrakenSolver {
1167
1335
  return [];
1168
1336
  }
1169
1337
  const blob = names.join(' ');
1170
- return selectors_1.VENDORS.filter(([, s]) => s.hosts.some((h) => blob.includes(h))).map(([vendor]) => vendor);
1338
+ return this.sel.vendors.filter(([, s]) => s.hosts.some((h) => blob.includes(h))).map(([vendor]) => vendor);
1171
1339
  }
1172
1340
  /** Vendors showing a frame no selector names. Only asked once detection has found nothing, and a passive frame (an invisible badge, an enclave) is not a widget, so what is left is markup the table no longer matches. */
1173
1341
  async unmatchedVendorFrames(page) {
1174
- const scopes = await hostScopes(page);
1175
- const counts = await Promise.all(selectors_1.VENDORS.map(async ([vendor, s]) => {
1342
+ const scopes = await hostScopes(page, this.sel.enclave);
1343
+ const counts = await Promise.all(this.sel.vendors.map(async ([vendor, s]) => {
1176
1344
  if (!s.hosts.length)
1177
1345
  return null;
1178
1346
  const [frames, passive] = await Promise.all([visibleIn(scopes, s.hosts.map((h) => `iframe[src*="${h}"]`)), visibleIn(scopes, s.passive ?? [])]);
@@ -1197,12 +1365,12 @@ class CaptchaKrakenSolver {
1197
1365
  * count: without them this port failed fast in under a second on every GeeTest/Yidun page and disagreed with Python in Tier 3.
1198
1366
  */
1199
1367
  async hasInteractiveWidgetInDom(page) {
1200
- return (await Promise.all(selectors_1.WIDGET_PROBES.map((p) => page.locator(p.selector).count().catch(() => 0)))).some((n) => n > 0);
1368
+ return (await Promise.all(this.sel.widgetProbes.map((p) => page.locator(p.selector).count().catch(() => 0)))).some((n) => n > 0);
1201
1369
  }
1202
1370
  /** Open challenges first, then unsolved checkboxes, then the inline vendors; every probe is queried at once. */
1203
1371
  async detectCaptcha(page) {
1204
- const scopes = await hostScopes(page);
1205
- const found = await Promise.all(selectors_1.WIDGET_PROBES.map(async (p) => {
1372
+ const scopes = await hostScopes(page, this.sel.enclave);
1373
+ const found = await Promise.all(this.sel.widgetProbes.map(async (p) => {
1206
1374
  const [at] = await visibleIn(scopes, [p.selector]);
1207
1375
  return at && !(await this.alreadyAccepted(scopes, at, p)) ? { ...p, at } : null;
1208
1376
  }));
@@ -1212,7 +1380,7 @@ class CaptchaKrakenSolver {
1212
1380
  }
1213
1381
  /** A checkbox the vendor has already accepted is not a captcha to solve. */
1214
1382
  async alreadyAccepted(scopes, at, { role, vendor }) {
1215
- const { response, checked } = selectors_1.SELECTORS[vendor];
1383
+ const { response, checked } = this.sel.selectors[vendor];
1216
1384
  if (role !== kinds_1.FrameRole.CHECKBOX)
1217
1385
  return false;
1218
1386
  const [token, box] = await Promise.all([!!response && this.hasValue(scopes, response), !!checked && this.checkedIn(at, checked)]);
@@ -1220,14 +1388,14 @@ class CaptchaKrakenSolver {
1220
1388
  }
1221
1389
  loraName(cliRoot) {
1222
1390
  if (this.loraNameCache === null)
1223
- this.loraNameCache = (0, model_name_1.resolveLoraName)({ cliRoot, baseUrl: process.env.VLLM_BASE_URL });
1391
+ this.loraNameCache = (0, model_name_1.resolveLoraName)({ cliRoot, baseUrl: this.baseUrl });
1224
1392
  return this.loraNameCache;
1225
1393
  }
1226
- /** The `--model` argv, or nothing: without VLLM_BASE_URL the CLI can see a credentials file this process cannot. */
1394
+ /** The `--model` argv, or nothing: without a base URL the CLI can see a credentials file this process cannot. */
1227
1395
  modelName(cliRoot) {
1228
1396
  const explicit = this.config.model ?? process.env.CAPTCHA_LORA_NAME;
1229
- const name = explicit || (process.env.VLLM_BASE_URL ? this.loraName(cliRoot) : undefined);
1230
- if (name && !this.legacyModelWarned && (0, model_name_1.isHostedEndpoint)(process.env.VLLM_BASE_URL)
1397
+ const name = explicit || (this.baseUrl ? this.loraName(cliRoot) : undefined);
1398
+ if (name && !this.legacyModelWarned && (0, model_name_1.isHostedEndpoint)(this.baseUrl)
1231
1399
  && (0, model_name_1.isLegacyHostedName)(name, cliRoot)) {
1232
1400
  this.legacyModelWarned = true;
1233
1401
  console.error(`[captchakraken] ${model_name_1.LEGACY_MODEL_WARNING.replace('{model}', name)}`);
@@ -1441,17 +1609,31 @@ class CaptchaKrakenSolver {
1441
1609
  unlink(f);
1442
1610
  }
1443
1611
  }
1612
+ /**
1613
+ * Hold the next ask until every tile has painted: a model asked about a half-loaded board answers it.
1614
+ *
1615
+ * Past the timeout a blank tile that has stopped changing is taken as painted (a pale photo, or a tile whose
1616
+ * image never came), and a board with no grid to read is let through. A tile still painting is waited on until
1617
+ * GRID_LOAD_CAP_FACTOR times that, or the solve's deadline.
1618
+ */
1444
1619
  async waitForGridCellsLoaded(captchaElement, opts) {
1445
1620
  const interval = opts?.intervalMs ?? this.config.gridLoadPollIntervalMs ?? 250;
1446
1621
  const timeout = opts?.timeoutMs ?? this.config.gridLoadTimeoutMs ?? 8000;
1447
- return (await this.pollFrames(captchaElement, timeout, interval, async (frames) => {
1622
+ const start = Date.now();
1623
+ const verdict = await this.pollFrames(captchaElement, timeout * exports.GRID_LOAD_CAP_FACTOR, interval, async (frames) => {
1448
1624
  if (frames.length < 2)
1449
1625
  return undefined;
1626
+ const late = Date.now() - start >= timeout;
1450
1627
  const res = await this.runCvTool('grid-cell-states', { a: frames[0], b: frames[1] }, ['grid-cell-states', frames[0], frames[1]]);
1451
- const ok = res && res.grid !== null && Array.isArray(res.loaded) && res.loaded.length > 0
1452
- && Array.isArray(res.empty) && res.empty.length === 0 && Array.isArray(res.changing) && res.changing.length === 0;
1453
- return ok ? true : undefined;
1454
- })) === true;
1628
+ if (!res || res.grid === null)
1629
+ return late ? false : undefined;
1630
+ if (!Array.isArray(res.loaded) || !res.loaded.length || !Array.isArray(res.changing) || res.changing.length)
1631
+ return undefined;
1632
+ return !Array.isArray(res.empty) || !res.empty.length || late ? true : undefined;
1633
+ });
1634
+ if (verdict !== true)
1635
+ console.log('[captchakraken] [recaptcha-grid] the board never finished loading; asking about it as it stands');
1636
+ return verdict === true;
1455
1637
  }
1456
1638
  async getGridBoxes(captchaElement) {
1457
1639
  const f = tmp('findgrid');
@@ -1480,12 +1662,25 @@ class CaptchaKrakenSolver {
1480
1662
  const [x1, y1, x2, y2] = session.gridBoxes[cell - 1];
1481
1663
  return { x: session.elementBox.x + (x1 + x2) / 2 * session.scaleX, y: session.elementBox.y + (y1 + y2) / 2 * session.scaleY };
1482
1664
  }
1665
+ /** Rest the hand on a tile: reach it once, then drift a few pixels inside it. */
1483
1666
  async hoverCell(page, session, cell) {
1484
1667
  if (!this.human.hovers)
1485
1668
  return;
1486
- const [x1, y1, x2, y2] = session.gridBoxes[0];
1487
- const center = this.cellCenterPage(cell, session);
1488
- await this.performSmoothMove(page, center.x + (Math.random() - 0.5) * (x2 - x1) * session.scaleX * 0.4, center.y + (Math.random() - 0.5) * (y2 - y1) * session.scaleY * 0.4);
1669
+ const [x1, y1, x2, y2] = session.gridBoxes[cell - 1];
1670
+ const left = session.elementBox.x + x1 * session.scaleX, top = session.elementBox.y + y1 * session.scaleY;
1671
+ const right = session.elementBox.x + x2 * session.scaleX, bottom = session.elementBox.y + y2 * session.scaleY;
1672
+ let [x, y] = this.human.at;
1673
+ if (x >= left && x <= right && y >= top && y <= bottom) {
1674
+ x += (0, humanize_js_1.gauss)(exports.IDLE_DRIFT_PX);
1675
+ y += (0, humanize_js_1.gauss)(exports.IDLE_DRIFT_PX);
1676
+ }
1677
+ else {
1678
+ const center = this.cellCenterPage(cell, session);
1679
+ x = center.x + (Math.random() - 0.5) * (right - left) * 0.4;
1680
+ y = center.y + (Math.random() - 0.5) * (bottom - top) * 0.4;
1681
+ }
1682
+ const inset = Math.min(4, (right - left) / 4, (bottom - top) / 4);
1683
+ await this.performSmoothMove(page, Math.min(Math.max(x, left + inset), right - inset), Math.min(Math.max(y, top + inset), bottom - inset));
1489
1684
  }
1490
1685
  /** Per-cell state with the session's cached boxes: a blank mid-refresh frame has no lattice to detect. */
1491
1686
  async gridCellStates(session, frameA, frameB) {
@@ -1502,8 +1697,13 @@ class CaptchaKrakenSolver {
1502
1697
  hoverLoop(page, session, cells) {
1503
1698
  if (!cells.length)
1504
1699
  return undefined;
1505
- let i = 0;
1506
- return async () => { await this.hoverCell(page, session, cells[i++ % cells.length]).catch(() => { }); };
1700
+ let due = 0;
1701
+ return async () => {
1702
+ if (Date.now() < due)
1703
+ return;
1704
+ due = Date.now() + exports.IDLE_DRIFT_EVERY_MS[0] + Math.random() * (exports.IDLE_DRIFT_EVERY_MS[1] - exports.IDLE_DRIFT_EVERY_MS[0]);
1705
+ await this.hoverCell(page, session, cells[0]).catch(() => { });
1706
+ };
1507
1707
  }
1508
1708
  /**
1509
1709
  * `chipped`: the photos were kept, press Verify. `loading`: those tiles are being swapped, read the board again.
@@ -1579,6 +1779,12 @@ class CaptchaKrakenSolver {
1579
1779
  });
1580
1780
  continue;
1581
1781
  }
1782
+ if (action.action === kinds_1.ActionKind.REFRESH) {
1783
+ // A new board replaces this one: nothing clicked on it is submitted, and the next loop reads the new one.
1784
+ const frame = await captchaElement.contentFrame();
1785
+ const refreshed = !!frame && await this.executeRefresh(page, captchaElement, frame, action, elementBox);
1786
+ return { didInteract: refreshed || performedAction, tokenUsage: allTokenUsage };
1787
+ }
1582
1788
  if (action.action !== kinds_1.ActionKind.CLICK) {
1583
1789
  console.warn(`[recaptcha-grid] round ${round}: unexpected action '${action.action}'; re-solving.`);
1584
1790
  continue;
@@ -1592,6 +1798,11 @@ class CaptchaKrakenSolver {
1592
1798
  const clickedThisRound = [];
1593
1799
  for (const bbox of bboxes) {
1594
1800
  const cell = this.bboxToCell(bbox, session.gridBoxes, session.screenshotW, session.screenshotH);
1801
+ if (cell != null && clickedThisRound.includes(cell)) {
1802
+ // Two points in one tile: a second press deselects what the first selected.
1803
+ console.log(`[recaptcha-grid] round ${round}: cell ${cell} is already clicked; not pressing it again.`);
1804
+ continue;
1805
+ }
1595
1806
  await this.executeClick(page, captchaElement, { action: kinds_1.ActionKind.CLICK, target_bounding_box: bbox }, elementBox);
1596
1807
  if (cell != null) {
1597
1808
  clickedOrder.push(cell);
@@ -1969,7 +2180,7 @@ class CaptchaKrakenSolver {
1969
2180
  /** Slice a burst into keyframes and solve them in one model request. The frame number is the guard. */
1970
2181
  async getAnimatedSolution(framesDir) {
1971
2182
  const { cliRoot, py } = this.resolveCli();
1972
- const { apiKey = process.env.CAPTCHA_KRAKEN_API_KEY ?? process.env.VLLM_API_KEY } = this.config;
2183
+ const { apiKey } = this;
1973
2184
  const m = this.modelName(cliRoot);
1974
2185
  const args = ['-m', 'captchakraken.cli', 'solve-animated', '--frames-dir', framesDir,
1975
2186
  '--fps', String(this.lastBurstFps ?? this.config.videoBurstFps ?? 10), ...(m ? ['--model', m] : [])];
@@ -1991,7 +2202,7 @@ class CaptchaKrakenSolver {
1991
2202
  }
1992
2203
  /** The routing values and the credential travel in the environment, never in argv. */
1993
2204
  solveEnvironment(cliRoot, apiKey) {
1994
- return (0, cli_invocation_1.solveEnv)(cliEnv(cliRoot, this.routingEnv()), apiKey, this.resampleLevel);
2205
+ return (0, cli_invocation_1.solveEnv)(cliEnv(cliRoot, this.routingEnv()), apiKey);
1995
2206
  }
1996
2207
  /** Exit 2 is an unsupported puzzle, exit 3 the hosted API's own refusal; both are relayed, not reworded. */
1997
2208
  cliError(error, unsupportedMessage, failMessage) {
@@ -2077,27 +2288,53 @@ class CaptchaKrakenSolver {
2077
2288
  console.log(`[animated] widget never matched the chosen keyframe within ${timeout}ms (closest diff=${best.toFixed(4)}); clicking on the model's coordinates anyway, and recording afresh next round`);
2078
2289
  return false;
2079
2290
  }
2291
+ dropPointerHistory() {
2292
+ for (const screen of this.pointerHistory)
2293
+ unlink(screen);
2294
+ this.pointerHistory = [];
2295
+ }
2296
+ /** A click board with no Verify submits itself at a count, so a plan of any other count leaves its tiles ticked
2297
+ * and never submits, and the next plan's clicks untick them. */
2298
+ keepPlansOfThisManyClicks(plan) {
2299
+ const clicks = (actions) => actions.length && actions.every((a) => a?.action === kinds_1.ActionKind.CLICK)
2300
+ ? actions.reduce((n, a) => n + (a.target_bounding_boxes?.length ?? 1), 0)
2301
+ : null;
2302
+ const count = clicks(plan);
2303
+ if (count === null)
2304
+ return;
2305
+ this.fallbackPlans = this.fallbackPlans.filter((other) => [null, count].includes(clicks(other)));
2306
+ }
2080
2307
  resetSolveState() {
2081
2308
  this.solutionCache.clear();
2309
+ this.fallbackPlans = [];
2310
+ this.fallbackBoardHash = null;
2311
+ this.fallbackArmed = false;
2312
+ this.dropPointerHistory();
2082
2313
  this.repeatedAnswerSeen = false;
2083
2314
  this.knownAnimated = false;
2084
2315
  this.animatedProbeDone = false;
2085
2316
  this.discardAnimatedPlan();
2086
2317
  this.lastSubmitFrameHash = null;
2318
+ this.submittedOnBoard = false;
2087
2319
  this.keyframeMode = null;
2088
2320
  this.keyframeSteadyScreens = 0;
2089
2321
  this.solveDeadlineAt = 0;
2090
2322
  this.lastAnswerSig = null;
2323
+ this.triedAnswers.clear();
2324
+ this.refreshedBoards.clear();
2091
2325
  this.noProgressRounds = 0;
2092
- this.resampleLevel = 0;
2093
2326
  this.videoBudgetMs = 0;
2094
2327
  this.videoBudgetGranted = false;
2095
2328
  }
2096
- /** Keyed on the retry mode too: the missed-tiles answer legitimately overlaps the previous one. */
2097
- static answerSignature(actions, retryMode) {
2329
+ /**
2330
+ * Keyed on the BOARD and the retry mode, not the answer alone. A paged puzzle's next arrow sits at the
2331
+ * same point on every page, so a signature over the actions alone would call a correct page turn a
2332
+ * repeated answer and stop pressing it.
2333
+ */
2334
+ static answerSignature(actions, retryMode, boardHash) {
2098
2335
  const round3 = (v) => typeof v === 'number' ? Math.round(v * 1000) / 1000 : Array.isArray(v) ? v.map(round3) : v ?? null;
2099
2336
  try {
2100
- return JSON.stringify([retryMode ?? null, actions.map((a) => [a?.action ?? null, round3(a?.target_bounding_boxes),
2337
+ return JSON.stringify([boardHash, retryMode ?? null, actions.map((a) => [a?.action ?? null, round3(a?.target_bounding_boxes),
2101
2338
  round3(a?.target_bounding_box), round3(a?.target_coordinates), round3(a?.source_bounding_box), a?.text ?? null])]);
2102
2339
  }
2103
2340
  catch {
@@ -2110,29 +2347,70 @@ class CaptchaKrakenSolver {
2110
2347
  if (plan?.response)
2111
2348
  this.animatedPlan = { burstDir: plan.burstDir, response: null };
2112
2349
  }
2113
- /** True when this answer already ran and changed nothing: resample, and let the recording path have a go. */
2114
- noteAnswer(actions, retryMode) {
2115
- const sig = CaptchaKrakenSolver.answerSignature(actions, retryMode);
2116
- if (sig !== null && sig === this.lastAnswerSig) {
2350
+ /** True when this answer already ran and changed nothing: re-ask, and let the recording path have a go. */
2351
+ noteAnswer(actions, retryMode, boardHash = null) {
2352
+ const sig = CaptchaKrakenSolver.answerSignature(actions, retryMode, boardHash);
2353
+ // A refresh asks for a different board, so the same refresh twice is two boards, not a refused answer. An
2354
+ // answer that could not be signed could not be read either, and is not looked into.
2355
+ if (sig !== null && actions.some((a) => a?.action === kinds_1.ActionKind.REFRESH)) {
2356
+ this.noProgressRounds = 0;
2357
+ this.lastAnswerSig = null;
2358
+ if (boardHash !== null)
2359
+ this.refreshedBoards.add(boardHash);
2360
+ return false;
2361
+ }
2362
+ // A vendor can deal the same picture again, which clears the plans but not what was pressed on it. A paged
2363
+ // puzzle is exempt: coming back to a page after the others is a new question, because the history differs.
2364
+ const tried = sig !== null && boardHash !== null && !this.pointerHistory.length && this.triedAnswers.has(sig);
2365
+ if (sig !== null && (sig === this.lastAnswerSig || tried)) {
2117
2366
  this.noProgressRounds++;
2118
2367
  console.log(`[no-progress] the model returned the same answer again (${this.noProgressRounds} in a row) — the previous one already ran and changed nothing`);
2119
- this.resampleLevel++;
2120
- // A raised sample never reaches the wire while the cached animated answer stands in front of it.
2368
+ // A re-ask never reaches the wire while the cached animated answer stands in front of it.
2121
2369
  this.invalidateAnimatedAnswer();
2122
2370
  this.repeatedAnswerSeen = true;
2371
+ // The plan at the front of the queue ran and changed nothing. A board that said nothing to its server has
2372
+ // refused nothing, though, and another plan is one more press.
2373
+ this.fallbackArmed = !this.boardStayedSilent();
2123
2374
  return true;
2124
2375
  }
2125
2376
  this.noProgressRounds = 0;
2126
2377
  this.lastAnswerSig = sig;
2378
+ if (sig !== null)
2379
+ this.triedAnswers.add(sig);
2380
+ this.exchangesAtAnswer = this.verdicts?.exchanges ?? 0;
2127
2381
  return false;
2128
2382
  }
2383
+ /** Whether the round after a repeat can differ. At temperature 0 the same question gets the same answer, so it
2384
+ * takes a different question (a retry mode, a film) or one of the model's other plans. */
2385
+ anotherMoveLeft(retryMode) {
2386
+ return !!retryMode || this.knownAnimated
2387
+ || (this.config.videoSolveEnabled !== false && this.repeatedAnswerSeen && !this.animatedProbeDone)
2388
+ || (this.fallbackArmed && this.fallbackPlans.length > 0);
2389
+ }
2390
+ boardStayedSilent() {
2391
+ const log = this.verdicts;
2392
+ return !!log?.listening && log.exchanges === this.exchangesAtAnswer;
2393
+ }
2394
+ /** The board's identity, always from a plain photograph: a filmed frame of an unchanged board differs in bytes. */
2395
+ async boardHash(element, shot, filmed) {
2396
+ if (!filmed)
2397
+ return sha1(shot);
2398
+ const plain = tmp('board');
2399
+ try {
2400
+ await this.shot(element, plain, this.config.elementScreenshotTimeoutMs ?? 8000);
2401
+ return sha1(plain);
2402
+ }
2403
+ finally {
2404
+ fs.rmSync(plain, { force: true });
2405
+ }
2406
+ }
2129
2407
  /** A cache hit costs no inference, and means the answer already ran: the board cycles. */
2130
2408
  async answerFor(cacheKey, ask) {
2131
2409
  const cached = this.solutionCache.get(cacheKey);
2132
2410
  if (cached) {
2133
2411
  console.log('[dedup] this exact picture was already answered and the answer already ran — the challenge is cycling, not still; re-solving it as animated.');
2134
2412
  this.repeatedAnswerSeen = true;
2135
- return { actions: cached.actions, token_usage: [] };
2413
+ return { ...cached, token_usage: [] };
2136
2414
  }
2137
2415
  const fresh = await ask();
2138
2416
  this.solutionCache.set(cacheKey, fresh);
@@ -2206,14 +2484,20 @@ class CaptchaKrakenSolver {
2206
2484
  catch {
2207
2485
  cacheKey = null;
2208
2486
  }
2209
- const ask = () => this.askModel(imagePath, puzzleSource, retryMode, textMode);
2487
+ const ask = () => this.askModel(imagePath, puzzleSource, retryMode, textMode).catch((e) => {
2488
+ // No answer is an empty plan, which presses Skip for a new board. An empty text box is a wrong answer.
2489
+ if (!e?.unsupported || textMode)
2490
+ throw e;
2491
+ return { actions: [], token_usage: [] };
2492
+ });
2210
2493
  return cacheKey ? this.answerFor(cacheKey, ask) : ask();
2211
2494
  }
2212
2495
  /** One inference through the CLI. */
2213
2496
  async askModel(imagePath, puzzleSource = kinds_1.Vendor.UNKNOWN, retryMode = null, textMode = false) {
2214
2497
  const { cliRoot, py } = this.resolveCli();
2215
- const { apiKey = process.env.CAPTCHA_KRAKEN_API_KEY ?? process.env.VLLM_API_KEY } = this.config;
2216
- const args = (0, cli_invocation_1.buildSolveArgs)({ imagePath, model: this.modelName(cliRoot), puzzleSource, retryMode, textMode, expert: this.config.expert });
2498
+ const { apiKey } = this;
2499
+ const args = (0, cli_invocation_1.buildSolveArgs)({ imagePath, model: this.modelName(cliRoot), puzzleSource, retryMode, textMode,
2500
+ expert: this.config.expert, history: this.pointerHistory });
2217
2501
  console.log(`Executing CaptchaKraken CLI: ${(0, cli_invocation_1.redactCommand)([py, ...args].join(' '), apiKey)}`);
2218
2502
  try {
2219
2503
  const { stdout, stderr } = await execFileAsync(py, args, { cwd: cliRoot, env: this.solveEnvironment(cliRoot, apiKey), maxBuffer: 10 * 1024 * 1024, timeout: this.askTimeoutMs(), killSignal: 'SIGKILL' });
@@ -2222,24 +2506,7 @@ class CaptchaKrakenSolver {
2222
2506
  console.error('CaptchaKraken CLI stderr:', stderr);
2223
2507
  if (!stdout.trim())
2224
2508
  throw new Error(`CLI returned empty output. Stderr: ${stderr}`);
2225
- let actions = [];
2226
- let tokenUsage = [];
2227
- for (const line of stdout.trim().split('\n')) {
2228
- try {
2229
- const parsed = JSON.parse(line);
2230
- if (parsed.actions !== undefined && parsed.token_usage !== undefined) {
2231
- actions = parsed.actions;
2232
- tokenUsage = parsed.token_usage;
2233
- break;
2234
- }
2235
- if (Array.isArray(parsed))
2236
- actions = parsed;
2237
- else if (parsed.action && (parsed.target_bounding_box || parsed.target_coordinates || parsed.action === kinds_1.ActionKind.WAIT))
2238
- actions = [parsed];
2239
- }
2240
- catch { /* not json or not relevant */ }
2241
- }
2242
- return { actions, token_usage: tokenUsage };
2509
+ return (0, cli_invocation_1.parseSolveOutput)(stdout);
2243
2510
  }
2244
2511
  catch (error) {
2245
2512
  throw this.cliError(error, 'Cannot solve this kind of captcha', 'Failed to execute captcha solver CLI');
@@ -2277,7 +2544,7 @@ class CaptchaKrakenSolver {
2277
2544
  }
2278
2545
  // While a speculative burst is filming, the film is the better answer to "the screen changed".
2279
2546
  if (opts.recordingInFlight)
2280
- return { actions: response.actions, token_usage: mergedUsage };
2547
+ return { ...response, token_usage: mergedUsage };
2281
2548
  let changedDuringInference = 0;
2282
2549
  for (let i = 0; i < maxReSolves; i++) {
2283
2550
  if (!(await this.captchaFrameChangedSince(captchaElement, currentPath, threshold)))
@@ -2285,7 +2552,7 @@ class CaptchaKrakenSolver {
2285
2552
  if (++changedDuringInference >= 2 && !this.actedOnBoard && untouched()) {
2286
2553
  this.repeatedAnswerSeen = true;
2287
2554
  console.log('[freshness] the frame changed twice during inference with nothing clicked — this board cycles; recording it rather than re-solving a screen that has gone.');
2288
- return { actions: response.actions, token_usage: mergedUsage };
2555
+ return { ...response, token_usage: mergedUsage };
2289
2556
  }
2290
2557
  await this.waitForBoardPainted(captchaElement);
2291
2558
  const fresh = tmp(`freshsolve_${Date.now()}`);
@@ -2301,7 +2568,7 @@ class CaptchaKrakenSolver {
2301
2568
  response = await runQuery(currentPath);
2302
2569
  mergedUsage.push(...response.token_usage);
2303
2570
  }
2304
- return { actions: response.actions, token_usage: mergedUsage };
2571
+ return { ...response, token_usage: mergedUsage };
2305
2572
  }
2306
2573
  finally {
2307
2574
  unlink(liveAnchor);
@@ -2476,6 +2743,17 @@ class CaptchaKrakenSolver {
2476
2743
  return [action.target_coordinates[0] * elementBox.width, action.target_coordinates[1] * elementBox.height];
2477
2744
  return null;
2478
2745
  }
2746
+ /**
2747
+ * The press point of a hold: the model's own point, the middle of its box. A hold can be on a bar narrower than that
2748
+ * box, and a random spot in it lands off the bar, or close enough to its edge that the hand's drift lets go: the
2749
+ * widget then never fills, and the same answer fails the same way.
2750
+ */
2751
+ holdPointFor(action, elementBox) {
2752
+ if (!action.target_bounding_box)
2753
+ return null;
2754
+ const [minX, minY, maxX, maxY] = action.target_bounding_box;
2755
+ return [(minX + maxX) / 2 * elementBox.width, (minY + maxY) / 2 * elementBox.height];
2756
+ }
2479
2757
  /** Park on the target first, so only a mouse-down separates the right screen from the click. */
2480
2758
  async clickWhenFrameMatches(page, element, action, elementBox, awaitKeyframe) {
2481
2759
  const rel = this.clickPointFor(action, elementBox);
@@ -2498,18 +2776,115 @@ class CaptchaKrakenSolver {
2498
2776
  }
2499
2777
  await this.gesture(() => this.human.click(page, [elementBox.x + rel[0], elementBox.y + rel[1]]));
2500
2778
  }
2501
- async executeDrag(page, _element, action, elementBox) {
2779
+ /**
2780
+ * Press-and-hold until the widget completes: true when it did, false when the cap ran out. Complete is the vendor's
2781
+ * done signal, the widget closing, or its picture having changed (the fill) and then sat still for HOLD_SETTLE_MS.
2782
+ * When the answer names the fill's line, the fill reaching its end is complete too: see `fillReached`. Except on a
2783
+ * widget that `completesItself` (HUMAN's button): it verifies only once it has shown its done state, and letting go
2784
+ * AT full resets it, so only its own signals end the hold.
2785
+ */
2786
+ async executeHold(page, element, action, elementBox, completesItself = false) {
2502
2787
  this.actedOnBoard = true;
2503
- const center = (bbox) => [elementBox.x + ((bbox[0] + bbox[2]) / 2) * elementBox.width, elementBox.y + ((bbox[1] + bbox[3]) / 2) * elementBox.height];
2504
- await this.gesture(() => this.human.drag(page, center(action.source_bounding_box), center(action.target_bounding_box)));
2788
+ const rel = this.holdPointFor(action, elementBox);
2789
+ if (!rel)
2790
+ return false;
2791
+ const first = await this.elementFrameHash(element);
2792
+ const seen = { hash: first, since: Date.now(), changed: false };
2793
+ const shots = action.fill_from_bounding_box && action.fill_to_bounding_box && !completesItself ? [tmp('hold_0'), tmp('hold_1')] : null;
2794
+ const looks = [];
2795
+ const done = async () => {
2796
+ if (await this.isCaptchaSolved(page))
2797
+ return true;
2798
+ if (!(await element.boundingBox().catch(() => null)))
2799
+ return true;
2800
+ const n = looks.length;
2801
+ if (shots && (await this.fillReached(element, action, elementBox, shots, looks)))
2802
+ return true;
2803
+ // A look that just shot the widget is its frame too: a second screenshot per look slowed the looks enough for
2804
+ // a fast fill to pass the mark unseen between two.
2805
+ const fresh = shots && looks.length > n ? shots[n === 0 ? 0 : 1] : null;
2806
+ const h = fresh ? sha1(fresh) : await this.elementFrameHash(element);
2807
+ if (h !== seen.hash) {
2808
+ Object.assign(seen, { hash: h, since: Date.now(), changed: seen.changed || h !== first });
2809
+ return false;
2810
+ }
2811
+ return seen.changed && Date.now() - seen.since >= exports.HOLD_SETTLE_MS;
2812
+ };
2813
+ try {
2814
+ return await this.gesture(() => (0, humanize_js_1.hold)(this.human, page, [elementBox.x + rel[0], elementBox.y + rel[1]], done, exports.HOLD_MAX_MS));
2815
+ }
2816
+ finally {
2817
+ for (const f of shots ?? [])
2818
+ unlink(f);
2819
+ }
2505
2820
  }
2506
2821
  /**
2507
- * The slider piece's box: the first visible match small enough to be a piece. Runs during detection, so it must
2508
- * not set `actedOnBoard`: marking there disabled `shouldSpeculate` on every slide solve.
2822
+ * One look at a held widget's fill (`fill_tracker.py` behind the CV worker): true once it reaches the answer's `to`,
2823
+ * after waiting out the tracker's `release_in` when the fill arrives between looks.
2824
+ * The first look only shoots the reference, after the press, so the button's own pressed look is not read as fill;
2825
+ * `looks` carries what each later look saw. A look that cannot be shot or read is not a release: the widget's own
2826
+ * signals still end the hold.
2509
2827
  */
2510
- async measurePieceBox(scope, widgetWidth) {
2511
- const boxes = await Promise.all((await handles(await visible(scope, selectors_1.PIECE_SELECTORS))).map((h) => h.boundingBox().catch(() => null)));
2512
- return boxes.find((b) => !!b && b.width >= slide_geometry_1.MIN_PIECE_PX && b.width <= widgetWidth * slide_geometry_1.MAX_PIECE_FRACTION) ?? null;
2828
+ async fillReached(element, action, elementBox, shots, looks) {
2829
+ const [first, current] = shots;
2830
+ const { width: w, height: h } = elementBox;
2831
+ const timeout = this.config.elementScreenshotTimeoutMs ?? 8000;
2832
+ try {
2833
+ if (!looks.length) {
2834
+ const before = performance.now() / 1000;
2835
+ await this.shot(element, first, timeout);
2836
+ looks.push({ t: (before + performance.now() / 1000) / 2, edge: null, k: this.shotScale(first, w) });
2837
+ return false;
2838
+ }
2839
+ // The shot was taken somewhere inside the call; its middle is the best guess, and the release is timed from it.
2840
+ const before = performance.now() / 1000;
2841
+ await this.shot(element, current, timeout);
2842
+ const shotAt = (before + performance.now() / 1000) / 2;
2843
+ const k = looks[0].k;
2844
+ const history = looks.slice(1).map(({ t, edge }) => ({ t, edge }));
2845
+ const at = (b) => [((b[0] + b[2]) / 2) * w * k, ((b[1] + b[3]) / 2) * h * k];
2846
+ const gaps = history.slice(1).map((o, i) => o.t - history[i].t);
2847
+ const request = {
2848
+ first, current, start: at(action.fill_from_bounding_box), end: at(action.fill_to_bounding_box), t: shotAt,
2849
+ history, first_t: looks[0].t, poll_s: gaps.length ? gaps.reduce((x, y) => x + y, 0) / gaps.length : 0.25, tol: SLIDE_TOLERANCE_PX * k,
2850
+ };
2851
+ const res = await this.runCvTool('fill-step', request, ['fill-step', JSON.stringify(request)]);
2852
+ if (!res?.observation)
2853
+ return false;
2854
+ looks.push(res.observation);
2855
+ if (res.release) {
2856
+ console.log(`[hold] the fill reaches its mark ${(res.release_in ?? 0).toFixed(2)}s after a look ${((res.length - res.edge) / k).toFixed(1)}px short of it`);
2857
+ const wait = (shotAt + (res.release_in ?? 0)) * 1000 - performance.now();
2858
+ if (wait > 0)
2859
+ await new Promise((r) => setTimeout(r, wait));
2860
+ }
2861
+ return Boolean(res.release);
2862
+ }
2863
+ catch {
2864
+ return false;
2865
+ }
2866
+ }
2867
+ /** Deal a new puzzle: the vendor's own control, else the model's point on one. False when there is neither. */
2868
+ async executeRefresh(page, element, scope, action, elementBox) {
2869
+ const control = await this.findControl(scope, this.sel.refresh);
2870
+ if (control) {
2871
+ this.actedOnBoard = true;
2872
+ await this.moveAndClick(page, control);
2873
+ }
2874
+ else if (action.target_bounding_box && elementBox) {
2875
+ await this.executeClick(page, element, { action: kinds_1.ActionKind.CLICK, target_bounding_box: action.target_bounding_box }, elementBox);
2876
+ }
2877
+ else {
2878
+ console.log('The model asked for a new puzzle, and there is no control to deal one; ending the round.');
2879
+ return false;
2880
+ }
2881
+ console.log('Asked for a new puzzle instead of solving this one.');
2882
+ return true;
2883
+ }
2884
+ async executeDrag(page, _element, action, elementBox) {
2885
+ this.actedOnBoard = true;
2886
+ const center = (bbox) => [elementBox.x + ((bbox[0] + bbox[2]) / 2) * elementBox.width, elementBox.y + ((bbox[1] + bbox[3]) / 2) * elementBox.height];
2887
+ await this.gesture(() => this.human.drag(page, center(action.source_bounding_box), center(action.target_bounding_box)));
2513
2888
  }
2514
2889
  async findControl(scope, selectors) {
2515
2890
  const [first] = await visible(scope, selectors);
@@ -2521,10 +2896,10 @@ class CaptchaKrakenSolver {
2521
2896
  * BotDetect, whose 280x50 `.BDC_CaptchaDiv` holds only the image while `#captchaCode` sits in a sibling div.
2522
2897
  */
2523
2898
  async answerBox(scope, at) {
2524
- const inside = await this.findControl(scope, selectors_1.TEXT_INPUT_SELECTORS);
2899
+ const inside = await this.findControl(scope, this.sel.textInput);
2525
2900
  if (inside !== null || !at)
2526
2901
  return inside;
2527
- const around = await Promise.all(['ancestor::fieldset[1]', 'ancestor::form[1]'].map((axis) => this.findControl(at.locator(`xpath=${axis}`), selectors_1.TEXT_INPUT_VENDOR_SELECTORS)));
2902
+ const around = await Promise.all(['ancestor::fieldset[1]', 'ancestor::form[1]'].map((axis) => this.findControl(at.locator(`xpath=${axis}`), this.sel.textInputVendor)));
2528
2903
  return around.find((h) => h !== null) ?? null;
2529
2904
  }
2530
2905
  async executeType(page, scope, action, at) {
@@ -2543,101 +2918,86 @@ class CaptchaKrakenSolver {
2543
2918
  console.log(`Typed ${text.length} character(s) into the captcha field.`);
2544
2919
  return true;
2545
2920
  }
2546
- async trackPiece(element, beforePath, afterPath, exclude, travel = 0) {
2547
- try {
2548
- await this.shot(element, afterPath, this.config.elementScreenshotTimeoutMs ?? 8000);
2549
- const res = await this.runCvTool('track-piece', { before: beforePath, after: afterPath, exclude, travel }, ['track-piece', beforePath, afterPath, JSON.stringify(exclude), String(travel)]);
2550
- return res && res.bbox ? { bbox: res.bbox, piece: res.piece ?? null } : null;
2551
- }
2552
- catch (e) {
2553
- console.warn('track-piece failed:', e);
2921
+ /** One look of the slide loop (`slide_tracker.py` behind the CV worker); null when the part cannot be followed. */
2922
+ async slideStep(request) {
2923
+ const res = await this.runCvTool('slide-step', request, ['slide-step', JSON.stringify(request)]);
2924
+ // A reply's `error` is the distance still to go; a failure is `{error: why}` with no position.
2925
+ if (!res?.pos) {
2926
+ console.warn(`[slide] ${res?.error ?? 'the tracker gave no answer'}; released where the handle is`);
2554
2927
  return null;
2555
2928
  }
2929
+ return res;
2556
2930
  }
2557
2931
  shotScale(shot, cssWidth) {
2558
2932
  const dims = readPngDimensions(shot);
2559
2933
  return dims && cssWidth > 0 ? dims.width / cssWidth : 1;
2560
2934
  }
2561
- /** Aim the handle at the slot once, then look and correct until the piece is home. Release is the submit. */
2935
+ /**
2936
+ * Press the handle, nudge it to see what it moves, then steer that part's point onto its target. Release is the submit.
2937
+ * A generation-3 answer names all three points, so nothing is read from the page but pixels. A generation-2 answer
2938
+ * names only the target: its handle comes from the selector table and the part it moves is whatever moved most.
2939
+ * That path goes when the last generation-2 model does.
2940
+ */
2562
2941
  async executeSlide(page, element, scope, action, elementBox) {
2563
2942
  this.actedOnBoard = true;
2564
- const tb = action.target_bounding_box;
2565
- const targetX = ((tb[0] + tb[2]) / 2) * elementBox.width;
2566
- const handle = await this.findControl(scope, selectors_1.SLIDER_HANDLE_SELECTORS);
2567
- if (!handle) {
2568
- const piece = await this.findControl(scope, selectors_1.PIECE_SELECTORS);
2569
- const box = piece ? await piece.boundingBox() : null;
2943
+ const { width: w, height: h } = elementBox;
2944
+ const at = (b) => [((b[0] + b[2]) / 2) * w, ((b[1] + b[3]) / 2) * h];
2945
+ const goal = at(action.target_bounding_box);
2946
+ let handle;
2947
+ let seed;
2948
+ if (action.action === kinds_1.ActionKind.SLIDE) {
2949
+ handle = at(action.handle_bounding_box);
2950
+ seed = at(action.source_bounding_box);
2951
+ }
2952
+ else {
2953
+ const slider = await this.findControl(scope, this.sel.sliderHandle);
2954
+ const piece = slider ? null : await this.findControl(scope, this.sel.piece);
2955
+ const box = await (slider ?? piece)?.boundingBox().catch(() => null);
2570
2956
  if (!box) {
2571
2957
  console.warn('Slide action, but the widget has neither a slider nor a draggable piece.');
2572
2958
  return false;
2573
2959
  }
2574
- const targetY = ((tb[1] + tb[3]) / 2) * elementBox.height;
2575
- console.log('No slider track; dragging the piece to the slot directly.');
2576
- await this.gesture(() => this.human.drag(page, [box.x + box.width / 2, box.y + box.height / 2], [elementBox.x + targetX, elementBox.y + targetY]));
2577
- return true;
2578
- }
2579
- const hbox = await handle.boundingBox();
2580
- if (!hbox)
2581
- return false;
2582
- const startX = hbox.x + hbox.width / 2;
2583
- const holdY = hbox.y + hbox.height / 2;
2584
- // Mask from the handle's band to the widget's bottom, not the handle's bottom: the filled track moves too, and
2585
- // masked to the band alone the loop once derived a 135.4px piece on Tencent's track against 42.0px masked to the bottom.
2586
- const pad = Math.max(4, hbox.height * 0.35);
2587
- const band = [0, hbox.y - elementBox.y - pad, elementBox.width, elementBox.height];
2588
- const shots = [tmp('slide_0'), tmp('slide_1')];
2960
+ handle = [box.x + box.width / 2 - elementBox.x, box.y + box.height / 2 - elementBox.y];
2961
+ // A piece with no track is the handle itself, and what it moves is it.
2962
+ seed = piece ? handle : null;
2963
+ }
2964
+ const press = [elementBox.x + handle[0], elementBox.y + handle[1]];
2965
+ const probe = goal[0] >= (seed ?? handle)[0] ? SLIDE_PROBE_PX : -SLIDE_PROBE_PX;
2966
+ const timeout = this.config.elementScreenshotTimeoutMs ?? 8000;
2967
+ const [first, probed, current] = [tmp('slide_0'), tmp('slide_1'), tmp('slide_2')];
2589
2968
  try {
2590
- await this.move(page, handle, { paddingPercentage: 30 });
2969
+ await this.performSmoothMove(page, ...press);
2591
2970
  await this.human.press(page);
2592
2971
  await this.human.pause(kinds_1.PauseKind.GRAB);
2593
- await this.shot(element, shots[0], this.config.elementScreenshotTimeoutMs ?? 8000);
2594
- const scale = this.shotScale(shots[0], elementBox.width);
2595
- const exclude = band.map((v) => v * scale);
2596
- const liveCentre = async () => {
2597
- const b = await this.measurePieceBox(scope, elementBox.width);
2598
- return b ? b.x + b.width / 2 - elementBox.x : null;
2972
+ await this.shot(element, first, timeout);
2973
+ const k = this.shotScale(first, w);
2974
+ await this.performSmoothMove(page, press[0] + probe, press[1]);
2975
+ await this.human.pause(kinds_1.PauseKind.PROBE);
2976
+ await this.shot(element, probed, timeout);
2977
+ const ask = {
2978
+ first, probe: probed, probe_s: probe * k, goal: [goal[0] * k, goal[1] * k], seed: seed && [seed[0] * k, seed[1] * k],
2979
+ handle: [handle[0] * k, handle[1] * k], bounds: [-handle[0] * k, (w - handle[0]) * k], tol: SLIDE_TOLERANCE_PX * k,
2599
2980
  };
2600
- const restCentre = (await liveCentre()) ?? (startX - elementBox.x);
2601
- let offset = targetX - restCentre;
2602
- await this.performSmoothMove(page, startX + offset, holdY);
2603
- const widths = [];
2604
- let lastBox = null;
2605
- let lastPiece = null;
2606
- let lastCentre = null;
2607
- let ratio = 1;
2608
- let i = 0;
2609
- for (; i < SLIDE_MAX_CORRECTIONS; i++) {
2610
- await this.human.pause(kinds_1.PauseKind.PROBE);
2611
- const seen = await this.trackPiece(element, shots[0], shots[1], exclude, offset * ratio * scale);
2612
- if (seen) {
2613
- widths.push([offset, (seen.bbox[2] - seen.bbox[0]) / scale]);
2614
- lastBox = seen.bbox;
2615
- lastPiece = seen.piece;
2616
- }
2617
- const solved = (0, slide_geometry_1.solveSlideGeometry)(widths, elementBox.width);
2618
- ratio = solved.ratio;
2619
- const live = await liveCentre();
2620
- const pieceCentre = live !== null ? live
2621
- : lastPiece !== null ? lastPiece.centre / scale
2622
- : (lastBox && solved.pieceWidth !== null ? lastBox[2] / scale - solved.pieceWidth / 2 : null);
2623
- if (pieceCentre === null)
2624
- continue;
2625
- lastCentre = pieceCentre;
2626
- const error = targetX - pieceCentre;
2627
- console.log(`[slide] ${live !== null ? 'the piece element' : lastPiece !== null ? 'the pixel diff' : "the diff's union"} puts it at ${Math.round(pieceCentre)}px, want ${Math.round(targetX)}px, ratio ${ratio.toFixed(2)}`);
2628
- if (Math.abs(error) <= SLIDE_TOLERANCE_PX)
2981
+ let look = probed;
2982
+ let s = probe * k;
2983
+ const history = [];
2984
+ for (let moves = 0;; moves++) {
2985
+ const reply = await this.slideStep({ ...ask, current: look, s, history });
2986
+ if (!reply)
2629
2987
  break;
2630
- offset += error / ratio;
2631
- await this.performSmoothMove(page, startX + offset, holdY);
2632
- }
2633
- if (lastCentre === null) {
2634
- console.warn('Slider: the piece never resolved on screen; released where the opening sweep put it.');
2635
- }
2636
- else if (i >= SLIDE_MAX_CORRECTIONS) {
2637
- const settled = await liveCentre();
2638
- const left = targetX - (settled !== null ? settled : lastCentre);
2639
- if (Math.abs(left) > SLIDE_TOLERANCE_PX)
2640
- console.warn(`[slide] out of corrections with ${left >= 0 ? '+' : ''}${left.toFixed(1)}px still to go — releasing off-target`);
2988
+ if (look !== probed)
2989
+ history.push(reply.observation);
2990
+ if (reply.done || moves >= SLIDE_MAX_CORRECTIONS) {
2991
+ if (reply.error > ask.tol)
2992
+ console.warn(`[slide] releasing ${(reply.error / k).toFixed(1)}px off target`);
2993
+ break;
2994
+ }
2995
+ s = reply.next;
2996
+ const [dx, dy] = Array.isArray(s) ? s : [s, 0];
2997
+ await this.performSmoothMove(page, press[0] + dx / k, press[1] + dy / k);
2998
+ await this.human.pause(kinds_1.PauseKind.PROBE);
2999
+ await this.shot(element, current, timeout);
3000
+ look = current;
2641
3001
  }
2642
3002
  await this.human.pause(kinds_1.PauseKind.SETTLE);
2643
3003
  }
@@ -2646,9 +3006,8 @@ class CaptchaKrakenSolver {
2646
3006
  await this.human.release(page);
2647
3007
  }
2648
3008
  catch { /* the page may have navigated */ }
2649
- this.lastInputAt = Date.now();
2650
- for (const s of shots)
2651
- unlink(s);
3009
+ for (const f of [first, probed, current])
3010
+ unlink(f);
2652
3011
  }
2653
3012
  return true;
2654
3013
  }