@trawlme/cli 3.9.0 → 3.9.1

package/dist/commands/doctor.d.ts

@@ -25,19 +25,31 @@ export interface Run {
         selectors?: Record<string, number>;
     } | null;
     blocked?: boolean;
+    block?: {
+        kind?: string | null;
+    } | null;
+    /** @deprecated trawl_node#1950 renamed this to `block.kind`. Kept as a read
+     * fallback (see detectWallVendor) for the window where this CLI is
+     * published ahead of the trawl_node prod tag — a prod backend served from
+     * the pre-#1950 tag still returns this flat field, not `block.kind`. Drop
+     * once prod is confirmed on a tag containing #1950. */
     blockType?: string | null;
     proxyTier?: string | null;
-    regressionDetected?: boolean;
     baselineLength?: number | null;
     fixVersionId?: string | null;
     createdAt?: string;
     time?: number | null;
     triggeredBy?: string | null;
+    /** trawl_node#1975 — freeform failure classification (`'auth'` = login-wall
+     * empty run, cookies are the fix). Not a TS union — trawl_node's own set
+     * is additive/open, so this CLI must stay read-safe against a future
+     * value it doesn't know about yet, same posture as `block.kind` above. */
+    failureKind?: string | null;
 }
 /**
- * Resolve a known anti-bot vendor (or auth) name from a worker `blockType`
- * string. Returns null when the run succeeded, when there is no `blockType`
- * signal at all, or when `blockType` names something other than a known
+ * Resolve a known anti-bot vendor name from a worker `block.kind`
+ * string. Returns null when the run succeeded, when there is no `block.kind`
+ * signal at all, or when `block.kind` names something other than a known
  * vendor (e.g. `proxy-domain-gate`, `rate_limited_per_host`) — those stay on
  * the genuine-error path since we can't honestly attribute them to a specific
  * "no reliable bypass" wall.
@@ -45,12 +57,18 @@ export interface Run {
  * `blocked` is deliberately NOT consulted here (#177): it is a mid-run,
  * attempt-level signal the worker only stamps on one envelope shape, so it
  * reads `false` on the great majority of genuinely walled runs (the early-block
- * throw path sets `blockType` but never `blocked` — 63 of 64 walled runs
+ * throw path sets `block.kind` but never `blocked` — 63 of 64 walled runs
  * measured on prod). `status === true` wins instead: a run that ultimately
- * returned data was not walled, even if an earlier tier's `blockType` stamp
+ * returned data was not walled, even if an earlier tier's `block.kind` stamp
  * survived on the row.
+ *
+ * trawl_node#1950 renamed the flat `blockType` field to nested `block.kind`.
+ * Read `block?.kind` first, falling back to the deprecated flat `blockType` —
+ * this CLI can be published (and talk to prod) before the trawl_node prod tag
+ * containing #1950 is cut (see the `Run.blockType` doc comment), so a prod
+ * response can still be the pre-#1950 flat shape for a while.
  */
-export declare function detectWallVendor(run: Pick<Run, 'status' | 'statusDetail' | 'blockType'>): string | null;
+export declare function detectWallVendor(run: Pick<Run, 'status' | 'statusDetail' | 'block' | 'blockType'>): string | null;
 /**
  * Autofix activity metadata — from the persisted ai_fix_end activity.
  * aiUsage (cost) is stripped server-side; all diagnostics are kept.

package/dist/commands/doctor.js

@@ -2,23 +2,25 @@ import { api } from '../lib/api.js';
 import chalk from 'chalk';
 import { formatDate } from '../lib/format.js';
 /**
- * Known anti-bot vendors (+ auth) that the worker's `blockType` field may name.
- * Ordered by first-match; `blockType` is a freeform worker string, not an enum
- * (see the `Run.blockType` doc comment), so this is a best-effort substring
- * match against the real field — never an invented/mocked value.
+ * Known anti-bot vendors that the worker's `block.kind` field may name
+ * (ex-flat `blockType`, trawl_node#1950). Ordered by first-match;
+ * `block.kind` is a freeform worker string, not an enum (see the `Run.block`
+ * doc comment), so this is a best-effort substring match against the real
+ * field — never an invented/mocked value.
  *
  * `datadome`/`perimeterx`/`akamai`/`cloudflare` are literal substrings the
- * current worker emits (detect.js). `kasada` and `auth` are forward-compatible:
- * the issue (#62 / wall-registry WS-7) names them as genuine walls, but the
- * current worker vocabulary does not yet emit them — these branches light up
- * automatically if/when a future worker classifier does, without a CLI change.
+ * current worker emits (detect.js). `kasada` is forward-compatible: the issue
+ * (#62 / wall-registry WS-7) names it as a genuine wall, but the current
+ * worker vocabulary does not yet emit it — this branch lights up
+ * automatically if/when a future worker classifier does, without a CLI
+ * change.
  *
- * The `auth` pattern is delimiter-anchored (`^`/`$`/`-_:`) rather than a bare
- * substring so it matches worker-style tokens (`auth`, `auth-wall`, `auth_wall`,
- * `login:auth`) without false-matching `oauth`/`authorization`/`author`. It uses
- * `[-_:]` (not `\b`) because `_` is a JS `\w` char, so `\bauth\b` would miss the
- * underscore-delimited `auth_wall` form the worker's `rate_limited_per_host`-style
- * naming favors.
+ * trawl_cli#182 — an `auth` entry used to live here, rendering a login wall
+ * as "walled: auth — no reliable bypass". Removed: unlike a real anti-bot
+ * vendor, a login wall's own reliable bypass is the user's session cookies,
+ * so that verdict was wrong, not just early. `failureKind==='auth'`
+ * (trawl_node#1975) is the honest signal for a login wall — see the
+ * dedicated branch in `formatDoctor` below.
  */
 const WALL_VENDOR_PATTERNS = [
     [/datadome/i, 'DataDome'],
@@ -26,12 +28,11 @@ const WALL_VENDOR_PATTERNS = [
     [/perimeterx/i, 'PerimeterX'],
     [/akamai/i, 'Akamai'],
     [/cloudflare/i, 'Cloudflare'],
-    [/(?:^|[-_:])auth(?:$|[-_:])/i, 'auth'],
 ];
 /**
- * Resolve a known anti-bot vendor (or auth) name from a worker `blockType`
- * string. Returns null when the run succeeded, when there is no `blockType`
- * signal at all, or when `blockType` names something other than a known
+ * Resolve a known anti-bot vendor name from a worker `block.kind`
+ * string. Returns null when the run succeeded, when there is no `block.kind`
+ * signal at all, or when `block.kind` names something other than a known
  * vendor (e.g. `proxy-domain-gate`, `rate_limited_per_host`) — those stay on
  * the genuine-error path since we can't honestly attribute them to a specific
  * "no reliable bypass" wall.
@@ -39,24 +40,31 @@ const WALL_VENDOR_PATTERNS = [
  * `blocked` is deliberately NOT consulted here (#177): it is a mid-run,
  * attempt-level signal the worker only stamps on one envelope shape, so it
  * reads `false` on the great majority of genuinely walled runs (the early-block
- * throw path sets `blockType` but never `blocked` — 63 of 64 walled runs
+ * throw path sets `block.kind` but never `blocked` — 63 of 64 walled runs
  * measured on prod). `status === true` wins instead: a run that ultimately
- * returned data was not walled, even if an earlier tier's `blockType` stamp
+ * returned data was not walled, even if an earlier tier's `block.kind` stamp
  * survived on the row.
+ *
+ * trawl_node#1950 renamed the flat `blockType` field to nested `block.kind`.
+ * Read `block?.kind` first, falling back to the deprecated flat `blockType` —
+ * this CLI can be published (and talk to prod) before the trawl_node prod tag
+ * containing #1950 is cut (see the `Run.blockType` doc comment), so a prod
+ * response can still be the pre-#1950 flat shape for a while.
  */
 export function detectWallVendor(run) {
     // A run that returned data was not walled. `status === true` is not the whole
     // predicate: trawl_node flips a degraded-but-non-empty run to
     // `status:false, statusDetail:'regression'` (scraps.service.js, #1112) via a
-    // patch that never clears `blockType` — so keying on `status` alone would
+    // patch that never clears `block.kind` — so keying on `status` alone would
     // print "no reliable bypass" next to "Regression: length N vs baseline M",
     // which is the same contradiction this guard exists to remove.
     if (run.status === true || run.statusDetail === 'regression')
         return null;
-    if (!run.blockType)
+    const kind = run.block?.kind ?? run.blockType ?? null;
+    if (!kind)
         return null; // no signal at all
     for (const [pattern, label] of WALL_VENDOR_PATTERNS) {
-        if (pattern.test(run.blockType))
+        if (pattern.test(kind))
             return label;
     }
     return null;
@@ -70,8 +78,8 @@ const TIER_LABELS = {
 };
 const RUN_ALLOWLIST = [
     '_id', 'status', 'statusDetail', 'length', 'errorMessage', 'errorSnapshot',
-    'emptyContext', 'blocked', 'blockType', 'proxyTier', 'regressionDetected', 'baselineLength',
-    'fixVersionId', 'createdAt', 'time', 'triggeredBy',
+    'emptyContext', 'blocked', 'block', 'blockType', 'proxyTier', 'baselineLength',
+    'fixVersionId', 'createdAt', 'time', 'triggeredBy', 'failureKind',
 ];
 const FIX_ALLOWLIST = [
     'outcome', 'classification', 'reason', 'fixDiff', 'dryRunResults',
@@ -124,14 +132,81 @@ export function formatDoctor(scrapTitle, run, fix = null, scrapId) {
     lines.push(`${chalk.bold(scrapTitle)}  ${badge}${run.statusDetail ? ` (${run.statusDetail})` : ''}`);
     lines.push(chalk.dim(`  Run ID:  ${run._id}`));
     // Error message — an honest accept-wall string for known-walled scraps
-    // (DataDome/Kasada/PerimeterX/Akamai/auth terminal verdict from the worker),
+    // (DataDome/Kasada/PerimeterX/Akamai terminal verdict from the worker),
     // otherwise the real error (genuine transient failure).
+    //
+    // trawl_cli#182 — the wall-vendor verdict (this CLI's own
+    // WALL_VENDOR_PATTERNS matched against block.kind/blockType) and the
+    // login-wall hint (node's failureKind==='auth', trawl_node#1975) are two
+    // INDEPENDENT server-derived signals with no shared contract: node's block
+    // classifier (modules/historys/helpers/failureKind.js BLOCK_TYPE_PATTERN)
+    // and this CLI's vendor table live in different repos and can legitimately
+    // disagree. Concretely, `kasada` matches this CLI's table but is absent
+    // from node's block pattern, and the deprecated flat `blockType` fallback
+    // is a field node's classifier no longer reads at all — so a run can come
+    // back `failureKind:'auth'` (the server says login wall) while also
+    // matching a vendor here (this CLI says Kasada/DataDome/etc), both true
+    // signals about the same run, neither one wrong.
+    //
+    // trawl_cli#169 precedent: never assert client-side what only the server
+    // knows. Silently picking a winner between two disagreeing signals is
+    // exactly that — printing "no reliable bypass" alone asserts the run is
+    // NOT an auth wall (may stop someone from trying a fixable cookie
+    // problem); printing the cookie hint alone asserts the run is NOT a
+    // vendor wall (may send someone chasing cookies against a wall no cookie
+    // fixes). This CLI cannot tell which is true without either importing
+    // node's BLOCK_TYPE_PATTERN (a second source of truth for the same list —
+    // how this defect got here) or re-detecting the login URL itself (the
+    // client-side re-derivation this fix deliberately rejects). So when both
+    // fire, render ONE hedged line naming both readings instead of picking:
+    // never emit the bare "no reliable bypass" verdict in that case — it is
+    // exactly the false precision this branch exists to avoid.
+    //
+    // `conflictingSignals` is deliberately NOT gated on `scrapId` the way the
+    // plain hint below is: the false "no reliable bypass" verdict is wrong
+    // regardless of whether we also have an id to build an actionable command
+    // for, so a scrapId-less conflicting run must still avoid it. Only the
+    // action line degrades (to a generic Settings pointer) when there's no id.
+    //
+    // Single-signal cases are untouched: a plain vendor match (no
+    // failureKind:'auth') still gets "no reliable bypass"; a plain
+    // failureKind:'auth' (no vendor match) still gets the plain hint block
+    // below. `errMsg` stays suppressed whenever any hint form (conflicting or
+    // plain) is about to render — node's own `errorMessage` for an auth run
+    // (historys.service.js ~line 774) IS the hint copy verbatim, literal
+    // `<id>` placeholder and all, so printing it here would just be a second
+    // rendering of the same information. If neither hint form renders (no
+    // vendor conflict and no scrapId for the plain hint), fall back to node's
+    // raw message rather than dropping the error entirely.
     const wallVendor = detectWallVendor(run);
     const errMsg = run.errorMessage ?? run.errorSnapshot?.errorMessage;
-    if (wallVendor) {
+    // trawl_cli#182 — same `status`/`statusDetail` guard as `detectWallVendor`
+    // above, and for the same reason (see its doc comment): `failureKind` is a
+    // terminal classification trawl_node stamps once
+    // (modules/historys/services/historys.service.js ~line 837), but
+    // `patchForRegression` (modules/scraps/services/scraps.service.js ~line
+    // 1972) can flip `status`/`statusDetail` to success/regression LATER,
+    // without ever clearing `failureKind`. Left unguarded, a run that
+    // ultimately succeeded or degraded could still carry a stale
+    // `failureKind:'auth'` and render the login-wall hint (or the
+    // conflicting-signals hedge) right next to a green "success" badge or a
+    // "Regression: length N vs baseline M" line — the exact contradiction
+    // `detectWallVendor`'s guard already exists to prevent for `block.kind`.
+    // One rule, not two coincidences: both checks guard the same two fields
+    // against the same after-the-fact patch.
+    const authWall = run.failureKind === 'auth' && run.status !== true && run.statusDetail !== 'regression';
+    const authHintWillRender = authWall && Boolean(scrapId);
+    const conflictingSignals = Boolean(wallVendor) && authWall;
+    if (conflictingSignals) {
+        lines.push(chalk.dim('  Error:   ') + chalk.yellow(`conflicting signals — the server reported ${wallVendor} on this run, but also classified it as a login wall; can't tell which is true here`));
+        lines.push(chalk.dim(scrapId
+            ? `  → cookies might help, but this may still be a genuine ${wallVendor} block: trawl scraps account session set ${scrapId} -c cookies.json`
+            : `  → cookies might help, but this may still be a genuine ${wallVendor} block (see app Settings → Account)`));
+    }
+    else if (wallVendor) {
         lines.push(chalk.dim('  Error:   ') + chalk.red(`walled: ${wallVendor} — no reliable bypass`));
     }
-    else if (errMsg) {
+    else if (errMsg && !authHintWillRender) {
         lines.push(chalk.dim('  Error:   ') + chalk.red(errMsg));
     }
     // Failed selector
@@ -149,8 +224,32 @@ export function formatDoctor(scrapTitle, run, fix = null, scrapId) {
         const page = run.emptyContext.page;
         lines.push(chalk.dim('  Empty context:   ') + `url=${page.url ?? '?'}  anchors=${page.totalAnchors ?? '?'}`);
     }
-    // Regression detail
-    if (run.regressionDetected) {
+    // Login-wall hint (trawl_node#1975 `failureKind==='auth'`) — trust node's
+    // verdict, never re-detect the login URL client-side. Hypothesis-framed
+    // ("likely fix"), not a promise: measured cases (Reddit/X/Instagram) had
+    // valid cookies and stayed walled. Gated on `scrapId` alone (via
+    // `authHintWillRender` above, shared with the generic error line so the
+    // two can't drift apart), NOT the `scrapId ?? run._id` fallback used
+    // elsewhere here — `run._id` is a history id, and the session-set command
+    // on one silently targets the wrong document.
+    //
+    // trawl_cli#182 — excludes `conflictingSignals`: when a wall-vendor match
+    // also fired, the combined hedged line above already covers the cookie
+    // suggestion. Rendering this plain, unhedged block too would stack a
+    // second verdict under the first (and directly contradict it, since this
+    // block asserts the run IS a login wall with no caveat) — exactly the
+    // two-verdicts-for-one-run defect this fix removes. Exactly one verdict
+    // block renders per run.
+    if (authHintWillRender && !conflictingSignals) {
+        lines.push('');
+        lines.push(chalk.yellow('  Login wall (hypothesis): ') + 'this looks like a login redirect — your own session cookies are the likely fix.');
+        lines.push(chalk.dim(`  → trawl scraps account session set ${scrapId} -c cookies.json   (or app Settings → Account)`));
+    }
+    // Regression detail. trawl_node#1950 dropped the redundant `regressionDetected`
+    // boolean — it was `true` on every row iff `statusDetail === 'regression'`,
+    // so read that directly (works against a pre- or post-#1950 backend alike,
+    // no CLI/backend sequencing gap: `statusDetail` isn't part of this rename).
+    if (run.statusDetail === 'regression') {
         lines.push(chalk.dim('  Regression:      ') + `length ${run.length ?? '?'} vs baseline ${run.baselineLength ?? '?'}`);
     }
     // Autofix summary (when fix exists)

package/dist/commands/scraps.js

@@ -1234,7 +1234,9 @@ export function attachHistoryCommand(parent, attachOpts = {}) {
             time: h.time ?? null,
             tier: h.proxyTier ?? null,
             failureKind: h.failureKind ?? null,
-            blockType: h.blockType ?? null,
+            // trawl_node#1950 renamed blockType to block.kind; fall back to the
+            // deprecated flat field for a prod backend not yet on that tag.
+            blockType: h.block?.kind ?? h.blockType ?? null,
             createdAt: h.createdAt ?? null,
         }));
         if (opts.json) {
@@ -1260,7 +1262,9 @@ export function attachRunInfoCommand(parent, attachOpts = {}) {
             time: h.time ?? null,
             tier: h.proxyTier ?? null,
             failureKind: h.failureKind ?? null,
-            blockType: h.blockType ?? null,
+            // trawl_node#1950 renamed blockType to block.kind; fall back to the
+            // deprecated flat field for a prod backend not yet on that tag.
+            blockType: h.block?.kind ?? h.blockType ?? null,
             errorMessage: h.errorSnapshot?.errorMessage ?? null,
             selector: h.errorSnapshot?.selector ?? null,
             emptyContext: h.emptyContext ?? null,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trawlme/cli",
-  "version": "3.9.0",
+  "version": "3.9.1",
   "description": "Trawl CLI — manage scraps from the terminal",
   "type": "module",
   "bin": {