@trawlme/cli 3.12.0 → 3.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/commands/create.d.ts +0 -28
- package/dist/commands/create.js +0 -89
- package/dist/commands/doctor.d.ts +0 -79
- package/dist/commands/doctor.js +1 -187
- package/dist/commands/login.js +0 -67
- package/dist/commands/ping.d.ts +0 -15
- package/dist/commands/ping.js +0 -15
- package/dist/commands/scraps.d.ts +0 -120
- package/dist/commands/scraps.js +10 -724
- package/dist/commands/skills.js +0 -22
- package/dist/commands/spec.d.ts +0 -85
- package/dist/commands/spec.js +0 -67
- package/dist/commands/telemetry.js +0 -4
- package/dist/commands/token.js +0 -28
- package/dist/commands/upgrade.js +0 -22
- package/dist/commands/whoami.d.ts +0 -12
- package/dist/commands/whoami.js +0 -6
- package/dist/index.d.ts +0 -188
- package/dist/index.js +0 -349
- package/dist/lib/api.d.ts +0 -78
- package/dist/lib/api.js +1 -320
- package/dist/lib/cdp-pipe.d.ts +0 -72
- package/dist/lib/cdp-pipe.js +1 -81
- package/dist/lib/chrome-discovery.d.ts +0 -11
- package/dist/lib/chrome-discovery.js +0 -19
- package/dist/lib/chrome-launch.d.ts +0 -40
- package/dist/lib/chrome-launch.js +0 -69
- package/dist/lib/config.d.ts +0 -53
- package/dist/lib/config.js +0 -55
- package/dist/lib/confirm.d.ts +0 -55
- package/dist/lib/confirm.js +0 -47
- package/dist/lib/docs.d.ts +0 -123
- package/dist/lib/docs.js +0 -169
- package/dist/lib/errors.d.ts +0 -134
- package/dist/lib/errors.js +0 -151
- package/dist/lib/format.d.ts +0 -6
- package/dist/lib/format.js +0 -6
- package/dist/lib/json.d.ts +0 -35
- package/dist/lib/json.js +0 -48
- package/dist/lib/jwt.d.ts +0 -7
- package/dist/lib/jwt.js +0 -7
- package/dist/lib/pinch.d.ts +0 -53
- package/dist/lib/pinch.js +6 -112
- package/dist/lib/pinchAnimation.d.ts +0 -16
- package/dist/lib/pinchAnimation.js +8 -29
- package/dist/lib/posthog.d.ts +0 -9
- package/dist/lib/posthog.js +0 -23
- package/dist/lib/prompt.js +1 -20
- package/dist/lib/secure-transport.d.ts +0 -7
- package/dist/lib/secure-transport.js +0 -24
- package/dist/lib/session-capture-guard.d.ts +0 -15
- package/dist/lib/session-capture-guard.js +0 -5
- package/dist/lib/session-capture.d.ts +0 -125
- package/dist/lib/session-capture.js +0 -281
- package/dist/lib/skills.d.ts +0 -175
- package/dist/lib/skills.js +1 -216
- package/dist/lib/skillsNudge.d.ts +0 -17
- package/dist/lib/skillsNudge.js +0 -83
- package/dist/lib/spinner.d.ts +0 -39
- package/dist/lib/spinner.js +0 -40
- package/dist/lib/storage-state.d.ts +0 -112
- package/dist/lib/storage-state.js +0 -131
- package/dist/lib/tips.d.ts +0 -38
- package/dist/lib/tips.js +0 -77
- package/dist/lib/updateCheckWorker.js +0 -14
- package/dist/lib/updateNotifier.d.ts +0 -17
- package/dist/lib/updateNotifier.js +0 -53
- package/dist/lib/validate.d.ts +0 -8
- package/dist/lib/validate.js +0 -8
- package/dist/lib/version.d.ts +0 -12
- package/dist/lib/version.js +1 -13
- package/package.json +2 -2
package/dist/lib/skillsNudge.js
CHANGED
|
@@ -1,61 +1,10 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* #184 — the "elsewhere: suggest, never install" half of the skills
|
|
3
|
-
* bootstrap story (lib/skills.ts's `bootstrapSkillsOnLogin` is the "install
|
|
4
|
-
* on login" half). Writing into `~/.claude/skills` as a side effect of an
|
|
5
|
-
* unrelated command is a filesystem mutation nobody asked for — this module
|
|
6
|
-
* only ever PRINTS a line naming the command that installs them. Never
|
|
7
|
-
* installs anything itself.
|
|
8
|
-
*
|
|
9
|
-
* Two nudges, one shared "at most once per process" guard:
|
|
10
|
-
* - `maybeSuggestSkillsInstall` — the generic, throttled nudge (once per
|
|
11
|
-
* NUDGE_THROTTLE_MS across ALL commands, mirrors lib/tips.ts's referral
|
|
12
|
-
* tip throttle exactly).
|
|
13
|
-
* - `maybeSuggestSkillsForAuthWall` — the safety-net nudge (#184 point 5):
|
|
14
|
-
* fires from `doctor`/`run-info` when the run they just showed carries a
|
|
15
|
-
* live `failureKind:'auth'` verdict and skills are absent. Deliberately
|
|
16
|
-
* UNTHROTTLED by the 7-day timestamp: sharing that throttle would mean a
|
|
17
|
-
* routine command on day 0 (stamping the generic nudge's timestamp)
|
|
18
|
-
* silently suppresses THIS nudge on day 1 when the user actually hits
|
|
19
|
-
* the auth wall the whole issue exists for — the exact failure this
|
|
20
|
-
* safety net is supposed to catch. It mirrors trawl_cli#182's own
|
|
21
|
-
* login-wall hint in `formatDoctor`, which also prints unthrottled every
|
|
22
|
-
* time the condition is true.
|
|
23
|
-
* - `nudgedThisProcess` still caps the two at "one nudge per invocation"
|
|
24
|
-
* combined: `doctor` on an auth-walled, skills-absent run can print the
|
|
25
|
-
* auth-wall nudge inside its own action, and index.ts's generic
|
|
26
|
-
* post-command nudge (which does not know `doctor` already said
|
|
27
|
-
* something) would otherwise print a second, redundant line right after
|
|
28
|
-
* it. Whichever fires first wins; reset per test via a fresh module
|
|
29
|
-
* import (see skillsNudge.test.ts's `freshImport` helper, mirroring
|
|
30
|
-
* tips.test.ts).
|
|
31
|
-
*
|
|
32
|
-
* Gate shape mirrors lib/tips.ts's isReferralTipDue/maybeShowReferralTip
|
|
33
|
-
* throughout (json/isTTY/opt-out/throttle, pure gate separated from the
|
|
34
|
-
* side-effecting caller, cheap checks before any filesystem read) — same
|
|
35
|
-
* reasoning applies verbatim.
|
|
36
|
-
*/
|
|
37
1
|
import chalk from 'chalk';
|
|
38
2
|
import config from './config.js';
|
|
39
3
|
import { listBundledSkills, isSkillInstalled, isSkillsActionAllowed, RESTART_CLAUDE_CODE_NOTE } from './skills.js';
|
|
40
|
-
/** Max once per 7 days — same window as lib/tips.ts's referral tip. */
|
|
41
4
|
const NUDGE_THROTTLE_MS = 7 * 24 * 60 * 60 * 1000;
|
|
42
|
-
// #184 constraint A — a FACT about CLI state plus the command name, never an
|
|
43
|
-
// instruction to the reader ("install the skills"/"you should run…"). This
|
|
44
|
-
// output is not only read by a human at a terminal: this platform can relay
|
|
45
|
-
// a CLI's own stdout/stderr into an AI agent's context (the incident this
|
|
46
|
-
// issue exists to prevent was exactly an agent driving this CLI), and
|
|
47
|
-
// third-party content returned into agent context must never read as an
|
|
48
|
-
// instruction the agent is being told to obey. State the fact, name the
|
|
49
|
-
// command, stop there.
|
|
50
5
|
export const SKILLS_NUDGE_TEXT = `Claude skills not installed — \`trawl skills install\` installs them (${RESTART_CLAUDE_CODE_NOTE})`;
|
|
51
6
|
export const AUTH_WALL_SKILLS_NUDGE_TEXT = `This looks like a login wall and Claude skills are not installed — \`trawl skills install\` installs the guidance for it too (${RESTART_CLAUDE_CODE_NOTE})`;
|
|
52
|
-
/** One nudge per process, whichever of the two below fires first. */
|
|
53
7
|
let nudgedThisProcess = false;
|
|
54
|
-
/**
|
|
55
|
-
* Pure gate — true when the throttled, generic "elsewhere" suggestion
|
|
56
|
-
* should print. Every external signal is a parameter, exactly like
|
|
57
|
-
* isReferralTipDue, so this is testable without mocking fs/env/Date.
|
|
58
|
-
*/
|
|
59
8
|
export function isSkillsNudgeDue(opts) {
|
|
60
9
|
if (!isSkillsActionAllowed({ json: opts.json, isTTY: opts.isTTY, optedOut: opts.optedOut }))
|
|
61
10
|
return false;
|
|
@@ -63,14 +12,6 @@ export function isSkillsNudgeDue(opts) {
|
|
|
63
12
|
return false;
|
|
64
13
|
return opts.now - opts.lastShownAt >= NUDGE_THROTTLE_MS;
|
|
65
14
|
}
|
|
66
|
-
/**
|
|
67
|
-
* Read-only — checks both scopes, mirrors `trawl skills list`'s own
|
|
68
|
-
* "installed anywhere" question. Failing toward `false` (not installed) on
|
|
69
|
-
* an unreadable skills dir is the safe default here: the worst case is one
|
|
70
|
-
* extra nudge line, never a missed one — the opposite failure (silently
|
|
71
|
-
* assuming "present" and staying quiet) is the exact gap this issue exists
|
|
72
|
-
* to close.
|
|
73
|
-
*/
|
|
74
15
|
function anyBundledSkillInstalled() {
|
|
75
16
|
try {
|
|
76
17
|
return listBundledSkills().some((name) => isSkillInstalled(name, 'user') || isSkillInstalled(name, 'local'));
|
|
@@ -79,15 +20,6 @@ function anyBundledSkillInstalled() {
|
|
|
79
20
|
return false;
|
|
80
21
|
}
|
|
81
22
|
}
|
|
82
|
-
/**
|
|
83
|
-
* Shared body for both exported nudges below: the cheap json/TTY/opt-out
|
|
84
|
-
* gate, the "one nudge per process" cap, the skills-presence check, and the
|
|
85
|
-
* actual print — everything the two nudges have in common. `useThrottle`
|
|
86
|
-
* is the one real difference between them (see the module doc comment for
|
|
87
|
-
* why the safety-net nudge deliberately opts out of it), so it stays an
|
|
88
|
-
* explicit parameter here rather than two near-identical function bodies.
|
|
89
|
-
* Never throws.
|
|
90
|
-
*/
|
|
91
23
|
function tryPrintNudge(opts, text, useThrottle) {
|
|
92
24
|
try {
|
|
93
25
|
if (nudgedThisProcess)
|
|
@@ -95,8 +27,6 @@ function tryPrintNudge(opts, text, useThrottle) {
|
|
|
95
27
|
const json = Boolean(opts.json);
|
|
96
28
|
const isTTY = Boolean(process.stdout.isTTY);
|
|
97
29
|
const optedOut = process.env['TRAWL_SKILLS_SYNC'] === '0';
|
|
98
|
-
// Cheap checks first (mirrors tips.ts #153's "never pay when not due")
|
|
99
|
-
// — only touch the filesystem once json/TTY/opt-out already say "maybe".
|
|
100
30
|
if (!isSkillsActionAllowed({ json, isTTY, optedOut }))
|
|
101
31
|
return;
|
|
102
32
|
if (useThrottle) {
|
|
@@ -114,24 +44,11 @@ function tryPrintNudge(opts, text, useThrottle) {
|
|
|
114
44
|
console.error(chalk.dim(text));
|
|
115
45
|
}
|
|
116
46
|
catch {
|
|
117
|
-
// silent — a broken nudge must never break a command
|
|
118
47
|
}
|
|
119
48
|
}
|
|
120
|
-
/**
|
|
121
|
-
* Best-effort generic nudge (#184 point 2) — called once per CLI invocation
|
|
122
|
-
* from index.ts's runCli, on every command. Never throws.
|
|
123
|
-
*/
|
|
124
49
|
export function maybeSuggestSkillsInstall(opts = {}) {
|
|
125
50
|
tryPrintNudge(opts, SKILLS_NUDGE_TEXT, true);
|
|
126
51
|
}
|
|
127
|
-
/**
|
|
128
|
-
* Safety-net nudge (#184 point 5) — called from `doctor`/`run-info` right
|
|
129
|
-
* when the run they just showed carries a live `failureKind:'auth'` verdict
|
|
130
|
-
* (see commands/doctor.ts's `isAuthWall`). Deliberately unthrottled by the
|
|
131
|
-
* 7-day timestamp — see this module's doc comment for why — but still
|
|
132
|
-
* capped to one nudge per process via the same `nudgedThisProcess` flag the
|
|
133
|
-
* generic nudge sets. Never throws.
|
|
134
|
-
*/
|
|
135
52
|
export function maybeSuggestSkillsForAuthWall(opts = {}) {
|
|
136
53
|
tryPrintNudge(opts, AUTH_WALL_SKILLS_NUDGE_TEXT, false);
|
|
137
54
|
}
|
package/dist/lib/spinner.d.ts
CHANGED
|
@@ -5,45 +5,6 @@ type SpinOptions<T> = string | {
|
|
|
5
5
|
failText?: string | ((error: Error) => string);
|
|
6
6
|
[key: string]: unknown;
|
|
7
7
|
};
|
|
8
|
-
/**
|
|
9
|
-
* Drop-in for `oraPromise` that emits NOTHING when stderr is not a TTY.
|
|
10
|
-
*
|
|
11
|
-
* #119 — under a pipe / non-TTY, `oraPromise` still printed the start text AND
|
|
12
|
-
* a persisted `✔ …` line, so `trawl list | cat` showed the spinner caption
|
|
13
|
-
* twice. An agent/CI never wants spinner chrome; here we just run the action.
|
|
14
|
-
* When stderr IS a TTY, behaviour is identical to `oraPromise` (spinner writes
|
|
15
|
-
* to stderr, stdout stays clean either way).
|
|
16
|
-
*/
|
|
17
8
|
export declare function spin<T>(action: Action<T>, options?: SpinOptions<T>): Promise<T>;
|
|
18
|
-
/**
|
|
19
|
-
* The counterpart to `spin()`'s silence — print a plain-text confirmation on
|
|
20
|
-
* stdout when, and only when, stdout is not a TTY.
|
|
21
|
-
*
|
|
22
|
-
* #160 then #166. `spin()` above deliberately emits nothing when stderr is not
|
|
23
|
-
* a TTY, so for every command whose only non-`--json` feedback was its
|
|
24
|
-
* `successText`, a piped or redirected invocation wrote **zero bytes to both
|
|
25
|
-
* streams while exiting 0**. Our ICP is agent builders, and agents pipe stdout:
|
|
26
|
-
* a verb that writes nothing on success is unusable from a script, because the
|
|
27
|
-
* caller cannot tell success from a no-op. Two of the affected commands were
|
|
28
|
-
* deletes, where that ambiguity is at its worst.
|
|
29
|
-
*
|
|
30
|
-
* Lives here, next to the silence it compensates for, because #160 fixed
|
|
31
|
-
* `trigger` by inlining this rule and #166 then found five siblings carrying
|
|
32
|
-
* the identical defect — five more inlined copies is how the next one gets
|
|
33
|
-
* missed. Callers pass the message; the gate lives in one place.
|
|
34
|
-
*
|
|
35
|
-
* Gated on **stdout**, not stderr: stdout is the stream a script actually reads
|
|
36
|
-
* (`$(trawl …)`, `> out.txt`, `| jq`), independent of whatever `spin()` decides
|
|
37
|
-
* about stderr. So an interactive session, where the ora spinner already
|
|
38
|
-
* confirmed on stderr, never gets a duplicate line here.
|
|
39
|
-
*
|
|
40
|
-
* Never call this on a `--json` path: under `--json`, stdout must stay exactly
|
|
41
|
-
* one parseable document.
|
|
42
|
-
*
|
|
43
|
-
* Known residual, unchanged from #160: stdout attached to a real terminal while
|
|
44
|
-
* stderr is separately redirected still prints nothing on either stream. That is
|
|
45
|
-
* not the reported or common shape (full redirection, or stdout-only capture),
|
|
46
|
-
* and widening the gate would put a duplicate line in front of interactive users.
|
|
47
|
-
*/
|
|
48
9
|
export declare function confirmNonTTY(message: string): void;
|
|
49
10
|
export {};
|
package/dist/lib/spinner.js
CHANGED
|
@@ -1,50 +1,10 @@
|
|
|
1
1
|
import { oraPromise } from 'ora';
|
|
2
|
-
/**
|
|
3
|
-
* Drop-in for `oraPromise` that emits NOTHING when stderr is not a TTY.
|
|
4
|
-
*
|
|
5
|
-
* #119 — under a pipe / non-TTY, `oraPromise` still printed the start text AND
|
|
6
|
-
* a persisted `✔ …` line, so `trawl list | cat` showed the spinner caption
|
|
7
|
-
* twice. An agent/CI never wants spinner chrome; here we just run the action.
|
|
8
|
-
* When stderr IS a TTY, behaviour is identical to `oraPromise` (spinner writes
|
|
9
|
-
* to stderr, stdout stays clean either way).
|
|
10
|
-
*/
|
|
11
2
|
export function spin(action, options) {
|
|
12
3
|
if (!process.stderr.isTTY) {
|
|
13
4
|
return Promise.resolve(typeof action === 'function' ? action() : action);
|
|
14
5
|
}
|
|
15
|
-
// oraPromise's own overloads accept (action, string) and (action, options).
|
|
16
6
|
return oraPromise(action, options);
|
|
17
7
|
}
|
|
18
|
-
/**
|
|
19
|
-
* The counterpart to `spin()`'s silence — print a plain-text confirmation on
|
|
20
|
-
* stdout when, and only when, stdout is not a TTY.
|
|
21
|
-
*
|
|
22
|
-
* #160 then #166. `spin()` above deliberately emits nothing when stderr is not
|
|
23
|
-
* a TTY, so for every command whose only non-`--json` feedback was its
|
|
24
|
-
* `successText`, a piped or redirected invocation wrote **zero bytes to both
|
|
25
|
-
* streams while exiting 0**. Our ICP is agent builders, and agents pipe stdout:
|
|
26
|
-
* a verb that writes nothing on success is unusable from a script, because the
|
|
27
|
-
* caller cannot tell success from a no-op. Two of the affected commands were
|
|
28
|
-
* deletes, where that ambiguity is at its worst.
|
|
29
|
-
*
|
|
30
|
-
* Lives here, next to the silence it compensates for, because #160 fixed
|
|
31
|
-
* `trigger` by inlining this rule and #166 then found five siblings carrying
|
|
32
|
-
* the identical defect — five more inlined copies is how the next one gets
|
|
33
|
-
* missed. Callers pass the message; the gate lives in one place.
|
|
34
|
-
*
|
|
35
|
-
* Gated on **stdout**, not stderr: stdout is the stream a script actually reads
|
|
36
|
-
* (`$(trawl …)`, `> out.txt`, `| jq`), independent of whatever `spin()` decides
|
|
37
|
-
* about stderr. So an interactive session, where the ora spinner already
|
|
38
|
-
* confirmed on stderr, never gets a duplicate line here.
|
|
39
|
-
*
|
|
40
|
-
* Never call this on a `--json` path: under `--json`, stdout must stay exactly
|
|
41
|
-
* one parseable document.
|
|
42
|
-
*
|
|
43
|
-
* Known residual, unchanged from #160: stdout attached to a real terminal while
|
|
44
|
-
* stderr is separately redirected still prints nothing on either stream. That is
|
|
45
|
-
* not the reported or common shape (full redirection, or stdout-only capture),
|
|
46
|
-
* and widening the gate would put a duplicate line in front of interactive users.
|
|
47
|
-
*/
|
|
48
8
|
export function confirmNonTTY(message) {
|
|
49
9
|
if (!process.stdout.isTTY)
|
|
50
10
|
console.log(message);
|
|
@@ -1,33 +1,14 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Pure CDP -> storageState mapping + domain scoping for `scraps account
|
|
3
|
-
* session capture` (trawl_cli#183). No I/O here — every function takes
|
|
4
|
-
* plain data and returns plain data, so the full cookie/localStorage
|
|
5
|
-
* matrix (sameSite casing, expires-in-seconds, domain scoping, …) is
|
|
6
|
-
* testable without a real Chrome.
|
|
7
|
-
*
|
|
8
|
-
* The output shape mirrors trawl_node#1976's ingest validator
|
|
9
|
-
* (modules/scraps/helpers/sessionShape.js) — CookieData fields, `sameSite`
|
|
10
|
-
* in `Strict|Lax|None`, `expires` in seconds, `domain` required. This file
|
|
11
|
-
* pre-normalizes to that shape so the server's validator never has to
|
|
12
|
-
* reject the WHOLE batch over one anomalous cookie the client could have
|
|
13
|
-
* caught itself (see normalizeCdpCookie's sameSite=None handling below).
|
|
14
|
-
*/
|
|
15
|
-
/** A cookie as CDP's `Storage.getCookies`/`Network.Cookie` returns it. */
|
|
16
1
|
export interface RawCdpCookie {
|
|
17
2
|
name: string;
|
|
18
3
|
value: string;
|
|
19
4
|
domain: string;
|
|
20
5
|
path?: string;
|
|
21
|
-
/** Unix seconds; -1 denotes a session cookie (matches chrome.cookies shape). */
|
|
22
6
|
expires?: number;
|
|
23
7
|
httpOnly?: boolean;
|
|
24
8
|
secure?: boolean;
|
|
25
|
-
/** CDP's `Network.CookieSameSite` enum — nominally `Strict|Lax|None`, but
|
|
26
|
-
* normalized defensively (case-insensitively) rather than trusted as-is. */
|
|
27
9
|
sameSite?: string;
|
|
28
10
|
[key: string]: unknown;
|
|
29
11
|
}
|
|
30
|
-
/** trawl_node sessionShape.js's `CookieData` — only these fields are ever forwarded. */
|
|
31
12
|
export interface CookieData {
|
|
32
13
|
name: string;
|
|
33
14
|
value: string;
|
|
@@ -38,7 +19,6 @@ export interface CookieData {
|
|
|
38
19
|
sameSite?: 'Strict' | 'Lax' | 'None';
|
|
39
20
|
expires?: number;
|
|
40
21
|
}
|
|
41
|
-
/** One origin's captured localStorage, pre-dedup/pre-scope. */
|
|
42
22
|
export interface RawOriginLocalStorage {
|
|
43
23
|
origin: string;
|
|
44
24
|
entries: Array<{
|
|
@@ -57,83 +37,8 @@ export interface StorageState {
|
|
|
57
37
|
cookies: CookieData[];
|
|
58
38
|
origins: OriginStorage[];
|
|
59
39
|
}
|
|
60
|
-
/**
|
|
61
|
-
* Domain scoping (trawl_cli#183 review finding 2 — cross-tenant leak). The
|
|
62
|
-
* previous version of this file peeled a hostname down to a "registrable
|
|
63
|
-
* domain" (last 2-3 labels, with a small hardcoded allowlist for
|
|
64
|
-
* `co.uk`-shaped suffixes) and treated any host sharing that suffix as in
|
|
65
|
-
* scope. That is wrong for any multi-tenant hosting domain NOT in the
|
|
66
|
-
* allowlist — `getRegistrableDomain('alice.github.io')` collapsed to
|
|
67
|
-
* `'github.io'`, so `isHostInScope('bob.github.io', 'github.io')` came back
|
|
68
|
-
* `true`: a scrap targeting `alice.github.io` would upload `bob.github.io`'s
|
|
69
|
-
* cookies too (same for `herokuapp.com`, `vercel.app`, `s3.amazonaws.com`,
|
|
70
|
-
* …) — every one of them would need its own allowlist entry, which is
|
|
71
|
-
* exactly the Public Suffix List this file deliberately avoids depending
|
|
72
|
-
* on.
|
|
73
|
-
*
|
|
74
|
-
* The fix NARROWS instead: scope every capture to RFC 6265 §5.1.3
|
|
75
|
-
* domain-matching against the scrap URL's own host (`getTargetHost`) —
|
|
76
|
-
* never a peeled/derived domain, so there is nothing left to get wrong
|
|
77
|
-
* about a given hosting provider's suffix.
|
|
78
|
-
*
|
|
79
|
-
* Verified against a real Chrome 152 (loopback probe, `--host-resolver-rules`
|
|
80
|
-
* remapping `alice.github.io`/`app.example.com`/etc. to 127.0.0.1, no real
|
|
81
|
-
* network traffic):
|
|
82
|
-
* - Chrome REFUSES to let a page at `alice.github.io` set a cookie with
|
|
83
|
-
* `Domain=github.io` at all — it never reaches the cookie jar. That's
|
|
84
|
-
* Chrome's own PSL enforcement on the SET side, and it's what makes the
|
|
85
|
-
* zero-PSL-dependency approach below safe: a public-suffix-scoped cookie
|
|
86
|
-
* simply cannot exist in a real jar to be mis-scoped in the first place.
|
|
87
|
-
* - `Storage.getCookies` DOES distinguish a host-only cookie from an
|
|
88
|
-
* explicit-`Domain=` one at the CDP layer, even when the explicit domain
|
|
89
|
-
* string equals the current host exactly: a bare `document.cookie="k=v"`
|
|
90
|
-
* on `alice.github.io` comes back with `domain:"alice.github.io"` (no
|
|
91
|
-
* leading dot); `document.cookie="k=v; domain=alice.github.io"` — same
|
|
92
|
-
* effective host — comes back `domain:".alice.github.io"` (leading dot).
|
|
93
|
-
* That leading dot is real, load-bearing information: it is exactly
|
|
94
|
-
* RFC 6265's own signal for "this is a domain-match cookie, not a
|
|
95
|
-
* host-only one" and `isCookieDomainInScope` below reads it as such
|
|
96
|
-
* rather than stripping it.
|
|
97
|
-
*
|
|
98
|
-
* One deliberate loss, not a bug: a host-only cookie set on a SIBLING host
|
|
99
|
-
* (e.g. `auth.example.com` while the scrap targets `app.example.com`) is
|
|
100
|
-
* out of scope here. A real browser would never send that cookie to
|
|
101
|
-
* `app.example.com` either — host-only means exactly that host — so
|
|
102
|
-
* nothing replay-relevant is lost; it would only matter if the scrap
|
|
103
|
-
* script itself later navigates to `auth.example.com`, which this capture
|
|
104
|
-
* has no visibility into anyway.
|
|
105
|
-
*/
|
|
106
|
-
/**
|
|
107
|
-
* @desc The exact host `targetUrl` will actually be requested at — the
|
|
108
|
-
* single anchor every scope decision in this file is relative to. NOT a
|
|
109
|
-
* peeled "registrable domain": no suffix list, no label-counting, no
|
|
110
|
-
* allowlist to keep up to date.
|
|
111
|
-
*/
|
|
112
40
|
export declare function getTargetHost(targetUrl: string): string;
|
|
113
|
-
/**
|
|
114
|
-
* @desc RFC 6265 §5.1.3 domain-matching for one captured cookie's `domain`
|
|
115
|
-
* attribute exactly as CDP's `Storage.getCookies` reports it (leading dot
|
|
116
|
-
* preserved when Chrome ever wrote one, absent for a host-only cookie —
|
|
117
|
-
* see the module doc comment for how that was verified against a real
|
|
118
|
-
* Chrome).
|
|
119
|
-
* - Leading dot (a domain-match cookie): in scope for the target host
|
|
120
|
-
* itself, or any subdomain of it.
|
|
121
|
-
* - No leading dot (a host-only cookie): in scope ONLY for that exact
|
|
122
|
-
* host — never a subdomain, never a sibling, never a parent. This is
|
|
123
|
-
* what excludes `bob.github.io` from a capture targeting
|
|
124
|
-
* `alice.github.io`, and `auth.example.com` from one targeting
|
|
125
|
-
* `app.example.com`.
|
|
126
|
-
*/
|
|
127
41
|
export declare function isCookieDomainInScope(rawDomain: string, targetHost: string): boolean;
|
|
128
|
-
/**
|
|
129
|
-
* @desc True when a captured page's own hostname (`originHost` — an
|
|
130
|
-
* origin's localStorage has no "domain attribute" concept, unlike a
|
|
131
|
-
* cookie) is the target host itself, or a subdomain of it. Anchored the
|
|
132
|
-
* opposite way from a host-only cookie check: here the TARGET is the root
|
|
133
|
-
* and the origin must fall under it, so a sibling (`auth.example.com` for
|
|
134
|
-
* a scrap on `app.example.com`) is still excluded, but a genuine
|
|
135
|
-
* subdomain page opened during the same login flow is not.
|
|
136
|
-
*/
|
|
137
42
|
export declare function isOriginHostInScope(originHost: string, targetHost: string): boolean;
|
|
138
43
|
export interface MapCookiesResult {
|
|
139
44
|
cookies: CookieData[];
|
|
@@ -141,27 +46,10 @@ export interface MapCookiesResult {
|
|
|
141
46
|
droppedOutOfScope: number;
|
|
142
47
|
droppedInvalid: number;
|
|
143
48
|
}
|
|
144
|
-
/**
|
|
145
|
-
* @desc Scope + normalize a raw CDP cookie jar down to the cookies a real
|
|
146
|
-
* browser would actually send to `targetUrl`'s host (`isCookieDomainInScope`
|
|
147
|
-
* — RFC 6265 domain-matching, never a peeled registrable domain). Never
|
|
148
|
-
* throws — a structurally broken cookie is counted and dropped, not fatal
|
|
149
|
-
* to the rest of the capture (see the "batch same class failures" rule:
|
|
150
|
-
* this IS the sweep of the offending class, applied once here rather than
|
|
151
|
-
* duplicated at each call site).
|
|
152
|
-
*/
|
|
153
49
|
export declare function mapCookies(rawCookies: RawCdpCookie[], targetUrl: string): MapCookiesResult;
|
|
154
50
|
export interface MapOriginsResult {
|
|
155
51
|
origins: OriginStorage[];
|
|
156
52
|
totalSeen: number;
|
|
157
53
|
droppedOutOfScope: number;
|
|
158
54
|
}
|
|
159
|
-
/**
|
|
160
|
-
* @desc Scope raw per-origin localStorage dumps down to origins that are
|
|
161
|
-
* `targetUrl`'s own host or a subdomain of it (`isOriginHostInScope`).
|
|
162
|
-
* `origin` is passed through unchanged (it is already a bare
|
|
163
|
-
* `scheme://host[:port]` by construction — callers build it via
|
|
164
|
-
* `new URL(pageUrl).origin`) so it satisfies sessionShape.js's bare-origin
|
|
165
|
-
* check untouched.
|
|
166
|
-
*/
|
|
167
55
|
export declare function mapOrigins(rawOrigins: RawOriginLocalStorage[], targetUrl: string): MapOriginsResult;
|
|
@@ -1,86 +1,6 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Pure CDP -> storageState mapping + domain scoping for `scraps account
|
|
3
|
-
* session capture` (trawl_cli#183). No I/O here — every function takes
|
|
4
|
-
* plain data and returns plain data, so the full cookie/localStorage
|
|
5
|
-
* matrix (sameSite casing, expires-in-seconds, domain scoping, …) is
|
|
6
|
-
* testable without a real Chrome.
|
|
7
|
-
*
|
|
8
|
-
* The output shape mirrors trawl_node#1976's ingest validator
|
|
9
|
-
* (modules/scraps/helpers/sessionShape.js) — CookieData fields, `sameSite`
|
|
10
|
-
* in `Strict|Lax|None`, `expires` in seconds, `domain` required. This file
|
|
11
|
-
* pre-normalizes to that shape so the server's validator never has to
|
|
12
|
-
* reject the WHOLE batch over one anomalous cookie the client could have
|
|
13
|
-
* caught itself (see normalizeCdpCookie's sameSite=None handling below).
|
|
14
|
-
*/
|
|
15
|
-
/**
|
|
16
|
-
* Domain scoping (trawl_cli#183 review finding 2 — cross-tenant leak). The
|
|
17
|
-
* previous version of this file peeled a hostname down to a "registrable
|
|
18
|
-
* domain" (last 2-3 labels, with a small hardcoded allowlist for
|
|
19
|
-
* `co.uk`-shaped suffixes) and treated any host sharing that suffix as in
|
|
20
|
-
* scope. That is wrong for any multi-tenant hosting domain NOT in the
|
|
21
|
-
* allowlist — `getRegistrableDomain('alice.github.io')` collapsed to
|
|
22
|
-
* `'github.io'`, so `isHostInScope('bob.github.io', 'github.io')` came back
|
|
23
|
-
* `true`: a scrap targeting `alice.github.io` would upload `bob.github.io`'s
|
|
24
|
-
* cookies too (same for `herokuapp.com`, `vercel.app`, `s3.amazonaws.com`,
|
|
25
|
-
* …) — every one of them would need its own allowlist entry, which is
|
|
26
|
-
* exactly the Public Suffix List this file deliberately avoids depending
|
|
27
|
-
* on.
|
|
28
|
-
*
|
|
29
|
-
* The fix NARROWS instead: scope every capture to RFC 6265 §5.1.3
|
|
30
|
-
* domain-matching against the scrap URL's own host (`getTargetHost`) —
|
|
31
|
-
* never a peeled/derived domain, so there is nothing left to get wrong
|
|
32
|
-
* about a given hosting provider's suffix.
|
|
33
|
-
*
|
|
34
|
-
* Verified against a real Chrome 152 (loopback probe, `--host-resolver-rules`
|
|
35
|
-
* remapping `alice.github.io`/`app.example.com`/etc. to 127.0.0.1, no real
|
|
36
|
-
* network traffic):
|
|
37
|
-
* - Chrome REFUSES to let a page at `alice.github.io` set a cookie with
|
|
38
|
-
* `Domain=github.io` at all — it never reaches the cookie jar. That's
|
|
39
|
-
* Chrome's own PSL enforcement on the SET side, and it's what makes the
|
|
40
|
-
* zero-PSL-dependency approach below safe: a public-suffix-scoped cookie
|
|
41
|
-
* simply cannot exist in a real jar to be mis-scoped in the first place.
|
|
42
|
-
* - `Storage.getCookies` DOES distinguish a host-only cookie from an
|
|
43
|
-
* explicit-`Domain=` one at the CDP layer, even when the explicit domain
|
|
44
|
-
* string equals the current host exactly: a bare `document.cookie="k=v"`
|
|
45
|
-
* on `alice.github.io` comes back with `domain:"alice.github.io"` (no
|
|
46
|
-
* leading dot); `document.cookie="k=v; domain=alice.github.io"` — same
|
|
47
|
-
* effective host — comes back `domain:".alice.github.io"` (leading dot).
|
|
48
|
-
* That leading dot is real, load-bearing information: it is exactly
|
|
49
|
-
* RFC 6265's own signal for "this is a domain-match cookie, not a
|
|
50
|
-
* host-only one" and `isCookieDomainInScope` below reads it as such
|
|
51
|
-
* rather than stripping it.
|
|
52
|
-
*
|
|
53
|
-
* One deliberate loss, not a bug: a host-only cookie set on a SIBLING host
|
|
54
|
-
* (e.g. `auth.example.com` while the scrap targets `app.example.com`) is
|
|
55
|
-
* out of scope here. A real browser would never send that cookie to
|
|
56
|
-
* `app.example.com` either — host-only means exactly that host — so
|
|
57
|
-
* nothing replay-relevant is lost; it would only matter if the scrap
|
|
58
|
-
* script itself later navigates to `auth.example.com`, which this capture
|
|
59
|
-
* has no visibility into anyway.
|
|
60
|
-
*/
|
|
61
|
-
/**
|
|
62
|
-
* @desc The exact host `targetUrl` will actually be requested at — the
|
|
63
|
-
* single anchor every scope decision in this file is relative to. NOT a
|
|
64
|
-
* peeled "registrable domain": no suffix list, no label-counting, no
|
|
65
|
-
* allowlist to keep up to date.
|
|
66
|
-
*/
|
|
67
1
|
export function getTargetHost(targetUrl) {
|
|
68
2
|
return new URL(targetUrl).hostname.toLowerCase();
|
|
69
3
|
}
|
|
70
|
-
/**
|
|
71
|
-
* @desc RFC 6265 §5.1.3 domain-matching for one captured cookie's `domain`
|
|
72
|
-
* attribute exactly as CDP's `Storage.getCookies` reports it (leading dot
|
|
73
|
-
* preserved when Chrome ever wrote one, absent for a host-only cookie —
|
|
74
|
-
* see the module doc comment for how that was verified against a real
|
|
75
|
-
* Chrome).
|
|
76
|
-
* - Leading dot (a domain-match cookie): in scope for the target host
|
|
77
|
-
* itself, or any subdomain of it.
|
|
78
|
-
* - No leading dot (a host-only cookie): in scope ONLY for that exact
|
|
79
|
-
* host — never a subdomain, never a sibling, never a parent. This is
|
|
80
|
-
* what excludes `bob.github.io` from a capture targeting
|
|
81
|
-
* `alice.github.io`, and `auth.example.com` from one targeting
|
|
82
|
-
* `app.example.com`.
|
|
83
|
-
*/
|
|
84
4
|
export function isCookieDomainInScope(rawDomain, targetHost) {
|
|
85
5
|
const host = targetHost.toLowerCase();
|
|
86
6
|
if (rawDomain.startsWith('.')) {
|
|
@@ -89,15 +9,6 @@ export function isCookieDomainInScope(rawDomain, targetHost) {
|
|
|
89
9
|
}
|
|
90
10
|
return host === rawDomain.toLowerCase();
|
|
91
11
|
}
|
|
92
|
-
/**
|
|
93
|
-
* @desc True when a captured page's own hostname (`originHost` — an
|
|
94
|
-
* origin's localStorage has no "domain attribute" concept, unlike a
|
|
95
|
-
* cookie) is the target host itself, or a subdomain of it. Anchored the
|
|
96
|
-
* opposite way from a host-only cookie check: here the TARGET is the root
|
|
97
|
-
* and the origin must fall under it, so a sibling (`auth.example.com` for
|
|
98
|
-
* a scrap on `app.example.com`) is still excluded, but a genuine
|
|
99
|
-
* subdomain page opened during the same login flow is not.
|
|
100
|
-
*/
|
|
101
12
|
export function isOriginHostInScope(originHost, targetHost) {
|
|
102
13
|
const h = originHost.toLowerCase();
|
|
103
14
|
const target = targetHost.toLowerCase();
|
|
@@ -108,29 +19,9 @@ const CANONICAL_SAME_SITE = {
|
|
|
108
19
|
lax: 'Lax',
|
|
109
20
|
none: 'None',
|
|
110
21
|
};
|
|
111
|
-
/**
|
|
112
|
-
* @desc Recase any CDP/browser sameSite spelling onto CookieData's exact
|
|
113
|
-
* casing. Returns null for anything unrecognized (CDP's own "unspecified"
|
|
114
|
-
* included) — the field is then omitted entirely rather than guessed,
|
|
115
|
-
* mirroring sessionShape.js's own 'unspecified' handling.
|
|
116
|
-
*/
|
|
117
22
|
function canonicalizeSameSite(value) {
|
|
118
23
|
return CANONICAL_SAME_SITE[value.toLowerCase()] ?? null;
|
|
119
24
|
}
|
|
120
|
-
/**
|
|
121
|
-
* @desc Map one CDP cookie onto CookieData, or null if it's structurally
|
|
122
|
-
* unusable (no domain, no name/value). A `sameSite=None` cookie missing
|
|
123
|
-
* `secure` is NOT dropped outright — dropping only the `sameSite`
|
|
124
|
-
* attribute (rather than the whole cookie) is deliberate: sessionShape.js's
|
|
125
|
-
* `normalizeCookies` throws on that combination, and since it processes
|
|
126
|
-
* the array with a single `.map()`, ONE such cookie would reject the
|
|
127
|
-
* ENTIRE upload with a 422 — a capture that otherwise succeeded would be
|
|
128
|
-
* thrown away over one anomalous cookie. A real, currently-live browser
|
|
129
|
-
* cookie can't actually be in this state (Chrome refuses to store a `None`
|
|
130
|
-
* cookie without `Secure`), so in practice this only guards a defensive
|
|
131
|
-
* edge; when it does happen, omitting `sameSite` keeps the cookie (and the
|
|
132
|
-
* rest of the batch) valid.
|
|
133
|
-
*/
|
|
134
25
|
function normalizeCdpCookie(raw) {
|
|
135
26
|
if (typeof raw.name !== 'string' || raw.name.length === 0)
|
|
136
27
|
return null;
|
|
@@ -156,15 +47,6 @@ function normalizeCdpCookie(raw) {
|
|
|
156
47
|
}
|
|
157
48
|
return cookie;
|
|
158
49
|
}
|
|
159
|
-
/**
|
|
160
|
-
* @desc Scope + normalize a raw CDP cookie jar down to the cookies a real
|
|
161
|
-
* browser would actually send to `targetUrl`'s host (`isCookieDomainInScope`
|
|
162
|
-
* — RFC 6265 domain-matching, never a peeled registrable domain). Never
|
|
163
|
-
* throws — a structurally broken cookie is counted and dropped, not fatal
|
|
164
|
-
* to the rest of the capture (see the "batch same class failures" rule:
|
|
165
|
-
* this IS the sweep of the offending class, applied once here rather than
|
|
166
|
-
* duplicated at each call site).
|
|
167
|
-
*/
|
|
168
50
|
export function mapCookies(rawCookies, targetUrl) {
|
|
169
51
|
const targetHost = getTargetHost(targetUrl);
|
|
170
52
|
let droppedOutOfScope = 0;
|
|
@@ -175,11 +57,6 @@ export function mapCookies(rawCookies, targetUrl) {
|
|
|
175
57
|
droppedInvalid++;
|
|
176
58
|
continue;
|
|
177
59
|
}
|
|
178
|
-
// The leading '.' (or its absence) is meaningful — see
|
|
179
|
-
// isCookieDomainInScope's own doc comment — so it is NOT stripped here;
|
|
180
|
-
// only the OUTPUT cookie's `domain` field stays untouched either way
|
|
181
|
-
// (normalizeCdpCookie passes raw.domain through as-is, dot included),
|
|
182
|
-
// since the server needs the exact original attribute to replay it.
|
|
183
60
|
if (!isCookieDomainInScope(raw.domain, targetHost)) {
|
|
184
61
|
droppedOutOfScope++;
|
|
185
62
|
continue;
|
|
@@ -193,14 +70,6 @@ export function mapCookies(rawCookies, targetUrl) {
|
|
|
193
70
|
}
|
|
194
71
|
return { cookies, totalSeen: rawCookies.length, droppedOutOfScope, droppedInvalid };
|
|
195
72
|
}
|
|
196
|
-
/**
|
|
197
|
-
* @desc Scope raw per-origin localStorage dumps down to origins that are
|
|
198
|
-
* `targetUrl`'s own host or a subdomain of it (`isOriginHostInScope`).
|
|
199
|
-
* `origin` is passed through unchanged (it is already a bare
|
|
200
|
-
* `scheme://host[:port]` by construction — callers build it via
|
|
201
|
-
* `new URL(pageUrl).origin`) so it satisfies sessionShape.js's bare-origin
|
|
202
|
-
* check untouched.
|
|
203
|
-
*/
|
|
204
73
|
export function mapOrigins(rawOrigins, targetUrl) {
|
|
205
74
|
const targetHost = getTargetHost(targetUrl);
|
|
206
75
|
let droppedOutOfScope = 0;
|
package/dist/lib/tips.d.ts
CHANGED
|
@@ -1,13 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Pure gate: true when the tip should print. Every external signal (json
|
|
3
|
-
* flag, TTY-ness, opt-out, persisted timestamp, current time) is a
|
|
4
|
-
* parameter rather than read internally, so this is trivially testable
|
|
5
|
-
* without mocking process.env/stdout/Date/config. Does NOT cover the
|
|
6
|
-
* server-side `invitations.userFacing` flag (#153) — that check requires a
|
|
7
|
-
* network round-trip, so it's kept out of this pure/sync gate and only
|
|
8
|
-
* consulted once this gate is already true (see
|
|
9
|
-
* isReferralProgramUserFacing + maybeShowReferralTip below).
|
|
10
|
-
*/
|
|
11
1
|
export declare function isReferralTipDue(opts: {
|
|
12
2
|
json?: boolean;
|
|
13
3
|
isTTY: boolean;
|
|
@@ -15,34 +5,6 @@ export declare function isReferralTipDue(opts: {
|
|
|
15
5
|
lastShownAt: number;
|
|
16
6
|
now: number;
|
|
17
7
|
}): boolean;
|
|
18
|
-
/**
|
|
19
|
-
* Best-effort: print one subtle line inviting the user to refer a builder,
|
|
20
|
-
* throttled to once per 7 days via a timestamp persisted in the CLI's
|
|
21
|
-
* existing Conf-backed config store (lib/config.ts — same store as
|
|
22
|
-
* apiUrl/token/telemetry/updateNotifier), AND gated on the server's
|
|
23
|
-
* `invitations.userFacing` flag (#153) once that throttle window is open.
|
|
24
|
-
*
|
|
25
|
-
* This only gates on throttle/TTY/`--json`/opt-out/server-flag — it does
|
|
26
|
-
* NOT know whether the underlying command actually succeeded. Callers must
|
|
27
|
-
* only invoke it on a genuine success (see commands/scraps.ts `run <id>`'s
|
|
28
|
-
* action, which also checks pollRunProgress's exit code under `--watch`
|
|
29
|
-
* before calling this). Any state read/write/network failure (corrupt
|
|
30
|
-
* config file, disk error, unreachable server, …) is swallowed silently —
|
|
31
|
-
* a broken tip must never break a run.
|
|
32
|
-
*
|
|
33
|
-
* #153 — the timestamp is now persisted BEFORE the print (#151 originally
|
|
34
|
-
* had it after). Two overlapping calls that both read `due=true` before
|
|
35
|
-
* either has persisted can still both decide to print (a file-based config
|
|
36
|
-
* store can't fully arbitrate that without a lock this feature doesn't
|
|
37
|
-
* warrant) — but persist-before-print closes the narrower, guaranteed-
|
|
38
|
-
* recurring failure mode: previously, if the print succeeded but the
|
|
39
|
-
* subsequent config.set failed (or simply hadn't landed yet when a second
|
|
40
|
-
* call read the same stale timestamp), the tip could print again on every
|
|
41
|
-
* later run forever, since the "already shown" state never advanced. With
|
|
42
|
-
* persist first, a lost race collapses to "tip skipped this time, timestamp
|
|
43
|
-
* still unset, next due window free to retry" — never a guaranteed
|
|
44
|
-
* repeat print.
|
|
45
|
-
*/
|
|
46
8
|
export declare function maybeShowReferralTip(opts?: {
|
|
47
9
|
json?: boolean;
|
|
48
10
|
}): Promise<void>;
|