@trawlme/cli 3.10.0 → 3.12.0

package/dist/lib/storage-state.d.ts

@@ -0,0 +1,167 @@
+/**
+ * Pure CDP -> storageState mapping + domain scoping for `scraps account
+ * session capture` (trawl_cli#183). No I/O here — every function takes
+ * plain data and returns plain data, so the full cookie/localStorage
+ * matrix (sameSite casing, expires-in-seconds, domain scoping, …) is
+ * testable without a real Chrome.
+ *
+ * The output shape mirrors trawl_node#1976's ingest validator
+ * (modules/scraps/helpers/sessionShape.js) — CookieData fields, `sameSite`
+ * in `Strict|Lax|None`, `expires` in seconds, `domain` required. This file
+ * pre-normalizes to that shape so the server's validator never has to
+ * reject the WHOLE batch over one anomalous cookie the client could have
+ * caught itself (see normalizeCdpCookie's sameSite=None handling below).
+ */
+/** A cookie as CDP's `Storage.getCookies`/`Network.Cookie` returns it. */
+export interface RawCdpCookie {
+    name: string;
+    value: string;
+    domain: string;
+    path?: string;
+    /** Unix seconds; -1 denotes a session cookie (matches chrome.cookies shape). */
+    expires?: number;
+    httpOnly?: boolean;
+    secure?: boolean;
+    /** CDP's `Network.CookieSameSite` enum — nominally `Strict|Lax|None`, but
+     *  normalized defensively (case-insensitively) rather than trusted as-is. */
+    sameSite?: string;
+    [key: string]: unknown;
+}
+/** trawl_node sessionShape.js's `CookieData` — only these fields are ever forwarded. */
+export interface CookieData {
+    name: string;
+    value: string;
+    domain: string;
+    path?: string;
+    secure?: boolean;
+    httpOnly?: boolean;
+    sameSite?: 'Strict' | 'Lax' | 'None';
+    expires?: number;
+}
+/** One origin's captured localStorage, pre-dedup/pre-scope. */
+export interface RawOriginLocalStorage {
+    origin: string;
+    entries: Array<{
+        name: string;
+        value: string;
+    }>;
+}
+export interface OriginStorage {
+    origin: string;
+    localStorage: Array<{
+        name: string;
+        value: string;
+    }>;
+}
+export interface StorageState {
+    cookies: CookieData[];
+    origins: OriginStorage[];
+}
+/**
+ * Domain scoping (trawl_cli#183 review finding 2 — cross-tenant leak). The
+ * previous version of this file peeled a hostname down to a "registrable
+ * domain" (last 2-3 labels, with a small hardcoded allowlist for
+ * `co.uk`-shaped suffixes) and treated any host sharing that suffix as in
+ * scope. That is wrong for any multi-tenant hosting domain NOT in the
+ * allowlist — `getRegistrableDomain('alice.github.io')` collapsed to
+ * `'github.io'`, so `isHostInScope('bob.github.io', 'github.io')` came back
+ * `true`: a scrap targeting `alice.github.io` would upload `bob.github.io`'s
+ * cookies too (same for `herokuapp.com`, `vercel.app`, `s3.amazonaws.com`,
+ * …) — every one of them would need its own allowlist entry, which is
+ * exactly the Public Suffix List this file deliberately avoids depending
+ * on.
+ *
+ * The fix NARROWS instead: scope every capture to RFC 6265 §5.1.3
+ * domain-matching against the scrap URL's own host (`getTargetHost`) —
+ * never a peeled/derived domain, so there is nothing left to get wrong
+ * about a given hosting provider's suffix.
+ *
+ * Verified against a real Chrome 152 (loopback probe, `--host-resolver-rules`
+ * remapping `alice.github.io`/`app.example.com`/etc. to 127.0.0.1, no real
+ * network traffic):
+ *  - Chrome REFUSES to let a page at `alice.github.io` set a cookie with
+ *    `Domain=github.io` at all — it never reaches the cookie jar. That's
+ *    Chrome's own PSL enforcement on the SET side, and it's what makes the
+ *    zero-PSL-dependency approach below safe: a public-suffix-scoped cookie
+ *    simply cannot exist in a real jar to be mis-scoped in the first place.
+ *  - `Storage.getCookies` DOES distinguish a host-only cookie from an
+ *    explicit-`Domain=` one at the CDP layer, even when the explicit domain
+ *    string equals the current host exactly: a bare `document.cookie="k=v"`
+ *    on `alice.github.io` comes back with `domain:"alice.github.io"` (no
+ *    leading dot); `document.cookie="k=v; domain=alice.github.io"` — same
+ *    effective host — comes back `domain:".alice.github.io"` (leading dot).
+ *    That leading dot is real, load-bearing information: it is exactly
+ *    RFC 6265's own signal for "this is a domain-match cookie, not a
+ *    host-only one" and `isCookieDomainInScope` below reads it as such
+ *    rather than stripping it.
+ *
+ * One deliberate loss, not a bug: a host-only cookie set on a SIBLING host
+ * (e.g. `auth.example.com` while the scrap targets `app.example.com`) is
+ * out of scope here. A real browser would never send that cookie to
+ * `app.example.com` either — host-only means exactly that host — so
+ * nothing replay-relevant is lost; it would only matter if the scrap
+ * script itself later navigates to `auth.example.com`, which this capture
+ * has no visibility into anyway.
+ */
+/**
+ * @desc The exact host `targetUrl` will actually be requested at — the
+ * single anchor every scope decision in this file is relative to. NOT a
+ * peeled "registrable domain": no suffix list, no label-counting, no
+ * allowlist to keep up to date.
+ */
+export declare function getTargetHost(targetUrl: string): string;
+/**
+ * @desc RFC 6265 §5.1.3 domain-matching for one captured cookie's `domain`
+ * attribute exactly as CDP's `Storage.getCookies` reports it (leading dot
+ * preserved when Chrome ever wrote one, absent for a host-only cookie —
+ * see the module doc comment for how that was verified against a real
+ * Chrome).
+ *  - Leading dot (a domain-match cookie): in scope for the target host
+ *    itself, or any subdomain of it.
+ *  - No leading dot (a host-only cookie): in scope ONLY for that exact
+ *    host — never a subdomain, never a sibling, never a parent. This is
+ *    what excludes `bob.github.io` from a capture targeting
+ *    `alice.github.io`, and `auth.example.com` from one targeting
+ *    `app.example.com`.
+ */
+export declare function isCookieDomainInScope(rawDomain: string, targetHost: string): boolean;
+/**
+ * @desc True when a captured page's own hostname (`originHost` — an
+ * origin's localStorage has no "domain attribute" concept, unlike a
+ * cookie) is the target host itself, or a subdomain of it. Anchored the
+ * opposite way from a host-only cookie check: here the TARGET is the root
+ * and the origin must fall under it, so a sibling (`auth.example.com` for
+ * a scrap on `app.example.com`) is still excluded, but a genuine
+ * subdomain page opened during the same login flow is not.
+ */
+export declare function isOriginHostInScope(originHost: string, targetHost: string): boolean;
+export interface MapCookiesResult {
+    cookies: CookieData[];
+    totalSeen: number;
+    droppedOutOfScope: number;
+    droppedInvalid: number;
+}
+/**
+ * @desc Scope + normalize a raw CDP cookie jar down to the cookies a real
+ * browser would actually send to `targetUrl`'s host (`isCookieDomainInScope`
+ * — RFC 6265 domain-matching, never a peeled registrable domain). Never
+ * throws — a structurally broken cookie is counted and dropped, not fatal
+ * to the rest of the capture (see the "batch same class failures" rule:
+ * this IS the sweep of the offending class, applied once here rather than
+ * duplicated at each call site).
+ */
+export declare function mapCookies(rawCookies: RawCdpCookie[], targetUrl: string): MapCookiesResult;
+export interface MapOriginsResult {
+    origins: OriginStorage[];
+    totalSeen: number;
+    droppedOutOfScope: number;
+}
+/**
+ * @desc Scope raw per-origin localStorage dumps down to origins that are
+ * `targetUrl`'s own host or a subdomain of it (`isOriginHostInScope`).
+ * `origin` is passed through unchanged (it is already a bare
+ * `scheme://host[:port]` by construction — callers build it via
+ * `new URL(pageUrl).origin`) so it satisfies sessionShape.js's bare-origin
+ * check untouched.
+ */
+export declare function mapOrigins(rawOrigins: RawOriginLocalStorage[], targetUrl: string): MapOriginsResult;

package/dist/lib/storage-state.js

@@ -0,0 +1,227 @@
+/**
+ * Pure CDP -> storageState mapping + domain scoping for `scraps account
+ * session capture` (trawl_cli#183). No I/O here — every function takes
+ * plain data and returns plain data, so the full cookie/localStorage
+ * matrix (sameSite casing, expires-in-seconds, domain scoping, …) is
+ * testable without a real Chrome.
+ *
+ * The output shape mirrors trawl_node#1976's ingest validator
+ * (modules/scraps/helpers/sessionShape.js) — CookieData fields, `sameSite`
+ * in `Strict|Lax|None`, `expires` in seconds, `domain` required. This file
+ * pre-normalizes to that shape so the server's validator never has to
+ * reject the WHOLE batch over one anomalous cookie the client could have
+ * caught itself (see normalizeCdpCookie's sameSite=None handling below).
+ */
+/**
+ * Domain scoping (trawl_cli#183 review finding 2 — cross-tenant leak). The
+ * previous version of this file peeled a hostname down to a "registrable
+ * domain" (last 2-3 labels, with a small hardcoded allowlist for
+ * `co.uk`-shaped suffixes) and treated any host sharing that suffix as in
+ * scope. That is wrong for any multi-tenant hosting domain NOT in the
+ * allowlist — `getRegistrableDomain('alice.github.io')` collapsed to
+ * `'github.io'`, so `isHostInScope('bob.github.io', 'github.io')` came back
+ * `true`: a scrap targeting `alice.github.io` would upload `bob.github.io`'s
+ * cookies too (same for `herokuapp.com`, `vercel.app`, `s3.amazonaws.com`,
+ * …) — every one of them would need its own allowlist entry, which is
+ * exactly the Public Suffix List this file deliberately avoids depending
+ * on.
+ *
+ * The fix NARROWS instead: scope every capture to RFC 6265 §5.1.3
+ * domain-matching against the scrap URL's own host (`getTargetHost`) —
+ * never a peeled/derived domain, so there is nothing left to get wrong
+ * about a given hosting provider's suffix.
+ *
+ * Verified against a real Chrome 152 (loopback probe, `--host-resolver-rules`
+ * remapping `alice.github.io`/`app.example.com`/etc. to 127.0.0.1, no real
+ * network traffic):
+ *  - Chrome REFUSES to let a page at `alice.github.io` set a cookie with
+ *    `Domain=github.io` at all — it never reaches the cookie jar. That's
+ *    Chrome's own PSL enforcement on the SET side, and it's what makes the
+ *    zero-PSL-dependency approach below safe: a public-suffix-scoped cookie
+ *    simply cannot exist in a real jar to be mis-scoped in the first place.
+ *  - `Storage.getCookies` DOES distinguish a host-only cookie from an
+ *    explicit-`Domain=` one at the CDP layer, even when the explicit domain
+ *    string equals the current host exactly: a bare `document.cookie="k=v"`
+ *    on `alice.github.io` comes back with `domain:"alice.github.io"` (no
+ *    leading dot); `document.cookie="k=v; domain=alice.github.io"` — same
+ *    effective host — comes back `domain:".alice.github.io"` (leading dot).
+ *    That leading dot is real, load-bearing information: it is exactly
+ *    RFC 6265's own signal for "this is a domain-match cookie, not a
+ *    host-only one" and `isCookieDomainInScope` below reads it as such
+ *    rather than stripping it.
+ *
+ * One deliberate loss, not a bug: a host-only cookie set on a SIBLING host
+ * (e.g. `auth.example.com` while the scrap targets `app.example.com`) is
+ * out of scope here. A real browser would never send that cookie to
+ * `app.example.com` either — host-only means exactly that host — so
+ * nothing replay-relevant is lost; it would only matter if the scrap
+ * script itself later navigates to `auth.example.com`, which this capture
+ * has no visibility into anyway.
+ */
+/**
+ * @desc The exact host `targetUrl` will actually be requested at — the
+ * single anchor every scope decision in this file is relative to. NOT a
+ * peeled "registrable domain": no suffix list, no label-counting, no
+ * allowlist to keep up to date.
+ */
+export function getTargetHost(targetUrl) {
+    return new URL(targetUrl).hostname.toLowerCase();
+}
+/**
+ * @desc RFC 6265 §5.1.3 domain-matching for one captured cookie's `domain`
+ * attribute exactly as CDP's `Storage.getCookies` reports it (leading dot
+ * preserved when Chrome ever wrote one, absent for a host-only cookie —
+ * see the module doc comment for how that was verified against a real
+ * Chrome).
+ *  - Leading dot (a domain-match cookie): in scope for the target host
+ *    itself, or any subdomain of it.
+ *  - No leading dot (a host-only cookie): in scope ONLY for that exact
+ *    host — never a subdomain, never a sibling, never a parent. This is
+ *    what excludes `bob.github.io` from a capture targeting
+ *    `alice.github.io`, and `auth.example.com` from one targeting
+ *    `app.example.com`.
+ */
+export function isCookieDomainInScope(rawDomain, targetHost) {
+    const host = targetHost.toLowerCase();
+    if (rawDomain.startsWith('.')) {
+        const domain = rawDomain.slice(1).toLowerCase();
+        return host === domain || host.endsWith(`.${domain}`);
+    }
+    return host === rawDomain.toLowerCase();
+}
+/**
+ * @desc True when a captured page's own hostname (`originHost` — an
+ * origin's localStorage has no "domain attribute" concept, unlike a
+ * cookie) is the target host itself, or a subdomain of it. Anchored the
+ * opposite way from a host-only cookie check: here the TARGET is the root
+ * and the origin must fall under it, so a sibling (`auth.example.com` for
+ * a scrap on `app.example.com`) is still excluded, but a genuine
+ * subdomain page opened during the same login flow is not.
+ */
+export function isOriginHostInScope(originHost, targetHost) {
+    const h = originHost.toLowerCase();
+    const target = targetHost.toLowerCase();
+    return h === target || h.endsWith(`.${target}`);
+}
+const CANONICAL_SAME_SITE = {
+    strict: 'Strict',
+    lax: 'Lax',
+    none: 'None',
+};
+/**
+ * @desc Recase any CDP/browser sameSite spelling onto CookieData's exact
+ * casing. Returns null for anything unrecognized (CDP's own "unspecified"
+ * included) — the field is then omitted entirely rather than guessed,
+ * mirroring sessionShape.js's own 'unspecified' handling.
+ */
+function canonicalizeSameSite(value) {
+    return CANONICAL_SAME_SITE[value.toLowerCase()] ?? null;
+}
+/**
+ * @desc Map one CDP cookie onto CookieData, or null if it's structurally
+ * unusable (no domain, no name/value). A `sameSite=None` cookie missing
+ * `secure` is NOT dropped outright — dropping only the `sameSite`
+ * attribute (rather than the whole cookie) is deliberate: sessionShape.js's
+ * `normalizeCookies` throws on that combination, and since it processes
+ * the array with a single `.map()`, ONE such cookie would reject the
+ * ENTIRE upload with a 422 — a capture that otherwise succeeded would be
+ * thrown away over one anomalous cookie. A real, currently-live browser
+ * cookie can't actually be in this state (Chrome refuses to store a `None`
+ * cookie without `Secure`), so in practice this only guards a defensive
+ * edge; when it does happen, omitting `sameSite` keeps the cookie (and the
+ * rest of the batch) valid.
+ */
+function normalizeCdpCookie(raw) {
+    if (typeof raw.name !== 'string' || raw.name.length === 0)
+        return null;
+    if (typeof raw.value !== 'string')
+        return null;
+    if (typeof raw.domain !== 'string' || raw.domain.length === 0)
+        return null;
+    const cookie = { name: raw.name, value: raw.value, domain: raw.domain };
+    if (typeof raw.path === 'string')
+        cookie.path = raw.path;
+    if (typeof raw.secure === 'boolean')
+        cookie.secure = raw.secure;
+    if (typeof raw.httpOnly === 'boolean')
+        cookie.httpOnly = raw.httpOnly;
+    if (typeof raw.sameSite === 'string' && raw.sameSite.length > 0) {
+        const canon = canonicalizeSameSite(raw.sameSite);
+        if (canon && !(canon === 'None' && cookie.secure !== true)) {
+            cookie.sameSite = canon;
+        }
+    }
+    if (typeof raw.expires === 'number' && !Number.isNaN(raw.expires) && raw.expires !== -1) {
+        cookie.expires = raw.expires;
+    }
+    return cookie;
+}
+/**
+ * @desc Scope + normalize a raw CDP cookie jar down to the cookies a real
+ * browser would actually send to `targetUrl`'s host (`isCookieDomainInScope`
+ * — RFC 6265 domain-matching, never a peeled registrable domain). Never
+ * throws — a structurally broken cookie is counted and dropped, not fatal
+ * to the rest of the capture (see the "batch same class failures" rule:
+ * this IS the sweep of the offending class, applied once here rather than
+ * duplicated at each call site).
+ */
+export function mapCookies(rawCookies, targetUrl) {
+    const targetHost = getTargetHost(targetUrl);
+    let droppedOutOfScope = 0;
+    let droppedInvalid = 0;
+    const cookies = [];
+    for (const raw of rawCookies) {
+        if (!raw || typeof raw.domain !== 'string' || raw.domain.length === 0) {
+            droppedInvalid++;
+            continue;
+        }
+        // The leading '.' (or its absence) is meaningful — see
+        // isCookieDomainInScope's own doc comment — so it is NOT stripped here;
+        // only the OUTPUT cookie's `domain` field stays untouched either way
+        // (normalizeCdpCookie passes raw.domain through as-is, dot included),
+        // since the server needs the exact original attribute to replay it.
+        if (!isCookieDomainInScope(raw.domain, targetHost)) {
+            droppedOutOfScope++;
+            continue;
+        }
+        const mapped = normalizeCdpCookie(raw);
+        if (!mapped) {
+            droppedInvalid++;
+            continue;
+        }
+        cookies.push(mapped);
+    }
+    return { cookies, totalSeen: rawCookies.length, droppedOutOfScope, droppedInvalid };
+}
+/**
+ * @desc Scope raw per-origin localStorage dumps down to origins that are
+ * `targetUrl`'s own host or a subdomain of it (`isOriginHostInScope`).
+ * `origin` is passed through unchanged (it is already a bare
+ * `scheme://host[:port]` by construction — callers build it via
+ * `new URL(pageUrl).origin`) so it satisfies sessionShape.js's bare-origin
+ * check untouched.
+ */
+export function mapOrigins(rawOrigins, targetUrl) {
+    const targetHost = getTargetHost(targetUrl);
+    let droppedOutOfScope = 0;
+    const origins = [];
+    for (const entry of rawOrigins) {
+        let host;
+        try {
+            host = new URL(entry.origin).hostname;
+        }
+        catch {
+            droppedOutOfScope++;
+            continue;
+        }
+        if (!isOriginHostInScope(host, targetHost)) {
+            droppedOutOfScope++;
+            continue;
+        }
+        const localStorage = entry.entries
+            .filter((kv) => typeof kv?.name === 'string' && typeof kv?.value === 'string')
+            .map((kv) => ({ name: kv.name, value: kv.value }));
+        origins.push({ origin: entry.origin, localStorage });
+    }
+    return { origins, totalSeen: rawOrigins.length, droppedOutOfScope };
+}

package/docs/agent-quickstart.md

@@ -166,7 +166,15 @@ just to look helpful, so don't treat its absence as an error.
 
 For the full, versioned machine description of every command (arguments,
 options, aliases, exit codes, error kinds) — instead of parsing this doc —
-run `trawl spec --json`.
+run `trawl spec --json`. It also carries `docsUrl` (resolved from the server
+when declared, else derived from a known first-party API host, else omitted
+— never a guessed URL) and, where a guide exists, a per-command `docs` deep
+link — but the deep link and `llmsUrl` are narrower than `docsUrl`: they only
+ever come from the known-first-party-host derivation, never from a
+server-declared `docsUrl`, since this CLI's own guide slugs have no reason
+to exist on a third party's own docs root. A `failureKind:"auth"` run's
+payload (`doctor`/`data --errors`/`run-info`) carries the same `docs` field,
+under the same rule.
 
 ## Non-interactive contract
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trawlme/cli",
-  "version": "3.10.0",
+  "version": "3.12.0",
   "description": "Trawl CLI — manage scraps from the terminal",
   "type": "module",
   "bin": {