@ultimat3/scraping 22.15.0 → 24.0.0

package/src/secrets.ts

@@ -7,13 +7,21 @@
 // logger. The third is not, because pixels cannot be redacted after the fact.
 
 import type { Secret } from '@ultimat3/core';
-import { revealSecret, secret, UltimateError } from '@ultimat3/core';
+import { escapeHtml, revealSecret, secret, UltimateError } from '@ultimat3/core';
 import type { ConsoleLine, NetworkEntry, PageError } from './rings';
 
 export interface ScrapeSecrets {
   readonly names: readonly string[];
   /** The boxed value. Reading it out is `revealSecret()`, one greppable call site. */
   get(name: string): Secret;
+  /**
+   * Add a value this run learned MID-RUN to the redaction set: a prompt answer, the credentials
+   * of the exit it dials, the connect URL of the browser it rented, a bearer the body read off a
+   * page. It has no name and cannot be read back — it exists only so `page.html()`, the rings and
+   * the failure artifact never carry it. The same floor applies: shorter than
+   * `MIN_REDACTABLE_LENGTH` is not redacted.
+   */
+  conceal(value: string): void;
 }
 
 /**
@@ -30,6 +38,7 @@ export function createSecretBag(
   resolve: SecretResolver = fromEnvironment,
 ): ScrapeSecrets {
   const boxed = new Map<string, Secret>();
+  const learned = new Set<string>();
   for (const name of names) {
     const value = resolve(name);
     if (value === undefined || value === '') {
@@ -42,8 +51,11 @@ export function createSecretBag(
     }
     boxed.set(name, secret(value, name));
   }
-  return {
+  const bag: ScrapeSecrets = {
     names: [...names],
+    conceal(value: string): void {
+      if (value !== '') learned.add(value);
+    },
     get(name: string): Secret {
       const found = boxed.get(name);
       if (found === undefined) {
@@ -57,8 +69,17 @@ export function createSecretBag(
       return found;
     },
   };
+  concealed.set(bag, learned);
+  return bag;
 }
 
+/**
+ * The values `conceal()` was handed, by bag — module-private, so the only way out of a bag is
+ * still `revealSecret(secrets.get(name))` for a DECLARED name. A `ScrapeSecrets` built by hand has
+ * no entry here and redacts its declared names alone.
+ */
+const concealed = new WeakMap<ScrapeSecrets, ReadonlySet<string>>();
+
 export const SECRET_PLACEHOLDER = '[redacted]';
 
 /**
@@ -74,6 +95,56 @@ export const SECRET_PLACEHOLDER = '[redacted]';
  */
 export const MIN_REDACTABLE_LENGTH = 4;
 
+/**
+ * The SUBSET a serialiser escaped, each character spelt by core's one table — a page escapes fewer
+ * characters than `escapeHtml` does (a browser's text serialiser leaves quotes alone), so the
+ * spelling is the table applied to that serialiser's set, never a second table.
+ */
+const escapeOnly = (value: string, chars: RegExp): string => value.replace(chars, escapeHtml);
+
+/**
+ * Every spelling a page or a URL writes one value in. The raw value alone missed the case
+ * `safeNetwork` exists for: a password in a query string arrives percent-encoded, or form-encoded
+ * (`+` for a space, `'` and `(` escaped), and a server-rendered page writes it HTML-escaped — as
+ * text (`&amp; &lt; &gt;`, what a browser's serialiser emits), as an attribute (`&amp; &quot;`),
+ * or through a template that escapes all five (`&#39;` or `&#x27;` for the apostrophe).
+ * Each spelling is at least as long as the value, so the length floor is decided on the value.
+ */
+function spellingsOf(value: string): readonly string[] {
+  const text = escapeOnly(value, /[&<>]/g);
+  const attribute = escapeOnly(value, /[&"]/g);
+  const full = escapeOnly(value, /[&<>"]/g);
+  return [
+    value,
+    encodeURIComponent(value),
+    new URLSearchParams([['', value]]).toString().slice(1),
+    text,
+    attribute,
+    full.replaceAll("'", '&#39;'),
+    full.replaceAll("'", '&#x27;'),
+  ];
+}
+
+const REGEX_SPECIAL = /[.*+?^${}()|[\]\\/]/g;
+
+/**
+ * One spelling as a pattern: literal throughout, except that the hex digits of a `%XX` escape
+ * match in either case — `%2F` and `%2f` are the same byte, and clients emit both. The value's own
+ * letters stay case-sensitive: `Hunter2` is not `hunter2`.
+ */
+const spellingPattern = (spelling: string): RegExp =>
+  new RegExp(
+    spelling
+      .split(/(%[0-9A-Fa-f]{2})/)
+      .map((part, index) =>
+        index % 2 === 1
+          ? part.replace(/[A-Fa-f]/g, (digit) => `[${digit.toUpperCase()}${digit.toLowerCase()}]`)
+          : part.replace(REGEX_SPECIAL, '\\$&'),
+      )
+      .join(''),
+    'g',
+  );
+
 /**
  * Redaction BY VALUE, over the text this package hands back or persists. Name-based redaction only
  * catches a secret travelling under a name somebody remembered to list, and a password pasted into
@@ -90,16 +161,22 @@ export const MIN_REDACTABLE_LENGTH = 4;
  * `MIN_REDACTABLE_LENGTH`, and pixels — which is why a secret TAINTS the page and captures are
  * refused outright.
  *
- * Longest first, so a secret that contains another one does not leave its tail behind.
+ * Every spelling of every value (`spellingsOf`), longest first, so a secret that contains another
+ * one does not leave its tail behind.
  */
 export function redactSecrets(text: string, secrets: ScrapeSecrets | undefined): string {
-  if (secrets === undefined || secrets.names.length === 0) return text;
-  const values = secrets.names
-    .map((name) => revealSecret(secrets.get(name)))
-    .filter((value) => value.length >= MIN_REDACTABLE_LENGTH)
-    .sort((a, b) => b.length - a.length);
+  if (secrets === undefined) return text;
+  const learned = concealed.get(secrets);
+  if (secrets.names.length === 0 && (learned === undefined || learned.size === 0)) return text;
+  const values = [
+    ...new Set(
+      [...secrets.names.map((name) => revealSecret(secrets.get(name))), ...(learned ?? [])]
+        .filter((value) => value.length >= MIN_REDACTABLE_LENGTH)
+        .flatMap(spellingsOf),
+    ),
+  ].sort((a, b) => b.length - a.length);
   let out = text;
-  for (const value of values) out = out.split(value).join(SECRET_PLACEHOLDER);
+  for (const value of values) out = out.replace(spellingPattern(value), SECRET_PLACEHOLDER);
   return out;
 }
 

package/src/session-state.ts

@@ -10,6 +10,8 @@
 // event field, an artifact or a screenshot, and it is summarised for logs by `sessionDigest()` —
 // counts and an origin, never a value.
 
+import type { SealKeySource } from '@ultimat3/core';
+import { isJsonObject, isSealed, openText, resolveSealKeys, seal } from '@ultimat3/core';
 import type { StorageDriver } from '@ultimat3/storage';
 import { assertSafeKey } from '@ultimat3/storage';
 import type { ScrapeCookie } from './target';
@@ -36,6 +38,13 @@ export interface SessionState extends SessionSnapshot {
   readonly key: string;
   /** ISO 8601. What a `maxAge` on reuse is measured against. */
   readonly savedAt: string;
+  /**
+   * Unique per save — what a burn or a tombstone compares to decide whether the record is still
+   * the one a run saw (`auth.ts`'s `stillSeen`). Never `savedAt`: two saves in one millisecond, or
+   * on one frozen clock, share it. Absent on a record written before it existed; `recordVersion`
+   * reads `savedAt` for that one.
+   */
+  readonly version?: string | undefined;
   /**
    * ISO 8601, set when this site REFUSED these credentials — and the reason the record survives
    * the failure instead of being deleted with it.
@@ -168,42 +177,103 @@ export function memorySessionStore(
 
 export const DEFAULT_SESSION_PREFIX = 'scrape-session';
 
+/** What a stored session is sealed FOR. Bound into the tag: nothing else sealed opens as one. */
+export const SESSION_SEAL_PURPOSE = 'scrape-session';
+
+export interface StorageSessionStoreOptions {
+  readonly prefix?: string | undefined;
+  /**
+   * Where the app's master key is read from — `seal()`'s own two fields, with its defaults: the
+   * process environment, then `.secrets.key` under the working directory.
+   */
+  readonly keySource?: SealKeySource | undefined;
+}
+
+/** The sealed string out of a stored object, or `undefined` for anything that is not one. */
+const sealedIn = (text: string): string | undefined => {
+  try {
+    const parsed: unknown = JSON.parse(text);
+    return isJsonObject(parsed) && isSealed(parsed['sealed']) ? parsed['sealed'] : undefined;
+  } catch {
+    return undefined;
+  }
+};
+
 /**
  * Sessions on the app's own disk, through `@ultimat3/storage` — the seam that already knows about
  * tenant-scoped keys and refuses one that escapes its prefix. This package owns no upload path.
  *
- * NOT encrypted at rest: the bytes are as sensitive as the bucket they sit in, so the bucket must
- * be private. Sealing them under core's secrets envelope is the obvious next step and is
- * deliberately not guessed at here — it needs a key the app declares, and a wrong guess about
- * where that key lives is worse than the honest note.
+ * SEALED at rest, under the app's master key — the one `x secrets init` writes — through core's
+ * `seal()`, purpose `SESSION_SEAL_PURPOSE`. The stored object is `{ "sealed": "x1.…" }`: no cookie
+ * value, no `localStorage` token and no origin is readable from the bucket. With no master key the
+ * store refuses (`X_SEAL_KEY_MISSING`) on `load()`, which a run calls BEFORE its browser opens —
+ * a login that could not be kept is a login repeated on every attempt.
+ *
+ * A session is a CACHE, so nothing is migrated: an object that is not a sealed session, does not
+ * open under a key this process declares, or opens to a record written for another key is burned
+ * and the run authenticates again.
+ *
+ * `storage` is a THUNK — `storageSessionStore(() => disk('sessions'))` — read on every call and
+ * never captured. A `scrape()` is declared when its module is evaluated; the app's disks exist
+ * once boot has run `defineStorage()`, which is later. A driver VALUE here could only be a
+ * `declare const` in a README.
  */
 export function storageSessionStore(
-  storage: StorageDriver,
-  options: { readonly prefix?: string | undefined } = {},
+  storage: () => StorageDriver,
+  options: StorageSessionStoreOptions = {},
 ): ScrapeSessionStore {
   const keyFor = (key: string): string => {
     const path = `${options.prefix ?? DEFAULT_SESSION_PREFIX}/${key}.json`;
     assertSafeKey(path);
     return path;
   };
+  /** Best effort: an object that could not be deleted is refused again by the next `load()`. */
+  const discard = async (path: string): Promise<undefined> => {
+    await storage()
+      .delete(path)
+      .catch(() => undefined);
+    return undefined;
+  };
   return {
     async load(key: string): Promise<SessionState | undefined> {
+      // The key ring FIRST, and outside every catch below: "no master key" is a deploy that cannot
+      // keep a session at all, and answering "no session" for it would log in, fail to save, and
+      // log in again on the retry.
+      const keys = await resolveSealKeys(options.keySource);
+      const path = keyFor(key);
+      let stored: string;
       try {
-        const read = await storage.get(keyFor(key));
-        return parseSessionState(JSON.parse(new TextDecoder().decode(read.bytes)) as unknown, key);
+        stored = new TextDecoder().decode((await storage().get(path)).bytes);
       } catch {
         // A session that cannot be read is a session that does not exist. Refusing the run over a
         // missing cache would make reuse — the fast path — the fragile path.
         return undefined;
       }
+      const sealed = sealedIn(stored);
+      if (sealed === undefined) return discard(path);
+      try {
+        const opened: unknown = JSON.parse(
+          await openText(sealed, { purpose: SESSION_SEAL_PURPOSE, keys }),
+        );
+        // The tag covers the record, and the record names the key it was saved under: an object
+        // copied onto another tenant's path opens, and is still not that tenant's session.
+        if (!isJsonObject(opened) || opened['key'] !== key) return discard(path);
+        return parseSessionState(opened, key) ?? discard(path);
+      } catch {
+        return discard(path);
+      }
     },
     async save(state: SessionState): Promise<void> {
-      await storage.put(keyFor(state.key), new TextEncoder().encode(JSON.stringify(state)), {
+      const sealed = await seal(JSON.stringify(state), {
+        purpose: SESSION_SEAL_PURPOSE,
+        ...options.keySource,
+      });
+      await storage().put(keyFor(state.key), new TextEncoder().encode(JSON.stringify({ sealed })), {
         contentType: 'application/json',
       });
     },
     async burn(key: string): Promise<void> {
-      await storage.delete(keyFor(key));
+      await storage().delete(keyFor(key));
     },
   };
 }
@@ -246,6 +316,7 @@ export function parseSessionState(raw: unknown, key: string): SessionState | und
   return {
     key,
     savedAt: value.savedAt,
+    ...(typeof value.version === 'string' ? { version: value.version } : {}),
     ...(typeof value.refusedAt === 'string' ? { refusedAt: value.refusedAt } : {}),
     cookies: value.cookies.flatMap((cookie: unknown) => toCookie(cookie) ?? []),
     headers: value.headers ?? {},
@@ -254,3 +325,6 @@ export function parseSessionState(raw: unknown, key: string): SessionState | und
     origin: value.origin ?? '',
   };
 }
+
+/** The identity a conditional burn or tombstone compares: the record's version, else its `savedAt`. */
+export const recordVersion = (state: SessionState): string => state.version ?? state.savedAt;

package/src/url-secrets.ts

@@ -0,0 +1,139 @@
+// What a URL that carries a credential may say about itself, and what it must hand the run's
+// secret set. A proxy URL holds its password in the userinfo; a provider's CDP connect URL holds
+// its access token in the query or the path. Both are named in errors and both pass through text
+// this package persists, so the split is made once, here.
+
+/**
+ * Scheme and host — `wss://connect.provider.test:443` — and nothing a credential can live in: no
+ * userinfo, no path, no query. A string that is not a URL answers a fixed phrase rather than
+ * itself, because the caller is about to print the answer.
+ */
+export function endpointLabel(url: string): string {
+  try {
+    const parsed = new URL(url);
+    return `${parsed.protocol}//${parsed.host}`;
+  } catch {
+    return 'an endpoint that is not a URL';
+  }
+}
+
+/** A lone `%` survives `URL` parsing and makes `decodeURIComponent` throw; the raw text is the answer then. */
+const decoded = (part: string): string => {
+  try {
+    return decodeURIComponent(part);
+  } catch {
+    return part;
+  }
+};
+
+/** True when the userinfo carries anything — the half `--proxy-server` cannot be handed. */
+export function hasCredentials(url: string): boolean {
+  try {
+    const parsed = new URL(url);
+    return parsed.username !== '' || parsed.password !== '';
+  } catch {
+    return false;
+  }
+}
+
+/** `http://user:pass@exit:8080` → `http://exit:8080` plus the pair, decoded as the proxy expects. */
+export function splitCredentials(url: string): {
+  readonly bare: string;
+  readonly username: string;
+  readonly password: string;
+} {
+  const parsed = new URL(url);
+  const username = decoded(parsed.username);
+  const password = decoded(parsed.password);
+  parsed.username = '';
+  parsed.password = '';
+  // `URL` renders a trailing slash for an empty path; a proxy argument never had one.
+  return { bare: parsed.toString().replace(/\/$/, ''), username, password };
+}
+
+/**
+ * The words that name a credential, matched against a query key's COMPONENTS — split on `_`, `-`,
+ * `.` and camelCase — never as substrings: `keyboard` holds `key` and `monkey` holds it too, and
+ * concealing their values blanked ordinary words out of every artifact. Read off the key and not
+ * the value, because a provider's token can be short and the key is the URL's own statement.
+ */
+const CREDENTIAL_WORDS = new Set([
+  'token',
+  'key',
+  'apikey',
+  'secret',
+  'auth',
+  'authorization',
+  'sig',
+  'signature',
+  'pass',
+  'password',
+  'passwd',
+  'pwd',
+  'cred',
+  'credential',
+  'credentials',
+  'session',
+  'sessionid',
+  'sid',
+  'jwt',
+  'bearer',
+]);
+
+const credentialKey = (key: string): boolean =>
+  key
+    .replace(/([a-z0-9])([A-Z])/g, '$1 $2')
+    .toLowerCase()
+    .split(/[\s_.-]+/)
+    .some((part) => CREDENTIAL_WORDS.has(part));
+
+/**
+ * A value that is a capability whatever it is called: long, one unbroken run, letters and digits
+ * mixed — a browser id, an access token. A word (`residential`, `true`, `eu`) is none of those.
+ */
+const credentialShaped = (value: string): boolean =>
+  value.length >= 16 && !/\s/.test(value) && /[0-9]/.test(value) && /[a-z]/i.test(value);
+
+/**
+ * Every value in the URL that is, or may be, a credential — for the run's redaction set.
+ *
+ * The whole URL first, then the parts a message quotes on their own: a launcher's error says
+ * `401 for /devtools/browser/<id>?token=…` as readily as it repeats the URL. Encoded and decoded
+ * forms both, since a message may carry either. The host is left out on purpose: it is what an
+ * error is allowed to say.
+ *
+ * A query VALUE or a path SEGMENT only when it is credential-shaped or sits under a
+ * credential-named key — never every value. Concealing `stealth=true&proxy=residential` blanked
+ * every `true` and `residential` in the page, the console and the network ring of the run: the
+ * unreadable artifact `MIN_REDACTABLE_LENGTH` exists to prevent.
+ */
+export function urlSecretValues(url: string): readonly string[] {
+  const values = new Set<string>([url]);
+  try {
+    const parsed = new URL(url);
+    // The password and not the username: a proxy username is an account and a zone, and redacting
+    // `admin` by value would blank the word out of every page artifact.
+    values.add(parsed.password);
+    values.add(decoded(parsed.password));
+    if (parsed.search !== '') {
+      values.add(`${parsed.pathname}${parsed.search}`);
+      values.add(parsed.search.slice(1));
+    }
+    const segments = parsed.pathname.split('/');
+    for (const [index, segment] of segments.entries()) {
+      // By POSITION as well as by shape: Chrome's `/devtools/<browser|page>/<id>` id IS the
+      // capability, and an id of letters alone is not credential-shaped.
+      const capability = index >= 2 && segments[index - 2] === 'devtools';
+      if (capability || credentialShaped(decoded(segment))) {
+        values.add(decoded(segment)).add(segment);
+      }
+    }
+    for (const [key, value] of parsed.searchParams) {
+      if (credentialKey(key) || credentialShaped(value)) values.add(value);
+    }
+  } catch {
+    // Not a URL: the whole string above is all there is to redact.
+  }
+  values.delete('');
+  return [...values];
+}

package/src/usage.ts

@@ -0,0 +1,101 @@
+// What one run USED, counted where the work happens and reported with its result. A measurement
+// and nothing else: no quota, no plan, no ledger — what a number here is worth, and what happens
+// when it is too high, is the app's (`docs/idea/19-mechanism-not-convention.md`).
+
+import type { MoneyValue } from '@ultimat3/schema';
+import type { ScrapeClock } from './clock';
+
+export interface ScrapeUsage {
+  /** Milliseconds the browser session was held, from before `driver.open()` to this report. */
+  readonly browserMs: number;
+  /** `page.goto()` calls that reached the driver — a refused host or a robots refusal is not one. */
+  readonly navigations: number;
+  /** Requests the HTTP leg put on the wire, one per redirect hop. */
+  readonly httpRequests: number;
+  /**
+   * Response-body bytes the HTTP leg read. The BROWSER leg's traffic is not in it: this package's
+   * CDP port subscribes to requests, not to their sizes, so a figure for it would be a guess.
+   */
+  readonly bytesIn: number;
+  readonly promptsAnswered: number;
+  /**
+   * What the rented browser cost, as the `CdpResolver` stated it — a `Money` value, never a float.
+   * Absent when the driver rented nothing or the provider prices at release.
+   */
+  readonly browserCost?: MoneyValue | undefined;
+}
+
+/**
+ * The counting half, handed to a driver as `SessionInit.usage`. `pageOverTarget`, `httpOverFetch`
+ * and `recordedHttp` each call it where they already report activity, so a third-party driver
+ * built on them counts by passing the meter through and one that is not counts by calling it.
+ */
+export interface UsageMeter {
+  navigation(): void;
+  /** One request on the wire. `bytes` is the body read for it — 0 for a hop that was followed. */
+  httpRequest(bytes: number): void;
+}
+
+export interface RunUsageMeter extends UsageMeter {
+  promptAnswered(): void;
+  /** The counts so far, with the session's age read from the clock this meter was built on. */
+  snapshot(browserCost?: MoneyValue | undefined): ScrapeUsage;
+}
+
+/** Built BEFORE `driver.open()`, so the time a rented browser took to arrive is time it was held. */
+export function createUsageMeter(clock: ScrapeClock): RunUsageMeter {
+  const startedAt = clock.monotonic();
+  let navigations = 0;
+  let httpRequests = 0;
+  let bytesIn = 0;
+  let promptsAnswered = 0;
+  return {
+    navigation(): void {
+      navigations += 1;
+    },
+    httpRequest(bytes: number): void {
+      httpRequests += 1;
+      bytesIn += bytes;
+    },
+    promptAnswered(): void {
+      promptsAnswered += 1;
+    },
+    snapshot(browserCost?: MoneyValue | undefined): ScrapeUsage {
+      return {
+        browserMs: Math.round(clock.monotonic() - startedAt),
+        navigations,
+        httpRequests,
+        bytesIn,
+        promptsAnswered,
+        ...(browserCost === undefined ? {} : { browserCost }),
+      };
+    },
+  };
+}
+
+/** Failed attempts remembered at once. A bound, not a ledger: the oldest goes first. */
+const MAX_FAILED_ATTEMPTS = 256;
+
+const failedAttempts = new Map<string, ScrapeUsage>();
+
+/**
+ * What a FAILED attempt used, held until its run is settled. A failed run has no report — the
+ * job's return value is the only thing the queue hands `onSettled` — and it was billed all the
+ * same, so the counts are kept by run id in the worker's own process, which is where the hook runs.
+ */
+export function rememberFailedUsage(runId: string, usage: ScrapeUsage): void {
+  failedAttempts.delete(runId);
+  failedAttempts.set(runId, usage);
+  while (failedAttempts.size > MAX_FAILED_ATTEMPTS) {
+    const oldest = failedAttempts.keys().next();
+    if (oldest.done === true) return;
+    failedAttempts.delete(oldest.value);
+  }
+}
+
+/** The last failed attempt's usage for this run, read once: taking it forgets it. */
+export function takeFailedUsage(runId: string): ScrapeUsage | undefined {
+  const usage = failedAttempts.get(runId);
+  failedAttempts.delete(runId);
+  return usage;
+}