@ultimat3/scraping 22.15.0 → 23.0.0

package/src/session-state.ts

@@ -10,6 +10,8 @@
 // event field, an artifact or a screenshot, and it is summarised for logs by `sessionDigest()` —
 // counts and an origin, never a value.
 
+import type { SealKeySource } from '@ultimat3/core';
+import { isJsonObject, isSealed, openText, resolveSealKeys, seal } from '@ultimat3/core';
 import type { StorageDriver } from '@ultimat3/storage';
 import { assertSafeKey } from '@ultimat3/storage';
 import type { ScrapeCookie } from './target';
@@ -168,42 +170,103 @@ export function memorySessionStore(
 
 export const DEFAULT_SESSION_PREFIX = 'scrape-session';
 
+/** What a stored session is sealed FOR. Bound into the tag: nothing else sealed opens as one. */
+export const SESSION_SEAL_PURPOSE = 'scrape-session';
+
+export interface StorageSessionStoreOptions {
+  readonly prefix?: string | undefined;
+  /**
+   * Where the app's master key is read from — `seal()`'s own two fields, with its defaults: the
+   * process environment, then `.secrets.key` under the working directory.
+   */
+  readonly keySource?: SealKeySource | undefined;
+}
+
+/** The sealed string out of a stored object, or `undefined` for anything that is not one. */
+const sealedIn = (text: string): string | undefined => {
+  try {
+    const parsed: unknown = JSON.parse(text);
+    return isJsonObject(parsed) && isSealed(parsed['sealed']) ? parsed['sealed'] : undefined;
+  } catch {
+    return undefined;
+  }
+};
+
 /**
  * Sessions on the app's own disk, through `@ultimat3/storage` — the seam that already knows about
  * tenant-scoped keys and refuses one that escapes its prefix. This package owns no upload path.
  *
- * NOT encrypted at rest: the bytes are as sensitive as the bucket they sit in, so the bucket must
- * be private. Sealing them under core's secrets envelope is the obvious next step and is
- * deliberately not guessed at here — it needs a key the app declares, and a wrong guess about
- * where that key lives is worse than the honest note.
+ * SEALED at rest, under the app's master key — the one `x secrets init` writes — through core's
+ * `seal()`, purpose `SESSION_SEAL_PURPOSE`. The stored object is `{ "sealed": "x1.…" }`: no cookie
+ * value, no `localStorage` token and no origin is readable from the bucket. With no master key the
+ * store refuses (`X_SEAL_KEY_MISSING`) on `load()`, which a run calls BEFORE its browser opens —
+ * a login that could not be kept is a login repeated on every attempt.
+ *
+ * A session is a CACHE, so nothing is migrated: an object that is not a sealed session, does not
+ * open under a key this process declares, or opens to a record written for another key is burned
+ * and the run authenticates again.
+ *
+ * `storage` is a THUNK — `storageSessionStore(() => disk('sessions'))` — read on every call and
+ * never captured. A `scrape()` is declared when its module is evaluated; the app's disks exist
+ * once boot has run `defineStorage()`, which is later. A driver VALUE here could only be a
+ * `declare const` in a README.
  */
 export function storageSessionStore(
-  storage: StorageDriver,
-  options: { readonly prefix?: string | undefined } = {},
+  storage: () => StorageDriver,
+  options: StorageSessionStoreOptions = {},
 ): ScrapeSessionStore {
   const keyFor = (key: string): string => {
     const path = `${options.prefix ?? DEFAULT_SESSION_PREFIX}/${key}.json`;
     assertSafeKey(path);
     return path;
   };
+  /** Best effort: an object that could not be deleted is refused again by the next `load()`. */
+  const discard = async (path: string): Promise<undefined> => {
+    await storage()
+      .delete(path)
+      .catch(() => undefined);
+    return undefined;
+  };
   return {
     async load(key: string): Promise<SessionState | undefined> {
+      // The key ring FIRST, and outside every catch below: "no master key" is a deploy that cannot
+      // keep a session at all, and answering "no session" for it would log in, fail to save, and
+      // log in again on the retry.
+      const keys = await resolveSealKeys(options.keySource);
+      const path = keyFor(key);
+      let stored: string;
       try {
-        const read = await storage.get(keyFor(key));
-        return parseSessionState(JSON.parse(new TextDecoder().decode(read.bytes)) as unknown, key);
+        stored = new TextDecoder().decode((await storage().get(path)).bytes);
       } catch {
         // A session that cannot be read is a session that does not exist. Refusing the run over a
         // missing cache would make reuse — the fast path — the fragile path.
         return undefined;
       }
+      const sealed = sealedIn(stored);
+      if (sealed === undefined) return discard(path);
+      try {
+        const opened: unknown = JSON.parse(
+          await openText(sealed, { purpose: SESSION_SEAL_PURPOSE, keys }),
+        );
+        // The tag covers the record, and the record names the key it was saved under: an object
+        // copied onto another tenant's path opens, and is still not that tenant's session.
+        if (!isJsonObject(opened) || opened['key'] !== key) return discard(path);
+        return parseSessionState(opened, key) ?? discard(path);
+      } catch {
+        return discard(path);
+      }
     },
     async save(state: SessionState): Promise<void> {
-      await storage.put(keyFor(state.key), new TextEncoder().encode(JSON.stringify(state)), {
+      const sealed = await seal(JSON.stringify(state), {
+        purpose: SESSION_SEAL_PURPOSE,
+        ...options.keySource,
+      });
+      await storage().put(keyFor(state.key), new TextEncoder().encode(JSON.stringify({ sealed })), {
         contentType: 'application/json',
       });
     },
     async burn(key: string): Promise<void> {
-      await storage.delete(keyFor(key));
+      await storage().delete(keyFor(key));
     },
   };
 }

package/src/url-secrets.ts

@@ -0,0 +1,78 @@
+// What a URL that carries a credential may say about itself, and what it must hand the run's
+// secret set. A proxy URL holds its password in the userinfo; a provider's CDP connect URL holds
+// its access token in the query or the path. Both are named in errors and both pass through text
+// this package persists, so the split is made once, here.
+
+/**
+ * Scheme and host — `wss://connect.provider.test:443` — and nothing a credential can live in: no
+ * userinfo, no path, no query. A string that is not a URL answers a fixed phrase rather than
+ * itself, because the caller is about to print the answer.
+ */
+export function endpointLabel(url: string): string {
+  try {
+    const parsed = new URL(url);
+    return `${parsed.protocol}//${parsed.host}`;
+  } catch {
+    return 'an endpoint that is not a URL';
+  }
+}
+
+/** A lone `%` survives `URL` parsing and makes `decodeURIComponent` throw; the raw text is the answer then. */
+const decoded = (part: string): string => {
+  try {
+    return decodeURIComponent(part);
+  } catch {
+    return part;
+  }
+};
+
+/** True when the userinfo carries anything — the half `--proxy-server` cannot be handed. */
+export function hasCredentials(url: string): boolean {
+  try {
+    const parsed = new URL(url);
+    return parsed.username !== '' || parsed.password !== '';
+  } catch {
+    return false;
+  }
+}
+
+/** `http://user:pass@exit:8080` → `http://exit:8080` plus the pair, decoded as the proxy expects. */
+export function splitCredentials(url: string): {
+  readonly bare: string;
+  readonly username: string;
+  readonly password: string;
+} {
+  const parsed = new URL(url);
+  const username = decoded(parsed.username);
+  const password = decoded(parsed.password);
+  parsed.username = '';
+  parsed.password = '';
+  // `URL` renders a trailing slash for an empty path; a proxy argument never had one.
+  return { bare: parsed.toString().replace(/\/$/, ''), username, password };
+}
+
+/**
+ * Every value in the URL that is, or may be, a credential — for the run's redaction set.
+ *
+ * The whole URL first, then the parts that can be quoted on their own: a launcher's error says
+ * `401 for /devtools/browser/<id>?token=…` as readily as it repeats the URL. The path counts
+ * because a browser id in a CDP path IS the capability. Encoded and decoded forms both, since a
+ * message may carry either. The host is left out on purpose: it is what an error is allowed to say.
+ */
+export function urlSecretValues(url: string): readonly string[] {
+  const values = new Set<string>([url]);
+  try {
+    const parsed = new URL(url);
+    // The password and not the username: a proxy username is an account and a zone, and redacting
+    // `admin` by value would blank the word out of every page artifact.
+    values.add(parsed.password);
+    values.add(decoded(parsed.password));
+    if (parsed.pathname !== '/') values.add(`${parsed.pathname}${parsed.search}`);
+    if (parsed.search !== '') values.add(parsed.search.slice(1));
+    for (const value of parsed.searchParams.values()) values.add(value);
+  } catch {
+    // Not a URL: the whole string above is all there is to redact.
+  }
+  values.delete('');
+  return [...values];
+}

package/src/usage.ts

@@ -0,0 +1,101 @@
+// What one run USED, counted where the work happens and reported with its result. A measurement
+// and nothing else: no quota, no plan, no ledger — what a number here is worth, and what happens
+// when it is too high, is the app's (`docs/idea/19-mechanism-not-convention.md`).
+
+import type { MoneyValue } from '@ultimat3/schema';
+import type { ScrapeClock } from './clock';
+
+export interface ScrapeUsage {
+  /** Milliseconds the browser session was held, from before `driver.open()` to this report. */
+  readonly browserMs: number;
+  /** `page.goto()` calls that reached the driver — a refused host or a robots refusal is not one. */
+  readonly navigations: number;
+  /** Requests the HTTP leg put on the wire, one per redirect hop. */
+  readonly httpRequests: number;
+  /**
+   * Response-body bytes the HTTP leg read. The BROWSER leg's traffic is not in it: this package's
+   * CDP port subscribes to requests, not to their sizes, so a figure for it would be a guess.
+   */
+  readonly bytesIn: number;
+  readonly promptsAnswered: number;
+  /**
+   * What the rented browser cost, as the `CdpResolver` stated it — a `Money` value, never a float.
+   * Absent when the driver rented nothing or the provider prices at release.
+   */
+  readonly browserCost?: MoneyValue | undefined;
+}
+
+/**
+ * The counting half, handed to a driver as `SessionInit.usage`. `pageOverTarget`, `httpOverFetch`
+ * and `recordedHttp` each call it where they already report activity, so a third-party driver
+ * built on them counts by passing the meter through and one that is not counts by calling it.
+ */
+export interface UsageMeter {
+  navigation(): void;
+  /** One request on the wire. `bytes` is the body read for it — 0 for a hop that was followed. */
+  httpRequest(bytes: number): void;
+}
+
+export interface RunUsageMeter extends UsageMeter {
+  promptAnswered(): void;
+  /** The counts so far, with the session's age read from the clock this meter was built on. */
+  snapshot(browserCost?: MoneyValue | undefined): ScrapeUsage;
+}
+
+/** Built BEFORE `driver.open()`, so the time a rented browser took to arrive is time it was held. */
+export function createUsageMeter(clock: ScrapeClock): RunUsageMeter {
+  const startedAt = clock.monotonic();
+  let navigations = 0;
+  let httpRequests = 0;
+  let bytesIn = 0;
+  let promptsAnswered = 0;
+  return {
+    navigation(): void {
+      navigations += 1;
+    },
+    httpRequest(bytes: number): void {
+      httpRequests += 1;
+      bytesIn += bytes;
+    },
+    promptAnswered(): void {
+      promptsAnswered += 1;
+    },
+    snapshot(browserCost?: MoneyValue | undefined): ScrapeUsage {
+      return {
+        browserMs: Math.round(clock.monotonic() - startedAt),
+        navigations,
+        httpRequests,
+        bytesIn,
+        promptsAnswered,
+        ...(browserCost === undefined ? {} : { browserCost }),
+      };
+    },
+  };
+}
+
+/** Failed attempts remembered at once. A bound, not a ledger: the oldest goes first. */
+const MAX_FAILED_ATTEMPTS = 256;
+
+const failedAttempts = new Map<string, ScrapeUsage>();
+
+/**
+ * What a FAILED attempt used, held until its run is settled. A failed run has no report — the
+ * job's return value is the only thing the queue hands `onSettled` — and it was billed all the
+ * same, so the counts are kept by run id in the worker's own process, which is where the hook runs.
+ */
+export function rememberFailedUsage(runId: string, usage: ScrapeUsage): void {
+  failedAttempts.delete(runId);
+  failedAttempts.set(runId, usage);
+  while (failedAttempts.size > MAX_FAILED_ATTEMPTS) {
+    const oldest = failedAttempts.keys().next();
+    if (oldest.done === true) return;
+    failedAttempts.delete(oldest.value);
+  }
+}
+
+/** The last failed attempt's usage for this run, read once: taking it forgets it. */
+export function takeFailedUsage(runId: string): ScrapeUsage | undefined {
+  const usage = failedAttempts.get(runId);
+  failedAttempts.delete(runId);
+  return usage;
+}