browser-commander 0.17.2 → 0.19.0

package/src/traces/redaction.js

@@ -0,0 +1,231 @@
+/**
+ * Redaction applied before a trace byte reaches disk (issue #87).
+ *
+ * Browser traces routinely contain credentials. Redaction therefore runs
+ * where the data is produced — inside the page for DOM and control state, and
+ * on every string this module touches for URLs, headers and messages — rather
+ * than as a pass over a file that has already been written.
+ */
+
+/** What replaces a redacted value. Recognizable, and never a valid secret. */
+export const REDACTED = '[redacted]';
+
+/** Controls whose value is a secret unless the caller says otherwise. */
+export const DEFAULT_REDACT_SELECTORS = Object.freeze([
+  'input[type=password]',
+  '[data-private]',
+  '[data-bc-redact]',
+]);
+
+/** Attributes and headers that carry credentials. */
+export const DEFAULT_REDACT_ATTRIBUTES = Object.freeze([
+  'authorization',
+  'proxy-authorization',
+  'cookie',
+  'set-cookie',
+  'x-api-key',
+  'x-auth-token',
+  'x-csrf-token',
+]);
+
+/** Query parameters that carry credentials in a URL. */
+export const DEFAULT_REDACT_QUERY_PARAMS = Object.freeze([
+  'access_token',
+  'api_key',
+  'apikey',
+  'auth',
+  'code',
+  'id_token',
+  'password',
+  'refresh_token',
+  'secret',
+  'session',
+  'signature',
+  'token',
+]);
+
+function lowerSet(values) {
+  return [...new Set(values.map((value) => String(value).toLowerCase()))];
+}
+
+/**
+ * Turn the caller's `privacy` option into the shape every writer uses.
+ *
+ * @param {Object} [privacy] - Caller privacy options
+ * @returns {Object} Normalized privacy options
+ */
+export function normalizePrivacyOptions(privacy = {}) {
+  if (privacy === null || typeof privacy !== 'object') {
+    throw new Error('trace privacy options must be an object');
+  }
+
+  const {
+    redactSelectors = [],
+    redactAttributes = [],
+    redactQueryParams = [],
+    redactPatterns = [],
+    redact = null,
+    // Defaults are additive: a caller adds to the safe set rather than
+    // silently replacing it, so a new selector cannot uncover passwords.
+    useDefaults = true,
+  } = privacy;
+
+  if (redact !== null && typeof redact !== 'function') {
+    throw new Error('privacy.redact must be a function');
+  }
+
+  return {
+    redactSelectors: lowerSet([
+      ...(useDefaults ? DEFAULT_REDACT_SELECTORS : []),
+      ...redactSelectors,
+    ]),
+    redactAttributes: lowerSet([
+      ...(useDefaults ? DEFAULT_REDACT_ATTRIBUTES : []),
+      ...redactAttributes,
+    ]),
+    redactQueryParams: lowerSet([
+      ...(useDefaults ? DEFAULT_REDACT_QUERY_PARAMS : []),
+      ...redactQueryParams,
+    ]),
+    redactPatterns: redactPatterns.map((pattern) =>
+      pattern instanceof RegExp ? pattern : new RegExp(String(pattern), 'g')
+    ),
+    redact,
+  };
+}
+
+/**
+ * Apply the caller's patterns and callback to a string.
+ *
+ * @param {string} value - Text about to be persisted
+ * @param {Object} privacy - Normalized privacy options
+ * @param {Object} [context] - What the text is, passed to the callback
+ * @returns {string} Redacted text
+ */
+export function redactText(value, privacy, context = {}) {
+  if (typeof value !== 'string' || value === '') {
+    return value;
+  }
+
+  let text = value;
+  for (const pattern of privacy.redactPatterns || []) {
+    // A caller's regex may be stateful; resetting keeps one trace's output
+    // from depending on the previous record's match position.
+    pattern.lastIndex = 0;
+    text = text.replace(pattern, REDACTED);
+  }
+
+  if (privacy.redact) {
+    const replaced = privacy.redact({ ...context, value: text });
+    if (typeof replaced === 'string') {
+      text = replaced;
+    }
+  }
+
+  return text;
+}
+
+/**
+ * Redact credentials carried in a URL.
+ *
+ * `https://user:pass@host/path?token=abc` becomes a URL that still identifies
+ * the page without handing over the session it belonged to.
+ *
+ * @param {string} url - URL about to be persisted
+ * @param {Object} privacy - Normalized privacy options
+ * @returns {string} Redacted URL
+ */
+export function redactUrl(url, privacy) {
+  if (typeof url !== 'string' || url === '') {
+    return url;
+  }
+
+  let text = url;
+  try {
+    const parsed = new URL(url);
+    if (parsed.username || parsed.password) {
+      parsed.username = parsed.username ? REDACTED : '';
+      parsed.password = parsed.password ? REDACTED : '';
+    }
+    for (const key of [...parsed.searchParams.keys()]) {
+      if (privacy.redactQueryParams.includes(key.toLowerCase())) {
+        parsed.searchParams.set(key, REDACTED);
+      }
+    }
+    if (parsed.hash) {
+      // Implicit OAuth flows put the token in the fragment.
+      const fragment = new URLSearchParams(parsed.hash.slice(1));
+      let changed = false;
+      for (const key of [...fragment.keys()]) {
+        if (privacy.redactQueryParams.includes(key.toLowerCase())) {
+          fragment.set(key, REDACTED);
+          changed = true;
+        }
+      }
+      if (changed) {
+        parsed.hash = `#${fragment.toString()}`;
+      }
+    }
+    // `URL` percent-encodes whatever it is given, so the marker is put back
+    // in readable form: a trace is read by people.
+    text = parsed.toString().split(encodeURIComponent(REDACTED)).join(REDACTED);
+  } catch {
+    // A relative or malformed URL is still worth keeping, just unparsed.
+  }
+
+  return redactText(text, privacy, { kind: 'url' });
+}
+
+/**
+ * Redact a header or attribute map.
+ *
+ * @param {Object} headers - Header name/value pairs
+ * @param {Object} privacy - Normalized privacy options
+ * @returns {Object} Map with sensitive values replaced
+ */
+export function redactHeaders(headers, privacy) {
+  if (!headers || typeof headers !== 'object') {
+    return headers;
+  }
+
+  const result = {};
+  for (const [name, value] of Object.entries(headers)) {
+    result[name] = privacy.redactAttributes.includes(name.toLowerCase())
+      ? REDACTED
+      : redactText(String(value), privacy, { kind: 'header', name });
+  }
+  return result;
+}
+
+/**
+ * Redact every string in an event payload.
+ *
+ * Applied to whatever a caller or engine put in an event, because a trace is
+ * only as private as its least careful field.
+ *
+ * @param {*} value - Any JSON-compatible value
+ * @param {Object} privacy - Normalized privacy options
+ * @param {string} [key] - Field name the value was found under
+ * @returns {*} The value with strings redacted
+ */
+export function redactValue(value, privacy, key = '') {
+  if (typeof value === 'string') {
+    if (privacy.redactAttributes.includes(key.toLowerCase())) {
+      return REDACTED;
+    }
+    return /url$/i.test(key)
+      ? redactUrl(value, privacy)
+      : redactText(value, privacy, { kind: 'field', name: key });
+  }
+  if (Array.isArray(value)) {
+    return value.map((entry) => redactValue(entry, privacy, key));
+  }
+  if (value && typeof value === 'object') {
+    const result = {};
+    for (const [name, entry] of Object.entries(value)) {
+      result[name] = redactValue(entry, privacy, name);
+    }
+    return result;
+  }
+  return value;
+}

package/src/traces/schema.js

@@ -0,0 +1,229 @@
+/**
+ * The portable browser trace format (issue #87).
+ *
+ * One engine-neutral, versioned layout is written by JavaScript and read by
+ * JavaScript, Python and Rust. Everything about the format that a reader has
+ * to agree on lives here, so a change to the layout is a change to one file.
+ */
+
+/**
+ * Schema version of the bundle.
+ *
+ * Readers refuse a major version they were not written for rather than
+ * guessing at the meaning of unknown records.
+ *
+ * Version 2 (issue #93) is what makes continuous replay complete: mutation
+ * batches name the frame they came from, child-list records carry the position
+ * a node was inserted at or removed from, and live control state - typing,
+ * checking, selecting, focus and scroll - is recorded as it happens instead of
+ * only at checkpoints. A version 1 reader given a version 2 bundle would
+ * quietly replay a different page, so it refuses instead.
+ */
+export const TRACE_SCHEMA_VERSION = 2;
+
+/** Name every reader looks for inside a bundle. */
+export const TRACE_FILES = Object.freeze({
+  MANIFEST: 'manifest.json',
+  EVENTS: 'events.ndjson',
+  CHECKPOINTS_DIR: 'checkpoints',
+  MUTATIONS_DIR: 'mutations',
+  ARTIFACTS_DIR: 'artifacts',
+  VIEWER: 'viewer.html',
+});
+
+/** What the recorder captures. */
+export const TRACE_MODE = Object.freeze({
+  OFF: 'off',
+  CHECKPOINTS: 'checkpoints',
+  CONTINUOUS: 'continuous',
+  RETAIN_ON_FAILURE: 'retain-on-failure',
+});
+
+/** Event kinds that share the one ordered timeline. */
+export const TRACE_EVENT = Object.freeze({
+  TRACE_START: 'trace.start',
+  TRACE_STOP: 'trace.stop',
+  CHECKPOINT: 'checkpoint',
+  MUTATIONS: 'mutations',
+  NAVIGATION: 'navigation',
+  INTERACTION: 'interaction',
+  CONSOLE: 'console',
+  PAGE_ERROR: 'pageerror',
+  DIALOG: 'dialog',
+  REQUEST_FAILED: 'requestfailed',
+  DOWNLOAD: 'download',
+  /** Something could not be recorded. The run continues; the gap is visible. */
+  DROPPED: 'dropped',
+});
+
+/** Record kinds inside a mutation batch. */
+export const TRACE_MUTATION_KIND = Object.freeze({
+  ATTRIBUTES: 'attributes',
+  CHARACTER_DATA: 'characterData',
+  CHILD_LIST: 'childList',
+  /**
+   * A change to what an element holds rather than to the document.
+   *
+   * Typing into an input, checking a box, choosing an option, moving focus and
+   * scrolling all change what the user sees and none of them mutate the DOM,
+   * so no observer reports them and replay skipped them entirely (issue #93).
+   */
+  LIVE_STATE: 'live-state',
+});
+
+/** What a `live-state` record changed. */
+export const TRACE_LIVE_STATE = Object.freeze({
+  VALUE: 'value',
+  CHECKED: 'checked',
+  SELECTED: 'selected',
+  FOCUS: 'focus',
+  SCROLL: 'scroll',
+});
+
+/** Why a checkpoint was taken. */
+export const TRACE_CHECKPOINT_REASON = Object.freeze({
+  /** The base snapshot every later record is a change against. */
+  INITIAL: 'initial',
+  /** A caller named this moment. */
+  CHECKPOINT: 'checkpoint',
+  /** The run failed here. */
+  FAILURE: 'failure',
+});
+
+/** Event families a caller can subscribe the recorder to. */
+export const TRACE_EVENT_SOURCES = Object.freeze([
+  'navigation',
+  'interaction',
+  'console',
+  'pageerror',
+  'dialog',
+  'requestfailed',
+  'download',
+]);
+
+/** Why a record was dropped. */
+export const TRACE_DROP_REASON = Object.freeze({
+  SIZE_LIMIT: 'size-limit',
+  WRITE_FAILED: 'write-failed',
+  CAPTURE_FAILED: 'capture-failed',
+  PAGE_CLOSED: 'page-closed',
+  TIMEOUT: 'timeout',
+});
+
+/** How a trace ended, recorded in the manifest. */
+export const TRACE_OUTCOME = Object.freeze({
+  /** `stop()` was called and every record was written. */
+  COMPLETE: 'complete',
+  /** The trace is readable but records are missing. */
+  PARTIAL: 'partial',
+  /** No manifest was ever written; readers repair this on open. */
+  TRUNCATED: 'truncated',
+});
+
+/** Default ceiling on a bundle, so a runaway page cannot fill a disk. */
+export const DEFAULT_MAX_BUNDLE_BYTES = 256 * 1024 * 1024;
+
+/** Default ceiling on any single member of a bundle. */
+export const DEFAULT_MAX_RESOURCE_BYTES = 32 * 1024 * 1024;
+
+/** How long a single capture may take before it is dropped. */
+export const DEFAULT_CAPTURE_TIMEOUT = 15000;
+
+/**
+ * Format a bundle member's sequence number.
+ *
+ * Zero padding keeps `ls` and any reader that sorts lexically in the same
+ * order as the sequence itself.
+ *
+ * @param {number} index - 1-based sequence number
+ * @returns {string} Zero-padded name, such as `0001`
+ */
+export function sequenceName(index) {
+  return String(index).padStart(4, '0');
+}
+
+/**
+ * What a bundle's records let a viewer reproduce.
+ *
+ * Issue #93 asked for the viewer to stop claiming more than it can do. Saying
+ * it in the manifest is better than saying it in the viewer's markup: a bundle
+ * recorded by an older version is honestly described by its own manifest, and
+ * any of the three readers can tell a caller what they are looking at.
+ */
+export const DEFAULT_REPLAY_SUPPORT = Object.freeze({
+  /** Checkpoints are snapshots, always replayable. */
+  checkpoints: true,
+  /** DOM mutations are recorded between checkpoints. */
+  mutations: false,
+  /** Child-list records say where a node went, so removals and moves replay. */
+  childListPositions: false,
+  /** Typing, checking, selecting, focus and scroll are recorded as they happen. */
+  liveState: false,
+  /** Every record names the context, page, navigation and frame it belongs to. */
+  identifiers: false,
+});
+
+/**
+ * Build the manifest of a trace.
+ *
+ * @param {Object} options - Manifest fields
+ * @returns {Object} The manifest as it is written to disk
+ */
+export function createManifest(options = {}) {
+  const {
+    mode,
+    startedAt,
+    stoppedAt = null,
+    outcome = TRACE_OUTCOME.COMPLETE,
+    commanderVersion = null,
+    engine = null,
+    browser = null,
+    platform = `${process.platform} ${process.arch}`,
+    runtime = `node ${process.versions?.node ?? 'unknown'}`,
+    counts = {},
+    limits = {},
+    privacy = {},
+    dom = {},
+    events = [],
+    dropped = 0,
+    replay = {},
+  } = options;
+
+  return {
+    schemaVersion: TRACE_SCHEMA_VERSION,
+    format: 'browser-commander-trace',
+    mode,
+    outcome,
+    startedAt,
+    stoppedAt,
+    commanderVersion,
+    engine,
+    browser,
+    platform,
+    runtime,
+    events: [...events],
+    dom: { ...dom },
+    replay: { ...DEFAULT_REPLAY_SUPPORT, ...replay },
+    privacy: { ...privacy },
+    limits: { ...limits },
+    counts: { checkpoints: 0, events: 0, mutationBatches: 0, ...counts },
+    dropped,
+  };
+}
+
+/**
+ * Check that a manifest can be read by this version of the format.
+ *
+ * @param {Object} manifest - Parsed manifest
+ * @throws {Error} When the bundle is not a readable trace
+ */
+export function assertReadableManifest(manifest) {
+  if (!manifest || manifest.format !== 'browser-commander-trace') {
+    throw new Error('not a Browser Commander trace bundle');
+  }
+  if (manifest.schemaVersion > TRACE_SCHEMA_VERSION) {
+    throw new Error(
+      `trace schema version ${manifest.schemaVersion} is newer than this reader (${TRACE_SCHEMA_VERSION})`
+    );
+  }
+}