@replayio-app-building/netlify-recorder 0.10.0 → 0.12.0

package/README.md

@@ -237,7 +237,6 @@ const handler: Handler = async (event) => {
 
   const recordingId = await ensureRequestRecording(requestId, {
     repositoryUrl: process.env.APP_REPOSITORY_URL!,
-    replayApiKey: process.env.RECORD_REPLAY_API_KEY!,
     lookupRequest: async (id) => {
       const [row] = await sql`
         SELECT blob_url, commit_sha, branch_name, handler_path
@@ -312,6 +311,10 @@ Self-hosted recording requires these environment variables:
 
 Wraps a Netlify handler function with automatic request recording. This is the recommended way to integrate — it handles `startRequest`/`finishRequest` and error cleanup internally.
 
+**Response timing:** When the Netlify Functions v2 `context` object is available (with `waitUntil`), the response is returned to the client **immediately** with a pre-generated `X-Replay-Request-Id` header. The blob upload and metadata storage continue in the background via `context.waitUntil()`, adding zero latency to the client response.
+
+When `context.waitUntil` is not available (v1 handlers or missing context), the wrapper falls back to awaiting `finishRequest` before returning.
+
 **Parameters:**
 - `handler` — Your async handler function `(event, context?) => Promise<{ statusCode, headers?, body? }>`
 - `options.callbacks` — Either `remoteCallbacks(serviceUrl)` or custom `{ uploadBlob, storeRequestData }` callbacks
@@ -322,6 +325,8 @@ Wraps a Netlify handler function with automatic request recording. This is the r
 
 **Returns:** A wrapped handler function with the same signature.
 
+**Callbacks note:** When using the `waitUntil` flow, `storeRequestData` receives a `requestId` field in its data parameter. Callbacks should use this as the row ID so the stored record matches the ID already sent to the client.
+
 ### `startRequest(event): RequestContext`
 
 Lower-level API. Begins capturing a Netlify handler execution. Patches `globalThis.fetch` and `process.env` to record all outbound network calls and environment variable reads. Use this with `finishRequest` when you need more control than `createRecordingRequestHandler` provides.
@@ -353,6 +358,7 @@ Logs a `console.warn` when the total duration exceeds 2 seconds or when individu
 - `options.commitSha` — Override `COMMIT_SHA` env var
 - `options.branchName` — Override `BRANCH_NAME` env var
 - `options.repositoryUrl` — Override `REPLAY_REPOSITORY_URL` env var
+- `options.requestId` — Pre-generated request ID (used by `createRecordingRequestHandler` in the `waitUntil` flow)
 
 ### `remoteCallbacks(serviceUrl): FinishRequestCallbacks`
 
@@ -368,7 +374,6 @@ Spawns a container via `@replayio/app-building` to create a Replay recording fro
 **Parameters:**
 - `requestId` — The request to create a recording for
 - `options.repositoryUrl` — Git repository URL for the container to clone
-- `options.replayApiKey` — Replay API key for recording upload
 - `options.lookupRequest(id)` — Fetches `{ blobUrl, commitSha, branchName, handlerPath }` from the database
 - `options.updateStatus(id, status, recordingId?)` — Updates the request status in the database
 

package/dist/index.d.ts

@@ -95,13 +95,22 @@ interface BlobData {
 interface FinishRequestCallbacks {
     /** Uploads serialized captured data and returns the blob URL. */
     uploadBlob: (data: string) => Promise<string>;
-    /** Stores request metadata in the database and returns the request ID. */
+    /**
+     * Stores request metadata in the database and returns the request ID.
+     *
+     * When `requestId` is provided (from `createRecordingRequestHandler`'s
+     * `waitUntil` flow), the callback should use it as the row ID so the
+     * client-facing header and the stored record match.  When omitted, the
+     * callback generates its own ID (backward-compatible).
+     */
     storeRequestData: (data: {
         blobUrl: string;
         commitSha: string;
         branchName: string;
         repositoryUrl: string;
         handlerPath: string;
+        /** Pre-generated request ID. Use as the row ID when provided. */
+        requestId?: string;
     }) => Promise<string>;
 }
 /**
@@ -126,7 +135,6 @@ interface ContainerInfraConfig {
 }
 interface EnsureRecordingOptions {
     repositoryUrl: string;
-    replayApiKey: string;
     /** Infrastructure credentials for starting the recording container. */
     infraConfig?: ContainerInfraConfig;
     /** Webhook URL the container can POST log entries to (optional). */
@@ -200,6 +208,15 @@ interface FinishRequestOptions {
     branchName?: string;
     /** Override REPLAY_REPOSITORY_URL env var. */
     repositoryUrl?: string;
+    /**
+     * Pre-generated request ID. When provided, this ID is passed to the
+     * `storeRequestData` callback so the stored row matches the ID already
+     * sent to the client in the `X-Replay-Request-Id` header.
+     *
+     * Used by `createRecordingRequestHandler` in the `waitUntil` flow where
+     * the response is returned before `finishRequest` runs.
+     */
+    requestId?: string;
 }
 /**
  * Called at the end of the handler execution.
@@ -232,9 +249,14 @@ interface CreateRecordingRequestHandlerOptions extends FinishRequestOptions {
  * after, capturing all outbound network calls and environment variable reads.
  * On error, interceptors are cleaned up and the error is re-thrown.
  *
- * **Important:** You must use the response returned by the wrapped handler
- * (not your original response object), because it includes the
- * `X-Replay-Request-Id` header set by `finishRequest`.
+ * **Response timing:** When the Netlify Functions v2 `context` object is
+ * available (with `waitUntil`), the response is returned to the client
+ * **immediately** with a pre-generated `X-Replay-Request-Id` header. The
+ * blob upload and metadata storage continue in the background via
+ * `context.waitUntil()`. This avoids adding latency to the client response.
+ *
+ * When `context.waitUntil` is not available (v1 handlers or missing context),
+ * the wrapper falls back to awaiting `finishRequest` before returning.
  *
  * For v2 handlers the request body is read from a clone internally — your
  * handler still receives the original request with an unconsumed body.
@@ -380,8 +402,6 @@ interface SpawnRecordingContainerOptions {
     branchName: string;
     /** Git repository URL for the app. */
     repositoryUrl: string;
-    /** Replay API key for uploading recordings. */
-    replayApiKey: string;
     /** Infrastructure credentials for Fly.io + Infisical. */
     infraConfig: ContainerInfraConfig;
     /** Optional webhook URL the container can POST log events to. */
@@ -401,4 +421,46 @@ interface SpawnRecordingContainerOptions {
  */
 declare function spawnRecordingContainer(options: SpawnRecordingContainerOptions): Promise<string>;
 
-export { type BlobData, type CapturedData, type ContainerInfraConfig, type CreateRecordingRequestHandlerOptions, type EnsureRecordingOptions, type EnvRead, type FinishRequestCallbacks, type FinishRequestOptions, type HandlerResponse$1 as HandlerResponse, type NetlifyEvent, type NetlifyV2Request, type NetworkCall, type RecordingResult, type RequestContext, type RequestInfo, type SpawnRecordingContainerOptions, createRecordingRequestHandler, createRequestRecording, ensureRequestRecording, finishRequest, readInfraConfigFromEnv, redactBlobData, remoteCallbacks, spawnRecordingContainer, startRequest };
+type SqlFunction = (...args: any[]) => Promise<any[]>;
+/**
+ * Creates the `audit_log` table and a generic PL/pgSQL trigger function
+ * (`audit_trigger_function`) that records INSERT, UPDATE, and DELETE
+ * operations on any monitored table.
+ *
+ * Call this once during schema initialization.
+ */
+declare function databaseAuditEnsureLogTable(sql: SqlFunction): Promise<void>;
+/**
+ * Creates a trigger on the specified table that calls
+ * `audit_trigger_function` for INSERT, UPDATE, and DELETE operations.
+ *
+ * Throws if `tableName` is `'audit_log'` (cannot monitor itself).
+ */
+declare function databaseAuditMonitorTable(sql: SqlFunction, tableName: string): Promise<void>;
+/**
+ * Returns all rows from the `audit_log` table, ordered by `performed_at` DESC.
+ */
+declare function databaseAuditDumpLogTable(sql: SqlFunction): Promise<Record<string, unknown>[]>;
+/**
+ * Wraps a Neon SQL tagged-template function so that after each query,
+ * any new `audit_log` rows (with NULL `replay_request_id`) are tagged
+ * with the current request ID and an incrementing call index.
+ *
+ * The request ID is read from module-level state set by
+ * `createRecordingRequestHandler`. If no request is active, the
+ * wrapper passes calls through without tagging.
+ *
+ * @example
+ * ```ts
+ * import { createAuditedSql } from "@replayio-app-building/netlify-recorder";
+ *
+ * const sql = createAuditedSql(getSql());
+ * await sql`INSERT INTO haikus (text) VALUES (${haiku})`;
+ * // audit_log entries from this INSERT are now tagged with the request ID
+ * ```
+ */
+declare function createAuditedSql(sql: SqlFunction): SqlFunction;
+
+declare function getCurrentRequestId(): string | null;
+
+export { type BlobData, type CapturedData, type ContainerInfraConfig, type CreateRecordingRequestHandlerOptions, type EnsureRecordingOptions, type EnvRead, type FinishRequestCallbacks, type FinishRequestOptions, type HandlerResponse$1 as HandlerResponse, type NetlifyEvent, type NetlifyV2Request, type NetworkCall, type RecordingResult, type RequestContext, type RequestInfo, type SpawnRecordingContainerOptions, createAuditedSql, createRecordingRequestHandler, createRequestRecording, databaseAuditDumpLogTable, databaseAuditEnsureLogTable, databaseAuditMonitorTable, ensureRequestRecording, finishRequest, getCurrentRequestId, readInfraConfigFromEnv, redactBlobData, remoteCallbacks, spawnRecordingContainer, startRequest };

package/dist/index.js

@@ -8,7 +8,7 @@ var __require = /* @__PURE__ */ ((x) => typeof require !== "undefined" ? require
 // src/interceptors/network.ts
 function installNetworkInterceptor(mode, calls) {
   const originalFetch = globalThis.fetch;
-  let replayCallIndex = 0;
+  const consumed = /* @__PURE__ */ new Set();
   if (mode === "capture") {
     const captureFetch = async (input, init) => {
       const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url;
@@ -40,16 +40,35 @@ function installNetworkInterceptor(mode, calls) {
     };
     globalThis.fetch = captureFetch;
   } else {
-    const replayFetch = async () => {
-      const idx = replayCallIndex++;
-      const call = calls[idx];
-      if (!call) {
+    const replayFetch = async (input, init) => {
+      const url = typeof input === "string" ? input : input instanceof URL ? input.href : typeof input === "object" && input !== null && "url" in input ? input.url : String(input);
+      const requestBody = typeof init?.body === "string" ? init.body : void 0;
+      let matchIdx = -1;
+      for (let i = 0; i < calls.length; i++) {
+        if (consumed.has(i)) continue;
+        const c = calls[i];
+        if (c && c.url === url && c.requestBody === requestBody) {
+          matchIdx = i;
+          break;
+        }
+      }
+      if (matchIdx === -1) {
+        for (let i = 0; i < calls.length; i++) {
+          if (!consumed.has(i)) {
+            matchIdx = i;
+            break;
+          }
+        }
+      }
+      const call = calls[matchIdx];
+      if (matchIdx === -1 || !call) {
         throw new Error(
           `No more recorded network calls to replay (exhausted ${calls.length} calls)`
         );
       }
+      consumed.add(matchIdx);
       console.log(
-        `  [network-replay] Consumed call ${idx + 1}/${calls.length}: ${call.method} ${call.url} => ${call.responseStatus}`
+        `  [network-replay] Consumed call ${consumed.size}/${calls.length}: ${call.method} ${call.url} => ${call.responseStatus}`
       );
       const body = call.responseBody ?? "";
       const status = call.responseStatus;
@@ -88,10 +107,17 @@ function installNetworkInterceptor(mode, calls) {
       globalThis.fetch = originalFetch;
     },
     consumedCount() {
-      return replayCallIndex;
+      return consumed.size;
     },
     totalCount() {
       return calls.length;
+    },
+    unconsumedIndices() {
+      const indices = [];
+      for (let i = 0; i < calls.length; i++) {
+        if (!consumed.has(i)) indices.push(i);
+      }
+      return indices;
     }
   };
 }
@@ -427,7 +453,8 @@ async function finishRequest(requestContext, callbacks, response, options) {
     commitSha,
     branchName,
     repositoryUrl,
-    handlerPath
+    handlerPath,
+    requestId: options?.requestId
   });
   const storeDuration = Date.now() - storeStart;
   if (storeDuration > SLOW_STEP_THRESHOLD_MS) {
@@ -450,17 +477,58 @@ async function finishRequest(requestContext, callbacks, response, options) {
   };
 }
 
+// src/createRecordingRequestHandler.ts
+import crypto from "crypto";
+
+// src/requestState.ts
+var _currentRequestId = null;
+function setCurrentRequestId(id) {
+  _currentRequestId = id;
+}
+function getCurrentRequestId() {
+  return _currentRequestId;
+}
+
 // src/createRecordingRequestHandler.ts
 function createRecordingRequestHandler(handler, options) {
   return async (event, context) => {
+    const requestId = crypto.randomUUID();
+    setCurrentRequestId(requestId);
     const reqContext = startRequest(event);
+    let response;
     try {
-      const response = await handler(event, context);
-      return await finishRequest(reqContext, options.callbacks, response, options);
+      response = await handler(event, context);
     } catch (err) {
+      setCurrentRequestId(null);
       reqContext.cleanup();
       throw err;
     }
+    reqContext.cleanup();
+    setCurrentRequestId(null);
+    const responseWithHeader = {
+      ...response,
+      headers: {
+        ...response.headers,
+        "X-Replay-Request-Id": requestId
+      }
+    };
+    const finishOpts = { ...options, requestId };
+    const ctx = context;
+    if (ctx && typeof ctx.waitUntil === "function") {
+      ctx.waitUntil(
+        finishRequest(reqContext, options.callbacks, response, finishOpts).catch(
+          (err) => {
+            console.error(
+              `netlify-recorder: background finishRequest failed (handler: ${options.handlerPath ?? "unknown"})`,
+              err
+            );
+          }
+        )
+      );
+      return responseWithHeader;
+    }
+    await finishRequest(reqContext, options.callbacks, response, finishOpts);
+    return responseWithHeader;
   };
 }
 
@@ -491,7 +559,8 @@ function remoteCallbacks(serviceUrl) {
             handlerPath: metadata.handlerPath,
             commitSha: metadata.commitSha,
             branchName: metadata.branchName,
-            repositoryUrl: metadata.repositoryUrl
+            repositoryUrl: metadata.repositoryUrl,
+            requestId: metadata.requestId
           })
         }
       );
@@ -515,7 +584,6 @@ async function spawnRecordingContainer(options) {
     commitSha,
     branchName,
     repositoryUrl,
-    replayApiKey,
     infraConfig,
     logWebhookUrl,
     onLog
@@ -605,7 +673,7 @@ async function spawnRecordingContainer(options) {
     `DB operations that were not in the original blob. Do NOT try to fix these.`,
     ``,
     `=== Step 6: Upload the recording ===`,
-    `RECORD_REPLAY_API_KEY=${replayApiKey} npx replayio upload --all 2>&1`,
+    `exec-secrets RECORD_REPLAY_API_KEY -- npx replayio upload --all 2>&1`,
     ``,
     `Find the recording ID (UUID) in the upload output and print:`,
     `  recording: <recording-id>`,
@@ -671,7 +739,7 @@ async function spawnRecordingContainer(options) {
 
 // src/ensureRequestRecording.ts
 async function ensureRequestRecording(requestId, options) {
-  const { repositoryUrl, replayApiKey, infraConfig, webhookUrl, lookupRequest, updateStatus, onLog } = options;
+  const { repositoryUrl, infraConfig, webhookUrl, lookupRequest, updateStatus, onLog } = options;
   const emit = async (level, message) => {
     if (onLog) {
       try {
@@ -696,7 +764,6 @@ async function ensureRequestRecording(requestId, options) {
       commitSha: requestData.commitSha,
       branchName: requestData.branchName,
       repositoryUrl,
-      replayApiKey,
       infraConfig,
       logWebhookUrl: webhookUrl,
       onLog
@@ -879,6 +946,12 @@ async function createRequestRecording(blobUrlOrData, handlerPath, requestInfo) {
         this.headers = new H(init?.headers);
         this.ok = this.status >= 200 && this.status < 300;
       }
+      static json(data, init) {
+        const body = JSON.stringify(data);
+        const headers = init?.headers ?? {};
+        headers["content-type"] = "application/json";
+        return new ResponseShim(body, { ...init, headers });
+      }
       async text() {
         return this._body ?? "";
       }
@@ -1002,17 +1075,18 @@ async function createRequestRecording(blobUrlOrData, handlerPath, requestInfo) {
       }
     }
   } finally {
-    const consumed = networkHandle.consumedCount();
+    const consumedCount = networkHandle.consumedCount();
     const total = networkHandle.totalCount();
-    if (consumed < total) {
+    if (consumedCount < total) {
       result.unconsumedNetworkCalls = true;
+      const unconsumed = networkHandle.unconsumedIndices();
       const details = [
-        `Consumed ${consumed} of ${total} calls. ${total - consumed} call(s) were never replayed.`
+        `Consumed ${consumedCount} of ${total} calls. ${unconsumed.length} call(s) were never replayed.`
       ];
       console.error(
-        `ERROR: Not all recorded network calls were consumed during replay. Consumed ${consumed} of ${total} calls. ${total - consumed} call(s) were never replayed.`
+        `ERROR: Not all recorded network calls were consumed during replay. Consumed ${consumedCount} of ${total} calls. ${unconsumed.length} call(s) were never replayed.`
       );
-      for (let i = consumed; i < total; i++) {
+      for (const i of unconsumed) {
         const call = blobData.capturedData.networkCalls[i];
         if (call) {
           details.push(`Unconsumed: ${call.method} ${call.url} => ${call.responseStatus}`);
@@ -1031,11 +1105,107 @@ async function createRequestRecording(blobUrlOrData, handlerPath, requestInfo) {
   }
   return result;
 }
+
+// src/databaseAudit.ts
+async function databaseAuditEnsureLogTable(sql) {
+  await sql`
+    CREATE TABLE IF NOT EXISTS audit_log (
+      id BIGSERIAL PRIMARY KEY,
+      table_name TEXT NOT NULL,
+      record_id TEXT NOT NULL,
+      action TEXT NOT NULL,
+      old_data JSONB,
+      new_data JSONB,
+      changed_fields TEXT[],
+      performed_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+      replay_request_id TEXT,
+      replay_request_call_index INTEGER
+    )
+  `;
+  await sql`CREATE INDEX IF NOT EXISTS idx_audit_log_table_name ON audit_log (table_name)`;
+  await sql`CREATE INDEX IF NOT EXISTS idx_audit_log_replay_request_id ON audit_log (replay_request_id)`;
+  await sql`CREATE INDEX IF NOT EXISTS idx_audit_log_performed_at ON audit_log (performed_at DESC)`;
+  await sql`
+    CREATE OR REPLACE FUNCTION audit_trigger_function()
+    RETURNS TRIGGER AS $$
+    DECLARE
+      changed_cols TEXT[];
+    BEGIN
+      IF TG_OP = 'INSERT' THEN
+        INSERT INTO audit_log (table_name, record_id, action, new_data)
+        VALUES (TG_TABLE_NAME, NEW.id::TEXT, 'INSERT', to_jsonb(NEW));
+        RETURN NEW;
+      ELSIF TG_OP = 'UPDATE' THEN
+        SELECT ARRAY_AGG(n.key) INTO changed_cols
+        FROM jsonb_each_text(to_jsonb(NEW)) n
+        LEFT JOIN jsonb_each_text(to_jsonb(OLD)) o ON n.key = o.key
+        WHERE o.value IS DISTINCT FROM n.value;
+
+        INSERT INTO audit_log (table_name, record_id, action, old_data, new_data, changed_fields)
+        VALUES (TG_TABLE_NAME, NEW.id::TEXT, 'UPDATE', to_jsonb(OLD), to_jsonb(NEW), COALESCE(changed_cols, ARRAY[]::TEXT[]));
+        RETURN NEW;
+      ELSIF TG_OP = 'DELETE' THEN
+        INSERT INTO audit_log (table_name, record_id, action, old_data)
+        VALUES (TG_TABLE_NAME, OLD.id::TEXT, 'DELETE', to_jsonb(OLD));
+        RETURN OLD;
+      END IF;
+      RETURN NULL;
+    END;
+    $$ LANGUAGE plpgsql
+  `;
+}
+async function databaseAuditMonitorTable(sql, tableName) {
+  if (tableName === "audit_log") {
+    throw new Error("Cannot monitor the audit_log table itself");
+  }
+  if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(tableName)) {
+    throw new Error(`Invalid table name: ${tableName}`);
+  }
+  const triggerName = `audit_trigger_${tableName}`;
+  await sql(
+    `DROP TRIGGER IF EXISTS ${triggerName} ON "${tableName}"`,
+    []
+  );
+  await sql(
+    `CREATE TRIGGER ${triggerName} AFTER INSERT OR UPDATE OR DELETE ON "${tableName}" FOR EACH ROW EXECUTE FUNCTION audit_trigger_function()`,
+    []
+  );
+}
+async function databaseAuditDumpLogTable(sql) {
+  const rows = await sql`SELECT * FROM audit_log ORDER BY performed_at DESC`;
+  return rows;
+}
+function createAuditedSql(sql) {
+  let callIndex = 0;
+  const requestId = getCurrentRequestId();
+  const auditedSql = async (strings, ...values) => {
+    const result = await sql(strings, ...values);
+    callIndex++;
+    if (requestId) {
+      try {
+        await sql`
+          UPDATE audit_log
+          SET replay_request_id = ${requestId}, replay_request_call_index = ${callIndex}
+          WHERE replay_request_id IS NULL
+        `;
+      } catch (err) {
+        console.warn("netlify-recorder: failed to update audit log:", err);
+      }
+    }
+    return result;
+  };
+  return auditedSql;
+}
 export {
+  createAuditedSql,
   createRecordingRequestHandler,
   createRequestRecording,
+  databaseAuditDumpLogTable,
+  databaseAuditEnsureLogTable,
+  databaseAuditMonitorTable,
   ensureRequestRecording,
   finishRequest,
+  getCurrentRequestId,
   readInfraConfigFromEnv,
   redactBlobData,
   remoteCallbacks,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@replayio-app-building/netlify-recorder",
-  "version": "0.10.0",
+  "version": "0.12.0",
   "description": "Capture and replay Netlify function executions as Replay recordings",
   "type": "module",
   "exports": {