@larc-iu/plaid-client 0.2.0-alpha.3 → 0.2.0-alpha.30

package/src/http.js

@@ -1,35 +1,172 @@
-import { transformRequest, transformResponse } from './transforms.js';
+import { markReplayed } from "./replayed.js";
+import { transformRequest, transformResponse } from "./transforms.js";
+import { nextIdempotencyKey } from "./operations.js";
+
+// ---------------------------------------------------------------------------
+// Batches: which calls a batch carries, and which go over the wire anyway.
+//
+// A batch is an OBJECT (`client.batch()`, or the argument `client.batched(fn)`
+// hands `fn`), a view of the client with the same bundles. A write made on the
+// batch queues; a call made on the client itself always goes over the wire,
+// whatever batches happen to be open. So the editor, the importer and the
+// app's chrome can share one client, and a write by code that knows nothing
+// about a batch can never land inside it. Every call reaching this layer is
+// one of three things, and the third column is what a new endpoint has to
+// pick.
+//
+// 1. A WRITE of project data: an entity, its metadata, a layer, a membership,
+//    a comment. This is what a batch is for, so on a batch it QUEUES. It is
+//    the default and needs no flag.
+//
+// 2. A READ. Every GET is answered from the wire even when made on a batch:
+//    the caller gets data rather than `{ batched: true }`, it takes no slot in
+//    the batch's results, and server-side it runs against the pool rather than
+//    the batch's transaction connection. A read that travels as a POST
+//    (`query`) says so with `outOfBand`.
+//
+// 3. An OUT-OF-BAND SIGNAL: stopping a service request, reporting a service's
+//    progress or result, taking or dropping a document lock, forgetting a
+//    service, an admin action on the server itself. It is shaped like a write
+//    but carries no project data, and its whole value is that it happens NOW.
+//    Pass `outOfBand`: made on a batch it still goes over the wire, and it
+//    never joins an open logical operation. The audit group is a label for
+//    the writes, and a signal that is never audited would mark the group
+//    written and leave the relabel PATCH 404ing on a group nothing ever
+//    created.
+//
+// A broadcast message (`messages.sendMessage`) sits between a write and a
+// signal: it stays batchable, so a message queued after the writes goes out
+// after them, but it writes nothing to the audit log either. `noOperation`
+// keeps it out of an open logical operation without making it out of band.
+//
+// Strict mode stamps the expected document-version onto every write, queued
+// or not. The server checks the first write of a batch request that names the
+// document and skips the rest of that document's, so the bump one op causes
+// does not 409 the next. A batch split past MAX_BATCH_OPS restamps each later
+// request with the version the one before it left (see submitBatch).
+//
+// `noBatch` is not part of that judgment. It marks the calls a batch cannot
+// carry at all (the multipart media and avatar uploads, the user-data store's
+// put and delete, and minting an API token or an invite, whose answer is a
+// secret the server refuses to keep with a keyed batch's answer) and raises
+// so the caller finds out. A
+// batch inside a batch is refused by the batch itself. Never put it on a read: it turns a swallowed read into a
+// thrown one, which is what the chrome hit when an unrelated import was
+// running. A blobless DELETE beside an upload is not one of them: it carries
+// nothing the transport cannot express, so it takes its class from the three
+// above like anything else.
+// ---------------------------------------------------------------------------
 
 // Default per-request timeout (ms). Applied to every request unless the client
 // is constructed with a different `timeout` (0 / null disables it). Note: this
 // also bounds media up/downloads — bump it (or disable) for very large files.
 export const DEFAULT_TIMEOUT_MS = 30000;
 
+// Batch submissions get their own, longer budget. A batch runs as ONE server
+// transaction holding the single SQLite write lock for its whole duration, and
+// aborting the HTTP request does not cancel it — so a short timeout buys
+// nothing and costs a retry stacked on a write that is still running.
+export const DEFAULT_BATCH_TIMEOUT_MS = 180000;
+
+// A query may run for core's own query limit (30 s, `*query-timeout-ms*` in
+// plaid.sql.query.exec), and core then answers 408. The client waits a little
+// longer than that, so the caller gets core's 408 for a query too broad and
+// not a timeout of its own that reads like a lost connection.
+export const DEFAULT_QUERY_TIMEOUT_MS = 35000;
+
+/**
+ * The timeout for a query: the client's own, raised to
+ * DEFAULT_QUERY_TIMEOUT_MS when it is shorter. A disabled timeout (0, null)
+ * stays disabled.
+ */
+export function queryTimeout(client) {
+  const t = client.timeout;
+  if (!t || t <= 0) return t;
+  return Math.max(t, DEFAULT_QUERY_TIMEOUT_MS);
+}
+
+/**
+ * Note how far the server's clock is from this machine's, from a response's
+ * Date header, for `client.serverNow()`. A time the server stamped (an audit
+ * entry's `ts`) has to be judged against the server's clock: a browser clock
+ * minutes off would misjudge it. A response with no readable Date header
+ * leaves the last offset as it was.
+ */
+export function noteServerClock(client, headers) {
+  const at = Date.parse(headers?.get?.("date") ?? "");
+  if (!Number.isNaN(at)) client.serverClockOffsetMs = at - Date.now();
+}
+
 /**
  * Extract and update document versions from response headers and body.
+ *
+ * `historical` marks a read made with `as-of`: its body carries the version
+ * the document HAD then, which is not what a strict-mode write must claim
+ * next, so the body is ignored and only the header (always the live
+ * version) is learned from. Without this, viewing an old state and then
+ * writing 409s on a version that stopped being current long ago.
  */
-export function extractDocumentVersions(client, responseHeaders, responseBody = null) {
-  const docVersionsHeader = responseHeaders.get('X-Document-Versions');
+export function extractDocumentVersions(
+  client,
+  responseHeaders,
+  responseBody = null,
+  { historical = false } = {},
+) {
+  // A replayed answer (see the Idempotency-Key note below) carries the
+  // versions right after the first send, and this page may have read newer
+  // ones since. Keep the higher of each.
+  const replayed = responseHeaders.get(REPLAYED_HEADER) === "true";
+  // Past fifty documents the server leaves the list out and says how many it
+  // left out. Any version held may be one of them, so all are forgotten, and
+  // makeRequest reads the strict-mode document's again (learnOmittedVersion).
+  if (responseHeaders.get("X-Document-Versions-Omitted")) {
+    client.documentVersions = {};
+    client.documentVersionsOmitted = true;
+  }
+  const docVersionsHeader = responseHeaders.get("X-Document-Versions");
   if (docVersionsHeader) {
     try {
       const versionsMap = JSON.parse(docVersionsHeader);
-      if (typeof versionsMap === 'object' && versionsMap !== null) {
+      if (typeof versionsMap === "object" && versionsMap !== null) {
         // Clone once, then assign — cloning inside the loop is O(n²) and pointless.
-        client.documentVersions = { ...client.documentVersions, ...versionsMap };
+        client.documentVersions = mergeVersions(
+          client.documentVersions,
+          versionsMap,
+          replayed,
+        );
       }
     } catch (e) {
-      console.warn('Failed to parse document versions header:', e);
+      console.warn("Failed to parse document versions header:", e);
     }
   }
 
-  if (responseBody && typeof responseBody === 'object') {
-    if (responseBody['document/id'] && responseBody['document/version']) {
-      client.documentVersions = { ...client.documentVersions };
-      client.documentVersions[responseBody['document/id']] = responseBody['document/version'];
+  if (!historical && responseBody && typeof responseBody === "object") {
+    if (responseBody["document/id"] && responseBody["document/version"]) {
+      client.documentVersions = mergeVersions(
+        client.documentVersions,
+        { [responseBody["document/id"]]: responseBody["document/version"] },
+        replayed,
+      );
     }
   }
 }
 
+/**
+ * `held` with `learned` merged in. A replayed answer only raises a version:
+ * it is what the document was right after the first send.
+ */
+export function mergeVersions(held, learned, replayed = false) {
+  if (!replayed) return { ...held, ...learned };
+  const merged = { ...held };
+  for (const [docId, version] of Object.entries(learned)) {
+    const current = merged[docId];
+    if (!(typeof current === "number" && current >= version)) {
+      merged[docId] = version;
+    }
+  }
+  return merged;
+}
+
 /**
  * Read a failed response's body as parsed JSON, falling back to text.
  */
@@ -37,7 +174,11 @@ export async function parseErrorBody(response) {
   try {
     return await response.json();
   } catch (_) {
-    return { message: await response.text().catch(() => 'Unable to read error response') };
+    return {
+      message: await response
+        .text()
+        .catch(() => "Unable to read error response"),
+    };
   }
 }
 
@@ -45,7 +186,11 @@ export async function parseErrorBody(response) {
  * Create an enriched error from a failed HTTP response.
  */
 export function makeHttpError(response, errorData, url, method) {
-  const serverMessage = errorData?.error || errorData?.message || response.statusText || 'Unknown error';
+  const serverMessage =
+    errorData?.error ||
+    errorData?.message ||
+    response.statusText ||
+    "Unknown error";
   const error = new Error(`HTTP ${response.status} ${serverMessage} at ${url}`);
   error.status = response.status;
   error.statusText = response.statusText;
@@ -59,7 +204,9 @@ export function makeHttpError(response, errorData, url, method) {
  * Create a network error (status 0). Timeout aborts get a clearer message.
  */
 export function makeNetworkError(originalError, url, method) {
-  const timedOut = originalError?.name === 'TimeoutError' || originalError?.name === 'AbortError';
+  const timedOut =
+    originalError?.name === "TimeoutError" ||
+    originalError?.name === "AbortError";
   const message = timedOut
     ? `Request timed out at ${url}`
     : `Network error: ${originalError.message} at ${url}`;
@@ -68,50 +215,274 @@ export function makeNetworkError(originalError, url, method) {
   error.url = url;
   error.method = method;
   error.originalError = originalError;
+  // The browser says it has no network, so the request most likely never
+  // left this machine. A timeout is not that: the request went, and may land.
+  if (!timedOut && globalThis.navigator?.onLine === false) error.offline = true;
   return error;
 }
 
+// Retry budget for 503 "Database busy" responses. Plaid serializes writers on
+// a single SQLite write lock; a writer that can't get it within the server's
+// busy_timeout is refused with 503. That refusal is definitive — the
+// transaction never opened, or was rolled back whole — so repeating the
+// request is safe, including for a batch (which is all-or-nothing by
+// construction). Retrying here is what lets a long import ride out someone
+// else's slow write instead of dying partway through.
+export const BUSY_RETRIES = 4;
+export const BUSY_BACKOFF_MS = 250;
+
+/**
+ * Run `attempt`, retrying while it rejects with a 503. Exponential backoff
+ * with full jitter, so two clients that collide don't march in lockstep and
+ * collide again on every retry.
+ *
+ * `attempt` must perform the whole request, not just await a prepared one:
+ * a fetch body and an AbortSignal.timeout are both single-use.
+ * @param {() => Promise<any>} attempt
+ * @param {{retries?: number, baseDelayMs?: number, onRetry?: (info: object) => void}} [options]
+ */
+export async function retryWhileBusy(
+  attempt,
+  { retries = BUSY_RETRIES, baseDelayMs = BUSY_BACKOFF_MS, onRetry } = {},
+) {
+  for (let i = 0; ; i += 1) {
+    try {
+      return await attempt();
+    } catch (error) {
+      if (error?.status !== 503 || i >= retries) throw error;
+      const delay = Math.round(baseDelayMs * 2 ** i * (0.5 + Math.random()));
+      onRetry?.({ attempt: i + 1, retries, delay, error });
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+  }
+}
+
+// ---------------------------------------------------------------------------
+// The Idempotency-Key.
+//
+// Every write that is not a signal (`outOfBand`) or a call a batch cannot
+// carry (`noBatch`: an upload, user data, a call that mints a secret) goes out
+// with an `Idempotency-Key` header, and so does every batch request (a queued
+// op has none: its batch request carries one). The server keeps the answer to a
+// keyed write for a day, and the same key sent again for the same request is
+// answered from it with `Idempotent-Replayed: true`, writing nothing. So a
+// write whose answer was lost (no response, 502, 504) can be sent again
+// safely: it lands once. `makeRequest` and `postBatch` do that three times
+// themselves (retryUnknown), and the error that escapes carries the key as
+// `idempotencyKey`.
+//
+// Inside a logical operation opened with `keys` (client.keySeed()), the nth
+// keyed request that joins it takes the key `<seed>.<n>` and, from its first
+// run, a pinned strict-mode stamp, so running the operation again from the
+// top sends byte for byte the requests the first run sent: those that landed
+// are replayed and the rest run. A request that does not join the operation
+// mints its own key.
+// ---------------------------------------------------------------------------
+
+export const IDEMPOTENCY_HEADER = "Idempotency-Key";
+export const REPLAYED_HEADER = "Idempotent-Replayed";
+
+// Delays before sending a keyed write again when its answer was lost.
+export const UNKNOWN_RETRY_DELAYS_MS = [1000, 3000, 9000];
+
+/**
+ * Whether `error` leaves the outcome unknown: no response came (status 0),
+ * or a proxy answered for the server (502, 504). The write may have landed.
+ */
+export function isUnknownOutcome(error) {
+  const status = error?.status;
+  return status === 0 || status === 502 || status === 504;
+}
+
+/**
+ * Whether a create refused with `status` and body `data` was made by an
+ * earlier send of the same work: `409 id-taken` for an id in `minted` (a Set
+ * of the ids the work minted for its creates), and not for a row deleted
+ * since (there is nothing to go on with). The one rule for a lost answer's
+ * create. The Python twin is `minted_taken`.
+ */
+export function mintedTaken(status, data, minted) {
+  return (
+    status === 409 &&
+    data?.error === "id-taken" &&
+    !data.deleted &&
+    !!minted?.has(data.id)
+  );
+}
+
+/**
+ * Run `attempt`, sending it again after each delay in `delaysMs` while it
+ * fails with an unknown outcome. Only for a keyed write, whose resend is
+ * answered from the first send when that one landed. Not when the browser
+ * says it is offline: then the request never left, and the caller waits for
+ * the network instead. Full jitter, as retryWhileBusy.
+ * @param {() => Promise<any>} attempt
+ * @param {{delaysMs?: number[], onRetry?: (info: object) => void}} [options]
+ */
+export async function retryUnknown(
+  attempt,
+  { delaysMs = UNKNOWN_RETRY_DELAYS_MS, onRetry } = {},
+) {
+  for (let i = 0; ; i += 1) {
+    try {
+      return await attempt();
+    } catch (error) {
+      if (!isUnknownOutcome(error) || error.offline || i >= delaysMs.length) {
+        throw error;
+      }
+      const delay = Math.round(delaysMs[i] * (0.5 + Math.random()));
+      onRetry?.({ attempt: i + 1, retries: delaysMs.length, delay, error });
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+  }
+}
+
 /**
  * Build a fetch AbortSignal that fires after `timeout` ms, or undefined when
  * timeouts are disabled / unsupported.
  */
 export function timeoutSignal(timeout) {
-  if (timeout && timeout > 0 && typeof AbortSignal !== 'undefined' && AbortSignal.timeout) {
+  if (
+    timeout &&
+    timeout > 0 &&
+    typeof AbortSignal !== "undefined" &&
+    AbortSignal.timeout
+  ) {
     return AbortSignal.timeout(timeout);
   }
   return undefined;
 }
 
 /**
- * Generic request method handling all fetch logic.
- *
- * Options:
- *   body            - Object body, run through transformRequest
- *   rawBody         - Body value passed directly (no transform). Mutually exclusive with body.
- *   formData        - If true, body is FormData; skip Content-Type header
- *   queryParams     - Object of query param key/values to append
- *   noBatch         - If true, throw when in batch mode
- *   skipResponseTransform - Return raw parsed JSON (no transformResponse)
- *   noAuth          - Skip Authorization header
- *   binaryResponse  - Return arrayBuffer instead of JSON/text
- *   timeout         - Per-request timeout in ms overriding client.timeout
- *                     for this call (0/null disables). Used for known-long
- *                     ops like project delete.
+ * Send a request through XMLHttpRequest so the caller can watch the bytes go
+ * up: fetch cannot report upload progress. Resolves to a Response, so the rest
+ * of the pipeline is the same either way. The timeout is a STALL timeout,
+ * re-armed by every progress event, so a large file on a slow link is never
+ * cut off while it is still moving, only once nothing has moved for `timeout`
+ * ms. Rejects the way fetch does: a TypeError for a network failure, an error
+ * named TimeoutError for a stall.
+ * @param {string} url
+ * @param {{method: string, headers: Record<string, string>, body: any}} request
+ * @param {{onUploadProgress?: (progress: {loaded: number, total: number | null}) => void, timeout?: number | null}} [options]
  */
-export async function makeRequest(client, method, path, options = {}) {
+export function xhrSend(
+  url,
+  { method, headers, body },
+  { onUploadProgress, timeout } = {},
+) {
+  return new Promise((resolve, reject) => {
+    const xhr = new XMLHttpRequest();
+    xhr.open(method, url);
+    xhr.responseType = "arraybuffer";
+    for (const [name, value] of Object.entries(headers || {}))
+      xhr.setRequestHeader(name, value);
+
+    let timer = null;
+    const disarm = () => clearTimeout(timer);
+    const arm = () => {
+      if (!timeout || timeout <= 0) return;
+      disarm();
+      timer = setTimeout(() => {
+        xhr.abort();
+        reject(
+          Object.assign(new Error(`Request timed out at ${url}`), {
+            name: "TimeoutError",
+          }),
+        );
+      }, timeout);
+    };
+
+    xhr.upload.onprogress = (e) => {
+      arm();
+      if (onUploadProgress) {
+        onUploadProgress({
+          loaded: e.loaded,
+          total: e.lengthComputable ? e.total : null,
+        });
+      }
+    };
+    xhr.onprogress = arm;
+    xhr.onload = () => {
+      disarm();
+      if (xhr.status === 0) {
+        reject(new TypeError(`Failed to fetch ${url}`));
+        return;
+      }
+      const responseHeaders = new Headers();
+      for (const line of (xhr.getAllResponseHeaders() || "")
+        .trim()
+        .split(/\r?\n/)) {
+        const i = line.indexOf(":");
+        if (i > 0)
+          responseHeaders.append(
+            line.slice(0, i).trim(),
+            line.slice(i + 1).trim(),
+          );
+      }
+      // A Response refuses a body for these statuses.
+      const responseBody = [204, 205, 304].includes(xhr.status)
+        ? null
+        : xhr.response;
+      resolve(
+        new Response(responseBody, {
+          status: xhr.status,
+          statusText: xhr.statusText,
+          headers: responseHeaders,
+        }),
+      );
+    };
+    xhr.onerror = () => {
+      disarm();
+      reject(new TypeError(`Failed to fetch ${url}`));
+    };
+    xhr.onabort = () => {
+      disarm();
+      reject(
+        Object.assign(new Error(`Request aborted at ${url}`), {
+          name: "AbortError",
+        }),
+      );
+    };
+
+    arm();
+    xhr.send(body);
+  });
+}
+
+/**
+ * Everything a request is before it goes anywhere: the URL with its query
+ * params and the stamps strict mode, a per-call audit message and an open
+ * logical operation add, plus the transformed body. Shared by the wire path
+ * (`makeRequest`) and the batch path (`queueRequest`), so a queued op is
+ * exactly the request that would have gone out. Strict mode stamps every
+ * write, queued or not, and `stampedDocument` names the document it stamped
+ * for (null when it stamped nothing). `stampedGroup` is the open logical
+ * operation it joined, if any, for `makeRequest` to mark written once the
+ * server has taken the request.
+ */
+export function prepareRequest(client, method, path, options = {}) {
   const {
     body,
     rawBody,
     formData,
     queryParams,
-    noBatch,
-    skipResponseTransform,
-    noAuth,
-    binaryResponse,
+    outOfBand,
+    noOperation,
     auditMessage,
-    timeout,
+    pinnedVersion,
+    versioned,
   } = options;
 
+  // A write must not go out on a lock that lapsed. `documents.locked()`
+  // records the loss here when its keep-alive cannot renew, and from that
+  // moment the block is holding nothing: an edit by somebody else can already
+  // have landed between the read the work was planned from and the write about
+  // to go out. Reads pass, and so do the lock routes themselves, which is how
+  // the block still releases on its way out.
+  if (client.documentLockLost && method !== "GET" && !path.endsWith("/lock")) {
+    throw client.documentLockLost;
+  }
+
   // Build URL
   let url = `${client.baseUrl}${path}`;
 
@@ -140,114 +511,459 @@ export async function makeRequest(client, method, path, options = {}) {
     requestBody = transformRequest(body);
   }
 
-  // Strict mode: append document-version for non-GET requests.
-  // Inside a batch, stamp ONLY the first write: batches run atomically
-  // server-side, so a version check on the first op gives whole-batch OCC
-  // semantics, while stamping every op would 409 the second op against the
-  // version bump the first op itself caused (every queued op captures the
-  // same pre-batch version).
-  if (client.strictModeDocumentId && method !== 'GET'
-      && !(client.isBatching && client.batchVersionStamped)) {
+  // Strict mode: append document-version for non-GET requests, on EVERY
+  // queued write of a batch. The server validates the first write it can
+  // (one whose route resolves a document) and skips the rest of that
+  // document's, so the version bump a sub-op causes does not 409 the next
+  // one. Stamping only the first write was silently no check at all
+  // whenever that write was one the route ignores, such as a vocabulary
+  // entry's metadata, which is exactly what the igt editor queues first.
+  // `stampedDocument` names the document the stamp is for, so a batch split
+  // into several requests can restamp its later ones (see submitBatch). An
+  // out-of-band signal (a lock, a service's progress, a query) is no write of
+  // the document and carries none.
+  // A resend of an operation's request claims what its first run claimed
+  // (`pinnedVersion`, null for no claim), so the two are the same request.
+  let stampedDocument = null;
+  let stampedVersion = null;
+  // A write with its own precondition (`versioned: false`, a text edit sent
+  // with the digest of the body it was made on) goes without the stamp.
+  if (
+    client.strictModeDocumentId &&
+    method !== "GET" &&
+    !outOfBand &&
+    versioned !== false
+  ) {
     const docId = client.strictModeDocumentId;
-    if (client.documentVersions[docId]) {
-      const docVersion = client.documentVersions[docId];
-      const separator = url.includes('?') ? '&' : '?';
+    const docVersion =
+      pinnedVersion !== undefined
+        ? pinnedVersion
+        : client.documentVersions[docId];
+    if (docVersion) {
+      const separator = url.includes("?") ? "&" : "?";
       url += `${separator}document-version=${encodeURIComponent(docVersion)}`;
-      if (client.isBatching) client.batchVersionStamped = true;
+      stampedDocument = docId;
+      stampedVersion = docVersion;
     }
   }
 
   // Per-call custom audit-log message (overrides the auto-generated
-  // description of THIS write). Unlike document-version this has no OCC
-  // self-conflict, so it is stamped on every queued batch op, not just the
-  // first. The server templates `{param}` placeholders against the endpoint's
-  // own path/query/body params.
-  if (auditMessage && method !== 'GET') {
-    const separator = url.includes('?') ? '&' : '?';
+  // description of THIS write), stamped on the one write it was given to. The
+  // server templates `{param}` placeholders against the endpoint's own
+  // path/query/body params.
+  if (auditMessage && method !== "GET") {
+    const separator = url.includes("?") ? "&" : "?";
     url += `${separator}audit-message=${encodeURIComponent(auditMessage)}`;
   }
 
   // Logical-operation group (see client.beginOperation): stamp every write
   // with the group id; the message rides along too so the server can label
-  // the group lazily on whichever tagged write lands first.
-  if (client.operationGroup && method !== 'GET') {
+  // the group lazily on whichever tagged write lands first. The group counts
+  // as written only once such a write has been taken (`makeRequest`): a group
+  // whose only write was refused was never made, and relabelling it 404s.
+  //
+  // An out-of-band signal is not one of those writes (see the note at the top
+  // of this file): it never lands in the audit log, so a stamp does nothing
+  // server-side while `written` promises a group that will never exist, and
+  // the relabel PATCH then 404s. A broadcast message (`noOperation`) is
+  // never audited either.
+  let stampedGroup = null;
+  if (
+    client.operationGroup &&
+    method !== "GET" &&
+    !outOfBand &&
+    !noOperation
+  ) {
     const group = client.operationGroup;
-    const separator = url.includes('?') ? '&' : '?';
+    const separator = url.includes("?") ? "&" : "?";
     url += `${separator}group-id=${encodeURIComponent(group.id)}`;
-    if (group.message) url += `&group-message=${encodeURIComponent(group.message)}`;
-    group.written = true;
+    if (group.message)
+      url += `&group-message=${encodeURIComponent(group.message)}`;
+    if (group.kind) url += `&group-kind=${encodeURIComponent(group.kind)}`;
+    if (group.ref) url += `&group-ref=${encodeURIComponent(group.ref)}`;
+    stampedGroup = group;
   }
 
-  // Batch mode
-  if (client.isBatching) {
-    if (noBatch) {
-      throw new Error(`This endpoint cannot be used in batch mode: ${path}`);
+  return { url, requestBody, stampedDocument, stampedVersion, stampedGroup };
+}
+
+/**
+ * Whether a write joins the open logical operation (see prepareRequest).
+ */
+export function joinsOperation(client, method, options = {}) {
+  return Boolean(
+    client.operationGroup &&
+      method !== "GET" &&
+      !options.outOfBand &&
+      !options.noOperation,
+  );
+}
+
+/**
+ * Whether a call made on the client goes out with an Idempotency-Key.
+ */
+export function takesIdempotencyKey(method, options = {}) {
+  return (
+    method !== "GET" &&
+    !options.outOfBand &&
+    !options.noBatch
+  );
+}
+
+/**
+ * `path` with its strict-mode `document-version` claim replaced by `version`,
+ * or dropped when `version` is unknown. For the later requests of a batch
+ * split past MAX_BATCH_OPS: each op was stamped with the version the document
+ * had when it was queued, and every request before it has since moved the
+ * document on.
+ */
+export function restampDocumentVersion(path, version) {
+  const [base, query = ""] = path.split(/\?(.*)/s);
+  const params = query
+    .split("&")
+    .filter((p) => p && !p.startsWith("document-version="));
+  if (version !== undefined && version !== null) {
+    params.push(`document-version=${encodeURIComponent(version)}`);
+  }
+  return params.length ? `${base}?${params.join("&")}` : base;
+}
+
+/**
+ * The strict-mode document whose version a response left out
+ * (X-Document-Versions-Omitted) and the client has not learned since, or
+ * null. Such a write would go out unstamped, which is no check at all.
+ */
+function omittedStrictDocument(client) {
+  const docId = client.strictModeDocumentId;
+  return docId &&
+    client.documentVersionsOmitted &&
+    client.documentVersions[docId] === undefined
+    ? docId
+    : null;
+}
+
+/**
+ * Read the strict-mode document once to learn the version a response left
+ * out. Done right after that response, so the client's own large write is
+ * not mistaken for somebody else's edit at the next save, and again before a
+ * strict write when that read failed.
+ */
+async function learnOmittedVersion(client) {
+  const docId = omittedStrictDocument(client);
+  if (!docId) return;
+  // One read at a time: a write made while it is in flight waits for it
+  // rather than going out unstamped.
+  if (!client._learningOmittedVersion) {
+    client._learningOmittedVersion = makeRequest(
+      client,
+      "GET",
+      `/api/v1/documents/${docId}`,
+      { learningOmittedVersion: true },
+    ).finally(() => {
+      client._learningOmittedVersion = null;
+    });
+  }
+  await client._learningOmittedVersion;
+}
+
+async function learnOmittedVersionQuietly(client) {
+  try {
+    await learnOmittedVersion(client);
+  } catch (_) {
+    /* the write went through; the next strict write asks again */
+  }
+}
+
+/**
+ * One send of a request by fetch, under a fresh timeout signal: an
+ * AbortSignal.timeout fires once and stays aborted, so a retry needs its own.
+ * `timeoutMs` 0 or null sends with none.
+ */
+export function fetchWithTimeout(url, fetchOptions, timeoutMs) {
+  const signal = timeoutSignal(timeoutMs);
+  return fetch(url, signal ? { ...fetchOptions, signal } : fetchOptions);
+}
+
+/**
+ * Run `attempt` with the client's retries: a 503 is retried (unless
+ * `noBusyRetry`), and a keyed request whose answer was lost is sent again
+ * under its key (retryUnknown). Each retry is told to `onRetry` listeners.
+ */
+export function withRetries(client, attempt, { keyed = false, noBusyRetry = false } = {}) {
+  const onRetry = (info) => client._noteRetry?.(info);
+  const busy = noBusyRetry ? { onRetry, retries: 0 } : { onRetry };
+  return keyed
+    ? retryUnknown(() => retryWhileBusy(attempt, busy), {
+        delaysMs: client.retryDelaysMs,
+        onRetry,
+      })
+    : retryWhileBusy(attempt, busy);
+}
+
+/**
+ * A 401 means the token is missing, expired or invalid. Fire the app's
+ * auth-error handler once per client (it discards the token and routes back
+ * to login). A 403 (authenticated but not permitted) deliberately does not.
+ */
+export function noteAuthError(client, error) {
+  if (
+    error?.status === 401 &&
+    typeof client.onAuthError === "function" &&
+    !client._authErrorFired
+  ) {
+    client._authErrorFired = true;
+    try {
+      client.onAuthError(error);
+    } catch (_) {
+      /* handler must not mask the original error */
     }
-    const operation = {
-      path: url.replace(client.baseUrl, ''),
-      method: method.toUpperCase(),
-    };
-    if (requestBody !== undefined) {
-      operation.body = requestBody;
+  }
+}
+
+/**
+ * Generic request method handling all fetch logic.
+ *
+ * Options:
+ *   body            - Object body, run through transformRequest
+ *   rawBody         - Body value passed directly (no transform). Mutually exclusive with body.
+ *   formData        - If true, body is FormData; skip Content-Type header
+ *   queryParams     - Object of query param key/values to append
+ *   noBatch         - If true, throw when made on a batch. Only for calls a
+ *                     batch cannot carry at all (see the note at the
+ *                     top of this file); never for a read.
+ *   outOfBand       - If true, the call is a signal rather than a write of
+ *                     project data: made on a batch it still goes over the
+ *                     wire, and it never joins an open logical operation (see
+ *                     the note at the top of this file).
+ *   noOperation     - If true, the call never joins an open logical operation
+ *                     but still queues on a batch like a write. For a
+ *                     broadcast message, which is never audited.
+ *   versioned       - If false, strict mode does not stamp the call with the
+ *                     document version. For a write that carries its own
+ *                     precondition (a text edit with `base`).
+ *   skipResponseTransform - Return raw parsed JSON (no transformResponse)
+ *   noAuth          - Skip Authorization header
+ *   binaryResponse  - Return arrayBuffer instead of JSON/text
+ *   blobResponse    - Return a Blob of the body, its `type` the response's
+ *                     Content-Type (a recording, whose reader needs both)
+ *   onDownloadProgress - With blobResponse, called with `{ loaded, total }`
+ *                     (bytes; total null when unknown) as the body arrives
+ *   noBusyRetry     - If true, a 503 is not retried. For a query: core
+ *                     answers 503 when its queue of large queries stays full
+ *                     until the query's deadline, and asking again at once
+ *                     only joins the same queue.
+ *   timeout         - Per-request timeout in ms overriding client.timeout
+ *                     for this call (0/null disables). Used for known-long
+ *                     ops like project delete.
+ *   onUploadProgress - Called with `{ loaded, total }` (bytes; total null when
+ *                     unknown) as the request body goes up. In a browser the
+ *                     request then travels by XMLHttpRequest (see xhrSend),
+ *                     where the timeout only fires when the upload stalls;
+ *                     elsewhere the callback is ignored and fetch is used.
+ */
+/**
+ * What a write about to go out needs to tell the lock keepers it renewed:
+ * each held document's keeper, its version now and the send time. Empty when
+ * no `documents.locked()` block on `client` keeps a lock. The server renews a
+ * live lock on every write its holder makes to the document, so a write that
+ * moved the held document's version renewed it (`writesLanded`).
+ * @private
+ */
+export function writesStarted(client) {
+  const keepers = client.lockKeepers;
+  if (!keepers || keepers.size === 0) return [];
+  return [...keepers.entries()].map(([docId, keeper]) => [
+    keeper,
+    client.documentVersions?.[docId],
+    keeper.now(),
+  ]);
+}
+
+/**
+ * The write `started` was read before has been taken, and its answer's
+ * versions learned. A held document whose version it moved was written by
+ * this run. A replayed answer was not taken again, and renewed nothing.
+ * @private
+ */
+export function writesLanded(client, started, replayed = false) {
+  if (replayed || !started?.length) return;
+  for (const [keeper, before, sent] of started) {
+    const after = client.documentVersions?.[keeper.documentId];
+    if (before != null && after != null && after !== before) {
+      keeper.noteWrite(sent);
     }
-    client.batchOperations.push(operation);
-    return { batched: true };
   }
+}
+
+/**
+ * A response's body as a Blob typed by its Content-Type, `onProgress` told
+ * `{ loaded, total }` after each piece when given (total null when the
+ * response has no Content-Length).
+ * @private
+ */
+async function readBlob(response, onProgress) {
+  const type = response.headers.get("content-type") ?? "";
+  const reader = onProgress ? response.body?.getReader?.() : null;
+  if (!reader) return new Blob([await response.arrayBuffer()], { type });
+  const length = Number(response.headers.get("content-length"));
+  const total = length > 0 ? length : null;
+  const parts = [];
+  let loaded = 0;
+  for (;;) {
+    const { done, value } = await reader.read();
+    if (done) break;
+    parts.push(value);
+    loaded += value.byteLength;
+    onProgress({ loaded, total });
+  }
+  return new Blob(parts, { type });
+}
+
+export async function makeRequest(client, method, path, options = {}) {
+  const {
+    formData,
+    skipResponseTransform,
+    noAuth,
+    binaryResponse,
+    blobResponse,
+    onDownloadProgress,
+    timeout,
+    onUploadProgress,
+    learningOmittedVersion,
+    noBusyRetry,
+  } = options;
+  if (method !== "GET" && omittedStrictDocument(client)) {
+    await learnOmittedVersion(client);
+  }
+  const keyed = takesIdempotencyKey(method, options);
+  const joins = joinsOperation(client, method, options);
+  // The ids the joined operation minted for what it creates (withOperation's
+  // `minted`): a create refused 409 id-taken for one of them was made by an
+  // earlier send of this operation, and answers as made.
+  const minted = joins ? client.operationGroup?.frames.at(-1)?.minted : null;
+  const { key, pin, record } = keyed ? nextIdempotencyKey(client, joins) : {};
+  const { url, requestBody, stampedGroup, stampedVersion } = prepareRequest(
+    client,
+    method,
+    path,
+    keyed ? { ...options, pinnedVersion: pin } : options,
+  );
+  if (keyed) record(stampedVersion);
 
   // Build fetch options
   const headers = {};
   if (!noAuth) {
-    headers['Authorization'] = `Bearer ${client.token}`;
+    headers["Authorization"] = `Bearer ${client.token}`;
   }
   if (!formData) {
-    headers['Content-Type'] = 'application/json';
+    headers["Content-Type"] = "application/json";
   }
+  if (keyed) headers[IDEMPOTENCY_HEADER] = key;
 
   const fetchOptions = { method, headers };
   if (requestBody !== undefined) {
     fetchOptions.body = formData ? requestBody : JSON.stringify(requestBody);
   }
-  const signal = timeoutSignal(timeout !== undefined ? timeout : client.timeout);
-  if (signal) fetchOptions.signal = signal;
+
+  // A fresh AbortSignal.timeout per attempt: the signal fires once and stays
+  // aborted, so reusing it would make every retry fail instantly.
+  const timeoutMs = timeout !== undefined ? timeout : client.timeout;
+  const send = () => {
+    if (onUploadProgress && typeof XMLHttpRequest !== "undefined") {
+      return xhrSend(url, fetchOptions, {
+        onUploadProgress,
+        timeout: timeoutMs,
+      });
+    }
+    return fetchWithTimeout(url, fetchOptions, timeoutMs);
+  };
+
+  // One attempt: a 503, and for a keyed write a 502 or 504, surfaces as a
+  // throw so the retries below can see it. Other failures are thrown from
+  // the checks after it.
+  // A write that lands on a held document renews its lock on the server, so
+  // its keeper hears of it (writesLanded).
+  const started =
+    method !== "GET" && !options.outOfBand ? writesStarted(client) : [];
+  const landed = (headers) =>
+    writesLanded(client, started, headers.get(REPLAYED_HEADER) === "true");
+
+  const attempt = async () => {
+    let res;
+    try {
+      res = await send();
+    } catch (e) {
+      throw makeNetworkError(e, url, method);
+    }
+    if (
+      res.status === 503 ||
+      (keyed && (res.status === 502 || res.status === 504))
+    ) {
+      throw makeHttpError(res, await parseErrorBody(res), url, method);
+    }
+    return res;
+  };
 
   try {
-    const response = await fetch(url, fetchOptions);
+    const response = await withRetries(client, attempt, { keyed, noBusyRetry });
+    noteServerClock(client, response.headers);
 
     if (!response.ok) {
-      const error = makeHttpError(response, await parseErrorBody(response), url, method);
-      // 401 means the token is missing/expired/invalid. Fire the app's auth-error
-      // handler once (it discards the token and routes back to login). 403
-      // (forbidden — authenticated but not permitted) deliberately does NOT.
-      if (response.status === 401 && typeof client.onAuthError === 'function' && !client._authErrorFired) {
-        client._authErrorFired = true;
-        try { client.onAuthError(error); } catch (_) { /* handler must not mask the original error */ }
+      const errorData = await parseErrorBody(response);
+      // Answered as made only for a single create, the one whose answer is
+      // the id alone. A bulk or a batch taken so is made whole (it is one
+      // transaction), and a caller that sends one decides what it answers.
+      if (
+        mintedTaken(response.status, errorData, minted) &&
+        requestBody?.id === errorData.id
+      ) {
+        if (stampedGroup) stampedGroup.written = true;
+        return transformResponse({ id: errorData.id });
       }
+      const error = makeHttpError(response, errorData, url, method);
+      noteAuthError(client, error);
       throw error;
     }
+    // The server took it, so the operation's group exists.
+    if (stampedGroup) stampedGroup.written = true;
 
-    // Binary response (getMedia)
+    // Binary response (users.getAvatar)
     if (binaryResponse) {
       extractDocumentVersions(client, response.headers);
+      landed(response.headers);
+      if (!learningOmittedVersion) await learnOmittedVersionQuietly(client);
       return await response.arrayBuffer();
     }
+    if (blobResponse) {
+      extractDocumentVersions(client, response.headers);
+      landed(response.headers);
+      if (!learningOmittedVersion) await learnOmittedVersionQuietly(client);
+      return await readBlob(response, onDownloadProgress);
+    }
 
     // JSON or text response
-    const contentType = response.headers.get('content-type');
-    if (contentType && contentType.includes('application/json')) {
+    const contentType = response.headers.get("content-type");
+    if (contentType && contentType.includes("application/json")) {
       const data = await response.json();
-      extractDocumentVersions(client, response.headers, data);
-      if (skipResponseTransform) {
-        return data;
-      }
-      return transformResponse(data);
+      extractDocumentVersions(client, response.headers, data, {
+        historical: /[?&]as-of=/.test(url),
+      });
+      landed(response.headers);
+      if (!learningOmittedVersion) await learnOmittedVersionQuietly(client);
+      const answer = skipResponseTransform ? data : transformResponse(data);
+      return response.headers.get(REPLAYED_HEADER) === "true"
+        ? markReplayed(answer)
+        : answer;
     } else {
       extractDocumentVersions(client, response.headers);
+      landed(response.headers);
+      if (!learningOmittedVersion) await learnOmittedVersionQuietly(client);
       return await response.text();
     }
   } catch (error) {
-    if (error.status !== undefined) {
-      throw error;
-    }
-    throw makeNetworkError(error, url, method);
+    const escaped =
+      error.status !== undefined ? error : makeNetworkError(error, url, method);
+    if (keyed) escaped.idempotencyKey = key;
+    throw escaped;
   }
 }