@larc-iu/plaid-client 0.2.0-alpha.18 → 0.2.0-alpha.20

package/src/http.js

@@ -1,4 +1,7 @@
+import { markReplayed } from "./replayed.js";
 import { transformRequest, transformResponse } from "./transforms.js";
+import { takeRefs } from "./batchRef.js";
+import { uuidv7 } from "./ids.js";
 
 // ---------------------------------------------------------------------------
 // Batches: which calls a batch carries, and which go over the wire anyway.
@@ -91,6 +94,10 @@ export function extractDocumentVersions(
   responseBody = null,
   { historical = false } = {},
 ) {
+  // A replayed answer (see the Idempotency-Key note below) carries the
+  // versions right after the first send, and this page may have read newer
+  // ones since. Keep the higher of each.
+  const replayed = responseHeaders.get(REPLAYED_HEADER) === "true";
   // Past fifty documents the server leaves the list out and says how many it
   // left out. Any version held may be one of them, so all are forgotten, and
   // makeRequest reads the strict-mode document's again (learnOmittedVersion).
@@ -104,10 +111,11 @@ export function extractDocumentVersions(
       const versionsMap = JSON.parse(docVersionsHeader);
       if (typeof versionsMap === "object" && versionsMap !== null) {
         // Clone once, then assign — cloning inside the loop is O(n²) and pointless.
-        client.documentVersions = {
-          ...client.documentVersions,
-          ...versionsMap,
-        };
+        client.documentVersions = mergeVersions(
+          client.documentVersions,
+          versionsMap,
+          replayed,
+        );
       }
     } catch (e) {
       console.warn("Failed to parse document versions header:", e);
@@ -116,13 +124,31 @@ export function extractDocumentVersions(
 
   if (!historical && responseBody && typeof responseBody === "object") {
     if (responseBody["document/id"] && responseBody["document/version"]) {
-      client.documentVersions = { ...client.documentVersions };
-      client.documentVersions[responseBody["document/id"]] =
-        responseBody["document/version"];
+      client.documentVersions = mergeVersions(
+        client.documentVersions,
+        { [responseBody["document/id"]]: responseBody["document/version"] },
+        replayed,
+      );
     }
   }
 }
 
+/**
+ * `held` with `learned` merged in. A replayed answer only raises a version:
+ * it is what the document was right after the first send.
+ */
+export function mergeVersions(held, learned, replayed = false) {
+  if (!replayed) return { ...held, ...learned };
+  const merged = { ...held };
+  for (const [docId, version] of Object.entries(learned)) {
+    const current = merged[docId];
+    if (!(typeof current === "number" && current >= version)) {
+      merged[docId] = version;
+    }
+  }
+  return merged;
+}
+
 /**
  * Read a failed response's body as parsed JSON, falling back to text.
  */
@@ -171,6 +197,9 @@ export function makeNetworkError(originalError, url, method) {
   error.url = url;
   error.method = method;
   error.originalError = originalError;
+  // The browser says it has no network, so the request most likely never
+  // left this machine. A timeout is not that: the request went, and may land.
+  if (!timedOut && globalThis.navigator?.onLine === false) error.offline = true;
   return error;
 }
 
@@ -208,6 +237,91 @@ export async function retryWhileBusy(
   }
 }
 
+// ---------------------------------------------------------------------------
+// The Idempotency-Key.
+//
+// Every write that is not a signal (`outOfBand`), an upload (`noBatch`) or a
+// call that mints a secret (`noIdempotency`) goes out with an
+// `Idempotency-Key` header, and so does every batch request (a queued op has
+// none: its batch request carries one). The server keeps the answer to a
+// keyed write for a day, and the same key sent again for the same request is
+// answered from it with `Idempotent-Replayed: true`, writing nothing. So a
+// write whose answer was lost (no response, 502, 504) can be sent again
+// safely: it lands once. `makeRequest` and `_postBatch` do that three times
+// themselves (retryUnknown), and the error that escapes carries the key as
+// `idempotencyKey`.
+//
+// Inside a logical operation opened with `keys` (client.keySeed()), the nth
+// keyed request that joins it takes the key `<seed>.<n>` and, from its first
+// run, a pinned strict-mode stamp, so running the operation again from the
+// top sends byte for byte the requests the first run sent: those that landed
+// are replayed and the rest run. A request that does not join the operation
+// mints its own key.
+// ---------------------------------------------------------------------------
+
+export const IDEMPOTENCY_HEADER = "Idempotency-Key";
+export const REPLAYED_HEADER = "Idempotent-Replayed";
+
+// Delays before sending a keyed write again when its answer was lost.
+export const UNKNOWN_RETRY_DELAYS_MS = [1000, 3000, 9000];
+
+/**
+ * Whether `error` leaves the outcome unknown: no response came (status 0),
+ * or a proxy answered for the server (502, 504). The write may have landed.
+ */
+export function isUnknownOutcome(error) {
+  const status = error?.status;
+  return status === 0 || status === 502 || status === 504;
+}
+
+/**
+ * Run `attempt`, sending it again after each delay in `delaysMs` while it
+ * fails with an unknown outcome. Only for a keyed write, whose resend is
+ * answered from the first send when that one landed. Not when the browser
+ * says it is offline: then the request never left, and the caller waits for
+ * the network instead. Full jitter, as retryWhileBusy.
+ */
+export async function retryUnknown(
+  attempt,
+  { delaysMs = UNKNOWN_RETRY_DELAYS_MS, onRetry } = {},
+) {
+  for (let i = 0; ; i += 1) {
+    try {
+      return await attempt();
+    } catch (error) {
+      if (!isUnknownOutcome(error) || error.offline || i >= delaysMs.length) {
+        throw error;
+      }
+      const delay = Math.round(delaysMs[i] * (0.5 + Math.random()));
+      onRetry?.({ attempt: i + 1, retries: delaysMs.length, delay, error });
+      await new Promise((resolve) => setTimeout(resolve, delay));
+    }
+  }
+}
+
+/**
+ * The key for the next keyed request, and the strict-mode stamp it must
+ * claim when an earlier run of its operation already sent it. `joins` says
+ * whether the request joins the open logical operation.
+ * Returns `{ key, pin, record }`: `pin` is undefined when nothing is pinned,
+ * and `record(version)` pins what this request claimed, the first time.
+ */
+export function nextIdempotencyKey(client, joins) {
+  const frame = joins ? client.operationGroup?.frames.at(-1) : null;
+  const keys = frame?.keys;
+  if (!keys) return { key: uuidv7(), pin: undefined, record: () => {} };
+  const n = frame.count;
+  frame.count += 1;
+  const pinned = keys.stamps.has(n);
+  return {
+    key: `${keys.seed}.${n}`,
+    pin: pinned ? keys.stamps.get(n) : undefined,
+    record: (version) => {
+      if (!pinned) keys.stamps.set(n, version ?? null);
+    },
+  };
+}
+
 /**
  * Build a fetch AbortSignal that fires after `timeout` ms, or undefined when
  * timeouts are disabled / unsupported.
@@ -324,7 +438,9 @@ export function xhrSend(
  * (`makeRequest`) and the batch path (`queueRequest`), so a queued op is
  * exactly the request that would have gone out. Strict mode stamps every
  * write, queued or not, and `stampedDocument` names the document it stamped
- * for (null when it stamped nothing).
+ * for (null when it stamped nothing). `stampedGroup` is the open logical
+ * operation it joined, if any, for `makeRequest` to mark written once the
+ * server has taken the request.
  */
 export function prepareRequest(client, method, path, options = {}) {
   const {
@@ -335,6 +451,8 @@ export function prepareRequest(client, method, path, options = {}) {
     outOfBand,
     noOperation,
     auditMessage,
+    pinnedVersion,
+    versioned,
   } = options;
 
   // A write must not go out on a lock that lapsed. `documents.locked()`
@@ -383,15 +501,31 @@ export function prepareRequest(client, method, path, options = {}) {
   // whenever that write was one the route ignores, such as a vocabulary
   // entry's metadata, which is exactly what the igt editor queues first.
   // `stampedDocument` names the document the stamp is for, so a batch split
-  // into several requests can restamp its later ones (see submitBatch).
+  // into several requests can restamp its later ones (see submitBatch). An
+  // out-of-band signal (a lock, a service's progress, a query) is no write of
+  // the document and carries none.
+  // A resend of an operation's request claims what its first run claimed
+  // (`pinnedVersion`, null for no claim), so the two are the same request.
   let stampedDocument = null;
-  if (client.strictModeDocumentId && method !== "GET") {
+  let stampedVersion = null;
+  // A write with its own precondition (`versioned: false`, a text edit sent
+  // with the digest of the body it was made on) goes without the stamp.
+  if (
+    client.strictModeDocumentId &&
+    method !== "GET" &&
+    !outOfBand &&
+    versioned !== false
+  ) {
     const docId = client.strictModeDocumentId;
-    if (client.documentVersions[docId]) {
-      const docVersion = client.documentVersions[docId];
+    const docVersion =
+      pinnedVersion !== undefined
+        ? pinnedVersion
+        : client.documentVersions[docId];
+    if (docVersion) {
       const separator = url.includes("?") ? "&" : "?";
       url += `${separator}document-version=${encodeURIComponent(docVersion)}`;
       stampedDocument = docId;
+      stampedVersion = docVersion;
     }
   }
 
@@ -406,13 +540,16 @@ export function prepareRequest(client, method, path, options = {}) {
 
   // Logical-operation group (see client.beginOperation): stamp every write
   // with the group id; the message rides along too so the server can label
-  // the group lazily on whichever tagged write lands first.
+  // the group lazily on whichever tagged write lands first. The group counts
+  // as written only once such a write has been taken (`makeRequest`): a group
+  // whose only write was refused was never made, and relabelling it 404s.
   //
   // An out-of-band signal is not one of those writes (see the note at the top
   // of this file): it never lands in the audit log, so a stamp does nothing
   // server-side while `written` promises a group that will never exist, and
   // the relabel PATCH then 404s. A broadcast message (`noOperation`) is
   // never audited either.
+  let stampedGroup = null;
   if (
     client.operationGroup &&
     method !== "GET" &&
@@ -426,10 +563,34 @@ export function prepareRequest(client, method, path, options = {}) {
       url += `&group-message=${encodeURIComponent(group.message)}`;
     if (group.kind) url += `&group-kind=${encodeURIComponent(group.kind)}`;
     if (group.ref) url += `&group-ref=${encodeURIComponent(group.ref)}`;
-    group.written = true;
+    stampedGroup = group;
   }
 
-  return { url, requestBody, stampedDocument };
+  return { url, requestBody, stampedDocument, stampedVersion, stampedGroup };
+}
+
+/**
+ * Whether a write joins the open logical operation (see prepareRequest).
+ */
+export function joinsOperation(client, method, options = {}) {
+  return Boolean(
+    client.operationGroup &&
+      method !== "GET" &&
+      !options.outOfBand &&
+      !options.noOperation,
+  );
+}
+
+/**
+ * Whether a call made on the client goes out with an Idempotency-Key.
+ */
+export function takesIdempotencyKey(method, options = {}) {
+  return (
+    method !== "GET" &&
+    !options.outOfBand &&
+    !options.noBatch &&
+    !options.noIdempotency
+  );
 }
 
 /**
@@ -513,7 +674,7 @@ export async function queueRequest(batch, method, path, options = {}) {
   if (options.noBatch) {
     throw new Error(`This endpoint cannot be used in a batch: ${path}`);
   }
-  const { url, requestBody, stampedDocument } = prepareRequest(
+  const { url, requestBody, stampedDocument, stampedGroup } = prepareRequest(
     batch.client,
     method,
     path,
@@ -524,10 +685,15 @@ export async function queueRequest(batch, method, path, options = {}) {
     method: method.toUpperCase(),
   };
   if (requestBody !== undefined) {
-    operation.body = requestBody;
+    // Each b.ref() in the body goes beside it, as the path to a null the
+    // server fills with the id (batchRef.js).
+    const { body, refs } = takeRefs(batch, requestBody);
+    operation.body = body;
+    if (refs) operation.refs = refs;
   }
   batch.operations.push(operation);
   batch.stampedDocuments.push(stampedDocument);
+  batch.stampedGroups.push(stampedGroup);
   return { batched: true };
 }
 
@@ -549,6 +715,9 @@ export async function queueRequest(batch, method, path, options = {}) {
  *   noOperation     - If true, the call never joins an open logical operation
  *                     but still queues on a batch like a write. For a
  *                     broadcast message, which is never audited.
+ *   versioned       - If false, strict mode does not stamp the call with the
+ *                     document version. For a write that carries its own
+ *                     precondition (a text edit with `base`).
  *   skipResponseTransform - Return raw parsed JSON (no transformResponse)
  *   noAuth          - Skip Authorization header
  *   binaryResponse  - Return arrayBuffer instead of JSON/text
@@ -574,7 +743,20 @@ export async function makeRequest(client, method, path, options = {}) {
   if (method !== "GET" && omittedStrictDocument(client)) {
     await learnOmittedVersion(client);
   }
-  const { url, requestBody } = prepareRequest(client, method, path, options);
+  const keyed = takesIdempotencyKey(method, options);
+  const joins = joinsOperation(client, method, options);
+  // The ids the joined operation minted for what it creates (withOperation's
+  // `minted`): a create refused 409 id-taken for one of them was made by an
+  // earlier send of this operation, and answers as made.
+  const minted = joins ? client.operationGroup?.frames.at(-1)?.minted : null;
+  const { key, pin, record } = keyed ? nextIdempotencyKey(client, joins) : {};
+  const { url, requestBody, stampedGroup, stampedVersion } = prepareRequest(
+    client,
+    method,
+    path,
+    keyed ? { ...options, pinnedVersion: pin } : options,
+  );
+  if (keyed) record(stampedVersion);
 
   // Build fetch options
   const headers = {};
@@ -584,6 +766,7 @@ export async function makeRequest(client, method, path, options = {}) {
   if (!formData) {
     headers["Content-Type"] = "application/json";
   }
+  if (keyed) headers[IDEMPOTENCY_HEADER] = key;
 
   const fetchOptions = { method, headers };
   if (requestBody !== undefined) {
@@ -604,25 +787,48 @@ export async function makeRequest(client, method, path, options = {}) {
     return fetch(url, signal ? { ...fetchOptions, signal } : fetchOptions);
   };
 
+  // One attempt: a 503, and for a keyed write a 502 or 504, surfaces as a
+  // throw so the retries below can see it. Other failures are thrown from
+  // the checks after it.
+  const attempt = async () => {
+    let res;
+    try {
+      res = await send();
+    } catch (e) {
+      throw makeNetworkError(e, url, method);
+    }
+    if (
+      res.status === 503 ||
+      (keyed && (res.status === 502 || res.status === 504))
+    ) {
+      throw makeHttpError(res, await parseErrorBody(res), url, method);
+    }
+    return res;
+  };
+
   try {
-    const response = await retryWhileBusy(async () => {
-      const res = await send();
-      // Surface a 503 as a throw so retryWhileBusy can see it; other failures
-      // are re-thrown from here and handled by the caller below.
-      if (res.status === 503) {
-        throw makeHttpError(res, await parseErrorBody(res), url, method);
-      }
-      return res;
-    });
+    const response = keyed
+      ? await retryUnknown(() => retryWhileBusy(attempt), {
+          delaysMs: client.retryDelaysMs,
+        })
+      : await retryWhileBusy(attempt);
     noteServerClock(client, response.headers);
 
     if (!response.ok) {
-      const error = makeHttpError(
-        response,
-        await parseErrorBody(response),
-        url,
-        method,
-      );
+      const errorData = await parseErrorBody(response);
+      // Only a single create: a bulk refused whole made none of its rows.
+      // Never a deleted row: there is nothing to open.
+      if (
+        response.status === 409 &&
+        errorData?.error === "id-taken" &&
+        !errorData.deleted &&
+        minted?.has(errorData.id) &&
+        requestBody?.id === errorData.id
+      ) {
+        if (stampedGroup) stampedGroup.written = true;
+        return transformResponse({ id: errorData.id });
+      }
+      const error = makeHttpError(response, errorData, url, method);
       // 401 means the token is missing/expired/invalid. Fire the app's auth-error
       // handler once (it discards the token and routes back to login). 403
       // (forbidden — authenticated but not permitted) deliberately does NOT.
@@ -640,6 +846,8 @@ export async function makeRequest(client, method, path, options = {}) {
       }
       throw error;
     }
+    // The server took it, so the operation's group exists.
+    if (stampedGroup) stampedGroup.written = true;
 
     // Binary response (getMedia)
     if (binaryResponse) {
@@ -656,19 +864,19 @@ export async function makeRequest(client, method, path, options = {}) {
         historical: /[?&]as-of=/.test(url),
       });
       if (!learningOmittedVersion) await learnOmittedVersionQuietly(client);
-      if (skipResponseTransform) {
-        return data;
-      }
-      return transformResponse(data);
+      const answer = skipResponseTransform ? data : transformResponse(data);
+      return response.headers.get(REPLAYED_HEADER) === "true"
+        ? markReplayed(answer)
+        : answer;
     } else {
       extractDocumentVersions(client, response.headers);
       if (!learningOmittedVersion) await learnOmittedVersionQuietly(client);
       return await response.text();
     }
   } catch (error) {
-    if (error.status !== undefined) {
-      throw error;
-    }
-    throw makeNetworkError(error, url, method);
+    const escaped =
+      error.status !== undefined ? error : makeNetworkError(error, url, method);
+    if (keyed) escaped.idempotencyKey = key;
+    throw escaped;
   }
 }

package/src/ids.js

@@ -0,0 +1,54 @@
+/**
+ * Ids a client mints for what it creates.
+ *
+ * A create may name the id of the row it makes, so a create sent again after
+ * its answer was lost lands under the same id or is told the id is taken
+ * (409 with `id-taken`). The server takes only a UUIDv7 (RFC 9562), shaped
+ * as its own ids are: a 48-bit millisecond timestamp, a 12-bit counter
+ * within the millisecond, then 62 random bits. Reads are id-ordered, so ids
+ * minted in one millisecond must still sort in the order they were made.
+ */
+
+let lastMs = 0;
+let counter = 0;
+
+const hex = Array.from({ length: 256 }, (_, i) =>
+  i.toString(16).padStart(2, "0"),
+);
+
+/**
+ * A fresh UUIDv7, later than every one this page minted before it.
+ * @returns {string}
+ */
+export function uuidv7() {
+  const now = Date.now();
+  if (now > lastMs) {
+    lastMs = now;
+    counter = 0;
+  } else if (counter < 0xfff) {
+    counter += 1;
+  } else {
+    // This millisecond is full: take the next one rather than repeat an order.
+    lastMs += 1;
+    counter = 0;
+  }
+  const b = new Uint8Array(16);
+  crypto.getRandomValues(b);
+  const high = Math.floor(lastMs / 0x100000000);
+  const low = lastMs >>> 0;
+  b[0] = (high >>> 8) & 0xff;
+  b[1] = high & 0xff;
+  b[2] = (low >>> 24) & 0xff;
+  b[3] = (low >>> 16) & 0xff;
+  b[4] = (low >>> 8) & 0xff;
+  b[5] = low & 0xff;
+  b[6] = 0x70 | (counter >>> 8);
+  b[7] = counter & 0xff;
+  b[8] = (b[8] & 0x3f) | 0x80;
+  let s = "";
+  for (let i = 0; i < 16; i++) {
+    s += hex[b[i]];
+    if (i === 3 || i === 5 || i === 7 || i === 9) s += "-";
+  }
+  return s;
+}