@larc-iu/plaid-client 0.2.0-alpha.14 → 0.2.0-alpha.17

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@larc-iu/plaid-client",
-  "version": "0.2.0-alpha.14",
+  "version": "0.2.0-alpha.17",
   "description": "JavaScript client for the Plaid annotation API",
   "type": "module",
   "repository": {

package/src/created.js

@@ -0,0 +1,29 @@
+/**
+ * The ids a create answered with. A create's response is `{id}` and a bulk
+ * create's is `{ids}`, read straight off the call or off its result in a
+ * batch (`{status, body}`). Every reader of a create response goes through
+ * these, so none guesses at the shape.
+ */
+
+const isId = (v) => typeof v === "string";
+
+/**
+ * The id a single create answered with, or undefined when it gave none.
+ * @param {Object|null|undefined} result - the create's response, or its batch result
+ * @returns {string|undefined}
+ */
+export const createdId = (result) => {
+  if (isId(result?.id)) return result.id;
+  return isId(result?.body?.id) ? result.body.id : undefined;
+};
+
+/**
+ * The ids a bulk create answered with, in input order, or an empty array
+ * when it gave none.
+ * @param {Object|null|undefined} result - the bulk create's response, or its batch result
+ * @returns {string[]}
+ */
+export const createdIds = (result) => {
+  if (Array.isArray(result?.ids)) return result.ids;
+  return Array.isArray(result?.body?.ids) ? result.body.ids : [];
+};

package/src/http.js

@@ -1,46 +1,52 @@
 import { transformRequest, transformResponse } from "./transforms.js";
 
 // ---------------------------------------------------------------------------
-// Batch mode: which calls join a batch, and which go over the wire anyway.
+// Batches: which calls a batch carries, and which go over the wire anyway.
 //
-// `client.batched()` sets ONE flag on the whole client, and every call made
-// while it is set is queued, whatever code made it. A browser client is shared
-// by an editor, an importer and the app's chrome at once, so the call that
-// lands inside someone else's batch is usually not the one that opened it.
-// Every call reaching this layer is one of three things, and the third column
-// is what a new endpoint has to pick.
+// A batch is an OBJECT (`client.batch()`, or the argument `client.batched(fn)`
+// hands `fn`), a view of the client with the same bundles. A write made on the
+// batch queues; a call made on the client itself always goes over the wire,
+// whatever batches happen to be open. So the editor, the importer and the
+// app's chrome can share one client, and a write by code that knows nothing
+// about a batch can never land inside it. Every call reaching this layer is
+// one of three things, and the third column is what a new endpoint has to
+// pick.
 //
 // 1. A WRITE of project data: an entity, its metadata, a layer, a membership,
-//    a comment. This is what a batch is for, so it QUEUES. It is the default
-//    and needs no flag.
+//    a comment. This is what a batch is for, so on a batch it QUEUES. It is
+//    the default and needs no flag.
 //
-// 2. A READ. Pass `bypassBatch`. A queued read is broken three ways: the
-//    caller gets `{ batched: true }` instead of data, the queued GET takes a
-//    slot in the batch's results array and shifts every positional read after
-//    it, and server-side the sub-request runs against the batch's transaction
-//    connection rather than the pool, where `/query` throws and 500s the whole
-//    batch, rolling back every write in it. Bypassed, the read is answered
-//    from the pool and sees exactly the state it would have seen had the batch
-//    not been open.
+// 2. A READ. Every GET is answered from the wire even when made on a batch:
+//    the caller gets data rather than `{ batched: true }`, it takes no slot in
+//    the batch's results, and server-side it runs against the pool rather than
+//    the batch's transaction connection. A read that travels as a POST
+//    (`query`) says so with `outOfBand`.
 //
 // 3. An OUT-OF-BAND SIGNAL: stopping a service request, reporting a service's
-//    progress or result, taking or dropping a document lock, an admin action
-//    on the server itself. It is shaped like a write but carries no project
-//    data, and its whole value is that it happens NOW. Queued, it happens at
-//    submit, or never if the batch aborts, while its caller reads success.
-//    Pass `bypassBatch` for these too.
+//    progress or result, taking or dropping a document lock, forgetting a
+//    service, an admin action on the server itself. It is shaped like a write
+//    but carries no project data, and its whole value is that it happens NOW.
+//    Pass `outOfBand`: made on a batch it still goes over the wire, and it
+//    never joins an open logical operation. The audit group is a label for
+//    the writes, and a signal that is never audited would mark the group
+//    written and leave the relabel PATCH 404ing on a group nothing ever
+//    created.
 //
-// A bypassed call neither joins the batch nor spends its stamp. Strict mode
-// marks the batch's one expected document-version onto the first QUEUED write,
-// and a call that went over the wire on its own is not that write. It does not
-// join an open logical operation either: the audit group is a label for the
-// writes, and a signal that is never audited would mark the group written and
-// leave the relabel PATCH 404ing on a group nothing ever created.
+// A broadcast message (`messages.sendMessage`) sits between a write and a
+// signal: it stays batchable, so a message queued after the writes goes out
+// after them, but it writes nothing to the audit log either. `noOperation`
+// keeps it out of an open logical operation without making it out of band.
 //
-// `noBatch` is not part of that judgment. It marks the five calls the batch
-// transport cannot carry at all (a batch inside a batch, the multipart media
-// and avatar uploads, the user-data store's put and delete) and raises so the
-// caller finds out. Never put it on a read: it turns a swallowed read into a
+// Strict mode stamps the expected document-version onto every write, queued
+// or not. The server checks the first write of a batch request that names the
+// document and skips the rest of that document's, so the bump one op causes
+// does not 409 the next. A batch split past MAX_BATCH_OPS restamps each later
+// request with the version the one before it left (see submitBatch).
+//
+// `noBatch` is not part of that judgment. It marks the four calls the batch
+// transport cannot carry at all (the multipart media and avatar uploads, the
+// user-data store's put and delete) and raises so the caller finds out. A
+// batch inside a batch is refused by the batch itself. Never put it on a read: it turns a swallowed read into a
 // thrown one, which is what the chrome hit when an unrelated import was
 // running. A blobless DELETE beside an upload is not one of them: it carries
 // nothing the transport cannot express, so it takes its class from the three
@@ -73,6 +79,13 @@ export function extractDocumentVersions(
   responseBody = null,
   { historical = false } = {},
 ) {
+  // Past fifty documents the server leaves the list out and says how many it
+  // left out. Any version held may be one of them, so all are forgotten, and
+  // makeRequest reads the strict-mode document's again (learnOmittedVersion).
+  if (responseHeaders.get("X-Document-Versions-Omitted")) {
+    client.documentVersions = {};
+    client.documentVersionsOmitted = true;
+  }
   const docVersionsHeader = responseHeaders.get("X-Document-Versions");
   if (docVersionsHeader) {
     try {
@@ -293,45 +306,23 @@ export function xhrSend(
 }
 
 /**
- * Generic request method handling all fetch logic.
- *
- * Options:
- *   body            - Object body, run through transformRequest
- *   rawBody         - Body value passed directly (no transform). Mutually exclusive with body.
- *   formData        - If true, body is FormData; skip Content-Type header
- *   queryParams     - Object of query param key/values to append
- *   noBatch         - If true, throw when in batch mode. Only for calls the
- *                     batch transport cannot carry at all (see the note at the
- *                     top of this file); never for a read.
- *   bypassBatch     - If true, go over the wire even while a batch is open.
- *                     Every read carries it, and so does every out-of-band
- *                     signal (see the note at the top of this file).
- *   skipResponseTransform - Return raw parsed JSON (no transformResponse)
- *   noAuth          - Skip Authorization header
- *   binaryResponse  - Return arrayBuffer instead of JSON/text
- *   timeout         - Per-request timeout in ms overriding client.timeout
- *                     for this call (0/null disables). Used for known-long
- *                     ops like project delete.
- *   onUploadProgress - Called with `{ loaded, total }` (bytes; total null when
- *                     unknown) as the request body goes up. In a browser the
- *                     request then travels by XMLHttpRequest (see xhrSend),
- *                     where the timeout only fires when the upload stalls;
- *                     elsewhere the callback is ignored and fetch is used.
+ * Everything a request is before it goes anywhere: the URL with its query
+ * params and the stamps strict mode, a per-call audit message and an open
+ * logical operation add, plus the transformed body. Shared by the wire path
+ * (`makeRequest`) and the batch path (`queueRequest`), so a queued op is
+ * exactly the request that would have gone out. Strict mode stamps every
+ * write, queued or not, and `stampedDocument` names the document it stamped
+ * for (null when it stamped nothing).
  */
-export async function makeRequest(client, method, path, options = {}) {
+export function prepareRequest(client, method, path, options = {}) {
   const {
     body,
     rawBody,
     formData,
     queryParams,
-    noBatch,
-    bypassBatch,
-    skipResponseTransform,
-    noAuth,
-    binaryResponse,
+    outOfBand,
+    noOperation,
     auditMessage,
-    timeout,
-    onUploadProgress,
   } = options;
 
   // A write must not go out on a lock that lapsed. `documents.locked()`
@@ -340,11 +331,7 @@ export async function makeRequest(client, method, path, options = {}) {
   // have landed between the read the work was planned from and the write about
   // to go out. Reads pass, and so do the lock routes themselves, which is how
   // the block still releases on its way out.
-  if (
-    client.documentLockLost &&
-    method !== "GET" &&
-    !path.endsWith("/lock")
-  ) {
+  if (client.documentLockLost && method !== "GET" && !path.endsWith("/lock")) {
     throw client.documentLockLost;
   }
 
@@ -376,35 +363,30 @@ export async function makeRequest(client, method, path, options = {}) {
     requestBody = transformRequest(body);
   }
 
-  // Strict mode: append document-version for non-GET requests.
-  // Inside a batch, stamp ONLY the first queued write: batches run atomically
-  // server-side, so a version check on the first op gives whole-batch OCC
-  // semantics, while stamping every op would 409 the second op against the
-  // version bump the first op itself caused (every queued op captures the
-  // same pre-batch version).
-  if (
-    client.strictModeDocumentId &&
-    method !== "GET" &&
-    !(client.isBatching && client.batchVersionStamped)
-  ) {
+  // Strict mode: append document-version for non-GET requests, on EVERY
+  // queued write of a batch. The server validates the first write it can
+  // (one whose route resolves a document) and skips the rest of that
+  // document's, so the version bump a sub-op causes does not 409 the next
+  // one. Stamping only the first write was silently no check at all
+  // whenever that write was one the route ignores, such as a vocabulary
+  // entry's metadata, which is exactly what the igt editor queues first.
+  // `stampedDocument` names the document the stamp is for, so a batch split
+  // into several requests can restamp its later ones (see submitBatch).
+  let stampedDocument = null;
+  if (client.strictModeDocumentId && method !== "GET") {
     const docId = client.strictModeDocumentId;
     if (client.documentVersions[docId]) {
       const docVersion = client.documentVersions[docId];
       const separator = url.includes("?") ? "&" : "?";
       url += `${separator}document-version=${encodeURIComponent(docVersion)}`;
-      // A bypassing call queues nothing, so it must not spend the batch's one
-      // stamp. `query` is a POST and lands here from app chrome while someone
-      // else's batch is open: marking the batch stamped there left the first
-      // real write unversioned and the batch unguarded.
-      if (client.isBatching && !bypassBatch) client.batchVersionStamped = true;
+      stampedDocument = docId;
     }
   }
 
   // Per-call custom audit-log message (overrides the auto-generated
-  // description of THIS write). Unlike document-version this has no OCC
-  // self-conflict, so it is stamped on every queued batch op, not just the
-  // first. The server templates `{param}` placeholders against the endpoint's
-  // own path/query/body params.
+  // description of THIS write), stamped on the one write it was given to. The
+  // server templates `{param}` placeholders against the endpoint's own
+  // path/query/body params.
   if (auditMessage && method !== "GET") {
     const separator = url.includes("?") ? "&" : "?";
     url += `${separator}audit-message=${encodeURIComponent(auditMessage)}`;
@@ -414,16 +396,17 @@ export async function makeRequest(client, method, path, options = {}) {
   // with the group id; the message rides along too so the server can label
   // the group lazily on whichever tagged write lands first.
   //
-  // A bypassing call is not one of those writes. The out-of-band signals are
-  // shaped like a write and carry no project data (a lock taken or renewed, a
-  // cancelled service request, a service reporting its progress, an admin
-  // control), and `query` is a read that travels as a POST. None of them lands
-  // in the audit log, so a stamp does nothing server-side while `written`
-  // promises a group that will never exist: the relabel PATCH then 404s. Same
-  // rule as the batch and the OCC stamp above, for the same reason: a
-  // bypassing call belongs to whoever made it, not to whatever operation
-  // happens to be open on this shared client.
-  if (client.operationGroup && method !== "GET" && !bypassBatch) {
+  // An out-of-band signal is not one of those writes (see the note at the top
+  // of this file): it never lands in the audit log, so a stamp does nothing
+  // server-side while `written` promises a group that will never exist, and
+  // the relabel PATCH then 404s. A broadcast message (`noOperation`) is
+  // never audited either.
+  if (
+    client.operationGroup &&
+    method !== "GET" &&
+    !outOfBand &&
+    !noOperation
+  ) {
     const group = client.operationGroup;
     const separator = url.includes("?") ? "&" : "?";
     url += `${separator}group-id=${encodeURIComponent(group.id)}`;
@@ -432,24 +415,152 @@ export async function makeRequest(client, method, path, options = {}) {
     group.written = true;
   }
 
-  // Batch mode. A `bypassBatch` call is deliberately NOT queued: it belongs to
-  // whoever made it, not to whatever batch happens to be open on this shared
-  // client (a document reconcile, an import). See the three classes at the top
-  // of this file.
-  if (client.isBatching && !bypassBatch) {
-    if (noBatch) {
-      throw new Error(`This endpoint cannot be used in batch mode: ${path}`);
-    }
-    const operation = {
-      path: url.replace(client.baseUrl, ""),
-      method: method.toUpperCase(),
-    };
-    if (requestBody !== undefined) {
-      operation.body = requestBody;
-    }
-    client.batchOperations.push(operation);
-    return { batched: true };
+  return { url, requestBody, stampedDocument };
+}
+
+/**
+ * `path` with its strict-mode `document-version` claim replaced by `version`,
+ * or dropped when `version` is unknown. For the later requests of a batch
+ * split past MAX_BATCH_OPS: each op was stamped with the version the document
+ * had when it was queued, and every request before it has since moved the
+ * document on.
+ */
+export function restampDocumentVersion(path, version) {
+  const [base, query = ""] = path.split(/\?(.*)/s);
+  const params = query
+    .split("&")
+    .filter((p) => p && !p.startsWith("document-version="));
+  if (version !== undefined && version !== null) {
+    params.push(`document-version=${encodeURIComponent(version)}`);
+  }
+  return params.length ? `${base}?${params.join("&")}` : base;
+}
+
+/**
+ * The strict-mode document whose version a response left out
+ * (X-Document-Versions-Omitted) and the client has not learned since, or
+ * null. Such a write would go out unstamped, which is no check at all.
+ */
+function omittedStrictDocument(client) {
+  const docId = client.strictModeDocumentId;
+  return docId &&
+    client.documentVersionsOmitted &&
+    client.documentVersions[docId] === undefined
+    ? docId
+    : null;
+}
+
+/**
+ * Read the strict-mode document once to learn the version a response left
+ * out. Done right after that response, so the client's own large write is
+ * not mistaken for somebody else's edit at the next save, and again before a
+ * strict write when that read failed.
+ */
+async function learnOmittedVersion(client) {
+  const docId = omittedStrictDocument(client);
+  if (!docId) return;
+  // One read at a time: a write made while it is in flight waits for it
+  // rather than going out unstamped.
+  if (!client._learningOmittedVersion) {
+    client._learningOmittedVersion = makeRequest(
+      client,
+      "GET",
+      `/api/v1/documents/${docId}`,
+      { learningOmittedVersion: true },
+    ).finally(() => {
+      client._learningOmittedVersion = null;
+    });
+  }
+  await client._learningOmittedVersion;
+}
+
+async function learnOmittedVersionQuietly(client) {
+  try {
+    await learnOmittedVersion(client);
+  } catch (_) {
+    /* the write went through; the next strict write asks again */
+  }
+}
+
+/**
+ * A call made on a batch (see `PlaidClient#batch`). A write of project data is
+ * queued as one operation of the batch and answers `{ batched: true }`; its
+ * result is the matching entry of what `submit()` resolves to. A read, and a
+ * signal marked `outOfBand`, is the client's to make and goes over the wire
+ * now, exactly as if it had been made on the client.
+ */
+export async function queueRequest(batch, method, path, options = {}) {
+  if (method === "GET" || options.outOfBand) {
+    return batch.client._request(method, path, options);
+  }
+  if (!batch.open) {
+    throw new Error(`This batch was already submitted or aborted: ${path}`);
+  }
+  if (options.noBatch) {
+    throw new Error(`This endpoint cannot be used in a batch: ${path}`);
+  }
+  const { url, requestBody, stampedDocument } = prepareRequest(
+    batch.client,
+    method,
+    path,
+    options,
+  );
+  const operation = {
+    path: url.replace(batch.client.baseUrl, ""),
+    method: method.toUpperCase(),
+  };
+  if (requestBody !== undefined) {
+    operation.body = requestBody;
+  }
+  batch.operations.push(operation);
+  batch.stampedDocuments.push(stampedDocument);
+  return { batched: true };
+}
+
+/**
+ * Generic request method handling all fetch logic.
+ *
+ * Options:
+ *   body            - Object body, run through transformRequest
+ *   rawBody         - Body value passed directly (no transform). Mutually exclusive with body.
+ *   formData        - If true, body is FormData; skip Content-Type header
+ *   queryParams     - Object of query param key/values to append
+ *   noBatch         - If true, throw when made on a batch. Only for calls the
+ *                     batch transport cannot carry at all (see the note at the
+ *                     top of this file); never for a read.
+ *   outOfBand       - If true, the call is a signal rather than a write of
+ *                     project data: made on a batch it still goes over the
+ *                     wire, and it never joins an open logical operation (see
+ *                     the note at the top of this file).
+ *   noOperation     - If true, the call never joins an open logical operation
+ *                     but still queues on a batch like a write. For a
+ *                     broadcast message, which is never audited.
+ *   skipResponseTransform - Return raw parsed JSON (no transformResponse)
+ *   noAuth          - Skip Authorization header
+ *   binaryResponse  - Return arrayBuffer instead of JSON/text
+ *   timeout         - Per-request timeout in ms overriding client.timeout
+ *                     for this call (0/null disables). Used for known-long
+ *                     ops like project delete.
+ *   onUploadProgress - Called with `{ loaded, total }` (bytes; total null when
+ *                     unknown) as the request body goes up. In a browser the
+ *                     request then travels by XMLHttpRequest (see xhrSend),
+ *                     where the timeout only fires when the upload stalls;
+ *                     elsewhere the callback is ignored and fetch is used.
+ */
+export async function makeRequest(client, method, path, options = {}) {
+  const {
+    formData,
+    skipResponseTransform,
+    noAuth,
+    binaryResponse,
+    timeout,
+    onUploadProgress,
+    learningOmittedVersion,
+  } = options;
+  if (method !== "GET" && omittedStrictDocument(client)) {
+    await learnOmittedVersion(client);
   }
+  const { url, requestBody } = prepareRequest(client, method, path, options);
 
   // Build fetch options
   const headers = {};
@@ -518,6 +629,7 @@ export async function makeRequest(client, method, path, options = {}) {
     // Binary response (getMedia)
     if (binaryResponse) {
       extractDocumentVersions(client, response.headers);
+      if (!learningOmittedVersion) await learnOmittedVersionQuietly(client);
       return await response.arrayBuffer();
     }
 
@@ -528,12 +640,14 @@ export async function makeRequest(client, method, path, options = {}) {
       extractDocumentVersions(client, response.headers, data, {
         historical: /[?&]as-of=/.test(url),
       });
+      if (!learningOmittedVersion) await learnOmittedVersionQuietly(client);
       if (skipResponseTransform) {
         return data;
       }
       return transformResponse(data);
     } else {
       extractDocumentVersions(client, response.headers);
+      if (!learningOmittedVersion) await learnOmittedVersionQuietly(client);
       return await response.text();
     }
   } catch (error) {