@lossless.org/client 1.22.2 → 1.23.1

package/ts/nosqldb/classes.writeobserver.ts

@@ -77,28 +77,37 @@ interface IWriteObserver {
 const maximumObservedFields = 64;
 const observersByDb = new WeakMap<SmartdataDb, Set<IWriteObserver>>();
 
-const normalizeObserverOptions = (optionsArg: unknown): IWriteObserver => {
-  const { fields, listener } = assertDataOptions(optionsArg, ['fields', 'listener']);
-  if (typeof listener !== 'function') {
-    throw new SmartdataPersistenceError('invalid_argument',
-      'A SmartData write observer requires a listener function.');
-  }
+/**
+ * @internal Validates the top-level field names a report carries: 1 to 64
+ * distinct names, no `.` and no leading `$`. `subjectArg` names what requires
+ * them in the refusal.
+ */
+export const normalizeObservedFields = (fieldsArg: unknown, subjectArg: string): readonly string[] => {
   if (
-    !Array.isArray(fields)
-    || fields.length === 0
-    || fields.length > maximumObservedFields
-    || fields.some((fieldArg) =>
+    !Array.isArray(fieldsArg)
+    || fieldsArg.length === 0
+    || fieldsArg.length > maximumObservedFields
+    || fieldsArg.some((fieldArg) =>
       typeof fieldArg !== 'string'
       || fieldArg.length === 0
       || fieldArg.includes('.')
       || fieldArg.startsWith('$'))
-    || new Set(fields).size !== fields.length
+    || new Set(fieldsArg).size !== fieldsArg.length
   ) {
     throw new SmartdataPersistenceError('invalid_argument',
-      `A SmartData write observer requires 1 to ${maximumObservedFields} distinct top-level field names.`);
+      `${subjectArg} requires 1 to ${maximumObservedFields} distinct top-level field names.`);
+  }
+  return Object.freeze([...fieldsArg] as string[]);
+};
+
+const normalizeObserverOptions = (optionsArg: unknown): IWriteObserver => {
+  const { fields, listener } = assertDataOptions(optionsArg, ['fields', 'listener']);
+  if (typeof listener !== 'function') {
+    throw new SmartdataPersistenceError('invalid_argument',
+      'A SmartData write observer requires a listener function.');
   }
   return {
-    fields: Object.freeze([...fields] as string[]),
+    fields: normalizeObservedFields(fields, 'A SmartData write observer'),
     listener: listener as IWriteObserver['listener'],
   };
 };
@@ -195,6 +204,41 @@ const collectSelectorAnchors = (
   }
 };
 
+/**
+ * @internal The report of one acknowledged write with the values of `fieldsArg`,
+ * copied now and frozen.
+ */
+export const reportOfSmartdataWrite = (
+  writeArg: ISmartdataWrite,
+  fieldsArg: readonly string[],
+): ISmartdataWriteReport => {
+  const anchors: Record<string, TSmartdataObservedValue> = {};
+  collectSelectorAnchors(writeArg.selector, new Set(fieldsArg), writeArg.identityDocumentIdField, anchors);
+  return Object.freeze({
+    collection: writeArg.collection,
+    kind: writeArg.kind,
+    documents: writeArg.documents === null
+      ? null
+      : Object.freeze(writeArg.documents.map((documentArg) =>
+        Object.freeze(pickDocumentFields(documentArg, fieldsArg)))),
+    anchors: Object.freeze(anchors),
+  });
+};
+
+/** @internal The same report with only the values of `fieldsArg`, which the report's fields include. */
+export const narrowSmartdataWriteReport = (
+  reportArg: ISmartdataWriteReport,
+  fieldsArg: readonly string[],
+): ISmartdataWriteReport => Object.freeze({
+  collection: reportArg.collection,
+  kind: reportArg.kind,
+  documents: reportArg.documents === null
+    ? null
+    : Object.freeze(reportArg.documents.map((documentArg) =>
+      Object.freeze(pickDocumentFields(documentArg, fieldsArg)))),
+  anchors: Object.freeze(pickDocumentFields(reportArg.anchors, fieldsArg)),
+});
+
 const describeError = (errorArg: unknown): string =>
   errorArg instanceof Error ? errorArg.message : String(errorArg);
 
@@ -212,25 +256,15 @@ export const captureSmartdataWrite = (
   const registered = observersByDb.get(dbArg);
   if (!registered || registered.size === 0) return undefined;
   const observers = [...registered];
-  const fields = [...new Set(observers.flatMap((observerArg) => observerArg.fields))];
-  const documents = writeArg.documents?.map((documentArg) =>
-    pickDocumentFields(documentArg, fields)) ?? null;
-  const anchors: Record<string, TSmartdataObservedValue> = {};
-  collectSelectorAnchors(writeArg.selector, new Set(fields), writeArg.identityDocumentIdField, anchors);
+  const report = reportOfSmartdataWrite(
+    writeArg,
+    [...new Set(observers.flatMap((observerArg) => observerArg.fields))],
+  );
   return () => {
     for (const observer of observers) {
       if (!observersByDb.get(dbArg)?.has(observer)) continue;
-      const report: ISmartdataWriteReport = Object.freeze({
-        collection: writeArg.collection,
-        kind: writeArg.kind,
-        documents: documents === null
-          ? null
-          : Object.freeze(documents.map((documentArg) =>
-            Object.freeze(pickDocumentFields(documentArg, observer.fields)))),
-        anchors: Object.freeze(pickDocumentFields(anchors, observer.fields)),
-      });
       try {
-        observer.listener(report);
+        observer.listener(narrowSmartdataWriteReport(report, observer.fields));
       } catch (errorArg) {
         logger.log('error',
           `A SmartData write observer threw on a write to "${writeArg.collection}": ${describeError(errorArg)}`);

package/ts/nosqldb/classes.writerevisions.ts

@@ -10,6 +10,13 @@ import { SmartdataPersistenceError, isMongoDuplicateKeyError } from './classes.p
  * no writes at all. Every model write bumps a counter per collection and
  * partition in the same transaction, and a read set compares the counters it
  * read before its reads with the current ones.
+ *
+ * A counter is its base document, which a writer bumps alone, and up to
+ * `counterShards - 1` shard documents that overlapping transactions bump
+ * instead, so they do not conflict on the base. A list document beside the
+ * base holds how many slots are in use, and the transaction that first bumps
+ * a slot raises it, so a snapshot reads the bases and lists in one query and
+ * only the listed shards of the counters that have any.
  */
 export interface ISmartdataWriteRevisionsOptions {
   /**
@@ -28,10 +35,21 @@ interface IWriteRevisionDocument {
   _id: string;
   n: number;
   generation?: string;
+  /** On a counter's list: the slots in use, the base included. */
+  slots?: number;
 }
 
 /** The counter every write revision belongs to; a new generation restarts all counters. */
 const epochId = '$epoch';
+/** Joins a counter's id and the slot of one of its shards, or ends the id of its list. No counter id contains it. */
+const shardSeparator = '\u0000';
+/**
+ * Slots per counter, the base included: as many transactions that overlap in
+ * time can bump one counter without conflicting on it.
+ */
+const counterShards = 16;
+/** Counters whose slots a database instance remembers, beyond those an open transaction holds. */
+const maximumRememberedCounters = 4096;
 const maximumPartitionLength = 512;
 const maximumPinnedPartitions = 64;
 /** Beyond this many documents, a write by selector bumps the shared counter instead. */
@@ -68,7 +86,10 @@ export const isSmartdataWriteRevisionsActive = (dbArg: SmartdataDb): boolean =>
 
 /** A string the counters can name, or `undefined`. */
 const partitionOf = (valueArg: unknown): string | undefined =>
-  typeof valueArg === 'string' && valueArg.length <= maximumPartitionLength && valueArg.isWellFormed()
+  typeof valueArg === 'string'
+  && valueArg.length <= maximumPartitionLength
+  && valueArg.isWellFormed()
+  && !valueArg.includes(shardSeparator)
     ? valueArg
     : undefined;
 
@@ -79,6 +100,13 @@ export const partitionRevisionId = (collectionArg: string, partitionArg: string)
 /** @internal The counter of a collection's writes that no partition can place. */
 export const sharedRevisionId = (collectionArg: string): string => `${collectionArg}$*`;
 
+/** @internal The document of a counter's slot: the base for slot 0, a shard otherwise. */
+export const revisionSlotId = (idArg: string, slotArg: number): string =>
+  slotArg === 0 ? idArg : `${idArg}${shardSeparator}${slotArg}`;
+
+/** @internal The document that lists how many slots of a counter are in use; none means only the base. */
+export const revisionSlotListId = (idArg: string): string => `${idArg}${shardSeparator}`;
+
 /**
  * @internal The counter a stored value of the partition field belongs to:
  * its partition's for a string the counters can name, the shared one otherwise.
@@ -177,17 +205,120 @@ export const writeRevisionBumpsOfKey = (keyArg: string, idsArg: ReadonlySet<stri
   return bumps;
 };
 
-export const beginSmartdataWriteRevisionCommit = (
-  dbArg: SmartdataDb,
-  idsArg: ReadonlySet<string>,
-): (() => void) => {
-  const commits = revisionCommitsOfDb(dbArg);
-  const ids = new Set(idsArg);
-  commits.pending.add(ids);
-  for (const snapshot of commits.snapshots) {
-    if (snapshot.keys.some((keyArg) => writeRevisionBumpsOfKey(keyArg, ids) > 0)) snapshot.lost = true;
+interface IRevisionSlot {
+  /** Attempts of this instance that bump the slot and have not ended yet. */
+  holders: number;
+  /** When the last such attempt ended, on the `performance.now()` clock. */
+  endedAt: number;
+}
+
+interface IRevisionCounterSlots {
+  /** The slot this instance's last committed bump took; a search for a free slot starts there. */
+  preferred: number;
+  slots: IRevisionSlot[];
+}
+
+/** Per database instance, in least recently bumped order. */
+const revisionSlotsOfDbs = new WeakMap<SmartdataDb, Map<string, IRevisionCounterSlots>>();
+
+/**
+ * The slot of a counter an attempt that started at `attemptStartedAtArg`
+ * bumps: from the preferred one on, the first that no attempt of this instance
+ * holds or ended since then, whose commit the attempt's snapshot might not
+ * see. Without one, the first that none holds, and the preferred one last.
+ * Transactions of other instances may still take the same slot; a conflict
+ * then fails the attempt, and its retry starts after the slot's end, so it
+ * moves on to another.
+ */
+const chooseRevisionSlot = (counterArg: IRevisionCounterSlots, attemptStartedAtArg: number): number => {
+  let unheld: number | undefined;
+  for (let offset = 0; offset < counterShards; offset++) {
+    const slot = (counterArg.preferred + offset) % counterShards;
+    const state = counterArg.slots[slot];
+    if (state.holders > 0) continue;
+    if (state.endedAt < attemptStartedAtArg) return slot;
+    unheld ??= slot;
   }
-  return () => { commits.pending.delete(ids); };
+  return unheld ?? counterArg.preferred;
+};
+
+/**
+ * @internal The slots one attempt of a transaction bumps. A counter takes its
+ * slot at its first bump in the attempt and holds it until the attempt ends,
+ * so no other attempt of this instance commits to it in between.
+ */
+export interface ISmartdataWriteRevisionAttempt {
+  /** The slot of each counter the attempt bumps. */
+  readonly slots: ReadonlyMap<string, number>;
+  /** Takes a slot of a counter the attempt has not bumped yet; once the attempt ended, none. */
+  bump(idArg: string): void;
+  /**
+   * Right before the commit: until the attempt ends, a snapshot of this
+   * instance that depends on its counters cannot tell its bumps from those of
+   * other processes.
+   */
+  commit(): void;
+  /**
+   * Ends the attempt: releases its slots and, after a commit, makes them the
+   * counters' preferred ones. Later calls do nothing.
+   */
+  finish(committedArg: boolean): void;
+}
+
+export const beginSmartdataWriteRevisionAttempt = (
+  dbArg: SmartdataDb,
+  attemptStartedAtArg: number,
+): ISmartdataWriteRevisionAttempt => {
+  const taken: [IRevisionCounterSlots, number][] = [];
+  const slots = new Map<string, number>();
+  let pending: ReadonlySet<string> | undefined;
+  let finished = false;
+  return {
+    slots,
+    bump: (idArg) => {
+      if (finished || slots.has(idArg)) return;
+      let counters = revisionSlotsOfDbs.get(dbArg);
+      if (!counters) {
+        counters = new Map();
+        revisionSlotsOfDbs.set(dbArg, counters);
+      }
+      const counter = counters.get(idArg) ?? {
+        preferred: 0,
+        slots: Array.from({ length: counterShards }, () => ({ holders: 0, endedAt: -Infinity })),
+      };
+      counters.delete(idArg);
+      counters.set(idArg, counter);
+      const slot = chooseRevisionSlot(counter, attemptStartedAtArg);
+      counter.slots[slot].holders++;
+      taken.push([counter, slot]);
+      slots.set(idArg, slot);
+      for (const [id, remembered] of counters) {
+        if (counters.size <= maximumRememberedCounters) break;
+        if (remembered.slots.every((slotArg) => slotArg.holders === 0)) counters.delete(id);
+      }
+    },
+    commit: () => {
+      if (finished || pending) return;
+      const commits = revisionCommitsOfDb(dbArg);
+      const ids = new Set(slots.keys());
+      pending = ids;
+      commits.pending.add(ids);
+      for (const snapshot of commits.snapshots) {
+        if (snapshot.keys.some((keyArg) => writeRevisionBumpsOfKey(keyArg, ids) > 0)) snapshot.lost = true;
+      }
+    },
+    finish: (committedArg) => {
+      if (finished) return;
+      finished = true;
+      if (pending) revisionCommitsOfDb(dbArg).pending.delete(pending);
+      const endedAt = performance.now();
+      for (const [counter, slot] of taken) {
+        counter.slots[slot].holders--;
+        counter.slots[slot].endedAt = endedAt;
+        if (committedArg) counter.preferred = slot;
+      }
+    },
+  };
 };
 
 /**
@@ -227,8 +358,12 @@ const connectedDb = (dbArg: SmartdataDb): plugins.mongodb.Db => {
 
 /**
  * @internal The current value of each snapshot key, or `undefined` for a key
- * whose value cannot be known: a missing epoch means the counters restarted.
- * Bounded by its own deadline.
+ * whose value cannot be known: a missing epoch means the counters restarted,
+ * and a counter with a value or slot count no writer stores stays unknown. One
+ * query reads the epoch, the bases and their lists, a second the listed
+ * shards. Every part only grows, and a slot is listed by the transaction that
+ * first bumps it, so a write that commits between the two queries shows in
+ * this snapshot or in the next, never in neither. Bounded by its own deadline.
  */
 const readStoredWriteRevisionSnapshot = async (
   dbArg: SmartdataDb,
@@ -242,36 +377,59 @@ const readStoredWriteRevisionSnapshot = async (
     timeoutMode: 'cursorLifetime' as const,
     signal,
   });
+  const readDocuments = async (idsArg: string[]) => new Map((await collection.find(
+    { _id: { $in: idsArg } },
+    { projection: { n: 1, generation: 1, slots: 1 }, ...options() },
+  ).toArray()).map((documentArg) => [documentArg._id, documentArg]));
+  // `$sum` skips a value that is not a number, so such values are counted apart.
+  const sumOf = async (collectionArg: string): Promise<string | undefined> => {
+    const [sum] = await collection.aggregate<{ n: unknown; invalid: unknown }>([
+      { $match: { _id: { $gte: `${collectionArg}$`, $lt: `${collectionArg}%` } } },
+      { $group: { _id: null, n: { $sum: '$n' }, invalid: { $sum: { $cond: [{ $isNumber: '$n' }, 0, 1] } } } },
+    ], options()).toArray();
+    if (sum === undefined) return '0';
+    return sum.invalid === 0 && typeof sum.n === 'number' && Number.isSafeInteger(sum.n)
+      ? String(sum.n)
+      : undefined;
+  };
   const counterIds = keysArg.flatMap((keyArg) => keyArg.startsWith('c') ? [keyArg.slice(1)] : []);
   const ranges = keysArg.flatMap((keyArg) => keyArg.startsWith('r') ? [keyArg.slice(1)] : []);
-  const [counters, ...sums] = await Promise.all([
-    collection.find(
-      { _id: { $in: [epochId, ...counterIds] } },
-      { projection: { n: 1, generation: 1 }, ...options() },
-    ).toArray(),
-    ...ranges.map((collectionArg) => collection.aggregate<{ n: unknown }>([
-      { $match: { _id: { $gte: `${collectionArg}$`, $lt: `${collectionArg}%` } } },
-      { $group: { _id: null, n: { $sum: '$n' } } },
-    ], options()).toArray()),
+  const [bases, ...sums] = await Promise.all([
+    readDocuments([epochId, ...counterIds.flatMap((idArg) => [idArg, revisionSlotListId(idArg)])]),
+    ...ranges.map(sumOf),
   ]);
-  const byId = new Map(counters.map((documentArg) => [documentArg._id, documentArg]));
+  /** The slots each counter's list holds, or `undefined` for a count no writer stores. */
+  const slotCounts = new Map(counterIds.map((idArg): [string, number | undefined] => {
+    const list = bases.get(revisionSlotListId(idArg));
+    const count: unknown = list === undefined ? 1 : list.slots;
+    return [idArg, Number.isSafeInteger(count) && (count as number) >= 1 && (count as number) <= counterShards
+      ? count as number
+      : undefined];
+  }));
+  const shardIds = counterIds.flatMap((idArg) => Array.from(
+    { length: (slotCounts.get(idArg) ?? 1) - 1 },
+    (_unusedArg, indexArg) => revisionSlotId(idArg, indexArg + 1),
+  ));
+  const shards = shardIds.length === 0 ? new Map<string, IWriteRevisionDocument>() : await readDocuments(shardIds);
   const snapshot = new Map<string, string | undefined>();
-  const epoch = byId.get(epochId);
+  const epoch = bases.get(epochId);
   snapshot.set(epochKey, typeof epoch?.generation === 'string' && Number.isSafeInteger(epoch.n)
     ? `${epoch.generation}:${epoch.n}`
     : undefined);
   for (const id of counterIds) {
-    const counter = byId.get(id);
-    snapshot.set(counterKey(id), counter === undefined
-      ? '0'
-      : Number.isSafeInteger(counter.n) ? String(counter.n) : undefined);
-  }
-  for (const [index, collectionArg] of ranges.entries()) {
-    const sum = sums[index].length === 0 ? 0 : sums[index][0].n;
-    snapshot.set(rangeKey(collectionArg), typeof sum === 'number' && Number.isSafeInteger(sum)
-      ? String(sum)
-      : undefined);
+    const count = slotCounts.get(id);
+    let sum: number | undefined = count === undefined ? undefined : 0;
+    for (let slot = 0; sum !== undefined && slot < count!; slot++) {
+      // A slot is missing until its first bump commits; a later one may commit first.
+      const document = slot === 0 ? bases.get(id) : shards.get(revisionSlotId(id, slot));
+      if (document === undefined) continue;
+      sum = Number.isSafeInteger(document.n) && Number.isSafeInteger(sum + document.n)
+        ? sum + document.n
+        : undefined;
+    }
+    snapshot.set(counterKey(id), sum === undefined ? undefined : String(sum));
   }
+  for (const [index, collectionName] of ranges.entries()) snapshot.set(rangeKey(collectionName), sums[index]);
   return snapshot;
 };
 
@@ -342,19 +500,45 @@ export const bumpSmartdataWriteRevisionEpoch = async (
 
 /**
  * @internal Bumps the counters a transaction's writes touched, inside that
- * transaction, right before its commit. Sorted, so concurrent transactions
- * take the counters in one order.
+ * transaction, right before its commit: each in the slot it took. A shard
+ * bump first makes sure the counter's list exists and holds the slot; once it
+ * does, neither update changes it, so later bumps of listed slots write only
+ * their own document. A slot whose value is no number stays as it is, which
+ * keeps its counter unknown, and the write still commits.
  */
 export const flushSmartdataWriteRevisions = async (
   mongoDbArg: plugins.mongodb.Db,
   rawSessionArg: plugins.mongodb.ClientSession,
-  idsArg: ReadonlySet<string>,
+  slotsArg: ReadonlyMap<string, number>,
   signalArg: AbortSignal | undefined,
 ): Promise<void> => {
-  if (idsArg.size === 0) return;
-  await revisionsCollection(mongoDbArg).bulkWrite([...idsArg].sort().map((idArg) => ({
-    updateOne: { filter: { _id: idArg }, update: { $inc: { n: 1 } }, upsert: true },
-  })), {
+  if (slotsArg.size === 0) return;
+  const operations: plugins.mongodb.AnyBulkWriteOperation<IWriteRevisionDocument>[] = [];
+  for (const [id, slot] of [...slotsArg].sort(([aArg], [bArg]) => aArg < bArg ? -1 : aArg > bArg ? 1 : 0)) {
+    if (slot > 0) {
+      const listId = revisionSlotListId(id);
+      operations.push(
+        { updateOne: { filter: { _id: listId }, update: { $setOnInsert: { n: 0 } }, upsert: true } },
+        {
+          updateOne: {
+            filter: { _id: listId, $or: [{ slots: { $exists: false } }, { slots: { $lt: slot + 1 } }] },
+            update: { $set: { slots: slot + 1 } },
+          },
+        },
+      );
+    }
+    const slotId = revisionSlotId(id, slot);
+    operations.push(
+      {
+        updateOne: {
+          filter: { _id: slotId, $or: [{ n: { $type: 'int' } }, { n: { $type: 'long' } }, { n: { $type: 'double' } }] },
+          update: { $inc: { n: 1 } },
+        },
+      },
+      { updateOne: { filter: { _id: slotId }, update: { $setOnInsert: { n: 1 } }, upsert: true } },
+    );
+  }
+  await revisionsCollection(mongoDbArg).bulkWrite(operations, {
     ordered: true,
     session: rawSessionArg,
     ...(signalArg === undefined ? {} : { signal: signalArg }),

package/ts/nosqldb/engineidentity.ts

@@ -32,6 +32,12 @@ export const unidentifiedSmartdataEngine: ISmartdataEngineIdentity = Object.free
  */
 const compositeGroupingEngineRelease = '10.5.0';
 
+/**
+ * The NoSQLDB release that publishes its durable commit journal through the
+ * `getCommittedWrites` command: @lossless.org/nosqldb 10.9.0.
+ */
+const commitJournalEngineRelease = '10.9.0';
+
 const parseEngineVersion = (versionArg: string): readonly number[] | undefined => {
   const parts = versionArg.split('.');
   if (parts.length !== 3) {
@@ -91,6 +97,16 @@ export const smartdataCompositeGrouping = (
   }
 };
 
+/**
+ * Whether the identified engine publishes a durable, commit-ordered journal of
+ * the transactions it committed. Only NoSQLDB does, from 10.9.0 on; MongoDB's
+ * change streams are a different contract and are not used for it.
+ */
+export const smartdataCommitJournal = (identityArg: ISmartdataEngineIdentity): boolean =>
+  identityArg.product === 'NoSQLDB'
+  && identityArg.version !== null
+  && engineVersionReaches(identityArg.version, commitJournalEngineRelease);
+
 export const composeSmartdataCapabilities = (
   backendArg: TSmartdataEngineBackend,
   compositeGroupingArg: TCompositeGrouping,