@lossless.org/client 1.22.1 → 1.23.0

package/ts/nosqldb/classes.writeobserver.ts

@@ -77,28 +77,37 @@ interface IWriteObserver {
 const maximumObservedFields = 64;
 const observersByDb = new WeakMap<SmartdataDb, Set<IWriteObserver>>();
 
-const normalizeObserverOptions = (optionsArg: unknown): IWriteObserver => {
-  const { fields, listener } = assertDataOptions(optionsArg, ['fields', 'listener']);
-  if (typeof listener !== 'function') {
-    throw new SmartdataPersistenceError('invalid_argument',
-      'A SmartData write observer requires a listener function.');
-  }
+/**
+ * @internal Validates the top-level field names a report carries: 1 to 64
+ * distinct names, no `.` and no leading `$`. `subjectArg` names what requires
+ * them in the refusal.
+ */
+export const normalizeObservedFields = (fieldsArg: unknown, subjectArg: string): readonly string[] => {
   if (
-    !Array.isArray(fields)
-    || fields.length === 0
-    || fields.length > maximumObservedFields
-    || fields.some((fieldArg) =>
+    !Array.isArray(fieldsArg)
+    || fieldsArg.length === 0
+    || fieldsArg.length > maximumObservedFields
+    || fieldsArg.some((fieldArg) =>
       typeof fieldArg !== 'string'
       || fieldArg.length === 0
       || fieldArg.includes('.')
       || fieldArg.startsWith('$'))
-    || new Set(fields).size !== fields.length
+    || new Set(fieldsArg).size !== fieldsArg.length
   ) {
     throw new SmartdataPersistenceError('invalid_argument',
-      `A SmartData write observer requires 1 to ${maximumObservedFields} distinct top-level field names.`);
+      `${subjectArg} requires 1 to ${maximumObservedFields} distinct top-level field names.`);
+  }
+  return Object.freeze([...fieldsArg] as string[]);
+};
+
+const normalizeObserverOptions = (optionsArg: unknown): IWriteObserver => {
+  const { fields, listener } = assertDataOptions(optionsArg, ['fields', 'listener']);
+  if (typeof listener !== 'function') {
+    throw new SmartdataPersistenceError('invalid_argument',
+      'A SmartData write observer requires a listener function.');
   }
   return {
-    fields: Object.freeze([...fields] as string[]),
+    fields: normalizeObservedFields(fields, 'A SmartData write observer'),
     listener: listener as IWriteObserver['listener'],
   };
 };
@@ -195,6 +204,41 @@ const collectSelectorAnchors = (
   }
 };
 
+/**
+ * @internal The report of one acknowledged write with the values of `fieldsArg`,
+ * copied now and frozen.
+ */
+export const reportOfSmartdataWrite = (
+  writeArg: ISmartdataWrite,
+  fieldsArg: readonly string[],
+): ISmartdataWriteReport => {
+  const anchors: Record<string, TSmartdataObservedValue> = {};
+  collectSelectorAnchors(writeArg.selector, new Set(fieldsArg), writeArg.identityDocumentIdField, anchors);
+  return Object.freeze({
+    collection: writeArg.collection,
+    kind: writeArg.kind,
+    documents: writeArg.documents === null
+      ? null
+      : Object.freeze(writeArg.documents.map((documentArg) =>
+        Object.freeze(pickDocumentFields(documentArg, fieldsArg)))),
+    anchors: Object.freeze(anchors),
+  });
+};
+
+/** @internal The same report with only the values of `fieldsArg`, which the report's fields include. */
+export const narrowSmartdataWriteReport = (
+  reportArg: ISmartdataWriteReport,
+  fieldsArg: readonly string[],
+): ISmartdataWriteReport => Object.freeze({
+  collection: reportArg.collection,
+  kind: reportArg.kind,
+  documents: reportArg.documents === null
+    ? null
+    : Object.freeze(reportArg.documents.map((documentArg) =>
+      Object.freeze(pickDocumentFields(documentArg, fieldsArg)))),
+  anchors: Object.freeze(pickDocumentFields(reportArg.anchors, fieldsArg)),
+});
+
 const describeError = (errorArg: unknown): string =>
   errorArg instanceof Error ? errorArg.message : String(errorArg);
 
@@ -212,25 +256,15 @@ export const captureSmartdataWrite = (
   const registered = observersByDb.get(dbArg);
   if (!registered || registered.size === 0) return undefined;
   const observers = [...registered];
-  const fields = [...new Set(observers.flatMap((observerArg) => observerArg.fields))];
-  const documents = writeArg.documents?.map((documentArg) =>
-    pickDocumentFields(documentArg, fields)) ?? null;
-  const anchors: Record<string, TSmartdataObservedValue> = {};
-  collectSelectorAnchors(writeArg.selector, new Set(fields), writeArg.identityDocumentIdField, anchors);
+  const report = reportOfSmartdataWrite(
+    writeArg,
+    [...new Set(observers.flatMap((observerArg) => observerArg.fields))],
+  );
   return () => {
     for (const observer of observers) {
       if (!observersByDb.get(dbArg)?.has(observer)) continue;
-      const report: ISmartdataWriteReport = Object.freeze({
-        collection: writeArg.collection,
-        kind: writeArg.kind,
-        documents: documents === null
-          ? null
-          : Object.freeze(documents.map((documentArg) =>
-            Object.freeze(pickDocumentFields(documentArg, observer.fields)))),
-        anchors: Object.freeze(pickDocumentFields(anchors, observer.fields)),
-      });
       try {
-        observer.listener(report);
+        observer.listener(narrowSmartdataWriteReport(report, observer.fields));
       } catch (errorArg) {
         logger.log('error',
           `A SmartData write observer threw on a write to "${writeArg.collection}": ${describeError(errorArg)}`);

package/ts/nosqldb/classes.writerevisions.ts

@@ -4,12 +4,18 @@ import type { SmartdataDb } from './classes.db.js';
 import { assertDataOptions } from './classes.operationbudget.js';
 import type { SmartdataOperationDeadline } from './classes.operationdeadline.js';
 import { SmartdataPersistenceError, isMongoDuplicateKeyError } from './classes.persistence.js';
+import { logger } from './logging.js';
 
 /**
  * Write revisions let a read set tell writes of other processes apart from
  * no writes at all. Every model write bumps a counter per collection and
  * partition in the same transaction, and a read set compares the counters it
  * read before its reads with the current ones.
+ *
+ * A bump inserts an entry of its own instead of incrementing a shared
+ * document, so concurrent transactions of one partition never conflict on its
+ * counter. A counter's value is its base document plus its entries; a
+ * background compaction folds entries into the base, which keeps the sum.
  */
 export interface ISmartdataWriteRevisionsOptions {
   /**
@@ -32,6 +38,17 @@ interface IWriteRevisionDocument {
 
 /** The counter every write revision belongs to; a new generation restarts all counters. */
 const epochId = '$epoch';
+/**
+ * Joins a counter's id and the suffix of one of its entries. No counter id
+ * contains it, so a counter's base and entries are exactly the ids from the
+ * counter's own up to, not including, the id followed by `entryRangeEnd`.
+ */
+const entrySeparator = '\u0000';
+const entryRangeEnd = '\u0001';
+/** Entries one process adds to a counter before it folds the counter's entries into its base. */
+const compactionEntries = 64;
+const maximumCompactedEntries = 1024;
+const compactionTimeoutMs = 5_000;
 const maximumPartitionLength = 512;
 const maximumPinnedPartitions = 64;
 /** Beyond this many documents, a write by selector bumps the shared counter instead. */
@@ -68,7 +85,10 @@ export const isSmartdataWriteRevisionsActive = (dbArg: SmartdataDb): boolean =>
 
 /** A string the counters can name, or `undefined`. */
 const partitionOf = (valueArg: unknown): string | undefined =>
-  typeof valueArg === 'string' && valueArg.length <= maximumPartitionLength && valueArg.isWellFormed()
+  typeof valueArg === 'string'
+  && valueArg.length <= maximumPartitionLength
+  && valueArg.isWellFormed()
+  && !valueArg.includes(entrySeparator)
     ? valueArg
     : undefined;
 
@@ -145,6 +165,51 @@ const epochKey = 'e';
 const counterKey = (idArg: string): string => `c${idArg}`;
 const rangeKey = (collectionArg: string): string => `r${collectionArg}`;
 
+interface ILocalRevisionSnapshot {
+  keys: readonly string[];
+  lost: boolean;
+}
+
+interface ILocalRevisionCommits {
+  pending: Set<ReadonlySet<string>>;
+  snapshots: Set<ILocalRevisionSnapshot>;
+}
+
+const localRevisionCommits = new WeakMap<SmartdataDb, ILocalRevisionCommits>();
+
+const revisionCommitsOfDb = (dbArg: SmartdataDb): ILocalRevisionCommits => {
+  let commits = localRevisionCommits.get(dbArg);
+  if (!commits) {
+    commits = { pending: new Set(), snapshots: new Set() };
+    localRevisionCommits.set(dbArg, commits);
+  }
+  return commits;
+};
+
+export const writeRevisionBumpsOfKey = (keyArg: string, idsArg: ReadonlySet<string>): number => {
+  if (keyArg.startsWith('c')) return idsArg.has(keyArg.slice(1)) ? 1 : 0;
+  if (!keyArg.startsWith('r')) return 0;
+  const collection = keyArg.slice(1);
+  let bumps = 0;
+  for (const id of idsArg) {
+    if (id >= `${collection}$` && id < `${collection}%`) bumps++;
+  }
+  return bumps;
+};
+
+export const beginSmartdataWriteRevisionCommit = (
+  dbArg: SmartdataDb,
+  idsArg: ReadonlySet<string>,
+): (() => void) => {
+  const commits = revisionCommitsOfDb(dbArg);
+  const ids = new Set(idsArg);
+  commits.pending.add(ids);
+  for (const snapshot of commits.snapshots) {
+    if (snapshot.keys.some((keyArg) => writeRevisionBumpsOfKey(keyArg, ids) > 0)) snapshot.lost = true;
+  }
+  return () => { commits.pending.delete(ids); };
+};
+
 /**
  * @internal The snapshot keys one read depends on. A read the field pins
  * depends on its partitions and the shared counter, which holds every
@@ -183,9 +248,10 @@ const connectedDb = (dbArg: SmartdataDb): plugins.mongodb.Db => {
 /**
  * @internal The current value of each snapshot key, or `undefined` for a key
  * whose value cannot be known: a missing epoch means the counters restarted.
- * Bounded by its own deadline.
+ * Every counter and collection sum is one read, so a compaction that moves
+ * entries into a base never shows half done. Bounded by its own deadline.
  */
-export const readWriteRevisionSnapshot = async (
+const readStoredWriteRevisionSnapshot = async (
   dbArg: SmartdataDb,
   keysArg: readonly string[],
 ): Promise<Map<string, string | undefined>> => {
@@ -197,39 +263,61 @@ export const readWriteRevisionSnapshot = async (
     timeoutMode: 'cursorLifetime' as const,
     signal,
   });
-  const counterIds = keysArg.flatMap((keyArg) => keyArg.startsWith('c') ? [keyArg.slice(1)] : []);
-  const ranges = keysArg.flatMap((keyArg) => keyArg.startsWith('r') ? [keyArg.slice(1)] : []);
-  const [counters, ...sums] = await Promise.all([
-    collection.find(
-      { _id: { $in: [epochId, ...counterIds] } },
-      { projection: { n: 1, generation: 1 }, ...options() },
-    ).toArray(),
-    ...ranges.map((collectionArg) => collection.aggregate<{ n: unknown }>([
-      { $match: { _id: { $gte: `${collectionArg}$`, $lt: `${collectionArg}%` } } },
-      { $group: { _id: null, n: { $sum: '$n' } } },
-    ], options()).toArray()),
+  // `$sum` skips a value that is not a number, so such values are counted apart.
+  const sumOf = async (fromArg: string, toArg: string): Promise<string | undefined> => {
+    const [sum] = await collection.aggregate<{ n: unknown; invalid: unknown }>([
+      { $match: { _id: { $gte: fromArg, $lt: toArg } } },
+      { $group: { _id: null, n: { $sum: '$n' }, invalid: { $sum: { $cond: [{ $isNumber: '$n' }, 0, 1] } } } },
+    ], options()).toArray();
+    if (sum === undefined) return '0';
+    return sum.invalid === 0 && typeof sum.n === 'number' && Number.isSafeInteger(sum.n)
+      ? String(sum.n)
+      : undefined;
+  };
+  const sums = keysArg.flatMap((keyArg): [string, Promise<string | undefined>][] => {
+    if (keyArg.startsWith('c')) {
+      const id = keyArg.slice(1);
+      return [[keyArg, sumOf(id, `${id}${entryRangeEnd}`)]];
+    }
+    if (keyArg.startsWith('r')) {
+      const collectionName = keyArg.slice(1);
+      return [[keyArg, sumOf(`${collectionName}$`, `${collectionName}%`)]];
+    }
+    return [];
+  });
+  const [epoch, ...values] = await Promise.all([
+    collection.findOne({ _id: epochId }, { projection: { n: 1, generation: 1 }, ...options() }),
+    ...sums.map(([, sumArg]) => sumArg),
   ]);
-  const byId = new Map(counters.map((documentArg) => [documentArg._id, documentArg]));
   const snapshot = new Map<string, string | undefined>();
-  const epoch = byId.get(epochId);
   snapshot.set(epochKey, typeof epoch?.generation === 'string' && Number.isSafeInteger(epoch.n)
     ? `${epoch.generation}:${epoch.n}`
     : undefined);
-  for (const id of counterIds) {
-    const counter = byId.get(id);
-    snapshot.set(counterKey(id), counter === undefined
-      ? '0'
-      : Number.isSafeInteger(counter.n) ? String(counter.n) : undefined);
-  }
-  for (const [index, collectionArg] of ranges.entries()) {
-    const sum = sums[index].length === 0 ? 0 : sums[index][0].n;
-    snapshot.set(rangeKey(collectionArg), typeof sum === 'number' && Number.isSafeInteger(sum)
-      ? String(sum)
-      : undefined);
-  }
+  for (const [index, [key]] of sums.entries()) snapshot.set(key, values[index]);
   return snapshot;
 };
 
+export const readWriteRevisionSnapshot = async (
+  dbArg: SmartdataDb,
+  keysArg: readonly string[],
+): Promise<Map<string, string | undefined>> => {
+  const commits = revisionCommitsOfDb(dbArg);
+  const snapshot: ILocalRevisionSnapshot = {
+    keys: keysArg,
+    lost: [...commits.pending].some((idsArg) =>
+      keysArg.some((keyArg) => writeRevisionBumpsOfKey(keyArg, idsArg) > 0)),
+  };
+  commits.snapshots.add(snapshot);
+  try {
+    const stored = await readStoredWriteRevisionSnapshot(dbArg, keysArg);
+    return snapshot.lost
+      ? new Map(keysArg.map((keyArg) => [keyArg, undefined]))
+      : stored;
+  } finally {
+    commits.snapshots.delete(snapshot);
+  }
+};
+
 /**
  * @internal Creates the epoch of a database that has none, before `init()`
  * publishes the connection. Another process creating it at the same time is
@@ -276,8 +364,8 @@ export const bumpSmartdataWriteRevisionEpoch = async (
 
 /**
  * @internal Bumps the counters a transaction's writes touched, inside that
- * transaction, right before its commit. Sorted, so concurrent transactions
- * take the counters in one order.
+ * transaction, right before its commit: one new entry per counter, which no
+ * concurrent transaction writes.
  */
 export const flushSmartdataWriteRevisions = async (
   mongoDbArg: plugins.mongodb.Db,
@@ -286,8 +374,9 @@ export const flushSmartdataWriteRevisions = async (
   signalArg: AbortSignal | undefined,
 ): Promise<void> => {
   if (idsArg.size === 0) return;
-  await revisionsCollection(mongoDbArg).bulkWrite([...idsArg].sort().map((idArg) => ({
-    updateOne: { filter: { _id: idArg }, update: { $inc: { n: 1 } }, upsert: true },
+  await revisionsCollection(mongoDbArg).insertMany([...idsArg].map((idArg) => ({
+    _id: `${idArg}${entrySeparator}${plugins.nodeCrypto.randomUUID()}`,
+    n: 1,
   })), {
     ordered: true,
     session: rawSessionArg,
@@ -295,6 +384,109 @@ export const flushSmartdataWriteRevisions = async (
   });
 };
 
+interface IWriteRevisionCompactions {
+  /** Committed entries this process added per counter since it last compacted it. */
+  entries: Map<string, number>;
+  running: Map<string, Promise<void>>;
+  controller: AbortController;
+}
+
+const revisionCompactions = new WeakMap<SmartdataDb, IWriteRevisionCompactions>();
+
+/**
+ * Folds up to `maximumCompactedEntries` entries of one counter into its base
+ * in one transaction, which keeps the counter's value. A concurrent compaction
+ * of the same counter conflicts on the base and on the entries, and the driver
+ * retries it within the deadline; one that still fails leaves the entries for
+ * the next.
+ */
+const compactWriteRevisionEntries = async (
+  dbArg: SmartdataDb,
+  idArg: string,
+  signalArg: AbortSignal,
+): Promise<void> => {
+  const collection = revisionsCollection(connectedDb(dbArg));
+  const abortable: plugins.mongodb.Abortable = { signal: signalArg };
+  const session = dbArg.mongoDbClient.startSession();
+  try {
+    await session.withTransaction(async () => {
+      const entries = await collection.find(
+        { _id: { $gt: `${idArg}${entrySeparator}`, $lt: `${idArg}${entryRangeEnd}` } },
+        { projection: { n: 1 }, limit: maximumCompactedEntries, session, ...abortable },
+      ).toArray();
+      if (entries.length === 0) return;
+      let sum = 0;
+      for (const entry of entries) {
+        if (!Number.isSafeInteger(entry.n)) {
+          throw new SmartdataPersistenceError('invalid_document',
+            `Write-revision entry "${entry._id}" holds no safe integer.`);
+        }
+        sum += entry.n;
+        if (!Number.isSafeInteger(sum)) {
+          throw new SmartdataPersistenceError('invalid_document',
+            `Write-revision entries of "${idArg}" do not sum to a safe integer.`);
+        }
+      }
+      await collection.deleteMany(
+        { _id: { $in: entries.map((entryArg) => entryArg._id) } },
+        { session, ...abortable },
+      );
+      await collection.updateOne(
+        { _id: idArg },
+        { $inc: { n: sum } },
+        { upsert: true, session, ...abortable },
+      );
+    }, { timeoutMS: compactionTimeoutMs });
+  } finally {
+    await session.endSession();
+  }
+};
+
+/**
+ * @internal Counts the entries a committed transaction of this process added,
+ * and starts a compaction of each counter that reached `compactionEntries`.
+ * Compactions run in the background; `stopSmartdataWriteRevisionCompactions`
+ * ends them.
+ */
+export const noteSmartdataWriteRevisionEntries = (
+  dbArg: SmartdataDb,
+  idsArg: ReadonlySet<string>,
+): void => {
+  if (idsArg.size === 0 || dbArg.status !== 'connected' || !isSmartdataWriteRevisionsActive(dbArg)) return;
+  let state = revisionCompactions.get(dbArg);
+  if (!state) {
+    state = { entries: new Map(), running: new Map(), controller: new AbortController() };
+    revisionCompactions.set(dbArg, state);
+  }
+  const compactions = state;
+  for (const id of idsArg) {
+    const entries = (compactions.entries.get(id) ?? 0) + 1;
+    if (entries < compactionEntries || compactions.running.has(id)) {
+      compactions.entries.set(id, entries);
+      continue;
+    }
+    compactions.entries.delete(id);
+    const signal = compactions.controller.signal;
+    const compaction = compactWriteRevisionEntries(dbArg, id, signal)
+      .catch((errorArg) => {
+        if (signal.aborted) return;
+        logger.log('warn', `Compacting the write revisions of "${id}" failed; a later compaction retries: ${
+          errorArg instanceof Error ? errorArg.message : String(errorArg)}`);
+      })
+      .finally(() => { compactions.running.delete(id); });
+    compactions.running.set(id, compaction);
+  }
+};
+
+/** @internal Aborts the database's compactions and waits until they settled; `close()` calls it. */
+export const stopSmartdataWriteRevisionCompactions = async (dbArg: SmartdataDb): Promise<void> => {
+  const state = revisionCompactions.get(dbArg);
+  if (!state) return;
+  revisionCompactions.delete(dbArg);
+  state.controller.abort();
+  await Promise.allSettled(state.running.values());
+};
+
 /**
  * @internal The counters of the documents a write by selector is about to
  * change, read in its transaction before it runs. Snapshot isolation makes

package/ts/nosqldb/engineidentity.ts

@@ -32,6 +32,12 @@ export const unidentifiedSmartdataEngine: ISmartdataEngineIdentity = Object.free
  */
 const compositeGroupingEngineRelease = '10.5.0';
 
+/**
+ * The NoSQLDB release that publishes its durable commit journal through the
+ * `getCommittedWrites` command: @lossless.org/nosqldb 10.9.0.
+ */
+const commitJournalEngineRelease = '10.9.0';
+
 const parseEngineVersion = (versionArg: string): readonly number[] | undefined => {
   const parts = versionArg.split('.');
   if (parts.length !== 3) {
@@ -91,6 +97,16 @@ export const smartdataCompositeGrouping = (
   }
 };
 
+/**
+ * Whether the identified engine publishes a durable, commit-ordered journal of
+ * the transactions it committed. Only NoSQLDB does, from 10.9.0 on; MongoDB's
+ * change streams are a different contract and are not used for it.
+ */
+export const smartdataCommitJournal = (identityArg: ISmartdataEngineIdentity): boolean =>
+  identityArg.product === 'NoSQLDB'
+  && identityArg.version !== null
+  && engineVersionReaches(identityArg.version, commitJournalEngineRelease);
+
 export const composeSmartdataCapabilities = (
   backendArg: TSmartdataEngineBackend,
   compositeGroupingArg: TCompositeGrouping,