@serfab/cadre-core 1.19.0 → 1.19.1

package/src/strand-membership-reconciler.ts

@@ -120,6 +120,15 @@
  * written nothing, and the pass retries on the ladder. A busy refusal is never read as a
  * dead invitation, and a busy burn leaves the invitation staged for the next pass.
  *
+ * A write refused because the strand's offline queue holds transactions
+ * (`StrandOfflineQueueBusyError`, docs/strands.md → "Offline mode") is a busy refusal too: it
+ * wrote nothing, and the ladder retries it until the queue drains. A membership commit queued
+ * behind an app commit anyway (the race that refusal leaves open) can fail with Optimystic's
+ * `OfflineCommitTimeoutError`, which is an unclassified failure here: a redemption keeps the
+ * invitation and retries, and a binding is retried by the next pass. Each pass reads the state
+ * before it writes (the member row, then the binding, which is insert-if-absent), so a write that
+ * landed after all (`sent: true`) is found by that pass rather than written twice.
+ *
  * ## Half-committed join
  *
  * On a networked strand optimystic commits `Member` and `ConsumedInvite` as separate
@@ -144,6 +153,7 @@ import type { Ed25519KeyPair } from './ed25519-key.js';
 import type { StrandMembershipInvite } from './types.js';
 import { strandMemberKeyPair } from './strand-member-key.js';
 import {
+  StrandOfflineQueueBusyError,
   StrandTransactionBusyError,
   burnInvite,
   consumeInvite,
@@ -231,7 +241,10 @@ export interface HalfCommittedJoin {
 /** Where {@link classifyConsumeFailure} routes a failed `consumeInvite` or `burnInvite`. */
 export type ConsumeFailure =
   | HalfCommittedJoin
-  /** The app had a transaction open, so nothing was tried — keep the invitation, retry. */
+  /**
+   * The app had a transaction open, or the strand's offline queue held transactions, so nothing
+   * was tried — keep the invitation, retry.
+   */
   | { kind: 'busy' }
   /** The strand is sealed — terminal for a non-member; a member's leftover invitation is merely dead. */
   | { kind: 'sealed' }
@@ -289,9 +302,13 @@ function halfCommittedJoin(error: unknown): HalfCommittedJoin | undefined {
   return undefined;
 }
 
-/** Whether a writer refused because another transaction was open (see "Sharing the database with the app"). */
+/**
+ * Whether a writer refused having written nothing, because another transaction was open or the
+ * offline queue held transactions (see "Sharing the database with the app").
+ */
 function isTransactionBusy(error: unknown): boolean {
-  return causeChain(error).some((link) => link instanceof StrandTransactionBusyError);
+  return causeChain(error).some((link) =>
+    link instanceof StrandTransactionBusyError || link instanceof StrandOfflineQueueBusyError);
 }
 
 function errorMessage(error: unknown): string {
@@ -712,8 +729,8 @@ export class StrandMembershipReconciler {
     const failure = classifyConsumeFailure(error);
     switch (failure.kind) {
       case 'busy':
-        log('[%s] burning the leftover invitation deferred — the app has a transaction open; it stays staged '
-          + 'for the next pass', this.deps.label);
+        log('[%s] burning the leftover invitation deferred — nothing was tried; it stays staged for the next '
+          + 'pass: %s', this.deps.label, errorMessage(error));
         return;
       case 'sealed':
       case 'dead-invite':
@@ -761,8 +778,8 @@ export class StrandMembershipReconciler {
     const failure = classifyConsumeFailure(error);
     switch (failure.kind) {
       case 'busy':
-        log('[%s] redeeming the staged invitation deferred — the app has a transaction open on the strand '
-          + 'database; retrying next tick', this.deps.label);
+        log('[%s] redeeming the staged invitation deferred — nothing was tried; retrying next tick: %s',
+          this.deps.label, errorMessage(error));
         return;
       case 'half-committed':
         this.reportHalfCommittedJoin(failure, error, invite);

package/src/strand-membership-writer.ts

@@ -3,6 +3,7 @@ import { toString as uint8ArrayToString } from 'uint8arrays';
 import { TransactionActiveError } from '@quereus/quereus';
 import type { Database, SqlValue } from '@quereus/quereus';
 import { digest, sign, verify, generatePrivateKey, getPublicKey, randomBytes } from '@optimystic/quereus-plugin-crypto';
+import { offlineControllerOf } from '@serfab/quereus-plugin-sereus';
 import type { SAppConfig } from './types.js';
 import type { Ed25519KeyPair } from './ed25519-key.js';
 import { canonicalDatetime } from './canonical-datetime.js';
@@ -146,6 +147,31 @@ export class StrandTransactionBusyError extends Error {
   }
 }
 
+/**
+ * A membership write refused because the strand's offline queue holds transactions its members
+ * have not taken yet (docs/strands.md → "Offline mode"). Every `Strand` table is network-acknowledged, and a
+ * network-acknowledged commit made while transactions are queued waits for them to settle while
+ * holding Quereus's execution lock, so every other statement on the strand database, the app's
+ * included, would wait with it for up to the plugin's `networkCommitTimeoutMs`. Nothing was
+ * written. Raised whether or not the write would have joined an open transaction: joining would
+ * make the app's own commit wait the same way.
+ */
+export class StrandOfflineQueueBusyError extends Error {
+  /** The queued transactions the write would have waited behind, in queue order. */
+  readonly transactionIds: readonly string[];
+
+  constructor(transactionIds: readonly string[]) {
+    super(
+      `The strand's offline queue holds ${transactionIds.length} transaction(s) its members have not taken yet `
+      + `(${transactionIds.join(', ')}). A membership write is acknowledged by the strand's members and would `
+      + 'wait behind them, holding the strand database, so nothing was written; await '
+      + 'strand.database.offline.whenDrained() and retry.',
+    );
+    this.name = 'StrandOfflineQueueBusyError';
+    this.transactionIds = transactionIds;
+  }
+}
+
 /** Options every transactional membership writer accepts. */
 export interface StrandWriteOptions {
   /**
@@ -207,6 +233,8 @@ function combineStatements(...fragments: StrandStatements[]): StrandStatements {
  * autocommit statement is merely in flight, which would make a background writer refuse
  * spuriously and a composing caller run its statements as separate autocommits.
  *
+ * - The strand's offline queue holds transactions: {@link StrandOfflineQueueBusyError}, before
+ *   anything runs, every writer and every option alike (see {@link refuseBehindOfflineQueue}).
  * - No transaction was open: the batch commits or fails as a unit, leaving nothing open.
  * - One was already open, so the batch refuses with `TransactionActiveError` having run
  *   nothing. With `joinOpenTransaction: false` that becomes a
@@ -219,6 +247,7 @@ async function execStrandTransaction(
   statements: StrandStatements,
   options?: StrandWriteOptions,
 ): Promise<void> {
+  refuseBehindOfflineQueue(db);
   try {
     await db.exec(statements.sql, statements.params, { transaction: true });
     return;
@@ -244,6 +273,24 @@ async function execStrandTransaction(
   await db.exec(statements.sql, statements.params);
 }
 
+/**
+ * Throw {@link StrandOfflineQueueBusyError} while the strand's offline queue lists any
+ * transaction. `entries()` also lists the journaled ones a restore has not put back yet, and a
+ * network-acknowledged commit waits behind those too.
+ *
+ * NOTE: racy by nature: an app commit queued between this check and the batch's `exec` makes the
+ * membership commit wait behind it, for up to `networkCommitTimeoutMs`, holding the execution
+ * lock. That is upstream's bounded wait in the rare case. If a frozen strand database is ever
+ * traced to a membership write that slipped past this check, hold the app's writes during
+ * membership commits instead.
+ */
+function refuseBehindOfflineQueue(db: Database): void {
+  const queued = offlineControllerOf(db)?.entries() ?? [];
+  if (queued.length > 0) {
+    throw new StrandOfflineQueueBusyError(queued.map((entry) => entry.transactionId));
+  }
+}
+
 /**
  * The `Strand.Revocation` tombstone insert that retires a deleted row's StampId.
  *
@@ -736,12 +783,12 @@ export async function issueInvite(db: Database, params: IssueInviteParams): Prom
   const managerSignature = signStrandPayload(payload, managerKeyPair.privateKeyB64);
   const inviteSignature = signStrandPayload(payload, invitePrivateKey);
 
-  await db.exec(
-    `insert into Strand.Invite (Key, Expiration)
-       with context ManagerKey = ?, ManagerSignature = ?, InviteSignature = ?
-       values (?, ?)`,
-    [managerKeyPair.publicKeyB64, managerSignature, inviteSignature, inviteKey, expirationColumn],
-  );
+  await execStrandTransaction(db, {
+    sql: `insert into Strand.Invite (Key, Expiration)
+       with context ManagerKey = :managerKey, ManagerSignature = :managerSignature, InviteSignature = :inviteSignature
+       values (:inviteKey, :expiration);`,
+    params: { managerKey: managerKeyPair.publicKeyB64, managerSignature, inviteSignature, inviteKey, expiration: expirationColumn },
+  });
   log('Issued invite %s (expires=%s)', inviteKey, expirationColumn ?? 'never');
 
   return { inviteKey, invitePrivateKey };
@@ -937,8 +984,8 @@ export interface CancelInviteParams {
  * `digest('Strand.CancelledInvite', 'cancel', InviteKey)`. The digest binds the exact
  * invite key, so an approval minted for one invitation cannot cancel another.
  *
- * One row, one statement — no transaction needed. `CancelledInvite.Authorized` is a
- * deferred (subquery-bearing) check, so it fires at this statement's own auto-commit.
+ * One row, one statement. `CancelledInvite.Authorized` is a deferred (subquery-bearing)
+ * check, so it fires at the batch's commit, or at the caller's when it joins an open one.
  *
  * Insert-if-absent, matching {@link registerMemberPeer}'s and {@link revokeMember}'s
  * restart-safe shape: a repeat cancellation logs and returns rather than throwing on the
@@ -973,12 +1020,12 @@ export async function cancelInvite(db: Database, params: CancelInviteParams): Pr
     ['Strand.CancelledInvite', 'cancel', inviteKey],
     managerKeyPair.privateKeyB64,
   );
-  await db.exec(
-    `insert into Strand.CancelledInvite (InviteKey)
-       with context ManagerKey = ?, ManagerSignature = ?
-       values (?)`,
-    [managerKeyPair.publicKeyB64, signature, inviteKey],
-  );
+  await execStrandTransaction(db, {
+    sql: `insert into Strand.CancelledInvite (InviteKey)
+       with context ManagerKey = :managerKey, ManagerSignature = :managerSignature
+       values (:inviteKey);`,
+    params: { managerKey: managerKeyPair.publicKeyB64, managerSignature: signature, inviteKey },
+  });
   log('Cancelled invite %s by manager %s', inviteKey, managerKeyPair.publicKeyB64);
 }
 

package/src/strand-offline.ts

@@ -0,0 +1,247 @@
+import debug from 'debug';
+import { EntryNotDiscardableError, UnknownEntryError } from '@optimystic/db-core';
+import type { IKVStore } from '@optimystic/db-p2p';
+import type {
+  AppSchemaOutcome, OfflineModeState, QueuedTransactionStatus, SereusPluginResult, StrandOfflineController,
+} from '@serfab/quereus-plugin-sereus';
+
+const log = debug('sereus:cadre:strand-offline');
+
+/**
+ * Which acknowledgement a strand's transactions get from the moment it is set: `'offline'`, a
+ * commit resolves once the transaction is queued on this machine; `'online'`, once the strand's
+ * members have taken it.
+ */
+export type StrandOfflineMode = 'online' | 'offline';
+
+/** `CadreNodeConfig.strandOffline`: what a node needs to offer offline mode on its strands. */
+export interface StrandOfflineConfig {
+  /**
+   * The journal for one strand: called once per database build with the strand id (a storage scope
+   * key). Queued transactions are saved in it before their commit resolves, so they survive a
+   * restart. A throw or rejection fails the build, as a storage provider's does.
+   *
+   * Nothing in cadre-core or Optimystic closes the store, and every rebuild (a wake, a retry while
+   * no seed peer answers) calls this again, so return the same store for the same scope and close
+   * it yourself after the node stops.
+   */
+  journal: (scope: string) => IKVStore | Promise<IKVStore>;
+  /** The mode every strand starts in until the app sets one. Default `'online'`. */
+  initialMode?: StrandOfflineMode;
+}
+
+/**
+ * `StrandDatabase.offline`: the strand connection's offline queue, less the mode switch, which
+ * goes through `CadreNode.setStrandOfflineMode` so that it outlives the database. Listeners taken
+ * here die with the database; the `CadreNode` events do not.
+ */
+export type StrandOfflineQueue = Pick<StrandOfflineController,
+  'queueState' | 'entries' | 'status' | 'outcomes' | 'acknowledge' | 'discard' | 'whenDrained' | 'onStatus' | 'onQueueState'>;
+
+/**
+ * `StrandDatabase.appSchemaState()`: whether the database runs the `App` schema its launch was
+ * given. `'waiting-for-queue'` while queued transactions hold the apply back (the database runs the
+ * earlier schema); `'failed'` when a later retry failed for another reason.
+ */
+export type StrandAppSchemaState = 'applied' | 'waiting-for-queue' | 'failed';
+
+/** `strand:offline-status`: one status of one queued transaction. */
+export interface StrandOfflineStatus {
+  readonly strandId: string;
+  readonly status: QueuedTransactionStatus;
+}
+
+/** `strand:queue-state`: the queue's state, for a status line. */
+export interface StrandQueueState {
+  readonly strandId: string;
+  readonly state: OfflineModeState;
+}
+
+/** `strand:app-schema-deferred`: the queued transactions that held the `App` schema apply back. */
+export interface StrandAppSchemaDeferred {
+  readonly strandId: string;
+  readonly transactionIds: readonly string[];
+}
+
+/**
+ * Where a strand database reports its offline queue and its app-schema apply. Retained with the
+ * launch config and handed to every database built for the launch, so the reports outlive
+ * hibernation and every other rebuild.
+ */
+export interface StrandOfflineListeners {
+  onOfflineStatus?: (event: StrandOfflineStatus) => void;
+  onQueueState?: (event: StrandQueueState) => void;
+  onAppSchemaDeferred?: (event: StrandAppSchemaDeferred) => void;
+  onAppSchemaApplied?: (event: { strandId: string }) => void;
+  onAppSchemaFailed?: (event: { strandId: string; error: Error }) => void;
+}
+
+/** Thrown by `CadreNode.setStrandOfflineMode` on a node configured without `strandOffline`. */
+export class StrandOfflineUnavailableError extends Error {
+  readonly strandId: string;
+
+  constructor(strandId: string) {
+    super(
+      `Cannot put strand ${strandId} in offline mode: this node has no offline journal ` +
+      '(CadreNodeConfig.strandOffline is not configured), and a queue lost with the process would ' +
+      'break the promise a local commit makes.'
+    );
+    this.name = 'StrandOfflineUnavailableError';
+    this.strandId = strandId;
+  }
+}
+
+/** Options for leaving a strand: `CadreNode.forgetJoinedStrand` and `CadreNode.unpublishStrand`. */
+export interface StrandLeaveOptions {
+  /**
+   * Withdraw the strand's queued offline transactions that were never sent, then leave. Without
+   * it a leave refuses with {@link StrandQueueNotEmptyError} while the queue holds any: the app was
+   * told each of them was acknowledged, so dropping them is the user's choice to make.
+   */
+  discardQueued?: boolean;
+}
+
+/**
+ * Thrown by a leave ({@link StrandLeaveOptions}) while the strand's offline queue holds
+ * transactions and `discardQueued` was not given. Nothing was left or removed.
+ */
+export class StrandQueueNotEmptyError extends Error {
+  readonly strandId: string;
+  /** The queued transactions, by the `StampId()` each one's statements returned. */
+  readonly transactionIds: readonly string[];
+
+  constructor(strandId: string, transactionIds: readonly string[]) {
+    super(
+      `Cannot leave strand ${strandId}: its offline queue holds ${transactionIds.length} transaction(s) its ` +
+      `members have not taken yet (${transactionIds.join(', ')}), so nothing was left or removed. Show the user ` +
+      'the unsent changes, then leave again with { discardQueued: true } to drop them, or once ' +
+      'strand.database.offline.whenDrained() resolves.'
+    );
+    this.name = 'StrandQueueNotEmptyError';
+    this.strandId = strandId;
+    this.transactionIds = transactionIds;
+  }
+}
+
+/**
+ * Withdraw every transaction in `queue` that was never sent, for a leave the app confirmed
+ * ({@link StrandLeaveOptions.discardQueued}). Returns the ones Optimystic refuses to withdraw
+ * (`EntryNotDiscardableError`): each was sent, or is being sent or restored, and the strand's
+ * members may already hold it, so it stays in the journal. One that settled meanwhile, or failed
+ * with a transaction it depended on, is already gone.
+ */
+export function discardQueued(queue: Pick<StrandOfflineController, 'entries' | 'discard'>): string[] {
+  const kept: string[] = [];
+  for (const { transactionId } of queue.entries()) {
+    try {
+      queue.discard(transactionId);
+    } catch (error) {
+      if (error instanceof EntryNotDiscardableError) {
+        kept.push(transactionId);
+      } else if (!(error instanceof UnknownEntryError)) {
+        throw error;
+      }
+    }
+  }
+  return kept;
+}
+
+/**
+ * Whether a status shows the queue sending, which counts as strand activity for hibernation. A
+ * transaction waiting for its cohort (`queued`) is not, or an offline phone would never hibernate.
+ */
+// NOTE: a send that keeps failing transiently while the cohort reads as reachable is re-tried at
+// least every 60 s (Optimystic's re-check cap), so such a strand never idles. If a phone is found
+// kept awake that way, count only `committed`, `partially-committed` and `failed`.
+export function isSendingStatus(status: QueuedTransactionStatus): boolean {
+  return status.kind === 'submitting' || status.kind === 'committed'
+    || status.kind === 'partially-committed' || status.kind === 'failed';
+}
+
+/**
+ * What one strand database reports about its offline queue and its app-schema apply. Built once
+ * `connectToStrand` has returned (a restored transaction's `queued` status was delivered during its
+ * `hydrate`, before this subscribes: {@link published} reports the queue's state for that reason)
+ * and released first when the database closes, so nothing from a closing connection reaches the app.
+ *
+ * App-schema events begin at publication, when the app gets the database: a retry that landed
+ * before then is not announced, and one that failed before then is announced at publication.
+ */
+export class StrandOfflineReport {
+  private appSchema: StrandAppSchemaState;
+  private appSchemaError: Error | undefined;
+  private wasPublished = false;
+  private released = false;
+  private readonly unsubscribe: () => void;
+
+  constructor(
+    private readonly strandId: string,
+    private readonly listeners: StrandOfflineListeners,
+    private readonly result: Pick<SereusPluginResult, 'offline' | 'appSchema' | 'appSchemaDeferredBy'>
+  ) {
+    // `appSchema` is absent without an `App` schema (a storage replica), which has nothing to defer.
+    this.appSchema = result.appSchemaDeferredBy ? 'waiting-for-queue' : 'applied';
+    const queue = result.offline;
+    const unsubscribeStatus = queue?.onStatus((status) => listeners.onOfflineStatus?.({ strandId, status }));
+    const unsubscribeState = queue?.onQueueState((state) => listeners.onQueueState?.({ strandId, state }));
+    this.unsubscribe = () => {
+      unsubscribeStatus?.();
+      unsubscribeState?.();
+    };
+    if (result.appSchemaDeferredBy) {
+      void result.appSchema?.then((outcome) => this.settleAppSchema(outcome)).catch((error: unknown) => {
+        log('strand %s: reporting the deferred app schema apply failed: %O', strandId, error);
+      });
+    }
+  }
+
+  appSchemaState(): StrandAppSchemaState {
+    return this.appSchema;
+  }
+
+  /** The database was handed to the app: report the queue's state, and an apply still waiting or already failed. */
+  published(): void {
+    if (this.released || this.wasPublished) {
+      return;
+    }
+    this.wasPublished = true;
+    const queue = this.result.offline;
+    if (queue) {
+      this.listeners.onQueueState?.({ strandId: this.strandId, state: queue.queueState() });
+    }
+    if (this.appSchema === 'waiting-for-queue') {
+      this.listeners.onAppSchemaDeferred?.({ strandId: this.strandId, transactionIds: this.result.appSchemaDeferredBy ?? [] });
+    } else if (this.appSchemaError) {
+      this.reportAppSchemaFailed(this.appSchemaError);
+    }
+  }
+
+  release(): void {
+    this.released = true;
+    this.unsubscribe();
+  }
+
+  /** A cancelled retry (the database closed first) reports nothing. */
+  private settleAppSchema(outcome: AppSchemaOutcome): void {
+    if (this.released || outcome.state === 'cancelled') {
+      return;
+    }
+    if (outcome.state === 'applied') {
+      this.appSchema = 'applied';
+      if (this.wasPublished) {
+        this.listeners.onAppSchemaApplied?.({ strandId: this.strandId });
+      }
+      return;
+    }
+    this.appSchema = 'failed';
+    this.appSchemaError = outcome.error instanceof Error ? outcome.error : new Error(String(outcome.error));
+    log('strand %s: the deferred app schema apply failed; the database keeps the earlier schema: %O', this.strandId, outcome.error);
+    if (this.wasPublished) {
+      this.reportAppSchemaFailed(this.appSchemaError);
+    }
+  }
+
+  private reportAppSchemaFailed(error: Error): void {
+    this.listeners.onAppSchemaFailed?.({ strandId: this.strandId, error });
+  }
+}