@optimystic/db-core 1.0.0-beta.1 → 1.0.0-beta.3

package/src/collections/tree/struct.ts

@@ -1,26 +1,116 @@
-import type { BlockId, CollectionHeaderBlock } from "../../index.js";
-import { registerBlockType } from "../../blocks/block-types.js";
-import { registerCollectionType } from "../../collection/collection-type-registry.js";
-import { nameof } from "../../utility/nameof.js";
-
-export const TreeHeaderBlockType = registerBlockType("TRE", "TreeHeaderBlock");
-
-registerCollectionType({
-	blockType: TreeHeaderBlockType,
-	name: "Tree",
-});
-
-export type TreeCollectionHeaderBlock = CollectionHeaderBlock & {
-	rootId: BlockId;
-};
-
-export const rootId$ = nameof<TreeCollectionHeaderBlock>("rootId");
-
-/** Represents a unit of change to a tree collection. */
-export type TreeReplaceAction<TKey, TEntry> = [
-	// The key to replace
-	key: TKey,
-	// The new entry to replace the old entry with (if not provided, the key is deleted)
-	entry?: TEntry,
-][];
-
+import type { BlockId, CollectionHeaderBlock, CollectionId } from "../../index.js";
+import type { KeyRange } from "../../btree/index.js";
+import { registerBlockType } from "../../blocks/block-types.js";
+import { registerCollectionType } from "../../collection/collection-type-registry.js";
+import { nameof } from "../../utility/nameof.js";
+
+export const TreeHeaderBlockType = registerBlockType("TRE", "TreeHeaderBlock");
+
+registerCollectionType({
+	blockType: TreeHeaderBlockType,
+	name: "Tree",
+});
+
+export type TreeCollectionHeaderBlock = CollectionHeaderBlock & {
+	rootId: BlockId;
+};
+
+export const rootId$ = nameof<TreeCollectionHeaderBlock>("rootId");
+
+/**
+ * Serializable statement of the INTENT behind a staged tree entry, enforced by the
+ * `replace` handler every time it runs — at initial staging AND at every conflict
+ * replay against a newly adopted committed revision. Without a guard, staging is a
+ * bare upsert, and a conflict replay silently overwrites whatever a rival writer
+ * committed at the same key (the concurrent-INSERT lost-uniqueness bug).
+ *
+ * - `absent` — the key must not exist (SQL INSERT): a hit throws {@link TreeKeyTakenError},
+ *   discarding the whole action's staged writes.
+ * - `keepExisting` — if the key exists, skip this entry silently (SQL INSERT OR IGNORE).
+ * - `absentRange` — no entry OTHER THAN this action's own key may exist in `range`
+ *   (secondary-UNIQUE enforcement, where uniqueness is a property of a framed key
+ *   PREFIX rather than one exact key: an index tree keys `indexKey ‖ primaryKey`, so
+ *   two rows sharing a unique value sit at different keys inside one prefix range). A
+ *   foreign hit throws {@link TreeRangeTakenError}. The entry being staged lands inside
+ *   its own range, so the scan excludes its exact key — otherwise every guarded
+ *   re-stage of a present key (a replay after a clean refresh) would refuse itself.
+ *
+ * MIXED VERSIONS: a peer running a build that predates guards destructures `[key, entry]`
+ * and ignores the third slot — its replays revert to today's silent overwrite. No version
+ * gating exists yet (see backlog ticket `debt-mixed-version-identify-incompatibility`).
+ */
+export type TreeEntryGuard<TKey> =
+	| { kind: 'absent' }
+	| { kind: 'keepExisting' }
+	| { kind: 'absentRange', range: KeyRange<TKey> };
+
+/** Represents a unit of change to a tree collection. */
+export type TreeReplaceAction<TKey, TEntry> = [
+	// The key to replace
+	key: TKey,
+	// The new entry to replace the old entry with (if not provided, the key is deleted)
+	entry?: TEntry,
+	// Optional uniqueness intent, re-checked on every handler run (see TreeEntryGuard).
+	// Absent = plain upsert, so existing callers and previously committed log entries
+	// deserialize and replay unchanged.
+	guard?: TreeEntryGuard<TKey>,
+][];
+
+/**
+ * Thrown by the tree `replace` handler when an entry guarded `absent` finds its key
+ * already present — at initial staging, or (the load-bearing case) at conflict replay
+ * after a rival writer's commit was adopted. The throw discards the whole action's
+ * staged writes (the handler runs inside an all-or-nothing Atomic wrapper) and
+ * propagates out of the sync/commit retry loops: it is not a StaleFailure, so no
+ * retry absorbs it, and it must never be downgraded to a retryable condition.
+ */
+export class TreeKeyTakenError<TKey = unknown> extends Error {
+	constructor(
+		/** The collection whose tree refused the entry. */
+		public readonly collectionId: CollectionId,
+		/** The key some other writer already committed (for the `absentRange` subclass: the
+		 * key this action was staging, whose claimed range a rival occupies). */
+		public readonly key: TKey,
+		/** Subclass override of the rendered message; the default names the exact-key refusal. */
+		message?: string,
+	) {
+		super(message ?? `Tree collection ${collectionId}: key ${renderKey(key)} is already taken by a committed entry`);
+		this.name = 'TreeKeyTakenError';
+	}
+}
+
+/**
+ * The `absentRange` refusal: the guarded entry's key is free, but some OTHER committed
+ * entry (`occupant`) sits inside the range the entry claims exclusively — for a unique
+ * index tree, a rival's row carrying the same unique value under a different primary
+ * key. A subclass of {@link TreeKeyTakenError} on purpose: every consumer that treats a
+ * key refusal as a non-retryable uniqueness failure (the sync/commit retry loops let it
+ * escape; the Quereus bridge maps it by `collectionId` to a `UNIQUE constraint failed`
+ * message) handles this one identically without a second arm. `collectionId` is the
+ * discriminator that names WHICH constraint fired — each unique index is its own
+ * collection — so the bridge needs nothing beyond it.
+ */
+export class TreeRangeTakenError<TKey = unknown> extends TreeKeyTakenError<TKey> {
+	constructor(
+		collectionId: CollectionId,
+		/** The key this action was staging (free — it is the range that is contested). */
+		key: TKey,
+		/** The range the entry claimed exclusively. */
+		public readonly range: KeyRange<TKey>,
+		/** The committed key found inside `range` that is not `key`. */
+		public readonly occupant: TKey,
+	) {
+		super(collectionId, key,
+			`Tree collection ${collectionId}: key ${renderKey(key)} is guarded unique over a key range `
+			+ `already occupied by committed entry ${renderKey(occupant)}`);
+		this.name = 'TreeRangeTakenError';
+	}
+}
+
+/** String keys render JSON-quoted so framing control bytes stay visible/escaped in logs;
+ * everything else via String() — JSON.stringify would throw on a bigint key, and an error
+ * constructor must never be the second failure. */
+function renderKey(key: unknown): string {
+	return typeof key === 'string' ? JSON.stringify(key) : String(key);
+}
+

package/src/collections/tree/tree.ts

@@ -2,7 +2,7 @@ import { Collection, type CollectionInitOptions, type CollectionId, type Collect
 import type { ITransactor, BlockId, BlockStore, IBlock, ActionId } from "../../index.js";
 import { BTree, type Path, type KeyRange } from "../../btree/index.js";
 import { CollectionTrunk } from "./collection-trunk.js";
-import { TreeHeaderBlockType, type TreeReplaceAction } from "./struct.js";
+import { TreeHeaderBlockType, TreeKeyTakenError, TreeRangeTakenError, type TreeReplaceAction } from "./struct.js";
 
 /**
  * Read-only surface of a tree: every navigation/lookup method a reader needs, with
@@ -105,8 +105,42 @@ export class Tree<TKey, TEntry> implements TreeReadView<TKey, TEntry> {
 						compare,
 						nodeCapacity,	// keep the write btree's fan-out in lock-step with the read btree
 					);
-					for (const [key, entry] of actions) {
+					for (const [key, entry, guard] of actions) {
 						if (entry) {
+							// Enforce the entry's guard (if any) on EVERY handler run — initial staging
+							// and every conflict replay — so the uniqueness decision is re-made against
+							// the newest adopted committed state, not just the stage-time snapshot.
+							// A throw here discards the whole action's staged writes (Atomic wrapper).
+							if (guard !== undefined) {
+								if (guard.kind === 'absentRange') {
+									// Secondary-UNIQUE: the claimed range (a unique index's framed
+									// value prefix) must hold no entry other than this action's own
+									// key. Two short descents (range start and end) plus an early-exit
+									// walk, over the SAME store the upsert below writes to, so it sees entries earlier
+									// actions of this replay already staged or deleted (an UPDATE's
+									// delete-old half runs before its guarded insert half). The guard
+									// is plain data (it is serialized into the log with its action), and
+									// BTree.range reads the range's fields only — so a KeyRange that has
+									// round-tripped through JSON is scanned exactly like a live instance.
+									for await (const path of actionTree.range(guard.range)) {
+										const occupant = actionTree.at(path);
+										if (occupant === undefined) continue;
+										const occupantKey = keyFromEntry(occupant);
+										if (compare(occupantKey, key) === 0) continue;	// self-exclusion
+										throw new TreeRangeTakenError(id, key, guard.range, occupantKey);
+									}
+								} else {
+									const found = await actionTree.find(key);
+									if (found.on) {
+										if (guard.kind === 'absent') {
+											throw new TreeKeyTakenError(id, key);
+										}
+										// keepExisting: leave the present (rival's) entry in place and
+										// skip this entry silently — the INSERT OR IGNORE disposition.
+										continue;
+									}
+								}
+							}
 							await actionTree.upsert(entry);
 						} else {
 							await actionTree.deleteAt((await actionTree.find(key)));

package/src/index.ts

@@ -6,6 +6,7 @@ export * from "./cohort-topic/index.js";
 export * from "./collection/index.js";
 export * from "./collections/index.js";
 export * from "./log/index.js";
+export * from "./logger-registry.js";
 export * from "./matchmaking/index.js";
 export * from "./network/index.js";
 export * from "./reactivity/index.js";

package/src/logger-registry.ts

@@ -0,0 +1,224 @@
+/*
+ * One switch for every `optimystic:*` log channel, whichever copy of `debug` it lives on.
+ *
+ * Why this exists. `debug` turns namespaces on only from `process.env.DEBUG` (Node) or
+ * `localStorage.debug` (its browser build, which is the one Metro bundles). React Native has
+ * neither, so every channel is silently off there — and an empty capture reads exactly like "that
+ * code never ran". Calling `debug.enable(...)` from app code does not fix it reliably either: it
+ * reaches only the copy of `debug` the caller resolved, and each Optimystic package may resolve its
+ * own (this repo's install, `nmHoistingLimits: workspaces`, gives every package a separate copy).
+ *
+ * So each package's `src/logger.ts` registers the copy IT imported, and `enableOptimysticLogging`
+ * drives all of them. Coverage follows from every package registering, not from how the install
+ * happened to be laid out. `docs/debugging.md` § "Turning logging on" is the user-facing statement.
+ *
+ * This module imports nothing. It sits below every logger in the package, and
+ * `test/barrel-import-cycle.spec.ts` holds registry modules to zero runtime imports.
+ */
+
+/** The slice of a `debug` module this registry drives. Structural, so db-core never type-imports a particular copy. */
+export interface DebugModule {
+	enable(namespaces: string): void;
+	/** `debug` 4.x returns the namespaces that were active. */
+	disable(): string;
+	log: (...args: any[]) => any;
+	/**
+	 * `debug`'s persistence hook, which `enable` calls with every new set: it writes
+	 * `process.env.DEBUG` on Node and `localStorage.debug` in the browser build. Suppressed around
+	 * every call this registry makes — see `withoutPersisting`. Optional so a test double need not
+	 * supply one, and because `@types/debug` does not declare it.
+	 */
+	save?: (namespaces: string) => void;
+}
+
+/** Where log lines go — the same shape as `debug`'s own `log` property. */
+export type LogSink = (...args: unknown[]) => void;
+
+export interface OptimysticLoggingOptions {
+	/**
+	 * Where log lines go. Default: leave each copy's own sink (stderr on Node; `console.debug` in the
+	 * browser/RN build). Pass one when the platform hides that sink (e.g. a device log filtered above
+	 * debug level) or to collect a capture in memory.
+	 */
+	log?: LogSink;
+}
+
+export interface OptimysticLoggingReport {
+	/** The Optimystic contribution, comma-joined — empty when the call enabled nothing. */
+	namespaces: string;
+	/** One entry per distinct `debug` copy: the packages that registered it. */
+	copies: string[][];
+}
+
+interface Entry {
+	module: DebugModule;
+	owners: string[];
+	/** What this copy had enabled, and its sink, before our first touch. Unset while untouched. */
+	original?: { namespaces: string; log: DebugModule['log'] };
+}
+
+interface Pending {
+	namespaces: string;
+	log?: LogSink;
+}
+
+/*
+ * NOTE: the state lives on `globalThis`, not in module scope, so a bundle that ends up with two
+ * copies of db-core still has ONE registry — a copy of a package that registered with the other
+ * db-core would otherwise be unreachable. The cost: two DIFFERENT db-core versions in one bundle
+ * share this object, so change its shape only additively.
+ */
+const REGISTRY_KEY = Symbol.for('@optimystic/logger-registry');
+
+interface RegistryState {
+	entries: Entry[];
+	/** The last enable call's settings, applied to copies that register afterwards. Unset when off. */
+	pending?: Pending;
+}
+
+function registry(): RegistryState {
+	const holder = globalThis as unknown as Record<symbol, RegistryState | undefined>;
+	return holder[REGISTRY_KEY] ??= { entries: [] };
+}
+
+/**
+ * Run `fn` with the copy's `save` hook stubbed out, so what we enable changes this process only.
+ *
+ * Without this, on Node every `enable` would write `process.env.DEBUG` — and a copy of `debug` that
+ * loads LATER reads that variable as its own starting set, so it would arrive with our namespaces
+ * baked into what `disableOptimysticLogging` later treats as its baseline (they could never be
+ * turned off on that copy). Capturing a baseline via `disable()` would also delete the user's
+ * `DEBUG`. In a browser, it would persist our namespaces into `localStorage.debug`, leaving logging
+ * on after a reload even without this call.
+ */
+function withoutPersisting(module: DebugModule, fn: () => void): void {
+	const save = module.save;
+	if (save) module.save = () => { };
+	try {
+		fn();
+	} finally {
+		if (save) module.save = save;
+	}
+}
+
+/**
+ * Bring one copy in line with `pending`: its baseline (captured on first touch) plus ours, and
+ * `pending.log` or its original sink.
+ *
+ * NOTE: ours is appended, so a `-optimystic:…` skip already in the copy's baseline (say
+ * `DEBUG='*,-optimystic:*'`) still wins — `debug` checks skips first. That is the user's explicit
+ * exclusion, so it is left alone; if it ever confuses someone, have the confirmation line name the
+ * conflicting skip rather than overriding it.
+ *
+ * NOTE: the baseline is captured once per enable/disable cycle, so an app that calls
+ * `debug.enable(...)` on a shared copy while ours are on has that change overwritten by our next
+ * enable or disable. Fine while apps set their channels once at start-up; if one ever toggles them
+ * at run time, re-derive the baseline from the copy's live set minus our contribution instead.
+ */
+function apply(entry: Entry, pending: Pending): void {
+	const module = entry.module;
+	if (!entry.original) {
+		let namespaces = '';
+		withoutPersisting(module, () => { namespaces = module.disable(); });
+		entry.original = { namespaces, log: module.log };
+	}
+	const combined = [entry.original.namespaces, pending.namespaces].filter(Boolean).join(',');
+	withoutPersisting(module, () => module.enable(combined));
+	module.log = pending.log ?? entry.original.log;
+}
+
+function restore(entry: Entry): void {
+	const original = entry.original;
+	if (!original) return;
+	withoutPersisting(entry.module, () => entry.module.enable(original.namespaces));
+	entry.module.log = original.log;
+	entry.original = undefined;
+}
+
+/** Comma-join, trimming each entry and dropping empty ones. A string may already hold a list. */
+function normalizeNamespaces(namespaces: string | readonly string[]): string {
+	const parts = typeof namespaces === 'string' ? [namespaces] : namespaces;
+	return parts
+		.flatMap(part => part.split(','))
+		.map(part => part.trim())
+		.filter(Boolean)
+		.join(',');
+}
+
+function confirmationLine(report: OptimysticLoggingReport): string {
+	const what = report.namespaces ? `"${report.namespaces}"` : 'nothing (empty namespace list)';
+	const trailer = 'libp2p:* is separate, see docs/debugging.md';
+	if (report.copies.length === 0) {
+		return `optimystic logging on: ${what}; no debug copies registered yet, namespaces will apply as Optimystic packages load; ${trailer}`;
+	}
+	const count = report.copies.length;
+	const groups = report.copies.map(owners => owners.join(', ')).join(' | ');
+	return `optimystic logging on: ${what} across ${count} debug ${count === 1 ? 'copy' : 'copies'} [${groups}]; ${trailer}`;
+}
+
+/**
+ * Called once from each package's `src/logger.ts` at module load, with the `debug` module that file
+ * imported. Idempotent per (owner, module); several owners sharing one copy are one entry.
+ *
+ * With no prior `enableOptimysticLogging` call this touches nothing, so `DEBUG=` on Node behaves
+ * exactly as it always has. After one, the arriving copy gets the same namespaces and sink at once
+ * (its own baseline captured first), silently — the enable call already confirmed.
+ */
+export function registerDebugModule(owner: string, module: DebugModule): void {
+	const state = registry();
+	const existing = state.entries.find(entry => entry.module === module);
+	if (existing) {
+		if (!existing.owners.includes(owner)) existing.owners.push(owner);
+		return;
+	}
+	const entry: Entry = { module, owners: [owner] };
+	state.entries.push(entry);
+	if (state.pending) apply(entry, state.pending);
+}
+
+/**
+ * Turn on the given namespaces on every registered `debug` copy, now and for copies that register
+ * later — the way to enable Optimystic logging on any runtime, React Native included, without
+ * environment variables.
+ *
+ * Adds to each copy's existing namespaces rather than replacing them, so an app's own channels
+ * sharing a copy stay on. A second call replaces the first call's namespaces and `log`, and does not
+ * accumulate. An empty list enables nothing of ours (and is not an error).
+ *
+ * Always writes one confirmation line — through `options.log` if given, else the first registered
+ * copy's sink, else `console.log` — because silence is exactly what cannot be told apart from "the
+ * code never ran". No line: this was not called, or the sink is swallowed. A line and no events: the
+ * code did not run, or the filter does not match. Events: it works.
+ */
+export function enableOptimysticLogging(
+	namespaces: string | readonly string[],
+	options?: OptimysticLoggingOptions,
+): OptimysticLoggingReport {
+	const state = registry();
+	const pending: Pending = { namespaces: normalizeNamespaces(namespaces), log: options?.log };
+	state.pending = pending;
+	for (const entry of state.entries) apply(entry, pending);
+
+	const report: OptimysticLoggingReport = {
+		namespaces: pending.namespaces,
+		copies: state.entries.map(entry => [...entry.owners]),
+	};
+	const line = confirmationLine(report);
+	const sink = options?.log ?? state.entries[0]?.module.log;
+	if (sink) {
+		sink(line);
+	} else {
+		// No Optimystic package has loaded yet, so there is no debug sink to write through — and the
+		// confirmation line must never be skipped (see above).
+		// eslint-disable-next-line no-console
+		console.log(line);
+	}
+	return report;
+}
+
+/** Undo it: every copy goes back to what it had enabled before the first enable call, and its original sink. Safe to call when nothing is enabled. */
+export function disableOptimysticLogging(): void {
+	const state = registry();
+	for (const entry of state.entries) restore(entry);
+	state.pending = undefined;
+}

package/src/logger.ts

@@ -1,7 +1,11 @@
 import debug from 'debug'
+import { registerDebugModule } from './logger-registry.js'
 
 const BASE_NAMESPACE = 'optimystic:db-core'
 
+// So `enableOptimysticLogging` reaches this package's copy of `debug`, which may be no one else's.
+registerDebugModule('db-core', debug)
+
 export function createLogger(subNamespace: string): debug.Debugger {
 	return debug(`${BASE_NAMESPACE}:${subNamespace}`)
 }

package/src/transactor/network-transactor.ts

@@ -154,9 +154,14 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
 		// missing block has already happened one layer down: CoordinatorRepo.get detects
 		// `isMissing` and consults cluster peers before it responds — and when that
 		// consult FAILS, the entry now says so via `unavailable` instead of posing as an
-		// authoritative absent. So by the time an unflagged absent reaches here there is
-		// nothing left for a transactor-level retry to discover, while a flagged entry
-		// earns the retry against a different peer that an absent deliberately does not.
+		// authoritative absent. An unflagged absent means the cohort confirmed the absence
+		// within the last `readRepairWindowMs` (a coordinator that is the block's whole
+		// cohort remembers its absence for one window — the same currency bound a held
+		// block's content carries; a multi-peer cohort's absence is re-asked on every read).
+		// So by the time an unflagged absent reaches here there is nothing left for a
+		// transactor-level retry to discover: another coordinator would find the same
+		// cohort's same answer. A flagged entry earns the retry against a different peer
+		// that an absent deliberately does not.
 		// See tickets txn-perf-authoritative-notfound and repo-reports-unavailable-vs-absent.
 		const hasValidResponse = (b: CoordinatorBatch<BlockId[], GetBlockResults>) => {
 			return b.request?.isResponse === true && b.request.response != null;