@astrosheep/square 0.3.72 → 0.3.73

package/dist/ledger-maintenance.d.ts

@@ -0,0 +1,30 @@
+import type { HostLedgerPort } from './host-ledger.js';
+/** The two host-ledger capabilities maintenance uses; no parallel ledger contract is owned here. */
+type MaintenanceLedger = Pick<HostLedgerPort, 'listEvidence' | 'retireEvidenceIfUnchanged'>;
+export interface LedgerMaintenanceOptions {
+    /** Prove eligible rows without retiring anything. */
+    readonly dryRun?: boolean;
+    readonly now?: number;
+}
+/**
+ * Counts are retired rows, not reclaimed bytes: SQLite reuses freed storage without necessarily
+ * shrinking its file. Retired Claude correlations leave indefinite recognition tokens; total
+ * token count remains unbounded. Maintenance does not run VACUUM.
+ */
+export interface LedgerMaintenanceReport {
+    readonly dryRun: boolean;
+    /** Old rows whose closure proof passed; exact-record CAS may still refuse. */
+    readonly eligible: number;
+    /** Rows actually removed by this run; always 0 in a dry run. */
+    readonly retired: number;
+    /** Eligible rows whose full-record compare-and-swap refused; always 0 in a dry run. */
+    readonly casMisses: number;
+    /** Canonical locations whose artifact could not be read; their rows were retained. */
+    readonly unavailableArtifacts: readonly string[];
+}
+/** Command/application entry: root selection uses the caller's captured environment. */
+export declare function gcHostLedger(options?: LedgerMaintenanceOptions & {
+    readonly env?: NodeJS.ProcessEnv;
+}): Promise<LedgerMaintenanceReport>;
+export declare function maintainHostLedger(ledger: MaintenanceLedger, options?: LedgerMaintenanceOptions): Promise<LedgerMaintenanceReport>;
+export {};

package/dist/ledger-maintenance.js

@@ -0,0 +1,110 @@
+import { createArtifactPort } from './artifact.js';
+import { canonicalPath } from './canonical-path.js';
+import { formatActivityId, parseActivityId } from './square-core.js';
+import { validateName } from './model.js';
+import { hostLedgerForEnv } from './registry.js';
+/**
+ * Explicit host-ledger evidence maintenance.
+ *
+ * Old wake/presentation rows are retired only when a typed artifact snapshot proves the
+ * recipient's attention is permanently closed: the activity reached them historically, and
+ * either catch recorded a receipt or the recipient joined again past it. Everything else is
+ * retained. Artifacts are read directly, one snapshot per canonical location, outside any
+ * ledger lock; retirement is the ledger's exact-record compare-and-swap. No SQL lives here.
+ */
+/** A row is old enough to consider once its landing time is at least this far behind now. */
+const RETIREMENT_AGE_MS = 7 * 24 * 60 * 60 * 1000;
+/** The canonical activity coordinate the row names; leading zeros and other spellings are invalid. */
+function activityIndex(activity) {
+    const index = parseActivityId(activity);
+    return index === undefined || formatActivityId(index) !== activity ? undefined : index;
+}
+/** Old, durably closed-looking wake/presentation rows with usable coordinates. */
+function isCandidate(record, now) {
+    if (record.kind !== 'wake' && record.kind !== 'presentation')
+        return false;
+    if (record.outcome === 'dispatching')
+        return false;
+    if (typeof record.at !== 'number' || !Number.isFinite(record.at) || record.at > now - RETIREMENT_AGE_MS)
+        return false;
+    if (typeof record.location !== 'string' || record.location.length === 0)
+        return false;
+    try {
+        validateName(record.participant);
+    }
+    catch {
+        return false;
+    }
+    return true;
+}
+/**
+ * One artifact snapshot for one location either proves each candidate closed or reports the
+ * whole location unavailable. A missing, unreadable, or non-artifact path returns undefined
+ * and its rows are retained.
+ */
+async function provenClosable(location, group) {
+    const artifact = createArtifactPort(location);
+    try {
+        return (await artifact.read((facts) => group.filter((candidate) => {
+            const participant = candidate.record.participant;
+            if (facts.directedAt(participant, candidate.index) === undefined)
+                return false;
+            if (facts.receipt(participant, candidate.index) !== undefined)
+                return true;
+            const lastJoin = facts.participant(participant)?.lastJoinIndex;
+            return lastJoin !== undefined && lastJoin > candidate.index;
+        }))).value;
+    }
+    catch {
+        return undefined;
+    }
+    finally {
+        await artifact.close();
+    }
+}
+/** Command/application entry: root selection uses the caller's captured environment. */
+export function gcHostLedger(options = {}) {
+    return maintainHostLedger(hostLedgerForEnv(options.env ?? process.env), options);
+}
+export async function maintainHostLedger(ledger, options = {}) {
+    const dryRun = options.dryRun === true;
+    const now = options.now ?? Date.now();
+    const records = await ledger.listEvidence({ includeReleased: true, includeExpired: true, now });
+    const groups = new Map();
+    for (const record of records) {
+        if (!isCandidate(record, now))
+            continue;
+        const index = activityIndex(record.activity);
+        if (index === undefined)
+            continue;
+        const location = await canonicalPath(record.location).catch(() => undefined);
+        if (location === undefined)
+            continue;
+        const group = groups.get(location);
+        if (group === undefined)
+            groups.set(location, [{ record, index }]);
+        else
+            group.push({ record, index });
+    }
+    let eligible = 0;
+    let retired = 0;
+    let casMisses = 0;
+    const unavailable = new Set();
+    for (const location of [...groups.keys()].sort()) {
+        const closable = await provenClosable(location, groups.get(location));
+        if (closable === undefined) {
+            unavailable.add(location);
+            continue;
+        }
+        eligible += closable.length;
+        if (dryRun)
+            continue;
+        for (const candidate of closable) {
+            if (await ledger.retireEvidenceIfUnchanged(candidate.record))
+                retired += 1;
+            else
+                casMisses += 1;
+        }
+    }
+    return { dryRun, eligible, retired, casMisses, unavailableArtifacts: [...unavailable].sort() };
+}

package/dist/ports.d.ts

@@ -1,5 +1,5 @@
 import type { Act, HardCap, InboxNotification, StoredAct, WakeRoute, WakeRouteKind, WatchLease } from './model.js';
-export type { ClaimResult, EvidenceClaim, EvidenceRelease, EvidenceGc, EvidenceLookup, EvidenceRecord, HostLedgerPort, PresenceKey, PresenceLookup, PresenceRecord, PresenceClaimResult, PresenceChannel, PresenceResult, } from './host-ledger.js';
+export type { ClaimResult, EvidenceClaim, EvidenceRelease, EvidenceLookup, EvidenceRecord, NativeTokenLookup, NativeTokenResult, HostLedgerPort, PresenceKey, PresenceLookup, PresenceRecord, PresenceClaimResult, PresenceChannel, PresenceResult, } from './host-ledger.js';
 import type { HostLedgerPort } from './host-ledger.js';
 export type ActivityIndex = number;
 export interface SquareHeader {
@@ -20,6 +20,11 @@ export interface ParticipantFact {
     readonly lastActiveAt?: number;
     readonly lastJoinIndex?: number;
 }
+export interface ParticipantCounts {
+    readonly known: number;
+    readonly joined: number;
+    readonly done: number;
+}
 export interface AttentionFact {
     readonly listener: string;
     readonly target: string;
@@ -98,6 +103,7 @@ export interface RouteFact {
 export interface SquareRead {
     header(): SquareHeader;
     participant(requestedName: string): ParticipantFact | undefined;
+    participantCounts(): ParticipantCounts;
     roster(): readonly ParticipantFact[];
     attention(listener: string): readonly AttentionFact[];
     incomingAttention(sender: string): readonly AttentionFact[];

package/dist/runtime.d.ts

@@ -1,6 +1,6 @@
 import { type ActivityId } from './square-core.js';
 import { type StoredAct, type WatchLease } from './model.js';
-import type { SquareRead } from './ports.js';
+import type { ParticipantCounts, ParticipantFact, SquareRead } from './ports.js';
 export declare const WATCH_HEARTBEAT_MS: number;
 export declare const SLEEP_MS: number;
 export declare const STALE_MS: number;
@@ -13,6 +13,7 @@ export declare function nowMs(): number;
 export declare function rosterNames(facts: SquareRead): string[];
 export declare function inSquareCount(facts: SquareRead): number;
 export declare function actId(actOrIndex: StoredAct | number): ActivityId;
+export declare function terminalStatusFor(current: ParticipantFact | undefined, hardCap: number | null, counts: ParticipantCounts): 'capped' | 'quorum' | undefined;
 export declare function watchTerminalStatus(facts: SquareRead, name: string): 'capped' | 'quorum' | undefined;
 /** A continuous prefix stops at the first unseen historical audience edge after join. */
 export declare function readCursor(facts: SquareRead, name: string): number;

package/dist/runtime.js

@@ -1,5 +1,5 @@
 import { formatActivityId } from './square-core.js';
-import { SquareError, sameName } from './model.js';
+import { SquareError } from './model.js';
 function parseIntegerEnv(name, fallback) {
     const raw = process.env[name];
     if (raw === undefined)
@@ -27,14 +27,20 @@ export function nowMs() {
     return value;
 }
 export function rosterNames(facts) { return facts.roster().map((participant) => participant.name); }
-export function inSquareCount(facts) { return facts.roster().filter((participant) => participant.joined).length; }
+export function inSquareCount(facts) { return facts.participantCounts().joined; }
 export function actId(actOrIndex) { return formatActivityId(typeof actOrIndex === 'number' ? actOrIndex : actOrIndex.index); }
+export function terminalStatusFor(current, hardCap, counts) {
+    if (hardCap !== null && (current?.sayCount ?? 0) >= hardCap)
+        return 'capped';
+    const peers = counts.known - Number(current !== undefined);
+    const donePeers = counts.done - Number(current?.done === true);
+    return peers > 0 && donePeers === peers ? 'quorum' : undefined;
+}
 export function watchTerminalStatus(facts, name) {
-    const cap = facts.header().hardCap;
-    if (cap !== null && (facts.participant(name)?.sayCount ?? 0) >= cap)
+    const cap = facts.header().hardCap, current = facts.participant(name);
+    if (cap !== null && (current?.sayCount ?? 0) >= cap)
         return 'capped';
-    const peers = facts.roster().filter((participant) => !sameName(participant.name, name));
-    return peers.length > 0 && peers.every((peer) => peer.done) ? 'quorum' : undefined;
+    return terminalStatusFor(current, cap, facts.participantCounts());
 }
 /** A continuous prefix stops at the first unseen historical audience edge after join. */
 export function readCursor(facts, name) {

package/dist/sqlite-transaction.d.ts

@@ -0,0 +1,5 @@
+import type { DatabaseSync } from 'node:sqlite';
+/** Yield rather than blocking the process while another connection owns SQLite. */
+export declare function retrySqliteBusy<T>(operation: () => T, signal?: AbortSignal, retryMs?: number | (() => number)): Promise<T>;
+/** Acquisition may retry; COMMIT retries retain the callback's result and transaction. */
+export declare function inSqliteTransaction<T, V>(db: DatabaseSync, write: boolean, validate: () => V, change: (validated: V) => T, signal?: AbortSignal): Promise<T>;

package/dist/sqlite-transaction.js

@@ -0,0 +1,72 @@
+import { setTimeout as sleep } from 'node:timers/promises';
+function abort(signal) {
+    if (signal?.aborted)
+        throw signal.reason ?? new Error('SQLite operation aborted');
+}
+function isBusy(error) {
+    return error instanceof Error && /database is locked|database is busy|SQLITE_BUSY/i.test(error.message);
+}
+/** Yield rather than blocking the process while another connection owns SQLite. */
+export async function retrySqliteBusy(operation, signal, retryMs = 25) {
+    for (;;) {
+        abort(signal);
+        try {
+            return operation();
+        }
+        catch (error) {
+            if (!isBusy(error))
+                throw error;
+            try {
+                await sleep(typeof retryMs === 'function' ? retryMs() : retryMs, undefined, { signal });
+            }
+            catch (error) {
+                abort(signal);
+                throw error;
+            }
+        }
+    }
+}
+/** Acquisition may retry; COMMIT retries retain the callback's result and transaction. */
+export async function inSqliteTransaction(db, write, validate, change, signal) {
+    let begun = false, backoffMs = 25;
+    // Spread writer contenders out without delaying a writer that already owns the lock.
+    const writerDelay = () => {
+        const delay = 1 + Math.floor(Math.random() * backoffMs);
+        backoffMs = Math.min(100, backoffMs * 2);
+        return delay;
+    };
+    try {
+        const validated = await retrySqliteBusy(() => {
+            db.exec(write ? 'BEGIN IMMEDIATE' : 'BEGIN');
+            begun = true;
+            try {
+                return validate();
+            }
+            catch (error) {
+                db.exec('ROLLBACK');
+                begun = false;
+                throw error;
+            }
+        }, signal, write ? writerDelay : 25);
+        abort(signal);
+        const result = change(validated);
+        if (result !== null && (typeof result === 'object' || typeof result === 'function') && typeof result.then === 'function') {
+            void Promise.resolve(result).catch(() => undefined);
+            throw new TypeError('SQLite transaction callbacks must be synchronous.');
+        }
+        // A busy commit still owns the writer lock. Retry promptly once readers leave;
+        // acquisition contenders keep the longer backoff above.
+        await retrySqliteBusy(() => db.exec('COMMIT'), signal, 1);
+        begun = false;
+        return result;
+    }
+    catch (error) {
+        if (begun) {
+            try {
+                db.exec('ROLLBACK');
+            }
+            catch { /* retain original failure */ }
+        }
+        throw error;
+    }
+}

package/dist/square-actions.js

@@ -2,7 +2,7 @@ import { coreDone, coreHold, coreIgnore, coreListen, coreListening, coreResume,
 import { isSquareError, nameKey, SquareError, validateName } from './model.js';
 import { participantIdentity } from './participant-identity.js';
 import { deliverPending } from './delivery-operations.js';
-import { decideCatch } from './catch-decisions.js';
+import { decideCatch, observeCatchSelection, selectCatchPage } from './catch-decisions.js';
 import { inSquareCount } from './runtime.js';
 import { claimSessionParticipant, claimSessionTakeover, presenceEpoch, releaseSessionParticipantClaim, readParticipantOwner } from './registry.js';
 import { assertLiveOwner, ensureLocalPresence, identityRouteDraft, processIdentity, publishCommittedIdentity, retireIdentityRoute } from './participant-host.js';
@@ -14,13 +14,21 @@ export async function catchUp(square, name, options = {}, control, leaseId) {
     throwIfAborted(control);
     if (!await assertLiveOwner(square, name))
         throw new SquareError('already_joined', `${participantIdentity(name)} already stands here — another session holds the name`);
-    const selected = (await square.artifact.transact((facts) => {
-        // Replacement and selection share the receipt transaction: an old waiter cannot
-        // consume a later activity between its heartbeat and its selection.
-        if (leaseId !== undefined && facts.lease(name)?.leaseId !== leaseId)
-            return null;
-        return decideCatch(facts, name, options, square.clock());
-    }, control?.signal)).value;
+    let selected;
+    if (leaseId === undefined && options.id === undefined) {
+        selected = (await square.artifact.read((facts) => {
+            const page = selectCatchPage(facts, name, options);
+            return page.indexes.length === 0 ? observeCatchSelection(facts, page.participant, [], page.filter, square.clock()) : undefined;
+        }, control?.signal)).value;
+    }
+    if (selected === undefined)
+        selected = (await square.artifact.transact((facts) => {
+            // Replacement and selection share the receipt transaction: an old waiter cannot
+            // consume a later activity between its heartbeat and its selection.
+            if (leaseId !== undefined && facts.lease(name)?.leaseId !== leaseId)
+                return null;
+            return decideCatch(facts, name, options, square.clock());
+        }, control?.signal)).value;
     if (selected !== null)
         await publishCommittedIdentity(square, selected.participant);
     return selected;

package/dist/square-migration.js

@@ -1,4 +1,5 @@
-import { inSqliteTransaction, openExistingSquareDatabase } from './artifact.js';
+import { openExistingSquareDatabase } from './artifact.js';
+import { inSqliteTransaction } from './sqlite-transaction.js';
 import { canonicalPath } from './canonical-path.js';
 import { isWakeRouteKind } from './model.js';
 import { parseActivityId } from './square-core.js';

package/dist/square-sqlite.d.ts

@@ -1,6 +1,6 @@
 import { type DatabaseSync } from 'node:sqlite';
 import { SquareError, type ActivityObservation, type HardCap, type StoredAct, type WakeRoute, type WatchLease } from './model.js';
-import type { SquareRead, RouteFact } from './ports.js';
+import type { SquareHeader, SquareRead, RouteFact } from './ports.js';
 import { type LandingWriter } from './square-domain.js';
 /** Explicit import/export DTOs, not a runtime transaction model. */
 export interface SquareRuntimeState {
@@ -43,7 +43,7 @@ export declare function invalidArtifact(detail: string): SquareError;
 export declare function validateStoredActivity(value: unknown): StoredAct;
 export declare function validateWatchLease(value: unknown): WatchLease;
 export declare function validateRoute(value: unknown): RouteFact;
-export declare function validateSqliteHeader(db: DatabaseSync, location?: string): void;
+export declare function validateSqliteHeader(db: DatabaseSync, location?: string): SquareHeader;
 export interface SqliteDriver extends SquareRead, LandingWriter {
     readonly changed: boolean;
 }

package/dist/square-sqlite.js

@@ -141,6 +141,7 @@ CREATE TABLE landed_audience (
   UNIQUE (act_index, recipient_order)
 ) WITHOUT ROWID;
 CREATE INDEX audience_recipient ON landed_audience(recipient_key_json, act_index);`;
+const EXPECTED_SCHEMA_SQL = SCHEMA_SQL.split(';').map(s => s.trim()).filter(Boolean).map(normalizedSql).sort();
 const encode = JSON.stringify;
 const key = (name) => encode(nameKey(name));
 export function invalidArtifact(detail) {
@@ -330,11 +331,10 @@ export function validateSqliteHeader(db, location = '') {
         throw new SquareError('invalid_args', `Square artifact needs migration.\n${migrationRecoveryCommand(location)}`, { reason: 'migration_required' });
     }
     requireFact(identity.user_version === 2, 'unsupported SQLite format version.');
-    const expected = SCHEMA_SQL.split(';').map(s => s.trim()).filter(Boolean).map(normalizedSql).sort();
     const actual = db.prepare("SELECT sql FROM sqlite_schema WHERE name NOT GLOB 'sqlite_*' AND sql IS NOT NULL").all().map(r => normalizedSql(string(r.sql))).sort();
-    requireFact(actual.length === expected.length && actual.every((sql, i) => sql === expected[i]), 'unsupported row schema.');
+    requireFact(actual.length === EXPECTED_SCHEMA_SQL.length && actual.every((sql, i) => sql === EXPECTED_SCHEMA_SQL[i]), 'unsupported row schema.');
     requireFact(db.prepare('PRAGMA journal_mode').get().journal_mode === (location === ':memory:' ? 'memory' : 'delete'), 'unsupported journal mode.');
-    createSqliteDriver(db).header();
+    return createSqliteDriver(db).header();
 }
 function activitySelection(filter, alias = 'a') {
     const terms = [], args = [];
@@ -418,6 +418,10 @@ export function createSqliteDriver(db) {
             const r = one('SELECT * FROM participants WHERE name_key_json=?', [key(name)]);
             return r ? participant(r) : undefined;
         },
+        participantCounts() {
+            const r = one('SELECT COUNT(*) AS known, COALESCE(SUM(joined),0) AS joined, COALESCE(SUM(done),0) AS done FROM participants');
+            return immutable({ known: integer(r.known), joined: integer(r.joined), done: integer(r.done) });
+        },
         roster: () => immutable(all('SELECT * FROM participants ORDER BY first_index').map(participant)),
         attention: name => immutable(attentionRows('t.listener_key_json=?', [key(name)])),
         incomingAttention: name => immutable(attentionRows('t.target_key_json=?', [key(name)])),
@@ -468,7 +472,14 @@ export function createSqliteDriver(db) {
         sayOrdinals(indexes) {
             indexes.forEach(v => integer(v));
             const result = {};
-            for (const r of all("SELECT a.act_index,(SELECT COUNT(*) FROM activities s WHERE s.kind='say' AND s.actor_key_json=a.actor_key_json AND s.act_index<=a.act_index) AS n FROM activities a WHERE a.kind='say' AND a.act_index IN (SELECT value FROM json_each(?))", [encode(indexes)]))
+            // Only an actor's latest say can use its authoritative lifetime count.
+            for (const r of all(`SELECT a.act_index, CASE
+                WHEN p.say_count IS NOT NULL AND a.act_index=(SELECT MAX(s.act_index) FROM activities s WHERE s.kind='say' AND s.actor_key_json=a.actor_key_json)
+                THEN p.say_count
+                ELSE (SELECT COUNT(*) FROM activities s WHERE s.kind='say' AND s.actor_key_json=a.actor_key_json AND s.act_index<=a.act_index)
+                END AS n
+                FROM activities a LEFT JOIN participants p ON p.name_key_json=a.actor_key_json
+                WHERE a.kind='say' AND a.act_index IN (SELECT value FROM json_each(?))`, [encode(indexes)]))
                 result[integer(r.act_index)] = integer(r.n, 1);
             return immutable(result);
         },

package/guides/catch-contention-performance.md

@@ -0,0 +1,157 @@
+# Catch contention: commit retries, empty reads, and shorter writes
+
+A busy SQLite COMMIT retains the write transaction while waiting for readers to leave. Previously it slept 25 ms between attempts, the same as contenders waiting to acquire a transaction. The retained writer also prevents those contenders from progressing.
+
+The change retries COMMIT after 1 ms while retaining the existing 25 ms acquisition backoff. It preserves the transaction, callback result, cancellation and rollback behavior. The shared helper applies to both artifact and host-ledger transactions. No journal mode, receipt semantics, workload, cache or background service changed.
+
+## Measurement
+
+The benchmark now times opening, recognition, first catch, second catch and closing separately. Both baseline and candidate used this instrumentation and the same command:
+
+```sh
+node --import ./test/sandbox-env.js scripts/benchmark-idle-wait.mjs --workers 300 --mode session --active --no-hints
+```
+
+Three sequential runs per implementation, each covering both history sizes. Baseline production is `f44f359`; candidate changes only COMMIT retry timing. Baselines ran before candidates, not in randomized order. These local measurements show improvement but are not confidence intervals or a universal latency guarantee.
+
+Values below are the **median of three run-level measurements**. For latency, each measurement is the p95 across 300 workers.
+
+| Metric | Before, 1,800 history | After, 1,800 | Before, 18,000 | After, 18,000 |
+| --- | ---: | ---: | ---: | ---: |
+| First catch API p95 (ms) | 6,596.63 | 4,619.84 | 7,302.29 | 4,781.38 |
+| Second catch API p95 (ms) | 3,620.93 | 2,413.91 | 4,034.01 | 2,613.97 |
+| Whole catch phase p95 (ms) | 7,262.52 | 5,464.51 | 8,126.17 | 5,653.63 |
+| Aggregate phase CPU (ms) | 23,466.80 | 20,205.76 | 27,313.38 | 21,241.62 |
+| Artifact busy observations | 47,578 | 30,824 | 53,418 | 32,265 |
+| Artifact COMMIT busy observations | 95 | 138 | 85 | 117 |
+
+First-catch p95 medians fell approximately 30% and 35%; whole-phase medians fell 25% and 30%. Aggregate CPU medians fell 14% and 22%. COMMIT retry counts increased, as expected from polling the retained writer sooner, while total busy observations and CPU medians decreased.
+
+Run-level first-catch p95s, in execution order:
+
+| History | Before (ms) | After (ms) |
+| --- | --- | --- |
+| 1,800 | 6,119.17 / 6,596.63 / 6,680.30 | 4,619.84 / 4,233.23 / 6,642.77 |
+| 18,000 | 7,785.32 / 7,302.29 / 5,966.01 | 4,781.38 / 4,995.59 / 4,694.29 |
+
+The third small-history candidate run was slower than its other two runs and overlapped the baseline range. Its whole-phase p95 was 7,602.57 ms, above all three small-history baseline runs. Contention and machine-load variability remain; this change does not remove multi-second waiting under 300 simultaneous processes.
+
+All twelve scenarios discovered 300 participants, consumed one activity each and consumed none again. All stderr files were empty. First-catch timing covers the full awaited API call, including ownership checks and postcommit identity publication. Whole-phase timing additionally includes open, recognize, the second catch and close; percentiles of individual steps must not be added.
+
+Raw local reports: `/tmp/square-catch-baseline-{1,2,3}.json` and `/tmp/square-catch-commit-{1,2,3}.json`. These measure isolated local storage, not native delivery or ordinary single-user latency.
+
+## Correctness boundaries
+
+Existing SQLite lifecycle tests exercise real blocked COMMITs, single callback execution, event-loop progress, cancellation rollback and cross-process receipt exclusion. Cancellation tests cover precommit rollback and postcommit success. The COMMIT-only measurements above retain the empty-catch write transaction. The subsequent optimization below measures its removal separately.
+
+## Follow-up: empty catches and repeated observation work
+
+Ordinary non-ID, non-leased catches now inspect pending indexes in a read transaction. An empty page returns its complete observation from that snapshot without acquiring the artifact writer or hydrating bodies. A nonempty page is selected again inside the existing receipt transaction; the probe never decides which activities get consumed. Exact-ID and leased catches keep their existing write path. Ownership checks and identity publication remain in place.
+
+Catch observations now reuse one roster and header for standing count and terminal status. Schema validation precomputes only the expected SQL constant; actual schema, identity, version, journal mode, and header checks still run inside each transaction. These three changes were measured together, so the results do not isolate each change's contribution.
+
+The same benchmark command ran three times on baseline `03623cf`, then three times on the candidate, covering both histories each time. These are fresh baselines, separate from the earlier COMMIT experiment. No workload ran alongside the benchmarks. As above, ordering was sequential, not randomized, and results describe this local 300-process workload.
+
+| Metric | Before, 1,800 history | After, 1,800 | Before, 18,000 | After, 18,000 |
+| --- | ---: | ---: | ---: | ---: |
+| First catch API p95 (ms) | 5,681.03 | 3,729.06 | 4,967.29 | 3,654.90 |
+| Second, empty catch API p95 (ms) | 2,868.83 | 79.80 | 2,608.11 | 55.08 |
+| Whole catch phase p95 (ms) | 6,315.96 | 4,111.88 | 5,701.10 | 3,887.74 |
+| Aggregate phase CPU (ms) | 22,441.95 | 17,401.83 | 21,456.88 | 16,184.47 |
+| Artifact busy observations | 34,542 | 18,780 | 32,986 | 18,638 |
+| Artifact COMMIT busy observations | 137 | 323 | 148 | 287 |
+
+Each cell is the median of three run-level measurements. Empty-catch p95 medians fell 97% and 98%; first-catch medians fell 34% and 26%; whole-phase medians fell 35% and 32%; CPU medians fell 22% and 25%. Total busy observations fell, while COMMIT busy observations increased. The extra read probe changes contention; fewer writer acquisitions do not imply every busy counter falls.
+
+Run-level p95s in execution order:
+
+| History | Call | Before (ms) | After (ms) |
+| --- | --- | --- | --- |
+| 1,800 | First | 7,655.42 / 4,857.49 / 5,681.03 | 4,274.17 / 3,729.06 / 3,557.04 |
+| 18,000 | First | 4,967.29 / 5,029.40 / 4,895.89 | 5,005.24 / 3,356.44 / 3,654.90 |
+| 1,800 | Empty | 4,495.46 / 2,348.03 / 2,868.83 | 56.35 / 79.80 / 82.83 |
+| 18,000 | Empty | 3,139.28 / 2,506.64 / 2,608.11 | 53.31 / 55.08 / 56.97 |
+
+The first large-history candidate's consuming catch overlaps the baseline range. Consuming catches still wait seconds under this load; this is not a guarantee of millisecond catch latency generally. The second catch measures an empty result immediately after consumption, not a long-running idle lease.
+
+All twelve scenarios discovered 300 participants, consumed exactly one activity per worker, and consumed none again: 3,600 successful worker results. Each scenario hydrated 300 bodies and issued 600 artifact SQL writes. Empty baseline transactions already wrote no rows; the improvement removes writer acquisition, not receipt writes. Successful artifact transactions per scenario increase from 1,800 to 2,100 because nonempty catches add a read probe. All stderr files were empty.
+
+Focused tests exercise empty snapshot completeness, absence of writer acquisition/body hydration/receipts, a competing consumer between probe and write selection, exact-ID repetition, lease fencing, validation, cancellation, and ownership publication. The existing performance assertion now checks that an empty catch makes no hydration request.
+
+Raw local reports: `/tmp/square-catch-empty-before-{1,2,3}.json` and `/tmp/square-catch-empty-after-{1,2,3}.json`. The final cap-first short circuit in `watchTerminalStatus()` restores its earlier read behavior; it is used by lease pulses and lies outside this benchmark's non-idle path.
+
+## Follow-up: shortening consuming transactions
+
+The next candidate adds transaction-local participant counts instead of decoding the whole roster for catch/terminal summaries. Latest-say ordinals use the existing participant lifetime count when the selected activity is that actor's latest say; older activities retain the indexed prefix count. Artifact acquisition also passes its already-validated header to the transaction callback instead of reading and decoding it again. Schema validation still occurs inside every transaction. There is no persistent schema change or cross-transaction cache.
+
+The benchmark now measures explicit transaction boundaries per connection and separates artifact from host-ledger connections:
+
+- Acquisition: first BEGIN attempt to successful BEGIN return, including retry gaps.
+- Hold: successful BEGIN return to first COMMIT attempt, including validation, decoding and off-CPU wall time.
+- Commit: first COMMIT attempt to successful COMMIT return, including retries.
+
+Only committed transactions contribute timing samples. Rollback resets the sample; unsuccessful read acquisition/validation retries are visible in counts but do not become committed timings. `BEGIN DEFERRED` returning does not imply a read lock has already been acquired. Concurrent read durations cannot be summed into a serial lock-occupancy number. Batched transaction controls without observable inner boundaries are counted separately and excluded from exact timings. Artifact SQL execution durations exclude preparation and JavaScript decoding. Bounded self-tests exercise busy BEGIN, busy COMMIT, rollback/reset and origin separation.
+
+Fresh baseline `13d1a36` and candidate each ran three times with the same new instrumentation and the command above, both histories per run. Baselines ran first; no test suite or second benchmark ran alongside the measurements. These changes were measured together, not as independent experiments.
+
+| Metric | Before, 1,800 history | After, 1,800 | Before, 18,000 | After, 18,000 |
+| --- | ---: | ---: | ---: | ---: |
+| First catch API p95 (ms) | 3,528.76 | 2,678.86 | 3,685.07 | 2,865.71 |
+| Empty catch API p95 (ms) | 68.23 | 89.97 | 58.31 | 91.11 |
+| Whole catch phase p95 (ms) | 3,831.22 | 2,998.22 | 4,165.47 | 3,177.27 |
+| Write acquisition p95 (ms) | 2,739.46 | 1,950.57 | 2,961.04 | 2,257.01 |
+| Write hold p95 (ms) | 8.33 | 5.32 | 10.05 | 5.88 |
+| Write commit p95 (ms) | 4.15 | 3.81 | 4.36 | 4.24 |
+| Aggregate phase CPU (ms) | 16,348.86 | 13,014.67 | 17,732.24 | 13,499.75 |
+| Artifact busy observations | 16,201 | 13,081 | 20,023 | 12,508 |
+
+Cells are medians of three run-level measurements. Each scenario has 300 successful consuming write transactions, so write percentiles span those 300 transactions. First-catch p95 medians fall 24% and 22%; whole-phase medians fall 22% and 24%; CPU falls 20% and 24%. Write acquisition falls 29% and 24%, and hold p95 falls 36% and 42%. Percentiles of these stages must not be added to reconstruct the API percentile.
+
+Empty-catch p95 regresses by 22 ms and 33 ms (32% and 56%) in this contended workload. The faster consuming path does not improve every operation. The current measurements do not isolate the cause of this regression; empty catches still use a read transaction, and the 25 ms acquisition retry interval is unchanged. The consuming-call improvement outweighs it in whole-phase time, but the regression remains part of the result.
+
+| History | Call | Before p95 runs (ms) | After p95 runs (ms) |
+| --- | --- | --- | --- |
+| 1,800 | First | 3,572.79 / 3,123.90 / 3,528.76 | 2,361.69 / 2,732.76 / 2,678.86 |
+| 18,000 | First | 3,577.88 / 3,685.07 / 4,165.72 | 2,865.71 / 2,975.57 / 2,813.31 |
+| 1,800 | Empty | 68.23 / 85.81 / 61.32 | 100.68 / 89.97 / 87.47 |
+| 18,000 | Empty | 57.38 / 59.12 / 58.31 | 108.55 / 83.11 / 91.11 |
+
+Every candidate first-catch p95 is below its history's baseline range, but these sequential local runs are not confidence intervals or a general latency guarantee. Acquisition still takes seconds near the tail under 300 simultaneous processes. Some individual hold/commit samples are much longer than their p95; wall-time instrumentation alone cannot attribute all of that time to SQL or process scheduling.
+
+All twelve scenarios discovered 300 participants, consumed one activity each, and consumed none again: 3,600 successful worker results. Each scenario retains 300 hydrated bodies, 600 SQL writes, 300 committed artifact writes and 1,800 committed artifact reads. All stderr files were empty. Tests cover aggregate read-your-writes and rollback, known-but-not-standing peers, missing/done caller quorum, cap precedence, and ordinal numbering across aliases, gaps, rejoins and subsequent writes.
+
+Raw local reports: `/tmp/square-catch-positive-before-{1,2,3}.json` and `/tmp/square-catch-positive-after-{1,2,3}.json`. Earlier reports lack these transaction timing fields and should not be mixed into this comparison.
+
+## Follow-up: spreading writer acquisition retries
+
+Writer acquisition now uses a per-transaction backoff window of 25, 50, then at most 100 ms. Each busy retry independently chooses an integer delay from 1 through that window. Read acquisition and other default retries remain fixed at 25 ms; a busy COMMIT still retries after 1 ms while retaining the writer. The shared helper applies this policy to artifact and host-ledger writes. Cancellation, rollback, synchronous callbacks, receipt selection and storage framing are unchanged.
+
+This changes both retry frequency and synchronization, not jitter alone. At the cap the mean requested delay is 50.5 ms rather than 25 ms. The intent is to reduce contender overhead, not to pretend queue waiting is idle time: serial write hold plus commit already accounts for most of the previous catch duration. Wall-time measurements cannot prove how much of the benefit comes from scheduling or filesystem contention. There is no fairness guarantee; an individual contended retry can sleep longer than before. Uncontended attempts do not sleep.
+
+Baseline `f004621` and the isolated writer-only candidate each ran three times with the same instrumentation and command above. Execution order was baseline 1, three rejected schema-normalization experiments, baselines 2–3, one broader retry pilot, then writer-only candidates 1–3. No build, test suite or second benchmark ran alongside a benchmark. This was not a randomized comparison; prior tables are separate experiments and should not be used as its baseline. The schema-normalization fast path showed no consistent benefit and was removed. The broader pilot is excluded from the results below.
+
+| Metric | Before, 1,800 history | After, 1,800 | Before, 18,000 | After, 18,000 |
+| --- | ---: | ---: | ---: | ---: |
+| First catch API p95 (ms) | 2,942.43 | 2,696.24 | 3,314.33 | 2,703.95 |
+| Empty catch API p95 (ms) | 114.52 | 83.35 | 107.99 | 85.06 |
+| Whole catch phase p95 (ms) | 3,281.04 | 3,089.50 | 3,797.85 | 3,008.00 |
+| Write acquisition p95 (ms) | 2,460.38 | 2,044.42 | 2,441.14 | 2,114.18 |
+| Write hold p95 (ms) | 6.43 | 5.46 | 7.20 | 5.39 |
+| Write commit p95 (ms) | 4.72 | 3.87 | 4.64 | 3.50 |
+| Aggregate phase CPU (ms) | 14,791.25 | 12,797.97 | 16,050.74 | 12,994.48 |
+| Artifact busy observations | 14,311 | 8,943 | 15,203 | 8,966 |
+
+Cells are medians of three run-level measurements. First-catch p95 medians improve 8% and 18%, whole-phase medians 6% and 21%, and CPU 13% and 19%. Empty-catch p95 medians improve 27% and 21%; total busy observations fall 38% and 41%. Median run-level maximum write acquisition falls from 2,835.81 to 2,626.85 ms and from 3,064.84 to 2,656.63 ms; this is not a bounded maximum or a starvation guarantee.
+
+| History | Call | Before p95 runs (ms) | After p95 runs (ms) |
+| --- | --- | --- | --- |
+| 1,800 | First | 3,518.32 / 2,942.43 / 2,797.10 | 2,479.32 / 2,696.24 / 2,829.63 |
+| 18,000 | First | 2,791.81 / 3,314.33 / 3,457.89 | 2,576.67 / 2,759.20 / 2,703.95 |
+| 1,800 | Empty | 114.52 / 114.76 / 100.27 | 83.35 / 84.59 / 82.22 |
+| 18,000 | Empty | 118.99 / 107.99 / 63.28 | 108.44 / 85.06 / 61.79 |
+
+Small-history consuming and whole-phase ranges overlap; large-history consuming ranges do not. Not every tail metric improves: the large-history median of each run's maximum COMMIT duration rises from 506.88 to 920.92 ms, despite the lower COMMIT p95. These local three-run results show lower contention overhead, not a universal latency improvement. Consuming catch still takes seconds near the tail with 300 simultaneous processes.
+
+All twelve scenarios discovered 300 participants, consumed exactly one activity each and consumed none again: 3,600 successful worker results. Each retains 300 hydrated bodies, 600 SQL writes, 300 committed artifact writes and 1,800 committed artifact reads. All stderr files were empty. A new real-lock test checks cancellation before writer acquisition and successful single-callback execution on a subsequent transaction; existing tests retain busy COMMIT cancellation, same-result retries and cross-process receipt exclusion.
+
+Raw local reports: `/tmp/square-lock-before-{1,2,3}.json` and `/tmp/square-lock-writer-{1,2,3}.json`. Discarded experiments are `/tmp/square-lock-schema-{1,2,3}.json` and `/tmp/square-lock-jitter-1.json`.

package/guides/change-waits.md

@@ -4,7 +4,7 @@ Square's artifact and host-ledger adapters own change detection. Notifications a
 
 - A process shares a revision reader per canonical artifact and a filesystem watcher per containing directory. Subscribers have independent baselines, deadlines, and cancellation. The final release closes the watcher, reconciliation timer, and any outstanding read.
 - Directory notifications invalidate lightweight reads. A two-second reconciliation checks revisions/ledger metadata even when notifications are missing. Missing directories, unsupported watches, and watcher failures retain that fallback; directory replacement is detected during reconciliation. Normal notification latency is not a hard guarantee: a missed hint may wait for reconciliation, plus scheduling and lock contention.
-- The artifact revision remains authoritative. No new sidecar, broker, or persistent cache is introduced. Changed projections read typed recipient facts and selected bodies, not the whole historical state. A missing artifact is a stable observation, not a repeated change: reconciliation remains quiet while it stays absent and detects its return without deleting its session bindings. Strict artifact reads still report absence as an error. Ledger metadata invalidates bindings and presentation projections; it is never treated as evidence of delivery.
+- The artifact revision remains authoritative. No new sidecar, broker, or persistent cache is introduced. Changed projections read typed recipient facts and selected bodies, not the whole historical state. A missing artifact is a stable observation, not a repeated change: reconciliation remains quiet while it stays absent and detects its return without deleting its session bindings. Strict artifact reads still report absence as an error. Host-ledger database metadata invalidates a process-shared read of its separate presence/evidence revisions; unchanged metadata requires no SQLite open. Revisions are read together in one short snapshot and cached against metadata captured before that read, so a concurrent mutation remains detectable. Failed reads are not cached. Missing ledger storage is stable until it appears. Neither metadata nor revisions are delivery evidence.
 - Each Pi session owns a long-lived pending observer, including when it has no bindings yet. Presence and evidence changes update its subscriptions. There is no repeated thirty-second restart and no per-second binding scan. A failed injection waits for a substantive change; its own failed evidence write does not trigger a retry loop.
 - Blocking CLI catch waits for a revision or its next heartbeat/quiet deadline. A held catch keeps its lease alive and pauses quiet expiry. Core catch, NDJSON stream, and delivery-receipt waits use the same change detector. Held express also waits for changes; throttling still waits for its eligibility time.
 - Catch lease expiry, ledger retention/claim expiry, and cancellation are independent wake conditions. No filesystem write is required for a time-based eligibility change.