@optimystic/db-p2p 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/cluster-policy.d.ts +13 -2
- package/dist/src/cluster/cluster-policy.d.ts.map +1 -1
- package/dist/src/cluster/cluster-policy.js +51 -4
- package/dist/src/cluster/cluster-policy.js.map +1 -1
- package/dist/src/cluster/cluster-repo.d.ts +3 -3
- package/dist/src/cluster/cluster-repo.js +3 -3
- package/dist/src/cluster/cluster-size-coupling.d.ts +28 -0
- package/dist/src/cluster/cluster-size-coupling.d.ts.map +1 -0
- package/dist/src/cluster/cluster-size-coupling.js +35 -0
- package/dist/src/cluster/cluster-size-coupling.js.map +1 -0
- package/dist/src/cluster/quorum-restore.d.ts +6 -0
- package/dist/src/cluster/quorum-restore.d.ts.map +1 -1
- package/dist/src/cluster/quorum-restore.js +1 -1
- package/dist/src/cluster/quorum-restore.js.map +1 -1
- package/dist/src/cluster/reconcile-block.d.ts.map +1 -1
- package/dist/src/cluster/reconcile-block.js +15 -3
- package/dist/src/cluster/reconcile-block.js.map +1 -1
- package/dist/src/index.d.ts +2 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +2 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/libp2p-key-network.d.ts +46 -5
- package/dist/src/libp2p-key-network.d.ts.map +1 -1
- package/dist/src/libp2p-key-network.js +40 -9
- package/dist/src/libp2p-key-network.js.map +1 -1
- package/dist/src/libp2p-node-base.d.ts +3 -2
- package/dist/src/libp2p-node-base.d.ts.map +1 -1
- package/dist/src/libp2p-node-base.js +834 -777
- package/dist/src/libp2p-node-base.js.map +1 -1
- package/dist/src/libp2p-node-rn.d.ts +2 -2
- package/dist/src/libp2p-node-rn.d.ts.map +1 -1
- package/dist/src/libp2p-node-rn.js.map +1 -1
- package/dist/src/libp2p-node.d.ts +2 -2
- package/dist/src/libp2p-node.d.ts.map +1 -1
- package/dist/src/libp2p-node.js.map +1 -1
- package/dist/src/network/network-manager-service.d.ts +2 -0
- package/dist/src/network/network-manager-service.d.ts.map +1 -1
- package/dist/src/network/network-manager-service.js +4 -0
- package/dist/src/network/network-manager-service.js.map +1 -1
- package/dist/src/optimystic-node.d.ts +35 -0
- package/dist/src/optimystic-node.d.ts.map +1 -0
- package/dist/src/optimystic-node.js +2 -0
- package/dist/src/optimystic-node.js.map +1 -0
- package/dist/src/repo/coordinator-repo.d.ts +16 -5
- package/dist/src/repo/coordinator-repo.d.ts.map +1 -1
- package/dist/src/repo/coordinator-repo.js +24 -8
- package/dist/src/repo/coordinator-repo.js.map +1 -1
- package/dist/src/rn.d.ts +2 -0
- package/dist/src/rn.d.ts.map +1 -1
- package/dist/src/rn.js +2 -0
- package/dist/src/rn.js.map +1 -1
- package/dist/src/storage/block-storage.d.ts.map +1 -1
- package/dist/src/storage/block-storage.js +57 -5
- package/dist/src/storage/block-storage.js.map +1 -1
- package/dist/src/storage/i-block-storage.d.ts +20 -1
- package/dist/src/storage/i-block-storage.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.d.ts +56 -3
- package/dist/src/storage/storage-repo.d.ts.map +1 -1
- package/dist/src/storage/storage-repo.js +124 -18
- package/dist/src/storage/storage-repo.js.map +1 -1
- package/dist/src/testing/raw-storage-conformance.d.ts.map +1 -1
- package/dist/src/testing/raw-storage-conformance.js +17 -0
- package/dist/src/testing/raw-storage-conformance.js.map +1 -1
- package/package.json +2 -2
- package/readme.md +41 -26
- package/src/cluster/cluster-policy.ts +55 -4
- package/src/cluster/cluster-repo.ts +3 -3
- package/src/cluster/cluster-size-coupling.ts +45 -0
- package/src/cluster/quorum-restore.ts +1 -1
- package/src/cluster/reconcile-block.ts +15 -3
- package/src/index.ts +2 -0
- package/src/libp2p-key-network.ts +41 -9
- package/src/libp2p-node-base.ts +907 -847
- package/src/libp2p-node-rn.ts +2 -2
- package/src/libp2p-node.ts +2 -2
- package/src/network/network-manager-service.ts +5 -0
- package/src/optimystic-node.ts +36 -0
- package/src/repo/coordinator-repo.ts +24 -8
- package/src/rn.ts +2 -0
- package/src/storage/block-storage.ts +59 -6
- package/src/storage/i-block-storage.ts +20 -1
- package/src/storage/storage-repo.ts +129 -18
- package/src/testing/raw-storage-conformance.ts +20 -0
package/src/libp2p-node-rn.ts
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
import type { Libp2p } from 'libp2p';
|
|
2
1
|
import {
|
|
3
2
|
createLibp2pNodeBase,
|
|
4
3
|
type Libp2pTransports,
|
|
5
4
|
type NodeOptions,
|
|
6
5
|
type RawStorageProvider,
|
|
7
6
|
} from './libp2p-node-base.js';
|
|
7
|
+
import type { OptimysticNode } from './optimystic-node.js';
|
|
8
8
|
|
|
9
9
|
export type { Libp2pTransports, NodeOptions, RawStorageProvider };
|
|
10
10
|
|
|
@@ -14,7 +14,7 @@ export type { Libp2pTransports, NodeOptions, RawStorageProvider };
|
|
|
14
14
|
* This entrypoint intentionally does not import Node-only transports (like `@libp2p/tcp`).
|
|
15
15
|
* Callers must provide `options.transports` (and typically `options.listenAddrs`).
|
|
16
16
|
*/
|
|
17
|
-
export async function createLibp2pNode(options: NodeOptions): Promise<
|
|
17
|
+
export async function createLibp2pNode(options: NodeOptions): Promise<OptimysticNode> {
|
|
18
18
|
const transports = options.transports;
|
|
19
19
|
if (!transports || transports.length === 0) {
|
|
20
20
|
throw new Error(
|
package/src/libp2p-node.ts
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import type { Libp2p } from 'libp2p';
|
|
2
1
|
import { tcp } from '@libp2p/tcp';
|
|
3
2
|
import { webSockets } from '@libp2p/websockets';
|
|
4
3
|
import { circuitRelayTransport } from '@libp2p/circuit-relay-v2';
|
|
@@ -8,10 +7,11 @@ import {
|
|
|
8
7
|
type NodeOptions,
|
|
9
8
|
type RawStorageProvider,
|
|
10
9
|
} from './libp2p-node-base.js';
|
|
10
|
+
import type { OptimysticNode } from './optimystic-node.js';
|
|
11
11
|
|
|
12
12
|
export type { Libp2pTransports, NodeOptions, RawStorageProvider };
|
|
13
13
|
|
|
14
|
-
export async function createLibp2pNode(options: NodeOptions): Promise<
|
|
14
|
+
export async function createLibp2pNode(options: NodeOptions): Promise<OptimysticNode> {
|
|
15
15
|
const port = options.port ?? 0;
|
|
16
16
|
const wsHost = options.wsHost ?? '0.0.0.0';
|
|
17
17
|
|
|
@@ -58,6 +58,11 @@ export class NetworkManagerService implements Startable {
|
|
|
58
58
|
}
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
+
/** The cluster size this instance actually resolved to, for `assertClusterSizeCoupling`. */
|
|
62
|
+
get effectiveClusterSize(): number {
|
|
63
|
+
return this.cfg.clusterSize;
|
|
64
|
+
}
|
|
65
|
+
|
|
61
66
|
setLibp2p(libp2p: Libp2p): void {
|
|
62
67
|
this.libp2pRef = libp2p;
|
|
63
68
|
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { Libp2p, PrivateKey } from '@libp2p/interface';
|
|
2
|
+
import type { IBlockChangeNotifier, IRepo } from '@optimystic/db-core';
|
|
3
|
+
import type { DisputeService } from './dispute/dispute-service.js';
|
|
4
|
+
import type { Libp2pKeyPeerNetwork } from './libp2p-key-network.js';
|
|
5
|
+
import type { PeerReputationService } from './reputation/peer-reputation.js';
|
|
6
|
+
import type { StorageRepo } from './storage/storage-repo.js';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The handles `createLibp2pNodeBase` attaches to the libp2p node it returns. This is the
|
|
10
|
+
* sanctioned in-process surface a host reads — declared once here so reaching it does not
|
|
11
|
+
* require a cast, and so a host cannot silently rebuild a component the node already owns.
|
|
12
|
+
*
|
|
13
|
+
* Deliberately NOT the full set of `(node as any).*` attachments made in `libp2p-node-base.ts`:
|
|
14
|
+
* the churn/rebalance/ring-shift monitors, the cohort-topic host and the reactivity registries
|
|
15
|
+
* are node-internal wiring, not a host-facing surface, and typing them is a separate job.
|
|
16
|
+
*/
|
|
17
|
+
export interface OptimysticNodeAttachments {
|
|
18
|
+
/**
|
|
19
|
+
* The node's ONE key network — built from its resolved cluster policy, network-namespaced
|
|
20
|
+
* protocol prefix, reputation tracker and persistence. A host that needs key/coordinator
|
|
21
|
+
* lookup uses THIS; constructing a second one gives peer selection a different cohort
|
|
22
|
+
* width and coordinator than the node's own consensus path uses for the same key.
|
|
23
|
+
*/
|
|
24
|
+
keyNetwork: Libp2pKeyPeerNetwork;
|
|
25
|
+
coordinatedRepo: IRepo;
|
|
26
|
+
storageRepo: StorageRepo;
|
|
27
|
+
/** Per-collection change origin. Replaced by the cohort-topic bridge notifier when enabled. */
|
|
28
|
+
blockChangeNotifier: IBlockChangeNotifier;
|
|
29
|
+
reputation: PeerReputationService;
|
|
30
|
+
/** Present only when the dispute subsystem is configured. */
|
|
31
|
+
disputeService?: DisputeService;
|
|
32
|
+
/** The node's libp2p Ed25519 identity key, for hosts binding a client-transaction signer. */
|
|
33
|
+
peerPrivateKey: PrivateKey;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export type OptimysticNode = Libp2p & OptimysticNodeAttachments;
|
|
@@ -9,6 +9,7 @@ import type { IPeerReputation } from "../reputation/types.js";
|
|
|
9
9
|
import { PenaltyReason } from "../reputation/types.js";
|
|
10
10
|
import type { ITransactionStateStore } from "../cluster/i-transaction-state-store.js";
|
|
11
11
|
import { quorumSize, corroboratorCapacity, selectQuorumRev, type RevClaim, type QuorumRev } from "../cluster/quorum-restore.js";
|
|
12
|
+
import { DEFAULT_CLUSTER_SIZE } from "../cluster/cluster-policy.js";
|
|
12
13
|
import { RECONCILE_TIMEOUT_MS } from "../cluster/reconcile-block.js";
|
|
13
14
|
import { isMissingBaseRevisionFailure, MISSING_BASE_REVISION_REASON } from "../storage/storage-repo.js";
|
|
14
15
|
import type { ReconcileBlockCallback } from "../cluster/cluster-repo.js";
|
|
@@ -191,7 +192,10 @@ export class CoordinatorRepo implements IRepo {
|
|
|
191
192
|
) {
|
|
192
193
|
this.localPeerId = localPeerId;
|
|
193
194
|
const policy: ClusterConsensusConfig & { clusterSize: number } = {
|
|
194
|
-
|
|
195
|
+
// Same constant `resolveClusterPolicy` gives a node that declares no clusterSize, not a
|
|
196
|
+
// second literal: a direct constructor (the readme's manual-wiring path) and the node
|
|
197
|
+
// assembly must land on the same width or the two disagree about the same key's cohort.
|
|
198
|
+
clusterSize: cfg?.clusterSize ?? DEFAULT_CLUSTER_SIZE,
|
|
195
199
|
assumedClusterSize: cfg?.assumedClusterSize,
|
|
196
200
|
superMajorityThreshold: cfg?.superMajorityThreshold ?? DEFAULT_SUPER_MAJORITY_THRESHOLD,
|
|
197
201
|
simpleMajorityThreshold: cfg?.simpleMajorityThreshold ?? 0.51,
|
|
@@ -393,12 +397,21 @@ export class CoordinatorRepo implements IRepo {
|
|
|
393
397
|
*
|
|
394
398
|
* No-op once the entry carries a real answer (the consult restored the block) or a sharper flag
|
|
395
399
|
* (storage's `'unmaterializable'`), so callers only need to establish that the answer is a guess.
|
|
400
|
+
*
|
|
401
|
+
* "Carries a real answer" is tested as `entry.block !== undefined`, NOT as `state.latest` being
|
|
402
|
+
* set. The two used to move together, so `state.latest` read as a serviceable proxy — but a
|
|
403
|
+
* pending-only insert (pended, not yet committed) served through the pending overlay has real
|
|
404
|
+
* CONTENT and no committed revision at all, so its `state.latest` is undefined. Flagging that
|
|
405
|
+
* entry would mark a block this node is positively holding as an unconfirmed absence, and
|
|
406
|
+
* `NetworkTransactor`'s `isAuthoritative` keys off the flag alone — the read would burn its
|
|
407
|
+
* retry budget re-asking other peers for content it already has. `state.latest` stays in the
|
|
408
|
+
* test as well so a stale-but-real committed answer is likewise never downgraded.
|
|
396
409
|
*/
|
|
397
410
|
private flagUnconfirmedAbsence(results: GetBlockResults, blockId: BlockId): void {
|
|
398
411
|
const entry = results[blockId];
|
|
399
412
|
if (!entry) {
|
|
400
413
|
results[blockId] = { state: {}, unavailable: 'peers-unreachable' };
|
|
401
|
-
} else if (!entry.state?.latest && entry.unavailable === undefined) {
|
|
414
|
+
} else if (entry.block === undefined && !entry.state?.latest && entry.unavailable === undefined) {
|
|
402
415
|
entry.unavailable = 'peers-unreachable';
|
|
403
416
|
}
|
|
404
417
|
}
|
|
@@ -603,11 +616,13 @@ export class CoordinatorRepo implements IRepo {
|
|
|
603
616
|
* the repair. Returns the local revision afterwards.
|
|
604
617
|
*
|
|
605
618
|
* A pending-only block (metadata seeded by `savePendingTransaction`, no committed revision) asked
|
|
606
|
-
* for a forward revision no promotion can reach used to throw out of `BlockStorage.ensureRevision
|
|
607
|
-
*
|
|
608
|
-
*
|
|
609
|
-
*
|
|
610
|
-
*
|
|
619
|
+
* for a forward revision no promotion can reach used to throw out of `BlockStorage.ensureRevision`.
|
|
620
|
+
* It no longer does: "no committed base here" is an absence, so that read comes back as a plain
|
|
621
|
+
* unflagged `{ state: {} }` and this method simply returns `undefined` — acquisition then supplies
|
|
622
|
+
* the revision. The `unavailable` arm below still fires for the shapes that ARE a guess (a `latest`
|
|
623
|
+
* this node cannot materialize, a missing-base promotion refusal); on THIS path those are an
|
|
624
|
+
* absence too rather than a read failure, so they are logged as `promote-unavailable` and stepped
|
|
625
|
+
* over rather than short-circuiting the caller. The catch stays for any other fault, same reason.
|
|
611
626
|
*/
|
|
612
627
|
private async promoteCorroborated(blockId: BlockId, corroborated: ActionRev): Promise<number | undefined> {
|
|
613
628
|
try {
|
|
@@ -715,7 +730,8 @@ export class CoordinatorRepo implements IRepo {
|
|
|
715
730
|
log('cluster-fetch:no-quorum', {
|
|
716
731
|
blockId,
|
|
717
732
|
responders: claims.length,
|
|
718
|
-
required: quorumSize(claims.length, this.simpleMajorityThreshold, capacity)
|
|
733
|
+
required: quorumSize(claims.length, this.simpleMajorityThreshold, capacity),
|
|
734
|
+
repairCorroborationClusterSize: this.repairCorroborationClusterSize
|
|
719
735
|
});
|
|
720
736
|
return { local, silent };
|
|
721
737
|
}
|
package/src/rn.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
export * from './cluster/client.js';
|
|
2
|
+
export * from './cluster/cluster-policy.js';
|
|
2
3
|
export * from './cluster/cluster-repo.js';
|
|
3
4
|
export * from './cluster/service.js';
|
|
4
5
|
export * from './protocol-client.js';
|
|
@@ -25,6 +26,7 @@ export * from './sync/service.js';
|
|
|
25
26
|
export * from './it-utility.js';
|
|
26
27
|
export * from './libp2p-key-network.js';
|
|
27
28
|
export * from './libp2p-node-rn.js';
|
|
29
|
+
export * from './optimystic-node.js';
|
|
28
30
|
export * from './routing/responsibility.js';
|
|
29
31
|
export * from './routing/libp2p-known-peers.js';
|
|
30
32
|
export * from './network/network-manager-service.js';
|
|
@@ -50,14 +50,44 @@ export class BlockStorage implements IBlockStorage {
|
|
|
50
50
|
return undefined;
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
-
// Pending-only state: metadata was seeded by savePendingTransaction but no
|
|
54
|
-
//
|
|
55
|
-
//
|
|
56
|
-
|
|
57
|
-
|
|
53
|
+
// Pending-only state: metadata was seeded by savePendingTransaction but no revision has been
|
|
54
|
+
// committed yet. "No committed base here" is an ABSENCE, not a fault — nothing is being
|
|
55
|
+
// FAILED to reconstruct — so both arms below answer `undefined` rather than throwing, whether
|
|
56
|
+
// or not the caller named a revision. StorageRepo.get then applies any pending overlay over
|
|
57
|
+
// that absent base; a throw here would instead be caught into `unavailable: 'unmaterializable'`
|
|
58
|
+
// and a writer reading back its own not-yet-committed insert would be told it is unreadable.
|
|
59
|
+
// `unmaterializable` must keep its one meaning: records prove the block exists and this node
|
|
60
|
+
// cannot reconstruct it.
|
|
61
|
+
if (meta.latest === undefined) {
|
|
62
|
+
if (rev === undefined) {
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
// A named rev still ATTEMPTS the restore: `restoreCallback` may be able to supply that
|
|
66
|
+
// revision even though nothing is committed locally, and a successful restore serves real
|
|
67
|
+
// content with `latest` still undefined. That capability is pinned by the 'getBlock for an
|
|
68
|
+
// absent revision fires restoreCallback (restore not short-circuited)' test in
|
|
69
|
+
// test/block-storage.spec.ts — do not short-circuit it away.
|
|
70
|
+
//
|
|
71
|
+
// Only ensureRevision's FAILURE is swallowed (no callback wired, or restore could not
|
|
72
|
+
// supply the rev): that is precisely the "no committed base here" absence. materializeBlock
|
|
73
|
+
// below is deliberately OUTSIDE the try — a throw from there means revision records exist
|
|
74
|
+
// with no materialization anywhere under them, which is genuine corruption and must keep
|
|
75
|
+
// reading as `unmaterializable`.
|
|
76
|
+
//
|
|
77
|
+
// NOTE: a contextful read of a pending-only block still attempts a network restore before
|
|
78
|
+
// falling back to absent (same cost as the pre-fix throw path); if pending-only read-backs
|
|
79
|
+
// ever show as hot, short-circuit when ranges are empty.
|
|
80
|
+
try {
|
|
81
|
+
await this.ensureRevision(meta, rev);
|
|
82
|
+
} catch (err) {
|
|
83
|
+
log('getBlock:no-committed-base blockId=%s rev=%d error=%s', this.blockId, rev,
|
|
84
|
+
err instanceof Error ? err.message : String(err));
|
|
85
|
+
return undefined;
|
|
86
|
+
}
|
|
87
|
+
return await this.materializeBlock(meta, rev);
|
|
58
88
|
}
|
|
59
89
|
|
|
60
|
-
const targetRev = rev ?? meta.latest
|
|
90
|
+
const targetRev = rev ?? meta.latest.rev;
|
|
61
91
|
await this.ensureRevision(meta, targetRev);
|
|
62
92
|
return await this.materializeBlock(meta, targetRev);
|
|
63
93
|
}
|
|
@@ -271,6 +301,29 @@ export class BlockStorage implements IBlockStorage {
|
|
|
271
301
|
};
|
|
272
302
|
await this.saveRestored(archive);
|
|
273
303
|
|
|
304
|
+
// INVARIANT P: a block never holds a pending record AND a committed record for the same
|
|
305
|
+
// action id. On the commit path `promotePendingTransaction` maintains it by MOVING the
|
|
306
|
+
// record atomically; this forward path writes the committed transform directly (via
|
|
307
|
+
// saveRestored above), so it owes the deletion itself. Without it, a node that pended the
|
|
308
|
+
// action but diverged before committing keeps a record nothing can ever promote — reported
|
|
309
|
+
// as a phantom conflicting action by every later `pend` on the block, which under
|
|
310
|
+
// `policy: 'f'` refuses that node's participation in the block's writes permanently.
|
|
311
|
+
//
|
|
312
|
+
// Deliberately on the WRITE path only: the monotonic guard above returns before here, and
|
|
313
|
+
// that early return must stay a true no-op (the earlier call that wrote the revision is the
|
|
314
|
+
// one that owed the deletion). Deliberately here rather than in `saveRestored`, which is
|
|
315
|
+
// also reached from ensureRevision's historical restore under a different latch, where a
|
|
316
|
+
// deletion could race a concurrent promotePendingTransaction; this path holds
|
|
317
|
+
// `BlockStorage.saveReplica:<id>` and (via StorageRepo.saveReplicatedBlock) the per-block
|
|
318
|
+
// commit latch, so it is already mutually exclusive with a live commit.
|
|
319
|
+
//
|
|
320
|
+
// NOTE: deletes only this revision's actionId, not every pending whose action is already
|
|
321
|
+
// committed. A broader sweep would repair records orphaned by routes that do not carry the
|
|
322
|
+
// committing actionId; if orphaned pendings ever show up in the field on blocks whose
|
|
323
|
+
// committing action id differs, widen to a sweep over listPendingTransactions filtered by
|
|
324
|
+
// getTransaction.
|
|
325
|
+
await this.storage.deletePendingTransaction(this.blockId, actionId);
|
|
326
|
+
|
|
274
327
|
// Seed metadata when absent, advance latest, and merge the covered range.
|
|
275
328
|
const prevRev = meta?.latest?.rev;
|
|
276
329
|
if (!meta) {
|
|
@@ -47,7 +47,18 @@ export interface IBlockStorage {
|
|
|
47
47
|
/** Saves a revision */
|
|
48
48
|
saveRevision(rev: number, actionId: ActionId): Promise<void>;
|
|
49
49
|
|
|
50
|
-
/**
|
|
50
|
+
/**
|
|
51
|
+
* Promotes a pending action to committed, MOVING the record from the pending namespace to the
|
|
52
|
+
* committed one in a single atomic step.
|
|
53
|
+
*
|
|
54
|
+
* **Invariant P** — a block never holds a pending record and a committed record for the same
|
|
55
|
+
* action id at the same time. This method maintains it on the commit path; every OTHER writer of
|
|
56
|
+
* a committed transform for a block ({@link saveReplica}, {@link saveDeletion}, and any forward
|
|
57
|
+
* path added later) must maintain it too, by deleting that action's pending record when it
|
|
58
|
+
* writes the committed one. A pending record left beside a committed one can never be promoted,
|
|
59
|
+
* and is reported as a phantom conflicting action by `StorageRepo.pend` on every later write to
|
|
60
|
+
* the block.
|
|
61
|
+
*/
|
|
51
62
|
promotePendingTransaction(actionId: ActionId): Promise<void>;
|
|
52
63
|
|
|
53
64
|
/** Sets the latest revision information */
|
|
@@ -62,6 +73,11 @@ export interface IBlockStorage {
|
|
|
62
73
|
* it falls back to `rev = 1` and a deterministic `actionId` derived from the block
|
|
63
74
|
* (so retries stay idempotent — never random).
|
|
64
75
|
*
|
|
76
|
+
* Maintains **Invariant P** (see {@link promotePendingTransaction}): writing the committed
|
|
77
|
+
* transform for `actionId` also deletes that action's pending record on this block, so a node
|
|
78
|
+
* that pended the action but diverged before committing it does not keep an unpromotable record.
|
|
79
|
+
* Only on the write path — the monotonic no-op below deletes nothing.
|
|
80
|
+
*
|
|
65
81
|
* No-op (still durable) when an equal-or-newer revision is already present: `latest`
|
|
66
82
|
* is never downgraded. Idempotent for a fixed `(rev, actionId)`. Returns the
|
|
67
83
|
* effective latest `ActionRev`.
|
|
@@ -75,6 +91,9 @@ export interface IBlockStorage {
|
|
|
75
91
|
* materialization as a deletion, so a `getBlock()` after a tombstone reads back as *absent*
|
|
76
92
|
* (`undefined`) while a historical `getBlock(creationRev)` still materializes the created content.
|
|
77
93
|
*
|
|
94
|
+
* Maintains **Invariant P** (see {@link promotePendingTransaction}) on the write path, exactly as
|
|
95
|
+
* {@link saveReplica} does: the tombstone's `actionId` loses its pending record on this block.
|
|
96
|
+
*
|
|
78
97
|
* Idempotent for a fixed `(rev, actionId)`; never downgrades `latest` (a no-op — still durable —
|
|
79
98
|
* when an equal-or-newer revision is already present). Returns the effective latest `ActionRev`.
|
|
80
99
|
*/
|
|
@@ -280,19 +280,30 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
|
|
|
280
280
|
}
|
|
281
281
|
|
|
282
282
|
// Include pending action if requested, applying the pending transform over whatever
|
|
283
|
-
// committed base getBlock() resolved (possibly none
|
|
284
|
-
//
|
|
285
|
-
// served here despite what this branch is shaped to do: `ActionContext.rev` is required,
|
|
286
|
-
// and BlockStorage.getBlock only tolerates a missing committed base when `rev` is
|
|
287
|
-
// undefined — so a contextful read of a block with no committed revision throws above and
|
|
288
|
-
// is caught into `unavailable: 'unmaterializable'` before reaching here. Asserted in
|
|
289
|
-
// storage-repo.spec.ts ('KNOWN GAP: a pending-only insert read WITH a context ...'); see
|
|
290
|
-
// tickets/backlog/debt-pending-only-insert-unreadable-with-context.
|
|
283
|
+
// committed base getBlock() resolved (possibly none — a pending-only insert has no
|
|
284
|
+
// committed revision under it and getBlock reports that as an absent base, not a fault).
|
|
291
285
|
if (context?.actionId !== undefined) {
|
|
292
286
|
const pendingTransform = await blockStorage.getPendingTransaction(context.actionId);
|
|
293
287
|
if (!pendingTransform) {
|
|
288
|
+
if (unavailable !== undefined) {
|
|
289
|
+
// The promotion refusal above deleted this very pending record
|
|
290
|
+
// (`refuseMissingBase` drops the pending it cannot promote). This node DID hold
|
|
291
|
+
// the record and dropped it, so the honest answer is an availability one — not
|
|
292
|
+
// a caller-contract violation, and never a throw that would fail the whole batch.
|
|
293
|
+
return [blockId, { state: {}, unavailable } as GetBlockResult];
|
|
294
|
+
}
|
|
294
295
|
// Caller-contract violation (the caller asserted a pending this repo never had, or
|
|
295
296
|
// cancelled) — an error, not an availability question. Deliberately NOT `unavailable`.
|
|
297
|
+
//
|
|
298
|
+
// It is NOT the only way to reach here. A context that both PROVES its own action
|
|
299
|
+
// (`committed` names it) and names it as the pending overlay (`actionId`) is
|
|
300
|
+
// self-contradictory, and the two halves of that contradiction land differently: if
|
|
301
|
+
// the read-driven promotion above REFUSED, the arm above answers gracefully; if it
|
|
302
|
+
// SUCCEEDED, `promotePendingTransaction` moved the record and we throw here — failing
|
|
303
|
+
// the whole batch for a request the refusal path tolerates. No production code sets
|
|
304
|
+
// `ActionContext.actionId` at all today, so neither is reachable except from tests or
|
|
305
|
+
// a peer that crafts the field on the wire. See
|
|
306
|
+
// tickets/blocked/repo-pending-overlay-has-no-producer.
|
|
296
307
|
throw new Error(`Pending action ${context.actionId} not found`);
|
|
297
308
|
}
|
|
298
309
|
const block = applyTransform(blockRev?.block, pendingTransform);
|
|
@@ -304,13 +315,21 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
|
|
|
304
315
|
},
|
|
305
316
|
// The COMMITTED revision underneath the pending overlay. A pending has no revision
|
|
306
317
|
// of its own, so the honest answer is the base it was applied to. Absent when there
|
|
307
|
-
// was no base
|
|
308
|
-
//
|
|
318
|
+
// was no base at all — a pending-only insert served over an absent committed base,
|
|
319
|
+
// where fabricating a revision would claim content this node never committed.
|
|
309
320
|
...(blockRev ? { materializedRev: blockRev.actionRev.rev } : {}),
|
|
310
321
|
// A pending applied to a missing base can materialize nothing (applyTransform drops
|
|
311
322
|
// updates with no block to apply them to) — that absence is a guess, and is flagged.
|
|
312
|
-
// A materialized block is a real answer regardless of the earlier refusal.
|
|
313
|
-
|
|
323
|
+
// A materialized block is a real answer regardless of the earlier refusal. TWO ways
|
|
324
|
+
// an empty result is a guess: the promotion refusal fired (`unavailable` set), or
|
|
325
|
+
// there was no committed base under the overlay at all (`blockRev === undefined`) —
|
|
326
|
+
// this node holds a pending record PROVING the block exists and produced nothing.
|
|
327
|
+
// The second clause's ABSENCE in the other direction is equally load-bearing: a
|
|
328
|
+
// pending DELETE over a real committed base also lands here with no block, and that
|
|
329
|
+
// is an authoritative tombstone which must stay unflagged.
|
|
330
|
+
...(block === undefined && (unavailable !== undefined || blockRev === undefined)
|
|
331
|
+
? { unavailable: unavailable ?? 'unmaterializable' }
|
|
332
|
+
: {})
|
|
314
333
|
} as GetBlockResult];
|
|
315
334
|
}
|
|
316
335
|
|
|
@@ -495,6 +514,34 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
|
|
|
495
514
|
}));
|
|
496
515
|
}
|
|
497
516
|
|
|
517
|
+
/**
|
|
518
|
+
* Commit a previously-pended action across its blocks, under the per-block commit latches.
|
|
519
|
+
*
|
|
520
|
+
* **Divergence vs genuine fault.** When the batch cannot be completed, the reason decides what
|
|
521
|
+
* happens to the pending records the pend left behind. `ClusterMember.applyConsensusOperation`
|
|
522
|
+
* makes the same split one layer up — it *tolerates* a divergence (and reconciles every
|
|
523
|
+
* `commit.blockIds` entry from a cohort peer) but *propagates* a genuine fault for retry — so this
|
|
524
|
+
* method must agree with it:
|
|
525
|
+
*
|
|
526
|
+
* - **Divergence** — this node is behind the agreed history, either because it holds no
|
|
527
|
+
* materializable base ({@link MissingBaseRevisionError}) or because it never received the pend
|
|
528
|
+
* (the `Pending action … not found` throw). Reconcile is guaranteed to follow and will advance
|
|
529
|
+
* every block in the batch past `request.rev`, so no pending record here can ever be promoted:
|
|
530
|
+
* {@link dropUnpromotablePendings} deletes them (see {@link refuseMissingBase}, which already
|
|
531
|
+
* accepts this tradeoff for the single refusing block).
|
|
532
|
+
* - **Genuine fault** — any other throw out of {@link internalCommit} (a raw-storage error, …).
|
|
533
|
+
* `ClusterMember` propagates it and the commit is retried, and a retry can still replay the
|
|
534
|
+
* pendings, so they are KEPT.
|
|
535
|
+
*
|
|
536
|
+
* The stale/`missedCommits` early return (this node is AHEAD — it already holds a revision at or
|
|
537
|
+
* past `request.rev`, committed under a different action) deliberately keeps pendings too, and its
|
|
538
|
+
* cure is the losing client's `cancel`: `CoordinatorRepo.cancel` runs through consensus, so every
|
|
539
|
+
* member drops the record, not just the coordinator. Replication cannot be the cure here — this
|
|
540
|
+
* node is already ahead, and a later forward write carries a DIFFERENT action id, which is not
|
|
541
|
+
* what `BlockStorage.saveForwardRevision` deletes. A client that dies between the stale result and
|
|
542
|
+
* its `cancel` therefore still strands the record; that is pre-existing and orthogonal to the
|
|
543
|
+
* divergence split above.
|
|
544
|
+
*/
|
|
498
545
|
async commit(request: CommitRequest, _options?: MessageOptions): Promise<CommitResult> {
|
|
499
546
|
log('commit actionId=%s rev=%d blockIds=%d', request.actionId, request.rev, request.blockIds.length);
|
|
500
547
|
const uniqueBlockIds = Array.from(new Set(request.blockIds)).sort();
|
|
@@ -620,6 +667,12 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
|
|
|
620
667
|
// pending-present — a never-pended block cannot coexist with it in one retry. If a path
|
|
621
668
|
// ever produces that mix, emit recovered blocks' events before throwing here.
|
|
622
669
|
if (missingPends.length) {
|
|
670
|
+
// Divergence (this node is behind): `ClusterMember` treats this throw as the canonical
|
|
671
|
+
// "behind" signal and reconciles EVERY block in the batch, advancing each past
|
|
672
|
+
// `request.rev`. Nothing can promote the pendings the other blocks still hold, so drop
|
|
673
|
+
// them here — while the latches are still held — before reporting. The thrown message
|
|
674
|
+
// must stay byte-identical: `ClusterMember.isMissingPendingActionError` matches on it.
|
|
675
|
+
await this.dropUnpromotablePendings(toCommit, request.actionId);
|
|
623
676
|
throw new Error(`Pending action ${request.actionId} not found for block(s): ${missingPends.map(p => p.blockId).join(', ')}`);
|
|
624
677
|
}
|
|
625
678
|
|
|
@@ -648,6 +701,10 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
|
|
|
648
701
|
// This loop will execute atomically for all blocks due to the acquired locks.
|
|
649
702
|
// Recovered (Crash-D3) blocks are already committed at request.rev and their pending is
|
|
650
703
|
// gone, so skip them — internalCommit would throw on the missing pending record.
|
|
704
|
+
//
|
|
705
|
+
// Set when the mid-loop failure was a divergence rather than a genuine fault — the split
|
|
706
|
+
// documented on commit() above, which decides the fate of the batch's pending records.
|
|
707
|
+
let divergentFailure = false;
|
|
651
708
|
for (const { blockId, storage } of toCommit) {
|
|
652
709
|
if (recovered.has(blockId)) {
|
|
653
710
|
continue;
|
|
@@ -666,9 +723,20 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
|
|
|
666
723
|
// treats them as idempotent no-ops and advances the remainder. Break
|
|
667
724
|
// instead of returning so locks release and those landings emit below.
|
|
668
725
|
failure = { reason: err instanceof Error ? err.message : 'Unknown error during commit' };
|
|
726
|
+
divergentFailure = err instanceof MissingBaseRevisionError;
|
|
669
727
|
break;
|
|
670
728
|
}
|
|
671
729
|
}
|
|
730
|
+
|
|
731
|
+
// The break left every not-yet-reached block still holding its pending record. Whether
|
|
732
|
+
// that record is still usable depends ENTIRELY on why we stopped — see the table on
|
|
733
|
+
// commit() above. Runs inside the try, so the per-block latches are still held.
|
|
734
|
+
// NOTE: a non-divergence fault deliberately KEEPS the batch's pendings so a retry can
|
|
735
|
+
// replay them. If ClusterMember ever stops retrying propagated commit faults, this arm
|
|
736
|
+
// becomes dead weight and the discriminator can collapse to "always drop".
|
|
737
|
+
if (divergentFailure) {
|
|
738
|
+
await this.dropUnpromotablePendings(toCommit, request.actionId);
|
|
739
|
+
}
|
|
672
740
|
}
|
|
673
741
|
finally {
|
|
674
742
|
// Release locks in reverse order of acquisition
|
|
@@ -683,6 +751,44 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
|
|
|
683
751
|
return failure ? { success: false, reason: failure.reason } : { success: true };
|
|
684
752
|
}
|
|
685
753
|
|
|
754
|
+
/**
|
|
755
|
+
* Delete `actionId`'s pending record from every given block, tolerating absence.
|
|
756
|
+
*
|
|
757
|
+
* Called by {@link commit} when it abandons a batch **because this node has diverged from the
|
|
758
|
+
* agreed history** — the caller has already made that determination; this helper does not
|
|
759
|
+
* re-derive it. Once `ClusterMember` reconciles the batch, every one of these blocks sits at or
|
|
760
|
+
* past `request.rev`, so a commit retry partitions them as already-done/stale and never revisits
|
|
761
|
+
* their pendings; left in place they are reported as phantom conflicting actions by {@link pend}
|
|
762
|
+
* for every later write to the block (under `policy: 'f'`, forever).
|
|
763
|
+
*
|
|
764
|
+
* No special-casing is needed for blocks that already landed (record promoted), that were
|
|
765
|
+
* `recovered` (record already gone), or for the refusing block itself
|
|
766
|
+
* ({@link refuseMissingBase} deleted its record): deleting an absent pending record is a no-op on
|
|
767
|
+
* every backend.
|
|
768
|
+
*
|
|
769
|
+
* Per-block failures are logged and swallowed rather than propagated: this cleanup must never
|
|
770
|
+
* replace the failure the caller is about to report — the pre-loop throw's message is pattern-
|
|
771
|
+
* matched by `ClusterMember.isMissingPendingActionError`, and a swapped error would misroute
|
|
772
|
+
* consensus. A leftover record only degrades this node's participation in that one block.
|
|
773
|
+
*/
|
|
774
|
+
private async dropUnpromotablePendings(
|
|
775
|
+
blocks: { blockId: BlockId, storage: IBlockStorage }[],
|
|
776
|
+
actionId: ActionId
|
|
777
|
+
): Promise<void> {
|
|
778
|
+
if (blocks.length === 0) {
|
|
779
|
+
return;
|
|
780
|
+
}
|
|
781
|
+
log('commit:drop-unpromotable-pendings actionId=%s blockIds=%d', actionId, blocks.length);
|
|
782
|
+
await Promise.all(blocks.map(async ({ blockId, storage }) => {
|
|
783
|
+
try {
|
|
784
|
+
await storage.deletePendingTransaction(actionId);
|
|
785
|
+
} catch (err) {
|
|
786
|
+
log('commit:drop-unpromotable-pending-failed blockId=%s actionId=%s error=%s', blockId, actionId,
|
|
787
|
+
err instanceof Error ? err.message : String(err));
|
|
788
|
+
}
|
|
789
|
+
}));
|
|
790
|
+
}
|
|
791
|
+
|
|
686
792
|
/**
|
|
687
793
|
* Reconciles `metadata.latest` for a single block with the highest contiguous
|
|
688
794
|
* fully-promoted revision in durable storage. Use after a crash between
|
|
@@ -830,9 +936,13 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
|
|
|
830
936
|
* BlockStorage reports every one of these as a bare `Error`, so they cannot be told apart here,
|
|
831
937
|
* and treating them as divergence is the safe default: this node genuinely cannot materialize the
|
|
832
938
|
* base right now, and the cluster's policy is to heal rather than throw out of consensus. The
|
|
833
|
-
* price is that a transient fault ALSO drops
|
|
834
|
-
*
|
|
835
|
-
*
|
|
939
|
+
* price is that a transient fault ALSO drops pending records — this block's (see
|
|
940
|
+
* {@link refuseMissingBase}) AND, because {@link commit} keys its cleanup off the same error type,
|
|
941
|
+
* every not-yet-reached block in the same batch — so those blocks converge by replication instead
|
|
942
|
+
* of by a replay the retry could have done. That is a wider blast radius than the per-block
|
|
943
|
+
* refusal alone, and it is why the discriminator must NOT be loosened beyond this error type.
|
|
944
|
+
* Narrowing this would require typed faults out of BlockStorage; until then, prefer the tolerant
|
|
945
|
+
* reading.
|
|
836
946
|
*/
|
|
837
947
|
private async readCommitBase(
|
|
838
948
|
blockId: BlockId,
|
|
@@ -881,9 +991,10 @@ export class StorageRepo implements IRepo, IBlockChangeNotifier, IBlockReplicaSt
|
|
|
881
991
|
* Converts list of missing actions per block into a list of missing actions across blocks.
|
|
882
992
|
*
|
|
883
993
|
* NOTE: relies on each (actionId, blockId) pair appearing at most once — one revision per action
|
|
884
|
-
* per block. If a block ever records two revisions under the same actionId, concatTransform
|
|
885
|
-
*
|
|
886
|
-
*
|
|
994
|
+
* per block. If a block ever records two revisions under the same actionId, concatTransform now
|
|
995
|
+
* concatenates both revisions' ops into one array rather than dropping the earlier one — still
|
|
996
|
+
* wrong, since ops from distinct revisions are not composable against a single base, but loud
|
|
997
|
+
* rather than silent. Group by (actionId, rev) instead if that case becomes reachable.
|
|
887
998
|
*/
|
|
888
999
|
function perBlockActionTransformsToPerAction(missing: { blockId: BlockId; transforms: ActionTransform[]; }[]) {
|
|
889
1000
|
const missingFlat = missing.flatMap(({ blockId, transforms }) =>
|
|
@@ -194,6 +194,26 @@ export function runRawStorageConformance(
|
|
|
194
194
|
expect(await collect(storage.listPendingTransactions(blockId))).to.deep.equal([]);
|
|
195
195
|
});
|
|
196
196
|
|
|
197
|
+
it('deletePendingTransaction on an absent entry is a tolerated no-op', async () => {
|
|
198
|
+
// Load-bearing across backends: the commit path deletes a pending record it has NOT
|
|
199
|
+
// checked for presence — `StorageRepo.dropUnpromotablePendings` over a whole batch, and
|
|
200
|
+
// `BlockStorage.saveForwardRevision` maintaining Invariant P — so a backend that threw
|
|
201
|
+
// (or rejected) on a miss would turn every ordinary commit into a logged failure.
|
|
202
|
+
const blockId = 'pend-del-absent' as BlockId;
|
|
203
|
+
await storage.deletePendingTransaction(blockId, 'never-pended' as ActionId);
|
|
204
|
+
|
|
205
|
+
// Also a no-op beside a LIVE pending for a different action: the delete must be scoped
|
|
206
|
+
// to its action id, never a per-block clear.
|
|
207
|
+
await storage.savePendingTransaction(blockId, 'a1' as ActionId, { delete: true });
|
|
208
|
+
await storage.deletePendingTransaction(blockId, 'other' as ActionId);
|
|
209
|
+
expect(await storage.getPendingTransaction(blockId, 'a1' as ActionId)).to.deep.equal({ delete: true });
|
|
210
|
+
|
|
211
|
+
// Idempotent: deleting the same entry twice is fine.
|
|
212
|
+
await storage.deletePendingTransaction(blockId, 'a1' as ActionId);
|
|
213
|
+
await storage.deletePendingTransaction(blockId, 'a1' as ActionId);
|
|
214
|
+
expect(await collect(storage.listPendingTransactions(blockId))).to.deep.equal([]);
|
|
215
|
+
});
|
|
216
|
+
|
|
197
217
|
it('listPendingTransactions tolerates awaits interleaved between yields (drain-before-yield)', async () => {
|
|
198
218
|
const blockId = 'drain-pend' as BlockId;
|
|
199
219
|
await storage.savePendingTransaction(blockId, 'a1' as ActionId, { delete: true });
|