@optimystic/db-core 1.10.1 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/cluster/structs.d.ts +35 -7
- package/dist/src/cluster/structs.d.ts.map +1 -1
- package/dist/src/cluster/structs.js +33 -13
- package/dist/src/cluster/structs.js.map +1 -1
- package/dist/src/cohort-topic/coldstart.d.ts +8 -4
- package/dist/src/cohort-topic/coldstart.d.ts.map +1 -1
- package/dist/src/cohort-topic/coldstart.js +3 -3
- package/dist/src/cohort-topic/coldstart.js.map +1 -1
- package/dist/src/cohort-topic/member-engine.d.ts +5 -1
- package/dist/src/cohort-topic/member-engine.d.ts.map +1 -1
- package/dist/src/cohort-topic/member-engine.js.map +1 -1
- package/dist/src/cohort-topic/willingness.d.ts +4 -3
- package/dist/src/cohort-topic/willingness.d.ts.map +1 -1
- package/dist/src/cohort-topic/willingness.js.map +1 -1
- package/dist/src/collection/collection.d.ts.map +1 -1
- package/dist/src/collection/collection.js +20 -1
- package/dist/src/collection/collection.js.map +1 -1
- package/dist/src/index.d.ts +1 -0
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +1 -0
- package/dist/src/index.js.map +1 -1
- package/dist/src/network/i-key-network.d.ts +8 -0
- package/dist/src/network/i-key-network.d.ts.map +1 -1
- package/dist/src/network/index.d.ts +1 -0
- package/dist/src/network/index.d.ts.map +1 -1
- package/dist/src/network/index.js +1 -0
- package/dist/src/network/index.js.map +1 -1
- package/dist/src/network/stale-failure.d.ts +12 -3
- package/dist/src/network/stale-failure.d.ts.map +1 -1
- package/dist/src/network/stale-failure.js +14 -1
- package/dist/src/network/stale-failure.js.map +1 -1
- package/dist/src/network/struct.d.ts +54 -4
- package/dist/src/network/struct.d.ts.map +1 -1
- package/dist/src/network/struct.js.map +1 -1
- package/dist/src/network/transaction-expired-error.d.ts +44 -0
- package/dist/src/network/transaction-expired-error.d.ts.map +1 -0
- package/dist/src/network/transaction-expired-error.js +77 -0
- package/dist/src/network/transaction-expired-error.js.map +1 -0
- package/dist/src/transaction/coordinator.d.ts +4 -1
- package/dist/src/transaction/coordinator.d.ts.map +1 -1
- package/dist/src/transaction/coordinator.js +31 -5
- package/dist/src/transaction/coordinator.js.map +1 -1
- package/dist/src/transactor/network-transactor.d.ts.map +1 -1
- package/dist/src/transactor/network-transactor.js +32 -39
- package/dist/src/transactor/network-transactor.js.map +1 -1
- package/dist/src/utility/format-instant.d.ts +8 -0
- package/dist/src/utility/format-instant.d.ts.map +1 -0
- package/dist/src/utility/format-instant.js +11 -0
- package/dist/src/utility/format-instant.js.map +1 -0
- package/package.json +1 -1
- package/src/cluster/structs.ts +59 -14
- package/src/cohort-topic/coldstart.ts +8 -4
- package/src/cohort-topic/member-engine.ts +5 -1
- package/src/cohort-topic/willingness.ts +4 -3
- package/src/collection/collection.ts +20 -1
- package/src/index.ts +1 -0
- package/src/network/i-key-network.ts +9 -0
- package/src/network/index.ts +1 -0
- package/src/network/stale-failure.ts +16 -3
- package/src/network/struct.ts +57 -4
- package/src/network/transaction-expired-error.ts +81 -0
- package/src/transaction/coordinator.ts +37 -7
- package/src/transactor/network-transactor.ts +36 -43
- package/src/utility/format-instant.ts +10 -0
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import type { ExpiredFailure, TransactionExpiry } from "./struct.js";
|
|
2
|
+
import { formatInstant } from "../utility/format-instant.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* A write the cohort refused because, by the refusing members' own clocks, the transaction had
|
|
6
|
+
* already expired when it reached them. Nobody judged the write and nobody was unreachable: the clock
|
|
7
|
+
* that set the expiration (the writer's) and the members' clocks disagree by more than the
|
|
8
|
+
* transaction timeout (30 s by default). Not retryable — a retry's expiration comes from the same
|
|
9
|
+
* clock, so it is refused the same way until one of the clocks is corrected. Nothing was saved.
|
|
10
|
+
*
|
|
11
|
+
* ONE class for every place it is raised, so the coordinator's report and the writer's cannot drift.
|
|
12
|
+
* The cluster coordinator (db-p2p `ClusterCoordinator`) raises it when every reject vote that sank a
|
|
13
|
+
* transaction was an expiry vote; the coordinating repo returns its facts as an
|
|
14
|
+
* {@link ExpiredFailure} ({@link toFailure}), which is what lets a remote writer receive them at all;
|
|
15
|
+
* and each write path (`Collection.sync`, `TransactionCoordinator.commit`) raises it again from that
|
|
16
|
+
* refusal, with its own clock as {@link localClock}. Deliberately not a db-p2p
|
|
17
|
+
* `ValidatorRejectionError`, which means "the cohort judged the write": the coordinating repo turns
|
|
18
|
+
* this one into its refusal before its validator-rejection classifiers run.
|
|
19
|
+
*/
|
|
20
|
+
export class TransactionExpiredError extends Error {
|
|
21
|
+
/** The transaction's expiration (unix ms), as the writer's transactor set it. */
|
|
22
|
+
readonly expiration: number;
|
|
23
|
+
/** Per refusing cohort member (peer-id string), the clock reading (unix ms) it signed. */
|
|
24
|
+
readonly memberClocks: Readonly<Record<string, number>>;
|
|
25
|
+
/** The coordinating node's clock (unix ms) when it concluded the refusal. */
|
|
26
|
+
readonly coordinatorClock: number;
|
|
27
|
+
/**
|
|
28
|
+
* How far ahead of {@link localClock} the most-skewed refusing member's clock appeared: the largest
|
|
29
|
+
* `memberClocks[peer] − localClock`. An estimate, low by the time the answer took to travel back
|
|
30
|
+
* (a member read its clock before then). Raised by a write path, this is how far the cohort's
|
|
31
|
+
* clocks appear to be from the writer's own — the number to act on whichever side is wrong. Zero or
|
|
32
|
+
* less means this node's clock has the expiration past as well.
|
|
33
|
+
*/
|
|
34
|
+
readonly apparentSkewMs: number;
|
|
35
|
+
|
|
36
|
+
constructor(
|
|
37
|
+
expiry: TransactionExpiry,
|
|
38
|
+
/** This node's clock (unix ms) when it raised the error: the coordinator's where the cluster
|
|
39
|
+
* raised it, the writer's where a write path raised it from a returned refusal. */
|
|
40
|
+
readonly localClock: number = Date.now(),
|
|
41
|
+
) {
|
|
42
|
+
// NOTE: `expiry` off the wire is trusted as shaped (`isExpiryFailure` checks only presence), so a
|
|
43
|
+
// malformed coordinator answer gives NaN here, or a TypeError when `memberClocks` is missing,
|
|
44
|
+
// instead of this error. Either still ends the write; if hostile coordinators become a concern,
|
|
45
|
+
// validate the shape where the refusal is received.
|
|
46
|
+
const apparentSkewMs = Math.max(...Object.values(expiry.memberClocks)) - localClock;
|
|
47
|
+
super(transactionExpiredMessage(expiry, localClock, apparentSkewMs));
|
|
48
|
+
this.name = 'TransactionExpiredError';
|
|
49
|
+
this.expiration = expiry.expiration;
|
|
50
|
+
this.memberClocks = { ...expiry.memberClocks };
|
|
51
|
+
this.coordinatorClock = expiry.coordinatorClock;
|
|
52
|
+
this.apparentSkewMs = apparentSkewMs;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/** The refusal a coordinating repo returns for this error, so it crosses the repo protocol as an
|
|
56
|
+
* answer rather than as a reset stream. */
|
|
57
|
+
toFailure(): ExpiredFailure {
|
|
58
|
+
return {
|
|
59
|
+
success: false,
|
|
60
|
+
conflict: false,
|
|
61
|
+
reason: this.message,
|
|
62
|
+
expired: { expiration: this.expiration, memberClocks: { ...this.memberClocks }, coordinatorClock: this.coordinatorClock },
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Written to be shown to a person. Deliberately free of the shortfall error's text ("Failed to get
|
|
69
|
+
* super-majority"), which a downstream repository matches to retry a silent cohort: a clock
|
|
70
|
+
* disagreement is neither silence nor transient.
|
|
71
|
+
*/
|
|
72
|
+
function transactionExpiredMessage(expiry: TransactionExpiry, localClock: number, apparentSkewMs: number): string {
|
|
73
|
+
const seconds = (ms: number) => (ms / 1000).toFixed(1);
|
|
74
|
+
const refusing = Object.keys(expiry.memberClocks).length;
|
|
75
|
+
const gap = apparentSkewMs > 0
|
|
76
|
+
? `Their clocks appear to be about ${seconds(apparentSkewMs)} s ahead of this device's.`
|
|
77
|
+
: `This device's clock has it ${seconds(localClock - expiry.expiration)} s past as well, so the clock that set the expiration may be behind, or the transaction outlived its deadline.`;
|
|
78
|
+
return `The transaction expired before the cohort could accept it: by their own clocks, ${refusing} cohort member(s) found its expiration (${formatInstant(expiry.expiration)}) already past. `
|
|
79
|
+
+ `${gap} This is a clock disagreement, not a conflict or an unreachable cohort, and retrying fails the same way until it is resolved: `
|
|
80
|
+
+ `correct the clock of whichever device is wrong — this one, or the refusing members.`;
|
|
81
|
+
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import type { ITransactor, BlockId, CollectionId, Transforms, PendRequest, CommitRequest, ActionId } from "../index.js";
|
|
2
2
|
import type { Transaction, ExecutionResult, ITransactionEngine, CollectionActions, ReadDependency } from "./transaction.js";
|
|
3
3
|
import type { PeerId } from "../network/types.js";
|
|
4
|
-
import { isConflictFailure } from "../network/stale-failure.js";
|
|
4
|
+
import { isConflictFailure, isExpiryFailure } from "../network/stale-failure.js";
|
|
5
|
+
import { TransactionExpiredError } from "../network/transaction-expired-error.js";
|
|
5
6
|
import type { Collection, CollectionSnapshot, RefreshReport } from "../collection/collection.js";
|
|
6
7
|
import type { SyncOptions } from "../collection/index.js";
|
|
7
8
|
import { TornActionError } from "../collection/struct.js";
|
|
@@ -304,7 +305,10 @@ export class TransactionCoordinator {
|
|
|
304
305
|
* still never re-drive a partial landing itself. A partial landing on a HARD failure (the
|
|
305
306
|
* transport budget spent, a structural rejection) escapes at once as
|
|
306
307
|
* {@link CoordinatorPartialCommitError}, and any other failure (expired transaction,
|
|
307
|
-
* unavailable transactor, unreachable cluster) also propagates without retry
|
|
308
|
+
* unavailable transactor, unreachable cluster) also propagates without retry — including a pend
|
|
309
|
+
* or commit the cohort refused because the transaction had expired by its members' clocks, which
|
|
310
|
+
* surfaces as {@link TransactionExpiredError} (inside a partial commit's `reason` once a sibling
|
|
311
|
+
* has landed).
|
|
308
312
|
*
|
|
309
313
|
* A participant can also be SAVED between attempts: an attempt that reported a loss may still
|
|
310
314
|
* have stored that participant's log tail, and the refresh before the next attempt then finishes
|
|
@@ -719,6 +723,7 @@ export class TransactionCoordinator {
|
|
|
719
723
|
committedCollections?: Set<CollectionId>;
|
|
720
724
|
failedCollections?: Set<CollectionId>;
|
|
721
725
|
staleLoss?: boolean;
|
|
726
|
+
expired?: TransactionExpiredError;
|
|
722
727
|
};
|
|
723
728
|
try {
|
|
724
729
|
for (const { collectionId, collection } of collectionData) {
|
|
@@ -835,7 +840,7 @@ export class TransactionCoordinator {
|
|
|
835
840
|
// stamp tracking (the success path does the same at the end) and surface the
|
|
836
841
|
// structured signal for reconciliation at once.
|
|
837
842
|
this.stampData.delete(transaction.stamp.id);
|
|
838
|
-
throw new CoordinatorPartialCommitError([...committed], failed, coordResult.error);
|
|
843
|
+
throw new CoordinatorPartialCommitError([...committed], failed, coordResult.expired ?? coordResult.error);
|
|
839
844
|
}
|
|
840
845
|
|
|
841
846
|
// EMPTY committed set: PEND failed, or the whole commit failed cleanly with
|
|
@@ -851,6 +856,11 @@ export class TransactionCoordinator {
|
|
|
851
856
|
if (coordResult.staleLoss) {
|
|
852
857
|
throw new CoordinatorStaleLossError([...(coordResult.failedCollections ?? new Set(allCollectionIds))], coordResult.error);
|
|
853
858
|
}
|
|
859
|
+
// An expiry is surfaced by name and never re-driven: the retry's pend would carry an
|
|
860
|
+
// expiration from the same clock.
|
|
861
|
+
if (coordResult.expired !== undefined) {
|
|
862
|
+
throw coordResult.expired;
|
|
863
|
+
}
|
|
854
864
|
throw new Error(`Transaction commit failed: ${coordResult.error}`);
|
|
855
865
|
}
|
|
856
866
|
|
|
@@ -1416,6 +1426,9 @@ export class TransactionCoordinator {
|
|
|
1416
1426
|
/** True when the failure was a clean optimistic-concurrency conflict (stale loss / pending
|
|
1417
1427
|
* contention) with nothing durable — i.e. safe to re-drive after a re-read. */
|
|
1418
1428
|
staleLoss?: boolean;
|
|
1429
|
+
/** Present when a pend or a commit was refused because the transaction had expired by the
|
|
1430
|
+
* cohort's clocks: the error the commit surfaces, never a stale loss. */
|
|
1431
|
+
expired?: TransactionExpiredError;
|
|
1419
1432
|
}> {
|
|
1420
1433
|
const trxId = transaction.id;
|
|
1421
1434
|
const t0 = Date.now();
|
|
@@ -1467,6 +1480,7 @@ export class TransactionCoordinator {
|
|
|
1467
1480
|
committedCollections: commitResult.committedCollections,
|
|
1468
1481
|
failedCollections: commitResult.failedCollections,
|
|
1469
1482
|
staleLoss: commitResult.staleLoss,
|
|
1483
|
+
...(commitResult.expired === undefined ? {} : { expired: commitResult.expired }),
|
|
1470
1484
|
};
|
|
1471
1485
|
}
|
|
1472
1486
|
|
|
@@ -1539,7 +1553,7 @@ export class TransactionCoordinator {
|
|
|
1539
1553
|
collectionTransforms: ReadonlyMap<CollectionId, Transforms>,
|
|
1540
1554
|
pendedRevs: ReadonlyMap<CollectionId, number>,
|
|
1541
1555
|
superclusterNominees: ReadonlySet<PeerId> | null
|
|
1542
|
-
): Promise<{ success: boolean; error?: string; pendedBlockIds?: Map<CollectionId, BlockId[]>; staleLoss?: boolean }> {
|
|
1556
|
+
): Promise<{ success: boolean; error?: string; pendedBlockIds?: Map<CollectionId, BlockId[]>; staleLoss?: boolean; expired?: TransactionExpiredError }> {
|
|
1543
1557
|
if (collectionTransforms.size === 0) {
|
|
1544
1558
|
return { success: false, error: 'No transforms to pend' };
|
|
1545
1559
|
}
|
|
@@ -1569,6 +1583,8 @@ export class TransactionCoordinator {
|
|
|
1569
1583
|
// re-driving it would just burn the retry budget — fail fast instead.
|
|
1570
1584
|
let anyConflict = false;
|
|
1571
1585
|
let anyHard = false;
|
|
1586
|
+
// An expiry is a hard failure too, and is carried whole: it is the error the commit surfaces.
|
|
1587
|
+
let expired: TransactionExpiredError | undefined;
|
|
1572
1588
|
for (const outcome of outcomes) {
|
|
1573
1589
|
if (outcome.status === 'fulfilled') {
|
|
1574
1590
|
pendedBlockIds.set(outcome.value.collectionId, outcome.value.blockIds);
|
|
@@ -1578,6 +1594,7 @@ export class TransactionCoordinator {
|
|
|
1578
1594
|
}
|
|
1579
1595
|
if (outcome.reason instanceof PendRejectedError && outcome.reason.conflict) anyConflict = true;
|
|
1580
1596
|
else anyHard = true;
|
|
1597
|
+
if (outcome.reason instanceof TransactionExpiredError) expired ??= outcome.reason;
|
|
1581
1598
|
}
|
|
1582
1599
|
}
|
|
1583
1600
|
|
|
@@ -1587,7 +1604,7 @@ export class TransactionCoordinator {
|
|
|
1587
1604
|
// only those started before the failure. Cancels are best-effort (cancelPhase
|
|
1588
1605
|
// swallows their errors) so they cannot mask the original pend failure.
|
|
1589
1606
|
await this.cancelPhase(actionId, pendedBlockIds);
|
|
1590
|
-
return { success: false, error: failure, staleLoss: anyConflict && !anyHard };
|
|
1607
|
+
return { success: false, error: failure, staleLoss: anyConflict && !anyHard, ...(expired === undefined ? {} : { expired }) };
|
|
1591
1608
|
}
|
|
1592
1609
|
|
|
1593
1610
|
return { success: true, pendedBlockIds };
|
|
@@ -1643,6 +1660,9 @@ export class TransactionCoordinator {
|
|
|
1643
1660
|
|
|
1644
1661
|
const pendResult = await this.transactor.pend(pendRequest);
|
|
1645
1662
|
if (!pendResult.success) {
|
|
1663
|
+
if (isExpiryFailure(pendResult)) {
|
|
1664
|
+
throw new TransactionExpiredError(pendResult.expired);
|
|
1665
|
+
}
|
|
1646
1666
|
// Retryability comes from the response itself: a producer that classified the failure sets
|
|
1647
1667
|
// `conflict`, and only where no producer set it do we fall back to inferring from
|
|
1648
1668
|
// `missing`/`pending`. Either way a conflict is an optimistic-concurrency loss, clearable
|
|
@@ -1671,6 +1691,7 @@ export class TransactionCoordinator {
|
|
|
1671
1691
|
committedCollections: Set<CollectionId>;
|
|
1672
1692
|
failedCollections: Set<CollectionId>;
|
|
1673
1693
|
staleLoss?: boolean;
|
|
1694
|
+
expired?: TransactionExpiredError;
|
|
1674
1695
|
}> {
|
|
1675
1696
|
// Fan out the independent per-collection commit-with-retry concurrently, then
|
|
1676
1697
|
// aggregate the committed/failed partition from the settled results.
|
|
@@ -1689,6 +1710,7 @@ export class TransactionCoordinator {
|
|
|
1689
1710
|
// re-driving.
|
|
1690
1711
|
let anyStale = false;
|
|
1691
1712
|
let anyHard = false;
|
|
1713
|
+
let expired: TransactionExpiredError | undefined;
|
|
1692
1714
|
for (const outcome of outcomes) {
|
|
1693
1715
|
if (outcome.status === 'fulfilled') {
|
|
1694
1716
|
const { collectionId, committed, error, stale } = outcome.value;
|
|
@@ -1698,6 +1720,7 @@ export class TransactionCoordinator {
|
|
|
1698
1720
|
failedCollections.add(collectionId);
|
|
1699
1721
|
if (error) errors.push(error);
|
|
1700
1722
|
if (stale) anyStale = true; else anyHard = true;
|
|
1723
|
+
expired ??= outcome.value.expired;
|
|
1701
1724
|
}
|
|
1702
1725
|
} else {
|
|
1703
1726
|
// commitCollection resolves rather than rejects, but treat any unexpected
|
|
@@ -1714,6 +1737,7 @@ export class TransactionCoordinator {
|
|
|
1714
1737
|
committedCollections,
|
|
1715
1738
|
failedCollections,
|
|
1716
1739
|
staleLoss: anyStale && !anyHard,
|
|
1740
|
+
...(expired === undefined ? {} : { expired }),
|
|
1717
1741
|
};
|
|
1718
1742
|
}
|
|
1719
1743
|
|
|
@@ -1734,7 +1758,7 @@ export class TransactionCoordinator {
|
|
|
1734
1758
|
* family as pendCollection's: recomputing `getNextRev()` here, after the pend round
|
|
1735
1759
|
* trips, could stamp the CommitRequest with a different number than the pend named. */
|
|
1736
1760
|
rev: number
|
|
1737
|
-
): Promise<{ collectionId: CollectionId; committed: boolean; error?: string; stale?: boolean }> {
|
|
1761
|
+
): Promise<{ collectionId: CollectionId; committed: boolean; error?: string; stale?: boolean; expired?: TransactionExpiredError }> {
|
|
1738
1762
|
const collection = this.collections.get(collectionId);
|
|
1739
1763
|
if (!collection) {
|
|
1740
1764
|
return { collectionId, committed: false, error: `Collection not found: ${collectionId}` };
|
|
@@ -1778,6 +1802,11 @@ export class TransactionCoordinator {
|
|
|
1778
1802
|
if (commitResult.success) {
|
|
1779
1803
|
return { collectionId, committed: true };
|
|
1780
1804
|
}
|
|
1805
|
+
if (isExpiryFailure(commitResult)) {
|
|
1806
|
+
// Not a stale loss: a re-drive's expiration comes from the same clock.
|
|
1807
|
+
const expired = new TransactionExpiredError(commitResult.expired);
|
|
1808
|
+
return { collectionId, committed: false, error: expired.message, expired };
|
|
1809
|
+
}
|
|
1781
1810
|
// Permanent stale failure: do not retry here. It IS a clean stale loss, though, so
|
|
1782
1811
|
// mark it retryable at the coordinator level (after a re-read advances the rev).
|
|
1783
1812
|
// NOTE: deliberately does NOT consult `isConflictFailure` / `StaleFailure.conflict`
|
|
@@ -1791,7 +1820,8 @@ export class TransactionCoordinator {
|
|
|
1791
1820
|
// stale-commit rejections as `{ success:false, conflict:true }` (returning, not
|
|
1792
1821
|
// throwing, is what keeps them out of the verbatim retry above) — but every returned
|
|
1793
1822
|
// failure still maps to `stale: true` here, and `isConflictFailure` covers that new
|
|
1794
|
-
// shape, so no behavior change is needed.
|
|
1823
|
+
// shape, so no behavior change is needed. The one returned refusal that is NOT a stale
|
|
1824
|
+
// loss, an expiry, is split off above. If a commit producer ever starts returning other
|
|
1795
1825
|
// hard commit rejections (validator policy, storage fault) as results too, gate
|
|
1796
1826
|
// `stale` on isConflictFailure here.
|
|
1797
1827
|
return {
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { peerIdFromString } from "../network/types.js";
|
|
2
2
|
import type { PeerId } from "../network/types.js";
|
|
3
|
-
import { highestStaleAt, isConflictFailure } from "../network/stale-failure.js";
|
|
3
|
+
import { highestStaleAt, isConflictFailure, isExpiryFailure } from "../network/stale-failure.js";
|
|
4
4
|
import { BlockUnavailableError, BlockPossiblyStaleError } from "../network/struct.js";
|
|
5
5
|
import { mergeDurability, withTornBlocks } from "../network/durability.js";
|
|
6
6
|
import { judgeCohortLineage, type CohortLineage, type MemberLineage } from "../network/lineage.js";
|
|
7
|
-
import type { ActionTransforms, ActionBlocks, ActionLineage, BlockActionStatus, ITransactor, PendSuccess, CommitSuccess, StaleFailure, IKeyNetwork, BlockId, GetBlockResults, PendResult, CommitResult, PendRequest, IRepo, BlockGets, Transforms, CommitRequest, ActionId, RepoCommitRequest, ClusterNomineesResult, CollectionId, IBlock, GetBlockResult, CoordinatorIntent, BlockUnavailableReason, BlockContentDigests, BlockBaseRevs, WriteDurability } from "../index.js";
|
|
7
|
+
import type { ActionTransforms, ActionBlocks, ActionLineage, BlockActionStatus, ITransactor, PendSuccess, CommitSuccess, StaleFailure, WriteFailure, IKeyNetwork, BlockId, GetBlockResults, PendResult, CommitResult, PendRequest, IRepo, BlockGets, Transforms, CommitRequest, ActionId, RepoCommitRequest, ClusterNomineesResult, CollectionId, IBlock, GetBlockResult, CoordinatorIntent, BlockUnavailableReason, BlockContentDigests, BlockBaseRevs, WriteDurability } from "../index.js";
|
|
8
8
|
import type { IBlockChangeNotifier, CollectionChangeListener } from "./change-notifier.js";
|
|
9
9
|
import { transformForBlockId, concatTransforms, concatTransform, transformsFromTransform, blockIdsForTransforms } from "../transform/helpers.js";
|
|
10
10
|
import { Tracker } from "../transform/tracker.js";
|
|
@@ -310,9 +310,9 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
310
310
|
// it: 'cohort-unreachable' is the one reason a caller may treat permissively (the
|
|
311
311
|
// answering node reached nobody, so its own view is all it has), and a partitioned
|
|
312
312
|
// coordinator answering first must not mask a well-connected one that positively
|
|
313
|
-
// established the block EXISTS ('claimed-elsewhere',
|
|
314
|
-
// held here). 'peers-unreachable' sits
|
|
315
|
-
// was reachable, without settling existence.
|
|
313
|
+
// established the block EXISTS ('claimed-elsewhere', 'named-by-log' — the caller's own
|
|
314
|
+
// floor names it — or 'unmaterializable' — records held here). 'peers-unreachable' sits
|
|
315
|
+
// between: it establishes that some of the cohort was reachable, without settling existence.
|
|
316
316
|
const unavailableRank = (reason: BlockUnavailableReason): number =>
|
|
317
317
|
reason === 'cohort-unreachable' ? 0 : reason === 'peers-unreachable' ? 1 : 2;
|
|
318
318
|
const rankOf = (r: unknown): number => {
|
|
@@ -727,35 +727,9 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
727
727
|
const stale = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && !b.request!.response!.success));
|
|
728
728
|
if (stale.length > 0) { // Any active stale failures should preempt reporting connection or other potential transient errors (we have information)
|
|
729
729
|
log('pend:stale actionId=%s staleCount=%d', blockAction.actionId, stale.length);
|
|
730
|
-
//
|
|
731
|
-
//
|
|
732
|
-
|
|
733
|
-
const reason = stale.map(b => (b.request!.response! as StaleFailure).reason).find(r => r !== undefined);
|
|
734
|
-
// This response is REBUILT from the per-batch ones rather than forwarded, so
|
|
735
|
-
// retryability has to be carried explicitly or it is lost: a batch whose failure was
|
|
736
|
-
// a confirmed lost race can arrive with neither `missing` nor `pending` (see
|
|
737
|
-
// CoordinatorRepo.classifyStaleRejection), and the aggregate would then look like a
|
|
738
|
-
// hard rejection to `isConflictFailure`. Any conflicting batch makes the aggregate a
|
|
739
|
-
// conflict — the pend failed as a whole, and a re-read/rebase can clear it.
|
|
740
|
-
// NOTE: `some`, not `every`, so a pend whose batches mix a lost race with a genuine hard
|
|
741
|
-
// rejection is reported retryable and burns its (bounded, backed-off) retry budget before
|
|
742
|
-
// failing. Deliberate: an unclassified reason-only response from an older peer is
|
|
743
|
-
// indistinguishable from a hard rejection here, and `every` would refuse to retry a real
|
|
744
|
-
// race whenever one batch came from such a peer. Revisit if every producer sets `conflict`
|
|
745
|
-
// (then `every` is both safe and tighter), or if mixed-outcome pends show up as wasted
|
|
746
|
-
// retry latency in practice.
|
|
747
|
-
const conflict = stale.some(b => isConflictFailure(b.request!.response! as StaleFailure));
|
|
748
|
-
// Deliberately NOT first-wins like `reason` above — `highestStaleAt` takes the largest
|
|
749
|
-
// confirmed revision, which is the binding constraint on the client's next request.
|
|
750
|
-
// Its doc comment carries the rule and the one-pend-one-collection assumption it rests on.
|
|
751
|
-
const staleAt = highestStaleAt(stale.map(b => (b.request!.response! as StaleFailure).staleAt));
|
|
752
|
-
return {
|
|
753
|
-
success: false,
|
|
754
|
-
conflict,
|
|
755
|
-
...(reason === undefined ? {} : { reason }),
|
|
756
|
-
...(staleAt === undefined ? {} : { staleAt }),
|
|
757
|
-
missing: distinctBlockActionTransforms(stale.flatMap(b => (b.request!.response! as StaleFailure).missing).filter((x): x is ActionTransforms => x !== undefined)),
|
|
758
|
-
};
|
|
730
|
+
// This response is REBUILT from the per-batch ones rather than forwarded, by the same
|
|
731
|
+
// rule the commit side uses — see `refusalFrom` for what is carried and why.
|
|
732
|
+
return refusalFrom(stale.map(b => b.request!.response! as WriteFailure));
|
|
759
733
|
}
|
|
760
734
|
throw error; // No stale failures, report the original error
|
|
761
735
|
}
|
|
@@ -1101,9 +1075,9 @@ export class NetworkTransactor implements ITransactor, IBlockChangeNotifier {
|
|
|
1101
1075
|
* sweep — both must distinguish a confirmed conflict (return it; the caller cancels and
|
|
1102
1076
|
* re-drives) from a transient fault (throw / tolerate).
|
|
1103
1077
|
*/
|
|
1104
|
-
private staleFromBatches(batches: CoordinatorBatch<BlockId[], CommitResult>[]):
|
|
1078
|
+
private staleFromBatches(batches: CoordinatorBatch<BlockId[], CommitResult>[]): WriteFailure | undefined {
|
|
1105
1079
|
const stale = Array.from(allBatches(batches, b => b.request?.isResponse as boolean && !b.request!.response!.success));
|
|
1106
|
-
return stale.length === 0 ? undefined : refusalFrom(stale.map(b => b.request!.response! as
|
|
1080
|
+
return stale.length === 0 ? undefined : refusalFrom(stale.map(b => b.request!.response! as WriteFailure));
|
|
1107
1081
|
}
|
|
1108
1082
|
|
|
1109
1083
|
/**
|
|
@@ -1440,14 +1414,33 @@ function confirmedDurabilities(batches: CoordinatorBatch<BlockId[], CommitResult
|
|
|
1440
1414
|
}
|
|
1441
1415
|
|
|
1442
1416
|
/**
|
|
1443
|
-
* One
|
|
1444
|
-
*
|
|
1445
|
-
*
|
|
1446
|
-
*
|
|
1447
|
-
*
|
|
1448
|
-
*
|
|
1417
|
+
* One refusal standing for every refused response in `responses` — the single rebuild both
|
|
1418
|
+
* {@link NetworkTransactor.pend} and the commit paths use, since the action-level answer is rebuilt
|
|
1419
|
+
* from per-batch ones rather than forwarded, and anything not carried here is lost.
|
|
1420
|
+
*
|
|
1421
|
+
* - An **expiry** anywhere ({@link isExpiryFailure}) is the answer, verbatim — the first one. No
|
|
1422
|
+
* retry can succeed while that cohort's clocks disagree with the writer's, so a conflict beside it
|
|
1423
|
+
* must not make the write look retryable, and its numbers must reach the writer whole.
|
|
1424
|
+
* - Otherwise a {@link StaleFailure}: `reason` is the first one any response gave (the only diagnostic
|
|
1425
|
+
* that survives an exhausted retry budget and the coordinator's error text — a refusal whose reason
|
|
1426
|
+
* is `commit-not-durable` must not read as "stale commit", a rival's win); `staleAt` is the highest
|
|
1427
|
+
* confirmed revision (`highestStaleAt`, the binding constraint on the next request); and `conflict`
|
|
1428
|
+
* holds when ANY response was a conflict, because a batch whose failure was a confirmed lost race
|
|
1429
|
+
* can arrive with neither `missing` nor `pending` (`CoordinatorRepo.classifyStaleRejection`), so
|
|
1430
|
+
* retryability has to be carried explicitly. A reason-only refusal comes out with `missing: []`.
|
|
1431
|
+
*
|
|
1432
|
+
* NOTE: `some`, not `every`, for `conflict`, so a write whose batches mix a lost race with a genuine
|
|
1433
|
+
* hard rejection is reported retryable and burns its (bounded, backed-off) retry budget before
|
|
1434
|
+
* failing. Deliberate: an unclassified reason-only response from an older peer is indistinguishable
|
|
1435
|
+
* from a hard rejection here, and `every` would refuse to retry a real race whenever one batch came
|
|
1436
|
+
* from such a peer. Revisit if every producer sets `conflict` (then `every` is both safe and tighter),
|
|
1437
|
+
* or if mixed-outcome writes show up as wasted retry latency in practice.
|
|
1449
1438
|
*/
|
|
1450
|
-
function refusalFrom(responses:
|
|
1439
|
+
function refusalFrom(responses: WriteFailure[]): WriteFailure {
|
|
1440
|
+
const expired = responses.find(isExpiryFailure);
|
|
1441
|
+
if (expired !== undefined) {
|
|
1442
|
+
return expired;
|
|
1443
|
+
}
|
|
1451
1444
|
const staleAt = highestStaleAt(responses.map(r => r.staleAt));
|
|
1452
1445
|
const reason = responses.map(r => r.reason).find(r => r !== undefined);
|
|
1453
1446
|
return {
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ISO-8601 for a representable instant (unix ms), the bare number otherwise. For log and error prose
|
|
3
|
+
* about instants that arrived off the wire: `Date.prototype.toISOString` throws on an invalid date
|
|
4
|
+
* (beyond ±8.64e15 ms, or not a number at all), and a throw while describing a refusal would replace
|
|
5
|
+
* the refusal.
|
|
6
|
+
*/
|
|
7
|
+
export function formatInstant(ms: number): string {
|
|
8
|
+
const date = new Date(ms);
|
|
9
|
+
return Number.isNaN(date.getTime()) ? String(ms) : date.toISOString();
|
|
10
|
+
}
|