@feltdb/core 0.5.1 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cell.d.ts +235 -0
- package/dist/cell.d.ts.map +1 -0
- package/dist/cell.js +1090 -0
- package/dist/cli/index.js +1 -1
- package/dist/collection.d.ts +4 -2
- package/dist/collection.d.ts.map +1 -1
- package/dist/collection.js +28 -25
- package/dist/create/package-versions.js +1 -1
- package/dist/create/server-source/crates/feltdb/src/lib.rs +173 -0
- package/dist/create/server-source/crates/feltdb/src/managed_cas_tests.rs +231 -0
- package/dist/create/server-source/crates/feltdb-server/src/main.rs +108 -1
- package/dist/create/server-source/crates/feltdb-server/src/request_telemetry.rs +1 -1
- package/dist/create/server-source/crates/feltdb-server/tests/revision_recovery_integration_test.rs +219 -0
- package/dist/db.d.ts +16 -0
- package/dist/db.d.ts.map +1 -1
- package/dist/db.js +21 -0
- package/dist/feltdb.d.ts +7 -1
- package/dist/feltdb.d.ts.map +1 -1
- package/dist/file-db.d.ts +5 -0
- package/dist/file-db.d.ts.map +1 -1
- package/dist/file-db.js +72 -9
- package/dist/http-db.d.ts +13 -0
- package/dist/http-db.d.ts.map +1 -1
- package/dist/http-db.js +41 -0
- package/dist/http-server.d.ts +42 -0
- package/dist/http-server.d.ts.map +1 -0
- package/dist/http-server.js +182 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/indexeddb-conformance.spec.d.ts +21 -0
- package/dist/indexeddb-conformance.spec.d.ts.map +1 -0
- package/dist/indexeddb-conformance.spec.js +103 -0
- package/dist/indexeddb-db.d.ts +8 -0
- package/dist/indexeddb-db.d.ts.map +1 -1
- package/dist/indexeddb-db.js +69 -0
- package/dist/memory-db.d.ts +2 -0
- package/dist/memory-db.d.ts.map +1 -1
- package/dist/memory-db.js +9 -3
- package/dist/studio-app/assets/{feltdb_wasm-DYPuS6Ky.js → feltdb_wasm-h9mxesnH.js} +1 -1
- package/dist/studio-app/assets/feltdb_wasm_bg-B8U4A1n1.wasm +0 -0
- package/dist/studio-app/assets/index-Z92yFC4z.js +28 -0
- package/dist/studio-app/index.html +1 -1
- package/dist/wasm/feltdb_wasm_bg.wasm +0 -0
- package/package.json +1 -1
- package/dist/studio-app/assets/feltdb_wasm_bg-DEwA82pF.wasm +0 -0
- package/dist/studio-app/assets/index-_qqVLfw1.js +0 -28
package/dist/cell.js
ADDED
|
@@ -0,0 +1,1090 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Durable Cell: independently-versioned mutation boundary
|
|
3
|
+
*
|
|
4
|
+
* A Cell is the smallest durable authority boundary in FeltDB.
|
|
5
|
+
* Each cell owns its own version, state, and mutation serialization.
|
|
6
|
+
*/
|
|
7
|
+
import { Collection } from './collection.js';
|
|
8
|
+
export const NO_CHANGE = Symbol('FELTDB_NO_CHANGE');
|
|
9
|
+
export const AUTHORITY_CONFLICT = Symbol('FELTDB_AUTHORITY_CONFLICT');
|
|
10
|
+
/**
|
|
11
|
+
* CellImpl implements the Cell interface with authority and lifecycle support.
|
|
12
|
+
* Uses per-cell locking to serialize mutations and authority transitions.
|
|
13
|
+
* Epoch-based fencing prevents stale authority from mutating.
|
|
14
|
+
*
|
|
15
|
+
* Replication: CellImpl can also track replica state separately.
|
|
16
|
+
* A replica is read-only: replication transfers knowledge, not authority.
|
|
17
|
+
*/
|
|
18
|
+
export class CellImpl {
|
|
19
|
+
constructor(jsDb, cellId) {
|
|
20
|
+
this.mutationQueue = [];
|
|
21
|
+
this.isLocked = false;
|
|
22
|
+
this.changeHandlers = [];
|
|
23
|
+
this.isReplica = false;
|
|
24
|
+
this.id = cellId;
|
|
25
|
+
this.jsDb = jsDb;
|
|
26
|
+
this.cellCollection = new Collection(jsDb, '_cells', undefined, undefined, false);
|
|
27
|
+
this.replicaCollection = new Collection(jsDb, '_replicas', undefined, undefined, false);
|
|
28
|
+
this.failureObservationCollection = new Collection(jsDb, '_failure_observations', undefined, undefined, false);
|
|
29
|
+
}
|
|
30
|
+
setReplica(isReplica) {
|
|
31
|
+
this.isReplica = isReplica;
|
|
32
|
+
}
|
|
33
|
+
isReplicaCell() {
|
|
34
|
+
return this.isReplica;
|
|
35
|
+
}
|
|
36
|
+
async applyReplication(snapshot) {
|
|
37
|
+
return this.acquireLock(async () => {
|
|
38
|
+
const current = await this.getUnlocked();
|
|
39
|
+
// Replication can only move forward
|
|
40
|
+
if (current && current.version > snapshot.version) {
|
|
41
|
+
throw new Error(`REPLICATION_CONFLICT: cannot apply version ${snapshot.version}, current is ${current.version}`);
|
|
42
|
+
}
|
|
43
|
+
// Same version must have identical state
|
|
44
|
+
if (current && current.version === snapshot.version) {
|
|
45
|
+
if (JSON.stringify(current.value) !== JSON.stringify(snapshot.value)) {
|
|
46
|
+
const conflictReason = `version ${snapshot.version} has divergent state`;
|
|
47
|
+
await this.replicaCollection.insert({
|
|
48
|
+
cellId: this.id,
|
|
49
|
+
version: current.version,
|
|
50
|
+
value: current.value,
|
|
51
|
+
updatedAt: current.updatedAt,
|
|
52
|
+
authority: current.authority,
|
|
53
|
+
placement: current.placement,
|
|
54
|
+
lease: current.lease,
|
|
55
|
+
divergent: true,
|
|
56
|
+
conflictReason,
|
|
57
|
+
}, this.encodeKey());
|
|
58
|
+
throw new Error(`REPLICATION_CONFLICT: ${conflictReason}`);
|
|
59
|
+
}
|
|
60
|
+
// Identical state at same version is idempotent, no-op
|
|
61
|
+
await this.clearDivergenceMarker();
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
// Apply replication: update the replicated state
|
|
65
|
+
// For PR 6A, we store replica state as regular cell state but marked as replica
|
|
66
|
+
const replicaState = {
|
|
67
|
+
id: snapshot.cellId,
|
|
68
|
+
version: snapshot.version,
|
|
69
|
+
value: snapshot.value,
|
|
70
|
+
updatedAt: snapshot.updatedAt,
|
|
71
|
+
authority: {
|
|
72
|
+
owner: snapshot.authority.owner,
|
|
73
|
+
epoch: snapshot.authority.epoch,
|
|
74
|
+
lifecycle: snapshot.authority.lifecycle,
|
|
75
|
+
},
|
|
76
|
+
placement: {
|
|
77
|
+
nodeId: snapshot.placement.nodeId,
|
|
78
|
+
},
|
|
79
|
+
lease: snapshot.lease,
|
|
80
|
+
};
|
|
81
|
+
const key = this.encodeKey();
|
|
82
|
+
if (!current) {
|
|
83
|
+
// New replica: insert the first version
|
|
84
|
+
await this.cellCollection.insert(replicaState, key);
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
// Update existing replica to new version
|
|
88
|
+
// Use version comparison for CAS (pass current version and epoch)
|
|
89
|
+
const result = await this.cellCollection.updateIfVersion(key, current.version, replicaState, current.authority.epoch);
|
|
90
|
+
if (!result.updated) {
|
|
91
|
+
// Concurrent replication delivery - just log and move forward
|
|
92
|
+
// This is expected in concurrent scenarios
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
// A successfully applied canonical snapshot repairs any older divergence
|
|
96
|
+
// marker and makes the replica eligible for later evaluation again.
|
|
97
|
+
await this.clearDivergenceMarker();
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
/** Removing an absent repair marker is an idempotent replication cleanup. */
|
|
101
|
+
async clearDivergenceMarker() {
|
|
102
|
+
try {
|
|
103
|
+
await this.replicaCollection.delete(this.encodeKey());
|
|
104
|
+
}
|
|
105
|
+
catch (error) {
|
|
106
|
+
if (error instanceof Error && /^Delete failed: record not found$/i.test(error.message)) {
|
|
107
|
+
return;
|
|
108
|
+
}
|
|
109
|
+
throw error;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
encodeKey() {
|
|
113
|
+
const hash = this.hashId(this.id);
|
|
114
|
+
return `${hash}`;
|
|
115
|
+
}
|
|
116
|
+
hashId(id) {
|
|
117
|
+
let hash = 0;
|
|
118
|
+
for (let i = 0; i < id.length; i++) {
|
|
119
|
+
const char = id.charCodeAt(i);
|
|
120
|
+
hash = ((hash << 5) - hash) + char;
|
|
121
|
+
hash = hash & hash;
|
|
122
|
+
}
|
|
123
|
+
return Math.abs(hash).toString(16);
|
|
124
|
+
}
|
|
125
|
+
async get() {
|
|
126
|
+
const key = this.encodeKey();
|
|
127
|
+
const state = await this.cellCollection.get(key);
|
|
128
|
+
if (!state)
|
|
129
|
+
return null;
|
|
130
|
+
return this.snapshotFromState(state);
|
|
131
|
+
}
|
|
132
|
+
async set(value, options) {
|
|
133
|
+
return this.acquireLock(async () => {
|
|
134
|
+
// Replicas are read-only
|
|
135
|
+
if (this.isReplica) {
|
|
136
|
+
throw new Error(`REPLICA_READ_ONLY: Cell ${this.id} is a read-only replica`);
|
|
137
|
+
}
|
|
138
|
+
const current = await this.getUnlocked();
|
|
139
|
+
// Check retirement
|
|
140
|
+
if (current?.authority.lifecycle === 'retired') {
|
|
141
|
+
throw new Error(`Cell ${this.id} is retired`);
|
|
142
|
+
}
|
|
143
|
+
// Capture current epoch for atomic fencing
|
|
144
|
+
// This epoch will be verified by backend CAS atomically with state mutation
|
|
145
|
+
const expectedEpoch = current?.authority.epoch ?? 0;
|
|
146
|
+
// Validate provided epoch if specified (optional, for explicit fencing)
|
|
147
|
+
if (options?.authorityEpoch !== undefined && current) {
|
|
148
|
+
if (options.authorityEpoch !== expectedEpoch) {
|
|
149
|
+
throw new Error(`AUTHORITY_CONFLICT: expected epoch ${expectedEpoch}, got ${options.authorityEpoch}`);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
// Validate lease if specified
|
|
153
|
+
if (options?.leaseId !== undefined) {
|
|
154
|
+
if (!current?.lease) {
|
|
155
|
+
throw new Error(`LEASE_INVALID: no active lease`);
|
|
156
|
+
}
|
|
157
|
+
if (current.lease.leaseId !== options.leaseId) {
|
|
158
|
+
throw new Error(`LEASE_INVALID: lease ID mismatch`);
|
|
159
|
+
}
|
|
160
|
+
if (current.lease.expiresAt <= Date.now()) {
|
|
161
|
+
throw new Error(`LEASE_EXPIRED: lease has expired`);
|
|
162
|
+
}
|
|
163
|
+
if (current.lease.epoch !== expectedEpoch) {
|
|
164
|
+
throw new Error(`LEASE_INVALID: lease epoch does not match authority epoch`);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
const expectedVersion = current?.version ?? 0;
|
|
168
|
+
const authority = current?.authority ?? { owner: null, epoch: 0, lifecycle: 'active' };
|
|
169
|
+
const placement = current?.placement ?? { nodeId: null };
|
|
170
|
+
const lease = current?.lease ?? null;
|
|
171
|
+
const newState = {
|
|
172
|
+
id: this.id,
|
|
173
|
+
version: expectedVersion + 1,
|
|
174
|
+
value,
|
|
175
|
+
updatedAt: new Date().toISOString(),
|
|
176
|
+
authority,
|
|
177
|
+
placement,
|
|
178
|
+
lease,
|
|
179
|
+
};
|
|
180
|
+
const key = this.encodeKey();
|
|
181
|
+
if (expectedVersion === 0) {
|
|
182
|
+
await this.cellCollection.insert(newState, key);
|
|
183
|
+
}
|
|
184
|
+
else {
|
|
185
|
+
// Pass captured epoch to backend CAS for atomic verification
|
|
186
|
+
const result = await this.cellCollection.updateIfVersion(key, expectedVersion, newState, expectedEpoch, options?.leaseId);
|
|
187
|
+
if (!result.updated) {
|
|
188
|
+
throw new Error(`Cell ${this.id} version conflict`);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
this.emitChange({
|
|
192
|
+
cellId: this.id,
|
|
193
|
+
version: newState.version,
|
|
194
|
+
value,
|
|
195
|
+
timestamp: newState.updatedAt,
|
|
196
|
+
epoch: newState.authority.epoch,
|
|
197
|
+
});
|
|
198
|
+
return this.snapshotFromState(newState);
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
async update(updater, options) {
|
|
202
|
+
return this.acquireLock(async () => {
|
|
203
|
+
// Replicas are read-only
|
|
204
|
+
if (this.isReplica) {
|
|
205
|
+
throw new Error(`REPLICA_READ_ONLY: Cell ${this.id} is a read-only replica`);
|
|
206
|
+
}
|
|
207
|
+
const current = await this.getUnlocked();
|
|
208
|
+
// Check retirement
|
|
209
|
+
if (current?.authority.lifecycle === 'retired') {
|
|
210
|
+
throw new Error(`Cell ${this.id} is retired`);
|
|
211
|
+
}
|
|
212
|
+
// Capture current epoch for atomic fencing
|
|
213
|
+
// This epoch will be verified by backend CAS atomically with state mutation
|
|
214
|
+
const expectedEpoch = current?.authority.epoch ?? 0;
|
|
215
|
+
// Validate provided epoch if specified (optional, for explicit fencing)
|
|
216
|
+
if (options?.authorityEpoch !== undefined && current) {
|
|
217
|
+
if (options.authorityEpoch !== expectedEpoch) {
|
|
218
|
+
throw new Error(`AUTHORITY_CONFLICT: expected epoch ${expectedEpoch}, got ${options.authorityEpoch}`);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
// Validate lease if specified
|
|
222
|
+
if (options?.leaseId !== undefined) {
|
|
223
|
+
if (!current?.lease) {
|
|
224
|
+
throw new Error(`LEASE_INVALID: no active lease`);
|
|
225
|
+
}
|
|
226
|
+
if (current.lease.leaseId !== options.leaseId) {
|
|
227
|
+
throw new Error(`LEASE_INVALID: lease ID mismatch`);
|
|
228
|
+
}
|
|
229
|
+
if (current.lease.expiresAt <= Date.now()) {
|
|
230
|
+
throw new Error(`LEASE_EXPIRED: lease has expired`);
|
|
231
|
+
}
|
|
232
|
+
if (current.lease.epoch !== expectedEpoch) {
|
|
233
|
+
throw new Error(`LEASE_INVALID: lease epoch does not match authority epoch`);
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
const currentValue = current?.value ?? null;
|
|
237
|
+
const currentVersion = current?.version ?? 0;
|
|
238
|
+
const nextValue = updater(currentValue);
|
|
239
|
+
if (nextValue === NO_CHANGE) {
|
|
240
|
+
if (current)
|
|
241
|
+
return this.snapshotFromState(current);
|
|
242
|
+
throw new Error(`Cell ${this.id} update returned NO_CHANGE but cell does not exist`);
|
|
243
|
+
}
|
|
244
|
+
const authority = current?.authority ?? { owner: null, epoch: 0, lifecycle: 'active' };
|
|
245
|
+
const placement = current?.placement ?? { nodeId: null };
|
|
246
|
+
const lease = current?.lease ?? null;
|
|
247
|
+
const newState = {
|
|
248
|
+
id: this.id,
|
|
249
|
+
version: currentVersion + 1,
|
|
250
|
+
value: nextValue,
|
|
251
|
+
updatedAt: new Date().toISOString(),
|
|
252
|
+
authority,
|
|
253
|
+
placement,
|
|
254
|
+
lease,
|
|
255
|
+
};
|
|
256
|
+
const key = this.encodeKey();
|
|
257
|
+
if (currentVersion === 0) {
|
|
258
|
+
await this.cellCollection.insert(newState, key);
|
|
259
|
+
}
|
|
260
|
+
else {
|
|
261
|
+
// Pass captured epoch to backend CAS for atomic verification
|
|
262
|
+
const result = await this.cellCollection.updateIfVersion(key, currentVersion, newState, expectedEpoch, options?.leaseId);
|
|
263
|
+
if (!result.updated) {
|
|
264
|
+
throw new Error(`Cell ${this.id} conflict during update`);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
this.emitChange({
|
|
268
|
+
cellId: this.id,
|
|
269
|
+
version: newState.version,
|
|
270
|
+
value: nextValue,
|
|
271
|
+
timestamp: newState.updatedAt,
|
|
272
|
+
epoch: newState.authority.epoch,
|
|
273
|
+
});
|
|
274
|
+
return this.snapshotFromState(newState);
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
async transact(fn) {
|
|
278
|
+
return this.acquireLock(async () => {
|
|
279
|
+
const tx = {
|
|
280
|
+
get: () => this.getUnlocked(),
|
|
281
|
+
set: (value, options) => this.setUnlocked(value, options),
|
|
282
|
+
update: (updater, options) => this.updateUnlocked(updater, options),
|
|
283
|
+
};
|
|
284
|
+
return fn(tx);
|
|
285
|
+
});
|
|
286
|
+
}
|
|
287
|
+
async transitionAuthority(options) {
|
|
288
|
+
return this.acquireLock(async () => {
|
|
289
|
+
// Replicas are read-only
|
|
290
|
+
if (this.isReplica) {
|
|
291
|
+
throw new Error(`REPLICA_READ_ONLY: Cell ${this.id} is a read-only replica`);
|
|
292
|
+
}
|
|
293
|
+
const current = await this.getUnlocked();
|
|
294
|
+
if (!current) {
|
|
295
|
+
throw new Error(`Cell ${this.id} does not exist`);
|
|
296
|
+
}
|
|
297
|
+
// Check retirement
|
|
298
|
+
if (current.authority.lifecycle === 'retired') {
|
|
299
|
+
throw new Error(`Cell ${this.id} is retired`);
|
|
300
|
+
}
|
|
301
|
+
// Verify epoch
|
|
302
|
+
if (current.authority.epoch !== options.expectedEpoch) {
|
|
303
|
+
throw new Error(`AUTHORITY_CONFLICT: expected epoch ${options.expectedEpoch}, got ${current.authority.epoch}`);
|
|
304
|
+
}
|
|
305
|
+
// Validate lifecycle transition if provided
|
|
306
|
+
if (options.lifecycle !== undefined) {
|
|
307
|
+
this.validateLifecycleTransition(current.authority.lifecycle, options.lifecycle);
|
|
308
|
+
}
|
|
309
|
+
// Create new authority state with incremented epoch
|
|
310
|
+
const newAuthority = {
|
|
311
|
+
owner: options.owner !== undefined ? options.owner : current.authority.owner,
|
|
312
|
+
epoch: current.authority.epoch + 1,
|
|
313
|
+
lifecycle: options.lifecycle !== undefined ? options.lifecycle : current.authority.lifecycle,
|
|
314
|
+
};
|
|
315
|
+
// Authority transition increments version, clears lease (old epoch invalid)
|
|
316
|
+
const newState = {
|
|
317
|
+
id: this.id,
|
|
318
|
+
version: current.version + 1,
|
|
319
|
+
value: current.value,
|
|
320
|
+
updatedAt: new Date().toISOString(),
|
|
321
|
+
authority: newAuthority,
|
|
322
|
+
placement: current.placement,
|
|
323
|
+
lease: null,
|
|
324
|
+
};
|
|
325
|
+
const key = this.encodeKey();
|
|
326
|
+
// Pass current epoch to backend CAS for atomic verification
|
|
327
|
+
const result = await this.cellCollection.updateIfVersion(key, current.version, newState, current.authority.epoch);
|
|
328
|
+
if (!result.updated) {
|
|
329
|
+
throw new Error(`AUTHORITY_CONFLICT: epoch changed during transition`);
|
|
330
|
+
}
|
|
331
|
+
// Emit change event for authority transition
|
|
332
|
+
this.emitChange({
|
|
333
|
+
cellId: this.id,
|
|
334
|
+
version: newState.version,
|
|
335
|
+
value: current.value,
|
|
336
|
+
timestamp: newState.updatedAt,
|
|
337
|
+
epoch: newState.authority.epoch,
|
|
338
|
+
});
|
|
339
|
+
return this.snapshotFromState(newState);
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
async delegate(options) {
|
|
343
|
+
return this.acquireLock(async () => {
|
|
344
|
+
// Replicas are read-only
|
|
345
|
+
if (this.isReplica) {
|
|
346
|
+
throw new Error(`REPLICA_READ_ONLY: Cell ${this.id} is a read-only replica`);
|
|
347
|
+
}
|
|
348
|
+
const current = await this.getUnlocked();
|
|
349
|
+
if (!current) {
|
|
350
|
+
throw new Error(`Cell ${this.id} does not exist`);
|
|
351
|
+
}
|
|
352
|
+
// Check retirement
|
|
353
|
+
if (current.authority.lifecycle === 'retired') {
|
|
354
|
+
throw new Error(`Cell ${this.id} is retired`);
|
|
355
|
+
}
|
|
356
|
+
// Verify epoch
|
|
357
|
+
if (current.authority.epoch !== options.expectedEpoch) {
|
|
358
|
+
throw new Error(`AUTHORITY_CONFLICT: expected epoch ${options.expectedEpoch}, got ${current.authority.epoch}`);
|
|
359
|
+
}
|
|
360
|
+
// Verify from owner
|
|
361
|
+
if (current.authority.owner !== options.from) {
|
|
362
|
+
throw new Error(`AUTHORITY_CONFLICT: expected owner ${options.from}, got ${current.authority.owner}`);
|
|
363
|
+
}
|
|
364
|
+
// Create new authority with transferred owner and incremented epoch
|
|
365
|
+
const newAuthority = {
|
|
366
|
+
owner: options.to,
|
|
367
|
+
epoch: current.authority.epoch + 1,
|
|
368
|
+
lifecycle: current.authority.lifecycle,
|
|
369
|
+
};
|
|
370
|
+
// Delegation increments version, clears lease (old epoch invalid)
|
|
371
|
+
const newState = {
|
|
372
|
+
id: this.id,
|
|
373
|
+
version: current.version + 1,
|
|
374
|
+
value: current.value,
|
|
375
|
+
updatedAt: new Date().toISOString(),
|
|
376
|
+
authority: newAuthority,
|
|
377
|
+
placement: current.placement,
|
|
378
|
+
lease: null,
|
|
379
|
+
};
|
|
380
|
+
const key = this.encodeKey();
|
|
381
|
+
// Pass current epoch to backend CAS for atomic verification
|
|
382
|
+
const result = await this.cellCollection.updateIfVersion(key, current.version, newState, current.authority.epoch);
|
|
383
|
+
if (!result.updated) {
|
|
384
|
+
throw new Error(`AUTHORITY_CONFLICT: authority changed during delegation`);
|
|
385
|
+
}
|
|
386
|
+
// Emit change event for delegation
|
|
387
|
+
this.emitChange({
|
|
388
|
+
cellId: this.id,
|
|
389
|
+
version: newState.version,
|
|
390
|
+
value: current.value,
|
|
391
|
+
timestamp: newState.updatedAt,
|
|
392
|
+
epoch: newState.authority.epoch,
|
|
393
|
+
});
|
|
394
|
+
return this.snapshotFromState(newState);
|
|
395
|
+
});
|
|
396
|
+
}
|
|
397
|
+
async updatePlacement(options) {
|
|
398
|
+
return this.acquireLock(async () => {
|
|
399
|
+
// Replicas are read-only
|
|
400
|
+
if (this.isReplica) {
|
|
401
|
+
throw new Error(`REPLICA_READ_ONLY: Cell ${this.id} is a read-only replica`);
|
|
402
|
+
}
|
|
403
|
+
const current = await this.getUnlocked();
|
|
404
|
+
if (!current) {
|
|
405
|
+
throw new Error(`Cell ${this.id} does not exist`);
|
|
406
|
+
}
|
|
407
|
+
// Check retirement
|
|
408
|
+
if (current.authority.lifecycle === 'retired') {
|
|
409
|
+
throw new Error(`Cell ${this.id} is retired`);
|
|
410
|
+
}
|
|
411
|
+
// Capture current epoch for atomic fencing during placement update
|
|
412
|
+
const expectedEpoch = current.authority.epoch;
|
|
413
|
+
// Create new placement
|
|
414
|
+
const newPlacement = {
|
|
415
|
+
nodeId: options.nodeId,
|
|
416
|
+
};
|
|
417
|
+
// Placement update increments version but does NOT change epoch, authority, or lease
|
|
418
|
+
const newState = {
|
|
419
|
+
id: this.id,
|
|
420
|
+
version: current.version + 1,
|
|
421
|
+
value: current.value,
|
|
422
|
+
updatedAt: new Date().toISOString(),
|
|
423
|
+
authority: current.authority,
|
|
424
|
+
placement: newPlacement,
|
|
425
|
+
lease: current.lease,
|
|
426
|
+
};
|
|
427
|
+
const key = this.encodeKey();
|
|
428
|
+
// Pass current epoch to backend CAS for atomic verification
|
|
429
|
+
const result = await this.cellCollection.updateIfVersion(key, current.version, newState, expectedEpoch);
|
|
430
|
+
if (!result.updated) {
|
|
431
|
+
throw new Error(`Cell ${this.id} conflict during placement update`);
|
|
432
|
+
}
|
|
433
|
+
// Emit change event for placement update
|
|
434
|
+
this.emitChange({
|
|
435
|
+
cellId: this.id,
|
|
436
|
+
version: newState.version,
|
|
437
|
+
value: current.value,
|
|
438
|
+
timestamp: newState.updatedAt,
|
|
439
|
+
epoch: newState.authority.epoch,
|
|
440
|
+
});
|
|
441
|
+
return this.snapshotFromState(newState);
|
|
442
|
+
});
|
|
443
|
+
}
|
|
444
|
+
async acquireLease(options) {
|
|
445
|
+
return this.acquireLock(async () => {
|
|
446
|
+
// Replicas are read-only
|
|
447
|
+
if (this.isReplica) {
|
|
448
|
+
throw new Error(`REPLICA_READ_ONLY: Cell ${this.id} is a read-only replica`);
|
|
449
|
+
}
|
|
450
|
+
const current = await this.getUnlocked();
|
|
451
|
+
if (!current) {
|
|
452
|
+
throw new Error(`Cell ${this.id} does not exist`);
|
|
453
|
+
}
|
|
454
|
+
// Cannot acquire lease on retired cell
|
|
455
|
+
if (current.authority.lifecycle === 'retired') {
|
|
456
|
+
throw new Error(`Cell ${this.id} is retired`);
|
|
457
|
+
}
|
|
458
|
+
// Cannot acquire lease on suspended cell
|
|
459
|
+
if (current.authority.lifecycle === 'suspended') {
|
|
460
|
+
throw new Error(`Cell ${this.id} is suspended`);
|
|
461
|
+
}
|
|
462
|
+
// Check if existing lease is still valid
|
|
463
|
+
if (current.lease && current.lease.expiresAt > Date.now()) {
|
|
464
|
+
throw new Error(`LEASE_CONFLICT: lease already held by ${current.lease.holder}`);
|
|
465
|
+
}
|
|
466
|
+
// Capture current epoch
|
|
467
|
+
const expectedEpoch = current.authority.epoch;
|
|
468
|
+
// Generate new lease ID
|
|
469
|
+
const leaseId = `${this.id}:${Date.now()}:${Math.random().toString(36).slice(2)}`;
|
|
470
|
+
// Create new lease
|
|
471
|
+
const newLease = {
|
|
472
|
+
holder: options.holder,
|
|
473
|
+
leaseId,
|
|
474
|
+
epoch: expectedEpoch,
|
|
475
|
+
expiresAt: Date.now() + options.durationMs,
|
|
476
|
+
};
|
|
477
|
+
// Lease acquisition increments version
|
|
478
|
+
const newState = {
|
|
479
|
+
id: this.id,
|
|
480
|
+
version: current.version + 1,
|
|
481
|
+
value: current.value,
|
|
482
|
+
updatedAt: new Date().toISOString(),
|
|
483
|
+
authority: current.authority,
|
|
484
|
+
placement: current.placement,
|
|
485
|
+
lease: newLease,
|
|
486
|
+
};
|
|
487
|
+
const key = this.encodeKey();
|
|
488
|
+
const result = await this.cellCollection.updateIfVersion(key, current.version, newState, expectedEpoch);
|
|
489
|
+
if (!result.updated) {
|
|
490
|
+
throw new Error(`Cell ${this.id} conflict during lease acquisition`);
|
|
491
|
+
}
|
|
492
|
+
this.emitChange({
|
|
493
|
+
cellId: this.id,
|
|
494
|
+
version: newState.version,
|
|
495
|
+
value: current.value,
|
|
496
|
+
timestamp: newState.updatedAt,
|
|
497
|
+
epoch: newState.authority.epoch,
|
|
498
|
+
});
|
|
499
|
+
return this.snapshotFromState(newState);
|
|
500
|
+
});
|
|
501
|
+
}
|
|
502
|
+
async renewLease(options) {
|
|
503
|
+
return this.acquireLock(async () => {
|
|
504
|
+
// Replicas are read-only
|
|
505
|
+
if (this.isReplica) {
|
|
506
|
+
throw new Error(`REPLICA_READ_ONLY: Cell ${this.id} is a read-only replica`);
|
|
507
|
+
}
|
|
508
|
+
const current = await this.getUnlocked();
|
|
509
|
+
if (!current) {
|
|
510
|
+
throw new Error(`Cell ${this.id} does not exist`);
|
|
511
|
+
}
|
|
512
|
+
if (!current.lease) {
|
|
513
|
+
throw new Error(`Cell ${this.id} has no active lease`);
|
|
514
|
+
}
|
|
515
|
+
// Verify lease ID matches
|
|
516
|
+
if (current.lease.leaseId !== options.leaseId) {
|
|
517
|
+
throw new Error(`LEASE_CONFLICT: lease ID mismatch`);
|
|
518
|
+
}
|
|
519
|
+
// Verify lease has not expired
|
|
520
|
+
if (current.lease.expiresAt <= Date.now()) {
|
|
521
|
+
throw new Error(`LEASE_EXPIRED: lease has expired`);
|
|
522
|
+
}
|
|
523
|
+
// Verify epoch has not changed (lease must be within same authority epoch)
|
|
524
|
+
if (current.lease.epoch !== current.authority.epoch) {
|
|
525
|
+
throw new Error(`AUTHORITY_CONFLICT: authority epoch changed since lease acquisition`);
|
|
526
|
+
}
|
|
527
|
+
const expectedEpoch = current.authority.epoch;
|
|
528
|
+
// Extend lease expiration
|
|
529
|
+
const renewedLease = {
|
|
530
|
+
holder: current.lease.holder,
|
|
531
|
+
leaseId: current.lease.leaseId,
|
|
532
|
+
epoch: current.lease.epoch,
|
|
533
|
+
expiresAt: Date.now() + options.durationMs,
|
|
534
|
+
};
|
|
535
|
+
// Lease renewal increments version
|
|
536
|
+
const newState = {
|
|
537
|
+
id: this.id,
|
|
538
|
+
version: current.version + 1,
|
|
539
|
+
value: current.value,
|
|
540
|
+
updatedAt: new Date().toISOString(),
|
|
541
|
+
authority: current.authority,
|
|
542
|
+
placement: current.placement,
|
|
543
|
+
lease: renewedLease,
|
|
544
|
+
};
|
|
545
|
+
const key = this.encodeKey();
|
|
546
|
+
const result = await this.cellCollection.updateIfVersion(key, current.version, newState, expectedEpoch);
|
|
547
|
+
if (!result.updated) {
|
|
548
|
+
throw new Error(`Cell ${this.id} conflict during lease renewal`);
|
|
549
|
+
}
|
|
550
|
+
this.emitChange({
|
|
551
|
+
cellId: this.id,
|
|
552
|
+
version: newState.version,
|
|
553
|
+
value: current.value,
|
|
554
|
+
timestamp: newState.updatedAt,
|
|
555
|
+
epoch: newState.authority.epoch,
|
|
556
|
+
});
|
|
557
|
+
return this.snapshotFromState(newState);
|
|
558
|
+
});
|
|
559
|
+
}
|
|
560
|
+
async releaseLease(options) {
|
|
561
|
+
return this.acquireLock(async () => {
|
|
562
|
+
// Replicas are read-only
|
|
563
|
+
if (this.isReplica) {
|
|
564
|
+
throw new Error(`REPLICA_READ_ONLY: Cell ${this.id} is a read-only replica`);
|
|
565
|
+
}
|
|
566
|
+
const current = await this.getUnlocked();
|
|
567
|
+
if (!current) {
|
|
568
|
+
throw new Error(`Cell ${this.id} does not exist`);
|
|
569
|
+
}
|
|
570
|
+
if (!current.lease) {
|
|
571
|
+
throw new Error(`Cell ${this.id} has no active lease`);
|
|
572
|
+
}
|
|
573
|
+
// Verify lease ID matches
|
|
574
|
+
if (current.lease.leaseId !== options.leaseId) {
|
|
575
|
+
throw new Error(`LEASE_CONFLICT: lease ID mismatch`);
|
|
576
|
+
}
|
|
577
|
+
const expectedEpoch = current.authority.epoch;
|
|
578
|
+
// Clear the lease
|
|
579
|
+
const newState = {
|
|
580
|
+
id: this.id,
|
|
581
|
+
version: current.version + 1,
|
|
582
|
+
value: current.value,
|
|
583
|
+
updatedAt: new Date().toISOString(),
|
|
584
|
+
authority: current.authority,
|
|
585
|
+
placement: current.placement,
|
|
586
|
+
lease: null,
|
|
587
|
+
};
|
|
588
|
+
const key = this.encodeKey();
|
|
589
|
+
const result = await this.cellCollection.updateIfVersion(key, current.version, newState, expectedEpoch);
|
|
590
|
+
if (!result.updated) {
|
|
591
|
+
throw new Error(`Cell ${this.id} conflict during lease release`);
|
|
592
|
+
}
|
|
593
|
+
this.emitChange({
|
|
594
|
+
cellId: this.id,
|
|
595
|
+
version: newState.version,
|
|
596
|
+
value: current.value,
|
|
597
|
+
timestamp: newState.updatedAt,
|
|
598
|
+
epoch: newState.authority.epoch,
|
|
599
|
+
});
|
|
600
|
+
return this.snapshotFromState(newState);
|
|
601
|
+
});
|
|
602
|
+
}
|
|
603
|
+
async promote(options) {
|
|
604
|
+
return this.acquireLock(async () => {
|
|
605
|
+
const current = await this.getUnlocked();
|
|
606
|
+
if (!current) {
|
|
607
|
+
throw new Error(`Cell ${this.id} does not exist`);
|
|
608
|
+
}
|
|
609
|
+
return this.promoteSnapshot(current, options);
|
|
610
|
+
});
|
|
611
|
+
}
|
|
612
|
+
async promoteSnapshot(current, options) {
|
|
613
|
+
// Check retirement
|
|
614
|
+
if (current.authority.lifecycle === 'retired') {
|
|
615
|
+
throw new Error(`Cell ${this.id} is retired`);
|
|
616
|
+
}
|
|
617
|
+
// Verify expected version
|
|
618
|
+
if (current.version !== options.expectedVersion) {
|
|
619
|
+
throw new Error(`PROMOTION_CONFLICT: expected version ${options.expectedVersion}, got ${current.version}`);
|
|
620
|
+
}
|
|
621
|
+
// Verify expected epoch
|
|
622
|
+
if (current.authority.epoch !== options.expectedEpoch) {
|
|
623
|
+
throw new Error(`PROMOTION_CONFLICT: expected epoch ${options.expectedEpoch}, got ${current.authority.epoch}`);
|
|
624
|
+
}
|
|
625
|
+
// Promotion atomically transitions:
|
|
626
|
+
// - owner (A → B)
|
|
627
|
+
// - epoch (7 → 8)
|
|
628
|
+
// - placement (if provided)
|
|
629
|
+
// - clears lease (old epoch invalid)
|
|
630
|
+
// - advances version because promotion is a durable concurrency mutation
|
|
631
|
+
// - keeps lifecycle the same
|
|
632
|
+
const newAuthority = {
|
|
633
|
+
owner: options.newOwner,
|
|
634
|
+
epoch: current.authority.epoch + 1,
|
|
635
|
+
lifecycle: current.authority.lifecycle,
|
|
636
|
+
};
|
|
637
|
+
const newPlacement = {
|
|
638
|
+
nodeId: options.newNodeId !== undefined ? options.newNodeId : current.placement.nodeId,
|
|
639
|
+
};
|
|
640
|
+
const newState = {
|
|
641
|
+
id: this.id,
|
|
642
|
+
version: current.version + 1,
|
|
643
|
+
value: current.value,
|
|
644
|
+
updatedAt: new Date().toISOString(),
|
|
645
|
+
authority: newAuthority,
|
|
646
|
+
placement: newPlacement,
|
|
647
|
+
lease: null,
|
|
648
|
+
};
|
|
649
|
+
const key = this.encodeKey();
|
|
650
|
+
// Promotion consumes an active lease as part of the same CAS that changes
|
|
651
|
+
// authority and clears it. An expired lease is not an active predicate.
|
|
652
|
+
const expectedLeaseId = current.lease && current.lease.expiresAt > Date.now()
|
|
653
|
+
? current.lease.leaseId
|
|
654
|
+
: undefined;
|
|
655
|
+
const result = await this.cellCollection.updateIfVersion(key, current.version, newState, current.authority.epoch, expectedLeaseId, true, current);
|
|
656
|
+
if (!result.updated) {
|
|
657
|
+
if (result.conflictCode === 'REPLICATION_CONFLICT') {
|
|
658
|
+
throw new Error(`REPLICATION_CONFLICT: divergent replica cannot be promoted`);
|
|
659
|
+
}
|
|
660
|
+
throw new Error(`PROMOTION_CONFLICT: cell state changed during promotion`);
|
|
661
|
+
}
|
|
662
|
+
// Emit change event for promotion
|
|
663
|
+
this.emitChange({
|
|
664
|
+
cellId: this.id,
|
|
665
|
+
version: newState.version,
|
|
666
|
+
value: current.value,
|
|
667
|
+
timestamp: newState.updatedAt,
|
|
668
|
+
epoch: newState.authority.epoch,
|
|
669
|
+
});
|
|
670
|
+
return this.snapshotFromState(newState);
|
|
671
|
+
}
|
|
672
|
+
async observeHealth(observation) {
|
|
673
|
+
// Observation does NOT use cell locking; it's a separate, concurrent observation stream
|
|
674
|
+
// Multiple observers can race; newer observations must win via CAS
|
|
675
|
+
const key = `observation_${this.id}_${observation.nodeId}`;
|
|
676
|
+
// Get current observation if one exists
|
|
677
|
+
const current = await this.failureObservationCollection.get(key);
|
|
678
|
+
// If current observation is newer, don't overwrite
|
|
679
|
+
if (current && current.observedAt > observation.observedAt) {
|
|
680
|
+
return; // Silently reject stale observation
|
|
681
|
+
}
|
|
682
|
+
// Use CAS to ensure two concurrent observers don't silently overwrite each other
|
|
683
|
+
if (current) {
|
|
684
|
+
const currentVersion = current.__version ?? 1;
|
|
685
|
+
const result = await this.failureObservationCollection.updateIfVersion(key, currentVersion, observation, 0);
|
|
686
|
+
if (!result.updated) {
|
|
687
|
+
// Another observer won the race; that's fine, failure detection is best-effort
|
|
688
|
+
return;
|
|
689
|
+
}
|
|
690
|
+
}
|
|
691
|
+
else {
|
|
692
|
+
// First observation for this node
|
|
693
|
+
await this.failureObservationCollection.insert(observation, key);
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
async evaluateFailure(options) {
|
|
697
|
+
// Evaluation is deterministic and read-only: no cell modifications whatsoever
|
|
698
|
+
const key = `observation_${this.id}_${options.nodeId}`;
|
|
699
|
+
const observation = await this.failureObservationCollection.get(key);
|
|
700
|
+
if (!observation) {
|
|
701
|
+
// No observation recorded; assume healthy
|
|
702
|
+
return {
|
|
703
|
+
status: 'healthy',
|
|
704
|
+
authorityEpoch: options.nodeId === (await this.getUnlocked())?.authority.owner ?
|
|
705
|
+
(await this.getUnlocked())?.authority.epoch ?? 0 : 0,
|
|
706
|
+
nodeId: options.nodeId,
|
|
707
|
+
};
|
|
708
|
+
}
|
|
709
|
+
// Check if observation is for a stale epoch
|
|
710
|
+
const current = await this.getUnlocked();
|
|
711
|
+
if (current && observation.authorityEpoch < current.authority.epoch) {
|
|
712
|
+
// Observation is from an old epoch; ignore it
|
|
713
|
+
return {
|
|
714
|
+
status: 'healthy',
|
|
715
|
+
authorityEpoch: current.authority.epoch,
|
|
716
|
+
nodeId: options.nodeId,
|
|
717
|
+
};
|
|
718
|
+
}
|
|
719
|
+
// Determine age of observation
|
|
720
|
+
const ageMs = options.now - observation.lastSeenAt;
|
|
721
|
+
// Deterministic classification
|
|
722
|
+
let status;
|
|
723
|
+
if (ageMs < options.suspectAfterMs) {
|
|
724
|
+
status = 'healthy';
|
|
725
|
+
}
|
|
726
|
+
else if (ageMs < options.failedAfterMs) {
|
|
727
|
+
status = 'suspect';
|
|
728
|
+
}
|
|
729
|
+
else {
|
|
730
|
+
status = 'failed';
|
|
731
|
+
}
|
|
732
|
+
return {
|
|
733
|
+
status,
|
|
734
|
+
authorityEpoch: observation.authorityEpoch,
|
|
735
|
+
nodeId: options.nodeId,
|
|
736
|
+
ageMs,
|
|
737
|
+
};
|
|
738
|
+
}
|
|
739
|
+
async attemptAutomaticFailover(options) {
|
|
740
|
+
// Automatic failover: bridge from failure detection to controlled promotion
|
|
741
|
+
// Critical invariant: failure detection itself never grants authority.
|
|
742
|
+
// Only the durable backend CAS (in promote) is the final authority.
|
|
743
|
+
// Step 1: Check if this is a replica
|
|
744
|
+
if (!this.isReplica) {
|
|
745
|
+
return {
|
|
746
|
+
attempted: false,
|
|
747
|
+
promoted: false,
|
|
748
|
+
reason: 'Not a replica',
|
|
749
|
+
diagnostics: {
|
|
750
|
+
isReplica: false,
|
|
751
|
+
eligibilityChecksPassed: false,
|
|
752
|
+
promotionAttempted: false,
|
|
753
|
+
promotionSucceeded: false,
|
|
754
|
+
},
|
|
755
|
+
};
|
|
756
|
+
}
|
|
757
|
+
// One authoritative Cell read is shared by eligibility and promotion. The
|
|
758
|
+
// caller may pass the observation it just recorded, avoiding another HTTP
|
|
759
|
+
// round trip; legacy callers still load it durably here.
|
|
760
|
+
const current = options.replicaSnapshot
|
|
761
|
+
?? await this.getUnlocked();
|
|
762
|
+
if (!current) {
|
|
763
|
+
return {
|
|
764
|
+
attempted: false,
|
|
765
|
+
promoted: false,
|
|
766
|
+
reason: 'Replica state not found',
|
|
767
|
+
diagnostics: {
|
|
768
|
+
isReplica: true,
|
|
769
|
+
eligibilityChecksPassed: false,
|
|
770
|
+
promotionAttempted: false,
|
|
771
|
+
promotionSucceeded: false,
|
|
772
|
+
},
|
|
773
|
+
};
|
|
774
|
+
}
|
|
775
|
+
const observationKey = `observation_${this.id}_${options.failureEvaluationOptions.nodeId}`;
|
|
776
|
+
const observation = options.failureObservation
|
|
777
|
+
?? await this.failureObservationCollection.get(observationKey);
|
|
778
|
+
const failureEval = this.evaluateFailureSnapshot(options.failureEvaluationOptions, observation, current);
|
|
779
|
+
if (failureEval.status !== 'failed') {
|
|
780
|
+
return {
|
|
781
|
+
attempted: false,
|
|
782
|
+
promoted: false,
|
|
783
|
+
reason: `Current authority not failed (status: ${failureEval.status})`,
|
|
784
|
+
diagnostics: {
|
|
785
|
+
isReplica: true,
|
|
786
|
+
failureStatus: failureEval.status,
|
|
787
|
+
eligibilityChecksPassed: false,
|
|
788
|
+
promotionAttempted: false,
|
|
789
|
+
promotionSucceeded: false,
|
|
790
|
+
},
|
|
791
|
+
};
|
|
792
|
+
}
|
|
793
|
+
// Step 4: Check lifecycle (retired cells cannot be promoted)
|
|
794
|
+
if (current.authority.lifecycle === 'retired') {
|
|
795
|
+
return {
|
|
796
|
+
attempted: false,
|
|
797
|
+
promoted: false,
|
|
798
|
+
reason: 'Cell is retired',
|
|
799
|
+
diagnostics: {
|
|
800
|
+
isReplica: true,
|
|
801
|
+
failureStatus: failureEval.status,
|
|
802
|
+
eligibilityChecksPassed: false,
|
|
803
|
+
promotionAttempted: false,
|
|
804
|
+
promotionSucceeded: false,
|
|
805
|
+
},
|
|
806
|
+
};
|
|
807
|
+
}
|
|
808
|
+
// Step 5: Verify replica authority epoch matches observed authority epoch
|
|
809
|
+
// This ensures the replica has the same authority state as what was observed as failed
|
|
810
|
+
if (current.authority.epoch !== failureEval.authorityEpoch) {
|
|
811
|
+
// Replica is from a different epoch (stale or ahead)
|
|
812
|
+
return {
|
|
813
|
+
attempted: false,
|
|
814
|
+
promoted: false,
|
|
815
|
+
reason: `Replica authority epoch mismatch: replica=${current.authority.epoch}, observed=${failureEval.authorityEpoch}`,
|
|
816
|
+
diagnostics: {
|
|
817
|
+
isReplica: true,
|
|
818
|
+
failureStatus: failureEval.status,
|
|
819
|
+
eligibilityChecksPassed: false,
|
|
820
|
+
promotionAttempted: false,
|
|
821
|
+
promotionSucceeded: false,
|
|
822
|
+
},
|
|
823
|
+
};
|
|
824
|
+
}
|
|
825
|
+
// Step 6: All eligibility checks passed; attempt atomic promotion
|
|
826
|
+
try {
|
|
827
|
+
await this.promoteSnapshot(current, {
|
|
828
|
+
expectedVersion: current.version,
|
|
829
|
+
expectedEpoch: current.authority.epoch,
|
|
830
|
+
newOwner: options.newOwner,
|
|
831
|
+
newNodeId: current.placement.nodeId,
|
|
832
|
+
});
|
|
833
|
+
return {
|
|
834
|
+
attempted: true,
|
|
835
|
+
promoted: true,
|
|
836
|
+
diagnostics: {
|
|
837
|
+
isReplica: true,
|
|
838
|
+
failureStatus: failureEval.status,
|
|
839
|
+
eligibilityChecksPassed: true,
|
|
840
|
+
promotionAttempted: true,
|
|
841
|
+
promotionSucceeded: true,
|
|
842
|
+
},
|
|
843
|
+
};
|
|
844
|
+
}
|
|
845
|
+
catch (err) {
|
|
846
|
+
// Promotion failed (likely CAS conflict due to concurrent writer)
|
|
847
|
+
return {
|
|
848
|
+
attempted: true,
|
|
849
|
+
promoted: false,
|
|
850
|
+
reason: `Promotion failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
851
|
+
diagnostics: {
|
|
852
|
+
isReplica: true,
|
|
853
|
+
failureStatus: failureEval.status,
|
|
854
|
+
eligibilityChecksPassed: true,
|
|
855
|
+
promotionAttempted: true,
|
|
856
|
+
promotionSucceeded: false,
|
|
857
|
+
},
|
|
858
|
+
};
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
evaluateFailureSnapshot(options, observation, current) {
|
|
862
|
+
if (!observation) {
|
|
863
|
+
return {
|
|
864
|
+
status: 'healthy',
|
|
865
|
+
authorityEpoch: options.nodeId === current.authority.owner ? current.authority.epoch : 0,
|
|
866
|
+
nodeId: options.nodeId,
|
|
867
|
+
};
|
|
868
|
+
}
|
|
869
|
+
if (observation.authorityEpoch < current.authority.epoch) {
|
|
870
|
+
return { status: 'healthy', authorityEpoch: current.authority.epoch, nodeId: options.nodeId };
|
|
871
|
+
}
|
|
872
|
+
const ageMs = options.now - observation.lastSeenAt;
|
|
873
|
+
const status = ageMs < options.suspectAfterMs
|
|
874
|
+
? 'healthy'
|
|
875
|
+
: ageMs < options.failedAfterMs ? 'suspect' : 'failed';
|
|
876
|
+
return { status, authorityEpoch: observation.authorityEpoch, nodeId: options.nodeId, ageMs };
|
|
877
|
+
}
|
|
878
|
+
validateLifecycleTransition(current, next) {
|
|
879
|
+
const validTransitions = {
|
|
880
|
+
active: ['suspended', 'retired'],
|
|
881
|
+
suspended: ['active', 'retired'],
|
|
882
|
+
retired: [],
|
|
883
|
+
};
|
|
884
|
+
if (!validTransitions[current].includes(next)) {
|
|
885
|
+
throw new Error(`Invalid lifecycle transition: ${current} → ${next}`);
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
async getUnlocked() {
|
|
889
|
+
const key = this.encodeKey();
|
|
890
|
+
const state = await this.cellCollection.get(key);
|
|
891
|
+
if (!state)
|
|
892
|
+
return null;
|
|
893
|
+
return this.snapshotFromState(state);
|
|
894
|
+
}
|
|
895
|
+
async setUnlocked(value, options) {
|
|
896
|
+
const current = await this.getUnlocked();
|
|
897
|
+
if (current?.authority.lifecycle === 'retired') {
|
|
898
|
+
throw new Error(`Cell ${this.id} is retired`);
|
|
899
|
+
}
|
|
900
|
+
const expectedEpoch = current?.authority.epoch ?? 0;
|
|
901
|
+
if (options?.authorityEpoch !== undefined && current) {
|
|
902
|
+
if (options.authorityEpoch !== expectedEpoch) {
|
|
903
|
+
throw new Error(`AUTHORITY_CONFLICT: expected epoch ${expectedEpoch}, got ${options.authorityEpoch}`);
|
|
904
|
+
}
|
|
905
|
+
}
|
|
906
|
+
// Validate lease if specified
|
|
907
|
+
if (options?.leaseId !== undefined) {
|
|
908
|
+
if (!current?.lease) {
|
|
909
|
+
throw new Error(`LEASE_INVALID: no active lease`);
|
|
910
|
+
}
|
|
911
|
+
if (current.lease.leaseId !== options.leaseId) {
|
|
912
|
+
throw new Error(`LEASE_INVALID: lease ID mismatch`);
|
|
913
|
+
}
|
|
914
|
+
if (current.lease.expiresAt <= Date.now()) {
|
|
915
|
+
throw new Error(`LEASE_EXPIRED: lease has expired`);
|
|
916
|
+
}
|
|
917
|
+
if (current.lease.epoch !== expectedEpoch) {
|
|
918
|
+
throw new Error(`LEASE_INVALID: lease epoch does not match authority epoch`);
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
const expectedVersion = current?.version ?? 0;
|
|
922
|
+
const authority = current?.authority ?? { owner: null, epoch: 0, lifecycle: 'active' };
|
|
923
|
+
const placement = current?.placement ?? { nodeId: null };
|
|
924
|
+
const lease = current?.lease ?? null;
|
|
925
|
+
const newState = {
|
|
926
|
+
id: this.id,
|
|
927
|
+
version: expectedVersion + 1,
|
|
928
|
+
value,
|
|
929
|
+
updatedAt: new Date().toISOString(),
|
|
930
|
+
authority,
|
|
931
|
+
placement,
|
|
932
|
+
lease,
|
|
933
|
+
};
|
|
934
|
+
const key = this.encodeKey();
|
|
935
|
+
if (expectedVersion === 0) {
|
|
936
|
+
await this.cellCollection.insert(newState, key);
|
|
937
|
+
}
|
|
938
|
+
else {
|
|
939
|
+
const result = await this.cellCollection.updateIfVersion(key, expectedVersion, newState, expectedEpoch);
|
|
940
|
+
if (!result.updated) {
|
|
941
|
+
throw new Error(`Cell ${this.id} version conflict`);
|
|
942
|
+
}
|
|
943
|
+
}
|
|
944
|
+
this.emitChange({
|
|
945
|
+
cellId: this.id,
|
|
946
|
+
version: newState.version,
|
|
947
|
+
value,
|
|
948
|
+
timestamp: newState.updatedAt,
|
|
949
|
+
epoch: newState.authority.epoch,
|
|
950
|
+
});
|
|
951
|
+
return this.snapshotFromState(newState);
|
|
952
|
+
}
|
|
953
|
+
async updateUnlocked(updater, options) {
|
|
954
|
+
const current = await this.getUnlocked();
|
|
955
|
+
if (current?.authority.lifecycle === 'retired') {
|
|
956
|
+
throw new Error(`Cell ${this.id} is retired`);
|
|
957
|
+
}
|
|
958
|
+
const expectedEpoch = current?.authority.epoch ?? 0;
|
|
959
|
+
if (options?.authorityEpoch !== undefined && current) {
|
|
960
|
+
if (options.authorityEpoch !== expectedEpoch) {
|
|
961
|
+
throw new Error(`AUTHORITY_CONFLICT: expected epoch ${expectedEpoch}, got ${options.authorityEpoch}`);
|
|
962
|
+
}
|
|
963
|
+
}
|
|
964
|
+
// Validate lease if specified
|
|
965
|
+
if (options?.leaseId !== undefined) {
|
|
966
|
+
if (!current?.lease) {
|
|
967
|
+
throw new Error(`LEASE_INVALID: no active lease`);
|
|
968
|
+
}
|
|
969
|
+
if (current.lease.leaseId !== options.leaseId) {
|
|
970
|
+
throw new Error(`LEASE_INVALID: lease ID mismatch`);
|
|
971
|
+
}
|
|
972
|
+
if (current.lease.expiresAt <= Date.now()) {
|
|
973
|
+
throw new Error(`LEASE_EXPIRED: lease has expired`);
|
|
974
|
+
}
|
|
975
|
+
if (current.lease.epoch !== expectedEpoch) {
|
|
976
|
+
throw new Error(`LEASE_INVALID: lease epoch does not match authority epoch`);
|
|
977
|
+
}
|
|
978
|
+
}
|
|
979
|
+
const currentValue = current?.value ?? null;
|
|
980
|
+
const currentVersion = current?.version ?? 0;
|
|
981
|
+
const nextValue = updater(currentValue);
|
|
982
|
+
if (nextValue === NO_CHANGE) {
|
|
983
|
+
if (current)
|
|
984
|
+
return this.snapshotFromState(current);
|
|
985
|
+
throw new Error(`Cell ${this.id} update returned NO_CHANGE but cell does not exist`);
|
|
986
|
+
}
|
|
987
|
+
const authority = current?.authority ?? { owner: null, epoch: 0, lifecycle: 'active' };
|
|
988
|
+
const placement = current?.placement ?? { nodeId: null };
|
|
989
|
+
const lease = current?.lease ?? null;
|
|
990
|
+
const newState = {
|
|
991
|
+
id: this.id,
|
|
992
|
+
version: currentVersion + 1,
|
|
993
|
+
value: nextValue,
|
|
994
|
+
updatedAt: new Date().toISOString(),
|
|
995
|
+
authority,
|
|
996
|
+
placement,
|
|
997
|
+
lease,
|
|
998
|
+
};
|
|
999
|
+
const key = this.encodeKey();
|
|
1000
|
+
if (currentVersion === 0) {
|
|
1001
|
+
await this.cellCollection.insert(newState, key);
|
|
1002
|
+
}
|
|
1003
|
+
else {
|
|
1004
|
+
const result = await this.cellCollection.updateIfVersion(key, currentVersion, newState, expectedEpoch);
|
|
1005
|
+
if (!result.updated) {
|
|
1006
|
+
throw new Error(`Cell ${this.id} conflict during update`);
|
|
1007
|
+
}
|
|
1008
|
+
}
|
|
1009
|
+
this.emitChange({
|
|
1010
|
+
cellId: this.id,
|
|
1011
|
+
version: newState.version,
|
|
1012
|
+
value: nextValue,
|
|
1013
|
+
timestamp: newState.updatedAt,
|
|
1014
|
+
epoch: newState.authority.epoch,
|
|
1015
|
+
});
|
|
1016
|
+
return this.snapshotFromState(newState);
|
|
1017
|
+
}
|
|
1018
|
+
snapshotFromState(state) {
|
|
1019
|
+
return {
|
|
1020
|
+
id: this.id,
|
|
1021
|
+
version: state.version,
|
|
1022
|
+
value: state.value,
|
|
1023
|
+
updatedAt: state.updatedAt,
|
|
1024
|
+
authority: state.authority,
|
|
1025
|
+
placement: state.placement,
|
|
1026
|
+
lease: state.lease,
|
|
1027
|
+
};
|
|
1028
|
+
}
|
|
1029
|
+
on(event, handler) {
|
|
1030
|
+
if (event === 'change') {
|
|
1031
|
+
this.changeHandlers.push(handler);
|
|
1032
|
+
}
|
|
1033
|
+
}
|
|
1034
|
+
off(event, handler) {
|
|
1035
|
+
if (event === 'change') {
|
|
1036
|
+
const idx = this.changeHandlers.indexOf(handler);
|
|
1037
|
+
if (idx >= 0)
|
|
1038
|
+
this.changeHandlers.splice(idx, 1);
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
emitChange(event) {
|
|
1042
|
+
for (const handler of this.changeHandlers) {
|
|
1043
|
+
try {
|
|
1044
|
+
handler(event);
|
|
1045
|
+
}
|
|
1046
|
+
catch (err) {
|
|
1047
|
+
console.error(`Error in cell change handler: ${err}`);
|
|
1048
|
+
}
|
|
1049
|
+
}
|
|
1050
|
+
}
|
|
1051
|
+
acquireLock(fn) {
|
|
1052
|
+
return new Promise((resolve, reject) => {
|
|
1053
|
+
const task = async () => {
|
|
1054
|
+
try {
|
|
1055
|
+
const result = await fn();
|
|
1056
|
+
resolve(result);
|
|
1057
|
+
}
|
|
1058
|
+
catch (err) {
|
|
1059
|
+
reject(err);
|
|
1060
|
+
}
|
|
1061
|
+
};
|
|
1062
|
+
this.mutationQueue.push(task);
|
|
1063
|
+
if (!this.isLocked) {
|
|
1064
|
+
this.processQueue();
|
|
1065
|
+
}
|
|
1066
|
+
});
|
|
1067
|
+
}
|
|
1068
|
+
async processQueue() {
|
|
1069
|
+
if (this.isLocked) {
|
|
1070
|
+
return;
|
|
1071
|
+
}
|
|
1072
|
+
if (this.mutationQueue.length === 0) {
|
|
1073
|
+
return;
|
|
1074
|
+
}
|
|
1075
|
+
this.isLocked = true;
|
|
1076
|
+
const task = this.mutationQueue.shift();
|
|
1077
|
+
if (task) {
|
|
1078
|
+
try {
|
|
1079
|
+
await task();
|
|
1080
|
+
}
|
|
1081
|
+
catch (err) {
|
|
1082
|
+
// Task already rejected its promise
|
|
1083
|
+
}
|
|
1084
|
+
finally {
|
|
1085
|
+
this.isLocked = false;
|
|
1086
|
+
this.processQueue();
|
|
1087
|
+
}
|
|
1088
|
+
}
|
|
1089
|
+
}
|
|
1090
|
+
}
|