castle-web-cli 0.4.246 → 0.4.247

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,7 +9,8 @@ function setup(t) {
9
9
  function instance(id) {
10
10
  const root = fixture.root(id), lease = create({ storage: root.storage }, true);
11
11
  t.after(() => lease.shutdown());
12
- return { ...root, lease, api: lease.api };
12
+ // The root itself, so a test can set its fault hooks (`checks`, `inTransit`).
13
+ return Object.assign(root, { lease, api: lease.api });
13
14
  }
14
15
  return { ...fixture, instance };
15
16
  }
@@ -84,14 +85,146 @@ test('concurrent fences share one check; transient failure is retryable by the c
84
85
  await a.api.fence('101');
85
86
  });
86
87
 
87
- test('stale access and dev writer-loss suffixes fail closed', async t => {
88
- for (const code of ['CAULDRON_STORAGE_STALE_ACCESS', 'STALE_ACCESS', 'CAULDRON_STORAGE_STALE_RESET_GENERATION', 'WRITER_LOST']) {
89
- const a = setup(t).instance(code); a.lease.join('p1', '101'); await a.api.ready('101');
90
- a.storage.transaction = async () => { throw Object.assign(Error(code), { code }); };
91
- await assert.rejects(a.api.deck('101').increment('coins'), lost);
88
+
89
+ test('writes carry the claim\'s writer check; a refusal loses the claim and is never resent', async t => {
90
+ for (const code of ['CAULDRON_STORAGE_WRITER_LOST', 'WRITER_LOST', 'CAULDRON_STORAGE_STALE_ACCESS', 'CAULDRON_STORAGE_STALE_RESET_GENERATION']) {
91
+ const world = setup(t), a = world.instance(code); a.lease.join('p1', '101'); await a.api.ready('101');
92
+ const deck = a.api.deck('101'), sent = [];
93
+ const set = a.storage.deck.set;
94
+ a.storage.deck.set = async (values, options) => {
95
+ sent.push(options);
96
+ if (values.refuse) throw Object.assign(Error(code), { code, committed: false, retryable: false });
97
+ return set(values, options);
98
+ };
99
+ await deck.set({ ok: 1 });
100
+ assert.deepEqual(sent[0].check, { target: { scope: 'user', subject: '101', prefix: 'hub:writer:' }, writer: world.writerOf('101') });
101
+ assert.equal(a.calls.transaction.length, 0, 'a checked write needs no separate fence');
102
+ await assert.rejects(deck.set({ refuse: 1 }), lost);
92
103
  assert.equal(a.api.lost('101'), true);
93
- assert.equal(a.calls.increment.length, 0);
104
+ await assert.rejects(deck.increment('coins'), lost);
105
+ await assert.rejects(deck.set({ refuse: 1 }), lost);
106
+ assert.equal(sent.length, 2, 'nothing is sent once the claim is lost');
94
107
  }
108
+ // remove and increment carry it too, after their own arguments.
109
+ const { instance } = setup(t), a = instance('a'), b = instance('b');
110
+ a.lease.join('p1', '101');
111
+ assert.equal(await a.api.deck('101').increment('n'), 1);
112
+ assert.equal(await a.api.user('101').increment('n', 5), 5);
113
+ await a.api.deck('101').remove(['n']);
114
+ assert.deepEqual(a.calls.checked.map(c => c.verb), ['increment', 'increment', 'remove']);
115
+ b.lease.join('p2', '101'); await b.api.ready('101');
116
+ await assert.rejects(a.api.deck('101').increment('n'), lost);
117
+ assert.equal(a.calls.increment.length, 2);
118
+ });
119
+
120
+ test('bucket transactions, which cannot name the writer target, are fenced first', async t => {
121
+ const { instance } = setup(t), a = instance('a'), b = instance('b');
122
+ const sent = [];
123
+ a.storage.deck.transaction = async input => { sent.push(input); return { operationId: input.operationId, access: 'y', revisions: [] }; };
124
+ a.lease.join('p1', '101');
125
+ await a.api.deck('101').transaction({ operationId: 'op1', checks: [{ prefix: 'x:', expectedRevision: 'r' }], changes: [] });
126
+ assert.deepEqual(sent[0].checks, [{ prefix: 'x:', expectedRevision: 'r' }], 'the bucket\'s own checks go unchanged');
127
+ assert.equal(a.calls.transaction.length, 1, 'one fence');
128
+ b.lease.join('p2', '101'); await b.api.ready('101');
129
+ await assert.rejects(a.api.deck('101').transaction({ operationId: 'op2', changes: [] }), lost);
130
+ assert.equal(sent.length, 1);
131
+ });
132
+
133
+ test('review round 2 orders: a late fence, a chained drain, an unleased queue, a moving claim', async t => {
134
+ // 1. A fence that answers after this instance has claimed anew sends nothing.
135
+ {
136
+ const { instance } = setup(t), a = instance('a');
137
+ const sent = [];
138
+ a.storage.deck.transaction = async input => { sent.push(input); return { operationId: input.operationId, access: 'y', revisions: [] }; };
139
+ a.lease.join('p1', '101'); await a.api.ready('101');
140
+ const fence = a.storage.transaction, reply = deferred();
141
+ let held = true;
142
+ a.storage.transaction = async input => { const r = await fence(input); if (held) { held = false; await reply.promise; } return r; };
143
+ const old = a.api.deck('101').transaction({ operationId: 'op1', changes: [] });
144
+ await flushMicrotasks();
145
+ a.lease.leave('p1'); a.lease.join('p2', '101');
146
+ await new Promise(r => setTimeout(r, 5100));
147
+ await a.api.ready('101');
148
+ assert.equal(a.calls.claim.length, 2, 'the return claimed after the drain cap');
149
+ reply.resolve();
150
+ await assert.rejects(old, lost);
151
+ assert.equal(sent.length, 0, 'the old transaction is never sent');
152
+ }
153
+ // 2. A return that leaves before claiming hands its drain to the next one.
154
+ {
155
+ const { instance, rows } = setup(t), a = instance('a');
156
+ a.lease.join('a1', '101'); await a.api.ready('101');
157
+ const gate = deferred();
158
+ a.inTransit = async (verb, values) => { if (values.end) await gate.promise; };
159
+ const end = a.api.deck('101').set({ end: 1 });
160
+ a.lease.leave('a1'); a.lease.join('a2', '101');
161
+ await flushMicrotasks();
162
+ a.lease.leave('a2'); a.lease.join('a3', '101');
163
+ await flushMicrotasks();
164
+ assert.equal(a.calls.claim.length, 1, 'a3 waits for a1\'s queue too');
165
+ gate.resolve();
166
+ await end;
167
+ await a.api.ready('101');
168
+ assert.equal(rows.get('deck:end'), 1);
169
+ assert.equal(a.calls.claim.length, 2);
170
+ }
171
+ // 3. An unleased account keeps one write queue across a return.
172
+ {
173
+ const { instance, rows } = setup(t), a = instance('a');
174
+ a.storage.claim = async () => { throw Object.assign(Error('no'), { code: 'CAULDRON_STORAGE_UNAVAILABLE', committed: false, retryable: false }); };
175
+ a.lease.join('a1', '101'); await a.api.ready('101');
176
+ const gate = deferred();
177
+ a.inTransit = async (verb, values) => { if (values.p === 1) await gate.promise; };
178
+ const first = a.api.deck('101').set({ p: 1 });
179
+ a.lease.leave('a1'); a.lease.join('a2', '101');
180
+ const second = a.api.deck('101').set({ p: 2 });
181
+ // Time for the second save to overtake the first, were it on a queue of its own.
182
+ await new Promise(r => setTimeout(r, 20));
183
+ gate.resolve();
184
+ await first; await second;
185
+ assert.equal(rows.get('deck:p'), 2, 'the later visit\'s save lands last');
186
+ }
187
+ // 4. holds waits until the account's claim stops changing.
188
+ {
189
+ const { instance } = setup(t), a = instance('a');
190
+ a.lease.join('a1', '101'); await a.api.ready('101');
191
+ const old = a.api.deck('101');
192
+ const claim = a.storage.claim, replies = [deferred(), deferred()];
193
+ let n = 0;
194
+ a.storage.claim = async input => { const r = replies[n++]; await r.promise; return claim(input); };
195
+ a.lease.leave('a1'); a.lease.join('a2', '101');
196
+ let settled = false;
197
+ const held = a.api.holds(old).then(v => { settled = true; return v; });
198
+ await new Promise(r => setTimeout(r, 20));
199
+ a.lease.leave('a2'); a.lease.join('a3', '101');
200
+ replies[0].resolve();
201
+ await new Promise(r => setTimeout(r, 20));
202
+ assert.equal(settled, false, 'still waiting for a3\'s claim');
203
+ replies[1].resolve();
204
+ assert.equal(await held, false);
205
+ }
206
+ });
207
+
208
+ test('where the platform cannot check a write it is fenced, then written unchecked, as before', async t => {
209
+ const { instance, rows } = setup(t), a = instance('a'), b = instance('b');
210
+ a.checks = false;
211
+ a.lease.join('p1', '101');
212
+ await a.api.deck('101').set({ x: 1 });
213
+ await a.api.deck('101').set({ x: 2 });
214
+ assert.equal(rows.get('deck:x'), 2);
215
+ assert.equal(a.calls.set.length, 2);
216
+ assert.equal(a.calls.transaction.length, 2, 'one fence per write');
217
+ b.lease.join('p2', '101'); await b.api.ready('101');
218
+ await assert.rejects(a.api.deck('101').set({ x: 3 }), lost);
219
+ assert.equal(rows.get('deck:x'), 2);
220
+ // Storage that is unavailable altogether is not mistaken for that.
221
+ const c = setup(t).instance('c'); c.lease.join('p1', '101'); await c.api.ready('101');
222
+ const real = c.storage.deck.set, checked = [];
223
+ c.storage.deck.set = async () => { throw Object.assign(Error('down'), { code: 'CAULDRON_STORAGE_UNAVAILABLE', committed: false, retryable: false }); };
224
+ await assert.rejects(c.api.deck('101').set({ x: 1 }), { code: 'CAULDRON_STORAGE_UNAVAILABLE' });
225
+ c.storage.deck.set = async (values, options) => { checked.push(options?.check !== undefined); return real(values, options); };
226
+ await c.api.deck('101').set({ x: 1 });
227
+ assert.deepEqual(checked, [true], 'still checking');
95
228
  });
96
229
 
97
230
  test('reads wait for claim settlement and ordinary writes stay in account order across buckets', async t => {
@@ -191,17 +324,6 @@ test('reads wait for the claim; a claim still failing after CLAIM_WAIT_MS stops
191
324
  assert.equal(a.calls.set.length, 0);
192
325
  });
193
326
 
194
- test('transactions through the view are fenced too', async t => {
195
- const { instance } = setup(t), a = instance('a'), b = instance('b');
196
- let sent = 0;
197
- a.storage.deck.transaction = async () => { sent++; return { operationId: 'x', access: 'y', revisions: [] }; };
198
- a.lease.join('p1', '101');
199
- await a.api.deck('101').transaction({ operationId: 'op1', changes: [] });
200
- assert.equal(sent, 1);
201
- b.lease.join('p2', '101'); await b.api.ready('101');
202
- await assert.rejects(a.api.deck('101').transaction({ operationId: 'op2', changes: [] }), lost);
203
- assert.equal(sent, 1);
204
- });
205
327
 
206
328
  test('a visit the place still holds keeps a lost account lost across a reconnect', async t => {
207
329
  const { instance } = setup(t), a = instance('a'), b = instance('b');
@@ -283,383 +405,185 @@ test('a fresh connection beside a connected lost visit preserves the stale accou
283
405
  await b.api.fence('101');
284
406
  });
285
407
 
286
- test('a lone visit back within the keep revalidates: kept claim reused, a stolen one reclaimed silently', async t => {
287
- t.mock.timers.enable({ apis: ['setTimeout'] });
288
- const { instance, rows } = setup(t), a = instance('a'), b = instance('b');
289
- const notices = [];
290
- a.api.onLost(user => notices.push(user));
291
- // Nobody else claimed: the visit back keeps the claim and its end saves land.
292
- a.lease.join('p1', '101'); await a.api.ready('101');
293
- const first = a.api.deck('101');
294
- assert.equal(await a.api.holds(b.api.deck('101')), true, 'another lease’s views are unrecognized');
295
- a.lease.leave('p1');
296
- a.lease.join('p2', '101');
297
- assert.equal(await a.api.holds(first), true);
298
- await first.set({ progress: 1 });
299
- assert.equal(a.calls.claim.length, 1);
300
- assert.equal(a.api.deck('101'), first, 'same claim, same view');
301
- // Town B takes the account; the player closes B and comes back here inside 60 s.
302
- a.lease.leave('p2');
303
- b.lease.join('p9', '101'); await b.api.deck('101').set({ progress: 2 });
304
- b.lease.leave('p9');
305
- t.mock.timers.tick(KEEP_MS - 1);
306
- a.lease.join('p3', '101');
307
- const returned = a.api.deck('101');
308
- assert.equal(await a.api.holds(first), false);
309
- assert.equal(await a.api.holds(returned), true);
310
- const loaded = await a.api.deck('101').get(['progress']);
311
- assert.deepEqual(loaded, { progress: 2 });
312
- assert.equal(a.calls.claim.length, 2, 'reclaimed on arrival');
313
- assert.equal(a.api.lost('101'), false);
314
- // The old visit's late write cannot land under the new claim.
315
- await assert.rejects(first.set({ progress: 1 }), lost);
316
- await a.api.deck('101').set({ progress: 3 });
317
- assert.equal(rows.get('deck:progress'), 3);
318
- assert.deepEqual(notices, []);
319
- assert.equal(a.api.lost('101'), false);
320
- await assert.rejects(b.api.fence('101'), lost);
321
- });
322
408
 
323
- test('a fresh connection replaces a held visit of a claimed account and revalidates', async t => {
324
- const { instance, rows } = setup(t), a = instance('a'), b = instance('b');
325
- a.lease.sync([['p1', '101']]);
326
- await a.api.deck('101').set({ progress: 1 });
327
- // The socket drops; resumePose retention holds p1. The player opens B, then returns.
328
- a.lease.sync([], ['p1']);
329
- b.lease.join('p9', '101'); await b.api.deck('101').set({ progress: 2 });
330
- b.lease.shutdown();
331
- const forgotten = [];
332
- a.lease.sync([['fresh', '101']], ['p1'], id => {
333
- forgotten.push(id);
334
- // The held visit's leave saves under its own, superseded claim.
335
- void a.api.deck('101').set({ progress: 1 }).catch(() => {});
336
- });
337
- assert.deepEqual(forgotten, ['p1']);
338
- assert.deepEqual(await a.api.deck('101').get(['progress']), { progress: 2 });
339
- await a.api.deck('101').set({ progress: 3 });
340
- assert.equal(rows.get('deck:progress'), 3);
341
- assert.equal(a.calls.claim.length, 2);
342
- assert.equal(a.api.lost('101'), false);
343
- });
344
-
345
- test('revalidation retries a rate-limited check, hides a late old-visit loss, and arrival views follow the new claim', async t => {
346
- t.mock.timers.enable({ apis: ['setTimeout'] });
347
- const { instance, rows } = setup(t), a = instance('a'), b = instance('b');
348
- const notices = [];
349
- a.api.onLost(user => notices.push(user));
350
- a.lease.join('p1', '101'); await a.api.ready('101');
351
- const old = a.api.deck('101');
352
- // The old visit's write is past its queue when the player leaves for B.
353
- const send = a.storage.transaction, hold = deferred();
354
- let checks = 0;
355
- a.storage.transaction = async input => {
356
- checks++;
357
- if (checks === 1) { await hold.promise; return send(input); }
358
- if (checks === 2) throw Object.assign(Error('busy'), { code: 'CAULDRON_STORAGE_RATE_LIMITED', retryable: true, committed: false });
359
- return send(input);
360
- };
361
- const late = old.set({ progress: 1 });
362
- a.lease.leave('p1');
363
- b.lease.join('p9', '101'); await b.api.deck('101').set({ progress: 2 }); b.lease.leave('p9');
364
- a.lease.join('p2', '101');
365
- // The arrival takes its view while the claim is being revalidated.
366
- const arrival = a.api.deck('101');
367
- assert.notEqual(arrival, old);
368
- const write = arrival.set({ progress: 3 });
369
- hold.resolve();
370
- await assert.rejects(late, lost);
371
- assert.equal(a.api.lost('101'), false, 'no notice while revalidating');
372
- for (let i = 0; i < 6; i++) await new Promise(r => setImmediate(r));
373
- t.mock.timers.tick(1000);
374
- await a.api.ready('101');
375
- await write;
376
- assert.equal(rows.get('deck:progress'), 3);
377
- assert.equal(a.calls.claim.length, 2, 'a rate-limited check retried, found the claim taken and reclaimed');
378
- assert.equal(a.api.lost('101'), false);
379
- assert.deepEqual(notices, []);
380
- await assert.rejects(old.set({ progress: 1 }), lost);
381
- });
382
-
383
- test('a view made during one revalidation never follows a later visit\'s reclaim', async t => {
384
- const { instance, rows } = setup(t), a = instance('a'), b = instance('b');
385
- a.lease.join('p1', '101'); await a.api.ready('101');
386
- a.lease.leave('p1');
387
- // Visit B comes back; nobody else claimed, so its round keeps the claim.
388
- a.lease.join('p2', '101');
389
- const viewB = a.api.deck('101');
390
- await a.api.ready('101');
391
- await viewB.set({ progress: 1 });
392
- // B goes, another instance takes the account, visit C comes back and reclaims.
393
- a.lease.leave('p2');
394
- b.lease.join('p9', '101'); await b.api.deck('101').set({ progress: 2 }); b.lease.leave('p9');
395
- a.lease.join('p3', '101');
396
- await a.api.ready('101');
397
- assert.equal(a.calls.claim.length, 2);
398
- await assert.rejects(viewB.set({ progress: 1 }), lost);
399
- assert.equal(rows.get('deck:progress'), 2);
400
- await a.api.deck('101').set({ progress: 3 });
401
- assert.equal(rows.get('deck:progress'), 3);
409
+ // Every order rounds 3 to 5 found, against storage that runs the writer check
410
+ // with the write. One trip A -> (B) -> A, with each concurrent event placed
411
+ // in each slot it can take:
412
+ // late A's first visit sends a write that sits on the wire until: never,
413
+ // before B claims, after B saved, or after the return to A.
414
+ // claim A's first claim answers: at once, or RATE_LIMITED once with the
415
+ // retry due while the player is still on A, or after they left.
416
+ // via the player goes through B (which claims and saves), or straight back.
417
+ // again the return's claim answers: at once, RATE_LIMITED once, or INTERNAL.
418
+ // drop while the return waits for the first visit's end save to land, the
419
+ // socket drops and: stays (none), resumes on the same player id before
420
+ // or after that save lands, or the player opens B instead.
421
+ // defer the place's arrival waits for `holds` of the old view before (or
422
+ // after) its own read starts.
423
+ // Each order must keep THE INVARIANT (writerLease.js): every write that
424
+ // lands carried the writer the platform held for the account as it landed.
425
+ // And: a refused write is never sent again; the first visit's end save lands
426
+ // before the return claims; the return reloads what storage holds; the newest
427
+ // place the player is in saves and the other does not; A never claims for a
428
+ // player who is not connected to it.
429
+ test('every interleaving of trips, late writes, retries, drops and deferred arrivals keeps the invariant', async t => {
430
+ const orders = [];
431
+ for (const late of ['none', 'beforeB', 'afterB', 'afterReturn'])
432
+ for (const claim of ['ok', 'retryHere', 'retryAway'])
433
+ for (const via of ['B', 'none'])
434
+ for (const again of ['ok', 'busy', 'refused'])
435
+ for (const drop of ['none', 'resume', 'resumeAfter', 'toB'])
436
+ for (const defer of [false, true]) orders.push({ late, claim, via, again, drop, defer });
437
+ assert.equal(orders.length, 576);
438
+ // ORDER=<substring of an order's JSON> runs only the matching ones.
439
+ for (const order of orders.filter(o => JSON.stringify(o).includes(process.env.ORDER ?? ''))) {
440
+ await t.test(JSON.stringify(order), async st => runOrder(st, order));
441
+ }
402
442
  });
403
443
 
404
- test('a second lone return while a revalidation is pending checks again after a takeover', async t => {
405
- const { instance, rows } = setup(t), a = instance('a'), b = instance('b');
444
+ async function runOrder(t, { late, claim, via, again, drop, defer }) {
445
+ t.mock.timers.enable({ apis: ['setTimeout', 'Date'] });
446
+ const world = setup(t), a = world.instance('a'), b = world.instance('b');
406
447
  const notices = [];
407
448
  a.api.onLost(user => notices.push(user));
408
- a.lease.join('p1', '101'); await a.api.ready('101');
409
- a.lease.leave('p1');
410
- // The first return's check succeeds at the broker, but its reply is late.
411
- const send = a.storage.transaction, reply = deferred();
412
- let checks = 0;
413
- a.storage.transaction = async input => {
414
- const result = send(input);
415
- if (++checks === 1) await reply.promise;
416
- return result;
449
+ const settle = async () => { for (let i = 0; i < 30; i++) await new Promise(r => setImmediate(r)); };
450
+ const drive = async (ms = 1000, n = 3) => { for (let i = 0; i < n; i++) { await settle(); t.mock.timers.tick(ms); } await settle(); };
451
+ // Failure injection.
452
+ const claimA = a.storage.claim;
453
+ let firstFails = claim === 'ok' ? 0 : 1, returning = false, returnFails = again === 'ok' ? 0 : 1;
454
+ a.storage.claim = async input => {
455
+ if (!returning && firstFails > 0) { firstFails--; throw Object.assign(Error('busy'), { code: 'CAULDRON_STORAGE_RATE_LIMITED', retryable: true, committed: false }); }
456
+ if (returning && returnFails > 0) {
457
+ returnFails--;
458
+ throw again === 'busy'
459
+ ? Object.assign(Error('busy'), { code: 'CAULDRON_STORAGE_RATE_LIMITED', retryable: true, committed: false })
460
+ : Object.assign(Error('bad'), { code: 'CAULDRON_STORAGE_INTERNAL', retryable: false, committed: false });
461
+ }
462
+ return claimA(input);
417
463
  };
418
- a.lease.join('p2', '101');
419
- await new Promise(r => setImmediate(r));
420
- assert.equal(checks, 1, 'the first round is in flight');
421
- // That visit leaves; another instance takes the account; the player comes back.
422
- a.lease.leave('p2');
423
- b.lease.join('p9', '101'); await b.api.deck('101').set({ progress: 2 }); b.lease.leave('p9');
424
- a.lease.join('p3', '101');
425
- const arrival = a.api.deck('101');
426
- reply.resolve();
427
- await a.api.ready('101');
428
- assert.deepEqual(await arrival.get(['progress']), { progress: 2 });
429
- await arrival.set({ progress: 3 });
430
- assert.equal(rows.get('deck:progress'), 3);
431
- assert.equal(a.calls.claim.length, 2, 'the second return reclaimed after the stale success');
432
- assert.equal(a.api.lost('101'), false);
433
- assert.deepEqual(notices, []);
434
- });
435
-
436
- test('an older pending round defers to a newer arrival already checking the replacement claim', async t => {
437
- const { instance, rows } = setup(t), a = instance('a'), b = instance('b');
438
- const notices = [];
439
- a.api.onLost(user => notices.push(user));
440
- a.lease.join('p1', '101'); await a.api.ready('101');
441
- a.lease.leave('p1');
442
- // Return 1's check waits before it is sent.
443
- const send = a.storage.transaction, gate = deferred();
444
- let checks = 0, refuse = false;
445
- a.storage.transaction = async input => {
446
- if (++checks === 1) await gate.promise;
447
- // Another instance took the replacement claim just after it settled.
448
- if (refuse && input.changes.length === 0) { refuse = false; throw Object.assign(Error('taken'), { code: 'CAULDRON_STORAGE_WRITER_LOST' }); }
449
- return send(input);
464
+ const lateGate = deferred(), endGate = deferred();
465
+ a.inTransit = async (verb, values) => {
466
+ if (values?.p === 'A1-late') await lateGate.promise;
467
+ // The first visit's end save is still on the wire as the return arrives.
468
+ if (values?.p === 'a1-end') await endGate.promise;
450
469
  };
451
- a.lease.join('p2', '101');
452
- await new Promise(r => setImmediate(r));
453
- a.lease.leave('p2');
454
- b.lease.join('p9', '101'); await b.api.deck('101').set({ progress: 2 }); b.lease.leave('p9');
455
- // Return 2 queues behind return 1, and goes.
456
- a.lease.join('p3', '101');
457
- const departed = a.api.deck('101');
458
- // Return 1 finds the claim taken and, with return 2 still here, reclaims;
459
- // that claim is slow to settle.
460
- const claim = a.storage.claim, held = deferred();
461
- let claims = 0;
462
- a.storage.claim = async input => { if (++claims === 1) await held.promise; return claim(input); };
463
- gate.resolve();
464
- for (let i = 0; i < 6; i++) await new Promise(r => setImmediate(r));
465
- assert.equal(claims, 1, 'return 1 is reclaiming');
466
- a.lease.leave('p3');
467
- // Return 3 arrives on the replacement claim while it settles.
468
- a.lease.join('p4', '101');
469
- const newest = a.api.deck('101');
470
- const before = checks;
471
- refuse = true;
472
- held.resolve();
473
- await newest.set({ progress: 4 });
474
- assert.equal(checks - before, 2, 'one round check of the replacement claim (return 3\'s), then the write\'s fence');
475
- assert.equal(rows.get('deck:progress'), 4, 'the newest arrival saves under the claim its round reopened');
476
- assert.equal(claims, 2, 'return 3 reclaimed; return 2 only waited for it');
477
- // Return 2 had gone: its view never writes under return 3's claim.
478
- await assert.rejects(departed.set({ progress: 3 }), lost);
479
- assert.equal(rows.get('deck:progress'), 4);
480
- assert.equal(a.api.lost('101'), false);
481
- assert.deepEqual(notices, []);
482
- });
483
-
484
- test('a lone visit back whose check is refused claims afresh instead of losing its first save', async t => {
485
- t.mock.timers.enable({ apis: ['setTimeout'] });
486
- for (const failure of ['refused', 'timed out']) {
487
- const { instance, rows } = setup(t), a = instance('a'), b = instance('b');
488
- const notices = [];
489
- a.api.onLost(user => notices.push(user));
490
- a.lease.join('p1', '101'); await a.api.ready('101');
491
- const old = a.api.deck('101');
492
- a.lease.leave('p1');
493
- // Town B takes the account; the player closes B and comes back alone inside 60 s.
494
- b.lease.join('p9', '101'); await b.api.deck('101').set({ progress: 2 }); b.lease.leave('p9');
495
- // A's return check fails without saying whether the claim is still A's;
496
- // storage recovers before the first save.
497
- const send = a.storage.transaction;
498
- let failing = true;
499
- a.storage.transaction = async input => {
500
- if (failing && input.changes.length === 0) {
501
- if (failure === 'refused') { failing = false; throw Object.assign(Error('bad'), { code: 'CAULDRON_STORAGE_INTERNAL', retryable: false, committed: false }); }
502
- throw Object.assign(Error('busy'), { code: 'CAULDRON_STORAGE_RATE_LIMITED', retryable: true, committed: false });
503
- }
504
- return send(input);
470
+ // Track every write a place sends: a refused one must never be sent again.
471
+ const sends = new Map();
472
+ for (const root of [a, b]) {
473
+ const set = root.storage.deck.set;
474
+ root.storage.deck.set = async (values, options) => {
475
+ sends.set(values.p, (sends.get(values.p) ?? 0) + 1);
476
+ return set(values, options);
505
477
  };
506
- a.lease.join('p2', '101');
507
- const arrival = a.api.deck('101');
508
- if (failure === 'timed out') {
509
- for (let i = 0; i < 40 && a.calls.claim.length < 2; i++) {
510
- for (let j = 0; j < 4; j++) await new Promise(r => setImmediate(r));
511
- t.mock.timers.tick(8000);
512
- }
513
- failing = false;
514
- }
515
- await a.api.ready('101');
516
- assert.equal(a.calls.claim.length, 2, `${failure}: the arrival claimed afresh`);
517
- assert.equal(await a.api.holds(old), false, `${failure}: the departed visit's view no longer holds`);
518
- assert.equal(await a.api.holds(arrival), true);
519
- assert.deepEqual(await arrival.get(['progress']), { progress: 2 }, `${failure}: the arrival loads B's save`);
520
- await arrival.set({ progress: 3 });
521
- assert.equal(rows.get('deck:progress'), 3, `${failure}: the first save lands`);
522
- await a.api.deck('101').set({ progress: 4 });
523
- assert.equal(rows.get('deck:progress'), 4, `${failure}: later saves land`);
524
- await assert.rejects(old.set({ progress: 1 }), lost);
525
- assert.equal(a.api.lost('101'), false);
526
- assert.deepEqual(notices, []);
527
- a.lease.shutdown(); b.lease.shutdown();
528
478
  }
529
- });
479
+ // A place: one visit at a time, retained (resume) when its socket drops.
480
+ const place = (inst) => ({ inst, roster: [], held: [], visit: null, ends: [] });
481
+ const A = place(a), B = place(b);
482
+ const sync = (p) => p.inst.lease.sync(p.roster, p.held, id => {
483
+ // The kit's leave hook of a replaced held visit: its end save, under its own view.
484
+ if (p.visit?.id === id) p.ends.push(p.visit.view.set({ p: `${id}-end` }).then(() => 'saved', () => 'refused'));
485
+ });
486
+ const arrive = (p, id) => {
487
+ const before = p.visit;
488
+ p.roster = [[id, '101']];
489
+ sync(p);
490
+ p.held = [];
491
+ const view = p.inst.api.deck('101');
492
+ p.visit = { id, view, before };
493
+ // The place reloads unless the claim its retained state lives under holds.
494
+ p.visit.loaded = (async () => {
495
+ if (defer && before) { if (await p.inst.api.holds(before.view)) return { kept: true }; }
496
+ const read = view.get(['p']);
497
+ const kept = before && !defer ? await p.inst.api.holds(before.view) : false;
498
+ return { kept, value: (await read).p };
499
+ })();
500
+ return p.visit;
501
+ };
502
+ const dropSocket = (p) => { p.held = [p.visit.id]; p.roster = []; sync(p); };
503
+ const resume = (p) => { p.roster = [[p.visit.id, '101']]; sync(p); };
504
+ const quit = (p) => { p.held = []; p.roster = []; sync(p); };
505
+ const save = (p, value) => p.visit.view.set({ p: value }).then(() => 'saved', e => (lost(e) ? 'refused' : Promise.reject(e)));
530
506
 
531
- test('a return that left again never reclaims from the instance the player is on now', async t => {
532
- for (const answer of ['refused', 'taken']) {
533
- const { instance, rows } = setup(t), a = instance('a'), b = instance('b');
534
- const notices = [];
535
- a.api.onLost(user => notices.push(user));
536
- a.lease.join('p1', '101'); await a.api.ready('101');
537
- a.lease.leave('p1');
538
- // The return's check is slow; the player leaves again and opens B.
539
- const send = a.storage.transaction, gate = deferred();
540
- let first = true;
541
- a.storage.transaction = async input => {
542
- if (first && input.changes.length === 0) {
543
- first = false; await gate.promise;
544
- if (answer === 'refused') throw Object.assign(Error('bad'), { code: 'CAULDRON_STORAGE_INTERNAL', retryable: false, committed: false });
545
- }
546
- return send(input);
547
- };
548
- a.lease.join('p2', '101');
549
- await new Promise(r => setImmediate(r));
550
- a.lease.leave('p2');
551
- b.lease.join('p9', '101'); await b.api.deck('101').set({ progress: 2 });
552
- gate.resolve();
553
- for (let i = 0; i < 8; i++) await new Promise(r => setImmediate(r));
554
- await a.api.ready('101');
555
- assert.equal(a.calls.claim.length, 1, `${answer}: A did not claim again`);
556
- await b.api.deck('101').set({ progress: 3 });
557
- assert.equal(rows.get('deck:progress'), 3, `${answer}: B keeps saving`);
507
+ // A, first visit.
508
+ arrive(A, 'a1');
509
+ const first = save(A, 'A1');
510
+ const lateWrite = late === 'none' ? null : save(A, 'A1-late');
511
+ if (late === 'beforeB') lateGate.resolve();
512
+ if (claim === 'retryHere') await drive(300);
513
+ await settle();
514
+ dropSocket(A);
515
+ await drive(300);
516
+ if (claim === 'retryAway') {
517
+ assert.equal(a.calls.claim.length, 0, 'a claim waiting on a gone socket sends nothing');
518
+ }
519
+ // Through B, or not.
520
+ if (via === 'B') {
521
+ arrive(B, 'b1');
522
+ assert.equal(await save(B, 'B1'), 'saved');
523
+ }
524
+ if (late === 'afterB') { lateGate.resolve(); await settle(); }
525
+ if (via === 'B') {
526
+ await drive(300);
527
+ assert.equal(world.rows.get('deck:p'), 'B1', 'nothing A sent lands after B saved');
528
+ quit(B);
529
+ }
530
+ // Lone return to A on a fresh connection.
531
+ const told = notices.length;
532
+ let claimsBefore;
533
+ returning = true;
534
+ const back = arrive(A, 'a2');
535
+ await settle();
536
+ if (drop !== 'none') {
537
+ dropSocket(A);
538
+ claimsBefore = a.calls.claim.length;
539
+ await settle();
540
+ if (drop === 'resume') resume(A);
541
+ }
542
+ let bWriter;
543
+ if (drop === 'resumeAfter') { endGate.resolve(); await drive(1000, 2); resume(A); }
544
+ if (drop === 'toB') {
545
+ arrive(B, 'b2');
546
+ assert.equal(await save(B, 'B2'), 'saved', 'the place the player went to saves');
547
+ bWriter = world.writerOf('101');
548
+ endGate.resolve();
549
+ if (late === 'afterReturn') lateGate.resolve();
550
+ await drive(1000, 6);
551
+ assert.equal(a.calls.claim.length, claimsBefore, 'A sends no claim once its player has gone');
552
+ assert.equal(world.writerOf('101'), bWriter);
553
+ assert.equal(world.rows.get('deck:p'), 'B2');
558
554
  assert.equal(b.api.lost('101'), false);
559
- assert.deepEqual(notices, []);
560
- a.lease.shutdown(); b.lease.shutdown();
555
+ } else {
556
+ endGate.resolve();
557
+ if (late === 'afterReturn') lateGate.resolve();
558
+ await drive(1000, 6);
559
+ const loaded = await back.loaded;
560
+ assert.equal(loaded.kept, false, 'every return reloads');
561
+ assert.equal(loaded.value, world.rows.get('deck:p'), 'the return reloads what storage holds');
562
+ if (via === 'B') assert.ok(['B1'].includes(loaded.value), `a return through B reloads B's save (${loaded.value})`);
563
+ if (again === 'refused') {
564
+ assert.equal(await save(A, 'A2'), 'refused', 'a refused claim does not save');
565
+ assert.equal(a.api.lost('101'), true);
566
+ } else {
567
+ assert.equal(await save(A, 'A2'), 'saved', 'the place the player came back to saves');
568
+ assert.equal(world.rows.get('deck:p'), 'A2');
569
+ assert.equal(a.api.lost('101'), false);
570
+ assert.equal(notices.length, told, 'the player here never sees the notice');
571
+ }
572
+ if (via === 'B' && again !== 'refused') await assert.rejects(b.api.deck('101').set({ p: 'B-stale' }), lost);
561
573
  }
562
- });
563
-
564
- test('an unanswered return check lets the previous visit\'s queued end save land before reclaiming', async t => {
565
- const { instance, rows } = setup(t), a = instance('a');
566
- const notices = [];
567
- a.api.onLost(user => notices.push(user));
568
- a.lease.join('p1', '101'); await a.api.ready('101');
569
- const old = a.api.deck('101');
570
- a.lease.leave('p1');
571
- // Nobody else claims. The return's check is refused once; storage recovers.
572
- const send = a.storage.transaction;
573
- let first = true;
574
- a.storage.transaction = async input => {
575
- if (first && input.changes.length === 0) { first = false; throw Object.assign(Error('bad'), { code: 'CAULDRON_STORAGE_INTERNAL', retryable: false, committed: false }); }
576
- return send(input);
577
- };
578
- a.lease.join('p2', '101');
579
- // The leave hook's end save is queued under the old claim as the return starts.
580
- const end = old.set({ progress: 1 });
581
- const arrival = a.api.deck('101');
582
- await end;
583
- assert.equal(rows.get('deck:progress'), 1, 'the end save landed under its own claim');
584
- await a.api.ready('101');
585
- assert.equal(a.calls.claim.length, 2, 'then the arrival claimed afresh');
586
- assert.deepEqual(await arrival.get(['progress']), { progress: 1 });
587
- await arrival.set({ progress: 2 });
588
- assert.equal(rows.get('deck:progress'), 2);
589
- assert.equal(a.api.lost('101'), false);
590
- assert.deepEqual(notices, []);
591
- });
592
-
593
- test('a return held after its connection went never reclaims from the instance the player moved to', async t => {
594
- const { instance, rows } = setup(t), a = instance('a'), b = instance('b');
595
- a.lease.sync([['p1', '101']]); await a.api.ready('101');
596
- a.lease.sync([]);
597
- const send = a.storage.transaction, gate = deferred();
598
- let first = true;
599
- a.storage.transaction = async input => {
600
- if (first && input.changes.length === 0) { first = false; await gate.promise; throw Object.assign(Error('bad'), { code: 'CAULDRON_STORAGE_INTERNAL', retryable: false, committed: false }); }
601
- return send(input);
602
- };
603
- a.lease.sync([['p2', '101']]);
604
- await new Promise(r => setImmediate(r));
605
- // The socket drops; the place still holds p2 (resumePose). The player opens B.
606
- a.lease.sync([], ['p2']);
607
- b.lease.join('p9', '101'); await b.api.deck('101').set({ progress: 2 });
608
- gate.resolve();
609
- for (let i = 0; i < 8; i++) await new Promise(r => setImmediate(r));
610
- await a.api.ready('101');
611
- assert.equal(a.calls.claim.length, 1, 'A did not claim again');
612
- await b.api.deck('101').set({ progress: 3 });
613
- assert.equal(rows.get('deck:progress'), 3);
614
- assert.equal(b.api.lost('101'), false);
615
- });
616
-
617
- test('a claim still retrying stops once every visit of the account has gone', async t => {
618
- t.mock.timers.enable({ apis: ['setTimeout'] });
619
- const { instance, rows } = setup(t), a = instance('a'), b = instance('b');
620
- const claim = a.storage.claim;
621
- let tries = 0;
622
- a.storage.claim = async input => {
623
- if (++tries === 1) throw Object.assign(Error('busy'), { code: 'CAULDRON_STORAGE_RATE_LIMITED', retryable: true, committed: false });
624
- return claim(input);
625
- };
626
- a.lease.join('p1', '101');
627
- await new Promise(r => setImmediate(r));
628
- a.lease.leave('p1');
629
- b.lease.join('p9', '101'); await b.api.deck('101').set({ progress: 2 });
630
- for (let i = 0; i < 4; i++) { t.mock.timers.tick(8000); await new Promise(r => setImmediate(r)); }
631
- await a.api.ready('101');
632
- assert.equal(tries, 1, 'no retry after the visit left');
633
- await b.api.deck('101').set({ progress: 3 });
634
- assert.equal(rows.get('deck:progress'), 3);
635
- assert.equal(b.api.lost('101'), false);
636
- });
637
-
638
- test('an unanswered return check drains end saves queued while earlier ones were in flight', async t => {
639
- const { instance, rows } = setup(t), a = instance('a');
640
- a.lease.join('p1', '101'); await a.api.ready('101');
641
- const old = a.api.deck('101');
642
- a.lease.leave('p1');
643
- const send = a.storage.transaction;
644
- let first = true;
645
- a.storage.transaction = async input => {
646
- if (first && input.changes.length === 0) { first = false; throw Object.assign(Error('bad'), { code: 'CAULDRON_STORAGE_INTERNAL', retryable: false, committed: false }); }
647
- return send(input);
648
- };
649
- // The first end save is slow to write; a second is queued while it is in flight.
650
- const set = a.storage.deck.set, slow = deferred();
651
- let writes = 0;
652
- a.storage.deck.set = async values => { if (++writes === 1) await slow.promise; return set(values); };
653
- a.lease.join('p2', '101');
654
- const one = old.set({ end: 1 });
655
- for (let i = 0; i < 6; i++) await new Promise(r => setImmediate(r));
656
- assert.equal(writes, 1, 'the first end save is in flight');
657
- const two = old.set({ end: 2 });
658
- slow.resolve();
659
- await one; await two;
660
- assert.equal(rows.get('deck:end'), 2, 'both end saves landed under their own claim');
661
- await a.api.ready('101');
662
- assert.equal(a.calls.claim.length, 2);
663
- await a.api.deck('101').set({ progress: 5 });
664
- assert.equal(rows.get('deck:progress'), 5);
665
- });
574
+ lateGate.resolve();
575
+ await drive(1000, 2);
576
+ await Promise.all([first, lateWrite, ...A.ends, ...B.ends]);
577
+ // THE INVARIANT, over every write that landed.
578
+ for (const commit of world.commits) {
579
+ assert.ok(commit.carried !== undefined, `${commit.values.p} landed without a writer check`);
580
+ assert.equal(commit.carried, commit.current, `${commit.values.p} landed under a claim the platform no longer held`);
581
+ }
582
+ for (const [value, count] of sends) assert.equal(count, 1, `${value} was sent ${count} times`);
583
+ if (via === 'B' && late !== 'none' && late !== 'beforeB') {
584
+ assert.equal(await lateWrite, 'refused', 'a write on the wire across B\'s claim is refused');
585
+ }
586
+ if (via === 'none' && claim !== 'retryAway' && late !== 'afterReturn' && drop !== 'toB') {
587
+ assert.equal(await A.ends[0], 'saved', 'the first visit\'s end save lands before the return claims');
588
+ }
589
+ }