durability 4.2.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/README.md +340 -12
  2. package/dist/dev.d.ts +22 -0
  3. package/dist/dev.js +149 -0
  4. package/dist/durability-alarms.d.ts +16 -3
  5. package/dist/durability-alarms.js +21 -7
  6. package/dist/durability.d.ts +31 -6
  7. package/dist/durability.js +42 -16
  8. package/dist/errors.d.ts +31 -0
  9. package/dist/errors.js +45 -0
  10. package/dist/experimental.d.ts +5 -0
  11. package/dist/experimental.js +3 -0
  12. package/dist/fanout.d.ts +6 -0
  13. package/dist/fanout.js +20 -9
  14. package/dist/index.d.ts +4 -2
  15. package/dist/index.js +3 -2
  16. package/dist/lifecycle.d.ts +6 -1
  17. package/dist/lifecycle.js +3 -2
  18. package/dist/log.js +61 -16
  19. package/dist/memoize.d.ts +34 -0
  20. package/dist/memoize.js +136 -0
  21. package/dist/migrations.d.ts +64 -2
  22. package/dist/migrations.js +179 -34
  23. package/dist/outbox.d.ts +13 -0
  24. package/dist/outbox.js +78 -30
  25. package/dist/scheduler.d.ts +9 -0
  26. package/dist/scheduler.js +53 -11
  27. package/dist/semaphore-leases.js +15 -9
  28. package/dist/semaphore-queue-store.d.ts +12 -2
  29. package/dist/semaphore-queue-store.js +32 -5
  30. package/dist/semaphore-queue.js +109 -43
  31. package/dist/semaphore-server.js +7 -9
  32. package/dist/socket.js +19 -10
  33. package/dist/storage.d.ts +1 -0
  34. package/dist/storage.js +1 -1
  35. package/dist/topology-client.d.ts +40 -0
  36. package/dist/topology-client.js +180 -0
  37. package/dist/topology-snapshot.d.ts +70 -0
  38. package/dist/topology-snapshot.js +48 -0
  39. package/dist/topology.d.ts +94 -0
  40. package/dist/topology.js +468 -0
  41. package/dist/tracked-source.d.ts +2 -0
  42. package/dist/tracker.js +2 -0
  43. package/package.json +3 -2
package/dist/scheduler.js CHANGED
@@ -1,8 +1,10 @@
1
1
  import { z } from 'zod';
2
+ import { trace, traceMethods, traceStorage } from './dev.js';
2
3
  import { DurableRetryPolicyError, isErrorInstance, isNonRetryable, reportFailure, serializeError, } from './errors.js';
3
4
  import { createEmitter, } from './lifecycle.js';
4
- import { migrate } from './migrations.js';
5
+ import { isSchemaCurrent, migrateCapability, } from './migrations.js';
5
6
  import { assertPositiveInteger } from './policy.js';
7
+ import { deleteMemos, deleteStaleMemos } from './memoize.js';
6
8
  import { claimAttempt, exhaustAttempts, updateAttempt, } from './storage.js';
7
9
  const minAlarmDelayMs = 1000;
8
10
  const maxAlarmDelayMs = 15000;
@@ -55,10 +57,12 @@ const decideRetry = (spec, attempt, error) => {
55
57
  export class Engine {
56
58
  constructor(context, options) {
57
59
  this.participants = [];
60
+ this.migrated = new Set();
58
61
  this.permitWaiters = [];
59
62
  this.watchers = new Map();
60
63
  this.context = context;
61
- this.sql = context.storage.sql;
64
+ this.storage = traceStorage(context.storage);
65
+ this.sql = this.storage.sql;
62
66
  this.alarmConcurrency = options.alarmConcurrency ?? 10;
63
67
  this.alarmMinDelayMs = options.alarmMinDelayMs ?? 15000;
64
68
  this.alarmHandoffMs = options.alarmHandoffMs ?? 14 * 60000;
@@ -89,16 +93,32 @@ export class Engine {
89
93
  onError(error);
90
94
  }
91
95
  }
96
+ /**
97
+ * The owning Durable Object's ID. The public context type omits it, but the
98
+ * `DurableObjectState` applications pass at runtime carries it.
99
+ */
100
+ objectId() {
101
+ const id = Reflect.get(this.context, 'id');
102
+ return id === undefined || id === null ? undefined : String(id);
103
+ }
92
104
  emitter(hook) {
93
105
  return createEmitter(hook, (promise) => this.context.waitUntil?.(promise));
94
106
  }
95
107
  /** Applies a helper's SQLite migrations. */
96
108
  migrateSchema(capability, migrations) {
97
- migrate(this.context.storage, capability, migrations, undefined);
109
+ // Helpers of one kind on one scheduler share their tables; checking the
110
+ // schema again for each would only add statements to every construction.
111
+ if (this.migrated.has(capability)) {
112
+ return;
113
+ }
114
+ if (!isSchemaCurrent(this.sql, capability, migrations)) {
115
+ migrateCapability(this.storage, capability, migrations, undefined);
116
+ }
117
+ this.migrated.add(capability);
98
118
  }
99
119
  /** Runs synchronous writes atomically without touching the physical alarm. */
100
120
  transactionSync(write) {
101
- return this.context.storage.transactionSync(() => write(this.sql));
121
+ return this.storage.transactionSync(() => write(this.sql));
102
122
  }
103
123
  /**
104
124
  * Runs the writes and reconciles the physical alarm in one storage
@@ -106,7 +126,7 @@ export class Engine {
106
126
  */
107
127
  async transaction(write) {
108
128
  let nextPendingAt;
109
- const result = await this.context.storage.transaction(async (transaction) => {
129
+ const result = await this.storage.transaction(async (transaction) => {
110
130
  const value = write(this.sql);
111
131
  nextPendingAt = await this.reconcile(transaction);
112
132
  return value;
@@ -165,6 +185,7 @@ export class Engine {
165
185
  return;
166
186
  }
167
187
  this.earlyAlarm = undefined;
188
+ trace('scheduler.early_alarm', { scheduledTime: timestamp });
168
189
  const onError = (error) => reportFailure('durability.background_execution.failed', { scheduledTime: timestamp }, error);
169
190
  const execution = this.alarm().catch(onError);
170
191
  this.waitUntil(execution, onError);
@@ -199,6 +220,7 @@ export class Engine {
199
220
  .exec(`SELECT COUNT(*) AS count FROM ${table.name} WHERE created_at < ?`, before)
200
221
  .toArray()[0]);
201
222
  sql.exec(`DELETE FROM ${table.name} WHERE created_at < ?`, before);
223
+ deleteStaleMemos(sql, table);
202
224
  return count;
203
225
  });
204
226
  if (removed > 0) {
@@ -326,7 +348,7 @@ export class Engine {
326
348
  /** Arms the physical alarm directly, outside any transaction; the next run reconciles properly. */
327
349
  async armFallbackAlarm() {
328
350
  try {
329
- await this.context.storage.setAlarm(Date.now() + this.alarmMinDelayMs);
351
+ await this.storage.setAlarm(Date.now() + this.alarmMinDelayMs);
330
352
  return true;
331
353
  }
332
354
  catch {
@@ -334,7 +356,7 @@ export class Engine {
334
356
  }
335
357
  }
336
358
  armForHandoff(timestamp) {
337
- return this.context.storage.transaction(async (physicalAlarm) => {
359
+ return this.storage.transaction(async (physicalAlarm) => {
338
360
  const currentAlarm = await physicalAlarm.getAlarm();
339
361
  const target = Math.max(timestamp, Date.now() + this.alarmMinDelayMs);
340
362
  if (currentAlarm === null || currentAlarm > target) {
@@ -403,7 +425,14 @@ export class Engine {
403
425
  }, policy.attemptTimeoutMs);
404
426
  });
405
427
  const result = await Promise.race([handlerResult, timeoutResult]);
406
- const timestamp = spec.complete(result, attempt);
428
+ // One transaction: a failed memo cleanup must undo the completion, not orphan the memos.
429
+ const timestamp = this.transactionSync((sql) => {
430
+ const completedAt = spec.complete(result, attempt);
431
+ if (completedAt !== undefined) {
432
+ deleteMemos(sql, spec.table, key);
433
+ }
434
+ return completedAt;
435
+ });
407
436
  if (timestamp !== undefined) {
408
437
  this.notify(spec.table, key, row);
409
438
  spec.emit({
@@ -413,7 +442,7 @@ export class Engine {
413
442
  attempt,
414
443
  durationMs: Math.max(0, timestamp - startedAt),
415
444
  outcome: 'completed',
416
- result,
445
+ ...(spec.includeResult ? { result } : {}),
417
446
  });
418
447
  }
419
448
  }
@@ -439,7 +468,13 @@ export class Engine {
439
468
  async failExhausted(spec) {
440
469
  const { key, row, policy } = spec;
441
470
  const exhausted = serializeError(spec.exhaustedError());
442
- const attempt = await this.transaction((sql) => exhaustAttempts(sql, spec.table, key, row.generation_id, policy.maxAttempts, exhausted));
471
+ const attempt = await this.transaction((sql) => {
472
+ const exhaustedAttempt = exhaustAttempts(sql, spec.table, key, row.generation_id, policy.maxAttempts, exhausted);
473
+ if (exhaustedAttempt !== undefined) {
474
+ deleteMemos(sql, spec.table, key);
475
+ }
476
+ return exhaustedAttempt;
477
+ });
443
478
  if (attempt === undefined) {
444
479
  return;
445
480
  }
@@ -457,7 +492,13 @@ export class Engine {
457
492
  const { key, row } = spec;
458
493
  const decision = decideRetry(spec, attempt, caught);
459
494
  const serializedError = serializeError(decision.error);
460
- const updated = await this.transaction((sql) => updateAttempt(sql, spec.table, { key, generation: row.generation_id, attempt }, 'status = ?, next_attempt_at = ?, last_error = ?, last_error_name = ?', decision.terminal ? 'failed' : 'pending', decision.nextAttemptAt, serializedError.message, serializedError.name));
495
+ const updated = await this.transaction((sql) => {
496
+ const owned = updateAttempt(sql, spec.table, { key, generation: row.generation_id, attempt }, 'status = ?, next_attempt_at = ?, last_error = ?, last_error_name = ?', decision.terminal ? 'failed' : 'pending', decision.nextAttemptAt, serializedError.message, serializedError.name);
497
+ if (owned && decision.terminal) {
498
+ deleteMemos(sql, spec.table, key);
499
+ }
500
+ return owned;
501
+ });
461
502
  if (!updated) {
462
503
  return;
463
504
  }
@@ -519,6 +560,7 @@ export class DurabilityScheduler {
519
560
  return engineFor({ scheduler: this }).alarm(info);
520
561
  }
521
562
  }
563
+ traceMethods(DurabilityScheduler.prototype, 'DurabilityScheduler');
522
564
  /**
523
565
  * Resolves the engine a helper should attach to: the shared one behind a
524
566
  * scheduler, or a private one built from the Durable Object context.
@@ -75,7 +75,7 @@ export const applyLeaseReports = (store, policy, clientId, reports, now) => {
75
75
  };
76
76
  for (const report of reports) {
77
77
  const lease = store.getLease(report.leaseId);
78
- if (!lease || lease.client_id !== clientId || lease.expires_at <= now) {
78
+ if (!lease || lease.client_id !== clientId) {
79
79
  outcome.echoes.push({
80
80
  leaseId: report.leaseId,
81
81
  group: lease?.grp ?? '',
@@ -90,13 +90,19 @@ export const applyLeaseReports = (store, policy, clientId, reports, now) => {
90
90
  const drawn = Math.min(Math.max(0, report.drawn), lease.permits);
91
91
  outcome.drawn.set(lease.lease_id, drawn);
92
92
  const released = Math.max(lease.released, Math.min(Math.max(0, report.released), drawn));
93
+ // An expired lease is retired, not forgotten: its permits may be running.
93
94
  // Only the client's own retired report shrinks a lease to what it drew:
94
95
  // until then it may still be drawing, whoever asked it to stop.
95
- const retired = lease.retired === 1 || report.retired === true;
96
+ const retired = lease.retired === 1 || report.retired === true || lease.expires_at <= now;
96
97
  const permits = report.retired === true ? Math.max(released, drawn) : lease.permits;
97
98
  if (released >= permits) {
98
99
  store.removeLease(lease.lease_id);
99
- outcome.echoes.push({ ...toLeaseGrant(lease, now), expiresAt: now });
100
+ // Echoed as settled (permits = released) so the client can drop the lease.
101
+ outcome.echoes.push({
102
+ ...toLeaseGrant(lease, now),
103
+ permits: released,
104
+ expiresAt: now,
105
+ });
100
106
  continue;
101
107
  }
102
108
  const refreshed = {
@@ -104,7 +110,7 @@ export const applyLeaseReports = (store, policy, clientId, reports, now) => {
104
110
  permits,
105
111
  released,
106
112
  retired: retired ? 1 : 0,
107
- expires_at: now + policy.ttlMs,
113
+ expires_at: retired ? lease.expires_at : now + policy.ttlMs,
108
114
  };
109
115
  store.updateLease(refreshed);
110
116
  outcome.echoes.push(toLeaseGrant(refreshed, now));
@@ -127,13 +133,13 @@ export const allocateLeases = (store, policy, input) => {
127
133
  const groupFootprint = new Map();
128
134
  const usable = new Set();
129
135
  for (const lease of store.listLeases()) {
130
- if (lease.expires_at <= now) {
131
- store.removeLease(lease.lease_id);
132
- continue;
133
- }
134
136
  const group = input.policies.get(lease.grp) ?? {};
135
137
  let current = lease;
136
- if (!lease.retired && !leaseEligible(policy, group)) {
138
+ // An unreported lease is retired, not swept: its client may be cut off
139
+ // with work still running, so its footprint stays until it reports, as a
140
+ // claimed grant permit stays until it is released.
141
+ if (!lease.retired &&
142
+ (lease.expires_at <= now || !leaseEligible(policy, group))) {
137
143
  // Stop draws now; the full footprint stays until the client confirms.
138
144
  current = { ...lease, retired: 1 };
139
145
  store.updateLease(current);
@@ -45,12 +45,16 @@ export declare const queueLeaseRowSchema: z.ZodObject<{
45
45
  released: z.ZodNumber;
46
46
  created_at: z.ZodNumber;
47
47
  expires_at: z.ZodNumber;
48
+ renew_at: z.ZodNumber;
49
+ renew_ms: z.ZodNumber;
50
+ confirmed: z.ZodNumber;
48
51
  }, z.core.$strip>;
49
52
  export type QueueLeaseRow = z.infer<typeof queueLeaseRowSchema>;
50
53
  /**
51
54
  * A semaphore client's queue: items, the grants it holds per key, and per-key
52
55
  * request pacing. Due work is a key whose request time has passed with items
53
- * queued, or an admitted item whose `admit()` notification is outstanding.
56
+ * queued, an admitted item whose `admit()` notification is outstanding, or a
57
+ * lease the holder must hear about again.
54
58
  */
55
59
  export declare class QueueStore {
56
60
  private readonly sql;
@@ -73,7 +77,13 @@ export declare class QueueStore {
73
77
  getLease(leaseId: string): QueueLeaseRow | undefined;
74
78
  insertLease(row: QueueLeaseRow): boolean;
75
79
  updateLease(row: QueueLeaseRow): void;
76
- /** Drops leases that stopped drawing before `before` and have every drawn permit back. */
80
+ /**
81
+ * Leases the holder must hear about again: those with permits in use, and
82
+ * stopped ones it has not settled. Both stay counted at the holder until
83
+ * reported, so a lost report must be repeated.
84
+ */
85
+ listRenewalsDue(now: number, limit: number): QueueLeaseRow[];
86
+ /** Drops settled leases that stopped drawing before `before`. */
77
87
  removeSpentLeases(before: number): void;
78
88
  listLeases(key: string): QueueLeaseRow[];
79
89
  getKey(key: string): QueueKeyRow | undefined;
@@ -45,6 +45,12 @@ export const queueLeaseRowSchema = z.object({
45
45
  released: z.number().int().nonnegative(),
46
46
  created_at: z.number().int().nonnegative(),
47
47
  expires_at: z.number().int().nonnegative(),
48
+ /** When to report the lease again while it has permits in use, so the holder keeps it. */
49
+ renew_at: z.number().int().nonnegative(),
50
+ /** Half the lease's lifetime at the holder, from its last refresh. */
51
+ renew_ms: z.number().int().positive(),
52
+ /** 1 once the holder has settled the stopped lease; until then it is reported again. */
53
+ confirmed: z.number().int().min(0).max(1),
48
54
  });
49
55
  const queueItemColumns = queueItemRowSchema.keyof().options;
50
56
  const queueGrantColumns = queueGrantRowSchema.keyof().options;
@@ -56,7 +62,8 @@ const statusCountSchema = z.object({
56
62
  /**
57
63
  * A semaphore client's queue: items, the grants it holds per key, and per-key
58
64
  * request pacing. Due work is a key whose request time has passed with items
59
- * queued, or an admitted item whose `admit()` notification is outstanding.
65
+ * queued, an admitted item whose `admit()` notification is outstanding, or a
66
+ * lease the holder must hear about again.
60
67
  */
61
68
  export class QueueStore {
62
69
  constructor(sql) {
@@ -162,13 +169,27 @@ export class QueueStore {
162
169
  }
163
170
  updateLease(row) {
164
171
  this.sql.exec(`UPDATE durability_semaphore_queue_leases
165
- SET permits = ?, drawn = ?, released = ?, expires_at = ?
166
- WHERE lease_id = ?`, row.permits, row.drawn, row.released, row.expires_at, row.lease_id);
172
+ SET permits = ?, drawn = ?, released = ?, expires_at = ?,
173
+ renew_at = ?, renew_ms = ?, confirmed = ?
174
+ WHERE lease_id = ?`, row.permits, row.drawn, row.released, row.expires_at, row.renew_at, row.renew_ms, row.confirmed, row.lease_id);
175
+ }
176
+ /**
177
+ * Leases the holder must hear about again: those with permits in use, and
178
+ * stopped ones it has not settled. Both stay counted at the holder until
179
+ * reported, so a lost report must be repeated.
180
+ */
181
+ listRenewalsDue(now, limit) {
182
+ return queueLeaseRowSchema.array().parse(this.sql
183
+ .exec(`SELECT * FROM durability_semaphore_queue_leases
184
+ WHERE confirmed = 0 AND renew_at <= ?
185
+ AND (released < drawn OR expires_at <= ?)
186
+ ORDER BY renew_at ASC LIMIT ?`, now, now, limit)
187
+ .toArray());
167
188
  }
168
- /** Drops leases that stopped drawing before `before` and have every drawn permit back. */
189
+ /** Drops settled leases that stopped drawing before `before`. */
169
190
  removeSpentLeases(before) {
170
191
  this.sql.exec(`DELETE FROM durability_semaphore_queue_leases
171
- WHERE expires_at < ? AND released >= drawn`, before);
192
+ WHERE confirmed = 1 AND expires_at < ? AND released >= drawn`, before);
172
193
  }
173
194
  listLeases(key) {
174
195
  return queueLeaseRowSchema.array().parse(this.sql
@@ -210,6 +231,12 @@ export class QueueStore {
210
231
  WHERE status = 'admitted' AND (notified = 0 OR ?)
211
232
  ORDER BY next_attempt_at ASC LIMIT 1`, checksFinished ? 1 : 0)
212
233
  .toArray()[0]?.['at'],
234
+ // An idle lease is due when it stops at its expiry, not before.
235
+ this.sql
236
+ .exec(`SELECT MIN(CASE WHEN released < drawn THEN renew_at
237
+ ELSE MAX(renew_at, expires_at) END) AS at
238
+ FROM durability_semaphore_queue_leases WHERE confirmed = 0`)
239
+ .toArray()[0]?.['at'],
213
240
  ].filter((value) => typeof value === 'number');
214
241
  return candidates.length === 0 ? undefined : Math.min(...candidates);
215
242
  }
@@ -9,9 +9,10 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
9
9
  if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
10
10
  return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
11
11
  };
12
- var _DurabilitySemaphoreQueue_instances, _DurabilitySemaphoreQueue_engine, _DurabilitySemaphoreQueue_store, _DurabilitySemaphoreQueue_server, _DurabilitySemaphoreQueue_clientId, _DurabilitySemaphoreQueue_admit, _DurabilitySemaphoreQueue_finished, _DurabilitySemaphoreQueue_finishedCheckMs, _DurabilitySemaphoreQueue_maxAdmitBatch, _DurabilitySemaphoreQueue_attemptTimeoutMs, _DurabilitySemaphoreQueue_delay, _DurabilitySemaphoreQueue_emit, _DurabilitySemaphoreQueue_outbox, _DurabilitySemaphoreQueue_requesting, _DurabilitySemaphoreQueue_blocked, _DurabilitySemaphoreQueue_notifying, _DurabilitySemaphoreQueue_checking, _DurabilitySemaphoreQueue_runDue, _DurabilitySemaphoreQueue_request, _DurabilitySemaphoreQueue_claimLocally, _DurabilitySemaphoreQueue_drawLeases, _DurabilitySemaphoreQueue_returnIdleLeases, _DurabilitySemaphoreQueue_applyLeaseGrants, _DurabilitySemaphoreQueue_applyResults, _DurabilitySemaphoreQueue_refreshKey, _DurabilitySemaphoreQueue_stageReport, _DurabilitySemaphoreQueue_stageLeaseReport, _DurabilitySemaphoreQueue_publishReports, _DurabilitySemaphoreQueue_releaseFinished, _DurabilitySemaphoreQueue_checkFinished, _DurabilitySemaphoreQueue_notify, _DurabilitySemaphoreQueue_notifyBatch, _DurabilitySemaphoreQueue_retryDelay, _DurabilitySemaphoreQueue_deadline;
12
+ var _DurabilitySemaphoreQueue_instances, _DurabilitySemaphoreQueue_engine, _DurabilitySemaphoreQueue_store, _DurabilitySemaphoreQueue_server, _DurabilitySemaphoreQueue_clientId, _DurabilitySemaphoreQueue_admit, _DurabilitySemaphoreQueue_finished, _DurabilitySemaphoreQueue_finishedCheckMs, _DurabilitySemaphoreQueue_maxAdmitBatch, _DurabilitySemaphoreQueue_attemptTimeoutMs, _DurabilitySemaphoreQueue_delay, _DurabilitySemaphoreQueue_emit, _DurabilitySemaphoreQueue_outbox, _DurabilitySemaphoreQueue_requesting, _DurabilitySemaphoreQueue_blocked, _DurabilitySemaphoreQueue_notifying, _DurabilitySemaphoreQueue_checking, _DurabilitySemaphoreQueue_runDue, _DurabilitySemaphoreQueue_retrySoon, _DurabilitySemaphoreQueue_request, _DurabilitySemaphoreQueue_claimLocally, _DurabilitySemaphoreQueue_drawLeases, _DurabilitySemaphoreQueue_returnIdleLeases, _DurabilitySemaphoreQueue_applyLeaseGrants, _DurabilitySemaphoreQueue_renewLeases, _DurabilitySemaphoreQueue_applyResults, _DurabilitySemaphoreQueue_refreshKey, _DurabilitySemaphoreQueue_stageReport, _DurabilitySemaphoreQueue_stageLeaseReport, _DurabilitySemaphoreQueue_publishReports, _DurabilitySemaphoreQueue_releaseFinished, _DurabilitySemaphoreQueue_checkFinished, _DurabilitySemaphoreQueue_notify, _DurabilitySemaphoreQueue_notifyBatch, _DurabilitySemaphoreQueue_retryDelay, _DurabilitySemaphoreQueue_deadline;
13
+ import { traceLifecycle, traceMethods } from './dev.js';
13
14
  import { reportFailure, serializeError } from './errors.js';
14
- import { durabilitySemaphoreQueueMigrations, migrate, } from './migrations.js';
15
+ import { durabilitySemaphoreQueueMigrations, migrateCapability, } from './migrations.js';
15
16
  import { DurabilityOutbox, stageWithin } from './outbox.js';
16
17
  import { assertPositiveInteger, } from './policy.js';
17
18
  import { DurabilityScheduler, engineFor, } from './scheduler.js';
@@ -23,6 +24,8 @@ const queueItemsTable = {
23
24
  key: 'id',
24
25
  };
25
26
  const outboxChannel = 'durability.semaphore-queue';
27
+ /** Half the holder's default grantTimeoutMs. */
28
+ const maxReportRetryMs = 5000;
26
29
  /** Queued items read per key when building a request's demand. */
27
30
  const demandScanLimit = 200;
28
31
  const maxDelayMs = 2 ** 31 - 1;
@@ -46,6 +49,15 @@ const leaseReport = (lease) => ({
46
49
  * draws this long before it bounds the window a skewed clock could open.
47
50
  */
48
51
  const leaseDrawMarginMs = 30000;
52
+ /**
53
+ * Renews at half the lifetime the holder last gave the lease. The holder
54
+ * sweeps a lease nobody reports, and a lease with permits in use must not be
55
+ * swept while its work runs: the holder would lease those permits again.
56
+ */
57
+ const renewal = (now, expiresAt) => {
58
+ const renewMs = Math.max(1000, Math.floor((expiresAt - now) / 2));
59
+ return { renew_at: now + renewMs, renew_ms: renewMs };
60
+ };
49
61
  const drawable = (lease, now) => lease.drawn < lease.permits && lease.expires_at - leaseDrawMarginMs > now;
50
62
  /**
51
63
  * A composable DO capability for the client side of {@link DurabilitySemaphore}:
@@ -105,12 +117,17 @@ export class DurabilitySemaphoreQueue {
105
117
  const scheduler = config.scheduler ?? new DurabilityScheduler(config);
106
118
  __classPrivateFieldSet(this, _DurabilitySemaphoreQueue_engine, engineFor({ scheduler }), "f");
107
119
  __classPrivateFieldSet(this, _DurabilitySemaphoreQueue_store, new QueueStore(__classPrivateFieldGet(this, _DurabilitySemaphoreQueue_engine, "f").sql), "f");
108
- __classPrivateFieldSet(this, _DurabilitySemaphoreQueue_emit, __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_engine, "f").emitter(config.onLifecycleEvent), "f");
120
+ __classPrivateFieldSet(this, _DurabilitySemaphoreQueue_emit, traceLifecycle(__classPrivateFieldGet(this, _DurabilitySemaphoreQueue_engine, "f").emitter(config.onLifecycleEvent)), "f");
109
121
  __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_engine, "f").migrateSchema('semaphoreQueue', durabilitySemaphoreQueueMigrations);
110
122
  __classPrivateFieldSet(this, _DurabilitySemaphoreQueue_outbox, new DurabilityOutbox({
111
123
  scheduler,
112
124
  channel: outboxChannel,
113
125
  publish: (entries) => __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_publishReports).call(this, entries),
126
+ // A claim the holder has not heard about within its grant window is
127
+ // reclaimed, so reports retry well inside it instead of backing off for minutes.
128
+ retries: {
129
+ delay: (attempt) => Math.min(__classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_retryDelay).call(this, attempt), maxReportRetryMs),
130
+ },
114
131
  }), "f");
115
132
  __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_engine, "f").register({
116
133
  table: queueItemsTable,
@@ -119,7 +136,7 @@ export class DurabilitySemaphoreQueue {
119
136
  });
120
137
  }
121
138
  static migrate(context, target) {
122
- return migrate(context.storage, 'semaphoreQueue', durabilitySemaphoreQueueMigrations, target);
139
+ return migrateCapability(context.storage, 'semaphoreQueue', durabilitySemaphoreQueueMigrations, target);
123
140
  }
124
141
  alarm(info) {
125
142
  return __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_engine, "f").alarm(info);
@@ -288,10 +305,27 @@ _DurabilitySemaphoreQueue_engine = new WeakMap(), _DurabilitySemaphoreQueue_stor
288
305
  __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_store, "f").removeSpentLeases(now - __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_attemptTimeoutMs, "f"));
289
306
  });
290
307
  const keys = __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_store, "f").listDueKeys(now);
291
- await Promise.all([__classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_notify).call(this), __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_releaseFinished).call(this, now)]);
308
+ await Promise.all([
309
+ __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_notify).call(this),
310
+ __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_releaseFinished).call(this, now),
311
+ __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_renewLeases).call(this, now),
312
+ ]);
292
313
  await Promise.all(keys
293
314
  .filter(({ key }) => !__classPrivateFieldGet(this, _DurabilitySemaphoreQueue_requesting, "f").has(key))
294
315
  .map(({ key }) => __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_request).call(this, key)));
316
+ }, _DurabilitySemaphoreQueue_retrySoon =
317
+ /** Best effort: if this write fails too, the request deadline still applies. */
318
+ async function _DurabilitySemaphoreQueue_retrySoon(key, queued) {
319
+ await __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_engine, "f")
320
+ .transaction(() => {
321
+ const current = __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_store, "f").getKey(key);
322
+ __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_store, "f").upsertKey({
323
+ key,
324
+ queued: current?.queued ?? queued,
325
+ next_request_at: Date.now() + __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_retryDelay).call(this, 1),
326
+ });
327
+ })
328
+ .catch(() => undefined);
295
329
  }, _DurabilitySemaphoreQueue_request =
296
330
  /** Asks the holder for permits covering this key's queued items. */
297
331
  async function _DurabilitySemaphoreQueue_request(key) {
@@ -342,45 +376,46 @@ async function _DurabilitySemaphoreQueue_request(key) {
342
376
  }
343
377
  catch (error) {
344
378
  reportFailure('durability.semaphore_queue.request_failed', { key }, error);
379
+ await __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_retrySoon).call(this, key, queued.length);
380
+ return;
381
+ }
382
+ try {
345
383
  await __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_engine, "f").transaction(() => {
346
- const current = __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_store, "f").getKey(key);
347
- __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_store, "f").upsertKey({
348
- key,
349
- queued: current?.queued ?? queued.length,
350
- next_request_at: Date.now() + __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_retryDelay).call(this, 1),
351
- });
384
+ const txStore = __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_store, "f");
385
+ for (const given of grant.grants) {
386
+ txStore.insertGrant({
387
+ grant_id: given.grantId,
388
+ key,
389
+ grp: given.group,
390
+ permits: given.permits,
391
+ claimed: 0,
392
+ confirmed: 0,
393
+ admitted: 0,
394
+ released: 0,
395
+ created_at: Date.now(),
396
+ });
397
+ }
398
+ __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_applyLeaseGrants).call(this, key, grant.leases ?? []);
399
+ __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_claimLocally).call(this, key);
400
+ __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_applyResults).call(this, key, grant.grantResults);
401
+ if (grant.contended) {
402
+ __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_returnIdleLeases).call(this, key);
403
+ }
404
+ const remaining = __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_refreshKey).call(this, key, grant.nextRequestAt);
405
+ for (const group of new Set(remaining.map((item) => item.grp))) {
406
+ __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_blocked, "f").delete(pairKey(key, group));
407
+ }
408
+ for (const { group, reason } of grant.blocked) {
409
+ __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_blocked, "f").set(pairKey(key, group), reason);
410
+ }
352
411
  });
353
- return;
354
412
  }
355
- await __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_engine, "f").transaction(() => {
356
- const txStore = __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_store, "f");
357
- for (const given of grant.grants) {
358
- txStore.insertGrant({
359
- grant_id: given.grantId,
360
- key,
361
- grp: given.group,
362
- permits: given.permits,
363
- claimed: 0,
364
- confirmed: 0,
365
- admitted: 0,
366
- released: 0,
367
- created_at: Date.now(),
368
- });
369
- }
370
- __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_applyLeaseGrants).call(this, key, grant.leases ?? []);
371
- __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_claimLocally).call(this, key);
372
- __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_applyResults).call(this, key, grant.grantResults);
373
- if (grant.contended) {
374
- __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_returnIdleLeases).call(this, key);
375
- }
376
- const remaining = __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_refreshKey).call(this, key, grant.nextRequestAt);
377
- for (const group of new Set(remaining.map((item) => item.grp))) {
378
- __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_blocked, "f").delete(pairKey(key, group));
379
- }
380
- for (const { group, reason } of grant.blocked) {
381
- __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_blocked, "f").set(pairKey(key, group), reason);
382
- }
383
- });
413
+ catch (error) {
414
+ // The grant is lost with the transaction; ask again soon rather than
415
+ // at the request deadline, which only covers a crash.
416
+ await __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_retrySoon).call(this, key, queued.length);
417
+ throw error;
418
+ }
384
419
  const counts = new Map();
385
420
  for (const item of store.listItems(key, 'queued', demandScanLimit)) {
386
421
  counts.set(item.grp, (counts.get(item.grp) ?? 0) + 1);
@@ -484,7 +519,15 @@ async function _DurabilitySemaphoreQueue_request(key) {
484
519
  for (const grant of grants) {
485
520
  const lease = store.getLease(grant.leaseId);
486
521
  if (grant.expiresAt <= now) {
487
- if (!lease || retiredLocally(lease, now)) {
522
+ if (!lease) {
523
+ continue;
524
+ }
525
+ if (retiredLocally(lease, now)) {
526
+ // The holder settles a stopped lease by echoing it with nothing left
527
+ // counted; until then this client keeps reporting it.
528
+ if (grant.permits <= lease.released && !lease.confirmed) {
529
+ store.updateLease({ ...lease, confirmed: 1 });
530
+ }
488
531
  continue;
489
532
  }
490
533
  // The holder keeps counting the whole lease until this report says
@@ -497,7 +540,11 @@ async function _DurabilitySemaphoreQueue_request(key) {
497
540
  if (lease) {
498
541
  // A refresh sent before this client retired the lease must not revive it.
499
542
  if (!retiredLocally(lease, now)) {
500
- store.updateLease({ ...lease, expires_at: grant.expiresAt });
543
+ store.updateLease({
544
+ ...lease,
545
+ expires_at: grant.expiresAt,
546
+ ...renewal(now, grant.expiresAt),
547
+ });
501
548
  }
502
549
  continue;
503
550
  }
@@ -510,8 +557,26 @@ async function _DurabilitySemaphoreQueue_request(key) {
510
557
  released: 0,
511
558
  created_at: now,
512
559
  expires_at: grant.expiresAt,
560
+ ...renewal(now, grant.expiresAt),
561
+ confirmed: 0,
513
562
  });
514
563
  }
564
+ }, _DurabilitySemaphoreQueue_renewLeases =
565
+ /**
566
+ * Reports leases with permits in use before the holder retires them, and
567
+ * stopped leases until the holder settles them.
568
+ */
569
+ async function _DurabilitySemaphoreQueue_renewLeases(now) {
570
+ // An idle run must not write: every transaction reconciles the alarm.
571
+ if (__classPrivateFieldGet(this, _DurabilitySemaphoreQueue_store, "f").listRenewalsDue(now, 1).length === 0) {
572
+ return;
573
+ }
574
+ await __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_engine, "f").transaction(() => {
575
+ for (const lease of __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_store, "f").listRenewalsDue(now, 100)) {
576
+ __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_store, "f").updateLease({ ...lease, renew_at: now + lease.renew_ms });
577
+ __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_instances, "m", _DurabilitySemaphoreQueue_stageLeaseReport).call(this, lease);
578
+ }
579
+ });
515
580
  }, _DurabilitySemaphoreQueue_applyResults = function _DurabilitySemaphoreQueue_applyResults(key, results) {
516
581
  const store = __classPrivateFieldGet(this, _DurabilitySemaphoreQueue_store, "f");
517
582
  const now = Date.now();
@@ -795,3 +860,4 @@ async function _DurabilitySemaphoreQueue_publishReports(entries) {
795
860
  }
796
861
  }
797
862
  };
863
+ traceMethods(DurabilitySemaphoreQueue.prototype, 'DurabilitySemaphoreQueue');
@@ -10,8 +10,9 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
10
10
  return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
11
11
  };
12
12
  var _DurabilitySemaphoreServer_instances, _DurabilitySemaphoreServer_engine, _DurabilitySemaphoreServer_store, _DurabilitySemaphoreServer_limit, _DurabilitySemaphoreServer_groupLimit, _DurabilitySemaphoreServer_canGrant, _DurabilitySemaphoreServer_roundWaitMs, _DurabilitySemaphoreServer_roundPeriodMs, _DurabilitySemaphoreServer_grantTimeoutMs, _DurabilitySemaphoreServer_leases, _DurabilitySemaphoreServer_emit, _DurabilitySemaphoreServer_pending, _DurabilitySemaphoreServer_roundTimer, _DurabilitySemaphoreServer_round, _DurabilitySemaphoreServer_lastRoundAt, _DurabilitySemaphoreServer_contended, _DurabilitySemaphoreServer_grantRoom, _DurabilitySemaphoreServer_scheduleRound, _DurabilitySemaphoreServer_runRound, _DurabilitySemaphoreServer_allocate, _DurabilitySemaphoreServer_sync, _DurabilitySemaphoreServer_applyReports, _DurabilitySemaphoreServer_reclaim, _DurabilitySemaphoreServer_reclaimExpired, _DurabilitySemaphoreServer_resolveLimit, _DurabilitySemaphoreServer_groupPolicy;
13
+ import { traceLifecycle, traceMethods } from './dev.js';
13
14
  import { SemaphoreProtocolError, reportFailure } from './errors.js';
14
- import { durabilitySemaphoreMigrations, migrate, } from './migrations.js';
15
+ import { durabilitySemaphoreMigrations, migrateCapability, } from './migrations.js';
15
16
  import { assertPositiveInteger } from './policy.js';
16
17
  import { allocateLeases, applyLeaseReports, budgets, leaseFootprint, resolveLeasePolicy, } from './semaphore-leases.js';
17
18
  import { engineFor, } from './scheduler.js';
@@ -97,7 +98,7 @@ export class DurabilitySemaphoreServer {
97
98
  }
98
99
  }
99
100
  __classPrivateFieldSet(this, _DurabilitySemaphoreServer_engine, engineFor(config), "f");
100
- __classPrivateFieldSet(this, _DurabilitySemaphoreServer_emit, __classPrivateFieldGet(this, _DurabilitySemaphoreServer_engine, "f").emitter(config.onLifecycleEvent), "f");
101
+ __classPrivateFieldSet(this, _DurabilitySemaphoreServer_emit, traceLifecycle(__classPrivateFieldGet(this, _DurabilitySemaphoreServer_engine, "f").emitter(config.onLifecycleEvent)), "f");
101
102
  __classPrivateFieldGet(this, _DurabilitySemaphoreServer_engine, "f").migrateSchema('semaphore', durabilitySemaphoreMigrations);
102
103
  __classPrivateFieldSet(this, _DurabilitySemaphoreServer_store, new HolderStore(__classPrivateFieldGet(this, _DurabilitySemaphoreServer_engine, "f").sql), "f");
103
104
  __classPrivateFieldGet(this, _DurabilitySemaphoreServer_engine, "f").register({
@@ -107,7 +108,7 @@ export class DurabilitySemaphoreServer {
107
108
  });
108
109
  }
109
110
  static migrate(context, target) {
110
- return migrate(context.storage, 'semaphore', durabilitySemaphoreMigrations, target);
111
+ return migrateCapability(context.storage, 'semaphore', durabilitySemaphoreMigrations, target);
111
112
  }
112
113
  alarm(info) {
113
114
  return __classPrivateFieldGet(this, _DurabilitySemaphoreServer_engine, "f").alarm(info);
@@ -148,9 +149,7 @@ export class DurabilitySemaphoreServer {
148
149
  }
149
150
  let leased = 0;
150
151
  for (const lease of leases) {
151
- if (lease.expires_at > now) {
152
- leased += leaseFootprint(lease);
153
- }
152
+ leased += leaseFootprint(lease);
154
153
  }
155
154
  await Promise.all([...groups.entries()].map(async ([name, group]) => {
156
155
  const policy = await __classPrivateFieldGet(this, _DurabilitySemaphoreServer_instances, "m", _DurabilitySemaphoreServer_groupPolicy).call(this, name);
@@ -287,9 +286,7 @@ _DurabilitySemaphoreServer_engine = new WeakMap(), _DurabilitySemaphoreServer_st
287
286
  let leased = 0;
288
287
  if (__classPrivateFieldGet(this, _DurabilitySemaphoreServer_leases, "f") && budget.shared) {
289
288
  for (const lease of store.listLeases()) {
290
- if (lease.expires_at > now) {
291
- leased += leaseFootprint(lease);
292
- }
289
+ leased += leaseFootprint(lease);
293
290
  }
294
291
  }
295
292
  let remaining = budget.grants - total - (budget.shared ? leased : 0);
@@ -518,3 +515,4 @@ async function _DurabilitySemaphoreServer_groupPolicy(group) {
518
515
  return { blocked: 'limit_resolution_error' };
519
516
  }
520
517
  };
518
+ traceMethods(DurabilitySemaphoreServer.prototype, 'DurabilitySemaphore');