durability 4.2.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +340 -12
- package/dist/dev.d.ts +22 -0
- package/dist/dev.js +149 -0
- package/dist/durability-alarms.d.ts +16 -3
- package/dist/durability-alarms.js +21 -7
- package/dist/durability.d.ts +31 -6
- package/dist/durability.js +42 -16
- package/dist/errors.d.ts +31 -0
- package/dist/errors.js +45 -0
- package/dist/experimental.d.ts +5 -0
- package/dist/experimental.js +3 -0
- package/dist/fanout.d.ts +6 -0
- package/dist/fanout.js +20 -9
- package/dist/index.d.ts +4 -2
- package/dist/index.js +3 -2
- package/dist/lifecycle.d.ts +6 -1
- package/dist/lifecycle.js +3 -2
- package/dist/log.js +61 -16
- package/dist/memoize.d.ts +34 -0
- package/dist/memoize.js +136 -0
- package/dist/migrations.d.ts +64 -2
- package/dist/migrations.js +179 -34
- package/dist/outbox.d.ts +13 -0
- package/dist/outbox.js +78 -30
- package/dist/scheduler.d.ts +9 -0
- package/dist/scheduler.js +53 -11
- package/dist/semaphore-leases.js +15 -9
- package/dist/semaphore-queue-store.d.ts +12 -2
- package/dist/semaphore-queue-store.js +32 -5
- package/dist/semaphore-queue.js +109 -43
- package/dist/semaphore-server.js +7 -9
- package/dist/socket.js +19 -10
- package/dist/storage.d.ts +1 -0
- package/dist/storage.js +1 -1
- package/dist/topology-client.d.ts +40 -0
- package/dist/topology-client.js +180 -0
- package/dist/topology-snapshot.d.ts +70 -0
- package/dist/topology-snapshot.js +48 -0
- package/dist/topology.d.ts +94 -0
- package/dist/topology.js +468 -0
- package/dist/tracked-source.d.ts +2 -0
- package/dist/tracker.js +2 -0
- package/package.json +3 -2
package/dist/fanout.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { type DurabilityLifecycleEvent, type LifecycleHook } from './lifecycle.js';
|
|
2
2
|
import { RoutingLoad, type EnqueueResult, type LoadSnapshot } from './load.js';
|
|
3
3
|
import type { MessageWrite } from './stored-message.js';
|
|
4
|
+
import { type DurableMemoize } from './memoize.js';
|
|
4
5
|
import { type ExecutionPolicyOptions } from './policy.js';
|
|
5
6
|
import { type SchedulerAttachment } from './scheduler.js';
|
|
6
7
|
export type FanoutInput<Body = unknown> = {
|
|
@@ -17,6 +18,11 @@ export type DurabilityFanoutMessage<Body = unknown> = {
|
|
|
17
18
|
body: Body;
|
|
18
19
|
attempt: number;
|
|
19
20
|
enqueuedAt: number;
|
|
21
|
+
/**
|
|
22
|
+
* Persists a callback's result by key for this message and target, so a
|
|
23
|
+
* retried delivery reuses it. Deleted when the delivery is acked or dead-lettered.
|
|
24
|
+
*/
|
|
25
|
+
memoize: DurableMemoize;
|
|
20
26
|
ack(): Promise<void>;
|
|
21
27
|
retry(delayMs?: number): Promise<void>;
|
|
22
28
|
deadLetter(): Promise<void>;
|
package/dist/fanout.js
CHANGED
|
@@ -11,10 +11,12 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
|
|
|
11
11
|
};
|
|
12
12
|
var _DurabilityFanout_instances, _DurabilityFanout_engine, _DurabilityFanout_routing, _DurabilityFanout_targets, _DurabilityFanout_policies, _DurabilityFanout_dlq, _DurabilityFanout_maxBatchSize, _DurabilityFanout_batchDelayMs, _DurabilityFanout_lifecycle, _DurabilityFanout_emit, _DurabilityFanout_running, _DurabilityFanout_pending, _DurabilityFanout_firstAttemptAt, _DurabilityFanout_runDue, _DurabilityFanout_runTarget, _DurabilityFanout_postpone, _DurabilityFanout_deliver, _DurabilityFanout_ackUnsettled, _DurabilityFanout_message, _DurabilityFanout_remove, _DurabilityFanout_removeDelivery, _DurabilityFanout_retry, _DurabilityFanout_deadLetter, _DurabilityFanout_emitTerminal, _DurabilityFanout_transaction, _DurabilityFanout_pendingCount, _DurabilityFanout_adjustPending, _DurabilityFanout_listDue, _DurabilityFanout_deadline;
|
|
13
13
|
import { z } from 'zod';
|
|
14
|
+
import { traceFunction, traceMethods } from './dev.js';
|
|
14
15
|
import { DurableRetryPolicyError, FanoutEnqueueError, FanoutSettlementError, FanoutTimeoutError, reportFailure, serializeError, } from './errors.js';
|
|
15
16
|
import { createLifecycleHooks, } from './lifecycle.js';
|
|
16
17
|
import { RoutingLoad } from './load.js';
|
|
17
|
-
import {
|
|
18
|
+
import { createMemoize, deleteMemos, deleteMemosOfDroppedTable, } from './memoize.js';
|
|
19
|
+
import { durabilityFanoutMigrations, durabilityMemoMigrations, migrateCapability, } from './migrations.js';
|
|
18
20
|
import { assertPositiveInteger, createPolicyResolver, methodOptionsSchema, } from './policy.js';
|
|
19
21
|
import { engineFor, } from './scheduler.js';
|
|
20
22
|
import { deserialize, serialize } from './serialization.js';
|
|
@@ -119,13 +121,16 @@ export class DurabilityFanout {
|
|
|
119
121
|
__classPrivateFieldSet(this, _DurabilityFanout_lifecycle, createLifecycleHooks((hook) => __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").emitter(hook), config.onLifecycleEvent), "f");
|
|
120
122
|
__classPrivateFieldSet(this, _DurabilityFanout_emit, __classPrivateFieldGet(this, _DurabilityFanout_lifecycle, "f").emit, "f");
|
|
121
123
|
__classPrivateFieldGet(this, _DurabilityFanout_engine, "f").migrateSchema('fanout', durabilityFanoutMigrations);
|
|
124
|
+
__classPrivateFieldGet(this, _DurabilityFanout_engine, "f").migrateSchema('memos', durabilityMemoMigrations);
|
|
122
125
|
__classPrivateFieldGet(this, _DurabilityFanout_engine, "f").register({
|
|
123
126
|
table: deliveriesTable,
|
|
124
127
|
runDue: () => __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_runDue).call(this),
|
|
125
128
|
});
|
|
126
129
|
}
|
|
127
130
|
static migrate(context, target) {
|
|
128
|
-
|
|
131
|
+
const result = migrateCapability(context.storage, 'fanout', durabilityFanoutMigrations, target);
|
|
132
|
+
deleteMemosOfDroppedTable(context.storage.sql, deliveriesTable);
|
|
133
|
+
return result;
|
|
129
134
|
}
|
|
130
135
|
alarm(info) {
|
|
131
136
|
return __classPrivateFieldGet(this, _DurabilityFanout_engine, "f").alarm(info);
|
|
@@ -501,6 +506,10 @@ _DurabilityFanout_engine = new WeakMap(), _DurabilityFanout_routing = new WeakMa
|
|
|
501
506
|
}
|
|
502
507
|
}, _DurabilityFanout_message = function _DurabilityFanout_message(state, policy, startedAt) {
|
|
503
508
|
const { row, message } = state.claim;
|
|
509
|
+
const identity = {
|
|
510
|
+
deliveryId: deliveryIdentity(row),
|
|
511
|
+
target: row.target_id,
|
|
512
|
+
};
|
|
504
513
|
const settle = (outcome, action) => {
|
|
505
514
|
if (state.closed || state.outcome !== undefined) {
|
|
506
515
|
return Promise.reject(new FanoutSettlementError(message.id, state.outcome));
|
|
@@ -519,12 +528,12 @@ _DurabilityFanout_engine = new WeakMap(), _DurabilityFanout_routing = new WeakMa
|
|
|
519
528
|
};
|
|
520
529
|
return {
|
|
521
530
|
id: message.id,
|
|
522
|
-
|
|
523
|
-
target: row.target_id,
|
|
531
|
+
...identity,
|
|
524
532
|
body: deserialize(message.payload),
|
|
525
533
|
enqueuedAt: message.created_at,
|
|
526
534
|
attempt: row.attempt,
|
|
527
|
-
|
|
535
|
+
memoize: createMemoize(__classPrivateFieldGet(this, _DurabilityFanout_engine, "f").sql, deliveriesTable, attemptOf(row)),
|
|
536
|
+
ack: traceFunction('DurabilityFanoutMessage.ack', () => settle('ack', async () => {
|
|
528
537
|
await __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_remove).call(this, row);
|
|
529
538
|
__classPrivateFieldGet(this, _DurabilityFanout_emit, "f").call(this, {
|
|
530
539
|
...entity(state.claim),
|
|
@@ -534,8 +543,8 @@ _DurabilityFanout_engine = new WeakMap(), _DurabilityFanout_routing = new WeakMa
|
|
|
534
543
|
durationMs: Math.max(0, Date.now() - startedAt),
|
|
535
544
|
outcome: 'completed',
|
|
536
545
|
});
|
|
537
|
-
}),
|
|
538
|
-
retry: (delayMs) => {
|
|
546
|
+
}), identity),
|
|
547
|
+
retry: traceFunction('DurabilityFanoutMessage.retry', (delayMs) => {
|
|
539
548
|
if (delayMs !== undefined &&
|
|
540
549
|
(!Number.isSafeInteger(delayMs) ||
|
|
541
550
|
delayMs < 0 ||
|
|
@@ -543,8 +552,8 @@ _DurabilityFanout_engine = new WeakMap(), _DurabilityFanout_routing = new WeakMa
|
|
|
543
552
|
return Promise.reject(new RangeError('delayMs must produce a non-negative safe-integer timestamp'));
|
|
544
553
|
}
|
|
545
554
|
return settle('retry', () => __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_retry).call(this, state.claim, policy, delayMs, { name: 'Retry', message: 'Retried by consumer' }, startedAt));
|
|
546
|
-
},
|
|
547
|
-
deadLetter: () => settle('deadLetter', () => __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_deadLetter).call(this, state.claim, 'explicit')),
|
|
555
|
+
}, identity),
|
|
556
|
+
deadLetter: traceFunction('DurabilityFanoutMessage.deadLetter', () => settle('deadLetter', () => __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_deadLetter).call(this, state.claim, 'explicit')), identity),
|
|
548
557
|
};
|
|
549
558
|
}, _DurabilityFanout_remove = async function _DurabilityFanout_remove(row) {
|
|
550
559
|
await __classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_transaction).call(this, (sql) => {
|
|
@@ -556,6 +565,7 @@ _DurabilityFanout_engine = new WeakMap(), _DurabilityFanout_routing = new WeakMa
|
|
|
556
565
|
if (!deleteAttempt(sql, deliveriesTable, attemptOf(row))) {
|
|
557
566
|
return false;
|
|
558
567
|
}
|
|
568
|
+
deleteMemos(sql, deliveriesTable, row.id);
|
|
559
569
|
__classPrivateFieldGet(this, _DurabilityFanout_instances, "m", _DurabilityFanout_adjustPending).call(this, -1);
|
|
560
570
|
sql.exec(`UPDATE durability_fanout_messages
|
|
561
571
|
SET remaining = remaining - 1
|
|
@@ -706,3 +716,4 @@ _DurabilityFanout_engine = new WeakMap(), _DurabilityFanout_routing = new WeakMa
|
|
|
706
716
|
}
|
|
707
717
|
}
|
|
708
718
|
};
|
|
719
|
+
traceMethods(DurabilityFanout.prototype, 'DurabilityFanout');
|
package/dist/index.d.ts
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
export { Durability, type DurabilityConfig, type DurabilityConstructor, type DurabilityOptions, type DurableCall, type DurableHandler, type DurableOperation, type DurableOperationResult, type DurabilityOperationEvent, type HandlerMap, } from './durability.js';
|
|
2
2
|
export { DurabilityAlarms, type DurabilityAlarmsConfig, type DurabilityAlarmsConstructor, type DurabilityAlarmsOptions, type DurabilityAlarmEvent, type DurableAlarmHandler, type DurableAlarmInfo, type DurableNamedAlarm, } from './durability-alarms.js';
|
|
3
|
+
export { maxMemoBytes, maxMemosPerRecord, type DurableMemoize, } from './memoize.js';
|
|
3
4
|
export { DurabilityFanout, type DurabilityFanoutConfig, type DurabilityFanoutEvent, type DurabilityFanoutMessage, type FanoutInput, type FanoutTarget, type FanoutEnqueueOptions, } from './fanout.js';
|
|
4
5
|
export { DurabilityLog, maxLogBytes, type DurabilityLogConfig, type LogAppendInput, type LogColdStorage, type LogLease, type LogLeaseOptions, type LogPage, type LogReadOptions, type LogRecord, type LogRetention, type LogSegment, type LogBounds, } from './log.js';
|
|
5
6
|
export { RoutingLoad, type EnqueueResult, type LoadSnapshot, type ProcessingLoad, } from './load.js';
|
|
6
7
|
export type { MessageWrite, StoredMessage } from './stored-message.js';
|
|
7
8
|
export { DurabilityScheduler, type DurabilityContext, type DurabilitySchedulerConfig, type DurabilitySchedulerOptions, type SchedulerAttachment, } from './scheduler.js';
|
|
8
|
-
export { DuplicateDurableCallError, DuplicateDurablePayloadError, DurableAlarmTimeoutError, DurableAttemptsExhaustedError, DurableAttemptTimeoutError, DurableResultSerializationError, DurableRetryPolicyError, NonRetryableError, FanoutTimeoutError, FanoutSettlementError, FanoutEnqueueError, LogAppendError, LogLeaseError, LogTruncatedError, RoutingError, } from './errors.js';
|
|
9
|
+
export { DuplicateDurableCallError, DuplicateDurablePayloadError, DurableAlarmTimeoutError, DurableAttemptsExhaustedError, DurableAttemptTimeoutError, DurableResultSerializationError, DurableRetryPolicyError, DurableMemoLimitError, DurableMemoSerializationError, NonRetryableError, FanoutTimeoutError, FanoutSettlementError, FanoutEnqueueError, LogAppendError, LogLeaseError, LogTruncatedError, RoutingError, } from './errors.js';
|
|
9
10
|
export type { DurabilityLifecycleEvent, LifecycleHook } from './lifecycle.js';
|
|
10
|
-
export { durabilityNamedAlarmMigrations, durabilityOperationMigrations, durabilityFanoutMigrations, durabilityLogLeaseMigrations, durabilityLogMigrations, type DurabilityMigrationResult, type MigrationCapability, } from './migrations.js';
|
|
11
|
+
export { durabilityNamedAlarmMigrations, durabilityOperationMigrations, durabilityFanoutMigrations, durabilityLogLeaseMigrations, durabilityLogMigrations, durabilityMemoMigrations, migrate, type DurabilityMigrationResult, type MigrateOptions, type MigrationCapability, } from './migrations.js';
|
|
12
|
+
export type { DurableMigrations } from '@durability/storage';
|
|
11
13
|
export type { DurabilityAlarmMethodOptions, DurabilityMethodOptions, DurabilityRetryOptions, } from './policy.js';
|
package/dist/index.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
export { Durability, } from './durability.js';
|
|
2
2
|
export { DurabilityAlarms, } from './durability-alarms.js';
|
|
3
|
+
export { maxMemoBytes, maxMemosPerRecord, } from './memoize.js';
|
|
3
4
|
export { DurabilityFanout, } from './fanout.js';
|
|
4
5
|
export { DurabilityLog, maxLogBytes, } from './log.js';
|
|
5
6
|
export { RoutingLoad, } from './load.js';
|
|
6
7
|
export { DurabilityScheduler, } from './scheduler.js';
|
|
7
|
-
export { DuplicateDurableCallError, DuplicateDurablePayloadError, DurableAlarmTimeoutError, DurableAttemptsExhaustedError, DurableAttemptTimeoutError, DurableResultSerializationError, DurableRetryPolicyError, NonRetryableError, FanoutTimeoutError, FanoutSettlementError, FanoutEnqueueError, LogAppendError, LogLeaseError, LogTruncatedError, RoutingError, } from './errors.js';
|
|
8
|
-
export { durabilityNamedAlarmMigrations, durabilityOperationMigrations, durabilityFanoutMigrations, durabilityLogLeaseMigrations, durabilityLogMigrations, } from './migrations.js';
|
|
8
|
+
export { DuplicateDurableCallError, DuplicateDurablePayloadError, DurableAlarmTimeoutError, DurableAttemptsExhaustedError, DurableAttemptTimeoutError, DurableResultSerializationError, DurableRetryPolicyError, DurableMemoLimitError, DurableMemoSerializationError, NonRetryableError, FanoutTimeoutError, FanoutSettlementError, FanoutEnqueueError, LogAppendError, LogLeaseError, LogTruncatedError, RoutingError, } from './errors.js';
|
|
9
|
+
export { durabilityNamedAlarmMigrations, durabilityOperationMigrations, durabilityFanoutMigrations, durabilityLogLeaseMigrations, durabilityLogMigrations, durabilityMemoMigrations, migrate, } from './migrations.js';
|
package/dist/lifecycle.d.ts
CHANGED
|
@@ -48,7 +48,12 @@ export type DurabilityLifecycleEvent = ((OperationLifecycleEntity | FanoutLifecy
|
|
|
48
48
|
attempt: number;
|
|
49
49
|
}) | (LifecycleEntity & AttemptSettled & {
|
|
50
50
|
outcome: 'completed';
|
|
51
|
-
/**
|
|
51
|
+
/**
|
|
52
|
+
* The handler's return value. Omitted by default to keep result
|
|
53
|
+
* content out of telemetry; present only when the emitting instance
|
|
54
|
+
* opts in (e.g. Durability's `includeResults`). Always absent for
|
|
55
|
+
* fanout acknowledgements.
|
|
56
|
+
*/
|
|
52
57
|
result?: unknown;
|
|
53
58
|
}) | (LifecycleEntity & AttemptSettled & {
|
|
54
59
|
outcome: 'retry_scheduled';
|
package/dist/lifecycle.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { traceLifecycle } from './dev.js';
|
|
1
2
|
import { reportFailure } from './errors.js';
|
|
2
3
|
/** Fans one helper's lifecycle events out to every attached hook. */
|
|
3
4
|
export const createLifecycleHooks = (emitter, initial) => {
|
|
@@ -13,11 +14,11 @@ export const createLifecycleHooks = (emitter, initial) => {
|
|
|
13
14
|
add(initial);
|
|
14
15
|
}
|
|
15
16
|
return {
|
|
16
|
-
emit: (event) => {
|
|
17
|
+
emit: traceLifecycle((event) => {
|
|
17
18
|
for (const emit of emits) {
|
|
18
19
|
emit(event);
|
|
19
20
|
}
|
|
20
|
-
},
|
|
21
|
+
}),
|
|
21
22
|
add,
|
|
22
23
|
};
|
|
23
24
|
};
|
package/dist/log.js
CHANGED
|
@@ -9,11 +9,12 @@ var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (
|
|
|
9
9
|
if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
|
|
10
10
|
return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
|
|
11
11
|
};
|
|
12
|
-
var _DurabilityLog_instances, _DurabilityLog_engine, _DurabilityLog_routing, _DurabilityLog_retention, _DurabilityLog_cold, _DurabilityLog_leases, _DurabilityLog_maxParallelism, _DurabilityLog_leaseMs, _DurabilityLog_maxBatchSize, _DurabilityLog_encoder, _DurabilityLog_rehydrate, _DurabilityLog_record, _DurabilityLog_settle, _DurabilityLog_advance, _DurabilityLog_between, _DurabilityLog_readFrom, _DurabilityLog_requireLeases;
|
|
12
|
+
var _DurabilityLog_instances, _DurabilityLog_engine, _DurabilityLog_routing, _DurabilityLog_retention, _DurabilityLog_cold, _DurabilityLog_leases, _DurabilityLog_maxParallelism, _DurabilityLog_leaseMs, _DurabilityLog_maxBatchSize, _DurabilityLog_encoder, _DurabilityLog_rehydrate, _DurabilityLog_record, _DurabilityLog_settle, _DurabilityLog_dropTrimmed, _DurabilityLog_advance, _DurabilityLog_between, _DurabilityLog_readFrom, _DurabilityLog_requireLeases;
|
|
13
13
|
import { z } from 'zod';
|
|
14
|
+
import { traceMethods } from './dev.js';
|
|
14
15
|
import { LogAppendError, LogLeaseError, LogTruncatedError, reportFailure, serializeError, } from './errors.js';
|
|
15
16
|
import { RoutingLoad } from './load.js';
|
|
16
|
-
import { durabilityLogLeaseMigrations, durabilityLogMigrations,
|
|
17
|
+
import { durabilityLogLeaseMigrations, durabilityLogMigrations, migrateCapability, } from './migrations.js';
|
|
17
18
|
import { assertPositiveInteger } from './policy.js';
|
|
18
19
|
import { engineFor, } from './scheduler.js';
|
|
19
20
|
import { deserialize, serialize } from './serialization.js';
|
|
@@ -251,7 +252,7 @@ const settleLease = (sql, batchId, state, expiresAt) => sql
|
|
|
251
252
|
*/
|
|
252
253
|
const leasesFrom = (sql, consumer, from, limit) => logLeaseRowSchema.array().parse(sql
|
|
253
254
|
.exec(`SELECT * FROM durability_log_leases
|
|
254
|
-
WHERE consumer = ? AND
|
|
255
|
+
WHERE consumer = ? AND last_offset >= ?
|
|
255
256
|
ORDER BY first_offset ASC
|
|
256
257
|
LIMIT ?`, consumer, from, limit)
|
|
257
258
|
.toArray());
|
|
@@ -321,7 +322,7 @@ export class DurabilityLog {
|
|
|
321
322
|
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").migrateSchema('log', durabilityLogMigrations);
|
|
322
323
|
}
|
|
323
324
|
static migrate(context, target) {
|
|
324
|
-
return
|
|
325
|
+
return migrateCapability(context.storage, 'log', durabilityLogMigrations, target);
|
|
325
326
|
}
|
|
326
327
|
/**
|
|
327
328
|
* Appends the batch atomically. Definite input rejections return
|
|
@@ -518,6 +519,11 @@ export class DurabilityLog {
|
|
|
518
519
|
const sql = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql;
|
|
519
520
|
const reclaimed = claimExpired(sql, consumer, now, crypto.randomUUID(), now + __classPrivateFieldGet(this, _DurabilityLog_leaseMs, "f"));
|
|
520
521
|
if (reclaimed) {
|
|
522
|
+
const { oldestOffset } = readBounds(sql);
|
|
523
|
+
if (reclaimed.first_offset < oldestOffset) {
|
|
524
|
+
__classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_dropTrimmed).call(this, consumer, reclaimed, oldestOffset);
|
|
525
|
+
throw new LogTruncatedError(reclaimed.first_offset, oldestOffset);
|
|
526
|
+
}
|
|
521
527
|
const poisoned = leases.maxAttempts !== undefined &&
|
|
522
528
|
leases.onPoison !== undefined &&
|
|
523
529
|
reclaimed.attempt > leases.maxAttempts;
|
|
@@ -535,7 +541,11 @@ export class DurabilityLog {
|
|
|
535
541
|
}
|
|
536
542
|
// Terminal intent is durable before the callback runs, so a failing
|
|
537
543
|
// handler cannot resurrect a range that already exhausted its attempts.
|
|
538
|
-
|
|
544
|
+
// A skipped range counts as settled, so the cursor moves past it as for an ack.
|
|
545
|
+
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync(() => {
|
|
546
|
+
__classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_settle).call(this, reclaimed.batch_id, 'skipped');
|
|
547
|
+
__classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_advance).call(this, consumer);
|
|
548
|
+
});
|
|
539
549
|
try {
|
|
540
550
|
await leases.onPoison(lease);
|
|
541
551
|
}
|
|
@@ -651,20 +661,41 @@ async function _DurabilityLog_rehydrate(from, limit, bounds) {
|
|
|
651
661
|
// Either the batch never existed, was fenced by a redelivery, or was
|
|
652
662
|
// already settled; all three are the caller holding a stale handle.
|
|
653
663
|
throw new LogLeaseError(batchId);
|
|
664
|
+
}, _DurabilityLog_dropTrimmed = function _DurabilityLog_dropTrimmed(consumer, lease, oldestOffset) {
|
|
665
|
+
__classPrivateFieldGet(this, _DurabilityLog_engine, "f").transactionSync((sql) => {
|
|
666
|
+
if (lease.last_offset < oldestOffset) {
|
|
667
|
+
deleteLeases(sql, [lease.batch_id]);
|
|
668
|
+
}
|
|
669
|
+
else {
|
|
670
|
+
sql.exec(`UPDATE durability_log_leases
|
|
671
|
+
SET first_offset = ?, state = 'pending', attempt = attempt - 1,
|
|
672
|
+
expires_at = 0
|
|
673
|
+
WHERE batch_id = ?`, oldestOffset, lease.batch_id);
|
|
674
|
+
}
|
|
675
|
+
__classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_advance).call(this, consumer);
|
|
676
|
+
});
|
|
654
677
|
}, _DurabilityLog_advance = function _DurabilityLog_advance(consumer) {
|
|
655
678
|
const sql = __classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql;
|
|
656
|
-
|
|
657
|
-
|
|
679
|
+
// Offsets below the retained window are gone for everyone; a cursor
|
|
680
|
+
// stranded there could never meet the ranges leased after them.
|
|
681
|
+
const from = Math.max(readCursor(sql, consumer) ?? 0, readBounds(sql).oldestOffset);
|
|
658
682
|
let cursor = from;
|
|
659
683
|
const settled = [];
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
684
|
+
// Ranges settled while an earlier one was outstanding wait behind it; when
|
|
685
|
+
// it settles the cursor must pass all of them, however many pages that is.
|
|
686
|
+
for (let page = true; page;) {
|
|
687
|
+
const candidates = leasesFrom(sql, consumer, cursor, __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f"));
|
|
688
|
+
page = candidates.length === __classPrivateFieldGet(this, _DurabilityLog_maxBatchSize, "f");
|
|
689
|
+
for (const row of candidates) {
|
|
690
|
+
const contiguous = row.first_offset <= cursor;
|
|
691
|
+
const done = row.state === 'acked' || row.state === 'skipped';
|
|
692
|
+
if (!contiguous || !done) {
|
|
693
|
+
page = false;
|
|
694
|
+
break;
|
|
695
|
+
}
|
|
696
|
+
cursor = Math.max(cursor, row.last_offset + 1);
|
|
697
|
+
settled.push(row.batch_id);
|
|
665
698
|
}
|
|
666
|
-
cursor = Math.max(cursor, row.last_offset + 1);
|
|
667
|
-
settled.push(row.batch_id);
|
|
668
699
|
}
|
|
669
700
|
if (settled.length === 0) {
|
|
670
701
|
return;
|
|
@@ -675,8 +706,21 @@ async function _DurabilityLog_rehydrate(from, limit, bounds) {
|
|
|
675
706
|
/** Reads a stored range, rehydrating it when retention already flushed it. */
|
|
676
707
|
async function _DurabilityLog_between(first, last) {
|
|
677
708
|
const bounds = readBounds(__classPrivateFieldGet(this, _DurabilityLog_engine, "f").sql);
|
|
678
|
-
|
|
679
|
-
|
|
709
|
+
// One read stops at the end of a cold segment, and a range redelivered
|
|
710
|
+
// after a flush can span several segments and the hot window.
|
|
711
|
+
const records = [];
|
|
712
|
+
let from = first;
|
|
713
|
+
while (from <= last) {
|
|
714
|
+
// eslint-disable-next-line no-await-in-loop
|
|
715
|
+
const page = await __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_readFrom).call(this, from, last - from + 1, bounds);
|
|
716
|
+
const inRange = page.filter((record) => record.offset <= last);
|
|
717
|
+
if (inRange.length === 0) {
|
|
718
|
+
break;
|
|
719
|
+
}
|
|
720
|
+
records.push(...inRange);
|
|
721
|
+
from = inRange[inRange.length - 1].offset + 1;
|
|
722
|
+
}
|
|
723
|
+
return records;
|
|
680
724
|
}, _DurabilityLog_readFrom = async function _DurabilityLog_readFrom(from, limit, bounds) {
|
|
681
725
|
if (from < bounds.hotOffset && from < bounds.nextOffset) {
|
|
682
726
|
return __classPrivateFieldGet(this, _DurabilityLog_instances, "m", _DurabilityLog_rehydrate).call(this, from, limit, bounds);
|
|
@@ -688,6 +732,7 @@ async function _DurabilityLog_between(first, last) {
|
|
|
688
732
|
}
|
|
689
733
|
return __classPrivateFieldGet(this, _DurabilityLog_leases, "f");
|
|
690
734
|
};
|
|
735
|
+
traceMethods(DurabilityLog.prototype, 'DurabilityLog');
|
|
691
736
|
const assertConsumer = (consumer) => {
|
|
692
737
|
if (typeof consumer !== 'string' || consumer.length === 0) {
|
|
693
738
|
throw new LogAppendError('Consumer must be a non-empty string');
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { type AttemptRef, type DurableTable } from './storage.js';
|
|
2
|
+
/**
|
|
3
|
+
* Runs `callback` once per key for a durable operation or named alarm
|
|
4
|
+
* occurrence and persists its result, so a retried attempt returns the result
|
|
5
|
+
* instead of running the callback again. A callback that throws persists
|
|
6
|
+
* nothing and runs again on the next call with that key.
|
|
7
|
+
*
|
|
8
|
+
* Calls with the same key return the same result, within one attempt and
|
|
9
|
+
* across attempts. A call for a key whose callback is still running rejects,
|
|
10
|
+
* including one made from that callback. Results are deleted when the
|
|
11
|
+
* operation, alarm occurrence, or fanout delivery settles.
|
|
12
|
+
*
|
|
13
|
+
* At most {@link maxMemosPerRecord} keys and {@link maxMemoBytes} per result;
|
|
14
|
+
* exceeding either fails the work with `DurableMemoLimitError`.
|
|
15
|
+
*/
|
|
16
|
+
export type DurableMemoize = <T>(key: string, callback: () => T | Promise<T>) => Promise<T>;
|
|
17
|
+
/** The most keys one operation, alarm occurrence, or fanout delivery may memoize. */
|
|
18
|
+
export declare const maxMemosPerRecord = 1000;
|
|
19
|
+
/** The largest serialized memoized result, in UTF-8 bytes. */
|
|
20
|
+
export declare const maxMemoBytes = 1048576;
|
|
21
|
+
/** Memoization for one attempt; only that attempt may persist new results. */
|
|
22
|
+
export declare const createMemoize: (sql: SqlStorage, table: DurableTable, ref: AttemptRef) => DurableMemoize;
|
|
23
|
+
/** Keys a record's current generation has results for, in the order they were saved. */
|
|
24
|
+
export declare const memoizedKeys: (sql: SqlStorage, table: DurableTable, key: string, generation: string) => string[];
|
|
25
|
+
/** Deletes every memoized result of a record, whichever generation wrote it. */
|
|
26
|
+
export declare const deleteMemos: (sql: SqlStorage, table: DurableTable, key: string) => void;
|
|
27
|
+
/** Deletes memoized results whose record generation is no longer pending. */
|
|
28
|
+
export declare const deleteStaleMemos: (sql: SqlStorage, table: DurableTable) => void;
|
|
29
|
+
/**
|
|
30
|
+
* Deletes the memoized results of a helper whose schema was reverted. The
|
|
31
|
+
* shared table stays for the other helper, but the removed records' results
|
|
32
|
+
* must not.
|
|
33
|
+
*/
|
|
34
|
+
export declare const deleteMemosOfDroppedTable: (sql: SqlStorage, table: DurableTable) => void;
|
package/dist/memoize.js
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { traceFunction } from './dev.js';
|
|
3
|
+
import { DurableMemoLimitError, DurableMemoSerializationError, } from './errors.js';
|
|
4
|
+
import { deserialize, serialize } from './serialization.js';
|
|
5
|
+
import { pendingGeneration, } from './storage.js';
|
|
6
|
+
/** The most keys one operation, alarm occurrence, or fanout delivery may memoize. */
|
|
7
|
+
export const maxMemosPerRecord = 1000;
|
|
8
|
+
/** The largest serialized memoized result, in UTF-8 bytes. */
|
|
9
|
+
export const maxMemoBytes = 1048576;
|
|
10
|
+
const memoLookupSchema = z.object({
|
|
11
|
+
result: z.string().nullable(),
|
|
12
|
+
owned: z.number().int(),
|
|
13
|
+
saved: z.number().int(),
|
|
14
|
+
});
|
|
15
|
+
const supersededError = (key) => new Error(`Memoized key ${JSON.stringify(key)} belongs to a superseded attempt`);
|
|
16
|
+
/** Memoization for one attempt; only that attempt may persist new results. */
|
|
17
|
+
export const createMemoize = (sql, table, ref) => {
|
|
18
|
+
const running = new Set();
|
|
19
|
+
/** Serializes, checks, and writes a callback's result, then returns its persisted form. */
|
|
20
|
+
const save = (key, value) => {
|
|
21
|
+
let serialized;
|
|
22
|
+
try {
|
|
23
|
+
serialized = serialize(value);
|
|
24
|
+
}
|
|
25
|
+
catch (error) {
|
|
26
|
+
throw new DurableMemoSerializationError(key, error);
|
|
27
|
+
}
|
|
28
|
+
const bytes = new TextEncoder().encode(serialized).byteLength;
|
|
29
|
+
if (bytes > maxMemoBytes) {
|
|
30
|
+
throw new DurableMemoLimitError(`Memoized result for ${JSON.stringify(key)} is ${bytes} bytes, over the ${maxMemoBytes}-byte limit`);
|
|
31
|
+
}
|
|
32
|
+
// A timed-out attempt that finishes late must not write over its successor.
|
|
33
|
+
const written = sql
|
|
34
|
+
.exec(`INSERT INTO durability_memos
|
|
35
|
+
(owner, key, generation_id, memo_key, result)
|
|
36
|
+
SELECT ?, ?, ?, ?, ?
|
|
37
|
+
WHERE EXISTS (
|
|
38
|
+
SELECT 1 FROM ${table.name}
|
|
39
|
+
WHERE ${pendingGeneration(table)} AND attempt = ?
|
|
40
|
+
)
|
|
41
|
+
RETURNING memo_key`, table.name, ref.key, ref.generation, key, serialized, ref.key, ref.generation, ref.attempt)
|
|
42
|
+
.toArray().length > 0;
|
|
43
|
+
if (!written) {
|
|
44
|
+
throw supersededError(key);
|
|
45
|
+
}
|
|
46
|
+
// Return what a replay would return, so later code sees one shape either way.
|
|
47
|
+
return deserialize(serialized);
|
|
48
|
+
};
|
|
49
|
+
// Everything up to the callback runs synchronously, so a concurrent call
|
|
50
|
+
// with the same key sees either the saved result or this key as running.
|
|
51
|
+
const run = async (key, callback) => {
|
|
52
|
+
if (typeof key !== 'string' || key === '') {
|
|
53
|
+
throw new TypeError('Memoize keys must be non-empty strings');
|
|
54
|
+
}
|
|
55
|
+
const lookup = memoLookupSchema.parse(sql
|
|
56
|
+
.exec(`SELECT
|
|
57
|
+
(SELECT result FROM durability_memos
|
|
58
|
+
WHERE owner = ? AND key = ? AND generation_id = ? AND memo_key = ?)
|
|
59
|
+
AS result,
|
|
60
|
+
EXISTS (
|
|
61
|
+
SELECT 1 FROM ${table.name}
|
|
62
|
+
WHERE ${pendingGeneration(table)} AND attempt = ?
|
|
63
|
+
) AS owned,
|
|
64
|
+
(SELECT COUNT(*) FROM durability_memos
|
|
65
|
+
WHERE owner = ? AND key = ? AND generation_id = ?) AS saved`, table.name, ref.key, ref.generation, key, ref.key, ref.generation, ref.attempt, table.name, ref.key, ref.generation)
|
|
66
|
+
.toArray()[0]);
|
|
67
|
+
if (lookup.result !== null) {
|
|
68
|
+
return deserialize(lookup.result);
|
|
69
|
+
}
|
|
70
|
+
// A rescheduled or purged attempt must not start side effects it can never record.
|
|
71
|
+
if (lookup.owned === 0) {
|
|
72
|
+
throw supersededError(key);
|
|
73
|
+
}
|
|
74
|
+
// Waiting on the running call would deadlock a callback that memoizes its own key.
|
|
75
|
+
if (running.has(key)) {
|
|
76
|
+
throw new Error(`Memoized key ${JSON.stringify(key)} is already running; await the first call instead`);
|
|
77
|
+
}
|
|
78
|
+
// Running callbacks count too: each may save a result when it settles.
|
|
79
|
+
if (lookup.saved + running.size >= maxMemosPerRecord) {
|
|
80
|
+
throw new DurableMemoLimitError(`Memoizing ${JSON.stringify(key)} would exceed ${maxMemosPerRecord} keys`);
|
|
81
|
+
}
|
|
82
|
+
// The key stays running through serialization, so a getter that memoizes
|
|
83
|
+
// it again is rejected instead of running the callback a second time.
|
|
84
|
+
running.add(key);
|
|
85
|
+
try {
|
|
86
|
+
return save(key, await callback());
|
|
87
|
+
}
|
|
88
|
+
finally {
|
|
89
|
+
running.delete(key);
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
return (key, callback) => traceFunction('memoize', () => run(key, callback), {
|
|
93
|
+
owner: table.name,
|
|
94
|
+
key: ref.key,
|
|
95
|
+
attempt: ref.attempt,
|
|
96
|
+
memoKey: key,
|
|
97
|
+
})();
|
|
98
|
+
};
|
|
99
|
+
const memoKeySchema = z.object({ memo_key: z.string() });
|
|
100
|
+
/** Keys a record's current generation has results for, in the order they were saved. */
|
|
101
|
+
export const memoizedKeys = (sql, table, key, generation) => memoKeySchema
|
|
102
|
+
.array()
|
|
103
|
+
.parse(sql
|
|
104
|
+
.exec(`SELECT memo_key FROM durability_memos
|
|
105
|
+
WHERE owner = ? AND key = ? AND generation_id = ?
|
|
106
|
+
ORDER BY rowid`, table.name, key, generation)
|
|
107
|
+
.toArray())
|
|
108
|
+
.map((row) => row.memo_key);
|
|
109
|
+
/** Deletes every memoized result of a record, whichever generation wrote it. */
|
|
110
|
+
export const deleteMemos = (sql, table, key) => {
|
|
111
|
+
sql.exec('DELETE FROM durability_memos WHERE owner = ? AND key = ?', table.name, key);
|
|
112
|
+
};
|
|
113
|
+
/** Deletes memoized results whose record generation is no longer pending. */
|
|
114
|
+
export const deleteStaleMemos = (sql, table) => {
|
|
115
|
+
sql.exec(`DELETE FROM durability_memos
|
|
116
|
+
WHERE owner = ? AND NOT EXISTS (
|
|
117
|
+
SELECT 1 FROM ${table.name} AS record
|
|
118
|
+
WHERE record.${table.key} = durability_memos.key
|
|
119
|
+
AND record.generation_id = durability_memos.generation_id
|
|
120
|
+
AND record.status = 'pending'
|
|
121
|
+
)`, table.name);
|
|
122
|
+
};
|
|
123
|
+
const tableExists = (sql, name) => sql
|
|
124
|
+
.exec(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?`, name)
|
|
125
|
+
.toArray().length > 0;
|
|
126
|
+
/**
|
|
127
|
+
* Deletes the memoized results of a helper whose schema was reverted. The
|
|
128
|
+
* shared table stays for the other helper, but the removed records' results
|
|
129
|
+
* must not.
|
|
130
|
+
*/
|
|
131
|
+
export const deleteMemosOfDroppedTable = (sql, table) => {
|
|
132
|
+
if (!tableExists(sql, 'durability_memos') || tableExists(sql, table.name)) {
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
sql.exec('DELETE FROM durability_memos WHERE owner = ?', table.name);
|
|
136
|
+
};
|
package/dist/migrations.d.ts
CHANGED
|
@@ -19,6 +19,16 @@ export declare const durabilityNamedAlarmMigrations: {
|
|
|
19
19
|
up: string;
|
|
20
20
|
down: string;
|
|
21
21
|
}[];
|
|
22
|
+
/**
|
|
23
|
+
* Ordered SQLite migrations for `durability_memos`, the memoized results that
|
|
24
|
+
* {@link Durability} and {@link DurabilityAlarms} share. Applied by whichever
|
|
25
|
+
* helper is constructed first.
|
|
26
|
+
*/
|
|
27
|
+
export declare const durabilityMemoMigrations: {
|
|
28
|
+
name: string;
|
|
29
|
+
up: string;
|
|
30
|
+
down: string;
|
|
31
|
+
}[];
|
|
22
32
|
/**
|
|
23
33
|
* Ordered SQLite migrations for fanout manifests and deliveries. Applied
|
|
24
34
|
* automatically on construction.
|
|
@@ -82,6 +92,16 @@ export declare const durabilitySemaphoreQueueMigrations: {
|
|
|
82
92
|
up: string;
|
|
83
93
|
down: string;
|
|
84
94
|
}[];
|
|
95
|
+
/**
|
|
96
|
+
* Ordered SQLite migrations for the experimental topology coordinator.
|
|
97
|
+
* Publication and sync ride on outbox channels. Applied automatically on
|
|
98
|
+
* construction.
|
|
99
|
+
*/
|
|
100
|
+
export declare const durabilityTopologyMigrations: {
|
|
101
|
+
name: string;
|
|
102
|
+
up: string;
|
|
103
|
+
down: string;
|
|
104
|
+
}[];
|
|
85
105
|
/** Migration names changed by one migrate call. */
|
|
86
106
|
export type DurabilityMigrationResult = {
|
|
87
107
|
/** Migrations applied in ascending order. */
|
|
@@ -89,8 +109,15 @@ export type DurabilityMigrationResult = {
|
|
|
89
109
|
/** Migrations reverted in descending order. */
|
|
90
110
|
rolledBack: string[];
|
|
91
111
|
};
|
|
92
|
-
export type MigrationCapability = 'operations' | 'namedAlarms' | 'fanout' | 'socket' | 'log' | 'logLeases' | 'outbox' | 'semaphore' | 'semaphoreQueue';
|
|
112
|
+
export type MigrationCapability = 'operations' | 'namedAlarms' | 'memos' | 'fanout' | 'socket' | 'log' | 'logLeases' | 'outbox' | 'semaphore' | 'semaphoreQueue' | 'topology';
|
|
93
113
|
type MigrationStorage = Pick<DurableObjectStorage, 'sql' | 'transactionSync'>;
|
|
114
|
+
/**
|
|
115
|
+
* Whether a capability's history already lists exactly these migrations with
|
|
116
|
+
* these `down` scripts, so construction can skip {@link migrateCapability}. Every Durable
|
|
117
|
+
* Object wake constructs its helpers, and each statement there is one more way
|
|
118
|
+
* for the wake to fail, so the common case costs one read.
|
|
119
|
+
*/
|
|
120
|
+
export declare const isSchemaCurrent: (sql: SqlStorage, capability: MigrationCapability, migrations: DurableMigrations) => boolean;
|
|
94
121
|
/**
|
|
95
122
|
* Moves one capability's schema to a target migration, applying or reverting as
|
|
96
123
|
* needed. `null` reverts every migration in the list; the shared migration
|
|
@@ -100,5 +127,40 @@ type MigrationStorage = Pick<DurableObjectStorage, 'sql' | 'transactionSync'>;
|
|
|
100
127
|
* downgrade of the library can still revert it. Batches of migrations apply or
|
|
101
128
|
* revert atomically.
|
|
102
129
|
*/
|
|
103
|
-
export declare const
|
|
130
|
+
export declare const migrateCapability: (storage: MigrationStorage, capability: MigrationCapability, migrations: DurableMigrations, target: string | null | undefined) => DurabilityMigrationResult;
|
|
131
|
+
/** Options for {@link migrate}. */
|
|
132
|
+
export type MigrateOptions = {
|
|
133
|
+
/** History table for these migrations. Defaults to `schema_migrations`. */
|
|
134
|
+
table?: string;
|
|
135
|
+
/**
|
|
136
|
+
* Migration to move the schema to. Defaults to the last one; `null` reverts
|
|
137
|
+
* every migration.
|
|
138
|
+
*/
|
|
139
|
+
target?: string | null;
|
|
140
|
+
};
|
|
141
|
+
/**
|
|
142
|
+
* Moves an application's own schema to a target migration, with the same
|
|
143
|
+
* guarantees durability gives its tables: each batch applies or reverts in one
|
|
144
|
+
* synchronous transaction, every history row stores its `down` script, and a
|
|
145
|
+
* Durable Object running older code reverts up to two migrations it does not
|
|
146
|
+
* know before refusing.
|
|
147
|
+
*
|
|
148
|
+
* The history lives in its own table, apart from `durability_migrations`, and
|
|
149
|
+
* is dropped once no migration remains applied.
|
|
150
|
+
*
|
|
151
|
+
* @example
|
|
152
|
+
* ```ts
|
|
153
|
+
* constructor(ctx: DurableObjectState, env: Env) {
|
|
154
|
+
* super(ctx, env);
|
|
155
|
+
* migrate(ctx.storage, [
|
|
156
|
+
* {
|
|
157
|
+
* name: '0001_create_jobs',
|
|
158
|
+
* up: 'CREATE TABLE jobs (id TEXT PRIMARY KEY);',
|
|
159
|
+
* down: 'DROP TABLE jobs;',
|
|
160
|
+
* },
|
|
161
|
+
* ]);
|
|
162
|
+
* }
|
|
163
|
+
* ```
|
|
164
|
+
*/
|
|
165
|
+
export declare const migrate: (storage: MigrationStorage, migrations: DurableMigrations, options?: MigrateOptions) => DurabilityMigrationResult;
|
|
104
166
|
export {};
|