@statewalker/cellb 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +323 -2
- package/dist/adapters/files-backup.d.ts +16 -0
- package/dist/adapters/files-backup.js +32 -0
- package/dist/adapters/files-backup.js.map +1 -0
- package/dist/adapters/files-changelog.d.ts +33 -0
- package/dist/adapters/files-changelog.js +96 -0
- package/dist/adapters/files-changelog.js.map +1 -0
- package/dist/adapters/files-ltx.d.ts +35 -0
- package/dist/adapters/files-ltx.js +9 -0
- package/dist/adapters/files-ltx.js.map +1 -0
- package/dist/adapters/ltx-vfs.d.ts +21 -0
- package/dist/adapters/ltx-vfs.js +286 -0
- package/dist/adapters/ltx-vfs.js.map +1 -0
- package/dist/adapters/node-sqlite.d.ts +45 -0
- package/dist/adapters/node-sqlite.js +254 -0
- package/dist/adapters/node-sqlite.js.map +1 -0
- package/dist/adapters/wasm-page-hook.d.ts +26 -0
- package/dist/adapters/wasm-page-hook.js +55 -0
- package/dist/adapters/wasm-page-hook.js.map +1 -0
- package/dist/adapters/wasm-session.d.ts +11 -0
- package/dist/adapters/wasm-session.js +76 -0
- package/dist/adapters/wasm-session.js.map +1 -0
- package/dist/adapters/wasm-sqlite.d.ts +89 -0
- package/dist/adapters/wasm-sqlite.js +258 -0
- package/dist/adapters/wasm-sqlite.js.map +1 -0
- package/dist/bindings/assets.d.ts +26 -0
- package/dist/bindings/assets.js +260 -0
- package/dist/bindings/assets.js.map +1 -0
- package/dist/bindings/cache.d.ts +16 -0
- package/dist/bindings/cache.js +34 -0
- package/dist/bindings/cache.js.map +1 -0
- package/dist/bindings/celld-kv.d.ts +26 -0
- package/dist/bindings/celld-kv.js +81 -0
- package/dist/bindings/celld-kv.js.map +1 -0
- package/dist/bindings/celld-queues.d.ts +56 -0
- package/dist/bindings/celld-queues.js +223 -0
- package/dist/bindings/celld-queues.js.map +1 -0
- package/dist/bindings/celld-workflows.d.ts +22 -0
- package/dist/bindings/celld-workflows.js +54 -0
- package/dist/bindings/celld-workflows.js.map +1 -0
- package/dist/bindings/cron.d.ts +27 -0
- package/dist/bindings/cron.js +232 -0
- package/dist/bindings/cron.js.map +1 -0
- package/dist/bindings/d1-migrations.d.ts +22 -0
- package/dist/bindings/d1-migrations.js +32 -0
- package/dist/bindings/d1-migrations.js.map +1 -0
- package/dist/bindings/html-rewriter.d.ts +8 -0
- package/dist/bindings/html-rewriter.js +75 -0
- package/dist/bindings/html-rewriter.js.map +1 -0
- package/dist/bindings/r2-celld.d.ts +3 -0
- package/dist/bindings/r2-celld.js +113 -0
- package/dist/bindings/r2-celld.js.map +1 -0
- package/dist/bindings/r2.d.ts +236 -0
- package/dist/bindings/r2.js +419 -0
- package/dist/bindings/r2.js.map +1 -0
- package/dist/bindings/worker-loader.d.ts +83 -0
- package/dist/bindings/worker-loader.js +183 -0
- package/dist/bindings/worker-loader.js.map +1 -0
- package/dist/bindings/workers-ai.d.ts +39 -0
- package/dist/bindings/workers-ai.js +175 -0
- package/dist/bindings/workers-ai.js.map +1 -0
- package/dist/browser.d.ts +7 -0
- package/dist/browser.js +8 -0
- package/dist/browser.js.map +1 -0
- package/dist/bucket/celld-cells.d.ts +86 -0
- package/dist/bucket/celld-cells.js +273 -0
- package/dist/bucket/celld-cells.js.map +1 -0
- package/dist/bucket/celld-format.d.ts +10 -0
- package/dist/bucket/celld-format.js +27 -0
- package/dist/bucket/celld-format.js.map +1 -0
- package/dist/bucket/idb-object-store.d.ts +9 -0
- package/dist/bucket/idb-object-store.js +73 -0
- package/dist/bucket/idb-object-store.js.map +1 -0
- package/dist/bucket/ltx-log.d.ts +34 -0
- package/dist/bucket/ltx-log.js +289 -0
- package/dist/bucket/ltx-log.js.map +1 -0
- package/dist/bucket/object-store.d.ts +51 -0
- package/dist/bucket/object-store.js +114 -0
- package/dist/bucket/object-store.js.map +1 -0
- package/dist/bucket/s3-object-store.d.ts +25 -0
- package/dist/bucket/s3-object-store.js +151 -0
- package/dist/bucket/s3-object-store.js.map +1 -0
- package/dist/bucket/sqlite-object-store.d.ts +10 -0
- package/dist/bucket/sqlite-object-store.js +107 -0
- package/dist/bucket/sqlite-object-store.js.map +1 -0
- package/dist/core/alarms.d.ts +27 -0
- package/dist/core/alarms.js +88 -0
- package/dist/core/alarms.js.map +1 -0
- package/dist/core/async-context.d.ts +34 -0
- package/dist/core/async-context.js +50 -0
- package/dist/core/async-context.js.map +1 -0
- package/dist/core/celld-schema.d.ts +14 -0
- package/dist/core/celld-schema.js +34 -0
- package/dist/core/celld-schema.js.map +1 -0
- package/dist/core/durable-object.d.ts +28 -0
- package/dist/core/durable-object.js +31 -0
- package/dist/core/durable-object.js.map +1 -0
- package/dist/core/gates.d.ts +33 -0
- package/dist/core/gates.js +89 -0
- package/dist/core/gates.js.map +1 -0
- package/dist/core/handles.d.ts +90 -0
- package/dist/core/handles.js +224 -0
- package/dist/core/handles.js.map +1 -0
- package/dist/core/id.d.ts +19 -0
- package/dist/core/id.js +64 -0
- package/dist/core/id.js.map +1 -0
- package/dist/core/namespace.d.ts +38 -0
- package/dist/core/namespace.js +141 -0
- package/dist/core/namespace.js.map +1 -0
- package/dist/core/object-host.d.ts +91 -0
- package/dist/core/object-host.js +616 -0
- package/dist/core/object-host.js.map +1 -0
- package/dist/core/request-body.d.ts +9 -0
- package/dist/core/request-body.js +55 -0
- package/dist/core/request-body.js.map +1 -0
- package/dist/core/runtime.d.ts +121 -0
- package/dist/core/runtime.js +366 -0
- package/dist/core/runtime.js.map +1 -0
- package/dist/core/shard.d.ts +7 -0
- package/dist/core/shard.js +10 -0
- package/dist/core/shard.js.map +1 -0
- package/dist/core/sql-split.d.ts +15 -0
- package/dist/core/sql-split.js +101 -0
- package/dist/core/sql-split.js.map +1 -0
- package/dist/core/sql-storage.d.ts +53 -0
- package/dist/core/sql-storage.js +407 -0
- package/dist/core/sql-storage.js.map +1 -0
- package/dist/core/state.d.ts +44 -0
- package/dist/core/state.js +93 -0
- package/dist/core/state.js.map +1 -0
- package/dist/core/storage.d.ts +103 -0
- package/dist/core/storage.js +351 -0
- package/dist/core/storage.js.map +1 -0
- package/dist/core/worker-rejections.d.ts +21 -0
- package/dist/core/worker-rejections.js +55 -0
- package/dist/core/worker-rejections.js.map +1 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.js +23 -0
- package/dist/index.js.map +1 -0
- package/dist/isolate/browser.d.ts +16 -0
- package/dist/isolate/browser.js +37 -0
- package/dist/isolate/browser.js.map +1 -0
- package/dist/isolate/host.d.ts +24 -0
- package/dist/isolate/host.js +74 -0
- package/dist/isolate/host.js.map +1 -0
- package/dist/isolate/isolate-worker.d.ts +1 -0
- package/dist/isolate/isolate-worker.js +53 -0
- package/dist/isolate/isolate-worker.js.map +1 -0
- package/dist/isolate/isolate.d.ts +10 -0
- package/dist/isolate/isolate.js +169 -0
- package/dist/isolate/isolate.js.map +1 -0
- package/dist/isolate/node-entry.d.ts +1 -0
- package/dist/isolate/node-entry.js +12 -0
- package/dist/isolate/node-entry.js.map +1 -0
- package/dist/isolate/protocol.d.ts +66 -0
- package/dist/isolate/protocol.js +205 -0
- package/dist/isolate/protocol.js.map +1 -0
- package/dist/ltx/crc64.d.ts +6 -0
- package/dist/ltx/crc64.js +46 -0
- package/dist/ltx/crc64.js.map +1 -0
- package/dist/ltx/ltx-page-server.d.ts +65 -0
- package/dist/ltx/ltx-page-server.js +309 -0
- package/dist/ltx/ltx-page-server.js.map +1 -0
- package/dist/ltx/ltx-pages.d.ts +81 -0
- package/dist/ltx/ltx-pages.js +315 -0
- package/dist/ltx/ltx-pages.js.map +1 -0
- package/dist/ltx/ltx-store-worker.d.ts +1 -0
- package/dist/ltx/ltx-store-worker.js +38 -0
- package/dist/ltx/ltx-store-worker.js.map +1 -0
- package/dist/ltx/ltx.d.ts +45 -0
- package/dist/ltx/ltx.js +203 -0
- package/dist/ltx/ltx.js.map +1 -0
- package/dist/ltx/lz4.d.ts +9 -0
- package/dist/ltx/lz4.js +156 -0
- package/dist/ltx/lz4.js.map +1 -0
- package/dist/ltx/opfs-sync-files.d.ts +23 -0
- package/dist/ltx/opfs-sync-files.js +191 -0
- package/dist/ltx/opfs-sync-files.js.map +1 -0
- package/dist/ltx/xxh32.d.ts +1 -0
- package/dist/ltx/xxh32.js +43 -0
- package/dist/ltx/xxh32.js.map +1 -0
- package/dist/miniflare-compat/blob-service.d.ts +13 -0
- package/dist/miniflare-compat/blob-service.js +66 -0
- package/dist/miniflare-compat/blob-service.js.map +1 -0
- package/dist/miniflare-compat/kv-binding.d.ts +44 -0
- package/dist/miniflare-compat/kv-binding.js +99 -0
- package/dist/miniflare-compat/kv-binding.js.map +1 -0
- package/dist/miniflare-compat/shims/assert.d.ts +15 -0
- package/dist/miniflare-compat/shims/assert.js +24 -0
- package/dist/miniflare-compat/shims/assert.js.map +1 -0
- package/dist/miniflare-compat/shims/workerd-cloudflare.d.ts +7 -0
- package/dist/miniflare-compat/shims/workerd-cloudflare.js +6 -0
- package/dist/miniflare-compat/shims/workerd-cloudflare.js.map +1 -0
- package/dist/miniflare-compat/shims/workerd-globals.d.ts +1 -0
- package/dist/miniflare-compat/shims/workerd-globals.js +30 -0
- package/dist/miniflare-compat/shims/workerd-globals.js.map +1 -0
- package/dist/miniflare-compat/shims/workerd-internal.d.ts +20 -0
- package/dist/miniflare-compat/shims/workerd-internal.js +23 -0
- package/dist/miniflare-compat/shims/workerd-internal.js.map +1 -0
- package/dist/miniflare-compat/shims/workerd-performance.d.ts +4 -0
- package/dist/miniflare-compat/shims/workerd-performance.js +17 -0
- package/dist/miniflare-compat/shims/workerd-performance.js.map +1 -0
- package/dist/miniflare-compat/simulator-fetcher.d.ts +9 -0
- package/dist/miniflare-compat/simulator-fetcher.js +14 -0
- package/dist/miniflare-compat/simulator-fetcher.js.map +1 -0
- package/dist/miniflare-compat/vendored.d.ts +15 -0
- package/dist/miniflare-compat/vendored.js +27 -0
- package/dist/miniflare-compat/vendored.js.map +1 -0
- package/dist/node-compat/async_hooks.d.ts +22 -0
- package/dist/node-compat/async_hooks.js +176 -0
- package/dist/node-compat/async_hooks.js.map +1 -0
- package/dist/node-compat/browser-registry.d.ts +3 -0
- package/dist/node-compat/browser-registry.js +39 -0
- package/dist/node-compat/browser-registry.js.map +1 -0
- package/dist/node-compat/crypto.d.ts +60 -0
- package/dist/node-compat/crypto.js +143 -0
- package/dist/node-compat/crypto.js.map +1 -0
- package/dist/node-compat/diagnostics_channel.d.ts +27 -0
- package/dist/node-compat/diagnostics_channel.js +46 -0
- package/dist/node-compat/diagnostics_channel.js.map +1 -0
- package/dist/node-compat/fs.d.ts +35 -0
- package/dist/node-compat/fs.js +40 -0
- package/dist/node-compat/fs.js.map +1 -0
- package/dist/node-compat/globals.d.ts +3 -0
- package/dist/node-compat/globals.js +4 -0
- package/dist/node-compat/globals.js.map +1 -0
- package/dist/node-compat/install.d.ts +5 -0
- package/dist/node-compat/install.js +12 -0
- package/dist/node-compat/install.js.map +1 -0
- package/dist/node-compat/module.d.ts +12 -0
- package/dist/node-compat/module.js +36 -0
- package/dist/node-compat/module.js.map +1 -0
- package/dist/node-compat/modules.d.ts +8 -0
- package/dist/node-compat/modules.js +23 -0
- package/dist/node-compat/modules.js.map +1 -0
- package/dist/node-compat/os.d.ts +36 -0
- package/dist/node-compat/os.js +36 -0
- package/dist/node-compat/os.js.map +1 -0
- package/dist/node-compat/process.d.ts +14 -0
- package/dist/node-compat/process.js +16 -0
- package/dist/node-compat/process.js.map +1 -0
- package/dist/node-compat/shims.d.ts +4 -0
- package/dist/node.d.ts +5 -0
- package/dist/node.js +6 -0
- package/dist/node.js.map +1 -0
- package/dist/peer/celld-peer.d.ts +53 -0
- package/dist/peer/celld-peer.js +555 -0
- package/dist/peer/celld-peer.js.map +1 -0
- package/dist/ports.d.ts +183 -0
- package/dist/ports.js +10 -0
- package/dist/ports.js.map +1 -0
- package/dist/project/browser-bundle.d.ts +17 -0
- package/dist/project/browser-bundle.js +125 -0
- package/dist/project/browser-bundle.js.map +1 -0
- package/dist/project/browser-cookies.d.ts +12 -0
- package/dist/project/browser-cookies.js +90 -0
- package/dist/project/browser-cookies.js.map +1 -0
- package/dist/project/browser-host.d.ts +173 -0
- package/dist/project/browser-host.js +200 -0
- package/dist/project/browser-host.js.map +1 -0
- package/dist/project/browser-modules.d.ts +17 -0
- package/dist/project/browser-modules.js +89 -0
- package/dist/project/browser-modules.js.map +1 -0
- package/dist/project/browser-shards.d.ts +62 -0
- package/dist/project/browser-shards.js +210 -0
- package/dist/project/browser-shards.js.map +1 -0
- package/dist/project/browser-tabs.d.ts +103 -0
- package/dist/project/browser-tabs.js +418 -0
- package/dist/project/browser-tabs.js.map +1 -0
- package/dist/project/cloudflare-email.d.ts +8 -0
- package/dist/project/cloudflare-email.js +11 -0
- package/dist/project/cloudflare-email.js.map +1 -0
- package/dist/project/cloudflare-workers.d.ts +12 -0
- package/dist/project/cloudflare-workers.js +16 -0
- package/dist/project/cloudflare-workers.js.map +1 -0
- package/dist/project/host-modules.d.ts +4 -0
- package/dist/project/host-modules.js +40 -0
- package/dist/project/host-modules.js.map +1 -0
- package/dist/project/jsonc.d.ts +2 -0
- package/dist/project/jsonc.js +27 -0
- package/dist/project/jsonc.js.map +1 -0
- package/dist/project/lower-await.d.ts +2 -0
- package/dist/project/lower-await.js +151 -0
- package/dist/project/lower-await.js.map +1 -0
- package/dist/project/node-project.d.ts +38 -0
- package/dist/project/node-project.js +362 -0
- package/dist/project/node-project.js.map +1 -0
- package/dist/project/own-module.d.ts +6 -0
- package/dist/project/own-module.js +13 -0
- package/dist/project/own-module.js.map +1 -0
- package/dist/project/scheduler.d.ts +2 -0
- package/dist/project/scheduler.js +3 -0
- package/dist/project/scheduler.js.map +1 -0
- package/dist/project/site.d.ts +74 -0
- package/dist/project/site.js +206 -0
- package/dist/project/site.js.map +1 -0
- package/dist/project/virtual-modules.d.ts +3 -0
- package/dist/project/worker-scope.d.ts +27 -0
- package/dist/project/worker-scope.js +46 -0
- package/dist/project/worker-scope.js.map +1 -0
- package/dist/project/workerd-message-channel.d.ts +13 -0
- package/dist/project/workerd-message-channel.js +137 -0
- package/dist/project/workerd-message-channel.js.map +1 -0
- package/dist/project/workers-global-names.d.ts +6 -0
- package/dist/project/workers-global-names.js +15 -0
- package/dist/project/workers-global-names.js.map +1 -0
- package/dist/project/workers-globals-module.d.ts +8 -0
- package/dist/project/workers-globals-module.js +9 -0
- package/dist/project/workers-globals-module.js.map +1 -0
- package/dist/project/workers-globals.d.ts +9 -0
- package/dist/project/workers-globals.js +19 -0
- package/dist/project/workers-globals.js.map +1 -0
- package/dist/project/wrangler.d.ts +116 -0
- package/dist/project/wrangler.js +342 -0
- package/dist/project/wrangler.js.map +1 -0
- package/dist/router/cell-router.d.ts +72 -0
- package/dist/router/cell-router.js +247 -0
- package/dist/router/cell-router.js.map +1 -0
- package/dist/router/shard-worker.d.ts +44 -0
- package/dist/router/shard-worker.js +163 -0
- package/dist/router/shard-worker.js.map +1 -0
- package/dist/transport/cells-http.d.ts +25 -0
- package/dist/transport/cells-http.js +141 -0
- package/dist/transport/cells-http.js.map +1 -0
- package/dist/transport/cells-link.d.ts +27 -0
- package/dist/transport/cells-link.js +71 -0
- package/dist/transport/cells-link.js.map +1 -0
- package/dist/transport/port.d.ts +26 -0
- package/dist/transport/port.js +100 -0
- package/dist/transport/port.js.map +1 -0
- package/dist/v8/codec.d.ts +6 -0
- package/dist/v8/codec.js +9 -0
- package/dist/v8/codec.js.map +1 -0
- package/dist/v8/serialize.d.ts +10 -0
- package/dist/v8/serialize.js +601 -0
- package/dist/v8/serialize.js.map +1 -0
- package/dist/ws/client.d.ts +25 -0
- package/dist/ws/client.js +203 -0
- package/dist/ws/client.js.map +1 -0
- package/dist/ws/control.d.ts +48 -0
- package/dist/ws/control.js +44 -0
- package/dist/ws/control.js.map +1 -0
- package/dist/ws/frames.d.ts +38 -0
- package/dist/ws/frames.js +39 -0
- package/dist/ws/frames.js.map +1 -0
- package/dist/ws/server.d.ts +99 -0
- package/dist/ws/server.js +295 -0
- package/dist/ws/server.js.map +1 -0
- package/dist/ws/sw-connector.d.ts +10 -0
- package/dist/ws/sw-connector.js +25 -0
- package/dist/ws/sw-connector.js.map +1 -0
- package/dist/ws/upgrade.d.ts +28 -0
- package/dist/ws/upgrade.js +75 -0
- package/dist/ws/upgrade.js.map +1 -0
- package/package.json +111 -3
- package/vendor/celld/LICENSE +202 -0
- package/vendor/celld/kv.js +1131 -0
- package/vendor/celld/queues.js +1656 -0
- package/vendor/celld/workflows.js +1610 -0
- package/vendor/html-rewriter-wasm/html_rewriter.js +925 -0
- package/vendor/html-rewriter-wasm/html_rewriter_bg.wasm +0 -0
- package/vendor/miniflare/d1-api.js +738 -0
- package/vendor/miniflare/d1-database.js +6978 -0
- package/vendor/miniflare/kv-namespace.js +3061 -0
|
@@ -0,0 +1,1656 @@
|
|
|
1
|
+
// GENERATED by scripts/vendor-celld.mjs from denoland/celld@8702f6d186c77698dbfb30cf29e3b1922a88f008 crates/celld/js/harness.js
|
|
2
|
+
// (Apache-2.0): the Queues section and __celld.__dispatchEntrypointQueue, unchanged, wrapped as
|
|
3
|
+
// queueService(host). Do not edit.
|
|
4
|
+
export function queueService(host) {
|
|
5
|
+
const { __cell, __queue_policy, __queue_dispatch, __queue_alarm_set_wait, __sc_encode, __sc_decode,
|
|
6
|
+
__cellSweepBatch, __dispatchEntrypointMethodInCurrentEvent } = host;
|
|
7
|
+
const $$btoa = host.btoa;
|
|
8
|
+
const __d = host.debug ?? (() => {});
|
|
9
|
+
const classes = {};
|
|
10
|
+
const __runtimeClass = (name, cls) => { classes[name] = cls; };
|
|
11
|
+
const __celld = {};
|
|
12
|
+
// A queue batch owns mutable settlement state only while its handler runs.
|
|
13
|
+
// Keeping a batch alive after dispatch must not keep authority to settle its
|
|
14
|
+
// lease, because that lease can have expired and been handed out again.
|
|
15
|
+
__celld.__dispatchEntrypointQueue = async (name, incoming) => {
|
|
16
|
+
let active = true;
|
|
17
|
+
let ackAll = false;
|
|
18
|
+
const explicitAcks = new Set();
|
|
19
|
+
const retries = new Map();
|
|
20
|
+
const retryBatch = { retry: false, delaySeconds: undefined };
|
|
21
|
+
const checkActive = () => {
|
|
22
|
+
if (!active)
|
|
23
|
+
throw new Error("Queue event methods cannot be called after the handler returns.");
|
|
24
|
+
};
|
|
25
|
+
const retryDelay = (options) => {
|
|
26
|
+
const value = options?.delaySeconds;
|
|
27
|
+
if (value === undefined) return undefined;
|
|
28
|
+
if (!Number.isInteger(value) || value < -0x8000_0000 || value > 0x7fff_ffff)
|
|
29
|
+
throw new TypeError("delaySeconds must be a 32-bit integer");
|
|
30
|
+
return value;
|
|
31
|
+
};
|
|
32
|
+
const decode = (message) => {
|
|
33
|
+
switch (message.contentType) {
|
|
34
|
+
case "text": return new TextDecoder().decode(message.body);
|
|
35
|
+
case "bytes": return message.body;
|
|
36
|
+
case "json": return JSON.parse(new TextDecoder().decode(message.body));
|
|
37
|
+
case "v8": return __sc_decode(message.body);
|
|
38
|
+
default: throw new TypeError(`Unknown queue content type: ${message.contentType}`);
|
|
39
|
+
}
|
|
40
|
+
};
|
|
41
|
+
const messages = incoming.messages.map((message) => ({
|
|
42
|
+
id: message.id,
|
|
43
|
+
timestamp: new Date(message.timestamp),
|
|
44
|
+
body: decode(message),
|
|
45
|
+
attempts: message.attempts,
|
|
46
|
+
ack() {
|
|
47
|
+
checkActive();
|
|
48
|
+
if (ackAll || retryBatch.retry || retries.has(message.id)) return;
|
|
49
|
+
explicitAcks.add(message.id);
|
|
50
|
+
},
|
|
51
|
+
retry(options) {
|
|
52
|
+
checkActive();
|
|
53
|
+
if (ackAll || explicitAcks.has(message.id)) return;
|
|
54
|
+
const delaySeconds = retryDelay(options);
|
|
55
|
+
if (delaySeconds !== undefined || !retries.has(message.id))
|
|
56
|
+
retries.set(message.id, delaySeconds);
|
|
57
|
+
},
|
|
58
|
+
}));
|
|
59
|
+
const oldest = incoming.metrics.oldestMessageTimestamp;
|
|
60
|
+
const batch = {
|
|
61
|
+
queue: incoming.queue,
|
|
62
|
+
messages,
|
|
63
|
+
metadata: {
|
|
64
|
+
metrics: {
|
|
65
|
+
backlogCount: incoming.metrics.backlogCount,
|
|
66
|
+
backlogBytes: incoming.metrics.backlogBytes,
|
|
67
|
+
oldestMessageTimestamp: oldest === undefined ? undefined : new Date(oldest),
|
|
68
|
+
},
|
|
69
|
+
},
|
|
70
|
+
ackAll() {
|
|
71
|
+
checkActive();
|
|
72
|
+
if (!retryBatch.retry) ackAll = true;
|
|
73
|
+
},
|
|
74
|
+
retryAll(options) {
|
|
75
|
+
checkActive();
|
|
76
|
+
if (ackAll) return;
|
|
77
|
+
retryBatch.retry = true;
|
|
78
|
+
const delaySeconds = retryDelay(options);
|
|
79
|
+
if (delaySeconds !== undefined) retryBatch.delaySeconds = delaySeconds;
|
|
80
|
+
},
|
|
81
|
+
};
|
|
82
|
+
let outcome = "ok";
|
|
83
|
+
let errorText;
|
|
84
|
+
try {
|
|
85
|
+
await __dispatchEntrypointMethodInCurrentEvent(name, "queue", batch);
|
|
86
|
+
} catch (error) {
|
|
87
|
+
// A handler exception retries the batch but keeps explicit acks that the
|
|
88
|
+
// handler made before it threw. The caller needs both facts together.
|
|
89
|
+
outcome = "exception";
|
|
90
|
+
try {
|
|
91
|
+
errorText = String(error);
|
|
92
|
+
} catch {
|
|
93
|
+
errorText = "queue handler rejected";
|
|
94
|
+
}
|
|
95
|
+
} finally {
|
|
96
|
+
active = false;
|
|
97
|
+
}
|
|
98
|
+
return {
|
|
99
|
+
outcome,
|
|
100
|
+
error: errorText,
|
|
101
|
+
ackAll,
|
|
102
|
+
retryBatch,
|
|
103
|
+
explicitAcks: [...explicitAcks],
|
|
104
|
+
retryMessages: [...retries].map(([msgId, delaySeconds]) => ({
|
|
105
|
+
msgId,
|
|
106
|
+
delaySeconds,
|
|
107
|
+
})),
|
|
108
|
+
};
|
|
109
|
+
};
|
|
110
|
+
|
|
111
|
+
// ---- Queues --------------------------------------------------------------
|
|
112
|
+
// A Queue is one runtime-supplied Durable Object. The producer binding writes
|
|
113
|
+
// bytes and a content tag through RPC, and the cell owns the durable order,
|
|
114
|
+
// deadlines, leases, and alarms. `celld_logic::queue` owns every transition
|
|
115
|
+
// that can invalidate a lease or choose a deadline; this code supplies SQL and
|
|
116
|
+
// the Cloudflare-compatible presentation.
|
|
117
|
+
|
|
118
|
+
const __QUEUE_MESSAGES = "__queue_messages";
|
|
119
|
+
const __QUEUE_META = "__queue_meta";
|
|
120
|
+
const __QUEUE_STATS = "__queue_stats";
|
|
121
|
+
const __QUEUE_TRANSFER_RECEIPTS = "__queue_transfer_receipts";
|
|
122
|
+
const __queueEncoder = new TextEncoder();
|
|
123
|
+
|
|
124
|
+
// Independent send() calls share the same economics as sendBatch() without
|
|
125
|
+
// changing the public promise boundary. The first call opens a four-millisecond
|
|
126
|
+
// owner-local group by default; later cell events append their already-copied
|
|
127
|
+
// entries, and park on their own promise. One event commits the bounded group,
|
|
128
|
+
// and every response still trails that commit through the ordinary output
|
|
129
|
+
// gate. The three caps bound heap retention and keep the persisted transaction
|
|
130
|
+
// inside the public sendBatch limits.
|
|
131
|
+
const __QUEUE_PRODUCER_GROUP_CALLS = 64;
|
|
132
|
+
const __QUEUE_PRODUCER_GROUP_MESSAGES = 100;
|
|
133
|
+
const __QUEUE_PRODUCER_GROUP_BYTES = 256_000;
|
|
134
|
+
|
|
135
|
+
const __queueError = (message) => new Error("QUEUE_ERROR: " + String(message));
|
|
136
|
+
// A Queue message id is a UUIDv7 (RFC 9562): 48 bits of enqueue time, a
|
|
137
|
+
// 12-bit sequence inside that millisecond, and 62 random bits. It keeps the
|
|
138
|
+
// UUID shape a consumer can parse, and it sorts in enqueue order, so the
|
|
139
|
+
// UNIQUE index on `id` appends. A random v4 id made every grouped send
|
|
140
|
+
// transaction dirty one index leaf per message (about 1.6 WAL frames per
|
|
141
|
+
// 64-byte message) and turned every passive checkpoint into a random write
|
|
142
|
+
// across the whole backlog, which is what made a deep Queue's checkpoint
|
|
143
|
+
// cost grow with its backlog. The sequence never moves backwards inside one
|
|
144
|
+
// process, so a clock step only widens the gap between consecutive ids.
|
|
145
|
+
let __queueIdMs = 0;
|
|
146
|
+
let __queueIdSeq = 0;
|
|
147
|
+
function __queueMessageId(now) {
|
|
148
|
+
if (now > __queueIdMs) {
|
|
149
|
+
__queueIdMs = now;
|
|
150
|
+
__queueIdSeq = 0;
|
|
151
|
+
} else {
|
|
152
|
+
__queueIdSeq += 1;
|
|
153
|
+
if (__queueIdSeq > 0xfff) {
|
|
154
|
+
__queueIdMs += 1;
|
|
155
|
+
__queueIdSeq = 0;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
const bytes = new Uint8Array(16);
|
|
159
|
+
crypto.getRandomValues(bytes);
|
|
160
|
+
const ms = __queueIdMs;
|
|
161
|
+
bytes[0] = Math.floor(ms / 0x10000000000) & 0xff;
|
|
162
|
+
bytes[1] = Math.floor(ms / 0x100000000) & 0xff;
|
|
163
|
+
bytes[2] = (ms >>> 24) & 0xff;
|
|
164
|
+
bytes[3] = (ms >>> 16) & 0xff;
|
|
165
|
+
bytes[4] = (ms >>> 8) & 0xff;
|
|
166
|
+
bytes[5] = ms & 0xff;
|
|
167
|
+
bytes[6] = 0x70 | (__queueIdSeq >> 8);
|
|
168
|
+
bytes[7] = __queueIdSeq & 0xff;
|
|
169
|
+
bytes[8] = (bytes[8] & 0x3f) | 0x80;
|
|
170
|
+
let hex = "";
|
|
171
|
+
for (let i = 0; i < 16; i++) hex += bytes[i].toString(16).padStart(2, "0");
|
|
172
|
+
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${
|
|
173
|
+
hex.slice(16, 20)
|
|
174
|
+
}-${hex.slice(20)}`;
|
|
175
|
+
}
|
|
176
|
+
const __queueLimits = () => __cell.queueLimits;
|
|
177
|
+
const __queuePolicy = (request) =>
|
|
178
|
+
JSON.parse(__queue_policy(JSON.stringify(request)));
|
|
179
|
+
|
|
180
|
+
const __queueDelay = (value, fallback = 0) => {
|
|
181
|
+
if (value === undefined || value === null) return fallback;
|
|
182
|
+
const seconds = Number(value);
|
|
183
|
+
if (!Number.isInteger(seconds) || seconds < 0 ||
|
|
184
|
+
seconds > __queueLimits().maxDelaySeconds) {
|
|
185
|
+
throw __queueError(
|
|
186
|
+
`delaySeconds must be an integer from 0 to ${__queueLimits().maxDelaySeconds}`,
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
return seconds;
|
|
190
|
+
};
|
|
191
|
+
|
|
192
|
+
const __queueBytes = (value) => {
|
|
193
|
+
if (value instanceof ArrayBuffer) {
|
|
194
|
+
return new Uint8Array(value.slice(0));
|
|
195
|
+
}
|
|
196
|
+
if (ArrayBuffer.isView(value)) {
|
|
197
|
+
return new Uint8Array(
|
|
198
|
+
value.buffer.slice(value.byteOffset, value.byteOffset + value.byteLength),
|
|
199
|
+
);
|
|
200
|
+
}
|
|
201
|
+
throw __queueError("a bytes message must be an ArrayBuffer or a typed array");
|
|
202
|
+
};
|
|
203
|
+
|
|
204
|
+
// Encode before entering the cell. The copy makes `sendBatch()` immune to a
|
|
205
|
+
// caller resizing or reusing a backing buffer while the durable write awaits.
|
|
206
|
+
const __queueEncode = (body, requestedType) => {
|
|
207
|
+
const contentType = requestedType ??
|
|
208
|
+
(__cell.compat.queueJsonMessages ? "json" : "v8");
|
|
209
|
+
let bytes;
|
|
210
|
+
switch (contentType) {
|
|
211
|
+
case "text":
|
|
212
|
+
bytes = __queueEncoder.encode(String(body));
|
|
213
|
+
break;
|
|
214
|
+
case "bytes":
|
|
215
|
+
bytes = __queueBytes(body);
|
|
216
|
+
break;
|
|
217
|
+
case "json": {
|
|
218
|
+
let json;
|
|
219
|
+
try {
|
|
220
|
+
json = JSON.stringify(body);
|
|
221
|
+
} catch (error) {
|
|
222
|
+
throw __queueError("the JSON message body cannot be serialized", error);
|
|
223
|
+
}
|
|
224
|
+
if (json === undefined) {
|
|
225
|
+
throw __queueError("the JSON message body cannot be undefined");
|
|
226
|
+
}
|
|
227
|
+
bytes = __queueEncoder.encode(json);
|
|
228
|
+
break;
|
|
229
|
+
}
|
|
230
|
+
case "v8":
|
|
231
|
+
try {
|
|
232
|
+
bytes = __sc_encode(body);
|
|
233
|
+
} catch (error) {
|
|
234
|
+
throw __queueError("the message body cannot be cloned", error);
|
|
235
|
+
}
|
|
236
|
+
break;
|
|
237
|
+
default:
|
|
238
|
+
throw __queueError(
|
|
239
|
+
`contentType must be "text", "bytes", "json", or "v8", got ${JSON.stringify(contentType)}`,
|
|
240
|
+
);
|
|
241
|
+
}
|
|
242
|
+
if (bytes.byteLength > __queueLimits().maxMessageBytes) {
|
|
243
|
+
throw __queueError(
|
|
244
|
+
`a message is at most ${__queueLimits().maxMessageBytes} bytes, got ${bytes.byteLength}`,
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
return { body: bytes, contentType, size: bytes.byteLength };
|
|
248
|
+
};
|
|
249
|
+
|
|
250
|
+
// A lease-scoped read must seek, not walk the backlog. The plan is the only
|
|
251
|
+
// evidence that says which, and it is cheap to ask for, so a gated test seam
|
|
252
|
+
// reads it from the shipped query rather than from a copy that can drift.
|
|
253
|
+
const __queueReportLeaseLookupPlan = (sql, query, leaseId) => {
|
|
254
|
+
if (typeof __test_queue_lease_lookup_plan !== "function") return;
|
|
255
|
+
const plan = sql.exec(`EXPLAIN QUERY PLAN ${query}`, leaseId)
|
|
256
|
+
.toArray()
|
|
257
|
+
.map((row) => row.detail)
|
|
258
|
+
.join("; ");
|
|
259
|
+
__test_queue_lease_lookup_plan(plan);
|
|
260
|
+
};
|
|
261
|
+
|
|
262
|
+
const __queueMetricsView = (metrics) => ({
|
|
263
|
+
backlogCount: metrics.backlogCount,
|
|
264
|
+
backlogBytes: metrics.backlogBytes,
|
|
265
|
+
oldestMessageTimestamp: metrics.oldestMessageTimestamp === null ||
|
|
266
|
+
metrics.oldestMessageTimestamp === undefined
|
|
267
|
+
? undefined
|
|
268
|
+
: new Date(metrics.oldestMessageTimestamp),
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
class __QueueCell {
|
|
272
|
+
constructor(ctx) {
|
|
273
|
+
this.ctx = ctx;
|
|
274
|
+
this.queue = ctx.id.name;
|
|
275
|
+
this._ready = false;
|
|
276
|
+
this._producerGroup = null;
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
_open() {
|
|
280
|
+
if (this._ready) return this.ctx.storage.sql;
|
|
281
|
+
const sql = this.ctx.storage.sql;
|
|
282
|
+
sql.exec(
|
|
283
|
+
`CREATE TABLE IF NOT EXISTS ${__QUEUE_MESSAGES} (
|
|
284
|
+
seq INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
285
|
+
id TEXT NOT NULL UNIQUE,
|
|
286
|
+
body BLOB NOT NULL,
|
|
287
|
+
content_type TEXT NOT NULL,
|
|
288
|
+
size INTEGER NOT NULL,
|
|
289
|
+
enqueued_at INTEGER NOT NULL,
|
|
290
|
+
visible_at INTEGER NOT NULL,
|
|
291
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
292
|
+
lease_id TEXT,
|
|
293
|
+
lease_generation INTEGER NOT NULL DEFAULT 0,
|
|
294
|
+
leased_until INTEGER,
|
|
295
|
+
purge_on_settle INTEGER NOT NULL DEFAULT 0,
|
|
296
|
+
dlq_target TEXT,
|
|
297
|
+
dlq_transfer_id TEXT,
|
|
298
|
+
transfer_kind TEXT,
|
|
299
|
+
dead_letter_source TEXT,
|
|
300
|
+
CHECK ((dlq_target IS NULL AND transfer_kind IS NULL)
|
|
301
|
+
OR (dlq_target IS NOT NULL
|
|
302
|
+
AND transfer_kind IN ('dead-letter', 'redrive')))
|
|
303
|
+
)`,
|
|
304
|
+
);
|
|
305
|
+
sql.exec(
|
|
306
|
+
`CREATE INDEX IF NOT EXISTS ${__QUEUE_MESSAGES}_visible
|
|
307
|
+
ON ${__QUEUE_MESSAGES} (visible_at, seq)`,
|
|
308
|
+
);
|
|
309
|
+
// Rearming is on every producer acknowledgement and settlement. Each
|
|
310
|
+
// scheduling lookup must therefore stop at one deadline or one batch;
|
|
311
|
+
// an aggregate over the backlog makes a large Queue quadratic to fill.
|
|
312
|
+
sql.exec(
|
|
313
|
+
`CREATE INDEX IF NOT EXISTS ${__QUEUE_MESSAGES}_ready
|
|
314
|
+
ON ${__QUEUE_MESSAGES} (visible_at, seq)
|
|
315
|
+
WHERE purge_on_settle = 0 AND dlq_target IS NULL
|
|
316
|
+
AND lease_id IS NULL`,
|
|
317
|
+
);
|
|
318
|
+
sql.exec(
|
|
319
|
+
`CREATE INDEX IF NOT EXISTS ${__QUEUE_MESSAGES}_enqueued
|
|
320
|
+
ON ${__QUEUE_MESSAGES} (enqueued_at, seq)`,
|
|
321
|
+
);
|
|
322
|
+
sql.exec(
|
|
323
|
+
`CREATE INDEX IF NOT EXISTS ${__QUEUE_MESSAGES}_retained
|
|
324
|
+
ON ${__QUEUE_MESSAGES} (enqueued_at, seq)
|
|
325
|
+
WHERE purge_on_settle = 0 AND dlq_target IS NULL`,
|
|
326
|
+
);
|
|
327
|
+
sql.exec(
|
|
328
|
+
`CREATE INDEX IF NOT EXISTS ${__QUEUE_MESSAGES}_lease
|
|
329
|
+
ON ${__QUEUE_MESSAGES} (leased_until, seq)
|
|
330
|
+
WHERE lease_id IS NOT NULL`,
|
|
331
|
+
);
|
|
332
|
+
sql.exec(
|
|
333
|
+
`CREATE INDEX IF NOT EXISTS ${__QUEUE_MESSAGES}_moving
|
|
334
|
+
ON ${__QUEUE_MESSAGES} (seq)
|
|
335
|
+
WHERE dlq_target IS NOT NULL`,
|
|
336
|
+
);
|
|
337
|
+
// Dispatch and settlement both select a whole batch by its lease. Without
|
|
338
|
+
// this index SQLite answers them by walking every row: the dispatch
|
|
339
|
+
// readback scans `_visible` and the settlement scan walks the table, so
|
|
340
|
+
// each delivered batch costs two passes over the backlog. The `_lease`
|
|
341
|
+
// index cannot serve them because it is keyed on `leased_until`.
|
|
342
|
+
// The partial predicate keeps only leased rows, which `max_concurrency`
|
|
343
|
+
// times `max_batch_size` bounds, and an inserted message has a null
|
|
344
|
+
// `lease_id`, so `send` pays nothing for it.
|
|
345
|
+
sql.exec(
|
|
346
|
+
`CREATE INDEX IF NOT EXISTS ${__QUEUE_MESSAGES}_lease_batch
|
|
347
|
+
ON ${__QUEUE_MESSAGES} (lease_id, visible_at, seq)
|
|
348
|
+
WHERE lease_id IS NOT NULL`,
|
|
349
|
+
);
|
|
350
|
+
// The live-row index catches an impossible duplicate while a received
|
|
351
|
+
// message remains queued. The receipt below is the durable deduplication
|
|
352
|
+
// record after a consumer or purge removes that message.
|
|
353
|
+
sql.exec(
|
|
354
|
+
`CREATE UNIQUE INDEX IF NOT EXISTS ${__QUEUE_MESSAGES}_dlq_transfer
|
|
355
|
+
ON ${__QUEUE_MESSAGES} (dlq_transfer_id)
|
|
356
|
+
WHERE dlq_transfer_id IS NOT NULL AND dlq_target IS NULL`,
|
|
357
|
+
);
|
|
358
|
+
sql.exec(
|
|
359
|
+
`CREATE TABLE IF NOT EXISTS ${__QUEUE_META} (
|
|
360
|
+
name TEXT PRIMARY KEY,
|
|
361
|
+
value TEXT NOT NULL
|
|
362
|
+
) WITHOUT ROWID`,
|
|
363
|
+
);
|
|
364
|
+
// The producer result and operator metrics are on hot paths. SQLite
|
|
365
|
+
// triggers keep the three values in the same transaction as every row
|
|
366
|
+
// transition, so a caller cannot observe a new message with old totals.
|
|
367
|
+
sql.exec(
|
|
368
|
+
`CREATE TABLE IF NOT EXISTS ${__QUEUE_STATS} (
|
|
369
|
+
id INTEGER PRIMARY KEY CHECK (id = 1),
|
|
370
|
+
stored_count INTEGER NOT NULL,
|
|
371
|
+
backlog_count INTEGER NOT NULL,
|
|
372
|
+
backlog_bytes INTEGER NOT NULL,
|
|
373
|
+
oldest_message_timestamp INTEGER
|
|
374
|
+
)`,
|
|
375
|
+
);
|
|
376
|
+
sql.exec(
|
|
377
|
+
`INSERT INTO ${__QUEUE_STATS}
|
|
378
|
+
(id, stored_count, backlog_count, backlog_bytes,
|
|
379
|
+
oldest_message_timestamp)
|
|
380
|
+
SELECT 1,
|
|
381
|
+
COUNT(*),
|
|
382
|
+
COALESCE(SUM(purge_on_settle = 0), 0),
|
|
383
|
+
COALESCE(SUM(CASE WHEN purge_on_settle = 0 THEN size ELSE 0 END), 0),
|
|
384
|
+
MIN(CASE WHEN purge_on_settle = 0 THEN enqueued_at END)
|
|
385
|
+
FROM ${__QUEUE_MESSAGES}
|
|
386
|
+
WHERE true
|
|
387
|
+
ON CONFLICT(id) DO NOTHING`,
|
|
388
|
+
);
|
|
389
|
+
sql.exec(
|
|
390
|
+
`CREATE TRIGGER IF NOT EXISTS ${__QUEUE_MESSAGES}_stored_insert
|
|
391
|
+
AFTER INSERT ON ${__QUEUE_MESSAGES}
|
|
392
|
+
BEGIN
|
|
393
|
+
UPDATE ${__QUEUE_STATS}
|
|
394
|
+
SET stored_count = stored_count + 1
|
|
395
|
+
WHERE id = 1;
|
|
396
|
+
END`,
|
|
397
|
+
);
|
|
398
|
+
sql.exec(
|
|
399
|
+
`CREATE TRIGGER IF NOT EXISTS ${__QUEUE_MESSAGES}_stored_delete
|
|
400
|
+
AFTER DELETE ON ${__QUEUE_MESSAGES}
|
|
401
|
+
BEGIN
|
|
402
|
+
UPDATE ${__QUEUE_STATS}
|
|
403
|
+
SET stored_count = stored_count - 1
|
|
404
|
+
WHERE id = 1;
|
|
405
|
+
END`,
|
|
406
|
+
);
|
|
407
|
+
sql.exec(
|
|
408
|
+
`CREATE TRIGGER IF NOT EXISTS ${__QUEUE_MESSAGES}_stats_insert
|
|
409
|
+
AFTER INSERT ON ${__QUEUE_MESSAGES}
|
|
410
|
+
WHEN new.purge_on_settle = 0
|
|
411
|
+
BEGIN
|
|
412
|
+
UPDATE ${__QUEUE_STATS}
|
|
413
|
+
SET backlog_count = backlog_count + 1,
|
|
414
|
+
backlog_bytes = backlog_bytes + new.size,
|
|
415
|
+
oldest_message_timestamp = CASE
|
|
416
|
+
WHEN oldest_message_timestamp IS NULL
|
|
417
|
+
OR new.enqueued_at < oldest_message_timestamp
|
|
418
|
+
THEN new.enqueued_at ELSE oldest_message_timestamp END
|
|
419
|
+
WHERE id = 1;
|
|
420
|
+
END`,
|
|
421
|
+
);
|
|
422
|
+
sql.exec(
|
|
423
|
+
`CREATE TRIGGER IF NOT EXISTS ${__QUEUE_MESSAGES}_stats_delete
|
|
424
|
+
AFTER DELETE ON ${__QUEUE_MESSAGES}
|
|
425
|
+
WHEN old.purge_on_settle = 0
|
|
426
|
+
BEGIN
|
|
427
|
+
UPDATE ${__QUEUE_STATS}
|
|
428
|
+
SET backlog_count = backlog_count - 1,
|
|
429
|
+
backlog_bytes = backlog_bytes - old.size,
|
|
430
|
+
oldest_message_timestamp = CASE
|
|
431
|
+
WHEN backlog_count <= 1 THEN NULL
|
|
432
|
+
WHEN oldest_message_timestamp = old.enqueued_at
|
|
433
|
+
THEN (SELECT MIN(enqueued_at) FROM ${__QUEUE_MESSAGES}
|
|
434
|
+
WHERE purge_on_settle = 0)
|
|
435
|
+
ELSE oldest_message_timestamp END
|
|
436
|
+
WHERE id = 1;
|
|
437
|
+
END`,
|
|
438
|
+
);
|
|
439
|
+
sql.exec(
|
|
440
|
+
`CREATE TRIGGER IF NOT EXISTS ${__QUEUE_MESSAGES}_stats_exclude
|
|
441
|
+
AFTER UPDATE OF purge_on_settle ON ${__QUEUE_MESSAGES}
|
|
442
|
+
WHEN old.purge_on_settle = 0 AND new.purge_on_settle != 0
|
|
443
|
+
BEGIN
|
|
444
|
+
UPDATE ${__QUEUE_STATS}
|
|
445
|
+
SET backlog_count = backlog_count - 1,
|
|
446
|
+
backlog_bytes = backlog_bytes - old.size,
|
|
447
|
+
oldest_message_timestamp = CASE
|
|
448
|
+
WHEN backlog_count <= 1 THEN NULL
|
|
449
|
+
WHEN oldest_message_timestamp = old.enqueued_at
|
|
450
|
+
THEN (SELECT MIN(enqueued_at) FROM ${__QUEUE_MESSAGES}
|
|
451
|
+
WHERE purge_on_settle = 0)
|
|
452
|
+
ELSE oldest_message_timestamp END
|
|
453
|
+
WHERE id = 1;
|
|
454
|
+
END`,
|
|
455
|
+
);
|
|
456
|
+
sql.exec(
|
|
457
|
+
`CREATE TRIGGER IF NOT EXISTS ${__QUEUE_MESSAGES}_stats_include
|
|
458
|
+
AFTER UPDATE OF purge_on_settle ON ${__QUEUE_MESSAGES}
|
|
459
|
+
WHEN old.purge_on_settle != 0 AND new.purge_on_settle = 0
|
|
460
|
+
BEGIN
|
|
461
|
+
UPDATE ${__QUEUE_STATS}
|
|
462
|
+
SET backlog_count = backlog_count + 1,
|
|
463
|
+
backlog_bytes = backlog_bytes + new.size,
|
|
464
|
+
oldest_message_timestamp = CASE
|
|
465
|
+
WHEN oldest_message_timestamp IS NULL
|
|
466
|
+
OR new.enqueued_at < oldest_message_timestamp
|
|
467
|
+
THEN new.enqueued_at ELSE oldest_message_timestamp END
|
|
468
|
+
WHERE id = 1;
|
|
469
|
+
END`,
|
|
470
|
+
);
|
|
471
|
+
sql.exec(
|
|
472
|
+
`CREATE TABLE IF NOT EXISTS ${__QUEUE_TRANSFER_RECEIPTS} (
|
|
473
|
+
transfer_id TEXT PRIMARY KEY,
|
|
474
|
+
message_id TEXT NOT NULL,
|
|
475
|
+
dead_letter_source TEXT,
|
|
476
|
+
expires_at INTEGER NOT NULL
|
|
477
|
+
) WITHOUT ROWID`,
|
|
478
|
+
);
|
|
479
|
+
sql.exec(
|
|
480
|
+
`CREATE INDEX IF NOT EXISTS ${__QUEUE_TRANSFER_RECEIPTS}_expires
|
|
481
|
+
ON ${__QUEUE_TRANSFER_RECEIPTS} (expires_at, transfer_id)`,
|
|
482
|
+
);
|
|
483
|
+
this._ready = true;
|
|
484
|
+
return sql;
|
|
485
|
+
}
|
|
486
|
+
|
|
487
|
+
_meta(name) {
|
|
488
|
+
return this._open().exec(
|
|
489
|
+
`SELECT value FROM ${__QUEUE_META} WHERE name = ?`, name,
|
|
490
|
+
).toArray()[0]?.value;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
_setMeta(name, value) {
|
|
494
|
+
if (value === null || value === undefined) {
|
|
495
|
+
this._open().exec(`DELETE FROM ${__QUEUE_META} WHERE name = ?`, name);
|
|
496
|
+
return;
|
|
497
|
+
}
|
|
498
|
+
this._open().exec(
|
|
499
|
+
`INSERT INTO ${__QUEUE_META} (name, value) VALUES (?, ?)
|
|
500
|
+
ON CONFLICT(name) DO UPDATE SET value = excluded.value`,
|
|
501
|
+
name,
|
|
502
|
+
String(value),
|
|
503
|
+
);
|
|
504
|
+
}
|
|
505
|
+
|
|
506
|
+
_paused() {
|
|
507
|
+
return this._meta("paused") === "1";
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
_metrics(now = Date.now(), waitingOnly = false) {
|
|
511
|
+
const expiresBefore = now - __queueLimits().retentionMs;
|
|
512
|
+
const sql = this._open();
|
|
513
|
+
const stats = sql.exec(
|
|
514
|
+
`SELECT backlog_count, backlog_bytes, oldest_message_timestamp
|
|
515
|
+
FROM ${__QUEUE_STATS} WHERE id = 1`,
|
|
516
|
+
).toArray()[0];
|
|
517
|
+
// Retention is a clock transition, not a row transition. Until the
|
|
518
|
+
// bounded sweep catches up, use the exact query only at that boundary;
|
|
519
|
+
// the normal hot path remains one materialized row at every depth.
|
|
520
|
+
if (stats.oldest_message_timestamp !== null &&
|
|
521
|
+
stats.oldest_message_timestamp <= expiresBefore) {
|
|
522
|
+
const row = sql.exec(
|
|
523
|
+
`SELECT COUNT(*) AS backlog_count,
|
|
524
|
+
COALESCE(SUM(size), 0) AS backlog_bytes,
|
|
525
|
+
MIN(enqueued_at) AS oldest_message_timestamp
|
|
526
|
+
FROM ${__QUEUE_MESSAGES}
|
|
527
|
+
WHERE purge_on_settle = 0 AND enqueued_at > ?
|
|
528
|
+
AND (? = 0 OR (lease_id IS NULL AND dlq_target IS NULL
|
|
529
|
+
AND visible_at <= ?))`,
|
|
530
|
+
expiresBefore,
|
|
531
|
+
waitingOnly ? 1 : 0,
|
|
532
|
+
now,
|
|
533
|
+
).toArray()[0];
|
|
534
|
+
return {
|
|
535
|
+
backlogCount: row.backlog_count,
|
|
536
|
+
backlogBytes: row.backlog_bytes,
|
|
537
|
+
oldestMessageTimestamp: row.oldest_message_timestamp ?? null,
|
|
538
|
+
};
|
|
539
|
+
}
|
|
540
|
+
if (!waitingOnly) {
|
|
541
|
+
if (typeof __test_queue_metrics_materialized === "function") {
|
|
542
|
+
__test_queue_metrics_materialized();
|
|
543
|
+
}
|
|
544
|
+
return {
|
|
545
|
+
backlogCount: stats.backlog_count,
|
|
546
|
+
backlogBytes: stats.backlog_bytes,
|
|
547
|
+
oldestMessageTimestamp: stats.oldest_message_timestamp ?? null,
|
|
548
|
+
};
|
|
549
|
+
}
|
|
550
|
+
const blocked = sql.exec(
|
|
551
|
+
`SELECT COALESCE(SUM(blocked_count), 0) AS blocked_count,
|
|
552
|
+
COALESCE(SUM(blocked_bytes), 0) AS blocked_bytes
|
|
553
|
+
FROM (
|
|
554
|
+
SELECT COUNT(*) AS blocked_count,
|
|
555
|
+
COALESCE(SUM(size), 0) AS blocked_bytes
|
|
556
|
+
FROM ${__QUEUE_MESSAGES} INDEXED BY ${__QUEUE_MESSAGES}_lease
|
|
557
|
+
WHERE purge_on_settle = 0 AND lease_id IS NOT NULL
|
|
558
|
+
UNION ALL
|
|
559
|
+
SELECT COUNT(*) AS blocked_count,
|
|
560
|
+
COALESCE(SUM(size), 0) AS blocked_bytes
|
|
561
|
+
FROM ${__QUEUE_MESSAGES} INDEXED BY ${__QUEUE_MESSAGES}_moving
|
|
562
|
+
WHERE purge_on_settle = 0 AND dlq_target IS NOT NULL
|
|
563
|
+
UNION ALL
|
|
564
|
+
SELECT COUNT(*) AS blocked_count,
|
|
565
|
+
COALESCE(SUM(size), 0) AS blocked_bytes
|
|
566
|
+
FROM ${__QUEUE_MESSAGES} INDEXED BY ${__QUEUE_MESSAGES}_ready
|
|
567
|
+
WHERE purge_on_settle = 0 AND dlq_target IS NULL
|
|
568
|
+
AND lease_id IS NULL AND visible_at > ?
|
|
569
|
+
)`,
|
|
570
|
+
now,
|
|
571
|
+
).toArray()[0];
|
|
572
|
+
const unleasedOldest = sql.exec(
|
|
573
|
+
`SELECT enqueued_at
|
|
574
|
+
FROM ${__QUEUE_MESSAGES} INDEXED BY ${__QUEUE_MESSAGES}_retained
|
|
575
|
+
WHERE purge_on_settle = 0 AND dlq_target IS NULL
|
|
576
|
+
AND lease_id IS NULL AND visible_at <= ?
|
|
577
|
+
ORDER BY enqueued_at, seq
|
|
578
|
+
LIMIT 1`,
|
|
579
|
+
now,
|
|
580
|
+
).toArray()[0]?.enqueued_at ?? null;
|
|
581
|
+
return {
|
|
582
|
+
backlogCount: stats.backlog_count - blocked.blocked_count,
|
|
583
|
+
backlogBytes: stats.backlog_bytes - blocked.blocked_bytes,
|
|
584
|
+
oldestMessageTimestamp: unleasedOldest,
|
|
585
|
+
};
|
|
586
|
+
}
|
|
587
|
+
|
|
588
|
+
async _rearm(now = Date.now(), waitForWake = false) {
|
|
589
|
+
const sql = this._open();
|
|
590
|
+
const consumer = __cell.queueConsumers?.[this.queue];
|
|
591
|
+
const paused = this._paused();
|
|
592
|
+
const active = sql.exec(
|
|
593
|
+
`SELECT COUNT(DISTINCT lease_id) AS count
|
|
594
|
+
FROM ${__QUEUE_MESSAGES} INDEXED BY ${__QUEUE_MESSAGES}_lease
|
|
595
|
+
WHERE lease_id IS NOT NULL AND leased_until > ?`,
|
|
596
|
+
now,
|
|
597
|
+
).toArray()[0].count;
|
|
598
|
+
const concurrency = consumer
|
|
599
|
+
? consumer.maxConcurrency ?? __queueLimits().maxConcurrency
|
|
600
|
+
: 0;
|
|
601
|
+
const hasCapacity = !paused && consumer !== undefined && __queuePolicy({
|
|
602
|
+
op: "capacity",
|
|
603
|
+
active,
|
|
604
|
+
maximum: concurrency,
|
|
605
|
+
});
|
|
606
|
+
// We need only enough ready rows to distinguish a partial batch from a
|
|
607
|
+
// full one. The two indexed arms cover a new row and an expired lease,
|
|
608
|
+
// and the outer LIMIT keeps the work independent of backlog depth.
|
|
609
|
+
let ready = [];
|
|
610
|
+
if (hasCapacity) {
|
|
611
|
+
const visibleReady = sql.exec(
|
|
612
|
+
`SELECT visible_at AS ready_at, seq
|
|
613
|
+
FROM ${__QUEUE_MESSAGES} INDEXED BY ${__QUEUE_MESSAGES}_ready
|
|
614
|
+
WHERE purge_on_settle = 0 AND dlq_target IS NULL
|
|
615
|
+
AND lease_id IS NULL AND visible_at <= ?
|
|
616
|
+
ORDER BY visible_at, seq
|
|
617
|
+
LIMIT ?`,
|
|
618
|
+
now,
|
|
619
|
+
consumer.maxBatchSize,
|
|
620
|
+
).toArray();
|
|
621
|
+
const expiredReady = sql.exec(
|
|
622
|
+
`SELECT leased_until AS ready_at, seq
|
|
623
|
+
FROM ${__QUEUE_MESSAGES} INDEXED BY ${__QUEUE_MESSAGES}_lease
|
|
624
|
+
WHERE purge_on_settle = 0 AND dlq_target IS NULL
|
|
625
|
+
AND lease_id IS NOT NULL AND leased_until <= ?
|
|
626
|
+
ORDER BY leased_until, seq
|
|
627
|
+
LIMIT ?`,
|
|
628
|
+
now,
|
|
629
|
+
consumer.maxBatchSize,
|
|
630
|
+
).toArray();
|
|
631
|
+
ready = [...visibleReady, ...expiredReady]
|
|
632
|
+
.sort((left, right) => left.ready_at - right.ready_at || left.seq - right.seq)
|
|
633
|
+
.slice(0, consumer.maxBatchSize);
|
|
634
|
+
}
|
|
635
|
+
if (typeof __test_queue_rearm_bounded === "function") {
|
|
636
|
+
__test_queue_rearm_bounded(
|
|
637
|
+
ready.length <= (consumer?.maxBatchSize ?? 0),
|
|
638
|
+
ready.length > 0,
|
|
639
|
+
);
|
|
640
|
+
}
|
|
641
|
+
const earliestVisible = hasCapacity
|
|
642
|
+
? sql.exec(
|
|
643
|
+
`SELECT visible_at
|
|
644
|
+
FROM ${__QUEUE_MESSAGES} INDEXED BY ${__QUEUE_MESSAGES}_ready
|
|
645
|
+
WHERE purge_on_settle = 0 AND dlq_target IS NULL
|
|
646
|
+
AND lease_id IS NULL AND visible_at > ?
|
|
647
|
+
ORDER BY visible_at, seq
|
|
648
|
+
LIMIT 1`,
|
|
649
|
+
now,
|
|
650
|
+
).toArray()[0]?.visible_at ?? null
|
|
651
|
+
: null;
|
|
652
|
+
const earliestLeaseExpiry = paused
|
|
653
|
+
? null
|
|
654
|
+
: sql.exec(
|
|
655
|
+
`SELECT leased_until
|
|
656
|
+
FROM ${__QUEUE_MESSAGES} INDEXED BY ${__QUEUE_MESSAGES}_lease
|
|
657
|
+
WHERE lease_id IS NOT NULL
|
|
658
|
+
ORDER BY leased_until
|
|
659
|
+
LIMIT 1`,
|
|
660
|
+
).toArray()[0]?.leased_until ?? null;
|
|
661
|
+
const nextSweep = sql.exec(
|
|
662
|
+
`SELECT enqueued_at + ? AS next_sweep
|
|
663
|
+
FROM ${__QUEUE_MESSAGES} INDEXED BY ${__QUEUE_MESSAGES}_retained
|
|
664
|
+
WHERE dlq_target IS NULL AND purge_on_settle = 0
|
|
665
|
+
ORDER BY enqueued_at, seq
|
|
666
|
+
LIMIT 1`,
|
|
667
|
+
__queueLimits().retentionMs,
|
|
668
|
+
).toArray()[0]?.next_sweep ?? null;
|
|
669
|
+
const dlqPending = sql.exec(
|
|
670
|
+
`SELECT 1 AS pending
|
|
671
|
+
FROM ${__QUEUE_MESSAGES} INDEXED BY ${__QUEUE_MESSAGES}_moving
|
|
672
|
+
WHERE dlq_target IS NOT NULL
|
|
673
|
+
LIMIT 1`,
|
|
674
|
+
).toArray().length !== 0;
|
|
675
|
+
const receiptSweep = sql.exec(
|
|
676
|
+
`SELECT expires_at AS next_sweep
|
|
677
|
+
FROM ${__QUEUE_TRANSFER_RECEIPTS}
|
|
678
|
+
ORDER BY expires_at, transfer_id
|
|
679
|
+
LIMIT 1`,
|
|
680
|
+
).toArray()[0]?.next_sweep ?? null;
|
|
681
|
+
// A queue without an attached consumer only wakes to reclaim retention.
|
|
682
|
+
// Consumer attachment will nudge the cell when the fleet catalog gains a
|
|
683
|
+
// registration; spinning on an already-visible row cannot make progress.
|
|
684
|
+
let batchDeadline = null;
|
|
685
|
+
if (ready.length > 0) {
|
|
686
|
+
batchDeadline = ready.length >= consumer.maxBatchSize
|
|
687
|
+
? now
|
|
688
|
+
: ready[0].ready_at + consumer.maxBatchTimeoutMs;
|
|
689
|
+
}
|
|
690
|
+
if (dlqPending) {
|
|
691
|
+
batchDeadline = batchDeadline === null ? now : Math.min(batchDeadline, now);
|
|
692
|
+
}
|
|
693
|
+
const sweepDeadlines = [nextSweep, receiptSweep]
|
|
694
|
+
.filter((deadline) => deadline !== null && deadline !== undefined);
|
|
695
|
+
const armAt = __queuePolicy({
|
|
696
|
+
op: "rearm",
|
|
697
|
+
now,
|
|
698
|
+
batchDeadline,
|
|
699
|
+
earliestVisible,
|
|
700
|
+
// A live handler can settle after delivery pauses. Do not reclaim its
|
|
701
|
+
// lease while paused; resume re-arms the expired deadline immediately
|
|
702
|
+
// and only then can a new generation replace it.
|
|
703
|
+
earliestLeaseExpiry,
|
|
704
|
+
nextSweep: sweepDeadlines.length === 0 ? null : Math.min(...sweepDeadlines),
|
|
705
|
+
});
|
|
706
|
+
if (armAt !== null && !Number.isFinite(armAt)) {
|
|
707
|
+
throw __queueError("the Queue sweep deadline is invalid");
|
|
708
|
+
}
|
|
709
|
+
if (armAt === null) await this.ctx.storage.deleteAlarm();
|
|
710
|
+
else if (waitForWake) {
|
|
711
|
+
await __queue_alarm_set_wait(this.ctx.storage._scope, armAt);
|
|
712
|
+
} else await this.ctx.storage.setAlarm(armAt);
|
|
713
|
+
}
|
|
714
|
+
|
|
715
|
+
_newProducerGroup() {
|
|
716
|
+
const group = {
|
|
717
|
+
calls: [],
|
|
718
|
+
entries: [],
|
|
719
|
+
bytes: 0,
|
|
720
|
+
flushing: false,
|
|
721
|
+
timer: null,
|
|
722
|
+
};
|
|
723
|
+
this._producerGroup = group;
|
|
724
|
+
group.timer = setTimeout(() => {
|
|
725
|
+
void this._flushProducerGroup(group);
|
|
726
|
+
}, __queueLimits().producerGroupMs);
|
|
727
|
+
return group;
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
async _flushProducerGroup(group) {
|
|
731
|
+
if (group.flushing) return;
|
|
732
|
+
group.flushing = true;
|
|
733
|
+
if (this._producerGroup === group) this._producerGroup = null;
|
|
734
|
+
if (group.timer !== null) {
|
|
735
|
+
clearTimeout(group.timer);
|
|
736
|
+
group.timer = null;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
const now = Date.now();
|
|
740
|
+
try {
|
|
741
|
+
const sql = this._open();
|
|
742
|
+
this.ctx.storage.transactionSync(() => {
|
|
743
|
+
for (const entry of group.entries) {
|
|
744
|
+
sql.exec(
|
|
745
|
+
`INSERT INTO ${__QUEUE_MESSAGES}
|
|
746
|
+
(id, body, content_type, size, enqueued_at, visible_at)
|
|
747
|
+
VALUES (?, ?, ?, ?, ?, ?)`,
|
|
748
|
+
__queueMessageId(now),
|
|
749
|
+
entry.body,
|
|
750
|
+
entry.contentType,
|
|
751
|
+
entry.size,
|
|
752
|
+
now,
|
|
753
|
+
now + entry.delaySeconds * 1000,
|
|
754
|
+
);
|
|
755
|
+
}
|
|
756
|
+
});
|
|
757
|
+
await this._rearm(now, true);
|
|
758
|
+
const metrics = this._metrics(now);
|
|
759
|
+
if (typeof __test_queue_producer_group === "function") {
|
|
760
|
+
__test_queue_producer_group(group.calls.length, group.entries.length);
|
|
761
|
+
}
|
|
762
|
+
for (const call of group.calls) call.resolve(metrics);
|
|
763
|
+
} catch (error) {
|
|
764
|
+
for (const call of group.calls) call.reject(error);
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
__queueSend({ entries }) {
|
|
769
|
+
const bytes = entries.reduce((total, entry) => total + entry.size, 0);
|
|
770
|
+
return new Promise((resolve, reject) => {
|
|
771
|
+
let group = this._producerGroup;
|
|
772
|
+
if (group !== null &&
|
|
773
|
+
(group.calls.length + 1 > __QUEUE_PRODUCER_GROUP_CALLS ||
|
|
774
|
+
group.entries.length + entries.length > __QUEUE_PRODUCER_GROUP_MESSAGES ||
|
|
775
|
+
group.bytes + bytes > __QUEUE_PRODUCER_GROUP_BYTES)) {
|
|
776
|
+
void this._flushProducerGroup(group);
|
|
777
|
+
group = null;
|
|
778
|
+
}
|
|
779
|
+
if (group === null) group = this._newProducerGroup();
|
|
780
|
+
group.calls.push({ resolve, reject });
|
|
781
|
+
group.entries.push(...entries);
|
|
782
|
+
group.bytes += bytes;
|
|
783
|
+
if (group.calls.length === __QUEUE_PRODUCER_GROUP_CALLS ||
|
|
784
|
+
group.entries.length === __QUEUE_PRODUCER_GROUP_MESSAGES ||
|
|
785
|
+
group.bytes === __QUEUE_PRODUCER_GROUP_BYTES) {
|
|
786
|
+
void this._flushProducerGroup(group);
|
|
787
|
+
}
|
|
788
|
+
});
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
__queueMetrics() {
|
|
792
|
+
return this._metrics();
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
// Retention decides visibility through `_metrics` immediately, while this
|
|
796
|
+
// bounded sweep only reclaims space. A live lease is marked instead of
|
|
797
|
+
// deleted, so its handler can finish but settlement or expiry removes it
|
|
798
|
+
// without a redelivery.
|
|
799
|
+
_sweep(now) {
|
|
800
|
+
const sql = this._open();
|
|
801
|
+
const cutoff = now - __queueLimits().retentionMs;
|
|
802
|
+
const messages = __cellSweepBatch(
|
|
803
|
+
this.ctx.storage,
|
|
804
|
+
(limit) => sql.exec(
|
|
805
|
+
`SELECT seq, lease_id, leased_until FROM ${__QUEUE_MESSAGES}
|
|
806
|
+
WHERE enqueued_at <= ?
|
|
807
|
+
AND (purge_on_settle = 0
|
|
808
|
+
OR lease_id IS NULL OR leased_until <= ?)
|
|
809
|
+
ORDER BY enqueued_at, seq
|
|
810
|
+
LIMIT ?`,
|
|
811
|
+
cutoff,
|
|
812
|
+
now,
|
|
813
|
+
limit,
|
|
814
|
+
).toArray(),
|
|
815
|
+
(row) => {
|
|
816
|
+
if (row.lease_id !== null && row.leased_until > now) {
|
|
817
|
+
sql.exec(
|
|
818
|
+
`UPDATE ${__QUEUE_MESSAGES} SET purge_on_settle = 1 WHERE seq = ?`,
|
|
819
|
+
row.seq,
|
|
820
|
+
);
|
|
821
|
+
} else {
|
|
822
|
+
sql.exec(`DELETE FROM ${__QUEUE_MESSAGES} WHERE seq = ?`, row.seq);
|
|
823
|
+
}
|
|
824
|
+
},
|
|
825
|
+
__queueLimits().sweepBatchRows,
|
|
826
|
+
);
|
|
827
|
+
const remaining = __queueLimits().sweepBatchRows - messages;
|
|
828
|
+
if (remaining === 0) return messages;
|
|
829
|
+
const receipts = __cellSweepBatch(
|
|
830
|
+
this.ctx.storage,
|
|
831
|
+
(limit) => sql.exec(
|
|
832
|
+
`SELECT transfer_id FROM ${__QUEUE_TRANSFER_RECEIPTS}
|
|
833
|
+
WHERE expires_at <= ?
|
|
834
|
+
ORDER BY expires_at, transfer_id
|
|
835
|
+
LIMIT ?`,
|
|
836
|
+
now,
|
|
837
|
+
limit,
|
|
838
|
+
).toArray(),
|
|
839
|
+
(row) => {
|
|
840
|
+
sql.exec(
|
|
841
|
+
`DELETE FROM ${__QUEUE_TRANSFER_RECEIPTS} WHERE transfer_id = ?`,
|
|
842
|
+
row.transfer_id,
|
|
843
|
+
);
|
|
844
|
+
},
|
|
845
|
+
remaining,
|
|
846
|
+
);
|
|
847
|
+
return messages + receipts;
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
_candidateRows(now, limit) {
|
|
851
|
+
return this._open().exec(
|
|
852
|
+
`SELECT seq, id, body, content_type, size, enqueued_at, visible_at,
|
|
853
|
+
attempts, lease_id, lease_generation, leased_until,
|
|
854
|
+
purge_on_settle
|
|
855
|
+
FROM ${__QUEUE_MESSAGES}
|
|
856
|
+
WHERE dlq_target IS NULL AND ((purge_on_settle = 1
|
|
857
|
+
AND (lease_id IS NULL OR leased_until <= ?))
|
|
858
|
+
OR (purge_on_settle = 0 AND visible_at <= ?
|
|
859
|
+
AND (lease_id IS NULL OR leased_until <= ?)))
|
|
860
|
+
ORDER BY visible_at, seq
|
|
861
|
+
LIMIT ?`,
|
|
862
|
+
now,
|
|
863
|
+
now,
|
|
864
|
+
now,
|
|
865
|
+
limit,
|
|
866
|
+
).toArray();
|
|
867
|
+
}
|
|
868
|
+
|
|
869
|
+
async _leaseBatch(now, consumer) {
|
|
870
|
+
const sql = this._open();
|
|
871
|
+
const active = sql.exec(
|
|
872
|
+
`SELECT COUNT(DISTINCT lease_id) AS count
|
|
873
|
+
FROM ${__QUEUE_MESSAGES}
|
|
874
|
+
WHERE lease_id IS NOT NULL AND leased_until > ?`,
|
|
875
|
+
now,
|
|
876
|
+
).toArray()[0].count;
|
|
877
|
+
const concurrency = consumer.maxConcurrency ?? __queueLimits().maxConcurrency;
|
|
878
|
+
if (!__queuePolicy({ op: "capacity", active, maximum: concurrency })) return null;
|
|
879
|
+
|
|
880
|
+
// Read extra rows so expired purge markers cannot hide a deliverable
|
|
881
|
+
// batch. Policy chooses at most `maxBatchSize` rows and returns every
|
|
882
|
+
// purge deletion that precedes them in this bounded scan.
|
|
883
|
+
const rows = this._candidateRows(now, consumer.maxBatchSize + 256);
|
|
884
|
+
const deliverable = rows.filter((row) => row.purge_on_settle === 0);
|
|
885
|
+
if (deliverable.length === 0) {
|
|
886
|
+
if (rows.length > 0) {
|
|
887
|
+
this.ctx.storage.transactionSync(() => {
|
|
888
|
+
for (const row of rows) {
|
|
889
|
+
sql.exec(`DELETE FROM ${__QUEUE_MESSAGES} WHERE seq = ?`, row.seq);
|
|
890
|
+
}
|
|
891
|
+
});
|
|
892
|
+
}
|
|
893
|
+
return null;
|
|
894
|
+
}
|
|
895
|
+
const readySince = Math.min(...deliverable.map((row) =>
|
|
896
|
+
row.lease_id === null ? row.visible_at : row.leased_until
|
|
897
|
+
));
|
|
898
|
+
if (deliverable.length < consumer.maxBatchSize &&
|
|
899
|
+
readySince + consumer.maxBatchTimeoutMs > now) {
|
|
900
|
+
return null;
|
|
901
|
+
}
|
|
902
|
+
|
|
903
|
+
const plan = __queuePolicy({
|
|
904
|
+
op: "batch",
|
|
905
|
+
now,
|
|
906
|
+
maxBatchSize: consumer.maxBatchSize,
|
|
907
|
+
rows: rows.map((row) => ({
|
|
908
|
+
seq: row.seq,
|
|
909
|
+
visibleAt: row.visible_at,
|
|
910
|
+
leaseGeneration: String(row.lease_generation),
|
|
911
|
+
leasedUntil: row.leased_until,
|
|
912
|
+
purgeOnSettle: row.purge_on_settle !== 0,
|
|
913
|
+
})),
|
|
914
|
+
});
|
|
915
|
+
if (plan.leases.length === 0 && plan.deletePurged.length === 0) return null;
|
|
916
|
+
const rowBySeq = new Map(rows.map((row) => [row.seq, row]));
|
|
917
|
+
const reclaimed = plan.leases.filter((lease) => lease.reclaimed);
|
|
918
|
+
const expiryPolicy = __queuePolicy({
|
|
919
|
+
op: "expiry",
|
|
920
|
+
now,
|
|
921
|
+
entries: reclaimed.map((lease) => {
|
|
922
|
+
const row = rowBySeq.get(lease.seq);
|
|
923
|
+
return {
|
|
924
|
+
priorFailures: row.attempts,
|
|
925
|
+
maxRetries: consumer.maxRetries,
|
|
926
|
+
configuredSeconds: consumer.retryDelaySeconds ?? null,
|
|
927
|
+
purgeOnSettle: row.purge_on_settle !== 0,
|
|
928
|
+
};
|
|
929
|
+
}),
|
|
930
|
+
});
|
|
931
|
+
const expiryBySeq = new Map(
|
|
932
|
+
reclaimed.map((lease, index) => [lease.seq, expiryPolicy[index]]),
|
|
933
|
+
);
|
|
934
|
+
const installs = plan.leases.filter((lease) => {
|
|
935
|
+
const expiry = expiryBySeq.get(lease.seq);
|
|
936
|
+
return expiry === undefined ||
|
|
937
|
+
(expiry.action.kind === "retry" && expiry.action.at <= now);
|
|
938
|
+
});
|
|
939
|
+
const exhaustedTransfers = new Map(
|
|
940
|
+
reclaimed
|
|
941
|
+
.filter((lease) =>
|
|
942
|
+
expiryBySeq.get(lease.seq).action.kind === "exhausted" &&
|
|
943
|
+
consumer.deadLetterQueue !== undefined
|
|
944
|
+
)
|
|
945
|
+
.map((lease) => [lease.seq, crypto.randomUUID()]),
|
|
946
|
+
);
|
|
947
|
+
const leaseId = crypto.randomUUID();
|
|
948
|
+
const leasedUntil = Math.min(
|
|
949
|
+
Number.MAX_SAFE_INTEGER,
|
|
950
|
+
now + __cell.queueLeaseDurationMs,
|
|
951
|
+
);
|
|
952
|
+
this.ctx.storage.transactionSync(() => {
|
|
953
|
+
for (const seq of plan.deletePurged) {
|
|
954
|
+
sql.exec(
|
|
955
|
+
`DELETE FROM ${__QUEUE_MESSAGES}
|
|
956
|
+
WHERE seq = ? AND purge_on_settle = 1
|
|
957
|
+
AND (lease_id IS NULL OR leased_until <= ?)`,
|
|
958
|
+
seq,
|
|
959
|
+
now,
|
|
960
|
+
);
|
|
961
|
+
}
|
|
962
|
+
for (const lease of reclaimed) {
|
|
963
|
+
const expiry = expiryBySeq.get(lease.seq);
|
|
964
|
+
if (expiry.action.kind === "delete-purged") {
|
|
965
|
+
sql.exec(
|
|
966
|
+
`DELETE FROM ${__QUEUE_MESSAGES}
|
|
967
|
+
WHERE seq = ? AND leased_until <= ?`,
|
|
968
|
+
lease.seq,
|
|
969
|
+
now,
|
|
970
|
+
);
|
|
971
|
+
} else if (expiry.action.kind === "exhausted") {
|
|
972
|
+
if (consumer.deadLetterQueue === undefined) {
|
|
973
|
+
sql.exec(
|
|
974
|
+
`DELETE FROM ${__QUEUE_MESSAGES}
|
|
975
|
+
WHERE seq = ? AND leased_until <= ?`,
|
|
976
|
+
lease.seq,
|
|
977
|
+
now,
|
|
978
|
+
);
|
|
979
|
+
} else {
|
|
980
|
+
sql.exec(
|
|
981
|
+
`UPDATE ${__QUEUE_MESSAGES}
|
|
982
|
+
SET attempts = ?, lease_id = NULL, leased_until = NULL,
|
|
983
|
+
dlq_target = ?, dlq_transfer_id = ?,
|
|
984
|
+
transfer_kind = 'dead-letter'
|
|
985
|
+
WHERE seq = ? AND leased_until <= ?`,
|
|
986
|
+
expiry.attempt,
|
|
987
|
+
consumer.deadLetterQueue,
|
|
988
|
+
exhaustedTransfers.get(lease.seq),
|
|
989
|
+
lease.seq,
|
|
990
|
+
now,
|
|
991
|
+
);
|
|
992
|
+
}
|
|
993
|
+
} else if (expiry.action.at > now) {
|
|
994
|
+
sql.exec(
|
|
995
|
+
`UPDATE ${__QUEUE_MESSAGES}
|
|
996
|
+
SET attempts = ?, visible_at = ?, lease_id = NULL,
|
|
997
|
+
leased_until = NULL
|
|
998
|
+
WHERE seq = ? AND leased_until <= ?`,
|
|
999
|
+
expiry.attempt,
|
|
1000
|
+
expiry.action.at,
|
|
1001
|
+
lease.seq,
|
|
1002
|
+
now,
|
|
1003
|
+
);
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
for (const lease of installs) {
|
|
1007
|
+
const generation = Number(lease.generation);
|
|
1008
|
+
if (!Number.isSafeInteger(generation)) {
|
|
1009
|
+
throw __queueError(`message ${lease.seq} exhausted its safe lease generation`);
|
|
1010
|
+
}
|
|
1011
|
+
sql.exec(
|
|
1012
|
+
`UPDATE ${__QUEUE_MESSAGES}
|
|
1013
|
+
SET lease_id = ?, lease_generation = ?, leased_until = ?,
|
|
1014
|
+
attempts = attempts + ?
|
|
1015
|
+
WHERE seq = ? AND lease_generation = ?
|
|
1016
|
+
AND (lease_id IS NULL OR leased_until <= ?)`,
|
|
1017
|
+
leaseId,
|
|
1018
|
+
generation,
|
|
1019
|
+
leasedUntil,
|
|
1020
|
+
lease.reclaimed ? 1 : 0,
|
|
1021
|
+
lease.seq,
|
|
1022
|
+
generation - 1,
|
|
1023
|
+
now,
|
|
1024
|
+
);
|
|
1025
|
+
}
|
|
1026
|
+
});
|
|
1027
|
+
if (installs.length === 0) return null;
|
|
1028
|
+
const leasedQuery =
|
|
1029
|
+
`SELECT seq, id, body, content_type, enqueued_at, attempts,
|
|
1030
|
+
lease_generation
|
|
1031
|
+
FROM ${__QUEUE_MESSAGES}
|
|
1032
|
+
WHERE lease_id = ?
|
|
1033
|
+
ORDER BY visible_at, seq`;
|
|
1034
|
+
__queueReportLeaseLookupPlan(sql, leasedQuery, leaseId);
|
|
1035
|
+
const leased = sql.exec(leasedQuery, leaseId).toArray();
|
|
1036
|
+
if (leased.length !== installs.length) {
|
|
1037
|
+
throw __queueError("the Queue lease transaction changed fewer rows than its plan");
|
|
1038
|
+
}
|
|
1039
|
+
return {
|
|
1040
|
+
leaseId,
|
|
1041
|
+
leases: leased.map((row) => ({
|
|
1042
|
+
message_id: row.id,
|
|
1043
|
+
seq: row.seq,
|
|
1044
|
+
generation: row.lease_generation,
|
|
1045
|
+
})),
|
|
1046
|
+
messages: leased.map((row) => ({
|
|
1047
|
+
id: row.id,
|
|
1048
|
+
timestampMs: row.enqueued_at,
|
|
1049
|
+
// SQLite answers a BLOB as an ArrayBuffer. The raw base64 op accepts
|
|
1050
|
+
// typed views, so normalize here; otherwise it encodes the text
|
|
1051
|
+
// "[object ArrayBuffer]" and every consumer decoder sees corrupt data.
|
|
1052
|
+
bodyBase64: $$btoa(
|
|
1053
|
+
row.body instanceof Uint8Array ? row.body : new Uint8Array(row.body),
|
|
1054
|
+
),
|
|
1055
|
+
contentType: row.content_type,
|
|
1056
|
+
attempts: row.attempts + 1,
|
|
1057
|
+
})),
|
|
1058
|
+
// Miniflare removes every dispatched batch from its backlog before it
|
|
1059
|
+
// invokes the consumer. Exclude every lease here for the same contract;
|
|
1060
|
+
// the operator aggregate still includes durable in-flight messages.
|
|
1061
|
+
metrics: this._metrics(now, true),
|
|
1062
|
+
};
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
// The target deduplicates the stable transfer identity. The source can
|
|
1066
|
+
// therefore repeat this call after a stop between the target commit and the
|
|
1067
|
+
// source delete without creating a second message.
|
|
1068
|
+
async __queueDlqAcceptBatch({ transfers }) {
|
|
1069
|
+
if (!Array.isArray(transfers) || transfers.length === 0 ||
|
|
1070
|
+
transfers.length > __queueLimits().maxBatchMessages) {
|
|
1071
|
+
throw __queueError("a Queue transfer batch has an invalid size");
|
|
1072
|
+
}
|
|
1073
|
+
for (const transfer of transfers) {
|
|
1074
|
+
if (transfer.deadLetterSource !== null &&
|
|
1075
|
+
typeof transfer.deadLetterSource !== "string") {
|
|
1076
|
+
throw __queueError("a Queue transfer has an invalid dead-letter source");
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
const now = Date.now();
|
|
1080
|
+
const sql = this._open();
|
|
1081
|
+
this.ctx.storage.transactionSync(() => {
|
|
1082
|
+
for (const { transferId, entry, deadLetterSource } of transfers) {
|
|
1083
|
+
const accepted = sql.exec(
|
|
1084
|
+
`SELECT message_id, dead_letter_source
|
|
1085
|
+
FROM ${__QUEUE_TRANSFER_RECEIPTS}
|
|
1086
|
+
WHERE transfer_id = ?`,
|
|
1087
|
+
transferId,
|
|
1088
|
+
).toArray()[0];
|
|
1089
|
+
if (accepted !== undefined && (accepted.message_id !== entry.id ||
|
|
1090
|
+
accepted.dead_letter_source !== deadLetterSource)) {
|
|
1091
|
+
throw __queueError("a DLQ transfer collided with another message identity");
|
|
1092
|
+
}
|
|
1093
|
+
if (accepted !== undefined) continue;
|
|
1094
|
+
sql.exec(
|
|
1095
|
+
`INSERT INTO ${__QUEUE_TRANSFER_RECEIPTS}
|
|
1096
|
+
(transfer_id, message_id, dead_letter_source, expires_at)
|
|
1097
|
+
VALUES (?, ?, ?, ?)`,
|
|
1098
|
+
transferId,
|
|
1099
|
+
entry.id,
|
|
1100
|
+
deadLetterSource,
|
|
1101
|
+
now + __queueLimits().retentionMs,
|
|
1102
|
+
);
|
|
1103
|
+
sql.exec(
|
|
1104
|
+
`INSERT INTO ${__QUEUE_MESSAGES}
|
|
1105
|
+
(id, body, content_type, size, enqueued_at, visible_at,
|
|
1106
|
+
attempts, dlq_transfer_id, dead_letter_source)
|
|
1107
|
+
VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?)`,
|
|
1108
|
+
entry.id,
|
|
1109
|
+
entry.body,
|
|
1110
|
+
entry.contentType,
|
|
1111
|
+
entry.size,
|
|
1112
|
+
now,
|
|
1113
|
+
now,
|
|
1114
|
+
transferId,
|
|
1115
|
+
deadLetterSource,
|
|
1116
|
+
);
|
|
1117
|
+
}
|
|
1118
|
+
});
|
|
1119
|
+
await this._rearm(now);
|
|
1120
|
+
return this._metrics(now);
|
|
1121
|
+
}
|
|
1122
|
+
|
|
1123
|
+
async _driveDlq(limit = 100) {
|
|
1124
|
+
const now = Date.now();
|
|
1125
|
+
const pending = this._open().exec(
|
|
1126
|
+
`SELECT seq, id, body, content_type, size, dlq_target, dlq_transfer_id,
|
|
1127
|
+
transfer_kind
|
|
1128
|
+
FROM ${__QUEUE_MESSAGES}
|
|
1129
|
+
WHERE dlq_target IS NOT NULL
|
|
1130
|
+
AND enqueued_at > ?
|
|
1131
|
+
ORDER BY seq
|
|
1132
|
+
LIMIT ?`,
|
|
1133
|
+
now - __queueLimits().retentionMs,
|
|
1134
|
+
limit,
|
|
1135
|
+
).toArray();
|
|
1136
|
+
const groups = new Map();
|
|
1137
|
+
for (const row of pending) {
|
|
1138
|
+
let group = groups.get(row.dlq_target);
|
|
1139
|
+
if (group === undefined) groups.set(row.dlq_target, group = []);
|
|
1140
|
+
group.push(row);
|
|
1141
|
+
}
|
|
1142
|
+
for (const [targetName, rows] of groups) {
|
|
1143
|
+
const target = __cell.makeNamespace("__Queue").getByName(targetName);
|
|
1144
|
+
await target.__queueDlqAcceptBatch({
|
|
1145
|
+
transfers: rows.map((row) => ({
|
|
1146
|
+
transferId: row.dlq_transfer_id,
|
|
1147
|
+
// A dead-letter transfer records where redrive returns the message.
|
|
1148
|
+
// A redrive returns it to ordinary queue state, so the source marker
|
|
1149
|
+
// must not follow it and make a later redrive bounce it backwards.
|
|
1150
|
+
deadLetterSource: row.transfer_kind === "dead-letter" ? this.queue : null,
|
|
1151
|
+
entry: {
|
|
1152
|
+
id: row.id,
|
|
1153
|
+
body: row.body,
|
|
1154
|
+
contentType: row.content_type,
|
|
1155
|
+
size: row.size,
|
|
1156
|
+
},
|
|
1157
|
+
})),
|
|
1158
|
+
});
|
|
1159
|
+
if (typeof __test_queue_dlq_accepted === "function") {
|
|
1160
|
+
__test_queue_dlq_accepted();
|
|
1161
|
+
}
|
|
1162
|
+
// The target committed every receipt before this reply. Delete the
|
|
1163
|
+
// corresponding source marks together; a stop before this transaction
|
|
1164
|
+
// leaves the whole group replayable through those stable receipts.
|
|
1165
|
+
this.ctx.storage.transactionSync(() => {
|
|
1166
|
+
for (const row of rows) {
|
|
1167
|
+
this._open().exec(
|
|
1168
|
+
`DELETE FROM ${__QUEUE_MESSAGES}
|
|
1169
|
+
WHERE seq = ? AND dlq_target = ? AND dlq_transfer_id = ?
|
|
1170
|
+
AND transfer_kind = ?`,
|
|
1171
|
+
row.seq,
|
|
1172
|
+
row.dlq_target,
|
|
1173
|
+
row.dlq_transfer_id,
|
|
1174
|
+
row.transfer_kind,
|
|
1175
|
+
);
|
|
1176
|
+
}
|
|
1177
|
+
});
|
|
1178
|
+
}
|
|
1179
|
+
return pending.length;
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
async alarm() {
|
|
1183
|
+
const now = Date.now();
|
|
1184
|
+
try {
|
|
1185
|
+
await this._driveDlq();
|
|
1186
|
+
} catch (error) {
|
|
1187
|
+
// The durable source mark is the retry record. An alarm stays armed at
|
|
1188
|
+
// `now`, and the target's transfer identity makes a replay idempotent.
|
|
1189
|
+
console.error("Queue DLQ transfer failed:", error);
|
|
1190
|
+
}
|
|
1191
|
+
const swept = this._sweep(now);
|
|
1192
|
+
const consumer = this._paused()
|
|
1193
|
+
? undefined
|
|
1194
|
+
: __cell.queueConsumers?.[this.queue];
|
|
1195
|
+
// Install every lease that the configured concurrency admits before the
|
|
1196
|
+
// first consumer can settle. One lease per alarm serialized full batches
|
|
1197
|
+
// because the next alarm depended on a later scheduler turn, so a queue
|
|
1198
|
+
// with maxConcurrency > 1 still used only one handler under backlog.
|
|
1199
|
+
// `_leaseBatch` counts the durable lease IDs on every iteration, which
|
|
1200
|
+
// makes the configured concurrency the structural bound for this loop.
|
|
1201
|
+
const batches = [];
|
|
1202
|
+
if (consumer !== undefined) {
|
|
1203
|
+
while (true) {
|
|
1204
|
+
const batch = await this._leaseBatch(now, consumer);
|
|
1205
|
+
if (batch === null) break;
|
|
1206
|
+
batches.push(batch);
|
|
1207
|
+
}
|
|
1208
|
+
}
|
|
1209
|
+
try {
|
|
1210
|
+
// `_leaseBatch` can exhaust an expired lease and create a new durable
|
|
1211
|
+
// DLQ mark. Drive that mark in the same wake; the first call above is
|
|
1212
|
+
// for a mark recovered from an earlier stopped process.
|
|
1213
|
+
await this._driveDlq();
|
|
1214
|
+
} catch (error) {
|
|
1215
|
+
console.error("Queue DLQ transfer failed:", error);
|
|
1216
|
+
}
|
|
1217
|
+
await this._rearm(now);
|
|
1218
|
+
if (swept >= __queueLimits().sweepBatchRows && batches.length === 0) {
|
|
1219
|
+
await this.ctx.storage.setAlarm(now + 1);
|
|
1220
|
+
return;
|
|
1221
|
+
}
|
|
1222
|
+
for (const batch of batches) {
|
|
1223
|
+
// This await covers only the output-gated handoff to the host. The
|
|
1224
|
+
// consumer runs after this alarm returns, and settlement comes back as
|
|
1225
|
+
// an independent cell event. All admitted leases are durable before the
|
|
1226
|
+
// first handoff, so a fast settlement cannot make this turn exceed the
|
|
1227
|
+
// configured concurrency.
|
|
1228
|
+
await __queue_dispatch(
|
|
1229
|
+
consumer.script,
|
|
1230
|
+
this.queue,
|
|
1231
|
+
JSON.stringify(batch),
|
|
1232
|
+
);
|
|
1233
|
+
}
|
|
1234
|
+
}
|
|
1235
|
+
|
|
1236
|
+
_matchingLease(body) {
|
|
1237
|
+
if (typeof body.leaseId !== "string" || !Array.isArray(body.leases)) return null;
|
|
1238
|
+
const matchQuery =
|
|
1239
|
+
`SELECT seq, id, attempts, lease_generation, purge_on_settle
|
|
1240
|
+
FROM ${__QUEUE_MESSAGES}
|
|
1241
|
+
WHERE lease_id = ?
|
|
1242
|
+
ORDER BY seq`;
|
|
1243
|
+
__queueReportLeaseLookupPlan(this._open(), matchQuery, body.leaseId);
|
|
1244
|
+
const rows = this._open().exec(matchQuery, body.leaseId).toArray();
|
|
1245
|
+
for (const lease of body.leases) {
|
|
1246
|
+
if (typeof lease?.message_id !== "string" ||
|
|
1247
|
+
!Number.isSafeInteger(lease.seq) ||
|
|
1248
|
+
!Number.isSafeInteger(lease.generation)) return null;
|
|
1249
|
+
}
|
|
1250
|
+
const matches = __queuePolicy({
|
|
1251
|
+
op: "settlement",
|
|
1252
|
+
current: rows.map((row) => ({
|
|
1253
|
+
seq: String(row.seq),
|
|
1254
|
+
messageId: row.id,
|
|
1255
|
+
generation: String(row.lease_generation),
|
|
1256
|
+
})),
|
|
1257
|
+
submitted: body.leases.map((lease) => ({
|
|
1258
|
+
seq: String(lease.seq),
|
|
1259
|
+
messageId: lease.message_id,
|
|
1260
|
+
generation: String(lease.generation),
|
|
1261
|
+
})),
|
|
1262
|
+
});
|
|
1263
|
+
return matches ? rows : null;
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
async _settle(body) {
|
|
1267
|
+
const rows = this._matchingLease(body);
|
|
1268
|
+
if (rows === null) return false;
|
|
1269
|
+
const consumer = __cell.queueConsumers?.[this.queue];
|
|
1270
|
+
if (consumer === undefined) return false;
|
|
1271
|
+
const explicitAcks = new Set(body.explicitAcks ?? []);
|
|
1272
|
+
const retryMessages = new Map(
|
|
1273
|
+
(body.retryMessages ?? []).map((retry) => [retry.msgId, retry.delaySeconds]),
|
|
1274
|
+
);
|
|
1275
|
+
const retryBatch = body.retryBatch?.retry === true;
|
|
1276
|
+
const batchDelay = body.retryBatch?.delaySeconds;
|
|
1277
|
+
const exception = body.outcome === "exception";
|
|
1278
|
+
// An explicit ack-all survives a later handler exception, so it must
|
|
1279
|
+
// override every batch-level and message-level retry signal.
|
|
1280
|
+
const ackAll = body.ackAll === true;
|
|
1281
|
+
const decisions = rows.map((row) => {
|
|
1282
|
+
const retry = !ackAll && !explicitAcks.has(row.id) && (
|
|
1283
|
+
retryMessages.has(row.id) || retryBatch || exception
|
|
1284
|
+
);
|
|
1285
|
+
return { row, retry };
|
|
1286
|
+
});
|
|
1287
|
+
const retrying = decisions.filter((decision) =>
|
|
1288
|
+
decision.retry && decision.row.purge_on_settle === 0
|
|
1289
|
+
);
|
|
1290
|
+
const retryPolicy = __queuePolicy({
|
|
1291
|
+
op: "retries",
|
|
1292
|
+
now: Date.now(),
|
|
1293
|
+
entries: retrying.map(({ row }) => ({
|
|
1294
|
+
attempt: row.attempts + 1,
|
|
1295
|
+
maxRetries: consumer.maxRetries,
|
|
1296
|
+
explicitSeconds: retryMessages.has(row.id)
|
|
1297
|
+
? retryMessages.get(row.id)
|
|
1298
|
+
: retryBatch ? batchDelay : null,
|
|
1299
|
+
configuredSeconds: consumer.retryDelaySeconds ?? null,
|
|
1300
|
+
})),
|
|
1301
|
+
});
|
|
1302
|
+
const retryById = new Map(
|
|
1303
|
+
retrying.map(({ row }, index) => [row.id, retryPolicy[index]]),
|
|
1304
|
+
);
|
|
1305
|
+
const dlqTransferById = new Map(
|
|
1306
|
+
retrying
|
|
1307
|
+
.filter(({ row }) =>
|
|
1308
|
+
retryById.get(row.id).exhausted && consumer.deadLetterQueue !== undefined
|
|
1309
|
+
)
|
|
1310
|
+
.map(({ row }) => [row.id, crypto.randomUUID()]),
|
|
1311
|
+
);
|
|
1312
|
+
const sql = this._open();
|
|
1313
|
+
this.ctx.storage.transactionSync(() => {
|
|
1314
|
+
for (const { row, retry } of decisions) {
|
|
1315
|
+
const policy = retryById.get(row.id);
|
|
1316
|
+
if (row.purge_on_settle !== 0 || !retry ||
|
|
1317
|
+
(policy.exhausted && consumer.deadLetterQueue === undefined)) {
|
|
1318
|
+
sql.exec(
|
|
1319
|
+
`DELETE FROM ${__QUEUE_MESSAGES}
|
|
1320
|
+
WHERE seq = ? AND lease_id = ? AND lease_generation = ?`,
|
|
1321
|
+
row.seq,
|
|
1322
|
+
body.leaseId,
|
|
1323
|
+
row.lease_generation,
|
|
1324
|
+
);
|
|
1325
|
+
continue;
|
|
1326
|
+
}
|
|
1327
|
+
if (policy.exhausted) {
|
|
1328
|
+
sql.exec(
|
|
1329
|
+
`UPDATE ${__QUEUE_MESSAGES}
|
|
1330
|
+
SET attempts = attempts + 1, lease_id = NULL,
|
|
1331
|
+
leased_until = NULL, dlq_target = ?, dlq_transfer_id = ?,
|
|
1332
|
+
transfer_kind = 'dead-letter'
|
|
1333
|
+
WHERE seq = ? AND lease_id = ? AND lease_generation = ?`,
|
|
1334
|
+
consumer.deadLetterQueue,
|
|
1335
|
+
dlqTransferById.get(row.id),
|
|
1336
|
+
row.seq,
|
|
1337
|
+
body.leaseId,
|
|
1338
|
+
row.lease_generation,
|
|
1339
|
+
);
|
|
1340
|
+
continue;
|
|
1341
|
+
}
|
|
1342
|
+
sql.exec(
|
|
1343
|
+
`UPDATE ${__QUEUE_MESSAGES}
|
|
1344
|
+
SET attempts = attempts + 1, visible_at = ?,
|
|
1345
|
+
lease_id = NULL, leased_until = NULL
|
|
1346
|
+
WHERE seq = ? AND lease_id = ? AND lease_generation = ?`,
|
|
1347
|
+
policy.at,
|
|
1348
|
+
row.seq,
|
|
1349
|
+
body.leaseId,
|
|
1350
|
+
row.lease_generation,
|
|
1351
|
+
);
|
|
1352
|
+
}
|
|
1353
|
+
});
|
|
1354
|
+
try {
|
|
1355
|
+
await this._driveDlq();
|
|
1356
|
+
} catch (error) {
|
|
1357
|
+
// Settlement is durable once the source mark commits. A failed transfer
|
|
1358
|
+
// is alarm work now, not a reason to put the message back in delivery.
|
|
1359
|
+
console.error("Queue DLQ transfer deferred:", error);
|
|
1360
|
+
}
|
|
1361
|
+
await this._rearm();
|
|
1362
|
+
return true;
|
|
1363
|
+
}
|
|
1364
|
+
|
|
1365
|
+
async _purge() {
|
|
1366
|
+
const now = Date.now();
|
|
1367
|
+
const sql = this._open();
|
|
1368
|
+
let deleted = 0;
|
|
1369
|
+
let leased = 0;
|
|
1370
|
+
this.ctx.storage.transactionSync(() => {
|
|
1371
|
+
let after = 0;
|
|
1372
|
+
while (true) {
|
|
1373
|
+
// Purge is necessarily proportional to the stored backlog, but its
|
|
1374
|
+
// policy envelope stays bounded instead of materializing the Queue in
|
|
1375
|
+
// V8 or Rust. One SQLite transaction keeps the visibility change
|
|
1376
|
+
// atomic across all chunks.
|
|
1377
|
+
const rows = sql.exec(
|
|
1378
|
+
`SELECT seq, lease_id, leased_until FROM ${__QUEUE_MESSAGES}
|
|
1379
|
+
WHERE seq > ?
|
|
1380
|
+
ORDER BY seq
|
|
1381
|
+
LIMIT 256`,
|
|
1382
|
+
after,
|
|
1383
|
+
).toArray();
|
|
1384
|
+
if (rows.length === 0) break;
|
|
1385
|
+
after = rows[rows.length - 1].seq;
|
|
1386
|
+
const plan = __queuePolicy({
|
|
1387
|
+
op: "purge",
|
|
1388
|
+
now,
|
|
1389
|
+
rows: rows.map((row) => ({
|
|
1390
|
+
seq: String(row.seq),
|
|
1391
|
+
leaseIdPresent: row.lease_id !== null,
|
|
1392
|
+
leasedUntil: row.leased_until,
|
|
1393
|
+
})),
|
|
1394
|
+
});
|
|
1395
|
+
for (const seq of plan.delete) {
|
|
1396
|
+
sql.exec(
|
|
1397
|
+
`DELETE FROM ${__QUEUE_MESSAGES}
|
|
1398
|
+
WHERE seq = ?
|
|
1399
|
+
AND (lease_id IS NULL OR leased_until IS NULL OR leased_until <= ?)`,
|
|
1400
|
+
Number(seq),
|
|
1401
|
+
now,
|
|
1402
|
+
);
|
|
1403
|
+
deleted += 1;
|
|
1404
|
+
}
|
|
1405
|
+
for (const seq of plan.markForSettle) {
|
|
1406
|
+
sql.exec(
|
|
1407
|
+
`UPDATE ${__QUEUE_MESSAGES} SET purge_on_settle = 1
|
|
1408
|
+
WHERE seq = ? AND lease_id IS NOT NULL AND leased_until > ?`,
|
|
1409
|
+
Number(seq),
|
|
1410
|
+
now,
|
|
1411
|
+
);
|
|
1412
|
+
leased += 1;
|
|
1413
|
+
}
|
|
1414
|
+
}
|
|
1415
|
+
});
|
|
1416
|
+
await this._rearm(now);
|
|
1417
|
+
return { deleted, leased, metrics: this._metrics(now) };
|
|
1418
|
+
}
|
|
1419
|
+
|
|
1420
|
+
async _setPaused(paused) {
|
|
1421
|
+
this._setMeta("paused", paused ? "1" : null);
|
|
1422
|
+
await this._rearm();
|
|
1423
|
+
return { paused, ...this._metrics() };
|
|
1424
|
+
}
|
|
1425
|
+
|
|
1426
|
+
_operatorLimit(value, fallback) {
|
|
1427
|
+
const limit = value ?? fallback;
|
|
1428
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 100) {
|
|
1429
|
+
throw __queueError("limit must be an integer from 1 through 100");
|
|
1430
|
+
}
|
|
1431
|
+
return limit;
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1434
|
+
_peek(limit) {
|
|
1435
|
+
const now = Date.now();
|
|
1436
|
+
const cutoff = now - __queueLimits().retentionMs;
|
|
1437
|
+
const rows = this._open().exec(
|
|
1438
|
+
`SELECT id, body, content_type, size, enqueued_at, visible_at, attempts,
|
|
1439
|
+
lease_id, leased_until, purge_on_settle, dlq_target,
|
|
1440
|
+
dead_letter_source
|
|
1441
|
+
FROM ${__QUEUE_MESSAGES}
|
|
1442
|
+
WHERE enqueued_at > ?
|
|
1443
|
+
ORDER BY visible_at, seq
|
|
1444
|
+
LIMIT ?`,
|
|
1445
|
+
cutoff,
|
|
1446
|
+
limit,
|
|
1447
|
+
).toArray();
|
|
1448
|
+
return {
|
|
1449
|
+
messages: rows.map((row) => ({
|
|
1450
|
+
id: row.id,
|
|
1451
|
+
bodyBase64: $$btoa(
|
|
1452
|
+
row.body instanceof Uint8Array ? row.body : new Uint8Array(row.body),
|
|
1453
|
+
),
|
|
1454
|
+
contentType: row.content_type,
|
|
1455
|
+
size: row.size,
|
|
1456
|
+
enqueuedAt: row.enqueued_at,
|
|
1457
|
+
visibleAt: row.visible_at,
|
|
1458
|
+
attempts: row.attempts,
|
|
1459
|
+
state: row.dlq_target !== null
|
|
1460
|
+
? "moving"
|
|
1461
|
+
: row.purge_on_settle !== 0
|
|
1462
|
+
? "purging"
|
|
1463
|
+
: row.lease_id !== null && row.leased_until > now
|
|
1464
|
+
? "leased"
|
|
1465
|
+
: row.visible_at <= now
|
|
1466
|
+
? "visible"
|
|
1467
|
+
: "delayed",
|
|
1468
|
+
leasedUntil: row.lease_id !== null && row.leased_until > now
|
|
1469
|
+
? row.leased_until
|
|
1470
|
+
: null,
|
|
1471
|
+
transferTarget: row.dlq_target,
|
|
1472
|
+
deadLetterSource: row.dead_letter_source,
|
|
1473
|
+
})),
|
|
1474
|
+
};
|
|
1475
|
+
}
|
|
1476
|
+
|
|
1477
|
+
async _redrive(limit) {
|
|
1478
|
+
// Finish an interrupted move before selecting more work. A transfer mark
|
|
1479
|
+
// is durable, so selecting it a second time would invent another identity
|
|
1480
|
+
// where replaying the existing identity is sufficient.
|
|
1481
|
+
await this._driveDlq(100);
|
|
1482
|
+
const now = Date.now();
|
|
1483
|
+
const cutoff = now - __queueLimits().retentionMs;
|
|
1484
|
+
const candidates = this._open().exec(
|
|
1485
|
+
`SELECT seq, dead_letter_source
|
|
1486
|
+
FROM ${__QUEUE_MESSAGES}
|
|
1487
|
+
WHERE dead_letter_source IS NOT NULL
|
|
1488
|
+
AND dlq_target IS NULL
|
|
1489
|
+
AND purge_on_settle = 0
|
|
1490
|
+
AND enqueued_at > ?
|
|
1491
|
+
AND (lease_id IS NULL OR leased_until <= ?)
|
|
1492
|
+
ORDER BY visible_at, seq
|
|
1493
|
+
LIMIT ?`,
|
|
1494
|
+
cutoff,
|
|
1495
|
+
now,
|
|
1496
|
+
limit,
|
|
1497
|
+
).toArray();
|
|
1498
|
+
const transfers = candidates.map((row) => ({
|
|
1499
|
+
...row,
|
|
1500
|
+
transferId: crypto.randomUUID(),
|
|
1501
|
+
}));
|
|
1502
|
+
this.ctx.storage.transactionSync(() => {
|
|
1503
|
+
for (const transfer of transfers) {
|
|
1504
|
+
this._open().exec(
|
|
1505
|
+
`UPDATE ${__QUEUE_MESSAGES}
|
|
1506
|
+
SET dlq_target = ?, dlq_transfer_id = ?, transfer_kind = 'redrive',
|
|
1507
|
+
lease_id = NULL, leased_until = NULL
|
|
1508
|
+
WHERE seq = ? AND dead_letter_source = ? AND dlq_target IS NULL
|
|
1509
|
+
AND purge_on_settle = 0
|
|
1510
|
+
AND (lease_id IS NULL OR leased_until <= ?)`,
|
|
1511
|
+
transfer.dead_letter_source,
|
|
1512
|
+
transfer.transferId,
|
|
1513
|
+
transfer.seq,
|
|
1514
|
+
transfer.dead_letter_source,
|
|
1515
|
+
now,
|
|
1516
|
+
);
|
|
1517
|
+
}
|
|
1518
|
+
});
|
|
1519
|
+
// Arm before the cross-cell call. If the process stops after the durable
|
|
1520
|
+
// mark, the alarm replays the same transfer identity.
|
|
1521
|
+
await this._rearm(now);
|
|
1522
|
+
await this._driveDlq(100);
|
|
1523
|
+
await this._rearm();
|
|
1524
|
+
return { redriven: transfers.length, metrics: this._metrics() };
|
|
1525
|
+
}
|
|
1526
|
+
|
|
1527
|
+
async fetch(request) {
|
|
1528
|
+
const path = new URL(request.url).pathname;
|
|
1529
|
+
if (request.method !== "POST") {
|
|
1530
|
+
return new Response("Queue route not found", { status: 404 });
|
|
1531
|
+
}
|
|
1532
|
+
let body;
|
|
1533
|
+
try {
|
|
1534
|
+
body = await request.json();
|
|
1535
|
+
} catch {
|
|
1536
|
+
return new Response("invalid Queue request", { status: 400 });
|
|
1537
|
+
}
|
|
1538
|
+
try {
|
|
1539
|
+
if (path === "/__qSettle") {
|
|
1540
|
+
return await this._settle(body)
|
|
1541
|
+
? new Response(null, { status: 204 })
|
|
1542
|
+
: new Response("stale Queue settlement", { status: 409 });
|
|
1543
|
+
}
|
|
1544
|
+
if (body.op === "info") {
|
|
1545
|
+
const metrics = this._metrics();
|
|
1546
|
+
const stored = this._open().exec(
|
|
1547
|
+
`SELECT stored_count FROM ${__QUEUE_STATS} WHERE id = 1`,
|
|
1548
|
+
).toArray()[0].stored_count;
|
|
1549
|
+
return Response.json({ result: { ...metrics, stored, paused: this._paused() } });
|
|
1550
|
+
}
|
|
1551
|
+
if (body.op === "purge") {
|
|
1552
|
+
return Response.json({ result: await this._purge() });
|
|
1553
|
+
}
|
|
1554
|
+
if (body.op === "pause") {
|
|
1555
|
+
return Response.json({ result: await this._setPaused(true) });
|
|
1556
|
+
}
|
|
1557
|
+
if (body.op === "resume") {
|
|
1558
|
+
return Response.json({ result: await this._setPaused(false) });
|
|
1559
|
+
}
|
|
1560
|
+
if (body.op === "peek") {
|
|
1561
|
+
return Response.json({
|
|
1562
|
+
result: this._peek(this._operatorLimit(body.limit, 10)),
|
|
1563
|
+
});
|
|
1564
|
+
}
|
|
1565
|
+
if (body.op === "redrive") {
|
|
1566
|
+
return Response.json({
|
|
1567
|
+
result: await this._redrive(this._operatorLimit(body.limit, 100)),
|
|
1568
|
+
});
|
|
1569
|
+
}
|
|
1570
|
+
return Response.json(
|
|
1571
|
+
{ error: `QUEUE_ERROR: unknown operation ${JSON.stringify(body.op)}` },
|
|
1572
|
+
{ status: 400 },
|
|
1573
|
+
);
|
|
1574
|
+
} catch (error) {
|
|
1575
|
+
return Response.json(
|
|
1576
|
+
{ error: String(error?.message ?? error) },
|
|
1577
|
+
{ status: 400 },
|
|
1578
|
+
);
|
|
1579
|
+
}
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
1582
|
+
|
|
1583
|
+
__runtimeClass("__Queue", __QueueCell);
|
|
1584
|
+
|
|
1585
|
+
class Queue {
|
|
1586
|
+
constructor(queue, cellName, deliveryDelay) {
|
|
1587
|
+
Object.defineProperties(this, {
|
|
1588
|
+
_queue: { value: queue },
|
|
1589
|
+
_cellName: { value: cellName },
|
|
1590
|
+
_deliveryDelay: { value: deliveryDelay },
|
|
1591
|
+
});
|
|
1592
|
+
}
|
|
1593
|
+
|
|
1594
|
+
get _stub() {
|
|
1595
|
+
return __cell.makeNamespace("__Queue").getByName(this._cellName);
|
|
1596
|
+
}
|
|
1597
|
+
|
|
1598
|
+
async _send(entries) {
|
|
1599
|
+
const metrics = await this._stub.__queueSend({ entries });
|
|
1600
|
+
return { metadata: { metrics: __queueMetricsView(metrics) } };
|
|
1601
|
+
}
|
|
1602
|
+
|
|
1603
|
+
send(body, options = {}) {
|
|
1604
|
+
const encoded = __queueEncode(body, options.contentType);
|
|
1605
|
+
return this._send([{
|
|
1606
|
+
...encoded,
|
|
1607
|
+
delaySeconds: __queueDelay(options.delaySeconds, this._deliveryDelay),
|
|
1608
|
+
}]);
|
|
1609
|
+
}
|
|
1610
|
+
|
|
1611
|
+
sendBatch(messages, options = {}) {
|
|
1612
|
+
if (messages === null || messages === undefined ||
|
|
1613
|
+
typeof messages[Symbol.iterator] !== "function") {
|
|
1614
|
+
throw __queueError("sendBatch() needs an iterable of messages");
|
|
1615
|
+
}
|
|
1616
|
+
const entries = [];
|
|
1617
|
+
let bytes = 0;
|
|
1618
|
+
const batchDelay = __queueDelay(options.delaySeconds, this._deliveryDelay);
|
|
1619
|
+
for (const message of messages) {
|
|
1620
|
+
if (entries.length >= __queueLimits().maxBatchMessages) {
|
|
1621
|
+
throw __queueError(
|
|
1622
|
+
`a batch has at most ${__queueLimits().maxBatchMessages} messages`,
|
|
1623
|
+
);
|
|
1624
|
+
}
|
|
1625
|
+
if (message === null || typeof message !== "object" || !("body" in message)) {
|
|
1626
|
+
throw __queueError("each batch entry needs a body");
|
|
1627
|
+
}
|
|
1628
|
+
const encoded = __queueEncode(message.body, message.contentType);
|
|
1629
|
+
bytes += encoded.size;
|
|
1630
|
+
if (bytes > __queueLimits().maxBatchBytes) {
|
|
1631
|
+
throw __queueError(
|
|
1632
|
+
`a batch is at most ${__queueLimits().maxBatchBytes} bytes, got ${bytes}`,
|
|
1633
|
+
);
|
|
1634
|
+
}
|
|
1635
|
+
entries.push({
|
|
1636
|
+
...encoded,
|
|
1637
|
+
delaySeconds: __queueDelay(message.delaySeconds, batchDelay),
|
|
1638
|
+
});
|
|
1639
|
+
}
|
|
1640
|
+
if (entries.length === 0) {
|
|
1641
|
+
throw __queueError("sendBatch() needs at least one message");
|
|
1642
|
+
}
|
|
1643
|
+
return this._send(entries);
|
|
1644
|
+
}
|
|
1645
|
+
|
|
1646
|
+
async metrics() {
|
|
1647
|
+
return __queueMetricsView(await this._stub.__queueMetrics());
|
|
1648
|
+
}
|
|
1649
|
+
}
|
|
1650
|
+
|
|
1651
|
+
__celld.__makeQueue = (queue, cellName, deliveryDelay) =>
|
|
1652
|
+
new Queue(queue, cellName, deliveryDelay);
|
|
1653
|
+
|
|
1654
|
+
|
|
1655
|
+
return { QueueCell: classes.__Queue, makeQueue: __celld.__makeQueue, dispatchQueue: __celld.__dispatchEntrypointQueue };
|
|
1656
|
+
}
|