experimental-a2 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +55 -0
- package/README.md +1 -1
- package/dist/{actor-C7b-1arn.d.ts → actor-CqibLgjQ.d.ts} +15 -7
- package/dist/actor-CqibLgjQ.d.ts.map +1 -0
- package/dist/actor-client.d.ts +1 -1
- package/dist/actor-client.js +2 -2
- package/dist/actor-react.d.ts +3 -3
- package/dist/actor-react.js +5 -5
- package/dist/{actor-shared-t7w8xIKP.js → actor-shared-DFET5q16.js} +3 -3
- package/dist/actor-shared-DFET5q16.js.map +1 -0
- package/dist/actor.d.ts +1 -1
- package/dist/actor.js +54 -43
- package/dist/actor.js.map +1 -1
- package/dist/{ai-Cz3hqVhk.d.ts → ai-DAu77CDr.d.ts} +8 -8
- package/dist/{ai-Cz3hqVhk.d.ts.map → ai-DAu77CDr.d.ts.map} +1 -1
- package/dist/{ai-mMiWul1T.js → ai-EEfKyc-z.js} +9 -9
- package/dist/ai-EEfKyc-z.js.map +1 -0
- package/dist/ai-client.d.ts +2 -2
- package/dist/ai-client.js +2 -2
- package/dist/ai-fx.d.ts +4 -2
- package/dist/ai-fx.d.ts.map +1 -1
- package/dist/ai-fx.js +214 -67
- package/dist/ai-fx.js.map +1 -1
- package/dist/{ai-harness-CV6rXbB3.d.ts → ai-harness-DTjkAr2T.d.ts} +2 -2
- package/dist/{ai-harness-CV6rXbB3.d.ts.map → ai-harness-DTjkAr2T.d.ts.map} +1 -1
- package/dist/{ai-harness-Clbg5ei7.js → ai-harness-dQeJaR1t.js} +7 -3
- package/dist/ai-harness-dQeJaR1t.js.map +1 -0
- package/dist/ai-pi.d.ts +1 -1
- package/dist/ai-pi.js +3 -3
- package/dist/ai-pi.js.map +1 -1
- package/dist/ai-react.d.ts +5 -5
- package/dist/ai-react.js +2 -2
- package/dist/ai-server.d.ts +14 -17
- package/dist/ai-server.d.ts.map +1 -1
- package/dist/ai-server.js +86 -73
- package/dist/ai-server.js.map +1 -1
- package/dist/ai.d.ts +2 -2
- package/dist/ai.js +2 -2
- package/dist/{client-BhHLoNKF.js → client-BQS4i_V9.js} +29 -6
- package/dist/client-BQS4i_V9.js.map +1 -0
- package/dist/{client-h_DKFAjj.d.ts → client-CUMbS_38.d.ts} +3 -3
- package/dist/{client-h_DKFAjj.d.ts.map → client-CUMbS_38.d.ts.map} +1 -1
- package/dist/client.d.ts +3 -3
- package/dist/client.js +1 -1
- package/dist/{contract-BYKASlCV.js → contract-CWukt01d.js} +2 -2
- package/dist/contract-CWukt01d.js.map +1 -0
- package/dist/index.d.ts +5 -5
- package/dist/index.js +3 -3
- package/dist/{initial-snapshot-Bx-c0pDP.d.ts → initial-snapshot-D3Lve43o.d.ts} +4 -4
- package/dist/initial-snapshot-D3Lve43o.d.ts.map +1 -0
- package/dist/{internal-CscSY7K_.js → internal-CvsrprnU.js} +40 -16
- package/dist/internal-CvsrprnU.js.map +1 -0
- package/dist/otel.d.ts +1 -1
- package/dist/otel.js +1 -1
- package/dist/otel.js.map +1 -1
- package/dist/{react-BBXgHWFc.d.ts → react-BoHvC3v9.d.ts} +4 -4
- package/dist/{react-BBXgHWFc.d.ts.map → react-BoHvC3v9.d.ts.map} +1 -1
- package/dist/react.d.ts +1 -1
- package/dist/react.js +1 -1
- package/dist/{reducer-CqA59Q6U.d.ts → reducer-CYPHsSmd.d.ts} +2 -2
- package/dist/{reducer-CqA59Q6U.d.ts.map → reducer-CYPHsSmd.d.ts.map} +1 -1
- package/dist/{reducer-BAXc8JyF.js → reducer-CjmvZSAb.js} +2 -2
- package/dist/{reducer-BAXc8JyF.js.map → reducer-CjmvZSAb.js.map} +1 -1
- package/dist/runtime-consumer-CIeL33wj.js +12 -0
- package/dist/runtime-consumer-CIeL33wj.js.map +1 -0
- package/dist/scheduler-qstash.d.ts +2 -2
- package/dist/scheduler-qstash.js +3 -3
- package/dist/scheduler-task-X1SToDjP.js.map +1 -1
- package/dist/scheduler-vercel.d.ts +2 -2
- package/dist/scheduler-vercel.js +3 -3
- package/dist/scheduler-vercel.js.map +1 -1
- package/dist/{server-qyb5aJJG.js → server-DtFrWxi3.js} +412 -116
- package/dist/server-DtFrWxi3.js.map +1 -0
- package/dist/{server-DYAATFuT.d.ts → server-qgez0Xny.d.ts} +108 -41
- package/dist/server-qgez0Xny.d.ts.map +1 -0
- package/dist/server.browser.js +1 -1
- package/dist/server.browser.js.map +1 -1
- package/dist/server.d.ts +4 -4
- package/dist/server.js +2 -2
- package/dist/{session-clear-C-QwTXV1.d.ts → session-clear-D1PFe7xl.d.ts} +2 -2
- package/dist/session-clear-D1PFe7xl.d.ts.map +1 -0
- package/dist/{session-clear-DGI4jTQ5.js → session-clear-wHd5GCpZ.js} +5 -4
- package/dist/session-clear-wHd5GCpZ.js.map +1 -0
- package/dist/store-D0MhBvg6.d.ts +532 -0
- package/dist/store-D0MhBvg6.d.ts.map +1 -0
- package/dist/store-N8PXxDAS.js.map +1 -1
- package/dist/store-codec-DwrWlgQx.js +65 -0
- package/dist/store-codec-DwrWlgQx.js.map +1 -0
- package/dist/store-memory.d.ts +1 -1
- package/dist/store-memory.d.ts.map +1 -1
- package/dist/store-memory.js +406 -225
- package/dist/store-memory.js.map +1 -1
- package/dist/{store-polling-7WJV0YWP.js → store-polling-BZt2Rf3D.js} +2 -2
- package/dist/{store-polling-7WJV0YWP.js.map → store-polling-BZt2Rf3D.js.map} +1 -1
- package/dist/store-postgres.d.ts +1 -1
- package/dist/store-postgres.d.ts.map +1 -1
- package/dist/store-postgres.js +1598 -587
- package/dist/store-postgres.js.map +1 -1
- package/dist/store-presence-polling-JuAy2Ck2.js +111 -0
- package/dist/store-presence-polling-JuAy2Ck2.js.map +1 -0
- package/dist/store-redis-http.d.ts +1 -1
- package/dist/store-redis-http.js +4 -4
- package/dist/store-redis-notify-CsJw_hfD.js +3060 -0
- package/dist/store-redis-notify-CsJw_hfD.js.map +1 -0
- package/dist/store-redis.d.ts +1 -1
- package/dist/store-redis.d.ts.map +1 -1
- package/dist/store-redis.js +4 -5
- package/dist/store-redis.js.map +1 -1
- package/dist/store-sqlite.d.ts +1 -1
- package/dist/store-sqlite.d.ts.map +1 -1
- package/dist/store-sqlite.js +527 -299
- package/dist/store-sqlite.js.map +1 -1
- package/dist/stream-buffer-DrTyiIuo.js +57 -0
- package/dist/stream-buffer-DrTyiIuo.js.map +1 -0
- package/dist/{telemetry-Cyfqo6Qt.d.ts → telemetry-CWwsh-Gx.d.ts} +3 -3
- package/dist/telemetry-CWwsh-Gx.d.ts.map +1 -0
- package/dist/{wire-BKxH1vbP.js → wire-CV-Z7wz-.js} +6 -5
- package/dist/wire-CV-Z7wz-.js.map +1 -0
- package/docs/01-quickstart.mdx +21 -15
- package/docs/02-examples.mdx +1 -1
- package/docs/actors/01-introduction.mdx +13 -2
- package/docs/actors/02-concurrency.mdx +1 -1
- package/docs/agents/01-introduction.mdx +1 -1
- package/docs/agents/02-tools.mdx +12 -11
- package/docs/agents/03-conversations.mdx +36 -25
- package/docs/agents/04-harnesses.mdx +14 -7
- package/docs/client/03-presence.mdx +18 -14
- package/docs/contract/01-events.mdx +2 -2
- package/docs/contract/02-state.mdx +18 -14
- package/docs/contract/03-sessions.mdx +32 -25
- package/docs/index.mdx +29 -24
- package/docs/reference/01-core.mdx +7 -6
- package/docs/reference/02-server.mdx +250 -191
- package/docs/reference/04-ai.mdx +33 -23
- package/docs/reference/05-telemetry.mdx +27 -13
- package/docs/reference/06-errors.mdx +25 -19
- package/docs/reference/meta.ts +1 -1
- package/docs/schedulers/01-overview.mdx +8 -8
- package/docs/schedulers/02-vercel-queues.mdx +2 -2
- package/docs/schedulers/03-qstash.mdx +4 -4
- package/docs/server/01-consumers.mdx +418 -0
- package/docs/server/02-timers.mdx +31 -21
- package/docs/server/03-cancellation.mdx +52 -44
- package/docs/server/04-routes.mdx +4 -1
- package/docs/server/05-sql-projections.mdx +391 -0
- package/docs/server/06-guarantees.mdx +61 -51
- package/docs/stores/01-overview.mdx +3 -3
- package/docs/stores/02-redis.mdx +24 -10
- package/docs/stores/03-redis-http.mdx +8 -6
- package/docs/stores/04-postgres.mdx +65 -14
- package/docs/stores/05-sqlite.mdx +5 -3
- package/docs/upgrades/01-overview.mdx +31 -0
- package/docs/upgrades/78-0-22.mdx +115 -0
- package/docs/upgrades/79-0-21.mdx +165 -0
- package/docs/upgrades/80-0-20.mdx +43 -0
- package/docs/upgrades/81-0-19.mdx +33 -0
- package/docs/upgrades/82-0-18.mdx +33 -0
- package/docs/upgrades/83-0-17.mdx +69 -0
- package/docs/upgrades/meta.ts +3 -0
- package/examples/playground/DEPLOY.md +39 -7
- package/examples/playground/app/a0/chat.test.tsx +2 -2
- package/examples/playground/app/a0/sandbox-log.ts +17 -15
- package/examples/playground/app/a0/sandbox.ts +58 -13
- package/examples/playground/app/a0/server.ts +4 -4
- package/examples/playground/app/a2slack/a2slack.css +1067 -0
- package/examples/playground/app/a2slack/c/[channel]/page.tsx +20 -0
- package/examples/playground/app/a2slack/consumers/agent.ts +63 -0
- package/examples/playground/app/a2slack/consumers/feed.ts +164 -0
- package/examples/playground/app/a2slack/consumers/notify.ts +116 -0
- package/examples/playground/app/a2slack/consumers/search.ts +37 -0
- package/examples/playground/app/a2slack/consumers/triage.ts +31 -0
- package/examples/playground/app/a2slack/consumers/welcome.ts +25 -0
- package/examples/playground/app/a2slack/db/a2.ts +209 -0
- package/examples/playground/app/a2slack/db/index.ts +73 -0
- package/examples/playground/app/a2slack/db/migrations/0000_extensions.sql +4 -0
- package/examples/playground/app/a2slack/db/migrations/0001_workspace.sql +74 -0
- package/examples/playground/app/a2slack/db/migrations/0002_a2.sql +215 -0
- package/examples/playground/app/a2slack/db/migrations/0003_a2_batch.sql +67 -0
- package/examples/playground/app/a2slack/db/migrations/0004_a2_session_event_ids.sql +2 -0
- package/examples/playground/app/a2slack/db/migrations/meta/0000_snapshot.json +18 -0
- package/examples/playground/app/a2slack/db/migrations/meta/0001_snapshot.json +500 -0
- package/examples/playground/app/a2slack/db/migrations/meta/0002_snapshot.json +1159 -0
- package/examples/playground/app/a2slack/db/migrations/meta/0003_snapshot.json +1159 -0
- package/examples/playground/app/a2slack/db/migrations/meta/0004_snapshot.json +1165 -0
- package/examples/playground/app/a2slack/db/migrations/meta/_journal.json +41 -0
- package/examples/playground/app/a2slack/db/queries.ts +263 -0
- package/examples/playground/app/a2slack/db/schema.ts +145 -0
- package/examples/playground/app/a2slack/drills/a2slack.drill.ts +708 -0
- package/examples/playground/app/a2slack/drills/harness.ts +244 -0
- package/examples/playground/app/a2slack/events/route.ts +36 -0
- package/examples/playground/app/a2slack/identity.ts +44 -0
- package/examples/playground/app/a2slack/inbox/route.ts +8 -0
- package/examples/playground/app/a2slack/lag/data/route.ts +8 -0
- package/examples/playground/app/a2slack/lag/page.tsx +101 -0
- package/examples/playground/app/a2slack/lag.ts +73 -0
- package/examples/playground/app/a2slack/layout.tsx +43 -0
- package/examples/playground/app/a2slack/messages/route.ts +17 -0
- package/examples/playground/app/a2slack/model.test.ts +208 -0
- package/examples/playground/app/a2slack/model.ts +369 -0
- package/examples/playground/app/a2slack/models.test.ts +72 -0
- package/examples/playground/app/a2slack/page.tsx +10 -0
- package/examples/playground/app/a2slack/scheduler/route.ts +5 -0
- package/examples/playground/app/a2slack/search/route.ts +40 -0
- package/examples/playground/app/a2slack/search-index.ts +108 -0
- package/examples/playground/app/a2slack/server.test.ts +315 -0
- package/examples/playground/app/a2slack/server.ts +55 -0
- package/examples/playground/app/a2slack/sessions.ts +50 -0
- package/examples/playground/app/a2slack/thread/route.ts +8 -0
- package/examples/playground/app/a2slack/ui/ChannelView.tsx +87 -0
- package/examples/playground/app/a2slack/ui/Composer.tsx +117 -0
- package/examples/playground/app/a2slack/ui/DotsShadow.tsx +42 -0
- package/examples/playground/app/a2slack/ui/Feed.tsx +157 -0
- package/examples/playground/app/a2slack/ui/InboxContext.tsx +63 -0
- package/examples/playground/app/a2slack/ui/InboxView.tsx +82 -0
- package/examples/playground/app/a2slack/ui/Menu.tsx +128 -0
- package/examples/playground/app/a2slack/ui/MessageRow.tsx +314 -0
- package/examples/playground/app/a2slack/ui/NameGate.tsx +35 -0
- package/examples/playground/app/a2slack/ui/SearchContext.tsx +146 -0
- package/examples/playground/app/a2slack/ui/SearchResults.tsx +97 -0
- package/examples/playground/app/a2slack/ui/Shell.tsx +34 -0
- package/examples/playground/app/a2slack/ui/Sidebar.tsx +199 -0
- package/examples/playground/app/a2slack/ui/ThreadPanel.tsx +134 -0
- package/examples/playground/app/a2slack/ui/useInbox.ts +71 -0
- package/examples/playground/app/agent/server.ts +3 -3
- package/examples/playground/app/chat/[chatId]/chat-client.tsx +1 -1
- package/examples/playground/app/chat/server.ts +45 -35
- package/examples/playground/app/components/demo-icons.tsx +7 -0
- package/examples/playground/app/components/demos.tsx +7 -0
- package/examples/playground/app/orders/[orderId]/order-client.tsx +2 -2
- package/examples/playground/app/orders/page.tsx +1 -1
- package/examples/playground/app/orders/server.ts +49 -30
- package/examples/playground/app/recovery/[recoveryId]/recovery-client.tsx +14 -15
- package/examples/playground/app/recovery/page.tsx +4 -4
- package/examples/playground/app/recovery/server.ts +23 -32
- package/examples/playground/drizzle.config.ts +14 -0
- package/examples/playground/lib/store.ts +1 -1
- package/examples/playground/next.config.ts +6 -1
- package/examples/playground/package.json +12 -1
- package/examples/playground/scripts/migrate.mjs +31 -0
- package/examples/playground/vercel.json +9 -0
- package/examples/playground/vitest.drills.config.ts +32 -0
- package/package.json +5 -2
- package/sql/postgres/drizzle-schema.ts +209 -0
- package/sql/postgres/schema.sql +227 -67
- package/src/actor-shared.ts +10 -2
- package/src/actor.ts +101 -71
- package/src/ai-agent.ts +162 -111
- package/src/ai-control-server.ts +8 -8
- package/src/ai-fx.ts +324 -82
- package/src/ai-harness.ts +17 -2
- package/src/ai-pi.ts +2 -2
- package/src/ai-server.ts +2 -2
- package/src/ai.ts +7 -7
- package/src/client.ts +19 -2
- package/src/contract.ts +1 -1
- package/src/deterministic-id.ts +27 -4
- package/src/errors.ts +8 -5
- package/src/idempotent-replay.ts +0 -1
- package/src/index.ts +1 -1
- package/src/internal.ts +57 -16
- package/src/otel.ts +1 -1
- package/src/postgres-schema.ts +98 -57
- package/src/presence-recovery.ts +18 -11
- package/src/presence-tombstones.ts +52 -0
- package/src/push-envelope.ts +20 -0
- package/src/runtime-consumer.ts +10 -0
- package/src/scheduler-task.ts +2 -2
- package/src/scheduler-vercel.ts +1 -1
- package/src/server.browser.ts +1 -1
- package/src/server.ts +819 -218
- package/src/session-socket.ts +1 -1
- package/src/store-codec.ts +66 -0
- package/src/store-memory.ts +604 -286
- package/src/store-postgres.ts +2006 -857
- package/src/store-presence-polling.ts +59 -34
- package/src/store-redis-core.ts +1961 -1333
- package/src/store-redis-notify.ts +76 -26
- package/src/store-redis.ts +2 -3
- package/src/store-sqlite.ts +915 -516
- package/src/store.ts +241 -102
- package/src/telemetry.ts +2 -1
- package/src/wire.ts +1 -0
- package/dist/actor-C7b-1arn.d.ts.map +0 -1
- package/dist/actor-shared-t7w8xIKP.js.map +0 -1
- package/dist/ai-harness-Clbg5ei7.js.map +0 -1
- package/dist/ai-mMiWul1T.js.map +0 -1
- package/dist/client-BhHLoNKF.js.map +0 -1
- package/dist/contract-BYKASlCV.js.map +0 -1
- package/dist/idempotent-replay-JdgMQRRm.js +0 -21
- package/dist/idempotent-replay-JdgMQRRm.js.map +0 -1
- package/dist/initial-snapshot-Bx-c0pDP.d.ts.map +0 -1
- package/dist/internal-CscSY7K_.js.map +0 -1
- package/dist/server-DYAATFuT.d.ts.map +0 -1
- package/dist/server-qyb5aJJG.js.map +0 -1
- package/dist/session-clear-C-QwTXV1.d.ts.map +0 -1
- package/dist/session-clear-DGI4jTQ5.js.map +0 -1
- package/dist/store-CyLUy86v.d.ts +0 -404
- package/dist/store-CyLUy86v.d.ts.map +0 -1
- package/dist/store-codec-DTG0Ftek.js +0 -10
- package/dist/store-codec-DTG0Ftek.js.map +0 -1
- package/dist/store-presence-polling-B1ixeqIS.js +0 -94
- package/dist/store-presence-polling-B1ixeqIS.js.map +0 -1
- package/dist/store-redis-notify-CM0AfLY0.js +0 -2544
- package/dist/store-redis-notify-CM0AfLY0.js.map +0 -1
- package/dist/stream-buffer-Cx7AAPGq.js +0 -23
- package/dist/stream-buffer-Cx7AAPGq.js.map +0 -1
- package/dist/telemetry-Cyfqo6Qt.d.ts.map +0 -1
- package/dist/wire-BKxH1vbP.js.map +0 -1
- package/docs/server/01-handlers.mdx +0 -227
package/src/store-redis-core.ts
CHANGED
|
@@ -5,26 +5,60 @@
|
|
|
5
5
|
* `store-redis-http` (provider REST APIs).
|
|
6
6
|
*
|
|
7
7
|
* The stream IS the store: each event is a stream entry whose ID is its
|
|
8
|
-
* dense a2 index (`${index}-0`), so `XRANGE` is the catch-up read.
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* Lua script.
|
|
12
|
-
* Global event-id ownership is one key per id, so no key grows with
|
|
13
|
-
* the number of sessions.
|
|
14
|
-
* The conformance suite in test/conformance is the executable
|
|
15
|
-
* contract, run against a real spawned `redis-server`.
|
|
8
|
+
* dense a2 index (`${index}-0`), so `XRANGE` is the catch-up read. Stream
|
|
9
|
+
* entries are immutable; dispatch bookkeeping lives on consumer cursors.
|
|
10
|
+
* Every write path is one atomic Lua script.
|
|
16
11
|
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
12
|
+
* Every key of a session is `{p}:{"<session id as JSON>"}:<name>`, one hash
|
|
13
|
+
* tag per session (`redisSessionKey`). Each script declares keys of one
|
|
14
|
+
* session only and derives the rest, so on Upstash a script that sets
|
|
15
|
+
* `allow-key-locking` locks only its session instead of the database. Redis
|
|
16
|
+
* and Valkey reject that flag, so the store asks the server once whether it
|
|
17
|
+
* accepts it.
|
|
18
|
+
*
|
|
19
|
+
* The cursor layout, per session:
|
|
20
|
+
*
|
|
21
|
+
* - `heads:{chunk}` holds one row per (head index, consumer) in hashes of
|
|
22
|
+
* META_CHUNK_SIZE consecutive indexes: a head's attempt and failure
|
|
23
|
+
* fields, its kind (`l`), its lane successor (`next`), its snapshot pins,
|
|
24
|
+
* and, once the cursor moves past it, its completion record. A new head
|
|
25
|
+
* is a new row, so moving a cursor resets its attempt fields for free,
|
|
26
|
+
* and an event cursor costs one row, as a handled event did.
|
|
27
|
+
* - `cursors` is one hash of every lane and log cursor: its head (`h`),
|
|
28
|
+
* position (`p`), lane tail (`t`), or log types (`ty`). A lane's queue is
|
|
29
|
+
* linked through the rows' `next` fields.
|
|
30
|
+
* - `runnable` is a zset of every cursor with a head, scored by the head's
|
|
31
|
+
* index, negated while the head is dead-lettered.
|
|
32
|
+
* - `pins` counts, per snapshot, the heads that pin it.
|
|
33
|
+
*
|
|
34
|
+
* A cursor id names a cursor within its session: the consumer's byte
|
|
35
|
+
* length, `:`, the consumer, then `#{index}` for an event cursor, `L{lane}`
|
|
36
|
+
* for a lane cursor, or `G{lane}` for a log cursor. A clear supersedes every
|
|
37
|
+
* head at or below its marker at once (the `runnable` set is dropped); lane and
|
|
38
|
+
* log cursor records are reset lazily against the marker and swept by
|
|
39
|
+
* reclaim.
|
|
40
|
+
*
|
|
41
|
+
* `{p}:schema` records the layout. A store claims it once, before its first
|
|
42
|
+
* script, and refuses a prefix holding another. Keys of an unrecorded layout
|
|
43
|
+
* have other names, so they are never read.
|
|
44
|
+
*
|
|
45
|
+
* Event ids are unique per session: `id:{event id}` holds the index an id
|
|
46
|
+
* names, and leaves with its session. The conformance suite in
|
|
47
|
+
* test/conformance is the executable contract, run against a real spawned
|
|
48
|
+
* `redis-server`, and over REST behind a shim that applies Upstash's
|
|
49
|
+
* key-locking rules.
|
|
50
|
+
*
|
|
51
|
+
* Timestamps and claim expiry come from the injected clock so tests can
|
|
52
|
+
* time-travel against a real server.
|
|
20
53
|
*/
|
|
21
54
|
|
|
22
55
|
import type { PresencePatch } from './contract.ts'
|
|
23
56
|
import { A2Error, supersededAttemptError } from './errors.ts'
|
|
24
57
|
import { CLEAR_EVENT_TYPE, sessionClearedError } from './session-clear.ts'
|
|
25
58
|
import { idempotentReplay } from './idempotent-replay.ts'
|
|
26
|
-
import { nullProtoRecord } from './internal.ts'
|
|
27
|
-
import {
|
|
59
|
+
import { NOTIFY_TIMINGS, nullProtoRecord } from './internal.ts'
|
|
60
|
+
import { retryableLazy } from './retryable-lazy.ts'
|
|
61
|
+
import { decodeCause } from './store-codec.ts'
|
|
28
62
|
import type { SerializedEvent } from './stream-buffer.ts'
|
|
29
63
|
import type {
|
|
30
64
|
A2Store,
|
|
@@ -32,8 +66,14 @@ import type {
|
|
|
32
66
|
CompleteAttemptResult,
|
|
33
67
|
Event,
|
|
34
68
|
EventCause,
|
|
69
|
+
EventTarget,
|
|
70
|
+
HeadDispatch,
|
|
71
|
+
HeadEvent,
|
|
72
|
+
HeadRef,
|
|
35
73
|
IdSource,
|
|
36
74
|
StoreClaimAvailableResult,
|
|
75
|
+
StoreConsumer,
|
|
76
|
+
StoredCursor,
|
|
37
77
|
PresenceRow,
|
|
38
78
|
StoredEvent,
|
|
39
79
|
} from './store.ts'
|
|
@@ -68,9 +108,9 @@ export type RedisStreamRead = (options: {
|
|
|
68
108
|
export type RedisStoreCore = Omit<A2Store, 'stream' | 'presence'> & {
|
|
69
109
|
/**
|
|
70
110
|
* The storage half of the presence capability, shared by both
|
|
71
|
-
* transports. `set` returns the applied patch for local loopback,
|
|
72
|
-
*
|
|
73
|
-
*
|
|
111
|
+
* transports. `set` returns the applied patch for local loopback, with
|
|
112
|
+
* the publish to remote subscribers already under way, or null when
|
|
113
|
+
* every field lost the LWW race.
|
|
74
114
|
*/
|
|
75
115
|
presence: {
|
|
76
116
|
set(
|
|
@@ -78,10 +118,15 @@ export type RedisStoreCore = Omit<A2Store, 'stream' | 'presence'> & {
|
|
|
78
118
|
participant: string,
|
|
79
119
|
values: Record<string, unknown | null>,
|
|
80
120
|
meta: { seen: number; at: Date; ttlMs: number },
|
|
81
|
-
): Promise<PresencePatch | null>
|
|
121
|
+
): Promise<{ patch: PresencePatch; published: Promise<void> } | null>
|
|
82
122
|
read(ns: string): Promise<PresenceRow[]>
|
|
83
123
|
}
|
|
84
|
-
/**
|
|
124
|
+
/**
|
|
125
|
+
* The latest clear marker's index, or 0. Entries below it are an old
|
|
126
|
+
* lifetime's until reclaim deletes them, so a feed starts no lower.
|
|
127
|
+
*/
|
|
128
|
+
clearIndex(sessionId: string): Promise<number>
|
|
129
|
+
/** XRANGE pages — the full-row catch-up read. */
|
|
85
130
|
readRange(
|
|
86
131
|
sessionId: string,
|
|
87
132
|
afterIndex: number,
|
|
@@ -106,7 +151,8 @@ export type RedisStoreCore = Omit<A2Store, 'stream' | 'presence'> & {
|
|
|
106
151
|
}) => AsyncGenerator<SerializedEvent[], void>
|
|
107
152
|
/**
|
|
108
153
|
* Events from `fromIndex` through `throughIndex` or the newest entry in
|
|
109
|
-
* one newest-first page, or null when the range doesn't fit in one
|
|
154
|
+
* one newest-first page, or null when the range doesn't fit in one or
|
|
155
|
+
* starts below the latest clear.
|
|
110
156
|
*/
|
|
111
157
|
readTail(
|
|
112
158
|
sessionId: string,
|
|
@@ -119,66 +165,31 @@ export const READ_PAGE_SIZE = 256
|
|
|
119
165
|
export const FEED_PAGE_SIZE = 1000
|
|
120
166
|
const READ_CONCURRENCY = 16
|
|
121
167
|
|
|
122
|
-
|
|
168
|
+
/** The key layout this core writes; any other is rejected. */
|
|
169
|
+
export const REDIS_LAYOUT = 'tagged-1'
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Upstash runs a script under a database-wide lock unless it opts into
|
|
173
|
+
* locking only its keys' hash tags. Redis and Valkey reject the flag, and
|
|
174
|
+
* have no such lock to opt out of: one thread runs every command.
|
|
175
|
+
*/
|
|
176
|
+
const KEY_LOCKING = '#!lua flags=allow-key-locking'
|
|
177
|
+
const READ_ONLY_KEY_LOCKING = '#!lua flags=no-writes,allow-key-locking'
|
|
178
|
+
const SHEBANG_REJECTED = /shebang|unexpected symbol near '#'/i
|
|
123
179
|
|
|
124
180
|
const META_CHUNK_SIZE = 1000
|
|
125
181
|
|
|
126
|
-
/**
|
|
127
|
-
const
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
'ri',
|
|
131
|
-
'ch',
|
|
132
|
-
'ce',
|
|
133
|
-
'f',
|
|
134
|
-
'fc',
|
|
135
|
-
'lc',
|
|
136
|
-
'lf',
|
|
137
|
-
'lfa',
|
|
138
|
-
'd',
|
|
139
|
-
'a',
|
|
140
|
-
'e',
|
|
141
|
-
] as const
|
|
182
|
+
/** A head row's dispatch columns, in the order scripts return them. */
|
|
183
|
+
const HEAD_COLUMNS = `{ 'd', 'fc', 'lc', 'ch', 'ce', 'a', 'lf', 'lfa', 'e', 'f' }`
|
|
184
|
+
|
|
185
|
+
// ── Lua scripts (each runs atomically) ───────────────────────────────
|
|
142
186
|
|
|
143
187
|
/**
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
* writes are invisible to reads until flushed, so a script must not read a
|
|
148
|
-
* field it has buffered. Per-event metadata lives in hashes of
|
|
149
|
-
* META_CHUNK_SIZE consecutive indexes, so no key grows with a session's
|
|
150
|
-
* history. Indexes at or below the split predate chunking and stay in the
|
|
151
|
-
* base hash, which also holds every session's lane pointers. An unset split
|
|
152
|
-
* means the current event count, so any script that adds events must write
|
|
153
|
-
* `split_write` pairs in its write phase. Including scripts bind COUNTER,
|
|
154
|
-
* META (the base hash), and SPLIT first.
|
|
188
|
+
* Issues a variadic command in calls of at most 1000 arguments, since Lua's
|
|
189
|
+
* `unpack` is bounded by its stack, and concatenates table replies. An even
|
|
190
|
+
* slice size keeps HSET's field/value pairs together.
|
|
155
191
|
*/
|
|
156
|
-
const
|
|
157
|
-
local split, split_stored = nil, false
|
|
158
|
-
-- The latest clear marker lives beside the counter, {prefix}:{session}:clear,
|
|
159
|
-
-- and is read in the same MGET, so a session never cleared pays nothing.
|
|
160
|
-
local CLEAR = string.sub(COUNTER, 1, -6) .. 'clear'
|
|
161
|
-
local boundary = 0
|
|
162
|
-
local function use_split(stored, count, cleared)
|
|
163
|
-
split_stored = stored ~= false
|
|
164
|
-
split = tonumber(stored or count or '0')
|
|
165
|
-
boundary = tonumber(cleared or '0')
|
|
166
|
-
end
|
|
167
|
-
local function mkey(field)
|
|
168
|
-
if split == nil then
|
|
169
|
-
local values = redis.call('MGET', SPLIT, COUNTER, CLEAR)
|
|
170
|
-
use_split(values[1], values[2], values[3])
|
|
171
|
-
end
|
|
172
|
-
local idx = tonumber(field)
|
|
173
|
-
if idx <= split then return META end
|
|
174
|
-
return META .. ':' .. tostring(math.floor((idx - 1) / ${META_CHUNK_SIZE}))
|
|
175
|
-
end
|
|
176
|
-
local function split_write(args)
|
|
177
|
-
if not split_stored then
|
|
178
|
-
args[#args + 1] = SPLIT
|
|
179
|
-
args[#args + 1] = tostring(split)
|
|
180
|
-
end
|
|
181
|
-
end
|
|
192
|
+
const SLICED_LUA = `
|
|
182
193
|
local function sliced(command, key, args)
|
|
183
194
|
local results = {}
|
|
184
195
|
for first = 1, #args, 1000 do
|
|
@@ -195,36 +206,109 @@ local function sliced(command, key, args)
|
|
|
195
206
|
end
|
|
196
207
|
return results
|
|
197
208
|
end
|
|
198
|
-
|
|
209
|
+
`
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Upstash bills every call a script makes, so scripts read the counter,
|
|
213
|
+
* clear marker, typed-type index, and event-id owners with one MGET
|
|
214
|
+
* (`read_bounds`), read rows with one HMGET per chunk
|
|
215
|
+
* (`read_rows`), and buffer hash writes (`put`, `drop`) into one HSET and
|
|
216
|
+
* one HDEL per hash (`flush`). Buffered writes are invisible to reads until
|
|
217
|
+
* flushed, so a script must not read a field it has buffered. Every script
|
|
218
|
+
* binds KEYS[1] to the session's counter and derives the session's other
|
|
219
|
+
* keys, which share its hash tag.
|
|
220
|
+
*/
|
|
221
|
+
const BASE_LUA = `${SLICED_LUA}
|
|
222
|
+
local COUNTER = KEYS[1]
|
|
223
|
+
local SESSION = string.sub(COUNTER, 1, -6)
|
|
224
|
+
local LOG, CLEAR = SESSION .. 'log', SESSION .. 'clear'
|
|
225
|
+
local META, READY, CURSORS = SESSION .. 'heads', SESSION .. 'runnable', SESSION .. 'cursors'
|
|
226
|
+
local PIN_COUNTS, TYPED = SESSION .. 'pins', SESSION .. 'typed'
|
|
227
|
+
local CLEAR_TYPE = '${CLEAR_EVENT_TYPE}'
|
|
228
|
+
local total, boundary = 0, 0
|
|
229
|
+
-- The event types typed log cursors read: type → { from, last }. The type's
|
|
230
|
+
-- index holds every event of it above \`from\`, and no event of it lies above
|
|
231
|
+
-- \`last\`.
|
|
232
|
+
local typed, typed_dirty = {}, false
|
|
233
|
+
-- One key per event id holds the index it names in this session. The
|
|
234
|
+
-- \`id:\` namespace keeps a raw id from spelling another session key.
|
|
235
|
+
local function id_key(id) return SESSION .. 'id:' .. id end
|
|
236
|
+
-- Reads the counter, the latest clear marker, the typed types, and the
|
|
237
|
+
-- owners of \`ids\` in one MGET. Returns each id's index, or false.
|
|
238
|
+
local function read_bounds(ids)
|
|
239
|
+
local keys = { COUNTER, CLEAR, TYPED }
|
|
240
|
+
for _, id in ipairs(ids or {}) do keys[#keys + 1] = id_key(id) end
|
|
241
|
+
local values = sliced('MGET', nil, keys)
|
|
242
|
+
total = tonumber(values[1] or values[2] or '0')
|
|
243
|
+
boundary = tonumber(values[2] or '0')
|
|
244
|
+
if values[3] then typed = cjson.decode(values[3]) end
|
|
245
|
+
local owners = {}
|
|
246
|
+
for i = 4, #keys do owners[i - 3] = values[i] end
|
|
247
|
+
return owners
|
|
248
|
+
end
|
|
249
|
+
-- Event-id owners from read_bounds: an id recorded below the latest clear
|
|
250
|
+
-- is free.
|
|
251
|
+
local function owners_of(values)
|
|
252
|
+
local owners = {}
|
|
253
|
+
for i = 1, #values do
|
|
254
|
+
local owner = values[i]
|
|
255
|
+
if owner and tonumber(owner) < boundary then owner = false end
|
|
256
|
+
owners[i] = owner
|
|
257
|
+
end
|
|
258
|
+
return owners
|
|
259
|
+
end
|
|
260
|
+
local function entry_index(id) return tonumber(string.match(id, '^(%d+)')) end
|
|
261
|
+
local function cid_of(consumer, tag)
|
|
262
|
+
return tostring(string.len(consumer)) .. ':' .. consumer .. tag
|
|
263
|
+
end
|
|
264
|
+
local function parse_cid(cid)
|
|
265
|
+
local colon = string.find(cid, ':', 1, true)
|
|
266
|
+
local n = tonumber(string.sub(cid, 1, colon - 1))
|
|
267
|
+
return string.sub(cid, colon + 1, colon + n), string.sub(cid, colon + n + 1)
|
|
268
|
+
end
|
|
269
|
+
-- A cursor's rows: its consumer, then '#' for an event cursor or 'L' .. lane
|
|
270
|
+
-- for a lane or log cursor, as requests name it.
|
|
271
|
+
local function row_of(consumer, tag)
|
|
272
|
+
local kind = string.sub(tag, 1, 1)
|
|
273
|
+
if kind == '#' then return cid_of(consumer, '#') end
|
|
274
|
+
if kind == 'G' then return cid_of(consumer, 'L' .. string.sub(tag, 2)) end
|
|
275
|
+
return cid_of(consumer, tag)
|
|
276
|
+
end
|
|
277
|
+
local function chunk_key(idx)
|
|
278
|
+
return META .. ':' .. tostring(math.floor((idx - 1) / ${META_CHUNK_SIZE}))
|
|
279
|
+
end
|
|
280
|
+
local function row_name(idx, row, suffix)
|
|
281
|
+
return tostring(idx) .. ':' .. row .. ':' .. suffix
|
|
282
|
+
end
|
|
283
|
+
-- Rows of (index, row) refs, aligned with refs; one HMGET per chunk.
|
|
284
|
+
local function read_rows(refs, suffixes)
|
|
199
285
|
local groups, order = {}, {}
|
|
200
|
-
for
|
|
201
|
-
local key =
|
|
286
|
+
for position, ref in ipairs(refs) do
|
|
287
|
+
local key = chunk_key(ref[1])
|
|
202
288
|
local group = groups[key]
|
|
203
289
|
if not group then
|
|
204
|
-
group = {
|
|
290
|
+
group = { positions = {}, names = {} }
|
|
205
291
|
groups[key] = group
|
|
206
292
|
order[#order + 1] = key
|
|
207
293
|
end
|
|
208
|
-
group.
|
|
294
|
+
group.positions[#group.positions + 1] = position
|
|
209
295
|
for _, suffix in ipairs(suffixes) do
|
|
210
|
-
group.names[#group.names + 1] =
|
|
296
|
+
group.names[#group.names + 1] = row_name(ref[1], ref[2], suffix)
|
|
211
297
|
end
|
|
212
298
|
end
|
|
213
299
|
local rows = {}
|
|
214
300
|
for _, key in ipairs(order) do
|
|
215
301
|
local group = groups[key]
|
|
216
302
|
local values = sliced('HMGET', key, group.names)
|
|
217
|
-
for
|
|
303
|
+
for n, position in ipairs(group.positions) do
|
|
218
304
|
local row = {}
|
|
219
|
-
for column = 1, #suffixes do
|
|
220
|
-
|
|
221
|
-
end
|
|
222
|
-
rows[field] = row
|
|
305
|
+
for column = 1, #suffixes do row[column] = values[(n - 1) * #suffixes + column] end
|
|
306
|
+
rows[position] = row
|
|
223
307
|
end
|
|
224
308
|
end
|
|
225
309
|
return rows
|
|
226
310
|
end
|
|
227
|
-
local writes, write_order = {}, {}
|
|
311
|
+
local writes, write_order, deletes, delete_order = {}, {}, {}, {}
|
|
228
312
|
local function put(key, field, value)
|
|
229
313
|
local fields = writes[key]
|
|
230
314
|
if not fields then
|
|
@@ -235,199 +319,410 @@ local function put(key, field, value)
|
|
|
235
319
|
fields[#fields + 1] = field
|
|
236
320
|
fields[#fields + 1] = value
|
|
237
321
|
end
|
|
322
|
+
local function drop(key, field)
|
|
323
|
+
local fields = deletes[key]
|
|
324
|
+
if not fields then
|
|
325
|
+
fields = {}
|
|
326
|
+
deletes[key] = fields
|
|
327
|
+
delete_order[#delete_order + 1] = key
|
|
328
|
+
end
|
|
329
|
+
fields[#fields + 1] = field
|
|
330
|
+
end
|
|
238
331
|
local function flush()
|
|
239
332
|
for _, key in ipairs(write_order) do sliced('HSET', key, writes[key]) end
|
|
240
|
-
|
|
333
|
+
for _, key in ipairs(delete_order) do sliced('HDEL', key, deletes[key]) end
|
|
334
|
+
writes, write_order, deletes, delete_order = {}, {}, {}, {}
|
|
335
|
+
end
|
|
336
|
+
local function put_row(idx, row, suffix, value)
|
|
337
|
+
put(chunk_key(idx), row_name(idx, row, suffix), value)
|
|
338
|
+
end
|
|
339
|
+
local function drop_row(idx, row, suffix)
|
|
340
|
+
drop(chunk_key(idx), row_name(idx, row, suffix))
|
|
341
|
+
end
|
|
342
|
+
-- A request names a cursor by tag: '#' for an event cursor, 'L' .. lane for a
|
|
343
|
+
-- lane or log cursor. A claimed row records its kind: '#', or the cursor's tag.
|
|
344
|
+
local function owns(kind, tag)
|
|
345
|
+
if not kind then return false end
|
|
346
|
+
if tag == '#' then return kind == '#' end
|
|
347
|
+
return kind == tag or kind == 'G' .. string.sub(tag, 2)
|
|
348
|
+
end
|
|
349
|
+
local function cid_for(consumer, kind, idx)
|
|
350
|
+
if kind == '#' then return cid_of(consumer, '#' .. tostring(idx)) end
|
|
351
|
+
return cid_of(consumer, kind)
|
|
352
|
+
end
|
|
353
|
+
-- One type's index, named by the type's bytes in hex so no type can spell
|
|
354
|
+
-- another key of the session.
|
|
355
|
+
local function type_key(name)
|
|
356
|
+
return SESSION .. 'type:' .. (string.gsub(name, '.', function(c)
|
|
357
|
+
return string.format('%02x', string.byte(c))
|
|
358
|
+
end))
|
|
359
|
+
end
|
|
360
|
+
local ready_changes, ready_order = {}, {}
|
|
361
|
+
local function ready_set(cid, score)
|
|
362
|
+
if ready_changes[cid] == nil then ready_order[#ready_order + 1] = cid end
|
|
363
|
+
ready_changes[cid] = score
|
|
364
|
+
end
|
|
365
|
+
local function flush_ready()
|
|
366
|
+
local adds, removes = {}, {}
|
|
367
|
+
for _, cid in ipairs(ready_order) do
|
|
368
|
+
local score = ready_changes[cid]
|
|
369
|
+
if score then
|
|
370
|
+
adds[#adds + 1] = score
|
|
371
|
+
adds[#adds + 1] = cid
|
|
372
|
+
else
|
|
373
|
+
removes[#removes + 1] = cid
|
|
374
|
+
end
|
|
375
|
+
end
|
|
376
|
+
sliced('ZADD', READY, adds)
|
|
377
|
+
sliced('ZREM', READY, removes)
|
|
378
|
+
ready_changes, ready_order = {}, {}
|
|
379
|
+
end
|
|
380
|
+
local function release_pins(idx, row, pins)
|
|
381
|
+
if not pins or pins == '' then return end
|
|
382
|
+
for _, ref in ipairs(cjson.decode(pins)) do
|
|
383
|
+
local count = redis.call('HINCRBY', PIN_COUNTS, ref, -1)
|
|
384
|
+
if count <= 0 then
|
|
385
|
+
redis.call('HDEL', PIN_COUNTS, ref)
|
|
386
|
+
local checkpoint = cjson.decode(ref)
|
|
387
|
+
redis.call('ZREMRANGEBYSCORE', checkpoint[1], checkpoint[2], checkpoint[2])
|
|
388
|
+
end
|
|
389
|
+
end
|
|
390
|
+
put_row(idx, row, 'pins', '')
|
|
241
391
|
end
|
|
242
392
|
`
|
|
243
393
|
|
|
244
394
|
/**
|
|
245
|
-
*
|
|
246
|
-
*
|
|
247
|
-
*
|
|
248
|
-
*
|
|
249
|
-
*
|
|
395
|
+
* Cursor movement for scripts that add events or move cursors. Lane and log
|
|
396
|
+
* cursor records are read once per script (`load_states`) and written back
|
|
397
|
+
* once (`flush_states`). A record whose head lies at or below the latest clear
|
|
398
|
+
* marker is idle, and a log cursor counts from the marker.
|
|
399
|
+
*
|
|
400
|
+
* A typed log cursor finds its next head in its types' indexes, one bounded
|
|
401
|
+
* lookup per type, skipped when `typed` shows no newer event of it. When an
|
|
402
|
+
* index does not reach back far enough yet (a typed backfill, or a type a
|
|
403
|
+
* cursor newly reads), the lookup returns the type, and the script returns
|
|
404
|
+
* `{ 'index', type, after }` before writing anything: the caller indexes
|
|
405
|
+
* that type down to `after` (INDEX_TYPE_LUA, one page per call) and runs the
|
|
406
|
+
* operation again.
|
|
250
407
|
*/
|
|
251
|
-
const
|
|
252
|
-
local
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
local
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
408
|
+
const CURSOR_LUA = `
|
|
409
|
+
local served_all, served_set, served_logs = true, {}, {}
|
|
410
|
+
local pending_fast = false
|
|
411
|
+
local function types_of(raw)
|
|
412
|
+
if not raw or raw == '*' then return nil end
|
|
413
|
+
local set = {}
|
|
414
|
+
for _, name in ipairs(cjson.decode(raw)) do set[name] = true end
|
|
415
|
+
return set
|
|
416
|
+
end
|
|
417
|
+
local function takes(types, kind)
|
|
418
|
+
return kind ~= CLEAR_TYPE and (types == nil or types[kind] == true)
|
|
419
|
+
end
|
|
420
|
+
-- The served consumers: '' for every consumer, else [[name] or
|
|
421
|
+
-- [name, lane, typesJson or '*', backfill]].
|
|
422
|
+
local function use_served(raw)
|
|
423
|
+
if raw == '' then return end
|
|
424
|
+
served_all = false
|
|
425
|
+
for _, entry in ipairs(cjson.decode(raw)) do
|
|
426
|
+
served_set[entry[1]] = true
|
|
427
|
+
if entry[2] then
|
|
428
|
+
served_logs[#served_logs + 1] = {
|
|
429
|
+
name = entry[1], lane = entry[2], ty = entry[3],
|
|
430
|
+
types = types_of(entry[3]), backfill = entry[4] == 1,
|
|
431
|
+
}
|
|
432
|
+
end
|
|
264
433
|
end
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
434
|
+
end
|
|
435
|
+
local function served_has(name) return served_all or served_set[name] == true end
|
|
436
|
+
-- Whether a served consumer has a live head. Newest heads come first: a
|
|
437
|
+
-- removed consumer's backlog is older than the work a deployment serves.
|
|
438
|
+
local function has_pending()
|
|
439
|
+
if pending_fast then return '1' end
|
|
440
|
+
if served_all then return redis.call('ZCOUNT', READY, 1, '+inf') > 0 and '1' or '0' end
|
|
441
|
+
if next(served_set) == nil then return '0' end
|
|
442
|
+
local page = redis.call('ZREVRANGEBYSCORE', READY, '+inf', 1, 'LIMIT', 0, 32)
|
|
443
|
+
for _, cid in ipairs(page) do
|
|
444
|
+
if served_set[(parse_cid(cid))] then return '1' end
|
|
271
445
|
end
|
|
272
|
-
if
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
put(META, lane_field(lane, 'head'), field)
|
|
276
|
-
ready_now[#ready_now + 1] = score
|
|
277
|
-
ready_now[#ready_now + 1] = field
|
|
446
|
+
if #page < 32 then return '0' end
|
|
447
|
+
for _, cid in ipairs(redis.call('ZREVRANGEBYSCORE', READY, '+inf', 1, 'LIMIT', 32, -1)) do
|
|
448
|
+
if served_set[(parse_cid(cid))] then return '1' end
|
|
278
449
|
end
|
|
279
|
-
|
|
280
|
-
end
|
|
281
|
-
local function
|
|
282
|
-
if
|
|
283
|
-
|
|
284
|
-
end
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
450
|
+
return '0'
|
|
451
|
+
end
|
|
452
|
+
local function targets_of(raw)
|
|
453
|
+
if raw == '' then return {} end
|
|
454
|
+
return cjson.decode(raw)
|
|
455
|
+
end
|
|
456
|
+
-- Starts indexing a type: every later event of it.
|
|
457
|
+
local function register(name, from)
|
|
458
|
+
typed[name] = { from, from }
|
|
459
|
+
typed_dirty = true
|
|
460
|
+
end
|
|
461
|
+
-- The first stored event in (after, total] of a type in \`types\` (nil: any
|
|
462
|
+
-- type), or false. Nil and a type when that type's index can't answer yet.
|
|
463
|
+
local function stored_head(after, types)
|
|
464
|
+
if types == nil then
|
|
465
|
+
if after + 1 <= total then return after + 1 end
|
|
466
|
+
return false
|
|
467
|
+
end
|
|
468
|
+
local best = false
|
|
469
|
+
for name in pairs(types) do
|
|
470
|
+
local entry = typed[name]
|
|
471
|
+
if entry == nil then
|
|
472
|
+
if after < total then return nil, name end
|
|
473
|
+
register(name, total)
|
|
474
|
+
elseif entry[2] > after then
|
|
475
|
+
if entry[1] > after then return nil, name end
|
|
476
|
+
local found = redis.call('ZRANGEBYSCORE', type_key(name),
|
|
477
|
+
'(' .. tostring(after), tostring(total), 'LIMIT', 0, 1)
|
|
478
|
+
if #found > 0 then
|
|
479
|
+
local idx = tonumber(found[1])
|
|
480
|
+
if not best or idx < best then best = idx end
|
|
481
|
+
end
|
|
482
|
+
end
|
|
483
|
+
end
|
|
484
|
+
return best
|
|
485
|
+
end
|
|
486
|
+
-- The first event after \`after\` a log cursor takes: stored, then in the
|
|
487
|
+
-- batch being appended.
|
|
488
|
+
local function next_log_head(after, types, batch)
|
|
489
|
+
local head, missing = stored_head(after, types)
|
|
490
|
+
if head == nil then return nil, missing end
|
|
491
|
+
if head then return head end
|
|
492
|
+
for _, event in ipairs(batch) do
|
|
493
|
+
if event.idx > after and takes(types, event.type) then return event.idx end
|
|
494
|
+
end
|
|
495
|
+
return false
|
|
496
|
+
end
|
|
497
|
+
-- Indexes a batch's events of the types typed log cursors read.
|
|
498
|
+
local function index_batch(batch)
|
|
499
|
+
local adds, order = {}, {}
|
|
500
|
+
for _, event in ipairs(batch) do
|
|
501
|
+
local entry = typed[event.type]
|
|
502
|
+
if entry then
|
|
503
|
+
local list = adds[event.type]
|
|
504
|
+
if not list then
|
|
505
|
+
list = {}
|
|
506
|
+
adds[event.type] = list
|
|
507
|
+
order[#order + 1] = event.type
|
|
508
|
+
end
|
|
509
|
+
list[#list + 1] = event.idx
|
|
510
|
+
list[#list + 1] = tostring(event.idx)
|
|
511
|
+
entry[2] = event.idx
|
|
512
|
+
typed_dirty = true
|
|
513
|
+
end
|
|
514
|
+
end
|
|
515
|
+
for _, name in ipairs(order) do sliced('ZADD', type_key(name), adds[name]) end
|
|
516
|
+
end
|
|
517
|
+
local states, state_order = {}, {}
|
|
518
|
+
local function load_states(cids)
|
|
519
|
+
local need, names = {}, {}
|
|
520
|
+
for _, cid in ipairs(cids) do
|
|
521
|
+
local state = states[cid]
|
|
522
|
+
if (state == nil or state.partial) and not need[cid] then
|
|
523
|
+
need[cid] = true
|
|
524
|
+
need[#need + 1] = cid
|
|
525
|
+
names[#names + 1] = cid .. ':h'
|
|
526
|
+
names[#names + 1] = cid .. ':p'
|
|
527
|
+
names[#names + 1] = cid .. ':t'
|
|
528
|
+
names[#names + 1] = cid .. ':ty'
|
|
529
|
+
end
|
|
530
|
+
end
|
|
531
|
+
if #need == 0 then return end
|
|
532
|
+
local values = sliced('HMGET', CURSORS, names)
|
|
533
|
+
for n, cid in ipairs(need) do
|
|
534
|
+
local h, p, t, ty = values[n * 4 - 3], values[n * 4 - 2], values[n * 4 - 1], values[n * 4]
|
|
535
|
+
local tail = (t and t ~= '') and tonumber(t) or false
|
|
536
|
+
local state = states[cid]
|
|
537
|
+
if state then
|
|
538
|
+
state.t = tail
|
|
539
|
+
state.partial = nil
|
|
289
540
|
else
|
|
290
|
-
|
|
541
|
+
local _, tag = parse_cid(cid)
|
|
542
|
+
state = {
|
|
543
|
+
kind = string.sub(tag, 1, 1), exists = p ~= false,
|
|
544
|
+
h = (h and h ~= '') and tonumber(h) or false,
|
|
545
|
+
p = tonumber(p or '0'), t = tail, ty = ty or false,
|
|
546
|
+
}
|
|
547
|
+
if state.h and state.h <= boundary then state.h = false end
|
|
548
|
+
if state.kind == 'G' and state.exists and state.p < boundary then state.p = boundary end
|
|
549
|
+
states[cid] = state
|
|
550
|
+
state_order[#state_order + 1] = cid
|
|
291
551
|
end
|
|
292
552
|
end
|
|
293
|
-
sliced('ZADD', PENDING, queued)
|
|
294
|
-
sliced('ZADD', READY, ready_now)
|
|
295
553
|
end
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
redis.call('ZREM', READY, field)
|
|
307
|
-
if lane == '' then return end
|
|
308
|
-
local head_key = lane_field(lane, 'head')
|
|
309
|
-
if redis.call('HGET', META, head_key) ~= field then return end
|
|
310
|
-
while next_field do
|
|
311
|
-
local row = read_fields({ next_field }, { 'p', 'f', 'next' })[next_field]
|
|
312
|
-
if not row[1] then
|
|
313
|
-
redis.call('HSET', META, head_key, next_field)
|
|
314
|
-
if not row[2] then
|
|
315
|
-
redis.call('ZADD', READY, tonumber(next_field), next_field)
|
|
554
|
+
local function flush_states()
|
|
555
|
+
for _, cid in ipairs(state_order) do
|
|
556
|
+
local state = states[cid]
|
|
557
|
+
if state.dirty then
|
|
558
|
+
put(CURSORS, cid .. ':h', state.h and tostring(state.h) or '')
|
|
559
|
+
put(CURSORS, cid .. ':p', tostring(state.p))
|
|
560
|
+
if state.kind == 'L' then
|
|
561
|
+
if not state.partial then put(CURSORS, cid .. ':t', state.t and tostring(state.t) or '') end
|
|
562
|
+
elseif state.ty then
|
|
563
|
+
put(CURSORS, cid .. ':ty', state.ty)
|
|
316
564
|
end
|
|
317
|
-
return
|
|
318
565
|
end
|
|
319
|
-
next_field = row[3]
|
|
320
566
|
end
|
|
321
|
-
clear_lane(lane)
|
|
322
567
|
end
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
local
|
|
333
|
-
local
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
local rec = cjson.decode(owner)
|
|
342
|
-
if rec[1] == session and tonumber(rec[2]) < boundary then owner = false end
|
|
568
|
+
-- Moves a cursor past its head \`idx\`. \`kind\` is the head row's kind;
|
|
569
|
+
-- \`batch\` is the returned events, appended after the stored log. Writes
|
|
570
|
+
-- nothing; returns nil and a type when that type's index must reach back
|
|
571
|
+
-- to \`idx\` first.
|
|
572
|
+
local function advance(consumer, kind, idx, row_next, batch)
|
|
573
|
+
if kind == '#' then
|
|
574
|
+
ready_set(cid_of(consumer, '#' .. tostring(idx)), false)
|
|
575
|
+
return true
|
|
576
|
+
end
|
|
577
|
+
local cid = cid_of(consumer, kind)
|
|
578
|
+
local state = states[cid]
|
|
579
|
+
local next_head
|
|
580
|
+
if string.sub(kind, 1, 1) == 'L' then
|
|
581
|
+
next_head = row_next and tonumber(row_next) or false
|
|
582
|
+
if not state then
|
|
583
|
+
state = { kind = 'L', exists = true, p = idx, h = false, t = false, ty = false, partial = true }
|
|
584
|
+
states[cid] = state
|
|
585
|
+
state_order[#state_order + 1] = cid
|
|
343
586
|
end
|
|
344
|
-
|
|
587
|
+
if not next_head then
|
|
588
|
+
state.t = false
|
|
589
|
+
state.partial = nil
|
|
590
|
+
end
|
|
591
|
+
else
|
|
592
|
+
local lane, raw = string.sub(kind, 2), nil
|
|
593
|
+
for _, spec in ipairs(served_logs) do
|
|
594
|
+
if spec.name == consumer and spec.lane == lane then raw = spec.ty end
|
|
595
|
+
end
|
|
596
|
+
if not raw then
|
|
597
|
+
load_states({ cid })
|
|
598
|
+
raw = states[cid].ty or '*'
|
|
599
|
+
end
|
|
600
|
+
local head, missing = next_log_head(idx, types_of(raw), batch)
|
|
601
|
+
if head == nil then return nil, missing end
|
|
602
|
+
next_head = head
|
|
603
|
+
state = states[cid]
|
|
604
|
+
if not state then
|
|
605
|
+
state = { kind = 'G', exists = true, p = idx, h = false, t = false, ty = raw }
|
|
606
|
+
states[cid] = state
|
|
607
|
+
state_order[#state_order + 1] = cid
|
|
608
|
+
end
|
|
609
|
+
state.ty = raw
|
|
345
610
|
end
|
|
346
|
-
|
|
611
|
+
state.h = next_head
|
|
612
|
+
state.p = idx
|
|
613
|
+
state.exists = true
|
|
614
|
+
state.dirty = true
|
|
615
|
+
ready_set(cid, next_head)
|
|
616
|
+
return true
|
|
347
617
|
end
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
*/
|
|
360
|
-
const WAKE_LUA = `
|
|
361
|
-
local wake_entries, wake_bytes = {}, 0
|
|
362
|
-
local function wake_add(field, id, typ, payload)
|
|
363
|
-
if wake_bytes + #payload > ${WAKE_PAYLOAD_LIMIT} then
|
|
364
|
-
wake_bytes = ${WAKE_PAYLOAD_LIMIT} + 1
|
|
365
|
-
return
|
|
618
|
+
-- Reads what an appended batch's cursor work needs: its lane targets'
|
|
619
|
+
-- records, and the head of each served log consumer whose types it carries.
|
|
620
|
+
-- Writes nothing; returns nil, a type, and an index when that type's index
|
|
621
|
+
-- must reach back to the index first.
|
|
622
|
+
local log_plan = {}
|
|
623
|
+
local function plan(batch, base)
|
|
624
|
+
local need = {}
|
|
625
|
+
for _, event in ipairs(batch) do
|
|
626
|
+
for _, target in ipairs(event.targets) do
|
|
627
|
+
if target[2] ~= '#' then need[#need + 1] = cid_of(target[1], target[2]) end
|
|
628
|
+
end
|
|
366
629
|
end
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
630
|
+
for _, spec in ipairs(served_logs) do
|
|
631
|
+
for _, event in ipairs(batch) do
|
|
632
|
+
if takes(spec.types, event.type) then
|
|
633
|
+
local cid = cid_of(spec.name, 'G' .. spec.lane)
|
|
634
|
+
log_plan[#log_plan + 1] = { spec = spec, first = event.idx, cid = cid }
|
|
635
|
+
need[#need + 1] = cid
|
|
636
|
+
break
|
|
637
|
+
end
|
|
638
|
+
end
|
|
372
639
|
end
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
640
|
+
load_states(need)
|
|
641
|
+
for _, log in ipairs(log_plan) do
|
|
642
|
+
local state, spec = states[log.cid], log.spec
|
|
643
|
+
if not state.exists then
|
|
644
|
+
if spec.backfill then
|
|
645
|
+
local head, missing = stored_head(boundary, spec.types)
|
|
646
|
+
if head == nil then return nil, missing, boundary end
|
|
647
|
+
log.p, log.h = boundary, head or log.first
|
|
648
|
+
else
|
|
649
|
+
-- Nothing it takes is stored after its start: index its types from there.
|
|
650
|
+
for name in pairs(spec.types or {}) do
|
|
651
|
+
if typed[name] == nil then register(name, base) end
|
|
652
|
+
end
|
|
653
|
+
log.p, log.h = base, log.first
|
|
654
|
+
end
|
|
655
|
+
elseif not state.h then
|
|
656
|
+
-- Not the batch's first: a deployment that doesn't serve this consumer
|
|
657
|
+
-- may have appended events it takes since it went idle.
|
|
658
|
+
local head, missing = stored_head(state.p, spec.types)
|
|
659
|
+
if head == nil then return nil, missing, state.p end
|
|
660
|
+
log.h = head or log.first
|
|
661
|
+
end
|
|
379
662
|
end
|
|
380
|
-
|
|
663
|
+
return true
|
|
381
664
|
end
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
665
|
+
-- Starts the cursor work \`plan\` read: each event's targets, and each
|
|
666
|
+
-- planned log cursor.
|
|
667
|
+
local function enqueue(batch)
|
|
668
|
+
for _, event in ipairs(batch) do
|
|
669
|
+
for _, target in ipairs(event.targets) do
|
|
670
|
+
local consumer, tag = target[1], target[2]
|
|
671
|
+
if served_has(consumer) then pending_fast = true end
|
|
672
|
+
if tag == '#' then
|
|
673
|
+
ready_set(cid_of(consumer, '#' .. tostring(event.idx)), event.idx)
|
|
674
|
+
else
|
|
675
|
+
local cid = cid_of(consumer, tag)
|
|
676
|
+
local state = states[cid]
|
|
677
|
+
if state.h and state.t then
|
|
678
|
+
put_row(state.t, row_of(consumer, tag), 'next', tostring(event.idx))
|
|
679
|
+
else
|
|
680
|
+
state.h = event.idx
|
|
681
|
+
if state.p < boundary then state.p = boundary end
|
|
682
|
+
ready_set(cid, event.idx)
|
|
683
|
+
end
|
|
684
|
+
state.t = event.idx
|
|
685
|
+
state.exists = true
|
|
686
|
+
state.dirty = true
|
|
687
|
+
end
|
|
394
688
|
end
|
|
395
689
|
end
|
|
396
|
-
|
|
690
|
+
for _, log in ipairs(log_plan) do
|
|
691
|
+
pending_fast = true
|
|
692
|
+
local state = states[log.cid]
|
|
693
|
+
if not state.exists then
|
|
694
|
+
state.exists, state.p, state.h = true, log.p, log.h
|
|
695
|
+
ready_set(log.cid, log.h)
|
|
696
|
+
elseif not state.h then
|
|
697
|
+
state.h = log.h
|
|
698
|
+
ready_set(log.cid, log.h)
|
|
699
|
+
end
|
|
700
|
+
state.ty = log.spec.ty
|
|
701
|
+
state.dirty = true
|
|
702
|
+
end
|
|
397
703
|
end
|
|
398
704
|
`
|
|
399
705
|
|
|
400
706
|
/**
|
|
401
|
-
* KEYS:
|
|
402
|
-
*
|
|
403
|
-
*
|
|
404
|
-
* id, type, payloadJson, causeJson, lane, settled; then notify channel and
|
|
405
|
-
* origin index or ''.
|
|
707
|
+
* KEYS: counter.
|
|
708
|
+
* ARGV: sessionId, nowMs, n, served consumers, origin index or '', then per
|
|
709
|
+
* event: id, type, payloadJson, causeJson or '', targets.
|
|
406
710
|
* Returns ['empty', hasPending] | ['ok', firstIndex, hasPending] |
|
|
407
711
|
* ['dup', hasPending, ...indexes] | ['partial', count] |
|
|
408
|
-
* ['
|
|
409
|
-
* ['
|
|
712
|
+
* ['superseded', position] | ['cleared', originIndex] |
|
|
713
|
+
* ['index', type, after].
|
|
410
714
|
*/
|
|
411
|
-
const APPEND_LUA =
|
|
412
|
-
local
|
|
413
|
-
|
|
414
|
-
${ENQUEUE_LUA}
|
|
415
|
-
${ID_OWNERS_LUA}
|
|
416
|
-
${WAKE_LUA}
|
|
417
|
-
local function has_pending()
|
|
418
|
-
return redis.call('ZCARD', PENDING) > 0 and '1' or '0'
|
|
419
|
-
end
|
|
420
|
-
local n = tonumber(ARGV[3])
|
|
715
|
+
const APPEND_LUA = `${BASE_LUA}${CURSOR_LUA}
|
|
716
|
+
local now, n = ARGV[2], tonumber(ARGV[3])
|
|
717
|
+
use_served(ARGV[4])
|
|
421
718
|
if n == 0 then return { 'empty', has_pending() } end
|
|
422
|
-
local
|
|
423
|
-
|
|
719
|
+
local ids = {}
|
|
720
|
+
for i = 1, n do ids[i] = ARGV[5 + (i - 1) * 5 + 1] end
|
|
721
|
+
local owners = owners_of(read_bounds(ids))
|
|
722
|
+
local base = total
|
|
424
723
|
local dups = {}
|
|
425
724
|
for i = 1, n do
|
|
426
|
-
if owners[i] then
|
|
427
|
-
local rec = cjson.decode(owners[i])
|
|
428
|
-
if rec[1] ~= ARGV[1] then return { 'foreign', ARGV[3 + (i - 1) * 6 + 1] } end
|
|
429
|
-
dups[#dups + 1] = tostring(rec[2])
|
|
430
|
-
end
|
|
725
|
+
if owners[i] then dups[#dups + 1] = owners[i] end
|
|
431
726
|
end
|
|
432
727
|
if #dups == n then
|
|
433
728
|
local res = { 'dup', has_pending() }
|
|
@@ -435,253 +730,251 @@ if #dups == n then
|
|
|
435
730
|
return res
|
|
436
731
|
end
|
|
437
732
|
if #dups > 0 then return { 'partial', tostring(#dups) } end
|
|
438
|
-
local origin = ARGV[5
|
|
439
|
-
if origin
|
|
440
|
-
|
|
733
|
+
local origin = ARGV[5]
|
|
734
|
+
if origin ~= '' and tonumber(origin) <= boundary then return { 'cleared', origin } end
|
|
735
|
+
-- The attempt-currency fence: a cause's head row must still be its cursor's
|
|
736
|
+
-- head, at that attempt, not dead-lettered.
|
|
737
|
+
local refs, fenced = {}, {}
|
|
738
|
+
for i = 1, n do
|
|
739
|
+
local raw = ARGV[5 + (i - 1) * 5 + 4]
|
|
740
|
+
if raw ~= '' then
|
|
741
|
+
local cause = cjson.decode(raw)
|
|
742
|
+
local index = tonumber(cause.index)
|
|
743
|
+
if index <= boundary then return { 'superseded', tostring(i) } end
|
|
744
|
+
local tag = type(cause.lane) == 'string' and ('L' .. cause.lane) or '#'
|
|
745
|
+
refs[#refs + 1] = { index, row_of(cause.consumer, tag) }
|
|
746
|
+
fenced[#fenced + 1] = {
|
|
747
|
+
position = i, consumer = cause.consumer, attempt = tonumber(cause.attempt), tag = tag,
|
|
748
|
+
}
|
|
749
|
+
end
|
|
441
750
|
end
|
|
442
|
-
local
|
|
751
|
+
local rows = read_rows(refs, { 'd', 'f', 'p', 'l' })
|
|
752
|
+
for k, cause in ipairs(fenced) do
|
|
753
|
+
local row = rows[k]
|
|
754
|
+
if tonumber(row[1] or '0') ~= cause.attempt or row[2] or row[3] or not owns(row[4], cause.tag) then
|
|
755
|
+
return { 'superseded', tostring(cause.position) }
|
|
756
|
+
end
|
|
757
|
+
if served_has(cause.consumer) then pending_fast = true end
|
|
758
|
+
end
|
|
759
|
+
local batch = {}
|
|
443
760
|
for i = 1, n do
|
|
444
|
-
local
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
end
|
|
451
|
-
end
|
|
452
|
-
local parent_rows = read_fields(parents, { 'd', 'f', 'p' })
|
|
453
|
-
local parent_pending = false
|
|
454
|
-
for _, parsed in ipairs(causes) do
|
|
455
|
-
local pidx = tonumber(parsed.index)
|
|
456
|
-
if pidx <= boundary then return { 'superseded', tostring(pidx) } end
|
|
457
|
-
if pidx < 1 or pidx > base then
|
|
458
|
-
return { 'no_parent', tostring(parsed.index) }
|
|
459
|
-
end
|
|
460
|
-
local row = parent_rows[tostring(pidx)]
|
|
461
|
-
if tonumber(row[1] or '0') ~= tonumber(parsed.attempt) or row[2] then
|
|
462
|
-
return { 'superseded', tostring(pidx) }
|
|
463
|
-
end
|
|
464
|
-
-- A claimed, unprocessed parent is still in the pending set.
|
|
465
|
-
if not row[3] then parent_pending = true end
|
|
466
|
-
end
|
|
467
|
-
local owner_writes = {}
|
|
761
|
+
local o = 5 + (i - 1) * 5
|
|
762
|
+
batch[i] = { idx = base + i, type = ARGV[o + 2], targets = targets_of(ARGV[o + 5]) }
|
|
763
|
+
end
|
|
764
|
+
local planned, missing, after = plan(batch, base)
|
|
765
|
+
if not planned then return { 'index', missing, tostring(after) } end
|
|
766
|
+
local marks = {}
|
|
468
767
|
for i = 1, n do
|
|
469
|
-
local o =
|
|
768
|
+
local o = 5 + (i - 1) * 5
|
|
470
769
|
local field = tostring(base + i)
|
|
471
|
-
redis.call('XADD',
|
|
770
|
+
redis.call('XADD', LOG, field .. '-0',
|
|
472
771
|
'id', ARGV[o + 1], 'type', ARGV[o + 2], 'payload', ARGV[o + 3],
|
|
473
|
-
'created_at',
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
flush_queue()
|
|
772
|
+
'created_at', now, 'cause', ARGV[o + 4])
|
|
773
|
+
marks[#marks + 1] = id_key(ARGV[o + 1])
|
|
774
|
+
marks[#marks + 1] = field
|
|
775
|
+
end
|
|
776
|
+
index_batch(batch)
|
|
777
|
+
enqueue(batch)
|
|
778
|
+
marks[#marks + 1] = COUNTER
|
|
779
|
+
marks[#marks + 1] = tostring(base + n)
|
|
780
|
+
if typed_dirty then
|
|
781
|
+
marks[#marks + 1] = TYPED
|
|
782
|
+
marks[#marks + 1] = cjson.encode(typed)
|
|
783
|
+
end
|
|
784
|
+
sliced('MSET', nil, marks)
|
|
785
|
+
flush_states()
|
|
488
786
|
flush()
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
return { 'ok', tostring(base + 1), pending_after }
|
|
787
|
+
flush_ready()
|
|
788
|
+
return { 'ok', tostring(base + 1), has_pending() }
|
|
492
789
|
`
|
|
493
790
|
|
|
494
791
|
/**
|
|
495
|
-
* KEYS: counter
|
|
496
|
-
*
|
|
497
|
-
*
|
|
792
|
+
* KEYS: counter.
|
|
793
|
+
* ARGV: type, after. Indexes one page of the stored log below the type's
|
|
794
|
+
* index, toward `after`, registering the type first if no cursor read it
|
|
795
|
+
* yet. Returns ['1'] once the index reaches `after`, else ['0'].
|
|
498
796
|
*/
|
|
499
|
-
const
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
local
|
|
511
|
-
|
|
512
|
-
local
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
if
|
|
517
|
-
if
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
else
|
|
535
|
-
redis.call('HDEL', meta_key, field .. ':ch', field .. ':ce')
|
|
536
|
-
end
|
|
537
|
-
if dead then
|
|
538
|
-
redis.call('ZREM', KEYS[4], field)
|
|
539
|
-
release_snapshot_pins(KEYS[5], KEYS[6])
|
|
540
|
-
return { 'dead_lettered', tostring(failures) }
|
|
797
|
+
const INDEX_TYPE_LUA = `${BASE_LUA}
|
|
798
|
+
read_bounds()
|
|
799
|
+
local name, after = ARGV[1], tonumber(ARGV[2])
|
|
800
|
+
local entry = typed[name]
|
|
801
|
+
if not entry then
|
|
802
|
+
entry = { total, total }
|
|
803
|
+
typed[name] = entry
|
|
804
|
+
end
|
|
805
|
+
-- Entries below the latest clear belong to an old lifetime: never indexed.
|
|
806
|
+
local floor = math.max(after, boundary - 1)
|
|
807
|
+
if entry[1] > floor then
|
|
808
|
+
local entries = redis.call('XREVRANGE', LOG, tostring(entry[1]) .. '-0',
|
|
809
|
+
tostring(floor + 1) .. '-0', 'COUNT', ${READ_PAGE_SIZE})
|
|
810
|
+
local adds = {}
|
|
811
|
+
for _, stored in ipairs(entries) do
|
|
812
|
+
local fields = stored[2]
|
|
813
|
+
for j = 1, #fields, 2 do
|
|
814
|
+
if fields[j] == 'type' then
|
|
815
|
+
if fields[j + 1] == name then
|
|
816
|
+
local idx = entry_index(stored[1])
|
|
817
|
+
adds[#adds + 1] = idx
|
|
818
|
+
adds[#adds + 1] = tostring(idx)
|
|
819
|
+
end
|
|
820
|
+
break
|
|
821
|
+
end
|
|
822
|
+
end
|
|
823
|
+
end
|
|
824
|
+
sliced('ZADD', type_key(name), adds)
|
|
825
|
+
if #entries < ${READ_PAGE_SIZE} then
|
|
826
|
+
entry[1] = after
|
|
827
|
+
else
|
|
828
|
+
entry[1] = entry_index(entries[#entries][1]) - 1
|
|
829
|
+
end
|
|
830
|
+
elseif entry[1] > after then
|
|
831
|
+
entry[1] = after
|
|
541
832
|
end
|
|
542
|
-
|
|
833
|
+
redis.call('SET', TYPED, cjson.encode(typed))
|
|
834
|
+
return { entry[1] <= after and '1' or '0' }
|
|
543
835
|
`
|
|
544
836
|
|
|
545
837
|
/**
|
|
546
|
-
* KEYS:
|
|
547
|
-
*
|
|
548
|
-
*
|
|
549
|
-
*
|
|
550
|
-
*
|
|
551
|
-
*
|
|
838
|
+
* KEYS: counter.
|
|
839
|
+
* ARGV: holder, nowMs, expiresAtMs, maxFailures or '', served consumers,
|
|
840
|
+
* the highest head to claim or '', excluded count, then per excluded head:
|
|
841
|
+
* consumer, tag.
|
|
842
|
+
* Live heads are the ready set's positive scores, so an empty one is enough
|
|
843
|
+
* to report 'settled'. A claimed result ends with the clear marker.
|
|
552
844
|
*/
|
|
553
|
-
const CLAIM_AVAILABLE_LUA =
|
|
554
|
-
local
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
local
|
|
845
|
+
const CLAIM_AVAILABLE_LUA = `${BASE_LUA}${CURSOR_LUA}
|
|
846
|
+
local ready = redis.call('ZRANGEBYSCORE', READY, 1, ARGV[6] == '' and '+inf' or ARGV[6], 'WITHSCORES')
|
|
847
|
+
if #ready == 0 then return { 'settled' } end
|
|
848
|
+
read_bounds()
|
|
849
|
+
local holder, now, expires_at = ARGV[1], tonumber(ARGV[2]), ARGV[3]
|
|
850
|
+
local max_failures = tonumber(ARGV[4])
|
|
851
|
+
use_served(ARGV[5])
|
|
558
852
|
local excluded = {}
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
853
|
+
for i = 1, tonumber(ARGV[7]) do
|
|
854
|
+
excluded[ARGV[7 + (i - 1) * 2 + 1] .. '\\0' .. ARGV[7 + (i - 1) * 2 + 2]] = true
|
|
855
|
+
end
|
|
562
856
|
local candidates = {}
|
|
563
|
-
for
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
local
|
|
567
|
-
local
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
857
|
+
for i = 1, #ready, 2 do
|
|
858
|
+
local cid, idx = ready[i], tonumber(ready[i + 1])
|
|
859
|
+
local consumer, tag = parse_cid(cid)
|
|
860
|
+
local kind = string.sub(tag, 1, 1)
|
|
861
|
+
local key = consumer .. '\\0' .. (kind == 'G' and ('L' .. string.sub(tag, 2)) or tag)
|
|
862
|
+
if served_has(consumer) and not excluded[key] then
|
|
863
|
+
candidates[#candidates + 1] = {
|
|
864
|
+
idx, row_of(consumer, tag), consumer = consumer, cid = cid,
|
|
865
|
+
kind = kind == '#' and '#' or tag,
|
|
866
|
+
}
|
|
867
|
+
end
|
|
868
|
+
end
|
|
869
|
+
local rows = read_rows(candidates, { 'd', 'fc', 'lc', 'ch', 'ce', 'a', 'lf', 'lfa', 'e', 'f', 'pins' })
|
|
870
|
+
local takes_over, retry_at, wanted = {}, nil, {}
|
|
871
|
+
for position, candidate in ipairs(candidates) do
|
|
872
|
+
local row = rows[position]
|
|
873
|
+
local expires = tonumber(row[5] or '0')
|
|
874
|
+
if expires > now then
|
|
574
875
|
if not retry_at or expires < retry_at then retry_at = expires end
|
|
575
876
|
else
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
local meta_key = mkey(field)
|
|
579
|
-
local attempt = tostring(tonumber(row[2] or '0') + 1)
|
|
580
|
-
-- An expiry that is not the current attempt's retry hold is a lapsed
|
|
581
|
-
-- claim: its attempt never finished.
|
|
582
|
-
local lapsed = row[1] and row[7] ~= row[2]
|
|
583
|
-
local failures = tonumber(row[4] or '0')
|
|
584
|
-
if lapsed then
|
|
585
|
-
failures = failures + 1
|
|
586
|
-
put(meta_key, field .. ':a', tostring(failures))
|
|
587
|
-
end
|
|
588
|
-
put(meta_key, field .. ':d', attempt)
|
|
589
|
-
if lapsed and max_failures and failures >= max_failures then
|
|
590
|
-
-- That takeover spent the last failed attempt: dead-letter, unclaimed.
|
|
591
|
-
-- Advancing the attempt fences the lapsed one; nothing runs.
|
|
592
|
-
put(meta_key, field .. ':f', ARGV[2])
|
|
593
|
-
dead[#dead + 1] = field
|
|
594
|
-
claimed[#claimed + 1] = {
|
|
595
|
-
attempt,
|
|
596
|
-
tostring(failures),
|
|
597
|
-
row[5] or false,
|
|
598
|
-
row[3] or false,
|
|
599
|
-
read_fields({ field }, { 'lc' })[field][1] or false,
|
|
600
|
-
row[6] or false,
|
|
601
|
-
row[7] or false,
|
|
602
|
-
false,
|
|
603
|
-
false,
|
|
604
|
-
entries[1],
|
|
605
|
-
ARGV[2]
|
|
606
|
-
}
|
|
607
|
-
else
|
|
608
|
-
local first_claimed = row[3] or ARGV[2]
|
|
609
|
-
put(meta_key, field .. ':fc', first_claimed)
|
|
610
|
-
put(meta_key, field .. ':lc', ARGV[2])
|
|
611
|
-
put(meta_key, field .. ':ch', ARGV[1])
|
|
612
|
-
put(meta_key, field .. ':ce', ARGV[3])
|
|
613
|
-
claimed[#claimed + 1] = {
|
|
614
|
-
attempt,
|
|
615
|
-
tostring(failures),
|
|
616
|
-
row[5] or false,
|
|
617
|
-
first_claimed,
|
|
618
|
-
ARGV[2],
|
|
619
|
-
row[6] or false,
|
|
620
|
-
row[7] or false,
|
|
621
|
-
ARGV[1],
|
|
622
|
-
ARGV[3],
|
|
623
|
-
entries[1],
|
|
624
|
-
false
|
|
625
|
-
}
|
|
626
|
-
end
|
|
877
|
+
takes_over[#takes_over + 1] = { candidate = candidate, row = row }
|
|
878
|
+
wanted[candidate[1]] = true
|
|
627
879
|
end
|
|
628
880
|
end
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
881
|
+
if #takes_over == 0 then
|
|
882
|
+
if retry_at then return { 'busy', tostring(retry_at) } end
|
|
883
|
+
return { 'settled' }
|
|
884
|
+
end
|
|
885
|
+
-- One XRANGE per run of consecutive claimed indexes.
|
|
886
|
+
local indexes, entries = {}, {}
|
|
887
|
+
for idx in pairs(wanted) do indexes[#indexes + 1] = idx end
|
|
888
|
+
table.sort(indexes)
|
|
889
|
+
local first = 1
|
|
890
|
+
while first <= #indexes do
|
|
891
|
+
local last = first
|
|
892
|
+
while last < #indexes and indexes[last + 1] == indexes[last] + 1 do last = last + 1 end
|
|
893
|
+
local range = redis.call('XRANGE', LOG, tostring(indexes[first]) .. '-0', tostring(indexes[last]) .. '-0')
|
|
894
|
+
for _, entry in ipairs(range) do entries[entry_index(entry[1])] = entry end
|
|
895
|
+
first = last + 1
|
|
896
|
+
end
|
|
897
|
+
for _, idx in ipairs(indexes) do
|
|
898
|
+
if not entries[idx] then return { 'missing', tostring(idx) } end
|
|
899
|
+
end
|
|
900
|
+
local result = { 'claimed', tostring(#takes_over) }
|
|
901
|
+
for _, take in ipairs(takes_over) do
|
|
902
|
+
local candidate, row = take.candidate, take.row
|
|
903
|
+
local idx, at = candidate[1], candidate[2]
|
|
904
|
+
local attempt = tostring(tonumber(row[1] or '0') + 1)
|
|
905
|
+
-- A claim still held here lapsed: its attempt never finished.
|
|
906
|
+
local lapsed = row[4] ~= false
|
|
907
|
+
local failures = tonumber(row[6] or '0')
|
|
908
|
+
if lapsed then
|
|
909
|
+
failures = failures + 1
|
|
910
|
+
put_row(idx, at, 'a', tostring(failures))
|
|
639
911
|
end
|
|
640
|
-
|
|
641
|
-
|
|
912
|
+
put_row(idx, at, 'd', attempt)
|
|
913
|
+
local columns
|
|
914
|
+
if lapsed and max_failures and failures >= max_failures then
|
|
915
|
+
-- That takeover spent the last failed attempt: dead-letter, unclaimed.
|
|
916
|
+
-- Advancing the attempt fences the lapsed one; nothing runs.
|
|
917
|
+
put_row(idx, at, 'f', ARGV[2])
|
|
918
|
+
drop_row(idx, at, 'ch')
|
|
919
|
+
drop_row(idx, at, 'ce')
|
|
920
|
+
ready_set(candidate.cid, -idx)
|
|
921
|
+
release_pins(idx, at, row[11])
|
|
922
|
+
columns = { attempt, row[2], row[3], false, false, tostring(failures), row[7], row[8], row[9], ARGV[2] }
|
|
923
|
+
else
|
|
924
|
+
local first_claimed = row[2] or ARGV[2]
|
|
925
|
+
put_row(idx, at, 'fc', first_claimed)
|
|
926
|
+
put_row(idx, at, 'lc', ARGV[2])
|
|
927
|
+
put_row(idx, at, 'ch', holder)
|
|
928
|
+
put_row(idx, at, 'ce', expires_at)
|
|
929
|
+
put_row(idx, at, 'l', candidate.kind)
|
|
930
|
+
columns = { attempt, first_claimed, ARGV[2], holder, expires_at, tostring(failures), row[7], row[8], row[9], false }
|
|
931
|
+
end
|
|
932
|
+
result[#result + 1] = { candidate.consumer, candidate.kind, tostring(idx), columns, entries[idx] }
|
|
642
933
|
end
|
|
643
|
-
|
|
644
|
-
|
|
934
|
+
flush()
|
|
935
|
+
flush_ready()
|
|
936
|
+
result[#result + 1] = tostring(boundary)
|
|
937
|
+
result[#result + 1] = tostring(total)
|
|
938
|
+
return result
|
|
645
939
|
`
|
|
646
940
|
|
|
647
941
|
/**
|
|
648
|
-
* KEYS: counter
|
|
649
|
-
* ARGV: holder, nowMs, expiresAtMs, n, then per claim: index,
|
|
650
|
-
* Returns [renewedCount, ...renewed, ...superseded]
|
|
942
|
+
* KEYS: counter.
|
|
943
|
+
* ARGV: holder, nowMs, expiresAtMs, n, then per claim: consumer, tag, index,
|
|
944
|
+
* attempt. Returns [renewedCount, ...renewed, ...superseded] as 1-based
|
|
945
|
+
* positions in the request.
|
|
651
946
|
*/
|
|
652
|
-
const RENEW_CLAIMS_LUA =
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
local
|
|
656
|
-
local total = tonumber(counts[1] or '0')
|
|
657
|
-
use_split(counts[2], counts[1], counts[3])
|
|
658
|
-
local claims, fields, cleared = {}, {}, {}
|
|
947
|
+
const RENEW_CLAIMS_LUA = `${BASE_LUA}
|
|
948
|
+
read_bounds()
|
|
949
|
+
local holder, now, expires_at = ARGV[1], tonumber(ARGV[2]), ARGV[3]
|
|
950
|
+
local refs, items, superseded = {}, {}, {}
|
|
659
951
|
for i = 1, tonumber(ARGV[4]) do
|
|
660
|
-
local
|
|
661
|
-
local idx = tonumber(
|
|
952
|
+
local o = 4 + (i - 1) * 4
|
|
953
|
+
local idx = tonumber(ARGV[o + 3])
|
|
662
954
|
if idx <= boundary then
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
955
|
+
superseded[#superseded + 1] = tostring(i)
|
|
956
|
+
else
|
|
957
|
+
local row = row_of(ARGV[o + 1], ARGV[o + 2])
|
|
958
|
+
refs[#refs + 1] = { idx, row }
|
|
959
|
+
items[#items + 1] = { position = i, idx = idx, row = row, tag = ARGV[o + 2], attempt = tonumber(ARGV[o + 4]) }
|
|
667
960
|
end
|
|
668
961
|
end
|
|
669
|
-
local rows =
|
|
962
|
+
local rows = read_rows(refs, { 'd', 'ce', 'p', 'f', 'ch', 'l' })
|
|
670
963
|
local renewed = {}
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
local
|
|
674
|
-
if
|
|
675
|
-
|
|
676
|
-
elseif
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
964
|
+
for k, item in ipairs(items) do
|
|
965
|
+
local row = rows[k]
|
|
966
|
+
local attempt = tonumber(row[1] or '0')
|
|
967
|
+
if row[6] and not owns(row[6], item.tag) then
|
|
968
|
+
-- Another cursor's head: this one never claimed it.
|
|
969
|
+
elseif row[3] or attempt > item.attempt then
|
|
970
|
+
superseded[#superseded + 1] = tostring(item.position)
|
|
971
|
+
elseif attempt == item.attempt and tonumber(expires_at) > now and
|
|
972
|
+
not row[4] and row[5] == holder and tonumber(row[2] or '0') > now then
|
|
973
|
+
put_row(item.idx, item.row, 'ce', expires_at)
|
|
974
|
+
renewed[#renewed + 1] = tostring(item.position)
|
|
681
975
|
end
|
|
682
976
|
end
|
|
683
977
|
flush()
|
|
684
|
-
for _, field in ipairs(cleared) do superseded[#superseded + 1] = field end
|
|
685
978
|
local result = { tostring(#renewed) }
|
|
686
979
|
for i = 1, #renewed do result[#result + 1] = renewed[i] end
|
|
687
980
|
for i = 1, #superseded do result[#result + 1] = superseded[i] end
|
|
@@ -689,228 +982,393 @@ return result
|
|
|
689
982
|
`
|
|
690
983
|
|
|
691
984
|
/**
|
|
692
|
-
* KEYS:
|
|
693
|
-
*
|
|
694
|
-
*
|
|
695
|
-
*
|
|
985
|
+
* KEYS: counter.
|
|
986
|
+
* ARGV: sessionId, nowMs, consumer, tag, index, attempt, child causeJson,
|
|
987
|
+
* served consumers, returned idsJson, n, then per child: id, type,
|
|
988
|
+
* payloadJson, targets.
|
|
696
989
|
*/
|
|
697
|
-
const COMPLETE_ATTEMPT_LUA =
|
|
698
|
-
local
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
local
|
|
708
|
-
local owners, count = read_owners(8, n, ARGV[1])
|
|
709
|
-
local total = tonumber(count or '0')
|
|
710
|
-
if tonumber(parent) <= boundary then return { 'superseded' } end
|
|
711
|
-
if tonumber(parent) < 1 or tonumber(parent) > total then return { 'missing' } end
|
|
712
|
-
local row = read_fields({ parent }, { 'd', 'p', 'pa', 'ri', 'ch', 'f', 'lfa', 'l', 'next' })[parent]
|
|
990
|
+
const COMPLETE_ATTEMPT_LUA = `${BASE_LUA}${CURSOR_LUA}
|
|
991
|
+
local now = ARGV[2]
|
|
992
|
+
local consumer, tag, idx, attempt = ARGV[3], ARGV[4], tonumber(ARGV[5]), tonumber(ARGV[6])
|
|
993
|
+
local at = row_of(consumer, tag)
|
|
994
|
+
local n = tonumber(ARGV[10])
|
|
995
|
+
local ids = {}
|
|
996
|
+
for i = 1, n do ids[i] = ARGV[10 + (i - 1) * 4 + 1] end
|
|
997
|
+
local owners = owners_of(read_bounds(ids))
|
|
998
|
+
if idx <= boundary then return { 'superseded' } end
|
|
999
|
+
if idx < 1 or idx > total then return { 'missing' } end
|
|
1000
|
+
local row = read_rows({ { idx, at } }, { 'd', 'p', 'pa', 'ri', 'ch', 'f', 'l', 'next', 'pins' })[1]
|
|
713
1001
|
if row[2] then
|
|
714
|
-
|
|
715
|
-
if not row[
|
|
716
|
-
|
|
717
|
-
|
|
1002
|
+
-- The cursor moved past this head: a retry of its completion replays.
|
|
1003
|
+
if not owns(row[7], tag) then return { 'superseded' } end
|
|
1004
|
+
if not row[3] or tonumber(row[3]) ~= attempt then return { 'superseded' } end
|
|
1005
|
+
local recorded = cjson.decode(row[4] or '[]')
|
|
1006
|
+
if #recorded ~= n then return { 'partial' } end
|
|
718
1007
|
for i = 1, n do
|
|
719
|
-
if
|
|
1008
|
+
if recorded[i] ~= ARGV[10 + (i - 1) * 4 + 1] then return { 'partial' } end
|
|
720
1009
|
end
|
|
721
|
-
local
|
|
1010
|
+
local result = { 'duplicate' }
|
|
722
1011
|
for i = 1, n do
|
|
723
1012
|
if not owners[i] then return { 'partial' } end
|
|
724
|
-
|
|
725
|
-
if rec[1] ~= ARGV[1] then return { 'partial' } end
|
|
726
|
-
local entries = redis.call('XRANGE', KEYS[1], tostring(rec[2]) .. '-0', tostring(rec[2]) .. '-0')
|
|
727
|
-
if #entries == 0 then return { 'partial' } end
|
|
728
|
-
local values = entries[1][2]
|
|
729
|
-
local cause = nil
|
|
730
|
-
for j = 1, #values, 2 do
|
|
731
|
-
if values[j] == 'cause' then cause = values[j + 1] break end
|
|
732
|
-
end
|
|
733
|
-
if not cause or cause == '' then return { 'partial' } end
|
|
734
|
-
local decoded = cjson.decode(cause)
|
|
735
|
-
if tonumber(decoded['index']) ~= tonumber(parent) or
|
|
736
|
-
tonumber(decoded['attempt']) ~= attempt then return { 'partial' } end
|
|
737
|
-
indexes[#indexes + 1] = tostring(rec[2])
|
|
1013
|
+
result[#result + 1] = owners[i]
|
|
738
1014
|
end
|
|
739
|
-
local result = { 'duplicate' }
|
|
740
|
-
for _, idx in ipairs(indexes) do result[#result + 1] = idx end
|
|
741
1015
|
return result
|
|
742
1016
|
end
|
|
743
|
-
if tonumber(row[1] or '0') ~= attempt
|
|
744
|
-
if not row[5] or row[6] or tonumber(row[7] or '0') == attempt then
|
|
1017
|
+
if tonumber(row[1] or '0') ~= attempt or not row[5] or row[6] or not owns(row[7], tag) then
|
|
745
1018
|
return { 'superseded' }
|
|
746
1019
|
end
|
|
747
1020
|
for i = 1, n do
|
|
748
1021
|
if owners[i] then return { 'partial' } end
|
|
749
1022
|
end
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
local returned_json = '[]'
|
|
753
|
-
if n > 0 then returned_json = cjson.encode(returned_ids) end
|
|
754
|
-
local parent_key = mkey(parent)
|
|
755
|
-
put(parent_key, parent .. ':p', ARGV[2])
|
|
756
|
-
put(parent_key, parent .. ':pa', ARGV[4])
|
|
757
|
-
put(parent_key, parent .. ':ri', returned_json)
|
|
758
|
-
redis.call('HDEL', parent_key, parent .. ':ch', parent .. ':ce')
|
|
759
|
-
settle(parent, row[8] or '', row[9])
|
|
760
|
-
release_snapshot_pins(KEYS[6], KEYS[7])
|
|
761
|
-
local cause = cjson.encode({ index = tonumber(parent), attempt = attempt })
|
|
762
|
-
local owner_writes = {}
|
|
1023
|
+
use_served(ARGV[8])
|
|
1024
|
+
local batch = {}
|
|
763
1025
|
for i = 1, n do
|
|
764
|
-
local o =
|
|
1026
|
+
local o = 10 + (i - 1) * 4
|
|
1027
|
+
batch[i] = { idx = total + i, type = ARGV[o + 2], targets = targets_of(ARGV[o + 4]) }
|
|
1028
|
+
end
|
|
1029
|
+
local moved, missing = advance(consumer, row[7], idx, row[8], batch)
|
|
1030
|
+
if not moved then return { 'index', missing, tostring(idx) } end
|
|
1031
|
+
local planned, unindexed, after = plan(batch, total)
|
|
1032
|
+
if not planned then return { 'index', unindexed, tostring(after) } end
|
|
1033
|
+
put_row(idx, at, 'p', now)
|
|
1034
|
+
put_row(idx, at, 'pa', ARGV[6])
|
|
1035
|
+
put_row(idx, at, 'ri', ARGV[9])
|
|
1036
|
+
release_pins(idx, at, row[9])
|
|
1037
|
+
local marks = {}
|
|
1038
|
+
for i = 1, n do
|
|
1039
|
+
local o = 10 + (i - 1) * 4
|
|
765
1040
|
local field = tostring(total + i)
|
|
766
|
-
redis.call('XADD',
|
|
1041
|
+
redis.call('XADD', LOG, field .. '-0',
|
|
767
1042
|
'id', ARGV[o + 1], 'type', ARGV[o + 2], 'payload', ARGV[o + 3],
|
|
768
|
-
'created_at',
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
wake_add(field, ARGV[o + 1], ARGV[o + 2], ARGV[o + 3])
|
|
772
|
-
if ARGV[o + 5] == '1' then
|
|
773
|
-
put(mkey(field), field .. ':p', ARGV[2])
|
|
774
|
-
else
|
|
775
|
-
enqueue(field, ARGV[o + 4])
|
|
776
|
-
end
|
|
1043
|
+
'created_at', now, 'cause', ARGV[7])
|
|
1044
|
+
marks[#marks + 1] = id_key(ARGV[o + 1])
|
|
1045
|
+
marks[#marks + 1] = field
|
|
777
1046
|
end
|
|
1047
|
+
index_batch(batch)
|
|
1048
|
+
enqueue(batch)
|
|
778
1049
|
if n > 0 then
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
split_write(owner_writes)
|
|
782
|
-
sliced('MSET', nil, owner_writes)
|
|
1050
|
+
marks[#marks + 1] = COUNTER
|
|
1051
|
+
marks[#marks + 1] = tostring(total + n)
|
|
783
1052
|
end
|
|
784
|
-
|
|
1053
|
+
if typed_dirty then
|
|
1054
|
+
marks[#marks + 1] = TYPED
|
|
1055
|
+
marks[#marks + 1] = cjson.encode(typed)
|
|
1056
|
+
end
|
|
1057
|
+
sliced('MSET', nil, marks)
|
|
1058
|
+
flush_states()
|
|
785
1059
|
flush()
|
|
786
|
-
|
|
1060
|
+
flush_ready()
|
|
787
1061
|
return { 'ok', tostring(total + 1) }
|
|
788
1062
|
`
|
|
789
1063
|
|
|
790
1064
|
/**
|
|
791
|
-
* KEYS: counter
|
|
792
|
-
* ARGV:
|
|
793
|
-
*
|
|
794
|
-
* event above the clear boundary. A retry keeps the event pending and makes
|
|
795
|
-
* it ready again when it is unlaned or its lane's head.
|
|
1065
|
+
* KEYS: counter.
|
|
1066
|
+
* ARGV: consumer, tag, index, attempt, error, maxFailures, nowMs, retryAtMs
|
|
1067
|
+
* or ''.
|
|
796
1068
|
*/
|
|
797
|
-
const
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
end
|
|
805
|
-
local
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
local
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
1069
|
+
const FAIL_ATTEMPT_LUA = `${BASE_LUA}
|
|
1070
|
+
read_bounds()
|
|
1071
|
+
local consumer, tag, idx, attempt = ARGV[1], ARGV[2], tonumber(ARGV[3]), tonumber(ARGV[4])
|
|
1072
|
+
local at = row_of(consumer, tag)
|
|
1073
|
+
if idx <= boundary then return { 'superseded', '0' } end
|
|
1074
|
+
if idx < 1 or idx > total then return { 'missing', '0' } end
|
|
1075
|
+
local row = read_rows({ { idx, at } }, { 'a', 'd', 'p', 'f', 'lfa', 'ch', 'l', 'pins' })[1]
|
|
1076
|
+
if row[3] or (row[7] and not owns(row[7], tag)) then return { 'superseded', '0' } end
|
|
1077
|
+
local failures = tonumber(row[1] or '0')
|
|
1078
|
+
if tonumber(row[2] or '0') ~= attempt then return { 'superseded', tostring(failures) } end
|
|
1079
|
+
if not row[6] and row[5] and tonumber(row[5]) == attempt then
|
|
1080
|
+
return { row[4] and 'dead_lettered' or 'failed', tostring(failures) }
|
|
1081
|
+
end
|
|
1082
|
+
if row[4] then return { 'dead_lettered', tostring(failures) } end
|
|
1083
|
+
if not row[6] then return { 'superseded', tostring(failures) } end
|
|
1084
|
+
failures = failures + 1
|
|
1085
|
+
local dead = failures >= tonumber(ARGV[6])
|
|
1086
|
+
put_row(idx, at, 'a', tostring(failures))
|
|
1087
|
+
put_row(idx, at, 'e', ARGV[5])
|
|
1088
|
+
put_row(idx, at, 'lf', ARGV[7])
|
|
1089
|
+
put_row(idx, at, 'lfa', ARGV[4])
|
|
1090
|
+
drop_row(idx, at, 'ch')
|
|
1091
|
+
if dead then
|
|
1092
|
+
put_row(idx, at, 'f', ARGV[7])
|
|
1093
|
+
drop_row(idx, at, 'ce')
|
|
1094
|
+
ready_set(cid_for(consumer, row[7], idx), -idx)
|
|
1095
|
+
release_pins(idx, at, row[8])
|
|
1096
|
+
elseif ARGV[8] ~= '' then
|
|
1097
|
+
put_row(idx, at, 'ce', ARGV[8])
|
|
820
1098
|
else
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
1099
|
+
drop_row(idx, at, 'ce')
|
|
1100
|
+
end
|
|
1101
|
+
flush()
|
|
1102
|
+
flush_ready()
|
|
1103
|
+
return { dead and 'dead_lettered' or 'failed', tostring(failures) }
|
|
1104
|
+
`
|
|
1105
|
+
|
|
1106
|
+
/**
|
|
1107
|
+
* KEYS: counter. ARGV: consumer, index, action, nowMs, served
|
|
1108
|
+
* consumers. Returns [redriven, hasPending] | ['index', type, after]. Acts
|
|
1109
|
+
* only on the consumer's dead-lettered head at the index, above the clear
|
|
1110
|
+
* marker.
|
|
1111
|
+
*/
|
|
1112
|
+
const REDRIVE_LUA = `${BASE_LUA}${CURSOR_LUA}
|
|
1113
|
+
read_bounds()
|
|
1114
|
+
use_served(ARGV[5])
|
|
1115
|
+
local consumer, idx = ARGV[1], tonumber(ARGV[2])
|
|
1116
|
+
local function result(redriven) return { redriven, has_pending() } end
|
|
1117
|
+
if idx <= boundary or idx < 1 or idx > total then return result('0') end
|
|
1118
|
+
local cid, tag = nil, nil
|
|
1119
|
+
for _, member in ipairs(redis.call('ZRANGEBYSCORE', READY, -idx, -idx)) do
|
|
1120
|
+
local owner, member_tag = parse_cid(member)
|
|
1121
|
+
if owner == consumer then
|
|
1122
|
+
cid, tag = member, member_tag
|
|
1123
|
+
break
|
|
824
1124
|
end
|
|
825
1125
|
end
|
|
1126
|
+
if not cid then return result('0') end
|
|
1127
|
+
local kind = string.sub(tag, 1, 1) == '#' and '#' or tag
|
|
1128
|
+
local at = row_of(consumer, tag)
|
|
1129
|
+
if ARGV[3] == 'skip' then
|
|
1130
|
+
local row = read_rows({ { idx, at } }, { 'next' })[1]
|
|
1131
|
+
local moved, missing = advance(consumer, kind, idx, row[1], {})
|
|
1132
|
+
if not moved then return { 'index', missing, tostring(idx) } end
|
|
1133
|
+
put_row(idx, at, 'p', ARGV[4])
|
|
1134
|
+
flush_states()
|
|
1135
|
+
if typed_dirty then redis.call('SET', TYPED, cjson.encode(typed)) end
|
|
1136
|
+
else
|
|
1137
|
+
drop_row(idx, at, 'f')
|
|
1138
|
+
drop_row(idx, at, 'a')
|
|
1139
|
+
ready_set(cid, idx)
|
|
1140
|
+
end
|
|
1141
|
+
flush()
|
|
1142
|
+
flush_ready()
|
|
826
1143
|
return result('1')
|
|
827
1144
|
`
|
|
828
1145
|
|
|
829
1146
|
/**
|
|
830
|
-
* KEYS:
|
|
831
|
-
*
|
|
832
|
-
*
|
|
833
|
-
*
|
|
834
|
-
*
|
|
1147
|
+
* KEYS: counter. ARGV: consumer, lane, types JSON or '*', backfill
|
|
1148
|
+
* ('1' or '0'). Brings one log cursor up to date without an append, as
|
|
1149
|
+
* `plan` and `enqueue` would for an append carrying its types: a missing
|
|
1150
|
+
* cursor starts at the clear marker with backfill, else at the head with
|
|
1151
|
+
* nothing to run; an idle one takes the next stored event of its types; one
|
|
1152
|
+
* with a head, claimed or not, keeps it. Records the types, and writes
|
|
1153
|
+
* nothing when nothing changed. Returns ['ok', position, head or nil, head
|
|
1154
|
+
* columns, session head] | ['index', type, after].
|
|
835
1155
|
*/
|
|
836
|
-
const
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
local
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
1156
|
+
const WAKE_LOG_LUA = `${BASE_LUA}${CURSOR_LUA}
|
|
1157
|
+
read_bounds()
|
|
1158
|
+
local consumer, lane, ty = ARGV[1], ARGV[2], ARGV[3]
|
|
1159
|
+
local types = types_of(ty)
|
|
1160
|
+
local cid = cid_of(consumer, 'G' .. lane)
|
|
1161
|
+
load_states({ cid })
|
|
1162
|
+
local state = states[cid]
|
|
1163
|
+
local had_head = state.h
|
|
1164
|
+
-- Nothing after the latest clear: a missing cursor stays missing.
|
|
1165
|
+
if not state.exists and total == boundary then
|
|
1166
|
+
return { 'ok', tostring(boundary), false, {}, tostring(total) }
|
|
1167
|
+
end
|
|
1168
|
+
if not state.exists then
|
|
1169
|
+
state.exists, state.p = true, total
|
|
1170
|
+
if ARGV[4] == '1' then
|
|
1171
|
+
local head, missing = stored_head(boundary, types)
|
|
1172
|
+
if head == nil then return { 'index', missing, tostring(boundary) } end
|
|
1173
|
+
state.p, state.h = boundary, head
|
|
1174
|
+
else
|
|
1175
|
+
-- Nothing it takes is stored after its start: index its types from there.
|
|
1176
|
+
for name in pairs(types or {}) do
|
|
1177
|
+
if typed[name] == nil then register(name, total) end
|
|
1178
|
+
end
|
|
1179
|
+
end
|
|
1180
|
+
state.dirty = true
|
|
1181
|
+
elseif not state.h then
|
|
1182
|
+
-- A deployment that doesn't serve this consumer may have appended events
|
|
1183
|
+
-- it takes since it went idle.
|
|
1184
|
+
local head, missing = stored_head(state.p, types)
|
|
1185
|
+
if head == nil then return { 'index', missing, tostring(state.p) } end
|
|
1186
|
+
if head then state.h, state.dirty = head, true end
|
|
1187
|
+
end
|
|
1188
|
+
if state.ty ~= ty then state.ty, state.dirty = ty, true end
|
|
1189
|
+
local columns = {}
|
|
1190
|
+
if had_head then
|
|
1191
|
+
columns = read_rows({ { state.h, row_of(consumer, 'G' .. lane) } }, ${HEAD_COLUMNS})[1]
|
|
1192
|
+
elseif state.h then
|
|
1193
|
+
ready_set(cid, state.h)
|
|
1194
|
+
end
|
|
1195
|
+
flush_states()
|
|
1196
|
+
if typed_dirty then redis.call('SET', TYPED, cjson.encode(typed)) end
|
|
1197
|
+
flush()
|
|
1198
|
+
flush_ready()
|
|
1199
|
+
return { 'ok', tostring(state.p), state.h and tostring(state.h) or false, columns, tostring(total) }
|
|
1200
|
+
`
|
|
1201
|
+
|
|
1202
|
+
/**
|
|
1203
|
+
* KEYS: counter. Returns the dead-lettered heads above the clear
|
|
1204
|
+
* marker as [consumer, tag, index, columns, entry] tuples.
|
|
1205
|
+
*/
|
|
1206
|
+
const DEAD_LETTERS_LUA = `${BASE_LUA}
|
|
1207
|
+
local dead = redis.call('ZRANGEBYSCORE', READY, '-inf', '(0', 'WITHSCORES')
|
|
1208
|
+
if #dead == 0 then return {} end
|
|
1209
|
+
read_bounds()
|
|
1210
|
+
local refs, items = {}, {}
|
|
1211
|
+
for i = 1, #dead, 2 do
|
|
1212
|
+
local consumer, tag = parse_cid(dead[i])
|
|
1213
|
+
local idx = -tonumber(dead[i + 1])
|
|
1214
|
+
if idx > boundary then
|
|
1215
|
+
refs[#refs + 1] = { idx, row_of(consumer, tag) }
|
|
1216
|
+
items[#items + 1] = { consumer, tag, idx }
|
|
1217
|
+
end
|
|
849
1218
|
end
|
|
850
|
-
local rows =
|
|
1219
|
+
local rows = read_rows(refs, ${HEAD_COLUMNS})
|
|
851
1220
|
local result = {}
|
|
852
|
-
for
|
|
853
|
-
local entries = redis.call('XRANGE',
|
|
1221
|
+
for k, item in ipairs(items) do
|
|
1222
|
+
local entries = redis.call('XRANGE', LOG, tostring(item[3]) .. '-0', tostring(item[3]) .. '-0')
|
|
854
1223
|
if #entries > 0 then
|
|
855
|
-
result[#result + 1] = entries[1]
|
|
856
|
-
|
|
1224
|
+
result[#result + 1] = { item[1], item[2], tostring(item[3]), rows[k], entries[1] }
|
|
1225
|
+
end
|
|
1226
|
+
end
|
|
1227
|
+
return result
|
|
1228
|
+
`
|
|
1229
|
+
|
|
1230
|
+
/**
|
|
1231
|
+
* KEYS: counter. Returns every listed cursor as [consumer, tag,
|
|
1232
|
+
* position, head or nil, columns]: lane and log cursors from the cursor
|
|
1233
|
+
* hash, read against the clear marker, then the event cursors with a head.
|
|
1234
|
+
*/
|
|
1235
|
+
const CURSORS_LUA = `${BASE_LUA}
|
|
1236
|
+
read_bounds()
|
|
1237
|
+
local listed = {}
|
|
1238
|
+
local fields = redis.call('HGETALL', CURSORS)
|
|
1239
|
+
local records, order = {}, {}
|
|
1240
|
+
for i = 1, #fields, 2 do
|
|
1241
|
+
local at = string.find(fields[i], ':[^:]*$')
|
|
1242
|
+
local cid, suffix = string.sub(fields[i], 1, at - 1), string.sub(fields[i], at + 1)
|
|
1243
|
+
if not records[cid] then
|
|
1244
|
+
records[cid] = {}
|
|
1245
|
+
order[#order + 1] = cid
|
|
1246
|
+
end
|
|
1247
|
+
records[cid][suffix] = fields[i + 1]
|
|
1248
|
+
end
|
|
1249
|
+
for _, cid in ipairs(order) do
|
|
1250
|
+
local record = records[cid]
|
|
1251
|
+
local consumer, tag = parse_cid(cid)
|
|
1252
|
+
local head = (record.h and record.h ~= '') and tonumber(record.h) or false
|
|
1253
|
+
local position = tonumber(record.p or '0')
|
|
1254
|
+
if head and head <= boundary then head = false end
|
|
1255
|
+
local kind = string.sub(tag, 1, 1)
|
|
1256
|
+
if kind == 'G' and position < boundary then position = boundary end
|
|
1257
|
+
if kind == 'G' or head or position > boundary then
|
|
1258
|
+
listed[#listed + 1] = { consumer, tag, position, head }
|
|
857
1259
|
end
|
|
858
1260
|
end
|
|
1261
|
+
local ready = redis.call('ZRANGE', READY, 0, -1, 'WITHSCORES')
|
|
1262
|
+
for i = 1, #ready, 2 do
|
|
1263
|
+
local consumer, tag = parse_cid(ready[i])
|
|
1264
|
+
if string.sub(tag, 1, 1) == '#' then
|
|
1265
|
+
local head = math.abs(tonumber(ready[i + 1]))
|
|
1266
|
+
listed[#listed + 1] = { consumer, '#', head - 1, head }
|
|
1267
|
+
end
|
|
1268
|
+
end
|
|
1269
|
+
local refs, positions = {}, {}
|
|
1270
|
+
for position, cursor in ipairs(listed) do
|
|
1271
|
+
if cursor[4] then
|
|
1272
|
+
refs[#refs + 1] = { cursor[4], row_of(cursor[1], cursor[2]) }
|
|
1273
|
+
positions[#refs] = position
|
|
1274
|
+
end
|
|
1275
|
+
end
|
|
1276
|
+
local rows = read_rows(refs, ${HEAD_COLUMNS})
|
|
1277
|
+
local columns = {}
|
|
1278
|
+
for k, position in ipairs(positions) do columns[position] = rows[k] end
|
|
1279
|
+
local result = {}
|
|
1280
|
+
for position, cursor in ipairs(listed) do
|
|
1281
|
+
result[#result + 1] = {
|
|
1282
|
+
cursor[1], cursor[2], tostring(cursor[3]),
|
|
1283
|
+
cursor[4] and tostring(cursor[4]) or false, columns[position] or {},
|
|
1284
|
+
}
|
|
1285
|
+
end
|
|
859
1286
|
return result
|
|
860
1287
|
`
|
|
861
1288
|
|
|
862
1289
|
/**
|
|
863
|
-
* KEYS:
|
|
864
|
-
* ARGV:
|
|
865
|
-
*
|
|
866
|
-
*
|
|
867
|
-
*
|
|
1290
|
+
* KEYS: counter, head, history.
|
|
1291
|
+
* ARGV: write count, then per write: index, snapshot JSON, pin count, then
|
|
1292
|
+
* per pin: consumer, tag, index; finally the reducer name. Writes from
|
|
1293
|
+
* before the latest clear are dropped, and a head from before it is
|
|
1294
|
+
* replaced as if absent. A pin holds only on an unfinished, live head: a
|
|
1295
|
+
* claimed row says so itself, an unclaimed head by its cursor's ready score.
|
|
868
1296
|
*/
|
|
869
|
-
const PUT_SNAPSHOTS_LUA =
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
local
|
|
873
|
-
local
|
|
874
|
-
|
|
875
|
-
local
|
|
876
|
-
local offset = 4
|
|
877
|
-
for i = 1, tonumber(ARGV[3]) do
|
|
1297
|
+
const PUT_SNAPSHOTS_LUA = `${BASE_LUA}
|
|
1298
|
+
read_bounds()
|
|
1299
|
+
local HEAD, HISTORY = KEYS[2], KEYS[3]
|
|
1300
|
+
local entries, refs, pins = {}, {}, {}
|
|
1301
|
+
local offset = 2
|
|
1302
|
+
for w = 1, tonumber(ARGV[1]) do
|
|
1303
|
+
local index = tonumber(ARGV[offset])
|
|
878
1304
|
local pin_count = tonumber(ARGV[offset + 2])
|
|
1305
|
+
local entry = { index = index, snapshot = ARGV[offset + 1], pins = {} }
|
|
879
1306
|
for p = 1, pin_count do
|
|
880
|
-
local
|
|
881
|
-
|
|
882
|
-
|
|
1307
|
+
local o = offset + 2 + (p - 1) * 3
|
|
1308
|
+
local pin = { consumer = ARGV[o + 1], tag = ARGV[o + 2], idx = tonumber(ARGV[o + 3]) }
|
|
1309
|
+
if index >= boundary and pin.idx > boundary and pin.idx <= total then
|
|
1310
|
+
pin.row = row_of(pin.consumer, pin.tag)
|
|
1311
|
+
refs[#refs + 1] = { pin.idx, pin.row }
|
|
1312
|
+
pins[#pins + 1] = pin
|
|
1313
|
+
entry.pins[#entry.pins + 1] = pin
|
|
1314
|
+
end
|
|
1315
|
+
end
|
|
1316
|
+
entries[w] = entry
|
|
1317
|
+
offset = offset + 3 + pin_count * 3
|
|
1318
|
+
end
|
|
1319
|
+
local function live(consumer, tag, idx)
|
|
1320
|
+
return tonumber(redis.call('ZSCORE', READY, cid_of(consumer, tag)) or '0') == idx
|
|
1321
|
+
end
|
|
1322
|
+
local rows = read_rows(refs, { 'pins', 'p', 'f', 'l' })
|
|
1323
|
+
local held, held_order = {}, {}
|
|
1324
|
+
for k, pin in ipairs(pins) do
|
|
1325
|
+
local row = rows[k]
|
|
1326
|
+
local valid
|
|
1327
|
+
if row[2] or row[3] then
|
|
1328
|
+
valid = false
|
|
1329
|
+
elseif row[4] then
|
|
1330
|
+
valid = owns(row[4], pin.tag)
|
|
1331
|
+
elseif pin.tag == '#' then
|
|
1332
|
+
valid = live(pin.consumer, '#' .. tostring(pin.idx), pin.idx)
|
|
1333
|
+
else
|
|
1334
|
+
valid = live(pin.consumer, pin.tag, pin.idx) or
|
|
1335
|
+
live(pin.consumer, 'G' .. string.sub(pin.tag, 2), pin.idx)
|
|
1336
|
+
end
|
|
1337
|
+
if valid then
|
|
1338
|
+
local key = tostring(pin.idx) .. ':' .. pin.row
|
|
1339
|
+
if not held[key] then
|
|
1340
|
+
local list = (row[1] and row[1] ~= '') and cjson.decode(row[1]) or {}
|
|
1341
|
+
local set = {}
|
|
1342
|
+
for _, ref in ipairs(list) do set[ref] = true end
|
|
1343
|
+
held[key] = { list = list, set = set, idx = pin.idx, row = pin.row }
|
|
1344
|
+
held_order[#held_order + 1] = key
|
|
883
1345
|
end
|
|
1346
|
+
pin.held = held[key]
|
|
884
1347
|
end
|
|
885
|
-
offset = offset + 3 + pin_count
|
|
886
1348
|
end
|
|
887
|
-
local status = read_fields(pinned, { 'p', 'f' })
|
|
888
1349
|
local snapshots = {}
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
local row = status[tostring(event_index)]
|
|
898
|
-
if index >= boundary and event_index > boundary and event_index <= total and
|
|
899
|
-
not row[1] and not row[2] then
|
|
900
|
-
local pin_key = ARGV[2] .. tostring(event_index)
|
|
901
|
-
if redis.call('SADD', pin_key, ref) == 1 then
|
|
902
|
-
redis.call('HINCRBY', KEYS[5], ref, 1)
|
|
903
|
-
end
|
|
1350
|
+
for _, entry in ipairs(entries) do
|
|
1351
|
+
local ref = cjson.encode({ HISTORY, tostring(entry.index) })
|
|
1352
|
+
for _, pin in ipairs(entry.pins) do
|
|
1353
|
+
if pin.held and not pin.held.set[ref] then
|
|
1354
|
+
pin.held.set[ref] = true
|
|
1355
|
+
pin.held.list[#pin.held.list + 1] = ref
|
|
1356
|
+
pin.held.dirty = true
|
|
1357
|
+
redis.call('HINCRBY', PIN_COUNTS, ref, 1)
|
|
904
1358
|
end
|
|
905
1359
|
end
|
|
906
|
-
if index >= boundary then
|
|
907
|
-
snapshots[#snapshots + 1] = { index = index, snapshot = snapshot, ref = ref }
|
|
1360
|
+
if entry.index >= boundary then
|
|
1361
|
+
snapshots[#snapshots + 1] = { index = entry.index, snapshot = entry.snapshot, ref = ref }
|
|
908
1362
|
end
|
|
909
|
-
offset = offset + 3 + pin_count
|
|
910
1363
|
end
|
|
1364
|
+
for _, key in ipairs(held_order) do
|
|
1365
|
+
local row = held[key]
|
|
1366
|
+
if row.dirty then put_row(row.idx, row.row, 'pins', cjson.encode(row.list)) end
|
|
1367
|
+
end
|
|
1368
|
+
flush()
|
|
911
1369
|
if #snapshots == 0 then return 1 end
|
|
912
1370
|
|
|
913
|
-
local head = redis.call('GET',
|
|
1371
|
+
local head = redis.call('GET', HEAD)
|
|
914
1372
|
local head_index = nil
|
|
915
1373
|
local registered = false
|
|
916
1374
|
if head then
|
|
@@ -920,109 +1378,112 @@ if head then
|
|
|
920
1378
|
end
|
|
921
1379
|
if head_index and head_index < boundary then
|
|
922
1380
|
head, head_index, registered = nil, nil, false
|
|
923
|
-
redis.call('ZREMRANGEBYSCORE',
|
|
1381
|
+
redis.call('ZREMRANGEBYSCORE', HISTORY, '-inf', '(' .. boundary)
|
|
924
1382
|
end
|
|
925
1383
|
-- Deletion finds each reducer's keys through this set instead of a scan.
|
|
926
1384
|
-- Heads written since carry the mark, so a reducer registers once.
|
|
927
1385
|
if not registered then
|
|
928
|
-
redis.call('SADD',
|
|
1386
|
+
redis.call('SADD', SESSION .. 'snap-reducers', ARGV[#ARGV])
|
|
929
1387
|
end
|
|
930
1388
|
local latest = snapshots[#snapshots]
|
|
931
1389
|
local writes_head = not head_index or latest.index >= head_index
|
|
932
1390
|
local advances = not head_index or latest.index > head_index
|
|
933
1391
|
local final_head_index = advances and latest.index or head_index
|
|
934
1392
|
|
|
935
|
-
if advances and head and tonumber(redis.call('HGET',
|
|
936
|
-
redis.call('ZREMRANGEBYSCORE',
|
|
937
|
-
redis.call('ZADD',
|
|
1393
|
+
if advances and head and tonumber(redis.call('HGET', PIN_COUNTS, cjson.encode({ HISTORY, tostring(head_index) })) or '0') > 0 then
|
|
1394
|
+
redis.call('ZREMRANGEBYSCORE', HISTORY, head_index, head_index)
|
|
1395
|
+
redis.call('ZADD', HISTORY, head_index, head)
|
|
938
1396
|
end
|
|
939
1397
|
|
|
940
1398
|
for _, write in ipairs(snapshots) do
|
|
941
1399
|
if write.index < final_head_index then
|
|
942
|
-
redis.call('ZREMRANGEBYSCORE',
|
|
943
|
-
if tonumber(redis.call('HGET',
|
|
944
|
-
redis.call('ZADD',
|
|
1400
|
+
redis.call('ZREMRANGEBYSCORE', HISTORY, write.index, write.index)
|
|
1401
|
+
if tonumber(redis.call('HGET', PIN_COUNTS, write.ref) or '0') > 0 then
|
|
1402
|
+
redis.call('ZADD', HISTORY, write.index, write.snapshot)
|
|
945
1403
|
end
|
|
946
1404
|
end
|
|
947
1405
|
end
|
|
948
1406
|
|
|
949
1407
|
if writes_head then
|
|
950
|
-
redis.call('SET',
|
|
951
|
-
redis.call('ZREMRANGEBYSCORE',
|
|
1408
|
+
redis.call('SET', HEAD, latest.snapshot)
|
|
1409
|
+
redis.call('ZREMRANGEBYSCORE', HISTORY, latest.index, latest.index)
|
|
952
1410
|
end
|
|
953
1411
|
return 1
|
|
954
1412
|
`
|
|
955
1413
|
|
|
956
1414
|
/**
|
|
957
|
-
*
|
|
958
|
-
*
|
|
959
|
-
*
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
1415
|
+
* Presence entries are `{"a":at,"e":expiresAt,"s":seen,"v":value}`, stamps
|
|
1416
|
+
* first and both integers, so scripts read them with a pattern instead of
|
|
1417
|
+
* decoding the value.
|
|
1418
|
+
*/
|
|
1419
|
+
const PRESENCE_STAMPS_LUA = `
|
|
1420
|
+
local function stamps(entry)
|
|
1421
|
+
local a, e = string.match(entry, '^{"a":(%-?%d+),"e":(%-?%d+),')
|
|
1422
|
+
return tonumber(a), tonumber(e)
|
|
1423
|
+
end
|
|
1424
|
+
`
|
|
1425
|
+
|
|
1426
|
+
/**
|
|
1427
|
+
* KEYS: presence hash. ARGV: atMs, ttlMs, nowMs, n, then per entry:
|
|
1428
|
+
* hashField, entryJson (a null value is a tombstone). Field-wise LWW by the
|
|
1429
|
+
* sender's `at`: an unexpired stored entry with a strictly newer stamp
|
|
1430
|
+
* survives; ties go to the incoming write (memory-tier parity). Returns the applied 1-based entry
|
|
1431
|
+
* positions. The whole-key PEXPIRE is the coarse bound, sound because core
|
|
1432
|
+
* sends one uniform ttlMs; the per-entry `e` stamp is the fine bound (the
|
|
1433
|
+
* injected storage clock at the applied set plus ttlMs — never the sender
|
|
1434
|
+
* stamp), filtered on read against the same clock.
|
|
968
1435
|
*/
|
|
969
|
-
const PRESENCE_SET_LUA =
|
|
970
|
-
local at = tonumber(ARGV[1])
|
|
971
|
-
local
|
|
972
|
-
|
|
973
|
-
local
|
|
1436
|
+
const PRESENCE_SET_LUA = `${SLICED_LUA}${PRESENCE_STAMPS_LUA}
|
|
1437
|
+
local at, now, n = tonumber(ARGV[1]), tonumber(ARGV[3]), tonumber(ARGV[4])
|
|
1438
|
+
local fields = {}
|
|
1439
|
+
for i = 1, n do fields[i] = ARGV[4 + (i - 1) * 2 + 1] end
|
|
1440
|
+
local stored = sliced('HMGET', KEYS[1], fields)
|
|
1441
|
+
local applied, sets = {}, {}
|
|
974
1442
|
for i = 1, n do
|
|
975
|
-
local
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
redis.call('HDEL', KEYS[1], field)
|
|
981
|
-
else
|
|
982
|
-
redis.call('HSET', KEYS[1], field, value)
|
|
983
|
-
end
|
|
1443
|
+
local a, e = false, false
|
|
1444
|
+
if stored[i] then a, e = stamps(stored[i]) end
|
|
1445
|
+
if not stored[i] or a <= at or e <= now then
|
|
1446
|
+
sets[#sets + 1] = fields[i]
|
|
1447
|
+
sets[#sets + 1] = ARGV[4 + (i - 1) * 2 + 2]
|
|
984
1448
|
applied[#applied + 1] = tostring(i)
|
|
985
|
-
published[#published + 1] = ARGV[3 + (i - 1) * 3 + 3]
|
|
986
1449
|
end
|
|
987
1450
|
end
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
redis.pcall('PUBLISH', ARGV[4 + n * 3], ARGV[5 + n * 3] .. table.concat(published, ',') .. '}}')
|
|
991
|
-
end
|
|
1451
|
+
sliced('HSET', KEYS[1], sets)
|
|
1452
|
+
if #applied > 0 then redis.call('PEXPIRE', KEYS[1], ARGV[2]) end
|
|
992
1453
|
return applied
|
|
993
1454
|
`
|
|
994
1455
|
|
|
995
1456
|
/**
|
|
996
|
-
* KEYS: presence hash. ARGV: nowMs
|
|
997
|
-
*
|
|
998
|
-
*
|
|
999
|
-
* expiry are HDELed in the same atomic step, so a stale reader can
|
|
1000
|
-
* never sweep a concurrently refreshed field.
|
|
1457
|
+
* KEYS: presence hash. ARGV: nowMs, then the fields a read saw expired.
|
|
1458
|
+
* Deletes those still at or past their expiry, re-checked in this atomic
|
|
1459
|
+
* step so a stale reader never sweeps a concurrently refreshed field.
|
|
1001
1460
|
*/
|
|
1002
|
-
const
|
|
1461
|
+
const PRESENCE_PRUNE_LUA = `${SLICED_LUA}${PRESENCE_STAMPS_LUA}
|
|
1003
1462
|
local now = tonumber(ARGV[1])
|
|
1004
|
-
local
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1463
|
+
local fields = {}
|
|
1464
|
+
for i = 2, #ARGV do fields[#fields + 1] = ARGV[i] end
|
|
1465
|
+
local stored = sliced('HMGET', KEYS[1], fields)
|
|
1466
|
+
local stale = {}
|
|
1467
|
+
for i = 1, #fields do
|
|
1468
|
+
if stored[i] then
|
|
1469
|
+
local _, e = stamps(stored[i])
|
|
1470
|
+
if e <= now then stale[#stale + 1] = fields[i] end
|
|
1012
1471
|
end
|
|
1013
1472
|
end
|
|
1014
|
-
|
|
1473
|
+
sliced('HDEL', KEYS[1], stale)
|
|
1474
|
+
return 1
|
|
1015
1475
|
`
|
|
1016
1476
|
|
|
1017
1477
|
/**
|
|
1018
|
-
* KEYS: head, history
|
|
1478
|
+
* KEYS: counter, head, history. ARGV: snapshot frontier or ''.
|
|
1019
1479
|
* Returns one atomic multiversion-cache-plus-log-frontier read, with the
|
|
1020
1480
|
* clear boundary; checkpoints from before it are ignored.
|
|
1021
1481
|
*/
|
|
1022
1482
|
const READ_STATE_LUA = `
|
|
1023
|
-
local
|
|
1483
|
+
local session = string.sub(KEYS[1], 1, -6)
|
|
1484
|
+
local counts = redis.call('MGET', KEYS[1], session .. 'clear')
|
|
1024
1485
|
local boundary = tonumber(counts[2] or '0')
|
|
1025
|
-
local head = redis.call('GET', KEYS[
|
|
1486
|
+
local head = redis.call('GET', KEYS[2])
|
|
1026
1487
|
local head_index = head and cjson.decode(head)['index'] or nil
|
|
1027
1488
|
if head and (type(head_index) ~= 'number' or head_index < 0 or head_index > 9007199254740991 or head_index ~= math.floor(head_index)) then
|
|
1028
1489
|
return redis.error_reply('invalid snapshot head index')
|
|
@@ -1037,63 +1498,138 @@ if head_index and (cap == '' or head_index <= tonumber(cap)) then
|
|
|
1037
1498
|
snapshot = head
|
|
1038
1499
|
else
|
|
1039
1500
|
local upper = cap == '' and '+inf' or cap
|
|
1040
|
-
local historical = redis.call('ZREVRANGEBYSCORE', KEYS[
|
|
1501
|
+
local historical = redis.call('ZREVRANGEBYSCORE', KEYS[3], upper, boundary, 'LIMIT', 0, 1)
|
|
1041
1502
|
if #historical > 0 then snapshot = historical[1] end
|
|
1042
1503
|
end
|
|
1043
|
-
return { head_index or '', snapshot or '', counts[1] or '0', tostring(boundary) }
|
|
1504
|
+
return { head_index or '', snapshot or '', counts[1] or counts[2] or '0', tostring(boundary) }
|
|
1044
1505
|
`
|
|
1045
1506
|
|
|
1046
1507
|
/**
|
|
1047
|
-
*
|
|
1048
|
-
*
|
|
1049
|
-
*
|
|
1508
|
+
* What reclaim removes once no entry is left below the latest clear: every
|
|
1509
|
+
* registered reducer's checkpoints below the latest clear and the pin counts
|
|
1510
|
+
* naming them (snapshot writes register each reducer in
|
|
1511
|
+
* {prefix}:{session}:snap-reducers, so no key scan is needed), every head
|
|
1512
|
+
* row below it (whole chunks, then the boundary chunk's older rows), the
|
|
1513
|
+
* type indexes' older entries, and the lane cursors it left idle.
|
|
1514
|
+
*
|
|
1515
|
+
* A lifetime that ended empty then collapses to the clear marker, the log
|
|
1516
|
+
* holding it (readers resuming before the marker still read it), and the log
|
|
1517
|
+
* cursors the sweep reset to the marker. The counter reads from the marker
|
|
1518
|
+
* when absent, type indexes rebuild lazily, and the marker's own id key goes
|
|
1519
|
+
* too.
|
|
1050
1520
|
*/
|
|
1051
|
-
const
|
|
1052
|
-
local function
|
|
1053
|
-
for _, reducer in ipairs(redis.call('SMEMBERS',
|
|
1054
|
-
local head_key =
|
|
1521
|
+
const SWEEP_LUA = `
|
|
1522
|
+
local function sweep(boundary)
|
|
1523
|
+
for _, reducer in ipairs(redis.call('SMEMBERS', SESSION .. 'snap-reducers')) do
|
|
1524
|
+
local head_key = SESSION .. 'snap:' .. reducer
|
|
1055
1525
|
local head = redis.call('GET', head_key)
|
|
1056
1526
|
if head and tonumber(cjson.decode(head)['index']) < boundary then
|
|
1057
1527
|
redis.call('DEL', head_key)
|
|
1058
1528
|
end
|
|
1059
|
-
redis.call('ZREMRANGEBYSCORE',
|
|
1529
|
+
redis.call('ZREMRANGEBYSCORE', SESSION .. 'snap-history:' .. reducer, '-inf', '(' .. boundary)
|
|
1060
1530
|
end
|
|
1061
|
-
local
|
|
1062
|
-
local counts = redis.call('HGETALL', counts_key)
|
|
1531
|
+
local counts = redis.call('HGETALL', PIN_COUNTS)
|
|
1063
1532
|
local stale = {}
|
|
1064
1533
|
for i = 1, #counts, 2 do
|
|
1065
1534
|
if tonumber(cjson.decode(counts[i])[2]) < boundary then stale[#stale + 1] = counts[i] end
|
|
1066
1535
|
end
|
|
1067
|
-
|
|
1536
|
+
sliced('HDEL', PIN_COUNTS, stale)
|
|
1537
|
+
local raw = redis.call('GET', TYPED)
|
|
1538
|
+
if raw then
|
|
1539
|
+
for name in pairs(cjson.decode(raw)) do
|
|
1540
|
+
redis.call('ZREMRANGEBYSCORE', type_key(name), '-inf', '(' .. boundary)
|
|
1541
|
+
end
|
|
1542
|
+
end
|
|
1543
|
+
local last = math.floor((boundary - 1) / ${META_CHUNK_SIZE})
|
|
1544
|
+
local chunks = {}
|
|
1545
|
+
for chunk = 0, last - 1 do chunks[#chunks + 1] = META .. ':' .. tostring(chunk) end
|
|
1546
|
+
sliced('UNLINK', nil, chunks)
|
|
1547
|
+
local older = {}
|
|
1548
|
+
for _, field in ipairs(redis.call('HKEYS', META .. ':' .. tostring(last))) do
|
|
1549
|
+
local idx = tonumber(string.match(field, '^(%d+):'))
|
|
1550
|
+
if idx and idx < boundary then older[#older + 1] = field end
|
|
1551
|
+
end
|
|
1552
|
+
sliced('HDEL', META .. ':' .. tostring(last), older)
|
|
1553
|
+
local fields = redis.call('HGETALL', CURSORS)
|
|
1554
|
+
local records, order = {}, {}
|
|
1555
|
+
for i = 1, #fields, 2 do
|
|
1556
|
+
local at = string.find(fields[i], ':[^:]*$')
|
|
1557
|
+
local cid, suffix = string.sub(fields[i], 1, at - 1), string.sub(fields[i], at + 1)
|
|
1558
|
+
if not records[cid] then
|
|
1559
|
+
records[cid] = {}
|
|
1560
|
+
order[#order + 1] = cid
|
|
1561
|
+
end
|
|
1562
|
+
records[cid][suffix] = fields[i + 1]
|
|
1563
|
+
end
|
|
1564
|
+
local idle, moved = {}, {}
|
|
1565
|
+
for _, cid in ipairs(order) do
|
|
1566
|
+
local record = records[cid]
|
|
1567
|
+
local _, tag = parse_cid(cid)
|
|
1568
|
+
local head = (record.h and record.h ~= '') and tonumber(record.h) or false
|
|
1569
|
+
local position = tonumber(record.p or '0')
|
|
1570
|
+
if head and head <= boundary then head = false end
|
|
1571
|
+
if string.sub(tag, 1, 1) == 'G' then
|
|
1572
|
+
if position < boundary then position = boundary end
|
|
1573
|
+
moved[#moved + 1] = cid .. ':h'
|
|
1574
|
+
moved[#moved + 1] = head and tostring(head) or ''
|
|
1575
|
+
moved[#moved + 1] = cid .. ':p'
|
|
1576
|
+
moved[#moved + 1] = tostring(position)
|
|
1577
|
+
elseif not head and position <= boundary then
|
|
1578
|
+
for suffix in pairs(record) do idle[#idle + 1] = cid .. ':' .. suffix end
|
|
1579
|
+
elseif not head and record.h ~= '' then
|
|
1580
|
+
moved[#moved + 1] = cid .. ':h'
|
|
1581
|
+
moved[#moved + 1] = ''
|
|
1582
|
+
end
|
|
1583
|
+
end
|
|
1584
|
+
sliced('HDEL', CURSORS, idle)
|
|
1585
|
+
sliced('HSET', CURSORS, moved)
|
|
1586
|
+
local count = redis.call('GET', COUNTER)
|
|
1587
|
+
if not count or tonumber(count) ~= boundary or redis.call('ZCARD', READY) > 0 then
|
|
1588
|
+
return
|
|
1589
|
+
end
|
|
1590
|
+
local stale = { COUNTER, TYPED, PIN_COUNTS, chunk_key(boundary) }
|
|
1591
|
+
local reducers = SESSION .. 'snap-reducers'
|
|
1592
|
+
for _, reducer in ipairs(redis.call('SMEMBERS', reducers)) do
|
|
1593
|
+
stale[#stale + 1] = SESSION .. 'snap:' .. reducer
|
|
1594
|
+
stale[#stale + 1] = SESSION .. 'snap-history:' .. reducer
|
|
1595
|
+
end
|
|
1596
|
+
stale[#stale + 1] = reducers
|
|
1597
|
+
local types = redis.call('GET', TYPED)
|
|
1598
|
+
if types then
|
|
1599
|
+
for name in pairs(cjson.decode(types)) do stale[#stale + 1] = type_key(name) end
|
|
1600
|
+
end
|
|
1601
|
+
local marker = redis.call('XRANGE', LOG, boundary .. '-0', boundary .. '-0')[1]
|
|
1602
|
+
if marker then
|
|
1603
|
+
for i = 1, #marker[2], 2 do
|
|
1604
|
+
if marker[2][i] == 'id' then stale[#stale + 1] = id_key(marker[2][i + 1]) end
|
|
1605
|
+
end
|
|
1606
|
+
end
|
|
1607
|
+
sliced('UNLINK', nil, stale)
|
|
1068
1608
|
end
|
|
1069
1609
|
`
|
|
1070
1610
|
|
|
1071
1611
|
/**
|
|
1072
|
-
* KEYS:
|
|
1073
|
-
*
|
|
1074
|
-
*
|
|
1075
|
-
*
|
|
1076
|
-
*
|
|
1077
|
-
*
|
|
1078
|
-
*
|
|
1079
|
-
*
|
|
1080
|
-
*
|
|
1612
|
+
* KEYS: counter.
|
|
1613
|
+
* ARGV: sessionId, nowMs, ifLastIndex or '', cause consumer or '', cause tag,
|
|
1614
|
+
* cause index, cause attempt, marker id.
|
|
1615
|
+
* Returns ['ok', cleared, head, boundary, appended?] | ['superseded'] |
|
|
1616
|
+
* ['foreign']; `appended` is '1' when this call appended the marker.
|
|
1617
|
+
* The marker joins the same log: the old lifetime's entries stay below it
|
|
1618
|
+
* until reclaim deletes them, and every read of the log starts at the
|
|
1619
|
+
* marker. (A stream can't be renamed aside: Upstash refuses RENAME on one.)
|
|
1620
|
+
* The ready set is unlinked, so claims and pending checks see only the new
|
|
1621
|
+
* lifetime. Type indexes keep their older entries until reclaim too:
|
|
1622
|
+
* lookups start above the marker.
|
|
1081
1623
|
*/
|
|
1082
|
-
const CLEAR_LUA =
|
|
1083
|
-
local
|
|
1084
|
-
|
|
1085
|
-
${WAKE_LUA}
|
|
1086
|
-
local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR, KEYS[5])
|
|
1087
|
-
local head = tonumber(counts[1] or '0')
|
|
1088
|
-
use_split(counts[2], counts[1], counts[3])
|
|
1624
|
+
const CLEAR_LUA = `${BASE_LUA}
|
|
1625
|
+
local marker_owner = owners_of(read_bounds({ ARGV[8] }))[1]
|
|
1626
|
+
local head = total
|
|
1089
1627
|
if ARGV[4] ~= '' then
|
|
1090
|
-
local
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
if
|
|
1094
|
-
|
|
1095
|
-
if tonumber(row[1] or '0') ~= tonumber(ARGV[5]) or row[2] then
|
|
1096
|
-
return { 'superseded', field }
|
|
1628
|
+
local idx = tonumber(ARGV[6])
|
|
1629
|
+
if idx <= boundary then return { 'superseded' } end
|
|
1630
|
+
local row = read_rows({ { idx, row_of(ARGV[4], ARGV[5]) } }, { 'd', 'f', 'p', 'l' })[1]
|
|
1631
|
+
if tonumber(row[1] or '0') ~= tonumber(ARGV[7]) or row[2] or row[3] or not owns(row[4], ARGV[5]) then
|
|
1632
|
+
return { 'superseded' }
|
|
1097
1633
|
end
|
|
1098
1634
|
end
|
|
1099
1635
|
local expected = ARGV[3]
|
|
@@ -1106,151 +1642,118 @@ end
|
|
|
1106
1642
|
if head == 0 or boundary == head then
|
|
1107
1643
|
return { 'ok', '1', tostring(head), tostring(boundary) }
|
|
1108
1644
|
end
|
|
1109
|
-
if
|
|
1645
|
+
if marker_owner then return { 'foreign' } end
|
|
1110
1646
|
local field = tostring(head + 1)
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
redis.call('
|
|
1116
|
-
|
|
1117
|
-
'id', ARGV[6], 'type', '${CLEAR_EVENT_TYPE}', 'payload', 'null',
|
|
1118
|
-
'created_at', ARGV[2], 'cause', '', 'lane', '')
|
|
1119
|
-
local writes = { KEYS[5], cjson.encode({ ARGV[1], head + 1 }), COUNTER, field, CLEAR, field }
|
|
1120
|
-
split_write(writes)
|
|
1121
|
-
redis.call('MSET', unpack(writes))
|
|
1122
|
-
put(mkey(field), field .. ':p', ARGV[2])
|
|
1123
|
-
flush()
|
|
1124
|
-
wake_add(field, ARGV[6], '${CLEAR_EVENT_TYPE}', 'null')
|
|
1125
|
-
wake(ARGV[7], field, ARGV[2])
|
|
1126
|
-
return { 'ok', '1', field, field }
|
|
1647
|
+
redis.call('UNLINK', READY)
|
|
1648
|
+
redis.call('XADD', LOG, field .. '-0',
|
|
1649
|
+
'id', ARGV[8], 'type', CLEAR_TYPE, 'payload', 'null',
|
|
1650
|
+
'created_at', ARGV[2], 'cause', '')
|
|
1651
|
+
redis.call('MSET', id_key(ARGV[8]), field, COUNTER, field, CLEAR, field)
|
|
1652
|
+
return { 'ok', '1', field, field, '1' }
|
|
1127
1653
|
`
|
|
1128
1654
|
|
|
1129
1655
|
/**
|
|
1130
1656
|
* KEYS: counter. ARGV: page size.
|
|
1131
|
-
* The next deletion batch:
|
|
1132
|
-
*
|
|
1133
|
-
*
|
|
1134
|
-
*
|
|
1657
|
+
* The next deletion batch: up to a page of [index, id] pairs of the log's
|
|
1658
|
+
* entries below the latest clear. With none left it sweeps what the old
|
|
1659
|
+
* lifetimes left and returns nothing, so a retry finishes an interrupted
|
|
1660
|
+
* deletion.
|
|
1135
1661
|
*/
|
|
1136
|
-
const RECLAIM_BATCH_LUA =
|
|
1137
|
-
|
|
1138
|
-
local session = string.sub(KEYS[1], 1, -6)
|
|
1139
|
-
local boundary = tonumber(redis.call('GET', session .. 'clear') or '0')
|
|
1662
|
+
const RECLAIM_BATCH_LUA = `${BASE_LUA}${SWEEP_LUA}
|
|
1663
|
+
local boundary = tonumber(redis.call('GET', CLEAR) or '0')
|
|
1140
1664
|
if boundary == 0 then return {} end
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1665
|
+
local entries = redis.call('XRANGE', LOG, '-', tostring(boundary - 1) .. '-0', 'COUNT', ARGV[1])
|
|
1666
|
+
if #entries == 0 then
|
|
1667
|
+
sweep(boundary)
|
|
1668
|
+
return {}
|
|
1669
|
+
end
|
|
1670
|
+
local batch = {}
|
|
1671
|
+
for _, entry in ipairs(entries) do
|
|
1672
|
+
local fields = entry[2]
|
|
1673
|
+
for i = 1, #fields, 2 do
|
|
1674
|
+
if fields[i] == 'id' then
|
|
1675
|
+
batch[#batch + 1] = string.match(entry[1], '^(%d+)')
|
|
1676
|
+
batch[#batch + 1] = fields[i + 1]
|
|
1677
|
+
break
|
|
1154
1678
|
end
|
|
1155
|
-
return batch
|
|
1156
1679
|
end
|
|
1157
|
-
redis.call('DEL', log)
|
|
1158
|
-
redis.call('SREM', session .. 'cleared', log)
|
|
1159
1680
|
end
|
|
1160
|
-
|
|
1161
|
-
return {}
|
|
1681
|
+
return batch
|
|
1162
1682
|
`
|
|
1163
1683
|
|
|
1164
1684
|
/**
|
|
1165
|
-
* KEYS:
|
|
1166
|
-
*
|
|
1167
|
-
*
|
|
1168
|
-
*
|
|
1169
|
-
*
|
|
1170
|
-
* reused in the new lifetime keeps its owner. Returns [deleted, done].
|
|
1685
|
+
* KEYS: counter.
|
|
1686
|
+
* ARGV: sessionId, n, then per entry: index, id.
|
|
1687
|
+
* Deletes the listed entries below the latest clear and their id records.
|
|
1688
|
+
* An id record survives when it names another event, so an id reused in the
|
|
1689
|
+
* new lifetime keeps its owner. Returns [deleted, done].
|
|
1171
1690
|
*/
|
|
1172
|
-
const RECLAIM_LUA =
|
|
1173
|
-
local
|
|
1174
|
-
|
|
1175
|
-
${SWEEP_CHECKPOINTS_LUA}
|
|
1176
|
-
local function lane_field(lane, suffix)
|
|
1177
|
-
return '__lane:' .. tostring(string.len(lane)) .. ':' .. lane .. ':' .. suffix
|
|
1178
|
-
end
|
|
1179
|
-
local session = string.sub(COUNTER, 1, -6)
|
|
1180
|
-
local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR)
|
|
1181
|
-
use_split(counts[2], counts[1], counts[3])
|
|
1182
|
-
local fields, id_keys, indexes, stream_ids, pin_keys, deleted = {}, {}, {}, {}, {}, {}
|
|
1691
|
+
const RECLAIM_LUA = `${BASE_LUA}${SWEEP_LUA}
|
|
1692
|
+
local boundary = tonumber(redis.call('GET', CLEAR) or '0')
|
|
1693
|
+
local ids, indexes, stream_ids = {}, {}, {}
|
|
1183
1694
|
-- A concurrent reclaim may list the same batch; only XDEL says who removed it.
|
|
1184
1695
|
local removed = 0
|
|
1185
|
-
for i = 1, tonumber(ARGV[
|
|
1186
|
-
local field = ARGV[
|
|
1696
|
+
for i = 1, tonumber(ARGV[2]) do
|
|
1697
|
+
local field = ARGV[2 + (i - 1) * 2 + 1]
|
|
1187
1698
|
if tonumber(field) < boundary then
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
stream_ids[#fields] = field .. '-0'
|
|
1192
|
-
pin_keys[#fields] = ARGV[2] .. field
|
|
1193
|
-
deleted[field] = true
|
|
1194
|
-
end
|
|
1195
|
-
end
|
|
1196
|
-
if #fields > 0 then
|
|
1197
|
-
local lanes, seen, pointers = read_fields(fields, { 'l' }), {}, {}
|
|
1198
|
-
for _, field in ipairs(fields) do
|
|
1199
|
-
local lane = lanes[field][1]
|
|
1200
|
-
if lane and lane ~= '' and not seen[lane] then
|
|
1201
|
-
seen[lane] = true
|
|
1202
|
-
pointers[#pointers + 1] = lane_field(lane, 'head')
|
|
1203
|
-
pointers[#pointers + 1] = lane_field(lane, 'tail')
|
|
1204
|
-
end
|
|
1205
|
-
end
|
|
1206
|
-
if #pointers > 0 then
|
|
1207
|
-
local values = redis.call('HMGET', META, unpack(pointers))
|
|
1208
|
-
local stale = {}
|
|
1209
|
-
for i = 1, #pointers do
|
|
1210
|
-
if values[i] and deleted[values[i]] then stale[#stale + 1] = pointers[i] end
|
|
1211
|
-
end
|
|
1212
|
-
if #stale > 0 then redis.call('HDEL', META, unpack(stale)) end
|
|
1213
|
-
end
|
|
1214
|
-
local suffixes = { 'l', 'next', 'p', 'pa', 'ri', 'ch', 'ce', 'f', 'fc', 'lc', 'lf', 'lfa', 'd', 'a', 'e' }
|
|
1215
|
-
local columns, order = {}, {}
|
|
1216
|
-
for _, field in ipairs(fields) do
|
|
1217
|
-
local key = mkey(field)
|
|
1218
|
-
if not columns[key] then
|
|
1219
|
-
columns[key] = {}
|
|
1220
|
-
order[#order + 1] = key
|
|
1221
|
-
end
|
|
1222
|
-
for _, suffix in ipairs(suffixes) do
|
|
1223
|
-
columns[key][#columns[key] + 1] = field .. ':' .. suffix
|
|
1224
|
-
end
|
|
1699
|
+
ids[#ids + 1] = ARGV[2 + (i - 1) * 2 + 2]
|
|
1700
|
+
indexes[#ids] = tonumber(field)
|
|
1701
|
+
stream_ids[#ids] = field .. '-0'
|
|
1225
1702
|
end
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
local
|
|
1229
|
-
for i = 1, #
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
end
|
|
1703
|
+
end
|
|
1704
|
+
if #ids > 0 then
|
|
1705
|
+
local keys = {}
|
|
1706
|
+
for i = 1, #ids do keys[i] = id_key(ids[i]) end
|
|
1707
|
+
local owners, stale = sliced('MGET', nil, keys), {}
|
|
1708
|
+
for i = 1, #ids do
|
|
1709
|
+
if owners[i] and tonumber(owners[i]) == indexes[i] then
|
|
1710
|
+
stale[#stale + 1] = keys[i]
|
|
1235
1711
|
end
|
|
1236
1712
|
end
|
|
1237
|
-
|
|
1238
|
-
removed = redis.call('XDEL',
|
|
1713
|
+
sliced('DEL', nil, stale)
|
|
1714
|
+
removed = redis.call('XDEL', LOG, unpack(stream_ids))
|
|
1239
1715
|
end
|
|
1240
1716
|
local done = 0
|
|
1241
|
-
if redis.call('
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
if redis.call('SCARD', session .. 'cleared') == 0 then
|
|
1245
|
-
sweep_checkpoints(session, boundary)
|
|
1246
|
-
done = 1
|
|
1247
|
-
end
|
|
1717
|
+
if #redis.call('XRANGE', LOG, '-', tostring(boundary - 1) .. '-0', 'COUNT', 1) == 0 then
|
|
1718
|
+
sweep(boundary)
|
|
1719
|
+
done = 1
|
|
1248
1720
|
end
|
|
1249
1721
|
return { removed, done }
|
|
1250
1722
|
`
|
|
1251
1723
|
|
|
1252
1724
|
// ── wire helpers ─────────────────────────────────────────────────────
|
|
1253
1725
|
|
|
1726
|
+
const WAKE_PAYLOAD_LIMIT = 32_768
|
|
1727
|
+
const encoder = new TextEncoder()
|
|
1728
|
+
|
|
1729
|
+
/**
|
|
1730
|
+
* A wake-up announcing events through `frontier`. It carries them, so a
|
|
1731
|
+
* caught-up reader applies them without reading the stream:
|
|
1732
|
+
* `{"f":frontier,"t":createdAt,"e":[[index,id,type,payload],...]}`, with
|
|
1733
|
+
* payloads inlined as their stored JSON text. One over WAKE_PAYLOAD_LIMIT
|
|
1734
|
+
* bytes, or one without entries, announces only `{"f":frontier}`, and
|
|
1735
|
+
* readers catch up with XRANGE.
|
|
1736
|
+
*/
|
|
1737
|
+
const wakeMessage = (
|
|
1738
|
+
frontier: number,
|
|
1739
|
+
createdAt: number,
|
|
1740
|
+
entries: ReadonlyArray<
|
|
1741
|
+
readonly [index: number, id: string, type: string, payloadJson: string]
|
|
1742
|
+
>,
|
|
1743
|
+
): string => {
|
|
1744
|
+
const bare = `{"f":${frontier}}`
|
|
1745
|
+
if (entries.length === 0) return bare
|
|
1746
|
+
const carried = `{"f":${frontier},"t":${createdAt},"e":[${entries
|
|
1747
|
+
.map(
|
|
1748
|
+
([index, id, type, payload]) =>
|
|
1749
|
+
`[${index},${JSON.stringify(id)},${JSON.stringify(type)},${payload}]`,
|
|
1750
|
+
)
|
|
1751
|
+
.join(',')}]}`
|
|
1752
|
+
return encoder.encode(carried).byteLength > WAKE_PAYLOAD_LIMIT
|
|
1753
|
+
? bare
|
|
1754
|
+
: carried
|
|
1755
|
+
}
|
|
1756
|
+
|
|
1254
1757
|
export type StreamEntry = [id: string, fields: string[]]
|
|
1255
1758
|
|
|
1256
1759
|
const fieldMap = (fields: string[]): Record<string, string> => {
|
|
@@ -1276,6 +1779,30 @@ const toEvent = (sessionId: string, entry: StreamEntry): Event => {
|
|
|
1276
1779
|
}
|
|
1277
1780
|
}
|
|
1278
1781
|
|
|
1782
|
+
/**
|
|
1783
|
+
* A stream entry's causal edge. Releases before consumers (0.17 to 0.20)
|
|
1784
|
+
* recorded only the parent's index and attempt: their edge reads as the
|
|
1785
|
+
* handlers table's, `a2.handlers`, as Postgres's upgrade names it, with no
|
|
1786
|
+
* lane, since nothing reads an earlier release's lane.
|
|
1787
|
+
*/
|
|
1788
|
+
const decodeStreamCause = (raw: string): EventCause | null => {
|
|
1789
|
+
const parsed = JSON.parse(raw) as { consumer?: unknown }
|
|
1790
|
+
return decodeCause(
|
|
1791
|
+
parsed.consumer === undefined
|
|
1792
|
+
? { consumer: 'a2.handlers', lane: null, ...parsed }
|
|
1793
|
+
: parsed,
|
|
1794
|
+
)
|
|
1795
|
+
}
|
|
1796
|
+
|
|
1797
|
+
/** A stream entry with its causal edge. */
|
|
1798
|
+
const toStoredEvent = (sessionId: string, entry: StreamEntry): StoredEvent => {
|
|
1799
|
+
const cause = fieldMap(entry[1])['cause']
|
|
1800
|
+
return Object.assign(toEvent(sessionId, entry), {
|
|
1801
|
+
cause:
|
|
1802
|
+
cause === undefined || cause === '' ? null : decodeStreamCause(cause),
|
|
1803
|
+
})
|
|
1804
|
+
}
|
|
1805
|
+
|
|
1279
1806
|
type ByteStreamEntry = [Uint8Array, Uint8Array[]]
|
|
1280
1807
|
|
|
1281
1808
|
const fieldBytes = (name: string): Uint8Array => new TextEncoder().encode(name)
|
|
@@ -1338,75 +1865,107 @@ const toSerializedEvent = (
|
|
|
1338
1865
|
|
|
1339
1866
|
const json = (value: unknown): string => JSON.stringify(value) ?? 'null'
|
|
1340
1867
|
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1868
|
+
/** How scripts name a cursor within its consumer: `#`, or `L` and its lane. */
|
|
1869
|
+
const tagOf = (lane: string | null): string =>
|
|
1870
|
+
lane === null ? '#' : `L${lane}`
|
|
1871
|
+
|
|
1872
|
+
/** The lane a cursor tag names: null for an event cursor. */
|
|
1873
|
+
const laneOf = (tag: string): string | null =>
|
|
1874
|
+
tag.startsWith('#') ? null : tag.slice(1)
|
|
1875
|
+
|
|
1876
|
+
const targetsArg = (targets: readonly EventTarget[] | undefined): string =>
|
|
1877
|
+
targets === undefined || targets.length === 0
|
|
1878
|
+
? ''
|
|
1879
|
+
: JSON.stringify(
|
|
1880
|
+
targets.map((target) => [target.consumer, tagOf(target.lane)]),
|
|
1881
|
+
)
|
|
1882
|
+
|
|
1883
|
+
/** A log consumer's types, as scripts read them; '*' takes every type. */
|
|
1884
|
+
const logTypesArg = (types: readonly string[] | null): string =>
|
|
1885
|
+
types === null ? '*' : JSON.stringify(types)
|
|
1886
|
+
|
|
1887
|
+
/** The served consumers, as scripts read them; '' serves every consumer. */
|
|
1888
|
+
const servedArg = (consumers: readonly StoreConsumer[] | undefined): string =>
|
|
1889
|
+
consumers === undefined
|
|
1890
|
+
? ''
|
|
1891
|
+
: JSON.stringify(
|
|
1892
|
+
consumers.map((consumer) =>
|
|
1893
|
+
consumer.log
|
|
1894
|
+
? [
|
|
1895
|
+
consumer.name,
|
|
1896
|
+
consumer.log.lane,
|
|
1897
|
+
logTypesArg(consumer.log.types),
|
|
1898
|
+
consumer.log.backfill ? 1 : 0,
|
|
1899
|
+
]
|
|
1900
|
+
: [consumer.name],
|
|
1901
|
+
),
|
|
1902
|
+
)
|
|
1903
|
+
|
|
1904
|
+
type Column = string | number | null | undefined
|
|
1905
|
+
|
|
1906
|
+
/** A head's dispatch state from a row's HEAD_COLUMNS. */
|
|
1907
|
+
const dispatchOf = (
|
|
1908
|
+
consumer: string,
|
|
1909
|
+
lane: string | null,
|
|
1910
|
+
columns: readonly Column[] | null | undefined,
|
|
1911
|
+
): HeadDispatch => {
|
|
1912
|
+
const column = (at: number): string | null => {
|
|
1913
|
+
const value = columns?.[at]
|
|
1914
|
+
return value === null || value === undefined ? null : String(value)
|
|
1347
1915
|
}
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
Number(value
|
|
1351
|
-
!Number.isInteger(value.attempt) ||
|
|
1352
|
-
Number(value.attempt) < 1
|
|
1353
|
-
) {
|
|
1354
|
-
throw new TypeError('stored event has an invalid cause')
|
|
1916
|
+
const date = (at: number): Date | null => {
|
|
1917
|
+
const value = column(at)
|
|
1918
|
+
return value === null ? null : new Date(Number(value))
|
|
1355
1919
|
}
|
|
1356
|
-
|
|
1357
|
-
value
|
|
1358
|
-
|
|
1359
|
-
) {
|
|
1360
|
-
throw new TypeError('stored event has an invalid cause')
|
|
1920
|
+
const count = (at: number): number | null => {
|
|
1921
|
+
const value = column(at)
|
|
1922
|
+
return value === null ? null : Number(value)
|
|
1361
1923
|
}
|
|
1362
1924
|
return {
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1925
|
+
consumer,
|
|
1926
|
+
lane,
|
|
1927
|
+
firstClaimedAt: date(1),
|
|
1928
|
+
lastClaimedAt: date(2),
|
|
1929
|
+
attemptCount: count(0) ?? 0,
|
|
1930
|
+
claimHolder: column(3),
|
|
1931
|
+
claimExpiresAt: date(4),
|
|
1932
|
+
failureCount: count(5) ?? 0,
|
|
1933
|
+
lastFailedAt: date(6),
|
|
1934
|
+
lastFailedAttempt: count(7),
|
|
1935
|
+
lastError: column(8),
|
|
1936
|
+
failedAt: date(9),
|
|
1368
1937
|
}
|
|
1369
1938
|
}
|
|
1370
1939
|
|
|
1371
|
-
type
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1940
|
+
type HeadTuple = [
|
|
1941
|
+
consumer: string,
|
|
1942
|
+
tag: string,
|
|
1943
|
+
index: string,
|
|
1944
|
+
columns: Column[],
|
|
1376
1945
|
entry: StreamEntry,
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
attemptCount: count('d') ?? 0,
|
|
1401
|
-
claimHolder: column('ch') ?? null,
|
|
1402
|
-
claimExpiresAt: date('ce'),
|
|
1403
|
-
failureCount: count('a') ?? 0,
|
|
1404
|
-
lastFailedAt: date('lf'),
|
|
1405
|
-
lastFailedAttempt: count('lfa'),
|
|
1406
|
-
lastError: column('e') ?? null,
|
|
1407
|
-
failedAt: date('f'),
|
|
1408
|
-
})
|
|
1409
|
-
}
|
|
1946
|
+
]
|
|
1947
|
+
|
|
1948
|
+
const toHeadEvent = (sessionId: string, head: HeadTuple): HeadEvent => ({
|
|
1949
|
+
...toStoredEvent(sessionId, head[4]),
|
|
1950
|
+
...dispatchOf(head[0], laneOf(head[1]), head[3]),
|
|
1951
|
+
})
|
|
1952
|
+
|
|
1953
|
+
const compareText = (a: string, b: string): number =>
|
|
1954
|
+
a < b ? -1 : a > b ? 1 : 0
|
|
1955
|
+
|
|
1956
|
+
const byHead = (a: HeadRef, b: HeadRef): number =>
|
|
1957
|
+
a.index - b.index || compareText(a.consumer, b.consumer)
|
|
1958
|
+
|
|
1959
|
+
/** Consumer, then lane, with concurrent cursors last by head. */
|
|
1960
|
+
const byCursor = (a: StoredCursor, b: StoredCursor): number =>
|
|
1961
|
+
compareText(a.consumer, b.consumer) ||
|
|
1962
|
+
(a.lane === null
|
|
1963
|
+
? b.lane === null
|
|
1964
|
+
? (a.head ?? 0) - (b.head ?? 0)
|
|
1965
|
+
: 1
|
|
1966
|
+
: b.lane === null
|
|
1967
|
+
? -1
|
|
1968
|
+
: compareText(a.lane, b.lane))
|
|
1410
1969
|
|
|
1411
1970
|
const toClaimAvailableResult = (
|
|
1412
1971
|
sessionId: string,
|
|
@@ -1415,57 +1974,13 @@ const toClaimAvailableResult = (
|
|
|
1415
1974
|
const outcome = String(reply[0])
|
|
1416
1975
|
if (outcome === 'claimed') {
|
|
1417
1976
|
const count = Number(reply[1])
|
|
1418
|
-
const events
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
const event = toEvent(sessionId, entry)
|
|
1423
|
-
const fields = fieldMap(entry[1])
|
|
1424
|
-
events.push(
|
|
1425
|
-
Object.assign(event, {
|
|
1426
|
-
cause: toCause(fields['cause']),
|
|
1427
|
-
lane: fields['lane'] === '' ? null : fields['lane']!,
|
|
1428
|
-
processedAt: null,
|
|
1429
|
-
processedByAttempt: null,
|
|
1430
|
-
returnedEventIds: null,
|
|
1431
|
-
firstClaimedAt:
|
|
1432
|
-
reply[offset + 3] === null
|
|
1433
|
-
? null
|
|
1434
|
-
: new Date(Number(reply[offset + 3])),
|
|
1435
|
-
lastClaimedAt:
|
|
1436
|
-
reply[offset + 4] === null
|
|
1437
|
-
? null
|
|
1438
|
-
: new Date(Number(reply[offset + 4])),
|
|
1439
|
-
attemptCount: Number(reply[offset]),
|
|
1440
|
-
claimHolder:
|
|
1441
|
-
reply[offset + 7] === null ? null : String(reply[offset + 7]),
|
|
1442
|
-
claimExpiresAt:
|
|
1443
|
-
reply[offset + 8] === null
|
|
1444
|
-
? null
|
|
1445
|
-
: new Date(Number(reply[offset + 8])),
|
|
1446
|
-
failureCount: Number(reply[offset + 1]),
|
|
1447
|
-
lastFailedAt:
|
|
1448
|
-
reply[offset + 5] === null
|
|
1449
|
-
? null
|
|
1450
|
-
: new Date(Number(reply[offset + 5])),
|
|
1451
|
-
lastFailedAttempt:
|
|
1452
|
-
reply[offset + 6] === null ? null : Number(reply[offset + 6]),
|
|
1453
|
-
lastError:
|
|
1454
|
-
reply[offset + 2] === null ? null : String(reply[offset + 2]),
|
|
1455
|
-
failedAt:
|
|
1456
|
-
reply[offset + 10] === null
|
|
1457
|
-
? null
|
|
1458
|
-
: new Date(Number(reply[offset + 10])),
|
|
1459
|
-
}),
|
|
1460
|
-
)
|
|
1461
|
-
}
|
|
1462
|
-
const clearIndex = Number(reply[2 + count * 11] ?? 0)
|
|
1977
|
+
const events = (reply.slice(2, 2 + count) as HeadTuple[])
|
|
1978
|
+
.map((head) => toHeadEvent(sessionId, head))
|
|
1979
|
+
.toSorted(byHead)
|
|
1980
|
+
const clearIndex = Number(reply[2 + count] ?? 0)
|
|
1463
1981
|
if (clearIndex > 0)
|
|
1464
1982
|
for (const event of events) event.clearIndex = clearIndex
|
|
1465
|
-
return {
|
|
1466
|
-
outcome: 'claimed',
|
|
1467
|
-
events,
|
|
1468
|
-
}
|
|
1983
|
+
return { outcome: 'claimed', events, head: Number(reply[3 + count]) }
|
|
1469
1984
|
}
|
|
1470
1985
|
if (outcome === 'busy') {
|
|
1471
1986
|
return { outcome, retryAt: new Date(Number(reply[1])) }
|
|
@@ -1496,9 +2011,9 @@ const toIndex = ({
|
|
|
1496
2011
|
}
|
|
1497
2012
|
|
|
1498
2013
|
/**
|
|
1499
|
-
* A range read that met a gap.
|
|
1500
|
-
* a read whose bounds predate
|
|
1501
|
-
* anywhere else a gap is corruption.
|
|
2014
|
+
* A range read that met a gap. Reclaim deletes the entries below a clear, so
|
|
2015
|
+
* a read whose bounds predate the clear finds nothing where it expected
|
|
2016
|
+
* events; anywhere else a gap is corruption.
|
|
1502
2017
|
*/
|
|
1503
2018
|
const rangeGap = (sessionId: string, message: string): A2Error =>
|
|
1504
2019
|
Object.assign(
|
|
@@ -1538,123 +2053,185 @@ const wrap = async <T>(fn: () => Promise<T>): Promise<T> => {
|
|
|
1538
2053
|
}
|
|
1539
2054
|
}
|
|
1540
2055
|
|
|
2056
|
+
/**
|
|
2057
|
+
* A session's key. Every key of a session shares one hash tag, so a script
|
|
2058
|
+
* locks only its session on Upstash and stays in one slot on a cluster. The
|
|
2059
|
+
* tag is the session id as JSON with its braces escaped: it can't end early,
|
|
2060
|
+
* and no session's tag spells another's.
|
|
2061
|
+
*/
|
|
2062
|
+
export const redisSessionKey = ({
|
|
2063
|
+
prefix,
|
|
2064
|
+
sessionId,
|
|
2065
|
+
suffix,
|
|
2066
|
+
}: {
|
|
2067
|
+
prefix: string
|
|
2068
|
+
sessionId: string
|
|
2069
|
+
suffix: string
|
|
2070
|
+
}): string =>
|
|
2071
|
+
`${prefix}:{${JSON.stringify(sessionId).replaceAll('{', '\\u007b').replaceAll('}', '\\u007d')}}:${suffix}`
|
|
2072
|
+
|
|
1541
2073
|
export function createRedisStoreCore(
|
|
1542
2074
|
options: RedisStoreCoreOptions,
|
|
1543
2075
|
): RedisStoreCore {
|
|
1544
2076
|
const { call, callBytes, clock } = options
|
|
1545
2077
|
const generateId = options.ids
|
|
1546
2078
|
const prefix = options.keyPrefix
|
|
2079
|
+
if (prefix.includes('{') || prefix.includes('}')) {
|
|
2080
|
+
throw new TypeError(
|
|
2081
|
+
'a redis keyPrefix cannot contain braces: they would set the hash tag of every key',
|
|
2082
|
+
)
|
|
2083
|
+
}
|
|
1547
2084
|
|
|
1548
|
-
const
|
|
1549
|
-
|
|
1550
|
-
const
|
|
1551
|
-
const
|
|
1552
|
-
|
|
1553
|
-
const
|
|
1554
|
-
const
|
|
1555
|
-
const
|
|
1556
|
-
const snapKey = (s: string, r: string): string => `${prefix}:${s}:snap:${r}`
|
|
2085
|
+
const sessionKey = (sessionId: string, suffix: string): string =>
|
|
2086
|
+
redisSessionKey({ prefix, sessionId, suffix })
|
|
2087
|
+
const storeKey = (s: string): string => sessionKey(s, 'log')
|
|
2088
|
+
const countKey = (s: string): string => sessionKey(s, 'count')
|
|
2089
|
+
// Scripts derive the session's other keys from the counter; keep them in step.
|
|
2090
|
+
const clearKey = (s: string): string => sessionKey(s, 'clear')
|
|
2091
|
+
const schemaKey = `${prefix}:schema`
|
|
2092
|
+
const snapKey = (s: string, r: string): string => sessionKey(s, `snap:${r}`)
|
|
1557
2093
|
const snapHistoryKey = (s: string, r: string): string =>
|
|
1558
|
-
|
|
1559
|
-
const
|
|
1560
|
-
|
|
1561
|
-
const
|
|
1562
|
-
|
|
1563
|
-
|
|
1564
|
-
|
|
1565
|
-
|
|
1566
|
-
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
2094
|
+
sessionKey(s, `snap-history:${r}`)
|
|
2095
|
+
const presenceKey = (s: string): string => sessionKey(s, 'presence')
|
|
2096
|
+
// Channels aren't keys; the notify store subscribes by these names.
|
|
2097
|
+
const notifyChannel = (s: string): string => `${prefix}:${s}:notify`
|
|
2098
|
+
const presenceChannel = (s: string): string => `${prefix}:${s}:presence`
|
|
2099
|
+
|
|
2100
|
+
const layoutError = (): A2Error =>
|
|
2101
|
+
new A2Error(
|
|
2102
|
+
'STORE_NOT_CONFIGURED',
|
|
2103
|
+
`the redis keys under prefix '${prefix}' hold an A2 layout other than '${REDIS_LAYOUT}', from another release; point the store at a fresh keyPrefix, or give the contract a new name so it starts new sessions`,
|
|
2104
|
+
)
|
|
2105
|
+
|
|
2106
|
+
/**
|
|
2107
|
+
* Once per store, before its first script: claim or check the prefix's
|
|
2108
|
+
* layout, and ask the server whether scripts may lock only their keys.
|
|
2109
|
+
* Only the server's own rejection of the flag means no; a transport error
|
|
2110
|
+
* fails this attempt, and the next operation asks again.
|
|
2111
|
+
*/
|
|
2112
|
+
const prepared = retryableLazy(async () => {
|
|
2113
|
+
const [, keyLocking] = await Promise.all([
|
|
2114
|
+
(async () => {
|
|
2115
|
+
await call('SET', schemaKey, REDIS_LAYOUT, 'NX')
|
|
2116
|
+
if ((await call('GET', schemaKey)) !== REDIS_LAYOUT) throw layoutError()
|
|
2117
|
+
})(),
|
|
2118
|
+
call('EVAL', `${KEY_LOCKING}\nreturn 1`, 1, schemaKey).then(
|
|
2119
|
+
() => true,
|
|
2120
|
+
(error: unknown) => {
|
|
2121
|
+
if (SHEBANG_REJECTED.test(String((error as Error)?.message)))
|
|
2122
|
+
return false
|
|
2123
|
+
throw error
|
|
2124
|
+
},
|
|
2125
|
+
),
|
|
2126
|
+
])
|
|
2127
|
+
return { keyLocking }
|
|
2128
|
+
})
|
|
2129
|
+
|
|
2130
|
+
/** Runs a script; a `readOnly` one takes Upstash's shared lock. */
|
|
2131
|
+
const runScript = async ({
|
|
2132
|
+
script,
|
|
2133
|
+
keys,
|
|
2134
|
+
args,
|
|
2135
|
+
readOnly,
|
|
2136
|
+
}: {
|
|
2137
|
+
script: string
|
|
2138
|
+
keys: string[]
|
|
2139
|
+
args: Array<string | number>
|
|
2140
|
+
readOnly: boolean
|
|
2141
|
+
}): Promise<unknown> => {
|
|
2142
|
+
const { keyLocking } = await prepared.get()
|
|
2143
|
+
let source = script
|
|
2144
|
+
if (keyLocking)
|
|
2145
|
+
source = `${readOnly ? READ_ONLY_KEY_LOCKING : KEY_LOCKING}\n${script}`
|
|
2146
|
+
return call('EVAL', source, keys.length, ...keys, ...args)
|
|
2147
|
+
}
|
|
1574
2148
|
|
|
2149
|
+
/** Runs a writing script over a session: KEYS start with its counter. */
|
|
1575
2150
|
const evalScript = (
|
|
1576
2151
|
script: string,
|
|
2152
|
+
sessionId: string,
|
|
1577
2153
|
keys: string[],
|
|
1578
2154
|
args: Array<string | number>,
|
|
1579
|
-
): Promise<unknown> =>
|
|
2155
|
+
): Promise<unknown> =>
|
|
2156
|
+
runScript({
|
|
2157
|
+
script,
|
|
2158
|
+
keys: [countKey(sessionId), ...keys],
|
|
2159
|
+
args,
|
|
2160
|
+
readOnly: false,
|
|
2161
|
+
})
|
|
2162
|
+
|
|
2163
|
+
/**
|
|
2164
|
+
* Publishes a write's wake-up after its script, never inside it: a script
|
|
2165
|
+
* holds its lock while PUBLISH waits on subscribers. Wake-ups are hints (a
|
|
2166
|
+
* lost one costs subscribers a catch-up read, never an event), so a failed
|
|
2167
|
+
* publish is dropped and the write waits for it at most
|
|
2168
|
+
* NOTIFY_TIMINGS.publishMs.
|
|
2169
|
+
*/
|
|
2170
|
+
const publish = async (channel: string, message: string): Promise<void> => {
|
|
2171
|
+
let timer: ReturnType<typeof setTimeout> | undefined
|
|
2172
|
+
await Promise.race([
|
|
2173
|
+
call('PUBLISH', channel, message).catch(() => undefined),
|
|
2174
|
+
new Promise<void>((resolve) => {
|
|
2175
|
+
timer = setTimeout(resolve, NOTIFY_TIMINGS.publishMs)
|
|
2176
|
+
;(timer as { unref?: () => void }).unref?.()
|
|
2177
|
+
}),
|
|
2178
|
+
])
|
|
2179
|
+
clearTimeout(timer)
|
|
2180
|
+
}
|
|
1580
2181
|
|
|
1581
|
-
|
|
2182
|
+
/**
|
|
2183
|
+
* Runs a write that may need a type's index to reach further back first.
|
|
2184
|
+
* Such a write returns ['index', type, after] having written nothing, and
|
|
2185
|
+
* runs again once INDEX_TYPE_LUA has indexed that history, a page per call.
|
|
2186
|
+
*/
|
|
2187
|
+
const indexing = async (
|
|
2188
|
+
sessionId: string,
|
|
2189
|
+
run: () => Promise<unknown>,
|
|
2190
|
+
): Promise<unknown[]> => {
|
|
2191
|
+
for (;;) {
|
|
2192
|
+
// oxlint-disable-next-line no-await-in-loop -- each run follows the indexing it asked for
|
|
2193
|
+
const reply = (await run()) as unknown[]
|
|
2194
|
+
if (reply[0] !== 'index') return reply
|
|
2195
|
+
for (;;) {
|
|
2196
|
+
// oxlint-disable-next-line no-await-in-loop -- each page lowers the index's floor
|
|
2197
|
+
const [done] = (await evalScript(
|
|
2198
|
+
INDEX_TYPE_LUA,
|
|
2199
|
+
sessionId,
|
|
2200
|
+
[],
|
|
2201
|
+
[String(reply[1]), Number(reply[2])],
|
|
2202
|
+
)) as [string]
|
|
2203
|
+
if (done === '1') break
|
|
2204
|
+
}
|
|
2205
|
+
}
|
|
2206
|
+
}
|
|
2207
|
+
|
|
2208
|
+
/** The frontier and clear marker, in one MGET. */
|
|
2209
|
+
const readBounds = async (
|
|
1582
2210
|
sessionId: string,
|
|
1583
|
-
): Promise<{ frontier: number;
|
|
1584
|
-
|
|
2211
|
+
): Promise<{ frontier: number; clearIndex: number }> => {
|
|
2212
|
+
await prepared.get()
|
|
2213
|
+
const [count, cleared] = (await call(
|
|
1585
2214
|
'MGET',
|
|
1586
2215
|
countKey(sessionId),
|
|
1587
|
-
metaSplitKey(sessionId),
|
|
1588
2216
|
clearKey(sessionId),
|
|
1589
|
-
)) as [unknown, unknown
|
|
1590
|
-
const frontier = toIndex({ raw: count, sessionId, name: 'event count' })
|
|
2217
|
+
)) as [unknown, unknown]
|
|
1591
2218
|
return {
|
|
1592
|
-
|
|
1593
|
-
|
|
1594
|
-
|
|
1595
|
-
|
|
1596
|
-
|
|
2219
|
+
// A collapsed session keeps only its clear marker.
|
|
2220
|
+
frontier: toIndex({
|
|
2221
|
+
raw: count ?? cleared,
|
|
2222
|
+
sessionId,
|
|
2223
|
+
name: 'event count',
|
|
2224
|
+
}),
|
|
1597
2225
|
clearIndex: toIndex({ raw: cleared, sessionId, name: 'clear boundary' }),
|
|
1598
2226
|
}
|
|
1599
2227
|
}
|
|
1600
2228
|
|
|
1601
|
-
const
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
sessionId: string
|
|
1608
|
-
afterIndex: number
|
|
1609
|
-
throughIndex: number
|
|
1610
|
-
metaSplit: number
|
|
1611
|
-
}): Promise<StoredEvent[]> => {
|
|
1612
|
-
if (throughIndex <= afterIndex) return []
|
|
1613
|
-
const entries = (await call(
|
|
1614
|
-
'XRANGE',
|
|
1615
|
-
storeKey(sessionId),
|
|
1616
|
-
`${afterIndex + 1}-0`,
|
|
1617
|
-
`${throughIndex}-0`,
|
|
1618
|
-
'COUNT',
|
|
1619
|
-
READ_PAGE_SIZE,
|
|
1620
|
-
)) as StreamEntry[]
|
|
1621
|
-
if (entries.length === 0) return []
|
|
1622
|
-
|
|
1623
|
-
const fieldsByKey = new Map<string, string[]>()
|
|
1624
|
-
for (const entry of entries) {
|
|
1625
|
-
const index = entryIndex(entry[0])
|
|
1626
|
-
const key =
|
|
1627
|
-
index <= metaSplit ? metaKey(sessionId) : metaChunkKey(sessionId, index)
|
|
1628
|
-
const fields = fieldsByKey.get(key) ?? []
|
|
1629
|
-
fields.push(
|
|
1630
|
-
...STORED_EVENT_META_SUFFIXES.map((suffix) => `${index}:${suffix}`),
|
|
1631
|
-
)
|
|
1632
|
-
fieldsByKey.set(key, fields)
|
|
1633
|
-
}
|
|
1634
|
-
const lookups = [...fieldsByKey]
|
|
1635
|
-
const results = await Promise.allSettled(
|
|
1636
|
-
lookups.map(([key, fields]) => call('HMGET', key, ...fields)),
|
|
1637
|
-
)
|
|
1638
|
-
const meta: Record<string, string> = {}
|
|
1639
|
-
for (let position = 0; position < lookups.length; position += 1) {
|
|
1640
|
-
const result = results[position]!
|
|
1641
|
-
if (result.status === 'rejected') throw result.reason
|
|
1642
|
-
const fields = lookups[position]![1]
|
|
1643
|
-
const values = (result.value as Array<string | null> | null) ?? []
|
|
1644
|
-
for (let i = 0; i < fields.length; i += 1) {
|
|
1645
|
-
const value = values[i]
|
|
1646
|
-
if (value !== null && value !== undefined) {
|
|
1647
|
-
meta[fields[i]!] = value
|
|
1648
|
-
}
|
|
1649
|
-
}
|
|
1650
|
-
}
|
|
1651
|
-
return entries.map((entry) => {
|
|
1652
|
-
const idx = entryIndex(entry[0])
|
|
1653
|
-
return toStoredEvent(
|
|
1654
|
-
sessionId,
|
|
1655
|
-
entry,
|
|
1656
|
-
(suffix) => meta[`${idx}:${suffix}`],
|
|
1657
|
-
)
|
|
2229
|
+
const readClearIndex = async (sessionId: string): Promise<number> => {
|
|
2230
|
+
await prepared.get()
|
|
2231
|
+
return toIndex({
|
|
2232
|
+
raw: await call('GET', clearKey(sessionId)),
|
|
2233
|
+
sessionId,
|
|
2234
|
+
name: 'clear boundary',
|
|
1658
2235
|
})
|
|
1659
2236
|
}
|
|
1660
2237
|
|
|
@@ -1744,11 +2321,8 @@ export function createRedisStoreCore(
|
|
|
1744
2321
|
): Promise<{ rows: StoredEvent[]; clearIndex: number }> => {
|
|
1745
2322
|
if (throughIndex !== undefined && throughIndex <= afterIndex)
|
|
1746
2323
|
return { rows: [], clearIndex: 0 }
|
|
1747
|
-
const {
|
|
1748
|
-
|
|
1749
|
-
metaSplit,
|
|
1750
|
-
clearIndex,
|
|
1751
|
-
} = await readRangeBounds(sessionId)
|
|
2324
|
+
const { frontier: currentFrontier, clearIndex } =
|
|
2325
|
+
await readBounds(sessionId)
|
|
1752
2326
|
const frontier = Math.min(throughIndex ?? currentFrontier, currentFrontier)
|
|
1753
2327
|
// Reads start at the latest clear marker.
|
|
1754
2328
|
const after = Math.max(afterIndex, clearIndex - 1)
|
|
@@ -1758,15 +2332,18 @@ export function createRedisStoreCore(
|
|
|
1758
2332
|
let nextIndex = after + 1
|
|
1759
2333
|
while (nextIndex <= frontier) {
|
|
1760
2334
|
// oxlint-disable-next-line no-await-in-loop -- each page advances the dense frozen range
|
|
1761
|
-
const
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
2335
|
+
const entries = (await call(
|
|
2336
|
+
'XRANGE',
|
|
2337
|
+
storeKey(sessionId),
|
|
2338
|
+
`${nextIndex}-0`,
|
|
2339
|
+
`${frontier}-0`,
|
|
2340
|
+
'COUNT',
|
|
2341
|
+
READ_PAGE_SIZE,
|
|
2342
|
+
)) as StreamEntry[]
|
|
2343
|
+
if (entries.length === 0) {
|
|
1768
2344
|
throw rangeGap(sessionId, `ended before index ${frontier}`)
|
|
1769
2345
|
}
|
|
2346
|
+
const page = entries.map((entry) => toStoredEvent(sessionId, entry))
|
|
1770
2347
|
for (let position = 0; position < page.length; position += 1) {
|
|
1771
2348
|
if (page[position]!.index !== nextIndex + position) {
|
|
1772
2349
|
throw rangeGap(sessionId, 'is not contiguous')
|
|
@@ -1836,62 +2413,44 @@ export function createRedisStoreCore(
|
|
|
1836
2413
|
type: e.type,
|
|
1837
2414
|
payload: e.payload,
|
|
1838
2415
|
cause: e.cause ? { ...e.cause } : null,
|
|
1839
|
-
|
|
1840
|
-
settled: e.settled === true,
|
|
2416
|
+
targets: e.targets,
|
|
1841
2417
|
}))
|
|
1842
|
-
const
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
pendingKey(sessionId),
|
|
1864
|
-
readyKey(sessionId),
|
|
1865
|
-
metaSplitKey(sessionId),
|
|
1866
|
-
...withIds.map((e) => idKey(e.id)),
|
|
1867
|
-
],
|
|
1868
|
-
args,
|
|
2418
|
+
const payloads = withIds.map((e) => json(e.payload))
|
|
2419
|
+
const reply = (await indexing(sessionId, () =>
|
|
2420
|
+
evalScript(
|
|
2421
|
+
APPEND_LUA,
|
|
2422
|
+
sessionId,
|
|
2423
|
+
[],
|
|
2424
|
+
[
|
|
2425
|
+
sessionId,
|
|
2426
|
+
now,
|
|
2427
|
+
withIds.length,
|
|
2428
|
+
servedArg(appendOptions?.consumers),
|
|
2429
|
+
appendOptions?.originIndex ?? '',
|
|
2430
|
+
...withIds.flatMap((e, i) => [
|
|
2431
|
+
e.id,
|
|
2432
|
+
e.type,
|
|
2433
|
+
payloads[i]!,
|
|
2434
|
+
e.cause ? json(e.cause) : '',
|
|
2435
|
+
targetsArg(e.targets),
|
|
2436
|
+
]),
|
|
2437
|
+
],
|
|
2438
|
+
),
|
|
1869
2439
|
)) as string[]
|
|
1870
2440
|
|
|
1871
2441
|
const [verdict] = reply
|
|
1872
2442
|
if (verdict === 'empty') {
|
|
1873
2443
|
return { events: [], hasPending: reply[1] === '1' }
|
|
1874
2444
|
}
|
|
1875
|
-
if (verdict === 'foreign') {
|
|
1876
|
-
throw new A2Error(
|
|
1877
|
-
'PARTIAL_DUPLICATE_BATCH',
|
|
1878
|
-
`event id '${reply[1]}' already exists in another session`,
|
|
1879
|
-
)
|
|
1880
|
-
}
|
|
1881
|
-
if (verdict === 'no_parent') {
|
|
1882
|
-
throw new TypeError(
|
|
1883
|
-
`no event at index ${reply[1]} in session '${sessionId}'`,
|
|
1884
|
-
)
|
|
1885
|
-
}
|
|
1886
2445
|
if (verdict === 'cleared') {
|
|
1887
2446
|
throw sessionClearedError(
|
|
1888
2447
|
`scheduled work from event ${reply[1]} was cleared in session '${sessionId}'`,
|
|
1889
2448
|
)
|
|
1890
2449
|
}
|
|
1891
2450
|
if (verdict === 'superseded') {
|
|
1892
|
-
throw
|
|
1893
|
-
|
|
1894
|
-
|
|
2451
|
+
throw supersededAttemptError(
|
|
2452
|
+
sessionId,
|
|
2453
|
+
withIds[Number(reply[1]) - 1]!.cause!,
|
|
1895
2454
|
)
|
|
1896
2455
|
}
|
|
1897
2456
|
if (verdict === 'partial') {
|
|
@@ -1918,12 +2477,23 @@ export function createRedisStoreCore(
|
|
|
1918
2477
|
`session '${sessionId}' was cleared while an append retry replayed; retry it`,
|
|
1919
2478
|
)
|
|
1920
2479
|
}
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
2480
|
+
const replayed = idempotentReplay(events, existing)
|
|
2481
|
+
// The first attempt may have committed without publishing.
|
|
2482
|
+
await publish(
|
|
2483
|
+
notifyChannel(sessionId),
|
|
2484
|
+
wakeMessage(firstIndex + events.length - 1, now, []),
|
|
2485
|
+
)
|
|
2486
|
+
return { events: replayed, hasPending }
|
|
1925
2487
|
}
|
|
1926
2488
|
const base = Number(reply[1]) - 1
|
|
2489
|
+
await publish(
|
|
2490
|
+
notifyChannel(sessionId),
|
|
2491
|
+
wakeMessage(
|
|
2492
|
+
base + withIds.length,
|
|
2493
|
+
now,
|
|
2494
|
+
withIds.map((e, i) => [base + 1 + i, e.id, e.type, payloads[i]!]),
|
|
2495
|
+
),
|
|
2496
|
+
)
|
|
1927
2497
|
return {
|
|
1928
2498
|
events: withIds.map((e, i) => ({
|
|
1929
2499
|
id: e.id,
|
|
@@ -1933,26 +2503,39 @@ export function createRedisStoreCore(
|
|
|
1933
2503
|
sessionId,
|
|
1934
2504
|
createdAt: new Date(now),
|
|
1935
2505
|
cause: e.cause,
|
|
1936
|
-
lane: e.lane,
|
|
1937
|
-
processedAt: e.settled ? new Date(now) : null,
|
|
1938
|
-
processedByAttempt: null,
|
|
1939
|
-
returnedEventIds: null,
|
|
1940
|
-
firstClaimedAt: null,
|
|
1941
|
-
lastClaimedAt: null,
|
|
1942
|
-
attemptCount: 0,
|
|
1943
|
-
claimHolder: null,
|
|
1944
|
-
claimExpiresAt: null,
|
|
1945
|
-
failureCount: 0,
|
|
1946
|
-
lastFailedAt: null,
|
|
1947
|
-
lastFailedAttempt: null,
|
|
1948
|
-
lastError: null,
|
|
1949
|
-
failedAt: null,
|
|
1950
2506
|
})),
|
|
1951
2507
|
hasPending: reply[2] === '1',
|
|
1952
2508
|
}
|
|
1953
2509
|
})
|
|
1954
2510
|
},
|
|
1955
2511
|
|
|
2512
|
+
async cursors(sessionId) {
|
|
2513
|
+
return wrap(async () => {
|
|
2514
|
+
const reply = (await runScript({
|
|
2515
|
+
script: CURSORS_LUA,
|
|
2516
|
+
keys: [countKey(sessionId)],
|
|
2517
|
+
args: [],
|
|
2518
|
+
readOnly: true,
|
|
2519
|
+
})) as Array<
|
|
2520
|
+
[
|
|
2521
|
+
consumer: string,
|
|
2522
|
+
tag: string,
|
|
2523
|
+
position: string,
|
|
2524
|
+
head: string | null,
|
|
2525
|
+
columns: Column[],
|
|
2526
|
+
]
|
|
2527
|
+
>
|
|
2528
|
+
return reply
|
|
2529
|
+
.map(([consumer, tag, position, head, columns]): StoredCursor =>
|
|
2530
|
+
Object.assign(dispatchOf(consumer, laneOf(tag), columns), {
|
|
2531
|
+
position: Number(position),
|
|
2532
|
+
head: head === null ? null : Number(head),
|
|
2533
|
+
}),
|
|
2534
|
+
)
|
|
2535
|
+
.toSorted(byCursor)
|
|
2536
|
+
})
|
|
2537
|
+
},
|
|
2538
|
+
|
|
1956
2539
|
async read(sessionId, opts) {
|
|
1957
2540
|
return wrap(() =>
|
|
1958
2541
|
retryAfterGap(() =>
|
|
@@ -1967,21 +2550,7 @@ export function createRedisStoreCore(
|
|
|
1967
2550
|
return []
|
|
1968
2551
|
return wrap(() =>
|
|
1969
2552
|
retryAfterGap(async () => {
|
|
1970
|
-
const
|
|
1971
|
-
'MGET',
|
|
1972
|
-
countKey(sessionId),
|
|
1973
|
-
clearKey(sessionId),
|
|
1974
|
-
)) as [unknown, unknown]
|
|
1975
|
-
const frontier = toIndex({
|
|
1976
|
-
raw: count,
|
|
1977
|
-
sessionId,
|
|
1978
|
-
name: 'event count',
|
|
1979
|
-
})
|
|
1980
|
-
const clearIndex = toIndex({
|
|
1981
|
-
raw: cleared,
|
|
1982
|
-
sessionId,
|
|
1983
|
-
name: 'clear boundary',
|
|
1984
|
-
})
|
|
2553
|
+
const { frontier, clearIndex } = await readBounds(sessionId)
|
|
1985
2554
|
// Reads start at the latest clear marker.
|
|
1986
2555
|
return readEventRange(
|
|
1987
2556
|
sessionId,
|
|
@@ -1997,32 +2566,31 @@ export function createRedisStoreCore(
|
|
|
1997
2566
|
holder,
|
|
1998
2567
|
ttlMs,
|
|
1999
2568
|
expiresAtMs,
|
|
2000
|
-
|
|
2569
|
+
exclude,
|
|
2570
|
+
consumers,
|
|
2001
2571
|
maxFailures,
|
|
2572
|
+
through,
|
|
2002
2573
|
}) {
|
|
2003
2574
|
return wrap(async () => {
|
|
2004
2575
|
const now = clock.now().getTime()
|
|
2005
2576
|
const expiresAt = expiresAtMs ?? now + ttlMs
|
|
2006
|
-
const excluded =
|
|
2577
|
+
const excluded = exclude ?? []
|
|
2007
2578
|
const reply = (await evalScript(
|
|
2008
2579
|
CLAIM_AVAILABLE_LUA,
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
countKey(sessionId),
|
|
2012
|
-
metaKey(sessionId),
|
|
2013
|
-
pendingKey(sessionId),
|
|
2014
|
-
readyKey(sessionId),
|
|
2015
|
-
metaSplitKey(sessionId),
|
|
2016
|
-
snapshotPinCountsKey(sessionId),
|
|
2017
|
-
],
|
|
2580
|
+
sessionId,
|
|
2581
|
+
[],
|
|
2018
2582
|
[
|
|
2019
2583
|
holder,
|
|
2020
2584
|
now,
|
|
2021
2585
|
expiresAt,
|
|
2022
|
-
excluded.length,
|
|
2023
|
-
...excluded,
|
|
2024
2586
|
maxFailures ?? '',
|
|
2025
|
-
|
|
2587
|
+
servedArg(consumers),
|
|
2588
|
+
through ?? '',
|
|
2589
|
+
excluded.length,
|
|
2590
|
+
...excluded.flatMap((ref) => [
|
|
2591
|
+
ref.consumer,
|
|
2592
|
+
ref.lane === null ? `#${ref.index}` : `L${ref.lane}`,
|
|
2593
|
+
]),
|
|
2026
2594
|
],
|
|
2027
2595
|
)) as unknown[]
|
|
2028
2596
|
return toClaimAvailableResult(sessionId, reply)
|
|
@@ -2030,30 +2598,48 @@ export function createRedisStoreCore(
|
|
|
2030
2598
|
},
|
|
2031
2599
|
|
|
2032
2600
|
async renewClaims({ sessionId, holder, claims, ttlMs, expiresAtMs }) {
|
|
2601
|
+
if (claims.length === 0) return { renewed: [], superseded: [] }
|
|
2033
2602
|
return wrap(async () => {
|
|
2034
2603
|
const now = clock.now().getTime()
|
|
2035
2604
|
const expiresAt = expiresAtMs ?? now + ttlMs
|
|
2036
2605
|
const reply = (await evalScript(
|
|
2037
2606
|
RENEW_CLAIMS_LUA,
|
|
2038
|
-
|
|
2607
|
+
sessionId,
|
|
2608
|
+
[],
|
|
2039
2609
|
[
|
|
2040
2610
|
holder,
|
|
2041
2611
|
now,
|
|
2042
2612
|
expiresAt,
|
|
2043
2613
|
claims.length,
|
|
2044
|
-
...claims.flatMap((claim) => [
|
|
2614
|
+
...claims.flatMap((claim) => [
|
|
2615
|
+
claim.consumer,
|
|
2616
|
+
tagOf(claim.lane),
|
|
2617
|
+
claim.index,
|
|
2618
|
+
claim.attempt,
|
|
2619
|
+
]),
|
|
2045
2620
|
],
|
|
2046
2621
|
)) as string[]
|
|
2047
2622
|
const renewedCount = Number(reply[0])
|
|
2048
|
-
const
|
|
2623
|
+
const refs = reply.slice(1).map((position): HeadRef => {
|
|
2624
|
+
const { consumer, lane, index } = claims[Number(position) - 1]!
|
|
2625
|
+
return { consumer, lane, index }
|
|
2626
|
+
})
|
|
2049
2627
|
return {
|
|
2050
|
-
renewed:
|
|
2051
|
-
superseded:
|
|
2628
|
+
renewed: refs.slice(0, renewedCount).toSorted(byHead),
|
|
2629
|
+
superseded: refs.slice(renewedCount).toSorted(byHead),
|
|
2052
2630
|
}
|
|
2053
2631
|
})
|
|
2054
2632
|
},
|
|
2055
2633
|
|
|
2056
|
-
async completeAttempt({
|
|
2634
|
+
async completeAttempt({
|
|
2635
|
+
sessionId,
|
|
2636
|
+
consumer,
|
|
2637
|
+
lane,
|
|
2638
|
+
index,
|
|
2639
|
+
attempt,
|
|
2640
|
+
events,
|
|
2641
|
+
consumers,
|
|
2642
|
+
}) {
|
|
2057
2643
|
return wrap(async () => {
|
|
2058
2644
|
const ids = events.map((event) => event.id)
|
|
2059
2645
|
if (new Set(ids).size !== ids.length) {
|
|
@@ -2063,34 +2649,32 @@ export function createRedisStoreCore(
|
|
|
2063
2649
|
)
|
|
2064
2650
|
}
|
|
2065
2651
|
const now = clock.now().getTime()
|
|
2066
|
-
const
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
metaKey(sessionId),
|
|
2072
|
-
pendingKey(sessionId),
|
|
2073
|
-
readyKey(sessionId),
|
|
2074
|
-
snapshotEventPinsKey(sessionId, index),
|
|
2075
|
-
snapshotPinCountsKey(sessionId),
|
|
2076
|
-
metaSplitKey(sessionId),
|
|
2077
|
-
...ids.map((id) => idKey(id)),
|
|
2078
|
-
],
|
|
2079
|
-
[
|
|
2652
|
+
const cause: EventCause = { consumer, lane, index, attempt }
|
|
2653
|
+
const payloads = events.map((event) => json(event.payload))
|
|
2654
|
+
const reply = (await indexing(sessionId, () =>
|
|
2655
|
+
evalScript(
|
|
2656
|
+
COMPLETE_ATTEMPT_LUA,
|
|
2080
2657
|
sessionId,
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2658
|
+
[],
|
|
2659
|
+
[
|
|
2660
|
+
sessionId,
|
|
2661
|
+
now,
|
|
2662
|
+
consumer,
|
|
2663
|
+
tagOf(lane),
|
|
2664
|
+
index,
|
|
2665
|
+
attempt,
|
|
2666
|
+
json(cause),
|
|
2667
|
+
servedArg(consumers),
|
|
2668
|
+
json(ids),
|
|
2669
|
+
events.length,
|
|
2670
|
+
...events.flatMap((event, i) => [
|
|
2671
|
+
event.id,
|
|
2672
|
+
event.type,
|
|
2673
|
+
payloads[i]!,
|
|
2674
|
+
targetsArg(event.targets),
|
|
2675
|
+
]),
|
|
2676
|
+
],
|
|
2677
|
+
),
|
|
2094
2678
|
)) as string[]
|
|
2095
2679
|
const outcome = reply[0]
|
|
2096
2680
|
if (outcome === 'missing') {
|
|
@@ -2106,9 +2690,23 @@ export function createRedisStoreCore(
|
|
|
2106
2690
|
}
|
|
2107
2691
|
if (outcome === 'superseded') return { outcome }
|
|
2108
2692
|
|
|
2109
|
-
const cause = { index, attempt }
|
|
2110
2693
|
if (outcome === 'ok') {
|
|
2111
2694
|
const base = Number(reply[1]) - 1
|
|
2695
|
+
if (events.length > 0) {
|
|
2696
|
+
await publish(
|
|
2697
|
+
notifyChannel(sessionId),
|
|
2698
|
+
wakeMessage(
|
|
2699
|
+
base + events.length,
|
|
2700
|
+
now,
|
|
2701
|
+
events.map((event, i) => [
|
|
2702
|
+
base + 1 + i,
|
|
2703
|
+
event.id,
|
|
2704
|
+
event.type,
|
|
2705
|
+
payloads[i]!,
|
|
2706
|
+
]),
|
|
2707
|
+
),
|
|
2708
|
+
)
|
|
2709
|
+
}
|
|
2112
2710
|
return {
|
|
2113
2711
|
outcome: 'completed',
|
|
2114
2712
|
events: events.map((event, i) => ({
|
|
@@ -2118,21 +2716,7 @@ export function createRedisStoreCore(
|
|
|
2118
2716
|
index: base + 1 + i,
|
|
2119
2717
|
sessionId,
|
|
2120
2718
|
createdAt: new Date(now),
|
|
2121
|
-
cause,
|
|
2122
|
-
lane: event.lane ?? null,
|
|
2123
|
-
processedAt: event.settled === true ? new Date(now) : null,
|
|
2124
|
-
processedByAttempt: null,
|
|
2125
|
-
returnedEventIds: null,
|
|
2126
|
-
firstClaimedAt: null,
|
|
2127
|
-
lastClaimedAt: null,
|
|
2128
|
-
attemptCount: 0,
|
|
2129
|
-
claimHolder: null,
|
|
2130
|
-
claimExpiresAt: null,
|
|
2131
|
-
failureCount: 0,
|
|
2132
|
-
lastFailedAt: null,
|
|
2133
|
-
lastFailedAttempt: null,
|
|
2134
|
-
lastError: null,
|
|
2135
|
-
failedAt: null,
|
|
2719
|
+
cause: { ...cause },
|
|
2136
2720
|
})),
|
|
2137
2721
|
} satisfies CompleteAttemptResult
|
|
2138
2722
|
}
|
|
@@ -2150,10 +2734,27 @@ export function createRedisStoreCore(
|
|
|
2150
2734
|
)
|
|
2151
2735
|
// Cleared after the script matched the retry.
|
|
2152
2736
|
if (index <= clearIndex) return { outcome: 'superseded' }
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2737
|
+
if (
|
|
2738
|
+
existing.some(
|
|
2739
|
+
(row) =>
|
|
2740
|
+
row.cause?.consumer !== consumer ||
|
|
2741
|
+
row.cause.lane !== lane ||
|
|
2742
|
+
row.cause.index !== index ||
|
|
2743
|
+
row.cause.attempt !== attempt,
|
|
2744
|
+
)
|
|
2745
|
+
) {
|
|
2746
|
+
throw new A2Error(
|
|
2747
|
+
'PARTIAL_DUPLICATE_BATCH',
|
|
2748
|
+
'returned event retry does not match the committed batch',
|
|
2749
|
+
)
|
|
2156
2750
|
}
|
|
2751
|
+
const replayed = idempotentReplay(events, existing)
|
|
2752
|
+
// The first attempt may have committed without publishing.
|
|
2753
|
+
await publish(
|
|
2754
|
+
notifyChannel(sessionId),
|
|
2755
|
+
wakeMessage(firstIndex + events.length - 1, now, []),
|
|
2756
|
+
)
|
|
2757
|
+
return { outcome: 'completed', events: replayed }
|
|
2157
2758
|
}
|
|
2158
2759
|
throw new TypeError(`unexpected completion outcome '${outcome}'`)
|
|
2159
2760
|
})
|
|
@@ -2161,6 +2762,8 @@ export function createRedisStoreCore(
|
|
|
2161
2762
|
|
|
2162
2763
|
async failAttempt({
|
|
2163
2764
|
sessionId,
|
|
2765
|
+
consumer,
|
|
2766
|
+
lane,
|
|
2164
2767
|
index,
|
|
2165
2768
|
attempt,
|
|
2166
2769
|
error,
|
|
@@ -2171,16 +2774,11 @@ export function createRedisStoreCore(
|
|
|
2171
2774
|
const now = clock.now().getTime()
|
|
2172
2775
|
const reply = (await evalScript(
|
|
2173
2776
|
FAIL_ATTEMPT_LUA,
|
|
2777
|
+
sessionId,
|
|
2778
|
+
[],
|
|
2174
2779
|
[
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
pendingKey(sessionId),
|
|
2178
|
-
readyKey(sessionId),
|
|
2179
|
-
snapshotEventPinsKey(sessionId, index),
|
|
2180
|
-
snapshotPinCountsKey(sessionId),
|
|
2181
|
-
metaSplitKey(sessionId),
|
|
2182
|
-
],
|
|
2183
|
-
[
|
|
2780
|
+
consumer,
|
|
2781
|
+
tagOf(lane),
|
|
2184
2782
|
index,
|
|
2185
2783
|
attempt,
|
|
2186
2784
|
error,
|
|
@@ -2190,8 +2788,6 @@ export function createRedisStoreCore(
|
|
|
2190
2788
|
],
|
|
2191
2789
|
)) as string[]
|
|
2192
2790
|
const outcome = reply[0]
|
|
2193
|
-
if (outcome === 'cleared')
|
|
2194
|
-
return { outcome: 'superseded', failureCount: 0 }
|
|
2195
2791
|
if (outcome === 'missing') {
|
|
2196
2792
|
throw new TypeError(
|
|
2197
2793
|
`no event at index ${index} in session '${sessionId}'`,
|
|
@@ -2206,67 +2802,84 @@ export function createRedisStoreCore(
|
|
|
2206
2802
|
|
|
2207
2803
|
async deadLetters(sessionId) {
|
|
2208
2804
|
return wrap(async () => {
|
|
2209
|
-
const reply = (await
|
|
2210
|
-
DEAD_LETTERS_LUA,
|
|
2211
|
-
[
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
[],
|
|
2219
|
-
)) as unknown[]
|
|
2220
|
-
const events: StoredEvent[] = []
|
|
2221
|
-
for (let i = 0; i + 1 < reply.length; i += 2) {
|
|
2222
|
-
const columns = reply[i + 1] as Array<string | null>
|
|
2223
|
-
events.push(
|
|
2224
|
-
toStoredEvent(
|
|
2225
|
-
sessionId,
|
|
2226
|
-
reply[i] as StreamEntry,
|
|
2227
|
-
(suffix) =>
|
|
2228
|
-
columns[STORED_EVENT_META_SUFFIXES.indexOf(suffix)] ??
|
|
2229
|
-
undefined,
|
|
2230
|
-
),
|
|
2231
|
-
)
|
|
2232
|
-
}
|
|
2233
|
-
return events
|
|
2805
|
+
const reply = (await runScript({
|
|
2806
|
+
script: DEAD_LETTERS_LUA,
|
|
2807
|
+
keys: [countKey(sessionId)],
|
|
2808
|
+
args: [],
|
|
2809
|
+
readOnly: true,
|
|
2810
|
+
})) as HeadTuple[]
|
|
2811
|
+
return reply
|
|
2812
|
+
.map((head) => toHeadEvent(sessionId, head))
|
|
2813
|
+
.toSorted(byHead)
|
|
2234
2814
|
})
|
|
2235
2815
|
},
|
|
2236
2816
|
|
|
2237
|
-
async redrive({ sessionId, index, action }) {
|
|
2817
|
+
async redrive({ sessionId, consumer, index, action, consumers }) {
|
|
2238
2818
|
return wrap(async () => {
|
|
2239
|
-
const [redriven, hasPending] = (await
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2819
|
+
const [redriven, hasPending] = (await indexing(sessionId, () =>
|
|
2820
|
+
evalScript(
|
|
2821
|
+
REDRIVE_LUA,
|
|
2822
|
+
sessionId,
|
|
2823
|
+
[],
|
|
2824
|
+
[
|
|
2825
|
+
consumer,
|
|
2826
|
+
index,
|
|
2827
|
+
action,
|
|
2828
|
+
clock.now().getTime(),
|
|
2829
|
+
servedArg(consumers),
|
|
2830
|
+
],
|
|
2831
|
+
),
|
|
2249
2832
|
)) as [string, string]
|
|
2250
2833
|
return { redriven: redriven === '1', hasPending: hasPending === '1' }
|
|
2251
2834
|
})
|
|
2252
2835
|
},
|
|
2253
2836
|
|
|
2837
|
+
async wakeLog({ sessionId, consumer }) {
|
|
2838
|
+
return wrap(async () => {
|
|
2839
|
+
const { name, log } = consumer
|
|
2840
|
+
const [, position, head, columns, sessionHead] = (await indexing(
|
|
2841
|
+
sessionId,
|
|
2842
|
+
() =>
|
|
2843
|
+
evalScript(
|
|
2844
|
+
WAKE_LOG_LUA,
|
|
2845
|
+
sessionId,
|
|
2846
|
+
[],
|
|
2847
|
+
[name, log.lane, logTypesArg(log.types), log.backfill ? 1 : 0],
|
|
2848
|
+
),
|
|
2849
|
+
)) as [
|
|
2850
|
+
ok: string,
|
|
2851
|
+
position: string,
|
|
2852
|
+
head: string | null,
|
|
2853
|
+
columns: Column[],
|
|
2854
|
+
sessionHead: string,
|
|
2855
|
+
]
|
|
2856
|
+
return {
|
|
2857
|
+
cursor: Object.assign(dispatchOf(name, log.lane, columns), {
|
|
2858
|
+
position: Number(position),
|
|
2859
|
+
head: head === null ? null : Number(head),
|
|
2860
|
+
}),
|
|
2861
|
+
head: Number(sessionHead),
|
|
2862
|
+
}
|
|
2863
|
+
})
|
|
2864
|
+
},
|
|
2865
|
+
|
|
2254
2866
|
async readState(sessionId, reducerName, stateOptions) {
|
|
2255
2867
|
return wrap(() =>
|
|
2256
2868
|
retryAfterGap(async () => {
|
|
2257
|
-
const reply = (await
|
|
2258
|
-
READ_STATE_LUA,
|
|
2259
|
-
[
|
|
2869
|
+
const reply = (await runScript({
|
|
2870
|
+
script: READ_STATE_LUA,
|
|
2871
|
+
keys: [
|
|
2872
|
+
countKey(sessionId),
|
|
2260
2873
|
snapKey(sessionId, reducerName),
|
|
2261
2874
|
snapHistoryKey(sessionId, reducerName),
|
|
2262
|
-
countKey(sessionId),
|
|
2263
2875
|
],
|
|
2264
|
-
[
|
|
2876
|
+
args: [
|
|
2265
2877
|
stateOptions?.snapshotThroughIndex ??
|
|
2266
2878
|
stateOptions?.throughIndex ??
|
|
2267
2879
|
'',
|
|
2268
2880
|
],
|
|
2269
|
-
|
|
2881
|
+
readOnly: true,
|
|
2882
|
+
})) as [number | string, string, string, string]
|
|
2270
2883
|
const rawHead = reply[0]
|
|
2271
2884
|
const raw = reply[1]
|
|
2272
2885
|
const headIndex = rawHead === '' ? null : Number(rawHead)
|
|
@@ -2315,33 +2928,28 @@ export function createRedisStoreCore(
|
|
|
2315
2928
|
async putSnapshots(sessionId, reducerName, writes) {
|
|
2316
2929
|
const byIndex = new Map<
|
|
2317
2930
|
number,
|
|
2318
|
-
{ index: number; state: unknown;
|
|
2931
|
+
{ index: number; state: unknown; pins: Map<string, HeadRef> }
|
|
2319
2932
|
>()
|
|
2320
2933
|
for (const write of writes) {
|
|
2321
2934
|
const existing = byIndex.get(write.index)
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2935
|
+
const pins = existing?.pins ?? new Map<string, HeadRef>()
|
|
2936
|
+
for (const pin of write.pins ?? []) {
|
|
2937
|
+
pins.set(JSON.stringify([pin.consumer, pin.lane, pin.index]), pin)
|
|
2938
|
+
}
|
|
2939
|
+
if (existing) existing.state = write.state
|
|
2940
|
+
else
|
|
2328
2941
|
byIndex.set(write.index, {
|
|
2329
2942
|
index: write.index,
|
|
2330
2943
|
state: write.state,
|
|
2331
|
-
|
|
2944
|
+
pins,
|
|
2332
2945
|
})
|
|
2333
|
-
}
|
|
2334
2946
|
}
|
|
2335
2947
|
const normalized = [...byIndex.values()].toSorted(
|
|
2336
2948
|
(a, b) => a.index - b.index,
|
|
2337
2949
|
)
|
|
2338
2950
|
if (normalized.length === 0) return
|
|
2339
2951
|
await wrap(async () => {
|
|
2340
|
-
const args: Array<string | number> = [
|
|
2341
|
-
clock.now().getTime(),
|
|
2342
|
-
snapshotEventPinsPrefix(sessionId),
|
|
2343
|
-
normalized.length,
|
|
2344
|
-
]
|
|
2952
|
+
const args: Array<string | number> = [normalized.length]
|
|
2345
2953
|
for (const write of normalized) {
|
|
2346
2954
|
args.push(
|
|
2347
2955
|
write.index,
|
|
@@ -2352,19 +2960,20 @@ export function createRedisStoreCore(
|
|
|
2352
2960
|
// This reducer is in the session's registry (PUT_SNAPSHOTS_LUA).
|
|
2353
2961
|
r: 1,
|
|
2354
2962
|
}),
|
|
2355
|
-
write.
|
|
2356
|
-
...write.
|
|
2963
|
+
write.pins.size,
|
|
2964
|
+
...[...write.pins.values()].flatMap((pin) => [
|
|
2965
|
+
pin.consumer,
|
|
2966
|
+
tagOf(pin.lane),
|
|
2967
|
+
pin.index,
|
|
2968
|
+
]),
|
|
2357
2969
|
)
|
|
2358
2970
|
}
|
|
2359
2971
|
await evalScript(
|
|
2360
2972
|
PUT_SNAPSHOTS_LUA,
|
|
2973
|
+
sessionId,
|
|
2361
2974
|
[
|
|
2362
2975
|
snapKey(sessionId, reducerName),
|
|
2363
2976
|
snapHistoryKey(sessionId, reducerName),
|
|
2364
|
-
countKey(sessionId),
|
|
2365
|
-
metaKey(sessionId),
|
|
2366
|
-
snapshotPinCountsKey(sessionId),
|
|
2367
|
-
metaSplitKey(sessionId),
|
|
2368
2977
|
],
|
|
2369
2978
|
[...args, reducerName],
|
|
2370
2979
|
)
|
|
@@ -2380,69 +2989,76 @@ export function createRedisStoreCore(
|
|
|
2380
2989
|
// Expiry anchors on the injected storage clock — the sender's
|
|
2381
2990
|
// `a` stamp orders writes, the `e` stamp bounds their
|
|
2382
2991
|
// lifetime, and the two clocks never mix.
|
|
2383
|
-
const
|
|
2992
|
+
const nowMs = clock.now().getTime()
|
|
2993
|
+
const expiresAtMs = nowMs + meta.ttlMs
|
|
2384
2994
|
// The hash field is JSON `[participant, field]`: both are
|
|
2385
2995
|
// caller strings, so no separator character is
|
|
2386
2996
|
// collision-safe — the JSON array is.
|
|
2387
|
-
const applied = (await
|
|
2388
|
-
PRESENCE_SET_LUA,
|
|
2389
|
-
[presenceKey(ns)],
|
|
2390
|
-
[
|
|
2997
|
+
const applied = (await runScript({
|
|
2998
|
+
script: PRESENCE_SET_LUA,
|
|
2999
|
+
keys: [presenceKey(ns)],
|
|
3000
|
+
args: [
|
|
2391
3001
|
atMs,
|
|
2392
3002
|
meta.ttlMs,
|
|
3003
|
+
nowMs,
|
|
2393
3004
|
entries.length,
|
|
3005
|
+
// A null is stored as a tombstone: it keeps its stamp, so an
|
|
3006
|
+
// older set that arrives after it still loses.
|
|
2394
3007
|
...entries.flatMap(([field, value]) => [
|
|
2395
3008
|
JSON.stringify([participant, field]),
|
|
2396
|
-
value
|
|
2397
|
-
? ''
|
|
2398
|
-
: json({
|
|
2399
|
-
v: value,
|
|
2400
|
-
s: meta.seen,
|
|
2401
|
-
a: atMs,
|
|
2402
|
-
e: expiresAtMs,
|
|
2403
|
-
}),
|
|
2404
|
-
`${json(field)}:${json(value)}`,
|
|
3009
|
+
json({ a: atMs, e: expiresAtMs, s: meta.seen, v: value }),
|
|
2405
3010
|
]),
|
|
2406
|
-
presenceKey(ns),
|
|
2407
|
-
`${json({ src: options.presenceSource, participant, seen: meta.seen, at: atMs }).slice(0, -1)},"values":{`,
|
|
2408
3011
|
],
|
|
2409
|
-
|
|
3012
|
+
readOnly: false,
|
|
3013
|
+
})) as string[]
|
|
2410
3014
|
if (applied.length === 0) return null
|
|
2411
3015
|
// Field-keyed and caller-named — null-prototype, like every
|
|
2412
3016
|
// presence map (see `nullProtoRecord`).
|
|
2413
3017
|
const appliedValues: Record<string, unknown> = nullProtoRecord()
|
|
3018
|
+
const published: string[] = []
|
|
2414
3019
|
for (const position of applied) {
|
|
2415
3020
|
const [field, value] = entries[Number(position) - 1]!
|
|
2416
3021
|
appliedValues[field] = structuredClone(value)
|
|
3022
|
+
published.push(`${json(field)}:${json(value)}`)
|
|
2417
3023
|
}
|
|
2418
3024
|
return {
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
3025
|
+
patch: {
|
|
3026
|
+
participant,
|
|
3027
|
+
values: appliedValues,
|
|
3028
|
+
seen: meta.seen,
|
|
3029
|
+
at: new Date(atMs),
|
|
3030
|
+
},
|
|
3031
|
+
published: publish(
|
|
3032
|
+
presenceChannel(ns),
|
|
3033
|
+
`${json({ src: options.presenceSource, participant, seen: meta.seen, at: atMs }).slice(0, -1)},"values":{${published.join(',')}}}`,
|
|
3034
|
+
),
|
|
2423
3035
|
}
|
|
2424
3036
|
})
|
|
2425
3037
|
},
|
|
2426
3038
|
|
|
2427
3039
|
async read(ns) {
|
|
2428
3040
|
return wrap(async () => {
|
|
2429
|
-
const
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
[clock.now().getTime()],
|
|
2433
|
-
)) as string[]
|
|
3041
|
+
const key = presenceKey(ns)
|
|
3042
|
+
const reply = (await call('HGETALL', key)) as string[]
|
|
3043
|
+
const nowMs = clock.now().getTime()
|
|
2434
3044
|
const rows: PresenceRow[] = []
|
|
3045
|
+
const expired: string[] = []
|
|
2435
3046
|
for (let i = 0; i + 1 < reply.length; i += 2) {
|
|
2436
|
-
const [participant, field] = JSON.parse(reply[i]!) as [
|
|
2437
|
-
string,
|
|
2438
|
-
string,
|
|
2439
|
-
]
|
|
2440
3047
|
const entry = JSON.parse(reply[i + 1]!) as {
|
|
2441
3048
|
v: unknown
|
|
2442
3049
|
s: number
|
|
2443
3050
|
a: number
|
|
2444
3051
|
e: number
|
|
2445
3052
|
}
|
|
3053
|
+
if (entry.e <= nowMs) {
|
|
3054
|
+
expired.push(reply[i]!)
|
|
3055
|
+
continue
|
|
3056
|
+
}
|
|
3057
|
+
if (entry.v === null) continue
|
|
3058
|
+
const [participant, field] = JSON.parse(reply[i]!) as [
|
|
3059
|
+
string,
|
|
3060
|
+
string,
|
|
3061
|
+
]
|
|
2446
3062
|
rows.push({
|
|
2447
3063
|
participant,
|
|
2448
3064
|
field,
|
|
@@ -2452,6 +3068,16 @@ export function createRedisStoreCore(
|
|
|
2452
3068
|
expiresAt: new Date(entry.e),
|
|
2453
3069
|
})
|
|
2454
3070
|
}
|
|
3071
|
+
// The rows are already filtered; a failed prune only defers the
|
|
3072
|
+
// deletion to the next read.
|
|
3073
|
+
if (expired.length > 0) {
|
|
3074
|
+
await runScript({
|
|
3075
|
+
script: PRESENCE_PRUNE_LUA,
|
|
3076
|
+
keys: [key],
|
|
3077
|
+
args: [nowMs, ...expired],
|
|
3078
|
+
readOnly: false,
|
|
3079
|
+
}).catch(() => undefined)
|
|
3080
|
+
}
|
|
2455
3081
|
return rows
|
|
2456
3082
|
})
|
|
2457
3083
|
},
|
|
@@ -2463,14 +3089,20 @@ export function createRedisStoreCore(
|
|
|
2463
3089
|
|
|
2464
3090
|
async readTail(sessionId, fromIndex, throughIndex) {
|
|
2465
3091
|
return wrap(async () => {
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
3092
|
+
// Entries below the latest clear are an old lifetime's, awaiting
|
|
3093
|
+
// reclaim: a tail that reaches them is the boundary-aware reads' job.
|
|
3094
|
+
const [newest, clearIndex] = await Promise.all([
|
|
3095
|
+
call(
|
|
3096
|
+
'XREVRANGE',
|
|
3097
|
+
storeKey(sessionId),
|
|
3098
|
+
throughIndex === undefined ? '+' : `${throughIndex}-0`,
|
|
3099
|
+
`${fromIndex}-0`,
|
|
3100
|
+
'COUNT',
|
|
3101
|
+
READ_PAGE_SIZE,
|
|
3102
|
+
) as Promise<StreamEntry[]>,
|
|
3103
|
+
readClearIndex(sessionId),
|
|
3104
|
+
])
|
|
3105
|
+
if (clearIndex > fromIndex) return null
|
|
2474
3106
|
const page = newest
|
|
2475
3107
|
.toReversed()
|
|
2476
3108
|
.map((entry) => toEvent(sessionId, entry))
|
|
@@ -2490,41 +3122,43 @@ export function createRedisStoreCore(
|
|
|
2490
3122
|
async clear({ sessionId, ifLastIndex, cause }) {
|
|
2491
3123
|
return wrap(async () => {
|
|
2492
3124
|
const markerId = generateId()
|
|
3125
|
+
const now = clock.now().getTime()
|
|
2493
3126
|
const reply = (await evalScript(
|
|
2494
3127
|
CLEAR_LUA,
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
countKey(sessionId),
|
|
2498
|
-
metaKey(sessionId),
|
|
2499
|
-
metaSplitKey(sessionId),
|
|
2500
|
-
idKey(markerId),
|
|
2501
|
-
pendingKey(sessionId),
|
|
2502
|
-
readyKey(sessionId),
|
|
2503
|
-
],
|
|
3128
|
+
sessionId,
|
|
3129
|
+
[],
|
|
2504
3130
|
[
|
|
2505
3131
|
sessionId,
|
|
2506
|
-
|
|
3132
|
+
now,
|
|
2507
3133
|
ifLastIndex ?? '',
|
|
3134
|
+
cause?.consumer ?? '',
|
|
3135
|
+
cause ? tagOf(cause.lane) : '',
|
|
2508
3136
|
cause?.index ?? '',
|
|
2509
3137
|
cause?.attempt ?? '',
|
|
2510
3138
|
markerId,
|
|
2511
|
-
`${prefix}:${sessionId}:notify`,
|
|
2512
3139
|
],
|
|
2513
3140
|
)) as string[]
|
|
2514
3141
|
if (reply[0] === 'superseded') {
|
|
2515
3142
|
throw supersededAttemptError(sessionId, cause!)
|
|
2516
3143
|
}
|
|
2517
|
-
if (reply[0] === 'no_parent') {
|
|
2518
|
-
throw new TypeError(
|
|
2519
|
-
`no event at index ${reply[1]} in session '${sessionId}'`,
|
|
2520
|
-
)
|
|
2521
|
-
}
|
|
2522
3144
|
if (reply[0] === 'foreign') {
|
|
2523
3145
|
throw new A2Error(
|
|
2524
3146
|
'PARTIAL_DUPLICATE_BATCH',
|
|
2525
3147
|
`the clear marker id '${markerId}' already exists`,
|
|
2526
3148
|
)
|
|
2527
3149
|
}
|
|
3150
|
+
const head = Number(reply[2])
|
|
3151
|
+
if (reply[4] === '1') {
|
|
3152
|
+
await publish(
|
|
3153
|
+
notifyChannel(sessionId),
|
|
3154
|
+
wakeMessage(head, now, [
|
|
3155
|
+
[head, markerId, CLEAR_EVENT_TYPE, 'null'],
|
|
3156
|
+
]),
|
|
3157
|
+
)
|
|
3158
|
+
} else if (reply[1] === '1' && head > 0) {
|
|
3159
|
+
// An earlier attempt may have appended the marker without publishing.
|
|
3160
|
+
await publish(notifyChannel(sessionId), wakeMessage(head, now, []))
|
|
3161
|
+
}
|
|
2528
3162
|
return {
|
|
2529
3163
|
cleared: reply[1] === '1',
|
|
2530
3164
|
index: Number(reply[2]),
|
|
@@ -2535,31 +3169,25 @@ export function createRedisStoreCore(
|
|
|
2535
3169
|
|
|
2536
3170
|
async reclaim(sessionId) {
|
|
2537
3171
|
return wrap(async () => {
|
|
2538
|
-
const
|
|
3172
|
+
const pairs = (await evalScript(
|
|
2539
3173
|
RECLAIM_BATCH_LUA,
|
|
2540
|
-
|
|
3174
|
+
sessionId,
|
|
3175
|
+
[],
|
|
2541
3176
|
[READ_PAGE_SIZE],
|
|
2542
3177
|
)) as string[]
|
|
2543
|
-
if (
|
|
2544
|
-
// Id keys are built here, like append's: they use JSON.stringify,
|
|
2545
|
-
// whose escaping cjson does not reproduce.
|
|
2546
|
-
const ids: string[] = []
|
|
2547
|
-
for (let i = 1; i < pairs.length; i += 2) ids.push(pairs[i]!)
|
|
3178
|
+
if (pairs.length === 0) return { deleted: 0, done: true }
|
|
2548
3179
|
const [deleted, done] = (await evalScript(
|
|
2549
3180
|
RECLAIM_LUA,
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
metaKey(sessionId),
|
|
2554
|
-
metaSplitKey(sessionId),
|
|
2555
|
-
...ids.map((id) => idKey(id)),
|
|
2556
|
-
],
|
|
2557
|
-
[sessionId, snapshotEventPinsPrefix(sessionId), ids.length, ...pairs],
|
|
3181
|
+
sessionId,
|
|
3182
|
+
[],
|
|
3183
|
+
[sessionId, pairs.length / 2, ...pairs],
|
|
2558
3184
|
)) as [number, number]
|
|
2559
3185
|
return { deleted: Number(deleted), done: Number(done) === 1 }
|
|
2560
3186
|
})
|
|
2561
3187
|
},
|
|
2562
3188
|
|
|
3189
|
+
clearIndex: (sessionId) => wrap(() => readClearIndex(sessionId)),
|
|
3190
|
+
|
|
2563
3191
|
async readEvents(sessionId, afterIndex) {
|
|
2564
3192
|
return wrap(async () => {
|
|
2565
3193
|
const entries = (await call(
|