experimental-a2 0.20.0 → 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/README.md +1 -1
- package/dist/{actor-C7b-1arn.d.ts → actor-HnGmrOZZ.d.ts} +5 -5
- package/dist/{actor-C7b-1arn.d.ts.map → actor-HnGmrOZZ.d.ts.map} +1 -1
- package/dist/actor-client.d.ts +1 -1
- package/dist/actor-client.js +2 -2
- package/dist/actor-react.d.ts +3 -3
- package/dist/actor-react.js +5 -5
- package/dist/{actor-shared-t7w8xIKP.js → actor-shared-DFET5q16.js} +3 -3
- package/dist/{actor-shared-t7w8xIKP.js.map → actor-shared-DFET5q16.js.map} +1 -1
- package/dist/actor.d.ts +1 -1
- package/dist/actor.js +54 -43
- package/dist/actor.js.map +1 -1
- package/dist/{ai-Cz3hqVhk.d.ts → ai-CvEnqVfo.d.ts} +8 -8
- package/dist/{ai-Cz3hqVhk.d.ts.map → ai-CvEnqVfo.d.ts.map} +1 -1
- package/dist/{ai-mMiWul1T.js → ai-CzE672Q3.js} +9 -9
- package/dist/ai-CzE672Q3.js.map +1 -0
- package/dist/ai-client.d.ts +2 -2
- package/dist/ai-client.js +2 -2
- package/dist/ai-fx.d.ts +1 -1
- package/dist/ai-fx.js +3 -6
- package/dist/ai-fx.js.map +1 -1
- package/dist/{ai-harness-CV6rXbB3.d.ts → ai-harness-Bfk-Lycn.d.ts} +2 -2
- package/dist/{ai-harness-CV6rXbB3.d.ts.map → ai-harness-Bfk-Lycn.d.ts.map} +1 -1
- package/dist/{ai-harness-Clbg5ei7.js → ai-harness-dQeJaR1t.js} +7 -3
- package/dist/ai-harness-dQeJaR1t.js.map +1 -0
- package/dist/ai-pi.d.ts +1 -1
- package/dist/ai-pi.js +3 -3
- package/dist/ai-pi.js.map +1 -1
- package/dist/ai-react.d.ts +5 -5
- package/dist/ai-react.js +2 -2
- package/dist/ai-server.d.ts +14 -17
- package/dist/ai-server.d.ts.map +1 -1
- package/dist/ai-server.js +80 -73
- package/dist/ai-server.js.map +1 -1
- package/dist/ai.d.ts +2 -2
- package/dist/ai.js +2 -2
- package/dist/{client-BhHLoNKF.js → client-9IeTsiAs.js} +5 -5
- package/dist/{client-BhHLoNKF.js.map → client-9IeTsiAs.js.map} +1 -1
- package/dist/{client-h_DKFAjj.d.ts → client-CUMbS_38.d.ts} +3 -3
- package/dist/{client-h_DKFAjj.d.ts.map → client-CUMbS_38.d.ts.map} +1 -1
- package/dist/client.d.ts +3 -3
- package/dist/client.js +1 -1
- package/dist/{contract-BYKASlCV.js → contract-CWukt01d.js} +2 -2
- package/dist/contract-CWukt01d.js.map +1 -0
- package/dist/index.d.ts +5 -5
- package/dist/index.js +3 -3
- package/dist/{initial-snapshot-Bx-c0pDP.d.ts → initial-snapshot-D3Lve43o.d.ts} +4 -4
- package/dist/initial-snapshot-D3Lve43o.d.ts.map +1 -0
- package/dist/{internal-CscSY7K_.js → internal-Bc4PvfHk.js} +10 -3
- package/dist/internal-Bc4PvfHk.js.map +1 -0
- package/dist/otel.d.ts +1 -1
- package/dist/otel.js +1 -1
- package/dist/otel.js.map +1 -1
- package/dist/{react-BBXgHWFc.d.ts → react-BoHvC3v9.d.ts} +4 -4
- package/dist/{react-BBXgHWFc.d.ts.map → react-BoHvC3v9.d.ts.map} +1 -1
- package/dist/react.d.ts +1 -1
- package/dist/react.js +1 -1
- package/dist/{reducer-CqA59Q6U.d.ts → reducer-CYPHsSmd.d.ts} +2 -2
- package/dist/{reducer-CqA59Q6U.d.ts.map → reducer-CYPHsSmd.d.ts.map} +1 -1
- package/dist/{reducer-BAXc8JyF.js → reducer-CjmvZSAb.js} +2 -2
- package/dist/{reducer-BAXc8JyF.js.map → reducer-CjmvZSAb.js.map} +1 -1
- package/dist/runtime-consumer-BWmht60z.js +12 -0
- package/dist/runtime-consumer-BWmht60z.js.map +1 -0
- package/dist/scheduler-qstash.d.ts +2 -2
- package/dist/scheduler-qstash.js +3 -3
- package/dist/scheduler-task-X1SToDjP.js.map +1 -1
- package/dist/scheduler-vercel.d.ts +2 -2
- package/dist/scheduler-vercel.js +3 -3
- package/dist/scheduler-vercel.js.map +1 -1
- package/dist/{server-DYAATFuT.d.ts → server-BshFZ0eL.d.ts} +108 -41
- package/dist/server-BshFZ0eL.d.ts.map +1 -0
- package/dist/{server-qyb5aJJG.js → server-DvIB8kj5.js} +387 -113
- package/dist/server-DvIB8kj5.js.map +1 -0
- package/dist/server.browser.js +1 -1
- package/dist/server.browser.js.map +1 -1
- package/dist/server.d.ts +4 -4
- package/dist/server.js +2 -2
- package/dist/{session-clear-C-QwTXV1.d.ts → session-clear-Bk_29eEw.d.ts} +2 -2
- package/dist/session-clear-Bk_29eEw.d.ts.map +1 -0
- package/dist/{session-clear-DGI4jTQ5.js → session-clear-wHd5GCpZ.js} +5 -4
- package/dist/session-clear-wHd5GCpZ.js.map +1 -0
- package/dist/store-N8PXxDAS.js.map +1 -1
- package/dist/store-Z2qjw50n.d.ts +523 -0
- package/dist/store-Z2qjw50n.d.ts.map +1 -0
- package/dist/store-codec-DwrWlgQx.js +65 -0
- package/dist/store-codec-DwrWlgQx.js.map +1 -0
- package/dist/store-memory.d.ts +1 -1
- package/dist/store-memory.d.ts.map +1 -1
- package/dist/store-memory.js +393 -210
- package/dist/store-memory.js.map +1 -1
- package/dist/{store-polling-7WJV0YWP.js → store-polling-DYNyxCev.js} +2 -2
- package/dist/{store-polling-7WJV0YWP.js.map → store-polling-DYNyxCev.js.map} +1 -1
- package/dist/store-postgres.d.ts +1 -1
- package/dist/store-postgres.d.ts.map +1 -1
- package/dist/store-postgres.js +1586 -566
- package/dist/store-postgres.js.map +1 -1
- package/dist/{store-presence-polling-B1ixeqIS.js → store-presence-polling-CzQ0wdtM.js} +3 -3
- package/dist/{store-presence-polling-B1ixeqIS.js.map → store-presence-polling-CzQ0wdtM.js.map} +1 -1
- package/dist/store-redis-http.d.ts +1 -1
- package/dist/store-redis-http.js +4 -4
- package/dist/store-redis-notify-IiVdlYHc.js +2919 -0
- package/dist/store-redis-notify-IiVdlYHc.js.map +1 -0
- package/dist/store-redis.d.ts +1 -1
- package/dist/store-redis.js +2 -2
- package/dist/store-sqlite.d.ts +1 -1
- package/dist/store-sqlite.d.ts.map +1 -1
- package/dist/store-sqlite.js +517 -287
- package/dist/store-sqlite.js.map +1 -1
- package/dist/{telemetry-Cyfqo6Qt.d.ts → telemetry-CWwsh-Gx.d.ts} +3 -3
- package/dist/telemetry-CWwsh-Gx.d.ts.map +1 -0
- package/dist/{wire-BKxH1vbP.js → wire-CZGqNZRx.js} +5 -4
- package/dist/wire-CZGqNZRx.js.map +1 -0
- package/docs/01-quickstart.mdx +21 -15
- package/docs/02-examples.mdx +1 -1
- package/docs/actors/01-introduction.mdx +8 -0
- package/docs/actors/02-concurrency.mdx +1 -1
- package/docs/agents/01-introduction.mdx +1 -1
- package/docs/agents/02-tools.mdx +12 -11
- package/docs/agents/03-conversations.mdx +30 -20
- package/docs/agents/04-harnesses.mdx +7 -3
- package/docs/client/03-presence.mdx +18 -14
- package/docs/contract/01-events.mdx +2 -2
- package/docs/contract/02-state.mdx +18 -14
- package/docs/contract/03-sessions.mdx +32 -25
- package/docs/index.mdx +29 -24
- package/docs/reference/01-core.mdx +4 -4
- package/docs/reference/02-server.mdx +245 -190
- package/docs/reference/04-ai.mdx +20 -19
- package/docs/reference/05-telemetry.mdx +27 -13
- package/docs/reference/06-errors.mdx +24 -18
- package/docs/reference/meta.ts +1 -1
- package/docs/schedulers/01-overview.mdx +8 -8
- package/docs/schedulers/02-vercel-queues.mdx +2 -2
- package/docs/schedulers/03-qstash.mdx +4 -4
- package/docs/server/01-consumers.mdx +415 -0
- package/docs/server/02-timers.mdx +31 -21
- package/docs/server/03-cancellation.mdx +52 -44
- package/docs/server/04-routes.mdx +4 -1
- package/docs/server/05-sql-projections.mdx +391 -0
- package/docs/server/06-guarantees.mdx +55 -48
- package/docs/stores/01-overview.mdx +3 -3
- package/docs/stores/02-redis.mdx +11 -3
- package/docs/stores/03-redis-http.mdx +2 -2
- package/docs/stores/04-postgres.mdx +65 -14
- package/docs/stores/05-sqlite.mdx +5 -3
- package/docs/upgrades/01-overview.mdx +30 -0
- package/docs/upgrades/79-0-21.mdx +165 -0
- package/docs/upgrades/80-0-20.mdx +43 -0
- package/docs/upgrades/81-0-19.mdx +33 -0
- package/docs/upgrades/82-0-18.mdx +33 -0
- package/docs/upgrades/83-0-17.mdx +69 -0
- package/docs/upgrades/meta.ts +3 -0
- package/examples/playground/DEPLOY.md +39 -7
- package/examples/playground/app/a0/chat.test.tsx +2 -2
- package/examples/playground/app/a0/sandbox-log.ts +17 -15
- package/examples/playground/app/a0/server.ts +4 -4
- package/examples/playground/app/a2slack/a2slack.css +1067 -0
- package/examples/playground/app/a2slack/c/[channel]/page.tsx +20 -0
- package/examples/playground/app/a2slack/consumers/agent.ts +63 -0
- package/examples/playground/app/a2slack/consumers/feed.ts +164 -0
- package/examples/playground/app/a2slack/consumers/notify.ts +116 -0
- package/examples/playground/app/a2slack/consumers/search.ts +37 -0
- package/examples/playground/app/a2slack/consumers/triage.ts +31 -0
- package/examples/playground/app/a2slack/consumers/welcome.ts +25 -0
- package/examples/playground/app/a2slack/db/a2.ts +206 -0
- package/examples/playground/app/a2slack/db/index.ts +73 -0
- package/examples/playground/app/a2slack/db/migrations/0000_extensions.sql +4 -0
- package/examples/playground/app/a2slack/db/migrations/0001_workspace.sql +74 -0
- package/examples/playground/app/a2slack/db/migrations/0002_a2.sql +215 -0
- package/examples/playground/app/a2slack/db/migrations/0003_a2_batch.sql +67 -0
- package/examples/playground/app/a2slack/db/migrations/meta/0000_snapshot.json +18 -0
- package/examples/playground/app/a2slack/db/migrations/meta/0001_snapshot.json +500 -0
- package/examples/playground/app/a2slack/db/migrations/meta/0002_snapshot.json +1159 -0
- package/examples/playground/app/a2slack/db/migrations/meta/0003_snapshot.json +1159 -0
- package/examples/playground/app/a2slack/db/migrations/meta/_journal.json +34 -0
- package/examples/playground/app/a2slack/db/queries.ts +263 -0
- package/examples/playground/app/a2slack/db/schema.ts +145 -0
- package/examples/playground/app/a2slack/drills/a2slack.drill.ts +708 -0
- package/examples/playground/app/a2slack/drills/harness.ts +244 -0
- package/examples/playground/app/a2slack/events/route.ts +36 -0
- package/examples/playground/app/a2slack/identity.ts +44 -0
- package/examples/playground/app/a2slack/inbox/route.ts +8 -0
- package/examples/playground/app/a2slack/lag/data/route.ts +8 -0
- package/examples/playground/app/a2slack/lag/page.tsx +101 -0
- package/examples/playground/app/a2slack/lag.ts +73 -0
- package/examples/playground/app/a2slack/layout.tsx +43 -0
- package/examples/playground/app/a2slack/messages/route.ts +17 -0
- package/examples/playground/app/a2slack/model.test.ts +208 -0
- package/examples/playground/app/a2slack/model.ts +369 -0
- package/examples/playground/app/a2slack/models.test.ts +72 -0
- package/examples/playground/app/a2slack/page.tsx +10 -0
- package/examples/playground/app/a2slack/scheduler/route.ts +5 -0
- package/examples/playground/app/a2slack/search/route.ts +40 -0
- package/examples/playground/app/a2slack/search-index.ts +108 -0
- package/examples/playground/app/a2slack/server.test.ts +315 -0
- package/examples/playground/app/a2slack/server.ts +55 -0
- package/examples/playground/app/a2slack/sessions.ts +50 -0
- package/examples/playground/app/a2slack/thread/route.ts +8 -0
- package/examples/playground/app/a2slack/ui/ChannelView.tsx +87 -0
- package/examples/playground/app/a2slack/ui/Composer.tsx +117 -0
- package/examples/playground/app/a2slack/ui/DotsShadow.tsx +42 -0
- package/examples/playground/app/a2slack/ui/Feed.tsx +157 -0
- package/examples/playground/app/a2slack/ui/InboxContext.tsx +63 -0
- package/examples/playground/app/a2slack/ui/InboxView.tsx +82 -0
- package/examples/playground/app/a2slack/ui/Menu.tsx +128 -0
- package/examples/playground/app/a2slack/ui/MessageRow.tsx +314 -0
- package/examples/playground/app/a2slack/ui/NameGate.tsx +35 -0
- package/examples/playground/app/a2slack/ui/SearchContext.tsx +146 -0
- package/examples/playground/app/a2slack/ui/SearchResults.tsx +97 -0
- package/examples/playground/app/a2slack/ui/Shell.tsx +34 -0
- package/examples/playground/app/a2slack/ui/Sidebar.tsx +199 -0
- package/examples/playground/app/a2slack/ui/ThreadPanel.tsx +134 -0
- package/examples/playground/app/a2slack/ui/useInbox.ts +71 -0
- package/examples/playground/app/agent/server.ts +3 -3
- package/examples/playground/app/chat/[chatId]/chat-client.tsx +1 -1
- package/examples/playground/app/chat/server.ts +45 -35
- package/examples/playground/app/components/demo-icons.tsx +7 -0
- package/examples/playground/app/components/demos.tsx +7 -0
- package/examples/playground/app/orders/[orderId]/order-client.tsx +2 -2
- package/examples/playground/app/orders/page.tsx +1 -1
- package/examples/playground/app/orders/server.ts +49 -30
- package/examples/playground/app/recovery/[recoveryId]/recovery-client.tsx +14 -15
- package/examples/playground/app/recovery/page.tsx +4 -4
- package/examples/playground/app/recovery/server.ts +23 -32
- package/examples/playground/drizzle.config.ts +14 -0
- package/examples/playground/next.config.ts +6 -1
- package/examples/playground/package.json +12 -1
- package/examples/playground/scripts/migrate.mjs +31 -0
- package/examples/playground/vercel.json +9 -0
- package/examples/playground/vitest.drills.config.ts +32 -0
- package/package.json +5 -2
- package/sql/postgres/drizzle-schema.ts +206 -0
- package/sql/postgres/schema.sql +206 -67
- package/src/actor.ts +101 -71
- package/src/ai-agent.ts +149 -111
- package/src/ai-control-server.ts +8 -8
- package/src/ai-fx.ts +4 -6
- package/src/ai-harness.ts +10 -2
- package/src/ai-pi.ts +2 -2
- package/src/ai-server.ts +2 -2
- package/src/ai.ts +7 -7
- package/src/contract.ts +1 -1
- package/src/deterministic-id.ts +26 -3
- package/src/errors.ts +8 -5
- package/src/idempotent-replay.ts +0 -1
- package/src/index.ts +1 -1
- package/src/internal.ts +15 -5
- package/src/otel.ts +1 -1
- package/src/postgres-schema.ts +98 -57
- package/src/push-envelope.ts +20 -0
- package/src/runtime-consumer.ts +10 -0
- package/src/scheduler-task.ts +2 -2
- package/src/scheduler-vercel.ts +1 -1
- package/src/server.browser.ts +1 -1
- package/src/server.ts +774 -207
- package/src/session-socket.ts +1 -1
- package/src/store-codec.ts +66 -0
- package/src/store-memory.ts +581 -255
- package/src/store-postgres.ts +1984 -825
- package/src/store-redis-core.ts +1624 -1152
- package/src/store-redis-notify.ts +2 -0
- package/src/store-sqlite.ts +891 -486
- package/src/store.ts +227 -97
- package/src/telemetry.ts +2 -1
- package/src/wire.ts +1 -0
- package/dist/ai-harness-Clbg5ei7.js.map +0 -1
- package/dist/ai-mMiWul1T.js.map +0 -1
- package/dist/contract-BYKASlCV.js.map +0 -1
- package/dist/idempotent-replay-JdgMQRRm.js +0 -21
- package/dist/idempotent-replay-JdgMQRRm.js.map +0 -1
- package/dist/initial-snapshot-Bx-c0pDP.d.ts.map +0 -1
- package/dist/internal-CscSY7K_.js.map +0 -1
- package/dist/server-DYAATFuT.d.ts.map +0 -1
- package/dist/server-qyb5aJJG.js.map +0 -1
- package/dist/session-clear-C-QwTXV1.d.ts.map +0 -1
- package/dist/session-clear-DGI4jTQ5.js.map +0 -1
- package/dist/store-CyLUy86v.d.ts +0 -404
- package/dist/store-CyLUy86v.d.ts.map +0 -1
- package/dist/store-codec-DTG0Ftek.js +0 -10
- package/dist/store-codec-DTG0Ftek.js.map +0 -1
- package/dist/store-redis-notify-CM0AfLY0.js +0 -2544
- package/dist/store-redis-notify-CM0AfLY0.js.map +0 -1
- package/dist/telemetry-Cyfqo6Qt.d.ts.map +0 -1
- package/dist/wire-BKxH1vbP.js.map +0 -1
- package/docs/server/01-handlers.mdx +0 -227
package/src/store-redis-core.ts
CHANGED
|
@@ -5,18 +5,46 @@
|
|
|
5
5
|
* `store-redis-http` (provider REST APIs).
|
|
6
6
|
*
|
|
7
7
|
* The stream IS the store: each event is a stream entry whose ID is its
|
|
8
|
-
* dense a2 index (`${index}-0`), so `XRANGE` is the catch-up read.
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* Lua script.
|
|
12
|
-
* Global event-id ownership is one key per id, so no key grows with
|
|
13
|
-
* the number of sessions.
|
|
14
|
-
* The conformance suite in test/conformance is the executable
|
|
15
|
-
* contract, run against a real spawned `redis-server`.
|
|
8
|
+
* dense a2 index (`${index}-0`), so `XRANGE` is the catch-up read. Stream
|
|
9
|
+
* entries are immutable; dispatch bookkeeping lives on consumer cursors.
|
|
10
|
+
* Every write path is one atomic Lua script.
|
|
16
11
|
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
12
|
+
* The cursor layout, per session `{p}:{s}:`:
|
|
13
|
+
*
|
|
14
|
+
* - `heads:{chunk}` holds one row per (head index, consumer) in hashes of
|
|
15
|
+
* META_CHUNK_SIZE consecutive indexes: a head's attempt and failure
|
|
16
|
+
* fields, its kind (`l`), its lane successor (`next`), its snapshot pins,
|
|
17
|
+
* and, once the cursor moves past it, its completion record. A new head
|
|
18
|
+
* is a new row, so moving a cursor resets its attempt fields for free,
|
|
19
|
+
* and an event cursor costs one row, as a handled event did.
|
|
20
|
+
* - `cursors` is one hash of every lane and log cursor: its head (`h`),
|
|
21
|
+
* position (`p`), lane tail (`t`), or log types (`ty`). A lane's queue is
|
|
22
|
+
* linked through the rows' `next` fields.
|
|
23
|
+
* - `runnable` is a zset of every cursor with a head, scored by the head's
|
|
24
|
+
* index, negated while the head is dead-lettered.
|
|
25
|
+
* - `pins` counts, per snapshot, the heads that pin it.
|
|
26
|
+
*
|
|
27
|
+
* A cursor id names a cursor within its session: the consumer's byte
|
|
28
|
+
* length, `:`, the consumer, then `#{index}` for an event cursor, `L{lane}`
|
|
29
|
+
* for a lane cursor, or `G{lane}` for a log cursor. A clear supersedes every
|
|
30
|
+
* head at or below its marker at once (the `runnable` set is dropped); lane and
|
|
31
|
+
* log cursor records are reset lazily against the marker and swept by
|
|
32
|
+
* reclaim.
|
|
33
|
+
*
|
|
34
|
+
* `{p}:schema` records the layout, and a prefix holding another one is
|
|
35
|
+
* rejected. A session from before cursors (0.17 to 0.20) opens as is: its
|
|
36
|
+
* log, counter, clear marker, snapshots, and ids read the same, and its
|
|
37
|
+
* dispatch keys (`meta`, `meta:{chunk}`, `meta-split`, `pending`, `ready`,
|
|
38
|
+
* `snapshot-pins:{index}`, `snapshot-pin-counts`) share no name with the
|
|
39
|
+
* cursor layout's, so they're never read. Its unfinished work is dropped.
|
|
40
|
+
*
|
|
41
|
+
* Global event-id ownership is one key per id, so no key grows with the
|
|
42
|
+
* number of sessions. The conformance suite in test/conformance is the
|
|
43
|
+
* executable contract, run against a real spawned `redis-server`.
|
|
44
|
+
*
|
|
45
|
+
* Timestamps and claim expiry come from the injected clock so tests can
|
|
46
|
+
* time-travel against a real server. Cluster mode is out: the atomic
|
|
47
|
+
* scripts span keys.
|
|
20
48
|
*/
|
|
21
49
|
|
|
22
50
|
import type { PresencePatch } from './contract.ts'
|
|
@@ -24,7 +52,7 @@ import { A2Error, supersededAttemptError } from './errors.ts'
|
|
|
24
52
|
import { CLEAR_EVENT_TYPE, sessionClearedError } from './session-clear.ts'
|
|
25
53
|
import { idempotentReplay } from './idempotent-replay.ts'
|
|
26
54
|
import { nullProtoRecord } from './internal.ts'
|
|
27
|
-
import {
|
|
55
|
+
import { decodeCause } from './store-codec.ts'
|
|
28
56
|
import type { SerializedEvent } from './stream-buffer.ts'
|
|
29
57
|
import type {
|
|
30
58
|
A2Store,
|
|
@@ -32,8 +60,14 @@ import type {
|
|
|
32
60
|
CompleteAttemptResult,
|
|
33
61
|
Event,
|
|
34
62
|
EventCause,
|
|
63
|
+
EventTarget,
|
|
64
|
+
HeadDispatch,
|
|
65
|
+
HeadEvent,
|
|
66
|
+
HeadRef,
|
|
35
67
|
IdSource,
|
|
36
68
|
StoreClaimAvailableResult,
|
|
69
|
+
StoreConsumer,
|
|
70
|
+
StoredCursor,
|
|
37
71
|
PresenceRow,
|
|
38
72
|
StoredEvent,
|
|
39
73
|
} from './store.ts'
|
|
@@ -81,7 +115,7 @@ export type RedisStoreCore = Omit<A2Store, 'stream' | 'presence'> & {
|
|
|
81
115
|
): Promise<PresencePatch | null>
|
|
82
116
|
read(ns: string): Promise<PresenceRow[]>
|
|
83
117
|
}
|
|
84
|
-
/** XRANGE
|
|
118
|
+
/** XRANGE pages — the full-row catch-up read. */
|
|
85
119
|
readRange(
|
|
86
120
|
sessionId: string,
|
|
87
121
|
afterIndex: number,
|
|
@@ -119,66 +153,28 @@ export const READ_PAGE_SIZE = 256
|
|
|
119
153
|
export const FEED_PAGE_SIZE = 1000
|
|
120
154
|
const READ_CONCURRENCY = 16
|
|
121
155
|
|
|
122
|
-
|
|
156
|
+
/** The key layout this core writes; any other is rejected. */
|
|
157
|
+
export const REDIS_LAYOUT = 'cursors-2'
|
|
158
|
+
const LAYOUT_ERROR = 'A2_LAYOUT'
|
|
123
159
|
|
|
124
160
|
const META_CHUNK_SIZE = 1000
|
|
125
161
|
|
|
126
|
-
/**
|
|
127
|
-
const
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
'ri',
|
|
131
|
-
'ch',
|
|
132
|
-
'ce',
|
|
133
|
-
'f',
|
|
134
|
-
'fc',
|
|
135
|
-
'lc',
|
|
136
|
-
'lf',
|
|
137
|
-
'lfa',
|
|
138
|
-
'd',
|
|
139
|
-
'a',
|
|
140
|
-
'e',
|
|
141
|
-
] as const
|
|
162
|
+
/** A head row's dispatch columns, in the order scripts return them. */
|
|
163
|
+
const HEAD_COLUMNS = `{ 'd', 'fc', 'lc', 'ch', 'ce', 'a', 'lf', 'lfa', 'e', 'f' }`
|
|
164
|
+
|
|
165
|
+
// ── Lua scripts (each runs atomically) ───────────────────────────────
|
|
142
166
|
|
|
143
167
|
/**
|
|
144
|
-
* Upstash bills every call a script makes, so scripts read the counter
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
* means the current event count, so any script that adds events must write
|
|
153
|
-
* `split_write` pairs in its write phase. Including scripts bind COUNTER,
|
|
154
|
-
* META (the base hash), and SPLIT first.
|
|
168
|
+
* Upstash bills every call a script makes, so scripts read the counter,
|
|
169
|
+
* clear marker, layout, typed-type index, and id keys with one MGET
|
|
170
|
+
* (`read_bounds`), read rows with one HMGET per chunk (`read_rows`), and
|
|
171
|
+
* buffer hash writes (`put`, `drop`) into one HSET and one HDEL per hash
|
|
172
|
+
* (`flush`). Buffered writes are invisible to reads until flushed, so a
|
|
173
|
+
* script must not read a field it has buffered. Every script binds KEYS[1]
|
|
174
|
+
* to the session's counter and KEYS[2] to the prefix's layout key, and
|
|
175
|
+
* derives the session's other keys.
|
|
155
176
|
*/
|
|
156
|
-
const
|
|
157
|
-
local split, split_stored = nil, false
|
|
158
|
-
-- The latest clear marker lives beside the counter, {prefix}:{session}:clear,
|
|
159
|
-
-- and is read in the same MGET, so a session never cleared pays nothing.
|
|
160
|
-
local CLEAR = string.sub(COUNTER, 1, -6) .. 'clear'
|
|
161
|
-
local boundary = 0
|
|
162
|
-
local function use_split(stored, count, cleared)
|
|
163
|
-
split_stored = stored ~= false
|
|
164
|
-
split = tonumber(stored or count or '0')
|
|
165
|
-
boundary = tonumber(cleared or '0')
|
|
166
|
-
end
|
|
167
|
-
local function mkey(field)
|
|
168
|
-
if split == nil then
|
|
169
|
-
local values = redis.call('MGET', SPLIT, COUNTER, CLEAR)
|
|
170
|
-
use_split(values[1], values[2], values[3])
|
|
171
|
-
end
|
|
172
|
-
local idx = tonumber(field)
|
|
173
|
-
if idx <= split then return META end
|
|
174
|
-
return META .. ':' .. tostring(math.floor((idx - 1) / ${META_CHUNK_SIZE}))
|
|
175
|
-
end
|
|
176
|
-
local function split_write(args)
|
|
177
|
-
if not split_stored then
|
|
178
|
-
args[#args + 1] = SPLIT
|
|
179
|
-
args[#args + 1] = tostring(split)
|
|
180
|
-
end
|
|
181
|
-
end
|
|
177
|
+
const BASE_LUA = `
|
|
182
178
|
local function sliced(command, key, args)
|
|
183
179
|
local results = {}
|
|
184
180
|
for first = 1, #args, 1000 do
|
|
@@ -195,36 +191,85 @@ local function sliced(command, key, args)
|
|
|
195
191
|
end
|
|
196
192
|
return results
|
|
197
193
|
end
|
|
198
|
-
local
|
|
194
|
+
local COUNTER, SCHEMA = KEYS[1], KEYS[2]
|
|
195
|
+
local SESSION = string.sub(COUNTER, 1, -6)
|
|
196
|
+
local LOG, CLEAR = SESSION .. 'log', SESSION .. 'clear'
|
|
197
|
+
local META, READY, CURSORS = SESSION .. 'heads', SESSION .. 'runnable', SESSION .. 'cursors'
|
|
198
|
+
local PIN_COUNTS, TYPED = SESSION .. 'pins', SESSION .. 'typed'
|
|
199
|
+
local CLEAR_TYPE = '${CLEAR_EVENT_TYPE}'
|
|
200
|
+
local total, boundary, schema_missing = 0, 0, false
|
|
201
|
+
-- The event types typed log cursors read: type → { from, last }. The type's
|
|
202
|
+
-- index holds every event of it above \`from\`, and no event of it lies above
|
|
203
|
+
-- \`last\`.
|
|
204
|
+
local typed, typed_dirty = {}, false
|
|
205
|
+
-- Reads the counter, the latest clear marker, the layout, the typed types,
|
|
206
|
+
-- and the extra keys in one MGET. Returns the extra keys' values, or nil when
|
|
207
|
+
-- the prefix holds another layout.
|
|
208
|
+
local function read_bounds(extra)
|
|
209
|
+
local keys = { COUNTER, CLEAR, SCHEMA, TYPED }
|
|
210
|
+
for _, key in ipairs(extra or {}) do keys[#keys + 1] = key end
|
|
211
|
+
local values = sliced('MGET', nil, keys)
|
|
212
|
+
if values[3] and values[3] ~= '${REDIS_LAYOUT}' then return nil end
|
|
213
|
+
total = tonumber(values[1] or '0')
|
|
214
|
+
boundary = tonumber(values[2] or '0')
|
|
215
|
+
schema_missing = not values[3]
|
|
216
|
+
if values[4] then typed = cjson.decode(values[4]) end
|
|
217
|
+
local rest = {}
|
|
218
|
+
for i = 5, #keys do rest[i - 4] = values[i] end
|
|
219
|
+
return rest
|
|
220
|
+
end
|
|
221
|
+
local function entry_index(id) return tonumber(string.match(id, '^(%d+)')) end
|
|
222
|
+
local function cid_of(consumer, tag)
|
|
223
|
+
return tostring(string.len(consumer)) .. ':' .. consumer .. tag
|
|
224
|
+
end
|
|
225
|
+
local function parse_cid(cid)
|
|
226
|
+
local colon = string.find(cid, ':', 1, true)
|
|
227
|
+
local n = tonumber(string.sub(cid, 1, colon - 1))
|
|
228
|
+
return string.sub(cid, colon + 1, colon + n), string.sub(cid, colon + n + 1)
|
|
229
|
+
end
|
|
230
|
+
-- A cursor's rows: its consumer, then '#' for an event cursor or 'L' .. lane
|
|
231
|
+
-- for a lane or log cursor, as requests name it.
|
|
232
|
+
local function row_of(consumer, tag)
|
|
233
|
+
local kind = string.sub(tag, 1, 1)
|
|
234
|
+
if kind == '#' then return cid_of(consumer, '#') end
|
|
235
|
+
if kind == 'G' then return cid_of(consumer, 'L' .. string.sub(tag, 2)) end
|
|
236
|
+
return cid_of(consumer, tag)
|
|
237
|
+
end
|
|
238
|
+
local function chunk_key(idx)
|
|
239
|
+
return META .. ':' .. tostring(math.floor((idx - 1) / ${META_CHUNK_SIZE}))
|
|
240
|
+
end
|
|
241
|
+
local function row_name(idx, row, suffix)
|
|
242
|
+
return tostring(idx) .. ':' .. row .. ':' .. suffix
|
|
243
|
+
end
|
|
244
|
+
-- Rows of (index, row) refs, aligned with refs; one HMGET per chunk.
|
|
245
|
+
local function read_rows(refs, suffixes)
|
|
199
246
|
local groups, order = {}, {}
|
|
200
|
-
for
|
|
201
|
-
local key =
|
|
247
|
+
for position, ref in ipairs(refs) do
|
|
248
|
+
local key = chunk_key(ref[1])
|
|
202
249
|
local group = groups[key]
|
|
203
250
|
if not group then
|
|
204
|
-
group = {
|
|
251
|
+
group = { positions = {}, names = {} }
|
|
205
252
|
groups[key] = group
|
|
206
253
|
order[#order + 1] = key
|
|
207
254
|
end
|
|
208
|
-
group.
|
|
255
|
+
group.positions[#group.positions + 1] = position
|
|
209
256
|
for _, suffix in ipairs(suffixes) do
|
|
210
|
-
group.names[#group.names + 1] =
|
|
257
|
+
group.names[#group.names + 1] = row_name(ref[1], ref[2], suffix)
|
|
211
258
|
end
|
|
212
259
|
end
|
|
213
260
|
local rows = {}
|
|
214
261
|
for _, key in ipairs(order) do
|
|
215
262
|
local group = groups[key]
|
|
216
263
|
local values = sliced('HMGET', key, group.names)
|
|
217
|
-
for
|
|
264
|
+
for n, position in ipairs(group.positions) do
|
|
218
265
|
local row = {}
|
|
219
|
-
for column = 1, #suffixes do
|
|
220
|
-
|
|
221
|
-
end
|
|
222
|
-
rows[field] = row
|
|
266
|
+
for column = 1, #suffixes do row[column] = values[(n - 1) * #suffixes + column] end
|
|
267
|
+
rows[position] = row
|
|
223
268
|
end
|
|
224
269
|
end
|
|
225
270
|
return rows
|
|
226
271
|
end
|
|
227
|
-
local writes, write_order = {}, {}
|
|
272
|
+
local writes, write_order, deletes, delete_order = {}, {}, {}, {}
|
|
228
273
|
local function put(key, field, value)
|
|
229
274
|
local fields = writes[key]
|
|
230
275
|
if not fields then
|
|
@@ -235,115 +280,401 @@ local function put(key, field, value)
|
|
|
235
280
|
fields[#fields + 1] = field
|
|
236
281
|
fields[#fields + 1] = value
|
|
237
282
|
end
|
|
283
|
+
local function drop(key, field)
|
|
284
|
+
local fields = deletes[key]
|
|
285
|
+
if not fields then
|
|
286
|
+
fields = {}
|
|
287
|
+
deletes[key] = fields
|
|
288
|
+
delete_order[#delete_order + 1] = key
|
|
289
|
+
end
|
|
290
|
+
fields[#fields + 1] = field
|
|
291
|
+
end
|
|
238
292
|
local function flush()
|
|
239
293
|
for _, key in ipairs(write_order) do sliced('HSET', key, writes[key]) end
|
|
240
|
-
|
|
294
|
+
for _, key in ipairs(delete_order) do sliced('HDEL', key, deletes[key]) end
|
|
295
|
+
writes, write_order, deletes, delete_order = {}, {}, {}, {}
|
|
296
|
+
end
|
|
297
|
+
local function put_row(idx, row, suffix, value)
|
|
298
|
+
put(chunk_key(idx), row_name(idx, row, suffix), value)
|
|
299
|
+
end
|
|
300
|
+
local function drop_row(idx, row, suffix)
|
|
301
|
+
drop(chunk_key(idx), row_name(idx, row, suffix))
|
|
302
|
+
end
|
|
303
|
+
-- A request names a cursor by tag: '#' for an event cursor, 'L' .. lane for a
|
|
304
|
+
-- lane or log cursor. A claimed row records its kind: '#', or the cursor's tag.
|
|
305
|
+
local function owns(kind, tag)
|
|
306
|
+
if not kind then return false end
|
|
307
|
+
if tag == '#' then return kind == '#' end
|
|
308
|
+
return kind == tag or kind == 'G' .. string.sub(tag, 2)
|
|
309
|
+
end
|
|
310
|
+
local function cid_for(consumer, kind, idx)
|
|
311
|
+
if kind == '#' then return cid_of(consumer, '#' .. tostring(idx)) end
|
|
312
|
+
return cid_of(consumer, kind)
|
|
313
|
+
end
|
|
314
|
+
-- One type's index, named by the type's bytes in hex so no type can spell
|
|
315
|
+
-- another key of the session.
|
|
316
|
+
local function type_key(name)
|
|
317
|
+
return SESSION .. 'type:' .. (string.gsub(name, '.', function(c)
|
|
318
|
+
return string.format('%02x', string.byte(c))
|
|
319
|
+
end))
|
|
320
|
+
end
|
|
321
|
+
local ready_changes, ready_order = {}, {}
|
|
322
|
+
local function ready_set(cid, score)
|
|
323
|
+
if ready_changes[cid] == nil then ready_order[#ready_order + 1] = cid end
|
|
324
|
+
ready_changes[cid] = score
|
|
325
|
+
end
|
|
326
|
+
local function flush_ready()
|
|
327
|
+
local adds, removes = {}, {}
|
|
328
|
+
for _, cid in ipairs(ready_order) do
|
|
329
|
+
local score = ready_changes[cid]
|
|
330
|
+
if score then
|
|
331
|
+
adds[#adds + 1] = score
|
|
332
|
+
adds[#adds + 1] = cid
|
|
333
|
+
else
|
|
334
|
+
removes[#removes + 1] = cid
|
|
335
|
+
end
|
|
336
|
+
end
|
|
337
|
+
sliced('ZADD', READY, adds)
|
|
338
|
+
sliced('ZREM', READY, removes)
|
|
339
|
+
ready_changes, ready_order = {}, {}
|
|
340
|
+
end
|
|
341
|
+
local function release_pins(idx, row, pins)
|
|
342
|
+
if not pins or pins == '' then return end
|
|
343
|
+
for _, ref in ipairs(cjson.decode(pins)) do
|
|
344
|
+
local count = redis.call('HINCRBY', PIN_COUNTS, ref, -1)
|
|
345
|
+
if count <= 0 then
|
|
346
|
+
redis.call('HDEL', PIN_COUNTS, ref)
|
|
347
|
+
local checkpoint = cjson.decode(ref)
|
|
348
|
+
redis.call('ZREMRANGEBYSCORE', checkpoint[1], checkpoint[2], checkpoint[2])
|
|
349
|
+
end
|
|
350
|
+
end
|
|
351
|
+
put_row(idx, row, 'pins', '')
|
|
241
352
|
end
|
|
242
353
|
`
|
|
243
354
|
|
|
244
355
|
/**
|
|
245
|
-
*
|
|
246
|
-
*
|
|
247
|
-
*
|
|
248
|
-
*
|
|
249
|
-
*
|
|
356
|
+
* Cursor movement for scripts that add events or move cursors. Lane and log
|
|
357
|
+
* cursor records are read once per script (`load_states`) and written back
|
|
358
|
+
* once (`flush_states`). A record whose head lies at or below the latest clear
|
|
359
|
+
* marker is idle, and a log cursor counts from the marker.
|
|
360
|
+
*
|
|
361
|
+
* A typed log cursor finds its next head in its types' indexes, one bounded
|
|
362
|
+
* lookup per type, skipped when `typed` shows no newer event of it. When an
|
|
363
|
+
* index does not reach back far enough yet (a typed backfill, or a type a
|
|
364
|
+
* cursor newly reads), the lookup returns the type, and the script returns
|
|
365
|
+
* `{ 'index', type, after }` before writing anything: the caller indexes
|
|
366
|
+
* that type down to `after` (INDEX_TYPE_LUA, one page per call) and runs the
|
|
367
|
+
* operation again.
|
|
250
368
|
*/
|
|
251
|
-
const
|
|
252
|
-
local
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
local
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
369
|
+
const CURSOR_LUA = `
|
|
370
|
+
local served_all, served_set, served_logs = true, {}, {}
|
|
371
|
+
local pending_fast = false
|
|
372
|
+
local function types_of(raw)
|
|
373
|
+
if not raw or raw == '*' then return nil end
|
|
374
|
+
local set = {}
|
|
375
|
+
for _, name in ipairs(cjson.decode(raw)) do set[name] = true end
|
|
376
|
+
return set
|
|
377
|
+
end
|
|
378
|
+
local function takes(types, kind)
|
|
379
|
+
return kind ~= CLEAR_TYPE and (types == nil or types[kind] == true)
|
|
380
|
+
end
|
|
381
|
+
-- The served consumers: '' for every consumer, else [[name] or
|
|
382
|
+
-- [name, lane, typesJson or '*', backfill]].
|
|
383
|
+
local function use_served(raw)
|
|
384
|
+
if raw == '' then return end
|
|
385
|
+
served_all = false
|
|
386
|
+
for _, entry in ipairs(cjson.decode(raw)) do
|
|
387
|
+
served_set[entry[1]] = true
|
|
388
|
+
if entry[2] then
|
|
389
|
+
served_logs[#served_logs + 1] = {
|
|
390
|
+
name = entry[1], lane = entry[2], ty = entry[3],
|
|
391
|
+
types = types_of(entry[3]), backfill = entry[4] == 1,
|
|
392
|
+
}
|
|
393
|
+
end
|
|
264
394
|
end
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
395
|
+
end
|
|
396
|
+
local function served_has(name) return served_all or served_set[name] == true end
|
|
397
|
+
-- Whether a served consumer has a live head. Newest heads come first: a
|
|
398
|
+
-- removed consumer's backlog is older than the work a deployment serves.
|
|
399
|
+
local function has_pending()
|
|
400
|
+
if pending_fast then return '1' end
|
|
401
|
+
if served_all then return redis.call('ZCOUNT', READY, 1, '+inf') > 0 and '1' or '0' end
|
|
402
|
+
if next(served_set) == nil then return '0' end
|
|
403
|
+
local page = redis.call('ZREVRANGEBYSCORE', READY, '+inf', 1, 'LIMIT', 0, 32)
|
|
404
|
+
for _, cid in ipairs(page) do
|
|
405
|
+
if served_set[(parse_cid(cid))] then return '1' end
|
|
271
406
|
end
|
|
272
|
-
if
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
407
|
+
if #page < 32 then return '0' end
|
|
408
|
+
for _, cid in ipairs(redis.call('ZREVRANGEBYSCORE', READY, '+inf', 1, 'LIMIT', 32, -1)) do
|
|
409
|
+
if served_set[(parse_cid(cid))] then return '1' end
|
|
410
|
+
end
|
|
411
|
+
return '0'
|
|
412
|
+
end
|
|
413
|
+
local function targets_of(raw)
|
|
414
|
+
if raw == '' then return {} end
|
|
415
|
+
return cjson.decode(raw)
|
|
416
|
+
end
|
|
417
|
+
-- Starts indexing a type: every later event of it.
|
|
418
|
+
local function register(name, from)
|
|
419
|
+
typed[name] = { from, from }
|
|
420
|
+
typed_dirty = true
|
|
421
|
+
end
|
|
422
|
+
-- The first stored event in (after, total] of a type in \`types\` (nil: any
|
|
423
|
+
-- type), or false. Nil and a type when that type's index can't answer yet.
|
|
424
|
+
local function stored_head(after, types)
|
|
425
|
+
if types == nil then
|
|
426
|
+
if after + 1 <= total then return after + 1 end
|
|
427
|
+
return false
|
|
428
|
+
end
|
|
429
|
+
local best = false
|
|
430
|
+
for name in pairs(types) do
|
|
431
|
+
local entry = typed[name]
|
|
432
|
+
if entry == nil then
|
|
433
|
+
if after < total then return nil, name end
|
|
434
|
+
register(name, total)
|
|
435
|
+
elseif entry[2] > after then
|
|
436
|
+
if entry[1] > after then return nil, name end
|
|
437
|
+
local found = redis.call('ZRANGEBYSCORE', type_key(name),
|
|
438
|
+
'(' .. tostring(after), tostring(total), 'LIMIT', 0, 1)
|
|
439
|
+
if #found > 0 then
|
|
440
|
+
local idx = tonumber(found[1])
|
|
441
|
+
if not best or idx < best then best = idx end
|
|
442
|
+
end
|
|
443
|
+
end
|
|
444
|
+
end
|
|
445
|
+
return best
|
|
446
|
+
end
|
|
447
|
+
-- The first event after \`after\` a log cursor takes: stored, then in the
|
|
448
|
+
-- batch being appended.
|
|
449
|
+
local function next_log_head(after, types, batch)
|
|
450
|
+
local head, missing = stored_head(after, types)
|
|
451
|
+
if head == nil then return nil, missing end
|
|
452
|
+
if head then return head end
|
|
453
|
+
for _, event in ipairs(batch) do
|
|
454
|
+
if event.idx > after and takes(types, event.type) then return event.idx end
|
|
455
|
+
end
|
|
456
|
+
return false
|
|
457
|
+
end
|
|
458
|
+
-- Indexes a batch's events of the types typed log cursors read.
|
|
459
|
+
local function index_batch(batch)
|
|
460
|
+
local adds, order = {}, {}
|
|
461
|
+
for _, event in ipairs(batch) do
|
|
462
|
+
local entry = typed[event.type]
|
|
463
|
+
if entry then
|
|
464
|
+
local list = adds[event.type]
|
|
465
|
+
if not list then
|
|
466
|
+
list = {}
|
|
467
|
+
adds[event.type] = list
|
|
468
|
+
order[#order + 1] = event.type
|
|
469
|
+
end
|
|
470
|
+
list[#list + 1] = event.idx
|
|
471
|
+
list[#list + 1] = tostring(event.idx)
|
|
472
|
+
entry[2] = event.idx
|
|
473
|
+
typed_dirty = true
|
|
474
|
+
end
|
|
475
|
+
end
|
|
476
|
+
for _, name in ipairs(order) do sliced('ZADD', type_key(name), adds[name]) end
|
|
477
|
+
end
|
|
478
|
+
local states, state_order = {}, {}
|
|
479
|
+
local function load_states(cids)
|
|
480
|
+
local need, names = {}, {}
|
|
481
|
+
for _, cid in ipairs(cids) do
|
|
482
|
+
local state = states[cid]
|
|
483
|
+
if (state == nil or state.partial) and not need[cid] then
|
|
484
|
+
need[cid] = true
|
|
485
|
+
need[#need + 1] = cid
|
|
486
|
+
names[#names + 1] = cid .. ':h'
|
|
487
|
+
names[#names + 1] = cid .. ':p'
|
|
488
|
+
names[#names + 1] = cid .. ':t'
|
|
489
|
+
names[#names + 1] = cid .. ':ty'
|
|
490
|
+
end
|
|
491
|
+
end
|
|
492
|
+
if #need == 0 then return end
|
|
493
|
+
local values = sliced('HMGET', CURSORS, names)
|
|
494
|
+
for n, cid in ipairs(need) do
|
|
495
|
+
local h, p, t, ty = values[n * 4 - 3], values[n * 4 - 2], values[n * 4 - 1], values[n * 4]
|
|
496
|
+
local tail = (t and t ~= '') and tonumber(t) or false
|
|
497
|
+
local state = states[cid]
|
|
498
|
+
if state then
|
|
499
|
+
state.t = tail
|
|
500
|
+
state.partial = nil
|
|
501
|
+
else
|
|
502
|
+
local _, tag = parse_cid(cid)
|
|
503
|
+
state = {
|
|
504
|
+
kind = string.sub(tag, 1, 1), exists = p ~= false,
|
|
505
|
+
h = (h and h ~= '') and tonumber(h) or false,
|
|
506
|
+
p = tonumber(p or '0'), t = tail, ty = ty or false,
|
|
507
|
+
}
|
|
508
|
+
if state.h and state.h <= boundary then state.h = false end
|
|
509
|
+
if state.kind == 'G' and state.exists and state.p < boundary then state.p = boundary end
|
|
510
|
+
states[cid] = state
|
|
511
|
+
state_order[#state_order + 1] = cid
|
|
512
|
+
end
|
|
278
513
|
end
|
|
279
|
-
lane_tails[lane] = field
|
|
280
514
|
end
|
|
281
|
-
local function
|
|
282
|
-
|
|
283
|
-
|
|
515
|
+
local function flush_states()
|
|
516
|
+
for _, cid in ipairs(state_order) do
|
|
517
|
+
local state = states[cid]
|
|
518
|
+
if state.dirty then
|
|
519
|
+
put(CURSORS, cid .. ':h', state.h and tostring(state.h) or '')
|
|
520
|
+
put(CURSORS, cid .. ':p', tostring(state.p))
|
|
521
|
+
if state.kind == 'L' then
|
|
522
|
+
if not state.partial then put(CURSORS, cid .. ':t', state.t and tostring(state.t) or '') end
|
|
523
|
+
elseif state.ty then
|
|
524
|
+
put(CURSORS, cid .. ':ty', state.ty)
|
|
525
|
+
end
|
|
526
|
+
end
|
|
527
|
+
end
|
|
284
528
|
end
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
529
|
+
-- Moves a cursor past its head \`idx\`. \`kind\` is the head row's kind;
|
|
530
|
+
-- \`batch\` is the returned events, appended after the stored log. Writes
|
|
531
|
+
-- nothing; returns nil and a type when that type's index must reach back
|
|
532
|
+
-- to \`idx\` first.
|
|
533
|
+
local function advance(consumer, kind, idx, row_next, batch)
|
|
534
|
+
if kind == '#' then
|
|
535
|
+
ready_set(cid_of(consumer, '#' .. tostring(idx)), false)
|
|
536
|
+
return true
|
|
537
|
+
end
|
|
538
|
+
local cid = cid_of(consumer, kind)
|
|
539
|
+
local state = states[cid]
|
|
540
|
+
local next_head
|
|
541
|
+
if string.sub(kind, 1, 1) == 'L' then
|
|
542
|
+
next_head = row_next and tonumber(row_next) or false
|
|
543
|
+
if not state then
|
|
544
|
+
state = { kind = 'L', exists = true, p = idx, h = false, t = false, ty = false, partial = true }
|
|
545
|
+
states[cid] = state
|
|
546
|
+
state_order[#state_order + 1] = cid
|
|
547
|
+
end
|
|
548
|
+
if not next_head then
|
|
549
|
+
state.t = false
|
|
550
|
+
state.partial = nil
|
|
551
|
+
end
|
|
552
|
+
else
|
|
553
|
+
local lane, raw = string.sub(kind, 2), nil
|
|
554
|
+
for _, spec in ipairs(served_logs) do
|
|
555
|
+
if spec.name == consumer and spec.lane == lane then raw = spec.ty end
|
|
556
|
+
end
|
|
557
|
+
if not raw then
|
|
558
|
+
load_states({ cid })
|
|
559
|
+
raw = states[cid].ty or '*'
|
|
291
560
|
end
|
|
561
|
+
local head, missing = next_log_head(idx, types_of(raw), batch)
|
|
562
|
+
if head == nil then return nil, missing end
|
|
563
|
+
next_head = head
|
|
564
|
+
state = states[cid]
|
|
565
|
+
if not state then
|
|
566
|
+
state = { kind = 'G', exists = true, p = idx, h = false, t = false, ty = raw }
|
|
567
|
+
states[cid] = state
|
|
568
|
+
state_order[#state_order + 1] = cid
|
|
569
|
+
end
|
|
570
|
+
state.ty = raw
|
|
292
571
|
end
|
|
293
|
-
|
|
294
|
-
|
|
572
|
+
state.h = next_head
|
|
573
|
+
state.p = idx
|
|
574
|
+
state.exists = true
|
|
575
|
+
state.dirty = true
|
|
576
|
+
ready_set(cid, next_head)
|
|
577
|
+
return true
|
|
295
578
|
end
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
redis.call('ZADD', READY, tonumber(next_field), next_field)
|
|
579
|
+
-- Reads what an appended batch's cursor work needs: its lane targets'
|
|
580
|
+
-- records, and the head of each served log consumer whose types it carries.
|
|
581
|
+
-- Writes nothing; returns nil, a type, and an index when that type's index
|
|
582
|
+
-- must reach back to the index first.
|
|
583
|
+
local log_plan = {}
|
|
584
|
+
local function plan(batch, base)
|
|
585
|
+
local need = {}
|
|
586
|
+
for _, event in ipairs(batch) do
|
|
587
|
+
for _, target in ipairs(event.targets) do
|
|
588
|
+
if target[2] ~= '#' then need[#need + 1] = cid_of(target[1], target[2]) end
|
|
589
|
+
end
|
|
590
|
+
end
|
|
591
|
+
for _, spec in ipairs(served_logs) do
|
|
592
|
+
for _, event in ipairs(batch) do
|
|
593
|
+
if takes(spec.types, event.type) then
|
|
594
|
+
local cid = cid_of(spec.name, 'G' .. spec.lane)
|
|
595
|
+
log_plan[#log_plan + 1] = { spec = spec, first = event.idx, cid = cid }
|
|
596
|
+
need[#need + 1] = cid
|
|
597
|
+
break
|
|
316
598
|
end
|
|
317
|
-
return
|
|
318
599
|
end
|
|
319
|
-
next_field = row[3]
|
|
320
600
|
end
|
|
321
|
-
|
|
601
|
+
load_states(need)
|
|
602
|
+
for _, log in ipairs(log_plan) do
|
|
603
|
+
local state, spec = states[log.cid], log.spec
|
|
604
|
+
if not state.exists then
|
|
605
|
+
if spec.backfill then
|
|
606
|
+
local head, missing = stored_head(boundary, spec.types)
|
|
607
|
+
if head == nil then return nil, missing, boundary end
|
|
608
|
+
log.p, log.h = boundary, head or log.first
|
|
609
|
+
else
|
|
610
|
+
-- Nothing it takes is stored after its start: index its types from there.
|
|
611
|
+
for name in pairs(spec.types or {}) do
|
|
612
|
+
if typed[name] == nil then register(name, base) end
|
|
613
|
+
end
|
|
614
|
+
log.p, log.h = base, log.first
|
|
615
|
+
end
|
|
616
|
+
elseif not state.h then
|
|
617
|
+
-- Not the batch's first: a deployment that doesn't serve this consumer
|
|
618
|
+
-- may have appended events it takes since it went idle.
|
|
619
|
+
local head, missing = stored_head(state.p, spec.types)
|
|
620
|
+
if head == nil then return nil, missing, state.p end
|
|
621
|
+
log.h = head or log.first
|
|
622
|
+
end
|
|
623
|
+
end
|
|
624
|
+
return true
|
|
322
625
|
end
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
626
|
+
-- Starts the cursor work \`plan\` read: each event's targets, and each
|
|
627
|
+
-- planned log cursor.
|
|
628
|
+
local function enqueue(batch)
|
|
629
|
+
for _, event in ipairs(batch) do
|
|
630
|
+
for _, target in ipairs(event.targets) do
|
|
631
|
+
local consumer, tag = target[1], target[2]
|
|
632
|
+
if served_has(consumer) then pending_fast = true end
|
|
633
|
+
if tag == '#' then
|
|
634
|
+
ready_set(cid_of(consumer, '#' .. tostring(event.idx)), event.idx)
|
|
635
|
+
else
|
|
636
|
+
local cid = cid_of(consumer, tag)
|
|
637
|
+
local state = states[cid]
|
|
638
|
+
if state.h and state.t then
|
|
639
|
+
put_row(state.t, row_of(consumer, tag), 'next', tostring(event.idx))
|
|
640
|
+
else
|
|
641
|
+
state.h = event.idx
|
|
642
|
+
if state.p < boundary then state.p = boundary end
|
|
643
|
+
ready_set(cid, event.idx)
|
|
644
|
+
end
|
|
645
|
+
state.t = event.idx
|
|
646
|
+
state.exists = true
|
|
647
|
+
state.dirty = true
|
|
648
|
+
end
|
|
649
|
+
end
|
|
650
|
+
end
|
|
651
|
+
for _, log in ipairs(log_plan) do
|
|
652
|
+
pending_fast = true
|
|
653
|
+
local state = states[log.cid]
|
|
654
|
+
if not state.exists then
|
|
655
|
+
state.exists, state.p, state.h = true, log.p, log.h
|
|
656
|
+
ready_set(log.cid, log.h)
|
|
657
|
+
elseif not state.h then
|
|
658
|
+
state.h = log.h
|
|
659
|
+
ready_set(log.cid, log.h)
|
|
660
|
+
end
|
|
661
|
+
state.ty = log.spec.ty
|
|
662
|
+
state.dirty = true
|
|
663
|
+
end
|
|
664
|
+
end
|
|
665
|
+
-- Event-id owners for the extra keys read_bounds returned. A record this
|
|
666
|
+
-- session made below its latest clear is free.
|
|
667
|
+
local function owners_of(values, session)
|
|
337
668
|
local owners = {}
|
|
338
|
-
for i = 1,
|
|
339
|
-
local owner = values[
|
|
669
|
+
for i = 1, #values do
|
|
670
|
+
local owner = values[i]
|
|
340
671
|
if owner then
|
|
341
672
|
local rec = cjson.decode(owner)
|
|
342
673
|
if rec[1] == session and tonumber(rec[2]) < boundary then owner = false end
|
|
343
674
|
end
|
|
344
675
|
owners[i] = owner
|
|
345
676
|
end
|
|
346
|
-
return owners
|
|
677
|
+
return owners
|
|
347
678
|
end
|
|
348
679
|
`
|
|
349
680
|
|
|
@@ -381,51 +712,30 @@ local function wake(channel, frontier, created_at)
|
|
|
381
712
|
end
|
|
382
713
|
`
|
|
383
714
|
|
|
384
|
-
const RELEASE_PINS_LUA = `
|
|
385
|
-
local function release_snapshot_pins(pins, counts)
|
|
386
|
-
local refs = redis.call('SMEMBERS', pins)
|
|
387
|
-
if #refs == 0 then return end
|
|
388
|
-
for _, ref in ipairs(refs) do
|
|
389
|
-
local count = redis.call('HINCRBY', counts, ref, -1)
|
|
390
|
-
if count <= 0 then
|
|
391
|
-
redis.call('HDEL', counts, ref)
|
|
392
|
-
local checkpoint = cjson.decode(ref)
|
|
393
|
-
redis.call('ZREMRANGEBYSCORE', checkpoint[1], checkpoint[2], checkpoint[2])
|
|
394
|
-
end
|
|
395
|
-
end
|
|
396
|
-
redis.call('DEL', pins)
|
|
397
|
-
end
|
|
398
|
-
`
|
|
399
|
-
|
|
400
715
|
/**
|
|
401
|
-
* KEYS:
|
|
402
|
-
*
|
|
403
|
-
*
|
|
404
|
-
* id, type, payloadJson, causeJson, lane, settled; then notify channel and
|
|
405
|
-
* origin index or ''.
|
|
716
|
+
* KEYS: counter, layout, then one id ownership key per event.
|
|
717
|
+
* ARGV: sessionId, nowMs, n, served consumers, notify channel, origin index
|
|
718
|
+
* or '', then per event: id, type, payloadJson, causeJson or '', targets.
|
|
406
719
|
* Returns ['empty', hasPending] | ['ok', firstIndex, hasPending] |
|
|
407
720
|
* ['dup', hasPending, ...indexes] | ['partial', count] |
|
|
408
|
-
* ['foreign', eventId] | ['
|
|
409
|
-
* ['
|
|
721
|
+
* ['foreign', eventId] | ['superseded', position] | ['cleared', originIndex] |
|
|
722
|
+
* ['index', type, after].
|
|
410
723
|
*/
|
|
411
|
-
const APPEND_LUA =
|
|
412
|
-
local
|
|
413
|
-
|
|
414
|
-
${ENQUEUE_LUA}
|
|
415
|
-
${ID_OWNERS_LUA}
|
|
416
|
-
${WAKE_LUA}
|
|
417
|
-
local function has_pending()
|
|
418
|
-
return redis.call('ZCARD', PENDING) > 0 and '1' or '0'
|
|
419
|
-
end
|
|
420
|
-
local n = tonumber(ARGV[3])
|
|
724
|
+
const APPEND_LUA = `${BASE_LUA}${CURSOR_LUA}${WAKE_LUA}
|
|
725
|
+
local session_id, now, n = ARGV[1], ARGV[2], tonumber(ARGV[3])
|
|
726
|
+
use_served(ARGV[4])
|
|
421
727
|
if n == 0 then return { 'empty', has_pending() } end
|
|
422
|
-
local
|
|
423
|
-
|
|
728
|
+
local id_keys = {}
|
|
729
|
+
for i = 1, n do id_keys[i] = KEYS[2 + i] end
|
|
730
|
+
local values = read_bounds(id_keys)
|
|
731
|
+
if not values then return redis.error_reply('${LAYOUT_ERROR}') end
|
|
732
|
+
local owners = owners_of(values, session_id)
|
|
733
|
+
local base = total
|
|
424
734
|
local dups = {}
|
|
425
735
|
for i = 1, n do
|
|
426
736
|
if owners[i] then
|
|
427
737
|
local rec = cjson.decode(owners[i])
|
|
428
|
-
if rec[1] ~=
|
|
738
|
+
if rec[1] ~= session_id then return { 'foreign', ARGV[6 + (i - 1) * 5 + 1] } end
|
|
429
739
|
dups[#dups + 1] = tostring(rec[2])
|
|
430
740
|
end
|
|
431
741
|
end
|
|
@@ -435,253 +745,254 @@ if #dups == n then
|
|
|
435
745
|
return res
|
|
436
746
|
end
|
|
437
747
|
if #dups > 0 then return { 'partial', tostring(#dups) } end
|
|
438
|
-
local origin = ARGV[
|
|
439
|
-
if origin
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
local
|
|
748
|
+
local origin = ARGV[6]
|
|
749
|
+
if origin ~= '' and tonumber(origin) <= boundary then return { 'cleared', origin } end
|
|
750
|
+
-- The attempt-currency fence: a cause's head row must still be its cursor's
|
|
751
|
+
-- head, at that attempt, not dead-lettered.
|
|
752
|
+
local refs, fenced = {}, {}
|
|
443
753
|
for i = 1, n do
|
|
444
|
-
local
|
|
445
|
-
if
|
|
446
|
-
local
|
|
447
|
-
local
|
|
448
|
-
|
|
449
|
-
|
|
754
|
+
local raw = ARGV[6 + (i - 1) * 5 + 4]
|
|
755
|
+
if raw ~= '' then
|
|
756
|
+
local cause = cjson.decode(raw)
|
|
757
|
+
local index = tonumber(cause.index)
|
|
758
|
+
if index <= boundary then return { 'superseded', tostring(i) } end
|
|
759
|
+
local tag = type(cause.lane) == 'string' and ('L' .. cause.lane) or '#'
|
|
760
|
+
refs[#refs + 1] = { index, row_of(cause.consumer, tag) }
|
|
761
|
+
fenced[#fenced + 1] = {
|
|
762
|
+
position = i, consumer = cause.consumer, attempt = tonumber(cause.attempt), tag = tag,
|
|
763
|
+
}
|
|
450
764
|
end
|
|
451
765
|
end
|
|
452
|
-
local
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
if pidx < 1 or pidx > base then
|
|
458
|
-
return { 'no_parent', tostring(parsed.index) }
|
|
766
|
+
local rows = read_rows(refs, { 'd', 'f', 'p', 'l' })
|
|
767
|
+
for k, cause in ipairs(fenced) do
|
|
768
|
+
local row = rows[k]
|
|
769
|
+
if tonumber(row[1] or '0') ~= cause.attempt or row[2] or row[3] or not owns(row[4], cause.tag) then
|
|
770
|
+
return { 'superseded', tostring(cause.position) }
|
|
459
771
|
end
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
772
|
+
if served_has(cause.consumer) then pending_fast = true end
|
|
773
|
+
end
|
|
774
|
+
local batch = {}
|
|
775
|
+
for i = 1, n do
|
|
776
|
+
local o = 6 + (i - 1) * 5
|
|
777
|
+
batch[i] = { idx = base + i, type = ARGV[o + 2], targets = targets_of(ARGV[o + 5]) }
|
|
466
778
|
end
|
|
779
|
+
local planned, missing, after = plan(batch, base)
|
|
780
|
+
if not planned then return { 'index', missing, tostring(after) } end
|
|
467
781
|
local owner_writes = {}
|
|
468
782
|
for i = 1, n do
|
|
469
|
-
local o =
|
|
783
|
+
local o = 6 + (i - 1) * 5
|
|
470
784
|
local field = tostring(base + i)
|
|
471
|
-
redis.call('XADD',
|
|
785
|
+
redis.call('XADD', LOG, field .. '-0',
|
|
472
786
|
'id', ARGV[o + 1], 'type', ARGV[o + 2], 'payload', ARGV[o + 3],
|
|
473
|
-
'created_at',
|
|
474
|
-
owner_writes[#owner_writes + 1] = KEYS[
|
|
475
|
-
owner_writes[#owner_writes + 1] = cjson.encode({
|
|
787
|
+
'created_at', now, 'cause', ARGV[o + 4])
|
|
788
|
+
owner_writes[#owner_writes + 1] = KEYS[2 + i]
|
|
789
|
+
owner_writes[#owner_writes + 1] = cjson.encode({ session_id, base + i })
|
|
476
790
|
wake_add(field, ARGV[o + 1], ARGV[o + 2], ARGV[o + 3])
|
|
477
|
-
if ARGV[o + 6] == '1' then
|
|
478
|
-
put(mkey(field), field .. ':p', ARGV[2])
|
|
479
|
-
else
|
|
480
|
-
enqueue(field, ARGV[o + 5])
|
|
481
|
-
end
|
|
482
791
|
end
|
|
792
|
+
index_batch(batch)
|
|
793
|
+
enqueue(batch)
|
|
483
794
|
owner_writes[#owner_writes + 1] = COUNTER
|
|
484
795
|
owner_writes[#owner_writes + 1] = tostring(base + n)
|
|
485
|
-
|
|
796
|
+
if schema_missing then
|
|
797
|
+
owner_writes[#owner_writes + 1] = SCHEMA
|
|
798
|
+
owner_writes[#owner_writes + 1] = '${REDIS_LAYOUT}'
|
|
799
|
+
end
|
|
800
|
+
if typed_dirty then
|
|
801
|
+
owner_writes[#owner_writes + 1] = TYPED
|
|
802
|
+
owner_writes[#owner_writes + 1] = cjson.encode(typed)
|
|
803
|
+
end
|
|
486
804
|
sliced('MSET', nil, owner_writes)
|
|
487
|
-
|
|
805
|
+
flush_states()
|
|
488
806
|
flush()
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
807
|
+
flush_ready()
|
|
808
|
+
local pending = has_pending()
|
|
809
|
+
wake(ARGV[5], tostring(base + n), now)
|
|
810
|
+
return { 'ok', tostring(base + 1), pending }
|
|
492
811
|
`
|
|
493
812
|
|
|
494
813
|
/**
|
|
495
|
-
* KEYS: counter,
|
|
496
|
-
*
|
|
497
|
-
*
|
|
814
|
+
* KEYS: counter, layout.
|
|
815
|
+
* ARGV: type, after. Indexes one page of the stored log below the type's
|
|
816
|
+
* index, toward `after`, registering the type first if no cursor read it
|
|
817
|
+
* yet. Returns ['1'] once the index reaches `after`, else ['0'].
|
|
498
818
|
*/
|
|
499
|
-
const
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
if
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
local
|
|
511
|
-
|
|
512
|
-
local
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
end
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
if hold then put(meta_key, field .. ':ce', ARGV[6]) end
|
|
531
|
-
flush()
|
|
532
|
-
if hold then
|
|
533
|
-
redis.call('HDEL', meta_key, field .. ':ch')
|
|
534
|
-
else
|
|
535
|
-
redis.call('HDEL', meta_key, field .. ':ch', field .. ':ce')
|
|
536
|
-
end
|
|
537
|
-
if dead then
|
|
538
|
-
redis.call('ZREM', KEYS[4], field)
|
|
539
|
-
release_snapshot_pins(KEYS[5], KEYS[6])
|
|
540
|
-
return { 'dead_lettered', tostring(failures) }
|
|
819
|
+
const INDEX_TYPE_LUA = `${BASE_LUA}
|
|
820
|
+
if not read_bounds() then return redis.error_reply('${LAYOUT_ERROR}') end
|
|
821
|
+
local name, after = ARGV[1], tonumber(ARGV[2])
|
|
822
|
+
local entry = typed[name]
|
|
823
|
+
if not entry then
|
|
824
|
+
entry = { total, total }
|
|
825
|
+
typed[name] = entry
|
|
826
|
+
end
|
|
827
|
+
if entry[1] > after then
|
|
828
|
+
local entries = redis.call('XREVRANGE', LOG, tostring(entry[1]) .. '-0',
|
|
829
|
+
tostring(after + 1) .. '-0', 'COUNT', ${READ_PAGE_SIZE})
|
|
830
|
+
local adds = {}
|
|
831
|
+
for _, stored in ipairs(entries) do
|
|
832
|
+
local fields = stored[2]
|
|
833
|
+
for j = 1, #fields, 2 do
|
|
834
|
+
if fields[j] == 'type' then
|
|
835
|
+
if fields[j + 1] == name then
|
|
836
|
+
local idx = entry_index(stored[1])
|
|
837
|
+
adds[#adds + 1] = idx
|
|
838
|
+
adds[#adds + 1] = tostring(idx)
|
|
839
|
+
end
|
|
840
|
+
break
|
|
841
|
+
end
|
|
842
|
+
end
|
|
843
|
+
end
|
|
844
|
+
sliced('ZADD', type_key(name), adds)
|
|
845
|
+
if #entries < ${READ_PAGE_SIZE} then
|
|
846
|
+
entry[1] = after
|
|
847
|
+
else
|
|
848
|
+
entry[1] = entry_index(entries[#entries][1]) - 1
|
|
849
|
+
end
|
|
541
850
|
end
|
|
542
|
-
|
|
851
|
+
redis.call('SET', TYPED, cjson.encode(typed))
|
|
852
|
+
return { entry[1] <= after and '1' or '0' }
|
|
543
853
|
`
|
|
544
854
|
|
|
545
855
|
/**
|
|
546
|
-
* KEYS:
|
|
547
|
-
*
|
|
548
|
-
*
|
|
549
|
-
*
|
|
550
|
-
*
|
|
551
|
-
*
|
|
856
|
+
* KEYS: counter, layout.
|
|
857
|
+
* ARGV: holder, nowMs, expiresAtMs, maxFailures or '', served consumers,
|
|
858
|
+
* the highest head to claim or '', excluded count, then per excluded head:
|
|
859
|
+
* consumer, tag.
|
|
860
|
+
* Live heads are the ready set's positive scores, so an empty one is enough
|
|
861
|
+
* to report 'settled'. A claimed result ends with the clear marker.
|
|
552
862
|
*/
|
|
553
|
-
const CLAIM_AVAILABLE_LUA =
|
|
554
|
-
local
|
|
555
|
-
|
|
556
|
-
${
|
|
557
|
-
local
|
|
863
|
+
const CLAIM_AVAILABLE_LUA = `${BASE_LUA}${CURSOR_LUA}
|
|
864
|
+
local ready = redis.call('ZRANGEBYSCORE', READY, 1, ARGV[6] == '' and '+inf' or ARGV[6], 'WITHSCORES')
|
|
865
|
+
if #ready == 0 then return { 'settled' } end
|
|
866
|
+
if not read_bounds() then return redis.error_reply('${LAYOUT_ERROR}') end
|
|
867
|
+
local holder, now, expires_at = ARGV[1], tonumber(ARGV[2]), ARGV[3]
|
|
868
|
+
local max_failures = tonumber(ARGV[4])
|
|
869
|
+
use_served(ARGV[5])
|
|
558
870
|
local excluded = {}
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
871
|
+
for i = 1, tonumber(ARGV[7]) do
|
|
872
|
+
excluded[ARGV[7 + (i - 1) * 2 + 1] .. '\\0' .. ARGV[7 + (i - 1) * 2 + 2]] = true
|
|
873
|
+
end
|
|
562
874
|
local candidates = {}
|
|
563
|
-
for
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
local
|
|
567
|
-
local
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
875
|
+
for i = 1, #ready, 2 do
|
|
876
|
+
local cid, idx = ready[i], tonumber(ready[i + 1])
|
|
877
|
+
local consumer, tag = parse_cid(cid)
|
|
878
|
+
local kind = string.sub(tag, 1, 1)
|
|
879
|
+
local key = consumer .. '\\0' .. (kind == 'G' and ('L' .. string.sub(tag, 2)) or tag)
|
|
880
|
+
if served_has(consumer) and not excluded[key] then
|
|
881
|
+
candidates[#candidates + 1] = {
|
|
882
|
+
idx, row_of(consumer, tag), consumer = consumer, cid = cid,
|
|
883
|
+
kind = kind == '#' and '#' or tag,
|
|
884
|
+
}
|
|
885
|
+
end
|
|
886
|
+
end
|
|
887
|
+
local rows = read_rows(candidates, { 'd', 'fc', 'lc', 'ch', 'ce', 'a', 'lf', 'lfa', 'e', 'f', 'pins' })
|
|
888
|
+
local takes_over, retry_at, wanted = {}, nil, {}
|
|
889
|
+
for position, candidate in ipairs(candidates) do
|
|
890
|
+
local row = rows[position]
|
|
891
|
+
local expires = tonumber(row[5] or '0')
|
|
892
|
+
if expires > now then
|
|
574
893
|
if not retry_at or expires < retry_at then retry_at = expires end
|
|
575
894
|
else
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
local meta_key = mkey(field)
|
|
579
|
-
local attempt = tostring(tonumber(row[2] or '0') + 1)
|
|
580
|
-
-- An expiry that is not the current attempt's retry hold is a lapsed
|
|
581
|
-
-- claim: its attempt never finished.
|
|
582
|
-
local lapsed = row[1] and row[7] ~= row[2]
|
|
583
|
-
local failures = tonumber(row[4] or '0')
|
|
584
|
-
if lapsed then
|
|
585
|
-
failures = failures + 1
|
|
586
|
-
put(meta_key, field .. ':a', tostring(failures))
|
|
587
|
-
end
|
|
588
|
-
put(meta_key, field .. ':d', attempt)
|
|
589
|
-
if lapsed and max_failures and failures >= max_failures then
|
|
590
|
-
-- That takeover spent the last failed attempt: dead-letter, unclaimed.
|
|
591
|
-
-- Advancing the attempt fences the lapsed one; nothing runs.
|
|
592
|
-
put(meta_key, field .. ':f', ARGV[2])
|
|
593
|
-
dead[#dead + 1] = field
|
|
594
|
-
claimed[#claimed + 1] = {
|
|
595
|
-
attempt,
|
|
596
|
-
tostring(failures),
|
|
597
|
-
row[5] or false,
|
|
598
|
-
row[3] or false,
|
|
599
|
-
read_fields({ field }, { 'lc' })[field][1] or false,
|
|
600
|
-
row[6] or false,
|
|
601
|
-
row[7] or false,
|
|
602
|
-
false,
|
|
603
|
-
false,
|
|
604
|
-
entries[1],
|
|
605
|
-
ARGV[2]
|
|
606
|
-
}
|
|
607
|
-
else
|
|
608
|
-
local first_claimed = row[3] or ARGV[2]
|
|
609
|
-
put(meta_key, field .. ':fc', first_claimed)
|
|
610
|
-
put(meta_key, field .. ':lc', ARGV[2])
|
|
611
|
-
put(meta_key, field .. ':ch', ARGV[1])
|
|
612
|
-
put(meta_key, field .. ':ce', ARGV[3])
|
|
613
|
-
claimed[#claimed + 1] = {
|
|
614
|
-
attempt,
|
|
615
|
-
tostring(failures),
|
|
616
|
-
row[5] or false,
|
|
617
|
-
first_claimed,
|
|
618
|
-
ARGV[2],
|
|
619
|
-
row[6] or false,
|
|
620
|
-
row[7] or false,
|
|
621
|
-
ARGV[1],
|
|
622
|
-
ARGV[3],
|
|
623
|
-
entries[1],
|
|
624
|
-
false
|
|
625
|
-
}
|
|
626
|
-
end
|
|
895
|
+
takes_over[#takes_over + 1] = { candidate = candidate, row = row }
|
|
896
|
+
wanted[candidate[1]] = true
|
|
627
897
|
end
|
|
628
898
|
end
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
899
|
+
if #takes_over == 0 then
|
|
900
|
+
if retry_at then return { 'busy', tostring(retry_at) } end
|
|
901
|
+
return { 'settled' }
|
|
902
|
+
end
|
|
903
|
+
-- One XRANGE per run of consecutive claimed indexes.
|
|
904
|
+
local indexes, entries = {}, {}
|
|
905
|
+
for idx in pairs(wanted) do indexes[#indexes + 1] = idx end
|
|
906
|
+
table.sort(indexes)
|
|
907
|
+
local first = 1
|
|
908
|
+
while first <= #indexes do
|
|
909
|
+
local last = first
|
|
910
|
+
while last < #indexes and indexes[last + 1] == indexes[last] + 1 do last = last + 1 end
|
|
911
|
+
local range = redis.call('XRANGE', LOG, tostring(indexes[first]) .. '-0', tostring(indexes[last]) .. '-0')
|
|
912
|
+
for _, entry in ipairs(range) do entries[entry_index(entry[1])] = entry end
|
|
913
|
+
first = last + 1
|
|
914
|
+
end
|
|
915
|
+
for _, idx in ipairs(indexes) do
|
|
916
|
+
if not entries[idx] then return { 'missing', tostring(idx) } end
|
|
917
|
+
end
|
|
918
|
+
local result = { 'claimed', tostring(#takes_over) }
|
|
919
|
+
for _, take in ipairs(takes_over) do
|
|
920
|
+
local candidate, row = take.candidate, take.row
|
|
921
|
+
local idx, at = candidate[1], candidate[2]
|
|
922
|
+
local attempt = tostring(tonumber(row[1] or '0') + 1)
|
|
923
|
+
-- A claim still held here lapsed: its attempt never finished.
|
|
924
|
+
local lapsed = row[4] ~= false
|
|
925
|
+
local failures = tonumber(row[6] or '0')
|
|
926
|
+
if lapsed then
|
|
927
|
+
failures = failures + 1
|
|
928
|
+
put_row(idx, at, 'a', tostring(failures))
|
|
639
929
|
end
|
|
640
|
-
|
|
641
|
-
|
|
930
|
+
put_row(idx, at, 'd', attempt)
|
|
931
|
+
local columns
|
|
932
|
+
if lapsed and max_failures and failures >= max_failures then
|
|
933
|
+
-- That takeover spent the last failed attempt: dead-letter, unclaimed.
|
|
934
|
+
-- Advancing the attempt fences the lapsed one; nothing runs.
|
|
935
|
+
put_row(idx, at, 'f', ARGV[2])
|
|
936
|
+
drop_row(idx, at, 'ch')
|
|
937
|
+
drop_row(idx, at, 'ce')
|
|
938
|
+
ready_set(candidate.cid, -idx)
|
|
939
|
+
release_pins(idx, at, row[11])
|
|
940
|
+
columns = { attempt, row[2], row[3], false, false, tostring(failures), row[7], row[8], row[9], ARGV[2] }
|
|
941
|
+
else
|
|
942
|
+
local first_claimed = row[2] or ARGV[2]
|
|
943
|
+
put_row(idx, at, 'fc', first_claimed)
|
|
944
|
+
put_row(idx, at, 'lc', ARGV[2])
|
|
945
|
+
put_row(idx, at, 'ch', holder)
|
|
946
|
+
put_row(idx, at, 'ce', expires_at)
|
|
947
|
+
put_row(idx, at, 'l', candidate.kind)
|
|
948
|
+
columns = { attempt, first_claimed, ARGV[2], holder, expires_at, tostring(failures), row[7], row[8], row[9], false }
|
|
949
|
+
end
|
|
950
|
+
result[#result + 1] = { candidate.consumer, candidate.kind, tostring(idx), columns, entries[idx] }
|
|
642
951
|
end
|
|
643
|
-
|
|
644
|
-
|
|
952
|
+
flush()
|
|
953
|
+
flush_ready()
|
|
954
|
+
result[#result + 1] = tostring(boundary)
|
|
955
|
+
result[#result + 1] = tostring(total)
|
|
956
|
+
return result
|
|
645
957
|
`
|
|
646
958
|
|
|
647
959
|
/**
|
|
648
|
-
* KEYS: counter,
|
|
649
|
-
* ARGV: holder, nowMs, expiresAtMs, n, then per claim: index,
|
|
650
|
-
* Returns [renewedCount, ...renewed, ...superseded]
|
|
960
|
+
* KEYS: counter, layout.
|
|
961
|
+
* ARGV: holder, nowMs, expiresAtMs, n, then per claim: consumer, tag, index,
|
|
962
|
+
* attempt. Returns [renewedCount, ...renewed, ...superseded] as 1-based
|
|
963
|
+
* positions in the request.
|
|
651
964
|
*/
|
|
652
|
-
const RENEW_CLAIMS_LUA =
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
local
|
|
656
|
-
local total = tonumber(counts[1] or '0')
|
|
657
|
-
use_split(counts[2], counts[1], counts[3])
|
|
658
|
-
local claims, fields, cleared = {}, {}, {}
|
|
965
|
+
const RENEW_CLAIMS_LUA = `${BASE_LUA}
|
|
966
|
+
if not read_bounds() then return redis.error_reply('${LAYOUT_ERROR}') end
|
|
967
|
+
local holder, now, expires_at = ARGV[1], tonumber(ARGV[2]), ARGV[3]
|
|
968
|
+
local refs, items, superseded = {}, {}, {}
|
|
659
969
|
for i = 1, tonumber(ARGV[4]) do
|
|
660
|
-
local
|
|
661
|
-
local idx = tonumber(
|
|
970
|
+
local o = 4 + (i - 1) * 4
|
|
971
|
+
local idx = tonumber(ARGV[o + 3])
|
|
662
972
|
if idx <= boundary then
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
973
|
+
superseded[#superseded + 1] = tostring(i)
|
|
974
|
+
else
|
|
975
|
+
local row = row_of(ARGV[o + 1], ARGV[o + 2])
|
|
976
|
+
refs[#refs + 1] = { idx, row }
|
|
977
|
+
items[#items + 1] = { position = i, idx = idx, row = row, tag = ARGV[o + 2], attempt = tonumber(ARGV[o + 4]) }
|
|
667
978
|
end
|
|
668
979
|
end
|
|
669
|
-
local rows =
|
|
980
|
+
local rows = read_rows(refs, { 'd', 'ce', 'p', 'f', 'ch', 'l' })
|
|
670
981
|
local renewed = {}
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
local
|
|
674
|
-
if
|
|
675
|
-
|
|
676
|
-
elseif
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
982
|
+
for k, item in ipairs(items) do
|
|
983
|
+
local row = rows[k]
|
|
984
|
+
local attempt = tonumber(row[1] or '0')
|
|
985
|
+
if row[6] and not owns(row[6], item.tag) then
|
|
986
|
+
-- Another cursor's head: this one never claimed it.
|
|
987
|
+
elseif row[3] or attempt > item.attempt then
|
|
988
|
+
superseded[#superseded + 1] = tostring(item.position)
|
|
989
|
+
elseif attempt == item.attempt and tonumber(expires_at) > now and
|
|
990
|
+
not row[4] and row[5] == holder and tonumber(row[2] or '0') > now then
|
|
991
|
+
put_row(item.idx, item.row, 'ce', expires_at)
|
|
992
|
+
renewed[#renewed + 1] = tostring(item.position)
|
|
681
993
|
end
|
|
682
994
|
end
|
|
683
995
|
flush()
|
|
684
|
-
for _, field in ipairs(cleared) do superseded[#superseded + 1] = field end
|
|
685
996
|
local result = { tostring(#renewed) }
|
|
686
997
|
for i = 1, #renewed do result[#result + 1] = renewed[i] end
|
|
687
998
|
for i = 1, #superseded do result[#result + 1] = superseded[i] end
|
|
@@ -689,228 +1000,403 @@ return result
|
|
|
689
1000
|
`
|
|
690
1001
|
|
|
691
1002
|
/**
|
|
692
|
-
* KEYS:
|
|
693
|
-
*
|
|
694
|
-
*
|
|
695
|
-
*
|
|
1003
|
+
* KEYS: counter, layout, then one id ownership key per child.
|
|
1004
|
+
* ARGV: sessionId, nowMs, consumer, tag, index, attempt, child causeJson,
|
|
1005
|
+
* served consumers, notify channel, returned idsJson, n, then per child: id,
|
|
1006
|
+
* type, payloadJson, targets.
|
|
696
1007
|
*/
|
|
697
|
-
const COMPLETE_ATTEMPT_LUA =
|
|
698
|
-
local
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
local
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
local
|
|
710
|
-
if tonumber(parent) <= boundary then return { 'superseded' } end
|
|
711
|
-
if tonumber(parent) < 1 or tonumber(parent) > total then return { 'missing' } end
|
|
712
|
-
local row = read_fields({ parent }, { 'd', 'p', 'pa', 'ri', 'ch', 'f', 'lfa', 'l', 'next' })[parent]
|
|
1008
|
+
const COMPLETE_ATTEMPT_LUA = `${BASE_LUA}${CURSOR_LUA}${WAKE_LUA}
|
|
1009
|
+
local session_id, now = ARGV[1], ARGV[2]
|
|
1010
|
+
local consumer, tag, idx, attempt = ARGV[3], ARGV[4], tonumber(ARGV[5]), tonumber(ARGV[6])
|
|
1011
|
+
local at = row_of(consumer, tag)
|
|
1012
|
+
local n = tonumber(ARGV[11])
|
|
1013
|
+
local id_keys = {}
|
|
1014
|
+
for i = 1, n do id_keys[i] = KEYS[2 + i] end
|
|
1015
|
+
local values = read_bounds(id_keys)
|
|
1016
|
+
if not values then return redis.error_reply('${LAYOUT_ERROR}') end
|
|
1017
|
+
local owners = owners_of(values, session_id)
|
|
1018
|
+
if idx <= boundary then return { 'superseded' } end
|
|
1019
|
+
if idx < 1 or idx > total then return { 'missing' } end
|
|
1020
|
+
local row = read_rows({ { idx, at } }, { 'd', 'p', 'pa', 'ri', 'ch', 'f', 'l', 'next', 'pins' })[1]
|
|
713
1021
|
if row[2] then
|
|
714
|
-
|
|
715
|
-
if not row[
|
|
716
|
-
|
|
717
|
-
|
|
1022
|
+
-- The cursor moved past this head: a retry of its completion replays.
|
|
1023
|
+
if not owns(row[7], tag) then return { 'superseded' } end
|
|
1024
|
+
if not row[3] or tonumber(row[3]) ~= attempt then return { 'superseded' } end
|
|
1025
|
+
local recorded = cjson.decode(row[4] or '[]')
|
|
1026
|
+
if #recorded ~= n then return { 'partial' } end
|
|
718
1027
|
for i = 1, n do
|
|
719
|
-
if
|
|
1028
|
+
if recorded[i] ~= ARGV[11 + (i - 1) * 4 + 1] then return { 'partial' } end
|
|
720
1029
|
end
|
|
721
|
-
local
|
|
1030
|
+
local result = { 'duplicate' }
|
|
722
1031
|
for i = 1, n do
|
|
723
1032
|
if not owners[i] then return { 'partial' } end
|
|
724
1033
|
local rec = cjson.decode(owners[i])
|
|
725
|
-
if rec[1] ~=
|
|
726
|
-
|
|
727
|
-
if #entries == 0 then return { 'partial' } end
|
|
728
|
-
local values = entries[1][2]
|
|
729
|
-
local cause = nil
|
|
730
|
-
for j = 1, #values, 2 do
|
|
731
|
-
if values[j] == 'cause' then cause = values[j + 1] break end
|
|
732
|
-
end
|
|
733
|
-
if not cause or cause == '' then return { 'partial' } end
|
|
734
|
-
local decoded = cjson.decode(cause)
|
|
735
|
-
if tonumber(decoded['index']) ~= tonumber(parent) or
|
|
736
|
-
tonumber(decoded['attempt']) ~= attempt then return { 'partial' } end
|
|
737
|
-
indexes[#indexes + 1] = tostring(rec[2])
|
|
1034
|
+
if rec[1] ~= session_id then return { 'partial' } end
|
|
1035
|
+
result[#result + 1] = tostring(rec[2])
|
|
738
1036
|
end
|
|
739
|
-
local result = { 'duplicate' }
|
|
740
|
-
for _, idx in ipairs(indexes) do result[#result + 1] = idx end
|
|
741
1037
|
return result
|
|
742
1038
|
end
|
|
743
|
-
if tonumber(row[1] or '0') ~= attempt
|
|
744
|
-
if not row[5] or row[6] or tonumber(row[7] or '0') == attempt then
|
|
1039
|
+
if tonumber(row[1] or '0') ~= attempt or not row[5] or row[6] or not owns(row[7], tag) then
|
|
745
1040
|
return { 'superseded' }
|
|
746
1041
|
end
|
|
747
1042
|
for i = 1, n do
|
|
748
1043
|
if owners[i] then return { 'partial' } end
|
|
749
1044
|
end
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
1045
|
+
use_served(ARGV[8])
|
|
1046
|
+
local batch = {}
|
|
1047
|
+
for i = 1, n do
|
|
1048
|
+
local o = 11 + (i - 1) * 4
|
|
1049
|
+
batch[i] = { idx = total + i, type = ARGV[o + 2], targets = targets_of(ARGV[o + 4]) }
|
|
1050
|
+
end
|
|
1051
|
+
local moved, missing = advance(consumer, row[7], idx, row[8], batch)
|
|
1052
|
+
if not moved then return { 'index', missing, tostring(idx) } end
|
|
1053
|
+
local planned, unindexed, after = plan(batch, total)
|
|
1054
|
+
if not planned then return { 'index', unindexed, tostring(after) } end
|
|
1055
|
+
put_row(idx, at, 'p', now)
|
|
1056
|
+
put_row(idx, at, 'pa', ARGV[6])
|
|
1057
|
+
put_row(idx, at, 'ri', ARGV[10])
|
|
1058
|
+
release_pins(idx, at, row[9])
|
|
762
1059
|
local owner_writes = {}
|
|
763
1060
|
for i = 1, n do
|
|
764
|
-
local o =
|
|
1061
|
+
local o = 11 + (i - 1) * 4
|
|
765
1062
|
local field = tostring(total + i)
|
|
766
|
-
redis.call('XADD',
|
|
1063
|
+
redis.call('XADD', LOG, field .. '-0',
|
|
767
1064
|
'id', ARGV[o + 1], 'type', ARGV[o + 2], 'payload', ARGV[o + 3],
|
|
768
|
-
'created_at',
|
|
769
|
-
owner_writes[#owner_writes + 1] = KEYS[
|
|
770
|
-
owner_writes[#owner_writes + 1] = cjson.encode({
|
|
1065
|
+
'created_at', now, 'cause', ARGV[7])
|
|
1066
|
+
owner_writes[#owner_writes + 1] = KEYS[2 + i]
|
|
1067
|
+
owner_writes[#owner_writes + 1] = cjson.encode({ session_id, total + i })
|
|
771
1068
|
wake_add(field, ARGV[o + 1], ARGV[o + 2], ARGV[o + 3])
|
|
772
|
-
if ARGV[o + 5] == '1' then
|
|
773
|
-
put(mkey(field), field .. ':p', ARGV[2])
|
|
774
|
-
else
|
|
775
|
-
enqueue(field, ARGV[o + 4])
|
|
776
|
-
end
|
|
777
1069
|
end
|
|
1070
|
+
index_batch(batch)
|
|
1071
|
+
enqueue(batch)
|
|
778
1072
|
if n > 0 then
|
|
779
1073
|
owner_writes[#owner_writes + 1] = COUNTER
|
|
780
1074
|
owner_writes[#owner_writes + 1] = tostring(total + n)
|
|
781
|
-
|
|
782
|
-
|
|
1075
|
+
if schema_missing then
|
|
1076
|
+
owner_writes[#owner_writes + 1] = SCHEMA
|
|
1077
|
+
owner_writes[#owner_writes + 1] = '${REDIS_LAYOUT}'
|
|
1078
|
+
end
|
|
1079
|
+
end
|
|
1080
|
+
if typed_dirty then
|
|
1081
|
+
owner_writes[#owner_writes + 1] = TYPED
|
|
1082
|
+
owner_writes[#owner_writes + 1] = cjson.encode(typed)
|
|
783
1083
|
end
|
|
784
|
-
|
|
1084
|
+
sliced('MSET', nil, owner_writes)
|
|
1085
|
+
flush_states()
|
|
785
1086
|
flush()
|
|
786
|
-
|
|
1087
|
+
flush_ready()
|
|
1088
|
+
if n > 0 then wake(ARGV[9], tostring(total + n), now) end
|
|
787
1089
|
return { 'ok', tostring(total + 1) }
|
|
788
1090
|
`
|
|
789
1091
|
|
|
790
1092
|
/**
|
|
791
|
-
* KEYS: counter,
|
|
792
|
-
* ARGV:
|
|
793
|
-
*
|
|
794
|
-
* event above the clear boundary. A retry keeps the event pending and makes
|
|
795
|
-
* it ready again when it is unlaned or its lane's head.
|
|
1093
|
+
* KEYS: counter, layout.
|
|
1094
|
+
* ARGV: consumer, tag, index, attempt, error, maxFailures, nowMs, retryAtMs
|
|
1095
|
+
* or ''.
|
|
796
1096
|
*/
|
|
797
|
-
const
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
end
|
|
805
|
-
local
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
local
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
1097
|
+
const FAIL_ATTEMPT_LUA = `${BASE_LUA}
|
|
1098
|
+
if not read_bounds() then return redis.error_reply('${LAYOUT_ERROR}') end
|
|
1099
|
+
local consumer, tag, idx, attempt = ARGV[1], ARGV[2], tonumber(ARGV[3]), tonumber(ARGV[4])
|
|
1100
|
+
local at = row_of(consumer, tag)
|
|
1101
|
+
if idx <= boundary then return { 'superseded', '0' } end
|
|
1102
|
+
if idx < 1 or idx > total then return { 'missing', '0' } end
|
|
1103
|
+
local row = read_rows({ { idx, at } }, { 'a', 'd', 'p', 'f', 'lfa', 'ch', 'l', 'pins' })[1]
|
|
1104
|
+
if row[3] or (row[7] and not owns(row[7], tag)) then return { 'superseded', '0' } end
|
|
1105
|
+
local failures = tonumber(row[1] or '0')
|
|
1106
|
+
if tonumber(row[2] or '0') ~= attempt then return { 'superseded', tostring(failures) } end
|
|
1107
|
+
if not row[6] and row[5] and tonumber(row[5]) == attempt then
|
|
1108
|
+
return { row[4] and 'dead_lettered' or 'failed', tostring(failures) }
|
|
1109
|
+
end
|
|
1110
|
+
if row[4] then return { 'dead_lettered', tostring(failures) } end
|
|
1111
|
+
if not row[6] then return { 'superseded', tostring(failures) } end
|
|
1112
|
+
failures = failures + 1
|
|
1113
|
+
local dead = failures >= tonumber(ARGV[6])
|
|
1114
|
+
put_row(idx, at, 'a', tostring(failures))
|
|
1115
|
+
put_row(idx, at, 'e', ARGV[5])
|
|
1116
|
+
put_row(idx, at, 'lf', ARGV[7])
|
|
1117
|
+
put_row(idx, at, 'lfa', ARGV[4])
|
|
1118
|
+
drop_row(idx, at, 'ch')
|
|
1119
|
+
if dead then
|
|
1120
|
+
put_row(idx, at, 'f', ARGV[7])
|
|
1121
|
+
drop_row(idx, at, 'ce')
|
|
1122
|
+
ready_set(cid_for(consumer, row[7], idx), -idx)
|
|
1123
|
+
release_pins(idx, at, row[8])
|
|
1124
|
+
elseif ARGV[8] ~= '' then
|
|
1125
|
+
put_row(idx, at, 'ce', ARGV[8])
|
|
820
1126
|
else
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
1127
|
+
drop_row(idx, at, 'ce')
|
|
1128
|
+
end
|
|
1129
|
+
flush()
|
|
1130
|
+
flush_ready()
|
|
1131
|
+
return { dead and 'dead_lettered' or 'failed', tostring(failures) }
|
|
1132
|
+
`
|
|
1133
|
+
|
|
1134
|
+
/**
|
|
1135
|
+
* KEYS: counter, layout. ARGV: consumer, index, action, nowMs, served
|
|
1136
|
+
* consumers. Returns [redriven, hasPending] | ['index', type, after]. Acts
|
|
1137
|
+
* only on the consumer's dead-lettered head at the index, above the clear
|
|
1138
|
+
* marker.
|
|
1139
|
+
*/
|
|
1140
|
+
const REDRIVE_LUA = `${BASE_LUA}${CURSOR_LUA}
|
|
1141
|
+
if not read_bounds() then return redis.error_reply('${LAYOUT_ERROR}') end
|
|
1142
|
+
use_served(ARGV[5])
|
|
1143
|
+
local consumer, idx = ARGV[1], tonumber(ARGV[2])
|
|
1144
|
+
local function result(redriven) return { redriven, has_pending() } end
|
|
1145
|
+
if idx <= boundary or idx < 1 or idx > total then return result('0') end
|
|
1146
|
+
local cid, tag = nil, nil
|
|
1147
|
+
for _, member in ipairs(redis.call('ZRANGEBYSCORE', READY, -idx, -idx)) do
|
|
1148
|
+
local owner, member_tag = parse_cid(member)
|
|
1149
|
+
if owner == consumer then
|
|
1150
|
+
cid, tag = member, member_tag
|
|
1151
|
+
break
|
|
824
1152
|
end
|
|
825
1153
|
end
|
|
1154
|
+
if not cid then return result('0') end
|
|
1155
|
+
local kind = string.sub(tag, 1, 1) == '#' and '#' or tag
|
|
1156
|
+
local at = row_of(consumer, tag)
|
|
1157
|
+
if ARGV[3] == 'skip' then
|
|
1158
|
+
local row = read_rows({ { idx, at } }, { 'next' })[1]
|
|
1159
|
+
local moved, missing = advance(consumer, kind, idx, row[1], {})
|
|
1160
|
+
if not moved then return { 'index', missing, tostring(idx) } end
|
|
1161
|
+
put_row(idx, at, 'p', ARGV[4])
|
|
1162
|
+
flush_states()
|
|
1163
|
+
if typed_dirty then redis.call('SET', TYPED, cjson.encode(typed)) end
|
|
1164
|
+
else
|
|
1165
|
+
drop_row(idx, at, 'f')
|
|
1166
|
+
drop_row(idx, at, 'a')
|
|
1167
|
+
ready_set(cid, idx)
|
|
1168
|
+
end
|
|
1169
|
+
flush()
|
|
1170
|
+
flush_ready()
|
|
826
1171
|
return result('1')
|
|
827
1172
|
`
|
|
828
1173
|
|
|
829
1174
|
/**
|
|
830
|
-
* KEYS:
|
|
831
|
-
*
|
|
832
|
-
*
|
|
833
|
-
*
|
|
834
|
-
*
|
|
1175
|
+
* KEYS: counter, layout. ARGV: consumer, lane, types JSON or '*', backfill
|
|
1176
|
+
* ('1' or '0'). Brings one log cursor up to date without an append, as
|
|
1177
|
+
* `plan` and `enqueue` would for an append carrying its types: a missing
|
|
1178
|
+
* cursor starts at the clear marker with backfill, else at the head with
|
|
1179
|
+
* nothing to run; an idle one takes the next stored event of its types; one
|
|
1180
|
+
* with a head, claimed or not, keeps it. Records the types, and writes
|
|
1181
|
+
* nothing when nothing changed. Returns ['ok', position, head or nil, head
|
|
1182
|
+
* columns, session head] | ['index', type, after].
|
|
835
1183
|
*/
|
|
836
|
-
const
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
local
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
1184
|
+
const WAKE_LOG_LUA = `${BASE_LUA}${CURSOR_LUA}
|
|
1185
|
+
if not read_bounds() then return redis.error_reply('${LAYOUT_ERROR}') end
|
|
1186
|
+
local consumer, lane, ty = ARGV[1], ARGV[2], ARGV[3]
|
|
1187
|
+
local types = types_of(ty)
|
|
1188
|
+
local cid = cid_of(consumer, 'G' .. lane)
|
|
1189
|
+
load_states({ cid })
|
|
1190
|
+
local state = states[cid]
|
|
1191
|
+
local had_head = state.h
|
|
1192
|
+
-- Nothing after the latest clear: a missing cursor stays missing.
|
|
1193
|
+
if not state.exists and total == boundary then
|
|
1194
|
+
return { 'ok', tostring(boundary), false, {}, tostring(total) }
|
|
1195
|
+
end
|
|
1196
|
+
if not state.exists then
|
|
1197
|
+
state.exists, state.p = true, total
|
|
1198
|
+
if ARGV[4] == '1' then
|
|
1199
|
+
local head, missing = stored_head(boundary, types)
|
|
1200
|
+
if head == nil then return { 'index', missing, tostring(boundary) } end
|
|
1201
|
+
state.p, state.h = boundary, head
|
|
1202
|
+
else
|
|
1203
|
+
-- Nothing it takes is stored after its start: index its types from there.
|
|
1204
|
+
for name in pairs(types or {}) do
|
|
1205
|
+
if typed[name] == nil then register(name, total) end
|
|
1206
|
+
end
|
|
1207
|
+
end
|
|
1208
|
+
state.dirty = true
|
|
1209
|
+
elseif not state.h then
|
|
1210
|
+
-- A deployment that doesn't serve this consumer may have appended events
|
|
1211
|
+
-- it takes since it went idle.
|
|
1212
|
+
local head, missing = stored_head(state.p, types)
|
|
1213
|
+
if head == nil then return { 'index', missing, tostring(state.p) } end
|
|
1214
|
+
if head then state.h, state.dirty = head, true end
|
|
1215
|
+
end
|
|
1216
|
+
if state.ty ~= ty then state.ty, state.dirty = ty, true end
|
|
1217
|
+
local columns = {}
|
|
1218
|
+
if had_head then
|
|
1219
|
+
columns = read_rows({ { state.h, row_of(consumer, 'G' .. lane) } }, ${HEAD_COLUMNS})[1]
|
|
1220
|
+
elseif state.h then
|
|
1221
|
+
ready_set(cid, state.h)
|
|
1222
|
+
end
|
|
1223
|
+
flush_states()
|
|
1224
|
+
if typed_dirty then redis.call('SET', TYPED, cjson.encode(typed)) end
|
|
1225
|
+
flush()
|
|
1226
|
+
flush_ready()
|
|
1227
|
+
return { 'ok', tostring(state.p), state.h and tostring(state.h) or false, columns, tostring(total) }
|
|
1228
|
+
`
|
|
1229
|
+
|
|
1230
|
+
/**
|
|
1231
|
+
* KEYS: counter, layout. Returns the dead-lettered heads above the clear
|
|
1232
|
+
* marker as [consumer, tag, index, columns, entry] tuples.
|
|
1233
|
+
*/
|
|
1234
|
+
const DEAD_LETTERS_LUA = `${BASE_LUA}
|
|
1235
|
+
local dead = redis.call('ZRANGEBYSCORE', READY, '-inf', '(0', 'WITHSCORES')
|
|
1236
|
+
if #dead == 0 then return {} end
|
|
1237
|
+
if not read_bounds() then return redis.error_reply('${LAYOUT_ERROR}') end
|
|
1238
|
+
local refs, items = {}, {}
|
|
1239
|
+
for i = 1, #dead, 2 do
|
|
1240
|
+
local consumer, tag = parse_cid(dead[i])
|
|
1241
|
+
local idx = -tonumber(dead[i + 1])
|
|
1242
|
+
if idx > boundary then
|
|
1243
|
+
refs[#refs + 1] = { idx, row_of(consumer, tag) }
|
|
1244
|
+
items[#items + 1] = { consumer, tag, idx }
|
|
1245
|
+
end
|
|
849
1246
|
end
|
|
850
|
-
local rows =
|
|
1247
|
+
local rows = read_rows(refs, ${HEAD_COLUMNS})
|
|
851
1248
|
local result = {}
|
|
852
|
-
for
|
|
853
|
-
local entries = redis.call('XRANGE',
|
|
1249
|
+
for k, item in ipairs(items) do
|
|
1250
|
+
local entries = redis.call('XRANGE', LOG, tostring(item[3]) .. '-0', tostring(item[3]) .. '-0')
|
|
854
1251
|
if #entries > 0 then
|
|
855
|
-
result[#result + 1] = entries[1]
|
|
856
|
-
|
|
1252
|
+
result[#result + 1] = { item[1], item[2], tostring(item[3]), rows[k], entries[1] }
|
|
1253
|
+
end
|
|
1254
|
+
end
|
|
1255
|
+
return result
|
|
1256
|
+
`
|
|
1257
|
+
|
|
1258
|
+
/**
|
|
1259
|
+
* KEYS: counter, layout. Returns every listed cursor as [consumer, tag,
|
|
1260
|
+
* position, head or nil, columns]: lane and log cursors from the cursor
|
|
1261
|
+
* hash, read against the clear marker, then the event cursors with a head.
|
|
1262
|
+
*/
|
|
1263
|
+
const CURSORS_LUA = `${BASE_LUA}
|
|
1264
|
+
if not read_bounds() then return redis.error_reply('${LAYOUT_ERROR}') end
|
|
1265
|
+
local listed = {}
|
|
1266
|
+
local fields = redis.call('HGETALL', CURSORS)
|
|
1267
|
+
local records, order = {}, {}
|
|
1268
|
+
for i = 1, #fields, 2 do
|
|
1269
|
+
local at = string.find(fields[i], ':[^:]*$')
|
|
1270
|
+
local cid, suffix = string.sub(fields[i], 1, at - 1), string.sub(fields[i], at + 1)
|
|
1271
|
+
if not records[cid] then
|
|
1272
|
+
records[cid] = {}
|
|
1273
|
+
order[#order + 1] = cid
|
|
1274
|
+
end
|
|
1275
|
+
records[cid][suffix] = fields[i + 1]
|
|
1276
|
+
end
|
|
1277
|
+
for _, cid in ipairs(order) do
|
|
1278
|
+
local record = records[cid]
|
|
1279
|
+
local consumer, tag = parse_cid(cid)
|
|
1280
|
+
local head = (record.h and record.h ~= '') and tonumber(record.h) or false
|
|
1281
|
+
local position = tonumber(record.p or '0')
|
|
1282
|
+
if head and head <= boundary then head = false end
|
|
1283
|
+
local kind = string.sub(tag, 1, 1)
|
|
1284
|
+
if kind == 'G' and position < boundary then position = boundary end
|
|
1285
|
+
if kind == 'G' or head or position > boundary then
|
|
1286
|
+
listed[#listed + 1] = { consumer, tag, position, head }
|
|
1287
|
+
end
|
|
1288
|
+
end
|
|
1289
|
+
local ready = redis.call('ZRANGE', READY, 0, -1, 'WITHSCORES')
|
|
1290
|
+
for i = 1, #ready, 2 do
|
|
1291
|
+
local consumer, tag = parse_cid(ready[i])
|
|
1292
|
+
if string.sub(tag, 1, 1) == '#' then
|
|
1293
|
+
local head = math.abs(tonumber(ready[i + 1]))
|
|
1294
|
+
listed[#listed + 1] = { consumer, '#', head - 1, head }
|
|
857
1295
|
end
|
|
858
1296
|
end
|
|
1297
|
+
local refs, positions = {}, {}
|
|
1298
|
+
for position, cursor in ipairs(listed) do
|
|
1299
|
+
if cursor[4] then
|
|
1300
|
+
refs[#refs + 1] = { cursor[4], row_of(cursor[1], cursor[2]) }
|
|
1301
|
+
positions[#refs] = position
|
|
1302
|
+
end
|
|
1303
|
+
end
|
|
1304
|
+
local rows = read_rows(refs, ${HEAD_COLUMNS})
|
|
1305
|
+
local columns = {}
|
|
1306
|
+
for k, position in ipairs(positions) do columns[position] = rows[k] end
|
|
1307
|
+
local result = {}
|
|
1308
|
+
for position, cursor in ipairs(listed) do
|
|
1309
|
+
result[#result + 1] = {
|
|
1310
|
+
cursor[1], cursor[2], tostring(cursor[3]),
|
|
1311
|
+
cursor[4] and tostring(cursor[4]) or false, columns[position] or {},
|
|
1312
|
+
}
|
|
1313
|
+
end
|
|
859
1314
|
return result
|
|
860
1315
|
`
|
|
861
1316
|
|
|
862
1317
|
/**
|
|
863
|
-
* KEYS:
|
|
864
|
-
* ARGV:
|
|
865
|
-
*
|
|
866
|
-
*
|
|
867
|
-
*
|
|
1318
|
+
* KEYS: counter, layout, head, history.
|
|
1319
|
+
* ARGV: write count, then per write: index, snapshot JSON, pin count, then
|
|
1320
|
+
* per pin: consumer, tag, index; finally the reducer name. Writes from
|
|
1321
|
+
* before the latest clear are dropped, and a head from before it is
|
|
1322
|
+
* replaced as if absent. A pin holds only on an unfinished, live head: a
|
|
1323
|
+
* claimed row says so itself, an unclaimed head by its cursor's ready score.
|
|
868
1324
|
*/
|
|
869
|
-
const PUT_SNAPSHOTS_LUA =
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
local
|
|
873
|
-
local
|
|
874
|
-
|
|
875
|
-
local
|
|
876
|
-
local offset = 4
|
|
877
|
-
for i = 1, tonumber(ARGV[3]) do
|
|
1325
|
+
const PUT_SNAPSHOTS_LUA = `${BASE_LUA}
|
|
1326
|
+
if not read_bounds() then return redis.error_reply('${LAYOUT_ERROR}') end
|
|
1327
|
+
local HEAD, HISTORY = KEYS[3], KEYS[4]
|
|
1328
|
+
local entries, refs, pins = {}, {}, {}
|
|
1329
|
+
local offset = 2
|
|
1330
|
+
for w = 1, tonumber(ARGV[1]) do
|
|
1331
|
+
local index = tonumber(ARGV[offset])
|
|
878
1332
|
local pin_count = tonumber(ARGV[offset + 2])
|
|
1333
|
+
local entry = { index = index, snapshot = ARGV[offset + 1], pins = {} }
|
|
879
1334
|
for p = 1, pin_count do
|
|
880
|
-
local
|
|
881
|
-
|
|
882
|
-
|
|
1335
|
+
local o = offset + 2 + (p - 1) * 3
|
|
1336
|
+
local pin = { consumer = ARGV[o + 1], tag = ARGV[o + 2], idx = tonumber(ARGV[o + 3]) }
|
|
1337
|
+
if index >= boundary and pin.idx > boundary and pin.idx <= total then
|
|
1338
|
+
pin.row = row_of(pin.consumer, pin.tag)
|
|
1339
|
+
refs[#refs + 1] = { pin.idx, pin.row }
|
|
1340
|
+
pins[#pins + 1] = pin
|
|
1341
|
+
entry.pins[#entry.pins + 1] = pin
|
|
1342
|
+
end
|
|
1343
|
+
end
|
|
1344
|
+
entries[w] = entry
|
|
1345
|
+
offset = offset + 3 + pin_count * 3
|
|
1346
|
+
end
|
|
1347
|
+
local function live(consumer, tag, idx)
|
|
1348
|
+
return tonumber(redis.call('ZSCORE', READY, cid_of(consumer, tag)) or '0') == idx
|
|
1349
|
+
end
|
|
1350
|
+
local rows = read_rows(refs, { 'pins', 'p', 'f', 'l' })
|
|
1351
|
+
local held, held_order = {}, {}
|
|
1352
|
+
for k, pin in ipairs(pins) do
|
|
1353
|
+
local row = rows[k]
|
|
1354
|
+
local valid
|
|
1355
|
+
if row[2] or row[3] then
|
|
1356
|
+
valid = false
|
|
1357
|
+
elseif row[4] then
|
|
1358
|
+
valid = owns(row[4], pin.tag)
|
|
1359
|
+
elseif pin.tag == '#' then
|
|
1360
|
+
valid = live(pin.consumer, '#' .. tostring(pin.idx), pin.idx)
|
|
1361
|
+
else
|
|
1362
|
+
valid = live(pin.consumer, pin.tag, pin.idx) or
|
|
1363
|
+
live(pin.consumer, 'G' .. string.sub(pin.tag, 2), pin.idx)
|
|
1364
|
+
end
|
|
1365
|
+
if valid then
|
|
1366
|
+
local key = tostring(pin.idx) .. ':' .. pin.row
|
|
1367
|
+
if not held[key] then
|
|
1368
|
+
local list = (row[1] and row[1] ~= '') and cjson.decode(row[1]) or {}
|
|
1369
|
+
local set = {}
|
|
1370
|
+
for _, ref in ipairs(list) do set[ref] = true end
|
|
1371
|
+
held[key] = { list = list, set = set, idx = pin.idx, row = pin.row }
|
|
1372
|
+
held_order[#held_order + 1] = key
|
|
883
1373
|
end
|
|
1374
|
+
pin.held = held[key]
|
|
884
1375
|
end
|
|
885
|
-
offset = offset + 3 + pin_count
|
|
886
1376
|
end
|
|
887
|
-
local status = read_fields(pinned, { 'p', 'f' })
|
|
888
1377
|
local snapshots = {}
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
local row = status[tostring(event_index)]
|
|
898
|
-
if index >= boundary and event_index > boundary and event_index <= total and
|
|
899
|
-
not row[1] and not row[2] then
|
|
900
|
-
local pin_key = ARGV[2] .. tostring(event_index)
|
|
901
|
-
if redis.call('SADD', pin_key, ref) == 1 then
|
|
902
|
-
redis.call('HINCRBY', KEYS[5], ref, 1)
|
|
903
|
-
end
|
|
1378
|
+
for _, entry in ipairs(entries) do
|
|
1379
|
+
local ref = cjson.encode({ HISTORY, tostring(entry.index) })
|
|
1380
|
+
for _, pin in ipairs(entry.pins) do
|
|
1381
|
+
if pin.held and not pin.held.set[ref] then
|
|
1382
|
+
pin.held.set[ref] = true
|
|
1383
|
+
pin.held.list[#pin.held.list + 1] = ref
|
|
1384
|
+
pin.held.dirty = true
|
|
1385
|
+
redis.call('HINCRBY', PIN_COUNTS, ref, 1)
|
|
904
1386
|
end
|
|
905
1387
|
end
|
|
906
|
-
if index >= boundary then
|
|
907
|
-
snapshots[#snapshots + 1] = { index = index, snapshot = snapshot, ref = ref }
|
|
1388
|
+
if entry.index >= boundary then
|
|
1389
|
+
snapshots[#snapshots + 1] = { index = entry.index, snapshot = entry.snapshot, ref = ref }
|
|
908
1390
|
end
|
|
909
|
-
offset = offset + 3 + pin_count
|
|
910
1391
|
end
|
|
1392
|
+
for _, key in ipairs(held_order) do
|
|
1393
|
+
local row = held[key]
|
|
1394
|
+
if row.dirty then put_row(row.idx, row.row, 'pins', cjson.encode(row.list)) end
|
|
1395
|
+
end
|
|
1396
|
+
flush()
|
|
911
1397
|
if #snapshots == 0 then return 1 end
|
|
912
1398
|
|
|
913
|
-
local head = redis.call('GET',
|
|
1399
|
+
local head = redis.call('GET', HEAD)
|
|
914
1400
|
local head_index = nil
|
|
915
1401
|
local registered = false
|
|
916
1402
|
if head then
|
|
@@ -920,35 +1406,35 @@ if head then
|
|
|
920
1406
|
end
|
|
921
1407
|
if head_index and head_index < boundary then
|
|
922
1408
|
head, head_index, registered = nil, nil, false
|
|
923
|
-
redis.call('ZREMRANGEBYSCORE',
|
|
1409
|
+
redis.call('ZREMRANGEBYSCORE', HISTORY, '-inf', '(' .. boundary)
|
|
924
1410
|
end
|
|
925
1411
|
-- Deletion finds each reducer's keys through this set instead of a scan.
|
|
926
1412
|
-- Heads written since carry the mark, so a reducer registers once.
|
|
927
1413
|
if not registered then
|
|
928
|
-
redis.call('SADD',
|
|
1414
|
+
redis.call('SADD', SESSION .. 'snap-reducers', ARGV[#ARGV])
|
|
929
1415
|
end
|
|
930
1416
|
local latest = snapshots[#snapshots]
|
|
931
1417
|
local writes_head = not head_index or latest.index >= head_index
|
|
932
1418
|
local advances = not head_index or latest.index > head_index
|
|
933
1419
|
local final_head_index = advances and latest.index or head_index
|
|
934
1420
|
|
|
935
|
-
if advances and head and tonumber(redis.call('HGET',
|
|
936
|
-
redis.call('ZREMRANGEBYSCORE',
|
|
937
|
-
redis.call('ZADD',
|
|
1421
|
+
if advances and head and tonumber(redis.call('HGET', PIN_COUNTS, cjson.encode({ HISTORY, tostring(head_index) })) or '0') > 0 then
|
|
1422
|
+
redis.call('ZREMRANGEBYSCORE', HISTORY, head_index, head_index)
|
|
1423
|
+
redis.call('ZADD', HISTORY, head_index, head)
|
|
938
1424
|
end
|
|
939
1425
|
|
|
940
1426
|
for _, write in ipairs(snapshots) do
|
|
941
1427
|
if write.index < final_head_index then
|
|
942
|
-
redis.call('ZREMRANGEBYSCORE',
|
|
943
|
-
if tonumber(redis.call('HGET',
|
|
944
|
-
redis.call('ZADD',
|
|
1428
|
+
redis.call('ZREMRANGEBYSCORE', HISTORY, write.index, write.index)
|
|
1429
|
+
if tonumber(redis.call('HGET', PIN_COUNTS, write.ref) or '0') > 0 then
|
|
1430
|
+
redis.call('ZADD', HISTORY, write.index, write.snapshot)
|
|
945
1431
|
end
|
|
946
1432
|
end
|
|
947
1433
|
end
|
|
948
1434
|
|
|
949
1435
|
if writes_head then
|
|
950
|
-
redis.call('SET',
|
|
951
|
-
redis.call('ZREMRANGEBYSCORE',
|
|
1436
|
+
redis.call('SET', HEAD, latest.snapshot)
|
|
1437
|
+
redis.call('ZREMRANGEBYSCORE', HISTORY, latest.index, latest.index)
|
|
952
1438
|
end
|
|
953
1439
|
return 1
|
|
954
1440
|
`
|
|
@@ -1015,14 +1501,18 @@ return live
|
|
|
1015
1501
|
`
|
|
1016
1502
|
|
|
1017
1503
|
/**
|
|
1018
|
-
* KEYS:
|
|
1504
|
+
* KEYS: counter, layout, head, history. ARGV: snapshot frontier or ''.
|
|
1019
1505
|
* Returns one atomic multiversion-cache-plus-log-frontier read, with the
|
|
1020
1506
|
* clear boundary; checkpoints from before it are ignored.
|
|
1021
1507
|
*/
|
|
1022
1508
|
const READ_STATE_LUA = `
|
|
1023
|
-
local
|
|
1509
|
+
local session = string.sub(KEYS[1], 1, -6)
|
|
1510
|
+
local counts = redis.call('MGET', KEYS[1], session .. 'clear', KEYS[2])
|
|
1511
|
+
if counts[3] and counts[3] ~= '${REDIS_LAYOUT}' then
|
|
1512
|
+
return redis.error_reply('${LAYOUT_ERROR}')
|
|
1513
|
+
end
|
|
1024
1514
|
local boundary = tonumber(counts[2] or '0')
|
|
1025
|
-
local head = redis.call('GET', KEYS[
|
|
1515
|
+
local head = redis.call('GET', KEYS[3])
|
|
1026
1516
|
local head_index = head and cjson.decode(head)['index'] or nil
|
|
1027
1517
|
if head and (type(head_index) ~= 'number' or head_index < 0 or head_index > 9007199254740991 or head_index ~= math.floor(head_index)) then
|
|
1028
1518
|
return redis.error_reply('invalid snapshot head index')
|
|
@@ -1037,63 +1527,109 @@ if head_index and (cap == '' or head_index <= tonumber(cap)) then
|
|
|
1037
1527
|
snapshot = head
|
|
1038
1528
|
else
|
|
1039
1529
|
local upper = cap == '' and '+inf' or cap
|
|
1040
|
-
local historical = redis.call('ZREVRANGEBYSCORE', KEYS[
|
|
1530
|
+
local historical = redis.call('ZREVRANGEBYSCORE', KEYS[4], upper, boundary, 'LIMIT', 0, 1)
|
|
1041
1531
|
if #historical > 0 then snapshot = historical[1] end
|
|
1042
1532
|
end
|
|
1043
1533
|
return { head_index or '', snapshot or '', counts[1] or '0', tostring(boundary) }
|
|
1044
1534
|
`
|
|
1045
1535
|
|
|
1046
1536
|
/**
|
|
1047
|
-
*
|
|
1048
|
-
*
|
|
1049
|
-
*
|
|
1537
|
+
* What reclaim removes once a session has no moved log left: every
|
|
1538
|
+
* registered reducer's checkpoints below the latest clear and the pin counts
|
|
1539
|
+
* naming them (snapshot writes register each reducer in
|
|
1540
|
+
* {prefix}:{session}:snap-reducers, so no key scan is needed), every head
|
|
1541
|
+
* row below it (whole chunks, then the boundary chunk's older rows), the
|
|
1542
|
+
* type indexes' older entries, and the lane cursors it left idle.
|
|
1050
1543
|
*/
|
|
1051
|
-
const
|
|
1052
|
-
local function
|
|
1053
|
-
for _, reducer in ipairs(redis.call('SMEMBERS',
|
|
1054
|
-
local head_key =
|
|
1544
|
+
const SWEEP_LUA = `
|
|
1545
|
+
local function sweep(boundary)
|
|
1546
|
+
for _, reducer in ipairs(redis.call('SMEMBERS', SESSION .. 'snap-reducers')) do
|
|
1547
|
+
local head_key = SESSION .. 'snap:' .. reducer
|
|
1055
1548
|
local head = redis.call('GET', head_key)
|
|
1056
1549
|
if head and tonumber(cjson.decode(head)['index']) < boundary then
|
|
1057
1550
|
redis.call('DEL', head_key)
|
|
1058
1551
|
end
|
|
1059
|
-
redis.call('ZREMRANGEBYSCORE',
|
|
1552
|
+
redis.call('ZREMRANGEBYSCORE', SESSION .. 'snap-history:' .. reducer, '-inf', '(' .. boundary)
|
|
1060
1553
|
end
|
|
1061
|
-
local
|
|
1062
|
-
local counts = redis.call('HGETALL', counts_key)
|
|
1554
|
+
local counts = redis.call('HGETALL', PIN_COUNTS)
|
|
1063
1555
|
local stale = {}
|
|
1064
1556
|
for i = 1, #counts, 2 do
|
|
1065
1557
|
if tonumber(cjson.decode(counts[i])[2]) < boundary then stale[#stale + 1] = counts[i] end
|
|
1066
1558
|
end
|
|
1067
|
-
|
|
1559
|
+
sliced('HDEL', PIN_COUNTS, stale)
|
|
1560
|
+
local raw = redis.call('GET', TYPED)
|
|
1561
|
+
if raw then
|
|
1562
|
+
for name in pairs(cjson.decode(raw)) do
|
|
1563
|
+
redis.call('ZREMRANGEBYSCORE', type_key(name), '-inf', '(' .. boundary)
|
|
1564
|
+
end
|
|
1565
|
+
end
|
|
1566
|
+
local last = math.floor((boundary - 1) / ${META_CHUNK_SIZE})
|
|
1567
|
+
local chunks = {}
|
|
1568
|
+
for chunk = 0, last - 1 do chunks[#chunks + 1] = META .. ':' .. tostring(chunk) end
|
|
1569
|
+
sliced('UNLINK', nil, chunks)
|
|
1570
|
+
local older = {}
|
|
1571
|
+
for _, field in ipairs(redis.call('HKEYS', META .. ':' .. tostring(last))) do
|
|
1572
|
+
local idx = tonumber(string.match(field, '^(%d+):'))
|
|
1573
|
+
if idx and idx < boundary then older[#older + 1] = field end
|
|
1574
|
+
end
|
|
1575
|
+
sliced('HDEL', META .. ':' .. tostring(last), older)
|
|
1576
|
+
local fields = redis.call('HGETALL', CURSORS)
|
|
1577
|
+
local records, order = {}, {}
|
|
1578
|
+
for i = 1, #fields, 2 do
|
|
1579
|
+
local at = string.find(fields[i], ':[^:]*$')
|
|
1580
|
+
local cid, suffix = string.sub(fields[i], 1, at - 1), string.sub(fields[i], at + 1)
|
|
1581
|
+
if not records[cid] then
|
|
1582
|
+
records[cid] = {}
|
|
1583
|
+
order[#order + 1] = cid
|
|
1584
|
+
end
|
|
1585
|
+
records[cid][suffix] = fields[i + 1]
|
|
1586
|
+
end
|
|
1587
|
+
local idle, moved = {}, {}
|
|
1588
|
+
for _, cid in ipairs(order) do
|
|
1589
|
+
local record = records[cid]
|
|
1590
|
+
local _, tag = parse_cid(cid)
|
|
1591
|
+
local head = (record.h and record.h ~= '') and tonumber(record.h) or false
|
|
1592
|
+
local position = tonumber(record.p or '0')
|
|
1593
|
+
if head and head <= boundary then head = false end
|
|
1594
|
+
if string.sub(tag, 1, 1) == 'G' then
|
|
1595
|
+
if position < boundary then position = boundary end
|
|
1596
|
+
moved[#moved + 1] = cid .. ':h'
|
|
1597
|
+
moved[#moved + 1] = head and tostring(head) or ''
|
|
1598
|
+
moved[#moved + 1] = cid .. ':p'
|
|
1599
|
+
moved[#moved + 1] = tostring(position)
|
|
1600
|
+
elseif not head and position <= boundary then
|
|
1601
|
+
for suffix in pairs(record) do idle[#idle + 1] = cid .. ':' .. suffix end
|
|
1602
|
+
elseif not head and record.h ~= '' then
|
|
1603
|
+
moved[#moved + 1] = cid .. ':h'
|
|
1604
|
+
moved[#moved + 1] = ''
|
|
1605
|
+
end
|
|
1606
|
+
end
|
|
1607
|
+
sliced('HDEL', CURSORS, idle)
|
|
1608
|
+
sliced('HSET', CURSORS, moved)
|
|
1068
1609
|
end
|
|
1069
1610
|
`
|
|
1070
1611
|
|
|
1071
1612
|
/**
|
|
1072
|
-
* KEYS:
|
|
1073
|
-
*
|
|
1074
|
-
*
|
|
1075
|
-
*
|
|
1076
|
-
*
|
|
1077
|
-
*
|
|
1078
|
-
*
|
|
1079
|
-
*
|
|
1080
|
-
*
|
|
1613
|
+
* KEYS: counter, layout, marker id ownership key.
|
|
1614
|
+
* ARGV: sessionId, nowMs, ifLastIndex or '', cause consumer or '', cause tag,
|
|
1615
|
+
* cause index, cause attempt, marker id, notify channel.
|
|
1616
|
+
* Returns ['ok', cleared, head, boundary] | ['superseded'] | ['foreign'].
|
|
1617
|
+
* The old lifetime's log moves aside in one RENAME, so every read of the log
|
|
1618
|
+
* starts at the marker, and its ready set is unlinked, so claims and pending
|
|
1619
|
+
* checks see only the new lifetime; deletion drains the moved log. Type
|
|
1620
|
+
* indexes keep their older entries until then: lookups start above the
|
|
1621
|
+
* marker.
|
|
1081
1622
|
*/
|
|
1082
|
-
const CLEAR_LUA =
|
|
1083
|
-
local
|
|
1084
|
-
${
|
|
1085
|
-
|
|
1086
|
-
local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR, KEYS[5])
|
|
1087
|
-
local head = tonumber(counts[1] or '0')
|
|
1088
|
-
use_split(counts[2], counts[1], counts[3])
|
|
1623
|
+
const CLEAR_LUA = `${BASE_LUA}${WAKE_LUA}
|
|
1624
|
+
local values = read_bounds({ KEYS[3] })
|
|
1625
|
+
if not values then return redis.error_reply('${LAYOUT_ERROR}') end
|
|
1626
|
+
local head = total
|
|
1089
1627
|
if ARGV[4] ~= '' then
|
|
1090
|
-
local
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
if
|
|
1094
|
-
|
|
1095
|
-
if tonumber(row[1] or '0') ~= tonumber(ARGV[5]) or row[2] then
|
|
1096
|
-
return { 'superseded', field }
|
|
1628
|
+
local idx = tonumber(ARGV[6])
|
|
1629
|
+
if idx <= boundary then return { 'superseded' } end
|
|
1630
|
+
local row = read_rows({ { idx, row_of(ARGV[4], ARGV[5]) } }, { 'd', 'f', 'p', 'l' })[1]
|
|
1631
|
+
if tonumber(row[1] or '0') ~= tonumber(ARGV[7]) or row[2] or row[3] or not owns(row[4], ARGV[5]) then
|
|
1632
|
+
return { 'superseded' }
|
|
1097
1633
|
end
|
|
1098
1634
|
end
|
|
1099
1635
|
local expected = ARGV[3]
|
|
@@ -1106,39 +1642,37 @@ end
|
|
|
1106
1642
|
if head == 0 or boundary == head then
|
|
1107
1643
|
return { 'ok', '1', tostring(head), tostring(boundary) }
|
|
1108
1644
|
end
|
|
1109
|
-
if
|
|
1645
|
+
if values[1] then return { 'foreign' } end
|
|
1110
1646
|
local field = tostring(head + 1)
|
|
1111
|
-
local
|
|
1112
|
-
|
|
1113
|
-
redis.call('
|
|
1114
|
-
redis.call('
|
|
1115
|
-
redis.call('
|
|
1116
|
-
|
|
1117
|
-
'
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
wake_add(field, ARGV[
|
|
1125
|
-
wake(ARGV[
|
|
1647
|
+
local moved = SESSION .. 'cleared:' .. field
|
|
1648
|
+
redis.call('RENAME', LOG, moved)
|
|
1649
|
+
redis.call('UNLINK', READY)
|
|
1650
|
+
redis.call('SADD', SESSION .. 'cleared', moved)
|
|
1651
|
+
redis.call('XADD', LOG, field .. '-0',
|
|
1652
|
+
'id', ARGV[8], 'type', CLEAR_TYPE, 'payload', 'null',
|
|
1653
|
+
'created_at', ARGV[2], 'cause', '')
|
|
1654
|
+
local marker = { KEYS[3], cjson.encode({ ARGV[1], head + 1 }), COUNTER, field, CLEAR, field }
|
|
1655
|
+
if schema_missing then
|
|
1656
|
+
marker[#marker + 1] = SCHEMA
|
|
1657
|
+
marker[#marker + 1] = '${REDIS_LAYOUT}'
|
|
1658
|
+
end
|
|
1659
|
+
redis.call('MSET', unpack(marker))
|
|
1660
|
+
wake_add(field, ARGV[8], CLEAR_TYPE, 'null')
|
|
1661
|
+
wake(ARGV[9], field, ARGV[2])
|
|
1126
1662
|
return { 'ok', '1', field, field }
|
|
1127
1663
|
`
|
|
1128
1664
|
|
|
1129
1665
|
/**
|
|
1130
|
-
* KEYS: counter. ARGV: page size.
|
|
1666
|
+
* KEYS: counter, layout. ARGV: page size.
|
|
1131
1667
|
* The next deletion batch: a moved log and up to a page of its [index, id]
|
|
1132
1668
|
* pairs, without payloads. Empty moved logs are dropped; with none left it
|
|
1133
|
-
* sweeps the old
|
|
1134
|
-
* interrupted deletion.
|
|
1669
|
+
* sweeps what the old lifetime left and returns nothing, so a retry
|
|
1670
|
+
* finishes an interrupted deletion.
|
|
1135
1671
|
*/
|
|
1136
|
-
const RECLAIM_BATCH_LUA =
|
|
1137
|
-
|
|
1138
|
-
local session = string.sub(KEYS[1], 1, -6)
|
|
1139
|
-
local boundary = tonumber(redis.call('GET', session .. 'clear') or '0')
|
|
1672
|
+
const RECLAIM_BATCH_LUA = `${BASE_LUA}${SWEEP_LUA}
|
|
1673
|
+
local boundary = tonumber(redis.call('GET', CLEAR) or '0')
|
|
1140
1674
|
if boundary == 0 then return {} end
|
|
1141
|
-
for _, log in ipairs(redis.call('SMEMBERS',
|
|
1675
|
+
for _, log in ipairs(redis.call('SMEMBERS', SESSION .. 'cleared')) do
|
|
1142
1676
|
local entries = redis.call('XRANGE', log, '-', '+', 'COUNT', ARGV[1])
|
|
1143
1677
|
if #entries > 0 then
|
|
1144
1678
|
local batch = { log }
|
|
@@ -1155,78 +1689,36 @@ for _, log in ipairs(redis.call('SMEMBERS', session .. 'cleared')) do
|
|
|
1155
1689
|
return batch
|
|
1156
1690
|
end
|
|
1157
1691
|
redis.call('DEL', log)
|
|
1158
|
-
redis.call('SREM',
|
|
1692
|
+
redis.call('SREM', SESSION .. 'cleared', log)
|
|
1159
1693
|
end
|
|
1160
|
-
|
|
1694
|
+
sweep(boundary)
|
|
1161
1695
|
return {}
|
|
1162
1696
|
`
|
|
1163
1697
|
|
|
1164
1698
|
/**
|
|
1165
|
-
* KEYS:
|
|
1166
|
-
*
|
|
1167
|
-
*
|
|
1168
|
-
*
|
|
1169
|
-
*
|
|
1170
|
-
* reused in the new lifetime keeps its owner. Returns [deleted, done].
|
|
1699
|
+
* KEYS: counter, layout, moved log, then one id ownership key per entry.
|
|
1700
|
+
* ARGV: sessionId, n, then per entry: index, id.
|
|
1701
|
+
* Deletes the listed entries below the latest clear and their id records.
|
|
1702
|
+
* An id record survives when it names another event, so an id reused in the
|
|
1703
|
+
* new lifetime keeps its owner. Returns [deleted, done].
|
|
1171
1704
|
*/
|
|
1172
|
-
const RECLAIM_LUA =
|
|
1173
|
-
local
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
local function lane_field(lane, suffix)
|
|
1177
|
-
return '__lane:' .. tostring(string.len(lane)) .. ':' .. lane .. ':' .. suffix
|
|
1178
|
-
end
|
|
1179
|
-
local session = string.sub(COUNTER, 1, -6)
|
|
1180
|
-
local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR)
|
|
1181
|
-
use_split(counts[2], counts[1], counts[3])
|
|
1182
|
-
local fields, id_keys, indexes, stream_ids, pin_keys, deleted = {}, {}, {}, {}, {}, {}
|
|
1705
|
+
const RECLAIM_LUA = `${BASE_LUA}${SWEEP_LUA}
|
|
1706
|
+
local boundary = tonumber(redis.call('GET', CLEAR) or '0')
|
|
1707
|
+
local MOVED = KEYS[3]
|
|
1708
|
+
local id_keys, indexes, stream_ids = {}, {}, {}
|
|
1183
1709
|
-- A concurrent reclaim may list the same batch; only XDEL says who removed it.
|
|
1184
1710
|
local removed = 0
|
|
1185
|
-
for i = 1, tonumber(ARGV[
|
|
1186
|
-
local field = ARGV[
|
|
1711
|
+
for i = 1, tonumber(ARGV[2]) do
|
|
1712
|
+
local field = ARGV[2 + (i - 1) * 2 + 1]
|
|
1187
1713
|
if tonumber(field) < boundary then
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
stream_ids[#fields] = field .. '-0'
|
|
1192
|
-
pin_keys[#fields] = ARGV[2] .. field
|
|
1193
|
-
deleted[field] = true
|
|
1194
|
-
end
|
|
1195
|
-
end
|
|
1196
|
-
if #fields > 0 then
|
|
1197
|
-
local lanes, seen, pointers = read_fields(fields, { 'l' }), {}, {}
|
|
1198
|
-
for _, field in ipairs(fields) do
|
|
1199
|
-
local lane = lanes[field][1]
|
|
1200
|
-
if lane and lane ~= '' and not seen[lane] then
|
|
1201
|
-
seen[lane] = true
|
|
1202
|
-
pointers[#pointers + 1] = lane_field(lane, 'head')
|
|
1203
|
-
pointers[#pointers + 1] = lane_field(lane, 'tail')
|
|
1204
|
-
end
|
|
1714
|
+
id_keys[#id_keys + 1] = KEYS[3 + i]
|
|
1715
|
+
indexes[#id_keys] = tonumber(field)
|
|
1716
|
+
stream_ids[#id_keys] = field .. '-0'
|
|
1205
1717
|
end
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
local stale = {}
|
|
1209
|
-
for i = 1, #pointers do
|
|
1210
|
-
if values[i] and deleted[values[i]] then stale[#stale + 1] = pointers[i] end
|
|
1211
|
-
end
|
|
1212
|
-
if #stale > 0 then redis.call('HDEL', META, unpack(stale)) end
|
|
1213
|
-
end
|
|
1214
|
-
local suffixes = { 'l', 'next', 'p', 'pa', 'ri', 'ch', 'ce', 'f', 'fc', 'lc', 'lf', 'lfa', 'd', 'a', 'e' }
|
|
1215
|
-
local columns, order = {}, {}
|
|
1216
|
-
for _, field in ipairs(fields) do
|
|
1217
|
-
local key = mkey(field)
|
|
1218
|
-
if not columns[key] then
|
|
1219
|
-
columns[key] = {}
|
|
1220
|
-
order[#order + 1] = key
|
|
1221
|
-
end
|
|
1222
|
-
for _, suffix in ipairs(suffixes) do
|
|
1223
|
-
columns[key][#columns[key] + 1] = field .. ':' .. suffix
|
|
1224
|
-
end
|
|
1225
|
-
end
|
|
1226
|
-
for _, key in ipairs(order) do sliced('HDEL', key, columns[key]) end
|
|
1227
|
-
sliced('DEL', nil, pin_keys)
|
|
1718
|
+
end
|
|
1719
|
+
if #id_keys > 0 then
|
|
1228
1720
|
local owners, stale_ids = sliced('MGET', nil, id_keys), {}
|
|
1229
|
-
for i = 1, #
|
|
1721
|
+
for i = 1, #id_keys do
|
|
1230
1722
|
if owners[i] then
|
|
1231
1723
|
local rec = cjson.decode(owners[i])
|
|
1232
1724
|
if rec[1] == ARGV[1] and tonumber(rec[2]) == indexes[i] then
|
|
@@ -1234,15 +1726,15 @@ if #fields > 0 then
|
|
|
1234
1726
|
end
|
|
1235
1727
|
end
|
|
1236
1728
|
end
|
|
1237
|
-
|
|
1238
|
-
removed = redis.call('XDEL',
|
|
1729
|
+
sliced('DEL', nil, stale_ids)
|
|
1730
|
+
removed = redis.call('XDEL', MOVED, unpack(stream_ids))
|
|
1239
1731
|
end
|
|
1240
1732
|
local done = 0
|
|
1241
|
-
if redis.call('XLEN',
|
|
1242
|
-
redis.call('DEL',
|
|
1243
|
-
redis.call('SREM',
|
|
1244
|
-
if redis.call('SCARD',
|
|
1245
|
-
|
|
1733
|
+
if redis.call('XLEN', MOVED) == 0 then
|
|
1734
|
+
redis.call('DEL', MOVED)
|
|
1735
|
+
redis.call('SREM', SESSION .. 'cleared', MOVED)
|
|
1736
|
+
if redis.call('SCARD', SESSION .. 'cleared') == 0 then
|
|
1737
|
+
sweep(boundary)
|
|
1246
1738
|
done = 1
|
|
1247
1739
|
end
|
|
1248
1740
|
end
|
|
@@ -1276,6 +1768,30 @@ const toEvent = (sessionId: string, entry: StreamEntry): Event => {
|
|
|
1276
1768
|
}
|
|
1277
1769
|
}
|
|
1278
1770
|
|
|
1771
|
+
/**
|
|
1772
|
+
* A stream entry's causal edge. Releases before consumers (0.17 to 0.20)
|
|
1773
|
+
* recorded only the parent's index and attempt: their edge reads as the
|
|
1774
|
+
* handlers table's, `a2.handlers`, as Postgres's upgrade names it, with no
|
|
1775
|
+
* lane, since nothing reads an earlier release's lane.
|
|
1776
|
+
*/
|
|
1777
|
+
const decodeStreamCause = (raw: string): EventCause | null => {
|
|
1778
|
+
const parsed = JSON.parse(raw) as { consumer?: unknown }
|
|
1779
|
+
return decodeCause(
|
|
1780
|
+
parsed.consumer === undefined
|
|
1781
|
+
? { consumer: 'a2.handlers', lane: null, ...parsed }
|
|
1782
|
+
: parsed,
|
|
1783
|
+
)
|
|
1784
|
+
}
|
|
1785
|
+
|
|
1786
|
+
/** A stream entry with its causal edge. */
|
|
1787
|
+
const toStoredEvent = (sessionId: string, entry: StreamEntry): StoredEvent => {
|
|
1788
|
+
const cause = fieldMap(entry[1])['cause']
|
|
1789
|
+
return Object.assign(toEvent(sessionId, entry), {
|
|
1790
|
+
cause:
|
|
1791
|
+
cause === undefined || cause === '' ? null : decodeStreamCause(cause),
|
|
1792
|
+
})
|
|
1793
|
+
}
|
|
1794
|
+
|
|
1279
1795
|
type ByteStreamEntry = [Uint8Array, Uint8Array[]]
|
|
1280
1796
|
|
|
1281
1797
|
const fieldBytes = (name: string): Uint8Array => new TextEncoder().encode(name)
|
|
@@ -1338,75 +1854,107 @@ const toSerializedEvent = (
|
|
|
1338
1854
|
|
|
1339
1855
|
const json = (value: unknown): string => JSON.stringify(value) ?? 'null'
|
|
1340
1856
|
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1857
|
+
/** How scripts name a cursor within its consumer: `#`, or `L` and its lane. */
|
|
1858
|
+
const tagOf = (lane: string | null): string =>
|
|
1859
|
+
lane === null ? '#' : `L${lane}`
|
|
1860
|
+
|
|
1861
|
+
/** The lane a cursor tag names: null for an event cursor. */
|
|
1862
|
+
const laneOf = (tag: string): string | null =>
|
|
1863
|
+
tag.startsWith('#') ? null : tag.slice(1)
|
|
1864
|
+
|
|
1865
|
+
const targetsArg = (targets: readonly EventTarget[] | undefined): string =>
|
|
1866
|
+
targets === undefined || targets.length === 0
|
|
1867
|
+
? ''
|
|
1868
|
+
: JSON.stringify(
|
|
1869
|
+
targets.map((target) => [target.consumer, tagOf(target.lane)]),
|
|
1870
|
+
)
|
|
1871
|
+
|
|
1872
|
+
/** A log consumer's types, as scripts read them; '*' takes every type. */
|
|
1873
|
+
const logTypesArg = (types: readonly string[] | null): string =>
|
|
1874
|
+
types === null ? '*' : JSON.stringify(types)
|
|
1875
|
+
|
|
1876
|
+
/** The served consumers, as scripts read them; '' serves every consumer. */
|
|
1877
|
+
const servedArg = (consumers: readonly StoreConsumer[] | undefined): string =>
|
|
1878
|
+
consumers === undefined
|
|
1879
|
+
? ''
|
|
1880
|
+
: JSON.stringify(
|
|
1881
|
+
consumers.map((consumer) =>
|
|
1882
|
+
consumer.log
|
|
1883
|
+
? [
|
|
1884
|
+
consumer.name,
|
|
1885
|
+
consumer.log.lane,
|
|
1886
|
+
logTypesArg(consumer.log.types),
|
|
1887
|
+
consumer.log.backfill ? 1 : 0,
|
|
1888
|
+
]
|
|
1889
|
+
: [consumer.name],
|
|
1890
|
+
),
|
|
1891
|
+
)
|
|
1892
|
+
|
|
1893
|
+
type Column = string | number | null | undefined
|
|
1894
|
+
|
|
1895
|
+
/** A head's dispatch state from a row's HEAD_COLUMNS. */
|
|
1896
|
+
const dispatchOf = (
|
|
1897
|
+
consumer: string,
|
|
1898
|
+
lane: string | null,
|
|
1899
|
+
columns: readonly Column[] | null | undefined,
|
|
1900
|
+
): HeadDispatch => {
|
|
1901
|
+
const column = (at: number): string | null => {
|
|
1902
|
+
const value = columns?.[at]
|
|
1903
|
+
return value === null || value === undefined ? null : String(value)
|
|
1347
1904
|
}
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
Number(value
|
|
1351
|
-
!Number.isInteger(value.attempt) ||
|
|
1352
|
-
Number(value.attempt) < 1
|
|
1353
|
-
) {
|
|
1354
|
-
throw new TypeError('stored event has an invalid cause')
|
|
1905
|
+
const date = (at: number): Date | null => {
|
|
1906
|
+
const value = column(at)
|
|
1907
|
+
return value === null ? null : new Date(Number(value))
|
|
1355
1908
|
}
|
|
1356
|
-
|
|
1357
|
-
value
|
|
1358
|
-
|
|
1359
|
-
) {
|
|
1360
|
-
throw new TypeError('stored event has an invalid cause')
|
|
1909
|
+
const count = (at: number): number | null => {
|
|
1910
|
+
const value = column(at)
|
|
1911
|
+
return value === null ? null : Number(value)
|
|
1361
1912
|
}
|
|
1362
1913
|
return {
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1914
|
+
consumer,
|
|
1915
|
+
lane,
|
|
1916
|
+
firstClaimedAt: date(1),
|
|
1917
|
+
lastClaimedAt: date(2),
|
|
1918
|
+
attemptCount: count(0) ?? 0,
|
|
1919
|
+
claimHolder: column(3),
|
|
1920
|
+
claimExpiresAt: date(4),
|
|
1921
|
+
failureCount: count(5) ?? 0,
|
|
1922
|
+
lastFailedAt: date(6),
|
|
1923
|
+
lastFailedAttempt: count(7),
|
|
1924
|
+
lastError: column(8),
|
|
1925
|
+
failedAt: date(9),
|
|
1368
1926
|
}
|
|
1369
1927
|
}
|
|
1370
1928
|
|
|
1371
|
-
type
|
|
1372
|
-
|
|
1373
|
-
|
|
1374
|
-
|
|
1375
|
-
|
|
1929
|
+
type HeadTuple = [
|
|
1930
|
+
consumer: string,
|
|
1931
|
+
tag: string,
|
|
1932
|
+
index: string,
|
|
1933
|
+
columns: Column[],
|
|
1376
1934
|
entry: StreamEntry,
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
attemptCount: count('d') ?? 0,
|
|
1401
|
-
claimHolder: column('ch') ?? null,
|
|
1402
|
-
claimExpiresAt: date('ce'),
|
|
1403
|
-
failureCount: count('a') ?? 0,
|
|
1404
|
-
lastFailedAt: date('lf'),
|
|
1405
|
-
lastFailedAttempt: count('lfa'),
|
|
1406
|
-
lastError: column('e') ?? null,
|
|
1407
|
-
failedAt: date('f'),
|
|
1408
|
-
})
|
|
1409
|
-
}
|
|
1935
|
+
]
|
|
1936
|
+
|
|
1937
|
+
const toHeadEvent = (sessionId: string, head: HeadTuple): HeadEvent => ({
|
|
1938
|
+
...toStoredEvent(sessionId, head[4]),
|
|
1939
|
+
...dispatchOf(head[0], laneOf(head[1]), head[3]),
|
|
1940
|
+
})
|
|
1941
|
+
|
|
1942
|
+
const compareText = (a: string, b: string): number =>
|
|
1943
|
+
a < b ? -1 : a > b ? 1 : 0
|
|
1944
|
+
|
|
1945
|
+
const byHead = (a: HeadRef, b: HeadRef): number =>
|
|
1946
|
+
a.index - b.index || compareText(a.consumer, b.consumer)
|
|
1947
|
+
|
|
1948
|
+
/** Consumer, then lane, with concurrent cursors last by head. */
|
|
1949
|
+
const byCursor = (a: StoredCursor, b: StoredCursor): number =>
|
|
1950
|
+
compareText(a.consumer, b.consumer) ||
|
|
1951
|
+
(a.lane === null
|
|
1952
|
+
? b.lane === null
|
|
1953
|
+
? (a.head ?? 0) - (b.head ?? 0)
|
|
1954
|
+
: 1
|
|
1955
|
+
: b.lane === null
|
|
1956
|
+
? -1
|
|
1957
|
+
: compareText(a.lane, b.lane))
|
|
1410
1958
|
|
|
1411
1959
|
const toClaimAvailableResult = (
|
|
1412
1960
|
sessionId: string,
|
|
@@ -1415,57 +1963,13 @@ const toClaimAvailableResult = (
|
|
|
1415
1963
|
const outcome = String(reply[0])
|
|
1416
1964
|
if (outcome === 'claimed') {
|
|
1417
1965
|
const count = Number(reply[1])
|
|
1418
|
-
const events
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
const event = toEvent(sessionId, entry)
|
|
1423
|
-
const fields = fieldMap(entry[1])
|
|
1424
|
-
events.push(
|
|
1425
|
-
Object.assign(event, {
|
|
1426
|
-
cause: toCause(fields['cause']),
|
|
1427
|
-
lane: fields['lane'] === '' ? null : fields['lane']!,
|
|
1428
|
-
processedAt: null,
|
|
1429
|
-
processedByAttempt: null,
|
|
1430
|
-
returnedEventIds: null,
|
|
1431
|
-
firstClaimedAt:
|
|
1432
|
-
reply[offset + 3] === null
|
|
1433
|
-
? null
|
|
1434
|
-
: new Date(Number(reply[offset + 3])),
|
|
1435
|
-
lastClaimedAt:
|
|
1436
|
-
reply[offset + 4] === null
|
|
1437
|
-
? null
|
|
1438
|
-
: new Date(Number(reply[offset + 4])),
|
|
1439
|
-
attemptCount: Number(reply[offset]),
|
|
1440
|
-
claimHolder:
|
|
1441
|
-
reply[offset + 7] === null ? null : String(reply[offset + 7]),
|
|
1442
|
-
claimExpiresAt:
|
|
1443
|
-
reply[offset + 8] === null
|
|
1444
|
-
? null
|
|
1445
|
-
: new Date(Number(reply[offset + 8])),
|
|
1446
|
-
failureCount: Number(reply[offset + 1]),
|
|
1447
|
-
lastFailedAt:
|
|
1448
|
-
reply[offset + 5] === null
|
|
1449
|
-
? null
|
|
1450
|
-
: new Date(Number(reply[offset + 5])),
|
|
1451
|
-
lastFailedAttempt:
|
|
1452
|
-
reply[offset + 6] === null ? null : Number(reply[offset + 6]),
|
|
1453
|
-
lastError:
|
|
1454
|
-
reply[offset + 2] === null ? null : String(reply[offset + 2]),
|
|
1455
|
-
failedAt:
|
|
1456
|
-
reply[offset + 10] === null
|
|
1457
|
-
? null
|
|
1458
|
-
: new Date(Number(reply[offset + 10])),
|
|
1459
|
-
}),
|
|
1460
|
-
)
|
|
1461
|
-
}
|
|
1462
|
-
const clearIndex = Number(reply[2 + count * 11] ?? 0)
|
|
1966
|
+
const events = (reply.slice(2, 2 + count) as HeadTuple[])
|
|
1967
|
+
.map((head) => toHeadEvent(sessionId, head))
|
|
1968
|
+
.toSorted(byHead)
|
|
1969
|
+
const clearIndex = Number(reply[2 + count] ?? 0)
|
|
1463
1970
|
if (clearIndex > 0)
|
|
1464
1971
|
for (const event of events) event.clearIndex = clearIndex
|
|
1465
|
-
return {
|
|
1466
|
-
outcome: 'claimed',
|
|
1467
|
-
events,
|
|
1468
|
-
}
|
|
1972
|
+
return { outcome: 'claimed', events, head: Number(reply[3 + count]) }
|
|
1469
1973
|
}
|
|
1470
1974
|
if (outcome === 'busy') {
|
|
1471
1975
|
return { outcome, retryAt: new Date(Number(reply[1])) }
|
|
@@ -1547,24 +2051,14 @@ export function createRedisStoreCore(
|
|
|
1547
2051
|
|
|
1548
2052
|
const storeKey = (s: string): string => `${prefix}:${s}:log`
|
|
1549
2053
|
const countKey = (s: string): string => `${prefix}:${s}:count`
|
|
1550
|
-
|
|
1551
|
-
const
|
|
1552
|
-
|
|
1553
|
-
const metaSplitKey = (s: string): string => `${prefix}:${s}:meta-split`
|
|
1554
|
-
const pendingKey = (s: string): string => `${prefix}:${s}:pending`
|
|
1555
|
-
const readyKey = (s: string): string => `${prefix}:${s}:ready`
|
|
2054
|
+
// Scripts derive the session's other keys from the counter; keep them in step.
|
|
2055
|
+
const clearKey = (s: string): string => `${prefix}:${s}:clear`
|
|
2056
|
+
const schemaKey = `${prefix}:schema`
|
|
1556
2057
|
const snapKey = (s: string, r: string): string => `${prefix}:${s}:snap:${r}`
|
|
1557
2058
|
const snapHistoryKey = (s: string, r: string): string =>
|
|
1558
2059
|
`${prefix}:${s}:snap-history:${r}`
|
|
1559
|
-
const snapshotEventPinsKey = (s: string, index: number): string =>
|
|
1560
|
-
`${prefix}:${s}:snapshot-pins:${index}`
|
|
1561
|
-
const snapshotEventPinsPrefix = (s: string): string =>
|
|
1562
|
-
`${prefix}:${s}:snapshot-pins:`
|
|
1563
|
-
const snapshotPinCountsKey = (s: string): string =>
|
|
1564
|
-
`${prefix}:${s}:snapshot-pin-counts`
|
|
1565
2060
|
const presenceKey = (s: string): string => `${prefix}:${s}:presence`
|
|
1566
|
-
|
|
1567
|
-
const clearKey = (s: string): string => `${prefix}:${s}:clear`
|
|
2061
|
+
const notifyChannel = (s: string): string => `${prefix}:${s}:notify`
|
|
1568
2062
|
// One ownership key per event id; reclaim deletes those naming cleared events.
|
|
1569
2063
|
// Session ids are raw and may contain colons, so a raw id could name a
|
|
1570
2064
|
// session key (`ids:count`). A JSON string with escaped colons starts with
|
|
@@ -1572,92 +2066,74 @@ export function createRedisStoreCore(
|
|
|
1572
2066
|
const idKey = (id: string): string =>
|
|
1573
2067
|
`${prefix}:ids:${JSON.stringify(id).replaceAll(':', '\\u003a')}`
|
|
1574
2068
|
|
|
1575
|
-
const
|
|
2069
|
+
const layoutError = (): A2Error =>
|
|
2070
|
+
new A2Error(
|
|
2071
|
+
'STORE_NOT_CONFIGURED',
|
|
2072
|
+
`the redis keys under prefix '${prefix}' hold an A2 layout other than '${REDIS_LAYOUT}', from another release; point the store at a fresh keyPrefix, or give the contract a new name so it starts new sessions`,
|
|
2073
|
+
)
|
|
2074
|
+
|
|
2075
|
+
/** Runs a script over a session: KEYS start with its counter and the layout key. */
|
|
2076
|
+
const evalScript = async (
|
|
1576
2077
|
script: string,
|
|
2078
|
+
sessionId: string,
|
|
1577
2079
|
keys: string[],
|
|
1578
2080
|
args: Array<string | number>,
|
|
1579
|
-
): Promise<unknown> =>
|
|
2081
|
+
): Promise<unknown> => {
|
|
2082
|
+
const allKeys = [countKey(sessionId), schemaKey, ...keys]
|
|
2083
|
+
try {
|
|
2084
|
+
return await call('EVAL', script, allKeys.length, ...allKeys, ...args)
|
|
2085
|
+
} catch (error) {
|
|
2086
|
+
if (String((error as Error)?.message).includes(LAYOUT_ERROR))
|
|
2087
|
+
throw layoutError()
|
|
2088
|
+
throw error
|
|
2089
|
+
}
|
|
2090
|
+
}
|
|
1580
2091
|
|
|
1581
|
-
|
|
2092
|
+
/**
|
|
2093
|
+
* Runs a write that may need a type's index to reach further back first.
|
|
2094
|
+
* Such a write returns ['index', type, after] having written nothing, and
|
|
2095
|
+
* runs again once INDEX_TYPE_LUA has indexed that history, a page per call.
|
|
2096
|
+
*/
|
|
2097
|
+
const indexing = async (
|
|
1582
2098
|
sessionId: string,
|
|
1583
|
-
|
|
1584
|
-
|
|
2099
|
+
run: () => Promise<unknown>,
|
|
2100
|
+
): Promise<unknown[]> => {
|
|
2101
|
+
for (;;) {
|
|
2102
|
+
// oxlint-disable-next-line no-await-in-loop -- each run follows the indexing it asked for
|
|
2103
|
+
const reply = (await run()) as unknown[]
|
|
2104
|
+
if (reply[0] !== 'index') return reply
|
|
2105
|
+
for (;;) {
|
|
2106
|
+
// oxlint-disable-next-line no-await-in-loop -- each page lowers the index's floor
|
|
2107
|
+
const [done] = (await evalScript(
|
|
2108
|
+
INDEX_TYPE_LUA,
|
|
2109
|
+
sessionId,
|
|
2110
|
+
[],
|
|
2111
|
+
[String(reply[1]), Number(reply[2])],
|
|
2112
|
+
)) as [string]
|
|
2113
|
+
if (done === '1') break
|
|
2114
|
+
}
|
|
2115
|
+
}
|
|
2116
|
+
}
|
|
2117
|
+
|
|
2118
|
+
/** The frontier and clear marker, checking the layout in the same MGET. */
|
|
2119
|
+
const readBounds = async (
|
|
2120
|
+
sessionId: string,
|
|
2121
|
+
): Promise<{ frontier: number; clearIndex: number }> => {
|
|
2122
|
+
const [count, cleared, schema] = (await call(
|
|
1585
2123
|
'MGET',
|
|
1586
2124
|
countKey(sessionId),
|
|
1587
|
-
metaSplitKey(sessionId),
|
|
1588
2125
|
clearKey(sessionId),
|
|
2126
|
+
schemaKey,
|
|
1589
2127
|
)) as [unknown, unknown, unknown]
|
|
1590
|
-
|
|
2128
|
+
if (schema !== null && schema !== undefined && schema !== REDIS_LAYOUT) {
|
|
2129
|
+
throw layoutError()
|
|
2130
|
+
}
|
|
1591
2131
|
return {
|
|
1592
|
-
frontier,
|
|
1593
|
-
metaSplit:
|
|
1594
|
-
split === null
|
|
1595
|
-
? frontier
|
|
1596
|
-
: toIndex({ raw: split, sessionId, name: 'metadata split' }),
|
|
2132
|
+
frontier: toIndex({ raw: count, sessionId, name: 'event count' }),
|
|
1597
2133
|
clearIndex: toIndex({ raw: cleared, sessionId, name: 'clear boundary' }),
|
|
1598
2134
|
}
|
|
1599
2135
|
}
|
|
1600
2136
|
|
|
1601
|
-
const readRangePage = async ({
|
|
1602
|
-
sessionId,
|
|
1603
|
-
afterIndex,
|
|
1604
|
-
throughIndex,
|
|
1605
|
-
metaSplit,
|
|
1606
|
-
}: {
|
|
1607
|
-
sessionId: string
|
|
1608
|
-
afterIndex: number
|
|
1609
|
-
throughIndex: number
|
|
1610
|
-
metaSplit: number
|
|
1611
|
-
}): Promise<StoredEvent[]> => {
|
|
1612
|
-
if (throughIndex <= afterIndex) return []
|
|
1613
|
-
const entries = (await call(
|
|
1614
|
-
'XRANGE',
|
|
1615
|
-
storeKey(sessionId),
|
|
1616
|
-
`${afterIndex + 1}-0`,
|
|
1617
|
-
`${throughIndex}-0`,
|
|
1618
|
-
'COUNT',
|
|
1619
|
-
READ_PAGE_SIZE,
|
|
1620
|
-
)) as StreamEntry[]
|
|
1621
|
-
if (entries.length === 0) return []
|
|
1622
|
-
|
|
1623
|
-
const fieldsByKey = new Map<string, string[]>()
|
|
1624
|
-
for (const entry of entries) {
|
|
1625
|
-
const index = entryIndex(entry[0])
|
|
1626
|
-
const key =
|
|
1627
|
-
index <= metaSplit ? metaKey(sessionId) : metaChunkKey(sessionId, index)
|
|
1628
|
-
const fields = fieldsByKey.get(key) ?? []
|
|
1629
|
-
fields.push(
|
|
1630
|
-
...STORED_EVENT_META_SUFFIXES.map((suffix) => `${index}:${suffix}`),
|
|
1631
|
-
)
|
|
1632
|
-
fieldsByKey.set(key, fields)
|
|
1633
|
-
}
|
|
1634
|
-
const lookups = [...fieldsByKey]
|
|
1635
|
-
const results = await Promise.allSettled(
|
|
1636
|
-
lookups.map(([key, fields]) => call('HMGET', key, ...fields)),
|
|
1637
|
-
)
|
|
1638
|
-
const meta: Record<string, string> = {}
|
|
1639
|
-
for (let position = 0; position < lookups.length; position += 1) {
|
|
1640
|
-
const result = results[position]!
|
|
1641
|
-
if (result.status === 'rejected') throw result.reason
|
|
1642
|
-
const fields = lookups[position]![1]
|
|
1643
|
-
const values = (result.value as Array<string | null> | null) ?? []
|
|
1644
|
-
for (let i = 0; i < fields.length; i += 1) {
|
|
1645
|
-
const value = values[i]
|
|
1646
|
-
if (value !== null && value !== undefined) {
|
|
1647
|
-
meta[fields[i]!] = value
|
|
1648
|
-
}
|
|
1649
|
-
}
|
|
1650
|
-
}
|
|
1651
|
-
return entries.map((entry) => {
|
|
1652
|
-
const idx = entryIndex(entry[0])
|
|
1653
|
-
return toStoredEvent(
|
|
1654
|
-
sessionId,
|
|
1655
|
-
entry,
|
|
1656
|
-
(suffix) => meta[`${idx}:${suffix}`],
|
|
1657
|
-
)
|
|
1658
|
-
})
|
|
1659
|
-
}
|
|
1660
|
-
|
|
1661
2137
|
const readEventPages = async (
|
|
1662
2138
|
sessionId: string,
|
|
1663
2139
|
afterIndex: number,
|
|
@@ -1744,11 +2220,8 @@ export function createRedisStoreCore(
|
|
|
1744
2220
|
): Promise<{ rows: StoredEvent[]; clearIndex: number }> => {
|
|
1745
2221
|
if (throughIndex !== undefined && throughIndex <= afterIndex)
|
|
1746
2222
|
return { rows: [], clearIndex: 0 }
|
|
1747
|
-
const {
|
|
1748
|
-
|
|
1749
|
-
metaSplit,
|
|
1750
|
-
clearIndex,
|
|
1751
|
-
} = await readRangeBounds(sessionId)
|
|
2223
|
+
const { frontier: currentFrontier, clearIndex } =
|
|
2224
|
+
await readBounds(sessionId)
|
|
1752
2225
|
const frontier = Math.min(throughIndex ?? currentFrontier, currentFrontier)
|
|
1753
2226
|
// Reads start at the latest clear marker.
|
|
1754
2227
|
const after = Math.max(afterIndex, clearIndex - 1)
|
|
@@ -1758,15 +2231,18 @@ export function createRedisStoreCore(
|
|
|
1758
2231
|
let nextIndex = after + 1
|
|
1759
2232
|
while (nextIndex <= frontier) {
|
|
1760
2233
|
// oxlint-disable-next-line no-await-in-loop -- each page advances the dense frozen range
|
|
1761
|
-
const
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
2234
|
+
const entries = (await call(
|
|
2235
|
+
'XRANGE',
|
|
2236
|
+
storeKey(sessionId),
|
|
2237
|
+
`${nextIndex}-0`,
|
|
2238
|
+
`${frontier}-0`,
|
|
2239
|
+
'COUNT',
|
|
2240
|
+
READ_PAGE_SIZE,
|
|
2241
|
+
)) as StreamEntry[]
|
|
2242
|
+
if (entries.length === 0) {
|
|
1768
2243
|
throw rangeGap(sessionId, `ended before index ${frontier}`)
|
|
1769
2244
|
}
|
|
2245
|
+
const page = entries.map((entry) => toStoredEvent(sessionId, entry))
|
|
1770
2246
|
for (let position = 0; position < page.length; position += 1) {
|
|
1771
2247
|
if (page[position]!.index !== nextIndex + position) {
|
|
1772
2248
|
throw rangeGap(sessionId, 'is not contiguous')
|
|
@@ -1836,36 +2312,29 @@ export function createRedisStoreCore(
|
|
|
1836
2312
|
type: e.type,
|
|
1837
2313
|
payload: e.payload,
|
|
1838
2314
|
cause: e.cause ? { ...e.cause } : null,
|
|
1839
|
-
|
|
1840
|
-
settled: e.settled === true,
|
|
2315
|
+
targets: e.targets,
|
|
1841
2316
|
}))
|
|
1842
|
-
const
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
pendingKey(sessionId),
|
|
1864
|
-
readyKey(sessionId),
|
|
1865
|
-
metaSplitKey(sessionId),
|
|
1866
|
-
...withIds.map((e) => idKey(e.id)),
|
|
1867
|
-
],
|
|
1868
|
-
args,
|
|
2317
|
+
const reply = (await indexing(sessionId, () =>
|
|
2318
|
+
evalScript(
|
|
2319
|
+
APPEND_LUA,
|
|
2320
|
+
sessionId,
|
|
2321
|
+
withIds.map((e) => idKey(e.id)),
|
|
2322
|
+
[
|
|
2323
|
+
sessionId,
|
|
2324
|
+
now,
|
|
2325
|
+
withIds.length,
|
|
2326
|
+
servedArg(appendOptions?.consumers),
|
|
2327
|
+
notifyChannel(sessionId),
|
|
2328
|
+
appendOptions?.originIndex ?? '',
|
|
2329
|
+
...withIds.flatMap((e) => [
|
|
2330
|
+
e.id,
|
|
2331
|
+
e.type,
|
|
2332
|
+
json(e.payload),
|
|
2333
|
+
e.cause ? json(e.cause) : '',
|
|
2334
|
+
targetsArg(e.targets),
|
|
2335
|
+
]),
|
|
2336
|
+
],
|
|
2337
|
+
),
|
|
1869
2338
|
)) as string[]
|
|
1870
2339
|
|
|
1871
2340
|
const [verdict] = reply
|
|
@@ -1878,20 +2347,15 @@ export function createRedisStoreCore(
|
|
|
1878
2347
|
`event id '${reply[1]}' already exists in another session`,
|
|
1879
2348
|
)
|
|
1880
2349
|
}
|
|
1881
|
-
if (verdict === 'no_parent') {
|
|
1882
|
-
throw new TypeError(
|
|
1883
|
-
`no event at index ${reply[1]} in session '${sessionId}'`,
|
|
1884
|
-
)
|
|
1885
|
-
}
|
|
1886
2350
|
if (verdict === 'cleared') {
|
|
1887
2351
|
throw sessionClearedError(
|
|
1888
2352
|
`scheduled work from event ${reply[1]} was cleared in session '${sessionId}'`,
|
|
1889
2353
|
)
|
|
1890
2354
|
}
|
|
1891
2355
|
if (verdict === 'superseded') {
|
|
1892
|
-
throw
|
|
1893
|
-
|
|
1894
|
-
|
|
2356
|
+
throw supersededAttemptError(
|
|
2357
|
+
sessionId,
|
|
2358
|
+
withIds[Number(reply[1]) - 1]!.cause!,
|
|
1895
2359
|
)
|
|
1896
2360
|
}
|
|
1897
2361
|
if (verdict === 'partial') {
|
|
@@ -1933,26 +2397,39 @@ export function createRedisStoreCore(
|
|
|
1933
2397
|
sessionId,
|
|
1934
2398
|
createdAt: new Date(now),
|
|
1935
2399
|
cause: e.cause,
|
|
1936
|
-
lane: e.lane,
|
|
1937
|
-
processedAt: e.settled ? new Date(now) : null,
|
|
1938
|
-
processedByAttempt: null,
|
|
1939
|
-
returnedEventIds: null,
|
|
1940
|
-
firstClaimedAt: null,
|
|
1941
|
-
lastClaimedAt: null,
|
|
1942
|
-
attemptCount: 0,
|
|
1943
|
-
claimHolder: null,
|
|
1944
|
-
claimExpiresAt: null,
|
|
1945
|
-
failureCount: 0,
|
|
1946
|
-
lastFailedAt: null,
|
|
1947
|
-
lastFailedAttempt: null,
|
|
1948
|
-
lastError: null,
|
|
1949
|
-
failedAt: null,
|
|
1950
2400
|
})),
|
|
1951
2401
|
hasPending: reply[2] === '1',
|
|
1952
2402
|
}
|
|
1953
2403
|
})
|
|
1954
2404
|
},
|
|
1955
2405
|
|
|
2406
|
+
async cursors(sessionId) {
|
|
2407
|
+
return wrap(async () => {
|
|
2408
|
+
const reply = (await evalScript(
|
|
2409
|
+
CURSORS_LUA,
|
|
2410
|
+
sessionId,
|
|
2411
|
+
[],
|
|
2412
|
+
[],
|
|
2413
|
+
)) as Array<
|
|
2414
|
+
[
|
|
2415
|
+
consumer: string,
|
|
2416
|
+
tag: string,
|
|
2417
|
+
position: string,
|
|
2418
|
+
head: string | null,
|
|
2419
|
+
columns: Column[],
|
|
2420
|
+
]
|
|
2421
|
+
>
|
|
2422
|
+
return reply
|
|
2423
|
+
.map(([consumer, tag, position, head, columns]): StoredCursor =>
|
|
2424
|
+
Object.assign(dispatchOf(consumer, laneOf(tag), columns), {
|
|
2425
|
+
position: Number(position),
|
|
2426
|
+
head: head === null ? null : Number(head),
|
|
2427
|
+
}),
|
|
2428
|
+
)
|
|
2429
|
+
.toSorted(byCursor)
|
|
2430
|
+
})
|
|
2431
|
+
},
|
|
2432
|
+
|
|
1956
2433
|
async read(sessionId, opts) {
|
|
1957
2434
|
return wrap(() =>
|
|
1958
2435
|
retryAfterGap(() =>
|
|
@@ -1967,21 +2444,7 @@ export function createRedisStoreCore(
|
|
|
1967
2444
|
return []
|
|
1968
2445
|
return wrap(() =>
|
|
1969
2446
|
retryAfterGap(async () => {
|
|
1970
|
-
const
|
|
1971
|
-
'MGET',
|
|
1972
|
-
countKey(sessionId),
|
|
1973
|
-
clearKey(sessionId),
|
|
1974
|
-
)) as [unknown, unknown]
|
|
1975
|
-
const frontier = toIndex({
|
|
1976
|
-
raw: count,
|
|
1977
|
-
sessionId,
|
|
1978
|
-
name: 'event count',
|
|
1979
|
-
})
|
|
1980
|
-
const clearIndex = toIndex({
|
|
1981
|
-
raw: cleared,
|
|
1982
|
-
sessionId,
|
|
1983
|
-
name: 'clear boundary',
|
|
1984
|
-
})
|
|
2447
|
+
const { frontier, clearIndex } = await readBounds(sessionId)
|
|
1985
2448
|
// Reads start at the latest clear marker.
|
|
1986
2449
|
return readEventRange(
|
|
1987
2450
|
sessionId,
|
|
@@ -1997,32 +2460,31 @@ export function createRedisStoreCore(
|
|
|
1997
2460
|
holder,
|
|
1998
2461
|
ttlMs,
|
|
1999
2462
|
expiresAtMs,
|
|
2000
|
-
|
|
2463
|
+
exclude,
|
|
2464
|
+
consumers,
|
|
2001
2465
|
maxFailures,
|
|
2466
|
+
through,
|
|
2002
2467
|
}) {
|
|
2003
2468
|
return wrap(async () => {
|
|
2004
2469
|
const now = clock.now().getTime()
|
|
2005
2470
|
const expiresAt = expiresAtMs ?? now + ttlMs
|
|
2006
|
-
const excluded =
|
|
2471
|
+
const excluded = exclude ?? []
|
|
2007
2472
|
const reply = (await evalScript(
|
|
2008
2473
|
CLAIM_AVAILABLE_LUA,
|
|
2009
|
-
|
|
2010
|
-
|
|
2011
|
-
countKey(sessionId),
|
|
2012
|
-
metaKey(sessionId),
|
|
2013
|
-
pendingKey(sessionId),
|
|
2014
|
-
readyKey(sessionId),
|
|
2015
|
-
metaSplitKey(sessionId),
|
|
2016
|
-
snapshotPinCountsKey(sessionId),
|
|
2017
|
-
],
|
|
2474
|
+
sessionId,
|
|
2475
|
+
[],
|
|
2018
2476
|
[
|
|
2019
2477
|
holder,
|
|
2020
2478
|
now,
|
|
2021
2479
|
expiresAt,
|
|
2022
|
-
excluded.length,
|
|
2023
|
-
...excluded,
|
|
2024
2480
|
maxFailures ?? '',
|
|
2025
|
-
|
|
2481
|
+
servedArg(consumers),
|
|
2482
|
+
through ?? '',
|
|
2483
|
+
excluded.length,
|
|
2484
|
+
...excluded.flatMap((ref) => [
|
|
2485
|
+
ref.consumer,
|
|
2486
|
+
ref.lane === null ? `#${ref.index}` : `L${ref.lane}`,
|
|
2487
|
+
]),
|
|
2026
2488
|
],
|
|
2027
2489
|
)) as unknown[]
|
|
2028
2490
|
return toClaimAvailableResult(sessionId, reply)
|
|
@@ -2030,30 +2492,48 @@ export function createRedisStoreCore(
|
|
|
2030
2492
|
},
|
|
2031
2493
|
|
|
2032
2494
|
async renewClaims({ sessionId, holder, claims, ttlMs, expiresAtMs }) {
|
|
2495
|
+
if (claims.length === 0) return { renewed: [], superseded: [] }
|
|
2033
2496
|
return wrap(async () => {
|
|
2034
2497
|
const now = clock.now().getTime()
|
|
2035
2498
|
const expiresAt = expiresAtMs ?? now + ttlMs
|
|
2036
2499
|
const reply = (await evalScript(
|
|
2037
2500
|
RENEW_CLAIMS_LUA,
|
|
2038
|
-
|
|
2501
|
+
sessionId,
|
|
2502
|
+
[],
|
|
2039
2503
|
[
|
|
2040
2504
|
holder,
|
|
2041
2505
|
now,
|
|
2042
2506
|
expiresAt,
|
|
2043
2507
|
claims.length,
|
|
2044
|
-
...claims.flatMap((claim) => [
|
|
2508
|
+
...claims.flatMap((claim) => [
|
|
2509
|
+
claim.consumer,
|
|
2510
|
+
tagOf(claim.lane),
|
|
2511
|
+
claim.index,
|
|
2512
|
+
claim.attempt,
|
|
2513
|
+
]),
|
|
2045
2514
|
],
|
|
2046
2515
|
)) as string[]
|
|
2047
2516
|
const renewedCount = Number(reply[0])
|
|
2048
|
-
const
|
|
2517
|
+
const refs = reply.slice(1).map((position): HeadRef => {
|
|
2518
|
+
const { consumer, lane, index } = claims[Number(position) - 1]!
|
|
2519
|
+
return { consumer, lane, index }
|
|
2520
|
+
})
|
|
2049
2521
|
return {
|
|
2050
|
-
renewed:
|
|
2051
|
-
superseded:
|
|
2522
|
+
renewed: refs.slice(0, renewedCount).toSorted(byHead),
|
|
2523
|
+
superseded: refs.slice(renewedCount).toSorted(byHead),
|
|
2052
2524
|
}
|
|
2053
2525
|
})
|
|
2054
2526
|
},
|
|
2055
2527
|
|
|
2056
|
-
async completeAttempt({
|
|
2528
|
+
async completeAttempt({
|
|
2529
|
+
sessionId,
|
|
2530
|
+
consumer,
|
|
2531
|
+
lane,
|
|
2532
|
+
index,
|
|
2533
|
+
attempt,
|
|
2534
|
+
events,
|
|
2535
|
+
consumers,
|
|
2536
|
+
}) {
|
|
2057
2537
|
return wrap(async () => {
|
|
2058
2538
|
const ids = events.map((event) => event.id)
|
|
2059
2539
|
if (new Set(ids).size !== ids.length) {
|
|
@@ -2063,34 +2543,32 @@ export function createRedisStoreCore(
|
|
|
2063
2543
|
)
|
|
2064
2544
|
}
|
|
2065
2545
|
const now = clock.now().getTime()
|
|
2066
|
-
const
|
|
2067
|
-
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
countKey(sessionId),
|
|
2071
|
-
metaKey(sessionId),
|
|
2072
|
-
pendingKey(sessionId),
|
|
2073
|
-
readyKey(sessionId),
|
|
2074
|
-
snapshotEventPinsKey(sessionId, index),
|
|
2075
|
-
snapshotPinCountsKey(sessionId),
|
|
2076
|
-
metaSplitKey(sessionId),
|
|
2077
|
-
...ids.map((id) => idKey(id)),
|
|
2078
|
-
],
|
|
2079
|
-
[
|
|
2546
|
+
const cause: EventCause = { consumer, lane, index, attempt }
|
|
2547
|
+
const reply = (await indexing(sessionId, () =>
|
|
2548
|
+
evalScript(
|
|
2549
|
+
COMPLETE_ATTEMPT_LUA,
|
|
2080
2550
|
sessionId,
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2551
|
+
ids.map((id) => idKey(id)),
|
|
2552
|
+
[
|
|
2553
|
+
sessionId,
|
|
2554
|
+
now,
|
|
2555
|
+
consumer,
|
|
2556
|
+
tagOf(lane),
|
|
2557
|
+
index,
|
|
2558
|
+
attempt,
|
|
2559
|
+
json(cause),
|
|
2560
|
+
servedArg(consumers),
|
|
2561
|
+
notifyChannel(sessionId),
|
|
2562
|
+
json(ids),
|
|
2563
|
+
events.length,
|
|
2564
|
+
...events.flatMap((event) => [
|
|
2565
|
+
event.id,
|
|
2566
|
+
event.type,
|
|
2567
|
+
json(event.payload),
|
|
2568
|
+
targetsArg(event.targets),
|
|
2569
|
+
]),
|
|
2570
|
+
],
|
|
2571
|
+
),
|
|
2094
2572
|
)) as string[]
|
|
2095
2573
|
const outcome = reply[0]
|
|
2096
2574
|
if (outcome === 'missing') {
|
|
@@ -2106,7 +2584,6 @@ export function createRedisStoreCore(
|
|
|
2106
2584
|
}
|
|
2107
2585
|
if (outcome === 'superseded') return { outcome }
|
|
2108
2586
|
|
|
2109
|
-
const cause = { index, attempt }
|
|
2110
2587
|
if (outcome === 'ok') {
|
|
2111
2588
|
const base = Number(reply[1]) - 1
|
|
2112
2589
|
return {
|
|
@@ -2118,21 +2595,7 @@ export function createRedisStoreCore(
|
|
|
2118
2595
|
index: base + 1 + i,
|
|
2119
2596
|
sessionId,
|
|
2120
2597
|
createdAt: new Date(now),
|
|
2121
|
-
cause,
|
|
2122
|
-
lane: event.lane ?? null,
|
|
2123
|
-
processedAt: event.settled === true ? new Date(now) : null,
|
|
2124
|
-
processedByAttempt: null,
|
|
2125
|
-
returnedEventIds: null,
|
|
2126
|
-
firstClaimedAt: null,
|
|
2127
|
-
lastClaimedAt: null,
|
|
2128
|
-
attemptCount: 0,
|
|
2129
|
-
claimHolder: null,
|
|
2130
|
-
claimExpiresAt: null,
|
|
2131
|
-
failureCount: 0,
|
|
2132
|
-
lastFailedAt: null,
|
|
2133
|
-
lastFailedAttempt: null,
|
|
2134
|
-
lastError: null,
|
|
2135
|
-
failedAt: null,
|
|
2598
|
+
cause: { ...cause },
|
|
2136
2599
|
})),
|
|
2137
2600
|
} satisfies CompleteAttemptResult
|
|
2138
2601
|
}
|
|
@@ -2150,6 +2613,20 @@ export function createRedisStoreCore(
|
|
|
2150
2613
|
)
|
|
2151
2614
|
// Cleared after the script matched the retry.
|
|
2152
2615
|
if (index <= clearIndex) return { outcome: 'superseded' }
|
|
2616
|
+
if (
|
|
2617
|
+
existing.some(
|
|
2618
|
+
(row) =>
|
|
2619
|
+
row.cause?.consumer !== consumer ||
|
|
2620
|
+
row.cause.lane !== lane ||
|
|
2621
|
+
row.cause.index !== index ||
|
|
2622
|
+
row.cause.attempt !== attempt,
|
|
2623
|
+
)
|
|
2624
|
+
) {
|
|
2625
|
+
throw new A2Error(
|
|
2626
|
+
'PARTIAL_DUPLICATE_BATCH',
|
|
2627
|
+
'returned event retry does not match the committed batch',
|
|
2628
|
+
)
|
|
2629
|
+
}
|
|
2153
2630
|
return {
|
|
2154
2631
|
outcome: 'completed',
|
|
2155
2632
|
events: idempotentReplay(events, existing),
|
|
@@ -2161,6 +2638,8 @@ export function createRedisStoreCore(
|
|
|
2161
2638
|
|
|
2162
2639
|
async failAttempt({
|
|
2163
2640
|
sessionId,
|
|
2641
|
+
consumer,
|
|
2642
|
+
lane,
|
|
2164
2643
|
index,
|
|
2165
2644
|
attempt,
|
|
2166
2645
|
error,
|
|
@@ -2171,16 +2650,11 @@ export function createRedisStoreCore(
|
|
|
2171
2650
|
const now = clock.now().getTime()
|
|
2172
2651
|
const reply = (await evalScript(
|
|
2173
2652
|
FAIL_ATTEMPT_LUA,
|
|
2653
|
+
sessionId,
|
|
2654
|
+
[],
|
|
2174
2655
|
[
|
|
2175
|
-
|
|
2176
|
-
|
|
2177
|
-
pendingKey(sessionId),
|
|
2178
|
-
readyKey(sessionId),
|
|
2179
|
-
snapshotEventPinsKey(sessionId, index),
|
|
2180
|
-
snapshotPinCountsKey(sessionId),
|
|
2181
|
-
metaSplitKey(sessionId),
|
|
2182
|
-
],
|
|
2183
|
-
[
|
|
2656
|
+
consumer,
|
|
2657
|
+
tagOf(lane),
|
|
2184
2658
|
index,
|
|
2185
2659
|
attempt,
|
|
2186
2660
|
error,
|
|
@@ -2190,8 +2664,6 @@ export function createRedisStoreCore(
|
|
|
2190
2664
|
],
|
|
2191
2665
|
)) as string[]
|
|
2192
2666
|
const outcome = reply[0]
|
|
2193
|
-
if (outcome === 'cleared')
|
|
2194
|
-
return { outcome: 'superseded', failureCount: 0 }
|
|
2195
2667
|
if (outcome === 'missing') {
|
|
2196
2668
|
throw new TypeError(
|
|
2197
2669
|
`no event at index ${index} in session '${sessionId}'`,
|
|
@@ -2208,58 +2680,74 @@ export function createRedisStoreCore(
|
|
|
2208
2680
|
return wrap(async () => {
|
|
2209
2681
|
const reply = (await evalScript(
|
|
2210
2682
|
DEAD_LETTERS_LUA,
|
|
2211
|
-
|
|
2212
|
-
storeKey(sessionId),
|
|
2213
|
-
countKey(sessionId),
|
|
2214
|
-
metaKey(sessionId),
|
|
2215
|
-
pendingKey(sessionId),
|
|
2216
|
-
metaSplitKey(sessionId),
|
|
2217
|
-
],
|
|
2683
|
+
sessionId,
|
|
2218
2684
|
[],
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
toStoredEvent(
|
|
2225
|
-
sessionId,
|
|
2226
|
-
reply[i] as StreamEntry,
|
|
2227
|
-
(suffix) =>
|
|
2228
|
-
columns[STORED_EVENT_META_SUFFIXES.indexOf(suffix)] ??
|
|
2229
|
-
undefined,
|
|
2230
|
-
),
|
|
2231
|
-
)
|
|
2232
|
-
}
|
|
2233
|
-
return events
|
|
2685
|
+
[],
|
|
2686
|
+
)) as HeadTuple[]
|
|
2687
|
+
return reply
|
|
2688
|
+
.map((head) => toHeadEvent(sessionId, head))
|
|
2689
|
+
.toSorted(byHead)
|
|
2234
2690
|
})
|
|
2235
2691
|
},
|
|
2236
2692
|
|
|
2237
|
-
async redrive({ sessionId, index, action }) {
|
|
2693
|
+
async redrive({ sessionId, consumer, index, action, consumers }) {
|
|
2238
2694
|
return wrap(async () => {
|
|
2239
|
-
const [redriven, hasPending] = (await
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
|
|
2245
|
-
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2695
|
+
const [redriven, hasPending] = (await indexing(sessionId, () =>
|
|
2696
|
+
evalScript(
|
|
2697
|
+
REDRIVE_LUA,
|
|
2698
|
+
sessionId,
|
|
2699
|
+
[],
|
|
2700
|
+
[
|
|
2701
|
+
consumer,
|
|
2702
|
+
index,
|
|
2703
|
+
action,
|
|
2704
|
+
clock.now().getTime(),
|
|
2705
|
+
servedArg(consumers),
|
|
2706
|
+
],
|
|
2707
|
+
),
|
|
2249
2708
|
)) as [string, string]
|
|
2250
2709
|
return { redriven: redriven === '1', hasPending: hasPending === '1' }
|
|
2251
2710
|
})
|
|
2252
2711
|
},
|
|
2253
2712
|
|
|
2713
|
+
async wakeLog({ sessionId, consumer }) {
|
|
2714
|
+
return wrap(async () => {
|
|
2715
|
+
const { name, log } = consumer
|
|
2716
|
+
const [, position, head, columns, sessionHead] = (await indexing(
|
|
2717
|
+
sessionId,
|
|
2718
|
+
() =>
|
|
2719
|
+
evalScript(
|
|
2720
|
+
WAKE_LOG_LUA,
|
|
2721
|
+
sessionId,
|
|
2722
|
+
[],
|
|
2723
|
+
[name, log.lane, logTypesArg(log.types), log.backfill ? 1 : 0],
|
|
2724
|
+
),
|
|
2725
|
+
)) as [
|
|
2726
|
+
ok: string,
|
|
2727
|
+
position: string,
|
|
2728
|
+
head: string | null,
|
|
2729
|
+
columns: Column[],
|
|
2730
|
+
sessionHead: string,
|
|
2731
|
+
]
|
|
2732
|
+
return {
|
|
2733
|
+
cursor: Object.assign(dispatchOf(name, log.lane, columns), {
|
|
2734
|
+
position: Number(position),
|
|
2735
|
+
head: head === null ? null : Number(head),
|
|
2736
|
+
}),
|
|
2737
|
+
head: Number(sessionHead),
|
|
2738
|
+
}
|
|
2739
|
+
})
|
|
2740
|
+
},
|
|
2741
|
+
|
|
2254
2742
|
async readState(sessionId, reducerName, stateOptions) {
|
|
2255
2743
|
return wrap(() =>
|
|
2256
2744
|
retryAfterGap(async () => {
|
|
2257
2745
|
const reply = (await evalScript(
|
|
2258
2746
|
READ_STATE_LUA,
|
|
2747
|
+
sessionId,
|
|
2259
2748
|
[
|
|
2260
2749
|
snapKey(sessionId, reducerName),
|
|
2261
2750
|
snapHistoryKey(sessionId, reducerName),
|
|
2262
|
-
countKey(sessionId),
|
|
2263
2751
|
],
|
|
2264
2752
|
[
|
|
2265
2753
|
stateOptions?.snapshotThroughIndex ??
|
|
@@ -2315,33 +2803,28 @@ export function createRedisStoreCore(
|
|
|
2315
2803
|
async putSnapshots(sessionId, reducerName, writes) {
|
|
2316
2804
|
const byIndex = new Map<
|
|
2317
2805
|
number,
|
|
2318
|
-
{ index: number; state: unknown;
|
|
2806
|
+
{ index: number; state: unknown; pins: Map<string, HeadRef> }
|
|
2319
2807
|
>()
|
|
2320
2808
|
for (const write of writes) {
|
|
2321
2809
|
const existing = byIndex.get(write.index)
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2810
|
+
const pins = existing?.pins ?? new Map<string, HeadRef>()
|
|
2811
|
+
for (const pin of write.pins ?? []) {
|
|
2812
|
+
pins.set(JSON.stringify([pin.consumer, pin.lane, pin.index]), pin)
|
|
2813
|
+
}
|
|
2814
|
+
if (existing) existing.state = write.state
|
|
2815
|
+
else
|
|
2328
2816
|
byIndex.set(write.index, {
|
|
2329
2817
|
index: write.index,
|
|
2330
2818
|
state: write.state,
|
|
2331
|
-
|
|
2819
|
+
pins,
|
|
2332
2820
|
})
|
|
2333
|
-
}
|
|
2334
2821
|
}
|
|
2335
2822
|
const normalized = [...byIndex.values()].toSorted(
|
|
2336
2823
|
(a, b) => a.index - b.index,
|
|
2337
2824
|
)
|
|
2338
2825
|
if (normalized.length === 0) return
|
|
2339
2826
|
await wrap(async () => {
|
|
2340
|
-
const args: Array<string | number> = [
|
|
2341
|
-
clock.now().getTime(),
|
|
2342
|
-
snapshotEventPinsPrefix(sessionId),
|
|
2343
|
-
normalized.length,
|
|
2344
|
-
]
|
|
2827
|
+
const args: Array<string | number> = [normalized.length]
|
|
2345
2828
|
for (const write of normalized) {
|
|
2346
2829
|
args.push(
|
|
2347
2830
|
write.index,
|
|
@@ -2352,19 +2835,20 @@ export function createRedisStoreCore(
|
|
|
2352
2835
|
// This reducer is in the session's registry (PUT_SNAPSHOTS_LUA).
|
|
2353
2836
|
r: 1,
|
|
2354
2837
|
}),
|
|
2355
|
-
write.
|
|
2356
|
-
...write.
|
|
2838
|
+
write.pins.size,
|
|
2839
|
+
...[...write.pins.values()].flatMap((pin) => [
|
|
2840
|
+
pin.consumer,
|
|
2841
|
+
tagOf(pin.lane),
|
|
2842
|
+
pin.index,
|
|
2843
|
+
]),
|
|
2357
2844
|
)
|
|
2358
2845
|
}
|
|
2359
2846
|
await evalScript(
|
|
2360
2847
|
PUT_SNAPSHOTS_LUA,
|
|
2848
|
+
sessionId,
|
|
2361
2849
|
[
|
|
2362
2850
|
snapKey(sessionId, reducerName),
|
|
2363
2851
|
snapHistoryKey(sessionId, reducerName),
|
|
2364
|
-
countKey(sessionId),
|
|
2365
|
-
metaKey(sessionId),
|
|
2366
|
-
snapshotPinCountsKey(sessionId),
|
|
2367
|
-
metaSplitKey(sessionId),
|
|
2368
2852
|
],
|
|
2369
2853
|
[...args, reducerName],
|
|
2370
2854
|
)
|
|
@@ -2384,28 +2868,28 @@ export function createRedisStoreCore(
|
|
|
2384
2868
|
// The hash field is JSON `[participant, field]`: both are
|
|
2385
2869
|
// caller strings, so no separator character is
|
|
2386
2870
|
// collision-safe — the JSON array is.
|
|
2387
|
-
const applied = (await
|
|
2871
|
+
const applied = (await call(
|
|
2872
|
+
'EVAL',
|
|
2388
2873
|
PRESENCE_SET_LUA,
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
],
|
|
2874
|
+
1,
|
|
2875
|
+
presenceKey(ns),
|
|
2876
|
+
atMs,
|
|
2877
|
+
meta.ttlMs,
|
|
2878
|
+
entries.length,
|
|
2879
|
+
...entries.flatMap(([field, value]) => [
|
|
2880
|
+
JSON.stringify([participant, field]),
|
|
2881
|
+
value === null
|
|
2882
|
+
? ''
|
|
2883
|
+
: json({
|
|
2884
|
+
v: value,
|
|
2885
|
+
s: meta.seen,
|
|
2886
|
+
a: atMs,
|
|
2887
|
+
e: expiresAtMs,
|
|
2888
|
+
}),
|
|
2889
|
+
`${json(field)}:${json(value)}`,
|
|
2890
|
+
]),
|
|
2891
|
+
presenceKey(ns),
|
|
2892
|
+
`${json({ src: options.presenceSource, participant, seen: meta.seen, at: atMs }).slice(0, -1)},"values":{`,
|
|
2409
2893
|
)) as string[]
|
|
2410
2894
|
if (applied.length === 0) return null
|
|
2411
2895
|
// Field-keyed and caller-named — null-prototype, like every
|
|
@@ -2426,10 +2910,12 @@ export function createRedisStoreCore(
|
|
|
2426
2910
|
|
|
2427
2911
|
async read(ns) {
|
|
2428
2912
|
return wrap(async () => {
|
|
2429
|
-
const reply = (await
|
|
2913
|
+
const reply = (await call(
|
|
2914
|
+
'EVAL',
|
|
2430
2915
|
PRESENCE_READ_LUA,
|
|
2431
|
-
|
|
2432
|
-
|
|
2916
|
+
1,
|
|
2917
|
+
presenceKey(ns),
|
|
2918
|
+
clock.now().getTime(),
|
|
2433
2919
|
)) as string[]
|
|
2434
2920
|
const rows: PresenceRow[] = []
|
|
2435
2921
|
for (let i = 0; i + 1 < reply.length; i += 2) {
|
|
@@ -2492,33 +2978,23 @@ export function createRedisStoreCore(
|
|
|
2492
2978
|
const markerId = generateId()
|
|
2493
2979
|
const reply = (await evalScript(
|
|
2494
2980
|
CLEAR_LUA,
|
|
2495
|
-
|
|
2496
|
-
|
|
2497
|
-
countKey(sessionId),
|
|
2498
|
-
metaKey(sessionId),
|
|
2499
|
-
metaSplitKey(sessionId),
|
|
2500
|
-
idKey(markerId),
|
|
2501
|
-
pendingKey(sessionId),
|
|
2502
|
-
readyKey(sessionId),
|
|
2503
|
-
],
|
|
2981
|
+
sessionId,
|
|
2982
|
+
[idKey(markerId)],
|
|
2504
2983
|
[
|
|
2505
2984
|
sessionId,
|
|
2506
2985
|
clock.now().getTime(),
|
|
2507
2986
|
ifLastIndex ?? '',
|
|
2987
|
+
cause?.consumer ?? '',
|
|
2988
|
+
cause ? tagOf(cause.lane) : '',
|
|
2508
2989
|
cause?.index ?? '',
|
|
2509
2990
|
cause?.attempt ?? '',
|
|
2510
2991
|
markerId,
|
|
2511
|
-
|
|
2992
|
+
notifyChannel(sessionId),
|
|
2512
2993
|
],
|
|
2513
2994
|
)) as string[]
|
|
2514
2995
|
if (reply[0] === 'superseded') {
|
|
2515
2996
|
throw supersededAttemptError(sessionId, cause!)
|
|
2516
2997
|
}
|
|
2517
|
-
if (reply[0] === 'no_parent') {
|
|
2518
|
-
throw new TypeError(
|
|
2519
|
-
`no event at index ${reply[1]} in session '${sessionId}'`,
|
|
2520
|
-
)
|
|
2521
|
-
}
|
|
2522
2998
|
if (reply[0] === 'foreign') {
|
|
2523
2999
|
throw new A2Error(
|
|
2524
3000
|
'PARTIAL_DUPLICATE_BATCH',
|
|
@@ -2537,7 +3013,8 @@ export function createRedisStoreCore(
|
|
|
2537
3013
|
return wrap(async () => {
|
|
2538
3014
|
const [log, ...pairs] = (await evalScript(
|
|
2539
3015
|
RECLAIM_BATCH_LUA,
|
|
2540
|
-
|
|
3016
|
+
sessionId,
|
|
3017
|
+
[],
|
|
2541
3018
|
[READ_PAGE_SIZE],
|
|
2542
3019
|
)) as string[]
|
|
2543
3020
|
if (log === undefined) return { deleted: 0, done: true }
|
|
@@ -2547,14 +3024,9 @@ export function createRedisStoreCore(
|
|
|
2547
3024
|
for (let i = 1; i < pairs.length; i += 2) ids.push(pairs[i]!)
|
|
2548
3025
|
const [deleted, done] = (await evalScript(
|
|
2549
3026
|
RECLAIM_LUA,
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
metaKey(sessionId),
|
|
2554
|
-
metaSplitKey(sessionId),
|
|
2555
|
-
...ids.map((id) => idKey(id)),
|
|
2556
|
-
],
|
|
2557
|
-
[sessionId, snapshotEventPinsPrefix(sessionId), ids.length, ...pairs],
|
|
3027
|
+
sessionId,
|
|
3028
|
+
[log, ...ids.map((id) => idKey(id))],
|
|
3029
|
+
[sessionId, ids.length, ...pairs],
|
|
2558
3030
|
)) as [number, number]
|
|
2559
3031
|
return { deleted: Number(deleted), done: Number(done) === 1 }
|
|
2560
3032
|
})
|