experimental-a2 0.19.0 → 0.21.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (320) hide show
  1. package/CHANGELOG.md +50 -0
  2. package/README.md +1 -1
  3. package/dist/{actor-C7b-1arn.d.ts → actor-HnGmrOZZ.d.ts} +5 -5
  4. package/dist/{actor-C7b-1arn.d.ts.map → actor-HnGmrOZZ.d.ts.map} +1 -1
  5. package/dist/actor-client.d.ts +1 -1
  6. package/dist/actor-client.js +2 -2
  7. package/dist/actor-react.d.ts +3 -3
  8. package/dist/actor-react.js +5 -5
  9. package/dist/{actor-shared-t7w8xIKP.js → actor-shared-DFET5q16.js} +3 -3
  10. package/dist/{actor-shared-t7w8xIKP.js.map → actor-shared-DFET5q16.js.map} +1 -1
  11. package/dist/actor.d.ts +1 -1
  12. package/dist/actor.js +54 -43
  13. package/dist/actor.js.map +1 -1
  14. package/dist/{ai-Cz3hqVhk.d.ts → ai-CvEnqVfo.d.ts} +8 -8
  15. package/dist/{ai-Cz3hqVhk.d.ts.map → ai-CvEnqVfo.d.ts.map} +1 -1
  16. package/dist/{ai-mMiWul1T.js → ai-CzE672Q3.js} +9 -9
  17. package/dist/ai-CzE672Q3.js.map +1 -0
  18. package/dist/ai-client.d.ts +2 -2
  19. package/dist/ai-client.js +2 -2
  20. package/dist/ai-fx.d.ts +1 -1
  21. package/dist/ai-fx.d.ts.map +1 -1
  22. package/dist/ai-fx.js +36 -9
  23. package/dist/ai-fx.js.map +1 -1
  24. package/dist/{ai-harness-CV6rXbB3.d.ts → ai-harness-Bfk-Lycn.d.ts} +2 -2
  25. package/dist/ai-harness-Bfk-Lycn.d.ts.map +1 -0
  26. package/dist/{ai-harness-BuuC-gJ3.js → ai-harness-dQeJaR1t.js} +17 -4
  27. package/dist/ai-harness-dQeJaR1t.js.map +1 -0
  28. package/dist/ai-pi.d.ts +1 -1
  29. package/dist/ai-pi.js +4 -4
  30. package/dist/ai-pi.js.map +1 -1
  31. package/dist/ai-react.d.ts +5 -5
  32. package/dist/ai-react.js +2 -2
  33. package/dist/ai-server.d.ts +14 -17
  34. package/dist/ai-server.d.ts.map +1 -1
  35. package/dist/ai-server.js +84 -75
  36. package/dist/ai-server.js.map +1 -1
  37. package/dist/ai.d.ts +2 -2
  38. package/dist/ai.js +2 -2
  39. package/dist/{client-BhHLoNKF.js → client-9IeTsiAs.js} +5 -5
  40. package/dist/{client-BhHLoNKF.js.map → client-9IeTsiAs.js.map} +1 -1
  41. package/dist/{client-h_DKFAjj.d.ts → client-CUMbS_38.d.ts} +3 -3
  42. package/dist/{client-h_DKFAjj.d.ts.map → client-CUMbS_38.d.ts.map} +1 -1
  43. package/dist/client.d.ts +3 -3
  44. package/dist/client.js +1 -1
  45. package/dist/{contract-BYKASlCV.js → contract-CWukt01d.js} +2 -2
  46. package/dist/contract-CWukt01d.js.map +1 -0
  47. package/dist/index.d.ts +5 -5
  48. package/dist/index.js +3 -3
  49. package/dist/{initial-snapshot-Bx-c0pDP.d.ts → initial-snapshot-D3Lve43o.d.ts} +4 -4
  50. package/dist/initial-snapshot-D3Lve43o.d.ts.map +1 -0
  51. package/dist/{internal-CscSY7K_.js → internal-Bc4PvfHk.js} +10 -3
  52. package/dist/internal-Bc4PvfHk.js.map +1 -0
  53. package/dist/otel.d.ts +1 -1
  54. package/dist/otel.js +1 -1
  55. package/dist/otel.js.map +1 -1
  56. package/dist/{react-BBXgHWFc.d.ts → react-BoHvC3v9.d.ts} +4 -4
  57. package/dist/{react-BBXgHWFc.d.ts.map → react-BoHvC3v9.d.ts.map} +1 -1
  58. package/dist/react.d.ts +1 -1
  59. package/dist/react.js +1 -1
  60. package/dist/{reducer-CqA59Q6U.d.ts → reducer-CYPHsSmd.d.ts} +2 -2
  61. package/dist/{reducer-CqA59Q6U.d.ts.map → reducer-CYPHsSmd.d.ts.map} +1 -1
  62. package/dist/{reducer-BAXc8JyF.js → reducer-CjmvZSAb.js} +2 -2
  63. package/dist/{reducer-BAXc8JyF.js.map → reducer-CjmvZSAb.js.map} +1 -1
  64. package/dist/runtime-consumer-BWmht60z.js +12 -0
  65. package/dist/runtime-consumer-BWmht60z.js.map +1 -0
  66. package/dist/scheduler-qstash.d.ts +2 -2
  67. package/dist/scheduler-qstash.d.ts.map +1 -1
  68. package/dist/scheduler-qstash.js +7 -7
  69. package/dist/scheduler-qstash.js.map +1 -1
  70. package/dist/scheduler-task-X1SToDjP.js.map +1 -1
  71. package/dist/scheduler-vercel.d.ts +2 -2
  72. package/dist/scheduler-vercel.d.ts.map +1 -1
  73. package/dist/scheduler-vercel.js +5 -4
  74. package/dist/scheduler-vercel.js.map +1 -1
  75. package/dist/{server-DYAATFuT.d.ts → server-BshFZ0eL.d.ts} +108 -41
  76. package/dist/server-BshFZ0eL.d.ts.map +1 -0
  77. package/dist/{server-D-Y4pqwu.js → server-DvIB8kj5.js} +423 -135
  78. package/dist/server-DvIB8kj5.js.map +1 -0
  79. package/dist/server.browser.js +1 -1
  80. package/dist/server.browser.js.map +1 -1
  81. package/dist/server.d.ts +4 -4
  82. package/dist/server.js +2 -2
  83. package/dist/{session-clear-C-QwTXV1.d.ts → session-clear-Bk_29eEw.d.ts} +2 -2
  84. package/dist/session-clear-Bk_29eEw.d.ts.map +1 -0
  85. package/dist/{session-clear-DGI4jTQ5.js → session-clear-wHd5GCpZ.js} +5 -4
  86. package/dist/session-clear-wHd5GCpZ.js.map +1 -0
  87. package/dist/store-N8PXxDAS.js.map +1 -1
  88. package/dist/store-Z2qjw50n.d.ts +523 -0
  89. package/dist/store-Z2qjw50n.d.ts.map +1 -0
  90. package/dist/store-codec-DwrWlgQx.js +65 -0
  91. package/dist/store-codec-DwrWlgQx.js.map +1 -0
  92. package/dist/store-memory.d.ts +1 -1
  93. package/dist/store-memory.d.ts.map +1 -1
  94. package/dist/store-memory.js +393 -210
  95. package/dist/store-memory.js.map +1 -1
  96. package/dist/{store-polling-7WJV0YWP.js → store-polling-DYNyxCev.js} +2 -2
  97. package/dist/{store-polling-7WJV0YWP.js.map → store-polling-DYNyxCev.js.map} +1 -1
  98. package/dist/store-postgres.d.ts +1 -1
  99. package/dist/store-postgres.d.ts.map +1 -1
  100. package/dist/store-postgres.js +1595 -673
  101. package/dist/store-postgres.js.map +1 -1
  102. package/dist/{store-presence-polling-B1ixeqIS.js → store-presence-polling-CzQ0wdtM.js} +3 -3
  103. package/dist/{store-presence-polling-B1ixeqIS.js.map → store-presence-polling-CzQ0wdtM.js.map} +1 -1
  104. package/dist/store-redis-http.d.ts +1 -1
  105. package/dist/store-redis-http.js +4 -4
  106. package/dist/store-redis-notify-IiVdlYHc.js +2919 -0
  107. package/dist/store-redis-notify-IiVdlYHc.js.map +1 -0
  108. package/dist/store-redis.d.ts +1 -1
  109. package/dist/store-redis.js +2 -2
  110. package/dist/store-sqlite.d.ts +1 -1
  111. package/dist/store-sqlite.d.ts.map +1 -1
  112. package/dist/store-sqlite.js +517 -287
  113. package/dist/store-sqlite.js.map +1 -1
  114. package/dist/{telemetry-Cyfqo6Qt.d.ts → telemetry-CWwsh-Gx.d.ts} +3 -3
  115. package/dist/telemetry-CWwsh-Gx.d.ts.map +1 -0
  116. package/dist/{wire-BKxH1vbP.js → wire-CZGqNZRx.js} +5 -4
  117. package/dist/wire-CZGqNZRx.js.map +1 -0
  118. package/docs/01-quickstart.mdx +28 -19
  119. package/docs/02-examples.mdx +1 -1
  120. package/docs/actors/01-introduction.mdx +9 -1
  121. package/docs/actors/02-concurrency.mdx +1 -1
  122. package/docs/actors/meta.ts +1 -1
  123. package/docs/agents/01-introduction.mdx +1 -1
  124. package/docs/agents/02-tools.mdx +13 -12
  125. package/docs/agents/03-conversations.mdx +31 -21
  126. package/docs/agents/04-harnesses.mdx +7 -3
  127. package/docs/agents/meta.ts +1 -1
  128. package/docs/client/03-presence.mdx +18 -14
  129. package/docs/client/meta.ts +1 -1
  130. package/docs/contract/01-events.mdx +2 -2
  131. package/docs/contract/02-state.mdx +18 -14
  132. package/docs/contract/03-sessions.mdx +32 -25
  133. package/docs/index.mdx +31 -26
  134. package/docs/reference/01-core.mdx +12 -12
  135. package/docs/reference/02-server.mdx +247 -192
  136. package/docs/reference/04-ai.mdx +29 -20
  137. package/docs/reference/05-telemetry.mdx +146 -0
  138. package/docs/reference/06-errors.mdx +24 -18
  139. package/docs/reference/meta.ts +1 -1
  140. package/docs/schedulers/01-overview.mdx +112 -0
  141. package/docs/schedulers/02-vercel-queues.mdx +87 -0
  142. package/docs/schedulers/03-qstash.mdx +123 -0
  143. package/docs/schedulers/meta.ts +1 -0
  144. package/docs/server/01-consumers.mdx +415 -0
  145. package/docs/server/02-timers.mdx +33 -23
  146. package/docs/server/03-cancellation.mdx +52 -44
  147. package/docs/server/04-routes.mdx +4 -1
  148. package/docs/server/05-sql-projections.mdx +391 -0
  149. package/docs/server/06-guarantees.mdx +60 -50
  150. package/docs/stores/01-overview.mdx +37 -0
  151. package/docs/stores/02-redis.mdx +57 -0
  152. package/docs/stores/03-redis-http.mdx +56 -0
  153. package/docs/stores/04-postgres.mdx +172 -0
  154. package/docs/stores/05-sqlite.mdx +36 -0
  155. package/docs/stores/06-memory.mdx +22 -0
  156. package/docs/stores/meta.ts +1 -0
  157. package/docs/upgrades/01-overview.mdx +30 -0
  158. package/docs/upgrades/79-0-21.mdx +165 -0
  159. package/docs/upgrades/80-0-20.mdx +43 -0
  160. package/docs/upgrades/81-0-19.mdx +33 -0
  161. package/docs/upgrades/82-0-18.mdx +33 -0
  162. package/docs/upgrades/83-0-17.mdx +69 -0
  163. package/docs/upgrades/meta.ts +3 -0
  164. package/examples/playground/DEPLOY.md +48 -11
  165. package/examples/playground/app/a0/apps/[appId]/route.ts +6 -10
  166. package/examples/playground/app/a0/chat.test.tsx +421 -330
  167. package/examples/playground/app/a0/sandbox-log.ts +17 -15
  168. package/examples/playground/app/a0/server.ts +4 -4
  169. package/examples/playground/app/a0/shell.tsx +1 -1
  170. package/examples/playground/app/a0/uploads/route.test.ts +59 -0
  171. package/examples/playground/app/a2slack/a2slack.css +1067 -0
  172. package/examples/playground/app/a2slack/c/[channel]/page.tsx +20 -0
  173. package/examples/playground/app/a2slack/consumers/agent.ts +63 -0
  174. package/examples/playground/app/a2slack/consumers/feed.ts +164 -0
  175. package/examples/playground/app/a2slack/consumers/notify.ts +116 -0
  176. package/examples/playground/app/a2slack/consumers/search.ts +37 -0
  177. package/examples/playground/app/a2slack/consumers/triage.ts +31 -0
  178. package/examples/playground/app/a2slack/consumers/welcome.ts +25 -0
  179. package/examples/playground/app/a2slack/db/a2.ts +206 -0
  180. package/examples/playground/app/a2slack/db/index.ts +73 -0
  181. package/examples/playground/app/a2slack/db/migrations/0000_extensions.sql +4 -0
  182. package/examples/playground/app/a2slack/db/migrations/0001_workspace.sql +74 -0
  183. package/examples/playground/app/a2slack/db/migrations/0002_a2.sql +215 -0
  184. package/examples/playground/app/a2slack/db/migrations/0003_a2_batch.sql +67 -0
  185. package/examples/playground/app/a2slack/db/migrations/meta/0000_snapshot.json +18 -0
  186. package/examples/playground/app/a2slack/db/migrations/meta/0001_snapshot.json +500 -0
  187. package/examples/playground/app/a2slack/db/migrations/meta/0002_snapshot.json +1159 -0
  188. package/examples/playground/app/a2slack/db/migrations/meta/0003_snapshot.json +1159 -0
  189. package/examples/playground/app/a2slack/db/migrations/meta/_journal.json +34 -0
  190. package/examples/playground/app/a2slack/db/queries.ts +263 -0
  191. package/examples/playground/app/a2slack/db/schema.ts +145 -0
  192. package/examples/playground/app/a2slack/drills/a2slack.drill.ts +708 -0
  193. package/examples/playground/app/a2slack/drills/harness.ts +244 -0
  194. package/examples/playground/app/a2slack/events/route.ts +36 -0
  195. package/examples/playground/app/a2slack/identity.ts +44 -0
  196. package/examples/playground/app/a2slack/inbox/route.ts +8 -0
  197. package/examples/playground/app/a2slack/lag/data/route.ts +8 -0
  198. package/examples/playground/app/a2slack/lag/page.tsx +101 -0
  199. package/examples/playground/app/a2slack/lag.ts +73 -0
  200. package/examples/playground/app/a2slack/layout.tsx +43 -0
  201. package/examples/playground/app/a2slack/messages/route.ts +17 -0
  202. package/examples/playground/app/a2slack/model.test.ts +208 -0
  203. package/examples/playground/app/a2slack/model.ts +369 -0
  204. package/examples/playground/app/a2slack/models.test.ts +72 -0
  205. package/examples/playground/app/a2slack/page.tsx +10 -0
  206. package/examples/playground/app/a2slack/scheduler/route.ts +5 -0
  207. package/examples/playground/app/a2slack/search/route.ts +40 -0
  208. package/examples/playground/app/a2slack/search-index.ts +108 -0
  209. package/examples/playground/app/a2slack/server.test.ts +315 -0
  210. package/examples/playground/app/a2slack/server.ts +55 -0
  211. package/examples/playground/app/a2slack/sessions.ts +50 -0
  212. package/examples/playground/app/a2slack/thread/route.ts +8 -0
  213. package/examples/playground/app/a2slack/ui/ChannelView.tsx +87 -0
  214. package/examples/playground/app/a2slack/ui/Composer.tsx +117 -0
  215. package/examples/playground/app/a2slack/ui/DotsShadow.tsx +42 -0
  216. package/examples/playground/app/a2slack/ui/Feed.tsx +157 -0
  217. package/examples/playground/app/a2slack/ui/InboxContext.tsx +63 -0
  218. package/examples/playground/app/a2slack/ui/InboxView.tsx +82 -0
  219. package/examples/playground/app/a2slack/ui/Menu.tsx +128 -0
  220. package/examples/playground/app/a2slack/ui/MessageRow.tsx +314 -0
  221. package/examples/playground/app/a2slack/ui/NameGate.tsx +35 -0
  222. package/examples/playground/app/a2slack/ui/SearchContext.tsx +146 -0
  223. package/examples/playground/app/a2slack/ui/SearchResults.tsx +97 -0
  224. package/examples/playground/app/a2slack/ui/Shell.tsx +34 -0
  225. package/examples/playground/app/a2slack/ui/Sidebar.tsx +199 -0
  226. package/examples/playground/app/a2slack/ui/ThreadPanel.tsx +134 -0
  227. package/examples/playground/app/a2slack/ui/useInbox.ts +71 -0
  228. package/examples/playground/app/agent/[agentId]/agent-queue.test.tsx +116 -90
  229. package/examples/playground/app/agent/[agentId]/compaction-panel.test.tsx +107 -76
  230. package/examples/playground/app/agent/ingress.test.ts +41 -17
  231. package/examples/playground/app/agent/model-input.test.tsx +92 -43
  232. package/examples/playground/app/agent/server.ts +3 -3
  233. package/examples/playground/app/chat/[chatId]/agent-stream-drawer.test.tsx +2 -2
  234. package/examples/playground/app/chat/[chatId]/chat-client.tsx +1 -1
  235. package/examples/playground/app/chat/server.test.ts +38 -65
  236. package/examples/playground/app/chat/server.ts +45 -35
  237. package/examples/playground/app/components/activity-history.test.tsx +60 -18
  238. package/examples/playground/app/components/demo-icons.tsx +7 -0
  239. package/examples/playground/app/components/demos.tsx +7 -0
  240. package/examples/playground/app/components/fast-link.test.tsx +3 -5
  241. package/examples/playground/app/initial-state.test.tsx +86 -45
  242. package/examples/playground/app/orders/[orderId]/order-client.tsx +2 -2
  243. package/examples/playground/app/orders/page.tsx +1 -1
  244. package/examples/playground/app/orders/server.ts +49 -30
  245. package/examples/playground/app/recovery/[recoveryId]/recovery-client.tsx +14 -15
  246. package/examples/playground/app/recovery/page.tsx +4 -4
  247. package/examples/playground/app/recovery/server.ts +23 -32
  248. package/examples/playground/drizzle.config.ts +14 -0
  249. package/examples/playground/next.config.ts +6 -1
  250. package/examples/playground/package.json +13 -1
  251. package/examples/playground/scripts/migrate.mjs +31 -0
  252. package/examples/playground/test/agent.ts +35 -0
  253. package/examples/playground/test/model.ts +149 -0
  254. package/examples/playground/test/serve.ts +184 -0
  255. package/examples/playground/test/server-only.ts +1 -0
  256. package/examples/playground/vercel.json +9 -0
  257. package/examples/playground/vitest.config.ts +27 -0
  258. package/examples/playground/vitest.drills.config.ts +32 -0
  259. package/package.json +7 -2
  260. package/sql/postgres/drizzle-schema.ts +206 -0
  261. package/sql/postgres/notifications.sql +24 -0
  262. package/sql/postgres/schema.sql +276 -0
  263. package/src/actor.ts +101 -71
  264. package/src/ai-agent.ts +154 -115
  265. package/src/ai-control-server.ts +8 -8
  266. package/src/ai-fx.ts +41 -9
  267. package/src/ai-harness.ts +28 -2
  268. package/src/ai-pi.ts +2 -2
  269. package/src/ai-server.ts +2 -2
  270. package/src/ai.ts +7 -7
  271. package/src/contract.ts +1 -1
  272. package/src/deterministic-id.ts +26 -3
  273. package/src/errors.ts +9 -6
  274. package/src/idempotent-replay.ts +0 -1
  275. package/src/index.ts +1 -1
  276. package/src/internal.ts +17 -6
  277. package/src/otel.ts +1 -1
  278. package/src/postgres-notifications.ts +0 -22
  279. package/src/postgres-resources.ts +6 -9
  280. package/src/postgres-schema.ts +121 -0
  281. package/src/push-envelope.ts +20 -0
  282. package/src/runtime-consumer.ts +10 -0
  283. package/src/scheduler-qstash.ts +12 -5
  284. package/src/scheduler-task.ts +2 -2
  285. package/src/scheduler-vercel.ts +10 -4
  286. package/src/server.browser.ts +1 -1
  287. package/src/server.ts +839 -238
  288. package/src/session-socket.ts +1 -1
  289. package/src/store-codec.ts +66 -0
  290. package/src/store-memory.ts +581 -255
  291. package/src/store-postgres.ts +1984 -943
  292. package/src/store-redis-core.ts +1624 -1152
  293. package/src/store-redis-notify.ts +2 -0
  294. package/src/store-sqlite.ts +891 -486
  295. package/src/store.ts +227 -97
  296. package/src/telemetry.ts +2 -1
  297. package/src/wire.ts +1 -0
  298. package/dist/ai-harness-BuuC-gJ3.js.map +0 -1
  299. package/dist/ai-harness-CV6rXbB3.d.ts.map +0 -1
  300. package/dist/ai-mMiWul1T.js.map +0 -1
  301. package/dist/contract-BYKASlCV.js.map +0 -1
  302. package/dist/idempotent-replay-JdgMQRRm.js +0 -21
  303. package/dist/idempotent-replay-JdgMQRRm.js.map +0 -1
  304. package/dist/initial-snapshot-Bx-c0pDP.d.ts.map +0 -1
  305. package/dist/internal-CscSY7K_.js.map +0 -1
  306. package/dist/server-D-Y4pqwu.js.map +0 -1
  307. package/dist/server-DYAATFuT.d.ts.map +0 -1
  308. package/dist/session-clear-C-QwTXV1.d.ts.map +0 -1
  309. package/dist/session-clear-DGI4jTQ5.js.map +0 -1
  310. package/dist/store-CyLUy86v.d.ts +0 -404
  311. package/dist/store-CyLUy86v.d.ts.map +0 -1
  312. package/dist/store-codec-DTG0Ftek.js +0 -10
  313. package/dist/store-codec-DTG0Ftek.js.map +0 -1
  314. package/dist/store-redis-notify-CM0AfLY0.js +0 -2544
  315. package/dist/store-redis-notify-CM0AfLY0.js.map +0 -1
  316. package/dist/telemetry-Cyfqo6Qt.d.ts.map +0 -1
  317. package/dist/wire-BKxH1vbP.js.map +0 -1
  318. package/docs/reference/05-adapters.mdx +0 -351
  319. package/docs/server/01-handlers.mdx +0 -200
  320. package/docs/server/05-production.mdx +0 -323
@@ -5,18 +5,46 @@
5
5
  * `store-redis-http` (provider REST APIs).
6
6
  *
7
7
  * The stream IS the store: each event is a stream entry whose ID is its
8
- * dense a2 index (`${index}-0`), so `XRANGE` is the catch-up read.
9
- * Mutable drain/failure bookkeeping lives in side hashes of fixed index
10
- * ranges (stream entries are immutable); every write path is one atomic
11
- * Lua script.
12
- * Global event-id ownership is one key per id, so no key grows with
13
- * the number of sessions.
14
- * The conformance suite in test/conformance is the executable
15
- * contract, run against a real spawned `redis-server`.
8
+ * dense a2 index (`${index}-0`), so `XRANGE` is the catch-up read. Stream
9
+ * entries are immutable; dispatch bookkeeping lives on consumer cursors.
10
+ * Every write path is one atomic Lua script.
16
11
  *
17
- * Timestamps and claim expiry come from the injected clock so tests
18
- * can time-travel against a real server. Cluster mode is out: the
19
- * atomic scripts span keys.
12
+ * The cursor layout, per session `{p}:{s}:`:
13
+ *
14
+ * - `heads:{chunk}` holds one row per (head index, consumer) in hashes of
15
+ * META_CHUNK_SIZE consecutive indexes: a head's attempt and failure
16
+ * fields, its kind (`l`), its lane successor (`next`), its snapshot pins,
17
+ * and, once the cursor moves past it, its completion record. A new head
18
+ * is a new row, so moving a cursor resets its attempt fields for free,
19
+ * and an event cursor costs one row, as a handled event did.
20
+ * - `cursors` is one hash of every lane and log cursor: its head (`h`),
21
+ * position (`p`), lane tail (`t`), or log types (`ty`). A lane's queue is
22
+ * linked through the rows' `next` fields.
23
+ * - `runnable` is a zset of every cursor with a head, scored by the head's
24
+ * index, negated while the head is dead-lettered.
25
+ * - `pins` counts, per snapshot, the heads that pin it.
26
+ *
27
+ * A cursor id names a cursor within its session: the consumer's byte
28
+ * length, `:`, the consumer, then `#{index}` for an event cursor, `L{lane}`
29
+ * for a lane cursor, or `G{lane}` for a log cursor. A clear supersedes every
30
+ * head at or below its marker at once (the `runnable` set is dropped); lane and
31
+ * log cursor records are reset lazily against the marker and swept by
32
+ * reclaim.
33
+ *
34
+ * `{p}:schema` records the layout, and a prefix holding another one is
35
+ * rejected. A session from before cursors (0.17 to 0.20) opens as is: its
36
+ * log, counter, clear marker, snapshots, and ids read the same, and its
37
+ * dispatch keys (`meta`, `meta:{chunk}`, `meta-split`, `pending`, `ready`,
38
+ * `snapshot-pins:{index}`, `snapshot-pin-counts`) share no name with the
39
+ * cursor layout's, so they're never read. Its unfinished work is dropped.
40
+ *
41
+ * Global event-id ownership is one key per id, so no key grows with the
42
+ * number of sessions. The conformance suite in test/conformance is the
43
+ * executable contract, run against a real spawned `redis-server`.
44
+ *
45
+ * Timestamps and claim expiry come from the injected clock so tests can
46
+ * time-travel against a real server. Cluster mode is out: the atomic
47
+ * scripts span keys.
20
48
  */
21
49
 
22
50
  import type { PresencePatch } from './contract.ts'
@@ -24,7 +52,7 @@ import { A2Error, supersededAttemptError } from './errors.ts'
24
52
  import { CLEAR_EVENT_TYPE, sessionClearedError } from './session-clear.ts'
25
53
  import { idempotentReplay } from './idempotent-replay.ts'
26
54
  import { nullProtoRecord } from './internal.ts'
27
- import { decodeReturnedEventIds } from './store-codec.ts'
55
+ import { decodeCause } from './store-codec.ts'
28
56
  import type { SerializedEvent } from './stream-buffer.ts'
29
57
  import type {
30
58
  A2Store,
@@ -32,8 +60,14 @@ import type {
32
60
  CompleteAttemptResult,
33
61
  Event,
34
62
  EventCause,
63
+ EventTarget,
64
+ HeadDispatch,
65
+ HeadEvent,
66
+ HeadRef,
35
67
  IdSource,
36
68
  StoreClaimAvailableResult,
69
+ StoreConsumer,
70
+ StoredCursor,
37
71
  PresenceRow,
38
72
  StoredEvent,
39
73
  } from './store.ts'
@@ -81,7 +115,7 @@ export type RedisStoreCore = Omit<A2Store, 'stream' | 'presence'> & {
81
115
  ): Promise<PresencePatch | null>
82
116
  read(ns: string): Promise<PresenceRow[]>
83
117
  }
84
- /** XRANGE + meta merge — the full-row catch-up read. */
118
+ /** XRANGE pages — the full-row catch-up read. */
85
119
  readRange(
86
120
  sessionId: string,
87
121
  afterIndex: number,
@@ -119,66 +153,28 @@ export const READ_PAGE_SIZE = 256
119
153
  export const FEED_PAGE_SIZE = 1000
120
154
  const READ_CONCURRENCY = 16
121
155
 
122
- // ── Lua scripts (each runs atomically) ───────────────────────────────
156
+ /** The key layout this core writes; any other is rejected. */
157
+ export const REDIS_LAYOUT = 'cursors-2'
158
+ const LAYOUT_ERROR = 'A2_LAYOUT'
123
159
 
124
160
  const META_CHUNK_SIZE = 1000
125
161
 
126
- /** The metadata columns of a full stored row. */
127
- const STORED_EVENT_META_SUFFIXES = [
128
- 'p',
129
- 'pa',
130
- 'ri',
131
- 'ch',
132
- 'ce',
133
- 'f',
134
- 'fc',
135
- 'lc',
136
- 'lf',
137
- 'lfa',
138
- 'd',
139
- 'a',
140
- 'e',
141
- ] as const
162
+ /** A head row's dispatch columns, in the order scripts return them. */
163
+ const HEAD_COLUMNS = `{ 'd', 'fc', 'lc', 'ch', 'ce', 'a', 'lf', 'lfa', 'e', 'f' }`
164
+
165
+ // ── Lua scripts (each runs atomically) ───────────────────────────────
142
166
 
143
167
  /**
144
- * Upstash bills every call a script makes, so scripts read the counter and
145
- * split with one MGET, read metadata with one HMGET per hash (`read_fields`),
146
- * and buffer hash writes (`put`) into one HSET per hash (`flush`). Buffered
147
- * writes are invisible to reads until flushed, so a script must not read a
148
- * field it has buffered. Per-event metadata lives in hashes of
149
- * META_CHUNK_SIZE consecutive indexes, so no key grows with a session's
150
- * history. Indexes at or below the split predate chunking and stay in the
151
- * base hash, which also holds every session's lane pointers. An unset split
152
- * means the current event count, so any script that adds events must write
153
- * `split_write` pairs in its write phase. Including scripts bind COUNTER,
154
- * META (the base hash), and SPLIT first.
168
+ * Upstash bills every call a script makes, so scripts read the counter,
169
+ * clear marker, layout, typed-type index, and id keys with one MGET
170
+ * (`read_bounds`), read rows with one HMGET per chunk (`read_rows`), and
171
+ * buffer hash writes (`put`, `drop`) into one HSET and one HDEL per hash
172
+ * (`flush`). Buffered writes are invisible to reads until flushed, so a
173
+ * script must not read a field it has buffered. Every script binds KEYS[1]
174
+ * to the session's counter and KEYS[2] to the prefix's layout key, and
175
+ * derives the session's other keys.
155
176
  */
156
- const META_LUA = `
157
- local split, split_stored = nil, false
158
- -- The latest clear marker lives beside the counter, {prefix}:{session}:clear,
159
- -- and is read in the same MGET, so a session never cleared pays nothing.
160
- local CLEAR = string.sub(COUNTER, 1, -6) .. 'clear'
161
- local boundary = 0
162
- local function use_split(stored, count, cleared)
163
- split_stored = stored ~= false
164
- split = tonumber(stored or count or '0')
165
- boundary = tonumber(cleared or '0')
166
- end
167
- local function mkey(field)
168
- if split == nil then
169
- local values = redis.call('MGET', SPLIT, COUNTER, CLEAR)
170
- use_split(values[1], values[2], values[3])
171
- end
172
- local idx = tonumber(field)
173
- if idx <= split then return META end
174
- return META .. ':' .. tostring(math.floor((idx - 1) / ${META_CHUNK_SIZE}))
175
- end
176
- local function split_write(args)
177
- if not split_stored then
178
- args[#args + 1] = SPLIT
179
- args[#args + 1] = tostring(split)
180
- end
181
- end
177
+ const BASE_LUA = `
182
178
  local function sliced(command, key, args)
183
179
  local results = {}
184
180
  for first = 1, #args, 1000 do
@@ -195,36 +191,85 @@ local function sliced(command, key, args)
195
191
  end
196
192
  return results
197
193
  end
198
- local function read_fields(fields, suffixes)
194
+ local COUNTER, SCHEMA = KEYS[1], KEYS[2]
195
+ local SESSION = string.sub(COUNTER, 1, -6)
196
+ local LOG, CLEAR = SESSION .. 'log', SESSION .. 'clear'
197
+ local META, READY, CURSORS = SESSION .. 'heads', SESSION .. 'runnable', SESSION .. 'cursors'
198
+ local PIN_COUNTS, TYPED = SESSION .. 'pins', SESSION .. 'typed'
199
+ local CLEAR_TYPE = '${CLEAR_EVENT_TYPE}'
200
+ local total, boundary, schema_missing = 0, 0, false
201
+ -- The event types typed log cursors read: type → { from, last }. The type's
202
+ -- index holds every event of it above \`from\`, and no event of it lies above
203
+ -- \`last\`.
204
+ local typed, typed_dirty = {}, false
205
+ -- Reads the counter, the latest clear marker, the layout, the typed types,
206
+ -- and the extra keys in one MGET. Returns the extra keys' values, or nil when
207
+ -- the prefix holds another layout.
208
+ local function read_bounds(extra)
209
+ local keys = { COUNTER, CLEAR, SCHEMA, TYPED }
210
+ for _, key in ipairs(extra or {}) do keys[#keys + 1] = key end
211
+ local values = sliced('MGET', nil, keys)
212
+ if values[3] and values[3] ~= '${REDIS_LAYOUT}' then return nil end
213
+ total = tonumber(values[1] or '0')
214
+ boundary = tonumber(values[2] or '0')
215
+ schema_missing = not values[3]
216
+ if values[4] then typed = cjson.decode(values[4]) end
217
+ local rest = {}
218
+ for i = 5, #keys do rest[i - 4] = values[i] end
219
+ return rest
220
+ end
221
+ local function entry_index(id) return tonumber(string.match(id, '^(%d+)')) end
222
+ local function cid_of(consumer, tag)
223
+ return tostring(string.len(consumer)) .. ':' .. consumer .. tag
224
+ end
225
+ local function parse_cid(cid)
226
+ local colon = string.find(cid, ':', 1, true)
227
+ local n = tonumber(string.sub(cid, 1, colon - 1))
228
+ return string.sub(cid, colon + 1, colon + n), string.sub(cid, colon + n + 1)
229
+ end
230
+ -- A cursor's rows: its consumer, then '#' for an event cursor or 'L' .. lane
231
+ -- for a lane or log cursor, as requests name it.
232
+ local function row_of(consumer, tag)
233
+ local kind = string.sub(tag, 1, 1)
234
+ if kind == '#' then return cid_of(consumer, '#') end
235
+ if kind == 'G' then return cid_of(consumer, 'L' .. string.sub(tag, 2)) end
236
+ return cid_of(consumer, tag)
237
+ end
238
+ local function chunk_key(idx)
239
+ return META .. ':' .. tostring(math.floor((idx - 1) / ${META_CHUNK_SIZE}))
240
+ end
241
+ local function row_name(idx, row, suffix)
242
+ return tostring(idx) .. ':' .. row .. ':' .. suffix
243
+ end
244
+ -- Rows of (index, row) refs, aligned with refs; one HMGET per chunk.
245
+ local function read_rows(refs, suffixes)
199
246
  local groups, order = {}, {}
200
- for _, field in ipairs(fields) do
201
- local key = mkey(field)
247
+ for position, ref in ipairs(refs) do
248
+ local key = chunk_key(ref[1])
202
249
  local group = groups[key]
203
250
  if not group then
204
- group = { fields = {}, names = {} }
251
+ group = { positions = {}, names = {} }
205
252
  groups[key] = group
206
253
  order[#order + 1] = key
207
254
  end
208
- group.fields[#group.fields + 1] = field
255
+ group.positions[#group.positions + 1] = position
209
256
  for _, suffix in ipairs(suffixes) do
210
- group.names[#group.names + 1] = field .. ':' .. suffix
257
+ group.names[#group.names + 1] = row_name(ref[1], ref[2], suffix)
211
258
  end
212
259
  end
213
260
  local rows = {}
214
261
  for _, key in ipairs(order) do
215
262
  local group = groups[key]
216
263
  local values = sliced('HMGET', key, group.names)
217
- for position, field in ipairs(group.fields) do
264
+ for n, position in ipairs(group.positions) do
218
265
  local row = {}
219
- for column = 1, #suffixes do
220
- row[column] = values[(position - 1) * #suffixes + column]
221
- end
222
- rows[field] = row
266
+ for column = 1, #suffixes do row[column] = values[(n - 1) * #suffixes + column] end
267
+ rows[position] = row
223
268
  end
224
269
  end
225
270
  return rows
226
271
  end
227
- local writes, write_order = {}, {}
272
+ local writes, write_order, deletes, delete_order = {}, {}, {}, {}
228
273
  local function put(key, field, value)
229
274
  local fields = writes[key]
230
275
  if not fields then
@@ -235,115 +280,401 @@ local function put(key, field, value)
235
280
  fields[#fields + 1] = field
236
281
  fields[#fields + 1] = value
237
282
  end
283
+ local function drop(key, field)
284
+ local fields = deletes[key]
285
+ if not fields then
286
+ fields = {}
287
+ deletes[key] = fields
288
+ delete_order[#delete_order + 1] = key
289
+ end
290
+ fields[#fields + 1] = field
291
+ end
238
292
  local function flush()
239
293
  for _, key in ipairs(write_order) do sliced('HSET', key, writes[key]) end
240
- writes, write_order = {}, {}
294
+ for _, key in ipairs(delete_order) do sliced('HDEL', key, deletes[key]) end
295
+ writes, write_order, deletes, delete_order = {}, {}, {}, {}
296
+ end
297
+ local function put_row(idx, row, suffix, value)
298
+ put(chunk_key(idx), row_name(idx, row, suffix), value)
299
+ end
300
+ local function drop_row(idx, row, suffix)
301
+ drop(chunk_key(idx), row_name(idx, row, suffix))
302
+ end
303
+ -- A request names a cursor by tag: '#' for an event cursor, 'L' .. lane for a
304
+ -- lane or log cursor. A claimed row records its kind: '#', or the cursor's tag.
305
+ local function owns(kind, tag)
306
+ if not kind then return false end
307
+ if tag == '#' then return kind == '#' end
308
+ return kind == tag or kind == 'G' .. string.sub(tag, 2)
309
+ end
310
+ local function cid_for(consumer, kind, idx)
311
+ if kind == '#' then return cid_of(consumer, '#' .. tostring(idx)) end
312
+ return cid_of(consumer, kind)
313
+ end
314
+ -- One type's index, named by the type's bytes in hex so no type can spell
315
+ -- another key of the session.
316
+ local function type_key(name)
317
+ return SESSION .. 'type:' .. (string.gsub(name, '.', function(c)
318
+ return string.format('%02x', string.byte(c))
319
+ end))
320
+ end
321
+ local ready_changes, ready_order = {}, {}
322
+ local function ready_set(cid, score)
323
+ if ready_changes[cid] == nil then ready_order[#ready_order + 1] = cid end
324
+ ready_changes[cid] = score
325
+ end
326
+ local function flush_ready()
327
+ local adds, removes = {}, {}
328
+ for _, cid in ipairs(ready_order) do
329
+ local score = ready_changes[cid]
330
+ if score then
331
+ adds[#adds + 1] = score
332
+ adds[#adds + 1] = cid
333
+ else
334
+ removes[#removes + 1] = cid
335
+ end
336
+ end
337
+ sliced('ZADD', READY, adds)
338
+ sliced('ZREM', READY, removes)
339
+ ready_changes, ready_order = {}, {}
340
+ end
341
+ local function release_pins(idx, row, pins)
342
+ if not pins or pins == '' then return end
343
+ for _, ref in ipairs(cjson.decode(pins)) do
344
+ local count = redis.call('HINCRBY', PIN_COUNTS, ref, -1)
345
+ if count <= 0 then
346
+ redis.call('HDEL', PIN_COUNTS, ref)
347
+ local checkpoint = cjson.decode(ref)
348
+ redis.call('ZREMRANGEBYSCORE', checkpoint[1], checkpoint[2], checkpoint[2])
349
+ end
350
+ end
351
+ put_row(idx, row, 'pins', '')
241
352
  end
242
353
  `
243
354
 
244
355
  /**
245
- * Lane-aware enqueueing for scripts that add events. Including scripts bind
246
- * PENDING and READY. Lane tails are read once per lane and tracked in
247
- * memory, so a batch in one lane links its events without re-reading. A
248
- * lane emptied earlier in the script (`clear_lane`) is known to have no tail,
249
- * and its pointers are deleted at `flush_queue` only if nothing re-filled it.
356
+ * Cursor movement for scripts that add events or move cursors. Lane and log
357
+ * cursor records are read once per script (`load_states`) and written back
358
+ * once (`flush_states`). A record whose head lies at or below the latest clear
359
+ * marker is idle, and a log cursor counts from the marker.
360
+ *
361
+ * A typed log cursor finds its next head in its types' indexes, one bounded
362
+ * lookup per type, skipped when `typed` shows no newer event of it. When an
363
+ * index does not reach back far enough yet (a typed backfill, or a type a
364
+ * cursor newly reads), the lookup returns the type, and the script returns
365
+ * `{ 'index', type, after }` before writing anything: the caller indexes
366
+ * that type down to `after` (INDEX_TYPE_LUA, one page per call) and runs the
367
+ * operation again.
250
368
  */
251
- const ENQUEUE_LUA = `
252
- local function lane_field(lane, suffix)
253
- return '__lane:' .. tostring(string.len(lane)) .. ':' .. lane .. ':' .. suffix
254
- end
255
- local queued, ready_now, lane_tails, lane_order = {}, {}, {}, {}
256
- local function enqueue(field, lane)
257
- local score = tonumber(field)
258
- queued[#queued + 1] = score
259
- queued[#queued + 1] = field
260
- if lane == '' then
261
- ready_now[#ready_now + 1] = score
262
- ready_now[#ready_now + 1] = field
263
- return
369
+ const CURSOR_LUA = `
370
+ local served_all, served_set, served_logs = true, {}, {}
371
+ local pending_fast = false
372
+ local function types_of(raw)
373
+ if not raw or raw == '*' then return nil end
374
+ local set = {}
375
+ for _, name in ipairs(cjson.decode(raw)) do set[name] = true end
376
+ return set
377
+ end
378
+ local function takes(types, kind)
379
+ return kind ~= CLEAR_TYPE and (types == nil or types[kind] == true)
380
+ end
381
+ -- The served consumers: '' for every consumer, else [[name] or
382
+ -- [name, lane, typesJson or '*', backfill]].
383
+ local function use_served(raw)
384
+ if raw == '' then return end
385
+ served_all = false
386
+ for _, entry in ipairs(cjson.decode(raw)) do
387
+ served_set[entry[1]] = true
388
+ if entry[2] then
389
+ served_logs[#served_logs + 1] = {
390
+ name = entry[1], lane = entry[2], ty = entry[3],
391
+ types = types_of(entry[3]), backfill = entry[4] == 1,
392
+ }
393
+ end
264
394
  end
265
- put(mkey(field), field .. ':l', lane)
266
- local tail = lane_tails[lane]
267
- if tail == nil then
268
- tail = redis.call('HGET', META, lane_field(lane, 'tail'))
269
- if tail and tonumber(tail) <= boundary then tail = false end
270
- lane_order[#lane_order + 1] = lane
395
+ end
396
+ local function served_has(name) return served_all or served_set[name] == true end
397
+ -- Whether a served consumer has a live head. Newest heads come first: a
398
+ -- removed consumer's backlog is older than the work a deployment serves.
399
+ local function has_pending()
400
+ if pending_fast then return '1' end
401
+ if served_all then return redis.call('ZCOUNT', READY, 1, '+inf') > 0 and '1' or '0' end
402
+ if next(served_set) == nil then return '0' end
403
+ local page = redis.call('ZREVRANGEBYSCORE', READY, '+inf', 1, 'LIMIT', 0, 32)
404
+ for _, cid in ipairs(page) do
405
+ if served_set[(parse_cid(cid))] then return '1' end
271
406
  end
272
- if tail then
273
- put(mkey(tail), tail .. ':next', field)
274
- else
275
- put(META, lane_field(lane, 'head'), field)
276
- ready_now[#ready_now + 1] = score
277
- ready_now[#ready_now + 1] = field
407
+ if #page < 32 then return '0' end
408
+ for _, cid in ipairs(redis.call('ZREVRANGEBYSCORE', READY, '+inf', 1, 'LIMIT', 32, -1)) do
409
+ if served_set[(parse_cid(cid))] then return '1' end
410
+ end
411
+ return '0'
412
+ end
413
+ local function targets_of(raw)
414
+ if raw == '' then return {} end
415
+ return cjson.decode(raw)
416
+ end
417
+ -- Starts indexing a type: every later event of it.
418
+ local function register(name, from)
419
+ typed[name] = { from, from }
420
+ typed_dirty = true
421
+ end
422
+ -- The first stored event in (after, total] of a type in \`types\` (nil: any
423
+ -- type), or false. Nil and a type when that type's index can't answer yet.
424
+ local function stored_head(after, types)
425
+ if types == nil then
426
+ if after + 1 <= total then return after + 1 end
427
+ return false
428
+ end
429
+ local best = false
430
+ for name in pairs(types) do
431
+ local entry = typed[name]
432
+ if entry == nil then
433
+ if after < total then return nil, name end
434
+ register(name, total)
435
+ elseif entry[2] > after then
436
+ if entry[1] > after then return nil, name end
437
+ local found = redis.call('ZRANGEBYSCORE', type_key(name),
438
+ '(' .. tostring(after), tostring(total), 'LIMIT', 0, 1)
439
+ if #found > 0 then
440
+ local idx = tonumber(found[1])
441
+ if not best or idx < best then best = idx end
442
+ end
443
+ end
444
+ end
445
+ return best
446
+ end
447
+ -- The first event after \`after\` a log cursor takes: stored, then in the
448
+ -- batch being appended.
449
+ local function next_log_head(after, types, batch)
450
+ local head, missing = stored_head(after, types)
451
+ if head == nil then return nil, missing end
452
+ if head then return head end
453
+ for _, event in ipairs(batch) do
454
+ if event.idx > after and takes(types, event.type) then return event.idx end
455
+ end
456
+ return false
457
+ end
458
+ -- Indexes a batch's events of the types typed log cursors read.
459
+ local function index_batch(batch)
460
+ local adds, order = {}, {}
461
+ for _, event in ipairs(batch) do
462
+ local entry = typed[event.type]
463
+ if entry then
464
+ local list = adds[event.type]
465
+ if not list then
466
+ list = {}
467
+ adds[event.type] = list
468
+ order[#order + 1] = event.type
469
+ end
470
+ list[#list + 1] = event.idx
471
+ list[#list + 1] = tostring(event.idx)
472
+ entry[2] = event.idx
473
+ typed_dirty = true
474
+ end
475
+ end
476
+ for _, name in ipairs(order) do sliced('ZADD', type_key(name), adds[name]) end
477
+ end
478
+ local states, state_order = {}, {}
479
+ local function load_states(cids)
480
+ local need, names = {}, {}
481
+ for _, cid in ipairs(cids) do
482
+ local state = states[cid]
483
+ if (state == nil or state.partial) and not need[cid] then
484
+ need[cid] = true
485
+ need[#need + 1] = cid
486
+ names[#names + 1] = cid .. ':h'
487
+ names[#names + 1] = cid .. ':p'
488
+ names[#names + 1] = cid .. ':t'
489
+ names[#names + 1] = cid .. ':ty'
490
+ end
491
+ end
492
+ if #need == 0 then return end
493
+ local values = sliced('HMGET', CURSORS, names)
494
+ for n, cid in ipairs(need) do
495
+ local h, p, t, ty = values[n * 4 - 3], values[n * 4 - 2], values[n * 4 - 1], values[n * 4]
496
+ local tail = (t and t ~= '') and tonumber(t) or false
497
+ local state = states[cid]
498
+ if state then
499
+ state.t = tail
500
+ state.partial = nil
501
+ else
502
+ local _, tag = parse_cid(cid)
503
+ state = {
504
+ kind = string.sub(tag, 1, 1), exists = p ~= false,
505
+ h = (h and h ~= '') and tonumber(h) or false,
506
+ p = tonumber(p or '0'), t = tail, ty = ty or false,
507
+ }
508
+ if state.h and state.h <= boundary then state.h = false end
509
+ if state.kind == 'G' and state.exists and state.p < boundary then state.p = boundary end
510
+ states[cid] = state
511
+ state_order[#state_order + 1] = cid
512
+ end
278
513
  end
279
- lane_tails[lane] = field
280
514
  end
281
- local function clear_lane(lane)
282
- if lane_tails[lane] == nil then lane_order[#lane_order + 1] = lane end
283
- lane_tails[lane] = false
515
+ local function flush_states()
516
+ for _, cid in ipairs(state_order) do
517
+ local state = states[cid]
518
+ if state.dirty then
519
+ put(CURSORS, cid .. ':h', state.h and tostring(state.h) or '')
520
+ put(CURSORS, cid .. ':p', tostring(state.p))
521
+ if state.kind == 'L' then
522
+ if not state.partial then put(CURSORS, cid .. ':t', state.t and tostring(state.t) or '') end
523
+ elseif state.ty then
524
+ put(CURSORS, cid .. ':ty', state.ty)
525
+ end
526
+ end
527
+ end
284
528
  end
285
- local function flush_queue()
286
- for _, lane in ipairs(lane_order) do
287
- if lane_tails[lane] then
288
- put(META, lane_field(lane, 'tail'), lane_tails[lane])
289
- else
290
- redis.call('HDEL', META, lane_field(lane, 'head'), lane_field(lane, 'tail'))
529
+ -- Moves a cursor past its head \`idx\`. \`kind\` is the head row's kind;
530
+ -- \`batch\` is the returned events, appended after the stored log. Writes
531
+ -- nothing; returns nil and a type when that type's index must reach back
532
+ -- to \`idx\` first.
533
+ local function advance(consumer, kind, idx, row_next, batch)
534
+ if kind == '#' then
535
+ ready_set(cid_of(consumer, '#' .. tostring(idx)), false)
536
+ return true
537
+ end
538
+ local cid = cid_of(consumer, kind)
539
+ local state = states[cid]
540
+ local next_head
541
+ if string.sub(kind, 1, 1) == 'L' then
542
+ next_head = row_next and tonumber(row_next) or false
543
+ if not state then
544
+ state = { kind = 'L', exists = true, p = idx, h = false, t = false, ty = false, partial = true }
545
+ states[cid] = state
546
+ state_order[#state_order + 1] = cid
547
+ end
548
+ if not next_head then
549
+ state.t = false
550
+ state.partial = nil
551
+ end
552
+ else
553
+ local lane, raw = string.sub(kind, 2), nil
554
+ for _, spec in ipairs(served_logs) do
555
+ if spec.name == consumer and spec.lane == lane then raw = spec.ty end
556
+ end
557
+ if not raw then
558
+ load_states({ cid })
559
+ raw = states[cid].ty or '*'
291
560
  end
561
+ local head, missing = next_log_head(idx, types_of(raw), batch)
562
+ if head == nil then return nil, missing end
563
+ next_head = head
564
+ state = states[cid]
565
+ if not state then
566
+ state = { kind = 'G', exists = true, p = idx, h = false, t = false, ty = raw }
567
+ states[cid] = state
568
+ state_order[#state_order + 1] = cid
569
+ end
570
+ state.ty = raw
292
571
  end
293
- sliced('ZADD', PENDING, queued)
294
- sliced('ZADD', READY, ready_now)
572
+ state.h = next_head
573
+ state.p = idx
574
+ state.exists = true
575
+ state.dirty = true
576
+ ready_set(cid, next_head)
577
+ return true
295
578
  end
296
- `
297
-
298
- /**
299
- * Settles one unfinished event for scripts that include ENQUEUE_LUA: it
300
- * leaves the pending and ready sets, and a lane head hands its lane to the
301
- * next unfinished event. A dead-lettered successor stays out of the ready set.
302
- */
303
- const SETTLE_LUA = `
304
- local function settle(field, lane, next_field)
305
- redis.call('ZREM', PENDING, field)
306
- redis.call('ZREM', READY, field)
307
- if lane == '' then return end
308
- local head_key = lane_field(lane, 'head')
309
- if redis.call('HGET', META, head_key) ~= field then return end
310
- while next_field do
311
- local row = read_fields({ next_field }, { 'p', 'f', 'next' })[next_field]
312
- if not row[1] then
313
- redis.call('HSET', META, head_key, next_field)
314
- if not row[2] then
315
- redis.call('ZADD', READY, tonumber(next_field), next_field)
579
+ -- Reads what an appended batch's cursor work needs: its lane targets'
580
+ -- records, and the head of each served log consumer whose types it carries.
581
+ -- Writes nothing; returns nil, a type, and an index when that type's index
582
+ -- must reach back to the index first.
583
+ local log_plan = {}
584
+ local function plan(batch, base)
585
+ local need = {}
586
+ for _, event in ipairs(batch) do
587
+ for _, target in ipairs(event.targets) do
588
+ if target[2] ~= '#' then need[#need + 1] = cid_of(target[1], target[2]) end
589
+ end
590
+ end
591
+ for _, spec in ipairs(served_logs) do
592
+ for _, event in ipairs(batch) do
593
+ if takes(spec.types, event.type) then
594
+ local cid = cid_of(spec.name, 'G' .. spec.lane)
595
+ log_plan[#log_plan + 1] = { spec = spec, first = event.idx, cid = cid }
596
+ need[#need + 1] = cid
597
+ break
316
598
  end
317
- return
318
599
  end
319
- next_field = row[3]
320
600
  end
321
- clear_lane(lane)
601
+ load_states(need)
602
+ for _, log in ipairs(log_plan) do
603
+ local state, spec = states[log.cid], log.spec
604
+ if not state.exists then
605
+ if spec.backfill then
606
+ local head, missing = stored_head(boundary, spec.types)
607
+ if head == nil then return nil, missing, boundary end
608
+ log.p, log.h = boundary, head or log.first
609
+ else
610
+ -- Nothing it takes is stored after its start: index its types from there.
611
+ for name in pairs(spec.types or {}) do
612
+ if typed[name] == nil then register(name, base) end
613
+ end
614
+ log.p, log.h = base, log.first
615
+ end
616
+ elseif not state.h then
617
+ -- Not the batch's first: a deployment that doesn't serve this consumer
618
+ -- may have appended events it takes since it went idle.
619
+ local head, missing = stored_head(state.p, spec.types)
620
+ if head == nil then return nil, missing, state.p end
621
+ log.h = head or log.first
622
+ end
623
+ end
624
+ return true
322
625
  end
323
- `
324
-
325
- /**
326
- * Event-id ownership for scripts that add events. Each script reads the
327
- * counter, split, clear boundary, and per-id keys with one MGET and applies
328
- * the first three. Returns the owner record, or false, per event, and the
329
- * counter. A record this session made below its latest clear is free.
330
- */
331
- const ID_OWNERS_LUA = `
332
- local function read_owners(key_offset, n, session)
333
- local keys = { COUNTER, SPLIT, CLEAR }
334
- for i = 1, n do keys[#keys + 1] = KEYS[key_offset + i] end
335
- local values = sliced('MGET', nil, keys)
336
- use_split(values[2], values[1], values[3])
626
+ -- Starts the cursor work \`plan\` read: each event's targets, and each
627
+ -- planned log cursor.
628
+ local function enqueue(batch)
629
+ for _, event in ipairs(batch) do
630
+ for _, target in ipairs(event.targets) do
631
+ local consumer, tag = target[1], target[2]
632
+ if served_has(consumer) then pending_fast = true end
633
+ if tag == '#' then
634
+ ready_set(cid_of(consumer, '#' .. tostring(event.idx)), event.idx)
635
+ else
636
+ local cid = cid_of(consumer, tag)
637
+ local state = states[cid]
638
+ if state.h and state.t then
639
+ put_row(state.t, row_of(consumer, tag), 'next', tostring(event.idx))
640
+ else
641
+ state.h = event.idx
642
+ if state.p < boundary then state.p = boundary end
643
+ ready_set(cid, event.idx)
644
+ end
645
+ state.t = event.idx
646
+ state.exists = true
647
+ state.dirty = true
648
+ end
649
+ end
650
+ end
651
+ for _, log in ipairs(log_plan) do
652
+ pending_fast = true
653
+ local state = states[log.cid]
654
+ if not state.exists then
655
+ state.exists, state.p, state.h = true, log.p, log.h
656
+ ready_set(log.cid, log.h)
657
+ elseif not state.h then
658
+ state.h = log.h
659
+ ready_set(log.cid, log.h)
660
+ end
661
+ state.ty = log.spec.ty
662
+ state.dirty = true
663
+ end
664
+ end
665
+ -- Event-id owners for the extra keys read_bounds returned. A record this
666
+ -- session made below its latest clear is free.
667
+ local function owners_of(values, session)
337
668
  local owners = {}
338
- for i = 1, n do
339
- local owner = values[3 + i]
669
+ for i = 1, #values do
670
+ local owner = values[i]
340
671
  if owner then
341
672
  local rec = cjson.decode(owner)
342
673
  if rec[1] == session and tonumber(rec[2]) < boundary then owner = false end
343
674
  end
344
675
  owners[i] = owner
345
676
  end
346
- return owners, values[1]
677
+ return owners
347
678
  end
348
679
  `
349
680
 
@@ -381,51 +712,30 @@ local function wake(channel, frontier, created_at)
381
712
  end
382
713
  `
383
714
 
384
- const RELEASE_PINS_LUA = `
385
- local function release_snapshot_pins(pins, counts)
386
- local refs = redis.call('SMEMBERS', pins)
387
- if #refs == 0 then return end
388
- for _, ref in ipairs(refs) do
389
- local count = redis.call('HINCRBY', counts, ref, -1)
390
- if count <= 0 then
391
- redis.call('HDEL', counts, ref)
392
- local checkpoint = cjson.decode(ref)
393
- redis.call('ZREMRANGEBYSCORE', checkpoint[1], checkpoint[2], checkpoint[2])
394
- end
395
- end
396
- redis.call('DEL', pins)
397
- end
398
- `
399
-
400
715
  /**
401
- * KEYS: store stream, counter, metadata, pending, ready, metadata split,
402
- * then one id ownership key per event.
403
- * ARGV: sessionId, nowMs, n, then per event:
404
- * id, type, payloadJson, causeJson, lane, settled; then notify channel and
405
- * origin index or ''.
716
+ * KEYS: counter, layout, then one id ownership key per event.
717
+ * ARGV: sessionId, nowMs, n, served consumers, notify channel, origin index
718
+ * or '', then per event: id, type, payloadJson, causeJson or '', targets.
406
719
  * Returns ['empty', hasPending] | ['ok', firstIndex, hasPending] |
407
720
  * ['dup', hasPending, ...indexes] | ['partial', count] |
408
- * ['foreign', eventId] | ['no_parent', idx] | ['superseded', idx] |
409
- * ['cleared', originIndex].
721
+ * ['foreign', eventId] | ['superseded', position] | ['cleared', originIndex] |
722
+ * ['index', type, after].
410
723
  */
411
- const APPEND_LUA = `
412
- local COUNTER, META, SPLIT, PENDING, READY = KEYS[2], KEYS[3], KEYS[6], KEYS[4], KEYS[5]
413
- ${META_LUA}
414
- ${ENQUEUE_LUA}
415
- ${ID_OWNERS_LUA}
416
- ${WAKE_LUA}
417
- local function has_pending()
418
- return redis.call('ZCARD', PENDING) > 0 and '1' or '0'
419
- end
420
- local n = tonumber(ARGV[3])
724
+ const APPEND_LUA = `${BASE_LUA}${CURSOR_LUA}${WAKE_LUA}
725
+ local session_id, now, n = ARGV[1], ARGV[2], tonumber(ARGV[3])
726
+ use_served(ARGV[4])
421
727
  if n == 0 then return { 'empty', has_pending() } end
422
- local owners, count = read_owners(6, n, ARGV[1])
423
- local base = tonumber(count or '0')
728
+ local id_keys = {}
729
+ for i = 1, n do id_keys[i] = KEYS[2 + i] end
730
+ local values = read_bounds(id_keys)
731
+ if not values then return redis.error_reply('${LAYOUT_ERROR}') end
732
+ local owners = owners_of(values, session_id)
733
+ local base = total
424
734
  local dups = {}
425
735
  for i = 1, n do
426
736
  if owners[i] then
427
737
  local rec = cjson.decode(owners[i])
428
- if rec[1] ~= ARGV[1] then return { 'foreign', ARGV[3 + (i - 1) * 6 + 1] } end
738
+ if rec[1] ~= session_id then return { 'foreign', ARGV[6 + (i - 1) * 5 + 1] } end
429
739
  dups[#dups + 1] = tostring(rec[2])
430
740
  end
431
741
  end
@@ -435,253 +745,254 @@ if #dups == n then
435
745
  return res
436
746
  end
437
747
  if #dups > 0 then return { 'partial', tostring(#dups) } end
438
- local origin = ARGV[5 + n * 6]
439
- if origin and origin ~= '' and tonumber(origin) <= boundary then
440
- return { 'cleared', origin }
441
- end
442
- local causes, parents = {}, {}
748
+ local origin = ARGV[6]
749
+ if origin ~= '' and tonumber(origin) <= boundary then return { 'cleared', origin } end
750
+ -- The attempt-currency fence: a cause's head row must still be its cursor's
751
+ -- head, at that attempt, not dead-lettered.
752
+ local refs, fenced = {}, {}
443
753
  for i = 1, n do
444
- local cause = ARGV[3 + (i - 1) * 6 + 4]
445
- if cause ~= '' then
446
- local parsed = cjson.decode(cause)
447
- local pidx = tonumber(parsed.index)
448
- causes[#causes + 1] = parsed
449
- if pidx >= 1 and pidx <= base then parents[#parents + 1] = tostring(pidx) end
754
+ local raw = ARGV[6 + (i - 1) * 5 + 4]
755
+ if raw ~= '' then
756
+ local cause = cjson.decode(raw)
757
+ local index = tonumber(cause.index)
758
+ if index <= boundary then return { 'superseded', tostring(i) } end
759
+ local tag = type(cause.lane) == 'string' and ('L' .. cause.lane) or '#'
760
+ refs[#refs + 1] = { index, row_of(cause.consumer, tag) }
761
+ fenced[#fenced + 1] = {
762
+ position = i, consumer = cause.consumer, attempt = tonumber(cause.attempt), tag = tag,
763
+ }
450
764
  end
451
765
  end
452
- local parent_rows = read_fields(parents, { 'd', 'f', 'p' })
453
- local parent_pending = false
454
- for _, parsed in ipairs(causes) do
455
- local pidx = tonumber(parsed.index)
456
- if pidx <= boundary then return { 'superseded', tostring(pidx) } end
457
- if pidx < 1 or pidx > base then
458
- return { 'no_parent', tostring(parsed.index) }
766
+ local rows = read_rows(refs, { 'd', 'f', 'p', 'l' })
767
+ for k, cause in ipairs(fenced) do
768
+ local row = rows[k]
769
+ if tonumber(row[1] or '0') ~= cause.attempt or row[2] or row[3] or not owns(row[4], cause.tag) then
770
+ return { 'superseded', tostring(cause.position) }
459
771
  end
460
- local row = parent_rows[tostring(pidx)]
461
- if tonumber(row[1] or '0') ~= tonumber(parsed.attempt) or row[2] then
462
- return { 'superseded', tostring(pidx) }
463
- end
464
- -- A claimed, unprocessed parent is still in the pending set.
465
- if not row[3] then parent_pending = true end
772
+ if served_has(cause.consumer) then pending_fast = true end
773
+ end
774
+ local batch = {}
775
+ for i = 1, n do
776
+ local o = 6 + (i - 1) * 5
777
+ batch[i] = { idx = base + i, type = ARGV[o + 2], targets = targets_of(ARGV[o + 5]) }
466
778
  end
779
+ local planned, missing, after = plan(batch, base)
780
+ if not planned then return { 'index', missing, tostring(after) } end
467
781
  local owner_writes = {}
468
782
  for i = 1, n do
469
- local o = 3 + (i - 1) * 6
783
+ local o = 6 + (i - 1) * 5
470
784
  local field = tostring(base + i)
471
- redis.call('XADD', KEYS[1], field .. '-0',
785
+ redis.call('XADD', LOG, field .. '-0',
472
786
  'id', ARGV[o + 1], 'type', ARGV[o + 2], 'payload', ARGV[o + 3],
473
- 'created_at', ARGV[2], 'cause', ARGV[o + 4], 'lane', ARGV[o + 5])
474
- owner_writes[#owner_writes + 1] = KEYS[6 + i]
475
- owner_writes[#owner_writes + 1] = cjson.encode({ ARGV[1], base + i })
787
+ 'created_at', now, 'cause', ARGV[o + 4])
788
+ owner_writes[#owner_writes + 1] = KEYS[2 + i]
789
+ owner_writes[#owner_writes + 1] = cjson.encode({ session_id, base + i })
476
790
  wake_add(field, ARGV[o + 1], ARGV[o + 2], ARGV[o + 3])
477
- if ARGV[o + 6] == '1' then
478
- put(mkey(field), field .. ':p', ARGV[2])
479
- else
480
- enqueue(field, ARGV[o + 5])
481
- end
482
791
  end
792
+ index_batch(batch)
793
+ enqueue(batch)
483
794
  owner_writes[#owner_writes + 1] = COUNTER
484
795
  owner_writes[#owner_writes + 1] = tostring(base + n)
485
- split_write(owner_writes)
796
+ if schema_missing then
797
+ owner_writes[#owner_writes + 1] = SCHEMA
798
+ owner_writes[#owner_writes + 1] = '${REDIS_LAYOUT}'
799
+ end
800
+ if typed_dirty then
801
+ owner_writes[#owner_writes + 1] = TYPED
802
+ owner_writes[#owner_writes + 1] = cjson.encode(typed)
803
+ end
486
804
  sliced('MSET', nil, owner_writes)
487
- flush_queue()
805
+ flush_states()
488
806
  flush()
489
- local pending_after = (#queued > 0 or parent_pending) and '1' or has_pending()
490
- wake(ARGV[4 + n * 6], tostring(base + n), ARGV[2])
491
- return { 'ok', tostring(base + 1), pending_after }
807
+ flush_ready()
808
+ local pending = has_pending()
809
+ wake(ARGV[5], tostring(base + n), now)
810
+ return { 'ok', tostring(base + 1), pending }
492
811
  `
493
812
 
494
813
  /**
495
- * KEYS: counter, meta, pending, ready, event snapshot pins, snapshot pin counts,
496
- * metadata split.
497
- * ARGV: idx, attempt, error, maxFailures, nowMs, retryAtMs ('' for none).
814
+ * KEYS: counter, layout.
815
+ * ARGV: type, after. Indexes one page of the stored log below the type's
816
+ * index, toward `after`, registering the type first if no cursor read it
817
+ * yet. Returns ['1'] once the index reaches `after`, else ['0'].
498
818
  */
499
- const FAIL_ATTEMPT_LUA = `
500
- local COUNTER, META, SPLIT = KEYS[1], KEYS[2], KEYS[7]
501
- ${META_LUA}
502
- ${RELEASE_PINS_LUA}
503
- local idx = tonumber(ARGV[1])
504
- local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR)
505
- local total = tonumber(counts[1] or '0')
506
- use_split(counts[2], counts[1], counts[3])
507
- if idx <= boundary then return { 'cleared', '0' } end
508
- if idx < 1 or idx > total then return { 'missing', '0' } end
509
- local field = ARGV[1]
510
- local attempt = tonumber(ARGV[2])
511
- local row = read_fields({ field }, { 'a', 'd', 'p', 'f', 'lfa', 'ch' })[field]
512
- local failures = tonumber(row[1] or '0')
513
- if row[3] or tonumber(row[2] or '0') ~= attempt then
514
- return { 'superseded', tostring(failures) }
515
- end
516
- if row[4] then return { 'dead_lettered', tostring(failures) } end
517
- if row[5] and tonumber(row[5]) == attempt and not row[6] then
518
- return { 'failed', tostring(failures) }
519
- end
520
- if not row[6] then return { 'superseded', tostring(failures) } end
521
- failures = failures + 1
522
- local dead = failures >= tonumber(ARGV[4])
523
- local meta_key = mkey(field)
524
- put(meta_key, field .. ':a', tostring(failures))
525
- put(meta_key, field .. ':e', ARGV[3])
526
- put(meta_key, field .. ':lf', ARGV[5])
527
- put(meta_key, field .. ':lfa', ARGV[2])
528
- if dead then put(meta_key, field .. ':f', ARGV[5]) end
529
- local hold = not dead and ARGV[6] ~= ''
530
- if hold then put(meta_key, field .. ':ce', ARGV[6]) end
531
- flush()
532
- if hold then
533
- redis.call('HDEL', meta_key, field .. ':ch')
534
- else
535
- redis.call('HDEL', meta_key, field .. ':ch', field .. ':ce')
536
- end
537
- if dead then
538
- redis.call('ZREM', KEYS[4], field)
539
- release_snapshot_pins(KEYS[5], KEYS[6])
540
- return { 'dead_lettered', tostring(failures) }
819
+ const INDEX_TYPE_LUA = `${BASE_LUA}
820
+ if not read_bounds() then return redis.error_reply('${LAYOUT_ERROR}') end
821
+ local name, after = ARGV[1], tonumber(ARGV[2])
822
+ local entry = typed[name]
823
+ if not entry then
824
+ entry = { total, total }
825
+ typed[name] = entry
826
+ end
827
+ if entry[1] > after then
828
+ local entries = redis.call('XREVRANGE', LOG, tostring(entry[1]) .. '-0',
829
+ tostring(after + 1) .. '-0', 'COUNT', ${READ_PAGE_SIZE})
830
+ local adds = {}
831
+ for _, stored in ipairs(entries) do
832
+ local fields = stored[2]
833
+ for j = 1, #fields, 2 do
834
+ if fields[j] == 'type' then
835
+ if fields[j + 1] == name then
836
+ local idx = entry_index(stored[1])
837
+ adds[#adds + 1] = idx
838
+ adds[#adds + 1] = tostring(idx)
839
+ end
840
+ break
841
+ end
842
+ end
843
+ end
844
+ sliced('ZADD', type_key(name), adds)
845
+ if #entries < ${READ_PAGE_SIZE} then
846
+ entry[1] = after
847
+ else
848
+ entry[1] = entry_index(entries[#entries][1]) - 1
849
+ end
541
850
  end
542
- return { 'failed', tostring(failures) }
851
+ redis.call('SET', TYPED, cjson.encode(typed))
852
+ return { entry[1] <= after and '1' or '0' }
543
853
  `
544
854
 
545
855
  /**
546
- * KEYS: store, counter, meta, pending, ready, metadata split, snapshot pin
547
- * counts.
548
- * ARGV: holder, nowMs, expiresAtMs, excludedCount, excluded indexes, then
549
- * maxFailures ('' for none) and the event snapshot-pins key prefix.
550
- * The ready set is a subset of the pending set, so an empty ready set is
551
- * enough to report 'settled'. A claimed result ends with the clear boundary.
856
+ * KEYS: counter, layout.
857
+ * ARGV: holder, nowMs, expiresAtMs, maxFailures or '', served consumers,
858
+ * the highest head to claim or '', excluded count, then per excluded head:
859
+ * consumer, tag.
860
+ * Live heads are the ready set's positive scores, so an empty one is enough
861
+ * to report 'settled'. A claimed result ends with the clear marker.
552
862
  */
553
- const CLAIM_AVAILABLE_LUA = `
554
- local COUNTER, META, SPLIT = KEYS[2], KEYS[3], KEYS[6]
555
- ${META_LUA}
556
- ${RELEASE_PINS_LUA}
557
- local ready = redis.call('ZRANGE', KEYS[5], 0, -1)
863
+ const CLAIM_AVAILABLE_LUA = `${BASE_LUA}${CURSOR_LUA}
864
+ local ready = redis.call('ZRANGEBYSCORE', READY, 1, ARGV[6] == '' and '+inf' or ARGV[6], 'WITHSCORES')
865
+ if #ready == 0 then return { 'settled' } end
866
+ if not read_bounds() then return redis.error_reply('${LAYOUT_ERROR}') end
867
+ local holder, now, expires_at = ARGV[1], tonumber(ARGV[2]), ARGV[3]
868
+ local max_failures = tonumber(ARGV[4])
869
+ use_served(ARGV[5])
558
870
  local excluded = {}
559
- local tail = 5 + tonumber(ARGV[4])
560
- for i = 1, tonumber(ARGV[4]) do excluded[ARGV[4 + i]] = true end
561
- local max_failures = tonumber(ARGV[tail])
871
+ for i = 1, tonumber(ARGV[7]) do
872
+ excluded[ARGV[7 + (i - 1) * 2 + 1] .. '\\0' .. ARGV[7 + (i - 1) * 2 + 2]] = true
873
+ end
562
874
  local candidates = {}
563
- for _, field in ipairs(ready) do
564
- if not excluded[field] then candidates[#candidates + 1] = field end
565
- end
566
- local rows = read_fields(candidates, { 'ce', 'd', 'fc', 'a', 'e', 'lf', 'lfa' })
567
- local claimed = {}
568
- local dead = {}
569
- local retry_at = nil
570
- for _, field in ipairs(candidates) do
571
- local row = rows[field]
572
- local expires = tonumber(row[1] or '0')
573
- if expires > tonumber(ARGV[2]) then
875
+ for i = 1, #ready, 2 do
876
+ local cid, idx = ready[i], tonumber(ready[i + 1])
877
+ local consumer, tag = parse_cid(cid)
878
+ local kind = string.sub(tag, 1, 1)
879
+ local key = consumer .. '\\0' .. (kind == 'G' and ('L' .. string.sub(tag, 2)) or tag)
880
+ if served_has(consumer) and not excluded[key] then
881
+ candidates[#candidates + 1] = {
882
+ idx, row_of(consumer, tag), consumer = consumer, cid = cid,
883
+ kind = kind == '#' and '#' or tag,
884
+ }
885
+ end
886
+ end
887
+ local rows = read_rows(candidates, { 'd', 'fc', 'lc', 'ch', 'ce', 'a', 'lf', 'lfa', 'e', 'f', 'pins' })
888
+ local takes_over, retry_at, wanted = {}, nil, {}
889
+ for position, candidate in ipairs(candidates) do
890
+ local row = rows[position]
891
+ local expires = tonumber(row[5] or '0')
892
+ if expires > now then
574
893
  if not retry_at or expires < retry_at then retry_at = expires end
575
894
  else
576
- local entries = redis.call('XRANGE', KEYS[1], field .. '-0', field .. '-0')
577
- if #entries == 0 then return { 'missing', field } end
578
- local meta_key = mkey(field)
579
- local attempt = tostring(tonumber(row[2] or '0') + 1)
580
- -- An expiry that is not the current attempt's retry hold is a lapsed
581
- -- claim: its attempt never finished.
582
- local lapsed = row[1] and row[7] ~= row[2]
583
- local failures = tonumber(row[4] or '0')
584
- if lapsed then
585
- failures = failures + 1
586
- put(meta_key, field .. ':a', tostring(failures))
587
- end
588
- put(meta_key, field .. ':d', attempt)
589
- if lapsed and max_failures and failures >= max_failures then
590
- -- That takeover spent the last failed attempt: dead-letter, unclaimed.
591
- -- Advancing the attempt fences the lapsed one; nothing runs.
592
- put(meta_key, field .. ':f', ARGV[2])
593
- dead[#dead + 1] = field
594
- claimed[#claimed + 1] = {
595
- attempt,
596
- tostring(failures),
597
- row[5] or false,
598
- row[3] or false,
599
- read_fields({ field }, { 'lc' })[field][1] or false,
600
- row[6] or false,
601
- row[7] or false,
602
- false,
603
- false,
604
- entries[1],
605
- ARGV[2]
606
- }
607
- else
608
- local first_claimed = row[3] or ARGV[2]
609
- put(meta_key, field .. ':fc', first_claimed)
610
- put(meta_key, field .. ':lc', ARGV[2])
611
- put(meta_key, field .. ':ch', ARGV[1])
612
- put(meta_key, field .. ':ce', ARGV[3])
613
- claimed[#claimed + 1] = {
614
- attempt,
615
- tostring(failures),
616
- row[5] or false,
617
- first_claimed,
618
- ARGV[2],
619
- row[6] or false,
620
- row[7] or false,
621
- ARGV[1],
622
- ARGV[3],
623
- entries[1],
624
- false
625
- }
626
- end
895
+ takes_over[#takes_over + 1] = { candidate = candidate, row = row }
896
+ wanted[candidate[1]] = true
627
897
  end
628
898
  end
629
- flush()
630
- for _, field in ipairs(dead) do
631
- redis.call('HDEL', mkey(field), field .. ':ch', field .. ':ce')
632
- redis.call('ZREM', KEYS[5], field)
633
- release_snapshot_pins(ARGV[tail + 1] .. field, KEYS[7])
634
- end
635
- if #claimed > 0 then
636
- local result = { 'claimed', tostring(#claimed) }
637
- for _, claim in ipairs(claimed) do
638
- for _, value in ipairs(claim) do result[#result + 1] = value end
899
+ if #takes_over == 0 then
900
+ if retry_at then return { 'busy', tostring(retry_at) } end
901
+ return { 'settled' }
902
+ end
903
+ -- One XRANGE per run of consecutive claimed indexes.
904
+ local indexes, entries = {}, {}
905
+ for idx in pairs(wanted) do indexes[#indexes + 1] = idx end
906
+ table.sort(indexes)
907
+ local first = 1
908
+ while first <= #indexes do
909
+ local last = first
910
+ while last < #indexes and indexes[last + 1] == indexes[last] + 1 do last = last + 1 end
911
+ local range = redis.call('XRANGE', LOG, tostring(indexes[first]) .. '-0', tostring(indexes[last]) .. '-0')
912
+ for _, entry in ipairs(range) do entries[entry_index(entry[1])] = entry end
913
+ first = last + 1
914
+ end
915
+ for _, idx in ipairs(indexes) do
916
+ if not entries[idx] then return { 'missing', tostring(idx) } end
917
+ end
918
+ local result = { 'claimed', tostring(#takes_over) }
919
+ for _, take in ipairs(takes_over) do
920
+ local candidate, row = take.candidate, take.row
921
+ local idx, at = candidate[1], candidate[2]
922
+ local attempt = tostring(tonumber(row[1] or '0') + 1)
923
+ -- A claim still held here lapsed: its attempt never finished.
924
+ local lapsed = row[4] ~= false
925
+ local failures = tonumber(row[6] or '0')
926
+ if lapsed then
927
+ failures = failures + 1
928
+ put_row(idx, at, 'a', tostring(failures))
639
929
  end
640
- result[#result + 1] = tostring(boundary)
641
- return result
930
+ put_row(idx, at, 'd', attempt)
931
+ local columns
932
+ if lapsed and max_failures and failures >= max_failures then
933
+ -- That takeover spent the last failed attempt: dead-letter, unclaimed.
934
+ -- Advancing the attempt fences the lapsed one; nothing runs.
935
+ put_row(idx, at, 'f', ARGV[2])
936
+ drop_row(idx, at, 'ch')
937
+ drop_row(idx, at, 'ce')
938
+ ready_set(candidate.cid, -idx)
939
+ release_pins(idx, at, row[11])
940
+ columns = { attempt, row[2], row[3], false, false, tostring(failures), row[7], row[8], row[9], ARGV[2] }
941
+ else
942
+ local first_claimed = row[2] or ARGV[2]
943
+ put_row(idx, at, 'fc', first_claimed)
944
+ put_row(idx, at, 'lc', ARGV[2])
945
+ put_row(idx, at, 'ch', holder)
946
+ put_row(idx, at, 'ce', expires_at)
947
+ put_row(idx, at, 'l', candidate.kind)
948
+ columns = { attempt, first_claimed, ARGV[2], holder, expires_at, tostring(failures), row[7], row[8], row[9], false }
949
+ end
950
+ result[#result + 1] = { candidate.consumer, candidate.kind, tostring(idx), columns, entries[idx] }
642
951
  end
643
- if retry_at then return { 'busy', tostring(retry_at) } end
644
- return { 'settled' }
952
+ flush()
953
+ flush_ready()
954
+ result[#result + 1] = tostring(boundary)
955
+ result[#result + 1] = tostring(total)
956
+ return result
645
957
  `
646
958
 
647
959
  /**
648
- * KEYS: counter, meta, metadata split.
649
- * ARGV: holder, nowMs, expiresAtMs, n, then per claim: index, attempt.
650
- * Returns [renewedCount, ...renewed, ...superseded].
960
+ * KEYS: counter, layout.
961
+ * ARGV: holder, nowMs, expiresAtMs, n, then per claim: consumer, tag, index,
962
+ * attempt. Returns [renewedCount, ...renewed, ...superseded] as 1-based
963
+ * positions in the request.
651
964
  */
652
- const RENEW_CLAIMS_LUA = `
653
- local COUNTER, META, SPLIT = KEYS[1], KEYS[2], KEYS[3]
654
- ${META_LUA}
655
- local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR)
656
- local total = tonumber(counts[1] or '0')
657
- use_split(counts[2], counts[1], counts[3])
658
- local claims, fields, cleared = {}, {}, {}
965
+ const RENEW_CLAIMS_LUA = `${BASE_LUA}
966
+ if not read_bounds() then return redis.error_reply('${LAYOUT_ERROR}') end
967
+ local holder, now, expires_at = ARGV[1], tonumber(ARGV[2]), ARGV[3]
968
+ local refs, items, superseded = {}, {}, {}
659
969
  for i = 1, tonumber(ARGV[4]) do
660
- local field = ARGV[4 + (i - 1) * 2 + 1]
661
- local idx = tonumber(field)
970
+ local o = 4 + (i - 1) * 4
971
+ local idx = tonumber(ARGV[o + 3])
662
972
  if idx <= boundary then
663
- cleared[#cleared + 1] = field
664
- elseif idx >= 1 and idx <= total then
665
- claims[#claims + 1] = { field = field, attempt = tonumber(ARGV[4 + (i - 1) * 2 + 2]) }
666
- fields[#fields + 1] = field
973
+ superseded[#superseded + 1] = tostring(i)
974
+ else
975
+ local row = row_of(ARGV[o + 1], ARGV[o + 2])
976
+ refs[#refs + 1] = { idx, row }
977
+ items[#items + 1] = { position = i, idx = idx, row = row, tag = ARGV[o + 2], attempt = tonumber(ARGV[o + 4]) }
667
978
  end
668
979
  end
669
- local rows = read_fields(fields, { 'd', 'ce', 'p', 'f', 'ch' })
980
+ local rows = read_rows(refs, { 'd', 'ce', 'p', 'f', 'ch', 'l' })
670
981
  local renewed = {}
671
- local superseded = {}
672
- for _, claim in ipairs(claims) do
673
- local row = rows[claim.field]
674
- if tonumber(row[1] or '0') > claim.attempt then
675
- superseded[#superseded + 1] = claim.field
676
- elseif tonumber(ARGV[3]) > tonumber(ARGV[2]) and
677
- not row[3] and not row[4] and row[5] == ARGV[1] and
678
- tonumber(row[2] or '0') > tonumber(ARGV[2]) then
679
- put(mkey(claim.field), claim.field .. ':ce', ARGV[3])
680
- renewed[#renewed + 1] = claim.field
982
+ for k, item in ipairs(items) do
983
+ local row = rows[k]
984
+ local attempt = tonumber(row[1] or '0')
985
+ if row[6] and not owns(row[6], item.tag) then
986
+ -- Another cursor's head: this one never claimed it.
987
+ elseif row[3] or attempt > item.attempt then
988
+ superseded[#superseded + 1] = tostring(item.position)
989
+ elseif attempt == item.attempt and tonumber(expires_at) > now and
990
+ not row[4] and row[5] == holder and tonumber(row[2] or '0') > now then
991
+ put_row(item.idx, item.row, 'ce', expires_at)
992
+ renewed[#renewed + 1] = tostring(item.position)
681
993
  end
682
994
  end
683
995
  flush()
684
- for _, field in ipairs(cleared) do superseded[#superseded + 1] = field end
685
996
  local result = { tostring(#renewed) }
686
997
  for i = 1, #renewed do result[#result + 1] = renewed[i] end
687
998
  for i = 1, #superseded do result[#result + 1] = superseded[i] end
@@ -689,228 +1000,403 @@ return result
689
1000
  `
690
1001
 
691
1002
  /**
692
- * KEYS: store, counter, meta, pending, ready, event snapshot pins, snapshot
693
- * pin counts, metadata split, then one id ownership key per child.
694
- * ARGV: sessionId, nowMs, parent index, attempt, n, then per child:
695
- * id, type, payloadJson, lane, settled; finally notify channel.
1003
+ * KEYS: counter, layout, then one id ownership key per child.
1004
+ * ARGV: sessionId, nowMs, consumer, tag, index, attempt, child causeJson,
1005
+ * served consumers, notify channel, returned idsJson, n, then per child: id,
1006
+ * type, payloadJson, targets.
696
1007
  */
697
- const COMPLETE_ATTEMPT_LUA = `
698
- local COUNTER, META, SPLIT, PENDING, READY = KEYS[2], KEYS[3], KEYS[8], KEYS[4], KEYS[5]
699
- ${META_LUA}
700
- ${ENQUEUE_LUA}
701
- ${ID_OWNERS_LUA}
702
- ${RELEASE_PINS_LUA}
703
- ${WAKE_LUA}
704
- ${SETTLE_LUA}
705
- local parent = ARGV[3]
706
- local attempt = tonumber(ARGV[4])
707
- local n = tonumber(ARGV[5])
708
- local owners, count = read_owners(8, n, ARGV[1])
709
- local total = tonumber(count or '0')
710
- if tonumber(parent) <= boundary then return { 'superseded' } end
711
- if tonumber(parent) < 1 or tonumber(parent) > total then return { 'missing' } end
712
- local row = read_fields({ parent }, { 'd', 'p', 'pa', 'ri', 'ch', 'f', 'lfa', 'l', 'next' })[parent]
1008
+ const COMPLETE_ATTEMPT_LUA = `${BASE_LUA}${CURSOR_LUA}${WAKE_LUA}
1009
+ local session_id, now = ARGV[1], ARGV[2]
1010
+ local consumer, tag, idx, attempt = ARGV[3], ARGV[4], tonumber(ARGV[5]), tonumber(ARGV[6])
1011
+ local at = row_of(consumer, tag)
1012
+ local n = tonumber(ARGV[11])
1013
+ local id_keys = {}
1014
+ for i = 1, n do id_keys[i] = KEYS[2 + i] end
1015
+ local values = read_bounds(id_keys)
1016
+ if not values then return redis.error_reply('${LAYOUT_ERROR}') end
1017
+ local owners = owners_of(values, session_id)
1018
+ if idx <= boundary then return { 'superseded' } end
1019
+ if idx < 1 or idx > total then return { 'missing' } end
1020
+ local row = read_rows({ { idx, at } }, { 'd', 'p', 'pa', 'ri', 'ch', 'f', 'l', 'next', 'pins' })[1]
713
1021
  if row[2] then
714
- if tonumber(row[3] or '0') ~= attempt then return { 'superseded' } end
715
- if not row[4] then return { 'partial' } end
716
- local recorded_ids = cjson.decode(row[4])
717
- if #recorded_ids ~= n then return { 'partial' } end
1022
+ -- The cursor moved past this head: a retry of its completion replays.
1023
+ if not owns(row[7], tag) then return { 'superseded' } end
1024
+ if not row[3] or tonumber(row[3]) ~= attempt then return { 'superseded' } end
1025
+ local recorded = cjson.decode(row[4] or '[]')
1026
+ if #recorded ~= n then return { 'partial' } end
718
1027
  for i = 1, n do
719
- if recorded_ids[i] ~= ARGV[5 + (i - 1) * 5 + 1] then return { 'partial' } end
1028
+ if recorded[i] ~= ARGV[11 + (i - 1) * 4 + 1] then return { 'partial' } end
720
1029
  end
721
- local indexes = {}
1030
+ local result = { 'duplicate' }
722
1031
  for i = 1, n do
723
1032
  if not owners[i] then return { 'partial' } end
724
1033
  local rec = cjson.decode(owners[i])
725
- if rec[1] ~= ARGV[1] then return { 'partial' } end
726
- local entries = redis.call('XRANGE', KEYS[1], tostring(rec[2]) .. '-0', tostring(rec[2]) .. '-0')
727
- if #entries == 0 then return { 'partial' } end
728
- local values = entries[1][2]
729
- local cause = nil
730
- for j = 1, #values, 2 do
731
- if values[j] == 'cause' then cause = values[j + 1] break end
732
- end
733
- if not cause or cause == '' then return { 'partial' } end
734
- local decoded = cjson.decode(cause)
735
- if tonumber(decoded['index']) ~= tonumber(parent) or
736
- tonumber(decoded['attempt']) ~= attempt then return { 'partial' } end
737
- indexes[#indexes + 1] = tostring(rec[2])
1034
+ if rec[1] ~= session_id then return { 'partial' } end
1035
+ result[#result + 1] = tostring(rec[2])
738
1036
  end
739
- local result = { 'duplicate' }
740
- for _, idx in ipairs(indexes) do result[#result + 1] = idx end
741
1037
  return result
742
1038
  end
743
- if tonumber(row[1] or '0') ~= attempt then return { 'superseded' } end
744
- if not row[5] or row[6] or tonumber(row[7] or '0') == attempt then
1039
+ if tonumber(row[1] or '0') ~= attempt or not row[5] or row[6] or not owns(row[7], tag) then
745
1040
  return { 'superseded' }
746
1041
  end
747
1042
  for i = 1, n do
748
1043
  if owners[i] then return { 'partial' } end
749
1044
  end
750
- local returned_ids = {}
751
- for i = 1, n do returned_ids[i] = ARGV[5 + (i - 1) * 5 + 1] end
752
- local returned_json = '[]'
753
- if n > 0 then returned_json = cjson.encode(returned_ids) end
754
- local parent_key = mkey(parent)
755
- put(parent_key, parent .. ':p', ARGV[2])
756
- put(parent_key, parent .. ':pa', ARGV[4])
757
- put(parent_key, parent .. ':ri', returned_json)
758
- redis.call('HDEL', parent_key, parent .. ':ch', parent .. ':ce')
759
- settle(parent, row[8] or '', row[9])
760
- release_snapshot_pins(KEYS[6], KEYS[7])
761
- local cause = cjson.encode({ index = tonumber(parent), attempt = attempt })
1045
+ use_served(ARGV[8])
1046
+ local batch = {}
1047
+ for i = 1, n do
1048
+ local o = 11 + (i - 1) * 4
1049
+ batch[i] = { idx = total + i, type = ARGV[o + 2], targets = targets_of(ARGV[o + 4]) }
1050
+ end
1051
+ local moved, missing = advance(consumer, row[7], idx, row[8], batch)
1052
+ if not moved then return { 'index', missing, tostring(idx) } end
1053
+ local planned, unindexed, after = plan(batch, total)
1054
+ if not planned then return { 'index', unindexed, tostring(after) } end
1055
+ put_row(idx, at, 'p', now)
1056
+ put_row(idx, at, 'pa', ARGV[6])
1057
+ put_row(idx, at, 'ri', ARGV[10])
1058
+ release_pins(idx, at, row[9])
762
1059
  local owner_writes = {}
763
1060
  for i = 1, n do
764
- local o = 5 + (i - 1) * 5
1061
+ local o = 11 + (i - 1) * 4
765
1062
  local field = tostring(total + i)
766
- redis.call('XADD', KEYS[1], field .. '-0',
1063
+ redis.call('XADD', LOG, field .. '-0',
767
1064
  'id', ARGV[o + 1], 'type', ARGV[o + 2], 'payload', ARGV[o + 3],
768
- 'created_at', ARGV[2], 'cause', cause, 'lane', ARGV[o + 4])
769
- owner_writes[#owner_writes + 1] = KEYS[8 + i]
770
- owner_writes[#owner_writes + 1] = cjson.encode({ ARGV[1], total + i })
1065
+ 'created_at', now, 'cause', ARGV[7])
1066
+ owner_writes[#owner_writes + 1] = KEYS[2 + i]
1067
+ owner_writes[#owner_writes + 1] = cjson.encode({ session_id, total + i })
771
1068
  wake_add(field, ARGV[o + 1], ARGV[o + 2], ARGV[o + 3])
772
- if ARGV[o + 5] == '1' then
773
- put(mkey(field), field .. ':p', ARGV[2])
774
- else
775
- enqueue(field, ARGV[o + 4])
776
- end
777
1069
  end
1070
+ index_batch(batch)
1071
+ enqueue(batch)
778
1072
  if n > 0 then
779
1073
  owner_writes[#owner_writes + 1] = COUNTER
780
1074
  owner_writes[#owner_writes + 1] = tostring(total + n)
781
- split_write(owner_writes)
782
- sliced('MSET', nil, owner_writes)
1075
+ if schema_missing then
1076
+ owner_writes[#owner_writes + 1] = SCHEMA
1077
+ owner_writes[#owner_writes + 1] = '${REDIS_LAYOUT}'
1078
+ end
1079
+ end
1080
+ if typed_dirty then
1081
+ owner_writes[#owner_writes + 1] = TYPED
1082
+ owner_writes[#owner_writes + 1] = cjson.encode(typed)
783
1083
  end
784
- flush_queue()
1084
+ sliced('MSET', nil, owner_writes)
1085
+ flush_states()
785
1086
  flush()
786
- if n > 0 then wake(ARGV[6 + n * 5], tostring(total + n), ARGV[2]) end
1087
+ flush_ready()
1088
+ if n > 0 then wake(ARGV[9], tostring(total + n), now) end
787
1089
  return { 'ok', tostring(total + 1) }
788
1090
  `
789
1091
 
790
1092
  /**
791
- * KEYS: counter, meta, pending, ready, metadata split.
792
- * ARGV: idx, action ('retry' | 'skip'), nowMs.
793
- * Returns [redriven, hasPending]. Acts only on an unfinished, dead-lettered
794
- * event above the clear boundary. A retry keeps the event pending and makes
795
- * it ready again when it is unlaned or its lane's head.
1093
+ * KEYS: counter, layout.
1094
+ * ARGV: consumer, tag, index, attempt, error, maxFailures, nowMs, retryAtMs
1095
+ * or ''.
796
1096
  */
797
- const REDRIVE_LUA = `
798
- local COUNTER, META, SPLIT, PENDING, READY = KEYS[1], KEYS[2], KEYS[5], KEYS[3], KEYS[4]
799
- ${META_LUA}
800
- ${ENQUEUE_LUA}
801
- ${SETTLE_LUA}
802
- local function result(redriven)
803
- return { redriven, redis.call('ZCARD', PENDING) > 0 and '1' or '0' }
804
- end
805
- local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR)
806
- local total = tonumber(counts[1] or '0')
807
- use_split(counts[2], counts[1], counts[3])
808
- local idx = tonumber(ARGV[1])
809
- if idx <= boundary or idx > total then return result('0') end
810
- local field = tostring(idx)
811
- local row = read_fields({ field }, { 'p', 'f', 'l', 'next' })[field]
812
- if row[1] or not row[2] then return result('0') end
813
- local meta_key = mkey(field)
814
- local lane = row[3] or ''
815
- if ARGV[2] == 'skip' then
816
- put(meta_key, field .. ':p', ARGV[3])
817
- settle(field, lane, row[4])
818
- flush_queue()
819
- flush()
1097
+ const FAIL_ATTEMPT_LUA = `${BASE_LUA}
1098
+ if not read_bounds() then return redis.error_reply('${LAYOUT_ERROR}') end
1099
+ local consumer, tag, idx, attempt = ARGV[1], ARGV[2], tonumber(ARGV[3]), tonumber(ARGV[4])
1100
+ local at = row_of(consumer, tag)
1101
+ if idx <= boundary then return { 'superseded', '0' } end
1102
+ if idx < 1 or idx > total then return { 'missing', '0' } end
1103
+ local row = read_rows({ { idx, at } }, { 'a', 'd', 'p', 'f', 'lfa', 'ch', 'l', 'pins' })[1]
1104
+ if row[3] or (row[7] and not owns(row[7], tag)) then return { 'superseded', '0' } end
1105
+ local failures = tonumber(row[1] or '0')
1106
+ if tonumber(row[2] or '0') ~= attempt then return { 'superseded', tostring(failures) } end
1107
+ if not row[6] and row[5] and tonumber(row[5]) == attempt then
1108
+ return { row[4] and 'dead_lettered' or 'failed', tostring(failures) }
1109
+ end
1110
+ if row[4] then return { 'dead_lettered', tostring(failures) } end
1111
+ if not row[6] then return { 'superseded', tostring(failures) } end
1112
+ failures = failures + 1
1113
+ local dead = failures >= tonumber(ARGV[6])
1114
+ put_row(idx, at, 'a', tostring(failures))
1115
+ put_row(idx, at, 'e', ARGV[5])
1116
+ put_row(idx, at, 'lf', ARGV[7])
1117
+ put_row(idx, at, 'lfa', ARGV[4])
1118
+ drop_row(idx, at, 'ch')
1119
+ if dead then
1120
+ put_row(idx, at, 'f', ARGV[7])
1121
+ drop_row(idx, at, 'ce')
1122
+ ready_set(cid_for(consumer, row[7], idx), -idx)
1123
+ release_pins(idx, at, row[8])
1124
+ elseif ARGV[8] ~= '' then
1125
+ put_row(idx, at, 'ce', ARGV[8])
820
1126
  else
821
- redis.call('HDEL', meta_key, field .. ':f', field .. ':a')
822
- if lane == '' or redis.call('HGET', META, lane_field(lane, 'head')) == field then
823
- redis.call('ZADD', READY, idx, field)
1127
+ drop_row(idx, at, 'ce')
1128
+ end
1129
+ flush()
1130
+ flush_ready()
1131
+ return { dead and 'dead_lettered' or 'failed', tostring(failures) }
1132
+ `
1133
+
1134
+ /**
1135
+ * KEYS: counter, layout. ARGV: consumer, index, action, nowMs, served
1136
+ * consumers. Returns [redriven, hasPending] | ['index', type, after]. Acts
1137
+ * only on the consumer's dead-lettered head at the index, above the clear
1138
+ * marker.
1139
+ */
1140
+ const REDRIVE_LUA = `${BASE_LUA}${CURSOR_LUA}
1141
+ if not read_bounds() then return redis.error_reply('${LAYOUT_ERROR}') end
1142
+ use_served(ARGV[5])
1143
+ local consumer, idx = ARGV[1], tonumber(ARGV[2])
1144
+ local function result(redriven) return { redriven, has_pending() } end
1145
+ if idx <= boundary or idx < 1 or idx > total then return result('0') end
1146
+ local cid, tag = nil, nil
1147
+ for _, member in ipairs(redis.call('ZRANGEBYSCORE', READY, -idx, -idx)) do
1148
+ local owner, member_tag = parse_cid(member)
1149
+ if owner == consumer then
1150
+ cid, tag = member, member_tag
1151
+ break
824
1152
  end
825
1153
  end
1154
+ if not cid then return result('0') end
1155
+ local kind = string.sub(tag, 1, 1) == '#' and '#' or tag
1156
+ local at = row_of(consumer, tag)
1157
+ if ARGV[3] == 'skip' then
1158
+ local row = read_rows({ { idx, at } }, { 'next' })[1]
1159
+ local moved, missing = advance(consumer, kind, idx, row[1], {})
1160
+ if not moved then return { 'index', missing, tostring(idx) } end
1161
+ put_row(idx, at, 'p', ARGV[4])
1162
+ flush_states()
1163
+ if typed_dirty then redis.call('SET', TYPED, cjson.encode(typed)) end
1164
+ else
1165
+ drop_row(idx, at, 'f')
1166
+ drop_row(idx, at, 'a')
1167
+ ready_set(cid, idx)
1168
+ end
1169
+ flush()
1170
+ flush_ready()
826
1171
  return result('1')
827
1172
  `
828
1173
 
829
1174
  /**
830
- * KEYS: store, counter, meta, pending, metadata split.
831
- * Returns the dead-lettered events above the clear boundary, oldest first,
832
- * as pairs of a stream entry and its STORED_EVENT_META_SUFFIXES columns.
833
- * Every unfinished event is in the pending set, so the scan is bounded by
834
- * unfinished work, not history; only dead letters read their full columns.
1175
+ * KEYS: counter, layout. ARGV: consumer, lane, types JSON or '*', backfill
1176
+ * ('1' or '0'). Brings one log cursor up to date without an append, as
1177
+ * `plan` and `enqueue` would for an append carrying its types: a missing
1178
+ * cursor starts at the clear marker with backfill, else at the head with
1179
+ * nothing to run; an idle one takes the next stored event of its types; one
1180
+ * with a head, claimed or not, keeps it. Records the types, and writes
1181
+ * nothing when nothing changed. Returns ['ok', position, head or nil, head
1182
+ * columns, session head] | ['index', type, after].
835
1183
  */
836
- const DEAD_LETTERS_LUA = `
837
- local COUNTER, META, SPLIT = KEYS[2], KEYS[3], KEYS[5]
838
- ${META_LUA}
839
- local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR)
840
- use_split(counts[2], counts[1], counts[3])
841
- local candidates = {}
842
- for _, field in ipairs(redis.call('ZRANGE', KEYS[4], 0, -1)) do
843
- if tonumber(field) > boundary then candidates[#candidates + 1] = field end
844
- end
845
- local status = read_fields(candidates, { 'p', 'f' })
846
- local dead = {}
847
- for _, field in ipairs(candidates) do
848
- if status[field][2] and not status[field][1] then dead[#dead + 1] = field end
1184
+ const WAKE_LOG_LUA = `${BASE_LUA}${CURSOR_LUA}
1185
+ if not read_bounds() then return redis.error_reply('${LAYOUT_ERROR}') end
1186
+ local consumer, lane, ty = ARGV[1], ARGV[2], ARGV[3]
1187
+ local types = types_of(ty)
1188
+ local cid = cid_of(consumer, 'G' .. lane)
1189
+ load_states({ cid })
1190
+ local state = states[cid]
1191
+ local had_head = state.h
1192
+ -- Nothing after the latest clear: a missing cursor stays missing.
1193
+ if not state.exists and total == boundary then
1194
+ return { 'ok', tostring(boundary), false, {}, tostring(total) }
1195
+ end
1196
+ if not state.exists then
1197
+ state.exists, state.p = true, total
1198
+ if ARGV[4] == '1' then
1199
+ local head, missing = stored_head(boundary, types)
1200
+ if head == nil then return { 'index', missing, tostring(boundary) } end
1201
+ state.p, state.h = boundary, head
1202
+ else
1203
+ -- Nothing it takes is stored after its start: index its types from there.
1204
+ for name in pairs(types or {}) do
1205
+ if typed[name] == nil then register(name, total) end
1206
+ end
1207
+ end
1208
+ state.dirty = true
1209
+ elseif not state.h then
1210
+ -- A deployment that doesn't serve this consumer may have appended events
1211
+ -- it takes since it went idle.
1212
+ local head, missing = stored_head(state.p, types)
1213
+ if head == nil then return { 'index', missing, tostring(state.p) } end
1214
+ if head then state.h, state.dirty = head, true end
1215
+ end
1216
+ if state.ty ~= ty then state.ty, state.dirty = ty, true end
1217
+ local columns = {}
1218
+ if had_head then
1219
+ columns = read_rows({ { state.h, row_of(consumer, 'G' .. lane) } }, ${HEAD_COLUMNS})[1]
1220
+ elseif state.h then
1221
+ ready_set(cid, state.h)
1222
+ end
1223
+ flush_states()
1224
+ if typed_dirty then redis.call('SET', TYPED, cjson.encode(typed)) end
1225
+ flush()
1226
+ flush_ready()
1227
+ return { 'ok', tostring(state.p), state.h and tostring(state.h) or false, columns, tostring(total) }
1228
+ `
1229
+
1230
+ /**
1231
+ * KEYS: counter, layout. Returns the dead-lettered heads above the clear
1232
+ * marker as [consumer, tag, index, columns, entry] tuples.
1233
+ */
1234
+ const DEAD_LETTERS_LUA = `${BASE_LUA}
1235
+ local dead = redis.call('ZRANGEBYSCORE', READY, '-inf', '(0', 'WITHSCORES')
1236
+ if #dead == 0 then return {} end
1237
+ if not read_bounds() then return redis.error_reply('${LAYOUT_ERROR}') end
1238
+ local refs, items = {}, {}
1239
+ for i = 1, #dead, 2 do
1240
+ local consumer, tag = parse_cid(dead[i])
1241
+ local idx = -tonumber(dead[i + 1])
1242
+ if idx > boundary then
1243
+ refs[#refs + 1] = { idx, row_of(consumer, tag) }
1244
+ items[#items + 1] = { consumer, tag, idx }
1245
+ end
849
1246
  end
850
- local rows = read_fields(dead, { ${STORED_EVENT_META_SUFFIXES.map((suffix) => `'${suffix}'`).join(', ')} })
1247
+ local rows = read_rows(refs, ${HEAD_COLUMNS})
851
1248
  local result = {}
852
- for _, field in ipairs(dead) do
853
- local entries = redis.call('XRANGE', KEYS[1], field .. '-0', field .. '-0')
1249
+ for k, item in ipairs(items) do
1250
+ local entries = redis.call('XRANGE', LOG, tostring(item[3]) .. '-0', tostring(item[3]) .. '-0')
854
1251
  if #entries > 0 then
855
- result[#result + 1] = entries[1]
856
- result[#result + 1] = rows[field]
1252
+ result[#result + 1] = { item[1], item[2], tostring(item[3]), rows[k], entries[1] }
1253
+ end
1254
+ end
1255
+ return result
1256
+ `
1257
+
1258
+ /**
1259
+ * KEYS: counter, layout. Returns every listed cursor as [consumer, tag,
1260
+ * position, head or nil, columns]: lane and log cursors from the cursor
1261
+ * hash, read against the clear marker, then the event cursors with a head.
1262
+ */
1263
+ const CURSORS_LUA = `${BASE_LUA}
1264
+ if not read_bounds() then return redis.error_reply('${LAYOUT_ERROR}') end
1265
+ local listed = {}
1266
+ local fields = redis.call('HGETALL', CURSORS)
1267
+ local records, order = {}, {}
1268
+ for i = 1, #fields, 2 do
1269
+ local at = string.find(fields[i], ':[^:]*$')
1270
+ local cid, suffix = string.sub(fields[i], 1, at - 1), string.sub(fields[i], at + 1)
1271
+ if not records[cid] then
1272
+ records[cid] = {}
1273
+ order[#order + 1] = cid
1274
+ end
1275
+ records[cid][suffix] = fields[i + 1]
1276
+ end
1277
+ for _, cid in ipairs(order) do
1278
+ local record = records[cid]
1279
+ local consumer, tag = parse_cid(cid)
1280
+ local head = (record.h and record.h ~= '') and tonumber(record.h) or false
1281
+ local position = tonumber(record.p or '0')
1282
+ if head and head <= boundary then head = false end
1283
+ local kind = string.sub(tag, 1, 1)
1284
+ if kind == 'G' and position < boundary then position = boundary end
1285
+ if kind == 'G' or head or position > boundary then
1286
+ listed[#listed + 1] = { consumer, tag, position, head }
1287
+ end
1288
+ end
1289
+ local ready = redis.call('ZRANGE', READY, 0, -1, 'WITHSCORES')
1290
+ for i = 1, #ready, 2 do
1291
+ local consumer, tag = parse_cid(ready[i])
1292
+ if string.sub(tag, 1, 1) == '#' then
1293
+ local head = math.abs(tonumber(ready[i + 1]))
1294
+ listed[#listed + 1] = { consumer, '#', head - 1, head }
857
1295
  end
858
1296
  end
1297
+ local refs, positions = {}, {}
1298
+ for position, cursor in ipairs(listed) do
1299
+ if cursor[4] then
1300
+ refs[#refs + 1] = { cursor[4], row_of(cursor[1], cursor[2]) }
1301
+ positions[#refs] = position
1302
+ end
1303
+ end
1304
+ local rows = read_rows(refs, ${HEAD_COLUMNS})
1305
+ local columns = {}
1306
+ for k, position in ipairs(positions) do columns[position] = rows[k] end
1307
+ local result = {}
1308
+ for position, cursor in ipairs(listed) do
1309
+ result[#result + 1] = {
1310
+ cursor[1], cursor[2], tostring(cursor[3]),
1311
+ cursor[4] and tostring(cursor[4]) or false, columns[position] or {},
1312
+ }
1313
+ end
859
1314
  return result
860
1315
  `
861
1316
 
862
1317
  /**
863
- * KEYS: head, history, event counter, event meta, pin counts, metadata split.
864
- * ARGV: nowMs, event pin key prefix, write count, then per write:
865
- * index, snapshot JSON, pin count, pin event indexes; finally the reducer
866
- * name. Writes from before the latest clear are dropped, and a head from
867
- * before it is replaced as if absent.
1318
+ * KEYS: counter, layout, head, history.
1319
+ * ARGV: write count, then per write: index, snapshot JSON, pin count, then
1320
+ * per pin: consumer, tag, index; finally the reducer name. Writes from
1321
+ * before the latest clear are dropped, and a head from before it is
1322
+ * replaced as if absent. A pin holds only on an unfinished, live head: a
1323
+ * claimed row says so itself, an unclaimed head by its cursor's ready score.
868
1324
  */
869
- const PUT_SNAPSHOTS_LUA = `
870
- local COUNTER, META, SPLIT = KEYS[3], KEYS[4], KEYS[6]
871
- ${META_LUA}
872
- local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR)
873
- local total = tonumber(counts[1] or '0')
874
- use_split(counts[2], counts[1], counts[3])
875
- local pinned = {}
876
- local offset = 4
877
- for i = 1, tonumber(ARGV[3]) do
1325
+ const PUT_SNAPSHOTS_LUA = `${BASE_LUA}
1326
+ if not read_bounds() then return redis.error_reply('${LAYOUT_ERROR}') end
1327
+ local HEAD, HISTORY = KEYS[3], KEYS[4]
1328
+ local entries, refs, pins = {}, {}, {}
1329
+ local offset = 2
1330
+ for w = 1, tonumber(ARGV[1]) do
1331
+ local index = tonumber(ARGV[offset])
878
1332
  local pin_count = tonumber(ARGV[offset + 2])
1333
+ local entry = { index = index, snapshot = ARGV[offset + 1], pins = {} }
879
1334
  for p = 1, pin_count do
880
- local event_index = tonumber(ARGV[offset + 2 + p])
881
- if event_index > boundary and event_index <= total then
882
- pinned[#pinned + 1] = tostring(event_index)
1335
+ local o = offset + 2 + (p - 1) * 3
1336
+ local pin = { consumer = ARGV[o + 1], tag = ARGV[o + 2], idx = tonumber(ARGV[o + 3]) }
1337
+ if index >= boundary and pin.idx > boundary and pin.idx <= total then
1338
+ pin.row = row_of(pin.consumer, pin.tag)
1339
+ refs[#refs + 1] = { pin.idx, pin.row }
1340
+ pins[#pins + 1] = pin
1341
+ entry.pins[#entry.pins + 1] = pin
1342
+ end
1343
+ end
1344
+ entries[w] = entry
1345
+ offset = offset + 3 + pin_count * 3
1346
+ end
1347
+ local function live(consumer, tag, idx)
1348
+ return tonumber(redis.call('ZSCORE', READY, cid_of(consumer, tag)) or '0') == idx
1349
+ end
1350
+ local rows = read_rows(refs, { 'pins', 'p', 'f', 'l' })
1351
+ local held, held_order = {}, {}
1352
+ for k, pin in ipairs(pins) do
1353
+ local row = rows[k]
1354
+ local valid
1355
+ if row[2] or row[3] then
1356
+ valid = false
1357
+ elseif row[4] then
1358
+ valid = owns(row[4], pin.tag)
1359
+ elseif pin.tag == '#' then
1360
+ valid = live(pin.consumer, '#' .. tostring(pin.idx), pin.idx)
1361
+ else
1362
+ valid = live(pin.consumer, pin.tag, pin.idx) or
1363
+ live(pin.consumer, 'G' .. string.sub(pin.tag, 2), pin.idx)
1364
+ end
1365
+ if valid then
1366
+ local key = tostring(pin.idx) .. ':' .. pin.row
1367
+ if not held[key] then
1368
+ local list = (row[1] and row[1] ~= '') and cjson.decode(row[1]) or {}
1369
+ local set = {}
1370
+ for _, ref in ipairs(list) do set[ref] = true end
1371
+ held[key] = { list = list, set = set, idx = pin.idx, row = pin.row }
1372
+ held_order[#held_order + 1] = key
883
1373
  end
1374
+ pin.held = held[key]
884
1375
  end
885
- offset = offset + 3 + pin_count
886
1376
  end
887
- local status = read_fields(pinned, { 'p', 'f' })
888
1377
  local snapshots = {}
889
- offset = 4
890
- for i = 1, tonumber(ARGV[3]) do
891
- local index = tonumber(ARGV[offset])
892
- local snapshot = ARGV[offset + 1]
893
- local pin_count = tonumber(ARGV[offset + 2])
894
- local ref = cjson.encode({ KEYS[2], tostring(index) })
895
- for p = 1, pin_count do
896
- local event_index = tonumber(ARGV[offset + 2 + p])
897
- local row = status[tostring(event_index)]
898
- if index >= boundary and event_index > boundary and event_index <= total and
899
- not row[1] and not row[2] then
900
- local pin_key = ARGV[2] .. tostring(event_index)
901
- if redis.call('SADD', pin_key, ref) == 1 then
902
- redis.call('HINCRBY', KEYS[5], ref, 1)
903
- end
1378
+ for _, entry in ipairs(entries) do
1379
+ local ref = cjson.encode({ HISTORY, tostring(entry.index) })
1380
+ for _, pin in ipairs(entry.pins) do
1381
+ if pin.held and not pin.held.set[ref] then
1382
+ pin.held.set[ref] = true
1383
+ pin.held.list[#pin.held.list + 1] = ref
1384
+ pin.held.dirty = true
1385
+ redis.call('HINCRBY', PIN_COUNTS, ref, 1)
904
1386
  end
905
1387
  end
906
- if index >= boundary then
907
- snapshots[#snapshots + 1] = { index = index, snapshot = snapshot, ref = ref }
1388
+ if entry.index >= boundary then
1389
+ snapshots[#snapshots + 1] = { index = entry.index, snapshot = entry.snapshot, ref = ref }
908
1390
  end
909
- offset = offset + 3 + pin_count
910
1391
  end
1392
+ for _, key in ipairs(held_order) do
1393
+ local row = held[key]
1394
+ if row.dirty then put_row(row.idx, row.row, 'pins', cjson.encode(row.list)) end
1395
+ end
1396
+ flush()
911
1397
  if #snapshots == 0 then return 1 end
912
1398
 
913
- local head = redis.call('GET', KEYS[1])
1399
+ local head = redis.call('GET', HEAD)
914
1400
  local head_index = nil
915
1401
  local registered = false
916
1402
  if head then
@@ -920,35 +1406,35 @@ if head then
920
1406
  end
921
1407
  if head_index and head_index < boundary then
922
1408
  head, head_index, registered = nil, nil, false
923
- redis.call('ZREMRANGEBYSCORE', KEYS[2], '-inf', '(' .. boundary)
1409
+ redis.call('ZREMRANGEBYSCORE', HISTORY, '-inf', '(' .. boundary)
924
1410
  end
925
1411
  -- Deletion finds each reducer's keys through this set instead of a scan.
926
1412
  -- Heads written since carry the mark, so a reducer registers once.
927
1413
  if not registered then
928
- redis.call('SADD', string.sub(COUNTER, 1, -6) .. 'snap-reducers', ARGV[#ARGV])
1414
+ redis.call('SADD', SESSION .. 'snap-reducers', ARGV[#ARGV])
929
1415
  end
930
1416
  local latest = snapshots[#snapshots]
931
1417
  local writes_head = not head_index or latest.index >= head_index
932
1418
  local advances = not head_index or latest.index > head_index
933
1419
  local final_head_index = advances and latest.index or head_index
934
1420
 
935
- if advances and head and tonumber(redis.call('HGET', KEYS[5], cjson.encode({ KEYS[2], tostring(head_index) })) or '0') > 0 then
936
- redis.call('ZREMRANGEBYSCORE', KEYS[2], head_index, head_index)
937
- redis.call('ZADD', KEYS[2], head_index, head)
1421
+ if advances and head and tonumber(redis.call('HGET', PIN_COUNTS, cjson.encode({ HISTORY, tostring(head_index) })) or '0') > 0 then
1422
+ redis.call('ZREMRANGEBYSCORE', HISTORY, head_index, head_index)
1423
+ redis.call('ZADD', HISTORY, head_index, head)
938
1424
  end
939
1425
 
940
1426
  for _, write in ipairs(snapshots) do
941
1427
  if write.index < final_head_index then
942
- redis.call('ZREMRANGEBYSCORE', KEYS[2], write.index, write.index)
943
- if tonumber(redis.call('HGET', KEYS[5], write.ref) or '0') > 0 then
944
- redis.call('ZADD', KEYS[2], write.index, write.snapshot)
1428
+ redis.call('ZREMRANGEBYSCORE', HISTORY, write.index, write.index)
1429
+ if tonumber(redis.call('HGET', PIN_COUNTS, write.ref) or '0') > 0 then
1430
+ redis.call('ZADD', HISTORY, write.index, write.snapshot)
945
1431
  end
946
1432
  end
947
1433
  end
948
1434
 
949
1435
  if writes_head then
950
- redis.call('SET', KEYS[1], latest.snapshot)
951
- redis.call('ZREMRANGEBYSCORE', KEYS[2], latest.index, latest.index)
1436
+ redis.call('SET', HEAD, latest.snapshot)
1437
+ redis.call('ZREMRANGEBYSCORE', HISTORY, latest.index, latest.index)
952
1438
  end
953
1439
  return 1
954
1440
  `
@@ -1015,14 +1501,18 @@ return live
1015
1501
  `
1016
1502
 
1017
1503
  /**
1018
- * KEYS: head, history, event counter. ARGV: snapshot frontier or ''.
1504
+ * KEYS: counter, layout, head, history. ARGV: snapshot frontier or ''.
1019
1505
  * Returns one atomic multiversion-cache-plus-log-frontier read, with the
1020
1506
  * clear boundary; checkpoints from before it are ignored.
1021
1507
  */
1022
1508
  const READ_STATE_LUA = `
1023
- local counts = redis.call('MGET', KEYS[3], string.sub(KEYS[3], 1, -6) .. 'clear')
1509
+ local session = string.sub(KEYS[1], 1, -6)
1510
+ local counts = redis.call('MGET', KEYS[1], session .. 'clear', KEYS[2])
1511
+ if counts[3] and counts[3] ~= '${REDIS_LAYOUT}' then
1512
+ return redis.error_reply('${LAYOUT_ERROR}')
1513
+ end
1024
1514
  local boundary = tonumber(counts[2] or '0')
1025
- local head = redis.call('GET', KEYS[1])
1515
+ local head = redis.call('GET', KEYS[3])
1026
1516
  local head_index = head and cjson.decode(head)['index'] or nil
1027
1517
  if head and (type(head_index) ~= 'number' or head_index < 0 or head_index > 9007199254740991 or head_index ~= math.floor(head_index)) then
1028
1518
  return redis.error_reply('invalid snapshot head index')
@@ -1037,63 +1527,109 @@ if head_index and (cap == '' or head_index <= tonumber(cap)) then
1037
1527
  snapshot = head
1038
1528
  else
1039
1529
  local upper = cap == '' and '+inf' or cap
1040
- local historical = redis.call('ZREVRANGEBYSCORE', KEYS[2], upper, boundary, 'LIMIT', 0, 1)
1530
+ local historical = redis.call('ZREVRANGEBYSCORE', KEYS[4], upper, boundary, 'LIMIT', 0, 1)
1041
1531
  if #historical > 0 then snapshot = historical[1] end
1042
1532
  end
1043
1533
  return { head_index or '', snapshot or '', counts[1] or '0', tostring(boundary) }
1044
1534
  `
1045
1535
 
1046
1536
  /**
1047
- * Removes every registered reducer's checkpoints below the latest clear, and
1048
- * the pin counts that name them. Snapshot writes register each reducer in
1049
- * {prefix}:{session}:snap-reducers, so no key scan is needed.
1537
+ * What reclaim removes once a session has no moved log left: every
1538
+ * registered reducer's checkpoints below the latest clear and the pin counts
1539
+ * naming them (snapshot writes register each reducer in
1540
+ * {prefix}:{session}:snap-reducers, so no key scan is needed), every head
1541
+ * row below it (whole chunks, then the boundary chunk's older rows), the
1542
+ * type indexes' older entries, and the lane cursors it left idle.
1050
1543
  */
1051
- const SWEEP_CHECKPOINTS_LUA = `
1052
- local function sweep_checkpoints(session, boundary)
1053
- for _, reducer in ipairs(redis.call('SMEMBERS', session .. 'snap-reducers')) do
1054
- local head_key = session .. 'snap:' .. reducer
1544
+ const SWEEP_LUA = `
1545
+ local function sweep(boundary)
1546
+ for _, reducer in ipairs(redis.call('SMEMBERS', SESSION .. 'snap-reducers')) do
1547
+ local head_key = SESSION .. 'snap:' .. reducer
1055
1548
  local head = redis.call('GET', head_key)
1056
1549
  if head and tonumber(cjson.decode(head)['index']) < boundary then
1057
1550
  redis.call('DEL', head_key)
1058
1551
  end
1059
- redis.call('ZREMRANGEBYSCORE', session .. 'snap-history:' .. reducer, '-inf', '(' .. boundary)
1552
+ redis.call('ZREMRANGEBYSCORE', SESSION .. 'snap-history:' .. reducer, '-inf', '(' .. boundary)
1060
1553
  end
1061
- local counts_key = session .. 'snapshot-pin-counts'
1062
- local counts = redis.call('HGETALL', counts_key)
1554
+ local counts = redis.call('HGETALL', PIN_COUNTS)
1063
1555
  local stale = {}
1064
1556
  for i = 1, #counts, 2 do
1065
1557
  if tonumber(cjson.decode(counts[i])[2]) < boundary then stale[#stale + 1] = counts[i] end
1066
1558
  end
1067
- if #stale > 0 then redis.call('HDEL', counts_key, unpack(stale)) end
1559
+ sliced('HDEL', PIN_COUNTS, stale)
1560
+ local raw = redis.call('GET', TYPED)
1561
+ if raw then
1562
+ for name in pairs(cjson.decode(raw)) do
1563
+ redis.call('ZREMRANGEBYSCORE', type_key(name), '-inf', '(' .. boundary)
1564
+ end
1565
+ end
1566
+ local last = math.floor((boundary - 1) / ${META_CHUNK_SIZE})
1567
+ local chunks = {}
1568
+ for chunk = 0, last - 1 do chunks[#chunks + 1] = META .. ':' .. tostring(chunk) end
1569
+ sliced('UNLINK', nil, chunks)
1570
+ local older = {}
1571
+ for _, field in ipairs(redis.call('HKEYS', META .. ':' .. tostring(last))) do
1572
+ local idx = tonumber(string.match(field, '^(%d+):'))
1573
+ if idx and idx < boundary then older[#older + 1] = field end
1574
+ end
1575
+ sliced('HDEL', META .. ':' .. tostring(last), older)
1576
+ local fields = redis.call('HGETALL', CURSORS)
1577
+ local records, order = {}, {}
1578
+ for i = 1, #fields, 2 do
1579
+ local at = string.find(fields[i], ':[^:]*$')
1580
+ local cid, suffix = string.sub(fields[i], 1, at - 1), string.sub(fields[i], at + 1)
1581
+ if not records[cid] then
1582
+ records[cid] = {}
1583
+ order[#order + 1] = cid
1584
+ end
1585
+ records[cid][suffix] = fields[i + 1]
1586
+ end
1587
+ local idle, moved = {}, {}
1588
+ for _, cid in ipairs(order) do
1589
+ local record = records[cid]
1590
+ local _, tag = parse_cid(cid)
1591
+ local head = (record.h and record.h ~= '') and tonumber(record.h) or false
1592
+ local position = tonumber(record.p or '0')
1593
+ if head and head <= boundary then head = false end
1594
+ if string.sub(tag, 1, 1) == 'G' then
1595
+ if position < boundary then position = boundary end
1596
+ moved[#moved + 1] = cid .. ':h'
1597
+ moved[#moved + 1] = head and tostring(head) or ''
1598
+ moved[#moved + 1] = cid .. ':p'
1599
+ moved[#moved + 1] = tostring(position)
1600
+ elseif not head and position <= boundary then
1601
+ for suffix in pairs(record) do idle[#idle + 1] = cid .. ':' .. suffix end
1602
+ elseif not head and record.h ~= '' then
1603
+ moved[#moved + 1] = cid .. ':h'
1604
+ moved[#moved + 1] = ''
1605
+ end
1606
+ end
1607
+ sliced('HDEL', CURSORS, idle)
1608
+ sliced('HSET', CURSORS, moved)
1068
1609
  end
1069
1610
  `
1070
1611
 
1071
1612
  /**
1072
- * KEYS: store stream, counter, metadata, metadata split, marker id ownership
1073
- * key, pending, ready.
1074
- * ARGV: sessionId, nowMs, ifLastIndex or '', cause index or '', cause
1075
- * attempt, marker id, notify channel.
1076
- * Returns ['ok', cleared, head, boundary] | ['no_parent', idx] |
1077
- * ['superseded', idx] | ['foreign']. The old lifetime's log moves aside in one
1078
- * RENAME, so every read of the log starts at the marker, and its queues are
1079
- * unlinked, so claims and pending checks see only the new lifetime; deletion
1080
- * drains the moved log.
1613
+ * KEYS: counter, layout, marker id ownership key.
1614
+ * ARGV: sessionId, nowMs, ifLastIndex or '', cause consumer or '', cause tag,
1615
+ * cause index, cause attempt, marker id, notify channel.
1616
+ * Returns ['ok', cleared, head, boundary] | ['superseded'] | ['foreign'].
1617
+ * The old lifetime's log moves aside in one RENAME, so every read of the log
1618
+ * starts at the marker, and its ready set is unlinked, so claims and pending
1619
+ * checks see only the new lifetime; deletion drains the moved log. Type
1620
+ * indexes keep their older entries until then: lookups start above the
1621
+ * marker.
1081
1622
  */
1082
- const CLEAR_LUA = `
1083
- local COUNTER, META, SPLIT = KEYS[2], KEYS[3], KEYS[4]
1084
- ${META_LUA}
1085
- ${WAKE_LUA}
1086
- local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR, KEYS[5])
1087
- local head = tonumber(counts[1] or '0')
1088
- use_split(counts[2], counts[1], counts[3])
1623
+ const CLEAR_LUA = `${BASE_LUA}${WAKE_LUA}
1624
+ local values = read_bounds({ KEYS[3] })
1625
+ if not values then return redis.error_reply('${LAYOUT_ERROR}') end
1626
+ local head = total
1089
1627
  if ARGV[4] ~= '' then
1090
- local field = ARGV[4]
1091
- local pidx = tonumber(field)
1092
- if pidx <= boundary then return { 'superseded', field } end
1093
- if pidx < 1 or pidx > head then return { 'no_parent', field } end
1094
- local row = read_fields({ field }, { 'd', 'f' })[field]
1095
- if tonumber(row[1] or '0') ~= tonumber(ARGV[5]) or row[2] then
1096
- return { 'superseded', field }
1628
+ local idx = tonumber(ARGV[6])
1629
+ if idx <= boundary then return { 'superseded' } end
1630
+ local row = read_rows({ { idx, row_of(ARGV[4], ARGV[5]) } }, { 'd', 'f', 'p', 'l' })[1]
1631
+ if tonumber(row[1] or '0') ~= tonumber(ARGV[7]) or row[2] or row[3] or not owns(row[4], ARGV[5]) then
1632
+ return { 'superseded' }
1097
1633
  end
1098
1634
  end
1099
1635
  local expected = ARGV[3]
@@ -1106,39 +1642,37 @@ end
1106
1642
  if head == 0 or boundary == head then
1107
1643
  return { 'ok', '1', tostring(head), tostring(boundary) }
1108
1644
  end
1109
- if counts[4] then return { 'foreign' } end
1645
+ if values[1] then return { 'foreign' } end
1110
1646
  local field = tostring(head + 1)
1111
- local session = string.sub(COUNTER, 1, -6)
1112
- local moved = session .. 'cleared:' .. field
1113
- redis.call('RENAME', KEYS[1], moved)
1114
- redis.call('UNLINK', KEYS[6], KEYS[7])
1115
- redis.call('SADD', session .. 'cleared', moved)
1116
- redis.call('XADD', KEYS[1], field .. '-0',
1117
- 'id', ARGV[6], 'type', '${CLEAR_EVENT_TYPE}', 'payload', 'null',
1118
- 'created_at', ARGV[2], 'cause', '', 'lane', '')
1119
- local writes = { KEYS[5], cjson.encode({ ARGV[1], head + 1 }), COUNTER, field, CLEAR, field }
1120
- split_write(writes)
1121
- redis.call('MSET', unpack(writes))
1122
- put(mkey(field), field .. ':p', ARGV[2])
1123
- flush()
1124
- wake_add(field, ARGV[6], '${CLEAR_EVENT_TYPE}', 'null')
1125
- wake(ARGV[7], field, ARGV[2])
1647
+ local moved = SESSION .. 'cleared:' .. field
1648
+ redis.call('RENAME', LOG, moved)
1649
+ redis.call('UNLINK', READY)
1650
+ redis.call('SADD', SESSION .. 'cleared', moved)
1651
+ redis.call('XADD', LOG, field .. '-0',
1652
+ 'id', ARGV[8], 'type', CLEAR_TYPE, 'payload', 'null',
1653
+ 'created_at', ARGV[2], 'cause', '')
1654
+ local marker = { KEYS[3], cjson.encode({ ARGV[1], head + 1 }), COUNTER, field, CLEAR, field }
1655
+ if schema_missing then
1656
+ marker[#marker + 1] = SCHEMA
1657
+ marker[#marker + 1] = '${REDIS_LAYOUT}'
1658
+ end
1659
+ redis.call('MSET', unpack(marker))
1660
+ wake_add(field, ARGV[8], CLEAR_TYPE, 'null')
1661
+ wake(ARGV[9], field, ARGV[2])
1126
1662
  return { 'ok', '1', field, field }
1127
1663
  `
1128
1664
 
1129
1665
  /**
1130
- * KEYS: counter. ARGV: page size.
1666
+ * KEYS: counter, layout. ARGV: page size.
1131
1667
  * The next deletion batch: a moved log and up to a page of its [index, id]
1132
1668
  * pairs, without payloads. Empty moved logs are dropped; with none left it
1133
- * sweeps the old checkpoints and returns nothing, so a retry finishes an
1134
- * interrupted deletion.
1669
+ * sweeps what the old lifetime left and returns nothing, so a retry
1670
+ * finishes an interrupted deletion.
1135
1671
  */
1136
- const RECLAIM_BATCH_LUA = `
1137
- ${SWEEP_CHECKPOINTS_LUA}
1138
- local session = string.sub(KEYS[1], 1, -6)
1139
- local boundary = tonumber(redis.call('GET', session .. 'clear') or '0')
1672
+ const RECLAIM_BATCH_LUA = `${BASE_LUA}${SWEEP_LUA}
1673
+ local boundary = tonumber(redis.call('GET', CLEAR) or '0')
1140
1674
  if boundary == 0 then return {} end
1141
- for _, log in ipairs(redis.call('SMEMBERS', session .. 'cleared')) do
1675
+ for _, log in ipairs(redis.call('SMEMBERS', SESSION .. 'cleared')) do
1142
1676
  local entries = redis.call('XRANGE', log, '-', '+', 'COUNT', ARGV[1])
1143
1677
  if #entries > 0 then
1144
1678
  local batch = { log }
@@ -1155,78 +1689,36 @@ for _, log in ipairs(redis.call('SMEMBERS', session .. 'cleared')) do
1155
1689
  return batch
1156
1690
  end
1157
1691
  redis.call('DEL', log)
1158
- redis.call('SREM', session .. 'cleared', log)
1692
+ redis.call('SREM', SESSION .. 'cleared', log)
1159
1693
  end
1160
- sweep_checkpoints(session, boundary)
1694
+ sweep(boundary)
1161
1695
  return {}
1162
1696
  `
1163
1697
 
1164
1698
  /**
1165
- * KEYS: moved log, counter, metadata, metadata split, then one id ownership
1166
- * key per entry.
1167
- * ARGV: sessionId, event pin key prefix, n, then per entry: index, id.
1168
- * Deletes the listed entries below the latest clear and everything that
1169
- * names them. An id record survives when it names another event, so an id
1170
- * reused in the new lifetime keeps its owner. Returns [deleted, done].
1699
+ * KEYS: counter, layout, moved log, then one id ownership key per entry.
1700
+ * ARGV: sessionId, n, then per entry: index, id.
1701
+ * Deletes the listed entries below the latest clear and their id records.
1702
+ * An id record survives when it names another event, so an id reused in the
1703
+ * new lifetime keeps its owner. Returns [deleted, done].
1171
1704
  */
1172
- const RECLAIM_LUA = `
1173
- local COUNTER, META, SPLIT = KEYS[2], KEYS[3], KEYS[4]
1174
- ${META_LUA}
1175
- ${SWEEP_CHECKPOINTS_LUA}
1176
- local function lane_field(lane, suffix)
1177
- return '__lane:' .. tostring(string.len(lane)) .. ':' .. lane .. ':' .. suffix
1178
- end
1179
- local session = string.sub(COUNTER, 1, -6)
1180
- local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR)
1181
- use_split(counts[2], counts[1], counts[3])
1182
- local fields, id_keys, indexes, stream_ids, pin_keys, deleted = {}, {}, {}, {}, {}, {}
1705
+ const RECLAIM_LUA = `${BASE_LUA}${SWEEP_LUA}
1706
+ local boundary = tonumber(redis.call('GET', CLEAR) or '0')
1707
+ local MOVED = KEYS[3]
1708
+ local id_keys, indexes, stream_ids = {}, {}, {}
1183
1709
  -- A concurrent reclaim may list the same batch; only XDEL says who removed it.
1184
1710
  local removed = 0
1185
- for i = 1, tonumber(ARGV[3]) do
1186
- local field = ARGV[3 + (i - 1) * 2 + 1]
1711
+ for i = 1, tonumber(ARGV[2]) do
1712
+ local field = ARGV[2 + (i - 1) * 2 + 1]
1187
1713
  if tonumber(field) < boundary then
1188
- fields[#fields + 1] = field
1189
- id_keys[#fields] = KEYS[4 + i]
1190
- indexes[#fields] = tonumber(field)
1191
- stream_ids[#fields] = field .. '-0'
1192
- pin_keys[#fields] = ARGV[2] .. field
1193
- deleted[field] = true
1194
- end
1195
- end
1196
- if #fields > 0 then
1197
- local lanes, seen, pointers = read_fields(fields, { 'l' }), {}, {}
1198
- for _, field in ipairs(fields) do
1199
- local lane = lanes[field][1]
1200
- if lane and lane ~= '' and not seen[lane] then
1201
- seen[lane] = true
1202
- pointers[#pointers + 1] = lane_field(lane, 'head')
1203
- pointers[#pointers + 1] = lane_field(lane, 'tail')
1204
- end
1714
+ id_keys[#id_keys + 1] = KEYS[3 + i]
1715
+ indexes[#id_keys] = tonumber(field)
1716
+ stream_ids[#id_keys] = field .. '-0'
1205
1717
  end
1206
- if #pointers > 0 then
1207
- local values = redis.call('HMGET', META, unpack(pointers))
1208
- local stale = {}
1209
- for i = 1, #pointers do
1210
- if values[i] and deleted[values[i]] then stale[#stale + 1] = pointers[i] end
1211
- end
1212
- if #stale > 0 then redis.call('HDEL', META, unpack(stale)) end
1213
- end
1214
- local suffixes = { 'l', 'next', 'p', 'pa', 'ri', 'ch', 'ce', 'f', 'fc', 'lc', 'lf', 'lfa', 'd', 'a', 'e' }
1215
- local columns, order = {}, {}
1216
- for _, field in ipairs(fields) do
1217
- local key = mkey(field)
1218
- if not columns[key] then
1219
- columns[key] = {}
1220
- order[#order + 1] = key
1221
- end
1222
- for _, suffix in ipairs(suffixes) do
1223
- columns[key][#columns[key] + 1] = field .. ':' .. suffix
1224
- end
1225
- end
1226
- for _, key in ipairs(order) do sliced('HDEL', key, columns[key]) end
1227
- sliced('DEL', nil, pin_keys)
1718
+ end
1719
+ if #id_keys > 0 then
1228
1720
  local owners, stale_ids = sliced('MGET', nil, id_keys), {}
1229
- for i = 1, #fields do
1721
+ for i = 1, #id_keys do
1230
1722
  if owners[i] then
1231
1723
  local rec = cjson.decode(owners[i])
1232
1724
  if rec[1] == ARGV[1] and tonumber(rec[2]) == indexes[i] then
@@ -1234,15 +1726,15 @@ if #fields > 0 then
1234
1726
  end
1235
1727
  end
1236
1728
  end
1237
- if #stale_ids > 0 then sliced('DEL', nil, stale_ids) end
1238
- removed = redis.call('XDEL', KEYS[1], unpack(stream_ids))
1729
+ sliced('DEL', nil, stale_ids)
1730
+ removed = redis.call('XDEL', MOVED, unpack(stream_ids))
1239
1731
  end
1240
1732
  local done = 0
1241
- if redis.call('XLEN', KEYS[1]) == 0 then
1242
- redis.call('DEL', KEYS[1])
1243
- redis.call('SREM', session .. 'cleared', KEYS[1])
1244
- if redis.call('SCARD', session .. 'cleared') == 0 then
1245
- sweep_checkpoints(session, boundary)
1733
+ if redis.call('XLEN', MOVED) == 0 then
1734
+ redis.call('DEL', MOVED)
1735
+ redis.call('SREM', SESSION .. 'cleared', MOVED)
1736
+ if redis.call('SCARD', SESSION .. 'cleared') == 0 then
1737
+ sweep(boundary)
1246
1738
  done = 1
1247
1739
  end
1248
1740
  end
@@ -1276,6 +1768,30 @@ const toEvent = (sessionId: string, entry: StreamEntry): Event => {
1276
1768
  }
1277
1769
  }
1278
1770
 
1771
+ /**
1772
+ * A stream entry's causal edge. Releases before consumers (0.17 to 0.20)
1773
+ * recorded only the parent's index and attempt: their edge reads as the
1774
+ * handlers table's, `a2.handlers`, as Postgres's upgrade names it, with no
1775
+ * lane, since nothing reads an earlier release's lane.
1776
+ */
1777
+ const decodeStreamCause = (raw: string): EventCause | null => {
1778
+ const parsed = JSON.parse(raw) as { consumer?: unknown }
1779
+ return decodeCause(
1780
+ parsed.consumer === undefined
1781
+ ? { consumer: 'a2.handlers', lane: null, ...parsed }
1782
+ : parsed,
1783
+ )
1784
+ }
1785
+
1786
+ /** A stream entry with its causal edge. */
1787
+ const toStoredEvent = (sessionId: string, entry: StreamEntry): StoredEvent => {
1788
+ const cause = fieldMap(entry[1])['cause']
1789
+ return Object.assign(toEvent(sessionId, entry), {
1790
+ cause:
1791
+ cause === undefined || cause === '' ? null : decodeStreamCause(cause),
1792
+ })
1793
+ }
1794
+
1279
1795
  type ByteStreamEntry = [Uint8Array, Uint8Array[]]
1280
1796
 
1281
1797
  const fieldBytes = (name: string): Uint8Array => new TextEncoder().encode(name)
@@ -1338,75 +1854,107 @@ const toSerializedEvent = (
1338
1854
 
1339
1855
  const json = (value: unknown): string => JSON.stringify(value) ?? 'null'
1340
1856
 
1341
- const toCause = (raw: string | undefined): EventCause | null => {
1342
- if (raw === '' || raw === undefined) return null
1343
- const value = JSON.parse(raw) as {
1344
- index?: unknown
1345
- attempt?: unknown
1346
- batchSize?: unknown
1857
+ /** How scripts name a cursor within its consumer: `#`, or `L` and its lane. */
1858
+ const tagOf = (lane: string | null): string =>
1859
+ lane === null ? '#' : `L${lane}`
1860
+
1861
+ /** The lane a cursor tag names: null for an event cursor. */
1862
+ const laneOf = (tag: string): string | null =>
1863
+ tag.startsWith('#') ? null : tag.slice(1)
1864
+
1865
+ const targetsArg = (targets: readonly EventTarget[] | undefined): string =>
1866
+ targets === undefined || targets.length === 0
1867
+ ? ''
1868
+ : JSON.stringify(
1869
+ targets.map((target) => [target.consumer, tagOf(target.lane)]),
1870
+ )
1871
+
1872
+ /** A log consumer's types, as scripts read them; '*' takes every type. */
1873
+ const logTypesArg = (types: readonly string[] | null): string =>
1874
+ types === null ? '*' : JSON.stringify(types)
1875
+
1876
+ /** The served consumers, as scripts read them; '' serves every consumer. */
1877
+ const servedArg = (consumers: readonly StoreConsumer[] | undefined): string =>
1878
+ consumers === undefined
1879
+ ? ''
1880
+ : JSON.stringify(
1881
+ consumers.map((consumer) =>
1882
+ consumer.log
1883
+ ? [
1884
+ consumer.name,
1885
+ consumer.log.lane,
1886
+ logTypesArg(consumer.log.types),
1887
+ consumer.log.backfill ? 1 : 0,
1888
+ ]
1889
+ : [consumer.name],
1890
+ ),
1891
+ )
1892
+
1893
+ type Column = string | number | null | undefined
1894
+
1895
+ /** A head's dispatch state from a row's HEAD_COLUMNS. */
1896
+ const dispatchOf = (
1897
+ consumer: string,
1898
+ lane: string | null,
1899
+ columns: readonly Column[] | null | undefined,
1900
+ ): HeadDispatch => {
1901
+ const column = (at: number): string | null => {
1902
+ const value = columns?.[at]
1903
+ return value === null || value === undefined ? null : String(value)
1347
1904
  }
1348
- if (
1349
- !Number.isInteger(value.index) ||
1350
- Number(value.index) < 1 ||
1351
- !Number.isInteger(value.attempt) ||
1352
- Number(value.attempt) < 1
1353
- ) {
1354
- throw new TypeError('stored event has an invalid cause')
1905
+ const date = (at: number): Date | null => {
1906
+ const value = column(at)
1907
+ return value === null ? null : new Date(Number(value))
1355
1908
  }
1356
- if (
1357
- value.batchSize !== undefined &&
1358
- (!Number.isInteger(value.batchSize) || Number(value.batchSize) < 1)
1359
- ) {
1360
- throw new TypeError('stored event has an invalid cause')
1909
+ const count = (at: number): number | null => {
1910
+ const value = column(at)
1911
+ return value === null ? null : Number(value)
1361
1912
  }
1362
1913
  return {
1363
- index: Number(value.index),
1364
- attempt: Number(value.attempt),
1365
- ...(value.batchSize === undefined
1366
- ? {}
1367
- : { batchSize: Number(value.batchSize) }),
1914
+ consumer,
1915
+ lane,
1916
+ firstClaimedAt: date(1),
1917
+ lastClaimedAt: date(2),
1918
+ attemptCount: count(0) ?? 0,
1919
+ claimHolder: column(3),
1920
+ claimExpiresAt: date(4),
1921
+ failureCount: count(5) ?? 0,
1922
+ lastFailedAt: date(6),
1923
+ lastFailedAttempt: count(7),
1924
+ lastError: column(8),
1925
+ failedAt: date(9),
1368
1926
  }
1369
1927
  }
1370
1928
 
1371
- type MetaSuffix = (typeof STORED_EVENT_META_SUFFIXES)[number]
1372
-
1373
- /** A stream entry joined with its metadata columns; absent columns are unset. */
1374
- const toStoredEvent = (
1375
- sessionId: string,
1929
+ type HeadTuple = [
1930
+ consumer: string,
1931
+ tag: string,
1932
+ index: string,
1933
+ columns: Column[],
1376
1934
  entry: StreamEntry,
1377
- column: (suffix: MetaSuffix) => string | undefined,
1378
- ): StoredEvent => {
1379
- const fields = fieldMap(entry[1])
1380
- const date = (suffix: MetaSuffix): Date | null => {
1381
- const value = column(suffix)
1382
- return value === undefined ? null : new Date(Number(value))
1383
- }
1384
- const count = (suffix: MetaSuffix): number | null => {
1385
- const value = column(suffix)
1386
- return value === undefined ? null : Number(value)
1387
- }
1388
- const returnedEventIds = column('ri')
1389
- return Object.assign(toEvent(sessionId, entry), {
1390
- cause: toCause(fields['cause']),
1391
- lane: fields['lane'] === '' ? null : fields['lane']!,
1392
- processedAt: date('p'),
1393
- processedByAttempt: count('pa'),
1394
- returnedEventIds:
1395
- returnedEventIds === undefined
1396
- ? null
1397
- : decodeReturnedEventIds(returnedEventIds),
1398
- firstClaimedAt: date('fc'),
1399
- lastClaimedAt: date('lc'),
1400
- attemptCount: count('d') ?? 0,
1401
- claimHolder: column('ch') ?? null,
1402
- claimExpiresAt: date('ce'),
1403
- failureCount: count('a') ?? 0,
1404
- lastFailedAt: date('lf'),
1405
- lastFailedAttempt: count('lfa'),
1406
- lastError: column('e') ?? null,
1407
- failedAt: date('f'),
1408
- })
1409
- }
1935
+ ]
1936
+
1937
+ const toHeadEvent = (sessionId: string, head: HeadTuple): HeadEvent => ({
1938
+ ...toStoredEvent(sessionId, head[4]),
1939
+ ...dispatchOf(head[0], laneOf(head[1]), head[3]),
1940
+ })
1941
+
1942
+ const compareText = (a: string, b: string): number =>
1943
+ a < b ? -1 : a > b ? 1 : 0
1944
+
1945
+ const byHead = (a: HeadRef, b: HeadRef): number =>
1946
+ a.index - b.index || compareText(a.consumer, b.consumer)
1947
+
1948
+ /** Consumer, then lane, with concurrent cursors last by head. */
1949
+ const byCursor = (a: StoredCursor, b: StoredCursor): number =>
1950
+ compareText(a.consumer, b.consumer) ||
1951
+ (a.lane === null
1952
+ ? b.lane === null
1953
+ ? (a.head ?? 0) - (b.head ?? 0)
1954
+ : 1
1955
+ : b.lane === null
1956
+ ? -1
1957
+ : compareText(a.lane, b.lane))
1410
1958
 
1411
1959
  const toClaimAvailableResult = (
1412
1960
  sessionId: string,
@@ -1415,57 +1963,13 @@ const toClaimAvailableResult = (
1415
1963
  const outcome = String(reply[0])
1416
1964
  if (outcome === 'claimed') {
1417
1965
  const count = Number(reply[1])
1418
- const events: StoredEvent[] = []
1419
- for (let i = 0; i < count; i++) {
1420
- const offset = 2 + i * 11
1421
- const entry = reply[offset + 9] as StreamEntry
1422
- const event = toEvent(sessionId, entry)
1423
- const fields = fieldMap(entry[1])
1424
- events.push(
1425
- Object.assign(event, {
1426
- cause: toCause(fields['cause']),
1427
- lane: fields['lane'] === '' ? null : fields['lane']!,
1428
- processedAt: null,
1429
- processedByAttempt: null,
1430
- returnedEventIds: null,
1431
- firstClaimedAt:
1432
- reply[offset + 3] === null
1433
- ? null
1434
- : new Date(Number(reply[offset + 3])),
1435
- lastClaimedAt:
1436
- reply[offset + 4] === null
1437
- ? null
1438
- : new Date(Number(reply[offset + 4])),
1439
- attemptCount: Number(reply[offset]),
1440
- claimHolder:
1441
- reply[offset + 7] === null ? null : String(reply[offset + 7]),
1442
- claimExpiresAt:
1443
- reply[offset + 8] === null
1444
- ? null
1445
- : new Date(Number(reply[offset + 8])),
1446
- failureCount: Number(reply[offset + 1]),
1447
- lastFailedAt:
1448
- reply[offset + 5] === null
1449
- ? null
1450
- : new Date(Number(reply[offset + 5])),
1451
- lastFailedAttempt:
1452
- reply[offset + 6] === null ? null : Number(reply[offset + 6]),
1453
- lastError:
1454
- reply[offset + 2] === null ? null : String(reply[offset + 2]),
1455
- failedAt:
1456
- reply[offset + 10] === null
1457
- ? null
1458
- : new Date(Number(reply[offset + 10])),
1459
- }),
1460
- )
1461
- }
1462
- const clearIndex = Number(reply[2 + count * 11] ?? 0)
1966
+ const events = (reply.slice(2, 2 + count) as HeadTuple[])
1967
+ .map((head) => toHeadEvent(sessionId, head))
1968
+ .toSorted(byHead)
1969
+ const clearIndex = Number(reply[2 + count] ?? 0)
1463
1970
  if (clearIndex > 0)
1464
1971
  for (const event of events) event.clearIndex = clearIndex
1465
- return {
1466
- outcome: 'claimed',
1467
- events,
1468
- }
1972
+ return { outcome: 'claimed', events, head: Number(reply[3 + count]) }
1469
1973
  }
1470
1974
  if (outcome === 'busy') {
1471
1975
  return { outcome, retryAt: new Date(Number(reply[1])) }
@@ -1547,24 +2051,14 @@ export function createRedisStoreCore(
1547
2051
 
1548
2052
  const storeKey = (s: string): string => `${prefix}:${s}:log`
1549
2053
  const countKey = (s: string): string => `${prefix}:${s}:count`
1550
- const metaKey = (s: string): string => `${prefix}:${s}:meta`
1551
- const metaChunkKey = (s: string, index: number): string =>
1552
- `${metaKey(s)}:${Math.floor((index - 1) / META_CHUNK_SIZE)}`
1553
- const metaSplitKey = (s: string): string => `${prefix}:${s}:meta-split`
1554
- const pendingKey = (s: string): string => `${prefix}:${s}:pending`
1555
- const readyKey = (s: string): string => `${prefix}:${s}:ready`
2054
+ // Scripts derive the session's other keys from the counter; keep them in step.
2055
+ const clearKey = (s: string): string => `${prefix}:${s}:clear`
2056
+ const schemaKey = `${prefix}:schema`
1556
2057
  const snapKey = (s: string, r: string): string => `${prefix}:${s}:snap:${r}`
1557
2058
  const snapHistoryKey = (s: string, r: string): string =>
1558
2059
  `${prefix}:${s}:snap-history:${r}`
1559
- const snapshotEventPinsKey = (s: string, index: number): string =>
1560
- `${prefix}:${s}:snapshot-pins:${index}`
1561
- const snapshotEventPinsPrefix = (s: string): string =>
1562
- `${prefix}:${s}:snapshot-pins:`
1563
- const snapshotPinCountsKey = (s: string): string =>
1564
- `${prefix}:${s}:snapshot-pin-counts`
1565
2060
  const presenceKey = (s: string): string => `${prefix}:${s}:presence`
1566
- // Scripts derive this from the counter key; keep the two in step.
1567
- const clearKey = (s: string): string => `${prefix}:${s}:clear`
2061
+ const notifyChannel = (s: string): string => `${prefix}:${s}:notify`
1568
2062
  // One ownership key per event id; reclaim deletes those naming cleared events.
1569
2063
  // Session ids are raw and may contain colons, so a raw id could name a
1570
2064
  // session key (`ids:count`). A JSON string with escaped colons starts with
@@ -1572,92 +2066,74 @@ export function createRedisStoreCore(
1572
2066
  const idKey = (id: string): string =>
1573
2067
  `${prefix}:ids:${JSON.stringify(id).replaceAll(':', '\\u003a')}`
1574
2068
 
1575
- const evalScript = (
2069
+ const layoutError = (): A2Error =>
2070
+ new A2Error(
2071
+ 'STORE_NOT_CONFIGURED',
2072
+ `the redis keys under prefix '${prefix}' hold an A2 layout other than '${REDIS_LAYOUT}', from another release; point the store at a fresh keyPrefix, or give the contract a new name so it starts new sessions`,
2073
+ )
2074
+
2075
+ /** Runs a script over a session: KEYS start with its counter and the layout key. */
2076
+ const evalScript = async (
1576
2077
  script: string,
2078
+ sessionId: string,
1577
2079
  keys: string[],
1578
2080
  args: Array<string | number>,
1579
- ): Promise<unknown> => call('EVAL', script, keys.length, ...keys, ...args)
2081
+ ): Promise<unknown> => {
2082
+ const allKeys = [countKey(sessionId), schemaKey, ...keys]
2083
+ try {
2084
+ return await call('EVAL', script, allKeys.length, ...allKeys, ...args)
2085
+ } catch (error) {
2086
+ if (String((error as Error)?.message).includes(LAYOUT_ERROR))
2087
+ throw layoutError()
2088
+ throw error
2089
+ }
2090
+ }
1580
2091
 
1581
- const readRangeBounds = async (
2092
+ /**
2093
+ * Runs a write that may need a type's index to reach further back first.
2094
+ * Such a write returns ['index', type, after] having written nothing, and
2095
+ * runs again once INDEX_TYPE_LUA has indexed that history, a page per call.
2096
+ */
2097
+ const indexing = async (
1582
2098
  sessionId: string,
1583
- ): Promise<{ frontier: number; metaSplit: number; clearIndex: number }> => {
1584
- const [count, split, cleared] = (await call(
2099
+ run: () => Promise<unknown>,
2100
+ ): Promise<unknown[]> => {
2101
+ for (;;) {
2102
+ // oxlint-disable-next-line no-await-in-loop -- each run follows the indexing it asked for
2103
+ const reply = (await run()) as unknown[]
2104
+ if (reply[0] !== 'index') return reply
2105
+ for (;;) {
2106
+ // oxlint-disable-next-line no-await-in-loop -- each page lowers the index's floor
2107
+ const [done] = (await evalScript(
2108
+ INDEX_TYPE_LUA,
2109
+ sessionId,
2110
+ [],
2111
+ [String(reply[1]), Number(reply[2])],
2112
+ )) as [string]
2113
+ if (done === '1') break
2114
+ }
2115
+ }
2116
+ }
2117
+
2118
+ /** The frontier and clear marker, checking the layout in the same MGET. */
2119
+ const readBounds = async (
2120
+ sessionId: string,
2121
+ ): Promise<{ frontier: number; clearIndex: number }> => {
2122
+ const [count, cleared, schema] = (await call(
1585
2123
  'MGET',
1586
2124
  countKey(sessionId),
1587
- metaSplitKey(sessionId),
1588
2125
  clearKey(sessionId),
2126
+ schemaKey,
1589
2127
  )) as [unknown, unknown, unknown]
1590
- const frontier = toIndex({ raw: count, sessionId, name: 'event count' })
2128
+ if (schema !== null && schema !== undefined && schema !== REDIS_LAYOUT) {
2129
+ throw layoutError()
2130
+ }
1591
2131
  return {
1592
- frontier,
1593
- metaSplit:
1594
- split === null
1595
- ? frontier
1596
- : toIndex({ raw: split, sessionId, name: 'metadata split' }),
2132
+ frontier: toIndex({ raw: count, sessionId, name: 'event count' }),
1597
2133
  clearIndex: toIndex({ raw: cleared, sessionId, name: 'clear boundary' }),
1598
2134
  }
1599
2135
  }
1600
2136
 
1601
- const readRangePage = async ({
1602
- sessionId,
1603
- afterIndex,
1604
- throughIndex,
1605
- metaSplit,
1606
- }: {
1607
- sessionId: string
1608
- afterIndex: number
1609
- throughIndex: number
1610
- metaSplit: number
1611
- }): Promise<StoredEvent[]> => {
1612
- if (throughIndex <= afterIndex) return []
1613
- const entries = (await call(
1614
- 'XRANGE',
1615
- storeKey(sessionId),
1616
- `${afterIndex + 1}-0`,
1617
- `${throughIndex}-0`,
1618
- 'COUNT',
1619
- READ_PAGE_SIZE,
1620
- )) as StreamEntry[]
1621
- if (entries.length === 0) return []
1622
-
1623
- const fieldsByKey = new Map<string, string[]>()
1624
- for (const entry of entries) {
1625
- const index = entryIndex(entry[0])
1626
- const key =
1627
- index <= metaSplit ? metaKey(sessionId) : metaChunkKey(sessionId, index)
1628
- const fields = fieldsByKey.get(key) ?? []
1629
- fields.push(
1630
- ...STORED_EVENT_META_SUFFIXES.map((suffix) => `${index}:${suffix}`),
1631
- )
1632
- fieldsByKey.set(key, fields)
1633
- }
1634
- const lookups = [...fieldsByKey]
1635
- const results = await Promise.allSettled(
1636
- lookups.map(([key, fields]) => call('HMGET', key, ...fields)),
1637
- )
1638
- const meta: Record<string, string> = {}
1639
- for (let position = 0; position < lookups.length; position += 1) {
1640
- const result = results[position]!
1641
- if (result.status === 'rejected') throw result.reason
1642
- const fields = lookups[position]![1]
1643
- const values = (result.value as Array<string | null> | null) ?? []
1644
- for (let i = 0; i < fields.length; i += 1) {
1645
- const value = values[i]
1646
- if (value !== null && value !== undefined) {
1647
- meta[fields[i]!] = value
1648
- }
1649
- }
1650
- }
1651
- return entries.map((entry) => {
1652
- const idx = entryIndex(entry[0])
1653
- return toStoredEvent(
1654
- sessionId,
1655
- entry,
1656
- (suffix) => meta[`${idx}:${suffix}`],
1657
- )
1658
- })
1659
- }
1660
-
1661
2137
  const readEventPages = async (
1662
2138
  sessionId: string,
1663
2139
  afterIndex: number,
@@ -1744,11 +2220,8 @@ export function createRedisStoreCore(
1744
2220
  ): Promise<{ rows: StoredEvent[]; clearIndex: number }> => {
1745
2221
  if (throughIndex !== undefined && throughIndex <= afterIndex)
1746
2222
  return { rows: [], clearIndex: 0 }
1747
- const {
1748
- frontier: currentFrontier,
1749
- metaSplit,
1750
- clearIndex,
1751
- } = await readRangeBounds(sessionId)
2223
+ const { frontier: currentFrontier, clearIndex } =
2224
+ await readBounds(sessionId)
1752
2225
  const frontier = Math.min(throughIndex ?? currentFrontier, currentFrontier)
1753
2226
  // Reads start at the latest clear marker.
1754
2227
  const after = Math.max(afterIndex, clearIndex - 1)
@@ -1758,15 +2231,18 @@ export function createRedisStoreCore(
1758
2231
  let nextIndex = after + 1
1759
2232
  while (nextIndex <= frontier) {
1760
2233
  // oxlint-disable-next-line no-await-in-loop -- each page advances the dense frozen range
1761
- const page = await readRangePage({
1762
- sessionId,
1763
- afterIndex: nextIndex - 1,
1764
- throughIndex: frontier,
1765
- metaSplit,
1766
- })
1767
- if (page.length === 0) {
2234
+ const entries = (await call(
2235
+ 'XRANGE',
2236
+ storeKey(sessionId),
2237
+ `${nextIndex}-0`,
2238
+ `${frontier}-0`,
2239
+ 'COUNT',
2240
+ READ_PAGE_SIZE,
2241
+ )) as StreamEntry[]
2242
+ if (entries.length === 0) {
1768
2243
  throw rangeGap(sessionId, `ended before index ${frontier}`)
1769
2244
  }
2245
+ const page = entries.map((entry) => toStoredEvent(sessionId, entry))
1770
2246
  for (let position = 0; position < page.length; position += 1) {
1771
2247
  if (page[position]!.index !== nextIndex + position) {
1772
2248
  throw rangeGap(sessionId, 'is not contiguous')
@@ -1836,36 +2312,29 @@ export function createRedisStoreCore(
1836
2312
  type: e.type,
1837
2313
  payload: e.payload,
1838
2314
  cause: e.cause ? { ...e.cause } : null,
1839
- lane: e.lane ?? null,
1840
- settled: e.settled === true,
2315
+ targets: e.targets,
1841
2316
  }))
1842
- const args: Array<string | number> = [
1843
- sessionId,
1844
- now,
1845
- withIds.length,
1846
- ...withIds.flatMap((e) => [
1847
- e.id,
1848
- e.type,
1849
- json(e.payload),
1850
- e.cause ? json(e.cause) : '',
1851
- e.lane ?? '',
1852
- e.settled ? '1' : '0',
1853
- ]),
1854
- `${prefix}:${sessionId}:notify`,
1855
- appendOptions?.originIndex ?? '',
1856
- ]
1857
- const reply = (await evalScript(
1858
- APPEND_LUA,
1859
- [
1860
- storeKey(sessionId),
1861
- countKey(sessionId),
1862
- metaKey(sessionId),
1863
- pendingKey(sessionId),
1864
- readyKey(sessionId),
1865
- metaSplitKey(sessionId),
1866
- ...withIds.map((e) => idKey(e.id)),
1867
- ],
1868
- args,
2317
+ const reply = (await indexing(sessionId, () =>
2318
+ evalScript(
2319
+ APPEND_LUA,
2320
+ sessionId,
2321
+ withIds.map((e) => idKey(e.id)),
2322
+ [
2323
+ sessionId,
2324
+ now,
2325
+ withIds.length,
2326
+ servedArg(appendOptions?.consumers),
2327
+ notifyChannel(sessionId),
2328
+ appendOptions?.originIndex ?? '',
2329
+ ...withIds.flatMap((e) => [
2330
+ e.id,
2331
+ e.type,
2332
+ json(e.payload),
2333
+ e.cause ? json(e.cause) : '',
2334
+ targetsArg(e.targets),
2335
+ ]),
2336
+ ],
2337
+ ),
1869
2338
  )) as string[]
1870
2339
 
1871
2340
  const [verdict] = reply
@@ -1878,20 +2347,15 @@ export function createRedisStoreCore(
1878
2347
  `event id '${reply[1]}' already exists in another session`,
1879
2348
  )
1880
2349
  }
1881
- if (verdict === 'no_parent') {
1882
- throw new TypeError(
1883
- `no event at index ${reply[1]} in session '${sessionId}'`,
1884
- )
1885
- }
1886
2350
  if (verdict === 'cleared') {
1887
2351
  throw sessionClearedError(
1888
2352
  `scheduled work from event ${reply[1]} was cleared in session '${sessionId}'`,
1889
2353
  )
1890
2354
  }
1891
2355
  if (verdict === 'superseded') {
1892
- throw new A2Error(
1893
- 'SUPERSEDED_ATTEMPT',
1894
- `a newer attempt owns event ${reply[1]} in session '${sessionId}'`,
2356
+ throw supersededAttemptError(
2357
+ sessionId,
2358
+ withIds[Number(reply[1]) - 1]!.cause!,
1895
2359
  )
1896
2360
  }
1897
2361
  if (verdict === 'partial') {
@@ -1933,26 +2397,39 @@ export function createRedisStoreCore(
1933
2397
  sessionId,
1934
2398
  createdAt: new Date(now),
1935
2399
  cause: e.cause,
1936
- lane: e.lane,
1937
- processedAt: e.settled ? new Date(now) : null,
1938
- processedByAttempt: null,
1939
- returnedEventIds: null,
1940
- firstClaimedAt: null,
1941
- lastClaimedAt: null,
1942
- attemptCount: 0,
1943
- claimHolder: null,
1944
- claimExpiresAt: null,
1945
- failureCount: 0,
1946
- lastFailedAt: null,
1947
- lastFailedAttempt: null,
1948
- lastError: null,
1949
- failedAt: null,
1950
2400
  })),
1951
2401
  hasPending: reply[2] === '1',
1952
2402
  }
1953
2403
  })
1954
2404
  },
1955
2405
 
2406
+ async cursors(sessionId) {
2407
+ return wrap(async () => {
2408
+ const reply = (await evalScript(
2409
+ CURSORS_LUA,
2410
+ sessionId,
2411
+ [],
2412
+ [],
2413
+ )) as Array<
2414
+ [
2415
+ consumer: string,
2416
+ tag: string,
2417
+ position: string,
2418
+ head: string | null,
2419
+ columns: Column[],
2420
+ ]
2421
+ >
2422
+ return reply
2423
+ .map(([consumer, tag, position, head, columns]): StoredCursor =>
2424
+ Object.assign(dispatchOf(consumer, laneOf(tag), columns), {
2425
+ position: Number(position),
2426
+ head: head === null ? null : Number(head),
2427
+ }),
2428
+ )
2429
+ .toSorted(byCursor)
2430
+ })
2431
+ },
2432
+
1956
2433
  async read(sessionId, opts) {
1957
2434
  return wrap(() =>
1958
2435
  retryAfterGap(() =>
@@ -1967,21 +2444,7 @@ export function createRedisStoreCore(
1967
2444
  return []
1968
2445
  return wrap(() =>
1969
2446
  retryAfterGap(async () => {
1970
- const [count, cleared] = (await call(
1971
- 'MGET',
1972
- countKey(sessionId),
1973
- clearKey(sessionId),
1974
- )) as [unknown, unknown]
1975
- const frontier = toIndex({
1976
- raw: count,
1977
- sessionId,
1978
- name: 'event count',
1979
- })
1980
- const clearIndex = toIndex({
1981
- raw: cleared,
1982
- sessionId,
1983
- name: 'clear boundary',
1984
- })
2447
+ const { frontier, clearIndex } = await readBounds(sessionId)
1985
2448
  // Reads start at the latest clear marker.
1986
2449
  return readEventRange(
1987
2450
  sessionId,
@@ -1997,32 +2460,31 @@ export function createRedisStoreCore(
1997
2460
  holder,
1998
2461
  ttlMs,
1999
2462
  expiresAtMs,
2000
- excludeIndexes,
2463
+ exclude,
2464
+ consumers,
2001
2465
  maxFailures,
2466
+ through,
2002
2467
  }) {
2003
2468
  return wrap(async () => {
2004
2469
  const now = clock.now().getTime()
2005
2470
  const expiresAt = expiresAtMs ?? now + ttlMs
2006
- const excluded = excludeIndexes ?? []
2471
+ const excluded = exclude ?? []
2007
2472
  const reply = (await evalScript(
2008
2473
  CLAIM_AVAILABLE_LUA,
2009
- [
2010
- storeKey(sessionId),
2011
- countKey(sessionId),
2012
- metaKey(sessionId),
2013
- pendingKey(sessionId),
2014
- readyKey(sessionId),
2015
- metaSplitKey(sessionId),
2016
- snapshotPinCountsKey(sessionId),
2017
- ],
2474
+ sessionId,
2475
+ [],
2018
2476
  [
2019
2477
  holder,
2020
2478
  now,
2021
2479
  expiresAt,
2022
- excluded.length,
2023
- ...excluded,
2024
2480
  maxFailures ?? '',
2025
- snapshotEventPinsPrefix(sessionId),
2481
+ servedArg(consumers),
2482
+ through ?? '',
2483
+ excluded.length,
2484
+ ...excluded.flatMap((ref) => [
2485
+ ref.consumer,
2486
+ ref.lane === null ? `#${ref.index}` : `L${ref.lane}`,
2487
+ ]),
2026
2488
  ],
2027
2489
  )) as unknown[]
2028
2490
  return toClaimAvailableResult(sessionId, reply)
@@ -2030,30 +2492,48 @@ export function createRedisStoreCore(
2030
2492
  },
2031
2493
 
2032
2494
  async renewClaims({ sessionId, holder, claims, ttlMs, expiresAtMs }) {
2495
+ if (claims.length === 0) return { renewed: [], superseded: [] }
2033
2496
  return wrap(async () => {
2034
2497
  const now = clock.now().getTime()
2035
2498
  const expiresAt = expiresAtMs ?? now + ttlMs
2036
2499
  const reply = (await evalScript(
2037
2500
  RENEW_CLAIMS_LUA,
2038
- [countKey(sessionId), metaKey(sessionId), metaSplitKey(sessionId)],
2501
+ sessionId,
2502
+ [],
2039
2503
  [
2040
2504
  holder,
2041
2505
  now,
2042
2506
  expiresAt,
2043
2507
  claims.length,
2044
- ...claims.flatMap((claim) => [claim.index, claim.attempt]),
2508
+ ...claims.flatMap((claim) => [
2509
+ claim.consumer,
2510
+ tagOf(claim.lane),
2511
+ claim.index,
2512
+ claim.attempt,
2513
+ ]),
2045
2514
  ],
2046
2515
  )) as string[]
2047
2516
  const renewedCount = Number(reply[0])
2048
- const values = reply.slice(1).map(Number)
2517
+ const refs = reply.slice(1).map((position): HeadRef => {
2518
+ const { consumer, lane, index } = claims[Number(position) - 1]!
2519
+ return { consumer, lane, index }
2520
+ })
2049
2521
  return {
2050
- renewed: values.slice(0, renewedCount).toSorted((a, b) => a - b),
2051
- superseded: values.slice(renewedCount).toSorted((a, b) => a - b),
2522
+ renewed: refs.slice(0, renewedCount).toSorted(byHead),
2523
+ superseded: refs.slice(renewedCount).toSorted(byHead),
2052
2524
  }
2053
2525
  })
2054
2526
  },
2055
2527
 
2056
- async completeAttempt({ sessionId, index, attempt, events }) {
2528
+ async completeAttempt({
2529
+ sessionId,
2530
+ consumer,
2531
+ lane,
2532
+ index,
2533
+ attempt,
2534
+ events,
2535
+ consumers,
2536
+ }) {
2057
2537
  return wrap(async () => {
2058
2538
  const ids = events.map((event) => event.id)
2059
2539
  if (new Set(ids).size !== ids.length) {
@@ -2063,34 +2543,32 @@ export function createRedisStoreCore(
2063
2543
  )
2064
2544
  }
2065
2545
  const now = clock.now().getTime()
2066
- const reply = (await evalScript(
2067
- COMPLETE_ATTEMPT_LUA,
2068
- [
2069
- storeKey(sessionId),
2070
- countKey(sessionId),
2071
- metaKey(sessionId),
2072
- pendingKey(sessionId),
2073
- readyKey(sessionId),
2074
- snapshotEventPinsKey(sessionId, index),
2075
- snapshotPinCountsKey(sessionId),
2076
- metaSplitKey(sessionId),
2077
- ...ids.map((id) => idKey(id)),
2078
- ],
2079
- [
2546
+ const cause: EventCause = { consumer, lane, index, attempt }
2547
+ const reply = (await indexing(sessionId, () =>
2548
+ evalScript(
2549
+ COMPLETE_ATTEMPT_LUA,
2080
2550
  sessionId,
2081
- now,
2082
- index,
2083
- attempt,
2084
- events.length,
2085
- ...events.flatMap((event) => [
2086
- event.id,
2087
- event.type,
2088
- json(event.payload),
2089
- event.lane ?? '',
2090
- event.settled === true ? '1' : '0',
2091
- ]),
2092
- `${prefix}:${sessionId}:notify`,
2093
- ],
2551
+ ids.map((id) => idKey(id)),
2552
+ [
2553
+ sessionId,
2554
+ now,
2555
+ consumer,
2556
+ tagOf(lane),
2557
+ index,
2558
+ attempt,
2559
+ json(cause),
2560
+ servedArg(consumers),
2561
+ notifyChannel(sessionId),
2562
+ json(ids),
2563
+ events.length,
2564
+ ...events.flatMap((event) => [
2565
+ event.id,
2566
+ event.type,
2567
+ json(event.payload),
2568
+ targetsArg(event.targets),
2569
+ ]),
2570
+ ],
2571
+ ),
2094
2572
  )) as string[]
2095
2573
  const outcome = reply[0]
2096
2574
  if (outcome === 'missing') {
@@ -2106,7 +2584,6 @@ export function createRedisStoreCore(
2106
2584
  }
2107
2585
  if (outcome === 'superseded') return { outcome }
2108
2586
 
2109
- const cause = { index, attempt }
2110
2587
  if (outcome === 'ok') {
2111
2588
  const base = Number(reply[1]) - 1
2112
2589
  return {
@@ -2118,21 +2595,7 @@ export function createRedisStoreCore(
2118
2595
  index: base + 1 + i,
2119
2596
  sessionId,
2120
2597
  createdAt: new Date(now),
2121
- cause,
2122
- lane: event.lane ?? null,
2123
- processedAt: event.settled === true ? new Date(now) : null,
2124
- processedByAttempt: null,
2125
- returnedEventIds: null,
2126
- firstClaimedAt: null,
2127
- lastClaimedAt: null,
2128
- attemptCount: 0,
2129
- claimHolder: null,
2130
- claimExpiresAt: null,
2131
- failureCount: 0,
2132
- lastFailedAt: null,
2133
- lastFailedAttempt: null,
2134
- lastError: null,
2135
- failedAt: null,
2598
+ cause: { ...cause },
2136
2599
  })),
2137
2600
  } satisfies CompleteAttemptResult
2138
2601
  }
@@ -2150,6 +2613,20 @@ export function createRedisStoreCore(
2150
2613
  )
2151
2614
  // Cleared after the script matched the retry.
2152
2615
  if (index <= clearIndex) return { outcome: 'superseded' }
2616
+ if (
2617
+ existing.some(
2618
+ (row) =>
2619
+ row.cause?.consumer !== consumer ||
2620
+ row.cause.lane !== lane ||
2621
+ row.cause.index !== index ||
2622
+ row.cause.attempt !== attempt,
2623
+ )
2624
+ ) {
2625
+ throw new A2Error(
2626
+ 'PARTIAL_DUPLICATE_BATCH',
2627
+ 'returned event retry does not match the committed batch',
2628
+ )
2629
+ }
2153
2630
  return {
2154
2631
  outcome: 'completed',
2155
2632
  events: idempotentReplay(events, existing),
@@ -2161,6 +2638,8 @@ export function createRedisStoreCore(
2161
2638
 
2162
2639
  async failAttempt({
2163
2640
  sessionId,
2641
+ consumer,
2642
+ lane,
2164
2643
  index,
2165
2644
  attempt,
2166
2645
  error,
@@ -2171,16 +2650,11 @@ export function createRedisStoreCore(
2171
2650
  const now = clock.now().getTime()
2172
2651
  const reply = (await evalScript(
2173
2652
  FAIL_ATTEMPT_LUA,
2653
+ sessionId,
2654
+ [],
2174
2655
  [
2175
- countKey(sessionId),
2176
- metaKey(sessionId),
2177
- pendingKey(sessionId),
2178
- readyKey(sessionId),
2179
- snapshotEventPinsKey(sessionId, index),
2180
- snapshotPinCountsKey(sessionId),
2181
- metaSplitKey(sessionId),
2182
- ],
2183
- [
2656
+ consumer,
2657
+ tagOf(lane),
2184
2658
  index,
2185
2659
  attempt,
2186
2660
  error,
@@ -2190,8 +2664,6 @@ export function createRedisStoreCore(
2190
2664
  ],
2191
2665
  )) as string[]
2192
2666
  const outcome = reply[0]
2193
- if (outcome === 'cleared')
2194
- return { outcome: 'superseded', failureCount: 0 }
2195
2667
  if (outcome === 'missing') {
2196
2668
  throw new TypeError(
2197
2669
  `no event at index ${index} in session '${sessionId}'`,
@@ -2208,58 +2680,74 @@ export function createRedisStoreCore(
2208
2680
  return wrap(async () => {
2209
2681
  const reply = (await evalScript(
2210
2682
  DEAD_LETTERS_LUA,
2211
- [
2212
- storeKey(sessionId),
2213
- countKey(sessionId),
2214
- metaKey(sessionId),
2215
- pendingKey(sessionId),
2216
- metaSplitKey(sessionId),
2217
- ],
2683
+ sessionId,
2218
2684
  [],
2219
- )) as unknown[]
2220
- const events: StoredEvent[] = []
2221
- for (let i = 0; i + 1 < reply.length; i += 2) {
2222
- const columns = reply[i + 1] as Array<string | null>
2223
- events.push(
2224
- toStoredEvent(
2225
- sessionId,
2226
- reply[i] as StreamEntry,
2227
- (suffix) =>
2228
- columns[STORED_EVENT_META_SUFFIXES.indexOf(suffix)] ??
2229
- undefined,
2230
- ),
2231
- )
2232
- }
2233
- return events
2685
+ [],
2686
+ )) as HeadTuple[]
2687
+ return reply
2688
+ .map((head) => toHeadEvent(sessionId, head))
2689
+ .toSorted(byHead)
2234
2690
  })
2235
2691
  },
2236
2692
 
2237
- async redrive({ sessionId, index, action }) {
2693
+ async redrive({ sessionId, consumer, index, action, consumers }) {
2238
2694
  return wrap(async () => {
2239
- const [redriven, hasPending] = (await evalScript(
2240
- REDRIVE_LUA,
2241
- [
2242
- countKey(sessionId),
2243
- metaKey(sessionId),
2244
- pendingKey(sessionId),
2245
- readyKey(sessionId),
2246
- metaSplitKey(sessionId),
2247
- ],
2248
- [index, action, clock.now().getTime()],
2695
+ const [redriven, hasPending] = (await indexing(sessionId, () =>
2696
+ evalScript(
2697
+ REDRIVE_LUA,
2698
+ sessionId,
2699
+ [],
2700
+ [
2701
+ consumer,
2702
+ index,
2703
+ action,
2704
+ clock.now().getTime(),
2705
+ servedArg(consumers),
2706
+ ],
2707
+ ),
2249
2708
  )) as [string, string]
2250
2709
  return { redriven: redriven === '1', hasPending: hasPending === '1' }
2251
2710
  })
2252
2711
  },
2253
2712
 
2713
+ async wakeLog({ sessionId, consumer }) {
2714
+ return wrap(async () => {
2715
+ const { name, log } = consumer
2716
+ const [, position, head, columns, sessionHead] = (await indexing(
2717
+ sessionId,
2718
+ () =>
2719
+ evalScript(
2720
+ WAKE_LOG_LUA,
2721
+ sessionId,
2722
+ [],
2723
+ [name, log.lane, logTypesArg(log.types), log.backfill ? 1 : 0],
2724
+ ),
2725
+ )) as [
2726
+ ok: string,
2727
+ position: string,
2728
+ head: string | null,
2729
+ columns: Column[],
2730
+ sessionHead: string,
2731
+ ]
2732
+ return {
2733
+ cursor: Object.assign(dispatchOf(name, log.lane, columns), {
2734
+ position: Number(position),
2735
+ head: head === null ? null : Number(head),
2736
+ }),
2737
+ head: Number(sessionHead),
2738
+ }
2739
+ })
2740
+ },
2741
+
2254
2742
  async readState(sessionId, reducerName, stateOptions) {
2255
2743
  return wrap(() =>
2256
2744
  retryAfterGap(async () => {
2257
2745
  const reply = (await evalScript(
2258
2746
  READ_STATE_LUA,
2747
+ sessionId,
2259
2748
  [
2260
2749
  snapKey(sessionId, reducerName),
2261
2750
  snapHistoryKey(sessionId, reducerName),
2262
- countKey(sessionId),
2263
2751
  ],
2264
2752
  [
2265
2753
  stateOptions?.snapshotThroughIndex ??
@@ -2315,33 +2803,28 @@ export function createRedisStoreCore(
2315
2803
  async putSnapshots(sessionId, reducerName, writes) {
2316
2804
  const byIndex = new Map<
2317
2805
  number,
2318
- { index: number; state: unknown; pinEventIndexes: Set<number> }
2806
+ { index: number; state: unknown; pins: Map<string, HeadRef> }
2319
2807
  >()
2320
2808
  for (const write of writes) {
2321
2809
  const existing = byIndex.get(write.index)
2322
- if (existing) {
2323
- existing.state = write.state
2324
- for (const eventIndex of write.pinEventIndexes ?? []) {
2325
- existing.pinEventIndexes.add(eventIndex)
2326
- }
2327
- } else {
2810
+ const pins = existing?.pins ?? new Map<string, HeadRef>()
2811
+ for (const pin of write.pins ?? []) {
2812
+ pins.set(JSON.stringify([pin.consumer, pin.lane, pin.index]), pin)
2813
+ }
2814
+ if (existing) existing.state = write.state
2815
+ else
2328
2816
  byIndex.set(write.index, {
2329
2817
  index: write.index,
2330
2818
  state: write.state,
2331
- pinEventIndexes: new Set(write.pinEventIndexes ?? []),
2819
+ pins,
2332
2820
  })
2333
- }
2334
2821
  }
2335
2822
  const normalized = [...byIndex.values()].toSorted(
2336
2823
  (a, b) => a.index - b.index,
2337
2824
  )
2338
2825
  if (normalized.length === 0) return
2339
2826
  await wrap(async () => {
2340
- const args: Array<string | number> = [
2341
- clock.now().getTime(),
2342
- snapshotEventPinsPrefix(sessionId),
2343
- normalized.length,
2344
- ]
2827
+ const args: Array<string | number> = [normalized.length]
2345
2828
  for (const write of normalized) {
2346
2829
  args.push(
2347
2830
  write.index,
@@ -2352,19 +2835,20 @@ export function createRedisStoreCore(
2352
2835
  // This reducer is in the session's registry (PUT_SNAPSHOTS_LUA).
2353
2836
  r: 1,
2354
2837
  }),
2355
- write.pinEventIndexes.size,
2356
- ...write.pinEventIndexes,
2838
+ write.pins.size,
2839
+ ...[...write.pins.values()].flatMap((pin) => [
2840
+ pin.consumer,
2841
+ tagOf(pin.lane),
2842
+ pin.index,
2843
+ ]),
2357
2844
  )
2358
2845
  }
2359
2846
  await evalScript(
2360
2847
  PUT_SNAPSHOTS_LUA,
2848
+ sessionId,
2361
2849
  [
2362
2850
  snapKey(sessionId, reducerName),
2363
2851
  snapHistoryKey(sessionId, reducerName),
2364
- countKey(sessionId),
2365
- metaKey(sessionId),
2366
- snapshotPinCountsKey(sessionId),
2367
- metaSplitKey(sessionId),
2368
2852
  ],
2369
2853
  [...args, reducerName],
2370
2854
  )
@@ -2384,28 +2868,28 @@ export function createRedisStoreCore(
2384
2868
  // The hash field is JSON `[participant, field]`: both are
2385
2869
  // caller strings, so no separator character is
2386
2870
  // collision-safe — the JSON array is.
2387
- const applied = (await evalScript(
2871
+ const applied = (await call(
2872
+ 'EVAL',
2388
2873
  PRESENCE_SET_LUA,
2389
- [presenceKey(ns)],
2390
- [
2391
- atMs,
2392
- meta.ttlMs,
2393
- entries.length,
2394
- ...entries.flatMap(([field, value]) => [
2395
- JSON.stringify([participant, field]),
2396
- value === null
2397
- ? ''
2398
- : json({
2399
- v: value,
2400
- s: meta.seen,
2401
- a: atMs,
2402
- e: expiresAtMs,
2403
- }),
2404
- `${json(field)}:${json(value)}`,
2405
- ]),
2406
- presenceKey(ns),
2407
- `${json({ src: options.presenceSource, participant, seen: meta.seen, at: atMs }).slice(0, -1)},"values":{`,
2408
- ],
2874
+ 1,
2875
+ presenceKey(ns),
2876
+ atMs,
2877
+ meta.ttlMs,
2878
+ entries.length,
2879
+ ...entries.flatMap(([field, value]) => [
2880
+ JSON.stringify([participant, field]),
2881
+ value === null
2882
+ ? ''
2883
+ : json({
2884
+ v: value,
2885
+ s: meta.seen,
2886
+ a: atMs,
2887
+ e: expiresAtMs,
2888
+ }),
2889
+ `${json(field)}:${json(value)}`,
2890
+ ]),
2891
+ presenceKey(ns),
2892
+ `${json({ src: options.presenceSource, participant, seen: meta.seen, at: atMs }).slice(0, -1)},"values":{`,
2409
2893
  )) as string[]
2410
2894
  if (applied.length === 0) return null
2411
2895
  // Field-keyed and caller-named — null-prototype, like every
@@ -2426,10 +2910,12 @@ export function createRedisStoreCore(
2426
2910
 
2427
2911
  async read(ns) {
2428
2912
  return wrap(async () => {
2429
- const reply = (await evalScript(
2913
+ const reply = (await call(
2914
+ 'EVAL',
2430
2915
  PRESENCE_READ_LUA,
2431
- [presenceKey(ns)],
2432
- [clock.now().getTime()],
2916
+ 1,
2917
+ presenceKey(ns),
2918
+ clock.now().getTime(),
2433
2919
  )) as string[]
2434
2920
  const rows: PresenceRow[] = []
2435
2921
  for (let i = 0; i + 1 < reply.length; i += 2) {
@@ -2492,33 +2978,23 @@ export function createRedisStoreCore(
2492
2978
  const markerId = generateId()
2493
2979
  const reply = (await evalScript(
2494
2980
  CLEAR_LUA,
2495
- [
2496
- storeKey(sessionId),
2497
- countKey(sessionId),
2498
- metaKey(sessionId),
2499
- metaSplitKey(sessionId),
2500
- idKey(markerId),
2501
- pendingKey(sessionId),
2502
- readyKey(sessionId),
2503
- ],
2981
+ sessionId,
2982
+ [idKey(markerId)],
2504
2983
  [
2505
2984
  sessionId,
2506
2985
  clock.now().getTime(),
2507
2986
  ifLastIndex ?? '',
2987
+ cause?.consumer ?? '',
2988
+ cause ? tagOf(cause.lane) : '',
2508
2989
  cause?.index ?? '',
2509
2990
  cause?.attempt ?? '',
2510
2991
  markerId,
2511
- `${prefix}:${sessionId}:notify`,
2992
+ notifyChannel(sessionId),
2512
2993
  ],
2513
2994
  )) as string[]
2514
2995
  if (reply[0] === 'superseded') {
2515
2996
  throw supersededAttemptError(sessionId, cause!)
2516
2997
  }
2517
- if (reply[0] === 'no_parent') {
2518
- throw new TypeError(
2519
- `no event at index ${reply[1]} in session '${sessionId}'`,
2520
- )
2521
- }
2522
2998
  if (reply[0] === 'foreign') {
2523
2999
  throw new A2Error(
2524
3000
  'PARTIAL_DUPLICATE_BATCH',
@@ -2537,7 +3013,8 @@ export function createRedisStoreCore(
2537
3013
  return wrap(async () => {
2538
3014
  const [log, ...pairs] = (await evalScript(
2539
3015
  RECLAIM_BATCH_LUA,
2540
- [countKey(sessionId)],
3016
+ sessionId,
3017
+ [],
2541
3018
  [READ_PAGE_SIZE],
2542
3019
  )) as string[]
2543
3020
  if (log === undefined) return { deleted: 0, done: true }
@@ -2547,14 +3024,9 @@ export function createRedisStoreCore(
2547
3024
  for (let i = 1; i < pairs.length; i += 2) ids.push(pairs[i]!)
2548
3025
  const [deleted, done] = (await evalScript(
2549
3026
  RECLAIM_LUA,
2550
- [
2551
- log,
2552
- countKey(sessionId),
2553
- metaKey(sessionId),
2554
- metaSplitKey(sessionId),
2555
- ...ids.map((id) => idKey(id)),
2556
- ],
2557
- [sessionId, snapshotEventPinsPrefix(sessionId), ids.length, ...pairs],
3027
+ sessionId,
3028
+ [log, ...ids.map((id) => idKey(id))],
3029
+ [sessionId, ids.length, ...pairs],
2558
3030
  )) as [number, number]
2559
3031
  return { deleted: Number(deleted), done: Number(done) === 1 }
2560
3032
  })