experimental-a2 0.20.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (309) hide show
  1. package/CHANGELOG.md +55 -0
  2. package/README.md +1 -1
  3. package/dist/{actor-C7b-1arn.d.ts → actor-CqibLgjQ.d.ts} +15 -7
  4. package/dist/actor-CqibLgjQ.d.ts.map +1 -0
  5. package/dist/actor-client.d.ts +1 -1
  6. package/dist/actor-client.js +2 -2
  7. package/dist/actor-react.d.ts +3 -3
  8. package/dist/actor-react.js +5 -5
  9. package/dist/{actor-shared-t7w8xIKP.js → actor-shared-DFET5q16.js} +3 -3
  10. package/dist/actor-shared-DFET5q16.js.map +1 -0
  11. package/dist/actor.d.ts +1 -1
  12. package/dist/actor.js +54 -43
  13. package/dist/actor.js.map +1 -1
  14. package/dist/{ai-Cz3hqVhk.d.ts → ai-DAu77CDr.d.ts} +8 -8
  15. package/dist/{ai-Cz3hqVhk.d.ts.map → ai-DAu77CDr.d.ts.map} +1 -1
  16. package/dist/{ai-mMiWul1T.js → ai-EEfKyc-z.js} +9 -9
  17. package/dist/ai-EEfKyc-z.js.map +1 -0
  18. package/dist/ai-client.d.ts +2 -2
  19. package/dist/ai-client.js +2 -2
  20. package/dist/ai-fx.d.ts +4 -2
  21. package/dist/ai-fx.d.ts.map +1 -1
  22. package/dist/ai-fx.js +214 -67
  23. package/dist/ai-fx.js.map +1 -1
  24. package/dist/{ai-harness-CV6rXbB3.d.ts → ai-harness-DTjkAr2T.d.ts} +2 -2
  25. package/dist/{ai-harness-CV6rXbB3.d.ts.map → ai-harness-DTjkAr2T.d.ts.map} +1 -1
  26. package/dist/{ai-harness-Clbg5ei7.js → ai-harness-dQeJaR1t.js} +7 -3
  27. package/dist/ai-harness-dQeJaR1t.js.map +1 -0
  28. package/dist/ai-pi.d.ts +1 -1
  29. package/dist/ai-pi.js +3 -3
  30. package/dist/ai-pi.js.map +1 -1
  31. package/dist/ai-react.d.ts +5 -5
  32. package/dist/ai-react.js +2 -2
  33. package/dist/ai-server.d.ts +14 -17
  34. package/dist/ai-server.d.ts.map +1 -1
  35. package/dist/ai-server.js +86 -73
  36. package/dist/ai-server.js.map +1 -1
  37. package/dist/ai.d.ts +2 -2
  38. package/dist/ai.js +2 -2
  39. package/dist/{client-BhHLoNKF.js → client-BQS4i_V9.js} +29 -6
  40. package/dist/client-BQS4i_V9.js.map +1 -0
  41. package/dist/{client-h_DKFAjj.d.ts → client-CUMbS_38.d.ts} +3 -3
  42. package/dist/{client-h_DKFAjj.d.ts.map → client-CUMbS_38.d.ts.map} +1 -1
  43. package/dist/client.d.ts +3 -3
  44. package/dist/client.js +1 -1
  45. package/dist/{contract-BYKASlCV.js → contract-CWukt01d.js} +2 -2
  46. package/dist/contract-CWukt01d.js.map +1 -0
  47. package/dist/index.d.ts +5 -5
  48. package/dist/index.js +3 -3
  49. package/dist/{initial-snapshot-Bx-c0pDP.d.ts → initial-snapshot-D3Lve43o.d.ts} +4 -4
  50. package/dist/initial-snapshot-D3Lve43o.d.ts.map +1 -0
  51. package/dist/{internal-CscSY7K_.js → internal-CvsrprnU.js} +40 -16
  52. package/dist/internal-CvsrprnU.js.map +1 -0
  53. package/dist/otel.d.ts +1 -1
  54. package/dist/otel.js +1 -1
  55. package/dist/otel.js.map +1 -1
  56. package/dist/{react-BBXgHWFc.d.ts → react-BoHvC3v9.d.ts} +4 -4
  57. package/dist/{react-BBXgHWFc.d.ts.map → react-BoHvC3v9.d.ts.map} +1 -1
  58. package/dist/react.d.ts +1 -1
  59. package/dist/react.js +1 -1
  60. package/dist/{reducer-CqA59Q6U.d.ts → reducer-CYPHsSmd.d.ts} +2 -2
  61. package/dist/{reducer-CqA59Q6U.d.ts.map → reducer-CYPHsSmd.d.ts.map} +1 -1
  62. package/dist/{reducer-BAXc8JyF.js → reducer-CjmvZSAb.js} +2 -2
  63. package/dist/{reducer-BAXc8JyF.js.map → reducer-CjmvZSAb.js.map} +1 -1
  64. package/dist/runtime-consumer-CIeL33wj.js +12 -0
  65. package/dist/runtime-consumer-CIeL33wj.js.map +1 -0
  66. package/dist/scheduler-qstash.d.ts +2 -2
  67. package/dist/scheduler-qstash.js +3 -3
  68. package/dist/scheduler-task-X1SToDjP.js.map +1 -1
  69. package/dist/scheduler-vercel.d.ts +2 -2
  70. package/dist/scheduler-vercel.js +3 -3
  71. package/dist/scheduler-vercel.js.map +1 -1
  72. package/dist/{server-qyb5aJJG.js → server-DtFrWxi3.js} +412 -116
  73. package/dist/server-DtFrWxi3.js.map +1 -0
  74. package/dist/{server-DYAATFuT.d.ts → server-qgez0Xny.d.ts} +108 -41
  75. package/dist/server-qgez0Xny.d.ts.map +1 -0
  76. package/dist/server.browser.js +1 -1
  77. package/dist/server.browser.js.map +1 -1
  78. package/dist/server.d.ts +4 -4
  79. package/dist/server.js +2 -2
  80. package/dist/{session-clear-C-QwTXV1.d.ts → session-clear-D1PFe7xl.d.ts} +2 -2
  81. package/dist/session-clear-D1PFe7xl.d.ts.map +1 -0
  82. package/dist/{session-clear-DGI4jTQ5.js → session-clear-wHd5GCpZ.js} +5 -4
  83. package/dist/session-clear-wHd5GCpZ.js.map +1 -0
  84. package/dist/store-D0MhBvg6.d.ts +532 -0
  85. package/dist/store-D0MhBvg6.d.ts.map +1 -0
  86. package/dist/store-N8PXxDAS.js.map +1 -1
  87. package/dist/store-codec-DwrWlgQx.js +65 -0
  88. package/dist/store-codec-DwrWlgQx.js.map +1 -0
  89. package/dist/store-memory.d.ts +1 -1
  90. package/dist/store-memory.d.ts.map +1 -1
  91. package/dist/store-memory.js +406 -225
  92. package/dist/store-memory.js.map +1 -1
  93. package/dist/{store-polling-7WJV0YWP.js → store-polling-BZt2Rf3D.js} +2 -2
  94. package/dist/{store-polling-7WJV0YWP.js.map → store-polling-BZt2Rf3D.js.map} +1 -1
  95. package/dist/store-postgres.d.ts +1 -1
  96. package/dist/store-postgres.d.ts.map +1 -1
  97. package/dist/store-postgres.js +1598 -587
  98. package/dist/store-postgres.js.map +1 -1
  99. package/dist/store-presence-polling-JuAy2Ck2.js +111 -0
  100. package/dist/store-presence-polling-JuAy2Ck2.js.map +1 -0
  101. package/dist/store-redis-http.d.ts +1 -1
  102. package/dist/store-redis-http.js +4 -4
  103. package/dist/store-redis-notify-CsJw_hfD.js +3060 -0
  104. package/dist/store-redis-notify-CsJw_hfD.js.map +1 -0
  105. package/dist/store-redis.d.ts +1 -1
  106. package/dist/store-redis.d.ts.map +1 -1
  107. package/dist/store-redis.js +4 -5
  108. package/dist/store-redis.js.map +1 -1
  109. package/dist/store-sqlite.d.ts +1 -1
  110. package/dist/store-sqlite.d.ts.map +1 -1
  111. package/dist/store-sqlite.js +527 -299
  112. package/dist/store-sqlite.js.map +1 -1
  113. package/dist/stream-buffer-DrTyiIuo.js +57 -0
  114. package/dist/stream-buffer-DrTyiIuo.js.map +1 -0
  115. package/dist/{telemetry-Cyfqo6Qt.d.ts → telemetry-CWwsh-Gx.d.ts} +3 -3
  116. package/dist/telemetry-CWwsh-Gx.d.ts.map +1 -0
  117. package/dist/{wire-BKxH1vbP.js → wire-CV-Z7wz-.js} +6 -5
  118. package/dist/wire-CV-Z7wz-.js.map +1 -0
  119. package/docs/01-quickstart.mdx +21 -15
  120. package/docs/02-examples.mdx +1 -1
  121. package/docs/actors/01-introduction.mdx +13 -2
  122. package/docs/actors/02-concurrency.mdx +1 -1
  123. package/docs/agents/01-introduction.mdx +1 -1
  124. package/docs/agents/02-tools.mdx +12 -11
  125. package/docs/agents/03-conversations.mdx +36 -25
  126. package/docs/agents/04-harnesses.mdx +14 -7
  127. package/docs/client/03-presence.mdx +18 -14
  128. package/docs/contract/01-events.mdx +2 -2
  129. package/docs/contract/02-state.mdx +18 -14
  130. package/docs/contract/03-sessions.mdx +32 -25
  131. package/docs/index.mdx +29 -24
  132. package/docs/reference/01-core.mdx +7 -6
  133. package/docs/reference/02-server.mdx +250 -191
  134. package/docs/reference/04-ai.mdx +33 -23
  135. package/docs/reference/05-telemetry.mdx +27 -13
  136. package/docs/reference/06-errors.mdx +25 -19
  137. package/docs/reference/meta.ts +1 -1
  138. package/docs/schedulers/01-overview.mdx +8 -8
  139. package/docs/schedulers/02-vercel-queues.mdx +2 -2
  140. package/docs/schedulers/03-qstash.mdx +4 -4
  141. package/docs/server/01-consumers.mdx +418 -0
  142. package/docs/server/02-timers.mdx +31 -21
  143. package/docs/server/03-cancellation.mdx +52 -44
  144. package/docs/server/04-routes.mdx +4 -1
  145. package/docs/server/05-sql-projections.mdx +391 -0
  146. package/docs/server/06-guarantees.mdx +61 -51
  147. package/docs/stores/01-overview.mdx +3 -3
  148. package/docs/stores/02-redis.mdx +24 -10
  149. package/docs/stores/03-redis-http.mdx +8 -6
  150. package/docs/stores/04-postgres.mdx +65 -14
  151. package/docs/stores/05-sqlite.mdx +5 -3
  152. package/docs/upgrades/01-overview.mdx +31 -0
  153. package/docs/upgrades/78-0-22.mdx +115 -0
  154. package/docs/upgrades/79-0-21.mdx +165 -0
  155. package/docs/upgrades/80-0-20.mdx +43 -0
  156. package/docs/upgrades/81-0-19.mdx +33 -0
  157. package/docs/upgrades/82-0-18.mdx +33 -0
  158. package/docs/upgrades/83-0-17.mdx +69 -0
  159. package/docs/upgrades/meta.ts +3 -0
  160. package/examples/playground/DEPLOY.md +39 -7
  161. package/examples/playground/app/a0/chat.test.tsx +2 -2
  162. package/examples/playground/app/a0/sandbox-log.ts +17 -15
  163. package/examples/playground/app/a0/sandbox.ts +58 -13
  164. package/examples/playground/app/a0/server.ts +4 -4
  165. package/examples/playground/app/a2slack/a2slack.css +1067 -0
  166. package/examples/playground/app/a2slack/c/[channel]/page.tsx +20 -0
  167. package/examples/playground/app/a2slack/consumers/agent.ts +63 -0
  168. package/examples/playground/app/a2slack/consumers/feed.ts +164 -0
  169. package/examples/playground/app/a2slack/consumers/notify.ts +116 -0
  170. package/examples/playground/app/a2slack/consumers/search.ts +37 -0
  171. package/examples/playground/app/a2slack/consumers/triage.ts +31 -0
  172. package/examples/playground/app/a2slack/consumers/welcome.ts +25 -0
  173. package/examples/playground/app/a2slack/db/a2.ts +209 -0
  174. package/examples/playground/app/a2slack/db/index.ts +73 -0
  175. package/examples/playground/app/a2slack/db/migrations/0000_extensions.sql +4 -0
  176. package/examples/playground/app/a2slack/db/migrations/0001_workspace.sql +74 -0
  177. package/examples/playground/app/a2slack/db/migrations/0002_a2.sql +215 -0
  178. package/examples/playground/app/a2slack/db/migrations/0003_a2_batch.sql +67 -0
  179. package/examples/playground/app/a2slack/db/migrations/0004_a2_session_event_ids.sql +2 -0
  180. package/examples/playground/app/a2slack/db/migrations/meta/0000_snapshot.json +18 -0
  181. package/examples/playground/app/a2slack/db/migrations/meta/0001_snapshot.json +500 -0
  182. package/examples/playground/app/a2slack/db/migrations/meta/0002_snapshot.json +1159 -0
  183. package/examples/playground/app/a2slack/db/migrations/meta/0003_snapshot.json +1159 -0
  184. package/examples/playground/app/a2slack/db/migrations/meta/0004_snapshot.json +1165 -0
  185. package/examples/playground/app/a2slack/db/migrations/meta/_journal.json +41 -0
  186. package/examples/playground/app/a2slack/db/queries.ts +263 -0
  187. package/examples/playground/app/a2slack/db/schema.ts +145 -0
  188. package/examples/playground/app/a2slack/drills/a2slack.drill.ts +708 -0
  189. package/examples/playground/app/a2slack/drills/harness.ts +244 -0
  190. package/examples/playground/app/a2slack/events/route.ts +36 -0
  191. package/examples/playground/app/a2slack/identity.ts +44 -0
  192. package/examples/playground/app/a2slack/inbox/route.ts +8 -0
  193. package/examples/playground/app/a2slack/lag/data/route.ts +8 -0
  194. package/examples/playground/app/a2slack/lag/page.tsx +101 -0
  195. package/examples/playground/app/a2slack/lag.ts +73 -0
  196. package/examples/playground/app/a2slack/layout.tsx +43 -0
  197. package/examples/playground/app/a2slack/messages/route.ts +17 -0
  198. package/examples/playground/app/a2slack/model.test.ts +208 -0
  199. package/examples/playground/app/a2slack/model.ts +369 -0
  200. package/examples/playground/app/a2slack/models.test.ts +72 -0
  201. package/examples/playground/app/a2slack/page.tsx +10 -0
  202. package/examples/playground/app/a2slack/scheduler/route.ts +5 -0
  203. package/examples/playground/app/a2slack/search/route.ts +40 -0
  204. package/examples/playground/app/a2slack/search-index.ts +108 -0
  205. package/examples/playground/app/a2slack/server.test.ts +315 -0
  206. package/examples/playground/app/a2slack/server.ts +55 -0
  207. package/examples/playground/app/a2slack/sessions.ts +50 -0
  208. package/examples/playground/app/a2slack/thread/route.ts +8 -0
  209. package/examples/playground/app/a2slack/ui/ChannelView.tsx +87 -0
  210. package/examples/playground/app/a2slack/ui/Composer.tsx +117 -0
  211. package/examples/playground/app/a2slack/ui/DotsShadow.tsx +42 -0
  212. package/examples/playground/app/a2slack/ui/Feed.tsx +157 -0
  213. package/examples/playground/app/a2slack/ui/InboxContext.tsx +63 -0
  214. package/examples/playground/app/a2slack/ui/InboxView.tsx +82 -0
  215. package/examples/playground/app/a2slack/ui/Menu.tsx +128 -0
  216. package/examples/playground/app/a2slack/ui/MessageRow.tsx +314 -0
  217. package/examples/playground/app/a2slack/ui/NameGate.tsx +35 -0
  218. package/examples/playground/app/a2slack/ui/SearchContext.tsx +146 -0
  219. package/examples/playground/app/a2slack/ui/SearchResults.tsx +97 -0
  220. package/examples/playground/app/a2slack/ui/Shell.tsx +34 -0
  221. package/examples/playground/app/a2slack/ui/Sidebar.tsx +199 -0
  222. package/examples/playground/app/a2slack/ui/ThreadPanel.tsx +134 -0
  223. package/examples/playground/app/a2slack/ui/useInbox.ts +71 -0
  224. package/examples/playground/app/agent/server.ts +3 -3
  225. package/examples/playground/app/chat/[chatId]/chat-client.tsx +1 -1
  226. package/examples/playground/app/chat/server.ts +45 -35
  227. package/examples/playground/app/components/demo-icons.tsx +7 -0
  228. package/examples/playground/app/components/demos.tsx +7 -0
  229. package/examples/playground/app/orders/[orderId]/order-client.tsx +2 -2
  230. package/examples/playground/app/orders/page.tsx +1 -1
  231. package/examples/playground/app/orders/server.ts +49 -30
  232. package/examples/playground/app/recovery/[recoveryId]/recovery-client.tsx +14 -15
  233. package/examples/playground/app/recovery/page.tsx +4 -4
  234. package/examples/playground/app/recovery/server.ts +23 -32
  235. package/examples/playground/drizzle.config.ts +14 -0
  236. package/examples/playground/lib/store.ts +1 -1
  237. package/examples/playground/next.config.ts +6 -1
  238. package/examples/playground/package.json +12 -1
  239. package/examples/playground/scripts/migrate.mjs +31 -0
  240. package/examples/playground/vercel.json +9 -0
  241. package/examples/playground/vitest.drills.config.ts +32 -0
  242. package/package.json +5 -2
  243. package/sql/postgres/drizzle-schema.ts +209 -0
  244. package/sql/postgres/schema.sql +227 -67
  245. package/src/actor-shared.ts +10 -2
  246. package/src/actor.ts +101 -71
  247. package/src/ai-agent.ts +162 -111
  248. package/src/ai-control-server.ts +8 -8
  249. package/src/ai-fx.ts +324 -82
  250. package/src/ai-harness.ts +17 -2
  251. package/src/ai-pi.ts +2 -2
  252. package/src/ai-server.ts +2 -2
  253. package/src/ai.ts +7 -7
  254. package/src/client.ts +19 -2
  255. package/src/contract.ts +1 -1
  256. package/src/deterministic-id.ts +27 -4
  257. package/src/errors.ts +8 -5
  258. package/src/idempotent-replay.ts +0 -1
  259. package/src/index.ts +1 -1
  260. package/src/internal.ts +57 -16
  261. package/src/otel.ts +1 -1
  262. package/src/postgres-schema.ts +98 -57
  263. package/src/presence-recovery.ts +18 -11
  264. package/src/presence-tombstones.ts +52 -0
  265. package/src/push-envelope.ts +20 -0
  266. package/src/runtime-consumer.ts +10 -0
  267. package/src/scheduler-task.ts +2 -2
  268. package/src/scheduler-vercel.ts +1 -1
  269. package/src/server.browser.ts +1 -1
  270. package/src/server.ts +819 -218
  271. package/src/session-socket.ts +1 -1
  272. package/src/store-codec.ts +66 -0
  273. package/src/store-memory.ts +604 -286
  274. package/src/store-postgres.ts +2006 -857
  275. package/src/store-presence-polling.ts +59 -34
  276. package/src/store-redis-core.ts +1961 -1333
  277. package/src/store-redis-notify.ts +76 -26
  278. package/src/store-redis.ts +2 -3
  279. package/src/store-sqlite.ts +915 -516
  280. package/src/store.ts +241 -102
  281. package/src/telemetry.ts +2 -1
  282. package/src/wire.ts +1 -0
  283. package/dist/actor-C7b-1arn.d.ts.map +0 -1
  284. package/dist/actor-shared-t7w8xIKP.js.map +0 -1
  285. package/dist/ai-harness-Clbg5ei7.js.map +0 -1
  286. package/dist/ai-mMiWul1T.js.map +0 -1
  287. package/dist/client-BhHLoNKF.js.map +0 -1
  288. package/dist/contract-BYKASlCV.js.map +0 -1
  289. package/dist/idempotent-replay-JdgMQRRm.js +0 -21
  290. package/dist/idempotent-replay-JdgMQRRm.js.map +0 -1
  291. package/dist/initial-snapshot-Bx-c0pDP.d.ts.map +0 -1
  292. package/dist/internal-CscSY7K_.js.map +0 -1
  293. package/dist/server-DYAATFuT.d.ts.map +0 -1
  294. package/dist/server-qyb5aJJG.js.map +0 -1
  295. package/dist/session-clear-C-QwTXV1.d.ts.map +0 -1
  296. package/dist/session-clear-DGI4jTQ5.js.map +0 -1
  297. package/dist/store-CyLUy86v.d.ts +0 -404
  298. package/dist/store-CyLUy86v.d.ts.map +0 -1
  299. package/dist/store-codec-DTG0Ftek.js +0 -10
  300. package/dist/store-codec-DTG0Ftek.js.map +0 -1
  301. package/dist/store-presence-polling-B1ixeqIS.js +0 -94
  302. package/dist/store-presence-polling-B1ixeqIS.js.map +0 -1
  303. package/dist/store-redis-notify-CM0AfLY0.js +0 -2544
  304. package/dist/store-redis-notify-CM0AfLY0.js.map +0 -1
  305. package/dist/stream-buffer-Cx7AAPGq.js +0 -23
  306. package/dist/stream-buffer-Cx7AAPGq.js.map +0 -1
  307. package/dist/telemetry-Cyfqo6Qt.d.ts.map +0 -1
  308. package/dist/wire-BKxH1vbP.js.map +0 -1
  309. package/docs/server/01-handlers.mdx +0 -227
@@ -5,26 +5,60 @@
5
5
  * `store-redis-http` (provider REST APIs).
6
6
  *
7
7
  * The stream IS the store: each event is a stream entry whose ID is its
8
- * dense a2 index (`${index}-0`), so `XRANGE` is the catch-up read.
9
- * Mutable drain/failure bookkeeping lives in side hashes of fixed index
10
- * ranges (stream entries are immutable); every write path is one atomic
11
- * Lua script.
12
- * Global event-id ownership is one key per id, so no key grows with
13
- * the number of sessions.
14
- * The conformance suite in test/conformance is the executable
15
- * contract, run against a real spawned `redis-server`.
8
+ * dense a2 index (`${index}-0`), so `XRANGE` is the catch-up read. Stream
9
+ * entries are immutable; dispatch bookkeeping lives on consumer cursors.
10
+ * Every write path is one atomic Lua script.
16
11
  *
17
- * Timestamps and claim expiry come from the injected clock so tests
18
- * can time-travel against a real server. Cluster mode is out: the
19
- * atomic scripts span keys.
12
+ * Every key of a session is `{p}:{"<session id as JSON>"}:<name>`, one hash
13
+ * tag per session (`redisSessionKey`). Each script declares keys of one
14
+ * session only and derives the rest, so on Upstash a script that sets
15
+ * `allow-key-locking` locks only its session instead of the database. Redis
16
+ * and Valkey reject that flag, so the store asks the server once whether it
17
+ * accepts it.
18
+ *
19
+ * The cursor layout, per session:
20
+ *
21
+ * - `heads:{chunk}` holds one row per (head index, consumer) in hashes of
22
+ * META_CHUNK_SIZE consecutive indexes: a head's attempt and failure
23
+ * fields, its kind (`l`), its lane successor (`next`), its snapshot pins,
24
+ * and, once the cursor moves past it, its completion record. A new head
25
+ * is a new row, so moving a cursor resets its attempt fields for free,
26
+ * and an event cursor costs one row, as a handled event did.
27
+ * - `cursors` is one hash of every lane and log cursor: its head (`h`),
28
+ * position (`p`), lane tail (`t`), or log types (`ty`). A lane's queue is
29
+ * linked through the rows' `next` fields.
30
+ * - `runnable` is a zset of every cursor with a head, scored by the head's
31
+ * index, negated while the head is dead-lettered.
32
+ * - `pins` counts, per snapshot, the heads that pin it.
33
+ *
34
+ * A cursor id names a cursor within its session: the consumer's byte
35
+ * length, `:`, the consumer, then `#{index}` for an event cursor, `L{lane}`
36
+ * for a lane cursor, or `G{lane}` for a log cursor. A clear supersedes every
37
+ * head at or below its marker at once (the `runnable` set is dropped); lane and
38
+ * log cursor records are reset lazily against the marker and swept by
39
+ * reclaim.
40
+ *
41
+ * `{p}:schema` records the layout. A store claims it once, before its first
42
+ * script, and refuses a prefix holding another. Keys of an unrecorded layout
43
+ * have other names, so they are never read.
44
+ *
45
+ * Event ids are unique per session: `id:{event id}` holds the index an id
46
+ * names, and leaves with its session. The conformance suite in
47
+ * test/conformance is the executable contract, run against a real spawned
48
+ * `redis-server`, and over REST behind a shim that applies Upstash's
49
+ * key-locking rules.
50
+ *
51
+ * Timestamps and claim expiry come from the injected clock so tests can
52
+ * time-travel against a real server.
20
53
  */
21
54
 
22
55
  import type { PresencePatch } from './contract.ts'
23
56
  import { A2Error, supersededAttemptError } from './errors.ts'
24
57
  import { CLEAR_EVENT_TYPE, sessionClearedError } from './session-clear.ts'
25
58
  import { idempotentReplay } from './idempotent-replay.ts'
26
- import { nullProtoRecord } from './internal.ts'
27
- import { decodeReturnedEventIds } from './store-codec.ts'
59
+ import { NOTIFY_TIMINGS, nullProtoRecord } from './internal.ts'
60
+ import { retryableLazy } from './retryable-lazy.ts'
61
+ import { decodeCause } from './store-codec.ts'
28
62
  import type { SerializedEvent } from './stream-buffer.ts'
29
63
  import type {
30
64
  A2Store,
@@ -32,8 +66,14 @@ import type {
32
66
  CompleteAttemptResult,
33
67
  Event,
34
68
  EventCause,
69
+ EventTarget,
70
+ HeadDispatch,
71
+ HeadEvent,
72
+ HeadRef,
35
73
  IdSource,
36
74
  StoreClaimAvailableResult,
75
+ StoreConsumer,
76
+ StoredCursor,
37
77
  PresenceRow,
38
78
  StoredEvent,
39
79
  } from './store.ts'
@@ -68,9 +108,9 @@ export type RedisStreamRead = (options: {
68
108
  export type RedisStoreCore = Omit<A2Store, 'stream' | 'presence'> & {
69
109
  /**
70
110
  * The storage half of the presence capability, shared by both
71
- * transports. `set` returns the applied patch for local loopback,
72
- * or null when every field lost the LWW race. The script publishes
73
- * the applied fields for remote subscribers.
111
+ * transports. `set` returns the applied patch for local loopback, with
112
+ * the publish to remote subscribers already under way, or null when
113
+ * every field lost the LWW race.
74
114
  */
75
115
  presence: {
76
116
  set(
@@ -78,10 +118,15 @@ export type RedisStoreCore = Omit<A2Store, 'stream' | 'presence'> & {
78
118
  participant: string,
79
119
  values: Record<string, unknown | null>,
80
120
  meta: { seen: number; at: Date; ttlMs: number },
81
- ): Promise<PresencePatch | null>
121
+ ): Promise<{ patch: PresencePatch; published: Promise<void> } | null>
82
122
  read(ns: string): Promise<PresenceRow[]>
83
123
  }
84
- /** XRANGE + meta merge — the full-row catch-up read. */
124
+ /**
125
+ * The latest clear marker's index, or 0. Entries below it are an old
126
+ * lifetime's until reclaim deletes them, so a feed starts no lower.
127
+ */
128
+ clearIndex(sessionId: string): Promise<number>
129
+ /** XRANGE pages — the full-row catch-up read. */
85
130
  readRange(
86
131
  sessionId: string,
87
132
  afterIndex: number,
@@ -106,7 +151,8 @@ export type RedisStoreCore = Omit<A2Store, 'stream' | 'presence'> & {
106
151
  }) => AsyncGenerator<SerializedEvent[], void>
107
152
  /**
108
153
  * Events from `fromIndex` through `throughIndex` or the newest entry in
109
- * one newest-first page, or null when the range doesn't fit in one.
154
+ * one newest-first page, or null when the range doesn't fit in one or
155
+ * starts below the latest clear.
110
156
  */
111
157
  readTail(
112
158
  sessionId: string,
@@ -119,66 +165,31 @@ export const READ_PAGE_SIZE = 256
119
165
  export const FEED_PAGE_SIZE = 1000
120
166
  const READ_CONCURRENCY = 16
121
167
 
122
- // ── Lua scripts (each runs atomically) ───────────────────────────────
168
+ /** The key layout this core writes; any other is rejected. */
169
+ export const REDIS_LAYOUT = 'tagged-1'
170
+
171
+ /**
172
+ * Upstash runs a script under a database-wide lock unless it opts into
173
+ * locking only its keys' hash tags. Redis and Valkey reject the flag, and
174
+ * have no such lock to opt out of: one thread runs every command.
175
+ */
176
+ const KEY_LOCKING = '#!lua flags=allow-key-locking'
177
+ const READ_ONLY_KEY_LOCKING = '#!lua flags=no-writes,allow-key-locking'
178
+ const SHEBANG_REJECTED = /shebang|unexpected symbol near '#'/i
123
179
 
124
180
  const META_CHUNK_SIZE = 1000
125
181
 
126
- /** The metadata columns of a full stored row. */
127
- const STORED_EVENT_META_SUFFIXES = [
128
- 'p',
129
- 'pa',
130
- 'ri',
131
- 'ch',
132
- 'ce',
133
- 'f',
134
- 'fc',
135
- 'lc',
136
- 'lf',
137
- 'lfa',
138
- 'd',
139
- 'a',
140
- 'e',
141
- ] as const
182
+ /** A head row's dispatch columns, in the order scripts return them. */
183
+ const HEAD_COLUMNS = `{ 'd', 'fc', 'lc', 'ch', 'ce', 'a', 'lf', 'lfa', 'e', 'f' }`
184
+
185
+ // ── Lua scripts (each runs atomically) ───────────────────────────────
142
186
 
143
187
  /**
144
- * Upstash bills every call a script makes, so scripts read the counter and
145
- * split with one MGET, read metadata with one HMGET per hash (`read_fields`),
146
- * and buffer hash writes (`put`) into one HSET per hash (`flush`). Buffered
147
- * writes are invisible to reads until flushed, so a script must not read a
148
- * field it has buffered. Per-event metadata lives in hashes of
149
- * META_CHUNK_SIZE consecutive indexes, so no key grows with a session's
150
- * history. Indexes at or below the split predate chunking and stay in the
151
- * base hash, which also holds every session's lane pointers. An unset split
152
- * means the current event count, so any script that adds events must write
153
- * `split_write` pairs in its write phase. Including scripts bind COUNTER,
154
- * META (the base hash), and SPLIT first.
188
+ * Issues a variadic command in calls of at most 1000 arguments, since Lua's
189
+ * `unpack` is bounded by its stack, and concatenates table replies. An even
190
+ * slice size keeps HSET's field/value pairs together.
155
191
  */
156
- const META_LUA = `
157
- local split, split_stored = nil, false
158
- -- The latest clear marker lives beside the counter, {prefix}:{session}:clear,
159
- -- and is read in the same MGET, so a session never cleared pays nothing.
160
- local CLEAR = string.sub(COUNTER, 1, -6) .. 'clear'
161
- local boundary = 0
162
- local function use_split(stored, count, cleared)
163
- split_stored = stored ~= false
164
- split = tonumber(stored or count or '0')
165
- boundary = tonumber(cleared or '0')
166
- end
167
- local function mkey(field)
168
- if split == nil then
169
- local values = redis.call('MGET', SPLIT, COUNTER, CLEAR)
170
- use_split(values[1], values[2], values[3])
171
- end
172
- local idx = tonumber(field)
173
- if idx <= split then return META end
174
- return META .. ':' .. tostring(math.floor((idx - 1) / ${META_CHUNK_SIZE}))
175
- end
176
- local function split_write(args)
177
- if not split_stored then
178
- args[#args + 1] = SPLIT
179
- args[#args + 1] = tostring(split)
180
- end
181
- end
192
+ const SLICED_LUA = `
182
193
  local function sliced(command, key, args)
183
194
  local results = {}
184
195
  for first = 1, #args, 1000 do
@@ -195,36 +206,109 @@ local function sliced(command, key, args)
195
206
  end
196
207
  return results
197
208
  end
198
- local function read_fields(fields, suffixes)
209
+ `
210
+
211
+ /**
212
+ * Upstash bills every call a script makes, so scripts read the counter,
213
+ * clear marker, typed-type index, and event-id owners with one MGET
214
+ * (`read_bounds`), read rows with one HMGET per chunk
215
+ * (`read_rows`), and buffer hash writes (`put`, `drop`) into one HSET and
216
+ * one HDEL per hash (`flush`). Buffered writes are invisible to reads until
217
+ * flushed, so a script must not read a field it has buffered. Every script
218
+ * binds KEYS[1] to the session's counter and derives the session's other
219
+ * keys, which share its hash tag.
220
+ */
221
+ const BASE_LUA = `${SLICED_LUA}
222
+ local COUNTER = KEYS[1]
223
+ local SESSION = string.sub(COUNTER, 1, -6)
224
+ local LOG, CLEAR = SESSION .. 'log', SESSION .. 'clear'
225
+ local META, READY, CURSORS = SESSION .. 'heads', SESSION .. 'runnable', SESSION .. 'cursors'
226
+ local PIN_COUNTS, TYPED = SESSION .. 'pins', SESSION .. 'typed'
227
+ local CLEAR_TYPE = '${CLEAR_EVENT_TYPE}'
228
+ local total, boundary = 0, 0
229
+ -- The event types typed log cursors read: type → { from, last }. The type's
230
+ -- index holds every event of it above \`from\`, and no event of it lies above
231
+ -- \`last\`.
232
+ local typed, typed_dirty = {}, false
233
+ -- One key per event id holds the index it names in this session. The
234
+ -- \`id:\` namespace keeps a raw id from spelling another session key.
235
+ local function id_key(id) return SESSION .. 'id:' .. id end
236
+ -- Reads the counter, the latest clear marker, the typed types, and the
237
+ -- owners of \`ids\` in one MGET. Returns each id's index, or false.
238
+ local function read_bounds(ids)
239
+ local keys = { COUNTER, CLEAR, TYPED }
240
+ for _, id in ipairs(ids or {}) do keys[#keys + 1] = id_key(id) end
241
+ local values = sliced('MGET', nil, keys)
242
+ total = tonumber(values[1] or values[2] or '0')
243
+ boundary = tonumber(values[2] or '0')
244
+ if values[3] then typed = cjson.decode(values[3]) end
245
+ local owners = {}
246
+ for i = 4, #keys do owners[i - 3] = values[i] end
247
+ return owners
248
+ end
249
+ -- Event-id owners from read_bounds: an id recorded below the latest clear
250
+ -- is free.
251
+ local function owners_of(values)
252
+ local owners = {}
253
+ for i = 1, #values do
254
+ local owner = values[i]
255
+ if owner and tonumber(owner) < boundary then owner = false end
256
+ owners[i] = owner
257
+ end
258
+ return owners
259
+ end
260
+ local function entry_index(id) return tonumber(string.match(id, '^(%d+)')) end
261
+ local function cid_of(consumer, tag)
262
+ return tostring(string.len(consumer)) .. ':' .. consumer .. tag
263
+ end
264
+ local function parse_cid(cid)
265
+ local colon = string.find(cid, ':', 1, true)
266
+ local n = tonumber(string.sub(cid, 1, colon - 1))
267
+ return string.sub(cid, colon + 1, colon + n), string.sub(cid, colon + n + 1)
268
+ end
269
+ -- A cursor's rows: its consumer, then '#' for an event cursor or 'L' .. lane
270
+ -- for a lane or log cursor, as requests name it.
271
+ local function row_of(consumer, tag)
272
+ local kind = string.sub(tag, 1, 1)
273
+ if kind == '#' then return cid_of(consumer, '#') end
274
+ if kind == 'G' then return cid_of(consumer, 'L' .. string.sub(tag, 2)) end
275
+ return cid_of(consumer, tag)
276
+ end
277
+ local function chunk_key(idx)
278
+ return META .. ':' .. tostring(math.floor((idx - 1) / ${META_CHUNK_SIZE}))
279
+ end
280
+ local function row_name(idx, row, suffix)
281
+ return tostring(idx) .. ':' .. row .. ':' .. suffix
282
+ end
283
+ -- Rows of (index, row) refs, aligned with refs; one HMGET per chunk.
284
+ local function read_rows(refs, suffixes)
199
285
  local groups, order = {}, {}
200
- for _, field in ipairs(fields) do
201
- local key = mkey(field)
286
+ for position, ref in ipairs(refs) do
287
+ local key = chunk_key(ref[1])
202
288
  local group = groups[key]
203
289
  if not group then
204
- group = { fields = {}, names = {} }
290
+ group = { positions = {}, names = {} }
205
291
  groups[key] = group
206
292
  order[#order + 1] = key
207
293
  end
208
- group.fields[#group.fields + 1] = field
294
+ group.positions[#group.positions + 1] = position
209
295
  for _, suffix in ipairs(suffixes) do
210
- group.names[#group.names + 1] = field .. ':' .. suffix
296
+ group.names[#group.names + 1] = row_name(ref[1], ref[2], suffix)
211
297
  end
212
298
  end
213
299
  local rows = {}
214
300
  for _, key in ipairs(order) do
215
301
  local group = groups[key]
216
302
  local values = sliced('HMGET', key, group.names)
217
- for position, field in ipairs(group.fields) do
303
+ for n, position in ipairs(group.positions) do
218
304
  local row = {}
219
- for column = 1, #suffixes do
220
- row[column] = values[(position - 1) * #suffixes + column]
221
- end
222
- rows[field] = row
305
+ for column = 1, #suffixes do row[column] = values[(n - 1) * #suffixes + column] end
306
+ rows[position] = row
223
307
  end
224
308
  end
225
309
  return rows
226
310
  end
227
- local writes, write_order = {}, {}
311
+ local writes, write_order, deletes, delete_order = {}, {}, {}, {}
228
312
  local function put(key, field, value)
229
313
  local fields = writes[key]
230
314
  if not fields then
@@ -235,199 +319,410 @@ local function put(key, field, value)
235
319
  fields[#fields + 1] = field
236
320
  fields[#fields + 1] = value
237
321
  end
322
+ local function drop(key, field)
323
+ local fields = deletes[key]
324
+ if not fields then
325
+ fields = {}
326
+ deletes[key] = fields
327
+ delete_order[#delete_order + 1] = key
328
+ end
329
+ fields[#fields + 1] = field
330
+ end
238
331
  local function flush()
239
332
  for _, key in ipairs(write_order) do sliced('HSET', key, writes[key]) end
240
- writes, write_order = {}, {}
333
+ for _, key in ipairs(delete_order) do sliced('HDEL', key, deletes[key]) end
334
+ writes, write_order, deletes, delete_order = {}, {}, {}, {}
335
+ end
336
+ local function put_row(idx, row, suffix, value)
337
+ put(chunk_key(idx), row_name(idx, row, suffix), value)
338
+ end
339
+ local function drop_row(idx, row, suffix)
340
+ drop(chunk_key(idx), row_name(idx, row, suffix))
341
+ end
342
+ -- A request names a cursor by tag: '#' for an event cursor, 'L' .. lane for a
343
+ -- lane or log cursor. A claimed row records its kind: '#', or the cursor's tag.
344
+ local function owns(kind, tag)
345
+ if not kind then return false end
346
+ if tag == '#' then return kind == '#' end
347
+ return kind == tag or kind == 'G' .. string.sub(tag, 2)
348
+ end
349
+ local function cid_for(consumer, kind, idx)
350
+ if kind == '#' then return cid_of(consumer, '#' .. tostring(idx)) end
351
+ return cid_of(consumer, kind)
352
+ end
353
+ -- One type's index, named by the type's bytes in hex so no type can spell
354
+ -- another key of the session.
355
+ local function type_key(name)
356
+ return SESSION .. 'type:' .. (string.gsub(name, '.', function(c)
357
+ return string.format('%02x', string.byte(c))
358
+ end))
359
+ end
360
+ local ready_changes, ready_order = {}, {}
361
+ local function ready_set(cid, score)
362
+ if ready_changes[cid] == nil then ready_order[#ready_order + 1] = cid end
363
+ ready_changes[cid] = score
364
+ end
365
+ local function flush_ready()
366
+ local adds, removes = {}, {}
367
+ for _, cid in ipairs(ready_order) do
368
+ local score = ready_changes[cid]
369
+ if score then
370
+ adds[#adds + 1] = score
371
+ adds[#adds + 1] = cid
372
+ else
373
+ removes[#removes + 1] = cid
374
+ end
375
+ end
376
+ sliced('ZADD', READY, adds)
377
+ sliced('ZREM', READY, removes)
378
+ ready_changes, ready_order = {}, {}
379
+ end
380
+ local function release_pins(idx, row, pins)
381
+ if not pins or pins == '' then return end
382
+ for _, ref in ipairs(cjson.decode(pins)) do
383
+ local count = redis.call('HINCRBY', PIN_COUNTS, ref, -1)
384
+ if count <= 0 then
385
+ redis.call('HDEL', PIN_COUNTS, ref)
386
+ local checkpoint = cjson.decode(ref)
387
+ redis.call('ZREMRANGEBYSCORE', checkpoint[1], checkpoint[2], checkpoint[2])
388
+ end
389
+ end
390
+ put_row(idx, row, 'pins', '')
241
391
  end
242
392
  `
243
393
 
244
394
  /**
245
- * Lane-aware enqueueing for scripts that add events. Including scripts bind
246
- * PENDING and READY. Lane tails are read once per lane and tracked in
247
- * memory, so a batch in one lane links its events without re-reading. A
248
- * lane emptied earlier in the script (`clear_lane`) is known to have no tail,
249
- * and its pointers are deleted at `flush_queue` only if nothing re-filled it.
395
+ * Cursor movement for scripts that add events or move cursors. Lane and log
396
+ * cursor records are read once per script (`load_states`) and written back
397
+ * once (`flush_states`). A record whose head lies at or below the latest clear
398
+ * marker is idle, and a log cursor counts from the marker.
399
+ *
400
+ * A typed log cursor finds its next head in its types' indexes, one bounded
401
+ * lookup per type, skipped when `typed` shows no newer event of it. When an
402
+ * index does not reach back far enough yet (a typed backfill, or a type a
403
+ * cursor newly reads), the lookup returns the type, and the script returns
404
+ * `{ 'index', type, after }` before writing anything: the caller indexes
405
+ * that type down to `after` (INDEX_TYPE_LUA, one page per call) and runs the
406
+ * operation again.
250
407
  */
251
- const ENQUEUE_LUA = `
252
- local function lane_field(lane, suffix)
253
- return '__lane:' .. tostring(string.len(lane)) .. ':' .. lane .. ':' .. suffix
254
- end
255
- local queued, ready_now, lane_tails, lane_order = {}, {}, {}, {}
256
- local function enqueue(field, lane)
257
- local score = tonumber(field)
258
- queued[#queued + 1] = score
259
- queued[#queued + 1] = field
260
- if lane == '' then
261
- ready_now[#ready_now + 1] = score
262
- ready_now[#ready_now + 1] = field
263
- return
408
+ const CURSOR_LUA = `
409
+ local served_all, served_set, served_logs = true, {}, {}
410
+ local pending_fast = false
411
+ local function types_of(raw)
412
+ if not raw or raw == '*' then return nil end
413
+ local set = {}
414
+ for _, name in ipairs(cjson.decode(raw)) do set[name] = true end
415
+ return set
416
+ end
417
+ local function takes(types, kind)
418
+ return kind ~= CLEAR_TYPE and (types == nil or types[kind] == true)
419
+ end
420
+ -- The served consumers: '' for every consumer, else [[name] or
421
+ -- [name, lane, typesJson or '*', backfill]].
422
+ local function use_served(raw)
423
+ if raw == '' then return end
424
+ served_all = false
425
+ for _, entry in ipairs(cjson.decode(raw)) do
426
+ served_set[entry[1]] = true
427
+ if entry[2] then
428
+ served_logs[#served_logs + 1] = {
429
+ name = entry[1], lane = entry[2], ty = entry[3],
430
+ types = types_of(entry[3]), backfill = entry[4] == 1,
431
+ }
432
+ end
264
433
  end
265
- put(mkey(field), field .. ':l', lane)
266
- local tail = lane_tails[lane]
267
- if tail == nil then
268
- tail = redis.call('HGET', META, lane_field(lane, 'tail'))
269
- if tail and tonumber(tail) <= boundary then tail = false end
270
- lane_order[#lane_order + 1] = lane
434
+ end
435
+ local function served_has(name) return served_all or served_set[name] == true end
436
+ -- Whether a served consumer has a live head. Newest heads come first: a
437
+ -- removed consumer's backlog is older than the work a deployment serves.
438
+ local function has_pending()
439
+ if pending_fast then return '1' end
440
+ if served_all then return redis.call('ZCOUNT', READY, 1, '+inf') > 0 and '1' or '0' end
441
+ if next(served_set) == nil then return '0' end
442
+ local page = redis.call('ZREVRANGEBYSCORE', READY, '+inf', 1, 'LIMIT', 0, 32)
443
+ for _, cid in ipairs(page) do
444
+ if served_set[(parse_cid(cid))] then return '1' end
271
445
  end
272
- if tail then
273
- put(mkey(tail), tail .. ':next', field)
274
- else
275
- put(META, lane_field(lane, 'head'), field)
276
- ready_now[#ready_now + 1] = score
277
- ready_now[#ready_now + 1] = field
446
+ if #page < 32 then return '0' end
447
+ for _, cid in ipairs(redis.call('ZREVRANGEBYSCORE', READY, '+inf', 1, 'LIMIT', 32, -1)) do
448
+ if served_set[(parse_cid(cid))] then return '1' end
278
449
  end
279
- lane_tails[lane] = field
280
- end
281
- local function clear_lane(lane)
282
- if lane_tails[lane] == nil then lane_order[#lane_order + 1] = lane end
283
- lane_tails[lane] = false
284
- end
285
- local function flush_queue()
286
- for _, lane in ipairs(lane_order) do
287
- if lane_tails[lane] then
288
- put(META, lane_field(lane, 'tail'), lane_tails[lane])
450
+ return '0'
451
+ end
452
+ local function targets_of(raw)
453
+ if raw == '' then return {} end
454
+ return cjson.decode(raw)
455
+ end
456
+ -- Starts indexing a type: every later event of it.
457
+ local function register(name, from)
458
+ typed[name] = { from, from }
459
+ typed_dirty = true
460
+ end
461
+ -- The first stored event in (after, total] of a type in \`types\` (nil: any
462
+ -- type), or false. Nil and a type when that type's index can't answer yet.
463
+ local function stored_head(after, types)
464
+ if types == nil then
465
+ if after + 1 <= total then return after + 1 end
466
+ return false
467
+ end
468
+ local best = false
469
+ for name in pairs(types) do
470
+ local entry = typed[name]
471
+ if entry == nil then
472
+ if after < total then return nil, name end
473
+ register(name, total)
474
+ elseif entry[2] > after then
475
+ if entry[1] > after then return nil, name end
476
+ local found = redis.call('ZRANGEBYSCORE', type_key(name),
477
+ '(' .. tostring(after), tostring(total), 'LIMIT', 0, 1)
478
+ if #found > 0 then
479
+ local idx = tonumber(found[1])
480
+ if not best or idx < best then best = idx end
481
+ end
482
+ end
483
+ end
484
+ return best
485
+ end
486
+ -- The first event after \`after\` a log cursor takes: stored, then in the
487
+ -- batch being appended.
488
+ local function next_log_head(after, types, batch)
489
+ local head, missing = stored_head(after, types)
490
+ if head == nil then return nil, missing end
491
+ if head then return head end
492
+ for _, event in ipairs(batch) do
493
+ if event.idx > after and takes(types, event.type) then return event.idx end
494
+ end
495
+ return false
496
+ end
497
+ -- Indexes a batch's events of the types typed log cursors read.
498
+ local function index_batch(batch)
499
+ local adds, order = {}, {}
500
+ for _, event in ipairs(batch) do
501
+ local entry = typed[event.type]
502
+ if entry then
503
+ local list = adds[event.type]
504
+ if not list then
505
+ list = {}
506
+ adds[event.type] = list
507
+ order[#order + 1] = event.type
508
+ end
509
+ list[#list + 1] = event.idx
510
+ list[#list + 1] = tostring(event.idx)
511
+ entry[2] = event.idx
512
+ typed_dirty = true
513
+ end
514
+ end
515
+ for _, name in ipairs(order) do sliced('ZADD', type_key(name), adds[name]) end
516
+ end
517
+ local states, state_order = {}, {}
518
+ local function load_states(cids)
519
+ local need, names = {}, {}
520
+ for _, cid in ipairs(cids) do
521
+ local state = states[cid]
522
+ if (state == nil or state.partial) and not need[cid] then
523
+ need[cid] = true
524
+ need[#need + 1] = cid
525
+ names[#names + 1] = cid .. ':h'
526
+ names[#names + 1] = cid .. ':p'
527
+ names[#names + 1] = cid .. ':t'
528
+ names[#names + 1] = cid .. ':ty'
529
+ end
530
+ end
531
+ if #need == 0 then return end
532
+ local values = sliced('HMGET', CURSORS, names)
533
+ for n, cid in ipairs(need) do
534
+ local h, p, t, ty = values[n * 4 - 3], values[n * 4 - 2], values[n * 4 - 1], values[n * 4]
535
+ local tail = (t and t ~= '') and tonumber(t) or false
536
+ local state = states[cid]
537
+ if state then
538
+ state.t = tail
539
+ state.partial = nil
289
540
  else
290
- redis.call('HDEL', META, lane_field(lane, 'head'), lane_field(lane, 'tail'))
541
+ local _, tag = parse_cid(cid)
542
+ state = {
543
+ kind = string.sub(tag, 1, 1), exists = p ~= false,
544
+ h = (h and h ~= '') and tonumber(h) or false,
545
+ p = tonumber(p or '0'), t = tail, ty = ty or false,
546
+ }
547
+ if state.h and state.h <= boundary then state.h = false end
548
+ if state.kind == 'G' and state.exists and state.p < boundary then state.p = boundary end
549
+ states[cid] = state
550
+ state_order[#state_order + 1] = cid
291
551
  end
292
552
  end
293
- sliced('ZADD', PENDING, queued)
294
- sliced('ZADD', READY, ready_now)
295
553
  end
296
- `
297
-
298
- /**
299
- * Settles one unfinished event for scripts that include ENQUEUE_LUA: it
300
- * leaves the pending and ready sets, and a lane head hands its lane to the
301
- * next unfinished event. A dead-lettered successor stays out of the ready set.
302
- */
303
- const SETTLE_LUA = `
304
- local function settle(field, lane, next_field)
305
- redis.call('ZREM', PENDING, field)
306
- redis.call('ZREM', READY, field)
307
- if lane == '' then return end
308
- local head_key = lane_field(lane, 'head')
309
- if redis.call('HGET', META, head_key) ~= field then return end
310
- while next_field do
311
- local row = read_fields({ next_field }, { 'p', 'f', 'next' })[next_field]
312
- if not row[1] then
313
- redis.call('HSET', META, head_key, next_field)
314
- if not row[2] then
315
- redis.call('ZADD', READY, tonumber(next_field), next_field)
554
+ local function flush_states()
555
+ for _, cid in ipairs(state_order) do
556
+ local state = states[cid]
557
+ if state.dirty then
558
+ put(CURSORS, cid .. ':h', state.h and tostring(state.h) or '')
559
+ put(CURSORS, cid .. ':p', tostring(state.p))
560
+ if state.kind == 'L' then
561
+ if not state.partial then put(CURSORS, cid .. ':t', state.t and tostring(state.t) or '') end
562
+ elseif state.ty then
563
+ put(CURSORS, cid .. ':ty', state.ty)
316
564
  end
317
- return
318
565
  end
319
- next_field = row[3]
320
566
  end
321
- clear_lane(lane)
322
567
  end
323
- `
324
-
325
- /**
326
- * Event-id ownership for scripts that add events. Each script reads the
327
- * counter, split, clear boundary, and per-id keys with one MGET and applies
328
- * the first three. Returns the owner record, or false, per event, and the
329
- * counter. A record this session made below its latest clear is free.
330
- */
331
- const ID_OWNERS_LUA = `
332
- local function read_owners(key_offset, n, session)
333
- local keys = { COUNTER, SPLIT, CLEAR }
334
- for i = 1, n do keys[#keys + 1] = KEYS[key_offset + i] end
335
- local values = sliced('MGET', nil, keys)
336
- use_split(values[2], values[1], values[3])
337
- local owners = {}
338
- for i = 1, n do
339
- local owner = values[3 + i]
340
- if owner then
341
- local rec = cjson.decode(owner)
342
- if rec[1] == session and tonumber(rec[2]) < boundary then owner = false end
568
+ -- Moves a cursor past its head \`idx\`. \`kind\` is the head row's kind;
569
+ -- \`batch\` is the returned events, appended after the stored log. Writes
570
+ -- nothing; returns nil and a type when that type's index must reach back
571
+ -- to \`idx\` first.
572
+ local function advance(consumer, kind, idx, row_next, batch)
573
+ if kind == '#' then
574
+ ready_set(cid_of(consumer, '#' .. tostring(idx)), false)
575
+ return true
576
+ end
577
+ local cid = cid_of(consumer, kind)
578
+ local state = states[cid]
579
+ local next_head
580
+ if string.sub(kind, 1, 1) == 'L' then
581
+ next_head = row_next and tonumber(row_next) or false
582
+ if not state then
583
+ state = { kind = 'L', exists = true, p = idx, h = false, t = false, ty = false, partial = true }
584
+ states[cid] = state
585
+ state_order[#state_order + 1] = cid
343
586
  end
344
- owners[i] = owner
587
+ if not next_head then
588
+ state.t = false
589
+ state.partial = nil
590
+ end
591
+ else
592
+ local lane, raw = string.sub(kind, 2), nil
593
+ for _, spec in ipairs(served_logs) do
594
+ if spec.name == consumer and spec.lane == lane then raw = spec.ty end
595
+ end
596
+ if not raw then
597
+ load_states({ cid })
598
+ raw = states[cid].ty or '*'
599
+ end
600
+ local head, missing = next_log_head(idx, types_of(raw), batch)
601
+ if head == nil then return nil, missing end
602
+ next_head = head
603
+ state = states[cid]
604
+ if not state then
605
+ state = { kind = 'G', exists = true, p = idx, h = false, t = false, ty = raw }
606
+ states[cid] = state
607
+ state_order[#state_order + 1] = cid
608
+ end
609
+ state.ty = raw
345
610
  end
346
- return owners, values[1]
611
+ state.h = next_head
612
+ state.p = idx
613
+ state.exists = true
614
+ state.dirty = true
615
+ ready_set(cid, next_head)
616
+ return true
347
617
  end
348
- `
349
-
350
- const WAKE_PAYLOAD_LIMIT = 32_768
351
-
352
- /**
353
- * Wake-up messages carry the events they announce, so a subscribed reader
354
- * that is caught up applies them without reading the stream:
355
- * `{"f":frontier,"t":createdAt,"e":[[index,id,type,payload],...]}`. Payloads
356
- * are the stored JSON text, inlined verbatim. A batch over
357
- * WAKE_PAYLOAD_LIMIT bytes publishes `{"f":frontier}` and readers catch up
358
- * with XRANGE.
359
- */
360
- const WAKE_LUA = `
361
- local wake_entries, wake_bytes = {}, 0
362
- local function wake_add(field, id, typ, payload)
363
- if wake_bytes + #payload > ${WAKE_PAYLOAD_LIMIT} then
364
- wake_bytes = ${WAKE_PAYLOAD_LIMIT} + 1
365
- return
618
+ -- Reads what an appended batch's cursor work needs: its lane targets'
619
+ -- records, and the head of each served log consumer whose types it carries.
620
+ -- Writes nothing; returns nil, a type, and an index when that type's index
621
+ -- must reach back to the index first.
622
+ local log_plan = {}
623
+ local function plan(batch, base)
624
+ local need = {}
625
+ for _, event in ipairs(batch) do
626
+ for _, target in ipairs(event.targets) do
627
+ if target[2] ~= '#' then need[#need + 1] = cid_of(target[1], target[2]) end
628
+ end
366
629
  end
367
- local entry = '[' .. field .. ',' .. cjson.encode(id) .. ',' ..
368
- cjson.encode(typ) .. ',' .. payload .. ']'
369
- wake_bytes = wake_bytes + #entry + 1
370
- if wake_bytes <= ${WAKE_PAYLOAD_LIMIT} then
371
- wake_entries[#wake_entries + 1] = entry
630
+ for _, spec in ipairs(served_logs) do
631
+ for _, event in ipairs(batch) do
632
+ if takes(spec.types, event.type) then
633
+ local cid = cid_of(spec.name, 'G' .. spec.lane)
634
+ log_plan[#log_plan + 1] = { spec = spec, first = event.idx, cid = cid }
635
+ need[#need + 1] = cid
636
+ break
637
+ end
638
+ end
372
639
  end
373
- end
374
- local function wake(channel, frontier, created_at)
375
- local message = '{"f":' .. frontier .. '}'
376
- if wake_bytes <= ${WAKE_PAYLOAD_LIMIT} then
377
- message = '{"f":' .. frontier .. ',"t":' .. created_at .. ',"e":[' ..
378
- table.concat(wake_entries, ',') .. ']}'
640
+ load_states(need)
641
+ for _, log in ipairs(log_plan) do
642
+ local state, spec = states[log.cid], log.spec
643
+ if not state.exists then
644
+ if spec.backfill then
645
+ local head, missing = stored_head(boundary, spec.types)
646
+ if head == nil then return nil, missing, boundary end
647
+ log.p, log.h = boundary, head or log.first
648
+ else
649
+ -- Nothing it takes is stored after its start: index its types from there.
650
+ for name in pairs(spec.types or {}) do
651
+ if typed[name] == nil then register(name, base) end
652
+ end
653
+ log.p, log.h = base, log.first
654
+ end
655
+ elseif not state.h then
656
+ -- Not the batch's first: a deployment that doesn't serve this consumer
657
+ -- may have appended events it takes since it went idle.
658
+ local head, missing = stored_head(state.p, spec.types)
659
+ if head == nil then return nil, missing, state.p end
660
+ log.h = head or log.first
661
+ end
379
662
  end
380
- redis.pcall('PUBLISH', channel, message)
663
+ return true
381
664
  end
382
- `
383
-
384
- const RELEASE_PINS_LUA = `
385
- local function release_snapshot_pins(pins, counts)
386
- local refs = redis.call('SMEMBERS', pins)
387
- if #refs == 0 then return end
388
- for _, ref in ipairs(refs) do
389
- local count = redis.call('HINCRBY', counts, ref, -1)
390
- if count <= 0 then
391
- redis.call('HDEL', counts, ref)
392
- local checkpoint = cjson.decode(ref)
393
- redis.call('ZREMRANGEBYSCORE', checkpoint[1], checkpoint[2], checkpoint[2])
665
+ -- Starts the cursor work \`plan\` read: each event's targets, and each
666
+ -- planned log cursor.
667
+ local function enqueue(batch)
668
+ for _, event in ipairs(batch) do
669
+ for _, target in ipairs(event.targets) do
670
+ local consumer, tag = target[1], target[2]
671
+ if served_has(consumer) then pending_fast = true end
672
+ if tag == '#' then
673
+ ready_set(cid_of(consumer, '#' .. tostring(event.idx)), event.idx)
674
+ else
675
+ local cid = cid_of(consumer, tag)
676
+ local state = states[cid]
677
+ if state.h and state.t then
678
+ put_row(state.t, row_of(consumer, tag), 'next', tostring(event.idx))
679
+ else
680
+ state.h = event.idx
681
+ if state.p < boundary then state.p = boundary end
682
+ ready_set(cid, event.idx)
683
+ end
684
+ state.t = event.idx
685
+ state.exists = true
686
+ state.dirty = true
687
+ end
394
688
  end
395
689
  end
396
- redis.call('DEL', pins)
690
+ for _, log in ipairs(log_plan) do
691
+ pending_fast = true
692
+ local state = states[log.cid]
693
+ if not state.exists then
694
+ state.exists, state.p, state.h = true, log.p, log.h
695
+ ready_set(log.cid, log.h)
696
+ elseif not state.h then
697
+ state.h = log.h
698
+ ready_set(log.cid, log.h)
699
+ end
700
+ state.ty = log.spec.ty
701
+ state.dirty = true
702
+ end
397
703
  end
398
704
  `
399
705
 
400
706
  /**
401
- * KEYS: store stream, counter, metadata, pending, ready, metadata split,
402
- * then one id ownership key per event.
403
- * ARGV: sessionId, nowMs, n, then per event:
404
- * id, type, payloadJson, causeJson, lane, settled; then notify channel and
405
- * origin index or ''.
707
+ * KEYS: counter.
708
+ * ARGV: sessionId, nowMs, n, served consumers, origin index or '', then per
709
+ * event: id, type, payloadJson, causeJson or '', targets.
406
710
  * Returns ['empty', hasPending] | ['ok', firstIndex, hasPending] |
407
711
  * ['dup', hasPending, ...indexes] | ['partial', count] |
408
- * ['foreign', eventId] | ['no_parent', idx] | ['superseded', idx] |
409
- * ['cleared', originIndex].
712
+ * ['superseded', position] | ['cleared', originIndex] |
713
+ * ['index', type, after].
410
714
  */
411
- const APPEND_LUA = `
412
- local COUNTER, META, SPLIT, PENDING, READY = KEYS[2], KEYS[3], KEYS[6], KEYS[4], KEYS[5]
413
- ${META_LUA}
414
- ${ENQUEUE_LUA}
415
- ${ID_OWNERS_LUA}
416
- ${WAKE_LUA}
417
- local function has_pending()
418
- return redis.call('ZCARD', PENDING) > 0 and '1' or '0'
419
- end
420
- local n = tonumber(ARGV[3])
715
+ const APPEND_LUA = `${BASE_LUA}${CURSOR_LUA}
716
+ local now, n = ARGV[2], tonumber(ARGV[3])
717
+ use_served(ARGV[4])
421
718
  if n == 0 then return { 'empty', has_pending() } end
422
- local owners, count = read_owners(6, n, ARGV[1])
423
- local base = tonumber(count or '0')
719
+ local ids = {}
720
+ for i = 1, n do ids[i] = ARGV[5 + (i - 1) * 5 + 1] end
721
+ local owners = owners_of(read_bounds(ids))
722
+ local base = total
424
723
  local dups = {}
425
724
  for i = 1, n do
426
- if owners[i] then
427
- local rec = cjson.decode(owners[i])
428
- if rec[1] ~= ARGV[1] then return { 'foreign', ARGV[3 + (i - 1) * 6 + 1] } end
429
- dups[#dups + 1] = tostring(rec[2])
430
- end
725
+ if owners[i] then dups[#dups + 1] = owners[i] end
431
726
  end
432
727
  if #dups == n then
433
728
  local res = { 'dup', has_pending() }
@@ -435,253 +730,251 @@ if #dups == n then
435
730
  return res
436
731
  end
437
732
  if #dups > 0 then return { 'partial', tostring(#dups) } end
438
- local origin = ARGV[5 + n * 6]
439
- if origin and origin ~= '' and tonumber(origin) <= boundary then
440
- return { 'cleared', origin }
733
+ local origin = ARGV[5]
734
+ if origin ~= '' and tonumber(origin) <= boundary then return { 'cleared', origin } end
735
+ -- The attempt-currency fence: a cause's head row must still be its cursor's
736
+ -- head, at that attempt, not dead-lettered.
737
+ local refs, fenced = {}, {}
738
+ for i = 1, n do
739
+ local raw = ARGV[5 + (i - 1) * 5 + 4]
740
+ if raw ~= '' then
741
+ local cause = cjson.decode(raw)
742
+ local index = tonumber(cause.index)
743
+ if index <= boundary then return { 'superseded', tostring(i) } end
744
+ local tag = type(cause.lane) == 'string' and ('L' .. cause.lane) or '#'
745
+ refs[#refs + 1] = { index, row_of(cause.consumer, tag) }
746
+ fenced[#fenced + 1] = {
747
+ position = i, consumer = cause.consumer, attempt = tonumber(cause.attempt), tag = tag,
748
+ }
749
+ end
441
750
  end
442
- local causes, parents = {}, {}
751
+ local rows = read_rows(refs, { 'd', 'f', 'p', 'l' })
752
+ for k, cause in ipairs(fenced) do
753
+ local row = rows[k]
754
+ if tonumber(row[1] or '0') ~= cause.attempt or row[2] or row[3] or not owns(row[4], cause.tag) then
755
+ return { 'superseded', tostring(cause.position) }
756
+ end
757
+ if served_has(cause.consumer) then pending_fast = true end
758
+ end
759
+ local batch = {}
443
760
  for i = 1, n do
444
- local cause = ARGV[3 + (i - 1) * 6 + 4]
445
- if cause ~= '' then
446
- local parsed = cjson.decode(cause)
447
- local pidx = tonumber(parsed.index)
448
- causes[#causes + 1] = parsed
449
- if pidx >= 1 and pidx <= base then parents[#parents + 1] = tostring(pidx) end
450
- end
451
- end
452
- local parent_rows = read_fields(parents, { 'd', 'f', 'p' })
453
- local parent_pending = false
454
- for _, parsed in ipairs(causes) do
455
- local pidx = tonumber(parsed.index)
456
- if pidx <= boundary then return { 'superseded', tostring(pidx) } end
457
- if pidx < 1 or pidx > base then
458
- return { 'no_parent', tostring(parsed.index) }
459
- end
460
- local row = parent_rows[tostring(pidx)]
461
- if tonumber(row[1] or '0') ~= tonumber(parsed.attempt) or row[2] then
462
- return { 'superseded', tostring(pidx) }
463
- end
464
- -- A claimed, unprocessed parent is still in the pending set.
465
- if not row[3] then parent_pending = true end
466
- end
467
- local owner_writes = {}
761
+ local o = 5 + (i - 1) * 5
762
+ batch[i] = { idx = base + i, type = ARGV[o + 2], targets = targets_of(ARGV[o + 5]) }
763
+ end
764
+ local planned, missing, after = plan(batch, base)
765
+ if not planned then return { 'index', missing, tostring(after) } end
766
+ local marks = {}
468
767
  for i = 1, n do
469
- local o = 3 + (i - 1) * 6
768
+ local o = 5 + (i - 1) * 5
470
769
  local field = tostring(base + i)
471
- redis.call('XADD', KEYS[1], field .. '-0',
770
+ redis.call('XADD', LOG, field .. '-0',
472
771
  'id', ARGV[o + 1], 'type', ARGV[o + 2], 'payload', ARGV[o + 3],
473
- 'created_at', ARGV[2], 'cause', ARGV[o + 4], 'lane', ARGV[o + 5])
474
- owner_writes[#owner_writes + 1] = KEYS[6 + i]
475
- owner_writes[#owner_writes + 1] = cjson.encode({ ARGV[1], base + i })
476
- wake_add(field, ARGV[o + 1], ARGV[o + 2], ARGV[o + 3])
477
- if ARGV[o + 6] == '1' then
478
- put(mkey(field), field .. ':p', ARGV[2])
479
- else
480
- enqueue(field, ARGV[o + 5])
481
- end
482
- end
483
- owner_writes[#owner_writes + 1] = COUNTER
484
- owner_writes[#owner_writes + 1] = tostring(base + n)
485
- split_write(owner_writes)
486
- sliced('MSET', nil, owner_writes)
487
- flush_queue()
772
+ 'created_at', now, 'cause', ARGV[o + 4])
773
+ marks[#marks + 1] = id_key(ARGV[o + 1])
774
+ marks[#marks + 1] = field
775
+ end
776
+ index_batch(batch)
777
+ enqueue(batch)
778
+ marks[#marks + 1] = COUNTER
779
+ marks[#marks + 1] = tostring(base + n)
780
+ if typed_dirty then
781
+ marks[#marks + 1] = TYPED
782
+ marks[#marks + 1] = cjson.encode(typed)
783
+ end
784
+ sliced('MSET', nil, marks)
785
+ flush_states()
488
786
  flush()
489
- local pending_after = (#queued > 0 or parent_pending) and '1' or has_pending()
490
- wake(ARGV[4 + n * 6], tostring(base + n), ARGV[2])
491
- return { 'ok', tostring(base + 1), pending_after }
787
+ flush_ready()
788
+ return { 'ok', tostring(base + 1), has_pending() }
492
789
  `
493
790
 
494
791
  /**
495
- * KEYS: counter, meta, pending, ready, event snapshot pins, snapshot pin counts,
496
- * metadata split.
497
- * ARGV: idx, attempt, error, maxFailures, nowMs, retryAtMs ('' for none).
792
+ * KEYS: counter.
793
+ * ARGV: type, after. Indexes one page of the stored log below the type's
794
+ * index, toward `after`, registering the type first if no cursor read it
795
+ * yet. Returns ['1'] once the index reaches `after`, else ['0'].
498
796
  */
499
- const FAIL_ATTEMPT_LUA = `
500
- local COUNTER, META, SPLIT = KEYS[1], KEYS[2], KEYS[7]
501
- ${META_LUA}
502
- ${RELEASE_PINS_LUA}
503
- local idx = tonumber(ARGV[1])
504
- local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR)
505
- local total = tonumber(counts[1] or '0')
506
- use_split(counts[2], counts[1], counts[3])
507
- if idx <= boundary then return { 'cleared', '0' } end
508
- if idx < 1 or idx > total then return { 'missing', '0' } end
509
- local field = ARGV[1]
510
- local attempt = tonumber(ARGV[2])
511
- local row = read_fields({ field }, { 'a', 'd', 'p', 'f', 'lfa', 'ch' })[field]
512
- local failures = tonumber(row[1] or '0')
513
- if row[3] or tonumber(row[2] or '0') ~= attempt then
514
- return { 'superseded', tostring(failures) }
515
- end
516
- if row[4] then return { 'dead_lettered', tostring(failures) } end
517
- if row[5] and tonumber(row[5]) == attempt and not row[6] then
518
- return { 'failed', tostring(failures) }
519
- end
520
- if not row[6] then return { 'superseded', tostring(failures) } end
521
- failures = failures + 1
522
- local dead = failures >= tonumber(ARGV[4])
523
- local meta_key = mkey(field)
524
- put(meta_key, field .. ':a', tostring(failures))
525
- put(meta_key, field .. ':e', ARGV[3])
526
- put(meta_key, field .. ':lf', ARGV[5])
527
- put(meta_key, field .. ':lfa', ARGV[2])
528
- if dead then put(meta_key, field .. ':f', ARGV[5]) end
529
- local hold = not dead and ARGV[6] ~= ''
530
- if hold then put(meta_key, field .. ':ce', ARGV[6]) end
531
- flush()
532
- if hold then
533
- redis.call('HDEL', meta_key, field .. ':ch')
534
- else
535
- redis.call('HDEL', meta_key, field .. ':ch', field .. ':ce')
536
- end
537
- if dead then
538
- redis.call('ZREM', KEYS[4], field)
539
- release_snapshot_pins(KEYS[5], KEYS[6])
540
- return { 'dead_lettered', tostring(failures) }
797
+ const INDEX_TYPE_LUA = `${BASE_LUA}
798
+ read_bounds()
799
+ local name, after = ARGV[1], tonumber(ARGV[2])
800
+ local entry = typed[name]
801
+ if not entry then
802
+ entry = { total, total }
803
+ typed[name] = entry
804
+ end
805
+ -- Entries below the latest clear belong to an old lifetime: never indexed.
806
+ local floor = math.max(after, boundary - 1)
807
+ if entry[1] > floor then
808
+ local entries = redis.call('XREVRANGE', LOG, tostring(entry[1]) .. '-0',
809
+ tostring(floor + 1) .. '-0', 'COUNT', ${READ_PAGE_SIZE})
810
+ local adds = {}
811
+ for _, stored in ipairs(entries) do
812
+ local fields = stored[2]
813
+ for j = 1, #fields, 2 do
814
+ if fields[j] == 'type' then
815
+ if fields[j + 1] == name then
816
+ local idx = entry_index(stored[1])
817
+ adds[#adds + 1] = idx
818
+ adds[#adds + 1] = tostring(idx)
819
+ end
820
+ break
821
+ end
822
+ end
823
+ end
824
+ sliced('ZADD', type_key(name), adds)
825
+ if #entries < ${READ_PAGE_SIZE} then
826
+ entry[1] = after
827
+ else
828
+ entry[1] = entry_index(entries[#entries][1]) - 1
829
+ end
830
+ elseif entry[1] > after then
831
+ entry[1] = after
541
832
  end
542
- return { 'failed', tostring(failures) }
833
+ redis.call('SET', TYPED, cjson.encode(typed))
834
+ return { entry[1] <= after and '1' or '0' }
543
835
  `
544
836
 
545
837
  /**
546
- * KEYS: store, counter, meta, pending, ready, metadata split, snapshot pin
547
- * counts.
548
- * ARGV: holder, nowMs, expiresAtMs, excludedCount, excluded indexes, then
549
- * maxFailures ('' for none) and the event snapshot-pins key prefix.
550
- * The ready set is a subset of the pending set, so an empty ready set is
551
- * enough to report 'settled'. A claimed result ends with the clear boundary.
838
+ * KEYS: counter.
839
+ * ARGV: holder, nowMs, expiresAtMs, maxFailures or '', served consumers,
840
+ * the highest head to claim or '', excluded count, then per excluded head:
841
+ * consumer, tag.
842
+ * Live heads are the ready set's positive scores, so an empty one is enough
843
+ * to report 'settled'. A claimed result ends with the clear marker.
552
844
  */
553
- const CLAIM_AVAILABLE_LUA = `
554
- local COUNTER, META, SPLIT = KEYS[2], KEYS[3], KEYS[6]
555
- ${META_LUA}
556
- ${RELEASE_PINS_LUA}
557
- local ready = redis.call('ZRANGE', KEYS[5], 0, -1)
845
+ const CLAIM_AVAILABLE_LUA = `${BASE_LUA}${CURSOR_LUA}
846
+ local ready = redis.call('ZRANGEBYSCORE', READY, 1, ARGV[6] == '' and '+inf' or ARGV[6], 'WITHSCORES')
847
+ if #ready == 0 then return { 'settled' } end
848
+ read_bounds()
849
+ local holder, now, expires_at = ARGV[1], tonumber(ARGV[2]), ARGV[3]
850
+ local max_failures = tonumber(ARGV[4])
851
+ use_served(ARGV[5])
558
852
  local excluded = {}
559
- local tail = 5 + tonumber(ARGV[4])
560
- for i = 1, tonumber(ARGV[4]) do excluded[ARGV[4 + i]] = true end
561
- local max_failures = tonumber(ARGV[tail])
853
+ for i = 1, tonumber(ARGV[7]) do
854
+ excluded[ARGV[7 + (i - 1) * 2 + 1] .. '\\0' .. ARGV[7 + (i - 1) * 2 + 2]] = true
855
+ end
562
856
  local candidates = {}
563
- for _, field in ipairs(ready) do
564
- if not excluded[field] then candidates[#candidates + 1] = field end
565
- end
566
- local rows = read_fields(candidates, { 'ce', 'd', 'fc', 'a', 'e', 'lf', 'lfa' })
567
- local claimed = {}
568
- local dead = {}
569
- local retry_at = nil
570
- for _, field in ipairs(candidates) do
571
- local row = rows[field]
572
- local expires = tonumber(row[1] or '0')
573
- if expires > tonumber(ARGV[2]) then
857
+ for i = 1, #ready, 2 do
858
+ local cid, idx = ready[i], tonumber(ready[i + 1])
859
+ local consumer, tag = parse_cid(cid)
860
+ local kind = string.sub(tag, 1, 1)
861
+ local key = consumer .. '\\0' .. (kind == 'G' and ('L' .. string.sub(tag, 2)) or tag)
862
+ if served_has(consumer) and not excluded[key] then
863
+ candidates[#candidates + 1] = {
864
+ idx, row_of(consumer, tag), consumer = consumer, cid = cid,
865
+ kind = kind == '#' and '#' or tag,
866
+ }
867
+ end
868
+ end
869
+ local rows = read_rows(candidates, { 'd', 'fc', 'lc', 'ch', 'ce', 'a', 'lf', 'lfa', 'e', 'f', 'pins' })
870
+ local takes_over, retry_at, wanted = {}, nil, {}
871
+ for position, candidate in ipairs(candidates) do
872
+ local row = rows[position]
873
+ local expires = tonumber(row[5] or '0')
874
+ if expires > now then
574
875
  if not retry_at or expires < retry_at then retry_at = expires end
575
876
  else
576
- local entries = redis.call('XRANGE', KEYS[1], field .. '-0', field .. '-0')
577
- if #entries == 0 then return { 'missing', field } end
578
- local meta_key = mkey(field)
579
- local attempt = tostring(tonumber(row[2] or '0') + 1)
580
- -- An expiry that is not the current attempt's retry hold is a lapsed
581
- -- claim: its attempt never finished.
582
- local lapsed = row[1] and row[7] ~= row[2]
583
- local failures = tonumber(row[4] or '0')
584
- if lapsed then
585
- failures = failures + 1
586
- put(meta_key, field .. ':a', tostring(failures))
587
- end
588
- put(meta_key, field .. ':d', attempt)
589
- if lapsed and max_failures and failures >= max_failures then
590
- -- That takeover spent the last failed attempt: dead-letter, unclaimed.
591
- -- Advancing the attempt fences the lapsed one; nothing runs.
592
- put(meta_key, field .. ':f', ARGV[2])
593
- dead[#dead + 1] = field
594
- claimed[#claimed + 1] = {
595
- attempt,
596
- tostring(failures),
597
- row[5] or false,
598
- row[3] or false,
599
- read_fields({ field }, { 'lc' })[field][1] or false,
600
- row[6] or false,
601
- row[7] or false,
602
- false,
603
- false,
604
- entries[1],
605
- ARGV[2]
606
- }
607
- else
608
- local first_claimed = row[3] or ARGV[2]
609
- put(meta_key, field .. ':fc', first_claimed)
610
- put(meta_key, field .. ':lc', ARGV[2])
611
- put(meta_key, field .. ':ch', ARGV[1])
612
- put(meta_key, field .. ':ce', ARGV[3])
613
- claimed[#claimed + 1] = {
614
- attempt,
615
- tostring(failures),
616
- row[5] or false,
617
- first_claimed,
618
- ARGV[2],
619
- row[6] or false,
620
- row[7] or false,
621
- ARGV[1],
622
- ARGV[3],
623
- entries[1],
624
- false
625
- }
626
- end
877
+ takes_over[#takes_over + 1] = { candidate = candidate, row = row }
878
+ wanted[candidate[1]] = true
627
879
  end
628
880
  end
629
- flush()
630
- for _, field in ipairs(dead) do
631
- redis.call('HDEL', mkey(field), field .. ':ch', field .. ':ce')
632
- redis.call('ZREM', KEYS[5], field)
633
- release_snapshot_pins(ARGV[tail + 1] .. field, KEYS[7])
634
- end
635
- if #claimed > 0 then
636
- local result = { 'claimed', tostring(#claimed) }
637
- for _, claim in ipairs(claimed) do
638
- for _, value in ipairs(claim) do result[#result + 1] = value end
881
+ if #takes_over == 0 then
882
+ if retry_at then return { 'busy', tostring(retry_at) } end
883
+ return { 'settled' }
884
+ end
885
+ -- One XRANGE per run of consecutive claimed indexes.
886
+ local indexes, entries = {}, {}
887
+ for idx in pairs(wanted) do indexes[#indexes + 1] = idx end
888
+ table.sort(indexes)
889
+ local first = 1
890
+ while first <= #indexes do
891
+ local last = first
892
+ while last < #indexes and indexes[last + 1] == indexes[last] + 1 do last = last + 1 end
893
+ local range = redis.call('XRANGE', LOG, tostring(indexes[first]) .. '-0', tostring(indexes[last]) .. '-0')
894
+ for _, entry in ipairs(range) do entries[entry_index(entry[1])] = entry end
895
+ first = last + 1
896
+ end
897
+ for _, idx in ipairs(indexes) do
898
+ if not entries[idx] then return { 'missing', tostring(idx) } end
899
+ end
900
+ local result = { 'claimed', tostring(#takes_over) }
901
+ for _, take in ipairs(takes_over) do
902
+ local candidate, row = take.candidate, take.row
903
+ local idx, at = candidate[1], candidate[2]
904
+ local attempt = tostring(tonumber(row[1] or '0') + 1)
905
+ -- A claim still held here lapsed: its attempt never finished.
906
+ local lapsed = row[4] ~= false
907
+ local failures = tonumber(row[6] or '0')
908
+ if lapsed then
909
+ failures = failures + 1
910
+ put_row(idx, at, 'a', tostring(failures))
639
911
  end
640
- result[#result + 1] = tostring(boundary)
641
- return result
912
+ put_row(idx, at, 'd', attempt)
913
+ local columns
914
+ if lapsed and max_failures and failures >= max_failures then
915
+ -- That takeover spent the last failed attempt: dead-letter, unclaimed.
916
+ -- Advancing the attempt fences the lapsed one; nothing runs.
917
+ put_row(idx, at, 'f', ARGV[2])
918
+ drop_row(idx, at, 'ch')
919
+ drop_row(idx, at, 'ce')
920
+ ready_set(candidate.cid, -idx)
921
+ release_pins(idx, at, row[11])
922
+ columns = { attempt, row[2], row[3], false, false, tostring(failures), row[7], row[8], row[9], ARGV[2] }
923
+ else
924
+ local first_claimed = row[2] or ARGV[2]
925
+ put_row(idx, at, 'fc', first_claimed)
926
+ put_row(idx, at, 'lc', ARGV[2])
927
+ put_row(idx, at, 'ch', holder)
928
+ put_row(idx, at, 'ce', expires_at)
929
+ put_row(idx, at, 'l', candidate.kind)
930
+ columns = { attempt, first_claimed, ARGV[2], holder, expires_at, tostring(failures), row[7], row[8], row[9], false }
931
+ end
932
+ result[#result + 1] = { candidate.consumer, candidate.kind, tostring(idx), columns, entries[idx] }
642
933
  end
643
- if retry_at then return { 'busy', tostring(retry_at) } end
644
- return { 'settled' }
934
+ flush()
935
+ flush_ready()
936
+ result[#result + 1] = tostring(boundary)
937
+ result[#result + 1] = tostring(total)
938
+ return result
645
939
  `
646
940
 
647
941
  /**
648
- * KEYS: counter, meta, metadata split.
649
- * ARGV: holder, nowMs, expiresAtMs, n, then per claim: index, attempt.
650
- * Returns [renewedCount, ...renewed, ...superseded].
942
+ * KEYS: counter.
943
+ * ARGV: holder, nowMs, expiresAtMs, n, then per claim: consumer, tag, index,
944
+ * attempt. Returns [renewedCount, ...renewed, ...superseded] as 1-based
945
+ * positions in the request.
651
946
  */
652
- const RENEW_CLAIMS_LUA = `
653
- local COUNTER, META, SPLIT = KEYS[1], KEYS[2], KEYS[3]
654
- ${META_LUA}
655
- local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR)
656
- local total = tonumber(counts[1] or '0')
657
- use_split(counts[2], counts[1], counts[3])
658
- local claims, fields, cleared = {}, {}, {}
947
+ const RENEW_CLAIMS_LUA = `${BASE_LUA}
948
+ read_bounds()
949
+ local holder, now, expires_at = ARGV[1], tonumber(ARGV[2]), ARGV[3]
950
+ local refs, items, superseded = {}, {}, {}
659
951
  for i = 1, tonumber(ARGV[4]) do
660
- local field = ARGV[4 + (i - 1) * 2 + 1]
661
- local idx = tonumber(field)
952
+ local o = 4 + (i - 1) * 4
953
+ local idx = tonumber(ARGV[o + 3])
662
954
  if idx <= boundary then
663
- cleared[#cleared + 1] = field
664
- elseif idx >= 1 and idx <= total then
665
- claims[#claims + 1] = { field = field, attempt = tonumber(ARGV[4 + (i - 1) * 2 + 2]) }
666
- fields[#fields + 1] = field
955
+ superseded[#superseded + 1] = tostring(i)
956
+ else
957
+ local row = row_of(ARGV[o + 1], ARGV[o + 2])
958
+ refs[#refs + 1] = { idx, row }
959
+ items[#items + 1] = { position = i, idx = idx, row = row, tag = ARGV[o + 2], attempt = tonumber(ARGV[o + 4]) }
667
960
  end
668
961
  end
669
- local rows = read_fields(fields, { 'd', 'ce', 'p', 'f', 'ch' })
962
+ local rows = read_rows(refs, { 'd', 'ce', 'p', 'f', 'ch', 'l' })
670
963
  local renewed = {}
671
- local superseded = {}
672
- for _, claim in ipairs(claims) do
673
- local row = rows[claim.field]
674
- if tonumber(row[1] or '0') > claim.attempt then
675
- superseded[#superseded + 1] = claim.field
676
- elseif tonumber(ARGV[3]) > tonumber(ARGV[2]) and
677
- not row[3] and not row[4] and row[5] == ARGV[1] and
678
- tonumber(row[2] or '0') > tonumber(ARGV[2]) then
679
- put(mkey(claim.field), claim.field .. ':ce', ARGV[3])
680
- renewed[#renewed + 1] = claim.field
964
+ for k, item in ipairs(items) do
965
+ local row = rows[k]
966
+ local attempt = tonumber(row[1] or '0')
967
+ if row[6] and not owns(row[6], item.tag) then
968
+ -- Another cursor's head: this one never claimed it.
969
+ elseif row[3] or attempt > item.attempt then
970
+ superseded[#superseded + 1] = tostring(item.position)
971
+ elseif attempt == item.attempt and tonumber(expires_at) > now and
972
+ not row[4] and row[5] == holder and tonumber(row[2] or '0') > now then
973
+ put_row(item.idx, item.row, 'ce', expires_at)
974
+ renewed[#renewed + 1] = tostring(item.position)
681
975
  end
682
976
  end
683
977
  flush()
684
- for _, field in ipairs(cleared) do superseded[#superseded + 1] = field end
685
978
  local result = { tostring(#renewed) }
686
979
  for i = 1, #renewed do result[#result + 1] = renewed[i] end
687
980
  for i = 1, #superseded do result[#result + 1] = superseded[i] end
@@ -689,228 +982,393 @@ return result
689
982
  `
690
983
 
691
984
  /**
692
- * KEYS: store, counter, meta, pending, ready, event snapshot pins, snapshot
693
- * pin counts, metadata split, then one id ownership key per child.
694
- * ARGV: sessionId, nowMs, parent index, attempt, n, then per child:
695
- * id, type, payloadJson, lane, settled; finally notify channel.
985
+ * KEYS: counter.
986
+ * ARGV: sessionId, nowMs, consumer, tag, index, attempt, child causeJson,
987
+ * served consumers, returned idsJson, n, then per child: id, type,
988
+ * payloadJson, targets.
696
989
  */
697
- const COMPLETE_ATTEMPT_LUA = `
698
- local COUNTER, META, SPLIT, PENDING, READY = KEYS[2], KEYS[3], KEYS[8], KEYS[4], KEYS[5]
699
- ${META_LUA}
700
- ${ENQUEUE_LUA}
701
- ${ID_OWNERS_LUA}
702
- ${RELEASE_PINS_LUA}
703
- ${WAKE_LUA}
704
- ${SETTLE_LUA}
705
- local parent = ARGV[3]
706
- local attempt = tonumber(ARGV[4])
707
- local n = tonumber(ARGV[5])
708
- local owners, count = read_owners(8, n, ARGV[1])
709
- local total = tonumber(count or '0')
710
- if tonumber(parent) <= boundary then return { 'superseded' } end
711
- if tonumber(parent) < 1 or tonumber(parent) > total then return { 'missing' } end
712
- local row = read_fields({ parent }, { 'd', 'p', 'pa', 'ri', 'ch', 'f', 'lfa', 'l', 'next' })[parent]
990
+ const COMPLETE_ATTEMPT_LUA = `${BASE_LUA}${CURSOR_LUA}
991
+ local now = ARGV[2]
992
+ local consumer, tag, idx, attempt = ARGV[3], ARGV[4], tonumber(ARGV[5]), tonumber(ARGV[6])
993
+ local at = row_of(consumer, tag)
994
+ local n = tonumber(ARGV[10])
995
+ local ids = {}
996
+ for i = 1, n do ids[i] = ARGV[10 + (i - 1) * 4 + 1] end
997
+ local owners = owners_of(read_bounds(ids))
998
+ if idx <= boundary then return { 'superseded' } end
999
+ if idx < 1 or idx > total then return { 'missing' } end
1000
+ local row = read_rows({ { idx, at } }, { 'd', 'p', 'pa', 'ri', 'ch', 'f', 'l', 'next', 'pins' })[1]
713
1001
  if row[2] then
714
- if tonumber(row[3] or '0') ~= attempt then return { 'superseded' } end
715
- if not row[4] then return { 'partial' } end
716
- local recorded_ids = cjson.decode(row[4])
717
- if #recorded_ids ~= n then return { 'partial' } end
1002
+ -- The cursor moved past this head: a retry of its completion replays.
1003
+ if not owns(row[7], tag) then return { 'superseded' } end
1004
+ if not row[3] or tonumber(row[3]) ~= attempt then return { 'superseded' } end
1005
+ local recorded = cjson.decode(row[4] or '[]')
1006
+ if #recorded ~= n then return { 'partial' } end
718
1007
  for i = 1, n do
719
- if recorded_ids[i] ~= ARGV[5 + (i - 1) * 5 + 1] then return { 'partial' } end
1008
+ if recorded[i] ~= ARGV[10 + (i - 1) * 4 + 1] then return { 'partial' } end
720
1009
  end
721
- local indexes = {}
1010
+ local result = { 'duplicate' }
722
1011
  for i = 1, n do
723
1012
  if not owners[i] then return { 'partial' } end
724
- local rec = cjson.decode(owners[i])
725
- if rec[1] ~= ARGV[1] then return { 'partial' } end
726
- local entries = redis.call('XRANGE', KEYS[1], tostring(rec[2]) .. '-0', tostring(rec[2]) .. '-0')
727
- if #entries == 0 then return { 'partial' } end
728
- local values = entries[1][2]
729
- local cause = nil
730
- for j = 1, #values, 2 do
731
- if values[j] == 'cause' then cause = values[j + 1] break end
732
- end
733
- if not cause or cause == '' then return { 'partial' } end
734
- local decoded = cjson.decode(cause)
735
- if tonumber(decoded['index']) ~= tonumber(parent) or
736
- tonumber(decoded['attempt']) ~= attempt then return { 'partial' } end
737
- indexes[#indexes + 1] = tostring(rec[2])
1013
+ result[#result + 1] = owners[i]
738
1014
  end
739
- local result = { 'duplicate' }
740
- for _, idx in ipairs(indexes) do result[#result + 1] = idx end
741
1015
  return result
742
1016
  end
743
- if tonumber(row[1] or '0') ~= attempt then return { 'superseded' } end
744
- if not row[5] or row[6] or tonumber(row[7] or '0') == attempt then
1017
+ if tonumber(row[1] or '0') ~= attempt or not row[5] or row[6] or not owns(row[7], tag) then
745
1018
  return { 'superseded' }
746
1019
  end
747
1020
  for i = 1, n do
748
1021
  if owners[i] then return { 'partial' } end
749
1022
  end
750
- local returned_ids = {}
751
- for i = 1, n do returned_ids[i] = ARGV[5 + (i - 1) * 5 + 1] end
752
- local returned_json = '[]'
753
- if n > 0 then returned_json = cjson.encode(returned_ids) end
754
- local parent_key = mkey(parent)
755
- put(parent_key, parent .. ':p', ARGV[2])
756
- put(parent_key, parent .. ':pa', ARGV[4])
757
- put(parent_key, parent .. ':ri', returned_json)
758
- redis.call('HDEL', parent_key, parent .. ':ch', parent .. ':ce')
759
- settle(parent, row[8] or '', row[9])
760
- release_snapshot_pins(KEYS[6], KEYS[7])
761
- local cause = cjson.encode({ index = tonumber(parent), attempt = attempt })
762
- local owner_writes = {}
1023
+ use_served(ARGV[8])
1024
+ local batch = {}
763
1025
  for i = 1, n do
764
- local o = 5 + (i - 1) * 5
1026
+ local o = 10 + (i - 1) * 4
1027
+ batch[i] = { idx = total + i, type = ARGV[o + 2], targets = targets_of(ARGV[o + 4]) }
1028
+ end
1029
+ local moved, missing = advance(consumer, row[7], idx, row[8], batch)
1030
+ if not moved then return { 'index', missing, tostring(idx) } end
1031
+ local planned, unindexed, after = plan(batch, total)
1032
+ if not planned then return { 'index', unindexed, tostring(after) } end
1033
+ put_row(idx, at, 'p', now)
1034
+ put_row(idx, at, 'pa', ARGV[6])
1035
+ put_row(idx, at, 'ri', ARGV[9])
1036
+ release_pins(idx, at, row[9])
1037
+ local marks = {}
1038
+ for i = 1, n do
1039
+ local o = 10 + (i - 1) * 4
765
1040
  local field = tostring(total + i)
766
- redis.call('XADD', KEYS[1], field .. '-0',
1041
+ redis.call('XADD', LOG, field .. '-0',
767
1042
  'id', ARGV[o + 1], 'type', ARGV[o + 2], 'payload', ARGV[o + 3],
768
- 'created_at', ARGV[2], 'cause', cause, 'lane', ARGV[o + 4])
769
- owner_writes[#owner_writes + 1] = KEYS[8 + i]
770
- owner_writes[#owner_writes + 1] = cjson.encode({ ARGV[1], total + i })
771
- wake_add(field, ARGV[o + 1], ARGV[o + 2], ARGV[o + 3])
772
- if ARGV[o + 5] == '1' then
773
- put(mkey(field), field .. ':p', ARGV[2])
774
- else
775
- enqueue(field, ARGV[o + 4])
776
- end
1043
+ 'created_at', now, 'cause', ARGV[7])
1044
+ marks[#marks + 1] = id_key(ARGV[o + 1])
1045
+ marks[#marks + 1] = field
777
1046
  end
1047
+ index_batch(batch)
1048
+ enqueue(batch)
778
1049
  if n > 0 then
779
- owner_writes[#owner_writes + 1] = COUNTER
780
- owner_writes[#owner_writes + 1] = tostring(total + n)
781
- split_write(owner_writes)
782
- sliced('MSET', nil, owner_writes)
1050
+ marks[#marks + 1] = COUNTER
1051
+ marks[#marks + 1] = tostring(total + n)
783
1052
  end
784
- flush_queue()
1053
+ if typed_dirty then
1054
+ marks[#marks + 1] = TYPED
1055
+ marks[#marks + 1] = cjson.encode(typed)
1056
+ end
1057
+ sliced('MSET', nil, marks)
1058
+ flush_states()
785
1059
  flush()
786
- if n > 0 then wake(ARGV[6 + n * 5], tostring(total + n), ARGV[2]) end
1060
+ flush_ready()
787
1061
  return { 'ok', tostring(total + 1) }
788
1062
  `
789
1063
 
790
1064
  /**
791
- * KEYS: counter, meta, pending, ready, metadata split.
792
- * ARGV: idx, action ('retry' | 'skip'), nowMs.
793
- * Returns [redriven, hasPending]. Acts only on an unfinished, dead-lettered
794
- * event above the clear boundary. A retry keeps the event pending and makes
795
- * it ready again when it is unlaned or its lane's head.
1065
+ * KEYS: counter.
1066
+ * ARGV: consumer, tag, index, attempt, error, maxFailures, nowMs, retryAtMs
1067
+ * or ''.
796
1068
  */
797
- const REDRIVE_LUA = `
798
- local COUNTER, META, SPLIT, PENDING, READY = KEYS[1], KEYS[2], KEYS[5], KEYS[3], KEYS[4]
799
- ${META_LUA}
800
- ${ENQUEUE_LUA}
801
- ${SETTLE_LUA}
802
- local function result(redriven)
803
- return { redriven, redis.call('ZCARD', PENDING) > 0 and '1' or '0' }
804
- end
805
- local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR)
806
- local total = tonumber(counts[1] or '0')
807
- use_split(counts[2], counts[1], counts[3])
808
- local idx = tonumber(ARGV[1])
809
- if idx <= boundary or idx > total then return result('0') end
810
- local field = tostring(idx)
811
- local row = read_fields({ field }, { 'p', 'f', 'l', 'next' })[field]
812
- if row[1] or not row[2] then return result('0') end
813
- local meta_key = mkey(field)
814
- local lane = row[3] or ''
815
- if ARGV[2] == 'skip' then
816
- put(meta_key, field .. ':p', ARGV[3])
817
- settle(field, lane, row[4])
818
- flush_queue()
819
- flush()
1069
+ const FAIL_ATTEMPT_LUA = `${BASE_LUA}
1070
+ read_bounds()
1071
+ local consumer, tag, idx, attempt = ARGV[1], ARGV[2], tonumber(ARGV[3]), tonumber(ARGV[4])
1072
+ local at = row_of(consumer, tag)
1073
+ if idx <= boundary then return { 'superseded', '0' } end
1074
+ if idx < 1 or idx > total then return { 'missing', '0' } end
1075
+ local row = read_rows({ { idx, at } }, { 'a', 'd', 'p', 'f', 'lfa', 'ch', 'l', 'pins' })[1]
1076
+ if row[3] or (row[7] and not owns(row[7], tag)) then return { 'superseded', '0' } end
1077
+ local failures = tonumber(row[1] or '0')
1078
+ if tonumber(row[2] or '0') ~= attempt then return { 'superseded', tostring(failures) } end
1079
+ if not row[6] and row[5] and tonumber(row[5]) == attempt then
1080
+ return { row[4] and 'dead_lettered' or 'failed', tostring(failures) }
1081
+ end
1082
+ if row[4] then return { 'dead_lettered', tostring(failures) } end
1083
+ if not row[6] then return { 'superseded', tostring(failures) } end
1084
+ failures = failures + 1
1085
+ local dead = failures >= tonumber(ARGV[6])
1086
+ put_row(idx, at, 'a', tostring(failures))
1087
+ put_row(idx, at, 'e', ARGV[5])
1088
+ put_row(idx, at, 'lf', ARGV[7])
1089
+ put_row(idx, at, 'lfa', ARGV[4])
1090
+ drop_row(idx, at, 'ch')
1091
+ if dead then
1092
+ put_row(idx, at, 'f', ARGV[7])
1093
+ drop_row(idx, at, 'ce')
1094
+ ready_set(cid_for(consumer, row[7], idx), -idx)
1095
+ release_pins(idx, at, row[8])
1096
+ elseif ARGV[8] ~= '' then
1097
+ put_row(idx, at, 'ce', ARGV[8])
820
1098
  else
821
- redis.call('HDEL', meta_key, field .. ':f', field .. ':a')
822
- if lane == '' or redis.call('HGET', META, lane_field(lane, 'head')) == field then
823
- redis.call('ZADD', READY, idx, field)
1099
+ drop_row(idx, at, 'ce')
1100
+ end
1101
+ flush()
1102
+ flush_ready()
1103
+ return { dead and 'dead_lettered' or 'failed', tostring(failures) }
1104
+ `
1105
+
1106
+ /**
1107
+ * KEYS: counter. ARGV: consumer, index, action, nowMs, served
1108
+ * consumers. Returns [redriven, hasPending] | ['index', type, after]. Acts
1109
+ * only on the consumer's dead-lettered head at the index, above the clear
1110
+ * marker.
1111
+ */
1112
+ const REDRIVE_LUA = `${BASE_LUA}${CURSOR_LUA}
1113
+ read_bounds()
1114
+ use_served(ARGV[5])
1115
+ local consumer, idx = ARGV[1], tonumber(ARGV[2])
1116
+ local function result(redriven) return { redriven, has_pending() } end
1117
+ if idx <= boundary or idx < 1 or idx > total then return result('0') end
1118
+ local cid, tag = nil, nil
1119
+ for _, member in ipairs(redis.call('ZRANGEBYSCORE', READY, -idx, -idx)) do
1120
+ local owner, member_tag = parse_cid(member)
1121
+ if owner == consumer then
1122
+ cid, tag = member, member_tag
1123
+ break
824
1124
  end
825
1125
  end
1126
+ if not cid then return result('0') end
1127
+ local kind = string.sub(tag, 1, 1) == '#' and '#' or tag
1128
+ local at = row_of(consumer, tag)
1129
+ if ARGV[3] == 'skip' then
1130
+ local row = read_rows({ { idx, at } }, { 'next' })[1]
1131
+ local moved, missing = advance(consumer, kind, idx, row[1], {})
1132
+ if not moved then return { 'index', missing, tostring(idx) } end
1133
+ put_row(idx, at, 'p', ARGV[4])
1134
+ flush_states()
1135
+ if typed_dirty then redis.call('SET', TYPED, cjson.encode(typed)) end
1136
+ else
1137
+ drop_row(idx, at, 'f')
1138
+ drop_row(idx, at, 'a')
1139
+ ready_set(cid, idx)
1140
+ end
1141
+ flush()
1142
+ flush_ready()
826
1143
  return result('1')
827
1144
  `
828
1145
 
829
1146
  /**
830
- * KEYS: store, counter, meta, pending, metadata split.
831
- * Returns the dead-lettered events above the clear boundary, oldest first,
832
- * as pairs of a stream entry and its STORED_EVENT_META_SUFFIXES columns.
833
- * Every unfinished event is in the pending set, so the scan is bounded by
834
- * unfinished work, not history; only dead letters read their full columns.
1147
+ * KEYS: counter. ARGV: consumer, lane, types JSON or '*', backfill
1148
+ * ('1' or '0'). Brings one log cursor up to date without an append, as
1149
+ * `plan` and `enqueue` would for an append carrying its types: a missing
1150
+ * cursor starts at the clear marker with backfill, else at the head with
1151
+ * nothing to run; an idle one takes the next stored event of its types; one
1152
+ * with a head, claimed or not, keeps it. Records the types, and writes
1153
+ * nothing when nothing changed. Returns ['ok', position, head or nil, head
1154
+ * columns, session head] | ['index', type, after].
835
1155
  */
836
- const DEAD_LETTERS_LUA = `
837
- local COUNTER, META, SPLIT = KEYS[2], KEYS[3], KEYS[5]
838
- ${META_LUA}
839
- local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR)
840
- use_split(counts[2], counts[1], counts[3])
841
- local candidates = {}
842
- for _, field in ipairs(redis.call('ZRANGE', KEYS[4], 0, -1)) do
843
- if tonumber(field) > boundary then candidates[#candidates + 1] = field end
844
- end
845
- local status = read_fields(candidates, { 'p', 'f' })
846
- local dead = {}
847
- for _, field in ipairs(candidates) do
848
- if status[field][2] and not status[field][1] then dead[#dead + 1] = field end
1156
+ const WAKE_LOG_LUA = `${BASE_LUA}${CURSOR_LUA}
1157
+ read_bounds()
1158
+ local consumer, lane, ty = ARGV[1], ARGV[2], ARGV[3]
1159
+ local types = types_of(ty)
1160
+ local cid = cid_of(consumer, 'G' .. lane)
1161
+ load_states({ cid })
1162
+ local state = states[cid]
1163
+ local had_head = state.h
1164
+ -- Nothing after the latest clear: a missing cursor stays missing.
1165
+ if not state.exists and total == boundary then
1166
+ return { 'ok', tostring(boundary), false, {}, tostring(total) }
1167
+ end
1168
+ if not state.exists then
1169
+ state.exists, state.p = true, total
1170
+ if ARGV[4] == '1' then
1171
+ local head, missing = stored_head(boundary, types)
1172
+ if head == nil then return { 'index', missing, tostring(boundary) } end
1173
+ state.p, state.h = boundary, head
1174
+ else
1175
+ -- Nothing it takes is stored after its start: index its types from there.
1176
+ for name in pairs(types or {}) do
1177
+ if typed[name] == nil then register(name, total) end
1178
+ end
1179
+ end
1180
+ state.dirty = true
1181
+ elseif not state.h then
1182
+ -- A deployment that doesn't serve this consumer may have appended events
1183
+ -- it takes since it went idle.
1184
+ local head, missing = stored_head(state.p, types)
1185
+ if head == nil then return { 'index', missing, tostring(state.p) } end
1186
+ if head then state.h, state.dirty = head, true end
1187
+ end
1188
+ if state.ty ~= ty then state.ty, state.dirty = ty, true end
1189
+ local columns = {}
1190
+ if had_head then
1191
+ columns = read_rows({ { state.h, row_of(consumer, 'G' .. lane) } }, ${HEAD_COLUMNS})[1]
1192
+ elseif state.h then
1193
+ ready_set(cid, state.h)
1194
+ end
1195
+ flush_states()
1196
+ if typed_dirty then redis.call('SET', TYPED, cjson.encode(typed)) end
1197
+ flush()
1198
+ flush_ready()
1199
+ return { 'ok', tostring(state.p), state.h and tostring(state.h) or false, columns, tostring(total) }
1200
+ `
1201
+
1202
+ /**
1203
+ * KEYS: counter. Returns the dead-lettered heads above the clear
1204
+ * marker as [consumer, tag, index, columns, entry] tuples.
1205
+ */
1206
+ const DEAD_LETTERS_LUA = `${BASE_LUA}
1207
+ local dead = redis.call('ZRANGEBYSCORE', READY, '-inf', '(0', 'WITHSCORES')
1208
+ if #dead == 0 then return {} end
1209
+ read_bounds()
1210
+ local refs, items = {}, {}
1211
+ for i = 1, #dead, 2 do
1212
+ local consumer, tag = parse_cid(dead[i])
1213
+ local idx = -tonumber(dead[i + 1])
1214
+ if idx > boundary then
1215
+ refs[#refs + 1] = { idx, row_of(consumer, tag) }
1216
+ items[#items + 1] = { consumer, tag, idx }
1217
+ end
849
1218
  end
850
- local rows = read_fields(dead, { ${STORED_EVENT_META_SUFFIXES.map((suffix) => `'${suffix}'`).join(', ')} })
1219
+ local rows = read_rows(refs, ${HEAD_COLUMNS})
851
1220
  local result = {}
852
- for _, field in ipairs(dead) do
853
- local entries = redis.call('XRANGE', KEYS[1], field .. '-0', field .. '-0')
1221
+ for k, item in ipairs(items) do
1222
+ local entries = redis.call('XRANGE', LOG, tostring(item[3]) .. '-0', tostring(item[3]) .. '-0')
854
1223
  if #entries > 0 then
855
- result[#result + 1] = entries[1]
856
- result[#result + 1] = rows[field]
1224
+ result[#result + 1] = { item[1], item[2], tostring(item[3]), rows[k], entries[1] }
1225
+ end
1226
+ end
1227
+ return result
1228
+ `
1229
+
1230
+ /**
1231
+ * KEYS: counter. Returns every listed cursor as [consumer, tag,
1232
+ * position, head or nil, columns]: lane and log cursors from the cursor
1233
+ * hash, read against the clear marker, then the event cursors with a head.
1234
+ */
1235
+ const CURSORS_LUA = `${BASE_LUA}
1236
+ read_bounds()
1237
+ local listed = {}
1238
+ local fields = redis.call('HGETALL', CURSORS)
1239
+ local records, order = {}, {}
1240
+ for i = 1, #fields, 2 do
1241
+ local at = string.find(fields[i], ':[^:]*$')
1242
+ local cid, suffix = string.sub(fields[i], 1, at - 1), string.sub(fields[i], at + 1)
1243
+ if not records[cid] then
1244
+ records[cid] = {}
1245
+ order[#order + 1] = cid
1246
+ end
1247
+ records[cid][suffix] = fields[i + 1]
1248
+ end
1249
+ for _, cid in ipairs(order) do
1250
+ local record = records[cid]
1251
+ local consumer, tag = parse_cid(cid)
1252
+ local head = (record.h and record.h ~= '') and tonumber(record.h) or false
1253
+ local position = tonumber(record.p or '0')
1254
+ if head and head <= boundary then head = false end
1255
+ local kind = string.sub(tag, 1, 1)
1256
+ if kind == 'G' and position < boundary then position = boundary end
1257
+ if kind == 'G' or head or position > boundary then
1258
+ listed[#listed + 1] = { consumer, tag, position, head }
857
1259
  end
858
1260
  end
1261
+ local ready = redis.call('ZRANGE', READY, 0, -1, 'WITHSCORES')
1262
+ for i = 1, #ready, 2 do
1263
+ local consumer, tag = parse_cid(ready[i])
1264
+ if string.sub(tag, 1, 1) == '#' then
1265
+ local head = math.abs(tonumber(ready[i + 1]))
1266
+ listed[#listed + 1] = { consumer, '#', head - 1, head }
1267
+ end
1268
+ end
1269
+ local refs, positions = {}, {}
1270
+ for position, cursor in ipairs(listed) do
1271
+ if cursor[4] then
1272
+ refs[#refs + 1] = { cursor[4], row_of(cursor[1], cursor[2]) }
1273
+ positions[#refs] = position
1274
+ end
1275
+ end
1276
+ local rows = read_rows(refs, ${HEAD_COLUMNS})
1277
+ local columns = {}
1278
+ for k, position in ipairs(positions) do columns[position] = rows[k] end
1279
+ local result = {}
1280
+ for position, cursor in ipairs(listed) do
1281
+ result[#result + 1] = {
1282
+ cursor[1], cursor[2], tostring(cursor[3]),
1283
+ cursor[4] and tostring(cursor[4]) or false, columns[position] or {},
1284
+ }
1285
+ end
859
1286
  return result
860
1287
  `
861
1288
 
862
1289
  /**
863
- * KEYS: head, history, event counter, event meta, pin counts, metadata split.
864
- * ARGV: nowMs, event pin key prefix, write count, then per write:
865
- * index, snapshot JSON, pin count, pin event indexes; finally the reducer
866
- * name. Writes from before the latest clear are dropped, and a head from
867
- * before it is replaced as if absent.
1290
+ * KEYS: counter, head, history.
1291
+ * ARGV: write count, then per write: index, snapshot JSON, pin count, then
1292
+ * per pin: consumer, tag, index; finally the reducer name. Writes from
1293
+ * before the latest clear are dropped, and a head from before it is
1294
+ * replaced as if absent. A pin holds only on an unfinished, live head: a
1295
+ * claimed row says so itself, an unclaimed head by its cursor's ready score.
868
1296
  */
869
- const PUT_SNAPSHOTS_LUA = `
870
- local COUNTER, META, SPLIT = KEYS[3], KEYS[4], KEYS[6]
871
- ${META_LUA}
872
- local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR)
873
- local total = tonumber(counts[1] or '0')
874
- use_split(counts[2], counts[1], counts[3])
875
- local pinned = {}
876
- local offset = 4
877
- for i = 1, tonumber(ARGV[3]) do
1297
+ const PUT_SNAPSHOTS_LUA = `${BASE_LUA}
1298
+ read_bounds()
1299
+ local HEAD, HISTORY = KEYS[2], KEYS[3]
1300
+ local entries, refs, pins = {}, {}, {}
1301
+ local offset = 2
1302
+ for w = 1, tonumber(ARGV[1]) do
1303
+ local index = tonumber(ARGV[offset])
878
1304
  local pin_count = tonumber(ARGV[offset + 2])
1305
+ local entry = { index = index, snapshot = ARGV[offset + 1], pins = {} }
879
1306
  for p = 1, pin_count do
880
- local event_index = tonumber(ARGV[offset + 2 + p])
881
- if event_index > boundary and event_index <= total then
882
- pinned[#pinned + 1] = tostring(event_index)
1307
+ local o = offset + 2 + (p - 1) * 3
1308
+ local pin = { consumer = ARGV[o + 1], tag = ARGV[o + 2], idx = tonumber(ARGV[o + 3]) }
1309
+ if index >= boundary and pin.idx > boundary and pin.idx <= total then
1310
+ pin.row = row_of(pin.consumer, pin.tag)
1311
+ refs[#refs + 1] = { pin.idx, pin.row }
1312
+ pins[#pins + 1] = pin
1313
+ entry.pins[#entry.pins + 1] = pin
1314
+ end
1315
+ end
1316
+ entries[w] = entry
1317
+ offset = offset + 3 + pin_count * 3
1318
+ end
1319
+ local function live(consumer, tag, idx)
1320
+ return tonumber(redis.call('ZSCORE', READY, cid_of(consumer, tag)) or '0') == idx
1321
+ end
1322
+ local rows = read_rows(refs, { 'pins', 'p', 'f', 'l' })
1323
+ local held, held_order = {}, {}
1324
+ for k, pin in ipairs(pins) do
1325
+ local row = rows[k]
1326
+ local valid
1327
+ if row[2] or row[3] then
1328
+ valid = false
1329
+ elseif row[4] then
1330
+ valid = owns(row[4], pin.tag)
1331
+ elseif pin.tag == '#' then
1332
+ valid = live(pin.consumer, '#' .. tostring(pin.idx), pin.idx)
1333
+ else
1334
+ valid = live(pin.consumer, pin.tag, pin.idx) or
1335
+ live(pin.consumer, 'G' .. string.sub(pin.tag, 2), pin.idx)
1336
+ end
1337
+ if valid then
1338
+ local key = tostring(pin.idx) .. ':' .. pin.row
1339
+ if not held[key] then
1340
+ local list = (row[1] and row[1] ~= '') and cjson.decode(row[1]) or {}
1341
+ local set = {}
1342
+ for _, ref in ipairs(list) do set[ref] = true end
1343
+ held[key] = { list = list, set = set, idx = pin.idx, row = pin.row }
1344
+ held_order[#held_order + 1] = key
883
1345
  end
1346
+ pin.held = held[key]
884
1347
  end
885
- offset = offset + 3 + pin_count
886
1348
  end
887
- local status = read_fields(pinned, { 'p', 'f' })
888
1349
  local snapshots = {}
889
- offset = 4
890
- for i = 1, tonumber(ARGV[3]) do
891
- local index = tonumber(ARGV[offset])
892
- local snapshot = ARGV[offset + 1]
893
- local pin_count = tonumber(ARGV[offset + 2])
894
- local ref = cjson.encode({ KEYS[2], tostring(index) })
895
- for p = 1, pin_count do
896
- local event_index = tonumber(ARGV[offset + 2 + p])
897
- local row = status[tostring(event_index)]
898
- if index >= boundary and event_index > boundary and event_index <= total and
899
- not row[1] and not row[2] then
900
- local pin_key = ARGV[2] .. tostring(event_index)
901
- if redis.call('SADD', pin_key, ref) == 1 then
902
- redis.call('HINCRBY', KEYS[5], ref, 1)
903
- end
1350
+ for _, entry in ipairs(entries) do
1351
+ local ref = cjson.encode({ HISTORY, tostring(entry.index) })
1352
+ for _, pin in ipairs(entry.pins) do
1353
+ if pin.held and not pin.held.set[ref] then
1354
+ pin.held.set[ref] = true
1355
+ pin.held.list[#pin.held.list + 1] = ref
1356
+ pin.held.dirty = true
1357
+ redis.call('HINCRBY', PIN_COUNTS, ref, 1)
904
1358
  end
905
1359
  end
906
- if index >= boundary then
907
- snapshots[#snapshots + 1] = { index = index, snapshot = snapshot, ref = ref }
1360
+ if entry.index >= boundary then
1361
+ snapshots[#snapshots + 1] = { index = entry.index, snapshot = entry.snapshot, ref = ref }
908
1362
  end
909
- offset = offset + 3 + pin_count
910
1363
  end
1364
+ for _, key in ipairs(held_order) do
1365
+ local row = held[key]
1366
+ if row.dirty then put_row(row.idx, row.row, 'pins', cjson.encode(row.list)) end
1367
+ end
1368
+ flush()
911
1369
  if #snapshots == 0 then return 1 end
912
1370
 
913
- local head = redis.call('GET', KEYS[1])
1371
+ local head = redis.call('GET', HEAD)
914
1372
  local head_index = nil
915
1373
  local registered = false
916
1374
  if head then
@@ -920,109 +1378,112 @@ if head then
920
1378
  end
921
1379
  if head_index and head_index < boundary then
922
1380
  head, head_index, registered = nil, nil, false
923
- redis.call('ZREMRANGEBYSCORE', KEYS[2], '-inf', '(' .. boundary)
1381
+ redis.call('ZREMRANGEBYSCORE', HISTORY, '-inf', '(' .. boundary)
924
1382
  end
925
1383
  -- Deletion finds each reducer's keys through this set instead of a scan.
926
1384
  -- Heads written since carry the mark, so a reducer registers once.
927
1385
  if not registered then
928
- redis.call('SADD', string.sub(COUNTER, 1, -6) .. 'snap-reducers', ARGV[#ARGV])
1386
+ redis.call('SADD', SESSION .. 'snap-reducers', ARGV[#ARGV])
929
1387
  end
930
1388
  local latest = snapshots[#snapshots]
931
1389
  local writes_head = not head_index or latest.index >= head_index
932
1390
  local advances = not head_index or latest.index > head_index
933
1391
  local final_head_index = advances and latest.index or head_index
934
1392
 
935
- if advances and head and tonumber(redis.call('HGET', KEYS[5], cjson.encode({ KEYS[2], tostring(head_index) })) or '0') > 0 then
936
- redis.call('ZREMRANGEBYSCORE', KEYS[2], head_index, head_index)
937
- redis.call('ZADD', KEYS[2], head_index, head)
1393
+ if advances and head and tonumber(redis.call('HGET', PIN_COUNTS, cjson.encode({ HISTORY, tostring(head_index) })) or '0') > 0 then
1394
+ redis.call('ZREMRANGEBYSCORE', HISTORY, head_index, head_index)
1395
+ redis.call('ZADD', HISTORY, head_index, head)
938
1396
  end
939
1397
 
940
1398
  for _, write in ipairs(snapshots) do
941
1399
  if write.index < final_head_index then
942
- redis.call('ZREMRANGEBYSCORE', KEYS[2], write.index, write.index)
943
- if tonumber(redis.call('HGET', KEYS[5], write.ref) or '0') > 0 then
944
- redis.call('ZADD', KEYS[2], write.index, write.snapshot)
1400
+ redis.call('ZREMRANGEBYSCORE', HISTORY, write.index, write.index)
1401
+ if tonumber(redis.call('HGET', PIN_COUNTS, write.ref) or '0') > 0 then
1402
+ redis.call('ZADD', HISTORY, write.index, write.snapshot)
945
1403
  end
946
1404
  end
947
1405
  end
948
1406
 
949
1407
  if writes_head then
950
- redis.call('SET', KEYS[1], latest.snapshot)
951
- redis.call('ZREMRANGEBYSCORE', KEYS[2], latest.index, latest.index)
1408
+ redis.call('SET', HEAD, latest.snapshot)
1409
+ redis.call('ZREMRANGEBYSCORE', HISTORY, latest.index, latest.index)
952
1410
  end
953
1411
  return 1
954
1412
  `
955
1413
 
956
1414
  /**
957
- * KEYS: presence hash. ARGV: atMs, ttlMs, n, then per entry:
958
- * hashField, valueJson ('' deletes), patchFieldJson; finally channel and
959
- * message prefix. Wire fragments stay JSON text to preserve arrays
960
- * and numeric precision through Lua. Field-wise LWW by the sender's
961
- * `at`: a stored entry with a strictly newer stamp survives; ties go
962
- * to the incoming write (memory-tier parity). Returns the applied
963
- * 1-based entry positions. The whole-key PEXPIRE is the coarse bound,
964
- * sound because core sends one uniform ttlMs; the per-entry `e` stamp
965
- * is the fine bound (the injected storage clock at the applied set
966
- * plus ttlMs — never the sender stamp), pruned on read against the
967
- * same clock.
1415
+ * Presence entries are `{"a":at,"e":expiresAt,"s":seen,"v":value}`, stamps
1416
+ * first and both integers, so scripts read them with a pattern instead of
1417
+ * decoding the value.
1418
+ */
1419
+ const PRESENCE_STAMPS_LUA = `
1420
+ local function stamps(entry)
1421
+ local a, e = string.match(entry, '^{"a":(%-?%d+),"e":(%-?%d+),')
1422
+ return tonumber(a), tonumber(e)
1423
+ end
1424
+ `
1425
+
1426
+ /**
1427
+ * KEYS: presence hash. ARGV: atMs, ttlMs, nowMs, n, then per entry:
1428
+ * hashField, entryJson (a null value is a tombstone). Field-wise LWW by the
1429
+ * sender's `at`: an unexpired stored entry with a strictly newer stamp
1430
+ * survives; ties go to the incoming write (memory-tier parity). Returns the applied 1-based entry
1431
+ * positions. The whole-key PEXPIRE is the coarse bound, sound because core
1432
+ * sends one uniform ttlMs; the per-entry `e` stamp is the fine bound (the
1433
+ * injected storage clock at the applied set plus ttlMs — never the sender
1434
+ * stamp), filtered on read against the same clock.
968
1435
  */
969
- const PRESENCE_SET_LUA = `
970
- local at = tonumber(ARGV[1])
971
- local applied = {}
972
- local published = {}
973
- local n = tonumber(ARGV[3])
1436
+ const PRESENCE_SET_LUA = `${SLICED_LUA}${PRESENCE_STAMPS_LUA}
1437
+ local at, now, n = tonumber(ARGV[1]), tonumber(ARGV[3]), tonumber(ARGV[4])
1438
+ local fields = {}
1439
+ for i = 1, n do fields[i] = ARGV[4 + (i - 1) * 2 + 1] end
1440
+ local stored = sliced('HMGET', KEYS[1], fields)
1441
+ local applied, sets = {}, {}
974
1442
  for i = 1, n do
975
- local field = ARGV[3 + (i - 1) * 3 + 1]
976
- local value = ARGV[3 + (i - 1) * 3 + 2]
977
- local existing = redis.call('HGET', KEYS[1], field)
978
- if not existing or tonumber(cjson.decode(existing)['a']) <= at then
979
- if value == '' then
980
- redis.call('HDEL', KEYS[1], field)
981
- else
982
- redis.call('HSET', KEYS[1], field, value)
983
- end
1443
+ local a, e = false, false
1444
+ if stored[i] then a, e = stamps(stored[i]) end
1445
+ if not stored[i] or a <= at or e <= now then
1446
+ sets[#sets + 1] = fields[i]
1447
+ sets[#sets + 1] = ARGV[4 + (i - 1) * 2 + 2]
984
1448
  applied[#applied + 1] = tostring(i)
985
- published[#published + 1] = ARGV[3 + (i - 1) * 3 + 3]
986
1449
  end
987
1450
  end
988
- if #applied > 0 then
989
- redis.call('PEXPIRE', KEYS[1], ARGV[2])
990
- redis.pcall('PUBLISH', ARGV[4 + n * 3], ARGV[5 + n * 3] .. table.concat(published, ',') .. '}}')
991
- end
1451
+ sliced('HSET', KEYS[1], sets)
1452
+ if #applied > 0 then redis.call('PEXPIRE', KEYS[1], ARGV[2]) end
992
1453
  return applied
993
1454
  `
994
1455
 
995
1456
  /**
996
- * KEYS: presence hash. ARGV: nowMs (the injected clock, so tests can
997
- * time-travel expiry against a real server). Returns the live
998
- * [field, value, ...] pairs; entries at or past their fine-bound
999
- * expiry are HDELed in the same atomic step, so a stale reader can
1000
- * never sweep a concurrently refreshed field.
1457
+ * KEYS: presence hash. ARGV: nowMs, then the fields a read saw expired.
1458
+ * Deletes those still at or past their expiry, re-checked in this atomic
1459
+ * step so a stale reader never sweeps a concurrently refreshed field.
1001
1460
  */
1002
- const PRESENCE_READ_LUA = `
1461
+ const PRESENCE_PRUNE_LUA = `${SLICED_LUA}${PRESENCE_STAMPS_LUA}
1003
1462
  local now = tonumber(ARGV[1])
1004
- local rows = redis.call('HGETALL', KEYS[1])
1005
- local live = {}
1006
- for i = 1, #rows, 2 do
1007
- if tonumber(cjson.decode(rows[i + 1])['e']) <= now then
1008
- redis.call('HDEL', KEYS[1], rows[i])
1009
- else
1010
- live[#live + 1] = rows[i]
1011
- live[#live + 1] = rows[i + 1]
1463
+ local fields = {}
1464
+ for i = 2, #ARGV do fields[#fields + 1] = ARGV[i] end
1465
+ local stored = sliced('HMGET', KEYS[1], fields)
1466
+ local stale = {}
1467
+ for i = 1, #fields do
1468
+ if stored[i] then
1469
+ local _, e = stamps(stored[i])
1470
+ if e <= now then stale[#stale + 1] = fields[i] end
1012
1471
  end
1013
1472
  end
1014
- return live
1473
+ sliced('HDEL', KEYS[1], stale)
1474
+ return 1
1015
1475
  `
1016
1476
 
1017
1477
  /**
1018
- * KEYS: head, history, event counter. ARGV: snapshot frontier or ''.
1478
+ * KEYS: counter, head, history. ARGV: snapshot frontier or ''.
1019
1479
  * Returns one atomic multiversion-cache-plus-log-frontier read, with the
1020
1480
  * clear boundary; checkpoints from before it are ignored.
1021
1481
  */
1022
1482
  const READ_STATE_LUA = `
1023
- local counts = redis.call('MGET', KEYS[3], string.sub(KEYS[3], 1, -6) .. 'clear')
1483
+ local session = string.sub(KEYS[1], 1, -6)
1484
+ local counts = redis.call('MGET', KEYS[1], session .. 'clear')
1024
1485
  local boundary = tonumber(counts[2] or '0')
1025
- local head = redis.call('GET', KEYS[1])
1486
+ local head = redis.call('GET', KEYS[2])
1026
1487
  local head_index = head and cjson.decode(head)['index'] or nil
1027
1488
  if head and (type(head_index) ~= 'number' or head_index < 0 or head_index > 9007199254740991 or head_index ~= math.floor(head_index)) then
1028
1489
  return redis.error_reply('invalid snapshot head index')
@@ -1037,63 +1498,138 @@ if head_index and (cap == '' or head_index <= tonumber(cap)) then
1037
1498
  snapshot = head
1038
1499
  else
1039
1500
  local upper = cap == '' and '+inf' or cap
1040
- local historical = redis.call('ZREVRANGEBYSCORE', KEYS[2], upper, boundary, 'LIMIT', 0, 1)
1501
+ local historical = redis.call('ZREVRANGEBYSCORE', KEYS[3], upper, boundary, 'LIMIT', 0, 1)
1041
1502
  if #historical > 0 then snapshot = historical[1] end
1042
1503
  end
1043
- return { head_index or '', snapshot or '', counts[1] or '0', tostring(boundary) }
1504
+ return { head_index or '', snapshot or '', counts[1] or counts[2] or '0', tostring(boundary) }
1044
1505
  `
1045
1506
 
1046
1507
  /**
1047
- * Removes every registered reducer's checkpoints below the latest clear, and
1048
- * the pin counts that name them. Snapshot writes register each reducer in
1049
- * {prefix}:{session}:snap-reducers, so no key scan is needed.
1508
+ * What reclaim removes once no entry is left below the latest clear: every
1509
+ * registered reducer's checkpoints below the latest clear and the pin counts
1510
+ * naming them (snapshot writes register each reducer in
1511
+ * {prefix}:{session}:snap-reducers, so no key scan is needed), every head
1512
+ * row below it (whole chunks, then the boundary chunk's older rows), the
1513
+ * type indexes' older entries, and the lane cursors it left idle.
1514
+ *
1515
+ * A lifetime that ended empty then collapses to the clear marker, the log
1516
+ * holding it (readers resuming before the marker still read it), and the log
1517
+ * cursors the sweep reset to the marker. The counter reads from the marker
1518
+ * when absent, type indexes rebuild lazily, and the marker's own id key goes
1519
+ * too.
1050
1520
  */
1051
- const SWEEP_CHECKPOINTS_LUA = `
1052
- local function sweep_checkpoints(session, boundary)
1053
- for _, reducer in ipairs(redis.call('SMEMBERS', session .. 'snap-reducers')) do
1054
- local head_key = session .. 'snap:' .. reducer
1521
+ const SWEEP_LUA = `
1522
+ local function sweep(boundary)
1523
+ for _, reducer in ipairs(redis.call('SMEMBERS', SESSION .. 'snap-reducers')) do
1524
+ local head_key = SESSION .. 'snap:' .. reducer
1055
1525
  local head = redis.call('GET', head_key)
1056
1526
  if head and tonumber(cjson.decode(head)['index']) < boundary then
1057
1527
  redis.call('DEL', head_key)
1058
1528
  end
1059
- redis.call('ZREMRANGEBYSCORE', session .. 'snap-history:' .. reducer, '-inf', '(' .. boundary)
1529
+ redis.call('ZREMRANGEBYSCORE', SESSION .. 'snap-history:' .. reducer, '-inf', '(' .. boundary)
1060
1530
  end
1061
- local counts_key = session .. 'snapshot-pin-counts'
1062
- local counts = redis.call('HGETALL', counts_key)
1531
+ local counts = redis.call('HGETALL', PIN_COUNTS)
1063
1532
  local stale = {}
1064
1533
  for i = 1, #counts, 2 do
1065
1534
  if tonumber(cjson.decode(counts[i])[2]) < boundary then stale[#stale + 1] = counts[i] end
1066
1535
  end
1067
- if #stale > 0 then redis.call('HDEL', counts_key, unpack(stale)) end
1536
+ sliced('HDEL', PIN_COUNTS, stale)
1537
+ local raw = redis.call('GET', TYPED)
1538
+ if raw then
1539
+ for name in pairs(cjson.decode(raw)) do
1540
+ redis.call('ZREMRANGEBYSCORE', type_key(name), '-inf', '(' .. boundary)
1541
+ end
1542
+ end
1543
+ local last = math.floor((boundary - 1) / ${META_CHUNK_SIZE})
1544
+ local chunks = {}
1545
+ for chunk = 0, last - 1 do chunks[#chunks + 1] = META .. ':' .. tostring(chunk) end
1546
+ sliced('UNLINK', nil, chunks)
1547
+ local older = {}
1548
+ for _, field in ipairs(redis.call('HKEYS', META .. ':' .. tostring(last))) do
1549
+ local idx = tonumber(string.match(field, '^(%d+):'))
1550
+ if idx and idx < boundary then older[#older + 1] = field end
1551
+ end
1552
+ sliced('HDEL', META .. ':' .. tostring(last), older)
1553
+ local fields = redis.call('HGETALL', CURSORS)
1554
+ local records, order = {}, {}
1555
+ for i = 1, #fields, 2 do
1556
+ local at = string.find(fields[i], ':[^:]*$')
1557
+ local cid, suffix = string.sub(fields[i], 1, at - 1), string.sub(fields[i], at + 1)
1558
+ if not records[cid] then
1559
+ records[cid] = {}
1560
+ order[#order + 1] = cid
1561
+ end
1562
+ records[cid][suffix] = fields[i + 1]
1563
+ end
1564
+ local idle, moved = {}, {}
1565
+ for _, cid in ipairs(order) do
1566
+ local record = records[cid]
1567
+ local _, tag = parse_cid(cid)
1568
+ local head = (record.h and record.h ~= '') and tonumber(record.h) or false
1569
+ local position = tonumber(record.p or '0')
1570
+ if head and head <= boundary then head = false end
1571
+ if string.sub(tag, 1, 1) == 'G' then
1572
+ if position < boundary then position = boundary end
1573
+ moved[#moved + 1] = cid .. ':h'
1574
+ moved[#moved + 1] = head and tostring(head) or ''
1575
+ moved[#moved + 1] = cid .. ':p'
1576
+ moved[#moved + 1] = tostring(position)
1577
+ elseif not head and position <= boundary then
1578
+ for suffix in pairs(record) do idle[#idle + 1] = cid .. ':' .. suffix end
1579
+ elseif not head and record.h ~= '' then
1580
+ moved[#moved + 1] = cid .. ':h'
1581
+ moved[#moved + 1] = ''
1582
+ end
1583
+ end
1584
+ sliced('HDEL', CURSORS, idle)
1585
+ sliced('HSET', CURSORS, moved)
1586
+ local count = redis.call('GET', COUNTER)
1587
+ if not count or tonumber(count) ~= boundary or redis.call('ZCARD', READY) > 0 then
1588
+ return
1589
+ end
1590
+ local stale = { COUNTER, TYPED, PIN_COUNTS, chunk_key(boundary) }
1591
+ local reducers = SESSION .. 'snap-reducers'
1592
+ for _, reducer in ipairs(redis.call('SMEMBERS', reducers)) do
1593
+ stale[#stale + 1] = SESSION .. 'snap:' .. reducer
1594
+ stale[#stale + 1] = SESSION .. 'snap-history:' .. reducer
1595
+ end
1596
+ stale[#stale + 1] = reducers
1597
+ local types = redis.call('GET', TYPED)
1598
+ if types then
1599
+ for name in pairs(cjson.decode(types)) do stale[#stale + 1] = type_key(name) end
1600
+ end
1601
+ local marker = redis.call('XRANGE', LOG, boundary .. '-0', boundary .. '-0')[1]
1602
+ if marker then
1603
+ for i = 1, #marker[2], 2 do
1604
+ if marker[2][i] == 'id' then stale[#stale + 1] = id_key(marker[2][i + 1]) end
1605
+ end
1606
+ end
1607
+ sliced('UNLINK', nil, stale)
1068
1608
  end
1069
1609
  `
1070
1610
 
1071
1611
  /**
1072
- * KEYS: store stream, counter, metadata, metadata split, marker id ownership
1073
- * key, pending, ready.
1074
- * ARGV: sessionId, nowMs, ifLastIndex or '', cause index or '', cause
1075
- * attempt, marker id, notify channel.
1076
- * Returns ['ok', cleared, head, boundary] | ['no_parent', idx] |
1077
- * ['superseded', idx] | ['foreign']. The old lifetime's log moves aside in one
1078
- * RENAME, so every read of the log starts at the marker, and its queues are
1079
- * unlinked, so claims and pending checks see only the new lifetime; deletion
1080
- * drains the moved log.
1612
+ * KEYS: counter.
1613
+ * ARGV: sessionId, nowMs, ifLastIndex or '', cause consumer or '', cause tag,
1614
+ * cause index, cause attempt, marker id.
1615
+ * Returns ['ok', cleared, head, boundary, appended?] | ['superseded'] |
1616
+ * ['foreign']; `appended` is '1' when this call appended the marker.
1617
+ * The marker joins the same log: the old lifetime's entries stay below it
1618
+ * until reclaim deletes them, and every read of the log starts at the
1619
+ * marker. (A stream can't be renamed aside: Upstash refuses RENAME on one.)
1620
+ * The ready set is unlinked, so claims and pending checks see only the new
1621
+ * lifetime. Type indexes keep their older entries until reclaim too:
1622
+ * lookups start above the marker.
1081
1623
  */
1082
- const CLEAR_LUA = `
1083
- local COUNTER, META, SPLIT = KEYS[2], KEYS[3], KEYS[4]
1084
- ${META_LUA}
1085
- ${WAKE_LUA}
1086
- local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR, KEYS[5])
1087
- local head = tonumber(counts[1] or '0')
1088
- use_split(counts[2], counts[1], counts[3])
1624
+ const CLEAR_LUA = `${BASE_LUA}
1625
+ local marker_owner = owners_of(read_bounds({ ARGV[8] }))[1]
1626
+ local head = total
1089
1627
  if ARGV[4] ~= '' then
1090
- local field = ARGV[4]
1091
- local pidx = tonumber(field)
1092
- if pidx <= boundary then return { 'superseded', field } end
1093
- if pidx < 1 or pidx > head then return { 'no_parent', field } end
1094
- local row = read_fields({ field }, { 'd', 'f' })[field]
1095
- if tonumber(row[1] or '0') ~= tonumber(ARGV[5]) or row[2] then
1096
- return { 'superseded', field }
1628
+ local idx = tonumber(ARGV[6])
1629
+ if idx <= boundary then return { 'superseded' } end
1630
+ local row = read_rows({ { idx, row_of(ARGV[4], ARGV[5]) } }, { 'd', 'f', 'p', 'l' })[1]
1631
+ if tonumber(row[1] or '0') ~= tonumber(ARGV[7]) or row[2] or row[3] or not owns(row[4], ARGV[5]) then
1632
+ return { 'superseded' }
1097
1633
  end
1098
1634
  end
1099
1635
  local expected = ARGV[3]
@@ -1106,151 +1642,118 @@ end
1106
1642
  if head == 0 or boundary == head then
1107
1643
  return { 'ok', '1', tostring(head), tostring(boundary) }
1108
1644
  end
1109
- if counts[4] then return { 'foreign' } end
1645
+ if marker_owner then return { 'foreign' } end
1110
1646
  local field = tostring(head + 1)
1111
- local session = string.sub(COUNTER, 1, -6)
1112
- local moved = session .. 'cleared:' .. field
1113
- redis.call('RENAME', KEYS[1], moved)
1114
- redis.call('UNLINK', KEYS[6], KEYS[7])
1115
- redis.call('SADD', session .. 'cleared', moved)
1116
- redis.call('XADD', KEYS[1], field .. '-0',
1117
- 'id', ARGV[6], 'type', '${CLEAR_EVENT_TYPE}', 'payload', 'null',
1118
- 'created_at', ARGV[2], 'cause', '', 'lane', '')
1119
- local writes = { KEYS[5], cjson.encode({ ARGV[1], head + 1 }), COUNTER, field, CLEAR, field }
1120
- split_write(writes)
1121
- redis.call('MSET', unpack(writes))
1122
- put(mkey(field), field .. ':p', ARGV[2])
1123
- flush()
1124
- wake_add(field, ARGV[6], '${CLEAR_EVENT_TYPE}', 'null')
1125
- wake(ARGV[7], field, ARGV[2])
1126
- return { 'ok', '1', field, field }
1647
+ redis.call('UNLINK', READY)
1648
+ redis.call('XADD', LOG, field .. '-0',
1649
+ 'id', ARGV[8], 'type', CLEAR_TYPE, 'payload', 'null',
1650
+ 'created_at', ARGV[2], 'cause', '')
1651
+ redis.call('MSET', id_key(ARGV[8]), field, COUNTER, field, CLEAR, field)
1652
+ return { 'ok', '1', field, field, '1' }
1127
1653
  `
1128
1654
 
1129
1655
  /**
1130
1656
  * KEYS: counter. ARGV: page size.
1131
- * The next deletion batch: a moved log and up to a page of its [index, id]
1132
- * pairs, without payloads. Empty moved logs are dropped; with none left it
1133
- * sweeps the old checkpoints and returns nothing, so a retry finishes an
1134
- * interrupted deletion.
1657
+ * The next deletion batch: up to a page of [index, id] pairs of the log's
1658
+ * entries below the latest clear. With none left it sweeps what the old
1659
+ * lifetimes left and returns nothing, so a retry finishes an interrupted
1660
+ * deletion.
1135
1661
  */
1136
- const RECLAIM_BATCH_LUA = `
1137
- ${SWEEP_CHECKPOINTS_LUA}
1138
- local session = string.sub(KEYS[1], 1, -6)
1139
- local boundary = tonumber(redis.call('GET', session .. 'clear') or '0')
1662
+ const RECLAIM_BATCH_LUA = `${BASE_LUA}${SWEEP_LUA}
1663
+ local boundary = tonumber(redis.call('GET', CLEAR) or '0')
1140
1664
  if boundary == 0 then return {} end
1141
- for _, log in ipairs(redis.call('SMEMBERS', session .. 'cleared')) do
1142
- local entries = redis.call('XRANGE', log, '-', '+', 'COUNT', ARGV[1])
1143
- if #entries > 0 then
1144
- local batch = { log }
1145
- for _, entry in ipairs(entries) do
1146
- local fields = entry[2]
1147
- for i = 1, #fields, 2 do
1148
- if fields[i] == 'id' then
1149
- batch[#batch + 1] = string.match(entry[1], '^(%d+)')
1150
- batch[#batch + 1] = fields[i + 1]
1151
- break
1152
- end
1153
- end
1665
+ local entries = redis.call('XRANGE', LOG, '-', tostring(boundary - 1) .. '-0', 'COUNT', ARGV[1])
1666
+ if #entries == 0 then
1667
+ sweep(boundary)
1668
+ return {}
1669
+ end
1670
+ local batch = {}
1671
+ for _, entry in ipairs(entries) do
1672
+ local fields = entry[2]
1673
+ for i = 1, #fields, 2 do
1674
+ if fields[i] == 'id' then
1675
+ batch[#batch + 1] = string.match(entry[1], '^(%d+)')
1676
+ batch[#batch + 1] = fields[i + 1]
1677
+ break
1154
1678
  end
1155
- return batch
1156
1679
  end
1157
- redis.call('DEL', log)
1158
- redis.call('SREM', session .. 'cleared', log)
1159
1680
  end
1160
- sweep_checkpoints(session, boundary)
1161
- return {}
1681
+ return batch
1162
1682
  `
1163
1683
 
1164
1684
  /**
1165
- * KEYS: moved log, counter, metadata, metadata split, then one id ownership
1166
- * key per entry.
1167
- * ARGV: sessionId, event pin key prefix, n, then per entry: index, id.
1168
- * Deletes the listed entries below the latest clear and everything that
1169
- * names them. An id record survives when it names another event, so an id
1170
- * reused in the new lifetime keeps its owner. Returns [deleted, done].
1685
+ * KEYS: counter.
1686
+ * ARGV: sessionId, n, then per entry: index, id.
1687
+ * Deletes the listed entries below the latest clear and their id records.
1688
+ * An id record survives when it names another event, so an id reused in the
1689
+ * new lifetime keeps its owner. Returns [deleted, done].
1171
1690
  */
1172
- const RECLAIM_LUA = `
1173
- local COUNTER, META, SPLIT = KEYS[2], KEYS[3], KEYS[4]
1174
- ${META_LUA}
1175
- ${SWEEP_CHECKPOINTS_LUA}
1176
- local function lane_field(lane, suffix)
1177
- return '__lane:' .. tostring(string.len(lane)) .. ':' .. lane .. ':' .. suffix
1178
- end
1179
- local session = string.sub(COUNTER, 1, -6)
1180
- local counts = redis.call('MGET', COUNTER, SPLIT, CLEAR)
1181
- use_split(counts[2], counts[1], counts[3])
1182
- local fields, id_keys, indexes, stream_ids, pin_keys, deleted = {}, {}, {}, {}, {}, {}
1691
+ const RECLAIM_LUA = `${BASE_LUA}${SWEEP_LUA}
1692
+ local boundary = tonumber(redis.call('GET', CLEAR) or '0')
1693
+ local ids, indexes, stream_ids = {}, {}, {}
1183
1694
  -- A concurrent reclaim may list the same batch; only XDEL says who removed it.
1184
1695
  local removed = 0
1185
- for i = 1, tonumber(ARGV[3]) do
1186
- local field = ARGV[3 + (i - 1) * 2 + 1]
1696
+ for i = 1, tonumber(ARGV[2]) do
1697
+ local field = ARGV[2 + (i - 1) * 2 + 1]
1187
1698
  if tonumber(field) < boundary then
1188
- fields[#fields + 1] = field
1189
- id_keys[#fields] = KEYS[4 + i]
1190
- indexes[#fields] = tonumber(field)
1191
- stream_ids[#fields] = field .. '-0'
1192
- pin_keys[#fields] = ARGV[2] .. field
1193
- deleted[field] = true
1194
- end
1195
- end
1196
- if #fields > 0 then
1197
- local lanes, seen, pointers = read_fields(fields, { 'l' }), {}, {}
1198
- for _, field in ipairs(fields) do
1199
- local lane = lanes[field][1]
1200
- if lane and lane ~= '' and not seen[lane] then
1201
- seen[lane] = true
1202
- pointers[#pointers + 1] = lane_field(lane, 'head')
1203
- pointers[#pointers + 1] = lane_field(lane, 'tail')
1204
- end
1205
- end
1206
- if #pointers > 0 then
1207
- local values = redis.call('HMGET', META, unpack(pointers))
1208
- local stale = {}
1209
- for i = 1, #pointers do
1210
- if values[i] and deleted[values[i]] then stale[#stale + 1] = pointers[i] end
1211
- end
1212
- if #stale > 0 then redis.call('HDEL', META, unpack(stale)) end
1213
- end
1214
- local suffixes = { 'l', 'next', 'p', 'pa', 'ri', 'ch', 'ce', 'f', 'fc', 'lc', 'lf', 'lfa', 'd', 'a', 'e' }
1215
- local columns, order = {}, {}
1216
- for _, field in ipairs(fields) do
1217
- local key = mkey(field)
1218
- if not columns[key] then
1219
- columns[key] = {}
1220
- order[#order + 1] = key
1221
- end
1222
- for _, suffix in ipairs(suffixes) do
1223
- columns[key][#columns[key] + 1] = field .. ':' .. suffix
1224
- end
1699
+ ids[#ids + 1] = ARGV[2 + (i - 1) * 2 + 2]
1700
+ indexes[#ids] = tonumber(field)
1701
+ stream_ids[#ids] = field .. '-0'
1225
1702
  end
1226
- for _, key in ipairs(order) do sliced('HDEL', key, columns[key]) end
1227
- sliced('DEL', nil, pin_keys)
1228
- local owners, stale_ids = sliced('MGET', nil, id_keys), {}
1229
- for i = 1, #fields do
1230
- if owners[i] then
1231
- local rec = cjson.decode(owners[i])
1232
- if rec[1] == ARGV[1] and tonumber(rec[2]) == indexes[i] then
1233
- stale_ids[#stale_ids + 1] = id_keys[i]
1234
- end
1703
+ end
1704
+ if #ids > 0 then
1705
+ local keys = {}
1706
+ for i = 1, #ids do keys[i] = id_key(ids[i]) end
1707
+ local owners, stale = sliced('MGET', nil, keys), {}
1708
+ for i = 1, #ids do
1709
+ if owners[i] and tonumber(owners[i]) == indexes[i] then
1710
+ stale[#stale + 1] = keys[i]
1235
1711
  end
1236
1712
  end
1237
- if #stale_ids > 0 then sliced('DEL', nil, stale_ids) end
1238
- removed = redis.call('XDEL', KEYS[1], unpack(stream_ids))
1713
+ sliced('DEL', nil, stale)
1714
+ removed = redis.call('XDEL', LOG, unpack(stream_ids))
1239
1715
  end
1240
1716
  local done = 0
1241
- if redis.call('XLEN', KEYS[1]) == 0 then
1242
- redis.call('DEL', KEYS[1])
1243
- redis.call('SREM', session .. 'cleared', KEYS[1])
1244
- if redis.call('SCARD', session .. 'cleared') == 0 then
1245
- sweep_checkpoints(session, boundary)
1246
- done = 1
1247
- end
1717
+ if #redis.call('XRANGE', LOG, '-', tostring(boundary - 1) .. '-0', 'COUNT', 1) == 0 then
1718
+ sweep(boundary)
1719
+ done = 1
1248
1720
  end
1249
1721
  return { removed, done }
1250
1722
  `
1251
1723
 
1252
1724
  // ── wire helpers ─────────────────────────────────────────────────────
1253
1725
 
1726
+ const WAKE_PAYLOAD_LIMIT = 32_768
1727
+ const encoder = new TextEncoder()
1728
+
1729
+ /**
1730
+ * A wake-up announcing events through `frontier`. It carries them, so a
1731
+ * caught-up reader applies them without reading the stream:
1732
+ * `{"f":frontier,"t":createdAt,"e":[[index,id,type,payload],...]}`, with
1733
+ * payloads inlined as their stored JSON text. One over WAKE_PAYLOAD_LIMIT
1734
+ * bytes, or one without entries, announces only `{"f":frontier}`, and
1735
+ * readers catch up with XRANGE.
1736
+ */
1737
+ const wakeMessage = (
1738
+ frontier: number,
1739
+ createdAt: number,
1740
+ entries: ReadonlyArray<
1741
+ readonly [index: number, id: string, type: string, payloadJson: string]
1742
+ >,
1743
+ ): string => {
1744
+ const bare = `{"f":${frontier}}`
1745
+ if (entries.length === 0) return bare
1746
+ const carried = `{"f":${frontier},"t":${createdAt},"e":[${entries
1747
+ .map(
1748
+ ([index, id, type, payload]) =>
1749
+ `[${index},${JSON.stringify(id)},${JSON.stringify(type)},${payload}]`,
1750
+ )
1751
+ .join(',')}]}`
1752
+ return encoder.encode(carried).byteLength > WAKE_PAYLOAD_LIMIT
1753
+ ? bare
1754
+ : carried
1755
+ }
1756
+
1254
1757
  export type StreamEntry = [id: string, fields: string[]]
1255
1758
 
1256
1759
  const fieldMap = (fields: string[]): Record<string, string> => {
@@ -1276,6 +1779,30 @@ const toEvent = (sessionId: string, entry: StreamEntry): Event => {
1276
1779
  }
1277
1780
  }
1278
1781
 
1782
+ /**
1783
+ * A stream entry's causal edge. Releases before consumers (0.17 to 0.20)
1784
+ * recorded only the parent's index and attempt: their edge reads as the
1785
+ * handlers table's, `a2.handlers`, as Postgres's upgrade names it, with no
1786
+ * lane, since nothing reads an earlier release's lane.
1787
+ */
1788
+ const decodeStreamCause = (raw: string): EventCause | null => {
1789
+ const parsed = JSON.parse(raw) as { consumer?: unknown }
1790
+ return decodeCause(
1791
+ parsed.consumer === undefined
1792
+ ? { consumer: 'a2.handlers', lane: null, ...parsed }
1793
+ : parsed,
1794
+ )
1795
+ }
1796
+
1797
+ /** A stream entry with its causal edge. */
1798
+ const toStoredEvent = (sessionId: string, entry: StreamEntry): StoredEvent => {
1799
+ const cause = fieldMap(entry[1])['cause']
1800
+ return Object.assign(toEvent(sessionId, entry), {
1801
+ cause:
1802
+ cause === undefined || cause === '' ? null : decodeStreamCause(cause),
1803
+ })
1804
+ }
1805
+
1279
1806
  type ByteStreamEntry = [Uint8Array, Uint8Array[]]
1280
1807
 
1281
1808
  const fieldBytes = (name: string): Uint8Array => new TextEncoder().encode(name)
@@ -1338,75 +1865,107 @@ const toSerializedEvent = (
1338
1865
 
1339
1866
  const json = (value: unknown): string => JSON.stringify(value) ?? 'null'
1340
1867
 
1341
- const toCause = (raw: string | undefined): EventCause | null => {
1342
- if (raw === '' || raw === undefined) return null
1343
- const value = JSON.parse(raw) as {
1344
- index?: unknown
1345
- attempt?: unknown
1346
- batchSize?: unknown
1868
+ /** How scripts name a cursor within its consumer: `#`, or `L` and its lane. */
1869
+ const tagOf = (lane: string | null): string =>
1870
+ lane === null ? '#' : `L${lane}`
1871
+
1872
+ /** The lane a cursor tag names: null for an event cursor. */
1873
+ const laneOf = (tag: string): string | null =>
1874
+ tag.startsWith('#') ? null : tag.slice(1)
1875
+
1876
+ const targetsArg = (targets: readonly EventTarget[] | undefined): string =>
1877
+ targets === undefined || targets.length === 0
1878
+ ? ''
1879
+ : JSON.stringify(
1880
+ targets.map((target) => [target.consumer, tagOf(target.lane)]),
1881
+ )
1882
+
1883
+ /** A log consumer's types, as scripts read them; '*' takes every type. */
1884
+ const logTypesArg = (types: readonly string[] | null): string =>
1885
+ types === null ? '*' : JSON.stringify(types)
1886
+
1887
+ /** The served consumers, as scripts read them; '' serves every consumer. */
1888
+ const servedArg = (consumers: readonly StoreConsumer[] | undefined): string =>
1889
+ consumers === undefined
1890
+ ? ''
1891
+ : JSON.stringify(
1892
+ consumers.map((consumer) =>
1893
+ consumer.log
1894
+ ? [
1895
+ consumer.name,
1896
+ consumer.log.lane,
1897
+ logTypesArg(consumer.log.types),
1898
+ consumer.log.backfill ? 1 : 0,
1899
+ ]
1900
+ : [consumer.name],
1901
+ ),
1902
+ )
1903
+
1904
+ type Column = string | number | null | undefined
1905
+
1906
+ /** A head's dispatch state from a row's HEAD_COLUMNS. */
1907
+ const dispatchOf = (
1908
+ consumer: string,
1909
+ lane: string | null,
1910
+ columns: readonly Column[] | null | undefined,
1911
+ ): HeadDispatch => {
1912
+ const column = (at: number): string | null => {
1913
+ const value = columns?.[at]
1914
+ return value === null || value === undefined ? null : String(value)
1347
1915
  }
1348
- if (
1349
- !Number.isInteger(value.index) ||
1350
- Number(value.index) < 1 ||
1351
- !Number.isInteger(value.attempt) ||
1352
- Number(value.attempt) < 1
1353
- ) {
1354
- throw new TypeError('stored event has an invalid cause')
1916
+ const date = (at: number): Date | null => {
1917
+ const value = column(at)
1918
+ return value === null ? null : new Date(Number(value))
1355
1919
  }
1356
- if (
1357
- value.batchSize !== undefined &&
1358
- (!Number.isInteger(value.batchSize) || Number(value.batchSize) < 1)
1359
- ) {
1360
- throw new TypeError('stored event has an invalid cause')
1920
+ const count = (at: number): number | null => {
1921
+ const value = column(at)
1922
+ return value === null ? null : Number(value)
1361
1923
  }
1362
1924
  return {
1363
- index: Number(value.index),
1364
- attempt: Number(value.attempt),
1365
- ...(value.batchSize === undefined
1366
- ? {}
1367
- : { batchSize: Number(value.batchSize) }),
1925
+ consumer,
1926
+ lane,
1927
+ firstClaimedAt: date(1),
1928
+ lastClaimedAt: date(2),
1929
+ attemptCount: count(0) ?? 0,
1930
+ claimHolder: column(3),
1931
+ claimExpiresAt: date(4),
1932
+ failureCount: count(5) ?? 0,
1933
+ lastFailedAt: date(6),
1934
+ lastFailedAttempt: count(7),
1935
+ lastError: column(8),
1936
+ failedAt: date(9),
1368
1937
  }
1369
1938
  }
1370
1939
 
1371
- type MetaSuffix = (typeof STORED_EVENT_META_SUFFIXES)[number]
1372
-
1373
- /** A stream entry joined with its metadata columns; absent columns are unset. */
1374
- const toStoredEvent = (
1375
- sessionId: string,
1940
+ type HeadTuple = [
1941
+ consumer: string,
1942
+ tag: string,
1943
+ index: string,
1944
+ columns: Column[],
1376
1945
  entry: StreamEntry,
1377
- column: (suffix: MetaSuffix) => string | undefined,
1378
- ): StoredEvent => {
1379
- const fields = fieldMap(entry[1])
1380
- const date = (suffix: MetaSuffix): Date | null => {
1381
- const value = column(suffix)
1382
- return value === undefined ? null : new Date(Number(value))
1383
- }
1384
- const count = (suffix: MetaSuffix): number | null => {
1385
- const value = column(suffix)
1386
- return value === undefined ? null : Number(value)
1387
- }
1388
- const returnedEventIds = column('ri')
1389
- return Object.assign(toEvent(sessionId, entry), {
1390
- cause: toCause(fields['cause']),
1391
- lane: fields['lane'] === '' ? null : fields['lane']!,
1392
- processedAt: date('p'),
1393
- processedByAttempt: count('pa'),
1394
- returnedEventIds:
1395
- returnedEventIds === undefined
1396
- ? null
1397
- : decodeReturnedEventIds(returnedEventIds),
1398
- firstClaimedAt: date('fc'),
1399
- lastClaimedAt: date('lc'),
1400
- attemptCount: count('d') ?? 0,
1401
- claimHolder: column('ch') ?? null,
1402
- claimExpiresAt: date('ce'),
1403
- failureCount: count('a') ?? 0,
1404
- lastFailedAt: date('lf'),
1405
- lastFailedAttempt: count('lfa'),
1406
- lastError: column('e') ?? null,
1407
- failedAt: date('f'),
1408
- })
1409
- }
1946
+ ]
1947
+
1948
+ const toHeadEvent = (sessionId: string, head: HeadTuple): HeadEvent => ({
1949
+ ...toStoredEvent(sessionId, head[4]),
1950
+ ...dispatchOf(head[0], laneOf(head[1]), head[3]),
1951
+ })
1952
+
1953
+ const compareText = (a: string, b: string): number =>
1954
+ a < b ? -1 : a > b ? 1 : 0
1955
+
1956
+ const byHead = (a: HeadRef, b: HeadRef): number =>
1957
+ a.index - b.index || compareText(a.consumer, b.consumer)
1958
+
1959
+ /** Consumer, then lane, with concurrent cursors last by head. */
1960
+ const byCursor = (a: StoredCursor, b: StoredCursor): number =>
1961
+ compareText(a.consumer, b.consumer) ||
1962
+ (a.lane === null
1963
+ ? b.lane === null
1964
+ ? (a.head ?? 0) - (b.head ?? 0)
1965
+ : 1
1966
+ : b.lane === null
1967
+ ? -1
1968
+ : compareText(a.lane, b.lane))
1410
1969
 
1411
1970
  const toClaimAvailableResult = (
1412
1971
  sessionId: string,
@@ -1415,57 +1974,13 @@ const toClaimAvailableResult = (
1415
1974
  const outcome = String(reply[0])
1416
1975
  if (outcome === 'claimed') {
1417
1976
  const count = Number(reply[1])
1418
- const events: StoredEvent[] = []
1419
- for (let i = 0; i < count; i++) {
1420
- const offset = 2 + i * 11
1421
- const entry = reply[offset + 9] as StreamEntry
1422
- const event = toEvent(sessionId, entry)
1423
- const fields = fieldMap(entry[1])
1424
- events.push(
1425
- Object.assign(event, {
1426
- cause: toCause(fields['cause']),
1427
- lane: fields['lane'] === '' ? null : fields['lane']!,
1428
- processedAt: null,
1429
- processedByAttempt: null,
1430
- returnedEventIds: null,
1431
- firstClaimedAt:
1432
- reply[offset + 3] === null
1433
- ? null
1434
- : new Date(Number(reply[offset + 3])),
1435
- lastClaimedAt:
1436
- reply[offset + 4] === null
1437
- ? null
1438
- : new Date(Number(reply[offset + 4])),
1439
- attemptCount: Number(reply[offset]),
1440
- claimHolder:
1441
- reply[offset + 7] === null ? null : String(reply[offset + 7]),
1442
- claimExpiresAt:
1443
- reply[offset + 8] === null
1444
- ? null
1445
- : new Date(Number(reply[offset + 8])),
1446
- failureCount: Number(reply[offset + 1]),
1447
- lastFailedAt:
1448
- reply[offset + 5] === null
1449
- ? null
1450
- : new Date(Number(reply[offset + 5])),
1451
- lastFailedAttempt:
1452
- reply[offset + 6] === null ? null : Number(reply[offset + 6]),
1453
- lastError:
1454
- reply[offset + 2] === null ? null : String(reply[offset + 2]),
1455
- failedAt:
1456
- reply[offset + 10] === null
1457
- ? null
1458
- : new Date(Number(reply[offset + 10])),
1459
- }),
1460
- )
1461
- }
1462
- const clearIndex = Number(reply[2 + count * 11] ?? 0)
1977
+ const events = (reply.slice(2, 2 + count) as HeadTuple[])
1978
+ .map((head) => toHeadEvent(sessionId, head))
1979
+ .toSorted(byHead)
1980
+ const clearIndex = Number(reply[2 + count] ?? 0)
1463
1981
  if (clearIndex > 0)
1464
1982
  for (const event of events) event.clearIndex = clearIndex
1465
- return {
1466
- outcome: 'claimed',
1467
- events,
1468
- }
1983
+ return { outcome: 'claimed', events, head: Number(reply[3 + count]) }
1469
1984
  }
1470
1985
  if (outcome === 'busy') {
1471
1986
  return { outcome, retryAt: new Date(Number(reply[1])) }
@@ -1496,9 +2011,9 @@ const toIndex = ({
1496
2011
  }
1497
2012
 
1498
2013
  /**
1499
- * A range read that met a gap. A clear moves the old lifetime's log aside, so
1500
- * a read whose bounds predate it finds nothing where it expected events;
1501
- * anywhere else a gap is corruption.
2014
+ * A range read that met a gap. Reclaim deletes the entries below a clear, so
2015
+ * a read whose bounds predate the clear finds nothing where it expected
2016
+ * events; anywhere else a gap is corruption.
1502
2017
  */
1503
2018
  const rangeGap = (sessionId: string, message: string): A2Error =>
1504
2019
  Object.assign(
@@ -1538,123 +2053,185 @@ const wrap = async <T>(fn: () => Promise<T>): Promise<T> => {
1538
2053
  }
1539
2054
  }
1540
2055
 
2056
+ /**
2057
+ * A session's key. Every key of a session shares one hash tag, so a script
2058
+ * locks only its session on Upstash and stays in one slot on a cluster. The
2059
+ * tag is the session id as JSON with its braces escaped: it can't end early,
2060
+ * and no session's tag spells another's.
2061
+ */
2062
+ export const redisSessionKey = ({
2063
+ prefix,
2064
+ sessionId,
2065
+ suffix,
2066
+ }: {
2067
+ prefix: string
2068
+ sessionId: string
2069
+ suffix: string
2070
+ }): string =>
2071
+ `${prefix}:{${JSON.stringify(sessionId).replaceAll('{', '\\u007b').replaceAll('}', '\\u007d')}}:${suffix}`
2072
+
1541
2073
  export function createRedisStoreCore(
1542
2074
  options: RedisStoreCoreOptions,
1543
2075
  ): RedisStoreCore {
1544
2076
  const { call, callBytes, clock } = options
1545
2077
  const generateId = options.ids
1546
2078
  const prefix = options.keyPrefix
2079
+ if (prefix.includes('{') || prefix.includes('}')) {
2080
+ throw new TypeError(
2081
+ 'a redis keyPrefix cannot contain braces: they would set the hash tag of every key',
2082
+ )
2083
+ }
1547
2084
 
1548
- const storeKey = (s: string): string => `${prefix}:${s}:log`
1549
- const countKey = (s: string): string => `${prefix}:${s}:count`
1550
- const metaKey = (s: string): string => `${prefix}:${s}:meta`
1551
- const metaChunkKey = (s: string, index: number): string =>
1552
- `${metaKey(s)}:${Math.floor((index - 1) / META_CHUNK_SIZE)}`
1553
- const metaSplitKey = (s: string): string => `${prefix}:${s}:meta-split`
1554
- const pendingKey = (s: string): string => `${prefix}:${s}:pending`
1555
- const readyKey = (s: string): string => `${prefix}:${s}:ready`
1556
- const snapKey = (s: string, r: string): string => `${prefix}:${s}:snap:${r}`
2085
+ const sessionKey = (sessionId: string, suffix: string): string =>
2086
+ redisSessionKey({ prefix, sessionId, suffix })
2087
+ const storeKey = (s: string): string => sessionKey(s, 'log')
2088
+ const countKey = (s: string): string => sessionKey(s, 'count')
2089
+ // Scripts derive the session's other keys from the counter; keep them in step.
2090
+ const clearKey = (s: string): string => sessionKey(s, 'clear')
2091
+ const schemaKey = `${prefix}:schema`
2092
+ const snapKey = (s: string, r: string): string => sessionKey(s, `snap:${r}`)
1557
2093
  const snapHistoryKey = (s: string, r: string): string =>
1558
- `${prefix}:${s}:snap-history:${r}`
1559
- const snapshotEventPinsKey = (s: string, index: number): string =>
1560
- `${prefix}:${s}:snapshot-pins:${index}`
1561
- const snapshotEventPinsPrefix = (s: string): string =>
1562
- `${prefix}:${s}:snapshot-pins:`
1563
- const snapshotPinCountsKey = (s: string): string =>
1564
- `${prefix}:${s}:snapshot-pin-counts`
1565
- const presenceKey = (s: string): string => `${prefix}:${s}:presence`
1566
- // Scripts derive this from the counter key; keep the two in step.
1567
- const clearKey = (s: string): string => `${prefix}:${s}:clear`
1568
- // One ownership key per event id; reclaim deletes those naming cleared events.
1569
- // Session ids are raw and may contain colons, so a raw id could name a
1570
- // session key (`ids:count`). A JSON string with escaped colons starts with
1571
- // a quote and has no colon, so it never matches a session key's suffix.
1572
- const idKey = (id: string): string =>
1573
- `${prefix}:ids:${JSON.stringify(id).replaceAll(':', '\\u003a')}`
2094
+ sessionKey(s, `snap-history:${r}`)
2095
+ const presenceKey = (s: string): string => sessionKey(s, 'presence')
2096
+ // Channels aren't keys; the notify store subscribes by these names.
2097
+ const notifyChannel = (s: string): string => `${prefix}:${s}:notify`
2098
+ const presenceChannel = (s: string): string => `${prefix}:${s}:presence`
2099
+
2100
+ const layoutError = (): A2Error =>
2101
+ new A2Error(
2102
+ 'STORE_NOT_CONFIGURED',
2103
+ `the redis keys under prefix '${prefix}' hold an A2 layout other than '${REDIS_LAYOUT}', from another release; point the store at a fresh keyPrefix, or give the contract a new name so it starts new sessions`,
2104
+ )
2105
+
2106
+ /**
2107
+ * Once per store, before its first script: claim or check the prefix's
2108
+ * layout, and ask the server whether scripts may lock only their keys.
2109
+ * Only the server's own rejection of the flag means no; a transport error
2110
+ * fails this attempt, and the next operation asks again.
2111
+ */
2112
+ const prepared = retryableLazy(async () => {
2113
+ const [, keyLocking] = await Promise.all([
2114
+ (async () => {
2115
+ await call('SET', schemaKey, REDIS_LAYOUT, 'NX')
2116
+ if ((await call('GET', schemaKey)) !== REDIS_LAYOUT) throw layoutError()
2117
+ })(),
2118
+ call('EVAL', `${KEY_LOCKING}\nreturn 1`, 1, schemaKey).then(
2119
+ () => true,
2120
+ (error: unknown) => {
2121
+ if (SHEBANG_REJECTED.test(String((error as Error)?.message)))
2122
+ return false
2123
+ throw error
2124
+ },
2125
+ ),
2126
+ ])
2127
+ return { keyLocking }
2128
+ })
2129
+
2130
+ /** Runs a script; a `readOnly` one takes Upstash's shared lock. */
2131
+ const runScript = async ({
2132
+ script,
2133
+ keys,
2134
+ args,
2135
+ readOnly,
2136
+ }: {
2137
+ script: string
2138
+ keys: string[]
2139
+ args: Array<string | number>
2140
+ readOnly: boolean
2141
+ }): Promise<unknown> => {
2142
+ const { keyLocking } = await prepared.get()
2143
+ let source = script
2144
+ if (keyLocking)
2145
+ source = `${readOnly ? READ_ONLY_KEY_LOCKING : KEY_LOCKING}\n${script}`
2146
+ return call('EVAL', source, keys.length, ...keys, ...args)
2147
+ }
1574
2148
 
2149
+ /** Runs a writing script over a session: KEYS start with its counter. */
1575
2150
  const evalScript = (
1576
2151
  script: string,
2152
+ sessionId: string,
1577
2153
  keys: string[],
1578
2154
  args: Array<string | number>,
1579
- ): Promise<unknown> => call('EVAL', script, keys.length, ...keys, ...args)
2155
+ ): Promise<unknown> =>
2156
+ runScript({
2157
+ script,
2158
+ keys: [countKey(sessionId), ...keys],
2159
+ args,
2160
+ readOnly: false,
2161
+ })
2162
+
2163
+ /**
2164
+ * Publishes a write's wake-up after its script, never inside it: a script
2165
+ * holds its lock while PUBLISH waits on subscribers. Wake-ups are hints (a
2166
+ * lost one costs subscribers a catch-up read, never an event), so a failed
2167
+ * publish is dropped and the write waits for it at most
2168
+ * NOTIFY_TIMINGS.publishMs.
2169
+ */
2170
+ const publish = async (channel: string, message: string): Promise<void> => {
2171
+ let timer: ReturnType<typeof setTimeout> | undefined
2172
+ await Promise.race([
2173
+ call('PUBLISH', channel, message).catch(() => undefined),
2174
+ new Promise<void>((resolve) => {
2175
+ timer = setTimeout(resolve, NOTIFY_TIMINGS.publishMs)
2176
+ ;(timer as { unref?: () => void }).unref?.()
2177
+ }),
2178
+ ])
2179
+ clearTimeout(timer)
2180
+ }
1580
2181
 
1581
- const readRangeBounds = async (
2182
+ /**
2183
+ * Runs a write that may need a type's index to reach further back first.
2184
+ * Such a write returns ['index', type, after] having written nothing, and
2185
+ * runs again once INDEX_TYPE_LUA has indexed that history, a page per call.
2186
+ */
2187
+ const indexing = async (
2188
+ sessionId: string,
2189
+ run: () => Promise<unknown>,
2190
+ ): Promise<unknown[]> => {
2191
+ for (;;) {
2192
+ // oxlint-disable-next-line no-await-in-loop -- each run follows the indexing it asked for
2193
+ const reply = (await run()) as unknown[]
2194
+ if (reply[0] !== 'index') return reply
2195
+ for (;;) {
2196
+ // oxlint-disable-next-line no-await-in-loop -- each page lowers the index's floor
2197
+ const [done] = (await evalScript(
2198
+ INDEX_TYPE_LUA,
2199
+ sessionId,
2200
+ [],
2201
+ [String(reply[1]), Number(reply[2])],
2202
+ )) as [string]
2203
+ if (done === '1') break
2204
+ }
2205
+ }
2206
+ }
2207
+
2208
+ /** The frontier and clear marker, in one MGET. */
2209
+ const readBounds = async (
1582
2210
  sessionId: string,
1583
- ): Promise<{ frontier: number; metaSplit: number; clearIndex: number }> => {
1584
- const [count, split, cleared] = (await call(
2211
+ ): Promise<{ frontier: number; clearIndex: number }> => {
2212
+ await prepared.get()
2213
+ const [count, cleared] = (await call(
1585
2214
  'MGET',
1586
2215
  countKey(sessionId),
1587
- metaSplitKey(sessionId),
1588
2216
  clearKey(sessionId),
1589
- )) as [unknown, unknown, unknown]
1590
- const frontier = toIndex({ raw: count, sessionId, name: 'event count' })
2217
+ )) as [unknown, unknown]
1591
2218
  return {
1592
- frontier,
1593
- metaSplit:
1594
- split === null
1595
- ? frontier
1596
- : toIndex({ raw: split, sessionId, name: 'metadata split' }),
2219
+ // A collapsed session keeps only its clear marker.
2220
+ frontier: toIndex({
2221
+ raw: count ?? cleared,
2222
+ sessionId,
2223
+ name: 'event count',
2224
+ }),
1597
2225
  clearIndex: toIndex({ raw: cleared, sessionId, name: 'clear boundary' }),
1598
2226
  }
1599
2227
  }
1600
2228
 
1601
- const readRangePage = async ({
1602
- sessionId,
1603
- afterIndex,
1604
- throughIndex,
1605
- metaSplit,
1606
- }: {
1607
- sessionId: string
1608
- afterIndex: number
1609
- throughIndex: number
1610
- metaSplit: number
1611
- }): Promise<StoredEvent[]> => {
1612
- if (throughIndex <= afterIndex) return []
1613
- const entries = (await call(
1614
- 'XRANGE',
1615
- storeKey(sessionId),
1616
- `${afterIndex + 1}-0`,
1617
- `${throughIndex}-0`,
1618
- 'COUNT',
1619
- READ_PAGE_SIZE,
1620
- )) as StreamEntry[]
1621
- if (entries.length === 0) return []
1622
-
1623
- const fieldsByKey = new Map<string, string[]>()
1624
- for (const entry of entries) {
1625
- const index = entryIndex(entry[0])
1626
- const key =
1627
- index <= metaSplit ? metaKey(sessionId) : metaChunkKey(sessionId, index)
1628
- const fields = fieldsByKey.get(key) ?? []
1629
- fields.push(
1630
- ...STORED_EVENT_META_SUFFIXES.map((suffix) => `${index}:${suffix}`),
1631
- )
1632
- fieldsByKey.set(key, fields)
1633
- }
1634
- const lookups = [...fieldsByKey]
1635
- const results = await Promise.allSettled(
1636
- lookups.map(([key, fields]) => call('HMGET', key, ...fields)),
1637
- )
1638
- const meta: Record<string, string> = {}
1639
- for (let position = 0; position < lookups.length; position += 1) {
1640
- const result = results[position]!
1641
- if (result.status === 'rejected') throw result.reason
1642
- const fields = lookups[position]![1]
1643
- const values = (result.value as Array<string | null> | null) ?? []
1644
- for (let i = 0; i < fields.length; i += 1) {
1645
- const value = values[i]
1646
- if (value !== null && value !== undefined) {
1647
- meta[fields[i]!] = value
1648
- }
1649
- }
1650
- }
1651
- return entries.map((entry) => {
1652
- const idx = entryIndex(entry[0])
1653
- return toStoredEvent(
1654
- sessionId,
1655
- entry,
1656
- (suffix) => meta[`${idx}:${suffix}`],
1657
- )
2229
+ const readClearIndex = async (sessionId: string): Promise<number> => {
2230
+ await prepared.get()
2231
+ return toIndex({
2232
+ raw: await call('GET', clearKey(sessionId)),
2233
+ sessionId,
2234
+ name: 'clear boundary',
1658
2235
  })
1659
2236
  }
1660
2237
 
@@ -1744,11 +2321,8 @@ export function createRedisStoreCore(
1744
2321
  ): Promise<{ rows: StoredEvent[]; clearIndex: number }> => {
1745
2322
  if (throughIndex !== undefined && throughIndex <= afterIndex)
1746
2323
  return { rows: [], clearIndex: 0 }
1747
- const {
1748
- frontier: currentFrontier,
1749
- metaSplit,
1750
- clearIndex,
1751
- } = await readRangeBounds(sessionId)
2324
+ const { frontier: currentFrontier, clearIndex } =
2325
+ await readBounds(sessionId)
1752
2326
  const frontier = Math.min(throughIndex ?? currentFrontier, currentFrontier)
1753
2327
  // Reads start at the latest clear marker.
1754
2328
  const after = Math.max(afterIndex, clearIndex - 1)
@@ -1758,15 +2332,18 @@ export function createRedisStoreCore(
1758
2332
  let nextIndex = after + 1
1759
2333
  while (nextIndex <= frontier) {
1760
2334
  // oxlint-disable-next-line no-await-in-loop -- each page advances the dense frozen range
1761
- const page = await readRangePage({
1762
- sessionId,
1763
- afterIndex: nextIndex - 1,
1764
- throughIndex: frontier,
1765
- metaSplit,
1766
- })
1767
- if (page.length === 0) {
2335
+ const entries = (await call(
2336
+ 'XRANGE',
2337
+ storeKey(sessionId),
2338
+ `${nextIndex}-0`,
2339
+ `${frontier}-0`,
2340
+ 'COUNT',
2341
+ READ_PAGE_SIZE,
2342
+ )) as StreamEntry[]
2343
+ if (entries.length === 0) {
1768
2344
  throw rangeGap(sessionId, `ended before index ${frontier}`)
1769
2345
  }
2346
+ const page = entries.map((entry) => toStoredEvent(sessionId, entry))
1770
2347
  for (let position = 0; position < page.length; position += 1) {
1771
2348
  if (page[position]!.index !== nextIndex + position) {
1772
2349
  throw rangeGap(sessionId, 'is not contiguous')
@@ -1836,62 +2413,44 @@ export function createRedisStoreCore(
1836
2413
  type: e.type,
1837
2414
  payload: e.payload,
1838
2415
  cause: e.cause ? { ...e.cause } : null,
1839
- lane: e.lane ?? null,
1840
- settled: e.settled === true,
2416
+ targets: e.targets,
1841
2417
  }))
1842
- const args: Array<string | number> = [
1843
- sessionId,
1844
- now,
1845
- withIds.length,
1846
- ...withIds.flatMap((e) => [
1847
- e.id,
1848
- e.type,
1849
- json(e.payload),
1850
- e.cause ? json(e.cause) : '',
1851
- e.lane ?? '',
1852
- e.settled ? '1' : '0',
1853
- ]),
1854
- `${prefix}:${sessionId}:notify`,
1855
- appendOptions?.originIndex ?? '',
1856
- ]
1857
- const reply = (await evalScript(
1858
- APPEND_LUA,
1859
- [
1860
- storeKey(sessionId),
1861
- countKey(sessionId),
1862
- metaKey(sessionId),
1863
- pendingKey(sessionId),
1864
- readyKey(sessionId),
1865
- metaSplitKey(sessionId),
1866
- ...withIds.map((e) => idKey(e.id)),
1867
- ],
1868
- args,
2418
+ const payloads = withIds.map((e) => json(e.payload))
2419
+ const reply = (await indexing(sessionId, () =>
2420
+ evalScript(
2421
+ APPEND_LUA,
2422
+ sessionId,
2423
+ [],
2424
+ [
2425
+ sessionId,
2426
+ now,
2427
+ withIds.length,
2428
+ servedArg(appendOptions?.consumers),
2429
+ appendOptions?.originIndex ?? '',
2430
+ ...withIds.flatMap((e, i) => [
2431
+ e.id,
2432
+ e.type,
2433
+ payloads[i]!,
2434
+ e.cause ? json(e.cause) : '',
2435
+ targetsArg(e.targets),
2436
+ ]),
2437
+ ],
2438
+ ),
1869
2439
  )) as string[]
1870
2440
 
1871
2441
  const [verdict] = reply
1872
2442
  if (verdict === 'empty') {
1873
2443
  return { events: [], hasPending: reply[1] === '1' }
1874
2444
  }
1875
- if (verdict === 'foreign') {
1876
- throw new A2Error(
1877
- 'PARTIAL_DUPLICATE_BATCH',
1878
- `event id '${reply[1]}' already exists in another session`,
1879
- )
1880
- }
1881
- if (verdict === 'no_parent') {
1882
- throw new TypeError(
1883
- `no event at index ${reply[1]} in session '${sessionId}'`,
1884
- )
1885
- }
1886
2445
  if (verdict === 'cleared') {
1887
2446
  throw sessionClearedError(
1888
2447
  `scheduled work from event ${reply[1]} was cleared in session '${sessionId}'`,
1889
2448
  )
1890
2449
  }
1891
2450
  if (verdict === 'superseded') {
1892
- throw new A2Error(
1893
- 'SUPERSEDED_ATTEMPT',
1894
- `a newer attempt owns event ${reply[1]} in session '${sessionId}'`,
2451
+ throw supersededAttemptError(
2452
+ sessionId,
2453
+ withIds[Number(reply[1]) - 1]!.cause!,
1895
2454
  )
1896
2455
  }
1897
2456
  if (verdict === 'partial') {
@@ -1918,12 +2477,23 @@ export function createRedisStoreCore(
1918
2477
  `session '${sessionId}' was cleared while an append retry replayed; retry it`,
1919
2478
  )
1920
2479
  }
1921
- return {
1922
- events: idempotentReplay(events, existing),
1923
- hasPending,
1924
- }
2480
+ const replayed = idempotentReplay(events, existing)
2481
+ // The first attempt may have committed without publishing.
2482
+ await publish(
2483
+ notifyChannel(sessionId),
2484
+ wakeMessage(firstIndex + events.length - 1, now, []),
2485
+ )
2486
+ return { events: replayed, hasPending }
1925
2487
  }
1926
2488
  const base = Number(reply[1]) - 1
2489
+ await publish(
2490
+ notifyChannel(sessionId),
2491
+ wakeMessage(
2492
+ base + withIds.length,
2493
+ now,
2494
+ withIds.map((e, i) => [base + 1 + i, e.id, e.type, payloads[i]!]),
2495
+ ),
2496
+ )
1927
2497
  return {
1928
2498
  events: withIds.map((e, i) => ({
1929
2499
  id: e.id,
@@ -1933,26 +2503,39 @@ export function createRedisStoreCore(
1933
2503
  sessionId,
1934
2504
  createdAt: new Date(now),
1935
2505
  cause: e.cause,
1936
- lane: e.lane,
1937
- processedAt: e.settled ? new Date(now) : null,
1938
- processedByAttempt: null,
1939
- returnedEventIds: null,
1940
- firstClaimedAt: null,
1941
- lastClaimedAt: null,
1942
- attemptCount: 0,
1943
- claimHolder: null,
1944
- claimExpiresAt: null,
1945
- failureCount: 0,
1946
- lastFailedAt: null,
1947
- lastFailedAttempt: null,
1948
- lastError: null,
1949
- failedAt: null,
1950
2506
  })),
1951
2507
  hasPending: reply[2] === '1',
1952
2508
  }
1953
2509
  })
1954
2510
  },
1955
2511
 
2512
+ async cursors(sessionId) {
2513
+ return wrap(async () => {
2514
+ const reply = (await runScript({
2515
+ script: CURSORS_LUA,
2516
+ keys: [countKey(sessionId)],
2517
+ args: [],
2518
+ readOnly: true,
2519
+ })) as Array<
2520
+ [
2521
+ consumer: string,
2522
+ tag: string,
2523
+ position: string,
2524
+ head: string | null,
2525
+ columns: Column[],
2526
+ ]
2527
+ >
2528
+ return reply
2529
+ .map(([consumer, tag, position, head, columns]): StoredCursor =>
2530
+ Object.assign(dispatchOf(consumer, laneOf(tag), columns), {
2531
+ position: Number(position),
2532
+ head: head === null ? null : Number(head),
2533
+ }),
2534
+ )
2535
+ .toSorted(byCursor)
2536
+ })
2537
+ },
2538
+
1956
2539
  async read(sessionId, opts) {
1957
2540
  return wrap(() =>
1958
2541
  retryAfterGap(() =>
@@ -1967,21 +2550,7 @@ export function createRedisStoreCore(
1967
2550
  return []
1968
2551
  return wrap(() =>
1969
2552
  retryAfterGap(async () => {
1970
- const [count, cleared] = (await call(
1971
- 'MGET',
1972
- countKey(sessionId),
1973
- clearKey(sessionId),
1974
- )) as [unknown, unknown]
1975
- const frontier = toIndex({
1976
- raw: count,
1977
- sessionId,
1978
- name: 'event count',
1979
- })
1980
- const clearIndex = toIndex({
1981
- raw: cleared,
1982
- sessionId,
1983
- name: 'clear boundary',
1984
- })
2553
+ const { frontier, clearIndex } = await readBounds(sessionId)
1985
2554
  // Reads start at the latest clear marker.
1986
2555
  return readEventRange(
1987
2556
  sessionId,
@@ -1997,32 +2566,31 @@ export function createRedisStoreCore(
1997
2566
  holder,
1998
2567
  ttlMs,
1999
2568
  expiresAtMs,
2000
- excludeIndexes,
2569
+ exclude,
2570
+ consumers,
2001
2571
  maxFailures,
2572
+ through,
2002
2573
  }) {
2003
2574
  return wrap(async () => {
2004
2575
  const now = clock.now().getTime()
2005
2576
  const expiresAt = expiresAtMs ?? now + ttlMs
2006
- const excluded = excludeIndexes ?? []
2577
+ const excluded = exclude ?? []
2007
2578
  const reply = (await evalScript(
2008
2579
  CLAIM_AVAILABLE_LUA,
2009
- [
2010
- storeKey(sessionId),
2011
- countKey(sessionId),
2012
- metaKey(sessionId),
2013
- pendingKey(sessionId),
2014
- readyKey(sessionId),
2015
- metaSplitKey(sessionId),
2016
- snapshotPinCountsKey(sessionId),
2017
- ],
2580
+ sessionId,
2581
+ [],
2018
2582
  [
2019
2583
  holder,
2020
2584
  now,
2021
2585
  expiresAt,
2022
- excluded.length,
2023
- ...excluded,
2024
2586
  maxFailures ?? '',
2025
- snapshotEventPinsPrefix(sessionId),
2587
+ servedArg(consumers),
2588
+ through ?? '',
2589
+ excluded.length,
2590
+ ...excluded.flatMap((ref) => [
2591
+ ref.consumer,
2592
+ ref.lane === null ? `#${ref.index}` : `L${ref.lane}`,
2593
+ ]),
2026
2594
  ],
2027
2595
  )) as unknown[]
2028
2596
  return toClaimAvailableResult(sessionId, reply)
@@ -2030,30 +2598,48 @@ export function createRedisStoreCore(
2030
2598
  },
2031
2599
 
2032
2600
  async renewClaims({ sessionId, holder, claims, ttlMs, expiresAtMs }) {
2601
+ if (claims.length === 0) return { renewed: [], superseded: [] }
2033
2602
  return wrap(async () => {
2034
2603
  const now = clock.now().getTime()
2035
2604
  const expiresAt = expiresAtMs ?? now + ttlMs
2036
2605
  const reply = (await evalScript(
2037
2606
  RENEW_CLAIMS_LUA,
2038
- [countKey(sessionId), metaKey(sessionId), metaSplitKey(sessionId)],
2607
+ sessionId,
2608
+ [],
2039
2609
  [
2040
2610
  holder,
2041
2611
  now,
2042
2612
  expiresAt,
2043
2613
  claims.length,
2044
- ...claims.flatMap((claim) => [claim.index, claim.attempt]),
2614
+ ...claims.flatMap((claim) => [
2615
+ claim.consumer,
2616
+ tagOf(claim.lane),
2617
+ claim.index,
2618
+ claim.attempt,
2619
+ ]),
2045
2620
  ],
2046
2621
  )) as string[]
2047
2622
  const renewedCount = Number(reply[0])
2048
- const values = reply.slice(1).map(Number)
2623
+ const refs = reply.slice(1).map((position): HeadRef => {
2624
+ const { consumer, lane, index } = claims[Number(position) - 1]!
2625
+ return { consumer, lane, index }
2626
+ })
2049
2627
  return {
2050
- renewed: values.slice(0, renewedCount).toSorted((a, b) => a - b),
2051
- superseded: values.slice(renewedCount).toSorted((a, b) => a - b),
2628
+ renewed: refs.slice(0, renewedCount).toSorted(byHead),
2629
+ superseded: refs.slice(renewedCount).toSorted(byHead),
2052
2630
  }
2053
2631
  })
2054
2632
  },
2055
2633
 
2056
- async completeAttempt({ sessionId, index, attempt, events }) {
2634
+ async completeAttempt({
2635
+ sessionId,
2636
+ consumer,
2637
+ lane,
2638
+ index,
2639
+ attempt,
2640
+ events,
2641
+ consumers,
2642
+ }) {
2057
2643
  return wrap(async () => {
2058
2644
  const ids = events.map((event) => event.id)
2059
2645
  if (new Set(ids).size !== ids.length) {
@@ -2063,34 +2649,32 @@ export function createRedisStoreCore(
2063
2649
  )
2064
2650
  }
2065
2651
  const now = clock.now().getTime()
2066
- const reply = (await evalScript(
2067
- COMPLETE_ATTEMPT_LUA,
2068
- [
2069
- storeKey(sessionId),
2070
- countKey(sessionId),
2071
- metaKey(sessionId),
2072
- pendingKey(sessionId),
2073
- readyKey(sessionId),
2074
- snapshotEventPinsKey(sessionId, index),
2075
- snapshotPinCountsKey(sessionId),
2076
- metaSplitKey(sessionId),
2077
- ...ids.map((id) => idKey(id)),
2078
- ],
2079
- [
2652
+ const cause: EventCause = { consumer, lane, index, attempt }
2653
+ const payloads = events.map((event) => json(event.payload))
2654
+ const reply = (await indexing(sessionId, () =>
2655
+ evalScript(
2656
+ COMPLETE_ATTEMPT_LUA,
2080
2657
  sessionId,
2081
- now,
2082
- index,
2083
- attempt,
2084
- events.length,
2085
- ...events.flatMap((event) => [
2086
- event.id,
2087
- event.type,
2088
- json(event.payload),
2089
- event.lane ?? '',
2090
- event.settled === true ? '1' : '0',
2091
- ]),
2092
- `${prefix}:${sessionId}:notify`,
2093
- ],
2658
+ [],
2659
+ [
2660
+ sessionId,
2661
+ now,
2662
+ consumer,
2663
+ tagOf(lane),
2664
+ index,
2665
+ attempt,
2666
+ json(cause),
2667
+ servedArg(consumers),
2668
+ json(ids),
2669
+ events.length,
2670
+ ...events.flatMap((event, i) => [
2671
+ event.id,
2672
+ event.type,
2673
+ payloads[i]!,
2674
+ targetsArg(event.targets),
2675
+ ]),
2676
+ ],
2677
+ ),
2094
2678
  )) as string[]
2095
2679
  const outcome = reply[0]
2096
2680
  if (outcome === 'missing') {
@@ -2106,9 +2690,23 @@ export function createRedisStoreCore(
2106
2690
  }
2107
2691
  if (outcome === 'superseded') return { outcome }
2108
2692
 
2109
- const cause = { index, attempt }
2110
2693
  if (outcome === 'ok') {
2111
2694
  const base = Number(reply[1]) - 1
2695
+ if (events.length > 0) {
2696
+ await publish(
2697
+ notifyChannel(sessionId),
2698
+ wakeMessage(
2699
+ base + events.length,
2700
+ now,
2701
+ events.map((event, i) => [
2702
+ base + 1 + i,
2703
+ event.id,
2704
+ event.type,
2705
+ payloads[i]!,
2706
+ ]),
2707
+ ),
2708
+ )
2709
+ }
2112
2710
  return {
2113
2711
  outcome: 'completed',
2114
2712
  events: events.map((event, i) => ({
@@ -2118,21 +2716,7 @@ export function createRedisStoreCore(
2118
2716
  index: base + 1 + i,
2119
2717
  sessionId,
2120
2718
  createdAt: new Date(now),
2121
- cause,
2122
- lane: event.lane ?? null,
2123
- processedAt: event.settled === true ? new Date(now) : null,
2124
- processedByAttempt: null,
2125
- returnedEventIds: null,
2126
- firstClaimedAt: null,
2127
- lastClaimedAt: null,
2128
- attemptCount: 0,
2129
- claimHolder: null,
2130
- claimExpiresAt: null,
2131
- failureCount: 0,
2132
- lastFailedAt: null,
2133
- lastFailedAttempt: null,
2134
- lastError: null,
2135
- failedAt: null,
2719
+ cause: { ...cause },
2136
2720
  })),
2137
2721
  } satisfies CompleteAttemptResult
2138
2722
  }
@@ -2150,10 +2734,27 @@ export function createRedisStoreCore(
2150
2734
  )
2151
2735
  // Cleared after the script matched the retry.
2152
2736
  if (index <= clearIndex) return { outcome: 'superseded' }
2153
- return {
2154
- outcome: 'completed',
2155
- events: idempotentReplay(events, existing),
2737
+ if (
2738
+ existing.some(
2739
+ (row) =>
2740
+ row.cause?.consumer !== consumer ||
2741
+ row.cause.lane !== lane ||
2742
+ row.cause.index !== index ||
2743
+ row.cause.attempt !== attempt,
2744
+ )
2745
+ ) {
2746
+ throw new A2Error(
2747
+ 'PARTIAL_DUPLICATE_BATCH',
2748
+ 'returned event retry does not match the committed batch',
2749
+ )
2156
2750
  }
2751
+ const replayed = idempotentReplay(events, existing)
2752
+ // The first attempt may have committed without publishing.
2753
+ await publish(
2754
+ notifyChannel(sessionId),
2755
+ wakeMessage(firstIndex + events.length - 1, now, []),
2756
+ )
2757
+ return { outcome: 'completed', events: replayed }
2157
2758
  }
2158
2759
  throw new TypeError(`unexpected completion outcome '${outcome}'`)
2159
2760
  })
@@ -2161,6 +2762,8 @@ export function createRedisStoreCore(
2161
2762
 
2162
2763
  async failAttempt({
2163
2764
  sessionId,
2765
+ consumer,
2766
+ lane,
2164
2767
  index,
2165
2768
  attempt,
2166
2769
  error,
@@ -2171,16 +2774,11 @@ export function createRedisStoreCore(
2171
2774
  const now = clock.now().getTime()
2172
2775
  const reply = (await evalScript(
2173
2776
  FAIL_ATTEMPT_LUA,
2777
+ sessionId,
2778
+ [],
2174
2779
  [
2175
- countKey(sessionId),
2176
- metaKey(sessionId),
2177
- pendingKey(sessionId),
2178
- readyKey(sessionId),
2179
- snapshotEventPinsKey(sessionId, index),
2180
- snapshotPinCountsKey(sessionId),
2181
- metaSplitKey(sessionId),
2182
- ],
2183
- [
2780
+ consumer,
2781
+ tagOf(lane),
2184
2782
  index,
2185
2783
  attempt,
2186
2784
  error,
@@ -2190,8 +2788,6 @@ export function createRedisStoreCore(
2190
2788
  ],
2191
2789
  )) as string[]
2192
2790
  const outcome = reply[0]
2193
- if (outcome === 'cleared')
2194
- return { outcome: 'superseded', failureCount: 0 }
2195
2791
  if (outcome === 'missing') {
2196
2792
  throw new TypeError(
2197
2793
  `no event at index ${index} in session '${sessionId}'`,
@@ -2206,67 +2802,84 @@ export function createRedisStoreCore(
2206
2802
 
2207
2803
  async deadLetters(sessionId) {
2208
2804
  return wrap(async () => {
2209
- const reply = (await evalScript(
2210
- DEAD_LETTERS_LUA,
2211
- [
2212
- storeKey(sessionId),
2213
- countKey(sessionId),
2214
- metaKey(sessionId),
2215
- pendingKey(sessionId),
2216
- metaSplitKey(sessionId),
2217
- ],
2218
- [],
2219
- )) as unknown[]
2220
- const events: StoredEvent[] = []
2221
- for (let i = 0; i + 1 < reply.length; i += 2) {
2222
- const columns = reply[i + 1] as Array<string | null>
2223
- events.push(
2224
- toStoredEvent(
2225
- sessionId,
2226
- reply[i] as StreamEntry,
2227
- (suffix) =>
2228
- columns[STORED_EVENT_META_SUFFIXES.indexOf(suffix)] ??
2229
- undefined,
2230
- ),
2231
- )
2232
- }
2233
- return events
2805
+ const reply = (await runScript({
2806
+ script: DEAD_LETTERS_LUA,
2807
+ keys: [countKey(sessionId)],
2808
+ args: [],
2809
+ readOnly: true,
2810
+ })) as HeadTuple[]
2811
+ return reply
2812
+ .map((head) => toHeadEvent(sessionId, head))
2813
+ .toSorted(byHead)
2234
2814
  })
2235
2815
  },
2236
2816
 
2237
- async redrive({ sessionId, index, action }) {
2817
+ async redrive({ sessionId, consumer, index, action, consumers }) {
2238
2818
  return wrap(async () => {
2239
- const [redriven, hasPending] = (await evalScript(
2240
- REDRIVE_LUA,
2241
- [
2242
- countKey(sessionId),
2243
- metaKey(sessionId),
2244
- pendingKey(sessionId),
2245
- readyKey(sessionId),
2246
- metaSplitKey(sessionId),
2247
- ],
2248
- [index, action, clock.now().getTime()],
2819
+ const [redriven, hasPending] = (await indexing(sessionId, () =>
2820
+ evalScript(
2821
+ REDRIVE_LUA,
2822
+ sessionId,
2823
+ [],
2824
+ [
2825
+ consumer,
2826
+ index,
2827
+ action,
2828
+ clock.now().getTime(),
2829
+ servedArg(consumers),
2830
+ ],
2831
+ ),
2249
2832
  )) as [string, string]
2250
2833
  return { redriven: redriven === '1', hasPending: hasPending === '1' }
2251
2834
  })
2252
2835
  },
2253
2836
 
2837
+ async wakeLog({ sessionId, consumer }) {
2838
+ return wrap(async () => {
2839
+ const { name, log } = consumer
2840
+ const [, position, head, columns, sessionHead] = (await indexing(
2841
+ sessionId,
2842
+ () =>
2843
+ evalScript(
2844
+ WAKE_LOG_LUA,
2845
+ sessionId,
2846
+ [],
2847
+ [name, log.lane, logTypesArg(log.types), log.backfill ? 1 : 0],
2848
+ ),
2849
+ )) as [
2850
+ ok: string,
2851
+ position: string,
2852
+ head: string | null,
2853
+ columns: Column[],
2854
+ sessionHead: string,
2855
+ ]
2856
+ return {
2857
+ cursor: Object.assign(dispatchOf(name, log.lane, columns), {
2858
+ position: Number(position),
2859
+ head: head === null ? null : Number(head),
2860
+ }),
2861
+ head: Number(sessionHead),
2862
+ }
2863
+ })
2864
+ },
2865
+
2254
2866
  async readState(sessionId, reducerName, stateOptions) {
2255
2867
  return wrap(() =>
2256
2868
  retryAfterGap(async () => {
2257
- const reply = (await evalScript(
2258
- READ_STATE_LUA,
2259
- [
2869
+ const reply = (await runScript({
2870
+ script: READ_STATE_LUA,
2871
+ keys: [
2872
+ countKey(sessionId),
2260
2873
  snapKey(sessionId, reducerName),
2261
2874
  snapHistoryKey(sessionId, reducerName),
2262
- countKey(sessionId),
2263
2875
  ],
2264
- [
2876
+ args: [
2265
2877
  stateOptions?.snapshotThroughIndex ??
2266
2878
  stateOptions?.throughIndex ??
2267
2879
  '',
2268
2880
  ],
2269
- )) as [number | string, string, string, string]
2881
+ readOnly: true,
2882
+ })) as [number | string, string, string, string]
2270
2883
  const rawHead = reply[0]
2271
2884
  const raw = reply[1]
2272
2885
  const headIndex = rawHead === '' ? null : Number(rawHead)
@@ -2315,33 +2928,28 @@ export function createRedisStoreCore(
2315
2928
  async putSnapshots(sessionId, reducerName, writes) {
2316
2929
  const byIndex = new Map<
2317
2930
  number,
2318
- { index: number; state: unknown; pinEventIndexes: Set<number> }
2931
+ { index: number; state: unknown; pins: Map<string, HeadRef> }
2319
2932
  >()
2320
2933
  for (const write of writes) {
2321
2934
  const existing = byIndex.get(write.index)
2322
- if (existing) {
2323
- existing.state = write.state
2324
- for (const eventIndex of write.pinEventIndexes ?? []) {
2325
- existing.pinEventIndexes.add(eventIndex)
2326
- }
2327
- } else {
2935
+ const pins = existing?.pins ?? new Map<string, HeadRef>()
2936
+ for (const pin of write.pins ?? []) {
2937
+ pins.set(JSON.stringify([pin.consumer, pin.lane, pin.index]), pin)
2938
+ }
2939
+ if (existing) existing.state = write.state
2940
+ else
2328
2941
  byIndex.set(write.index, {
2329
2942
  index: write.index,
2330
2943
  state: write.state,
2331
- pinEventIndexes: new Set(write.pinEventIndexes ?? []),
2944
+ pins,
2332
2945
  })
2333
- }
2334
2946
  }
2335
2947
  const normalized = [...byIndex.values()].toSorted(
2336
2948
  (a, b) => a.index - b.index,
2337
2949
  )
2338
2950
  if (normalized.length === 0) return
2339
2951
  await wrap(async () => {
2340
- const args: Array<string | number> = [
2341
- clock.now().getTime(),
2342
- snapshotEventPinsPrefix(sessionId),
2343
- normalized.length,
2344
- ]
2952
+ const args: Array<string | number> = [normalized.length]
2345
2953
  for (const write of normalized) {
2346
2954
  args.push(
2347
2955
  write.index,
@@ -2352,19 +2960,20 @@ export function createRedisStoreCore(
2352
2960
  // This reducer is in the session's registry (PUT_SNAPSHOTS_LUA).
2353
2961
  r: 1,
2354
2962
  }),
2355
- write.pinEventIndexes.size,
2356
- ...write.pinEventIndexes,
2963
+ write.pins.size,
2964
+ ...[...write.pins.values()].flatMap((pin) => [
2965
+ pin.consumer,
2966
+ tagOf(pin.lane),
2967
+ pin.index,
2968
+ ]),
2357
2969
  )
2358
2970
  }
2359
2971
  await evalScript(
2360
2972
  PUT_SNAPSHOTS_LUA,
2973
+ sessionId,
2361
2974
  [
2362
2975
  snapKey(sessionId, reducerName),
2363
2976
  snapHistoryKey(sessionId, reducerName),
2364
- countKey(sessionId),
2365
- metaKey(sessionId),
2366
- snapshotPinCountsKey(sessionId),
2367
- metaSplitKey(sessionId),
2368
2977
  ],
2369
2978
  [...args, reducerName],
2370
2979
  )
@@ -2380,69 +2989,76 @@ export function createRedisStoreCore(
2380
2989
  // Expiry anchors on the injected storage clock — the sender's
2381
2990
  // `a` stamp orders writes, the `e` stamp bounds their
2382
2991
  // lifetime, and the two clocks never mix.
2383
- const expiresAtMs = clock.now().getTime() + meta.ttlMs
2992
+ const nowMs = clock.now().getTime()
2993
+ const expiresAtMs = nowMs + meta.ttlMs
2384
2994
  // The hash field is JSON `[participant, field]`: both are
2385
2995
  // caller strings, so no separator character is
2386
2996
  // collision-safe — the JSON array is.
2387
- const applied = (await evalScript(
2388
- PRESENCE_SET_LUA,
2389
- [presenceKey(ns)],
2390
- [
2997
+ const applied = (await runScript({
2998
+ script: PRESENCE_SET_LUA,
2999
+ keys: [presenceKey(ns)],
3000
+ args: [
2391
3001
  atMs,
2392
3002
  meta.ttlMs,
3003
+ nowMs,
2393
3004
  entries.length,
3005
+ // A null is stored as a tombstone: it keeps its stamp, so an
3006
+ // older set that arrives after it still loses.
2394
3007
  ...entries.flatMap(([field, value]) => [
2395
3008
  JSON.stringify([participant, field]),
2396
- value === null
2397
- ? ''
2398
- : json({
2399
- v: value,
2400
- s: meta.seen,
2401
- a: atMs,
2402
- e: expiresAtMs,
2403
- }),
2404
- `${json(field)}:${json(value)}`,
3009
+ json({ a: atMs, e: expiresAtMs, s: meta.seen, v: value }),
2405
3010
  ]),
2406
- presenceKey(ns),
2407
- `${json({ src: options.presenceSource, participant, seen: meta.seen, at: atMs }).slice(0, -1)},"values":{`,
2408
3011
  ],
2409
- )) as string[]
3012
+ readOnly: false,
3013
+ })) as string[]
2410
3014
  if (applied.length === 0) return null
2411
3015
  // Field-keyed and caller-named — null-prototype, like every
2412
3016
  // presence map (see `nullProtoRecord`).
2413
3017
  const appliedValues: Record<string, unknown> = nullProtoRecord()
3018
+ const published: string[] = []
2414
3019
  for (const position of applied) {
2415
3020
  const [field, value] = entries[Number(position) - 1]!
2416
3021
  appliedValues[field] = structuredClone(value)
3022
+ published.push(`${json(field)}:${json(value)}`)
2417
3023
  }
2418
3024
  return {
2419
- participant,
2420
- values: appliedValues,
2421
- seen: meta.seen,
2422
- at: new Date(atMs),
3025
+ patch: {
3026
+ participant,
3027
+ values: appliedValues,
3028
+ seen: meta.seen,
3029
+ at: new Date(atMs),
3030
+ },
3031
+ published: publish(
3032
+ presenceChannel(ns),
3033
+ `${json({ src: options.presenceSource, participant, seen: meta.seen, at: atMs }).slice(0, -1)},"values":{${published.join(',')}}}`,
3034
+ ),
2423
3035
  }
2424
3036
  })
2425
3037
  },
2426
3038
 
2427
3039
  async read(ns) {
2428
3040
  return wrap(async () => {
2429
- const reply = (await evalScript(
2430
- PRESENCE_READ_LUA,
2431
- [presenceKey(ns)],
2432
- [clock.now().getTime()],
2433
- )) as string[]
3041
+ const key = presenceKey(ns)
3042
+ const reply = (await call('HGETALL', key)) as string[]
3043
+ const nowMs = clock.now().getTime()
2434
3044
  const rows: PresenceRow[] = []
3045
+ const expired: string[] = []
2435
3046
  for (let i = 0; i + 1 < reply.length; i += 2) {
2436
- const [participant, field] = JSON.parse(reply[i]!) as [
2437
- string,
2438
- string,
2439
- ]
2440
3047
  const entry = JSON.parse(reply[i + 1]!) as {
2441
3048
  v: unknown
2442
3049
  s: number
2443
3050
  a: number
2444
3051
  e: number
2445
3052
  }
3053
+ if (entry.e <= nowMs) {
3054
+ expired.push(reply[i]!)
3055
+ continue
3056
+ }
3057
+ if (entry.v === null) continue
3058
+ const [participant, field] = JSON.parse(reply[i]!) as [
3059
+ string,
3060
+ string,
3061
+ ]
2446
3062
  rows.push({
2447
3063
  participant,
2448
3064
  field,
@@ -2452,6 +3068,16 @@ export function createRedisStoreCore(
2452
3068
  expiresAt: new Date(entry.e),
2453
3069
  })
2454
3070
  }
3071
+ // The rows are already filtered; a failed prune only defers the
3072
+ // deletion to the next read.
3073
+ if (expired.length > 0) {
3074
+ await runScript({
3075
+ script: PRESENCE_PRUNE_LUA,
3076
+ keys: [key],
3077
+ args: [nowMs, ...expired],
3078
+ readOnly: false,
3079
+ }).catch(() => undefined)
3080
+ }
2455
3081
  return rows
2456
3082
  })
2457
3083
  },
@@ -2463,14 +3089,20 @@ export function createRedisStoreCore(
2463
3089
 
2464
3090
  async readTail(sessionId, fromIndex, throughIndex) {
2465
3091
  return wrap(async () => {
2466
- const newest = (await call(
2467
- 'XREVRANGE',
2468
- storeKey(sessionId),
2469
- throughIndex === undefined ? '+' : `${throughIndex}-0`,
2470
- `${fromIndex}-0`,
2471
- 'COUNT',
2472
- READ_PAGE_SIZE,
2473
- )) as StreamEntry[]
3092
+ // Entries below the latest clear are an old lifetime's, awaiting
3093
+ // reclaim: a tail that reaches them is the boundary-aware reads' job.
3094
+ const [newest, clearIndex] = await Promise.all([
3095
+ call(
3096
+ 'XREVRANGE',
3097
+ storeKey(sessionId),
3098
+ throughIndex === undefined ? '+' : `${throughIndex}-0`,
3099
+ `${fromIndex}-0`,
3100
+ 'COUNT',
3101
+ READ_PAGE_SIZE,
3102
+ ) as Promise<StreamEntry[]>,
3103
+ readClearIndex(sessionId),
3104
+ ])
3105
+ if (clearIndex > fromIndex) return null
2474
3106
  const page = newest
2475
3107
  .toReversed()
2476
3108
  .map((entry) => toEvent(sessionId, entry))
@@ -2490,41 +3122,43 @@ export function createRedisStoreCore(
2490
3122
  async clear({ sessionId, ifLastIndex, cause }) {
2491
3123
  return wrap(async () => {
2492
3124
  const markerId = generateId()
3125
+ const now = clock.now().getTime()
2493
3126
  const reply = (await evalScript(
2494
3127
  CLEAR_LUA,
2495
- [
2496
- storeKey(sessionId),
2497
- countKey(sessionId),
2498
- metaKey(sessionId),
2499
- metaSplitKey(sessionId),
2500
- idKey(markerId),
2501
- pendingKey(sessionId),
2502
- readyKey(sessionId),
2503
- ],
3128
+ sessionId,
3129
+ [],
2504
3130
  [
2505
3131
  sessionId,
2506
- clock.now().getTime(),
3132
+ now,
2507
3133
  ifLastIndex ?? '',
3134
+ cause?.consumer ?? '',
3135
+ cause ? tagOf(cause.lane) : '',
2508
3136
  cause?.index ?? '',
2509
3137
  cause?.attempt ?? '',
2510
3138
  markerId,
2511
- `${prefix}:${sessionId}:notify`,
2512
3139
  ],
2513
3140
  )) as string[]
2514
3141
  if (reply[0] === 'superseded') {
2515
3142
  throw supersededAttemptError(sessionId, cause!)
2516
3143
  }
2517
- if (reply[0] === 'no_parent') {
2518
- throw new TypeError(
2519
- `no event at index ${reply[1]} in session '${sessionId}'`,
2520
- )
2521
- }
2522
3144
  if (reply[0] === 'foreign') {
2523
3145
  throw new A2Error(
2524
3146
  'PARTIAL_DUPLICATE_BATCH',
2525
3147
  `the clear marker id '${markerId}' already exists`,
2526
3148
  )
2527
3149
  }
3150
+ const head = Number(reply[2])
3151
+ if (reply[4] === '1') {
3152
+ await publish(
3153
+ notifyChannel(sessionId),
3154
+ wakeMessage(head, now, [
3155
+ [head, markerId, CLEAR_EVENT_TYPE, 'null'],
3156
+ ]),
3157
+ )
3158
+ } else if (reply[1] === '1' && head > 0) {
3159
+ // An earlier attempt may have appended the marker without publishing.
3160
+ await publish(notifyChannel(sessionId), wakeMessage(head, now, []))
3161
+ }
2528
3162
  return {
2529
3163
  cleared: reply[1] === '1',
2530
3164
  index: Number(reply[2]),
@@ -2535,31 +3169,25 @@ export function createRedisStoreCore(
2535
3169
 
2536
3170
  async reclaim(sessionId) {
2537
3171
  return wrap(async () => {
2538
- const [log, ...pairs] = (await evalScript(
3172
+ const pairs = (await evalScript(
2539
3173
  RECLAIM_BATCH_LUA,
2540
- [countKey(sessionId)],
3174
+ sessionId,
3175
+ [],
2541
3176
  [READ_PAGE_SIZE],
2542
3177
  )) as string[]
2543
- if (log === undefined) return { deleted: 0, done: true }
2544
- // Id keys are built here, like append's: they use JSON.stringify,
2545
- // whose escaping cjson does not reproduce.
2546
- const ids: string[] = []
2547
- for (let i = 1; i < pairs.length; i += 2) ids.push(pairs[i]!)
3178
+ if (pairs.length === 0) return { deleted: 0, done: true }
2548
3179
  const [deleted, done] = (await evalScript(
2549
3180
  RECLAIM_LUA,
2550
- [
2551
- log,
2552
- countKey(sessionId),
2553
- metaKey(sessionId),
2554
- metaSplitKey(sessionId),
2555
- ...ids.map((id) => idKey(id)),
2556
- ],
2557
- [sessionId, snapshotEventPinsPrefix(sessionId), ids.length, ...pairs],
3181
+ sessionId,
3182
+ [],
3183
+ [sessionId, pairs.length / 2, ...pairs],
2558
3184
  )) as [number, number]
2559
3185
  return { deleted: Number(deleted), done: Number(done) === 1 }
2560
3186
  })
2561
3187
  },
2562
3188
 
3189
+ clearIndex: (sessionId) => wrap(() => readClearIndex(sessionId)),
3190
+
2563
3191
  async readEvents(sessionId, afterIndex) {
2564
3192
  return wrap(async () => {
2565
3193
  const entries = (await call(