experimental-a2 0.19.0 → 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -0
- package/README.md +1 -1
- package/dist/{actor-C7b-1arn.d.ts → actor-HnGmrOZZ.d.ts} +5 -5
- package/dist/{actor-C7b-1arn.d.ts.map → actor-HnGmrOZZ.d.ts.map} +1 -1
- package/dist/actor-client.d.ts +1 -1
- package/dist/actor-client.js +2 -2
- package/dist/actor-react.d.ts +3 -3
- package/dist/actor-react.js +5 -5
- package/dist/{actor-shared-t7w8xIKP.js → actor-shared-DFET5q16.js} +3 -3
- package/dist/{actor-shared-t7w8xIKP.js.map → actor-shared-DFET5q16.js.map} +1 -1
- package/dist/actor.d.ts +1 -1
- package/dist/actor.js +54 -43
- package/dist/actor.js.map +1 -1
- package/dist/{ai-Cz3hqVhk.d.ts → ai-CvEnqVfo.d.ts} +8 -8
- package/dist/{ai-Cz3hqVhk.d.ts.map → ai-CvEnqVfo.d.ts.map} +1 -1
- package/dist/{ai-mMiWul1T.js → ai-CzE672Q3.js} +9 -9
- package/dist/ai-CzE672Q3.js.map +1 -0
- package/dist/ai-client.d.ts +2 -2
- package/dist/ai-client.js +2 -2
- package/dist/ai-fx.d.ts +1 -1
- package/dist/ai-fx.d.ts.map +1 -1
- package/dist/ai-fx.js +36 -9
- package/dist/ai-fx.js.map +1 -1
- package/dist/{ai-harness-CV6rXbB3.d.ts → ai-harness-Bfk-Lycn.d.ts} +2 -2
- package/dist/ai-harness-Bfk-Lycn.d.ts.map +1 -0
- package/dist/{ai-harness-BuuC-gJ3.js → ai-harness-dQeJaR1t.js} +17 -4
- package/dist/ai-harness-dQeJaR1t.js.map +1 -0
- package/dist/ai-pi.d.ts +1 -1
- package/dist/ai-pi.js +4 -4
- package/dist/ai-pi.js.map +1 -1
- package/dist/ai-react.d.ts +5 -5
- package/dist/ai-react.js +2 -2
- package/dist/ai-server.d.ts +14 -17
- package/dist/ai-server.d.ts.map +1 -1
- package/dist/ai-server.js +84 -75
- package/dist/ai-server.js.map +1 -1
- package/dist/ai.d.ts +2 -2
- package/dist/ai.js +2 -2
- package/dist/{client-BhHLoNKF.js → client-9IeTsiAs.js} +5 -5
- package/dist/{client-BhHLoNKF.js.map → client-9IeTsiAs.js.map} +1 -1
- package/dist/{client-h_DKFAjj.d.ts → client-CUMbS_38.d.ts} +3 -3
- package/dist/{client-h_DKFAjj.d.ts.map → client-CUMbS_38.d.ts.map} +1 -1
- package/dist/client.d.ts +3 -3
- package/dist/client.js +1 -1
- package/dist/{contract-BYKASlCV.js → contract-CWukt01d.js} +2 -2
- package/dist/contract-CWukt01d.js.map +1 -0
- package/dist/index.d.ts +5 -5
- package/dist/index.js +3 -3
- package/dist/{initial-snapshot-Bx-c0pDP.d.ts → initial-snapshot-D3Lve43o.d.ts} +4 -4
- package/dist/initial-snapshot-D3Lve43o.d.ts.map +1 -0
- package/dist/{internal-CscSY7K_.js → internal-Bc4PvfHk.js} +10 -3
- package/dist/internal-Bc4PvfHk.js.map +1 -0
- package/dist/otel.d.ts +1 -1
- package/dist/otel.js +1 -1
- package/dist/otel.js.map +1 -1
- package/dist/{react-BBXgHWFc.d.ts → react-BoHvC3v9.d.ts} +4 -4
- package/dist/{react-BBXgHWFc.d.ts.map → react-BoHvC3v9.d.ts.map} +1 -1
- package/dist/react.d.ts +1 -1
- package/dist/react.js +1 -1
- package/dist/{reducer-CqA59Q6U.d.ts → reducer-CYPHsSmd.d.ts} +2 -2
- package/dist/{reducer-CqA59Q6U.d.ts.map → reducer-CYPHsSmd.d.ts.map} +1 -1
- package/dist/{reducer-BAXc8JyF.js → reducer-CjmvZSAb.js} +2 -2
- package/dist/{reducer-BAXc8JyF.js.map → reducer-CjmvZSAb.js.map} +1 -1
- package/dist/runtime-consumer-BWmht60z.js +12 -0
- package/dist/runtime-consumer-BWmht60z.js.map +1 -0
- package/dist/scheduler-qstash.d.ts +2 -2
- package/dist/scheduler-qstash.d.ts.map +1 -1
- package/dist/scheduler-qstash.js +7 -7
- package/dist/scheduler-qstash.js.map +1 -1
- package/dist/scheduler-task-X1SToDjP.js.map +1 -1
- package/dist/scheduler-vercel.d.ts +2 -2
- package/dist/scheduler-vercel.d.ts.map +1 -1
- package/dist/scheduler-vercel.js +5 -4
- package/dist/scheduler-vercel.js.map +1 -1
- package/dist/{server-DYAATFuT.d.ts → server-BshFZ0eL.d.ts} +108 -41
- package/dist/server-BshFZ0eL.d.ts.map +1 -0
- package/dist/{server-D-Y4pqwu.js → server-DvIB8kj5.js} +423 -135
- package/dist/server-DvIB8kj5.js.map +1 -0
- package/dist/server.browser.js +1 -1
- package/dist/server.browser.js.map +1 -1
- package/dist/server.d.ts +4 -4
- package/dist/server.js +2 -2
- package/dist/{session-clear-C-QwTXV1.d.ts → session-clear-Bk_29eEw.d.ts} +2 -2
- package/dist/session-clear-Bk_29eEw.d.ts.map +1 -0
- package/dist/{session-clear-DGI4jTQ5.js → session-clear-wHd5GCpZ.js} +5 -4
- package/dist/session-clear-wHd5GCpZ.js.map +1 -0
- package/dist/store-N8PXxDAS.js.map +1 -1
- package/dist/store-Z2qjw50n.d.ts +523 -0
- package/dist/store-Z2qjw50n.d.ts.map +1 -0
- package/dist/store-codec-DwrWlgQx.js +65 -0
- package/dist/store-codec-DwrWlgQx.js.map +1 -0
- package/dist/store-memory.d.ts +1 -1
- package/dist/store-memory.d.ts.map +1 -1
- package/dist/store-memory.js +393 -210
- package/dist/store-memory.js.map +1 -1
- package/dist/{store-polling-7WJV0YWP.js → store-polling-DYNyxCev.js} +2 -2
- package/dist/{store-polling-7WJV0YWP.js.map → store-polling-DYNyxCev.js.map} +1 -1
- package/dist/store-postgres.d.ts +1 -1
- package/dist/store-postgres.d.ts.map +1 -1
- package/dist/store-postgres.js +1595 -673
- package/dist/store-postgres.js.map +1 -1
- package/dist/{store-presence-polling-B1ixeqIS.js → store-presence-polling-CzQ0wdtM.js} +3 -3
- package/dist/{store-presence-polling-B1ixeqIS.js.map → store-presence-polling-CzQ0wdtM.js.map} +1 -1
- package/dist/store-redis-http.d.ts +1 -1
- package/dist/store-redis-http.js +4 -4
- package/dist/store-redis-notify-IiVdlYHc.js +2919 -0
- package/dist/store-redis-notify-IiVdlYHc.js.map +1 -0
- package/dist/store-redis.d.ts +1 -1
- package/dist/store-redis.js +2 -2
- package/dist/store-sqlite.d.ts +1 -1
- package/dist/store-sqlite.d.ts.map +1 -1
- package/dist/store-sqlite.js +517 -287
- package/dist/store-sqlite.js.map +1 -1
- package/dist/{telemetry-Cyfqo6Qt.d.ts → telemetry-CWwsh-Gx.d.ts} +3 -3
- package/dist/telemetry-CWwsh-Gx.d.ts.map +1 -0
- package/dist/{wire-BKxH1vbP.js → wire-CZGqNZRx.js} +5 -4
- package/dist/wire-CZGqNZRx.js.map +1 -0
- package/docs/01-quickstart.mdx +28 -19
- package/docs/02-examples.mdx +1 -1
- package/docs/actors/01-introduction.mdx +9 -1
- package/docs/actors/02-concurrency.mdx +1 -1
- package/docs/actors/meta.ts +1 -1
- package/docs/agents/01-introduction.mdx +1 -1
- package/docs/agents/02-tools.mdx +13 -12
- package/docs/agents/03-conversations.mdx +31 -21
- package/docs/agents/04-harnesses.mdx +7 -3
- package/docs/agents/meta.ts +1 -1
- package/docs/client/03-presence.mdx +18 -14
- package/docs/client/meta.ts +1 -1
- package/docs/contract/01-events.mdx +2 -2
- package/docs/contract/02-state.mdx +18 -14
- package/docs/contract/03-sessions.mdx +32 -25
- package/docs/index.mdx +31 -26
- package/docs/reference/01-core.mdx +12 -12
- package/docs/reference/02-server.mdx +247 -192
- package/docs/reference/04-ai.mdx +29 -20
- package/docs/reference/05-telemetry.mdx +146 -0
- package/docs/reference/06-errors.mdx +24 -18
- package/docs/reference/meta.ts +1 -1
- package/docs/schedulers/01-overview.mdx +112 -0
- package/docs/schedulers/02-vercel-queues.mdx +87 -0
- package/docs/schedulers/03-qstash.mdx +123 -0
- package/docs/schedulers/meta.ts +1 -0
- package/docs/server/01-consumers.mdx +415 -0
- package/docs/server/02-timers.mdx +33 -23
- package/docs/server/03-cancellation.mdx +52 -44
- package/docs/server/04-routes.mdx +4 -1
- package/docs/server/05-sql-projections.mdx +391 -0
- package/docs/server/06-guarantees.mdx +60 -50
- package/docs/stores/01-overview.mdx +37 -0
- package/docs/stores/02-redis.mdx +57 -0
- package/docs/stores/03-redis-http.mdx +56 -0
- package/docs/stores/04-postgres.mdx +172 -0
- package/docs/stores/05-sqlite.mdx +36 -0
- package/docs/stores/06-memory.mdx +22 -0
- package/docs/stores/meta.ts +1 -0
- package/docs/upgrades/01-overview.mdx +30 -0
- package/docs/upgrades/79-0-21.mdx +165 -0
- package/docs/upgrades/80-0-20.mdx +43 -0
- package/docs/upgrades/81-0-19.mdx +33 -0
- package/docs/upgrades/82-0-18.mdx +33 -0
- package/docs/upgrades/83-0-17.mdx +69 -0
- package/docs/upgrades/meta.ts +3 -0
- package/examples/playground/DEPLOY.md +48 -11
- package/examples/playground/app/a0/apps/[appId]/route.ts +6 -10
- package/examples/playground/app/a0/chat.test.tsx +421 -330
- package/examples/playground/app/a0/sandbox-log.ts +17 -15
- package/examples/playground/app/a0/server.ts +4 -4
- package/examples/playground/app/a0/shell.tsx +1 -1
- package/examples/playground/app/a0/uploads/route.test.ts +59 -0
- package/examples/playground/app/a2slack/a2slack.css +1067 -0
- package/examples/playground/app/a2slack/c/[channel]/page.tsx +20 -0
- package/examples/playground/app/a2slack/consumers/agent.ts +63 -0
- package/examples/playground/app/a2slack/consumers/feed.ts +164 -0
- package/examples/playground/app/a2slack/consumers/notify.ts +116 -0
- package/examples/playground/app/a2slack/consumers/search.ts +37 -0
- package/examples/playground/app/a2slack/consumers/triage.ts +31 -0
- package/examples/playground/app/a2slack/consumers/welcome.ts +25 -0
- package/examples/playground/app/a2slack/db/a2.ts +206 -0
- package/examples/playground/app/a2slack/db/index.ts +73 -0
- package/examples/playground/app/a2slack/db/migrations/0000_extensions.sql +4 -0
- package/examples/playground/app/a2slack/db/migrations/0001_workspace.sql +74 -0
- package/examples/playground/app/a2slack/db/migrations/0002_a2.sql +215 -0
- package/examples/playground/app/a2slack/db/migrations/0003_a2_batch.sql +67 -0
- package/examples/playground/app/a2slack/db/migrations/meta/0000_snapshot.json +18 -0
- package/examples/playground/app/a2slack/db/migrations/meta/0001_snapshot.json +500 -0
- package/examples/playground/app/a2slack/db/migrations/meta/0002_snapshot.json +1159 -0
- package/examples/playground/app/a2slack/db/migrations/meta/0003_snapshot.json +1159 -0
- package/examples/playground/app/a2slack/db/migrations/meta/_journal.json +34 -0
- package/examples/playground/app/a2slack/db/queries.ts +263 -0
- package/examples/playground/app/a2slack/db/schema.ts +145 -0
- package/examples/playground/app/a2slack/drills/a2slack.drill.ts +708 -0
- package/examples/playground/app/a2slack/drills/harness.ts +244 -0
- package/examples/playground/app/a2slack/events/route.ts +36 -0
- package/examples/playground/app/a2slack/identity.ts +44 -0
- package/examples/playground/app/a2slack/inbox/route.ts +8 -0
- package/examples/playground/app/a2slack/lag/data/route.ts +8 -0
- package/examples/playground/app/a2slack/lag/page.tsx +101 -0
- package/examples/playground/app/a2slack/lag.ts +73 -0
- package/examples/playground/app/a2slack/layout.tsx +43 -0
- package/examples/playground/app/a2slack/messages/route.ts +17 -0
- package/examples/playground/app/a2slack/model.test.ts +208 -0
- package/examples/playground/app/a2slack/model.ts +369 -0
- package/examples/playground/app/a2slack/models.test.ts +72 -0
- package/examples/playground/app/a2slack/page.tsx +10 -0
- package/examples/playground/app/a2slack/scheduler/route.ts +5 -0
- package/examples/playground/app/a2slack/search/route.ts +40 -0
- package/examples/playground/app/a2slack/search-index.ts +108 -0
- package/examples/playground/app/a2slack/server.test.ts +315 -0
- package/examples/playground/app/a2slack/server.ts +55 -0
- package/examples/playground/app/a2slack/sessions.ts +50 -0
- package/examples/playground/app/a2slack/thread/route.ts +8 -0
- package/examples/playground/app/a2slack/ui/ChannelView.tsx +87 -0
- package/examples/playground/app/a2slack/ui/Composer.tsx +117 -0
- package/examples/playground/app/a2slack/ui/DotsShadow.tsx +42 -0
- package/examples/playground/app/a2slack/ui/Feed.tsx +157 -0
- package/examples/playground/app/a2slack/ui/InboxContext.tsx +63 -0
- package/examples/playground/app/a2slack/ui/InboxView.tsx +82 -0
- package/examples/playground/app/a2slack/ui/Menu.tsx +128 -0
- package/examples/playground/app/a2slack/ui/MessageRow.tsx +314 -0
- package/examples/playground/app/a2slack/ui/NameGate.tsx +35 -0
- package/examples/playground/app/a2slack/ui/SearchContext.tsx +146 -0
- package/examples/playground/app/a2slack/ui/SearchResults.tsx +97 -0
- package/examples/playground/app/a2slack/ui/Shell.tsx +34 -0
- package/examples/playground/app/a2slack/ui/Sidebar.tsx +199 -0
- package/examples/playground/app/a2slack/ui/ThreadPanel.tsx +134 -0
- package/examples/playground/app/a2slack/ui/useInbox.ts +71 -0
- package/examples/playground/app/agent/[agentId]/agent-queue.test.tsx +116 -90
- package/examples/playground/app/agent/[agentId]/compaction-panel.test.tsx +107 -76
- package/examples/playground/app/agent/ingress.test.ts +41 -17
- package/examples/playground/app/agent/model-input.test.tsx +92 -43
- package/examples/playground/app/agent/server.ts +3 -3
- package/examples/playground/app/chat/[chatId]/agent-stream-drawer.test.tsx +2 -2
- package/examples/playground/app/chat/[chatId]/chat-client.tsx +1 -1
- package/examples/playground/app/chat/server.test.ts +38 -65
- package/examples/playground/app/chat/server.ts +45 -35
- package/examples/playground/app/components/activity-history.test.tsx +60 -18
- package/examples/playground/app/components/demo-icons.tsx +7 -0
- package/examples/playground/app/components/demos.tsx +7 -0
- package/examples/playground/app/components/fast-link.test.tsx +3 -5
- package/examples/playground/app/initial-state.test.tsx +86 -45
- package/examples/playground/app/orders/[orderId]/order-client.tsx +2 -2
- package/examples/playground/app/orders/page.tsx +1 -1
- package/examples/playground/app/orders/server.ts +49 -30
- package/examples/playground/app/recovery/[recoveryId]/recovery-client.tsx +14 -15
- package/examples/playground/app/recovery/page.tsx +4 -4
- package/examples/playground/app/recovery/server.ts +23 -32
- package/examples/playground/drizzle.config.ts +14 -0
- package/examples/playground/next.config.ts +6 -1
- package/examples/playground/package.json +13 -1
- package/examples/playground/scripts/migrate.mjs +31 -0
- package/examples/playground/test/agent.ts +35 -0
- package/examples/playground/test/model.ts +149 -0
- package/examples/playground/test/serve.ts +184 -0
- package/examples/playground/test/server-only.ts +1 -0
- package/examples/playground/vercel.json +9 -0
- package/examples/playground/vitest.config.ts +27 -0
- package/examples/playground/vitest.drills.config.ts +32 -0
- package/package.json +7 -2
- package/sql/postgres/drizzle-schema.ts +206 -0
- package/sql/postgres/notifications.sql +24 -0
- package/sql/postgres/schema.sql +276 -0
- package/src/actor.ts +101 -71
- package/src/ai-agent.ts +154 -115
- package/src/ai-control-server.ts +8 -8
- package/src/ai-fx.ts +41 -9
- package/src/ai-harness.ts +28 -2
- package/src/ai-pi.ts +2 -2
- package/src/ai-server.ts +2 -2
- package/src/ai.ts +7 -7
- package/src/contract.ts +1 -1
- package/src/deterministic-id.ts +26 -3
- package/src/errors.ts +9 -6
- package/src/idempotent-replay.ts +0 -1
- package/src/index.ts +1 -1
- package/src/internal.ts +17 -6
- package/src/otel.ts +1 -1
- package/src/postgres-notifications.ts +0 -22
- package/src/postgres-resources.ts +6 -9
- package/src/postgres-schema.ts +121 -0
- package/src/push-envelope.ts +20 -0
- package/src/runtime-consumer.ts +10 -0
- package/src/scheduler-qstash.ts +12 -5
- package/src/scheduler-task.ts +2 -2
- package/src/scheduler-vercel.ts +10 -4
- package/src/server.browser.ts +1 -1
- package/src/server.ts +839 -238
- package/src/session-socket.ts +1 -1
- package/src/store-codec.ts +66 -0
- package/src/store-memory.ts +581 -255
- package/src/store-postgres.ts +1984 -943
- package/src/store-redis-core.ts +1624 -1152
- package/src/store-redis-notify.ts +2 -0
- package/src/store-sqlite.ts +891 -486
- package/src/store.ts +227 -97
- package/src/telemetry.ts +2 -1
- package/src/wire.ts +1 -0
- package/dist/ai-harness-BuuC-gJ3.js.map +0 -1
- package/dist/ai-harness-CV6rXbB3.d.ts.map +0 -1
- package/dist/ai-mMiWul1T.js.map +0 -1
- package/dist/contract-BYKASlCV.js.map +0 -1
- package/dist/idempotent-replay-JdgMQRRm.js +0 -21
- package/dist/idempotent-replay-JdgMQRRm.js.map +0 -1
- package/dist/initial-snapshot-Bx-c0pDP.d.ts.map +0 -1
- package/dist/internal-CscSY7K_.js.map +0 -1
- package/dist/server-D-Y4pqwu.js.map +0 -1
- package/dist/server-DYAATFuT.d.ts.map +0 -1
- package/dist/session-clear-C-QwTXV1.d.ts.map +0 -1
- package/dist/session-clear-DGI4jTQ5.js.map +0 -1
- package/dist/store-CyLUy86v.d.ts +0 -404
- package/dist/store-CyLUy86v.d.ts.map +0 -1
- package/dist/store-codec-DTG0Ftek.js +0 -10
- package/dist/store-codec-DTG0Ftek.js.map +0 -1
- package/dist/store-redis-notify-CM0AfLY0.js +0 -2544
- package/dist/store-redis-notify-CM0AfLY0.js.map +0 -1
- package/dist/telemetry-Cyfqo6Qt.d.ts.map +0 -1
- package/dist/wire-BKxH1vbP.js.map +0 -1
- package/docs/reference/05-adapters.mdx +0 -351
- package/docs/server/01-handlers.mdx +0 -200
- package/docs/server/05-production.mdx +0 -323
package/src/store-postgres.ts
CHANGED
|
@@ -10,16 +10,34 @@
|
|
|
10
10
|
*
|
|
11
11
|
* Semantics mirror the sqlite backend: `index` is stored as `idx`,
|
|
12
12
|
* timestamps are epoch milliseconds written from the injected clock —
|
|
13
|
-
* never SQL `now()
|
|
14
|
-
*
|
|
13
|
+
* never SQL `now()`, and dispatch state lives on consumer cursors, not on
|
|
14
|
+
* events. Streams poll by default; an optional direct connection enables
|
|
15
|
+
* LISTEN/NOTIFY for events and presence. Appends serialize per session via
|
|
15
16
|
* `pg_advisory_xact_lock`, so concurrent writers can't race the index
|
|
16
|
-
* assignment
|
|
17
|
+
* assignment, and each runs in one round trip (`a2_batch`, below), so the
|
|
18
|
+
* lock is held only while the server runs it.
|
|
19
|
+
*
|
|
20
|
+
* Lock discipline. Completions lock their cursor first, then the session lock
|
|
21
|
+
* when they append, or when their cursor goes idle (`completeAttempt`);
|
|
22
|
+
* everything else that takes the session lock takes it first. Every
|
|
23
|
+
* operation that locks a cursor row locks it only in the exact state it acts
|
|
24
|
+
* on (a head, an attempt). Under the session lock, an append never writes a
|
|
25
|
+
* cursor row that exists: a lane or log cursor's runs are rows keyed by the
|
|
26
|
+
* head each run started at (its epoch), so starting an idle cursor inserts a
|
|
27
|
+
* new run and retires the idle row with `skip locked`. An append therefore
|
|
28
|
+
* never waits on a cursor row, in particular not on one a completion waiting
|
|
29
|
+
* for the session lock holds, or one a clear left behind.
|
|
17
30
|
*/
|
|
18
31
|
|
|
19
32
|
import { A2Error, supersededAttemptError } from './errors.ts'
|
|
20
33
|
import { idempotentReplay } from './idempotent-replay.ts'
|
|
21
34
|
import { CLEAR_EVENT_TYPE, sessionClearedError } from './session-clear.ts'
|
|
22
|
-
import {
|
|
35
|
+
import {
|
|
36
|
+
cursorKeyOf,
|
|
37
|
+
decodeCause,
|
|
38
|
+
decodeReturnedEventIds,
|
|
39
|
+
idleLogCursor,
|
|
40
|
+
} from './store-codec.ts'
|
|
23
41
|
import { pollingStream, defaultSleep, type Sleeper } from './store-polling.ts'
|
|
24
42
|
import { NOTIFY_TIMINGS, nullProtoRecord } from './internal.ts'
|
|
25
43
|
import type { PresencePatch } from './contract.ts'
|
|
@@ -27,9 +45,13 @@ import { pollPresence } from './store-presence-polling.ts'
|
|
|
27
45
|
import {
|
|
28
46
|
postgresNotifications,
|
|
29
47
|
postgresNotificationChannel,
|
|
30
|
-
POSTGRES_NOTIFICATIONS_SQL,
|
|
31
48
|
type PostgresListenerConnection,
|
|
32
49
|
} from './postgres-notifications.ts'
|
|
50
|
+
import {
|
|
51
|
+
checkPostgresTriggers,
|
|
52
|
+
explainingPostgresClient,
|
|
53
|
+
explainPostgresError,
|
|
54
|
+
} from './postgres-schema.ts'
|
|
33
55
|
import { retryableLazy } from './retryable-lazy.ts'
|
|
34
56
|
import { postgresNotificationScope } from './postgres-notification-scope.ts'
|
|
35
57
|
import { postgresPoolClient } from './postgres-pool.ts'
|
|
@@ -43,10 +65,18 @@ import {
|
|
|
43
65
|
RANDOM_IDS,
|
|
44
66
|
SYSTEM_CLOCK,
|
|
45
67
|
type A2Store,
|
|
68
|
+
type AppendEvent,
|
|
46
69
|
type Clock,
|
|
70
|
+
type CompleteAttemptResult,
|
|
47
71
|
type Event,
|
|
48
72
|
type EventCause,
|
|
73
|
+
type HeadDispatch,
|
|
74
|
+
type HeadEvent,
|
|
75
|
+
type HeadRef,
|
|
49
76
|
type IdSource,
|
|
77
|
+
type ReturnedEvent,
|
|
78
|
+
type StoreConsumer,
|
|
79
|
+
type StoredCursor,
|
|
50
80
|
type StoredEvent,
|
|
51
81
|
type StoreStateReadRequest,
|
|
52
82
|
type StoreStateRead,
|
|
@@ -92,93 +122,11 @@ export type PostgresStore = A2Store & {
|
|
|
92
122
|
close(): Promise<void>
|
|
93
123
|
}
|
|
94
124
|
|
|
95
|
-
const SCHEMA = `
|
|
96
|
-
create table if not exists a2_events (
|
|
97
|
-
session_id text not null,
|
|
98
|
-
idx bigint not null,
|
|
99
|
-
event_type text not null,
|
|
100
|
-
payload jsonb not null,
|
|
101
|
-
event_id text not null,
|
|
102
|
-
created_at bigint not null,
|
|
103
|
-
cause jsonb,
|
|
104
|
-
lane text,
|
|
105
|
-
lane_ready boolean not null default false,
|
|
106
|
-
processed_at bigint,
|
|
107
|
-
processed_by_attempt integer,
|
|
108
|
-
returned_event_ids jsonb,
|
|
109
|
-
first_claimed_at bigint,
|
|
110
|
-
last_claimed_at bigint,
|
|
111
|
-
attempt_count integer not null default 0,
|
|
112
|
-
failure_count integer not null default 0,
|
|
113
|
-
last_failed_at bigint,
|
|
114
|
-
last_failed_attempt integer,
|
|
115
|
-
last_error text,
|
|
116
|
-
failed_at bigint,
|
|
117
|
-
claim_holder text,
|
|
118
|
-
claim_expires_at bigint,
|
|
119
|
-
primary key (session_id, idx)
|
|
120
|
-
);
|
|
121
|
-
|
|
122
|
-
create unique index if not exists a2_events_event_id on a2_events (event_id);
|
|
123
|
-
create index if not exists a2_events_unprocessed
|
|
124
|
-
on a2_events (session_id, idx) where processed_at is null;
|
|
125
|
-
create index if not exists a2_events_lane_pending
|
|
126
|
-
on a2_events (session_id, lane, idx) where processed_at is null;
|
|
127
|
-
create index if not exists a2_events_dispatch_ready
|
|
128
|
-
on a2_events (session_id, idx)
|
|
129
|
-
where lane_ready and processed_at is null and failed_at is null;
|
|
130
|
-
create index if not exists a2_events_claim_expiry
|
|
131
|
-
on a2_events (session_id, claim_expires_at)
|
|
132
|
-
where lane_ready and processed_at is null and failed_at is null;
|
|
133
|
-
|
|
134
|
-
create table if not exists a2_snapshots (
|
|
135
|
-
session_id text not null,
|
|
136
|
-
reducer_name text not null,
|
|
137
|
-
up_to_index bigint not null,
|
|
138
|
-
state jsonb not null,
|
|
139
|
-
updated_at bigint not null,
|
|
140
|
-
primary key (session_id, reducer_name)
|
|
141
|
-
);
|
|
142
|
-
|
|
143
|
-
create table if not exists a2_snapshot_history (
|
|
144
|
-
session_id text not null,
|
|
145
|
-
reducer_name text not null,
|
|
146
|
-
up_to_index bigint not null,
|
|
147
|
-
state jsonb not null,
|
|
148
|
-
updated_at bigint not null,
|
|
149
|
-
primary key (session_id, reducer_name, up_to_index)
|
|
150
|
-
);
|
|
151
|
-
|
|
152
|
-
create table if not exists a2_snapshot_pins (
|
|
153
|
-
session_id text not null,
|
|
154
|
-
event_index bigint not null,
|
|
155
|
-
reducer_name text not null,
|
|
156
|
-
up_to_index bigint not null,
|
|
157
|
-
primary key (session_id, event_index, reducer_name, up_to_index)
|
|
158
|
-
);
|
|
159
|
-
|
|
160
|
-
create index if not exists a2_snapshot_pins_checkpoint
|
|
161
|
-
on a2_snapshot_pins (session_id, reducer_name, up_to_index);
|
|
162
|
-
|
|
163
|
-
create table if not exists a2_clears (
|
|
164
|
-
session_id text primary key,
|
|
165
|
-
clear_index bigint not null
|
|
166
|
-
);
|
|
167
|
-
|
|
168
|
-
create table if not exists a2_presence (
|
|
169
|
-
ns text not null,
|
|
170
|
-
participant text not null,
|
|
171
|
-
field text not null,
|
|
172
|
-
value jsonb not null,
|
|
173
|
-
seen bigint not null,
|
|
174
|
-
at bigint not null,
|
|
175
|
-
expires_at bigint not null,
|
|
176
|
-
primary key (ns, participant, field)
|
|
177
|
-
);
|
|
178
|
-
`
|
|
179
|
-
|
|
180
125
|
const asNumber = (value: unknown): number => Number(value) // bigint columns arrive as strings
|
|
181
126
|
|
|
127
|
+
const asNullableNumber = (value: unknown): number | null =>
|
|
128
|
+
value === null || value === undefined ? null : asNumber(value)
|
|
129
|
+
|
|
182
130
|
/**
|
|
183
131
|
* The session's latest clear marker, or 0, as a primary-key subquery. Read
|
|
184
132
|
* it in a statement that runs after the operation's lock: a statement's
|
|
@@ -188,86 +136,232 @@ const asNumber = (value: unknown): number => Number(value) // bigint columns arr
|
|
|
188
136
|
const clearIndexOf = (session: string): string =>
|
|
189
137
|
`coalesce((select clear_index from a2_clears where session_id = ${session}), 0)`
|
|
190
138
|
|
|
139
|
+
/**
|
|
140
|
+
* Whether a served consumer has a cursor with a head in the session. A null
|
|
141
|
+
* `served` array serves every consumer.
|
|
142
|
+
*/
|
|
143
|
+
const pendingIn = (session: string, served: string): string =>
|
|
144
|
+
`exists(
|
|
145
|
+
select 1 from a2_cursors pending
|
|
146
|
+
where pending.session_id = ${session}
|
|
147
|
+
and pending.head > ${clearIndexOf(session)}
|
|
148
|
+
and pending.failed_at is null
|
|
149
|
+
and (${served}::text[] is null or pending.consumer = any(${served}::text[]))
|
|
150
|
+
)`
|
|
151
|
+
|
|
152
|
+
/** A fresh head's dispatch columns, for `set` clauses. */
|
|
153
|
+
const FRESH_DISPATCH = `
|
|
154
|
+
attempt_count = 0, first_claimed_at = null, last_claimed_at = null,
|
|
155
|
+
claim_holder = null, claim_expires_at = null, failure_count = 0,
|
|
156
|
+
last_failed_at = null, last_failed_attempt = null, last_error = null,
|
|
157
|
+
failed_at = null`
|
|
158
|
+
|
|
159
|
+
/** The head event's columns a claim, dead letter, or listing joins in, prefixed `e_`. */
|
|
160
|
+
const HEAD_COLUMNS = `
|
|
161
|
+
event.idx as e_idx, event.event_type as e_event_type,
|
|
162
|
+
event.payload as e_payload, event.event_id as e_event_id,
|
|
163
|
+
event.created_at as e_created_at, event.cause as e_cause`
|
|
164
|
+
|
|
191
165
|
const RECLAIM_BATCH = 256
|
|
192
166
|
|
|
193
|
-
|
|
194
|
-
|
|
167
|
+
/**
|
|
168
|
+
* A cursor run reclaim deletes once the session is cleared through `$2`:
|
|
169
|
+
* one from before the clear, or one a later run of its cursor replaced. A
|
|
170
|
+
* log cursor's last run stays: it records that the consumer counts from the
|
|
171
|
+
* marker.
|
|
172
|
+
*/
|
|
173
|
+
const SUPERSEDED_CURSOR = `
|
|
174
|
+
cur.session_id = $1 and (
|
|
175
|
+
(cur.epoch < $2 and cur.kind <> 'log')
|
|
176
|
+
or ((cur.head is null or cur.head <= $2) and exists (
|
|
177
|
+
select 1 from a2_cursors later
|
|
178
|
+
where later.session_id = cur.session_id
|
|
179
|
+
and later.cursor_key = cur.cursor_key
|
|
180
|
+
and later.consumer = cur.consumer
|
|
181
|
+
and later.epoch > cur.epoch
|
|
182
|
+
))
|
|
183
|
+
)`
|
|
195
184
|
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
}
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
}
|
|
211
|
-
if (
|
|
212
|
-
value.batchSize !== undefined &&
|
|
213
|
-
(!Number.isInteger(value.batchSize) || Number(value.batchSize) < 1)
|
|
214
|
-
) {
|
|
215
|
-
throw new TypeError('stored event has an invalid cause')
|
|
216
|
-
}
|
|
217
|
-
return {
|
|
218
|
-
index: Number(value.index),
|
|
219
|
-
attempt: Number(value.attempt),
|
|
220
|
-
...(value.batchSize === undefined
|
|
221
|
-
? {}
|
|
222
|
-
: { batchSize: Number(value.batchSize) }),
|
|
223
|
-
}
|
|
185
|
+
/**
|
|
186
|
+
* A value of `a2_batch`'s one parameter, `$1`: a JSON object the store builds
|
|
187
|
+
* for each call. JSON null reads as SQL null.
|
|
188
|
+
*/
|
|
189
|
+
const param = {
|
|
190
|
+
text: (key: string): string => `($1->>'${key}')`,
|
|
191
|
+
bigint: (key: string): string => `($1->>'${key}')::bigint`,
|
|
192
|
+
int: (key: string): string => `($1->>'${key}')::int`,
|
|
193
|
+
boolean: (key: string): string => `($1->>'${key}')::boolean`,
|
|
194
|
+
jsonb: (key: string): string => `nullif($1->'${key}', 'null'::jsonb)`,
|
|
195
|
+
/** A JSON array of strings as text[], or null when the value isn't an array. */
|
|
196
|
+
texts: (key: string): string =>
|
|
197
|
+
`(case when jsonb_typeof($1->'${key}') = 'array'
|
|
198
|
+
then array(select jsonb_array_elements_text($1->'${key}')) end)`,
|
|
224
199
|
}
|
|
225
200
|
|
|
226
|
-
|
|
201
|
+
/** Every batch names its session. */
|
|
202
|
+
const SESSION = param.text('session')
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* The next event after `after` a log cursor takes, as a scalar subquery:
|
|
206
|
+
* `types` is its JSON array, or null for every type but the clear marker
|
|
207
|
+
* (`clearType`). Each listed type is one probe of `a2_events_types`, so a
|
|
208
|
+
* consumer of rare types never scans the events it skips.
|
|
209
|
+
*/
|
|
210
|
+
const nextLogHead = ({
|
|
211
|
+
types,
|
|
212
|
+
after,
|
|
213
|
+
clearType = param.text('clearType'),
|
|
214
|
+
}: {
|
|
215
|
+
types: string
|
|
216
|
+
after: string
|
|
217
|
+
clearType?: string
|
|
218
|
+
}): string => `
|
|
219
|
+
case when ${types} is null then (
|
|
220
|
+
select event.idx from a2_events event
|
|
221
|
+
where event.session_id = ${SESSION} and event.idx > ${after}
|
|
222
|
+
and event.event_type <> ${clearType}
|
|
223
|
+
order by event.idx limit 1
|
|
224
|
+
) else (
|
|
225
|
+
select min(next.idx)
|
|
226
|
+
from jsonb_array_elements_text(${types}) as wanted(event_type)
|
|
227
|
+
cross join lateral (
|
|
228
|
+
select event.idx from a2_events event
|
|
229
|
+
where event.session_id = ${SESSION} and event.event_type = wanted.event_type
|
|
230
|
+
and event.idx > ${after}
|
|
231
|
+
order by event.idx limit 1
|
|
232
|
+
) next
|
|
233
|
+
where wanted.event_type <> ${clearType}
|
|
234
|
+
) end`
|
|
235
|
+
|
|
236
|
+
/** The latest run of `target`'s cursor, as a lateral subquery. */
|
|
237
|
+
const LATEST_RUN = (target: string): string => `
|
|
238
|
+
select cur.epoch, cur.position, cur.head from a2_cursors cur
|
|
239
|
+
where cur.session_id = ${SESSION} and cur.cursor_key = ${target}.cursor_key
|
|
240
|
+
and cur.consumer = ${target}.consumer
|
|
241
|
+
order by cur.epoch desc limit 1`
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* Retires the runs `target` replaces, those idle or superseded by the clear
|
|
245
|
+
* marker `clearIndex`, skipping any another operation holds: a later reclaim
|
|
246
|
+
* deletes those.
|
|
247
|
+
*/
|
|
248
|
+
const RETIRE_RUNS = (target: string, clearIndex: string): string => `
|
|
249
|
+
idle as materialized (
|
|
250
|
+
select cur.ctid from a2_cursors cur
|
|
251
|
+
join ${target} on cur.cursor_key = ${target}.cursor_key
|
|
252
|
+
and cur.consumer = ${target}.consumer
|
|
253
|
+
where cur.session_id = ${SESSION}
|
|
254
|
+
and (cur.head is null or cur.head <= ${clearIndex})
|
|
255
|
+
for update of cur skip locked
|
|
256
|
+
), retired as (
|
|
257
|
+
delete from a2_cursors old using idle where old.ctid = idle.ctid
|
|
258
|
+
)`
|
|
259
|
+
|
|
260
|
+
type Row = Record<string, unknown>
|
|
261
|
+
|
|
262
|
+
const toDate = (ms: unknown): Date | null =>
|
|
263
|
+
ms === null || ms === undefined ? null : new Date(asNumber(ms))
|
|
264
|
+
|
|
265
|
+
const toEvent = (row: Row): Event => ({
|
|
227
266
|
id: row['event_id'] as string,
|
|
228
267
|
type: row['event_type'] as string,
|
|
229
268
|
payload: row['payload'],
|
|
230
269
|
index: asNumber(row['idx']),
|
|
231
270
|
sessionId: row['session_id'] as string,
|
|
232
271
|
createdAt: new Date(asNumber(row['created_at'])),
|
|
233
|
-
|
|
272
|
+
})
|
|
273
|
+
|
|
274
|
+
const toStored = (row: Row): StoredEvent => ({
|
|
275
|
+
...toEvent(row),
|
|
276
|
+
cause: decodeCause(row['cause']),
|
|
277
|
+
})
|
|
278
|
+
|
|
279
|
+
const toDispatch = (row: Row): HeadDispatch => ({
|
|
280
|
+
consumer: row['consumer'] as string,
|
|
234
281
|
lane: (row['lane'] as string | null) ?? null,
|
|
235
|
-
processedAt: toDate(row['processed_at']),
|
|
236
|
-
processedByAttempt:
|
|
237
|
-
row['processed_by_attempt'] === null ||
|
|
238
|
-
row['processed_by_attempt'] === undefined
|
|
239
|
-
? null
|
|
240
|
-
: asNumber(row['processed_by_attempt']),
|
|
241
|
-
returnedEventIds:
|
|
242
|
-
row['returned_event_ids'] === null ||
|
|
243
|
-
row['returned_event_ids'] === undefined
|
|
244
|
-
? null
|
|
245
|
-
: decodeReturnedEventIds(row['returned_event_ids']),
|
|
246
282
|
firstClaimedAt: toDate(row['first_claimed_at']),
|
|
247
283
|
lastClaimedAt: toDate(row['last_claimed_at']),
|
|
248
284
|
attemptCount: asNumber(row['attempt_count']),
|
|
285
|
+
claimHolder: (row['claim_holder'] as string | null) ?? null,
|
|
286
|
+
claimExpiresAt: toDate(row['claim_expires_at']),
|
|
249
287
|
failureCount: asNumber(row['failure_count']),
|
|
250
288
|
lastFailedAt: toDate(row['last_failed_at']),
|
|
251
|
-
lastFailedAttempt:
|
|
252
|
-
row['last_failed_attempt'] === null ||
|
|
253
|
-
row['last_failed_attempt'] === undefined
|
|
254
|
-
? null
|
|
255
|
-
: asNumber(row['last_failed_attempt']),
|
|
289
|
+
lastFailedAttempt: asNullableNumber(row['last_failed_attempt']),
|
|
256
290
|
lastError: (row['last_error'] as string | null) ?? null,
|
|
257
291
|
failedAt: toDate(row['failed_at']),
|
|
258
|
-
claimHolder: (row['claim_holder'] as string | null) ?? null,
|
|
259
|
-
claimExpiresAt: toDate(row['claim_expires_at']),
|
|
260
292
|
})
|
|
261
293
|
|
|
262
|
-
const
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
294
|
+
const freshDispatch = (
|
|
295
|
+
consumer: string,
|
|
296
|
+
lane: string | null,
|
|
297
|
+
): HeadDispatch => ({
|
|
298
|
+
consumer,
|
|
299
|
+
lane,
|
|
300
|
+
firstClaimedAt: null,
|
|
301
|
+
lastClaimedAt: null,
|
|
302
|
+
attemptCount: 0,
|
|
303
|
+
claimHolder: null,
|
|
304
|
+
claimExpiresAt: null,
|
|
305
|
+
failureCount: 0,
|
|
306
|
+
lastFailedAt: null,
|
|
307
|
+
lastFailedAttempt: null,
|
|
308
|
+
lastError: null,
|
|
309
|
+
failedAt: null,
|
|
310
|
+
})
|
|
311
|
+
|
|
312
|
+
/** A cursor row joined with its head event's `e_` columns. */
|
|
313
|
+
const toHead = (row: Row): HeadEvent => ({
|
|
314
|
+
...toStored({
|
|
315
|
+
session_id: row['session_id'],
|
|
316
|
+
idx: row['e_idx'],
|
|
317
|
+
event_type: row['e_event_type'],
|
|
318
|
+
payload: row['e_payload'],
|
|
319
|
+
event_id: row['e_event_id'],
|
|
320
|
+
created_at: row['e_created_at'],
|
|
321
|
+
cause: row['e_cause'],
|
|
322
|
+
}),
|
|
323
|
+
...toDispatch(row),
|
|
269
324
|
})
|
|
270
325
|
|
|
326
|
+
/** A cursor run as `cursors()` reports it, given the session's clear marker. */
|
|
327
|
+
const toCursor = (row: Row, clearIndex: number): StoredCursor => {
|
|
328
|
+
const position = asNumber(row['position'])
|
|
329
|
+
if (row['kind'] !== 'log' || asNumber(row['epoch']) > clearIndex) {
|
|
330
|
+
return Object.assign(toDispatch(row), {
|
|
331
|
+
position,
|
|
332
|
+
head: asNullableNumber(row['head']),
|
|
333
|
+
})
|
|
334
|
+
}
|
|
335
|
+
// A clear supersedes a log cursor without touching its row: it counts
|
|
336
|
+
// from the marker, with nothing to run.
|
|
337
|
+
return Object.assign(
|
|
338
|
+
freshDispatch(row['consumer'] as string, row['lane'] as string),
|
|
339
|
+
{ position: Math.max(position, clearIndex), head: null },
|
|
340
|
+
)
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
const compareText = (a: string, b: string): number =>
|
|
344
|
+
a < b ? -1 : a > b ? 1 : 0
|
|
345
|
+
|
|
346
|
+
const byHead = (a: HeadEvent, b: HeadEvent): number =>
|
|
347
|
+
a.index - b.index || compareText(a.consumer, b.consumer)
|
|
348
|
+
|
|
349
|
+
/** Consumer, then lane, with concurrent cursors last by head. */
|
|
350
|
+
const byCursor = (a: StoredCursor, b: StoredCursor): number =>
|
|
351
|
+
compareText(a.consumer, b.consumer) ||
|
|
352
|
+
(a.lane === null
|
|
353
|
+
? b.lane === null
|
|
354
|
+
? (a.head ?? 0) - (b.head ?? 0)
|
|
355
|
+
: 1
|
|
356
|
+
: b.lane === null
|
|
357
|
+
? -1
|
|
358
|
+
: compareText(a.lane, b.lane))
|
|
359
|
+
|
|
360
|
+
const servedNames = (
|
|
361
|
+
consumers: readonly StoreConsumer[] | undefined,
|
|
362
|
+
): string[] | null =>
|
|
363
|
+
consumers === undefined ? null : consumers.map((consumer) => consumer.name)
|
|
364
|
+
|
|
271
365
|
const wrap = async <T>(fn: () => Promise<T>): Promise<T> => {
|
|
272
366
|
try {
|
|
273
367
|
return await fn()
|
|
@@ -281,37 +375,6 @@ const wrap = async <T>(fn: () => Promise<T>): Promise<T> => {
|
|
|
281
375
|
|
|
282
376
|
const jsonb = (value: unknown): string => JSON.stringify(value) ?? 'null'
|
|
283
377
|
|
|
284
|
-
const initializeSchema = async ({
|
|
285
|
-
client,
|
|
286
|
-
listen,
|
|
287
|
-
}: {
|
|
288
|
-
client: PostgresClient
|
|
289
|
-
listen: boolean
|
|
290
|
-
}): Promise<void> => {
|
|
291
|
-
const schema = SCHEMA + (listen ? POSTGRES_NOTIFICATIONS_SQL : '')
|
|
292
|
-
if (!client.connect) {
|
|
293
|
-
await client.query(schema)
|
|
294
|
-
return
|
|
295
|
-
}
|
|
296
|
-
|
|
297
|
-
const conn = await client.connect()
|
|
298
|
-
try {
|
|
299
|
-
await conn.query('begin')
|
|
300
|
-
// `if not exists` does not serialize concurrent catalog writes. Every
|
|
301
|
-
// serverless isolate takes the same transaction-scoped lock before DDL.
|
|
302
|
-
await conn.query(
|
|
303
|
-
"select pg_advisory_xact_lock(hashtext('a2:postgres:schema'))",
|
|
304
|
-
)
|
|
305
|
-
await conn.query(schema)
|
|
306
|
-
await conn.query('commit')
|
|
307
|
-
} catch (err) {
|
|
308
|
-
await conn.query('rollback').catch(() => {})
|
|
309
|
-
throw err
|
|
310
|
-
} finally {
|
|
311
|
-
conn.release()
|
|
312
|
-
}
|
|
313
|
-
}
|
|
314
|
-
|
|
315
378
|
const loadPg = async () =>
|
|
316
379
|
import('pg').catch(() => {
|
|
317
380
|
throw new A2Error(
|
|
@@ -350,20 +413,27 @@ const createListener = ({
|
|
|
350
413
|
return listenerConnection
|
|
351
414
|
})
|
|
352
415
|
|
|
353
|
-
|
|
416
|
+
type Query = PostgresConnection['query']
|
|
417
|
+
|
|
418
|
+
/** A finished or dead head's snapshot pins, and history no pin keeps. */
|
|
354
419
|
const releasePins = async ({
|
|
355
420
|
query,
|
|
356
421
|
sessionId,
|
|
422
|
+
consumer,
|
|
423
|
+
cursorKey,
|
|
357
424
|
index,
|
|
358
425
|
}: {
|
|
359
|
-
query:
|
|
426
|
+
query: Query
|
|
360
427
|
sessionId: string
|
|
428
|
+
consumer: string
|
|
429
|
+
cursorKey: string
|
|
361
430
|
index: number
|
|
362
431
|
}): Promise<void> => {
|
|
363
432
|
await query(
|
|
364
433
|
`delete from a2_snapshot_pins
|
|
365
|
-
where session_id = $1 and event_index = $2
|
|
366
|
-
|
|
434
|
+
where session_id = $1 and event_index = $2
|
|
435
|
+
and consumer = $3 and cursor_key = $4`,
|
|
436
|
+
[sessionId, index, consumer, cursorKey],
|
|
367
437
|
)
|
|
368
438
|
await query(
|
|
369
439
|
`delete from a2_snapshot_history history
|
|
@@ -378,6 +448,925 @@ const releasePins = async ({
|
|
|
378
448
|
)
|
|
379
449
|
}
|
|
380
450
|
|
|
451
|
+
const lockSession = (query: Query, sessionId: string) =>
|
|
452
|
+
query('select pg_advisory_xact_lock(hashtext($1))', [sessionId])
|
|
453
|
+
|
|
454
|
+
/** Whether the session has an event at `index`. */
|
|
455
|
+
const hasEvent = async (
|
|
456
|
+
query: Query,
|
|
457
|
+
sessionId: string,
|
|
458
|
+
index: number,
|
|
459
|
+
): Promise<boolean> =>
|
|
460
|
+
(
|
|
461
|
+
await query('select 1 from a2_events where session_id = $1 and idx = $2', [
|
|
462
|
+
sessionId,
|
|
463
|
+
index,
|
|
464
|
+
])
|
|
465
|
+
).rows.length > 0
|
|
466
|
+
|
|
467
|
+
/**
|
|
468
|
+
* The attempt-currency fence shared by consumer appends and clears, under
|
|
469
|
+
* the session lock. The cursor is read locked (`for share` by an append,
|
|
470
|
+
* `for update` by a clear, which then releases the head): it serializes
|
|
471
|
+
* with every writer of the cursor (claims, completions, failures), so a
|
|
472
|
+
* stale read cannot pass the fence against a takeover committing beside
|
|
473
|
+
* it. The lock is only taken while the head and attempt still match.
|
|
474
|
+
*/
|
|
475
|
+
const assertCurrentCause = async ({
|
|
476
|
+
query,
|
|
477
|
+
sessionId,
|
|
478
|
+
cause,
|
|
479
|
+
clearIndex,
|
|
480
|
+
lock,
|
|
481
|
+
}: {
|
|
482
|
+
query: Query
|
|
483
|
+
sessionId: string
|
|
484
|
+
cause: EventCause
|
|
485
|
+
clearIndex: number
|
|
486
|
+
lock: 'share' | 'update'
|
|
487
|
+
}): Promise<void> => {
|
|
488
|
+
if (cause.index <= clearIndex) throw supersededAttemptError(sessionId, cause)
|
|
489
|
+
const { rows } = await query(
|
|
490
|
+
`select 1 from a2_cursors
|
|
491
|
+
where session_id = $1 and cursor_key = $2 and consumer = $3
|
|
492
|
+
and head = $4 and attempt_count = $5 and failed_at is null
|
|
493
|
+
for ${lock}`,
|
|
494
|
+
[
|
|
495
|
+
sessionId,
|
|
496
|
+
cursorKeyOf(cause.lane, cause.index),
|
|
497
|
+
cause.consumer,
|
|
498
|
+
cause.index,
|
|
499
|
+
cause.attempt,
|
|
500
|
+
],
|
|
501
|
+
)
|
|
502
|
+
if (rows.length === 0) throw supersededAttemptError(sessionId, cause)
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
// ── one-round-trip operations ─────────────────────────────────────────
|
|
506
|
+
//
|
|
507
|
+
// An operation that writes under the session lock runs as one call of
|
|
508
|
+
// `a2_batch` (schema.sql): its statements, in order, in one transaction and
|
|
509
|
+
// one round trip, the lock held only while the server runs them. The function
|
|
510
|
+
// is what makes that correct. A statement's snapshot is taken before any lock
|
|
511
|
+
// it waits for, so a single statement that took the lock would still miss
|
|
512
|
+
// what the lock's last holder committed; inside the function each statement
|
|
513
|
+
// takes its snapshot when it starts. Every statement is a select of jsonb rows
|
|
514
|
+
// reading its values from `$1` (`param`). A row with `abort` fails the call and
|
|
515
|
+
// rolls it back (`abortedBatch`), and one with `stop` ends it there.
|
|
516
|
+
|
|
517
|
+
const BATCH = 'select r from a2_batch($1::text[], $2::jsonb) as r'
|
|
518
|
+
|
|
519
|
+
/**
|
|
520
|
+
* Each statement as a batch sends it, whitespace collapsed: a batch sends all
|
|
521
|
+
* its statements on every call. They hold no comments or spaced literals, and
|
|
522
|
+
* they're this module's constants, so the cache stays this size.
|
|
523
|
+
*/
|
|
524
|
+
const compacted = new Map<string, string>()
|
|
525
|
+
const compact = (statement: string): string => {
|
|
526
|
+
let sent = compacted.get(statement)
|
|
527
|
+
if (sent === undefined) {
|
|
528
|
+
// A line comment would swallow the rest of the statement once it's one line.
|
|
529
|
+
if (statement.includes('--'))
|
|
530
|
+
throw new TypeError('a batch statement holds a comment')
|
|
531
|
+
sent = statement.replace(/\s+/g, ' ').trim()
|
|
532
|
+
compacted.set(statement, sent)
|
|
533
|
+
}
|
|
534
|
+
return sent
|
|
535
|
+
}
|
|
536
|
+
|
|
537
|
+
/** The SQLSTATE `a2_batch` raises an `abort` row with, as its detail. */
|
|
538
|
+
const BATCH_ABORTED = 'A2001'
|
|
539
|
+
|
|
540
|
+
/** The row an aborted `a2_batch` call raised, from `error` or its cause chain. */
|
|
541
|
+
const abortedBatch = (error: unknown): Row | undefined => {
|
|
542
|
+
for (let current = error, depth = 0; depth < 5; depth += 1) {
|
|
543
|
+
if (typeof current !== 'object' || current === null) return undefined
|
|
544
|
+
const { code, detail, cause } = current as {
|
|
545
|
+
code?: unknown
|
|
546
|
+
detail?: unknown
|
|
547
|
+
cause?: unknown
|
|
548
|
+
}
|
|
549
|
+
if (code === BATCH_ABORTED && typeof detail === 'string')
|
|
550
|
+
return JSON.parse(detail) as Row
|
|
551
|
+
current = cause
|
|
552
|
+
}
|
|
553
|
+
return undefined
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
const LOCK_SESSION = `
|
|
557
|
+
select jsonb_build_object('locked', true)
|
|
558
|
+
from pg_advisory_xact_lock(hashtext(${SESSION}))`
|
|
559
|
+
|
|
560
|
+
const INDEX = param.bigint('index')
|
|
561
|
+
const CONSUMER = param.text('consumer')
|
|
562
|
+
|
|
563
|
+
/**
|
|
564
|
+
* The index before the events this call inserted, the last `count` of the
|
|
565
|
+
* session's: read under the session lock, after the insert.
|
|
566
|
+
*/
|
|
567
|
+
const INSERTED_AFTER = `(
|
|
568
|
+
select coalesce(max(idx), 0) - ${param.bigint('count')}
|
|
569
|
+
from a2_events where session_id = ${SESSION})`
|
|
570
|
+
|
|
571
|
+
/**
|
|
572
|
+
* An append's checks, under the session lock: the session's head and pending
|
|
573
|
+
* state, and what its supplied `ids` say. When every event's id belongs to
|
|
574
|
+
* this session already, the batch is a retry: it stops with the committed
|
|
575
|
+
* rows. Some owned, or one in another session, aborts, as does scheduled work
|
|
576
|
+
* from at or below the clear marker (`origin`). Rows below the marker own no
|
|
577
|
+
* id.
|
|
578
|
+
*/
|
|
579
|
+
const APPEND_CHECKS = `
|
|
580
|
+
with boundary as materialized (
|
|
581
|
+
select coalesce(max(idx), 0) as base,
|
|
582
|
+
${clearIndexOf(SESSION)} as clear_index,
|
|
583
|
+
${pendingIn(SESSION, param.texts('served'))} as has_pending
|
|
584
|
+
from a2_events where session_id = ${SESSION}
|
|
585
|
+
), owned as materialized (
|
|
586
|
+
select event.session_id, event.idx, event.event_type, event.payload,
|
|
587
|
+
event.event_id, event.created_at, event.cause
|
|
588
|
+
from a2_events event, boundary
|
|
589
|
+
where event.event_id = any(${param.texts('ids')})
|
|
590
|
+
and not (event.session_id = ${SESSION}
|
|
591
|
+
and event.idx < boundary.clear_index)
|
|
592
|
+
)
|
|
593
|
+
select case
|
|
594
|
+
when exists (select 1 from owned where session_id <> ${SESSION}) then
|
|
595
|
+
jsonb_build_object('abort', 'foreign', 'event_id', (
|
|
596
|
+
select event_id from owned where session_id <> ${SESSION}
|
|
597
|
+
order by idx limit 1))
|
|
598
|
+
when (select count(*) from owned) = ${param.bigint('count')} then
|
|
599
|
+
jsonb_build_object('stop', 'replay',
|
|
600
|
+
'has_pending', boundary.has_pending,
|
|
601
|
+
'events', (select jsonb_agg(to_jsonb(owned) order by owned.idx)
|
|
602
|
+
from owned))
|
|
603
|
+
when exists (select 1 from owned) then
|
|
604
|
+
jsonb_build_object('abort', 'mixed', 'owned',
|
|
605
|
+
(select count(*) from owned))
|
|
606
|
+
when ${param.bigint('origin')} <= boundary.clear_index then
|
|
607
|
+
jsonb_build_object('abort', 'cleared')
|
|
608
|
+
else jsonb_build_object('base', boundary.base,
|
|
609
|
+
'has_pending', boundary.has_pending)
|
|
610
|
+
end
|
|
611
|
+
from boundary`
|
|
612
|
+
|
|
613
|
+
/**
|
|
614
|
+
* The attempt-currency fence of a consumer append, under the session lock.
|
|
615
|
+
* Each cause's cursor is read `for share`: that serializes with every writer
|
|
616
|
+
* of the cursor (claims, completions, failures), so a stale read cannot pass
|
|
617
|
+
* the fence against a takeover committing beside it. A row is locked only
|
|
618
|
+
* while its head and attempt still match. The first cause that fails aborts.
|
|
619
|
+
*/
|
|
620
|
+
const FENCE = `
|
|
621
|
+
with cause as (
|
|
622
|
+
select * from jsonb_to_recordset($1->'causes')
|
|
623
|
+
as cause(ord int, consumer text, cursor_key text, idx bigint, attempt int)
|
|
624
|
+
), held as materialized (
|
|
625
|
+
select cause.ord from cause
|
|
626
|
+
join a2_cursors cur
|
|
627
|
+
on cur.session_id = ${SESSION} and cur.cursor_key = cause.cursor_key
|
|
628
|
+
and cur.consumer = cause.consumer and cur.head = cause.idx
|
|
629
|
+
and cur.attempt_count = cause.attempt and cur.failed_at is null
|
|
630
|
+
where cause.idx > ${clearIndexOf(SESSION)}
|
|
631
|
+
for share of cur
|
|
632
|
+
)
|
|
633
|
+
select jsonb_build_object('abort', 'superseded', 'ord', cause.ord)
|
|
634
|
+
from cause where cause.ord not in (select ord from held)
|
|
635
|
+
order by cause.ord limit 1`
|
|
636
|
+
|
|
637
|
+
/**
|
|
638
|
+
* Frees the `ids` this session's rows below its clear marker hold, so the
|
|
639
|
+
* batch can take them. It never waits on a row: a reclaim deleting one may
|
|
640
|
+
* hold it, and one it skips aborts.
|
|
641
|
+
*/
|
|
642
|
+
const REUSE_IDS = `
|
|
643
|
+
with stale as materialized (
|
|
644
|
+
select event.idx from a2_events event
|
|
645
|
+
where event.session_id = ${SESSION}
|
|
646
|
+
and event.event_id = any(${param.texts('ids')})
|
|
647
|
+
and event.idx < ${clearIndexOf(SESSION)}
|
|
648
|
+
order by event.idx
|
|
649
|
+
for update skip locked
|
|
650
|
+
), deleted as (
|
|
651
|
+
delete from a2_events event using stale
|
|
652
|
+
where event.session_id = ${SESSION} and event.idx = stale.idx
|
|
653
|
+
returning event.idx
|
|
654
|
+
)
|
|
655
|
+
select jsonb_build_object('abort', 'held')
|
|
656
|
+
where (select count(*) from deleted) < (
|
|
657
|
+
select count(*) from a2_events event
|
|
658
|
+
where event.session_id = ${SESSION}
|
|
659
|
+
and event.event_id = any(${param.texts('ids')})
|
|
660
|
+
and event.idx < ${clearIndexOf(SESSION)})`
|
|
661
|
+
|
|
662
|
+
/** Inserts `events` after the session's head, under the session lock. */
|
|
663
|
+
const INSERT_EVENTS = `
|
|
664
|
+
with inserted as (
|
|
665
|
+
insert into a2_events
|
|
666
|
+
(session_id, idx, event_type, payload, event_id, created_at, cause)
|
|
667
|
+
select ${SESSION}, base.base + event.ord, event.type,
|
|
668
|
+
coalesce(event.payload, 'null'::jsonb), event.id,
|
|
669
|
+
${param.bigint('now')}, event.cause
|
|
670
|
+
from (select coalesce(max(idx), 0) as base from a2_events
|
|
671
|
+
where session_id = ${SESSION}) base,
|
|
672
|
+
jsonb_to_recordset($1->'events')
|
|
673
|
+
as event(ord bigint, type text, payload jsonb, id text, cause jsonb)
|
|
674
|
+
returning 1
|
|
675
|
+
)
|
|
676
|
+
select jsonb_build_object('inserted', count(*)) from inserted`
|
|
677
|
+
|
|
678
|
+
/** A cursor for each concurrent target: the event alone. */
|
|
679
|
+
const START_EVENTS = `
|
|
680
|
+
with started as (
|
|
681
|
+
insert into a2_cursors
|
|
682
|
+
(session_id, consumer, cursor_key, epoch, lane, kind, position, head)
|
|
683
|
+
select ${SESSION}, target.consumer, '#' || target.idx, target.idx, null,
|
|
684
|
+
'event', target.idx - 1, target.idx
|
|
685
|
+
from (
|
|
686
|
+
select single.consumer, ${INSERTED_AFTER} + single.ord as idx
|
|
687
|
+
from jsonb_to_recordset($1->'singles')
|
|
688
|
+
as single(consumer text, ord bigint)
|
|
689
|
+
) target
|
|
690
|
+
on conflict do nothing
|
|
691
|
+
returning 1
|
|
692
|
+
)
|
|
693
|
+
select jsonb_build_object('started', count(*)) from started`
|
|
694
|
+
|
|
695
|
+
/** Queues each lane target's event in its lane. */
|
|
696
|
+
const QUEUE_LANES = `
|
|
697
|
+
with queued as (
|
|
698
|
+
insert into a2_lane_events (session_id, consumer, lane, idx)
|
|
699
|
+
select ${SESSION}, queued.consumer, queued.lane,
|
|
700
|
+
${INSERTED_AFTER} + queued.ord
|
|
701
|
+
from jsonb_to_recordset($1->'queued')
|
|
702
|
+
as queued(consumer text, lane text, ord bigint)
|
|
703
|
+
on conflict do nothing
|
|
704
|
+
returning 1
|
|
705
|
+
)
|
|
706
|
+
select jsonb_build_object('queued', count(*)) from queued`
|
|
707
|
+
|
|
708
|
+
/**
|
|
709
|
+
* Starts a run for each lane cursor the batch queues work for that has none
|
|
710
|
+
* running: one that is idle, superseded by the clear marker, or new. It starts
|
|
711
|
+
* at the lane's first event in the batch and counts from the marker.
|
|
712
|
+
*/
|
|
713
|
+
const START_LANES = `
|
|
714
|
+
with boundary as materialized (
|
|
715
|
+
select ${INSERTED_AFTER} as base, ${clearIndexOf(SESSION)} as clear_index
|
|
716
|
+
), target as (
|
|
717
|
+
select first.consumer, first.cursor_key, first.lane,
|
|
718
|
+
boundary.base + first.ord as idx
|
|
719
|
+
from boundary, jsonb_to_recordset($1->'lanes')
|
|
720
|
+
as first(consumer text, cursor_key text, lane text, ord bigint)
|
|
721
|
+
), started as materialized (
|
|
722
|
+
select target.*, latest.position
|
|
723
|
+
from target
|
|
724
|
+
left join lateral (${LATEST_RUN('target')}) latest on true
|
|
725
|
+
where latest.head is null
|
|
726
|
+
or latest.head <= (select clear_index from boundary)
|
|
727
|
+
), ${RETIRE_RUNS('started', '(select clear_index from boundary)')},
|
|
728
|
+
inserted as (
|
|
729
|
+
insert into a2_cursors
|
|
730
|
+
(session_id, consumer, cursor_key, epoch, lane, kind, position, head)
|
|
731
|
+
select ${SESSION}, started.consumer, started.cursor_key, started.idx,
|
|
732
|
+
started.lane, 'lane',
|
|
733
|
+
greatest(coalesce(started.position, 0),
|
|
734
|
+
(select clear_index from boundary)),
|
|
735
|
+
started.idx
|
|
736
|
+
from started
|
|
737
|
+
on conflict do nothing
|
|
738
|
+
returning 1
|
|
739
|
+
)
|
|
740
|
+
select jsonb_build_object('lanes', count(*)) from inserted`
|
|
741
|
+
|
|
742
|
+
/**
|
|
743
|
+
* Starts a run for each served log cursor the batch carries types for that
|
|
744
|
+
* has none running. A missing cursor starts at the batch's first event of its
|
|
745
|
+
* types, or at the beginning of the log with backfill. One that exists starts
|
|
746
|
+
* at the next event it takes after its position: a deployment that does not
|
|
747
|
+
* serve the consumer may have appended some while it was idle. A cursor from
|
|
748
|
+
* before the clear marker counts from it.
|
|
749
|
+
*/
|
|
750
|
+
const WAKE_LOGS = `
|
|
751
|
+
with boundary as materialized (
|
|
752
|
+
select ${INSERTED_AFTER} as base, ${clearIndexOf(SESSION)} as clear_index
|
|
753
|
+
), target as (
|
|
754
|
+
select log.consumer, log.cursor_key, log.lane,
|
|
755
|
+
boundary.base + log.ord as first, log.types, log.backfill
|
|
756
|
+
from boundary, jsonb_to_recordset($1->'logs')
|
|
757
|
+
as log(consumer text, cursor_key text, lane text, ord bigint,
|
|
758
|
+
types jsonb, backfill boolean)
|
|
759
|
+
), woken as materialized (
|
|
760
|
+
select target.*,
|
|
761
|
+
case when latest.epoch is not null
|
|
762
|
+
then greatest(latest.position,
|
|
763
|
+
(select clear_index from boundary))
|
|
764
|
+
when target.backfill then (select clear_index from boundary)
|
|
765
|
+
else (select base from boundary) end as position,
|
|
766
|
+
latest.epoch is not null or target.backfill as resumes
|
|
767
|
+
from target
|
|
768
|
+
left join lateral (${LATEST_RUN('target')}) latest on true
|
|
769
|
+
where latest.head is null
|
|
770
|
+
or latest.head <= (select clear_index from boundary)
|
|
771
|
+
), ${RETIRE_RUNS('woken', '(select clear_index from boundary)')}, run as (
|
|
772
|
+
select woken.*,
|
|
773
|
+
case when woken.resumes then coalesce(${nextLogHead({
|
|
774
|
+
types: 'woken.types',
|
|
775
|
+
after: 'woken.position',
|
|
776
|
+
})}, woken.first) else woken.first end as head
|
|
777
|
+
from woken
|
|
778
|
+
), inserted as (
|
|
779
|
+
insert into a2_cursors
|
|
780
|
+
(session_id, consumer, cursor_key, epoch, lane, kind, position, head,
|
|
781
|
+
types)
|
|
782
|
+
select ${SESSION}, run.consumer, run.cursor_key, run.head, run.lane, 'log',
|
|
783
|
+
run.position, run.head, run.types
|
|
784
|
+
from run
|
|
785
|
+
on conflict do nothing
|
|
786
|
+
returning 1
|
|
787
|
+
)
|
|
788
|
+
select jsonb_build_object('logs', count(*)) from inserted`
|
|
789
|
+
|
|
790
|
+
/**
|
|
791
|
+
* The heads a claim may take, up to `through` when there is one, as
|
|
792
|
+
* `from ... where` for `cur`. `for update` makes a claim racing another wait
|
|
793
|
+
* and then re-check the row it waited for.
|
|
794
|
+
*/
|
|
795
|
+
const ELIGIBLE = `
|
|
796
|
+
from a2_cursors cur
|
|
797
|
+
where cur.session_id = ${SESSION}
|
|
798
|
+
and cur.head > ${clearIndexOf(SESSION)}
|
|
799
|
+
and (${param.bigint('through')} is null
|
|
800
|
+
or cur.head <= ${param.bigint('through')})
|
|
801
|
+
and cur.failed_at is null
|
|
802
|
+
and (${param.texts('served')} is null
|
|
803
|
+
or cur.consumer = any(${param.texts('served')}))
|
|
804
|
+
and not exists (
|
|
805
|
+
select 1 from jsonb_to_recordset($1->'excluded')
|
|
806
|
+
as skip(consumer text, cursor_key text)
|
|
807
|
+
where skip.consumer = cur.consumer and skip.cursor_key = cur.cursor_key
|
|
808
|
+
)`
|
|
809
|
+
|
|
810
|
+
/** A lapsed claim's takeover that spends the last failed attempt dead-letters the head instead. */
|
|
811
|
+
const EXHAUSTED = `(cur.claim_holder is not null
|
|
812
|
+
and cur.failure_count + 1 >= ${param.int('maxFailures')})`
|
|
813
|
+
|
|
814
|
+
/**
|
|
815
|
+
* Claims every eligible head, returning each with its event. A head the claim
|
|
816
|
+
* waited on may have moved to an event its snapshot predates: that row comes
|
|
817
|
+
* back without one (`e_idx` null), for `claimAvailable` to read. It stops
|
|
818
|
+
* there unless one dead-lettered, whose pins `RELEASE_DEAD` then releases, or
|
|
819
|
+
* none was claimed, for `RETRY_AT`.
|
|
820
|
+
*/
|
|
821
|
+
const CLAIM = `
|
|
822
|
+
with eligible as materialized (
|
|
823
|
+
select cur.consumer, cur.cursor_key, cur.epoch
|
|
824
|
+
${ELIGIBLE}
|
|
825
|
+
and (cur.claim_expires_at is null
|
|
826
|
+
or cur.claim_expires_at <= ${param.bigint('now')})
|
|
827
|
+
order by cur.head, cur.consumer
|
|
828
|
+
for update of cur
|
|
829
|
+
), claimed as (
|
|
830
|
+
update a2_cursors cur set
|
|
831
|
+
attempt_count = cur.attempt_count + 1,
|
|
832
|
+
failure_count = cur.failure_count
|
|
833
|
+
+ case when cur.claim_holder is null then 0 else 1 end,
|
|
834
|
+
failed_at = case when ${EXHAUSTED} then ${param.bigint('now')}
|
|
835
|
+
else cur.failed_at end,
|
|
836
|
+
first_claimed_at = case when ${EXHAUSTED} then cur.first_claimed_at
|
|
837
|
+
else coalesce(cur.first_claimed_at, ${param.bigint('now')}) end,
|
|
838
|
+
last_claimed_at = case when ${EXHAUSTED} then cur.last_claimed_at
|
|
839
|
+
else ${param.bigint('now')} end,
|
|
840
|
+
claim_holder = case when ${EXHAUSTED} then null
|
|
841
|
+
else ${param.text('holder')} end,
|
|
842
|
+
claim_expires_at = case when ${EXHAUSTED} then null
|
|
843
|
+
else ${param.bigint('expiresAt')} end
|
|
844
|
+
from eligible
|
|
845
|
+
where cur.session_id = ${SESSION}
|
|
846
|
+
and cur.cursor_key = eligible.cursor_key
|
|
847
|
+
and cur.consumer = eligible.consumer
|
|
848
|
+
and cur.epoch = eligible.epoch
|
|
849
|
+
returning cur.*
|
|
850
|
+
), heads as (
|
|
851
|
+
select claimed.*, ${HEAD_COLUMNS},
|
|
852
|
+
${clearIndexOf(SESSION)} as clear_index,
|
|
853
|
+
(select coalesce(max(idx), 0) from a2_events
|
|
854
|
+
where session_id = ${SESSION}) as session_head
|
|
855
|
+
from claimed
|
|
856
|
+
left join a2_events event
|
|
857
|
+
on event.session_id = ${SESSION} and event.idx = claimed.head
|
|
858
|
+
)
|
|
859
|
+
select r from (
|
|
860
|
+
select 0 as ord, to_jsonb(heads) as r from heads
|
|
861
|
+
union all
|
|
862
|
+
select 1, jsonb_build_object('stop', 'claimed')
|
|
863
|
+
where exists (select 1 from heads)
|
|
864
|
+
and not exists (select 1 from heads where failed_at is not null)
|
|
865
|
+
) claim
|
|
866
|
+
order by ord`
|
|
867
|
+
|
|
868
|
+
/**
|
|
869
|
+
* Releases the pins of the heads dead-lettered at `now`, the claim's among
|
|
870
|
+
* them, in a snapshot after the claim's locks, with the history no other pin
|
|
871
|
+
* keeps.
|
|
872
|
+
*/
|
|
873
|
+
const RELEASE_DEAD = `
|
|
874
|
+
with dead as materialized (
|
|
875
|
+
select cur.consumer, cur.cursor_key, cur.head from a2_cursors cur
|
|
876
|
+
where cur.session_id = ${SESSION} and cur.head is not null
|
|
877
|
+
and cur.failed_at = ${param.bigint('now')} and cur.claim_holder is null
|
|
878
|
+
), unpinned as (
|
|
879
|
+
delete from a2_snapshot_pins pin using dead
|
|
880
|
+
where pin.session_id = ${SESSION} and pin.event_index = dead.head
|
|
881
|
+
and pin.consumer = dead.consumer and pin.cursor_key = dead.cursor_key
|
|
882
|
+
returning 1
|
|
883
|
+
), unkept as (
|
|
884
|
+
delete from a2_snapshot_history history
|
|
885
|
+
where history.session_id = ${SESSION}
|
|
886
|
+
and exists (select 1 from dead)
|
|
887
|
+
and not exists (
|
|
888
|
+
select 1 from a2_snapshot_pins pin
|
|
889
|
+
where pin.session_id = history.session_id
|
|
890
|
+
and pin.reducer_name = history.reducer_name
|
|
891
|
+
and pin.up_to_index = history.up_to_index
|
|
892
|
+
and not exists (
|
|
893
|
+
select 1 from dead
|
|
894
|
+
where pin.event_index = dead.head
|
|
895
|
+
and pin.consumer = dead.consumer
|
|
896
|
+
and pin.cursor_key = dead.cursor_key)
|
|
897
|
+
)
|
|
898
|
+
returning 1
|
|
899
|
+
)
|
|
900
|
+
select jsonb_build_object('released', (select count(*) from unpinned))`
|
|
901
|
+
|
|
902
|
+
/** When nothing was claimed: the earliest live claim or retry hold that blocks a head. */
|
|
903
|
+
const RETRY_AT = `
|
|
904
|
+
select jsonb_build_object('retry_at', min(cur.claim_expires_at))
|
|
905
|
+
${ELIGIBLE}
|
|
906
|
+
and cur.claim_expires_at > ${param.bigint('now')}`
|
|
907
|
+
|
|
908
|
+
/** The cursor row `alias` whose head this attempt holds. */
|
|
909
|
+
const heldBy = (alias: string): string => `
|
|
910
|
+
${alias}.session_id = ${SESSION}
|
|
911
|
+
and ${alias}.cursor_key = ${param.text('cursorKey')}
|
|
912
|
+
and ${alias}.consumer = ${CONSUMER} and ${alias}.head = ${INDEX}
|
|
913
|
+
and ${alias}.attempt_count = ${param.int('attempt')}
|
|
914
|
+
and ${alias}.failed_at is null and ${alias}.claim_holder is not null`
|
|
915
|
+
|
|
916
|
+
/**
|
|
917
|
+
* Locks the cursor while this attempt holds its head, and stops unless it
|
|
918
|
+
* does: a completion locks its cursor first. A cleared head is never locked.
|
|
919
|
+
*/
|
|
920
|
+
const HOLD = `
|
|
921
|
+
with boundary as materialized (
|
|
922
|
+
select ${clearIndexOf(SESSION)} as clear_index
|
|
923
|
+
), held as materialized (
|
|
924
|
+
select cur.kind from a2_cursors cur, boundary
|
|
925
|
+
where ${heldBy('cur')} and ${INDEX} > boundary.clear_index
|
|
926
|
+
for update of cur
|
|
927
|
+
)
|
|
928
|
+
select case
|
|
929
|
+
when ${INDEX} <= boundary.clear_index then
|
|
930
|
+
jsonb_build_object('stop', 'superseded')
|
|
931
|
+
when not exists (select 1 from held) then
|
|
932
|
+
jsonb_build_object('stop', 'unheld')
|
|
933
|
+
else jsonb_build_object('held', (select kind from held limit 1))
|
|
934
|
+
end
|
|
935
|
+
from boundary`
|
|
936
|
+
|
|
937
|
+
/** A log cursor's types once it moves: the served consumer's (`types`, when `retype`), else its own. */
|
|
938
|
+
const logTypesOf = (cur: string): string =>
|
|
939
|
+
`(case when ${param.boolean('retype')} then ${param.jsonb('types')}
|
|
940
|
+
else ${cur}.types end)`
|
|
941
|
+
|
|
942
|
+
/**
|
|
943
|
+
* The head cursor `cur` moves to from its head: its lane's next unfinished
|
|
944
|
+
* event, the next event of a log cursor's types, or none for a concurrent
|
|
945
|
+
* cursor. The lane's lookup is ordered by its whole key, so the lane's
|
|
946
|
+
* primary key serves it.
|
|
947
|
+
*/
|
|
948
|
+
const nextHeadOf = (cur: string): string => `
|
|
949
|
+
case ${cur}.kind
|
|
950
|
+
when 'lane' then (
|
|
951
|
+
select lane.idx from a2_lane_events lane
|
|
952
|
+
where lane.session_id = ${SESSION} and lane.consumer = ${cur}.consumer
|
|
953
|
+
and lane.lane = ${cur}.lane and not lane.done
|
|
954
|
+
and lane.idx > ${cur}.head
|
|
955
|
+
order by lane.consumer, lane.lane, lane.idx limit 1)
|
|
956
|
+
when 'log' then (${nextLogHead({
|
|
957
|
+
types: logTypesOf(cur),
|
|
958
|
+
after: `${cur}.head`,
|
|
959
|
+
})})
|
|
960
|
+
end`
|
|
961
|
+
|
|
962
|
+
/**
|
|
963
|
+
* Takes the session lock when the held cursor would go idle: a lane or log
|
|
964
|
+
* cursor with no next head. See `completeAttempt` for why a completion that
|
|
965
|
+
* moves to a next head needs no lock.
|
|
966
|
+
*/
|
|
967
|
+
const LOCK_IF_IDLE = `
|
|
968
|
+
select jsonb_build_object('locked',
|
|
969
|
+
pg_advisory_xact_lock(hashtext(${SESSION}))::text is not null)
|
|
970
|
+
from a2_cursors cur
|
|
971
|
+
where ${heldBy('cur')} and cur.kind <> 'event'
|
|
972
|
+
and (${nextHeadOf('cur')}) is null`
|
|
973
|
+
|
|
974
|
+
/**
|
|
975
|
+
* Moves the cursor `target` selects past its head `index`, if its head lies
|
|
976
|
+
* above the clear marker: marks the lane's event done (with this attempt's
|
|
977
|
+
* receipt, `receipt`), releases the head's snapshot pins with the history no
|
|
978
|
+
* other pin keeps, and gives the cursor its next head with fresh dispatch
|
|
979
|
+
* columns (a log or concurrent cursor keeps the receipt on the run). The
|
|
980
|
+
* marker and the next head are read in one snapshot, after the cursor's lock.
|
|
981
|
+
* Without a row it writes nothing, and stops if `stop`.
|
|
982
|
+
*/
|
|
983
|
+
const advance = ({
|
|
984
|
+
target,
|
|
985
|
+
receipt,
|
|
986
|
+
missing,
|
|
987
|
+
}: {
|
|
988
|
+
target: string
|
|
989
|
+
receipt: boolean
|
|
990
|
+
missing: 'stop' | 'abort' | 'continue'
|
|
991
|
+
}): string => `
|
|
992
|
+
with target as materialized (
|
|
993
|
+
select cur.* from a2_cursors cur
|
|
994
|
+
where ${target} and ${INDEX} > ${clearIndexOf(SESSION)}
|
|
995
|
+
order by cur.cursor_key limit 1
|
|
996
|
+
for update of cur
|
|
997
|
+
), done as (
|
|
998
|
+
update a2_lane_events lane set done = true${
|
|
999
|
+
receipt
|
|
1000
|
+
? `, completed_attempt = ${param.int('attempt')},
|
|
1001
|
+
returned_event_ids = ${param.jsonb('ids')}`
|
|
1002
|
+
: ''
|
|
1003
|
+
}
|
|
1004
|
+
from target
|
|
1005
|
+
where target.kind = 'lane' and lane.session_id = ${SESSION}
|
|
1006
|
+
and lane.consumer = target.consumer and lane.lane = target.lane
|
|
1007
|
+
and lane.idx = target.head
|
|
1008
|
+
returning 1
|
|
1009
|
+
), unpinned as (
|
|
1010
|
+
delete from a2_snapshot_pins pin using target
|
|
1011
|
+
where pin.session_id = ${SESSION} and pin.event_index = target.head
|
|
1012
|
+
and pin.consumer = target.consumer and pin.cursor_key = target.cursor_key
|
|
1013
|
+
returning 1
|
|
1014
|
+
), unkept as (
|
|
1015
|
+
delete from a2_snapshot_history history using target
|
|
1016
|
+
where history.session_id = ${SESSION}
|
|
1017
|
+
and not exists (
|
|
1018
|
+
select 1 from a2_snapshot_pins pin
|
|
1019
|
+
where pin.session_id = history.session_id
|
|
1020
|
+
and pin.reducer_name = history.reducer_name
|
|
1021
|
+
and pin.up_to_index = history.up_to_index
|
|
1022
|
+
and not (pin.event_index = target.head
|
|
1023
|
+
and pin.consumer = target.consumer
|
|
1024
|
+
and pin.cursor_key = target.cursor_key)
|
|
1025
|
+
)
|
|
1026
|
+
returning 1
|
|
1027
|
+
), moved as (
|
|
1028
|
+
update a2_cursors cur
|
|
1029
|
+
set position = target.head, head = ${nextHeadOf('target')},
|
|
1030
|
+
${FRESH_DISPATCH},
|
|
1031
|
+
types = case when target.kind = 'log'
|
|
1032
|
+
then ${logTypesOf('target')} else cur.types end${
|
|
1033
|
+
receipt
|
|
1034
|
+
? `,
|
|
1035
|
+
completed_idx = case when target.kind = 'lane'
|
|
1036
|
+
then cur.completed_idx else target.head end,
|
|
1037
|
+
completed_attempt = case when target.kind = 'lane'
|
|
1038
|
+
then cur.completed_attempt
|
|
1039
|
+
else ${param.int('attempt')} end,
|
|
1040
|
+
returned_event_ids = case when target.kind = 'lane'
|
|
1041
|
+
then cur.returned_event_ids
|
|
1042
|
+
else ${param.jsonb('ids')} end`
|
|
1043
|
+
: ''
|
|
1044
|
+
}
|
|
1045
|
+
from target
|
|
1046
|
+
where cur.session_id = ${SESSION} and cur.cursor_key = target.cursor_key
|
|
1047
|
+
and cur.consumer = target.consumer and cur.epoch = target.epoch
|
|
1048
|
+
returning cur.head
|
|
1049
|
+
)
|
|
1050
|
+
select case when exists (select 1 from moved)
|
|
1051
|
+
then jsonb_build_object('advanced', true)
|
|
1052
|
+
else jsonb_build_object(${
|
|
1053
|
+
missing === 'continue'
|
|
1054
|
+
? `'advanced', false`
|
|
1055
|
+
: `'${missing}', 'superseded'`
|
|
1056
|
+
}) end`
|
|
1057
|
+
|
|
1058
|
+
/** Completing a held head, once the cursor's lock is taken. */
|
|
1059
|
+
const COMPLETE = advance({
|
|
1060
|
+
target: heldBy('cur'),
|
|
1061
|
+
receipt: true,
|
|
1062
|
+
missing: 'stop',
|
|
1063
|
+
})
|
|
1064
|
+
|
|
1065
|
+
/**
|
|
1066
|
+
* Completing a held head after appending its events, under the session lock:
|
|
1067
|
+
* the head can't have moved, but if it had, the events must not stand.
|
|
1068
|
+
*/
|
|
1069
|
+
const COMPLETE_APPENDED = advance({
|
|
1070
|
+
target: heldBy('cur'),
|
|
1071
|
+
receipt: true,
|
|
1072
|
+
missing: 'abort',
|
|
1073
|
+
})
|
|
1074
|
+
|
|
1075
|
+
/**
|
|
1076
|
+
* A completion's checks before it appends, under the session lock, which the
|
|
1077
|
+
* cursor's lock preceded: a head the clear marker passed meanwhile stops, and
|
|
1078
|
+
* a returned id a live row owns aborts.
|
|
1079
|
+
*/
|
|
1080
|
+
const RETURN_CHECKS = `
|
|
1081
|
+
select case
|
|
1082
|
+
when ${INDEX} <= ${clearIndexOf(SESSION)} then
|
|
1083
|
+
jsonb_build_object('stop', 'superseded')
|
|
1084
|
+
when exists (
|
|
1085
|
+
select 1 from a2_events event
|
|
1086
|
+
where event.event_id = any(${param.texts('ids')})
|
|
1087
|
+
and not (event.session_id = ${SESSION}
|
|
1088
|
+
and event.idx < ${clearIndexOf(SESSION)})
|
|
1089
|
+
) then jsonb_build_object('abort', 'owned')
|
|
1090
|
+
else jsonb_build_object('base', (
|
|
1091
|
+
select coalesce(max(idx), 0) from a2_events
|
|
1092
|
+
where session_id = ${SESSION}))
|
|
1093
|
+
end`
|
|
1094
|
+
|
|
1095
|
+
/** The consumer's dead-lettered head at `index`, locked. */
|
|
1096
|
+
const deadAt = (alias: string): string => `
|
|
1097
|
+
${alias}.session_id = ${SESSION} and ${alias}.consumer = ${CONSUMER}
|
|
1098
|
+
and ${alias}.head = ${INDEX} and ${alias}.failed_at is not null`
|
|
1099
|
+
|
|
1100
|
+
/**
|
|
1101
|
+
* Locks the dead letter, or stops: the statements after it act only on the
|
|
1102
|
+
* row this one locked.
|
|
1103
|
+
*/
|
|
1104
|
+
const LOCK_DEAD = `
|
|
1105
|
+
with dead as materialized (
|
|
1106
|
+
select cur.cursor_key from a2_cursors cur
|
|
1107
|
+
where ${deadAt('cur')} and ${INDEX} > ${clearIndexOf(SESSION)}
|
|
1108
|
+
order by cur.cursor_key limit 1
|
|
1109
|
+
for update
|
|
1110
|
+
)
|
|
1111
|
+
select case when exists (select 1 from dead)
|
|
1112
|
+
then jsonb_build_object('redriven', true)
|
|
1113
|
+
else jsonb_build_object('stop', 'alive', 'redriven', false,
|
|
1114
|
+
'has_pending',
|
|
1115
|
+
${pendingIn(SESSION, param.texts('served'))}) end`
|
|
1116
|
+
|
|
1117
|
+
/** Clears a dead letter, keeping its claim history and last failure. */
|
|
1118
|
+
const RETRY_DEAD = `
|
|
1119
|
+
with target as materialized (
|
|
1120
|
+
select cur.cursor_key, cur.epoch from a2_cursors cur
|
|
1121
|
+
where ${deadAt('cur')} and ${INDEX} > ${clearIndexOf(SESSION)}
|
|
1122
|
+
order by cur.cursor_key limit 1
|
|
1123
|
+
for update of cur
|
|
1124
|
+
), retried as (
|
|
1125
|
+
update a2_cursors cur set failed_at = null, failure_count = 0
|
|
1126
|
+
from target
|
|
1127
|
+
where cur.session_id = ${SESSION} and cur.cursor_key = target.cursor_key
|
|
1128
|
+
and cur.consumer = ${CONSUMER} and cur.epoch = target.epoch
|
|
1129
|
+
returning 1
|
|
1130
|
+
)
|
|
1131
|
+
select jsonb_build_object('retried', count(*)) from retried`
|
|
1132
|
+
|
|
1133
|
+
const SKIP_DEAD = advance({
|
|
1134
|
+
target: deadAt('cur'),
|
|
1135
|
+
receipt: false,
|
|
1136
|
+
missing: 'continue',
|
|
1137
|
+
})
|
|
1138
|
+
|
|
1139
|
+
const PENDING = `
|
|
1140
|
+
select jsonb_build_object('has_pending',
|
|
1141
|
+
${pendingIn(SESSION, param.texts('served'))})`
|
|
1142
|
+
|
|
1143
|
+
/**
|
|
1144
|
+
* One log cursor as `wakeLog` finds it: the session's head and clear marker,
|
|
1145
|
+
* the cursor's latest run, where a run would start (`start`), and the next
|
|
1146
|
+
* event it takes from there unless the run has a head. `retyped` says whether
|
|
1147
|
+
* the run's types are stale. CTEs, for a `with`.
|
|
1148
|
+
*/
|
|
1149
|
+
const LOG_CURSOR = `
|
|
1150
|
+
boundary as materialized (
|
|
1151
|
+
select coalesce(max(idx), 0) as session_head,
|
|
1152
|
+
${clearIndexOf(SESSION)} as clear_index
|
|
1153
|
+
from a2_events where session_id = ${SESSION}
|
|
1154
|
+
), latest as materialized (
|
|
1155
|
+
select boundary.*, cur.*,
|
|
1156
|
+
case when cur.epoch is not null
|
|
1157
|
+
then greatest(cur.position, boundary.clear_index)
|
|
1158
|
+
when ${param.boolean('backfill')} then boundary.clear_index
|
|
1159
|
+
else boundary.session_head end as start
|
|
1160
|
+
from boundary
|
|
1161
|
+
left join lateral (
|
|
1162
|
+
select * from a2_cursors cur
|
|
1163
|
+
where cur.session_id = ${SESSION}
|
|
1164
|
+
and cur.cursor_key = ${param.text('cursorKey')}
|
|
1165
|
+
and cur.consumer = ${CONSUMER}
|
|
1166
|
+
order by cur.epoch desc limit 1
|
|
1167
|
+
) cur on true
|
|
1168
|
+
), found as materialized (
|
|
1169
|
+
select latest.*,
|
|
1170
|
+
latest.types is distinct from ${param.jsonb('types')} as retyped,
|
|
1171
|
+
case when latest.head > latest.clear_index then null
|
|
1172
|
+
else ${nextLogHead({
|
|
1173
|
+
types: param.jsonb('types'),
|
|
1174
|
+
after: 'latest.start',
|
|
1175
|
+
})} end as next_head
|
|
1176
|
+
from latest
|
|
1177
|
+
)`
|
|
1178
|
+
|
|
1179
|
+
/**
|
|
1180
|
+
* A run with a head gets no next one: it is left as it is, claim and all,
|
|
1181
|
+
* and so is an idle run with nothing new to take. Only their types change.
|
|
1182
|
+
*/
|
|
1183
|
+
const KEEPS_RUN = 'found.epoch is not null and found.next_head is null'
|
|
1184
|
+
|
|
1185
|
+
/** No run yet, and nothing after the latest clear: none is started. */
|
|
1186
|
+
const NO_RUN_NEEDED =
|
|
1187
|
+
'found.epoch is null and found.session_head = found.clear_index'
|
|
1188
|
+
|
|
1189
|
+
/** `wakeLog` without the lock: stops when there is nothing to write. */
|
|
1190
|
+
const READ_LOG = `
|
|
1191
|
+
with ${LOG_CURSOR}
|
|
1192
|
+
select case when ${KEEPS_RUN} and not found.retyped
|
|
1193
|
+
then jsonb_build_object('stop', 'unchanged',
|
|
1194
|
+
'cursor', to_jsonb(found))
|
|
1195
|
+
when ${NO_RUN_NEEDED}
|
|
1196
|
+
then jsonb_build_object('stop', 'empty', 'empty', true,
|
|
1197
|
+
'head', found.session_head)
|
|
1198
|
+
else jsonb_build_object('stale', true) end
|
|
1199
|
+
from found`
|
|
1200
|
+
|
|
1201
|
+
/**
|
|
1202
|
+
* `wakeLog` under the session lock. A kept run takes the consumer's types,
|
|
1203
|
+
* unless another operation holds its row: the session lock is held, so this
|
|
1204
|
+
* waits for none. Otherwise a run starts at `start`, retiring the runs it
|
|
1205
|
+
* replaces. One with a head is keyed by it; one with nothing to run is idle
|
|
1206
|
+
* at its position, as a run that finished its last head is, keyed by that
|
|
1207
|
+
* position so the next run, which starts after it, sorts later.
|
|
1208
|
+
*/
|
|
1209
|
+
const WAKE_LOG = `
|
|
1210
|
+
with ${LOG_CURSOR}, held as materialized (
|
|
1211
|
+
select cur.ctid from a2_cursors cur, found
|
|
1212
|
+
where ${KEEPS_RUN} and found.retyped
|
|
1213
|
+
and cur.session_id = ${SESSION}
|
|
1214
|
+
and cur.cursor_key = ${param.text('cursorKey')}
|
|
1215
|
+
and cur.consumer = ${CONSUMER} and cur.epoch = found.epoch
|
|
1216
|
+
and cur.types is distinct from ${param.jsonb('types')}
|
|
1217
|
+
for update of cur skip locked
|
|
1218
|
+
), retype as (
|
|
1219
|
+
update a2_cursors cur set types = ${param.jsonb('types')} from held
|
|
1220
|
+
where cur.ctid = held.ctid
|
|
1221
|
+
), target as (
|
|
1222
|
+
select ${param.text('cursorKey')} as cursor_key, ${CONSUMER} as consumer
|
|
1223
|
+
from found where not (${KEEPS_RUN}) and not (${NO_RUN_NEEDED})
|
|
1224
|
+
), ${RETIRE_RUNS('target', '(select clear_index from found)')}, started as (
|
|
1225
|
+
insert into a2_cursors
|
|
1226
|
+
(session_id, consumer, cursor_key, epoch, lane, kind, position, head,
|
|
1227
|
+
types)
|
|
1228
|
+
select ${SESSION}, target.consumer, target.cursor_key,
|
|
1229
|
+
coalesce(found.next_head, found.start), ${param.text('lane')}, 'log',
|
|
1230
|
+
found.start, found.next_head, ${param.jsonb('types')}
|
|
1231
|
+
from found, target
|
|
1232
|
+
returning *
|
|
1233
|
+
)
|
|
1234
|
+
select jsonb_build_object(
|
|
1235
|
+
'cursor', coalesce((select to_jsonb(started) from started),
|
|
1236
|
+
to_jsonb(found)),
|
|
1237
|
+
'head', found.session_head, 'clear_index', found.clear_index,
|
|
1238
|
+
'empty', ${NO_RUN_NEEDED})
|
|
1239
|
+
from found`
|
|
1240
|
+
|
|
1241
|
+
/**
|
|
1242
|
+
* The statements that start the cursor work a batch of `events` carries,
|
|
1243
|
+
* with their parameters: each event by its position in the batch, from 1,
|
|
1244
|
+
* since its index is assigned under the lock. `started` says whether it
|
|
1245
|
+
* starts work for a served consumer.
|
|
1246
|
+
*/
|
|
1247
|
+
const enqueue = (
|
|
1248
|
+
events: readonly AppendEvent[],
|
|
1249
|
+
consumers: readonly StoreConsumer[] | undefined,
|
|
1250
|
+
): {
|
|
1251
|
+
statements: (string | false)[]
|
|
1252
|
+
params: Record<string, unknown>
|
|
1253
|
+
started: boolean
|
|
1254
|
+
} => {
|
|
1255
|
+
const served = consumers && new Set(consumers.map(({ name }) => name))
|
|
1256
|
+
let started = false
|
|
1257
|
+
const singles: { consumer: string; ord: number }[] = []
|
|
1258
|
+
const queued: { consumer: string; lane: string; ord: number }[] = []
|
|
1259
|
+
const lanes = new Map<
|
|
1260
|
+
string,
|
|
1261
|
+
{ consumer: string; cursor_key: string; lane: string; ord: number }
|
|
1262
|
+
>()
|
|
1263
|
+
events.forEach((event, offset) => {
|
|
1264
|
+
const ord = offset + 1
|
|
1265
|
+
for (const target of event.targets ?? []) {
|
|
1266
|
+
if (!served || served.has(target.consumer)) started = true
|
|
1267
|
+
if (target.lane === null) {
|
|
1268
|
+
singles.push({ consumer: target.consumer, ord })
|
|
1269
|
+
continue
|
|
1270
|
+
}
|
|
1271
|
+
queued.push({ consumer: target.consumer, lane: target.lane, ord })
|
|
1272
|
+
const key = cursorKeyOf(target.lane, ord)
|
|
1273
|
+
const first = `${target.consumer}\u0000${key}`
|
|
1274
|
+
if (!lanes.has(first)) {
|
|
1275
|
+
lanes.set(first, {
|
|
1276
|
+
consumer: target.consumer,
|
|
1277
|
+
cursor_key: key,
|
|
1278
|
+
lane: target.lane,
|
|
1279
|
+
ord,
|
|
1280
|
+
})
|
|
1281
|
+
}
|
|
1282
|
+
}
|
|
1283
|
+
})
|
|
1284
|
+
const logs: {
|
|
1285
|
+
consumer: string
|
|
1286
|
+
cursor_key: string
|
|
1287
|
+
lane: string
|
|
1288
|
+
ord: number
|
|
1289
|
+
types: readonly string[] | null
|
|
1290
|
+
backfill: boolean
|
|
1291
|
+
}[] = []
|
|
1292
|
+
for (const consumer of consumers ?? []) {
|
|
1293
|
+
const log = consumer.log
|
|
1294
|
+
if (!log) continue
|
|
1295
|
+
const offset = events.findIndex(
|
|
1296
|
+
(event) =>
|
|
1297
|
+
event.type !== CLEAR_EVENT_TYPE &&
|
|
1298
|
+
(log.types === null || log.types.includes(event.type)),
|
|
1299
|
+
)
|
|
1300
|
+
if (offset === -1) continue
|
|
1301
|
+
logs.push({
|
|
1302
|
+
consumer: consumer.name,
|
|
1303
|
+
cursor_key: cursorKeyOf(log.lane, 0),
|
|
1304
|
+
lane: log.lane,
|
|
1305
|
+
ord: offset + 1,
|
|
1306
|
+
types: log.types,
|
|
1307
|
+
backfill: log.backfill,
|
|
1308
|
+
})
|
|
1309
|
+
}
|
|
1310
|
+
if (logs.length > 0) started = true
|
|
1311
|
+
return {
|
|
1312
|
+
statements: [
|
|
1313
|
+
singles.length > 0 && START_EVENTS,
|
|
1314
|
+
queued.length > 0 && QUEUE_LANES,
|
|
1315
|
+
queued.length > 0 && START_LANES,
|
|
1316
|
+
logs.length > 0 && WAKE_LOGS,
|
|
1317
|
+
],
|
|
1318
|
+
params: {
|
|
1319
|
+
singles,
|
|
1320
|
+
queued,
|
|
1321
|
+
lanes: [...lanes.values()],
|
|
1322
|
+
logs,
|
|
1323
|
+
clearType: CLEAR_EVENT_TYPE,
|
|
1324
|
+
},
|
|
1325
|
+
started,
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
|
|
1329
|
+
/**
|
|
1330
|
+
* The rows a batch inserted at `base + 1`, as the store returns them.
|
|
1331
|
+
* `events` are what the batch sent.
|
|
1332
|
+
*/
|
|
1333
|
+
const insertedRows = ({
|
|
1334
|
+
sessionId,
|
|
1335
|
+
events,
|
|
1336
|
+
base,
|
|
1337
|
+
now,
|
|
1338
|
+
}: {
|
|
1339
|
+
sessionId: string
|
|
1340
|
+
events: readonly {
|
|
1341
|
+
id: string
|
|
1342
|
+
type: string
|
|
1343
|
+
payload: unknown
|
|
1344
|
+
cause: EventCause | null
|
|
1345
|
+
}[]
|
|
1346
|
+
base: number
|
|
1347
|
+
now: number
|
|
1348
|
+
}): StoredEvent[] =>
|
|
1349
|
+
events.map((event, offset) => ({
|
|
1350
|
+
id: event.id,
|
|
1351
|
+
type: event.type,
|
|
1352
|
+
payload: structuredClone(event.payload),
|
|
1353
|
+
index: base + 1 + offset,
|
|
1354
|
+
sessionId,
|
|
1355
|
+
createdAt: new Date(now),
|
|
1356
|
+
cause: event.cause ? { ...event.cause } : null,
|
|
1357
|
+
}))
|
|
1358
|
+
|
|
1359
|
+
/** A served log consumer's types, for a cursor that moves past its head. */
|
|
1360
|
+
const retypeOf = (
|
|
1361
|
+
consumer: string,
|
|
1362
|
+
consumers: readonly StoreConsumer[] | undefined,
|
|
1363
|
+
): { retype: boolean; types: readonly string[] | null } => {
|
|
1364
|
+
const log = consumers?.find(({ name }) => name === consumer)?.log
|
|
1365
|
+
return log
|
|
1366
|
+
? { retype: true, types: log.types }
|
|
1367
|
+
: { retype: false, types: null }
|
|
1368
|
+
}
|
|
1369
|
+
|
|
381
1370
|
export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
382
1371
|
const clock = options.clock ?? SYSTEM_CLOCK
|
|
383
1372
|
const generateId = options.ids ?? RANDOM_IDS
|
|
@@ -417,31 +1406,23 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
417
1406
|
},
|
|
418
1407
|
})
|
|
419
1408
|
}
|
|
420
|
-
|
|
421
|
-
const listen = Boolean(options.listenConnectionString)
|
|
422
|
-
const resource = lease.value
|
|
423
|
-
if (injected) await initializeSchema({ client: resource.client, listen })
|
|
424
|
-
else
|
|
425
|
-
await resource.initialize({
|
|
426
|
-
schema: SCHEMA + (listen ? POSTGRES_NOTIFICATIONS_SQL : ''),
|
|
427
|
-
run: () => initializeSchema({ client: resource.client, listen }),
|
|
428
|
-
})
|
|
429
|
-
return lease
|
|
430
|
-
} catch (error) {
|
|
431
|
-
if (!injected) await lease.release().catch(() => {})
|
|
432
|
-
throw error
|
|
433
|
-
}
|
|
1409
|
+
return { lease, client: explainingPostgresClient(lease.value.client) }
|
|
434
1410
|
})
|
|
435
1411
|
const client = async (): Promise<PostgresClient> => {
|
|
436
1412
|
if (closed) throw new A2Error('STORE_UNAVAILABLE', 'the store was closed')
|
|
437
|
-
const
|
|
1413
|
+
const { client: explaining } = await connection.get()
|
|
438
1414
|
if (closed) throw new A2Error('STORE_UNAVAILABLE', 'the store was closed')
|
|
439
|
-
return
|
|
1415
|
+
return explaining
|
|
440
1416
|
}
|
|
441
1417
|
|
|
442
1418
|
const notifications = options.listenConnectionString
|
|
443
1419
|
? postgresNotificationScope(async () => {
|
|
444
|
-
await client()
|
|
1420
|
+
const c = await client()
|
|
1421
|
+
const resource = (await connection.get()).lease.value
|
|
1422
|
+
await resource.initialize({
|
|
1423
|
+
key: 'triggers',
|
|
1424
|
+
run: () => checkPostgresTriggers(c),
|
|
1425
|
+
})
|
|
445
1426
|
const pg = await loadPg()
|
|
446
1427
|
const connectionString = options.listenConnectionString!
|
|
447
1428
|
const key = postgresConnectionKey({
|
|
@@ -576,9 +1557,7 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
576
1557
|
const begin = notifications
|
|
577
1558
|
? `begin; set local a2.presence_source = '${instanceId}'`
|
|
578
1559
|
: 'begin'
|
|
579
|
-
const withTx = async <T>(
|
|
580
|
-
fn: (query: PostgresConnection['query']) => Promise<T>,
|
|
581
|
-
): Promise<T> => {
|
|
1560
|
+
const withTx = async <T>(fn: (query: Query) => Promise<T>): Promise<T> => {
|
|
582
1561
|
const c = await client()
|
|
583
1562
|
if (c.connect) {
|
|
584
1563
|
const conn = await c.connect()
|
|
@@ -594,12 +1573,16 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
594
1573
|
conn.release()
|
|
595
1574
|
}
|
|
596
1575
|
}
|
|
597
|
-
const resource = (await connection.get()).value
|
|
1576
|
+
const resource = (await connection.get()).lease.value
|
|
598
1577
|
return resource.exclusive(async () => {
|
|
599
1578
|
const raw = resource.raw
|
|
600
1579
|
await raw.query(begin)
|
|
601
1580
|
try {
|
|
602
|
-
const result = await fn((text, values) =>
|
|
1581
|
+
const result = await fn((text, values) =>
|
|
1582
|
+
raw.query(text, values).catch((error: unknown) => {
|
|
1583
|
+
throw explainPostgresError(error)
|
|
1584
|
+
}),
|
|
1585
|
+
)
|
|
603
1586
|
await raw.query('commit')
|
|
604
1587
|
return result
|
|
605
1588
|
} catch (err) {
|
|
@@ -609,54 +1592,37 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
609
1592
|
})
|
|
610
1593
|
}
|
|
611
1594
|
|
|
612
|
-
type Query = PostgresConnection['query']
|
|
613
|
-
|
|
614
1595
|
/**
|
|
615
|
-
*
|
|
616
|
-
*
|
|
1596
|
+
* Runs `statements` (the falsy ones dropped) as one `a2_batch` call with
|
|
1597
|
+
* `params`, and the session's id as `session`. Its rows come back in order;
|
|
1598
|
+
* an aborted call throws what `abort` makes of the aborting row.
|
|
617
1599
|
*/
|
|
618
|
-
const
|
|
619
|
-
query,
|
|
1600
|
+
const run = async ({
|
|
620
1601
|
sessionId,
|
|
621
|
-
|
|
622
|
-
|
|
1602
|
+
statements,
|
|
1603
|
+
params,
|
|
1604
|
+
abort = () =>
|
|
1605
|
+
new A2Error('STORE_UNAVAILABLE', 'postgres store operation failed'),
|
|
623
1606
|
}: {
|
|
624
|
-
query: Query
|
|
625
1607
|
sessionId: string
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
const
|
|
631
|
-
|
|
632
|
-
[
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
select idx from a2_events
|
|
644
|
-
where session_id = $1 and event_id = any($2::text[])
|
|
645
|
-
order by idx
|
|
646
|
-
for update skip locked
|
|
647
|
-
)
|
|
648
|
-
delete from a2_events event using stale
|
|
649
|
-
where event.session_id = $1 and event.idx = stale.idx
|
|
650
|
-
returning event.idx`,
|
|
651
|
-
[sessionId, stale.map((row) => row['event_id'])],
|
|
652
|
-
)
|
|
653
|
-
if (deleted.length < stale.length) {
|
|
654
|
-
throw new A2Error(
|
|
655
|
-
'STORE_UNAVAILABLE',
|
|
656
|
-
`an event id being reused is still held by cleared work in session '${sessionId}'`,
|
|
657
|
-
)
|
|
1608
|
+
statements: readonly (string | false)[]
|
|
1609
|
+
params: Record<string, unknown>
|
|
1610
|
+
abort?: (reason: Row) => Error
|
|
1611
|
+
}): Promise<Row[]> => {
|
|
1612
|
+
const c = await client()
|
|
1613
|
+
try {
|
|
1614
|
+
const { rows } = await c.query(BATCH, [
|
|
1615
|
+
statements.flatMap((statement) =>
|
|
1616
|
+
statement === false ? [] : [compact(statement)],
|
|
1617
|
+
),
|
|
1618
|
+
jsonb({ ...params, session: sessionId }),
|
|
1619
|
+
])
|
|
1620
|
+
return rows.map((row) => row['r'] as Row)
|
|
1621
|
+
} catch (error) {
|
|
1622
|
+
const reason = abortedBatch(error)
|
|
1623
|
+
if (reason) throw abort(reason)
|
|
1624
|
+
throw error
|
|
658
1625
|
}
|
|
659
|
-
return rows.filter((row) => !stale.includes(row))
|
|
660
1626
|
}
|
|
661
1627
|
|
|
662
1628
|
/** Whether `index` is at or below the boundary, read in a fresh statement. */
|
|
@@ -676,240 +1642,345 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
676
1642
|
return index <= asNumber(rows[0]?.['clear_index'])
|
|
677
1643
|
}
|
|
678
1644
|
|
|
679
|
-
/**
|
|
680
|
-
|
|
1645
|
+
/**
|
|
1646
|
+
* A completion that does not hold its head: a stale attempt, or a retry
|
|
1647
|
+
* after an unknown acknowledgement, which replays the committed children.
|
|
1648
|
+
* Takes no lock.
|
|
1649
|
+
*/
|
|
1650
|
+
const settleUnheld = async ({
|
|
681
1651
|
query,
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
1652
|
+
sessionId,
|
|
1653
|
+
consumer,
|
|
1654
|
+
lane,
|
|
1655
|
+
index,
|
|
1656
|
+
attempt,
|
|
1657
|
+
events,
|
|
685
1658
|
}: {
|
|
686
1659
|
query: Query
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
1660
|
+
sessionId: string
|
|
1661
|
+
consumer: string
|
|
1662
|
+
lane: string | null
|
|
1663
|
+
index: number
|
|
1664
|
+
attempt: number
|
|
1665
|
+
events: readonly ReturnedEvent[]
|
|
1666
|
+
}): Promise<CompleteAttemptResult> => {
|
|
1667
|
+
const superseded: CompleteAttemptResult = { outcome: 'superseded' }
|
|
1668
|
+
// The cursor's latest run, and the run the head belonged to: the latest
|
|
1669
|
+
// one started at or before it. A lane keeps its completions per event;
|
|
1670
|
+
// a log cursor keeps its latest on the run.
|
|
1671
|
+
const { rows: cursorRows } = await query(
|
|
1672
|
+
`select latest.kind, latest.head, covering.completed_idx,
|
|
1673
|
+
covering.completed_attempt, covering.returned_event_ids
|
|
1674
|
+
from (
|
|
1675
|
+
select kind, head from a2_cursors
|
|
1676
|
+
where session_id = $1 and cursor_key = $2 and consumer = $3
|
|
1677
|
+
order by epoch desc limit 1
|
|
1678
|
+
) latest
|
|
1679
|
+
left join lateral (
|
|
1680
|
+
select completed_idx, completed_attempt, returned_event_ids
|
|
1681
|
+
from a2_cursors
|
|
1682
|
+
where session_id = $1 and cursor_key = $2 and consumer = $3
|
|
1683
|
+
and epoch <= $4
|
|
1684
|
+
order by epoch desc limit 1
|
|
1685
|
+
) covering on true`,
|
|
1686
|
+
[sessionId, cursorKeyOf(lane, index), consumer, index],
|
|
1687
|
+
)
|
|
1688
|
+
const cursor = cursorRows[0]
|
|
1689
|
+
if (!cursor) {
|
|
1690
|
+
if (await hasEvent(query, sessionId, index)) return superseded
|
|
1691
|
+
// Deleted while this statement ran, after a clear.
|
|
1692
|
+
if (await clearedAt({ query, sessionId, index })) return superseded
|
|
1693
|
+
throw new TypeError(
|
|
1694
|
+
`no event at index ${index} in session '${sessionId}'`,
|
|
1695
|
+
)
|
|
697
1696
|
}
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
1697
|
+
const ids = events.map((event) => event.id)
|
|
1698
|
+
if (new Set(ids).size !== ids.length) {
|
|
1699
|
+
throw new A2Error(
|
|
1700
|
+
'PARTIAL_DUPLICATE_BATCH',
|
|
1701
|
+
'returned event batch contains the same event id more than once',
|
|
1702
|
+
)
|
|
1703
|
+
}
|
|
1704
|
+
if (asNullableNumber(cursor['head']) === index) return superseded
|
|
1705
|
+
let completion: { attempt: number | null; ids: unknown } | undefined
|
|
1706
|
+
if (cursor['kind'] === 'lane') {
|
|
1707
|
+
const { rows } = await query(
|
|
1708
|
+
`select completed_attempt, returned_event_ids from a2_lane_events
|
|
1709
|
+
where session_id = $1 and consumer = $2 and lane = $3 and idx = $4`,
|
|
1710
|
+
[sessionId, consumer, lane, index],
|
|
1711
|
+
)
|
|
1712
|
+
completion = rows[0] && {
|
|
1713
|
+
attempt: asNullableNumber(rows[0]['completed_attempt']),
|
|
1714
|
+
ids: rows[0]['returned_event_ids'],
|
|
1715
|
+
}
|
|
1716
|
+
} else if (asNullableNumber(cursor['completed_idx']) === index) {
|
|
1717
|
+
completion = {
|
|
1718
|
+
attempt: asNullableNumber(cursor['completed_attempt']),
|
|
1719
|
+
ids: cursor['returned_event_ids'],
|
|
1720
|
+
}
|
|
1721
|
+
}
|
|
1722
|
+
if (!completion || completion.attempt !== attempt) return superseded
|
|
1723
|
+
const returnedIds =
|
|
1724
|
+
completion.ids === null || completion.ids === undefined
|
|
1725
|
+
? null
|
|
1726
|
+
: decodeReturnedEventIds(completion.ids)
|
|
1727
|
+
if (
|
|
1728
|
+
returnedIds === null ||
|
|
1729
|
+
returnedIds.length !== ids.length ||
|
|
1730
|
+
returnedIds.some((id, offset) => id !== ids[offset])
|
|
1731
|
+
) {
|
|
1732
|
+
throw new A2Error(
|
|
1733
|
+
'PARTIAL_DUPLICATE_BATCH',
|
|
1734
|
+
'completed attempt does not match the returned event batch',
|
|
1735
|
+
)
|
|
1736
|
+
}
|
|
1737
|
+
if (events.length === 0) return { outcome: 'completed', events: [] }
|
|
1738
|
+
const { rows: existing } = await query(
|
|
1739
|
+
`select session_id, idx, event_type, payload, event_id, created_at, cause
|
|
1740
|
+
from a2_events where event_id = any($1::text[])`,
|
|
1741
|
+
[ids],
|
|
1742
|
+
)
|
|
1743
|
+
const byId = new Map(
|
|
1744
|
+
existing.map((row) => [row['event_id'] as string, row]),
|
|
1745
|
+
)
|
|
1746
|
+
const ordered = ids.map((id) => byId.get(id))
|
|
1747
|
+
if (
|
|
1748
|
+
ordered.some((row) => !row) &&
|
|
1749
|
+
(await clearedAt({ query, sessionId, index }))
|
|
726
1750
|
)
|
|
727
|
-
|
|
1751
|
+
return superseded
|
|
1752
|
+
if (
|
|
1753
|
+
ordered.some((row) => {
|
|
1754
|
+
const cause = row ? decodeCause(row['cause']) : null
|
|
1755
|
+
return (
|
|
1756
|
+
!row ||
|
|
1757
|
+
row['session_id'] !== sessionId ||
|
|
1758
|
+
cause?.consumer !== consumer ||
|
|
1759
|
+
cause.lane !== lane ||
|
|
1760
|
+
cause.index !== index ||
|
|
1761
|
+
cause.attempt !== attempt
|
|
1762
|
+
)
|
|
1763
|
+
})
|
|
1764
|
+
) {
|
|
1765
|
+
throw new A2Error(
|
|
1766
|
+
'PARTIAL_DUPLICATE_BATCH',
|
|
1767
|
+
'returned event retry does not match the committed batch',
|
|
1768
|
+
)
|
|
1769
|
+
}
|
|
1770
|
+
return {
|
|
1771
|
+
outcome: 'completed',
|
|
1772
|
+
events: idempotentReplay(events, (ordered as Row[]).map(toStored)),
|
|
1773
|
+
}
|
|
728
1774
|
}
|
|
729
1775
|
|
|
730
1776
|
const store: PostgresStore = {
|
|
731
1777
|
async append(sessionId, events, appendOptions) {
|
|
1778
|
+
const consumers = appendOptions?.consumers
|
|
1779
|
+
const served = servedNames(consumers)
|
|
732
1780
|
if (events.length === 0) {
|
|
733
|
-
return wrap(async () => {
|
|
734
|
-
const c = await client()
|
|
735
|
-
const { rows } = await c.query(
|
|
736
|
-
`select
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
[
|
|
742
|
-
)
|
|
743
|
-
return {
|
|
744
|
-
events: [],
|
|
745
|
-
hasPending: rows[0]?.['has_pending'] === true,
|
|
746
|
-
}
|
|
747
|
-
})
|
|
748
|
-
}
|
|
749
|
-
return wrap(() =>
|
|
750
|
-
withTx(async (query) => {
|
|
751
|
-
// Serialize appends per session — index assignment must not
|
|
752
|
-
// race across writers. Transaction-scoped, self-releasing.
|
|
753
|
-
await query('select pg_advisory_xact_lock(hashtext($1))', [sessionId])
|
|
754
|
-
const { rows: statusRows } = await query(
|
|
755
|
-
`select coalesce(max(idx), 0) as max,
|
|
756
|
-
${clearIndexOf('$1')} as clear_index,
|
|
757
|
-
exists(
|
|
758
|
-
select 1 from a2_events pending
|
|
759
|
-
where pending.session_id = $1
|
|
760
|
-
and pending.processed_at is null
|
|
761
|
-
and pending.idx > ${clearIndexOf('$1')}
|
|
762
|
-
) as session_has_pending
|
|
763
|
-
from a2_events
|
|
764
|
-
where session_id = $1`,
|
|
765
|
-
[sessionId],
|
|
766
|
-
)
|
|
767
|
-
const base = asNumber(statusRows[0]?.['max'])
|
|
768
|
-
const clearIndex = asNumber(statusRows[0]?.['clear_index'])
|
|
769
|
-
const sessionHasPending =
|
|
770
|
-
statusRows[0]?.['session_has_pending'] === true
|
|
771
|
-
|
|
772
|
-
const supplied = events.filter((e) => e.id !== undefined)
|
|
773
|
-
const suppliedIds = supplied.map((e) => e.id as string)
|
|
774
|
-
if (new Set(suppliedIds).size !== suppliedIds.length) {
|
|
775
|
-
throw new A2Error(
|
|
776
|
-
'PARTIAL_DUPLICATE_BATCH',
|
|
777
|
-
'batch contains the same event id more than once',
|
|
778
|
-
)
|
|
779
|
-
}
|
|
780
|
-
if (suppliedIds.length > 0) {
|
|
781
|
-
const existing = await liveOwnersOf({
|
|
782
|
-
query,
|
|
783
|
-
sessionId,
|
|
784
|
-
clearIndex,
|
|
785
|
-
ids: suppliedIds,
|
|
786
|
-
})
|
|
787
|
-
if (existing.length > 0) {
|
|
788
|
-
const foreign = existing.find(
|
|
789
|
-
(row) => row['session_id'] !== sessionId,
|
|
790
|
-
)
|
|
791
|
-
if (foreign) {
|
|
792
|
-
throw new A2Error(
|
|
793
|
-
'PARTIAL_DUPLICATE_BATCH',
|
|
794
|
-
`event id '${String(foreign['event_id'])}' already exists in another session`,
|
|
795
|
-
)
|
|
796
|
-
}
|
|
797
|
-
if (existing.length === events.length) {
|
|
798
|
-
return {
|
|
799
|
-
events: idempotentReplay(events, existing.map(toStored)),
|
|
800
|
-
hasPending: sessionHasPending,
|
|
801
|
-
}
|
|
802
|
-
}
|
|
803
|
-
throw new A2Error(
|
|
804
|
-
'PARTIAL_DUPLICATE_BATCH',
|
|
805
|
-
`batch mixes ${existing.length} already-appended and ${events.length - existing.length} fresh events`,
|
|
806
|
-
)
|
|
807
|
-
}
|
|
808
|
-
}
|
|
809
|
-
|
|
810
|
-
const originIndex = appendOptions?.originIndex
|
|
811
|
-
if (originIndex !== undefined && originIndex <= clearIndex) {
|
|
812
|
-
throw sessionClearedError(
|
|
813
|
-
`scheduled work from event ${originIndex} was cleared in session '${sessionId}'`,
|
|
814
|
-
)
|
|
815
|
-
}
|
|
816
|
-
for (const e of events) {
|
|
817
|
-
if (e.cause && e.cause.index <= clearIndex)
|
|
818
|
-
throw supersededAttemptError(sessionId, e.cause)
|
|
1781
|
+
return wrap(async () => {
|
|
1782
|
+
const c = await client()
|
|
1783
|
+
const { rows } = await c.query(
|
|
1784
|
+
`select ${pendingIn('$1', '$2')} as has_pending`,
|
|
1785
|
+
[sessionId, served],
|
|
1786
|
+
)
|
|
1787
|
+
return {
|
|
1788
|
+
events: [],
|
|
1789
|
+
hasPending: rows[0]?.['has_pending'] === true,
|
|
819
1790
|
}
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
1791
|
+
})
|
|
1792
|
+
}
|
|
1793
|
+
return wrap(async () => {
|
|
1794
|
+
const supplied = events.filter((e) => e.id !== undefined)
|
|
1795
|
+
const suppliedIds = supplied.map((e) => e.id as string)
|
|
1796
|
+
if (new Set(suppliedIds).size !== suppliedIds.length) {
|
|
1797
|
+
throw new A2Error(
|
|
1798
|
+
'PARTIAL_DUPLICATE_BATCH',
|
|
1799
|
+
'batch contains the same event id more than once',
|
|
1800
|
+
)
|
|
1801
|
+
}
|
|
1802
|
+
const now = clock.now().getTime()
|
|
1803
|
+
const rows = events.map((event, offset) => ({
|
|
1804
|
+
ord: offset + 1,
|
|
1805
|
+
id: event.id ?? generateId(),
|
|
1806
|
+
type: event.type,
|
|
1807
|
+
payload: event.payload,
|
|
1808
|
+
cause: event.cause ?? null,
|
|
1809
|
+
}))
|
|
1810
|
+
const causes = [
|
|
1811
|
+
...new Map(
|
|
1812
|
+
events.flatMap((event) =>
|
|
1813
|
+
event.cause ? [[jsonb(event.cause), event.cause] as const] : [],
|
|
830
1814
|
),
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
1815
|
+
).values(),
|
|
1816
|
+
]
|
|
1817
|
+
const work = enqueue(events, consumers)
|
|
1818
|
+
const originIndex = appendOptions?.originIndex
|
|
1819
|
+
// One round trip, holding the session lock only while the server
|
|
1820
|
+
// runs it: index assignment must not race across writers.
|
|
1821
|
+
const results = await run({
|
|
1822
|
+
sessionId,
|
|
1823
|
+
statements: [
|
|
1824
|
+
LOCK_SESSION,
|
|
1825
|
+
APPEND_CHECKS,
|
|
1826
|
+
causes.length > 0 && FENCE,
|
|
1827
|
+
suppliedIds.length > 0 && REUSE_IDS,
|
|
1828
|
+
INSERT_EVENTS,
|
|
1829
|
+
...work.statements,
|
|
1830
|
+
],
|
|
1831
|
+
params: {
|
|
1832
|
+
...work.params,
|
|
1833
|
+
served,
|
|
1834
|
+
ids: suppliedIds,
|
|
1835
|
+
count: events.length,
|
|
1836
|
+
origin: originIndex ?? null,
|
|
1837
|
+
// Attempt-currency fence: a fresh consumer append commits only
|
|
1838
|
+
// while its causal attempt still owns the cursor's head
|
|
1839
|
+
// (store.ts `append`).
|
|
1840
|
+
causes: causes.map((cause, offset) => ({
|
|
1841
|
+
ord: offset + 1,
|
|
1842
|
+
consumer: cause.consumer,
|
|
1843
|
+
cursor_key: cursorKeyOf(cause.lane, cause.index),
|
|
1844
|
+
idx: cause.index,
|
|
1845
|
+
attempt: cause.attempt,
|
|
1846
|
+
})),
|
|
1847
|
+
events: rows,
|
|
1848
|
+
now,
|
|
1849
|
+
},
|
|
1850
|
+
abort: (reason) => {
|
|
1851
|
+
switch (reason['abort']) {
|
|
1852
|
+
case 'foreign':
|
|
1853
|
+
return new A2Error(
|
|
1854
|
+
'PARTIAL_DUPLICATE_BATCH',
|
|
1855
|
+
`event id '${String(reason['event_id'])}' already exists in another session`,
|
|
1856
|
+
)
|
|
1857
|
+
case 'mixed': {
|
|
1858
|
+
const owned = asNumber(reason['owned'])
|
|
1859
|
+
return new A2Error(
|
|
1860
|
+
'PARTIAL_DUPLICATE_BATCH',
|
|
1861
|
+
`batch mixes ${owned} already-appended and ${events.length - owned} fresh events`,
|
|
848
1862
|
)
|
|
849
1863
|
}
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
1864
|
+
case 'cleared':
|
|
1865
|
+
return sessionClearedError(
|
|
1866
|
+
`scheduled work from event ${originIndex} was cleared in session '${sessionId}'`,
|
|
1867
|
+
)
|
|
1868
|
+
case 'superseded':
|
|
1869
|
+
return supersededAttemptError(
|
|
1870
|
+
sessionId,
|
|
1871
|
+
causes[asNumber(reason['ord']) - 1]!,
|
|
1872
|
+
)
|
|
1873
|
+
case 'held':
|
|
1874
|
+
return new A2Error(
|
|
1875
|
+
'STORE_UNAVAILABLE',
|
|
1876
|
+
`an event id being reused is still held by cleared work in session '${sessionId}'`,
|
|
1877
|
+
)
|
|
1878
|
+
default:
|
|
1879
|
+
return new A2Error(
|
|
1880
|
+
'STORE_UNAVAILABLE',
|
|
1881
|
+
'postgres store operation failed',
|
|
1882
|
+
)
|
|
856
1883
|
}
|
|
1884
|
+
},
|
|
1885
|
+
})
|
|
1886
|
+
const checked = results.find(
|
|
1887
|
+
(row) => row['stop'] === 'replay' || 'base' in row,
|
|
1888
|
+
)!
|
|
1889
|
+
if (checked['stop'] === 'replay') {
|
|
1890
|
+
return {
|
|
1891
|
+
events: idempotentReplay(
|
|
1892
|
+
events,
|
|
1893
|
+
(checked['events'] as Row[]).map(toStored),
|
|
1894
|
+
),
|
|
1895
|
+
hasPending: checked['has_pending'] === true,
|
|
857
1896
|
}
|
|
1897
|
+
}
|
|
1898
|
+
return {
|
|
1899
|
+
events: insertedRows({
|
|
1900
|
+
sessionId,
|
|
1901
|
+
events: rows,
|
|
1902
|
+
base: asNumber(checked['base']),
|
|
1903
|
+
now,
|
|
1904
|
+
}),
|
|
1905
|
+
hasPending: checked['has_pending'] === true || work.started,
|
|
1906
|
+
}
|
|
1907
|
+
})
|
|
1908
|
+
},
|
|
858
1909
|
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
id,
|
|
883
|
-
type: e.type,
|
|
884
|
-
payload: structuredClone(e.payload),
|
|
885
|
-
index,
|
|
886
|
-
sessionId,
|
|
887
|
-
createdAt: new Date(now),
|
|
888
|
-
cause: e.cause ? { ...e.cause } : null,
|
|
889
|
-
lane: e.lane ?? null,
|
|
890
|
-
processedAt: e.settled ? new Date(now) : null,
|
|
891
|
-
processedByAttempt: null,
|
|
892
|
-
returnedEventIds: null,
|
|
893
|
-
firstClaimedAt: null,
|
|
894
|
-
lastClaimedAt: null,
|
|
895
|
-
attemptCount: 0,
|
|
896
|
-
failureCount: 0,
|
|
897
|
-
lastFailedAt: null,
|
|
898
|
-
lastFailedAttempt: null,
|
|
899
|
-
lastError: null,
|
|
900
|
-
failedAt: null,
|
|
901
|
-
claimHolder: null,
|
|
902
|
-
claimExpiresAt: null,
|
|
903
|
-
})
|
|
904
|
-
}
|
|
1910
|
+
async wakeLog({ sessionId, consumer }) {
|
|
1911
|
+
const { log } = consumer
|
|
1912
|
+
return wrap(async () => {
|
|
1913
|
+
// A read that finds nothing to write needs no lock: the cursor and
|
|
1914
|
+
// the head come from one snapshot. Otherwise the session lock comes
|
|
1915
|
+
// first, as an append takes it: the log and the cursor's runs hold
|
|
1916
|
+
// still, and no cursor row is waited on after.
|
|
1917
|
+
const results = await run({
|
|
1918
|
+
sessionId,
|
|
1919
|
+
statements: [READ_LOG, LOCK_SESSION, WAKE_LOG],
|
|
1920
|
+
params: {
|
|
1921
|
+
cursorKey: cursorKeyOf(log.lane, 0),
|
|
1922
|
+
consumer: consumer.name,
|
|
1923
|
+
lane: log.lane,
|
|
1924
|
+
types: log.types,
|
|
1925
|
+
backfill: log.backfill,
|
|
1926
|
+
clearType: CLEAR_EVENT_TYPE,
|
|
1927
|
+
},
|
|
1928
|
+
})
|
|
1929
|
+
const found = results.at(-1)!
|
|
1930
|
+
// Nothing after the latest clear: a missing cursor stays missing.
|
|
1931
|
+
if (found['empty'] === true) {
|
|
1932
|
+
const head = asNumber(found['head'])
|
|
905
1933
|
return {
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
sessionHasPending ||
|
|
909
|
-
events.some((event) => event.settled !== true),
|
|
1934
|
+
cursor: idleLogCursor(consumer.name, log.lane, head),
|
|
1935
|
+
head,
|
|
910
1936
|
}
|
|
911
|
-
}
|
|
912
|
-
|
|
1937
|
+
}
|
|
1938
|
+
const cursor = found['cursor'] as Row
|
|
1939
|
+
const clearIndex = asNumber(
|
|
1940
|
+
found['stop'] === 'unchanged'
|
|
1941
|
+
? cursor['clear_index']
|
|
1942
|
+
: found['clear_index'],
|
|
1943
|
+
)
|
|
1944
|
+
return {
|
|
1945
|
+
cursor: toCursor(cursor, clearIndex),
|
|
1946
|
+
head: asNumber(
|
|
1947
|
+
found['stop'] === 'unchanged'
|
|
1948
|
+
? cursor['session_head']
|
|
1949
|
+
: found['head'],
|
|
1950
|
+
),
|
|
1951
|
+
}
|
|
1952
|
+
})
|
|
1953
|
+
},
|
|
1954
|
+
|
|
1955
|
+
async cursors(sessionId) {
|
|
1956
|
+
return wrap(async () => {
|
|
1957
|
+
const c = await client()
|
|
1958
|
+
const { rows } = await c.query(
|
|
1959
|
+
`with boundary as materialized (
|
|
1960
|
+
select ${clearIndexOf('$1')} as clear_index
|
|
1961
|
+
)
|
|
1962
|
+
select cur.*, boundary.clear_index
|
|
1963
|
+
from a2_cursors cur, boundary
|
|
1964
|
+
where cur.session_id = $1
|
|
1965
|
+
and (
|
|
1966
|
+
cur.head > boundary.clear_index
|
|
1967
|
+
or (cur.kind <> 'event' and not exists (
|
|
1968
|
+
select 1 from a2_cursors later
|
|
1969
|
+
where later.session_id = cur.session_id
|
|
1970
|
+
and later.cursor_key = cur.cursor_key
|
|
1971
|
+
and later.consumer = cur.consumer
|
|
1972
|
+
and later.epoch > cur.epoch
|
|
1973
|
+
) and (
|
|
1974
|
+
cur.kind = 'log'
|
|
1975
|
+
or (cur.head is null and cur.position > boundary.clear_index)
|
|
1976
|
+
))
|
|
1977
|
+
)`,
|
|
1978
|
+
[sessionId],
|
|
1979
|
+
)
|
|
1980
|
+
return rows
|
|
1981
|
+
.map((row) => toCursor(row, asNumber(row['clear_index'])))
|
|
1982
|
+
.toSorted(byCursor)
|
|
1983
|
+
})
|
|
913
1984
|
},
|
|
914
1985
|
|
|
915
1986
|
async read(sessionId, opts) {
|
|
@@ -928,7 +1999,8 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
928
1999
|
// Reads start at the latest clear marker.
|
|
929
2000
|
conditions.push(`idx >= ${clearIndexOf('$1')}`)
|
|
930
2001
|
const { rows } = await c.query(
|
|
931
|
-
`select
|
|
2002
|
+
`select session_id, idx, event_type, payload, event_id, created_at, cause
|
|
2003
|
+
from a2_events where ${conditions.join(' and ')} order by idx`,
|
|
932
2004
|
params,
|
|
933
2005
|
)
|
|
934
2006
|
return rows.map(toStored)
|
|
@@ -969,91 +2041,71 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
969
2041
|
holder,
|
|
970
2042
|
ttlMs,
|
|
971
2043
|
expiresAtMs,
|
|
972
|
-
|
|
2044
|
+
exclude = [],
|
|
2045
|
+
consumers,
|
|
973
2046
|
maxFailures,
|
|
2047
|
+
through,
|
|
974
2048
|
}) {
|
|
975
|
-
return wrap(() =>
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
|
|
1003
|
-
|
|
1004
|
-
last_claimed_at = case when ${exhausted}
|
|
1005
|
-
then claimed.last_claimed_at else $2::bigint end,
|
|
1006
|
-
claim_holder = case when ${exhausted} then null else $3::text end,
|
|
1007
|
-
claim_expires_at = case when ${exhausted} then null else $4::bigint end
|
|
1008
|
-
from eligible
|
|
1009
|
-
where claimed.session_id = $1 and claimed.idx = eligible.idx
|
|
1010
|
-
returning claimed.*, ${clearIndexOf('$1')} as clear_index`,
|
|
1011
|
-
[
|
|
1012
|
-
sessionId,
|
|
1013
|
-
now,
|
|
1014
|
-
holder,
|
|
1015
|
-
expiresAt,
|
|
1016
|
-
[...excludeIndexes],
|
|
1017
|
-
maxFailures ?? null,
|
|
1018
|
-
],
|
|
2049
|
+
return wrap(async () => {
|
|
2050
|
+
const now = clock.now().getTime()
|
|
2051
|
+
const results = await run({
|
|
2052
|
+
sessionId,
|
|
2053
|
+
statements: [CLAIM, RELEASE_DEAD, RETRY_AT],
|
|
2054
|
+
params: {
|
|
2055
|
+
now,
|
|
2056
|
+
served: servedNames(consumers),
|
|
2057
|
+
excluded: exclude.map((ref) => ({
|
|
2058
|
+
consumer: ref.consumer,
|
|
2059
|
+
cursor_key: cursorKeyOf(ref.lane, ref.index),
|
|
2060
|
+
})),
|
|
2061
|
+
holder,
|
|
2062
|
+
expiresAt: expiresAtMs ?? now + ttlMs,
|
|
2063
|
+
maxFailures: maxFailures ?? null,
|
|
2064
|
+
through: through ?? null,
|
|
2065
|
+
},
|
|
2066
|
+
})
|
|
2067
|
+
const rows = results.filter((row) => 'e_idx' in row)
|
|
2068
|
+
const unseen = rows.filter((row) => row['e_idx'] === null)
|
|
2069
|
+
if (unseen.length > 0) {
|
|
2070
|
+
// The claim waited on these rows while they moved to events that
|
|
2071
|
+
// committed after its snapshot. A fresh read finds them; one it
|
|
2072
|
+
// doesn't was cleared and reclaimed, and its claim is superseded.
|
|
2073
|
+
const c = await client()
|
|
2074
|
+
const { rows: events } = await c.query(
|
|
2075
|
+
`select ${HEAD_COLUMNS} from a2_events event
|
|
2076
|
+
where event.session_id = $1 and event.idx = any($2::bigint[])`,
|
|
2077
|
+
[sessionId, unseen.map((row) => row['head'])],
|
|
1019
2078
|
)
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
await releasePins({
|
|
1023
|
-
query,
|
|
1024
|
-
sessionId,
|
|
1025
|
-
index: asNumber(row['idx']),
|
|
1026
|
-
})
|
|
1027
|
-
}
|
|
1028
|
-
if (rows.length > 0) {
|
|
1029
|
-
const events = rows
|
|
1030
|
-
.map((row) => {
|
|
1031
|
-
const stored = toStored(row)
|
|
1032
|
-
const clearIndex = asNumber(row['clear_index'])
|
|
1033
|
-
if (clearIndex > 0) stored.clearIndex = clearIndex
|
|
1034
|
-
return stored
|
|
1035
|
-
})
|
|
1036
|
-
.toSorted((a, b) => a.index - b.index)
|
|
1037
|
-
return { outcome: 'claimed', events }
|
|
1038
|
-
}
|
|
1039
|
-
const { rows: activeRows } = await query(
|
|
1040
|
-
`select min(claim_expires_at) as retry_at
|
|
1041
|
-
from a2_events
|
|
1042
|
-
where session_id = $1
|
|
1043
|
-
and processed_at is null
|
|
1044
|
-
and failed_at is null
|
|
1045
|
-
and lane_ready
|
|
1046
|
-
and idx > ${clearIndexOf('$1')}
|
|
1047
|
-
and claim_expires_at > $2
|
|
1048
|
-
and not (idx = any($3::bigint[]))`,
|
|
1049
|
-
[sessionId, now, [...excludeIndexes]],
|
|
2079
|
+
const byIndex = new Map(
|
|
2080
|
+
events.map((event) => [asNumber(event['e_idx']), event]),
|
|
1050
2081
|
)
|
|
1051
|
-
const
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
2082
|
+
for (const row of unseen)
|
|
2083
|
+
Object.assign(row, byIndex.get(asNumber(row['head'])))
|
|
2084
|
+
}
|
|
2085
|
+
const heads = rows.filter((row) => row['e_idx'] !== null)
|
|
2086
|
+
if (heads.length > 0) {
|
|
2087
|
+
const events = heads
|
|
2088
|
+
.map((row) => {
|
|
2089
|
+
const head = toHead(row)
|
|
2090
|
+
const clearIndex = asNumber(row['clear_index'])
|
|
2091
|
+
if (clearIndex > 0) head.clearIndex = clearIndex
|
|
2092
|
+
return head
|
|
2093
|
+
})
|
|
2094
|
+
.toSorted(byHead)
|
|
2095
|
+
return {
|
|
2096
|
+
outcome: 'claimed',
|
|
2097
|
+
events,
|
|
2098
|
+
head: Math.max(
|
|
2099
|
+
...heads.map((row) => asNumber(row['session_head'])),
|
|
2100
|
+
...events.map((event) => event.index),
|
|
2101
|
+
),
|
|
2102
|
+
}
|
|
2103
|
+
}
|
|
2104
|
+
const retryAt = results.at(-1)!['retry_at']
|
|
2105
|
+
return retryAt === null || retryAt === undefined
|
|
2106
|
+
? { outcome: 'settled' }
|
|
2107
|
+
: { outcome: 'busy', retryAt: new Date(asNumber(retryAt)) }
|
|
2108
|
+
})
|
|
1057
2109
|
},
|
|
1058
2110
|
|
|
1059
2111
|
async renewClaims({ sessionId, holder, claims, ttlMs, expiresAtMs }) {
|
|
@@ -1062,320 +2114,274 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
1062
2114
|
const c = await client()
|
|
1063
2115
|
const now = clock.now().getTime()
|
|
1064
2116
|
const expiresAt = expiresAtMs ?? now + ttlMs
|
|
1065
|
-
const
|
|
1066
|
-
|
|
1067
|
-
claims.map((claim) => [claim.index, claim.attempt]),
|
|
2117
|
+
const ordered = claims.toSorted(
|
|
2118
|
+
(a, b) => a.index - b.index || compareText(a.consumer, b.consumer),
|
|
1068
2119
|
)
|
|
1069
2120
|
const { rows } = await c.query(
|
|
1070
2121
|
`with listed as (
|
|
1071
|
-
select
|
|
1072
|
-
|
|
1073
|
-
),
|
|
1074
|
-
|
|
1075
|
-
update a2_events event set claim_expires_at = $1
|
|
2122
|
+
select * from unnest($3::text[], $4::text[], $5::bigint[], $6::int[])
|
|
2123
|
+
with ordinality as listed(consumer, cursor_key, idx, attempt, ord)
|
|
2124
|
+
), renewed as (
|
|
2125
|
+
update a2_cursors cur set claim_expires_at = $7
|
|
1076
2126
|
from listed
|
|
1077
|
-
where
|
|
1078
|
-
and
|
|
1079
|
-
and
|
|
1080
|
-
and
|
|
1081
|
-
and
|
|
1082
|
-
and
|
|
1083
|
-
and
|
|
1084
|
-
and
|
|
1085
|
-
|
|
2127
|
+
where cur.session_id = $1
|
|
2128
|
+
and cur.cursor_key = listed.cursor_key
|
|
2129
|
+
and cur.consumer = listed.consumer
|
|
2130
|
+
and cur.head = listed.idx
|
|
2131
|
+
and listed.idx > ${clearIndexOf('$1')}
|
|
2132
|
+
and cur.attempt_count = listed.attempt
|
|
2133
|
+
and cur.failed_at is null
|
|
2134
|
+
and cur.claim_holder = $2
|
|
2135
|
+
and cur.claim_expires_at > $8
|
|
2136
|
+
returning cur.cursor_key, cur.consumer, cur.head
|
|
1086
2137
|
)
|
|
1087
|
-
select listed.
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
2138
|
+
select listed.ord, latest.head, latest.position, latest.attempt_count,
|
|
2139
|
+
latest.epoch is not null as found,
|
|
2140
|
+
exists(
|
|
2141
|
+
select 1 from renewed
|
|
2142
|
+
where renewed.cursor_key = listed.cursor_key
|
|
2143
|
+
and renewed.consumer = listed.consumer
|
|
2144
|
+
and renewed.head = listed.idx
|
|
2145
|
+
) as renewed,
|
|
2146
|
+
listed.idx <= ${clearIndexOf('$1')} as cleared
|
|
1091
2147
|
from listed
|
|
1092
|
-
left join
|
|
1093
|
-
|
|
2148
|
+
left join lateral (
|
|
2149
|
+
select cur.epoch, cur.head, cur.position, cur.attempt_count
|
|
2150
|
+
from a2_cursors cur
|
|
2151
|
+
where cur.session_id = $1
|
|
2152
|
+
and cur.cursor_key = listed.cursor_key
|
|
2153
|
+
and cur.consumer = listed.consumer
|
|
2154
|
+
order by cur.epoch desc limit 1
|
|
2155
|
+
) latest on true
|
|
2156
|
+
order by listed.ord`,
|
|
1094
2157
|
[
|
|
1095
|
-
expiresAt,
|
|
1096
2158
|
sessionId,
|
|
1097
|
-
indexes,
|
|
1098
2159
|
holder,
|
|
2160
|
+
ordered.map((claim) => claim.consumer),
|
|
2161
|
+
ordered.map((claim) => cursorKeyOf(claim.lane, claim.index)),
|
|
2162
|
+
ordered.map((claim) => claim.index),
|
|
2163
|
+
ordered.map((claim) => claim.attempt),
|
|
2164
|
+
expiresAt,
|
|
1099
2165
|
now,
|
|
1100
|
-
claims.map((claim) => claim.attempt),
|
|
1101
2166
|
],
|
|
1102
2167
|
)
|
|
1103
|
-
const renewed:
|
|
1104
|
-
const superseded:
|
|
2168
|
+
const renewed: HeadRef[] = []
|
|
2169
|
+
const superseded: HeadRef[] = []
|
|
1105
2170
|
for (const row of rows) {
|
|
1106
|
-
const
|
|
1107
|
-
if (row['renewed'] === true)
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
(row['attempt_count'] !== null &&
|
|
1111
|
-
asNumber(row['attempt_count']) > (attempts.get(index) ?? 0))
|
|
1112
|
-
) {
|
|
1113
|
-
superseded.push(index)
|
|
2171
|
+
const { attempt, ...ref } = ordered[asNumber(row['ord']) - 1]!
|
|
2172
|
+
if (row['renewed'] === true) {
|
|
2173
|
+
renewed.push(ref)
|
|
2174
|
+
continue
|
|
1114
2175
|
}
|
|
2176
|
+
if (row['cleared'] === true) {
|
|
2177
|
+
superseded.push(ref)
|
|
2178
|
+
continue
|
|
2179
|
+
}
|
|
2180
|
+
if (row['found'] !== true) continue
|
|
2181
|
+
const head = asNullableNumber(row['head'])
|
|
2182
|
+
// An event the cursor has not reached yet holds no claim to renew.
|
|
2183
|
+
if (head !== ref.index && asNumber(row['position']) < ref.index)
|
|
2184
|
+
continue
|
|
2185
|
+
if (head !== ref.index || asNumber(row['attempt_count']) > attempt)
|
|
2186
|
+
superseded.push(ref)
|
|
1115
2187
|
}
|
|
1116
|
-
return {
|
|
1117
|
-
renewed: renewed.toSorted((a, b) => a - b),
|
|
1118
|
-
superseded: superseded.toSorted((a, b) => a - b),
|
|
1119
|
-
}
|
|
2188
|
+
return { renewed, superseded }
|
|
1120
2189
|
})
|
|
1121
2190
|
},
|
|
1122
2191
|
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
2192
|
+
/*
|
|
2193
|
+
* One round trip. The cursor's lock comes first, and only while this
|
|
2194
|
+
* attempt holds the head (`HOLD`): a completion waiting on a write-back
|
|
2195
|
+
* holds nothing a push needs, and a cleared head is never locked. A
|
|
2196
|
+
* completion that returns events then takes the session lock to give them
|
|
2197
|
+
* indexes, as an append does.
|
|
2198
|
+
*
|
|
2199
|
+
* One that returns none takes the session lock only when its lane or log
|
|
2200
|
+
* cursor would go idle (`LOCK_IF_IDLE`), the one case that needs it: an
|
|
2201
|
+
* append that finds a cursor with a head leaves its events to the
|
|
2202
|
+
* cursor's next head, so a cursor going idle must see every append that
|
|
2203
|
+
* committed before, or that work would wait with no head to run it.
|
|
2204
|
+
* Moving to a next head needs no lock, because the head it reads is the
|
|
2205
|
+
* one it would read under the lock:
|
|
2206
|
+
*
|
|
2207
|
+
* - Every insert into `a2_events` and `a2_lane_events` holds the session
|
|
2208
|
+
* lock through its commit, so they commit in index order: a snapshot
|
|
2209
|
+
* that sees the next head sees every event before it, and nothing
|
|
2210
|
+
* committed later lies before it.
|
|
2211
|
+
* - Only a holder of the cursor's lock marks its lane's events done, and
|
|
2212
|
+
* rows go away only below a clear marker, which `COMPLETE` reads in the
|
|
2213
|
+
* snapshot it reads the next head in: cleared, it writes nothing. A
|
|
2214
|
+
* clear that commits after that snapshot puts its marker past the next
|
|
2215
|
+
* head, which a later append then treats as cleared.
|
|
2216
|
+
*
|
|
2217
|
+
* A concurrent cursor has no next head and no append touches its row, so
|
|
2218
|
+
* it never takes the session lock.
|
|
2219
|
+
*/
|
|
2220
|
+
async completeAttempt({
|
|
2221
|
+
sessionId,
|
|
2222
|
+
consumer,
|
|
2223
|
+
lane,
|
|
2224
|
+
index,
|
|
2225
|
+
attempt,
|
|
2226
|
+
events,
|
|
2227
|
+
consumers,
|
|
2228
|
+
}) {
|
|
2229
|
+
return wrap(async () => {
|
|
2230
|
+
const ids = events.map((event) => event.id)
|
|
2231
|
+
const duplicated = new Set(ids).size !== ids.length
|
|
2232
|
+
const cause: EventCause = { consumer, lane, index, attempt }
|
|
2233
|
+
const now = clock.now().getTime()
|
|
2234
|
+
const rows = events.map((event, offset) => ({
|
|
2235
|
+
ord: offset + 1,
|
|
2236
|
+
id: event.id,
|
|
2237
|
+
type: event.type,
|
|
2238
|
+
payload: event.payload,
|
|
2239
|
+
cause,
|
|
2240
|
+
}))
|
|
2241
|
+
const work =
|
|
2242
|
+
events.length > 0 && !duplicated
|
|
2243
|
+
? enqueue(events, consumers)
|
|
2244
|
+
: undefined
|
|
2245
|
+
const results = await run({
|
|
2246
|
+
sessionId,
|
|
2247
|
+
statements: duplicated
|
|
2248
|
+
? [HOLD]
|
|
2249
|
+
: work
|
|
2250
|
+
? [
|
|
2251
|
+
HOLD,
|
|
2252
|
+
LOCK_SESSION,
|
|
2253
|
+
RETURN_CHECKS,
|
|
2254
|
+
REUSE_IDS,
|
|
2255
|
+
INSERT_EVENTS,
|
|
2256
|
+
...work.statements,
|
|
2257
|
+
COMPLETE_APPENDED,
|
|
2258
|
+
]
|
|
2259
|
+
: [HOLD, LOCK_IF_IDLE, COMPLETE],
|
|
2260
|
+
params: {
|
|
2261
|
+
...work?.params,
|
|
2262
|
+
...retypeOf(consumer, consumers),
|
|
2263
|
+
cursorKey: cursorKeyOf(lane, index),
|
|
2264
|
+
consumer,
|
|
2265
|
+
index,
|
|
2266
|
+
attempt,
|
|
2267
|
+
ids,
|
|
2268
|
+
count: events.length,
|
|
2269
|
+
events: rows,
|
|
2270
|
+
now,
|
|
2271
|
+
clearType: CLEAR_EVENT_TYPE,
|
|
2272
|
+
},
|
|
2273
|
+
abort: (reason) =>
|
|
2274
|
+
reason['abort'] === 'owned'
|
|
2275
|
+
? new A2Error(
|
|
2276
|
+
'PARTIAL_DUPLICATE_BATCH',
|
|
2277
|
+
'returned event batch contains an already-appended event id',
|
|
1199
2278
|
)
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
// lock sees every clear that committed while this one waited.
|
|
1226
|
-
await query('select pg_advisory_xact_lock(hashtext($1))', [sessionId])
|
|
1227
|
-
const { rows: maxRows } = await query(
|
|
1228
|
-
`select coalesce(max(idx), 0) as max, ${clearIndexOf('$1')} as clear_index
|
|
1229
|
-
from a2_events where session_id = $1`,
|
|
1230
|
-
[sessionId],
|
|
2279
|
+
: new A2Error(
|
|
2280
|
+
'STORE_UNAVAILABLE',
|
|
2281
|
+
reason['abort'] === 'held'
|
|
2282
|
+
? `an event id being reused is still held by cleared work in session '${sessionId}'`
|
|
2283
|
+
: 'postgres store operation failed',
|
|
2284
|
+
),
|
|
2285
|
+
})
|
|
2286
|
+
const held = results[0]!
|
|
2287
|
+
if (held['stop'] === 'superseded') return { outcome: 'superseded' }
|
|
2288
|
+
if (held['stop'] === 'unheld') {
|
|
2289
|
+
const c = await client()
|
|
2290
|
+
return settleUnheld({
|
|
2291
|
+
query: (text, values) => c.query(text, values),
|
|
2292
|
+
sessionId,
|
|
2293
|
+
consumer,
|
|
2294
|
+
lane,
|
|
2295
|
+
index,
|
|
2296
|
+
attempt,
|
|
2297
|
+
events,
|
|
2298
|
+
})
|
|
2299
|
+
}
|
|
2300
|
+
if (duplicated) {
|
|
2301
|
+
throw new A2Error(
|
|
2302
|
+
'PARTIAL_DUPLICATE_BATCH',
|
|
2303
|
+
'returned event batch contains the same event id more than once',
|
|
1231
2304
|
)
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
})
|
|
1241
|
-
if (existing.length > 0) {
|
|
1242
|
-
throw new A2Error(
|
|
1243
|
-
'PARTIAL_DUPLICATE_BATCH',
|
|
1244
|
-
'returned event batch contains an already-appended event id',
|
|
1245
|
-
)
|
|
1246
|
-
}
|
|
1247
|
-
}
|
|
1248
|
-
|
|
1249
|
-
const base = asNumber(maxRows[0]?.['max'])
|
|
1250
|
-
const now = clock.now().getTime()
|
|
1251
|
-
const inserted: StoredEvent[] = []
|
|
1252
|
-
for (let offset = 0; offset < events.length; offset += 1) {
|
|
1253
|
-
const event = events[offset]!
|
|
1254
|
-
const childIndex = base + offset + 1
|
|
1255
|
-
// oxlint-disable-next-line no-await-in-loop -- one tx, ordered inserts
|
|
1256
|
-
const row = await insertEvent({
|
|
1257
|
-
query,
|
|
1258
|
-
row: {
|
|
2305
|
+
}
|
|
2306
|
+
if (results.at(-1)!['stop'] === 'superseded')
|
|
2307
|
+
return { outcome: 'superseded' }
|
|
2308
|
+
const checked = results.find((row) => 'base' in row)
|
|
2309
|
+
return {
|
|
2310
|
+
outcome: 'completed',
|
|
2311
|
+
events: checked
|
|
2312
|
+
? insertedRows({
|
|
1259
2313
|
sessionId,
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
processedAt: event.settled ? now : null,
|
|
1268
|
-
},
|
|
1269
|
-
clearIndex,
|
|
1270
|
-
returning: true,
|
|
1271
|
-
})
|
|
1272
|
-
inserted.push(toStored(row!))
|
|
1273
|
-
}
|
|
1274
|
-
await query(
|
|
1275
|
-
`update a2_events set
|
|
1276
|
-
processed_at = $1,
|
|
1277
|
-
processed_by_attempt = $2,
|
|
1278
|
-
returned_event_ids = $5::jsonb,
|
|
1279
|
-
lane_ready = false,
|
|
1280
|
-
claim_holder = null,
|
|
1281
|
-
claim_expires_at = null
|
|
1282
|
-
where session_id = $3 and idx = $4`,
|
|
1283
|
-
[now, attempt, sessionId, index, jsonb(ids)],
|
|
1284
|
-
)
|
|
1285
|
-
await query(
|
|
1286
|
-
`delete from a2_snapshot_pins
|
|
1287
|
-
where session_id = $1 and event_index = $2`,
|
|
1288
|
-
[sessionId, index],
|
|
1289
|
-
)
|
|
1290
|
-
await query(
|
|
1291
|
-
`delete from a2_snapshot_history history
|
|
1292
|
-
where history.session_id = $1
|
|
1293
|
-
and not exists (
|
|
1294
|
-
select 1 from a2_snapshot_pins pin
|
|
1295
|
-
where pin.session_id = history.session_id
|
|
1296
|
-
and pin.reducer_name = history.reducer_name
|
|
1297
|
-
and pin.up_to_index = history.up_to_index
|
|
1298
|
-
)`,
|
|
1299
|
-
[sessionId],
|
|
1300
|
-
)
|
|
1301
|
-
const parentLane = (parent['lane'] as string | null) ?? null
|
|
1302
|
-
if (parentLane !== null) {
|
|
1303
|
-
await query(
|
|
1304
|
-
`update a2_events set lane_ready = true
|
|
1305
|
-
where session_id = $1
|
|
1306
|
-
and idx = (
|
|
1307
|
-
select idx from a2_events
|
|
1308
|
-
where session_id = $1 and lane = $2 and processed_at is null
|
|
1309
|
-
and idx > $3
|
|
1310
|
-
order by idx limit 1
|
|
1311
|
-
)`,
|
|
1312
|
-
[sessionId, parentLane, clearIndex],
|
|
1313
|
-
)
|
|
1314
|
-
}
|
|
1315
|
-
return { outcome: 'completed', events: inserted }
|
|
1316
|
-
}),
|
|
1317
|
-
)
|
|
2314
|
+
events: rows,
|
|
2315
|
+
base: asNumber(checked['base']),
|
|
2316
|
+
now,
|
|
2317
|
+
})
|
|
2318
|
+
: [],
|
|
2319
|
+
}
|
|
2320
|
+
})
|
|
1318
2321
|
},
|
|
1319
2322
|
|
|
1320
2323
|
async failAttempt({
|
|
1321
2324
|
sessionId,
|
|
2325
|
+
consumer,
|
|
2326
|
+
lane,
|
|
1322
2327
|
index,
|
|
1323
2328
|
attempt,
|
|
1324
2329
|
error,
|
|
1325
2330
|
maxFailures,
|
|
1326
2331
|
retryAfterMs,
|
|
1327
2332
|
}) {
|
|
1328
|
-
const now = clock.now().getTime()
|
|
1329
2333
|
return wrap(() =>
|
|
1330
2334
|
withTx(async (query) => {
|
|
2335
|
+
const now = clock.now().getTime()
|
|
2336
|
+
const cursorKey = cursorKeyOf(lane, index)
|
|
1331
2337
|
const { rows } = await query(
|
|
1332
2338
|
`with current as materialized (
|
|
1333
|
-
select
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
2339
|
+
select cur.* from a2_cursors cur
|
|
2340
|
+
where cur.session_id = $1 and cur.cursor_key = $2
|
|
2341
|
+
and cur.consumer = $3 and cur.head = $4
|
|
2342
|
+
and cur.head > ${clearIndexOf('$1')}
|
|
2343
|
+
for update of cur
|
|
1337
2344
|
), updated as (
|
|
1338
|
-
update
|
|
1339
|
-
failure_count =
|
|
1340
|
-
last_error = $
|
|
1341
|
-
last_failed_at = $
|
|
1342
|
-
last_failed_attempt = $
|
|
2345
|
+
update a2_cursors cur set
|
|
2346
|
+
failure_count = cur.failure_count + 1,
|
|
2347
|
+
last_error = $6,
|
|
2348
|
+
last_failed_at = $8,
|
|
2349
|
+
last_failed_attempt = $5,
|
|
1343
2350
|
failed_at = case
|
|
1344
|
-
when
|
|
1345
|
-
else
|
|
2351
|
+
when cur.failure_count + 1 >= $7 then $8
|
|
2352
|
+
else cur.failed_at
|
|
1346
2353
|
end,
|
|
1347
2354
|
claim_holder = null,
|
|
1348
2355
|
claim_expires_at = case
|
|
1349
|
-
when
|
|
1350
|
-
else $
|
|
2356
|
+
when cur.failure_count + 1 >= $7 then null
|
|
2357
|
+
else $9::bigint
|
|
1351
2358
|
end
|
|
1352
2359
|
from current
|
|
1353
|
-
where
|
|
1354
|
-
and
|
|
1355
|
-
and current.
|
|
2360
|
+
where cur.session_id = $1 and cur.cursor_key = $2
|
|
2361
|
+
and cur.consumer = $3 and cur.epoch = current.epoch
|
|
2362
|
+
and current.attempt_count = $5
|
|
1356
2363
|
and current.failed_at is null
|
|
1357
|
-
and current.attempt_count = $3
|
|
1358
2364
|
and current.claim_holder is not null
|
|
1359
|
-
returning
|
|
2365
|
+
returning cur.failure_count, cur.failed_at
|
|
1360
2366
|
)
|
|
1361
2367
|
select case
|
|
1362
|
-
when $
|
|
2368
|
+
when $4 <= ${clearIndexOf('$1')} then 'cleared'
|
|
1363
2369
|
when not exists (select 1 from current) then 'missing'
|
|
1364
2370
|
when exists (
|
|
1365
2371
|
select 1 from updated where failed_at is not null
|
|
1366
2372
|
) then 'dead_lettered'
|
|
1367
2373
|
when exists (select 1 from updated) then 'failed'
|
|
1368
2374
|
when exists (
|
|
1369
|
-
select 1 from current
|
|
1370
|
-
where processed_at is not null or attempt_count <> $3
|
|
2375
|
+
select 1 from current where attempt_count <> $5
|
|
1371
2376
|
) then 'superseded'
|
|
1372
|
-
when exists (
|
|
1373
|
-
select 1 from current where failed_at is not null
|
|
1374
|
-
) then 'dead_lettered'
|
|
1375
2377
|
when exists (
|
|
1376
2378
|
select 1 from current
|
|
1377
|
-
where claim_holder is null and last_failed_attempt = $
|
|
2379
|
+
where claim_holder is null and last_failed_attempt = $5
|
|
2380
|
+
and failed_at is null
|
|
1378
2381
|
) then 'failed'
|
|
2382
|
+
when exists (
|
|
2383
|
+
select 1 from current where failed_at is not null
|
|
2384
|
+
) then 'dead_lettered'
|
|
1379
2385
|
else 'superseded'
|
|
1380
2386
|
end as outcome,
|
|
1381
2387
|
coalesce(
|
|
@@ -1384,6 +2390,8 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
1384
2390
|
) as failure_count`,
|
|
1385
2391
|
[
|
|
1386
2392
|
sessionId,
|
|
2393
|
+
cursorKey,
|
|
2394
|
+
consumer,
|
|
1387
2395
|
index,
|
|
1388
2396
|
attempt,
|
|
1389
2397
|
error,
|
|
@@ -1397,15 +2405,24 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
1397
2405
|
return { outcome: 'superseded', failureCount: 0 }
|
|
1398
2406
|
}
|
|
1399
2407
|
if (!row || row['outcome'] === 'missing') {
|
|
1400
|
-
|
|
1401
|
-
|
|
2408
|
+
const { rows: cursorRows } = await query(
|
|
2409
|
+
`select 1 from a2_cursors
|
|
2410
|
+
where session_id = $1 and cursor_key = $2 and consumer = $3`,
|
|
2411
|
+
[sessionId, cursorKey, consumer],
|
|
2412
|
+
)
|
|
2413
|
+
if (
|
|
2414
|
+
cursorRows.length > 0 ||
|
|
2415
|
+
(await hasEvent(query, sessionId, index)) ||
|
|
2416
|
+
// Deleted while this statement waited for its lock, after a clear.
|
|
2417
|
+
(await clearedAt({ query, sessionId, index }))
|
|
2418
|
+
)
|
|
1402
2419
|
return { outcome: 'superseded', failureCount: 0 }
|
|
1403
2420
|
throw new TypeError(
|
|
1404
2421
|
`no event at index ${index} in session '${sessionId}'`,
|
|
1405
2422
|
)
|
|
1406
2423
|
}
|
|
1407
2424
|
if (row['outcome'] === 'dead_lettered')
|
|
1408
|
-
await releasePins({ query, sessionId, index })
|
|
2425
|
+
await releasePins({ query, sessionId, consumer, cursorKey, index })
|
|
1409
2426
|
return {
|
|
1410
2427
|
outcome: row['outcome'] as
|
|
1411
2428
|
'failed' | 'dead_lettered' | 'superseded',
|
|
@@ -1419,80 +2436,47 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
1419
2436
|
return wrap(async () => {
|
|
1420
2437
|
const c = await client()
|
|
1421
2438
|
const { rows } = await c.query(
|
|
1422
|
-
`select
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
2439
|
+
`select cur.*, ${HEAD_COLUMNS}
|
|
2440
|
+
from a2_cursors cur
|
|
2441
|
+
join a2_events event
|
|
2442
|
+
on event.session_id = cur.session_id and event.idx = cur.head
|
|
2443
|
+
where cur.session_id = $1
|
|
2444
|
+
and cur.failed_at is not null
|
|
2445
|
+
and cur.head > ${clearIndexOf('$1')}`,
|
|
1428
2446
|
[sessionId],
|
|
1429
2447
|
)
|
|
1430
|
-
return rows.map(
|
|
2448
|
+
return rows.map(toHead).toSorted(byHead)
|
|
1431
2449
|
})
|
|
1432
2450
|
},
|
|
1433
2451
|
|
|
1434
|
-
async redrive({ sessionId, index, action }) {
|
|
1435
|
-
return wrap(() =>
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
await query(
|
|
1463
|
-
`update a2_events set processed_at = $3, lane_ready = false
|
|
1464
|
-
where session_id = $1 and idx = $2`,
|
|
1465
|
-
[sessionId, index, clock.now().getTime()],
|
|
1466
|
-
)
|
|
1467
|
-
const lane = (row['lane'] as string | null) ?? null
|
|
1468
|
-
if (lane !== null) {
|
|
1469
|
-
await query(
|
|
1470
|
-
`update a2_events set lane_ready = true
|
|
1471
|
-
where session_id = $1
|
|
1472
|
-
and idx = (
|
|
1473
|
-
select idx from a2_events
|
|
1474
|
-
where session_id = $1 and lane = $2 and processed_at is null
|
|
1475
|
-
and idx > ${clearIndexOf('$1')}
|
|
1476
|
-
order by idx limit 1
|
|
1477
|
-
)`,
|
|
1478
|
-
[sessionId, lane],
|
|
1479
|
-
)
|
|
1480
|
-
}
|
|
1481
|
-
}
|
|
1482
|
-
const { rows: status } = await query(
|
|
1483
|
-
`select exists(
|
|
1484
|
-
select 1 from a2_events
|
|
1485
|
-
where session_id = $1 and processed_at is null
|
|
1486
|
-
and idx > ${clearIndexOf('$1')}
|
|
1487
|
-
) as has_pending`,
|
|
1488
|
-
[sessionId],
|
|
1489
|
-
)
|
|
1490
|
-
return {
|
|
1491
|
-
redriven: row !== undefined,
|
|
1492
|
-
hasPending: status[0]?.['has_pending'] === true,
|
|
1493
|
-
}
|
|
1494
|
-
}),
|
|
1495
|
-
)
|
|
2452
|
+
async redrive({ sessionId, consumer, index, action, consumers }) {
|
|
2453
|
+
return wrap(async () => {
|
|
2454
|
+
// Lane promotion must see every append that committed before it,
|
|
2455
|
+
// so a skip takes the session lock first, in append's order. The
|
|
2456
|
+
// cursor lock serializes both actions with claims and consumer
|
|
2457
|
+
// appends.
|
|
2458
|
+
const results = await run({
|
|
2459
|
+
sessionId,
|
|
2460
|
+
statements: [
|
|
2461
|
+
action === 'skip' && LOCK_SESSION,
|
|
2462
|
+
LOCK_DEAD,
|
|
2463
|
+
action === 'skip' ? SKIP_DEAD : RETRY_DEAD,
|
|
2464
|
+
PENDING,
|
|
2465
|
+
],
|
|
2466
|
+
params: {
|
|
2467
|
+
...retypeOf(consumer, consumers),
|
|
2468
|
+
consumer,
|
|
2469
|
+
index,
|
|
2470
|
+
served: servedNames(consumers),
|
|
2471
|
+
clearType: CLEAR_EVENT_TYPE,
|
|
2472
|
+
},
|
|
2473
|
+
})
|
|
2474
|
+
return {
|
|
2475
|
+
redriven:
|
|
2476
|
+
results.find((row) => 'redriven' in row)?.['redriven'] === true,
|
|
2477
|
+
hasPending: results.at(-1)!['has_pending'] === true,
|
|
2478
|
+
}
|
|
2479
|
+
})
|
|
1496
2480
|
},
|
|
1497
2481
|
|
|
1498
2482
|
async readState(sessionId, reducerName, stateOptions) {
|
|
@@ -1555,39 +2539,65 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
1555
2539
|
headRows[0]?.['up_to_index'] == null ? undefined : headRows[0]
|
|
1556
2540
|
|
|
1557
2541
|
const requestedPins = normalized.flatMap((write) =>
|
|
1558
|
-
(write.
|
|
1559
|
-
eventIndex,
|
|
2542
|
+
(write.pins ?? []).map((pin) => ({
|
|
2543
|
+
eventIndex: pin.index,
|
|
2544
|
+
consumer: pin.consumer,
|
|
2545
|
+
cursorKey: cursorKeyOf(pin.lane, pin.index),
|
|
1560
2546
|
snapshotIndex: write.index,
|
|
1561
2547
|
})),
|
|
1562
2548
|
)
|
|
1563
2549
|
if (requestedPins.length > 0) {
|
|
1564
|
-
const
|
|
1565
|
-
...new
|
|
2550
|
+
const heads = [
|
|
2551
|
+
...new Map(
|
|
2552
|
+
requestedPins.map((pin) => [
|
|
2553
|
+
`${pin.consumer}\u0000${pin.cursorKey}\u0000${pin.eventIndex}`,
|
|
2554
|
+
pin,
|
|
2555
|
+
]),
|
|
2556
|
+
).values(),
|
|
1566
2557
|
]
|
|
1567
|
-
|
|
1568
|
-
|
|
1569
|
-
|
|
1570
|
-
|
|
1571
|
-
|
|
2558
|
+
// Held `for share` so a completion or dead letter releasing the
|
|
2559
|
+
// head's pins waits for these to commit, and sees them.
|
|
2560
|
+
const { rows: live } = await query(
|
|
2561
|
+
`select cur.consumer, cur.cursor_key, cur.head
|
|
2562
|
+
from a2_cursors cur
|
|
2563
|
+
join unnest($2::text[], $3::text[], $4::bigint[])
|
|
2564
|
+
as pin(consumer, cursor_key, idx)
|
|
2565
|
+
on cur.consumer = pin.consumer
|
|
2566
|
+
and cur.cursor_key = pin.cursor_key
|
|
2567
|
+
and cur.head = pin.idx
|
|
2568
|
+
where cur.session_id = $1 and cur.head > $5
|
|
2569
|
+
and cur.failed_at is null
|
|
2570
|
+
order by cur.cursor_key, cur.consumer
|
|
2571
|
+
for share of cur`,
|
|
2572
|
+
[
|
|
2573
|
+
sessionId,
|
|
2574
|
+
heads.map((pin) => pin.consumer),
|
|
2575
|
+
heads.map((pin) => pin.cursorKey),
|
|
2576
|
+
heads.map((pin) => pin.eventIndex),
|
|
2577
|
+
clearIndex,
|
|
2578
|
+
],
|
|
1572
2579
|
)
|
|
1573
2580
|
const unfinished = new Set(
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
(row)
|
|
1577
|
-
|
|
1578
|
-
)
|
|
1579
|
-
.map((row) => asNumber(row['idx'])),
|
|
2581
|
+
live.map(
|
|
2582
|
+
(row) =>
|
|
2583
|
+
`${String(row['consumer'])}\u0000${String(row['cursor_key'])}\u0000${asNumber(row['head'])}`,
|
|
2584
|
+
),
|
|
1580
2585
|
)
|
|
1581
2586
|
const accepted = requestedPins.filter((pin) =>
|
|
1582
|
-
unfinished.has(
|
|
2587
|
+
unfinished.has(
|
|
2588
|
+
`${pin.consumer}\u0000${pin.cursorKey}\u0000${pin.eventIndex}`,
|
|
2589
|
+
),
|
|
1583
2590
|
)
|
|
1584
2591
|
if (accepted.length > 0) {
|
|
1585
2592
|
await query(
|
|
1586
2593
|
`insert into a2_snapshot_pins
|
|
1587
|
-
(session_id, event_index, reducer_name, up_to_index)
|
|
1588
|
-
select $1, pin."eventIndex",
|
|
2594
|
+
(session_id, event_index, consumer, cursor_key, reducer_name, up_to_index)
|
|
2595
|
+
select $1, pin."eventIndex", pin.consumer, pin."cursorKey",
|
|
2596
|
+
$2, pin."snapshotIndex"
|
|
1589
2597
|
from jsonb_to_recordset($3::jsonb) as pin(
|
|
1590
2598
|
"eventIndex" bigint,
|
|
2599
|
+
consumer text,
|
|
2600
|
+
"cursorKey" text,
|
|
1591
2601
|
"snapshotIndex" bigint
|
|
1592
2602
|
)
|
|
1593
2603
|
on conflict do nothing`,
|
|
@@ -1702,9 +2712,7 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
1702
2712
|
// orders writes but never anchors their lifetime.
|
|
1703
2713
|
const expiresAtMs = clock.now().getTime() + meta.ttlMs
|
|
1704
2714
|
const applied: Record<string, unknown> = nullProtoRecord()
|
|
1705
|
-
const write = async (
|
|
1706
|
-
query: PostgresConnection['query'],
|
|
1707
|
-
): Promise<void> => {
|
|
2715
|
+
const write = async (query: Query): Promise<void> => {
|
|
1708
2716
|
for (const [field, value] of entries) {
|
|
1709
2717
|
// Field-wise LWW: a stored row with a strictly newer `at`
|
|
1710
2718
|
// survives; ties go to the incoming write (memory parity).
|
|
@@ -1819,7 +2827,7 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
1819
2827
|
async clear({ sessionId, ifLastIndex, cause }) {
|
|
1820
2828
|
return wrap(() =>
|
|
1821
2829
|
withTx(async (query) => {
|
|
1822
|
-
await query
|
|
2830
|
+
await lockSession(query, sessionId)
|
|
1823
2831
|
const { rows: headRows } = await query(
|
|
1824
2832
|
`select coalesce(max(idx), 0) as head,
|
|
1825
2833
|
${clearIndexOf('$1')} as clear_index
|
|
@@ -1829,26 +2837,13 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
1829
2837
|
const head = asNumber(headRows[0]?.['head'])
|
|
1830
2838
|
const clearIndex = asNumber(headRows[0]?.['clear_index'])
|
|
1831
2839
|
if (cause) {
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
)
|
|
1840
|
-
const parent = parents[0]
|
|
1841
|
-
if (!parent) {
|
|
1842
|
-
throw new TypeError(
|
|
1843
|
-
`no event at index ${cause.index} in session '${sessionId}'`,
|
|
1844
|
-
)
|
|
1845
|
-
}
|
|
1846
|
-
if (
|
|
1847
|
-
asNumber(parent['attempt_count']) !== cause.attempt ||
|
|
1848
|
-
parent['failed_at'] !== null
|
|
1849
|
-
) {
|
|
1850
|
-
throw supersededAttemptError(sessionId, cause)
|
|
1851
|
-
}
|
|
2840
|
+
await assertCurrentCause({
|
|
2841
|
+
query,
|
|
2842
|
+
sessionId,
|
|
2843
|
+
cause,
|
|
2844
|
+
clearIndex,
|
|
2845
|
+
lock: 'update',
|
|
2846
|
+
})
|
|
1852
2847
|
}
|
|
1853
2848
|
if (ifLastIndex !== undefined && clearIndex === ifLastIndex + 1) {
|
|
1854
2849
|
return { cleared: true, index: head, clearIndex }
|
|
@@ -1860,33 +2855,38 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
1860
2855
|
return { cleared: true, index: head, clearIndex }
|
|
1861
2856
|
}
|
|
1862
2857
|
const marker = head + 1
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
2858
|
+
await query(
|
|
2859
|
+
`insert into a2_events
|
|
2860
|
+
(session_id, idx, event_type, payload, event_id, created_at, cause)
|
|
2861
|
+
values ($1, $2, $3, 'null'::jsonb, $4, $5, null)`,
|
|
2862
|
+
[
|
|
1867
2863
|
sessionId,
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
})
|
|
1879
|
-
// Releasing a clearing handler's claim makes a claim waiting on its
|
|
1880
|
-
// row skip it once this commits, instead of re-running the handler.
|
|
2864
|
+
marker,
|
|
2865
|
+
CLEAR_EVENT_TYPE,
|
|
2866
|
+
generateId(),
|
|
2867
|
+
clock.now().getTime(),
|
|
2868
|
+
],
|
|
2869
|
+
)
|
|
2870
|
+
// Every cursor at or below the marker is superseded by the marker
|
|
2871
|
+
// alone, and reclaim deletes it. Only a clearing consumer's own
|
|
2872
|
+
// cursor is written: releasing its head makes a claim waiting on
|
|
2873
|
+
// the row skip it once this commits, instead of re-running it.
|
|
1881
2874
|
await query(
|
|
1882
2875
|
`with released as (
|
|
1883
|
-
update
|
|
1884
|
-
set
|
|
1885
|
-
where session_id = $1 and
|
|
2876
|
+
update a2_cursors
|
|
2877
|
+
set head = null, ${FRESH_DISPATCH}
|
|
2878
|
+
where session_id = $1 and cursor_key = $3 and consumer = $4
|
|
2879
|
+
and head = $5
|
|
1886
2880
|
)
|
|
1887
2881
|
insert into a2_clears (session_id, clear_index) values ($1, $2)
|
|
1888
2882
|
on conflict (session_id) do update set clear_index = excluded.clear_index`,
|
|
1889
|
-
[
|
|
2883
|
+
[
|
|
2884
|
+
sessionId,
|
|
2885
|
+
marker,
|
|
2886
|
+
cause ? cursorKeyOf(cause.lane, cause.index) : null,
|
|
2887
|
+
cause?.consumer ?? null,
|
|
2888
|
+
cause?.index ?? null,
|
|
2889
|
+
],
|
|
1890
2890
|
)
|
|
1891
2891
|
return { cleared: true, index: marker, clearIndex: marker }
|
|
1892
2892
|
}),
|
|
@@ -1915,6 +2915,26 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
1915
2915
|
), pins as (
|
|
1916
2916
|
delete from a2_snapshot_pins pin using deleted
|
|
1917
2917
|
where pin.session_id = $1 and pin.event_index = deleted.idx
|
|
2918
|
+
), lanes as materialized (
|
|
2919
|
+
-- Each lane's queue through the batch, by the lane's own key.
|
|
2920
|
+
select lane.ctid from a2_cursors cur
|
|
2921
|
+
join a2_lane_events lane
|
|
2922
|
+
on lane.session_id = cur.session_id
|
|
2923
|
+
and lane.consumer = cur.consumer and lane.lane = cur.lane
|
|
2924
|
+
where cur.session_id = $1 and cur.kind = 'lane'
|
|
2925
|
+
and lane.idx <= (select max(idx) from batch)
|
|
2926
|
+
for update of lane skip locked
|
|
2927
|
+
), gone_lanes as (
|
|
2928
|
+
delete from a2_lane_events target using lanes
|
|
2929
|
+
where target.ctid = lanes.ctid
|
|
2930
|
+
), singles as materialized (
|
|
2931
|
+
select cur.ctid from a2_cursors cur
|
|
2932
|
+
join batch on cur.cursor_key = '#' || batch.idx
|
|
2933
|
+
where cur.session_id = $1 and cur.kind = 'event'
|
|
2934
|
+
for update of cur skip locked
|
|
2935
|
+
), gone_singles as (
|
|
2936
|
+
delete from a2_cursors target using singles
|
|
2937
|
+
where target.ctid = singles.ctid
|
|
1918
2938
|
)
|
|
1919
2939
|
select (select count(*) from deleted) as deleted,
|
|
1920
2940
|
(select clear_index from boundary) as clear_index`,
|
|
@@ -1923,14 +2943,23 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
1923
2943
|
const deleted = asNumber(rows[0]?.['deleted'])
|
|
1924
2944
|
const clearIndex = asNumber(rows[0]?.['clear_index'])
|
|
1925
2945
|
if (clearIndex === 0) return { deleted: 0, done: true }
|
|
1926
|
-
if (deleted > 0) return { deleted, done: false }
|
|
1927
2946
|
const { rows: remaining } = await query(
|
|
1928
2947
|
'select 1 from a2_events where session_id = $1 and idx < $2 limit 1',
|
|
1929
2948
|
[sessionId, clearIndex],
|
|
1930
2949
|
)
|
|
1931
|
-
if (remaining.length > 0) return { deleted
|
|
2950
|
+
if (remaining.length > 0) return { deleted, done: false }
|
|
2951
|
+
// Writes serialize with the clear, so none lands below the marker
|
|
2952
|
+
// later: one sweep of what the batches left finishes the session.
|
|
1932
2953
|
const { rows: swept } = await query(
|
|
1933
|
-
`with
|
|
2954
|
+
`with lanes as materialized (
|
|
2955
|
+
select ctid from a2_lane_events
|
|
2956
|
+
where session_id = $1 and idx < $2
|
|
2957
|
+
for update skip locked
|
|
2958
|
+
), cursors as materialized (
|
|
2959
|
+
select cur.ctid from a2_cursors cur
|
|
2960
|
+
where ${SUPERSEDED_CURSOR}
|
|
2961
|
+
for update skip locked
|
|
2962
|
+
), pins as materialized (
|
|
1934
2963
|
select ctid from a2_snapshot_pins
|
|
1935
2964
|
where session_id = $1 and up_to_index < $2
|
|
1936
2965
|
for update skip locked
|
|
@@ -1942,6 +2971,12 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
1942
2971
|
select ctid from a2_snapshots
|
|
1943
2972
|
where session_id = $1 and up_to_index < $2
|
|
1944
2973
|
for update skip locked
|
|
2974
|
+
), gone_lanes as (
|
|
2975
|
+
delete from a2_lane_events target using lanes
|
|
2976
|
+
where target.ctid = lanes.ctid
|
|
2977
|
+
), gone_cursors as (
|
|
2978
|
+
delete from a2_cursors target using cursors
|
|
2979
|
+
where target.ctid = cursors.ctid
|
|
1945
2980
|
), gone_pins as (
|
|
1946
2981
|
delete from a2_snapshot_pins target using pins where target.ctid = pins.ctid
|
|
1947
2982
|
), gone_history as (
|
|
@@ -1950,18 +2985,24 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
1950
2985
|
), gone_heads as (
|
|
1951
2986
|
delete from a2_snapshots target using heads where target.ctid = heads.ctid
|
|
1952
2987
|
)
|
|
1953
|
-
select (select count(*) from
|
|
2988
|
+
select (select count(*) from a2_lane_events
|
|
2989
|
+
where session_id = $1 and idx < $2)
|
|
2990
|
+
+ (select count(*) from a2_cursors cur
|
|
2991
|
+
where ${SUPERSEDED_CURSOR})
|
|
2992
|
+
+ (select count(*) from a2_snapshot_pins
|
|
1954
2993
|
where session_id = $1 and up_to_index < $2)
|
|
1955
2994
|
+ (select count(*) from a2_snapshot_history
|
|
1956
2995
|
where session_id = $1 and up_to_index < $2)
|
|
1957
2996
|
+ (select count(*) from a2_snapshots
|
|
1958
2997
|
where session_id = $1 and up_to_index < $2)
|
|
2998
|
+
- (select count(*) from lanes)
|
|
2999
|
+
- (select count(*) from cursors)
|
|
1959
3000
|
- (select count(*) from pins)
|
|
1960
3001
|
- (select count(*) from history)
|
|
1961
3002
|
- (select count(*) from heads) as skipped`,
|
|
1962
3003
|
[sessionId, clearIndex],
|
|
1963
3004
|
)
|
|
1964
|
-
return { deleted
|
|
3005
|
+
return { deleted, done: asNumber(swept[0]?.['skipped']) === 0 }
|
|
1965
3006
|
}),
|
|
1966
3007
|
)
|
|
1967
3008
|
},
|
|
@@ -1973,8 +3014,8 @@ export function postgres(options: PostgresStoreOptions = {}): PostgresStore {
|
|
|
1973
3014
|
subscription.stop()
|
|
1974
3015
|
presenceSubscriptions.clear()
|
|
1975
3016
|
await notifications?.close()
|
|
1976
|
-
const
|
|
1977
|
-
await (injected ?? lease)?.release()
|
|
3017
|
+
const opened = await connection.peek()?.catch(() => undefined)
|
|
3018
|
+
await (injected ?? opened?.lease)?.release()
|
|
1978
3019
|
},
|
|
1979
3020
|
}
|
|
1980
3021
|
if (notifications) {
|