@nylorun/runtime 0.13.0-beta → 0.15.0-beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (309) hide show
  1. package/CHANGELOG.md +434 -0
  2. package/README.md +137 -41
  3. package/dist/adapters/execution/restate.d.ts +7 -2
  4. package/dist/adapters/execution/restate.js +40 -10
  5. package/dist/adapters/replication/pgoutput.d.ts +21 -0
  6. package/dist/adapters/replication/pgoutput.js +170 -0
  7. package/dist/adapters/streams/s2.js +37 -20
  8. package/dist/admin-openapi.json +570 -0
  9. package/dist/api/a2a/card.d.ts +15 -0
  10. package/dist/api/a2a/card.js +24 -0
  11. package/dist/api/a2a/endpoint.d.ts +8 -0
  12. package/dist/api/a2a/endpoint.js +81 -0
  13. package/dist/api/a2a/protocol.d.ts +144 -0
  14. package/dist/api/a2a/protocol.js +188 -0
  15. package/dist/api/a2a/routes.d.ts +19 -0
  16. package/dist/api/a2a/routes.js +289 -0
  17. package/dist/api/a2a/tasks.d.ts +50 -0
  18. package/dist/api/a2a/tasks.js +186 -0
  19. package/dist/api/ag-ui/endpoint.d.ts +8 -0
  20. package/dist/api/ag-ui/endpoint.js +126 -0
  21. package/dist/api/ag-ui/history.d.ts +7 -0
  22. package/dist/api/ag-ui/history.js +60 -0
  23. package/dist/api/ag-ui/routes.d.ts +32 -0
  24. package/dist/api/ag-ui/routes.js +305 -0
  25. package/dist/api/ag-ui/session-id.d.ts +1 -0
  26. package/dist/api/ag-ui/session-id.js +12 -0
  27. package/dist/api/ag-ui/sse.d.ts +9 -0
  28. package/dist/api/ag-ui/sse.js +7 -0
  29. package/dist/api/ag-ui/translate.d.ts +38 -0
  30. package/dist/api/ag-ui/translate.js +222 -0
  31. package/dist/api/components.d.ts +8065 -0
  32. package/dist/api/components.js +82 -0
  33. package/dist/api/gate/keys.d.ts +14 -0
  34. package/dist/api/gate/keys.js +59 -0
  35. package/dist/api/gate/routes.d.ts +34 -0
  36. package/dist/api/gate/routes.js +235 -0
  37. package/dist/api/http/app.d.ts +18 -0
  38. package/dist/api/http/app.js +58 -0
  39. package/dist/api/http/body.d.ts +4 -0
  40. package/dist/api/http/body.js +37 -0
  41. package/dist/api/http/define.d.ts +39 -0
  42. package/dist/api/http/define.js +122 -0
  43. package/dist/api/http/respond.d.ts +9 -0
  44. package/dist/api/http/respond.js +44 -0
  45. package/dist/api/http/routes/access.d.ts +24 -0
  46. package/dist/api/http/routes/access.js +266 -0
  47. package/dist/api/http/routes/actions.d.ts +8 -0
  48. package/dist/api/http/routes/actions.js +76 -0
  49. package/dist/api/http/routes/endpoints.d.ts +7 -0
  50. package/dist/api/http/routes/endpoints.js +100 -0
  51. package/dist/api/http/routes/sessions.d.ts +3 -0
  52. package/dist/api/http/routes/sessions.js +205 -0
  53. package/dist/api/http/routes/tenant.d.ts +11 -0
  54. package/dist/api/http/routes/tenant.js +244 -0
  55. package/dist/api/http/routes/vaults.d.ts +8 -0
  56. package/dist/api/http/routes/vaults.js +167 -0
  57. package/dist/api/openapi.d.ts +19 -0
  58. package/dist/api/openapi.js +137 -0
  59. package/dist/api/route.d.ts +12 -0
  60. package/dist/api/route.js +4 -0
  61. package/dist/core/bearer.d.ts +8 -0
  62. package/dist/core/bearer.js +11 -0
  63. package/dist/core/flow-host.d.ts +7 -15
  64. package/dist/core/flow-host.js +16 -41
  65. package/dist/core/sandbox-routes.d.ts +12 -7
  66. package/dist/core/sandbox-routes.js +24 -25
  67. package/dist/execution/memory.d.ts +8 -5
  68. package/dist/execution/memory.js +33 -13
  69. package/dist/execution/types.d.ts +38 -6
  70. package/dist/execution/types.js +12 -3
  71. package/dist/gates/contract.d.ts +51 -0
  72. package/dist/gates/contract.js +37 -0
  73. package/dist/gates/handler.d.ts +21 -0
  74. package/dist/gates/handler.js +55 -0
  75. package/dist/gates/http-client.d.ts +12 -0
  76. package/dist/gates/http-client.js +160 -0
  77. package/dist/gates/in-process.d.ts +32 -0
  78. package/dist/gates/in-process.js +42 -0
  79. package/dist/gates/inflight.d.ts +39 -0
  80. package/dist/gates/inflight.js +70 -0
  81. package/dist/gates/mcp-handler.d.ts +35 -0
  82. package/dist/gates/mcp-handler.js +142 -0
  83. package/dist/gates/meter.d.ts +24 -0
  84. package/dist/gates/meter.js +165 -0
  85. package/dist/gates/model-gate.d.ts +45 -0
  86. package/dist/gates/model-gate.js +1 -0
  87. package/dist/gates/tenant-vaults.d.ts +64 -0
  88. package/dist/gates/tenant-vaults.js +94 -0
  89. package/dist/gates/tool-calls.d.ts +51 -0
  90. package/dist/gates/tool-calls.js +64 -0
  91. package/dist/gates/tool-client.d.ts +12 -0
  92. package/dist/gates/tool-client.js +201 -0
  93. package/dist/gates/tool-contract.d.ts +116 -0
  94. package/dist/gates/tool-contract.js +64 -0
  95. package/dist/gates/tool-gate.d.ts +45 -0
  96. package/dist/gates/tool-gate.js +10 -0
  97. package/dist/host/admin-api.d.ts +23 -0
  98. package/dist/host/admin-api.js +74 -0
  99. package/dist/host/app.d.ts +34 -0
  100. package/dist/host/app.js +181 -0
  101. package/dist/host/config-for.d.ts +3 -1
  102. package/dist/host/config-for.js +3 -2
  103. package/dist/host/config.d.ts +11 -0
  104. package/dist/host/cors.d.ts +27 -0
  105. package/dist/host/cors.js +64 -0
  106. package/dist/host/create-host.d.ts +36 -8
  107. package/dist/host/create-host.js +161 -330
  108. package/dist/host/execution.d.ts +20 -18
  109. package/dist/host/execution.js +2 -2
  110. package/dist/host/gates.d.ts +58 -0
  111. package/dist/host/gates.js +127 -0
  112. package/dist/host/http.d.ts +20 -4
  113. package/dist/host/http.js +81 -23
  114. package/dist/host/index.d.ts +1 -1
  115. package/dist/host/main.js +189 -58
  116. package/dist/host/stack-config.d.ts +91 -19
  117. package/dist/host/stack-config.js +246 -17
  118. package/dist/infra/database.d.ts +2 -2
  119. package/dist/infra/execution.d.ts +5 -5
  120. package/dist/infra/execution.js +8 -6
  121. package/dist/infra/index.d.ts +1 -1
  122. package/dist/infra/workers.d.ts +17 -17
  123. package/dist/infra/workers.js +9 -9
  124. package/dist/keys/client.d.ts +9 -0
  125. package/dist/keys/client.js +70 -0
  126. package/dist/keys/contract.d.ts +24 -0
  127. package/dist/keys/contract.js +12 -0
  128. package/dist/keys/keys.d.ts +41 -0
  129. package/dist/keys/keys.js +50 -0
  130. package/dist/mcp/connect.d.ts +43 -3
  131. package/dist/mcp/connect.js +12 -5
  132. package/dist/mcp/pool.d.ts +39 -3
  133. package/dist/mcp/pool.js +74 -11
  134. package/dist/model/catalog.d.ts +2 -10
  135. package/dist/model/classify.d.ts +13 -0
  136. package/dist/model/classify.js +61 -0
  137. package/dist/model/models.d.ts +10 -1
  138. package/dist/model/models.js +14 -6
  139. package/dist/model/pi-model.d.ts +17 -2
  140. package/dist/model/pi-model.js +366 -222
  141. package/dist/openapi.json +25889 -0
  142. package/dist/record/append.d.ts +51 -0
  143. package/dist/record/append.js +20 -0
  144. package/dist/record/envelope.d.ts +19 -0
  145. package/dist/record/envelope.js +49 -0
  146. package/dist/record/index.d.ts +11 -0
  147. package/dist/record/index.js +11 -0
  148. package/dist/sandbox/manager.d.ts +13 -1
  149. package/dist/sandbox/manager.js +8 -11
  150. package/dist/sandbox/resolve.d.ts +33 -0
  151. package/dist/sandbox/resolve.js +81 -0
  152. package/dist/sandbox/session-sandbox.d.ts +11 -0
  153. package/dist/sandbox/session-sandbox.js +51 -0
  154. package/dist/sandbox/share.d.ts +14 -2
  155. package/dist/sandbox/share.js +33 -3
  156. package/dist/sandbox/tenant-config.d.ts +37 -0
  157. package/dist/sandbox/tenant-config.js +51 -0
  158. package/dist/sandbox/tools.d.ts +7 -9
  159. package/dist/sandbox/tools.js +11 -3
  160. package/dist/sandbox/types.d.ts +8 -2
  161. package/dist/store/postgres/connect.d.ts +25 -9
  162. package/dist/store/postgres/connect.js +27 -10
  163. package/dist/store/postgres/db.d.ts +16 -0
  164. package/dist/store/postgres/db.js +28 -0
  165. package/dist/store/postgres/drizzle/0000_foundation.sql +22 -0
  166. package/dist/store/postgres/drizzle/0001_baseline.sql +255 -0
  167. package/dist/store/postgres/drizzle/0002_stream_relay.sql +4 -0
  168. package/dist/store/postgres/drizzle/0003_tool_crossings.sql +9 -0
  169. package/dist/store/postgres/drizzle/meta/_journal.json +34 -0
  170. package/dist/store/postgres/migrate.d.ts +68 -0
  171. package/dist/store/postgres/migrate.js +183 -0
  172. package/dist/store/postgres/record-writer.d.ts +4 -0
  173. package/dist/store/postgres/record-writer.js +34 -0
  174. package/dist/store/postgres/record.d.ts +5 -0
  175. package/dist/store/postgres/record.js +60 -0
  176. package/dist/store/postgres/schema.d.ts +3096 -0
  177. package/dist/store/postgres/schema.js +363 -0
  178. package/dist/store/postgres/store.d.ts +2 -4
  179. package/dist/store/postgres/store.js +590 -434
  180. package/dist/store/postgres/tenant.d.ts +56 -0
  181. package/dist/store/postgres/tenant.js +154 -0
  182. package/dist/store/types.d.ts +183 -132
  183. package/dist/store/types.js +0 -1
  184. package/dist/streams/basin.d.ts +18 -1
  185. package/dist/streams/basin.js +34 -2
  186. package/dist/streams/control.d.ts +18 -0
  187. package/dist/streams/control.js +33 -0
  188. package/dist/streams/memory.d.ts +3 -1
  189. package/dist/streams/memory.js +14 -8
  190. package/dist/streams/relay/core.d.ts +35 -0
  191. package/dist/streams/relay/core.js +366 -0
  192. package/dist/streams/relay/types.d.ts +78 -0
  193. package/dist/streams/relay/types.js +8 -0
  194. package/dist/streams/types.d.ts +71 -36
  195. package/dist/streams/types.js +18 -53
  196. package/dist/tenant/access-policy.d.ts +30 -0
  197. package/dist/tenant/access-policy.js +52 -0
  198. package/dist/tenant/actions.d.ts +10 -149
  199. package/dist/tenant/actions.js +5 -167
  200. package/dist/tenant/advance.d.ts +7 -2
  201. package/dist/tenant/advance.js +125 -14
  202. package/dist/tenant/auth.d.ts +19 -17
  203. package/dist/tenant/auth.js +90 -93
  204. package/dist/tenant/browser.d.ts +15 -0
  205. package/dist/tenant/browser.js +39 -0
  206. package/dist/tenant/cause.d.ts +25 -0
  207. package/dist/tenant/cause.js +45 -0
  208. package/dist/tenant/commands.d.ts +18 -2
  209. package/dist/tenant/commands.js +111 -115
  210. package/dist/tenant/context.d.ts +101 -21
  211. package/dist/tenant/context.js +14 -10
  212. package/dist/tenant/delivery-token.d.ts +41 -0
  213. package/dist/tenant/delivery-token.js +104 -0
  214. package/dist/tenant/delivery.d.ts +62 -0
  215. package/dist/tenant/delivery.js +481 -0
  216. package/dist/tenant/effects.d.ts +20 -1
  217. package/dist/tenant/effects.js +159 -42
  218. package/dist/tenant/endpoints.d.ts +67 -0
  219. package/dist/tenant/endpoints.js +90 -0
  220. package/dist/tenant/ephemeral.d.ts +22 -7
  221. package/dist/tenant/ephemeral.js +83 -125
  222. package/dist/tenant/history.d.ts +64 -0
  223. package/dist/tenant/history.js +120 -0
  224. package/dist/tenant/http.d.ts +8 -9
  225. package/dist/tenant/http.js +13 -27
  226. package/dist/tenant/jwt.d.ts +16 -0
  227. package/dist/tenant/jwt.js +50 -0
  228. package/dist/tenant/model-setting.d.ts +2 -2
  229. package/dist/tenant/module.d.ts +17 -28
  230. package/dist/tenant/module.js +133 -279
  231. package/dist/tenant/outbound.d.ts +44 -0
  232. package/dist/tenant/outbound.js +127 -0
  233. package/dist/tenant/paths.d.ts +11 -16
  234. package/dist/tenant/paths.js +24 -68
  235. package/dist/tenant/pool.d.ts +1 -3
  236. package/dist/tenant/pool.js +1 -19
  237. package/dist/tenant/principals.d.ts +23 -11
  238. package/dist/tenant/principals.js +61 -49
  239. package/dist/tenant/reset.d.ts +0 -4
  240. package/dist/tenant/reset.js +0 -5
  241. package/dist/tenant/runtime.d.ts +28 -20
  242. package/dist/tenant/runtime.js +95 -64
  243. package/dist/tenant/scheduler.d.ts +10 -3
  244. package/dist/tenant/scheduler.js +9 -5
  245. package/dist/tenant/session-streams.d.ts +127 -0
  246. package/dist/tenant/session-streams.js +404 -0
  247. package/dist/tenant/sessions.d.ts +28 -4
  248. package/dist/tenant/sessions.js +118 -24
  249. package/dist/tenant/signing-keys.d.ts +55 -0
  250. package/dist/tenant/signing-keys.js +172 -0
  251. package/dist/tenant/slim.d.ts +8 -0
  252. package/dist/tenant/slim.js +21 -0
  253. package/dist/tenant/status.d.ts +3 -3
  254. package/dist/tenant/status.js +20 -13
  255. package/dist/tenant/store-pg.d.ts +9 -6
  256. package/dist/tenant/store-pg.js +60 -125
  257. package/dist/tenant/streams.d.ts +62 -85
  258. package/dist/tenant/streams.js +143 -226
  259. package/dist/tenant/subject-limits.d.ts +16 -0
  260. package/dist/tenant/subject-limits.js +45 -0
  261. package/dist/tenant/sweep.d.ts +1 -10
  262. package/dist/tenant/sweep.js +29 -69
  263. package/dist/tenant/tokens.d.ts +14 -0
  264. package/dist/tenant/tokens.js +180 -0
  265. package/dist/tenant/transcript.d.ts +48 -5
  266. package/dist/tenant/transcript.js +58 -1
  267. package/dist/tenant/types.d.ts +69 -98
  268. package/dist/tenant/types.js +4 -15
  269. package/dist/tenant/worker.d.ts +3 -1
  270. package/dist/tenant/worker.js +13 -7
  271. package/dist/vault/host-model.d.ts +31 -0
  272. package/dist/vault/host-model.js +57 -0
  273. package/dist/vault/service.d.ts +44 -17
  274. package/dist/vault/service.js +126 -121
  275. package/dist/version.d.ts +1 -1
  276. package/dist/version.js +1 -1
  277. package/package.json +25 -9
  278. package/dist/core/executors.d.ts +0 -44
  279. package/dist/core/executors.js +0 -95
  280. package/dist/store/memory.d.ts +0 -57
  281. package/dist/store/memory.js +0 -645
  282. package/dist/store/postgres/migrations/001_initial.d.ts +0 -27
  283. package/dist/store/postgres/migrations/001_initial.js +0 -196
  284. package/dist/store/postgres/migrations/002_session_owner.d.ts +0 -6
  285. package/dist/store/postgres/migrations/002_session_owner.js +0 -13
  286. package/dist/store/postgres/migrations/index.d.ts +0 -49
  287. package/dist/store/postgres/migrations/index.js +0 -76
  288. package/dist/store/postgres/names.d.ts +0 -11
  289. package/dist/store/postgres/names.js +0 -42
  290. package/dist/store/postgres/tenants.d.ts +0 -90
  291. package/dist/store/postgres/tenants.js +0 -214
  292. package/dist/streams/relay.d.ts +0 -37
  293. package/dist/streams/relay.js +0 -254
  294. package/dist/tenant/envelope.d.ts +0 -7
  295. package/dist/tenant/envelope.js +0 -22
  296. package/dist/tenant/live.d.ts +0 -92
  297. package/dist/tenant/live.js +0 -290
  298. package/dist/tenant/quarantine-error.d.ts +0 -6
  299. package/dist/tenant/quarantine-error.js +0 -9
  300. package/dist/tenant/quarantine.d.ts +0 -27
  301. package/dist/tenant/quarantine.js +0 -72
  302. package/dist/tenant/routes-tenant.d.ts +0 -12
  303. package/dist/tenant/routes-tenant.js +0 -137
  304. package/dist/tenant/routes.d.ts +0 -9
  305. package/dist/tenant/routes.js +0 -151
  306. package/dist/tenant/store-memory.d.ts +0 -16
  307. package/dist/tenant/store-memory.js +0 -111
  308. /package/dist/{store → record}/cursor.d.ts +0 -0
  309. /package/dist/{store → record}/cursor.js +0 -0
@@ -1,3 +1,4 @@
1
+ import { parseBasin } from "./basin.js";
1
2
  /**
2
3
  * In-memory `DurableStreams` for unit tests and a local development Host.
3
4
  * Not durable: history does not survive a restart. Keeps the seam's
@@ -71,14 +72,19 @@ export class MemoryStreams {
71
72
  if (!this.tenants.has(tenantId))
72
73
  this.tenants.set(tenantId, new Map());
73
74
  }
74
- async deleteTenant(tenantId) {
75
- const streams = this.tenants.get(tenantId);
76
- this.tenants.delete(tenantId);
77
- for (const stream of streams?.keys() ?? [])
78
- this.notify(tenantId, stream);
79
- for (const key of this.waiters.keys())
80
- if (key.startsWith(`${tenantId}\u0000`))
81
- this.notifyKey(key);
75
+ async deleteTenant(tenantId, options = {}) {
76
+ const basins = options.allGenerations
77
+ ? [...this.tenants.keys()].filter((key) => parseBasin(key).tenantId === tenantId)
78
+ : [tenantId];
79
+ for (const basin of basins) {
80
+ const streams = this.tenants.get(basin);
81
+ this.tenants.delete(basin);
82
+ for (const stream of streams?.keys() ?? [])
83
+ this.notify(basin, stream);
84
+ for (const key of this.waiters.keys())
85
+ if (key.startsWith(`${basin}\u0000`))
86
+ this.notifyKey(key);
87
+ }
82
88
  }
83
89
  async deleteStream(tenantId, stream) {
84
90
  this.tenants.get(tenantId)?.delete(stream);
@@ -0,0 +1,35 @@
1
+ import { type DurableStreams } from "../types.js";
2
+ import type { ChangeSource, RecordReader } from "./types.js";
3
+ export interface StreamRelayOptions {
4
+ source: ChangeSource;
5
+ record: RecordReader;
6
+ streams: DurableStreams;
7
+ /** Relay events: lag, errors, reconciliations. */
8
+ log?: (message: string, fields?: Record<string, unknown>) => void;
9
+ /** Records per append (S2 allows 1000). */
10
+ maxBatchRecords?: number;
11
+ /** Bytes per append (S2 allows 1 MiB). */
12
+ maxBatchBytes?: number;
13
+ /** Sessions reconciled at once. */
14
+ reconcileConcurrency?: number;
15
+ }
16
+ export interface StreamRelayStatus {
17
+ /** This process holds the source (the replication slot). */
18
+ active: boolean;
19
+ /** Committed transactions not yet fully in S2. */
20
+ pendingTxs: number;
21
+ /** Rows queued for S2. */
22
+ pendingRows: number;
23
+ /** The last position acknowledged to the source. */
24
+ confirmed: string | null;
25
+ reconciliations: number;
26
+ lastError: string | null;
27
+ }
28
+ export interface StreamRelay {
29
+ start(): void;
30
+ stop(): Promise<void>;
31
+ status(): StreamRelayStatus;
32
+ /** Resolves once nothing is pending (tests and shutdown). */
33
+ idle(): Promise<void>;
34
+ }
35
+ export declare function createStreamRelay(options: StreamRelayOptions): StreamRelay;
@@ -0,0 +1,366 @@
1
+ /**
2
+ * The stream relay (Durable Streams §7): feeds S2 from the record, exactly once and in order
3
+ * per session.
4
+ *
5
+ * - **Input.** Committed transactions from a `ChangeSource`, in commit order. The source
6
+ * keeps each one until it is acknowledged, so a crash replays what S2 may not have.
7
+ * - **Pumps.** Each session has a queue and one pump. The pump appends the contiguous rows
8
+ * at the front of the queue as one batch to `sessions/<id>` in the row's basin generation,
9
+ * with `matchSeq` = the first row's `seq`:
10
+ * - `ok`: the rows are done;
11
+ * - tail past the batch: S2 already has those rows (a replay); they are done;
12
+ * - tail before the batch: S2 is missing rows (a lost stream, or rows the source never
13
+ * delivered); they are read from the record and put in front;
14
+ * - an error: the rows' generation is checked against the Tenant's (a reset or Tenant
15
+ * deletion makes them obsolete: done), the basin is ensured, and the batch is retried
16
+ * with backoff (50 ms to 2 s). The pump is per session, so order is kept.
17
+ * - **Acknowledgement.** A transaction is acknowledged once all its rows are done, in commit
18
+ * order, never past one that is not.
19
+ * - **Reconciliation.** When the source starts fresh (a new or lost slot), every session's S2
20
+ * tail is compared with its log head and the difference is re-sent from the record.
21
+ * `matchSeq` makes overlap with the live input harmless.
22
+ *
23
+ * The relay never writes the record and never deletes from S2.
24
+ */
25
+ import { basinOf } from "../basin.js";
26
+ import { sessionStream } from "../types.js";
27
+ const RETRY_MIN_MS = 50;
28
+ const RETRY_MAX_MS = 2000;
29
+ const HEADS_PAGE = 500;
30
+ export function createStreamRelay(options) {
31
+ const { source, record, streams } = options;
32
+ const log = options.log ?? (() => { });
33
+ const maxRecords = options.maxBatchRecords ?? 500;
34
+ const maxBytes = options.maxBatchBytes ?? 768 * 1024;
35
+ const concurrency = options.reconcileConcurrency ?? 8;
36
+ /** Committed transactions not yet fully in S2, in commit order. */
37
+ const txs = [];
38
+ const sessions = new Map();
39
+ let confirmed = null;
40
+ let active = false;
41
+ let stopped = false;
42
+ let reconciliations = 0;
43
+ let lastError = null;
44
+ let idleWaiters = [];
45
+ /** Woken when queued rows settle or the activation ends (`reconcile` waits on its rows). */
46
+ let progressWaiters = [];
47
+ /** Moves on with every activation and every loss of the source. */
48
+ let activation = 0;
49
+ const stopping = new AbortController();
50
+ const keyOf = (tenantId, sessionId) => `${tenantId}\u0000${sessionId}`;
51
+ function queueOf(tenantId, sessionId) {
52
+ const key = keyOf(tenantId, sessionId);
53
+ let queue = sessions.get(key);
54
+ if (!queue)
55
+ sessions.set(key, (queue = { tenantId, sessionId, rows: [], running: false }));
56
+ return queue;
57
+ }
58
+ function queued(row, tx) {
59
+ const body = JSON.stringify(row.body) ?? "null";
60
+ return { ...row, seq: Number(row.seq), tx, bytes: Buffer.byteLength(body) };
61
+ }
62
+ function onTx(committed) {
63
+ if (stopped)
64
+ return;
65
+ const tx = { endLsn: committed.endLsn, remaining: committed.rows.length };
66
+ txs.push(tx);
67
+ // Queue every row before pumping, so a transaction's rows go out in as few batches as fit.
68
+ const touched = new Set();
69
+ for (const row of committed.rows) {
70
+ const queue = queueOf(row.tenantId, row.sessionId);
71
+ queue.rows.push(queued(row, tx));
72
+ touched.add(queue);
73
+ }
74
+ for (const queue of touched)
75
+ pumpSoon(queue);
76
+ advance();
77
+ }
78
+ function pumpSoon(queue) {
79
+ if (!queue.running)
80
+ void pump(queue);
81
+ }
82
+ function done(row) {
83
+ row.tx.remaining -= 1;
84
+ }
85
+ /** The contiguous rows at the front of the queue, within the batch limits. */
86
+ function takeBatch(rows) {
87
+ const batch = [rows[0]];
88
+ let bytes = rows[0].bytes;
89
+ for (let i = 1; i < rows.length && batch.length < maxRecords; i += 1) {
90
+ const row = rows[i];
91
+ const previous = batch[batch.length - 1];
92
+ if (row.seq !== previous.seq + 1 || row.generation !== previous.generation)
93
+ break;
94
+ if (bytes + row.bytes > maxBytes)
95
+ break;
96
+ bytes += row.bytes;
97
+ batch.push(row);
98
+ }
99
+ return batch;
100
+ }
101
+ async function pump(queue) {
102
+ queue.running = true;
103
+ let delay = RETRY_MIN_MS;
104
+ try {
105
+ while (queue.rows.length > 0 && !stopped) {
106
+ // Duplicates (a replay overlapping a refill) and stale rows sort out here.
107
+ queue.rows.sort((a, b) => a.generation - b.generation || a.seq - b.seq);
108
+ const batch = takeBatch(queue.rows);
109
+ const first = batch[0];
110
+ const basin = basinOf(first.tenantId, first.generation);
111
+ try {
112
+ const result = await streams.append(basin, sessionStream(first.sessionId), batch.map((row) => row.body), { matchSeq: first.seq });
113
+ if (result.status === "ok") {
114
+ settle(queue, (row) => batch.includes(row));
115
+ }
116
+ else if (result.tail > first.seq) {
117
+ // A replay: S2 already has every row below its tail.
118
+ settle(queue, (row) => row.generation === first.generation && row.seq < result.tail);
119
+ }
120
+ else {
121
+ // S2 is behind the record: re-send the gap first.
122
+ await refill(queue, first.generation, result.tail, first.seq);
123
+ }
124
+ delay = RETRY_MIN_MS;
125
+ lastError = null;
126
+ }
127
+ catch (error) {
128
+ if (stopped)
129
+ return;
130
+ lastError = messageOf(error);
131
+ if (await obsolete(first)) {
132
+ settle(queue, (row) => row.generation <= first.generation);
133
+ continue;
134
+ }
135
+ log("stream relay append failed; retrying", {
136
+ tenantId: first.tenantId,
137
+ sessionId: first.sessionId,
138
+ seq: first.seq,
139
+ retryInMs: delay,
140
+ message: lastError,
141
+ });
142
+ // The basin may be missing (S2 lost it, or it was never created).
143
+ await streams.ensureTenant(basin).catch(() => undefined);
144
+ await sleep(delay, stopping.signal);
145
+ delay = Math.min(delay * 2, RETRY_MAX_MS);
146
+ }
147
+ }
148
+ }
149
+ finally {
150
+ queue.running = false;
151
+ if (queue.rows.length === 0)
152
+ sessions.delete(keyOf(queue.tenantId, queue.sessionId));
153
+ else if (!stopped)
154
+ pumpSoon(queue);
155
+ advance();
156
+ }
157
+ }
158
+ /** Marks the queue's rows matching `match` done and drops them. */
159
+ function settle(queue, match) {
160
+ const keep = [];
161
+ for (const row of queue.rows) {
162
+ if (match(row))
163
+ done(row);
164
+ else
165
+ keep.push(row);
166
+ }
167
+ queue.rows = keep;
168
+ progressed();
169
+ }
170
+ function progressed() {
171
+ const waiters = progressWaiters;
172
+ progressWaiters = [];
173
+ for (const wake of waiters)
174
+ wake();
175
+ }
176
+ /** The rows' generation is no longer the Tenant's (a reset), or the Tenant is gone. */
177
+ async function obsolete(row) {
178
+ try {
179
+ const current = await record.generation(row.tenantId);
180
+ return current === undefined || current > row.generation;
181
+ }
182
+ catch {
183
+ return false;
184
+ }
185
+ }
186
+ /**
187
+ * Puts record rows `[from, to)` in front of the queue (not part of the ack order). Returns
188
+ * their pending count, which reaches 0 once they are all in S2.
189
+ */
190
+ async function refill(queue, generation, from, to) {
191
+ const rows = await record.readRange(queue.tenantId, queue.sessionId, from, to);
192
+ const own = rows.filter((row) => row.generation === generation);
193
+ if (own.length < to - from) {
194
+ // The record no longer has them (a reset deleted the session): nothing to re-send.
195
+ if (await obsolete({ ...queue, seq: from, generation, body: null })) {
196
+ settle(queue, (row) => row.generation <= generation);
197
+ return undefined;
198
+ }
199
+ throw new Error(`The record is missing rows ${from}..${to - 1} of session ${queue.sessionId}`);
200
+ }
201
+ const tx = { endLsn: undefined, remaining: own.length };
202
+ log("stream relay refilling a session from the record", {
203
+ tenantId: queue.tenantId,
204
+ sessionId: queue.sessionId,
205
+ from,
206
+ to,
207
+ });
208
+ queue.rows.unshift(...own.map((row) => queued(row, tx)));
209
+ return tx;
210
+ }
211
+ /** Acknowledges the newest transaction whose rows, and all before it, are in S2. */
212
+ function advance() {
213
+ let lsn;
214
+ while (txs.length > 0 && txs[0].remaining <= 0)
215
+ lsn = txs.shift().endLsn ?? lsn;
216
+ if (lsn)
217
+ acknowledge(lsn);
218
+ if (txs.length === 0 && sessions.size === 0) {
219
+ const waiters = idleWaiters;
220
+ idleWaiters = [];
221
+ for (const wake of waiters)
222
+ wake();
223
+ }
224
+ }
225
+ function acknowledge(lsn) {
226
+ confirmed = lsn;
227
+ source.acknowledge(lsn);
228
+ }
229
+ /**
230
+ * Re-sends, for every session, what the record has beyond its S2 tail, and resolves true
231
+ * once all of it is in S2. False when activation `of` ended first (the source was lost, so
232
+ * the queued rows were dropped) or the relay stopped: the slot must stay unreconciled.
233
+ */
234
+ async function reconcile(of) {
235
+ reconciliations += 1;
236
+ const started = Date.now();
237
+ const current = () => !stopped && activation === of;
238
+ let sessionsChecked = 0;
239
+ let resent = 0;
240
+ const refilled = [];
241
+ let after;
242
+ for (;;) {
243
+ if (!current())
244
+ return false;
245
+ const page = await record.heads(after, HEADS_PAGE);
246
+ if (page.length === 0)
247
+ break;
248
+ let next = 0;
249
+ await Promise.all(Array.from({ length: Math.min(concurrency, page.length) }, async () => {
250
+ while (next < page.length && current()) {
251
+ const head = page[next++];
252
+ sessionsChecked += 1;
253
+ const tx = await reconcileOne(head);
254
+ if (tx) {
255
+ refilled.push(tx);
256
+ resent += tx.remaining;
257
+ }
258
+ }
259
+ }));
260
+ const last = page[page.length - 1];
261
+ after = { tenantId: last.tenantId, sessionId: last.sessionId };
262
+ }
263
+ // Queued is not delivered: wait until S2 has every re-sent row.
264
+ while (current() && refilled.some((tx) => tx.remaining > 0))
265
+ await new Promise((resolve) => progressWaiters.push(resolve));
266
+ if (!current())
267
+ return false;
268
+ log("stream relay reconciled the record with S2", {
269
+ sessions: sessionsChecked,
270
+ resent,
271
+ durationMs: Date.now() - started,
272
+ });
273
+ return true;
274
+ }
275
+ async function reconcileOne(head) {
276
+ const current = await record.generation(head.tenantId);
277
+ if (current === undefined || current !== head.generation)
278
+ return undefined;
279
+ const basin = basinOf(head.tenantId, head.generation);
280
+ await streams.ensureTenant(basin);
281
+ const tail = await streams.tail(basin, sessionStream(head.sessionId));
282
+ if (tail >= head.head)
283
+ return undefined;
284
+ const queue = queueOf(head.tenantId, head.sessionId);
285
+ const tx = await refill(queue, head.generation, tail, head.head);
286
+ pumpSoon(queue);
287
+ return tx;
288
+ }
289
+ return {
290
+ start() {
291
+ source.start({
292
+ onActive({ fresh }) {
293
+ active = true;
294
+ const of = (activation += 1);
295
+ log("stream relay active", { fresh });
296
+ if (fresh)
297
+ void reconcile(of)
298
+ .then((complete) => (complete ? source.reconciled() : undefined))
299
+ .catch((error) => {
300
+ lastError = messageOf(error);
301
+ log("stream relay reconciliation failed", { message: lastError });
302
+ });
303
+ },
304
+ onTx,
305
+ onKeepalive(lsn) {
306
+ // Caught up: let the source move past WAL that holds none of our rows.
307
+ if (txs.length === 0) {
308
+ confirmed = lsn;
309
+ return lsn;
310
+ }
311
+ return undefined;
312
+ },
313
+ onInactive(error) {
314
+ active = false;
315
+ activation += 1;
316
+ // The source replays everything not acknowledged; what is queued is dropped.
317
+ txs.length = 0;
318
+ for (const queue of sessions.values())
319
+ queue.rows = [];
320
+ progressed();
321
+ if (error) {
322
+ lastError = messageOf(error);
323
+ log("stream relay lost its source; retrying", { message: lastError });
324
+ }
325
+ },
326
+ });
327
+ },
328
+ async stop() {
329
+ stopped = true;
330
+ stopping.abort();
331
+ progressed();
332
+ await source.stop();
333
+ const waiters = idleWaiters;
334
+ idleWaiters = [];
335
+ for (const wake of waiters)
336
+ wake();
337
+ },
338
+ status: () => ({
339
+ active,
340
+ pendingTxs: txs.length,
341
+ pendingRows: [...sessions.values()].reduce((n, queue) => n + queue.rows.length, 0),
342
+ confirmed,
343
+ reconciliations,
344
+ lastError,
345
+ }),
346
+ idle() {
347
+ if (txs.length === 0 && sessions.size === 0)
348
+ return Promise.resolve();
349
+ return new Promise((resolve) => idleWaiters.push(resolve));
350
+ },
351
+ };
352
+ }
353
+ const messageOf = (error) => (error instanceof Error ? error.message : String(error));
354
+ function sleep(ms, signal) {
355
+ return new Promise((resolve) => {
356
+ if (signal.aborted)
357
+ return resolve();
358
+ const done = () => {
359
+ clearTimeout(timer);
360
+ signal.removeEventListener("abort", done);
361
+ resolve();
362
+ };
363
+ const timer = setTimeout(done, ms);
364
+ signal.addEventListener("abort", done, { once: true });
365
+ });
366
+ }
@@ -0,0 +1,78 @@
1
+ /**
2
+ * Contracts of the stream relay (Durable Streams §7): what it reads committed events from,
3
+ * and how it reads the record back to refill and reconcile. Postgres implements both with
4
+ * logical replication (`adapters/replication/pgoutput.ts`) and the record tables
5
+ * (`store/postgres/record.ts`); the relay core's unit tests use an in-memory pair
6
+ * (`test/support/relay-record.ts`).
7
+ */
8
+ /** One committed row of the record: a session event and the basin generation it goes to. */
9
+ export interface RecordRow {
10
+ tenantId: string;
11
+ sessionId: string;
12
+ seq: number;
13
+ generation: number;
14
+ /** The event, as stored. */
15
+ body: unknown;
16
+ }
17
+ /** One committed transaction's record rows, in commit order. */
18
+ export interface CommittedTx {
19
+ /** Where the source resumes after this transaction once it is acknowledged. */
20
+ readonly endLsn: string;
21
+ readonly rows: readonly RecordRow[];
22
+ }
23
+ export interface ChangeHandlers {
24
+ /**
25
+ * This process now reads the source (the slot is ours). `fresh` while the source has no
26
+ * history to replay for rows committed before it existed (a new or lost slot): the record
27
+ * must be reconciled with S2, then `reconciled()` called.
28
+ */
29
+ onActive(state: {
30
+ fresh: boolean;
31
+ }): void;
32
+ /** A committed transaction, in commit order. Replayed after a restart until acknowledged. */
33
+ onTx(tx: CommittedTx): void;
34
+ /**
35
+ * The source asks how far it may move on: return the position to acknowledge (the
36
+ * server's `lsn` when nothing is pending), or undefined to acknowledge nothing new.
37
+ */
38
+ onKeepalive(lsn: string): string | undefined;
39
+ /** This process stopped reading the source (connection lost); it tries again. */
40
+ onInactive(error?: unknown): void;
41
+ }
42
+ /**
43
+ * Committed record rows in commit order, kept by the source until acknowledged: a
44
+ * transaction not acknowledged before a crash is delivered again.
45
+ */
46
+ export interface ChangeSource {
47
+ /** Starts reading; keeps trying (another process may hold the source) until `stop`. */
48
+ start(handlers: ChangeHandlers): void;
49
+ /** Everything up to `lsn` is delivered; the source may forget it. */
50
+ acknowledge(lsn: string): void;
51
+ /**
52
+ * The relay reconciled the record after a fresh start. Until this is called, every process
53
+ * that becomes active is told `fresh` again (the reconciliation may have been cut short).
54
+ */
55
+ reconciled(): Promise<void>;
56
+ stop(): Promise<void>;
57
+ /** How far the source is behind, in bytes, when known. */
58
+ lag?(): Promise<number | undefined>;
59
+ }
60
+ /** A session's log head: the next `seq` to allocate. */
61
+ export interface LogHead {
62
+ tenantId: string;
63
+ sessionId: string;
64
+ generation: number;
65
+ head: number;
66
+ }
67
+ /** Reads the record back (refill and reconciliation). */
68
+ export interface RecordReader {
69
+ /** Rows `[from, to)` of a session, in `seq` order. */
70
+ readRange(tenantId: string, sessionId: string, from: number, to: number): Promise<RecordRow[]>;
71
+ /** Log heads with `head > 0`, in (Tenant, session) order, after `after`, at most `limit`. */
72
+ heads(after: {
73
+ tenantId: string;
74
+ sessionId: string;
75
+ } | undefined, limit: number): Promise<LogHead[]>;
76
+ /** The Tenant's current basin generation; undefined when the Tenant is gone. */
77
+ generation(tenantId: string): Promise<number | undefined>;
78
+ }
@@ -0,0 +1,8 @@
1
+ /**
2
+ * Contracts of the stream relay (Durable Streams §7): what it reads committed events from,
3
+ * and how it reads the record back to refill and reconcile. Postgres implements both with
4
+ * logical replication (`adapters/replication/pgoutput.ts`) and the record tables
5
+ * (`store/postgres/record.ts`); the relay core's unit tests use an in-memory pair
6
+ * (`test/support/relay-record.ts`).
7
+ */
8
+ export {};
@@ -1,3 +1,43 @@
1
+ /**
2
+ * The Durable Streams seam (Durable Streams §8).
3
+ *
4
+ * Durable Streams deliver what the record holds. Each Tenant has one S2 basin per basin
5
+ * generation (`streams/basin.ts`); inside it:
6
+ *
7
+ * | Stream | Contents | Written by | Read by |
8
+ * | --- | --- | --- | --- |
9
+ * | `sessions/<sessionId>` | every event of the session, in sequence | the stream relay, from the record | history and session SSE |
10
+ * | `tenant/control` | `session.cancel`, `sessions.reset` and `subject.revoked` signals | API nodes | every process with the Tenant open |
11
+ *
12
+ * **Basin generations.** A session id is unique within its Tenant's basin generation. A
13
+ * Tenant reset, the only path that frees ids, moves the Tenant to the next generation, so
14
+ * a session created again with the same id starts in an empty basin at sequence 0; the old
15
+ * basin is deleted afterwards. The cursor is `base64url("<sessionId>:<seq>")`.
16
+ *
17
+ * S2 is the supported implementation (`adapters/streams/s2.ts`, the only file
18
+ * importing the S2 SDK); `streams/memory.ts` is the in-memory fake.
19
+ *
20
+ * ## Guarantees every implementation keeps
21
+ *
22
+ * - **Sequence numbers.** Each stream numbers its records from 0 without gaps.
23
+ * The tail is the sequence the next appended record gets (0 for a stream
24
+ * that does not exist yet).
25
+ * - **Conditional appends.** With `matchSeq`, a batch is appended only when the
26
+ * tail equals `matchSeq`; otherwise nothing is written and the result is a
27
+ * `SeqMismatch` carrying the current tail. The relay appends a session
28
+ * stream with `matchSeq` set to the record row's sequence, so a retried
29
+ * append after an unacknowledged success is detected and never duplicated.
30
+ * - **Batches are atomic.** A batch is appended entirely or not at all.
31
+ * - **Reads resume.** `read` from `fromSeq` yields every record with
32
+ * `seq >= fromSeq` in order, with no gap between history and the live tail.
33
+ * - **Tenant scope.** Methods take a Tenant id, or a basin key from `basinOf`
34
+ * for a later generation. `ensureTenant` creates the basin (idempotent) and
35
+ * must run before appends; streams are created on first append or read.
36
+ * `deleteTenant` removes the basin and every stream in it (with
37
+ * `allGenerations`, every generation's basin of the Tenant).
38
+ * - **Records are JSON.** Bodies are JSON-serializable values; implementations
39
+ * encode them as they need.
40
+ */
1
41
  /** A record as read from a stream. */
2
42
  export interface StreamRecord<T = unknown> {
3
43
  seq: number;
@@ -48,9 +88,14 @@ export interface DurableStreams {
48
88
  tail(tenantId: string, stream: string): Promise<number>;
49
89
  /** Creates the Tenant's basin. Idempotent. */
50
90
  ensureTenant(tenantId: string): Promise<void>;
51
- /** Deletes the Tenant's basin and all its streams. Idempotent. */
52
- deleteTenant(tenantId: string): Promise<void>;
53
- /** Deletes one stream (an abandoned session incarnation). Idempotent. */
91
+ /**
92
+ * Deletes the basin and all its streams. Idempotent. With `allGenerations`, given a Tenant
93
+ * id, also every later generation's basin (Tenant deletion).
94
+ */
95
+ deleteTenant(tenantId: string, options?: {
96
+ allGenerations?: boolean;
97
+ }): Promise<void>;
98
+ /** Deletes one stream. Idempotent. */
54
99
  deleteStream(tenantId: string, stream: string): Promise<void>;
55
100
  /**
56
101
  * Names of the Tenant's streams starting with `prefix`, in name order,
@@ -66,50 +111,40 @@ export interface DurableStreams {
66
111
  probe?(signal: AbortSignal): Promise<void>;
67
112
  }
68
113
  export declare const SESSION_STREAM_PREFIX = "sessions/";
69
- /** `work_available` signals for executor connections. */
70
- export declare const WORK_STREAM = "tenant/work";
71
114
  /** `session.cancel` and `sessions.reset` signals for every process with the Tenant open. */
72
115
  export declare const CONTROL_STREAM = "tenant/control";
116
+ /** The stream of a session: `sessions/<sessionId>`. */
117
+ export declare function sessionStream(sessionId: string): string;
73
118
  /**
74
- * The incarnation of a session document written before incarnations existed.
75
- * Random incarnations are 12 characters, so never this.
76
- */
77
- export declare const LEGACY_INCARNATION = "0";
78
- /** A new session incarnation: 12 random base64url characters. */
79
- export declare function newStreamIncarnation(): string;
80
- /** The stream of one incarnation of a session: `sessions/<sessionId>/<incarnation>`. */
81
- export declare function sessionStream(sessionId: string, incarnation: string): string;
82
- /** The fields of a session document that name its stream. */
83
- export interface SessionStreamRef {
84
- id: string;
85
- /** Set when the session is created (`newStreamIncarnation`); never changed. */
86
- streamIncarnation?: string;
87
- }
88
- /** The stream of a session as stored now. */
89
- export declare function streamOfSession(session: SessionStreamRef): string;
90
- /** The session id and incarnation of a session stream, or undefined for other streams. */
91
- export declare function parseSessionStream(stream: string): {
92
- sessionId: string;
93
- incarnation: string;
94
- } | undefined;
95
- /**
96
- * Signals are not canonical events: they are appended without the outbox, and
97
- * a lost signal costs latency, not correctness.
119
+ * Signals are not events: they are appended directly, never recorded, and a
120
+ * lost signal costs latency, not correctness.
98
121
  */
99
- export interface WorkSignal {
100
- type: "work_available";
101
- }
102
122
  /** Ends the advance of `sessionId` on the process running it. */
103
123
  export interface SessionCancelSignal {
104
124
  type: "session.cancel";
105
125
  sessionId: string;
126
+ /**
127
+ * The cancelled turn. Only an advance of this turn stops, so a signal delivered late (an
128
+ * append retried after S2 returns) never stops a later turn. Absent: any advance stops.
129
+ */
130
+ turnId?: string;
106
131
  }
107
132
  /**
108
- * Session streams were abandoned (a Tenant reset): each process checks its
109
- * session feeds and ends those whose session is gone or has a new incarnation.
133
+ * The Tenant was reset and moved to basin generation `generation`. Appended to the old
134
+ * generation's `tenant/control`: each process switches its readers to the new basin and
135
+ * ends the streams of sessions the reset deleted.
110
136
  */
111
137
  export interface SessionsResetSignal {
112
138
  type: "sessions.reset";
139
+ generation?: number;
140
+ }
141
+ /**
142
+ * A subject's tokens older than `epoch` were revoked: each process ends that subject's
143
+ * streams opened with them.
144
+ */
145
+ export interface SubjectRevokedSignal {
146
+ type: "subject.revoked";
147
+ subject: string;
148
+ epoch: number;
113
149
  }
114
- export type ControlSignal = SessionCancelSignal | SessionsResetSignal;
115
- export declare const WORK_AVAILABLE: Readonly<WorkSignal>;
150
+ export type ControlSignal = SessionCancelSignal | SessionsResetSignal | SubjectRevokedSignal;