@nylorun/runtime 0.22.0-beta → 0.24.0-beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (273) hide show
  1. package/CHANGELOG.md +161 -0
  2. package/README.md +95 -30
  3. package/dist/adapters/execution/restate.d.ts +15 -3
  4. package/dist/adapters/execution/restate.js +78 -84
  5. package/dist/adapters/replication/pgoutput.d.ts +2 -0
  6. package/dist/adapters/replication/pgoutput.js +6 -3
  7. package/dist/adapters/sandbox/local.d.ts +9 -1
  8. package/dist/adapters/sandbox/local.js +16 -4
  9. package/dist/adapters/sandbox/virtual.js +4 -1
  10. package/dist/adapters/streams/s2.js +1 -13
  11. package/dist/api/a2a/endpoint.js +5 -1
  12. package/dist/api/a2a/routes.d.ts +2 -0
  13. package/dist/api/a2a/routes.js +2 -2
  14. package/dist/api/a2a/tasks.js +2 -1
  15. package/dist/api/ag-ui/endpoint.js +4 -2
  16. package/dist/api/ag-ui/history.js +4 -1
  17. package/dist/api/ag-ui/routes.d.ts +4 -1
  18. package/dist/api/ag-ui/routes.js +11 -10
  19. package/dist/api/ag-ui/translate.js +3 -2
  20. package/dist/api/components.d.ts +4953 -1204
  21. package/dist/api/components.js +25 -16
  22. package/dist/api/gate/keys.js +7 -1
  23. package/dist/api/gate/routes.d.ts +6 -0
  24. package/dist/api/gate/routes.js +39 -3
  25. package/dist/api/http/app.js +4 -4
  26. package/dist/api/http/define.d.ts +6 -0
  27. package/dist/api/http/define.js +13 -1
  28. package/dist/api/http/respond.js +1 -1
  29. package/dist/api/http/routes/artifacts.js +35 -8
  30. package/dist/api/http/routes/coverage.d.ts +9 -0
  31. package/dist/api/http/routes/coverage.js +34 -0
  32. package/dist/api/http/routes/environments.d.ts +11 -0
  33. package/dist/api/http/routes/environments.js +283 -0
  34. package/dist/api/http/routes/mcp.d.ts +1 -1
  35. package/dist/api/http/routes/mcp.js +2 -2
  36. package/dist/api/http/routes/me.js +1 -1
  37. package/dist/api/http/routes/reads.d.ts +45 -9
  38. package/dist/api/http/routes/reads.js +54 -5
  39. package/dist/api/http/routes/sessions.js +11 -6
  40. package/dist/api/http/routes/tenant.js +32 -10
  41. package/dist/api/openapi.js +31 -26
  42. package/dist/artifacts/disk.d.ts +59 -0
  43. package/dist/artifacts/disk.js +563 -0
  44. package/dist/artifacts/export.d.ts +1 -1
  45. package/dist/artifacts/export.js +2 -3
  46. package/dist/artifacts/folders.d.ts +6 -3
  47. package/dist/artifacts/folders.js +17 -12
  48. package/dist/artifacts/service.d.ts +2 -1
  49. package/dist/artifacts/service.js +6 -1
  50. package/dist/artifacts/workspace.d.ts +1 -1
  51. package/dist/artifacts/workspace.js +1 -1
  52. package/dist/core/flow-host.d.ts +3 -22
  53. package/dist/core/flow-host.js +11 -37
  54. package/dist/core/provider.d.ts +3 -2
  55. package/dist/core/provider.js +5 -37
  56. package/dist/core/sandbox-routes.d.ts +2 -21
  57. package/dist/core/sandbox-routes.js +10 -43
  58. package/dist/execution/types.d.ts +11 -9
  59. package/dist/execution/types.js +5 -7
  60. package/dist/gates/contract.d.ts +3 -24
  61. package/dist/gates/contract.js +1 -36
  62. package/dist/gates/egress-credentials.d.ts +9 -0
  63. package/dist/gates/egress-credentials.js +11 -4
  64. package/dist/gates/egress.d.ts +28 -1
  65. package/dist/gates/egress.js +115 -10
  66. package/dist/gates/fixture-model.d.ts +28 -0
  67. package/dist/gates/fixture-model.js +92 -0
  68. package/dist/gates/handler.js +4 -4
  69. package/dist/gates/http-tool.d.ts +6 -2
  70. package/dist/gates/http-tool.js +3 -5
  71. package/dist/gates/in-process.d.ts +5 -0
  72. package/dist/gates/in-process.js +12 -1
  73. package/dist/gates/journal.d.ts +30 -0
  74. package/dist/gates/journal.js +52 -0
  75. package/dist/gates/mcp-handler.d.ts +4 -0
  76. package/dist/gates/mcp-handler.js +5 -1
  77. package/dist/gates/meter.d.ts +10 -2
  78. package/dist/gates/meter.js +20 -5
  79. package/dist/gates/policy.d.ts +49 -0
  80. package/dist/gates/policy.js +134 -0
  81. package/dist/gates/tenant-vaults.d.ts +7 -7
  82. package/dist/gates/tenant-vaults.js +5 -5
  83. package/dist/gates/tool-calls.d.ts +10 -5
  84. package/dist/gates/tool-calls.js +28 -10
  85. package/dist/gates/tool-client.d.ts +8 -0
  86. package/dist/gates/tool-client.js +27 -11
  87. package/dist/gates/tool-contract.d.ts +10 -68
  88. package/dist/gates/tool-contract.js +1 -85
  89. package/dist/gates/tool-gate.d.ts +1 -1
  90. package/dist/harness/calls.d.ts +12 -5
  91. package/dist/harness/calls.js +21 -12
  92. package/dist/harness/executors.js +12 -9
  93. package/dist/harness/main.js +18 -5
  94. package/dist/harness/pod.d.ts +34 -0
  95. package/dist/harness/pod.js +41 -0
  96. package/dist/harness/service.d.ts +0 -1
  97. package/dist/harness/service.js +13 -5
  98. package/dist/harness/setup.d.ts +27 -0
  99. package/dist/harness/setup.js +150 -0
  100. package/dist/harness/ws-client.js +3 -2
  101. package/dist/harness/ws-port.d.ts +1 -5
  102. package/dist/harness/ws-port.js +1 -5
  103. package/dist/harness-api/in-process.js +3 -3
  104. package/dist/harness-api/record.d.ts +15 -1
  105. package/dist/harness-api/record.js +174 -44
  106. package/dist/harness-api/redact.d.ts +3 -0
  107. package/dist/harness-api/redact.js +45 -0
  108. package/dist/harness-api/server.d.ts +8 -0
  109. package/dist/harness-api/server.js +93 -13
  110. package/dist/harness-api/start.d.ts +10 -3
  111. package/dist/harness-api/start.js +40 -10
  112. package/dist/harness-api/tool-journal.d.ts +96 -0
  113. package/dist/harness-api/tool-journal.js +102 -0
  114. package/dist/harness-api/tool-status.d.ts +13 -0
  115. package/dist/harness-api/tool-status.js +26 -0
  116. package/dist/harness-api/workspace.d.ts +6 -3
  117. package/dist/harness-api/workspace.js +27 -11
  118. package/dist/harness-api/ws-server.d.ts +11 -3
  119. package/dist/harness-api/ws-server.js +94 -15
  120. package/dist/host/execution.d.ts +8 -6
  121. package/dist/host/gates.d.ts +2 -1
  122. package/dist/host/gates.js +10 -2
  123. package/dist/host/main.js +64 -21
  124. package/dist/host/operate.js +67 -5
  125. package/dist/host/stack-config.d.ts +70 -0
  126. package/dist/host/stack-config.js +67 -1
  127. package/dist/infra/execution.d.ts +18 -1
  128. package/dist/infra/execution.js +28 -4
  129. package/dist/infra/index.d.ts +1 -1
  130. package/dist/keys/client.js +4 -1
  131. package/dist/keys/keys.d.ts +5 -5
  132. package/dist/keys/keys.js +5 -7
  133. package/dist/management-openapi.json +999 -2211
  134. package/dist/mcp/connect.d.ts +25 -20
  135. package/dist/mcp/connect.js +29 -30
  136. package/dist/mcp/pool.d.ts +2 -2
  137. package/dist/mcp/pool.js +1 -1
  138. package/dist/mcp/preview.d.ts +3 -3
  139. package/dist/mcp/preview.js +10 -20
  140. package/dist/mcp/snapshot.d.ts +5 -11
  141. package/dist/mcp/snapshot.js +6 -7
  142. package/dist/model/pi-model.js +8 -0
  143. package/dist/openapi.json +6955 -2526
  144. package/dist/reads/types.d.ts +16 -3
  145. package/dist/record/append.d.ts +3 -1
  146. package/dist/record/append.js +1 -0
  147. package/dist/record/envelope.d.ts +5 -1
  148. package/dist/record/envelope.js +7 -3
  149. package/dist/record/environment.d.ts +25 -0
  150. package/dist/record/environment.js +44 -0
  151. package/dist/record/index.d.ts +1 -1
  152. package/dist/record/index.js +1 -1
  153. package/dist/retry.d.ts +21 -0
  154. package/dist/retry.js +34 -0
  155. package/dist/sandbox/disk-fs.d.ts +95 -0
  156. package/dist/sandbox/disk-fs.js +206 -0
  157. package/dist/sandbox/join.d.ts +17 -18
  158. package/dist/sandbox/join.js +46 -20
  159. package/dist/sandbox/manager.d.ts +16 -8
  160. package/dist/sandbox/manager.js +33 -17
  161. package/dist/sandbox/placement.d.ts +13 -12
  162. package/dist/sandbox/placement.js +29 -12
  163. package/dist/sandbox/pods/client.d.ts +6 -1
  164. package/dist/sandbox/pods/lifecycle.d.ts +28 -27
  165. package/dist/sandbox/pods/lifecycle.js +14 -26
  166. package/dist/sandbox/pods/reconcile.js +21 -15
  167. package/dist/sandbox/pods/spec.d.ts +15 -13
  168. package/dist/sandbox/pods/spec.js +16 -20
  169. package/dist/sandbox/records.d.ts +2 -2
  170. package/dist/sandbox/records.js +3 -3
  171. package/dist/sandbox/resolve.d.ts +35 -27
  172. package/dist/sandbox/resolve.js +66 -41
  173. package/dist/sandbox/session-sandbox.js +1 -1
  174. package/dist/sandbox/setup.d.ts +61 -0
  175. package/dist/sandbox/setup.js +123 -0
  176. package/dist/sandbox/share.d.ts +11 -21
  177. package/dist/sandbox/share.js +12 -41
  178. package/dist/sandbox/tenant-config.d.ts +12 -14
  179. package/dist/sandbox/tenant-config.js +12 -17
  180. package/dist/sandbox/types.d.ts +6 -0
  181. package/dist/store/postgres/control.js +8 -1
  182. package/dist/store/postgres/drizzle/0018_wake_outbox_retries.sql +3 -0
  183. package/dist/store/postgres/drizzle/0019_sandbox_signal_outbox.sql +18 -0
  184. package/dist/store/postgres/drizzle/0020_environments.sql +63 -0
  185. package/dist/store/postgres/drizzle/0021_disk_versions.sql +7 -0
  186. package/dist/store/postgres/drizzle/0022_model_usage_timing.sql +5 -0
  187. package/dist/store/postgres/drizzle/0023_artifact_pages.sql +1 -0
  188. package/dist/store/postgres/drizzle/meta/_journal.json +42 -0
  189. package/dist/store/postgres/reads.js +117 -17
  190. package/dist/store/postgres/record-writer.d.ts +4 -4
  191. package/dist/store/postgres/record-writer.js +9 -9
  192. package/dist/store/postgres/record.d.ts +7 -0
  193. package/dist/store/postgres/record.js +23 -2
  194. package/dist/store/postgres/schema.d.ts +1213 -779
  195. package/dist/store/postgres/schema.js +136 -76
  196. package/dist/store/postgres/store.js +283 -117
  197. package/dist/store/types.d.ts +206 -88
  198. package/dist/streams/relay/core.js +1 -13
  199. package/dist/tenant/advance.d.ts +7 -6
  200. package/dist/tenant/advance.js +121 -49
  201. package/dist/tenant/artifact-tool.js +3 -4
  202. package/dist/tenant/commands.d.ts +15 -2
  203. package/dist/tenant/commands.js +94 -14
  204. package/dist/tenant/context.d.ts +67 -33
  205. package/dist/tenant/context.js +4 -15
  206. package/dist/tenant/control.d.ts +8 -1
  207. package/dist/tenant/control.js +2 -0
  208. package/dist/tenant/effects.d.ts +1 -1
  209. package/dist/tenant/effects.js +37 -20
  210. package/dist/tenant/environments.d.ts +167 -0
  211. package/dist/tenant/environments.js +823 -0
  212. package/dist/tenant/harness-tokens.d.ts +21 -0
  213. package/dist/tenant/harness-tokens.js +61 -0
  214. package/dist/tenant/host-token.d.ts +1 -1
  215. package/dist/tenant/host-token.js +2 -4
  216. package/dist/tenant/identity-file.d.ts +4 -4
  217. package/dist/tenant/identity-file.js +17 -8
  218. package/dist/tenant/issuers.js +3 -3
  219. package/dist/tenant/principals.js +1 -1
  220. package/dist/tenant/reset.js +3 -3
  221. package/dist/tenant/runtime.d.ts +6 -1
  222. package/dist/tenant/runtime.js +30 -4
  223. package/dist/tenant/scheduler.d.ts +7 -0
  224. package/dist/tenant/scheduler.js +38 -7
  225. package/dist/tenant/session-streams.d.ts +0 -1
  226. package/dist/tenant/session-streams.js +1 -13
  227. package/dist/tenant/session.d.ts +3 -3
  228. package/dist/tenant/session.js +0 -14
  229. package/dist/tenant/sessions.d.ts +13 -13
  230. package/dist/tenant/sessions.js +61 -138
  231. package/dist/tenant/signing-keys.d.ts +5 -1
  232. package/dist/tenant/status.js +3 -3
  233. package/dist/tenant/streams.js +14 -29
  234. package/dist/tenant/sweep.d.ts +39 -12
  235. package/dist/tenant/sweep.js +168 -63
  236. package/dist/tenant/transcript.d.ts +210 -20
  237. package/dist/tenant/transcript.js +62 -11
  238. package/dist/tenant/turn-timing.d.ts +43 -0
  239. package/dist/tenant/turn-timing.js +39 -0
  240. package/dist/tenant/types.d.ts +2 -0
  241. package/dist/traces/ids.d.ts +42 -0
  242. package/dist/traces/ids.js +85 -0
  243. package/dist/traces/key.d.ts +6 -0
  244. package/dist/traces/key.js +37 -0
  245. package/dist/traces/map.d.ts +113 -0
  246. package/dist/traces/map.js +317 -0
  247. package/dist/traces/options.d.ts +7 -0
  248. package/dist/traces/options.js +1 -0
  249. package/dist/traces/otlp.d.ts +42 -0
  250. package/dist/traces/otlp.js +84 -0
  251. package/dist/traces/projector.d.ts +58 -0
  252. package/dist/traces/projector.js +200 -0
  253. package/dist/traces/read.d.ts +23 -0
  254. package/dist/traces/read.js +16 -0
  255. package/dist/vault/coverage.d.ts +26 -0
  256. package/dist/vault/coverage.js +64 -0
  257. package/dist/vault/error.d.ts +11 -1
  258. package/dist/vault/error.js +5 -1
  259. package/dist/vault/service.d.ts +47 -48
  260. package/dist/vault/service.js +321 -382
  261. package/dist/vault/sources.d.ts +5 -16
  262. package/dist/vault/sources.js +6 -8
  263. package/dist/version.d.ts +1 -1
  264. package/dist/version.js +2 -2
  265. package/package.json +14 -13
  266. package/dist/api/http/routes/sandboxes.d.ts +0 -9
  267. package/dist/api/http/routes/sandboxes.js +0 -153
  268. package/dist/api/http/routes/vaults.d.ts +0 -10
  269. package/dist/api/http/routes/vaults.js +0 -139
  270. package/dist/record/sandbox.d.ts +0 -25
  271. package/dist/record/sandbox.js +0 -43
  272. package/dist/tenant/sandboxes.d.ts +0 -122
  273. package/dist/tenant/sandboxes.js +0 -562
@@ -25,37 +25,28 @@
25
25
  * What a run answers arms its timers and its retry.
26
26
  *
27
27
  * Wakes, sandbox reconciles, timers and sweep arming go through the ingress as one-way
28
- * sends, so an API node can call them without ever calling `start`. A wake a commit asked for
29
- * comes from the Session Store's wake outbox (`Tx.wake`) with an idempotency key, so a send
30
- * repeated after its answer was lost causes no second advance.
28
+ * sends (`@restatedev/restate-sdk-clients`), so an API node can call them without ever
29
+ * calling `start`. Every send carries an idempotency key, which the client's retries need: a
30
+ * wake's dedupe key (the outbox gives every committed wake one), a reconcile's outbox key, a
31
+ * timer's `<key>@<at>`, a sandbox timer's `<timer>@<at>`, or one made for the call. A send
32
+ * repeated after its answer was lost therefore starts no second invocation.
31
33
  *
32
- * ## What Restate owns, and what the Session Store owns
33
- *
34
- * Restate owns when a session runs: it delivers each wake at least once, deduped by its
35
- * key for `dedupeRetentionMs`; runs at most one invocation per key at a time; retries an
36
- * invocation whose attempt failed, so an advance whose Worker died runs again and takes the
37
- * session over (§11.4); and keeps every timer: the busy re-wake, `NylorunTimer`,
38
- * `NylorunSandbox` and the `NylorunTenant` sweep chain. The Session Store (Postgres) owns
39
- * what happened: the checkpoint, the effect journal (`harness-api/record.ts`), the events
40
- * (the Record), and the wake outbox that hands each committed wake to Restate. Restate
41
- * journals no effect: an effect's intent and outcome commit with the session's events in
42
- * one transaction, under the advance's lease, and remote harnesses and pod engines write
43
- * them too, over the Harness API.
44
- *
45
- * Two Session Store mechanisms overlap Restate on purpose:
46
- *
47
- * - **The lease and its epoch** (§10.6). Restate's one invocation per key holds for an
48
- * attempt, not for the code it started: an attempt that ends (abort timeout, a dropped
49
- * connection) is retried while the previous attempt's handler may still run on its Worker,
50
- * and a remote harness or pod engine may still hold the run. The epoch fences their writes,
51
- * and `busy` answers the overlap.
52
- * - **The advance deadline** (`tenant/worker.ts`). Restate's abort timeout does not stop a
53
- * running handler, so the Worker bounds each advance itself, below `timeouts`.
34
+ * What Restate owns and what the Session Store owns, and where the two overlap on purpose,
35
+ * is said once, under "Durable Session Execution" in `src/CONTEXT.md`.
54
36
  */
37
+ import { randomUUID } from "node:crypto";
55
38
  import { createServer } from "node:http2";
56
39
  import * as restate from "@restatedev/restate-sdk";
40
+ import * as clients from "@restatedev/restate-sdk-clients";
41
+ import { retry } from "../../retry.js";
57
42
  import { DEFAULT_STOP_GRACE_MS, WAKE_REASONS, parseSessionKey, sessionKey, settleWithin, } from "../../execution/types.js";
58
43
  const HOUR_MS = 60 * 60 * 1000;
44
+ /**
45
+ * How long one send through the ingress keeps retrying while the ingress is unreachable,
46
+ * overloaded, or answers 404 because a just-registered service has not reached it yet. The
47
+ * outbox's grace (`OUTBOX_GRACE_MS` in `tenant/sweep.ts`) is longer.
48
+ */
49
+ export const SEND_RETRY_MS = 10_000;
59
50
  export function createRestateExecution(options) {
60
51
  return new RestateExecution(options);
61
52
  }
@@ -69,6 +60,7 @@ export class RestateExecution {
69
60
  inflight = new Set();
70
61
  names;
71
62
  ingressUrl;
63
+ ingress;
72
64
  adminUrl;
73
65
  sweepIntervalMs;
74
66
  constructor(options) {
@@ -83,6 +75,18 @@ export class RestateExecution {
83
75
  sandbox: `${prefix}NylorunSandbox`,
84
76
  };
85
77
  this.ingressUrl = options.ingressUrl.replace(/\/+$/, "");
78
+ this.ingress = clients.connect({
79
+ url: this.ingressUrl,
80
+ retry: {
81
+ maxAttempts: false,
82
+ maxDuration: SEND_RETRY_MS,
83
+ initialInterval: 100,
84
+ maxInterval: 2000,
85
+ // A just-registered service can take a moment to reach the ingress.
86
+ shouldRetry: (failure) => clients.defaultShouldRetry(failure) ||
87
+ (failure.kind === "response" && failure.status === 404),
88
+ },
89
+ });
86
90
  this.adminUrl = options.adminUrl.replace(/\/+$/, "");
87
91
  this.sweepIntervalMs = options.sweepIntervalMs ?? 5000;
88
92
  }
@@ -93,28 +97,27 @@ export class RestateExecution {
93
97
  async wake(tenantId, sessionId, wake) {
94
98
  if (!WAKE_REASONS.includes(wake.reason))
95
99
  throw new Error(`Unknown wake reason: ${String(wake.reason)}`);
96
- await this.send(this.names.session, sessionKey(tenantId, sessionId), "advance", { reason: wake.reason }, wake.dedupeKey);
100
+ await this.send(this.names.session, sessionKey(tenantId, sessionId), "advance", { reason: wake.reason }, wake.dedupeKey ?? randomUUID());
97
101
  }
98
102
  async sandbox(tenantId, sandboxId, signal) {
99
103
  const key = sessionKey(tenantId, sandboxId);
100
104
  if (signal.kind === "reconcile")
101
- await this.send(this.names.sandbox, key, "reconcile", {});
105
+ await this.send(this.names.sandbox, key, "reconcile", {}, signal.key ?? randomUUID());
102
106
  else
103
- await this.send(this.names.sandbox, key, "arm", {
104
- timer: signal.timer,
105
- at: signal.at,
106
- });
107
+ await this.send(this.names.sandbox, key, "arm", { timer: signal.timer, at: signal.at },
108
+ // Arming a timer for the same time again changes nothing.
109
+ `${signal.timer}@${signal.at}`);
107
110
  }
108
111
  async timer(tenantId, key, at) {
109
- await this.send(this.names.timer, sessionKey(tenantId, key), "set", {
110
- at: at.getTime(),
111
- });
112
+ const ms = at.getTime();
113
+ await this.send(this.names.timer, sessionKey(tenantId, key), "set", { at: ms }, `${key}@${ms}`);
112
114
  }
113
115
  async armSweep(tenantId) {
114
- await this.send(this.names.tenant, tenantKey(tenantId), "arm", {});
116
+ // A key of its own: a later arm (a Host opening the Tenant again) must still run.
117
+ await this.send(this.names.tenant, tenantKey(tenantId), "arm", {}, randomUUID());
115
118
  }
116
119
  async disarmSweep(tenantId) {
117
- await this.send(this.names.tenant, tenantKey(tenantId), "disarm", {});
120
+ await this.send(this.names.tenant, tenantKey(tenantId), "disarm", {}, randomUUID());
118
121
  }
119
122
  /**
120
123
  * Readiness: the admin API (`/health`), which `start` registers with, and the
@@ -311,7 +314,7 @@ export class RestateExecution {
311
314
  if ((await ctx.get(`at:${input.timer}`)) !== input.at)
312
315
  return;
313
316
  ctx.clear(`at:${input.timer}`);
314
- await runSandbox(ctx, input.timer === "retry" ? "reconcile" : input.timer);
317
+ await runSandbox(ctx, input.timer === "idle" ? "idle" : "reconcile");
315
318
  },
316
319
  },
317
320
  options: serviceOptions,
@@ -366,51 +369,62 @@ export class RestateExecution {
366
369
  },
367
370
  };
368
371
  }
369
- /** One-way send through the ingress. Retries while the ingress is unreachable or overloaded. */
372
+ /**
373
+ * One-way send through the ingress (`@restatedev/restate-sdk-clients`). Every send carries
374
+ * an idempotency key, so the client's retries (`SEND_RETRY_MS`), and a send repeated after
375
+ * its answer was lost, start one invocation: the request's own key where it has one (an
376
+ * outbox row's, a timer's time), or one made for this call.
377
+ */
370
378
  async send(service, key, handler, body, idempotencyKey) {
371
- const url = `${this.ingressUrl}/${service}/${encodeURIComponent(key)}/${handler}/send`;
372
- const headers = { "content-type": "application/json" };
373
- if (idempotencyKey !== undefined)
374
- headers["idempotency-key"] = idempotencyKey;
375
- await withRetry(`Restate send to ${service}/${handler}`, 10_000, async () => {
376
- const response = await fetch(url, {
377
- method: "POST",
378
- headers,
379
- body: JSON.stringify(body),
380
- signal: AbortSignal.timeout(10_000),
379
+ try {
380
+ await this.ingress.send({
381
+ service,
382
+ handler,
383
+ key,
384
+ parameter: body,
385
+ opts: clients.SendOpts.from({ idempotencyKey, timeout: 10_000 }),
381
386
  });
382
- if (response.ok) {
383
- await response.body?.cancel();
384
- return;
385
- }
386
- const text = await response.text().catch(() => "");
387
- const error = new Error(`Restate ingress ${response.status} for ${service}/${handler}: ${text}`);
388
- // A just-registered service can take a moment to reach the ingress.
389
- throw response.status >= 500 || response.status === 404 || response.status === 429
390
- ? error
391
- : new PermanentError(error.message);
392
- });
387
+ }
388
+ catch (error) {
389
+ const message = error instanceof clients.HttpCallError
390
+ ? `Restate ingress ${error.status}: ${error.responseText}`
391
+ : error instanceof Error
392
+ ? error.message
393
+ : String(error);
394
+ throw new Error(`Restate send to ${service}/${handler} failed: ${message}`, { cause: error });
395
+ }
393
396
  }
394
397
  /** Registers the Worker endpoint with the admin API, retrying until Restate is up. */
395
398
  async register(uri) {
396
399
  const force = this.options.forceRegistration ?? true;
397
- await withRetry("Restate deployment registration", this.options.registrationTimeoutMs ?? 60_000, async () => {
400
+ const registered = await retry(async () => {
398
401
  const response = await fetch(`${this.adminUrl}/deployments`, {
399
402
  method: "POST",
400
403
  headers: { "content-type": "application/json" },
401
404
  body: JSON.stringify({ uri, force }),
402
405
  signal: AbortSignal.timeout(30_000),
403
406
  });
404
- if (response.ok) {
405
- await response.body?.cancel();
406
- return;
407
- }
407
+ if (response.ok)
408
+ return (await response.json().catch(() => ({})))
409
+ .services;
408
410
  const text = await response.text().catch(() => "");
409
411
  const message = `Restate admin ${response.status} registering ${uri}: ${text}`;
410
412
  // Discovery failures (Restate cannot reach the endpoint yet) come back
411
413
  // as 4xx/5xx; keep trying until the deadline.
412
414
  throw new Error(message);
415
+ }, { minMs: 100, maxMs: 2000, deadlineMs: this.options.registrationTimeoutMs ?? 60_000 }).catch((error) => {
416
+ throw new Error(`Restate deployment registration failed: ${error.message}`, {
417
+ cause: error,
418
+ });
413
419
  });
420
+ // Without `force`, Restate answers a URL already registered with the deployment it has
421
+ // there, without discovering the endpoint again: one with other services is refused here.
422
+ const names = new Set((registered ?? []).map((service) => service.name));
423
+ const missing = Object.values(this.names).filter((name) => !names.has(name));
424
+ if (registered && missing.length > 0)
425
+ throw new Error(`Restate deployment registration failed: ${uri} is registered with other services ` +
426
+ `(missing ${missing.join(", ")}); register this Worker at a URL of its own ` +
427
+ "(NYLORUN_WORKER_VERSION)");
414
428
  }
415
429
  }
416
430
  /**
@@ -501,26 +515,6 @@ function tenantKey(tenantId) {
501
515
  function sqlString(value) {
502
516
  return `'${value.replaceAll("'", "''")}'`;
503
517
  }
504
- class PermanentError extends Error {
505
- }
506
- async function withRetry(what, timeoutMs, attempt) {
507
- const deadline = Date.now() + timeoutMs;
508
- let delay = 100;
509
- for (;;) {
510
- try {
511
- await attempt();
512
- return;
513
- }
514
- catch (error) {
515
- if (error instanceof PermanentError || Date.now() + delay > deadline)
516
- throw new Error(`${what} failed: ${error.message}`, {
517
- cause: error,
518
- });
519
- await new Promise((resolve) => setTimeout(resolve, delay));
520
- delay = Math.min(delay * 2, 2000);
521
- }
522
- }
523
- }
524
518
  function sdkLogger(log) {
525
519
  return (meta, message, ...rest) => {
526
520
  const parts = [message, ...rest].map((part) => part instanceof Error ? part.stack ?? part.message : String(part));
@@ -13,6 +13,8 @@ export interface PgoutputSourceOptions {
13
13
  publication?: string;
14
14
  /** The schema of the record tables. Default `nylorun_streams`. */
15
15
  schema?: string;
16
+ /** The connections' `application_name`. Default `nylorun-stream-relay`. */
17
+ applicationName?: string;
16
18
  /** Wait between attempts to take the slot. Default 5 s. */
17
19
  retryMs?: number;
18
20
  log?: (message: string, fields?: Record<string, unknown>) => void;
@@ -1,7 +1,9 @@
1
1
  /**
2
2
  * The stream relay's change source on Postgres logical replication (Durable Streams §7.1,
3
3
  * §7.4, §7.6): a persistent `pgoutput` slot on the `nylorun_stream_relay` publication, read
4
- * with `pg-logical-replication`. The only file importing it (with `pg`, which it needs).
4
+ * with `pg-logical-replication`. The only file importing it (with `pg`, which it needs). The
5
+ * trace projector (R7.2) reads the same publication from a slot of its own
6
+ * (`traces/projector.ts`); for it a fresh slot means nothing to reconcile.
5
7
  *
6
8
  * - **One reader per slot.** Postgres lets one connection hold a slot, so every Runtime
7
9
  * process can run a source and exactly one is active; the others retry every few seconds
@@ -25,12 +27,13 @@ export function createPgoutputSource(options) {
25
27
  const publication = options.publication ?? "nylorun_stream_relay";
26
28
  const schema = options.schema ?? "nylorun_streams";
27
29
  const retryMs = options.retryMs ?? 5000;
30
+ const applicationName = options.applicationName ?? "nylorun-stream-relay";
28
31
  const { tenantId } = options;
29
32
  const log = options.log ?? (() => { });
30
33
  const pool = new pg.Pool({
31
34
  connectionString: options.connectionString,
32
35
  max: 2,
33
- application_name: "nylorun-stream-relay",
36
+ application_name: applicationName,
34
37
  });
35
38
  let service;
36
39
  let stopped = false;
@@ -72,7 +75,7 @@ export function createPgoutputSource(options) {
72
75
  // `stop()` during `prepareSlot` had no service to stop: do not start one it cannot.
73
76
  if (stopped)
74
77
  return;
75
- const current = new LogicalReplicationService({ connectionString: options.connectionString, application_name: "nylorun-stream-relay" }, { acknowledge: { auto: false, timeoutSeconds: 0 }, flowControl: { enabled: false } });
78
+ const current = new LogicalReplicationService({ connectionString: options.connectionString, application_name: applicationName }, { acknowledge: { auto: false, timeoutSeconds: 0 }, flowControl: { enabled: false } });
76
79
  service = current;
77
80
  let decoding;
78
81
  current.on("start", () => handlers.onActive({ fresh }));
@@ -1,4 +1,4 @@
1
- import type { SandboxBackend } from "../../sandbox/types.js";
1
+ import type { ExecRequest, ExecResult, SandboxBackend } from "../../sandbox/types.js";
2
2
  export interface LocalBackendOptions {
3
3
  /** Default `/workspace`. */
4
4
  readonly workspace?: string;
@@ -6,5 +6,13 @@ export interface LocalBackendOptions {
6
6
  readonly proxyEnv?: () => Readonly<Record<string, string>>;
7
7
  /** The base environment of commands (the engine's allowlisted one); `NYLORUN_*` is dropped. */
8
8
  readonly env: Readonly<Record<string, string | undefined>>;
9
+ /** The engine's `/harness` on the same volume, a disk root of its versions (R2d step 3). */
10
+ readonly harnessRoot?: string;
9
11
  }
12
+ /**
13
+ * How the pod runs a command: `/bin/sh -c` in its own process group, with the base environment,
14
+ * the request's variables and the proxy of the moment. The sandbox tools and the disk's setup
15
+ * (`harness/setup.ts`) both run commands this way.
16
+ */
17
+ export declare function localExec(options: LocalBackendOptions): (request: ExecRequest, signal: AbortSignal) => Promise<ExecResult>;
10
18
  export declare function localBackend(options: LocalBackendOptions): SandboxBackend;
@@ -49,10 +49,14 @@ function collector() {
49
49
  },
50
50
  };
51
51
  }
52
- export function localBackend(options) {
53
- const workspace = options.workspace ?? SANDBOX_WORKSPACE;
54
- const env = baseEnv(options.env, workspace);
55
- const exec = (request, signal) => new Promise((resolve, reject) => {
52
+ /**
53
+ * How the pod runs a command: `/bin/sh -c` in its own process group, with the base environment,
54
+ * the request's variables and the proxy of the moment. The sandbox tools and the disk's setup
55
+ * (`harness/setup.ts`) both run commands this way.
56
+ */
57
+ export function localExec(options) {
58
+ const env = baseEnv(options.env, options.workspace ?? SANDBOX_WORKSPACE);
59
+ return (request, signal) => new Promise((resolve, reject) => {
56
60
  if (signal.aborted)
57
61
  return reject(signal.reason ?? new Error("Turn cancelled"));
58
62
  const child = spawn("/bin/sh", ["-c", request.command], {
@@ -102,10 +106,18 @@ export function localBackend(options) {
102
106
  });
103
107
  });
104
108
  });
109
+ }
110
+ export function localBackend(options) {
111
+ const workspace = options.workspace ?? SANDBOX_WORKSPACE;
112
+ const exec = localExec(options);
105
113
  const missing = (error) => error?.code === "ENOENT" || error?.code === "ENOTDIR";
106
114
  const handle = {
107
115
  workspace,
108
116
  exec,
117
+ diskRoots: {
118
+ workspace,
119
+ ...(options.harnessRoot === undefined ? {} : { harness: options.harnessRoot }),
120
+ },
109
121
  async readFile(path) {
110
122
  try {
111
123
  return await readFile(path, "utf8");
@@ -1,5 +1,6 @@
1
1
  /**
2
- * Virtual backend: an emulated bash with a virtual filesystem, running in the Runtime process.
2
+ * Virtual backend: an emulated bash with a virtual filesystem, running in the harness's process
3
+ * (the harness container, or the Runtime process with NYLORUN_HARNESS=in-process).
3
4
  * `/workspace` is backed by a host directory so files survive stop and Runtime restarts.
4
5
  * It is not a VM boundary; it exists so the first run and CI work on any machine.
5
6
  */
@@ -67,6 +68,8 @@ export function virtualBackend(options) {
67
68
  ...(network ? { network } : {}),
68
69
  });
69
70
  return {
71
+ // A virtual workspace's disk is its host directory: `/workspace` only.
72
+ diskRoots: { workspace: root },
70
73
  async exec(request, signal) {
71
74
  const controller = new AbortController();
72
75
  let timedOut = false;
@@ -58,6 +58,7 @@
58
58
  */
59
59
  import { AppendInput, AppendRecord, RangeNotSatisfiableError, S2, S2Error, SeqNumMismatchError, } from "@s2-dev/streamstore";
60
60
  import { parseBasin, tenantBasinName, validateBasinPrefix } from "../../streams/basin.js";
61
+ import { sleep } from "../../retry.js";
61
62
  import {} from "../../streams/types.js";
62
63
  /** Records per unary read while reading history. */
63
64
  const HISTORY_PAGE = 1000;
@@ -357,16 +358,3 @@ function escapeRegExp(text) {
357
358
  function anySignal(...signals) {
358
359
  return AbortSignal.any(signals.filter((s) => s !== undefined));
359
360
  }
360
- function sleep(ms, signal) {
361
- return new Promise((resolve) => {
362
- if (signal.aborted)
363
- return resolve();
364
- const done = () => {
365
- clearTimeout(timer);
366
- signal.removeEventListener("abort", done);
367
- resolve();
368
- };
369
- const timer = setTimeout(done, ms);
370
- signal.addEventListener("abort", done, { once: true });
371
- });
372
- }
@@ -1,6 +1,7 @@
1
1
  import { z } from "zod";
2
2
  import { readText } from "../http/body.js";
3
- import { tenantRoute } from "../http/define.js";
3
+ import { TraceHeaders, tenantRoute } from "../http/define.js";
4
+ import { traceContextOf } from "../../traces/ids.js";
4
5
  import { jsonResponse } from "../http/respond.js";
5
6
  import { a2aCall, a2aCard } from "./routes.js";
6
7
  const A2A_SPECIFICATION = {
@@ -63,6 +64,7 @@ export function a2aRoutes(api) {
63
64
  "A2A-Version": z.string().optional().meta({ description: "Or the `A2A-Version` header: `1.0`" }),
64
65
  }),
65
66
  body: { required: true, content: { "application/json": { schema: JsonRpcRequest } } },
67
+ headers: TraceHeaders,
66
68
  },
67
69
  responses: {
68
70
  200: {
@@ -73,9 +75,11 @@ export function a2aRoutes(api) {
73
75
  }, async (c) => {
74
76
  const scope = c.get("scope");
75
77
  const version = c.env.incoming.headers["a2a-version"];
78
+ const trace = traceContextOf(c.req.raw.headers);
76
79
  return jsonResponse(200, await a2aCall(c.env.tenant, scope, c.req.param("agentId"), {
77
80
  body: () => readText(c.req.raw),
78
81
  version: typeof version === "string" ? version : (c.req.query("A2A-Version") ?? null),
82
+ ...(trace ? { trace } : {}),
79
83
  }, c.env.outgoing));
80
84
  });
81
85
  }
@@ -1,5 +1,6 @@
1
1
  import type { ServerResponse } from "node:http";
2
2
  import { type AuthScope, type TenantContext } from "../../tenant/context.js";
3
+ import type { TraceContext } from "../../traces/ids.js";
3
4
  /**
4
5
  * How long a blocking `SendMessage` waits for the task to end or ask for input. After it, the
5
6
  * task is returned `WORKING` and keeps running; the caller polls `GetTask`.
@@ -16,4 +17,5 @@ export declare function a2aCard(ctx: TenantContext, scope: AuthScope, agentId: s
16
17
  export declare function a2aCall(ctx: TenantContext, scope: AuthScope, agentId: string, input: {
17
18
  body: () => Promise<string>;
18
19
  version: string | null;
20
+ trace?: TraceContext;
19
21
  }, response: ServerResponse): Promise<unknown>;
@@ -50,7 +50,7 @@ export async function a2aCall(ctx, scope, agentId, input, response) {
50
50
  // A blocking call stops waiting when the caller hangs up; the task keeps running.
51
51
  const hangup = new AbortController();
52
52
  response.on("close", () => hangup.abort());
53
- const caller = { ctx, scope, access, agentId };
53
+ const caller = { ctx, scope, access, agentId, ...(input.trace ? { trace: input.trace } : {}) };
54
54
  const { id } = envelope.request;
55
55
  try {
56
56
  checkVersion(input.version);
@@ -137,7 +137,7 @@ async function run(caller, sessionId, input,
137
137
  /** The error for a `409` conflict: the session is busy or no longer waiting. */
138
138
  conflict) {
139
139
  try {
140
- const accepted = (await command(caller.ctx, sessionId, SessionCommandSchema.parse(input), caller.scope));
140
+ const accepted = (await command(caller.ctx, sessionId, SessionCommandSchema.parse(input), caller.scope, caller.trace ? { trace: caller.trace } : {}));
141
141
  return { turnId: accepted.turnId, cursor: accepted.cursor };
142
142
  }
143
143
  catch (error) {
@@ -97,7 +97,8 @@ export function historyOf(events, contextId, taskId) {
97
97
  add(clientMessageId(event), "ROLE_USER", [valuePart(payload.value)]);
98
98
  break;
99
99
  case "message.assistant":
100
- // Agents used as tools speak inside the turn; only the agent's own words are history.
100
+ // Agents used as tools speak inside the turn; only the agent's own words are history. A
101
+ // step that said nothing (`text: ""`, R7.1) is none.
101
102
  if (!payload.agent && typeof payload.text === "string" && payload.text.trim() !== "")
102
103
  add(event.eventId, "ROLE_AGENT", [{ text: payload.text }]);
103
104
  break;
@@ -2,7 +2,8 @@ import { RESPONSE_ALREADY_SENT } from "@hono/node-server/utils/response";
2
2
  import { z } from "zod";
3
3
  import { EventSchemas, MessageSchema, RunAgentInputSchema } from "@ag-ui/core/schemas";
4
4
  import { readJson } from "../http/body.js";
5
- import { tenantRoute } from "../http/define.js";
5
+ import { TraceHeaders, tenantRoute } from "../http/define.js";
6
+ import { traceContextOf } from "../../traces/ids.js";
6
7
  import { jsonResponse } from "../http/respond.js";
7
8
  import { agUiCaller, cancelRun, reattachRun, startRun, threadMessages } from "./routes.js";
8
9
  import { sessionIdFor } from "./session-id.js";
@@ -56,11 +57,12 @@ export function agUiRoutes(api) {
56
57
  required: true,
57
58
  content: { "application/json": { schema: ref("AgUiRunAgentInput") } },
58
59
  },
60
+ headers: TraceHeaders,
59
61
  },
60
62
  responses: { 200: events("The run's events, from RUN_STARTED to its end") },
61
63
  }, async (c) => {
62
64
  const { scope, subject, access } = callerOf(c);
63
- await startRun(c.env.tenant, scope, c.req.param("agentId"), { subject, access }, await readJson(c.req.raw), c.env.outgoing);
65
+ await startRun(c.env.tenant, scope, c.req.param("agentId"), { subject, access }, await readJson(c.req.raw), c.env.outgoing, traceContextOf(c.req.raw.headers));
64
66
  return RESPONSE_ALREADY_SENT;
65
67
  });
66
68
  tenantRoute(api, FOR_A_PERSON, {
@@ -1,4 +1,4 @@
1
- import { parseTranscriptEvent } from "@nylorun/core/contracts";
1
+ import { isEmptyAssistantMessage, parseTranscriptEvent, } from "@nylorun/core/contracts";
2
2
  import { outputText, toolResultOf } from "./translate.js";
3
3
  export function messagesFromEvents(items) {
4
4
  const messages = [];
@@ -37,6 +37,9 @@ export function messagesFromEvents(items) {
37
37
  });
38
38
  continue;
39
39
  }
40
+ // A step that said nothing is in the log for its timing (R7.1), never a message.
41
+ if (event.type === "message.assistant" && isEmptyAssistantMessage(event.payload))
42
+ continue;
40
43
  if (event.type === "message.assistant" && !event.payload.agent) {
41
44
  const p = event.payload;
42
45
  sawAssistant = true;
@@ -1,6 +1,7 @@
1
1
  import type { ServerResponse } from "node:http";
2
2
  import { type Message } from "@ag-ui/core";
3
3
  import { type AuthScope, type SessionAccess, type TenantContext } from "../../tenant/context.js";
4
+ import type { TraceContext } from "../../traces/ids.js";
4
5
  /**
5
6
  * Who an AG-UI request acts for, once its agent is known: a person (an issuer token's subject,
6
7
  * or `Nylorun-Subject`), who may use the agent; else 400, or the 404 of a missing agent.
@@ -13,7 +14,9 @@ export declare function agUiCaller(scope: AuthScope, agentId: string): {
13
14
  export declare function startRun(ctx: TenantContext, scope: AuthScope, agentId: string, caller: {
14
15
  subject: string;
15
16
  access: SessionAccess | undefined;
16
- }, body: unknown, response: ServerResponse): Promise<void>;
17
+ }, body: unknown, response: ServerResponse,
18
+ /** The request's trace context (R7.4): the turn the run starts joins the caller's trace. */
19
+ trace?: TraceContext): Promise<void>;
17
20
  /** `GET …/threads/:thread/messages`: the thread's messages; none before its first run. */
18
21
  export declare function threadMessages(ctx: TenantContext, id: string, access: SessionAccess | undefined): Promise<Message[]>;
19
22
  /**
@@ -33,7 +33,7 @@ import { fail, HttpError } from "../../tenant/http.js";
33
33
  import { observeSession, readHistory, StreamClosed, streamHolderOf, } from "../../tenant/session-streams.js";
34
34
  import { putSession, sessionView } from "../../tenant/sessions.js";
35
35
  import { mayUseAgent } from "../../tenant/auth.js";
36
- import { sandboxGrantsOf } from "../../tenant/sandboxes.js";
36
+ import { environmentGrantsOf } from "../../tenant/environments.js";
37
37
  const HEARTBEAT_MS = 15_000;
38
38
  const TERMINAL = new Set([
39
39
  "turn.completed",
@@ -83,8 +83,7 @@ function sessionOptions(input, scope) {
83
83
  return {};
84
84
  const parsed = PutSessionRequestSchema.pick({
85
85
  info: true,
86
- vaultIds: true,
87
- credentialSelections: true,
86
+ environment: true,
88
87
  })
89
88
  .strict()
90
89
  .safeParse(raw);
@@ -99,11 +98,11 @@ function sessionOptions(input, scope) {
99
98
  });
100
99
  return parsed.data;
101
100
  }
102
- async function accepted(ctx, id, body, scope) {
103
- const result = (await command(ctx, id, body, scope));
101
+ async function accepted(ctx, id, body, scope, trace) {
102
+ const result = (await command(ctx, id, body, scope, trace ? { trace } : {}));
104
103
  return result?.cursor ?? null;
105
104
  }
106
- async function submit(ctx, id, input, scope) {
105
+ async function submit(ctx, id, input, scope, trace) {
107
106
  if (input.resume?.length) {
108
107
  // The interaction's kind is on the session's pending view, not in its id.
109
108
  const view = (await ctx.store.tx(async (t) => sessionView(t, await sessionOf(t, id, accessOf(scope)))));
@@ -144,7 +143,7 @@ async function submit(ctx, id, input, scope) {
144
143
  requestId: randomUUID(),
145
144
  idempotencyKey: message.id,
146
145
  content: message.text,
147
- }, scope);
146
+ }, scope, trace);
148
147
  }
149
148
  /**
150
149
  * Streams translated events until the run ends. Closing the connection stops the read, never
@@ -229,7 +228,9 @@ export function agUiCaller(scope, agentId) {
229
228
  return { subject, access: accessOf(scope) };
230
229
  }
231
230
  /** `POST /v1/ag-ui/agents/:agent`: start a run, and stream it on `response`. */
232
- export async function startRun(ctx, scope, agentId, caller, body, response) {
231
+ export async function startRun(ctx, scope, agentId, caller, body, response,
232
+ /** The request's trace context (R7.4): the turn the run starts joins the caller's trace. */
233
+ trace) {
233
234
  const parsed = RunAgentInputSchema.safeParse(body);
234
235
  if (!parsed.success)
235
236
  return fail(400, "Invalid RunAgentInput", { code: "invalid_request" });
@@ -245,12 +246,12 @@ export async function startRun(ctx, scope, agentId, caller, body, response) {
245
246
  ...sessionOptions(input, scope),
246
247
  }, caller.access, {
247
248
  createOnly: true,
248
- ...(sandboxGrantsOf(scope) === undefined ? {} : { sandboxGrants: sandboxGrantsOf(scope) }),
249
+ ...(environmentGrantsOf(scope) === undefined ? {} : { environmentGrants: environmentGrantsOf(scope) }),
249
250
  });
250
251
  const translator = new RunTranslator(input.threadId, input.runId);
251
252
  let cursor;
252
253
  try {
253
- cursor = await submit(ctx, id, input, scope);
254
+ cursor = await submit(ctx, id, input, scope, trace);
254
255
  }
255
256
  catch (error) {
256
257
  // A busy session or a limit is a run that could not start, not a transport failure.
@@ -4,7 +4,7 @@
4
4
  * `<callId>:result`, so live runs and `messagesFromEvents` agree.
5
5
  */
6
6
  import { EventType } from "@ag-ui/core";
7
- import { parseTranscriptEvent, } from "@nylorun/core/contracts";
7
+ import { isEmptyAssistantMessage, parseTranscriptEvent, } from "@nylorun/core/contracts";
8
8
  /** The text a tool result shows; `undefined` while there is no result yet. */
9
9
  export function toolResultText(result) {
10
10
  const value = result;
@@ -89,7 +89,8 @@ export class RunTranslator {
89
89
  switch (event.type) {
90
90
  case "message.assistant": {
91
91
  const p = event.payload;
92
- if (p.agent)
92
+ // A step that said nothing is in the log for its timing (R7.1), never a message.
93
+ if (p.agent || isEmptyAssistantMessage(p))
93
94
  return none;
94
95
  this.sawAssistant = true;
95
96
  return { events: assistantEvents(p), finished: false };