@nylorun/runtime 0.21.1-beta → 0.23.0-beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/CHANGELOG.md +32 -0
  2. package/README.md +17 -3
  3. package/dist/adapters/execution/restate.d.ts +15 -3
  4. package/dist/adapters/execution/restate.js +77 -83
  5. package/dist/adapters/sandbox/local.js +1 -1
  6. package/dist/adapters/sandbox/virtual.js +1 -1
  7. package/dist/adapters/streams/s2.js +1 -13
  8. package/dist/api/components.d.ts +42 -2
  9. package/dist/core/flow-host.d.ts +1 -19
  10. package/dist/core/flow-host.js +8 -37
  11. package/dist/execution/types.d.ts +10 -8
  12. package/dist/execution/types.js +5 -7
  13. package/dist/gates/egress-credentials.d.ts +48 -0
  14. package/dist/gates/egress-credentials.js +147 -0
  15. package/dist/gates/egress.d.ts +3 -0
  16. package/dist/gates/egress.js +14 -1
  17. package/dist/gates/tenant-vaults.d.ts +4 -0
  18. package/dist/gates/tenant-vaults.js +2 -0
  19. package/dist/harness/calls.d.ts +2 -0
  20. package/dist/harness/calls.js +1 -0
  21. package/dist/harness/executors.js +1 -0
  22. package/dist/harness/main.js +4 -1
  23. package/dist/harness/pod.d.ts +2 -0
  24. package/dist/harness/pod.js +36 -1
  25. package/dist/harness-api/start.d.ts +2 -0
  26. package/dist/harness-api/start.js +4 -2
  27. package/dist/harness-api/ws-server.js +2 -2
  28. package/dist/host/execution.d.ts +8 -6
  29. package/dist/host/gates.js +11 -0
  30. package/dist/host/main.js +10 -16
  31. package/dist/host/stack-config.d.ts +6 -0
  32. package/dist/host/stack-config.js +6 -0
  33. package/dist/infra/execution.d.ts +18 -1
  34. package/dist/infra/execution.js +28 -4
  35. package/dist/infra/index.d.ts +1 -1
  36. package/dist/keys/egress-ca.d.ts +15 -0
  37. package/dist/keys/egress-ca.js +66 -0
  38. package/dist/keys/keys.d.ts +8 -1
  39. package/dist/keys/keys.js +6 -0
  40. package/dist/keys/x509.d.ts +29 -0
  41. package/dist/keys/x509.js +160 -0
  42. package/dist/management-openapi.json +195 -9
  43. package/dist/openapi.json +2 -1
  44. package/dist/retry.d.ts +21 -0
  45. package/dist/retry.js +34 -0
  46. package/dist/sandbox/join.d.ts +2 -0
  47. package/dist/sandbox/join.js +14 -10
  48. package/dist/sandbox/manager.d.ts +2 -0
  49. package/dist/sandbox/manager.js +1 -1
  50. package/dist/sandbox/pods/reconcile.js +3 -3
  51. package/dist/sandbox/tools.d.ts +3 -1
  52. package/dist/sandbox/tools.js +4 -2
  53. package/dist/sandbox/types.d.ts +5 -0
  54. package/dist/store/postgres/control.js +8 -1
  55. package/dist/store/postgres/drizzle/0018_wake_outbox_retries.sql +3 -0
  56. package/dist/store/postgres/drizzle/0019_sandbox_signal_outbox.sql +18 -0
  57. package/dist/store/postgres/drizzle/meta/_journal.json +14 -0
  58. package/dist/store/postgres/schema.d.ts +265 -4
  59. package/dist/store/postgres/schema.js +35 -4
  60. package/dist/store/postgres/store.js +165 -52
  61. package/dist/store/types.d.ts +96 -39
  62. package/dist/streams/relay/core.js +1 -13
  63. package/dist/tenant/advance.js +6 -7
  64. package/dist/tenant/commands.js +1 -2
  65. package/dist/tenant/context.d.ts +19 -11
  66. package/dist/tenant/control.d.ts +8 -1
  67. package/dist/tenant/control.js +2 -0
  68. package/dist/tenant/runtime.js +23 -3
  69. package/dist/tenant/sandboxes.d.ts +2 -1
  70. package/dist/tenant/sandboxes.js +23 -19
  71. package/dist/tenant/scheduler.d.ts +7 -0
  72. package/dist/tenant/scheduler.js +38 -7
  73. package/dist/tenant/session-streams.d.ts +0 -1
  74. package/dist/tenant/session-streams.js +1 -13
  75. package/dist/tenant/sessions.js +26 -0
  76. package/dist/tenant/streams.js +14 -29
  77. package/dist/tenant/sweep.d.ts +39 -12
  78. package/dist/tenant/sweep.js +157 -58
  79. package/dist/tenant/transcript.d.ts +8 -8
  80. package/dist/vault/service.d.ts +34 -0
  81. package/dist/vault/service.js +148 -15
  82. package/dist/version.d.ts +1 -1
  83. package/dist/version.js +1 -1
  84. package/package.json +4 -3
package/CHANGELOG.md CHANGED
@@ -1,5 +1,37 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.23.0-beta
4
+
5
+ ### Minor Changes
6
+
7
+ - 72a4b52: The outbox no longer stalls the Tenant sweep, holds up commands or loses a pod sandbox's signals, and Workers register versioned Restate deployments.
8
+
9
+ - A wake that keeps failing (Restate refuses it, or an older Runtime does not know its reason) is retried with backoff and, after ten failures, parked: logged once at error and kept in the outbox for an operator. It no longer fails the sweep, so Restate never pauses the Tenant's sweep object over it. Migration 0018 adds `attempts`, `retry_at` and `parked_at` to `nylorun.wakes`.
10
+ - The sweep drains the whole outbox and pages through every orphaned session in each pass, within a 2 s budget, instead of 100 rows per pass.
11
+ - A command's wake is sent after commit without the request waiting for it; a failed send is logged at warn and sent again by the sweep's next pass. The sweep's grace is compared on the database's clock and is now 30 s.
12
+ - A pod sandbox's reconcile and idle timer go through the outbox (migration 0019, `nylorun.sandbox_signals`), so a crash after commit no longer loses them, and a host revocation goes over the control bus (`host.revoked`), so it reaches every process with the Tenant open. `Tx.afterCommit` is removed.
13
+ - Sends to Restate's ingress use `@restatedev/restate-sdk-clients` 1.17.2, each with an idempotency key.
14
+ - A Worker registers `<NYLORUN_WORKER_URL>/nylorun/<version>`, the version being the new `NYLORUN_WORKER_VERSION` or the Runtime's own, without forcing in a container, so a rolling upgrade keeps the old deployment for the invocations it started. A build whose Worker code changed under the same Runtime version sets its own `NYLORUN_WORKER_VERSION`.
15
+
16
+ ## 0.22.0-beta
17
+
18
+ ### Minor Changes
19
+
20
+ - 212364b: **Credentials for skills (R2c).** A skill's CLI in a pod sandbox authenticates with a vault credential whose value never enters the sandbox.
21
+
22
+ - `@nylorun/core`: vault credentials gain `environment_secret` (`secretName`, `secretValue`, `allowedHosts`, optional `inject: { header, format }`) and `environment_variable` (`variableName`, `variableValue`), in create, rotate and `CredentialInfo` (whose `binding.url` is now optional). `ENVIRONMENT_SECRET_SENTINEL` (`nylorun-managed`), `ENVIRONMENT_SECRET_DEFAULT_INJECT`, `isInjectFormat`, `renderInjectFormat`, `EnvironmentNameSchema` and `RESERVED_ENVIRONMENT_NAMES` are exported; `ERROR_CODES` gains `credential_conflict`; the Harness API's `RunRouting.sandbox` gains `environment`.
23
+ - `@nylorun/runtime`: every sandbox command gets each secret's name set to `nylorun-managed` and each variable's value. egress-gate terminates TLS for port 443 of a host an `environment_secret` of the sandbox's sessions is bound to, with a fresh leaf signed by the installation's egress CA (held by keys; its certificate reaches the pod in the join answer, and the engine points `SSL_CERT_FILE`, `NODE_EXTRA_CA_CERTS`, `REQUESTS_CA_BUNDLE`, `CURL_CA_BUNDLE`, `GIT_SSL_CAINFO` and `AWS_CA_BUNDLE` at a bundle with it), and sets the credential's header on every request from the vault, uncached. Every other host stays an opaque tunnel. Session open answers `409 credential_conflict` for two credentials with one variable name or one host, or two owners' secrets on one sandbox. No protocol change.
24
+ - `@nylorun/studio`: the Credentials page lists shell credentials (their variable and hosts) and deletes them; create and rotate them with the Management API.
25
+
26
+ ### Patch Changes
27
+
28
+ - Pin core to the tested release.
29
+ - Pin harness to the tested release.
30
+ - Updated dependencies [212364b]
31
+ - Updated dependencies
32
+ - @nylorun/core@0.19.0-beta
33
+ - @nylorun/harness@0.25.1-beta
34
+
3
35
  ## 0.21.1-beta
4
36
 
5
37
  ### Patch Changes
package/README.md CHANGED
@@ -44,8 +44,9 @@ or deleted key stops authenticating at once. A database written by a Runtime tha
44
44
  database (`tenant_<id>` schemas), or by a pre-release build of one Tenant per database
45
45
  (`schema_version` tables), is refused: this release starts fresh on a new database. Restate runs one advance of a session at a time, delivers its wakes and holds
46
46
  its timers and the Tenant's sweep (Durable Session Execution), and journals no effect:
47
- checkpoints, effects, events and the wake outbox are the Session Store's, and a wake is
48
- written to the outbox in the transaction that causes it, so a commit never loses it; every session's events are relayed from the record to
47
+ checkpoints, effects, events and the outbox are the Session Store's, and a wake or a pod
48
+ sandbox's reconcile is written to the outbox in the transaction that causes it, so a commit
49
+ never loses it ([src/CONTEXT.md](src/CONTEXT.md)); every session's events are relayed from the record to
49
50
  its own S2 stream, which history and SSE read (Durable Streams). One image runs every **service**,
50
51
  and `--service` picks what a process runs: `core` serves the Runtime API, the
51
52
  Management API and SSE and runs the stream relay, `loop` runs advances (the Worker), and
@@ -69,10 +70,23 @@ gateway ([DEPLOYMENT.md](../DEPLOYMENT.md#the-harness-agent-turns-mcp-servers-an
69
70
 
70
71
  The container is configured by its environment, which a local Tenant's Compose file
71
72
  sets: `NYLORUN_DATABASE_URL` (required), `NYLORUN_RESTATE_INGRESS_URL`,
72
- `NYLORUN_RESTATE_ADMIN_URL`, `NYLORUN_WORKER_URL` and
73
+ `NYLORUN_RESTATE_ADMIN_URL`, `NYLORUN_WORKER_URL`, `NYLORUN_WORKER_VERSION` and
73
74
  `NYLORUN_RESTATE_IDENTITY_KEY` (Restate), `NYLORUN_S2_ENDPOINT` and
74
75
  `NYLORUN_S2_TOKEN`, and in container mode `NYLORUN_LISTEN_HOST`,
75
76
  `NYLORUN_LISTEN_PORT`, `NYLORUN_ALLOWED_HOSTS` and `NYLORUN_PUBLIC_URL`.
77
+ A Worker registers its Restate deployment at a versioned URL,
78
+ `<NYLORUN_WORKER_URL>/nylorun/<version>`, where the version is `NYLORUN_WORKER_VERSION`
79
+ (letters, digits, `.`, `_`, `-`) or, by default, the Runtime's own. In a container it never
80
+ replaces a deployment already registered there (`force: false`): during a rolling upgrade
81
+ the new version's Workers register next to the old ones, Restate sends new invocations to
82
+ the new deployment and lets the old one finish what it started, and every Worker of one
83
+ version shares its deployment through one load-balanced `NYLORUN_WORKER_URL`. A build that
84
+ changes the Worker's code without changing the Runtime's version (an image built from a
85
+ branch) sets its own `NYLORUN_WORKER_VERSION`: without force Restate keeps the deployment it
86
+ already has at a URL and does not discover the Worker again, so the changed code would run
87
+ under the old deployment's registration, and a Worker whose services that deployment lacks
88
+ refuses to start. Outside a container (a development
89
+ Host) the registration replaces the deployment at that URL.
76
90
  `NYLORUN_MANAGEMENT_KEY_FILE` names the bootstrap secret (below).
77
91
  `NYLORUN_IDENTITY_FILE` names the identity
78
92
  file, a YAML list of the trusted issuers whose JWTs the Runtime API accepts
@@ -1,4 +1,10 @@
1
1
  import { type DurableExecution, type SandboxSignal, type StuckInvocation, type Wake, type WorkerHandlers } from "../../execution/types.js";
2
+ /**
3
+ * How long one send through the ingress keeps retrying while the ingress is unreachable,
4
+ * overloaded, or answers 404 because a just-registered service has not reached it yet. The
5
+ * outbox's grace (`OUTBOX_GRACE_MS` in `tenant/sweep.ts`) is longer.
6
+ */
7
+ export declare const SEND_RETRY_MS = 10000;
2
8
  export interface RestateTimeouts {
3
9
  /**
4
10
  * How long an invocation may run without journal progress before Restate
@@ -60,8 +66,8 @@ export interface RestateExecutionOptions {
60
66
  /**
61
67
  * Overwrite a deployment already registered at `workerAdvertisedUrl`, so a
62
68
  * restarted Worker with changed code takes effect. Default true, which suits
63
- * a stable URL on a developer machine. Rolling upgrades should advertise a
64
- * versioned URL instead (§14.6).
69
+ * a stable URL on a developer machine. A Host in a container advertises a
70
+ * versioned URL without it (`workerDeployment` in `infra/execution.ts`, §14.6).
65
71
  */
66
72
  forceRegistration?: boolean;
67
73
  /** How long `start` keeps retrying registration while Restate comes up. Default 60000. */
@@ -88,6 +94,7 @@ export declare class RestateExecution implements DurableExecution {
88
94
  private readonly inflight;
89
95
  private readonly names;
90
96
  private readonly ingressUrl;
97
+ private readonly ingress;
91
98
  private readonly adminUrl;
92
99
  private readonly sweepIntervalMs;
93
100
  constructor(options: RestateExecutionOptions);
@@ -123,7 +130,12 @@ export declare class RestateExecution implements DurableExecution {
123
130
  private runTracked;
124
131
  private staleSweepMs;
125
132
  private serviceOptions;
126
- /** One-way send through the ingress. Retries while the ingress is unreachable or overloaded. */
133
+ /**
134
+ * One-way send through the ingress (`@restatedev/restate-sdk-clients`). Every send carries
135
+ * an idempotency key, so the client's retries (`SEND_RETRY_MS`), and a send repeated after
136
+ * its answer was lost, start one invocation: the request's own key where it has one (an
137
+ * outbox row's, a timer's time), or one made for this call.
138
+ */
127
139
  private send;
128
140
  /** Registers the Worker endpoint with the admin API, retrying until Restate is up. */
129
141
  private register;
@@ -25,37 +25,28 @@
25
25
  * What a run answers arms its timers and its retry.
26
26
  *
27
27
  * Wakes, sandbox reconciles, timers and sweep arming go through the ingress as one-way
28
- * sends, so an API node can call them without ever calling `start`. A wake a commit asked for
29
- * comes from the Session Store's wake outbox (`Tx.wake`) with an idempotency key, so a send
30
- * repeated after its answer was lost causes no second advance.
28
+ * sends (`@restatedev/restate-sdk-clients`), so an API node can call them without ever
29
+ * calling `start`. Every send carries an idempotency key, which the client's retries need: a
30
+ * wake's dedupe key (the outbox gives every committed wake one), a reconcile's outbox key, a
31
+ * timer's `<key>@<at>`, a sandbox timer's `<timer>@<at>`, or one made for the call. A send
32
+ * repeated after its answer was lost therefore starts no second invocation.
31
33
  *
32
- * ## What Restate owns, and what the Session Store owns
33
- *
34
- * Restate owns when a session runs: it delivers each wake at least once, deduped by its
35
- * key for `dedupeRetentionMs`; runs at most one invocation per key at a time; retries an
36
- * invocation whose attempt failed, so an advance whose Worker died runs again and takes the
37
- * session over (§11.4); and keeps every timer: the busy re-wake, `NylorunTimer`,
38
- * `NylorunSandbox` and the `NylorunTenant` sweep chain. The Session Store (Postgres) owns
39
- * what happened: the checkpoint, the effect journal (`harness-api/record.ts`), the events
40
- * (the Record), and the wake outbox that hands each committed wake to Restate. Restate
41
- * journals no effect: an effect's intent and outcome commit with the session's events in
42
- * one transaction, under the advance's lease, and remote harnesses and pod engines write
43
- * them too, over the Harness API.
44
- *
45
- * Two Session Store mechanisms overlap Restate on purpose:
46
- *
47
- * - **The lease and its epoch** (§10.6). Restate's one invocation per key holds for an
48
- * attempt, not for the code it started: an attempt that ends (abort timeout, a dropped
49
- * connection) is retried while the previous attempt's handler may still run on its Worker,
50
- * and a remote harness or pod engine may still hold the run. The epoch fences their writes,
51
- * and `busy` answers the overlap.
52
- * - **The advance deadline** (`tenant/worker.ts`). Restate's abort timeout does not stop a
53
- * running handler, so the Worker bounds each advance itself, below `timeouts`.
34
+ * What Restate owns and what the Session Store owns, and where the two overlap on purpose,
35
+ * is said once, under "Durable Session Execution" in `src/CONTEXT.md`.
54
36
  */
37
+ import { randomUUID } from "node:crypto";
55
38
  import { createServer } from "node:http2";
56
39
  import * as restate from "@restatedev/restate-sdk";
40
+ import * as clients from "@restatedev/restate-sdk-clients";
41
+ import { retry } from "../../retry.js";
57
42
  import { DEFAULT_STOP_GRACE_MS, WAKE_REASONS, parseSessionKey, sessionKey, settleWithin, } from "../../execution/types.js";
58
43
  const HOUR_MS = 60 * 60 * 1000;
44
+ /**
45
+ * How long one send through the ingress keeps retrying while the ingress is unreachable,
46
+ * overloaded, or answers 404 because a just-registered service has not reached it yet. The
47
+ * outbox's grace (`OUTBOX_GRACE_MS` in `tenant/sweep.ts`) is longer.
48
+ */
49
+ export const SEND_RETRY_MS = 10_000;
59
50
  export function createRestateExecution(options) {
60
51
  return new RestateExecution(options);
61
52
  }
@@ -69,6 +60,7 @@ export class RestateExecution {
69
60
  inflight = new Set();
70
61
  names;
71
62
  ingressUrl;
63
+ ingress;
72
64
  adminUrl;
73
65
  sweepIntervalMs;
74
66
  constructor(options) {
@@ -83,6 +75,18 @@ export class RestateExecution {
83
75
  sandbox: `${prefix}NylorunSandbox`,
84
76
  };
85
77
  this.ingressUrl = options.ingressUrl.replace(/\/+$/, "");
78
+ this.ingress = clients.connect({
79
+ url: this.ingressUrl,
80
+ retry: {
81
+ maxAttempts: false,
82
+ maxDuration: SEND_RETRY_MS,
83
+ initialInterval: 100,
84
+ maxInterval: 2000,
85
+ // A just-registered service can take a moment to reach the ingress.
86
+ shouldRetry: (failure) => clients.defaultShouldRetry(failure) ||
87
+ (failure.kind === "response" && failure.status === 404),
88
+ },
89
+ });
86
90
  this.adminUrl = options.adminUrl.replace(/\/+$/, "");
87
91
  this.sweepIntervalMs = options.sweepIntervalMs ?? 5000;
88
92
  }
@@ -93,28 +97,27 @@ export class RestateExecution {
93
97
  async wake(tenantId, sessionId, wake) {
94
98
  if (!WAKE_REASONS.includes(wake.reason))
95
99
  throw new Error(`Unknown wake reason: ${String(wake.reason)}`);
96
- await this.send(this.names.session, sessionKey(tenantId, sessionId), "advance", { reason: wake.reason }, wake.dedupeKey);
100
+ await this.send(this.names.session, sessionKey(tenantId, sessionId), "advance", { reason: wake.reason }, wake.dedupeKey ?? randomUUID());
97
101
  }
98
102
  async sandbox(tenantId, sandboxId, signal) {
99
103
  const key = sessionKey(tenantId, sandboxId);
100
104
  if (signal.kind === "reconcile")
101
- await this.send(this.names.sandbox, key, "reconcile", {});
105
+ await this.send(this.names.sandbox, key, "reconcile", {}, signal.key ?? randomUUID());
102
106
  else
103
- await this.send(this.names.sandbox, key, "arm", {
104
- timer: signal.timer,
105
- at: signal.at,
106
- });
107
+ await this.send(this.names.sandbox, key, "arm", { timer: signal.timer, at: signal.at },
108
+ // Arming a timer for the same time again changes nothing.
109
+ `${signal.timer}@${signal.at}`);
107
110
  }
108
111
  async timer(tenantId, key, at) {
109
- await this.send(this.names.timer, sessionKey(tenantId, key), "set", {
110
- at: at.getTime(),
111
- });
112
+ const ms = at.getTime();
113
+ await this.send(this.names.timer, sessionKey(tenantId, key), "set", { at: ms }, `${key}@${ms}`);
112
114
  }
113
115
  async armSweep(tenantId) {
114
- await this.send(this.names.tenant, tenantKey(tenantId), "arm", {});
116
+ // A key of its own: a later arm (a Host opening the Tenant again) must still run.
117
+ await this.send(this.names.tenant, tenantKey(tenantId), "arm", {}, randomUUID());
115
118
  }
116
119
  async disarmSweep(tenantId) {
117
- await this.send(this.names.tenant, tenantKey(tenantId), "disarm", {});
120
+ await this.send(this.names.tenant, tenantKey(tenantId), "disarm", {}, randomUUID());
118
121
  }
119
122
  /**
120
123
  * Readiness: the admin API (`/health`), which `start` registers with, and the
@@ -366,51 +369,62 @@ export class RestateExecution {
366
369
  },
367
370
  };
368
371
  }
369
- /** One-way send through the ingress. Retries while the ingress is unreachable or overloaded. */
372
+ /**
373
+ * One-way send through the ingress (`@restatedev/restate-sdk-clients`). Every send carries
374
+ * an idempotency key, so the client's retries (`SEND_RETRY_MS`), and a send repeated after
375
+ * its answer was lost, start one invocation: the request's own key where it has one (an
376
+ * outbox row's, a timer's time), or one made for this call.
377
+ */
370
378
  async send(service, key, handler, body, idempotencyKey) {
371
- const url = `${this.ingressUrl}/${service}/${encodeURIComponent(key)}/${handler}/send`;
372
- const headers = { "content-type": "application/json" };
373
- if (idempotencyKey !== undefined)
374
- headers["idempotency-key"] = idempotencyKey;
375
- await withRetry(`Restate send to ${service}/${handler}`, 10_000, async () => {
376
- const response = await fetch(url, {
377
- method: "POST",
378
- headers,
379
- body: JSON.stringify(body),
380
- signal: AbortSignal.timeout(10_000),
379
+ try {
380
+ await this.ingress.send({
381
+ service,
382
+ handler,
383
+ key,
384
+ parameter: body,
385
+ opts: clients.SendOpts.from({ idempotencyKey, timeout: 10_000 }),
381
386
  });
382
- if (response.ok) {
383
- await response.body?.cancel();
384
- return;
385
- }
386
- const text = await response.text().catch(() => "");
387
- const error = new Error(`Restate ingress ${response.status} for ${service}/${handler}: ${text}`);
388
- // A just-registered service can take a moment to reach the ingress.
389
- throw response.status >= 500 || response.status === 404 || response.status === 429
390
- ? error
391
- : new PermanentError(error.message);
392
- });
387
+ }
388
+ catch (error) {
389
+ const message = error instanceof clients.HttpCallError
390
+ ? `Restate ingress ${error.status}: ${error.responseText}`
391
+ : error instanceof Error
392
+ ? error.message
393
+ : String(error);
394
+ throw new Error(`Restate send to ${service}/${handler} failed: ${message}`, { cause: error });
395
+ }
393
396
  }
394
397
  /** Registers the Worker endpoint with the admin API, retrying until Restate is up. */
395
398
  async register(uri) {
396
399
  const force = this.options.forceRegistration ?? true;
397
- await withRetry("Restate deployment registration", this.options.registrationTimeoutMs ?? 60_000, async () => {
400
+ const registered = await retry(async () => {
398
401
  const response = await fetch(`${this.adminUrl}/deployments`, {
399
402
  method: "POST",
400
403
  headers: { "content-type": "application/json" },
401
404
  body: JSON.stringify({ uri, force }),
402
405
  signal: AbortSignal.timeout(30_000),
403
406
  });
404
- if (response.ok) {
405
- await response.body?.cancel();
406
- return;
407
- }
407
+ if (response.ok)
408
+ return (await response.json().catch(() => ({})))
409
+ .services;
408
410
  const text = await response.text().catch(() => "");
409
411
  const message = `Restate admin ${response.status} registering ${uri}: ${text}`;
410
412
  // Discovery failures (Restate cannot reach the endpoint yet) come back
411
413
  // as 4xx/5xx; keep trying until the deadline.
412
414
  throw new Error(message);
415
+ }, { minMs: 100, maxMs: 2000, deadlineMs: this.options.registrationTimeoutMs ?? 60_000 }).catch((error) => {
416
+ throw new Error(`Restate deployment registration failed: ${error.message}`, {
417
+ cause: error,
418
+ });
413
419
  });
420
+ // Without `force`, Restate answers a URL already registered with the deployment it has
421
+ // there, without discovering the endpoint again: one with other services is refused here.
422
+ const names = new Set((registered ?? []).map((service) => service.name));
423
+ const missing = Object.values(this.names).filter((name) => !names.has(name));
424
+ if (registered && missing.length > 0)
425
+ throw new Error(`Restate deployment registration failed: ${uri} is registered with other services ` +
426
+ `(missing ${missing.join(", ")}); register this Worker at a URL of its own ` +
427
+ "(NYLORUN_WORKER_VERSION)");
414
428
  }
415
429
  }
416
430
  /**
@@ -501,26 +515,6 @@ function tenantKey(tenantId) {
501
515
  function sqlString(value) {
502
516
  return `'${value.replaceAll("'", "''")}'`;
503
517
  }
504
- class PermanentError extends Error {
505
- }
506
- async function withRetry(what, timeoutMs, attempt) {
507
- const deadline = Date.now() + timeoutMs;
508
- let delay = 100;
509
- for (;;) {
510
- try {
511
- await attempt();
512
- return;
513
- }
514
- catch (error) {
515
- if (error instanceof PermanentError || Date.now() + delay > deadline)
516
- throw new Error(`${what} failed: ${error.message}`, {
517
- cause: error,
518
- });
519
- await new Promise((resolve) => setTimeout(resolve, delay));
520
- delay = Math.min(delay * 2, 2000);
521
- }
522
- }
523
- }
524
518
  function sdkLogger(log) {
525
519
  return (meta, message, ...rest) => {
526
520
  const parts = [message, ...rest].map((part) => part instanceof Error ? part.stack ?? part.message : String(part));
@@ -57,7 +57,7 @@ export function localBackend(options) {
57
57
  return reject(signal.reason ?? new Error("Turn cancelled"));
58
58
  const child = spawn("/bin/sh", ["-c", request.command], {
59
59
  cwd: request.cwd,
60
- env: { ...env, ...(options.proxyEnv?.() ?? {}) },
60
+ env: { ...env, ...(request.env ?? {}), ...(options.proxyEnv?.() ?? {}) },
61
61
  stdio: ["ignore", "pipe", "pipe"],
62
62
  // Its own process group, so a timeout or a cancel kills everything it started.
63
63
  detached: true,
@@ -81,7 +81,7 @@ export function virtualBackend(options) {
81
81
  try {
82
82
  const result = await bash.exec(request.command, {
83
83
  cwd: request.cwd,
84
- env: { ...ENV },
84
+ env: { ...(request.env ?? {}), ...ENV },
85
85
  replaceEnv: true,
86
86
  rawScript: true,
87
87
  signal: controller.signal,
@@ -58,6 +58,7 @@
58
58
  */
59
59
  import { AppendInput, AppendRecord, RangeNotSatisfiableError, S2, S2Error, SeqNumMismatchError, } from "@s2-dev/streamstore";
60
60
  import { parseBasin, tenantBasinName, validateBasinPrefix } from "../../streams/basin.js";
61
+ import { sleep } from "../../retry.js";
61
62
  import {} from "../../streams/types.js";
62
63
  /** Records per unary read while reading history. */
63
64
  const HISTORY_PAGE = 1000;
@@ -357,16 +358,3 @@ function escapeRegExp(text) {
357
358
  function anySignal(...signals) {
358
359
  return AbortSignal.any(signals.filter((s) => s !== undefined));
359
360
  }
360
- function sleep(ms, signal) {
361
- return new Promise((resolve) => {
362
- if (signal.aborted)
363
- return resolve();
364
- const done = () => {
365
- clearTimeout(timer);
366
- signal.removeEventListener("abort", done);
367
- resolve();
368
- };
369
- const timer = setTimeout(done, ms);
370
- signal.addEventListener("abort", done, { once: true });
371
- });
372
- }
@@ -10,6 +10,7 @@ export declare const Rejected: z.ZodObject<{
10
10
  status: z.ZodLiteral<"rejected">;
11
11
  code: z.ZodEnum<{
12
12
  connection_missing: "connection_missing";
13
+ credential_conflict: "credential_conflict";
13
14
  credential_invalid: "credential_invalid";
14
15
  credential_rejected: "credential_rejected";
15
16
  credential_required: "credential_required";
@@ -8376,6 +8377,19 @@ export declare const CreateCredentialRequest: z.ZodObject<{
8376
8377
  type: z.ZodLiteral<"headers">;
8377
8378
  url: z.ZodString;
8378
8379
  headers: z.ZodRecord<z.ZodString, z.ZodString>;
8380
+ }, z.core.$strict>, z.ZodObject<{
8381
+ type: z.ZodLiteral<"environment_secret">;
8382
+ secretName: z.ZodString;
8383
+ secretValue: z.ZodString;
8384
+ allowedHosts: z.ZodArray<z.ZodString>;
8385
+ inject: z.ZodOptional<z.ZodObject<{
8386
+ header: z.ZodString;
8387
+ format: z.ZodString;
8388
+ }, z.core.$strict>>;
8389
+ }, z.core.$strict>, z.ZodObject<{
8390
+ type: z.ZodLiteral<"environment_variable">;
8391
+ variableName: z.ZodString;
8392
+ variableValue: z.ZodString;
8379
8393
  }, z.core.$strict>], "type">;
8380
8394
  }, z.core.$strict>;
8381
8395
  export declare const CredentialInfo: z.ZodObject<{
@@ -8384,10 +8398,20 @@ export declare const CredentialInfo: z.ZodObject<{
8384
8398
  name: z.ZodString;
8385
8399
  type: z.ZodEnum<{
8386
8400
  bearer: "bearer";
8401
+ environment_secret: "environment_secret";
8402
+ environment_variable: "environment_variable";
8387
8403
  headers: "headers";
8388
8404
  }>;
8389
8405
  binding: z.ZodObject<{
8390
- url: z.ZodString;
8406
+ url: z.ZodOptional<z.ZodString>;
8407
+ secretName: z.ZodOptional<z.ZodString>;
8408
+ allowedHosts: z.ZodOptional<z.ZodArray<z.ZodString>>;
8409
+ inject: z.ZodOptional<z.ZodObject<{
8410
+ header: z.ZodString;
8411
+ format: z.ZodString;
8412
+ }, z.core.$strict>>;
8413
+ variableName: z.ZodOptional<z.ZodString>;
8414
+ variableValue: z.ZodOptional<z.ZodString>;
8391
8415
  }, z.core.$strict>;
8392
8416
  headerNames: z.ZodOptional<z.ZodArray<z.ZodString>>;
8393
8417
  via: z.ZodOptional<z.ZodString>;
@@ -8404,10 +8428,20 @@ export declare const ListCredentialsResponse: z.ZodObject<{
8404
8428
  name: z.ZodString;
8405
8429
  type: z.ZodEnum<{
8406
8430
  bearer: "bearer";
8431
+ environment_secret: "environment_secret";
8432
+ environment_variable: "environment_variable";
8407
8433
  headers: "headers";
8408
8434
  }>;
8409
8435
  binding: z.ZodObject<{
8410
- url: z.ZodString;
8436
+ url: z.ZodOptional<z.ZodString>;
8437
+ secretName: z.ZodOptional<z.ZodString>;
8438
+ allowedHosts: z.ZodOptional<z.ZodArray<z.ZodString>>;
8439
+ inject: z.ZodOptional<z.ZodObject<{
8440
+ header: z.ZodString;
8441
+ format: z.ZodString;
8442
+ }, z.core.$strict>>;
8443
+ variableName: z.ZodOptional<z.ZodString>;
8444
+ variableValue: z.ZodOptional<z.ZodString>;
8411
8445
  }, z.core.$strict>;
8412
8446
  headerNames: z.ZodOptional<z.ZodArray<z.ZodString>>;
8413
8447
  via: z.ZodOptional<z.ZodString>;
@@ -8435,6 +8469,12 @@ export declare const RotateCredentialRequest: z.ZodObject<{
8435
8469
  }, z.core.$strict>>>;
8436
8470
  type: z.ZodLiteral<"headers">;
8437
8471
  headers: z.ZodRecord<z.ZodString, z.ZodString>;
8472
+ }, z.core.$strict>, z.ZodObject<{
8473
+ type: z.ZodLiteral<"environment_secret">;
8474
+ secretValue: z.ZodString;
8475
+ }, z.core.$strict>, z.ZodObject<{
8476
+ type: z.ZodLiteral<"environment_variable">;
8477
+ variableValue: z.ZodString;
8438
8478
  }, z.core.$strict>], "type">;
8439
8479
  }, z.core.$strict>;
8440
8480
  export declare const DeletedResponse: z.ZodObject<{
@@ -1,8 +1,7 @@
1
1
  import type { EffectOutcome } from "@nylorun/core/contracts";
2
2
  import { type JsonValue, type WorkflowManifest } from "@nylorun/core/define";
3
3
  import type { HostEffect } from "@nylorun/harness/run";
4
- import type { EffectDoc, Tx, WakeDelivery } from "../store/types.js";
5
- import { type FlowLimits } from "./limits.js";
4
+ import type { EffectDoc, Tx } from "../store/types.js";
6
5
  /** Deterministic agent session id: derive(workflowSessionId, path, …parts). */
7
6
  export declare function deriveSessionId(workflowSessionId: string, path: string, ...parts: string[]): string;
8
7
  /**
@@ -97,8 +96,6 @@ export type CascadeCancelPlan = {
97
96
  /** Linked agent sessions to cancel, deepest path first. */
98
97
  readonly agentSessionIds: string[];
99
98
  };
100
- /** Wakes a session after commit (`DurableExecution.wake` through the Tenant context). */
101
- type Schedule = WakeDelivery;
102
99
  /** Effect documents as the flow host writes them. */
103
100
  export type FlowEffect = EffectDoc & {
104
101
  agentSessionId?: string;
@@ -135,7 +132,6 @@ export declare function wakeLinkedWorkflow(input: {
135
132
  readonly failed?: boolean;
136
133
  readonly cancelled?: boolean;
137
134
  readonly error?: string;
138
- readonly schedule: Schedule;
139
135
  }): Promise<void>;
140
136
  /** Pending workflow `agent` effects with a linked session, for the Tenant sweep. */
141
137
  export declare function pendingAgentEffects(t: Tx): Promise<FlowEffect[]>;
@@ -152,7 +148,6 @@ export declare function pendingAgentEffects(t: Tx): Promise<FlowEffect[]>;
152
148
  export declare function reconcilePendingAgentEffect(input: {
153
149
  readonly t: Tx;
154
150
  readonly effectId: string;
155
- readonly schedule: Schedule;
156
151
  }): Promise<boolean>;
157
152
  /** Path depth for deepest-first cancel ordering. */
158
153
  export declare function pathDepth(path: string): number;
@@ -222,16 +217,3 @@ export declare function foreignInteractionConflict(input: {
222
217
  message: string;
223
218
  ownerSessionId: string;
224
219
  } | undefined>;
225
- /**
226
- * When concurrency slots free, re-enter the workflow so queued effects can start
227
- * (status stays `queued` until the effect resolver dispatches them). Returns
228
- * true when a wake was scheduled.
229
- */
230
- export declare function wakeForQueuedEffects(input: {
231
- readonly t: Tx;
232
- readonly workflowSessionId: string;
233
- readonly turnId: string;
234
- readonly limits: FlowLimits;
235
- readonly schedule: Schedule;
236
- }): Promise<boolean>;
237
- export {};
@@ -4,8 +4,8 @@
4
4
  * Every function that touches state takes the caller's transaction `t: Tx` and
5
5
  * is async. Nothing here publishes, notifies or schedules directly (seam rule
6
6
  * 1): events go through `t.event(...)`, which the store publishes after
7
- * commit, and wakes go through `t.wake(id, wake, schedule)`, the wake outbox, which
8
- * calls `schedule` after commit. Functions
7
+ * commit, and wakes go through `t.wake(id, wake)`, the wake outbox, which the store
8
+ * delivers after commit. Functions
9
9
  * that rewrite a session-scoped document lock that session first with
10
10
  * `t.lockSession` (a no-op when the caller already holds it).
11
11
  *
@@ -24,9 +24,9 @@
24
24
  * - `linkedMessageKey(request): string`
25
25
  * - `linkedTurnOf(t, agentSessionId, request): Promise<string | undefined>`
26
26
  * - `linkedTurnEnd(t, effect, agent): Promise<LinkedTurnEnd | undefined>`
27
- * - `wakeLinkedWorkflow({ t, agentSessionId, turnId, output?, failed?, cancelled?, error?, schedule }): Promise<void>`
27
+ * - `wakeLinkedWorkflow({ t, agentSessionId, turnId, output?, failed?, cancelled?, error? }): Promise<void>`
28
28
  * - `pendingAgentEffects(t): Promise<FlowEffect[]>`
29
- * - `reconcilePendingAgentEffect({ t, effectId, schedule }): Promise<boolean>`
29
+ * - `reconcilePendingAgentEffect({ t, effectId }): Promise<boolean>`
30
30
  * - `planCancelCascade({ t, workflowSessionId, turnId }): Promise<CascadeCancelPlan>`
31
31
  * - `cancelSiblingWork({ t, workflowSessionId, turnId, siblingPaths?, cancelEffectIds? }): Promise<CancelSiblingResult>`
32
32
  * - `cancelQueuedEffects({ t, workflowSessionId, turnId }): Promise<string[]>`
@@ -34,11 +34,10 @@
34
34
  * - `flowInteractionOf(waits, workflowSessionId, interactionId): { id; kind } | undefined`
35
35
  * - `findInteractionOwner({ t, workflowSessionId, interactionId }): Promise<{ sessionId; path } | undefined>`
36
36
  * - `foreignInteractionConflict({ t, workflowSessionId, interactionId }): Promise<{ status: 409; message; ownerSessionId } | undefined>`
37
- * - `wakeForQueuedEffects({ t, workflowSessionId, turnId, limits, schedule }): Promise<boolean>`
38
37
  *
39
- * `schedule` (a `WakeDelivery`, the Tenant's `ctx.wake`) runs after commit, never inside `t`;
40
- * `wake` carries the reason (`linked`, `flow`) and, where the cause has one, a dedupe key. A
41
- * wake `schedule` does not deliver stays in the outbox for the Tenant sweep (`Tx.wake`).
38
+ * A wake carries the reason (`linked`, `flow`) and, where the cause has one, a dedupe key.
39
+ * The store delivers it after commit, never inside `t`; one it does not deliver stays in the
40
+ * outbox for the Tenant sweep (`Tx.wake`).
42
41
  *
43
42
  * An `agent` effect settles only from the linked turn it started. A linked agent session is
44
43
  * reused by every iteration of a Loop (its id derives from the workflow and the path), so its
@@ -51,7 +50,6 @@
51
50
  */
52
51
  import { createHash } from "node:crypto";
53
52
  import { isVerdict } from "@nylorun/core/define";
54
- import { mayDispatchMore } from "./limits.js";
55
53
  /** Deterministic agent session id: derive(workflowSessionId, path, …parts). */
56
54
  export function deriveSessionId(workflowSessionId, path, ...parts) {
57
55
  const digest = createHash("sha256")
@@ -140,10 +138,6 @@ export async function recordVerdict(t, request, value) {
140
138
  ...(verdict.data !== undefined ? { data: verdict.data } : {}),
141
139
  });
142
140
  }
143
- /** Writes the wake to the outbox; `schedule` delivers it after commit. */
144
- function requestWake(t, schedule, id, wake) {
145
- return t.wake(id, wake, schedule);
146
- }
147
141
  /** Active agent turns + tool nodes in flight for one workflow turn. */
148
142
  export async function countActiveFlowWork(t, workflowSessionId, turnId) {
149
143
  const effects = await t.effectsForTurn(workflowSessionId, turnId);
@@ -245,7 +239,7 @@ export async function wakeLinkedWorkflow(input) {
245
239
  await settleAgentEffect({ t, effect, outcome });
246
240
  workflow.status = "runnable";
247
241
  await t.put("sessions", workflow.id, workflow);
248
- await requestWake(t, input.schedule, workflow.id, {
242
+ await t.wake(workflow.id, {
249
243
  reason: "linked",
250
244
  dedupeKey: `linked:${link.turnId}:${link.effectId}`,
251
245
  });
@@ -288,7 +282,6 @@ export async function reconcilePendingAgentEffect(input) {
288
282
  : end.status === "failed"
289
283
  ? { failed: true, error: end.error }
290
284
  : { cancelled: true, error: end.error }),
291
- schedule: input.schedule,
292
285
  });
293
286
  return true;
294
287
  }
@@ -457,25 +450,3 @@ export async function foreignInteractionConflict(input) {
457
450
  ownerSessionId: owner.sessionId,
458
451
  };
459
452
  }
460
- /**
461
- * When concurrency slots free, re-enter the workflow so queued effects can start
462
- * (status stays `queued` until the effect resolver dispatches them). Returns
463
- * true when a wake was scheduled.
464
- */
465
- export async function wakeForQueuedEffects(input) {
466
- const { t } = input;
467
- if (!mayDispatchMore(await countActiveFlowWork(t, input.workflowSessionId, input.turnId), input.limits))
468
- return false;
469
- const queued = await t.effectsForTurn(input.workflowSessionId, input.turnId, ["queued"]);
470
- if (queued.length === 0)
471
- return false;
472
- const workflow = await t.lockSession(input.workflowSessionId);
473
- if (workflow && workflow.status === "waiting") {
474
- workflow.status = "runnable";
475
- await t.put("sessions", workflow.id, workflow);
476
- }
477
- await requestWake(t, input.schedule, input.workflowSessionId, {
478
- reason: "flow",
479
- });
480
- return true;
481
- }