@nylorun/runtime 0.15.1-beta → 0.16.0-beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (220) hide show
  1. package/CHANGELOG.md +82 -0
  2. package/README.md +33 -6
  3. package/dist/adapters/execution/restate.d.ts +9 -1
  4. package/dist/adapters/execution/restate.js +64 -5
  5. package/dist/adapters/replication/pgoutput.js +3 -1
  6. package/dist/adapters/sandbox/local.d.ts +10 -0
  7. package/dist/adapters/sandbox/local.js +195 -0
  8. package/dist/adapters/sandbox/virtual.js +36 -1
  9. package/dist/admin-openapi.json +37 -3
  10. package/dist/api/a2a/tasks.js +6 -1
  11. package/dist/api/ag-ui/history.js +6 -1
  12. package/dist/api/ag-ui/routes.js +5 -1
  13. package/dist/api/components.d.ts +2673 -655
  14. package/dist/api/components.js +17 -1
  15. package/dist/api/gate/keys.d.ts +4 -3
  16. package/dist/api/gate/routes.d.ts +19 -3
  17. package/dist/api/gate/routes.js +162 -53
  18. package/dist/api/http/app.js +4 -0
  19. package/dist/api/http/define.d.ts +10 -0
  20. package/dist/api/http/define.js +2 -1
  21. package/dist/api/http/routes/artifacts.d.ts +11 -0
  22. package/dist/api/http/routes/artifacts.js +518 -0
  23. package/dist/api/http/routes/sandboxes.d.ts +9 -0
  24. package/dist/api/http/routes/sandboxes.js +152 -0
  25. package/dist/api/http/routes/sessions.js +3 -1
  26. package/dist/api/http/routes/tenant.js +36 -2
  27. package/dist/artifacts/config.d.ts +23 -0
  28. package/dist/artifacts/config.js +27 -0
  29. package/dist/artifacts/export.d.ts +8 -0
  30. package/dist/artifacts/export.js +205 -0
  31. package/dist/artifacts/files.d.ts +36 -0
  32. package/dist/artifacts/files.js +45 -0
  33. package/dist/artifacts/folders.d.ts +60 -0
  34. package/dist/artifacts/folders.js +209 -0
  35. package/dist/artifacts/links.d.ts +28 -0
  36. package/dist/artifacts/links.js +92 -0
  37. package/dist/artifacts/media-types.d.ts +13 -0
  38. package/dist/artifacts/media-types.js +80 -0
  39. package/dist/artifacts/parts.d.ts +34 -0
  40. package/dist/artifacts/parts.js +38 -0
  41. package/dist/artifacts/service.d.ts +74 -0
  42. package/dist/artifacts/service.js +301 -0
  43. package/dist/artifacts/workspace.d.ts +60 -0
  44. package/dist/artifacts/workspace.js +57 -0
  45. package/dist/artifacts/zip.d.ts +10 -0
  46. package/dist/artifacts/zip.js +126 -0
  47. package/dist/blob/fs.d.ts +13 -0
  48. package/dist/blob/fs.js +194 -0
  49. package/dist/blob/index.d.ts +9 -0
  50. package/dist/blob/index.js +9 -0
  51. package/dist/blob/s3.d.ts +42 -0
  52. package/dist/blob/s3.js +366 -0
  53. package/dist/blob/types.d.ts +108 -0
  54. package/dist/blob/types.js +85 -0
  55. package/dist/core/sandbox-routes.d.ts +2 -2
  56. package/dist/core/sandbox-routes.js +4 -3
  57. package/dist/execution/memory.d.ts +11 -1
  58. package/dist/execution/memory.js +77 -3
  59. package/dist/execution/types.d.ts +54 -1
  60. package/dist/execution/types.js +31 -0
  61. package/dist/gates/contract.d.ts +14 -13
  62. package/dist/gates/contract.js +13 -12
  63. package/dist/gates/egress.d.ts +72 -0
  64. package/dist/gates/egress.js +319 -0
  65. package/dist/gates/handler.d.ts +3 -0
  66. package/dist/gates/handler.js +3 -2
  67. package/dist/gates/http-client.d.ts +3 -2
  68. package/dist/gates/http-client.js +17 -6
  69. package/dist/gates/in-process.d.ts +13 -1
  70. package/dist/gates/in-process.js +11 -0
  71. package/dist/gates/inflight.d.ts +23 -4
  72. package/dist/gates/inflight.js +18 -4
  73. package/dist/gates/model-gate.d.ts +6 -2
  74. package/dist/gates/tool-calls.d.ts +9 -5
  75. package/dist/gates/tool-calls.js +6 -6
  76. package/dist/gates/tool-client.d.ts +9 -2
  77. package/dist/gates/tool-client.js +56 -22
  78. package/dist/gates/tool-contract.d.ts +12 -7
  79. package/dist/gates/tool-contract.js +10 -4
  80. package/dist/gates/tool-gate.d.ts +1 -0
  81. package/dist/harness/calls.d.ts +52 -0
  82. package/dist/harness/calls.js +113 -0
  83. package/dist/harness/executors.d.ts +22 -0
  84. package/dist/harness/executors.js +96 -0
  85. package/dist/harness/main.d.ts +6 -0
  86. package/dist/harness/main.js +103 -0
  87. package/dist/harness/pod.d.ts +21 -0
  88. package/dist/harness/pod.js +119 -0
  89. package/dist/harness/run-tokens.d.ts +18 -0
  90. package/dist/harness/run-tokens.js +15 -0
  91. package/dist/harness/service.d.ts +46 -0
  92. package/dist/harness/service.js +179 -0
  93. package/dist/harness/ws-client.d.ts +43 -0
  94. package/dist/harness/ws-client.js +122 -0
  95. package/dist/harness/ws-port.d.ts +22 -0
  96. package/dist/harness/ws-port.js +78 -0
  97. package/dist/harness-api/in-process.d.ts +24 -0
  98. package/dist/harness-api/in-process.js +119 -0
  99. package/dist/harness-api/record.d.ts +44 -0
  100. package/dist/harness-api/record.js +250 -0
  101. package/dist/harness-api/renew.d.ts +37 -0
  102. package/dist/harness-api/renew.js +77 -0
  103. package/dist/harness-api/server.d.ts +104 -0
  104. package/dist/harness-api/server.js +408 -0
  105. package/dist/harness-api/start.d.ts +36 -0
  106. package/dist/harness-api/start.js +118 -0
  107. package/dist/harness-api/workspace.d.ts +53 -0
  108. package/dist/harness-api/workspace.js +201 -0
  109. package/dist/harness-api/ws-server.d.ts +34 -0
  110. package/dist/harness-api/ws-server.js +227 -0
  111. package/dist/host/app.js +23 -2
  112. package/dist/host/gates.d.ts +27 -2
  113. package/dist/host/gates.js +46 -1
  114. package/dist/host/http.d.ts +4 -1
  115. package/dist/host/http.js +6 -1
  116. package/dist/host/main.js +135 -27
  117. package/dist/host/stack-config.d.ts +101 -1
  118. package/dist/host/stack-config.js +199 -8
  119. package/dist/mcp/snapshot.d.ts +8 -1
  120. package/dist/mcp/snapshot.js +28 -0
  121. package/dist/media.d.ts +5 -24
  122. package/dist/media.js +5 -0
  123. package/dist/model/pi-model.d.ts +6 -2
  124. package/dist/model/pi-model.js +39 -22
  125. package/dist/node/index.d.ts +1 -1
  126. package/dist/node/index.js +1 -1
  127. package/dist/openapi.json +15245 -8850
  128. package/dist/record/index.d.ts +1 -0
  129. package/dist/record/index.js +1 -0
  130. package/dist/record/sandbox.d.ts +25 -0
  131. package/dist/record/sandbox.js +43 -0
  132. package/dist/sandbox/capability.d.ts +3 -0
  133. package/dist/sandbox/capability.js +10 -0
  134. package/dist/sandbox/egress-token.d.ts +26 -0
  135. package/dist/sandbox/egress-token.js +8 -0
  136. package/dist/sandbox/join.d.ts +37 -0
  137. package/dist/sandbox/join.js +165 -0
  138. package/dist/sandbox/manager.d.ts +99 -34
  139. package/dist/sandbox/manager.js +216 -84
  140. package/dist/sandbox/placement.d.ts +27 -0
  141. package/dist/sandbox/placement.js +44 -0
  142. package/dist/sandbox/pods/client.d.ts +90 -0
  143. package/dist/sandbox/pods/client.js +79 -0
  144. package/dist/sandbox/pods/lifecycle.d.ts +91 -0
  145. package/dist/sandbox/pods/lifecycle.js +178 -0
  146. package/dist/sandbox/pods/name.d.ts +1 -0
  147. package/dist/sandbox/pods/name.js +28 -0
  148. package/dist/sandbox/pods/network-gate.d.ts +25 -0
  149. package/dist/sandbox/pods/network-gate.js +71 -0
  150. package/dist/sandbox/pods/reconcile.d.ts +11 -0
  151. package/dist/sandbox/pods/reconcile.js +152 -0
  152. package/dist/sandbox/pods/spec.d.ts +45 -0
  153. package/dist/sandbox/pods/spec.js +62 -0
  154. package/dist/sandbox/records.d.ts +29 -0
  155. package/dist/sandbox/records.js +79 -0
  156. package/dist/sandbox/resolve.d.ts +5 -0
  157. package/dist/sandbox/resolve.js +3 -2
  158. package/dist/sandbox/session-sandbox.js +7 -5
  159. package/dist/sandbox/share.d.ts +11 -0
  160. package/dist/sandbox/share.js +10 -0
  161. package/dist/sandbox/tenant-config.d.ts +16 -1
  162. package/dist/sandbox/tenant-config.js +9 -0
  163. package/dist/sandbox/types.d.ts +29 -1
  164. package/dist/sandbox/types.js +13 -1
  165. package/dist/store/postgres/drizzle/0004_sandbox_resources.sql +20 -0
  166. package/dist/store/postgres/drizzle/0005_artifacts.sql +27 -0
  167. package/dist/store/postgres/drizzle/0006_folder_artifacts.sql +12 -0
  168. package/dist/store/postgres/drizzle/0007_sandbox_pods.sql +14 -0
  169. package/dist/store/postgres/drizzle/meta/_journal.json +28 -0
  170. package/dist/store/postgres/record-writer.d.ts +6 -1
  171. package/dist/store/postgres/record-writer.js +20 -2
  172. package/dist/store/postgres/schema.d.ts +894 -0
  173. package/dist/store/postgres/schema.js +130 -2
  174. package/dist/store/postgres/store.js +289 -4
  175. package/dist/store/types.d.ts +143 -6
  176. package/dist/streams/control.d.ts +2 -0
  177. package/dist/streams/control.js +5 -0
  178. package/dist/streams/types.d.ts +10 -1
  179. package/dist/tenant/advance.js +218 -160
  180. package/dist/tenant/artifact-tool.d.ts +5 -0
  181. package/dist/tenant/artifact-tool.js +119 -0
  182. package/dist/tenant/commands.js +30 -5
  183. package/dist/tenant/context.d.ts +64 -5
  184. package/dist/tenant/effects.d.ts +19 -13
  185. package/dist/tenant/effects.js +5 -382
  186. package/dist/tenant/history.d.ts +13 -23
  187. package/dist/tenant/history.js +14 -62
  188. package/dist/tenant/host-token.d.ts +92 -0
  189. package/dist/tenant/host-token.js +179 -0
  190. package/dist/tenant/module.js +1 -0
  191. package/dist/tenant/paths.d.ts +2 -1
  192. package/dist/tenant/paths.js +3 -1
  193. package/dist/tenant/reset.d.ts +5 -2
  194. package/dist/tenant/reset.js +22 -2
  195. package/dist/tenant/run-grants.d.ts +49 -0
  196. package/dist/tenant/run-grants.js +62 -0
  197. package/dist/tenant/run-token.d.ts +70 -0
  198. package/dist/tenant/run-token.js +138 -0
  199. package/dist/tenant/runtime.d.ts +38 -0
  200. package/dist/tenant/runtime.js +120 -32
  201. package/dist/tenant/sandboxes.d.ts +119 -0
  202. package/dist/tenant/sandboxes.js +551 -0
  203. package/dist/tenant/session.d.ts +2 -13
  204. package/dist/tenant/session.js +4 -28
  205. package/dist/tenant/sessions.d.ts +5 -0
  206. package/dist/tenant/sessions.js +28 -1
  207. package/dist/tenant/status.d.ts +8 -3
  208. package/dist/tenant/status.js +5 -0
  209. package/dist/tenant/streams.d.ts +7 -0
  210. package/dist/tenant/streams.js +24 -1
  211. package/dist/tenant/sweep.js +37 -4
  212. package/dist/tenant/tokens.js +9 -2
  213. package/dist/tenant/types.d.ts +18 -1
  214. package/dist/tenant/worker.d.ts +11 -5
  215. package/dist/tenant/worker.js +12 -6
  216. package/dist/version.d.ts +1 -1
  217. package/dist/version.js +1 -1
  218. package/package.json +13 -10
  219. package/dist/adapters/media.d.ts +0 -24
  220. package/dist/adapters/media.js +0 -111
@@ -0,0 +1,91 @@
1
+ /**
2
+ * A pod sandbox's lifecycle (blueprint D34, D36), as one pure decision: from the row, the
3
+ * Tenant's lifecycle settings, the Sandbox's status as the sandboxes service sees it and the
4
+ * clock, what to ask of the service, what to write to the row, which events to record, and
5
+ * when to look again. `reconcile.ts` gathers the facts, carries the decision out and runs it
6
+ * again until nothing is left to do; the `Sandbox` object (`execution/`) serializes its runs
7
+ * per sandbox and keeps its timers.
8
+ *
9
+ * - **Create** is eager: a `PUT` of a new pod sandbox asks for `Running` at once. One that is
10
+ * not ready within `READY_DEADLINE_MS` is `failed`, with the pod's reason; the next turn
11
+ * start tries again.
12
+ * - **Every change** the Runtime asks for bumps `rev`; the `PUT` carries `opId =
13
+ * <name>.r<rev>`, so a repeated apply is a no-op at the service, and a Sandbox whose `opId`
14
+ * is not the row's is applied again.
15
+ * - **Idle** (`trigger: idle`): no turn on an attached session, and none ended within the
16
+ * Tenant's `limits.idle` → `Suspended` (`sandbox.suspended`, reason `idle`). The volume is
17
+ * kept; the next turn start asks for `Running` again.
18
+ * - **TTL**: the Sandbox carries `shutdownTime = createdAt + lifecycle.ttl` and the Tenant's
19
+ * `onExpiry` as its `shutdownPolicy`. Once it passes, the sandbox is `expired`
20
+ * (`sandbox.expired`), its host epoch moves (its tokens stop working) and turns are refused
21
+ * (`sandbox_expired`); with `onExpiry: delete` it is deleted. A `PUT` with a longer TTL moves
22
+ * `expiresAt`, and the next apply revives it with the later `shutdownTime`.
23
+ * - **Lost**: an incarnation that has run (a pod joined) whose Sandbox is gone or whose volume
24
+ * is missing is `lost` (`sandbox.lost`): its host epoch moves, its Sandbox is deleted, and
25
+ * only a reset brings the sandbox back. agent-sandbox recreates a deleted claim empty, so the
26
+ * engine's join also checks the id it keeps on the volume (`join.ts`): a new volume is lost
27
+ * too. A pod replaced on the same volume is not lost: the new pod's join bumps the epoch
28
+ * (`sandbox.relaunched`).
29
+ * - **Delete** (`desired: deleted`): the Sandbox is deleted, and the row once it is gone.
30
+ * - **Reset** (`tenant/sandboxes.ts`) bumps the volume generation: a new name, so a new
31
+ * Sandbox and volume; the old name is `retiring` and deleted here.
32
+ */
33
+ import type { SandboxEventPayload, SandboxEventType } from "@nylorun/core/contracts";
34
+ import type { SandboxPodPatch, SandboxPodState, SandboxResource } from "../../store/types.js";
35
+ import type { PodStatus } from "./client.js";
36
+ /** A pod not ready this long after it was asked to run has failed. */
37
+ export declare const READY_DEADLINE_MS: number;
38
+ /** How soon to look again while the pod changes. */
39
+ export declare const POLL_MS = 2000;
40
+ /** How soon to look again when the sandboxes service does not answer. */
41
+ export declare const UNREACHABLE_RETRY_MS = 5000;
42
+ /** The Tenant's lifecycle settings (D36), read each time a decision is made. */
43
+ export interface PodLifecycleConfig {
44
+ readonly onExpiry: "retain" | "delete";
45
+ readonly idleMs: number;
46
+ }
47
+ /** Why the decision is made: a reconcile, or one of the sandbox's timers firing. */
48
+ export type PodTrigger = "reconcile" | "idle" | "ttl";
49
+ export type PodTimer = "idle" | "ttl";
50
+ export type PodRow = SandboxResource & {
51
+ readonly pod: SandboxPodState;
52
+ };
53
+ export interface PodFacts {
54
+ readonly row: PodRow;
55
+ readonly config: PodLifecycleConfig;
56
+ /** The Sandbox's status; undefined when the sandboxes service did not answer. */
57
+ readonly status: PodStatus | undefined;
58
+ /** The retiring Sandbox's status, when the row names one. */
59
+ readonly retiring?: PodStatus | undefined;
60
+ /** A session attached to the sandbox has a turn running. */
61
+ readonly busy: boolean;
62
+ readonly now: number;
63
+ readonly trigger: PodTrigger;
64
+ }
65
+ export interface PodEvent<T extends SandboxEventType = SandboxEventType> {
66
+ readonly type: T;
67
+ readonly payload: SandboxEventPayload<T>;
68
+ }
69
+ export interface PodDecision {
70
+ /** Apply the row's spec (after `patch`) in this mode; `rotateJoin` mints a new join token. */
71
+ readonly apply?: {
72
+ readonly mode: "Running" | "Suspended";
73
+ readonly rotateJoin: boolean;
74
+ };
75
+ /** Sandboxes to delete, by name. */
76
+ readonly delete: readonly string[];
77
+ readonly patch: SandboxPodPatch;
78
+ readonly events: readonly PodEvent[];
79
+ /** The Sandbox is gone and the sandbox deleted: remove the row. */
80
+ readonly removeRow?: true;
81
+ /** Look again this soon. */
82
+ readonly retryAfterMs?: number;
83
+ /** Timers to (re)arm, at ms since the epoch. */
84
+ readonly arm: readonly {
85
+ readonly timer: PodTimer;
86
+ readonly at: number;
87
+ }[];
88
+ }
89
+ /** The service's operation id of the row's current spec. */
90
+ export declare function opIdOf(pod: Pick<SandboxPodState, "k8sName" | "rev">): string;
91
+ export declare function decide(facts: PodFacts): PodDecision;
@@ -0,0 +1,178 @@
1
+ /** A pod not ready this long after it was asked to run has failed. */
2
+ export const READY_DEADLINE_MS = 5 * 60_000;
3
+ /** How soon to look again while the pod changes. */
4
+ export const POLL_MS = 2_000;
5
+ /** How soon to look again when the sandboxes service does not answer. */
6
+ export const UNREACHABLE_RETRY_MS = 5_000;
7
+ /** The service's operation id of the row's current spec. */
8
+ export function opIdOf(pod) {
9
+ return `${pod.k8sName}.r${pod.rev}`;
10
+ }
11
+ /** No Sandbox, pod or volume is left. */
12
+ function gone(status) {
13
+ return !status.exists && !status.podPhase && status.volume === "missing";
14
+ }
15
+ const event = (type, payload) => ({ type, payload });
16
+ export function decide(facts) {
17
+ const { row, status, now } = facts;
18
+ const pod = row.pod;
19
+ const patch = {};
20
+ const events = [];
21
+ const arm = [];
22
+ const deletes = [];
23
+ let retryAfterMs;
24
+ const again = (ms) => {
25
+ retryAfterMs = retryAfterMs === undefined ? ms : Math.min(retryAfterMs, ms);
26
+ };
27
+ const done = (extra = {}) => ({
28
+ ...extra,
29
+ delete: deletes,
30
+ patch,
31
+ events,
32
+ arm,
33
+ ...(retryAfterMs === undefined ? {} : { retryAfterMs }),
34
+ });
35
+ // An earlier incarnation's Sandbox (reset, loss) is deleted until it is gone.
36
+ if (pod.retiring !== undefined) {
37
+ const retiring = facts.retiring;
38
+ if (retiring === undefined)
39
+ again(UNREACHABLE_RETRY_MS);
40
+ else if (gone(retiring))
41
+ patch.retiring = null;
42
+ else {
43
+ if (retiring.exists && !retiring.deleting)
44
+ deletes.push(pod.retiring);
45
+ again(POLL_MS);
46
+ }
47
+ }
48
+ if (status === undefined) {
49
+ again(UNREACHABLE_RETRY_MS);
50
+ return done();
51
+ }
52
+ if (pod.desired === "deleted") {
53
+ if (gone(status) && (pod.retiring === undefined || patch.retiring === null))
54
+ return done({ removeRow: true });
55
+ if (status.exists && !status.deleting)
56
+ deletes.push(pod.k8sName);
57
+ if (pod.observed !== "deleting")
58
+ patch.observed = "deleting";
59
+ again(POLL_MS);
60
+ return done();
61
+ }
62
+ // Lost is terminal: only a reset (a new name) leaves it.
63
+ if (pod.observed === "lost") {
64
+ if (status.exists && !status.deleting)
65
+ deletes.push(pod.k8sName);
66
+ if (status.exists)
67
+ again(POLL_MS);
68
+ return done();
69
+ }
70
+ const expiresAt = pod.expiresAt === undefined ? undefined : Date.parse(pod.expiresAt);
71
+ if (expiresAt !== undefined && now >= expiresAt) {
72
+ if (pod.observed !== "expired") {
73
+ patch.observed = "expired";
74
+ patch.hostEpoch = pod.hostEpoch + 1;
75
+ events.push(event("sandbox.expired", { onExpiry: facts.config.onExpiry }));
76
+ }
77
+ if (facts.config.onExpiry === "delete") {
78
+ patch.desired = "deleted";
79
+ patch.rev = pod.rev + 1;
80
+ if (status.exists && !status.deleting)
81
+ deletes.push(pod.k8sName);
82
+ again(POLL_MS);
83
+ }
84
+ return done();
85
+ }
86
+ if (expiresAt !== undefined)
87
+ arm.push({ timer: "ttl", at: expiresAt });
88
+ let desired = pod.desired;
89
+ let rev = pod.rev;
90
+ if (facts.trigger === "idle" && desired === "running" && !facts.busy) {
91
+ const last = Date.parse(pod.lastActiveAt ?? row.createdAt);
92
+ const due = last + facts.config.idleMs;
93
+ if (now >= due) {
94
+ desired = "suspended";
95
+ rev += 1;
96
+ patch.desired = desired;
97
+ patch.rev = rev;
98
+ patch.hostEpoch = pod.hostEpoch + 1;
99
+ events.push(event("sandbox.suspended", { reason: "idle" }));
100
+ }
101
+ else
102
+ arm.push({ timer: "idle", at: due });
103
+ }
104
+ // An incarnation a pod has joined, whose Sandbox or volume is gone: lost.
105
+ if (pod.podUid !== undefined && (!status.exists || status.deleting || status.volume === "missing")) {
106
+ const reason = !status.exists
107
+ ? "The sandbox's pod and volume were deleted outside the Runtime"
108
+ : status.deleting
109
+ ? "The sandbox is being deleted outside the Runtime"
110
+ : "The sandbox's volume is gone";
111
+ patch.observed = "lost";
112
+ patch.reason = reason;
113
+ patch.hostEpoch = (patch.hostEpoch ?? pod.hostEpoch) + 1;
114
+ events.push(event("sandbox.lost", { reason }));
115
+ if (status.exists && !status.deleting)
116
+ deletes.push(pod.k8sName);
117
+ if (status.exists)
118
+ again(POLL_MS);
119
+ return done();
120
+ }
121
+ if (status.deleting) {
122
+ // The previous incarnation of this name is still going (a sandbox deleted and created again).
123
+ again(POLL_MS);
124
+ return done();
125
+ }
126
+ const opId = opIdOf({ k8sName: pod.k8sName, rev });
127
+ if (desired === "suspended" && !status.exists) {
128
+ if (pod.observed !== "suspended")
129
+ patch.observed = "suspended";
130
+ return done();
131
+ }
132
+ if (status.opId !== opId) {
133
+ if (desired === "running" && (pod.observed !== "creating" || pod.startedAt === undefined)) {
134
+ patch.observed = "creating";
135
+ patch.startedAt = new Date(now).toISOString();
136
+ patch.reason = null;
137
+ }
138
+ // A pod started by this apply (a create, a resume) is the sandbox's first, not a relaunch.
139
+ if (desired === "running" && pod.podUid !== undefined)
140
+ patch.podUid = null;
141
+ again(POLL_MS);
142
+ return done({
143
+ apply: { mode: desired === "running" ? "Running" : "Suspended", rotateJoin: !status.exists },
144
+ });
145
+ }
146
+ if (desired === "running") {
147
+ if (status.ready) {
148
+ if (pod.observed !== "running") {
149
+ patch.observed = "running";
150
+ patch.reason = null;
151
+ }
152
+ return done();
153
+ }
154
+ const startedAt = Date.parse(pod.startedAt ?? row.updatedAt);
155
+ if (pod.observed === "failed")
156
+ return done();
157
+ if (now - startedAt >= READY_DEADLINE_MS) {
158
+ const reason = [status.reason, status.message].filter(Boolean).join(": ") ||
159
+ `The pod was not ready within ${READY_DEADLINE_MS / 60_000} minutes`;
160
+ patch.observed = "failed";
161
+ patch.reason = reason;
162
+ events.push(event("sandbox.failed", { reason }));
163
+ return done();
164
+ }
165
+ if (pod.observed !== "creating")
166
+ patch.observed = "creating";
167
+ again(POLL_MS);
168
+ return done();
169
+ }
170
+ // Suspended, applied: suspended once its pod is gone.
171
+ if (status.suspended || !status.podPhase) {
172
+ if (pod.observed !== "suspended")
173
+ patch.observed = "suspended";
174
+ return done();
175
+ }
176
+ again(POLL_MS);
177
+ return done();
178
+ }
@@ -0,0 +1 @@
1
+ export declare function podName(tenantId: string, sandboxId: string, volumeGen: number): string;
@@ -0,0 +1,28 @@
1
+ /**
2
+ * The Kubernetes name of a pod sandbox's incarnation: `sbx-<lowercase base32(sha256(
3
+ * "<tenant>/<id>"))[:16]>-g<volumeGen>`, as the sandboxes service computes it (`driver.Name` in
4
+ * Go; `podName("shop", "sbx_01", 0)` is `sbx-ub6g5m7mvjlct6r7-g0`). A reset bumps the volume
5
+ * generation, so each volume belongs to one name.
6
+ */
7
+ import { createHash } from "node:crypto";
8
+ const ALPHABET = "abcdefghijklmnopqrstuvwxyz234567";
9
+ function base32(bytes) {
10
+ let bits = 0;
11
+ let value = 0;
12
+ let out = "";
13
+ for (const byte of bytes) {
14
+ value = ((value << 8) | byte) & 0xffff;
15
+ bits += 8;
16
+ while (bits >= 5) {
17
+ out += ALPHABET[(value >>> (bits - 5)) & 31];
18
+ bits -= 5;
19
+ }
20
+ }
21
+ if (bits > 0)
22
+ out += ALPHABET[(value << (5 - bits)) & 31];
23
+ return out;
24
+ }
25
+ export function podName(tenantId, sandboxId, volumeGen) {
26
+ const digest = createHash("sha256").update(`${tenantId}/${sandboxId}`).digest();
27
+ return `sbx-${base32(digest).slice(0, 16)}-g${volumeGen}`;
28
+ }
@@ -0,0 +1,25 @@
1
+ export interface BlockedAddress {
2
+ readonly host: string;
3
+ readonly port: number;
4
+ }
5
+ /** Consecutive failed connections that prove an address blocked. */
6
+ export declare const CONFIRMATIONS = 3;
7
+ /** The addresses a pod's NetworkPolicy must block: the API server's service address. */
8
+ export declare function blockedAddresses(env: Readonly<Record<string, string | undefined>>): BlockedAddress[];
9
+ /** Whether a TCP connection to `address` opens within `timeoutMs`. */
10
+ export declare function opens(address: BlockedAddress, timeoutMs: number): Promise<boolean>;
11
+ export interface NetworkGateOptions {
12
+ readonly addresses: readonly BlockedAddress[];
13
+ /** Give up after this long. Default 120 s. */
14
+ readonly timeoutMs?: number;
15
+ /** One connection attempt. Default 1 s. */
16
+ readonly attemptMs?: number;
17
+ /** Between attempts after a connection opened. Default 250 ms. */
18
+ readonly pauseMs?: number;
19
+ readonly confirmations?: number;
20
+ readonly log?: (message: string, fields: Record<string, unknown>) => void;
21
+ }
22
+ /** Resolves once every address is blocked; rejects past the timeout. */
23
+ export declare function awaitNetworkPolicy(options: NetworkGateOptions): Promise<{
24
+ waitedMs: number;
25
+ }>;
@@ -0,0 +1,71 @@
1
+ /**
2
+ * The engine in a pod sandbox stays off the network until the pod's NetworkPolicy is in force
3
+ * (F7.2). kindnet applies a new pod's policy asynchronously: a fresh pod reached the API server
4
+ * (`10.96.0.1:443`) for 0.5 to 5 s after it started (8a). So before the engine joins, or runs
5
+ * anything, it connects to addresses the policy must block (the API server's service address,
6
+ * `KUBERNETES_SERVICE_HOST:KUBERNETES_SERVICE_PORT`, which the kubelet always sets) until each
7
+ * fails `CONFIRMATIONS` times in a row (refused, unreachable or timed out). A connection that
8
+ * succeeds starts the count again. Past `timeoutMs` the engine exits, and the pod restarts it.
9
+ */
10
+ import { connect } from "node:net";
11
+ /** Consecutive failed connections that prove an address blocked. */
12
+ export const CONFIRMATIONS = 3;
13
+ /** The addresses a pod's NetworkPolicy must block: the API server's service address. */
14
+ export function blockedAddresses(env) {
15
+ const override = env.NYLORUN_SANDBOX_BLOCKED_PROBE;
16
+ if (override) {
17
+ return override
18
+ .split(",")
19
+ .map((entry) => entry.trim())
20
+ .filter(Boolean)
21
+ .map((entry) => {
22
+ const at = entry.lastIndexOf(":");
23
+ return { host: entry.slice(0, at).replace(/^\[|\]$/g, ""), port: Number(entry.slice(at + 1)) };
24
+ });
25
+ }
26
+ const host = env.KUBERNETES_SERVICE_HOST ?? "10.96.0.1";
27
+ const port = Number(env.KUBERNETES_SERVICE_PORT ?? "443");
28
+ return [{ host, port }];
29
+ }
30
+ /** Whether a TCP connection to `address` opens within `timeoutMs`. */
31
+ export function opens(address, timeoutMs) {
32
+ return new Promise((resolve) => {
33
+ const socket = connect({ host: address.host, port: address.port });
34
+ const done = (open) => {
35
+ clearTimeout(timer);
36
+ socket.removeAllListeners();
37
+ socket.on("error", () => undefined);
38
+ socket.destroy();
39
+ resolve(open);
40
+ };
41
+ const timer = setTimeout(() => done(false), timeoutMs);
42
+ socket.once("connect", () => done(true));
43
+ socket.once("error", () => done(false));
44
+ });
45
+ }
46
+ /** Resolves once every address is blocked; rejects past the timeout. */
47
+ export async function awaitNetworkPolicy(options) {
48
+ const started = Date.now();
49
+ const deadline = started + (options.timeoutMs ?? 120_000);
50
+ const needed = options.confirmations ?? CONFIRMATIONS;
51
+ for (const address of options.addresses) {
52
+ let failures = 0;
53
+ let opened = 0;
54
+ while (failures < needed) {
55
+ if (Date.now() > deadline)
56
+ throw new Error(`The pod's NetworkPolicy is not in force: ${address.host}:${address.port} still answered ${opened} time(s)`);
57
+ if (await opens(address, options.attemptMs ?? 1_000)) {
58
+ failures = 0;
59
+ opened += 1;
60
+ await new Promise((resolve) => setTimeout(resolve, options.pauseMs ?? 250));
61
+ }
62
+ else
63
+ failures += 1;
64
+ }
65
+ options.log?.("sandbox_network_policy_checked", {
66
+ address: `${address.host}:${address.port}`,
67
+ openedBefore: opened,
68
+ });
69
+ }
70
+ return { waitedMs: Date.now() - started };
71
+ }
@@ -0,0 +1,11 @@
1
+ import type { TenantContext } from "../../tenant/context.js";
2
+ import { type PodTimer, type PodTrigger } from "./lifecycle.js";
3
+ /** What a reconcile asks of the `Sandbox` object next. */
4
+ export interface SandboxReconcileResult {
5
+ readonly retryAfterMs?: number;
6
+ readonly arm?: readonly {
7
+ readonly timer: PodTimer;
8
+ readonly at: number;
9
+ }[];
10
+ }
11
+ export declare function reconcileSandbox(ctx: TenantContext, sandboxId: string, trigger: PodTrigger): Promise<SandboxReconcileResult>;
@@ -0,0 +1,152 @@
1
+ /**
2
+ * One reconcile of a pod sandbox (S5): the `Sandbox` object's handler
3
+ * (`WorkerHandlers.sandbox`, serialized per sandbox). It reads the row and the Tenant's
4
+ * settings, asks the sandboxes service for the Sandbox's status, decides (`lifecycle.ts`),
5
+ * then carries the decision out:
6
+ *
7
+ * 1. a join token rotation is written first (its hash and a new `rev`), so the token the
8
+ * service writes to the join Secret is always the one the row names;
9
+ * 2. deletes and the apply go to the service, outside any transaction;
10
+ * 3. the row's patch and the events are written under the row's lock, unless the row changed
11
+ * since it was read (an API call or a join): then nothing is written and the reconcile runs
12
+ * again at once. A host epoch the decision moves is recomputed from the locked row.
13
+ *
14
+ * It answers when to look again (`retryAfterMs`) and the timers to arm (idle, TTL). Only
15
+ * infrastructure errors throw; Restate retries those.
16
+ */
17
+ import { randomBytes } from "node:crypto";
18
+ import { readSandboxConfig } from "../tenant-config.js";
19
+ import { sha256Hex } from "../join.js";
20
+ import { decide } from "./lifecycle.js";
21
+ import { podLifecycleConfig, podSpecOf } from "./spec.js";
22
+ export async function reconcileSandbox(ctx, sandboxId, trigger) {
23
+ const pods = ctx.pods;
24
+ if (!pods || ctx.closed)
25
+ return {};
26
+ const read = await ctx.store.tx(async (t) => {
27
+ const row = await t.sandboxResource(sandboxId);
28
+ if (!row || row.kind !== "pod" || !row.pod)
29
+ return undefined;
30
+ const config = await readSandboxConfig(t);
31
+ const busy = (await t.sessionsOnSandbox(sandboxId)).some((session) => session.activeTurnId != null);
32
+ return { row: row, config, busy };
33
+ });
34
+ if (!read)
35
+ return {};
36
+ const { row, config } = read;
37
+ const status = async (name) => {
38
+ try {
39
+ return await pods.client.status(name);
40
+ }
41
+ catch (error) {
42
+ ctx.config.logger.warn("sandboxes service status failed", {
43
+ sandboxId,
44
+ name,
45
+ message: error instanceof Error ? error.message : String(error),
46
+ });
47
+ return undefined;
48
+ }
49
+ };
50
+ const lifecycle = podLifecycleConfig(config);
51
+ const decision = decide({
52
+ row,
53
+ config: lifecycle,
54
+ status: await status(row.pod.k8sName),
55
+ ...(row.pod.retiring ? { retiring: await status(row.pod.retiring) } : {}),
56
+ busy: read.busy,
57
+ now: Date.now(),
58
+ trigger,
59
+ });
60
+ let patch = { ...decision.patch };
61
+ const unchanged = (current) => current !== undefined &&
62
+ current.rev === row.pod.rev &&
63
+ current.k8sName === row.pod.k8sName &&
64
+ current.desired === row.pod.desired;
65
+ // A new join token is recorded before the service writes it to the Sandbox's Secret.
66
+ let joinToken;
67
+ let rev = patch.rev ?? row.pod.rev;
68
+ if (decision.apply?.rotateJoin) {
69
+ joinToken = randomBytes(32).toString("base64url");
70
+ rev += 1;
71
+ const recorded = await ctx.store.tx(async (t) => {
72
+ const current = await t.sandboxResource(sandboxId, { lock: true });
73
+ if (!unchanged(current?.pod))
74
+ return false;
75
+ await t.updateSandboxPod(sandboxId, { joinTokenHash: sha256Hex(joinToken), rev }, new Date().toISOString());
76
+ return true;
77
+ });
78
+ if (!recorded)
79
+ return { retryAfterMs: 0 };
80
+ row.pod.rev = rev;
81
+ row.pod.joinTokenHash = sha256Hex(joinToken);
82
+ patch = { ...patch, rev };
83
+ }
84
+ for (const name of decision.delete) {
85
+ try {
86
+ await pods.client.delete(name, `${name}.delete`);
87
+ }
88
+ catch (error) {
89
+ ctx.config.logger.warn("sandboxes service delete failed", {
90
+ sandboxId,
91
+ name,
92
+ message: error instanceof Error ? error.message : String(error),
93
+ });
94
+ }
95
+ }
96
+ if (decision.apply) {
97
+ const spec = podSpecOf({
98
+ sandboxId,
99
+ spec: row.spec,
100
+ pod: { ...row.pod, rev },
101
+ config,
102
+ mode: decision.apply.mode,
103
+ harnessImage: pods.harnessImage,
104
+ ...(joinToken ? { joinToken } : {}),
105
+ });
106
+ try {
107
+ await pods.client.put(row.pod.k8sName, spec);
108
+ }
109
+ catch (error) {
110
+ ctx.config.logger.warn("sandboxes service apply failed", {
111
+ sandboxId,
112
+ name: row.pod.k8sName,
113
+ message: error instanceof Error ? error.message : String(error),
114
+ });
115
+ // Nothing of the decision is written: the next reconcile decides again.
116
+ return { retryAfterMs: decision.retryAfterMs ?? 5_000 };
117
+ }
118
+ }
119
+ const written = await ctx.store.tx(async (t) => {
120
+ const current = await t.sandboxResource(sandboxId, { lock: true });
121
+ if (!current?.pod)
122
+ return { kind: "gone" };
123
+ if (!unchanged(current.pod))
124
+ return { kind: "changed" };
125
+ if (decision.removeRow) {
126
+ await t.deleteSandboxResource(sandboxId);
127
+ return { kind: "removed" };
128
+ }
129
+ const epochMoved = patch.hostEpoch !== undefined && patch.hostEpoch !== null;
130
+ const finalPatch = {
131
+ ...patch,
132
+ ...(epochMoved ? { hostEpoch: current.pod.hostEpoch + 1 } : {}),
133
+ };
134
+ if (Object.keys(finalPatch).length > 0)
135
+ await t.updateSandboxPod(sandboxId, finalPatch, new Date().toISOString());
136
+ for (const item of decision.events)
137
+ await t.sandboxEvent(sandboxId, item.type, item.payload);
138
+ return { kind: "written", epoch: epochMoved ? current.pod.hostEpoch + 1 : undefined };
139
+ });
140
+ if (written.kind === "changed")
141
+ return { retryAfterMs: 0 };
142
+ if (written.kind === "removed") {
143
+ await ctx.sandbox.removeSandbox(sandboxId).catch(() => undefined);
144
+ return {};
145
+ }
146
+ if (written.kind === "written" && written.epoch !== undefined)
147
+ ctx.harness.revokeHost(sandboxId, written.epoch);
148
+ return {
149
+ ...(decision.retryAfterMs === undefined ? {} : { retryAfterMs: decision.retryAfterMs }),
150
+ ...(decision.arm.length > 0 ? { arm: decision.arm } : {}),
151
+ };
152
+ }
@@ -0,0 +1,45 @@
1
+ /**
2
+ * The `PUT /v1/pods/{name}` body of a pod sandbox: its resolved spec, the Tenant's lifecycle
3
+ * settings (D36, read at every apply) and the incarnation's fences. The pod shape itself is
4
+ * rendered only by the sandboxes service.
5
+ */
6
+ import { type SandboxManifest } from "@nylorun/core/define";
7
+ import type { TenantSandboxConfig } from "@nylorun/core/contracts";
8
+ import type { SandboxPodState } from "../../store/types.js";
9
+ import type { PodSpec } from "./client.js";
10
+ import { type PodLifecycleConfig } from "./lifecycle.js";
11
+ /** The pod's grace period to stop when the Tenant sets none (D34). */
12
+ export declare const DEFAULT_STOP_GRACE_SECONDS = 10;
13
+ /** The volume's size when the sandbox asks for none. */
14
+ export declare const DEFAULT_STORAGE_GIB = 5;
15
+ /** Where the engine in the pod keeps its records, plugin data and home (on the volume). */
16
+ export declare const POD_HARNESS_ROOT = "/harness";
17
+ /** What a pod sandbox's spec holds beyond a virtual one's. */
18
+ export type PodSandboxSpec = SandboxManifest & {
19
+ readonly storage?: string;
20
+ readonly lifecycle?: {
21
+ readonly ttl?: string;
22
+ };
23
+ };
24
+ /** The Tenant's lifecycle settings with their defaults. */
25
+ export declare function podLifecycleConfig(config: TenantSandboxConfig): PodLifecycleConfig & {
26
+ readonly stopGraceSeconds: number;
27
+ };
28
+ /** The volume's size in GiB. */
29
+ export declare function storageGiBOf(spec: PodSandboxSpec): number;
30
+ /** The Tenant's default image, when its default sandbox names one. */
31
+ export declare function tenantDefaultImage(config: TenantSandboxConfig): string | undefined;
32
+ export interface PodSpecInput {
33
+ readonly sandboxId: string;
34
+ readonly spec: PodSandboxSpec;
35
+ readonly pod: SandboxPodState;
36
+ readonly config: TenantSandboxConfig;
37
+ readonly mode: "Running" | "Suspended";
38
+ /** The Runtime image the engine is copied from. */
39
+ readonly harnessImage: string;
40
+ /** The join token, when this apply rotates it. */
41
+ readonly joinToken?: string;
42
+ }
43
+ export declare function podSpecOf(input: PodSpecInput): PodSpec;
44
+ /** When a sandbox with this TTL, created at `createdAt`, expires. */
45
+ export declare function expiresAtOf(createdAt: string, ttl: string | undefined): string | undefined;
@@ -0,0 +1,62 @@
1
+ /**
2
+ * The `PUT /v1/pods/{name}` body of a pod sandbox: its resolved spec, the Tenant's lifecycle
3
+ * settings (D36, read at every apply) and the incarnation's fences. The pod shape itself is
4
+ * rendered only by the sandboxes service.
5
+ */
6
+ import { parseSandboxDuration, parseSandboxSize } from "@nylorun/core/define";
7
+ import { memoryMiBOf } from "../policy.js";
8
+ import { opIdOf } from "./lifecycle.js";
9
+ /** The pod's grace period to stop when the Tenant sets none (D34). */
10
+ export const DEFAULT_STOP_GRACE_SECONDS = 10;
11
+ /** The volume's size when the sandbox asks for none. */
12
+ export const DEFAULT_STORAGE_GIB = 5;
13
+ /** Where the engine in the pod keeps its records, plugin data and home (on the volume). */
14
+ export const POD_HARNESS_ROOT = "/harness";
15
+ /** The Tenant's lifecycle settings with their defaults. */
16
+ export function podLifecycleConfig(config) {
17
+ const stopGrace = config.lifecycle?.stopGrace;
18
+ return {
19
+ onExpiry: config.lifecycle?.onExpiry ?? "retain",
20
+ idleMs: parseSandboxDuration(config.limits?.idle ?? "15m") ?? 15 * 60_000,
21
+ stopGraceSeconds: stopGrace === undefined
22
+ ? DEFAULT_STOP_GRACE_SECONDS
23
+ : Math.max(1, Math.min(30, Math.round((parseSandboxDuration(stopGrace) ?? 10_000) / 1000))),
24
+ };
25
+ }
26
+ /** The volume's size in GiB. */
27
+ export function storageGiBOf(spec) {
28
+ const bytes = spec.storage === undefined ? undefined : parseSandboxSize(spec.storage);
29
+ return bytes === undefined ? DEFAULT_STORAGE_GIB : Math.max(1, Math.ceil(bytes / 1024 ** 3));
30
+ }
31
+ /** The Tenant's default image, when its default sandbox names one. */
32
+ export function tenantDefaultImage(config) {
33
+ const fallback = config.default;
34
+ return typeof fallback === "object" ? fallback.image : undefined;
35
+ }
36
+ export function podSpecOf(input) {
37
+ const { spec, pod, config } = input;
38
+ const lifecycle = podLifecycleConfig(config);
39
+ const image = spec.image ?? tenantDefaultImage(config);
40
+ return {
41
+ opId: opIdOf(pod),
42
+ mode: input.mode,
43
+ ...(image ? { image } : {}),
44
+ harnessImage: input.harnessImage,
45
+ cpus: spec.resources?.cpus ?? 1,
46
+ memoryMiB: memoryMiBOf(spec),
47
+ storageGiB: storageGiBOf(spec),
48
+ stopGraceSeconds: lifecycle.stopGraceSeconds,
49
+ ...(pod.expiresAt ? { shutdownTime: new Date(pod.expiresAt).toISOString().replace(/\.\d{3}Z$/, "Z") } : {}),
50
+ shutdownPolicy: lifecycle.onExpiry === "delete" ? "Delete" : "Retain",
51
+ env: {
52
+ NYLORUN_SANDBOX_ID: input.sandboxId,
53
+ NYLORUN_HARNESS_ROOT: POD_HARNESS_ROOT,
54
+ },
55
+ ...(input.joinToken ? { joinToken: input.joinToken } : {}),
56
+ };
57
+ }
58
+ /** When a sandbox with this TTL, created at `createdAt`, expires. */
59
+ export function expiresAtOf(createdAt, ttl) {
60
+ const ms = ttl === undefined ? undefined : parseSandboxDuration(ttl);
61
+ return ms === undefined ? undefined : new Date(Date.parse(createdAt) + ms).toISOString();
62
+ }