@nylorun/runtime 0.17.0-beta → 0.19.1-beta

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (223) hide show
  1. package/CHANGELOG.md +155 -0
  2. package/README.md +89 -58
  3. package/dist/adapters/execution/restate.d.ts +1 -3
  4. package/dist/adapters/execution/restate.js +4 -31
  5. package/dist/adapters/replication/pgoutput.js +1 -1
  6. package/dist/adapters/sandbox/local.js +1 -1
  7. package/dist/adapters/sandbox/virtual.js +1 -1
  8. package/dist/api/ag-ui/routes.js +1 -1
  9. package/dist/api/ag-ui/translate.d.ts +1 -3
  10. package/dist/api/ag-ui/translate.js +2 -17
  11. package/dist/api/components.d.ts +4871 -2614
  12. package/dist/api/components.js +12 -11
  13. package/dist/api/gate/routes.d.ts +7 -8
  14. package/dist/api/gate/routes.js +58 -52
  15. package/dist/api/http/app.d.ts +6 -0
  16. package/dist/api/http/app.js +18 -14
  17. package/dist/api/http/define.d.ts +14 -2
  18. package/dist/api/http/define.js +34 -15
  19. package/dist/api/http/routes/access.d.ts +3 -3
  20. package/dist/api/http/routes/access.js +18 -17
  21. package/dist/api/http/routes/artifacts.js +2 -4
  22. package/dist/api/http/routes/files.d.ts +8 -0
  23. package/dist/api/http/routes/files.js +77 -0
  24. package/dist/api/http/routes/keys.d.ts +10 -0
  25. package/dist/api/http/routes/keys.js +64 -0
  26. package/dist/api/http/routes/me.js +5 -2
  27. package/dist/api/http/routes/reads.d.ts +66 -0
  28. package/dist/api/http/routes/reads.js +127 -0
  29. package/dist/api/http/routes/sandboxes.js +7 -4
  30. package/dist/api/http/routes/sessions.js +25 -8
  31. package/dist/api/http/routes/tenant.js +15 -18
  32. package/dist/api/http/routes/vaults.d.ts +10 -9
  33. package/dist/api/http/routes/vaults.js +27 -23
  34. package/dist/api/openapi.d.ts +10 -11
  35. package/dist/api/openapi.js +288 -63
  36. package/dist/artifacts/links.js +1 -1
  37. package/dist/core/flow-host.d.ts +44 -23
  38. package/dist/core/flow-host.js +88 -106
  39. package/dist/core/provider.d.ts +4 -1
  40. package/dist/core/provider.js +6 -1
  41. package/dist/core/sandbox-routes.d.ts +0 -9
  42. package/dist/core/sandbox-routes.js +2 -21
  43. package/dist/execution/memory.d.ts +4 -5
  44. package/dist/execution/memory.js +11 -25
  45. package/dist/execution/types.d.ts +3 -33
  46. package/dist/execution/types.js +2 -12
  47. package/dist/gates/http-tool.d.ts +86 -0
  48. package/dist/gates/http-tool.js +197 -0
  49. package/dist/gates/mcp-handler.d.ts +3 -0
  50. package/dist/gates/mcp-handler.js +1 -3
  51. package/dist/gates/meter.d.ts +2 -2
  52. package/dist/gates/meter.js +6 -0
  53. package/dist/gates/tenant-vaults.d.ts +5 -9
  54. package/dist/gates/tool-calls.d.ts +7 -15
  55. package/dist/gates/tool-calls.js +4 -4
  56. package/dist/gates/tool-client.d.ts +1 -1
  57. package/dist/gates/tool-client.js +23 -47
  58. package/dist/gates/tool-contract.d.ts +27 -39
  59. package/dist/gates/tool-contract.js +30 -21
  60. package/dist/gates/tool-gate.d.ts +25 -24
  61. package/dist/gates/tool-gate.js +11 -7
  62. package/dist/harness/calls.d.ts +10 -4
  63. package/dist/harness/calls.js +15 -6
  64. package/dist/harness/executors.d.ts +1 -1
  65. package/dist/harness/executors.js +17 -12
  66. package/dist/harness/main.d.ts +2 -2
  67. package/dist/harness/main.js +5 -13
  68. package/dist/harness/service.d.ts +4 -5
  69. package/dist/harness/service.js +9 -2
  70. package/dist/harness-api/in-process.js +2 -2
  71. package/dist/harness-api/record.d.ts +6 -3
  72. package/dist/harness-api/record.js +65 -72
  73. package/dist/harness-api/server.d.ts +1 -9
  74. package/dist/harness-api/server.js +19 -17
  75. package/dist/harness-api/start.js +1 -5
  76. package/dist/harness-api/workspace.js +2 -3
  77. package/dist/harness-api/ws-server.js +1 -1
  78. package/dist/host/app.d.ts +2 -9
  79. package/dist/host/app.js +15 -25
  80. package/dist/host/config-for.d.ts +1 -5
  81. package/dist/host/config-for.js +0 -3
  82. package/dist/host/config.d.ts +0 -5
  83. package/dist/host/create-host.d.ts +3 -33
  84. package/dist/host/create-host.js +18 -94
  85. package/dist/host/environment.d.ts +0 -3
  86. package/dist/host/environment.js +0 -9
  87. package/dist/host/gates.d.ts +4 -1
  88. package/dist/host/gates.js +3 -2
  89. package/dist/host/http.d.ts +1 -2
  90. package/dist/host/http.js +1 -7
  91. package/dist/host/index.d.ts +2 -2
  92. package/dist/host/index.js +2 -2
  93. package/dist/host/main.js +27 -34
  94. package/dist/host/operate.d.ts +29 -0
  95. package/dist/host/operate.js +223 -0
  96. package/dist/host/stack-config.d.ts +12 -12
  97. package/dist/host/stack-config.js +5 -32
  98. package/dist/index.d.ts +1 -1
  99. package/dist/index.js +1 -1
  100. package/dist/infra/workers.d.ts +1 -1
  101. package/dist/keys/keys.js +2 -2
  102. package/dist/management-openapi.json +6491 -0
  103. package/dist/mcp/connect.d.ts +8 -4
  104. package/dist/mcp/connect.js +6 -26
  105. package/dist/mcp/pool.d.ts +5 -9
  106. package/dist/mcp/pool.js +8 -19
  107. package/dist/mcp/snapshot.d.ts +8 -2
  108. package/dist/mcp/snapshot.js +29 -2
  109. package/dist/media.d.ts +1 -1
  110. package/dist/media.js +1 -1
  111. package/dist/model/pi-model.js +3 -0
  112. package/dist/openapi.json +9372 -17030
  113. package/dist/reads/access.d.ts +4 -0
  114. package/dist/reads/access.js +12 -0
  115. package/dist/reads/cursor.d.ts +11 -0
  116. package/dist/reads/cursor.js +80 -0
  117. package/dist/reads/types.d.ts +25 -0
  118. package/dist/reads/types.js +1 -0
  119. package/dist/sandbox/manager.d.ts +16 -0
  120. package/dist/sandbox/manager.js +40 -0
  121. package/dist/sandbox/session-sandbox.js +13 -3
  122. package/dist/sandbox/skills.d.ts +13 -0
  123. package/dist/sandbox/skills.js +46 -0
  124. package/dist/sandbox/types.d.ts +2 -2
  125. package/dist/store/ownership.d.ts +9 -0
  126. package/dist/store/ownership.js +13 -0
  127. package/dist/store/postgres/connect.d.ts +2 -0
  128. package/dist/store/postgres/connect.js +29 -0
  129. package/dist/store/postgres/drizzle/0011_key_roles.sql +3 -0
  130. package/dist/store/postgres/drizzle/0012_session_reads.sql +8 -0
  131. package/dist/store/postgres/drizzle/0013_definition_files.sql +16 -0
  132. package/dist/store/postgres/drizzle/0014_action_endpoints_removed.sql +2 -0
  133. package/dist/store/postgres/drizzle/meta/_journal.json +28 -0
  134. package/dist/store/postgres/reads.d.ts +3 -0
  135. package/dist/store/postgres/reads.js +272 -0
  136. package/dist/store/postgres/schema.d.ts +294 -489
  137. package/dist/store/postgres/schema.js +41 -76
  138. package/dist/store/postgres/store.js +51 -154
  139. package/dist/store/postgres/tenant.d.ts +34 -2
  140. package/dist/store/postgres/tenant.js +16 -32
  141. package/dist/store/types.d.ts +27 -96
  142. package/dist/store/types.js +0 -1
  143. package/dist/streams/control.d.ts +0 -2
  144. package/dist/streams/control.js +0 -5
  145. package/dist/streams/types.d.ts +1 -10
  146. package/dist/tenant/advance.d.ts +3 -3
  147. package/dist/tenant/advance.js +10 -7
  148. package/dist/tenant/auth.d.ts +14 -9
  149. package/dist/tenant/auth.js +24 -28
  150. package/dist/tenant/cause.d.ts +3 -3
  151. package/dist/tenant/cause.js +1 -1
  152. package/dist/tenant/commands.d.ts +2 -18
  153. package/dist/tenant/commands.js +35 -172
  154. package/dist/tenant/context.d.ts +14 -24
  155. package/dist/tenant/definition-files.d.ts +24 -0
  156. package/dist/tenant/definition-files.js +103 -0
  157. package/dist/tenant/effects.d.ts +18 -20
  158. package/dist/tenant/effects.js +56 -119
  159. package/dist/tenant/ephemeral.d.ts +7 -6
  160. package/dist/tenant/ephemeral.js +3 -11
  161. package/dist/tenant/host-token.js +1 -1
  162. package/dist/tenant/jwt.js +1 -1
  163. package/dist/tenant/module.d.ts +1 -4
  164. package/dist/tenant/module.js +3 -12
  165. package/dist/tenant/operator-keys.d.ts +34 -8
  166. package/dist/tenant/operator-keys.js +50 -18
  167. package/dist/tenant/outbound.d.ts +10 -1
  168. package/dist/tenant/outbound.js +44 -27
  169. package/dist/tenant/paths.d.ts +1 -1
  170. package/dist/tenant/paths.js +1 -3
  171. package/dist/tenant/principals.d.ts +11 -2
  172. package/dist/tenant/principals.js +27 -8
  173. package/dist/tenant/run-token.js +3 -3
  174. package/dist/tenant/runtime.d.ts +4 -6
  175. package/dist/tenant/runtime.js +19 -18
  176. package/dist/tenant/sandboxes.d.ts +4 -1
  177. package/dist/tenant/sandboxes.js +11 -0
  178. package/dist/tenant/scheduler.d.ts +1 -5
  179. package/dist/tenant/scheduler.js +2 -5
  180. package/dist/tenant/session-streams.d.ts +2 -1
  181. package/dist/tenant/session-streams.js +39 -5
  182. package/dist/tenant/session.d.ts +8 -41
  183. package/dist/tenant/session.js +0 -37
  184. package/dist/tenant/sessions.js +16 -8
  185. package/dist/tenant/signing-keys.d.ts +3 -3
  186. package/dist/tenant/signing-keys.js +5 -5
  187. package/dist/tenant/skill-tool.d.ts +20 -0
  188. package/dist/tenant/skill-tool.js +72 -0
  189. package/dist/tenant/slim.d.ts +1 -1
  190. package/dist/tenant/slim.js +1 -1
  191. package/dist/tenant/status.js +7 -12
  192. package/dist/tenant/store-pg.d.ts +15 -0
  193. package/dist/tenant/store-pg.js +7 -3
  194. package/dist/tenant/streams.d.ts +3 -10
  195. package/dist/tenant/streams.js +1 -24
  196. package/dist/tenant/sweep.js +0 -5
  197. package/dist/tenant/transcript.d.ts +2 -2
  198. package/dist/tenant/transcript.js +2 -2
  199. package/dist/tenant/types.d.ts +15 -22
  200. package/dist/tenant/worker.d.ts +1 -3
  201. package/dist/tenant/worker.js +3 -8
  202. package/dist/vault/sources.d.ts +7 -1
  203. package/dist/vault/sources.js +22 -11
  204. package/dist/version.d.ts +1 -1
  205. package/dist/version.js +1 -1
  206. package/package.json +4 -4
  207. package/dist/admin-openapi.json +0 -904
  208. package/dist/api/http/routes/actions.d.ts +0 -8
  209. package/dist/api/http/routes/actions.js +0 -75
  210. package/dist/api/http/routes/endpoints.d.ts +0 -7
  211. package/dist/api/http/routes/endpoints.js +0 -100
  212. package/dist/host/admin-api.d.ts +0 -30
  213. package/dist/host/admin-api.js +0 -140
  214. package/dist/mcp/stdio.d.ts +0 -17
  215. package/dist/mcp/stdio.js +0 -82
  216. package/dist/tenant/actions.d.ts +0 -14
  217. package/dist/tenant/actions.js +0 -29
  218. package/dist/tenant/delivery-token.d.ts +0 -41
  219. package/dist/tenant/delivery-token.js +0 -104
  220. package/dist/tenant/delivery.d.ts +0 -62
  221. package/dist/tenant/delivery.js +0 -481
  222. package/dist/tenant/endpoints.d.ts +0 -67
  223. package/dist/tenant/endpoints.js +0 -90
@@ -1,39 +1,3 @@
1
- /**
2
- * The tables of a Tenant database (session-store.md §2–§3): the source drizzle-kit generates
3
- * the migrations from (`drizzle/`, `CONTRIBUTING.md` "Adding a migration") and the query
4
- * builder's view of them. Column names are the camelCase keys in snake_case (`casing:
5
- * "snake_case"`, here, in `db.ts` and in `drizzle.config.ts`).
6
- *
7
- * - `nylorun` holds the Tenant's state: the one `tenant` row, the document tables, Action
8
- * endpoints, principals, vaults and credentials, signing keys, settings, the model usage
9
- * ledger, the model budgets, the Tool Gate's crossings and file artifacts with their versions.
10
- * - `nylorun_streams` holds the record (Durable Streams §6): `session_events`,
11
- * `session_log_heads` and the relay's `relay_slots`. The relay's publication is custom SQL
12
- * (`drizzle/0002_stream_relay.sql`).
13
- *
14
- * What the columns keep:
15
- *
16
- * - **Bodies are `json`, not `jsonb`** (`jsonText`), written as the text `JSON.stringify`
17
- * wrote, so every string round-trips exactly, key order included. `jsonb` rejects the
18
- * escapes `\u0000` (`22P05`) and unpaired surrogates (`\ud800`), which tool output can
19
- * contain; `json` stores the text verbatim and checks only its syntax.
20
- * - **Indexed fields of a body are stored generated columns** read through
21
- * `nylorun.doc(body)` (custom SQL, `drizzle/0000_foundation.sql`), which casts to `jsonb`
22
- * after replacing those two escapes with `\ufffd`: the body is untouched, and only an indexed
23
- * field that itself contained one (never an id, status or timestamp) would see U+FFFD.
24
- * Queries never apply `->`/`->>` to a `json` body (Postgres de-escapes the whole text and
25
- * fails on the same escapes); they read the generated columns.
26
- * - **Ids and timestamps compared as text are `COLLATE "C"`** (`textC`), so they order by code
27
- * point whatever the database's collation.
28
- * - **Timestamps are ISO text**, except the session lease `owner_expires_at` and the record's
29
- * `committed_at` (`timestamptz`).
30
- * - **Sealed secrets are `bytea` columns** (`bytes`), never inside a JSON body.
31
- * - Sessions carry the store-managed ownership columns (`owner`, `epoch`,
32
- * `owner_expires_at`), which are never in `body`.
33
- *
34
- * Columns are listed in the order the tables have them, so a database created by these
35
- * migrations and one created by the hand-written migrations they replaced dump the same.
36
- */
37
1
  import { sql } from "drizzle-orm";
38
2
  import { bigint, boolean, check, customType, doublePrecision, foreignKey, index, integer, pgSchema, primaryKey, text, timestamp, uniqueIndex, } from "drizzle-orm/pg-core";
39
3
  export const TENANT_SCHEMA = "nylorun";
@@ -91,11 +55,14 @@ export const tenant = nylorun.table("tenant", {
91
55
  retiredGenerations: integer().array().notNull().default(sql `'{}'`),
92
56
  }, () => [check("tenant_singleton_check", sql `singleton`)]);
93
57
  export const definitions = documents("definitions");
58
+ const xid8 = customType({ dataType: () => "xid8" });
94
59
  export const sessions = nylorun.table("sessions", {
95
60
  id: textC().primaryKey(),
96
61
  body: jsonText().notNull(),
97
62
  status: field("->>'status'"),
98
63
  agentId: field("->>'agentId'"),
64
+ /** Unknown for legacy sessions; assigned only on insert. */
65
+ createdAt: timestamp({ withTimezone: true, mode: "string" }).defaultNow(),
99
66
  /** The lease holder; store-managed, never in `body`. */
100
67
  owner: text(),
101
68
  epoch: bigint({ mode: "number" }).notNull().default(0),
@@ -105,6 +72,7 @@ export const sessions = nylorun.table("sessions", {
105
72
  /** The sandbox resource the session is attached to (`sandbox: { id }`). */
106
73
  sandboxId: field("->>'sandboxId'"),
107
74
  }, (t) => [
75
+ index("sessions_created").on(t.createdAt.desc().nullsLast(), t.id.desc().nullsFirst()),
108
76
  index("sessions_status").on(t.status, t.ownerExpiresAt),
109
77
  index("sessions_agent").on(t.agentId),
110
78
  index("sessions_owner_user").on(t.ownerUserId),
@@ -122,29 +90,16 @@ export const effects = nylorun.table("effects", {
122
90
  index("effects_session").on(t.sessionId, t.turnId, t.status),
123
91
  index("effects_status").on(t.status, t.kind),
124
92
  ]);
125
- export const actions = nylorun.table("actions", {
126
- id: textC().primaryKey(),
127
- body: jsonText().notNull(),
128
- sessionId: field("->>'sessionId'"),
129
- turnId: field("->>'turnId'"),
130
- agentId: field("->>'agentId'"),
131
- status: field("->>'status'"),
132
- kind: field("->>'kind'"),
133
- /** When the Action being delivered is lost (ISO text). */
134
- deadlineAt: field("->>'deadlineAt'"),
135
- }, (t) => [
136
- index("actions_session").on(t.sessionId, t.turnId, t.status),
137
- index("actions_agent").on(t.agentId, t.status),
138
- index("actions_status").on(t.status, t.kind),
139
- index("actions_deadline").on(t.status, t.deadlineAt),
140
- ]);
141
93
  export const sandboxes = documents("sandboxes");
142
94
  export const links = nylorun.table("links", {
143
95
  id: textC().primaryKey(),
144
96
  body: jsonText().notNull(),
145
97
  workflowSessionId: field("->>'workflowSessionId'"),
146
98
  }, (t) => [index("links_workflow").on(t.workflowSessionId)]);
147
- /** An application principal: who holds a key, by its SHA-256. */
99
+ /**
100
+ * A principal: who holds a key, by its SHA-256, and what the key reaches (`role`, protocol 8):
101
+ * `application` the Runtime API, `management` the Management API, `studio` both.
102
+ */
148
103
  export const principals = nylorun.table("principals", {
149
104
  id: textC().primaryKey(),
150
105
  role: text().$type().notNull(),
@@ -249,7 +204,7 @@ export const tenantSettings = nylorun.table("tenant_settings", {
249
204
  value: text().notNull(),
250
205
  });
251
206
  /**
252
- * The Tenant's signing keys (delivery tokens, capability links, run and host tokens): the
207
+ * The Tenant's signing keys (capability links, run and host tokens): the
253
208
  * private half sealed, the public JWK not.
254
209
  */
255
210
  export const signingKeys = nylorun.table("signing_keys", {
@@ -273,28 +228,6 @@ export const signingKeys = nylorun.table("signing_keys", {
273
228
  uniqueIndex("signing_keys_one_current").on(t.state).where(sql `state = 'current'`),
274
229
  uniqueIndex("signing_keys_one_previous").on(t.state).where(sql `state = 'previous'`),
275
230
  ]);
276
- /**
277
- * Action endpoints: where the Runtime delivers each agent's Actions over HTTP, and the health
278
- * recent deliveries and the last ping report.
279
- */
280
- export const endpoints = nylorun.table("endpoints", {
281
- agentId: textC().primaryKey(),
282
- url: text().notNull(),
283
- implementationVersion: text().notNull(),
284
- manifestHash: text(),
285
- timeoutMs: integer().notNull(),
286
- maxConcurrent: integer().notNull(),
287
- principalId: text(),
288
- lastDeliveryAt: text(),
289
- lastSuccessAt: text(),
290
- lastErrorCode: text(),
291
- lastErrorMessage: text(),
292
- consecutiveFailures: integer().notNull().default(0),
293
- servedImplementationVersion: text(),
294
- servedManifestHash: text(),
295
- createdAt: text().notNull(),
296
- updatedAt: text().notNull(),
297
- });
298
231
  /**
299
232
  * The model usage ledger (P1.3): one row per model call a gate served, written after the call.
300
233
  * A second row for the same effect (a call the gate ran again after a gateway restart) is
@@ -305,6 +238,9 @@ export const modelUsage = nylorun.table("model_usage", {
305
238
  id: textC().primaryKey(),
306
239
  /** The model effect's id; unique per Tenant, since it carries the turn id. */
307
240
  effectKey: textC().notNull(),
241
+ tokensReported: boolean(),
242
+ costKnown: boolean(),
243
+ txid: xid8().notNull().default(sql `pg_current_xact_id()`),
308
244
  sessionId: textC().notNull(),
309
245
  turnId: textC().notNull(),
310
246
  agentId: textC().notNull(),
@@ -321,6 +257,8 @@ export const modelUsage = nylorun.table("model_usage", {
321
257
  duplicate: boolean().notNull(),
322
258
  createdAt: text().notNull(),
323
259
  }, (t) => [
260
+ index("model_usage_session").on(t.sessionId, t.createdAt, t.id),
261
+ index("model_usage_export").on(t.txid, t.id),
324
262
  index("model_usage_effect").on(t.effectKey),
325
263
  index("model_usage_agent").on(t.agentId, t.createdAt),
326
264
  index("model_usage_turn").on(t.turnId),
@@ -470,6 +408,33 @@ export const artifactContent = nylorun.table("artifact_content", {
470
408
  }).onDelete("cascade"),
471
409
  index("artifact_content_sha256").on(t.sha256),
472
410
  ]);
411
+ /**
412
+ * The definition files the Tenant holds (track R2 M4): files a definition names by the SHA-256
413
+ * of their bytes (a skill's folder), uploaded with `PUT /v1/files/sha256:<hex>`. The bytes are
414
+ * in the Object store at `definitions/sha256/<hex>`; a row counts them only once they are
415
+ * stored. `sha256` is `sha256:<hex>`.
416
+ */
417
+ export const definitionFiles = nylorun.table("definition_files", {
418
+ sha256: textC().primaryKey(),
419
+ /** What the file is part of: `skill` for now. */
420
+ kind: text().notNull(),
421
+ size: bigint({ mode: "number" }).notNull(),
422
+ contentType: text(),
423
+ createdAt: textC().notNull(),
424
+ });
425
+ /**
426
+ * Which definition version names which definition file: one row per agent id, manifest hash
427
+ * and file, written when the definition is stored. A later sweep deletes the files no
428
+ * definition uses.
429
+ */
430
+ export const definitionFileUses = nylorun.table("definition_file_uses", {
431
+ agentId: textC().notNull(),
432
+ manifestHash: textC().notNull(),
433
+ sha256: textC().notNull(),
434
+ }, (t) => [
435
+ primaryKey({ name: "definition_file_uses_pkey", columns: [t.agentId, t.manifestHash, t.sha256] }),
436
+ index("definition_file_uses_sha256").on(t.sha256),
437
+ ]);
473
438
  // ---------------------------------------------------------------------------
474
439
  // nylorun_streams: the record
475
440
  /**
@@ -8,7 +8,7 @@
8
8
  *
9
9
  * Plain reads and writes use Drizzle's query builder: `.set(patch)` and `and(...)` skip
10
10
  * `undefined`, rows come back in the camelCase of `schema.ts`. Statements whose logic is SQL
11
- * stay `sql` fragments inside it: the endpoint upsert's `CASE` arms, the log-head upsert,
11
+ * stay `sql` fragments inside it: the log-head upsert,
12
12
  * `orphanedSessions`' ordering, `counts()`, advisory locks and array updates.
13
13
  *
14
14
  * ## Transactions
@@ -28,7 +28,7 @@
28
28
  * ends. To stay deadlock-free:
29
29
  *
30
30
  * 1. A session-scoped transaction locks its session row before writing any
31
- * other row of that session (effects, actions, its record rows and log
31
+ * other row of that session (effects, its record rows and log
32
32
  * head).
33
33
  * 2. A transaction that touches several sessions locks them in ascending id
34
34
  * order, with `lockSessions` (`./locking.ts`), before writing any of them.
@@ -51,12 +51,12 @@
51
51
  import { AsyncLocalStorage } from "node:async_hooks";
52
52
  import { and, count, eq, getTableColumns, gte, inArray, isNotNull, isNull, lt, lte, ne, or, sql, } from "drizzle-orm";
53
53
  import { appendEvent, appendSandboxEvent } from "../../record/index.js";
54
- import { OwnershipLostError } from "../ownership.js";
54
+ import { OwnershipLostError, PrincipalRoleConflict } from "../ownership.js";
55
55
  import { database, driverError } from "./db.js";
56
56
  import { expectedSchemaVersion, readSchemaVersion } from "./migrate.js";
57
57
  import { createPostgresRecordReader } from "./record.js";
58
58
  import { postgresRecordWriter, postgresSandboxRecordWriter } from "./record-writer.js";
59
- import { actions, artifacts, artifactVersions, artifactContent, commands, definitions, effects, endpoints, links, modelBudgets, modelUsage, principals, sandboxes, sandboxEvents, sandboxResources, sessionEvents, sessionLogHeads, sessions, toJson, toolCrossings, signingKeys, tenant, tenantSettings, vaultAudit, vaultCredentials, vaultIdempotency, vaults, oauthPending, } from "./schema.js";
59
+ import { artifacts, artifactVersions, artifactContent, definitionFiles, definitionFileUses, commands, definitions, effects, links, modelBudgets, modelUsage, principals, sandboxes, sandboxEvents, sandboxResources, sessionEvents, sessionLogHeads, sessions, toJson, toolCrossings, signingKeys, tenant, tenantSettings, vaultAudit, vaultCredentials, vaultIdempotency, vaults, oauthPending, } from "./schema.js";
60
60
  export function createPostgresSessionStore(options) {
61
61
  return new PostgresSessionStore(options);
62
62
  }
@@ -68,7 +68,6 @@ const DOCUMENTS = {
68
68
  sessions: sessions,
69
69
  commands,
70
70
  effects: effects,
71
- actions: actions,
72
71
  sandboxes,
73
72
  links: links,
74
73
  };
@@ -243,10 +242,6 @@ function podColumns(patch) {
243
242
  };
244
243
  return Object.fromEntries(Object.entries(columns).filter(([, value]) => value !== undefined));
245
244
  }
246
- /** An endpoint row with its unset (null) columns left out, as `EndpointRow` has them. */
247
- function endpointRow(row) {
248
- return Object.fromEntries(Object.entries(row).filter(([, value]) => value !== null));
249
- }
250
245
  /** The time column a signing key's new state stamps. */
251
246
  const SIGNING_KEY_STAMP = {
252
247
  current: "activatedAt",
@@ -255,11 +250,9 @@ const SIGNING_KEY_STAMP = {
255
250
  };
256
251
  /** The audit columns a row has (`ord` only orders them). */
257
252
  const { ord: _ord, ...AUDIT } = getTableColumns(vaultAudit);
258
- const SESSION_TABLES = [sessions, commands, effects, actions, links];
253
+ const SESSION_TABLES = [sessions, commands, effects, links];
259
254
  /** The inserted row's `column` (`ON CONFLICT … DO UPDATE`). */
260
255
  const excluded = (column) => sql.raw(`excluded.${column}`);
261
- /** Keeps an endpoint's health `column` when its URL stays; a new URL starts with `fresh`. */
262
- const sameUrl = (column, fresh = sql `NULL`) => sql `CASE WHEN ${endpoints.url} = excluded.url THEN ${column} ELSE ${fresh} END`;
263
256
  class PostgresTx {
264
257
  db;
265
258
  tenantId;
@@ -536,56 +529,6 @@ class PostgresTx {
536
529
  .limit(options.limit ?? 1000);
537
530
  return rows.map((row) => row.body);
538
531
  }
539
- async actionBodies(where) {
540
- const rows = await this.db
541
- .select({ body: actions.body })
542
- .from(actions)
543
- .where(where)
544
- .orderBy(actions.id);
545
- return rows.map((row) => row.body);
546
- }
547
- async pendingActions(agentId) {
548
- this.check();
549
- return this.actionBodies(and(eq(actions.agentId, agentId), eq(actions.status, "pending")));
550
- }
551
- async deliveringCount(agentId) {
552
- this.check();
553
- const [row] = await this.db
554
- .select({ n: count() })
555
- .from(actions)
556
- .where(and(eq(actions.agentId, agentId), eq(actions.status, "delivering")));
557
- return row.n;
558
- }
559
- async pendingActionsWithEndpoint(limit) {
560
- this.check();
561
- const rows = await this.db
562
- .select({ body: actions.body })
563
- .from(actions)
564
- .innerJoin(endpoints, eq(endpoints.agentId, actions.agentId))
565
- .where(eq(actions.status, "pending"))
566
- .orderBy(actions.id)
567
- .limit(limit);
568
- return rows.map((row) => row.body);
569
- }
570
- async expiredDeliveries(now, limit) {
571
- this.check();
572
- const deadline = sql `${actions.deadlineAt}::timestamptz`;
573
- const rows = await this.db
574
- .select({ body: actions.body })
575
- .from(actions)
576
- .where(and(eq(actions.status, "delivering"), isNotNull(actions.deadlineAt), sql `${deadline} <= ${now.toISOString()}::timestamptz`))
577
- .orderBy(deadline, actions.id)
578
- .limit(limit);
579
- return rows.map((row) => row.body);
580
- }
581
- async actionsForSession(sessionId, filter = {}) {
582
- this.check();
583
- return this.actionBodies(and(eq(actions.sessionId, sessionId), filter.turnId === undefined ? undefined : eq(actions.turnId, filter.turnId), filter.statuses === undefined ? undefined : inArray(actions.status, filter.statuses)));
584
- }
585
- async actionsWithStatus(statuses, filter = {}) {
586
- this.check();
587
- return this.actionBodies(and(inArray(actions.status, statuses), filter.kinds === undefined ? undefined : inArray(actions.kind, filter.kinds)));
588
- }
589
532
  async effectBodies(where) {
590
533
  const rows = await this.db
591
534
  .select({ body: effects.body })
@@ -629,15 +572,12 @@ class PostgresTx {
629
572
  (SELECT count(*) FROM ${sessions})::int AS sessions,
630
573
  (SELECT count(*) FROM ${sessions}
631
574
  WHERE ${sessions.status} IN ('running', 'runnable'))::int AS running,
632
- (SELECT count(*) FROM ${actions}
633
- WHERE ${actions.status} IN ('pending', 'delivering'))::int AS actions,
634
575
  (SELECT count(*) FROM ${effects} WHERE ${effects.status} = 'uncertain')::int AS uncertain,
635
576
  (SELECT count(*) FROM ${sandboxes})::int AS sandboxes,
636
577
  (SELECT count(*) FROM ${definitions})::int AS definitions`);
637
578
  return {
638
579
  sessions: row.sessions,
639
580
  runningSessions: row.running,
640
- pendingActions: row.actions,
641
581
  uncertainEffects: row.uncertain,
642
582
  sandboxes: row.sandboxes,
643
583
  definitions: row.definitions,
@@ -658,81 +598,6 @@ class PostgresTx {
658
598
  retiredGenerations: sql `array_remove(${tenant.retiredGenerations}, ${generation}::int)`,
659
599
  });
660
600
  }
661
- // --- Action endpoints -----------------------------------------------------
662
- async listEndpoints() {
663
- this.check();
664
- const rows = await this.db.select().from(endpoints).orderBy(endpoints.agentId);
665
- return rows.map(endpointRow);
666
- }
667
- async getEndpoint(agentId) {
668
- this.check();
669
- const [row] = await this.db.select().from(endpoints).where(eq(endpoints.agentId, agentId));
670
- return row && endpointRow(row);
671
- }
672
- async putEndpoint(row) {
673
- this.check();
674
- // Health belongs to a URL: a new URL starts with none.
675
- await this.db
676
- .insert(endpoints)
677
- .values({
678
- agentId: row.agentId,
679
- url: row.url,
680
- implementationVersion: row.implementationVersion,
681
- manifestHash: row.manifestHash ?? null,
682
- timeoutMs: row.timeoutMs,
683
- maxConcurrent: row.maxConcurrent,
684
- principalId: row.principalId ?? null,
685
- consecutiveFailures: 0,
686
- createdAt: row.updatedAt,
687
- updatedAt: row.updatedAt,
688
- })
689
- .onConflictDoUpdate({
690
- target: endpoints.agentId,
691
- set: {
692
- url: excluded("url"),
693
- implementationVersion: excluded("implementation_version"),
694
- manifestHash: excluded("manifest_hash"),
695
- timeoutMs: excluded("timeout_ms"),
696
- maxConcurrent: excluded("max_concurrent"),
697
- principalId: excluded("principal_id"),
698
- updatedAt: excluded("updated_at"),
699
- lastDeliveryAt: sameUrl(sql `${endpoints.lastDeliveryAt}`),
700
- lastSuccessAt: sameUrl(sql `${endpoints.lastSuccessAt}`),
701
- lastErrorCode: sameUrl(sql `${endpoints.lastErrorCode}`),
702
- lastErrorMessage: sameUrl(sql `${endpoints.lastErrorMessage}`),
703
- consecutiveFailures: sameUrl(sql `${endpoints.consecutiveFailures}`, sql `0`),
704
- servedImplementationVersion: sameUrl(sql `${endpoints.servedImplementationVersion}`),
705
- servedManifestHash: sameUrl(sql `${endpoints.servedManifestHash}`),
706
- },
707
- });
708
- }
709
- async deleteEndpoint(agentId) {
710
- this.check();
711
- await this.db.delete(endpoints).where(eq(endpoints.agentId, agentId));
712
- }
713
- async recordEndpointHealth(agentId, update) {
714
- this.check();
715
- const set = update.kind === "success"
716
- ? {
717
- lastDeliveryAt: update.at,
718
- lastSuccessAt: update.at,
719
- consecutiveFailures: 0,
720
- lastErrorCode: null,
721
- lastErrorMessage: null,
722
- }
723
- : update.kind === "failure"
724
- ? {
725
- lastDeliveryAt: update.at,
726
- lastErrorCode: update.code,
727
- lastErrorMessage: update.message,
728
- consecutiveFailures: sql `${endpoints.consecutiveFailures} + 1`,
729
- }
730
- : {
731
- servedImplementationVersion: update.implementationVersion,
732
- servedManifestHash: update.manifestHash ?? null,
733
- };
734
- await this.db.update(endpoints).set(set).where(eq(endpoints.agentId, agentId));
735
- }
736
601
  // --- principals ----------------------------------------------------------
737
602
  async insertPrincipal(row) {
738
603
  this.check();
@@ -751,29 +616,24 @@ class PostgresTx {
751
616
  const [row] = await this.db.select().from(principals).where(eq(principals.id, id));
752
617
  return row;
753
618
  }
754
- async applicationTokenHashes() {
755
- this.check();
756
- const rows = await this.db
757
- .select({ tokenHash: principals.tokenHash })
758
- .from(principals)
759
- .where(eq(principals.role, "application"))
760
- .orderBy(principals.id);
761
- return rows.map((row) => row.tokenHash);
762
- }
763
619
  async listPrincipals() {
764
620
  this.check();
765
621
  return await this.db.select().from(principals).orderBy(principals.id);
766
622
  }
767
- async putPrincipal(id, tokenHash, createdAt) {
623
+ async putPrincipal(id, tokenHash, createdAt, role = "application") {
768
624
  this.check();
769
625
  const [row] = await this.db
770
626
  .insert(principals)
771
- .values({ id, role: "application", tokenHash, idempotencyKey: null, createdAt })
627
+ .values({ id, role, tokenHash, idempotencyKey: null, createdAt })
772
628
  .onConflictDoUpdate({
773
629
  target: principals.id,
774
630
  set: { tokenHash, createdAt, idempotencyKey: null },
631
+ // A key keeps its role: rotating never turns one kind of key into another.
632
+ setWhere: eq(principals.role, role),
775
633
  })
776
634
  .returning();
635
+ if (!row)
636
+ throw new PrincipalRoleConflict(id);
777
637
  return row;
778
638
  }
779
639
  async deletePrincipal(id) {
@@ -953,8 +813,8 @@ class PostgresTx {
953
813
  ...row,
954
814
  duplicate: sql `EXISTS (SELECT 1 FROM ${modelUsage} WHERE ${modelUsage.effectKey} = ${row.effectKey})`,
955
815
  })
956
- .returning({ duplicate: modelUsage.duplicate });
957
- return { ...row, duplicate: inserted.duplicate };
816
+ .returning();
817
+ return inserted;
958
818
  }
959
819
  async modelUsageTotals(query) {
960
820
  this.check();
@@ -1137,6 +997,42 @@ class PostgresTx {
1137
997
  .where(scope === "sessions" ? isNotNull(artifacts.sessionId) : undefined);
1138
998
  return rows.map((row) => row.blobKey);
1139
999
  }
1000
+ // --- definition files -----------------------------------------------------
1001
+ async definitionFile(sha256) {
1002
+ this.check();
1003
+ const [row] = await this.db.select().from(definitionFiles).where(eq(definitionFiles.sha256, sha256));
1004
+ return row;
1005
+ }
1006
+ async insertDefinitionFile(row) {
1007
+ this.check();
1008
+ const inserted = await this.db
1009
+ .insert(definitionFiles)
1010
+ .values(row)
1011
+ .onConflictDoNothing()
1012
+ .returning({ sha256: definitionFiles.sha256 });
1013
+ return inserted.length === 1;
1014
+ }
1015
+ async heldDefinitionFiles(shas) {
1016
+ this.check();
1017
+ const found = new Set();
1018
+ for (let at = 0; at < shas.length; at += CONTENT_BATCH) {
1019
+ const rows = await this.db
1020
+ .select({ sha256: definitionFiles.sha256 })
1021
+ .from(definitionFiles)
1022
+ .where(inArray(definitionFiles.sha256, shas.slice(at, at + CONTENT_BATCH)));
1023
+ for (const row of rows)
1024
+ found.add(row.sha256);
1025
+ }
1026
+ return found;
1027
+ }
1028
+ async insertDefinitionFileUses(agentId, manifestHash, shas) {
1029
+ this.check();
1030
+ for (let at = 0; at < shas.length; at += CONTENT_BATCH)
1031
+ await this.db
1032
+ .insert(definitionFileUses)
1033
+ .values(shas.slice(at, at + CONTENT_BATCH).map((sha256) => ({ agentId, manifestHash, sha256 })))
1034
+ .onConflictDoNothing();
1035
+ }
1140
1036
  // --- settings ------------------------------------------------------------
1141
1037
  async getSetting(key) {
1142
1038
  this.check();
@@ -1179,7 +1075,8 @@ class PostgresTx {
1179
1075
  }
1180
1076
  if (scope === "all") {
1181
1077
  await db.delete(definitions);
1182
- await db.delete(endpoints);
1078
+ // The files stay, as their bytes do; only what used them goes.
1079
+ await db.delete(definitionFileUses);
1183
1080
  await db.delete(modelUsage);
1184
1081
  await db.delete(modelBudgets);
1185
1082
  await db.delete(artifacts);
@@ -1,3 +1,27 @@
1
+ /**
2
+ * The Tenant a database holds (tenancy.md §1, §4; session-store.md §2). One database is one
3
+ * Tenant: its state in the schema `nylorun`, its record in `nylorun_streams`, and its
4
+ * envelope in the one row of `nylorun.tenant`. There is no catalog.
5
+ *
6
+ * `openTenantDatabase` runs in one transaction, under the database's migration lock
7
+ * (`migrate.ts`), so several processes starting at once migrate and create once:
8
+ *
9
+ * 1. Refuse an old layout (`database-layout-old`): `tenant_<id>` schemas or a record keyed by
10
+ * Tenant (a Runtime from before one Tenant per installation), or the `schema_version`
11
+ * tables of a pre-release build of it. It is never touched: this release starts fresh.
12
+ * 2. Apply the missing migrations (`migrate.ts`). A database with migrations this Runtime
13
+ * does not know is `schema-too-new`; a migration Postgres rejects is `migration-failed`.
14
+ * 3. When `nylorun.tenant` is empty, create the Tenant: its row (id, name, created time) and
15
+ * its first principals. A database whose row is gone but that holds a Tenant's data
16
+ * (principals, sessions, agents, vaults, keys) is `envelope-invalid`, never given a new
17
+ * Tenant. Principals given later that the Tenant does not have yet are added (the
18
+ * ephemeral Runtime's application key); existing ones are kept, as ordinary keys.
19
+ * 4. Read the envelope back (`envelope-invalid` when it does not parse), and open the store.
20
+ *
21
+ * A lost connection or an unavailable server says nothing about the Tenant: it is thrown as
22
+ * it is, never as a cause.
23
+ */
24
+ import type { KeyRole } from "@nylorun/core/compatibility";
1
25
  import type { Sql } from "postgres";
2
26
  import { type TenantEnvelope } from "@nylorun/core/contracts";
3
27
  import type { SessionStore } from "../types.js";
@@ -8,6 +32,14 @@ export interface InitialPrincipal {
8
32
  id: string;
9
33
  /** SHA-256 hex of the principal's key. */
10
34
  credentialHash: string;
35
+ /** What the key reaches; `application` when absent. */
36
+ role?: KeyRole;
37
+ /**
38
+ * Its key comes from outside the database (the bootstrap file, or Studio's key derived from
39
+ * the admin key): on later opens a principal of the same role takes this hash. Otherwise an
40
+ * existing principal is kept as it is.
41
+ */
42
+ replace?: boolean;
11
43
  }
12
44
  /** Who the Tenant is when the database holds none yet. */
13
45
  export interface TenantCreation {
@@ -16,8 +48,8 @@ export interface TenantCreation {
16
48
  /** Ignored when the Tenant exists. */
17
49
  name: string;
18
50
  /**
19
- * The principals the Tenant must have, given its id (Studio's derived key depends on it). Created
20
- * with the Tenant; on later opens, the missing ones are added.
51
+ * The principals the Tenant must have, given its id. Created with the Tenant; on later
52
+ * opens, the missing ones are added.
21
53
  */
22
54
  principals?(tenantId: string): readonly InitialPrincipal[];
23
55
  }
@@ -1,27 +1,4 @@
1
- /**
2
- * The Tenant a database holds (tenancy.md §1, §4; session-store.md §2). One database is one
3
- * Tenant: its state in the schema `nylorun`, its record in `nylorun_streams`, and its
4
- * envelope in the one row of `nylorun.tenant`. There is no catalog.
5
- *
6
- * `openTenantDatabase` runs in one transaction, under the database's migration lock
7
- * (`migrate.ts`), so several processes starting at once migrate and create once:
8
- *
9
- * 1. Refuse an old layout (`database-layout-old`): `tenant_<id>` schemas or a record keyed by
10
- * Tenant (a Runtime from before one Tenant per installation), or the `schema_version`
11
- * tables of a pre-release build of it. It is never touched: this release starts fresh.
12
- * 2. Apply the missing migrations (`migrate.ts`). A database with migrations this Runtime
13
- * does not know is `schema-too-new`; a migration Postgres rejects is `migration-failed`.
14
- * 3. When `nylorun.tenant` is empty, create the Tenant: its row (id, name, created time) and
15
- * its first principals. A database whose row is gone but that holds a Tenant's data
16
- * (principals, sessions, agents, vaults, keys) is `envelope-invalid`, never given a new
17
- * Tenant. Principals given later that the Tenant does not have yet are added (the
18
- * ephemeral Runtime's application key); existing ones are kept, as ordinary keys.
19
- * 4. Read the envelope back (`envelope-invalid` when it does not parse), and open the store.
20
- *
21
- * A lost connection or an unavailable server says nothing about the Tenant: it is thrown as
22
- * it is, never as a cause.
23
- */
24
- import { sql } from "drizzle-orm";
1
+ import { and, eq, ne, sql } from "drizzle-orm";
25
2
  import { isTenantId, newTenantId } from "@nylorun/core/compatibility";
26
3
  import { TenantEnvelopeSchema, } from "@nylorun/core/contracts";
27
4
  import { openError, TenantOpenError } from "../../tenant/cause.js";
@@ -138,17 +115,24 @@ async function ensureTenant(tx, create, migrated, now) {
138
115
  else if (migrated.from !== migrated.to) {
139
116
  await tx.update(tenant).set({ schemaVersion: migrated.to });
140
117
  }
141
- for (const principal of create.principals?.(tenantId) ?? [])
142
- // A principal with this id or this key already exists: it is kept as it is.
143
- await tx
144
- .insert(principals)
145
- .values({
118
+ for (const principal of create.principals?.(tenantId) ?? []) {
119
+ const insert = tx.insert(principals).values({
146
120
  id: principal.id,
147
- role: "application",
121
+ role: principal.role ?? "application",
148
122
  tokenHash: principal.credentialHash,
149
123
  idempotencyKey: null,
150
124
  createdAt,
151
- })
152
- .onConflictDoNothing();
125
+ });
126
+ if (principal.replace)
127
+ // The bootstrap key or Studio's: a principal of the same role takes the current key.
128
+ await insert.onConflictDoUpdate({
129
+ target: principals.id,
130
+ set: { tokenHash: principal.credentialHash, createdAt },
131
+ setWhere: and(eq(principals.role, principal.role ?? "application"), ne(principals.tokenHash, principal.credentialHash)),
132
+ });
133
+ // A principal with this id or this key already exists: it is kept as it is.
134
+ else
135
+ await insert.onConflictDoNothing();
136
+ }
153
137
  return existing === undefined;
154
138
  }