@capybara-hq/kernel 0.1.0-rc.1 → 0.1.0-rc.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bootstrap.json +85 -5
- package/dist/admin/admin.controller.d.ts +52 -71
- package/dist/admin/admin.controller.js +117 -63
- package/dist/admin/admin.module.d.ts +3 -0
- package/dist/admin/admin.module.js +22 -0
- package/dist/admin/admin.schemas.d.ts +60 -0
- package/dist/admin/admin.schemas.js +50 -0
- package/dist/admin/extensions.controller.d.ts +47 -16
- package/dist/admin/extensions.controller.js +94 -23
- package/dist/admin/extensions.schemas.d.ts +7 -0
- package/dist/admin/extensions.schemas.js +10 -0
- package/dist/admin/limit-defaults.d.ts +20 -0
- package/dist/admin/limit-defaults.js +10 -0
- package/dist/admin/setup-lock.d.ts +16 -0
- package/dist/admin/setup-lock.js +47 -0
- package/dist/admin/setup.controller.d.ts +9 -9
- package/dist/admin/setup.controller.js +28 -26
- package/dist/admin/setup.service.d.ts +20 -28
- package/dist/admin/setup.service.js +65 -65
- package/dist/agents/agent-lookup.d.ts +72 -0
- package/dist/agents/agent-lookup.js +208 -0
- package/dist/agents/agent-profile.service.d.ts +31 -0
- package/dist/agents/agent-profile.service.js +122 -0
- package/dist/agents/agents.controller.d.ts +54 -169
- package/dist/agents/agents.controller.js +172 -37
- package/dist/agents/agents.module.d.ts +10 -0
- package/dist/agents/agents.module.js +71 -0
- package/dist/agents/agents.schemas.d.ts +389 -0
- package/dist/agents/agents.schemas.js +126 -0
- package/dist/agents/agents.service.d.ts +104 -76
- package/dist/agents/agents.service.js +509 -383
- package/dist/agents/definition.d.ts +133 -21
- package/dist/agents/definition.js +361 -41
- package/dist/agents/from-template.d.ts +106 -0
- package/dist/agents/from-template.js +680 -0
- package/dist/agents/from-template.service.d.ts +72 -0
- package/dist/agents/from-template.service.js +178 -0
- package/dist/agents/instruction-files.d.ts +60 -0
- package/dist/agents/instruction-files.js +111 -0
- package/dist/agents/models.controller.d.ts +44 -6
- package/dist/agents/models.controller.js +84 -8
- package/dist/agents/org-templates.d.ts +186 -0
- package/dist/agents/org-templates.js +868 -0
- package/dist/agents/pending-connector-requests.d.ts +79 -0
- package/dist/agents/pending-connector-requests.js +215 -0
- package/dist/agents/pending-connectors.controller.d.ts +39 -0
- package/dist/agents/pending-connectors.controller.js +91 -0
- package/dist/agents/pending-connectors.service.d.ts +62 -0
- package/dist/agents/pending-connectors.service.js +282 -0
- package/dist/agents/profile.d.ts +114 -0
- package/dist/agents/profile.js +55 -0
- package/dist/agents/readiness.controller.d.ts +26 -0
- package/dist/agents/readiness.controller.js +56 -0
- package/dist/agents/templates.controller.d.ts +78 -6
- package/dist/agents/templates.controller.js +154 -8
- package/dist/agents/templates.d.ts +132 -30
- package/dist/agents/templates.js +500 -46
- package/dist/agents/templates.schemas.d.ts +20 -0
- package/dist/agents/templates.schemas.js +45 -0
- package/dist/app.d.ts +93 -0
- package/dist/app.js +150 -0
- package/dist/background/background.controller.d.ts +38 -0
- package/dist/background/background.controller.js +88 -0
- package/dist/background/background.module.d.ts +10 -0
- package/dist/background/background.module.js +39 -0
- package/dist/background/background.schemas.d.ts +10 -0
- package/dist/background/background.schemas.js +16 -0
- package/dist/background/finish.d.ts +44 -0
- package/dist/background/finish.js +95 -0
- package/dist/background/launch-lock.d.ts +15 -0
- package/dist/background/launch-lock.js +40 -0
- package/dist/background/launch.d.ts +42 -0
- package/dist/background/launch.js +77 -0
- package/dist/background/poll.job.d.ts +56 -0
- package/dist/background/poll.job.js +195 -0
- package/dist/background/render.d.ts +23 -0
- package/dist/background/render.js +48 -0
- package/dist/background/session.d.ts +48 -0
- package/dist/background/session.js +80 -0
- package/dist/background/status.d.ts +35 -0
- package/dist/background/status.js +57 -0
- package/dist/background/stop.d.ts +68 -0
- package/dist/background/stop.js +244 -0
- package/dist/background/store.d.ts +167 -0
- package/dist/background/store.js +361 -0
- package/dist/background/store.module.d.ts +6 -0
- package/dist/background/store.module.js +22 -0
- package/dist/background/tasks.service.d.ts +77 -0
- package/dist/background/tasks.service.js +216 -0
- package/dist/background/wake-rules.d.ts +66 -0
- package/dist/background/wake-rules.js +113 -0
- package/dist/background/wake.d.ts +30 -0
- package/dist/background/wake.js +63 -0
- package/dist/catalog/admin-catalog.controller.d.ts +102 -52
- package/dist/catalog/admin-catalog.controller.js +222 -67
- package/dist/catalog/admin-catalog.schemas.d.ts +72 -0
- package/dist/catalog/admin-catalog.schemas.js +92 -0
- package/dist/catalog/catalog-content.d.ts +264 -0
- package/dist/catalog/catalog-content.js +430 -0
- package/dist/catalog/catalog-files.d.ts +41 -0
- package/dist/catalog/catalog-files.js +86 -0
- package/dist/catalog/catalog-import.module.d.ts +7 -0
- package/dist/catalog/catalog-import.module.js +60 -0
- package/dist/catalog/catalog-items.d.ts +290 -0
- package/dist/catalog/catalog-items.js +677 -0
- package/dist/catalog/catalog-scope.service.d.ts +286 -0
- package/dist/catalog/catalog-scope.service.js +1196 -0
- package/dist/catalog/catalog.module.d.ts +12 -0
- package/dist/catalog/catalog.module.js +81 -0
- package/dist/catalog/catalog.service.d.ts +144 -207
- package/dist/catalog/catalog.service.js +258 -528
- package/dist/catalog/claude/agents.js +15 -10
- package/dist/catalog/claude/translate.js +3 -0
- package/dist/catalog/cli.d.ts +14 -3
- package/dist/catalog/cli.js +39 -32
- package/dist/catalog/component-promotion.d.ts +96 -0
- package/dist/catalog/component-promotion.js +395 -0
- package/dist/catalog/component-scope.controller.d.ts +90 -0
- package/dist/catalog/component-scope.controller.js +234 -0
- package/dist/catalog/component-scope.schemas.d.ts +118 -0
- package/dist/catalog/component-scope.schemas.js +97 -0
- package/dist/catalog/component-scope.service.d.ts +366 -0
- package/dist/catalog/component-scope.service.js +1115 -0
- package/dist/catalog/component-state.d.ts +197 -0
- package/dist/catalog/component-state.js +473 -0
- package/dist/catalog/git-source.d.ts +9 -1
- package/dist/catalog/git-source.js +29 -2
- package/dist/catalog/job-ids.d.ts +45 -0
- package/dist/catalog/job-ids.js +40 -0
- package/dist/catalog/jobs.d.ts +27 -0
- package/dist/catalog/jobs.js +70 -0
- package/dist/catalog/locks.d.ts +24 -0
- package/dist/catalog/locks.js +63 -0
- package/dist/catalog/marketplace-fetch.d.ts +1 -1
- package/dist/catalog/marketplace-icons.d.ts +64 -0
- package/dist/catalog/marketplace-icons.js +131 -0
- package/dist/catalog/marketplaces.controller.d.ts +53 -44
- package/dist/catalog/marketplaces.controller.js +146 -51
- package/dist/catalog/marketplaces.schemas.d.ts +48 -0
- package/dist/catalog/marketplaces.schemas.js +53 -0
- package/dist/catalog/marketplaces.service.d.ts +136 -26
- package/dist/catalog/marketplaces.service.js +663 -281
- package/dist/catalog/org-catalog.controller.d.ts +115 -63
- package/dist/catalog/org-catalog.controller.js +207 -100
- package/dist/catalog/org-catalog.schemas.d.ts +88 -0
- package/dist/catalog/org-catalog.schemas.js +93 -0
- package/dist/catalog/org-catalog.service.d.ts +89 -216
- package/dist/catalog/org-catalog.service.js +434 -472
- package/dist/catalog/org-extension-switch.d.ts +41 -0
- package/dist/catalog/org-extension-switch.js +115 -0
- package/dist/catalog/org-extension.controller.d.ts +113 -0
- package/dist/catalog/org-extension.controller.js +271 -0
- package/dist/catalog/org-extension.schemas.d.ts +15 -0
- package/dist/catalog/org-extension.schemas.js +22 -0
- package/dist/catalog/org-plugin-uploads.controller.d.ts +77 -0
- package/dist/catalog/org-plugin-uploads.controller.js +200 -0
- package/dist/catalog/org-plugin-uploads.d.ts +71 -0
- package/dist/catalog/org-plugin-uploads.js +122 -0
- package/dist/catalog/org-plugin-uploads.schemas.d.ts +20 -0
- package/dist/catalog/org-plugin-uploads.schemas.js +32 -0
- package/dist/catalog/owned-items.d.ts +188 -0
- package/dist/catalog/owned-items.js +808 -0
- package/dist/catalog/owned-plugin-cleanup.d.ts +56 -0
- package/dist/catalog/owned-plugin-cleanup.js +194 -0
- package/dist/catalog/ownership.d.ts +172 -0
- package/dist/catalog/ownership.js +326 -0
- package/dist/catalog/platform-components.d.ts +243 -0
- package/dist/catalog/platform-components.js +872 -0
- package/dist/catalog/platform-migration.d.ts +115 -0
- package/dist/catalog/platform-migration.js +583 -0
- package/dist/catalog/plugin-import.d.ts +293 -71
- package/dist/catalog/plugin-import.js +1039 -281
- package/dist/catalog/plugin-install.d.ts +271 -66
- package/dist/catalog/plugin-install.js +1198 -344
- package/dist/catalog/plugin-package.d.ts +41 -4
- package/dist/catalog/plugin-package.js +140 -7
- package/dist/catalog/plugin-upgrade.d.ts +312 -188
- package/dist/catalog/plugin-upgrade.js +1371 -1065
- package/dist/catalog/request-rules.d.ts +65 -0
- package/dist/catalog/request-rules.js +217 -0
- package/dist/catalog/requests.service.d.ts +205 -117
- package/dist/catalog/requests.service.js +612 -369
- package/dist/catalog/reserved-names.d.ts +8 -0
- package/dist/catalog/reserved-names.js +42 -0
- package/dist/catalog/scope-orgs.d.ts +21 -0
- package/dist/catalog/scope-orgs.js +67 -0
- package/dist/catalog/scope-state.d.ts +35 -0
- package/dist/catalog/scope-state.js +87 -0
- package/dist/catalog/scope-sync-workers.d.ts +7 -0
- package/dist/catalog/scope-sync-workers.js +29 -0
- package/dist/catalog/scope-sync.d.ts +176 -0
- package/dist/catalog/scope-sync.js +576 -0
- package/dist/catalog/scope.d.ts +42 -0
- package/dist/catalog/scope.js +42 -0
- package/dist/catalog/unshare.d.ts +22 -0
- package/dist/catalog/unshare.js +119 -0
- package/dist/channels/actions.d.ts +6 -4
- package/dist/channels/actions.js +11 -4
- package/dist/channels/cards.d.ts +33 -11
- package/dist/channels/cards.js +96 -24
- package/dist/channels/channels.module.d.ts +40 -0
- package/dist/channels/channels.module.js +130 -0
- package/dist/channels/channels.schemas.d.ts +37 -0
- package/dist/channels/channels.schemas.js +38 -0
- package/dist/channels/connections.controller.d.ts +44 -37
- package/dist/channels/connections.controller.js +83 -38
- package/dist/channels/connections.service.d.ts +101 -30
- package/dist/channels/connections.service.js +380 -147
- package/dist/channels/inbound.d.ts +44 -38
- package/dist/channels/inbound.js +190 -44
- package/dist/channels/job-ids.d.ts +10 -0
- package/dist/channels/job-ids.js +10 -0
- package/dist/channels/jobs.d.ts +85 -0
- package/dist/channels/jobs.js +157 -0
- package/dist/channels/outbound.d.ts +71 -80
- package/dist/channels/outbound.js +219 -202
- package/dist/channels/phrases.d.ts +33 -2
- package/dist/channels/phrases.js +29 -1
- package/dist/channels/supervisor.d.ts +88 -46
- package/dist/channels/supervisor.js +230 -132
- package/dist/channels/threads.d.ts +34 -8
- package/dist/channels/threads.js +79 -13
- package/dist/channels/webhook.d.ts +20 -14
- package/dist/channels/webhook.js +51 -20
- package/dist/config/docs.js +5 -1
- package/dist/config/load.d.ts +3 -2
- package/dist/config/load.js +3 -2
- package/dist/config/nacos-source.js +11 -2
- package/dist/config/schema.d.ts +36 -2
- package/dist/config/schema.js +226 -25
- package/dist/conversations/access.d.ts +31 -0
- package/dist/conversations/access.js +110 -0
- package/dist/conversations/agent-tree.d.ts +46 -0
- package/dist/conversations/agent-tree.js +92 -0
- package/dist/conversations/artifacts.d.ts +11 -11
- package/dist/conversations/artifacts.js +36 -10
- package/dist/conversations/attachments.controller.d.ts +1 -1
- package/dist/conversations/attachments.controller.js +8 -6
- package/dist/conversations/attachments.module.d.ts +7 -0
- package/dist/conversations/attachments.module.js +25 -0
- package/dist/conversations/attachments.service.d.ts +60 -22
- package/dist/conversations/attachments.service.js +249 -51
- package/dist/conversations/conversations.controller.d.ts +86 -50
- package/dist/conversations/conversations.controller.js +184 -103
- package/dist/conversations/conversations.module.d.ts +8 -0
- package/dist/conversations/conversations.module.js +72 -0
- package/dist/conversations/conversations.schemas.d.ts +52 -0
- package/dist/conversations/conversations.schemas.js +79 -0
- package/dist/conversations/conversations.service.d.ts +166 -49
- package/dist/conversations/conversations.service.js +383 -179
- package/dist/conversations/environment-status.d.ts +24 -0
- package/dist/conversations/environment-status.js +71 -0
- package/dist/conversations/event-log.d.ts +64 -27
- package/dist/conversations/event-log.js +167 -31
- package/dist/conversations/events.d.ts +252 -9
- package/dist/conversations/events.js +243 -32
- package/dist/conversations/files.d.ts +10 -10
- package/dist/conversations/files.js +37 -11
- package/dist/conversations/idem-keys.d.ts +5 -0
- package/dist/conversations/idem-keys.js +5 -0
- package/dist/conversations/job-ids.d.ts +6 -0
- package/dist/conversations/job-ids.js +6 -0
- package/dist/conversations/message-context.d.ts +79 -0
- package/dist/conversations/message-context.js +145 -0
- package/dist/conversations/outputs-read.d.ts +11 -11
- package/dist/conversations/outputs-read.js +31 -7
- package/dist/conversations/queue.d.ts +53 -58
- package/dist/conversations/queue.js +37 -293
- package/dist/conversations/queue.service.d.ts +60 -0
- package/dist/conversations/queue.service.js +350 -0
- package/dist/conversations/titles.d.ts +83 -27
- package/dist/conversations/titles.js +195 -71
- package/dist/conversations/tree.d.ts +17 -0
- package/dist/conversations/tree.js +42 -0
- package/dist/ctx/bus.js +9 -3
- package/dist/ctx/create-ctx.d.ts +26 -1
- package/dist/ctx/create-ctx.js +45 -16
- package/dist/ctx/lock.d.ts +7 -3
- package/dist/ctx/lock.js +9 -20
- package/dist/ctx/secrets.d.ts +10 -2
- package/dist/ctx/secrets.js +7 -3
- package/dist/db/kernel-schema.d.ts +589 -27
- package/dist/db/pool.d.ts +10 -0
- package/dist/db/pool.js +4 -0
- package/dist/devices/device-auth.controller.d.ts +42 -0
- package/dist/devices/device-auth.controller.js +116 -0
- package/dist/devices/device-auth.d.ts +119 -0
- package/dist/devices/device-auth.js +403 -0
- package/dist/devices/device-auth.module.d.ts +7 -0
- package/dist/devices/device-auth.module.js +25 -0
- package/dist/devices/device-gateway.d.ts +128 -0
- package/dist/devices/device-gateway.js +530 -0
- package/dist/devices/device-provider.d.ts +89 -0
- package/dist/devices/device-provider.js +151 -0
- package/dist/devices/device-routes.d.ts +36 -0
- package/dist/devices/device-routes.js +90 -0
- package/dist/devices/devices.module.d.ts +16 -0
- package/dist/devices/devices.module.js +44 -0
- package/dist/devices/dto.d.ts +66 -0
- package/dist/devices/dto.js +44 -0
- package/dist/devices/keys.d.ts +25 -0
- package/dist/devices/keys.js +25 -0
- package/dist/devices/machines.d.ts +118 -0
- package/dist/devices/machines.js +366 -0
- package/dist/devices/relay.d.ts +100 -0
- package/dist/devices/relay.js +373 -0
- package/dist/devices/targets.d.ts +28 -0
- package/dist/devices/targets.js +73 -0
- package/dist/environments/access.d.ts +66 -0
- package/dist/environments/access.js +113 -0
- package/dist/environments/conversation-environments.d.ts +75 -0
- package/dist/environments/conversation-environments.js +231 -0
- package/dist/environments/dto.d.ts +25 -0
- package/dist/environments/dto.js +12 -0
- package/dist/environments/environments.controller.d.ts +36 -0
- package/dist/environments/environments.controller.js +124 -0
- package/dist/environments/environments.module.d.ts +8 -0
- package/dist/environments/environments.module.js +31 -0
- package/dist/environments/environments.schemas.d.ts +3 -0
- package/dist/environments/environments.schemas.js +14 -0
- package/dist/environments/environments.service.d.ts +251 -0
- package/dist/environments/environments.service.js +791 -0
- package/dist/environments/label.d.ts +11 -0
- package/dist/environments/label.js +25 -0
- package/dist/environments/legacy-cleanup.job.d.ts +34 -0
- package/dist/environments/legacy-cleanup.job.js +141 -0
- package/dist/errors.d.ts +160 -2
- package/dist/errors.js +160 -2
- package/dist/evals/budget.d.ts +42 -21
- package/dist/evals/budget.js +114 -62
- package/dist/evals/evals.module.d.ts +31 -0
- package/dist/evals/evals.module.js +114 -0
- package/dist/evals/evals.schemas.d.ts +110 -0
- package/dist/evals/evals.schemas.js +107 -0
- package/dist/evals/jobs.d.ts +20 -0
- package/dist/evals/jobs.js +45 -0
- package/dist/evals/reconstruct.d.ts +18 -9
- package/dist/evals/reconstruct.js +36 -10
- package/dist/evals/recorded-tools.d.ts +37 -1
- package/dist/evals/recorded-tools.js +120 -10
- package/dist/evals/run-replay.d.ts +75 -10
- package/dist/evals/run-replay.js +218 -94
- package/dist/evals/runs.controller.d.ts +3 -58
- package/dist/evals/runs.controller.js +18 -50
- package/dist/evals/runs.service.d.ts +31 -52
- package/dist/evals/runs.service.js +281 -255
- package/dist/evals/sets.controller.d.ts +3 -29
- package/dist/evals/sets.controller.js +18 -39
- package/dist/evals/sets.service.d.ts +29 -13
- package/dist/evals/sets.service.js +262 -202
- package/dist/evals/snapshot.d.ts +5 -4
- package/dist/evals/snapshot.js +17 -4
- package/dist/evals/step-replay.d.ts +6 -8
- package/dist/evals/step-replay.js +17 -10
- package/dist/evals/trajectory.controller.d.ts +1 -1
- package/dist/evals/trajectory.controller.js +11 -7
- package/dist/evals/trajectory.d.ts +2 -2
- package/dist/evals/trajectory.js +1 -1
- package/dist/evals/trajectory.service.d.ts +13 -7
- package/dist/evals/trajectory.service.js +56 -21
- package/dist/evals/worker.d.ts +48 -31
- package/dist/evals/worker.js +229 -124
- package/dist/extensions/admin-ops.d.ts +133 -41
- package/dist/extensions/admin-ops.js +378 -172
- package/dist/extensions/admin-view.d.ts +82 -32
- package/dist/extensions/admin-view.js +284 -71
- package/dist/extensions/approvals.d.ts +24 -10
- package/dist/extensions/approvals.js +73 -46
- package/dist/extensions/blobs.d.ts +27 -10
- package/dist/extensions/blobs.js +112 -45
- package/dist/extensions/bootstrap.d.ts +99 -36
- package/dist/extensions/bootstrap.js +290 -134
- package/dist/extensions/cli.d.ts +32 -10
- package/dist/extensions/cli.js +52 -98
- package/dist/extensions/config-hash.d.ts +10 -0
- package/dist/extensions/config-hash.js +38 -0
- package/dist/extensions/config-issues.d.ts +19 -0
- package/dist/extensions/config-issues.js +48 -0
- package/dist/extensions/config-json-schema.d.ts +43 -7
- package/dist/extensions/config-json-schema.js +124 -58
- package/dist/extensions/deployment-config-store.d.ts +55 -0
- package/dist/extensions/deployment-config-store.js +133 -0
- package/dist/extensions/deployment-config.d.ts +146 -70
- package/dist/extensions/deployment-config.js +439 -246
- package/dist/extensions/display-name.d.ts +22 -0
- package/dist/extensions/display-name.js +77 -0
- package/dist/extensions/effective-config.d.ts +127 -0
- package/dist/extensions/effective-config.js +368 -0
- package/dist/extensions/enabled.d.ts +5 -5
- package/dist/extensions/extensions.module.d.ts +8 -0
- package/dist/extensions/extensions.module.js +96 -0
- package/dist/extensions/hooks.d.ts +40 -6
- package/dist/extensions/hooks.js +83 -17
- package/dist/extensions/host.d.ts +161 -80
- package/dist/extensions/host.js +505 -213
- package/dist/extensions/installs.d.ts +73 -19
- package/dist/extensions/installs.js +174 -37
- package/dist/extensions/job-ids.d.ts +6 -0
- package/dist/extensions/job-ids.js +6 -0
- package/dist/extensions/lifecycle-topics.d.ts +18 -0
- package/dist/extensions/lifecycle-topics.js +18 -0
- package/dist/extensions/lifecycle.d.ts +35 -47
- package/dist/extensions/lifecycle.js +103 -39
- package/dist/extensions/locks.d.ts +45 -0
- package/dist/extensions/locks.js +112 -0
- package/dist/extensions/migrate-lock.d.ts +3 -4
- package/dist/extensions/org-config.d.ts +141 -49
- package/dist/extensions/org-config.js +471 -243
- package/dist/extensions/org-levels.d.ts +37 -0
- package/dist/extensions/org-levels.js +71 -0
- package/dist/extensions/org-overrides.d.ts +151 -0
- package/dist/extensions/org-overrides.js +524 -0
- package/dist/extensions/package.d.ts +14 -1
- package/dist/extensions/package.js +51 -2
- package/dist/extensions/permission-view.d.ts +59 -6
- package/dist/extensions/permission-view.js +7 -5
- package/dist/extensions/plugin-guides.d.ts +129 -0
- package/dist/extensions/plugin-guides.js +283 -0
- package/dist/extensions/plugin-icons.d.ts +54 -0
- package/dist/extensions/plugin-icons.js +125 -0
- package/dist/extensions/plugin-items.d.ts +21 -0
- package/dist/extensions/plugin-items.js +42 -0
- package/dist/extensions/point-defaults.controller.d.ts +28 -0
- package/dist/extensions/point-defaults.controller.js +69 -0
- package/dist/extensions/point-defaults.d.ts +73 -0
- package/dist/extensions/point-defaults.js +236 -0
- package/dist/extensions/point-defaults.schemas.d.ts +9 -0
- package/dist/extensions/point-defaults.schemas.js +14 -0
- package/dist/extensions/registry.d.ts +50 -2
- package/dist/extensions/registry.js +91 -4
- package/dist/extensions/resolve.d.ts +51 -1
- package/dist/extensions/resolve.js +85 -3
- package/dist/extensions/single-provider.d.ts +92 -0
- package/dist/extensions/single-provider.js +180 -0
- package/dist/extensions/sources.d.ts +19 -10
- package/dist/extensions/sources.js +34 -13
- package/dist/extensions/state.module.d.ts +9 -0
- package/dist/extensions/state.module.js +38 -0
- package/dist/extensions/unavailable.d.ts +50 -0
- package/dist/extensions/unavailable.js +115 -0
- package/dist/files/files.module.d.ts +3 -0
- package/dist/files/files.module.js +22 -0
- package/dist/files/previewers.controller.d.ts +7 -2
- package/dist/files/previewers.controller.js +19 -12
- package/dist/files/previewers.d.ts +21 -19
- package/dist/files/previewers.js +114 -38
- package/dist/files/previewers.schemas.d.ts +10 -0
- package/dist/files/previewers.schemas.js +9 -0
- package/dist/http/adapter.d.ts +3 -0
- package/dist/http/adapter.js +25 -0
- package/dist/http/api.module.d.ts +3 -110
- package/dist/http/api.module.js +93 -358
- package/dist/http/app.d.ts +22 -17
- package/dist/http/app.js +46 -65
- package/dist/http/error-filter.module.d.ts +5 -0
- package/dist/http/error-filter.module.js +18 -0
- package/dist/http/error.filter.js +2 -2
- package/dist/http/{gateway.module.d.ts → gateway-routes.d.ts} +7 -3
- package/dist/http/gateway-routes.js +16 -0
- package/dist/http/nest-logger.d.ts +6 -5
- package/dist/http/nest-logger.js +24 -10
- package/dist/http/plugin-dispatch.d.ts +4 -1
- package/dist/http/plugin-dispatch.js +14 -2
- package/dist/identity/accounts.d.ts +137 -38
- package/dist/identity/accounts.js +386 -186
- package/dist/identity/api-tokens.d.ts +11 -7
- package/dist/identity/api-tokens.js +71 -44
- package/dist/identity/audit.d.ts +9 -5
- package/dist/identity/audit.js +5 -1
- package/dist/identity/auth.controller.d.ts +11 -10
- package/dist/identity/auth.controller.js +40 -30
- package/dist/identity/auth.guard.d.ts +16 -8
- package/dist/identity/auth.guard.js +41 -13
- package/dist/identity/binding.d.ts +9 -11
- package/dist/identity/binding.js +41 -13
- package/dist/identity/cookies.d.ts +26 -5
- package/dist/identity/cookies.js +58 -6
- package/dist/identity/external-identity-lock.d.ts +17 -0
- package/dist/identity/external-identity-lock.js +39 -0
- package/dist/identity/external.d.ts +19 -15
- package/dist/identity/external.js +163 -125
- package/dist/identity/identity.module.d.ts +6 -0
- package/dist/identity/identity.module.js +59 -0
- package/dist/identity/initial-passwords.controller.d.ts +22 -0
- package/dist/identity/initial-passwords.controller.js +49 -0
- package/dist/identity/initial-passwords.d.ts +61 -0
- package/dist/identity/initial-passwords.job.d.ts +16 -0
- package/dist/identity/initial-passwords.job.js +46 -0
- package/dist/identity/initial-passwords.js +159 -0
- package/dist/identity/me.controller.d.ts +32 -15
- package/dist/identity/me.controller.js +80 -33
- package/dist/identity/me.schemas.d.ts +24 -0
- package/dist/identity/me.schemas.js +25 -0
- package/dist/identity/principal.d.ts +11 -3
- package/dist/identity/principal.js +11 -3
- package/dist/identity/sessions.d.ts +24 -10
- package/dist/identity/sessions.js +119 -42
- package/dist/identity/settings.d.ts +69 -23
- package/dist/identity/settings.js +201 -72
- package/dist/identity/sso.controller.d.ts +7 -7
- package/dist/identity/sso.controller.js +13 -9
- package/dist/identity/sso.d.ts +9 -12
- package/dist/identity/sso.js +54 -24
- package/dist/identity/super-admin.d.ts +20 -0
- package/dist/identity/super-admin.js +81 -0
- package/dist/inbox/inbox.module.d.ts +6 -0
- package/dist/inbox/inbox.module.js +26 -0
- package/dist/inbox/inbox.schemas.d.ts +8 -0
- package/dist/inbox/inbox.schemas.js +11 -0
- package/dist/inbox/inbox.service.d.ts +68 -11
- package/dist/inbox/inbox.service.js +104 -13
- package/dist/inbox/org-notices.d.ts +63 -0
- package/dist/inbox/org-notices.js +91 -0
- package/dist/inbox/platform-inbox.controller.d.ts +23 -0
- package/dist/inbox/platform-inbox.controller.js +58 -0
- package/dist/inbox/platform-inbox.service.d.ts +151 -0
- package/dist/inbox/platform-inbox.service.js +263 -0
- package/dist/inbox/platform-notices.d.ts +19 -0
- package/dist/inbox/platform-notices.js +9 -0
- package/dist/index.d.ts +22 -13
- package/dist/index.js +16 -7
- package/dist/infra/cache/cache.module.d.ts +3 -0
- package/dist/infra/cache/cache.module.js +16 -0
- package/dist/infra/cache/cache.service.d.ts +21 -0
- package/dist/infra/cache/cache.service.js +80 -0
- package/dist/infra/cache/cacheable.decorator.d.ts +32 -0
- package/dist/infra/cache/cacheable.decorator.js +83 -0
- package/dist/infra/cli/kernel-cli.module.d.ts +66 -0
- package/dist/infra/cli/kernel-cli.module.js +173 -0
- package/dist/infra/close-within.d.ts +15 -0
- package/dist/infra/close-within.js +36 -0
- package/dist/infra/config/config.module.d.ts +13 -0
- package/dist/infra/config/config.module.js +39 -0
- package/dist/infra/config/kernel-config.service.d.ts +25 -0
- package/dist/infra/config/kernel-config.service.js +157 -0
- package/dist/infra/database/after-commit.d.ts +6 -0
- package/dist/infra/database/after-commit.js +9 -0
- package/dist/infra/database/database.module.d.ts +25 -0
- package/dist/infra/database/database.module.js +91 -0
- package/dist/infra/database/kernel-db.d.ts +22 -0
- package/dist/infra/database/kernel-db.js +65 -0
- package/dist/infra/database/transaction-host.d.ts +50 -0
- package/dist/infra/database/transaction-host.js +125 -0
- package/dist/infra/database/transactional.decorator.d.ts +15 -0
- package/dist/infra/database/transactional.decorator.js +27 -0
- package/dist/infra/database/xact-lock.d.ts +17 -0
- package/dist/infra/database/xact-lock.js +43 -0
- package/dist/infra/events/kernel-events.d.ts +121 -0
- package/dist/infra/events/kernel-events.js +65 -0
- package/dist/infra/health/health.controller.d.ts +26 -0
- package/dist/infra/health/health.controller.js +104 -0
- package/dist/infra/health/health.module.d.ts +9 -0
- package/dist/infra/health/health.module.js +24 -0
- package/dist/infra/health/health.options.d.ts +10 -0
- package/dist/infra/health/health.options.js +15 -0
- package/dist/infra/health/shutdown-state.d.ts +6 -0
- package/dist/infra/health/shutdown-state.js +2 -0
- package/dist/infra/lifecycle/graceful-shutdown.module.d.ts +6 -0
- package/dist/infra/lifecycle/graceful-shutdown.module.js +23 -0
- package/dist/infra/lifecycle/graceful-shutdown.service.d.ts +70 -0
- package/dist/infra/lifecycle/graceful-shutdown.service.js +135 -0
- package/dist/infra/lifecycle/process-guards.d.ts +52 -0
- package/dist/infra/lifecycle/process-guards.js +90 -0
- package/dist/infra/lifecycle/shutdown-signals.d.ts +30 -0
- package/dist/infra/lifecycle/shutdown-signals.js +48 -0
- package/dist/infra/lock/key-template.d.ts +13 -0
- package/dist/infra/lock/key-template.js +46 -0
- package/dist/infra/lock/lock.module.d.ts +3 -0
- package/dist/infra/lock/lock.module.js +16 -0
- package/dist/infra/lock/lock.service.d.ts +120 -0
- package/dist/infra/lock/lock.service.js +343 -0
- package/dist/infra/lock/shared-lease.d.ts +44 -0
- package/dist/infra/lock/shared-lease.js +121 -0
- package/dist/infra/lock/with-lock.decorator.d.ts +28 -0
- package/dist/infra/lock/with-lock.decorator.js +49 -0
- package/dist/infra/logging/logging.module.d.ts +18 -0
- package/dist/infra/logging/logging.module.js +52 -0
- package/dist/infra/queue/add-job-once.d.ts +14 -0
- package/dist/infra/queue/add-job-once.js +16 -0
- package/dist/infra/queue/configured-processor.d.ts +18 -0
- package/dist/infra/queue/configured-processor.js +23 -0
- package/dist/infra/queue/job-failure.d.ts +18 -0
- package/dist/infra/queue/job-failure.js +26 -0
- package/dist/infra/queue/queue-shutdown.d.ts +27 -0
- package/dist/infra/queue/queue-shutdown.js +81 -0
- package/dist/infra/queue/queue.module.d.ts +16 -0
- package/dist/infra/queue/queue.module.js +61 -0
- package/dist/infra/queue/redis-connection.d.ts +13 -0
- package/dist/infra/queue/redis-connection.js +25 -0
- package/dist/infra/queue/retired-schedulers.d.ts +7 -0
- package/dist/infra/queue/retired-schedulers.js +9 -0
- package/dist/infra/queue/run-job.d.ts +16 -0
- package/dist/infra/queue/run-job.js +19 -0
- package/dist/infra/redis/redis.module.d.ts +24 -0
- package/dist/infra/redis/redis.module.js +91 -0
- package/dist/infra/schedule/cron-tick.d.ts +12 -0
- package/dist/infra/schedule/cron-tick.js +25 -0
- package/dist/infra/schedule/locked-cron.decorator.d.ts +27 -0
- package/dist/infra/schedule/locked-cron.decorator.js +53 -0
- package/dist/infra/schedule/locked-interval.d.ts +69 -0
- package/dist/infra/schedule/locked-interval.js +190 -0
- package/dist/infra/schedule/locked-tick.d.ts +90 -0
- package/dist/infra/schedule/locked-tick.js +137 -0
- package/dist/infra/schedule/schedule.module.d.ts +8 -0
- package/dist/infra/schedule/schedule.module.js +27 -0
- package/dist/infra/schedule/scheduled-runs.d.ts +69 -0
- package/dist/infra/schedule/scheduled-runs.js +163 -0
- package/dist/infra/swagger/swagger.d.ts +16 -0
- package/dist/infra/swagger/swagger.js +31 -0
- package/dist/infra/throttler/credential-throttler.d.ts +35 -0
- package/dist/infra/throttler/credential-throttler.js +77 -0
- package/dist/infra/throttler/named-throttlers.d.ts +36 -0
- package/dist/infra/throttler/named-throttlers.js +157 -0
- package/dist/infra/throttler/redis-throttler-storage.d.ts +38 -0
- package/dist/infra/throttler/redis-throttler-storage.js +130 -0
- package/dist/infra/throttler/throttler.guard.d.ts +23 -0
- package/dist/infra/throttler/throttler.guard.js +68 -0
- package/dist/infra/throttler/throttler.module.d.ts +15 -0
- package/dist/infra/throttler/throttler.module.js +53 -0
- package/dist/infra/tokens.d.ts +20 -0
- package/dist/infra/tokens.js +20 -0
- package/dist/infra/validation/create-zod-dto.d.ts +23 -0
- package/dist/infra/validation/create-zod-dto.js +35 -0
- package/dist/infra/validation/parse-with-zod.d.ts +7 -0
- package/dist/infra/validation/parse-with-zod.js +12 -0
- package/dist/infra/validation/validation.module.d.ts +3 -0
- package/dist/infra/validation/validation.module.js +16 -0
- package/dist/infra/validation/zod-params.d.ts +7 -0
- package/dist/infra/validation/zod-params.js +42 -0
- package/dist/infra/validation/zod-validation.pipe.d.ts +11 -0
- package/dist/infra/validation/zod-validation.pipe.js +31 -0
- package/dist/kernel.d.ts +9 -244
- package/dist/kernel.js +8 -1305
- package/dist/kernel.module.d.ts +17 -0
- package/dist/kernel.module.js +145 -0
- package/dist/lifecycle/kernel-lifecycle.d.ts +140 -0
- package/dist/lifecycle/kernel-lifecycle.js +465 -0
- package/dist/lifecycle/tokens.d.ts +3 -0
- package/dist/lifecycle/tokens.js +3 -0
- package/dist/log-context.d.ts +2 -0
- package/dist/log-context.js +4 -0
- package/dist/log.d.ts +13 -1
- package/dist/log.js +93 -17
- package/dist/models/embeddings.d.ts +82 -0
- package/dist/models/embeddings.js +326 -0
- package/dist/models/enablement.controller.d.ts +40 -0
- package/dist/models/enablement.controller.js +74 -0
- package/dist/models/enablement.d.ts +101 -0
- package/dist/models/enablement.js +264 -0
- package/dist/models/models.module.d.ts +7 -0
- package/dist/models/models.module.js +36 -0
- package/dist/models/org-models.schemas.d.ts +8 -0
- package/dist/models/org-models.schemas.js +22 -0
- package/dist/models/prices.d.ts +13 -5
- package/dist/models/prices.js +69 -39
- package/dist/models/purposes.controller.d.ts +6 -4
- package/dist/models/purposes.controller.js +16 -11
- package/dist/models/purposes.d.ts +103 -23
- package/dist/models/purposes.js +208 -77
- package/dist/orgs/directory-sync.controller.d.ts +1 -1
- package/dist/orgs/directory-sync.controller.js +2 -2
- package/dist/orgs/directory-sync.d.ts +74 -35
- package/dist/orgs/directory-sync.js +243 -132
- package/dist/orgs/invitations.controller.d.ts +8 -8
- package/dist/orgs/invitations.controller.js +27 -26
- package/dist/orgs/invitations.service.d.ts +12 -9
- package/dist/orgs/invitations.service.js +130 -92
- package/dist/orgs/job-ids.d.ts +8 -0
- package/dist/orgs/job-ids.js +8 -0
- package/dist/orgs/members.service.d.ts +19 -12
- package/dist/orgs/members.service.js +161 -117
- package/dist/orgs/offboarding.d.ts +61 -60
- package/dist/orgs/offboarding.js +206 -95
- package/dist/orgs/org-lock.d.ts +33 -19
- package/dist/orgs/org-lock.js +94 -35
- package/dist/orgs/org.guard.d.ts +11 -1
- package/dist/orgs/org.guard.js +45 -34
- package/dist/orgs/orgs.controller.d.ts +13 -16
- package/dist/orgs/orgs.controller.js +29 -20
- package/dist/orgs/orgs.module.d.ts +6 -0
- package/dist/orgs/orgs.module.js +71 -0
- package/dist/orgs/orgs.schemas.d.ts +8 -0
- package/dist/orgs/orgs.schemas.js +16 -0
- package/dist/orgs/orgs.service.d.ts +47 -21
- package/dist/orgs/orgs.service.js +166 -91
- package/dist/orgs/roles.d.ts +1 -1
- package/dist/orgs/roles.js +5 -1
- package/dist/orgs/settings.controller.d.ts +2 -2
- package/dist/orgs/settings.controller.js +8 -6
- package/dist/orgs/settings.service.d.ts +76 -14
- package/dist/orgs/settings.service.js +140 -51
- package/dist/overrides.d.ts +63 -0
- package/dist/overrides.js +34 -0
- package/dist/platform/actor.d.ts +48 -0
- package/dist/platform/actor.js +100 -0
- package/dist/platform/agent-docs.d.ts +18 -0
- package/dist/platform/agent-docs.js +59 -0
- package/dist/platform/availability.d.ts +14 -0
- package/dist/platform/availability.js +43 -0
- package/dist/platform/errors.d.ts +28 -0
- package/dist/platform/errors.js +65 -0
- package/dist/platform/exclusions.d.ts +11 -0
- package/dist/platform/exclusions.js +683 -0
- package/dist/platform/method.d.ts +6 -0
- package/dist/platform/method.js +15 -0
- package/dist/platform/operation.d.ts +169 -0
- package/dist/platform/operation.js +94 -0
- package/dist/platform/ops/accounts.ops.d.ts +32 -0
- package/dist/platform/ops/accounts.ops.js +291 -0
- package/dist/platform/ops/admin.ops.d.ts +38 -0
- package/dist/platform/ops/admin.ops.js +407 -0
- package/dist/platform/ops/agent-skills.ops.d.ts +24 -0
- package/dist/platform/ops/agent-skills.ops.js +226 -0
- package/dist/platform/ops/agent-templates.ops.d.ts +17 -0
- package/dist/platform/ops/agent-templates.ops.js +272 -0
- package/dist/platform/ops/agents.ops.d.ts +37 -0
- package/dist/platform/ops/agents.ops.js +717 -0
- package/dist/platform/ops/approvals.ops.d.ts +30 -0
- package/dist/platform/ops/approvals.ops.js +116 -0
- package/dist/platform/ops/background.ops.d.ts +14 -0
- package/dist/platform/ops/background.ops.js +134 -0
- package/dist/platform/ops/catalog-admin.ops.d.ts +74 -0
- package/dist/platform/ops/catalog-admin.ops.js +599 -0
- package/dist/platform/ops/channels.ops.d.ts +16 -0
- package/dist/platform/ops/channels.ops.js +195 -0
- package/dist/platform/ops/components.ops.d.ts +46 -0
- package/dist/platform/ops/components.ops.js +854 -0
- package/dist/platform/ops/connectors.ops.d.ts +17 -0
- package/dist/platform/ops/connectors.ops.js +188 -0
- package/dist/platform/ops/conversations.ops.d.ts +30 -0
- package/dist/platform/ops/conversations.ops.js +470 -0
- package/dist/platform/ops/environments.ops.d.ts +26 -0
- package/dist/platform/ops/environments.ops.js +311 -0
- package/dist/platform/ops/errors.d.ts +16 -0
- package/dist/platform/ops/errors.js +30 -0
- package/dist/platform/ops/evals.ops.d.ts +28 -0
- package/dist/platform/ops/evals.ops.js +783 -0
- package/dist/platform/ops/extensions-admin.ops.d.ts +25 -0
- package/dist/platform/ops/extensions-admin.ops.js +372 -0
- package/dist/platform/ops/extensions.ops.d.ts +25 -0
- package/dist/platform/ops/extensions.ops.js +347 -0
- package/dist/platform/ops/inbox.ops.d.ts +25 -0
- package/dist/platform/ops/inbox.ops.js +378 -0
- package/dist/platform/ops/marketplaces.ops.d.ts +22 -0
- package/dist/platform/ops/marketplaces.ops.js +466 -0
- package/dist/platform/ops/models.ops.d.ts +25 -0
- package/dist/platform/ops/models.ops.js +232 -0
- package/dist/platform/ops/org-plugins.ops.d.ts +29 -0
- package/dist/platform/ops/org-plugins.ops.js +329 -0
- package/dist/platform/ops/org-settings.ops.d.ts +18 -0
- package/dist/platform/ops/org-settings.ops.js +127 -0
- package/dist/platform/ops/orgs.ops.d.ts +28 -0
- package/dist/platform/ops/orgs.ops.js +296 -0
- package/dist/platform/ops/other-orgs.d.ts +20 -0
- package/dist/platform/ops/other-orgs.js +72 -0
- package/dist/platform/ops/paging.d.ts +14 -0
- package/dist/platform/ops/paging.js +20 -0
- package/dist/platform/ops/platform.ops.d.ts +30 -0
- package/dist/platform/ops/platform.ops.js +231 -0
- package/dist/platform/ops/plugins.ops.d.ts +35 -0
- package/dist/platform/ops/plugins.ops.js +476 -0
- package/dist/platform/ops/previewers.ops.d.ts +14 -0
- package/dist/platform/ops/previewers.ops.js +109 -0
- package/dist/platform/ops/requests.ops.d.ts +14 -0
- package/dist/platform/ops/requests.ops.js +191 -0
- package/dist/platform/ops/role.d.ts +4 -0
- package/dist/platform/ops/role.js +10 -0
- package/dist/platform/ops/skills.ops.d.ts +17 -0
- package/dist/platform/ops/skills.ops.js +216 -0
- package/dist/platform/ops/summary.d.ts +10 -0
- package/dist/platform/ops/summary.js +18 -0
- package/dist/platform/ops/themes.ops.d.ts +14 -0
- package/dist/platform/ops/themes.ops.js +47 -0
- package/dist/platform/ops/unavailable.d.ts +23 -0
- package/dist/platform/ops/unavailable.js +118 -0
- package/dist/platform/parity.d.ts +33 -0
- package/dist/platform/parity.js +272 -0
- package/dist/platform/platform-ops.module.d.ts +40 -0
- package/dist/platform/platform-ops.module.js +127 -0
- package/dist/platform/platform.module.d.ts +8 -0
- package/dist/platform/platform.module.js +25 -0
- package/dist/platform/registry.d.ts +30 -0
- package/dist/platform/registry.js +111 -0
- package/dist/platform/route-schemas.d.ts +18 -0
- package/dist/platform/route-schemas.js +66 -0
- package/dist/roles/gateway-role.d.ts +54 -0
- package/dist/roles/gateway-role.js +118 -0
- package/dist/roles/gateway.module.d.ts +9 -0
- package/dist/roles/gateway.module.js +31 -0
- package/dist/roles/worker-role.d.ts +65 -0
- package/dist/roles/worker-role.js +151 -0
- package/dist/roles/worker.module.d.ts +14 -0
- package/dist/roles/worker.module.js +106 -0
- package/dist/runtime/abort.js +19 -2
- package/dist/runtime/checkpointer.d.ts +8 -3
- package/dist/runtime/checkpointer.js +87 -55
- package/dist/runtime/context/build.d.ts +21 -13
- package/dist/runtime/context/build.js +106 -29
- package/dist/runtime/context/claude-tools.d.ts +4 -1
- package/dist/runtime/context/claude-tools.js +13 -12
- package/dist/runtime/context/compaction.js +24 -1
- package/dist/runtime/context/environment-paths.d.ts +40 -0
- package/dist/runtime/context/environment-paths.js +48 -0
- package/dist/runtime/context/legacy-text.d.ts +22 -0
- package/dist/runtime/context/legacy-text.js +37 -0
- package/dist/runtime/context/reminders.d.ts +2 -0
- package/dist/runtime/context/reminders.js +2 -0
- package/dist/runtime/context/run-context.d.ts +65 -0
- package/dist/runtime/context/run-context.js +215 -0
- package/dist/runtime/context/skills.d.ts +29 -11
- package/dist/runtime/context/skills.js +46 -32
- package/dist/runtime/context/tool-changes.d.ts +27 -0
- package/dist/runtime/context/tool-changes.js +60 -0
- package/dist/runtime/conversation-lock.d.ts +20 -11
- package/dist/runtime/conversation-lock.js +62 -72
- package/dist/runtime/hook-calls.d.ts +11 -4
- package/dist/runtime/hook-calls.js +41 -13
- package/dist/runtime/job-ids.d.ts +10 -0
- package/dist/runtime/job-ids.js +10 -0
- package/dist/runtime/langgraph-loop.d.ts +3 -1
- package/dist/runtime/langgraph-loop.js +37 -18
- package/dist/runtime/loop.d.ts +42 -6
- package/dist/runtime/loop.js +14 -0
- package/dist/runtime/maintenance-queue.d.ts +10 -0
- package/dist/runtime/maintenance-queue.js +10 -0
- package/dist/runtime/maintenance.d.ts +50 -74
- package/dist/runtime/maintenance.js +166 -137
- package/dist/runtime/model-call.d.ts +61 -15
- package/dist/runtime/model-call.js +128 -84
- package/dist/runtime/providers.d.ts +158 -27
- package/dist/runtime/providers.js +373 -63
- package/dist/runtime/quota.d.ts +17 -3
- package/dist/runtime/quota.js +27 -2
- package/dist/runtime/retired-schedulers.d.ts +7 -0
- package/dist/runtime/retired-schedulers.js +31 -0
- package/dist/runtime/run-context.service.d.ts +65 -0
- package/dist/runtime/run-context.service.js +236 -0
- package/dist/runtime/run-control.d.ts +43 -0
- package/dist/runtime/run-control.js +89 -0
- package/dist/runtime/run-drafts.d.ts +85 -0
- package/dist/runtime/run-drafts.js +230 -0
- package/dist/runtime/run-finish.d.ts +27 -18
- package/dist/runtime/run-finish.js +98 -51
- package/dist/runtime/run-jobs.d.ts +46 -0
- package/dist/runtime/run-jobs.js +91 -0
- package/dist/runtime/run-processors.d.ts +60 -0
- package/dist/runtime/run-processors.js +185 -0
- package/dist/runtime/run-queue.d.ts +90 -58
- package/dist/runtime/run-queue.js +566 -297
- package/dist/runtime/run-steps.d.ts +39 -0
- package/dist/runtime/run-steps.js +96 -0
- package/dist/runtime/run-worker.d.ts +134 -77
- package/dist/runtime/run-worker.js +412 -237
- package/dist/runtime/runtime.module.d.ts +10 -0
- package/dist/runtime/runtime.module.js +122 -0
- package/dist/runtime/step-log.d.ts +51 -0
- package/dist/runtime/step-log.js +111 -0
- package/dist/runtime/stream.controller.d.ts +28 -9
- package/dist/runtime/stream.controller.js +72 -13
- package/dist/runtime/stream.d.ts +27 -6
- package/dist/runtime/stream.js +102 -7
- package/dist/runtime/subagents/child.d.ts +58 -51
- package/dist/runtime/subagents/child.js +105 -86
- package/dist/runtime/subagents/definition.d.ts +32 -28
- package/dist/runtime/subagents/definition.js +89 -143
- package/dist/runtime/subagents/deliveries.d.ts +71 -0
- package/dist/runtime/subagents/deliveries.js +228 -0
- package/dist/runtime/subagents/dispatch.d.ts +197 -27
- package/dist/runtime/subagents/dispatch.js +336 -270
- package/dist/runtime/subagents/limits.d.ts +32 -8
- package/dist/runtime/subagents/limits.js +44 -8
- package/dist/runtime/subagents/messages.d.ts +95 -0
- package/dist/runtime/subagents/messages.js +328 -0
- package/dist/runtime/subagents/org-agents.d.ts +4 -4
- package/dist/runtime/subagents/org-agents.js +3 -2
- package/dist/runtime/subagents/preamble.d.ts +4 -4
- package/dist/runtime/subagents/preamble.js +9 -7
- package/dist/runtime/subagents/progress.d.ts +10 -8
- package/dist/runtime/subagents/progress.js +11 -6
- package/dist/runtime/subagents/render.d.ts +30 -0
- package/dist/runtime/subagents/render.js +53 -0
- package/dist/runtime/subagents/roster.d.ts +27 -10
- package/dist/runtime/subagents/roster.js +49 -13
- package/dist/runtime/subagents/skill-fork.d.ts +10 -10
- package/dist/runtime/subagents/skill-fork.js +23 -8
- package/dist/runtime/subagents/status.d.ts +14 -0
- package/dist/runtime/subagents/status.js +23 -0
- package/dist/runtime/subagents/subagents.service.d.ts +43 -0
- package/dist/runtime/subagents/subagents.service.js +106 -0
- package/dist/runtime/usage-report-jobs.d.ts +88 -0
- package/dist/runtime/usage-report-jobs.js +158 -0
- package/dist/runtime/usage-report.d.ts +121 -83
- package/dist/runtime/usage-report.js +355 -181
- package/dist/runtime/waits.d.ts +258 -81
- package/dist/runtime/waits.js +800 -263
- package/dist/sandbox/job-ids.d.ts +10 -0
- package/dist/sandbox/job-ids.js +10 -0
- package/dist/sandbox/jobs.d.ts +68 -0
- package/dist/sandbox/jobs.js +150 -0
- package/dist/sandbox/leases.d.ts +73 -82
- package/dist/sandbox/leases.js +162 -346
- package/dist/sandbox/providers.d.ts +46 -0
- package/dist/sandbox/providers.js +83 -0
- package/dist/sandbox/sandbox.module.d.ts +9 -0
- package/dist/sandbox/sandbox.module.js +36 -0
- package/dist/sandbox/sandbox.service.d.ts +359 -151
- package/dist/sandbox/sandbox.service.js +950 -435
- package/dist/sandbox/unreachable.d.ts +30 -0
- package/dist/sandbox/unreachable.js +108 -0
- package/dist/skills/catalog.d.ts +40 -26
- package/dist/skills/catalog.js +120 -99
- package/dist/skills/changes.controller.d.ts +11 -4
- package/dist/skills/changes.controller.js +34 -9
- package/dist/skills/changes.service.d.ts +31 -21
- package/dist/skills/changes.service.js +426 -368
- package/dist/skills/invocable.controller.d.ts +2 -2
- package/dist/skills/invocable.controller.js +10 -7
- package/dist/skills/invocation-keys.d.ts +1 -1
- package/dist/skills/invocation-keys.js +1 -1
- package/dist/skills/invoke.d.ts +19 -18
- package/dist/skills/invoke.js +68 -24
- package/dist/skills/package.d.ts +20 -1
- package/dist/skills/package.js +28 -1
- package/dist/skills/skills.controller.d.ts +33 -7
- package/dist/skills/skills.controller.js +79 -12
- package/dist/skills/skills.module.d.ts +6 -0
- package/dist/skills/skills.module.js +39 -0
- package/dist/skills/skills.schemas.d.ts +48 -0
- package/dist/skills/skills.schemas.js +61 -0
- package/dist/skills/skills.service.d.ts +151 -44
- package/dist/skills/skills.service.js +636 -204
- package/dist/skills/usable.d.ts +81 -0
- package/dist/skills/usable.js +379 -0
- package/dist/storage/job-ids.d.ts +6 -0
- package/dist/storage/job-ids.js +6 -0
- package/dist/storage/purge.d.ts +22 -7
- package/dist/storage/purge.js +55 -3
- package/dist/storage/storage.module.d.ts +6 -0
- package/dist/storage/storage.module.js +23 -0
- package/dist/storage/storage.service.d.ts +18 -4
- package/dist/storage/storage.service.js +44 -3
- package/dist/themes/render-css.d.ts +12 -0
- package/dist/themes/render-css.js +132 -0
- package/dist/themes/theme-delivery.service.d.ts +34 -0
- package/dist/themes/theme-delivery.service.js +111 -0
- package/dist/themes/theme-state.d.ts +95 -0
- package/dist/themes/theme-state.js +203 -0
- package/dist/themes/theme-usage.d.ts +61 -0
- package/dist/themes/theme-usage.js +98 -0
- package/dist/themes/themes-public.controller.d.ts +28 -0
- package/dist/themes/themes-public.controller.js +92 -0
- package/dist/themes/themes.controller.d.ts +13 -0
- package/dist/themes/themes.controller.js +38 -0
- package/dist/themes/themes.module.d.ts +3 -0
- package/dist/themes/themes.module.js +21 -0
- package/dist/themes/themes.schemas.d.ts +3 -0
- package/dist/themes/themes.schemas.js +2 -0
- package/dist/themes/themes.service.d.ts +50 -0
- package/dist/themes/themes.service.js +99 -0
- package/dist/time-zone.d.ts +5 -0
- package/dist/time-zone.js +19 -0
- package/dist/tools/approvals.controller.d.ts +1 -1
- package/dist/tools/approvals.controller.js +14 -11
- package/dist/tools/approvals.schemas.d.ts +21 -0
- package/dist/tools/approvals.schemas.js +15 -0
- package/dist/tools/approvals.service.d.ts +53 -27
- package/dist/tools/approvals.service.js +365 -260
- package/dist/tools/breaker.d.ts +2 -2
- package/dist/tools/breaker.js +21 -3
- package/dist/tools/builtin-names.d.ts +109 -0
- package/dist/tools/builtin-names.js +173 -0
- package/dist/tools/builtins/agent.d.ts +27 -25
- package/dist/tools/builtins/agent.js +66 -63
- package/dist/tools/builtins/ask-user.d.ts +3 -3
- package/dist/tools/builtins/ask-user.js +3 -3
- package/dist/tools/builtins/builtin-tools.d.ts +41 -0
- package/dist/tools/builtins/builtin-tools.js +151 -0
- package/dist/tools/builtins/current-time.d.ts +26 -0
- package/dist/tools/builtins/current-time.js +99 -0
- package/dist/tools/builtins/handback.d.ts +28 -0
- package/dist/tools/builtins/handback.js +53 -0
- package/dist/tools/builtins/platform.d.ts +51 -0
- package/dist/tools/builtins/platform.js +451 -0
- package/dist/tools/builtins/read-tool-output.d.ts +3 -3
- package/dist/tools/builtins/read-tool-output.js +3 -3
- package/dist/tools/builtins/sandbox/bash.d.ts +5 -5
- package/dist/tools/builtins/sandbox/bash.js +293 -41
- package/dist/tools/builtins/sandbox/common.d.ts +137 -37
- package/dist/tools/builtins/sandbox/common.js +222 -62
- package/dist/tools/builtins/sandbox/deliver.d.ts +1 -1
- package/dist/tools/builtins/sandbox/deliver.js +21 -18
- package/dist/tools/builtins/sandbox/edit.d.ts +1 -1
- package/dist/tools/builtins/sandbox/edit.js +65 -51
- package/dist/tools/builtins/sandbox/file-state.d.ts +34 -0
- package/dist/tools/builtins/sandbox/file-state.js +34 -0
- package/dist/tools/builtins/sandbox/glob-match.d.ts +1 -1
- package/dist/tools/builtins/sandbox/glob-match.js +1 -1
- package/dist/tools/builtins/sandbox/glob.d.ts +1 -1
- package/dist/tools/builtins/sandbox/glob.js +19 -14
- package/dist/tools/builtins/sandbox/grep.d.ts +7 -4
- package/dist/tools/builtins/sandbox/grep.js +155 -36
- package/dist/tools/builtins/sandbox/index.d.ts +9 -5
- package/dist/tools/builtins/sandbox/index.js +13 -5
- package/dist/tools/builtins/sandbox/read.d.ts +2 -2
- package/dist/tools/builtins/sandbox/read.js +30 -20
- package/dist/tools/builtins/sandbox/task-output.d.ts +79 -0
- package/dist/tools/builtins/sandbox/task-output.js +334 -0
- package/dist/tools/builtins/sandbox/task-stop.d.ts +10 -0
- package/dist/tools/builtins/sandbox/task-stop.js +113 -0
- package/dist/tools/builtins/sandbox/write.d.ts +1 -1
- package/dist/tools/builtins/sandbox/write.js +52 -45
- package/dist/tools/builtins/send-message.d.ts +56 -0
- package/dist/tools/builtins/send-message.js +100 -0
- package/dist/tools/builtins/skill-load.d.ts +32 -21
- package/dist/tools/builtins/skill-load.js +125 -74
- package/dist/tools/builtins/skill-write.d.ts +4 -4
- package/dist/tools/builtins/skill-write.js +12 -12
- package/dist/tools/builtins/tool-search.d.ts +13 -9
- package/dist/tools/builtins/tool-search.js +25 -14
- package/dist/tools/builtins/web/errors.d.ts +8 -0
- package/dist/tools/builtins/web/errors.js +53 -0
- package/dist/tools/builtins/web/extract.d.ts +62 -0
- package/dist/tools/builtins/web/extract.js +271 -0
- package/dist/tools/builtins/web/fetch.d.ts +55 -0
- package/dist/tools/builtins/web/fetch.js +175 -0
- package/dist/tools/builtins/web/index.d.ts +13 -0
- package/dist/tools/builtins/web/index.js +12 -0
- package/dist/tools/builtins/web/page-cache.d.ts +25 -0
- package/dist/tools/builtins/web/page-cache.js +77 -0
- package/dist/tools/builtins/web/search.d.ts +25 -0
- package/dist/tools/builtins/web/search.js +210 -0
- package/dist/tools/builtins/web/url.d.ts +17 -0
- package/dist/tools/builtins/web/url.js +43 -0
- package/dist/tools/canonical-json.d.ts +6 -0
- package/dist/tools/canonical-json.js +16 -0
- package/dist/tools/connector-content.controller.d.ts +1 -1
- package/dist/tools/connector-content.controller.js +11 -6
- package/dist/tools/connector-content.d.ts +12 -15
- package/dist/tools/connector-content.js +53 -18
- package/dist/tools/connectors.controller.d.ts +16 -3
- package/dist/tools/connectors.controller.js +34 -14
- package/dist/tools/connectors.schemas.d.ts +12 -0
- package/dist/tools/connectors.schemas.js +19 -0
- package/dist/tools/connectors.service.d.ts +80 -18
- package/dist/tools/connectors.service.js +529 -358
- package/dist/tools/credentials.d.ts +20 -20
- package/dist/tools/credentials.js +55 -34
- package/dist/tools/display-names.d.ts +9 -0
- package/dist/tools/display-names.js +24 -0
- package/dist/tools/drift.d.ts +29 -41
- package/dist/tools/drift.js +116 -84
- package/dist/tools/elicitation.js +1 -1
- package/dist/tools/extension-tools.controller.d.ts +14 -0
- package/dist/tools/extension-tools.controller.js +37 -0
- package/dist/tools/info.d.ts +73 -13
- package/dist/tools/info.js +151 -39
- package/dist/tools/job-ids.d.ts +13 -0
- package/dist/tools/job-ids.js +15 -0
- package/dist/tools/jobs.d.ts +24 -0
- package/dist/tools/jobs.js +69 -0
- package/dist/tools/legacy-data.d.ts +34 -0
- package/dist/tools/legacy-data.js +76 -0
- package/dist/tools/legacy-keys.d.ts +51 -0
- package/dist/tools/legacy-keys.js +115 -0
- package/dist/tools/naming.d.ts +1 -5
- package/dist/tools/naming.js +1 -7
- package/dist/tools/oauth/flow-paths.d.ts +16 -0
- package/dist/tools/oauth/flow-paths.js +38 -0
- package/dist/tools/oauth/flow.d.ts +30 -30
- package/dist/tools/oauth/flow.js +109 -81
- package/dist/tools/oauth/oauth.controller.d.ts +3 -3
- package/dist/tools/oauth/oauth.controller.js +12 -9
- package/dist/tools/oauth/tokens.d.ts +45 -10
- package/dist/tools/oauth/tokens.js +70 -24
- package/dist/tools/org-tool-switches.d.ts +88 -0
- package/dist/tools/org-tool-switches.js +201 -0
- package/dist/tools/outputs.d.ts +13 -4
- package/dist/tools/outputs.js +48 -4
- package/dist/tools/pipeline.d.ts +108 -80
- package/dist/tools/pipeline.js +396 -409
- package/dist/tools/policy.d.ts +4 -17
- package/dist/tools/policy.js +49 -35
- package/dist/tools/questions.controller.d.ts +2 -2
- package/dist/tools/questions.controller.js +9 -6
- package/dist/tools/questions.service.d.ts +26 -25
- package/dist/tools/questions.service.js +201 -142
- package/dist/tools/registry.d.ts +215 -84
- package/dist/tools/registry.js +382 -264
- package/dist/tools/resources.d.ts +1 -1
- package/dist/tools/resources.js +1 -1
- package/dist/tools/specs.d.ts +5 -4
- package/dist/tools/specs.js +3 -1
- package/dist/tools/tools.module.d.ts +10 -0
- package/dist/tools/tools.module.js +113 -0
- package/dist/tools/validate.js +1 -1
- package/dist/ui/call-extension.d.ts +16 -0
- package/dist/ui/call-extension.js +38 -0
- package/dist/ui/embed-token.d.ts +9 -5
- package/dist/ui/embed-token.js +34 -6
- package/dist/ui/ui-stream.d.ts +22 -17
- package/dist/ui/ui-stream.js +72 -12
- package/dist/ui/ui.module.d.ts +7 -0
- package/dist/ui/ui.module.js +28 -0
- package/dist/ui/views.controller.d.ts +14 -12
- package/dist/ui/views.controller.js +32 -25
- package/dist/ui/views.d.ts +10 -57
- package/dist/ui/views.js +58 -46
- package/package.json +13 -3
- package/src/migrations/0063_tool_keys_claude_code.sql +220 -0
- package/src/migrations/0064_account_last_org.sql +7 -0
- package/src/migrations/0065_conversation_archive.sql +8 -0
- package/src/migrations/0066_extension_display_name.sql +4 -0
- package/src/migrations/0067_agent_profiles.sql +20 -0
- package/src/migrations/0068_agent_avatar_uploads.sql +7 -0
- package/src/migrations/0069_attachments_agent_idx.sql +3 -0
- package/src/migrations/0070_background_tasks.sql +37 -0
- package/src/migrations/0071_background_tasks_reserve.sql +6 -0
- package/src/migrations/0072_background_tasks_stopping.sql +5 -0
- package/src/migrations/0073_org_extension_point_defaults.sql +11 -0
- package/src/migrations/0074_subagent_tree.sql +50 -0
- package/src/migrations/0075_plugin_scope.sql +61 -0
- package/src/migrations/0076_skills_created_by_nullable.sql +6 -0
- package/src/migrations/0077_scope_requests.sql +52 -0
- package/src/migrations/0078_scope_grant_switches.sql +18 -0
- package/src/migrations/0079_scope_live_install_switches.sql +19 -0
- package/src/migrations/0080_rename_enterprise_admin.sql +18 -0
- package/src/migrations/0081_platform_components.sql +129 -0
- package/src/migrations/0082_platform_migration.sql +33 -0
- package/src/migrations/0083_plugin_scope_sync_failures.sql +15 -0
- package/src/migrations/0084_org_owned_items.sql +85 -0
- package/src/migrations/0085_agent_templates_phase4.sql +45 -0
- package/src/migrations/0086_org_layering.sql +30 -0
- package/src/migrations/0087_approval_summary.sql +6 -0
- package/src/migrations/0088_environments.sql +80 -0
- package/src/migrations/0089_model_kinds_usage.sql +38 -0
- package/src/migrations/0090_initial_passwords.sql +27 -0
- package/src/migrations/0091_account_time_zone.sql +4 -0
- package/src/migrations/0092_tools_default_on.sql +27 -0
- package/src/migrations/0093_connected_environments.sql +37 -0
- package/src/migrations/0094_environment_paths_idx.sql +7 -0
- package/src/migrations/0095_environment_files_resets.sql +5 -0
- package/src/migrations/0096_environment_demoted_default.sql +18 -0
- package/src/migrations/0097_extension_installs_enabled_at.sql +16 -0
- package/src/migrations/0098_single_provider_running_first.sql +41 -0
- package/src/migrations/0099_channel_connection_guide_ack.sql +6 -0
- package/src/migrations/0100_catalog_files.sql +13 -0
- package/src/migrations/0101_marketplace_icons.sql +32 -0
- package/src/migrations/0102_themes.sql +64 -0
- package/src/migrations/0103_ui_theme_setting.sql +7 -0
- package/dist/catalog/upgrade-targets.d.ts +0 -59
- package/dist/catalog/upgrade-targets.js +0 -90
- package/dist/conversations/sandbox-status.d.ts +0 -21
- package/dist/conversations/sandbox-status.js +0 -24
- package/dist/extensions/org-config.controller.d.ts +0 -24
- package/dist/extensions/org-config.controller.js +0 -59
- package/dist/http/gateway.module.js +0 -13
- package/dist/http/health.controller.d.ts +0 -26
- package/dist/http/health.controller.js +0 -79
- package/dist/http/tokens.d.ts +0 -131
- package/dist/http/tokens.js +0 -131
- package/dist/http/zod.pipe.d.ts +0 -8
- package/dist/http/zod.pipe.js +0 -14
- package/dist/identity/enterprise-admin.d.ts +0 -14
- package/dist/identity/enterprise-admin.js +0 -27
- package/dist/identity/rate-limit.d.ts +0 -55
- package/dist/identity/rate-limit.js +0 -121
- package/dist/sandbox/lifecycle.d.ts +0 -21
- package/dist/sandbox/lifecycle.js +0 -68
- package/dist/tools/builtins/sandbox/names.d.ts +0 -7
- package/dist/tools/builtins/sandbox/names.js +0 -20
|
@@ -1,29 +1,60 @@
|
|
|
1
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
};
|
|
7
|
+
var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
8
|
+
return function (target, key) { decorator(target, key, paramIndex); }
|
|
9
|
+
};
|
|
1
10
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import { isSandboxGoneError, } from "@capybara-hq/plugin-sdk";
|
|
11
|
+
import { isFileSystemGoneError, isSandboxGoneError, rootsOf, } from "@capybara-hq/plugin-sdk";
|
|
12
|
+
import { Inject, Injectable } from "@nestjs/common";
|
|
3
13
|
import { sql, } from "kysely";
|
|
14
|
+
import { BackgroundTaskStore } from "../background/store.js";
|
|
4
15
|
import { DEFAULT_MAX_COMMAND_SECONDS, DEFAULT_MAX_RUN_SECONDS, } from "../config/schema.js";
|
|
5
16
|
import { ApiError } from "../errors.js";
|
|
6
17
|
import { readSandboxLimitOverride } from "../identity/settings.js";
|
|
18
|
+
import { KernelConfigService } from "../infra/config/kernel-config.service.js";
|
|
19
|
+
import { KernelDb } from "../infra/database/kernel-db.js";
|
|
20
|
+
import { KERNEL_LOGGER, REDIS } from "../infra/tokens.js";
|
|
7
21
|
import { effectiveSandboxEgress, effectiveSandboxLimits, readOrgSettings, } from "../orgs/settings.service.js";
|
|
8
|
-
import { canonicalJson } from "../tools/
|
|
9
|
-
import {
|
|
22
|
+
import { canonicalJson } from "../tools/canonical-json.js";
|
|
23
|
+
import { SandboxLeases, SandboxLockLostError, sandboxBusy, } from "./leases.js";
|
|
24
|
+
import { DEVICE_PROVIDER, providerForEnvironment, SandboxProviders, } from "./providers.js";
|
|
10
25
|
// Moved to leases.ts (phase C); re-exported for existing importers
|
|
11
|
-
export {
|
|
26
|
+
export { SANDBOX_LOCK_PREFIX, SANDBOX_LOCK_TTL_MS, SandboxLockLostError, sandboxBusy, } from "./leases.js";
|
|
12
27
|
/** Margin (ms) of a holder record's TTL beyond the longest command: by then the in-sandbox limit has ended its commands. */
|
|
13
28
|
const HOLDER_TTL_MARGIN_MS = 10 * 60_000;
|
|
14
29
|
/** How long the kernel's own read-modify-write of one file waits for another holder's (ms, spec §10.4). */
|
|
15
30
|
const PATH_LEASE_WAIT_MS = 30_000;
|
|
16
|
-
/**
|
|
31
|
+
/** Time limit of one provider call in maintenance (suspend, destroy). */
|
|
17
32
|
const SWEEP_OP_TIMEOUT_MS = 120_000;
|
|
18
33
|
/** How long the suspension sweep holds the intent on a sandbox past `maxRunningMinutes` (ms). */
|
|
19
34
|
const SWEEP_INTENT_WAIT_MS = 1000;
|
|
20
|
-
/**
|
|
35
|
+
/** Rows one sweep handles at most, and the provider calls it runs at once. */
|
|
21
36
|
const SWEEP_BATCH = 200;
|
|
22
37
|
const SWEEP_CONCURRENCY = 4;
|
|
23
|
-
/**
|
|
24
|
-
const
|
|
25
|
-
/**
|
|
38
|
+
/** How long `destroyEnvironment` / `resetFileSystem` wait for the exclusive lease (running commands). */
|
|
39
|
+
const EXCLUSIVE_WAIT_MS = 10 * 60_000;
|
|
40
|
+
/** The time limit of the exec that kills lost background tasks after a resume (ms). */
|
|
41
|
+
const PENDING_KILL_TIMEOUT_MS = 30_000;
|
|
42
|
+
/** A background task's session id as `echo $!` printed it; anything else is never put in a command. */
|
|
43
|
+
const SID_RE = /^[1-9][0-9]{0,9}$/;
|
|
44
|
+
/** The signal of one provider call in maintenance: aborted on timeout or when the lease is lost. */
|
|
26
45
|
const opSignal = (lease) => AbortSignal.any([lease.signal, AbortSignal.timeout(SWEEP_OP_TIMEOUT_MS)]);
|
|
46
|
+
/** The provider-side name of an environment's file system and instance. */
|
|
47
|
+
export const environmentResourceName = (environmentId) => `cap-env-${environmentId}`;
|
|
48
|
+
/** The environment is missing, deleted, or not in the caller's org (404; ids cannot be probed). */
|
|
49
|
+
export const environmentNotFound = () => new ApiError("environment.not_found", "This environment does not exist or was deleted.");
|
|
50
|
+
/**
|
|
51
|
+
* The environment's file system is gone at the provider (409); never replaced silently. On a
|
|
52
|
+
* connected machine (`device`, details `[{ source: "device" }]`) the directory is the machine
|
|
53
|
+
* owner's: `capybara-device logout`, then `login` into the environment, recreates it (ruling R24).
|
|
54
|
+
*/
|
|
55
|
+
export const environmentFilesLost = (device = false) => device
|
|
56
|
+
? new ApiError("environment.files_lost", "The environment's directory is gone from its connected machine; run capybara-device logout, then capybara-device login into this environment, on that machine to recreate it.", [{ source: "device" }])
|
|
57
|
+
: new ApiError("environment.files_lost", "The environment's files are gone; reset its files from the environments page to start with an empty workspace.");
|
|
27
58
|
/**
|
|
28
59
|
* The row still records the instance `gone` (same handle and same `created_at`). Handles compare
|
|
29
60
|
* canonically: a jsonb round trip reorders a multi-key handle's keys (final review m9).
|
|
@@ -33,42 +64,31 @@ export const isInstance = (row, gone) => gone !== null &&
|
|
|
33
64
|
canonicalJson(row.handle) === canonicalJson(gone.handle) &&
|
|
34
65
|
row.created_at.getTime() === gone.createdAt;
|
|
35
66
|
/**
|
|
36
|
-
*
|
|
37
|
-
*
|
|
67
|
+
* `impl` with every `exec` labelled with the holder's tag (and `actor` when given); `before` runs
|
|
68
|
+
* first (it refreshes the holder's record, so a callback that runs several long commands in a row
|
|
69
|
+
* stays recorded).
|
|
38
70
|
*/
|
|
39
|
-
|
|
40
|
-
return impl.concurrentExec === true && typeof impl.killTagged === "function" ? max : 1;
|
|
41
|
-
}
|
|
42
|
-
/**
|
|
43
|
-
* `impl` with every `exec` labelled with the holder's tag; `before` runs first (it refreshes the
|
|
44
|
-
* holder's record, so a callback that runs several long commands in a row stays recorded).
|
|
45
|
-
*/
|
|
46
|
-
function tagged(impl, tag, before) {
|
|
71
|
+
function tagged(impl, tag, actor, before) {
|
|
47
72
|
const wrapped = Object.create(impl);
|
|
48
73
|
Object.defineProperty(wrapped, "exec", {
|
|
49
74
|
value: async (h, args) => {
|
|
50
75
|
await before();
|
|
51
|
-
return impl.exec(h, { ...args, tag });
|
|
76
|
+
return impl.exec(h, { ...args, tag, ...(actor !== undefined && { actor }) });
|
|
52
77
|
},
|
|
53
78
|
});
|
|
54
79
|
return wrapped;
|
|
55
80
|
}
|
|
56
81
|
/**
|
|
57
|
-
*
|
|
58
|
-
* - `runtime.maxRunSeconds`
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
82
|
+
* Sandbox config problems to tell operators at startup (English log lines):
|
|
83
|
+
* - `runtime.maxRunSeconds` below `sandbox.limits.maxCommandSeconds`: long commands are cut by the
|
|
84
|
+
* run time limit (Ruling M20);
|
|
85
|
+
* - `sandbox.limits.egress` is open but the current provider plugin has no `allowOpenEgress`: every
|
|
86
|
+
* instance creation is refused.
|
|
87
|
+
* With both time limits at their defaults (900 < 7200, a known trade-off) it is only info; warn
|
|
88
|
+
* when an operator changed one and they still disagree.
|
|
63
89
|
*/
|
|
64
|
-
export function sandboxConfigWarnings(cfg, providerPlugin
|
|
90
|
+
export function sandboxConfigWarnings(cfg, providerPlugin) {
|
|
65
91
|
const out = [];
|
|
66
|
-
if (impl && (impl.concurrentExec === true) !== (typeof impl.killTagged === "function")) {
|
|
67
|
-
out.push({
|
|
68
|
-
level: "warn",
|
|
69
|
-
message: `the sandbox plugin ${providerPlugin ?? ""} declares only one of concurrentExec and killTagged; concurrent commands need both, so its sandboxes run one command at a time`,
|
|
70
|
-
});
|
|
71
|
-
}
|
|
72
92
|
if (cfg.sandbox.limits.egress === "open" && providerPlugin !== undefined) {
|
|
73
93
|
const pc = cfg.extensionConfig[providerPlugin];
|
|
74
94
|
if (pc?.allowOpenEgress !== true) {
|
|
@@ -89,8 +109,13 @@ export function sandboxConfigWarnings(cfg, providerPlugin, impl) {
|
|
|
89
109
|
}
|
|
90
110
|
return out;
|
|
91
111
|
}
|
|
92
|
-
/**
|
|
112
|
+
/** No sandbox provider (503). */
|
|
93
113
|
export const sandboxUnavailable = () => new ApiError("sandbox.unavailable", "Sandbox is not available in this deployment; install a sandbox plugin");
|
|
114
|
+
/**
|
|
115
|
+
* The environment's files are kept by a provider that is not the current one (503); its files are
|
|
116
|
+
* not migrated, so it stays unusable until that provider is back (spec §4.2 item 13).
|
|
117
|
+
*/
|
|
118
|
+
const otherProvider = (provider) => new ApiError("environment.unavailable", `This environment's files are kept by the sandbox provider ${provider}, which is not the current provider; it can be used again once ${provider} is the deployment's sandbox provider`);
|
|
94
119
|
const abortReason = (signal) => signal.reason ?? new DOMException("This operation was aborted", "AbortError");
|
|
95
120
|
function sleep(ms, signal) {
|
|
96
121
|
return new Promise((resolve, reject) => {
|
|
@@ -109,7 +134,7 @@ function sleep(ms, signal) {
|
|
|
109
134
|
signal.addEventListener("abort", onAbort, { once: true });
|
|
110
135
|
});
|
|
111
136
|
}
|
|
112
|
-
/**
|
|
137
|
+
/** Runs `fn` on `items`, `limit` at a time. */
|
|
113
138
|
async function inPool(items, limit, fn) {
|
|
114
139
|
let next = 0;
|
|
115
140
|
const worker = async () => {
|
|
@@ -121,77 +146,156 @@ async function inPool(items, limit, fn) {
|
|
|
121
146
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
|
122
147
|
}
|
|
123
148
|
/**
|
|
124
|
-
*
|
|
149
|
+
* The kernel's sandbox service over the `sandbox@2` provider, keyed by **environment** (spec
|
|
150
|
+
* 2026-11-07-environments-design §4.2, §5.1): one file system kept until the environment is
|
|
151
|
+
* deleted, plus one instance attached to it that comes and goes; all conversations bound to the
|
|
152
|
+
* environment (and their sub-agents) share both.
|
|
125
153
|
*
|
|
126
|
-
* -
|
|
127
|
-
*
|
|
128
|
-
*
|
|
129
|
-
*
|
|
130
|
-
*
|
|
131
|
-
*
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
*
|
|
136
|
-
*
|
|
137
|
-
*
|
|
138
|
-
*
|
|
139
|
-
*
|
|
140
|
-
*
|
|
141
|
-
*
|
|
142
|
-
*
|
|
143
|
-
*
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
150
|
-
* `
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
*
|
|
154
|
+
* - **Lazy creation**: the first {@link withSandbox} creates the file system
|
|
155
|
+
* (`createFileSystem`, name `cap-env-<environmentId>`) and then the instance on it; state lives
|
|
156
|
+
* in `environment_instances`. A file system gone at the provider is never replaced silently:
|
|
157
|
+
* `fs_status` becomes `lost` and uses fail with `environment.files_lost` until
|
|
158
|
+
* {@link resetFileSystem}.
|
|
159
|
+
* - **Leases** (phase C, spec §10, {@link SandboxLeases}), keyed by environment id: using an
|
|
160
|
+
* instance recorded `running` with the current provider takes a **shared** lease, so up to
|
|
161
|
+
* `sandbox.maxConcurrentCommands` holders (all sessions of the environment) run at once. Every
|
|
162
|
+
* lifecycle change (create, resume, rebuild, suspend, destroy, reset) takes the **exclusive**
|
|
163
|
+
* lease, alone; a caller that needs one on the use path sets the intent so new shared leases wait
|
|
164
|
+
* for it, and downgrades to a shared lease to run its own command once the instance runs.
|
|
165
|
+
* Waiting is capped at `sandbox.lockWaitSeconds` (`sandbox.busy`); two replicas using it for the
|
|
166
|
+
* first time at once still build only one.
|
|
167
|
+
* - **Fencing**: exclusive holders read `generation` and compare-and-increment it with every state
|
|
168
|
+
* write ({@link SandboxLockLostError}); every acquisition (shared ones too) bumps it first, so a
|
|
169
|
+
* stale exclusive holder's later writes fail. Shared holders write nothing else but
|
|
170
|
+
* `last_used_at` (monotonic). A lost lease aborts the signal passed to `fn` and to maintenance
|
|
171
|
+
* provider calls.
|
|
172
|
+
* - **Suspension and destruction**: the sweeps (`SandboxSweepJobs`, crons) call
|
|
173
|
+
* {@link suspendIdle} / {@link destroyInactive}; they try the exclusive lease once and skip an
|
|
174
|
+
* environment with any holder; one past `maxRunningMinutes` gets the intent for a short window
|
|
175
|
+
* (about a second) so the suspension can get in when its commands end. Inactivity destroys only
|
|
176
|
+
* the instance (files kept); a deleted environment or org loses its files too.
|
|
177
|
+
* - **Transparent recovery**: a suspended instance is resumed on next use; when `resume` returns
|
|
178
|
+
* `gone`, the instance was destroyed, or `fn` throws `SandboxGoneError` (retried once), a new
|
|
179
|
+
* instance is created on the kept files and `restarted` is true. Several holders hitting `gone`
|
|
180
|
+
* together rebuild once: the others find the new instance.
|
|
181
|
+
* - **Takeover** (spec §10.5; environments spec §4.2): every holder records itself in
|
|
182
|
+
* `capk:sandbox-holders:<env>` (again before each `exec`) and its commands carry its lease token
|
|
183
|
+
* as `tag`. An acquirer that finds a record without a live lease (a crashed holder) kills only
|
|
184
|
+
* that holder's commands (`killTagged`): other conversations of the environment may be running
|
|
185
|
+
* commands. `killAll` runs only under the exclusive lease, before the downgrade, on an instance
|
|
186
|
+
* this holder just resumed from `suspended` (nobody else can hold it; a provider that keeps
|
|
187
|
+
* memory across a pause brings old commands back); it also covers crashed holders' records,
|
|
188
|
+
* which it claims. A freshly created instance has nothing to kill. Failures are only warned (the
|
|
189
|
+
* in-sandbox time limit is the backstop).
|
|
155
190
|
* - **Instance identity**: a holder that hits `gone` names the instance by its handle and the
|
|
156
191
|
* row's `created_at` (set by the fenced write after each `create`); the handle alone is the same
|
|
157
192
|
* after a rebuild on providers whose handle is the instance name (sandbox-docker).
|
|
158
193
|
*
|
|
159
|
-
*
|
|
160
|
-
*
|
|
194
|
+
* - **Provider per environment** (environments spec §6): each call resolves the provider from
|
|
195
|
+
* the environment's `environments.provider` ({@link providerForEnvironment}): a `device`
|
|
196
|
+
* environment runs on the kernel-internal `@device` provider (never the cloud one), a cloud
|
|
197
|
+
* environment on the loaded `sandbox@2` extension. A `@device` instance is never suspended or
|
|
198
|
+
* destroyed for inactivity; a deleted `@device` environment (or one of a deleted org) is
|
|
199
|
+
* destroyed by the sweep like any other.
|
|
200
|
+
*
|
|
201
|
+
* Files kept by another provider than the current one make the environment unusable
|
|
202
|
+
* (`environment.unavailable`) until that provider is back; deleting it still works while that
|
|
203
|
+
* provider is loaded ({@link SandboxProviders.forName}). Without the environment's provider
|
|
204
|
+
* {@link availableFor} is false and uses throw `sandbox.unavailable` (503). No in-process state:
|
|
205
|
+
* replicas share it through PG and Redis.
|
|
161
206
|
*/
|
|
162
|
-
|
|
163
|
-
|
|
207
|
+
let SandboxService = class SandboxService {
|
|
208
|
+
db;
|
|
209
|
+
redis;
|
|
164
210
|
leases;
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
211
|
+
providers;
|
|
212
|
+
config;
|
|
213
|
+
log;
|
|
214
|
+
background;
|
|
215
|
+
constructor(db,
|
|
216
|
+
/** The record of the instance each holder last used (`capk:sandbox-seen:…`). */
|
|
217
|
+
redis, leases,
|
|
218
|
+
/** The providers (read at every call: providers are ready after the extension load). */
|
|
219
|
+
providers, config, log,
|
|
220
|
+
/** Background tasks lost while their process may live on (killed after a resume). */
|
|
221
|
+
background) {
|
|
222
|
+
this.db = db;
|
|
223
|
+
this.redis = redis;
|
|
224
|
+
this.leases = leases;
|
|
225
|
+
this.providers = providers;
|
|
226
|
+
this.config = config;
|
|
227
|
+
this.log = log;
|
|
228
|
+
this.background = background;
|
|
229
|
+
}
|
|
230
|
+
/** Poll interval while waiting for a lease (the leases' own). */
|
|
231
|
+
get pollMs() {
|
|
232
|
+
return this.leases.pollMs;
|
|
177
233
|
}
|
|
178
|
-
/**
|
|
179
|
-
|
|
180
|
-
|
|
234
|
+
/**
|
|
235
|
+
* Whether there is a cloud sandbox provider (a loaded `sandbox@2` extension). For gates that have
|
|
236
|
+
* no environment yet (tool listing, skill files); a call on an environment asks
|
|
237
|
+
* {@link availableFor}.
|
|
238
|
+
*/
|
|
239
|
+
get cloudAvailable() {
|
|
240
|
+
return this.providers.cloud() !== undefined;
|
|
181
241
|
}
|
|
182
|
-
/**
|
|
183
|
-
|
|
184
|
-
return
|
|
242
|
+
/** Whether the environment's provider is there (`@device` registered, or a cloud provider). */
|
|
243
|
+
async availableFor(environmentId) {
|
|
244
|
+
return (await this.providerOf(environmentId)) !== undefined;
|
|
185
245
|
}
|
|
186
246
|
/**
|
|
187
|
-
*
|
|
188
|
-
*
|
|
247
|
+
* The provider the environment works with, read from its row ({@link providerForEnvironment});
|
|
248
|
+
* a missing row (or one of another org, with `orgId`) reads as a cloud environment, whose use
|
|
249
|
+
* then fails as not found.
|
|
189
250
|
*/
|
|
190
|
-
async
|
|
191
|
-
const row = await this.
|
|
192
|
-
.selectFrom("
|
|
251
|
+
async providerOf(environmentId, orgId) {
|
|
252
|
+
const row = await this.db
|
|
253
|
+
.selectFrom("environments")
|
|
254
|
+
.select("provider")
|
|
255
|
+
.where("id", "=", environmentId)
|
|
256
|
+
.$if(orgId !== undefined, (q) => q.where("org_id", "=", orgId))
|
|
257
|
+
.executeTakeFirst();
|
|
258
|
+
return providerForEnvironment(this.providers, row?.provider ?? null);
|
|
259
|
+
}
|
|
260
|
+
/** {@link providerOf}, or `sandbox.unavailable` (503) without it. */
|
|
261
|
+
async require(environmentId, orgId) {
|
|
262
|
+
const p = await this.providerOf(environmentId, orgId);
|
|
263
|
+
if (!p)
|
|
264
|
+
throw sandboxUnavailable();
|
|
265
|
+
return p;
|
|
266
|
+
}
|
|
267
|
+
/** Concurrent holders of one environment (`sandbox.maxConcurrentCommands`, read at every use). */
|
|
268
|
+
get cap() {
|
|
269
|
+
return this.config.get().sandbox.maxConcurrentCommands;
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* The org's effective sandbox limits (spec §8.4): the deployment's `sandbox.limits`, egress
|
|
273
|
+
* narrowed by the org setting `sandboxEgress` (Ruling M19), cpu / memory per the super admin's
|
|
274
|
+
* org override (C19 / Ruling M16). Instances are created with them; phase 1 environments use
|
|
275
|
+
* them unchanged.
|
|
276
|
+
*/
|
|
277
|
+
async orgLimits(orgId) {
|
|
278
|
+
const cfg = this.config.get().sandbox;
|
|
279
|
+
const egress = effectiveSandboxEgress(cfg.limits, (await readOrgSettings(this.db.db, orgId)).sandboxEgress);
|
|
280
|
+
const sized = effectiveSandboxLimits({ ...cfg.limits, ...egress }, await readSandboxLimitOverride(this.db.db, orgId));
|
|
281
|
+
return {
|
|
282
|
+
cpu: sized.cpu,
|
|
283
|
+
memoryMb: sized.memoryMb,
|
|
284
|
+
diskMb: sized.diskMb,
|
|
285
|
+
egress: sized.egress,
|
|
286
|
+
allowHosts: sized.allowHosts,
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* The environment's instance state as shown to people (read only, no lease): `running` /
|
|
291
|
+
* `suspended` and the last use. No instance, a destroyed one, or one stuck in `creating` (being
|
|
292
|
+
* created, or a failed create the next use retries) is null.
|
|
293
|
+
*/
|
|
294
|
+
async status(orgId, environmentId) {
|
|
295
|
+
const row = await this.db
|
|
296
|
+
.selectFrom("environment_instances")
|
|
193
297
|
.select(["status", "last_used_at"])
|
|
194
|
-
.where("
|
|
298
|
+
.where("environment_id", "=", environmentId)
|
|
195
299
|
.where("org_id", "=", orgId)
|
|
196
300
|
.executeTakeFirst();
|
|
197
301
|
if (!row || (row.status !== "running" && row.status !== "suspended"))
|
|
@@ -199,91 +303,224 @@ export class SandboxService {
|
|
|
199
303
|
return { status: row.status, lastUsedAt: row.last_used_at };
|
|
200
304
|
}
|
|
201
305
|
/**
|
|
202
|
-
*
|
|
203
|
-
*
|
|
204
|
-
*
|
|
306
|
+
* The instance the environment records now (its `created_at`), while it has one that runs or is
|
|
307
|
+
* suspended; null otherwise. Read without a lease: a caller inside `withSandbox` reads the
|
|
308
|
+
* instance its own lease keeps.
|
|
309
|
+
*/
|
|
310
|
+
async currentInstance(environmentId) {
|
|
311
|
+
const row = await this.db
|
|
312
|
+
.selectFrom("environment_instances")
|
|
313
|
+
.select(["status", "handle", "created_at"])
|
|
314
|
+
.where("environment_id", "=", environmentId)
|
|
315
|
+
.executeTakeFirst();
|
|
316
|
+
if (!row || row.handle === null)
|
|
317
|
+
return null;
|
|
318
|
+
if (row.status !== "running" && row.status !== "suspended")
|
|
319
|
+
return null;
|
|
320
|
+
return { createdAt: row.created_at };
|
|
321
|
+
}
|
|
322
|
+
/**
|
|
323
|
+
* Runs `fn` on the environment's instance only while it runs the instance created at
|
|
324
|
+
* `instance.createdAt` (background tasks, spec 2026-11-05-background-bash-design §4.3). It never
|
|
325
|
+
* creates, resumes or rebuilds one: an instance that is missing, suspended, being created or
|
|
326
|
+
* destroyed (or recorded with another provider) is `not_running`, one that records another
|
|
327
|
+
* instance is `other_instance`. Otherwise `fn` runs under a shared lease, as a `withSandbox`
|
|
328
|
+
* holder does (its tag on every exec, crashed holders' commands killed), and `last_used_at`
|
|
329
|
+
* moves forward when it ends. Waits for the lease up to `sandbox.lockWaitSeconds`
|
|
330
|
+
* (`sandbox.busy`); `signal` (optional) ends the wait with its abort reason. The signal passed
|
|
331
|
+
* to `fn` aborts with `signal` or when the lease is lost. Errors of `fn` (`SandboxGoneError`
|
|
332
|
+
* too) are thrown as they are.
|
|
333
|
+
*/
|
|
334
|
+
async ifRunning(environmentId, instance, fn, signal = new AbortController().signal) {
|
|
335
|
+
const p = await this.require(environmentId);
|
|
336
|
+
const env = environmentId;
|
|
337
|
+
const deadline = Date.now() + this.config.get().sandbox.lockWaitSeconds * 1000;
|
|
338
|
+
const token = randomUUID();
|
|
339
|
+
for (;;) {
|
|
340
|
+
if (signal.aborted)
|
|
341
|
+
throw abortReason(signal);
|
|
342
|
+
const row = await this.db
|
|
343
|
+
.selectFrom("environment_instances")
|
|
344
|
+
.selectAll()
|
|
345
|
+
.where("environment_id", "=", env)
|
|
346
|
+
.executeTakeFirst();
|
|
347
|
+
if (!row)
|
|
348
|
+
return { kind: "not_running", status: "gone" };
|
|
349
|
+
if (row.created_at.getTime() !== instance.createdAt.getTime()) {
|
|
350
|
+
return { kind: "other_instance" };
|
|
351
|
+
}
|
|
352
|
+
if (!this.usable(row, p, null))
|
|
353
|
+
return { kind: "not_running", status: row.status };
|
|
354
|
+
// the instance this check saw: its handle plus `created_at` (never `generation`, which
|
|
355
|
+
// every lease bumps)
|
|
356
|
+
const want = { handle: row.handle, createdAt: row.created_at.getTime() };
|
|
357
|
+
const lease = await this.leases.tryShared(env, this.cap, token);
|
|
358
|
+
if (lease) {
|
|
359
|
+
const e = await this.asShared({ orgId: row.org_id, environmentId: env }, p, lease, (r) => this.usable(r, p, null) && isInstance(r, want), signal);
|
|
360
|
+
if (e) {
|
|
361
|
+
try {
|
|
362
|
+
const impl = tagged(p.impl, lease.token, undefined, () => this.register(env, lease.token));
|
|
363
|
+
const sig = AbortSignal.any([signal, lease.signal]);
|
|
364
|
+
try {
|
|
365
|
+
return { kind: "ran", value: await fn(impl, e.handle, sig) };
|
|
366
|
+
}
|
|
367
|
+
catch (err) {
|
|
368
|
+
if (isFileSystemGoneError(err))
|
|
369
|
+
throw await this.filesGone(env, p, e);
|
|
370
|
+
throw err;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
finally {
|
|
374
|
+
await this.leave(env, e);
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
// the row changed between the check and the lease: classify it again
|
|
378
|
+
continue;
|
|
379
|
+
}
|
|
380
|
+
if (Date.now() + this.pollMs > deadline)
|
|
381
|
+
throw sandboxBusy();
|
|
382
|
+
await sleep(this.pollMs + Math.floor(Math.random() * this.pollMs), signal);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
/**
|
|
386
|
+
* A short lease on one path of the environment around the kernel's own read-modify-write of
|
|
387
|
+
* that file (spec §10.4); waits up to 30 s, then throws `SandboxPathBusyError`. Its `signal`
|
|
388
|
+
* aborts when a renewal finds it lost.
|
|
205
389
|
*/
|
|
206
|
-
pathLease(
|
|
207
|
-
return this.leases.pathLease(
|
|
390
|
+
pathLease(environmentId, path, signal) {
|
|
391
|
+
return this.leases.pathLease(environmentId, path, signal, PATH_LEASE_WAIT_MS);
|
|
208
392
|
}
|
|
209
393
|
/**
|
|
210
|
-
*
|
|
211
|
-
*
|
|
212
|
-
* `last_used_at` → release.
|
|
213
|
-
* -
|
|
214
|
-
*
|
|
215
|
-
* - The
|
|
216
|
-
*
|
|
217
|
-
*
|
|
218
|
-
* - `
|
|
219
|
-
*
|
|
220
|
-
*
|
|
221
|
-
*
|
|
394
|
+
* Runs `fn` on the environment's instance: take a lease → make sure the instance runs (file
|
|
395
|
+
* system, create / resume / rebuild, under the exclusive lease) → `fn` under a shared lease →
|
|
396
|
+
* update `last_used_at` → release.
|
|
397
|
+
* - Waiting longer than `sandbox.lockWaitSeconds` throws `sandbox.busy` (409); `signal` aborts
|
|
398
|
+
* the wait and the creation / resume.
|
|
399
|
+
* - The fourth argument of `fn` combines `signal` with the lease's: use it for commands, so they
|
|
400
|
+
* stop when the lease is lost.
|
|
401
|
+
* - The provider passed to `fn` labels every `exec` with this holder's tag and `ctx.actor`.
|
|
402
|
+
* - When `fn` throws `SandboxGoneError` (the instance is gone) the instance is rebuilt on the
|
|
403
|
+
* kept files (or the one another holder rebuilt is taken) and `fn` runs once more with
|
|
404
|
+
* `restarted: true`; a second `gone` is thrown. Other errors are thrown as they are.
|
|
405
|
+
* - Lost files throw `environment.files_lost` (409), also when `fn` throws
|
|
406
|
+
* `FileSystemGoneError` (the files are then recorded lost, ruling R24).
|
|
407
|
+
* - When `ctx.holderConversationId` last used another instance (another holder rebuilt it
|
|
408
|
+
* meanwhile), `info.restarted` is true, so it learns its processes are gone (final review m5).
|
|
222
409
|
*/
|
|
223
410
|
async withSandbox(ctx, fn, signal) {
|
|
224
|
-
const p = this.require();
|
|
225
|
-
const
|
|
226
|
-
|
|
411
|
+
const p = await this.require(ctx.environmentId, ctx.orgId);
|
|
412
|
+
const env = ctx.environmentId;
|
|
413
|
+
const holder = ctx.holderConversationId;
|
|
414
|
+
let e = await this.seenBy(env, holder, await this.enter(ctx, p, signal, null));
|
|
227
415
|
try {
|
|
228
416
|
try {
|
|
229
|
-
return await this.runAs(e, p, fn, signal);
|
|
417
|
+
return await this.runAs(e, ctx, p, fn, signal);
|
|
230
418
|
}
|
|
231
419
|
catch (err) {
|
|
420
|
+
if (isFileSystemGoneError(err))
|
|
421
|
+
throw await this.filesGone(env, p, e);
|
|
232
422
|
if (!isSandboxGoneError(err))
|
|
233
423
|
throw err;
|
|
234
|
-
this.
|
|
235
|
-
conversationId: ctx.conversationId,
|
|
236
|
-
});
|
|
424
|
+
this.log.info("sandbox instance is gone during use; recreating", { environmentId: env });
|
|
237
425
|
const gone = { handle: e.handle, createdAt: e.createdAt };
|
|
238
426
|
const prev = e;
|
|
239
|
-
await this.leave(
|
|
240
|
-
e = await this.seenBy(
|
|
241
|
-
|
|
427
|
+
await this.leave(env, prev);
|
|
428
|
+
e = await this.seenBy(env, holder, await this.enter(ctx, p, signal, gone));
|
|
429
|
+
// a reset seen on the first try still has to be told (it covers the restart)
|
|
430
|
+
const info = prev.info.filesReset
|
|
431
|
+
? { fresh: true, restarted: false, filesReset: true }
|
|
432
|
+
: { fresh: true, restarted: true };
|
|
433
|
+
try {
|
|
434
|
+
return await this.runAs({ ...e, info }, ctx, p, fn, signal);
|
|
435
|
+
}
|
|
436
|
+
catch (err2) {
|
|
437
|
+
if (isFileSystemGoneError(err2))
|
|
438
|
+
throw await this.filesGone(env, p, e);
|
|
439
|
+
throw err2;
|
|
440
|
+
}
|
|
242
441
|
}
|
|
243
442
|
}
|
|
244
443
|
finally {
|
|
245
|
-
await this.leave(
|
|
444
|
+
await this.leave(env, e);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
/**
|
|
448
|
+
* Ruling R24: an operation on the held instance `e` found its file system gone. Records
|
|
449
|
+
* `fs_status = 'lost'` while the row still records that instance on its file system (the
|
|
450
|
+
* instance handle stays, so a reset still destroys it), moving the fence on, and returns
|
|
451
|
+
* `environment.files_lost` to throw. Uses then fail until the files are reset (a connected
|
|
452
|
+
* machine: until a new login, which resets the row). A failed write only logs.
|
|
453
|
+
*/
|
|
454
|
+
async filesGone(env, p, e) {
|
|
455
|
+
this.log.warn("environment file system is gone at the provider; its files are lost", {
|
|
456
|
+
environmentId: env,
|
|
457
|
+
provider: p.plugin,
|
|
458
|
+
});
|
|
459
|
+
try {
|
|
460
|
+
await this.db
|
|
461
|
+
.updateTable("environment_instances")
|
|
462
|
+
.set((eb) => ({ fs_status: "lost", generation: eb("generation", "+", 1) }))
|
|
463
|
+
.where("environment_id", "=", env)
|
|
464
|
+
.where("fs_status", "=", "ready")
|
|
465
|
+
// `createdAt` is the row's value read back in milliseconds
|
|
466
|
+
.where(sql `date_trunc('milliseconds', created_at)`, "=", new Date(e.createdAt))
|
|
467
|
+
.execute();
|
|
468
|
+
}
|
|
469
|
+
catch (err) {
|
|
470
|
+
this.log.warn("cannot record lost environment files", { environmentId: env, err });
|
|
246
471
|
}
|
|
472
|
+
return environmentFilesLost(p.plugin === DEVICE_PROVIDER);
|
|
247
473
|
}
|
|
248
474
|
/**
|
|
249
|
-
* Records that `holder` uses the instance of `e` (`capk:sandbox-seen:<
|
|
250
|
-
*
|
|
251
|
-
*
|
|
475
|
+
* Records that `holder` uses the instance of `e` (`capk:sandbox-seen:<env>:<holder>` =
|
|
476
|
+
* `<created_at>:<fs_resets>`, with a TTL of the inactivity destruction plus a day). When the
|
|
477
|
+
* holder last used another instance under the same reset count `e` is marked restarted (files
|
|
478
|
+
* kept); under another count it is marked `filesReset` instead. The count, not the provider's
|
|
479
|
+
* file system id, tells them apart: a provider may give the new file system the old id
|
|
480
|
+
* (sandbox-docker's volumes are named after the environment). Best effort: a Redis error only
|
|
481
|
+
* logs.
|
|
252
482
|
*/
|
|
253
|
-
async seenBy(
|
|
483
|
+
async seenBy(environmentId, holder, e) {
|
|
254
484
|
try {
|
|
255
|
-
const ttl = (this.
|
|
256
|
-
const
|
|
257
|
-
|
|
258
|
-
|
|
485
|
+
const ttl = (this.config.get().sandbox.limits.destroyAfterInactiveDays + 1) * 86_400;
|
|
486
|
+
const seen = `${e.createdAt}:${e.fsResets}`;
|
|
487
|
+
const prev = await this.redis.set(`capk:sandbox-seen:${environmentId}:${holder}`, seen, "EX", ttl, "GET");
|
|
488
|
+
if (prev !== null && prev !== seen) {
|
|
489
|
+
// `<createdAt>:<fsResets>`: another reset count means the files were reset since; another
|
|
490
|
+
// instance under the same count is a restart on kept files
|
|
491
|
+
const sameFiles = prev.slice(prev.indexOf(":") + 1) === String(e.fsResets);
|
|
492
|
+
if (!sameFiles) {
|
|
493
|
+
return { ...e, info: { fresh: e.info.fresh, restarted: false, filesReset: true } };
|
|
494
|
+
}
|
|
495
|
+
if (!e.info.restarted)
|
|
496
|
+
return { ...e, info: { fresh: e.info.fresh, restarted: true } };
|
|
259
497
|
}
|
|
260
498
|
}
|
|
261
499
|
catch (err) {
|
|
262
|
-
this.
|
|
263
|
-
|
|
500
|
+
this.log.warn("cannot record the sandbox instance a holder uses", {
|
|
501
|
+
environmentId,
|
|
264
502
|
holder,
|
|
265
503
|
err,
|
|
266
504
|
});
|
|
267
505
|
}
|
|
268
506
|
return e;
|
|
269
507
|
}
|
|
270
|
-
/** Runs `fn` as the holder `e`: its tag on every exec, its lease's signal. */
|
|
271
|
-
runAs(e, p, fn, signal) {
|
|
508
|
+
/** Runs `fn` as the holder `e`: its tag and the caller's actor on every exec, its lease's signal. */
|
|
509
|
+
runAs(e, ctx, p, fn, signal) {
|
|
272
510
|
const sig = AbortSignal.any([signal, e.lease.signal]);
|
|
273
|
-
const
|
|
274
|
-
const impl = tagged(p.impl, e.lease.token, () => this.register(
|
|
275
|
-
return fn(e.handle, impl, e.info, sig);
|
|
511
|
+
const env = ctx.environmentId;
|
|
512
|
+
const impl = tagged(p.impl, e.lease.token, ctx.actor, () => this.register(env, e.lease.token));
|
|
513
|
+
return fn(e.handle, impl, { ...e.info, roots: rootsOf(e.handle) }, sig);
|
|
276
514
|
}
|
|
277
515
|
/**
|
|
278
516
|
* Becomes a holder: loops until it has a shared lease on a running instance, or `sandbox.busy`.
|
|
279
|
-
*
|
|
280
|
-
*
|
|
281
|
-
*
|
|
517
|
+
* An instance recorded `running` (with the current provider, not the `gone` instance) is used
|
|
518
|
+
* under a shared lease; otherwise the exclusive lease (with intent) makes it run and is
|
|
519
|
+
* downgraded. One token serves both, so the waiter passes its own intent.
|
|
282
520
|
*/
|
|
283
521
|
async enter(ctx, p, signal, gone) {
|
|
284
|
-
const
|
|
285
|
-
const
|
|
286
|
-
const deadline = Date.now() + this.d.config().sandbox.lockWaitSeconds * 1000;
|
|
522
|
+
const env = ctx.environmentId;
|
|
523
|
+
const deadline = Date.now() + this.config.get().sandbox.lockWaitSeconds * 1000;
|
|
287
524
|
const token = randomUUID();
|
|
288
525
|
let intent = false;
|
|
289
526
|
try {
|
|
@@ -291,19 +528,19 @@ export class SandboxService {
|
|
|
291
528
|
if (signal.aborted)
|
|
292
529
|
throw abortReason(signal);
|
|
293
530
|
const row = await this.readRow(ctx);
|
|
294
|
-
if (this.usable(row, p, gone)
|
|
295
|
-
const lease = await this.leases.tryShared(
|
|
531
|
+
if (this.usable(row, p, gone)) {
|
|
532
|
+
const lease = await this.leases.tryShared(env, this.cap, token);
|
|
296
533
|
if (lease) {
|
|
297
534
|
if (intent)
|
|
298
|
-
await this.clearIntent(
|
|
535
|
+
await this.clearIntent(env, token);
|
|
299
536
|
intent = false;
|
|
300
|
-
const e = await this.asShared(ctx, p, lease, gone, signal);
|
|
537
|
+
const e = await this.asShared(ctx, p, lease, (r) => this.usable(r, p, gone), signal);
|
|
301
538
|
if (e)
|
|
302
539
|
return e;
|
|
303
540
|
}
|
|
304
541
|
}
|
|
305
542
|
else {
|
|
306
|
-
const lease = await this.leases.tryExclusive(
|
|
543
|
+
const lease = await this.leases.tryExclusive(env, { token, intent: true });
|
|
307
544
|
intent = lease === null;
|
|
308
545
|
if (lease)
|
|
309
546
|
return await this.asExclusive(ctx, p, lease, gone, signal);
|
|
@@ -315,181 +552,193 @@ export class SandboxService {
|
|
|
315
552
|
}
|
|
316
553
|
catch (err) {
|
|
317
554
|
if (intent)
|
|
318
|
-
await this.clearIntent(
|
|
555
|
+
await this.clearIntent(env, token);
|
|
319
556
|
throw err;
|
|
320
557
|
}
|
|
321
558
|
}
|
|
322
|
-
async clearIntent(
|
|
559
|
+
async clearIntent(env, token) {
|
|
323
560
|
await this.leases
|
|
324
|
-
.clearIntent(
|
|
325
|
-
.catch((err) => this.
|
|
561
|
+
.clearIntent(env, token)
|
|
562
|
+
.catch((err) => this.log.warn("cannot clear sandbox intent", { environmentId: env, err }));
|
|
326
563
|
}
|
|
327
564
|
async readRow(ctx) {
|
|
328
|
-
const row = await this.
|
|
329
|
-
.selectFrom("
|
|
565
|
+
const row = await this.db
|
|
566
|
+
.selectFrom("environment_instances")
|
|
330
567
|
.selectAll()
|
|
331
|
-
.where("
|
|
568
|
+
.where("environment_id", "=", ctx.environmentId)
|
|
332
569
|
.executeTakeFirst();
|
|
333
|
-
if (row && row.org_id !== ctx.orgId)
|
|
334
|
-
throw
|
|
335
|
-
}
|
|
570
|
+
if (row && row.org_id !== ctx.orgId)
|
|
571
|
+
throw this.foreign(ctx);
|
|
336
572
|
return row;
|
|
337
573
|
}
|
|
338
|
-
/**
|
|
574
|
+
/** The instance row belongs to another org: reported as not found, so ids cannot be probed. */
|
|
575
|
+
foreign(_ctx) {
|
|
576
|
+
return environmentNotFound();
|
|
577
|
+
}
|
|
578
|
+
/**
|
|
579
|
+
* Recorded running with the current provider on files not lost (lost ones go through the
|
|
580
|
+
* exclusive path, which refuses them), and not the instance found gone.
|
|
581
|
+
*/
|
|
339
582
|
usable(row, p, gone) {
|
|
340
583
|
return (row !== undefined &&
|
|
341
584
|
row.status === "running" &&
|
|
585
|
+
row.fs_status !== "lost" &&
|
|
342
586
|
row.provider === p.plugin &&
|
|
343
587
|
row.handle !== null &&
|
|
344
588
|
!isInstance(row, gone));
|
|
345
589
|
}
|
|
346
590
|
/**
|
|
347
|
-
* Under a just-taken shared lease: confirms the row is still
|
|
348
|
-
* null), records the holder and kills crashed holders' commands.
|
|
591
|
+
* Under a just-taken shared lease: confirms the row is still `accept`ed (else releases and
|
|
592
|
+
* returns null), records the holder and kills crashed holders' commands.
|
|
349
593
|
*/
|
|
350
|
-
async asShared(ctx, p, lease,
|
|
351
|
-
const
|
|
594
|
+
async asShared(ctx, p, lease, accept, signal) {
|
|
595
|
+
const env = ctx.environmentId;
|
|
352
596
|
try {
|
|
353
597
|
const row = await this.readRow(ctx);
|
|
354
|
-
if (!
|
|
598
|
+
if (!accept(row)) {
|
|
355
599
|
await lease.release();
|
|
356
600
|
return null;
|
|
357
601
|
}
|
|
358
602
|
// Bump the fence as every acquisition did before phase C: an exclusive holder whose lease
|
|
359
603
|
// expired unnoticed (a stalled sweep) then fails its next write instead of suspending or
|
|
360
604
|
// destroying the instance under this holder. Shared holders never compare it.
|
|
361
|
-
await this.
|
|
362
|
-
.updateTable("
|
|
605
|
+
await this.db
|
|
606
|
+
.updateTable("environment_instances")
|
|
363
607
|
.set((eb) => ({ generation: eb("generation", "+", 1) }))
|
|
364
|
-
.where("
|
|
608
|
+
.where("environment_id", "=", env)
|
|
365
609
|
.execute();
|
|
366
|
-
await this.register(
|
|
367
|
-
//
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
const sig = AbortSignal.any([signal, lease.signal]);
|
|
371
|
-
const impl = p.impl;
|
|
372
|
-
if (impl.killTagged) {
|
|
373
|
-
// by tag whenever the provider can, even with cap 1: while a hot change of
|
|
374
|
-
// maxConcurrentCommands spreads, a replica on a larger cap may have let a sibling in
|
|
375
|
-
await this.kill(conv, "takeover", async () => impl.killTagged?.(row.handle, dead.tags, sig), sig);
|
|
376
|
-
}
|
|
377
|
-
else if ((await this.leases.sharedCount(conv)) <= 1) {
|
|
378
|
-
// cap 1 and this holder is alone: killing everything cannot hit a sibling
|
|
379
|
-
await this.killLeftovers(conv, row.handle, p, sig, "takeover");
|
|
380
|
-
}
|
|
381
|
-
else {
|
|
382
|
-
this.d.log.warn("cannot kill leftover sandbox commands: the provider has no killTagged and another holder is live; the in-sandbox time limit ends them", { conversationId: conv });
|
|
383
|
-
}
|
|
384
|
-
}
|
|
610
|
+
await this.register(env, lease.token);
|
|
611
|
+
// other holders (any conversation of the environment) may be running: only the dead
|
|
612
|
+
// holders' own commands are killed
|
|
613
|
+
await this.killTagged(env, row.handle, p, await this.claim(env), signal, lease);
|
|
385
614
|
return {
|
|
386
615
|
lease,
|
|
387
616
|
handle: row.handle,
|
|
388
617
|
createdAt: row.created_at.getTime(),
|
|
389
|
-
|
|
618
|
+
fsResets: row.fs_resets,
|
|
619
|
+
info: { fresh: false, restarted: false },
|
|
390
620
|
};
|
|
391
621
|
}
|
|
392
622
|
catch (err) {
|
|
393
|
-
await this.unregister(
|
|
623
|
+
await this.unregister(env, lease.token);
|
|
394
624
|
await lease.release();
|
|
395
625
|
throw err;
|
|
396
626
|
}
|
|
397
627
|
}
|
|
398
628
|
/**
|
|
399
|
-
* Under a just-taken exclusive lease: makes the
|
|
400
|
-
* recorded instance is the one found gone), kills leftovers, then downgrades to
|
|
629
|
+
* Under a just-taken exclusive lease: makes the instance run (file system, create / resume /
|
|
630
|
+
* rebuild when the recorded instance is the one found gone), kills leftovers, then downgrades to
|
|
631
|
+
* a shared lease. Crashed holders' records are claimed here: on an instance just resumed from
|
|
632
|
+
* `suspended` one `killAll` covers them and anything else left (nobody else holds the instance),
|
|
633
|
+
* on a new instance there is nothing to kill, and on one that already ran their tags are killed.
|
|
401
634
|
*/
|
|
402
635
|
async asExclusive(ctx, p, lease, gone, signal) {
|
|
403
|
-
const
|
|
636
|
+
const env = ctx.environmentId;
|
|
404
637
|
const sig = AbortSignal.any([signal, lease.signal]);
|
|
405
638
|
try {
|
|
406
639
|
const s = await this.ensure(ctx, p, lease, sig, gone);
|
|
407
|
-
const dead = await this.claim(
|
|
408
|
-
if (
|
|
409
|
-
await this.killLeftovers(
|
|
640
|
+
const dead = await this.claim(env);
|
|
641
|
+
if (s.resumed)
|
|
642
|
+
await this.killLeftovers(env, s.handle, p, sig, dead);
|
|
643
|
+
else if (!s.info.fresh)
|
|
644
|
+
await this.killTagged(env, s.handle, p, dead, signal, lease);
|
|
645
|
+
if (s.resumed || s.info.fresh) {
|
|
646
|
+
await this.killPendingTasks(env, s.handle, s.createdAt, p, lease.token, ctx.actor, sig);
|
|
410
647
|
}
|
|
411
648
|
const shared = await lease.downgrade();
|
|
412
|
-
await this.register(
|
|
413
|
-
return {
|
|
649
|
+
await this.register(env, shared.token);
|
|
650
|
+
return {
|
|
651
|
+
lease: shared,
|
|
652
|
+
handle: s.handle,
|
|
653
|
+
createdAt: s.createdAt,
|
|
654
|
+
fsResets: s.fsResets,
|
|
655
|
+
info: s.info,
|
|
656
|
+
};
|
|
414
657
|
}
|
|
415
658
|
catch (err) {
|
|
416
|
-
await this.unregister(
|
|
659
|
+
await this.unregister(env, lease.token);
|
|
417
660
|
// releases the lease in whatever kind it has now (also after a downgrade)
|
|
418
661
|
await lease.release();
|
|
419
662
|
throw err;
|
|
420
663
|
}
|
|
421
664
|
}
|
|
422
665
|
/** Ends a holder: unregisters it before releasing its lease, then moves `last_used_at` forward. */
|
|
423
|
-
async leave(
|
|
424
|
-
await this.unregister(
|
|
666
|
+
async leave(env, e) {
|
|
667
|
+
await this.unregister(env, e.lease.token);
|
|
425
668
|
await e.lease.release();
|
|
426
|
-
await this.
|
|
427
|
-
.updateTable("
|
|
669
|
+
await this.db
|
|
670
|
+
.updateTable("environment_instances")
|
|
428
671
|
.set({ last_used_at: sql `GREATEST(last_used_at, now())` })
|
|
429
|
-
.where("
|
|
672
|
+
.where("environment_id", "=", env)
|
|
430
673
|
.where("status", "=", "running")
|
|
431
674
|
.execute()
|
|
432
|
-
.catch((err) => this.
|
|
675
|
+
.catch((err) => this.log.warn("cannot update sandbox last_used_at", { environmentId: env, err }));
|
|
433
676
|
}
|
|
434
|
-
async register(
|
|
435
|
-
const ttl = this.
|
|
677
|
+
async register(env, token) {
|
|
678
|
+
const ttl = this.config.get().sandbox.limits.maxCommandSeconds * 1000 + HOLDER_TTL_MARGIN_MS;
|
|
436
679
|
await this.leases
|
|
437
|
-
.registerHolder(
|
|
438
|
-
.catch((err) => this.
|
|
680
|
+
.registerHolder(env, token, ttl)
|
|
681
|
+
.catch((err) => this.log.warn("cannot record sandbox holder", { environmentId: env, err }));
|
|
439
682
|
}
|
|
440
|
-
async unregister(
|
|
683
|
+
async unregister(env, token) {
|
|
441
684
|
await this.leases
|
|
442
|
-
.unregisterHolder(
|
|
443
|
-
.catch((err) => this.
|
|
685
|
+
.unregisterHolder(env, token)
|
|
686
|
+
.catch((err) => this.log.warn("cannot clear sandbox holder record", { environmentId: env, err }));
|
|
444
687
|
}
|
|
445
|
-
/**
|
|
446
|
-
|
|
447
|
-
* the caller's exclusive lease; a Redis error counts as none.
|
|
448
|
-
*/
|
|
449
|
-
async claim(conv, exclusiveToken) {
|
|
688
|
+
/** Crashed holders' tags (claimed); a Redis error counts as none. */
|
|
689
|
+
async claim(env) {
|
|
450
690
|
try {
|
|
451
|
-
return await this.leases.claimDeadHolders(
|
|
691
|
+
return await this.leases.claimDeadHolders(env);
|
|
452
692
|
}
|
|
453
693
|
catch (err) {
|
|
454
|
-
this.
|
|
455
|
-
return
|
|
694
|
+
this.log.warn("cannot read sandbox holder records", { environmentId: env, err });
|
|
695
|
+
return [];
|
|
456
696
|
}
|
|
457
697
|
}
|
|
698
|
+
/** Kills the commands of crashed holders by their tags (`killTagged`); nothing without tags. */
|
|
699
|
+
async killTagged(env, handle, p, tags, signal, lease) {
|
|
700
|
+
if (tags.length === 0)
|
|
701
|
+
return;
|
|
702
|
+
const sig = AbortSignal.any([signal, lease.signal]);
|
|
703
|
+
await this.kill(env, "takeover", () => p.impl.killTagged(handle, tags, sig), sig);
|
|
704
|
+
}
|
|
458
705
|
/** Runs one kill call: gone and abort are thrown, other failures only warned. */
|
|
459
|
-
async kill(
|
|
706
|
+
async kill(environmentId, why, call, signal) {
|
|
460
707
|
try {
|
|
461
708
|
await call();
|
|
462
709
|
if (why === "takeover") {
|
|
463
|
-
this.
|
|
710
|
+
this.log.info("sandbox taken over from a holder that did not finish; leftover commands killed", { environmentId });
|
|
464
711
|
}
|
|
465
712
|
}
|
|
466
713
|
catch (err) {
|
|
467
714
|
if (signal?.aborted || isSandboxGoneError(err))
|
|
468
715
|
throw err;
|
|
469
|
-
this.
|
|
716
|
+
this.log.warn("cannot kill leftover sandbox commands", { environmentId, err });
|
|
470
717
|
}
|
|
471
718
|
}
|
|
472
719
|
/**
|
|
473
|
-
*
|
|
474
|
-
* Skips
|
|
475
|
-
* window (about a second) first.
|
|
720
|
+
* Suspends running instances idle longer than `suspendAfterIdleMinutes` or running continuously
|
|
721
|
+
* longer than `maxRunningMinutes`. Skips an environment with any holder; one past
|
|
722
|
+
* `maxRunningMinutes` gets the intent for one poll window (about a second) first. Returns how
|
|
723
|
+
* many it suspended. Called every minute by the `sandbox-suspend-idle` cron.
|
|
476
724
|
*/
|
|
477
725
|
async suspendIdle(now) {
|
|
478
|
-
|
|
726
|
+
// cloud only: suspendDue keeps to this provider's rows, so `@device` is never suspended here
|
|
727
|
+
const p = this.providers.cloud();
|
|
479
728
|
if (!p)
|
|
480
729
|
return 0;
|
|
481
730
|
const due = this.suspendDue(now, p.plugin);
|
|
482
|
-
const runningBefore = new Date(now.getTime() - this.
|
|
731
|
+
const runningBefore = new Date(now.getTime() - this.config.get().sandbox.limits.maxRunningMinutes * 60_000);
|
|
483
732
|
const rows = await this.candidates(due);
|
|
484
733
|
let n = 0;
|
|
485
734
|
await inPool(rows, SWEEP_CONCURRENCY, async (c) => {
|
|
486
735
|
const overdue = c.running_since !== null && c.running_since < runningBefore;
|
|
487
|
-
const done = await this.tryLocked(c.
|
|
488
|
-
const id = c.
|
|
489
|
-
const row = await this.
|
|
490
|
-
.selectFrom("
|
|
736
|
+
const done = await this.tryLocked(c.environment_id, async (lease) => {
|
|
737
|
+
const id = c.environment_id;
|
|
738
|
+
const row = await this.db
|
|
739
|
+
.selectFrom("environment_instances")
|
|
491
740
|
.selectAll()
|
|
492
|
-
.where("
|
|
741
|
+
.where("environment_id", "=", id)
|
|
493
742
|
.where(due)
|
|
494
743
|
.executeTakeFirst();
|
|
495
744
|
if (!row?.handle)
|
|
@@ -504,7 +753,7 @@ export class SandboxService {
|
|
|
504
753
|
running_since: null,
|
|
505
754
|
sweep_failed_at: null,
|
|
506
755
|
});
|
|
507
|
-
this.
|
|
756
|
+
this.log.info("sandbox suspended", { environmentId: id });
|
|
508
757
|
return true;
|
|
509
758
|
}, overdue);
|
|
510
759
|
if (done)
|
|
@@ -513,106 +762,217 @@ export class SandboxService {
|
|
|
513
762
|
return n;
|
|
514
763
|
}
|
|
515
764
|
/**
|
|
516
|
-
*
|
|
517
|
-
*
|
|
765
|
+
* Destruction sweep: an instance unused for longer than `destroyAfterInactiveDays` is destroyed
|
|
766
|
+
* (the instance only; the file system is kept and the next use builds a new instance on it); an
|
|
767
|
+
* environment that was deleted, or whose org was deleted, loses its instance and its file system
|
|
768
|
+
* and its row is deleted (as {@link destroyEnvironment} does). Environments with any holder are
|
|
769
|
+
* skipped. Inactivity never destroys a `@device` instance; a gone `@device` environment is
|
|
770
|
+
* destroyed like any other while `@device` is registered. Returns how many it destroyed. Called
|
|
771
|
+
* hourly by the `sandbox-destroy-inactive` cron.
|
|
518
772
|
*/
|
|
519
773
|
async destroyInactive(now) {
|
|
520
|
-
const p = this.
|
|
521
|
-
|
|
522
|
-
return 0;
|
|
523
|
-
const due = this.destroyDue(now);
|
|
524
|
-
const rows = await this.candidates(due);
|
|
774
|
+
const p = this.providers.cloud();
|
|
775
|
+
const device = this.providers.forName(DEVICE_PROVIDER) !== undefined;
|
|
525
776
|
let n = 0;
|
|
526
|
-
|
|
777
|
+
// Gone environments (deleted, or of a deleted org) on every provider that is here, `@device`
|
|
778
|
+
// too (its destroy calls never need the machine online); rows whose provider is missing wait.
|
|
779
|
+
const here = (eb) => eb.or([
|
|
780
|
+
...(p ? [this.cloudRow(eb)] : []),
|
|
781
|
+
...(device ? [eb("provider", "=", DEVICE_PROVIDER)] : []),
|
|
782
|
+
]);
|
|
783
|
+
const gone = (eb) => eb.and([here(eb), this.environmentGone(eb)]);
|
|
784
|
+
if (p || device) {
|
|
785
|
+
await inPool(await this.candidates(gone), SWEEP_CONCURRENCY, async ({ environment_id: id }) => {
|
|
786
|
+
const done = await this.tryLocked(id, async (lease) => {
|
|
787
|
+
const row = await this.db
|
|
788
|
+
.selectFrom("environment_instances")
|
|
789
|
+
.selectAll()
|
|
790
|
+
.where("environment_id", "=", id)
|
|
791
|
+
.where(gone)
|
|
792
|
+
.executeTakeFirst();
|
|
793
|
+
const owner = row && this.rowProvider(row);
|
|
794
|
+
if (!row || !owner)
|
|
795
|
+
return false;
|
|
796
|
+
await this.destroyRow(row, owner, lease, "environment");
|
|
797
|
+
return true;
|
|
798
|
+
});
|
|
799
|
+
if (done)
|
|
800
|
+
n++;
|
|
801
|
+
});
|
|
802
|
+
}
|
|
803
|
+
// inactivity destroys cloud instances only: `@device` instances are never swept
|
|
804
|
+
if (!p)
|
|
805
|
+
return n;
|
|
806
|
+
const due = this.instanceDue(now);
|
|
807
|
+
const idle = (eb) => eb.and([this.cloudRow(eb), due(eb)]);
|
|
808
|
+
await inPool(await this.candidates(idle), SWEEP_CONCURRENCY, async ({ environment_id: id }) => {
|
|
527
809
|
const done = await this.tryLocked(id, async (lease) => {
|
|
528
|
-
const row = await this.
|
|
529
|
-
.selectFrom("
|
|
810
|
+
const row = await this.db
|
|
811
|
+
.selectFrom("environment_instances")
|
|
530
812
|
.selectAll()
|
|
531
|
-
.where("
|
|
532
|
-
.where(
|
|
813
|
+
.where("environment_id", "=", id)
|
|
814
|
+
.where(idle)
|
|
533
815
|
.executeTakeFirst();
|
|
534
816
|
if (!row)
|
|
535
817
|
return false;
|
|
536
|
-
await this.destroyRow(row, p, lease);
|
|
818
|
+
await this.destroyRow(row, p, lease, "instance");
|
|
537
819
|
return true;
|
|
538
820
|
});
|
|
539
821
|
if (done)
|
|
540
822
|
n++;
|
|
541
823
|
});
|
|
542
|
-
// 早先已销毁、对话后来才删除、且没有遗留旧实例的记录
|
|
543
|
-
await this.d.db
|
|
544
|
-
.deleteFrom("sandboxes")
|
|
545
|
-
.where("status", "=", "destroyed")
|
|
546
|
-
.where("prev_handle", "is", null)
|
|
547
|
-
.where(({ not, exists, selectFrom }) => not(exists(selectFrom("conversations")
|
|
548
|
-
.select("conversations.id")
|
|
549
|
-
.whereRef("conversations.id", "=", "sandboxes.conversation_id"))))
|
|
550
|
-
.execute();
|
|
551
824
|
return n;
|
|
552
825
|
}
|
|
553
826
|
/**
|
|
554
|
-
*
|
|
555
|
-
*
|
|
556
|
-
* to 10 minutes.
|
|
827
|
+
* Destroys the environment's instance, then its file system, then deletes its instance row
|
|
828
|
+
* (environment deletion, offboarding); nothing to do without a row. Waits for the exclusive lease
|
|
829
|
+
* (with intent, so running commands finish and new ones wait) up to 10 minutes. Without the
|
|
830
|
+
* environment's provider it throws and the row stays, marked `sweep_failed_at` (the job is
|
|
831
|
+
* retried, and the destruction sweep finishes a deleted environment once the provider is back).
|
|
832
|
+
* Files kept by another provider that is not loaded cannot be destroyed: it throws and the row
|
|
833
|
+
* stays.
|
|
557
834
|
*/
|
|
558
|
-
async
|
|
559
|
-
const row = await this.
|
|
560
|
-
.selectFrom("
|
|
561
|
-
.select("
|
|
562
|
-
.where("
|
|
835
|
+
async destroyEnvironment(environmentId) {
|
|
836
|
+
const row = await this.db
|
|
837
|
+
.selectFrom("environment_instances")
|
|
838
|
+
.select("environment_id")
|
|
839
|
+
.where("environment_id", "=", environmentId)
|
|
563
840
|
.executeTakeFirst();
|
|
564
|
-
if (!row
|
|
841
|
+
if (!row)
|
|
565
842
|
return;
|
|
566
|
-
const p = this.
|
|
843
|
+
const p = await this.providerOf(environmentId);
|
|
567
844
|
if (!p) {
|
|
568
|
-
this.
|
|
569
|
-
|
|
845
|
+
await this.db
|
|
846
|
+
.updateTable("environment_instances")
|
|
847
|
+
.set({ sweep_failed_at: sql `now()` })
|
|
848
|
+
.where("environment_id", "=", environmentId)
|
|
849
|
+
.execute();
|
|
850
|
+
throw new Error(`cannot destroy environment ${environmentId}: its sandbox provider is not there; ` +
|
|
851
|
+
"the destruction sweep retries it");
|
|
570
852
|
}
|
|
571
|
-
const lease = await this.leases.acquireExclusive(
|
|
853
|
+
const lease = await this.leases.acquireExclusive(environmentId, AbortSignal.timeout(EXCLUSIVE_WAIT_MS), undefined, { intent: true });
|
|
572
854
|
try {
|
|
573
|
-
const current = await this.
|
|
574
|
-
.selectFrom("
|
|
855
|
+
const current = await this.db
|
|
856
|
+
.selectFrom("environment_instances")
|
|
575
857
|
.selectAll()
|
|
576
|
-
.where("
|
|
577
|
-
.where("status", "<>", "destroyed")
|
|
858
|
+
.where("environment_id", "=", environmentId)
|
|
578
859
|
.executeTakeFirst();
|
|
579
860
|
if (current)
|
|
580
|
-
await this.destroyRow(current, p, lease);
|
|
861
|
+
await this.destroyRow(current, p, lease, "environment");
|
|
581
862
|
}
|
|
582
863
|
finally {
|
|
583
864
|
await lease.release();
|
|
584
865
|
}
|
|
585
866
|
}
|
|
586
867
|
/**
|
|
587
|
-
*
|
|
588
|
-
*
|
|
868
|
+
* Starts the environment over with empty files (after `environment.files_lost`, or on request):
|
|
869
|
+
* under the exclusive lease (waited for like {@link destroyEnvironment}) destroys any instance and
|
|
870
|
+
* the file system (a `lost` one is only dropped), and records no file system; the next use
|
|
871
|
+
* creates a new one. Nothing to do without a row. Throws `sandbox.unavailable` without a provider.
|
|
589
872
|
*/
|
|
590
|
-
async
|
|
591
|
-
const
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
873
|
+
async resetFileSystem(environmentId) {
|
|
874
|
+
const p = await this.require(environmentId);
|
|
875
|
+
const lease = await this.leases.acquireExclusive(environmentId, AbortSignal.timeout(EXCLUSIVE_WAIT_MS), undefined, { intent: true });
|
|
876
|
+
try {
|
|
877
|
+
const row = await this.db
|
|
878
|
+
.selectFrom("environment_instances")
|
|
879
|
+
.selectAll()
|
|
880
|
+
.where("environment_id", "=", environmentId)
|
|
881
|
+
.executeTakeFirst();
|
|
882
|
+
if (!row)
|
|
883
|
+
return;
|
|
884
|
+
const fence = { gen: row.generation };
|
|
885
|
+
const signal = opSignal(lease);
|
|
886
|
+
const prev = await this.destroyInstance(row, p, signal);
|
|
887
|
+
await this.destroyFiles(row, p, signal, true);
|
|
888
|
+
if (!(await lease.renew()))
|
|
889
|
+
throw new SandboxLockLostError(environmentId);
|
|
890
|
+
await this.fenced(lease, environmentId, fence, {
|
|
891
|
+
fs_handle: null,
|
|
892
|
+
fs_status: "none",
|
|
893
|
+
handle: null,
|
|
894
|
+
status: "none",
|
|
895
|
+
running_since: null,
|
|
896
|
+
suspended_at: null,
|
|
897
|
+
destroyed_at: sql `now()`,
|
|
898
|
+
sweep_failed_at: null,
|
|
899
|
+
// conversations that used the old files are told they were reset (seenBy)
|
|
900
|
+
fs_resets: sql `fs_resets + 1`,
|
|
901
|
+
...prev,
|
|
902
|
+
});
|
|
903
|
+
this.log.info("environment files reset", { environmentId });
|
|
904
|
+
}
|
|
905
|
+
finally {
|
|
906
|
+
await lease.release();
|
|
597
907
|
}
|
|
598
|
-
await this.kill(conversationId, why, () => killAll.call(p.impl, handle, signal), signal);
|
|
599
908
|
}
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
909
|
+
/**
|
|
910
|
+
* Under the exclusive lease (before the downgrade), on an instance just resumed from
|
|
911
|
+
* `suspended`: kills every command left in it (`killAll`), crashed holders' (`dead`, already
|
|
912
|
+
* claimed) included. Never on the shared path: other holders may be running commands. A gone
|
|
913
|
+
* instance (`SandboxGoneError`) and an abort are thrown (`withSandbox` rebuilds / ends); other
|
|
914
|
+
* failures are only warned.
|
|
915
|
+
*/
|
|
916
|
+
async killLeftovers(environmentId, handle, p, signal, dead) {
|
|
917
|
+
await this.kill(environmentId, dead.length > 0 ? "takeover" : "resume", () => p.impl.killAll(handle, signal), signal);
|
|
918
|
+
}
|
|
919
|
+
/**
|
|
920
|
+
* Under the exclusive lease, right after a resume or a (re)build: kills the sessions of the
|
|
921
|
+
* background tasks marked lost while their process may live on (`kill_pending`), in one exec of
|
|
922
|
+
* `pkill -KILL -s <sid>`, then clears their flags. Tasks of another instance have no process
|
|
923
|
+
* here and are only cleared. A gone instance and an abort are thrown; another failure is warned
|
|
924
|
+
* and leaves the flags of this instance's tasks for the next resume.
|
|
925
|
+
*/
|
|
926
|
+
async killPendingTasks(environmentId, handle, createdAt, p, tag, actor, signal) {
|
|
927
|
+
const rows = await this.background.pendingKills(environmentId);
|
|
928
|
+
if (rows.length === 0)
|
|
929
|
+
return;
|
|
930
|
+
const mine = rows.filter((r) => r.sandboxCreatedAt?.getTime() === createdAt && r.sid !== null && SID_RE.test(r.sid));
|
|
931
|
+
let cleared = rows;
|
|
932
|
+
if (mine.length > 0) {
|
|
933
|
+
// pkill exits 1 when the session has no process left: only a higher status is a failure
|
|
934
|
+
const command = `rc=0; for sid in ${mine.map((r) => r.sid).join(" ")}; do pkill -KILL -s "$sid"; c=$?; [ "$c" -le 1 ] || rc=$c; done; exit $rc`;
|
|
935
|
+
try {
|
|
936
|
+
const r = await p.impl.exec(handle, {
|
|
937
|
+
command,
|
|
938
|
+
timeoutMs: PENDING_KILL_TIMEOUT_MS,
|
|
939
|
+
maxOutputBytes: 4096,
|
|
940
|
+
signal,
|
|
941
|
+
tag,
|
|
942
|
+
actor,
|
|
943
|
+
});
|
|
944
|
+
if (r.exitCode !== 0) {
|
|
945
|
+
throw new Error(`pkill exited with ${r.exitCode === null ? "no status" : r.exitCode}: ${r.stderr.slice(0, 500)}`);
|
|
946
|
+
}
|
|
947
|
+
this.log.info("lost background tasks killed after a sandbox resume", {
|
|
948
|
+
environmentId,
|
|
949
|
+
tasks: mine.map((t) => t.id),
|
|
950
|
+
});
|
|
951
|
+
}
|
|
952
|
+
catch (err) {
|
|
953
|
+
if (signal.aborted || isSandboxGoneError(err))
|
|
954
|
+
throw err;
|
|
955
|
+
this.log.warn("cannot kill lost background tasks after a sandbox resume", {
|
|
956
|
+
environmentId,
|
|
957
|
+
err,
|
|
958
|
+
});
|
|
959
|
+
const kept = new Set(mine.map((r) => r.id));
|
|
960
|
+
cleared = rows.filter((r) => !kept.has(r.id));
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
await this.background.clearKill(cleared.map((r) => r.id));
|
|
605
964
|
}
|
|
606
965
|
/**
|
|
607
|
-
*
|
|
966
|
+
* A fenced write: only while the lease holds and the row's generation is still the one read
|
|
967
|
+
* under it (and incremented); otherwise {@link SandboxLockLostError}.
|
|
608
968
|
*/
|
|
609
969
|
async fenced(lease, id, fence, set) {
|
|
610
970
|
if (lease.signal.aborted)
|
|
611
971
|
throw new SandboxLockLostError(id);
|
|
612
|
-
const r = await this.
|
|
613
|
-
.updateTable("
|
|
972
|
+
const r = await this.db
|
|
973
|
+
.updateTable("environment_instances")
|
|
614
974
|
.set({ ...set, generation: fence.gen + 1 })
|
|
615
|
-
.where("
|
|
975
|
+
.where("environment_id", "=", id)
|
|
616
976
|
.where("generation", "=", fence.gen)
|
|
617
977
|
.returning("created_at")
|
|
618
978
|
.executeTakeFirst();
|
|
@@ -621,11 +981,11 @@ export class SandboxService {
|
|
|
621
981
|
fence.gen++;
|
|
622
982
|
return r.created_at;
|
|
623
983
|
}
|
|
624
|
-
/**
|
|
984
|
+
/** Sweep candidates: those whose last sweep failed come last, the rest least recently used first. */
|
|
625
985
|
async candidates(due) {
|
|
626
|
-
return await this.
|
|
627
|
-
.selectFrom("
|
|
628
|
-
.select(["
|
|
986
|
+
return await this.db
|
|
987
|
+
.selectFrom("environment_instances")
|
|
988
|
+
.select(["environment_id", "running_since"])
|
|
629
989
|
.where(due)
|
|
630
990
|
.orderBy(sql `sweep_failed_at ASC NULLS FIRST`)
|
|
631
991
|
.orderBy("last_used_at")
|
|
@@ -634,17 +994,17 @@ export class SandboxService {
|
|
|
634
994
|
}
|
|
635
995
|
/**
|
|
636
996
|
* One try at the exclusive lease: null → false; else runs `fn`. With `overdue` (past
|
|
637
|
-
* `maxRunningMinutes`) a busy
|
|
638
|
-
* briefly and the sweep gets in if the running ones end. Failures are warned, recorded in
|
|
997
|
+
* `maxRunningMinutes`) a busy environment gets the intent for about a second, so new commands
|
|
998
|
+
* wait briefly and the sweep gets in if the running ones end. Failures are warned, recorded in
|
|
639
999
|
* `sweep_failed_at` (sorted last next time) and return false.
|
|
640
1000
|
*/
|
|
641
|
-
async tryLocked(
|
|
1001
|
+
async tryLocked(environmentId, fn, overdue = false) {
|
|
642
1002
|
let lease;
|
|
643
1003
|
try {
|
|
644
|
-
lease = await this.leases.tryExclusive(
|
|
1004
|
+
lease = await this.leases.tryExclusive(environmentId);
|
|
645
1005
|
if (!lease && overdue) {
|
|
646
1006
|
lease = await this.leases
|
|
647
|
-
.acquireExclusive(
|
|
1007
|
+
.acquireExclusive(environmentId, AbortSignal.timeout(SWEEP_OP_TIMEOUT_MS), Math.max(this.pollMs * 2, SWEEP_INTENT_WAIT_MS), { intent: true })
|
|
648
1008
|
.catch((err) => {
|
|
649
1009
|
if (err instanceof ApiError && err.code === "sandbox.busy")
|
|
650
1010
|
return null;
|
|
@@ -653,7 +1013,7 @@ export class SandboxService {
|
|
|
653
1013
|
}
|
|
654
1014
|
}
|
|
655
1015
|
catch (err) {
|
|
656
|
-
this.
|
|
1016
|
+
this.log.warn("cannot take sandbox lock", { environmentId, err });
|
|
657
1017
|
return false;
|
|
658
1018
|
}
|
|
659
1019
|
if (!lease)
|
|
@@ -662,11 +1022,11 @@ export class SandboxService {
|
|
|
662
1022
|
return await fn(lease);
|
|
663
1023
|
}
|
|
664
1024
|
catch (err) {
|
|
665
|
-
this.
|
|
666
|
-
await this.
|
|
667
|
-
.updateTable("
|
|
1025
|
+
this.log.warn("sandbox maintenance failed; will retry", { environmentId, err });
|
|
1026
|
+
await this.db
|
|
1027
|
+
.updateTable("environment_instances")
|
|
668
1028
|
.set({ sweep_failed_at: sql `now()` })
|
|
669
|
-
.where("
|
|
1029
|
+
.where("environment_id", "=", environmentId)
|
|
670
1030
|
.execute()
|
|
671
1031
|
.catch(() => { });
|
|
672
1032
|
return false;
|
|
@@ -676,82 +1036,121 @@ export class SandboxService {
|
|
|
676
1036
|
}
|
|
677
1037
|
}
|
|
678
1038
|
/**
|
|
679
|
-
* Under the exclusive lease: make the
|
|
1039
|
+
* Under the exclusive lease: make the environment's instance run. `gone`: the instance a holder
|
|
680
1040
|
* found gone; when it is still the recorded one it is rebuilt, when another holder already
|
|
681
1041
|
* rebuilt it the new one is used.
|
|
682
1042
|
*/
|
|
683
1043
|
async ensure(ctx, p, held, signal, gone) {
|
|
684
|
-
const id = ctx.
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
.
|
|
1044
|
+
const id = ctx.environmentId;
|
|
1045
|
+
// Only a live environment of the caller's org gets a row, with the environment's own org; a
|
|
1046
|
+
// row made earlier (another use) keeps its file system handle.
|
|
1047
|
+
await this.db
|
|
1048
|
+
.insertInto("environment_instances")
|
|
1049
|
+
.columns(["environment_id", "org_id", "provider", "status"])
|
|
1050
|
+
.expression((eb) => eb
|
|
1051
|
+
.selectFrom("environments")
|
|
1052
|
+
.select((e) => [
|
|
1053
|
+
"environments.id",
|
|
1054
|
+
"environments.org_id",
|
|
1055
|
+
e.val(p.plugin).as("provider"),
|
|
1056
|
+
e.val("creating").as("status"),
|
|
1057
|
+
])
|
|
1058
|
+
.where("environments.id", "=", id)
|
|
1059
|
+
.where("environments.org_id", "=", ctx.orgId)
|
|
1060
|
+
.where("environments.deleted_at", "is", null))
|
|
1061
|
+
.onConflict((oc) => oc.column("environment_id").doNothing())
|
|
689
1062
|
.execute();
|
|
690
|
-
|
|
691
|
-
|
|
1063
|
+
// Under the lease: a use that waited behind destroyEnvironment (or raced a deletion) must not
|
|
1064
|
+
// bring the environment's provider resources back.
|
|
1065
|
+
const live = await this.db
|
|
1066
|
+
.selectFrom("environments")
|
|
1067
|
+
.select("id")
|
|
1068
|
+
.where("id", "=", id)
|
|
1069
|
+
.where("org_id", "=", ctx.orgId)
|
|
1070
|
+
.where("deleted_at", "is", null)
|
|
1071
|
+
.executeTakeFirst();
|
|
1072
|
+
if (!live)
|
|
1073
|
+
throw environmentNotFound();
|
|
1074
|
+
const row = await this.db
|
|
1075
|
+
.selectFrom("environment_instances")
|
|
692
1076
|
.selectAll()
|
|
693
|
-
.where("
|
|
694
|
-
.
|
|
695
|
-
if (row
|
|
696
|
-
throw
|
|
697
|
-
|
|
1077
|
+
.where("environment_id", "=", id)
|
|
1078
|
+
.executeTakeFirst();
|
|
1079
|
+
if (!row)
|
|
1080
|
+
throw environmentNotFound();
|
|
1081
|
+
if (row.org_id !== ctx.orgId)
|
|
1082
|
+
throw this.foreign(ctx);
|
|
698
1083
|
const fence = { gen: row.generation };
|
|
699
|
-
//
|
|
1084
|
+
// Every acquisition first does a fenced increment: an old holder whose lease expired while it
|
|
1085
|
+
// kept running (a stalled sweep) then fails its next write.
|
|
700
1086
|
await this.fenced(held, id, fence, {});
|
|
1087
|
+
if (row.fs_status === "lost")
|
|
1088
|
+
throw environmentFilesLost(p.plugin === DEVICE_PROVIDER);
|
|
1089
|
+
if (row.provider !== null &&
|
|
1090
|
+
row.provider !== p.plugin &&
|
|
1091
|
+
(row.fs_status !== "none" || row.handle !== null)) {
|
|
1092
|
+
throw otherProvider(row.provider);
|
|
1093
|
+
}
|
|
1094
|
+
if (row.provider !== null &&
|
|
1095
|
+
row.provider !== p.plugin &&
|
|
1096
|
+
row.fs_status === "none" &&
|
|
1097
|
+
row.status === "creating") {
|
|
1098
|
+
// a create by the previous provider ended before its file system was recorded: that
|
|
1099
|
+
// provider may have made it by name, and nothing else would ever destroy it
|
|
1100
|
+
await this.dropUnrecordedFiles(id, row.provider, signal);
|
|
1101
|
+
}
|
|
701
1102
|
const handle = row.handle;
|
|
702
1103
|
const createdAt = row.created_at.getTime();
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
1104
|
+
const fsResets = row.fs_resets;
|
|
1105
|
+
if (handle && isInstance(row, gone) && row.status === "running") {
|
|
1106
|
+
const created = await this.create(ctx, p, held, row, fence, signal);
|
|
1107
|
+
return { ...created, fsResets, info: { fresh: true, restarted: true }, fence };
|
|
706
1108
|
}
|
|
707
|
-
if (handle && row.
|
|
708
|
-
|
|
709
|
-
return { handle, createdAt, info: { fresh: false, recreated: false }, fence };
|
|
710
|
-
}
|
|
711
|
-
if (row.status === "suspended") {
|
|
712
|
-
const r = await p.impl.resume(handle, signal);
|
|
713
|
-
if (r === "resumed") {
|
|
714
|
-
await this.fenced(held, id, fence, {
|
|
715
|
-
status: "running",
|
|
716
|
-
running_since: sql `now()`,
|
|
717
|
-
suspended_at: null,
|
|
718
|
-
sweep_failed_at: null,
|
|
719
|
-
});
|
|
720
|
-
return {
|
|
721
|
-
handle,
|
|
722
|
-
createdAt,
|
|
723
|
-
info: { fresh: false, recreated: false },
|
|
724
|
-
fence,
|
|
725
|
-
resumed: true,
|
|
726
|
-
};
|
|
727
|
-
}
|
|
728
|
-
this.d.log.info("sandbox is gone at the provider; recreating", { conversationId: id });
|
|
729
|
-
}
|
|
1109
|
+
if (handle && row.status === "running") {
|
|
1110
|
+
return { handle, createdAt, fsResets, info: { fresh: false, restarted: false }, fence };
|
|
730
1111
|
}
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
1112
|
+
if (handle && row.status === "suspended") {
|
|
1113
|
+
const r = await p.impl.resume(handle, signal);
|
|
1114
|
+
if (r === "resumed") {
|
|
1115
|
+
await this.fenced(held, id, fence, {
|
|
1116
|
+
status: "running",
|
|
1117
|
+
running_since: sql `now()`,
|
|
1118
|
+
suspended_at: null,
|
|
1119
|
+
sweep_failed_at: null,
|
|
1120
|
+
});
|
|
1121
|
+
return {
|
|
1122
|
+
handle,
|
|
1123
|
+
createdAt,
|
|
1124
|
+
fsResets,
|
|
1125
|
+
info: { fresh: false, restarted: false },
|
|
1126
|
+
fence,
|
|
1127
|
+
resumed: true,
|
|
1128
|
+
};
|
|
743
1129
|
}
|
|
1130
|
+
this.log.info("sandbox instance is gone at the provider; recreating", {
|
|
1131
|
+
environmentId: id,
|
|
1132
|
+
});
|
|
744
1133
|
}
|
|
745
|
-
const created = await this.create(ctx, p, held, fence, signal
|
|
746
|
-
// had_instance
|
|
747
|
-
|
|
1134
|
+
const created = await this.create(ctx, p, held, row, fence, signal);
|
|
1135
|
+
// had_instance only ever becomes true: any earlier instance (even if a rebuild failed since)
|
|
1136
|
+
// makes this one a restart on the kept files
|
|
1137
|
+
return {
|
|
1138
|
+
...created,
|
|
1139
|
+
fsResets,
|
|
1140
|
+
info: { fresh: true, restarted: row.had_instance && row.fs_status === "ready" },
|
|
1141
|
+
fence,
|
|
1142
|
+
};
|
|
748
1143
|
}
|
|
749
1144
|
/**
|
|
750
|
-
*
|
|
751
|
-
*
|
|
1145
|
+
* Under the lease: fenced reset to `creating`, the file system when there is none
|
|
1146
|
+
* (`createFileSystem`, its handle written fenced), then the instance on it, then a fenced write
|
|
1147
|
+
* of the instance handle. `FileSystemGoneError` from `create` records `fs_status = 'lost'` and
|
|
1148
|
+
* throws `environment.files_lost`. When the instance handle cannot be written (lease lost), the
|
|
1149
|
+
* new instance is destroyed (best effort) before {@link SandboxLockLostError} is thrown.
|
|
752
1150
|
*/
|
|
753
|
-
async create(ctx, p, held, fence, signal
|
|
754
|
-
const id = ctx.
|
|
1151
|
+
async create(ctx, p, held, row, fence, signal) {
|
|
1152
|
+
const id = ctx.environmentId;
|
|
1153
|
+
const name = environmentResourceName(id);
|
|
755
1154
|
await this.fenced(held, id, fence, {
|
|
756
1155
|
org_id: ctx.orgId,
|
|
757
1156
|
provider: p.plugin,
|
|
@@ -762,21 +1161,45 @@ export class SandboxService {
|
|
|
762
1161
|
suspended_at: null,
|
|
763
1162
|
destroyed_at: null,
|
|
764
1163
|
sweep_failed_at: null,
|
|
765
|
-
...extra,
|
|
766
1164
|
});
|
|
767
|
-
|
|
768
|
-
//
|
|
769
|
-
const
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
1165
|
+
// the org's effective limits apply at creation only, so a changed setting reaches only
|
|
1166
|
+
// instances created afterwards
|
|
1167
|
+
const limits = await this.orgLimits(ctx.orgId);
|
|
1168
|
+
const cfg = this.config.get().sandbox;
|
|
1169
|
+
let fs = row.fs_status === "ready" && (row.provider === null || row.provider === p.plugin)
|
|
1170
|
+
? row.fs_handle
|
|
1171
|
+
: null;
|
|
1172
|
+
if (fs === null) {
|
|
1173
|
+
fs = await p.impl.createFileSystem({ name, limits, signal });
|
|
1174
|
+
// A lost lease fails this write and leaves the file system alone: createFileSystem returns
|
|
1175
|
+
// the existing one for a name, so the new holder may already use it, files and all.
|
|
1176
|
+
await this.fenced(held, id, fence, {
|
|
1177
|
+
provider: p.plugin,
|
|
1178
|
+
fs_handle: JSON.stringify(fs),
|
|
1179
|
+
fs_status: "ready",
|
|
1180
|
+
});
|
|
1181
|
+
this.log.info("environment file system created", { environmentId: id, provider: p.plugin });
|
|
1182
|
+
}
|
|
1183
|
+
let handle;
|
|
1184
|
+
try {
|
|
1185
|
+
handle = await p.impl.create({
|
|
1186
|
+
name,
|
|
1187
|
+
...(cfg.image !== undefined && { image: cfg.image }),
|
|
1188
|
+
limits,
|
|
1189
|
+
fileSystem: fs,
|
|
1190
|
+
signal,
|
|
1191
|
+
});
|
|
1192
|
+
}
|
|
1193
|
+
catch (err) {
|
|
1194
|
+
if (!isFileSystemGoneError(err))
|
|
1195
|
+
throw err;
|
|
1196
|
+
this.log.warn("environment file system is gone at the provider; its files are lost", {
|
|
1197
|
+
environmentId: id,
|
|
1198
|
+
provider: p.plugin,
|
|
1199
|
+
});
|
|
1200
|
+
await this.fenced(held, id, fence, { fs_status: "lost", status: "none" });
|
|
1201
|
+
throw environmentFilesLost(p.plugin === DEVICE_PROVIDER);
|
|
1202
|
+
}
|
|
780
1203
|
let createdAt;
|
|
781
1204
|
try {
|
|
782
1205
|
createdAt = await this.fenced(held, id, fence, {
|
|
@@ -789,72 +1212,138 @@ export class SandboxService {
|
|
|
789
1212
|
});
|
|
790
1213
|
}
|
|
791
1214
|
catch (e) {
|
|
792
|
-
//
|
|
793
|
-
//
|
|
794
|
-
//
|
|
795
|
-
|
|
796
|
-
|
|
1215
|
+
// Lease lost before the handle was recorded: destroy the new instance (best effort) so it
|
|
1216
|
+
// does not leak. A provider that reuses instances by name (Ruling M10) may then hit the one
|
|
1217
|
+
// the new holder just resumed or created; that heals: its next provider call throws
|
|
1218
|
+
// SandboxGoneError (or resume returns gone) and withSandbox rebuilds in place. The file
|
|
1219
|
+
// system is never touched here.
|
|
1220
|
+
await p.impl.destroy(handle, AbortSignal.timeout(SWEEP_OP_TIMEOUT_MS)).catch((err) => this.log.warn("cannot destroy sandbox created after the lock was lost", {
|
|
1221
|
+
environmentId: id,
|
|
797
1222
|
err,
|
|
798
1223
|
}));
|
|
799
1224
|
throw e;
|
|
800
1225
|
}
|
|
801
|
-
this.
|
|
1226
|
+
this.log.info("sandbox created", { environmentId: id, provider: p.plugin });
|
|
802
1227
|
return { handle, createdAt: createdAt.getTime() };
|
|
803
1228
|
}
|
|
804
1229
|
/**
|
|
805
|
-
*
|
|
1230
|
+
* Destroys the file system another provider may have made under the environment's name without
|
|
1231
|
+
* the kernel recording it (`destroyFileSystemByName`; not there is success). That provider not
|
|
1232
|
+
* loaded, or a failure, is only warned (the files may leak there) so the environment stays
|
|
1233
|
+
* usable; an abort is thrown.
|
|
1234
|
+
*/
|
|
1235
|
+
async dropUnrecordedFiles(id, provider, signal) {
|
|
1236
|
+
const old = this.providers.forName(provider)?.impl;
|
|
1237
|
+
if (!old) {
|
|
1238
|
+
this.log.warn("environment file system may have been made by a provider that is not loaded; it may leak", { environmentId: id, provider });
|
|
1239
|
+
return;
|
|
1240
|
+
}
|
|
1241
|
+
try {
|
|
1242
|
+
await old.destroyFileSystemByName(environmentResourceName(id), signal);
|
|
1243
|
+
}
|
|
1244
|
+
catch (err) {
|
|
1245
|
+
if (signal.aborted)
|
|
1246
|
+
throw err;
|
|
1247
|
+
this.log.warn("cannot destroy the file system of the previous sandbox provider", {
|
|
1248
|
+
environmentId: id,
|
|
1249
|
+
provider,
|
|
1250
|
+
err,
|
|
1251
|
+
});
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
1254
|
+
/**
|
|
1255
|
+
* Destroys an instance of another provider with that provider: still loaded and destroyed →
|
|
1256
|
+
* false; otherwise warns and returns true (the record must be kept).
|
|
806
1257
|
*/
|
|
807
1258
|
async retire(id, provider, handle, signal) {
|
|
808
|
-
const old = this.
|
|
1259
|
+
const old = this.providers.forName(provider)?.impl;
|
|
809
1260
|
if (old) {
|
|
810
1261
|
try {
|
|
811
1262
|
await old.destroy(handle, AbortSignal.any([signal, AbortSignal.timeout(SWEEP_OP_TIMEOUT_MS)]));
|
|
812
1263
|
return false;
|
|
813
1264
|
}
|
|
814
1265
|
catch (err) {
|
|
815
|
-
this.
|
|
816
|
-
|
|
1266
|
+
this.log.warn("cannot destroy sandbox instance of the previous provider", {
|
|
1267
|
+
environmentId: id,
|
|
817
1268
|
provider,
|
|
818
1269
|
err,
|
|
819
1270
|
});
|
|
820
1271
|
return true;
|
|
821
1272
|
}
|
|
822
1273
|
}
|
|
823
|
-
this.
|
|
1274
|
+
this.log.warn("sandbox was created by another provider that is not loaded; its instance may leak", { environmentId: id, provider });
|
|
824
1275
|
return true;
|
|
825
1276
|
}
|
|
826
|
-
/**
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
1277
|
+
/**
|
|
1278
|
+
* Under the lease: destroys the row's instance (and a leaked one of an earlier provider). An
|
|
1279
|
+
* instance without a handle (a create that ended before the handle was recorded) is destroyed by
|
|
1280
|
+
* name. Returns the `prev_*` patch for instances that could not be destroyed.
|
|
1281
|
+
*/
|
|
1282
|
+
async destroyInstance(row, p, signal) {
|
|
1283
|
+
const id = row.environment_id;
|
|
1284
|
+
const owner = row.provider ?? p.plugin;
|
|
831
1285
|
let prev = {};
|
|
832
|
-
if (row.handle &&
|
|
1286
|
+
if (row.handle && owner === p.plugin) {
|
|
833
1287
|
await p.impl.destroy(row.handle, signal);
|
|
834
1288
|
}
|
|
835
1289
|
else if (row.handle && row.status !== "destroyed") {
|
|
836
|
-
if (await this.retire(id,
|
|
837
|
-
prev = { prev_provider:
|
|
1290
|
+
if (await this.retire(id, owner, row.handle, signal)) {
|
|
1291
|
+
prev = { prev_provider: owner, prev_handle: JSON.stringify(row.handle) };
|
|
838
1292
|
}
|
|
839
1293
|
}
|
|
840
|
-
else if (!row.handle && row.
|
|
841
|
-
//
|
|
842
|
-
|
|
1294
|
+
else if (!row.handle && row.status === "creating") {
|
|
1295
|
+
// the provider may have built it already: only the name finds it
|
|
1296
|
+
const impl = owner === p.plugin ? p.impl : this.providers.forName(owner)?.impl;
|
|
1297
|
+
await impl?.destroyByName(environmentResourceName(id), signal);
|
|
843
1298
|
}
|
|
844
1299
|
if (row.prev_handle && row.prev_provider && !("prev_handle" in prev)) {
|
|
845
1300
|
if (!(await this.retire(id, row.prev_provider, row.prev_handle, signal))) {
|
|
846
1301
|
prev = { prev_provider: null, prev_handle: null };
|
|
847
1302
|
}
|
|
848
1303
|
}
|
|
1304
|
+
return prev;
|
|
1305
|
+
}
|
|
1306
|
+
/**
|
|
1307
|
+
* Under the lease: destroys the row's file system with the provider that made it (by name when
|
|
1308
|
+
* no handle was recorded: `createFileSystem` may have finished before the kernel stopped). With
|
|
1309
|
+
* that provider not loaded it throws (the row must stay for a retry), or with `lenient` only
|
|
1310
|
+
* warns (the files leak at that provider).
|
|
1311
|
+
*/
|
|
1312
|
+
async destroyFiles(row, p, signal, lenient = false) {
|
|
1313
|
+
const id = row.environment_id;
|
|
1314
|
+
const owner = row.provider ?? p.plugin;
|
|
1315
|
+
const impl = owner === p.plugin ? p.impl : this.providers.forName(owner)?.impl;
|
|
1316
|
+
if (!impl) {
|
|
1317
|
+
if (row.fs_handle === null)
|
|
1318
|
+
return;
|
|
1319
|
+
if (!lenient) {
|
|
1320
|
+
throw new Error(`cannot destroy the file system of environment ${id}: its sandbox provider ${owner} is not loaded`);
|
|
1321
|
+
}
|
|
1322
|
+
this.log.warn("environment file system of a provider that is not loaded is leaked", {
|
|
1323
|
+
environmentId: id,
|
|
1324
|
+
provider: owner,
|
|
1325
|
+
});
|
|
1326
|
+
return;
|
|
1327
|
+
}
|
|
1328
|
+
if (row.fs_handle !== null)
|
|
1329
|
+
await impl.destroyFileSystem(row.fs_handle, signal);
|
|
1330
|
+
else
|
|
1331
|
+
await impl.destroyFileSystemByName(environmentResourceName(id), signal);
|
|
1332
|
+
}
|
|
1333
|
+
/**
|
|
1334
|
+
* Under the lease: `instance` destroys the instance and marks the row `destroyed` (the file
|
|
1335
|
+
* system stays); `environment` also destroys the file system and deletes the row.
|
|
1336
|
+
*/
|
|
1337
|
+
async destroyRow(row, p, held, mode) {
|
|
1338
|
+
const id = row.environment_id;
|
|
1339
|
+
const fence = { gen: row.generation };
|
|
1340
|
+
const signal = opSignal(held);
|
|
1341
|
+
const prev = await this.destroyInstance(row, p, signal);
|
|
1342
|
+
if (mode === "environment")
|
|
1343
|
+
await this.destroyFiles(row, p, signal);
|
|
849
1344
|
if (!(await held.renew()))
|
|
850
1345
|
throw new SandboxLockLostError(id);
|
|
851
|
-
|
|
852
|
-
.selectFrom("conversations")
|
|
853
|
-
.select("id")
|
|
854
|
-
.where("id", "=", id)
|
|
855
|
-
.executeTakeFirst();
|
|
856
|
-
const keepRow = conversation !== undefined || (prev.prev_handle ?? row.prev_handle) != null;
|
|
857
|
-
if (keepRow) {
|
|
1346
|
+
if (mode === "instance") {
|
|
858
1347
|
await this.fenced(held, id, fence, {
|
|
859
1348
|
status: "destroyed",
|
|
860
1349
|
destroyed_at: sql `now()`,
|
|
@@ -862,21 +1351,21 @@ export class SandboxService {
|
|
|
862
1351
|
sweep_failed_at: null,
|
|
863
1352
|
...prev,
|
|
864
1353
|
});
|
|
1354
|
+
this.log.info("sandbox instance destroyed", { environmentId: id });
|
|
1355
|
+
return;
|
|
865
1356
|
}
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
}
|
|
875
|
-
this.d.log.info("sandbox destroyed", { conversationId: id });
|
|
1357
|
+
const r = await this.db
|
|
1358
|
+
.deleteFrom("environment_instances")
|
|
1359
|
+
.where("environment_id", "=", id)
|
|
1360
|
+
.where("generation", "=", fence.gen)
|
|
1361
|
+
.executeTakeFirst();
|
|
1362
|
+
if (Number(r.numDeletedRows) !== 1)
|
|
1363
|
+
throw new SandboxLockLostError(id, "generation");
|
|
1364
|
+
this.log.info("environment destroyed", { environmentId: id });
|
|
876
1365
|
}
|
|
877
|
-
/**
|
|
1366
|
+
/** Due for suspension: made by this provider, running, and idle or running for too long. */
|
|
878
1367
|
suspendDue(now, plugin) {
|
|
879
|
-
const lim = this.
|
|
1368
|
+
const lim = this.config.get().sandbox.limits;
|
|
880
1369
|
const idleBefore = new Date(now.getTime() - lim.suspendAfterIdleMinutes * 60_000);
|
|
881
1370
|
const runningBefore = new Date(now.getTime() - lim.maxRunningMinutes * 60_000);
|
|
882
1371
|
return (eb) => eb.and([
|
|
@@ -885,24 +1374,50 @@ export class SandboxService {
|
|
|
885
1374
|
eb.or([eb("last_used_at", "<", idleBefore), eb("running_since", "<", runningBefore)]),
|
|
886
1375
|
]);
|
|
887
1376
|
}
|
|
888
|
-
/**
|
|
889
|
-
|
|
890
|
-
|
|
1377
|
+
/** The provider of an instance row: an internal one (`@device`) by name, else the cloud one. */
|
|
1378
|
+
rowProvider(row) {
|
|
1379
|
+
return providerForEnvironment(this.providers, row.provider);
|
|
1380
|
+
}
|
|
1381
|
+
/**
|
|
1382
|
+
* A row the idle sweeps may touch: not on a kernel-internal provider (`@device` instances live on
|
|
1383
|
+
* people's machines and are never suspended or destroyed for inactivity).
|
|
1384
|
+
*/
|
|
1385
|
+
cloudRow(eb) {
|
|
1386
|
+
return eb.or([eb("provider", "is", null), eb("provider", "not like", "@%")]);
|
|
1387
|
+
}
|
|
1388
|
+
/** The environment was deleted, or its org was: the instance and the files go. */
|
|
1389
|
+
environmentGone(eb) {
|
|
1390
|
+
return eb.or([
|
|
1391
|
+
eb.exists(eb
|
|
1392
|
+
.selectFrom("environments")
|
|
1393
|
+
.select("environments.id")
|
|
1394
|
+
.whereRef("environments.id", "=", "environment_instances.environment_id")
|
|
1395
|
+
.where("environments.deleted_at", "is not", null)),
|
|
1396
|
+
eb.exists(eb
|
|
1397
|
+
.selectFrom("orgs")
|
|
1398
|
+
.select("orgs.id")
|
|
1399
|
+
.whereRef("orgs.id", "=", "environment_instances.org_id")
|
|
1400
|
+
.where("orgs.deleted_at", "is not", null)),
|
|
1401
|
+
]);
|
|
1402
|
+
}
|
|
1403
|
+
/** Due for instance destruction: an instance (or a create in progress) unused for too long. */
|
|
1404
|
+
instanceDue(now) {
|
|
1405
|
+
const days = this.config.get().sandbox.limits.destroyAfterInactiveDays;
|
|
891
1406
|
const before = new Date(now.getTime() - days * 24 * 3_600_000);
|
|
892
1407
|
return (eb) => eb.and([
|
|
893
|
-
eb("status", "
|
|
894
|
-
eb
|
|
895
|
-
eb("last_used_at", "<", before),
|
|
896
|
-
eb.exists(eb
|
|
897
|
-
.selectFrom("orgs")
|
|
898
|
-
.select("orgs.id")
|
|
899
|
-
.whereRef("orgs.id", "=", "sandboxes.org_id")
|
|
900
|
-
.where("orgs.deleted_at", "is not", null)),
|
|
901
|
-
eb.not(eb.exists(eb
|
|
902
|
-
.selectFrom("conversations")
|
|
903
|
-
.select("conversations.id")
|
|
904
|
-
.whereRef("conversations.id", "=", "sandboxes.conversation_id"))),
|
|
905
|
-
]),
|
|
1408
|
+
eb("status", "in", ["creating", "running", "suspended"]),
|
|
1409
|
+
eb("last_used_at", "<", before),
|
|
906
1410
|
]);
|
|
907
1411
|
}
|
|
908
|
-
}
|
|
1412
|
+
};
|
|
1413
|
+
SandboxService = __decorate([
|
|
1414
|
+
Injectable(),
|
|
1415
|
+
__param(0, Inject(KernelDb)),
|
|
1416
|
+
__param(1, Inject(REDIS)),
|
|
1417
|
+
__param(2, Inject(SandboxLeases)),
|
|
1418
|
+
__param(3, Inject(SandboxProviders)),
|
|
1419
|
+
__param(4, Inject(KernelConfigService)),
|
|
1420
|
+
__param(5, Inject(KERNEL_LOGGER)),
|
|
1421
|
+
__param(6, Inject(BackgroundTaskStore))
|
|
1422
|
+
], SandboxService);
|
|
1423
|
+
export { SandboxService };
|