@capybara-hq/kernel 0.1.0-rc.3 → 0.1.0-rc.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bootstrap.json +61 -5
- package/dist/admin/admin.controller.d.ts +47 -16
- package/dist/admin/admin.controller.js +44 -19
- package/dist/admin/extensions.controller.d.ts +24 -3
- package/dist/admin/extensions.controller.js +48 -8
- package/dist/admin/setup-lock.d.ts +1 -1
- package/dist/admin/setup-lock.js +1 -1
- package/dist/admin/setup.controller.d.ts +1 -1
- package/dist/admin/setup.controller.js +1 -1
- package/dist/admin/setup.service.d.ts +4 -4
- package/dist/admin/setup.service.js +8 -8
- package/dist/agents/agent-lookup.d.ts +19 -0
- package/dist/agents/agent-lookup.js +21 -0
- package/dist/agents/agent-profile.service.d.ts +29 -0
- package/dist/agents/agent-profile.service.js +117 -0
- package/dist/agents/agents.controller.d.ts +168 -3
- package/dist/agents/agents.controller.js +201 -12
- package/dist/agents/agents.module.d.ts +4 -1
- package/dist/agents/agents.module.js +36 -6
- package/dist/agents/agents.service.d.ts +64 -27
- package/dist/agents/agents.service.js +199 -74
- package/dist/agents/definition.d.ts +53 -0
- package/dist/agents/definition.js +55 -0
- package/dist/agents/from-template.d.ts +106 -0
- package/dist/agents/from-template.js +649 -0
- package/dist/agents/from-template.service.d.ts +60 -0
- package/dist/agents/from-template.service.js +161 -0
- package/dist/agents/instruction-files.d.ts +60 -0
- package/dist/agents/instruction-files.js +111 -0
- package/dist/agents/models.controller.d.ts +9 -3
- package/dist/agents/models.controller.js +19 -4
- package/dist/agents/org-templates.d.ts +176 -0
- package/dist/agents/org-templates.js +827 -0
- package/dist/agents/pending-connector-requests.d.ts +77 -0
- package/dist/agents/pending-connector-requests.js +215 -0
- package/dist/agents/pending-connectors.controller.d.ts +39 -0
- package/dist/agents/pending-connectors.controller.js +91 -0
- package/dist/agents/pending-connectors.service.d.ts +62 -0
- package/dist/agents/pending-connectors.service.js +282 -0
- package/dist/agents/profile.d.ts +96 -0
- package/dist/agents/profile.js +51 -0
- package/dist/agents/readiness.controller.d.ts +26 -0
- package/dist/agents/readiness.controller.js +56 -0
- package/dist/agents/templates.controller.d.ts +86 -5
- package/dist/agents/templates.controller.js +171 -7
- package/dist/agents/templates.d.ts +127 -23
- package/dist/agents/templates.js +476 -53
- package/dist/background/background.controller.d.ts +38 -0
- package/dist/background/background.controller.js +87 -0
- package/dist/background/background.module.d.ts +10 -0
- package/dist/background/background.module.js +39 -0
- package/dist/background/finish.d.ts +44 -0
- package/dist/background/finish.js +94 -0
- package/dist/background/launch-lock.d.ts +15 -0
- package/dist/background/launch-lock.js +40 -0
- package/dist/background/launch.d.ts +36 -0
- package/dist/background/launch.js +70 -0
- package/dist/background/poll.job.d.ts +56 -0
- package/dist/background/poll.job.js +190 -0
- package/dist/background/render.d.ts +23 -0
- package/dist/background/render.js +48 -0
- package/dist/background/session.d.ts +48 -0
- package/dist/background/session.js +79 -0
- package/dist/background/status.d.ts +35 -0
- package/dist/background/status.js +57 -0
- package/dist/background/stop.d.ts +68 -0
- package/dist/background/stop.js +243 -0
- package/dist/background/store.d.ts +162 -0
- package/dist/background/store.js +357 -0
- package/dist/background/store.module.d.ts +6 -0
- package/dist/background/store.module.js +22 -0
- package/dist/background/tasks.service.d.ts +77 -0
- package/dist/background/tasks.service.js +211 -0
- package/dist/background/wake-rules.d.ts +66 -0
- package/dist/background/wake-rules.js +113 -0
- package/dist/background/wake.d.ts +30 -0
- package/dist/background/wake.js +63 -0
- package/dist/catalog/admin-catalog.controller.d.ts +129 -30
- package/dist/catalog/admin-catalog.controller.js +213 -42
- package/dist/catalog/catalog-content.d.ts +58 -32
- package/dist/catalog/catalog-content.js +34 -8
- package/dist/catalog/catalog-import.module.d.ts +7 -0
- package/dist/catalog/catalog-import.module.js +60 -0
- package/dist/catalog/catalog-items.d.ts +111 -7
- package/dist/catalog/catalog-items.js +279 -13
- package/dist/catalog/catalog-scope.service.d.ts +260 -0
- package/dist/catalog/catalog-scope.service.js +1116 -0
- package/dist/catalog/catalog.module.d.ts +7 -9
- package/dist/catalog/catalog.module.js +41 -31
- package/dist/catalog/catalog.service.d.ts +101 -29
- package/dist/catalog/catalog.service.js +104 -106
- package/dist/catalog/claude/agents.js +2 -0
- package/dist/catalog/cli.d.ts +11 -1
- package/dist/catalog/cli.js +26 -1
- package/dist/catalog/component-promotion.d.ts +96 -0
- package/dist/catalog/component-promotion.js +395 -0
- package/dist/catalog/component-scope.controller.d.ts +152 -0
- package/dist/catalog/component-scope.controller.js +287 -0
- package/dist/catalog/component-scope.service.d.ts +303 -0
- package/dist/catalog/component-scope.service.js +936 -0
- package/dist/catalog/component-state.d.ts +174 -0
- package/dist/catalog/component-state.js +437 -0
- package/dist/catalog/job-ids.d.ts +30 -0
- package/dist/catalog/job-ids.js +25 -0
- package/dist/catalog/marketplace-fetch.d.ts +1 -1
- package/dist/catalog/marketplaces.controller.d.ts +22 -5
- package/dist/catalog/marketplaces.controller.js +56 -7
- package/dist/catalog/marketplaces.service.d.ts +54 -14
- package/dist/catalog/marketplaces.service.js +71 -22
- package/dist/catalog/org-catalog.controller.d.ts +125 -34
- package/dist/catalog/org-catalog.controller.js +181 -65
- package/dist/catalog/org-catalog.service.d.ts +65 -33
- package/dist/catalog/org-catalog.service.js +324 -157
- package/dist/catalog/org-extension-switch.d.ts +41 -0
- package/dist/catalog/org-extension-switch.js +115 -0
- package/dist/catalog/org-extension.controller.d.ts +119 -0
- package/dist/catalog/org-extension.controller.js +266 -0
- package/dist/catalog/org-plugin-uploads.controller.d.ts +98 -0
- package/dist/catalog/org-plugin-uploads.controller.js +213 -0
- package/dist/catalog/org-plugin-uploads.d.ts +63 -0
- package/dist/catalog/org-plugin-uploads.js +116 -0
- package/dist/catalog/owned-items.d.ts +184 -0
- package/dist/catalog/owned-items.js +789 -0
- package/dist/catalog/owned-plugin-cleanup.d.ts +56 -0
- package/dist/catalog/owned-plugin-cleanup.js +194 -0
- package/dist/catalog/ownership.d.ts +172 -0
- package/dist/catalog/ownership.js +326 -0
- package/dist/catalog/platform-components.d.ts +221 -0
- package/dist/catalog/platform-components.js +761 -0
- package/dist/catalog/platform-migration.d.ts +115 -0
- package/dist/catalog/platform-migration.js +582 -0
- package/dist/catalog/plugin-import.d.ts +178 -34
- package/dist/catalog/plugin-import.js +623 -83
- package/dist/catalog/plugin-install.d.ts +185 -41
- package/dist/catalog/plugin-install.js +758 -126
- package/dist/catalog/plugin-upgrade.d.ts +260 -225
- package/dist/catalog/plugin-upgrade.js +1148 -1137
- package/dist/catalog/request-rules.d.ts +65 -0
- package/dist/catalog/request-rules.js +217 -0
- package/dist/catalog/requests.service.d.ts +193 -118
- package/dist/catalog/requests.service.js +526 -343
- package/dist/catalog/reserved-names.d.ts +8 -0
- package/dist/catalog/reserved-names.js +42 -0
- package/dist/catalog/scope-orgs.d.ts +21 -0
- package/dist/catalog/scope-orgs.js +67 -0
- package/dist/catalog/scope-state.d.ts +35 -0
- package/dist/catalog/scope-state.js +87 -0
- package/dist/catalog/scope-sync-workers.d.ts +7 -0
- package/dist/catalog/scope-sync-workers.js +29 -0
- package/dist/catalog/scope-sync.d.ts +176 -0
- package/dist/catalog/scope-sync.js +576 -0
- package/dist/catalog/scope.d.ts +42 -0
- package/dist/catalog/scope.js +42 -0
- package/dist/catalog/unshare.d.ts +22 -0
- package/dist/catalog/unshare.js +118 -0
- package/dist/channels/actions.js +1 -0
- package/dist/channels/cards.js +21 -3
- package/dist/channels/channels.module.js +2 -0
- package/dist/channels/connections.service.d.ts +16 -1
- package/dist/channels/connections.service.js +38 -4
- package/dist/channels/inbound.d.ts +9 -1
- package/dist/channels/inbound.js +67 -5
- package/dist/channels/phrases.d.ts +35 -0
- package/dist/channels/phrases.js +29 -0
- package/dist/channels/threads.d.ts +5 -0
- package/dist/channels/threads.js +8 -0
- package/dist/config/docs.js +3 -1
- package/dist/config/schema.d.ts +24 -2
- package/dist/config/schema.js +145 -19
- package/dist/conversations/access.d.ts +6 -4
- package/dist/conversations/access.js +29 -5
- package/dist/conversations/agent-tree.d.ts +46 -0
- package/dist/conversations/agent-tree.js +92 -0
- package/dist/conversations/attachments.service.d.ts +30 -0
- package/dist/conversations/attachments.service.js +130 -5
- package/dist/conversations/conversations.controller.d.ts +79 -23
- package/dist/conversations/conversations.controller.js +141 -50
- package/dist/conversations/conversations.module.d.ts +1 -1
- package/dist/conversations/conversations.module.js +10 -7
- package/dist/conversations/conversations.service.d.ts +108 -9
- package/dist/conversations/conversations.service.js +237 -30
- package/dist/conversations/environment-status.d.ts +24 -0
- package/dist/conversations/environment-status.js +71 -0
- package/dist/conversations/event-log.d.ts +21 -6
- package/dist/conversations/event-log.js +70 -1
- package/dist/conversations/events.d.ts +140 -0
- package/dist/conversations/events.js +142 -13
- package/dist/conversations/queue.d.ts +3 -1
- package/dist/conversations/queue.js +13 -2
- package/dist/conversations/queue.service.d.ts +1 -1
- package/dist/conversations/queue.service.js +8 -1
- package/dist/conversations/titles.d.ts +12 -3
- package/dist/conversations/titles.js +36 -8
- package/dist/conversations/tree.d.ts +17 -0
- package/dist/conversations/tree.js +42 -0
- package/dist/ctx/create-ctx.d.ts +5 -0
- package/dist/ctx/create-ctx.js +15 -5
- package/dist/db/kernel-schema.d.ts +420 -20
- package/dist/db/pool.d.ts +5 -0
- package/dist/db/pool.js +3 -0
- package/dist/environments/access.d.ts +41 -0
- package/dist/environments/access.js +77 -0
- package/dist/environments/conversation-environments.d.ts +61 -0
- package/dist/environments/conversation-environments.js +170 -0
- package/dist/environments/dto.d.ts +25 -0
- package/dist/environments/dto.js +12 -0
- package/dist/environments/environments.controller.d.ts +33 -0
- package/dist/environments/environments.controller.js +110 -0
- package/dist/environments/environments.module.d.ts +8 -0
- package/dist/environments/environments.module.js +27 -0
- package/dist/environments/environments.service.d.ts +153 -0
- package/dist/environments/environments.service.js +541 -0
- package/dist/environments/legacy-cleanup.job.d.ts +34 -0
- package/dist/environments/legacy-cleanup.job.js +141 -0
- package/dist/errors.d.ts +90 -2
- package/dist/errors.js +90 -2
- package/dist/evals/reconstruct.d.ts +18 -9
- package/dist/evals/reconstruct.js +26 -7
- package/dist/evals/run-replay.d.ts +7 -2
- package/dist/evals/run-replay.js +42 -7
- package/dist/evals/step-replay.d.ts +2 -5
- package/dist/evals/step-replay.js +13 -7
- package/dist/evals/trajectory.js +1 -1
- package/dist/evals/trajectory.service.d.ts +8 -3
- package/dist/evals/trajectory.service.js +26 -14
- package/dist/evals/worker.d.ts +5 -5
- package/dist/evals/worker.js +9 -3
- package/dist/extensions/admin-ops.d.ts +36 -10
- package/dist/extensions/admin-ops.js +67 -42
- package/dist/extensions/admin-view.d.ts +39 -10
- package/dist/extensions/admin-view.js +111 -20
- package/dist/extensions/blobs.js +1 -0
- package/dist/extensions/bootstrap.d.ts +17 -4
- package/dist/extensions/bootstrap.js +70 -9
- package/dist/extensions/cli.d.ts +12 -0
- package/dist/extensions/cli.js +11 -2
- package/dist/extensions/config-issues.d.ts +13 -1
- package/dist/extensions/config-issues.js +22 -5
- package/dist/extensions/config-json-schema.d.ts +43 -7
- package/dist/extensions/config-json-schema.js +124 -58
- package/dist/extensions/deployment-config-store.d.ts +6 -0
- package/dist/extensions/deployment-config.d.ts +78 -8
- package/dist/extensions/deployment-config.js +172 -34
- package/dist/extensions/display-name.d.ts +22 -0
- package/dist/extensions/display-name.js +77 -0
- package/dist/extensions/effective-config.d.ts +127 -0
- package/dist/extensions/effective-config.js +368 -0
- package/dist/extensions/extensions.module.js +14 -2
- package/dist/extensions/hooks.d.ts +37 -4
- package/dist/extensions/hooks.js +82 -16
- package/dist/extensions/host.d.ts +45 -2
- package/dist/extensions/host.js +112 -16
- package/dist/extensions/installs.d.ts +20 -8
- package/dist/extensions/installs.js +37 -15
- package/dist/extensions/org-config.d.ts +81 -6
- package/dist/extensions/org-config.js +195 -44
- package/dist/extensions/org-levels.d.ts +37 -0
- package/dist/extensions/org-levels.js +71 -0
- package/dist/extensions/org-overrides.d.ts +151 -0
- package/dist/extensions/org-overrides.js +524 -0
- package/dist/extensions/package.d.ts +14 -1
- package/dist/extensions/package.js +51 -2
- package/dist/extensions/permission-view.d.ts +19 -4
- package/dist/extensions/permission-view.js +4 -4
- package/dist/extensions/plugin-items.d.ts +21 -0
- package/dist/extensions/plugin-items.js +42 -0
- package/dist/extensions/point-defaults.controller.d.ts +34 -0
- package/dist/extensions/point-defaults.controller.js +71 -0
- package/dist/extensions/point-defaults.d.ts +71 -0
- package/dist/extensions/point-defaults.js +234 -0
- package/dist/extensions/registry.d.ts +1 -1
- package/dist/extensions/registry.js +2 -0
- package/dist/extensions/resolve.d.ts +5 -1
- package/dist/extensions/resolve.js +15 -2
- package/dist/files/previewers.d.ts +8 -1
- package/dist/files/previewers.js +35 -6
- package/dist/http/api.module.js +31 -3
- package/dist/http/app.d.ts +11 -2
- package/dist/http/app.js +31 -5
- package/dist/http/nest-logger.d.ts +6 -5
- package/dist/http/nest-logger.js +24 -10
- package/dist/identity/accounts.d.ts +29 -29
- package/dist/identity/accounts.js +47 -47
- package/dist/identity/audit.d.ts +2 -1
- package/dist/identity/auth.controller.js +9 -5
- package/dist/identity/auth.guard.d.ts +13 -5
- package/dist/identity/auth.guard.js +26 -9
- package/dist/identity/cookies.d.ts +20 -5
- package/dist/identity/cookies.js +44 -6
- package/dist/identity/identity.module.d.ts +1 -1
- package/dist/identity/identity.module.js +4 -4
- package/dist/identity/me.controller.d.ts +5 -1
- package/dist/identity/me.controller.js +12 -3
- package/dist/identity/principal.d.ts +3 -3
- package/dist/identity/principal.js +3 -3
- package/dist/identity/sessions.d.ts +12 -0
- package/dist/identity/sessions.js +45 -0
- package/dist/identity/settings.d.ts +22 -11
- package/dist/identity/settings.js +53 -18
- package/dist/identity/{enterprise-admin.d.ts → super-admin.d.ts} +5 -5
- package/dist/identity/{enterprise-admin.js → super-admin.js} +24 -24
- package/dist/inbox/inbox.module.d.ts +4 -1
- package/dist/inbox/inbox.module.js +7 -3
- package/dist/inbox/inbox.service.d.ts +62 -3
- package/dist/inbox/inbox.service.js +74 -4
- package/dist/inbox/org-notices.d.ts +63 -0
- package/dist/inbox/org-notices.js +91 -0
- package/dist/inbox/platform-inbox.controller.d.ts +23 -0
- package/dist/inbox/platform-inbox.controller.js +58 -0
- package/dist/inbox/platform-inbox.service.d.ts +151 -0
- package/dist/inbox/platform-inbox.service.js +263 -0
- package/dist/inbox/platform-notices.d.ts +19 -0
- package/dist/inbox/platform-notices.js +9 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +1 -1
- package/dist/infra/cli/kernel-cli.module.js +9 -1
- package/dist/infra/config/kernel-config.service.js +27 -1
- package/dist/infra/database/database.module.js +1 -0
- package/dist/infra/database/xact-lock.d.ts +1 -1
- package/dist/infra/database/xact-lock.js +1 -1
- package/dist/infra/events/kernel-events.d.ts +19 -2
- package/dist/infra/lifecycle/process-guards.js +4 -0
- package/dist/infra/logging/logging.module.d.ts +4 -2
- package/dist/infra/logging/logging.module.js +24 -6
- package/dist/infra/queue/add-job-once.d.ts +14 -0
- package/dist/infra/queue/add-job-once.js +16 -0
- package/dist/infra/queue/redis-connection.d.ts +6 -0
- package/dist/infra/queue/redis-connection.js +7 -0
- package/dist/infra/redis/redis.module.js +6 -2
- package/dist/infra/tokens.d.ts +6 -1
- package/dist/infra/tokens.js +6 -1
- package/dist/kernel.module.js +6 -0
- package/dist/lifecycle/kernel-lifecycle.d.ts +1 -1
- package/dist/lifecycle/kernel-lifecycle.js +4 -4
- package/dist/log.d.ts +13 -1
- package/dist/log.js +93 -17
- package/dist/models/prices.d.ts +5 -5
- package/dist/models/prices.js +4 -4
- package/dist/models/purposes.controller.d.ts +6 -4
- package/dist/models/purposes.controller.js +13 -7
- package/dist/models/purposes.d.ts +43 -9
- package/dist/models/purposes.js +48 -18
- package/dist/orgs/directory-sync.d.ts +14 -5
- package/dist/orgs/directory-sync.js +41 -17
- package/dist/orgs/offboarding.d.ts +12 -8
- package/dist/orgs/offboarding.js +54 -33
- package/dist/orgs/org-lock.d.ts +5 -0
- package/dist/orgs/org-lock.js +13 -0
- package/dist/orgs/org.guard.d.ts +10 -0
- package/dist/orgs/org.guard.js +43 -32
- package/dist/orgs/orgs.module.js +3 -0
- package/dist/orgs/orgs.service.d.ts +20 -7
- package/dist/orgs/orgs.service.js +32 -12
- package/dist/orgs/roles.d.ts +1 -1
- package/dist/orgs/roles.js +5 -1
- package/dist/orgs/settings.controller.d.ts +1 -1
- package/dist/orgs/settings.service.d.ts +20 -3
- package/dist/orgs/settings.service.js +19 -4
- package/dist/overrides.d.ts +3 -0
- package/dist/overrides.js +2 -0
- package/dist/platform/actor.d.ts +48 -0
- package/dist/platform/actor.js +100 -0
- package/dist/platform/agent-docs.d.ts +18 -0
- package/dist/platform/agent-docs.js +59 -0
- package/dist/platform/availability.d.ts +14 -0
- package/dist/platform/availability.js +43 -0
- package/dist/platform/errors.d.ts +11 -0
- package/dist/platform/errors.js +26 -0
- package/dist/platform/exclusions.d.ts +12 -0
- package/dist/platform/exclusions.js +1387 -0
- package/dist/platform/method.d.ts +6 -0
- package/dist/platform/method.js +15 -0
- package/dist/platform/operation.d.ts +92 -0
- package/dist/platform/operation.js +35 -0
- package/dist/platform/ops/agent-templates.ops.d.ts +14 -0
- package/dist/platform/ops/agent-templates.ops.js +92 -0
- package/dist/platform/ops/agents.ops.d.ts +19 -0
- package/dist/platform/ops/agents.ops.js +294 -0
- package/dist/platform/ops/connectors.ops.d.ts +15 -0
- package/dist/platform/ops/connectors.ops.js +126 -0
- package/dist/platform/ops/errors.d.ts +11 -0
- package/dist/platform/ops/errors.js +22 -0
- package/dist/platform/ops/extensions.ops.d.ts +19 -0
- package/dist/platform/ops/extensions.ops.js +116 -0
- package/dist/platform/ops/marketplaces.ops.d.ts +16 -0
- package/dist/platform/ops/marketplaces.ops.js +157 -0
- package/dist/platform/ops/paging.d.ts +14 -0
- package/dist/platform/ops/paging.js +20 -0
- package/dist/platform/ops/platform.ops.d.ts +23 -0
- package/dist/platform/ops/platform.ops.js +158 -0
- package/dist/platform/ops/plugins.ops.d.ts +33 -0
- package/dist/platform/ops/plugins.ops.js +377 -0
- package/dist/platform/ops/requests.ops.d.ts +14 -0
- package/dist/platform/ops/requests.ops.js +153 -0
- package/dist/platform/ops/role.d.ts +4 -0
- package/dist/platform/ops/role.js +10 -0
- package/dist/platform/ops/skills.ops.d.ts +13 -0
- package/dist/platform/ops/skills.ops.js +98 -0
- package/dist/platform/ops/summary.d.ts +10 -0
- package/dist/platform/ops/summary.js +18 -0
- package/dist/platform/platform-ops.module.d.ts +7 -0
- package/dist/platform/platform-ops.module.js +55 -0
- package/dist/platform/platform.module.d.ts +8 -0
- package/dist/platform/platform.module.js +25 -0
- package/dist/platform/redact-orgs.d.ts +7 -0
- package/dist/platform/redact-orgs.js +52 -0
- package/dist/platform/registry.d.ts +30 -0
- package/dist/platform/registry.js +111 -0
- package/dist/roles/worker-role.d.ts +6 -4
- package/dist/roles/worker-role.js +14 -7
- package/dist/roles/worker.module.d.ts +1 -1
- package/dist/roles/worker.module.js +15 -1
- package/dist/runtime/context/build.d.ts +11 -7
- package/dist/runtime/context/build.js +46 -19
- package/dist/runtime/context/claude-tools.d.ts +3 -1
- package/dist/runtime/context/claude-tools.js +8 -6
- package/dist/runtime/context/compaction.js +13 -0
- package/dist/runtime/context/reminders.d.ts +2 -0
- package/dist/runtime/context/reminders.js +2 -0
- package/dist/runtime/context/skills.js +9 -2
- package/dist/runtime/context/tool-changes.d.ts +27 -0
- package/dist/runtime/context/tool-changes.js +60 -0
- package/dist/runtime/hook-calls.js +2 -0
- package/dist/runtime/langgraph-loop.d.ts +3 -1
- package/dist/runtime/langgraph-loop.js +36 -17
- package/dist/runtime/loop.d.ts +34 -5
- package/dist/runtime/maintenance.d.ts +9 -4
- package/dist/runtime/maintenance.js +35 -8
- package/dist/runtime/model-call.d.ts +46 -8
- package/dist/runtime/model-call.js +72 -21
- package/dist/runtime/providers.d.ts +6 -0
- package/dist/runtime/providers.js +8 -0
- package/dist/runtime/run-control.d.ts +11 -1
- package/dist/runtime/run-control.js +12 -0
- package/dist/runtime/run-drafts.d.ts +85 -0
- package/dist/runtime/run-drafts.js +230 -0
- package/dist/runtime/run-jobs.d.ts +3 -5
- package/dist/runtime/run-jobs.js +3 -7
- package/dist/runtime/run-queue.d.ts +59 -2
- package/dist/runtime/run-queue.js +311 -38
- package/dist/runtime/run-worker.d.ts +11 -5
- package/dist/runtime/run-worker.js +69 -33
- package/dist/runtime/runtime.module.js +11 -3
- package/dist/runtime/step-log.d.ts +15 -4
- package/dist/runtime/step-log.js +24 -7
- package/dist/runtime/stream.controller.d.ts +23 -1
- package/dist/runtime/stream.controller.js +60 -3
- package/dist/runtime/stream.d.ts +4 -1
- package/dist/runtime/stream.js +27 -3
- package/dist/runtime/subagents/child.d.ts +55 -7
- package/dist/runtime/subagents/child.js +105 -7
- package/dist/runtime/subagents/definition.d.ts +12 -3
- package/dist/runtime/subagents/definition.js +44 -28
- package/dist/runtime/subagents/deliveries.d.ts +71 -0
- package/dist/runtime/subagents/deliveries.js +228 -0
- package/dist/runtime/subagents/dispatch.d.ts +115 -26
- package/dist/runtime/subagents/dispatch.js +212 -141
- package/dist/runtime/subagents/limits.d.ts +24 -5
- package/dist/runtime/subagents/limits.js +40 -6
- package/dist/runtime/subagents/messages.d.ts +95 -0
- package/dist/runtime/subagents/messages.js +328 -0
- package/dist/runtime/subagents/preamble.d.ts +4 -4
- package/dist/runtime/subagents/preamble.js +9 -7
- package/dist/runtime/subagents/progress.d.ts +8 -6
- package/dist/runtime/subagents/progress.js +11 -6
- package/dist/runtime/subagents/render.d.ts +30 -0
- package/dist/runtime/subagents/render.js +53 -0
- package/dist/runtime/subagents/roster.d.ts +24 -7
- package/dist/runtime/subagents/roster.js +47 -11
- package/dist/runtime/subagents/skill-fork.d.ts +3 -2
- package/dist/runtime/subagents/skill-fork.js +8 -4
- package/dist/runtime/subagents/status.d.ts +14 -0
- package/dist/runtime/subagents/status.js +23 -0
- package/dist/runtime/subagents/subagents.service.d.ts +7 -4
- package/dist/runtime/subagents/subagents.service.js +11 -6
- package/dist/runtime/usage-report.d.ts +2 -1
- package/dist/runtime/usage-report.js +19 -13
- package/dist/runtime/waits.d.ts +145 -36
- package/dist/runtime/waits.js +423 -109
- package/dist/sandbox/jobs.d.ts +40 -13
- package/dist/sandbox/jobs.js +57 -20
- package/dist/sandbox/leases.d.ts +37 -54
- package/dist/sandbox/leases.js +71 -115
- package/dist/sandbox/providers.d.ts +2 -2
- package/dist/sandbox/providers.js +1 -1
- package/dist/sandbox/sandbox.module.d.ts +3 -2
- package/dist/sandbox/sandbox.module.js +10 -4
- package/dist/sandbox/sandbox.service.d.ts +282 -129
- package/dist/sandbox/sandbox.service.js +736 -381
- package/dist/skills/catalog.d.ts +3 -2
- package/dist/skills/catalog.js +21 -43
- package/dist/skills/changes.controller.d.ts +7 -0
- package/dist/skills/changes.controller.js +19 -1
- package/dist/skills/invoke.js +8 -3
- package/dist/skills/package.d.ts +20 -1
- package/dist/skills/package.js +28 -1
- package/dist/skills/skills.controller.d.ts +39 -4
- package/dist/skills/skills.controller.js +88 -10
- package/dist/skills/skills.module.js +10 -1
- package/dist/skills/skills.service.d.ts +106 -26
- package/dist/skills/skills.service.js +421 -75
- package/dist/skills/usable.d.ts +81 -0
- package/dist/skills/usable.js +379 -0
- package/dist/tools/approvals.service.d.ts +20 -2
- package/dist/tools/approvals.service.js +25 -4
- package/dist/tools/builtin-names.d.ts +41 -7
- package/dist/tools/builtin-names.js +59 -7
- package/dist/tools/builtins/agent.d.ts +23 -21
- package/dist/tools/builtins/agent.js +62 -55
- package/dist/tools/builtins/builtin-tools.d.ts +19 -1
- package/dist/tools/builtins/builtin-tools.js +59 -2
- package/dist/tools/builtins/handback.d.ts +28 -0
- package/dist/tools/builtins/handback.js +53 -0
- package/dist/tools/builtins/platform.d.ts +34 -0
- package/dist/tools/builtins/platform.js +362 -0
- package/dist/tools/builtins/sandbox/bash.d.ts +1 -1
- package/dist/tools/builtins/sandbox/bash.js +241 -4
- package/dist/tools/builtins/sandbox/common.d.ts +69 -27
- package/dist/tools/builtins/sandbox/common.js +87 -48
- package/dist/tools/builtins/sandbox/deliver.js +1 -1
- package/dist/tools/builtins/sandbox/edit.js +2 -2
- package/dist/tools/builtins/sandbox/glob.js +2 -2
- package/dist/tools/builtins/sandbox/grep.js +2 -4
- package/dist/tools/builtins/sandbox/index.d.ts +3 -1
- package/dist/tools/builtins/sandbox/index.js +7 -1
- package/dist/tools/builtins/sandbox/read.js +1 -1
- package/dist/tools/builtins/sandbox/task-output.d.ts +74 -0
- package/dist/tools/builtins/sandbox/task-output.js +329 -0
- package/dist/tools/builtins/sandbox/task-stop.d.ts +10 -0
- package/dist/tools/builtins/sandbox/task-stop.js +113 -0
- package/dist/tools/builtins/sandbox/write.js +2 -2
- package/dist/tools/builtins/send-message.d.ts +56 -0
- package/dist/tools/builtins/send-message.js +100 -0
- package/dist/tools/builtins/skill-load.d.ts +22 -14
- package/dist/tools/builtins/skill-load.js +86 -57
- package/dist/tools/builtins/skill-write.js +1 -1
- package/dist/tools/builtins/web/errors.d.ts +8 -0
- package/dist/tools/builtins/web/errors.js +53 -0
- package/dist/tools/builtins/web/extract.d.ts +62 -0
- package/dist/tools/builtins/web/extract.js +271 -0
- package/dist/tools/builtins/web/fetch.d.ts +55 -0
- package/dist/tools/builtins/web/fetch.js +175 -0
- package/dist/tools/builtins/web/index.d.ts +13 -0
- package/dist/tools/builtins/web/index.js +12 -0
- package/dist/tools/builtins/web/page-cache.d.ts +25 -0
- package/dist/tools/builtins/web/page-cache.js +77 -0
- package/dist/tools/builtins/web/search.d.ts +25 -0
- package/dist/tools/builtins/web/search.js +210 -0
- package/dist/tools/builtins/web/url.d.ts +17 -0
- package/dist/tools/builtins/web/url.js +43 -0
- package/dist/tools/connectors.controller.d.ts +10 -0
- package/dist/tools/connectors.controller.js +12 -0
- package/dist/tools/connectors.service.d.ts +46 -5
- package/dist/tools/connectors.service.js +94 -15
- package/dist/tools/extension-tools.controller.d.ts +14 -0
- package/dist/tools/extension-tools.controller.js +37 -0
- package/dist/tools/info.d.ts +42 -1
- package/dist/tools/info.js +64 -1
- package/dist/tools/oauth/flow-paths.d.ts +1 -1
- package/dist/tools/oauth/flow-paths.js +1 -1
- package/dist/tools/org-tool-switches.d.ts +74 -0
- package/dist/tools/org-tool-switches.js +184 -0
- package/dist/tools/pipeline.d.ts +46 -40
- package/dist/tools/pipeline.js +202 -364
- package/dist/tools/policy.d.ts +1 -1
- package/dist/tools/policy.js +36 -16
- package/dist/tools/registry.d.ts +120 -8
- package/dist/tools/registry.js +112 -19
- package/dist/tools/tools.module.js +24 -2
- package/dist/tools/validate.js +1 -1
- package/dist/ui/views.d.ts +2 -1
- package/dist/ui/views.js +17 -3
- package/package.json +3 -3
- package/src/migrations/0064_account_last_org.sql +7 -0
- package/src/migrations/0065_conversation_archive.sql +8 -0
- package/src/migrations/0066_extension_display_name.sql +4 -0
- package/src/migrations/0067_agent_profiles.sql +20 -0
- package/src/migrations/0068_agent_avatar_uploads.sql +7 -0
- package/src/migrations/0069_attachments_agent_idx.sql +3 -0
- package/src/migrations/0070_background_tasks.sql +37 -0
- package/src/migrations/0071_background_tasks_reserve.sql +6 -0
- package/src/migrations/0072_background_tasks_stopping.sql +5 -0
- package/src/migrations/0073_org_extension_point_defaults.sql +11 -0
- package/src/migrations/0074_subagent_tree.sql +50 -0
- package/src/migrations/0075_plugin_scope.sql +61 -0
- package/src/migrations/0076_skills_created_by_nullable.sql +6 -0
- package/src/migrations/0077_scope_requests.sql +52 -0
- package/src/migrations/0078_scope_grant_switches.sql +18 -0
- package/src/migrations/0079_scope_live_install_switches.sql +19 -0
- package/src/migrations/0080_rename_enterprise_admin.sql +18 -0
- package/src/migrations/0081_platform_components.sql +129 -0
- package/src/migrations/0082_platform_migration.sql +33 -0
- package/src/migrations/0083_plugin_scope_sync_failures.sql +15 -0
- package/src/migrations/0084_org_owned_items.sql +85 -0
- package/src/migrations/0085_agent_templates_phase4.sql +45 -0
- package/src/migrations/0086_org_layering.sql +30 -0
- package/src/migrations/0087_approval_summary.sql +6 -0
- package/src/migrations/0088_environments.sql +80 -0
- package/dist/catalog/upgrade-targets.d.ts +0 -56
- package/dist/catalog/upgrade-targets.js +0 -90
- package/dist/conversations/sandbox-status.d.ts +0 -24
- package/dist/conversations/sandbox-status.js +0 -52
- package/dist/extensions/org-config.controller.d.ts +0 -24
- package/dist/extensions/org-config.controller.js +0 -62
|
@@ -8,9 +8,10 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
|
8
8
|
return function (target, key) { decorator(target, key, paramIndex); }
|
|
9
9
|
};
|
|
10
10
|
import { randomUUID } from "node:crypto";
|
|
11
|
-
import { isSandboxGoneError, } from "@capybara-hq/plugin-sdk";
|
|
11
|
+
import { isFileSystemGoneError, isSandboxGoneError, } from "@capybara-hq/plugin-sdk";
|
|
12
12
|
import { Inject, Injectable } from "@nestjs/common";
|
|
13
13
|
import { sql, } from "kysely";
|
|
14
|
+
import { BackgroundTaskStore } from "../background/store.js";
|
|
14
15
|
import { DEFAULT_MAX_COMMAND_SECONDS, DEFAULT_MAX_RUN_SECONDS, } from "../config/schema.js";
|
|
15
16
|
import { ApiError } from "../errors.js";
|
|
16
17
|
import { readSandboxLimitOverride } from "../identity/settings.js";
|
|
@@ -22,22 +23,32 @@ import { canonicalJson } from "../tools/canonical-json.js";
|
|
|
22
23
|
import { SandboxLeases, SandboxLockLostError, sandboxBusy, } from "./leases.js";
|
|
23
24
|
import { SandboxProviders } from "./providers.js";
|
|
24
25
|
// Moved to leases.ts (phase C); re-exported for existing importers
|
|
25
|
-
export {
|
|
26
|
+
export { SANDBOX_LOCK_PREFIX, SANDBOX_LOCK_TTL_MS, SandboxLockLostError, sandboxBusy, } from "./leases.js";
|
|
26
27
|
/** Margin (ms) of a holder record's TTL beyond the longest command: by then the in-sandbox limit has ended its commands. */
|
|
27
28
|
const HOLDER_TTL_MARGIN_MS = 10 * 60_000;
|
|
28
29
|
/** How long the kernel's own read-modify-write of one file waits for another holder's (ms, spec §10.4). */
|
|
29
30
|
const PATH_LEASE_WAIT_MS = 30_000;
|
|
30
|
-
/**
|
|
31
|
+
/** Time limit of one provider call in maintenance (suspend, destroy). */
|
|
31
32
|
const SWEEP_OP_TIMEOUT_MS = 120_000;
|
|
32
33
|
/** How long the suspension sweep holds the intent on a sandbox past `maxRunningMinutes` (ms). */
|
|
33
34
|
const SWEEP_INTENT_WAIT_MS = 1000;
|
|
34
|
-
/**
|
|
35
|
+
/** Rows one sweep handles at most, and the provider calls it runs at once. */
|
|
35
36
|
const SWEEP_BATCH = 200;
|
|
36
37
|
const SWEEP_CONCURRENCY = 4;
|
|
37
|
-
/**
|
|
38
|
-
const
|
|
39
|
-
/**
|
|
38
|
+
/** How long `destroyEnvironment` / `resetFileSystem` wait for the exclusive lease (running commands). */
|
|
39
|
+
const EXCLUSIVE_WAIT_MS = 10 * 60_000;
|
|
40
|
+
/** The time limit of the exec that kills lost background tasks after a resume (ms). */
|
|
41
|
+
const PENDING_KILL_TIMEOUT_MS = 30_000;
|
|
42
|
+
/** A background task's session id as `echo $!` printed it; anything else is never put in a command. */
|
|
43
|
+
const SID_RE = /^[1-9][0-9]{0,9}$/;
|
|
44
|
+
/** The signal of one provider call in maintenance: aborted on timeout or when the lease is lost. */
|
|
40
45
|
const opSignal = (lease) => AbortSignal.any([lease.signal, AbortSignal.timeout(SWEEP_OP_TIMEOUT_MS)]);
|
|
46
|
+
/** The provider-side name of an environment's file system and instance. */
|
|
47
|
+
export const environmentResourceName = (environmentId) => `cap-env-${environmentId}`;
|
|
48
|
+
/** The environment is missing, deleted, or not in the caller's org (404; ids cannot be probed). */
|
|
49
|
+
export const environmentNotFound = () => new ApiError("environment.not_found", "This environment does not exist or was deleted.");
|
|
50
|
+
/** The environment's file system is gone at the provider (409); never replaced silently. */
|
|
51
|
+
export const environmentFilesLost = () => new ApiError("environment.files_lost", "The environment's files are gone; reset its files from the environments page to start with an empty workspace.");
|
|
41
52
|
/**
|
|
42
53
|
* The row still records the instance `gone` (same handle and same `created_at`). Handles compare
|
|
43
54
|
* canonically: a jsonb round trip reorders a multi-key handle's keys (final review m9).
|
|
@@ -47,42 +58,31 @@ export const isInstance = (row, gone) => gone !== null &&
|
|
|
47
58
|
canonicalJson(row.handle) === canonicalJson(gone.handle) &&
|
|
48
59
|
row.created_at.getTime() === gone.createdAt;
|
|
49
60
|
/**
|
|
50
|
-
*
|
|
51
|
-
*
|
|
61
|
+
* `impl` with every `exec` labelled with the holder's tag (and `actor` when given); `before` runs
|
|
62
|
+
* first (it refreshes the holder's record, so a callback that runs several long commands in a row
|
|
63
|
+
* stays recorded).
|
|
52
64
|
*/
|
|
53
|
-
|
|
54
|
-
return impl.concurrentExec === true && typeof impl.killTagged === "function" ? max : 1;
|
|
55
|
-
}
|
|
56
|
-
/**
|
|
57
|
-
* `impl` with every `exec` labelled with the holder's tag; `before` runs first (it refreshes the
|
|
58
|
-
* holder's record, so a callback that runs several long commands in a row stays recorded).
|
|
59
|
-
*/
|
|
60
|
-
function tagged(impl, tag, before) {
|
|
65
|
+
function tagged(impl, tag, actor, before) {
|
|
61
66
|
const wrapped = Object.create(impl);
|
|
62
67
|
Object.defineProperty(wrapped, "exec", {
|
|
63
68
|
value: async (h, args) => {
|
|
64
69
|
await before();
|
|
65
|
-
return impl.exec(h, { ...args, tag });
|
|
70
|
+
return impl.exec(h, { ...args, tag, ...(actor !== undefined && { actor }) });
|
|
66
71
|
},
|
|
67
72
|
});
|
|
68
73
|
return wrapped;
|
|
69
74
|
}
|
|
70
75
|
/**
|
|
71
|
-
*
|
|
72
|
-
* - `runtime.maxRunSeconds`
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
76
|
+
* Sandbox config problems to tell operators at startup (English log lines):
|
|
77
|
+
* - `runtime.maxRunSeconds` below `sandbox.limits.maxCommandSeconds`: long commands are cut by the
|
|
78
|
+
* run time limit (Ruling M20);
|
|
79
|
+
* - `sandbox.limits.egress` is open but the current provider plugin has no `allowOpenEgress`: every
|
|
80
|
+
* instance creation is refused.
|
|
81
|
+
* With both time limits at their defaults (900 < 7200, a known trade-off) it is only info; warn
|
|
82
|
+
* when an operator changed one and they still disagree.
|
|
77
83
|
*/
|
|
78
|
-
export function sandboxConfigWarnings(cfg, providerPlugin
|
|
84
|
+
export function sandboxConfigWarnings(cfg, providerPlugin) {
|
|
79
85
|
const out = [];
|
|
80
|
-
if (impl && (impl.concurrentExec === true) !== (typeof impl.killTagged === "function")) {
|
|
81
|
-
out.push({
|
|
82
|
-
level: "warn",
|
|
83
|
-
message: `the sandbox plugin ${providerPlugin ?? ""} declares only one of concurrentExec and killTagged; concurrent commands need both, so its sandboxes run one command at a time`,
|
|
84
|
-
});
|
|
85
|
-
}
|
|
86
86
|
if (cfg.sandbox.limits.egress === "open" && providerPlugin !== undefined) {
|
|
87
87
|
const pc = cfg.extensionConfig[providerPlugin];
|
|
88
88
|
if (pc?.allowOpenEgress !== true) {
|
|
@@ -103,8 +103,13 @@ export function sandboxConfigWarnings(cfg, providerPlugin, impl) {
|
|
|
103
103
|
}
|
|
104
104
|
return out;
|
|
105
105
|
}
|
|
106
|
-
/**
|
|
106
|
+
/** No sandbox provider (503). */
|
|
107
107
|
export const sandboxUnavailable = () => new ApiError("sandbox.unavailable", "Sandbox is not available in this deployment; install a sandbox plugin");
|
|
108
|
+
/**
|
|
109
|
+
* The environment's files are kept by a provider that is not the current one (503); its files are
|
|
110
|
+
* not migrated, so it stays unusable until that provider is back (spec §4.2 item 13).
|
|
111
|
+
*/
|
|
112
|
+
const otherProvider = (provider) => new ApiError("environment.unavailable", `This environment's files are kept by the sandbox provider ${provider}, which is not the current provider; it can be used again once ${provider} is the deployment's sandbox provider`);
|
|
108
113
|
const abortReason = (signal) => signal.reason ?? new DOMException("This operation was aborted", "AbortError");
|
|
109
114
|
function sleep(ms, signal) {
|
|
110
115
|
return new Promise((resolve, reject) => {
|
|
@@ -123,7 +128,7 @@ function sleep(ms, signal) {
|
|
|
123
128
|
signal.addEventListener("abort", onAbort, { once: true });
|
|
124
129
|
});
|
|
125
130
|
}
|
|
126
|
-
/**
|
|
131
|
+
/** Runs `fn` on `items`, `limit` at a time. */
|
|
127
132
|
async function inPool(items, limit, fn) {
|
|
128
133
|
let next = 0;
|
|
129
134
|
const worker = async () => {
|
|
@@ -135,43 +140,56 @@ async function inPool(items, limit, fn) {
|
|
|
135
140
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
|
136
141
|
}
|
|
137
142
|
/**
|
|
138
|
-
*
|
|
143
|
+
* The kernel's sandbox service over the `sandbox@2` provider, keyed by **environment** (spec
|
|
144
|
+
* 2026-11-07-environments-design §4.2, §5.1): one file system kept until the environment is
|
|
145
|
+
* deleted, plus one instance attached to it that comes and goes; all conversations bound to the
|
|
146
|
+
* environment (and their sub-agents) share both.
|
|
139
147
|
*
|
|
140
|
-
* -
|
|
141
|
-
*
|
|
142
|
-
*
|
|
143
|
-
*
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
*
|
|
155
|
-
*
|
|
156
|
-
*
|
|
157
|
-
*
|
|
158
|
-
*
|
|
159
|
-
*
|
|
160
|
-
*
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
-
*
|
|
164
|
-
* `
|
|
165
|
-
*
|
|
166
|
-
*
|
|
167
|
-
*
|
|
168
|
-
*
|
|
148
|
+
* - **Lazy creation**: the first {@link withSandbox} creates the file system
|
|
149
|
+
* (`createFileSystem`, name `cap-env-<environmentId>`) and then the instance on it; state lives
|
|
150
|
+
* in `environment_instances`. A file system gone at the provider is never replaced silently:
|
|
151
|
+
* `fs_status` becomes `lost` and uses fail with `environment.files_lost` until
|
|
152
|
+
* {@link resetFileSystem}.
|
|
153
|
+
* - **Leases** (phase C, spec §10, {@link SandboxLeases}), keyed by environment id: using an
|
|
154
|
+
* instance recorded `running` with the current provider takes a **shared** lease, so up to
|
|
155
|
+
* `sandbox.maxConcurrentCommands` holders (all sessions of the environment) run at once. Every
|
|
156
|
+
* lifecycle change (create, resume, rebuild, suspend, destroy, reset) takes the **exclusive**
|
|
157
|
+
* lease, alone; a caller that needs one on the use path sets the intent so new shared leases wait
|
|
158
|
+
* for it, and downgrades to a shared lease to run its own command once the instance runs.
|
|
159
|
+
* Waiting is capped at `sandbox.lockWaitSeconds` (`sandbox.busy`); two replicas using it for the
|
|
160
|
+
* first time at once still build only one.
|
|
161
|
+
* - **Fencing**: exclusive holders read `generation` and compare-and-increment it with every state
|
|
162
|
+
* write ({@link SandboxLockLostError}); every acquisition (shared ones too) bumps it first, so a
|
|
163
|
+
* stale exclusive holder's later writes fail. Shared holders write nothing else but
|
|
164
|
+
* `last_used_at` (monotonic). A lost lease aborts the signal passed to `fn` and to maintenance
|
|
165
|
+
* provider calls.
|
|
166
|
+
* - **Suspension and destruction**: the sweeps (`SandboxSweepJobs`, crons) call
|
|
167
|
+
* {@link suspendIdle} / {@link destroyInactive}; they try the exclusive lease once and skip an
|
|
168
|
+
* environment with any holder; one past `maxRunningMinutes` gets the intent for a short window
|
|
169
|
+
* (about a second) so the suspension can get in when its commands end. Inactivity destroys only
|
|
170
|
+
* the instance (files kept); a deleted environment or org loses its files too.
|
|
171
|
+
* - **Transparent recovery**: a suspended instance is resumed on next use; when `resume` returns
|
|
172
|
+
* `gone`, the instance was destroyed, or `fn` throws `SandboxGoneError` (retried once), a new
|
|
173
|
+
* instance is created on the kept files and `restarted` is true. Several holders hitting `gone`
|
|
174
|
+
* together rebuild once: the others find the new instance.
|
|
175
|
+
* - **Takeover** (spec §10.5; environments spec §4.2): every holder records itself in
|
|
176
|
+
* `capk:sandbox-holders:<env>` (again before each `exec`) and its commands carry its lease token
|
|
177
|
+
* as `tag`. An acquirer that finds a record without a live lease (a crashed holder) kills only
|
|
178
|
+
* that holder's commands (`killTagged`): other conversations of the environment may be running
|
|
179
|
+
* commands. `killAll` runs only under the exclusive lease, before the downgrade, on an instance
|
|
180
|
+
* this holder just resumed from `suspended` (nobody else can hold it; a provider that keeps
|
|
181
|
+
* memory across a pause brings old commands back); it also covers crashed holders' records,
|
|
182
|
+
* which it claims. A freshly created instance has nothing to kill. Failures are only warned (the
|
|
183
|
+
* in-sandbox time limit is the backstop).
|
|
169
184
|
* - **Instance identity**: a holder that hits `gone` names the instance by its handle and the
|
|
170
185
|
* row's `created_at` (set by the fenced write after each `create`); the handle alone is the same
|
|
171
186
|
* after a rebuild on providers whose handle is the instance name (sandbox-docker).
|
|
172
187
|
*
|
|
173
|
-
*
|
|
174
|
-
*
|
|
188
|
+
* Files kept by another provider than the current one make the environment unusable
|
|
189
|
+
* (`environment.unavailable`) until that provider is back; deleting it still works while that
|
|
190
|
+
* provider is loaded ({@link SandboxProviders.lookup}). Without a provider {@link available} is false and
|
|
191
|
+
* uses throw `sandbox.unavailable` (503). No in-process state: replicas share it through PG and
|
|
192
|
+
* Redis.
|
|
175
193
|
*/
|
|
176
194
|
let SandboxService = class SandboxService {
|
|
177
195
|
db;
|
|
@@ -180,39 +198,62 @@ let SandboxService = class SandboxService {
|
|
|
180
198
|
providers;
|
|
181
199
|
config;
|
|
182
200
|
log;
|
|
201
|
+
background;
|
|
183
202
|
constructor(db,
|
|
184
203
|
/** The record of the instance each holder last used (`capk:sandbox-seen:…`). */
|
|
185
204
|
redis, leases,
|
|
186
|
-
/** The `sandbox@
|
|
187
|
-
providers, config, log
|
|
205
|
+
/** The `sandbox@2` provider (read at every call: providers are ready after the extension load). */
|
|
206
|
+
providers, config, log,
|
|
207
|
+
/** Background tasks lost while their process may live on (killed after a resume). */
|
|
208
|
+
background) {
|
|
188
209
|
this.db = db;
|
|
189
210
|
this.redis = redis;
|
|
190
211
|
this.leases = leases;
|
|
191
212
|
this.providers = providers;
|
|
192
213
|
this.config = config;
|
|
193
214
|
this.log = log;
|
|
215
|
+
this.background = background;
|
|
194
216
|
}
|
|
195
217
|
/** Poll interval while waiting for a lease (the leases' own). */
|
|
196
218
|
get pollMs() {
|
|
197
219
|
return this.leases.pollMs;
|
|
198
220
|
}
|
|
199
|
-
/**
|
|
221
|
+
/** Whether there is a sandbox provider. */
|
|
200
222
|
get available() {
|
|
201
223
|
return this.providers.current() !== undefined;
|
|
202
224
|
}
|
|
203
|
-
/** Concurrent holders
|
|
204
|
-
|
|
205
|
-
return
|
|
225
|
+
/** Concurrent holders of one environment (`sandbox.maxConcurrentCommands`, read at every use). */
|
|
226
|
+
get cap() {
|
|
227
|
+
return this.config.get().sandbox.maxConcurrentCommands;
|
|
228
|
+
}
|
|
229
|
+
/**
|
|
230
|
+
* The org's effective sandbox limits (spec §8.4): the deployment's `sandbox.limits`, egress
|
|
231
|
+
* narrowed by the org setting `sandboxEgress` (Ruling M19), cpu / memory per the super admin's
|
|
232
|
+
* org override (C19 / Ruling M16). Instances are created with them; phase 1 environments use
|
|
233
|
+
* them unchanged.
|
|
234
|
+
*/
|
|
235
|
+
async orgLimits(orgId) {
|
|
236
|
+
const cfg = this.config.get().sandbox;
|
|
237
|
+
const egress = effectiveSandboxEgress(cfg.limits, (await readOrgSettings(this.db.db, orgId)).sandboxEgress);
|
|
238
|
+
const sized = effectiveSandboxLimits({ ...cfg.limits, ...egress }, await readSandboxLimitOverride(this.db.db, orgId));
|
|
239
|
+
return {
|
|
240
|
+
cpu: sized.cpu,
|
|
241
|
+
memoryMb: sized.memoryMb,
|
|
242
|
+
diskMb: sized.diskMb,
|
|
243
|
+
egress: sized.egress,
|
|
244
|
+
allowHosts: sized.allowHosts,
|
|
245
|
+
};
|
|
206
246
|
}
|
|
207
247
|
/**
|
|
208
|
-
*
|
|
209
|
-
*
|
|
248
|
+
* The environment's instance state as shown to people (read only, no lease): `running` /
|
|
249
|
+
* `suspended` and the last use. No instance, a destroyed one, or one stuck in `creating` (being
|
|
250
|
+
* created, or a failed create the next use retries) is null.
|
|
210
251
|
*/
|
|
211
|
-
async status(orgId,
|
|
252
|
+
async status(orgId, environmentId) {
|
|
212
253
|
const row = await this.db
|
|
213
|
-
.selectFrom("
|
|
254
|
+
.selectFrom("environment_instances")
|
|
214
255
|
.select(["status", "last_used_at"])
|
|
215
|
-
.where("
|
|
256
|
+
.where("environment_id", "=", environmentId)
|
|
216
257
|
.where("org_id", "=", orgId)
|
|
217
258
|
.executeTakeFirst();
|
|
218
259
|
if (!row || (row.status !== "running" && row.status !== "suspended"))
|
|
@@ -220,90 +261,170 @@ let SandboxService = class SandboxService {
|
|
|
220
261
|
return { status: row.status, lastUsedAt: row.last_used_at };
|
|
221
262
|
}
|
|
222
263
|
/**
|
|
223
|
-
*
|
|
224
|
-
*
|
|
225
|
-
*
|
|
264
|
+
* The instance the environment records now (its `created_at`), while it has one that runs or is
|
|
265
|
+
* suspended; null otherwise. Read without a lease: a caller inside `withSandbox` reads the
|
|
266
|
+
* instance its own lease keeps.
|
|
267
|
+
*/
|
|
268
|
+
async currentInstance(environmentId) {
|
|
269
|
+
const row = await this.db
|
|
270
|
+
.selectFrom("environment_instances")
|
|
271
|
+
.select(["status", "handle", "created_at"])
|
|
272
|
+
.where("environment_id", "=", environmentId)
|
|
273
|
+
.executeTakeFirst();
|
|
274
|
+
if (!row || row.handle === null)
|
|
275
|
+
return null;
|
|
276
|
+
if (row.status !== "running" && row.status !== "suspended")
|
|
277
|
+
return null;
|
|
278
|
+
return { createdAt: row.created_at };
|
|
279
|
+
}
|
|
280
|
+
/**
|
|
281
|
+
* Runs `fn` on the environment's instance only while it runs the instance created at
|
|
282
|
+
* `instance.createdAt` (background tasks, spec 2026-11-05-background-bash-design §4.3). It never
|
|
283
|
+
* creates, resumes or rebuilds one: an instance that is missing, suspended, being created or
|
|
284
|
+
* destroyed (or recorded with another provider) is `not_running`, one that records another
|
|
285
|
+
* instance is `other_instance`. Otherwise `fn` runs under a shared lease, as a `withSandbox`
|
|
286
|
+
* holder does (its tag on every exec, crashed holders' commands killed), and `last_used_at`
|
|
287
|
+
* moves forward when it ends. Waits for the lease up to `sandbox.lockWaitSeconds`
|
|
288
|
+
* (`sandbox.busy`); `signal` (optional) ends the wait with its abort reason. The signal passed
|
|
289
|
+
* to `fn` aborts with `signal` or when the lease is lost. Errors of `fn` (`SandboxGoneError`
|
|
290
|
+
* too) are thrown as they are.
|
|
291
|
+
*/
|
|
292
|
+
async ifRunning(environmentId, instance, fn, signal = new AbortController().signal) {
|
|
293
|
+
const p = this.require();
|
|
294
|
+
const env = environmentId;
|
|
295
|
+
const deadline = Date.now() + this.config.get().sandbox.lockWaitSeconds * 1000;
|
|
296
|
+
const token = randomUUID();
|
|
297
|
+
for (;;) {
|
|
298
|
+
if (signal.aborted)
|
|
299
|
+
throw abortReason(signal);
|
|
300
|
+
const row = await this.db
|
|
301
|
+
.selectFrom("environment_instances")
|
|
302
|
+
.selectAll()
|
|
303
|
+
.where("environment_id", "=", env)
|
|
304
|
+
.executeTakeFirst();
|
|
305
|
+
if (!row)
|
|
306
|
+
return { kind: "not_running", status: "gone" };
|
|
307
|
+
if (row.created_at.getTime() !== instance.createdAt.getTime()) {
|
|
308
|
+
return { kind: "other_instance" };
|
|
309
|
+
}
|
|
310
|
+
if (!this.usable(row, p, null))
|
|
311
|
+
return { kind: "not_running", status: row.status };
|
|
312
|
+
// the instance this check saw: its handle plus `created_at` (never `generation`, which
|
|
313
|
+
// every lease bumps)
|
|
314
|
+
const want = { handle: row.handle, createdAt: row.created_at.getTime() };
|
|
315
|
+
const lease = await this.leases.tryShared(env, this.cap, token);
|
|
316
|
+
if (lease) {
|
|
317
|
+
const e = await this.asShared({ orgId: row.org_id, environmentId: env }, p, lease, (r) => this.usable(r, p, null) && isInstance(r, want), signal);
|
|
318
|
+
if (e) {
|
|
319
|
+
try {
|
|
320
|
+
const impl = tagged(p.impl, lease.token, undefined, () => this.register(env, lease.token));
|
|
321
|
+
const sig = AbortSignal.any([signal, lease.signal]);
|
|
322
|
+
return { kind: "ran", value: await fn(impl, e.handle, sig) };
|
|
323
|
+
}
|
|
324
|
+
finally {
|
|
325
|
+
await this.leave(env, e);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
// the row changed between the check and the lease: classify it again
|
|
329
|
+
continue;
|
|
330
|
+
}
|
|
331
|
+
if (Date.now() + this.pollMs > deadline)
|
|
332
|
+
throw sandboxBusy();
|
|
333
|
+
await sleep(this.pollMs + Math.floor(Math.random() * this.pollMs), signal);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* A short lease on one path of the environment around the kernel's own read-modify-write of
|
|
338
|
+
* that file (spec §10.4); waits up to 30 s, then throws `SandboxPathBusyError`. Its `signal`
|
|
339
|
+
* aborts when a renewal finds it lost.
|
|
226
340
|
*/
|
|
227
|
-
pathLease(
|
|
228
|
-
return this.leases.pathLease(
|
|
341
|
+
pathLease(environmentId, path, signal) {
|
|
342
|
+
return this.leases.pathLease(environmentId, path, signal, PATH_LEASE_WAIT_MS);
|
|
229
343
|
}
|
|
230
344
|
/**
|
|
231
|
-
*
|
|
232
|
-
*
|
|
233
|
-
* `last_used_at` → release.
|
|
234
|
-
* -
|
|
235
|
-
*
|
|
236
|
-
* - The
|
|
237
|
-
*
|
|
238
|
-
*
|
|
239
|
-
* - `
|
|
240
|
-
*
|
|
241
|
-
*
|
|
242
|
-
*
|
|
345
|
+
* Runs `fn` on the environment's instance: take a lease → make sure the instance runs (file
|
|
346
|
+
* system, create / resume / rebuild, under the exclusive lease) → `fn` under a shared lease →
|
|
347
|
+
* update `last_used_at` → release.
|
|
348
|
+
* - Waiting longer than `sandbox.lockWaitSeconds` throws `sandbox.busy` (409); `signal` aborts
|
|
349
|
+
* the wait and the creation / resume.
|
|
350
|
+
* - The fourth argument of `fn` combines `signal` with the lease's: use it for commands, so they
|
|
351
|
+
* stop when the lease is lost.
|
|
352
|
+
* - The provider passed to `fn` labels every `exec` with this holder's tag and `ctx.actor`.
|
|
353
|
+
* - When `fn` throws `SandboxGoneError` (the instance is gone) the instance is rebuilt on the
|
|
354
|
+
* kept files (or the one another holder rebuilt is taken) and `fn` runs once more with
|
|
355
|
+
* `restarted: true`; a second `gone` is thrown. Other errors are thrown as they are.
|
|
356
|
+
* - Lost files throw `environment.files_lost` (409).
|
|
357
|
+
* - When `ctx.holderConversationId` last used another instance (another holder rebuilt it
|
|
358
|
+
* meanwhile), `info.restarted` is true, so it learns its processes are gone (final review m5).
|
|
243
359
|
*/
|
|
244
360
|
async withSandbox(ctx, fn, signal) {
|
|
245
361
|
const p = this.require();
|
|
246
|
-
const
|
|
247
|
-
|
|
362
|
+
const env = ctx.environmentId;
|
|
363
|
+
const holder = ctx.holderConversationId;
|
|
364
|
+
let e = await this.seenBy(env, holder, await this.enter(ctx, p, signal, null));
|
|
248
365
|
try {
|
|
249
366
|
try {
|
|
250
|
-
return await this.runAs(e, p, fn, signal);
|
|
367
|
+
return await this.runAs(e, ctx, p, fn, signal);
|
|
251
368
|
}
|
|
252
369
|
catch (err) {
|
|
253
370
|
if (!isSandboxGoneError(err))
|
|
254
371
|
throw err;
|
|
255
|
-
this.log.info("sandbox instance is gone during use; recreating", {
|
|
256
|
-
conversationId: ctx.conversationId,
|
|
257
|
-
});
|
|
372
|
+
this.log.info("sandbox instance is gone during use; recreating", { environmentId: env });
|
|
258
373
|
const gone = { handle: e.handle, createdAt: e.createdAt };
|
|
259
374
|
const prev = e;
|
|
260
|
-
await this.leave(
|
|
261
|
-
e = await this.seenBy(
|
|
262
|
-
return await this.runAs({ ...e, info: { fresh: true,
|
|
375
|
+
await this.leave(env, prev);
|
|
376
|
+
e = await this.seenBy(env, holder, await this.enter(ctx, p, signal, gone));
|
|
377
|
+
return await this.runAs({ ...e, info: { fresh: true, restarted: true } }, ctx, p, fn, signal);
|
|
263
378
|
}
|
|
264
379
|
}
|
|
265
380
|
finally {
|
|
266
|
-
await this.leave(
|
|
381
|
+
await this.leave(env, e);
|
|
267
382
|
}
|
|
268
383
|
}
|
|
269
384
|
/**
|
|
270
|
-
* Records that `holder` uses the instance of `e` (`capk:sandbox-seen:<
|
|
271
|
-
*
|
|
272
|
-
*
|
|
385
|
+
* Records that `holder` uses the instance of `e` (`capk:sandbox-seen:<env>:<holder>` =
|
|
386
|
+
* `<created_at>:<file system id>`, with a TTL of the inactivity destruction plus a day). When the
|
|
387
|
+
* holder last used another instance on the same file system `e` is marked restarted; on another
|
|
388
|
+
* file system (the files were reset) it is not. Best effort: a Redis error only logs.
|
|
273
389
|
*/
|
|
274
|
-
async seenBy(
|
|
390
|
+
async seenBy(environmentId, holder, e) {
|
|
275
391
|
try {
|
|
276
392
|
const ttl = (this.config.get().sandbox.limits.destroyAfterInactiveDays + 1) * 86_400;
|
|
277
|
-
const
|
|
278
|
-
|
|
279
|
-
|
|
393
|
+
const seen = `${e.createdAt}:${e.fsId}`;
|
|
394
|
+
const prev = await this.redis.set(`capk:sandbox-seen:${environmentId}:${holder}`, seen, "EX", ttl, "GET");
|
|
395
|
+
if (prev !== null && prev !== seen) {
|
|
396
|
+
// `<createdAt>:<fsId>`: another file system (a reset) means a fresh start, not a restart on
|
|
397
|
+
// kept files; another instance on the same file system is a restart
|
|
398
|
+
const sameFiles = prev.slice(prev.indexOf(":") + 1) === e.fsId;
|
|
399
|
+
if (sameFiles !== e.info.restarted) {
|
|
400
|
+
return { ...e, info: { fresh: e.info.fresh, restarted: sameFiles } };
|
|
401
|
+
}
|
|
280
402
|
}
|
|
281
403
|
}
|
|
282
404
|
catch (err) {
|
|
283
405
|
this.log.warn("cannot record the sandbox instance a holder uses", {
|
|
284
|
-
|
|
406
|
+
environmentId,
|
|
285
407
|
holder,
|
|
286
408
|
err,
|
|
287
409
|
});
|
|
288
410
|
}
|
|
289
411
|
return e;
|
|
290
412
|
}
|
|
291
|
-
/** Runs `fn` as the holder `e`: its tag on every exec, its lease's signal. */
|
|
292
|
-
runAs(e, p, fn, signal) {
|
|
413
|
+
/** Runs `fn` as the holder `e`: its tag and the caller's actor on every exec, its lease's signal. */
|
|
414
|
+
runAs(e, ctx, p, fn, signal) {
|
|
293
415
|
const sig = AbortSignal.any([signal, e.lease.signal]);
|
|
294
|
-
const
|
|
295
|
-
const impl = tagged(p.impl, e.lease.token, () => this.register(
|
|
416
|
+
const env = ctx.environmentId;
|
|
417
|
+
const impl = tagged(p.impl, e.lease.token, ctx.actor, () => this.register(env, e.lease.token));
|
|
296
418
|
return fn(e.handle, impl, e.info, sig);
|
|
297
419
|
}
|
|
298
420
|
/**
|
|
299
421
|
* Becomes a holder: loops until it has a shared lease on a running instance, or `sandbox.busy`.
|
|
300
|
-
*
|
|
301
|
-
*
|
|
302
|
-
*
|
|
422
|
+
* An instance recorded `running` (with the current provider, not the `gone` instance) is used
|
|
423
|
+
* under a shared lease; otherwise the exclusive lease (with intent) makes it run and is
|
|
424
|
+
* downgraded. One token serves both, so the waiter passes its own intent.
|
|
303
425
|
*/
|
|
304
426
|
async enter(ctx, p, signal, gone) {
|
|
305
|
-
const
|
|
306
|
-
const cap = this.effectiveCap(p);
|
|
427
|
+
const env = ctx.environmentId;
|
|
307
428
|
const deadline = Date.now() + this.config.get().sandbox.lockWaitSeconds * 1000;
|
|
308
429
|
const token = randomUUID();
|
|
309
430
|
let intent = false;
|
|
@@ -312,19 +433,19 @@ let SandboxService = class SandboxService {
|
|
|
312
433
|
if (signal.aborted)
|
|
313
434
|
throw abortReason(signal);
|
|
314
435
|
const row = await this.readRow(ctx);
|
|
315
|
-
if (this.usable(row, p, gone)
|
|
316
|
-
const lease = await this.leases.tryShared(
|
|
436
|
+
if (this.usable(row, p, gone)) {
|
|
437
|
+
const lease = await this.leases.tryShared(env, this.cap, token);
|
|
317
438
|
if (lease) {
|
|
318
439
|
if (intent)
|
|
319
|
-
await this.clearIntent(
|
|
440
|
+
await this.clearIntent(env, token);
|
|
320
441
|
intent = false;
|
|
321
|
-
const e = await this.asShared(ctx, p, lease, gone, signal);
|
|
442
|
+
const e = await this.asShared(ctx, p, lease, (r) => this.usable(r, p, gone), signal);
|
|
322
443
|
if (e)
|
|
323
444
|
return e;
|
|
324
445
|
}
|
|
325
446
|
}
|
|
326
447
|
else {
|
|
327
|
-
const lease = await this.leases.tryExclusive(
|
|
448
|
+
const lease = await this.leases.tryExclusive(env, { token, intent: true });
|
|
328
449
|
intent = lease === null;
|
|
329
450
|
if (lease)
|
|
330
451
|
return await this.asExclusive(ctx, p, lease, gone, signal);
|
|
@@ -336,26 +457,29 @@ let SandboxService = class SandboxService {
|
|
|
336
457
|
}
|
|
337
458
|
catch (err) {
|
|
338
459
|
if (intent)
|
|
339
|
-
await this.clearIntent(
|
|
460
|
+
await this.clearIntent(env, token);
|
|
340
461
|
throw err;
|
|
341
462
|
}
|
|
342
463
|
}
|
|
343
|
-
async clearIntent(
|
|
464
|
+
async clearIntent(env, token) {
|
|
344
465
|
await this.leases
|
|
345
|
-
.clearIntent(
|
|
346
|
-
.catch((err) => this.log.warn("cannot clear sandbox intent", {
|
|
466
|
+
.clearIntent(env, token)
|
|
467
|
+
.catch((err) => this.log.warn("cannot clear sandbox intent", { environmentId: env, err }));
|
|
347
468
|
}
|
|
348
469
|
async readRow(ctx) {
|
|
349
470
|
const row = await this.db
|
|
350
|
-
.selectFrom("
|
|
471
|
+
.selectFrom("environment_instances")
|
|
351
472
|
.selectAll()
|
|
352
|
-
.where("
|
|
473
|
+
.where("environment_id", "=", ctx.environmentId)
|
|
353
474
|
.executeTakeFirst();
|
|
354
|
-
if (row && row.org_id !== ctx.orgId)
|
|
355
|
-
throw
|
|
356
|
-
}
|
|
475
|
+
if (row && row.org_id !== ctx.orgId)
|
|
476
|
+
throw this.foreign(ctx);
|
|
357
477
|
return row;
|
|
358
478
|
}
|
|
479
|
+
/** The instance row belongs to another org: reported as not found, so ids cannot be probed. */
|
|
480
|
+
foreign(_ctx) {
|
|
481
|
+
return environmentNotFound();
|
|
482
|
+
}
|
|
359
483
|
/** Recorded running with the current provider, and not the instance found gone. */
|
|
360
484
|
usable(row, p, gone) {
|
|
361
485
|
return (row !== undefined &&
|
|
@@ -365,14 +489,14 @@ let SandboxService = class SandboxService {
|
|
|
365
489
|
!isInstance(row, gone));
|
|
366
490
|
}
|
|
367
491
|
/**
|
|
368
|
-
* Under a just-taken shared lease: confirms the row is still
|
|
369
|
-
* null), records the holder and kills crashed holders' commands.
|
|
492
|
+
* Under a just-taken shared lease: confirms the row is still `accept`ed (else releases and
|
|
493
|
+
* returns null), records the holder and kills crashed holders' commands.
|
|
370
494
|
*/
|
|
371
|
-
async asShared(ctx, p, lease,
|
|
372
|
-
const
|
|
495
|
+
async asShared(ctx, p, lease, accept, signal) {
|
|
496
|
+
const env = ctx.environmentId;
|
|
373
497
|
try {
|
|
374
498
|
const row = await this.readRow(ctx);
|
|
375
|
-
if (!
|
|
499
|
+
if (!accept(row)) {
|
|
376
500
|
await lease.release();
|
|
377
501
|
return null;
|
|
378
502
|
}
|
|
@@ -380,120 +504,124 @@ let SandboxService = class SandboxService {
|
|
|
380
504
|
// expired unnoticed (a stalled sweep) then fails its next write instead of suspending or
|
|
381
505
|
// destroying the instance under this holder. Shared holders never compare it.
|
|
382
506
|
await this.db
|
|
383
|
-
.updateTable("
|
|
507
|
+
.updateTable("environment_instances")
|
|
384
508
|
.set((eb) => ({ generation: eb("generation", "+", 1) }))
|
|
385
|
-
.where("
|
|
509
|
+
.where("environment_id", "=", env)
|
|
386
510
|
.execute();
|
|
387
|
-
await this.register(
|
|
388
|
-
//
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
const sig = AbortSignal.any([signal, lease.signal]);
|
|
392
|
-
const impl = p.impl;
|
|
393
|
-
if (impl.killTagged) {
|
|
394
|
-
// by tag whenever the provider can, even with cap 1: while a hot change of
|
|
395
|
-
// maxConcurrentCommands spreads, a replica on a larger cap may have let a sibling in
|
|
396
|
-
await this.kill(conv, "takeover", async () => impl.killTagged?.(row.handle, dead.tags, sig), sig);
|
|
397
|
-
}
|
|
398
|
-
else if ((await this.leases.sharedCount(conv)) <= 1) {
|
|
399
|
-
// cap 1 and this holder is alone: killing everything cannot hit a sibling
|
|
400
|
-
await this.killLeftovers(conv, row.handle, p, sig, "takeover");
|
|
401
|
-
}
|
|
402
|
-
else {
|
|
403
|
-
this.log.warn("cannot kill leftover sandbox commands: the provider has no killTagged and another holder is live; the in-sandbox time limit ends them", { conversationId: conv });
|
|
404
|
-
}
|
|
405
|
-
}
|
|
511
|
+
await this.register(env, lease.token);
|
|
512
|
+
// other holders (any conversation of the environment) may be running: only the dead
|
|
513
|
+
// holders' own commands are killed
|
|
514
|
+
await this.killTagged(env, row.handle, p, await this.claim(env), signal, lease);
|
|
406
515
|
return {
|
|
407
516
|
lease,
|
|
408
517
|
handle: row.handle,
|
|
409
518
|
createdAt: row.created_at.getTime(),
|
|
410
|
-
|
|
519
|
+
fsId: row.fs_handle?.id ?? "",
|
|
520
|
+
info: { fresh: false, restarted: false },
|
|
411
521
|
};
|
|
412
522
|
}
|
|
413
523
|
catch (err) {
|
|
414
|
-
await this.unregister(
|
|
524
|
+
await this.unregister(env, lease.token);
|
|
415
525
|
await lease.release();
|
|
416
526
|
throw err;
|
|
417
527
|
}
|
|
418
528
|
}
|
|
419
529
|
/**
|
|
420
|
-
* Under a just-taken exclusive lease: makes the
|
|
421
|
-
* recorded instance is the one found gone), kills leftovers, then downgrades to
|
|
530
|
+
* Under a just-taken exclusive lease: makes the instance run (file system, create / resume /
|
|
531
|
+
* rebuild when the recorded instance is the one found gone), kills leftovers, then downgrades to
|
|
532
|
+
* a shared lease. Crashed holders' records are claimed here: on an instance just resumed from
|
|
533
|
+
* `suspended` one `killAll` covers them and anything else left (nobody else holds the instance),
|
|
534
|
+
* on a new instance there is nothing to kill, and on one that already ran their tags are killed.
|
|
422
535
|
*/
|
|
423
536
|
async asExclusive(ctx, p, lease, gone, signal) {
|
|
424
|
-
const
|
|
537
|
+
const env = ctx.environmentId;
|
|
425
538
|
const sig = AbortSignal.any([signal, lease.signal]);
|
|
426
539
|
try {
|
|
427
540
|
const s = await this.ensure(ctx, p, lease, sig, gone);
|
|
428
|
-
const dead = await this.claim(
|
|
429
|
-
if (
|
|
430
|
-
await this.killLeftovers(
|
|
541
|
+
const dead = await this.claim(env);
|
|
542
|
+
if (s.resumed)
|
|
543
|
+
await this.killLeftovers(env, s.handle, p, sig, dead);
|
|
544
|
+
else if (!s.info.fresh)
|
|
545
|
+
await this.killTagged(env, s.handle, p, dead, signal, lease);
|
|
546
|
+
if (s.resumed || s.info.fresh) {
|
|
547
|
+
await this.killPendingTasks(env, s.handle, s.createdAt, p, lease.token, ctx.actor, sig);
|
|
431
548
|
}
|
|
432
549
|
const shared = await lease.downgrade();
|
|
433
|
-
await this.register(
|
|
434
|
-
return {
|
|
550
|
+
await this.register(env, shared.token);
|
|
551
|
+
return {
|
|
552
|
+
lease: shared,
|
|
553
|
+
handle: s.handle,
|
|
554
|
+
createdAt: s.createdAt,
|
|
555
|
+
fsId: s.fsId,
|
|
556
|
+
info: s.info,
|
|
557
|
+
};
|
|
435
558
|
}
|
|
436
559
|
catch (err) {
|
|
437
|
-
await this.unregister(
|
|
560
|
+
await this.unregister(env, lease.token);
|
|
438
561
|
// releases the lease in whatever kind it has now (also after a downgrade)
|
|
439
562
|
await lease.release();
|
|
440
563
|
throw err;
|
|
441
564
|
}
|
|
442
565
|
}
|
|
443
566
|
/** Ends a holder: unregisters it before releasing its lease, then moves `last_used_at` forward. */
|
|
444
|
-
async leave(
|
|
445
|
-
await this.unregister(
|
|
567
|
+
async leave(env, e) {
|
|
568
|
+
await this.unregister(env, e.lease.token);
|
|
446
569
|
await e.lease.release();
|
|
447
570
|
await this.db
|
|
448
|
-
.updateTable("
|
|
571
|
+
.updateTable("environment_instances")
|
|
449
572
|
.set({ last_used_at: sql `GREATEST(last_used_at, now())` })
|
|
450
|
-
.where("
|
|
573
|
+
.where("environment_id", "=", env)
|
|
451
574
|
.where("status", "=", "running")
|
|
452
575
|
.execute()
|
|
453
|
-
.catch((err) => this.log.warn("cannot update sandbox last_used_at", {
|
|
576
|
+
.catch((err) => this.log.warn("cannot update sandbox last_used_at", { environmentId: env, err }));
|
|
454
577
|
}
|
|
455
|
-
async register(
|
|
578
|
+
async register(env, token) {
|
|
456
579
|
const ttl = this.config.get().sandbox.limits.maxCommandSeconds * 1000 + HOLDER_TTL_MARGIN_MS;
|
|
457
580
|
await this.leases
|
|
458
|
-
.registerHolder(
|
|
459
|
-
.catch((err) => this.log.warn("cannot record sandbox holder", {
|
|
581
|
+
.registerHolder(env, token, ttl)
|
|
582
|
+
.catch((err) => this.log.warn("cannot record sandbox holder", { environmentId: env, err }));
|
|
460
583
|
}
|
|
461
|
-
async unregister(
|
|
584
|
+
async unregister(env, token) {
|
|
462
585
|
await this.leases
|
|
463
|
-
.unregisterHolder(
|
|
464
|
-
.catch((err) => this.log.warn("cannot clear sandbox holder record", {
|
|
586
|
+
.unregisterHolder(env, token)
|
|
587
|
+
.catch((err) => this.log.warn("cannot clear sandbox holder record", { environmentId: env, err }));
|
|
465
588
|
}
|
|
466
|
-
/**
|
|
467
|
-
|
|
468
|
-
* the caller's exclusive lease; a Redis error counts as none.
|
|
469
|
-
*/
|
|
470
|
-
async claim(conv, exclusiveToken) {
|
|
589
|
+
/** Crashed holders' tags (claimed); a Redis error counts as none. */
|
|
590
|
+
async claim(env) {
|
|
471
591
|
try {
|
|
472
|
-
return await this.leases.claimDeadHolders(
|
|
592
|
+
return await this.leases.claimDeadHolders(env);
|
|
473
593
|
}
|
|
474
594
|
catch (err) {
|
|
475
|
-
this.log.warn("cannot read sandbox holder records", {
|
|
476
|
-
return
|
|
595
|
+
this.log.warn("cannot read sandbox holder records", { environmentId: env, err });
|
|
596
|
+
return [];
|
|
477
597
|
}
|
|
478
598
|
}
|
|
599
|
+
/** Kills the commands of crashed holders by their tags (`killTagged`); nothing without tags. */
|
|
600
|
+
async killTagged(env, handle, p, tags, signal, lease) {
|
|
601
|
+
if (tags.length === 0)
|
|
602
|
+
return;
|
|
603
|
+
const sig = AbortSignal.any([signal, lease.signal]);
|
|
604
|
+
await this.kill(env, "takeover", () => p.impl.killTagged(handle, tags, sig), sig);
|
|
605
|
+
}
|
|
479
606
|
/** Runs one kill call: gone and abort are thrown, other failures only warned. */
|
|
480
|
-
async kill(
|
|
607
|
+
async kill(environmentId, why, call, signal) {
|
|
481
608
|
try {
|
|
482
609
|
await call();
|
|
483
610
|
if (why === "takeover") {
|
|
484
|
-
this.log.info("sandbox taken over from a holder that did not finish; leftover commands killed", {
|
|
611
|
+
this.log.info("sandbox taken over from a holder that did not finish; leftover commands killed", { environmentId });
|
|
485
612
|
}
|
|
486
613
|
}
|
|
487
614
|
catch (err) {
|
|
488
615
|
if (signal?.aborted || isSandboxGoneError(err))
|
|
489
616
|
throw err;
|
|
490
|
-
this.log.warn("cannot kill leftover sandbox commands", {
|
|
617
|
+
this.log.warn("cannot kill leftover sandbox commands", { environmentId, err });
|
|
491
618
|
}
|
|
492
619
|
}
|
|
493
620
|
/**
|
|
494
|
-
*
|
|
495
|
-
* Skips
|
|
496
|
-
* window (about a second) first.
|
|
621
|
+
* Suspends running instances idle longer than `suspendAfterIdleMinutes` or running continuously
|
|
622
|
+
* longer than `maxRunningMinutes`. Skips an environment with any holder; one past
|
|
623
|
+
* `maxRunningMinutes` gets the intent for one poll window (about a second) first. Returns how
|
|
624
|
+
* many it suspended. Called every minute by the `sandbox-suspend-idle` cron.
|
|
497
625
|
*/
|
|
498
626
|
async suspendIdle(now) {
|
|
499
627
|
const p = this.providers.current();
|
|
@@ -505,12 +633,12 @@ let SandboxService = class SandboxService {
|
|
|
505
633
|
let n = 0;
|
|
506
634
|
await inPool(rows, SWEEP_CONCURRENCY, async (c) => {
|
|
507
635
|
const overdue = c.running_since !== null && c.running_since < runningBefore;
|
|
508
|
-
const done = await this.tryLocked(c.
|
|
509
|
-
const id = c.
|
|
636
|
+
const done = await this.tryLocked(c.environment_id, async (lease) => {
|
|
637
|
+
const id = c.environment_id;
|
|
510
638
|
const row = await this.db
|
|
511
|
-
.selectFrom("
|
|
639
|
+
.selectFrom("environment_instances")
|
|
512
640
|
.selectAll()
|
|
513
|
-
.where("
|
|
641
|
+
.where("environment_id", "=", id)
|
|
514
642
|
.where(due)
|
|
515
643
|
.executeTakeFirst();
|
|
516
644
|
if (!row?.handle)
|
|
@@ -525,7 +653,7 @@ let SandboxService = class SandboxService {
|
|
|
525
653
|
running_since: null,
|
|
526
654
|
sweep_failed_at: null,
|
|
527
655
|
});
|
|
528
|
-
this.log.info("sandbox suspended", {
|
|
656
|
+
this.log.info("sandbox suspended", { environmentId: id });
|
|
529
657
|
return true;
|
|
530
658
|
}, overdue);
|
|
531
659
|
if (done)
|
|
@@ -534,89 +662,182 @@ let SandboxService = class SandboxService {
|
|
|
534
662
|
return n;
|
|
535
663
|
}
|
|
536
664
|
/**
|
|
537
|
-
*
|
|
538
|
-
*
|
|
665
|
+
* Destruction sweep: an instance unused for longer than `destroyAfterInactiveDays` is destroyed
|
|
666
|
+
* (the instance only; the file system is kept and the next use builds a new instance on it); an
|
|
667
|
+
* environment that was deleted, or whose org was deleted, loses its instance and its file system
|
|
668
|
+
* and its row is deleted (as {@link destroyEnvironment} does). Environments with any holder are
|
|
669
|
+
* skipped. Returns how many it destroyed. Called hourly by the `sandbox-destroy-inactive` cron.
|
|
539
670
|
*/
|
|
540
671
|
async destroyInactive(now) {
|
|
541
672
|
const p = this.providers.current();
|
|
542
673
|
if (!p)
|
|
543
674
|
return 0;
|
|
544
|
-
const due = this.destroyDue(now);
|
|
545
|
-
const rows = await this.candidates(due);
|
|
546
675
|
let n = 0;
|
|
547
|
-
|
|
676
|
+
const gone = (eb) => this.environmentGone(eb);
|
|
677
|
+
await inPool(await this.candidates(gone), SWEEP_CONCURRENCY, async ({ environment_id: id }) => {
|
|
548
678
|
const done = await this.tryLocked(id, async (lease) => {
|
|
549
679
|
const row = await this.db
|
|
550
|
-
.selectFrom("
|
|
680
|
+
.selectFrom("environment_instances")
|
|
551
681
|
.selectAll()
|
|
552
|
-
.where("
|
|
553
|
-
.where(
|
|
682
|
+
.where("environment_id", "=", id)
|
|
683
|
+
.where(gone)
|
|
554
684
|
.executeTakeFirst();
|
|
555
685
|
if (!row)
|
|
556
686
|
return false;
|
|
557
|
-
await this.destroyRow(row, p, lease);
|
|
687
|
+
await this.destroyRow(row, p, lease, "environment");
|
|
688
|
+
return true;
|
|
689
|
+
});
|
|
690
|
+
if (done)
|
|
691
|
+
n++;
|
|
692
|
+
});
|
|
693
|
+
const idle = this.instanceDue(now);
|
|
694
|
+
await inPool(await this.candidates(idle), SWEEP_CONCURRENCY, async ({ environment_id: id }) => {
|
|
695
|
+
const done = await this.tryLocked(id, async (lease) => {
|
|
696
|
+
const row = await this.db
|
|
697
|
+
.selectFrom("environment_instances")
|
|
698
|
+
.selectAll()
|
|
699
|
+
.where("environment_id", "=", id)
|
|
700
|
+
.where(idle)
|
|
701
|
+
.executeTakeFirst();
|
|
702
|
+
if (!row)
|
|
703
|
+
return false;
|
|
704
|
+
await this.destroyRow(row, p, lease, "instance");
|
|
558
705
|
return true;
|
|
559
706
|
});
|
|
560
707
|
if (done)
|
|
561
708
|
n++;
|
|
562
709
|
});
|
|
563
|
-
// 早先已销毁、对话后来才删除、且没有遗留旧实例的记录
|
|
564
|
-
await this.db
|
|
565
|
-
.deleteFrom("sandboxes")
|
|
566
|
-
.where("status", "=", "destroyed")
|
|
567
|
-
.where("prev_handle", "is", null)
|
|
568
|
-
.where(({ not, exists, selectFrom }) => not(exists(selectFrom("conversations")
|
|
569
|
-
.select("conversations.id")
|
|
570
|
-
.whereRef("conversations.id", "=", "sandboxes.conversation_id"))))
|
|
571
|
-
.execute();
|
|
572
710
|
return n;
|
|
573
711
|
}
|
|
574
712
|
/**
|
|
575
|
-
*
|
|
576
|
-
*
|
|
577
|
-
* to 10 minutes.
|
|
713
|
+
* Destroys the environment's instance, then its file system, then deletes its instance row
|
|
714
|
+
* (environment deletion, offboarding); nothing to do without a row. Waits for the exclusive lease
|
|
715
|
+
* (with intent, so running commands finish and new ones wait) up to 10 minutes. Without a
|
|
716
|
+
* provider it only warns (the destruction sweep finishes a deleted environment later). Files kept
|
|
717
|
+
* by another provider that is not loaded cannot be destroyed: it throws and the row stays.
|
|
578
718
|
*/
|
|
579
|
-
async
|
|
719
|
+
async destroyEnvironment(environmentId) {
|
|
580
720
|
const row = await this.db
|
|
581
|
-
.selectFrom("
|
|
582
|
-
.select("
|
|
583
|
-
.where("
|
|
721
|
+
.selectFrom("environment_instances")
|
|
722
|
+
.select("environment_id")
|
|
723
|
+
.where("environment_id", "=", environmentId)
|
|
584
724
|
.executeTakeFirst();
|
|
585
|
-
if (!row
|
|
725
|
+
if (!row)
|
|
586
726
|
return;
|
|
587
727
|
const p = this.providers.current();
|
|
588
728
|
if (!p) {
|
|
589
|
-
this.log.warn("cannot destroy
|
|
729
|
+
this.log.warn("cannot destroy environment: no sandbox provider", { environmentId });
|
|
590
730
|
return;
|
|
591
731
|
}
|
|
592
|
-
const lease = await this.leases.acquireExclusive(
|
|
732
|
+
const lease = await this.leases.acquireExclusive(environmentId, AbortSignal.timeout(EXCLUSIVE_WAIT_MS), undefined, { intent: true });
|
|
593
733
|
try {
|
|
594
734
|
const current = await this.db
|
|
595
|
-
.selectFrom("
|
|
735
|
+
.selectFrom("environment_instances")
|
|
596
736
|
.selectAll()
|
|
597
|
-
.where("
|
|
598
|
-
.where("status", "<>", "destroyed")
|
|
737
|
+
.where("environment_id", "=", environmentId)
|
|
599
738
|
.executeTakeFirst();
|
|
600
739
|
if (current)
|
|
601
|
-
await this.destroyRow(current, p, lease);
|
|
740
|
+
await this.destroyRow(current, p, lease, "environment");
|
|
602
741
|
}
|
|
603
742
|
finally {
|
|
604
743
|
await lease.release();
|
|
605
744
|
}
|
|
606
745
|
}
|
|
607
746
|
/**
|
|
608
|
-
*
|
|
609
|
-
*
|
|
747
|
+
* Starts the environment over with empty files (after `environment.files_lost`, or on request):
|
|
748
|
+
* under the exclusive lease (waited for like {@link destroyEnvironment}) destroys any instance and
|
|
749
|
+
* the file system (a `lost` one is only dropped), and records no file system; the next use
|
|
750
|
+
* creates a new one. Nothing to do without a row. Throws `sandbox.unavailable` without a provider.
|
|
610
751
|
*/
|
|
611
|
-
async
|
|
612
|
-
const
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
752
|
+
async resetFileSystem(environmentId) {
|
|
753
|
+
const p = this.require();
|
|
754
|
+
const lease = await this.leases.acquireExclusive(environmentId, AbortSignal.timeout(EXCLUSIVE_WAIT_MS), undefined, { intent: true });
|
|
755
|
+
try {
|
|
756
|
+
const row = await this.db
|
|
757
|
+
.selectFrom("environment_instances")
|
|
758
|
+
.selectAll()
|
|
759
|
+
.where("environment_id", "=", environmentId)
|
|
760
|
+
.executeTakeFirst();
|
|
761
|
+
if (!row)
|
|
762
|
+
return;
|
|
763
|
+
const fence = { gen: row.generation };
|
|
764
|
+
const signal = opSignal(lease);
|
|
765
|
+
const prev = await this.destroyInstance(row, p, signal);
|
|
766
|
+
await this.destroyFiles(row, p, signal, true);
|
|
767
|
+
if (!(await lease.renew()))
|
|
768
|
+
throw new SandboxLockLostError(environmentId);
|
|
769
|
+
await this.fenced(lease, environmentId, fence, {
|
|
770
|
+
fs_handle: null,
|
|
771
|
+
fs_status: "none",
|
|
772
|
+
handle: null,
|
|
773
|
+
status: "none",
|
|
774
|
+
running_since: null,
|
|
775
|
+
suspended_at: null,
|
|
776
|
+
destroyed_at: sql `now()`,
|
|
777
|
+
sweep_failed_at: null,
|
|
778
|
+
...prev,
|
|
779
|
+
});
|
|
780
|
+
this.log.info("environment files reset", { environmentId });
|
|
781
|
+
}
|
|
782
|
+
finally {
|
|
783
|
+
await lease.release();
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
/**
|
|
787
|
+
* Under the exclusive lease (before the downgrade), on an instance just resumed from
|
|
788
|
+
* `suspended`: kills every command left in it (`killAll`), crashed holders' (`dead`, already
|
|
789
|
+
* claimed) included. Never on the shared path: other holders may be running commands. A gone
|
|
790
|
+
* instance (`SandboxGoneError`) and an abort are thrown (`withSandbox` rebuilds / ends); other
|
|
791
|
+
* failures are only warned.
|
|
792
|
+
*/
|
|
793
|
+
async killLeftovers(environmentId, handle, p, signal, dead) {
|
|
794
|
+
await this.kill(environmentId, dead.length > 0 ? "takeover" : "resume", () => p.impl.killAll(handle, signal), signal);
|
|
795
|
+
}
|
|
796
|
+
/**
|
|
797
|
+
* Under the exclusive lease, right after a resume or a (re)build: kills the sessions of the
|
|
798
|
+
* background tasks marked lost while their process may live on (`kill_pending`), in one exec of
|
|
799
|
+
* `pkill -KILL -s <sid>`, then clears their flags. Tasks of another instance have no process
|
|
800
|
+
* here and are only cleared. A gone instance and an abort are thrown; another failure is warned
|
|
801
|
+
* and leaves the flags of this instance's tasks for the next resume.
|
|
802
|
+
*/
|
|
803
|
+
async killPendingTasks(environmentId, handle, createdAt, p, tag, actor, signal) {
|
|
804
|
+
const rows = await this.background.pendingKills(environmentId);
|
|
805
|
+
if (rows.length === 0)
|
|
617
806
|
return;
|
|
807
|
+
const mine = rows.filter((r) => r.sandboxCreatedAt?.getTime() === createdAt && r.sid !== null && SID_RE.test(r.sid));
|
|
808
|
+
let cleared = rows;
|
|
809
|
+
if (mine.length > 0) {
|
|
810
|
+
// pkill exits 1 when the session has no process left: only a higher status is a failure
|
|
811
|
+
const command = `rc=0; for sid in ${mine.map((r) => r.sid).join(" ")}; do pkill -KILL -s "$sid"; c=$?; [ "$c" -le 1 ] || rc=$c; done; exit $rc`;
|
|
812
|
+
try {
|
|
813
|
+
const r = await p.impl.exec(handle, {
|
|
814
|
+
command,
|
|
815
|
+
timeoutMs: PENDING_KILL_TIMEOUT_MS,
|
|
816
|
+
maxOutputBytes: 4096,
|
|
817
|
+
signal,
|
|
818
|
+
tag,
|
|
819
|
+
actor,
|
|
820
|
+
});
|
|
821
|
+
if (r.exitCode !== 0) {
|
|
822
|
+
throw new Error(`pkill exited with ${r.exitCode === null ? "no status" : r.exitCode}: ${r.stderr.slice(0, 500)}`);
|
|
823
|
+
}
|
|
824
|
+
this.log.info("lost background tasks killed after a sandbox resume", {
|
|
825
|
+
environmentId,
|
|
826
|
+
tasks: mine.map((t) => t.id),
|
|
827
|
+
});
|
|
828
|
+
}
|
|
829
|
+
catch (err) {
|
|
830
|
+
if (signal.aborted || isSandboxGoneError(err))
|
|
831
|
+
throw err;
|
|
832
|
+
this.log.warn("cannot kill lost background tasks after a sandbox resume", {
|
|
833
|
+
environmentId,
|
|
834
|
+
err,
|
|
835
|
+
});
|
|
836
|
+
const kept = new Set(mine.map((r) => r.id));
|
|
837
|
+
cleared = rows.filter((r) => !kept.has(r.id));
|
|
838
|
+
}
|
|
618
839
|
}
|
|
619
|
-
await this.
|
|
840
|
+
await this.background.clearKill(cleared.map((r) => r.id));
|
|
620
841
|
}
|
|
621
842
|
require() {
|
|
622
843
|
const p = this.providers.current();
|
|
@@ -625,15 +846,16 @@ let SandboxService = class SandboxService {
|
|
|
625
846
|
return p;
|
|
626
847
|
}
|
|
627
848
|
/**
|
|
628
|
-
*
|
|
849
|
+
* A fenced write: only while the lease holds and the row's generation is still the one read
|
|
850
|
+
* under it (and incremented); otherwise {@link SandboxLockLostError}.
|
|
629
851
|
*/
|
|
630
852
|
async fenced(lease, id, fence, set) {
|
|
631
853
|
if (lease.signal.aborted)
|
|
632
854
|
throw new SandboxLockLostError(id);
|
|
633
855
|
const r = await this.db
|
|
634
|
-
.updateTable("
|
|
856
|
+
.updateTable("environment_instances")
|
|
635
857
|
.set({ ...set, generation: fence.gen + 1 })
|
|
636
|
-
.where("
|
|
858
|
+
.where("environment_id", "=", id)
|
|
637
859
|
.where("generation", "=", fence.gen)
|
|
638
860
|
.returning("created_at")
|
|
639
861
|
.executeTakeFirst();
|
|
@@ -642,11 +864,11 @@ let SandboxService = class SandboxService {
|
|
|
642
864
|
fence.gen++;
|
|
643
865
|
return r.created_at;
|
|
644
866
|
}
|
|
645
|
-
/**
|
|
867
|
+
/** Sweep candidates: those whose last sweep failed come last, the rest least recently used first. */
|
|
646
868
|
async candidates(due) {
|
|
647
869
|
return await this.db
|
|
648
|
-
.selectFrom("
|
|
649
|
-
.select(["
|
|
870
|
+
.selectFrom("environment_instances")
|
|
871
|
+
.select(["environment_id", "running_since"])
|
|
650
872
|
.where(due)
|
|
651
873
|
.orderBy(sql `sweep_failed_at ASC NULLS FIRST`)
|
|
652
874
|
.orderBy("last_used_at")
|
|
@@ -655,17 +877,17 @@ let SandboxService = class SandboxService {
|
|
|
655
877
|
}
|
|
656
878
|
/**
|
|
657
879
|
* One try at the exclusive lease: null → false; else runs `fn`. With `overdue` (past
|
|
658
|
-
* `maxRunningMinutes`) a busy
|
|
659
|
-
* briefly and the sweep gets in if the running ones end. Failures are warned, recorded in
|
|
880
|
+
* `maxRunningMinutes`) a busy environment gets the intent for about a second, so new commands
|
|
881
|
+
* wait briefly and the sweep gets in if the running ones end. Failures are warned, recorded in
|
|
660
882
|
* `sweep_failed_at` (sorted last next time) and return false.
|
|
661
883
|
*/
|
|
662
|
-
async tryLocked(
|
|
884
|
+
async tryLocked(environmentId, fn, overdue = false) {
|
|
663
885
|
let lease;
|
|
664
886
|
try {
|
|
665
|
-
lease = await this.leases.tryExclusive(
|
|
887
|
+
lease = await this.leases.tryExclusive(environmentId);
|
|
666
888
|
if (!lease && overdue) {
|
|
667
889
|
lease = await this.leases
|
|
668
|
-
.acquireExclusive(
|
|
890
|
+
.acquireExclusive(environmentId, AbortSignal.timeout(SWEEP_OP_TIMEOUT_MS), Math.max(this.pollMs * 2, SWEEP_INTENT_WAIT_MS), { intent: true })
|
|
669
891
|
.catch((err) => {
|
|
670
892
|
if (err instanceof ApiError && err.code === "sandbox.busy")
|
|
671
893
|
return null;
|
|
@@ -674,7 +896,7 @@ let SandboxService = class SandboxService {
|
|
|
674
896
|
}
|
|
675
897
|
}
|
|
676
898
|
catch (err) {
|
|
677
|
-
this.log.warn("cannot take sandbox lock", {
|
|
899
|
+
this.log.warn("cannot take sandbox lock", { environmentId, err });
|
|
678
900
|
return false;
|
|
679
901
|
}
|
|
680
902
|
if (!lease)
|
|
@@ -683,11 +905,11 @@ let SandboxService = class SandboxService {
|
|
|
683
905
|
return await fn(lease);
|
|
684
906
|
}
|
|
685
907
|
catch (err) {
|
|
686
|
-
this.log.warn("sandbox maintenance failed; will retry", {
|
|
908
|
+
this.log.warn("sandbox maintenance failed; will retry", { environmentId, err });
|
|
687
909
|
await this.db
|
|
688
|
-
.updateTable("
|
|
910
|
+
.updateTable("environment_instances")
|
|
689
911
|
.set({ sweep_failed_at: sql `now()` })
|
|
690
|
-
.where("
|
|
912
|
+
.where("environment_id", "=", environmentId)
|
|
691
913
|
.execute()
|
|
692
914
|
.catch(() => { });
|
|
693
915
|
return false;
|
|
@@ -697,82 +919,120 @@ let SandboxService = class SandboxService {
|
|
|
697
919
|
}
|
|
698
920
|
}
|
|
699
921
|
/**
|
|
700
|
-
* Under the exclusive lease: make the
|
|
922
|
+
* Under the exclusive lease: make the environment's instance run. `gone`: the instance a holder
|
|
701
923
|
* found gone; when it is still the recorded one it is rebuilt, when another holder already
|
|
702
924
|
* rebuilt it the new one is used.
|
|
703
925
|
*/
|
|
704
926
|
async ensure(ctx, p, held, signal, gone) {
|
|
705
|
-
const id = ctx.
|
|
927
|
+
const id = ctx.environmentId;
|
|
928
|
+
// Only a live environment of the caller's org gets a row, with the environment's own org; a
|
|
929
|
+
// row made earlier (another use) keeps its file system handle.
|
|
706
930
|
await this.db
|
|
707
|
-
.insertInto("
|
|
708
|
-
.
|
|
709
|
-
.
|
|
931
|
+
.insertInto("environment_instances")
|
|
932
|
+
.columns(["environment_id", "org_id", "provider", "status"])
|
|
933
|
+
.expression((eb) => eb
|
|
934
|
+
.selectFrom("environments")
|
|
935
|
+
.select((e) => [
|
|
936
|
+
"environments.id",
|
|
937
|
+
"environments.org_id",
|
|
938
|
+
e.val(p.plugin).as("provider"),
|
|
939
|
+
e.val("creating").as("status"),
|
|
940
|
+
])
|
|
941
|
+
.where("environments.id", "=", id)
|
|
942
|
+
.where("environments.org_id", "=", ctx.orgId)
|
|
943
|
+
.where("environments.deleted_at", "is", null))
|
|
944
|
+
.onConflict((oc) => oc.column("environment_id").doNothing())
|
|
710
945
|
.execute();
|
|
946
|
+
// Under the lease: a use that waited behind destroyEnvironment (or raced a deletion) must not
|
|
947
|
+
// bring the environment's provider resources back.
|
|
948
|
+
const live = await this.db
|
|
949
|
+
.selectFrom("environments")
|
|
950
|
+
.select("id")
|
|
951
|
+
.where("id", "=", id)
|
|
952
|
+
.where("org_id", "=", ctx.orgId)
|
|
953
|
+
.where("deleted_at", "is", null)
|
|
954
|
+
.executeTakeFirst();
|
|
955
|
+
if (!live)
|
|
956
|
+
throw environmentNotFound();
|
|
711
957
|
const row = await this.db
|
|
712
|
-
.selectFrom("
|
|
958
|
+
.selectFrom("environment_instances")
|
|
713
959
|
.selectAll()
|
|
714
|
-
.where("
|
|
715
|
-
.
|
|
716
|
-
if (row
|
|
717
|
-
throw
|
|
718
|
-
|
|
960
|
+
.where("environment_id", "=", id)
|
|
961
|
+
.executeTakeFirst();
|
|
962
|
+
if (!row)
|
|
963
|
+
throw environmentNotFound();
|
|
964
|
+
if (row.org_id !== ctx.orgId)
|
|
965
|
+
throw this.foreign(ctx);
|
|
719
966
|
const fence = { gen: row.generation };
|
|
720
|
-
//
|
|
967
|
+
// Every acquisition first does a fenced increment: an old holder whose lease expired while it
|
|
968
|
+
// kept running (a stalled sweep) then fails its next write.
|
|
721
969
|
await this.fenced(held, id, fence, {});
|
|
970
|
+
if (row.fs_status === "lost")
|
|
971
|
+
throw environmentFilesLost();
|
|
972
|
+
if (row.provider !== null &&
|
|
973
|
+
row.provider !== p.plugin &&
|
|
974
|
+
(row.fs_status !== "none" || row.handle !== null)) {
|
|
975
|
+
throw otherProvider(row.provider);
|
|
976
|
+
}
|
|
977
|
+
if (row.provider !== null &&
|
|
978
|
+
row.provider !== p.plugin &&
|
|
979
|
+
row.fs_status === "none" &&
|
|
980
|
+
row.status === "creating") {
|
|
981
|
+
// a create by the previous provider ended before its file system was recorded: that
|
|
982
|
+
// provider may have made it by name, and nothing else would ever destroy it
|
|
983
|
+
await this.dropUnrecordedFiles(id, row.provider, signal);
|
|
984
|
+
}
|
|
722
985
|
const handle = row.handle;
|
|
723
986
|
const createdAt = row.created_at.getTime();
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
987
|
+
const fsId = row.fs_handle?.id ?? "";
|
|
988
|
+
if (handle && isInstance(row, gone) && row.status === "running") {
|
|
989
|
+
const created = await this.create(ctx, p, held, row, fence, signal);
|
|
990
|
+
return { ...created, info: { fresh: true, restarted: true }, fence };
|
|
727
991
|
}
|
|
728
|
-
if (handle && row.
|
|
729
|
-
|
|
730
|
-
return { handle, createdAt, info: { fresh: false, recreated: false }, fence };
|
|
731
|
-
}
|
|
732
|
-
if (row.status === "suspended") {
|
|
733
|
-
const r = await p.impl.resume(handle, signal);
|
|
734
|
-
if (r === "resumed") {
|
|
735
|
-
await this.fenced(held, id, fence, {
|
|
736
|
-
status: "running",
|
|
737
|
-
running_since: sql `now()`,
|
|
738
|
-
suspended_at: null,
|
|
739
|
-
sweep_failed_at: null,
|
|
740
|
-
});
|
|
741
|
-
return {
|
|
742
|
-
handle,
|
|
743
|
-
createdAt,
|
|
744
|
-
info: { fresh: false, recreated: false },
|
|
745
|
-
fence,
|
|
746
|
-
resumed: true,
|
|
747
|
-
};
|
|
748
|
-
}
|
|
749
|
-
this.log.info("sandbox is gone at the provider; recreating", { conversationId: id });
|
|
750
|
-
}
|
|
992
|
+
if (handle && row.status === "running") {
|
|
993
|
+
return { handle, createdAt, fsId, info: { fresh: false, restarted: false }, fence };
|
|
751
994
|
}
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
995
|
+
if (handle && row.status === "suspended") {
|
|
996
|
+
const r = await p.impl.resume(handle, signal);
|
|
997
|
+
if (r === "resumed") {
|
|
998
|
+
await this.fenced(held, id, fence, {
|
|
999
|
+
status: "running",
|
|
1000
|
+
running_since: sql `now()`,
|
|
1001
|
+
suspended_at: null,
|
|
1002
|
+
sweep_failed_at: null,
|
|
1003
|
+
});
|
|
1004
|
+
return {
|
|
1005
|
+
handle,
|
|
1006
|
+
createdAt,
|
|
1007
|
+
fsId,
|
|
1008
|
+
info: { fresh: false, restarted: false },
|
|
1009
|
+
fence,
|
|
1010
|
+
resumed: true,
|
|
1011
|
+
};
|
|
764
1012
|
}
|
|
1013
|
+
this.log.info("sandbox instance is gone at the provider; recreating", {
|
|
1014
|
+
environmentId: id,
|
|
1015
|
+
});
|
|
765
1016
|
}
|
|
766
|
-
const created = await this.create(ctx, p, held, fence, signal
|
|
767
|
-
// had_instance
|
|
768
|
-
|
|
1017
|
+
const created = await this.create(ctx, p, held, row, fence, signal);
|
|
1018
|
+
// had_instance only ever becomes true: any earlier instance (even if a rebuild failed since)
|
|
1019
|
+
// makes this one a restart on the kept files
|
|
1020
|
+
return {
|
|
1021
|
+
...created,
|
|
1022
|
+
info: { fresh: true, restarted: row.had_instance && row.fs_status === "ready" },
|
|
1023
|
+
fence,
|
|
1024
|
+
};
|
|
769
1025
|
}
|
|
770
1026
|
/**
|
|
771
|
-
*
|
|
772
|
-
*
|
|
1027
|
+
* Under the lease: fenced reset to `creating`, the file system when there is none
|
|
1028
|
+
* (`createFileSystem`, its handle written fenced), then the instance on it, then a fenced write
|
|
1029
|
+
* of the instance handle. `FileSystemGoneError` from `create` records `fs_status = 'lost'` and
|
|
1030
|
+
* throws `environment.files_lost`. When the instance handle cannot be written (lease lost), the
|
|
1031
|
+
* new instance is destroyed (best effort) before {@link SandboxLockLostError} is thrown.
|
|
773
1032
|
*/
|
|
774
|
-
async create(ctx, p, held, fence, signal
|
|
775
|
-
const id = ctx.
|
|
1033
|
+
async create(ctx, p, held, row, fence, signal) {
|
|
1034
|
+
const id = ctx.environmentId;
|
|
1035
|
+
const name = environmentResourceName(id);
|
|
776
1036
|
await this.fenced(held, id, fence, {
|
|
777
1037
|
org_id: ctx.orgId,
|
|
778
1038
|
provider: p.plugin,
|
|
@@ -783,21 +1043,45 @@ let SandboxService = class SandboxService {
|
|
|
783
1043
|
suspended_at: null,
|
|
784
1044
|
destroyed_at: null,
|
|
785
1045
|
sweep_failed_at: null,
|
|
786
|
-
...extra,
|
|
787
1046
|
});
|
|
1047
|
+
// the org's effective limits apply at creation only, so a changed setting reaches only
|
|
1048
|
+
// instances created afterwards
|
|
1049
|
+
const limits = await this.orgLimits(ctx.orgId);
|
|
788
1050
|
const cfg = this.config.get().sandbox;
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
1051
|
+
let fs = row.fs_status === "ready" && (row.provider === null || row.provider === p.plugin)
|
|
1052
|
+
? row.fs_handle
|
|
1053
|
+
: null;
|
|
1054
|
+
if (fs === null) {
|
|
1055
|
+
fs = await p.impl.createFileSystem({ name, limits, signal });
|
|
1056
|
+
// A lost lease fails this write and leaves the file system alone: createFileSystem returns
|
|
1057
|
+
// the existing one for a name, so the new holder may already use it, files and all.
|
|
1058
|
+
await this.fenced(held, id, fence, {
|
|
1059
|
+
provider: p.plugin,
|
|
1060
|
+
fs_handle: JSON.stringify(fs),
|
|
1061
|
+
fs_status: "ready",
|
|
1062
|
+
});
|
|
1063
|
+
this.log.info("environment file system created", { environmentId: id, provider: p.plugin });
|
|
1064
|
+
}
|
|
1065
|
+
let handle;
|
|
1066
|
+
try {
|
|
1067
|
+
handle = await p.impl.create({
|
|
1068
|
+
name,
|
|
1069
|
+
...(cfg.image !== undefined && { image: cfg.image }),
|
|
1070
|
+
limits,
|
|
1071
|
+
fileSystem: fs,
|
|
1072
|
+
signal,
|
|
1073
|
+
});
|
|
1074
|
+
}
|
|
1075
|
+
catch (err) {
|
|
1076
|
+
if (!isFileSystemGoneError(err))
|
|
1077
|
+
throw err;
|
|
1078
|
+
this.log.warn("environment file system is gone at the provider; its files are lost", {
|
|
1079
|
+
environmentId: id,
|
|
1080
|
+
provider: p.plugin,
|
|
1081
|
+
});
|
|
1082
|
+
await this.fenced(held, id, fence, { fs_status: "lost", status: "none" });
|
|
1083
|
+
throw environmentFilesLost();
|
|
1084
|
+
}
|
|
801
1085
|
let createdAt;
|
|
802
1086
|
try {
|
|
803
1087
|
createdAt = await this.fenced(held, id, fence, {
|
|
@@ -810,20 +1094,48 @@ let SandboxService = class SandboxService {
|
|
|
810
1094
|
});
|
|
811
1095
|
}
|
|
812
1096
|
catch (e) {
|
|
813
|
-
//
|
|
814
|
-
//
|
|
815
|
-
//
|
|
1097
|
+
// Lease lost before the handle was recorded: destroy the new instance (best effort) so it
|
|
1098
|
+
// does not leak. A provider that reuses instances by name (Ruling M10) may then hit the one
|
|
1099
|
+
// the new holder just resumed or created; that heals: its next provider call throws
|
|
1100
|
+
// SandboxGoneError (or resume returns gone) and withSandbox rebuilds in place. The file
|
|
1101
|
+
// system is never touched here.
|
|
816
1102
|
await p.impl.destroy(handle, AbortSignal.timeout(SWEEP_OP_TIMEOUT_MS)).catch((err) => this.log.warn("cannot destroy sandbox created after the lock was lost", {
|
|
817
|
-
|
|
1103
|
+
environmentId: id,
|
|
818
1104
|
err,
|
|
819
1105
|
}));
|
|
820
1106
|
throw e;
|
|
821
1107
|
}
|
|
822
|
-
this.log.info("sandbox created", {
|
|
823
|
-
return { handle, createdAt: createdAt.getTime() };
|
|
1108
|
+
this.log.info("sandbox created", { environmentId: id, provider: p.plugin });
|
|
1109
|
+
return { handle, createdAt: createdAt.getTime(), fsId: fs.id };
|
|
1110
|
+
}
|
|
1111
|
+
/**
|
|
1112
|
+
* Destroys the file system another provider may have made under the environment's name without
|
|
1113
|
+
* the kernel recording it (`destroyFileSystemByName`; not there is success). That provider not
|
|
1114
|
+
* loaded, or a failure, is only warned (the files may leak there) so the environment stays
|
|
1115
|
+
* usable; an abort is thrown.
|
|
1116
|
+
*/
|
|
1117
|
+
async dropUnrecordedFiles(id, provider, signal) {
|
|
1118
|
+
const old = this.providers.lookup(provider);
|
|
1119
|
+
if (!old) {
|
|
1120
|
+
this.log.warn("environment file system may have been made by a provider that is not loaded; it may leak", { environmentId: id, provider });
|
|
1121
|
+
return;
|
|
1122
|
+
}
|
|
1123
|
+
try {
|
|
1124
|
+
await old.destroyFileSystemByName(environmentResourceName(id), signal);
|
|
1125
|
+
}
|
|
1126
|
+
catch (err) {
|
|
1127
|
+
if (signal.aborted)
|
|
1128
|
+
throw err;
|
|
1129
|
+
this.log.warn("cannot destroy the file system of the previous sandbox provider", {
|
|
1130
|
+
environmentId: id,
|
|
1131
|
+
provider,
|
|
1132
|
+
err,
|
|
1133
|
+
});
|
|
1134
|
+
}
|
|
824
1135
|
}
|
|
825
1136
|
/**
|
|
826
|
-
*
|
|
1137
|
+
* Destroys an instance of another provider with that provider: still loaded and destroyed →
|
|
1138
|
+
* false; otherwise warns and returns true (the record must be kept).
|
|
827
1139
|
*/
|
|
828
1140
|
async retire(id, provider, handle, signal) {
|
|
829
1141
|
const old = this.providers.lookup(provider);
|
|
@@ -834,48 +1146,86 @@ let SandboxService = class SandboxService {
|
|
|
834
1146
|
}
|
|
835
1147
|
catch (err) {
|
|
836
1148
|
this.log.warn("cannot destroy sandbox instance of the previous provider", {
|
|
837
|
-
|
|
1149
|
+
environmentId: id,
|
|
838
1150
|
provider,
|
|
839
1151
|
err,
|
|
840
1152
|
});
|
|
841
1153
|
return true;
|
|
842
1154
|
}
|
|
843
1155
|
}
|
|
844
|
-
this.log.warn("sandbox was created by another provider that is not loaded; its instance may leak", {
|
|
1156
|
+
this.log.warn("sandbox was created by another provider that is not loaded; its instance may leak", { environmentId: id, provider });
|
|
845
1157
|
return true;
|
|
846
1158
|
}
|
|
847
|
-
/**
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
1159
|
+
/**
|
|
1160
|
+
* Under the lease: destroys the row's instance (and a leaked one of an earlier provider). An
|
|
1161
|
+
* instance without a handle (a create that ended before the handle was recorded) is destroyed by
|
|
1162
|
+
* name. Returns the `prev_*` patch for instances that could not be destroyed.
|
|
1163
|
+
*/
|
|
1164
|
+
async destroyInstance(row, p, signal) {
|
|
1165
|
+
const id = row.environment_id;
|
|
1166
|
+
const owner = row.provider ?? p.plugin;
|
|
852
1167
|
let prev = {};
|
|
853
|
-
if (row.handle &&
|
|
1168
|
+
if (row.handle && owner === p.plugin) {
|
|
854
1169
|
await p.impl.destroy(row.handle, signal);
|
|
855
1170
|
}
|
|
856
1171
|
else if (row.handle && row.status !== "destroyed") {
|
|
857
|
-
if (await this.retire(id,
|
|
858
|
-
prev = { prev_provider:
|
|
1172
|
+
if (await this.retire(id, owner, row.handle, signal)) {
|
|
1173
|
+
prev = { prev_provider: owner, prev_handle: JSON.stringify(row.handle) };
|
|
859
1174
|
}
|
|
860
1175
|
}
|
|
861
|
-
else if (!row.handle && row.
|
|
862
|
-
//
|
|
863
|
-
|
|
1176
|
+
else if (!row.handle && row.status === "creating") {
|
|
1177
|
+
// the provider may have built it already: only the name finds it
|
|
1178
|
+
const impl = owner === p.plugin ? p.impl : this.providers.lookup(owner);
|
|
1179
|
+
await impl?.destroyByName(environmentResourceName(id), signal);
|
|
864
1180
|
}
|
|
865
1181
|
if (row.prev_handle && row.prev_provider && !("prev_handle" in prev)) {
|
|
866
1182
|
if (!(await this.retire(id, row.prev_provider, row.prev_handle, signal))) {
|
|
867
1183
|
prev = { prev_provider: null, prev_handle: null };
|
|
868
1184
|
}
|
|
869
1185
|
}
|
|
1186
|
+
return prev;
|
|
1187
|
+
}
|
|
1188
|
+
/**
|
|
1189
|
+
* Under the lease: destroys the row's file system with the provider that made it (by name when
|
|
1190
|
+
* no handle was recorded: `createFileSystem` may have finished before the kernel stopped). With
|
|
1191
|
+
* that provider not loaded it throws (the row must stay for a retry), or with `lenient` only
|
|
1192
|
+
* warns (the files leak at that provider).
|
|
1193
|
+
*/
|
|
1194
|
+
async destroyFiles(row, p, signal, lenient = false) {
|
|
1195
|
+
const id = row.environment_id;
|
|
1196
|
+
const owner = row.provider ?? p.plugin;
|
|
1197
|
+
const impl = owner === p.plugin ? p.impl : this.providers.lookup(owner);
|
|
1198
|
+
if (!impl) {
|
|
1199
|
+
if (row.fs_handle === null)
|
|
1200
|
+
return;
|
|
1201
|
+
if (!lenient) {
|
|
1202
|
+
throw new Error(`cannot destroy the file system of environment ${id}: its sandbox provider ${owner} is not loaded`);
|
|
1203
|
+
}
|
|
1204
|
+
this.log.warn("environment file system of a provider that is not loaded is leaked", {
|
|
1205
|
+
environmentId: id,
|
|
1206
|
+
provider: owner,
|
|
1207
|
+
});
|
|
1208
|
+
return;
|
|
1209
|
+
}
|
|
1210
|
+
if (row.fs_handle !== null)
|
|
1211
|
+
await impl.destroyFileSystem(row.fs_handle, signal);
|
|
1212
|
+
else
|
|
1213
|
+
await impl.destroyFileSystemByName(environmentResourceName(id), signal);
|
|
1214
|
+
}
|
|
1215
|
+
/**
|
|
1216
|
+
* Under the lease: `instance` destroys the instance and marks the row `destroyed` (the file
|
|
1217
|
+
* system stays); `environment` also destroys the file system and deletes the row.
|
|
1218
|
+
*/
|
|
1219
|
+
async destroyRow(row, p, held, mode) {
|
|
1220
|
+
const id = row.environment_id;
|
|
1221
|
+
const fence = { gen: row.generation };
|
|
1222
|
+
const signal = opSignal(held);
|
|
1223
|
+
const prev = await this.destroyInstance(row, p, signal);
|
|
1224
|
+
if (mode === "environment")
|
|
1225
|
+
await this.destroyFiles(row, p, signal);
|
|
870
1226
|
if (!(await held.renew()))
|
|
871
1227
|
throw new SandboxLockLostError(id);
|
|
872
|
-
|
|
873
|
-
.selectFrom("conversations")
|
|
874
|
-
.select("id")
|
|
875
|
-
.where("id", "=", id)
|
|
876
|
-
.executeTakeFirst();
|
|
877
|
-
const keepRow = conversation !== undefined || (prev.prev_handle ?? row.prev_handle) != null;
|
|
878
|
-
if (keepRow) {
|
|
1228
|
+
if (mode === "instance") {
|
|
879
1229
|
await this.fenced(held, id, fence, {
|
|
880
1230
|
status: "destroyed",
|
|
881
1231
|
destroyed_at: sql `now()`,
|
|
@@ -883,19 +1233,19 @@ let SandboxService = class SandboxService {
|
|
|
883
1233
|
sweep_failed_at: null,
|
|
884
1234
|
...prev,
|
|
885
1235
|
});
|
|
1236
|
+
this.log.info("sandbox instance destroyed", { environmentId: id });
|
|
1237
|
+
return;
|
|
886
1238
|
}
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
}
|
|
896
|
-
this.log.info("sandbox destroyed", { conversationId: id });
|
|
1239
|
+
const r = await this.db
|
|
1240
|
+
.deleteFrom("environment_instances")
|
|
1241
|
+
.where("environment_id", "=", id)
|
|
1242
|
+
.where("generation", "=", fence.gen)
|
|
1243
|
+
.executeTakeFirst();
|
|
1244
|
+
if (Number(r.numDeletedRows) !== 1)
|
|
1245
|
+
throw new SandboxLockLostError(id, "generation");
|
|
1246
|
+
this.log.info("environment destroyed", { environmentId: id });
|
|
897
1247
|
}
|
|
898
|
-
/**
|
|
1248
|
+
/** Due for suspension: made by this provider, running, and idle or running for too long. */
|
|
899
1249
|
suspendDue(now, plugin) {
|
|
900
1250
|
const lim = this.config.get().sandbox.limits;
|
|
901
1251
|
const idleBefore = new Date(now.getTime() - lim.suspendAfterIdleMinutes * 60_000);
|
|
@@ -906,24 +1256,28 @@ let SandboxService = class SandboxService {
|
|
|
906
1256
|
eb.or([eb("last_used_at", "<", idleBefore), eb("running_since", "<", runningBefore)]),
|
|
907
1257
|
]);
|
|
908
1258
|
}
|
|
909
|
-
/**
|
|
910
|
-
|
|
1259
|
+
/** The environment was deleted, or its org was: the instance and the files go. */
|
|
1260
|
+
environmentGone(eb) {
|
|
1261
|
+
return eb.or([
|
|
1262
|
+
eb.exists(eb
|
|
1263
|
+
.selectFrom("environments")
|
|
1264
|
+
.select("environments.id")
|
|
1265
|
+
.whereRef("environments.id", "=", "environment_instances.environment_id")
|
|
1266
|
+
.where("environments.deleted_at", "is not", null)),
|
|
1267
|
+
eb.exists(eb
|
|
1268
|
+
.selectFrom("orgs")
|
|
1269
|
+
.select("orgs.id")
|
|
1270
|
+
.whereRef("orgs.id", "=", "environment_instances.org_id")
|
|
1271
|
+
.where("orgs.deleted_at", "is not", null)),
|
|
1272
|
+
]);
|
|
1273
|
+
}
|
|
1274
|
+
/** Due for instance destruction: an instance (or a create in progress) unused for too long. */
|
|
1275
|
+
instanceDue(now) {
|
|
911
1276
|
const days = this.config.get().sandbox.limits.destroyAfterInactiveDays;
|
|
912
1277
|
const before = new Date(now.getTime() - days * 24 * 3_600_000);
|
|
913
1278
|
return (eb) => eb.and([
|
|
914
|
-
eb("status", "
|
|
915
|
-
eb
|
|
916
|
-
eb("last_used_at", "<", before),
|
|
917
|
-
eb.exists(eb
|
|
918
|
-
.selectFrom("orgs")
|
|
919
|
-
.select("orgs.id")
|
|
920
|
-
.whereRef("orgs.id", "=", "sandboxes.org_id")
|
|
921
|
-
.where("orgs.deleted_at", "is not", null)),
|
|
922
|
-
eb.not(eb.exists(eb
|
|
923
|
-
.selectFrom("conversations")
|
|
924
|
-
.select("conversations.id")
|
|
925
|
-
.whereRef("conversations.id", "=", "sandboxes.conversation_id"))),
|
|
926
|
-
]),
|
|
1279
|
+
eb("status", "in", ["creating", "running", "suspended"]),
|
|
1280
|
+
eb("last_used_at", "<", before),
|
|
927
1281
|
]);
|
|
928
1282
|
}
|
|
929
1283
|
};
|
|
@@ -934,6 +1288,7 @@ SandboxService = __decorate([
|
|
|
934
1288
|
__param(2, Inject(SandboxLeases)),
|
|
935
1289
|
__param(3, Inject(SandboxProviders)),
|
|
936
1290
|
__param(4, Inject(KernelConfigService)),
|
|
937
|
-
__param(5, Inject(KERNEL_LOGGER))
|
|
1291
|
+
__param(5, Inject(KERNEL_LOGGER)),
|
|
1292
|
+
__param(6, Inject(BackgroundTaskStore))
|
|
938
1293
|
], SandboxService);
|
|
939
1294
|
export { SandboxService };
|