@capybara-hq/kernel 0.1.0-rc.4 → 0.1.0-rc.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bootstrap.json +61 -5
- package/dist/admin/admin.controller.d.ts +39 -13
- package/dist/admin/admin.controller.js +33 -17
- package/dist/admin/extensions.controller.d.ts +16 -2
- package/dist/admin/extensions.controller.js +26 -6
- package/dist/admin/setup-lock.d.ts +1 -1
- package/dist/admin/setup-lock.js +1 -1
- package/dist/admin/setup.controller.d.ts +1 -1
- package/dist/admin/setup.controller.js +1 -1
- package/dist/admin/setup.service.d.ts +4 -4
- package/dist/admin/setup.service.js +8 -8
- package/dist/agents/agent-lookup.d.ts +16 -0
- package/dist/agents/agent-lookup.js +11 -0
- package/dist/agents/agents.controller.d.ts +76 -1
- package/dist/agents/agents.controller.js +90 -11
- package/dist/agents/agents.module.d.ts +4 -1
- package/dist/agents/agents.module.js +32 -6
- package/dist/agents/agents.service.d.ts +51 -22
- package/dist/agents/agents.service.js +172 -70
- package/dist/agents/definition.d.ts +49 -0
- package/dist/agents/definition.js +45 -0
- package/dist/agents/from-template.d.ts +106 -0
- package/dist/agents/from-template.js +649 -0
- package/dist/agents/from-template.service.d.ts +60 -0
- package/dist/agents/from-template.service.js +161 -0
- package/dist/agents/org-templates.d.ts +176 -0
- package/dist/agents/org-templates.js +827 -0
- package/dist/agents/pending-connector-requests.d.ts +77 -0
- package/dist/agents/pending-connector-requests.js +215 -0
- package/dist/agents/pending-connectors.controller.d.ts +39 -0
- package/dist/agents/pending-connectors.controller.js +91 -0
- package/dist/agents/pending-connectors.service.d.ts +62 -0
- package/dist/agents/pending-connectors.service.js +282 -0
- package/dist/agents/templates.controller.d.ts +86 -5
- package/dist/agents/templates.controller.js +171 -7
- package/dist/agents/templates.d.ts +127 -23
- package/dist/agents/templates.js +475 -52
- package/dist/background/background.module.js +8 -1
- package/dist/background/finish.d.ts +3 -2
- package/dist/background/launch-lock.d.ts +4 -3
- package/dist/background/launch-lock.js +4 -3
- package/dist/background/poll.job.d.ts +9 -6
- package/dist/background/poll.job.js +30 -22
- package/dist/background/session.d.ts +13 -2
- package/dist/background/session.js +13 -0
- package/dist/background/stop.d.ts +12 -6
- package/dist/background/stop.js +57 -39
- package/dist/background/store.d.ts +18 -6
- package/dist/background/store.js +12 -6
- package/dist/background/tasks.service.d.ts +15 -5
- package/dist/background/tasks.service.js +42 -17
- package/dist/background/wake-rules.d.ts +27 -18
- package/dist/background/wake-rules.js +52 -22
- package/dist/background/wake.d.ts +13 -11
- package/dist/background/wake.js +20 -14
- package/dist/catalog/admin-catalog.controller.d.ts +129 -30
- package/dist/catalog/admin-catalog.controller.js +213 -42
- package/dist/catalog/catalog-content.d.ts +58 -32
- package/dist/catalog/catalog-content.js +34 -8
- package/dist/catalog/catalog-import.module.d.ts +7 -0
- package/dist/catalog/catalog-import.module.js +60 -0
- package/dist/catalog/catalog-items.d.ts +111 -7
- package/dist/catalog/catalog-items.js +279 -13
- package/dist/catalog/catalog-scope.service.d.ts +260 -0
- package/dist/catalog/catalog-scope.service.js +1116 -0
- package/dist/catalog/catalog.module.d.ts +7 -9
- package/dist/catalog/catalog.module.js +41 -31
- package/dist/catalog/catalog.service.d.ts +101 -29
- package/dist/catalog/catalog.service.js +104 -106
- package/dist/catalog/cli.d.ts +11 -1
- package/dist/catalog/cli.js +26 -1
- package/dist/catalog/component-promotion.d.ts +96 -0
- package/dist/catalog/component-promotion.js +395 -0
- package/dist/catalog/component-scope.controller.d.ts +152 -0
- package/dist/catalog/component-scope.controller.js +287 -0
- package/dist/catalog/component-scope.service.d.ts +303 -0
- package/dist/catalog/component-scope.service.js +936 -0
- package/dist/catalog/component-state.d.ts +174 -0
- package/dist/catalog/component-state.js +437 -0
- package/dist/catalog/job-ids.d.ts +30 -0
- package/dist/catalog/job-ids.js +25 -0
- package/dist/catalog/marketplace-fetch.d.ts +1 -1
- package/dist/catalog/marketplaces.controller.d.ts +22 -5
- package/dist/catalog/marketplaces.controller.js +56 -7
- package/dist/catalog/marketplaces.service.d.ts +54 -14
- package/dist/catalog/marketplaces.service.js +71 -22
- package/dist/catalog/org-catalog.controller.d.ts +125 -34
- package/dist/catalog/org-catalog.controller.js +181 -65
- package/dist/catalog/org-catalog.service.d.ts +65 -33
- package/dist/catalog/org-catalog.service.js +324 -157
- package/dist/catalog/org-extension-switch.d.ts +41 -0
- package/dist/catalog/org-extension-switch.js +115 -0
- package/dist/catalog/org-extension.controller.d.ts +119 -0
- package/dist/catalog/org-extension.controller.js +266 -0
- package/dist/catalog/org-plugin-uploads.controller.d.ts +98 -0
- package/dist/catalog/org-plugin-uploads.controller.js +213 -0
- package/dist/catalog/org-plugin-uploads.d.ts +63 -0
- package/dist/catalog/org-plugin-uploads.js +116 -0
- package/dist/catalog/owned-items.d.ts +184 -0
- package/dist/catalog/owned-items.js +789 -0
- package/dist/catalog/owned-plugin-cleanup.d.ts +56 -0
- package/dist/catalog/owned-plugin-cleanup.js +194 -0
- package/dist/catalog/ownership.d.ts +172 -0
- package/dist/catalog/ownership.js +326 -0
- package/dist/catalog/platform-components.d.ts +221 -0
- package/dist/catalog/platform-components.js +761 -0
- package/dist/catalog/platform-migration.d.ts +115 -0
- package/dist/catalog/platform-migration.js +582 -0
- package/dist/catalog/plugin-import.d.ts +178 -34
- package/dist/catalog/plugin-import.js +622 -82
- package/dist/catalog/plugin-install.d.ts +185 -41
- package/dist/catalog/plugin-install.js +758 -126
- package/dist/catalog/plugin-upgrade.d.ts +260 -225
- package/dist/catalog/plugin-upgrade.js +1147 -1136
- package/dist/catalog/request-rules.d.ts +65 -0
- package/dist/catalog/request-rules.js +217 -0
- package/dist/catalog/requests.service.d.ts +193 -118
- package/dist/catalog/requests.service.js +526 -343
- package/dist/catalog/reserved-names.d.ts +8 -0
- package/dist/catalog/reserved-names.js +42 -0
- package/dist/catalog/scope-orgs.d.ts +21 -0
- package/dist/catalog/scope-orgs.js +67 -0
- package/dist/catalog/scope-state.d.ts +35 -0
- package/dist/catalog/scope-state.js +87 -0
- package/dist/catalog/scope-sync-workers.d.ts +7 -0
- package/dist/catalog/scope-sync-workers.js +29 -0
- package/dist/catalog/scope-sync.d.ts +176 -0
- package/dist/catalog/scope-sync.js +576 -0
- package/dist/catalog/scope.d.ts +42 -0
- package/dist/catalog/scope.js +42 -0
- package/dist/catalog/unshare.d.ts +22 -0
- package/dist/catalog/unshare.js +118 -0
- package/dist/channels/actions.js +1 -0
- package/dist/channels/cards.js +20 -2
- package/dist/channels/channels.module.js +2 -0
- package/dist/channels/inbound.d.ts +9 -1
- package/dist/channels/inbound.js +67 -5
- package/dist/channels/phrases.d.ts +35 -0
- package/dist/channels/phrases.js +29 -0
- package/dist/config/docs.js +3 -1
- package/dist/config/schema.d.ts +6 -2
- package/dist/config/schema.js +46 -22
- package/dist/conversations/access.d.ts +6 -4
- package/dist/conversations/access.js +27 -5
- package/dist/conversations/agent-tree.d.ts +46 -0
- package/dist/conversations/agent-tree.js +92 -0
- package/dist/conversations/conversations.controller.d.ts +43 -18
- package/dist/conversations/conversations.controller.js +79 -38
- package/dist/conversations/conversations.module.d.ts +1 -1
- package/dist/conversations/conversations.module.js +7 -7
- package/dist/conversations/conversations.service.d.ts +69 -12
- package/dist/conversations/conversations.service.js +106 -27
- package/dist/conversations/environment-status.d.ts +24 -0
- package/dist/conversations/environment-status.js +71 -0
- package/dist/conversations/event-log.d.ts +20 -5
- package/dist/conversations/event-log.js +60 -0
- package/dist/conversations/events.d.ts +84 -2
- package/dist/conversations/events.js +91 -17
- package/dist/conversations/queue.service.d.ts +1 -1
- package/dist/conversations/queue.service.js +5 -0
- package/dist/conversations/titles.d.ts +5 -3
- package/dist/conversations/titles.js +21 -8
- package/dist/conversations/tree.d.ts +17 -0
- package/dist/conversations/tree.js +42 -0
- package/dist/ctx/create-ctx.d.ts +5 -0
- package/dist/ctx/create-ctx.js +15 -5
- package/dist/db/kernel-schema.d.ts +362 -18
- package/dist/environments/access.d.ts +41 -0
- package/dist/environments/access.js +77 -0
- package/dist/environments/conversation-environments.d.ts +61 -0
- package/dist/environments/conversation-environments.js +170 -0
- package/dist/environments/dto.d.ts +25 -0
- package/dist/environments/dto.js +12 -0
- package/dist/environments/environments.controller.d.ts +33 -0
- package/dist/environments/environments.controller.js +110 -0
- package/dist/environments/environments.module.d.ts +8 -0
- package/dist/environments/environments.module.js +27 -0
- package/dist/environments/environments.service.d.ts +153 -0
- package/dist/environments/environments.service.js +541 -0
- package/dist/environments/legacy-cleanup.job.d.ts +34 -0
- package/dist/environments/legacy-cleanup.job.js +141 -0
- package/dist/errors.d.ts +82 -2
- package/dist/errors.js +82 -2
- package/dist/evals/reconstruct.d.ts +18 -9
- package/dist/evals/reconstruct.js +26 -7
- package/dist/evals/run-replay.d.ts +4 -2
- package/dist/evals/run-replay.js +16 -4
- package/dist/evals/step-replay.d.ts +2 -5
- package/dist/evals/step-replay.js +1 -1
- package/dist/evals/trajectory.service.d.ts +8 -3
- package/dist/evals/trajectory.service.js +26 -14
- package/dist/evals/worker.d.ts +5 -5
- package/dist/evals/worker.js +9 -3
- package/dist/extensions/admin-ops.d.ts +36 -10
- package/dist/extensions/admin-ops.js +65 -42
- package/dist/extensions/admin-view.d.ts +37 -10
- package/dist/extensions/admin-view.js +103 -20
- package/dist/extensions/bootstrap.d.ts +17 -4
- package/dist/extensions/bootstrap.js +70 -9
- package/dist/extensions/cli.d.ts +12 -0
- package/dist/extensions/cli.js +11 -2
- package/dist/extensions/deployment-config.d.ts +28 -5
- package/dist/extensions/deployment-config.js +48 -7
- package/dist/extensions/effective-config.d.ts +127 -0
- package/dist/extensions/effective-config.js +368 -0
- package/dist/extensions/extensions.module.js +8 -2
- package/dist/extensions/hooks.d.ts +37 -4
- package/dist/extensions/hooks.js +82 -16
- package/dist/extensions/host.d.ts +19 -1
- package/dist/extensions/host.js +30 -7
- package/dist/extensions/installs.d.ts +20 -8
- package/dist/extensions/installs.js +37 -15
- package/dist/extensions/org-config.d.ts +75 -6
- package/dist/extensions/org-config.js +184 -32
- package/dist/extensions/org-levels.d.ts +37 -0
- package/dist/extensions/org-levels.js +71 -0
- package/dist/extensions/org-overrides.d.ts +151 -0
- package/dist/extensions/org-overrides.js +524 -0
- package/dist/extensions/permission-view.d.ts +16 -4
- package/dist/extensions/permission-view.js +4 -4
- package/dist/extensions/plugin-items.d.ts +21 -0
- package/dist/extensions/plugin-items.js +42 -0
- package/dist/extensions/point-defaults.d.ts +12 -5
- package/dist/extensions/point-defaults.js +37 -9
- package/dist/extensions/resolve.d.ts +5 -1
- package/dist/extensions/resolve.js +15 -2
- package/dist/files/previewers.d.ts +8 -1
- package/dist/files/previewers.js +35 -6
- package/dist/http/api.module.js +23 -3
- package/dist/http/app.d.ts +11 -2
- package/dist/http/app.js +31 -5
- package/dist/identity/accounts.d.ts +29 -29
- package/dist/identity/accounts.js +47 -47
- package/dist/identity/audit.d.ts +2 -1
- package/dist/identity/auth.guard.d.ts +13 -5
- package/dist/identity/auth.guard.js +26 -9
- package/dist/identity/identity.module.d.ts +1 -1
- package/dist/identity/identity.module.js +4 -4
- package/dist/identity/me.controller.js +1 -1
- package/dist/identity/principal.d.ts +3 -3
- package/dist/identity/principal.js +3 -3
- package/dist/identity/settings.d.ts +22 -11
- package/dist/identity/settings.js +53 -18
- package/dist/identity/{enterprise-admin.d.ts → super-admin.d.ts} +5 -5
- package/dist/identity/{enterprise-admin.js → super-admin.js} +24 -24
- package/dist/inbox/inbox.module.d.ts +4 -1
- package/dist/inbox/inbox.module.js +7 -3
- package/dist/inbox/inbox.service.d.ts +62 -3
- package/dist/inbox/inbox.service.js +74 -4
- package/dist/inbox/org-notices.d.ts +63 -0
- package/dist/inbox/org-notices.js +91 -0
- package/dist/inbox/platform-inbox.controller.d.ts +23 -0
- package/dist/inbox/platform-inbox.controller.js +58 -0
- package/dist/inbox/platform-inbox.service.d.ts +151 -0
- package/dist/inbox/platform-inbox.service.js +263 -0
- package/dist/inbox/platform-notices.d.ts +19 -0
- package/dist/inbox/platform-notices.js +9 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/infra/cli/kernel-cli.module.js +9 -1
- package/dist/infra/config/kernel-config.service.js +21 -1
- package/dist/infra/database/xact-lock.d.ts +1 -1
- package/dist/infra/database/xact-lock.js +1 -1
- package/dist/infra/events/kernel-events.d.ts +11 -2
- package/dist/infra/queue/add-job-once.d.ts +14 -0
- package/dist/infra/queue/add-job-once.js +16 -0
- package/dist/kernel.module.js +6 -0
- package/dist/lifecycle/kernel-lifecycle.d.ts +1 -1
- package/dist/lifecycle/kernel-lifecycle.js +4 -4
- package/dist/models/prices.d.ts +5 -5
- package/dist/models/prices.js +4 -4
- package/dist/models/purposes.controller.d.ts +6 -4
- package/dist/models/purposes.controller.js +13 -7
- package/dist/models/purposes.d.ts +20 -9
- package/dist/models/purposes.js +17 -8
- package/dist/orgs/directory-sync.d.ts +14 -5
- package/dist/orgs/directory-sync.js +41 -17
- package/dist/orgs/offboarding.d.ts +9 -7
- package/dist/orgs/offboarding.js +28 -17
- package/dist/orgs/org-lock.d.ts +5 -0
- package/dist/orgs/org-lock.js +13 -0
- package/dist/orgs/org.guard.d.ts +10 -0
- package/dist/orgs/org.guard.js +43 -32
- package/dist/orgs/orgs.module.js +3 -0
- package/dist/orgs/orgs.service.d.ts +20 -7
- package/dist/orgs/orgs.service.js +32 -12
- package/dist/orgs/roles.d.ts +1 -1
- package/dist/orgs/roles.js +5 -1
- package/dist/orgs/settings.controller.d.ts +1 -1
- package/dist/orgs/settings.service.d.ts +15 -3
- package/dist/orgs/settings.service.js +6 -2
- package/dist/overrides.d.ts +3 -0
- package/dist/overrides.js +2 -0
- package/dist/platform/actor.d.ts +48 -0
- package/dist/platform/actor.js +100 -0
- package/dist/platform/agent-docs.d.ts +18 -0
- package/dist/platform/agent-docs.js +59 -0
- package/dist/platform/availability.d.ts +14 -0
- package/dist/platform/availability.js +43 -0
- package/dist/platform/errors.d.ts +11 -0
- package/dist/platform/errors.js +26 -0
- package/dist/platform/exclusions.d.ts +12 -0
- package/dist/platform/exclusions.js +1387 -0
- package/dist/platform/method.d.ts +6 -0
- package/dist/platform/method.js +15 -0
- package/dist/platform/operation.d.ts +92 -0
- package/dist/platform/operation.js +35 -0
- package/dist/platform/ops/agent-templates.ops.d.ts +14 -0
- package/dist/platform/ops/agent-templates.ops.js +92 -0
- package/dist/platform/ops/agents.ops.d.ts +19 -0
- package/dist/platform/ops/agents.ops.js +294 -0
- package/dist/platform/ops/connectors.ops.d.ts +15 -0
- package/dist/platform/ops/connectors.ops.js +126 -0
- package/dist/platform/ops/errors.d.ts +11 -0
- package/dist/platform/ops/errors.js +22 -0
- package/dist/platform/ops/extensions.ops.d.ts +19 -0
- package/dist/platform/ops/extensions.ops.js +116 -0
- package/dist/platform/ops/marketplaces.ops.d.ts +16 -0
- package/dist/platform/ops/marketplaces.ops.js +157 -0
- package/dist/platform/ops/paging.d.ts +14 -0
- package/dist/platform/ops/paging.js +20 -0
- package/dist/platform/ops/platform.ops.d.ts +23 -0
- package/dist/platform/ops/platform.ops.js +158 -0
- package/dist/platform/ops/plugins.ops.d.ts +33 -0
- package/dist/platform/ops/plugins.ops.js +377 -0
- package/dist/platform/ops/requests.ops.d.ts +14 -0
- package/dist/platform/ops/requests.ops.js +153 -0
- package/dist/platform/ops/role.d.ts +4 -0
- package/dist/platform/ops/role.js +10 -0
- package/dist/platform/ops/skills.ops.d.ts +13 -0
- package/dist/platform/ops/skills.ops.js +98 -0
- package/dist/platform/ops/summary.d.ts +10 -0
- package/dist/platform/ops/summary.js +18 -0
- package/dist/platform/platform-ops.module.d.ts +7 -0
- package/dist/platform/platform-ops.module.js +55 -0
- package/dist/platform/platform.module.d.ts +8 -0
- package/dist/platform/platform.module.js +25 -0
- package/dist/platform/redact-orgs.d.ts +7 -0
- package/dist/platform/redact-orgs.js +52 -0
- package/dist/platform/registry.d.ts +30 -0
- package/dist/platform/registry.js +111 -0
- package/dist/roles/worker-role.d.ts +6 -4
- package/dist/roles/worker-role.js +14 -7
- package/dist/roles/worker.module.d.ts +1 -1
- package/dist/roles/worker.module.js +9 -1
- package/dist/runtime/context/build.d.ts +6 -4
- package/dist/runtime/context/build.js +26 -7
- package/dist/runtime/context/claude-tools.d.ts +2 -1
- package/dist/runtime/context/claude-tools.js +5 -4
- package/dist/runtime/context/compaction.js +9 -0
- package/dist/runtime/context/reminders.d.ts +2 -0
- package/dist/runtime/context/reminders.js +2 -0
- package/dist/runtime/context/skills.js +9 -2
- package/dist/runtime/context/tool-changes.d.ts +27 -0
- package/dist/runtime/context/tool-changes.js +60 -0
- package/dist/runtime/hook-calls.js +2 -0
- package/dist/runtime/langgraph-loop.d.ts +3 -1
- package/dist/runtime/langgraph-loop.js +35 -17
- package/dist/runtime/loop.d.ts +32 -4
- package/dist/runtime/maintenance.d.ts +7 -4
- package/dist/runtime/maintenance.js +23 -8
- package/dist/runtime/model-call.d.ts +30 -8
- package/dist/runtime/model-call.js +69 -19
- package/dist/runtime/run-control.d.ts +11 -1
- package/dist/runtime/run-control.js +12 -0
- package/dist/runtime/run-drafts.d.ts +85 -0
- package/dist/runtime/run-drafts.js +230 -0
- package/dist/runtime/run-jobs.d.ts +3 -5
- package/dist/runtime/run-jobs.js +3 -7
- package/dist/runtime/run-queue.d.ts +37 -3
- package/dist/runtime/run-queue.js +188 -11
- package/dist/runtime/run-worker.d.ts +11 -5
- package/dist/runtime/run-worker.js +54 -31
- package/dist/runtime/runtime.module.js +3 -0
- package/dist/runtime/step-log.d.ts +9 -4
- package/dist/runtime/step-log.js +8 -5
- package/dist/runtime/stream.controller.d.ts +23 -1
- package/dist/runtime/stream.controller.js +60 -3
- package/dist/runtime/stream.d.ts +4 -1
- package/dist/runtime/stream.js +27 -3
- package/dist/runtime/subagents/child.d.ts +55 -7
- package/dist/runtime/subagents/child.js +105 -7
- package/dist/runtime/subagents/definition.d.ts +12 -3
- package/dist/runtime/subagents/definition.js +43 -29
- package/dist/runtime/subagents/deliveries.d.ts +71 -0
- package/dist/runtime/subagents/deliveries.js +228 -0
- package/dist/runtime/subagents/dispatch.d.ts +115 -26
- package/dist/runtime/subagents/dispatch.js +212 -141
- package/dist/runtime/subagents/limits.d.ts +24 -5
- package/dist/runtime/subagents/limits.js +40 -6
- package/dist/runtime/subagents/messages.d.ts +95 -0
- package/dist/runtime/subagents/messages.js +328 -0
- package/dist/runtime/subagents/preamble.d.ts +4 -4
- package/dist/runtime/subagents/preamble.js +9 -7
- package/dist/runtime/subagents/progress.d.ts +8 -6
- package/dist/runtime/subagents/progress.js +11 -6
- package/dist/runtime/subagents/render.d.ts +30 -0
- package/dist/runtime/subagents/render.js +53 -0
- package/dist/runtime/subagents/roster.d.ts +24 -7
- package/dist/runtime/subagents/roster.js +47 -11
- package/dist/runtime/subagents/skill-fork.d.ts +3 -2
- package/dist/runtime/subagents/skill-fork.js +8 -4
- package/dist/runtime/subagents/status.d.ts +14 -0
- package/dist/runtime/subagents/status.js +23 -0
- package/dist/runtime/subagents/subagents.service.d.ts +7 -4
- package/dist/runtime/subagents/subagents.service.js +11 -6
- package/dist/runtime/usage-report.js +17 -12
- package/dist/runtime/waits.d.ts +140 -37
- package/dist/runtime/waits.js +394 -106
- package/dist/sandbox/jobs.d.ts +40 -13
- package/dist/sandbox/jobs.js +57 -20
- package/dist/sandbox/leases.d.ts +37 -54
- package/dist/sandbox/leases.js +71 -115
- package/dist/sandbox/providers.d.ts +2 -2
- package/dist/sandbox/providers.js +1 -1
- package/dist/sandbox/sandbox.module.d.ts +3 -2
- package/dist/sandbox/sandbox.module.js +4 -3
- package/dist/sandbox/sandbox.service.d.ts +229 -134
- package/dist/sandbox/sandbox.service.js +630 -409
- package/dist/skills/catalog.d.ts +3 -2
- package/dist/skills/catalog.js +21 -43
- package/dist/skills/changes.controller.d.ts +7 -0
- package/dist/skills/changes.controller.js +19 -1
- package/dist/skills/invoke.js +8 -3
- package/dist/skills/package.d.ts +20 -1
- package/dist/skills/package.js +28 -1
- package/dist/skills/skills.controller.d.ts +39 -4
- package/dist/skills/skills.controller.js +88 -10
- package/dist/skills/skills.module.js +10 -1
- package/dist/skills/skills.service.d.ts +106 -26
- package/dist/skills/skills.service.js +421 -75
- package/dist/skills/usable.d.ts +81 -0
- package/dist/skills/usable.js +379 -0
- package/dist/tools/approvals.service.d.ts +20 -2
- package/dist/tools/approvals.service.js +23 -4
- package/dist/tools/builtin-names.d.ts +21 -5
- package/dist/tools/builtin-names.js +30 -5
- package/dist/tools/builtins/agent.d.ts +23 -21
- package/dist/tools/builtins/agent.js +62 -55
- package/dist/tools/builtins/builtin-tools.d.ts +6 -2
- package/dist/tools/builtins/builtin-tools.js +37 -12
- package/dist/tools/builtins/handback.d.ts +28 -0
- package/dist/tools/builtins/handback.js +53 -0
- package/dist/tools/builtins/platform.d.ts +34 -0
- package/dist/tools/builtins/platform.js +362 -0
- package/dist/tools/builtins/sandbox/bash.js +13 -3
- package/dist/tools/builtins/sandbox/common.d.ts +43 -24
- package/dist/tools/builtins/sandbox/common.js +80 -45
- package/dist/tools/builtins/sandbox/edit.js +2 -2
- package/dist/tools/builtins/sandbox/glob.js +2 -2
- package/dist/tools/builtins/sandbox/grep.js +2 -4
- package/dist/tools/builtins/sandbox/index.d.ts +1 -1
- package/dist/tools/builtins/sandbox/index.js +1 -1
- package/dist/tools/builtins/sandbox/read.js +1 -1
- package/dist/tools/builtins/sandbox/task-output.d.ts +14 -3
- package/dist/tools/builtins/sandbox/task-output.js +30 -5
- package/dist/tools/builtins/sandbox/task-stop.js +2 -1
- package/dist/tools/builtins/sandbox/write.js +2 -2
- package/dist/tools/builtins/send-message.d.ts +56 -0
- package/dist/tools/builtins/send-message.js +100 -0
- package/dist/tools/builtins/skill-load.d.ts +22 -14
- package/dist/tools/builtins/skill-load.js +86 -57
- package/dist/tools/builtins/skill-write.js +1 -1
- package/dist/tools/builtins/web/extract.js +5 -1
- package/dist/tools/builtins/web/fetch.js +1 -0
- package/dist/tools/builtins/web/search.js +1 -0
- package/dist/tools/connectors.controller.d.ts +10 -0
- package/dist/tools/connectors.controller.js +12 -0
- package/dist/tools/connectors.service.d.ts +46 -5
- package/dist/tools/connectors.service.js +94 -15
- package/dist/tools/extension-tools.controller.d.ts +14 -0
- package/dist/tools/extension-tools.controller.js +37 -0
- package/dist/tools/info.d.ts +42 -1
- package/dist/tools/info.js +64 -1
- package/dist/tools/org-tool-switches.d.ts +74 -0
- package/dist/tools/org-tool-switches.js +184 -0
- package/dist/tools/pipeline.d.ts +46 -40
- package/dist/tools/pipeline.js +187 -364
- package/dist/tools/policy.d.ts +1 -1
- package/dist/tools/policy.js +36 -16
- package/dist/tools/registry.d.ts +98 -5
- package/dist/tools/registry.js +87 -9
- package/dist/tools/tools.module.js +12 -2
- package/dist/ui/views.d.ts +2 -1
- package/dist/ui/views.js +17 -3
- package/package.json +3 -3
- package/src/migrations/0074_subagent_tree.sql +50 -0
- package/src/migrations/0075_plugin_scope.sql +61 -0
- package/src/migrations/0076_skills_created_by_nullable.sql +6 -0
- package/src/migrations/0077_scope_requests.sql +52 -0
- package/src/migrations/0078_scope_grant_switches.sql +18 -0
- package/src/migrations/0079_scope_live_install_switches.sql +19 -0
- package/src/migrations/0080_rename_enterprise_admin.sql +18 -0
- package/src/migrations/0081_platform_components.sql +129 -0
- package/src/migrations/0082_platform_migration.sql +33 -0
- package/src/migrations/0083_plugin_scope_sync_failures.sql +15 -0
- package/src/migrations/0084_org_owned_items.sql +85 -0
- package/src/migrations/0085_agent_templates_phase4.sql +45 -0
- package/src/migrations/0086_org_layering.sql +30 -0
- package/src/migrations/0087_approval_summary.sql +6 -0
- package/src/migrations/0088_environments.sql +80 -0
- package/dist/catalog/upgrade-targets.d.ts +0 -56
- package/dist/catalog/upgrade-targets.js +0 -90
- package/dist/conversations/sandbox-status.d.ts +0 -24
- package/dist/conversations/sandbox-status.js +0 -52
- package/dist/extensions/org-config.controller.d.ts +0 -32
- package/dist/extensions/org-config.controller.js +0 -70
|
@@ -8,7 +8,7 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
|
8
8
|
return function (target, key) { decorator(target, key, paramIndex); }
|
|
9
9
|
};
|
|
10
10
|
import { randomUUID } from "node:crypto";
|
|
11
|
-
import { isSandboxGoneError, } from "@capybara-hq/plugin-sdk";
|
|
11
|
+
import { isFileSystemGoneError, isSandboxGoneError, } from "@capybara-hq/plugin-sdk";
|
|
12
12
|
import { Inject, Injectable } from "@nestjs/common";
|
|
13
13
|
import { sql, } from "kysely";
|
|
14
14
|
import { BackgroundTaskStore } from "../background/store.js";
|
|
@@ -23,26 +23,32 @@ import { canonicalJson } from "../tools/canonical-json.js";
|
|
|
23
23
|
import { SandboxLeases, SandboxLockLostError, sandboxBusy, } from "./leases.js";
|
|
24
24
|
import { SandboxProviders } from "./providers.js";
|
|
25
25
|
// Moved to leases.ts (phase C); re-exported for existing importers
|
|
26
|
-
export {
|
|
26
|
+
export { SANDBOX_LOCK_PREFIX, SANDBOX_LOCK_TTL_MS, SandboxLockLostError, sandboxBusy, } from "./leases.js";
|
|
27
27
|
/** Margin (ms) of a holder record's TTL beyond the longest command: by then the in-sandbox limit has ended its commands. */
|
|
28
28
|
const HOLDER_TTL_MARGIN_MS = 10 * 60_000;
|
|
29
29
|
/** How long the kernel's own read-modify-write of one file waits for another holder's (ms, spec §10.4). */
|
|
30
30
|
const PATH_LEASE_WAIT_MS = 30_000;
|
|
31
|
-
/**
|
|
31
|
+
/** Time limit of one provider call in maintenance (suspend, destroy). */
|
|
32
32
|
const SWEEP_OP_TIMEOUT_MS = 120_000;
|
|
33
33
|
/** How long the suspension sweep holds the intent on a sandbox past `maxRunningMinutes` (ms). */
|
|
34
34
|
const SWEEP_INTENT_WAIT_MS = 1000;
|
|
35
|
-
/**
|
|
35
|
+
/** Rows one sweep handles at most, and the provider calls it runs at once. */
|
|
36
36
|
const SWEEP_BATCH = 200;
|
|
37
37
|
const SWEEP_CONCURRENCY = 4;
|
|
38
|
-
/**
|
|
39
|
-
const
|
|
38
|
+
/** How long `destroyEnvironment` / `resetFileSystem` wait for the exclusive lease (running commands). */
|
|
39
|
+
const EXCLUSIVE_WAIT_MS = 10 * 60_000;
|
|
40
40
|
/** The time limit of the exec that kills lost background tasks after a resume (ms). */
|
|
41
41
|
const PENDING_KILL_TIMEOUT_MS = 30_000;
|
|
42
42
|
/** A background task's session id as `echo $!` printed it; anything else is never put in a command. */
|
|
43
43
|
const SID_RE = /^[1-9][0-9]{0,9}$/;
|
|
44
|
-
/**
|
|
44
|
+
/** The signal of one provider call in maintenance: aborted on timeout or when the lease is lost. */
|
|
45
45
|
const opSignal = (lease) => AbortSignal.any([lease.signal, AbortSignal.timeout(SWEEP_OP_TIMEOUT_MS)]);
|
|
46
|
+
/** The provider-side name of an environment's file system and instance. */
|
|
47
|
+
export const environmentResourceName = (environmentId) => `cap-env-${environmentId}`;
|
|
48
|
+
/** The environment is missing, deleted, or not in the caller's org (404; ids cannot be probed). */
|
|
49
|
+
export const environmentNotFound = () => new ApiError("environment.not_found", "This environment does not exist or was deleted.");
|
|
50
|
+
/** The environment's file system is gone at the provider (409); never replaced silently. */
|
|
51
|
+
export const environmentFilesLost = () => new ApiError("environment.files_lost", "The environment's files are gone; reset its files from the environments page to start with an empty workspace.");
|
|
46
52
|
/**
|
|
47
53
|
* The row still records the instance `gone` (same handle and same `created_at`). Handles compare
|
|
48
54
|
* canonically: a jsonb round trip reorders a multi-key handle's keys (final review m9).
|
|
@@ -52,42 +58,31 @@ export const isInstance = (row, gone) => gone !== null &&
|
|
|
52
58
|
canonicalJson(row.handle) === canonicalJson(gone.handle) &&
|
|
53
59
|
row.created_at.getTime() === gone.createdAt;
|
|
54
60
|
/**
|
|
55
|
-
*
|
|
56
|
-
*
|
|
61
|
+
* `impl` with every `exec` labelled with the holder's tag (and `actor` when given); `before` runs
|
|
62
|
+
* first (it refreshes the holder's record, so a callback that runs several long commands in a row
|
|
63
|
+
* stays recorded).
|
|
57
64
|
*/
|
|
58
|
-
|
|
59
|
-
return impl.concurrentExec === true && typeof impl.killTagged === "function" ? max : 1;
|
|
60
|
-
}
|
|
61
|
-
/**
|
|
62
|
-
* `impl` with every `exec` labelled with the holder's tag; `before` runs first (it refreshes the
|
|
63
|
-
* holder's record, so a callback that runs several long commands in a row stays recorded).
|
|
64
|
-
*/
|
|
65
|
-
function tagged(impl, tag, before) {
|
|
65
|
+
function tagged(impl, tag, actor, before) {
|
|
66
66
|
const wrapped = Object.create(impl);
|
|
67
67
|
Object.defineProperty(wrapped, "exec", {
|
|
68
68
|
value: async (h, args) => {
|
|
69
69
|
await before();
|
|
70
|
-
return impl.exec(h, { ...args, tag });
|
|
70
|
+
return impl.exec(h, { ...args, tag, ...(actor !== undefined && { actor }) });
|
|
71
71
|
},
|
|
72
72
|
});
|
|
73
73
|
return wrapped;
|
|
74
74
|
}
|
|
75
75
|
/**
|
|
76
|
-
*
|
|
77
|
-
* - `runtime.maxRunSeconds`
|
|
78
|
-
*
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
76
|
+
* Sandbox config problems to tell operators at startup (English log lines):
|
|
77
|
+
* - `runtime.maxRunSeconds` below `sandbox.limits.maxCommandSeconds`: long commands are cut by the
|
|
78
|
+
* run time limit (Ruling M20);
|
|
79
|
+
* - `sandbox.limits.egress` is open but the current provider plugin has no `allowOpenEgress`: every
|
|
80
|
+
* instance creation is refused.
|
|
81
|
+
* With both time limits at their defaults (900 < 7200, a known trade-off) it is only info; warn
|
|
82
|
+
* when an operator changed one and they still disagree.
|
|
82
83
|
*/
|
|
83
|
-
export function sandboxConfigWarnings(cfg, providerPlugin
|
|
84
|
+
export function sandboxConfigWarnings(cfg, providerPlugin) {
|
|
84
85
|
const out = [];
|
|
85
|
-
if (impl && (impl.concurrentExec === true) !== (typeof impl.killTagged === "function")) {
|
|
86
|
-
out.push({
|
|
87
|
-
level: "warn",
|
|
88
|
-
message: `the sandbox plugin ${providerPlugin ?? ""} declares only one of concurrentExec and killTagged; concurrent commands need both, so its sandboxes run one command at a time`,
|
|
89
|
-
});
|
|
90
|
-
}
|
|
91
86
|
if (cfg.sandbox.limits.egress === "open" && providerPlugin !== undefined) {
|
|
92
87
|
const pc = cfg.extensionConfig[providerPlugin];
|
|
93
88
|
if (pc?.allowOpenEgress !== true) {
|
|
@@ -108,8 +103,13 @@ export function sandboxConfigWarnings(cfg, providerPlugin, impl) {
|
|
|
108
103
|
}
|
|
109
104
|
return out;
|
|
110
105
|
}
|
|
111
|
-
/**
|
|
106
|
+
/** No sandbox provider (503). */
|
|
112
107
|
export const sandboxUnavailable = () => new ApiError("sandbox.unavailable", "Sandbox is not available in this deployment; install a sandbox plugin");
|
|
108
|
+
/**
|
|
109
|
+
* The environment's files are kept by a provider that is not the current one (503); its files are
|
|
110
|
+
* not migrated, so it stays unusable until that provider is back (spec §4.2 item 13).
|
|
111
|
+
*/
|
|
112
|
+
const otherProvider = (provider) => new ApiError("environment.unavailable", `This environment's files are kept by the sandbox provider ${provider}, which is not the current provider; it can be used again once ${provider} is the deployment's sandbox provider`);
|
|
113
113
|
const abortReason = (signal) => signal.reason ?? new DOMException("This operation was aborted", "AbortError");
|
|
114
114
|
function sleep(ms, signal) {
|
|
115
115
|
return new Promise((resolve, reject) => {
|
|
@@ -128,7 +128,7 @@ function sleep(ms, signal) {
|
|
|
128
128
|
signal.addEventListener("abort", onAbort, { once: true });
|
|
129
129
|
});
|
|
130
130
|
}
|
|
131
|
-
/**
|
|
131
|
+
/** Runs `fn` on `items`, `limit` at a time. */
|
|
132
132
|
async function inPool(items, limit, fn) {
|
|
133
133
|
let next = 0;
|
|
134
134
|
const worker = async () => {
|
|
@@ -140,43 +140,56 @@ async function inPool(items, limit, fn) {
|
|
|
140
140
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
|
141
141
|
}
|
|
142
142
|
/**
|
|
143
|
-
*
|
|
143
|
+
* The kernel's sandbox service over the `sandbox@2` provider, keyed by **environment** (spec
|
|
144
|
+
* 2026-11-07-environments-design §4.2, §5.1): one file system kept until the environment is
|
|
145
|
+
* deleted, plus one instance attached to it that comes and goes; all conversations bound to the
|
|
146
|
+
* environment (and their sub-agents) share both.
|
|
144
147
|
*
|
|
145
|
-
* -
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
*
|
|
155
|
-
*
|
|
156
|
-
*
|
|
157
|
-
*
|
|
158
|
-
*
|
|
159
|
-
*
|
|
160
|
-
*
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
-
*
|
|
164
|
-
*
|
|
165
|
-
*
|
|
166
|
-
*
|
|
167
|
-
*
|
|
168
|
-
*
|
|
169
|
-
* `
|
|
170
|
-
*
|
|
171
|
-
*
|
|
172
|
-
*
|
|
173
|
-
*
|
|
148
|
+
* - **Lazy creation**: the first {@link withSandbox} creates the file system
|
|
149
|
+
* (`createFileSystem`, name `cap-env-<environmentId>`) and then the instance on it; state lives
|
|
150
|
+
* in `environment_instances`. A file system gone at the provider is never replaced silently:
|
|
151
|
+
* `fs_status` becomes `lost` and uses fail with `environment.files_lost` until
|
|
152
|
+
* {@link resetFileSystem}.
|
|
153
|
+
* - **Leases** (phase C, spec §10, {@link SandboxLeases}), keyed by environment id: using an
|
|
154
|
+
* instance recorded `running` with the current provider takes a **shared** lease, so up to
|
|
155
|
+
* `sandbox.maxConcurrentCommands` holders (all sessions of the environment) run at once. Every
|
|
156
|
+
* lifecycle change (create, resume, rebuild, suspend, destroy, reset) takes the **exclusive**
|
|
157
|
+
* lease, alone; a caller that needs one on the use path sets the intent so new shared leases wait
|
|
158
|
+
* for it, and downgrades to a shared lease to run its own command once the instance runs.
|
|
159
|
+
* Waiting is capped at `sandbox.lockWaitSeconds` (`sandbox.busy`); two replicas using it for the
|
|
160
|
+
* first time at once still build only one.
|
|
161
|
+
* - **Fencing**: exclusive holders read `generation` and compare-and-increment it with every state
|
|
162
|
+
* write ({@link SandboxLockLostError}); every acquisition (shared ones too) bumps it first, so a
|
|
163
|
+
* stale exclusive holder's later writes fail. Shared holders write nothing else but
|
|
164
|
+
* `last_used_at` (monotonic). A lost lease aborts the signal passed to `fn` and to maintenance
|
|
165
|
+
* provider calls.
|
|
166
|
+
* - **Suspension and destruction**: the sweeps (`SandboxSweepJobs`, crons) call
|
|
167
|
+
* {@link suspendIdle} / {@link destroyInactive}; they try the exclusive lease once and skip an
|
|
168
|
+
* environment with any holder; one past `maxRunningMinutes` gets the intent for a short window
|
|
169
|
+
* (about a second) so the suspension can get in when its commands end. Inactivity destroys only
|
|
170
|
+
* the instance (files kept); a deleted environment or org loses its files too.
|
|
171
|
+
* - **Transparent recovery**: a suspended instance is resumed on next use; when `resume` returns
|
|
172
|
+
* `gone`, the instance was destroyed, or `fn` throws `SandboxGoneError` (retried once), a new
|
|
173
|
+
* instance is created on the kept files and `restarted` is true. Several holders hitting `gone`
|
|
174
|
+
* together rebuild once: the others find the new instance.
|
|
175
|
+
* - **Takeover** (spec §10.5; environments spec §4.2): every holder records itself in
|
|
176
|
+
* `capk:sandbox-holders:<env>` (again before each `exec`) and its commands carry its lease token
|
|
177
|
+
* as `tag`. An acquirer that finds a record without a live lease (a crashed holder) kills only
|
|
178
|
+
* that holder's commands (`killTagged`): other conversations of the environment may be running
|
|
179
|
+
* commands. `killAll` runs only under the exclusive lease, before the downgrade, on an instance
|
|
180
|
+
* this holder just resumed from `suspended` (nobody else can hold it; a provider that keeps
|
|
181
|
+
* memory across a pause brings old commands back); it also covers crashed holders' records,
|
|
182
|
+
* which it claims. A freshly created instance has nothing to kill. Failures are only warned (the
|
|
183
|
+
* in-sandbox time limit is the backstop).
|
|
174
184
|
* - **Instance identity**: a holder that hits `gone` names the instance by its handle and the
|
|
175
185
|
* row's `created_at` (set by the fenced write after each `create`); the handle alone is the same
|
|
176
186
|
* after a rebuild on providers whose handle is the instance name (sandbox-docker).
|
|
177
187
|
*
|
|
178
|
-
*
|
|
179
|
-
*
|
|
188
|
+
* Files kept by another provider than the current one make the environment unusable
|
|
189
|
+
* (`environment.unavailable`) until that provider is back; deleting it still works while that
|
|
190
|
+
* provider is loaded ({@link SandboxProviders.lookup}). Without a provider {@link available} is false and
|
|
191
|
+
* uses throw `sandbox.unavailable` (503). No in-process state: replicas share it through PG and
|
|
192
|
+
* Redis.
|
|
180
193
|
*/
|
|
181
194
|
let SandboxService = class SandboxService {
|
|
182
195
|
db;
|
|
@@ -189,7 +202,7 @@ let SandboxService = class SandboxService {
|
|
|
189
202
|
constructor(db,
|
|
190
203
|
/** The record of the instance each holder last used (`capk:sandbox-seen:…`). */
|
|
191
204
|
redis, leases,
|
|
192
|
-
/** The `sandbox@
|
|
205
|
+
/** The `sandbox@2` provider (read at every call: providers are ready after the extension load). */
|
|
193
206
|
providers, config, log,
|
|
194
207
|
/** Background tasks lost while their process may live on (killed after a resume). */
|
|
195
208
|
background) {
|
|
@@ -205,23 +218,42 @@ let SandboxService = class SandboxService {
|
|
|
205
218
|
get pollMs() {
|
|
206
219
|
return this.leases.pollMs;
|
|
207
220
|
}
|
|
208
|
-
/**
|
|
221
|
+
/** Whether there is a sandbox provider. */
|
|
209
222
|
get available() {
|
|
210
223
|
return this.providers.current() !== undefined;
|
|
211
224
|
}
|
|
212
|
-
/** Concurrent holders
|
|
213
|
-
|
|
214
|
-
return
|
|
225
|
+
/** Concurrent holders of one environment (`sandbox.maxConcurrentCommands`, read at every use). */
|
|
226
|
+
get cap() {
|
|
227
|
+
return this.config.get().sandbox.maxConcurrentCommands;
|
|
215
228
|
}
|
|
216
229
|
/**
|
|
217
|
-
*
|
|
218
|
-
*
|
|
230
|
+
* The org's effective sandbox limits (spec §8.4): the deployment's `sandbox.limits`, egress
|
|
231
|
+
* narrowed by the org setting `sandboxEgress` (Ruling M19), cpu / memory per the super admin's
|
|
232
|
+
* org override (C19 / Ruling M16). Instances are created with them; phase 1 environments use
|
|
233
|
+
* them unchanged.
|
|
219
234
|
*/
|
|
220
|
-
async
|
|
235
|
+
async orgLimits(orgId) {
|
|
236
|
+
const cfg = this.config.get().sandbox;
|
|
237
|
+
const egress = effectiveSandboxEgress(cfg.limits, (await readOrgSettings(this.db.db, orgId)).sandboxEgress);
|
|
238
|
+
const sized = effectiveSandboxLimits({ ...cfg.limits, ...egress }, await readSandboxLimitOverride(this.db.db, orgId));
|
|
239
|
+
return {
|
|
240
|
+
cpu: sized.cpu,
|
|
241
|
+
memoryMb: sized.memoryMb,
|
|
242
|
+
diskMb: sized.diskMb,
|
|
243
|
+
egress: sized.egress,
|
|
244
|
+
allowHosts: sized.allowHosts,
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* The environment's instance state as shown to people (read only, no lease): `running` /
|
|
249
|
+
* `suspended` and the last use. No instance, a destroyed one, or one stuck in `creating` (being
|
|
250
|
+
* created, or a failed create the next use retries) is null.
|
|
251
|
+
*/
|
|
252
|
+
async status(orgId, environmentId) {
|
|
221
253
|
const row = await this.db
|
|
222
|
-
.selectFrom("
|
|
254
|
+
.selectFrom("environment_instances")
|
|
223
255
|
.select(["status", "last_used_at"])
|
|
224
|
-
.where("
|
|
256
|
+
.where("environment_id", "=", environmentId)
|
|
225
257
|
.where("org_id", "=", orgId)
|
|
226
258
|
.executeTakeFirst();
|
|
227
259
|
if (!row || (row.status !== "running" && row.status !== "suspended"))
|
|
@@ -229,15 +261,15 @@ let SandboxService = class SandboxService {
|
|
|
229
261
|
return { status: row.status, lastUsedAt: row.last_used_at };
|
|
230
262
|
}
|
|
231
263
|
/**
|
|
232
|
-
* The instance the
|
|
233
|
-
*
|
|
234
|
-
*
|
|
264
|
+
* The instance the environment records now (its `created_at`), while it has one that runs or is
|
|
265
|
+
* suspended; null otherwise. Read without a lease: a caller inside `withSandbox` reads the
|
|
266
|
+
* instance its own lease keeps.
|
|
235
267
|
*/
|
|
236
|
-
async currentInstance(
|
|
268
|
+
async currentInstance(environmentId) {
|
|
237
269
|
const row = await this.db
|
|
238
|
-
.selectFrom("
|
|
270
|
+
.selectFrom("environment_instances")
|
|
239
271
|
.select(["status", "handle", "created_at"])
|
|
240
|
-
.where("
|
|
272
|
+
.where("environment_id", "=", environmentId)
|
|
241
273
|
.executeTakeFirst();
|
|
242
274
|
if (!row || row.handle === null)
|
|
243
275
|
return null;
|
|
@@ -246,9 +278,9 @@ let SandboxService = class SandboxService {
|
|
|
246
278
|
return { createdAt: row.created_at };
|
|
247
279
|
}
|
|
248
280
|
/**
|
|
249
|
-
* Runs `fn` on the
|
|
281
|
+
* Runs `fn` on the environment's instance only while it runs the instance created at
|
|
250
282
|
* `instance.createdAt` (background tasks, spec 2026-11-05-background-bash-design §4.3). It never
|
|
251
|
-
* creates, resumes or rebuilds
|
|
283
|
+
* creates, resumes or rebuilds one: an instance that is missing, suspended, being created or
|
|
252
284
|
* destroyed (or recorded with another provider) is `not_running`, one that records another
|
|
253
285
|
* instance is `other_instance`. Otherwise `fn` runs under a shared lease, as a `withSandbox`
|
|
254
286
|
* holder does (its tag on every exec, crashed holders' commands killed), and `last_used_at`
|
|
@@ -257,19 +289,18 @@ let SandboxService = class SandboxService {
|
|
|
257
289
|
* to `fn` aborts with `signal` or when the lease is lost. Errors of `fn` (`SandboxGoneError`
|
|
258
290
|
* too) are thrown as they are.
|
|
259
291
|
*/
|
|
260
|
-
async ifRunning(
|
|
292
|
+
async ifRunning(environmentId, instance, fn, signal = new AbortController().signal) {
|
|
261
293
|
const p = this.require();
|
|
262
|
-
const
|
|
263
|
-
const cap = this.effectiveCap(p);
|
|
294
|
+
const env = environmentId;
|
|
264
295
|
const deadline = Date.now() + this.config.get().sandbox.lockWaitSeconds * 1000;
|
|
265
296
|
const token = randomUUID();
|
|
266
297
|
for (;;) {
|
|
267
298
|
if (signal.aborted)
|
|
268
299
|
throw abortReason(signal);
|
|
269
300
|
const row = await this.db
|
|
270
|
-
.selectFrom("
|
|
301
|
+
.selectFrom("environment_instances")
|
|
271
302
|
.selectAll()
|
|
272
|
-
.where("
|
|
303
|
+
.where("environment_id", "=", env)
|
|
273
304
|
.executeTakeFirst();
|
|
274
305
|
if (!row)
|
|
275
306
|
return { kind: "not_running", status: "gone" };
|
|
@@ -281,23 +312,21 @@ let SandboxService = class SandboxService {
|
|
|
281
312
|
// the instance this check saw: its handle plus `created_at` (never `generation`, which
|
|
282
313
|
// every lease bumps)
|
|
283
314
|
const want = { handle: row.handle, createdAt: row.created_at.getTime() };
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
await this.leave(conv, e);
|
|
296
|
-
}
|
|
315
|
+
const lease = await this.leases.tryShared(env, this.cap, token);
|
|
316
|
+
if (lease) {
|
|
317
|
+
const e = await this.asShared({ orgId: row.org_id, environmentId: env }, p, lease, (r) => this.usable(r, p, null) && isInstance(r, want), signal);
|
|
318
|
+
if (e) {
|
|
319
|
+
try {
|
|
320
|
+
const impl = tagged(p.impl, lease.token, undefined, () => this.register(env, lease.token));
|
|
321
|
+
const sig = AbortSignal.any([signal, lease.signal]);
|
|
322
|
+
return { kind: "ran", value: await fn(impl, e.handle, sig) };
|
|
323
|
+
}
|
|
324
|
+
finally {
|
|
325
|
+
await this.leave(env, e);
|
|
297
326
|
}
|
|
298
|
-
// the row changed between the check and the lease: classify it again
|
|
299
|
-
continue;
|
|
300
327
|
}
|
|
328
|
+
// the row changed between the check and the lease: classify it again
|
|
329
|
+
continue;
|
|
301
330
|
}
|
|
302
331
|
if (Date.now() + this.pollMs > deadline)
|
|
303
332
|
throw sandboxBusy();
|
|
@@ -305,90 +334,97 @@ let SandboxService = class SandboxService {
|
|
|
305
334
|
}
|
|
306
335
|
}
|
|
307
336
|
/**
|
|
308
|
-
* A short lease on one path of the
|
|
309
|
-
*
|
|
310
|
-
*
|
|
337
|
+
* A short lease on one path of the environment around the kernel's own read-modify-write of
|
|
338
|
+
* that file (spec §10.4); waits up to 30 s, then throws `SandboxPathBusyError`. Its `signal`
|
|
339
|
+
* aborts when a renewal finds it lost.
|
|
311
340
|
*/
|
|
312
|
-
pathLease(
|
|
313
|
-
return this.leases.pathLease(
|
|
341
|
+
pathLease(environmentId, path, signal) {
|
|
342
|
+
return this.leases.pathLease(environmentId, path, signal, PATH_LEASE_WAIT_MS);
|
|
314
343
|
}
|
|
315
344
|
/**
|
|
316
|
-
*
|
|
317
|
-
*
|
|
318
|
-
* `last_used_at` → release.
|
|
319
|
-
* -
|
|
320
|
-
*
|
|
321
|
-
* - The
|
|
322
|
-
*
|
|
323
|
-
*
|
|
324
|
-
* - `
|
|
325
|
-
*
|
|
326
|
-
*
|
|
327
|
-
*
|
|
345
|
+
* Runs `fn` on the environment's instance: take a lease → make sure the instance runs (file
|
|
346
|
+
* system, create / resume / rebuild, under the exclusive lease) → `fn` under a shared lease →
|
|
347
|
+
* update `last_used_at` → release.
|
|
348
|
+
* - Waiting longer than `sandbox.lockWaitSeconds` throws `sandbox.busy` (409); `signal` aborts
|
|
349
|
+
* the wait and the creation / resume.
|
|
350
|
+
* - The fourth argument of `fn` combines `signal` with the lease's: use it for commands, so they
|
|
351
|
+
* stop when the lease is lost.
|
|
352
|
+
* - The provider passed to `fn` labels every `exec` with this holder's tag and `ctx.actor`.
|
|
353
|
+
* - When `fn` throws `SandboxGoneError` (the instance is gone) the instance is rebuilt on the
|
|
354
|
+
* kept files (or the one another holder rebuilt is taken) and `fn` runs once more with
|
|
355
|
+
* `restarted: true`; a second `gone` is thrown. Other errors are thrown as they are.
|
|
356
|
+
* - Lost files throw `environment.files_lost` (409).
|
|
357
|
+
* - When `ctx.holderConversationId` last used another instance (another holder rebuilt it
|
|
358
|
+
* meanwhile), `info.restarted` is true, so it learns its processes are gone (final review m5).
|
|
328
359
|
*/
|
|
329
360
|
async withSandbox(ctx, fn, signal) {
|
|
330
361
|
const p = this.require();
|
|
331
|
-
const
|
|
332
|
-
|
|
362
|
+
const env = ctx.environmentId;
|
|
363
|
+
const holder = ctx.holderConversationId;
|
|
364
|
+
let e = await this.seenBy(env, holder, await this.enter(ctx, p, signal, null));
|
|
333
365
|
try {
|
|
334
366
|
try {
|
|
335
|
-
return await this.runAs(e, p, fn, signal);
|
|
367
|
+
return await this.runAs(e, ctx, p, fn, signal);
|
|
336
368
|
}
|
|
337
369
|
catch (err) {
|
|
338
370
|
if (!isSandboxGoneError(err))
|
|
339
371
|
throw err;
|
|
340
|
-
this.log.info("sandbox instance is gone during use; recreating", {
|
|
341
|
-
conversationId: ctx.conversationId,
|
|
342
|
-
});
|
|
372
|
+
this.log.info("sandbox instance is gone during use; recreating", { environmentId: env });
|
|
343
373
|
const gone = { handle: e.handle, createdAt: e.createdAt };
|
|
344
374
|
const prev = e;
|
|
345
|
-
await this.leave(
|
|
346
|
-
e = await this.seenBy(
|
|
347
|
-
return await this.runAs({ ...e, info: { fresh: true,
|
|
375
|
+
await this.leave(env, prev);
|
|
376
|
+
e = await this.seenBy(env, holder, await this.enter(ctx, p, signal, gone));
|
|
377
|
+
return await this.runAs({ ...e, info: { fresh: true, restarted: true } }, ctx, p, fn, signal);
|
|
348
378
|
}
|
|
349
379
|
}
|
|
350
380
|
finally {
|
|
351
|
-
await this.leave(
|
|
381
|
+
await this.leave(env, e);
|
|
352
382
|
}
|
|
353
383
|
}
|
|
354
384
|
/**
|
|
355
|
-
* Records that `holder` uses the instance of `e` (`capk:sandbox-seen:<
|
|
356
|
-
*
|
|
357
|
-
*
|
|
385
|
+
* Records that `holder` uses the instance of `e` (`capk:sandbox-seen:<env>:<holder>` =
|
|
386
|
+
* `<created_at>:<file system id>`, with a TTL of the inactivity destruction plus a day). When the
|
|
387
|
+
* holder last used another instance on the same file system `e` is marked restarted; on another
|
|
388
|
+
* file system (the files were reset) it is not. Best effort: a Redis error only logs.
|
|
358
389
|
*/
|
|
359
|
-
async seenBy(
|
|
390
|
+
async seenBy(environmentId, holder, e) {
|
|
360
391
|
try {
|
|
361
392
|
const ttl = (this.config.get().sandbox.limits.destroyAfterInactiveDays + 1) * 86_400;
|
|
362
|
-
const
|
|
363
|
-
|
|
364
|
-
|
|
393
|
+
const seen = `${e.createdAt}:${e.fsId}`;
|
|
394
|
+
const prev = await this.redis.set(`capk:sandbox-seen:${environmentId}:${holder}`, seen, "EX", ttl, "GET");
|
|
395
|
+
if (prev !== null && prev !== seen) {
|
|
396
|
+
// `<createdAt>:<fsId>`: another file system (a reset) means a fresh start, not a restart on
|
|
397
|
+
// kept files; another instance on the same file system is a restart
|
|
398
|
+
const sameFiles = prev.slice(prev.indexOf(":") + 1) === e.fsId;
|
|
399
|
+
if (sameFiles !== e.info.restarted) {
|
|
400
|
+
return { ...e, info: { fresh: e.info.fresh, restarted: sameFiles } };
|
|
401
|
+
}
|
|
365
402
|
}
|
|
366
403
|
}
|
|
367
404
|
catch (err) {
|
|
368
405
|
this.log.warn("cannot record the sandbox instance a holder uses", {
|
|
369
|
-
|
|
406
|
+
environmentId,
|
|
370
407
|
holder,
|
|
371
408
|
err,
|
|
372
409
|
});
|
|
373
410
|
}
|
|
374
411
|
return e;
|
|
375
412
|
}
|
|
376
|
-
/** Runs `fn` as the holder `e`: its tag on every exec, its lease's signal. */
|
|
377
|
-
runAs(e, p, fn, signal) {
|
|
413
|
+
/** Runs `fn` as the holder `e`: its tag and the caller's actor on every exec, its lease's signal. */
|
|
414
|
+
runAs(e, ctx, p, fn, signal) {
|
|
378
415
|
const sig = AbortSignal.any([signal, e.lease.signal]);
|
|
379
|
-
const
|
|
380
|
-
const impl = tagged(p.impl, e.lease.token, () => this.register(
|
|
416
|
+
const env = ctx.environmentId;
|
|
417
|
+
const impl = tagged(p.impl, e.lease.token, ctx.actor, () => this.register(env, e.lease.token));
|
|
381
418
|
return fn(e.handle, impl, e.info, sig);
|
|
382
419
|
}
|
|
383
420
|
/**
|
|
384
421
|
* Becomes a holder: loops until it has a shared lease on a running instance, or `sandbox.busy`.
|
|
385
|
-
*
|
|
386
|
-
*
|
|
387
|
-
*
|
|
422
|
+
* An instance recorded `running` (with the current provider, not the `gone` instance) is used
|
|
423
|
+
* under a shared lease; otherwise the exclusive lease (with intent) makes it run and is
|
|
424
|
+
* downgraded. One token serves both, so the waiter passes its own intent.
|
|
388
425
|
*/
|
|
389
426
|
async enter(ctx, p, signal, gone) {
|
|
390
|
-
const
|
|
391
|
-
const cap = this.effectiveCap(p);
|
|
427
|
+
const env = ctx.environmentId;
|
|
392
428
|
const deadline = Date.now() + this.config.get().sandbox.lockWaitSeconds * 1000;
|
|
393
429
|
const token = randomUUID();
|
|
394
430
|
let intent = false;
|
|
@@ -397,11 +433,11 @@ let SandboxService = class SandboxService {
|
|
|
397
433
|
if (signal.aborted)
|
|
398
434
|
throw abortReason(signal);
|
|
399
435
|
const row = await this.readRow(ctx);
|
|
400
|
-
if (this.usable(row, p, gone)
|
|
401
|
-
const lease = await this.leases.tryShared(
|
|
436
|
+
if (this.usable(row, p, gone)) {
|
|
437
|
+
const lease = await this.leases.tryShared(env, this.cap, token);
|
|
402
438
|
if (lease) {
|
|
403
439
|
if (intent)
|
|
404
|
-
await this.clearIntent(
|
|
440
|
+
await this.clearIntent(env, token);
|
|
405
441
|
intent = false;
|
|
406
442
|
const e = await this.asShared(ctx, p, lease, (r) => this.usable(r, p, gone), signal);
|
|
407
443
|
if (e)
|
|
@@ -409,7 +445,7 @@ let SandboxService = class SandboxService {
|
|
|
409
445
|
}
|
|
410
446
|
}
|
|
411
447
|
else {
|
|
412
|
-
const lease = await this.leases.tryExclusive(
|
|
448
|
+
const lease = await this.leases.tryExclusive(env, { token, intent: true });
|
|
413
449
|
intent = lease === null;
|
|
414
450
|
if (lease)
|
|
415
451
|
return await this.asExclusive(ctx, p, lease, gone, signal);
|
|
@@ -421,26 +457,29 @@ let SandboxService = class SandboxService {
|
|
|
421
457
|
}
|
|
422
458
|
catch (err) {
|
|
423
459
|
if (intent)
|
|
424
|
-
await this.clearIntent(
|
|
460
|
+
await this.clearIntent(env, token);
|
|
425
461
|
throw err;
|
|
426
462
|
}
|
|
427
463
|
}
|
|
428
|
-
async clearIntent(
|
|
464
|
+
async clearIntent(env, token) {
|
|
429
465
|
await this.leases
|
|
430
|
-
.clearIntent(
|
|
431
|
-
.catch((err) => this.log.warn("cannot clear sandbox intent", {
|
|
466
|
+
.clearIntent(env, token)
|
|
467
|
+
.catch((err) => this.log.warn("cannot clear sandbox intent", { environmentId: env, err }));
|
|
432
468
|
}
|
|
433
469
|
async readRow(ctx) {
|
|
434
470
|
const row = await this.db
|
|
435
|
-
.selectFrom("
|
|
471
|
+
.selectFrom("environment_instances")
|
|
436
472
|
.selectAll()
|
|
437
|
-
.where("
|
|
473
|
+
.where("environment_id", "=", ctx.environmentId)
|
|
438
474
|
.executeTakeFirst();
|
|
439
|
-
if (row && row.org_id !== ctx.orgId)
|
|
440
|
-
throw
|
|
441
|
-
}
|
|
475
|
+
if (row && row.org_id !== ctx.orgId)
|
|
476
|
+
throw this.foreign(ctx);
|
|
442
477
|
return row;
|
|
443
478
|
}
|
|
479
|
+
/** The instance row belongs to another org: reported as not found, so ids cannot be probed. */
|
|
480
|
+
foreign(_ctx) {
|
|
481
|
+
return environmentNotFound();
|
|
482
|
+
}
|
|
444
483
|
/** Recorded running with the current provider, and not the instance found gone. */
|
|
445
484
|
usable(row, p, gone) {
|
|
446
485
|
return (row !== undefined &&
|
|
@@ -454,7 +493,7 @@ let SandboxService = class SandboxService {
|
|
|
454
493
|
* returns null), records the holder and kills crashed holders' commands.
|
|
455
494
|
*/
|
|
456
495
|
async asShared(ctx, p, lease, accept, signal) {
|
|
457
|
-
const
|
|
496
|
+
const env = ctx.environmentId;
|
|
458
497
|
try {
|
|
459
498
|
const row = await this.readRow(ctx);
|
|
460
499
|
if (!accept(row)) {
|
|
@@ -465,123 +504,124 @@ let SandboxService = class SandboxService {
|
|
|
465
504
|
// expired unnoticed (a stalled sweep) then fails its next write instead of suspending or
|
|
466
505
|
// destroying the instance under this holder. Shared holders never compare it.
|
|
467
506
|
await this.db
|
|
468
|
-
.updateTable("
|
|
507
|
+
.updateTable("environment_instances")
|
|
469
508
|
.set((eb) => ({ generation: eb("generation", "+", 1) }))
|
|
470
|
-
.where("
|
|
509
|
+
.where("environment_id", "=", env)
|
|
471
510
|
.execute();
|
|
472
|
-
await this.register(
|
|
473
|
-
//
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
const sig = AbortSignal.any([signal, lease.signal]);
|
|
477
|
-
const impl = p.impl;
|
|
478
|
-
if (impl.killTagged) {
|
|
479
|
-
// by tag whenever the provider can, even with cap 1: while a hot change of
|
|
480
|
-
// maxConcurrentCommands spreads, a replica on a larger cap may have let a sibling in
|
|
481
|
-
await this.kill(conv, "takeover", async () => impl.killTagged?.(row.handle, dead.tags, sig), sig);
|
|
482
|
-
}
|
|
483
|
-
else if ((await this.leases.sharedCount(conv)) <= 1) {
|
|
484
|
-
// cap 1 and this holder is alone: killing everything cannot hit a sibling
|
|
485
|
-
await this.killLeftovers(conv, row.handle, p, sig, "takeover");
|
|
486
|
-
}
|
|
487
|
-
else {
|
|
488
|
-
this.log.warn("cannot kill leftover sandbox commands: the provider has no killTagged and another holder is live; the in-sandbox time limit ends them", { conversationId: conv });
|
|
489
|
-
}
|
|
490
|
-
}
|
|
511
|
+
await this.register(env, lease.token);
|
|
512
|
+
// other holders (any conversation of the environment) may be running: only the dead
|
|
513
|
+
// holders' own commands are killed
|
|
514
|
+
await this.killTagged(env, row.handle, p, await this.claim(env), signal, lease);
|
|
491
515
|
return {
|
|
492
516
|
lease,
|
|
493
517
|
handle: row.handle,
|
|
494
518
|
createdAt: row.created_at.getTime(),
|
|
495
|
-
|
|
519
|
+
fsId: row.fs_handle?.id ?? "",
|
|
520
|
+
info: { fresh: false, restarted: false },
|
|
496
521
|
};
|
|
497
522
|
}
|
|
498
523
|
catch (err) {
|
|
499
|
-
await this.unregister(
|
|
524
|
+
await this.unregister(env, lease.token);
|
|
500
525
|
await lease.release();
|
|
501
526
|
throw err;
|
|
502
527
|
}
|
|
503
528
|
}
|
|
504
529
|
/**
|
|
505
|
-
* Under a just-taken exclusive lease: makes the
|
|
506
|
-
* recorded instance is the one found gone), kills leftovers, then downgrades to
|
|
530
|
+
* Under a just-taken exclusive lease: makes the instance run (file system, create / resume /
|
|
531
|
+
* rebuild when the recorded instance is the one found gone), kills leftovers, then downgrades to
|
|
532
|
+
* a shared lease. Crashed holders' records are claimed here: on an instance just resumed from
|
|
533
|
+
* `suspended` one `killAll` covers them and anything else left (nobody else holds the instance),
|
|
534
|
+
* on a new instance there is nothing to kill, and on one that already ran their tags are killed.
|
|
507
535
|
*/
|
|
508
536
|
async asExclusive(ctx, p, lease, gone, signal) {
|
|
509
|
-
const
|
|
537
|
+
const env = ctx.environmentId;
|
|
510
538
|
const sig = AbortSignal.any([signal, lease.signal]);
|
|
511
539
|
try {
|
|
512
540
|
const s = await this.ensure(ctx, p, lease, sig, gone);
|
|
513
|
-
const dead = await this.claim(
|
|
514
|
-
if (
|
|
515
|
-
await this.killLeftovers(
|
|
516
|
-
|
|
541
|
+
const dead = await this.claim(env);
|
|
542
|
+
if (s.resumed)
|
|
543
|
+
await this.killLeftovers(env, s.handle, p, sig, dead);
|
|
544
|
+
else if (!s.info.fresh)
|
|
545
|
+
await this.killTagged(env, s.handle, p, dead, signal, lease);
|
|
517
546
|
if (s.resumed || s.info.fresh) {
|
|
518
|
-
await this.killPendingTasks(
|
|
547
|
+
await this.killPendingTasks(env, s.handle, s.createdAt, p, lease.token, ctx.actor, sig);
|
|
519
548
|
}
|
|
520
549
|
const shared = await lease.downgrade();
|
|
521
|
-
await this.register(
|
|
522
|
-
return {
|
|
550
|
+
await this.register(env, shared.token);
|
|
551
|
+
return {
|
|
552
|
+
lease: shared,
|
|
553
|
+
handle: s.handle,
|
|
554
|
+
createdAt: s.createdAt,
|
|
555
|
+
fsId: s.fsId,
|
|
556
|
+
info: s.info,
|
|
557
|
+
};
|
|
523
558
|
}
|
|
524
559
|
catch (err) {
|
|
525
|
-
await this.unregister(
|
|
560
|
+
await this.unregister(env, lease.token);
|
|
526
561
|
// releases the lease in whatever kind it has now (also after a downgrade)
|
|
527
562
|
await lease.release();
|
|
528
563
|
throw err;
|
|
529
564
|
}
|
|
530
565
|
}
|
|
531
566
|
/** Ends a holder: unregisters it before releasing its lease, then moves `last_used_at` forward. */
|
|
532
|
-
async leave(
|
|
533
|
-
await this.unregister(
|
|
567
|
+
async leave(env, e) {
|
|
568
|
+
await this.unregister(env, e.lease.token);
|
|
534
569
|
await e.lease.release();
|
|
535
570
|
await this.db
|
|
536
|
-
.updateTable("
|
|
571
|
+
.updateTable("environment_instances")
|
|
537
572
|
.set({ last_used_at: sql `GREATEST(last_used_at, now())` })
|
|
538
|
-
.where("
|
|
573
|
+
.where("environment_id", "=", env)
|
|
539
574
|
.where("status", "=", "running")
|
|
540
575
|
.execute()
|
|
541
|
-
.catch((err) => this.log.warn("cannot update sandbox last_used_at", {
|
|
576
|
+
.catch((err) => this.log.warn("cannot update sandbox last_used_at", { environmentId: env, err }));
|
|
542
577
|
}
|
|
543
|
-
async register(
|
|
578
|
+
async register(env, token) {
|
|
544
579
|
const ttl = this.config.get().sandbox.limits.maxCommandSeconds * 1000 + HOLDER_TTL_MARGIN_MS;
|
|
545
580
|
await this.leases
|
|
546
|
-
.registerHolder(
|
|
547
|
-
.catch((err) => this.log.warn("cannot record sandbox holder", {
|
|
581
|
+
.registerHolder(env, token, ttl)
|
|
582
|
+
.catch((err) => this.log.warn("cannot record sandbox holder", { environmentId: env, err }));
|
|
548
583
|
}
|
|
549
|
-
async unregister(
|
|
584
|
+
async unregister(env, token) {
|
|
550
585
|
await this.leases
|
|
551
|
-
.unregisterHolder(
|
|
552
|
-
.catch((err) => this.log.warn("cannot clear sandbox holder record", {
|
|
586
|
+
.unregisterHolder(env, token)
|
|
587
|
+
.catch((err) => this.log.warn("cannot clear sandbox holder record", { environmentId: env, err }));
|
|
553
588
|
}
|
|
554
|
-
/**
|
|
555
|
-
|
|
556
|
-
* the caller's exclusive lease; a Redis error counts as none.
|
|
557
|
-
*/
|
|
558
|
-
async claim(conv, exclusiveToken) {
|
|
589
|
+
/** Crashed holders' tags (claimed); a Redis error counts as none. */
|
|
590
|
+
async claim(env) {
|
|
559
591
|
try {
|
|
560
|
-
return await this.leases.claimDeadHolders(
|
|
592
|
+
return await this.leases.claimDeadHolders(env);
|
|
561
593
|
}
|
|
562
594
|
catch (err) {
|
|
563
|
-
this.log.warn("cannot read sandbox holder records", {
|
|
564
|
-
return
|
|
595
|
+
this.log.warn("cannot read sandbox holder records", { environmentId: env, err });
|
|
596
|
+
return [];
|
|
565
597
|
}
|
|
566
598
|
}
|
|
599
|
+
/** Kills the commands of crashed holders by their tags (`killTagged`); nothing without tags. */
|
|
600
|
+
async killTagged(env, handle, p, tags, signal, lease) {
|
|
601
|
+
if (tags.length === 0)
|
|
602
|
+
return;
|
|
603
|
+
const sig = AbortSignal.any([signal, lease.signal]);
|
|
604
|
+
await this.kill(env, "takeover", () => p.impl.killTagged(handle, tags, sig), sig);
|
|
605
|
+
}
|
|
567
606
|
/** Runs one kill call: gone and abort are thrown, other failures only warned. */
|
|
568
|
-
async kill(
|
|
607
|
+
async kill(environmentId, why, call, signal) {
|
|
569
608
|
try {
|
|
570
609
|
await call();
|
|
571
610
|
if (why === "takeover") {
|
|
572
|
-
this.log.info("sandbox taken over from a holder that did not finish; leftover commands killed", {
|
|
611
|
+
this.log.info("sandbox taken over from a holder that did not finish; leftover commands killed", { environmentId });
|
|
573
612
|
}
|
|
574
613
|
}
|
|
575
614
|
catch (err) {
|
|
576
615
|
if (signal?.aborted || isSandboxGoneError(err))
|
|
577
616
|
throw err;
|
|
578
|
-
this.log.warn("cannot kill leftover sandbox commands", {
|
|
617
|
+
this.log.warn("cannot kill leftover sandbox commands", { environmentId, err });
|
|
579
618
|
}
|
|
580
619
|
}
|
|
581
620
|
/**
|
|
582
|
-
*
|
|
583
|
-
* Skips
|
|
584
|
-
* window (about a second) first.
|
|
621
|
+
* Suspends running instances idle longer than `suspendAfterIdleMinutes` or running continuously
|
|
622
|
+
* longer than `maxRunningMinutes`. Skips an environment with any holder; one past
|
|
623
|
+
* `maxRunningMinutes` gets the intent for one poll window (about a second) first. Returns how
|
|
624
|
+
* many it suspended. Called every minute by the `sandbox-suspend-idle` cron.
|
|
585
625
|
*/
|
|
586
626
|
async suspendIdle(now) {
|
|
587
627
|
const p = this.providers.current();
|
|
@@ -593,12 +633,12 @@ let SandboxService = class SandboxService {
|
|
|
593
633
|
let n = 0;
|
|
594
634
|
await inPool(rows, SWEEP_CONCURRENCY, async (c) => {
|
|
595
635
|
const overdue = c.running_since !== null && c.running_since < runningBefore;
|
|
596
|
-
const done = await this.tryLocked(c.
|
|
597
|
-
const id = c.
|
|
636
|
+
const done = await this.tryLocked(c.environment_id, async (lease) => {
|
|
637
|
+
const id = c.environment_id;
|
|
598
638
|
const row = await this.db
|
|
599
|
-
.selectFrom("
|
|
639
|
+
.selectFrom("environment_instances")
|
|
600
640
|
.selectAll()
|
|
601
|
-
.where("
|
|
641
|
+
.where("environment_id", "=", id)
|
|
602
642
|
.where(due)
|
|
603
643
|
.executeTakeFirst();
|
|
604
644
|
if (!row?.handle)
|
|
@@ -613,7 +653,7 @@ let SandboxService = class SandboxService {
|
|
|
613
653
|
running_since: null,
|
|
614
654
|
sweep_failed_at: null,
|
|
615
655
|
});
|
|
616
|
-
this.log.info("sandbox suspended", {
|
|
656
|
+
this.log.info("sandbox suspended", { environmentId: id });
|
|
617
657
|
return true;
|
|
618
658
|
}, overdue);
|
|
619
659
|
if (done)
|
|
@@ -622,89 +662,136 @@ let SandboxService = class SandboxService {
|
|
|
622
662
|
return n;
|
|
623
663
|
}
|
|
624
664
|
/**
|
|
625
|
-
*
|
|
626
|
-
*
|
|
665
|
+
* Destruction sweep: an instance unused for longer than `destroyAfterInactiveDays` is destroyed
|
|
666
|
+
* (the instance only; the file system is kept and the next use builds a new instance on it); an
|
|
667
|
+
* environment that was deleted, or whose org was deleted, loses its instance and its file system
|
|
668
|
+
* and its row is deleted (as {@link destroyEnvironment} does). Environments with any holder are
|
|
669
|
+
* skipped. Returns how many it destroyed. Called hourly by the `sandbox-destroy-inactive` cron.
|
|
627
670
|
*/
|
|
628
671
|
async destroyInactive(now) {
|
|
629
672
|
const p = this.providers.current();
|
|
630
673
|
if (!p)
|
|
631
674
|
return 0;
|
|
632
|
-
const due = this.destroyDue(now);
|
|
633
|
-
const rows = await this.candidates(due);
|
|
634
675
|
let n = 0;
|
|
635
|
-
|
|
676
|
+
const gone = (eb) => this.environmentGone(eb);
|
|
677
|
+
await inPool(await this.candidates(gone), SWEEP_CONCURRENCY, async ({ environment_id: id }) => {
|
|
636
678
|
const done = await this.tryLocked(id, async (lease) => {
|
|
637
679
|
const row = await this.db
|
|
638
|
-
.selectFrom("
|
|
680
|
+
.selectFrom("environment_instances")
|
|
639
681
|
.selectAll()
|
|
640
|
-
.where("
|
|
641
|
-
.where(
|
|
682
|
+
.where("environment_id", "=", id)
|
|
683
|
+
.where(gone)
|
|
642
684
|
.executeTakeFirst();
|
|
643
685
|
if (!row)
|
|
644
686
|
return false;
|
|
645
|
-
await this.destroyRow(row, p, lease);
|
|
687
|
+
await this.destroyRow(row, p, lease, "environment");
|
|
688
|
+
return true;
|
|
689
|
+
});
|
|
690
|
+
if (done)
|
|
691
|
+
n++;
|
|
692
|
+
});
|
|
693
|
+
const idle = this.instanceDue(now);
|
|
694
|
+
await inPool(await this.candidates(idle), SWEEP_CONCURRENCY, async ({ environment_id: id }) => {
|
|
695
|
+
const done = await this.tryLocked(id, async (lease) => {
|
|
696
|
+
const row = await this.db
|
|
697
|
+
.selectFrom("environment_instances")
|
|
698
|
+
.selectAll()
|
|
699
|
+
.where("environment_id", "=", id)
|
|
700
|
+
.where(idle)
|
|
701
|
+
.executeTakeFirst();
|
|
702
|
+
if (!row)
|
|
703
|
+
return false;
|
|
704
|
+
await this.destroyRow(row, p, lease, "instance");
|
|
646
705
|
return true;
|
|
647
706
|
});
|
|
648
707
|
if (done)
|
|
649
708
|
n++;
|
|
650
709
|
});
|
|
651
|
-
// 早先已销毁、对话后来才删除、且没有遗留旧实例的记录
|
|
652
|
-
await this.db
|
|
653
|
-
.deleteFrom("sandboxes")
|
|
654
|
-
.where("status", "=", "destroyed")
|
|
655
|
-
.where("prev_handle", "is", null)
|
|
656
|
-
.where(({ not, exists, selectFrom }) => not(exists(selectFrom("conversations")
|
|
657
|
-
.select("conversations.id")
|
|
658
|
-
.whereRef("conversations.id", "=", "sandboxes.conversation_id"))))
|
|
659
|
-
.execute();
|
|
660
710
|
return n;
|
|
661
711
|
}
|
|
662
712
|
/**
|
|
663
|
-
*
|
|
664
|
-
*
|
|
665
|
-
* to 10 minutes.
|
|
713
|
+
* Destroys the environment's instance, then its file system, then deletes its instance row
|
|
714
|
+
* (environment deletion, offboarding); nothing to do without a row. Waits for the exclusive lease
|
|
715
|
+
* (with intent, so running commands finish and new ones wait) up to 10 minutes. Without a
|
|
716
|
+
* provider it only warns (the destruction sweep finishes a deleted environment later). Files kept
|
|
717
|
+
* by another provider that is not loaded cannot be destroyed: it throws and the row stays.
|
|
666
718
|
*/
|
|
667
|
-
async
|
|
719
|
+
async destroyEnvironment(environmentId) {
|
|
668
720
|
const row = await this.db
|
|
669
|
-
.selectFrom("
|
|
670
|
-
.select("
|
|
671
|
-
.where("
|
|
721
|
+
.selectFrom("environment_instances")
|
|
722
|
+
.select("environment_id")
|
|
723
|
+
.where("environment_id", "=", environmentId)
|
|
672
724
|
.executeTakeFirst();
|
|
673
|
-
if (!row
|
|
725
|
+
if (!row)
|
|
674
726
|
return;
|
|
675
727
|
const p = this.providers.current();
|
|
676
728
|
if (!p) {
|
|
677
|
-
this.log.warn("cannot destroy
|
|
729
|
+
this.log.warn("cannot destroy environment: no sandbox provider", { environmentId });
|
|
678
730
|
return;
|
|
679
731
|
}
|
|
680
|
-
const lease = await this.leases.acquireExclusive(
|
|
732
|
+
const lease = await this.leases.acquireExclusive(environmentId, AbortSignal.timeout(EXCLUSIVE_WAIT_MS), undefined, { intent: true });
|
|
681
733
|
try {
|
|
682
734
|
const current = await this.db
|
|
683
|
-
.selectFrom("
|
|
735
|
+
.selectFrom("environment_instances")
|
|
684
736
|
.selectAll()
|
|
685
|
-
.where("
|
|
686
|
-
.where("status", "<>", "destroyed")
|
|
737
|
+
.where("environment_id", "=", environmentId)
|
|
687
738
|
.executeTakeFirst();
|
|
688
739
|
if (current)
|
|
689
|
-
await this.destroyRow(current, p, lease);
|
|
740
|
+
await this.destroyRow(current, p, lease, "environment");
|
|
690
741
|
}
|
|
691
742
|
finally {
|
|
692
743
|
await lease.release();
|
|
693
744
|
}
|
|
694
745
|
}
|
|
695
746
|
/**
|
|
696
|
-
*
|
|
697
|
-
*
|
|
747
|
+
* Starts the environment over with empty files (after `environment.files_lost`, or on request):
|
|
748
|
+
* under the exclusive lease (waited for like {@link destroyEnvironment}) destroys any instance and
|
|
749
|
+
* the file system (a `lost` one is only dropped), and records no file system; the next use
|
|
750
|
+
* creates a new one. Nothing to do without a row. Throws `sandbox.unavailable` without a provider.
|
|
698
751
|
*/
|
|
699
|
-
async
|
|
700
|
-
const
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
752
|
+
async resetFileSystem(environmentId) {
|
|
753
|
+
const p = this.require();
|
|
754
|
+
const lease = await this.leases.acquireExclusive(environmentId, AbortSignal.timeout(EXCLUSIVE_WAIT_MS), undefined, { intent: true });
|
|
755
|
+
try {
|
|
756
|
+
const row = await this.db
|
|
757
|
+
.selectFrom("environment_instances")
|
|
758
|
+
.selectAll()
|
|
759
|
+
.where("environment_id", "=", environmentId)
|
|
760
|
+
.executeTakeFirst();
|
|
761
|
+
if (!row)
|
|
762
|
+
return;
|
|
763
|
+
const fence = { gen: row.generation };
|
|
764
|
+
const signal = opSignal(lease);
|
|
765
|
+
const prev = await this.destroyInstance(row, p, signal);
|
|
766
|
+
await this.destroyFiles(row, p, signal, true);
|
|
767
|
+
if (!(await lease.renew()))
|
|
768
|
+
throw new SandboxLockLostError(environmentId);
|
|
769
|
+
await this.fenced(lease, environmentId, fence, {
|
|
770
|
+
fs_handle: null,
|
|
771
|
+
fs_status: "none",
|
|
772
|
+
handle: null,
|
|
773
|
+
status: "none",
|
|
774
|
+
running_since: null,
|
|
775
|
+
suspended_at: null,
|
|
776
|
+
destroyed_at: sql `now()`,
|
|
777
|
+
sweep_failed_at: null,
|
|
778
|
+
...prev,
|
|
779
|
+
});
|
|
780
|
+
this.log.info("environment files reset", { environmentId });
|
|
781
|
+
}
|
|
782
|
+
finally {
|
|
783
|
+
await lease.release();
|
|
706
784
|
}
|
|
707
|
-
|
|
785
|
+
}
|
|
786
|
+
/**
|
|
787
|
+
* Under the exclusive lease (before the downgrade), on an instance just resumed from
|
|
788
|
+
* `suspended`: kills every command left in it (`killAll`), crashed holders' (`dead`, already
|
|
789
|
+
* claimed) included. Never on the shared path: other holders may be running commands. A gone
|
|
790
|
+
* instance (`SandboxGoneError`) and an abort are thrown (`withSandbox` rebuilds / ends); other
|
|
791
|
+
* failures are only warned.
|
|
792
|
+
*/
|
|
793
|
+
async killLeftovers(environmentId, handle, p, signal, dead) {
|
|
794
|
+
await this.kill(environmentId, dead.length > 0 ? "takeover" : "resume", () => p.impl.killAll(handle, signal), signal);
|
|
708
795
|
}
|
|
709
796
|
/**
|
|
710
797
|
* Under the exclusive lease, right after a resume or a (re)build: kills the sessions of the
|
|
@@ -713,8 +800,8 @@ let SandboxService = class SandboxService {
|
|
|
713
800
|
* here and are only cleared. A gone instance and an abort are thrown; another failure is warned
|
|
714
801
|
* and leaves the flags of this instance's tasks for the next resume.
|
|
715
802
|
*/
|
|
716
|
-
async killPendingTasks(
|
|
717
|
-
const rows = await this.background.pendingKills(
|
|
803
|
+
async killPendingTasks(environmentId, handle, createdAt, p, tag, actor, signal) {
|
|
804
|
+
const rows = await this.background.pendingKills(environmentId);
|
|
718
805
|
if (rows.length === 0)
|
|
719
806
|
return;
|
|
720
807
|
const mine = rows.filter((r) => r.sandboxCreatedAt?.getTime() === createdAt && r.sid !== null && SID_RE.test(r.sid));
|
|
@@ -729,12 +816,13 @@ let SandboxService = class SandboxService {
|
|
|
729
816
|
maxOutputBytes: 4096,
|
|
730
817
|
signal,
|
|
731
818
|
tag,
|
|
819
|
+
actor,
|
|
732
820
|
});
|
|
733
821
|
if (r.exitCode !== 0) {
|
|
734
822
|
throw new Error(`pkill exited with ${r.exitCode === null ? "no status" : r.exitCode}: ${r.stderr.slice(0, 500)}`);
|
|
735
823
|
}
|
|
736
824
|
this.log.info("lost background tasks killed after a sandbox resume", {
|
|
737
|
-
|
|
825
|
+
environmentId,
|
|
738
826
|
tasks: mine.map((t) => t.id),
|
|
739
827
|
});
|
|
740
828
|
}
|
|
@@ -742,7 +830,7 @@ let SandboxService = class SandboxService {
|
|
|
742
830
|
if (signal.aborted || isSandboxGoneError(err))
|
|
743
831
|
throw err;
|
|
744
832
|
this.log.warn("cannot kill lost background tasks after a sandbox resume", {
|
|
745
|
-
|
|
833
|
+
environmentId,
|
|
746
834
|
err,
|
|
747
835
|
});
|
|
748
836
|
const kept = new Set(mine.map((r) => r.id));
|
|
@@ -758,15 +846,16 @@ let SandboxService = class SandboxService {
|
|
|
758
846
|
return p;
|
|
759
847
|
}
|
|
760
848
|
/**
|
|
761
|
-
*
|
|
849
|
+
* A fenced write: only while the lease holds and the row's generation is still the one read
|
|
850
|
+
* under it (and incremented); otherwise {@link SandboxLockLostError}.
|
|
762
851
|
*/
|
|
763
852
|
async fenced(lease, id, fence, set) {
|
|
764
853
|
if (lease.signal.aborted)
|
|
765
854
|
throw new SandboxLockLostError(id);
|
|
766
855
|
const r = await this.db
|
|
767
|
-
.updateTable("
|
|
856
|
+
.updateTable("environment_instances")
|
|
768
857
|
.set({ ...set, generation: fence.gen + 1 })
|
|
769
|
-
.where("
|
|
858
|
+
.where("environment_id", "=", id)
|
|
770
859
|
.where("generation", "=", fence.gen)
|
|
771
860
|
.returning("created_at")
|
|
772
861
|
.executeTakeFirst();
|
|
@@ -775,11 +864,11 @@ let SandboxService = class SandboxService {
|
|
|
775
864
|
fence.gen++;
|
|
776
865
|
return r.created_at;
|
|
777
866
|
}
|
|
778
|
-
/**
|
|
867
|
+
/** Sweep candidates: those whose last sweep failed come last, the rest least recently used first. */
|
|
779
868
|
async candidates(due) {
|
|
780
869
|
return await this.db
|
|
781
|
-
.selectFrom("
|
|
782
|
-
.select(["
|
|
870
|
+
.selectFrom("environment_instances")
|
|
871
|
+
.select(["environment_id", "running_since"])
|
|
783
872
|
.where(due)
|
|
784
873
|
.orderBy(sql `sweep_failed_at ASC NULLS FIRST`)
|
|
785
874
|
.orderBy("last_used_at")
|
|
@@ -788,17 +877,17 @@ let SandboxService = class SandboxService {
|
|
|
788
877
|
}
|
|
789
878
|
/**
|
|
790
879
|
* One try at the exclusive lease: null → false; else runs `fn`. With `overdue` (past
|
|
791
|
-
* `maxRunningMinutes`) a busy
|
|
792
|
-
* briefly and the sweep gets in if the running ones end. Failures are warned, recorded in
|
|
880
|
+
* `maxRunningMinutes`) a busy environment gets the intent for about a second, so new commands
|
|
881
|
+
* wait briefly and the sweep gets in if the running ones end. Failures are warned, recorded in
|
|
793
882
|
* `sweep_failed_at` (sorted last next time) and return false.
|
|
794
883
|
*/
|
|
795
|
-
async tryLocked(
|
|
884
|
+
async tryLocked(environmentId, fn, overdue = false) {
|
|
796
885
|
let lease;
|
|
797
886
|
try {
|
|
798
|
-
lease = await this.leases.tryExclusive(
|
|
887
|
+
lease = await this.leases.tryExclusive(environmentId);
|
|
799
888
|
if (!lease && overdue) {
|
|
800
889
|
lease = await this.leases
|
|
801
|
-
.acquireExclusive(
|
|
890
|
+
.acquireExclusive(environmentId, AbortSignal.timeout(SWEEP_OP_TIMEOUT_MS), Math.max(this.pollMs * 2, SWEEP_INTENT_WAIT_MS), { intent: true })
|
|
802
891
|
.catch((err) => {
|
|
803
892
|
if (err instanceof ApiError && err.code === "sandbox.busy")
|
|
804
893
|
return null;
|
|
@@ -807,7 +896,7 @@ let SandboxService = class SandboxService {
|
|
|
807
896
|
}
|
|
808
897
|
}
|
|
809
898
|
catch (err) {
|
|
810
|
-
this.log.warn("cannot take sandbox lock", {
|
|
899
|
+
this.log.warn("cannot take sandbox lock", { environmentId, err });
|
|
811
900
|
return false;
|
|
812
901
|
}
|
|
813
902
|
if (!lease)
|
|
@@ -816,11 +905,11 @@ let SandboxService = class SandboxService {
|
|
|
816
905
|
return await fn(lease);
|
|
817
906
|
}
|
|
818
907
|
catch (err) {
|
|
819
|
-
this.log.warn("sandbox maintenance failed; will retry", {
|
|
908
|
+
this.log.warn("sandbox maintenance failed; will retry", { environmentId, err });
|
|
820
909
|
await this.db
|
|
821
|
-
.updateTable("
|
|
910
|
+
.updateTable("environment_instances")
|
|
822
911
|
.set({ sweep_failed_at: sql `now()` })
|
|
823
|
-
.where("
|
|
912
|
+
.where("environment_id", "=", environmentId)
|
|
824
913
|
.execute()
|
|
825
914
|
.catch(() => { });
|
|
826
915
|
return false;
|
|
@@ -830,82 +919,120 @@ let SandboxService = class SandboxService {
|
|
|
830
919
|
}
|
|
831
920
|
}
|
|
832
921
|
/**
|
|
833
|
-
* Under the exclusive lease: make the
|
|
922
|
+
* Under the exclusive lease: make the environment's instance run. `gone`: the instance a holder
|
|
834
923
|
* found gone; when it is still the recorded one it is rebuilt, when another holder already
|
|
835
924
|
* rebuilt it the new one is used.
|
|
836
925
|
*/
|
|
837
926
|
async ensure(ctx, p, held, signal, gone) {
|
|
838
|
-
const id = ctx.
|
|
927
|
+
const id = ctx.environmentId;
|
|
928
|
+
// Only a live environment of the caller's org gets a row, with the environment's own org; a
|
|
929
|
+
// row made earlier (another use) keeps its file system handle.
|
|
839
930
|
await this.db
|
|
840
|
-
.insertInto("
|
|
841
|
-
.
|
|
842
|
-
.
|
|
931
|
+
.insertInto("environment_instances")
|
|
932
|
+
.columns(["environment_id", "org_id", "provider", "status"])
|
|
933
|
+
.expression((eb) => eb
|
|
934
|
+
.selectFrom("environments")
|
|
935
|
+
.select((e) => [
|
|
936
|
+
"environments.id",
|
|
937
|
+
"environments.org_id",
|
|
938
|
+
e.val(p.plugin).as("provider"),
|
|
939
|
+
e.val("creating").as("status"),
|
|
940
|
+
])
|
|
941
|
+
.where("environments.id", "=", id)
|
|
942
|
+
.where("environments.org_id", "=", ctx.orgId)
|
|
943
|
+
.where("environments.deleted_at", "is", null))
|
|
944
|
+
.onConflict((oc) => oc.column("environment_id").doNothing())
|
|
843
945
|
.execute();
|
|
946
|
+
// Under the lease: a use that waited behind destroyEnvironment (or raced a deletion) must not
|
|
947
|
+
// bring the environment's provider resources back.
|
|
948
|
+
const live = await this.db
|
|
949
|
+
.selectFrom("environments")
|
|
950
|
+
.select("id")
|
|
951
|
+
.where("id", "=", id)
|
|
952
|
+
.where("org_id", "=", ctx.orgId)
|
|
953
|
+
.where("deleted_at", "is", null)
|
|
954
|
+
.executeTakeFirst();
|
|
955
|
+
if (!live)
|
|
956
|
+
throw environmentNotFound();
|
|
844
957
|
const row = await this.db
|
|
845
|
-
.selectFrom("
|
|
958
|
+
.selectFrom("environment_instances")
|
|
846
959
|
.selectAll()
|
|
847
|
-
.where("
|
|
848
|
-
.
|
|
849
|
-
if (row
|
|
850
|
-
throw
|
|
851
|
-
|
|
960
|
+
.where("environment_id", "=", id)
|
|
961
|
+
.executeTakeFirst();
|
|
962
|
+
if (!row)
|
|
963
|
+
throw environmentNotFound();
|
|
964
|
+
if (row.org_id !== ctx.orgId)
|
|
965
|
+
throw this.foreign(ctx);
|
|
852
966
|
const fence = { gen: row.generation };
|
|
853
|
-
//
|
|
967
|
+
// Every acquisition first does a fenced increment: an old holder whose lease expired while it
|
|
968
|
+
// kept running (a stalled sweep) then fails its next write.
|
|
854
969
|
await this.fenced(held, id, fence, {});
|
|
970
|
+
if (row.fs_status === "lost")
|
|
971
|
+
throw environmentFilesLost();
|
|
972
|
+
if (row.provider !== null &&
|
|
973
|
+
row.provider !== p.plugin &&
|
|
974
|
+
(row.fs_status !== "none" || row.handle !== null)) {
|
|
975
|
+
throw otherProvider(row.provider);
|
|
976
|
+
}
|
|
977
|
+
if (row.provider !== null &&
|
|
978
|
+
row.provider !== p.plugin &&
|
|
979
|
+
row.fs_status === "none" &&
|
|
980
|
+
row.status === "creating") {
|
|
981
|
+
// a create by the previous provider ended before its file system was recorded: that
|
|
982
|
+
// provider may have made it by name, and nothing else would ever destroy it
|
|
983
|
+
await this.dropUnrecordedFiles(id, row.provider, signal);
|
|
984
|
+
}
|
|
855
985
|
const handle = row.handle;
|
|
856
986
|
const createdAt = row.created_at.getTime();
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
987
|
+
const fsId = row.fs_handle?.id ?? "";
|
|
988
|
+
if (handle && isInstance(row, gone) && row.status === "running") {
|
|
989
|
+
const created = await this.create(ctx, p, held, row, fence, signal);
|
|
990
|
+
return { ...created, info: { fresh: true, restarted: true }, fence };
|
|
860
991
|
}
|
|
861
|
-
if (handle && row.
|
|
862
|
-
|
|
863
|
-
return { handle, createdAt, info: { fresh: false, recreated: false }, fence };
|
|
864
|
-
}
|
|
865
|
-
if (row.status === "suspended") {
|
|
866
|
-
const r = await p.impl.resume(handle, signal);
|
|
867
|
-
if (r === "resumed") {
|
|
868
|
-
await this.fenced(held, id, fence, {
|
|
869
|
-
status: "running",
|
|
870
|
-
running_since: sql `now()`,
|
|
871
|
-
suspended_at: null,
|
|
872
|
-
sweep_failed_at: null,
|
|
873
|
-
});
|
|
874
|
-
return {
|
|
875
|
-
handle,
|
|
876
|
-
createdAt,
|
|
877
|
-
info: { fresh: false, recreated: false },
|
|
878
|
-
fence,
|
|
879
|
-
resumed: true,
|
|
880
|
-
};
|
|
881
|
-
}
|
|
882
|
-
this.log.info("sandbox is gone at the provider; recreating", { conversationId: id });
|
|
883
|
-
}
|
|
992
|
+
if (handle && row.status === "running") {
|
|
993
|
+
return { handle, createdAt, fsId, info: { fresh: false, restarted: false }, fence };
|
|
884
994
|
}
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
995
|
+
if (handle && row.status === "suspended") {
|
|
996
|
+
const r = await p.impl.resume(handle, signal);
|
|
997
|
+
if (r === "resumed") {
|
|
998
|
+
await this.fenced(held, id, fence, {
|
|
999
|
+
status: "running",
|
|
1000
|
+
running_since: sql `now()`,
|
|
1001
|
+
suspended_at: null,
|
|
1002
|
+
sweep_failed_at: null,
|
|
1003
|
+
});
|
|
1004
|
+
return {
|
|
1005
|
+
handle,
|
|
1006
|
+
createdAt,
|
|
1007
|
+
fsId,
|
|
1008
|
+
info: { fresh: false, restarted: false },
|
|
1009
|
+
fence,
|
|
1010
|
+
resumed: true,
|
|
1011
|
+
};
|
|
897
1012
|
}
|
|
1013
|
+
this.log.info("sandbox instance is gone at the provider; recreating", {
|
|
1014
|
+
environmentId: id,
|
|
1015
|
+
});
|
|
898
1016
|
}
|
|
899
|
-
const created = await this.create(ctx, p, held, fence, signal
|
|
900
|
-
// had_instance
|
|
901
|
-
|
|
1017
|
+
const created = await this.create(ctx, p, held, row, fence, signal);
|
|
1018
|
+
// had_instance only ever becomes true: any earlier instance (even if a rebuild failed since)
|
|
1019
|
+
// makes this one a restart on the kept files
|
|
1020
|
+
return {
|
|
1021
|
+
...created,
|
|
1022
|
+
info: { fresh: true, restarted: row.had_instance && row.fs_status === "ready" },
|
|
1023
|
+
fence,
|
|
1024
|
+
};
|
|
902
1025
|
}
|
|
903
1026
|
/**
|
|
904
|
-
*
|
|
905
|
-
*
|
|
1027
|
+
* Under the lease: fenced reset to `creating`, the file system when there is none
|
|
1028
|
+
* (`createFileSystem`, its handle written fenced), then the instance on it, then a fenced write
|
|
1029
|
+
* of the instance handle. `FileSystemGoneError` from `create` records `fs_status = 'lost'` and
|
|
1030
|
+
* throws `environment.files_lost`. When the instance handle cannot be written (lease lost), the
|
|
1031
|
+
* new instance is destroyed (best effort) before {@link SandboxLockLostError} is thrown.
|
|
906
1032
|
*/
|
|
907
|
-
async create(ctx, p, held, fence, signal
|
|
908
|
-
const id = ctx.
|
|
1033
|
+
async create(ctx, p, held, row, fence, signal) {
|
|
1034
|
+
const id = ctx.environmentId;
|
|
1035
|
+
const name = environmentResourceName(id);
|
|
909
1036
|
await this.fenced(held, id, fence, {
|
|
910
1037
|
org_id: ctx.orgId,
|
|
911
1038
|
provider: p.plugin,
|
|
@@ -916,21 +1043,45 @@ let SandboxService = class SandboxService {
|
|
|
916
1043
|
suspended_at: null,
|
|
917
1044
|
destroyed_at: null,
|
|
918
1045
|
sweep_failed_at: null,
|
|
919
|
-
...extra,
|
|
920
1046
|
});
|
|
1047
|
+
// the org's effective limits apply at creation only, so a changed setting reaches only
|
|
1048
|
+
// instances created afterwards
|
|
1049
|
+
const limits = await this.orgLimits(ctx.orgId);
|
|
921
1050
|
const cfg = this.config.get().sandbox;
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
1051
|
+
let fs = row.fs_status === "ready" && (row.provider === null || row.provider === p.plugin)
|
|
1052
|
+
? row.fs_handle
|
|
1053
|
+
: null;
|
|
1054
|
+
if (fs === null) {
|
|
1055
|
+
fs = await p.impl.createFileSystem({ name, limits, signal });
|
|
1056
|
+
// A lost lease fails this write and leaves the file system alone: createFileSystem returns
|
|
1057
|
+
// the existing one for a name, so the new holder may already use it, files and all.
|
|
1058
|
+
await this.fenced(held, id, fence, {
|
|
1059
|
+
provider: p.plugin,
|
|
1060
|
+
fs_handle: JSON.stringify(fs),
|
|
1061
|
+
fs_status: "ready",
|
|
1062
|
+
});
|
|
1063
|
+
this.log.info("environment file system created", { environmentId: id, provider: p.plugin });
|
|
1064
|
+
}
|
|
1065
|
+
let handle;
|
|
1066
|
+
try {
|
|
1067
|
+
handle = await p.impl.create({
|
|
1068
|
+
name,
|
|
1069
|
+
...(cfg.image !== undefined && { image: cfg.image }),
|
|
1070
|
+
limits,
|
|
1071
|
+
fileSystem: fs,
|
|
1072
|
+
signal,
|
|
1073
|
+
});
|
|
1074
|
+
}
|
|
1075
|
+
catch (err) {
|
|
1076
|
+
if (!isFileSystemGoneError(err))
|
|
1077
|
+
throw err;
|
|
1078
|
+
this.log.warn("environment file system is gone at the provider; its files are lost", {
|
|
1079
|
+
environmentId: id,
|
|
1080
|
+
provider: p.plugin,
|
|
1081
|
+
});
|
|
1082
|
+
await this.fenced(held, id, fence, { fs_status: "lost", status: "none" });
|
|
1083
|
+
throw environmentFilesLost();
|
|
1084
|
+
}
|
|
934
1085
|
let createdAt;
|
|
935
1086
|
try {
|
|
936
1087
|
createdAt = await this.fenced(held, id, fence, {
|
|
@@ -943,20 +1094,48 @@ let SandboxService = class SandboxService {
|
|
|
943
1094
|
});
|
|
944
1095
|
}
|
|
945
1096
|
catch (e) {
|
|
946
|
-
//
|
|
947
|
-
//
|
|
948
|
-
//
|
|
1097
|
+
// Lease lost before the handle was recorded: destroy the new instance (best effort) so it
|
|
1098
|
+
// does not leak. A provider that reuses instances by name (Ruling M10) may then hit the one
|
|
1099
|
+
// the new holder just resumed or created; that heals: its next provider call throws
|
|
1100
|
+
// SandboxGoneError (or resume returns gone) and withSandbox rebuilds in place. The file
|
|
1101
|
+
// system is never touched here.
|
|
949
1102
|
await p.impl.destroy(handle, AbortSignal.timeout(SWEEP_OP_TIMEOUT_MS)).catch((err) => this.log.warn("cannot destroy sandbox created after the lock was lost", {
|
|
950
|
-
|
|
1103
|
+
environmentId: id,
|
|
951
1104
|
err,
|
|
952
1105
|
}));
|
|
953
1106
|
throw e;
|
|
954
1107
|
}
|
|
955
|
-
this.log.info("sandbox created", {
|
|
956
|
-
return { handle, createdAt: createdAt.getTime() };
|
|
1108
|
+
this.log.info("sandbox created", { environmentId: id, provider: p.plugin });
|
|
1109
|
+
return { handle, createdAt: createdAt.getTime(), fsId: fs.id };
|
|
1110
|
+
}
|
|
1111
|
+
/**
|
|
1112
|
+
* Destroys the file system another provider may have made under the environment's name without
|
|
1113
|
+
* the kernel recording it (`destroyFileSystemByName`; not there is success). That provider not
|
|
1114
|
+
* loaded, or a failure, is only warned (the files may leak there) so the environment stays
|
|
1115
|
+
* usable; an abort is thrown.
|
|
1116
|
+
*/
|
|
1117
|
+
async dropUnrecordedFiles(id, provider, signal) {
|
|
1118
|
+
const old = this.providers.lookup(provider);
|
|
1119
|
+
if (!old) {
|
|
1120
|
+
this.log.warn("environment file system may have been made by a provider that is not loaded; it may leak", { environmentId: id, provider });
|
|
1121
|
+
return;
|
|
1122
|
+
}
|
|
1123
|
+
try {
|
|
1124
|
+
await old.destroyFileSystemByName(environmentResourceName(id), signal);
|
|
1125
|
+
}
|
|
1126
|
+
catch (err) {
|
|
1127
|
+
if (signal.aborted)
|
|
1128
|
+
throw err;
|
|
1129
|
+
this.log.warn("cannot destroy the file system of the previous sandbox provider", {
|
|
1130
|
+
environmentId: id,
|
|
1131
|
+
provider,
|
|
1132
|
+
err,
|
|
1133
|
+
});
|
|
1134
|
+
}
|
|
957
1135
|
}
|
|
958
1136
|
/**
|
|
959
|
-
*
|
|
1137
|
+
* Destroys an instance of another provider with that provider: still loaded and destroyed →
|
|
1138
|
+
* false; otherwise warns and returns true (the record must be kept).
|
|
960
1139
|
*/
|
|
961
1140
|
async retire(id, provider, handle, signal) {
|
|
962
1141
|
const old = this.providers.lookup(provider);
|
|
@@ -967,48 +1146,86 @@ let SandboxService = class SandboxService {
|
|
|
967
1146
|
}
|
|
968
1147
|
catch (err) {
|
|
969
1148
|
this.log.warn("cannot destroy sandbox instance of the previous provider", {
|
|
970
|
-
|
|
1149
|
+
environmentId: id,
|
|
971
1150
|
provider,
|
|
972
1151
|
err,
|
|
973
1152
|
});
|
|
974
1153
|
return true;
|
|
975
1154
|
}
|
|
976
1155
|
}
|
|
977
|
-
this.log.warn("sandbox was created by another provider that is not loaded; its instance may leak", {
|
|
1156
|
+
this.log.warn("sandbox was created by another provider that is not loaded; its instance may leak", { environmentId: id, provider });
|
|
978
1157
|
return true;
|
|
979
1158
|
}
|
|
980
|
-
/**
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
1159
|
+
/**
|
|
1160
|
+
* Under the lease: destroys the row's instance (and a leaked one of an earlier provider). An
|
|
1161
|
+
* instance without a handle (a create that ended before the handle was recorded) is destroyed by
|
|
1162
|
+
* name. Returns the `prev_*` patch for instances that could not be destroyed.
|
|
1163
|
+
*/
|
|
1164
|
+
async destroyInstance(row, p, signal) {
|
|
1165
|
+
const id = row.environment_id;
|
|
1166
|
+
const owner = row.provider ?? p.plugin;
|
|
985
1167
|
let prev = {};
|
|
986
|
-
if (row.handle &&
|
|
1168
|
+
if (row.handle && owner === p.plugin) {
|
|
987
1169
|
await p.impl.destroy(row.handle, signal);
|
|
988
1170
|
}
|
|
989
1171
|
else if (row.handle && row.status !== "destroyed") {
|
|
990
|
-
if (await this.retire(id,
|
|
991
|
-
prev = { prev_provider:
|
|
1172
|
+
if (await this.retire(id, owner, row.handle, signal)) {
|
|
1173
|
+
prev = { prev_provider: owner, prev_handle: JSON.stringify(row.handle) };
|
|
992
1174
|
}
|
|
993
1175
|
}
|
|
994
|
-
else if (!row.handle && row.
|
|
995
|
-
//
|
|
996
|
-
|
|
1176
|
+
else if (!row.handle && row.status === "creating") {
|
|
1177
|
+
// the provider may have built it already: only the name finds it
|
|
1178
|
+
const impl = owner === p.plugin ? p.impl : this.providers.lookup(owner);
|
|
1179
|
+
await impl?.destroyByName(environmentResourceName(id), signal);
|
|
997
1180
|
}
|
|
998
1181
|
if (row.prev_handle && row.prev_provider && !("prev_handle" in prev)) {
|
|
999
1182
|
if (!(await this.retire(id, row.prev_provider, row.prev_handle, signal))) {
|
|
1000
1183
|
prev = { prev_provider: null, prev_handle: null };
|
|
1001
1184
|
}
|
|
1002
1185
|
}
|
|
1186
|
+
return prev;
|
|
1187
|
+
}
|
|
1188
|
+
/**
|
|
1189
|
+
* Under the lease: destroys the row's file system with the provider that made it (by name when
|
|
1190
|
+
* no handle was recorded: `createFileSystem` may have finished before the kernel stopped). With
|
|
1191
|
+
* that provider not loaded it throws (the row must stay for a retry), or with `lenient` only
|
|
1192
|
+
* warns (the files leak at that provider).
|
|
1193
|
+
*/
|
|
1194
|
+
async destroyFiles(row, p, signal, lenient = false) {
|
|
1195
|
+
const id = row.environment_id;
|
|
1196
|
+
const owner = row.provider ?? p.plugin;
|
|
1197
|
+
const impl = owner === p.plugin ? p.impl : this.providers.lookup(owner);
|
|
1198
|
+
if (!impl) {
|
|
1199
|
+
if (row.fs_handle === null)
|
|
1200
|
+
return;
|
|
1201
|
+
if (!lenient) {
|
|
1202
|
+
throw new Error(`cannot destroy the file system of environment ${id}: its sandbox provider ${owner} is not loaded`);
|
|
1203
|
+
}
|
|
1204
|
+
this.log.warn("environment file system of a provider that is not loaded is leaked", {
|
|
1205
|
+
environmentId: id,
|
|
1206
|
+
provider: owner,
|
|
1207
|
+
});
|
|
1208
|
+
return;
|
|
1209
|
+
}
|
|
1210
|
+
if (row.fs_handle !== null)
|
|
1211
|
+
await impl.destroyFileSystem(row.fs_handle, signal);
|
|
1212
|
+
else
|
|
1213
|
+
await impl.destroyFileSystemByName(environmentResourceName(id), signal);
|
|
1214
|
+
}
|
|
1215
|
+
/**
|
|
1216
|
+
* Under the lease: `instance` destroys the instance and marks the row `destroyed` (the file
|
|
1217
|
+
* system stays); `environment` also destroys the file system and deletes the row.
|
|
1218
|
+
*/
|
|
1219
|
+
async destroyRow(row, p, held, mode) {
|
|
1220
|
+
const id = row.environment_id;
|
|
1221
|
+
const fence = { gen: row.generation };
|
|
1222
|
+
const signal = opSignal(held);
|
|
1223
|
+
const prev = await this.destroyInstance(row, p, signal);
|
|
1224
|
+
if (mode === "environment")
|
|
1225
|
+
await this.destroyFiles(row, p, signal);
|
|
1003
1226
|
if (!(await held.renew()))
|
|
1004
1227
|
throw new SandboxLockLostError(id);
|
|
1005
|
-
|
|
1006
|
-
.selectFrom("conversations")
|
|
1007
|
-
.select("id")
|
|
1008
|
-
.where("id", "=", id)
|
|
1009
|
-
.executeTakeFirst();
|
|
1010
|
-
const keepRow = conversation !== undefined || (prev.prev_handle ?? row.prev_handle) != null;
|
|
1011
|
-
if (keepRow) {
|
|
1228
|
+
if (mode === "instance") {
|
|
1012
1229
|
await this.fenced(held, id, fence, {
|
|
1013
1230
|
status: "destroyed",
|
|
1014
1231
|
destroyed_at: sql `now()`,
|
|
@@ -1016,19 +1233,19 @@ let SandboxService = class SandboxService {
|
|
|
1016
1233
|
sweep_failed_at: null,
|
|
1017
1234
|
...prev,
|
|
1018
1235
|
});
|
|
1236
|
+
this.log.info("sandbox instance destroyed", { environmentId: id });
|
|
1237
|
+
return;
|
|
1019
1238
|
}
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
}
|
|
1029
|
-
this.log.info("sandbox destroyed", { conversationId: id });
|
|
1239
|
+
const r = await this.db
|
|
1240
|
+
.deleteFrom("environment_instances")
|
|
1241
|
+
.where("environment_id", "=", id)
|
|
1242
|
+
.where("generation", "=", fence.gen)
|
|
1243
|
+
.executeTakeFirst();
|
|
1244
|
+
if (Number(r.numDeletedRows) !== 1)
|
|
1245
|
+
throw new SandboxLockLostError(id, "generation");
|
|
1246
|
+
this.log.info("environment destroyed", { environmentId: id });
|
|
1030
1247
|
}
|
|
1031
|
-
/**
|
|
1248
|
+
/** Due for suspension: made by this provider, running, and idle or running for too long. */
|
|
1032
1249
|
suspendDue(now, plugin) {
|
|
1033
1250
|
const lim = this.config.get().sandbox.limits;
|
|
1034
1251
|
const idleBefore = new Date(now.getTime() - lim.suspendAfterIdleMinutes * 60_000);
|
|
@@ -1039,24 +1256,28 @@ let SandboxService = class SandboxService {
|
|
|
1039
1256
|
eb.or([eb("last_used_at", "<", idleBefore), eb("running_since", "<", runningBefore)]),
|
|
1040
1257
|
]);
|
|
1041
1258
|
}
|
|
1042
|
-
/**
|
|
1043
|
-
|
|
1259
|
+
/** The environment was deleted, or its org was: the instance and the files go. */
|
|
1260
|
+
environmentGone(eb) {
|
|
1261
|
+
return eb.or([
|
|
1262
|
+
eb.exists(eb
|
|
1263
|
+
.selectFrom("environments")
|
|
1264
|
+
.select("environments.id")
|
|
1265
|
+
.whereRef("environments.id", "=", "environment_instances.environment_id")
|
|
1266
|
+
.where("environments.deleted_at", "is not", null)),
|
|
1267
|
+
eb.exists(eb
|
|
1268
|
+
.selectFrom("orgs")
|
|
1269
|
+
.select("orgs.id")
|
|
1270
|
+
.whereRef("orgs.id", "=", "environment_instances.org_id")
|
|
1271
|
+
.where("orgs.deleted_at", "is not", null)),
|
|
1272
|
+
]);
|
|
1273
|
+
}
|
|
1274
|
+
/** Due for instance destruction: an instance (or a create in progress) unused for too long. */
|
|
1275
|
+
instanceDue(now) {
|
|
1044
1276
|
const days = this.config.get().sandbox.limits.destroyAfterInactiveDays;
|
|
1045
1277
|
const before = new Date(now.getTime() - days * 24 * 3_600_000);
|
|
1046
1278
|
return (eb) => eb.and([
|
|
1047
|
-
eb("status", "
|
|
1048
|
-
eb
|
|
1049
|
-
eb("last_used_at", "<", before),
|
|
1050
|
-
eb.exists(eb
|
|
1051
|
-
.selectFrom("orgs")
|
|
1052
|
-
.select("orgs.id")
|
|
1053
|
-
.whereRef("orgs.id", "=", "sandboxes.org_id")
|
|
1054
|
-
.where("orgs.deleted_at", "is not", null)),
|
|
1055
|
-
eb.not(eb.exists(eb
|
|
1056
|
-
.selectFrom("conversations")
|
|
1057
|
-
.select("conversations.id")
|
|
1058
|
-
.whereRef("conversations.id", "=", "sandboxes.conversation_id"))),
|
|
1059
|
-
]),
|
|
1279
|
+
eb("status", "in", ["creating", "running", "suspended"]),
|
|
1280
|
+
eb("last_used_at", "<", before),
|
|
1060
1281
|
]);
|
|
1061
1282
|
}
|
|
1062
1283
|
};
|