@capybara-hq/kernel 0.1.0-rc.2 → 0.1.0-rc.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bootstrap.json +85 -5
- package/dist/admin/admin-agents.controller.d.ts +28 -0
- package/dist/admin/admin-agents.controller.js +79 -0
- package/dist/admin/admin-connectors.controller.d.ts +29 -0
- package/dist/admin/admin-connectors.controller.js +79 -0
- package/dist/admin/admin-items.schemas.d.ts +42 -0
- package/dist/admin/admin-items.schemas.js +48 -0
- package/dist/admin/admin.controller.d.ts +52 -80
- package/dist/admin/admin.controller.js +112 -81
- package/dist/admin/admin.module.d.ts +4 -1
- package/dist/admin/admin.module.js +50 -4
- package/dist/admin/admin.schemas.d.ts +69 -0
- package/dist/admin/admin.schemas.js +53 -0
- package/dist/admin/audit.controller.d.ts +14 -0
- package/dist/admin/audit.controller.js +42 -0
- package/dist/admin/audit.schemas.d.ts +8 -0
- package/dist/admin/audit.schemas.js +17 -0
- package/dist/admin/extensions.controller.d.ts +34 -3
- package/dist/admin/extensions.controller.js +66 -8
- package/dist/admin/extensions.schemas.d.ts +7 -0
- package/dist/admin/extensions.schemas.js +10 -0
- package/dist/admin/health.service.d.ts +104 -0
- package/dist/admin/health.service.js +246 -0
- package/dist/admin/limit-defaults.d.ts +20 -0
- package/dist/admin/limit-defaults.js +10 -0
- package/dist/admin/overview.controller.d.ts +31 -0
- package/dist/admin/overview.controller.js +78 -0
- package/dist/admin/overview.schemas.d.ts +8 -0
- package/dist/admin/overview.schemas.js +13 -0
- package/dist/admin/overview.service.d.ts +112 -0
- package/dist/admin/overview.service.js +223 -0
- package/dist/admin/platform-audit.service.d.ts +41 -0
- package/dist/admin/platform-audit.service.js +70 -0
- package/dist/admin/setup-lock.d.ts +1 -1
- package/dist/admin/setup-lock.js +1 -1
- package/dist/admin/setup.controller.d.ts +1 -1
- package/dist/admin/setup.controller.js +1 -1
- package/dist/admin/setup.service.d.ts +4 -4
- package/dist/admin/setup.service.js +8 -8
- package/dist/admin/usage.service.d.ts +149 -0
- package/dist/admin/usage.service.js +321 -0
- package/dist/agents/admin-agents.d.ts +92 -0
- package/dist/agents/admin-agents.js +170 -0
- package/dist/agents/agent-lookup.d.ts +24 -0
- package/dist/agents/agent-lookup.js +46 -2
- package/dist/agents/agent-profile.service.d.ts +31 -0
- package/dist/agents/agent-profile.service.js +122 -0
- package/dist/agents/agents.controller.d.ts +53 -168
- package/dist/agents/agents.controller.js +163 -34
- package/dist/agents/agents.module.d.ts +6 -2
- package/dist/agents/agents.module.js +41 -7
- package/dist/agents/agents.schemas.d.ts +392 -0
- package/dist/agents/agents.schemas.js +126 -0
- package/dist/agents/agents.service.d.ts +121 -36
- package/dist/agents/agents.service.js +411 -121
- package/dist/agents/definition.d.ts +142 -21
- package/dist/agents/definition.js +389 -43
- package/dist/agents/from-template.d.ts +106 -0
- package/dist/agents/from-template.js +680 -0
- package/dist/agents/from-template.service.d.ts +72 -0
- package/dist/agents/from-template.service.js +178 -0
- package/dist/agents/instruction-files.d.ts +60 -0
- package/dist/agents/instruction-files.js +111 -0
- package/dist/agents/models.controller.d.ts +44 -6
- package/dist/agents/models.controller.js +84 -8
- package/dist/agents/org-templates.d.ts +186 -0
- package/dist/agents/org-templates.js +876 -0
- package/dist/agents/pending-connector-requests.d.ts +79 -0
- package/dist/agents/pending-connector-requests.js +215 -0
- package/dist/agents/pending-connectors.controller.d.ts +39 -0
- package/dist/agents/pending-connectors.controller.js +91 -0
- package/dist/agents/pending-connectors.service.d.ts +62 -0
- package/dist/agents/pending-connectors.service.js +282 -0
- package/dist/agents/profile.d.ts +114 -0
- package/dist/agents/profile.js +55 -0
- package/dist/agents/readiness.controller.d.ts +26 -0
- package/dist/agents/readiness.controller.js +56 -0
- package/dist/agents/templates.controller.d.ts +77 -5
- package/dist/agents/templates.controller.js +153 -7
- package/dist/agents/templates.d.ts +127 -23
- package/dist/agents/templates.js +482 -44
- package/dist/agents/templates.schemas.d.ts +20 -0
- package/dist/agents/templates.schemas.js +45 -0
- package/dist/background/background.controller.d.ts +38 -0
- package/dist/background/background.controller.js +88 -0
- package/dist/background/background.module.d.ts +10 -0
- package/dist/background/background.module.js +39 -0
- package/dist/background/background.schemas.d.ts +10 -0
- package/dist/background/background.schemas.js +16 -0
- package/dist/background/finish.d.ts +44 -0
- package/dist/background/finish.js +95 -0
- package/dist/background/launch-lock.d.ts +15 -0
- package/dist/background/launch-lock.js +40 -0
- package/dist/background/launch.d.ts +42 -0
- package/dist/background/launch.js +77 -0
- package/dist/background/poll.job.d.ts +56 -0
- package/dist/background/poll.job.js +195 -0
- package/dist/background/render.d.ts +23 -0
- package/dist/background/render.js +48 -0
- package/dist/background/session.d.ts +48 -0
- package/dist/background/session.js +80 -0
- package/dist/background/status.d.ts +35 -0
- package/dist/background/status.js +57 -0
- package/dist/background/stop.d.ts +68 -0
- package/dist/background/stop.js +244 -0
- package/dist/background/store.d.ts +167 -0
- package/dist/background/store.js +361 -0
- package/dist/background/store.module.d.ts +6 -0
- package/dist/background/store.module.js +22 -0
- package/dist/background/tasks.service.d.ts +109 -0
- package/dist/background/tasks.service.js +271 -0
- package/dist/background/wake-rules.d.ts +66 -0
- package/dist/background/wake-rules.js +113 -0
- package/dist/background/wake.d.ts +30 -0
- package/dist/background/wake.js +63 -0
- package/dist/catalog/admin-catalog.controller.d.ts +136 -52
- package/dist/catalog/admin-catalog.controller.js +275 -65
- package/dist/catalog/admin-catalog.schemas.d.ts +82 -0
- package/dist/catalog/admin-catalog.schemas.js +99 -0
- package/dist/catalog/appearance.schemas.d.ts +10 -0
- package/dist/catalog/appearance.schemas.js +20 -0
- package/dist/catalog/appearance.service.d.ts +105 -0
- package/dist/catalog/appearance.service.js +306 -0
- package/dist/catalog/catalog-content.d.ts +122 -35
- package/dist/catalog/catalog-content.js +122 -14
- package/dist/catalog/catalog-files.d.ts +42 -0
- package/dist/catalog/catalog-files.js +89 -0
- package/dist/catalog/catalog-import.module.d.ts +7 -0
- package/dist/catalog/catalog-import.module.js +63 -0
- package/dist/catalog/catalog-items.d.ts +122 -7
- package/dist/catalog/catalog-items.js +301 -13
- package/dist/catalog/catalog-scope.service.d.ts +293 -0
- package/dist/catalog/catalog-scope.service.js +1236 -0
- package/dist/catalog/catalog.module.d.ts +7 -9
- package/dist/catalog/catalog.module.js +47 -31
- package/dist/catalog/catalog.service.d.ts +122 -31
- package/dist/catalog/catalog.service.js +177 -105
- package/dist/catalog/claude/agents.js +15 -10
- package/dist/catalog/claude/translate.js +3 -0
- package/dist/catalog/cli.d.ts +11 -1
- package/dist/catalog/cli.js +29 -1
- package/dist/catalog/component-promotion.d.ts +96 -0
- package/dist/catalog/component-promotion.js +395 -0
- package/dist/catalog/component-scope.controller.d.ts +102 -0
- package/dist/catalog/component-scope.controller.js +257 -0
- package/dist/catalog/component-scope.schemas.d.ts +118 -0
- package/dist/catalog/component-scope.schemas.js +97 -0
- package/dist/catalog/component-scope.service.d.ts +394 -0
- package/dist/catalog/component-scope.service.js +1133 -0
- package/dist/catalog/component-state.d.ts +207 -0
- package/dist/catalog/component-state.js +484 -0
- package/dist/catalog/git-source.d.ts +9 -1
- package/dist/catalog/git-source.js +29 -2
- package/dist/catalog/job-ids.d.ts +30 -0
- package/dist/catalog/job-ids.js +25 -0
- package/dist/catalog/marketplace-fetch.d.ts +1 -1
- package/dist/catalog/marketplace-icons.d.ts +64 -0
- package/dist/catalog/marketplace-icons.js +131 -0
- package/dist/catalog/marketplaces.controller.d.ts +50 -41
- package/dist/catalog/marketplaces.controller.js +130 -46
- package/dist/catalog/marketplaces.schemas.d.ts +48 -0
- package/dist/catalog/marketplaces.schemas.js +53 -0
- package/dist/catalog/marketplaces.service.d.ts +110 -17
- package/dist/catalog/marketplaces.service.js +337 -29
- package/dist/catalog/org-catalog.controller.d.ts +114 -62
- package/dist/catalog/org-catalog.controller.js +191 -98
- package/dist/catalog/org-catalog.schemas.d.ts +104 -0
- package/dist/catalog/org-catalog.schemas.js +107 -0
- package/dist/catalog/org-catalog.service.d.ts +72 -33
- package/dist/catalog/org-catalog.service.js +367 -158
- package/dist/catalog/org-extension-switch.d.ts +41 -0
- package/dist/catalog/org-extension-switch.js +115 -0
- package/dist/catalog/org-extension.controller.d.ts +133 -0
- package/dist/catalog/org-extension.controller.js +313 -0
- package/dist/catalog/org-extension.schemas.d.ts +15 -0
- package/dist/catalog/org-extension.schemas.js +22 -0
- package/dist/catalog/org-plugin-uploads.controller.d.ts +77 -0
- package/dist/catalog/org-plugin-uploads.controller.js +200 -0
- package/dist/catalog/org-plugin-uploads.d.ts +71 -0
- package/dist/catalog/org-plugin-uploads.js +122 -0
- package/dist/catalog/org-plugin-uploads.schemas.d.ts +20 -0
- package/dist/catalog/org-plugin-uploads.schemas.js +32 -0
- package/dist/catalog/owned-items.d.ts +188 -0
- package/dist/catalog/owned-items.js +808 -0
- package/dist/catalog/owned-plugin-cleanup.d.ts +56 -0
- package/dist/catalog/owned-plugin-cleanup.js +194 -0
- package/dist/catalog/ownership.d.ts +172 -0
- package/dist/catalog/ownership.js +326 -0
- package/dist/catalog/platform-components.d.ts +243 -0
- package/dist/catalog/platform-components.js +872 -0
- package/dist/catalog/platform-migration.d.ts +115 -0
- package/dist/catalog/platform-migration.js +583 -0
- package/dist/catalog/plugin-import.d.ts +254 -47
- package/dist/catalog/plugin-import.js +810 -112
- package/dist/catalog/plugin-install.d.ts +205 -42
- package/dist/catalog/plugin-install.js +828 -126
- package/dist/catalog/plugin-package.d.ts +43 -4
- package/dist/catalog/plugin-package.js +194 -7
- package/dist/catalog/plugin-upgrade.d.ts +308 -225
- package/dist/catalog/plugin-upgrade.js +1325 -1131
- package/dist/catalog/request-rules.d.ts +65 -0
- package/dist/catalog/request-rules.js +217 -0
- package/dist/catalog/requests.service.d.ts +194 -118
- package/dist/catalog/requests.service.js +530 -343
- package/dist/catalog/reserved-names.d.ts +8 -0
- package/dist/catalog/reserved-names.js +42 -0
- package/dist/catalog/scope-orgs.d.ts +21 -0
- package/dist/catalog/scope-orgs.js +67 -0
- package/dist/catalog/scope-state.d.ts +35 -0
- package/dist/catalog/scope-state.js +87 -0
- package/dist/catalog/scope-sync-workers.d.ts +7 -0
- package/dist/catalog/scope-sync-workers.js +29 -0
- package/dist/catalog/scope-sync.d.ts +176 -0
- package/dist/catalog/scope-sync.js +576 -0
- package/dist/catalog/scope.d.ts +42 -0
- package/dist/catalog/scope.js +42 -0
- package/dist/catalog/unshare.d.ts +22 -0
- package/dist/catalog/unshare.js +119 -0
- package/dist/channels/actions.js +10 -3
- package/dist/channels/cards.d.ts +36 -10
- package/dist/channels/cards.js +100 -25
- package/dist/channels/channels.module.js +13 -2
- package/dist/channels/channels.schemas.d.ts +41 -0
- package/dist/channels/channels.schemas.js +40 -0
- package/dist/channels/connections.controller.d.ts +46 -37
- package/dist/channels/connections.controller.js +76 -34
- package/dist/channels/connections.service.d.ts +102 -10
- package/dist/channels/connections.service.js +271 -24
- package/dist/channels/inbound.d.ts +11 -1
- package/dist/channels/inbound.js +87 -7
- package/dist/channels/outbound.d.ts +14 -1
- package/dist/channels/outbound.js +64 -15
- package/dist/channels/phrases.d.ts +33 -2
- package/dist/channels/phrases.js +29 -1
- package/dist/channels/supervisor.d.ts +2 -0
- package/dist/channels/supervisor.js +4 -0
- package/dist/channels/threads.d.ts +5 -0
- package/dist/channels/threads.js +8 -0
- package/dist/channels/webhook.d.ts +21 -1
- package/dist/channels/webhook.js +39 -4
- package/dist/client/client-access.service.d.ts +102 -0
- package/dist/client/client-access.service.js +303 -0
- package/dist/client/client-builds.service.d.ts +109 -0
- package/dist/client/client-builds.service.js +229 -0
- package/dist/client/client-delivery.controller.d.ts +25 -0
- package/dist/client/client-delivery.controller.js +66 -0
- package/dist/client/client-files.d.ts +48 -0
- package/dist/client/client-files.js +136 -0
- package/dist/client/client-reports.cron.d.ts +13 -0
- package/dist/client/client-reports.cron.js +42 -0
- package/dist/client/client-reports.service.d.ts +30 -0
- package/dist/client/client-reports.service.js +102 -0
- package/dist/client/client-sign-in.service.d.ts +21 -0
- package/dist/client/client-sign-in.service.js +49 -0
- package/dist/client/client.controller.d.ts +49 -0
- package/dist/client/client.controller.js +129 -0
- package/dist/client/client.module.d.ts +8 -0
- package/dist/client/client.module.js +28 -0
- package/dist/client/client.schemas.d.ts +61 -0
- package/dist/client/client.schemas.js +27 -0
- package/dist/config/docs.js +5 -1
- package/dist/config/schema.d.ts +35 -2
- package/dist/config/schema.js +216 -21
- package/dist/conversations/access.d.ts +6 -4
- package/dist/conversations/access.js +29 -5
- package/dist/conversations/agent-tree.d.ts +46 -0
- package/dist/conversations/agent-tree.js +92 -0
- package/dist/conversations/artifacts.d.ts +1 -1
- package/dist/conversations/artifacts.js +1 -1
- package/dist/conversations/attachments.service.d.ts +30 -0
- package/dist/conversations/attachments.service.js +130 -5
- package/dist/conversations/conversations.controller.d.ts +79 -44
- package/dist/conversations/conversations.controller.js +133 -82
- package/dist/conversations/conversations.module.d.ts +1 -1
- package/dist/conversations/conversations.module.js +10 -7
- package/dist/conversations/conversations.schemas.d.ts +52 -0
- package/dist/conversations/conversations.schemas.js +79 -0
- package/dist/conversations/conversations.service.d.ts +161 -13
- package/dist/conversations/conversations.service.js +329 -57
- package/dist/conversations/device-status.d.ts +24 -0
- package/dist/conversations/device-status.js +71 -0
- package/dist/conversations/event-log.d.ts +45 -9
- package/dist/conversations/event-log.js +113 -6
- package/dist/conversations/events.d.ts +255 -9
- package/dist/conversations/events.js +250 -34
- package/dist/conversations/message-context.d.ts +79 -0
- package/dist/conversations/message-context.js +145 -0
- package/dist/conversations/queue.d.ts +5 -2
- package/dist/conversations/queue.js +24 -4
- package/dist/conversations/queue.service.d.ts +2 -2
- package/dist/conversations/queue.service.js +21 -3
- package/dist/conversations/titles.d.ts +12 -3
- package/dist/conversations/titles.js +40 -16
- package/dist/conversations/tree.d.ts +17 -0
- package/dist/conversations/tree.js +42 -0
- package/dist/ctx/create-ctx.d.ts +48 -5
- package/dist/ctx/create-ctx.js +174 -34
- package/dist/db/kernel-schema.d.ts +668 -27
- package/dist/db/pool.d.ts +5 -0
- package/dist/db/pool.js +3 -0
- package/dist/devices/access.d.ts +66 -0
- package/dist/devices/access.js +113 -0
- package/dist/devices/conversation-devices.d.ts +75 -0
- package/dist/devices/conversation-devices.js +230 -0
- package/dist/devices/devices.controller.d.ts +36 -0
- package/dist/devices/devices.controller.js +124 -0
- package/dist/devices/devices.module.d.ts +8 -0
- package/dist/devices/devices.module.js +34 -0
- package/dist/devices/devices.schemas.d.ts +3 -0
- package/dist/devices/devices.schemas.js +14 -0
- package/dist/devices/devices.service.d.ts +251 -0
- package/dist/devices/devices.service.js +791 -0
- package/dist/devices/dto.d.ts +25 -0
- package/dist/devices/dto.js +12 -0
- package/dist/devices/label.d.ts +11 -0
- package/dist/devices/label.js +25 -0
- package/dist/devices/legacy-cleanup.job.d.ts +34 -0
- package/dist/devices/legacy-cleanup.job.js +141 -0
- package/dist/errors.d.ts +215 -3
- package/dist/errors.js +215 -3
- package/dist/evals/evals.module.js +11 -2
- package/dist/evals/evals.schemas.d.ts +110 -0
- package/dist/evals/evals.schemas.js +107 -0
- package/dist/evals/reconstruct.d.ts +18 -9
- package/dist/evals/reconstruct.js +36 -10
- package/dist/evals/recorded-tools.d.ts +37 -1
- package/dist/evals/recorded-tools.js +120 -10
- package/dist/evals/run-replay.d.ts +20 -3
- package/dist/evals/run-replay.js +78 -14
- package/dist/evals/runs.controller.d.ts +2 -57
- package/dist/evals/runs.controller.js +1 -42
- package/dist/evals/runs.service.d.ts +8 -7
- package/dist/evals/runs.service.js +45 -30
- package/dist/evals/sets.controller.d.ts +2 -28
- package/dist/evals/sets.controller.js +1 -31
- package/dist/evals/sets.service.d.ts +7 -7
- package/dist/evals/sets.service.js +23 -17
- package/dist/evals/snapshot.d.ts +3 -2
- package/dist/evals/snapshot.js +17 -4
- package/dist/evals/step-replay.d.ts +4 -6
- package/dist/evals/step-replay.js +16 -9
- package/dist/evals/trajectory.d.ts +5 -3
- package/dist/evals/trajectory.js +4 -1
- package/dist/evals/trajectory.service.d.ts +8 -3
- package/dist/evals/trajectory.service.js +26 -14
- package/dist/evals/worker.d.ts +5 -5
- package/dist/evals/worker.js +9 -3
- package/dist/extensions/admin-ops.d.ts +81 -14
- package/dist/extensions/admin-ops.js +181 -47
- package/dist/extensions/admin-view.d.ts +57 -10
- package/dist/extensions/admin-view.js +249 -27
- package/dist/extensions/approvals.d.ts +12 -2
- package/dist/extensions/blobs.d.ts +14 -1
- package/dist/extensions/blobs.js +49 -5
- package/dist/extensions/bootstrap.d.ts +41 -5
- package/dist/extensions/bootstrap.js +121 -19
- package/dist/extensions/cli.d.ts +12 -0
- package/dist/extensions/cli.js +11 -2
- package/dist/extensions/config-hash.d.ts +10 -0
- package/dist/extensions/config-hash.js +38 -0
- package/dist/extensions/config-issues.d.ts +13 -1
- package/dist/extensions/config-issues.js +22 -5
- package/dist/extensions/config-json-schema.d.ts +43 -7
- package/dist/extensions/config-json-schema.js +124 -58
- package/dist/extensions/deployment-config-store.d.ts +6 -0
- package/dist/extensions/deployment-config.d.ts +94 -9
- package/dist/extensions/deployment-config.js +255 -34
- package/dist/extensions/display-name.d.ts +22 -0
- package/dist/extensions/display-name.js +77 -0
- package/dist/extensions/effective-config.d.ts +133 -0
- package/dist/extensions/effective-config.js +379 -0
- package/dist/extensions/extensions.module.js +31 -12
- package/dist/extensions/guide-places.d.ts +68 -0
- package/dist/extensions/guide-places.js +152 -0
- package/dist/extensions/hooks.d.ts +37 -4
- package/dist/extensions/hooks.js +82 -16
- package/dist/extensions/host.d.ts +105 -12
- package/dist/extensions/host.js +372 -87
- package/dist/extensions/installs.d.ts +30 -9
- package/dist/extensions/installs.js +57 -16
- package/dist/extensions/instance.d.ts +17 -2
- package/dist/extensions/instance.js +21 -0
- package/dist/extensions/ledger.d.ts +1 -1
- package/dist/extensions/locks.d.ts +12 -0
- package/dist/extensions/locks.js +26 -0
- package/dist/extensions/org-config.d.ts +97 -7
- package/dist/extensions/org-config.js +215 -45
- package/dist/extensions/org-levels.d.ts +37 -0
- package/dist/extensions/org-levels.js +71 -0
- package/dist/extensions/org-overrides.d.ts +151 -0
- package/dist/extensions/org-overrides.js +524 -0
- package/dist/extensions/package.d.ts +18 -4
- package/dist/extensions/package.js +56 -3
- package/dist/extensions/permission-set.d.ts +35 -5
- package/dist/extensions/permission-set.js +36 -5
- package/dist/extensions/permission-view.d.ts +106 -13
- package/dist/extensions/permission-view.js +33 -15
- package/dist/extensions/plugin-guides.d.ts +282 -0
- package/dist/extensions/plugin-guides.js +554 -0
- package/dist/extensions/plugin-icons.d.ts +54 -0
- package/dist/extensions/plugin-icons.js +125 -0
- package/dist/extensions/plugin-items.d.ts +21 -0
- package/dist/extensions/plugin-items.js +42 -0
- package/dist/extensions/point-defaults.controller.d.ts +28 -0
- package/dist/extensions/point-defaults.controller.js +69 -0
- package/dist/extensions/point-defaults.d.ts +73 -0
- package/dist/extensions/point-defaults.js +236 -0
- package/dist/extensions/point-defaults.schemas.d.ts +9 -0
- package/dist/extensions/point-defaults.schemas.js +14 -0
- package/dist/extensions/registry.d.ts +1 -1
- package/dist/extensions/registry.js +2 -0
- package/dist/extensions/resolve.d.ts +51 -1
- package/dist/extensions/resolve.js +85 -3
- package/dist/extensions/single-provider.d.ts +92 -0
- package/dist/extensions/single-provider.js +180 -0
- package/dist/extensions/sources.d.ts +13 -2
- package/dist/extensions/sources.js +2 -0
- package/dist/extensions/state.module.d.ts +9 -0
- package/dist/extensions/state.module.js +46 -0
- package/dist/extensions/unavailable.d.ts +50 -0
- package/dist/extensions/unavailable.js +115 -0
- package/dist/files/previewers.controller.js +2 -3
- package/dist/files/previewers.d.ts +10 -3
- package/dist/files/previewers.js +40 -11
- package/dist/files/previewers.schemas.d.ts +10 -0
- package/dist/files/previewers.schemas.js +9 -0
- package/dist/http/api.module.js +76 -4
- package/dist/http/app.d.ts +11 -2
- package/dist/http/app.js +31 -5
- package/dist/http/gateway-routes.d.ts +7 -3
- package/dist/http/gateway-routes.js +6 -3
- package/dist/http/nest-logger.d.ts +6 -5
- package/dist/http/nest-logger.js +24 -10
- package/dist/http/plugin-dispatch.d.ts +4 -1
- package/dist/http/plugin-dispatch.js +14 -2
- package/dist/identity/accounts.d.ts +77 -31
- package/dist/identity/accounts.js +133 -52
- package/dist/identity/audit.d.ts +2 -1
- package/dist/identity/auth.controller.js +9 -5
- package/dist/identity/auth.guard.d.ts +13 -5
- package/dist/identity/auth.guard.js +35 -9
- package/dist/identity/cookies.d.ts +26 -5
- package/dist/identity/cookies.js +58 -6
- package/dist/identity/identity.module.d.ts +1 -1
- package/dist/identity/identity.module.js +7 -4
- package/dist/identity/initial-passwords.controller.d.ts +22 -0
- package/dist/identity/initial-passwords.controller.js +49 -0
- package/dist/identity/initial-passwords.d.ts +61 -0
- package/dist/identity/initial-passwords.job.d.ts +16 -0
- package/dist/identity/initial-passwords.job.js +46 -0
- package/dist/identity/initial-passwords.js +159 -0
- package/dist/identity/me.controller.d.ts +17 -1
- package/dist/identity/me.controller.js +45 -4
- package/dist/identity/me.schemas.d.ts +24 -0
- package/dist/identity/me.schemas.js +25 -0
- package/dist/identity/principal.d.ts +11 -3
- package/dist/identity/principal.js +11 -3
- package/dist/identity/sessions.d.ts +12 -0
- package/dist/identity/sessions.js +45 -0
- package/dist/identity/settings.d.ts +99 -12
- package/dist/identity/settings.js +234 -25
- package/dist/identity/{enterprise-admin.d.ts → super-admin.d.ts} +5 -5
- package/dist/identity/{enterprise-admin.js → super-admin.js} +24 -24
- package/dist/identity/write-actor.d.ts +11 -0
- package/dist/identity/write-actor.js +7 -0
- package/dist/inbox/inbox.module.d.ts +4 -1
- package/dist/inbox/inbox.module.js +7 -3
- package/dist/inbox/inbox.schemas.d.ts +8 -0
- package/dist/inbox/inbox.schemas.js +11 -0
- package/dist/inbox/inbox.service.d.ts +64 -4
- package/dist/inbox/inbox.service.js +74 -4
- package/dist/inbox/org-notices.d.ts +63 -0
- package/dist/inbox/org-notices.js +91 -0
- package/dist/inbox/platform-inbox.controller.d.ts +23 -0
- package/dist/inbox/platform-inbox.controller.js +58 -0
- package/dist/inbox/platform-inbox.service.d.ts +157 -0
- package/dist/inbox/platform-inbox.service.js +281 -0
- package/dist/inbox/platform-notices.d.ts +19 -0
- package/dist/inbox/platform-notices.js +9 -0
- package/dist/index.d.ts +6 -2
- package/dist/index.js +7 -1
- package/dist/infra/cli/kernel-cli.module.js +9 -1
- package/dist/infra/config/kernel-config.service.js +32 -1
- package/dist/infra/database/database.module.js +1 -0
- package/dist/infra/database/xact-lock.d.ts +1 -1
- package/dist/infra/database/xact-lock.js +1 -1
- package/dist/infra/events/kernel-events.d.ts +31 -3
- package/dist/infra/lifecycle/process-guards.js +4 -0
- package/dist/infra/lock/lock.service.d.ts +8 -0
- package/dist/infra/lock/lock.service.js +27 -0
- package/dist/infra/logging/logging.module.d.ts +4 -2
- package/dist/infra/logging/logging.module.js +24 -6
- package/dist/infra/presence/presence.module.d.ts +6 -0
- package/dist/infra/presence/presence.module.js +22 -0
- package/dist/infra/presence/replica-presence.d.ts +87 -0
- package/dist/infra/presence/replica-presence.js +189 -0
- package/dist/infra/queue/add-job-once.d.ts +14 -0
- package/dist/infra/queue/add-job-once.js +16 -0
- package/dist/infra/queue/redis-connection.d.ts +6 -0
- package/dist/infra/queue/redis-connection.js +7 -0
- package/dist/infra/redis/redis.module.js +6 -2
- package/dist/infra/throttler/named-throttlers.d.ts +12 -0
- package/dist/infra/throttler/named-throttlers.js +67 -0
- package/dist/infra/throttler/throttler.guard.js +1 -1
- package/dist/infra/tokens.d.ts +6 -1
- package/dist/infra/tokens.js +6 -1
- package/dist/kernel.module.js +15 -0
- package/dist/lifecycle/kernel-lifecycle.d.ts +4 -2
- package/dist/lifecycle/kernel-lifecycle.js +35 -7
- package/dist/local-devices/device-auth.controller.d.ts +42 -0
- package/dist/local-devices/device-auth.controller.js +116 -0
- package/dist/local-devices/device-auth.d.ts +119 -0
- package/dist/local-devices/device-auth.js +401 -0
- package/dist/local-devices/device-auth.module.d.ts +7 -0
- package/dist/local-devices/device-auth.module.js +25 -0
- package/dist/local-devices/device-routes.d.ts +36 -0
- package/dist/local-devices/device-routes.js +90 -0
- package/dist/local-devices/dto.d.ts +66 -0
- package/dist/local-devices/dto.js +44 -0
- package/dist/local-devices/keys.d.ts +25 -0
- package/dist/local-devices/keys.js +25 -0
- package/dist/local-devices/local-device-gateway.d.ts +134 -0
- package/dist/local-devices/local-device-gateway.js +542 -0
- package/dist/local-devices/local-device-provider.d.ts +89 -0
- package/dist/local-devices/local-device-provider.js +151 -0
- package/dist/local-devices/local-devices.module.d.ts +16 -0
- package/dist/local-devices/local-devices.module.js +44 -0
- package/dist/local-devices/machines.d.ts +118 -0
- package/dist/local-devices/machines.js +366 -0
- package/dist/local-devices/relay.d.ts +100 -0
- package/dist/local-devices/relay.js +373 -0
- package/dist/local-devices/targets.d.ts +28 -0
- package/dist/local-devices/targets.js +73 -0
- package/dist/log.d.ts +13 -1
- package/dist/log.js +93 -17
- package/dist/models/embeddings.d.ts +82 -0
- package/dist/models/embeddings.js +326 -0
- package/dist/models/enablement.controller.d.ts +40 -0
- package/dist/models/enablement.controller.js +74 -0
- package/dist/models/enablement.d.ts +101 -0
- package/dist/models/enablement.js +264 -0
- package/dist/models/models.module.d.ts +5 -1
- package/dist/models/models.module.js +16 -4
- package/dist/models/org-models.schemas.d.ts +8 -0
- package/dist/models/org-models.schemas.js +22 -0
- package/dist/models/prices.d.ts +8 -6
- package/dist/models/prices.js +11 -9
- package/dist/models/purposes.controller.d.ts +6 -4
- package/dist/models/purposes.controller.js +13 -7
- package/dist/models/purposes.d.ts +72 -14
- package/dist/models/purposes.js +115 -34
- package/dist/orgs/directory-sync.d.ts +14 -5
- package/dist/orgs/directory-sync.js +41 -17
- package/dist/orgs/offboarding.d.ts +12 -8
- package/dist/orgs/offboarding.js +54 -33
- package/dist/orgs/org-lock.d.ts +5 -0
- package/dist/orgs/org-lock.js +13 -0
- package/dist/orgs/org.guard.d.ts +10 -0
- package/dist/orgs/org.guard.js +43 -32
- package/dist/orgs/orgs.controller.d.ts +2 -5
- package/dist/orgs/orgs.controller.js +3 -6
- package/dist/orgs/orgs.module.js +3 -0
- package/dist/orgs/orgs.schemas.d.ts +8 -0
- package/dist/orgs/orgs.schemas.js +16 -0
- package/dist/orgs/orgs.service.d.ts +23 -8
- package/dist/orgs/orgs.service.js +38 -16
- package/dist/orgs/roles.d.ts +1 -1
- package/dist/orgs/roles.js +5 -1
- package/dist/orgs/settings.controller.d.ts +1 -1
- package/dist/orgs/settings.service.d.ts +66 -6
- package/dist/orgs/settings.service.js +89 -10
- package/dist/overrides.d.ts +16 -1
- package/dist/overrides.js +10 -0
- package/dist/platform/actor.d.ts +48 -0
- package/dist/platform/actor.js +100 -0
- package/dist/platform/agent-docs.d.ts +18 -0
- package/dist/platform/agent-docs.js +59 -0
- package/dist/platform/availability.d.ts +14 -0
- package/dist/platform/availability.js +43 -0
- package/dist/platform/errors.d.ts +28 -0
- package/dist/platform/errors.js +65 -0
- package/dist/platform/exclusions.d.ts +11 -0
- package/dist/platform/exclusions.js +774 -0
- package/dist/platform/method.d.ts +6 -0
- package/dist/platform/method.js +15 -0
- package/dist/platform/operation.d.ts +169 -0
- package/dist/platform/operation.js +102 -0
- package/dist/platform/ops/accounts.ops.d.ts +32 -0
- package/dist/platform/ops/accounts.ops.js +291 -0
- package/dist/platform/ops/admin-items.ops.d.ts +37 -0
- package/dist/platform/ops/admin-items.ops.js +278 -0
- package/dist/platform/ops/admin.ops.d.ts +38 -0
- package/dist/platform/ops/admin.ops.js +497 -0
- package/dist/platform/ops/agent-skills.ops.d.ts +50 -0
- package/dist/platform/ops/agent-skills.ops.js +509 -0
- package/dist/platform/ops/agent-templates.ops.d.ts +17 -0
- package/dist/platform/ops/agent-templates.ops.js +272 -0
- package/dist/platform/ops/agents.ops.d.ts +37 -0
- package/dist/platform/ops/agents.ops.js +730 -0
- package/dist/platform/ops/approvals.ops.d.ts +30 -0
- package/dist/platform/ops/approvals.ops.js +116 -0
- package/dist/platform/ops/background.ops.d.ts +14 -0
- package/dist/platform/ops/background.ops.js +134 -0
- package/dist/platform/ops/catalog-admin.ops.d.ts +92 -0
- package/dist/platform/ops/catalog-admin.ops.js +682 -0
- package/dist/platform/ops/channels.ops.d.ts +16 -0
- package/dist/platform/ops/channels.ops.js +195 -0
- package/dist/platform/ops/components.ops.d.ts +46 -0
- package/dist/platform/ops/components.ops.js +854 -0
- package/dist/platform/ops/connectors.ops.d.ts +17 -0
- package/dist/platform/ops/connectors.ops.js +190 -0
- package/dist/platform/ops/conversations.ops.d.ts +30 -0
- package/dist/platform/ops/conversations.ops.js +470 -0
- package/dist/platform/ops/devices.ops.d.ts +26 -0
- package/dist/platform/ops/devices.ops.js +311 -0
- package/dist/platform/ops/errors.d.ts +16 -0
- package/dist/platform/ops/errors.js +30 -0
- package/dist/platform/ops/evals.ops.d.ts +28 -0
- package/dist/platform/ops/evals.ops.js +783 -0
- package/dist/platform/ops/extensions-admin.ops.d.ts +25 -0
- package/dist/platform/ops/extensions-admin.ops.js +372 -0
- package/dist/platform/ops/extensions.ops.d.ts +25 -0
- package/dist/platform/ops/extensions.ops.js +347 -0
- package/dist/platform/ops/inbox.ops.d.ts +25 -0
- package/dist/platform/ops/inbox.ops.js +381 -0
- package/dist/platform/ops/marketplaces.ops.d.ts +22 -0
- package/dist/platform/ops/marketplaces.ops.js +466 -0
- package/dist/platform/ops/models.ops.d.ts +25 -0
- package/dist/platform/ops/models.ops.js +232 -0
- package/dist/platform/ops/org-plugins.ops.d.ts +35 -0
- package/dist/platform/ops/org-plugins.ops.js +388 -0
- package/dist/platform/ops/org-settings.ops.d.ts +18 -0
- package/dist/platform/ops/org-settings.ops.js +127 -0
- package/dist/platform/ops/orgs.ops.d.ts +28 -0
- package/dist/platform/ops/orgs.ops.js +298 -0
- package/dist/platform/ops/other-orgs.d.ts +20 -0
- package/dist/platform/ops/other-orgs.js +72 -0
- package/dist/platform/ops/paging.d.ts +14 -0
- package/dist/platform/ops/paging.js +20 -0
- package/dist/platform/ops/platform.ops.d.ts +30 -0
- package/dist/platform/ops/platform.ops.js +231 -0
- package/dist/platform/ops/plugins.ops.d.ts +35 -0
- package/dist/platform/ops/plugins.ops.js +476 -0
- package/dist/platform/ops/previewers.ops.d.ts +14 -0
- package/dist/platform/ops/previewers.ops.js +109 -0
- package/dist/platform/ops/requests.ops.d.ts +14 -0
- package/dist/platform/ops/requests.ops.js +191 -0
- package/dist/platform/ops/role.d.ts +4 -0
- package/dist/platform/ops/role.js +10 -0
- package/dist/platform/ops/setup.ops.d.ts +24 -0
- package/dist/platform/ops/setup.ops.js +212 -0
- package/dist/platform/ops/skills.ops.d.ts +17 -0
- package/dist/platform/ops/skills.ops.js +160 -0
- package/dist/platform/ops/summary.d.ts +10 -0
- package/dist/platform/ops/summary.js +18 -0
- package/dist/platform/ops/themes.ops.d.ts +14 -0
- package/dist/platform/ops/themes.ops.js +47 -0
- package/dist/platform/ops/ui.ops.d.ts +8 -0
- package/dist/platform/ops/ui.ops.js +41 -0
- package/dist/platform/ops/unavailable.d.ts +23 -0
- package/dist/platform/ops/unavailable.js +118 -0
- package/dist/platform/parity.d.ts +33 -0
- package/dist/platform/parity.js +272 -0
- package/dist/platform/platform-ops.module.d.ts +43 -0
- package/dist/platform/platform-ops.module.js +142 -0
- package/dist/platform/platform.module.d.ts +8 -0
- package/dist/platform/platform.module.js +25 -0
- package/dist/platform/registry.d.ts +30 -0
- package/dist/platform/registry.js +111 -0
- package/dist/platform/route-schemas.d.ts +18 -0
- package/dist/platform/route-schemas.js +85 -0
- package/dist/roles/gateway-role.d.ts +18 -3
- package/dist/roles/gateway-role.js +33 -5
- package/dist/roles/gateway.module.d.ts +2 -1
- package/dist/roles/gateway.module.js +6 -3
- package/dist/roles/worker-role.d.ts +6 -4
- package/dist/roles/worker-role.js +14 -7
- package/dist/roles/worker.module.d.ts +1 -1
- package/dist/roles/worker.module.js +21 -1
- package/dist/runtime/context/build.d.ts +21 -13
- package/dist/runtime/context/build.js +106 -29
- package/dist/runtime/context/claude-tools.d.ts +4 -1
- package/dist/runtime/context/claude-tools.js +13 -12
- package/dist/runtime/context/compaction.js +24 -1
- package/dist/runtime/context/device-paths.d.ts +40 -0
- package/dist/runtime/context/device-paths.js +48 -0
- package/dist/runtime/context/legacy-text.d.ts +22 -0
- package/dist/runtime/context/legacy-text.js +37 -0
- package/dist/runtime/context/reminders.d.ts +2 -0
- package/dist/runtime/context/reminders.js +2 -0
- package/dist/runtime/context/run-context.d.ts +65 -0
- package/dist/runtime/context/run-context.js +215 -0
- package/dist/runtime/context/skills.d.ts +29 -11
- package/dist/runtime/context/skills.js +46 -32
- package/dist/runtime/context/tool-changes.d.ts +27 -0
- package/dist/runtime/context/tool-changes.js +60 -0
- package/dist/runtime/hook-calls.d.ts +8 -1
- package/dist/runtime/hook-calls.js +40 -7
- package/dist/runtime/langgraph-loop.d.ts +3 -1
- package/dist/runtime/langgraph-loop.js +36 -17
- package/dist/runtime/loop.d.ts +35 -6
- package/dist/runtime/maintenance.d.ts +9 -4
- package/dist/runtime/maintenance.js +35 -8
- package/dist/runtime/model-call.d.ts +54 -11
- package/dist/runtime/model-call.js +105 -28
- package/dist/runtime/providers.d.ts +149 -29
- package/dist/runtime/providers.js +332 -63
- package/dist/runtime/run-context.service.d.ts +65 -0
- package/dist/runtime/run-context.service.js +236 -0
- package/dist/runtime/run-control.d.ts +11 -1
- package/dist/runtime/run-control.js +12 -0
- package/dist/runtime/run-drafts.d.ts +85 -0
- package/dist/runtime/run-drafts.js +230 -0
- package/dist/runtime/run-finish.d.ts +1 -1
- package/dist/runtime/run-jobs.d.ts +3 -5
- package/dist/runtime/run-jobs.js +7 -9
- package/dist/runtime/run-queue.d.ts +62 -5
- package/dist/runtime/run-queue.js +313 -40
- package/dist/runtime/run-worker.d.ts +19 -8
- package/dist/runtime/run-worker.js +105 -44
- package/dist/runtime/runtime.module.js +13 -3
- package/dist/runtime/step-log.d.ts +15 -4
- package/dist/runtime/step-log.js +24 -7
- package/dist/runtime/stream.controller.d.ts +23 -1
- package/dist/runtime/stream.controller.js +60 -3
- package/dist/runtime/stream.d.ts +4 -1
- package/dist/runtime/stream.js +27 -3
- package/dist/runtime/subagents/child.d.ts +55 -7
- package/dist/runtime/subagents/child.js +105 -7
- package/dist/runtime/subagents/definition.d.ts +31 -26
- package/dist/runtime/subagents/definition.js +89 -142
- package/dist/runtime/subagents/deliveries.d.ts +71 -0
- package/dist/runtime/subagents/deliveries.js +228 -0
- package/dist/runtime/subagents/dispatch.d.ts +116 -27
- package/dist/runtime/subagents/dispatch.js +213 -142
- package/dist/runtime/subagents/limits.d.ts +24 -5
- package/dist/runtime/subagents/limits.js +40 -6
- package/dist/runtime/subagents/messages.d.ts +95 -0
- package/dist/runtime/subagents/messages.js +328 -0
- package/dist/runtime/subagents/org-agents.js +2 -1
- package/dist/runtime/subagents/preamble.d.ts +4 -4
- package/dist/runtime/subagents/preamble.js +9 -7
- package/dist/runtime/subagents/progress.d.ts +8 -6
- package/dist/runtime/subagents/progress.js +11 -6
- package/dist/runtime/subagents/render.d.ts +30 -0
- package/dist/runtime/subagents/render.js +53 -0
- package/dist/runtime/subagents/roster.d.ts +25 -8
- package/dist/runtime/subagents/roster.js +48 -12
- package/dist/runtime/subagents/skill-fork.d.ts +4 -3
- package/dist/runtime/subagents/skill-fork.js +9 -5
- package/dist/runtime/subagents/status.d.ts +14 -0
- package/dist/runtime/subagents/status.js +23 -0
- package/dist/runtime/subagents/subagents.service.d.ts +9 -6
- package/dist/runtime/subagents/subagents.service.js +13 -8
- package/dist/runtime/usage-report-jobs.d.ts +88 -0
- package/dist/runtime/usage-report-jobs.js +158 -0
- package/dist/runtime/usage-report.d.ts +35 -39
- package/dist/runtime/usage-report.js +171 -97
- package/dist/runtime/waits.d.ts +148 -39
- package/dist/runtime/waits.js +426 -112
- package/dist/sandbox/jobs.d.ts +40 -13
- package/dist/sandbox/jobs.js +57 -20
- package/dist/sandbox/leases.d.ts +37 -54
- package/dist/sandbox/leases.js +71 -115
- package/dist/sandbox/providers.d.ts +34 -7
- package/dist/sandbox/providers.js +54 -9
- package/dist/sandbox/sandbox-file-reader.d.ts +35 -0
- package/dist/sandbox/sandbox-file-reader.js +98 -0
- package/dist/sandbox/sandbox.module.d.ts +3 -2
- package/dist/sandbox/sandbox.module.js +10 -4
- package/dist/sandbox/sandbox.service.d.ts +345 -132
- package/dist/sandbox/sandbox.service.js +883 -399
- package/dist/sandbox/unreachable.d.ts +30 -0
- package/dist/sandbox/unreachable.js +108 -0
- package/dist/setup/kernel-checks.d.ts +91 -0
- package/dist/setup/kernel-checks.js +295 -0
- package/dist/setup/oauth-token-probe.d.ts +47 -0
- package/dist/setup/oauth-token-probe.js +108 -0
- package/dist/setup/redact-check.d.ts +18 -0
- package/dist/setup/redact-check.js +46 -0
- package/dist/setup/setup-check-registry.d.ts +86 -0
- package/dist/setup/setup-check-registry.js +75 -0
- package/dist/setup/setup-checks.controller.d.ts +28 -0
- package/dist/setup/setup-checks.controller.js +78 -0
- package/dist/setup/setup-checks.schemas.d.ts +38 -0
- package/dist/setup/setup-checks.schemas.js +43 -0
- package/dist/setup/setup-checks.service.d.ts +144 -0
- package/dist/setup/setup-checks.service.js +683 -0
- package/dist/setup/setup-guides.controller.d.ts +18 -0
- package/dist/setup/setup-guides.controller.js +48 -0
- package/dist/setup/setup-guides.schemas.d.ts +10 -0
- package/dist/setup/setup-guides.schemas.js +6 -0
- package/dist/setup/setup-guides.service.d.ts +126 -0
- package/dist/setup/setup-guides.service.js +315 -0
- package/dist/setup/setup.module.d.ts +8 -0
- package/dist/setup/setup.module.js +31 -0
- package/dist/skills/agent-skills.controller.d.ts +48 -0
- package/dist/skills/agent-skills.controller.js +131 -0
- package/dist/skills/agent-skills.service.d.ts +262 -0
- package/dist/skills/agent-skills.service.js +996 -0
- package/dist/skills/catalog.d.ts +14 -11
- package/dist/skills/catalog.js +32 -50
- package/dist/skills/changes.controller.d.ts +12 -22
- package/dist/skills/changes.controller.js +27 -56
- package/dist/skills/changes.service.d.ts +69 -115
- package/dist/skills/changes.service.js +249 -598
- package/dist/skills/conversation-catalog.d.ts +53 -0
- package/dist/skills/conversation-catalog.js +149 -0
- package/dist/skills/invocable.controller.d.ts +1 -1
- package/dist/skills/invocable.controller.js +2 -2
- package/dist/skills/invocation-keys.d.ts +1 -1
- package/dist/skills/invocation-keys.js +1 -1
- package/dist/skills/invoke.d.ts +3 -3
- package/dist/skills/invoke.js +21 -8
- package/dist/skills/package.d.ts +15 -1
- package/dist/skills/package.js +14 -1
- package/dist/skills/skills.controller.d.ts +32 -6
- package/dist/skills/skills.controller.js +64 -11
- package/dist/skills/skills.module.js +25 -3
- package/dist/skills/skills.schemas.d.ts +44 -0
- package/dist/skills/skills.schemas.js +68 -0
- package/dist/skills/skills.service.d.ts +123 -26
- package/dist/skills/skills.service.js +493 -76
- package/dist/skills/usable.d.ts +82 -0
- package/dist/skills/usable.js +388 -0
- package/dist/themes/render-css.d.ts +13 -0
- package/dist/themes/render-css.js +135 -0
- package/dist/themes/theme-delivery.service.d.ts +36 -0
- package/dist/themes/theme-delivery.service.js +116 -0
- package/dist/themes/theme-state.d.ts +116 -0
- package/dist/themes/theme-state.js +221 -0
- package/dist/themes/theme-usage.d.ts +61 -0
- package/dist/themes/theme-usage.js +98 -0
- package/dist/themes/themes-public.controller.d.ts +33 -0
- package/dist/themes/themes-public.controller.js +98 -0
- package/dist/themes/themes.controller.d.ts +13 -0
- package/dist/themes/themes.controller.js +38 -0
- package/dist/themes/themes.module.d.ts +3 -0
- package/dist/themes/themes.module.js +21 -0
- package/dist/themes/themes.schemas.d.ts +3 -0
- package/dist/themes/themes.schemas.js +2 -0
- package/dist/themes/themes.service.d.ts +50 -0
- package/dist/themes/themes.service.js +99 -0
- package/dist/time-zone.d.ts +5 -0
- package/dist/time-zone.js +19 -0
- package/dist/tools/admin-connectors.d.ts +69 -0
- package/dist/tools/admin-connectors.js +135 -0
- package/dist/tools/approvals.controller.js +2 -3
- package/dist/tools/approvals.schemas.d.ts +21 -0
- package/dist/tools/approvals.schemas.js +15 -0
- package/dist/tools/approvals.service.d.ts +26 -2
- package/dist/tools/approvals.service.js +44 -5
- package/dist/tools/builtin-names.d.ts +104 -0
- package/dist/tools/builtin-names.js +164 -0
- package/dist/tools/builtins/agent.d.ts +27 -25
- package/dist/tools/builtins/agent.js +66 -63
- package/dist/tools/builtins/ask-user.d.ts +1 -1
- package/dist/tools/builtins/ask-user.js +5 -4
- package/dist/tools/builtins/builtin-tools.d.ts +23 -4
- package/dist/tools/builtins/builtin-tools.js +78 -12
- package/dist/tools/builtins/current-time.d.ts +26 -0
- package/dist/tools/builtins/current-time.js +99 -0
- package/dist/tools/builtins/handback.d.ts +28 -0
- package/dist/tools/builtins/handback.js +53 -0
- package/dist/tools/builtins/platform.d.ts +51 -0
- package/dist/tools/builtins/platform.js +451 -0
- package/dist/tools/builtins/read-tool-output.d.ts +1 -1
- package/dist/tools/builtins/read-tool-output.js +2 -2
- package/dist/tools/builtins/sandbox/bash.d.ts +5 -5
- package/dist/tools/builtins/sandbox/bash.js +291 -39
- package/dist/tools/builtins/sandbox/common.d.ts +135 -36
- package/dist/tools/builtins/sandbox/common.js +222 -62
- package/dist/tools/builtins/sandbox/deliver.d.ts +1 -1
- package/dist/tools/builtins/sandbox/deliver.js +19 -16
- package/dist/tools/builtins/sandbox/edit.d.ts +1 -1
- package/dist/tools/builtins/sandbox/edit.js +64 -50
- package/dist/tools/builtins/sandbox/file-state.d.ts +34 -0
- package/dist/tools/builtins/sandbox/file-state.js +34 -0
- package/dist/tools/builtins/sandbox/glob-match.d.ts +1 -1
- package/dist/tools/builtins/sandbox/glob-match.js +1 -1
- package/dist/tools/builtins/sandbox/glob.d.ts +1 -1
- package/dist/tools/builtins/sandbox/glob.js +17 -12
- package/dist/tools/builtins/sandbox/grep.d.ts +7 -4
- package/dist/tools/builtins/sandbox/grep.js +153 -34
- package/dist/tools/builtins/sandbox/index.d.ts +9 -5
- package/dist/tools/builtins/sandbox/index.js +13 -5
- package/dist/tools/builtins/sandbox/read.d.ts +2 -2
- package/dist/tools/builtins/sandbox/read.js +29 -19
- package/dist/tools/builtins/sandbox/task-output.d.ts +79 -0
- package/dist/tools/builtins/sandbox/task-output.js +334 -0
- package/dist/tools/builtins/sandbox/task-stop.d.ts +10 -0
- package/dist/tools/builtins/sandbox/task-stop.js +113 -0
- package/dist/tools/builtins/sandbox/write.d.ts +1 -1
- package/dist/tools/builtins/sandbox/write.js +51 -44
- package/dist/tools/builtins/send-message.d.ts +56 -0
- package/dist/tools/builtins/send-message.js +100 -0
- package/dist/tools/builtins/skill-load.d.ts +29 -18
- package/dist/tools/builtins/skill-load.js +125 -73
- package/dist/tools/builtins/tool-search.d.ts +13 -9
- package/dist/tools/builtins/tool-search.js +25 -14
- package/dist/tools/builtins/web/errors.d.ts +8 -0
- package/dist/tools/builtins/web/errors.js +53 -0
- package/dist/tools/builtins/web/extract.d.ts +62 -0
- package/dist/tools/builtins/web/extract.js +271 -0
- package/dist/tools/builtins/web/fetch.d.ts +55 -0
- package/dist/tools/builtins/web/fetch.js +175 -0
- package/dist/tools/builtins/web/index.d.ts +13 -0
- package/dist/tools/builtins/web/index.js +12 -0
- package/dist/tools/builtins/web/page-cache.d.ts +25 -0
- package/dist/tools/builtins/web/page-cache.js +77 -0
- package/dist/tools/builtins/web/search.d.ts +25 -0
- package/dist/tools/builtins/web/search.js +210 -0
- package/dist/tools/builtins/web/url.d.ts +17 -0
- package/dist/tools/builtins/web/url.js +43 -0
- package/dist/tools/connector-account-policies.d.ts +51 -0
- package/dist/tools/connector-account-policies.js +165 -0
- package/dist/tools/connectors.controller.d.ts +74 -9
- package/dist/tools/connectors.controller.js +133 -16
- package/dist/tools/connectors.schemas.d.ts +27 -0
- package/dist/tools/connectors.schemas.js +34 -0
- package/dist/tools/connectors.service.d.ts +101 -8
- package/dist/tools/connectors.service.js +266 -28
- package/dist/tools/credentials.d.ts +23 -0
- package/dist/tools/credentials.js +90 -1
- package/dist/tools/display-names.d.ts +9 -0
- package/dist/tools/display-names.js +24 -0
- package/dist/tools/drift.js +5 -5
- package/dist/tools/elicitation.js +1 -1
- package/dist/tools/extension-tools.controller.d.ts +14 -0
- package/dist/tools/extension-tools.controller.js +37 -0
- package/dist/tools/info.d.ts +61 -4
- package/dist/tools/info.js +119 -33
- package/dist/tools/legacy-data.d.ts +34 -0
- package/dist/tools/legacy-data.js +76 -0
- package/dist/tools/legacy-keys.d.ts +51 -0
- package/dist/tools/legacy-keys.js +114 -0
- package/dist/tools/naming.d.ts +1 -5
- package/dist/tools/naming.js +1 -7
- package/dist/tools/oauth/flow-paths.d.ts +1 -1
- package/dist/tools/oauth/flow-paths.js +1 -1
- package/dist/tools/oauth/flow.js +1 -1
- package/dist/tools/oauth/tokens.d.ts +54 -7
- package/dist/tools/oauth/tokens.js +87 -17
- package/dist/tools/org-tool-switches.d.ts +89 -0
- package/dist/tools/org-tool-switches.js +201 -0
- package/dist/tools/outputs.js +1 -1
- package/dist/tools/pipeline.d.ts +67 -47
- package/dist/tools/pipeline.js +319 -380
- package/dist/tools/policy.d.ts +30 -23
- package/dist/tools/policy.js +142 -64
- package/dist/tools/questions.controller.d.ts +1 -1
- package/dist/tools/questions.controller.js +1 -1
- package/dist/tools/questions.service.d.ts +23 -2
- package/dist/tools/questions.service.js +104 -2
- package/dist/tools/registry.d.ts +189 -68
- package/dist/tools/registry.js +318 -236
- package/dist/tools/resources.d.ts +1 -1
- package/dist/tools/resources.js +1 -1
- package/dist/tools/tools.module.js +31 -3
- package/dist/tools/validate.js +1 -1
- package/dist/ui/admin-ui.controller.d.ts +56 -0
- package/dist/ui/admin-ui.controller.js +192 -0
- package/dist/ui/arrangement.d.ts +65 -0
- package/dist/ui/arrangement.js +171 -0
- package/dist/ui/embed-token.d.ts +2 -2
- package/dist/ui/embed-token.js +30 -5
- package/dist/ui/registry.d.ts +42 -17
- package/dist/ui/registry.js +46 -14
- package/dist/ui/slots.controller.d.ts +11 -0
- package/dist/ui/slots.controller.js +25 -0
- package/dist/ui/ui-stream.d.ts +20 -3
- package/dist/ui/ui-stream.js +25 -7
- package/dist/ui/ui.schemas.d.ts +70 -0
- package/dist/ui/ui.schemas.js +65 -0
- package/dist/ui/views.controller.d.ts +35 -49
- package/dist/ui/views.controller.js +48 -70
- package/dist/ui/views.d.ts +112 -18
- package/dist/ui/views.js +254 -47
- package/package.json +6 -3
- package/src/migrations/0063_tool_keys_claude_code.sql +220 -0
- package/src/migrations/0064_account_last_org.sql +7 -0
- package/src/migrations/0065_conversation_archive.sql +8 -0
- package/src/migrations/0066_extension_display_name.sql +4 -0
- package/src/migrations/0067_agent_profiles.sql +20 -0
- package/src/migrations/0068_agent_avatar_uploads.sql +7 -0
- package/src/migrations/0069_attachments_agent_idx.sql +3 -0
- package/src/migrations/0070_background_tasks.sql +37 -0
- package/src/migrations/0071_background_tasks_reserve.sql +6 -0
- package/src/migrations/0072_background_tasks_stopping.sql +5 -0
- package/src/migrations/0073_org_extension_point_defaults.sql +11 -0
- package/src/migrations/0074_subagent_tree.sql +50 -0
- package/src/migrations/0075_plugin_scope.sql +61 -0
- package/src/migrations/0076_skills_created_by_nullable.sql +6 -0
- package/src/migrations/0077_scope_requests.sql +52 -0
- package/src/migrations/0078_scope_grant_switches.sql +18 -0
- package/src/migrations/0079_scope_live_install_switches.sql +19 -0
- package/src/migrations/0080_rename_enterprise_admin.sql +18 -0
- package/src/migrations/0081_platform_components.sql +129 -0
- package/src/migrations/0082_platform_migration.sql +33 -0
- package/src/migrations/0083_plugin_scope_sync_failures.sql +15 -0
- package/src/migrations/0084_org_owned_items.sql +85 -0
- package/src/migrations/0085_agent_templates_phase4.sql +45 -0
- package/src/migrations/0086_org_layering.sql +30 -0
- package/src/migrations/0087_approval_summary.sql +6 -0
- package/src/migrations/0088_environments.sql +80 -0
- package/src/migrations/0089_model_kinds_usage.sql +38 -0
- package/src/migrations/0090_initial_passwords.sql +27 -0
- package/src/migrations/0091_account_time_zone.sql +4 -0
- package/src/migrations/0092_tools_default_on.sql +27 -0
- package/src/migrations/0093_connected_environments.sql +37 -0
- package/src/migrations/0094_environment_paths_idx.sql +7 -0
- package/src/migrations/0095_environment_files_resets.sql +5 -0
- package/src/migrations/0096_environment_demoted_default.sql +18 -0
- package/src/migrations/0097_extension_installs_enabled_at.sql +16 -0
- package/src/migrations/0098_single_provider_running_first.sql +41 -0
- package/src/migrations/0099_channel_connection_guide_ack.sql +6 -0
- package/src/migrations/0100_catalog_files.sql +13 -0
- package/src/migrations/0101_marketplace_icons.sql +32 -0
- package/src/migrations/0102_themes.sql +64 -0
- package/src/migrations/0103_ui_theme_setting.sql +7 -0
- package/src/migrations/0104_connector_tool_account_policies.sql +16 -0
- package/src/migrations/0105_devices_rename.sql +136 -0
- package/src/migrations/0106_setup_guides_v2.sql +10 -0
- package/src/migrations/0107_extension_client_builds.sql +50 -0
- package/src/migrations/0108_question_skipped.sql +6 -0
- package/src/migrations/0109_audit_platform_idx.sql +5 -0
- package/src/migrations/0110_usage_time_idx.sql +8 -0
- package/src/migrations/0111_agent_skill_sharing.sql +22 -0
- package/src/migrations/0112_skill_share_requests.sql +4 -0
- package/dist/catalog/upgrade-targets.d.ts +0 -56
- package/dist/catalog/upgrade-targets.js +0 -90
- package/dist/conversations/sandbox-status.d.ts +0 -24
- package/dist/conversations/sandbox-status.js +0 -52
- package/dist/extensions/org-config.controller.d.ts +0 -24
- package/dist/extensions/org-config.controller.js +0 -62
- package/dist/tools/builtins/sandbox/names.d.ts +0 -7
- package/dist/tools/builtins/sandbox/names.js +0 -20
- package/dist/tools/builtins/skill-write.d.ts +0 -59
- package/dist/tools/builtins/skill-write.js +0 -244
|
@@ -8,9 +8,10 @@ var __param = (this && this.__param) || function (paramIndex, decorator) {
|
|
|
8
8
|
return function (target, key) { decorator(target, key, paramIndex); }
|
|
9
9
|
};
|
|
10
10
|
import { randomUUID } from "node:crypto";
|
|
11
|
-
import { isSandboxGoneError, } from "@capybara-hq/plugin-sdk";
|
|
11
|
+
import { isFileSystemGoneError, isSandboxGoneError, rootsOf, } from "@capybara-hq/plugin-sdk";
|
|
12
12
|
import { Inject, Injectable } from "@nestjs/common";
|
|
13
13
|
import { sql, } from "kysely";
|
|
14
|
+
import { BackgroundTaskStore } from "../background/store.js";
|
|
14
15
|
import { DEFAULT_MAX_COMMAND_SECONDS, DEFAULT_MAX_RUN_SECONDS, } from "../config/schema.js";
|
|
15
16
|
import { ApiError } from "../errors.js";
|
|
16
17
|
import { readSandboxLimitOverride } from "../identity/settings.js";
|
|
@@ -20,24 +21,40 @@ import { KERNEL_LOGGER, REDIS } from "../infra/tokens.js";
|
|
|
20
21
|
import { effectiveSandboxEgress, effectiveSandboxLimits, readOrgSettings, } from "../orgs/settings.service.js";
|
|
21
22
|
import { canonicalJson } from "../tools/canonical-json.js";
|
|
22
23
|
import { SandboxLeases, SandboxLockLostError, sandboxBusy, } from "./leases.js";
|
|
23
|
-
import { SandboxProviders } from "./providers.js";
|
|
24
|
+
import { LOCAL_PROVIDER, providerForDevice, SandboxProviders, } from "./providers.js";
|
|
24
25
|
// Moved to leases.ts (phase C); re-exported for existing importers
|
|
25
|
-
export {
|
|
26
|
+
export { SANDBOX_LOCK_PREFIX, SANDBOX_LOCK_TTL_MS, SandboxLockLostError, sandboxBusy, } from "./leases.js";
|
|
26
27
|
/** Margin (ms) of a holder record's TTL beyond the longest command: by then the in-sandbox limit has ended its commands. */
|
|
27
28
|
const HOLDER_TTL_MARGIN_MS = 10 * 60_000;
|
|
28
29
|
/** How long the kernel's own read-modify-write of one file waits for another holder's (ms, spec §10.4). */
|
|
29
30
|
const PATH_LEASE_WAIT_MS = 30_000;
|
|
30
|
-
/**
|
|
31
|
+
/** Time limit of one provider call in maintenance (suspend, destroy). */
|
|
31
32
|
const SWEEP_OP_TIMEOUT_MS = 120_000;
|
|
32
33
|
/** How long the suspension sweep holds the intent on a sandbox past `maxRunningMinutes` (ms). */
|
|
33
34
|
const SWEEP_INTENT_WAIT_MS = 1000;
|
|
34
|
-
/**
|
|
35
|
+
/** Rows one sweep handles at most, and the provider calls it runs at once. */
|
|
35
36
|
const SWEEP_BATCH = 200;
|
|
36
37
|
const SWEEP_CONCURRENCY = 4;
|
|
37
|
-
/**
|
|
38
|
-
const
|
|
39
|
-
/**
|
|
38
|
+
/** How long `destroyDevice` / `resetFileSystem` wait for the exclusive lease (running commands). */
|
|
39
|
+
const EXCLUSIVE_WAIT_MS = 10 * 60_000;
|
|
40
|
+
/** The time limit of the exec that kills lost background tasks after a resume (ms). */
|
|
41
|
+
const PENDING_KILL_TIMEOUT_MS = 30_000;
|
|
42
|
+
/** A background task's session id as `echo $!` printed it; anything else is never put in a command. */
|
|
43
|
+
const SID_RE = /^[1-9][0-9]{0,9}$/;
|
|
44
|
+
/** The signal of one provider call in maintenance: aborted on timeout or when the lease is lost. */
|
|
40
45
|
const opSignal = (lease) => AbortSignal.any([lease.signal, AbortSignal.timeout(SWEEP_OP_TIMEOUT_MS)]);
|
|
46
|
+
/** The provider-side name of a device's file system and instance. */
|
|
47
|
+
export const deviceResourceName = (deviceId) => `cap-device-${deviceId}`;
|
|
48
|
+
/** The device is missing, deleted, or not in the caller's org (404; ids cannot be probed). */
|
|
49
|
+
export const deviceNotFound = () => new ApiError("device.not_found", "This device does not exist or was deleted.");
|
|
50
|
+
/**
|
|
51
|
+
* The device's file system is gone at the provider (409); never replaced silently. On a
|
|
52
|
+
* connected machine (`local`, details `[{ source: "local" }]`) the directory is the machine
|
|
53
|
+
* owner's: `capybara-device logout`, then `login` into the device, recreates it (ruling R24).
|
|
54
|
+
*/
|
|
55
|
+
export const deviceFilesLost = (local = false) => local
|
|
56
|
+
? new ApiError("device.files_lost", "The local device's directory is gone from its machine; run capybara-device logout, then capybara-device login into this device, on that machine to recreate it.", [{ source: "local" }])
|
|
57
|
+
: new ApiError("device.files_lost", "The device's files are gone; reset its files from the devices page to start with an empty workspace.");
|
|
41
58
|
/**
|
|
42
59
|
* The row still records the instance `gone` (same handle and same `created_at`). Handles compare
|
|
43
60
|
* canonically: a jsonb round trip reorders a multi-key handle's keys (final review m9).
|
|
@@ -47,42 +64,31 @@ export const isInstance = (row, gone) => gone !== null &&
|
|
|
47
64
|
canonicalJson(row.handle) === canonicalJson(gone.handle) &&
|
|
48
65
|
row.created_at.getTime() === gone.createdAt;
|
|
49
66
|
/**
|
|
50
|
-
*
|
|
51
|
-
*
|
|
67
|
+
* `impl` with every `exec` labelled with the holder's tag (and `actor` when given); `before` runs
|
|
68
|
+
* first (it refreshes the holder's record, so a callback that runs several long commands in a row
|
|
69
|
+
* stays recorded).
|
|
52
70
|
*/
|
|
53
|
-
|
|
54
|
-
return impl.concurrentExec === true && typeof impl.killTagged === "function" ? max : 1;
|
|
55
|
-
}
|
|
56
|
-
/**
|
|
57
|
-
* `impl` with every `exec` labelled with the holder's tag; `before` runs first (it refreshes the
|
|
58
|
-
* holder's record, so a callback that runs several long commands in a row stays recorded).
|
|
59
|
-
*/
|
|
60
|
-
function tagged(impl, tag, before) {
|
|
71
|
+
function tagged(impl, tag, actor, before) {
|
|
61
72
|
const wrapped = Object.create(impl);
|
|
62
73
|
Object.defineProperty(wrapped, "exec", {
|
|
63
74
|
value: async (h, args) => {
|
|
64
75
|
await before();
|
|
65
|
-
return impl.exec(h, { ...args, tag });
|
|
76
|
+
return impl.exec(h, { ...args, tag, ...(actor !== undefined && { actor }) });
|
|
66
77
|
},
|
|
67
78
|
});
|
|
68
79
|
return wrapped;
|
|
69
80
|
}
|
|
70
81
|
/**
|
|
71
|
-
*
|
|
72
|
-
* - `runtime.maxRunSeconds`
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
*
|
|
82
|
+
* Sandbox config problems to tell operators at startup (English log lines):
|
|
83
|
+
* - `runtime.maxRunSeconds` below `sandbox.limits.maxCommandSeconds`: long commands are cut by the
|
|
84
|
+
* run time limit (Ruling M20);
|
|
85
|
+
* - `sandbox.limits.egress` is open but the current provider plugin has no `allowOpenEgress`: every
|
|
86
|
+
* instance creation is refused.
|
|
87
|
+
* With both time limits at their defaults (900 < 7200, a known trade-off) it is only info; warn
|
|
88
|
+
* when an operator changed one and they still disagree.
|
|
77
89
|
*/
|
|
78
|
-
export function sandboxConfigWarnings(cfg, providerPlugin
|
|
90
|
+
export function sandboxConfigWarnings(cfg, providerPlugin) {
|
|
79
91
|
const out = [];
|
|
80
|
-
if (impl && (impl.concurrentExec === true) !== (typeof impl.killTagged === "function")) {
|
|
81
|
-
out.push({
|
|
82
|
-
level: "warn",
|
|
83
|
-
message: `the sandbox plugin ${providerPlugin ?? ""} declares only one of concurrentExec and killTagged; concurrent commands need both, so its sandboxes run one command at a time`,
|
|
84
|
-
});
|
|
85
|
-
}
|
|
86
92
|
if (cfg.sandbox.limits.egress === "open" && providerPlugin !== undefined) {
|
|
87
93
|
const pc = cfg.extensionConfig[providerPlugin];
|
|
88
94
|
if (pc?.allowOpenEgress !== true) {
|
|
@@ -103,8 +109,13 @@ export function sandboxConfigWarnings(cfg, providerPlugin, impl) {
|
|
|
103
109
|
}
|
|
104
110
|
return out;
|
|
105
111
|
}
|
|
106
|
-
/**
|
|
112
|
+
/** No sandbox provider (503). */
|
|
107
113
|
export const sandboxUnavailable = () => new ApiError("sandbox.unavailable", "Sandbox is not available in this deployment; install a sandbox plugin");
|
|
114
|
+
/**
|
|
115
|
+
* The device's files are kept by a provider that is not the current one (503); its files are
|
|
116
|
+
* not migrated, so it stays unusable until that provider is back (spec §4.2 item 13).
|
|
117
|
+
*/
|
|
118
|
+
const otherProvider = (provider) => new ApiError("device.unavailable", `This device's files are kept by the sandbox provider ${provider}, which is not the current provider; it can be used again once ${provider} is the deployment's sandbox provider`);
|
|
108
119
|
const abortReason = (signal) => signal.reason ?? new DOMException("This operation was aborted", "AbortError");
|
|
109
120
|
function sleep(ms, signal) {
|
|
110
121
|
return new Promise((resolve, reject) => {
|
|
@@ -123,7 +134,7 @@ function sleep(ms, signal) {
|
|
|
123
134
|
signal.addEventListener("abort", onAbort, { once: true });
|
|
124
135
|
});
|
|
125
136
|
}
|
|
126
|
-
/**
|
|
137
|
+
/** Runs `fn` on `items`, `limit` at a time. */
|
|
127
138
|
async function inPool(items, limit, fn) {
|
|
128
139
|
let next = 0;
|
|
129
140
|
const worker = async () => {
|
|
@@ -135,43 +146,63 @@ async function inPool(items, limit, fn) {
|
|
|
135
146
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
|
136
147
|
}
|
|
137
148
|
/**
|
|
138
|
-
*
|
|
149
|
+
* The kernel's sandbox service over the `sandbox@2` provider, keyed by **device** (spec
|
|
150
|
+
* 2026-11-07-environments-design §4.2, §5.1): one file system kept until the device is
|
|
151
|
+
* deleted, plus one instance attached to it that comes and goes; all conversations bound to the
|
|
152
|
+
* device (and their sub-agents) share both.
|
|
139
153
|
*
|
|
140
|
-
* -
|
|
141
|
-
*
|
|
142
|
-
*
|
|
143
|
-
*
|
|
144
|
-
*
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
*
|
|
148
|
-
*
|
|
149
|
-
*
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
*
|
|
155
|
-
*
|
|
156
|
-
*
|
|
157
|
-
*
|
|
158
|
-
*
|
|
159
|
-
*
|
|
160
|
-
*
|
|
161
|
-
*
|
|
162
|
-
*
|
|
163
|
-
*
|
|
164
|
-
* `
|
|
165
|
-
*
|
|
166
|
-
*
|
|
167
|
-
*
|
|
168
|
-
*
|
|
154
|
+
* - **Lazy creation**: the first {@link withSandbox} creates the file system
|
|
155
|
+
* (`createFileSystem`, name `cap-device-<deviceId>`) and then the instance on it; state lives
|
|
156
|
+
* in `device_instances`. A file system gone at the provider is never replaced silently:
|
|
157
|
+
* `fs_status` becomes `lost` and uses fail with `device.files_lost` until
|
|
158
|
+
* {@link resetFileSystem}.
|
|
159
|
+
* - **Leases** (phase C, spec §10, {@link SandboxLeases}), keyed by device id: using an
|
|
160
|
+
* instance recorded `running` with the current provider takes a **shared** lease, so up to
|
|
161
|
+
* `sandbox.maxConcurrentCommands` holders (all sessions of the device) run at once. Every
|
|
162
|
+
* lifecycle change (create, resume, rebuild, suspend, destroy, reset) takes the **exclusive**
|
|
163
|
+
* lease, alone; a caller that needs one on the use path sets the intent so new shared leases wait
|
|
164
|
+
* for it, and downgrades to a shared lease to run its own command once the instance runs.
|
|
165
|
+
* Waiting is capped at `sandbox.lockWaitSeconds` (`sandbox.busy`); two replicas using it for the
|
|
166
|
+
* first time at once still build only one.
|
|
167
|
+
* - **Fencing**: exclusive holders read `generation` and compare-and-increment it with every state
|
|
168
|
+
* write ({@link SandboxLockLostError}); every acquisition (shared ones too) bumps it first, so a
|
|
169
|
+
* stale exclusive holder's later writes fail. Shared holders write nothing else but
|
|
170
|
+
* `last_used_at` (monotonic). A lost lease aborts the signal passed to `fn` and to maintenance
|
|
171
|
+
* provider calls.
|
|
172
|
+
* - **Suspension and destruction**: the sweeps (`SandboxSweepJobs`, crons) call
|
|
173
|
+
* {@link suspendIdle} / {@link destroyInactive}; they try the exclusive lease once and skip an
|
|
174
|
+
* device with any holder; one past `maxRunningMinutes` gets the intent for a short window
|
|
175
|
+
* (about a second) so the suspension can get in when its commands end. Inactivity destroys only
|
|
176
|
+
* the instance (files kept); a deleted device or org loses its files too.
|
|
177
|
+
* - **Transparent recovery**: a suspended instance is resumed on next use; when `resume` returns
|
|
178
|
+
* `gone`, the instance was destroyed, or `fn` throws `SandboxGoneError` (retried once), a new
|
|
179
|
+
* instance is created on the kept files and `restarted` is true. Several holders hitting `gone`
|
|
180
|
+
* together rebuild once: the others find the new instance.
|
|
181
|
+
* - **Takeover** (spec §10.5; environments spec §4.2): every holder records itself in
|
|
182
|
+
* `capk:sandbox-holders:<env>` (again before each `exec`) and its commands carry its lease token
|
|
183
|
+
* as `tag`. An acquirer that finds a record without a live lease (a crashed holder) kills only
|
|
184
|
+
* that holder's commands (`killTagged`): other conversations of the device may be running
|
|
185
|
+
* commands. `killAll` runs only under the exclusive lease, before the downgrade, on an instance
|
|
186
|
+
* this holder just resumed from `suspended` (nobody else can hold it; a provider that keeps
|
|
187
|
+
* memory across a pause brings old commands back); it also covers crashed holders' records,
|
|
188
|
+
* which it claims. A freshly created instance has nothing to kill. Failures are only warned (the
|
|
189
|
+
* in-sandbox time limit is the backstop).
|
|
169
190
|
* - **Instance identity**: a holder that hits `gone` names the instance by its handle and the
|
|
170
191
|
* row's `created_at` (set by the fenced write after each `create`); the handle alone is the same
|
|
171
192
|
* after a rebuild on providers whose handle is the instance name (sandbox-docker).
|
|
172
193
|
*
|
|
173
|
-
*
|
|
174
|
-
*
|
|
194
|
+
* - **Provider per device** (environments spec §6): each call resolves the provider from
|
|
195
|
+
* the device's `devices.provider` ({@link providerForDevice}): a `device`
|
|
196
|
+
* device runs on the kernel-internal `@local` provider (never the cloud one), a cloud
|
|
197
|
+
* device on the loaded `sandbox@2` extension. A `@local` instance is never suspended or
|
|
198
|
+
* destroyed for inactivity; a deleted `@local` device (or one of a deleted org) is
|
|
199
|
+
* destroyed by the sweep like any other.
|
|
200
|
+
*
|
|
201
|
+
* Files kept by another provider than the current one make the device unusable
|
|
202
|
+
* (`device.unavailable`) until that provider is back; deleting it still works while that
|
|
203
|
+
* provider is loaded ({@link SandboxProviders.forName}). Without the device's provider
|
|
204
|
+
* {@link availableFor} is false and uses throw `sandbox.unavailable` (503). No in-process state:
|
|
205
|
+
* replicas share it through PG and Redis.
|
|
175
206
|
*/
|
|
176
207
|
let SandboxService = class SandboxService {
|
|
177
208
|
db;
|
|
@@ -180,39 +211,91 @@ let SandboxService = class SandboxService {
|
|
|
180
211
|
providers;
|
|
181
212
|
config;
|
|
182
213
|
log;
|
|
214
|
+
background;
|
|
183
215
|
constructor(db,
|
|
184
216
|
/** The record of the instance each holder last used (`capk:sandbox-seen:…`). */
|
|
185
217
|
redis, leases,
|
|
186
|
-
/** The
|
|
187
|
-
providers, config, log
|
|
218
|
+
/** The providers (read at every call: providers are ready after the extension load). */
|
|
219
|
+
providers, config, log,
|
|
220
|
+
/** Background tasks lost while their process may live on (killed after a resume). */
|
|
221
|
+
background) {
|
|
188
222
|
this.db = db;
|
|
189
223
|
this.redis = redis;
|
|
190
224
|
this.leases = leases;
|
|
191
225
|
this.providers = providers;
|
|
192
226
|
this.config = config;
|
|
193
227
|
this.log = log;
|
|
228
|
+
this.background = background;
|
|
194
229
|
}
|
|
195
230
|
/** Poll interval while waiting for a lease (the leases' own). */
|
|
196
231
|
get pollMs() {
|
|
197
232
|
return this.leases.pollMs;
|
|
198
233
|
}
|
|
199
|
-
/**
|
|
200
|
-
|
|
201
|
-
|
|
234
|
+
/**
|
|
235
|
+
* Whether there is a cloud sandbox provider (a loaded `sandbox@2` extension). For gates that have
|
|
236
|
+
* no device yet (tool listing, skill files); a call on a device asks
|
|
237
|
+
* {@link availableFor}.
|
|
238
|
+
*/
|
|
239
|
+
get cloudAvailable() {
|
|
240
|
+
return this.providers.cloud() !== undefined;
|
|
202
241
|
}
|
|
203
|
-
/**
|
|
204
|
-
|
|
205
|
-
return
|
|
242
|
+
/** Whether the device's provider is there (`@local` registered, or a cloud provider). */
|
|
243
|
+
async availableFor(deviceId) {
|
|
244
|
+
return (await this.providerOf(deviceId)) !== undefined;
|
|
206
245
|
}
|
|
207
246
|
/**
|
|
208
|
-
*
|
|
209
|
-
*
|
|
247
|
+
* The provider the device works with, read from its row ({@link providerForDevice});
|
|
248
|
+
* a missing row (or one of another org, with `orgId`) reads as a cloud device, whose use
|
|
249
|
+
* then fails as not found.
|
|
210
250
|
*/
|
|
211
|
-
async
|
|
251
|
+
async providerOf(deviceId, orgId) {
|
|
212
252
|
const row = await this.db
|
|
213
|
-
.selectFrom("
|
|
253
|
+
.selectFrom("devices")
|
|
254
|
+
.select("provider")
|
|
255
|
+
.where("id", "=", deviceId)
|
|
256
|
+
.$if(orgId !== undefined, (q) => q.where("org_id", "=", orgId))
|
|
257
|
+
.executeTakeFirst();
|
|
258
|
+
return providerForDevice(this.providers, row?.provider ?? null);
|
|
259
|
+
}
|
|
260
|
+
/** {@link providerOf}, or `sandbox.unavailable` (503) without it. */
|
|
261
|
+
async require(deviceId, orgId) {
|
|
262
|
+
const p = await this.providerOf(deviceId, orgId);
|
|
263
|
+
if (!p)
|
|
264
|
+
throw sandboxUnavailable();
|
|
265
|
+
return p;
|
|
266
|
+
}
|
|
267
|
+
/** Concurrent holders of one device (`sandbox.maxConcurrentCommands`, read at every use). */
|
|
268
|
+
get cap() {
|
|
269
|
+
return this.config.get().sandbox.maxConcurrentCommands;
|
|
270
|
+
}
|
|
271
|
+
/**
|
|
272
|
+
* The org's effective sandbox limits (spec §8.4): the deployment's `sandbox.limits`, egress
|
|
273
|
+
* narrowed by the org setting `sandboxEgress` (Ruling M19), cpu / memory per the super admin's
|
|
274
|
+
* org override (C19 / Ruling M16). Instances are created with them; phase 1 devices use
|
|
275
|
+
* them unchanged.
|
|
276
|
+
*/
|
|
277
|
+
async orgLimits(orgId) {
|
|
278
|
+
const cfg = this.config.get().sandbox;
|
|
279
|
+
const egress = effectiveSandboxEgress(cfg.limits, (await readOrgSettings(this.db.db, orgId)).sandboxEgress);
|
|
280
|
+
const sized = effectiveSandboxLimits({ ...cfg.limits, ...egress }, await readSandboxLimitOverride(this.db.db, orgId));
|
|
281
|
+
return {
|
|
282
|
+
cpu: sized.cpu,
|
|
283
|
+
memoryMb: sized.memoryMb,
|
|
284
|
+
diskMb: sized.diskMb,
|
|
285
|
+
egress: sized.egress,
|
|
286
|
+
allowHosts: sized.allowHosts,
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* The device's instance state as shown to people (read only, no lease): `running` /
|
|
291
|
+
* `suspended` and the last use. No instance, a destroyed one, or one stuck in `creating` (being
|
|
292
|
+
* created, or a failed create the next use retries) is null.
|
|
293
|
+
*/
|
|
294
|
+
async status(orgId, deviceId) {
|
|
295
|
+
const row = await this.db
|
|
296
|
+
.selectFrom("device_instances")
|
|
214
297
|
.select(["status", "last_used_at"])
|
|
215
|
-
.where("
|
|
298
|
+
.where("device_id", "=", deviceId)
|
|
216
299
|
.where("org_id", "=", orgId)
|
|
217
300
|
.executeTakeFirst();
|
|
218
301
|
if (!row || (row.status !== "running" && row.status !== "suspended"))
|
|
@@ -220,90 +303,223 @@ let SandboxService = class SandboxService {
|
|
|
220
303
|
return { status: row.status, lastUsedAt: row.last_used_at };
|
|
221
304
|
}
|
|
222
305
|
/**
|
|
223
|
-
*
|
|
224
|
-
*
|
|
225
|
-
*
|
|
306
|
+
* The instance the device records now (its `created_at`), while it has one that runs or is
|
|
307
|
+
* suspended; null otherwise. Read without a lease: a caller inside `withSandbox` reads the
|
|
308
|
+
* instance its own lease keeps.
|
|
309
|
+
*/
|
|
310
|
+
async currentInstance(deviceId) {
|
|
311
|
+
const row = await this.db
|
|
312
|
+
.selectFrom("device_instances")
|
|
313
|
+
.select(["status", "handle", "created_at"])
|
|
314
|
+
.where("device_id", "=", deviceId)
|
|
315
|
+
.executeTakeFirst();
|
|
316
|
+
if (!row || row.handle === null)
|
|
317
|
+
return null;
|
|
318
|
+
if (row.status !== "running" && row.status !== "suspended")
|
|
319
|
+
return null;
|
|
320
|
+
return { createdAt: row.created_at };
|
|
321
|
+
}
|
|
322
|
+
/**
|
|
323
|
+
* Runs `fn` on the device's instance only while it runs the instance created at
|
|
324
|
+
* `instance.createdAt` (background tasks, spec 2026-11-05-background-bash-design §4.3). It never
|
|
325
|
+
* creates, resumes or rebuilds one: an instance that is missing, suspended, being created or
|
|
326
|
+
* destroyed (or recorded with another provider) is `not_running`, one that records another
|
|
327
|
+
* instance is `other_instance`. Otherwise `fn` runs under a shared lease, as a `withSandbox`
|
|
328
|
+
* holder does (its tag on every exec, crashed holders' commands killed), and `last_used_at`
|
|
329
|
+
* moves forward when it ends. Waits for the lease up to `sandbox.lockWaitSeconds`
|
|
330
|
+
* (`sandbox.busy`); `signal` (optional) ends the wait with its abort reason. The signal passed
|
|
331
|
+
* to `fn` aborts with `signal` or when the lease is lost. Errors of `fn` (`SandboxGoneError`
|
|
332
|
+
* too) are thrown as they are.
|
|
333
|
+
*/
|
|
334
|
+
async ifRunning(deviceId, instance, fn, signal = new AbortController().signal) {
|
|
335
|
+
const p = await this.require(deviceId);
|
|
336
|
+
const env = deviceId;
|
|
337
|
+
const deadline = Date.now() + this.config.get().sandbox.lockWaitSeconds * 1000;
|
|
338
|
+
const token = randomUUID();
|
|
339
|
+
for (;;) {
|
|
340
|
+
if (signal.aborted)
|
|
341
|
+
throw abortReason(signal);
|
|
342
|
+
const row = await this.db
|
|
343
|
+
.selectFrom("device_instances")
|
|
344
|
+
.selectAll()
|
|
345
|
+
.where("device_id", "=", env)
|
|
346
|
+
.executeTakeFirst();
|
|
347
|
+
if (!row)
|
|
348
|
+
return { kind: "not_running", status: "gone" };
|
|
349
|
+
if (row.created_at.getTime() !== instance.createdAt.getTime()) {
|
|
350
|
+
return { kind: "other_instance" };
|
|
351
|
+
}
|
|
352
|
+
if (!this.usable(row, p, null))
|
|
353
|
+
return { kind: "not_running", status: row.status };
|
|
354
|
+
// the instance this check saw: its handle plus `created_at` (never `generation`, which
|
|
355
|
+
// every lease bumps)
|
|
356
|
+
const want = { handle: row.handle, createdAt: row.created_at.getTime() };
|
|
357
|
+
const lease = await this.leases.tryShared(env, this.cap, token);
|
|
358
|
+
if (lease) {
|
|
359
|
+
const e = await this.asShared({ orgId: row.org_id, deviceId: env }, p, lease, (r) => this.usable(r, p, null) && isInstance(r, want), signal);
|
|
360
|
+
if (e) {
|
|
361
|
+
try {
|
|
362
|
+
const impl = tagged(p.impl, lease.token, undefined, () => this.register(env, lease.token));
|
|
363
|
+
const sig = AbortSignal.any([signal, lease.signal]);
|
|
364
|
+
try {
|
|
365
|
+
return { kind: "ran", value: await fn(impl, e.handle, sig) };
|
|
366
|
+
}
|
|
367
|
+
catch (err) {
|
|
368
|
+
if (isFileSystemGoneError(err))
|
|
369
|
+
throw await this.filesGone(env, p, e);
|
|
370
|
+
throw err;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
finally {
|
|
374
|
+
await this.leave(env, e);
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
// the row changed between the check and the lease: classify it again
|
|
378
|
+
continue;
|
|
379
|
+
}
|
|
380
|
+
if (Date.now() + this.pollMs > deadline)
|
|
381
|
+
throw sandboxBusy();
|
|
382
|
+
await sleep(this.pollMs + Math.floor(Math.random() * this.pollMs), signal);
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
/**
|
|
386
|
+
* A short lease on one path of the device around the kernel's own read-modify-write of
|
|
387
|
+
* that file (spec §10.4); waits up to 30 s, then throws `SandboxPathBusyError`. Its `signal`
|
|
388
|
+
* aborts when a renewal finds it lost.
|
|
226
389
|
*/
|
|
227
|
-
pathLease(
|
|
228
|
-
return this.leases.pathLease(
|
|
390
|
+
pathLease(deviceId, path, signal) {
|
|
391
|
+
return this.leases.pathLease(deviceId, path, signal, PATH_LEASE_WAIT_MS);
|
|
229
392
|
}
|
|
230
393
|
/**
|
|
231
|
-
*
|
|
232
|
-
*
|
|
233
|
-
* `last_used_at` → release.
|
|
234
|
-
* -
|
|
235
|
-
*
|
|
236
|
-
* - The
|
|
237
|
-
*
|
|
238
|
-
*
|
|
239
|
-
* - `
|
|
240
|
-
*
|
|
241
|
-
*
|
|
242
|
-
*
|
|
394
|
+
* Runs `fn` on the device's instance: take a lease → make sure the instance runs (file
|
|
395
|
+
* system, create / resume / rebuild, under the exclusive lease) → `fn` under a shared lease →
|
|
396
|
+
* update `last_used_at` → release.
|
|
397
|
+
* - Waiting longer than `sandbox.lockWaitSeconds` throws `sandbox.busy` (409); `signal` aborts
|
|
398
|
+
* the wait and the creation / resume.
|
|
399
|
+
* - The fourth argument of `fn` combines `signal` with the lease's: use it for commands, so they
|
|
400
|
+
* stop when the lease is lost.
|
|
401
|
+
* - The provider passed to `fn` labels every `exec` with this holder's tag and `ctx.actor`.
|
|
402
|
+
* - When `fn` throws `SandboxGoneError` (the instance is gone) the instance is rebuilt on the
|
|
403
|
+
* kept files (or the one another holder rebuilt is taken) and `fn` runs once more with
|
|
404
|
+
* `restarted: true`; a second `gone` is thrown. Other errors are thrown as they are.
|
|
405
|
+
* - Lost files throw `device.files_lost` (409), also when `fn` throws
|
|
406
|
+
* `FileSystemGoneError` (the files are then recorded lost, ruling R24).
|
|
407
|
+
* - When `ctx.holderConversationId` last used another instance (another holder rebuilt it
|
|
408
|
+
* meanwhile), `info.restarted` is true, so it learns its processes are gone (final review m5).
|
|
243
409
|
*/
|
|
244
410
|
async withSandbox(ctx, fn, signal) {
|
|
245
|
-
const p = this.require();
|
|
246
|
-
const
|
|
247
|
-
|
|
411
|
+
const p = await this.require(ctx.deviceId, ctx.orgId);
|
|
412
|
+
const env = ctx.deviceId;
|
|
413
|
+
const holder = ctx.holderConversationId;
|
|
414
|
+
let e = await this.seenBy(env, holder, await this.enter(ctx, p, signal, null));
|
|
248
415
|
try {
|
|
249
416
|
try {
|
|
250
|
-
return await this.runAs(e, p, fn, signal);
|
|
417
|
+
return await this.runAs(e, ctx, p, fn, signal);
|
|
251
418
|
}
|
|
252
419
|
catch (err) {
|
|
420
|
+
if (isFileSystemGoneError(err))
|
|
421
|
+
throw await this.filesGone(env, p, e);
|
|
253
422
|
if (!isSandboxGoneError(err))
|
|
254
423
|
throw err;
|
|
255
|
-
this.log.info("sandbox instance is gone during use; recreating", {
|
|
256
|
-
conversationId: ctx.conversationId,
|
|
257
|
-
});
|
|
424
|
+
this.log.info("sandbox instance is gone during use; recreating", { deviceId: env });
|
|
258
425
|
const gone = { handle: e.handle, createdAt: e.createdAt };
|
|
259
426
|
const prev = e;
|
|
260
|
-
await this.leave(
|
|
261
|
-
e = await this.seenBy(
|
|
262
|
-
|
|
427
|
+
await this.leave(env, prev);
|
|
428
|
+
e = await this.seenBy(env, holder, await this.enter(ctx, p, signal, gone));
|
|
429
|
+
// a reset seen on the first try still has to be told (it covers the restart)
|
|
430
|
+
const info = prev.info.filesReset
|
|
431
|
+
? { fresh: true, restarted: false, filesReset: true }
|
|
432
|
+
: { fresh: true, restarted: true };
|
|
433
|
+
try {
|
|
434
|
+
return await this.runAs({ ...e, info }, ctx, p, fn, signal);
|
|
435
|
+
}
|
|
436
|
+
catch (err2) {
|
|
437
|
+
if (isFileSystemGoneError(err2))
|
|
438
|
+
throw await this.filesGone(env, p, e);
|
|
439
|
+
throw err2;
|
|
440
|
+
}
|
|
263
441
|
}
|
|
264
442
|
}
|
|
265
443
|
finally {
|
|
266
|
-
await this.leave(
|
|
444
|
+
await this.leave(env, e);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
/**
|
|
448
|
+
* Ruling R24: an operation on the held instance `e` found its file system gone. Records
|
|
449
|
+
* `fs_status = 'lost'` while the row still records that instance on its file system (the
|
|
450
|
+
* instance handle stays, so a reset still destroys it), moving the fence on, and returns
|
|
451
|
+
* `device.files_lost` to throw. Uses then fail until the files are reset (a connected
|
|
452
|
+
* machine: until a new login, which resets the row). A failed write only logs.
|
|
453
|
+
*/
|
|
454
|
+
async filesGone(env, p, e) {
|
|
455
|
+
this.log.warn("device file system is gone at the provider; its files are lost", {
|
|
456
|
+
deviceId: env,
|
|
457
|
+
provider: p.plugin,
|
|
458
|
+
});
|
|
459
|
+
try {
|
|
460
|
+
await this.db
|
|
461
|
+
.updateTable("device_instances")
|
|
462
|
+
.set((eb) => ({ fs_status: "lost", generation: eb("generation", "+", 1) }))
|
|
463
|
+
.where("device_id", "=", env)
|
|
464
|
+
.where("fs_status", "=", "ready")
|
|
465
|
+
// `createdAt` is the row's value read back in milliseconds
|
|
466
|
+
.where(sql `date_trunc('milliseconds', created_at)`, "=", new Date(e.createdAt))
|
|
467
|
+
.execute();
|
|
468
|
+
}
|
|
469
|
+
catch (err) {
|
|
470
|
+
this.log.warn("cannot record lost device files", { deviceId: env, err });
|
|
267
471
|
}
|
|
472
|
+
return deviceFilesLost(p.plugin === LOCAL_PROVIDER);
|
|
268
473
|
}
|
|
269
474
|
/**
|
|
270
|
-
* Records that `holder` uses the instance of `e` (`capk:sandbox-seen:<
|
|
271
|
-
*
|
|
272
|
-
*
|
|
475
|
+
* Records that `holder` uses the instance of `e` (`capk:sandbox-seen:<env>:<holder>` =
|
|
476
|
+
* `<created_at>:<fs_resets>`, with a TTL of the inactivity destruction plus a day). When the
|
|
477
|
+
* holder last used another instance under the same reset count `e` is marked restarted (files
|
|
478
|
+
* kept); under another count it is marked `filesReset` instead. The count, not the provider's
|
|
479
|
+
* file system id, tells them apart: a provider may give the new file system the old id
|
|
480
|
+
* (sandbox-docker's volumes are named after the device). Best effort: a Redis error only
|
|
481
|
+
* logs.
|
|
273
482
|
*/
|
|
274
|
-
async seenBy(
|
|
483
|
+
async seenBy(deviceId, holder, e) {
|
|
275
484
|
try {
|
|
276
485
|
const ttl = (this.config.get().sandbox.limits.destroyAfterInactiveDays + 1) * 86_400;
|
|
277
|
-
const
|
|
278
|
-
|
|
279
|
-
|
|
486
|
+
const seen = `${e.createdAt}:${e.fsResets}`;
|
|
487
|
+
const prev = await this.redis.set(`capk:sandbox-seen:${deviceId}:${holder}`, seen, "EX", ttl, "GET");
|
|
488
|
+
if (prev !== null && prev !== seen) {
|
|
489
|
+
// `<createdAt>:<fsResets>`: another reset count means the files were reset since; another
|
|
490
|
+
// instance under the same count is a restart on kept files
|
|
491
|
+
const sameFiles = prev.slice(prev.indexOf(":") + 1) === String(e.fsResets);
|
|
492
|
+
if (!sameFiles) {
|
|
493
|
+
return { ...e, info: { fresh: e.info.fresh, restarted: false, filesReset: true } };
|
|
494
|
+
}
|
|
495
|
+
if (!e.info.restarted)
|
|
496
|
+
return { ...e, info: { fresh: e.info.fresh, restarted: true } };
|
|
280
497
|
}
|
|
281
498
|
}
|
|
282
499
|
catch (err) {
|
|
283
500
|
this.log.warn("cannot record the sandbox instance a holder uses", {
|
|
284
|
-
|
|
501
|
+
deviceId,
|
|
285
502
|
holder,
|
|
286
503
|
err,
|
|
287
504
|
});
|
|
288
505
|
}
|
|
289
506
|
return e;
|
|
290
507
|
}
|
|
291
|
-
/** Runs `fn` as the holder `e`: its tag on every exec, its lease's signal. */
|
|
292
|
-
runAs(e, p, fn, signal) {
|
|
508
|
+
/** Runs `fn` as the holder `e`: its tag and the caller's actor on every exec, its lease's signal. */
|
|
509
|
+
runAs(e, ctx, p, fn, signal) {
|
|
293
510
|
const sig = AbortSignal.any([signal, e.lease.signal]);
|
|
294
|
-
const
|
|
295
|
-
const impl = tagged(p.impl, e.lease.token, () => this.register(
|
|
296
|
-
return fn(e.handle, impl, e.info, sig);
|
|
511
|
+
const env = ctx.deviceId;
|
|
512
|
+
const impl = tagged(p.impl, e.lease.token, ctx.actor, () => this.register(env, e.lease.token));
|
|
513
|
+
return fn(e.handle, impl, { ...e.info, roots: rootsOf(e.handle) }, sig);
|
|
297
514
|
}
|
|
298
515
|
/**
|
|
299
516
|
* Becomes a holder: loops until it has a shared lease on a running instance, or `sandbox.busy`.
|
|
300
|
-
*
|
|
301
|
-
*
|
|
302
|
-
*
|
|
517
|
+
* An instance recorded `running` (with the current provider, not the `gone` instance) is used
|
|
518
|
+
* under a shared lease; otherwise the exclusive lease (with intent) makes it run and is
|
|
519
|
+
* downgraded. One token serves both, so the waiter passes its own intent.
|
|
303
520
|
*/
|
|
304
521
|
async enter(ctx, p, signal, gone) {
|
|
305
|
-
const
|
|
306
|
-
const cap = this.effectiveCap(p);
|
|
522
|
+
const env = ctx.deviceId;
|
|
307
523
|
const deadline = Date.now() + this.config.get().sandbox.lockWaitSeconds * 1000;
|
|
308
524
|
const token = randomUUID();
|
|
309
525
|
let intent = false;
|
|
@@ -312,19 +528,19 @@ let SandboxService = class SandboxService {
|
|
|
312
528
|
if (signal.aborted)
|
|
313
529
|
throw abortReason(signal);
|
|
314
530
|
const row = await this.readRow(ctx);
|
|
315
|
-
if (this.usable(row, p, gone)
|
|
316
|
-
const lease = await this.leases.tryShared(
|
|
531
|
+
if (this.usable(row, p, gone)) {
|
|
532
|
+
const lease = await this.leases.tryShared(env, this.cap, token);
|
|
317
533
|
if (lease) {
|
|
318
534
|
if (intent)
|
|
319
|
-
await this.clearIntent(
|
|
535
|
+
await this.clearIntent(env, token);
|
|
320
536
|
intent = false;
|
|
321
|
-
const e = await this.asShared(ctx, p, lease, gone, signal);
|
|
537
|
+
const e = await this.asShared(ctx, p, lease, (r) => this.usable(r, p, gone), signal);
|
|
322
538
|
if (e)
|
|
323
539
|
return e;
|
|
324
540
|
}
|
|
325
541
|
}
|
|
326
542
|
else {
|
|
327
|
-
const lease = await this.leases.tryExclusive(
|
|
543
|
+
const lease = await this.leases.tryExclusive(env, { token, intent: true });
|
|
328
544
|
intent = lease === null;
|
|
329
545
|
if (lease)
|
|
330
546
|
return await this.asExclusive(ctx, p, lease, gone, signal);
|
|
@@ -336,43 +552,50 @@ let SandboxService = class SandboxService {
|
|
|
336
552
|
}
|
|
337
553
|
catch (err) {
|
|
338
554
|
if (intent)
|
|
339
|
-
await this.clearIntent(
|
|
555
|
+
await this.clearIntent(env, token);
|
|
340
556
|
throw err;
|
|
341
557
|
}
|
|
342
558
|
}
|
|
343
|
-
async clearIntent(
|
|
559
|
+
async clearIntent(env, token) {
|
|
344
560
|
await this.leases
|
|
345
|
-
.clearIntent(
|
|
346
|
-
.catch((err) => this.log.warn("cannot clear sandbox intent", {
|
|
561
|
+
.clearIntent(env, token)
|
|
562
|
+
.catch((err) => this.log.warn("cannot clear sandbox intent", { deviceId: env, err }));
|
|
347
563
|
}
|
|
348
564
|
async readRow(ctx) {
|
|
349
565
|
const row = await this.db
|
|
350
|
-
.selectFrom("
|
|
566
|
+
.selectFrom("device_instances")
|
|
351
567
|
.selectAll()
|
|
352
|
-
.where("
|
|
568
|
+
.where("device_id", "=", ctx.deviceId)
|
|
353
569
|
.executeTakeFirst();
|
|
354
|
-
if (row && row.org_id !== ctx.orgId)
|
|
355
|
-
throw
|
|
356
|
-
}
|
|
570
|
+
if (row && row.org_id !== ctx.orgId)
|
|
571
|
+
throw this.foreign(ctx);
|
|
357
572
|
return row;
|
|
358
573
|
}
|
|
359
|
-
/**
|
|
574
|
+
/** The instance row belongs to another org: reported as not found, so ids cannot be probed. */
|
|
575
|
+
foreign(_ctx) {
|
|
576
|
+
return deviceNotFound();
|
|
577
|
+
}
|
|
578
|
+
/**
|
|
579
|
+
* Recorded running with the current provider on files not lost (lost ones go through the
|
|
580
|
+
* exclusive path, which refuses them), and not the instance found gone.
|
|
581
|
+
*/
|
|
360
582
|
usable(row, p, gone) {
|
|
361
583
|
return (row !== undefined &&
|
|
362
584
|
row.status === "running" &&
|
|
585
|
+
row.fs_status !== "lost" &&
|
|
363
586
|
row.provider === p.plugin &&
|
|
364
587
|
row.handle !== null &&
|
|
365
588
|
!isInstance(row, gone));
|
|
366
589
|
}
|
|
367
590
|
/**
|
|
368
|
-
* Under a just-taken shared lease: confirms the row is still
|
|
369
|
-
* null), records the holder and kills crashed holders' commands.
|
|
591
|
+
* Under a just-taken shared lease: confirms the row is still `accept`ed (else releases and
|
|
592
|
+
* returns null), records the holder and kills crashed holders' commands.
|
|
370
593
|
*/
|
|
371
|
-
async asShared(ctx, p, lease,
|
|
372
|
-
const
|
|
594
|
+
async asShared(ctx, p, lease, accept, signal) {
|
|
595
|
+
const env = ctx.deviceId;
|
|
373
596
|
try {
|
|
374
597
|
const row = await this.readRow(ctx);
|
|
375
|
-
if (!
|
|
598
|
+
if (!accept(row)) {
|
|
376
599
|
await lease.release();
|
|
377
600
|
return null;
|
|
378
601
|
}
|
|
@@ -380,123 +603,128 @@ let SandboxService = class SandboxService {
|
|
|
380
603
|
// expired unnoticed (a stalled sweep) then fails its next write instead of suspending or
|
|
381
604
|
// destroying the instance under this holder. Shared holders never compare it.
|
|
382
605
|
await this.db
|
|
383
|
-
.updateTable("
|
|
606
|
+
.updateTable("device_instances")
|
|
384
607
|
.set((eb) => ({ generation: eb("generation", "+", 1) }))
|
|
385
|
-
.where("
|
|
608
|
+
.where("device_id", "=", env)
|
|
386
609
|
.execute();
|
|
387
|
-
await this.register(
|
|
388
|
-
//
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
const sig = AbortSignal.any([signal, lease.signal]);
|
|
392
|
-
const impl = p.impl;
|
|
393
|
-
if (impl.killTagged) {
|
|
394
|
-
// by tag whenever the provider can, even with cap 1: while a hot change of
|
|
395
|
-
// maxConcurrentCommands spreads, a replica on a larger cap may have let a sibling in
|
|
396
|
-
await this.kill(conv, "takeover", async () => impl.killTagged?.(row.handle, dead.tags, sig), sig);
|
|
397
|
-
}
|
|
398
|
-
else if ((await this.leases.sharedCount(conv)) <= 1) {
|
|
399
|
-
// cap 1 and this holder is alone: killing everything cannot hit a sibling
|
|
400
|
-
await this.killLeftovers(conv, row.handle, p, sig, "takeover");
|
|
401
|
-
}
|
|
402
|
-
else {
|
|
403
|
-
this.log.warn("cannot kill leftover sandbox commands: the provider has no killTagged and another holder is live; the in-sandbox time limit ends them", { conversationId: conv });
|
|
404
|
-
}
|
|
405
|
-
}
|
|
610
|
+
await this.register(env, lease.token);
|
|
611
|
+
// other holders (any conversation of the device) may be running: only the dead
|
|
612
|
+
// holders' own commands are killed
|
|
613
|
+
await this.killTagged(env, row.handle, p, await this.claim(env), signal, lease);
|
|
406
614
|
return {
|
|
407
615
|
lease,
|
|
408
616
|
handle: row.handle,
|
|
409
617
|
createdAt: row.created_at.getTime(),
|
|
410
|
-
|
|
618
|
+
fsResets: row.fs_resets,
|
|
619
|
+
info: { fresh: false, restarted: false },
|
|
411
620
|
};
|
|
412
621
|
}
|
|
413
622
|
catch (err) {
|
|
414
|
-
await this.unregister(
|
|
623
|
+
await this.unregister(env, lease.token);
|
|
415
624
|
await lease.release();
|
|
416
625
|
throw err;
|
|
417
626
|
}
|
|
418
627
|
}
|
|
419
628
|
/**
|
|
420
|
-
* Under a just-taken exclusive lease: makes the
|
|
421
|
-
* recorded instance is the one found gone), kills leftovers, then downgrades to
|
|
629
|
+
* Under a just-taken exclusive lease: makes the instance run (file system, create / resume /
|
|
630
|
+
* rebuild when the recorded instance is the one found gone), kills leftovers, then downgrades to
|
|
631
|
+
* a shared lease. Crashed holders' records are claimed here: on an instance just resumed from
|
|
632
|
+
* `suspended` one `killAll` covers them and anything else left (nobody else holds the instance),
|
|
633
|
+
* on a new instance there is nothing to kill, and on one that already ran their tags are killed.
|
|
422
634
|
*/
|
|
423
635
|
async asExclusive(ctx, p, lease, gone, signal) {
|
|
424
|
-
const
|
|
636
|
+
const env = ctx.deviceId;
|
|
425
637
|
const sig = AbortSignal.any([signal, lease.signal]);
|
|
426
638
|
try {
|
|
427
639
|
const s = await this.ensure(ctx, p, lease, sig, gone);
|
|
428
|
-
const dead = await this.claim(
|
|
429
|
-
if (
|
|
430
|
-
await this.killLeftovers(
|
|
640
|
+
const dead = await this.claim(env);
|
|
641
|
+
if (s.resumed)
|
|
642
|
+
await this.killLeftovers(env, s.handle, p, sig, dead);
|
|
643
|
+
else if (!s.info.fresh)
|
|
644
|
+
await this.killTagged(env, s.handle, p, dead, signal, lease);
|
|
645
|
+
if (s.resumed || s.info.fresh) {
|
|
646
|
+
await this.killPendingTasks(env, s.handle, s.createdAt, p, lease.token, ctx.actor, sig);
|
|
431
647
|
}
|
|
432
648
|
const shared = await lease.downgrade();
|
|
433
|
-
await this.register(
|
|
434
|
-
return {
|
|
649
|
+
await this.register(env, shared.token);
|
|
650
|
+
return {
|
|
651
|
+
lease: shared,
|
|
652
|
+
handle: s.handle,
|
|
653
|
+
createdAt: s.createdAt,
|
|
654
|
+
fsResets: s.fsResets,
|
|
655
|
+
info: s.info,
|
|
656
|
+
};
|
|
435
657
|
}
|
|
436
658
|
catch (err) {
|
|
437
|
-
await this.unregister(
|
|
659
|
+
await this.unregister(env, lease.token);
|
|
438
660
|
// releases the lease in whatever kind it has now (also after a downgrade)
|
|
439
661
|
await lease.release();
|
|
440
662
|
throw err;
|
|
441
663
|
}
|
|
442
664
|
}
|
|
443
665
|
/** Ends a holder: unregisters it before releasing its lease, then moves `last_used_at` forward. */
|
|
444
|
-
async leave(
|
|
445
|
-
await this.unregister(
|
|
666
|
+
async leave(env, e) {
|
|
667
|
+
await this.unregister(env, e.lease.token);
|
|
446
668
|
await e.lease.release();
|
|
447
669
|
await this.db
|
|
448
|
-
.updateTable("
|
|
670
|
+
.updateTable("device_instances")
|
|
449
671
|
.set({ last_used_at: sql `GREATEST(last_used_at, now())` })
|
|
450
|
-
.where("
|
|
672
|
+
.where("device_id", "=", env)
|
|
451
673
|
.where("status", "=", "running")
|
|
452
674
|
.execute()
|
|
453
|
-
.catch((err) => this.log.warn("cannot update sandbox last_used_at", {
|
|
675
|
+
.catch((err) => this.log.warn("cannot update sandbox last_used_at", { deviceId: env, err }));
|
|
454
676
|
}
|
|
455
|
-
async register(
|
|
677
|
+
async register(env, token) {
|
|
456
678
|
const ttl = this.config.get().sandbox.limits.maxCommandSeconds * 1000 + HOLDER_TTL_MARGIN_MS;
|
|
457
679
|
await this.leases
|
|
458
|
-
.registerHolder(
|
|
459
|
-
.catch((err) => this.log.warn("cannot record sandbox holder", {
|
|
680
|
+
.registerHolder(env, token, ttl)
|
|
681
|
+
.catch((err) => this.log.warn("cannot record sandbox holder", { deviceId: env, err }));
|
|
460
682
|
}
|
|
461
|
-
async unregister(
|
|
683
|
+
async unregister(env, token) {
|
|
462
684
|
await this.leases
|
|
463
|
-
.unregisterHolder(
|
|
464
|
-
.catch((err) => this.log.warn("cannot clear sandbox holder record", {
|
|
685
|
+
.unregisterHolder(env, token)
|
|
686
|
+
.catch((err) => this.log.warn("cannot clear sandbox holder record", { deviceId: env, err }));
|
|
465
687
|
}
|
|
466
|
-
/**
|
|
467
|
-
|
|
468
|
-
* the caller's exclusive lease; a Redis error counts as none.
|
|
469
|
-
*/
|
|
470
|
-
async claim(conv, exclusiveToken) {
|
|
688
|
+
/** Crashed holders' tags (claimed); a Redis error counts as none. */
|
|
689
|
+
async claim(env) {
|
|
471
690
|
try {
|
|
472
|
-
return await this.leases.claimDeadHolders(
|
|
691
|
+
return await this.leases.claimDeadHolders(env);
|
|
473
692
|
}
|
|
474
693
|
catch (err) {
|
|
475
|
-
this.log.warn("cannot read sandbox holder records", {
|
|
476
|
-
return
|
|
694
|
+
this.log.warn("cannot read sandbox holder records", { deviceId: env, err });
|
|
695
|
+
return [];
|
|
477
696
|
}
|
|
478
697
|
}
|
|
698
|
+
/** Kills the commands of crashed holders by their tags (`killTagged`); nothing without tags. */
|
|
699
|
+
async killTagged(env, handle, p, tags, signal, lease) {
|
|
700
|
+
if (tags.length === 0)
|
|
701
|
+
return;
|
|
702
|
+
const sig = AbortSignal.any([signal, lease.signal]);
|
|
703
|
+
await this.kill(env, "takeover", () => p.impl.killTagged(handle, tags, sig), sig);
|
|
704
|
+
}
|
|
479
705
|
/** Runs one kill call: gone and abort are thrown, other failures only warned. */
|
|
480
|
-
async kill(
|
|
706
|
+
async kill(deviceId, why, call, signal) {
|
|
481
707
|
try {
|
|
482
708
|
await call();
|
|
483
709
|
if (why === "takeover") {
|
|
484
|
-
this.log.info("sandbox taken over from a holder that did not finish; leftover commands killed", {
|
|
710
|
+
this.log.info("sandbox taken over from a holder that did not finish; leftover commands killed", { deviceId });
|
|
485
711
|
}
|
|
486
712
|
}
|
|
487
713
|
catch (err) {
|
|
488
714
|
if (signal?.aborted || isSandboxGoneError(err))
|
|
489
715
|
throw err;
|
|
490
|
-
this.log.warn("cannot kill leftover sandbox commands", {
|
|
716
|
+
this.log.warn("cannot kill leftover sandbox commands", { deviceId, err });
|
|
491
717
|
}
|
|
492
718
|
}
|
|
493
719
|
/**
|
|
494
|
-
*
|
|
495
|
-
* Skips a
|
|
496
|
-
* window (about a second) first.
|
|
720
|
+
* Suspends running instances idle longer than `suspendAfterIdleMinutes` or running continuously
|
|
721
|
+
* longer than `maxRunningMinutes`. Skips a device with any holder; one past
|
|
722
|
+
* `maxRunningMinutes` gets the intent for one poll window (about a second) first. Returns how
|
|
723
|
+
* many it suspended. Called every minute by the `sandbox-suspend-idle` cron.
|
|
497
724
|
*/
|
|
498
725
|
async suspendIdle(now) {
|
|
499
|
-
|
|
726
|
+
// cloud only: suspendDue keeps to this provider's rows, so `@local` is never suspended here
|
|
727
|
+
const p = this.providers.cloud();
|
|
500
728
|
if (!p)
|
|
501
729
|
return 0;
|
|
502
730
|
const due = this.suspendDue(now, p.plugin);
|
|
@@ -505,12 +733,12 @@ let SandboxService = class SandboxService {
|
|
|
505
733
|
let n = 0;
|
|
506
734
|
await inPool(rows, SWEEP_CONCURRENCY, async (c) => {
|
|
507
735
|
const overdue = c.running_since !== null && c.running_since < runningBefore;
|
|
508
|
-
const done = await this.tryLocked(c.
|
|
509
|
-
const id = c.
|
|
736
|
+
const done = await this.tryLocked(c.device_id, async (lease) => {
|
|
737
|
+
const id = c.device_id;
|
|
510
738
|
const row = await this.db
|
|
511
|
-
.selectFrom("
|
|
739
|
+
.selectFrom("device_instances")
|
|
512
740
|
.selectAll()
|
|
513
|
-
.where("
|
|
741
|
+
.where("device_id", "=", id)
|
|
514
742
|
.where(due)
|
|
515
743
|
.executeTakeFirst();
|
|
516
744
|
if (!row?.handle)
|
|
@@ -525,7 +753,7 @@ let SandboxService = class SandboxService {
|
|
|
525
753
|
running_since: null,
|
|
526
754
|
sweep_failed_at: null,
|
|
527
755
|
});
|
|
528
|
-
this.log.info("sandbox suspended", {
|
|
756
|
+
this.log.info("sandbox suspended", { deviceId: id });
|
|
529
757
|
return true;
|
|
530
758
|
}, overdue);
|
|
531
759
|
if (done)
|
|
@@ -534,106 +762,217 @@ let SandboxService = class SandboxService {
|
|
|
534
762
|
return n;
|
|
535
763
|
}
|
|
536
764
|
/**
|
|
537
|
-
*
|
|
538
|
-
*
|
|
765
|
+
* Destruction sweep: an instance unused for longer than `destroyAfterInactiveDays` is destroyed
|
|
766
|
+
* (the instance only; the file system is kept and the next use builds a new instance on it); an
|
|
767
|
+
* device that was deleted, or whose org was deleted, loses its instance and its file system
|
|
768
|
+
* and its row is deleted (as {@link destroyDevice} does). Devices with any holder are
|
|
769
|
+
* skipped. Inactivity never destroys a `@local` instance; a gone `@local` device is
|
|
770
|
+
* destroyed like any other while `@local` is registered. Returns how many it destroyed. Called
|
|
771
|
+
* hourly by the `sandbox-destroy-inactive` cron.
|
|
539
772
|
*/
|
|
540
773
|
async destroyInactive(now) {
|
|
541
|
-
const p = this.providers.
|
|
542
|
-
|
|
543
|
-
return 0;
|
|
544
|
-
const due = this.destroyDue(now);
|
|
545
|
-
const rows = await this.candidates(due);
|
|
774
|
+
const p = this.providers.cloud();
|
|
775
|
+
const local = this.providers.forName(LOCAL_PROVIDER) !== undefined;
|
|
546
776
|
let n = 0;
|
|
547
|
-
|
|
777
|
+
// Gone devices (deleted, or of a deleted org) on every provider that is here, `@local`
|
|
778
|
+
// too (its destroy calls never need the machine online); rows whose provider is missing wait.
|
|
779
|
+
const here = (eb) => eb.or([
|
|
780
|
+
...(p ? [this.cloudRow(eb)] : []),
|
|
781
|
+
...(local ? [eb("provider", "=", LOCAL_PROVIDER)] : []),
|
|
782
|
+
]);
|
|
783
|
+
const gone = (eb) => eb.and([here(eb), this.deviceGone(eb)]);
|
|
784
|
+
if (p || local) {
|
|
785
|
+
await inPool(await this.candidates(gone), SWEEP_CONCURRENCY, async ({ device_id: id }) => {
|
|
786
|
+
const done = await this.tryLocked(id, async (lease) => {
|
|
787
|
+
const row = await this.db
|
|
788
|
+
.selectFrom("device_instances")
|
|
789
|
+
.selectAll()
|
|
790
|
+
.where("device_id", "=", id)
|
|
791
|
+
.where(gone)
|
|
792
|
+
.executeTakeFirst();
|
|
793
|
+
const owner = row && this.rowProvider(row);
|
|
794
|
+
if (!row || !owner)
|
|
795
|
+
return false;
|
|
796
|
+
await this.destroyRow(row, owner, lease, "device");
|
|
797
|
+
return true;
|
|
798
|
+
});
|
|
799
|
+
if (done)
|
|
800
|
+
n++;
|
|
801
|
+
});
|
|
802
|
+
}
|
|
803
|
+
// inactivity destroys cloud instances only: `@local` instances are never swept
|
|
804
|
+
if (!p)
|
|
805
|
+
return n;
|
|
806
|
+
const due = this.instanceDue(now);
|
|
807
|
+
const idle = (eb) => eb.and([this.cloudRow(eb), due(eb)]);
|
|
808
|
+
await inPool(await this.candidates(idle), SWEEP_CONCURRENCY, async ({ device_id: id }) => {
|
|
548
809
|
const done = await this.tryLocked(id, async (lease) => {
|
|
549
810
|
const row = await this.db
|
|
550
|
-
.selectFrom("
|
|
811
|
+
.selectFrom("device_instances")
|
|
551
812
|
.selectAll()
|
|
552
|
-
.where("
|
|
553
|
-
.where(
|
|
813
|
+
.where("device_id", "=", id)
|
|
814
|
+
.where(idle)
|
|
554
815
|
.executeTakeFirst();
|
|
555
816
|
if (!row)
|
|
556
817
|
return false;
|
|
557
|
-
await this.destroyRow(row, p, lease);
|
|
818
|
+
await this.destroyRow(row, p, lease, "instance");
|
|
558
819
|
return true;
|
|
559
820
|
});
|
|
560
821
|
if (done)
|
|
561
822
|
n++;
|
|
562
823
|
});
|
|
563
|
-
// 早先已销毁、对话后来才删除、且没有遗留旧实例的记录
|
|
564
|
-
await this.db
|
|
565
|
-
.deleteFrom("sandboxes")
|
|
566
|
-
.where("status", "=", "destroyed")
|
|
567
|
-
.where("prev_handle", "is", null)
|
|
568
|
-
.where(({ not, exists, selectFrom }) => not(exists(selectFrom("conversations")
|
|
569
|
-
.select("conversations.id")
|
|
570
|
-
.whereRef("conversations.id", "=", "sandboxes.conversation_id"))))
|
|
571
|
-
.execute();
|
|
572
824
|
return n;
|
|
573
825
|
}
|
|
574
826
|
/**
|
|
575
|
-
*
|
|
576
|
-
*
|
|
577
|
-
* to 10 minutes.
|
|
827
|
+
* Destroys the device's instance, then its file system, then deletes its instance row
|
|
828
|
+
* (device deletion, offboarding); nothing to do without a row. Waits for the exclusive lease
|
|
829
|
+
* (with intent, so running commands finish and new ones wait) up to 10 minutes. Without the
|
|
830
|
+
* device's provider it throws and the row stays, marked `sweep_failed_at` (the job is
|
|
831
|
+
* retried, and the destruction sweep finishes a deleted device once the provider is back).
|
|
832
|
+
* Files kept by another provider that is not loaded cannot be destroyed: it throws and the row
|
|
833
|
+
* stays.
|
|
578
834
|
*/
|
|
579
|
-
async
|
|
835
|
+
async destroyDevice(deviceId) {
|
|
580
836
|
const row = await this.db
|
|
581
|
-
.selectFrom("
|
|
582
|
-
.select("
|
|
583
|
-
.where("
|
|
837
|
+
.selectFrom("device_instances")
|
|
838
|
+
.select("device_id")
|
|
839
|
+
.where("device_id", "=", deviceId)
|
|
584
840
|
.executeTakeFirst();
|
|
585
|
-
if (!row
|
|
841
|
+
if (!row)
|
|
586
842
|
return;
|
|
587
|
-
const p = this.
|
|
843
|
+
const p = await this.providerOf(deviceId);
|
|
588
844
|
if (!p) {
|
|
589
|
-
this.
|
|
590
|
-
|
|
845
|
+
await this.db
|
|
846
|
+
.updateTable("device_instances")
|
|
847
|
+
.set({ sweep_failed_at: sql `now()` })
|
|
848
|
+
.where("device_id", "=", deviceId)
|
|
849
|
+
.execute();
|
|
850
|
+
throw new Error(`cannot destroy device ${deviceId}: its sandbox provider is not there; ` +
|
|
851
|
+
"the destruction sweep retries it");
|
|
591
852
|
}
|
|
592
|
-
const lease = await this.leases.acquireExclusive(
|
|
853
|
+
const lease = await this.leases.acquireExclusive(deviceId, AbortSignal.timeout(EXCLUSIVE_WAIT_MS), undefined, { intent: true });
|
|
593
854
|
try {
|
|
594
855
|
const current = await this.db
|
|
595
|
-
.selectFrom("
|
|
856
|
+
.selectFrom("device_instances")
|
|
596
857
|
.selectAll()
|
|
597
|
-
.where("
|
|
598
|
-
.where("status", "<>", "destroyed")
|
|
858
|
+
.where("device_id", "=", deviceId)
|
|
599
859
|
.executeTakeFirst();
|
|
600
860
|
if (current)
|
|
601
|
-
await this.destroyRow(current, p, lease);
|
|
861
|
+
await this.destroyRow(current, p, lease, "device");
|
|
602
862
|
}
|
|
603
863
|
finally {
|
|
604
864
|
await lease.release();
|
|
605
865
|
}
|
|
606
866
|
}
|
|
607
867
|
/**
|
|
608
|
-
*
|
|
609
|
-
*
|
|
868
|
+
* Starts the device over with empty files (after `device.files_lost`, or on request):
|
|
869
|
+
* under the exclusive lease (waited for like {@link destroyDevice}) destroys any instance and
|
|
870
|
+
* the file system (a `lost` one is only dropped), and records no file system; the next use
|
|
871
|
+
* creates a new one. Nothing to do without a row. Throws `sandbox.unavailable` without a provider.
|
|
610
872
|
*/
|
|
611
|
-
async
|
|
612
|
-
const
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
873
|
+
async resetFileSystem(deviceId) {
|
|
874
|
+
const p = await this.require(deviceId);
|
|
875
|
+
const lease = await this.leases.acquireExclusive(deviceId, AbortSignal.timeout(EXCLUSIVE_WAIT_MS), undefined, { intent: true });
|
|
876
|
+
try {
|
|
877
|
+
const row = await this.db
|
|
878
|
+
.selectFrom("device_instances")
|
|
879
|
+
.selectAll()
|
|
880
|
+
.where("device_id", "=", deviceId)
|
|
881
|
+
.executeTakeFirst();
|
|
882
|
+
if (!row)
|
|
883
|
+
return;
|
|
884
|
+
const fence = { gen: row.generation };
|
|
885
|
+
const signal = opSignal(lease);
|
|
886
|
+
const prev = await this.destroyInstance(row, p, signal);
|
|
887
|
+
await this.destroyFiles(row, p, signal, true);
|
|
888
|
+
if (!(await lease.renew()))
|
|
889
|
+
throw new SandboxLockLostError(deviceId);
|
|
890
|
+
await this.fenced(lease, deviceId, fence, {
|
|
891
|
+
fs_handle: null,
|
|
892
|
+
fs_status: "none",
|
|
893
|
+
handle: null,
|
|
894
|
+
status: "none",
|
|
895
|
+
running_since: null,
|
|
896
|
+
suspended_at: null,
|
|
897
|
+
destroyed_at: sql `now()`,
|
|
898
|
+
sweep_failed_at: null,
|
|
899
|
+
// conversations that used the old files are told they were reset (seenBy)
|
|
900
|
+
fs_resets: sql `fs_resets + 1`,
|
|
901
|
+
...prev,
|
|
902
|
+
});
|
|
903
|
+
this.log.info("device files reset", { deviceId });
|
|
904
|
+
}
|
|
905
|
+
finally {
|
|
906
|
+
await lease.release();
|
|
618
907
|
}
|
|
619
|
-
await this.kill(conversationId, why, () => killAll.call(p.impl, handle, signal), signal);
|
|
620
908
|
}
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
909
|
+
/**
|
|
910
|
+
* Under the exclusive lease (before the downgrade), on an instance just resumed from
|
|
911
|
+
* `suspended`: kills every command left in it (`killAll`), crashed holders' (`dead`, already
|
|
912
|
+
* claimed) included. Never on the shared path: other holders may be running commands. A gone
|
|
913
|
+
* instance (`SandboxGoneError`) and an abort are thrown (`withSandbox` rebuilds / ends); other
|
|
914
|
+
* failures are only warned.
|
|
915
|
+
*/
|
|
916
|
+
async killLeftovers(deviceId, handle, p, signal, dead) {
|
|
917
|
+
await this.kill(deviceId, dead.length > 0 ? "takeover" : "resume", () => p.impl.killAll(handle, signal), signal);
|
|
918
|
+
}
|
|
919
|
+
/**
|
|
920
|
+
* Under the exclusive lease, right after a resume or a (re)build: kills the sessions of the
|
|
921
|
+
* background tasks marked lost while their process may live on (`kill_pending`), in one exec of
|
|
922
|
+
* `pkill -KILL -s <sid>`, then clears their flags. Tasks of another instance have no process
|
|
923
|
+
* here and are only cleared. A gone instance and an abort are thrown; another failure is warned
|
|
924
|
+
* and leaves the flags of this instance's tasks for the next resume.
|
|
925
|
+
*/
|
|
926
|
+
async killPendingTasks(deviceId, handle, createdAt, p, tag, actor, signal) {
|
|
927
|
+
const rows = await this.background.pendingKills(deviceId);
|
|
928
|
+
if (rows.length === 0)
|
|
929
|
+
return;
|
|
930
|
+
const mine = rows.filter((r) => r.sandboxCreatedAt?.getTime() === createdAt && r.sid !== null && SID_RE.test(r.sid));
|
|
931
|
+
let cleared = rows;
|
|
932
|
+
if (mine.length > 0) {
|
|
933
|
+
// pkill exits 1 when the session has no process left: only a higher status is a failure
|
|
934
|
+
const command = `rc=0; for sid in ${mine.map((r) => r.sid).join(" ")}; do pkill -KILL -s "$sid"; c=$?; [ "$c" -le 1 ] || rc=$c; done; exit $rc`;
|
|
935
|
+
try {
|
|
936
|
+
const r = await p.impl.exec(handle, {
|
|
937
|
+
command,
|
|
938
|
+
timeoutMs: PENDING_KILL_TIMEOUT_MS,
|
|
939
|
+
maxOutputBytes: 4096,
|
|
940
|
+
signal,
|
|
941
|
+
tag,
|
|
942
|
+
actor,
|
|
943
|
+
});
|
|
944
|
+
if (r.exitCode !== 0) {
|
|
945
|
+
throw new Error(`pkill exited with ${r.exitCode === null ? "no status" : r.exitCode}: ${r.stderr.slice(0, 500)}`);
|
|
946
|
+
}
|
|
947
|
+
this.log.info("lost background tasks killed after a sandbox resume", {
|
|
948
|
+
deviceId,
|
|
949
|
+
tasks: mine.map((t) => t.id),
|
|
950
|
+
});
|
|
951
|
+
}
|
|
952
|
+
catch (err) {
|
|
953
|
+
if (signal.aborted || isSandboxGoneError(err))
|
|
954
|
+
throw err;
|
|
955
|
+
this.log.warn("cannot kill lost background tasks after a sandbox resume", {
|
|
956
|
+
deviceId,
|
|
957
|
+
err,
|
|
958
|
+
});
|
|
959
|
+
const kept = new Set(mine.map((r) => r.id));
|
|
960
|
+
cleared = rows.filter((r) => !kept.has(r.id));
|
|
961
|
+
}
|
|
962
|
+
}
|
|
963
|
+
await this.background.clearKill(cleared.map((r) => r.id));
|
|
626
964
|
}
|
|
627
965
|
/**
|
|
628
|
-
*
|
|
966
|
+
* A fenced write: only while the lease holds and the row's generation is still the one read
|
|
967
|
+
* under it (and incremented); otherwise {@link SandboxLockLostError}.
|
|
629
968
|
*/
|
|
630
969
|
async fenced(lease, id, fence, set) {
|
|
631
970
|
if (lease.signal.aborted)
|
|
632
971
|
throw new SandboxLockLostError(id);
|
|
633
972
|
const r = await this.db
|
|
634
|
-
.updateTable("
|
|
973
|
+
.updateTable("device_instances")
|
|
635
974
|
.set({ ...set, generation: fence.gen + 1 })
|
|
636
|
-
.where("
|
|
975
|
+
.where("device_id", "=", id)
|
|
637
976
|
.where("generation", "=", fence.gen)
|
|
638
977
|
.returning("created_at")
|
|
639
978
|
.executeTakeFirst();
|
|
@@ -642,11 +981,11 @@ let SandboxService = class SandboxService {
|
|
|
642
981
|
fence.gen++;
|
|
643
982
|
return r.created_at;
|
|
644
983
|
}
|
|
645
|
-
/**
|
|
984
|
+
/** Sweep candidates: those whose last sweep failed come last, the rest least recently used first. */
|
|
646
985
|
async candidates(due) {
|
|
647
986
|
return await this.db
|
|
648
|
-
.selectFrom("
|
|
649
|
-
.select(["
|
|
987
|
+
.selectFrom("device_instances")
|
|
988
|
+
.select(["device_id", "running_since"])
|
|
650
989
|
.where(due)
|
|
651
990
|
.orderBy(sql `sweep_failed_at ASC NULLS FIRST`)
|
|
652
991
|
.orderBy("last_used_at")
|
|
@@ -655,17 +994,17 @@ let SandboxService = class SandboxService {
|
|
|
655
994
|
}
|
|
656
995
|
/**
|
|
657
996
|
* One try at the exclusive lease: null → false; else runs `fn`. With `overdue` (past
|
|
658
|
-
* `maxRunningMinutes`) a busy
|
|
659
|
-
* briefly and the sweep gets in if the running ones end. Failures are warned, recorded in
|
|
997
|
+
* `maxRunningMinutes`) a busy device gets the intent for about a second, so new commands
|
|
998
|
+
* wait briefly and the sweep gets in if the running ones end. Failures are warned, recorded in
|
|
660
999
|
* `sweep_failed_at` (sorted last next time) and return false.
|
|
661
1000
|
*/
|
|
662
|
-
async tryLocked(
|
|
1001
|
+
async tryLocked(deviceId, fn, overdue = false) {
|
|
663
1002
|
let lease;
|
|
664
1003
|
try {
|
|
665
|
-
lease = await this.leases.tryExclusive(
|
|
1004
|
+
lease = await this.leases.tryExclusive(deviceId);
|
|
666
1005
|
if (!lease && overdue) {
|
|
667
1006
|
lease = await this.leases
|
|
668
|
-
.acquireExclusive(
|
|
1007
|
+
.acquireExclusive(deviceId, AbortSignal.timeout(SWEEP_OP_TIMEOUT_MS), Math.max(this.pollMs * 2, SWEEP_INTENT_WAIT_MS), { intent: true })
|
|
669
1008
|
.catch((err) => {
|
|
670
1009
|
if (err instanceof ApiError && err.code === "sandbox.busy")
|
|
671
1010
|
return null;
|
|
@@ -674,7 +1013,7 @@ let SandboxService = class SandboxService {
|
|
|
674
1013
|
}
|
|
675
1014
|
}
|
|
676
1015
|
catch (err) {
|
|
677
|
-
this.log.warn("cannot take sandbox lock", {
|
|
1016
|
+
this.log.warn("cannot take sandbox lock", { deviceId, err });
|
|
678
1017
|
return false;
|
|
679
1018
|
}
|
|
680
1019
|
if (!lease)
|
|
@@ -683,11 +1022,11 @@ let SandboxService = class SandboxService {
|
|
|
683
1022
|
return await fn(lease);
|
|
684
1023
|
}
|
|
685
1024
|
catch (err) {
|
|
686
|
-
this.log.warn("sandbox maintenance failed; will retry", {
|
|
1025
|
+
this.log.warn("sandbox maintenance failed; will retry", { deviceId, err });
|
|
687
1026
|
await this.db
|
|
688
|
-
.updateTable("
|
|
1027
|
+
.updateTable("device_instances")
|
|
689
1028
|
.set({ sweep_failed_at: sql `now()` })
|
|
690
|
-
.where("
|
|
1029
|
+
.where("device_id", "=", deviceId)
|
|
691
1030
|
.execute()
|
|
692
1031
|
.catch(() => { });
|
|
693
1032
|
return false;
|
|
@@ -697,82 +1036,121 @@ let SandboxService = class SandboxService {
|
|
|
697
1036
|
}
|
|
698
1037
|
}
|
|
699
1038
|
/**
|
|
700
|
-
* Under the exclusive lease: make the
|
|
1039
|
+
* Under the exclusive lease: make the device's instance run. `gone`: the instance a holder
|
|
701
1040
|
* found gone; when it is still the recorded one it is rebuilt, when another holder already
|
|
702
1041
|
* rebuilt it the new one is used.
|
|
703
1042
|
*/
|
|
704
1043
|
async ensure(ctx, p, held, signal, gone) {
|
|
705
|
-
const id = ctx.
|
|
1044
|
+
const id = ctx.deviceId;
|
|
1045
|
+
// Only a live device of the caller's org gets a row, with the device's own org; a
|
|
1046
|
+
// row made earlier (another use) keeps its file system handle.
|
|
706
1047
|
await this.db
|
|
707
|
-
.insertInto("
|
|
708
|
-
.
|
|
709
|
-
.
|
|
1048
|
+
.insertInto("device_instances")
|
|
1049
|
+
.columns(["device_id", "org_id", "provider", "status"])
|
|
1050
|
+
.expression((eb) => eb
|
|
1051
|
+
.selectFrom("devices")
|
|
1052
|
+
.select((e) => [
|
|
1053
|
+
"devices.id",
|
|
1054
|
+
"devices.org_id",
|
|
1055
|
+
e.val(p.plugin).as("provider"),
|
|
1056
|
+
e.val("creating").as("status"),
|
|
1057
|
+
])
|
|
1058
|
+
.where("devices.id", "=", id)
|
|
1059
|
+
.where("devices.org_id", "=", ctx.orgId)
|
|
1060
|
+
.where("devices.deleted_at", "is", null))
|
|
1061
|
+
.onConflict((oc) => oc.column("device_id").doNothing())
|
|
710
1062
|
.execute();
|
|
1063
|
+
// Under the lease: a use that waited behind destroyDevice (or raced a deletion) must not
|
|
1064
|
+
// bring the device's provider resources back.
|
|
1065
|
+
const live = await this.db
|
|
1066
|
+
.selectFrom("devices")
|
|
1067
|
+
.select("id")
|
|
1068
|
+
.where("id", "=", id)
|
|
1069
|
+
.where("org_id", "=", ctx.orgId)
|
|
1070
|
+
.where("deleted_at", "is", null)
|
|
1071
|
+
.executeTakeFirst();
|
|
1072
|
+
if (!live)
|
|
1073
|
+
throw deviceNotFound();
|
|
711
1074
|
const row = await this.db
|
|
712
|
-
.selectFrom("
|
|
1075
|
+
.selectFrom("device_instances")
|
|
713
1076
|
.selectAll()
|
|
714
|
-
.where("
|
|
715
|
-
.
|
|
716
|
-
if (row
|
|
717
|
-
throw
|
|
718
|
-
|
|
1077
|
+
.where("device_id", "=", id)
|
|
1078
|
+
.executeTakeFirst();
|
|
1079
|
+
if (!row)
|
|
1080
|
+
throw deviceNotFound();
|
|
1081
|
+
if (row.org_id !== ctx.orgId)
|
|
1082
|
+
throw this.foreign(ctx);
|
|
719
1083
|
const fence = { gen: row.generation };
|
|
720
|
-
//
|
|
1084
|
+
// Every acquisition first does a fenced increment: an old holder whose lease expired while it
|
|
1085
|
+
// kept running (a stalled sweep) then fails its next write.
|
|
721
1086
|
await this.fenced(held, id, fence, {});
|
|
1087
|
+
if (row.fs_status === "lost")
|
|
1088
|
+
throw deviceFilesLost(p.plugin === LOCAL_PROVIDER);
|
|
1089
|
+
if (row.provider !== null &&
|
|
1090
|
+
row.provider !== p.plugin &&
|
|
1091
|
+
(row.fs_status !== "none" || row.handle !== null)) {
|
|
1092
|
+
throw otherProvider(row.provider);
|
|
1093
|
+
}
|
|
1094
|
+
if (row.provider !== null &&
|
|
1095
|
+
row.provider !== p.plugin &&
|
|
1096
|
+
row.fs_status === "none" &&
|
|
1097
|
+
row.status === "creating") {
|
|
1098
|
+
// a create by the previous provider ended before its file system was recorded: that
|
|
1099
|
+
// provider may have made it by name, and nothing else would ever destroy it
|
|
1100
|
+
await this.dropUnrecordedFiles(id, row.provider, signal);
|
|
1101
|
+
}
|
|
722
1102
|
const handle = row.handle;
|
|
723
1103
|
const createdAt = row.created_at.getTime();
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
1104
|
+
const fsResets = row.fs_resets;
|
|
1105
|
+
if (handle && isInstance(row, gone) && row.status === "running") {
|
|
1106
|
+
const created = await this.create(ctx, p, held, row, fence, signal);
|
|
1107
|
+
return { ...created, fsResets, info: { fresh: true, restarted: true }, fence };
|
|
727
1108
|
}
|
|
728
|
-
if (handle && row.
|
|
729
|
-
|
|
730
|
-
return { handle, createdAt, info: { fresh: false, recreated: false }, fence };
|
|
731
|
-
}
|
|
732
|
-
if (row.status === "suspended") {
|
|
733
|
-
const r = await p.impl.resume(handle, signal);
|
|
734
|
-
if (r === "resumed") {
|
|
735
|
-
await this.fenced(held, id, fence, {
|
|
736
|
-
status: "running",
|
|
737
|
-
running_since: sql `now()`,
|
|
738
|
-
suspended_at: null,
|
|
739
|
-
sweep_failed_at: null,
|
|
740
|
-
});
|
|
741
|
-
return {
|
|
742
|
-
handle,
|
|
743
|
-
createdAt,
|
|
744
|
-
info: { fresh: false, recreated: false },
|
|
745
|
-
fence,
|
|
746
|
-
resumed: true,
|
|
747
|
-
};
|
|
748
|
-
}
|
|
749
|
-
this.log.info("sandbox is gone at the provider; recreating", { conversationId: id });
|
|
750
|
-
}
|
|
1109
|
+
if (handle && row.status === "running") {
|
|
1110
|
+
return { handle, createdAt, fsResets, info: { fresh: false, restarted: false }, fence };
|
|
751
1111
|
}
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
1112
|
+
if (handle && row.status === "suspended") {
|
|
1113
|
+
const r = await p.impl.resume(handle, signal);
|
|
1114
|
+
if (r === "resumed") {
|
|
1115
|
+
await this.fenced(held, id, fence, {
|
|
1116
|
+
status: "running",
|
|
1117
|
+
running_since: sql `now()`,
|
|
1118
|
+
suspended_at: null,
|
|
1119
|
+
sweep_failed_at: null,
|
|
1120
|
+
});
|
|
1121
|
+
return {
|
|
1122
|
+
handle,
|
|
1123
|
+
createdAt,
|
|
1124
|
+
fsResets,
|
|
1125
|
+
info: { fresh: false, restarted: false },
|
|
1126
|
+
fence,
|
|
1127
|
+
resumed: true,
|
|
1128
|
+
};
|
|
764
1129
|
}
|
|
1130
|
+
this.log.info("sandbox instance is gone at the provider; recreating", {
|
|
1131
|
+
deviceId: id,
|
|
1132
|
+
});
|
|
765
1133
|
}
|
|
766
|
-
const created = await this.create(ctx, p, held, fence, signal
|
|
767
|
-
// had_instance
|
|
768
|
-
|
|
1134
|
+
const created = await this.create(ctx, p, held, row, fence, signal);
|
|
1135
|
+
// had_instance only ever becomes true: any earlier instance (even if a rebuild failed since)
|
|
1136
|
+
// makes this one a restart on the kept files
|
|
1137
|
+
return {
|
|
1138
|
+
...created,
|
|
1139
|
+
fsResets,
|
|
1140
|
+
info: { fresh: true, restarted: row.had_instance && row.fs_status === "ready" },
|
|
1141
|
+
fence,
|
|
1142
|
+
};
|
|
769
1143
|
}
|
|
770
1144
|
/**
|
|
771
|
-
*
|
|
772
|
-
*
|
|
1145
|
+
* Under the lease: fenced reset to `creating`, the file system when there is none
|
|
1146
|
+
* (`createFileSystem`, its handle written fenced), then the instance on it, then a fenced write
|
|
1147
|
+
* of the instance handle. `FileSystemGoneError` from `create` records `fs_status = 'lost'` and
|
|
1148
|
+
* throws `device.files_lost`. When the instance handle cannot be written (lease lost), the
|
|
1149
|
+
* new instance is destroyed (best effort) before {@link SandboxLockLostError} is thrown.
|
|
773
1150
|
*/
|
|
774
|
-
async create(ctx, p, held, fence, signal
|
|
775
|
-
const id = ctx.
|
|
1151
|
+
async create(ctx, p, held, row, fence, signal) {
|
|
1152
|
+
const id = ctx.deviceId;
|
|
1153
|
+
const name = deviceResourceName(id);
|
|
776
1154
|
await this.fenced(held, id, fence, {
|
|
777
1155
|
org_id: ctx.orgId,
|
|
778
1156
|
provider: p.plugin,
|
|
@@ -783,21 +1161,45 @@ let SandboxService = class SandboxService {
|
|
|
783
1161
|
suspended_at: null,
|
|
784
1162
|
destroyed_at: null,
|
|
785
1163
|
sweep_failed_at: null,
|
|
786
|
-
...extra,
|
|
787
1164
|
});
|
|
1165
|
+
// the org's effective limits apply at creation only, so a changed setting reaches only
|
|
1166
|
+
// instances created afterwards
|
|
1167
|
+
const limits = await this.orgLimits(ctx.orgId);
|
|
788
1168
|
const cfg = this.config.get().sandbox;
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
1169
|
+
let fs = row.fs_status === "ready" && (row.provider === null || row.provider === p.plugin)
|
|
1170
|
+
? row.fs_handle
|
|
1171
|
+
: null;
|
|
1172
|
+
if (fs === null) {
|
|
1173
|
+
fs = await p.impl.createFileSystem({ name, limits, signal });
|
|
1174
|
+
// A lost lease fails this write and leaves the file system alone: createFileSystem returns
|
|
1175
|
+
// the existing one for a name, so the new holder may already use it, files and all.
|
|
1176
|
+
await this.fenced(held, id, fence, {
|
|
1177
|
+
provider: p.plugin,
|
|
1178
|
+
fs_handle: JSON.stringify(fs),
|
|
1179
|
+
fs_status: "ready",
|
|
1180
|
+
});
|
|
1181
|
+
this.log.info("device file system created", { deviceId: id, provider: p.plugin });
|
|
1182
|
+
}
|
|
1183
|
+
let handle;
|
|
1184
|
+
try {
|
|
1185
|
+
handle = await p.impl.create({
|
|
1186
|
+
name,
|
|
1187
|
+
...(cfg.image !== undefined && { image: cfg.image }),
|
|
1188
|
+
limits,
|
|
1189
|
+
fileSystem: fs,
|
|
1190
|
+
signal,
|
|
1191
|
+
});
|
|
1192
|
+
}
|
|
1193
|
+
catch (err) {
|
|
1194
|
+
if (!isFileSystemGoneError(err))
|
|
1195
|
+
throw err;
|
|
1196
|
+
this.log.warn("device file system is gone at the provider; its files are lost", {
|
|
1197
|
+
deviceId: id,
|
|
1198
|
+
provider: p.plugin,
|
|
1199
|
+
});
|
|
1200
|
+
await this.fenced(held, id, fence, { fs_status: "lost", status: "none" });
|
|
1201
|
+
throw deviceFilesLost(p.plugin === LOCAL_PROVIDER);
|
|
1202
|
+
}
|
|
801
1203
|
let createdAt;
|
|
802
1204
|
try {
|
|
803
1205
|
createdAt = await this.fenced(held, id, fence, {
|
|
@@ -810,23 +1212,51 @@ let SandboxService = class SandboxService {
|
|
|
810
1212
|
});
|
|
811
1213
|
}
|
|
812
1214
|
catch (e) {
|
|
813
|
-
//
|
|
814
|
-
//
|
|
815
|
-
//
|
|
1215
|
+
// Lease lost before the handle was recorded: destroy the new instance (best effort) so it
|
|
1216
|
+
// does not leak. A provider that reuses instances by name (Ruling M10) may then hit the one
|
|
1217
|
+
// the new holder just resumed or created; that heals: its next provider call throws
|
|
1218
|
+
// SandboxGoneError (or resume returns gone) and withSandbox rebuilds in place. The file
|
|
1219
|
+
// system is never touched here.
|
|
816
1220
|
await p.impl.destroy(handle, AbortSignal.timeout(SWEEP_OP_TIMEOUT_MS)).catch((err) => this.log.warn("cannot destroy sandbox created after the lock was lost", {
|
|
817
|
-
|
|
1221
|
+
deviceId: id,
|
|
818
1222
|
err,
|
|
819
1223
|
}));
|
|
820
1224
|
throw e;
|
|
821
1225
|
}
|
|
822
|
-
this.log.info("sandbox created", {
|
|
1226
|
+
this.log.info("sandbox created", { deviceId: id, provider: p.plugin });
|
|
823
1227
|
return { handle, createdAt: createdAt.getTime() };
|
|
824
1228
|
}
|
|
825
1229
|
/**
|
|
826
|
-
*
|
|
1230
|
+
* Destroys the file system another provider may have made under the device's name without
|
|
1231
|
+
* the kernel recording it (`destroyFileSystemByName`; not there is success). That provider not
|
|
1232
|
+
* loaded, or a failure, is only warned (the files may leak there) so the device stays
|
|
1233
|
+
* usable; an abort is thrown.
|
|
1234
|
+
*/
|
|
1235
|
+
async dropUnrecordedFiles(id, provider, signal) {
|
|
1236
|
+
const old = this.providers.forName(provider)?.impl;
|
|
1237
|
+
if (!old) {
|
|
1238
|
+
this.log.warn("device file system may have been made by a provider that is not loaded; it may leak", { deviceId: id, provider });
|
|
1239
|
+
return;
|
|
1240
|
+
}
|
|
1241
|
+
try {
|
|
1242
|
+
await old.destroyFileSystemByName(deviceResourceName(id), signal);
|
|
1243
|
+
}
|
|
1244
|
+
catch (err) {
|
|
1245
|
+
if (signal.aborted)
|
|
1246
|
+
throw err;
|
|
1247
|
+
this.log.warn("cannot destroy the file system of the previous sandbox provider", {
|
|
1248
|
+
deviceId: id,
|
|
1249
|
+
provider,
|
|
1250
|
+
err,
|
|
1251
|
+
});
|
|
1252
|
+
}
|
|
1253
|
+
}
|
|
1254
|
+
/**
|
|
1255
|
+
* Destroys an instance of another provider with that provider: still loaded and destroyed →
|
|
1256
|
+
* false; otherwise warns and returns true (the record must be kept).
|
|
827
1257
|
*/
|
|
828
1258
|
async retire(id, provider, handle, signal) {
|
|
829
|
-
const old = this.providers.
|
|
1259
|
+
const old = this.providers.forName(provider)?.impl;
|
|
830
1260
|
if (old) {
|
|
831
1261
|
try {
|
|
832
1262
|
await old.destroy(handle, AbortSignal.any([signal, AbortSignal.timeout(SWEEP_OP_TIMEOUT_MS)]));
|
|
@@ -834,48 +1264,86 @@ let SandboxService = class SandboxService {
|
|
|
834
1264
|
}
|
|
835
1265
|
catch (err) {
|
|
836
1266
|
this.log.warn("cannot destroy sandbox instance of the previous provider", {
|
|
837
|
-
|
|
1267
|
+
deviceId: id,
|
|
838
1268
|
provider,
|
|
839
1269
|
err,
|
|
840
1270
|
});
|
|
841
1271
|
return true;
|
|
842
1272
|
}
|
|
843
1273
|
}
|
|
844
|
-
this.log.warn("sandbox was created by another provider that is not loaded; its instance may leak", {
|
|
1274
|
+
this.log.warn("sandbox was created by another provider that is not loaded; its instance may leak", { deviceId: id, provider });
|
|
845
1275
|
return true;
|
|
846
1276
|
}
|
|
847
|
-
/**
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
1277
|
+
/**
|
|
1278
|
+
* Under the lease: destroys the row's instance (and a leaked one of an earlier provider). An
|
|
1279
|
+
* instance without a handle (a create that ended before the handle was recorded) is destroyed by
|
|
1280
|
+
* name. Returns the `prev_*` patch for instances that could not be destroyed.
|
|
1281
|
+
*/
|
|
1282
|
+
async destroyInstance(row, p, signal) {
|
|
1283
|
+
const id = row.device_id;
|
|
1284
|
+
const owner = row.provider ?? p.plugin;
|
|
852
1285
|
let prev = {};
|
|
853
|
-
if (row.handle &&
|
|
1286
|
+
if (row.handle && owner === p.plugin) {
|
|
854
1287
|
await p.impl.destroy(row.handle, signal);
|
|
855
1288
|
}
|
|
856
1289
|
else if (row.handle && row.status !== "destroyed") {
|
|
857
|
-
if (await this.retire(id,
|
|
858
|
-
prev = { prev_provider:
|
|
1290
|
+
if (await this.retire(id, owner, row.handle, signal)) {
|
|
1291
|
+
prev = { prev_provider: owner, prev_handle: JSON.stringify(row.handle) };
|
|
859
1292
|
}
|
|
860
1293
|
}
|
|
861
|
-
else if (!row.handle && row.
|
|
862
|
-
//
|
|
863
|
-
|
|
1294
|
+
else if (!row.handle && row.status === "creating") {
|
|
1295
|
+
// the provider may have built it already: only the name finds it
|
|
1296
|
+
const impl = owner === p.plugin ? p.impl : this.providers.forName(owner)?.impl;
|
|
1297
|
+
await impl?.destroyByName(deviceResourceName(id), signal);
|
|
864
1298
|
}
|
|
865
1299
|
if (row.prev_handle && row.prev_provider && !("prev_handle" in prev)) {
|
|
866
1300
|
if (!(await this.retire(id, row.prev_provider, row.prev_handle, signal))) {
|
|
867
1301
|
prev = { prev_provider: null, prev_handle: null };
|
|
868
1302
|
}
|
|
869
1303
|
}
|
|
1304
|
+
return prev;
|
|
1305
|
+
}
|
|
1306
|
+
/**
|
|
1307
|
+
* Under the lease: destroys the row's file system with the provider that made it (by name when
|
|
1308
|
+
* no handle was recorded: `createFileSystem` may have finished before the kernel stopped). With
|
|
1309
|
+
* that provider not loaded it throws (the row must stay for a retry), or with `lenient` only
|
|
1310
|
+
* warns (the files leak at that provider).
|
|
1311
|
+
*/
|
|
1312
|
+
async destroyFiles(row, p, signal, lenient = false) {
|
|
1313
|
+
const id = row.device_id;
|
|
1314
|
+
const owner = row.provider ?? p.plugin;
|
|
1315
|
+
const impl = owner === p.plugin ? p.impl : this.providers.forName(owner)?.impl;
|
|
1316
|
+
if (!impl) {
|
|
1317
|
+
if (row.fs_handle === null)
|
|
1318
|
+
return;
|
|
1319
|
+
if (!lenient) {
|
|
1320
|
+
throw new Error(`cannot destroy the file system of device ${id}: its sandbox provider ${owner} is not loaded`);
|
|
1321
|
+
}
|
|
1322
|
+
this.log.warn("device file system of a provider that is not loaded is leaked", {
|
|
1323
|
+
deviceId: id,
|
|
1324
|
+
provider: owner,
|
|
1325
|
+
});
|
|
1326
|
+
return;
|
|
1327
|
+
}
|
|
1328
|
+
if (row.fs_handle !== null)
|
|
1329
|
+
await impl.destroyFileSystem(row.fs_handle, signal);
|
|
1330
|
+
else
|
|
1331
|
+
await impl.destroyFileSystemByName(deviceResourceName(id), signal);
|
|
1332
|
+
}
|
|
1333
|
+
/**
|
|
1334
|
+
* Under the lease: `instance` destroys the instance and marks the row `destroyed` (the file
|
|
1335
|
+
* system stays); `device` also destroys the file system and deletes the row.
|
|
1336
|
+
*/
|
|
1337
|
+
async destroyRow(row, p, held, mode) {
|
|
1338
|
+
const id = row.device_id;
|
|
1339
|
+
const fence = { gen: row.generation };
|
|
1340
|
+
const signal = opSignal(held);
|
|
1341
|
+
const prev = await this.destroyInstance(row, p, signal);
|
|
1342
|
+
if (mode === "device")
|
|
1343
|
+
await this.destroyFiles(row, p, signal);
|
|
870
1344
|
if (!(await held.renew()))
|
|
871
1345
|
throw new SandboxLockLostError(id);
|
|
872
|
-
|
|
873
|
-
.selectFrom("conversations")
|
|
874
|
-
.select("id")
|
|
875
|
-
.where("id", "=", id)
|
|
876
|
-
.executeTakeFirst();
|
|
877
|
-
const keepRow = conversation !== undefined || (prev.prev_handle ?? row.prev_handle) != null;
|
|
878
|
-
if (keepRow) {
|
|
1346
|
+
if (mode === "instance") {
|
|
879
1347
|
await this.fenced(held, id, fence, {
|
|
880
1348
|
status: "destroyed",
|
|
881
1349
|
destroyed_at: sql `now()`,
|
|
@@ -883,19 +1351,19 @@ let SandboxService = class SandboxService {
|
|
|
883
1351
|
sweep_failed_at: null,
|
|
884
1352
|
...prev,
|
|
885
1353
|
});
|
|
1354
|
+
this.log.info("sandbox instance destroyed", { deviceId: id });
|
|
1355
|
+
return;
|
|
886
1356
|
}
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
}
|
|
896
|
-
this.log.info("sandbox destroyed", { conversationId: id });
|
|
1357
|
+
const r = await this.db
|
|
1358
|
+
.deleteFrom("device_instances")
|
|
1359
|
+
.where("device_id", "=", id)
|
|
1360
|
+
.where("generation", "=", fence.gen)
|
|
1361
|
+
.executeTakeFirst();
|
|
1362
|
+
if (Number(r.numDeletedRows) !== 1)
|
|
1363
|
+
throw new SandboxLockLostError(id, "generation");
|
|
1364
|
+
this.log.info("device destroyed", { deviceId: id });
|
|
897
1365
|
}
|
|
898
|
-
/**
|
|
1366
|
+
/** Due for suspension: made by this provider, running, and idle or running for too long. */
|
|
899
1367
|
suspendDue(now, plugin) {
|
|
900
1368
|
const lim = this.config.get().sandbox.limits;
|
|
901
1369
|
const idleBefore = new Date(now.getTime() - lim.suspendAfterIdleMinutes * 60_000);
|
|
@@ -906,24 +1374,39 @@ let SandboxService = class SandboxService {
|
|
|
906
1374
|
eb.or([eb("last_used_at", "<", idleBefore), eb("running_since", "<", runningBefore)]),
|
|
907
1375
|
]);
|
|
908
1376
|
}
|
|
909
|
-
/**
|
|
910
|
-
|
|
1377
|
+
/** The provider of an instance row: an internal one (`@local`) by name, else the cloud one. */
|
|
1378
|
+
rowProvider(row) {
|
|
1379
|
+
return providerForDevice(this.providers, row.provider);
|
|
1380
|
+
}
|
|
1381
|
+
/**
|
|
1382
|
+
* A row the idle sweeps may touch: not on a kernel-internal provider (`@local` instances live on
|
|
1383
|
+
* people's machines and are never suspended or destroyed for inactivity).
|
|
1384
|
+
*/
|
|
1385
|
+
cloudRow(eb) {
|
|
1386
|
+
return eb.or([eb("provider", "is", null), eb("provider", "not like", "@%")]);
|
|
1387
|
+
}
|
|
1388
|
+
/** The device was deleted, or its org was: the instance and the files go. */
|
|
1389
|
+
deviceGone(eb) {
|
|
1390
|
+
return eb.or([
|
|
1391
|
+
eb.exists(eb
|
|
1392
|
+
.selectFrom("devices")
|
|
1393
|
+
.select("devices.id")
|
|
1394
|
+
.whereRef("devices.id", "=", "device_instances.device_id")
|
|
1395
|
+
.where("devices.deleted_at", "is not", null)),
|
|
1396
|
+
eb.exists(eb
|
|
1397
|
+
.selectFrom("orgs")
|
|
1398
|
+
.select("orgs.id")
|
|
1399
|
+
.whereRef("orgs.id", "=", "device_instances.org_id")
|
|
1400
|
+
.where("orgs.deleted_at", "is not", null)),
|
|
1401
|
+
]);
|
|
1402
|
+
}
|
|
1403
|
+
/** Due for instance destruction: an instance (or a create in progress) unused for too long. */
|
|
1404
|
+
instanceDue(now) {
|
|
911
1405
|
const days = this.config.get().sandbox.limits.destroyAfterInactiveDays;
|
|
912
1406
|
const before = new Date(now.getTime() - days * 24 * 3_600_000);
|
|
913
1407
|
return (eb) => eb.and([
|
|
914
|
-
eb("status", "
|
|
915
|
-
eb
|
|
916
|
-
eb("last_used_at", "<", before),
|
|
917
|
-
eb.exists(eb
|
|
918
|
-
.selectFrom("orgs")
|
|
919
|
-
.select("orgs.id")
|
|
920
|
-
.whereRef("orgs.id", "=", "sandboxes.org_id")
|
|
921
|
-
.where("orgs.deleted_at", "is not", null)),
|
|
922
|
-
eb.not(eb.exists(eb
|
|
923
|
-
.selectFrom("conversations")
|
|
924
|
-
.select("conversations.id")
|
|
925
|
-
.whereRef("conversations.id", "=", "sandboxes.conversation_id"))),
|
|
926
|
-
]),
|
|
1408
|
+
eb("status", "in", ["creating", "running", "suspended"]),
|
|
1409
|
+
eb("last_used_at", "<", before),
|
|
927
1410
|
]);
|
|
928
1411
|
}
|
|
929
1412
|
};
|
|
@@ -934,6 +1417,7 @@ SandboxService = __decorate([
|
|
|
934
1417
|
__param(2, Inject(SandboxLeases)),
|
|
935
1418
|
__param(3, Inject(SandboxProviders)),
|
|
936
1419
|
__param(4, Inject(KernelConfigService)),
|
|
937
|
-
__param(5, Inject(KERNEL_LOGGER))
|
|
1420
|
+
__param(5, Inject(KERNEL_LOGGER)),
|
|
1421
|
+
__param(6, Inject(BackgroundTaskStore))
|
|
938
1422
|
], SandboxService);
|
|
939
1423
|
export { SandboxService };
|