@capybara-hq/kernel 0.1.0-rc.11 → 0.1.0-rc.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/admin/admin.schemas.d.ts +13 -0
- package/dist/agents/agent-lookup.d.ts +5 -0
- package/dist/agents/agent-lookup.js +15 -0
- package/dist/agents/agents.schemas.d.ts +3 -0
- package/dist/agents/agents.service.js +11 -1
- package/dist/agents/definition.d.ts +1 -0
- package/dist/agents/definition.js +7 -3
- package/dist/agents/org-templates.js +4 -0
- package/dist/agents/pending-connectors.service.js +1 -1
- package/dist/background/background.module.js +2 -2
- package/dist/background/launch-lock.d.ts +1 -1
- package/dist/background/launch-lock.js +1 -1
- package/dist/background/poll.job.d.ts +2 -2
- package/dist/background/poll.job.js +10 -10
- package/dist/background/stop.d.ts +1 -1
- package/dist/background/stop.js +3 -3
- package/dist/background/store.d.ts +5 -5
- package/dist/background/store.js +5 -5
- package/dist/background/tasks.service.d.ts +6 -6
- package/dist/background/tasks.service.js +13 -13
- package/dist/catalog/component-scope.controller.d.ts +18 -6
- package/dist/catalog/component-scope.controller.js +36 -13
- package/dist/catalog/component-scope.service.d.ts +11 -0
- package/dist/catalog/component-scope.service.js +4 -0
- package/dist/catalog/component-state.d.ts +8 -0
- package/dist/catalog/component-state.js +19 -11
- package/dist/catalog/marketplaces.service.d.ts +1 -1
- package/dist/catalog/plugin-import.js +4 -1
- package/dist/channels/channels.module.js +2 -2
- package/dist/channels/inbound.d.ts +8 -8
- package/dist/channels/inbound.js +31 -31
- package/dist/channels/phrases.d.ts +27 -27
- package/dist/channels/phrases.js +27 -27
- package/dist/config/docs.js +1 -1
- package/dist/config/schema.d.ts +2 -2
- package/dist/config/schema.js +21 -21
- package/dist/conversations/conversations.controller.d.ts +11 -11
- package/dist/conversations/conversations.controller.js +15 -15
- package/dist/conversations/conversations.module.d.ts +1 -1
- package/dist/conversations/conversations.module.js +7 -7
- package/dist/conversations/conversations.schemas.js +1 -1
- package/dist/conversations/conversations.service.d.ts +23 -14
- package/dist/conversations/conversations.service.js +28 -21
- package/dist/conversations/{environment-status.d.ts → device-status.d.ts} +9 -9
- package/dist/conversations/{environment-status.js → device-status.js} +18 -18
- package/dist/conversations/event-log.d.ts +1 -1
- package/dist/conversations/event-log.js +1 -1
- package/dist/conversations/events.d.ts +2 -1
- package/dist/conversations/events.js +2 -2
- package/dist/ctx/create-ctx.d.ts +10 -1
- package/dist/ctx/create-ctx.js +86 -25
- package/dist/db/kernel-schema.d.ts +39 -23
- package/dist/{environments → devices}/access.d.ts +19 -19
- package/dist/{environments → devices}/access.js +28 -28
- package/dist/devices/conversation-devices.d.ts +75 -0
- package/dist/{environments/conversation-environments.js → devices/conversation-devices.js} +71 -72
- package/dist/devices/devices.controller.d.ts +36 -0
- package/dist/{environments/environments.controller.js → devices/devices.controller.js} +37 -37
- package/dist/devices/devices.module.d.ts +5 -13
- package/dist/devices/devices.module.js +16 -29
- package/dist/devices/devices.schemas.d.ts +3 -0
- package/dist/devices/devices.schemas.js +14 -0
- package/dist/{environments/environments.service.d.ts → devices/devices.service.d.ts} +70 -70
- package/dist/{environments/environments.service.js → devices/devices.service.js} +122 -122
- package/dist/devices/dto.d.ts +13 -54
- package/dist/devices/dto.js +7 -39
- package/dist/devices/label.d.ts +11 -0
- package/dist/{environments → devices}/label.js +7 -7
- package/dist/{environments → devices}/legacy-cleanup.job.d.ts +3 -3
- package/dist/{environments → devices}/legacy-cleanup.job.js +3 -3
- package/dist/errors.d.ts +27 -25
- package/dist/errors.js +27 -25
- package/dist/evals/recorded-tools.d.ts +2 -2
- package/dist/evals/recorded-tools.js +6 -6
- package/dist/extensions/admin-ops.js +1 -0
- package/dist/extensions/admin-view.js +2 -0
- package/dist/extensions/effective-config.d.ts +6 -0
- package/dist/extensions/effective-config.js +18 -7
- package/dist/extensions/host.d.ts +13 -2
- package/dist/extensions/host.js +24 -3
- package/dist/extensions/instance.d.ts +12 -1
- package/dist/extensions/instance.js +13 -0
- package/dist/extensions/permission-view.d.ts +10 -1
- package/dist/files/previewers.d.ts +1 -1
- package/dist/files/previewers.js +1 -1
- package/dist/http/api.module.js +10 -6
- package/dist/http/gateway-routes.d.ts +1 -1
- package/dist/http/gateway-routes.js +1 -1
- package/dist/identity/settings.d.ts +32 -0
- package/dist/identity/settings.js +26 -2
- package/dist/infra/config/kernel-config.service.js +4 -4
- package/dist/kernel.module.js +3 -3
- package/dist/{devices → local-devices}/device-auth.controller.d.ts +6 -6
- package/dist/{devices → local-devices}/device-auth.controller.js +5 -5
- package/dist/{devices → local-devices}/device-auth.d.ts +20 -20
- package/dist/{devices → local-devices}/device-auth.js +51 -53
- package/dist/local-devices/device-auth.module.d.ts +7 -0
- package/dist/{devices → local-devices}/device-auth.module.js +5 -5
- package/dist/{devices → local-devices}/device-routes.d.ts +3 -3
- package/dist/{devices → local-devices}/device-routes.js +2 -2
- package/dist/local-devices/dto.d.ts +66 -0
- package/dist/local-devices/dto.js +44 -0
- package/dist/{devices → local-devices}/keys.d.ts +11 -11
- package/dist/{devices → local-devices}/keys.js +11 -11
- package/dist/{devices/device-gateway.d.ts → local-devices/local-device-gateway.d.ts} +14 -14
- package/dist/{devices/device-gateway.js → local-devices/local-device-gateway.js} +44 -44
- package/dist/{devices/device-provider.d.ts → local-devices/local-device-provider.d.ts} +15 -15
- package/dist/{devices/device-provider.js → local-devices/local-device-provider.js} +23 -23
- package/dist/local-devices/local-devices.module.d.ts +16 -0
- package/dist/local-devices/local-devices.module.js +44 -0
- package/dist/{devices → local-devices}/machines.d.ts +26 -26
- package/dist/{devices → local-devices}/machines.js +71 -71
- package/dist/{devices → local-devices}/relay.d.ts +15 -15
- package/dist/{devices → local-devices}/relay.js +55 -55
- package/dist/{devices → local-devices}/targets.d.ts +8 -8
- package/dist/{devices → local-devices}/targets.js +14 -14
- package/dist/orgs/offboarding.d.ts +6 -6
- package/dist/orgs/offboarding.js +12 -12
- package/dist/orgs/settings.service.d.ts +28 -0
- package/dist/orgs/settings.service.js +20 -1
- package/dist/platform/exclusions.js +48 -3
- package/dist/platform/operation.js +6 -2
- package/dist/platform/ops/admin.ops.js +8 -2
- package/dist/platform/ops/agents.ops.js +19 -6
- package/dist/platform/ops/conversations.ops.d.ts +3 -3
- package/dist/platform/ops/conversations.ops.js +9 -9
- package/dist/platform/ops/devices.ops.d.ts +26 -0
- package/dist/platform/ops/devices.ops.js +311 -0
- package/dist/platform/ops/org-settings.ops.js +2 -2
- package/dist/platform/ops/orgs.ops.js +1 -0
- package/dist/platform/ops/ui.ops.d.ts +8 -0
- package/dist/platform/ops/ui.ops.js +41 -0
- package/dist/platform/platform-ops.module.d.ts +3 -2
- package/dist/platform/platform-ops.module.js +8 -5
- package/dist/platform/route-schemas.js +6 -3
- package/dist/roles/gateway-role.d.ts +3 -3
- package/dist/roles/gateway-role.js +4 -4
- package/dist/roles/gateway.module.d.ts +1 -1
- package/dist/roles/gateway.module.js +5 -5
- package/dist/roles/worker.module.js +1 -1
- package/dist/runtime/context/build.d.ts +1 -1
- package/dist/runtime/context/build.js +6 -6
- package/dist/runtime/context/compaction.js +1 -1
- package/dist/runtime/context/{environment-paths.d.ts → device-paths.d.ts} +11 -11
- package/dist/runtime/context/{environment-paths.js → device-paths.js} +13 -13
- package/dist/runtime/maintenance.d.ts +1 -1
- package/dist/runtime/subagents/definition.d.ts +3 -2
- package/dist/runtime/subagents/definition.js +2 -1
- package/dist/sandbox/jobs.d.ts +13 -13
- package/dist/sandbox/jobs.js +20 -20
- package/dist/sandbox/leases.d.ts +9 -9
- package/dist/sandbox/leases.js +14 -14
- package/dist/sandbox/providers.d.ts +8 -8
- package/dist/sandbox/providers.js +8 -8
- package/dist/sandbox/sandbox.module.d.ts +1 -1
- package/dist/sandbox/sandbox.module.js +1 -1
- package/dist/sandbox/sandbox.service.d.ts +76 -76
- package/dist/sandbox/sandbox.service.js +230 -230
- package/dist/tools/builtins/builtin-tools.d.ts +2 -2
- package/dist/tools/builtins/builtin-tools.js +5 -5
- package/dist/tools/builtins/platform.js +1 -1
- package/dist/tools/builtins/sandbox/bash.js +6 -6
- package/dist/tools/builtins/sandbox/common.d.ts +25 -25
- package/dist/tools/builtins/sandbox/common.js +51 -51
- package/dist/tools/builtins/sandbox/edit.js +2 -2
- package/dist/tools/builtins/sandbox/index.d.ts +1 -1
- package/dist/tools/builtins/sandbox/index.js +1 -1
- package/dist/tools/builtins/sandbox/task-output.d.ts +6 -6
- package/dist/tools/builtins/sandbox/task-output.js +10 -10
- package/dist/tools/builtins/sandbox/task-stop.js +2 -2
- package/dist/tools/builtins/sandbox/write.js +2 -2
- package/dist/tools/builtins/skill-load.d.ts +1 -1
- package/dist/tools/builtins/skill-load.js +2 -2
- package/dist/tools/connector-account-policies.d.ts +51 -0
- package/dist/tools/connector-account-policies.js +165 -0
- package/dist/tools/connectors.controller.d.ts +35 -4
- package/dist/tools/connectors.controller.js +66 -7
- package/dist/tools/connectors.schemas.d.ts +11 -0
- package/dist/tools/connectors.schemas.js +11 -0
- package/dist/tools/connectors.service.d.ts +2 -0
- package/dist/tools/connectors.service.js +1 -0
- package/dist/tools/pipeline.d.ts +1 -1
- package/dist/tools/pipeline.js +4 -3
- package/dist/tools/policy.d.ts +23 -1
- package/dist/tools/policy.js +99 -24
- package/dist/tools/registry.d.ts +2 -2
- package/dist/tools/tools.module.js +6 -3
- package/dist/ui/admin-ui.controller.d.ts +56 -0
- package/dist/ui/admin-ui.controller.js +192 -0
- package/dist/ui/arrangement.d.ts +74 -0
- package/dist/ui/arrangement.js +98 -0
- package/dist/ui/embed-token.d.ts +2 -2
- package/dist/ui/embed-token.js +30 -5
- package/dist/ui/registry.d.ts +29 -14
- package/dist/ui/registry.js +31 -10
- package/dist/ui/slots.controller.d.ts +11 -0
- package/dist/ui/slots.controller.js +25 -0
- package/dist/ui/ui-stream.d.ts +20 -3
- package/dist/ui/ui-stream.js +25 -7
- package/dist/ui/ui.schemas.d.ts +66 -0
- package/dist/ui/ui.schemas.js +65 -0
- package/dist/ui/views.controller.d.ts +35 -49
- package/dist/ui/views.controller.js +48 -70
- package/dist/ui/views.d.ts +84 -18
- package/dist/ui/views.js +201 -38
- package/package.json +3 -3
- package/src/migrations/0104_connector_tool_account_policies.sql +16 -0
- package/src/migrations/0105_devices_rename.sql +136 -0
- package/dist/devices/device-auth.module.d.ts +0 -7
- package/dist/environments/conversation-environments.d.ts +0 -75
- package/dist/environments/dto.d.ts +0 -25
- package/dist/environments/dto.js +0 -12
- package/dist/environments/environments.controller.d.ts +0 -36
- package/dist/environments/environments.module.d.ts +0 -8
- package/dist/environments/environments.module.js +0 -31
- package/dist/environments/environments.schemas.d.ts +0 -3
- package/dist/environments/environments.schemas.js +0 -14
- package/dist/environments/label.d.ts +0 -11
- package/dist/platform/ops/environments.ops.d.ts +0 -26
- package/dist/platform/ops/environments.ops.js +0 -311
|
@@ -21,7 +21,7 @@ import { KERNEL_LOGGER, REDIS } from "../infra/tokens.js";
|
|
|
21
21
|
import { effectiveSandboxEgress, effectiveSandboxLimits, readOrgSettings, } from "../orgs/settings.service.js";
|
|
22
22
|
import { canonicalJson } from "../tools/canonical-json.js";
|
|
23
23
|
import { SandboxLeases, SandboxLockLostError, sandboxBusy, } from "./leases.js";
|
|
24
|
-
import {
|
|
24
|
+
import { LOCAL_PROVIDER, providerForDevice, SandboxProviders, } from "./providers.js";
|
|
25
25
|
// Moved to leases.ts (phase C); re-exported for existing importers
|
|
26
26
|
export { SANDBOX_LOCK_PREFIX, SANDBOX_LOCK_TTL_MS, SandboxLockLostError, sandboxBusy, } from "./leases.js";
|
|
27
27
|
/** Margin (ms) of a holder record's TTL beyond the longest command: by then the in-sandbox limit has ended its commands. */
|
|
@@ -35,7 +35,7 @@ const SWEEP_INTENT_WAIT_MS = 1000;
|
|
|
35
35
|
/** Rows one sweep handles at most, and the provider calls it runs at once. */
|
|
36
36
|
const SWEEP_BATCH = 200;
|
|
37
37
|
const SWEEP_CONCURRENCY = 4;
|
|
38
|
-
/** How long `
|
|
38
|
+
/** How long `destroyDevice` / `resetFileSystem` wait for the exclusive lease (running commands). */
|
|
39
39
|
const EXCLUSIVE_WAIT_MS = 10 * 60_000;
|
|
40
40
|
/** The time limit of the exec that kills lost background tasks after a resume (ms). */
|
|
41
41
|
const PENDING_KILL_TIMEOUT_MS = 30_000;
|
|
@@ -43,18 +43,18 @@ const PENDING_KILL_TIMEOUT_MS = 30_000;
|
|
|
43
43
|
const SID_RE = /^[1-9][0-9]{0,9}$/;
|
|
44
44
|
/** The signal of one provider call in maintenance: aborted on timeout or when the lease is lost. */
|
|
45
45
|
const opSignal = (lease) => AbortSignal.any([lease.signal, AbortSignal.timeout(SWEEP_OP_TIMEOUT_MS)]);
|
|
46
|
-
/** The provider-side name of
|
|
47
|
-
export const
|
|
48
|
-
/** The
|
|
49
|
-
export const
|
|
46
|
+
/** The provider-side name of a device's file system and instance. */
|
|
47
|
+
export const deviceResourceName = (deviceId) => `cap-device-${deviceId}`;
|
|
48
|
+
/** The device is missing, deleted, or not in the caller's org (404; ids cannot be probed). */
|
|
49
|
+
export const deviceNotFound = () => new ApiError("device.not_found", "This device does not exist or was deleted.");
|
|
50
50
|
/**
|
|
51
|
-
* The
|
|
52
|
-
* connected machine (`
|
|
53
|
-
* owner's: `capybara-device logout`, then `login` into the
|
|
51
|
+
* The device's file system is gone at the provider (409); never replaced silently. On a
|
|
52
|
+
* connected machine (`local`, details `[{ source: "local" }]`) the directory is the machine
|
|
53
|
+
* owner's: `capybara-device logout`, then `login` into the device, recreates it (ruling R24).
|
|
54
54
|
*/
|
|
55
|
-
export const
|
|
56
|
-
? new ApiError("
|
|
57
|
-
: new ApiError("
|
|
55
|
+
export const deviceFilesLost = (local = false) => local
|
|
56
|
+
? new ApiError("device.files_lost", "The local device's directory is gone from its machine; run capybara-device logout, then capybara-device login into this device, on that machine to recreate it.", [{ source: "local" }])
|
|
57
|
+
: new ApiError("device.files_lost", "The device's files are gone; reset its files from the devices page to start with an empty workspace.");
|
|
58
58
|
/**
|
|
59
59
|
* The row still records the instance `gone` (same handle and same `created_at`). Handles compare
|
|
60
60
|
* canonically: a jsonb round trip reorders a multi-key handle's keys (final review m9).
|
|
@@ -112,10 +112,10 @@ export function sandboxConfigWarnings(cfg, providerPlugin) {
|
|
|
112
112
|
/** No sandbox provider (503). */
|
|
113
113
|
export const sandboxUnavailable = () => new ApiError("sandbox.unavailable", "Sandbox is not available in this deployment; install a sandbox plugin");
|
|
114
114
|
/**
|
|
115
|
-
* The
|
|
115
|
+
* The device's files are kept by a provider that is not the current one (503); its files are
|
|
116
116
|
* not migrated, so it stays unusable until that provider is back (spec §4.2 item 13).
|
|
117
117
|
*/
|
|
118
|
-
const otherProvider = (provider) => new ApiError("
|
|
118
|
+
const otherProvider = (provider) => new ApiError("device.unavailable", `This device's files are kept by the sandbox provider ${provider}, which is not the current provider; it can be used again once ${provider} is the deployment's sandbox provider`);
|
|
119
119
|
const abortReason = (signal) => signal.reason ?? new DOMException("This operation was aborted", "AbortError");
|
|
120
120
|
function sleep(ms, signal) {
|
|
121
121
|
return new Promise((resolve, reject) => {
|
|
@@ -146,19 +146,19 @@ async function inPool(items, limit, fn) {
|
|
|
146
146
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
|
147
147
|
}
|
|
148
148
|
/**
|
|
149
|
-
* The kernel's sandbox service over the `sandbox@2` provider, keyed by **
|
|
150
|
-
* 2026-11-07-environments-design §4.2, §5.1): one file system kept until the
|
|
149
|
+
* The kernel's sandbox service over the `sandbox@2` provider, keyed by **device** (spec
|
|
150
|
+
* 2026-11-07-environments-design §4.2, §5.1): one file system kept until the device is
|
|
151
151
|
* deleted, plus one instance attached to it that comes and goes; all conversations bound to the
|
|
152
|
-
*
|
|
152
|
+
* device (and their sub-agents) share both.
|
|
153
153
|
*
|
|
154
154
|
* - **Lazy creation**: the first {@link withSandbox} creates the file system
|
|
155
|
-
* (`createFileSystem`, name `cap-
|
|
156
|
-
* in `
|
|
157
|
-
* `fs_status` becomes `lost` and uses fail with `
|
|
155
|
+
* (`createFileSystem`, name `cap-device-<deviceId>`) and then the instance on it; state lives
|
|
156
|
+
* in `device_instances`. A file system gone at the provider is never replaced silently:
|
|
157
|
+
* `fs_status` becomes `lost` and uses fail with `device.files_lost` until
|
|
158
158
|
* {@link resetFileSystem}.
|
|
159
|
-
* - **Leases** (phase C, spec §10, {@link SandboxLeases}), keyed by
|
|
159
|
+
* - **Leases** (phase C, spec §10, {@link SandboxLeases}), keyed by device id: using an
|
|
160
160
|
* instance recorded `running` with the current provider takes a **shared** lease, so up to
|
|
161
|
-
* `sandbox.maxConcurrentCommands` holders (all sessions of the
|
|
161
|
+
* `sandbox.maxConcurrentCommands` holders (all sessions of the device) run at once. Every
|
|
162
162
|
* lifecycle change (create, resume, rebuild, suspend, destroy, reset) takes the **exclusive**
|
|
163
163
|
* lease, alone; a caller that needs one on the use path sets the intent so new shared leases wait
|
|
164
164
|
* for it, and downgrades to a shared lease to run its own command once the instance runs.
|
|
@@ -171,9 +171,9 @@ async function inPool(items, limit, fn) {
|
|
|
171
171
|
* provider calls.
|
|
172
172
|
* - **Suspension and destruction**: the sweeps (`SandboxSweepJobs`, crons) call
|
|
173
173
|
* {@link suspendIdle} / {@link destroyInactive}; they try the exclusive lease once and skip an
|
|
174
|
-
*
|
|
174
|
+
* device with any holder; one past `maxRunningMinutes` gets the intent for a short window
|
|
175
175
|
* (about a second) so the suspension can get in when its commands end. Inactivity destroys only
|
|
176
|
-
* the instance (files kept); a deleted
|
|
176
|
+
* the instance (files kept); a deleted device or org loses its files too.
|
|
177
177
|
* - **Transparent recovery**: a suspended instance is resumed on next use; when `resume` returns
|
|
178
178
|
* `gone`, the instance was destroyed, or `fn` throws `SandboxGoneError` (retried once), a new
|
|
179
179
|
* instance is created on the kept files and `restarted` is true. Several holders hitting `gone`
|
|
@@ -181,7 +181,7 @@ async function inPool(items, limit, fn) {
|
|
|
181
181
|
* - **Takeover** (spec §10.5; environments spec §4.2): every holder records itself in
|
|
182
182
|
* `capk:sandbox-holders:<env>` (again before each `exec`) and its commands carry its lease token
|
|
183
183
|
* as `tag`. An acquirer that finds a record without a live lease (a crashed holder) kills only
|
|
184
|
-
* that holder's commands (`killTagged`): other conversations of the
|
|
184
|
+
* that holder's commands (`killTagged`): other conversations of the device may be running
|
|
185
185
|
* commands. `killAll` runs only under the exclusive lease, before the downgrade, on an instance
|
|
186
186
|
* this holder just resumed from `suspended` (nobody else can hold it; a provider that keeps
|
|
187
187
|
* memory across a pause brings old commands back); it also covers crashed holders' records,
|
|
@@ -191,16 +191,16 @@ async function inPool(items, limit, fn) {
|
|
|
191
191
|
* row's `created_at` (set by the fenced write after each `create`); the handle alone is the same
|
|
192
192
|
* after a rebuild on providers whose handle is the instance name (sandbox-docker).
|
|
193
193
|
*
|
|
194
|
-
* - **Provider per
|
|
195
|
-
* the
|
|
196
|
-
*
|
|
197
|
-
*
|
|
198
|
-
* destroyed for inactivity; a deleted `@
|
|
194
|
+
* - **Provider per device** (environments spec §6): each call resolves the provider from
|
|
195
|
+
* the device's `devices.provider` ({@link providerForDevice}): a `device`
|
|
196
|
+
* device runs on the kernel-internal `@local` provider (never the cloud one), a cloud
|
|
197
|
+
* device on the loaded `sandbox@2` extension. A `@local` instance is never suspended or
|
|
198
|
+
* destroyed for inactivity; a deleted `@local` device (or one of a deleted org) is
|
|
199
199
|
* destroyed by the sweep like any other.
|
|
200
200
|
*
|
|
201
|
-
* Files kept by another provider than the current one make the
|
|
202
|
-
* (`
|
|
203
|
-
* provider is loaded ({@link SandboxProviders.forName}). Without the
|
|
201
|
+
* Files kept by another provider than the current one make the device unusable
|
|
202
|
+
* (`device.unavailable`) until that provider is back; deleting it still works while that
|
|
203
|
+
* provider is loaded ({@link SandboxProviders.forName}). Without the device's provider
|
|
204
204
|
* {@link availableFor} is false and uses throw `sandbox.unavailable` (503). No in-process state:
|
|
205
205
|
* replicas share it through PG and Redis.
|
|
206
206
|
*/
|
|
@@ -233,45 +233,45 @@ let SandboxService = class SandboxService {
|
|
|
233
233
|
}
|
|
234
234
|
/**
|
|
235
235
|
* Whether there is a cloud sandbox provider (a loaded `sandbox@2` extension). For gates that have
|
|
236
|
-
* no
|
|
236
|
+
* no device yet (tool listing, skill files); a call on a device asks
|
|
237
237
|
* {@link availableFor}.
|
|
238
238
|
*/
|
|
239
239
|
get cloudAvailable() {
|
|
240
240
|
return this.providers.cloud() !== undefined;
|
|
241
241
|
}
|
|
242
|
-
/** Whether the
|
|
243
|
-
async availableFor(
|
|
244
|
-
return (await this.providerOf(
|
|
242
|
+
/** Whether the device's provider is there (`@local` registered, or a cloud provider). */
|
|
243
|
+
async availableFor(deviceId) {
|
|
244
|
+
return (await this.providerOf(deviceId)) !== undefined;
|
|
245
245
|
}
|
|
246
246
|
/**
|
|
247
|
-
* The provider the
|
|
248
|
-
* a missing row (or one of another org, with `orgId`) reads as a cloud
|
|
247
|
+
* The provider the device works with, read from its row ({@link providerForDevice});
|
|
248
|
+
* a missing row (or one of another org, with `orgId`) reads as a cloud device, whose use
|
|
249
249
|
* then fails as not found.
|
|
250
250
|
*/
|
|
251
|
-
async providerOf(
|
|
251
|
+
async providerOf(deviceId, orgId) {
|
|
252
252
|
const row = await this.db
|
|
253
|
-
.selectFrom("
|
|
253
|
+
.selectFrom("devices")
|
|
254
254
|
.select("provider")
|
|
255
|
-
.where("id", "=",
|
|
255
|
+
.where("id", "=", deviceId)
|
|
256
256
|
.$if(orgId !== undefined, (q) => q.where("org_id", "=", orgId))
|
|
257
257
|
.executeTakeFirst();
|
|
258
|
-
return
|
|
258
|
+
return providerForDevice(this.providers, row?.provider ?? null);
|
|
259
259
|
}
|
|
260
260
|
/** {@link providerOf}, or `sandbox.unavailable` (503) without it. */
|
|
261
|
-
async require(
|
|
262
|
-
const p = await this.providerOf(
|
|
261
|
+
async require(deviceId, orgId) {
|
|
262
|
+
const p = await this.providerOf(deviceId, orgId);
|
|
263
263
|
if (!p)
|
|
264
264
|
throw sandboxUnavailable();
|
|
265
265
|
return p;
|
|
266
266
|
}
|
|
267
|
-
/** Concurrent holders of one
|
|
267
|
+
/** Concurrent holders of one device (`sandbox.maxConcurrentCommands`, read at every use). */
|
|
268
268
|
get cap() {
|
|
269
269
|
return this.config.get().sandbox.maxConcurrentCommands;
|
|
270
270
|
}
|
|
271
271
|
/**
|
|
272
272
|
* The org's effective sandbox limits (spec §8.4): the deployment's `sandbox.limits`, egress
|
|
273
273
|
* narrowed by the org setting `sandboxEgress` (Ruling M19), cpu / memory per the super admin's
|
|
274
|
-
* org override (C19 / Ruling M16). Instances are created with them; phase 1
|
|
274
|
+
* org override (C19 / Ruling M16). Instances are created with them; phase 1 devices use
|
|
275
275
|
* them unchanged.
|
|
276
276
|
*/
|
|
277
277
|
async orgLimits(orgId) {
|
|
@@ -287,15 +287,15 @@ let SandboxService = class SandboxService {
|
|
|
287
287
|
};
|
|
288
288
|
}
|
|
289
289
|
/**
|
|
290
|
-
* The
|
|
290
|
+
* The device's instance state as shown to people (read only, no lease): `running` /
|
|
291
291
|
* `suspended` and the last use. No instance, a destroyed one, or one stuck in `creating` (being
|
|
292
292
|
* created, or a failed create the next use retries) is null.
|
|
293
293
|
*/
|
|
294
|
-
async status(orgId,
|
|
294
|
+
async status(orgId, deviceId) {
|
|
295
295
|
const row = await this.db
|
|
296
|
-
.selectFrom("
|
|
296
|
+
.selectFrom("device_instances")
|
|
297
297
|
.select(["status", "last_used_at"])
|
|
298
|
-
.where("
|
|
298
|
+
.where("device_id", "=", deviceId)
|
|
299
299
|
.where("org_id", "=", orgId)
|
|
300
300
|
.executeTakeFirst();
|
|
301
301
|
if (!row || (row.status !== "running" && row.status !== "suspended"))
|
|
@@ -303,15 +303,15 @@ let SandboxService = class SandboxService {
|
|
|
303
303
|
return { status: row.status, lastUsedAt: row.last_used_at };
|
|
304
304
|
}
|
|
305
305
|
/**
|
|
306
|
-
* The instance the
|
|
306
|
+
* The instance the device records now (its `created_at`), while it has one that runs or is
|
|
307
307
|
* suspended; null otherwise. Read without a lease: a caller inside `withSandbox` reads the
|
|
308
308
|
* instance its own lease keeps.
|
|
309
309
|
*/
|
|
310
|
-
async currentInstance(
|
|
310
|
+
async currentInstance(deviceId) {
|
|
311
311
|
const row = await this.db
|
|
312
|
-
.selectFrom("
|
|
312
|
+
.selectFrom("device_instances")
|
|
313
313
|
.select(["status", "handle", "created_at"])
|
|
314
|
-
.where("
|
|
314
|
+
.where("device_id", "=", deviceId)
|
|
315
315
|
.executeTakeFirst();
|
|
316
316
|
if (!row || row.handle === null)
|
|
317
317
|
return null;
|
|
@@ -320,7 +320,7 @@ let SandboxService = class SandboxService {
|
|
|
320
320
|
return { createdAt: row.created_at };
|
|
321
321
|
}
|
|
322
322
|
/**
|
|
323
|
-
* Runs `fn` on the
|
|
323
|
+
* Runs `fn` on the device's instance only while it runs the instance created at
|
|
324
324
|
* `instance.createdAt` (background tasks, spec 2026-11-05-background-bash-design §4.3). It never
|
|
325
325
|
* creates, resumes or rebuilds one: an instance that is missing, suspended, being created or
|
|
326
326
|
* destroyed (or recorded with another provider) is `not_running`, one that records another
|
|
@@ -331,18 +331,18 @@ let SandboxService = class SandboxService {
|
|
|
331
331
|
* to `fn` aborts with `signal` or when the lease is lost. Errors of `fn` (`SandboxGoneError`
|
|
332
332
|
* too) are thrown as they are.
|
|
333
333
|
*/
|
|
334
|
-
async ifRunning(
|
|
335
|
-
const p = await this.require(
|
|
336
|
-
const env =
|
|
334
|
+
async ifRunning(deviceId, instance, fn, signal = new AbortController().signal) {
|
|
335
|
+
const p = await this.require(deviceId);
|
|
336
|
+
const env = deviceId;
|
|
337
337
|
const deadline = Date.now() + this.config.get().sandbox.lockWaitSeconds * 1000;
|
|
338
338
|
const token = randomUUID();
|
|
339
339
|
for (;;) {
|
|
340
340
|
if (signal.aborted)
|
|
341
341
|
throw abortReason(signal);
|
|
342
342
|
const row = await this.db
|
|
343
|
-
.selectFrom("
|
|
343
|
+
.selectFrom("device_instances")
|
|
344
344
|
.selectAll()
|
|
345
|
-
.where("
|
|
345
|
+
.where("device_id", "=", env)
|
|
346
346
|
.executeTakeFirst();
|
|
347
347
|
if (!row)
|
|
348
348
|
return { kind: "not_running", status: "gone" };
|
|
@@ -356,7 +356,7 @@ let SandboxService = class SandboxService {
|
|
|
356
356
|
const want = { handle: row.handle, createdAt: row.created_at.getTime() };
|
|
357
357
|
const lease = await this.leases.tryShared(env, this.cap, token);
|
|
358
358
|
if (lease) {
|
|
359
|
-
const e = await this.asShared({ orgId: row.org_id,
|
|
359
|
+
const e = await this.asShared({ orgId: row.org_id, deviceId: env }, p, lease, (r) => this.usable(r, p, null) && isInstance(r, want), signal);
|
|
360
360
|
if (e) {
|
|
361
361
|
try {
|
|
362
362
|
const impl = tagged(p.impl, lease.token, undefined, () => this.register(env, lease.token));
|
|
@@ -383,15 +383,15 @@ let SandboxService = class SandboxService {
|
|
|
383
383
|
}
|
|
384
384
|
}
|
|
385
385
|
/**
|
|
386
|
-
* A short lease on one path of the
|
|
386
|
+
* A short lease on one path of the device around the kernel's own read-modify-write of
|
|
387
387
|
* that file (spec §10.4); waits up to 30 s, then throws `SandboxPathBusyError`. Its `signal`
|
|
388
388
|
* aborts when a renewal finds it lost.
|
|
389
389
|
*/
|
|
390
|
-
pathLease(
|
|
391
|
-
return this.leases.pathLease(
|
|
390
|
+
pathLease(deviceId, path, signal) {
|
|
391
|
+
return this.leases.pathLease(deviceId, path, signal, PATH_LEASE_WAIT_MS);
|
|
392
392
|
}
|
|
393
393
|
/**
|
|
394
|
-
* Runs `fn` on the
|
|
394
|
+
* Runs `fn` on the device's instance: take a lease → make sure the instance runs (file
|
|
395
395
|
* system, create / resume / rebuild, under the exclusive lease) → `fn` under a shared lease →
|
|
396
396
|
* update `last_used_at` → release.
|
|
397
397
|
* - Waiting longer than `sandbox.lockWaitSeconds` throws `sandbox.busy` (409); `signal` aborts
|
|
@@ -402,14 +402,14 @@ let SandboxService = class SandboxService {
|
|
|
402
402
|
* - When `fn` throws `SandboxGoneError` (the instance is gone) the instance is rebuilt on the
|
|
403
403
|
* kept files (or the one another holder rebuilt is taken) and `fn` runs once more with
|
|
404
404
|
* `restarted: true`; a second `gone` is thrown. Other errors are thrown as they are.
|
|
405
|
-
* - Lost files throw `
|
|
405
|
+
* - Lost files throw `device.files_lost` (409), also when `fn` throws
|
|
406
406
|
* `FileSystemGoneError` (the files are then recorded lost, ruling R24).
|
|
407
407
|
* - When `ctx.holderConversationId` last used another instance (another holder rebuilt it
|
|
408
408
|
* meanwhile), `info.restarted` is true, so it learns its processes are gone (final review m5).
|
|
409
409
|
*/
|
|
410
410
|
async withSandbox(ctx, fn, signal) {
|
|
411
|
-
const p = await this.require(ctx.
|
|
412
|
-
const env = ctx.
|
|
411
|
+
const p = await this.require(ctx.deviceId, ctx.orgId);
|
|
412
|
+
const env = ctx.deviceId;
|
|
413
413
|
const holder = ctx.holderConversationId;
|
|
414
414
|
let e = await this.seenBy(env, holder, await this.enter(ctx, p, signal, null));
|
|
415
415
|
try {
|
|
@@ -421,7 +421,7 @@ let SandboxService = class SandboxService {
|
|
|
421
421
|
throw await this.filesGone(env, p, e);
|
|
422
422
|
if (!isSandboxGoneError(err))
|
|
423
423
|
throw err;
|
|
424
|
-
this.log.info("sandbox instance is gone during use; recreating", {
|
|
424
|
+
this.log.info("sandbox instance is gone during use; recreating", { deviceId: env });
|
|
425
425
|
const gone = { handle: e.handle, createdAt: e.createdAt };
|
|
426
426
|
const prev = e;
|
|
427
427
|
await this.leave(env, prev);
|
|
@@ -448,28 +448,28 @@ let SandboxService = class SandboxService {
|
|
|
448
448
|
* Ruling R24: an operation on the held instance `e` found its file system gone. Records
|
|
449
449
|
* `fs_status = 'lost'` while the row still records that instance on its file system (the
|
|
450
450
|
* instance handle stays, so a reset still destroys it), moving the fence on, and returns
|
|
451
|
-
* `
|
|
451
|
+
* `device.files_lost` to throw. Uses then fail until the files are reset (a connected
|
|
452
452
|
* machine: until a new login, which resets the row). A failed write only logs.
|
|
453
453
|
*/
|
|
454
454
|
async filesGone(env, p, e) {
|
|
455
|
-
this.log.warn("
|
|
456
|
-
|
|
455
|
+
this.log.warn("device file system is gone at the provider; its files are lost", {
|
|
456
|
+
deviceId: env,
|
|
457
457
|
provider: p.plugin,
|
|
458
458
|
});
|
|
459
459
|
try {
|
|
460
460
|
await this.db
|
|
461
|
-
.updateTable("
|
|
461
|
+
.updateTable("device_instances")
|
|
462
462
|
.set((eb) => ({ fs_status: "lost", generation: eb("generation", "+", 1) }))
|
|
463
|
-
.where("
|
|
463
|
+
.where("device_id", "=", env)
|
|
464
464
|
.where("fs_status", "=", "ready")
|
|
465
465
|
// `createdAt` is the row's value read back in milliseconds
|
|
466
466
|
.where(sql `date_trunc('milliseconds', created_at)`, "=", new Date(e.createdAt))
|
|
467
467
|
.execute();
|
|
468
468
|
}
|
|
469
469
|
catch (err) {
|
|
470
|
-
this.log.warn("cannot record lost
|
|
470
|
+
this.log.warn("cannot record lost device files", { deviceId: env, err });
|
|
471
471
|
}
|
|
472
|
-
return
|
|
472
|
+
return deviceFilesLost(p.plugin === LOCAL_PROVIDER);
|
|
473
473
|
}
|
|
474
474
|
/**
|
|
475
475
|
* Records that `holder` uses the instance of `e` (`capk:sandbox-seen:<env>:<holder>` =
|
|
@@ -477,14 +477,14 @@ let SandboxService = class SandboxService {
|
|
|
477
477
|
* holder last used another instance under the same reset count `e` is marked restarted (files
|
|
478
478
|
* kept); under another count it is marked `filesReset` instead. The count, not the provider's
|
|
479
479
|
* file system id, tells them apart: a provider may give the new file system the old id
|
|
480
|
-
* (sandbox-docker's volumes are named after the
|
|
480
|
+
* (sandbox-docker's volumes are named after the device). Best effort: a Redis error only
|
|
481
481
|
* logs.
|
|
482
482
|
*/
|
|
483
|
-
async seenBy(
|
|
483
|
+
async seenBy(deviceId, holder, e) {
|
|
484
484
|
try {
|
|
485
485
|
const ttl = (this.config.get().sandbox.limits.destroyAfterInactiveDays + 1) * 86_400;
|
|
486
486
|
const seen = `${e.createdAt}:${e.fsResets}`;
|
|
487
|
-
const prev = await this.redis.set(`capk:sandbox-seen:${
|
|
487
|
+
const prev = await this.redis.set(`capk:sandbox-seen:${deviceId}:${holder}`, seen, "EX", ttl, "GET");
|
|
488
488
|
if (prev !== null && prev !== seen) {
|
|
489
489
|
// `<createdAt>:<fsResets>`: another reset count means the files were reset since; another
|
|
490
490
|
// instance under the same count is a restart on kept files
|
|
@@ -498,7 +498,7 @@ let SandboxService = class SandboxService {
|
|
|
498
498
|
}
|
|
499
499
|
catch (err) {
|
|
500
500
|
this.log.warn("cannot record the sandbox instance a holder uses", {
|
|
501
|
-
|
|
501
|
+
deviceId,
|
|
502
502
|
holder,
|
|
503
503
|
err,
|
|
504
504
|
});
|
|
@@ -508,7 +508,7 @@ let SandboxService = class SandboxService {
|
|
|
508
508
|
/** Runs `fn` as the holder `e`: its tag and the caller's actor on every exec, its lease's signal. */
|
|
509
509
|
runAs(e, ctx, p, fn, signal) {
|
|
510
510
|
const sig = AbortSignal.any([signal, e.lease.signal]);
|
|
511
|
-
const env = ctx.
|
|
511
|
+
const env = ctx.deviceId;
|
|
512
512
|
const impl = tagged(p.impl, e.lease.token, ctx.actor, () => this.register(env, e.lease.token));
|
|
513
513
|
return fn(e.handle, impl, { ...e.info, roots: rootsOf(e.handle) }, sig);
|
|
514
514
|
}
|
|
@@ -519,7 +519,7 @@ let SandboxService = class SandboxService {
|
|
|
519
519
|
* downgraded. One token serves both, so the waiter passes its own intent.
|
|
520
520
|
*/
|
|
521
521
|
async enter(ctx, p, signal, gone) {
|
|
522
|
-
const env = ctx.
|
|
522
|
+
const env = ctx.deviceId;
|
|
523
523
|
const deadline = Date.now() + this.config.get().sandbox.lockWaitSeconds * 1000;
|
|
524
524
|
const token = randomUUID();
|
|
525
525
|
let intent = false;
|
|
@@ -559,13 +559,13 @@ let SandboxService = class SandboxService {
|
|
|
559
559
|
async clearIntent(env, token) {
|
|
560
560
|
await this.leases
|
|
561
561
|
.clearIntent(env, token)
|
|
562
|
-
.catch((err) => this.log.warn("cannot clear sandbox intent", {
|
|
562
|
+
.catch((err) => this.log.warn("cannot clear sandbox intent", { deviceId: env, err }));
|
|
563
563
|
}
|
|
564
564
|
async readRow(ctx) {
|
|
565
565
|
const row = await this.db
|
|
566
|
-
.selectFrom("
|
|
566
|
+
.selectFrom("device_instances")
|
|
567
567
|
.selectAll()
|
|
568
|
-
.where("
|
|
568
|
+
.where("device_id", "=", ctx.deviceId)
|
|
569
569
|
.executeTakeFirst();
|
|
570
570
|
if (row && row.org_id !== ctx.orgId)
|
|
571
571
|
throw this.foreign(ctx);
|
|
@@ -573,7 +573,7 @@ let SandboxService = class SandboxService {
|
|
|
573
573
|
}
|
|
574
574
|
/** The instance row belongs to another org: reported as not found, so ids cannot be probed. */
|
|
575
575
|
foreign(_ctx) {
|
|
576
|
-
return
|
|
576
|
+
return deviceNotFound();
|
|
577
577
|
}
|
|
578
578
|
/**
|
|
579
579
|
* Recorded running with the current provider on files not lost (lost ones go through the
|
|
@@ -592,7 +592,7 @@ let SandboxService = class SandboxService {
|
|
|
592
592
|
* returns null), records the holder and kills crashed holders' commands.
|
|
593
593
|
*/
|
|
594
594
|
async asShared(ctx, p, lease, accept, signal) {
|
|
595
|
-
const env = ctx.
|
|
595
|
+
const env = ctx.deviceId;
|
|
596
596
|
try {
|
|
597
597
|
const row = await this.readRow(ctx);
|
|
598
598
|
if (!accept(row)) {
|
|
@@ -603,12 +603,12 @@ let SandboxService = class SandboxService {
|
|
|
603
603
|
// expired unnoticed (a stalled sweep) then fails its next write instead of suspending or
|
|
604
604
|
// destroying the instance under this holder. Shared holders never compare it.
|
|
605
605
|
await this.db
|
|
606
|
-
.updateTable("
|
|
606
|
+
.updateTable("device_instances")
|
|
607
607
|
.set((eb) => ({ generation: eb("generation", "+", 1) }))
|
|
608
|
-
.where("
|
|
608
|
+
.where("device_id", "=", env)
|
|
609
609
|
.execute();
|
|
610
610
|
await this.register(env, lease.token);
|
|
611
|
-
// other holders (any conversation of the
|
|
611
|
+
// other holders (any conversation of the device) may be running: only the dead
|
|
612
612
|
// holders' own commands are killed
|
|
613
613
|
await this.killTagged(env, row.handle, p, await this.claim(env), signal, lease);
|
|
614
614
|
return {
|
|
@@ -633,7 +633,7 @@ let SandboxService = class SandboxService {
|
|
|
633
633
|
* on a new instance there is nothing to kill, and on one that already ran their tags are killed.
|
|
634
634
|
*/
|
|
635
635
|
async asExclusive(ctx, p, lease, gone, signal) {
|
|
636
|
-
const env = ctx.
|
|
636
|
+
const env = ctx.deviceId;
|
|
637
637
|
const sig = AbortSignal.any([signal, lease.signal]);
|
|
638
638
|
try {
|
|
639
639
|
const s = await this.ensure(ctx, p, lease, sig, gone);
|
|
@@ -667,23 +667,23 @@ let SandboxService = class SandboxService {
|
|
|
667
667
|
await this.unregister(env, e.lease.token);
|
|
668
668
|
await e.lease.release();
|
|
669
669
|
await this.db
|
|
670
|
-
.updateTable("
|
|
670
|
+
.updateTable("device_instances")
|
|
671
671
|
.set({ last_used_at: sql `GREATEST(last_used_at, now())` })
|
|
672
|
-
.where("
|
|
672
|
+
.where("device_id", "=", env)
|
|
673
673
|
.where("status", "=", "running")
|
|
674
674
|
.execute()
|
|
675
|
-
.catch((err) => this.log.warn("cannot update sandbox last_used_at", {
|
|
675
|
+
.catch((err) => this.log.warn("cannot update sandbox last_used_at", { deviceId: env, err }));
|
|
676
676
|
}
|
|
677
677
|
async register(env, token) {
|
|
678
678
|
const ttl = this.config.get().sandbox.limits.maxCommandSeconds * 1000 + HOLDER_TTL_MARGIN_MS;
|
|
679
679
|
await this.leases
|
|
680
680
|
.registerHolder(env, token, ttl)
|
|
681
|
-
.catch((err) => this.log.warn("cannot record sandbox holder", {
|
|
681
|
+
.catch((err) => this.log.warn("cannot record sandbox holder", { deviceId: env, err }));
|
|
682
682
|
}
|
|
683
683
|
async unregister(env, token) {
|
|
684
684
|
await this.leases
|
|
685
685
|
.unregisterHolder(env, token)
|
|
686
|
-
.catch((err) => this.log.warn("cannot clear sandbox holder record", {
|
|
686
|
+
.catch((err) => this.log.warn("cannot clear sandbox holder record", { deviceId: env, err }));
|
|
687
687
|
}
|
|
688
688
|
/** Crashed holders' tags (claimed); a Redis error counts as none. */
|
|
689
689
|
async claim(env) {
|
|
@@ -691,7 +691,7 @@ let SandboxService = class SandboxService {
|
|
|
691
691
|
return await this.leases.claimDeadHolders(env);
|
|
692
692
|
}
|
|
693
693
|
catch (err) {
|
|
694
|
-
this.log.warn("cannot read sandbox holder records", {
|
|
694
|
+
this.log.warn("cannot read sandbox holder records", { deviceId: env, err });
|
|
695
695
|
return [];
|
|
696
696
|
}
|
|
697
697
|
}
|
|
@@ -703,27 +703,27 @@ let SandboxService = class SandboxService {
|
|
|
703
703
|
await this.kill(env, "takeover", () => p.impl.killTagged(handle, tags, sig), sig);
|
|
704
704
|
}
|
|
705
705
|
/** Runs one kill call: gone and abort are thrown, other failures only warned. */
|
|
706
|
-
async kill(
|
|
706
|
+
async kill(deviceId, why, call, signal) {
|
|
707
707
|
try {
|
|
708
708
|
await call();
|
|
709
709
|
if (why === "takeover") {
|
|
710
|
-
this.log.info("sandbox taken over from a holder that did not finish; leftover commands killed", {
|
|
710
|
+
this.log.info("sandbox taken over from a holder that did not finish; leftover commands killed", { deviceId });
|
|
711
711
|
}
|
|
712
712
|
}
|
|
713
713
|
catch (err) {
|
|
714
714
|
if (signal?.aborted || isSandboxGoneError(err))
|
|
715
715
|
throw err;
|
|
716
|
-
this.log.warn("cannot kill leftover sandbox commands", {
|
|
716
|
+
this.log.warn("cannot kill leftover sandbox commands", { deviceId, err });
|
|
717
717
|
}
|
|
718
718
|
}
|
|
719
719
|
/**
|
|
720
720
|
* Suspends running instances idle longer than `suspendAfterIdleMinutes` or running continuously
|
|
721
|
-
* longer than `maxRunningMinutes`. Skips
|
|
721
|
+
* longer than `maxRunningMinutes`. Skips a device with any holder; one past
|
|
722
722
|
* `maxRunningMinutes` gets the intent for one poll window (about a second) first. Returns how
|
|
723
723
|
* many it suspended. Called every minute by the `sandbox-suspend-idle` cron.
|
|
724
724
|
*/
|
|
725
725
|
async suspendIdle(now) {
|
|
726
|
-
// cloud only: suspendDue keeps to this provider's rows, so `@
|
|
726
|
+
// cloud only: suspendDue keeps to this provider's rows, so `@local` is never suspended here
|
|
727
727
|
const p = this.providers.cloud();
|
|
728
728
|
if (!p)
|
|
729
729
|
return 0;
|
|
@@ -733,12 +733,12 @@ let SandboxService = class SandboxService {
|
|
|
733
733
|
let n = 0;
|
|
734
734
|
await inPool(rows, SWEEP_CONCURRENCY, async (c) => {
|
|
735
735
|
const overdue = c.running_since !== null && c.running_since < runningBefore;
|
|
736
|
-
const done = await this.tryLocked(c.
|
|
737
|
-
const id = c.
|
|
736
|
+
const done = await this.tryLocked(c.device_id, async (lease) => {
|
|
737
|
+
const id = c.device_id;
|
|
738
738
|
const row = await this.db
|
|
739
|
-
.selectFrom("
|
|
739
|
+
.selectFrom("device_instances")
|
|
740
740
|
.selectAll()
|
|
741
|
-
.where("
|
|
741
|
+
.where("device_id", "=", id)
|
|
742
742
|
.where(due)
|
|
743
743
|
.executeTakeFirst();
|
|
744
744
|
if (!row?.handle)
|
|
@@ -753,7 +753,7 @@ let SandboxService = class SandboxService {
|
|
|
753
753
|
running_since: null,
|
|
754
754
|
sweep_failed_at: null,
|
|
755
755
|
});
|
|
756
|
-
this.log.info("sandbox suspended", {
|
|
756
|
+
this.log.info("sandbox suspended", { deviceId: id });
|
|
757
757
|
return true;
|
|
758
758
|
}, overdue);
|
|
759
759
|
if (done)
|
|
@@ -764,53 +764,53 @@ let SandboxService = class SandboxService {
|
|
|
764
764
|
/**
|
|
765
765
|
* Destruction sweep: an instance unused for longer than `destroyAfterInactiveDays` is destroyed
|
|
766
766
|
* (the instance only; the file system is kept and the next use builds a new instance on it); an
|
|
767
|
-
*
|
|
768
|
-
* and its row is deleted (as {@link
|
|
769
|
-
* skipped. Inactivity never destroys a `@
|
|
770
|
-
* destroyed like any other while `@
|
|
767
|
+
* device that was deleted, or whose org was deleted, loses its instance and its file system
|
|
768
|
+
* and its row is deleted (as {@link destroyDevice} does). Devices with any holder are
|
|
769
|
+
* skipped. Inactivity never destroys a `@local` instance; a gone `@local` device is
|
|
770
|
+
* destroyed like any other while `@local` is registered. Returns how many it destroyed. Called
|
|
771
771
|
* hourly by the `sandbox-destroy-inactive` cron.
|
|
772
772
|
*/
|
|
773
773
|
async destroyInactive(now) {
|
|
774
774
|
const p = this.providers.cloud();
|
|
775
|
-
const
|
|
775
|
+
const local = this.providers.forName(LOCAL_PROVIDER) !== undefined;
|
|
776
776
|
let n = 0;
|
|
777
|
-
// Gone
|
|
777
|
+
// Gone devices (deleted, or of a deleted org) on every provider that is here, `@local`
|
|
778
778
|
// too (its destroy calls never need the machine online); rows whose provider is missing wait.
|
|
779
779
|
const here = (eb) => eb.or([
|
|
780
780
|
...(p ? [this.cloudRow(eb)] : []),
|
|
781
|
-
...(
|
|
781
|
+
...(local ? [eb("provider", "=", LOCAL_PROVIDER)] : []),
|
|
782
782
|
]);
|
|
783
|
-
const gone = (eb) => eb.and([here(eb), this.
|
|
784
|
-
if (p ||
|
|
785
|
-
await inPool(await this.candidates(gone), SWEEP_CONCURRENCY, async ({
|
|
783
|
+
const gone = (eb) => eb.and([here(eb), this.deviceGone(eb)]);
|
|
784
|
+
if (p || local) {
|
|
785
|
+
await inPool(await this.candidates(gone), SWEEP_CONCURRENCY, async ({ device_id: id }) => {
|
|
786
786
|
const done = await this.tryLocked(id, async (lease) => {
|
|
787
787
|
const row = await this.db
|
|
788
|
-
.selectFrom("
|
|
788
|
+
.selectFrom("device_instances")
|
|
789
789
|
.selectAll()
|
|
790
|
-
.where("
|
|
790
|
+
.where("device_id", "=", id)
|
|
791
791
|
.where(gone)
|
|
792
792
|
.executeTakeFirst();
|
|
793
793
|
const owner = row && this.rowProvider(row);
|
|
794
794
|
if (!row || !owner)
|
|
795
795
|
return false;
|
|
796
|
-
await this.destroyRow(row, owner, lease, "
|
|
796
|
+
await this.destroyRow(row, owner, lease, "device");
|
|
797
797
|
return true;
|
|
798
798
|
});
|
|
799
799
|
if (done)
|
|
800
800
|
n++;
|
|
801
801
|
});
|
|
802
802
|
}
|
|
803
|
-
// inactivity destroys cloud instances only: `@
|
|
803
|
+
// inactivity destroys cloud instances only: `@local` instances are never swept
|
|
804
804
|
if (!p)
|
|
805
805
|
return n;
|
|
806
806
|
const due = this.instanceDue(now);
|
|
807
807
|
const idle = (eb) => eb.and([this.cloudRow(eb), due(eb)]);
|
|
808
|
-
await inPool(await this.candidates(idle), SWEEP_CONCURRENCY, async ({
|
|
808
|
+
await inPool(await this.candidates(idle), SWEEP_CONCURRENCY, async ({ device_id: id }) => {
|
|
809
809
|
const done = await this.tryLocked(id, async (lease) => {
|
|
810
810
|
const row = await this.db
|
|
811
|
-
.selectFrom("
|
|
811
|
+
.selectFrom("device_instances")
|
|
812
812
|
.selectAll()
|
|
813
|
-
.where("
|
|
813
|
+
.where("device_id", "=", id)
|
|
814
814
|
.where(idle)
|
|
815
815
|
.executeTakeFirst();
|
|
816
816
|
if (!row)
|
|
@@ -824,60 +824,60 @@ let SandboxService = class SandboxService {
|
|
|
824
824
|
return n;
|
|
825
825
|
}
|
|
826
826
|
/**
|
|
827
|
-
* Destroys the
|
|
828
|
-
* (
|
|
827
|
+
* Destroys the device's instance, then its file system, then deletes its instance row
|
|
828
|
+
* (device deletion, offboarding); nothing to do without a row. Waits for the exclusive lease
|
|
829
829
|
* (with intent, so running commands finish and new ones wait) up to 10 minutes. Without the
|
|
830
|
-
*
|
|
831
|
-
* retried, and the destruction sweep finishes a deleted
|
|
830
|
+
* device's provider it throws and the row stays, marked `sweep_failed_at` (the job is
|
|
831
|
+
* retried, and the destruction sweep finishes a deleted device once the provider is back).
|
|
832
832
|
* Files kept by another provider that is not loaded cannot be destroyed: it throws and the row
|
|
833
833
|
* stays.
|
|
834
834
|
*/
|
|
835
|
-
async
|
|
835
|
+
async destroyDevice(deviceId) {
|
|
836
836
|
const row = await this.db
|
|
837
|
-
.selectFrom("
|
|
838
|
-
.select("
|
|
839
|
-
.where("
|
|
837
|
+
.selectFrom("device_instances")
|
|
838
|
+
.select("device_id")
|
|
839
|
+
.where("device_id", "=", deviceId)
|
|
840
840
|
.executeTakeFirst();
|
|
841
841
|
if (!row)
|
|
842
842
|
return;
|
|
843
|
-
const p = await this.providerOf(
|
|
843
|
+
const p = await this.providerOf(deviceId);
|
|
844
844
|
if (!p) {
|
|
845
845
|
await this.db
|
|
846
|
-
.updateTable("
|
|
846
|
+
.updateTable("device_instances")
|
|
847
847
|
.set({ sweep_failed_at: sql `now()` })
|
|
848
|
-
.where("
|
|
848
|
+
.where("device_id", "=", deviceId)
|
|
849
849
|
.execute();
|
|
850
|
-
throw new Error(`cannot destroy
|
|
850
|
+
throw new Error(`cannot destroy device ${deviceId}: its sandbox provider is not there; ` +
|
|
851
851
|
"the destruction sweep retries it");
|
|
852
852
|
}
|
|
853
|
-
const lease = await this.leases.acquireExclusive(
|
|
853
|
+
const lease = await this.leases.acquireExclusive(deviceId, AbortSignal.timeout(EXCLUSIVE_WAIT_MS), undefined, { intent: true });
|
|
854
854
|
try {
|
|
855
855
|
const current = await this.db
|
|
856
|
-
.selectFrom("
|
|
856
|
+
.selectFrom("device_instances")
|
|
857
857
|
.selectAll()
|
|
858
|
-
.where("
|
|
858
|
+
.where("device_id", "=", deviceId)
|
|
859
859
|
.executeTakeFirst();
|
|
860
860
|
if (current)
|
|
861
|
-
await this.destroyRow(current, p, lease, "
|
|
861
|
+
await this.destroyRow(current, p, lease, "device");
|
|
862
862
|
}
|
|
863
863
|
finally {
|
|
864
864
|
await lease.release();
|
|
865
865
|
}
|
|
866
866
|
}
|
|
867
867
|
/**
|
|
868
|
-
* Starts the
|
|
869
|
-
* under the exclusive lease (waited for like {@link
|
|
868
|
+
* Starts the device over with empty files (after `device.files_lost`, or on request):
|
|
869
|
+
* under the exclusive lease (waited for like {@link destroyDevice}) destroys any instance and
|
|
870
870
|
* the file system (a `lost` one is only dropped), and records no file system; the next use
|
|
871
871
|
* creates a new one. Nothing to do without a row. Throws `sandbox.unavailable` without a provider.
|
|
872
872
|
*/
|
|
873
|
-
async resetFileSystem(
|
|
874
|
-
const p = await this.require(
|
|
875
|
-
const lease = await this.leases.acquireExclusive(
|
|
873
|
+
async resetFileSystem(deviceId) {
|
|
874
|
+
const p = await this.require(deviceId);
|
|
875
|
+
const lease = await this.leases.acquireExclusive(deviceId, AbortSignal.timeout(EXCLUSIVE_WAIT_MS), undefined, { intent: true });
|
|
876
876
|
try {
|
|
877
877
|
const row = await this.db
|
|
878
|
-
.selectFrom("
|
|
878
|
+
.selectFrom("device_instances")
|
|
879
879
|
.selectAll()
|
|
880
|
-
.where("
|
|
880
|
+
.where("device_id", "=", deviceId)
|
|
881
881
|
.executeTakeFirst();
|
|
882
882
|
if (!row)
|
|
883
883
|
return;
|
|
@@ -886,8 +886,8 @@ let SandboxService = class SandboxService {
|
|
|
886
886
|
const prev = await this.destroyInstance(row, p, signal);
|
|
887
887
|
await this.destroyFiles(row, p, signal, true);
|
|
888
888
|
if (!(await lease.renew()))
|
|
889
|
-
throw new SandboxLockLostError(
|
|
890
|
-
await this.fenced(lease,
|
|
889
|
+
throw new SandboxLockLostError(deviceId);
|
|
890
|
+
await this.fenced(lease, deviceId, fence, {
|
|
891
891
|
fs_handle: null,
|
|
892
892
|
fs_status: "none",
|
|
893
893
|
handle: null,
|
|
@@ -900,7 +900,7 @@ let SandboxService = class SandboxService {
|
|
|
900
900
|
fs_resets: sql `fs_resets + 1`,
|
|
901
901
|
...prev,
|
|
902
902
|
});
|
|
903
|
-
this.log.info("
|
|
903
|
+
this.log.info("device files reset", { deviceId });
|
|
904
904
|
}
|
|
905
905
|
finally {
|
|
906
906
|
await lease.release();
|
|
@@ -913,8 +913,8 @@ let SandboxService = class SandboxService {
|
|
|
913
913
|
* instance (`SandboxGoneError`) and an abort are thrown (`withSandbox` rebuilds / ends); other
|
|
914
914
|
* failures are only warned.
|
|
915
915
|
*/
|
|
916
|
-
async killLeftovers(
|
|
917
|
-
await this.kill(
|
|
916
|
+
async killLeftovers(deviceId, handle, p, signal, dead) {
|
|
917
|
+
await this.kill(deviceId, dead.length > 0 ? "takeover" : "resume", () => p.impl.killAll(handle, signal), signal);
|
|
918
918
|
}
|
|
919
919
|
/**
|
|
920
920
|
* Under the exclusive lease, right after a resume or a (re)build: kills the sessions of the
|
|
@@ -923,8 +923,8 @@ let SandboxService = class SandboxService {
|
|
|
923
923
|
* here and are only cleared. A gone instance and an abort are thrown; another failure is warned
|
|
924
924
|
* and leaves the flags of this instance's tasks for the next resume.
|
|
925
925
|
*/
|
|
926
|
-
async killPendingTasks(
|
|
927
|
-
const rows = await this.background.pendingKills(
|
|
926
|
+
async killPendingTasks(deviceId, handle, createdAt, p, tag, actor, signal) {
|
|
927
|
+
const rows = await this.background.pendingKills(deviceId);
|
|
928
928
|
if (rows.length === 0)
|
|
929
929
|
return;
|
|
930
930
|
const mine = rows.filter((r) => r.sandboxCreatedAt?.getTime() === createdAt && r.sid !== null && SID_RE.test(r.sid));
|
|
@@ -945,7 +945,7 @@ let SandboxService = class SandboxService {
|
|
|
945
945
|
throw new Error(`pkill exited with ${r.exitCode === null ? "no status" : r.exitCode}: ${r.stderr.slice(0, 500)}`);
|
|
946
946
|
}
|
|
947
947
|
this.log.info("lost background tasks killed after a sandbox resume", {
|
|
948
|
-
|
|
948
|
+
deviceId,
|
|
949
949
|
tasks: mine.map((t) => t.id),
|
|
950
950
|
});
|
|
951
951
|
}
|
|
@@ -953,7 +953,7 @@ let SandboxService = class SandboxService {
|
|
|
953
953
|
if (signal.aborted || isSandboxGoneError(err))
|
|
954
954
|
throw err;
|
|
955
955
|
this.log.warn("cannot kill lost background tasks after a sandbox resume", {
|
|
956
|
-
|
|
956
|
+
deviceId,
|
|
957
957
|
err,
|
|
958
958
|
});
|
|
959
959
|
const kept = new Set(mine.map((r) => r.id));
|
|
@@ -970,9 +970,9 @@ let SandboxService = class SandboxService {
|
|
|
970
970
|
if (lease.signal.aborted)
|
|
971
971
|
throw new SandboxLockLostError(id);
|
|
972
972
|
const r = await this.db
|
|
973
|
-
.updateTable("
|
|
973
|
+
.updateTable("device_instances")
|
|
974
974
|
.set({ ...set, generation: fence.gen + 1 })
|
|
975
|
-
.where("
|
|
975
|
+
.where("device_id", "=", id)
|
|
976
976
|
.where("generation", "=", fence.gen)
|
|
977
977
|
.returning("created_at")
|
|
978
978
|
.executeTakeFirst();
|
|
@@ -984,8 +984,8 @@ let SandboxService = class SandboxService {
|
|
|
984
984
|
/** Sweep candidates: those whose last sweep failed come last, the rest least recently used first. */
|
|
985
985
|
async candidates(due) {
|
|
986
986
|
return await this.db
|
|
987
|
-
.selectFrom("
|
|
988
|
-
.select(["
|
|
987
|
+
.selectFrom("device_instances")
|
|
988
|
+
.select(["device_id", "running_since"])
|
|
989
989
|
.where(due)
|
|
990
990
|
.orderBy(sql `sweep_failed_at ASC NULLS FIRST`)
|
|
991
991
|
.orderBy("last_used_at")
|
|
@@ -994,17 +994,17 @@ let SandboxService = class SandboxService {
|
|
|
994
994
|
}
|
|
995
995
|
/**
|
|
996
996
|
* One try at the exclusive lease: null → false; else runs `fn`. With `overdue` (past
|
|
997
|
-
* `maxRunningMinutes`) a busy
|
|
997
|
+
* `maxRunningMinutes`) a busy device gets the intent for about a second, so new commands
|
|
998
998
|
* wait briefly and the sweep gets in if the running ones end. Failures are warned, recorded in
|
|
999
999
|
* `sweep_failed_at` (sorted last next time) and return false.
|
|
1000
1000
|
*/
|
|
1001
|
-
async tryLocked(
|
|
1001
|
+
async tryLocked(deviceId, fn, overdue = false) {
|
|
1002
1002
|
let lease;
|
|
1003
1003
|
try {
|
|
1004
|
-
lease = await this.leases.tryExclusive(
|
|
1004
|
+
lease = await this.leases.tryExclusive(deviceId);
|
|
1005
1005
|
if (!lease && overdue) {
|
|
1006
1006
|
lease = await this.leases
|
|
1007
|
-
.acquireExclusive(
|
|
1007
|
+
.acquireExclusive(deviceId, AbortSignal.timeout(SWEEP_OP_TIMEOUT_MS), Math.max(this.pollMs * 2, SWEEP_INTENT_WAIT_MS), { intent: true })
|
|
1008
1008
|
.catch((err) => {
|
|
1009
1009
|
if (err instanceof ApiError && err.code === "sandbox.busy")
|
|
1010
1010
|
return null;
|
|
@@ -1013,7 +1013,7 @@ let SandboxService = class SandboxService {
|
|
|
1013
1013
|
}
|
|
1014
1014
|
}
|
|
1015
1015
|
catch (err) {
|
|
1016
|
-
this.log.warn("cannot take sandbox lock", {
|
|
1016
|
+
this.log.warn("cannot take sandbox lock", { deviceId, err });
|
|
1017
1017
|
return false;
|
|
1018
1018
|
}
|
|
1019
1019
|
if (!lease)
|
|
@@ -1022,11 +1022,11 @@ let SandboxService = class SandboxService {
|
|
|
1022
1022
|
return await fn(lease);
|
|
1023
1023
|
}
|
|
1024
1024
|
catch (err) {
|
|
1025
|
-
this.log.warn("sandbox maintenance failed; will retry", {
|
|
1025
|
+
this.log.warn("sandbox maintenance failed; will retry", { deviceId, err });
|
|
1026
1026
|
await this.db
|
|
1027
|
-
.updateTable("
|
|
1027
|
+
.updateTable("device_instances")
|
|
1028
1028
|
.set({ sweep_failed_at: sql `now()` })
|
|
1029
|
-
.where("
|
|
1029
|
+
.where("device_id", "=", deviceId)
|
|
1030
1030
|
.execute()
|
|
1031
1031
|
.catch(() => { });
|
|
1032
1032
|
return false;
|
|
@@ -1036,48 +1036,48 @@ let SandboxService = class SandboxService {
|
|
|
1036
1036
|
}
|
|
1037
1037
|
}
|
|
1038
1038
|
/**
|
|
1039
|
-
* Under the exclusive lease: make the
|
|
1039
|
+
* Under the exclusive lease: make the device's instance run. `gone`: the instance a holder
|
|
1040
1040
|
* found gone; when it is still the recorded one it is rebuilt, when another holder already
|
|
1041
1041
|
* rebuilt it the new one is used.
|
|
1042
1042
|
*/
|
|
1043
1043
|
async ensure(ctx, p, held, signal, gone) {
|
|
1044
|
-
const id = ctx.
|
|
1045
|
-
// Only a live
|
|
1044
|
+
const id = ctx.deviceId;
|
|
1045
|
+
// Only a live device of the caller's org gets a row, with the device's own org; a
|
|
1046
1046
|
// row made earlier (another use) keeps its file system handle.
|
|
1047
1047
|
await this.db
|
|
1048
|
-
.insertInto("
|
|
1049
|
-
.columns(["
|
|
1048
|
+
.insertInto("device_instances")
|
|
1049
|
+
.columns(["device_id", "org_id", "provider", "status"])
|
|
1050
1050
|
.expression((eb) => eb
|
|
1051
|
-
.selectFrom("
|
|
1051
|
+
.selectFrom("devices")
|
|
1052
1052
|
.select((e) => [
|
|
1053
|
-
"
|
|
1054
|
-
"
|
|
1053
|
+
"devices.id",
|
|
1054
|
+
"devices.org_id",
|
|
1055
1055
|
e.val(p.plugin).as("provider"),
|
|
1056
1056
|
e.val("creating").as("status"),
|
|
1057
1057
|
])
|
|
1058
|
-
.where("
|
|
1059
|
-
.where("
|
|
1060
|
-
.where("
|
|
1061
|
-
.onConflict((oc) => oc.column("
|
|
1058
|
+
.where("devices.id", "=", id)
|
|
1059
|
+
.where("devices.org_id", "=", ctx.orgId)
|
|
1060
|
+
.where("devices.deleted_at", "is", null))
|
|
1061
|
+
.onConflict((oc) => oc.column("device_id").doNothing())
|
|
1062
1062
|
.execute();
|
|
1063
|
-
// Under the lease: a use that waited behind
|
|
1064
|
-
// bring the
|
|
1063
|
+
// Under the lease: a use that waited behind destroyDevice (or raced a deletion) must not
|
|
1064
|
+
// bring the device's provider resources back.
|
|
1065
1065
|
const live = await this.db
|
|
1066
|
-
.selectFrom("
|
|
1066
|
+
.selectFrom("devices")
|
|
1067
1067
|
.select("id")
|
|
1068
1068
|
.where("id", "=", id)
|
|
1069
1069
|
.where("org_id", "=", ctx.orgId)
|
|
1070
1070
|
.where("deleted_at", "is", null)
|
|
1071
1071
|
.executeTakeFirst();
|
|
1072
1072
|
if (!live)
|
|
1073
|
-
throw
|
|
1073
|
+
throw deviceNotFound();
|
|
1074
1074
|
const row = await this.db
|
|
1075
|
-
.selectFrom("
|
|
1075
|
+
.selectFrom("device_instances")
|
|
1076
1076
|
.selectAll()
|
|
1077
|
-
.where("
|
|
1077
|
+
.where("device_id", "=", id)
|
|
1078
1078
|
.executeTakeFirst();
|
|
1079
1079
|
if (!row)
|
|
1080
|
-
throw
|
|
1080
|
+
throw deviceNotFound();
|
|
1081
1081
|
if (row.org_id !== ctx.orgId)
|
|
1082
1082
|
throw this.foreign(ctx);
|
|
1083
1083
|
const fence = { gen: row.generation };
|
|
@@ -1085,7 +1085,7 @@ let SandboxService = class SandboxService {
|
|
|
1085
1085
|
// kept running (a stalled sweep) then fails its next write.
|
|
1086
1086
|
await this.fenced(held, id, fence, {});
|
|
1087
1087
|
if (row.fs_status === "lost")
|
|
1088
|
-
throw
|
|
1088
|
+
throw deviceFilesLost(p.plugin === LOCAL_PROVIDER);
|
|
1089
1089
|
if (row.provider !== null &&
|
|
1090
1090
|
row.provider !== p.plugin &&
|
|
1091
1091
|
(row.fs_status !== "none" || row.handle !== null)) {
|
|
@@ -1128,7 +1128,7 @@ let SandboxService = class SandboxService {
|
|
|
1128
1128
|
};
|
|
1129
1129
|
}
|
|
1130
1130
|
this.log.info("sandbox instance is gone at the provider; recreating", {
|
|
1131
|
-
|
|
1131
|
+
deviceId: id,
|
|
1132
1132
|
});
|
|
1133
1133
|
}
|
|
1134
1134
|
const created = await this.create(ctx, p, held, row, fence, signal);
|
|
@@ -1145,12 +1145,12 @@ let SandboxService = class SandboxService {
|
|
|
1145
1145
|
* Under the lease: fenced reset to `creating`, the file system when there is none
|
|
1146
1146
|
* (`createFileSystem`, its handle written fenced), then the instance on it, then a fenced write
|
|
1147
1147
|
* of the instance handle. `FileSystemGoneError` from `create` records `fs_status = 'lost'` and
|
|
1148
|
-
* throws `
|
|
1148
|
+
* throws `device.files_lost`. When the instance handle cannot be written (lease lost), the
|
|
1149
1149
|
* new instance is destroyed (best effort) before {@link SandboxLockLostError} is thrown.
|
|
1150
1150
|
*/
|
|
1151
1151
|
async create(ctx, p, held, row, fence, signal) {
|
|
1152
|
-
const id = ctx.
|
|
1153
|
-
const name =
|
|
1152
|
+
const id = ctx.deviceId;
|
|
1153
|
+
const name = deviceResourceName(id);
|
|
1154
1154
|
await this.fenced(held, id, fence, {
|
|
1155
1155
|
org_id: ctx.orgId,
|
|
1156
1156
|
provider: p.plugin,
|
|
@@ -1178,7 +1178,7 @@ let SandboxService = class SandboxService {
|
|
|
1178
1178
|
fs_handle: JSON.stringify(fs),
|
|
1179
1179
|
fs_status: "ready",
|
|
1180
1180
|
});
|
|
1181
|
-
this.log.info("
|
|
1181
|
+
this.log.info("device file system created", { deviceId: id, provider: p.plugin });
|
|
1182
1182
|
}
|
|
1183
1183
|
let handle;
|
|
1184
1184
|
try {
|
|
@@ -1193,12 +1193,12 @@ let SandboxService = class SandboxService {
|
|
|
1193
1193
|
catch (err) {
|
|
1194
1194
|
if (!isFileSystemGoneError(err))
|
|
1195
1195
|
throw err;
|
|
1196
|
-
this.log.warn("
|
|
1197
|
-
|
|
1196
|
+
this.log.warn("device file system is gone at the provider; its files are lost", {
|
|
1197
|
+
deviceId: id,
|
|
1198
1198
|
provider: p.plugin,
|
|
1199
1199
|
});
|
|
1200
1200
|
await this.fenced(held, id, fence, { fs_status: "lost", status: "none" });
|
|
1201
|
-
throw
|
|
1201
|
+
throw deviceFilesLost(p.plugin === LOCAL_PROVIDER);
|
|
1202
1202
|
}
|
|
1203
1203
|
let createdAt;
|
|
1204
1204
|
try {
|
|
@@ -1218,34 +1218,34 @@ let SandboxService = class SandboxService {
|
|
|
1218
1218
|
// SandboxGoneError (or resume returns gone) and withSandbox rebuilds in place. The file
|
|
1219
1219
|
// system is never touched here.
|
|
1220
1220
|
await p.impl.destroy(handle, AbortSignal.timeout(SWEEP_OP_TIMEOUT_MS)).catch((err) => this.log.warn("cannot destroy sandbox created after the lock was lost", {
|
|
1221
|
-
|
|
1221
|
+
deviceId: id,
|
|
1222
1222
|
err,
|
|
1223
1223
|
}));
|
|
1224
1224
|
throw e;
|
|
1225
1225
|
}
|
|
1226
|
-
this.log.info("sandbox created", {
|
|
1226
|
+
this.log.info("sandbox created", { deviceId: id, provider: p.plugin });
|
|
1227
1227
|
return { handle, createdAt: createdAt.getTime() };
|
|
1228
1228
|
}
|
|
1229
1229
|
/**
|
|
1230
|
-
* Destroys the file system another provider may have made under the
|
|
1230
|
+
* Destroys the file system another provider may have made under the device's name without
|
|
1231
1231
|
* the kernel recording it (`destroyFileSystemByName`; not there is success). That provider not
|
|
1232
|
-
* loaded, or a failure, is only warned (the files may leak there) so the
|
|
1232
|
+
* loaded, or a failure, is only warned (the files may leak there) so the device stays
|
|
1233
1233
|
* usable; an abort is thrown.
|
|
1234
1234
|
*/
|
|
1235
1235
|
async dropUnrecordedFiles(id, provider, signal) {
|
|
1236
1236
|
const old = this.providers.forName(provider)?.impl;
|
|
1237
1237
|
if (!old) {
|
|
1238
|
-
this.log.warn("
|
|
1238
|
+
this.log.warn("device file system may have been made by a provider that is not loaded; it may leak", { deviceId: id, provider });
|
|
1239
1239
|
return;
|
|
1240
1240
|
}
|
|
1241
1241
|
try {
|
|
1242
|
-
await old.destroyFileSystemByName(
|
|
1242
|
+
await old.destroyFileSystemByName(deviceResourceName(id), signal);
|
|
1243
1243
|
}
|
|
1244
1244
|
catch (err) {
|
|
1245
1245
|
if (signal.aborted)
|
|
1246
1246
|
throw err;
|
|
1247
1247
|
this.log.warn("cannot destroy the file system of the previous sandbox provider", {
|
|
1248
|
-
|
|
1248
|
+
deviceId: id,
|
|
1249
1249
|
provider,
|
|
1250
1250
|
err,
|
|
1251
1251
|
});
|
|
@@ -1264,14 +1264,14 @@ let SandboxService = class SandboxService {
|
|
|
1264
1264
|
}
|
|
1265
1265
|
catch (err) {
|
|
1266
1266
|
this.log.warn("cannot destroy sandbox instance of the previous provider", {
|
|
1267
|
-
|
|
1267
|
+
deviceId: id,
|
|
1268
1268
|
provider,
|
|
1269
1269
|
err,
|
|
1270
1270
|
});
|
|
1271
1271
|
return true;
|
|
1272
1272
|
}
|
|
1273
1273
|
}
|
|
1274
|
-
this.log.warn("sandbox was created by another provider that is not loaded; its instance may leak", {
|
|
1274
|
+
this.log.warn("sandbox was created by another provider that is not loaded; its instance may leak", { deviceId: id, provider });
|
|
1275
1275
|
return true;
|
|
1276
1276
|
}
|
|
1277
1277
|
/**
|
|
@@ -1280,7 +1280,7 @@ let SandboxService = class SandboxService {
|
|
|
1280
1280
|
* name. Returns the `prev_*` patch for instances that could not be destroyed.
|
|
1281
1281
|
*/
|
|
1282
1282
|
async destroyInstance(row, p, signal) {
|
|
1283
|
-
const id = row.
|
|
1283
|
+
const id = row.device_id;
|
|
1284
1284
|
const owner = row.provider ?? p.plugin;
|
|
1285
1285
|
let prev = {};
|
|
1286
1286
|
if (row.handle && owner === p.plugin) {
|
|
@@ -1294,7 +1294,7 @@ let SandboxService = class SandboxService {
|
|
|
1294
1294
|
else if (!row.handle && row.status === "creating") {
|
|
1295
1295
|
// the provider may have built it already: only the name finds it
|
|
1296
1296
|
const impl = owner === p.plugin ? p.impl : this.providers.forName(owner)?.impl;
|
|
1297
|
-
await impl?.destroyByName(
|
|
1297
|
+
await impl?.destroyByName(deviceResourceName(id), signal);
|
|
1298
1298
|
}
|
|
1299
1299
|
if (row.prev_handle && row.prev_provider && !("prev_handle" in prev)) {
|
|
1300
1300
|
if (!(await this.retire(id, row.prev_provider, row.prev_handle, signal))) {
|
|
@@ -1310,17 +1310,17 @@ let SandboxService = class SandboxService {
|
|
|
1310
1310
|
* warns (the files leak at that provider).
|
|
1311
1311
|
*/
|
|
1312
1312
|
async destroyFiles(row, p, signal, lenient = false) {
|
|
1313
|
-
const id = row.
|
|
1313
|
+
const id = row.device_id;
|
|
1314
1314
|
const owner = row.provider ?? p.plugin;
|
|
1315
1315
|
const impl = owner === p.plugin ? p.impl : this.providers.forName(owner)?.impl;
|
|
1316
1316
|
if (!impl) {
|
|
1317
1317
|
if (row.fs_handle === null)
|
|
1318
1318
|
return;
|
|
1319
1319
|
if (!lenient) {
|
|
1320
|
-
throw new Error(`cannot destroy the file system of
|
|
1320
|
+
throw new Error(`cannot destroy the file system of device ${id}: its sandbox provider ${owner} is not loaded`);
|
|
1321
1321
|
}
|
|
1322
|
-
this.log.warn("
|
|
1323
|
-
|
|
1322
|
+
this.log.warn("device file system of a provider that is not loaded is leaked", {
|
|
1323
|
+
deviceId: id,
|
|
1324
1324
|
provider: owner,
|
|
1325
1325
|
});
|
|
1326
1326
|
return;
|
|
@@ -1328,18 +1328,18 @@ let SandboxService = class SandboxService {
|
|
|
1328
1328
|
if (row.fs_handle !== null)
|
|
1329
1329
|
await impl.destroyFileSystem(row.fs_handle, signal);
|
|
1330
1330
|
else
|
|
1331
|
-
await impl.destroyFileSystemByName(
|
|
1331
|
+
await impl.destroyFileSystemByName(deviceResourceName(id), signal);
|
|
1332
1332
|
}
|
|
1333
1333
|
/**
|
|
1334
1334
|
* Under the lease: `instance` destroys the instance and marks the row `destroyed` (the file
|
|
1335
|
-
* system stays); `
|
|
1335
|
+
* system stays); `device` also destroys the file system and deletes the row.
|
|
1336
1336
|
*/
|
|
1337
1337
|
async destroyRow(row, p, held, mode) {
|
|
1338
|
-
const id = row.
|
|
1338
|
+
const id = row.device_id;
|
|
1339
1339
|
const fence = { gen: row.generation };
|
|
1340
1340
|
const signal = opSignal(held);
|
|
1341
1341
|
const prev = await this.destroyInstance(row, p, signal);
|
|
1342
|
-
if (mode === "
|
|
1342
|
+
if (mode === "device")
|
|
1343
1343
|
await this.destroyFiles(row, p, signal);
|
|
1344
1344
|
if (!(await held.renew()))
|
|
1345
1345
|
throw new SandboxLockLostError(id);
|
|
@@ -1351,17 +1351,17 @@ let SandboxService = class SandboxService {
|
|
|
1351
1351
|
sweep_failed_at: null,
|
|
1352
1352
|
...prev,
|
|
1353
1353
|
});
|
|
1354
|
-
this.log.info("sandbox instance destroyed", {
|
|
1354
|
+
this.log.info("sandbox instance destroyed", { deviceId: id });
|
|
1355
1355
|
return;
|
|
1356
1356
|
}
|
|
1357
1357
|
const r = await this.db
|
|
1358
|
-
.deleteFrom("
|
|
1359
|
-
.where("
|
|
1358
|
+
.deleteFrom("device_instances")
|
|
1359
|
+
.where("device_id", "=", id)
|
|
1360
1360
|
.where("generation", "=", fence.gen)
|
|
1361
1361
|
.executeTakeFirst();
|
|
1362
1362
|
if (Number(r.numDeletedRows) !== 1)
|
|
1363
1363
|
throw new SandboxLockLostError(id, "generation");
|
|
1364
|
-
this.log.info("
|
|
1364
|
+
this.log.info("device destroyed", { deviceId: id });
|
|
1365
1365
|
}
|
|
1366
1366
|
/** Due for suspension: made by this provider, running, and idle or running for too long. */
|
|
1367
1367
|
suspendDue(now, plugin) {
|
|
@@ -1374,29 +1374,29 @@ let SandboxService = class SandboxService {
|
|
|
1374
1374
|
eb.or([eb("last_used_at", "<", idleBefore), eb("running_since", "<", runningBefore)]),
|
|
1375
1375
|
]);
|
|
1376
1376
|
}
|
|
1377
|
-
/** The provider of an instance row: an internal one (`@
|
|
1377
|
+
/** The provider of an instance row: an internal one (`@local`) by name, else the cloud one. */
|
|
1378
1378
|
rowProvider(row) {
|
|
1379
|
-
return
|
|
1379
|
+
return providerForDevice(this.providers, row.provider);
|
|
1380
1380
|
}
|
|
1381
1381
|
/**
|
|
1382
|
-
* A row the idle sweeps may touch: not on a kernel-internal provider (`@
|
|
1382
|
+
* A row the idle sweeps may touch: not on a kernel-internal provider (`@local` instances live on
|
|
1383
1383
|
* people's machines and are never suspended or destroyed for inactivity).
|
|
1384
1384
|
*/
|
|
1385
1385
|
cloudRow(eb) {
|
|
1386
1386
|
return eb.or([eb("provider", "is", null), eb("provider", "not like", "@%")]);
|
|
1387
1387
|
}
|
|
1388
|
-
/** The
|
|
1389
|
-
|
|
1388
|
+
/** The device was deleted, or its org was: the instance and the files go. */
|
|
1389
|
+
deviceGone(eb) {
|
|
1390
1390
|
return eb.or([
|
|
1391
1391
|
eb.exists(eb
|
|
1392
|
-
.selectFrom("
|
|
1393
|
-
.select("
|
|
1394
|
-
.whereRef("
|
|
1395
|
-
.where("
|
|
1392
|
+
.selectFrom("devices")
|
|
1393
|
+
.select("devices.id")
|
|
1394
|
+
.whereRef("devices.id", "=", "device_instances.device_id")
|
|
1395
|
+
.where("devices.deleted_at", "is not", null)),
|
|
1396
1396
|
eb.exists(eb
|
|
1397
1397
|
.selectFrom("orgs")
|
|
1398
1398
|
.select("orgs.id")
|
|
1399
|
-
.whereRef("orgs.id", "=", "
|
|
1399
|
+
.whereRef("orgs.id", "=", "device_instances.org_id")
|
|
1400
1400
|
.where("orgs.deleted_at", "is not", null)),
|
|
1401
1401
|
]);
|
|
1402
1402
|
}
|