libre-webui 0.19.5 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/backend/dist/db.d.ts.map +1 -1
- package/backend/dist/db.js +29 -0
- package/backend/dist/db.js.map +1 -1
- package/backend/dist/index.d.ts.map +1 -1
- package/backend/dist/index.js +12 -10
- package/backend/dist/index.js.map +1 -1
- package/backend/dist/routes/work.d.ts.map +1 -1
- package/backend/dist/routes/work.js +138 -4
- package/backend/dist/routes/work.js.map +1 -1
- package/backend/dist/services/systemDiagnosticsService.d.ts +4 -3
- package/backend/dist/services/systemDiagnosticsService.d.ts.map +1 -1
- package/backend/dist/services/systemDiagnosticsService.js +35 -34
- package/backend/dist/services/systemDiagnosticsService.js.map +1 -1
- package/backend/dist/services/workAccessService.d.ts +15 -0
- package/backend/dist/services/workAccessService.d.ts.map +1 -0
- package/backend/dist/services/workAccessService.js +67 -0
- package/backend/dist/services/workAccessService.js.map +1 -0
- package/backend/dist/services/workAdminService.d.ts +67 -0
- package/backend/dist/services/workAdminService.d.ts.map +1 -0
- package/backend/dist/services/workAdminService.js +92 -0
- package/backend/dist/services/workAdminService.js.map +1 -0
- package/backend/dist/services/workKubernetesDriver.d.ts +74 -0
- package/backend/dist/services/workKubernetesDriver.d.ts.map +1 -0
- package/backend/dist/services/workKubernetesDriver.js +659 -0
- package/backend/dist/services/workKubernetesDriver.js.map +1 -0
- package/backend/dist/services/workPolicyService.d.ts +58 -0
- package/backend/dist/services/workPolicyService.d.ts.map +1 -0
- package/backend/dist/services/workPolicyService.js +249 -0
- package/backend/dist/services/workPolicyService.js.map +1 -0
- package/backend/dist/services/workPreviewProxyService.d.ts +7 -1
- package/backend/dist/services/workPreviewProxyService.d.ts.map +1 -1
- package/backend/dist/services/workPreviewProxyService.js +36 -8
- package/backend/dist/services/workPreviewProxyService.js.map +1 -1
- package/backend/dist/services/workRuntimeDriver.d.ts +166 -0
- package/backend/dist/services/workRuntimeDriver.d.ts.map +1 -0
- package/backend/dist/services/workRuntimeDriver.js +737 -0
- package/backend/dist/services/workRuntimeDriver.js.map +1 -0
- package/backend/dist/services/workRuntimeService.d.ts +60 -51
- package/backend/dist/services/workRuntimeService.d.ts.map +1 -1
- package/backend/dist/services/workRuntimeService.js +310 -658
- package/backend/dist/services/workRuntimeService.js.map +1 -1
- package/backend/dist/services/workRuntimeShared.d.ts +83 -0
- package/backend/dist/services/workRuntimeShared.d.ts.map +1 -0
- package/backend/dist/services/workRuntimeShared.js +136 -0
- package/backend/dist/services/workRuntimeShared.js.map +1 -0
- package/backend/dist/services/workTaskService.d.ts +6 -1
- package/backend/dist/services/workTaskService.d.ts.map +1 -1
- package/backend/dist/services/workTaskService.js +18 -3
- package/backend/dist/services/workTaskService.js.map +1 -1
- package/backend/dist/services/workTerminalService.d.ts +2 -22
- package/backend/dist/services/workTerminalService.d.ts.map +1 -1
- package/backend/dist/services/workTerminalService.js +10 -127
- package/backend/dist/services/workTerminalService.js.map +1 -1
- package/backend/dist/types/work.d.ts +4 -0
- package/backend/dist/types/work.d.ts.map +1 -1
- package/backend/dist/utils/dockerEndpoint.d.ts +37 -0
- package/backend/dist/utils/dockerEndpoint.d.ts.map +1 -0
- package/backend/dist/utils/dockerEndpoint.js +72 -0
- package/backend/dist/utils/dockerEndpoint.js.map +1 -0
- package/backend/dist/workTerminalServer.d.ts.map +1 -1
- package/backend/dist/workTerminalServer.js +6 -5
- package/backend/dist/workTerminalServer.js.map +1 -1
- package/backend/package.json +2 -1
- package/electron/main.js +196 -11
- package/frontend/dist/artifact-runtime/.build-fingerprint +1 -1
- package/frontend/dist/assets/index-Z4-oLWDC.js +2 -0
- package/frontend/dist/assets/index-tBUyDbHh.css +1 -0
- package/frontend/dist/index.html +11 -10
- package/frontend/dist/js/{ArtifactContainer-BRqFYglf.js → ArtifactContainer-DQRF4rh-.js} +1 -1
- package/frontend/dist/js/{ArtifactDemoPage-CQj2XRxx.js → ArtifactDemoPage-BOlB1lS-.js} +1 -1
- package/frontend/dist/js/{ArtifactSandboxFrame-BsrybHW3.js → ArtifactSandboxFrame-DqU5GDrI.js} +2 -2
- package/frontend/dist/js/{ArtifactSlideOutPanel-BcDkHDZY.js → ArtifactSlideOutPanel-B0I-hCux.js} +1 -1
- package/frontend/dist/js/ChatPage-DyV0_oCt.js +16 -0
- package/frontend/dist/js/{FirstTimeSetup-CX3zIxZR.js → FirstTimeSetup-ButTmjp0.js} +1 -1
- package/frontend/dist/js/{GalleryPage-Dv5IJVVY.js → GalleryPage-CaT63b-s.js} +1 -1
- package/frontend/dist/js/HomePage-BWyg3nHu.js +1 -0
- package/frontend/dist/js/{LibreClawPage-iEHU_8KO.js → LibreClawPage-DUd3tjQE.js} +1 -1
- package/frontend/dist/js/ModelSelector-COFcJnMJ.js +1 -0
- package/frontend/dist/js/{ModelsPage-BU-VYLs1.js → ModelsPage-BRbTEXNL.js} +1 -1
- package/frontend/dist/js/{NotesPage-DKXMxPjC.js → NotesPage-CTutDN50.js} +1 -1
- package/frontend/dist/js/{PersonasPage-DnjO6raM.js → PersonasPage-C6iTMI3N.js} +2 -2
- package/frontend/dist/js/{PluginUsagePage-DNihHooM.js → PluginUsagePage-BQQ0-_PP.js} +1 -1
- package/frontend/dist/js/{RichMessageContent-DDJDx7_n.js → RichMessageContent-DPqeqxkQ.js} +1 -1
- package/frontend/dist/js/SettingsModal-C_1k75Kv.js +1 -0
- package/frontend/dist/js/SettingsToggle-bZ5W4CCf.js +1 -0
- package/frontend/dist/js/SystemPage-CUecyRPh.js +1 -0
- package/frontend/dist/js/UserManagementPage-CZ4KMOY9.js +1 -0
- package/frontend/dist/js/WorkPage-BqDcfXV6.js +5 -0
- package/frontend/dist/js/api-iCpOEijB.js +14 -0
- package/frontend/dist/js/{appStore-hNTtRnlO.js → appStore-DTd6dWg2.js} +2 -2
- package/frontend/dist/js/{ar-ClMsdoTH.js → ar-C8YV99Y-.js} +1 -1
- package/frontend/dist/js/authStore-CuK4HUVh.js +4 -0
- package/frontend/dist/js/{bn-LyPol_AY.js → bn-C9dcls7_.js} +1 -1
- package/frontend/dist/js/chatStore-VNT0eHRB.js +2 -0
- package/frontend/dist/js/{config-cqTxMjWo.js → config--Xp9rZtJ.js} +1 -1
- package/frontend/dist/js/{cs-D98fZaKF.js → cs-DtL8VeeX.js} +1 -1
- package/frontend/dist/js/{da-M7ND0GQR.js → da-U3aNn1aF.js} +1 -1
- package/frontend/dist/js/{de-BV0q1kzb.js → de-DzuyeVpX.js} +1 -1
- package/frontend/dist/js/{es-C7FvsVoN.js → es-OiU8plii.js} +1 -1
- package/frontend/dist/js/{estree-BAxGO-Uf.js → estree-BAQkciRX.js} +1 -1
- package/frontend/dist/js/{fr-5SloN0ok.js → fr-DDa3R8bs.js} +1 -1
- package/frontend/dist/js/{hi-C6VeRgeR.js → hi-D8Y0hl34.js} +1 -1
- package/frontend/dist/js/{html-DL8k6nha.js → html-DxA-Bcvi.js} +1 -1
- package/frontend/dist/js/{i18n-CLd1FZPx.js → i18n-BqWNFeNo.js} +1 -1
- package/frontend/dist/js/{id-CkVp7Z1Z.js → id-BwcWtkBv.js} +1 -1
- package/frontend/dist/js/{is-DqxnBX4W.js → is-DzfQHQih.js} +1 -1
- package/frontend/dist/js/{it-CiW4_O5V.js → it-C5VU3_Je.js} +1 -1
- package/frontend/dist/js/{ja-BZ3Sdund.js → ja-DLhOhxvf.js} +1 -1
- package/frontend/dist/js/{ko-CS7LAhfJ.js → ko-CCKiyQD6.js} +1 -1
- package/frontend/dist/js/{markdown-CKunEqMI.js → markdown-IsZpJ5ke.js} +1 -1
- package/frontend/dist/js/{messageCodeStyles-Cva_Rdk3.js → messageCodeStyles-CsV3zwyK.js} +1 -1
- package/frontend/dist/js/{ms-URvcYqgz.js → ms-CXXkPvWw.js} +1 -1
- package/frontend/dist/js/{nl-CQJpW7wK.js → nl-CA_e0Epl.js} +1 -1
- package/frontend/dist/js/{pl-DDMgPMAz.js → pl-BG32wdpx.js} +1 -1
- package/frontend/dist/js/{pt-JCqqmGn_.js → pt-BzRGWQXO.js} +1 -1
- package/frontend/dist/js/{ru-C_dwMbiK.js → ru-DBjn_Q-Q.js} +1 -1
- package/frontend/dist/js/{standalone-qHCG6_XI.js → standalone-CK3b5Ktq.js} +1 -1
- package/frontend/dist/js/{sv-CW3c9NGx.js → sv-Hz_iRGdF.js} +1 -1
- package/frontend/dist/js/{th-9ARgClBr.js → th-MsyXlGH9.js} +1 -1
- package/frontend/dist/js/{tr-DNY5_DAg.js → tr-Cofmy_ef.js} +1 -1
- package/frontend/dist/js/{typescript-ClQp1SgP.js → typescript-BMYmVohw.js} +1 -1
- package/frontend/dist/js/{ui-ChYGf2Rg.js → ui-C2kNYwfG.js} +2 -2
- package/frontend/dist/js/{uk-XDh4_7IL.js → uk-B2o05LCR.js} +1 -1
- package/frontend/dist/js/{vi-BaoJBXnt.js → vi-CZh8uJgV.js} +1 -1
- package/frontend/dist/js/workApi-C33ZrPws.js +6 -0
- package/frontend/dist/js/{zh-B202vYJt.js → zh-DhKXi8ZA.js} +1 -1
- package/package.json +7 -3
- package/frontend/dist/assets/index-DtkGuqvC.css +0 -1
- package/frontend/dist/assets/index-yeE8bw17.js +0 -22
- package/frontend/dist/js/ChatPage-CtN25kO5.js +0 -16
- package/frontend/dist/js/HomePage-C7JjQJSz.js +0 -1
- package/frontend/dist/js/ModelSelector-DhRVhkDT.js +0 -1
- package/frontend/dist/js/SettingsModal-Djij7CKl.js +0 -1
- package/frontend/dist/js/SystemPage-CDBXVff3.js +0 -1
- package/frontend/dist/js/UserManagementPage-CZJLqvwO.js +0 -1
- package/frontend/dist/js/WorkPage-EpH5lfJx.js +0 -5
- package/frontend/dist/js/api-DVeiqNG9.js +0 -19
- package/frontend/dist/js/authStore-Cl_7F0Si.js +0 -3
- package/frontend/dist/js/chatStore-Bp5u1tTv.js +0 -2
|
@@ -14,83 +14,26 @@
|
|
|
14
14
|
* See the License for the specific language governing permissions and
|
|
15
15
|
* limitations under the License.
|
|
16
16
|
*/
|
|
17
|
-
import { spawn } from 'node:child_process';
|
|
18
|
-
import { createHash } from 'node:crypto';
|
|
19
17
|
import path from 'node:path';
|
|
20
18
|
import { getDatabase } from '../db.js';
|
|
21
19
|
import { createLogger } from '../utils/logger.js';
|
|
22
20
|
import { buildWorkGitCommand, parseWorkGitBranches, parseWorkGitLog, parseWorkGitStatus, validateWorkGitBranchName, validateWorkGitRepositoryPaths, } from '../utils/workGit.js';
|
|
21
|
+
import { userHasWorkAccess } from './workAccessService.js';
|
|
22
|
+
import workPolicyService from './workPolicyService.js';
|
|
23
23
|
import workPreviewProxyService from './workPreviewProxyService.js';
|
|
24
|
+
import workTaskService from './workTaskService.js';
|
|
25
|
+
import { KubernetesWorkRuntimeDriver } from './workKubernetesDriver.js';
|
|
26
|
+
import { DockerWorkRuntimeDriver, } from './workRuntimeDriver.js';
|
|
27
|
+
import { WorkRuntimeError, workRuntimeConfig as config, } from './workRuntimeShared.js';
|
|
28
|
+
// The public surface of the Work runtime is unchanged by the driver split:
|
|
29
|
+
// consumers and tests keep importing everything from this module.
|
|
30
|
+
export { WORK_RUNTIME_ADMISSION_DEFAULTS, WORK_RUNTIME_DEFAULTS, WorkRuntimeError, parseDnsServers, } from './workRuntimeShared.js';
|
|
31
|
+
export { DockerWorkRuntimeDriver, buildWorkContainerRunArgs, describeDockerUnavailable, formatPreviewHost, parseManagedContainerList, parsePublishedPort, } from './workRuntimeDriver.js';
|
|
24
32
|
const logger = createLogger('services:work-runtime');
|
|
25
|
-
const activeDockerProcesses = new Set();
|
|
26
|
-
export const WORK_RUNTIME_DEFAULTS = {
|
|
27
|
-
image: 'node:22.22-bookworm@sha256:2d178f2785b96dfbf62a416ca2e40f50e30150b4ff3320d706f0d96e90600eb3',
|
|
28
|
-
dockerCommand: 'docker',
|
|
29
|
-
commandTimeoutMs: 120000,
|
|
30
|
-
maxOutputChars: 50000,
|
|
31
|
-
maxAgentRounds: 48,
|
|
32
|
-
memoryLimit: '2g',
|
|
33
|
-
cpuLimit: '2',
|
|
34
|
-
pidsLimit: 256,
|
|
35
|
-
previewPort: 4173,
|
|
36
|
-
previewBind: '127.0.0.1',
|
|
37
|
-
networkName: 'libre-webui-work',
|
|
38
|
-
};
|
|
39
|
-
// Two runtimes per administrator so a second task does not have to wait for
|
|
40
|
-
// the first, three per instance to bound the worst case at three containers'
|
|
41
|
-
// resource caps. Both remain tunable through WORK_MAX_ACTIVE_RUNTIMES_*.
|
|
42
|
-
export const WORK_RUNTIME_ADMISSION_DEFAULTS = {
|
|
43
|
-
maxActiveRuntimesGlobal: 3,
|
|
44
|
-
maxActiveRuntimesPerUser: 2,
|
|
45
|
-
};
|
|
46
|
-
const config = {
|
|
47
|
-
image: process.env.WORK_RUNTIME_IMAGE || WORK_RUNTIME_DEFAULTS.image,
|
|
48
|
-
dockerCommand: process.env.WORK_DOCKER_COMMAND || WORK_RUNTIME_DEFAULTS.dockerCommand,
|
|
49
|
-
commandTimeoutMs: positiveInteger(process.env.WORK_COMMAND_TIMEOUT_MS, WORK_RUNTIME_DEFAULTS.commandTimeoutMs),
|
|
50
|
-
maxOutputChars: positiveInteger(process.env.WORK_MAX_OUTPUT_CHARS, WORK_RUNTIME_DEFAULTS.maxOutputChars),
|
|
51
|
-
maxAgentRounds: positiveInteger(process.env.WORK_MAX_AGENT_ROUNDS, WORK_RUNTIME_DEFAULTS.maxAgentRounds),
|
|
52
|
-
memoryLimit: process.env.WORK_MEMORY_LIMIT || WORK_RUNTIME_DEFAULTS.memoryLimit,
|
|
53
|
-
cpuLimit: process.env.WORK_CPU_LIMIT || WORK_RUNTIME_DEFAULTS.cpuLimit,
|
|
54
|
-
pidsLimit: positiveInteger(process.env.WORK_PIDS_LIMIT, WORK_RUNTIME_DEFAULTS.pidsLimit),
|
|
55
|
-
previewPort: positiveInteger(process.env.WORK_PREVIEW_PORT, WORK_RUNTIME_DEFAULTS.previewPort),
|
|
56
|
-
// Interface the daemon publishes a task preview on. Loopback keeps a preview
|
|
57
|
-
// private to the Docker host, which is correct when the browser runs there.
|
|
58
|
-
previewBind: process.env.WORK_PREVIEW_BIND || WORK_RUNTIME_DEFAULTS.previewBind,
|
|
59
|
-
maxActiveRuntimesGlobal: positiveInteger(process.env.WORK_MAX_ACTIVE_RUNTIMES_GLOBAL, WORK_RUNTIME_ADMISSION_DEFAULTS.maxActiveRuntimesGlobal),
|
|
60
|
-
maxActiveRuntimesPerUser: positiveInteger(process.env.WORK_MAX_ACTIVE_RUNTIMES_PER_USER, WORK_RUNTIME_ADMISSION_DEFAULTS.maxActiveRuntimesPerUser),
|
|
61
|
-
// Dedicated bridge network for networked Work tasks. It is created with
|
|
62
|
-
// inter-container communication disabled, so a sandbox cannot reach other
|
|
63
|
-
// Work sandboxes or the deployment's own containers on the default bridge.
|
|
64
|
-
networkName: process.env.WORK_NETWORK_NAME || WORK_RUNTIME_DEFAULTS.networkName,
|
|
65
|
-
// Optional resolvers forced onto every networked sandbox. Pointing this at
|
|
66
|
-
// a filtering resolver is the supported egress-policy hook.
|
|
67
|
-
dnsServers: parseDnsServers(process.env.WORK_RUNTIME_DNS),
|
|
68
|
-
};
|
|
69
33
|
const PREVIEW_READY_TIMEOUT_MS = 15000;
|
|
70
34
|
const PREVIEW_POLL_INTERVAL_MS = 250;
|
|
71
|
-
const runtimePolicyFingerprint = createHash('sha256')
|
|
72
|
-
.update(JSON.stringify({
|
|
73
|
-
version: 2,
|
|
74
|
-
memoryLimit: config.memoryLimit,
|
|
75
|
-
cpuLimit: config.cpuLimit,
|
|
76
|
-
pidsLimit: config.pidsLimit,
|
|
77
|
-
previewPort: config.previewPort,
|
|
78
|
-
previewBind: config.previewBind,
|
|
79
|
-
networkName: config.networkName,
|
|
80
|
-
dnsServers: config.dnsServers,
|
|
81
|
-
memorySwapPinned: true,
|
|
82
|
-
}))
|
|
83
|
-
.digest('hex');
|
|
84
|
-
export class WorkRuntimeError extends Error {
|
|
85
|
-
constructor(message, status = 503, code = 'WORK_RUNTIME_UNAVAILABLE') {
|
|
86
|
-
super(message);
|
|
87
|
-
this.name = 'WorkRuntimeError';
|
|
88
|
-
this.status = status;
|
|
89
|
-
this.code = code;
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
35
|
export class WorkRuntimeService {
|
|
93
|
-
constructor() {
|
|
36
|
+
constructor(driver = new DockerWorkRuntimeDriver()) {
|
|
94
37
|
this.image = config.image;
|
|
95
38
|
this.previewPort = config.previewPort;
|
|
96
39
|
this.limits = {
|
|
@@ -100,6 +43,7 @@ export class WorkRuntimeService {
|
|
|
100
43
|
maxActiveRuntimesGlobal: config.maxActiveRuntimesGlobal,
|
|
101
44
|
maxActiveRuntimesPerUser: config.maxActiveRuntimesPerUser,
|
|
102
45
|
};
|
|
46
|
+
this.imagePreparations = new Map();
|
|
103
47
|
this.lifecycleTails = new Map();
|
|
104
48
|
this.preparations = new Map();
|
|
105
49
|
this.retiringTasks = new Set();
|
|
@@ -110,13 +54,33 @@ export class WorkRuntimeService {
|
|
|
110
54
|
this.previewLeaseReleases = new Map();
|
|
111
55
|
this.terminalHolds = new Map();
|
|
112
56
|
this.recoveryTasks = new Map();
|
|
57
|
+
this.recoveryOrphans = new Map();
|
|
58
|
+
// Sweeps left for the empty-inventory case before giving up on a daemon
|
|
59
|
+
// that never appears (30 × 10s: covers a socket proxy starting late
|
|
60
|
+
// without probing a Docker-less deployment forever).
|
|
61
|
+
this.emptyInventorySweepsLeft = 30;
|
|
113
62
|
this.shuttingDown = false;
|
|
63
|
+
// Last observed activity per task, feeding the idle sweep: a finished
|
|
64
|
+
// command, a terminal session ending, or a preview fetch. In-memory only —
|
|
65
|
+
// after a restart the clock restarts from the first sighting.
|
|
66
|
+
this.taskActivity = new Map();
|
|
67
|
+
this.driver = driver;
|
|
68
|
+
// Authorized preview traffic keeps its task's idle clock fresh.
|
|
69
|
+
workPreviewProxyService.onPreviewActivity(taskId => this.noteTaskActivity(taskId));
|
|
70
|
+
this.scheduleIdleSweep();
|
|
71
|
+
}
|
|
72
|
+
noteTaskActivity(taskId) {
|
|
73
|
+
this.taskActivity.set(taskId, Date.now());
|
|
114
74
|
}
|
|
115
75
|
get recoveryPending() {
|
|
116
|
-
return this.
|
|
76
|
+
return this.recoveryPendingCount > 0;
|
|
117
77
|
}
|
|
118
78
|
get recoveryPendingCount() {
|
|
119
|
-
|
|
79
|
+
// Before reconciliation has run, every inventoried task counts as
|
|
80
|
+
// pending: nothing has been proven about its container yet.
|
|
81
|
+
return ((this.recoveryInventory?.length ?? 0) +
|
|
82
|
+
this.recoveryTasks.size +
|
|
83
|
+
this.recoveryOrphans.size);
|
|
120
84
|
}
|
|
121
85
|
assertAcceptingWork() {
|
|
122
86
|
if (this.shuttingDown) {
|
|
@@ -128,14 +92,13 @@ export class WorkRuntimeService {
|
|
|
128
92
|
}
|
|
129
93
|
async isDockerAvailable() {
|
|
130
94
|
try {
|
|
131
|
-
await this.
|
|
132
|
-
timeoutMs: 5000,
|
|
133
|
-
});
|
|
95
|
+
await this.driver.probe();
|
|
134
96
|
this.lastDockerUnavailableReason = null;
|
|
135
97
|
return true;
|
|
136
98
|
}
|
|
137
99
|
catch (error) {
|
|
138
|
-
this.lastDockerUnavailableReason =
|
|
100
|
+
this.lastDockerUnavailableReason =
|
|
101
|
+
error instanceof Error ? error.message : String(error);
|
|
139
102
|
return false;
|
|
140
103
|
}
|
|
141
104
|
}
|
|
@@ -162,9 +125,7 @@ export class WorkRuntimeService {
|
|
|
162
125
|
return { global: this.runtimeLeases.size, user };
|
|
163
126
|
}
|
|
164
127
|
async beginRecovery(tasks) {
|
|
165
|
-
|
|
166
|
-
this.recoveryTasks.set(task.id, task);
|
|
167
|
-
}
|
|
128
|
+
this.recoveryInventory = tasks;
|
|
168
129
|
const result = await this.sweepRecoveryTasks();
|
|
169
130
|
this.scheduleRecoverySweep();
|
|
170
131
|
return result;
|
|
@@ -233,7 +194,7 @@ export class WorkRuntimeService {
|
|
|
233
194
|
}
|
|
234
195
|
let tracked;
|
|
235
196
|
tracked = (async () => {
|
|
236
|
-
await waitForAbortSignal(this.ensureImage(), signal);
|
|
197
|
+
await waitForAbortSignal(this.ensureImage(task), signal);
|
|
237
198
|
this.assertTaskIsActive(task);
|
|
238
199
|
await this.withLifecycleLock(task.id, async () => {
|
|
239
200
|
this.assertTaskIsActive(task);
|
|
@@ -272,6 +233,7 @@ export class WorkRuntimeService {
|
|
|
272
233
|
if (released)
|
|
273
234
|
return;
|
|
274
235
|
released = true;
|
|
236
|
+
this.noteTaskActivity(task.id);
|
|
275
237
|
const remaining = (this.terminalHolds.get(task.id) ?? 1) - 1;
|
|
276
238
|
if (remaining <= 0) {
|
|
277
239
|
this.terminalHolds.delete(task.id);
|
|
@@ -301,9 +263,11 @@ export class WorkRuntimeService {
|
|
|
301
263
|
clearTimeout(this.recoveryTimer);
|
|
302
264
|
this.recoveryTimer = undefined;
|
|
303
265
|
}
|
|
304
|
-
|
|
305
|
-
|
|
266
|
+
if (this.idleTimer) {
|
|
267
|
+
clearTimeout(this.idleTimer);
|
|
268
|
+
this.idleTimer = undefined;
|
|
306
269
|
}
|
|
270
|
+
this.driver.shutdown();
|
|
307
271
|
}
|
|
308
272
|
beginTaskSuspension(taskId) {
|
|
309
273
|
if (this.retiringTasks.has(taskId)) {
|
|
@@ -315,16 +279,63 @@ export class WorkRuntimeService {
|
|
|
315
279
|
this.retiringTasks.delete(taskId);
|
|
316
280
|
}
|
|
317
281
|
async sweepRecoveryTasks() {
|
|
318
|
-
if (this.
|
|
282
|
+
if (this.recoveryInventory === undefined &&
|
|
283
|
+
this.recoveryTasks.size === 0 &&
|
|
284
|
+
this.recoveryOrphans.size === 0) {
|
|
319
285
|
return { stopped: 0, failed: 0 };
|
|
320
286
|
}
|
|
321
287
|
if (this.shuttingDown || !(await this.isDockerAvailable())) {
|
|
322
|
-
|
|
288
|
+
if (this.recoveryInventory?.length === 0) {
|
|
289
|
+
// No tasks to supervise, so nothing fail-closes — but leftover
|
|
290
|
+
// containers may still exist (a restored database, a daemon or
|
|
291
|
+
// socket proxy that comes up after the backend). Keep retrying the
|
|
292
|
+
// reconciliation quietly for a bounded window, then give up so a
|
|
293
|
+
// deployment without Docker is not probed forever.
|
|
294
|
+
if (this.shuttingDown || this.emptyInventorySweepsLeft <= 0) {
|
|
295
|
+
this.recoveryInventory = undefined;
|
|
296
|
+
return { stopped: 0, failed: 0 };
|
|
297
|
+
}
|
|
298
|
+
this.emptyInventorySweepsLeft -= 1;
|
|
299
|
+
return { stopped: 0, failed: 0 };
|
|
300
|
+
}
|
|
301
|
+
return { stopped: 0, failed: this.recoveryPendingCount };
|
|
302
|
+
}
|
|
303
|
+
if (this.recoveryInventory !== undefined) {
|
|
304
|
+
// Reconcile against what actually exists instead of stopping every
|
|
305
|
+
// known task blind: one labeled listing decides which containers are
|
|
306
|
+
// running unsupervised, which are at rest, and which are orphans left
|
|
307
|
+
// by a task whose database row is gone.
|
|
308
|
+
let discovered;
|
|
309
|
+
try {
|
|
310
|
+
discovered = await this.driver.listManaged();
|
|
311
|
+
}
|
|
312
|
+
catch (error) {
|
|
313
|
+
logger.warn('Could not list Work containers for startup reconciliation:', error);
|
|
314
|
+
return { stopped: 0, failed: this.recoveryPendingCount };
|
|
315
|
+
}
|
|
316
|
+
const plan = planStartupReconciliation(this.recoveryInventory, discovered);
|
|
317
|
+
for (const task of plan.stop) {
|
|
318
|
+
this.recoveryTasks.set(task.id, task);
|
|
319
|
+
}
|
|
320
|
+
for (const orphan of plan.removeOrphans) {
|
|
321
|
+
this.recoveryOrphans.set(orphan.name, orphan);
|
|
322
|
+
}
|
|
323
|
+
await this.reportOrphanWorkspaces(this.recoveryInventory);
|
|
324
|
+
this.recoveryInventory = undefined;
|
|
325
|
+
if (this.recoveryPendingCount > 0 || plan.atRest > 0) {
|
|
326
|
+
logger.info(`Work startup reconciliation: ${plan.stop.length} running container(s) to stop, ` +
|
|
327
|
+
`${plan.removeOrphans.length} orphaned container(s) to remove, ` +
|
|
328
|
+
`${plan.atRest} container(s) already at rest.`);
|
|
329
|
+
}
|
|
323
330
|
}
|
|
324
331
|
const tasks = [...this.recoveryTasks.values()];
|
|
325
|
-
const
|
|
332
|
+
const orphans = [...this.recoveryOrphans.values()];
|
|
333
|
+
const [taskResults, orphanResults] = await Promise.all([
|
|
334
|
+
Promise.allSettled(tasks.map(task => this.stopContainer(task))),
|
|
335
|
+
Promise.allSettled(orphans.map(orphan => this.driver.removeOrphan(orphan.name))),
|
|
336
|
+
]);
|
|
326
337
|
let stopped = 0;
|
|
327
|
-
|
|
338
|
+
taskResults.forEach((result, index) => {
|
|
328
339
|
if (result.status === 'fulfilled') {
|
|
329
340
|
this.recoveryTasks.delete(tasks[index].id);
|
|
330
341
|
stopped += 1;
|
|
@@ -333,14 +344,135 @@ export class WorkRuntimeService {
|
|
|
333
344
|
logger.warn(`Could not stop Work container ${tasks[index].containerName} during recovery:`, result.reason);
|
|
334
345
|
}
|
|
335
346
|
});
|
|
336
|
-
|
|
347
|
+
orphanResults.forEach((result, index) => {
|
|
348
|
+
if (result.status === 'fulfilled') {
|
|
349
|
+
this.recoveryOrphans.delete(orphans[index].name);
|
|
350
|
+
stopped += 1;
|
|
351
|
+
}
|
|
352
|
+
else {
|
|
353
|
+
logger.warn(`Could not remove orphaned Work container ${orphans[index].name} during recovery:`, result.reason);
|
|
354
|
+
}
|
|
355
|
+
});
|
|
356
|
+
if (this.recoveryPendingCount === 0 && (tasks.length || orphans.length)) {
|
|
337
357
|
logger.info('Work startup container recovery completed.');
|
|
338
358
|
}
|
|
339
|
-
return { stopped, failed: this.
|
|
359
|
+
return { stopped, failed: this.recoveryPendingCount };
|
|
360
|
+
}
|
|
361
|
+
/**
|
|
362
|
+
* Workspaces whose task record no longer exists are reported, never
|
|
363
|
+
* auto-deleted: a workspace is the durable half of a task, so removal is
|
|
364
|
+
* always an explicit operator decision.
|
|
365
|
+
*/
|
|
366
|
+
async reportOrphanWorkspaces(inventory) {
|
|
367
|
+
if (!this.driver.listWorkspaces)
|
|
368
|
+
return;
|
|
369
|
+
try {
|
|
370
|
+
const known = new Set(inventory.map(task => task.id));
|
|
371
|
+
const orphaned = (await this.driver.listWorkspaces()).filter(workspace => !known.has(workspace.taskId));
|
|
372
|
+
if (orphaned.length > 0) {
|
|
373
|
+
logger.warn(`Work found ${orphaned.length} workspace(s) with no task record, left in place: ` +
|
|
374
|
+
orphaned.map(workspace => workspace.name).join(', '));
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
catch (error) {
|
|
378
|
+
logger.warn('Could not list Work workspaces for orphan reporting:', error);
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
/**
|
|
382
|
+
* Stop sandboxes that have seen no activity — no command finished, no
|
|
383
|
+
* terminal attached, no preview fetch — for WORK_RUNTIME_IDLE_TIMEOUT_MS.
|
|
384
|
+
* Stopping is cheap and the workspace persists, so the sweep only spends
|
|
385
|
+
* an admission slot holder that nobody is using. Busy tasks (active
|
|
386
|
+
* command, terminal, or a non-preview operation lease) refresh their
|
|
387
|
+
* clock instead of being considered. A running sandbox seen for the
|
|
388
|
+
* first time starts its clock at the sighting rather than being stopped
|
|
389
|
+
* on a guess.
|
|
390
|
+
*/
|
|
391
|
+
async sweepIdleRuntimes(now = Date.now()) {
|
|
392
|
+
if (this.shuttingDown || this.recoveryPending) {
|
|
393
|
+
return { stopped: 0 };
|
|
394
|
+
}
|
|
395
|
+
// Idle-stop can come from the global knob or from any named policy.
|
|
396
|
+
if (config.idleTimeoutMs <= 0 &&
|
|
397
|
+
!workPolicyService.anyIdleTimeoutConfigured()) {
|
|
398
|
+
return { stopped: 0 };
|
|
399
|
+
}
|
|
400
|
+
let discovered;
|
|
401
|
+
try {
|
|
402
|
+
discovered = await this.driver.listManaged();
|
|
403
|
+
}
|
|
404
|
+
catch {
|
|
405
|
+
// The runtime is unreachable; nothing can be stopped anyway.
|
|
406
|
+
return { stopped: 0 };
|
|
407
|
+
}
|
|
408
|
+
const records = new Map(workTaskService.listAllTaskRecords().map(task => [task.id, task]));
|
|
409
|
+
let stopped = 0;
|
|
410
|
+
for (const entry of discovered) {
|
|
411
|
+
if (!entry.running)
|
|
412
|
+
continue;
|
|
413
|
+
const task = records.get(entry.taskId);
|
|
414
|
+
// Containers without a task row are startup reconciliation's business.
|
|
415
|
+
if (!task)
|
|
416
|
+
continue;
|
|
417
|
+
const busy = this.activeCommands.has(task.id) ||
|
|
418
|
+
(this.terminalHolds.get(task.id) ?? 0) > 0 ||
|
|
419
|
+
(this.runtimeLeases.has(task.id) &&
|
|
420
|
+
!this.previewLeaseReleases.has(task.id));
|
|
421
|
+
const lastActivity = this.taskActivity.get(task.id);
|
|
422
|
+
if (busy || lastActivity === undefined) {
|
|
423
|
+
this.noteTaskActivity(task.id);
|
|
424
|
+
continue;
|
|
425
|
+
}
|
|
426
|
+
const idleAfterMs = workPolicyService.resolve(task.policyId).idleTimeoutMs;
|
|
427
|
+
if (idleAfterMs <= 0 || now - lastActivity < idleAfterMs)
|
|
428
|
+
continue;
|
|
429
|
+
try {
|
|
430
|
+
if (this.previewLeaseReleases.has(task.id)) {
|
|
431
|
+
await this.stopPreview(task, {
|
|
432
|
+
onStopped: () => workTaskService.updatePreview(task.id, 'stopped'),
|
|
433
|
+
});
|
|
434
|
+
}
|
|
435
|
+
else {
|
|
436
|
+
await this.stopContainer(task);
|
|
437
|
+
}
|
|
438
|
+
this.taskActivity.delete(task.id);
|
|
439
|
+
stopped += 1;
|
|
440
|
+
logger.info(`Stopped idle Work sandbox ${task.containerName} after ${idleAfterMs}ms of inactivity.`);
|
|
441
|
+
}
|
|
442
|
+
catch (error) {
|
|
443
|
+
logger.warn(`Could not stop idle Work sandbox ${task.containerName}:`, error);
|
|
444
|
+
}
|
|
445
|
+
}
|
|
446
|
+
return { stopped };
|
|
447
|
+
}
|
|
448
|
+
scheduleIdleSweep() {
|
|
449
|
+
if (this.shuttingDown || this.idleTimer) {
|
|
450
|
+
return;
|
|
451
|
+
}
|
|
452
|
+
// Policies can enable idle-stop at runtime even when the global knob is
|
|
453
|
+
// off, so the timer always runs; the sweep itself exits in one cheap
|
|
454
|
+
// query when nothing configures a timeout. With a global timeout the
|
|
455
|
+
// interval sits well inside it, so an idle sandbox overshoots its
|
|
456
|
+
// deadline by a fraction, not a multiple.
|
|
457
|
+
const interval = config.idleTimeoutMs > 0
|
|
458
|
+
? Math.max(15000, Math.min(60000, Math.floor(config.idleTimeoutMs / 4)))
|
|
459
|
+
: 60000;
|
|
460
|
+
this.idleTimer = setTimeout(() => {
|
|
461
|
+
this.idleTimer = undefined;
|
|
462
|
+
void this.sweepIdleRuntimes()
|
|
463
|
+
.catch(error => {
|
|
464
|
+
logger.warn('Work idle sweep failed:', error);
|
|
465
|
+
})
|
|
466
|
+
.finally(() => {
|
|
467
|
+
this.scheduleIdleSweep();
|
|
468
|
+
});
|
|
469
|
+
}, interval);
|
|
470
|
+
this.idleTimer.unref();
|
|
340
471
|
}
|
|
341
472
|
scheduleRecoverySweep() {
|
|
342
473
|
if (this.shuttingDown ||
|
|
343
|
-
this.
|
|
474
|
+
(this.recoveryPendingCount === 0 &&
|
|
475
|
+
this.recoveryInventory === undefined) ||
|
|
344
476
|
this.recoveryTimer) {
|
|
345
477
|
return;
|
|
346
478
|
}
|
|
@@ -363,7 +495,7 @@ export class WorkRuntimeService {
|
|
|
363
495
|
}
|
|
364
496
|
completeRecoveryTask(taskId) {
|
|
365
497
|
const removed = this.recoveryTasks.delete(taskId);
|
|
366
|
-
if (removed && this.
|
|
498
|
+
if (removed && this.recoveryPendingCount === 0) {
|
|
367
499
|
logger.info('Pending Work container cleanup completed.');
|
|
368
500
|
}
|
|
369
501
|
}
|
|
@@ -383,72 +515,19 @@ export class WorkRuntimeService {
|
|
|
383
515
|
this.assertRuntimeLease(task);
|
|
384
516
|
this.assertCurrentNetworkPolicy(task);
|
|
385
517
|
this.assertTaskIsActive(task);
|
|
386
|
-
await this.
|
|
518
|
+
await this.driver.ensureWorkspace(task);
|
|
387
519
|
this.assertTaskIsActive(task);
|
|
388
|
-
|
|
520
|
+
this.assertRuntimeLease(task);
|
|
521
|
+
await this.driver.ensureRuntime(task);
|
|
389
522
|
if (this.shuttingDown) {
|
|
390
523
|
await this.stopContainerWithLock(task);
|
|
391
524
|
this.assertTaskIsActive(task);
|
|
392
525
|
}
|
|
393
526
|
}
|
|
394
|
-
async ensureWorkNetwork(task) {
|
|
395
|
-
if (!task.networkEnabled)
|
|
396
|
-
return;
|
|
397
|
-
const inspect = await this.docker([
|
|
398
|
-
'network',
|
|
399
|
-
'inspect',
|
|
400
|
-
'--format',
|
|
401
|
-
'{{index .Labels "ai.libre-webui.managed"}} {{index .Options "com.docker.network.bridge.enable_icc"}}',
|
|
402
|
-
config.networkName,
|
|
403
|
-
], { acceptFailure: true });
|
|
404
|
-
if (inspect.exitCode === 0) {
|
|
405
|
-
const [managed, icc] = inspect.stdout.trim().split(' ');
|
|
406
|
-
if (managed !== 'true' || icc !== 'false') {
|
|
407
|
-
throw new WorkRuntimeError(`Docker network "${config.networkName}" exists but is not the managed Work sandbox network (label ai.libre-webui.managed=true with inter-container communication disabled). Remove it or point WORK_NETWORK_NAME at an unused name.`, 503, 'WORK_NETWORK_CONFLICT');
|
|
408
|
-
}
|
|
409
|
-
return;
|
|
410
|
-
}
|
|
411
|
-
const created = await this.docker([
|
|
412
|
-
'network',
|
|
413
|
-
'create',
|
|
414
|
-
'--label',
|
|
415
|
-
'ai.libre-webui.managed=true',
|
|
416
|
-
'--opt',
|
|
417
|
-
'com.docker.network.bridge.enable_icc=false',
|
|
418
|
-
config.networkName,
|
|
419
|
-
], { acceptFailure: true });
|
|
420
|
-
if (created.exitCode !== 0 && !/already exists/i.test(created.stderr)) {
|
|
421
|
-
throw new WorkRuntimeError(`Could not create the Work sandbox network "${config.networkName}": ${created.stderr.trim()}`, 503, 'WORK_NETWORK_UNAVAILABLE');
|
|
422
|
-
}
|
|
423
|
-
}
|
|
424
|
-
async ensureContainer(task) {
|
|
425
|
-
this.assertRuntimeLease(task);
|
|
426
|
-
await this.ensureWorkNetwork(task);
|
|
427
|
-
if (await this.containerExists(task.containerName)) {
|
|
428
|
-
await this.assertManagedContainer(task);
|
|
429
|
-
if (!(await this.containerMatchesTaskPolicy(task))) {
|
|
430
|
-
logger.warn(`Recreating Work container ${task.containerName} because its isolation policy is stale.`);
|
|
431
|
-
await this.docker(['rm', '-f', task.containerName]);
|
|
432
|
-
await this.docker(buildWorkContainerRunArgs(task, this.image));
|
|
433
|
-
return;
|
|
434
|
-
}
|
|
435
|
-
const state = await this.docker([
|
|
436
|
-
'inspect',
|
|
437
|
-
'--format',
|
|
438
|
-
'{{.State.Running}}',
|
|
439
|
-
task.containerName,
|
|
440
|
-
]);
|
|
441
|
-
if (state.stdout.trim() !== 'true') {
|
|
442
|
-
await this.docker(['start', task.containerName]);
|
|
443
|
-
}
|
|
444
|
-
return;
|
|
445
|
-
}
|
|
446
|
-
await this.docker(buildWorkContainerRunArgs(task, this.image));
|
|
447
|
-
}
|
|
448
527
|
async recreateContainer(task) {
|
|
449
528
|
const releaseLease = this.acquireRuntimeLease(task);
|
|
450
529
|
try {
|
|
451
|
-
await this.ensureImage();
|
|
530
|
+
await this.ensureImage(task);
|
|
452
531
|
this.assertTaskIsActive(task);
|
|
453
532
|
await this.withLifecycleLock(task.id, async () => {
|
|
454
533
|
this.assertTaskIsActive(task);
|
|
@@ -477,7 +556,7 @@ export class WorkRuntimeService {
|
|
|
477
556
|
const releaseLease = this.acquireRuntimeLease(before);
|
|
478
557
|
let previewStopped = false;
|
|
479
558
|
try {
|
|
480
|
-
await this.ensureImage();
|
|
559
|
+
await this.ensureImage(before);
|
|
481
560
|
this.assertTaskIsActive(before);
|
|
482
561
|
return await this.withLifecycleLock(before.id, async () => {
|
|
483
562
|
this.assertTaskIsActive(before);
|
|
@@ -515,13 +594,10 @@ export class WorkRuntimeService {
|
|
|
515
594
|
async recreateContainerWithLock(task) {
|
|
516
595
|
this.assertRuntimeLease(task);
|
|
517
596
|
this.assertTaskIsActive(task);
|
|
518
|
-
await this.
|
|
597
|
+
await this.driver.ensureWorkspace(task);
|
|
519
598
|
this.assertTaskIsActive(task);
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
await this.docker(['rm', '-f', task.containerName]);
|
|
523
|
-
}
|
|
524
|
-
await this.ensureContainer(task);
|
|
599
|
+
await this.driver.removeRuntime(task);
|
|
600
|
+
await this.driver.ensureRuntime(task);
|
|
525
601
|
if (this.shuttingDown) {
|
|
526
602
|
await this.stopContainerWithLock(task);
|
|
527
603
|
this.assertTaskIsActive(task);
|
|
@@ -547,12 +623,7 @@ export class WorkRuntimeService {
|
|
|
547
623
|
}
|
|
548
624
|
async stopContainerWithLock(task) {
|
|
549
625
|
try {
|
|
550
|
-
|
|
551
|
-
return;
|
|
552
|
-
await this.assertManagedContainer(task);
|
|
553
|
-
await this.docker(['stop', '--time', '1', task.containerName], {
|
|
554
|
-
timeoutMs: 10000,
|
|
555
|
-
});
|
|
626
|
+
await this.driver.stopRuntime(task);
|
|
556
627
|
}
|
|
557
628
|
catch (error) {
|
|
558
629
|
this.queueFailedCleanup(task, error);
|
|
@@ -564,34 +635,11 @@ export class WorkRuntimeService {
|
|
|
564
635
|
this.assertTaskStillOwned(task);
|
|
565
636
|
}
|
|
566
637
|
else {
|
|
567
|
-
this.
|
|
638
|
+
this.assertTaskOwnerHasWorkAccess(task);
|
|
568
639
|
}
|
|
569
640
|
this.retiringTasks.add(task.id);
|
|
570
641
|
try {
|
|
571
|
-
await this.withLifecycleLock(task.id,
|
|
572
|
-
const [hasContainer, hasVolume] = await Promise.all([
|
|
573
|
-
this.containerExists(task.containerName),
|
|
574
|
-
this.volumeExists(task.volumeName),
|
|
575
|
-
]);
|
|
576
|
-
// Validate every destructive target before removing either one, so a
|
|
577
|
-
// conflicting unmanaged Docker resource cannot cause partial cleanup.
|
|
578
|
-
if (hasContainer) {
|
|
579
|
-
await this.assertManagedContainer(task);
|
|
580
|
-
}
|
|
581
|
-
if (hasVolume) {
|
|
582
|
-
await this.assertManagedVolume(task);
|
|
583
|
-
}
|
|
584
|
-
if (hasContainer) {
|
|
585
|
-
await this.docker(['rm', '-f', task.containerName], {
|
|
586
|
-
timeoutMs: 15000,
|
|
587
|
-
});
|
|
588
|
-
}
|
|
589
|
-
if (hasVolume) {
|
|
590
|
-
await this.docker(['volume', 'rm', task.volumeName], {
|
|
591
|
-
timeoutMs: 15000,
|
|
592
|
-
});
|
|
593
|
-
}
|
|
594
|
-
});
|
|
642
|
+
await this.withLifecycleLock(task.id, () => this.driver.removeTaskResources(task));
|
|
595
643
|
}
|
|
596
644
|
catch (error) {
|
|
597
645
|
this.retiringTasks.delete(task.id);
|
|
@@ -605,6 +653,8 @@ export class WorkRuntimeService {
|
|
|
605
653
|
this.activeCommands.delete(taskId);
|
|
606
654
|
this.releasePreviewLease(taskId);
|
|
607
655
|
this.runtimeLeases.delete(taskId);
|
|
656
|
+
this.taskActivity.delete(taskId);
|
|
657
|
+
workPreviewProxyService.clearPreviewUpstream(taskId);
|
|
608
658
|
}
|
|
609
659
|
async listFiles(task, requestedPath = '.') {
|
|
610
660
|
const workspacePath = validateWorkspacePath(requestedPath);
|
|
@@ -617,7 +667,7 @@ export class WorkRuntimeService {
|
|
|
617
667
|
async withWorkspaceHelperContainer(task, operation) {
|
|
618
668
|
const releaseLease = this.acquireRuntimeLease(task);
|
|
619
669
|
try {
|
|
620
|
-
await this.ensureImage();
|
|
670
|
+
await this.ensureImage(task);
|
|
621
671
|
this.assertTaskIsActive(task);
|
|
622
672
|
return await this.withLifecycleLock(task.id, async () => {
|
|
623
673
|
this.assertTaskIsActive(task);
|
|
@@ -883,7 +933,7 @@ export class WorkRuntimeService {
|
|
|
883
933
|
const releaseLease = this.acquireRuntimeLease(task);
|
|
884
934
|
let commandRegistered = false;
|
|
885
935
|
try {
|
|
886
|
-
await this.ensureImage();
|
|
936
|
+
await this.ensureImage(task);
|
|
887
937
|
this.assertTaskIsActive(task);
|
|
888
938
|
await this.withLifecycleLock(task.id, async () => {
|
|
889
939
|
this.assertTaskIsActive(task);
|
|
@@ -946,7 +996,7 @@ export class WorkRuntimeService {
|
|
|
946
996
|
const releaseLease = this.acquireRuntimeLease(task);
|
|
947
997
|
let leaseRetained = false;
|
|
948
998
|
try {
|
|
949
|
-
await this.ensureImage();
|
|
999
|
+
await this.ensureImage(task);
|
|
950
1000
|
this.assertTaskIsActive(task);
|
|
951
1001
|
const url = await this.withLifecycleLock(task.id, async () => {
|
|
952
1002
|
this.assertTaskIsActive(task);
|
|
@@ -993,19 +1043,7 @@ export class WorkRuntimeService {
|
|
|
993
1043
|
}
|
|
994
1044
|
async detectPreviewLaunch(task) {
|
|
995
1045
|
try {
|
|
996
|
-
const result = await this.
|
|
997
|
-
'exec',
|
|
998
|
-
'--user',
|
|
999
|
-
'1000:1000',
|
|
1000
|
-
'--workdir',
|
|
1001
|
-
'/workspace',
|
|
1002
|
-
task.containerName,
|
|
1003
|
-
'node',
|
|
1004
|
-
'-e',
|
|
1005
|
-
PREVIEW_TARGET_SCRIPT,
|
|
1006
|
-
'--',
|
|
1007
|
-
'/workspace',
|
|
1008
|
-
], {
|
|
1046
|
+
const result = await this.driver.exec(task, ['node', '-e', PREVIEW_TARGET_SCRIPT, '--', '/workspace'], {
|
|
1009
1047
|
acceptFailure: true,
|
|
1010
1048
|
timeoutMs: 5000,
|
|
1011
1049
|
maxOutputChars: 10000,
|
|
@@ -1056,13 +1094,7 @@ export class WorkRuntimeService {
|
|
|
1056
1094
|
}
|
|
1057
1095
|
async startPreviewPrepared(task, previewLaunch) {
|
|
1058
1096
|
try {
|
|
1059
|
-
const launch = await this.
|
|
1060
|
-
'exec',
|
|
1061
|
-
'--user',
|
|
1062
|
-
'1000:1000',
|
|
1063
|
-
'--workdir',
|
|
1064
|
-
previewLaunch.workdir,
|
|
1065
|
-
task.containerName,
|
|
1097
|
+
const launch = await this.driver.exec(task, [
|
|
1066
1098
|
'/bin/bash',
|
|
1067
1099
|
'-lc',
|
|
1068
1100
|
`if [ -s /tmp/libre-work-preview.pid ] &&
|
|
@@ -1085,17 +1117,17 @@ export class WorkRuntimeService {
|
|
|
1085
1117
|
? STATIC_PREVIEW_SERVER_SCRIPT
|
|
1086
1118
|
: previewLaunch.command,
|
|
1087
1119
|
String(config.previewPort),
|
|
1088
|
-
], {
|
|
1120
|
+
], {
|
|
1121
|
+
workdir: previewLaunch.workdir,
|
|
1122
|
+
acceptFailure: true,
|
|
1123
|
+
timeoutMs: 5000,
|
|
1124
|
+
});
|
|
1089
1125
|
if (launch.exitCode !== 0 && launch.exitCode !== 17) {
|
|
1090
1126
|
throw await this.previewStartError(task, 'Preview process could not be launched.', launch.stderr);
|
|
1091
1127
|
}
|
|
1092
1128
|
const deadline = Date.now() + PREVIEW_READY_TIMEOUT_MS;
|
|
1093
1129
|
while (true) {
|
|
1094
|
-
const readiness = await this.
|
|
1095
|
-
'exec',
|
|
1096
|
-
'--user',
|
|
1097
|
-
'1000:1000',
|
|
1098
|
-
task.containerName,
|
|
1130
|
+
const readiness = await this.driver.exec(task, [
|
|
1099
1131
|
'node',
|
|
1100
1132
|
'-e',
|
|
1101
1133
|
PREVIEW_READY_SCRIPT,
|
|
@@ -1112,16 +1144,11 @@ export class WorkRuntimeService {
|
|
|
1112
1144
|
}
|
|
1113
1145
|
await new Promise(resolve => setTimeout(resolve, PREVIEW_POLL_INTERVAL_MS));
|
|
1114
1146
|
}
|
|
1115
|
-
const
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
`${config.previewPort}/tcp`,
|
|
1119
|
-
]);
|
|
1120
|
-
const publishedPort = parsePublishedPort(portResult.stdout, config.previewPort);
|
|
1121
|
-
if (!publishedPort) {
|
|
1122
|
-
throw new WorkRuntimeError('Docker did not publish the preview port.', 503, 'WORK_PREVIEW_PORT_UNAVAILABLE');
|
|
1147
|
+
const endpoint = await this.driver.previewEndpoint(task);
|
|
1148
|
+
if (!endpoint) {
|
|
1149
|
+
throw new WorkRuntimeError('The runtime did not expose the preview port.', 503, 'WORK_PREVIEW_PORT_UNAVAILABLE');
|
|
1123
1150
|
}
|
|
1124
|
-
return workPreviewProxyService.createPreviewUrl(task.id,
|
|
1151
|
+
return workPreviewProxyService.createPreviewUrl(task.id, endpoint.port, endpoint.host);
|
|
1125
1152
|
}
|
|
1126
1153
|
catch (error) {
|
|
1127
1154
|
try {
|
|
@@ -1134,11 +1161,7 @@ export class WorkRuntimeService {
|
|
|
1134
1161
|
}
|
|
1135
1162
|
}
|
|
1136
1163
|
async previewStartError(task, reason, detail = '') {
|
|
1137
|
-
const log = await this.
|
|
1138
|
-
'exec',
|
|
1139
|
-
'--user',
|
|
1140
|
-
'1000:1000',
|
|
1141
|
-
task.containerName,
|
|
1164
|
+
const log = await this.driver.exec(task, [
|
|
1142
1165
|
'/bin/bash',
|
|
1143
1166
|
'-lc',
|
|
1144
1167
|
'tail -c 4000 /tmp/libre-work-preview.log 2>/dev/null || true',
|
|
@@ -1189,22 +1212,9 @@ export class WorkRuntimeService {
|
|
|
1189
1212
|
});
|
|
1190
1213
|
}
|
|
1191
1214
|
async previewProcessCheckWithLock(task) {
|
|
1192
|
-
if (
|
|
1215
|
+
if ((await this.driver.runtimeState(task)) !== 'running')
|
|
1193
1216
|
return 'absent';
|
|
1194
|
-
await this.
|
|
1195
|
-
if (!(await this.containerIsRunning(task.containerName)))
|
|
1196
|
-
return 'absent';
|
|
1197
|
-
const readiness = await this.docker([
|
|
1198
|
-
'exec',
|
|
1199
|
-
'--user',
|
|
1200
|
-
'1000:1000',
|
|
1201
|
-
task.containerName,
|
|
1202
|
-
'node',
|
|
1203
|
-
'-e',
|
|
1204
|
-
PREVIEW_READY_SCRIPT,
|
|
1205
|
-
'--',
|
|
1206
|
-
String(config.previewPort),
|
|
1207
|
-
], { acceptFailure: true, timeoutMs: 2000, maxOutputChars: 2000 });
|
|
1217
|
+
const readiness = await this.driver.exec(task, ['node', '-e', PREVIEW_READY_SCRIPT, '--', String(config.previewPort)], { acceptFailure: true, timeoutMs: 2000, maxOutputChars: 2000 });
|
|
1208
1218
|
if (readiness.exitCode === 0)
|
|
1209
1219
|
return 'ready';
|
|
1210
1220
|
if (readiness.exitCode === 3)
|
|
@@ -1214,208 +1224,37 @@ export class WorkRuntimeService {
|
|
|
1214
1224
|
throw new WorkRuntimeError(readiness.stderr.trim() || 'Could not inspect the Work preview.', 503, 'WORK_PREVIEW_INSPECT_FAILED');
|
|
1215
1225
|
}
|
|
1216
1226
|
async stopPreviewPrepared(task) {
|
|
1227
|
+
workPreviewProxyService.clearPreviewUpstream(task.id);
|
|
1217
1228
|
// Stop the container, not only the recorded process group. A custom
|
|
1218
1229
|
// preview command can intentionally double-fork or create a new session;
|
|
1219
1230
|
// Docker's container boundary guarantees those descendants are gone.
|
|
1220
1231
|
// The named /workspace volume remains persistent across the restart.
|
|
1221
1232
|
await this.stopContainerWithLock(task);
|
|
1222
1233
|
}
|
|
1223
|
-
async ensureImage() {
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
return;
|
|
1232
|
-
logger.info(`Pulling Work runtime image ${this.image}`);
|
|
1233
|
-
await this.docker(['pull', this.image], { timeoutMs: 900000 });
|
|
1234
|
-
})().catch(error => {
|
|
1235
|
-
this.imagePreparation = undefined;
|
|
1234
|
+
async ensureImage(task) {
|
|
1235
|
+
// Pulls are deduplicated per image, not globally: tasks under different
|
|
1236
|
+
// policies may run different images.
|
|
1237
|
+
const image = workPolicyService.resolve(task.policyId).image;
|
|
1238
|
+
let preparation = this.imagePreparations.get(image);
|
|
1239
|
+
if (!preparation) {
|
|
1240
|
+
preparation = this.driver.ensureImage(image).catch(error => {
|
|
1241
|
+
this.imagePreparations.delete(image);
|
|
1236
1242
|
throw error;
|
|
1237
1243
|
});
|
|
1244
|
+
this.imagePreparations.set(image, preparation);
|
|
1238
1245
|
}
|
|
1239
|
-
await
|
|
1240
|
-
}
|
|
1241
|
-
async initializeVolume(task) {
|
|
1242
|
-
await this.docker([
|
|
1243
|
-
'run',
|
|
1244
|
-
'--rm',
|
|
1245
|
-
'--network',
|
|
1246
|
-
'none',
|
|
1247
|
-
'--read-only',
|
|
1248
|
-
'--tmpfs',
|
|
1249
|
-
'/tmp:rw,noexec,nosuid,size=64m',
|
|
1250
|
-
'--cap-drop',
|
|
1251
|
-
'ALL',
|
|
1252
|
-
'--cap-add',
|
|
1253
|
-
'CHOWN',
|
|
1254
|
-
'--mount',
|
|
1255
|
-
`type=volume,src=${task.volumeName},dst=/workspace`,
|
|
1256
|
-
this.image,
|
|
1257
|
-
'chown',
|
|
1258
|
-
'-R',
|
|
1259
|
-
'1000:1000',
|
|
1260
|
-
'/workspace',
|
|
1261
|
-
]);
|
|
1262
|
-
}
|
|
1263
|
-
async ensureVolume(task) {
|
|
1264
|
-
if (await this.volumeExists(task.volumeName)) {
|
|
1265
|
-
await this.assertManagedVolume(task);
|
|
1266
|
-
return;
|
|
1267
|
-
}
|
|
1268
|
-
await this.docker([
|
|
1269
|
-
'volume',
|
|
1270
|
-
'create',
|
|
1271
|
-
'--label',
|
|
1272
|
-
'ai.libre-webui.managed=true',
|
|
1273
|
-
'--label',
|
|
1274
|
-
`ai.libre-webui.task=${task.id}`,
|
|
1275
|
-
task.volumeName,
|
|
1276
|
-
]);
|
|
1277
|
-
// Docker returns an existing volume from `volume create` if another
|
|
1278
|
-
// process wins the name race, so prove ownership before mounting it.
|
|
1279
|
-
await this.assertManagedVolume(task);
|
|
1280
|
-
try {
|
|
1281
|
-
await this.initializeVolume(task);
|
|
1282
|
-
}
|
|
1283
|
-
catch (error) {
|
|
1284
|
-
await this.docker(['volume', 'rm', task.volumeName], {
|
|
1285
|
-
acceptFailure: true,
|
|
1286
|
-
});
|
|
1287
|
-
throw error;
|
|
1288
|
-
}
|
|
1289
|
-
}
|
|
1290
|
-
async volumeExists(name) {
|
|
1291
|
-
const result = await this.docker(['volume', 'inspect', name], {
|
|
1292
|
-
timeoutMs: 5000,
|
|
1293
|
-
acceptFailure: true,
|
|
1294
|
-
});
|
|
1295
|
-
if (result.exitCode === 0)
|
|
1296
|
-
return true;
|
|
1297
|
-
if (/no such volume/i.test(`${result.stderr}\n${result.stdout}`)) {
|
|
1298
|
-
return false;
|
|
1299
|
-
}
|
|
1300
|
-
throw new WorkRuntimeError(`Could not inspect Work volume "${name}": ${result.stderr.trim() || result.stdout.trim() || `Docker exited with code ${result.exitCode}.`}`, 503, 'WORK_DOCKER_INSPECT_FAILED');
|
|
1301
|
-
}
|
|
1302
|
-
async containerExists(name) {
|
|
1303
|
-
const result = await this.docker(['container', 'inspect', name], {
|
|
1304
|
-
timeoutMs: 5000,
|
|
1305
|
-
acceptFailure: true,
|
|
1306
|
-
});
|
|
1307
|
-
if (result.exitCode === 0)
|
|
1308
|
-
return true;
|
|
1309
|
-
if (/no such (?:container|object)/i.test(`${result.stderr}\n${result.stdout}`)) {
|
|
1310
|
-
return false;
|
|
1311
|
-
}
|
|
1312
|
-
throw new WorkRuntimeError(`Could not inspect Work container "${name}": ${result.stderr.trim() || result.stdout.trim() || `Docker exited with code ${result.exitCode}.`}`, 503, 'WORK_DOCKER_INSPECT_FAILED');
|
|
1313
|
-
}
|
|
1314
|
-
async containerIsRunning(name) {
|
|
1315
|
-
const result = await this.docker([
|
|
1316
|
-
'inspect',
|
|
1317
|
-
'--format',
|
|
1318
|
-
'{{.State.Running}}',
|
|
1319
|
-
name,
|
|
1320
|
-
]);
|
|
1321
|
-
const state = result.stdout.trim();
|
|
1322
|
-
if (state === 'true')
|
|
1323
|
-
return true;
|
|
1324
|
-
if (state === 'false')
|
|
1325
|
-
return false;
|
|
1326
|
-
throw new WorkRuntimeError(`Docker returned an invalid state for Work container "${name}".`, 503, 'WORK_DOCKER_INSPECT_FAILED');
|
|
1327
|
-
}
|
|
1328
|
-
async assertManagedContainer(task) {
|
|
1329
|
-
const result = await this.docker([
|
|
1330
|
-
'inspect',
|
|
1331
|
-
'--format',
|
|
1332
|
-
'{{ index .Config.Labels "ai.libre-webui.task" }}',
|
|
1333
|
-
task.containerName,
|
|
1334
|
-
]);
|
|
1335
|
-
if (result.stdout.trim() !== task.id) {
|
|
1336
|
-
throw new WorkRuntimeError(`Refusing to operate unmanaged container "${task.containerName}".`, 409, 'WORK_CONTAINER_NAME_CONFLICT');
|
|
1337
|
-
}
|
|
1338
|
-
}
|
|
1339
|
-
async containerMatchesTaskPolicy(task) {
|
|
1340
|
-
const result = await this.docker([
|
|
1341
|
-
'inspect',
|
|
1342
|
-
'--format',
|
|
1343
|
-
'{{json .}}',
|
|
1344
|
-
task.containerName,
|
|
1345
|
-
]);
|
|
1346
|
-
let inspected;
|
|
1347
|
-
try {
|
|
1348
|
-
inspected = JSON.parse(result.stdout);
|
|
1349
|
-
}
|
|
1350
|
-
catch {
|
|
1351
|
-
return false;
|
|
1352
|
-
}
|
|
1353
|
-
const containerConfig = objectRecord(inspected.Config);
|
|
1354
|
-
const hostConfig = objectRecord(inspected.HostConfig);
|
|
1355
|
-
const labels = objectRecord(containerConfig.Labels);
|
|
1356
|
-
const env = stringArray(containerConfig.Env);
|
|
1357
|
-
const capDrop = stringArray(hostConfig.CapDrop);
|
|
1358
|
-
const securityOpt = stringArray(hostConfig.SecurityOpt);
|
|
1359
|
-
const command = stringArray(containerConfig.Cmd);
|
|
1360
|
-
const mounts = Array.isArray(inspected.Mounts)
|
|
1361
|
-
? inspected.Mounts.map(objectRecord)
|
|
1362
|
-
: [];
|
|
1363
|
-
const workspaceMount = mounts.find(mount => mount.Destination === '/workspace');
|
|
1364
|
-
const expectedNetwork = task.networkEnabled ? config.networkName : 'none';
|
|
1365
|
-
const portBindings = objectRecord(hostConfig.PortBindings);
|
|
1366
|
-
const previewBindings = Array.isArray(portBindings[`${config.previewPort}/tcp`])
|
|
1367
|
-
? portBindings[`${config.previewPort}/tcp`].map(objectRecord)
|
|
1368
|
-
: [];
|
|
1369
|
-
const portPolicyMatches = task.networkEnabled
|
|
1370
|
-
? Object.keys(portBindings).length === 1 &&
|
|
1371
|
-
previewBindings.length === 1 &&
|
|
1372
|
-
previewBindings[0]?.HostIp === config.previewBind
|
|
1373
|
-
: Object.keys(portBindings).length === 0;
|
|
1374
|
-
return (labels['ai.libre-webui.managed'] === 'true' &&
|
|
1375
|
-
labels['ai.libre-webui.task'] === task.id &&
|
|
1376
|
-
labels['ai.libre-webui.policy'] === runtimePolicyFingerprint &&
|
|
1377
|
-
containerConfig.Image === this.image &&
|
|
1378
|
-
containerConfig.User === '1000:1000' &&
|
|
1379
|
-
containerConfig.WorkingDir === '/workspace' &&
|
|
1380
|
-
command.join('\0') === ['tail', '-f', '/dev/null'].join('\0') &&
|
|
1381
|
-
env.includes('HOME=/tmp') &&
|
|
1382
|
-
env.includes('NPM_CONFIG_CACHE=/tmp/npm-cache') &&
|
|
1383
|
-
hostConfig.ReadonlyRootfs === true &&
|
|
1384
|
-
hostConfig.Privileged === false &&
|
|
1385
|
-
capDrop.includes('ALL') &&
|
|
1386
|
-
securityOpt.includes('no-new-privileges') &&
|
|
1387
|
-
Number(hostConfig.PidsLimit) === config.pidsLimit &&
|
|
1388
|
-
Number(hostConfig.Memory) > 0 &&
|
|
1389
|
-
Number(hostConfig.MemorySwap) === Number(hostConfig.Memory) &&
|
|
1390
|
-
Number(hostConfig.NanoCpus) > 0 &&
|
|
1391
|
-
hostConfig.NetworkMode === expectedNetwork &&
|
|
1392
|
-
portPolicyMatches &&
|
|
1393
|
-
workspaceMount?.Type === 'volume' &&
|
|
1394
|
-
workspaceMount.Name === task.volumeName &&
|
|
1395
|
-
workspaceMount.RW === true);
|
|
1396
|
-
}
|
|
1397
|
-
async assertManagedVolume(task) {
|
|
1398
|
-
const result = await this.docker([
|
|
1399
|
-
'volume',
|
|
1400
|
-
'inspect',
|
|
1401
|
-
'--format',
|
|
1402
|
-
'{{ index .Labels "ai.libre-webui.task" }}',
|
|
1403
|
-
task.volumeName,
|
|
1404
|
-
]);
|
|
1405
|
-
if (result.stdout.trim() !== task.id) {
|
|
1406
|
-
throw new WorkRuntimeError(`Refusing to operate unmanaged volume "${task.volumeName}".`, 409, 'WORK_VOLUME_NAME_CONFLICT');
|
|
1407
|
-
}
|
|
1246
|
+
await preparation;
|
|
1408
1247
|
}
|
|
1409
1248
|
assertTaskIsActive(task) {
|
|
1410
1249
|
this.assertAcceptingWork();
|
|
1411
1250
|
if (this.retiringTasks.has(task.id)) {
|
|
1412
1251
|
throw new WorkRuntimeError('This Work task is being deleted.', 409, 'WORK_TASK_REMOVING');
|
|
1413
1252
|
}
|
|
1414
|
-
this.
|
|
1253
|
+
this.assertTaskOwnerHasWorkAccess(task);
|
|
1415
1254
|
}
|
|
1416
|
-
|
|
1255
|
+
assertTaskOwnerHasWorkAccess(task) {
|
|
1417
1256
|
const access = getDatabase()
|
|
1418
|
-
.prepare(`SELECT users.role
|
|
1257
|
+
.prepare(`SELECT users.role AS role, users.account_status AS status
|
|
1419
1258
|
FROM work_tasks
|
|
1420
1259
|
JOIN users ON users.id = work_tasks.user_id
|
|
1421
1260
|
WHERE work_tasks.id = ? AND work_tasks.user_id = ?`)
|
|
@@ -1423,8 +1262,8 @@ export class WorkRuntimeService {
|
|
|
1423
1262
|
if (!access) {
|
|
1424
1263
|
throw new WorkRuntimeError('This Work task no longer exists.', 409, 'WORK_TASK_REMOVING');
|
|
1425
1264
|
}
|
|
1426
|
-
if (access
|
|
1427
|
-
throw new WorkRuntimeError('
|
|
1265
|
+
if (!userHasWorkAccess(access)) {
|
|
1266
|
+
throw new WorkRuntimeError('Work access for this task was revoked.', 403, 'WORK_ACCESS_REVOKED');
|
|
1428
1267
|
}
|
|
1429
1268
|
}
|
|
1430
1269
|
assertTaskStillOwned(task) {
|
|
@@ -1593,106 +1432,41 @@ export class WorkRuntimeService {
|
|
|
1593
1432
|
return this.exec(task, buildWorkGitCommand(args), options);
|
|
1594
1433
|
}
|
|
1595
1434
|
async exec(task, command, options = {}) {
|
|
1596
|
-
const args = [
|
|
1597
|
-
'exec',
|
|
1598
|
-
...(options.input !== undefined ? ['--interactive'] : []),
|
|
1599
|
-
'--user',
|
|
1600
|
-
'1000:1000',
|
|
1601
|
-
'--workdir',
|
|
1602
|
-
'/workspace',
|
|
1603
|
-
task.containerName,
|
|
1604
|
-
...command,
|
|
1605
|
-
];
|
|
1606
|
-
return this.docker(args, options);
|
|
1607
|
-
}
|
|
1608
|
-
async docker(args, options = {}) {
|
|
1609
1435
|
try {
|
|
1610
|
-
return await
|
|
1436
|
+
return await this.driver.exec(task, command, options);
|
|
1611
1437
|
}
|
|
1612
|
-
|
|
1613
|
-
|
|
1614
|
-
throw error;
|
|
1615
|
-
throw new WorkRuntimeError(error instanceof Error ? error.message : 'Docker command failed.');
|
|
1438
|
+
finally {
|
|
1439
|
+
this.noteTaskActivity(task.id);
|
|
1616
1440
|
}
|
|
1617
1441
|
}
|
|
1618
1442
|
}
|
|
1619
|
-
/** Bracket a bare IPv6 literal so it can carry a port in a URL. */
|
|
1620
|
-
export function formatPreviewHost(host) {
|
|
1621
|
-
return host.includes(':') && !host.startsWith('[') ? `[${host}]` : host;
|
|
1622
|
-
}
|
|
1623
1443
|
/**
|
|
1624
|
-
*
|
|
1625
|
-
*
|
|
1626
|
-
*
|
|
1627
|
-
*
|
|
1444
|
+
* Decide the minimal startup cleanup from the task inventory and the labeled
|
|
1445
|
+
* containers Docker actually has. A task without a container needs nothing —
|
|
1446
|
+
* which is the common case, and why recovery is no longer O(tasks) docker
|
|
1447
|
+
* calls. Ownership is decided by the task label alone (stamped at creation
|
|
1448
|
+
* together with the managed label): a managed container whose label matches
|
|
1449
|
+
* no inventory row is unowned, and stopping it through a task record would
|
|
1450
|
+
* be refused by the ownership assertion anyway.
|
|
1628
1451
|
*/
|
|
1629
|
-
export function
|
|
1630
|
-
const
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1634
|
-
|
|
1635
|
-
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1452
|
+
export function planStartupReconciliation(tasks, containers) {
|
|
1453
|
+
const byId = new Map(tasks.map(task => [task.id, task]));
|
|
1454
|
+
const stop = new Map();
|
|
1455
|
+
const removeOrphans = [];
|
|
1456
|
+
let atRest = 0;
|
|
1457
|
+
for (const container of containers) {
|
|
1458
|
+
const task = byId.get(container.taskId);
|
|
1459
|
+
if (!task) {
|
|
1460
|
+
removeOrphans.push(container);
|
|
1461
|
+
}
|
|
1462
|
+
else if (container.running) {
|
|
1463
|
+
stop.set(task.id, task);
|
|
1464
|
+
}
|
|
1465
|
+
else {
|
|
1466
|
+
atRest += 1;
|
|
1467
|
+
}
|
|
1641
1468
|
}
|
|
1642
|
-
return
|
|
1643
|
-
}
|
|
1644
|
-
export function buildWorkContainerRunArgs(task, image = config.image) {
|
|
1645
|
-
const args = [
|
|
1646
|
-
'run',
|
|
1647
|
-
'--detach',
|
|
1648
|
-
'--name',
|
|
1649
|
-
task.containerName,
|
|
1650
|
-
'--init',
|
|
1651
|
-
'--label',
|
|
1652
|
-
'ai.libre-webui.managed=true',
|
|
1653
|
-
'--label',
|
|
1654
|
-
`ai.libre-webui.task=${task.id}`,
|
|
1655
|
-
'--label',
|
|
1656
|
-
`ai.libre-webui.policy=${runtimePolicyFingerprint}`,
|
|
1657
|
-
'--user',
|
|
1658
|
-
'1000:1000',
|
|
1659
|
-
'--workdir',
|
|
1660
|
-
'/workspace',
|
|
1661
|
-
'--read-only',
|
|
1662
|
-
'--tmpfs',
|
|
1663
|
-
'/tmp:rw,nosuid,size=512m',
|
|
1664
|
-
'--env',
|
|
1665
|
-
'HOME=/tmp',
|
|
1666
|
-
'--env',
|
|
1667
|
-
'NPM_CONFIG_CACHE=/tmp/npm-cache',
|
|
1668
|
-
'--cap-drop',
|
|
1669
|
-
'ALL',
|
|
1670
|
-
'--security-opt',
|
|
1671
|
-
'no-new-privileges',
|
|
1672
|
-
'--pids-limit',
|
|
1673
|
-
String(config.pidsLimit),
|
|
1674
|
-
'--memory',
|
|
1675
|
-
config.memoryLimit,
|
|
1676
|
-
// Same value as --memory: the memory cap cannot be sidestepped via swap.
|
|
1677
|
-
'--memory-swap',
|
|
1678
|
-
config.memoryLimit,
|
|
1679
|
-
'--cpus',
|
|
1680
|
-
config.cpuLimit,
|
|
1681
|
-
'--network',
|
|
1682
|
-
task.networkEnabled ? config.networkName : 'none',
|
|
1683
|
-
];
|
|
1684
|
-
if (task.networkEnabled) {
|
|
1685
|
-
args.push('--publish', `${config.previewBind}::${config.previewPort}`);
|
|
1686
|
-
for (const server of config.dnsServers) {
|
|
1687
|
-
args.push('--dns', server);
|
|
1688
|
-
}
|
|
1689
|
-
}
|
|
1690
|
-
args.push('--mount', task.hostPath
|
|
1691
|
-
? // Host-folder workspaces are opt-in per deployment and validated
|
|
1692
|
-
// against an allowlist before ever reaching this point.
|
|
1693
|
-
`type=bind,src=${task.hostPath},dst=/workspace`
|
|
1694
|
-
: `type=volume,src=${task.volumeName},dst=/workspace,volume-nocopy`, image, 'tail', '-f', '/dev/null');
|
|
1695
|
-
return args;
|
|
1469
|
+
return { stop: [...stop.values()], removeOrphans, atRest };
|
|
1696
1470
|
}
|
|
1697
1471
|
export function validateWorkspacePath(input, allowRoot = true) {
|
|
1698
1472
|
const value = String(input || '.').trim();
|
|
@@ -1730,22 +1504,6 @@ function validateGitIdentity(value, field) {
|
|
|
1730
1504
|
}
|
|
1731
1505
|
return value;
|
|
1732
1506
|
}
|
|
1733
|
-
export function parsePublishedPort(output, _containerPort = config.previewPort, bindAddress = config.previewBind) {
|
|
1734
|
-
// Only a binding on the interface this deployment asked for counts. The
|
|
1735
|
-
// default stays loopback, so a stray wildcard binding is still rejected.
|
|
1736
|
-
const allowed = new Set(bindAddress === WORK_RUNTIME_DEFAULTS.previewBind
|
|
1737
|
-
? ['127.0.0.1', '[::1]']
|
|
1738
|
-
: [bindAddress, formatPreviewHost(bindAddress)]);
|
|
1739
|
-
for (const line of output.trim().split(/\r?\n/)) {
|
|
1740
|
-
const match = line.match(/^(.*):(\d+)$/);
|
|
1741
|
-
if (!match || !allowed.has(match[1]))
|
|
1742
|
-
continue;
|
|
1743
|
-
const port = Number(match[2]);
|
|
1744
|
-
if (Number.isInteger(port) && port > 0 && port <= 65535)
|
|
1745
|
-
return port;
|
|
1746
|
-
}
|
|
1747
|
-
return undefined;
|
|
1748
|
-
}
|
|
1749
1507
|
async function waitForAbortSignal(promise, signal) {
|
|
1750
1508
|
if (!signal)
|
|
1751
1509
|
return promise;
|
|
@@ -1767,98 +1525,6 @@ async function waitForAbortSignal(promise, signal) {
|
|
|
1767
1525
|
});
|
|
1768
1526
|
});
|
|
1769
1527
|
}
|
|
1770
|
-
async function runProcess(command, args, options) {
|
|
1771
|
-
const timeoutMs = options.timeoutMs ?? config.commandTimeoutMs;
|
|
1772
|
-
const maxOutputChars = options.maxOutputChars ?? config.maxOutputChars;
|
|
1773
|
-
return new Promise((resolve, reject) => {
|
|
1774
|
-
const child = spawn(command, args, {
|
|
1775
|
-
stdio: ['pipe', 'pipe', 'pipe'],
|
|
1776
|
-
env: process.env,
|
|
1777
|
-
});
|
|
1778
|
-
activeDockerProcesses.add(child);
|
|
1779
|
-
let stdout = '';
|
|
1780
|
-
let stderr = '';
|
|
1781
|
-
let truncated = false;
|
|
1782
|
-
let settled = false;
|
|
1783
|
-
let stdinError;
|
|
1784
|
-
let timer;
|
|
1785
|
-
const claimSettlement = () => {
|
|
1786
|
-
if (settled)
|
|
1787
|
-
return false;
|
|
1788
|
-
settled = true;
|
|
1789
|
-
if (timer)
|
|
1790
|
-
clearTimeout(timer);
|
|
1791
|
-
return true;
|
|
1792
|
-
};
|
|
1793
|
-
const append = (current, chunk) => {
|
|
1794
|
-
const next = current + chunk.toString('utf8');
|
|
1795
|
-
if (next.length <= maxOutputChars)
|
|
1796
|
-
return next;
|
|
1797
|
-
truncated = true;
|
|
1798
|
-
const half = Math.floor(maxOutputChars / 2);
|
|
1799
|
-
return `${next.slice(0, half)}\n... output truncated ...\n${next.slice(-half)}`;
|
|
1800
|
-
};
|
|
1801
|
-
child.stdout.on('data', chunk => {
|
|
1802
|
-
stdout = append(stdout, chunk);
|
|
1803
|
-
});
|
|
1804
|
-
child.stderr.on('data', chunk => {
|
|
1805
|
-
stderr = append(stderr, chunk);
|
|
1806
|
-
});
|
|
1807
|
-
child.stdin.on('error', error => {
|
|
1808
|
-
// A helper may reject the request and close stdin before a large input
|
|
1809
|
-
// has finished writing. EPIPE is then expected; the process exit carries
|
|
1810
|
-
// the useful error. Other stdin failures are reported when it closes.
|
|
1811
|
-
if (error.code !== 'EPIPE') {
|
|
1812
|
-
stdinError = error;
|
|
1813
|
-
}
|
|
1814
|
-
});
|
|
1815
|
-
child.on('error', error => {
|
|
1816
|
-
activeDockerProcesses.delete(child);
|
|
1817
|
-
if (!claimSettlement())
|
|
1818
|
-
return;
|
|
1819
|
-
reject(new WorkRuntimeError(`Could not start ${command}: ${error.message}`, 503, 'WORK_DOCKER_UNAVAILABLE'));
|
|
1820
|
-
});
|
|
1821
|
-
child.on('close', (code, signal) => {
|
|
1822
|
-
activeDockerProcesses.delete(child);
|
|
1823
|
-
const result = {
|
|
1824
|
-
exitCode: code ?? -1,
|
|
1825
|
-
stdout,
|
|
1826
|
-
stderr,
|
|
1827
|
-
truncated,
|
|
1828
|
-
signal: signal || undefined,
|
|
1829
|
-
};
|
|
1830
|
-
if (stdinError && !options.acceptFailure) {
|
|
1831
|
-
if (!claimSettlement())
|
|
1832
|
-
return;
|
|
1833
|
-
reject(new WorkRuntimeError(`Could not write command input: ${stdinError.message}`, 503, 'WORK_DOCKER_STDIN_FAILED'));
|
|
1834
|
-
return;
|
|
1835
|
-
}
|
|
1836
|
-
if (result.exitCode !== 0 && !options.acceptFailure) {
|
|
1837
|
-
if (!claimSettlement())
|
|
1838
|
-
return;
|
|
1839
|
-
reject(new WorkRuntimeError(stderr.trim() ||
|
|
1840
|
-
stdout.trim() ||
|
|
1841
|
-
`${command} exited with code ${result.exitCode}.`, 503, 'WORK_DOCKER_COMMAND_FAILED'));
|
|
1842
|
-
return;
|
|
1843
|
-
}
|
|
1844
|
-
if (claimSettlement())
|
|
1845
|
-
resolve(result);
|
|
1846
|
-
});
|
|
1847
|
-
timer = setTimeout(() => {
|
|
1848
|
-
child.kill('SIGKILL');
|
|
1849
|
-
if (!claimSettlement())
|
|
1850
|
-
return;
|
|
1851
|
-
reject(new WorkRuntimeError(`Command timed out after ${timeoutMs}ms.`, 504, 'WORK_COMMAND_TIMEOUT'));
|
|
1852
|
-
}, timeoutMs);
|
|
1853
|
-
timer.unref();
|
|
1854
|
-
if (options.input !== undefined) {
|
|
1855
|
-
child.stdin.end(options.input);
|
|
1856
|
-
}
|
|
1857
|
-
else {
|
|
1858
|
-
child.stdin.end();
|
|
1859
|
-
}
|
|
1860
|
-
});
|
|
1861
|
-
}
|
|
1862
1528
|
function parseJsonOutput(output) {
|
|
1863
1529
|
try {
|
|
1864
1530
|
return JSON.parse(output.trim());
|
|
@@ -1867,35 +1533,6 @@ function parseJsonOutput(output) {
|
|
|
1867
1533
|
throw new WorkRuntimeError('Workspace helper returned an invalid response.', 500, 'WORK_HELPER_INVALID_RESPONSE');
|
|
1868
1534
|
}
|
|
1869
1535
|
}
|
|
1870
|
-
function positiveInteger(value, fallback) {
|
|
1871
|
-
const parsed = Number(value);
|
|
1872
|
-
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
|
1873
|
-
}
|
|
1874
|
-
export function parseDnsServers(value) {
|
|
1875
|
-
if (!value)
|
|
1876
|
-
return [];
|
|
1877
|
-
return value
|
|
1878
|
-
.split(',')
|
|
1879
|
-
.map(server => server.trim())
|
|
1880
|
-
.filter(server => {
|
|
1881
|
-
if (!server)
|
|
1882
|
-
return false;
|
|
1883
|
-
if (/^[0-9a-fA-F:.]+$/.test(server))
|
|
1884
|
-
return true;
|
|
1885
|
-
logger.warn(`Ignoring WORK_RUNTIME_DNS entry "${server}": not an IPv4/IPv6 address.`);
|
|
1886
|
-
return false;
|
|
1887
|
-
});
|
|
1888
|
-
}
|
|
1889
|
-
function objectRecord(value) {
|
|
1890
|
-
return value && typeof value === 'object'
|
|
1891
|
-
? value
|
|
1892
|
-
: {};
|
|
1893
|
-
}
|
|
1894
|
-
function stringArray(value) {
|
|
1895
|
-
return Array.isArray(value)
|
|
1896
|
-
? value.filter((item) => typeof item === 'string')
|
|
1897
|
-
: [];
|
|
1898
|
-
}
|
|
1899
1536
|
function isSafePreviewWorkdir(value) {
|
|
1900
1537
|
if (typeof value !== 'string' ||
|
|
1901
1538
|
value.length > 4096 ||
|
|
@@ -2428,6 +2065,21 @@ const walk = directory => {
|
|
|
2428
2065
|
walk(targetReal);
|
|
2429
2066
|
process.stdout.write(results.join('\\n'));
|
|
2430
2067
|
`;
|
|
2431
|
-
|
|
2068
|
+
/**
|
|
2069
|
+
* Select the runtime backend for this deployment. Docker remains the
|
|
2070
|
+
* default; WORK_RUNTIME_BACKEND=kubernetes runs sandboxes as Pods in a
|
|
2071
|
+
* namespace instead (no Docker daemon or socket anywhere). An unknown value
|
|
2072
|
+
* fails startup loudly rather than silently running the wrong backend.
|
|
2073
|
+
*/
|
|
2074
|
+
export function createWorkRuntimeDriver(backend = process.env.WORK_RUNTIME_BACKEND) {
|
|
2075
|
+
const selected = backend?.trim() || 'docker';
|
|
2076
|
+
if (selected === 'docker')
|
|
2077
|
+
return new DockerWorkRuntimeDriver();
|
|
2078
|
+
if (selected === 'kubernetes')
|
|
2079
|
+
return new KubernetesWorkRuntimeDriver();
|
|
2080
|
+
throw new WorkRuntimeError(`Unknown WORK_RUNTIME_BACKEND "${selected}". Use "docker" or "kubernetes".`, 503, 'WORK_RUNTIME_BACKEND_INVALID');
|
|
2081
|
+
}
|
|
2082
|
+
export { KubernetesWorkRuntimeDriver } from './workKubernetesDriver.js';
|
|
2083
|
+
export const workRuntimeService = new WorkRuntimeService(createWorkRuntimeDriver());
|
|
2432
2084
|
export default workRuntimeService;
|
|
2433
2085
|
//# sourceMappingURL=workRuntimeService.js.map
|