@askexenow/exe-os 0.9.286 → 0.9.287
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/deploy/stack-manifests/v0.9.json +1 -1
- package/dist/bin/stack-update.js +2 -2
- package/dist/bin/vps-health-gate.js +1 -1
- package/dist/chunk-ASL5PHCT.js +1934 -0
- package/dist/chunk-XLF5F55U.js +230 -0
- package/dist/hooks/manifest.json +1 -1
- package/dist/stack-update-45SNTDZ6.js +76 -0
- package/package.json +1 -1
- package/release-notes.json +43 -43
|
@@ -0,0 +1,1934 @@
|
|
|
1
|
+
import {
|
|
2
|
+
loadLicense
|
|
3
|
+
} from "./chunk-YMKUXZIG.js";
|
|
4
|
+
|
|
5
|
+
// src/lib/stack-update.ts
|
|
6
|
+
import { execFileSync, spawnSync } from "child_process";
|
|
7
|
+
import { createVerify, randomBytes, verify as verifySignature } from "crypto";
|
|
8
|
+
import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, writeFileSync } from "fs";
|
|
9
|
+
import http from "http";
|
|
10
|
+
import https from "https";
|
|
11
|
+
import path from "path";
|
|
12
|
+
import { fileURLToPath } from "url";
|
|
13
|
+
function serviceSelectionPath(envFile) {
|
|
14
|
+
return path.join(path.dirname(envFile), ".stack-services.json");
|
|
15
|
+
}
|
|
16
|
+
function loadServiceSelection(envFile) {
|
|
17
|
+
const selPath = serviceSelectionPath(envFile);
|
|
18
|
+
if (!existsSync(selPath)) return void 0;
|
|
19
|
+
try {
|
|
20
|
+
return JSON.parse(readFileSync(selPath, "utf8"));
|
|
21
|
+
} catch {
|
|
22
|
+
return void 0;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
function saveServiceSelection(envFile, selectedServices) {
|
|
26
|
+
const selPath = serviceSelectionPath(envFile);
|
|
27
|
+
mkdirSync(path.dirname(selPath), { recursive: true });
|
|
28
|
+
const data = {
|
|
29
|
+
selectedServices,
|
|
30
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
31
|
+
};
|
|
32
|
+
writeFileSync(selPath, JSON.stringify(data, null, 2) + "\n", { mode: 384 });
|
|
33
|
+
}
|
|
34
|
+
function filterServicesBySelection(release, selection) {
|
|
35
|
+
if (!selection) return release;
|
|
36
|
+
const selected = new Set(selection.selectedServices);
|
|
37
|
+
const filteredServices = {};
|
|
38
|
+
for (const [name, service] of Object.entries(release.services)) {
|
|
39
|
+
if (service.required) {
|
|
40
|
+
filteredServices[name] = service;
|
|
41
|
+
continue;
|
|
42
|
+
}
|
|
43
|
+
if (selected.has(name) || selected.has(service.composeService ?? name)) {
|
|
44
|
+
filteredServices[name] = service;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return { ...release, services: filteredServices };
|
|
48
|
+
}
|
|
49
|
+
function buildDefaultServiceSelection(release) {
|
|
50
|
+
return Object.keys(release.services);
|
|
51
|
+
}
|
|
52
|
+
var ProgressReporter = class {
|
|
53
|
+
totalServices;
|
|
54
|
+
completedServices;
|
|
55
|
+
currentPhase;
|
|
56
|
+
phaseStart;
|
|
57
|
+
overallStart;
|
|
58
|
+
constructor(serviceNames) {
|
|
59
|
+
this.totalServices = serviceNames.length;
|
|
60
|
+
this.completedServices = 0;
|
|
61
|
+
this.currentPhase = "";
|
|
62
|
+
this.phaseStart = Date.now();
|
|
63
|
+
this.overallStart = Date.now();
|
|
64
|
+
}
|
|
65
|
+
/** Log the start of a named phase (pull, stop, start, verify, etc.) */
|
|
66
|
+
startPhase(phase) {
|
|
67
|
+
this.currentPhase = phase;
|
|
68
|
+
this.phaseStart = Date.now();
|
|
69
|
+
console.log(`[stack-update] ${(/* @__PURE__ */ new Date()).toISOString()} \u25B6 Phase: ${phase}`);
|
|
70
|
+
}
|
|
71
|
+
/** Log the end of the current phase with elapsed time */
|
|
72
|
+
endPhase() {
|
|
73
|
+
const elapsed = ((Date.now() - this.phaseStart) / 1e3).toFixed(1);
|
|
74
|
+
console.log(`[stack-update] ${(/* @__PURE__ */ new Date()).toISOString()} \u2713 Phase: ${this.currentPhase} completed in ${elapsed}s`);
|
|
75
|
+
}
|
|
76
|
+
/** Log service-level progress with N/M counter and percentage */
|
|
77
|
+
logServiceProgress(serviceName, action) {
|
|
78
|
+
this.completedServices++;
|
|
79
|
+
const pct = Math.round(this.completedServices / this.totalServices * 100);
|
|
80
|
+
console.log(
|
|
81
|
+
`[stack-update] [${this.completedServices}/${this.totalServices}] (${pct}%) ${action} service: ${serviceName}`
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
/** Log a service-level step without incrementing the counter */
|
|
85
|
+
logServiceStep(index, serviceName, action) {
|
|
86
|
+
const pct = Math.round(index / this.totalServices * 100);
|
|
87
|
+
console.log(
|
|
88
|
+
`[stack-update] [${index}/${this.totalServices}] (${pct}%) ${action}: ${serviceName}`
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
/** Log a clean, actionable error when a service fails */
|
|
92
|
+
logServiceFailure(serviceName, error, composeFile) {
|
|
93
|
+
const border = "\u2500".repeat(60);
|
|
94
|
+
console.error(`
|
|
95
|
+
${border}`);
|
|
96
|
+
console.error(` SERVICE FAILURE: ${serviceName}`);
|
|
97
|
+
console.error(`${border}`);
|
|
98
|
+
console.error(` Error: ${error}`);
|
|
99
|
+
console.error("");
|
|
100
|
+
console.error(" Suggested actions:");
|
|
101
|
+
console.error(` 1. Check logs: docker compose logs ${serviceName}`);
|
|
102
|
+
console.error(` 2. Inspect: docker inspect ${serviceName}`);
|
|
103
|
+
if (composeFile) {
|
|
104
|
+
console.error(` 3. Restart: docker compose --file ${composeFile} up -d ${serviceName}`);
|
|
105
|
+
}
|
|
106
|
+
console.error(` 4. Resume: exe-os stack-update --resume --yes`);
|
|
107
|
+
console.error(`${border}
|
|
108
|
+
`);
|
|
109
|
+
}
|
|
110
|
+
/** Print the overall elapsed time */
|
|
111
|
+
logOverallSummary() {
|
|
112
|
+
const elapsed = ((Date.now() - this.overallStart) / 1e3).toFixed(1);
|
|
113
|
+
console.log(`[stack-update] Total update time: ${elapsed}s for ${this.totalServices} services`);
|
|
114
|
+
}
|
|
115
|
+
/** Reset counter for a new phase (e.g., going from pull to restart) */
|
|
116
|
+
resetCounter(newTotal) {
|
|
117
|
+
this.completedServices = 0;
|
|
118
|
+
if (newTotal !== void 0) this.totalServices = newTotal;
|
|
119
|
+
}
|
|
120
|
+
};
|
|
121
|
+
function isSignedEnvelope(value) {
|
|
122
|
+
return !!value && typeof value === "object" && "manifest" in value && "signature" in value;
|
|
123
|
+
}
|
|
124
|
+
function canonicalizeStackManifest(manifest) {
|
|
125
|
+
const clone = JSON.parse(JSON.stringify(manifest));
|
|
126
|
+
delete clone.signature;
|
|
127
|
+
return stableJson(clone);
|
|
128
|
+
}
|
|
129
|
+
function verifyStackManifestSignature(manifest, publicKeyPem) {
|
|
130
|
+
const signature = manifest.signature;
|
|
131
|
+
if (!signature) throw new Error("Stack manifest signature required but missing");
|
|
132
|
+
const payload = Buffer.from(canonicalizeStackManifest(manifest));
|
|
133
|
+
const sig = Buffer.from(signature.signature, "base64");
|
|
134
|
+
let ok = false;
|
|
135
|
+
if (signature.alg === "ed25519") {
|
|
136
|
+
ok = verifySignature(null, payload, publicKeyPem, sig);
|
|
137
|
+
} else if (signature.alg === "rsa-sha256") {
|
|
138
|
+
const verifier = createVerify("RSA-SHA256");
|
|
139
|
+
verifier.update(payload);
|
|
140
|
+
verifier.end();
|
|
141
|
+
ok = verifier.verify(publicKeyPem, sig);
|
|
142
|
+
} else {
|
|
143
|
+
throw new Error(`Unsupported stack manifest signature alg: ${signature.alg}`);
|
|
144
|
+
}
|
|
145
|
+
if (!ok) throw new Error("Stack manifest signature verification failed");
|
|
146
|
+
}
|
|
147
|
+
function stableJson(value) {
|
|
148
|
+
if (value === null || typeof value !== "object") return JSON.stringify(value);
|
|
149
|
+
if (Array.isArray(value)) return `[${value.map(stableJson).join(",")}]`;
|
|
150
|
+
const obj = value;
|
|
151
|
+
return `{${Object.keys(obj).sort().map((key) => `${JSON.stringify(key)}:${stableJson(obj[key])}`).join(",")}}`;
|
|
152
|
+
}
|
|
153
|
+
function findLatestBackupEnvFile(envFile) {
|
|
154
|
+
const backupDir = path.join(path.dirname(envFile), ".exe-stack-backups");
|
|
155
|
+
if (!existsSync(backupDir)) return null;
|
|
156
|
+
const backups = readdirSync(backupDir).filter((name) => name.startsWith("env-") && name.endsWith(".bak")).sort();
|
|
157
|
+
const latest = backups.at(-1);
|
|
158
|
+
return latest ? path.join(backupDir, latest) : null;
|
|
159
|
+
}
|
|
160
|
+
async function rollbackStackUpdate(options) {
|
|
161
|
+
const exec = options.exec ?? defaultExec;
|
|
162
|
+
const lockFilePath = options.lockFile ?? path.join(path.dirname(options.envFile), ".exe-stack-lock.json");
|
|
163
|
+
const lockData = readStackUpdateLock(lockFilePath);
|
|
164
|
+
const backupEnvFile = lockData?.backupEnvFile;
|
|
165
|
+
const rollbackEnv = backupEnvFile && existsSync(backupEnvFile) ? backupEnvFile : findLatestBackupEnvFile(options.envFile);
|
|
166
|
+
if (!rollbackEnv) throw new Error(`No stack backup env found beside ${options.envFile}`);
|
|
167
|
+
console.error(`[stack-update] ROLLBACK: restoring ${options.envFile} from ${rollbackEnv}`);
|
|
168
|
+
console.log("[stack-update] Starting rollback\u2026");
|
|
169
|
+
const preRollbackBackup = options.envFile + `.pre-rollback-${Date.now()}`;
|
|
170
|
+
try {
|
|
171
|
+
if (existsSync(options.envFile)) copyFileSync(options.envFile, preRollbackBackup);
|
|
172
|
+
} catch {
|
|
173
|
+
}
|
|
174
|
+
const scopedServiceNames = options.serviceName ? Object.entries(lockData?.services ?? {}).filter(([name, service]) => name === options.serviceName || (service.composeService ?? name) === options.serviceName).map(([name]) => name) : [];
|
|
175
|
+
const rollbackEnvRaw = readFileSync(rollbackEnv, "utf8");
|
|
176
|
+
if (scopedServiceNames.length > 0 && lockData?.services) {
|
|
177
|
+
const rollbackEnvMap = parseEnv(rollbackEnvRaw);
|
|
178
|
+
const scopedUpdates = Object.fromEntries(scopedServiceNames.map((serviceName) => {
|
|
179
|
+
const key = lockData.services[serviceName].env;
|
|
180
|
+
return [key, rollbackEnvMap.get(key) ?? ""];
|
|
181
|
+
}));
|
|
182
|
+
console.log(`[stack-update] Restoring env keys for service ${scopedServiceNames.join(", ")} from ${rollbackEnv}`);
|
|
183
|
+
writeFileSync(options.envFile, patchEnv(readFileSync(options.envFile, "utf8"), scopedUpdates), { mode: 384 });
|
|
184
|
+
} else {
|
|
185
|
+
console.log(`[stack-update] Restoring env from ${rollbackEnv}`);
|
|
186
|
+
writeFileSync(options.envFile, rollbackEnvRaw, { mode: 384 });
|
|
187
|
+
}
|
|
188
|
+
const composeArgs = ["compose", "--file", options.composeFile, "--env-file", options.envFile];
|
|
189
|
+
if (lockData?.services) {
|
|
190
|
+
for (const [serviceName, service] of Object.entries(lockData.services)) {
|
|
191
|
+
if (scopedServiceNames.length > 0 && !scopedServiceNames.includes(serviceName)) continue;
|
|
192
|
+
if (!service.migrations?.rollback) continue;
|
|
193
|
+
const composeServiceName = service.composeService ?? serviceName;
|
|
194
|
+
console.log(`[stack-update] Running rollback migration for ${composeServiceName}: ${service.migrations.rollback}`);
|
|
195
|
+
try {
|
|
196
|
+
const migrationArgs = service.migrations.rollback.split(/\s+/);
|
|
197
|
+
exec("docker", [
|
|
198
|
+
...composeArgs,
|
|
199
|
+
"run",
|
|
200
|
+
"--rm",
|
|
201
|
+
"--no-deps",
|
|
202
|
+
composeServiceName,
|
|
203
|
+
...migrationArgs
|
|
204
|
+
]);
|
|
205
|
+
console.log(`[stack-update] \u2713 Rollback migration for ${composeServiceName} completed`);
|
|
206
|
+
} catch (migErr) {
|
|
207
|
+
const reason = migErr instanceof Error ? migErr.message : String(migErr);
|
|
208
|
+
console.warn(`[stack-update] \u26A0 Rollback migration failed for ${composeServiceName}: ${reason} \u2014 continuing rollback`);
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
console.log("[stack-update] Restarting services with previous images\u2026");
|
|
213
|
+
if (scopedServiceNames.length > 0 && lockData?.services) {
|
|
214
|
+
for (const serviceName of scopedServiceNames) {
|
|
215
|
+
const composeServiceName = lockData.services[serviceName]?.composeService ?? serviceName;
|
|
216
|
+
exec("docker", [...composeArgs, "up", "-d", "--remove-orphans", "--no-deps", composeServiceName]);
|
|
217
|
+
}
|
|
218
|
+
} else {
|
|
219
|
+
exec("docker", [...composeArgs, "up", "-d", "--remove-orphans"]);
|
|
220
|
+
}
|
|
221
|
+
console.log("[stack-update] Rollback complete.");
|
|
222
|
+
return { status: "rolled_back", targetVersion: "previous", changes: [], backupEnvFile: rollbackEnv, lockFile: lockFilePath, serviceStatus: lockData?.serviceStatus };
|
|
223
|
+
}
|
|
224
|
+
function manifestImagesForPull(release) {
|
|
225
|
+
return Array.from(new Set(
|
|
226
|
+
Object.values(release.services).map((service) => service.image.trim()).filter(Boolean)
|
|
227
|
+
));
|
|
228
|
+
}
|
|
229
|
+
function cloneReleaseWithServices(release, serviceNames) {
|
|
230
|
+
const wanted = new Set(serviceNames);
|
|
231
|
+
const services = {};
|
|
232
|
+
for (const [name, service] of Object.entries(release.services)) {
|
|
233
|
+
if (wanted.has(name) || wanted.has(service.composeService ?? name)) {
|
|
234
|
+
services[name] = service;
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
const missing = serviceNames.filter((name) => !Object.entries(release.services).some(([svcName, svc]) => svcName === name || (svc.composeService ?? svcName) === name));
|
|
238
|
+
if (missing.length > 0) throw new Error(`Stack service not found in manifest: ${missing.join(", ")}`);
|
|
239
|
+
return { ...release, services };
|
|
240
|
+
}
|
|
241
|
+
function readStackUpdateLock(lockFile) {
|
|
242
|
+
if (!existsSync(lockFile)) return void 0;
|
|
243
|
+
try {
|
|
244
|
+
return JSON.parse(readFileSync(lockFile, "utf8"));
|
|
245
|
+
} catch {
|
|
246
|
+
return void 0;
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
function writeStackUpdateLock(lockFile, data) {
|
|
250
|
+
writeFileSync(lockFile, JSON.stringify(data, null, 2) + "\n");
|
|
251
|
+
}
|
|
252
|
+
function buildInitialServiceStatus(plan, envRaw, now, activeServices) {
|
|
253
|
+
const env = parseEnv(envRaw);
|
|
254
|
+
const at = now().toISOString();
|
|
255
|
+
return Object.fromEntries(Object.entries(plan.release.services).map(([serviceName, service]) => [
|
|
256
|
+
serviceName,
|
|
257
|
+
{
|
|
258
|
+
service: serviceName,
|
|
259
|
+
image: service.image,
|
|
260
|
+
env: service.env,
|
|
261
|
+
before: env.get(service.env),
|
|
262
|
+
after: service.image,
|
|
263
|
+
status: activeServices && !activeServices.has(serviceName) ? "skipped" : "pending",
|
|
264
|
+
error: activeServices && !activeServices.has(serviceName) ? "not selected for this service-scoped update" : void 0,
|
|
265
|
+
updatedAt: at
|
|
266
|
+
}
|
|
267
|
+
]));
|
|
268
|
+
}
|
|
269
|
+
function updateServiceStatus(lockFile, serviceName, status, now, error) {
|
|
270
|
+
const lock = readStackUpdateLock(lockFile);
|
|
271
|
+
if (!lock?.serviceStatus?.[serviceName]) return lock?.serviceStatus;
|
|
272
|
+
lock.serviceStatus[serviceName] = {
|
|
273
|
+
...lock.serviceStatus[serviceName],
|
|
274
|
+
status,
|
|
275
|
+
error,
|
|
276
|
+
updatedAt: now().toISOString()
|
|
277
|
+
};
|
|
278
|
+
lock.updatedAt = now().toISOString();
|
|
279
|
+
writeStackUpdateLock(lockFile, lock);
|
|
280
|
+
return lock.serviceStatus;
|
|
281
|
+
}
|
|
282
|
+
function incompleteServicesFromLock(lockFile) {
|
|
283
|
+
const lock = readStackUpdateLock(lockFile);
|
|
284
|
+
const serviceStatus = lock?.serviceStatus ?? {};
|
|
285
|
+
return Object.entries(serviceStatus).filter(([, status]) => status.status !== "completed" && status.status !== "skipped").map(([service]) => service);
|
|
286
|
+
}
|
|
287
|
+
function parseStackManifest(raw, publicKey) {
|
|
288
|
+
const parsedRaw = JSON.parse(raw);
|
|
289
|
+
const parsed = isSignedEnvelope(parsedRaw) ? { ...parsedRaw.manifest, signature: parsedRaw.signature } : parsedRaw;
|
|
290
|
+
if (publicKey) verifyStackManifestSignature(parsed, publicKey);
|
|
291
|
+
if (parsed.schemaVersion !== 1) throw new Error("Unsupported stack manifest schemaVersion");
|
|
292
|
+
if (!parsed.latest || !parsed.stacks || typeof parsed.stacks !== "object") {
|
|
293
|
+
throw new Error("Invalid stack manifest: latest and stacks are required");
|
|
294
|
+
}
|
|
295
|
+
for (const [version, release] of Object.entries(parsed.stacks)) {
|
|
296
|
+
if (!release.version) release.version = version;
|
|
297
|
+
if (!release.services || typeof release.services !== "object") {
|
|
298
|
+
throw new Error(`Invalid stack manifest: release ${version} has no services`);
|
|
299
|
+
}
|
|
300
|
+
for (const [serviceName, service] of Object.entries(release.services)) {
|
|
301
|
+
if (!service.image || !service.env) {
|
|
302
|
+
throw new Error(`Invalid stack manifest: ${version}.${serviceName} requires image and env`);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
return parsed;
|
|
307
|
+
}
|
|
308
|
+
async function loadStackManifest(ref, fetchText = defaultFetchText, publicKey, authToken) {
|
|
309
|
+
if (/^https?:\/\//.test(ref)) return parseStackManifest(await fetchTextWithAuth(ref, fetchText, authToken), publicKey);
|
|
310
|
+
return parseStackManifest(readFileSync(ref, "utf8"), publicKey);
|
|
311
|
+
}
|
|
312
|
+
async function fetchTextWithAuth(ref, fetchText, authToken) {
|
|
313
|
+
if (!authToken || fetchText !== defaultFetchText) return fetchText(ref);
|
|
314
|
+
return defaultFetchText(ref, authToken);
|
|
315
|
+
}
|
|
316
|
+
function parseEnv(raw) {
|
|
317
|
+
const env = /* @__PURE__ */ new Map();
|
|
318
|
+
for (const line of raw.split(/\r?\n/)) {
|
|
319
|
+
const trimmed = line.trim();
|
|
320
|
+
if (!trimmed || trimmed.startsWith("#")) continue;
|
|
321
|
+
const idx = line.indexOf("=");
|
|
322
|
+
if (idx <= 0) continue;
|
|
323
|
+
env.set(line.slice(0, idx).trim(), line.slice(idx + 1));
|
|
324
|
+
}
|
|
325
|
+
return env;
|
|
326
|
+
}
|
|
327
|
+
function patchEnv(raw, updates) {
|
|
328
|
+
const seen = /* @__PURE__ */ new Set();
|
|
329
|
+
const lines = raw.replace(/\n$/, "").split(/\r?\n/);
|
|
330
|
+
const patched = lines.map((line) => {
|
|
331
|
+
const idx = line.indexOf("=");
|
|
332
|
+
if (idx <= 0 || line.trim().startsWith("#")) return line;
|
|
333
|
+
const key = line.slice(0, idx).trim();
|
|
334
|
+
if (!(key in updates)) return line;
|
|
335
|
+
seen.add(key);
|
|
336
|
+
return `${key}=${updates[key]}`;
|
|
337
|
+
});
|
|
338
|
+
for (const [key, value] of Object.entries(updates)) {
|
|
339
|
+
if (!seen.has(key)) patched.push(`${key}=${value}`);
|
|
340
|
+
}
|
|
341
|
+
return patched.join("\n").replace(/\n*$/, "\n");
|
|
342
|
+
}
|
|
343
|
+
function createStackUpdatePlan(manifest, envRaw, targetVersion, serviceNames, selection) {
|
|
344
|
+
const version = targetVersion ?? manifest.latest;
|
|
345
|
+
const manifestRelease = manifest.stacks[version];
|
|
346
|
+
if (!manifestRelease) throw new Error(`Stack version ${version} not found in manifest`);
|
|
347
|
+
const selectedRelease = filterServicesBySelection(manifestRelease, selection);
|
|
348
|
+
const release = serviceNames && serviceNames.length > 0 ? cloneReleaseWithServices(selectedRelease, serviceNames) : selectedRelease;
|
|
349
|
+
const env = parseEnv(envRaw);
|
|
350
|
+
const changes = [];
|
|
351
|
+
for (const [serviceName, service] of Object.entries(release.services)) {
|
|
352
|
+
const before = env.get(service.env);
|
|
353
|
+
if (before !== service.image) {
|
|
354
|
+
changes.push({ key: service.env, before, after: service.image, service: serviceName });
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
return {
|
|
358
|
+
manifest,
|
|
359
|
+
release,
|
|
360
|
+
targetVersion: version,
|
|
361
|
+
changes,
|
|
362
|
+
breakingChanges: release.breakingChanges ?? [],
|
|
363
|
+
scopedServices: serviceNames && serviceNames.length > 0 ? Object.keys(release.services) : void 0
|
|
364
|
+
};
|
|
365
|
+
}
|
|
366
|
+
var ASKEXE_GHCR_IMAGE = /^(?:ghcr\.io\/askexe|update\.askexe\.com\/askexe)\/[a-z0-9._/-]+(?:(?::[^:@$/{]+)(?:@sha256:[a-f0-9]{64})?|@sha256:[a-f0-9]{64})$/i;
|
|
367
|
+
var OFFICIAL_COMPOSE_IMAGE = /^(?:postgres|pgvector\/pgvector|clickhouse\/clickhouse-server|redis|nginx|postgrest\/postgrest|supabase\/gotrue|cloudflare\/cloudflared|otel\/opentelemetry-collector-contrib)(?:(?::[^:@$/{]+)(?:@sha256:[a-f0-9]{64})?|@sha256:[a-f0-9]{64})$/i;
|
|
368
|
+
function validatePinnedGhcrImage(image, label, requireDigest = false) {
|
|
369
|
+
const trimmed = image.trim().replace(/^['"]|['"]$/g, "");
|
|
370
|
+
if (!trimmed) return `${label} is empty`;
|
|
371
|
+
if (trimmed.includes("${")) return null;
|
|
372
|
+
if (!trimmed.startsWith("ghcr.io/askexe/") && !trimmed.startsWith("update.askexe.com/askexe/")) return `${label} must use ghcr.io/askexe/* or update.askexe.com/askexe/*, got ${trimmed}`;
|
|
373
|
+
if (/:latest(?:$|[\s#])/.test(trimmed)) return `${label} must not use :latest (${trimmed})`;
|
|
374
|
+
if (!ASKEXE_GHCR_IMAGE.test(trimmed)) return `${label} must be pinned with an explicit tag or sha256 digest from ghcr.io/askexe or update.askexe.com/askexe, got ${trimmed}`;
|
|
375
|
+
if (requireDigest && !/@sha256:[a-f0-9]{64}/.test(trimmed)) return `${label} must include @sha256: digest for supply-chain integrity, got ${trimmed}`;
|
|
376
|
+
return null;
|
|
377
|
+
}
|
|
378
|
+
function validateComposeImageLiteral(image, label) {
|
|
379
|
+
const trimmed = image.trim().replace(/^['"]|['"]$/g, "");
|
|
380
|
+
if (!trimmed) return `${label} is empty`;
|
|
381
|
+
if (trimmed.startsWith("ghcr.io/askexe/") || trimmed.startsWith("update.askexe.com/askexe/")) return validatePinnedGhcrImage(trimmed, label);
|
|
382
|
+
if (OFFICIAL_COMPOSE_IMAGE.test(trimmed)) return null;
|
|
383
|
+
return `${label} uses unsupported non-AskExe image ${trimmed}; customer app images must come from pinned update.askexe.com/askexe or ghcr.io/askexe images`;
|
|
384
|
+
}
|
|
385
|
+
function resolveComposeImageExpression(image, env) {
|
|
386
|
+
let resolved = image.trim().replace(/^['"]|['"]$/g, "");
|
|
387
|
+
for (let i = 0; i < 12 && resolved.includes("${"); i++) {
|
|
388
|
+
const next = resolved.replace(/\$\{([^${}]+)\}/g, (_match, body) => {
|
|
389
|
+
const fallbackIdx = body.indexOf(":-");
|
|
390
|
+
const key = (fallbackIdx >= 0 ? body.slice(0, fallbackIdx) : body).trim();
|
|
391
|
+
const fallback = fallbackIdx >= 0 ? body.slice(fallbackIdx + 2) : "";
|
|
392
|
+
const envValue = env.get(key);
|
|
393
|
+
return envValue && envValue.trim() ? envValue.trim() : fallback;
|
|
394
|
+
});
|
|
395
|
+
if (next === resolved) break;
|
|
396
|
+
resolved = next.trim().replace(/^['"]|['"]$/g, "");
|
|
397
|
+
}
|
|
398
|
+
if (resolved.includes("${")) return null;
|
|
399
|
+
return resolved || null;
|
|
400
|
+
}
|
|
401
|
+
function collectProductionDeployGateIssues(plan, envRaw, composeRaw) {
|
|
402
|
+
const issues = [];
|
|
403
|
+
const env = parseEnv(envRaw);
|
|
404
|
+
for (const [serviceName, service] of Object.entries(plan.release.services)) {
|
|
405
|
+
const manifestIssue = validatePinnedGhcrImage(service.image, `manifest ${plan.targetVersion}.${serviceName}.image`);
|
|
406
|
+
if (manifestIssue) issues.push({ kind: "manifest-image", message: manifestIssue });
|
|
407
|
+
const envImage = env.get(service.env);
|
|
408
|
+
if (envImage) {
|
|
409
|
+
const envIssue = validatePinnedGhcrImage(envImage, `env ${service.env}`);
|
|
410
|
+
if (envIssue) issues.push({ kind: "env-image", message: envIssue });
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
const lines = composeRaw.split(/\r?\n/);
|
|
414
|
+
lines.forEach((line, index) => {
|
|
415
|
+
if (/^\s*build\s*:/.test(line)) {
|
|
416
|
+
issues.push({ kind: "compose-build", message: `compose line ${index + 1} contains build:, production deploys must pull images` });
|
|
417
|
+
}
|
|
418
|
+
const imageMatch = line.match(/^\s*image\s*:\s*(.+?)\s*(?:#.*)?$/);
|
|
419
|
+
if (imageMatch) {
|
|
420
|
+
const image = imageMatch[1].trim();
|
|
421
|
+
if (image.includes("${")) {
|
|
422
|
+
const resolved = resolveComposeImageExpression(image, env);
|
|
423
|
+
if (resolved) {
|
|
424
|
+
const composeIssue = validateComposeImageLiteral(resolved, `compose image resolved value on line ${index + 1}`);
|
|
425
|
+
if (composeIssue) issues.push({ kind: "compose-image", message: composeIssue });
|
|
426
|
+
}
|
|
427
|
+
} else {
|
|
428
|
+
const composeIssue = validateComposeImageLiteral(image, `compose image on line ${index + 1}`);
|
|
429
|
+
if (composeIssue) issues.push({ kind: "compose-image", message: composeIssue });
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
});
|
|
433
|
+
return issues;
|
|
434
|
+
}
|
|
435
|
+
function collectImageAvailabilityIssues(plan) {
|
|
436
|
+
const issues = [];
|
|
437
|
+
const checked = /* @__PURE__ */ new Set();
|
|
438
|
+
for (const [serviceName, service] of Object.entries(plan.release.services)) {
|
|
439
|
+
const image = service.image;
|
|
440
|
+
if (checked.has(image)) continue;
|
|
441
|
+
checked.add(image);
|
|
442
|
+
try {
|
|
443
|
+
const result = spawnSync("docker", ["manifest", "inspect", image, "--verbose"], {
|
|
444
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
445
|
+
timeout: 3e4
|
|
446
|
+
});
|
|
447
|
+
if (result.status !== 0) {
|
|
448
|
+
const stderr = result.stderr?.toString().trim() ?? "";
|
|
449
|
+
issues.push({
|
|
450
|
+
kind: "image-not-pullable",
|
|
451
|
+
message: `${serviceName} image ${image} not found on registry. ${stderr.slice(0, 200)}`
|
|
452
|
+
});
|
|
453
|
+
continue;
|
|
454
|
+
}
|
|
455
|
+
const output = result.stdout?.toString() ?? "";
|
|
456
|
+
try {
|
|
457
|
+
const parsed = JSON.parse(output);
|
|
458
|
+
const manifests = Array.isArray(parsed) ? parsed : [parsed];
|
|
459
|
+
const hasAmd64 = manifests.some((m) => {
|
|
460
|
+
const platform = m.platform;
|
|
461
|
+
return platform?.architecture === "amd64" && platform?.os === "linux";
|
|
462
|
+
});
|
|
463
|
+
if (!hasAmd64 && manifests.length > 0 && manifests[0]?.platform) {
|
|
464
|
+
const platforms = manifests.map((m) => {
|
|
465
|
+
const p = m.platform;
|
|
466
|
+
return `${p?.os ?? "?"}/${p?.architecture ?? "?"}`;
|
|
467
|
+
}).join(", ");
|
|
468
|
+
issues.push({
|
|
469
|
+
kind: "platform-mismatch",
|
|
470
|
+
message: `${serviceName} image ${image} has no linux/amd64 manifest (available: ${platforms}). Rebuild with: docker buildx build --platform linux/amd64 --push`
|
|
471
|
+
});
|
|
472
|
+
}
|
|
473
|
+
} catch {
|
|
474
|
+
}
|
|
475
|
+
} catch {
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
return issues;
|
|
479
|
+
}
|
|
480
|
+
function assertDeploymentScopeAllowed(plan, persona = "customer") {
|
|
481
|
+
if (persona === "askexe-control-plane") return;
|
|
482
|
+
const blocked = Object.entries(plan.release.services).filter(([, service]) => service.deploymentScope === "askexe-control-plane").map(([name]) => name);
|
|
483
|
+
if (blocked.length > 0) {
|
|
484
|
+
throw new Error(
|
|
485
|
+
`Customer deployment manifest includes AskExe control-plane service(s): ${blocked.join(", ")}. Customer VPSs may deploy customer services and optional agents only.`
|
|
486
|
+
);
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
function assertProductionDeployGate(plan, envRaw, composeRaw, options = {}) {
|
|
490
|
+
const issues = collectProductionDeployGateIssues(plan, envRaw, composeRaw);
|
|
491
|
+
if (issues.length === 0 && !options.breakGlassReason?.trim()) {
|
|
492
|
+
const hasPrivateImages = Object.values(plan.release.services).some(
|
|
493
|
+
(s) => s.image.startsWith("update.askexe.com")
|
|
494
|
+
);
|
|
495
|
+
if (!hasPrivateImages) {
|
|
496
|
+
try {
|
|
497
|
+
const availabilityIssues = collectImageAvailabilityIssues(plan);
|
|
498
|
+
issues.push(...availabilityIssues);
|
|
499
|
+
} catch {
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
if (issues.length === 0) return;
|
|
504
|
+
if (options.breakGlassReason?.trim()) {
|
|
505
|
+
writeBreakGlassAudit(plan, issues, options);
|
|
506
|
+
return;
|
|
507
|
+
}
|
|
508
|
+
const details = issues.map((issue) => `- [${issue.kind}] ${issue.message}`).join("\n");
|
|
509
|
+
throw new Error(
|
|
510
|
+
`Production deploy gate failed. Exe OS deploys must use pinned ghcr.io/askexe or update.askexe.com/askexe images and must not build from source on the VPS.
|
|
511
|
+
${details}
|
|
512
|
+
Emergency override requires --break-glass <reason> and writes an audit file.`
|
|
513
|
+
);
|
|
514
|
+
}
|
|
515
|
+
function writeBreakGlassAudit(plan, issues, options) {
|
|
516
|
+
const now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
517
|
+
const stamp = now().toISOString().replace(/[:.]/g, "-");
|
|
518
|
+
const defaultDir = existsSync("exe/output") ? "exe/output" : path.dirname(options.envFile ?? ".");
|
|
519
|
+
const auditFile = options.breakGlassAuditFile ?? path.join(defaultDir, `stack-update-break-glass-${stamp}.md`);
|
|
520
|
+
mkdirSync(path.dirname(auditFile), { recursive: true });
|
|
521
|
+
const body = [
|
|
522
|
+
`# Stack Update Break-Glass Audit \u2014 ${now().toISOString()}`,
|
|
523
|
+
"",
|
|
524
|
+
`Target version: ${plan.targetVersion}`,
|
|
525
|
+
`Reason: ${options.breakGlassReason?.trim()}`,
|
|
526
|
+
"",
|
|
527
|
+
"## Gate failures overridden",
|
|
528
|
+
...issues.map((issue) => `- [${issue.kind}] ${issue.message}`),
|
|
529
|
+
"",
|
|
530
|
+
"## Required follow-up",
|
|
531
|
+
"Return this deployment to the standard pinned GHCR image path immediately after the emergency is resolved.",
|
|
532
|
+
""
|
|
533
|
+
].join("\n");
|
|
534
|
+
writeFileSync(auditFile, body, { mode: 384 });
|
|
535
|
+
console.warn(`[stack-update] BREAK-GLASS deploy override recorded: ${auditFile}`);
|
|
536
|
+
}
|
|
537
|
+
function assertBreakingChangesAllowed(plan, allowedIds) {
|
|
538
|
+
const required = plan.breakingChanges.filter((c) => c.requiresConfirmation !== false);
|
|
539
|
+
const missing = required.filter((c) => !allowedIds.includes(c.id));
|
|
540
|
+
if (missing.length > 0) {
|
|
541
|
+
const details = missing.map((c) => `- ${c.id}: ${c.title}
|
|
542
|
+
${c.description}
|
|
543
|
+
Action: ${c.requiredAction ?? "Review release notes."}`).join("\n");
|
|
544
|
+
throw new Error(
|
|
545
|
+
`Stack ${plan.targetVersion} has breaking changes that require confirmation:
|
|
546
|
+
${details}
|
|
547
|
+
Re-run with: exe-os stack-update --target ${plan.targetVersion} --allow-breaking ${missing.map((c) => c.id).join(",")} --yes`
|
|
548
|
+
);
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
function registryFromImage(image) {
|
|
552
|
+
const first = image.split("/")[0] ?? "";
|
|
553
|
+
return first.includes(".") || first.includes(":") ? first : "docker.io";
|
|
554
|
+
}
|
|
555
|
+
function parseManualRegistryCredentials(raw, sampleImage) {
|
|
556
|
+
if (!raw) return void 0;
|
|
557
|
+
const [username, ...rest] = raw.split(":");
|
|
558
|
+
const password = rest.join(":");
|
|
559
|
+
if (!username || !password) return void 0;
|
|
560
|
+
return { registry: registryFromImage(sampleImage ?? ""), username, password };
|
|
561
|
+
}
|
|
562
|
+
async function verifyManifestImagesAvailable(plan, options) {
|
|
563
|
+
const exec = options.exec ?? defaultExec;
|
|
564
|
+
const images = manifestImagesForPull(plan.release);
|
|
565
|
+
const sampleImage = images[0];
|
|
566
|
+
let registryForLogout;
|
|
567
|
+
const creds = await fetchImageCredentials(options) ?? parseManualRegistryCredentials(options.registryCredentials || process.env.EXE_REGISTRY_CREDENTIALS, sampleImage);
|
|
568
|
+
if (creds) {
|
|
569
|
+
(options.dockerLogin ?? defaultDockerLogin)(creds);
|
|
570
|
+
registryForLogout = creds.registry;
|
|
571
|
+
}
|
|
572
|
+
try {
|
|
573
|
+
return images.map((image) => {
|
|
574
|
+
try {
|
|
575
|
+
exec("docker", ["manifest", "inspect", image]);
|
|
576
|
+
return { image, ok: true };
|
|
577
|
+
} catch (err) {
|
|
578
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
579
|
+
return { image, ok: false, error: reason };
|
|
580
|
+
}
|
|
581
|
+
});
|
|
582
|
+
} finally {
|
|
583
|
+
if (registryForLogout) {
|
|
584
|
+
try {
|
|
585
|
+
(options.dockerLogout ?? defaultDockerLogout)(registryForLogout);
|
|
586
|
+
} catch {
|
|
587
|
+
}
|
|
588
|
+
}
|
|
589
|
+
}
|
|
590
|
+
}
|
|
591
|
+
function commandSucceeds(cmd, args = []) {
|
|
592
|
+
const res = spawnSync(cmd, args, { stdio: "ignore" });
|
|
593
|
+
return res.status === 0;
|
|
594
|
+
}
|
|
595
|
+
function shellSucceeds(command) {
|
|
596
|
+
const res = spawnSync("sh", ["-lc", command], { stdio: "ignore" });
|
|
597
|
+
return res.status === 0;
|
|
598
|
+
}
|
|
599
|
+
function resolvePackageRoot() {
|
|
600
|
+
const here = path.dirname(fileURLToPath(import.meta.url));
|
|
601
|
+
const candidates = [
|
|
602
|
+
path.resolve(here, "..", ".."),
|
|
603
|
+
path.resolve(here, ".."),
|
|
604
|
+
process.cwd()
|
|
605
|
+
];
|
|
606
|
+
for (const c of candidates) {
|
|
607
|
+
if (existsSync(path.join(c, "package.json")) && existsSync(path.join(c, "deploy", "compose", "docker-compose.yml"))) return c;
|
|
608
|
+
}
|
|
609
|
+
return process.cwd();
|
|
610
|
+
}
|
|
611
|
+
function copyTemplate(srcRel, dest, created, options = {}) {
|
|
612
|
+
const existed = existsSync(dest);
|
|
613
|
+
if (existed && !options.overwrite) return;
|
|
614
|
+
const src = path.join(resolvePackageRoot(), srcRel);
|
|
615
|
+
if (!existsSync(src)) throw new Error(`Missing packaged stack template: ${srcRel}. Reinstall/update exe-os and retry.`);
|
|
616
|
+
try {
|
|
617
|
+
mkdirSync(path.dirname(dest), { recursive: true });
|
|
618
|
+
} catch (err) {
|
|
619
|
+
if (err.code === "EACCES") {
|
|
620
|
+
const dir = path.dirname(dest);
|
|
621
|
+
throw new Error(
|
|
622
|
+
`Permission denied creating ${dir}. Run this first:
|
|
623
|
+
|
|
624
|
+
sudo mkdir -p ${dir} && sudo chown $(whoami) ${dir}
|
|
625
|
+
|
|
626
|
+
Then re-run stack-update.`
|
|
627
|
+
);
|
|
628
|
+
}
|
|
629
|
+
throw err;
|
|
630
|
+
}
|
|
631
|
+
copyFileSync(src, dest);
|
|
632
|
+
if (!existed) created.push(dest);
|
|
633
|
+
}
|
|
634
|
+
function copyTemplateIfMissing(srcRel, dest, created) {
|
|
635
|
+
copyTemplate(srcRel, dest, created);
|
|
636
|
+
}
|
|
637
|
+
function refreshPackagedInitSql(dest, created = []) {
|
|
638
|
+
copyTemplate("deploy/compose/init-db.sql", dest, created, { overwrite: true });
|
|
639
|
+
}
|
|
640
|
+
function installDockerUbuntu(exec) {
|
|
641
|
+
if (process.platform !== "linux") throw new Error("Docker auto-install is only supported on Linux. Install Docker manually, then retry.");
|
|
642
|
+
if (!existsSync("/etc/os-release")) throw new Error("Cannot detect Linux distro; install Docker manually, then retry.");
|
|
643
|
+
const osRelease = readFileSync("/etc/os-release", "utf8");
|
|
644
|
+
if (!/ID=(ubuntu|debian)|ID_LIKE=.*debian/.test(osRelease)) {
|
|
645
|
+
throw new Error("Docker auto-install currently supports Ubuntu/Debian only. Install Docker manually, then retry.");
|
|
646
|
+
}
|
|
647
|
+
const script = [
|
|
648
|
+
"set -e",
|
|
649
|
+
"sudo apt-get update",
|
|
650
|
+
"sudo apt-get install -y ca-certificates curl gnupg",
|
|
651
|
+
"sudo install -m 0755 -d /etc/apt/keyrings",
|
|
652
|
+
"if [ ! -f /etc/apt/keyrings/docker.gpg ]; then curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg; sudo chmod a+r /etc/apt/keyrings/docker.gpg; fi",
|
|
653
|
+
". /etc/os-release",
|
|
654
|
+
'CODENAME="${VERSION_CODENAME:-bookworm}"',
|
|
655
|
+
'if [ "${ID:-}" = "debian" ]; then DOCKER_DISTRO=debian; else DOCKER_DISTRO=ubuntu; fi',
|
|
656
|
+
`printf 'deb [arch=%s signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/%s %s stable\\n' "$(dpkg --print-architecture)" "$DOCKER_DISTRO" "$CODENAME" | sudo tee /etc/apt/sources.list.d/docker.list >/dev/null`,
|
|
657
|
+
"sudo apt-get update",
|
|
658
|
+
"sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin",
|
|
659
|
+
"sudo systemctl enable --now docker",
|
|
660
|
+
"sudo docker version >/dev/null",
|
|
661
|
+
"sudo docker compose version >/dev/null"
|
|
662
|
+
].join("\n");
|
|
663
|
+
exec("sh", ["-lc", script]);
|
|
664
|
+
}
|
|
665
|
+
function randomSecret(bytes = 32) {
|
|
666
|
+
return randomBytes(bytes).toString("base64url");
|
|
667
|
+
}
|
|
668
|
+
function randomHexSecret(bytes = 24) {
|
|
669
|
+
return randomBytes(bytes).toString("hex");
|
|
670
|
+
}
|
|
671
|
+
function hydrateEnv(raw, opts) {
|
|
672
|
+
let next = raw;
|
|
673
|
+
const license = opts.licenseKey || process.env.EXE_LICENSE_KEY || loadLicense() || "";
|
|
674
|
+
const domain = opts.domain || process.env.EXE_STACK_DOMAIN || process.env.CUSTOMER_DOMAIN || "";
|
|
675
|
+
const replacements = {};
|
|
676
|
+
const env = parseEnv(raw);
|
|
677
|
+
for (const [key, value] of env.entries()) {
|
|
678
|
+
if (!/CHANGEME/.test(value)) continue;
|
|
679
|
+
if (key === "EXE_LICENSE_KEY" && license) replacements[key] = license;
|
|
680
|
+
else if (key === "MONITOR_AGENT_TOKEN" || key === "MONITOR_AGENT_KEY") continue;
|
|
681
|
+
else if (key === "CLOUDFLARE_TUNNEL_TOKEN") continue;
|
|
682
|
+
else if (key === "EXE_GATEWAY_WS_RELAY_AUTH_TOKEN") replacements[key] = randomHexSecret(24);
|
|
683
|
+
else if (key.endsWith("_PASSWORD")) replacements[key] = randomSecret(24);
|
|
684
|
+
else if (key.endsWith("_TOKEN")) replacements[key] = randomHexSecret(32);
|
|
685
|
+
else if (key.endsWith("_SECRET") || key.endsWith("_SALT")) replacements[key] = randomSecret(32);
|
|
686
|
+
else if (key.endsWith("_KEY") && key !== "EXE_LICENSE_KEY") continue;
|
|
687
|
+
}
|
|
688
|
+
if (domain) next = next.replaceAll("CHANGEME_DOMAIN", domain);
|
|
689
|
+
next = patchEnv(next, replacements);
|
|
690
|
+
const remaining = [...parseEnv(next).entries()].filter(([, value]) => /CHANGEME/.test(value)).map(([key, value]) => `${key}=${value}`);
|
|
691
|
+
return { raw: next, hadPlaceholders: /CHANGEME/.test(raw), remaining: [...new Set(remaining)] };
|
|
692
|
+
}
|
|
693
|
+
async function pairMonitorAgent(hubUrl, licenseKey, domain, envFile) {
|
|
694
|
+
if (!hubUrl || !licenseKey || !domain) {
|
|
695
|
+
return { paired: false, error: "Missing hubUrl, licenseKey, or domain for monitor pairing" };
|
|
696
|
+
}
|
|
697
|
+
const registrationUrl = `${hubUrl.replace(/\/+$/, "")}/api/register-agent`;
|
|
698
|
+
try {
|
|
699
|
+
const res = await fetch(registrationUrl, {
|
|
700
|
+
method: "POST",
|
|
701
|
+
headers: {
|
|
702
|
+
"content-type": "application/json",
|
|
703
|
+
authorization: `Bearer ${licenseKey}`
|
|
704
|
+
},
|
|
705
|
+
body: JSON.stringify({ name: domain, host: domain, port: 45876 }),
|
|
706
|
+
signal: AbortSignal.timeout(15e3)
|
|
707
|
+
});
|
|
708
|
+
if (!res.ok) {
|
|
709
|
+
const body = await res.text().catch(() => "");
|
|
710
|
+
return { paired: false, error: `Monitor hub returned HTTP ${res.status}: ${body}` };
|
|
711
|
+
}
|
|
712
|
+
const data = await res.json();
|
|
713
|
+
if (!data.token || !data.key) {
|
|
714
|
+
return { paired: false, error: "Monitor hub response missing token or key" };
|
|
715
|
+
}
|
|
716
|
+
if (existsSync(envFile)) {
|
|
717
|
+
const envRaw = readFileSync(envFile, "utf8");
|
|
718
|
+
const patched = patchEnv(envRaw, {
|
|
719
|
+
MONITOR_AGENT_TOKEN: data.token,
|
|
720
|
+
MONITOR_AGENT_KEY: data.key
|
|
721
|
+
});
|
|
722
|
+
if (patched !== envRaw) {
|
|
723
|
+
writeFileSync(envFile, patched, { mode: 384 });
|
|
724
|
+
}
|
|
725
|
+
}
|
|
726
|
+
return { paired: true, systemName: data.name ?? domain };
|
|
727
|
+
} catch (err) {
|
|
728
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
729
|
+
return { paired: false, error: `Monitor hub unreachable: ${reason}` };
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
function bootstrapStackHost(options) {
|
|
733
|
+
const exec = options.exec ?? defaultExec;
|
|
734
|
+
const createdFiles = [];
|
|
735
|
+
const actions = [];
|
|
736
|
+
let dockerInstalled = commandSucceeds("docker", ["version"]);
|
|
737
|
+
let dockerComposeInstalled = shellSucceeds("docker compose version");
|
|
738
|
+
if ((!dockerInstalled || !dockerComposeInstalled) && options.installDocker) {
|
|
739
|
+
actions.push("install_docker");
|
|
740
|
+
installDockerUbuntu(exec);
|
|
741
|
+
dockerInstalled = commandSucceeds("docker", ["version"]);
|
|
742
|
+
dockerComposeInstalled = shellSucceeds("docker compose version");
|
|
743
|
+
}
|
|
744
|
+
copyTemplateIfMissing("deploy/compose/docker-compose.yml", options.composeFile, createdFiles);
|
|
745
|
+
copyTemplateIfMissing("deploy/compose/.env.customer.example", options.envFile, createdFiles);
|
|
746
|
+
refreshPackagedInitSql(path.join(path.dirname(options.composeFile), "init-db.sql"), createdFiles);
|
|
747
|
+
copyTemplateIfMissing(
|
|
748
|
+
"deploy/compose/clickhouse-config.xml",
|
|
749
|
+
path.join(path.dirname(options.envFile), "clickhouse-config.xml"),
|
|
750
|
+
createdFiles
|
|
751
|
+
);
|
|
752
|
+
const brandingDest = path.join(path.dirname(options.envFile), "branding.json");
|
|
753
|
+
copyTemplateIfMissing("deploy/compose/gateway.json", path.join(path.dirname(options.envFile), "gateway.json"), createdFiles);
|
|
754
|
+
if (!existsSync(brandingDest)) writeFileSync(brandingDest, JSON.stringify({ brandName: "Exe", productName: "Exe OS" }, null, 2) + "\n", { mode: 384 });
|
|
755
|
+
copyTemplateIfMissing(
|
|
756
|
+
"deploy/compose/cloudflared/config.yml.example",
|
|
757
|
+
path.join(path.dirname(options.composeFile), "cloudflared", "config.yml.example"),
|
|
758
|
+
createdFiles
|
|
759
|
+
);
|
|
760
|
+
let envHadPlaceholders = false;
|
|
761
|
+
let envRemainingPlaceholders = [];
|
|
762
|
+
if (existsSync(options.envFile)) {
|
|
763
|
+
const envRaw = readFileSync(options.envFile, "utf8");
|
|
764
|
+
const hydrated = hydrateEnv(envRaw, options);
|
|
765
|
+
envHadPlaceholders = hydrated.hadPlaceholders;
|
|
766
|
+
envRemainingPlaceholders = hydrated.remaining;
|
|
767
|
+
if (hydrated.raw !== envRaw) {
|
|
768
|
+
const backupPath = options.envFile + `.bak-${Date.now()}`;
|
|
769
|
+
try {
|
|
770
|
+
copyFileSync(options.envFile, backupPath);
|
|
771
|
+
} catch {
|
|
772
|
+
}
|
|
773
|
+
writeFileSync(options.envFile, hydrated.raw, { mode: 384 });
|
|
774
|
+
actions.push("hydrate_env_secrets");
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
return {
|
|
778
|
+
dockerInstalled,
|
|
779
|
+
dockerComposeInstalled,
|
|
780
|
+
composeFileExists: existsSync(options.composeFile),
|
|
781
|
+
envFileExists: existsSync(options.envFile),
|
|
782
|
+
envHadPlaceholders,
|
|
783
|
+
envRemainingPlaceholders,
|
|
784
|
+
licensePresent: !!(options.licenseKey || process.env.EXE_LICENSE_KEY || loadLicense()),
|
|
785
|
+
createdFiles,
|
|
786
|
+
actions
|
|
787
|
+
};
|
|
788
|
+
}
|
|
789
|
+
function assertHostReadyForApply(report) {
|
|
790
|
+
const blockers = [];
|
|
791
|
+
if (!report.dockerInstalled) blockers.push("Docker is not installed/running. Re-run with --yes to auto-install on Ubuntu/Debian, or install Docker manually.");
|
|
792
|
+
if (!report.dockerComposeInstalled) blockers.push("Docker Compose plugin is missing. Re-run with --yes to auto-install on Ubuntu/Debian, or install docker-compose-plugin manually.");
|
|
793
|
+
if (!report.composeFileExists) blockers.push("docker-compose.yml is missing and could not be created.");
|
|
794
|
+
if (!report.envFileExists) blockers.push(".env is missing and could not be created.");
|
|
795
|
+
if (!report.licensePresent) blockers.push("Exe OS license key is missing. Run `exe-os setup`, `exe-os --activate <key>`, or pass --license-key.");
|
|
796
|
+
const hardPlaceholders = report.envRemainingPlaceholders.filter((p) => !/WHATSAPP|API_ROUTER|MONITOR_AGENT|MONITOR_HUB_ADMIN|CLOUDFLARE_TUNNEL_TOKEN/.test(p));
|
|
797
|
+
if (hardPlaceholders.length > 0) blockers.push(`Required .env placeholders remain: ${hardPlaceholders.join(", ")}`);
|
|
798
|
+
if (blockers.length > 0) throw new Error(`Stack host is not ready:
|
|
799
|
+
- ${blockers.join("\n- ")}`);
|
|
800
|
+
}
|
|
801
|
+
function hardenHost(exec) {
|
|
802
|
+
const run = exec ?? defaultExec;
|
|
803
|
+
const result = {
|
|
804
|
+
skipped: false,
|
|
805
|
+
ufw: { changed: false, actions: [] },
|
|
806
|
+
ssh: { changed: false, actions: [] }
|
|
807
|
+
};
|
|
808
|
+
if (process.platform !== "linux") {
|
|
809
|
+
result.skipped = true;
|
|
810
|
+
result.reason = "Host hardening only runs on Linux (skipped on " + process.platform + ")";
|
|
811
|
+
console.log(`[stack-update] ${result.reason}`);
|
|
812
|
+
return result;
|
|
813
|
+
}
|
|
814
|
+
hardenUfw(run, result);
|
|
815
|
+
hardenSsh(run, result);
|
|
816
|
+
return result;
|
|
817
|
+
}
|
|
818
|
+
function hardenUfw(exec, result) {
|
|
819
|
+
const whichUfw = spawnSync("which", ["ufw"], { stdio: ["pipe", "pipe", "pipe"], timeout: 5e3 });
|
|
820
|
+
const ufwInstalled = whichUfw.status === 0;
|
|
821
|
+
if (!ufwInstalled) {
|
|
822
|
+
if (!existsSync("/etc/os-release")) {
|
|
823
|
+
result.ufw.actions.push("ufw_not_installed_unknown_distro");
|
|
824
|
+
console.warn("[stack-update] UFW not installed and OS not detected \u2014 skipping firewall hardening");
|
|
825
|
+
return;
|
|
826
|
+
}
|
|
827
|
+
const osRelease = readFileSync("/etc/os-release", "utf8");
|
|
828
|
+
if (!/ID=(ubuntu|debian)|ID_LIKE=.*debian/.test(osRelease)) {
|
|
829
|
+
result.ufw.actions.push("ufw_not_installed_unsupported_distro");
|
|
830
|
+
console.warn("[stack-update] UFW not installed on non-Debian OS \u2014 skipping firewall hardening");
|
|
831
|
+
return;
|
|
832
|
+
}
|
|
833
|
+
console.log("[stack-update] Installing UFW...");
|
|
834
|
+
try {
|
|
835
|
+
exec("apt-get", ["install", "-y", "ufw"]);
|
|
836
|
+
result.ufw.actions.push("installed_ufw");
|
|
837
|
+
} catch (err) {
|
|
838
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
839
|
+
console.warn(`[stack-update] Failed to install UFW: ${reason}`);
|
|
840
|
+
result.ufw.actions.push("install_failed");
|
|
841
|
+
return;
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
const statusResult = spawnSync("ufw", ["status", "verbose"], {
|
|
845
|
+
encoding: "utf8",
|
|
846
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
847
|
+
timeout: 1e4
|
|
848
|
+
});
|
|
849
|
+
const statusOutput = statusResult.stdout ?? "";
|
|
850
|
+
const isActive = /Status:\s*active/i.test(statusOutput);
|
|
851
|
+
const has22 = /22\/tcp\s+ALLOW/i.test(statusOutput);
|
|
852
|
+
const has80 = /80\/tcp\s+ALLOW/i.test(statusOutput);
|
|
853
|
+
const has443 = /443\/tcp\s+ALLOW/i.test(statusOutput);
|
|
854
|
+
const defaultDeny = /Default:\s*deny\s*\(incoming\)/i.test(statusOutput);
|
|
855
|
+
if (isActive && has22 && has80 && has443 && defaultDeny) {
|
|
856
|
+
result.ufw.actions.push("already_hardened");
|
|
857
|
+
console.log("[stack-update] UFW already active with correct rules \u2014 skipping");
|
|
858
|
+
return;
|
|
859
|
+
}
|
|
860
|
+
console.log("[stack-update] Configuring UFW firewall...");
|
|
861
|
+
try {
|
|
862
|
+
if (!defaultDeny) {
|
|
863
|
+
exec("ufw", ["default", "deny", "incoming"]);
|
|
864
|
+
result.ufw.actions.push("set_default_deny_incoming");
|
|
865
|
+
}
|
|
866
|
+
exec("ufw", ["default", "allow", "outgoing"]);
|
|
867
|
+
result.ufw.actions.push("set_default_allow_outgoing");
|
|
868
|
+
if (!has22) {
|
|
869
|
+
exec("ufw", ["allow", "22/tcp"]);
|
|
870
|
+
result.ufw.actions.push("allow_22_tcp");
|
|
871
|
+
}
|
|
872
|
+
if (!has80) {
|
|
873
|
+
exec("ufw", ["allow", "80/tcp"]);
|
|
874
|
+
result.ufw.actions.push("allow_80_tcp");
|
|
875
|
+
}
|
|
876
|
+
if (!has443) {
|
|
877
|
+
exec("ufw", ["allow", "443/tcp"]);
|
|
878
|
+
result.ufw.actions.push("allow_443_tcp");
|
|
879
|
+
}
|
|
880
|
+
if (!isActive) {
|
|
881
|
+
exec("ufw", ["--force", "enable"]);
|
|
882
|
+
result.ufw.actions.push("enabled_ufw");
|
|
883
|
+
}
|
|
884
|
+
result.ufw.changed = true;
|
|
885
|
+
console.log("[stack-update] \u2713 UFW firewall configured: deny incoming, allow 22/80/443");
|
|
886
|
+
} catch (err) {
|
|
887
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
888
|
+
console.warn(`[stack-update] UFW configuration failed (non-fatal): ${reason}`);
|
|
889
|
+
result.ufw.actions.push("configuration_failed");
|
|
890
|
+
}
|
|
891
|
+
}
|
|
892
|
+
function hardenSsh(exec, result) {
|
|
893
|
+
const sshdConfig = "/etc/ssh/sshd_config";
|
|
894
|
+
if (!existsSync(sshdConfig)) {
|
|
895
|
+
result.ssh.actions.push("sshd_config_not_found");
|
|
896
|
+
console.warn("[stack-update] /etc/ssh/sshd_config not found \u2014 skipping SSH hardening");
|
|
897
|
+
return;
|
|
898
|
+
}
|
|
899
|
+
let configRaw;
|
|
900
|
+
try {
|
|
901
|
+
configRaw = readFileSync(sshdConfig, "utf8");
|
|
902
|
+
} catch (err) {
|
|
903
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
904
|
+
result.ssh.actions.push("sshd_config_unreadable");
|
|
905
|
+
console.warn(`[stack-update] Cannot read sshd_config: ${reason}`);
|
|
906
|
+
return;
|
|
907
|
+
}
|
|
908
|
+
let modified = configRaw;
|
|
909
|
+
let needsReload = false;
|
|
910
|
+
const passwordAuthMatch = modified.match(/^\s*PasswordAuthentication\s+(yes|no)\s*$/m);
|
|
911
|
+
const passwordAuthCurrently = passwordAuthMatch?.[1]?.toLowerCase();
|
|
912
|
+
if (passwordAuthCurrently === "yes") {
|
|
913
|
+
modified = modified.replace(
|
|
914
|
+
/^\s*PasswordAuthentication\s+yes\s*$/m,
|
|
915
|
+
"PasswordAuthentication no"
|
|
916
|
+
);
|
|
917
|
+
result.ssh.actions.push("set_password_auth_no");
|
|
918
|
+
needsReload = true;
|
|
919
|
+
} else if (!passwordAuthMatch) {
|
|
920
|
+
if (/^#\s*PasswordAuthentication\s/m.test(modified)) {
|
|
921
|
+
modified = modified.replace(
|
|
922
|
+
/^#\s*PasswordAuthentication\s.*$/m,
|
|
923
|
+
"PasswordAuthentication no"
|
|
924
|
+
);
|
|
925
|
+
} else {
|
|
926
|
+
modified += "\nPasswordAuthentication no\n";
|
|
927
|
+
}
|
|
928
|
+
result.ssh.actions.push("set_password_auth_no");
|
|
929
|
+
needsReload = true;
|
|
930
|
+
} else {
|
|
931
|
+
result.ssh.actions.push("password_auth_already_no");
|
|
932
|
+
}
|
|
933
|
+
const rootLoginMatch = modified.match(/^\s*PermitRootLogin\s+(\S+)\s*$/m);
|
|
934
|
+
const rootLoginCurrently = rootLoginMatch?.[1]?.toLowerCase();
|
|
935
|
+
if (rootLoginCurrently === "yes") {
|
|
936
|
+
modified = modified.replace(
|
|
937
|
+
/^\s*PermitRootLogin\s+yes\s*$/m,
|
|
938
|
+
"PermitRootLogin prohibit-password"
|
|
939
|
+
);
|
|
940
|
+
result.ssh.actions.push("set_root_login_prohibit_password");
|
|
941
|
+
needsReload = true;
|
|
942
|
+
} else if (!rootLoginMatch) {
|
|
943
|
+
if (/^#\s*PermitRootLogin\s/m.test(modified)) {
|
|
944
|
+
modified = modified.replace(
|
|
945
|
+
/^#\s*PermitRootLogin\s.*$/m,
|
|
946
|
+
"PermitRootLogin prohibit-password"
|
|
947
|
+
);
|
|
948
|
+
} else {
|
|
949
|
+
modified += "\nPermitRootLogin prohibit-password\n";
|
|
950
|
+
}
|
|
951
|
+
result.ssh.actions.push("set_root_login_prohibit_password");
|
|
952
|
+
needsReload = true;
|
|
953
|
+
} else if (rootLoginCurrently === "prohibit-password" || rootLoginCurrently === "without-password" || rootLoginCurrently === "no") {
|
|
954
|
+
result.ssh.actions.push("root_login_already_restricted");
|
|
955
|
+
} else {
|
|
956
|
+
modified = modified.replace(
|
|
957
|
+
/^\s*PermitRootLogin\s+\S+\s*$/m,
|
|
958
|
+
"PermitRootLogin prohibit-password"
|
|
959
|
+
);
|
|
960
|
+
result.ssh.actions.push("set_root_login_prohibit_password");
|
|
961
|
+
needsReload = true;
|
|
962
|
+
}
|
|
963
|
+
if (!needsReload) {
|
|
964
|
+
console.log("[stack-update] SSH already hardened \u2014 skipping");
|
|
965
|
+
return;
|
|
966
|
+
}
|
|
967
|
+
try {
|
|
968
|
+
writeFileSync(sshdConfig, modified, { mode: 384 });
|
|
969
|
+
result.ssh.changed = true;
|
|
970
|
+
} catch (err) {
|
|
971
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
972
|
+
console.warn(`[stack-update] Failed to write sshd_config: ${reason}`);
|
|
973
|
+
result.ssh.actions.push("write_failed");
|
|
974
|
+
return;
|
|
975
|
+
}
|
|
976
|
+
try {
|
|
977
|
+
exec("systemctl", ["reload", "ssh"]);
|
|
978
|
+
result.ssh.actions.push("reloaded_ssh");
|
|
979
|
+
} catch {
|
|
980
|
+
try {
|
|
981
|
+
exec("systemctl", ["reload", "sshd"]);
|
|
982
|
+
result.ssh.actions.push("reloaded_sshd");
|
|
983
|
+
} catch (err) {
|
|
984
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
985
|
+
console.warn(`[stack-update] SSH reload failed (changes will apply on next restart): ${reason}`);
|
|
986
|
+
result.ssh.actions.push("reload_failed");
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
const sshSummary = result.ssh.actions.filter((a) => a.startsWith("set_")).map((a) => a.replace("set_", "").replaceAll("_", " ")).join(", ");
|
|
990
|
+
if (sshSummary) {
|
|
991
|
+
console.log(`[stack-update] \u2713 SSH hardened: ${sshSummary}`);
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
function areStackContainersRunning(composeFile, envFile) {
|
|
995
|
+
try {
|
|
996
|
+
const result = spawnSync("docker", ["compose", "--file", composeFile, "--env-file", envFile, "ps", "-q"], {
|
|
997
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
998
|
+
timeout: 15e3
|
|
999
|
+
});
|
|
1000
|
+
return result.status === 0 && (result.stdout?.toString().trim().length ?? 0) > 0;
|
|
1001
|
+
} catch {
|
|
1002
|
+
return false;
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
var CRITICAL_BIND_MOUNTS = [
|
|
1006
|
+
{ file: ".env", description: "secrets and image tags" },
|
|
1007
|
+
{ file: "gateway.json", description: "gateway connector config" },
|
|
1008
|
+
{ file: "clickhouse-config.xml", description: "ClickHouse memory/log guardrails" }
|
|
1009
|
+
];
|
|
1010
|
+
var CRITICAL_VOLUMES = [
|
|
1011
|
+
"postgres_data",
|
|
1012
|
+
"gateway_data",
|
|
1013
|
+
"gateway_whatsapp_auth",
|
|
1014
|
+
"wiki_data",
|
|
1015
|
+
"exe_os_data",
|
|
1016
|
+
"crm_data",
|
|
1017
|
+
"monitor_hub_data",
|
|
1018
|
+
"monitor_agent_data",
|
|
1019
|
+
"redis_data",
|
|
1020
|
+
"clickhouse_data",
|
|
1021
|
+
"erp_sites"
|
|
1022
|
+
];
|
|
1023
|
+
function assertBindMountsExist(stackDir) {
|
|
1024
|
+
const missing = [];
|
|
1025
|
+
for (const mount of CRITICAL_BIND_MOUNTS) {
|
|
1026
|
+
const fullPath = path.join(stackDir, mount.file);
|
|
1027
|
+
if (!existsSync(fullPath)) {
|
|
1028
|
+
missing.push(`${mount.file} (${mount.description})`);
|
|
1029
|
+
}
|
|
1030
|
+
}
|
|
1031
|
+
if (missing.length > 0) {
|
|
1032
|
+
console.warn(`[stack-update] \u26A0 Missing critical bind mounts:
|
|
1033
|
+
- ${missing.join("\n - ")}`);
|
|
1034
|
+
console.warn("[stack-update] These files contain customer config that will be lost if not present.");
|
|
1035
|
+
}
|
|
1036
|
+
}
|
|
1037
|
+
function assertVolumesIntact() {
|
|
1038
|
+
const missing = [];
|
|
1039
|
+
for (const vol of CRITICAL_VOLUMES) {
|
|
1040
|
+
const result = spawnSync("docker", ["volume", "inspect", `exe-os_${vol}`], {
|
|
1041
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
1042
|
+
timeout: 5e3
|
|
1043
|
+
});
|
|
1044
|
+
if (result.status !== 0) {
|
|
1045
|
+
missing.push(vol);
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
if (missing.length > 0) {
|
|
1049
|
+
console.warn(`[stack-update] \u26A0 Critical volumes missing after update: ${missing.join(", ")}`);
|
|
1050
|
+
console.warn("[stack-update] Data may have been lost. Check if 'docker compose down -v' was used.");
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
function warnLegacyDaemonVolume() {
|
|
1054
|
+
const legacy = spawnSync("docker", ["volume", "inspect", "exe-os_exed_data"], {
|
|
1055
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
1056
|
+
timeout: 5e3
|
|
1057
|
+
});
|
|
1058
|
+
const current = spawnSync("docker", ["volume", "inspect", "exe-os_exe_os_data"], {
|
|
1059
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
1060
|
+
timeout: 5e3
|
|
1061
|
+
});
|
|
1062
|
+
if (legacy.status === 0 && current.status === 0) {
|
|
1063
|
+
console.warn("[stack-update] \u26A0 Legacy volume exe-os_exed_data exists alongside exe-os_exe_os_data.");
|
|
1064
|
+
console.warn("[stack-update] Keeping both volumes intact; after verifying exe_os_data has current daemon state, remove exed_data manually if desired.");
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
function applyPostgresInitSql(composeArgs, envFile, composeFile, exec) {
|
|
1068
|
+
const initSql = path.join(path.dirname(composeFile), "init-db.sql");
|
|
1069
|
+
refreshPackagedInitSql(initSql);
|
|
1070
|
+
if (!existsSync(initSql)) return;
|
|
1071
|
+
const env = parseEnv(readFileSync(envFile, "utf8"));
|
|
1072
|
+
const postgresUser = env.get("POSTGRES_USER") || "exe";
|
|
1073
|
+
const postgresDb = env.get("POSTGRES_DB") || "exedb";
|
|
1074
|
+
console.log("[stack-update] Applying idempotent Postgres init SQL for existing volumes...");
|
|
1075
|
+
exec("docker", [
|
|
1076
|
+
...composeArgs,
|
|
1077
|
+
"exec",
|
|
1078
|
+
"-T",
|
|
1079
|
+
"exe-db",
|
|
1080
|
+
"psql",
|
|
1081
|
+
"-v",
|
|
1082
|
+
"ON_ERROR_STOP=1",
|
|
1083
|
+
"-U",
|
|
1084
|
+
postgresUser,
|
|
1085
|
+
"-d",
|
|
1086
|
+
postgresDb,
|
|
1087
|
+
"-f",
|
|
1088
|
+
"/docker-entrypoint-initdb.d/01-init.sql"
|
|
1089
|
+
]);
|
|
1090
|
+
console.log("[stack-update] \u2713 Postgres init SQL applied");
|
|
1091
|
+
}
|
|
1092
|
+
async function runStackUpdate(options) {
|
|
1093
|
+
const exec = options.exec ?? defaultExec;
|
|
1094
|
+
const now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
1095
|
+
const phaseDurations = [];
|
|
1096
|
+
const startPhase = (name) => {
|
|
1097
|
+
const started = Date.now();
|
|
1098
|
+
console.log(`[stack-update] ${now().toISOString()} \u25B6 ${name} started`);
|
|
1099
|
+
return () => {
|
|
1100
|
+
const durationMs = Date.now() - started;
|
|
1101
|
+
phaseDurations.push({ name, durationMs });
|
|
1102
|
+
console.log(`[stack-update] ${now().toISOString()} \u2713 ${name} completed in ${(durationMs / 1e3).toFixed(1)}s`);
|
|
1103
|
+
};
|
|
1104
|
+
};
|
|
1105
|
+
const printPhaseSummary = () => {
|
|
1106
|
+
if (phaseDurations.length === 0) return;
|
|
1107
|
+
console.log("[stack-update] Phase duration summary:");
|
|
1108
|
+
for (const phase of phaseDurations) {
|
|
1109
|
+
console.log(` - ${phase.name}: ${(phase.durationMs / 1e3).toFixed(1)}s`);
|
|
1110
|
+
}
|
|
1111
|
+
};
|
|
1112
|
+
if (options.rollback) return rollbackStackUpdate(options);
|
|
1113
|
+
if (options.bootstrap !== false) {
|
|
1114
|
+
const report = bootstrapStackHost({ ...options, installDocker: options.installDocker ?? !!options.yes });
|
|
1115
|
+
if (!options.dryRun) assertHostReadyForApply(report);
|
|
1116
|
+
}
|
|
1117
|
+
assertBindMountsExist(path.dirname(options.envFile));
|
|
1118
|
+
warnLegacyDaemonVolume();
|
|
1119
|
+
if (!options.dryRun) {
|
|
1120
|
+
try {
|
|
1121
|
+
const endPhase = startPhase("preflight");
|
|
1122
|
+
if (!options.preflight) {
|
|
1123
|
+
throw new Error("preflight hook not provided");
|
|
1124
|
+
}
|
|
1125
|
+
const preflightReport = options.preflight({
|
|
1126
|
+
composeFile: options.composeFile,
|
|
1127
|
+
envFile: options.envFile
|
|
1128
|
+
});
|
|
1129
|
+
if (!preflightReport.passed) {
|
|
1130
|
+
const failures = preflightReport.results.filter((r) => r.status === "fail").map((r) => `${r.check}: ${r.message}`);
|
|
1131
|
+
throw new Error(`Preflight blocked deploy:
|
|
1132
|
+
- ${failures.join("\n- ")}`);
|
|
1133
|
+
}
|
|
1134
|
+
console.log("[stack-update] \u2713 Preflight passed");
|
|
1135
|
+
endPhase();
|
|
1136
|
+
} catch (preflightErr) {
|
|
1137
|
+
if (preflightErr instanceof Error && preflightErr.message.startsWith("Preflight blocked")) {
|
|
1138
|
+
throw preflightErr;
|
|
1139
|
+
}
|
|
1140
|
+
console.warn("[stack-update] Preflight check skipped (module not available)");
|
|
1141
|
+
}
|
|
1142
|
+
}
|
|
1143
|
+
const hubUrl = options.monitorHubUrl || parseEnv(readFileSync(options.envFile, "utf8")).get("MONITOR_HUB_URL") || "";
|
|
1144
|
+
const pairLicense = options.licenseKey || process.env.EXE_LICENSE_KEY || loadLicense() || "";
|
|
1145
|
+
const pairDomain = options.domain || process.env.EXE_STACK_DOMAIN || process.env.CUSTOMER_DOMAIN || "";
|
|
1146
|
+
if (hubUrl && pairLicense && pairDomain) {
|
|
1147
|
+
const envBefore = readFileSync(options.envFile, "utf8");
|
|
1148
|
+
const hasPlaceholder = /CHANGEME/.test(parseEnv(envBefore).get("MONITOR_AGENT_TOKEN") ?? "");
|
|
1149
|
+
if (hasPlaceholder) {
|
|
1150
|
+
const pair = options.pairMonitor ? options.pairMonitor(hubUrl, pairLicense, pairDomain, options.envFile) : pairMonitorAgent(hubUrl, pairLicense, pairDomain, options.envFile);
|
|
1151
|
+
const result = await pair;
|
|
1152
|
+
if (result.paired) {
|
|
1153
|
+
console.log(`[stack-update] Monitor agent paired: ${result.systemName}`);
|
|
1154
|
+
} else {
|
|
1155
|
+
console.warn(`[stack-update] Monitor pairing skipped: ${result.error}`);
|
|
1156
|
+
}
|
|
1157
|
+
}
|
|
1158
|
+
}
|
|
1159
|
+
const manifest = await loadStackManifest(options.manifestRef, options.fetchText, options.manifestPublicKey, options.manifestAuthToken);
|
|
1160
|
+
if (options.addService || options.removeService) {
|
|
1161
|
+
const effectiveVersion = options.targetVersion ?? manifest.latest;
|
|
1162
|
+
const manifestRelease = manifest.stacks[effectiveVersion];
|
|
1163
|
+
if (!manifestRelease) throw new Error(`Stack version ${effectiveVersion} not found in manifest`);
|
|
1164
|
+
const currentSelection = loadServiceSelection(options.envFile);
|
|
1165
|
+
const currentServices = new Set(currentSelection?.selectedServices ?? buildDefaultServiceSelection(manifestRelease));
|
|
1166
|
+
if (options.addService) {
|
|
1167
|
+
if (!manifestRelease.services[options.addService]) {
|
|
1168
|
+
throw new Error(`Service "${options.addService}" not found in manifest ${effectiveVersion}. Available: ${Object.keys(manifestRelease.services).join(", ")}`);
|
|
1169
|
+
}
|
|
1170
|
+
currentServices.add(options.addService);
|
|
1171
|
+
console.log(`[stack-update] Added service: ${options.addService}`);
|
|
1172
|
+
}
|
|
1173
|
+
if (options.removeService) {
|
|
1174
|
+
const svc = manifestRelease.services[options.removeService];
|
|
1175
|
+
if (svc?.required) throw new Error(`Cannot remove required service: ${options.removeService}`);
|
|
1176
|
+
currentServices.delete(options.removeService);
|
|
1177
|
+
console.log(`[stack-update] Removed service: ${options.removeService}`);
|
|
1178
|
+
}
|
|
1179
|
+
saveServiceSelection(options.envFile, [...currentServices]);
|
|
1180
|
+
console.log(`[stack-update] Saved service selection: ${[...currentServices].join(", ")}`);
|
|
1181
|
+
}
|
|
1182
|
+
if (options.services && options.services.length > 0) {
|
|
1183
|
+
saveServiceSelection(options.envFile, options.services);
|
|
1184
|
+
console.log(`[stack-update] Service selection overridden via --services: ${options.services.join(", ")}`);
|
|
1185
|
+
}
|
|
1186
|
+
const serviceSelection = loadServiceSelection(options.envFile);
|
|
1187
|
+
const envRaw = readFileSync(options.envFile, "utf8");
|
|
1188
|
+
const lockFile = options.lockFile ?? path.join(path.dirname(options.envFile), ".exe-stack-lock.json");
|
|
1189
|
+
const resumeLock = options.resume ? readStackUpdateLock(lockFile) : void 0;
|
|
1190
|
+
if (options.resume && !resumeLock) throw new Error(`Cannot resume stack update: lock file not found or unreadable at ${lockFile}`);
|
|
1191
|
+
const resumeServices = options.resume ? incompleteServicesFromLock(lockFile) : [];
|
|
1192
|
+
if (options.resume && resumeServices.length === 0) throw new Error("Cannot resume stack update: no incomplete services found in lock file");
|
|
1193
|
+
const scopedServiceNames = options.serviceName ? [options.serviceName] : resumeServices;
|
|
1194
|
+
const effectiveTargetVersion = options.targetVersion ?? resumeLock?.stackVersion;
|
|
1195
|
+
const plan = createStackUpdatePlan(manifest, envRaw, effectiveTargetVersion, scopedServiceNames, serviceSelection);
|
|
1196
|
+
const fullTargetPlan = scopedServiceNames.length > 0 ? createStackUpdatePlan(manifest, envRaw, effectiveTargetVersion, void 0, serviceSelection) : plan;
|
|
1197
|
+
assertBreakingChangesAllowed(plan, options.allowedBreakingChangeIds ?? []);
|
|
1198
|
+
assertDeploymentScopeAllowed(plan, options.deploymentPersona ?? "customer");
|
|
1199
|
+
const plannedEnvRaw = patchEnv(envRaw, Object.fromEntries(plan.changes.map((c) => [c.key, c.after])));
|
|
1200
|
+
const composeRaw = readFileSync(options.composeFile, "utf8");
|
|
1201
|
+
assertProductionDeployGate(plan, plannedEnvRaw, composeRaw, {
|
|
1202
|
+
breakGlassReason: options.breakGlassReason,
|
|
1203
|
+
breakGlassAuditFile: options.breakGlassAuditFile,
|
|
1204
|
+
now,
|
|
1205
|
+
envFile: options.envFile
|
|
1206
|
+
});
|
|
1207
|
+
const previousVersion = readCurrentStackVersion(lockFile);
|
|
1208
|
+
if (options.verifyImages) {
|
|
1209
|
+
const endPhase = startPhase("verify images");
|
|
1210
|
+
const results = await verifyManifestImagesAvailable(plan, options);
|
|
1211
|
+
endPhase();
|
|
1212
|
+
const failed = results.filter((r) => !r.ok);
|
|
1213
|
+
for (const result of results) {
|
|
1214
|
+
console.log(`[stack-update] ${result.ok ? "\u2713" : "\u2717"} image ${result.image}${result.error ? ` \u2014 ${result.error}` : ""}`);
|
|
1215
|
+
}
|
|
1216
|
+
printPhaseSummary();
|
|
1217
|
+
if (failed.length > 0) {
|
|
1218
|
+
throw new Error(`Image verification failed for ${failed.length} image(s).`);
|
|
1219
|
+
}
|
|
1220
|
+
return { status: "planned", targetVersion: plan.targetVersion, changes: plan.changes, lockFile };
|
|
1221
|
+
}
|
|
1222
|
+
const containersRunning = plan.changes.length === 0 ? areStackContainersRunning(options.composeFile, options.envFile) : true;
|
|
1223
|
+
if (options.dryRun || !options.resume && plan.changes.length === 0 && containersRunning) {
|
|
1224
|
+
return { status: "planned", targetVersion: plan.targetVersion, changes: plan.changes, lockFile };
|
|
1225
|
+
}
|
|
1226
|
+
await postDeployAudit(options, "started", plan.targetVersion, previousVersion);
|
|
1227
|
+
try {
|
|
1228
|
+
if (!options.preDeployBackup) {
|
|
1229
|
+
throw new Error("preDeployBackup hook not provided");
|
|
1230
|
+
}
|
|
1231
|
+
options.preDeployBackup(plan.targetVersion);
|
|
1232
|
+
} catch {
|
|
1233
|
+
}
|
|
1234
|
+
const stackDir = path.dirname(options.envFile);
|
|
1235
|
+
const backupDir = path.join(stackDir, ".exe-stack-backups");
|
|
1236
|
+
mkdirSync(backupDir, { recursive: true });
|
|
1237
|
+
const stamp = now().toISOString().replace(/[:.]/g, "-");
|
|
1238
|
+
const updateBackupDir = path.join(backupDir, `pre-update-${stamp}`);
|
|
1239
|
+
mkdirSync(updateBackupDir, { recursive: true });
|
|
1240
|
+
const backupEnvFile = path.join(updateBackupDir, "env.bak");
|
|
1241
|
+
writeFileSync(backupEnvFile, envRaw, { mode: 384 });
|
|
1242
|
+
const protectedFiles = ["gateway.json", "branding.json", "clickhouse-config.xml"];
|
|
1243
|
+
for (const f of protectedFiles) {
|
|
1244
|
+
const src = path.join(stackDir, f);
|
|
1245
|
+
try {
|
|
1246
|
+
if (existsSync(src)) {
|
|
1247
|
+
copyFileSync(src, path.join(updateBackupDir, f));
|
|
1248
|
+
}
|
|
1249
|
+
} catch {
|
|
1250
|
+
}
|
|
1251
|
+
}
|
|
1252
|
+
const cfDir = path.join(stackDir, "cloudflared");
|
|
1253
|
+
try {
|
|
1254
|
+
if (existsSync(cfDir)) {
|
|
1255
|
+
const cfBackup = path.join(updateBackupDir, "cloudflared");
|
|
1256
|
+
mkdirSync(cfBackup, { recursive: true });
|
|
1257
|
+
for (const f of readdirSync(cfDir)) {
|
|
1258
|
+
copyFileSync(path.join(cfDir, f), path.join(cfBackup, f));
|
|
1259
|
+
}
|
|
1260
|
+
}
|
|
1261
|
+
} catch {
|
|
1262
|
+
}
|
|
1263
|
+
console.log(`[stack-update] Config backed up to ${updateBackupDir}`);
|
|
1264
|
+
const initialServiceStatus = options.resume && resumeLock?.serviceStatus ? resumeLock.serviceStatus : buildInitialServiceStatus(fullTargetPlan, envRaw, now, new Set(Object.keys(plan.release.services)));
|
|
1265
|
+
writeStackUpdateLock(lockFile, {
|
|
1266
|
+
stackVersion: plan.targetVersion,
|
|
1267
|
+
previousVersion,
|
|
1268
|
+
updatedAt: now().toISOString(),
|
|
1269
|
+
backupEnvFile: resumeLock?.backupEnvFile ?? backupEnvFile,
|
|
1270
|
+
services: { ...fullTargetPlan.release.services, ...resumeLock?.services ?? {}, ...plan.release.services },
|
|
1271
|
+
serviceStatus: initialServiceStatus
|
|
1272
|
+
});
|
|
1273
|
+
try {
|
|
1274
|
+
const { spawnSync: sp } = await import("child_process");
|
|
1275
|
+
const backupSh = path.join(stackDir, "backup.sh");
|
|
1276
|
+
if (existsSync(backupSh)) {
|
|
1277
|
+
console.log("[stack-update] Uploading pre-update snapshot to R2...");
|
|
1278
|
+
const r2Result = sp("bash", [backupSh, "--upload-r2"], {
|
|
1279
|
+
encoding: "utf8",
|
|
1280
|
+
timeout: 3e5,
|
|
1281
|
+
// 5 min max
|
|
1282
|
+
cwd: stackDir,
|
|
1283
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
1284
|
+
});
|
|
1285
|
+
if (r2Result.status === 0) {
|
|
1286
|
+
console.log("[stack-update] \u2713 Pre-update snapshot uploaded to R2");
|
|
1287
|
+
} else {
|
|
1288
|
+
console.warn("[stack-update] R2 upload failed (non-blocking) \u2014 local backup preserved");
|
|
1289
|
+
}
|
|
1290
|
+
}
|
|
1291
|
+
} catch {
|
|
1292
|
+
}
|
|
1293
|
+
const updates = Object.fromEntries(plan.changes.map((c) => [c.key, c.after]));
|
|
1294
|
+
const patched = patchEnv(envRaw, updates);
|
|
1295
|
+
const tmp = `${options.envFile}.tmp-${process.pid}`;
|
|
1296
|
+
writeFileSync(tmp, patched, { mode: 384 });
|
|
1297
|
+
renameSync(tmp, options.envFile);
|
|
1298
|
+
for (const serviceName of Object.keys(plan.release.services)) {
|
|
1299
|
+
updateServiceStatus(lockFile, serviceName, "env_patched", now);
|
|
1300
|
+
}
|
|
1301
|
+
const composeArgs = ["compose", "--file", options.composeFile, "--env-file", options.envFile];
|
|
1302
|
+
let registryForLogout;
|
|
1303
|
+
try {
|
|
1304
|
+
let creds = await fetchImageCredentials(options);
|
|
1305
|
+
if (!creds) {
|
|
1306
|
+
const manual = options.registryCredentials || process.env.EXE_REGISTRY_CREDENTIALS;
|
|
1307
|
+
if (manual) {
|
|
1308
|
+
const [username, ...rest] = manual.split(":");
|
|
1309
|
+
const password = rest.join(":");
|
|
1310
|
+
if (username && password) {
|
|
1311
|
+
const sampleImage = Object.values(plan.release.services)[0]?.image ?? "";
|
|
1312
|
+
const registry = sampleImage.startsWith("update.askexe.com") ? "update.askexe.com" : "ghcr.io";
|
|
1313
|
+
creds = { registry, username, password };
|
|
1314
|
+
console.log(`[stack-update] Using manual registry credentials for ${registry}.`);
|
|
1315
|
+
}
|
|
1316
|
+
}
|
|
1317
|
+
}
|
|
1318
|
+
if (!creds) {
|
|
1319
|
+
const sampleImage = Object.values(plan.release.services)[0]?.image ?? "";
|
|
1320
|
+
if (sampleImage.startsWith("update.askexe.com")) {
|
|
1321
|
+
const stackEnvPullTokensRaw = process.env.EXE_REGISTRY_PROXY_PULL_TOKENS || (existsSync(options.envFile) ? (() => {
|
|
1322
|
+
const envMap = parseEnv(readFileSync(options.envFile, "utf8"));
|
|
1323
|
+
return envMap.get("EXE_REGISTRY_PROXY_PULL_TOKENS") ?? "";
|
|
1324
|
+
})() : "");
|
|
1325
|
+
const firstPullToken = stackEnvPullTokensRaw.split(/[\n,]/).map((s) => s.trim()).find(Boolean);
|
|
1326
|
+
if (firstPullToken) {
|
|
1327
|
+
creds = { registry: "update.askexe.com", username: "token", password: firstPullToken };
|
|
1328
|
+
console.log("[stack-update] Using EXE_REGISTRY_PROXY_PULL_TOKENS for update.askexe.com registry auth.");
|
|
1329
|
+
}
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
if (!creds) {
|
|
1333
|
+
const sampleImage = Object.values(plan.release.services)[0]?.image ?? "";
|
|
1334
|
+
if (sampleImage.startsWith("update.askexe.com")) {
|
|
1335
|
+
const licenseKey = options.manifestAuthToken || options.licenseKey || process.env.EXE_LICENSE_KEY;
|
|
1336
|
+
if (licenseKey) {
|
|
1337
|
+
creds = { registry: "update.askexe.com", username: "license", password: licenseKey };
|
|
1338
|
+
console.log("[stack-update] Using license key for update.askexe.com registry auth.");
|
|
1339
|
+
}
|
|
1340
|
+
}
|
|
1341
|
+
}
|
|
1342
|
+
if (creds) {
|
|
1343
|
+
(options.dockerLogin ?? defaultDockerLogin)(creds);
|
|
1344
|
+
registryForLogout = creds.registry;
|
|
1345
|
+
}
|
|
1346
|
+
const imagesToPull = manifestImagesForPull(plan.release);
|
|
1347
|
+
const allServiceNames = Object.keys(plan.release.services);
|
|
1348
|
+
const progress = new ProgressReporter(allServiceNames);
|
|
1349
|
+
progress.resetCounter(imagesToPull.length);
|
|
1350
|
+
progress.startPhase("pull");
|
|
1351
|
+
for (let imgIdx = 0; imgIdx < imagesToPull.length; imgIdx++) {
|
|
1352
|
+
const image = imagesToPull[imgIdx];
|
|
1353
|
+
const endPull = startPhase(`pull ${image}`);
|
|
1354
|
+
progress.logServiceStep(imgIdx + 1, image, "Pulling image");
|
|
1355
|
+
exec("docker", ["pull", image]);
|
|
1356
|
+
for (const [serviceName, service] of Object.entries(plan.release.services)) {
|
|
1357
|
+
if (service.image === image) updateServiceStatus(lockFile, serviceName, "pulled", now);
|
|
1358
|
+
}
|
|
1359
|
+
endPull();
|
|
1360
|
+
}
|
|
1361
|
+
const endComposePull = startPhase("docker compose pull");
|
|
1362
|
+
exec("docker", [...composeArgs, "pull"]);
|
|
1363
|
+
endComposePull();
|
|
1364
|
+
progress.endPhase();
|
|
1365
|
+
const serviceEntries = Object.entries(plan.release.services);
|
|
1366
|
+
progress.resetCounter(serviceEntries.length);
|
|
1367
|
+
progress.startPhase("verify");
|
|
1368
|
+
for (let svcIdx = 0; svcIdx < serviceEntries.length; svcIdx++) {
|
|
1369
|
+
const [serviceName, service] = serviceEntries[svcIdx];
|
|
1370
|
+
const endVerify = startPhase(`verify ${serviceName} image`);
|
|
1371
|
+
const image = service.image;
|
|
1372
|
+
progress.logServiceStep(svcIdx + 1, serviceName, "Verifying image");
|
|
1373
|
+
try {
|
|
1374
|
+
exec("docker", ["image", "inspect", image, "--format", "{{.Id}}"]);
|
|
1375
|
+
} catch {
|
|
1376
|
+
progress.logServiceFailure(
|
|
1377
|
+
serviceName,
|
|
1378
|
+
`Image ${image} was not pulled successfully. Check registry authentication \u2014 if the image is on update.askexe.com, ensure EXE_LICENSE_KEY or EXE_REGISTRY_PROXY_PULL_TOKENS is set.`,
|
|
1379
|
+
options.composeFile
|
|
1380
|
+
);
|
|
1381
|
+
throw new Error(
|
|
1382
|
+
`Image ${image} for service ${serviceName} was not pulled successfully. Check registry authentication \u2014 if the image is on update.askexe.com, ensure EXE_LICENSE_KEY or EXE_REGISTRY_PROXY_PULL_TOKENS is set.`
|
|
1383
|
+
);
|
|
1384
|
+
}
|
|
1385
|
+
updateServiceStatus(lockFile, serviceName, "verified", now);
|
|
1386
|
+
endVerify();
|
|
1387
|
+
}
|
|
1388
|
+
progress.endPhase();
|
|
1389
|
+
const migratable = Object.entries(plan.release.services).filter(([, s]) => s.migrations?.command);
|
|
1390
|
+
if (migratable.length > 0) {
|
|
1391
|
+
progress.resetCounter(migratable.length);
|
|
1392
|
+
progress.startPhase("migrate");
|
|
1393
|
+
}
|
|
1394
|
+
for (const [serviceName, service] of Object.entries(plan.release.services)) {
|
|
1395
|
+
if (!service.migrations?.command) continue;
|
|
1396
|
+
const composeServiceName = service.composeService ?? serviceName;
|
|
1397
|
+
const endMigration = startPhase(`migrate ${composeServiceName}`);
|
|
1398
|
+
progress.logServiceProgress(composeServiceName, "Migrating");
|
|
1399
|
+
try {
|
|
1400
|
+
const migrationArgs = service.migrations.command.split(/\s+/);
|
|
1401
|
+
exec("docker", [
|
|
1402
|
+
...composeArgs,
|
|
1403
|
+
"run",
|
|
1404
|
+
"--rm",
|
|
1405
|
+
"--no-deps",
|
|
1406
|
+
composeServiceName,
|
|
1407
|
+
...migrationArgs
|
|
1408
|
+
]);
|
|
1409
|
+
console.log(`[stack-update] \u2713 Migrations for ${composeServiceName} completed`);
|
|
1410
|
+
updateServiceStatus(lockFile, serviceName, "migrated", now);
|
|
1411
|
+
endMigration();
|
|
1412
|
+
} catch (migErr) {
|
|
1413
|
+
const reason = migErr instanceof Error ? migErr.message : String(migErr);
|
|
1414
|
+
progress.logServiceFailure(composeServiceName, `Migration failed: ${reason}`, options.composeFile);
|
|
1415
|
+
updateServiceStatus(lockFile, serviceName, "failed", now, reason);
|
|
1416
|
+
throw new Error(`Migration failed for ${composeServiceName} \u2014 aborting update. Fix the migration and retry. Error: ${reason}`);
|
|
1417
|
+
}
|
|
1418
|
+
}
|
|
1419
|
+
if (migratable.length > 0) {
|
|
1420
|
+
progress.endPhase();
|
|
1421
|
+
}
|
|
1422
|
+
const RESTART_ORDER = [
|
|
1423
|
+
"exe-db",
|
|
1424
|
+
// data layer — must be healthy before apps
|
|
1425
|
+
"gotrue",
|
|
1426
|
+
// auth — depends on idempotent init-db.sql applied after exe-db
|
|
1427
|
+
"clickhouse",
|
|
1428
|
+
"redis",
|
|
1429
|
+
"exe-os",
|
|
1430
|
+
// daemon — has its own health endpoint
|
|
1431
|
+
"exe-crm",
|
|
1432
|
+
// CRM app
|
|
1433
|
+
"exe-crm-worker",
|
|
1434
|
+
// CRM background worker
|
|
1435
|
+
"exe-gateway",
|
|
1436
|
+
// gateway — WhatsApp connections
|
|
1437
|
+
"exe-wiki",
|
|
1438
|
+
// wiki
|
|
1439
|
+
"exe-erp",
|
|
1440
|
+
// ERP — depends on exe-db + redis (dbs 3/4/5)
|
|
1441
|
+
"exe-erp-websocket",
|
|
1442
|
+
// ERP WebSocket (Socket.io)
|
|
1443
|
+
"exe-erp-queue",
|
|
1444
|
+
// ERP background workers (RQ)
|
|
1445
|
+
"exe-erp-scheduler",
|
|
1446
|
+
// ERP scheduled tasks
|
|
1447
|
+
"cloudflared"
|
|
1448
|
+
// tunnel — restarted last so services are healthy before it connects
|
|
1449
|
+
];
|
|
1450
|
+
const scopedComposeServices = Object.entries(plan.release.services).map(([serviceName, service]) => service.composeService ?? serviceName);
|
|
1451
|
+
const restartOrder = [
|
|
1452
|
+
...RESTART_ORDER.filter((service) => scopedComposeServices.includes(service)),
|
|
1453
|
+
...scopedComposeServices.filter((service) => !RESTART_ORDER.includes(service))
|
|
1454
|
+
];
|
|
1455
|
+
const preSnapshot = spawnSync("docker", ["ps", "--format", "json"], { encoding: "utf8", timeout: 1e4 });
|
|
1456
|
+
const preContainerCount = preSnapshot.stdout?.split("\n").filter(Boolean).length ?? 0;
|
|
1457
|
+
console.log(`[stack-update] Pre-update snapshot: ${preContainerCount} containers`);
|
|
1458
|
+
let restartedAnyService = false;
|
|
1459
|
+
progress.resetCounter(restartOrder.length);
|
|
1460
|
+
progress.startPhase("restart");
|
|
1461
|
+
let restartIndex = 0;
|
|
1462
|
+
for (const service of restartOrder) {
|
|
1463
|
+
restartIndex++;
|
|
1464
|
+
const endRestart = startPhase(`restart ${service}`);
|
|
1465
|
+
const manifestServiceName = Object.entries(plan.release.services).find(([name, svc]) => (svc.composeService ?? name) === service)?.[0] ?? service;
|
|
1466
|
+
const psResult = spawnSync("docker", [...composeArgs, "ps", "--quiet", service], {
|
|
1467
|
+
encoding: "utf8",
|
|
1468
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
1469
|
+
timeout: 1e4
|
|
1470
|
+
});
|
|
1471
|
+
if (psResult.status !== 0) continue;
|
|
1472
|
+
progress.logServiceStep(restartIndex, service, "Restarting");
|
|
1473
|
+
exec("docker", [...composeArgs, "up", "-d", "--remove-orphans", "--no-deps", service]);
|
|
1474
|
+
restartedAnyService = true;
|
|
1475
|
+
updateServiceStatus(lockFile, manifestServiceName, "restarted", now);
|
|
1476
|
+
const maxWait = options.healthTimeoutSecs ?? 180;
|
|
1477
|
+
let healthy = false;
|
|
1478
|
+
for (let i = 0; i < maxWait; i++) {
|
|
1479
|
+
const inspectResult = spawnSync(
|
|
1480
|
+
"docker",
|
|
1481
|
+
["inspect", "--format", "{{.State.Health.Status}}", service],
|
|
1482
|
+
{ encoding: "utf8", timeout: 5e3 }
|
|
1483
|
+
);
|
|
1484
|
+
const status = inspectResult.stdout?.trim();
|
|
1485
|
+
if (status === "healthy") {
|
|
1486
|
+
healthy = true;
|
|
1487
|
+
break;
|
|
1488
|
+
}
|
|
1489
|
+
if (status === "" || inspectResult.status !== 0) {
|
|
1490
|
+
healthy = true;
|
|
1491
|
+
break;
|
|
1492
|
+
}
|
|
1493
|
+
await new Promise((r) => setTimeout(r, 1e3));
|
|
1494
|
+
}
|
|
1495
|
+
if (!healthy) {
|
|
1496
|
+
const failMsg = `Service ${service} failed health check after ${maxWait}s`;
|
|
1497
|
+
progress.logServiceFailure(service, failMsg, options.composeFile);
|
|
1498
|
+
updateServiceStatus(lockFile, manifestServiceName, "failed", now, failMsg);
|
|
1499
|
+
throw new Error(`${failMsg} \u2014 aborting update`);
|
|
1500
|
+
}
|
|
1501
|
+
if (service === "exe-db") {
|
|
1502
|
+
applyPostgresInitSql(composeArgs, options.envFile, options.composeFile, exec);
|
|
1503
|
+
}
|
|
1504
|
+
progress.logServiceProgress(service, "Restarted");
|
|
1505
|
+
updateServiceStatus(lockFile, manifestServiceName, "healthy", now);
|
|
1506
|
+
endRestart();
|
|
1507
|
+
}
|
|
1508
|
+
progress.endPhase();
|
|
1509
|
+
if (!restartedAnyService) {
|
|
1510
|
+
exec("docker", [...composeArgs, "up", "-d", "--remove-orphans"]);
|
|
1511
|
+
}
|
|
1512
|
+
const postSnapshot = spawnSync("docker", ["ps", "--format", "json"], { encoding: "utf8", timeout: 1e4 });
|
|
1513
|
+
const postContainerCount = postSnapshot.stdout?.split("\n").filter(Boolean).length ?? 0;
|
|
1514
|
+
console.log(`[stack-update] Post-update snapshot: ${postContainerCount} containers (was ${preContainerCount})`);
|
|
1515
|
+
if (postContainerCount < preContainerCount) {
|
|
1516
|
+
console.warn(`[stack-update] \u26A0 Container count dropped from ${preContainerCount} to ${postContainerCount}`);
|
|
1517
|
+
}
|
|
1518
|
+
await verifyReleaseHealth(plan.release, options.healthRetries ?? 12, options.healthDelayMs ?? 5e3);
|
|
1519
|
+
for (const serviceName of Object.keys(plan.release.services)) {
|
|
1520
|
+
updateServiceStatus(lockFile, serviceName, "completed", now);
|
|
1521
|
+
}
|
|
1522
|
+
try {
|
|
1523
|
+
if (!options.healthGate) {
|
|
1524
|
+
throw new Error("healthGate hook not provided");
|
|
1525
|
+
}
|
|
1526
|
+
const gateResult = await options.healthGate();
|
|
1527
|
+
options.logHealthGateResult?.(gateResult);
|
|
1528
|
+
if (!gateResult.passed) {
|
|
1529
|
+
const failed = gateResult.results.filter((r) => r.status === "fail").map((r) => r.check);
|
|
1530
|
+
throw new Error(`Health gate failed: ${failed.join(", ")}`);
|
|
1531
|
+
}
|
|
1532
|
+
console.log("[stack-update] \u2713 Health gate passed");
|
|
1533
|
+
} catch (gateErr) {
|
|
1534
|
+
if (gateErr instanceof Error && gateErr.message.startsWith("Health gate failed")) {
|
|
1535
|
+
throw gateErr;
|
|
1536
|
+
}
|
|
1537
|
+
console.warn("[stack-update] Health gate check skipped (module not available)");
|
|
1538
|
+
}
|
|
1539
|
+
assertVolumesIntact();
|
|
1540
|
+
try {
|
|
1541
|
+
if (!options.verifyStack) {
|
|
1542
|
+
throw new Error("verifyStack hook not provided");
|
|
1543
|
+
}
|
|
1544
|
+
const verifyReport = await options.verifyStack({ composeFile: options.composeFile, envFile: options.envFile });
|
|
1545
|
+
const securityFailures = [];
|
|
1546
|
+
for (const r of verifyReport.results) {
|
|
1547
|
+
if (r.status === "fail") {
|
|
1548
|
+
console.warn(`[verify-stack] FAIL: ${r.check}: ${r.message}`);
|
|
1549
|
+
if (/security|auth|signup|password|ssl|tls|bind|exposed/i.test(r.check)) {
|
|
1550
|
+
securityFailures.push(`${r.check}: ${r.message}`);
|
|
1551
|
+
}
|
|
1552
|
+
}
|
|
1553
|
+
}
|
|
1554
|
+
if (securityFailures.length > 0) {
|
|
1555
|
+
throw new Error(`Security verification failed:
|
|
1556
|
+
- ${securityFailures.join("\n - ")}
|
|
1557
|
+
Fix these before the stack is safe to operate.`);
|
|
1558
|
+
}
|
|
1559
|
+
} catch (verifyErr) {
|
|
1560
|
+
if (verifyErr instanceof Error && verifyErr.message.startsWith("Security verification failed")) {
|
|
1561
|
+
throw verifyErr;
|
|
1562
|
+
}
|
|
1563
|
+
console.warn("[stack-update] Post-deploy verification skipped (module not available)");
|
|
1564
|
+
}
|
|
1565
|
+
if (!options.skipHardening) {
|
|
1566
|
+
try {
|
|
1567
|
+
const hardenResult = hardenHost(exec);
|
|
1568
|
+
if (!hardenResult.skipped) {
|
|
1569
|
+
const actions = [...hardenResult.ufw.actions, ...hardenResult.ssh.actions].filter(Boolean);
|
|
1570
|
+
if (actions.length > 0) {
|
|
1571
|
+
console.log(`[stack-update] Host hardening: ${actions.join(", ")}`);
|
|
1572
|
+
}
|
|
1573
|
+
}
|
|
1574
|
+
} catch (hardenErr) {
|
|
1575
|
+
const reason = hardenErr instanceof Error ? hardenErr.message : String(hardenErr);
|
|
1576
|
+
console.warn(`[stack-update] Host hardening failed (non-fatal): ${reason}`);
|
|
1577
|
+
}
|
|
1578
|
+
} else {
|
|
1579
|
+
console.log("[stack-update] Host hardening skipped (--skip-hardening)");
|
|
1580
|
+
}
|
|
1581
|
+
progress.logOverallSummary();
|
|
1582
|
+
printPhaseSummary();
|
|
1583
|
+
const finalLock = readStackUpdateLock(lockFile) ?? {};
|
|
1584
|
+
writeStackUpdateLock(lockFile, { ...finalLock, stackVersion: plan.targetVersion, previousVersion, updatedAt: now().toISOString(), backupEnvFile: finalLock.backupEnvFile ?? backupEnvFile, services: { ...fullTargetPlan.release.services, ...finalLock.services ?? {}, ...plan.release.services } });
|
|
1585
|
+
await postDeployAudit(options, "success", plan.targetVersion, previousVersion, void 0, { changes: plan.changes.length });
|
|
1586
|
+
return { status: "updated", targetVersion: plan.targetVersion, changes: plan.changes, backupEnvFile, lockFile, serviceStatus: readStackUpdateLock(lockFile)?.serviceStatus };
|
|
1587
|
+
} catch (err) {
|
|
1588
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
1589
|
+
for (const serviceName of Object.keys(plan.release.services)) {
|
|
1590
|
+
const currentStatus = readStackUpdateLock(lockFile)?.serviceStatus?.[serviceName]?.status;
|
|
1591
|
+
if (currentStatus !== "completed") updateServiceStatus(lockFile, serviceName, "failed", now, reason);
|
|
1592
|
+
}
|
|
1593
|
+
console.error(`[stack-update] ROLLBACK starting for ${plan.targetVersion}: ${reason}`);
|
|
1594
|
+
if (plan.release.services && Object.values(plan.release.services).some((s) => s.migrations?.command)) {
|
|
1595
|
+
console.error(
|
|
1596
|
+
"[stack-update] ROLLBACK warning: pre-swap migrations may not be reversible. Running configured rollback migrations where available; verify schema compatibility before re-running old images."
|
|
1597
|
+
);
|
|
1598
|
+
}
|
|
1599
|
+
console.log("[stack-update] Attempting automatic rollback\u2026");
|
|
1600
|
+
try {
|
|
1601
|
+
await rollbackStackUpdate({
|
|
1602
|
+
...options,
|
|
1603
|
+
lockFile,
|
|
1604
|
+
serviceName: scopedServiceNames.length === 1 ? scopedServiceNames[0] : options.serviceName
|
|
1605
|
+
});
|
|
1606
|
+
console.log("[stack-update] Automatic rollback completed.");
|
|
1607
|
+
} catch (rollbackErr) {
|
|
1608
|
+
const rbReason = rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr);
|
|
1609
|
+
console.error(`[stack-update] Rollback via rollbackStackUpdate failed: ${rbReason}`);
|
|
1610
|
+
console.log("[stack-update] Falling back to basic env restore\u2026");
|
|
1611
|
+
writeFileSync(options.envFile, envRaw, { mode: 384 });
|
|
1612
|
+
try {
|
|
1613
|
+
exec("docker", [...composeArgs, "up", "-d", "--remove-orphans"]);
|
|
1614
|
+
} catch {
|
|
1615
|
+
}
|
|
1616
|
+
}
|
|
1617
|
+
await postDeployAudit(options, "failed", plan.targetVersion, previousVersion, reason, { rollbackAttempted: true });
|
|
1618
|
+
throw new Error(`Stack update failed and rollback was attempted: ${reason}`);
|
|
1619
|
+
} finally {
|
|
1620
|
+
if (registryForLogout) {
|
|
1621
|
+
try {
|
|
1622
|
+
(options.dockerLogout ?? defaultDockerLogout)(registryForLogout);
|
|
1623
|
+
} catch {
|
|
1624
|
+
}
|
|
1625
|
+
}
|
|
1626
|
+
}
|
|
1627
|
+
}
|
|
1628
|
+
async function fetchImageCredentials(options) {
|
|
1629
|
+
if (!options.imageCredentialsUrl) return null;
|
|
1630
|
+
try {
|
|
1631
|
+
const res = await fetch(options.imageCredentialsUrl, {
|
|
1632
|
+
method: "POST",
|
|
1633
|
+
headers: {
|
|
1634
|
+
"content-type": "application/json",
|
|
1635
|
+
...options.manifestAuthToken ? { authorization: `Bearer ${options.manifestAuthToken}` } : {}
|
|
1636
|
+
},
|
|
1637
|
+
body: JSON.stringify({ deviceId: options.deviceId, licenseKey: options.licenseKey }),
|
|
1638
|
+
signal: AbortSignal.timeout(1e4)
|
|
1639
|
+
});
|
|
1640
|
+
if (!res.ok) {
|
|
1641
|
+
let body = "";
|
|
1642
|
+
try {
|
|
1643
|
+
body = (await res.text()).slice(0, 200);
|
|
1644
|
+
} catch {
|
|
1645
|
+
}
|
|
1646
|
+
const suffix = body ? `: ${body}` : "";
|
|
1647
|
+
if (res.status === 401 || res.status === 403) {
|
|
1648
|
+
console.warn(
|
|
1649
|
+
`[stack-update] Image credentials denied (HTTP ${res.status}${suffix}). AskExe must provision registry entitlement for this license/device, or provide EXE_REGISTRY_CREDENTIALS=user:token. Docker pull will use existing auth.`
|
|
1650
|
+
);
|
|
1651
|
+
} else {
|
|
1652
|
+
console.warn(`[stack-update] Image credentials unavailable (HTTP ${res.status}${suffix}). Docker pull will use existing auth.`);
|
|
1653
|
+
}
|
|
1654
|
+
return null;
|
|
1655
|
+
}
|
|
1656
|
+
return await res.json();
|
|
1657
|
+
} catch (err) {
|
|
1658
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
1659
|
+
console.warn(`[stack-update] Image credentials fetch failed: ${reason}. Docker pull will use existing auth.`);
|
|
1660
|
+
return null;
|
|
1661
|
+
}
|
|
1662
|
+
}
|
|
1663
|
+
function readCurrentStackVersion(lockFile) {
|
|
1664
|
+
if (!existsSync(lockFile)) return void 0;
|
|
1665
|
+
try {
|
|
1666
|
+
const parsed = JSON.parse(readFileSync(lockFile, "utf8"));
|
|
1667
|
+
return parsed.stackVersion;
|
|
1668
|
+
} catch {
|
|
1669
|
+
return void 0;
|
|
1670
|
+
}
|
|
1671
|
+
}
|
|
1672
|
+
function listAvailableVersions(manifest, currentVersion) {
|
|
1673
|
+
const versions = Object.keys(manifest.stacks).sort((a, b) => compareVersions(a, b));
|
|
1674
|
+
return versions.map((v) => ({
|
|
1675
|
+
version: v,
|
|
1676
|
+
releasedAt: manifest.stacks[v]?.releasedAt,
|
|
1677
|
+
notes: manifest.stacks[v]?.notes,
|
|
1678
|
+
isCurrent: v === currentVersion,
|
|
1679
|
+
isLatest: v === manifest.latest
|
|
1680
|
+
}));
|
|
1681
|
+
}
|
|
1682
|
+
function compareVersions(a, b) {
|
|
1683
|
+
const pa = a.split(".").map(Number);
|
|
1684
|
+
const pb = b.split(".").map(Number);
|
|
1685
|
+
for (let i = 0; i < Math.max(pa.length, pb.length); i++) {
|
|
1686
|
+
const diff = (pa[i] ?? 0) - (pb[i] ?? 0);
|
|
1687
|
+
if (diff !== 0) return diff;
|
|
1688
|
+
}
|
|
1689
|
+
return 0;
|
|
1690
|
+
}
|
|
1691
|
+
async function postDeployAudit(options, status, stackVersion, previousVersion, error, metadata) {
|
|
1692
|
+
if (!options.auditUrl) return;
|
|
1693
|
+
const postJson = options.postJson ?? defaultPostJson;
|
|
1694
|
+
try {
|
|
1695
|
+
await postJson(options.auditUrl, {
|
|
1696
|
+
stackVersion,
|
|
1697
|
+
previousVersion,
|
|
1698
|
+
status,
|
|
1699
|
+
error,
|
|
1700
|
+
metadata,
|
|
1701
|
+
deviceId: options.deviceId,
|
|
1702
|
+
licenseKey: options.licenseKey
|
|
1703
|
+
}, options.manifestAuthToken);
|
|
1704
|
+
} catch (err) {
|
|
1705
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
1706
|
+
console.warn(`[stack-update] deploy audit failed: ${reason}`);
|
|
1707
|
+
}
|
|
1708
|
+
}
|
|
1709
|
+
async function verifyReleaseHealth(release, retries, delayMs) {
|
|
1710
|
+
for (const [serviceName, service] of Object.entries(release.services)) {
|
|
1711
|
+
if (!service.healthUrl) continue;
|
|
1712
|
+
await waitForHttpOk(service.healthUrl, retries, delayMs, serviceName);
|
|
1713
|
+
}
|
|
1714
|
+
}
|
|
1715
|
+
async function waitForHttpOk(url, retries, delayMs, label) {
|
|
1716
|
+
let last = "";
|
|
1717
|
+
for (let i = 0; i < retries; i++) {
|
|
1718
|
+
try {
|
|
1719
|
+
const status = await httpStatus(url);
|
|
1720
|
+
if (status >= 200 && status < 300) return;
|
|
1721
|
+
last = `HTTP ${status}`;
|
|
1722
|
+
} catch (err) {
|
|
1723
|
+
last = err instanceof Error ? err.message : String(err);
|
|
1724
|
+
}
|
|
1725
|
+
if (i < retries - 1) await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
1726
|
+
}
|
|
1727
|
+
throw new Error(`Health check failed for ${label} (${url}): ${last}`);
|
|
1728
|
+
}
|
|
1729
|
+
function httpStatus(urlString) {
|
|
1730
|
+
return new Promise((resolve, reject) => {
|
|
1731
|
+
const url = new URL(urlString);
|
|
1732
|
+
const mod = url.protocol === "https:" ? https : http;
|
|
1733
|
+
const req = mod.request(url, { method: "GET", timeout: 5e3 }, (res) => {
|
|
1734
|
+
res.resume();
|
|
1735
|
+
resolve(res.statusCode ?? 0);
|
|
1736
|
+
});
|
|
1737
|
+
req.on("timeout", () => req.destroy(new Error("timeout")));
|
|
1738
|
+
req.on("error", reject);
|
|
1739
|
+
req.end();
|
|
1740
|
+
});
|
|
1741
|
+
}
|
|
1742
|
+
function defaultExec(cmd, args, opts) {
|
|
1743
|
+
execFileSync(cmd, args, { stdio: "inherit", cwd: opts?.cwd });
|
|
1744
|
+
}
|
|
1745
|
+
function defaultDockerLogin(creds) {
|
|
1746
|
+
execFileSync("docker", ["login", creds.registry, "-u", creds.username, "--password-stdin"], {
|
|
1747
|
+
input: creds.password,
|
|
1748
|
+
stdio: ["pipe", "inherit", "inherit"]
|
|
1749
|
+
});
|
|
1750
|
+
}
|
|
1751
|
+
function defaultDockerLogout(registry) {
|
|
1752
|
+
execFileSync("docker", ["logout", registry], { stdio: "ignore" });
|
|
1753
|
+
}
|
|
1754
|
+
async function defaultFetchText(ref, authToken) {
|
|
1755
|
+
const res = await fetch(ref, {
|
|
1756
|
+
headers: authToken ? { authorization: `Bearer ${authToken}` } : void 0
|
|
1757
|
+
});
|
|
1758
|
+
if (!res.ok) throw new Error(`Failed to fetch ${ref}: HTTP ${res.status}`);
|
|
1759
|
+
return res.text();
|
|
1760
|
+
}
|
|
1761
|
+
async function defaultPostJson(url, body, authToken) {
|
|
1762
|
+
const res = await fetch(url, {
|
|
1763
|
+
method: "POST",
|
|
1764
|
+
headers: {
|
|
1765
|
+
"content-type": "application/json",
|
|
1766
|
+
...authToken ? { authorization: `Bearer ${authToken}` } : {}
|
|
1767
|
+
},
|
|
1768
|
+
body: JSON.stringify(body)
|
|
1769
|
+
});
|
|
1770
|
+
if (!res.ok) throw new Error(`Failed to POST ${url}: HTTP ${res.status}`);
|
|
1771
|
+
}
|
|
1772
|
+
function defaultStackPaths() {
|
|
1773
|
+
const cwdCompose = path.resolve("docker-compose.yml");
|
|
1774
|
+
const cwdEnv = path.resolve(".env");
|
|
1775
|
+
const packagedManifest = path.join(resolvePackageRoot(), "deploy", "stack-manifests", "v0.9.json");
|
|
1776
|
+
const manifestRef = process.env.EXE_STACK_MANIFEST || (existsSync(packagedManifest) ? packagedManifest : "https://update.askexe.com/v1/stack-manifest.json");
|
|
1777
|
+
const licenseKey = process.env.EXE_LICENSE_KEY || loadLicense() || process.env.EXE_STACK_UPDATE_TOKEN || void 0;
|
|
1778
|
+
const isRemoteManifest = /^https?:\/\//.test(manifestRef);
|
|
1779
|
+
const hasLicenseForRemote = !!licenseKey;
|
|
1780
|
+
const apiBaseUrl = process.env.EXE_STACK_API_URL || process.env.EXE_CLOUD_ENDPOINT || "https://api.askexe.com";
|
|
1781
|
+
return {
|
|
1782
|
+
composeFile: process.env.EXE_STACK_COMPOSE_FILE || (existsSync(cwdCompose) ? cwdCompose : "/opt/exe-stack/docker-compose.yml"),
|
|
1783
|
+
envFile: process.env.EXE_STACK_ENV_FILE || (existsSync(cwdEnv) ? cwdEnv : "/opt/exe-stack/.env"),
|
|
1784
|
+
manifestRef,
|
|
1785
|
+
auditUrl: process.env.EXE_STACK_AUDIT_URL || (isRemoteManifest || hasLicenseForRemote ? `${apiBaseUrl}/v1/deploy-audits` : void 0),
|
|
1786
|
+
imageCredentialsUrl: process.env.EXE_STACK_IMAGE_CREDENTIALS_URL || (isRemoteManifest || hasLicenseForRemote ? `${apiBaseUrl}/v1/image-credentials` : void 0),
|
|
1787
|
+
// License key IS the auth token for update.askexe.com — no separate update token needed.
|
|
1788
|
+
// EXE_STACK_UPDATE_TOKEN kept as legacy fallback during migration.
|
|
1789
|
+
manifestAuthToken: licenseKey,
|
|
1790
|
+
manifestPublicKey: loadDefaultPublicKey()
|
|
1791
|
+
};
|
|
1792
|
+
}
|
|
1793
|
+
function detectDanglingVolumes() {
|
|
1794
|
+
const result = spawnSync(
|
|
1795
|
+
"docker",
|
|
1796
|
+
["volume", "ls", "--filter", "dangling=true", "--format", "{{.Name}} {{.Driver}}"],
|
|
1797
|
+
{ encoding: "utf8", stdio: ["pipe", "pipe", "pipe"], timeout: 3e4 }
|
|
1798
|
+
);
|
|
1799
|
+
if (result.status !== 0 || !result.stdout?.trim()) return [];
|
|
1800
|
+
const volumes = [];
|
|
1801
|
+
for (const line of result.stdout.trim().split("\n")) {
|
|
1802
|
+
if (!line.trim()) continue;
|
|
1803
|
+
const [name, driver] = line.split(" ");
|
|
1804
|
+
if (!name) continue;
|
|
1805
|
+
const size = getVolumeSize(name);
|
|
1806
|
+
volumes.push({ name, driver: driver ?? "local", size });
|
|
1807
|
+
}
|
|
1808
|
+
return volumes;
|
|
1809
|
+
}
|
|
1810
|
+
function getVolumeSize(volumeName) {
|
|
1811
|
+
try {
|
|
1812
|
+
const inspectResult = spawnSync(
|
|
1813
|
+
"docker",
|
|
1814
|
+
["volume", "inspect", volumeName, "--format", "{{.Mountpoint}}"],
|
|
1815
|
+
{ encoding: "utf8", stdio: ["pipe", "pipe", "pipe"], timeout: 1e4 }
|
|
1816
|
+
);
|
|
1817
|
+
const mountpoint = inspectResult.stdout?.trim();
|
|
1818
|
+
if (!mountpoint || inspectResult.status !== 0) return "unknown";
|
|
1819
|
+
const duResult = spawnSync(
|
|
1820
|
+
"du",
|
|
1821
|
+
["-sh", mountpoint],
|
|
1822
|
+
{ encoding: "utf8", stdio: ["pipe", "pipe", "pipe"], timeout: 1e4 }
|
|
1823
|
+
);
|
|
1824
|
+
if (duResult.status === 0 && duResult.stdout?.trim()) {
|
|
1825
|
+
return duResult.stdout.trim().split(/\s+/)[0] ?? "unknown";
|
|
1826
|
+
}
|
|
1827
|
+
return "unknown";
|
|
1828
|
+
} catch {
|
|
1829
|
+
return "unknown";
|
|
1830
|
+
}
|
|
1831
|
+
}
|
|
1832
|
+
function removeDanglingVolumes(volumes) {
|
|
1833
|
+
const result = {
|
|
1834
|
+
detected: volumes,
|
|
1835
|
+
removed: [],
|
|
1836
|
+
failed: [],
|
|
1837
|
+
skipped: []
|
|
1838
|
+
};
|
|
1839
|
+
for (const vol of volumes) {
|
|
1840
|
+
const checkResult = spawnSync(
|
|
1841
|
+
"docker",
|
|
1842
|
+
["volume", "ls", "--filter", "dangling=true", "--filter", `name=^${vol.name}$`, "--format", "{{.Name}}"],
|
|
1843
|
+
{ encoding: "utf8", stdio: ["pipe", "pipe", "pipe"], timeout: 1e4 }
|
|
1844
|
+
);
|
|
1845
|
+
const stillDangling = checkResult.stdout?.trim().split("\n").some((n) => n.trim() === vol.name);
|
|
1846
|
+
if (!stillDangling) {
|
|
1847
|
+
result.skipped.push(vol);
|
|
1848
|
+
console.log(`[stack-update] Skipping ${vol.name} \u2014 no longer dangling`);
|
|
1849
|
+
continue;
|
|
1850
|
+
}
|
|
1851
|
+
try {
|
|
1852
|
+
execFileSync("docker", ["volume", "rm", vol.name], {
|
|
1853
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
1854
|
+
timeout: 3e4
|
|
1855
|
+
});
|
|
1856
|
+
result.removed.push(vol);
|
|
1857
|
+
console.log(`[stack-update] Removed volume: ${vol.name} (${vol.size})`);
|
|
1858
|
+
} catch (err) {
|
|
1859
|
+
const reason = err instanceof Error ? err.message : String(err);
|
|
1860
|
+
result.failed.push({ volume: vol, error: reason });
|
|
1861
|
+
console.warn(`[stack-update] Failed to remove volume ${vol.name}: ${reason}`);
|
|
1862
|
+
}
|
|
1863
|
+
}
|
|
1864
|
+
return result;
|
|
1865
|
+
}
|
|
1866
|
+
function printDanglingVolumes(volumes) {
|
|
1867
|
+
if (volumes.length === 0) {
|
|
1868
|
+
console.log("[stack-update] No orphaned volumes detected.");
|
|
1869
|
+
return;
|
|
1870
|
+
}
|
|
1871
|
+
console.log(`
|
|
1872
|
+
[stack-update] Detected ${volumes.length} orphaned volume(s):`);
|
|
1873
|
+
for (const vol of volumes) {
|
|
1874
|
+
console.log(` - ${vol.name} (${vol.size}, driver: ${vol.driver})`);
|
|
1875
|
+
}
|
|
1876
|
+
console.log("");
|
|
1877
|
+
}
|
|
1878
|
+
function printVolumeCleanupResult(result) {
|
|
1879
|
+
const { removed, failed, skipped } = result;
|
|
1880
|
+
console.log(`
|
|
1881
|
+
[stack-update] Volume cleanup summary:`);
|
|
1882
|
+
console.log(` Removed: ${removed.length}`);
|
|
1883
|
+
if (skipped.length > 0) console.log(` Skipped (no longer dangling): ${skipped.length}`);
|
|
1884
|
+
if (failed.length > 0) {
|
|
1885
|
+
console.log(` Failed: ${failed.length}`);
|
|
1886
|
+
for (const f of failed) {
|
|
1887
|
+
console.log(` - ${f.volume.name}: ${f.error}`);
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1890
|
+
}
|
|
1891
|
+
function loadDefaultPublicKey() {
|
|
1892
|
+
if (process.env.EXE_STACK_PUBLIC_KEY) return process.env.EXE_STACK_PUBLIC_KEY;
|
|
1893
|
+
if (process.env.EXE_STACK_PUBLIC_KEY_FILE && existsSync(process.env.EXE_STACK_PUBLIC_KEY_FILE)) {
|
|
1894
|
+
return readFileSync(process.env.EXE_STACK_PUBLIC_KEY_FILE, "utf8");
|
|
1895
|
+
}
|
|
1896
|
+
return void 0;
|
|
1897
|
+
}
|
|
1898
|
+
|
|
1899
|
+
export {
|
|
1900
|
+
serviceSelectionPath,
|
|
1901
|
+
loadServiceSelection,
|
|
1902
|
+
saveServiceSelection,
|
|
1903
|
+
filterServicesBySelection,
|
|
1904
|
+
buildDefaultServiceSelection,
|
|
1905
|
+
ProgressReporter,
|
|
1906
|
+
canonicalizeStackManifest,
|
|
1907
|
+
verifyStackManifestSignature,
|
|
1908
|
+
findLatestBackupEnvFile,
|
|
1909
|
+
rollbackStackUpdate,
|
|
1910
|
+
parseStackManifest,
|
|
1911
|
+
loadStackManifest,
|
|
1912
|
+
parseEnv,
|
|
1913
|
+
patchEnv,
|
|
1914
|
+
createStackUpdatePlan,
|
|
1915
|
+
collectProductionDeployGateIssues,
|
|
1916
|
+
collectImageAvailabilityIssues,
|
|
1917
|
+
assertDeploymentScopeAllowed,
|
|
1918
|
+
assertProductionDeployGate,
|
|
1919
|
+
assertBreakingChangesAllowed,
|
|
1920
|
+
verifyManifestImagesAvailable,
|
|
1921
|
+
pairMonitorAgent,
|
|
1922
|
+
bootstrapStackHost,
|
|
1923
|
+
assertHostReadyForApply,
|
|
1924
|
+
hardenHost,
|
|
1925
|
+
runStackUpdate,
|
|
1926
|
+
readCurrentStackVersion,
|
|
1927
|
+
listAvailableVersions,
|
|
1928
|
+
verifyReleaseHealth,
|
|
1929
|
+
defaultStackPaths,
|
|
1930
|
+
detectDanglingVolumes,
|
|
1931
|
+
removeDanglingVolumes,
|
|
1932
|
+
printDanglingVolumes,
|
|
1933
|
+
printVolumeCleanupResult
|
|
1934
|
+
};
|