openship 0.6.6 → 0.6.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bare-FA66N7GC.js +29 -0
- package/dist/{chunk-77QDF6UD.js → chunk-3MP6ACKM.js} +587 -156
- package/dist/{chunk-YIMOCZBZ.js → chunk-5DJOB7IX.js} +3 -3
- package/dist/{chunk-WVAML4H6.js → chunk-5I3COAFH.js} +274 -65
- package/dist/{chunk-5DMWSTOM.js → chunk-5UXHT6ED.js} +940 -224
- package/dist/{chunk-LUKYPVJ4.js → chunk-6GU6JSK6.js} +329 -466
- package/dist/{chunk-RSR7PFIQ.js → chunk-7PEXM2UI.js} +213 -45
- package/dist/{chunk-VTGPEC6B.js → chunk-7YBWXEM6.js} +6 -4
- package/dist/chunk-AHJYP7DG.js +938 -0
- package/dist/chunk-BO6CY4V2.js +265 -0
- package/dist/{chunk-FE7ZZQSE.js → chunk-D3OEMXYP.js} +3 -3
- package/dist/chunk-D6HU4VUZ.js +249 -0
- package/dist/{chunk-LFE5KQ7Y.js → chunk-ENKJ7XIF.js} +152 -57
- package/dist/{chunk-YMMQRE5J.js → chunk-GNOMEMNI.js} +1 -1
- package/dist/{chunk-QVW76WDL.js → chunk-IAMOUDHY.js} +190 -64
- package/dist/{chunk-OBPKYXQL.js → chunk-MUWVC5R4.js} +219 -58
- package/dist/chunk-N5YZMO33.js +111 -0
- package/dist/{chunk-A4GXYE4L.js → chunk-NIYD7MD2.js} +206 -52
- package/dist/{chunk-UVNMQM7W.js → chunk-NPSRGIHV.js} +8 -6
- package/dist/{chunk-2GWYYHOR.js → chunk-NRU2EBYZ.js} +2 -2
- package/dist/{chunk-CBLD2HS5.js → chunk-NUT6HKK6.js} +8 -5
- package/dist/{chunk-ZFJXPCQP.js → chunk-P7RCYELG.js} +4 -3
- package/dist/chunk-R4VDPFLS.js +106 -0
- package/dist/{chunk-HKRQXPTI.js → chunk-R5DC5PEF.js} +1 -1
- package/dist/{chunk-NK5N5LXN.js → chunk-RCMERVBL.js} +24 -9
- package/dist/{chunk-RXQBD4ZY.js → chunk-RHLNVZMS.js} +2 -2
- package/dist/chunk-SK4FFCFR.js +70 -0
- package/dist/{chunk-4I2BM2DT.js → chunk-T46BQOLY.js} +3 -3
- package/dist/{chunk-6FH4AXVY.js → chunk-T6DVUX4F.js} +1 -1
- package/dist/{chunk-PCPVTH6M.js → chunk-TAZZZKYO.js} +6 -4
- package/dist/{chunk-MLYSEZQB.js → chunk-UAWY6X7V.js} +43 -72
- package/dist/{chunk-Q7WGTQQ5.js → chunk-VAI5VOOI.js} +1 -1
- package/dist/{chunk-UJ6JZM65.js → chunk-WEZFTL4V.js} +1325 -513
- package/dist/{chunk-2YM3NI2E.js → chunk-X53JXRS4.js} +38 -44
- package/dist/{cloud-KRTZ2IL5.js → cloud-2CRDKOWQ.js} +1 -1
- package/dist/cloud-REV3DPJH.js +42 -0
- package/dist/{detect-OIE5V3VZ.js → detect-BBFT5GEU.js} +9 -6
- package/dist/{docker-VTR2GK5Y.js → docker-EG3QSE6I.js} +11 -8
- package/dist/{docker-edge-executor-22IBXNYJ.js → docker-edge-executor-E5TOTLMR.js} +8 -6
- package/dist/edge-import-YJVJUGYC.js +56 -0
- package/dist/{edge-preflight-GBNCPNVU.js → edge-preflight-QUW4BM7N.js} +15 -12
- package/dist/{ensure-container-edge-KPI2CVXJ.js → ensure-container-edge-FOEFQ3PI.js} +15 -10
- package/dist/{executor-7LXXZPBR.js → executor-NTWDJ3DB.js} +8 -6
- package/dist/host-channel-preflight-OMCZMMKR.js +54 -0
- package/dist/index.js +198 -76
- package/dist/{local-executor-WOFZD72X.js → local-executor-OQ2GRN7K.js} +3 -2
- package/dist/{nginx-KUFMLMAC.js → nginx-4U4C2XUS.js} +11 -8
- package/dist/{noop-NJ72F46T.js → noop-CLLJHCPL.js} +1 -1
- package/dist/{openresty-lua-ATTH35BE.js → openresty-lua-LLWT5XQW.js} +8 -5
- package/dist/{privilege-I4W3ZNZK.js → privilege-MNDVMDST.js} +6 -5
- package/dist/server/index.js +40049 -30378
- package/dist/server/migrations/0109_service_build_args.sql +9 -0
- package/dist/server/migrations/0110_service_deployment_host_ports.sql +18 -0
- package/dist/server/migrations/0111_host_port_claim.sql +381 -0
- package/dist/server/migrations/0112_backup_run_execution_lease.sql +12 -0
- package/dist/server/migrations/0113_docker_migration_execution_lease.sql +11 -0
- package/dist/server/migrations/0114_build_session_execution_lease_index.sql +6 -0
- package/dist/server/migrations/0115_orphan_target_key.sql +3 -0
- package/dist/server/migrations/0116_domain_ssl_challenge.sql +2 -0
- package/dist/server/migrations/meta/_journal.json +56 -0
- package/dist/setup-IFIMPQZJ.js +25 -0
- package/dist/{ssh-client-JBMA7SN4.js → ssh-client-2XWZCMGI.js} +3 -2
- package/dist/up-HQS7IR7H.js +61 -0
- package/package.json +1 -1
- package/dist/bare-OXJVW3AK.js +0 -25
- package/dist/chunk-LH2PLRQO.js +0 -434
- package/dist/chunk-SM26HDS2.js +0 -185
- package/dist/cloud-75VIT4KV.js +0 -37
- package/dist/edge-import-WJUB4M7F.js +0 -52
- package/dist/host-channel-preflight-KE6OONYK.js +0 -50
- package/dist/setup-XGLGUUFW.js +0 -22
- package/dist/up-K4BV7466.js +0 -57
|
@@ -134,41 +134,50 @@ import {
|
|
|
134
134
|
} from "./chunk-ZWLDLTJP.js";
|
|
135
135
|
import {
|
|
136
136
|
CloudRuntime
|
|
137
|
-
} from "./chunk-
|
|
137
|
+
} from "./chunk-MUWVC5R4.js";
|
|
138
138
|
import {
|
|
139
139
|
OPENRESTY_DEFAULT_PATHS
|
|
140
|
-
} from "./chunk-
|
|
140
|
+
} from "./chunk-7PEXM2UI.js";
|
|
141
141
|
import {
|
|
142
142
|
DockerRuntime,
|
|
143
143
|
daemonConnectionFrom,
|
|
144
|
+
ensureScopedVolumeName,
|
|
144
145
|
isHostPathSource,
|
|
145
146
|
resolveExecExitCode,
|
|
146
|
-
scopedVolumeName,
|
|
147
147
|
startAttachStream,
|
|
148
148
|
startExecStream
|
|
149
|
-
} from "./chunk-
|
|
149
|
+
} from "./chunk-3MP6ACKM.js";
|
|
150
150
|
import {
|
|
151
151
|
BareRuntime
|
|
152
|
-
} from "./chunk-
|
|
152
|
+
} from "./chunk-ENKJ7XIF.js";
|
|
153
153
|
import {
|
|
154
154
|
createExecutor
|
|
155
|
-
} from "./chunk-
|
|
155
|
+
} from "./chunk-UAWY6X7V.js";
|
|
156
156
|
import {
|
|
157
157
|
envOps
|
|
158
|
-
} from "./chunk-
|
|
158
|
+
} from "./chunk-GNOMEMNI.js";
|
|
159
159
|
import {
|
|
160
160
|
ENVIRONMENT_PROBE_SCRIPT,
|
|
161
161
|
ENVIRONMENT_PROBE_TIMEOUT_MS,
|
|
162
162
|
cacheEnvironment,
|
|
163
163
|
parseEnvironmentProbe,
|
|
164
164
|
profileIsFresh
|
|
165
|
-
} from "./chunk-
|
|
165
|
+
} from "./chunk-NPSRGIHV.js";
|
|
166
|
+
import {
|
|
167
|
+
MAIL_HOST_STATE_DIR
|
|
168
|
+
} from "./chunk-D6HU4VUZ.js";
|
|
166
169
|
import {
|
|
167
170
|
FAMILY_BY_DISTRO,
|
|
168
171
|
isAwsS3Endpoint,
|
|
172
|
+
isDbImage,
|
|
173
|
+
normalizeBackupPath,
|
|
174
|
+
payloadSpec,
|
|
169
175
|
safeErrorMessage,
|
|
176
|
+
shellQuote,
|
|
177
|
+
validateBackupPath,
|
|
178
|
+
validateClearPath,
|
|
170
179
|
withTimeout
|
|
171
|
-
} from "./chunk-
|
|
180
|
+
} from "./chunk-WEZFTL4V.js";
|
|
172
181
|
|
|
173
182
|
// ../../packages/adapters/src/runtime/image-transfer.ts
|
|
174
183
|
import { Transform } from "stream";
|
|
@@ -176,42 +185,6 @@ import { Transform } from "stream";
|
|
|
176
185
|
// ../../packages/adapters/src/infra/edge-baked-conf.ts
|
|
177
186
|
var LOGS_DIR = OPENRESTY_DEFAULT_PATHS.pidPath.replace(/\/[^/]+$/, "");
|
|
178
187
|
|
|
179
|
-
// ../../packages/adapters/src/infra/mail-container.ts
|
|
180
|
-
var MAIL_HOST_STATE_DIR = "/var/lib/openship/mail";
|
|
181
|
-
var MAIL_CONTAINER_MOUNTS = [
|
|
182
|
-
{ host: `${MAIL_HOST_STATE_DIR}/vmail`, container: "/var/vmail" },
|
|
183
|
-
// The Postfix queue. A recreate on an ephemeral layer would drop in-flight and
|
|
184
|
-
// deferred mail without this — keep it on the host.
|
|
185
|
-
{ host: `${MAIL_HOST_STATE_DIR}/spool`, container: "/var/spool/postfix" },
|
|
186
|
-
// DKIM private keys. NEVER baked, never regenerated on pull — a fresh selector
|
|
187
|
-
// breaks DMARC until DNS repropagates.
|
|
188
|
-
{ host: `${MAIL_HOST_STATE_DIR}/dkim`, container: "/var/lib/dkim" },
|
|
189
|
-
// Mutable daemon config the admin layer edits (sasl_passwd, relayhost maps,
|
|
190
|
-
// amavis 50-user). Seeded from baked defaults on first boot.
|
|
191
|
-
{ host: `${MAIL_HOST_STATE_DIR}/config/postfix`, container: "/etc/postfix", seed: true },
|
|
192
|
-
{ host: `${MAIL_HOST_STATE_DIR}/config/dovecot`, container: "/etc/dovecot", seed: true },
|
|
193
|
-
{ host: `${MAIL_HOST_STATE_DIR}/config/amavis`, container: "/etc/amavis/conf.d", seed: true },
|
|
194
|
-
// ClamAV signatures — bind-mounted so a pull doesn't force a multi-hundred-MB
|
|
195
|
-
// freshclam re-download and delay readiness. `seed: true` because that same mount
|
|
196
|
-
// HIDES the database baked into the image: with nothing copying it across, clamd
|
|
197
|
-
// finds an empty database directory and exits (issue #565).
|
|
198
|
-
{ host: `${MAIL_HOST_STATE_DIR}/clamav`, container: "/var/lib/clamav", seed: true },
|
|
199
|
-
{ host: "/etc/letsencrypt", container: "/etc/letsencrypt", readonly: true }
|
|
200
|
-
];
|
|
201
|
-
var MAIL_DB_HOST_DATA_DIR = `${MAIL_HOST_STATE_DIR}/pgdata`;
|
|
202
|
-
var MAIL_DB_CONTAINER_DATA_DIR = "/var/lib/postgresql/data";
|
|
203
|
-
var MAIL_DB_PGDATA = `${MAIL_DB_CONTAINER_DATA_DIR}/pgdata`;
|
|
204
|
-
var MAIL_HOST_PATHS = {
|
|
205
|
-
saslPasswd: `${MAIL_HOST_STATE_DIR}/config/postfix/sasl_passwd`,
|
|
206
|
-
senderRelayhost: `${MAIL_HOST_STATE_DIR}/config/postfix/sender_relayhost`,
|
|
207
|
-
// Per-nexthop TLS policy. Only split ("selected"-scope) relaying writes it: the
|
|
208
|
-
// relay hop must be `encrypt` so SASL credentials never cross a plaintext
|
|
209
|
-
// connection, while direct-to-MX delivery for the un-relayed domains stays
|
|
210
|
-
// opportunistic. A GLOBAL smtp_tls_security_level can't express both.
|
|
211
|
-
relayTlsPolicy: `${MAIL_HOST_STATE_DIR}/config/postfix/openship_tls_policy`,
|
|
212
|
-
amavisUserConf: `${MAIL_HOST_STATE_DIR}/config/amavis/50-user`
|
|
213
|
-
};
|
|
214
|
-
|
|
215
188
|
// ../../packages/adapters/src/system/mail/ensure-container-mail.ts
|
|
216
189
|
var MAIL_DB_IMAGE = process.env.OPENSHIP_MAIL_DB_IMAGE?.trim() || "postgres:16-alpine";
|
|
217
190
|
var ENGINE_ENV_FILE = `${MAIL_HOST_STATE_DIR}/engine.env`;
|
|
@@ -342,6 +315,102 @@ function decryptCredential(stored) {
|
|
|
342
315
|
import { createHash as createHash3 } from "crypto";
|
|
343
316
|
import { Transform as Transform2 } from "stream";
|
|
344
317
|
|
|
318
|
+
// ../../packages/adapters/src/backup/common/dump-pipeline.ts
|
|
319
|
+
function codecSuffix(codec) {
|
|
320
|
+
return codec === "zstd" ? ".zst" : codec === "gzip" ? ".gz" : "";
|
|
321
|
+
}
|
|
322
|
+
function preludeBlock(prelude) {
|
|
323
|
+
return `{
|
|
324
|
+
${prelude}
|
|
325
|
+
} >&2`;
|
|
326
|
+
}
|
|
327
|
+
function compressor(codec) {
|
|
328
|
+
if (codec === "zstd") return "zstd -c -3";
|
|
329
|
+
if (codec === "gzip") return "gzip -c";
|
|
330
|
+
return null;
|
|
331
|
+
}
|
|
332
|
+
function decompressor(codec) {
|
|
333
|
+
if (codec === "zstd") return "zstd -d -c";
|
|
334
|
+
if (codec === "gzip") return "gzip -d -c";
|
|
335
|
+
return null;
|
|
336
|
+
}
|
|
337
|
+
function recordedCodec(value) {
|
|
338
|
+
return value === "gzip" || value === "none" || value === "zstd" ? value : "zstd";
|
|
339
|
+
}
|
|
340
|
+
async function detectDumpCodec(service, executor) {
|
|
341
|
+
try {
|
|
342
|
+
const { stdout, awaitExit } = await executor.execStream(service, [
|
|
343
|
+
"sh",
|
|
344
|
+
"-c",
|
|
345
|
+
"if command -v zstd >/dev/null 2>&1; then echo zstd; elif command -v gzip >/dev/null 2>&1; then echo gzip; else echo none; fi"
|
|
346
|
+
]);
|
|
347
|
+
const chunks = [];
|
|
348
|
+
stdout.on("data", (c2) => {
|
|
349
|
+
if (chunks.length < 8) chunks.push(c2);
|
|
350
|
+
});
|
|
351
|
+
stdout.on("error", () => {
|
|
352
|
+
});
|
|
353
|
+
const exit = await awaitExit;
|
|
354
|
+
await new Promise((resolve2) => setImmediate(resolve2));
|
|
355
|
+
if (exit.code !== 0) return "none";
|
|
356
|
+
const answer = Buffer.concat(chunks).toString("utf8").trim();
|
|
357
|
+
return answer === "zstd" || answer === "gzip" ? answer : "none";
|
|
358
|
+
} catch {
|
|
359
|
+
return "none";
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
function safePipelineFragment(left, right, labels) {
|
|
363
|
+
return [
|
|
364
|
+
// Duplicating stdout is harmless to repeat, so a caller may use more than one of
|
|
365
|
+
// these in a script.
|
|
366
|
+
"exec 4>&1",
|
|
367
|
+
// `{ right; } >&4` and not `right >&4`: a right-hand side containing `&&` (the redis
|
|
368
|
+
// loader is `cat > file && chmod …`) would otherwise attach the redirect to its LAST
|
|
369
|
+
// command only. Grouping covers the whole side whatever the caller passes.
|
|
370
|
+
`res=$( { { if ${left}; then echo "d=0" >&3; else echo "d=$?" >&3; fi; } | { if { ${right}; } >&4; then echo "c=0" >&3; else echo "c=$?" >&3; fi; } } 3>&1 )`,
|
|
371
|
+
// Both default to FAILURE, so a report that never arrived cannot read as success.
|
|
372
|
+
"d=1; c=1",
|
|
373
|
+
// Unquoted `$res` word-splits on IFS, which includes the newline separating the two
|
|
374
|
+
// reports — no `sed`/`awk` (a minimal image may have neither), and order-independent.
|
|
375
|
+
// The two writes are single sub-PIPE_BUF lines on one pipe, so they cannot interleave.
|
|
376
|
+
'for kv in $res; do case "$kv" in d=*) d=${kv#d=} ;; c=*) c=${kv#c=} ;; esac; done',
|
|
377
|
+
// A status we could not parse is a failure, never an implicit success.
|
|
378
|
+
`case "$d" in ''|*[!0-9]*) d=1 ;; esac`,
|
|
379
|
+
`case "$c" in ''|*[!0-9]*) c=1 ;; esac`,
|
|
380
|
+
`[ "$c" = 0 ] || { echo "openship: ${labels.right} failed with status $c" >&2; exit "$c"; }`,
|
|
381
|
+
`[ "$d" = 0 ] || { echo "openship: ${labels.left} failed with status $d" >&2; exit "$d"; }`
|
|
382
|
+
].join("\n");
|
|
383
|
+
}
|
|
384
|
+
function safePipelineCommand(left, right, labels, prelude) {
|
|
385
|
+
const script = [
|
|
386
|
+
...prelude ? [preludeBlock(prelude)] : [],
|
|
387
|
+
safePipelineFragment(left, right, labels)
|
|
388
|
+
].join("\n");
|
|
389
|
+
return ["sh", "-c", script];
|
|
390
|
+
}
|
|
391
|
+
function safeDumpCommand(dump, codec, prelude) {
|
|
392
|
+
const comp = compressor(codec);
|
|
393
|
+
if (!comp) return ["sh", "-c", prelude ? `${preludeBlock(prelude)}
|
|
394
|
+
${dump}` : dump];
|
|
395
|
+
return safePipelineCommand(
|
|
396
|
+
dump,
|
|
397
|
+
comp,
|
|
398
|
+
{ left: "the dump command", right: `the compressor '${comp}'` },
|
|
399
|
+
prelude
|
|
400
|
+
);
|
|
401
|
+
}
|
|
402
|
+
function safeRestoreCommand(codec, load, prelude) {
|
|
403
|
+
const decomp = decompressor(codec);
|
|
404
|
+
if (!decomp) return ["sh", "-c", prelude ? `${preludeBlock(prelude)}
|
|
405
|
+
${load}` : load];
|
|
406
|
+
return safePipelineCommand(
|
|
407
|
+
decomp,
|
|
408
|
+
load,
|
|
409
|
+
{ left: `the decompressor '${decomp}'`, right: "the restore command" },
|
|
410
|
+
prelude
|
|
411
|
+
);
|
|
412
|
+
}
|
|
413
|
+
|
|
345
414
|
// ../../packages/adapters/src/backup/common/source-match.ts
|
|
346
415
|
function matchBackupSource(sources, targetSourceId) {
|
|
347
416
|
const wanted = targetSourceId.trim();
|
|
@@ -361,40 +430,53 @@ function matchBackupSource(sources, targetSourceId) {
|
|
|
361
430
|
return null;
|
|
362
431
|
}
|
|
363
432
|
|
|
433
|
+
// ../../packages/adapters/src/backup/common/artifact-metadata.ts
|
|
434
|
+
var REDACTED_USERINFO_MARKER = "://***@";
|
|
435
|
+
function isRedactedCommand(command) {
|
|
436
|
+
return command.includes(REDACTED_USERINFO_MARKER);
|
|
437
|
+
}
|
|
438
|
+
|
|
364
439
|
// ../../packages/adapters/src/backup/volume-transfer.ts
|
|
365
440
|
import { Transform as Transform3 } from "stream";
|
|
366
441
|
|
|
442
|
+
// ../../packages/adapters/src/backup/common/exec-target.ts
|
|
443
|
+
function canExecInService(service) {
|
|
444
|
+
return !!service.containerId && service.containerRunning !== false;
|
|
445
|
+
}
|
|
446
|
+
|
|
367
447
|
// ../../packages/adapters/src/backup/producers/pg-dump.ts
|
|
368
|
-
var POSTGRES_IMAGE_RE = /^(postgres|postgis\/postgis):/i;
|
|
369
448
|
function envOr(service, key, fallback) {
|
|
370
449
|
return service.env[key] ?? fallback;
|
|
371
450
|
}
|
|
372
451
|
var PgDumpProducerImpl = class {
|
|
373
452
|
kind = "pg_dump";
|
|
374
453
|
detects(service) {
|
|
375
|
-
if (!service.image
|
|
376
|
-
|
|
377
|
-
return !!db;
|
|
454
|
+
if (!isDbImage(service.image, "pg_dump")) return false;
|
|
455
|
+
return canExecInService(service);
|
|
378
456
|
}
|
|
379
457
|
async *produce(service, executor, _opts) {
|
|
380
458
|
const user = envOr(service, "POSTGRES_USER", "postgres");
|
|
381
|
-
const db = service.env.POSTGRES_DB ?? service.env.PGDATABASE ?? "postgres";
|
|
459
|
+
const db = service.env.POSTGRES_DB ?? service.env.PGDATABASE ?? service.env.POSTGRES_USER ?? "postgres";
|
|
382
460
|
const password = service.env.POSTGRES_PASSWORD ?? service.env.PGPASSWORD ?? "";
|
|
383
|
-
const cmd =
|
|
384
|
-
|
|
385
|
-
"
|
|
386
|
-
`export PGPASSWORD=${
|
|
387
|
-
|
|
461
|
+
const cmd = safeDumpCommand(
|
|
462
|
+
`pg_dump -Fc -U ${shellQuote(user)} ${shellQuote(db)}`,
|
|
463
|
+
"none",
|
|
464
|
+
`export PGPASSWORD=${shellQuote(password)}`
|
|
465
|
+
);
|
|
388
466
|
const { stdout, awaitExit } = await executor.execStream(service, cmd);
|
|
389
467
|
yield {
|
|
390
|
-
|
|
468
|
+
// These bytes are pg_dump's own container format, which pg_restore reads
|
|
469
|
+
// directly. `.dump` is what the Postgres docs call it.
|
|
470
|
+
name: "pg-dump.dump",
|
|
391
471
|
stream: stdout,
|
|
392
472
|
payloadKind: "pg_dump",
|
|
393
473
|
metadata: {
|
|
394
474
|
postgresUser: user,
|
|
395
475
|
postgresDb: db,
|
|
396
476
|
format: "custom",
|
|
397
|
-
|
|
477
|
+
// The EXTERNAL codec only. The custom format's internal zlib is pg_restore's
|
|
478
|
+
// own business, so a `-Fc` artifact records "none".
|
|
479
|
+
compression: "none"
|
|
398
480
|
}
|
|
399
481
|
};
|
|
400
482
|
const exit = await awaitExit;
|
|
@@ -406,17 +488,18 @@ var PgDumpProducerImpl = class {
|
|
|
406
488
|
}
|
|
407
489
|
async restore(service, executor, artifact, _opts) {
|
|
408
490
|
const user = envOr(service, "POSTGRES_USER", "postgres");
|
|
409
|
-
const db = artifact.metadata.postgresDb ?? service.env.POSTGRES_DB ?? service.env.PGDATABASE ?? "postgres";
|
|
410
491
|
const password = service.env.POSTGRES_PASSWORD ?? service.env.PGPASSWORD ?? "";
|
|
411
|
-
const
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
492
|
+
const codec = recordedCodec(artifact.metadata.compression);
|
|
493
|
+
const db = artifact.metadata.postgresDb ?? service.env.POSTGRES_DB ?? service.env.PGDATABASE ?? service.env.POSTGRES_USER ?? "postgres";
|
|
494
|
+
const cmd = safeRestoreCommand(
|
|
495
|
+
codec,
|
|
496
|
+
`pg_restore --clean --if-exists --no-owner --no-acl --single-transaction -U ${shellQuote(user)} -d ${shellQuote(db)}`,
|
|
497
|
+
`export PGPASSWORD=${shellQuote(password)}`
|
|
498
|
+
);
|
|
416
499
|
const body = await artifact.open();
|
|
417
500
|
const exit = await executor.pipeIntoCommand(service, cmd, body, {
|
|
418
|
-
|
|
419
|
-
|
|
501
|
+
// Ceiling from the catalog, so the number is not a per-producer literal.
|
|
502
|
+
timeoutMs: payloadSpec("pg_dump").restoreTimeoutMs
|
|
420
503
|
});
|
|
421
504
|
if (exit.code !== 0) {
|
|
422
505
|
throw new Error(
|
|
@@ -425,17 +508,10 @@ var PgDumpProducerImpl = class {
|
|
|
425
508
|
}
|
|
426
509
|
}
|
|
427
510
|
};
|
|
428
|
-
function shellEscape(s2) {
|
|
429
|
-
return `'${s2.replace(/'/g, "'\\''")}'`;
|
|
430
|
-
}
|
|
431
511
|
var PgDumpProducer = new PgDumpProducerImpl();
|
|
432
512
|
registerProducer(PgDumpProducer);
|
|
433
513
|
|
|
434
514
|
// ../../packages/adapters/src/backup/producers/mysql-dump.ts
|
|
435
|
-
var MYSQL_IMAGE_RE = /^(mysql|mariadb|percona\/percona-server):/i;
|
|
436
|
-
function shellEscape2(s2) {
|
|
437
|
-
return `'${s2.replace(/'/g, "'\\''")}'`;
|
|
438
|
-
}
|
|
439
515
|
function mysqlCredentials(env) {
|
|
440
516
|
if (env.MYSQL_ROOT_PASSWORD) {
|
|
441
517
|
return { user: "root", password: env.MYSQL_ROOT_PASSWORD };
|
|
@@ -445,27 +521,32 @@ function mysqlCredentials(env) {
|
|
|
445
521
|
var MysqlDumpProducerImpl = class {
|
|
446
522
|
kind = "mysql_dump";
|
|
447
523
|
detects(service) {
|
|
448
|
-
if (!service.image
|
|
524
|
+
if (!isDbImage(service.image, "mysql_dump")) return false;
|
|
525
|
+
if (!canExecInService(service)) return false;
|
|
449
526
|
return !!(service.env.MYSQL_ROOT_PASSWORD ?? service.env.MYSQL_PASSWORD);
|
|
450
527
|
}
|
|
451
528
|
async *produce(service, executor, _opts) {
|
|
452
529
|
const { user, password } = mysqlCredentials(service.env);
|
|
453
530
|
const db = service.env.MYSQL_DATABASE ?? "";
|
|
454
|
-
const dbArg = db ? `--databases ${
|
|
455
|
-
const
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
531
|
+
const dbArg = db ? `--databases ${shellQuote(db)}` : "--all-databases";
|
|
532
|
+
const codec = await detectDumpCodec(service, executor);
|
|
533
|
+
const cmd = safeDumpCommand(
|
|
534
|
+
`mysqldump --single-transaction --routines --triggers -u ${shellQuote(user)} ${dbArg}`,
|
|
535
|
+
codec,
|
|
536
|
+
// `export`, not the `VAR=x cmd` prefix: with a pipeline the prefix form sets
|
|
537
|
+
// the variable on mysqldump only, and the safe-pipeline wrapper puts the
|
|
538
|
+
// compressor in the same shell.
|
|
539
|
+
`export MYSQL_PWD=${shellQuote(password)}`
|
|
540
|
+
);
|
|
460
541
|
const { stdout, awaitExit } = await executor.execStream(service, cmd);
|
|
461
542
|
yield {
|
|
462
|
-
name:
|
|
543
|
+
name: `mysql-dump.sql${codecSuffix(codec)}`,
|
|
463
544
|
stream: stdout,
|
|
464
545
|
payloadKind: "mysql_dump",
|
|
465
546
|
metadata: {
|
|
466
547
|
mysqlUser: user,
|
|
467
548
|
mysqlDatabase: db || "(all)",
|
|
468
|
-
compression:
|
|
549
|
+
compression: codec
|
|
469
550
|
}
|
|
470
551
|
};
|
|
471
552
|
const exit = await awaitExit;
|
|
@@ -475,14 +556,16 @@ var MysqlDumpProducerImpl = class {
|
|
|
475
556
|
}
|
|
476
557
|
async restore(service, executor, artifact, _opts) {
|
|
477
558
|
const { user, password } = mysqlCredentials(service.env);
|
|
478
|
-
const
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
`
|
|
482
|
-
|
|
559
|
+
const codec = recordedCodec(artifact.metadata.compression);
|
|
560
|
+
const cmd = safeRestoreCommand(
|
|
561
|
+
codec,
|
|
562
|
+
`mysql -u ${shellQuote(user)}`,
|
|
563
|
+
`export MYSQL_PWD=${shellQuote(password)}`
|
|
564
|
+
);
|
|
483
565
|
const body = await artifact.open();
|
|
484
566
|
const exit = await executor.pipeIntoCommand(service, cmd, body, {
|
|
485
|
-
|
|
567
|
+
// Ceiling from the catalog, so the number is not a per-producer literal.
|
|
568
|
+
timeoutMs: payloadSpec("mysql_dump").restoreTimeoutMs
|
|
486
569
|
});
|
|
487
570
|
if (exit.code !== 0) {
|
|
488
571
|
throw new Error(`mysql restore exited ${exit.code}: ${exit.stderr.slice(0, 500)}`);
|
|
@@ -493,36 +576,110 @@ var MysqlDumpProducer = new MysqlDumpProducerImpl();
|
|
|
493
576
|
registerProducer(MysqlDumpProducer);
|
|
494
577
|
|
|
495
578
|
// ../../packages/adapters/src/backup/producers/redis.ts
|
|
496
|
-
|
|
579
|
+
async function probeOutput(res) {
|
|
580
|
+
const chunks = [];
|
|
581
|
+
let size = 0;
|
|
582
|
+
const drained = new Promise((resolve2) => {
|
|
583
|
+
res.stdout.on("data", (c2) => {
|
|
584
|
+
if (size < 4096) {
|
|
585
|
+
chunks.push(Buffer.from(c2));
|
|
586
|
+
size += c2.byteLength;
|
|
587
|
+
}
|
|
588
|
+
});
|
|
589
|
+
res.stdout.once("end", () => resolve2());
|
|
590
|
+
res.stdout.once("error", () => resolve2());
|
|
591
|
+
res.stdout.once("close", () => resolve2());
|
|
592
|
+
});
|
|
593
|
+
await withTimeout(drained, PROBE_TIMEOUT_MS, "redis config probe timed out").catch(
|
|
594
|
+
() => void 0
|
|
595
|
+
);
|
|
596
|
+
await res.awaitExit.catch(() => void 0);
|
|
597
|
+
return Buffer.concat(chunks).toString("utf8");
|
|
598
|
+
}
|
|
599
|
+
var PROBE_TIMEOUT_MS = 3e4;
|
|
600
|
+
var BGSAVE_WAIT_SECONDS = 300;
|
|
497
601
|
var RedisRdbProducerImpl = class {
|
|
498
602
|
kind = "redis_rdb";
|
|
499
603
|
detects(service) {
|
|
500
|
-
|
|
604
|
+
if (!isDbImage(service.image, "redis_rdb")) return false;
|
|
605
|
+
return canExecInService(service);
|
|
501
606
|
}
|
|
502
607
|
async *produce(service, executor, _opts) {
|
|
503
|
-
const
|
|
504
|
-
const
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
for
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
608
|
+
const cli = this.cli(service);
|
|
609
|
+
const waitSeconds = BGSAVE_WAIT_SECONDS;
|
|
610
|
+
const codec = await detectDumpCodec(service, executor);
|
|
611
|
+
const prelude = [
|
|
612
|
+
// One reader for both LASTSAVE calls, because reading it correctly is not a
|
|
613
|
+
// one-liner: `redis-cli -a` has printed an auth warning on EVERY invocation since
|
|
614
|
+
// 4.0, so the capture is two lines and folding it into the timestamp with a bare
|
|
615
|
+
// `2>&1` would make every password-protected Redis look unreachable. Take the last
|
|
616
|
+
// all-digits LINE as the reply, and keep the rest as the diagnostic.
|
|
617
|
+
`lastsave() {`,
|
|
618
|
+
` RAW=$(${cli} LASTSAVE 2>&1 | tr -d '\\r')`,
|
|
619
|
+
` TS=$(printf '%s\\n' "$RAW" | grep -E '^[0-9]+$' | tail -n 1)`,
|
|
620
|
+
`}`,
|
|
621
|
+
`lastsave`,
|
|
622
|
+
`[ -n "$TS" ] || {`,
|
|
623
|
+
` echo "openship: could not read Redis LASTSAVE, so a captured dump.rdb could not be proven fresh \u2014 refusing to archive a possibly stale one. redis-cli said: $RAW" >&2`,
|
|
624
|
+
` exit 91`,
|
|
625
|
+
`}`,
|
|
626
|
+
`LAST=$TS`,
|
|
627
|
+
// Both halves of "did BGSAVE start", because which one answers depends on the
|
|
628
|
+
// version: Redis 6+ exits non-zero on an error reply, older builds exit 0 and only
|
|
629
|
+
// print it. An already-RUNNING save is not a failure — it will advance LASTSAVE, so
|
|
630
|
+
// the wait below covers it — and its reply contains ERR, which is why that arm comes
|
|
631
|
+
// first.
|
|
632
|
+
//
|
|
633
|
+
// `already in progress` and not `in progress`: Redis has a SECOND error with that
|
|
634
|
+
// phrase — "Can't BGSAVE while AOF log rewriting is in progress" — and it means the
|
|
635
|
+
// opposite, that no save was started at all. Matching the loose phrase waved it
|
|
636
|
+
// through and left the wait polling for a timestamp nothing was going to move.
|
|
637
|
+
`BG=$(${cli} BGSAVE 2>&1 | tr -d '\\r') || BGFAIL=1`,
|
|
638
|
+
`case "$BG" in`,
|
|
639
|
+
` *"already in progress"*) ;;`,
|
|
640
|
+
` *ERR*) BGFAIL=1 ;;`,
|
|
641
|
+
`esac`,
|
|
642
|
+
`[ -z "$BGFAIL" ] || {`,
|
|
643
|
+
` echo "openship: Redis would not start a background save, so /data/dump.rdb is whatever an earlier save left there \u2014 refusing to archive it as this backup. redis-cli said: $BG" >&2`,
|
|
644
|
+
` exit 92`,
|
|
645
|
+
`}`,
|
|
646
|
+
`i=0; SAVED=0`,
|
|
647
|
+
`while [ "$i" -lt ${waitSeconds} ]; do`,
|
|
648
|
+
` lastsave`,
|
|
649
|
+
` [ -n "$TS" ] || {`,
|
|
650
|
+
` echo "openship: lost contact with Redis while waiting for BGSAVE to finish. redis-cli said: $RAW" >&2`,
|
|
651
|
+
` exit 91`,
|
|
652
|
+
` }`,
|
|
653
|
+
` if [ "$TS" != "$LAST" ]; then SAVED=1; break; fi`,
|
|
654
|
+
// Fail on a save Redis has already given up on rather than waiting out the whole
|
|
655
|
+
// window for a timestamp that is never going to move. Gated on
|
|
656
|
+
// `rdb_bgsave_in_progress:0` because the status field still reports the PREVIOUS
|
|
657
|
+
// save's result while a new one runs. A missing `tr`/`grep`/`INFO` leaves this
|
|
658
|
+
// empty, which matches nothing and falls through to the timeout below — the checks
|
|
659
|
+
// here only ever ADD a reason to fail, never a reason to pass.
|
|
660
|
+
` INFO=$(${cli} INFO persistence 2>/dev/null | tr -d '\\r')`,
|
|
661
|
+
` case "$INFO" in *rdb_bgsave_in_progress:0*)`,
|
|
662
|
+
` case "$INFO" in *rdb_last_bgsave_status:err*)`,
|
|
663
|
+
` echo "openship: Redis reported its background save FAILED (rdb_last_bgsave_status:err) \u2014 usually no free space where /data lives. Nothing fresh was written, so refusing to archive the previous save." >&2`,
|
|
664
|
+
` exit 93 ;;`,
|
|
665
|
+
` esac ;;`,
|
|
666
|
+
` esac`,
|
|
667
|
+
` i=$((i+1)); sleep 1`,
|
|
668
|
+
`done`,
|
|
669
|
+
`[ "$SAVED" = 1 ] || {`,
|
|
670
|
+
` echo "openship: Redis BGSAVE did not finish within ${waitSeconds}s \u2014 LASTSAVE never advanced, so /data/dump.rdb is still the PREVIOUS save. Refusing to archive it as this backup; use a volume payload for a dataset this large, or retry." >&2`,
|
|
671
|
+
` exit 94`,
|
|
672
|
+
`}`
|
|
673
|
+
].join("\n");
|
|
674
|
+
const cmd = safeDumpCommand("cat /data/dump.rdb", codec, prelude);
|
|
675
|
+
const { stdout, awaitExit } = await executor.execStream(service, cmd);
|
|
519
676
|
yield {
|
|
520
|
-
name:
|
|
677
|
+
name: `redis-dump.rdb${codecSuffix(codec)}`,
|
|
521
678
|
stream: stdout,
|
|
522
679
|
payloadKind: "redis_rdb",
|
|
523
680
|
metadata: {
|
|
524
681
|
rdbPath: "/data/dump.rdb",
|
|
525
|
-
compression:
|
|
682
|
+
compression: codec
|
|
526
683
|
}
|
|
527
684
|
};
|
|
528
685
|
const exit = await awaitExit;
|
|
@@ -530,15 +687,31 @@ var RedisRdbProducerImpl = class {
|
|
|
530
687
|
throw new Error(`redis BGSAVE/capture exited ${exit.code}: ${exit.stderr.slice(0, 500)}`);
|
|
531
688
|
}
|
|
532
689
|
}
|
|
690
|
+
/** `redis-cli` with auth, when a password is discoverable. */
|
|
691
|
+
cli(service) {
|
|
692
|
+
const pass = service.env.REDIS_PASSWORD;
|
|
693
|
+
return pass ? `redis-cli -a ${shellQuote(pass)}` : "redis-cli";
|
|
694
|
+
}
|
|
533
695
|
async restore(service, executor, artifact, _opts) {
|
|
534
|
-
const
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
696
|
+
const aofValue = await probeOutput(
|
|
697
|
+
await executor.execStream(service, [
|
|
698
|
+
"sh",
|
|
699
|
+
"-c",
|
|
700
|
+
`${this.cli(service)} CONFIG GET appendonly 2>/dev/null | tail -n 1`
|
|
701
|
+
])
|
|
702
|
+
);
|
|
703
|
+
if (aofValue.trim().toLowerCase() === "yes") {
|
|
704
|
+
throw new Error(
|
|
705
|
+
`This Redis has AOF persistence enabled (appendonly yes), so at startup it loads its append-only file INSTEAD of dump.rdb \u2014 restoring this RDB artifact would change nothing. Disable appendonly on the service and redeploy before restoring an RDB, or restore from a volume backup, which replaces /data wholesale.`
|
|
706
|
+
);
|
|
707
|
+
}
|
|
708
|
+
await executor.execStream(service, ["sh", "-c", `${this.cli(service)} CONFIG SET save '' >/dev/null 2>&1`]).then((r2) => r2.awaitExit).catch(() => void 0);
|
|
709
|
+
const codec = recordedCodec(artifact.metadata.compression);
|
|
710
|
+
const cmd = safeRestoreCommand(codec, "cat > /data/dump.rdb && chmod 644 /data/dump.rdb");
|
|
539
711
|
const body = await artifact.open();
|
|
540
712
|
const exit = await executor.pipeIntoCommand(service, cmd, body, {
|
|
541
|
-
|
|
713
|
+
// Ceiling from the catalog, so the number is not a per-producer literal.
|
|
714
|
+
timeoutMs: payloadSpec("redis_rdb").restoreTimeoutMs
|
|
542
715
|
});
|
|
543
716
|
if (exit.code !== 0) {
|
|
544
717
|
throw new Error(`redis restore exited ${exit.code}: ${exit.stderr.slice(0, 500)}`);
|
|
@@ -549,20 +722,17 @@ var RedisRdbProducer = new RedisRdbProducerImpl();
|
|
|
549
722
|
registerProducer(RedisRdbProducer);
|
|
550
723
|
|
|
551
724
|
// ../../packages/adapters/src/backup/producers/mongo.ts
|
|
552
|
-
var MONGO_IMAGE_RE = /^(mongo|percona\/percona-server-mongodb):/i;
|
|
553
|
-
function shellEscape3(s2) {
|
|
554
|
-
return `'${s2.replace(/'/g, "'\\''")}'`;
|
|
555
|
-
}
|
|
556
725
|
var MongoDumpProducerImpl = class {
|
|
557
726
|
kind = "mongo_dump";
|
|
558
727
|
detects(service) {
|
|
559
|
-
|
|
728
|
+
if (!isDbImage(service.image, "mongo_dump")) return false;
|
|
729
|
+
return canExecInService(service);
|
|
560
730
|
}
|
|
561
731
|
authArgs(service) {
|
|
562
732
|
const user = service.env.MONGO_INITDB_ROOT_USERNAME ?? service.env.MONGODB_ROOT_USERNAME ?? "";
|
|
563
733
|
const pass = service.env.MONGO_INITDB_ROOT_PASSWORD ?? service.env.MONGODB_ROOT_PASSWORD ?? "";
|
|
564
734
|
if (!user || !pass) return "";
|
|
565
|
-
return `-u ${
|
|
735
|
+
return `-u ${shellQuote(user)} -p ${shellQuote(pass)} --authenticationDatabase admin`;
|
|
566
736
|
}
|
|
567
737
|
async *produce(service, executor, _opts) {
|
|
568
738
|
const auth = this.authArgs(service);
|
|
@@ -588,7 +758,8 @@ var MongoDumpProducerImpl = class {
|
|
|
588
758
|
];
|
|
589
759
|
const body = await artifact.open();
|
|
590
760
|
const exit = await executor.pipeIntoCommand(service, cmd, body, {
|
|
591
|
-
|
|
761
|
+
// Ceiling from the catalog, so the number is not a per-producer literal.
|
|
762
|
+
timeoutMs: payloadSpec("mongo_dump").restoreTimeoutMs
|
|
592
763
|
});
|
|
593
764
|
if (exit.code !== 0) {
|
|
594
765
|
throw new Error(`mongorestore exited ${exit.code}: ${exit.stderr.slice(0, 500)}`);
|
|
@@ -638,12 +809,18 @@ var CustomCommandProducerImpl = class {
|
|
|
638
809
|
"Backup has no restoreCommand recorded \u2014 restore is not possible for this artifact. Re-run the backup policy to capture a restorable one."
|
|
639
810
|
);
|
|
640
811
|
}
|
|
812
|
+
if (isRedactedCommand(restoreCommand)) {
|
|
813
|
+
throw new Error(
|
|
814
|
+
"Backup's recorded restoreCommand had its credentials redacted when it was captured, so running it would fail authentication partway through the restore. Re-run the backup policy to capture a restorable artifact (or restore by hand using the command on the policy)."
|
|
815
|
+
);
|
|
816
|
+
}
|
|
641
817
|
const body = await artifact.open();
|
|
642
818
|
const exit = await executor.pipeIntoCommand(
|
|
643
819
|
service,
|
|
644
820
|
["sh", "-c", restoreCommand],
|
|
645
821
|
body,
|
|
646
|
-
|
|
822
|
+
// Ceiling from the catalog, so the number is not a per-producer literal.
|
|
823
|
+
{ timeoutMs: payloadSpec("custom_command").restoreTimeoutMs }
|
|
647
824
|
);
|
|
648
825
|
if (exit.code !== 0) {
|
|
649
826
|
throw new Error(
|
|
@@ -655,6 +832,130 @@ var CustomCommandProducerImpl = class {
|
|
|
655
832
|
var CustomCommandProducer = new CustomCommandProducerImpl();
|
|
656
833
|
registerProducer(CustomCommandProducer);
|
|
657
834
|
|
|
835
|
+
// ../../packages/adapters/src/backup/producers/path.ts
|
|
836
|
+
function guardSourcePath(path, serviceName) {
|
|
837
|
+
const q2 = shellQuote(path);
|
|
838
|
+
return [
|
|
839
|
+
`if [ ! -d ${q2} ]; then`,
|
|
840
|
+
` echo "openship: ${path} does not exist inside service ${serviceName} (or is not a directory), so there is nothing to archive. Check the path, or point the policy at a volume instead." >&2`,
|
|
841
|
+
` exit 66`,
|
|
842
|
+
`fi`,
|
|
843
|
+
`if [ -z "$(find ${q2} -mindepth 1 -print -quit 2>/dev/null)" ]; then`,
|
|
844
|
+
` echo "openship: ${path} exists inside service ${serviceName} but is empty. Refusing to record an empty archive as a backup \u2014 a directory that should hold data and does not usually means the data lives somewhere else (a volume mounted one level off, or an app that moved its data directory). Fix the path, or remove it from the policy if it is expected to be empty." >&2`,
|
|
845
|
+
` exit 67`,
|
|
846
|
+
`fi`
|
|
847
|
+
].join("\n");
|
|
848
|
+
}
|
|
849
|
+
function guardRestorePath(path, codec, clear) {
|
|
850
|
+
const q2 = shellQuote(path);
|
|
851
|
+
const tool = codec === "zstd" ? "zstd" : codec === "gzip" ? "gzip" : null;
|
|
852
|
+
const lines = [];
|
|
853
|
+
if (tool && clear) {
|
|
854
|
+
lines.push(
|
|
855
|
+
`command -v ${tool} >/dev/null 2>&1 || { echo "openship: ${tool} is not available inside this service, so the archive cannot be decompressed. Refusing to clear ${path}: nothing was written and your data is untouched." >&2; exit 90; }`
|
|
856
|
+
);
|
|
857
|
+
}
|
|
858
|
+
lines.push(`mkdir -p ${q2} || exit 91`);
|
|
859
|
+
if (clear) {
|
|
860
|
+
lines.push(`find ${q2} -mindepth 1 -maxdepth 1 -exec rm -rf {} + 2>/dev/null || true`);
|
|
861
|
+
}
|
|
862
|
+
return lines.join("\n");
|
|
863
|
+
}
|
|
864
|
+
function pathSlug(path) {
|
|
865
|
+
return path.replace(/^\/+/, "").replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 80) || "root";
|
|
866
|
+
}
|
|
867
|
+
var PathArchiveProducerImpl = class {
|
|
868
|
+
kind = "path";
|
|
869
|
+
// No detects() — explicit selection only. See the module doc.
|
|
870
|
+
async *produce(service, executor, opts) {
|
|
871
|
+
const requested = opts.paths ?? [];
|
|
872
|
+
if (requested.length === 0) {
|
|
873
|
+
throw new Error(
|
|
874
|
+
`Nothing to back up for service "${service.name}": this policy archives files and folders but names none. Set \`payloadConfig.paths\` to one or more absolute paths inside the service.`
|
|
875
|
+
);
|
|
876
|
+
}
|
|
877
|
+
const paths = [];
|
|
878
|
+
for (const entry of requested) {
|
|
879
|
+
const problem = validateBackupPath(entry);
|
|
880
|
+
if (problem) throw new Error(problem);
|
|
881
|
+
const normalized = normalizeBackupPath(entry);
|
|
882
|
+
if (!paths.includes(normalized)) paths.push(normalized);
|
|
883
|
+
}
|
|
884
|
+
const codec = opts.compression ?? await detectDumpCodec(service, executor);
|
|
885
|
+
for (const path of paths) {
|
|
886
|
+
const { stdout, awaitExit } = await executor.execStream(
|
|
887
|
+
service,
|
|
888
|
+
safeDumpCommand(
|
|
889
|
+
// `-C <dir> .` rather than `tar -c <dir>`: the archive holds the directory's
|
|
890
|
+
// CONTENTS at relative paths, so it can be extracted into a different
|
|
891
|
+
// location — which is what makes an artifact captured from /data/app
|
|
892
|
+
// restorable after the app moved to /srv/app.
|
|
893
|
+
`tar -c -C ${shellQuote(path)} ${(opts.exclude ?? []).map((p2) => `--exclude=${shellQuote(p2)}`).join(" ")} .`.replace(/\s+/g, " "),
|
|
894
|
+
codec,
|
|
895
|
+
guardSourcePath(path, service.name)
|
|
896
|
+
)
|
|
897
|
+
);
|
|
898
|
+
yield {
|
|
899
|
+
name: `path-${pathSlug(path)}.tar${codecSuffix(codec)}`,
|
|
900
|
+
stream: stdout,
|
|
901
|
+
payloadKind: "path",
|
|
902
|
+
metadata: {
|
|
903
|
+
// The restore target. Recorded per-artifact rather than read back off the
|
|
904
|
+
// policy, because a policy is editable and soft-deletable: an artifact has to
|
|
905
|
+
// carry everything needed to put it back, or it stops being a restore point
|
|
906
|
+
// the moment somebody edits the schedule.
|
|
907
|
+
path,
|
|
908
|
+
compression: codec,
|
|
909
|
+
// See the module doc: always crash-consistent, and the artifact says so
|
|
910
|
+
// rather than leaving an operator to assume otherwise.
|
|
911
|
+
consistency: "crash"
|
|
912
|
+
}
|
|
913
|
+
};
|
|
914
|
+
const exit = await awaitExit;
|
|
915
|
+
if (exit.code !== 0) {
|
|
916
|
+
throw new Error(
|
|
917
|
+
`Archiving ${path} from service "${service.name}" failed (exit ${exit.code}): ${exit.stderr.slice(0, 500)}`
|
|
918
|
+
);
|
|
919
|
+
}
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
async restore(service, executor, artifact, opts) {
|
|
923
|
+
const recorded = artifact.metadata.path;
|
|
924
|
+
if (typeof recorded !== "string" || recorded.trim() === "") {
|
|
925
|
+
throw new Error(
|
|
926
|
+
`Artifact ${artifact.key} cannot restore: metadata.path missing. This artifact may have been produced by a different producer.`
|
|
927
|
+
);
|
|
928
|
+
}
|
|
929
|
+
const path = normalizeBackupPath(recorded);
|
|
930
|
+
const problem = validateBackupPath(path);
|
|
931
|
+
if (problem) throw new Error(`Artifact ${artifact.key} records an unusable path. ${problem}`);
|
|
932
|
+
const codec = recordedCodec(artifact.metadata.compression);
|
|
933
|
+
const wantsClear = opts.clearTarget === true;
|
|
934
|
+
if (wantsClear) {
|
|
935
|
+
const unsafe = validateClearPath(path);
|
|
936
|
+
if (unsafe) throw new Error(unsafe);
|
|
937
|
+
}
|
|
938
|
+
const body = await artifact.open();
|
|
939
|
+
const exit = await executor.pipeIntoCommand(
|
|
940
|
+
service,
|
|
941
|
+
safeRestoreCommand(
|
|
942
|
+
codec,
|
|
943
|
+
`tar -x -C ${shellQuote(path)}`,
|
|
944
|
+
guardRestorePath(path, codec, wantsClear)
|
|
945
|
+
),
|
|
946
|
+
body,
|
|
947
|
+
{ timeoutMs: payloadSpec("path").restoreTimeoutMs }
|
|
948
|
+
);
|
|
949
|
+
if (exit.code !== 0) {
|
|
950
|
+
throw new Error(
|
|
951
|
+
`Restoring ${path} into service "${service.name}" failed (exit ${exit.code}): ${exit.stderr.slice(0, 500)}`
|
|
952
|
+
);
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
};
|
|
956
|
+
var PathArchiveProducer = new PathArchiveProducerImpl();
|
|
957
|
+
registerProducer(PathArchiveProducer);
|
|
958
|
+
|
|
658
959
|
// ../../packages/adapters/src/backup/producers/volume.ts
|
|
659
960
|
var VolumeCopyProducerImpl = class {
|
|
660
961
|
kind = "volume";
|
|
@@ -663,16 +964,26 @@ var VolumeCopyProducerImpl = class {
|
|
|
663
964
|
async *produce(service, executor, opts) {
|
|
664
965
|
const sources = await executor.listSources(service);
|
|
665
966
|
if (sources.length === 0) {
|
|
666
|
-
|
|
967
|
+
throw new Error(
|
|
968
|
+
`Nothing to back up: no volumes or bind mounts found for service "${service.name}". ` + (service.containerId ? `Its live container (${service.containerId.slice(0, 12)}) reports no volume or bind mounts, so there is no persistent data here to capture.` : `No container was resolved for it, so only the service's recorded volumes could be checked and it has none. If this service does hold data, redeploy it so Openship records its volumes, or point the policy at a database payload instead of a volume snapshot.`)
|
|
969
|
+
);
|
|
667
970
|
}
|
|
668
971
|
const selected = opts.sourceIds && opts.sourceIds.length > 0 ? sources.filter((s2) => opts.sourceIds.includes(s2.id)) : sources.filter((s2) => s2.type !== "tmpfs");
|
|
972
|
+
if (selected.length === 0) {
|
|
973
|
+
const available = sources.map((s2) => `${s2.id} (${s2.type})`).join(", ");
|
|
974
|
+
throw new Error(
|
|
975
|
+
`Nothing to back up for service "${service.name}": ` + (opts.sourceIds && opts.sourceIds.length > 0 ? `the policy selects ${opts.sourceIds.join(", ")}, none of which is a source on this service. Available: ${available}.` : `its only mounts are tmpfs, which hold no data across a restart. Found: ${available}.`)
|
|
976
|
+
);
|
|
977
|
+
}
|
|
669
978
|
for (const source of selected) {
|
|
979
|
+
const compression = opts.compression ?? "zstd";
|
|
670
980
|
const { stdout, awaitExit } = await executor.streamPath(service, source.id, {
|
|
671
|
-
compression
|
|
672
|
-
exclude: opts.exclude
|
|
981
|
+
compression,
|
|
982
|
+
exclude: opts.exclude,
|
|
983
|
+
quiesce: opts.quiesce
|
|
673
984
|
});
|
|
674
985
|
yield {
|
|
675
|
-
name: `volume-${sanitizeArtifactName(source.id)}.tar.zst`,
|
|
986
|
+
name: `volume-${sanitizeArtifactName(source.id)}.tar${compression === "zstd" ? ".zst" : compression === "gzip" ? ".gz" : ""}`,
|
|
676
987
|
stream: stdout,
|
|
677
988
|
payloadKind: "volume",
|
|
678
989
|
sizeHint: source.sizeHint,
|
|
@@ -681,7 +992,12 @@ var VolumeCopyProducerImpl = class {
|
|
|
681
992
|
volumeSource: source.source,
|
|
682
993
|
volumeTarget: source.target,
|
|
683
994
|
volumeType: source.type,
|
|
684
|
-
compression
|
|
995
|
+
compression,
|
|
996
|
+
// Recorded so the artifact says what it IS, rather than leaving an operator to
|
|
997
|
+
// assume. `crash` means the service kept writing while tar walked the tree —
|
|
998
|
+
// usable for most payloads, and the reason a database deserves a logical dump
|
|
999
|
+
// instead. `quiesced` means the container was frozen for the copy.
|
|
1000
|
+
consistency: opts.quiesce ? "quiesced" : "crash"
|
|
685
1001
|
}
|
|
686
1002
|
};
|
|
687
1003
|
const exit = await awaitExit;
|
|
@@ -701,7 +1017,12 @@ var VolumeCopyProducerImpl = class {
|
|
|
701
1017
|
}
|
|
702
1018
|
const stream = await artifact.open();
|
|
703
1019
|
await executor.receiveStream(service, volumeId, stream, {
|
|
704
|
-
|
|
1020
|
+
// Through the recorded-codec reader rather than a cast: the cast trusted jsonb to
|
|
1021
|
+
// hold one of three strings, and anything else (a typo in a hand-written policy,
|
|
1022
|
+
// a value from a newer build) reached the restore helper as a codec it cannot
|
|
1023
|
+
// build a pipeline for. `recordedCodec` maps the unknown to `zstd` — the only
|
|
1024
|
+
// safe guess, and for the same documented reason.
|
|
1025
|
+
compression: recordedCodec(artifact.metadata.compression),
|
|
705
1026
|
clearTarget: opts.clearTarget ?? true,
|
|
706
1027
|
signal: opts.signal
|
|
707
1028
|
});
|
|
@@ -715,10 +1036,116 @@ registerProducer(VolumeCopyProducer);
|
|
|
715
1036
|
|
|
716
1037
|
// ../../packages/adapters/src/backup/executors/docker.ts
|
|
717
1038
|
import { PassThrough } from "stream";
|
|
1039
|
+
|
|
1040
|
+
// ../../packages/adapters/src/runtime/docker-demux.ts
|
|
1041
|
+
var HEADER_BYTES = 8;
|
|
1042
|
+
var MAX_FRAME_BYTES = 64 * 1024 * 1024;
|
|
1043
|
+
var STREAM_STDIN = 0;
|
|
1044
|
+
var STREAM_STDOUT = 1;
|
|
1045
|
+
var STREAM_STDERR = 2;
|
|
1046
|
+
function gateSink(sink, onBlock, onUnblock, onWriteAfterEnd) {
|
|
1047
|
+
let blocked = false;
|
|
1048
|
+
return {
|
|
1049
|
+
write(chunk) {
|
|
1050
|
+
if (sink.destroyed || sink.writableEnded) return onWriteAfterEnd();
|
|
1051
|
+
if (sink.write(chunk) !== false || blocked) return;
|
|
1052
|
+
blocked = true;
|
|
1053
|
+
onBlock();
|
|
1054
|
+
sink.once("drain", () => {
|
|
1055
|
+
blocked = false;
|
|
1056
|
+
onUnblock();
|
|
1057
|
+
});
|
|
1058
|
+
}
|
|
1059
|
+
};
|
|
1060
|
+
}
|
|
1061
|
+
function demuxDockerStream(source, stdout, stderr, onError) {
|
|
1062
|
+
const src = source;
|
|
1063
|
+
let blockedSinks = 0;
|
|
1064
|
+
let pausedByUs = false;
|
|
1065
|
+
const onBlock = () => {
|
|
1066
|
+
blockedSinks += 1;
|
|
1067
|
+
if (pausedByUs) return;
|
|
1068
|
+
pausedByUs = true;
|
|
1069
|
+
src.pause();
|
|
1070
|
+
};
|
|
1071
|
+
const onUnblock = () => {
|
|
1072
|
+
blockedSinks -= 1;
|
|
1073
|
+
if (blockedSinks > 0 || !pausedByUs) return;
|
|
1074
|
+
pausedByUs = false;
|
|
1075
|
+
src.resume();
|
|
1076
|
+
};
|
|
1077
|
+
const onWriteAfterEnd = () => fail(
|
|
1078
|
+
new Error(
|
|
1079
|
+
"Docker stream was ended while payload was still arriving: the sink had already closed, so those bytes were dropped. What was written is not a complete artifact."
|
|
1080
|
+
)
|
|
1081
|
+
);
|
|
1082
|
+
const out = gateSink(stdout, onBlock, onUnblock, onWriteAfterEnd);
|
|
1083
|
+
const err = gateSink(stderr, onBlock, onUnblock, onWriteAfterEnd);
|
|
1084
|
+
let carry = null;
|
|
1085
|
+
let failed = false;
|
|
1086
|
+
const fail = (cause) => {
|
|
1087
|
+
if (failed) return;
|
|
1088
|
+
failed = true;
|
|
1089
|
+
carry = null;
|
|
1090
|
+
src.pause();
|
|
1091
|
+
onError?.(cause);
|
|
1092
|
+
stdout.destroy(cause);
|
|
1093
|
+
};
|
|
1094
|
+
src.on("data", (chunk) => {
|
|
1095
|
+
if (failed) return;
|
|
1096
|
+
let buf = carry === null ? chunk : Buffer.concat([carry, chunk]);
|
|
1097
|
+
carry = null;
|
|
1098
|
+
let offset = 0;
|
|
1099
|
+
while (buf.length - offset >= HEADER_BYTES) {
|
|
1100
|
+
const type = buf[offset];
|
|
1101
|
+
const length = buf.readUInt32BE(offset + 4);
|
|
1102
|
+
if (type !== STREAM_STDIN && type !== STREAM_STDOUT && type !== STREAM_STDERR) {
|
|
1103
|
+
fail(
|
|
1104
|
+
new Error(
|
|
1105
|
+
`Docker stream desynchronized: frame type ${type} at offset ${offset} is not stdin/stdout/stderr. Refusing to guess where the payload resumes \u2014 the bytes already read cannot be trusted as a complete artifact.`
|
|
1106
|
+
)
|
|
1107
|
+
);
|
|
1108
|
+
return;
|
|
1109
|
+
}
|
|
1110
|
+
if (length > MAX_FRAME_BYTES) {
|
|
1111
|
+
fail(
|
|
1112
|
+
new Error(
|
|
1113
|
+
`Docker stream desynchronized: frame claims ${length} bytes, over the ${MAX_FRAME_BYTES}-byte sanity ceiling. Treating as a lost frame boundary rather than waiting for input that will never arrive.`
|
|
1114
|
+
)
|
|
1115
|
+
);
|
|
1116
|
+
return;
|
|
1117
|
+
}
|
|
1118
|
+
if (buf.length - offset - HEADER_BYTES < length) break;
|
|
1119
|
+
const payload = buf.subarray(offset + HEADER_BYTES, offset + HEADER_BYTES + length);
|
|
1120
|
+
if (type === STREAM_STDOUT) out.write(payload);
|
|
1121
|
+
else if (type === STREAM_STDERR) err.write(payload);
|
|
1122
|
+
offset += HEADER_BYTES + length;
|
|
1123
|
+
}
|
|
1124
|
+
if (offset < buf.length) {
|
|
1125
|
+
carry = Buffer.from(buf.subarray(offset));
|
|
1126
|
+
}
|
|
1127
|
+
});
|
|
1128
|
+
src.on("error", (cause) => fail(cause));
|
|
1129
|
+
src.on("end", () => {
|
|
1130
|
+
if (failed || carry === null) return;
|
|
1131
|
+
fail(
|
|
1132
|
+
new Error(
|
|
1133
|
+
`Docker stream ended mid-frame with ${carry.length} bytes of an incomplete frame buffered. The payload was cut short, so what was read is not a complete artifact.`
|
|
1134
|
+
)
|
|
1135
|
+
);
|
|
1136
|
+
});
|
|
1137
|
+
return {
|
|
1138
|
+
pending: () => carry !== null || src.readableLength > 0 || blockedSinks > 0
|
|
1139
|
+
};
|
|
1140
|
+
}
|
|
1141
|
+
|
|
1142
|
+
// ../../packages/adapters/src/backup/executors/docker.ts
|
|
718
1143
|
var HELPER_IMAGE = "alpine:3";
|
|
1144
|
+
var ARTIFACT_BUFFER_BYTES = 4 * 1024 * 1024;
|
|
719
1145
|
var DEFAULT_HELPER_IDLE_MS = 10 * 60 * 1e3;
|
|
720
1146
|
var DEFAULT_HELPER_TIMEOUT_MS = 6 * 60 * 60 * 1e3;
|
|
721
1147
|
var EXIT_POLL_INTERVAL_MS = 2e3;
|
|
1148
|
+
var ATTACH_DRAIN_QUIET_MS = 3e3;
|
|
722
1149
|
function createIdleWatchdog(ms, message) {
|
|
723
1150
|
let timer;
|
|
724
1151
|
let done = false;
|
|
@@ -768,9 +1195,6 @@ function destroyQuietly(stream) {
|
|
|
768
1195
|
} catch {
|
|
769
1196
|
}
|
|
770
1197
|
}
|
|
771
|
-
function shellEscape4(s2) {
|
|
772
|
-
return `'${s2.replace(/'/g, "'\\''")}'`;
|
|
773
|
-
}
|
|
774
1198
|
function compressionFlag(compression) {
|
|
775
1199
|
switch (compression) {
|
|
776
1200
|
case "gzip":
|
|
@@ -800,6 +1224,26 @@ var DockerBackupExecutor = class {
|
|
|
800
1224
|
get dockerode() {
|
|
801
1225
|
return this.runtime.docker;
|
|
802
1226
|
}
|
|
1227
|
+
/**
|
|
1228
|
+
* Freeze / thaw the target container around a copy — Docker's native cgroup freezer.
|
|
1229
|
+
*
|
|
1230
|
+
* `unpause` is best-effort and deliberately swallows "not paused": the thaw runs from a
|
|
1231
|
+
* `finally`, and a container the daemon already restarted (or that a human unpaused)
|
|
1232
|
+
* must not turn a completed backup into a failed one. `pause` is NOT best-effort — see
|
|
1233
|
+
* streamPath.
|
|
1234
|
+
*/
|
|
1235
|
+
async freeze(containerId) {
|
|
1236
|
+
await this.dockerode.getContainer(containerId).pause();
|
|
1237
|
+
}
|
|
1238
|
+
async thaw(containerId, label) {
|
|
1239
|
+
try {
|
|
1240
|
+
await this.dockerode.getContainer(containerId).unpause();
|
|
1241
|
+
} catch (err) {
|
|
1242
|
+
console.error(
|
|
1243
|
+
`[backup] ${label}: could not unpause container ${containerId.slice(0, 12)} after the copy \u2014 the service may still be FROZEN and needs \`docker unpause\`: ${err?.message ?? String(err)}`
|
|
1244
|
+
);
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
803
1247
|
async listSources(service) {
|
|
804
1248
|
if (service.containerId) {
|
|
805
1249
|
try {
|
|
@@ -819,7 +1263,7 @@ var DockerBackupExecutor = class {
|
|
|
819
1263
|
return service.volumes.map((spec, i2) => {
|
|
820
1264
|
const parsed = parseVolumeSpec(spec);
|
|
821
1265
|
if (!parsed || !parsed.source) return null;
|
|
822
|
-
const source = parsed.type === "volume" && service.namespaceVolumes ?
|
|
1266
|
+
const source = parsed.type === "volume" && service.namespaceVolumes ? ensureScopedVolumeName(service.projectSlug, parsed.source) : parsed.source;
|
|
823
1267
|
return {
|
|
824
1268
|
id: `${source}-${i2}`,
|
|
825
1269
|
source,
|
|
@@ -828,6 +1272,31 @@ var DockerBackupExecutor = class {
|
|
|
828
1272
|
};
|
|
829
1273
|
}).filter((x2) => x2 !== null);
|
|
830
1274
|
}
|
|
1275
|
+
/**
|
|
1276
|
+
* The container's own `Config.Env`, as `KEY=value` pairs already split.
|
|
1277
|
+
*
|
|
1278
|
+
* This is the credentials source for every service Openship did not deploy
|
|
1279
|
+
* itself — see `BackupExecutor.readContainerEnv`. Returns `{}` rather than
|
|
1280
|
+
* throwing on any failure: it is an ENRICHMENT of the handle's env, so a
|
|
1281
|
+
* container that has gone away, or a daemon that refuses the inspect, must leave
|
|
1282
|
+
* the caller exactly as well off as before it asked.
|
|
1283
|
+
*/
|
|
1284
|
+
async readContainerEnv(service) {
|
|
1285
|
+
if (!service.containerId) return {};
|
|
1286
|
+
try {
|
|
1287
|
+
const data = await this.dockerode.getContainer(service.containerId).inspect();
|
|
1288
|
+
const pairs = data.Config?.Env ?? [] ?? [];
|
|
1289
|
+
const env = {};
|
|
1290
|
+
for (const pair of pairs) {
|
|
1291
|
+
const eq = pair.indexOf("=");
|
|
1292
|
+
if (eq <= 0) continue;
|
|
1293
|
+
env[pair.slice(0, eq)] = pair.slice(eq + 1);
|
|
1294
|
+
}
|
|
1295
|
+
return env;
|
|
1296
|
+
} catch {
|
|
1297
|
+
return {};
|
|
1298
|
+
}
|
|
1299
|
+
}
|
|
831
1300
|
async execStream(service, cmd, opts) {
|
|
832
1301
|
if (!service.containerId) {
|
|
833
1302
|
throw new Error(
|
|
@@ -843,7 +1312,10 @@ var DockerBackupExecutor = class {
|
|
|
843
1312
|
WorkingDir: opts?.cwd,
|
|
844
1313
|
Env: opts?.env ? Object.entries(opts.env).map(([k2, v2]) => `${k2}=${v2}`) : void 0
|
|
845
1314
|
});
|
|
846
|
-
const stream = await exec.
|
|
1315
|
+
const stream = await startExecStream(daemonConnectionFrom(this.dockerode), exec.id, {
|
|
1316
|
+
tty: false,
|
|
1317
|
+
stdin: false
|
|
1318
|
+
});
|
|
847
1319
|
return this.attachDemuxed(this.dockerode, exec.id, stream, opts, {
|
|
848
1320
|
label: `exec in service ${service.name}`
|
|
849
1321
|
});
|
|
@@ -858,9 +1330,9 @@ var DockerBackupExecutor = class {
|
|
|
858
1330
|
throw new Error(`Backup source "${sourceId}" is tmpfs \u2014 not backupable`);
|
|
859
1331
|
}
|
|
860
1332
|
const compression = opts?.compression ?? "zstd";
|
|
861
|
-
const excludeArgs = (opts?.exclude ?? []).flatMap((p2) => ["--exclude",
|
|
1333
|
+
const excludeArgs = (opts?.exclude ?? []).flatMap((p2) => ["--exclude", shellQuote(p2)]);
|
|
862
1334
|
const tarFlags = compressionFlag(compression);
|
|
863
|
-
const tarCmd =
|
|
1335
|
+
const tarCmd = `tar -c${tarFlags} -C /mnt ${excludeArgs.join(" ")} .`;
|
|
864
1336
|
const helperImage = compression === "zstd" ? "alpine:3" : HELPER_IMAGE;
|
|
865
1337
|
await this.ensureImage(helperImage);
|
|
866
1338
|
const hostConfig = {
|
|
@@ -880,14 +1352,52 @@ var DockerBackupExecutor = class {
|
|
|
880
1352
|
// the helper exits. Nothing ever reads these logs.
|
|
881
1353
|
LogConfig: { Type: "none", Config: {} }
|
|
882
1354
|
};
|
|
883
|
-
|
|
1355
|
+
const quiesceId = opts?.quiesce ? service.containerId : null;
|
|
1356
|
+
if (opts?.quiesce && !quiesceId) {
|
|
1357
|
+
throw new Error(
|
|
1358
|
+
`Cannot quiesce service "${service.name}" for a consistent copy: it has no running container to freeze. Either deploy it, or take the copy without quiesce and accept a crash-consistent archive.`
|
|
1359
|
+
);
|
|
1360
|
+
}
|
|
1361
|
+
if (quiesceId) await this.freeze(quiesceId);
|
|
1362
|
+
const handed = await this.handOffHelper(
|
|
884
1363
|
{
|
|
885
1364
|
Image: helperImage,
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
1365
|
+
// Built by the SAME helper the DB producers use, for the same reason: a shell
|
|
1366
|
+
// pipeline exits with its LAST command's status, so `tar -c … | zstd` reported
|
|
1367
|
+
// ZSTD's success and a tar that failed mid-archive — a permissions error, a
|
|
1368
|
+
// vanishing file, ENOSPC in the pipe — came back as exit 0 over a truncated
|
|
1369
|
+
// archive that the run then recorded as a complete backup. Closing that for
|
|
1370
|
+
// `pg_dump | zstd` and leaving it here would have been the same bug with a
|
|
1371
|
+
// different payload, and the volume producer is the DEFAULT one.
|
|
1372
|
+
//
|
|
1373
|
+
// gzip and none need no pipeline at all (busybox tar compresses in-process via
|
|
1374
|
+
// `tarFlags`), so those pass `"none"` and keep tar's own status directly.
|
|
1375
|
+
Cmd: safeDumpCommand(
|
|
1376
|
+
tarCmd,
|
|
1377
|
+
compression === "zstd" ? "zstd" : "none",
|
|
1378
|
+
[
|
|
1379
|
+
// REFUSE an empty mount before doing anything else.
|
|
1380
|
+
//
|
|
1381
|
+
// The daemon CREATES a missing bind directory and a missing named volume
|
|
1382
|
+
// when it honors HostConfig.Binds, so a source that is not on this host
|
|
1383
|
+
// produces a helper that starts fine and a `tar -c` of nothing that exits
|
|
1384
|
+
// 0. That archive is ~350 bytes compressed, so the orchestrator's
|
|
1385
|
+
// zero-byte guard passes and the run is recorded as a successful backup —
|
|
1386
|
+
// a nightly green tick over an external disk that failed to mount, a data
|
|
1387
|
+
// directory that was renamed under a still-running container, or a volume
|
|
1388
|
+
// that `docker system prune --volumes` took.
|
|
1389
|
+
//
|
|
1390
|
+
// An intentionally-empty source fails too, which is the deliberate half of
|
|
1391
|
+
// the trade: an archive of nothing is not a restore point either way, and
|
|
1392
|
+
// the operator has `payloadConfig.sourceIds` to narrow the policy. Silence
|
|
1393
|
+
// is the only outcome that cannot be recovered from.
|
|
1394
|
+
`SRC=${shellQuote(sourceId)}`,
|
|
1395
|
+
`[ -n "$(ls -A /mnt 2>/dev/null)" ] || { echo "openship: backup source $SRC is empty or missing on this host, so this archive would contain nothing. Refusing to record it as a backup. If it is a mount, check that it is mounted; if it is empty on purpose, narrow the policy's sourceIds." >&2; exit 91; }`,
|
|
1396
|
+
// zstd is not in alpine:3, so the helper installs it. Second, so a doomed
|
|
1397
|
+
// capture does not pay the ~20s fetch first.
|
|
1398
|
+
compression === "zstd" ? "apk add --no-cache zstd >/dev/null 2>&1" : null
|
|
1399
|
+
].filter(Boolean).join("\n")
|
|
1400
|
+
),
|
|
891
1401
|
HostConfig: hostConfig,
|
|
892
1402
|
AttachStdout: true,
|
|
893
1403
|
AttachStderr: true,
|
|
@@ -897,8 +1407,8 @@ var DockerBackupExecutor = class {
|
|
|
897
1407
|
NetworkDisabled: compression !== "zstd"
|
|
898
1408
|
},
|
|
899
1409
|
async (helper) => {
|
|
900
|
-
const stream = await helper.
|
|
901
|
-
|
|
1410
|
+
const stream = await startAttachStream(daemonConnectionFrom(this.dockerode), helper.id, {
|
|
1411
|
+
stdin: false,
|
|
902
1412
|
stdout: true,
|
|
903
1413
|
stderr: true
|
|
904
1414
|
});
|
|
@@ -909,7 +1419,22 @@ var DockerBackupExecutor = class {
|
|
|
909
1419
|
label: `Backup of "${sourceId}" on service ${service.name}`
|
|
910
1420
|
});
|
|
911
1421
|
}
|
|
912
|
-
)
|
|
1422
|
+
).catch(async (err) => {
|
|
1423
|
+
if (quiesceId) await this.thaw(quiesceId, `backup of "${sourceId}"`);
|
|
1424
|
+
throw err;
|
|
1425
|
+
});
|
|
1426
|
+
if (!quiesceId) return handed;
|
|
1427
|
+
return {
|
|
1428
|
+
stdout: handed.stdout,
|
|
1429
|
+
// Thaw when the copy is DONE, however it ends. `awaitExit` settles once the helper
|
|
1430
|
+
// has exited and its output has drained, which is precisely the window the freeze has
|
|
1431
|
+
// to cover; unfreezing when `streamPath` returned would thaw before a single byte had
|
|
1432
|
+
// been read. The original promise is handed back so a caller still sees its value or
|
|
1433
|
+
// its rejection.
|
|
1434
|
+
awaitExit: handed.awaitExit.finally(
|
|
1435
|
+
() => this.thaw(quiesceId, `backup of "${sourceId}"`)
|
|
1436
|
+
)
|
|
1437
|
+
};
|
|
913
1438
|
}
|
|
914
1439
|
async receiveStream(service, targetSourceId, body, opts) {
|
|
915
1440
|
opts?.signal?.throwIfAborted();
|
|
@@ -924,16 +1449,18 @@ var DockerBackupExecutor = class {
|
|
|
924
1449
|
const compression = opts?.compression ?? "zstd";
|
|
925
1450
|
const helperImage = compression === "zstd" ? "alpine:3" : HELPER_IMAGE;
|
|
926
1451
|
await this.ensureImage(helperImage);
|
|
927
|
-
const
|
|
928
|
-
const
|
|
1452
|
+
const restoreCodec = compression === "none" ? "none" : compression;
|
|
1453
|
+
const prelude = [
|
|
1454
|
+
compression === "zstd" ? "apk add --no-cache zstd >/dev/null 2>&1 || true" : null,
|
|
1455
|
+
compression === "zstd" ? 'command -v zstd >/dev/null 2>&1 || { echo "openship: zstd is not available in the restore helper and could not be installed (the host may have no egress). Refusing to clear the target: nothing was written and your data is untouched." >&2; exit 90; }' : null,
|
|
1456
|
+
compression === "gzip" ? 'command -v gzip >/dev/null 2>&1 || { echo "openship: gzip is not available in the restore helper. Refusing to clear the target; your data is untouched." >&2; exit 90; }' : null,
|
|
1457
|
+
// Only now, with every tool proven present.
|
|
1458
|
+
opts?.clearTarget ? "find /mnt -mindepth 1 -delete 2>/dev/null || true" : null
|
|
1459
|
+
].filter(Boolean).join("\n");
|
|
929
1460
|
return this.withHelper(
|
|
930
1461
|
{
|
|
931
1462
|
Image: helperImage,
|
|
932
|
-
Cmd:
|
|
933
|
-
"sh",
|
|
934
|
-
"-c",
|
|
935
|
-
compression === "zstd" ? `apk add --no-cache zstd >/dev/null 2>&1; ${untarCmd}` : untarCmd
|
|
936
|
-
],
|
|
1463
|
+
Cmd: safeRestoreCommand(restoreCodec, "tar -x -C /mnt", prelude),
|
|
937
1464
|
// AutoRemove is OFF deliberately, and this is the fix for the `404 no such
|
|
938
1465
|
// container` in #434: with it on, the daemon could reap the helper before
|
|
939
1466
|
// /containers/{id}/wait answered, and the exit status was then unknowable.
|
|
@@ -1218,47 +1745,69 @@ var DockerBackupExecutor = class {
|
|
|
1218
1745
|
const { PassThrough: PassThrough3 } = await import("stream");
|
|
1219
1746
|
const stdoutSink = new PassThrough3();
|
|
1220
1747
|
stdoutSink.resume();
|
|
1748
|
+
stdoutSink.on("error", () => {
|
|
1749
|
+
});
|
|
1221
1750
|
const stderrSink = new PassThrough3();
|
|
1222
1751
|
stderrSink.on("data", (chunk) => {
|
|
1223
1752
|
if (stderrChunks.length < 16) stderrChunks.push(chunk);
|
|
1224
1753
|
});
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
const
|
|
1231
|
-
try {
|
|
1232
|
-
stream.end?.();
|
|
1233
|
-
} catch {
|
|
1234
|
-
}
|
|
1235
|
-
}, opts.timeoutMs) : null;
|
|
1236
|
-
return new Promise((resolve2, reject) => {
|
|
1754
|
+
demuxDockerStream(stream, stdoutSink, stderrSink);
|
|
1755
|
+
const stderrTail = () => Buffer.concat(stderrChunks).toString("utf8").slice(0, 16 * 1024);
|
|
1756
|
+
const timeoutMs = opts?.timeoutMs ?? DEFAULT_HELPER_TIMEOUT_MS;
|
|
1757
|
+
let bodyEnded = false;
|
|
1758
|
+
let polling = true;
|
|
1759
|
+
const attachEnded = new Promise((resolve2, reject) => {
|
|
1237
1760
|
body.on("error", (err) => {
|
|
1238
|
-
if (timer) clearTimeout(timer);
|
|
1239
1761
|
try {
|
|
1240
1762
|
stream.end?.();
|
|
1241
1763
|
} catch {
|
|
1242
1764
|
}
|
|
1243
1765
|
reject(err);
|
|
1244
1766
|
});
|
|
1767
|
+
body.on("end", () => {
|
|
1768
|
+
bodyEnded = true;
|
|
1769
|
+
});
|
|
1245
1770
|
stream.on("end", async () => {
|
|
1246
|
-
if (timer) clearTimeout(timer);
|
|
1247
1771
|
try {
|
|
1248
|
-
resolve2({
|
|
1249
|
-
code: await resolveExecExitCode(exec, exec.id),
|
|
1250
|
-
stderr: Buffer.concat(stderrChunks).toString("utf8").slice(0, 16 * 1024)
|
|
1251
|
-
});
|
|
1772
|
+
resolve2({ code: await resolveExecExitCode(exec, exec.id), stderr: stderrTail() });
|
|
1252
1773
|
} catch (err) {
|
|
1253
1774
|
reject(err);
|
|
1254
1775
|
}
|
|
1255
1776
|
});
|
|
1256
|
-
stream.on("error", (err) =>
|
|
1257
|
-
if (timer) clearTimeout(timer);
|
|
1258
|
-
reject(err);
|
|
1259
|
-
});
|
|
1777
|
+
stream.on("error", (err) => reject(err));
|
|
1260
1778
|
body.pipe(stream);
|
|
1261
1779
|
});
|
|
1780
|
+
const polledExit = async () => {
|
|
1781
|
+
while (polling) {
|
|
1782
|
+
await new Promise((r2) => {
|
|
1783
|
+
const t8 = setTimeout(r2, EXIT_POLL_INTERVAL_MS);
|
|
1784
|
+
t8.unref?.();
|
|
1785
|
+
});
|
|
1786
|
+
if (!polling || !bodyEnded) continue;
|
|
1787
|
+
let info;
|
|
1788
|
+
try {
|
|
1789
|
+
info = await exec.inspect();
|
|
1790
|
+
} catch {
|
|
1791
|
+
continue;
|
|
1792
|
+
}
|
|
1793
|
+
if (info.Running === false && typeof info.ExitCode === "number") {
|
|
1794
|
+
return { code: info.ExitCode, stderr: stderrTail() };
|
|
1795
|
+
}
|
|
1796
|
+
}
|
|
1797
|
+
return new Promise(() => {
|
|
1798
|
+
});
|
|
1799
|
+
};
|
|
1800
|
+
try {
|
|
1801
|
+
return await withTimeout(
|
|
1802
|
+
Promise.race([attachEnded, polledExit()]),
|
|
1803
|
+
timeoutMs,
|
|
1804
|
+
`Restore into service ${service.name} exceeded its ${Math.round(timeoutMs / 1e3)}s ceiling and was abandoned. The target may hold partial data.`
|
|
1805
|
+
);
|
|
1806
|
+
} finally {
|
|
1807
|
+
polling = false;
|
|
1808
|
+
destroyQuietly(stream);
|
|
1809
|
+
destroyQuietly(body);
|
|
1810
|
+
}
|
|
1262
1811
|
}
|
|
1263
1812
|
async stopService(service) {
|
|
1264
1813
|
if (!service.containerId) return;
|
|
@@ -1297,13 +1846,18 @@ var DockerBackupExecutor = class {
|
|
|
1297
1846
|
const label = meta?.label ?? "exec";
|
|
1298
1847
|
const idleMs = opts?.idleTimeoutMs ?? DEFAULT_HELPER_IDLE_MS;
|
|
1299
1848
|
const timeoutMs = opts?.timeoutMs ?? DEFAULT_HELPER_TIMEOUT_MS;
|
|
1300
|
-
const stdout = new PassThrough();
|
|
1849
|
+
const stdout = new PassThrough({ highWaterMark: ARTIFACT_BUFFER_BYTES });
|
|
1301
1850
|
const stderrChunks = [];
|
|
1302
1851
|
const stderrSink = new PassThrough();
|
|
1303
1852
|
stderrSink.on("data", (chunk) => {
|
|
1304
1853
|
if (stderrChunks.length < 16) stderrChunks.push(chunk);
|
|
1305
1854
|
});
|
|
1306
|
-
|
|
1855
|
+
let demuxError = null;
|
|
1856
|
+
let rejectExit = null;
|
|
1857
|
+
demuxDockerStream(stream, stdout, stderrSink, (err) => {
|
|
1858
|
+
demuxError ??= err;
|
|
1859
|
+
rejectExit?.(err);
|
|
1860
|
+
});
|
|
1307
1861
|
let sinksEnded = false;
|
|
1308
1862
|
const endSinks = () => {
|
|
1309
1863
|
if (sinksEnded) return;
|
|
@@ -1318,9 +1872,13 @@ var DockerBackupExecutor = class {
|
|
|
1318
1872
|
`${label} produced no data for ${Math.round(idleMs / 1e3)}s and was abandoned.`
|
|
1319
1873
|
);
|
|
1320
1874
|
stream.on("data", () => watchdog.touch());
|
|
1875
|
+
stdout.on("drain", () => watchdog.touch());
|
|
1321
1876
|
const awaitExit = (async () => {
|
|
1322
1877
|
const onEnd = new Promise((resolve2, reject) => {
|
|
1878
|
+
rejectExit = reject;
|
|
1879
|
+
if (demuxError) reject(demuxError);
|
|
1323
1880
|
stream.on("end", async () => {
|
|
1881
|
+
if (demuxError) return reject(demuxError);
|
|
1324
1882
|
try {
|
|
1325
1883
|
resolve2({
|
|
1326
1884
|
code: await resolveExecExitCode(docker.getExec(execId), execId),
|
|
@@ -1340,21 +1898,33 @@ var DockerBackupExecutor = class {
|
|
|
1340
1898
|
);
|
|
1341
1899
|
} catch (err) {
|
|
1342
1900
|
stdout.destroy(err);
|
|
1901
|
+
destroyQuietly(stream);
|
|
1343
1902
|
throw err;
|
|
1344
1903
|
} finally {
|
|
1345
1904
|
watchdog.dispose();
|
|
1346
1905
|
}
|
|
1347
1906
|
})();
|
|
1907
|
+
void awaitExit.catch(() => {
|
|
1908
|
+
});
|
|
1348
1909
|
return { stdout, awaitExit };
|
|
1349
1910
|
}
|
|
1350
1911
|
demuxContainerStream(container, stream, opts) {
|
|
1351
|
-
const stdout = new PassThrough();
|
|
1912
|
+
const stdout = new PassThrough({ highWaterMark: ARTIFACT_BUFFER_BYTES });
|
|
1352
1913
|
const stderrChunks = [];
|
|
1353
1914
|
const stderrSink = new PassThrough();
|
|
1354
1915
|
stderrSink.on("data", (chunk) => {
|
|
1355
1916
|
if (stderrChunks.length < 16) stderrChunks.push(chunk);
|
|
1356
1917
|
});
|
|
1357
|
-
|
|
1918
|
+
let demuxError = null;
|
|
1919
|
+
const demux = demuxDockerStream(
|
|
1920
|
+
stream,
|
|
1921
|
+
stdout,
|
|
1922
|
+
stderrSink,
|
|
1923
|
+
(err) => {
|
|
1924
|
+
demuxError = err;
|
|
1925
|
+
}
|
|
1926
|
+
);
|
|
1927
|
+
let lastProgressAt = Date.now();
|
|
1358
1928
|
let sinksEnded = false;
|
|
1359
1929
|
let exited = false;
|
|
1360
1930
|
const reapIfDone = () => {
|
|
@@ -1374,7 +1944,14 @@ var DockerBackupExecutor = class {
|
|
|
1374
1944
|
opts.idleTimeoutMs,
|
|
1375
1945
|
`${opts.label} produced no data for ${Math.round(opts.idleTimeoutMs / 1e3)}s and was abandoned. Nothing was written; the archive is incomplete and was not kept.`
|
|
1376
1946
|
);
|
|
1377
|
-
stream.on("data", () =>
|
|
1947
|
+
stream.on("data", () => {
|
|
1948
|
+
lastProgressAt = Date.now();
|
|
1949
|
+
watchdog.touch();
|
|
1950
|
+
});
|
|
1951
|
+
stdout.on("drain", () => {
|
|
1952
|
+
lastProgressAt = Date.now();
|
|
1953
|
+
watchdog.touch();
|
|
1954
|
+
});
|
|
1378
1955
|
const awaitExit = (async () => {
|
|
1379
1956
|
try {
|
|
1380
1957
|
const res = await this.awaitHelperExit(container, watchdog, () => sinksEnded, {
|
|
@@ -1383,8 +1960,19 @@ var DockerBackupExecutor = class {
|
|
|
1383
1960
|
note: "Nothing was written; the archive is incomplete and was not kept."
|
|
1384
1961
|
});
|
|
1385
1962
|
exited = true;
|
|
1386
|
-
|
|
1963
|
+
const armEndBackstop = () => {
|
|
1964
|
+
const timer = setTimeout(() => {
|
|
1965
|
+
if (sinksEnded) return;
|
|
1966
|
+
if (Date.now() - lastProgressAt < ATTACH_DRAIN_QUIET_MS || demux.pending()) {
|
|
1967
|
+
return armEndBackstop();
|
|
1968
|
+
}
|
|
1969
|
+
endSinks();
|
|
1970
|
+
}, ATTACH_DRAIN_QUIET_MS);
|
|
1971
|
+
timer.unref?.();
|
|
1972
|
+
};
|
|
1973
|
+
armEndBackstop();
|
|
1387
1974
|
reapIfDone();
|
|
1975
|
+
if (demuxError) throw demuxError;
|
|
1388
1976
|
return {
|
|
1389
1977
|
code: res.StatusCode,
|
|
1390
1978
|
stderr: Buffer.concat(stderrChunks).toString("utf8").slice(0, 16 * 1024)
|
|
@@ -1442,8 +2030,21 @@ var CloudBackupExecutor = class {
|
|
|
1442
2030
|
const ws = this.client.workspace(service.containerId);
|
|
1443
2031
|
const rt = await ws.runtime();
|
|
1444
2032
|
const fullCmd = this.composeShellCommand(cmd, opts);
|
|
1445
|
-
|
|
1446
|
-
|
|
2033
|
+
let resumeRead = null;
|
|
2034
|
+
const stdout = new Readable2({
|
|
2035
|
+
read() {
|
|
2036
|
+
const resume = resumeRead;
|
|
2037
|
+
resumeRead = null;
|
|
2038
|
+
resume?.();
|
|
2039
|
+
}
|
|
2040
|
+
});
|
|
2041
|
+
const pushWithBackpressure = async (chunk) => {
|
|
2042
|
+
if (stdout.push(chunk)) return;
|
|
2043
|
+
if (stdout.destroyed) return;
|
|
2044
|
+
await new Promise((resolve2) => {
|
|
2045
|
+
resumeRead = resolve2;
|
|
2046
|
+
});
|
|
2047
|
+
};
|
|
1447
2048
|
let stderrBuf = "";
|
|
1448
2049
|
const awaitExit = new Promise((resolve2, reject) => {
|
|
1449
2050
|
const timer = opts?.timeoutMs ? setTimeout(() => {
|
|
@@ -1459,7 +2060,7 @@ var CloudBackupExecutor = class {
|
|
|
1459
2060
|
for await (const ev of stream) {
|
|
1460
2061
|
switch (ev.event) {
|
|
1461
2062
|
case "stdout":
|
|
1462
|
-
|
|
2063
|
+
await pushWithBackpressure(Buffer.from(ev.data, "base64"));
|
|
1463
2064
|
break;
|
|
1464
2065
|
case "stderr": {
|
|
1465
2066
|
const decoded = Buffer.from(ev.data, "base64").toString("utf8");
|
|
@@ -1483,6 +2084,10 @@ var CloudBackupExecutor = class {
|
|
|
1483
2084
|
if (timer) clearTimeout(timer);
|
|
1484
2085
|
stdout.destroy(err);
|
|
1485
2086
|
reject(err);
|
|
2087
|
+
} finally {
|
|
2088
|
+
const resume = resumeRead;
|
|
2089
|
+
resumeRead = null;
|
|
2090
|
+
resume?.();
|
|
1486
2091
|
}
|
|
1487
2092
|
};
|
|
1488
2093
|
void pump();
|
|
@@ -1492,9 +2097,9 @@ var CloudBackupExecutor = class {
|
|
|
1492
2097
|
/** Bake env + cwd into a shell command since Oblien's ExecRunParams
|
|
1493
2098
|
* doesn't carry them. */
|
|
1494
2099
|
composeShellCommand(cmd, opts) {
|
|
1495
|
-
const envPrefix = opts?.env ? Object.entries(opts.env).map(([k2, v2]) => `${k2}=${
|
|
1496
|
-
const cwdPrefix = opts?.cwd ? `cd ${
|
|
1497
|
-
const quotedCmd = cmd.map((arg) =>
|
|
2100
|
+
const envPrefix = opts?.env ? Object.entries(opts.env).map(([k2, v2]) => `${k2}=${shellQuote(v2)}`).join(" ") + " " : "";
|
|
2101
|
+
const cwdPrefix = opts?.cwd ? `cd ${shellQuote(opts.cwd)} && ` : "";
|
|
2102
|
+
const quotedCmd = cmd.map((arg) => shellQuote(arg)).join(" ");
|
|
1498
2103
|
return ["sh", "-c", `${cwdPrefix}${envPrefix}${quotedCmd}`];
|
|
1499
2104
|
}
|
|
1500
2105
|
async streamPath(service, sourceId, opts) {
|
|
@@ -1502,10 +2107,13 @@ var CloudBackupExecutor = class {
|
|
|
1502
2107
|
const compression = opts?.compression ?? "zstd";
|
|
1503
2108
|
const excludeArgs = (opts?.exclude ?? []).flatMap((p2) => [
|
|
1504
2109
|
"--exclude",
|
|
1505
|
-
|
|
2110
|
+
shellQuote(p2)
|
|
1506
2111
|
]);
|
|
1507
|
-
const tarCmd = compression === "
|
|
1508
|
-
return this.execStream(
|
|
2112
|
+
const tarCmd = compression === "gzip" ? `tar -cz -C ${shellQuote(path)} ${excludeArgs.join(" ")} .` : `tar -c -C ${shellQuote(path)} ${excludeArgs.join(" ")} .`;
|
|
2113
|
+
return this.execStream(
|
|
2114
|
+
service,
|
|
2115
|
+
safeDumpCommand(tarCmd, compression === "zstd" ? "zstd" : "none")
|
|
2116
|
+
);
|
|
1509
2117
|
}
|
|
1510
2118
|
async receiveStream(service, targetSourceId, body, opts) {
|
|
1511
2119
|
if (!service.containerId) {
|
|
@@ -1542,7 +2150,7 @@ var CloudBackupExecutor = class {
|
|
|
1542
2150
|
body,
|
|
1543
2151
|
dest: tmpPath
|
|
1544
2152
|
});
|
|
1545
|
-
const wrapped = `${this.composeShellCommand(cmd, opts).slice(2).join(" ")} < ${
|
|
2153
|
+
const wrapped = `${this.composeShellCommand(cmd, opts).slice(2).join(" ")} < ${shellQuote(tmpPath)}; ec=$?; rm -f ${shellQuote(tmpPath)}; exit $ec`;
|
|
1546
2154
|
const result = await this.execStream(service, ["sh", "-c", wrapped]);
|
|
1547
2155
|
result.stdout.resume();
|
|
1548
2156
|
return result.awaitExit;
|
|
@@ -1570,9 +2178,6 @@ var CloudBackupExecutor = class {
|
|
|
1570
2178
|
}
|
|
1571
2179
|
}
|
|
1572
2180
|
};
|
|
1573
|
-
function shellEscape5(s2) {
|
|
1574
|
-
return `'${s2.replace(/'/g, "'\\''")}'`;
|
|
1575
|
-
}
|
|
1576
2181
|
registerExecutor("cloud", (runtime) => {
|
|
1577
2182
|
if (!(runtime instanceof CloudRuntime)) {
|
|
1578
2183
|
throw new Error(
|
|
@@ -1588,6 +2193,8 @@ import { createWriteStream, promises as fs } from "fs";
|
|
|
1588
2193
|
import { randomBytes as randomBytes2 } from "crypto";
|
|
1589
2194
|
import { tmpdir } from "os";
|
|
1590
2195
|
import { join } from "path";
|
|
2196
|
+
var DEFAULT_RESTORE_TIMEOUT_MS = 6 * 60 * 60 * 1e3;
|
|
2197
|
+
var KILL_GRACE_MS = 3e4;
|
|
1591
2198
|
var BareBackupExecutor = class {
|
|
1592
2199
|
constructor(runtime) {
|
|
1593
2200
|
this.runtime = runtime;
|
|
@@ -1635,22 +2242,33 @@ var BareBackupExecutor = class {
|
|
|
1635
2242
|
const compression = opts?.compression ?? "zstd";
|
|
1636
2243
|
const excludeArgs = (opts?.exclude ?? []).flatMap((p2) => [
|
|
1637
2244
|
"--exclude",
|
|
1638
|
-
|
|
2245
|
+
shellQuote(p2)
|
|
1639
2246
|
]);
|
|
1640
2247
|
const path = sourceId;
|
|
1641
|
-
const tarCmd = compression === "zstd" ? `tar -c -C ${
|
|
2248
|
+
const tarCmd = compression === "zstd" ? `tar -c -C ${shellQuote(path)} ${excludeArgs.join(" ")} . | zstd -c -3` : compression === "gzip" ? `tar -cz -C ${shellQuote(path)} ${excludeArgs.join(" ")} .` : `tar -c -C ${shellQuote(path)} ${excludeArgs.join(" ")} .`;
|
|
1642
2249
|
return this.execStream(service, ["sh", "-c", tarCmd], { timeoutMs: opts?.timeoutMs });
|
|
1643
2250
|
}
|
|
1644
2251
|
async receiveStream(service, targetSourceId, body, opts) {
|
|
1645
2252
|
const compression = opts?.compression ?? "zstd";
|
|
1646
|
-
const
|
|
1647
|
-
const
|
|
1648
|
-
|
|
2253
|
+
const tool = compression === "zstd" ? "zstd" : compression === "gzip" ? "gzip" : null;
|
|
2254
|
+
const prelude = [
|
|
2255
|
+
`mkdir -p ${shellQuote(targetSourceId)}`,
|
|
2256
|
+
tool ? `command -v ${tool} >/dev/null 2>&1 || { echo "openship: '${tool}' is not installed on this host, so the artifact cannot be decompressed. Refusing to clear the target: nothing was written and your data is untouched." >&2; exit 90; }` : null,
|
|
2257
|
+
// Only now, with the decompressor proven present.
|
|
2258
|
+
opts?.clearTarget ? `rm -rf ${shellQuote(targetSourceId)}/* ${shellQuote(targetSourceId)}/.[!.]* 2>/dev/null || true` : null
|
|
2259
|
+
].filter(Boolean).join("\n");
|
|
2260
|
+
const cmd = safeRestoreCommand(
|
|
2261
|
+
compression === "none" ? "none" : compression,
|
|
2262
|
+
`tar -x -C ${shellQuote(targetSourceId)}`,
|
|
2263
|
+
prelude
|
|
2264
|
+
);
|
|
1649
2265
|
let bytesWritten = 0;
|
|
1650
2266
|
body.on("data", (chunk) => {
|
|
1651
2267
|
bytesWritten += chunk.byteLength;
|
|
1652
2268
|
});
|
|
1653
|
-
const exit = await this.pipeIntoCommand(service,
|
|
2269
|
+
const exit = await this.pipeIntoCommand(service, cmd, body, {
|
|
2270
|
+
timeoutMs: opts?.timeoutMs
|
|
2271
|
+
});
|
|
1654
2272
|
if (exit.code !== 0) {
|
|
1655
2273
|
throw new Error(
|
|
1656
2274
|
`receiveStream tar-extract exited ${exit.code}: ${exit.stderr.slice(0, 500)}`
|
|
@@ -1665,15 +2283,30 @@ var BareBackupExecutor = class {
|
|
|
1665
2283
|
const localFile = join(localDir, "artifact.bin");
|
|
1666
2284
|
const remoteDir = `/tmp/openship-bare-restore-${id}`;
|
|
1667
2285
|
const remoteFile = `${remoteDir}/artifact.bin`;
|
|
2286
|
+
const timeoutMs = opts?.timeoutMs ?? DEFAULT_RESTORE_TIMEOUT_MS;
|
|
2287
|
+
const ceiling = () => `Restore into ${service.name} exceeded its ${Math.round(timeoutMs / 1e3)}s ceiling and was abandoned. The target may hold partial data.`;
|
|
2288
|
+
if (exec.execWithInput) {
|
|
2289
|
+
const run = exec.execWithInput(`sh -c ${shellQuote(shellScript(cmd, opts))}`, body);
|
|
2290
|
+
const result = await withTimeout(run, timeoutMs, ceiling()).catch((err) => {
|
|
2291
|
+
body.destroy();
|
|
2292
|
+
throw err;
|
|
2293
|
+
});
|
|
2294
|
+
return { code: result.code, stderr: result.stderr };
|
|
2295
|
+
}
|
|
2296
|
+
const deadline = Date.now() + timeoutMs;
|
|
2297
|
+
const remaining = () => Math.max(1e3, deadline - Date.now());
|
|
1668
2298
|
await fs.mkdir(localDir, { recursive: true });
|
|
1669
2299
|
try {
|
|
1670
|
-
await pipeline(body, createWriteStream(localFile));
|
|
1671
|
-
await exec.transferIn(localDir, remoteDir);
|
|
1672
|
-
const wrapped = `${
|
|
1673
|
-
const runner = `sh -c ${
|
|
1674
|
-
const { stdout, awaitExit } = await this.execStream(service, ["sh", "-c", runner],
|
|
2300
|
+
await withTimeout(pipeline(body, createWriteStream(localFile)), remaining(), ceiling());
|
|
2301
|
+
await withTimeout(exec.transferIn(localDir, remoteDir), remaining(), ceiling());
|
|
2302
|
+
const wrapped = `${shellScript(cmd, opts)} < ${shellQuote(remoteFile)}`;
|
|
2303
|
+
const runner = `sh -c ${shellQuote(wrapped)}; ec=$?; rm -rf ${shellQuote(remoteDir)}; exit $ec`;
|
|
2304
|
+
const { stdout, awaitExit } = await this.execStream(service, ["sh", "-c", runner], {
|
|
2305
|
+
...opts,
|
|
2306
|
+
timeoutMs: remaining()
|
|
2307
|
+
});
|
|
1675
2308
|
stdout.resume();
|
|
1676
|
-
return await awaitExit;
|
|
2309
|
+
return await withTimeout(awaitExit, remaining() + KILL_GRACE_MS, ceiling());
|
|
1677
2310
|
} finally {
|
|
1678
2311
|
await fs.rm(localDir, { recursive: true, force: true }).catch(() => {
|
|
1679
2312
|
});
|
|
@@ -1690,14 +2323,14 @@ var BareBackupExecutor = class {
|
|
|
1690
2323
|
return true;
|
|
1691
2324
|
}
|
|
1692
2325
|
};
|
|
1693
|
-
function
|
|
1694
|
-
const envPrefix = opts?.env ? Object.entries(opts.env).map(([k2, v2]) => `${k2}=${
|
|
1695
|
-
const cwdPrefix = opts?.cwd ? `cd ${
|
|
1696
|
-
const inner = cmd.length >= 3 && cmd[0] === "sh" && cmd[1] === "-c" ? cmd.slice(2).join(" ") : cmd.map((a2) =>
|
|
1697
|
-
return
|
|
2326
|
+
function shellScript(cmd, opts) {
|
|
2327
|
+
const envPrefix = opts?.env ? Object.entries(opts.env).map(([k2, v2]) => `${k2}=${shellQuote(v2)}`).join(" ") + " " : "";
|
|
2328
|
+
const cwdPrefix = opts?.cwd ? `cd ${shellQuote(opts.cwd)} && ` : "";
|
|
2329
|
+
const inner = cmd.length >= 3 && cmd[0] === "sh" && cmd[1] === "-c" ? cmd.slice(2).join(" ") : cmd.map((a2) => shellQuote(a2)).join(" ");
|
|
2330
|
+
return `${cwdPrefix}${envPrefix}${inner}`;
|
|
1698
2331
|
}
|
|
1699
|
-
function
|
|
1700
|
-
return `
|
|
2332
|
+
function composeShell(cmd, opts) {
|
|
2333
|
+
return `sh -c ${shellQuote(shellScript(cmd, opts))}`;
|
|
1701
2334
|
}
|
|
1702
2335
|
registerExecutor("bare", (runtime) => {
|
|
1703
2336
|
if (!(runtime instanceof BareRuntime)) {
|
|
@@ -1723,11 +2356,19 @@ var LocalDestinationImpl = class {
|
|
|
1723
2356
|
kind = "local";
|
|
1724
2357
|
capabilities = CAPS;
|
|
1725
2358
|
root;
|
|
2359
|
+
/**
|
|
2360
|
+
* Where artifacts written BEFORE `pathPrefix` moved out of the key builder live, or
|
|
2361
|
+
* null when this destination has no prefix and the two are the same place.
|
|
2362
|
+
*/
|
|
2363
|
+
legacyRoot;
|
|
1726
2364
|
constructor(row) {
|
|
1727
2365
|
if (!row.endpoint) {
|
|
1728
2366
|
throw new Error(`LocalDestination "${row.name}" has no endpoint (root path) configured`);
|
|
1729
2367
|
}
|
|
1730
|
-
|
|
2368
|
+
const base = resolve(row.endpoint);
|
|
2369
|
+
const prefix = (row.pathPrefix ?? "").replace(/^\/+|\/+$/g, "");
|
|
2370
|
+
this.root = prefix ? resolve(base, prefix) : base;
|
|
2371
|
+
this.legacyRoot = prefix ? base : null;
|
|
1731
2372
|
}
|
|
1732
2373
|
resolveKey(key) {
|
|
1733
2374
|
const normalized = normalize(key);
|
|
@@ -1736,6 +2377,38 @@ var LocalDestinationImpl = class {
|
|
|
1736
2377
|
}
|
|
1737
2378
|
return join2(this.root, normalized);
|
|
1738
2379
|
}
|
|
2380
|
+
/**
|
|
2381
|
+
* The path to READ `key` from, preferring the current layout and falling back to the
|
|
2382
|
+
* pre-prefix one.
|
|
2383
|
+
*
|
|
2384
|
+
* A run captured before this change recorded a key that already CONTAINS the prefix
|
|
2385
|
+
* (the key builder put it there), and its bytes sit at `endpoint/<prefix>/<key…>` —
|
|
2386
|
+
* which is `legacyRoot + key`, not `root + key`. Resolving it against the new root
|
|
2387
|
+
* would look for `endpoint/<prefix>/<prefix>/…` and find nothing, so an existing
|
|
2388
|
+
* restore point would quietly become unrestorable.
|
|
2389
|
+
*
|
|
2390
|
+
* Deliberately an existence check rather than a "does the key already start with the
|
|
2391
|
+
* prefix" string test: the key builder's first literal segment is `openship`, so a
|
|
2392
|
+
* destination whose prefix is also `openship` would make that test ambiguous. The
|
|
2393
|
+
* filesystem is not ambiguous. Reads only — `put` always writes the current layout,
|
|
2394
|
+
* so this shim shrinks to nothing as old runs age out of retention.
|
|
2395
|
+
*/
|
|
2396
|
+
async resolveKeyForRead(key) {
|
|
2397
|
+
const current = this.resolveKey(key);
|
|
2398
|
+
if (this.legacyRoot === null) return current;
|
|
2399
|
+
try {
|
|
2400
|
+
await fs2.access(current);
|
|
2401
|
+
return current;
|
|
2402
|
+
} catch {
|
|
2403
|
+
const legacy = join2(this.legacyRoot, normalize(key));
|
|
2404
|
+
try {
|
|
2405
|
+
await fs2.access(legacy);
|
|
2406
|
+
return legacy;
|
|
2407
|
+
} catch {
|
|
2408
|
+
return current;
|
|
2409
|
+
}
|
|
2410
|
+
}
|
|
2411
|
+
}
|
|
1739
2412
|
async preflight() {
|
|
1740
2413
|
try {
|
|
1741
2414
|
await fs2.mkdir(this.root, { recursive: true });
|
|
@@ -1782,12 +2455,11 @@ var LocalDestinationImpl = class {
|
|
|
1782
2455
|
return { bytesWritten, etag: digest };
|
|
1783
2456
|
}
|
|
1784
2457
|
async get(key) {
|
|
1785
|
-
|
|
1786
|
-
return createReadStream(path);
|
|
2458
|
+
return createReadStream(await this.resolveKeyForRead(key));
|
|
1787
2459
|
}
|
|
1788
2460
|
async head(key) {
|
|
1789
2461
|
try {
|
|
1790
|
-
const stat = await fs2.stat(this.
|
|
2462
|
+
const stat = await fs2.stat(await this.resolveKeyForRead(key));
|
|
1791
2463
|
return {
|
|
1792
2464
|
sizeBytes: stat.size,
|
|
1793
2465
|
uploadedAt: stat.mtime
|
|
@@ -1799,7 +2471,7 @@ var LocalDestinationImpl = class {
|
|
|
1799
2471
|
}
|
|
1800
2472
|
}
|
|
1801
2473
|
async list(prefix, opts) {
|
|
1802
|
-
const root2 = this.
|
|
2474
|
+
const root2 = await this.resolveKeyForRead(prefix);
|
|
1803
2475
|
const entries = [];
|
|
1804
2476
|
const limit = opts?.limit ?? 1e3;
|
|
1805
2477
|
async function walk(dir, relBase) {
|
|
@@ -1832,7 +2504,7 @@ var LocalDestinationImpl = class {
|
|
|
1832
2504
|
}
|
|
1833
2505
|
async delete(key) {
|
|
1834
2506
|
try {
|
|
1835
|
-
await fs2.unlink(this.
|
|
2507
|
+
await fs2.unlink(await this.resolveKeyForRead(key));
|
|
1836
2508
|
} catch (err) {
|
|
1837
2509
|
const e2 = err;
|
|
1838
2510
|
if (e2?.code !== "ENOENT") throw err;
|
|
@@ -12563,7 +13235,22 @@ var CAPS2 = /* @__PURE__ */ new Set([
|
|
|
12563
13235
|
]);
|
|
12564
13236
|
var PRESIGN_TTL_DEFAULT = 60 * 60;
|
|
12565
13237
|
var MULTIPART_PART_SIZE = 16 * 1024 * 1024;
|
|
12566
|
-
var
|
|
13238
|
+
var MULTIPART_MEMORY_BUDGET = 64 * 1024 * 1024;
|
|
13239
|
+
var MULTIPART_MAX_PARTS = 9e3;
|
|
13240
|
+
var UNKNOWN_SIZE_PART_SIZE = 32 * 1024 * 1024;
|
|
13241
|
+
function partSizeFor(sizeBytes) {
|
|
13242
|
+
if (!sizeBytes || sizeBytes <= 0) return UNKNOWN_SIZE_PART_SIZE;
|
|
13243
|
+
return Math.max(MULTIPART_PART_SIZE, Math.ceil(sizeBytes / MULTIPART_MAX_PARTS));
|
|
13244
|
+
}
|
|
13245
|
+
function queueSizeFor(partSize) {
|
|
13246
|
+
return Math.max(1, Math.floor(MULTIPART_MEMORY_BUDGET / partSize));
|
|
13247
|
+
}
|
|
13248
|
+
function partLimitCeiling(partSize) {
|
|
13249
|
+
return partSize * MULTIPART_MAX_PARTS;
|
|
13250
|
+
}
|
|
13251
|
+
function gib(bytes) {
|
|
13252
|
+
return Math.round(bytes / (1024 * 1024 * 1024));
|
|
13253
|
+
}
|
|
12567
13254
|
var S3DestinationImpl = class {
|
|
12568
13255
|
kind = "s3_compatible";
|
|
12569
13256
|
capabilities = CAPS2;
|
|
@@ -12616,6 +13303,7 @@ var S3DestinationImpl = class {
|
|
|
12616
13303
|
}
|
|
12617
13304
|
async put(key, body, opts) {
|
|
12618
13305
|
const fullKey = this.fullKey(key);
|
|
13306
|
+
const partSize = partSizeFor(opts.size);
|
|
12619
13307
|
const upload = new Upload({
|
|
12620
13308
|
client: this.client,
|
|
12621
13309
|
params: {
|
|
@@ -12625,15 +13313,34 @@ var S3DestinationImpl = class {
|
|
|
12625
13313
|
ContentType: opts.contentType ?? "application/octet-stream",
|
|
12626
13314
|
Metadata: opts.metadata
|
|
12627
13315
|
},
|
|
12628
|
-
|
|
12629
|
-
|
|
13316
|
+
// Derived, not fixed — see partSizeFor. An artifact whose size we know never trips
|
|
13317
|
+
// S3's 10,000-part limit; one we don't (a DB dump has no sizeHint) gets
|
|
13318
|
+
// UNKNOWN_SIZE_PART_SIZE, whose ceiling is enforced below rather than discovered
|
|
13319
|
+
// at CompleteMultipartUpload.
|
|
13320
|
+
partSize,
|
|
13321
|
+
queueSize: queueSizeFor(partSize),
|
|
12630
13322
|
leavePartsOnError: false
|
|
12631
13323
|
});
|
|
13324
|
+
const ceiling = partLimitCeiling(partSize);
|
|
12632
13325
|
let bytesWritten = 0;
|
|
13326
|
+
let ceilingError = null;
|
|
12633
13327
|
upload.on("httpUploadProgress", (progress) => {
|
|
12634
13328
|
if (typeof progress.loaded === "number") bytesWritten = progress.loaded;
|
|
13329
|
+
if (!ceilingError && bytesWritten > ceiling) {
|
|
13330
|
+
ceilingError = new Error(
|
|
13331
|
+
`Artifact exceeds what one S3 multipart upload can hold at this part size: ${gib(bytesWritten)} GiB transferred, ceiling ${gib(ceiling)} GiB (${MULTIPART_MAX_PARTS} parts \xD7 ${Math.round(partSize / (1024 * 1024))} MiB). Aborted early rather than failing at CompleteMultipartUpload after transferring everything. A source whose size is known scales its part size automatically; a streamed database dump cannot, so this artifact needs to be split or sent to a destination without a part limit.`
|
|
13332
|
+
);
|
|
13333
|
+
void upload.abort().catch(() => {
|
|
13334
|
+
});
|
|
13335
|
+
}
|
|
12635
13336
|
});
|
|
12636
|
-
|
|
13337
|
+
let result;
|
|
13338
|
+
try {
|
|
13339
|
+
result = await upload.done();
|
|
13340
|
+
} catch (err) {
|
|
13341
|
+
if (ceilingError) throw ceilingError;
|
|
13342
|
+
throw err;
|
|
13343
|
+
}
|
|
12637
13344
|
return {
|
|
12638
13345
|
bytesWritten,
|
|
12639
13346
|
etag: result.ETag?.replace(/"/g, "") ?? void 0
|
|
@@ -12768,6 +13475,7 @@ var CAPS3 = /* @__PURE__ */ new Set([
|
|
|
12768
13475
|
]);
|
|
12769
13476
|
var UPLOAD_STALL_TIMEOUT_MS = 10 * 60 * 1e3;
|
|
12770
13477
|
var UPLOAD_STALL_PROBE_MS = 30 * 1e3;
|
|
13478
|
+
var UPLOAD_BUFFER_BYTES = 8 * 1024 * 1024;
|
|
12771
13479
|
var SftpDestinationImpl = class {
|
|
12772
13480
|
kind;
|
|
12773
13481
|
capabilities = CAPS3;
|
|
@@ -12896,14 +13604,18 @@ var SftpDestinationImpl = class {
|
|
|
12896
13604
|
await this.withSftp(async (sftp) => {
|
|
12897
13605
|
await this.ensureDir(sftp, posix2.dirname(target));
|
|
12898
13606
|
await new Promise((resolve2, reject) => {
|
|
12899
|
-
const ws = sftp.createWriteStream(tmp);
|
|
13607
|
+
const ws = sftp.createWriteStream(tmp, { highWaterMark: UPLOAD_BUFFER_BYTES });
|
|
12900
13608
|
let lastProgressAt = Date.now();
|
|
12901
13609
|
let settled = false;
|
|
12902
13610
|
let watchdog;
|
|
13611
|
+
const onSettled = [];
|
|
12903
13612
|
const finish = (err) => {
|
|
12904
13613
|
if (settled) return;
|
|
12905
13614
|
settled = true;
|
|
12906
13615
|
if (watchdog) clearInterval(watchdog);
|
|
13616
|
+
for (const stop of onSettled) stop();
|
|
13617
|
+
const acked = ws.bytesWritten ?? 0;
|
|
13618
|
+
if (acked > bytesWritten) bytesWritten = acked;
|
|
12907
13619
|
if (err) {
|
|
12908
13620
|
ws.destroy();
|
|
12909
13621
|
body.destroy();
|
|
@@ -12923,10 +13635,14 @@ var SftpDestinationImpl = class {
|
|
|
12923
13635
|
watchdog.unref?.();
|
|
12924
13636
|
ws.on("error", (err) => finish(err));
|
|
12925
13637
|
ws.on("close", () => finish());
|
|
12926
|
-
|
|
12927
|
-
bytesWritten
|
|
13638
|
+
const trackProgress = setInterval(() => {
|
|
13639
|
+
const acked = ws.bytesWritten ?? 0;
|
|
13640
|
+
if (acked <= bytesWritten) return;
|
|
13641
|
+
bytesWritten = acked;
|
|
12928
13642
|
lastProgressAt = Date.now();
|
|
12929
|
-
});
|
|
13643
|
+
}, 1e3);
|
|
13644
|
+
trackProgress.unref?.();
|
|
13645
|
+
onSettled.push(() => clearInterval(trackProgress));
|
|
12930
13646
|
body.on("error", (err) => finish(err));
|
|
12931
13647
|
body.pipe(ws);
|
|
12932
13648
|
});
|