@magland/mochi 0.3.1 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -6
- package/dist/agefile.js +47 -0
- package/dist/agescript.js +343 -0
- package/dist/api/admin.js +66 -28
- package/dist/api/auth.js +3 -2
- package/dist/api/backup.js +227 -192
- package/dist/api/contents.js +46 -16
- package/dist/api/repos.js +60 -4
- package/dist/api/write.js +138 -8
- package/dist/api.js +34 -1
- package/dist/browse.js +171 -33
- package/dist/build-info.json +2 -2
- package/dist/ci/api.js +138 -4
- package/dist/ci/artifacts.js +82 -2
- package/dist/ci/engine.js +30 -17
- package/dist/ci/expr.js +14 -0
- package/dist/ci/runners.js +60 -1
- package/dist/ci/runs.js +21 -3
- package/dist/ci/trigger.js +6 -4
- package/dist/ci/views.js +24 -3
- package/dist/ci/web.js +48 -0
- package/dist/cli/admin-cmd.js +40 -3
- package/dist/cli/backup-cmd.js +166 -120
- package/dist/cli/gitrun.js +50 -0
- package/dist/cli/pr-cmd.js +155 -0
- package/dist/cli/repo-cmd.js +134 -6
- package/dist/cli/sync-cmd.js +128 -0
- package/dist/cli/target.js +7 -3
- package/dist/config.js +10 -0
- package/dist/credentials.js +9 -8
- package/dist/deploy-cli.js +214 -109
- package/dist/deploy-runner-cli.js +125 -49
- package/dist/domains.js +259 -0
- package/dist/egress.js +23 -6
- package/dist/forms.js +703 -32
- package/dist/git.js +124 -13
- package/dist/githttp.js +80 -23
- package/dist/githubauth.js +185 -0
- package/dist/icons.js +14 -0
- package/dist/import-cli.js +40 -25
- package/dist/index.js +110 -4
- package/dist/issueweb.js +10 -6
- package/dist/lfs.js +9 -2
- package/dist/limit.js +2 -2
- package/dist/logincodes.js +31 -0
- package/dist/maintenance.js +297 -0
- package/dist/markdown.js +193 -3
- package/dist/mdedit.js +29 -0
- package/dist/migrate.js +48 -0
- package/dist/naming.js +25 -0
- package/dist/onetime.js +112 -0
- package/dist/ops.js +293 -16
- package/dist/pagescript.js +355 -24
- package/dist/perms.js +1 -0
- package/dist/profile.js +2 -0
- package/dist/pulls.js +21 -1
- package/dist/pullweb.js +25 -6
- package/dist/releases.js +10 -1
- package/dist/runner/actions.js +12 -1
- package/dist/runner/client.js +40 -2
- package/dist/runner/docker.js +40 -28
- package/dist/runner/download.js +77 -0
- package/dist/runner/externals.js +12 -1
- package/dist/runner/job.js +26 -11
- package/dist/runner/steps.js +42 -7
- package/dist/runner-cli.js +89 -0
- package/dist/scan.js +58 -2
- package/dist/server.js +107 -26
- package/dist/session.js +82 -40
- package/dist/site.js +110 -27
- package/dist/sitepublish.js +177 -0
- package/dist/sitesettings.js +420 -0
- package/dist/siteshost.js +116 -28
- package/dist/source.js +20 -0
- package/dist/style.js +162 -3
- package/dist/topics.js +165 -0
- package/dist/vault.js +399 -6
- package/dist/vendor-age.js +15 -0
- package/dist/views.js +242 -57
- package/dist/web.js +11 -1
- package/dist/webauthn.js +347 -0
- package/dist/webops.js +1263 -33
- package/package.json +4 -4
package/dist/cli/backup-cmd.js
CHANGED
|
@@ -33,16 +33,18 @@ var __importStar = (this && this.__importStar) || (function () {
|
|
|
33
33
|
};
|
|
34
34
|
})();
|
|
35
35
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.backupCommands = void 0;
|
|
36
|
+
exports.backupCommands = exports.MOCHI_BACKUP = void 0;
|
|
37
37
|
exports.backupsIndexPath = backupsIndexPath;
|
|
38
38
|
exports.knownBackups = knownBackups;
|
|
39
39
|
exports.backupLineFor = backupLineFor;
|
|
40
|
+
exports.makeBackupCommands = makeBackupCommands;
|
|
40
41
|
const child_process_1 = require("child_process");
|
|
41
42
|
const crypto = __importStar(require("crypto"));
|
|
42
43
|
const fs = __importStar(require("fs"));
|
|
43
44
|
const os = __importStar(require("os"));
|
|
44
45
|
const path = __importStar(require("path"));
|
|
45
46
|
const atomic_1 = require("../atomic");
|
|
47
|
+
const naming_1 = require("../naming");
|
|
46
48
|
const scan_1 = require("../scan");
|
|
47
49
|
const exit_1 = require("./exit");
|
|
48
50
|
const output_1 = require("./output");
|
|
@@ -275,8 +277,8 @@ function isVaultRelative(p) {
|
|
|
275
277
|
}
|
|
276
278
|
/** How a refused path is reported, in one place since two kinds of line carry one. */
|
|
277
279
|
function refusedPath(p) {
|
|
278
|
-
return (`The
|
|
279
|
-
|
|
280
|
+
return (`The ${naming_1.naming.rootNoun} named a path this backup will not write: ${JSON.stringify(p)}. ` +
|
|
281
|
+
`A manifest path must be relative to the ${naming_1.naming.rootNoun} and must not climb out of it, so nothing was copied.`);
|
|
280
282
|
}
|
|
281
283
|
async function fetchManifest(target, exclude, hash) {
|
|
282
284
|
const query = [];
|
|
@@ -308,8 +310,8 @@ async function fetchManifest(target, exclude, hash) {
|
|
|
308
310
|
// URL and the token, neither of which is the problem.
|
|
309
311
|
if (resp.status === 404) {
|
|
310
312
|
message =
|
|
311
|
-
`${target.host} has no /api/backup/manifest route, so it is running a
|
|
312
|
-
|
|
313
|
+
`${target.host} has no /api/backup/manifest route, so it is running a ${naming_1.naming.product} older than this ` +
|
|
314
|
+
`command. Deploy the ${naming_1.naming.rootNoun} again from a version that has it, then run this.`;
|
|
313
315
|
}
|
|
314
316
|
// The same status-to-code mapping every other command uses, so that a
|
|
315
317
|
// caller branching on the exit code does not have to learn a second table.
|
|
@@ -350,13 +352,13 @@ async function fetchManifest(target, exclude, hash) {
|
|
|
350
352
|
ended = true;
|
|
351
353
|
}
|
|
352
354
|
else if (kind === 'error') {
|
|
353
|
-
throw new exit_1.CliError(`The
|
|
355
|
+
throw new exit_1.CliError(`The ${naming_1.naming.rootNoun} could not finish the manifest: ${String(line.error)}`);
|
|
354
356
|
}
|
|
355
357
|
}
|
|
356
358
|
// The end line is what says the walk completed. Acting on a truncated
|
|
357
359
|
// manifest would delete every path the vault did not get around to listing.
|
|
358
360
|
if (!ended) {
|
|
359
|
-
throw new exit_1.CliError(
|
|
361
|
+
throw new exit_1.CliError(`The manifest ended early, so what the ${naming_1.naming.rootNoun} holds is not fully known. Nothing was deleted; try again.`);
|
|
360
362
|
}
|
|
361
363
|
return manifest;
|
|
362
364
|
}
|
|
@@ -428,7 +430,7 @@ class FrameReader {
|
|
|
428
430
|
let left = n;
|
|
429
431
|
while (left > 0) {
|
|
430
432
|
if (this.buf.length === 0 && !(await this.more())) {
|
|
431
|
-
throw new exit_1.CliError(
|
|
433
|
+
throw new exit_1.CliError(`The ${naming_1.naming.rootNoun} closed the connection part way through a file. Nothing was left half-written.`);
|
|
432
434
|
}
|
|
433
435
|
const take = Math.min(left, this.buf.length);
|
|
434
436
|
sink(this.buf.subarray(0, take));
|
|
@@ -530,6 +532,29 @@ function walkFiles(dir, rel = '', out = []) {
|
|
|
530
532
|
}
|
|
531
533
|
return out;
|
|
532
534
|
}
|
|
535
|
+
/**
|
|
536
|
+
* Every directory under dir, as paths relative to it, each after its parent.
|
|
537
|
+
* Listed apart from the files because an empty directory can matter: a bare
|
|
538
|
+
* repository whose refs are all packed has an empty refs/, and git does not
|
|
539
|
+
* take a directory without refs/ for a repository.
|
|
540
|
+
*/
|
|
541
|
+
function walkDirs(dir, rel = '', out = []) {
|
|
542
|
+
let entries;
|
|
543
|
+
try {
|
|
544
|
+
entries = fs.readdirSync(path.join(dir, rel), { withFileTypes: true });
|
|
545
|
+
}
|
|
546
|
+
catch {
|
|
547
|
+
return out;
|
|
548
|
+
}
|
|
549
|
+
for (const e of entries) {
|
|
550
|
+
if (!e.isDirectory())
|
|
551
|
+
continue;
|
|
552
|
+
const child = rel ? `${rel}/${e.name}` : e.name;
|
|
553
|
+
out.push(child);
|
|
554
|
+
walkDirs(dir, child, out);
|
|
555
|
+
}
|
|
556
|
+
return out;
|
|
557
|
+
}
|
|
533
558
|
/**
|
|
534
559
|
* A snapshot of current/ as a directory of hardlinks: one inode per file and no
|
|
535
560
|
* data, and still a servable vault.
|
|
@@ -549,11 +574,16 @@ function takeSnapshot(dir, quiet) {
|
|
|
549
574
|
if (fs.existsSync(to)) {
|
|
550
575
|
throw new exit_1.CliError(`A snapshot named ${name} is already there, so this second is left alone.`, exit_1.EXIT_CONFLICT);
|
|
551
576
|
}
|
|
577
|
+
const dirs = walkDirs(from);
|
|
552
578
|
const files = walkFiles(from);
|
|
553
579
|
const maxLinks = before + 1;
|
|
554
580
|
fs.mkdirSync(to, { recursive: true });
|
|
555
581
|
let linked = 0;
|
|
556
582
|
try {
|
|
583
|
+
// Every directory, not only those a file lands in, so that the snapshot has
|
|
584
|
+
// the empty ones too and its repositories are still repositories.
|
|
585
|
+
for (const rel of dirs)
|
|
586
|
+
fs.mkdirSync(path.join(to, rel), { recursive: true });
|
|
557
587
|
for (const rel of files) {
|
|
558
588
|
const src = path.join(from, rel);
|
|
559
589
|
const st = fs.lstatSync(src);
|
|
@@ -670,18 +700,58 @@ const RETENTION_OPTIONS = [
|
|
|
670
700
|
summary: `Monthly snapshots to keep (default ${DEFAULT_RETENTION.monthly})`,
|
|
671
701
|
},
|
|
672
702
|
];
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
703
|
+
exports.MOCHI_BACKUP = {
|
|
704
|
+
exclusions: [
|
|
705
|
+
{ category: 'runs', summary: 'Leave out workflow run history (<repo>.runs)' },
|
|
706
|
+
{ category: 'sites', summary: 'Leave out published sites (<repo>.site)' },
|
|
707
|
+
{ category: 'lfs', summary: 'Leave out LFS objects on the volume (<repo>.lfs)' },
|
|
708
|
+
{ category: 'secrets', summary: 'Leave out vault.json, runners.json, .secret, and .github-secret' },
|
|
709
|
+
],
|
|
710
|
+
repos: true,
|
|
711
|
+
description: `A vault is a directory, so a backup of one is a directory too, and this makes it
|
|
712
|
+
over HTTP: it needs no shell on the server, no flyctl, and no rsync at the far
|
|
713
|
+
end, so it works the same against a Fly app, a VPS, a Docker deployment, and
|
|
714
|
+
127.0.0.1:3000.
|
|
715
|
+
|
|
716
|
+
<dir>/current a servable vault. Restoring is: mochi serve <dir>/current
|
|
717
|
+
<dir>/snapshots hardlinked copies, each one also a servable vault
|
|
718
|
+
<dir>/backup.json which vault, what is left out, and how each run went
|
|
719
|
+
|
|
720
|
+
Repositories come across as mirrors, so a second run moves only the objects it
|
|
721
|
+
does not have and skips a repository nothing was pushed to. Everything beside
|
|
722
|
+
them - issues, pull requests, releases, sites, run history, LFS objects on the
|
|
723
|
+
volume, and the vault's state files - is compared by size and modification time
|
|
724
|
+
and fetched only where it differs.
|
|
725
|
+
|
|
726
|
+
The token needs to belong to a site admin, because the copy includes
|
|
727
|
+
vault.json. The vault URL, the exclusions, and the retention policy are recorded
|
|
728
|
+
in backup.json, so a cron entry is this command and a directory.
|
|
729
|
+
|
|
730
|
+
There is no vault-wide point-in-time image: the server holds no lock a client
|
|
731
|
+
could take, so a run is a walk of a live tree and can catch a mixed vintage.
|
|
732
|
+
Every individual file in a backup is one that really existed. See docs/backup.md.
|
|
733
|
+
|
|
734
|
+
Related: mochi backup list, verify, prune.`,
|
|
735
|
+
verifyDescription: `Runs git fsck --connectivity-only over every mirror, asks the vault for hashes,
|
|
736
|
+
and reports anything missing, extra, or different. Exits non-zero when there is
|
|
737
|
+
something to report, so it can be run from cron.`,
|
|
738
|
+
pruneDescription: `Grandfather-father-son: the newest snapshot of each of the last N days, weeks,
|
|
739
|
+
and months is kept and the rest are removed, evaluated in UTC. The newest
|
|
740
|
+
snapshot is always kept.
|
|
741
|
+
|
|
742
|
+
A snapshot pins the packfiles that were current when it was taken, so a repack
|
|
743
|
+
in a busy repository leaves the old pack on disk until the last snapshot
|
|
744
|
+
referring to it is pruned. This is what reclaims that space.`,
|
|
745
|
+
};
|
|
746
|
+
function excludeOptions(profile) {
|
|
747
|
+
return profile.exclusions.map((e) => ({ name: `no-${e.category}`, type: 'boolean', summary: e.summary }));
|
|
748
|
+
}
|
|
679
749
|
const QUIET_OPTION = { name: 'quiet', type: 'boolean', summary: 'Say nothing on success' };
|
|
680
750
|
/** The backup directory a command was given, made if it is not there yet. */
|
|
681
751
|
function backupDirectory(inv, create) {
|
|
682
752
|
const given = inv.args[0];
|
|
683
753
|
if (!given)
|
|
684
|
-
throw new exit_1.CliError(
|
|
754
|
+
throw new exit_1.CliError(`Which directory? Usage: ${naming_1.naming.product} backup <dir>`, exit_1.EXIT_USAGE);
|
|
685
755
|
const dir = path.resolve(given);
|
|
686
756
|
if (!fs.existsSync(dir)) {
|
|
687
757
|
if (!create)
|
|
@@ -697,7 +767,7 @@ function backupDirectory(inv, create) {
|
|
|
697
767
|
function existingBackup(inv) {
|
|
698
768
|
const dir = backupDirectory(inv, false);
|
|
699
769
|
if (!fs.existsSync(statePath(dir))) {
|
|
700
|
-
throw new exit_1.CliError(`${dir} holds no backup (no ${STATE_FILE}). Make one first:
|
|
770
|
+
throw new exit_1.CliError(`${dir} holds no backup (no ${STATE_FILE}). Make one first: ${naming_1.naming.product} backup ${inv.args[0]}`, exit_1.EXIT_USAGE);
|
|
701
771
|
}
|
|
702
772
|
return { dir, state: loadState(dir) };
|
|
703
773
|
}
|
|
@@ -707,16 +777,8 @@ function existingBackup(inv) {
|
|
|
707
777
|
* last run used, and naming any at all replaces the set, which is how a
|
|
708
778
|
* category can be put back.
|
|
709
779
|
*/
|
|
710
|
-
function exclusionsFor(inv, state) {
|
|
711
|
-
const given =
|
|
712
|
-
if (inv.bool('no-runs'))
|
|
713
|
-
given.push('runs');
|
|
714
|
-
if (inv.bool('no-sites'))
|
|
715
|
-
given.push('sites');
|
|
716
|
-
if (inv.bool('no-lfs'))
|
|
717
|
-
given.push('lfs');
|
|
718
|
-
if (inv.bool('no-secrets'))
|
|
719
|
-
given.push('secrets');
|
|
780
|
+
function exclusionsFor(inv, state, profile) {
|
|
781
|
+
const given = profile.exclusions.map((e) => e.category).filter((c) => inv.bool(`no-${c}`));
|
|
720
782
|
return given.length ? given : state.excluded;
|
|
721
783
|
}
|
|
722
784
|
function retentionFor(inv, state) {
|
|
@@ -884,14 +946,14 @@ async function fetchChunk(target, current, chunk, state, wantHashes) {
|
|
|
884
946
|
throw new exit_1.CliError(message, (0, exit_1.exitCodeForStatus)(resp.status));
|
|
885
947
|
}
|
|
886
948
|
if (!resp.body)
|
|
887
|
-
throw new exit_1.CliError(
|
|
949
|
+
throw new exit_1.CliError(`The ${naming_1.naming.rootNoun} answered a fetch with no body.`);
|
|
888
950
|
const reader = new FrameReader(resp.body);
|
|
889
951
|
let bytes = 0;
|
|
890
952
|
let missing = [];
|
|
891
953
|
for (;;) {
|
|
892
954
|
const line = await reader.line();
|
|
893
955
|
if (line === null)
|
|
894
|
-
throw new exit_1.CliError(
|
|
956
|
+
throw new exit_1.CliError(`The ${naming_1.naming.rootNoun} ended a fetch without saying it had finished.`);
|
|
895
957
|
const frame = JSON.parse(line);
|
|
896
958
|
if (frame.end) {
|
|
897
959
|
missing = frame.missing ?? [];
|
|
@@ -900,7 +962,7 @@ async function fetchChunk(target, current, chunk, state, wantHashes) {
|
|
|
900
962
|
const rel = frame.path;
|
|
901
963
|
const size = frame.size;
|
|
902
964
|
if (typeof rel !== 'string' || typeof size !== 'number' || !byPath.has(rel)) {
|
|
903
|
-
throw new exit_1.CliError(`The
|
|
965
|
+
throw new exit_1.CliError(`The ${naming_1.naming.rootNoun} sent a file this run did not ask for: ${String(rel)}`);
|
|
904
966
|
}
|
|
905
967
|
const wanted = byPath.get(rel);
|
|
906
968
|
const dest = path.join(current, ...rel.split('/'));
|
|
@@ -993,7 +1055,7 @@ function emptyDirs(dir, rel = '', out = []) {
|
|
|
993
1055
|
}
|
|
994
1056
|
return empty;
|
|
995
1057
|
}
|
|
996
|
-
async function syncCmd(inv) {
|
|
1058
|
+
async function syncCmd(inv, profile) {
|
|
997
1059
|
const json = (0, output_1.jsonMode)(inv);
|
|
998
1060
|
// Two kinds of silence, and they are not the same. --json puts one JSON value
|
|
999
1061
|
// on stdout, so the running commentary has to go, but a warning is a
|
|
@@ -1004,7 +1066,7 @@ async function syncCmd(inv) {
|
|
|
1004
1066
|
const dir = backupDirectory(inv, true);
|
|
1005
1067
|
const state = loadState(dir);
|
|
1006
1068
|
const target = await targetForBackup(inv, state);
|
|
1007
|
-
const exclude = exclusionsFor(inv, state);
|
|
1069
|
+
const exclude = exclusionsFor(inv, state, profile);
|
|
1008
1070
|
const retention = retentionFor(inv, state);
|
|
1009
1071
|
const checksum = inv.bool('checksum');
|
|
1010
1072
|
const current = path.join(dir, CURRENT);
|
|
@@ -1078,7 +1140,7 @@ async function syncCmd(inv) {
|
|
|
1078
1140
|
delete state.repos[known];
|
|
1079
1141
|
summary.repos.removed++;
|
|
1080
1142
|
if (!quiet)
|
|
1081
|
-
console.error(`Removed ${known}, which the
|
|
1143
|
+
console.error(`Removed ${known}, which the ${naming_1.naming.rootNoun} no longer holds`);
|
|
1082
1144
|
}
|
|
1083
1145
|
// Files.
|
|
1084
1146
|
const changed = [];
|
|
@@ -1094,7 +1156,7 @@ async function syncCmd(inv) {
|
|
|
1094
1156
|
for (const gone of r.missing) {
|
|
1095
1157
|
delete state.files[gone];
|
|
1096
1158
|
if (!quiet)
|
|
1097
|
-
console.error(`${gone} vanished from the
|
|
1159
|
+
console.error(`${gone} vanished from the ${naming_1.naming.rootNoun} while this run was reading it`);
|
|
1098
1160
|
}
|
|
1099
1161
|
}
|
|
1100
1162
|
// A mirror's config is one of the files the manifest names, so the copy just
|
|
@@ -1204,15 +1266,17 @@ async function syncCmd(inv) {
|
|
|
1204
1266
|
if (quiet)
|
|
1205
1267
|
return;
|
|
1206
1268
|
const r = summary.repos;
|
|
1207
|
-
|
|
1208
|
-
(r.
|
|
1269
|
+
if (profile.repos) {
|
|
1270
|
+
console.log(`${r.total} repositories: ${r.cloned} cloned, ${r.fetched} fetched, ${r.skipped} unchanged` +
|
|
1271
|
+
(r.removed ? `, ${r.removed} removed` : ''));
|
|
1272
|
+
}
|
|
1209
1273
|
console.log(`${summary.files.total} files: ${summary.files.fetched} fetched (${human(summary.files.bytes)})` +
|
|
1210
1274
|
(summary.files.removed ? `, ${summary.files.removed} removed` : ''));
|
|
1211
1275
|
if (summary.snapshot)
|
|
1212
1276
|
console.log(`Snapshot ${summary.snapshot}`);
|
|
1213
1277
|
console.log('');
|
|
1214
|
-
console.log(`Serve this backup to look at it, or to stand the
|
|
1215
|
-
console.log(`
|
|
1278
|
+
console.log(`Serve this backup to look at it, or to stand the ${naming_1.naming.rootNoun} back up:`);
|
|
1279
|
+
console.log(` ${naming_1.naming.product} serve ${path.join(dir, CURRENT)}`);
|
|
1216
1280
|
}
|
|
1217
1281
|
// ---- list, prune, verify ----
|
|
1218
1282
|
function listCmd(inv) {
|
|
@@ -1236,7 +1300,7 @@ function listCmd(inv) {
|
|
|
1236
1300
|
return;
|
|
1237
1301
|
}
|
|
1238
1302
|
console.log(`${dir}`);
|
|
1239
|
-
console.log(`
|
|
1303
|
+
console.log(` ${naming_1.naming.rootNoun.padEnd(10)} ${state.host || '(unknown)'}`);
|
|
1240
1304
|
console.log(` current ${human(apparentSize(path.join(dir, CURRENT)))} apparent`);
|
|
1241
1305
|
console.log(` excluded ${state.excluded.length ? state.excluded.join(', ') : 'nothing'}`);
|
|
1242
1306
|
console.log(` retention ${state.retention.daily} daily, ${state.retention.weekly} weekly, ${state.retention.monthly} monthly`);
|
|
@@ -1246,7 +1310,7 @@ function listCmd(inv) {
|
|
|
1246
1310
|
}
|
|
1247
1311
|
console.log('');
|
|
1248
1312
|
if (snapshots.length === 0) {
|
|
1249
|
-
console.log(
|
|
1313
|
+
console.log(`No snapshots. '${naming_1.naming.product} backup <dir> --snapshot' takes one after a sync.`);
|
|
1250
1314
|
return;
|
|
1251
1315
|
}
|
|
1252
1316
|
// Apparent size rather than disk use: a snapshot is hardlinked, so what it
|
|
@@ -1277,7 +1341,7 @@ function pruneCmd(inv) {
|
|
|
1277
1341
|
console.log(dropped.length ? `Pruned ${dropped.length}: ${dropped.join(', ')}` : 'Nothing to prune under this retention.');
|
|
1278
1342
|
console.log(`${kept.length} snapshot${kept.length === 1 ? '' : 's'} kept.`);
|
|
1279
1343
|
}
|
|
1280
|
-
async function verifyCmd(inv) {
|
|
1344
|
+
async function verifyCmd(inv, profile) {
|
|
1281
1345
|
const { dir, state } = existingBackup(inv);
|
|
1282
1346
|
const json = (0, output_1.jsonMode)(inv);
|
|
1283
1347
|
const quiet = inv.bool('quiet') || json.enabled;
|
|
@@ -1304,7 +1368,7 @@ async function verifyCmd(inv) {
|
|
|
1304
1368
|
// The files, against hashes the vault computes now. This is the part a
|
|
1305
1369
|
// size-and-mtime sync cannot check on its own.
|
|
1306
1370
|
if (!quiet)
|
|
1307
|
-
console.error(
|
|
1371
|
+
console.error(`Asking the ${naming_1.naming.rootNoun} for hashes`);
|
|
1308
1372
|
const manifest = await fetchManifest(target, state.excluded, true);
|
|
1309
1373
|
for (const f of manifest.files.values()) {
|
|
1310
1374
|
const dest = path.join(current, ...f.path.split('/'));
|
|
@@ -1317,11 +1381,11 @@ async function verifyCmd(inv) {
|
|
|
1317
1381
|
continue;
|
|
1318
1382
|
}
|
|
1319
1383
|
if (st.size !== f.size) {
|
|
1320
|
-
problems.push({ path: f.path, problem: `size ${st.size}, the
|
|
1384
|
+
problems.push({ path: f.path, problem: `size ${st.size}, the ${naming_1.naming.rootNoun} has ${f.size}` });
|
|
1321
1385
|
continue;
|
|
1322
1386
|
}
|
|
1323
1387
|
if (f.sha256 && sha256Of(dest) !== f.sha256) {
|
|
1324
|
-
problems.push({ path: f.path, problem:
|
|
1388
|
+
problems.push({ path: f.path, problem: `contents differ from the ${naming_1.naming.rootNoun}` });
|
|
1325
1389
|
}
|
|
1326
1390
|
}
|
|
1327
1391
|
const mirrors = repos.map((p) => p.split('/').join(path.sep));
|
|
@@ -1331,11 +1395,11 @@ async function verifyCmd(inv) {
|
|
|
1331
1395
|
if (insideMirror(asPath))
|
|
1332
1396
|
continue;
|
|
1333
1397
|
if (!manifest.files.has(rel))
|
|
1334
|
-
problems.push({ path: rel, problem:
|
|
1398
|
+
problems.push({ path: rel, problem: `in the backup, not in the ${naming_1.naming.rootNoun}` });
|
|
1335
1399
|
}
|
|
1336
1400
|
for (const entry of manifest.repos) {
|
|
1337
1401
|
if (!state.repos[entry.path])
|
|
1338
|
-
problems.push({ path: entry.path, problem:
|
|
1402
|
+
problems.push({ path: entry.path, problem: `in the ${naming_1.naming.rootNoun}, not in the backup` });
|
|
1339
1403
|
}
|
|
1340
1404
|
// The hardlink invariant the snapshots rest on. A file with more links than
|
|
1341
1405
|
// current/ plus the snapshots can account for is shared with something
|
|
@@ -1356,7 +1420,9 @@ async function verifyCmd(inv) {
|
|
|
1356
1420
|
}
|
|
1357
1421
|
else if (problems.length === 0) {
|
|
1358
1422
|
if (!inv.bool('quiet')) {
|
|
1359
|
-
console.log(
|
|
1423
|
+
console.log(profile.repos
|
|
1424
|
+
? `${repos.length} mirrors and ${manifest.files.size} files check out against ${target.host}.`
|
|
1425
|
+
: `${manifest.files.size} files check out against ${target.host}.`);
|
|
1360
1426
|
}
|
|
1361
1427
|
}
|
|
1362
1428
|
else {
|
|
@@ -1378,7 +1444,7 @@ async function verifyCmd(inv) {
|
|
|
1378
1444
|
*/
|
|
1379
1445
|
function backupsIndexPath() {
|
|
1380
1446
|
const base = process.env.XDG_CONFIG_HOME ?? path.join(os.homedir(), '.config');
|
|
1381
|
-
return path.join(base,
|
|
1447
|
+
return path.join(base, naming_1.naming.configDirName, 'backups.json');
|
|
1382
1448
|
}
|
|
1383
1449
|
function knownBackups() {
|
|
1384
1450
|
try {
|
|
@@ -1424,77 +1490,57 @@ function backupLineFor(hosts) {
|
|
|
1424
1490
|
}
|
|
1425
1491
|
// ---- the commands ----
|
|
1426
1492
|
const COMMON = [output_1.JSON_OPTION, QUIET_OPTION, ...target_1.TARGET_OPTIONS];
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1433
|
-
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
Every individual file in a backup is one that really existed. See docs/backup.md.
|
|
1453
|
-
|
|
1454
|
-
Related: mochi backup list, verify, prune.`,
|
|
1455
|
-
args: [{ name: 'dir', required: true }],
|
|
1456
|
-
options: [
|
|
1457
|
-
{ name: 'snapshot', type: 'boolean', summary: 'Take a snapshot after a successful sync, then prune' },
|
|
1458
|
-
...RETENTION_OPTIONS,
|
|
1459
|
-
...EXCLUDE_OPTIONS,
|
|
1460
|
-
{ name: 'checksum', type: 'boolean', summary: 'Compare hashes rather than size and modification time' },
|
|
1461
|
-
...COMMON,
|
|
1462
|
-
],
|
|
1463
|
-
async run(inv) {
|
|
1464
|
-
await syncCmd(inv);
|
|
1465
|
-
const dir = path.resolve(inv.args[0]);
|
|
1466
|
-
rememberBackup(dir, loadState(dir).host);
|
|
1493
|
+
/**
|
|
1494
|
+
* The backup commands for one application. Mochi's are `backupCommands`
|
|
1495
|
+
* below; a sibling application built on these modules (dango) passes its own
|
|
1496
|
+
* profile and gets the same four commands with its own exclusions and help.
|
|
1497
|
+
*/
|
|
1498
|
+
function makeBackupCommands(profile) {
|
|
1499
|
+
const noun = naming_1.naming.rootNoun;
|
|
1500
|
+
return [
|
|
1501
|
+
{
|
|
1502
|
+
path: ['backup'],
|
|
1503
|
+
summary: `Copy a whole ${noun} to a directory on this machine, incrementally`,
|
|
1504
|
+
description: profile.description,
|
|
1505
|
+
args: [{ name: 'dir', required: true }],
|
|
1506
|
+
options: [
|
|
1507
|
+
{ name: 'snapshot', type: 'boolean', summary: 'Take a snapshot after a successful sync, then prune' },
|
|
1508
|
+
...RETENTION_OPTIONS,
|
|
1509
|
+
...excludeOptions(profile),
|
|
1510
|
+
{ name: 'checksum', type: 'boolean', summary: 'Compare hashes rather than size and modification time' },
|
|
1511
|
+
...COMMON,
|
|
1512
|
+
],
|
|
1513
|
+
async run(inv) {
|
|
1514
|
+
await syncCmd(inv, profile);
|
|
1515
|
+
const dir = path.resolve(inv.args[0]);
|
|
1516
|
+
rememberBackup(dir, loadState(dir).host);
|
|
1517
|
+
},
|
|
1467
1518
|
},
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
args: [{ name: 'dir', required: true }],
|
|
1497
|
-
options: [...RETENTION_OPTIONS, output_1.JSON_OPTION, QUIET_OPTION],
|
|
1498
|
-
run: pruneCmd,
|
|
1499
|
-
},
|
|
1500
|
-
];
|
|
1519
|
+
{
|
|
1520
|
+
path: ['backup', 'list'],
|
|
1521
|
+
summary: "Show a backup's snapshots, and how the last run went",
|
|
1522
|
+
args: [{ name: 'dir', required: true }],
|
|
1523
|
+
options: [output_1.JSON_OPTION],
|
|
1524
|
+
run: listCmd,
|
|
1525
|
+
},
|
|
1526
|
+
{
|
|
1527
|
+
path: ['backup', 'verify'],
|
|
1528
|
+
summary: profile.repos
|
|
1529
|
+
? `Check a backup against the ${noun}, and its mirrors against git`
|
|
1530
|
+
: `Check a backup against the ${noun}`,
|
|
1531
|
+
description: profile.verifyDescription,
|
|
1532
|
+
args: [{ name: 'dir', required: true }],
|
|
1533
|
+
options: [...COMMON],
|
|
1534
|
+
run: (inv) => verifyCmd(inv, profile),
|
|
1535
|
+
},
|
|
1536
|
+
{
|
|
1537
|
+
path: ['backup', 'prune'],
|
|
1538
|
+
summary: 'Apply the retention policy to the snapshots, without syncing',
|
|
1539
|
+
description: profile.pruneDescription,
|
|
1540
|
+
args: [{ name: 'dir', required: true }],
|
|
1541
|
+
options: [...RETENTION_OPTIONS, output_1.JSON_OPTION, QUIET_OPTION],
|
|
1542
|
+
run: pruneCmd,
|
|
1543
|
+
},
|
|
1544
|
+
];
|
|
1545
|
+
}
|
|
1546
|
+
exports.backupCommands = makeBackupCommands(exports.MOCHI_BACKUP);
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.run = run;
|
|
4
|
+
exports.capture = capture;
|
|
5
|
+
exports.mochiCredentialEnv = mochiCredentialEnv;
|
|
6
|
+
exports.mochiCredentialArgs = mochiCredentialArgs;
|
|
7
|
+
exports.ghCredentialArgs = ghCredentialArgs;
|
|
8
|
+
const child_process_1 = require("child_process");
|
|
9
|
+
// Client-side git, shared by the commands that move repositories between a
|
|
10
|
+
// vault and somewhere else: `mochi import` and `mochi fork` (clone from a
|
|
11
|
+
// source, push to the vault), `mochi sync` (fetch from an upstream, push to
|
|
12
|
+
// the vault), and `mochi pr export` (clone from the vault, push to GitHub).
|
|
13
|
+
// All of them run on the operator's machine, which is the point: the vault
|
|
14
|
+
// holds no credential for any system but itself.
|
|
15
|
+
/** Run a command with its output going straight to the terminal, so a long clone shows progress. */
|
|
16
|
+
function run(cmd, args, env) {
|
|
17
|
+
return new Promise((resolve, reject) => {
|
|
18
|
+
const child = (0, child_process_1.spawn)(cmd, args, { stdio: 'inherit', env: env ?? process.env });
|
|
19
|
+
child.on('error', (e) => reject(new Error(e.code === 'ENOENT' ? `${cmd} is not on PATH` : String(e.message))));
|
|
20
|
+
child.on('close', (code) => resolve(code ?? 1));
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
/** Run a command quietly and capture its stdout, or null when it fails to run or exits nonzero. */
|
|
24
|
+
function capture(cmd, args, env) {
|
|
25
|
+
return new Promise((resolve) => {
|
|
26
|
+
(0, child_process_1.execFile)(cmd, args, { maxBuffer: 4 * 1024 * 1024, env: env ?? process.env }, (err, stdout) => {
|
|
27
|
+
resolve(err ? null : String(stdout));
|
|
28
|
+
});
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
// The token is handed to git through a helper that reads it from the
|
|
32
|
+
// environment rather than through the command line, where every process on the
|
|
33
|
+
// machine could read it. The empty helper first clears the list, so the answer
|
|
34
|
+
// comes from here and nowhere else, and no prompt can appear.
|
|
35
|
+
function mochiCredentialEnv(username, token) {
|
|
36
|
+
return { ...process.env, MOCHI_USER: username, MOCHI_TOKEN: token, GIT_TERMINAL_PROMPT: '0' };
|
|
37
|
+
}
|
|
38
|
+
const MOCHI_CREDENTIAL_HELPER = `!f() { test "$1" = get && printf 'username=%s\\npassword=%s\\n' "$MOCHI_USER" "$MOCHI_TOKEN"; }; f`;
|
|
39
|
+
/** git options that make a vault URL authenticate from mochiCredentialEnv and nothing else. */
|
|
40
|
+
function mochiCredentialArgs() {
|
|
41
|
+
return ['-c', 'credential.helper=', '-c', `credential.helper=${MOCHI_CREDENTIAL_HELPER}`];
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* git options that make a github.com URL authenticate through `gh`'s own
|
|
45
|
+
* credential helper. `pr export` pushes with these, so having run
|
|
46
|
+
* `gh auth login` once is the only GitHub credential setup it asks for.
|
|
47
|
+
*/
|
|
48
|
+
function ghCredentialArgs() {
|
|
49
|
+
return ['-c', 'credential.helper=', '-c', 'credential.helper=!gh auth git-credential'];
|
|
50
|
+
}
|