@magland/mochi 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.md +7 -6
  2. package/dist/agefile.js +47 -0
  3. package/dist/agescript.js +343 -0
  4. package/dist/api/admin.js +66 -28
  5. package/dist/api/auth.js +3 -2
  6. package/dist/api/backup.js +227 -192
  7. package/dist/api/contents.js +46 -16
  8. package/dist/api/repos.js +60 -4
  9. package/dist/api/write.js +138 -8
  10. package/dist/api.js +34 -1
  11. package/dist/browse.js +171 -33
  12. package/dist/build-info.json +2 -2
  13. package/dist/ci/api.js +138 -4
  14. package/dist/ci/artifacts.js +82 -2
  15. package/dist/ci/engine.js +30 -17
  16. package/dist/ci/expr.js +14 -0
  17. package/dist/ci/runners.js +60 -1
  18. package/dist/ci/runs.js +21 -3
  19. package/dist/ci/trigger.js +6 -4
  20. package/dist/ci/views.js +24 -3
  21. package/dist/ci/web.js +48 -0
  22. package/dist/cli/admin-cmd.js +40 -3
  23. package/dist/cli/backup-cmd.js +138 -120
  24. package/dist/cli/gitrun.js +50 -0
  25. package/dist/cli/pr-cmd.js +155 -0
  26. package/dist/cli/repo-cmd.js +134 -6
  27. package/dist/cli/sync-cmd.js +128 -0
  28. package/dist/cli/target.js +7 -3
  29. package/dist/config.js +10 -0
  30. package/dist/credentials.js +9 -8
  31. package/dist/deploy-cli.js +214 -109
  32. package/dist/deploy-runner-cli.js +125 -49
  33. package/dist/domains.js +259 -0
  34. package/dist/egress.js +23 -6
  35. package/dist/forms.js +703 -32
  36. package/dist/git.js +124 -13
  37. package/dist/githttp.js +80 -23
  38. package/dist/githubauth.js +185 -0
  39. package/dist/icons.js +14 -0
  40. package/dist/import-cli.js +40 -25
  41. package/dist/index.js +110 -4
  42. package/dist/issueweb.js +10 -6
  43. package/dist/lfs.js +9 -2
  44. package/dist/limit.js +2 -2
  45. package/dist/logincodes.js +31 -0
  46. package/dist/maintenance.js +297 -0
  47. package/dist/markdown.js +192 -3
  48. package/dist/mdedit.js +29 -0
  49. package/dist/migrate.js +48 -0
  50. package/dist/naming.js +25 -0
  51. package/dist/onetime.js +112 -0
  52. package/dist/ops.js +293 -16
  53. package/dist/pagescript.js +355 -24
  54. package/dist/perms.js +1 -0
  55. package/dist/profile.js +2 -0
  56. package/dist/pulls.js +21 -1
  57. package/dist/pullweb.js +25 -6
  58. package/dist/releases.js +10 -1
  59. package/dist/runner/actions.js +12 -1
  60. package/dist/runner/client.js +40 -2
  61. package/dist/runner/docker.js +40 -28
  62. package/dist/runner/download.js +77 -0
  63. package/dist/runner/externals.js +12 -1
  64. package/dist/runner/job.js +26 -11
  65. package/dist/runner/steps.js +42 -7
  66. package/dist/runner-cli.js +89 -0
  67. package/dist/scan.js +58 -2
  68. package/dist/server.js +107 -26
  69. package/dist/session.js +82 -40
  70. package/dist/site.js +110 -27
  71. package/dist/sitepublish.js +177 -0
  72. package/dist/sitesettings.js +420 -0
  73. package/dist/siteshost.js +116 -28
  74. package/dist/source.js +20 -0
  75. package/dist/style.js +162 -3
  76. package/dist/topics.js +165 -0
  77. package/dist/vault.js +399 -6
  78. package/dist/vendor-age.js +15 -0
  79. package/dist/views.js +242 -57
  80. package/dist/web.js +11 -1
  81. package/dist/webauthn.js +347 -0
  82. package/dist/webops.js +1263 -33
  83. package/package.json +4 -4
@@ -33,16 +33,18 @@ var __importStar = (this && this.__importStar) || (function () {
33
33
  };
34
34
  })();
35
35
  Object.defineProperty(exports, "__esModule", { value: true });
36
- exports.backupCommands = void 0;
36
+ exports.backupCommands = exports.MOCHI_BACKUP = void 0;
37
37
  exports.backupsIndexPath = backupsIndexPath;
38
38
  exports.knownBackups = knownBackups;
39
39
  exports.backupLineFor = backupLineFor;
40
+ exports.makeBackupCommands = makeBackupCommands;
40
41
  const child_process_1 = require("child_process");
41
42
  const crypto = __importStar(require("crypto"));
42
43
  const fs = __importStar(require("fs"));
43
44
  const os = __importStar(require("os"));
44
45
  const path = __importStar(require("path"));
45
46
  const atomic_1 = require("../atomic");
47
+ const naming_1 = require("../naming");
46
48
  const scan_1 = require("../scan");
47
49
  const exit_1 = require("./exit");
48
50
  const output_1 = require("./output");
@@ -275,8 +277,8 @@ function isVaultRelative(p) {
275
277
  }
276
278
  /** How a refused path is reported, in one place since two kinds of line carry one. */
277
279
  function refusedPath(p) {
278
- return (`The vault named a path this backup will not write: ${JSON.stringify(p)}. ` +
279
- 'A manifest path must be relative to the vault and must not climb out of it, so nothing was copied.');
280
+ return (`The ${naming_1.naming.rootNoun} named a path this backup will not write: ${JSON.stringify(p)}. ` +
281
+ `A manifest path must be relative to the ${naming_1.naming.rootNoun} and must not climb out of it, so nothing was copied.`);
280
282
  }
281
283
  async function fetchManifest(target, exclude, hash) {
282
284
  const query = [];
@@ -308,8 +310,8 @@ async function fetchManifest(target, exclude, hash) {
308
310
  // URL and the token, neither of which is the problem.
309
311
  if (resp.status === 404) {
310
312
  message =
311
- `${target.host} has no /api/backup/manifest route, so it is running a mochi older than this ` +
312
- 'command. Deploy the vault again from a version that has it, then run this.';
313
+ `${target.host} has no /api/backup/manifest route, so it is running a ${naming_1.naming.product} older than this ` +
314
+ `command. Deploy the ${naming_1.naming.rootNoun} again from a version that has it, then run this.`;
313
315
  }
314
316
  // The same status-to-code mapping every other command uses, so that a
315
317
  // caller branching on the exit code does not have to learn a second table.
@@ -350,13 +352,13 @@ async function fetchManifest(target, exclude, hash) {
350
352
  ended = true;
351
353
  }
352
354
  else if (kind === 'error') {
353
- throw new exit_1.CliError(`The vault could not finish the manifest: ${String(line.error)}`);
355
+ throw new exit_1.CliError(`The ${naming_1.naming.rootNoun} could not finish the manifest: ${String(line.error)}`);
354
356
  }
355
357
  }
356
358
  // The end line is what says the walk completed. Acting on a truncated
357
359
  // manifest would delete every path the vault did not get around to listing.
358
360
  if (!ended) {
359
- throw new exit_1.CliError('The manifest ended early, so what the vault holds is not fully known. Nothing was deleted; try again.');
361
+ throw new exit_1.CliError(`The manifest ended early, so what the ${naming_1.naming.rootNoun} holds is not fully known. Nothing was deleted; try again.`);
360
362
  }
361
363
  return manifest;
362
364
  }
@@ -428,7 +430,7 @@ class FrameReader {
428
430
  let left = n;
429
431
  while (left > 0) {
430
432
  if (this.buf.length === 0 && !(await this.more())) {
431
- throw new exit_1.CliError('The vault closed the connection part way through a file. Nothing was left half-written.');
433
+ throw new exit_1.CliError(`The ${naming_1.naming.rootNoun} closed the connection part way through a file. Nothing was left half-written.`);
432
434
  }
433
435
  const take = Math.min(left, this.buf.length);
434
436
  sink(this.buf.subarray(0, take));
@@ -670,18 +672,58 @@ const RETENTION_OPTIONS = [
670
672
  summary: `Monthly snapshots to keep (default ${DEFAULT_RETENTION.monthly})`,
671
673
  },
672
674
  ];
673
- const EXCLUDE_OPTIONS = [
674
- { name: 'no-runs', type: 'boolean', summary: 'Leave out workflow run history (<repo>.runs)' },
675
- { name: 'no-sites', type: 'boolean', summary: 'Leave out published sites (<repo>.site)' },
676
- { name: 'no-lfs', type: 'boolean', summary: 'Leave out LFS objects on the volume (<repo>.lfs)' },
677
- { name: 'no-secrets', type: 'boolean', summary: 'Leave out vault.json, runners.json, and .secret' },
678
- ];
675
+ exports.MOCHI_BACKUP = {
676
+ exclusions: [
677
+ { category: 'runs', summary: 'Leave out workflow run history (<repo>.runs)' },
678
+ { category: 'sites', summary: 'Leave out published sites (<repo>.site)' },
679
+ { category: 'lfs', summary: 'Leave out LFS objects on the volume (<repo>.lfs)' },
680
+ { category: 'secrets', summary: 'Leave out vault.json, runners.json, .secret, and .github-secret' },
681
+ ],
682
+ repos: true,
683
+ description: `A vault is a directory, so a backup of one is a directory too, and this makes it
684
+ over HTTP: it needs no shell on the server, no flyctl, and no rsync at the far
685
+ end, so it works the same against a Fly app, a VPS, a Docker deployment, and
686
+ 127.0.0.1:3000.
687
+
688
+ <dir>/current a servable vault. Restoring is: mochi serve <dir>/current
689
+ <dir>/snapshots hardlinked copies, each one also a servable vault
690
+ <dir>/backup.json which vault, what is left out, and how each run went
691
+
692
+ Repositories come across as mirrors, so a second run moves only the objects it
693
+ does not have and skips a repository nothing was pushed to. Everything beside
694
+ them - issues, pull requests, releases, sites, run history, LFS objects on the
695
+ volume, and the vault's state files - is compared by size and modification time
696
+ and fetched only where it differs.
697
+
698
+ The token needs to belong to a site admin, because the copy includes
699
+ vault.json. The vault URL, the exclusions, and the retention policy are recorded
700
+ in backup.json, so a cron entry is this command and a directory.
701
+
702
+ There is no vault-wide point-in-time image: the server holds no lock a client
703
+ could take, so a run is a walk of a live tree and can catch a mixed vintage.
704
+ Every individual file in a backup is one that really existed. See docs/backup.md.
705
+
706
+ Related: mochi backup list, verify, prune.`,
707
+ verifyDescription: `Runs git fsck --connectivity-only over every mirror, asks the vault for hashes,
708
+ and reports anything missing, extra, or different. Exits non-zero when there is
709
+ something to report, so it can be run from cron.`,
710
+ pruneDescription: `Grandfather-father-son: the newest snapshot of each of the last N days, weeks,
711
+ and months is kept and the rest are removed, evaluated in UTC. The newest
712
+ snapshot is always kept.
713
+
714
+ A snapshot pins the packfiles that were current when it was taken, so a repack
715
+ in a busy repository leaves the old pack on disk until the last snapshot
716
+ referring to it is pruned. This is what reclaims that space.`,
717
+ };
718
+ function excludeOptions(profile) {
719
+ return profile.exclusions.map((e) => ({ name: `no-${e.category}`, type: 'boolean', summary: e.summary }));
720
+ }
679
721
  const QUIET_OPTION = { name: 'quiet', type: 'boolean', summary: 'Say nothing on success' };
680
722
  /** The backup directory a command was given, made if it is not there yet. */
681
723
  function backupDirectory(inv, create) {
682
724
  const given = inv.args[0];
683
725
  if (!given)
684
- throw new exit_1.CliError('Which directory? Usage: mochi backup <dir>', exit_1.EXIT_USAGE);
726
+ throw new exit_1.CliError(`Which directory? Usage: ${naming_1.naming.product} backup <dir>`, exit_1.EXIT_USAGE);
685
727
  const dir = path.resolve(given);
686
728
  if (!fs.existsSync(dir)) {
687
729
  if (!create)
@@ -697,7 +739,7 @@ function backupDirectory(inv, create) {
697
739
  function existingBackup(inv) {
698
740
  const dir = backupDirectory(inv, false);
699
741
  if (!fs.existsSync(statePath(dir))) {
700
- throw new exit_1.CliError(`${dir} holds no backup (no ${STATE_FILE}). Make one first: mochi backup ${inv.args[0]}`, exit_1.EXIT_USAGE);
742
+ throw new exit_1.CliError(`${dir} holds no backup (no ${STATE_FILE}). Make one first: ${naming_1.naming.product} backup ${inv.args[0]}`, exit_1.EXIT_USAGE);
701
743
  }
702
744
  return { dir, state: loadState(dir) };
703
745
  }
@@ -707,16 +749,8 @@ function existingBackup(inv) {
707
749
  * last run used, and naming any at all replaces the set, which is how a
708
750
  * category can be put back.
709
751
  */
710
- function exclusionsFor(inv, state) {
711
- const given = [];
712
- if (inv.bool('no-runs'))
713
- given.push('runs');
714
- if (inv.bool('no-sites'))
715
- given.push('sites');
716
- if (inv.bool('no-lfs'))
717
- given.push('lfs');
718
- if (inv.bool('no-secrets'))
719
- given.push('secrets');
752
+ function exclusionsFor(inv, state, profile) {
753
+ const given = profile.exclusions.map((e) => e.category).filter((c) => inv.bool(`no-${c}`));
720
754
  return given.length ? given : state.excluded;
721
755
  }
722
756
  function retentionFor(inv, state) {
@@ -884,14 +918,14 @@ async function fetchChunk(target, current, chunk, state, wantHashes) {
884
918
  throw new exit_1.CliError(message, (0, exit_1.exitCodeForStatus)(resp.status));
885
919
  }
886
920
  if (!resp.body)
887
- throw new exit_1.CliError('The vault answered a fetch with no body.');
921
+ throw new exit_1.CliError(`The ${naming_1.naming.rootNoun} answered a fetch with no body.`);
888
922
  const reader = new FrameReader(resp.body);
889
923
  let bytes = 0;
890
924
  let missing = [];
891
925
  for (;;) {
892
926
  const line = await reader.line();
893
927
  if (line === null)
894
- throw new exit_1.CliError('The vault ended a fetch without saying it had finished.');
928
+ throw new exit_1.CliError(`The ${naming_1.naming.rootNoun} ended a fetch without saying it had finished.`);
895
929
  const frame = JSON.parse(line);
896
930
  if (frame.end) {
897
931
  missing = frame.missing ?? [];
@@ -900,7 +934,7 @@ async function fetchChunk(target, current, chunk, state, wantHashes) {
900
934
  const rel = frame.path;
901
935
  const size = frame.size;
902
936
  if (typeof rel !== 'string' || typeof size !== 'number' || !byPath.has(rel)) {
903
- throw new exit_1.CliError(`The vault sent a file this run did not ask for: ${String(rel)}`);
937
+ throw new exit_1.CliError(`The ${naming_1.naming.rootNoun} sent a file this run did not ask for: ${String(rel)}`);
904
938
  }
905
939
  const wanted = byPath.get(rel);
906
940
  const dest = path.join(current, ...rel.split('/'));
@@ -993,7 +1027,7 @@ function emptyDirs(dir, rel = '', out = []) {
993
1027
  }
994
1028
  return empty;
995
1029
  }
996
- async function syncCmd(inv) {
1030
+ async function syncCmd(inv, profile) {
997
1031
  const json = (0, output_1.jsonMode)(inv);
998
1032
  // Two kinds of silence, and they are not the same. --json puts one JSON value
999
1033
  // on stdout, so the running commentary has to go, but a warning is a
@@ -1004,7 +1038,7 @@ async function syncCmd(inv) {
1004
1038
  const dir = backupDirectory(inv, true);
1005
1039
  const state = loadState(dir);
1006
1040
  const target = await targetForBackup(inv, state);
1007
- const exclude = exclusionsFor(inv, state);
1041
+ const exclude = exclusionsFor(inv, state, profile);
1008
1042
  const retention = retentionFor(inv, state);
1009
1043
  const checksum = inv.bool('checksum');
1010
1044
  const current = path.join(dir, CURRENT);
@@ -1078,7 +1112,7 @@ async function syncCmd(inv) {
1078
1112
  delete state.repos[known];
1079
1113
  summary.repos.removed++;
1080
1114
  if (!quiet)
1081
- console.error(`Removed ${known}, which the vault no longer holds`);
1115
+ console.error(`Removed ${known}, which the ${naming_1.naming.rootNoun} no longer holds`);
1082
1116
  }
1083
1117
  // Files.
1084
1118
  const changed = [];
@@ -1094,7 +1128,7 @@ async function syncCmd(inv) {
1094
1128
  for (const gone of r.missing) {
1095
1129
  delete state.files[gone];
1096
1130
  if (!quiet)
1097
- console.error(`${gone} vanished from the vault while this run was reading it`);
1131
+ console.error(`${gone} vanished from the ${naming_1.naming.rootNoun} while this run was reading it`);
1098
1132
  }
1099
1133
  }
1100
1134
  // A mirror's config is one of the files the manifest names, so the copy just
@@ -1204,15 +1238,17 @@ async function syncCmd(inv) {
1204
1238
  if (quiet)
1205
1239
  return;
1206
1240
  const r = summary.repos;
1207
- console.log(`${r.total} repositories: ${r.cloned} cloned, ${r.fetched} fetched, ${r.skipped} unchanged` +
1208
- (r.removed ? `, ${r.removed} removed` : ''));
1241
+ if (profile.repos) {
1242
+ console.log(`${r.total} repositories: ${r.cloned} cloned, ${r.fetched} fetched, ${r.skipped} unchanged` +
1243
+ (r.removed ? `, ${r.removed} removed` : ''));
1244
+ }
1209
1245
  console.log(`${summary.files.total} files: ${summary.files.fetched} fetched (${human(summary.files.bytes)})` +
1210
1246
  (summary.files.removed ? `, ${summary.files.removed} removed` : ''));
1211
1247
  if (summary.snapshot)
1212
1248
  console.log(`Snapshot ${summary.snapshot}`);
1213
1249
  console.log('');
1214
- console.log(`Serve this backup to look at it, or to stand the vault back up:`);
1215
- console.log(` mochi serve ${path.join(dir, CURRENT)}`);
1250
+ console.log(`Serve this backup to look at it, or to stand the ${naming_1.naming.rootNoun} back up:`);
1251
+ console.log(` ${naming_1.naming.product} serve ${path.join(dir, CURRENT)}`);
1216
1252
  }
1217
1253
  // ---- list, prune, verify ----
1218
1254
  function listCmd(inv) {
@@ -1236,7 +1272,7 @@ function listCmd(inv) {
1236
1272
  return;
1237
1273
  }
1238
1274
  console.log(`${dir}`);
1239
- console.log(` vault ${state.host || '(unknown)'}`);
1275
+ console.log(` ${naming_1.naming.rootNoun.padEnd(10)} ${state.host || '(unknown)'}`);
1240
1276
  console.log(` current ${human(apparentSize(path.join(dir, CURRENT)))} apparent`);
1241
1277
  console.log(` excluded ${state.excluded.length ? state.excluded.join(', ') : 'nothing'}`);
1242
1278
  console.log(` retention ${state.retention.daily} daily, ${state.retention.weekly} weekly, ${state.retention.monthly} monthly`);
@@ -1246,7 +1282,7 @@ function listCmd(inv) {
1246
1282
  }
1247
1283
  console.log('');
1248
1284
  if (snapshots.length === 0) {
1249
- console.log('No snapshots. `mochi backup <dir> --snapshot` takes one after a sync.');
1285
+ console.log(`No snapshots. '${naming_1.naming.product} backup <dir> --snapshot' takes one after a sync.`);
1250
1286
  return;
1251
1287
  }
1252
1288
  // Apparent size rather than disk use: a snapshot is hardlinked, so what it
@@ -1277,7 +1313,7 @@ function pruneCmd(inv) {
1277
1313
  console.log(dropped.length ? `Pruned ${dropped.length}: ${dropped.join(', ')}` : 'Nothing to prune under this retention.');
1278
1314
  console.log(`${kept.length} snapshot${kept.length === 1 ? '' : 's'} kept.`);
1279
1315
  }
1280
- async function verifyCmd(inv) {
1316
+ async function verifyCmd(inv, profile) {
1281
1317
  const { dir, state } = existingBackup(inv);
1282
1318
  const json = (0, output_1.jsonMode)(inv);
1283
1319
  const quiet = inv.bool('quiet') || json.enabled;
@@ -1304,7 +1340,7 @@ async function verifyCmd(inv) {
1304
1340
  // The files, against hashes the vault computes now. This is the part a
1305
1341
  // size-and-mtime sync cannot check on its own.
1306
1342
  if (!quiet)
1307
- console.error('Asking the vault for hashes');
1343
+ console.error(`Asking the ${naming_1.naming.rootNoun} for hashes`);
1308
1344
  const manifest = await fetchManifest(target, state.excluded, true);
1309
1345
  for (const f of manifest.files.values()) {
1310
1346
  const dest = path.join(current, ...f.path.split('/'));
@@ -1317,11 +1353,11 @@ async function verifyCmd(inv) {
1317
1353
  continue;
1318
1354
  }
1319
1355
  if (st.size !== f.size) {
1320
- problems.push({ path: f.path, problem: `size ${st.size}, the vault has ${f.size}` });
1356
+ problems.push({ path: f.path, problem: `size ${st.size}, the ${naming_1.naming.rootNoun} has ${f.size}` });
1321
1357
  continue;
1322
1358
  }
1323
1359
  if (f.sha256 && sha256Of(dest) !== f.sha256) {
1324
- problems.push({ path: f.path, problem: 'contents differ from the vault' });
1360
+ problems.push({ path: f.path, problem: `contents differ from the ${naming_1.naming.rootNoun}` });
1325
1361
  }
1326
1362
  }
1327
1363
  const mirrors = repos.map((p) => p.split('/').join(path.sep));
@@ -1331,11 +1367,11 @@ async function verifyCmd(inv) {
1331
1367
  if (insideMirror(asPath))
1332
1368
  continue;
1333
1369
  if (!manifest.files.has(rel))
1334
- problems.push({ path: rel, problem: 'in the backup, not in the vault' });
1370
+ problems.push({ path: rel, problem: `in the backup, not in the ${naming_1.naming.rootNoun}` });
1335
1371
  }
1336
1372
  for (const entry of manifest.repos) {
1337
1373
  if (!state.repos[entry.path])
1338
- problems.push({ path: entry.path, problem: 'in the vault, not in the backup' });
1374
+ problems.push({ path: entry.path, problem: `in the ${naming_1.naming.rootNoun}, not in the backup` });
1339
1375
  }
1340
1376
  // The hardlink invariant the snapshots rest on. A file with more links than
1341
1377
  // current/ plus the snapshots can account for is shared with something
@@ -1356,7 +1392,9 @@ async function verifyCmd(inv) {
1356
1392
  }
1357
1393
  else if (problems.length === 0) {
1358
1394
  if (!inv.bool('quiet')) {
1359
- console.log(`${repos.length} mirrors and ${manifest.files.size} files check out against ${target.host}.`);
1395
+ console.log(profile.repos
1396
+ ? `${repos.length} mirrors and ${manifest.files.size} files check out against ${target.host}.`
1397
+ : `${manifest.files.size} files check out against ${target.host}.`);
1360
1398
  }
1361
1399
  }
1362
1400
  else {
@@ -1378,7 +1416,7 @@ async function verifyCmd(inv) {
1378
1416
  */
1379
1417
  function backupsIndexPath() {
1380
1418
  const base = process.env.XDG_CONFIG_HOME ?? path.join(os.homedir(), '.config');
1381
- return path.join(base, 'mochi', 'backups.json');
1419
+ return path.join(base, naming_1.naming.configDirName, 'backups.json');
1382
1420
  }
1383
1421
  function knownBackups() {
1384
1422
  try {
@@ -1424,77 +1462,57 @@ function backupLineFor(hosts) {
1424
1462
  }
1425
1463
  // ---- the commands ----
1426
1464
  const COMMON = [output_1.JSON_OPTION, QUIET_OPTION, ...target_1.TARGET_OPTIONS];
1427
- exports.backupCommands = [
1428
- {
1429
- path: ['backup'],
1430
- summary: 'Copy a whole vault to a directory on this machine, incrementally',
1431
- description: `A vault is a directory, so a backup of one is a directory too, and this makes it
1432
- over HTTP: it needs no shell on the server, no flyctl, and no rsync at the far
1433
- end, so it works the same against a Fly app, a VPS, a Docker deployment, and
1434
- 127.0.0.1:3000.
1435
-
1436
- <dir>/current a servable vault. Restoring is: mochi serve <dir>/current
1437
- <dir>/snapshots hardlinked copies, each one also a servable vault
1438
- <dir>/backup.json which vault, what is left out, and how each run went
1439
-
1440
- Repositories come across as mirrors, so a second run moves only the objects it
1441
- does not have and skips a repository nothing was pushed to. Everything beside
1442
- them - issues, pull requests, releases, sites, run history, LFS objects on the
1443
- volume, and the vault's state files - is compared by size and modification time
1444
- and fetched only where it differs.
1445
-
1446
- The token needs to belong to a site admin, because the copy includes
1447
- vault.json. The vault URL, the exclusions, and the retention policy are recorded
1448
- in backup.json, so a cron entry is this command and a directory.
1449
-
1450
- There is no vault-wide point-in-time image: the server holds no lock a client
1451
- could take, so a run is a walk of a live tree and can catch a mixed vintage.
1452
- Every individual file in a backup is one that really existed. See docs/backup.md.
1453
-
1454
- Related: mochi backup list, verify, prune.`,
1455
- args: [{ name: 'dir', required: true }],
1456
- options: [
1457
- { name: 'snapshot', type: 'boolean', summary: 'Take a snapshot after a successful sync, then prune' },
1458
- ...RETENTION_OPTIONS,
1459
- ...EXCLUDE_OPTIONS,
1460
- { name: 'checksum', type: 'boolean', summary: 'Compare hashes rather than size and modification time' },
1461
- ...COMMON,
1462
- ],
1463
- async run(inv) {
1464
- await syncCmd(inv);
1465
- const dir = path.resolve(inv.args[0]);
1466
- rememberBackup(dir, loadState(dir).host);
1465
+ /**
1466
+ * The backup commands for one application. Mochi's are `backupCommands`
1467
+ * below; a sibling application built on these modules (dango) passes its own
1468
+ * profile and gets the same four commands with its own exclusions and help.
1469
+ */
1470
+ function makeBackupCommands(profile) {
1471
+ const noun = naming_1.naming.rootNoun;
1472
+ return [
1473
+ {
1474
+ path: ['backup'],
1475
+ summary: `Copy a whole ${noun} to a directory on this machine, incrementally`,
1476
+ description: profile.description,
1477
+ args: [{ name: 'dir', required: true }],
1478
+ options: [
1479
+ { name: 'snapshot', type: 'boolean', summary: 'Take a snapshot after a successful sync, then prune' },
1480
+ ...RETENTION_OPTIONS,
1481
+ ...excludeOptions(profile),
1482
+ { name: 'checksum', type: 'boolean', summary: 'Compare hashes rather than size and modification time' },
1483
+ ...COMMON,
1484
+ ],
1485
+ async run(inv) {
1486
+ await syncCmd(inv, profile);
1487
+ const dir = path.resolve(inv.args[0]);
1488
+ rememberBackup(dir, loadState(dir).host);
1489
+ },
1467
1490
  },
1468
- },
1469
- {
1470
- path: ['backup', 'list'],
1471
- summary: "Show a backup's snapshots, and how the last run went",
1472
- args: [{ name: 'dir', required: true }],
1473
- options: [output_1.JSON_OPTION],
1474
- run: listCmd,
1475
- },
1476
- {
1477
- path: ['backup', 'verify'],
1478
- summary: 'Check a backup against the vault, and its mirrors against git',
1479
- description: `Runs git fsck --connectivity-only over every mirror, asks the vault for hashes,
1480
- and reports anything missing, extra, or different. Exits non-zero when there is
1481
- something to report, so it can be run from cron.`,
1482
- args: [{ name: 'dir', required: true }],
1483
- options: [...COMMON],
1484
- run: verifyCmd,
1485
- },
1486
- {
1487
- path: ['backup', 'prune'],
1488
- summary: 'Apply the retention policy to the snapshots, without syncing',
1489
- description: `Grandfather-father-son: the newest snapshot of each of the last N days, weeks,
1490
- and months is kept and the rest are removed, evaluated in UTC. The newest
1491
- snapshot is always kept.
1492
-
1493
- A snapshot pins the packfiles that were current when it was taken, so a repack
1494
- in a busy repository leaves the old pack on disk until the last snapshot
1495
- referring to it is pruned. This is what reclaims that space.`,
1496
- args: [{ name: 'dir', required: true }],
1497
- options: [...RETENTION_OPTIONS, output_1.JSON_OPTION, QUIET_OPTION],
1498
- run: pruneCmd,
1499
- },
1500
- ];
1491
+ {
1492
+ path: ['backup', 'list'],
1493
+ summary: "Show a backup's snapshots, and how the last run went",
1494
+ args: [{ name: 'dir', required: true }],
1495
+ options: [output_1.JSON_OPTION],
1496
+ run: listCmd,
1497
+ },
1498
+ {
1499
+ path: ['backup', 'verify'],
1500
+ summary: profile.repos
1501
+ ? `Check a backup against the ${noun}, and its mirrors against git`
1502
+ : `Check a backup against the ${noun}`,
1503
+ description: profile.verifyDescription,
1504
+ args: [{ name: 'dir', required: true }],
1505
+ options: [...COMMON],
1506
+ run: (inv) => verifyCmd(inv, profile),
1507
+ },
1508
+ {
1509
+ path: ['backup', 'prune'],
1510
+ summary: 'Apply the retention policy to the snapshots, without syncing',
1511
+ description: profile.pruneDescription,
1512
+ args: [{ name: 'dir', required: true }],
1513
+ options: [...RETENTION_OPTIONS, output_1.JSON_OPTION, QUIET_OPTION],
1514
+ run: pruneCmd,
1515
+ },
1516
+ ];
1517
+ }
1518
+ exports.backupCommands = makeBackupCommands(exports.MOCHI_BACKUP);
@@ -0,0 +1,50 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.run = run;
4
+ exports.capture = capture;
5
+ exports.mochiCredentialEnv = mochiCredentialEnv;
6
+ exports.mochiCredentialArgs = mochiCredentialArgs;
7
+ exports.ghCredentialArgs = ghCredentialArgs;
8
+ const child_process_1 = require("child_process");
9
+ // Client-side git, shared by the commands that move repositories between a
10
+ // vault and somewhere else: `mochi import` and `mochi fork` (clone from a
11
+ // source, push to the vault), `mochi sync` (fetch from an upstream, push to
12
+ // the vault), and `mochi pr export` (clone from the vault, push to GitHub).
13
+ // All of them run on the operator's machine, which is the point: the vault
14
+ // holds no credential for any system but itself.
15
+ /** Run a command with its output going straight to the terminal, so a long clone shows progress. */
16
+ function run(cmd, args, env) {
17
+ return new Promise((resolve, reject) => {
18
+ const child = (0, child_process_1.spawn)(cmd, args, { stdio: 'inherit', env: env ?? process.env });
19
+ child.on('error', (e) => reject(new Error(e.code === 'ENOENT' ? `${cmd} is not on PATH` : String(e.message))));
20
+ child.on('close', (code) => resolve(code ?? 1));
21
+ });
22
+ }
23
+ /** Run a command quietly and capture its stdout, or null when it fails to run or exits nonzero. */
24
+ function capture(cmd, args, env) {
25
+ return new Promise((resolve) => {
26
+ (0, child_process_1.execFile)(cmd, args, { maxBuffer: 4 * 1024 * 1024, env: env ?? process.env }, (err, stdout) => {
27
+ resolve(err ? null : String(stdout));
28
+ });
29
+ });
30
+ }
31
+ // The token is handed to git through a helper that reads it from the
32
+ // environment rather than through the command line, where every process on the
33
+ // machine could read it. The empty helper first clears the list, so the answer
34
+ // comes from here and nowhere else, and no prompt can appear.
35
+ function mochiCredentialEnv(username, token) {
36
+ return { ...process.env, MOCHI_USER: username, MOCHI_TOKEN: token, GIT_TERMINAL_PROMPT: '0' };
37
+ }
38
+ const MOCHI_CREDENTIAL_HELPER = `!f() { test "$1" = get && printf 'username=%s\\npassword=%s\\n' "$MOCHI_USER" "$MOCHI_TOKEN"; }; f`;
39
+ /** git options that make a vault URL authenticate from mochiCredentialEnv and nothing else. */
40
+ function mochiCredentialArgs() {
41
+ return ['-c', 'credential.helper=', '-c', `credential.helper=${MOCHI_CREDENTIAL_HELPER}`];
42
+ }
43
+ /**
44
+ * git options that make a github.com URL authenticate through `gh`'s own
45
+ * credential helper. `pr export` pushes with these, so having run
46
+ * `gh auth login` once is the only GitHub credential setup it asks for.
47
+ */
48
+ function ghCredentialArgs() {
49
+ return ['-c', 'credential.helper=', '-c', 'credential.helper=!gh auth git-credential'];
50
+ }