@ps-aux/nodebup 0.8.1 → 0.9.2

package/lib/bup/pg/PgBackupController.js

@@ -59,7 +59,7 @@ let PgBackupController = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.m
 
     _defineProperty(this, "outFileName", 'backup.sql');
 
-    _defineProperty(this, "backup", ({
+    _defineProperty(this, "backup", async ({
       pgUrl,
       pgVersion
     }) => {
@@ -69,16 +69,20 @@ let PgBackupController = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.m
       this.log.info(`Backing up Postgres database, version=${version}`);
       const connParams = parseConnectionUrl(pgUrl);
       const pass = connParams.password;
-      this.fs.inTmpDir('pg-backup', dir => {
+      await this.fs.inTmpDir('pg-backup', async dir => {
         const outputDir = _Path.AbsPath.from(dir);
 
         this.log.info('Dumping database to a file');
         const bashCmds = [`echo "*:*:*:*:${pass}" > ~/.pgpass`, `chmod 400 ~/.pgpass`, `pg_dumpall -d ${pgUrl}`]; // Don't forget that this itself might be run in docker
         // therefore volume mounts are not usable (will apply to the location at host)
 
-        const b = this.sh.execAndReturnBuffer(`docker run --network host ` + `postgres:${version} ` + `bash -c '${bashCmds.join(' && ')}'`);
         const dumpOut = outputDir.resolve(this.outFileName);
-        this.fs.writeFile(dumpOut, b);
+        const f = this.fs.writeStream(dumpOut);
+        await this.sh.asyncExec(`docker run --rm --network host ` + `postgres:${version} ` + `bash -c '${bashCmds.join(' && ')}'`, // f.write
+        data => {
+          f.write(data);
+        });
+        f.end();
         this.log.info('Compressing');
         const zipOutputName = `pg-backup-${this.now()}.zip`;
         this.zip.zipDir(outputDir, outputDir.resolve(zipOutputName));
@@ -88,7 +92,7 @@ let PgBackupController = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.m
       });
     });
 
-    _defineProperty(this, "restore", ({
+    _defineProperty(this, "restore", async ({
       pgUrl,
       pgVersion
     }) => {
@@ -97,7 +101,7 @@ let PgBackupController = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.m
 
       parseConnectionUrl(pgUrl);
       this.log.info(`Restoring Postgres database, version=${version}`);
-      this.fs.inTmpDir('pg-restore', dirStr => {
+      await this.fs.inTmpDir('pg-restore', dirStr => {
         const dir = _Path.AbsPath.from(dirStr);
 
         storage.restore(dir); // TODO check if the dir contains the with the expected name

package/lib/cli/app.js

@@ -101,14 +101,22 @@ const createApp = () => _nclif.CliApp.of({
 
 exports.createApp = createApp;
 const restic = (0, _nclif.cmdGroup)({
+  options: singleStorageOptions,
   commands: {
     'init-repo': (0, _nclif.cmd)({
-      options: singleStorageOptions,
       run: (_, c) => c.get(_ResticController.ResticController).initRepo()
     }),
     snapshots: (0, _nclif.cmd)({
-      options: singleStorageOptions,
       run: (_, c, p) => c.get(_ResticController.ResticController).listSnapshots(p.stdout)
+    }),
+    cmd: (0, _nclif.cmd)({
+      positionals: [{
+        name: 'cmd',
+        required: true
+      }],
+      run: ({
+        cmd
+      }, c, p) => c.get(_ResticController.ResticController).runResticCmd(cmd, p.stdout)
     })
   }
 });

package/lib/fs/Fs.js

@@ -37,6 +37,11 @@ let Fs = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.metadata("design:
       _fs.default.writeFileSync(path.str(), content);
     });
 
+    _defineProperty(this, "writeStream", path => {
+      this.log.debug('Writing into file', path);
+      return _fs.default.createWriteStream(path.str());
+    });
+
     _defineProperty(this, "isFile", path => !this.isDir(path));
 
     _defineProperty(this, "ensureIsFile", p => {
@@ -59,11 +64,11 @@ let Fs = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.metadata("design:
       return _fs.default.readdirSync(path.str()).map(f => path.resolve(f));
     });
 
-    _defineProperty(this, "inTmpDir", (name, withDir) => {
+    _defineProperty(this, "inTmpDir", async (name, withDir) => {
       const dir = this.mkTmpDir(name);
 
       try {
-        withDir(dir);
+        await withDir(dir);
       } finally {
         this.rmDir(_Path.AbsPath.from(dir));
       }

package/lib/fs/Fs.test.js

@@ -0,0 +1,20 @@
+"use strict";
+
+var _Fs = require("./Fs");
+
+var _Path = require("./path/Path");
+
+var _AppLogger = require("../log/AppLogger");
+
+it.skip('withTmpDir', async () => {
+  const fs = new _Fs.Fs(_Path.AbsPath.from(__dirname), new _AppLogger.AppLogger());
+  await fs.inTmpDir('foo', async () => {
+    console.log('before');
+    const wait = new Promise((res, rej) => {
+      setTimeout(res, 1000);
+    });
+    await wait;
+    console.log('after');
+  });
+  console.log('done');
+});

package/lib/storage/restic/ResticClient.js

@@ -34,6 +34,12 @@ class ResticClient {
       });
     });
 
+    _defineProperty(this, "runCmd", cmd => {
+      return this.shell.execAndReturnString(`restic ${cmd}`, {
+        env: this.env()
+      });
+    });
+
     _defineProperty(this, "backup", (cwd, from, tags = []) => {
       this.log.debug(`Running backup for repo=${this.url}`);
       let cmd = `restic backup ${from.str()}`;
@@ -56,12 +62,12 @@ class ResticClient {
     _defineProperty(this, "forget", () => {
       this.log.debug(`Pruning repo=${this.url}`);
       const cfg = {
-        hourly: 12,
+        hourly: 6,
         daily: 7,
         weekly: 8,
-        monthly: 24
+        monthly: 12
       };
-      this.shell.exec(`restic forget --prune ` + `--tag '' ` + // Ignore the tagged snapshots
+      this.shell.exec(`restic forget --prune ` + `--group-by tags ` + // The paths are often different when using tmp dirs
       `--keep-hourly ${cfg.hourly}  --keep-daily ${cfg.daily} ` + `--keep-weekly ${cfg.weekly} --keep-monthly ${cfg.monthly}`, {
         env: this.env()
       });

package/lib/storage/restic/ResticController.js

@@ -36,11 +36,12 @@ let ResticController = (_dec = (0, _inversify.injectable)(), _dec2 = function (t
     _defineProperty(this, "client", () => {
       const props = this.storageConfigProvider.provide();
       if (props.type !== _types.StorageType.Restic) throw new Error('Storage is not Restic storage');
-      this.log.info('Initializing repo', props.repo);
       return this.restFact.createClient(props);
     });
 
     _defineProperty(this, "initRepo", () => {
+      const props = this.storageConfigProvider.provide();
+      this.log.info(`Initializing Restic repo ${props.repo}`);
       this.client().prepareRepo();
     });
 
@@ -48,6 +49,11 @@ let ResticController = (_dec = (0, _inversify.injectable)(), _dec2 = function (t
       const res = this.client().snapshots();
       print(JSON.stringify(res, null, 4));
     });
+
+    _defineProperty(this, "runResticCmd", (cmd, print) => {
+      const res = this.client().runCmd(cmd);
+      print(res.toString());
+    });
   }
 
 }) || _class) || _class) || _class) || _class);

package/lib/tools/shell/Shell.js

@@ -11,13 +11,16 @@ var _inversify = require("inversify");
 
 var _AppLogger = require("../../log/AppLogger");
 
+var _child_process = require("child_process");
+
 var _dec, _dec2, _dec3, _class;
 
 function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
 
 // TODO copy pasted from my other project
+// TODO don't log the full command as it contains secrets
 let Shell = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.metadata("design:type", Function), _dec3 = Reflect.metadata("design:paramtypes", [typeof _AppLogger.AppLogger === "undefined" ? Object : _AppLogger.AppLogger]), _dec(_class = _dec2(_class = _dec3(_class = class Shell {
-  constructor(log) {
+  constructor(log = (0, _AppLogger.logger)('shell')) {
     this.log = log;
 
     _defineProperty(this, "exec", (cmd, opts = {}) => {
@@ -38,19 +41,31 @@ let Shell = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.metadata("desi
     });
 
     _defineProperty(this, "execAndReturnString", (cmd, ops = {}) => {
-      this.log.debug(cmd, '[stdout consumed]');
+      // this.log.debug(cmd, '[stdout consumed]')
       return (0, _shellCmd.shellCmd)(cmd, {
         returnStdout: 'string',
         ...ops
       });
     });
 
-    _defineProperty(this, "execAndReturnBuffer", (cmd, ops = {}) => {
-      this.log.debug(cmd, '[stdout consumed]');
-      return (0, _shellCmd.shellCmd)(cmd, {
-        returnStdout: 'buffer',
-        ...ops
+    _defineProperty(this, "asyncExec", (cmd, onStdout, ops = {}) => {
+      let onDone;
+      let onError;
+      const p = new Promise((res, rej) => {
+        onDone = res;
+        onError = rej;
+      });
+      const r = (0, _child_process.spawn)(cmd, {
+        shell: true,
+        cwd: ops.cwd,
+        env: ops.env
+      });
+      r.stdout.on('data', onStdout);
+      r.stderr.on('data', data => console.error(data.toString()));
+      r.on('exit', status => {
+        if (status === 0) onDone();else onError(new Error('Exited with non zero return code: ' + status));
       });
+      return p;
     });
   }
 

package/lib/tools/shell/Shell.test.js

@@ -0,0 +1,21 @@
+"use strict";
+
+var _Shell = require("./Shell");
+
+const bigStdOutCme = () => {
+  const times = 5_00_000;
+  return `bash -c 'for i in {1..${times}}; do echo "Looong data"; done; echo Done'`;
+}; // Not to be run on CI
+
+
+it.skip('execAndReturnStream', async () => {
+  const sh = new _Shell.Shell(); // The output is too big for the exec buffer
+
+  expect(() => sh.execAndReturnString(bigStdOutCme())).toThrow('spawnSync /bin/sh ENOBUFS'); //
+
+  let out = '';
+  await sh.asyncExec(bigStdOutCme(), data => {
+    out = data.toString().trim();
+  });
+  expect(out).toEndWith('Done');
+}, 5000_000);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ps-aux/nodebup",
-  "version": "0.8.1",
+  "version": "0.9.2",
   "description": "",
   "module": "lib/index.js",
   "main": "lib/index.js",