@ps-aux/nodebup 0.9.1 → 0.9.3

package/lib/bup/pg/PgBackupController.js

@@ -26,7 +26,8 @@ function _defineProperty(obj, key, value) { if (key in obj) { Object.definePrope
 const parseConnectionUrl = url => {
   var _url$match;
 
-  const regex = /postgres:\/\/(?<username>.*):(?<password>.*)@(?<host>.*):(?<port>\d*)$/;
+  // TODO what if there are query params in the end?
+  const regex = /postgres:\/\/(?<username>.*):(?<password>.*)@(?<host>.*):(?<port>\d*)(\/(?<database>.*))?$/;
   const match = (_url$match = url.match(regex)) === null || _url$match === void 0 ? void 0 : _url$match.groups;
   if (!match || !match.username || !match.password || !match.host || !match.port) throw new Error(`The Postgres connection URL does not match required regex: ${regex.toString()}`);
   return {
@@ -59,7 +60,7 @@ let PgBackupController = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.m
 
     _defineProperty(this, "outFileName", 'backup.sql');
 
-    _defineProperty(this, "backup", ({
+    _defineProperty(this, "backup", async ({
       pgUrl,
       pgVersion
     }) => {
@@ -69,16 +70,20 @@ let PgBackupController = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.m
       this.log.info(`Backing up Postgres database, version=${version}`);
       const connParams = parseConnectionUrl(pgUrl);
       const pass = connParams.password;
-      this.fs.inTmpDir('pg-backup', dir => {
+      await this.fs.inTmpDir('pg-backup', async dir => {
         const outputDir = _Path.AbsPath.from(dir);
 
         this.log.info('Dumping database to a file');
         const bashCmds = [`echo "*:*:*:*:${pass}" > ~/.pgpass`, `chmod 400 ~/.pgpass`, `pg_dumpall -d ${pgUrl}`]; // Don't forget that this itself might be run in docker
         // therefore volume mounts are not usable (will apply to the location at host)
 
-        const b = this.sh.execAndReturnBuffer(`docker run --network host ` + `postgres:${version} ` + `bash -c '${bashCmds.join(' && ')}'`);
         const dumpOut = outputDir.resolve(this.outFileName);
-        this.fs.writeFile(dumpOut, b);
+        const f = this.fs.writeStream(dumpOut);
+        await this.sh.asyncExec(`docker run --rm --network host ` + `postgres:${version} ` + `bash -c '${bashCmds.join(' && ')}'`, // f.write
+        data => {
+          f.write(data);
+        });
+        f.end();
         this.log.info('Compressing');
         const zipOutputName = `pg-backup-${this.now()}.zip`;
         this.zip.zipDir(outputDir, outputDir.resolve(zipOutputName));
@@ -88,7 +93,7 @@ let PgBackupController = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.m
       });
     });
 
-    _defineProperty(this, "restore", ({
+    _defineProperty(this, "restore", async ({
       pgUrl,
       pgVersion
     }) => {
@@ -97,7 +102,7 @@ let PgBackupController = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.m
 
       parseConnectionUrl(pgUrl);
       this.log.info(`Restoring Postgres database, version=${version}`);
-      this.fs.inTmpDir('pg-restore', dirStr => {
+      await this.fs.inTmpDir('pg-restore', dirStr => {
         const dir = _Path.AbsPath.from(dirStr);
 
         storage.restore(dir); // TODO check if the dir contains the with the expected name

package/lib/fs/Fs.js

@@ -37,6 +37,11 @@ let Fs = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.metadata("design:
       _fs.default.writeFileSync(path.str(), content);
     });
 
+    _defineProperty(this, "writeStream", path => {
+      this.log.debug('Writing into file', path);
+      return _fs.default.createWriteStream(path.str());
+    });
+
     _defineProperty(this, "isFile", path => !this.isDir(path));
 
     _defineProperty(this, "ensureIsFile", p => {
@@ -59,11 +64,11 @@ let Fs = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.metadata("design:
       return _fs.default.readdirSync(path.str()).map(f => path.resolve(f));
     });
 
-    _defineProperty(this, "inTmpDir", (name, withDir) => {
+    _defineProperty(this, "inTmpDir", async (name, withDir) => {
       const dir = this.mkTmpDir(name);
 
       try {
-        withDir(dir);
+        await withDir(dir);
       } finally {
         this.rmDir(_Path.AbsPath.from(dir));
       }

package/lib/fs/Fs.test.js

@@ -0,0 +1,20 @@
+"use strict";
+
+var _Fs = require("./Fs");
+
+var _Path = require("./path/Path");
+
+var _AppLogger = require("../log/AppLogger");
+
+it.skip('withTmpDir', async () => {
+  const fs = new _Fs.Fs(_Path.AbsPath.from(__dirname), new _AppLogger.AppLogger());
+  await fs.inTmpDir('foo', async () => {
+    console.log('before');
+    const wait = new Promise((res, rej) => {
+      setTimeout(res, 1000);
+    });
+    await wait;
+    console.log('after');
+  });
+  console.log('done');
+});

package/lib/storage/restic/ResticClient.js

@@ -35,7 +35,7 @@ class ResticClient {
     });
 
     _defineProperty(this, "runCmd", cmd => {
-      return this.shell.execAndReturnBuffer(`restic ${cmd}`, {
+      return this.shell.execAndReturnString(`restic ${cmd}`, {
         env: this.env()
       });
     });

package/lib/tools/shell/Shell.js

@@ -11,13 +11,16 @@ var _inversify = require("inversify");
 
 var _AppLogger = require("../../log/AppLogger");
 
+var _child_process = require("child_process");
+
 var _dec, _dec2, _dec3, _class;
 
 function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
 
 // TODO copy pasted from my other project
+// TODO don't log the full command as it contains secrets
 let Shell = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.metadata("design:type", Function), _dec3 = Reflect.metadata("design:paramtypes", [typeof _AppLogger.AppLogger === "undefined" ? Object : _AppLogger.AppLogger]), _dec(_class = _dec2(_class = _dec3(_class = class Shell {
-  constructor(log) {
+  constructor(log = (0, _AppLogger.logger)('shell')) {
     this.log = log;
 
     _defineProperty(this, "exec", (cmd, opts = {}) => {
@@ -38,19 +41,31 @@ let Shell = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.metadata("desi
     });
 
     _defineProperty(this, "execAndReturnString", (cmd, ops = {}) => {
-      this.log.debug(cmd, '[stdout consumed]');
+      // this.log.debug(cmd, '[stdout consumed]')
       return (0, _shellCmd.shellCmd)(cmd, {
         returnStdout: 'string',
         ...ops
       });
     });
 
-    _defineProperty(this, "execAndReturnBuffer", (cmd, ops = {}) => {
-      this.log.debug(cmd, '[stdout consumed]');
-      return (0, _shellCmd.shellCmd)(cmd, {
-        returnStdout: 'buffer',
-        ...ops
+    _defineProperty(this, "asyncExec", (cmd, onStdout, ops = {}) => {
+      let onDone;
+      let onError;
+      const p = new Promise((res, rej) => {
+        onDone = res;
+        onError = rej;
+      });
+      const r = (0, _child_process.spawn)(cmd, {
+        shell: true,
+        cwd: ops.cwd,
+        env: ops.env
+      });
+      r.stdout.on('data', onStdout);
+      r.stderr.on('data', data => console.error(data.toString()));
+      r.on('exit', status => {
+        if (status === 0) onDone();else onError(new Error('Exited with non zero return code: ' + status));
       });
+      return p;
     });
   }
 

package/lib/tools/shell/Shell.test.js

@@ -0,0 +1,21 @@
+"use strict";
+
+var _Shell = require("./Shell");
+
+const bigStdOutCme = () => {
+  const times = 5_00_000;
+  return `bash -c 'for i in {1..${times}}; do echo "Looong data"; done; echo Done'`;
+}; // Not to be run on CI
+
+
+it.skip('execAndReturnStream', async () => {
+  const sh = new _Shell.Shell(); // The output is too big for the exec buffer
+
+  expect(() => sh.execAndReturnString(bigStdOutCme())).toThrow('spawnSync /bin/sh ENOBUFS'); //
+
+  let out = '';
+  await sh.asyncExec(bigStdOutCme(), data => {
+    out = data.toString().trim();
+  });
+  expect(out).toEndWith('Done');
+}, 5000_000);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ps-aux/nodebup",
-  "version": "0.9.1",
+  "version": "0.9.3",
   "description": "",
   "module": "lib/index.js",
   "main": "lib/index.js",