@ps-aux/nodebup 0.8.1 → 0.9.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/bup/pg/PgBackupController.js +10 -6
- package/lib/cli/app.js +10 -2
- package/lib/fs/Fs.js +7 -2
- package/lib/fs/Fs.test.js +20 -0
- package/lib/storage/restic/ResticClient.js +9 -3
- package/lib/storage/restic/ResticController.js +7 -1
- package/lib/tools/shell/Shell.js +22 -7
- package/lib/tools/shell/Shell.test.js +21 -0
- package/package.json +1 -1
@@ -59,7 +59,7 @@ let PgBackupController = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.m
|
|
59
59
|
|
60
60
|
_defineProperty(this, "outFileName", 'backup.sql');
|
61
61
|
|
62
|
-
_defineProperty(this, "backup", ({
|
62
|
+
_defineProperty(this, "backup", async ({
|
63
63
|
pgUrl,
|
64
64
|
pgVersion
|
65
65
|
}) => {
|
@@ -69,16 +69,20 @@ let PgBackupController = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.m
|
|
69
69
|
this.log.info(`Backing up Postgres database, version=${version}`);
|
70
70
|
const connParams = parseConnectionUrl(pgUrl);
|
71
71
|
const pass = connParams.password;
|
72
|
-
this.fs.inTmpDir('pg-backup', dir => {
|
72
|
+
await this.fs.inTmpDir('pg-backup', async dir => {
|
73
73
|
const outputDir = _Path.AbsPath.from(dir);
|
74
74
|
|
75
75
|
this.log.info('Dumping database to a file');
|
76
76
|
const bashCmds = [`echo "*:*:*:*:${pass}" > ~/.pgpass`, `chmod 400 ~/.pgpass`, `pg_dumpall -d ${pgUrl}`]; // Don't forget that this itself might be run in docker
|
77
77
|
// therefore volume mounts are not usable (will apply to the location at host)
|
78
78
|
|
79
|
-
const b = this.sh.execAndReturnBuffer(`docker run --network host ` + `postgres:${version} ` + `bash -c '${bashCmds.join(' && ')}'`);
|
80
79
|
const dumpOut = outputDir.resolve(this.outFileName);
|
81
|
-
this.fs.
|
80
|
+
const f = this.fs.writeStream(dumpOut);
|
81
|
+
await this.sh.asyncExec(`docker run --rm --network host ` + `postgres:${version} ` + `bash -c '${bashCmds.join(' && ')}'`, // f.write
|
82
|
+
data => {
|
83
|
+
f.write(data);
|
84
|
+
});
|
85
|
+
f.end();
|
82
86
|
this.log.info('Compressing');
|
83
87
|
const zipOutputName = `pg-backup-${this.now()}.zip`;
|
84
88
|
this.zip.zipDir(outputDir, outputDir.resolve(zipOutputName));
|
@@ -88,7 +92,7 @@ let PgBackupController = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.m
|
|
88
92
|
});
|
89
93
|
});
|
90
94
|
|
91
|
-
_defineProperty(this, "restore", ({
|
95
|
+
_defineProperty(this, "restore", async ({
|
92
96
|
pgUrl,
|
93
97
|
pgVersion
|
94
98
|
}) => {
|
@@ -97,7 +101,7 @@ let PgBackupController = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.m
|
|
97
101
|
|
98
102
|
parseConnectionUrl(pgUrl);
|
99
103
|
this.log.info(`Restoring Postgres database, version=${version}`);
|
100
|
-
this.fs.inTmpDir('pg-restore', dirStr => {
|
104
|
+
await this.fs.inTmpDir('pg-restore', dirStr => {
|
101
105
|
const dir = _Path.AbsPath.from(dirStr);
|
102
106
|
|
103
107
|
storage.restore(dir); // TODO check if the dir contains the with the expected name
|
package/lib/cli/app.js
CHANGED
@@ -101,14 +101,22 @@ const createApp = () => _nclif.CliApp.of({
|
|
101
101
|
|
102
102
|
exports.createApp = createApp;
|
103
103
|
const restic = (0, _nclif.cmdGroup)({
|
104
|
+
options: singleStorageOptions,
|
104
105
|
commands: {
|
105
106
|
'init-repo': (0, _nclif.cmd)({
|
106
|
-
options: singleStorageOptions,
|
107
107
|
run: (_, c) => c.get(_ResticController.ResticController).initRepo()
|
108
108
|
}),
|
109
109
|
snapshots: (0, _nclif.cmd)({
|
110
|
-
options: singleStorageOptions,
|
111
110
|
run: (_, c, p) => c.get(_ResticController.ResticController).listSnapshots(p.stdout)
|
111
|
+
}),
|
112
|
+
cmd: (0, _nclif.cmd)({
|
113
|
+
positionals: [{
|
114
|
+
name: 'cmd',
|
115
|
+
required: true
|
116
|
+
}],
|
117
|
+
run: ({
|
118
|
+
cmd
|
119
|
+
}, c, p) => c.get(_ResticController.ResticController).runResticCmd(cmd, p.stdout)
|
112
120
|
})
|
113
121
|
}
|
114
122
|
});
|
package/lib/fs/Fs.js
CHANGED
@@ -37,6 +37,11 @@ let Fs = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.metadata("design:
|
|
37
37
|
_fs.default.writeFileSync(path.str(), content);
|
38
38
|
});
|
39
39
|
|
40
|
+
_defineProperty(this, "writeStream", path => {
|
41
|
+
this.log.debug('Writing into file', path);
|
42
|
+
return _fs.default.createWriteStream(path.str());
|
43
|
+
});
|
44
|
+
|
40
45
|
_defineProperty(this, "isFile", path => !this.isDir(path));
|
41
46
|
|
42
47
|
_defineProperty(this, "ensureIsFile", p => {
|
@@ -59,11 +64,11 @@ let Fs = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.metadata("design:
|
|
59
64
|
return _fs.default.readdirSync(path.str()).map(f => path.resolve(f));
|
60
65
|
});
|
61
66
|
|
62
|
-
_defineProperty(this, "inTmpDir", (name, withDir) => {
|
67
|
+
_defineProperty(this, "inTmpDir", async (name, withDir) => {
|
63
68
|
const dir = this.mkTmpDir(name);
|
64
69
|
|
65
70
|
try {
|
66
|
-
withDir(dir);
|
71
|
+
await withDir(dir);
|
67
72
|
} finally {
|
68
73
|
this.rmDir(_Path.AbsPath.from(dir));
|
69
74
|
}
|
@@ -0,0 +1,20 @@
|
|
1
|
+
"use strict";
|
2
|
+
|
3
|
+
var _Fs = require("./Fs");
|
4
|
+
|
5
|
+
var _Path = require("./path/Path");
|
6
|
+
|
7
|
+
var _AppLogger = require("../log/AppLogger");
|
8
|
+
|
9
|
+
it.skip('withTmpDir', async () => {
|
10
|
+
const fs = new _Fs.Fs(_Path.AbsPath.from(__dirname), new _AppLogger.AppLogger());
|
11
|
+
await fs.inTmpDir('foo', async () => {
|
12
|
+
console.log('before');
|
13
|
+
const wait = new Promise((res, rej) => {
|
14
|
+
setTimeout(res, 1000);
|
15
|
+
});
|
16
|
+
await wait;
|
17
|
+
console.log('after');
|
18
|
+
});
|
19
|
+
console.log('done');
|
20
|
+
});
|
@@ -34,6 +34,12 @@ class ResticClient {
|
|
34
34
|
});
|
35
35
|
});
|
36
36
|
|
37
|
+
_defineProperty(this, "runCmd", cmd => {
|
38
|
+
return this.shell.execAndReturnString(`restic ${cmd}`, {
|
39
|
+
env: this.env()
|
40
|
+
});
|
41
|
+
});
|
42
|
+
|
37
43
|
_defineProperty(this, "backup", (cwd, from, tags = []) => {
|
38
44
|
this.log.debug(`Running backup for repo=${this.url}`);
|
39
45
|
let cmd = `restic backup ${from.str()}`;
|
@@ -56,12 +62,12 @@ class ResticClient {
|
|
56
62
|
_defineProperty(this, "forget", () => {
|
57
63
|
this.log.debug(`Pruning repo=${this.url}`);
|
58
64
|
const cfg = {
|
59
|
-
hourly:
|
65
|
+
hourly: 6,
|
60
66
|
daily: 7,
|
61
67
|
weekly: 8,
|
62
|
-
monthly:
|
68
|
+
monthly: 12
|
63
69
|
};
|
64
|
-
this.shell.exec(`restic forget --prune ` + `--
|
70
|
+
this.shell.exec(`restic forget --prune ` + `--group-by tags ` + // The paths are often different when using tmp dirs
|
65
71
|
`--keep-hourly ${cfg.hourly} --keep-daily ${cfg.daily} ` + `--keep-weekly ${cfg.weekly} --keep-monthly ${cfg.monthly}`, {
|
66
72
|
env: this.env()
|
67
73
|
});
|
@@ -36,11 +36,12 @@ let ResticController = (_dec = (0, _inversify.injectable)(), _dec2 = function (t
|
|
36
36
|
_defineProperty(this, "client", () => {
|
37
37
|
const props = this.storageConfigProvider.provide();
|
38
38
|
if (props.type !== _types.StorageType.Restic) throw new Error('Storage is not Restic storage');
|
39
|
-
this.log.info('Initializing repo', props.repo);
|
40
39
|
return this.restFact.createClient(props);
|
41
40
|
});
|
42
41
|
|
43
42
|
_defineProperty(this, "initRepo", () => {
|
43
|
+
const props = this.storageConfigProvider.provide();
|
44
|
+
this.log.info(`Initializing Restic repo ${props.repo}`);
|
44
45
|
this.client().prepareRepo();
|
45
46
|
});
|
46
47
|
|
@@ -48,6 +49,11 @@ let ResticController = (_dec = (0, _inversify.injectable)(), _dec2 = function (t
|
|
48
49
|
const res = this.client().snapshots();
|
49
50
|
print(JSON.stringify(res, null, 4));
|
50
51
|
});
|
52
|
+
|
53
|
+
_defineProperty(this, "runResticCmd", (cmd, print) => {
|
54
|
+
const res = this.client().runCmd(cmd);
|
55
|
+
print(res.toString());
|
56
|
+
});
|
51
57
|
}
|
52
58
|
|
53
59
|
}) || _class) || _class) || _class) || _class);
|
package/lib/tools/shell/Shell.js
CHANGED
@@ -11,13 +11,16 @@ var _inversify = require("inversify");
|
|
11
11
|
|
12
12
|
var _AppLogger = require("../../log/AppLogger");
|
13
13
|
|
14
|
+
var _child_process = require("child_process");
|
15
|
+
|
14
16
|
var _dec, _dec2, _dec3, _class;
|
15
17
|
|
16
18
|
function _defineProperty(obj, key, value) { if (key in obj) { Object.defineProperty(obj, key, { value: value, enumerable: true, configurable: true, writable: true }); } else { obj[key] = value; } return obj; }
|
17
19
|
|
18
20
|
// TODO copy pasted from my other project
|
21
|
+
// TODO don't log the full command as it contains secrets
|
19
22
|
let Shell = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.metadata("design:type", Function), _dec3 = Reflect.metadata("design:paramtypes", [typeof _AppLogger.AppLogger === "undefined" ? Object : _AppLogger.AppLogger]), _dec(_class = _dec2(_class = _dec3(_class = class Shell {
|
20
|
-
constructor(log) {
|
23
|
+
constructor(log = (0, _AppLogger.logger)('shell')) {
|
21
24
|
this.log = log;
|
22
25
|
|
23
26
|
_defineProperty(this, "exec", (cmd, opts = {}) => {
|
@@ -38,19 +41,31 @@ let Shell = (_dec = (0, _inversify.injectable)(), _dec2 = Reflect.metadata("desi
|
|
38
41
|
});
|
39
42
|
|
40
43
|
_defineProperty(this, "execAndReturnString", (cmd, ops = {}) => {
|
41
|
-
this.log.debug(cmd, '[stdout consumed]')
|
44
|
+
// this.log.debug(cmd, '[stdout consumed]')
|
42
45
|
return (0, _shellCmd.shellCmd)(cmd, {
|
43
46
|
returnStdout: 'string',
|
44
47
|
...ops
|
45
48
|
});
|
46
49
|
});
|
47
50
|
|
48
|
-
_defineProperty(this, "
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
51
|
+
_defineProperty(this, "asyncExec", (cmd, onStdout, ops = {}) => {
|
52
|
+
let onDone;
|
53
|
+
let onError;
|
54
|
+
const p = new Promise((res, rej) => {
|
55
|
+
onDone = res;
|
56
|
+
onError = rej;
|
57
|
+
});
|
58
|
+
const r = (0, _child_process.spawn)(cmd, {
|
59
|
+
shell: true,
|
60
|
+
cwd: ops.cwd,
|
61
|
+
env: ops.env
|
62
|
+
});
|
63
|
+
r.stdout.on('data', onStdout);
|
64
|
+
r.stderr.on('data', data => console.error(data.toString()));
|
65
|
+
r.on('exit', status => {
|
66
|
+
if (status === 0) onDone();else onError(new Error('Exited with non zero return code: ' + status));
|
53
67
|
});
|
68
|
+
return p;
|
54
69
|
});
|
55
70
|
}
|
56
71
|
|
@@ -0,0 +1,21 @@
|
|
1
|
+
"use strict";
|
2
|
+
|
3
|
+
var _Shell = require("./Shell");
|
4
|
+
|
5
|
+
const bigStdOutCme = () => {
|
6
|
+
const times = 5_00_000;
|
7
|
+
return `bash -c 'for i in {1..${times}}; do echo "Looong data"; done; echo Done'`;
|
8
|
+
}; // Not to be run on CI
|
9
|
+
|
10
|
+
|
11
|
+
it.skip('execAndReturnStream', async () => {
|
12
|
+
const sh = new _Shell.Shell(); // The output is too big for the exec buffer
|
13
|
+
|
14
|
+
expect(() => sh.execAndReturnString(bigStdOutCme())).toThrow('spawnSync /bin/sh ENOBUFS'); //
|
15
|
+
|
16
|
+
let out = '';
|
17
|
+
await sh.asyncExec(bigStdOutCme(), data => {
|
18
|
+
out = data.toString().trim();
|
19
|
+
});
|
20
|
+
expect(out).toEndWith('Done');
|
21
|
+
}, 5000_000);
|