@cloudant/couchbackup 2.7.1-SNAPSHOT.55 → 2.7.1-SNAPSHOT.59
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.secrets.baseline +4 -4
- package/CHANGES.md +3 -0
- package/app.js +7 -7
- package/bin/couchbackup.bin.js +5 -5
- package/bin/couchrestore.bin.js +3 -3
- package/includes/backup.js +37 -27
- package/includes/change.js +3 -3
- package/includes/config.js +3 -3
- package/includes/liner.js +5 -5
- package/includes/logfilegetbatches.js +10 -7
- package/includes/logfilesummary.js +9 -9
- package/includes/parser.js +2 -2
- package/includes/request.js +1 -1
- package/includes/restore.js +2 -2
- package/includes/shallowbackup.js +6 -6
- package/includes/spoolchanges.js +52 -33
- package/includes/writer.js +7 -7
- package/package.json +5 -9
package/.secrets.baseline
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
"files": "package-lock.json|test/fixtures|^.secrets.baseline$",
|
|
4
4
|
"lines": null
|
|
5
5
|
},
|
|
6
|
-
"generated_at": "2021-11-
|
|
6
|
+
"generated_at": "2021-11-16T14:34:16Z",
|
|
7
7
|
"plugins_used": [
|
|
8
8
|
{
|
|
9
9
|
"name": "AWSKeyDetector"
|
|
@@ -82,7 +82,7 @@
|
|
|
82
82
|
"hashed_secret": "90c240c0ecfb254c589319f022de059fc739c54a",
|
|
83
83
|
"is_secret": false,
|
|
84
84
|
"is_verified": false,
|
|
85
|
-
"line_number":
|
|
85
|
+
"line_number": 152,
|
|
86
86
|
"type": "NPM tokens",
|
|
87
87
|
"verified_result": null
|
|
88
88
|
}
|
|
@@ -200,7 +200,7 @@
|
|
|
200
200
|
"hashed_secret": "9d4e1e23bd5b727046a9e3b4b7db57bd8d6ee684",
|
|
201
201
|
"is_secret": false,
|
|
202
202
|
"is_verified": false,
|
|
203
|
-
"line_number":
|
|
203
|
+
"line_number": 143,
|
|
204
204
|
"type": "Basic Auth Credentials",
|
|
205
205
|
"verified_result": null
|
|
206
206
|
},
|
|
@@ -208,7 +208,7 @@
|
|
|
208
208
|
"hashed_secret": "e9d71f5ee7c92d6dc9e92ffdad17b8bd49418f98",
|
|
209
209
|
"is_secret": false,
|
|
210
210
|
"is_verified": false,
|
|
211
|
-
"line_number":
|
|
211
|
+
"line_number": 305,
|
|
212
212
|
"type": "Basic Auth Credentials",
|
|
213
213
|
"verified_result": null
|
|
214
214
|
}
|
package/CHANGES.md
CHANGED
|
@@ -3,6 +3,9 @@
|
|
|
3
3
|
- [FIXED] Restore of shallow backups created with versions <=2.4.2.
|
|
4
4
|
- [IMPROVED] Added quiet option to backup and restore to suppress batch messages.
|
|
5
5
|
- [IMPROVED] Added a preflight check for restore function to make sure that a target database is new and empty.
|
|
6
|
+
- [IMPROVED] Added handling for errors reading log file.
|
|
7
|
+
- [IMPROVED] Split changes spooling to improve reliability on databases with
|
|
8
|
+
millions of documents.
|
|
6
9
|
|
|
7
10
|
# 2.7.0 (2021-09-14)
|
|
8
11
|
- [UPGRADED] Cloudant client dependency from `@cloudant/cloudant` to `@ibm-cloud/cloudant`.
|
package/app.js
CHANGED
|
@@ -178,8 +178,8 @@ function proceedIfRestoreDbValid(db, callback) {
|
|
|
178
178
|
// The system databases can have a validation ddoc(s) injected in them on creation.
|
|
179
179
|
// This sets the doc count off, so we just complitely exclude the system databases from this check.
|
|
180
180
|
// The assumption here is that users restoring system databases know what they are doing.
|
|
181
|
-
if (!db.db.startsWith(
|
|
182
|
-
|
|
181
|
+
if (!db.db.startsWith('_') && (docCount !== 0 || deletedDocCount !== 0)) {
|
|
182
|
+
const notEmptyDBErr = new Error(`Target database ${db.url}${db.db} is not empty.`);
|
|
183
183
|
notEmptyDBErr.name = 'DatabaseNotEmpty';
|
|
184
184
|
callback(notEmptyDBErr);
|
|
185
185
|
} else {
|
|
@@ -200,10 +200,10 @@ function proceedIfRestoreDbValid(db, callback) {
|
|
|
200
200
|
function parseIfDbValidResponseError(db, err) {
|
|
201
201
|
if (err && err.status === 404) {
|
|
202
202
|
// Override the error type and message for the DB not found case
|
|
203
|
-
|
|
203
|
+
const msg = `Database ${db.url}` +
|
|
204
204
|
`${db.db} does not exist. ` +
|
|
205
205
|
'Check the URL and database name have been specified correctly.';
|
|
206
|
-
|
|
206
|
+
const noDBErr = new Error(msg);
|
|
207
207
|
noDBErr.name = 'DatabaseNotFound';
|
|
208
208
|
return noDBErr;
|
|
209
209
|
}
|
|
@@ -229,7 +229,7 @@ module.exports = {
|
|
|
229
229
|
* @param {backupRestoreCallback} callback - Called on completion.
|
|
230
230
|
*/
|
|
231
231
|
backup: function(srcUrl, targetStream, opts, callback) {
|
|
232
|
-
|
|
232
|
+
const listenerErrorIndicator = { errored: false };
|
|
233
233
|
if (typeof callback === 'undefined' && typeof opts === 'function') {
|
|
234
234
|
callback = opts;
|
|
235
235
|
opts = {};
|
|
@@ -263,7 +263,7 @@ module.exports = {
|
|
|
263
263
|
callback(err);
|
|
264
264
|
return;
|
|
265
265
|
}
|
|
266
|
-
|
|
266
|
+
let backup = null;
|
|
267
267
|
if (opts.mode === 'shallow') {
|
|
268
268
|
backup = backupShallow;
|
|
269
269
|
} else { // full mode
|
|
@@ -348,7 +348,7 @@ module.exports = {
|
|
|
348
348
|
* @param {backupRestoreCallback} callback - Called on completion.
|
|
349
349
|
*/
|
|
350
350
|
restore: function(srcStream, targetUrl, opts, callback) {
|
|
351
|
-
|
|
351
|
+
const listenerErrorIndicator = { errored: false };
|
|
352
352
|
if (typeof callback === 'undefined' && typeof opts === 'function') {
|
|
353
353
|
callback = opts;
|
|
354
354
|
opts = {};
|
package/bin/couchbackup.bin.js
CHANGED
|
@@ -25,10 +25,10 @@ const backupBatchDebug = debug('couchbackup:backup:batch');
|
|
|
25
25
|
|
|
26
26
|
backupDebug.enabled = true;
|
|
27
27
|
|
|
28
|
-
|
|
28
|
+
const program = parser.parseBackupArgs();
|
|
29
29
|
|
|
30
|
-
|
|
31
|
-
|
|
30
|
+
const databaseUrl = cliutils.databaseUrl(program.url, program.db);
|
|
31
|
+
const opts = {
|
|
32
32
|
bufferSize: program.bufferSize,
|
|
33
33
|
log: program.log,
|
|
34
34
|
mode: program.mode,
|
|
@@ -47,11 +47,11 @@ console.error('='.repeat(80));
|
|
|
47
47
|
|
|
48
48
|
backupBatchDebug.enabled = !program.quiet;
|
|
49
49
|
|
|
50
|
-
|
|
50
|
+
let ws = process.stdout;
|
|
51
51
|
|
|
52
52
|
// open output file
|
|
53
53
|
if (program.output) {
|
|
54
|
-
|
|
54
|
+
let flags = 'w';
|
|
55
55
|
if (program.log && program.resume) {
|
|
56
56
|
flags = 'a';
|
|
57
57
|
}
|
package/bin/couchrestore.bin.js
CHANGED
|
@@ -24,9 +24,9 @@ const restoreBatchDebug = debug('couchbackup:restore:batch');
|
|
|
24
24
|
|
|
25
25
|
restoreDebug.enabled = true;
|
|
26
26
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
27
|
+
const program = parser.parseRestoreArgs();
|
|
28
|
+
const databaseUrl = cliutils.databaseUrl(program.url, program.db);
|
|
29
|
+
const opts = {
|
|
30
30
|
bufferSize: program.bufferSize,
|
|
31
31
|
parallelism: program.parallelism,
|
|
32
32
|
requestTimeout: program.requestTimeout,
|
package/includes/backup.js
CHANGED
|
@@ -105,8 +105,8 @@ function validateBulkGetSupport(db, callback) {
|
|
|
105
105
|
* (err, {batches: batch, docs: doccount}) {@see spoolchanges}.
|
|
106
106
|
*/
|
|
107
107
|
function downloadRemainingBatches(log, db, ee, startTime, batchesPerDownloadSession, parallelism) {
|
|
108
|
-
|
|
109
|
-
|
|
108
|
+
let total = 0; // running total of documents downloaded so far
|
|
109
|
+
let noRemainingBatches = false;
|
|
110
110
|
|
|
111
111
|
// Generate a set of batches (up to batchesPerDownloadSession) to download from the
|
|
112
112
|
// log file and download them. Set noRemainingBatches to `true` for last batch.
|
|
@@ -114,12 +114,18 @@ function downloadRemainingBatches(log, db, ee, startTime, batchesPerDownloadSess
|
|
|
114
114
|
// Fetch the doc IDs for the batches in the current set to
|
|
115
115
|
// download them.
|
|
116
116
|
function batchSetComplete(err, data) {
|
|
117
|
-
|
|
118
|
-
|
|
117
|
+
if (!err) {
|
|
118
|
+
total = data.total;
|
|
119
|
+
}
|
|
120
|
+
done(err);
|
|
119
121
|
}
|
|
120
122
|
function processRetrievedBatches(err, batches) {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
+
if (!err) {
|
|
124
|
+
// process them in parallelised queue
|
|
125
|
+
processBatchSet(db, parallelism, log, batches, ee, startTime, total, batchSetComplete);
|
|
126
|
+
} else {
|
|
127
|
+
batchSetComplete(err);
|
|
128
|
+
}
|
|
123
129
|
}
|
|
124
130
|
|
|
125
131
|
readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, function(err, batchSetIds) {
|
|
@@ -157,18 +163,22 @@ function downloadRemainingBatches(log, db, ee, startTime, batchesPerDownloadSess
|
|
|
157
163
|
*/
|
|
158
164
|
function readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, callback) {
|
|
159
165
|
logfilesummary(log, function processSummary(err, summary) {
|
|
160
|
-
if (!
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
166
|
+
if (!err) {
|
|
167
|
+
if (!summary.changesComplete) {
|
|
168
|
+
callback(new error.BackupError('IncompleteChangesInLogFile',
|
|
169
|
+
'WARNING: Changes did not finish spooling'));
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
if (Object.keys(summary.batches).length === 0) {
|
|
173
|
+
return callback(null, []);
|
|
174
|
+
}
|
|
168
175
|
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
176
|
+
// batch IDs are the property names of summary.batches
|
|
177
|
+
const batchSetIds = getPropertyNames(summary.batches, batchesPerDownloadSession);
|
|
178
|
+
callback(null, batchSetIds);
|
|
179
|
+
} else {
|
|
180
|
+
callback(err);
|
|
181
|
+
}
|
|
172
182
|
});
|
|
173
183
|
}
|
|
174
184
|
|
|
@@ -188,13 +198,13 @@ function readBatchSetIdsFromLogFile(log, batchesPerDownloadSession, callback) {
|
|
|
188
198
|
* @param {any} callback - completion callback, (err, {total: number}).
|
|
189
199
|
*/
|
|
190
200
|
function processBatchSet(db, parallelism, log, batches, ee, start, grandtotal, callback) {
|
|
191
|
-
|
|
192
|
-
|
|
201
|
+
let hasErrored = false;
|
|
202
|
+
let total = grandtotal;
|
|
193
203
|
|
|
194
204
|
// queue to process the fetch requests in an orderly fashion using _bulk_get
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
205
|
+
const q = async.queue(function(payload, done) {
|
|
206
|
+
const output = [];
|
|
207
|
+
const thisBatch = payload.batch;
|
|
198
208
|
delete payload.batch;
|
|
199
209
|
delete payload.command;
|
|
200
210
|
|
|
@@ -223,7 +233,7 @@ function processBatchSet(db, parallelism, log, batches, ee, start, grandtotal, c
|
|
|
223
233
|
}
|
|
224
234
|
});
|
|
225
235
|
total += output.length;
|
|
226
|
-
|
|
236
|
+
const t = (new Date().getTime() - start) / 1000;
|
|
227
237
|
ee.emit('received', {
|
|
228
238
|
batch: thisBatch,
|
|
229
239
|
data: output,
|
|
@@ -243,7 +253,7 @@ function processBatchSet(db, parallelism, log, batches, ee, start, grandtotal, c
|
|
|
243
253
|
});
|
|
244
254
|
}, parallelism);
|
|
245
255
|
|
|
246
|
-
for (
|
|
256
|
+
for (const i in batches) {
|
|
247
257
|
q.push(batches[i]);
|
|
248
258
|
}
|
|
249
259
|
|
|
@@ -260,9 +270,9 @@ function processBatchSet(db, parallelism, log, batches, ee, start, grandtotal, c
|
|
|
260
270
|
*/
|
|
261
271
|
function getPropertyNames(obj, count) {
|
|
262
272
|
// decide which batch numbers to deal with
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
for (
|
|
273
|
+
const batchestofetch = [];
|
|
274
|
+
let j = 0;
|
|
275
|
+
for (const i in obj) {
|
|
266
276
|
batchestofetch.push(parseInt(i));
|
|
267
277
|
j++;
|
|
268
278
|
if (j >= count) break;
|
package/includes/change.js
CHANGED
|
@@ -14,13 +14,13 @@
|
|
|
14
14
|
'use strict';
|
|
15
15
|
|
|
16
16
|
// stolen from http://strongloop.com/strongblog/practical-examples-of-the-new-node-js-streams-api/
|
|
17
|
-
|
|
17
|
+
const stream = require('stream');
|
|
18
18
|
|
|
19
19
|
module.exports = function(onChange) {
|
|
20
|
-
|
|
20
|
+
const change = new stream.Transform({ objectMode: true });
|
|
21
21
|
|
|
22
22
|
change._transform = function(line, encoding, done) {
|
|
23
|
-
|
|
23
|
+
let obj = null;
|
|
24
24
|
|
|
25
25
|
// one change per line - remove the trailing comma
|
|
26
26
|
line = line.trim().replace(/,$/, '');
|
package/includes/config.js
CHANGED
|
@@ -13,8 +13,8 @@
|
|
|
13
13
|
// limitations under the License.
|
|
14
14
|
'use strict';
|
|
15
15
|
|
|
16
|
-
|
|
17
|
-
|
|
16
|
+
const path = require('path');
|
|
17
|
+
const tmp = require('tmp');
|
|
18
18
|
|
|
19
19
|
/**
|
|
20
20
|
Return API default settings.
|
|
@@ -34,7 +34,7 @@ function apiDefaults() {
|
|
|
34
34
|
Return CLI default settings.
|
|
35
35
|
*/
|
|
36
36
|
function cliDefaults() {
|
|
37
|
-
|
|
37
|
+
const defaults = apiDefaults();
|
|
38
38
|
|
|
39
39
|
// add additional legacy settings
|
|
40
40
|
defaults.db = 'test';
|
package/includes/liner.js
CHANGED
|
@@ -14,21 +14,21 @@
|
|
|
14
14
|
'use strict';
|
|
15
15
|
|
|
16
16
|
// stolen from http://strongloop.com/strongblog/practical-examples-of-the-new-node-js-streams-api/
|
|
17
|
-
|
|
17
|
+
const stream = require('stream');
|
|
18
18
|
|
|
19
19
|
module.exports = function() {
|
|
20
|
-
|
|
20
|
+
const liner = new stream.Transform({ objectMode: true });
|
|
21
21
|
|
|
22
22
|
liner._transform = function(chunk, encoding, done) {
|
|
23
|
-
|
|
23
|
+
let data = chunk.toString();
|
|
24
24
|
if (this._lastLineData) {
|
|
25
25
|
data = this._lastLineData + data;
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
-
|
|
28
|
+
const lines = data.split('\n');
|
|
29
29
|
this._lastLineData = lines.splice(lines.length - 1, 1)[0];
|
|
30
30
|
|
|
31
|
-
for (
|
|
31
|
+
for (const i in lines) {
|
|
32
32
|
this.push(lines[i]);
|
|
33
33
|
}
|
|
34
34
|
done();
|
|
@@ -17,17 +17,17 @@ const fs = require('fs');
|
|
|
17
17
|
const stream = require('stream');
|
|
18
18
|
const liner = require('./liner.js');
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
const onLine = function(onCommand, batches) {
|
|
21
|
+
const change = new stream.Transform({ objectMode: true });
|
|
22
22
|
change._transform = function(line, encoding, done) {
|
|
23
23
|
if (line && line[0] === ':') {
|
|
24
|
-
|
|
24
|
+
const obj = {
|
|
25
25
|
command: null,
|
|
26
26
|
batch: null,
|
|
27
27
|
docs: []
|
|
28
28
|
};
|
|
29
29
|
|
|
30
|
-
|
|
30
|
+
let matches;
|
|
31
31
|
|
|
32
32
|
// extract command
|
|
33
33
|
matches = line.match(/^:([a-z_]+) ?/);
|
|
@@ -43,7 +43,7 @@ var onLine = function(onCommand, batches) {
|
|
|
43
43
|
|
|
44
44
|
// if this is one we want
|
|
45
45
|
if (obj.command === 't' && batches.indexOf(obj.batch) > -1) {
|
|
46
|
-
|
|
46
|
+
const json = line.replace(/^.* batch[0-9]+ /, '').trim();
|
|
47
47
|
obj.docs = JSON.parse(json);
|
|
48
48
|
onCommand(obj);
|
|
49
49
|
}
|
|
@@ -55,10 +55,10 @@ var onLine = function(onCommand, batches) {
|
|
|
55
55
|
|
|
56
56
|
module.exports = function(log, batches, callback) {
|
|
57
57
|
// our sense of state
|
|
58
|
-
|
|
58
|
+
const retval = { };
|
|
59
59
|
|
|
60
60
|
// called with each line from the log file
|
|
61
|
-
|
|
61
|
+
const onCommand = function(obj) {
|
|
62
62
|
retval[obj.batch] = obj;
|
|
63
63
|
};
|
|
64
64
|
|
|
@@ -66,6 +66,9 @@ module.exports = function(log, batches, callback) {
|
|
|
66
66
|
fs.createReadStream(log)
|
|
67
67
|
.pipe(liner())
|
|
68
68
|
.pipe(onLine(onCommand, batches))
|
|
69
|
+
.on('error', function(err) {
|
|
70
|
+
callback(err);
|
|
71
|
+
})
|
|
69
72
|
.on('finish', function() {
|
|
70
73
|
callback(null, retval);
|
|
71
74
|
});
|
|
@@ -17,18 +17,18 @@ const fs = require('fs');
|
|
|
17
17
|
const stream = require('stream');
|
|
18
18
|
const liner = require('./liner.js');
|
|
19
19
|
|
|
20
|
-
|
|
21
|
-
|
|
20
|
+
const onLine = function(onCommand, getDocs) {
|
|
21
|
+
const change = new stream.Transform({ objectMode: true });
|
|
22
22
|
|
|
23
23
|
change._transform = function(line, encoding, done) {
|
|
24
24
|
if (line && line[0] === ':') {
|
|
25
|
-
|
|
25
|
+
const obj = {
|
|
26
26
|
command: null,
|
|
27
27
|
batch: null,
|
|
28
28
|
docs: []
|
|
29
29
|
};
|
|
30
30
|
|
|
31
|
-
|
|
31
|
+
let matches;
|
|
32
32
|
|
|
33
33
|
// extract command
|
|
34
34
|
matches = line.match(/^:([a-z_]+) ?/);
|
|
@@ -44,7 +44,7 @@ var onLine = function(onCommand, getDocs) {
|
|
|
44
44
|
|
|
45
45
|
// extract doc ids
|
|
46
46
|
if (getDocs && obj.command === 't') {
|
|
47
|
-
|
|
47
|
+
const json = line.replace(/^.* batch[0-9]+ /, '').trim();
|
|
48
48
|
obj.docs = JSON.parse(json);
|
|
49
49
|
}
|
|
50
50
|
onCommand(obj);
|
|
@@ -65,13 +65,13 @@ var onLine = function(onCommand, getDocs) {
|
|
|
65
65
|
*/
|
|
66
66
|
module.exports = function(log, callback) {
|
|
67
67
|
// our sense of state
|
|
68
|
-
|
|
68
|
+
const state = {
|
|
69
69
|
|
|
70
70
|
};
|
|
71
|
-
|
|
71
|
+
let changesComplete = false;
|
|
72
72
|
|
|
73
73
|
// called with each line from the log file
|
|
74
|
-
|
|
74
|
+
const onCommand = function(obj) {
|
|
75
75
|
if (obj.command === 't') {
|
|
76
76
|
state[obj.batch] = true;
|
|
77
77
|
} else if (obj.command === 'd') {
|
|
@@ -86,7 +86,7 @@ module.exports = function(log, callback) {
|
|
|
86
86
|
.pipe(liner())
|
|
87
87
|
.pipe(onLine(onCommand, false))
|
|
88
88
|
.on('finish', function() {
|
|
89
|
-
|
|
89
|
+
const obj = { changesComplete: changesComplete, batches: state };
|
|
90
90
|
callback(null, obj);
|
|
91
91
|
});
|
|
92
92
|
};
|
package/includes/parser.js
CHANGED
|
@@ -20,7 +20,7 @@ const path = require('path');
|
|
|
20
20
|
const pkg = require('../package.json');
|
|
21
21
|
|
|
22
22
|
function parseBackupArgs() {
|
|
23
|
-
|
|
23
|
+
const program = require('commander');
|
|
24
24
|
|
|
25
25
|
// Option CLI defaults
|
|
26
26
|
const defaults = config.cliDefaults();
|
|
@@ -84,7 +84,7 @@ function parseBackupArgs() {
|
|
|
84
84
|
}
|
|
85
85
|
|
|
86
86
|
function parseRestoreArgs() {
|
|
87
|
-
|
|
87
|
+
const program = require('commander');
|
|
88
88
|
|
|
89
89
|
// Option CLI defaults
|
|
90
90
|
const defaults = config.cliDefaults();
|
package/includes/request.js
CHANGED
|
@@ -102,7 +102,7 @@ const errorHelper = async function(err) {
|
|
|
102
102
|
module.exports = {
|
|
103
103
|
client: function(rawUrl, opts) {
|
|
104
104
|
const url = new URL(rawUrl);
|
|
105
|
-
|
|
105
|
+
const protocol = (url.protocol.match(/^https/)) ? https : http;
|
|
106
106
|
const keepAliveAgent = new protocol.Agent({
|
|
107
107
|
keepAlive: true,
|
|
108
108
|
keepAliveMsecs: 30000,
|
package/includes/restore.js
CHANGED
|
@@ -14,8 +14,8 @@
|
|
|
14
14
|
'use strict';
|
|
15
15
|
|
|
16
16
|
module.exports = function(db, options, readstream, ee, callback) {
|
|
17
|
-
|
|
18
|
-
|
|
17
|
+
const liner = require('../includes/liner.js')();
|
|
18
|
+
const writer = require('../includes/writer.js')(db, options.bufferSize, options.parallelism, ee);
|
|
19
19
|
|
|
20
20
|
// pipe the input to the output, via transformation functions
|
|
21
21
|
readstream
|
|
@@ -20,16 +20,16 @@ const events = require('events');
|
|
|
20
20
|
module.exports = function(db, options) {
|
|
21
21
|
const ee = new events.EventEmitter();
|
|
22
22
|
const start = new Date().getTime();
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
23
|
+
let batch = 0;
|
|
24
|
+
let hasErrored = false;
|
|
25
|
+
let startKey = null;
|
|
26
|
+
let total = 0;
|
|
27
27
|
|
|
28
28
|
async.doUntil(
|
|
29
29
|
function(callback) {
|
|
30
30
|
// Note, include_docs: true is set automatically when using the
|
|
31
31
|
// fetch function.
|
|
32
|
-
|
|
32
|
+
const opts = { db: db.db, limit: options.bufferSize, includeDocs: true };
|
|
33
33
|
|
|
34
34
|
// To avoid double fetching a document solely for the purposes of getting
|
|
35
35
|
// the next ID to use as a startkey for the next page we instead use the
|
|
@@ -49,7 +49,7 @@ module.exports = function(db, options) {
|
|
|
49
49
|
startKey = body.rows[opts.limit - 1].id;
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
-
|
|
52
|
+
const docs = [];
|
|
53
53
|
body.rows.forEach(function(doc) {
|
|
54
54
|
docs.push(doc.doc);
|
|
55
55
|
});
|
package/includes/spoolchanges.js
CHANGED
|
@@ -30,16 +30,19 @@ const debug = require('debug')('couchbackup:spoolchanges');
|
|
|
30
30
|
*/
|
|
31
31
|
module.exports = function(db, log, bufferSize, ee, callback) {
|
|
32
32
|
// list of document ids to process
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
33
|
+
const buffer = [];
|
|
34
|
+
let batch = 0;
|
|
35
|
+
let lastSeq = null;
|
|
36
|
+
const logStream = fs.createWriteStream(log);
|
|
37
|
+
let pending = 0;
|
|
38
|
+
// The number of changes to fetch per request
|
|
39
|
+
const limit = 100000;
|
|
37
40
|
|
|
38
41
|
// send documents ids to the queue in batches of bufferSize + the last batch
|
|
39
|
-
|
|
42
|
+
const processBuffer = function(lastOne) {
|
|
40
43
|
if (buffer.length >= bufferSize || (lastOne && buffer.length > 0)) {
|
|
41
44
|
debug('writing', buffer.length, 'changes to the backup file');
|
|
42
|
-
|
|
45
|
+
const b = { docs: buffer.splice(0, bufferSize), batch: batch };
|
|
43
46
|
logStream.write(':t batch' + batch + ' ' + JSON.stringify(b.docs) + '\n');
|
|
44
47
|
ee.emit('changes', batch);
|
|
45
48
|
batch++;
|
|
@@ -47,46 +50,62 @@ module.exports = function(db, log, bufferSize, ee, callback) {
|
|
|
47
50
|
};
|
|
48
51
|
|
|
49
52
|
// called once per received change
|
|
50
|
-
|
|
53
|
+
const onChange = function(c) {
|
|
51
54
|
if (c) {
|
|
52
55
|
if (c.error) {
|
|
53
56
|
ee.emit('error', new error.BackupError('InvalidChange', `Received invalid change: ${c}`));
|
|
54
57
|
} else if (c.changes) {
|
|
55
|
-
|
|
58
|
+
const obj = { id: c.id };
|
|
56
59
|
buffer.push(obj);
|
|
57
60
|
processBuffer(false);
|
|
58
61
|
} else if (c.last_seq) {
|
|
59
62
|
lastSeq = c.last_seq;
|
|
63
|
+
pending = c.pending;
|
|
60
64
|
}
|
|
61
65
|
}
|
|
62
66
|
};
|
|
63
67
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
.
|
|
68
|
-
|
|
68
|
+
function getChanges(since = 0) {
|
|
69
|
+
debug('making changes request since ' + since);
|
|
70
|
+
return db.service.postChangesAsStream({ db: db.db, since: since, limit: limit, seq_interval: limit })
|
|
71
|
+
.then(response => {
|
|
72
|
+
response.result.pipe(liner())
|
|
73
|
+
.on('error', function(err) {
|
|
74
|
+
logStream.end();
|
|
75
|
+
callback(err);
|
|
76
|
+
})
|
|
77
|
+
.pipe(change(onChange))
|
|
78
|
+
.on('error', function(err) {
|
|
79
|
+
logStream.end();
|
|
80
|
+
callback(err);
|
|
81
|
+
})
|
|
82
|
+
.on('finish', function() {
|
|
83
|
+
processBuffer(true);
|
|
84
|
+
if (!lastSeq) {
|
|
85
|
+
logStream.end();
|
|
86
|
+
debug('changes request terminated before last_seq was sent');
|
|
87
|
+
callback(new error.BackupError('SpoolChangesError', 'Changes request terminated before last_seq was sent'));
|
|
88
|
+
} else {
|
|
89
|
+
debug(`changes request completed with last_seq: ${lastSeq} and ${pending} changes pending.`);
|
|
90
|
+
if (pending > 0) {
|
|
91
|
+
// Return the next promise
|
|
92
|
+
return getChanges(lastSeq);
|
|
93
|
+
} else {
|
|
94
|
+
debug('finished streaming database changes');
|
|
95
|
+
logStream.end(':changes_complete ' + lastSeq + '\n', 'utf8', callback);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
});
|
|
69
99
|
})
|
|
70
|
-
.
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
if (!lastSeq) {
|
|
77
|
-
logStream.end();
|
|
78
|
-
debug('changes request terminated before last_seq was sent');
|
|
79
|
-
callback(new error.BackupError('SpoolChangesError', 'Changes request terminated before last_seq was sent'));
|
|
80
|
-
} else {
|
|
81
|
-
debug('finished streaming database changes');
|
|
82
|
-
logStream.end(':changes_complete ' + lastSeq + '\n', 'utf8', callback);
|
|
100
|
+
.catch(err => {
|
|
101
|
+
logStream.end();
|
|
102
|
+
if (err.status && err.status >= 400) {
|
|
103
|
+
callback(error.convertResponseError(err));
|
|
104
|
+
} else if (err.name !== 'SpoolChangesError') {
|
|
105
|
+
callback(new error.BackupError('SpoolChangesError', `Failed changes request - ${err.message}`));
|
|
83
106
|
}
|
|
84
107
|
});
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
} else {
|
|
89
|
-
callback(new error.BackupError('SpoolChangesError', `Failed changes request - ${err.message}`));
|
|
90
|
-
}
|
|
91
|
-
});
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
getChanges();
|
|
92
111
|
};
|
package/includes/writer.js
CHANGED
|
@@ -20,15 +20,15 @@ const debug = require('debug')('couchbackup:writer');
|
|
|
20
20
|
|
|
21
21
|
module.exports = function(db, bufferSize, parallelism, ee) {
|
|
22
22
|
const writer = new stream.Transform({ objectMode: true });
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
23
|
+
let buffer = [];
|
|
24
|
+
let written = 0;
|
|
25
|
+
let linenumber = 0;
|
|
26
26
|
|
|
27
27
|
// this is the queue of chunks that are written to the database
|
|
28
28
|
// the queue's payload will be an array of documents to be written,
|
|
29
29
|
// the size of the array will be bufferSize. The variable parallelism
|
|
30
30
|
// determines how many HTTP requests will occur at any one time.
|
|
31
|
-
|
|
31
|
+
const q = async.queue(function(payload, cb) {
|
|
32
32
|
// if we are restoring known revisions, we need to supply new_edits=false
|
|
33
33
|
if (payload.docs && payload.docs[0] && payload.docs[0]._rev) {
|
|
34
34
|
payload.new_edits = false;
|
|
@@ -55,7 +55,7 @@ module.exports = function(db, bufferSize, parallelism, ee) {
|
|
|
55
55
|
}
|
|
56
56
|
}, parallelism);
|
|
57
57
|
|
|
58
|
-
|
|
58
|
+
let didError = false;
|
|
59
59
|
|
|
60
60
|
// write the contents of the buffer to CouchDB in blocks of bufferSize
|
|
61
61
|
function processBuffer(flush, callback) {
|
|
@@ -73,7 +73,7 @@ module.exports = function(db, bufferSize, parallelism, ee) {
|
|
|
73
73
|
// and feed the chunks to the queue
|
|
74
74
|
do {
|
|
75
75
|
// split the buffer into bufferSize chunks
|
|
76
|
-
|
|
76
|
+
const toSend = buffer.splice(0, bufferSize);
|
|
77
77
|
|
|
78
78
|
// and add the chunk to the queue
|
|
79
79
|
debug(`Adding ${toSend.length} to the write queue.`);
|
|
@@ -126,7 +126,7 @@ module.exports = function(db, bufferSize, parallelism, ee) {
|
|
|
126
126
|
if (!didError && obj !== '') {
|
|
127
127
|
// see if it parses as JSON
|
|
128
128
|
try {
|
|
129
|
-
|
|
129
|
+
const arr = JSON.parse(obj);
|
|
130
130
|
|
|
131
131
|
// if it's an array with a length
|
|
132
132
|
if (typeof arr === 'object' && arr.length > 0) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cloudant/couchbackup",
|
|
3
|
-
"version": "2.7.1-SNAPSHOT.
|
|
3
|
+
"version": "2.7.1-SNAPSHOT.59",
|
|
4
4
|
"description": "CouchBackup - command-line backup utility for Cloudant/CouchDB",
|
|
5
5
|
"homepage": "https://github.com/cloudant/couchbackup",
|
|
6
6
|
"repository": "https://github.com/cloudant/couchbackup.git",
|
|
@@ -36,13 +36,12 @@
|
|
|
36
36
|
"devDependencies": {
|
|
37
37
|
"eslint": "^8.2.0",
|
|
38
38
|
"eslint-config-semistandard": "^16.0.0",
|
|
39
|
-
"eslint-config-standard": "^
|
|
39
|
+
"eslint-config-standard": "^16.0.3",
|
|
40
40
|
"eslint-plugin-header": "^3.0.0",
|
|
41
41
|
"eslint-plugin-import": "^2.8.0",
|
|
42
42
|
"eslint-plugin-node": "^11.0.0",
|
|
43
43
|
"eslint-plugin-promise": "^4.0.0",
|
|
44
44
|
"eslint-plugin-react": "^7.14.2",
|
|
45
|
-
"eslint-plugin-standard": "^5.0.0",
|
|
46
45
|
"http-proxy": "^1.16.2",
|
|
47
46
|
"jsdoc": "^3.6.7",
|
|
48
47
|
"mocha": "^9.1.3",
|
|
@@ -52,11 +51,8 @@
|
|
|
52
51
|
"uuid": "^8.3.2"
|
|
53
52
|
},
|
|
54
53
|
"scripts": {
|
|
55
|
-
"
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
"ignore": [
|
|
59
|
-
"eslint"
|
|
60
|
-
]
|
|
54
|
+
"lint": "eslint --ignore-path .gitignore .",
|
|
55
|
+
"unit": "mocha --grep \"#unit\"",
|
|
56
|
+
"test": "npm run lint && npm run unit"
|
|
61
57
|
}
|
|
62
58
|
}
|