@cloudant/couchbackup 2.6.2-SNAPSHOT.21 → 2.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGES.md +4 -1
- package/README.md +5 -4
- package/app.js +6 -7
- package/includes/backup.js +44 -51
- package/includes/error.js +10 -18
- package/includes/request.js +150 -23
- package/includes/shallowbackup.js +10 -9
- package/includes/spoolchanges.js +27 -30
- package/includes/writer.js +16 -53
- package/package.json +4 -2
package/CHANGES.md
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
|
-
#
|
|
1
|
+
# 2.7.0 (2021-09-14)
|
|
2
|
+
- [UPGRADED] Cloudant client dependency from `@cloudant/cloudant` to `@ibm-cloud/cloudant`.
|
|
3
|
+
|
|
4
|
+
# 2.6.2 (2021-08-27)
|
|
2
5
|
- [FIXED] `Invalid document ID: _bulk_get` error when using `@cloudant/cloudant`
|
|
3
6
|
version `4.5.0`.
|
|
4
7
|
- [UPGRADED] Upgraded `@cloudant/cloudant` dependency to version `4.5.0`.
|
package/README.md
CHANGED
|
@@ -122,7 +122,7 @@ couchbackup --db animaldb --log animaldb.log --resume true --output animaldb.txt
|
|
|
122
122
|
|
|
123
123
|
## Restore
|
|
124
124
|
|
|
125
|
-
Now we have our backup text file, we can restore it to
|
|
125
|
+
Now that we have our backup text file, we can restore it to a new, empty, existing database using the `couchrestore`:
|
|
126
126
|
|
|
127
127
|
```sh
|
|
128
128
|
cat animaldb.txt | couchrestore
|
|
@@ -245,7 +245,7 @@ Then you can import the library into your code:
|
|
|
245
245
|
The library exports two main functions:
|
|
246
246
|
|
|
247
247
|
1. `backup` - backup from a database to a writable stream.
|
|
248
|
-
2. `restore` - restore from a readable stream to
|
|
248
|
+
2. `restore` - restore from a readable stream to an empty database.
|
|
249
249
|
|
|
250
250
|
### Examples
|
|
251
251
|
|
|
@@ -319,8 +319,9 @@ couchbackup.backup(
|
|
|
319
319
|
### Restore
|
|
320
320
|
|
|
321
321
|
The `restore` function takes a readable stream containing the data emitted
|
|
322
|
-
by the `backup` function
|
|
323
|
-
|
|
322
|
+
by the `backup` function and uploads that to a Cloudant database.
|
|
323
|
+
|
|
324
|
+
_Note:_ A target database must be a **new and empty** database.
|
|
324
325
|
|
|
325
326
|
```javascript
|
|
326
327
|
restore: function(srcStream, targetUrl, opts, callback) { /* ... */ }
|
package/app.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// Copyright © 2017,
|
|
1
|
+
// Copyright © 2017, 2021 IBM Corp. All rights reserved.
|
|
2
2
|
//
|
|
3
3
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
// you may not use this file except in compliance with the License.
|
|
@@ -109,7 +109,7 @@ function validateArgs(url, opts, cb) {
|
|
|
109
109
|
cb(new error.BackupError('InvalidOption', 'Invalid URL, missing path element (no database).'));
|
|
110
110
|
return;
|
|
111
111
|
}
|
|
112
|
-
if (opts && opts.iamApiKey && (urlObject.username ||
|
|
112
|
+
if (opts && opts.iamApiKey && (urlObject.username || urlObject.password)) {
|
|
113
113
|
cb(new error.BackupError('InvalidOption', 'URL user information must not be supplied when using IAM API key.'));
|
|
114
114
|
return;
|
|
115
115
|
}
|
|
@@ -160,12 +160,12 @@ function addEventListener(indicator, emitter, event, f) {
|
|
|
160
160
|
@param {function(err)} callback - error is undefined if DB exists
|
|
161
161
|
*/
|
|
162
162
|
function proceedIfDbValid(db, callback) {
|
|
163
|
-
db.
|
|
163
|
+
db.service.headDatabase({ db: db.db }).then(() => callback()).catch(err => {
|
|
164
164
|
err = error.convertResponseError(err, function(err) {
|
|
165
|
-
if (err && err.
|
|
165
|
+
if (err && err.status === 404) {
|
|
166
166
|
// Override the error type and mesasge for the DB not found case
|
|
167
|
-
var msg = `Database ${db.
|
|
168
|
-
|
|
167
|
+
var msg = `Database ${db.url}` +
|
|
168
|
+
`${db.db} does not exist. ` +
|
|
169
169
|
'Check the URL and database name have been specified correctly.';
|
|
170
170
|
var noDBErr = new Error(msg);
|
|
171
171
|
noDBErr.name = 'DatabaseNotFound';
|
|
@@ -175,7 +175,6 @@ function proceedIfDbValid(db, callback) {
|
|
|
175
175
|
return error.convertResponseError(err);
|
|
176
176
|
}
|
|
177
177
|
});
|
|
178
|
-
// Callback with or without (i.e. undefined) error
|
|
179
178
|
callback(err);
|
|
180
179
|
});
|
|
181
180
|
}
|
package/includes/backup.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// Copyright © 2017,
|
|
1
|
+
// Copyright © 2017, 2021 IBM Corp. All rights reserved.
|
|
2
2
|
//
|
|
3
3
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
// you may not use this file except in compliance with the License.
|
|
@@ -73,24 +73,20 @@ module.exports = function(db, options) {
|
|
|
73
73
|
* @param {function} callback - called on completion with signature (err)
|
|
74
74
|
*/
|
|
75
75
|
function validateBulkGetSupport(db, callback) {
|
|
76
|
-
db.
|
|
77
|
-
function(err) {
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
return;
|
|
88
|
-
default:
|
|
89
|
-
return new error.HTTPError(err);
|
|
90
|
-
}
|
|
91
|
-
});
|
|
92
|
-
callback(err);
|
|
76
|
+
db.service.postBulkGet({ db: db.db, docs: [] }).then(() => { callback(); }).catch(err => {
|
|
77
|
+
err = error.convertResponseError(err, function(err) {
|
|
78
|
+
switch (err.status) {
|
|
79
|
+
case undefined:
|
|
80
|
+
// There was no status code on the error
|
|
81
|
+
return err;
|
|
82
|
+
case 404:
|
|
83
|
+
return new error.BackupError('BulkGetError', 'Database does not support /_bulk_get endpoint');
|
|
84
|
+
default:
|
|
85
|
+
return new error.HTTPError(err);
|
|
86
|
+
}
|
|
93
87
|
});
|
|
88
|
+
callback(err);
|
|
89
|
+
});
|
|
94
90
|
}
|
|
95
91
|
|
|
96
92
|
/**
|
|
@@ -211,43 +207,40 @@ function processBatchSet(db, parallelism, log, batches, ee, start, grandtotal, c
|
|
|
211
207
|
}
|
|
212
208
|
|
|
213
209
|
// do the /db/_bulk_get request
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
q.kill();
|
|
226
|
-
ee.emit('error', err);
|
|
227
|
-
}
|
|
228
|
-
done();
|
|
229
|
-
} else {
|
|
230
|
-
// create an output array with the docs returned
|
|
231
|
-
body.results.forEach(function(d) {
|
|
232
|
-
if (d.docs) {
|
|
233
|
-
d.docs.forEach(function(doc) {
|
|
234
|
-
if (doc.ok) {
|
|
235
|
-
output.push(doc.ok);
|
|
236
|
-
}
|
|
237
|
-
});
|
|
210
|
+
db.service.postBulkGet({
|
|
211
|
+
db: db.db,
|
|
212
|
+
revs: true,
|
|
213
|
+
docs: payload.docs
|
|
214
|
+
}).then(response => {
|
|
215
|
+
// create an output array with the docs returned
|
|
216
|
+
response.result.results.forEach(function(d) {
|
|
217
|
+
if (d.docs) {
|
|
218
|
+
d.docs.forEach(function(doc) {
|
|
219
|
+
if (doc.ok) {
|
|
220
|
+
output.push(doc.ok);
|
|
238
221
|
}
|
|
239
222
|
});
|
|
240
|
-
total += output.length;
|
|
241
|
-
var t = (new Date().getTime() - start) / 1000;
|
|
242
|
-
ee.emit('received', {
|
|
243
|
-
batch: thisBatch,
|
|
244
|
-
data: output,
|
|
245
|
-
length: output.length,
|
|
246
|
-
time: t,
|
|
247
|
-
total: total
|
|
248
|
-
}, q, logCompletedBatch);
|
|
249
223
|
}
|
|
250
224
|
});
|
|
225
|
+
total += output.length;
|
|
226
|
+
var t = (new Date().getTime() - start) / 1000;
|
|
227
|
+
ee.emit('received', {
|
|
228
|
+
batch: thisBatch,
|
|
229
|
+
data: output,
|
|
230
|
+
length: output.length,
|
|
231
|
+
time: t,
|
|
232
|
+
total: total
|
|
233
|
+
}, q, logCompletedBatch);
|
|
234
|
+
}).catch(err => {
|
|
235
|
+
if (!hasErrored) {
|
|
236
|
+
hasErrored = true;
|
|
237
|
+
err = error.convertResponseError(err);
|
|
238
|
+
// Kill the queue for fatal errors
|
|
239
|
+
q.kill();
|
|
240
|
+
ee.emit('error', err);
|
|
241
|
+
}
|
|
242
|
+
done();
|
|
243
|
+
});
|
|
251
244
|
}, parallelism);
|
|
252
245
|
|
|
253
246
|
for (var i in batches) {
|
package/includes/error.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// Copyright © 2017,
|
|
1
|
+
// Copyright © 2017, 2021 IBM Corp. All rights reserved.
|
|
2
2
|
//
|
|
3
3
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
// you may not use this file except in compliance with the License.
|
|
@@ -37,13 +37,8 @@ class BackupError extends Error {
|
|
|
37
37
|
|
|
38
38
|
class HTTPError extends BackupError {
|
|
39
39
|
constructor(responseError, name) {
|
|
40
|
-
var errMsg = `${responseError.statusCode} ${responseError.statusMessage || ''}: ` +
|
|
41
|
-
`${responseError.request.method} ${(typeof responseError.request.uri === 'object') ? responseError.request.uri.href : responseError.request.uri}`;
|
|
42
|
-
if (responseError.error && responseError.reason) {
|
|
43
|
-
errMsg += ` - Error: ${responseError.error}, Reason: ${responseError.reason}`;
|
|
44
|
-
}
|
|
45
40
|
// Special case some names for more useful error messages
|
|
46
|
-
switch (responseError.
|
|
41
|
+
switch (responseError.status) {
|
|
47
42
|
case 401:
|
|
48
43
|
name = 'Unauthorized';
|
|
49
44
|
break;
|
|
@@ -53,7 +48,7 @@ class HTTPError extends BackupError {
|
|
|
53
48
|
default:
|
|
54
49
|
name = name || 'HTTPFatalError';
|
|
55
50
|
}
|
|
56
|
-
super(name,
|
|
51
|
+
super(name, responseError.message);
|
|
57
52
|
}
|
|
58
53
|
}
|
|
59
54
|
|
|
@@ -65,7 +60,7 @@ function checkResponse(err) {
|
|
|
65
60
|
if (err) {
|
|
66
61
|
// Construct an HTTPError if there is request information on the error
|
|
67
62
|
// Codes < 400 are considered OK
|
|
68
|
-
if (err.
|
|
63
|
+
if (err.status >= 400) {
|
|
69
64
|
return new HTTPError(err);
|
|
70
65
|
} else {
|
|
71
66
|
// Send it back again if there was no status code, e.g. a cxn error
|
|
@@ -84,10 +79,9 @@ function convertResponseError(responseError, errorFactory) {
|
|
|
84
79
|
function augmentMessage(err) {
|
|
85
80
|
// For errors that don't have a status code, we are likely looking at a cxn
|
|
86
81
|
// error.
|
|
87
|
-
// Try to augment the message with more detail
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
err.message = `${err.message} ${err.code}`;
|
|
82
|
+
// Try to augment the message with more detail (core puts the code in statusText)
|
|
83
|
+
if (err && err.statusText) {
|
|
84
|
+
err.message = `${err.message} ${err.statusText}`;
|
|
91
85
|
}
|
|
92
86
|
if (err && err.description) {
|
|
93
87
|
err.message = `${err.message} ${err.description}`;
|
|
@@ -101,11 +95,9 @@ module.exports = {
|
|
|
101
95
|
convertResponseError: convertResponseError,
|
|
102
96
|
terminationCallback: function terminationCallback(err, data) {
|
|
103
97
|
if (err) {
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
});
|
|
108
|
-
throw err;
|
|
98
|
+
console.error(`ERROR: ${err.message}`);
|
|
99
|
+
process.exitCode = codes[err.name] || 1;
|
|
100
|
+
process.exit();
|
|
109
101
|
}
|
|
110
102
|
}
|
|
111
103
|
};
|
package/includes/request.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// Copyright © 2017,
|
|
1
|
+
// Copyright © 2017, 2021 IBM Corp. All rights reserved.
|
|
2
2
|
//
|
|
3
3
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
// you may not use this file except in compliance with the License.
|
|
@@ -16,43 +16,170 @@
|
|
|
16
16
|
const pkg = require('../package.json');
|
|
17
17
|
const http = require('http');
|
|
18
18
|
const https = require('https');
|
|
19
|
-
const
|
|
19
|
+
const stream = require('stream');
|
|
20
|
+
const { CloudantV1, CouchdbSessionAuthenticator } = require('@ibm-cloud/cloudant');
|
|
21
|
+
const { IamAuthenticator, NoAuthAuthenticator } = require('ibm-cloud-sdk-core');
|
|
22
|
+
const retryPlugin = require('retry-axios');
|
|
20
23
|
|
|
21
24
|
const userAgent = 'couchbackup-cloudant/' + pkg.version + ' (Node.js ' +
|
|
22
25
|
process.version + ')';
|
|
23
26
|
|
|
27
|
+
// Class for streaming _changes error responses into
|
|
28
|
+
// In general the response is a small error/reason JSON object
|
|
29
|
+
// so it is OK to have this in memory.
|
|
30
|
+
class ResponseWriteable extends stream.Writable {
|
|
31
|
+
constructor(options) {
|
|
32
|
+
super(options);
|
|
33
|
+
this.data = [];
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
_write(chunk, encoding, callback) {
|
|
37
|
+
this.data.push(chunk);
|
|
38
|
+
callback();
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
stringBody() {
|
|
42
|
+
return Buffer.concat(this.data).toString();
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// An interceptor function to help augment error bodies with a little
|
|
47
|
+
// extra information so we can continue to use consistent messaging
|
|
48
|
+
// after the ugprade to @ibm-cloud/cloudant
|
|
49
|
+
const errorHelper = async function(err) {
|
|
50
|
+
let method;
|
|
51
|
+
let requestUrl;
|
|
52
|
+
if (err.response) {
|
|
53
|
+
if (err.response.config.url) {
|
|
54
|
+
requestUrl = err.response.config.url;
|
|
55
|
+
method = err.response.config.method;
|
|
56
|
+
}
|
|
57
|
+
// Override the status text with an improved message
|
|
58
|
+
let errorMsg = `${err.response.status} ${err.response.statusText || ''}: ` +
|
|
59
|
+
`${method} ${requestUrl}`;
|
|
60
|
+
if (err.response.data) {
|
|
61
|
+
// Check if we have a JSON response and try to get the error/reason
|
|
62
|
+
if (err.response.headers['content-type'] === 'application/json') {
|
|
63
|
+
if (!err.response.data.error && err.response.data.pipe) {
|
|
64
|
+
// If we didn't find a JSON object with `error` then we might have a stream response.
|
|
65
|
+
// Detect the stream by the presence of `pipe` and use it to get the body and parse
|
|
66
|
+
// the error information.
|
|
67
|
+
const p = new Promise((resolve, reject) => {
|
|
68
|
+
const errorBody = new ResponseWriteable();
|
|
69
|
+
err.response.data.pipe(errorBody)
|
|
70
|
+
.on('finish', () => { resolve(JSON.parse(errorBody.stringBody())); })
|
|
71
|
+
.on('error', () => { reject(err); });
|
|
72
|
+
});
|
|
73
|
+
// Replace the stream on the response with the parsed object
|
|
74
|
+
err.response.data = await p;
|
|
75
|
+
}
|
|
76
|
+
// Append the error/reason if available
|
|
77
|
+
if (err.response.data.error) {
|
|
78
|
+
// Override the status text with our more complete message
|
|
79
|
+
errorMsg += ` - Error: ${err.response.data.error}`;
|
|
80
|
+
if (err.response.data.reason) {
|
|
81
|
+
errorMsg += `, Reason: ${err.response.data.reason}`;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
} else {
|
|
85
|
+
errorMsg += err.response.data;
|
|
86
|
+
}
|
|
87
|
+
// Set a new message for use by the node-sdk-core
|
|
88
|
+
// We use the errors array because it gets processed
|
|
89
|
+
// ahead of all other service errors.
|
|
90
|
+
err.response.data.errors = [{ message: errorMsg }];
|
|
91
|
+
}
|
|
92
|
+
} else if (err.request) {
|
|
93
|
+
if (!err.message.includes(err.config.url)) {
|
|
94
|
+
// Augment the message with the URL and method
|
|
95
|
+
// but don't do it again if we already have the URL.
|
|
96
|
+
err.message = `${err.message}: ${err.config.method} ${err.config.url}`;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return Promise.reject(err);
|
|
100
|
+
};
|
|
101
|
+
|
|
24
102
|
module.exports = {
|
|
25
|
-
client: function(
|
|
26
|
-
|
|
103
|
+
client: function(rawUrl, opts) {
|
|
104
|
+
const url = new URL(rawUrl);
|
|
105
|
+
var protocol = (url.protocol.match(/^https/)) ? https : http;
|
|
27
106
|
const keepAliveAgent = new protocol.Agent({
|
|
28
107
|
keepAlive: true,
|
|
29
108
|
keepAliveMsecs: 30000,
|
|
30
109
|
maxSockets: opts.parallelism
|
|
31
110
|
});
|
|
32
|
-
// Split the URL
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
111
|
+
// Split the URL to separate service from database
|
|
112
|
+
// Use origin as the "base" to remove auth elements
|
|
113
|
+
const actUrl = new URL(url.pathname.substr(0, url.pathname.lastIndexOf('/')), url.origin);
|
|
114
|
+
const dbName = url.pathname.substr(url.pathname.lastIndexOf('/') + 1);
|
|
115
|
+
let authenticator;
|
|
37
116
|
// Default to cookieauth unless an IAM key is provided
|
|
38
117
|
if (opts.iamApiKey) {
|
|
39
|
-
const
|
|
118
|
+
const iamAuthOpts = { apikey: opts.iamApiKey };
|
|
40
119
|
if (opts.iamTokenUrl) {
|
|
41
|
-
|
|
120
|
+
iamAuthOpts.url = opts.iamTokenUrl;
|
|
42
121
|
}
|
|
43
|
-
|
|
122
|
+
authenticator = new IamAuthenticator(iamAuthOpts);
|
|
123
|
+
} else if (url.username) {
|
|
124
|
+
authenticator = new CouchdbSessionAuthenticator({
|
|
125
|
+
username: url.username,
|
|
126
|
+
password: url.password
|
|
127
|
+
});
|
|
44
128
|
} else {
|
|
45
|
-
|
|
129
|
+
authenticator = new NoAuthAuthenticator();
|
|
46
130
|
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
}
|
|
131
|
+
const serviceOpts = {
|
|
132
|
+
authenticator: authenticator,
|
|
133
|
+
timeout: opts.requestTimeout,
|
|
134
|
+
headers: { 'User-Agent': userAgent },
|
|
135
|
+
// Axios performance options
|
|
136
|
+
maxContentLength: -1
|
|
137
|
+
};
|
|
138
|
+
if (url.protocol === 'https') {
|
|
139
|
+
serviceOpts.httpsAgent = keepAliveAgent;
|
|
140
|
+
} else {
|
|
141
|
+
serviceOpts.httpAgent = keepAliveAgent;
|
|
142
|
+
}
|
|
143
|
+
const service = new CloudantV1(serviceOpts);
|
|
144
|
+
// Configure retries
|
|
145
|
+
const maxRetries = 2; // for 3 total attempts
|
|
146
|
+
service.getHttpClient().defaults.raxConfig = {
|
|
147
|
+
// retries for status codes
|
|
148
|
+
retry: maxRetries,
|
|
149
|
+
// retries for non-response e.g. ETIMEDOUT
|
|
150
|
+
noResponseRetries: maxRetries,
|
|
151
|
+
backoffType: 'exponential',
|
|
152
|
+
httpMethodsToRetry: ['GET', 'HEAD', 'POST'],
|
|
153
|
+
statusCodesToRetry: [
|
|
154
|
+
[429, 429],
|
|
155
|
+
[500, 599]
|
|
156
|
+
],
|
|
157
|
+
shouldRetry: err => {
|
|
158
|
+
const cfg = retryPlugin.getConfig(err);
|
|
159
|
+
// cap at max retries regardless of response/non-response type
|
|
160
|
+
if (cfg.currentRetryAttempt >= maxRetries) {
|
|
161
|
+
return false;
|
|
162
|
+
} else {
|
|
163
|
+
return retryPlugin.shouldRetryRequest(err);
|
|
164
|
+
}
|
|
165
|
+
},
|
|
166
|
+
instance: service.getHttpClient()
|
|
167
|
+
};
|
|
168
|
+
retryPlugin.attach(service.getHttpClient());
|
|
169
|
+
|
|
170
|
+
service.setServiceUrl(actUrl.toString());
|
|
171
|
+
if (authenticator instanceof CouchdbSessionAuthenticator) {
|
|
172
|
+
// Awkward workaround for known Couch issue with compression on _session requests
|
|
173
|
+
// It is not feasible to disable compression on all requests with the amount of
|
|
174
|
+
// data this lib needs to move, so override the property in the tokenManager instance.
|
|
175
|
+
authenticator.tokenManager.requestWrapperInstance.compressRequestData = false;
|
|
176
|
+
}
|
|
177
|
+
if (authenticator.tokenManager && authenticator.tokenManager.requestWrapperInstance) {
|
|
178
|
+
authenticator.tokenManager.requestWrapperInstance.axiosInstance.interceptors.response.use(null, errorHelper);
|
|
179
|
+
}
|
|
180
|
+
// Add error interceptors to put URLs in error messages
|
|
181
|
+
service.getHttpClient().interceptors.response.use(null, errorHelper);
|
|
182
|
+
|
|
183
|
+
return { service: service, db: dbName, url: actUrl.toString() };
|
|
57
184
|
}
|
|
58
185
|
};
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// Copyright © 2017,
|
|
1
|
+
// Copyright © 2017, 2021 IBM Corp. All rights reserved.
|
|
2
2
|
//
|
|
3
3
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
// you may not use this file except in compliance with the License.
|
|
@@ -29,20 +29,16 @@ module.exports = function(db, options) {
|
|
|
29
29
|
function(callback) {
|
|
30
30
|
// Note, include_docs: true is set automatically when using the
|
|
31
31
|
// fetch function.
|
|
32
|
-
var opts = { limit: options.bufferSize,
|
|
32
|
+
var opts = { db: db.db, limit: options.bufferSize, includeDocs: true };
|
|
33
33
|
|
|
34
34
|
// To avoid double fetching a document solely for the purposes of getting
|
|
35
35
|
// the next ID to use as a startkey for the next page we instead use the
|
|
36
36
|
// last ID of the current page and append the lowest unicode sort
|
|
37
37
|
// character.
|
|
38
38
|
if (startKey) opts.startkey = `${startKey}\0`;
|
|
39
|
-
db.
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
ee.emit('error', err);
|
|
43
|
-
hasErrored = true;
|
|
44
|
-
callback();
|
|
45
|
-
} else if (!body.rows) {
|
|
39
|
+
db.service.postAllDocs(opts).then(response => {
|
|
40
|
+
const body = response.result;
|
|
41
|
+
if (!body.rows) {
|
|
46
42
|
ee.emit('error', new error.BackupError(
|
|
47
43
|
'AllDocsError', 'ERROR: Invalid all docs response'));
|
|
48
44
|
callback();
|
|
@@ -69,6 +65,11 @@ module.exports = function(db, options) {
|
|
|
69
65
|
}
|
|
70
66
|
callback();
|
|
71
67
|
}
|
|
68
|
+
}).catch(err => {
|
|
69
|
+
err = error.convertResponseError(err);
|
|
70
|
+
ee.emit('error', err);
|
|
71
|
+
hasErrored = true;
|
|
72
|
+
callback();
|
|
72
73
|
});
|
|
73
74
|
},
|
|
74
75
|
function(callback) { callback(null, hasErrored || startKey == null); },
|
package/includes/spoolchanges.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// Copyright © 2017,
|
|
1
|
+
// Copyright © 2017, 2021 IBM Corp. All rights reserved.
|
|
2
2
|
//
|
|
3
3
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
// you may not use this file except in compliance with the License.
|
|
@@ -62,34 +62,31 @@ module.exports = function(db, log, bufferSize, ee, callback) {
|
|
|
62
62
|
};
|
|
63
63
|
|
|
64
64
|
// stream the changes feed to disk
|
|
65
|
-
|
|
66
|
-
.
|
|
65
|
+
db.service.postChangesAsStream({ db: db.db, seq_interval: 10000 }).then(response => {
|
|
66
|
+
response.result.pipe(liner())
|
|
67
|
+
.on('error', function(err) {
|
|
68
|
+
callback(err);
|
|
69
|
+
})
|
|
70
|
+
.pipe(change(onChange))
|
|
71
|
+
.on('error', function(err) {
|
|
72
|
+
callback(err);
|
|
73
|
+
})
|
|
74
|
+
.on('finish', function() {
|
|
75
|
+
processBuffer(true);
|
|
76
|
+
if (!lastSeq) {
|
|
77
|
+
logStream.end();
|
|
78
|
+
debug('changes request terminated before last_seq was sent');
|
|
79
|
+
callback(new error.BackupError('SpoolChangesError', 'Changes request terminated before last_seq was sent'));
|
|
80
|
+
} else {
|
|
81
|
+
debug('finished streaming database changes');
|
|
82
|
+
logStream.end(':changes_complete ' + lastSeq + '\n', 'utf8', callback);
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
}).catch(err => {
|
|
86
|
+
if (err.status && err.status >= 400) {
|
|
87
|
+
callback(error.convertResponseError(err));
|
|
88
|
+
} else {
|
|
67
89
|
callback(new error.BackupError('SpoolChangesError', `Failed changes request - ${err.message}`));
|
|
68
|
-
}
|
|
69
|
-
|
|
70
|
-
if (resp.statusCode >= 400) {
|
|
71
|
-
changesRequest.abort();
|
|
72
|
-
callback(error.convertResponseError(resp));
|
|
73
|
-
} else {
|
|
74
|
-
changesRequest.pipe(liner())
|
|
75
|
-
.on('error', function(err) {
|
|
76
|
-
callback(err);
|
|
77
|
-
})
|
|
78
|
-
.pipe(change(onChange))
|
|
79
|
-
.on('error', function(err) {
|
|
80
|
-
callback(err);
|
|
81
|
-
})
|
|
82
|
-
.on('finish', function() {
|
|
83
|
-
processBuffer(true);
|
|
84
|
-
if (!lastSeq) {
|
|
85
|
-
logStream.end();
|
|
86
|
-
debug('changes request terminated before last_seq was sent');
|
|
87
|
-
callback(new error.BackupError('SpoolChangesError', 'Changes request terminated before last_seq was sent'));
|
|
88
|
-
} else {
|
|
89
|
-
debug('finished streaming database changes');
|
|
90
|
-
logStream.end(':changes_complete ' + lastSeq + '\n', 'utf8', callback);
|
|
91
|
-
}
|
|
92
|
-
});
|
|
93
|
-
}
|
|
94
|
-
});
|
|
90
|
+
}
|
|
91
|
+
});
|
|
95
92
|
};
|
package/includes/writer.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// Copyright © 2017,
|
|
1
|
+
// Copyright © 2017, 2021 IBM Corp. All rights reserved.
|
|
2
2
|
//
|
|
3
3
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
// you may not use this file except in compliance with the License.
|
|
@@ -15,7 +15,6 @@
|
|
|
15
15
|
|
|
16
16
|
const async = require('async');
|
|
17
17
|
const stream = require('stream');
|
|
18
|
-
const zlib = require('zlib');
|
|
19
18
|
const error = require('./error.js');
|
|
20
19
|
const debug = require('debug')('couchbackup:writer');
|
|
21
20
|
|
|
@@ -35,58 +34,22 @@ module.exports = function(db, bufferSize, parallelism, ee) {
|
|
|
35
34
|
payload.new_edits = false;
|
|
36
35
|
}
|
|
37
36
|
|
|
38
|
-
// Stream the payload through a zip stream to the server
|
|
39
|
-
const payloadStream = new stream.PassThrough();
|
|
40
|
-
payloadStream.end(Buffer.from(JSON.stringify(payload), 'utf8'));
|
|
41
|
-
const zipstream = zlib.createGzip();
|
|
42
|
-
|
|
43
|
-
// Class for streaming _bulk_docs responses into
|
|
44
|
-
// In general the response is [] or a small error/reason JSON object
|
|
45
|
-
// so it is OK to have this in memory.
|
|
46
|
-
class ResponseWriteable extends stream.Writable {
|
|
47
|
-
constructor(options) {
|
|
48
|
-
super(options);
|
|
49
|
-
this.data = [];
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
_write(chunk, encoding, callback) {
|
|
53
|
-
this.data.push(chunk);
|
|
54
|
-
callback();
|
|
55
|
-
}
|
|
56
|
-
|
|
57
|
-
asJson() {
|
|
58
|
-
return JSON.parse(Buffer.concat(this.data).toString());
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
|
|
62
37
|
if (!didError) {
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
debug(`Error writing docs ${err.name} ${err.message}`);
|
|
79
|
-
cb(err, payload);
|
|
80
|
-
} else {
|
|
81
|
-
written += payload.docs.length;
|
|
82
|
-
writer.emit('restored', { documents: payload.docs.length, total: written });
|
|
83
|
-
cb();
|
|
84
|
-
}
|
|
85
|
-
});
|
|
86
|
-
// Pipe the payload into the request object to POST to _bulk_docs
|
|
87
|
-
payloadStream.pipe(zipstream).pipe(req);
|
|
88
|
-
// Pipe the request object's response into our bulkDocsResponse
|
|
89
|
-
req.pipe(responseBody);
|
|
38
|
+
db.service.postBulkDocs({
|
|
39
|
+
db: db.db,
|
|
40
|
+
bulkDocs: payload
|
|
41
|
+
}).then(response => {
|
|
42
|
+
if (!response.result || response.result.length > 0) {
|
|
43
|
+
throw new Error('Error writing batch.');
|
|
44
|
+
}
|
|
45
|
+
written += payload.docs.length;
|
|
46
|
+
writer.emit('restored', { documents: payload.docs.length, total: written });
|
|
47
|
+
cb();
|
|
48
|
+
}).catch(err => {
|
|
49
|
+
err = error.convertResponseError(err);
|
|
50
|
+
debug(`Error writing docs ${err.name} ${err.message}`);
|
|
51
|
+
cb(err, payload);
|
|
52
|
+
});
|
|
90
53
|
}
|
|
91
54
|
}, parallelism);
|
|
92
55
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cloudant/couchbackup",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.7.0",
|
|
4
4
|
"description": "CouchBackup - command-line backup utility for Cloudant/CouchDB",
|
|
5
5
|
"homepage": "https://github.com/cloudant/couchbackup",
|
|
6
6
|
"repository": "https://github.com/cloudant/couchbackup.git",
|
|
@@ -20,7 +20,9 @@
|
|
|
20
20
|
"node": ">=12"
|
|
21
21
|
},
|
|
22
22
|
"dependencies": {
|
|
23
|
-
"@
|
|
23
|
+
"@ibm-cloud/cloudant": "0.0.18",
|
|
24
|
+
"tough-cookie": "^4.0.0",
|
|
25
|
+
"retry-axios": "^2.4.0",
|
|
24
26
|
"async": "^3.1.0",
|
|
25
27
|
"commander": "^5.0.0",
|
|
26
28
|
"debug": "~4.1.0",
|