@cloudant/couchbackup 2.6.2-SNAPSHOT.21 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGES.md CHANGED
@@ -1,4 +1,7 @@
1
- # Unreleased
1
+ # 2.7.0 (2021-09-14)
2
+ - [UPGRADED] Cloudant client dependency from `@cloudant/cloudant` to `@ibm-cloud/cloudant`.
3
+
4
+ # 2.6.2 (2021-08-27)
2
5
  - [FIXED] `Invalid document ID: _bulk_get` error when using `@cloudant/cloudant`
3
6
  version `4.5.0`.
4
7
  - [UPGRADED] Upgraded `@cloudant/cloudant` dependency to version `4.5.0`.
package/README.md CHANGED
@@ -122,7 +122,7 @@ couchbackup --db animaldb --log animaldb.log --resume true --output animaldb.txt
122
122
 
123
123
  ## Restore
124
124
 
125
- Now we have our backup text file, we can restore it to an existing database using the `couchrestore`:
125
+ Now that we have our backup text file, we can restore it to a new, empty, existing database using the `couchrestore`:
126
126
 
127
127
  ```sh
128
128
  cat animaldb.txt | couchrestore
@@ -245,7 +245,7 @@ Then you can import the library into your code:
245
245
  The library exports two main functions:
246
246
 
247
247
  1. `backup` - backup from a database to a writable stream.
248
- 2. `restore` - restore from a readable stream to a database.
248
+ 2. `restore` - restore from a readable stream to an empty database.
249
249
 
250
250
  ### Examples
251
251
 
@@ -319,8 +319,9 @@ couchbackup.backup(
319
319
  ### Restore
320
320
 
321
321
  The `restore` function takes a readable stream containing the data emitted
322
- by the `backup` function. It uploads that to a Cloudant database, which
323
- should be a **new** database.
322
+ by the `backup` function and uploads that to a Cloudant database.
323
+
324
+ _Note:_ A target database must be a **new and empty** database.
324
325
 
325
326
  ```javascript
326
327
  restore: function(srcStream, targetUrl, opts, callback) { /* ... */ }
package/app.js CHANGED
@@ -1,4 +1,4 @@
1
- // Copyright © 2017, 2020 IBM Corp. All rights reserved.
1
+ // Copyright © 2017, 2021 IBM Corp. All rights reserved.
2
2
  //
3
3
  // Licensed under the Apache License, Version 2.0 (the "License");
4
4
  // you may not use this file except in compliance with the License.
@@ -109,7 +109,7 @@ function validateArgs(url, opts, cb) {
109
109
  cb(new error.BackupError('InvalidOption', 'Invalid URL, missing path element (no database).'));
110
110
  return;
111
111
  }
112
- if (opts && opts.iamApiKey && (urlObject.username || url.password)) {
112
+ if (opts && opts.iamApiKey && (urlObject.username || urlObject.password)) {
113
113
  cb(new error.BackupError('InvalidOption', 'URL user information must not be supplied when using IAM API key.'));
114
114
  return;
115
115
  }
@@ -160,12 +160,12 @@ function addEventListener(indicator, emitter, event, f) {
160
160
  @param {function(err)} callback - error is undefined if DB exists
161
161
  */
162
162
  function proceedIfDbValid(db, callback) {
163
- db.server.request({ db: db.config.db, method: 'HEAD' }, function(err) {
163
+ db.service.headDatabase({ db: db.db }).then(() => callback()).catch(err => {
164
164
  err = error.convertResponseError(err, function(err) {
165
- if (err && err.statusCode === 404) {
165
+ if (err && err.status === 404) {
166
166
  // Override the error type and mesasge for the DB not found case
167
- var msg = `Database ${db.config.url.replace(/\/\/.+@/g, '//****:****@')}` +
168
- `/${db.config.db} does not exist. ` +
167
+ var msg = `Database ${db.url}` +
168
+ `${db.db} does not exist. ` +
169
169
  'Check the URL and database name have been specified correctly.';
170
170
  var noDBErr = new Error(msg);
171
171
  noDBErr.name = 'DatabaseNotFound';
@@ -175,7 +175,6 @@ function proceedIfDbValid(db, callback) {
175
175
  return error.convertResponseError(err);
176
176
  }
177
177
  });
178
- // Callback with or without (i.e. undefined) error
179
178
  callback(err);
180
179
  });
181
180
  }
@@ -1,4 +1,4 @@
1
- // Copyright © 2017, 2019 IBM Corp. All rights reserved.
1
+ // Copyright © 2017, 2021 IBM Corp. All rights reserved.
2
2
  //
3
3
  // Licensed under the Apache License, Version 2.0 (the "License");
4
4
  // you may not use this file except in compliance with the License.
@@ -73,24 +73,20 @@ module.exports = function(db, options) {
73
73
  * @param {function} callback - called on completion with signature (err)
74
74
  */
75
75
  function validateBulkGetSupport(db, callback) {
76
- db.server.request({ method: 'HEAD', db: db.config.db, path: '_bulk_get' },
77
- function(err) {
78
- err = error.convertResponseError(err, function(err) {
79
- switch (err.statusCode) {
80
- case undefined:
81
- // There was no status code on the error
82
- return err;
83
- case 404:
84
- return new error.BackupError('BulkGetError', 'Database does not support /_bulk_get endpoint');
85
- case 405:
86
- // => supports /_bulk_get endpoint
87
- return;
88
- default:
89
- return new error.HTTPError(err);
90
- }
91
- });
92
- callback(err);
76
+ db.service.postBulkGet({ db: db.db, docs: [] }).then(() => { callback(); }).catch(err => {
77
+ err = error.convertResponseError(err, function(err) {
78
+ switch (err.status) {
79
+ case undefined:
80
+ // There was no status code on the error
81
+ return err;
82
+ case 404:
83
+ return new error.BackupError('BulkGetError', 'Database does not support /_bulk_get endpoint');
84
+ default:
85
+ return new error.HTTPError(err);
86
+ }
93
87
  });
88
+ callback(err);
89
+ });
94
90
  }
95
91
 
96
92
  /**
@@ -211,43 +207,40 @@ function processBatchSet(db, parallelism, log, batches, ee, start, grandtotal, c
211
207
  }
212
208
 
213
209
  // do the /db/_bulk_get request
214
- // Note: this should use built-in _bulk_get, but revs is not accepted as
215
- // part of the request body by the server yet. Working around using request
216
- // method to POST with a query string.
217
- db.server.request(
218
- { method: 'POST', db: db.config.db, path: '_bulk_get', qs: { revs: true }, body: payload },
219
- function(err, body) {
220
- if (err) {
221
- if (!hasErrored) {
222
- hasErrored = true;
223
- err = error.convertResponseError(err);
224
- // Kill the queue for fatal errors
225
- q.kill();
226
- ee.emit('error', err);
227
- }
228
- done();
229
- } else {
230
- // create an output array with the docs returned
231
- body.results.forEach(function(d) {
232
- if (d.docs) {
233
- d.docs.forEach(function(doc) {
234
- if (doc.ok) {
235
- output.push(doc.ok);
236
- }
237
- });
210
+ db.service.postBulkGet({
211
+ db: db.db,
212
+ revs: true,
213
+ docs: payload.docs
214
+ }).then(response => {
215
+ // create an output array with the docs returned
216
+ response.result.results.forEach(function(d) {
217
+ if (d.docs) {
218
+ d.docs.forEach(function(doc) {
219
+ if (doc.ok) {
220
+ output.push(doc.ok);
238
221
  }
239
222
  });
240
- total += output.length;
241
- var t = (new Date().getTime() - start) / 1000;
242
- ee.emit('received', {
243
- batch: thisBatch,
244
- data: output,
245
- length: output.length,
246
- time: t,
247
- total: total
248
- }, q, logCompletedBatch);
249
223
  }
250
224
  });
225
+ total += output.length;
226
+ var t = (new Date().getTime() - start) / 1000;
227
+ ee.emit('received', {
228
+ batch: thisBatch,
229
+ data: output,
230
+ length: output.length,
231
+ time: t,
232
+ total: total
233
+ }, q, logCompletedBatch);
234
+ }).catch(err => {
235
+ if (!hasErrored) {
236
+ hasErrored = true;
237
+ err = error.convertResponseError(err);
238
+ // Kill the queue for fatal errors
239
+ q.kill();
240
+ ee.emit('error', err);
241
+ }
242
+ done();
243
+ });
251
244
  }, parallelism);
252
245
 
253
246
  for (var i in batches) {
package/includes/error.js CHANGED
@@ -1,4 +1,4 @@
1
- // Copyright © 2017, 2018 IBM Corp. All rights reserved.
1
+ // Copyright © 2017, 2021 IBM Corp. All rights reserved.
2
2
  //
3
3
  // Licensed under the Apache License, Version 2.0 (the "License");
4
4
  // you may not use this file except in compliance with the License.
@@ -37,13 +37,8 @@ class BackupError extends Error {
37
37
 
38
38
  class HTTPError extends BackupError {
39
39
  constructor(responseError, name) {
40
- var errMsg = `${responseError.statusCode} ${responseError.statusMessage || ''}: ` +
41
- `${responseError.request.method} ${(typeof responseError.request.uri === 'object') ? responseError.request.uri.href : responseError.request.uri}`;
42
- if (responseError.error && responseError.reason) {
43
- errMsg += ` - Error: ${responseError.error}, Reason: ${responseError.reason}`;
44
- }
45
40
  // Special case some names for more useful error messages
46
- switch (responseError.statusCode) {
41
+ switch (responseError.status) {
47
42
  case 401:
48
43
  name = 'Unauthorized';
49
44
  break;
@@ -53,7 +48,7 @@ class HTTPError extends BackupError {
53
48
  default:
54
49
  name = name || 'HTTPFatalError';
55
50
  }
56
- super(name, errMsg);
51
+ super(name, responseError.message);
57
52
  }
58
53
  }
59
54
 
@@ -65,7 +60,7 @@ function checkResponse(err) {
65
60
  if (err) {
66
61
  // Construct an HTTPError if there is request information on the error
67
62
  // Codes < 400 are considered OK
68
- if (err.statusCode >= 400) {
63
+ if (err.status >= 400) {
69
64
  return new HTTPError(err);
70
65
  } else {
71
66
  // Send it back again if there was no status code, e.g. a cxn error
@@ -84,10 +79,9 @@ function convertResponseError(responseError, errorFactory) {
84
79
  function augmentMessage(err) {
85
80
  // For errors that don't have a status code, we are likely looking at a cxn
86
81
  // error.
87
- // Try to augment the message with more detail
88
- // TODO add this extra message detail to nano?
89
- if (err && err.code) {
90
- err.message = `${err.message} ${err.code}`;
82
+ // Try to augment the message with more detail (core puts the code in statusText)
83
+ if (err && err.statusText) {
84
+ err.message = `${err.message} ${err.statusText}`;
91
85
  }
92
86
  if (err && err.description) {
93
87
  err.message = `${err.message} ${err.description}`;
@@ -101,11 +95,9 @@ module.exports = {
101
95
  convertResponseError: convertResponseError,
102
96
  terminationCallback: function terminationCallback(err, data) {
103
97
  if (err) {
104
- process.on('uncaughtException', function(err) {
105
- console.error(`ERROR: ${err.message}`);
106
- process.exitCode = codes[err.name] || 1;
107
- });
108
- throw err;
98
+ console.error(`ERROR: ${err.message}`);
99
+ process.exitCode = codes[err.name] || 1;
100
+ process.exit();
109
101
  }
110
102
  }
111
103
  };
@@ -1,4 +1,4 @@
1
- // Copyright © 2017, 2018 IBM Corp. All rights reserved.
1
+ // Copyright © 2017, 2021 IBM Corp. All rights reserved.
2
2
  //
3
3
  // Licensed under the Apache License, Version 2.0 (the "License");
4
4
  // you may not use this file except in compliance with the License.
@@ -16,43 +16,170 @@
16
16
  const pkg = require('../package.json');
17
17
  const http = require('http');
18
18
  const https = require('https');
19
- const cloudant = require('@cloudant/cloudant');
19
+ const stream = require('stream');
20
+ const { CloudantV1, CouchdbSessionAuthenticator } = require('@ibm-cloud/cloudant');
21
+ const { IamAuthenticator, NoAuthAuthenticator } = require('ibm-cloud-sdk-core');
22
+ const retryPlugin = require('retry-axios');
20
23
 
21
24
  const userAgent = 'couchbackup-cloudant/' + pkg.version + ' (Node.js ' +
22
25
  process.version + ')';
23
26
 
27
+ // Class for streaming _changes error responses into
28
+ // In general the response is a small error/reason JSON object
29
+ // so it is OK to have this in memory.
30
+ class ResponseWriteable extends stream.Writable {
31
+ constructor(options) {
32
+ super(options);
33
+ this.data = [];
34
+ }
35
+
36
+ _write(chunk, encoding, callback) {
37
+ this.data.push(chunk);
38
+ callback();
39
+ }
40
+
41
+ stringBody() {
42
+ return Buffer.concat(this.data).toString();
43
+ }
44
+ }
45
+
46
+ // An interceptor function to help augment error bodies with a little
47
+ // extra information so we can continue to use consistent messaging
48
+ // after the ugprade to @ibm-cloud/cloudant
49
+ const errorHelper = async function(err) {
50
+ let method;
51
+ let requestUrl;
52
+ if (err.response) {
53
+ if (err.response.config.url) {
54
+ requestUrl = err.response.config.url;
55
+ method = err.response.config.method;
56
+ }
57
+ // Override the status text with an improved message
58
+ let errorMsg = `${err.response.status} ${err.response.statusText || ''}: ` +
59
+ `${method} ${requestUrl}`;
60
+ if (err.response.data) {
61
+ // Check if we have a JSON response and try to get the error/reason
62
+ if (err.response.headers['content-type'] === 'application/json') {
63
+ if (!err.response.data.error && err.response.data.pipe) {
64
+ // If we didn't find a JSON object with `error` then we might have a stream response.
65
+ // Detect the stream by the presence of `pipe` and use it to get the body and parse
66
+ // the error information.
67
+ const p = new Promise((resolve, reject) => {
68
+ const errorBody = new ResponseWriteable();
69
+ err.response.data.pipe(errorBody)
70
+ .on('finish', () => { resolve(JSON.parse(errorBody.stringBody())); })
71
+ .on('error', () => { reject(err); });
72
+ });
73
+ // Replace the stream on the response with the parsed object
74
+ err.response.data = await p;
75
+ }
76
+ // Append the error/reason if available
77
+ if (err.response.data.error) {
78
+ // Override the status text with our more complete message
79
+ errorMsg += ` - Error: ${err.response.data.error}`;
80
+ if (err.response.data.reason) {
81
+ errorMsg += `, Reason: ${err.response.data.reason}`;
82
+ }
83
+ }
84
+ } else {
85
+ errorMsg += err.response.data;
86
+ }
87
+ // Set a new message for use by the node-sdk-core
88
+ // We use the errors array because it gets processed
89
+ // ahead of all other service errors.
90
+ err.response.data.errors = [{ message: errorMsg }];
91
+ }
92
+ } else if (err.request) {
93
+ if (!err.message.includes(err.config.url)) {
94
+ // Augment the message with the URL and method
95
+ // but don't do it again if we already have the URL.
96
+ err.message = `${err.message}: ${err.config.method} ${err.config.url}`;
97
+ }
98
+ }
99
+ return Promise.reject(err);
100
+ };
101
+
24
102
  module.exports = {
25
- client: function(url, opts) {
26
- var protocol = (url.match(/^https/)) ? https : http;
103
+ client: function(rawUrl, opts) {
104
+ const url = new URL(rawUrl);
105
+ var protocol = (url.protocol.match(/^https/)) ? https : http;
27
106
  const keepAliveAgent = new protocol.Agent({
28
107
  keepAlive: true,
29
108
  keepAliveMsecs: 30000,
30
109
  maxSockets: opts.parallelism
31
110
  });
32
- // Split the URL for use with nodejs-cloudant
33
- var actUrl = url.substr(0, url.lastIndexOf('/'));
34
- var dbName = url.substr(url.lastIndexOf('/') + 1);
35
- // Default set of plugins includes retry
36
- var pluginsToUse = ['retry'];
111
+ // Split the URL to separate service from database
112
+ // Use origin as the "base" to remove auth elements
113
+ const actUrl = new URL(url.pathname.substr(0, url.pathname.lastIndexOf('/')), url.origin);
114
+ const dbName = url.pathname.substr(url.pathname.lastIndexOf('/') + 1);
115
+ let authenticator;
37
116
  // Default to cookieauth unless an IAM key is provided
38
117
  if (opts.iamApiKey) {
39
- const iamPluginConfig = { iamApiKey: opts.iamApiKey };
118
+ const iamAuthOpts = { apikey: opts.iamApiKey };
40
119
  if (opts.iamTokenUrl) {
41
- iamPluginConfig.iamTokenUrl = opts.iamTokenUrl;
120
+ iamAuthOpts.url = opts.iamTokenUrl;
42
121
  }
43
- pluginsToUse.push({ iamauth: iamPluginConfig });
122
+ authenticator = new IamAuthenticator(iamAuthOpts);
123
+ } else if (url.username) {
124
+ authenticator = new CouchdbSessionAuthenticator({
125
+ username: url.username,
126
+ password: url.password
127
+ });
44
128
  } else {
45
- pluginsToUse.push({ cookieauth: { errorOnNoCreds: false } });
129
+ authenticator = new NoAuthAuthenticator();
46
130
  }
47
- return cloudant({
48
- url: actUrl,
49
- plugins: pluginsToUse,
50
- requestDefaults: {
51
- agent: keepAliveAgent,
52
- headers: { 'User-Agent': userAgent },
53
- gzip: true,
54
- timeout: opts.requestTimeout
55
- }
56
- }).use(dbName);
131
+ const serviceOpts = {
132
+ authenticator: authenticator,
133
+ timeout: opts.requestTimeout,
134
+ headers: { 'User-Agent': userAgent },
135
+ // Axios performance options
136
+ maxContentLength: -1
137
+ };
138
+ if (url.protocol === 'https') {
139
+ serviceOpts.httpsAgent = keepAliveAgent;
140
+ } else {
141
+ serviceOpts.httpAgent = keepAliveAgent;
142
+ }
143
+ const service = new CloudantV1(serviceOpts);
144
+ // Configure retries
145
+ const maxRetries = 2; // for 3 total attempts
146
+ service.getHttpClient().defaults.raxConfig = {
147
+ // retries for status codes
148
+ retry: maxRetries,
149
+ // retries for non-response e.g. ETIMEDOUT
150
+ noResponseRetries: maxRetries,
151
+ backoffType: 'exponential',
152
+ httpMethodsToRetry: ['GET', 'HEAD', 'POST'],
153
+ statusCodesToRetry: [
154
+ [429, 429],
155
+ [500, 599]
156
+ ],
157
+ shouldRetry: err => {
158
+ const cfg = retryPlugin.getConfig(err);
159
+ // cap at max retries regardless of response/non-response type
160
+ if (cfg.currentRetryAttempt >= maxRetries) {
161
+ return false;
162
+ } else {
163
+ return retryPlugin.shouldRetryRequest(err);
164
+ }
165
+ },
166
+ instance: service.getHttpClient()
167
+ };
168
+ retryPlugin.attach(service.getHttpClient());
169
+
170
+ service.setServiceUrl(actUrl.toString());
171
+ if (authenticator instanceof CouchdbSessionAuthenticator) {
172
+ // Awkward workaround for known Couch issue with compression on _session requests
173
+ // It is not feasible to disable compression on all requests with the amount of
174
+ // data this lib needs to move, so override the property in the tokenManager instance.
175
+ authenticator.tokenManager.requestWrapperInstance.compressRequestData = false;
176
+ }
177
+ if (authenticator.tokenManager && authenticator.tokenManager.requestWrapperInstance) {
178
+ authenticator.tokenManager.requestWrapperInstance.axiosInstance.interceptors.response.use(null, errorHelper);
179
+ }
180
+ // Add error interceptors to put URLs in error messages
181
+ service.getHttpClient().interceptors.response.use(null, errorHelper);
182
+
183
+ return { service: service, db: dbName, url: actUrl.toString() };
57
184
  }
58
185
  };
@@ -1,4 +1,4 @@
1
- // Copyright © 2017, 2019 IBM Corp. All rights reserved.
1
+ // Copyright © 2017, 2021 IBM Corp. All rights reserved.
2
2
  //
3
3
  // Licensed under the Apache License, Version 2.0 (the "License");
4
4
  // you may not use this file except in compliance with the License.
@@ -29,20 +29,16 @@ module.exports = function(db, options) {
29
29
  function(callback) {
30
30
  // Note, include_docs: true is set automatically when using the
31
31
  // fetch function.
32
- var opts = { limit: options.bufferSize, include_docs: true };
32
+ var opts = { db: db.db, limit: options.bufferSize, includeDocs: true };
33
33
 
34
34
  // To avoid double fetching a document solely for the purposes of getting
35
35
  // the next ID to use as a startkey for the next page we instead use the
36
36
  // last ID of the current page and append the lowest unicode sort
37
37
  // character.
38
38
  if (startKey) opts.startkey = `${startKey}\0`;
39
- db.list(opts, function(err, body) {
40
- if (err) {
41
- err = error.convertResponseError(err);
42
- ee.emit('error', err);
43
- hasErrored = true;
44
- callback();
45
- } else if (!body.rows) {
39
+ db.service.postAllDocs(opts).then(response => {
40
+ const body = response.result;
41
+ if (!body.rows) {
46
42
  ee.emit('error', new error.BackupError(
47
43
  'AllDocsError', 'ERROR: Invalid all docs response'));
48
44
  callback();
@@ -69,6 +65,11 @@ module.exports = function(db, options) {
69
65
  }
70
66
  callback();
71
67
  }
68
+ }).catch(err => {
69
+ err = error.convertResponseError(err);
70
+ ee.emit('error', err);
71
+ hasErrored = true;
72
+ callback();
72
73
  });
73
74
  },
74
75
  function(callback) { callback(null, hasErrored || startKey == null); },
@@ -1,4 +1,4 @@
1
- // Copyright © 2017, 2018 IBM Corp. All rights reserved.
1
+ // Copyright © 2017, 2021 IBM Corp. All rights reserved.
2
2
  //
3
3
  // Licensed under the Apache License, Version 2.0 (the "License");
4
4
  // you may not use this file except in compliance with the License.
@@ -62,34 +62,31 @@ module.exports = function(db, log, bufferSize, ee, callback) {
62
62
  };
63
63
 
64
64
  // stream the changes feed to disk
65
- var changesRequest = db.changesAsStream({ seq_interval: 10000 })
66
- .on('error', function(err) {
65
+ db.service.postChangesAsStream({ db: db.db, seq_interval: 10000 }).then(response => {
66
+ response.result.pipe(liner())
67
+ .on('error', function(err) {
68
+ callback(err);
69
+ })
70
+ .pipe(change(onChange))
71
+ .on('error', function(err) {
72
+ callback(err);
73
+ })
74
+ .on('finish', function() {
75
+ processBuffer(true);
76
+ if (!lastSeq) {
77
+ logStream.end();
78
+ debug('changes request terminated before last_seq was sent');
79
+ callback(new error.BackupError('SpoolChangesError', 'Changes request terminated before last_seq was sent'));
80
+ } else {
81
+ debug('finished streaming database changes');
82
+ logStream.end(':changes_complete ' + lastSeq + '\n', 'utf8', callback);
83
+ }
84
+ });
85
+ }).catch(err => {
86
+ if (err.status && err.status >= 400) {
87
+ callback(error.convertResponseError(err));
88
+ } else {
67
89
  callback(new error.BackupError('SpoolChangesError', `Failed changes request - ${err.message}`));
68
- })
69
- .on('response', function(resp) {
70
- if (resp.statusCode >= 400) {
71
- changesRequest.abort();
72
- callback(error.convertResponseError(resp));
73
- } else {
74
- changesRequest.pipe(liner())
75
- .on('error', function(err) {
76
- callback(err);
77
- })
78
- .pipe(change(onChange))
79
- .on('error', function(err) {
80
- callback(err);
81
- })
82
- .on('finish', function() {
83
- processBuffer(true);
84
- if (!lastSeq) {
85
- logStream.end();
86
- debug('changes request terminated before last_seq was sent');
87
- callback(new error.BackupError('SpoolChangesError', 'Changes request terminated before last_seq was sent'));
88
- } else {
89
- debug('finished streaming database changes');
90
- logStream.end(':changes_complete ' + lastSeq + '\n', 'utf8', callback);
91
- }
92
- });
93
- }
94
- });
90
+ }
91
+ });
95
92
  };
@@ -1,4 +1,4 @@
1
- // Copyright © 2017, 2019 IBM Corp. All rights reserved.
1
+ // Copyright © 2017, 2021 IBM Corp. All rights reserved.
2
2
  //
3
3
  // Licensed under the Apache License, Version 2.0 (the "License");
4
4
  // you may not use this file except in compliance with the License.
@@ -15,7 +15,6 @@
15
15
 
16
16
  const async = require('async');
17
17
  const stream = require('stream');
18
- const zlib = require('zlib');
19
18
  const error = require('./error.js');
20
19
  const debug = require('debug')('couchbackup:writer');
21
20
 
@@ -35,58 +34,22 @@ module.exports = function(db, bufferSize, parallelism, ee) {
35
34
  payload.new_edits = false;
36
35
  }
37
36
 
38
- // Stream the payload through a zip stream to the server
39
- const payloadStream = new stream.PassThrough();
40
- payloadStream.end(Buffer.from(JSON.stringify(payload), 'utf8'));
41
- const zipstream = zlib.createGzip();
42
-
43
- // Class for streaming _bulk_docs responses into
44
- // In general the response is [] or a small error/reason JSON object
45
- // so it is OK to have this in memory.
46
- class ResponseWriteable extends stream.Writable {
47
- constructor(options) {
48
- super(options);
49
- this.data = [];
50
- }
51
-
52
- _write(chunk, encoding, callback) {
53
- this.data.push(chunk);
54
- callback();
55
- }
56
-
57
- asJson() {
58
- return JSON.parse(Buffer.concat(this.data).toString());
59
- }
60
- }
61
-
62
37
  if (!didError) {
63
- var response;
64
- const responseBody = new ResponseWriteable();
65
- const req = db.server.request({
66
- db: db.config.db,
67
- path: '_bulk_docs',
68
- method: 'POST',
69
- headers: { 'content-encoding': 'gzip' },
70
- stream: true
71
- })
72
- .on('response', function(resp) {
73
- response = resp;
74
- })
75
- .on('end', function() {
76
- if (response.statusCode >= 400) {
77
- const err = error.convertResponseError(Object.assign({}, response, responseBody.asJson()));
78
- debug(`Error writing docs ${err.name} ${err.message}`);
79
- cb(err, payload);
80
- } else {
81
- written += payload.docs.length;
82
- writer.emit('restored', { documents: payload.docs.length, total: written });
83
- cb();
84
- }
85
- });
86
- // Pipe the payload into the request object to POST to _bulk_docs
87
- payloadStream.pipe(zipstream).pipe(req);
88
- // Pipe the request object's response into our bulkDocsResponse
89
- req.pipe(responseBody);
38
+ db.service.postBulkDocs({
39
+ db: db.db,
40
+ bulkDocs: payload
41
+ }).then(response => {
42
+ if (!response.result || response.result.length > 0) {
43
+ throw new Error('Error writing batch.');
44
+ }
45
+ written += payload.docs.length;
46
+ writer.emit('restored', { documents: payload.docs.length, total: written });
47
+ cb();
48
+ }).catch(err => {
49
+ err = error.convertResponseError(err);
50
+ debug(`Error writing docs ${err.name} ${err.message}`);
51
+ cb(err, payload);
52
+ });
90
53
  }
91
54
  }, parallelism);
92
55
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@cloudant/couchbackup",
3
- "version": "2.6.2-SNAPSHOT.21",
3
+ "version": "2.7.0",
4
4
  "description": "CouchBackup - command-line backup utility for Cloudant/CouchDB",
5
5
  "homepage": "https://github.com/cloudant/couchbackup",
6
6
  "repository": "https://github.com/cloudant/couchbackup.git",
@@ -20,7 +20,9 @@
20
20
  "node": ">=12"
21
21
  },
22
22
  "dependencies": {
23
- "@cloudant/cloudant": "^4.5.0",
23
+ "@ibm-cloud/cloudant": "0.0.18",
24
+ "tough-cookie": "^4.0.0",
25
+ "retry-axios": "^2.4.0",
24
26
  "async": "^3.1.0",
25
27
  "commander": "^5.0.0",
26
28
  "debug": "~4.1.0",