@cloudant/couchbackup 2.11.12-SNAPSHOT-384 → 2.11.12-SNAPSHOT-386
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/includes/liner.js +12 -35
- package/includes/restore.js +1 -1
- package/package.json +2 -2
package/includes/liner.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
// Copyright © 2017,
|
|
1
|
+
// Copyright © 2017, 2024 IBM Corp. All rights reserved.
|
|
2
2
|
//
|
|
3
3
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
4
4
|
// you may not use this file except in compliance with the License.
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
// limitations under the License.
|
|
14
14
|
|
|
15
15
|
const { createInterface } = require('node:readline');
|
|
16
|
-
const {
|
|
16
|
+
const { PassThrough, Duplex } = require('node:stream');
|
|
17
17
|
const debug = require('debug');
|
|
18
18
|
|
|
19
19
|
/**
|
|
@@ -32,14 +32,16 @@ class Liner extends Duplex {
|
|
|
32
32
|
log = debug(('couchbackup:liner'));
|
|
33
33
|
// Flag for whether the readline interface is running
|
|
34
34
|
isRunning = true;
|
|
35
|
-
// Flag for whether the readline interface is closed
|
|
36
|
-
isClosed = false;
|
|
37
35
|
// Line number state
|
|
38
36
|
lineNumber = 0;
|
|
39
37
|
// Buffer of processed lines
|
|
40
38
|
lines = [];
|
|
39
|
+
// Stream of bytes that will be processed to lines.
|
|
40
|
+
inStream = new PassThrough({ objectMode: false })
|
|
41
|
+
// if there is an error destroy this Duplex with it
|
|
42
|
+
.on('error', e => this.destroy(e));
|
|
41
43
|
|
|
42
|
-
constructor(
|
|
44
|
+
constructor() {
|
|
43
45
|
// Configuration of this Duplex:
|
|
44
46
|
// objectMode: false on the writable input (file chunks), true on the readable output (line objects)
|
|
45
47
|
// The readableHighWaterMark controls the number of lines buffered after this implementation calls
|
|
@@ -47,25 +49,6 @@ class Liner extends Duplex {
|
|
|
47
49
|
// there is additional buffering downstream and file processing is faster than the network ops
|
|
48
50
|
// we don't bottleneck here even without a large buffer.
|
|
49
51
|
super({ readableObjectMode: true, readableHighWaterMark: 0, writableObjectMode: false });
|
|
50
|
-
// Set up the stream of bytes that will be processed to lines.
|
|
51
|
-
if (sanitize) {
|
|
52
|
-
// Handle unescaped unicode "newlines" by escaping them before passing to readline
|
|
53
|
-
this.inStream = new Transform({
|
|
54
|
-
objectMode: false,
|
|
55
|
-
transform(chunk, encoding, callback) {
|
|
56
|
-
try {
|
|
57
|
-
this.push(chunk.toString('utf-8').replace(/\u2028/, '\\u2028').replace(/\u2029/, '\\u2029'));
|
|
58
|
-
callback();
|
|
59
|
-
} catch (e) {
|
|
60
|
-
callback(e);
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
});
|
|
64
|
-
} else {
|
|
65
|
-
this.inStream = new PassThrough({ objectMode: false });
|
|
66
|
-
}
|
|
67
|
-
// if there is an error destroy this Duplex with it
|
|
68
|
-
this.inStream.on('error', e => this.destroy(e));
|
|
69
52
|
// Built-in readline interface over the inStream
|
|
70
53
|
this.readlineInterface = createInterface({
|
|
71
54
|
input: this.inStream, // the writable side of Liner, passed through
|
|
@@ -77,8 +60,7 @@ class Liner extends Duplex {
|
|
|
77
60
|
const bufferedLines = this.lines.push(this.wrapLine(line));
|
|
78
61
|
this.log(`Liner processed line ${this.lineNumber}. Buffered lines available: ${bufferedLines}.`);
|
|
79
62
|
this.pushAvailable();
|
|
80
|
-
}).
|
|
81
|
-
this.isClosed = true;
|
|
63
|
+
}).on('close', () => {
|
|
82
64
|
this.log('Liner readline interface closed.');
|
|
83
65
|
// Push null onto our lines buffer to signal EOF to downstream consumers.
|
|
84
66
|
this.lines.push(null);
|
|
@@ -105,16 +87,13 @@ class Liner extends Duplex {
|
|
|
105
87
|
// Check readline is running flag and whether there is content to push.
|
|
106
88
|
while (this.isRunning && this.lines.length > 0) {
|
|
107
89
|
if (!this.push(this.lines.shift())) {
|
|
108
|
-
this.log(`Back-pressure from push. Buffered lines available: ${this.lines.length}.`);
|
|
109
90
|
// Push returned false, this indicates downstream back-pressure.
|
|
110
91
|
// Pause the readline interface to stop pushing more lines downstream.
|
|
111
92
|
// Resumption is triggered by downstream calling _read which happens
|
|
112
93
|
// when it is ready for more data.
|
|
94
|
+
this.log(`Liner pausing after back-pressure from push. Buffered lines available: ${this.lines.length}.`);
|
|
113
95
|
this.isRunning = false;
|
|
114
|
-
|
|
115
|
-
this.log('Liner pausing.');
|
|
116
|
-
this.readlineInterface.pause();
|
|
117
|
-
}
|
|
96
|
+
this.readlineInterface.pause();
|
|
118
97
|
break;
|
|
119
98
|
} else {
|
|
120
99
|
this.log(`Liner pushed. Buffered lines available: ${this.lines.length}.`);
|
|
@@ -135,11 +114,9 @@ class Liner extends Duplex {
|
|
|
135
114
|
// is called to ensure that pushes are able to happen (and thereby trigger)
|
|
136
115
|
// subsequent reads.
|
|
137
116
|
if (!this.isRunning) {
|
|
117
|
+
this.log('Liner resuming after read.');
|
|
138
118
|
this.isRunning = true;
|
|
139
|
-
|
|
140
|
-
this.log('Liner resuming after read.');
|
|
141
|
-
this.readlineInterface.resume();
|
|
142
|
-
}
|
|
119
|
+
this.readlineInterface.resume();
|
|
143
120
|
}
|
|
144
121
|
this.pushAvailable();
|
|
145
122
|
}
|
package/includes/restore.js
CHANGED
|
@@ -50,7 +50,7 @@ module.exports = function(dbClient, options, readstream, ee) {
|
|
|
50
50
|
|
|
51
51
|
const batchPreparationStreams = [
|
|
52
52
|
readstream, // the backup file
|
|
53
|
-
new Liner(
|
|
53
|
+
new Liner(), // line by line
|
|
54
54
|
new MappingStream(restore.backupLineToDocsArray), // convert line to a docs array
|
|
55
55
|
new BatchingStream(options.bufferSize, true), // make new arrays of the correct buffer size
|
|
56
56
|
new MappingStream(restore.docsToRestoreBatch) // make a restore batch
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cloudant/couchbackup",
|
|
3
|
-
"version": "2.11.12-SNAPSHOT-
|
|
3
|
+
"version": "2.11.12-SNAPSHOT-386",
|
|
4
4
|
"description": "CouchBackup - command-line backup utility for Cloudant/CouchDB",
|
|
5
5
|
"homepage": "https://github.com/IBM/couchbackup",
|
|
6
6
|
"repository": {
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
},
|
|
21
21
|
"license": "Apache-2.0",
|
|
22
22
|
"engines": {
|
|
23
|
-
"node": "^20 || ^22
|
|
23
|
+
"node": "^20 || ^22"
|
|
24
24
|
},
|
|
25
25
|
"dependencies": {
|
|
26
26
|
"@ibm-cloud/cloudant": "0.12.11",
|