@push.rocks/smartstream 3.0.19 → 3.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
export const commitinfo = {
|
|
5
5
|
name: '@push.rocks/smartstream',
|
|
6
|
-
version: '3.0.
|
|
6
|
+
version: '3.0.20',
|
|
7
7
|
description: 'simplifies access to node streams'
|
|
8
8
|
};
|
|
9
9
|
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiMDBfY29tbWl0aW5mb19kYXRhLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vdHMvMDBfY29tbWl0aW5mb19kYXRhLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBOztHQUVHO0FBQ0gsTUFBTSxDQUFDLE1BQU0sVUFBVSxHQUFHO0lBQ3hCLElBQUksRUFBRSx5QkFBeUI7SUFDL0IsT0FBTyxFQUFFLFFBQVE7SUFDakIsV0FBVyxFQUFFLG1DQUFtQztDQUNqRCxDQUFBIn0=
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
/// <reference types="node" resolution-mode="require"/>
|
|
2
2
|
/// <reference types="node" resolution-mode="require"/>
|
|
3
|
-
import * as plugins from './smartstream.plugins.js';
|
|
4
3
|
import { Duplex, type DuplexOptions } from 'stream';
|
|
5
4
|
export interface IStreamTools {
|
|
6
5
|
truncate: () => void;
|
|
@@ -13,6 +12,8 @@ export interface IStreamFinalFunction<rT> {
|
|
|
13
12
|
(toolsArg: IStreamTools): Promise<rT>;
|
|
14
13
|
}
|
|
15
14
|
export interface ISmartDuplexOptions<TInput, TOutput> extends DuplexOptions {
|
|
15
|
+
debug?: boolean;
|
|
16
|
+
name?: string;
|
|
16
17
|
handleBackpressure?: boolean;
|
|
17
18
|
readFunction?: () => Promise<void>;
|
|
18
19
|
writeFunction?: IStreamWriteFunction<TInput, TOutput>;
|
|
@@ -20,13 +21,10 @@ export interface ISmartDuplexOptions<TInput, TOutput> extends DuplexOptions {
|
|
|
20
21
|
}
|
|
21
22
|
export declare class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
|
|
22
23
|
static fromBuffer(buffer: Buffer, options?: ISmartDuplexOptions<any, any>): SmartDuplex;
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
private readFunction?;
|
|
26
|
-
private handleBackpressure;
|
|
27
|
-
private writeFunction?;
|
|
28
|
-
private finalFunction?;
|
|
24
|
+
private backpressuredArray;
|
|
25
|
+
options: ISmartDuplexOptions<TInput, TOutput>;
|
|
29
26
|
private observableSubscription?;
|
|
27
|
+
private debugLog;
|
|
30
28
|
constructor(optionsArg?: ISmartDuplexOptions<TInput, TOutput>);
|
|
31
29
|
_read(size: number): Promise<void>;
|
|
32
30
|
private asyncWritePromiseObjectmap;
|
|
@@ -10,119 +10,64 @@ export class SmartDuplex extends Duplex {
|
|
|
10
10
|
});
|
|
11
11
|
return smartDuplex;
|
|
12
12
|
}
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
if (!smartStream.push(data) && smartStream.handleBackpressure) {
|
|
18
|
-
// Pause the observable if the stream buffer is full
|
|
19
|
-
smartStream.observableSubscription?.unsubscribe();
|
|
20
|
-
smartStream.once('drain', () => {
|
|
21
|
-
// Resume the observable when the stream buffer is drained
|
|
22
|
-
smartStream.observableSubscription?.unsubscribe();
|
|
23
|
-
smartStream.observableSubscription = observable.subscribe((data) => {
|
|
24
|
-
smartStream.push(data);
|
|
25
|
-
});
|
|
26
|
-
});
|
|
27
|
-
}
|
|
28
|
-
},
|
|
29
|
-
error: (err) => {
|
|
30
|
-
smartStream.emit('error', err);
|
|
31
|
-
},
|
|
32
|
-
complete: () => {
|
|
33
|
-
smartStream.push(null); // Signal the end of the data
|
|
34
|
-
},
|
|
35
|
-
});
|
|
36
|
-
return smartStream;
|
|
37
|
-
}
|
|
38
|
-
static fromReplaySubject(replaySubject, options) {
|
|
39
|
-
const smartStream = new SmartDuplex(options);
|
|
40
|
-
let isBackpressured = false;
|
|
41
|
-
// Subscribe to the ReplaySubject
|
|
42
|
-
const subscription = replaySubject.subscribe({
|
|
43
|
-
next: (data) => {
|
|
44
|
-
const canPush = smartStream.push(data);
|
|
45
|
-
if (!canPush) {
|
|
46
|
-
// If push returns false, pause the subscription because of backpressure
|
|
47
|
-
isBackpressured = true;
|
|
48
|
-
subscription.unsubscribe();
|
|
49
|
-
}
|
|
50
|
-
},
|
|
51
|
-
error: (err) => {
|
|
52
|
-
smartStream.emit('error', err);
|
|
53
|
-
},
|
|
54
|
-
complete: () => {
|
|
55
|
-
smartStream.push(null); // End the stream when the ReplaySubject completes
|
|
56
|
-
},
|
|
57
|
-
});
|
|
58
|
-
// Listen for 'drain' event to resume the subscription if it was paused
|
|
59
|
-
smartStream.on('drain', () => {
|
|
60
|
-
if (isBackpressured) {
|
|
61
|
-
isBackpressured = false;
|
|
62
|
-
// Resubscribe to the ReplaySubject since we previously paused
|
|
63
|
-
smartStream.observableSubscription = replaySubject.subscribe({
|
|
64
|
-
next: (data) => {
|
|
65
|
-
if (!smartStream.push(data)) {
|
|
66
|
-
smartStream.observableSubscription?.unsubscribe();
|
|
67
|
-
isBackpressured = true;
|
|
68
|
-
}
|
|
69
|
-
},
|
|
70
|
-
// No need to repeat error and complete handling here because it's already set up above
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
});
|
|
74
|
-
return smartStream;
|
|
13
|
+
debugLog(messageArg) {
|
|
14
|
+
if (this.options.debug) {
|
|
15
|
+
console.log(messageArg);
|
|
16
|
+
}
|
|
75
17
|
}
|
|
76
18
|
constructor(optionsArg) {
|
|
77
19
|
super(optionsArg);
|
|
20
|
+
// INSTANCE
|
|
21
|
+
this.backpressuredArray = new plugins.lik.BackpressuredArray();
|
|
78
22
|
this.asyncWritePromiseObjectmap = new plugins.lik.ObjectMap();
|
|
79
|
-
this.
|
|
80
|
-
this.writeFunction = optionsArg?.writeFunction;
|
|
81
|
-
this.finalFunction = optionsArg?.finalFunction;
|
|
82
|
-
this.handleBackpressure = optionsArg?.handleBackpressure ?? true;
|
|
23
|
+
this.options = optionsArg;
|
|
83
24
|
}
|
|
84
25
|
async _read(size) {
|
|
85
|
-
|
|
86
|
-
|
|
26
|
+
await this.backpressuredArray.waitForItems();
|
|
27
|
+
this.debugLog(`${this.options.name}: read was called`);
|
|
28
|
+
if (this.options.readFunction) {
|
|
29
|
+
await this.options.readFunction();
|
|
30
|
+
}
|
|
31
|
+
let canPushMore = true;
|
|
32
|
+
while (this.backpressuredArray.data.length > 0 && canPushMore) {
|
|
33
|
+
const nextChunk = this.backpressuredArray.shift();
|
|
34
|
+
if (nextChunk) {
|
|
35
|
+
canPushMore = this.push(nextChunk);
|
|
36
|
+
}
|
|
87
37
|
}
|
|
88
38
|
}
|
|
89
39
|
// Ensure the _write method types the chunk as TInput and encodes TOutput
|
|
90
40
|
async _write(chunk, encoding, callback) {
|
|
91
|
-
if (!this.writeFunction) {
|
|
41
|
+
if (!this.options.writeFunction) {
|
|
92
42
|
return callback(new Error('No stream function provided'));
|
|
93
43
|
}
|
|
44
|
+
let isTruncated = false;
|
|
94
45
|
const tools = {
|
|
95
46
|
truncate: () => {
|
|
96
47
|
this.push(null);
|
|
48
|
+
isTruncated = true;
|
|
97
49
|
callback();
|
|
98
50
|
},
|
|
99
|
-
push: (pushArg) =>
|
|
51
|
+
push: (pushArg) => {
|
|
52
|
+
this.backpressuredArray.push(pushArg);
|
|
53
|
+
},
|
|
100
54
|
};
|
|
101
55
|
try {
|
|
102
56
|
const writeDeferred = plugins.smartpromise.defer();
|
|
103
57
|
this.asyncWritePromiseObjectmap.add(writeDeferred.promise);
|
|
104
|
-
const modifiedChunk = await this.writeFunction(chunk, tools);
|
|
58
|
+
const modifiedChunk = await this.options.writeFunction(chunk, tools);
|
|
59
|
+
if (isTruncated) {
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
105
62
|
if (modifiedChunk) {
|
|
106
|
-
const
|
|
107
|
-
this.once('drain', () => {
|
|
108
|
-
drainDeferred.resolve();
|
|
109
|
-
});
|
|
110
|
-
const canPushMore = this.push(modifiedChunk);
|
|
63
|
+
const canPushMore = this.backpressuredArray.push(modifiedChunk);
|
|
111
64
|
if (!canPushMore) {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
writeDeferred.resolve();
|
|
116
|
-
}
|
|
117
|
-
else {
|
|
118
|
-
callback();
|
|
119
|
-
writeDeferred.resolve();
|
|
65
|
+
this.debugLog(`${this.options.name}: cannot push more`);
|
|
66
|
+
await this.backpressuredArray.waitForSpace();
|
|
67
|
+
this.debugLog(`${this.options.name}: can push more again`);
|
|
120
68
|
}
|
|
121
69
|
}
|
|
122
|
-
|
|
123
|
-
callback();
|
|
124
|
-
writeDeferred.resolve();
|
|
125
|
-
}
|
|
70
|
+
callback();
|
|
126
71
|
writeDeferred.resolve();
|
|
127
72
|
writeDeferred.promise.then(() => {
|
|
128
73
|
this.asyncWritePromiseObjectmap.remove(writeDeferred.promise);
|
|
@@ -134,13 +79,13 @@ export class SmartDuplex extends Duplex {
|
|
|
134
79
|
}
|
|
135
80
|
async _final(callback) {
|
|
136
81
|
await Promise.all(this.asyncWritePromiseObjectmap.getArray());
|
|
137
|
-
if (this.finalFunction) {
|
|
82
|
+
if (this.options.finalFunction) {
|
|
138
83
|
const tools = {
|
|
139
84
|
truncate: () => callback(),
|
|
140
85
|
push: (pipeObject) => this.push(pipeObject),
|
|
141
86
|
};
|
|
142
87
|
try {
|
|
143
|
-
const finalChunk = await this.finalFunction(tools);
|
|
88
|
+
const finalChunk = await this.options.finalFunction(tools);
|
|
144
89
|
if (finalChunk) {
|
|
145
90
|
this.push(finalChunk);
|
|
146
91
|
}
|
|
@@ -155,4 +100,4 @@ export class SmartDuplex extends Duplex {
|
|
|
155
100
|
callback();
|
|
156
101
|
}
|
|
157
102
|
}
|
|
158
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
103
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic21hcnRzdHJlYW0uY2xhc3Nlcy5zbWFydGR1cGxleC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3RzL3NtYXJ0c3RyZWFtLmNsYXNzZXMuc21hcnRkdXBsZXgudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLE9BQU8sTUFBTSwwQkFBMEIsQ0FBQztBQUNwRCxPQUFPLEVBQUUsTUFBTSxFQUFzQixNQUFNLFFBQVEsQ0FBQztBQXlCcEQsTUFBTSxPQUFPLFdBQXlDLFNBQVEsTUFBTTtJQUNsRSxTQUFTO0lBQ1QsTUFBTSxDQUFDLFVBQVUsQ0FBQyxNQUFjLEVBQUUsT0FBdUM7UUFDdkUsTUFBTSxXQUFXLEdBQUcsSUFBSSxXQUFXLENBQUMsT0FBTyxDQUFDLENBQUM7UUFDN0MsT0FBTyxDQUFDLFFBQVEsQ0FBQyxHQUFHLEVBQUU7WUFDcEIsV0FBVyxDQUFDLElBQUksQ0FBQyxNQUFNLENBQUMsQ0FBQztZQUN6QixXQUFXLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsNkJBQTZCO1FBQ3ZELENBQUMsQ0FBQyxDQUFDO1FBQ0gsT0FBTyxXQUFXLENBQUM7SUFDckIsQ0FBQztJQU1PLFFBQVEsQ0FBQyxVQUFrQjtRQUNqQyxJQUFJLElBQUksQ0FBQyxPQUFPLENBQUMsS0FBSyxFQUFFO1lBQ3RCLE9BQU8sQ0FBQyxHQUFHLENBQUMsVUFBVSxDQUFDLENBQUM7U0FDekI7SUFDSCxDQUFDO0lBRUQsWUFBWSxVQUFpRDtRQUMzRCxLQUFLLENBQUMsVUFBVSxDQUFDLENBQUM7UUFYcEIsV0FBVztRQUNILHVCQUFrQixHQUFHLElBQUksT0FBTyxDQUFDLEdBQUcsQ0FBQyxrQkFBa0IsRUFBVyxDQUFDO1FBNkJuRSwrQkFBMEIsR0FBRyxJQUFJLE9BQU8sQ0FBQyxHQUFHLENBQUMsU0FBUyxFQUFnQixDQUFDO1FBbEI3RSxJQUFJLENBQUMsT0FBTyxHQUFHLFVBQVUsQ0FBQztJQUM1QixDQUFDO0lBRU0sS0FBSyxDQUFDLEtBQUssQ0FBQyxJQUFZO1FBQzdCLE1BQU0sSUFBSSxDQUFDLGtCQUFrQixDQUFDLFlBQVksRUFBRSxDQUFDO1FBQzdDLElBQUksQ0FBQyxRQUFRLENBQUMsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLElBQUksbUJBQW1CLENBQUMsQ0FBQztRQUN2RCxJQUFJLElBQUksQ0FBQyxPQUFPLENBQUMsWUFBWSxFQUFFO1lBQzdCLE1BQU0sSUFBSSxDQUFDLE9BQU8sQ0FBQyxZQUFZLEVBQUUsQ0FBQztTQUNuQztRQUNELElBQUksV0FBVyxHQUFHLElBQUksQ0FBQztRQUN2QixPQUFNLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxJQUFJLENBQUMsTUFBTSxHQUFHLENBQUMsSUFBSSxXQUFXLEVBQUU7WUFDNUQsTUFBTSxTQUFTLEdBQUcsSUFBSSxDQUFDLGtCQUFrQixDQUFDLEtBQUssRUFBRSxDQUFDO1lBQ2xELElBQUksU0FBUyxFQUFFO2dCQUNiLFdBQVcsR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxDQUFDO2FBQ3BDO1NBQ0Y7SUFDSCxDQUFDO0lBR0QseUVBQXlFO0lBQ2xFLEtBQUssQ0FBQyxNQUFNLENBQUMsS0FBYSxFQUFFLFFBQWdCLEVBQUUsUUFBd0M7UUFDM0YsSUFBSSxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsYUFBYSxFQUFFO1lBQy9CLE9BQU8sUUFBUSxDQUFDLElBQUksS0FBSyxDQUFDLDZCQUE2QixDQUFDLENBQUMsQ0FBQztTQUMzRDtRQUVELElBQUksV0FBVyxHQUFHLEtBQUssQ0FBQztRQUN4QixNQUFNLEtBQUssR0FBaUI7WUFDMUIsUUFBUSxFQUFFLEdBQUcsRUFBRTtnQkFDYixJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO2dCQUNoQixXQUFXLEdBQUcsSUFBSSxDQUFDO2dCQUNuQixRQUFRLEVBQUUsQ0FBQztZQUNiLENBQUM7WUFDRCxJQUFJLEVBQUUsQ0FBQyxPQUFnQixFQUFFLEVBQUU7Z0JBQ3pCLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFDLENBQUM7WUFDeEMsQ0FBQztTQUNGLENBQUM7UUFFRixJQUFJO1lBQ0YsTUFBTSxhQUFhLEdBQUcsT0FBTyxDQUFDLFlBQVksQ0FBQyxLQUFLLEVBQUUsQ0FBQztZQUNuRCxJQUFJLENBQUMsMEJBQTBCLENBQUMsR0FBRyxDQUFDLGFBQWEsQ0FBQyxPQUFPLENBQUMsQ0FBQztZQUMzRCxNQUFNLGFBQWEsR0FBRyxNQUFNLElBQUksQ0FBQyxPQUFPLENBQUMsYUFBYSxDQUFDLEtBQUssRUFBRSxLQUFLLENBQUMsQ0FBQztZQUNyRSxJQUFJLFdBQVcsRUFBRTtnQkFDZixPQUFPO2FBQ1I7WUFDRCxJQUFJLGFBQWEsRUFBRTtnQkFDakIsTUFBTSxXQUFXLEdBQUcsSUFBSSxDQUFDLGtCQUFrQixDQUFDLElBQUksQ0FBQyxhQUFhLENBQUMsQ0FBQztnQkFDaEUsSUFBSSxDQUFDLFdBQVcsRUFBRTtvQkFDaEIsSUFBSSxDQUFDLFFBQVEsQ0FBQyxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsSUFBSSxvQkFBb0IsQ0FBQyxDQUFDO29CQUN4RCxNQUFNLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxZQUFZLEVBQUUsQ0FBQztvQkFDN0MsSUFBSSxDQUFDLFFBQVEsQ0FBQyxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsSUFBSSx1QkFBdUIsQ0FBQyxDQUFDO2lCQUM1RDthQUNGO1lBQ0QsUUFBUSxFQUFFLENBQUM7WUFDWCxhQUFhLENBQUMsT0FBTyxFQUFFLENBQUM7WUFDeEIsYUFBYSxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMsR0FBRyxFQUFFO2dCQUM5QixJQUFJLENBQUMsMEJBQTBCLENBQUMsTUFBTSxDQUFDLGFBQWEsQ0FBQyxPQUFPLENBQUMsQ0FBQztZQUNoRSxDQUFDLENBQUMsQ0FBQztTQUNKO1FBQUMsT0FBTyxHQUFHLEVBQUU7WUFDWixRQUFRLENBQUMsR0FBRyxDQUFDLENBQUM7U0FDZjtJQUNILENBQUM7SUFFTSxLQUFLLENBQUMsTUFBTSxDQUFDLFFBQXdDO1FBQzFELE1BQU0sT0FBTyxDQUFDLEdBQUcsQ0FBQyxJQUFJLENBQUMsMEJBQTBCLENBQUMsUUFBUSxFQUFFLENBQUMsQ0FBQztRQUM5RCxJQUFJLElBQUksQ0FBQyxPQUFPLENBQUMsYUFBYSxFQUFFO1lBQzlCLE1BQU0sS0FBSyxHQUFpQjtnQkFDMUIsUUFBUSxFQUFFLEdBQUcsRUFBRSxDQUFDLFFBQVEsRUFBRTtnQkFDMUIsSUFBSSxFQUFFLENBQUMsVUFBVSxFQUFFLEVBQUUsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQzthQUM1QyxDQUFDO1lBRUYsSUFBSTtnQkFDRixNQUFNLFVBQVUsR0FBRyxNQUFNLElBQUksQ0FBQyxPQUFPLENBQUMsYUFBYSxDQUFDLEtBQUssQ0FBQyxDQUFDO2dCQUMzRCxJQUFJLFVBQVUsRUFBRTtvQkFDZCxJQUFJLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDO2lCQUN2QjthQUNGO1lBQUMsT0FBTyxHQUFHLEVBQUU7Z0JBQ1osSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztnQkFDaEIsUUFBUSxDQUFDLEdBQUcsQ0FBQyxDQUFDO2dCQUNkLE9BQU87YUFDUjtTQUNGO1FBQ0QsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztRQUNoQixRQUFRLEVBQUUsQ0FBQztJQUNiLENBQUM7Q0FDRiJ9
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@push.rocks/smartstream",
|
|
3
|
-
"version": "3.0.
|
|
3
|
+
"version": "3.0.20",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "simplifies access to node streams",
|
|
6
6
|
"main": "dist_ts/index.js",
|
|
@@ -31,7 +31,7 @@
|
|
|
31
31
|
"@types/node": "^20.9.0"
|
|
32
32
|
},
|
|
33
33
|
"dependencies": {
|
|
34
|
-
"@push.rocks/lik": "^6.0.
|
|
34
|
+
"@push.rocks/lik": "^6.0.12",
|
|
35
35
|
"@push.rocks/smartpromise": "^4.0.3",
|
|
36
36
|
"@push.rocks/smartrx": "^3.0.7"
|
|
37
37
|
},
|
package/ts/00_commitinfo_data.ts
CHANGED
|
@@ -15,6 +15,8 @@ export interface IStreamFinalFunction<rT> {
|
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
export interface ISmartDuplexOptions<TInput, TOutput> extends DuplexOptions {
|
|
18
|
+
debug?: boolean;
|
|
19
|
+
name?: string;
|
|
18
20
|
handleBackpressure?: boolean;
|
|
19
21
|
readFunction?: () => Promise<void>;
|
|
20
22
|
writeFunction?: IStreamWriteFunction<TInput, TOutput>;
|
|
@@ -33,141 +35,71 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
|
|
|
33
35
|
return smartDuplex;
|
|
34
36
|
}
|
|
35
37
|
|
|
36
|
-
static fromObservable(
|
|
37
|
-
observable: plugins.smartrx.rxjs.Observable<any>,
|
|
38
|
-
options?: ISmartDuplexOptions<any, any>
|
|
39
|
-
): SmartDuplex {
|
|
40
|
-
const smartStream = new SmartDuplex(options);
|
|
41
|
-
smartStream.observableSubscription = observable.subscribe({
|
|
42
|
-
next: (data) => {
|
|
43
|
-
if (!smartStream.push(data) && smartStream.handleBackpressure) {
|
|
44
|
-
// Pause the observable if the stream buffer is full
|
|
45
|
-
smartStream.observableSubscription?.unsubscribe();
|
|
46
|
-
smartStream.once('drain', () => {
|
|
47
|
-
// Resume the observable when the stream buffer is drained
|
|
48
|
-
smartStream.observableSubscription?.unsubscribe();
|
|
49
|
-
smartStream.observableSubscription = observable.subscribe((data) => {
|
|
50
|
-
smartStream.push(data);
|
|
51
|
-
});
|
|
52
|
-
});
|
|
53
|
-
}
|
|
54
|
-
},
|
|
55
|
-
error: (err) => {
|
|
56
|
-
smartStream.emit('error', err);
|
|
57
|
-
},
|
|
58
|
-
complete: () => {
|
|
59
|
-
smartStream.push(null); // Signal the end of the data
|
|
60
|
-
},
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
return smartStream;
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
static fromReplaySubject(
|
|
67
|
-
replaySubject: plugins.smartrx.rxjs.ReplaySubject<any>,
|
|
68
|
-
options?: DuplexOptions
|
|
69
|
-
): SmartDuplex {
|
|
70
|
-
const smartStream = new SmartDuplex(options);
|
|
71
|
-
let isBackpressured = false;
|
|
72
|
-
|
|
73
|
-
// Subscribe to the ReplaySubject
|
|
74
|
-
const subscription = replaySubject.subscribe({
|
|
75
|
-
next: (data) => {
|
|
76
|
-
const canPush = smartStream.push(data);
|
|
77
|
-
if (!canPush) {
|
|
78
|
-
// If push returns false, pause the subscription because of backpressure
|
|
79
|
-
isBackpressured = true;
|
|
80
|
-
subscription.unsubscribe();
|
|
81
|
-
}
|
|
82
|
-
},
|
|
83
|
-
error: (err) => {
|
|
84
|
-
smartStream.emit('error', err);
|
|
85
|
-
},
|
|
86
|
-
complete: () => {
|
|
87
|
-
smartStream.push(null); // End the stream when the ReplaySubject completes
|
|
88
|
-
},
|
|
89
|
-
});
|
|
90
|
-
|
|
91
|
-
// Listen for 'drain' event to resume the subscription if it was paused
|
|
92
|
-
smartStream.on('drain', () => {
|
|
93
|
-
if (isBackpressured) {
|
|
94
|
-
isBackpressured = false;
|
|
95
|
-
// Resubscribe to the ReplaySubject since we previously paused
|
|
96
|
-
smartStream.observableSubscription = replaySubject.subscribe({
|
|
97
|
-
next: (data) => {
|
|
98
|
-
if (!smartStream.push(data)) {
|
|
99
|
-
smartStream.observableSubscription?.unsubscribe();
|
|
100
|
-
isBackpressured = true;
|
|
101
|
-
}
|
|
102
|
-
},
|
|
103
|
-
// No need to repeat error and complete handling here because it's already set up above
|
|
104
|
-
});
|
|
105
|
-
}
|
|
106
|
-
});
|
|
107
|
-
|
|
108
|
-
return smartStream;
|
|
109
|
-
}
|
|
110
|
-
|
|
111
38
|
// INSTANCE
|
|
112
|
-
private
|
|
113
|
-
|
|
114
|
-
private writeFunction?: IStreamWriteFunction<TInput, TOutput>;
|
|
115
|
-
private finalFunction?: IStreamFinalFunction<TOutput>;
|
|
39
|
+
private backpressuredArray = new plugins.lik.BackpressuredArray<TOutput>();
|
|
40
|
+
public options: ISmartDuplexOptions<TInput, TOutput>;
|
|
116
41
|
private observableSubscription?: plugins.smartrx.rxjs.Subscription;
|
|
42
|
+
private debugLog(messageArg: string) {
|
|
43
|
+
if (this.options.debug) {
|
|
44
|
+
console.log(messageArg);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
117
47
|
|
|
118
48
|
constructor(optionsArg?: ISmartDuplexOptions<TInput, TOutput>) {
|
|
119
49
|
super(optionsArg);
|
|
120
|
-
this.
|
|
121
|
-
this.writeFunction = optionsArg?.writeFunction;
|
|
122
|
-
this.finalFunction = optionsArg?.finalFunction;
|
|
123
|
-
this.handleBackpressure = optionsArg?.handleBackpressure ?? true;
|
|
50
|
+
this.options = optionsArg;
|
|
124
51
|
}
|
|
125
52
|
|
|
126
53
|
public async _read(size: number): Promise<void> {
|
|
127
|
-
|
|
128
|
-
|
|
54
|
+
await this.backpressuredArray.waitForItems();
|
|
55
|
+
this.debugLog(`${this.options.name}: read was called`);
|
|
56
|
+
if (this.options.readFunction) {
|
|
57
|
+
await this.options.readFunction();
|
|
58
|
+
}
|
|
59
|
+
let canPushMore = true;
|
|
60
|
+
while(this.backpressuredArray.data.length > 0 && canPushMore) {
|
|
61
|
+
const nextChunk = this.backpressuredArray.shift();
|
|
62
|
+
if (nextChunk) {
|
|
63
|
+
canPushMore = this.push(nextChunk);
|
|
64
|
+
}
|
|
129
65
|
}
|
|
130
66
|
}
|
|
131
67
|
|
|
132
68
|
private asyncWritePromiseObjectmap = new plugins.lik.ObjectMap<Promise<any>>();
|
|
133
|
-
|
|
134
69
|
// Ensure the _write method types the chunk as TInput and encodes TOutput
|
|
135
70
|
public async _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) {
|
|
136
|
-
if (!this.writeFunction) {
|
|
71
|
+
if (!this.options.writeFunction) {
|
|
137
72
|
return callback(new Error('No stream function provided'));
|
|
138
73
|
}
|
|
139
74
|
|
|
75
|
+
let isTruncated = false;
|
|
140
76
|
const tools: IStreamTools = {
|
|
141
77
|
truncate: () => {
|
|
142
78
|
this.push(null);
|
|
79
|
+
isTruncated = true;
|
|
143
80
|
callback();
|
|
144
81
|
},
|
|
145
|
-
push: (pushArg: TOutput) =>
|
|
82
|
+
push: (pushArg: TOutput) => {
|
|
83
|
+
this.backpressuredArray.push(pushArg);
|
|
84
|
+
},
|
|
146
85
|
};
|
|
147
86
|
|
|
148
87
|
try {
|
|
149
88
|
const writeDeferred = plugins.smartpromise.defer();
|
|
150
89
|
this.asyncWritePromiseObjectmap.add(writeDeferred.promise);
|
|
151
|
-
const modifiedChunk = await this.writeFunction(chunk, tools);
|
|
90
|
+
const modifiedChunk = await this.options.writeFunction(chunk, tools);
|
|
91
|
+
if (isTruncated) {
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
152
94
|
if (modifiedChunk) {
|
|
153
|
-
const
|
|
154
|
-
this.once('drain', () => {
|
|
155
|
-
drainDeferred.resolve();
|
|
156
|
-
});
|
|
157
|
-
const canPushMore = this.push(modifiedChunk);
|
|
95
|
+
const canPushMore = this.backpressuredArray.push(modifiedChunk);
|
|
158
96
|
if (!canPushMore) {
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
writeDeferred.resolve();
|
|
163
|
-
} else {
|
|
164
|
-
callback();
|
|
165
|
-
writeDeferred.resolve();
|
|
97
|
+
this.debugLog(`${this.options.name}: cannot push more`);
|
|
98
|
+
await this.backpressuredArray.waitForSpace();
|
|
99
|
+
this.debugLog(`${this.options.name}: can push more again`);
|
|
166
100
|
}
|
|
167
|
-
} else {
|
|
168
|
-
callback();
|
|
169
|
-
writeDeferred.resolve();
|
|
170
101
|
}
|
|
102
|
+
callback();
|
|
171
103
|
writeDeferred.resolve();
|
|
172
104
|
writeDeferred.promise.then(() => {
|
|
173
105
|
this.asyncWritePromiseObjectmap.remove(writeDeferred.promise);
|
|
@@ -179,14 +111,14 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
|
|
|
179
111
|
|
|
180
112
|
public async _final(callback: (error?: Error | null) => void) {
|
|
181
113
|
await Promise.all(this.asyncWritePromiseObjectmap.getArray());
|
|
182
|
-
if (this.finalFunction) {
|
|
114
|
+
if (this.options.finalFunction) {
|
|
183
115
|
const tools: IStreamTools = {
|
|
184
116
|
truncate: () => callback(),
|
|
185
117
|
push: (pipeObject) => this.push(pipeObject),
|
|
186
118
|
};
|
|
187
119
|
|
|
188
120
|
try {
|
|
189
|
-
const finalChunk = await this.finalFunction(tools);
|
|
121
|
+
const finalChunk = await this.options.finalFunction(tools);
|
|
190
122
|
if (finalChunk) {
|
|
191
123
|
this.push(finalChunk);
|
|
192
124
|
}
|