@push.rocks/smartstream 3.0.19 → 3.0.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
export const commitinfo = {
|
|
5
5
|
name: '@push.rocks/smartstream',
|
|
6
|
-
version: '3.0.
|
|
6
|
+
version: '3.0.21',
|
|
7
7
|
description: 'simplifies access to node streams'
|
|
8
8
|
};
|
|
9
9
|
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiMDBfY29tbWl0aW5mb19kYXRhLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vdHMvMDBfY29tbWl0aW5mb19kYXRhLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBOztHQUVHO0FBQ0gsTUFBTSxDQUFDLE1BQU0sVUFBVSxHQUFHO0lBQ3hCLElBQUksRUFBRSx5QkFBeUI7SUFDL0IsT0FBTyxFQUFFLFFBQVE7SUFDakIsV0FBVyxFQUFFLG1DQUFtQztDQUNqRCxDQUFBIn0=
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
/// <reference types="node" resolution-mode="require"/>
|
|
2
2
|
/// <reference types="node" resolution-mode="require"/>
|
|
3
|
-
import * as plugins from './smartstream.plugins.js';
|
|
4
3
|
import { Duplex, type DuplexOptions } from 'stream';
|
|
5
4
|
export interface IStreamTools {
|
|
6
5
|
truncate: () => void;
|
|
@@ -13,6 +12,8 @@ export interface IStreamFinalFunction<rT> {
|
|
|
13
12
|
(toolsArg: IStreamTools): Promise<rT>;
|
|
14
13
|
}
|
|
15
14
|
export interface ISmartDuplexOptions<TInput, TOutput> extends DuplexOptions {
|
|
15
|
+
debug?: boolean;
|
|
16
|
+
name?: string;
|
|
16
17
|
handleBackpressure?: boolean;
|
|
17
18
|
readFunction?: () => Promise<void>;
|
|
18
19
|
writeFunction?: IStreamWriteFunction<TInput, TOutput>;
|
|
@@ -20,13 +21,10 @@ export interface ISmartDuplexOptions<TInput, TOutput> extends DuplexOptions {
|
|
|
20
21
|
}
|
|
21
22
|
export declare class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
|
|
22
23
|
static fromBuffer(buffer: Buffer, options?: ISmartDuplexOptions<any, any>): SmartDuplex;
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
private readFunction?;
|
|
26
|
-
private handleBackpressure;
|
|
27
|
-
private writeFunction?;
|
|
28
|
-
private finalFunction?;
|
|
24
|
+
private backpressuredArray;
|
|
25
|
+
options: ISmartDuplexOptions<TInput, TOutput>;
|
|
29
26
|
private observableSubscription?;
|
|
27
|
+
private debugLog;
|
|
30
28
|
constructor(optionsArg?: ISmartDuplexOptions<TInput, TOutput>);
|
|
31
29
|
_read(size: number): Promise<void>;
|
|
32
30
|
private asyncWritePromiseObjectmap;
|
|
@@ -10,119 +10,65 @@ export class SmartDuplex extends Duplex {
|
|
|
10
10
|
});
|
|
11
11
|
return smartDuplex;
|
|
12
12
|
}
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
if (!smartStream.push(data) && smartStream.handleBackpressure) {
|
|
18
|
-
// Pause the observable if the stream buffer is full
|
|
19
|
-
smartStream.observableSubscription?.unsubscribe();
|
|
20
|
-
smartStream.once('drain', () => {
|
|
21
|
-
// Resume the observable when the stream buffer is drained
|
|
22
|
-
smartStream.observableSubscription?.unsubscribe();
|
|
23
|
-
smartStream.observableSubscription = observable.subscribe((data) => {
|
|
24
|
-
smartStream.push(data);
|
|
25
|
-
});
|
|
26
|
-
});
|
|
27
|
-
}
|
|
28
|
-
},
|
|
29
|
-
error: (err) => {
|
|
30
|
-
smartStream.emit('error', err);
|
|
31
|
-
},
|
|
32
|
-
complete: () => {
|
|
33
|
-
smartStream.push(null); // Signal the end of the data
|
|
34
|
-
},
|
|
35
|
-
});
|
|
36
|
-
return smartStream;
|
|
37
|
-
}
|
|
38
|
-
static fromReplaySubject(replaySubject, options) {
|
|
39
|
-
const smartStream = new SmartDuplex(options);
|
|
40
|
-
let isBackpressured = false;
|
|
41
|
-
// Subscribe to the ReplaySubject
|
|
42
|
-
const subscription = replaySubject.subscribe({
|
|
43
|
-
next: (data) => {
|
|
44
|
-
const canPush = smartStream.push(data);
|
|
45
|
-
if (!canPush) {
|
|
46
|
-
// If push returns false, pause the subscription because of backpressure
|
|
47
|
-
isBackpressured = true;
|
|
48
|
-
subscription.unsubscribe();
|
|
49
|
-
}
|
|
50
|
-
},
|
|
51
|
-
error: (err) => {
|
|
52
|
-
smartStream.emit('error', err);
|
|
53
|
-
},
|
|
54
|
-
complete: () => {
|
|
55
|
-
smartStream.push(null); // End the stream when the ReplaySubject completes
|
|
56
|
-
},
|
|
57
|
-
});
|
|
58
|
-
// Listen for 'drain' event to resume the subscription if it was paused
|
|
59
|
-
smartStream.on('drain', () => {
|
|
60
|
-
if (isBackpressured) {
|
|
61
|
-
isBackpressured = false;
|
|
62
|
-
// Resubscribe to the ReplaySubject since we previously paused
|
|
63
|
-
smartStream.observableSubscription = replaySubject.subscribe({
|
|
64
|
-
next: (data) => {
|
|
65
|
-
if (!smartStream.push(data)) {
|
|
66
|
-
smartStream.observableSubscription?.unsubscribe();
|
|
67
|
-
isBackpressured = true;
|
|
68
|
-
}
|
|
69
|
-
},
|
|
70
|
-
// No need to repeat error and complete handling here because it's already set up above
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
});
|
|
74
|
-
return smartStream;
|
|
13
|
+
debugLog(messageArg) {
|
|
14
|
+
if (this.options.debug) {
|
|
15
|
+
console.log(messageArg);
|
|
16
|
+
}
|
|
75
17
|
}
|
|
76
18
|
constructor(optionsArg) {
|
|
77
19
|
super(optionsArg);
|
|
20
|
+
// INSTANCE
|
|
21
|
+
this.backpressuredArray = new plugins.lik.BackpressuredArray();
|
|
78
22
|
this.asyncWritePromiseObjectmap = new plugins.lik.ObjectMap();
|
|
79
|
-
this.
|
|
80
|
-
this.writeFunction = optionsArg?.writeFunction;
|
|
81
|
-
this.finalFunction = optionsArg?.finalFunction;
|
|
82
|
-
this.handleBackpressure = optionsArg?.handleBackpressure ?? true;
|
|
23
|
+
this.options = optionsArg;
|
|
83
24
|
}
|
|
84
25
|
async _read(size) {
|
|
85
|
-
|
|
86
|
-
|
|
26
|
+
this.debugLog(`${this.options.name}: read was called`);
|
|
27
|
+
await this.backpressuredArray.waitForItems();
|
|
28
|
+
this.debugLog(`${this.options.name}: successfully waited for items.`);
|
|
29
|
+
if (this.options.readFunction) {
|
|
30
|
+
await this.options.readFunction();
|
|
31
|
+
}
|
|
32
|
+
let canPushMore = true;
|
|
33
|
+
while (this.backpressuredArray.data.length > 0 && canPushMore) {
|
|
34
|
+
const nextChunk = this.backpressuredArray.shift();
|
|
35
|
+
if (nextChunk) {
|
|
36
|
+
canPushMore = this.push(nextChunk);
|
|
37
|
+
}
|
|
87
38
|
}
|
|
88
39
|
}
|
|
89
40
|
// Ensure the _write method types the chunk as TInput and encodes TOutput
|
|
90
41
|
async _write(chunk, encoding, callback) {
|
|
91
|
-
if (!this.writeFunction) {
|
|
42
|
+
if (!this.options.writeFunction) {
|
|
92
43
|
return callback(new Error('No stream function provided'));
|
|
93
44
|
}
|
|
45
|
+
let isTruncated = false;
|
|
94
46
|
const tools = {
|
|
95
47
|
truncate: () => {
|
|
96
48
|
this.push(null);
|
|
49
|
+
isTruncated = true;
|
|
97
50
|
callback();
|
|
98
51
|
},
|
|
99
|
-
push: (pushArg) =>
|
|
52
|
+
push: (pushArg) => {
|
|
53
|
+
this.backpressuredArray.push(pushArg);
|
|
54
|
+
},
|
|
100
55
|
};
|
|
101
56
|
try {
|
|
102
57
|
const writeDeferred = plugins.smartpromise.defer();
|
|
103
58
|
this.asyncWritePromiseObjectmap.add(writeDeferred.promise);
|
|
104
|
-
const modifiedChunk = await this.writeFunction(chunk, tools);
|
|
59
|
+
const modifiedChunk = await this.options.writeFunction(chunk, tools);
|
|
60
|
+
if (isTruncated) {
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
105
63
|
if (modifiedChunk) {
|
|
106
|
-
const
|
|
107
|
-
this.once('drain', () => {
|
|
108
|
-
drainDeferred.resolve();
|
|
109
|
-
});
|
|
110
|
-
const canPushMore = this.push(modifiedChunk);
|
|
64
|
+
const canPushMore = this.backpressuredArray.push(modifiedChunk);
|
|
111
65
|
if (!canPushMore) {
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
writeDeferred.resolve();
|
|
116
|
-
}
|
|
117
|
-
else {
|
|
118
|
-
callback();
|
|
119
|
-
writeDeferred.resolve();
|
|
66
|
+
this.debugLog(`${this.options.name}: cannot push more`);
|
|
67
|
+
await this.backpressuredArray.waitForSpace();
|
|
68
|
+
this.debugLog(`${this.options.name}: can push more again`);
|
|
120
69
|
}
|
|
121
70
|
}
|
|
122
|
-
|
|
123
|
-
callback();
|
|
124
|
-
writeDeferred.resolve();
|
|
125
|
-
}
|
|
71
|
+
callback();
|
|
126
72
|
writeDeferred.resolve();
|
|
127
73
|
writeDeferred.promise.then(() => {
|
|
128
74
|
this.asyncWritePromiseObjectmap.remove(writeDeferred.promise);
|
|
@@ -134,13 +80,13 @@ export class SmartDuplex extends Duplex {
|
|
|
134
80
|
}
|
|
135
81
|
async _final(callback) {
|
|
136
82
|
await Promise.all(this.asyncWritePromiseObjectmap.getArray());
|
|
137
|
-
if (this.finalFunction) {
|
|
83
|
+
if (this.options.finalFunction) {
|
|
138
84
|
const tools = {
|
|
139
85
|
truncate: () => callback(),
|
|
140
86
|
push: (pipeObject) => this.push(pipeObject),
|
|
141
87
|
};
|
|
142
88
|
try {
|
|
143
|
-
const finalChunk = await this.finalFunction(tools);
|
|
89
|
+
const finalChunk = await this.options.finalFunction(tools);
|
|
144
90
|
if (finalChunk) {
|
|
145
91
|
this.push(finalChunk);
|
|
146
92
|
}
|
|
@@ -155,4 +101,4 @@ export class SmartDuplex extends Duplex {
|
|
|
155
101
|
callback();
|
|
156
102
|
}
|
|
157
103
|
}
|
|
158
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
104
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic21hcnRzdHJlYW0uY2xhc3Nlcy5zbWFydGR1cGxleC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3RzL3NtYXJ0c3RyZWFtLmNsYXNzZXMuc21hcnRkdXBsZXgudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLE9BQU8sTUFBTSwwQkFBMEIsQ0FBQztBQUNwRCxPQUFPLEVBQUUsTUFBTSxFQUFzQixNQUFNLFFBQVEsQ0FBQztBQXlCcEQsTUFBTSxPQUFPLFdBQXlDLFNBQVEsTUFBTTtJQUNsRSxTQUFTO0lBQ1QsTUFBTSxDQUFDLFVBQVUsQ0FBQyxNQUFjLEVBQUUsT0FBdUM7UUFDdkUsTUFBTSxXQUFXLEdBQUcsSUFBSSxXQUFXLENBQUMsT0FBTyxDQUFDLENBQUM7UUFDN0MsT0FBTyxDQUFDLFFBQVEsQ0FBQyxHQUFHLEVBQUU7WUFDcEIsV0FBVyxDQUFDLElBQUksQ0FBQyxNQUFNLENBQUMsQ0FBQztZQUN6QixXQUFXLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsNkJBQTZCO1FBQ3ZELENBQUMsQ0FBQyxDQUFDO1FBQ0gsT0FBTyxXQUFXLENBQUM7SUFDckIsQ0FBQztJQU1PLFFBQVEsQ0FBQyxVQUFrQjtRQUNqQyxJQUFJLElBQUksQ0FBQyxPQUFPLENBQUMsS0FBSyxFQUFFO1lBQ3RCLE9BQU8sQ0FBQyxHQUFHLENBQUMsVUFBVSxDQUFDLENBQUM7U0FDekI7SUFDSCxDQUFDO0lBRUQsWUFBWSxVQUFpRDtRQUMzRCxLQUFLLENBQUMsVUFBVSxDQUFDLENBQUM7UUFYcEIsV0FBVztRQUNILHVCQUFrQixHQUFHLElBQUksT0FBTyxDQUFDLEdBQUcsQ0FBQyxrQkFBa0IsRUFBVyxDQUFDO1FBOEJuRSwrQkFBMEIsR0FBRyxJQUFJLE9BQU8sQ0FBQyxHQUFHLENBQUMsU0FBUyxFQUFnQixDQUFDO1FBbkI3RSxJQUFJLENBQUMsT0FBTyxHQUFHLFVBQVUsQ0FBQztJQUM1QixDQUFDO0lBRU0sS0FBSyxDQUFDLEtBQUssQ0FBQyxJQUFZO1FBQzdCLElBQUksQ0FBQyxRQUFRLENBQUMsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLElBQUksbUJBQW1CLENBQUMsQ0FBQztRQUN2RCxNQUFNLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxZQUFZLEVBQUUsQ0FBQztRQUM3QyxJQUFJLENBQUMsUUFBUSxDQUFDLEdBQUcsSUFBSSxDQUFDLE9BQU8sQ0FBQyxJQUFJLGtDQUFrQyxDQUFDLENBQUM7UUFDdEUsSUFBSSxJQUFJLENBQUMsT0FBTyxDQUFDLFlBQVksRUFBRTtZQUM3QixNQUFNLElBQUksQ0FBQyxPQUFPLENBQUMsWUFBWSxFQUFFLENBQUM7U0FDbkM7UUFDRCxJQUFJLFdBQVcsR0FBRyxJQUFJLENBQUM7UUFDdkIsT0FBTSxJQUFJLENBQUMsa0JBQWtCLENBQUMsSUFBSSxDQUFDLE1BQU0sR0FBRyxDQUFDLElBQUksV0FBVyxFQUFFO1lBQzVELE1BQU0sU0FBUyxHQUFHLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxLQUFLLEVBQUUsQ0FBQztZQUNsRCxJQUFJLFNBQVMsRUFBRTtnQkFDYixXQUFXLEdBQUcsSUFBSSxDQUFDLElBQUksQ0FBQyxTQUFTLENBQUMsQ0FBQzthQUNwQztTQUNGO0lBQ0gsQ0FBQztJQUdELHlFQUF5RTtJQUNsRSxLQUFLLENBQUMsTUFBTSxDQUFDLEtBQWEsRUFBRSxRQUFnQixFQUFFLFFBQXdDO1FBQzNGLElBQUksQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFDLGFBQWEsRUFBRTtZQUMvQixPQUFPLFFBQVEsQ0FBQyxJQUFJLEtBQUssQ0FBQyw2QkFBNkIsQ0FBQyxDQUFDLENBQUM7U0FDM0Q7UUFFRCxJQUFJLFdBQVcsR0FBRyxLQUFLLENBQUM7UUFDeEIsTUFBTSxLQUFLLEdBQWlCO1lBQzFCLFFBQVEsRUFBRSxHQUFHLEVBQUU7Z0JBQ2IsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztnQkFDaEIsV0FBVyxHQUFHLElBQUksQ0FBQztnQkFDbkIsUUFBUSxFQUFFLENBQUM7WUFDYixDQUFDO1lBQ0QsSUFBSSxFQUFFLENBQUMsT0FBZ0IsRUFBRSxFQUFFO2dCQUN6QixJQUFJLENBQUMsa0JBQWtCLENBQUMsSUFBSSxDQUFDLE9BQU8sQ0FBQyxDQUFDO1lBQ3hDLENBQUM7U0FDRixDQUFDO1FBRUYsSUFBSTtZQUNGLE1BQU0sYUFBYSxHQUFHLE9BQU8sQ0FBQyxZQUFZLENBQUMsS0FBSyxFQUFFLENBQUM7WUFDbkQsSUFBSSxDQUFDLDBCQUEwQixDQUFDLEdBQUcsQ0FBQyxhQUFhLENBQUMsT0FBTyxDQUFDLENBQUM7WUFDM0QsTUFBTSxhQUFhLEdBQUcsTUFBTSxJQUFJLENBQUMsT0FBTyxDQUFDLGFBQWEsQ0FBQyxLQUFLLEVBQUUsS0FBSyxDQUFDLENBQUM7WUFDckUsSUFBSSxXQUFXLEVBQUU7Z0JBQ2YsT0FBTzthQUNSO1lBQ0QsSUFBSSxhQUFhLEVBQUU7Z0JBQ2pCLE1BQU0sV0FBVyxHQUFHLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxJQUFJLENBQUMsYUFBYSxDQUFDLENBQUM7Z0JBQ2hFLElBQUksQ0FBQyxXQUFXLEVBQUU7b0JBQ2hCLElBQUksQ0FBQyxRQUFRLENBQUMsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLElBQUksb0JBQW9CLENBQUMsQ0FBQztvQkFDeEQsTUFBTSxJQUFJLENBQUMsa0JBQWtCLENBQUMsWUFBWSxFQUFFLENBQUM7b0JBQzdDLElBQUksQ0FBQyxRQUFRLENBQUMsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLElBQUksdUJBQXVCLENBQUMsQ0FBQztpQkFDNUQ7YUFDRjtZQUNELFFBQVEsRUFBRSxDQUFDO1lBQ1gsYUFBYSxDQUFDLE9BQU8sRUFBRSxDQUFDO1lBQ3hCLGFBQWEsQ0FBQyxPQUFPLENBQUMsSUFBSSxDQUFDLEdBQUcsRUFBRTtnQkFDOUIsSUFBSSxDQUFDLDBCQUEwQixDQUFDLE1BQU0sQ0FBQyxhQUFhLENBQUMsT0FBTyxDQUFDLENBQUM7WUFDaEUsQ0FBQyxDQUFDLENBQUM7U0FDSjtRQUFDLE9BQU8sR0FBRyxFQUFFO1lBQ1osUUFBUSxDQUFDLEdBQUcsQ0FBQyxDQUFDO1NBQ2Y7SUFDSCxDQUFDO0lBRU0sS0FBSyxDQUFDLE1BQU0sQ0FBQyxRQUF3QztRQUMxRCxNQUFNLE9BQU8sQ0FBQyxHQUFHLENBQUMsSUFBSSxDQUFDLDBCQUEwQixDQUFDLFFBQVEsRUFBRSxDQUFDLENBQUM7UUFDOUQsSUFBSSxJQUFJLENBQUMsT0FBTyxDQUFDLGFBQWEsRUFBRTtZQUM5QixNQUFNLEtBQUssR0FBaUI7Z0JBQzFCLFFBQVEsRUFBRSxHQUFHLEVBQUUsQ0FBQyxRQUFRLEVBQUU7Z0JBQzFCLElBQUksRUFBRSxDQUFDLFVBQVUsRUFBRSxFQUFFLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUM7YUFDNUMsQ0FBQztZQUVGLElBQUk7Z0JBQ0YsTUFBTSxVQUFVLEdBQUcsTUFBTSxJQUFJLENBQUMsT0FBTyxDQUFDLGFBQWEsQ0FBQyxLQUFLLENBQUMsQ0FBQztnQkFDM0QsSUFBSSxVQUFVLEVBQUU7b0JBQ2QsSUFBSSxDQUFDLElBQUksQ0FBQyxVQUFVLENBQUMsQ0FBQztpQkFDdkI7YUFDRjtZQUFDLE9BQU8sR0FBRyxFQUFFO2dCQUNaLElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7Z0JBQ2hCLFFBQVEsQ0FBQyxHQUFHLENBQUMsQ0FBQztnQkFDZCxPQUFPO2FBQ1I7U0FDRjtRQUNELElBQUksQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLENBQUM7UUFDaEIsUUFBUSxFQUFFLENBQUM7SUFDYixDQUFDO0NBQ0YifQ==
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@push.rocks/smartstream",
|
|
3
|
-
"version": "3.0.
|
|
3
|
+
"version": "3.0.21",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "simplifies access to node streams",
|
|
6
6
|
"main": "dist_ts/index.js",
|
|
@@ -25,13 +25,12 @@
|
|
|
25
25
|
"@git.zone/tsbuild": "^2.1.66",
|
|
26
26
|
"@git.zone/tsrun": "^1.2.44",
|
|
27
27
|
"@git.zone/tstest": "^1.0.84",
|
|
28
|
-
"@push.rocks/smartdelay": "^3.0.5",
|
|
29
28
|
"@push.rocks/smartfile": "^11.0.0",
|
|
30
29
|
"@push.rocks/tapbundle": "^5.0.15",
|
|
31
30
|
"@types/node": "^20.9.0"
|
|
32
31
|
},
|
|
33
32
|
"dependencies": {
|
|
34
|
-
"@push.rocks/lik": "^6.0.
|
|
33
|
+
"@push.rocks/lik": "^6.0.12",
|
|
35
34
|
"@push.rocks/smartpromise": "^4.0.3",
|
|
36
35
|
"@push.rocks/smartrx": "^3.0.7"
|
|
37
36
|
},
|
package/ts/00_commitinfo_data.ts
CHANGED
|
@@ -15,6 +15,8 @@ export interface IStreamFinalFunction<rT> {
|
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
export interface ISmartDuplexOptions<TInput, TOutput> extends DuplexOptions {
|
|
18
|
+
debug?: boolean;
|
|
19
|
+
name?: string;
|
|
18
20
|
handleBackpressure?: boolean;
|
|
19
21
|
readFunction?: () => Promise<void>;
|
|
20
22
|
writeFunction?: IStreamWriteFunction<TInput, TOutput>;
|
|
@@ -33,141 +35,72 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
|
|
|
33
35
|
return smartDuplex;
|
|
34
36
|
}
|
|
35
37
|
|
|
36
|
-
static fromObservable(
|
|
37
|
-
observable: plugins.smartrx.rxjs.Observable<any>,
|
|
38
|
-
options?: ISmartDuplexOptions<any, any>
|
|
39
|
-
): SmartDuplex {
|
|
40
|
-
const smartStream = new SmartDuplex(options);
|
|
41
|
-
smartStream.observableSubscription = observable.subscribe({
|
|
42
|
-
next: (data) => {
|
|
43
|
-
if (!smartStream.push(data) && smartStream.handleBackpressure) {
|
|
44
|
-
// Pause the observable if the stream buffer is full
|
|
45
|
-
smartStream.observableSubscription?.unsubscribe();
|
|
46
|
-
smartStream.once('drain', () => {
|
|
47
|
-
// Resume the observable when the stream buffer is drained
|
|
48
|
-
smartStream.observableSubscription?.unsubscribe();
|
|
49
|
-
smartStream.observableSubscription = observable.subscribe((data) => {
|
|
50
|
-
smartStream.push(data);
|
|
51
|
-
});
|
|
52
|
-
});
|
|
53
|
-
}
|
|
54
|
-
},
|
|
55
|
-
error: (err) => {
|
|
56
|
-
smartStream.emit('error', err);
|
|
57
|
-
},
|
|
58
|
-
complete: () => {
|
|
59
|
-
smartStream.push(null); // Signal the end of the data
|
|
60
|
-
},
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
return smartStream;
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
static fromReplaySubject(
|
|
67
|
-
replaySubject: plugins.smartrx.rxjs.ReplaySubject<any>,
|
|
68
|
-
options?: DuplexOptions
|
|
69
|
-
): SmartDuplex {
|
|
70
|
-
const smartStream = new SmartDuplex(options);
|
|
71
|
-
let isBackpressured = false;
|
|
72
|
-
|
|
73
|
-
// Subscribe to the ReplaySubject
|
|
74
|
-
const subscription = replaySubject.subscribe({
|
|
75
|
-
next: (data) => {
|
|
76
|
-
const canPush = smartStream.push(data);
|
|
77
|
-
if (!canPush) {
|
|
78
|
-
// If push returns false, pause the subscription because of backpressure
|
|
79
|
-
isBackpressured = true;
|
|
80
|
-
subscription.unsubscribe();
|
|
81
|
-
}
|
|
82
|
-
},
|
|
83
|
-
error: (err) => {
|
|
84
|
-
smartStream.emit('error', err);
|
|
85
|
-
},
|
|
86
|
-
complete: () => {
|
|
87
|
-
smartStream.push(null); // End the stream when the ReplaySubject completes
|
|
88
|
-
},
|
|
89
|
-
});
|
|
90
|
-
|
|
91
|
-
// Listen for 'drain' event to resume the subscription if it was paused
|
|
92
|
-
smartStream.on('drain', () => {
|
|
93
|
-
if (isBackpressured) {
|
|
94
|
-
isBackpressured = false;
|
|
95
|
-
// Resubscribe to the ReplaySubject since we previously paused
|
|
96
|
-
smartStream.observableSubscription = replaySubject.subscribe({
|
|
97
|
-
next: (data) => {
|
|
98
|
-
if (!smartStream.push(data)) {
|
|
99
|
-
smartStream.observableSubscription?.unsubscribe();
|
|
100
|
-
isBackpressured = true;
|
|
101
|
-
}
|
|
102
|
-
},
|
|
103
|
-
// No need to repeat error and complete handling here because it's already set up above
|
|
104
|
-
});
|
|
105
|
-
}
|
|
106
|
-
});
|
|
107
|
-
|
|
108
|
-
return smartStream;
|
|
109
|
-
}
|
|
110
|
-
|
|
111
38
|
// INSTANCE
|
|
112
|
-
private
|
|
113
|
-
|
|
114
|
-
private writeFunction?: IStreamWriteFunction<TInput, TOutput>;
|
|
115
|
-
private finalFunction?: IStreamFinalFunction<TOutput>;
|
|
39
|
+
private backpressuredArray = new plugins.lik.BackpressuredArray<TOutput>();
|
|
40
|
+
public options: ISmartDuplexOptions<TInput, TOutput>;
|
|
116
41
|
private observableSubscription?: plugins.smartrx.rxjs.Subscription;
|
|
42
|
+
private debugLog(messageArg: string) {
|
|
43
|
+
if (this.options.debug) {
|
|
44
|
+
console.log(messageArg);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
117
47
|
|
|
118
48
|
constructor(optionsArg?: ISmartDuplexOptions<TInput, TOutput>) {
|
|
119
49
|
super(optionsArg);
|
|
120
|
-
this.
|
|
121
|
-
this.writeFunction = optionsArg?.writeFunction;
|
|
122
|
-
this.finalFunction = optionsArg?.finalFunction;
|
|
123
|
-
this.handleBackpressure = optionsArg?.handleBackpressure ?? true;
|
|
50
|
+
this.options = optionsArg;
|
|
124
51
|
}
|
|
125
52
|
|
|
126
53
|
public async _read(size: number): Promise<void> {
|
|
127
|
-
|
|
128
|
-
|
|
54
|
+
this.debugLog(`${this.options.name}: read was called`);
|
|
55
|
+
await this.backpressuredArray.waitForItems();
|
|
56
|
+
this.debugLog(`${this.options.name}: successfully waited for items.`);
|
|
57
|
+
if (this.options.readFunction) {
|
|
58
|
+
await this.options.readFunction();
|
|
59
|
+
}
|
|
60
|
+
let canPushMore = true;
|
|
61
|
+
while(this.backpressuredArray.data.length > 0 && canPushMore) {
|
|
62
|
+
const nextChunk = this.backpressuredArray.shift();
|
|
63
|
+
if (nextChunk) {
|
|
64
|
+
canPushMore = this.push(nextChunk);
|
|
65
|
+
}
|
|
129
66
|
}
|
|
130
67
|
}
|
|
131
68
|
|
|
132
69
|
private asyncWritePromiseObjectmap = new plugins.lik.ObjectMap<Promise<any>>();
|
|
133
|
-
|
|
134
70
|
// Ensure the _write method types the chunk as TInput and encodes TOutput
|
|
135
71
|
public async _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) {
|
|
136
|
-
if (!this.writeFunction) {
|
|
72
|
+
if (!this.options.writeFunction) {
|
|
137
73
|
return callback(new Error('No stream function provided'));
|
|
138
74
|
}
|
|
139
75
|
|
|
76
|
+
let isTruncated = false;
|
|
140
77
|
const tools: IStreamTools = {
|
|
141
78
|
truncate: () => {
|
|
142
79
|
this.push(null);
|
|
80
|
+
isTruncated = true;
|
|
143
81
|
callback();
|
|
144
82
|
},
|
|
145
|
-
push: (pushArg: TOutput) =>
|
|
83
|
+
push: (pushArg: TOutput) => {
|
|
84
|
+
this.backpressuredArray.push(pushArg);
|
|
85
|
+
},
|
|
146
86
|
};
|
|
147
87
|
|
|
148
88
|
try {
|
|
149
89
|
const writeDeferred = plugins.smartpromise.defer();
|
|
150
90
|
this.asyncWritePromiseObjectmap.add(writeDeferred.promise);
|
|
151
|
-
const modifiedChunk = await this.writeFunction(chunk, tools);
|
|
91
|
+
const modifiedChunk = await this.options.writeFunction(chunk, tools);
|
|
92
|
+
if (isTruncated) {
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
152
95
|
if (modifiedChunk) {
|
|
153
|
-
const
|
|
154
|
-
this.once('drain', () => {
|
|
155
|
-
drainDeferred.resolve();
|
|
156
|
-
});
|
|
157
|
-
const canPushMore = this.push(modifiedChunk);
|
|
96
|
+
const canPushMore = this.backpressuredArray.push(modifiedChunk);
|
|
158
97
|
if (!canPushMore) {
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
writeDeferred.resolve();
|
|
163
|
-
} else {
|
|
164
|
-
callback();
|
|
165
|
-
writeDeferred.resolve();
|
|
98
|
+
this.debugLog(`${this.options.name}: cannot push more`);
|
|
99
|
+
await this.backpressuredArray.waitForSpace();
|
|
100
|
+
this.debugLog(`${this.options.name}: can push more again`);
|
|
166
101
|
}
|
|
167
|
-
} else {
|
|
168
|
-
callback();
|
|
169
|
-
writeDeferred.resolve();
|
|
170
102
|
}
|
|
103
|
+
callback();
|
|
171
104
|
writeDeferred.resolve();
|
|
172
105
|
writeDeferred.promise.then(() => {
|
|
173
106
|
this.asyncWritePromiseObjectmap.remove(writeDeferred.promise);
|
|
@@ -179,14 +112,14 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
|
|
|
179
112
|
|
|
180
113
|
public async _final(callback: (error?: Error | null) => void) {
|
|
181
114
|
await Promise.all(this.asyncWritePromiseObjectmap.getArray());
|
|
182
|
-
if (this.finalFunction) {
|
|
115
|
+
if (this.options.finalFunction) {
|
|
183
116
|
const tools: IStreamTools = {
|
|
184
117
|
truncate: () => callback(),
|
|
185
118
|
push: (pipeObject) => this.push(pipeObject),
|
|
186
119
|
};
|
|
187
120
|
|
|
188
121
|
try {
|
|
189
|
-
const finalChunk = await this.finalFunction(tools);
|
|
122
|
+
const finalChunk = await this.options.finalFunction(tools);
|
|
190
123
|
if (finalChunk) {
|
|
191
124
|
this.push(finalChunk);
|
|
192
125
|
}
|