@push.rocks/smartstream 3.0.18 → 3.0.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist_ts/00_commitinfo_data.js +1 -1
- package/dist_ts/smartstream.classes.smartduplex.d.ts +7 -11
- package/dist_ts/smartstream.classes.smartduplex.js +46 -105
- package/dist_ts/smartstream.plugins.d.ts +2 -1
- package/dist_ts/smartstream.plugins.js +3 -2
- package/package.json +3 -1
- package/ts/00_commitinfo_data.ts +1 -1
- package/ts/smartstream.classes.smartduplex.ts +49 -120
- package/ts/smartstream.plugins.ts +2 -1
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
*/
|
|
4
4
|
export const commitinfo = {
|
|
5
5
|
name: '@push.rocks/smartstream',
|
|
6
|
-
version: '3.0.
|
|
6
|
+
version: '3.0.20',
|
|
7
7
|
description: 'simplifies access to node streams'
|
|
8
8
|
};
|
|
9
9
|
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiMDBfY29tbWl0aW5mb19kYXRhLmpzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vdHMvMDBfY29tbWl0aW5mb19kYXRhLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBOztHQUVHO0FBQ0gsTUFBTSxDQUFDLE1BQU0sVUFBVSxHQUFHO0lBQ3hCLElBQUksRUFBRSx5QkFBeUI7SUFDL0IsT0FBTyxFQUFFLFFBQVE7SUFDakIsV0FBVyxFQUFFLG1DQUFtQztDQUNqRCxDQUFBIn0=
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
/// <reference types="node" resolution-mode="require"/>
|
|
2
2
|
/// <reference types="node" resolution-mode="require"/>
|
|
3
|
-
import * as plugins from './smartstream.plugins.js';
|
|
4
3
|
import { Duplex, type DuplexOptions } from 'stream';
|
|
5
4
|
export interface IStreamTools {
|
|
6
5
|
truncate: () => void;
|
|
@@ -13,6 +12,8 @@ export interface IStreamFinalFunction<rT> {
|
|
|
13
12
|
(toolsArg: IStreamTools): Promise<rT>;
|
|
14
13
|
}
|
|
15
14
|
export interface ISmartDuplexOptions<TInput, TOutput> extends DuplexOptions {
|
|
15
|
+
debug?: boolean;
|
|
16
|
+
name?: string;
|
|
16
17
|
handleBackpressure?: boolean;
|
|
17
18
|
readFunction?: () => Promise<void>;
|
|
18
19
|
writeFunction?: IStreamWriteFunction<TInput, TOutput>;
|
|
@@ -20,18 +21,13 @@ export interface ISmartDuplexOptions<TInput, TOutput> extends DuplexOptions {
|
|
|
20
21
|
}
|
|
21
22
|
export declare class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
|
|
22
23
|
static fromBuffer(buffer: Buffer, options?: ISmartDuplexOptions<any, any>): SmartDuplex;
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
private readFunction?;
|
|
26
|
-
private handleBackpressure;
|
|
27
|
-
private writeFunction?;
|
|
28
|
-
private finalFunction?;
|
|
24
|
+
private backpressuredArray;
|
|
25
|
+
options: ISmartDuplexOptions<TInput, TOutput>;
|
|
29
26
|
private observableSubscription?;
|
|
27
|
+
private debugLog;
|
|
30
28
|
constructor(optionsArg?: ISmartDuplexOptions<TInput, TOutput>);
|
|
31
29
|
_read(size: number): Promise<void>;
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
push(chunkArg?: TOutput | null): boolean;
|
|
35
|
-
_write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void): Promise<boolean | void>;
|
|
30
|
+
private asyncWritePromiseObjectmap;
|
|
31
|
+
_write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void): Promise<void>;
|
|
36
32
|
_final(callback: (error?: Error | null) => void): Promise<void>;
|
|
37
33
|
}
|
|
@@ -10,153 +10,94 @@ export class SmartDuplex extends Duplex {
|
|
|
10
10
|
});
|
|
11
11
|
return smartDuplex;
|
|
12
12
|
}
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
if (!smartStream.push(data) && smartStream.handleBackpressure) {
|
|
18
|
-
// Pause the observable if the stream buffer is full
|
|
19
|
-
smartStream.observableSubscription?.unsubscribe();
|
|
20
|
-
smartStream.once('drain', () => {
|
|
21
|
-
// Resume the observable when the stream buffer is drained
|
|
22
|
-
smartStream.observableSubscription?.unsubscribe();
|
|
23
|
-
smartStream.observableSubscription = observable.subscribe((data) => {
|
|
24
|
-
smartStream.push(data);
|
|
25
|
-
});
|
|
26
|
-
});
|
|
27
|
-
}
|
|
28
|
-
},
|
|
29
|
-
error: (err) => {
|
|
30
|
-
smartStream.emit('error', err);
|
|
31
|
-
},
|
|
32
|
-
complete: () => {
|
|
33
|
-
smartStream.push(null); // Signal the end of the data
|
|
34
|
-
},
|
|
35
|
-
});
|
|
36
|
-
return smartStream;
|
|
37
|
-
}
|
|
38
|
-
static fromReplaySubject(replaySubject, options) {
|
|
39
|
-
const smartStream = new SmartDuplex(options);
|
|
40
|
-
let isBackpressured = false;
|
|
41
|
-
// Subscribe to the ReplaySubject
|
|
42
|
-
const subscription = replaySubject.subscribe({
|
|
43
|
-
next: (data) => {
|
|
44
|
-
const canPush = smartStream.push(data);
|
|
45
|
-
if (!canPush) {
|
|
46
|
-
// If push returns false, pause the subscription because of backpressure
|
|
47
|
-
isBackpressured = true;
|
|
48
|
-
subscription.unsubscribe();
|
|
49
|
-
}
|
|
50
|
-
},
|
|
51
|
-
error: (err) => {
|
|
52
|
-
smartStream.emit('error', err);
|
|
53
|
-
},
|
|
54
|
-
complete: () => {
|
|
55
|
-
smartStream.push(null); // End the stream when the ReplaySubject completes
|
|
56
|
-
},
|
|
57
|
-
});
|
|
58
|
-
// Listen for 'drain' event to resume the subscription if it was paused
|
|
59
|
-
smartStream.on('drain', () => {
|
|
60
|
-
if (isBackpressured) {
|
|
61
|
-
isBackpressured = false;
|
|
62
|
-
// Resubscribe to the ReplaySubject since we previously paused
|
|
63
|
-
smartStream.observableSubscription = replaySubject.subscribe({
|
|
64
|
-
next: (data) => {
|
|
65
|
-
if (!smartStream.push(data)) {
|
|
66
|
-
smartStream.observableSubscription?.unsubscribe();
|
|
67
|
-
isBackpressured = true;
|
|
68
|
-
}
|
|
69
|
-
},
|
|
70
|
-
// No need to repeat error and complete handling here because it's already set up above
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
});
|
|
74
|
-
return smartStream;
|
|
13
|
+
debugLog(messageArg) {
|
|
14
|
+
if (this.options.debug) {
|
|
15
|
+
console.log(messageArg);
|
|
16
|
+
}
|
|
75
17
|
}
|
|
76
18
|
constructor(optionsArg) {
|
|
77
19
|
super(optionsArg);
|
|
78
|
-
|
|
79
|
-
this.
|
|
80
|
-
this.
|
|
81
|
-
this.
|
|
82
|
-
this.handleBackpressure = optionsArg?.handleBackpressure ?? true;
|
|
20
|
+
// INSTANCE
|
|
21
|
+
this.backpressuredArray = new plugins.lik.BackpressuredArray();
|
|
22
|
+
this.asyncWritePromiseObjectmap = new plugins.lik.ObjectMap();
|
|
23
|
+
this.options = optionsArg;
|
|
83
24
|
}
|
|
84
25
|
async _read(size) {
|
|
85
|
-
|
|
86
|
-
|
|
26
|
+
await this.backpressuredArray.waitForItems();
|
|
27
|
+
this.debugLog(`${this.options.name}: read was called`);
|
|
28
|
+
if (this.options.readFunction) {
|
|
29
|
+
await this.options.readFunction();
|
|
87
30
|
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
if (!result && this.handleBackpressure) {
|
|
95
|
-
this.notBackpressured = false;
|
|
96
|
-
this.pause();
|
|
97
|
-
// Listen for 'drain' event to resume
|
|
98
|
-
this.once('drain', () => {
|
|
99
|
-
this.notBackpressured = true;
|
|
100
|
-
this.resume(); // Resume the source of data
|
|
101
|
-
});
|
|
31
|
+
let canPushMore = true;
|
|
32
|
+
while (this.backpressuredArray.data.length > 0 && canPushMore) {
|
|
33
|
+
const nextChunk = this.backpressuredArray.shift();
|
|
34
|
+
if (nextChunk) {
|
|
35
|
+
canPushMore = this.push(nextChunk);
|
|
36
|
+
}
|
|
102
37
|
}
|
|
103
|
-
return result;
|
|
104
38
|
}
|
|
105
39
|
// Ensure the _write method types the chunk as TInput and encodes TOutput
|
|
106
40
|
async _write(chunk, encoding, callback) {
|
|
107
|
-
if (!this.writeFunction) {
|
|
41
|
+
if (!this.options.writeFunction) {
|
|
108
42
|
return callback(new Error('No stream function provided'));
|
|
109
43
|
}
|
|
44
|
+
let isTruncated = false;
|
|
110
45
|
const tools = {
|
|
111
46
|
truncate: () => {
|
|
112
47
|
this.push(null);
|
|
48
|
+
isTruncated = true;
|
|
113
49
|
callback();
|
|
114
50
|
},
|
|
115
|
-
push: (pushArg) =>
|
|
51
|
+
push: (pushArg) => {
|
|
52
|
+
this.backpressuredArray.push(pushArg);
|
|
53
|
+
},
|
|
116
54
|
};
|
|
117
55
|
try {
|
|
118
|
-
const
|
|
56
|
+
const writeDeferred = plugins.smartpromise.defer();
|
|
57
|
+
this.asyncWritePromiseObjectmap.add(writeDeferred.promise);
|
|
58
|
+
const modifiedChunk = await this.options.writeFunction(chunk, tools);
|
|
59
|
+
if (isTruncated) {
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
119
62
|
if (modifiedChunk) {
|
|
120
|
-
this.push(modifiedChunk);
|
|
121
|
-
if (
|
|
122
|
-
this.
|
|
123
|
-
|
|
124
|
-
});
|
|
63
|
+
const canPushMore = this.backpressuredArray.push(modifiedChunk);
|
|
64
|
+
if (!canPushMore) {
|
|
65
|
+
this.debugLog(`${this.options.name}: cannot push more`);
|
|
66
|
+
await this.backpressuredArray.waitForSpace();
|
|
67
|
+
this.debugLog(`${this.options.name}: can push more again`);
|
|
125
68
|
}
|
|
126
|
-
else {
|
|
127
|
-
callback();
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
else {
|
|
131
|
-
callback();
|
|
132
69
|
}
|
|
70
|
+
callback();
|
|
71
|
+
writeDeferred.resolve();
|
|
72
|
+
writeDeferred.promise.then(() => {
|
|
73
|
+
this.asyncWritePromiseObjectmap.remove(writeDeferred.promise);
|
|
74
|
+
});
|
|
133
75
|
}
|
|
134
76
|
catch (err) {
|
|
135
77
|
callback(err);
|
|
136
78
|
}
|
|
137
|
-
return this.notBackpressured;
|
|
138
79
|
}
|
|
139
80
|
async _final(callback) {
|
|
140
|
-
|
|
81
|
+
await Promise.all(this.asyncWritePromiseObjectmap.getArray());
|
|
82
|
+
if (this.options.finalFunction) {
|
|
141
83
|
const tools = {
|
|
142
84
|
truncate: () => callback(),
|
|
143
85
|
push: (pipeObject) => this.push(pipeObject),
|
|
144
86
|
};
|
|
145
87
|
try {
|
|
146
|
-
const finalChunk = await this.finalFunction(tools);
|
|
88
|
+
const finalChunk = await this.options.finalFunction(tools);
|
|
147
89
|
if (finalChunk) {
|
|
148
90
|
this.push(finalChunk);
|
|
149
91
|
}
|
|
150
92
|
}
|
|
151
93
|
catch (err) {
|
|
94
|
+
this.push(null);
|
|
152
95
|
callback(err);
|
|
96
|
+
return;
|
|
153
97
|
}
|
|
154
98
|
}
|
|
155
|
-
else {
|
|
156
|
-
// nothing here
|
|
157
|
-
}
|
|
158
99
|
this.push(null);
|
|
159
100
|
callback();
|
|
160
101
|
}
|
|
161
102
|
}
|
|
162
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
103
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic21hcnRzdHJlYW0uY2xhc3Nlcy5zbWFydGR1cGxleC5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3RzL3NtYXJ0c3RyZWFtLmNsYXNzZXMuc21hcnRkdXBsZXgudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLE9BQU8sTUFBTSwwQkFBMEIsQ0FBQztBQUNwRCxPQUFPLEVBQUUsTUFBTSxFQUFzQixNQUFNLFFBQVEsQ0FBQztBQXlCcEQsTUFBTSxPQUFPLFdBQXlDLFNBQVEsTUFBTTtJQUNsRSxTQUFTO0lBQ1QsTUFBTSxDQUFDLFVBQVUsQ0FBQyxNQUFjLEVBQUUsT0FBdUM7UUFDdkUsTUFBTSxXQUFXLEdBQUcsSUFBSSxXQUFXLENBQUMsT0FBTyxDQUFDLENBQUM7UUFDN0MsT0FBTyxDQUFDLFFBQVEsQ0FBQyxHQUFHLEVBQUU7WUFDcEIsV0FBVyxDQUFDLElBQUksQ0FBQyxNQUFNLENBQUMsQ0FBQztZQUN6QixXQUFXLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDLENBQUMsNkJBQTZCO1FBQ3ZELENBQUMsQ0FBQyxDQUFDO1FBQ0gsT0FBTyxXQUFXLENBQUM7SUFDckIsQ0FBQztJQU1PLFFBQVEsQ0FBQyxVQUFrQjtRQUNqQyxJQUFJLElBQUksQ0FBQyxPQUFPLENBQUMsS0FBSyxFQUFFO1lBQ3RCLE9BQU8sQ0FBQyxHQUFHLENBQUMsVUFBVSxDQUFDLENBQUM7U0FDekI7SUFDSCxDQUFDO0lBRUQsWUFBWSxVQUFpRDtRQUMzRCxLQUFLLENBQUMsVUFBVSxDQUFDLENBQUM7UUFYcEIsV0FBVztRQUNILHVCQUFrQixHQUFHLElBQUksT0FBTyxDQUFDLEdBQUcsQ0FBQyxrQkFBa0IsRUFBVyxDQUFDO1FBNkJuRSwrQkFBMEIsR0FBRyxJQUFJLE9BQU8sQ0FBQyxHQUFHLENBQUMsU0FBUyxFQUFnQixDQUFDO1FBbEI3RSxJQUFJLENBQUMsT0FBTyxHQUFHLFVBQVUsQ0FBQztJQUM1QixDQUFDO0lBRU0sS0FBSyxDQUFDLEtBQUssQ0FBQyxJQUFZO1FBQzdCLE1BQU0sSUFBSSxDQUFDLGtCQUFrQixDQUFDLFlBQVksRUFBRSxDQUFDO1FBQzdDLElBQUksQ0FBQyxRQUFRLENBQUMsR0FBRyxJQUFJLENBQUMsT0FBTyxDQUFDLElBQUksbUJBQW1CLENBQUMsQ0FBQztRQUN2RCxJQUFJLElBQUksQ0FBQyxPQUFPLENBQUMsWUFBWSxFQUFFO1lBQzdCLE1BQU0sSUFBSSxDQUFDLE9BQU8sQ0FBQyxZQUFZLEVBQUUsQ0FBQztTQUNuQztRQUNELElBQUksV0FBVyxHQUFHLElBQUksQ0FBQztRQUN2QixPQUFNLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxJQUFJLENBQUMsTUFBTSxHQUFHLENBQUMsSUFBSSxXQUFXLEVBQUU7WUFDNUQsTUFBTSxTQUFTLEdBQUcsSUFBSSxDQUFDLGtCQUFrQixDQUFDLEtBQUssRUFBRSxDQUFDO1lBQ2xELElBQUksU0FBUyxFQUFFO2dCQUNiLFdBQVcsR0FBRyxJQUFJLENBQUMsSUFBSSxDQUFDLFNBQVMsQ0FBQyxDQUFDO2FBQ3BDO1NBQ0Y7SUFDSCxDQUFDO0lBR0QseUVBQXlFO0lBQ2xFLEtBQUssQ0FBQyxNQUFNLENBQUMsS0FBYSxFQUFFLFFBQWdCLEVBQUUsUUFBd0M7UUFDM0YsSUFBSSxDQUFDLElBQUksQ0FBQyxPQUFPLENBQUMsYUFBYSxFQUFFO1lBQy9CLE9BQU8sUUFBUSxDQUFDLElBQUksS0FBSyxDQUFDLDZCQUE2QixDQUFDLENBQUMsQ0FBQztTQUMzRDtRQUVELElBQUksV0FBVyxHQUFHLEtBQUssQ0FBQztRQUN4QixNQUFNLEtBQUssR0FBaUI7WUFDMUIsUUFBUSxFQUFFLEdBQUcsRUFBRTtnQkFDYixJQUFJLENBQUMsSUFBSSxDQUFDLElBQUksQ0FBQyxDQUFDO2dCQUNoQixXQUFXLEdBQUcsSUFBSSxDQUFDO2dCQUNuQixRQUFRLEVBQUUsQ0FBQztZQUNiLENBQUM7WUFDRCxJQUFJLEVBQUUsQ0FBQyxPQUFnQixFQUFFLEVBQUU7Z0JBQ3pCLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxJQUFJLENBQUMsT0FBTyxDQUFDLENBQUM7WUFDeEMsQ0FBQztTQUNGLENBQUM7UUFFRixJQUFJO1lBQ0YsTUFBTSxhQUFhLEdBQUcsT0FBTyxDQUFDLFlBQVksQ0FBQyxLQUFLLEVBQUUsQ0FBQztZQUNuRCxJQUFJLENBQUMsMEJBQTBCLENBQUMsR0FBRyxDQUFDLGFBQWEsQ0FBQyxPQUFPLENBQUMsQ0FBQztZQUMzRCxNQUFNLGFBQWEsR0FBRyxNQUFNLElBQUksQ0FBQyxPQUFPLENBQUMsYUFBYSxDQUFDLEtBQUssRUFBRSxLQUFLLENBQUMsQ0FBQztZQUNyRSxJQUFJLFdBQVcsRUFBRTtnQkFDZixPQUFPO2FBQ1I7WUFDRCxJQUFJLGFBQWEsRUFBRTtnQkFDakIsTUFBTSxXQUFXLEdBQUcsSUFBSSxDQUFDLGtCQUFrQixDQUFDLElBQUksQ0FBQyxhQUFhLENBQUMsQ0FBQztnQkFDaEUsSUFBSSxDQUFDLFdBQVcsRUFBRTtvQkFDaEIsSUFBSSxDQUFDLFFBQVEsQ0FBQyxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsSUFBSSxvQkFBb0IsQ0FBQyxDQUFDO29CQUN4RCxNQUFNLElBQUksQ0FBQyxrQkFBa0IsQ0FBQyxZQUFZLEVBQUUsQ0FBQztvQkFDN0MsSUFBSSxDQUFDLFFBQVEsQ0FBQyxHQUFHLElBQUksQ0FBQyxPQUFPLENBQUMsSUFBSSx1QkFBdUIsQ0FBQyxDQUFDO2lCQUM1RDthQUNGO1lBQ0QsUUFBUSxFQUFFLENBQUM7WUFDWCxhQUFhLENBQUMsT0FBTyxFQUFFLENBQUM7WUFDeEIsYUFBYSxDQUFDLE9BQU8sQ0FBQyxJQUFJLENBQUMsR0FBRyxFQUFFO2dCQUM5QixJQUFJLENBQUMsMEJBQTBCLENBQUMsTUFBTSxDQUFDLGFBQWEsQ0FBQyxPQUFPLENBQUMsQ0FBQztZQUNoRSxDQUFDLENBQUMsQ0FBQztTQUNKO1FBQUMsT0FBTyxHQUFHLEVBQUU7WUFDWixRQUFRLENBQUMsR0FBRyxDQUFDLENBQUM7U0FDZjtJQUNILENBQUM7SUFFTSxLQUFLLENBQUMsTUFBTSxDQUFDLFFBQXdDO1FBQzFELE1BQU0sT0FBTyxDQUFDLEdBQUcsQ0FBQyxJQUFJLENBQUMsMEJBQTBCLENBQUMsUUFBUSxFQUFFLENBQUMsQ0FBQztRQUM5RCxJQUFJLElBQUksQ0FBQyxPQUFPLENBQUMsYUFBYSxFQUFFO1lBQzlCLE1BQU0sS0FBSyxHQUFpQjtnQkFDMUIsUUFBUSxFQUFFLEdBQUcsRUFBRSxDQUFDLFFBQVEsRUFBRTtnQkFDMUIsSUFBSSxFQUFFLENBQUMsVUFBVSxFQUFFLEVBQUUsQ0FBQyxJQUFJLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQzthQUM1QyxDQUFDO1lBRUYsSUFBSTtnQkFDRixNQUFNLFVBQVUsR0FBRyxNQUFNLElBQUksQ0FBQyxPQUFPLENBQUMsYUFBYSxDQUFDLEtBQUssQ0FBQyxDQUFDO2dCQUMzRCxJQUFJLFVBQVUsRUFBRTtvQkFDZCxJQUFJLENBQUMsSUFBSSxDQUFDLFVBQVUsQ0FBQyxDQUFDO2lCQUN2QjthQUNGO1lBQUMsT0FBTyxHQUFHLEVBQUU7Z0JBQ1osSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztnQkFDaEIsUUFBUSxDQUFDLEdBQUcsQ0FBQyxDQUFDO2dCQUNkLE9BQU87YUFDUjtTQUNGO1FBQ0QsSUFBSSxDQUFDLElBQUksQ0FBQyxJQUFJLENBQUMsQ0FBQztRQUNoQixRQUFRLEVBQUUsQ0FBQztJQUNiLENBQUM7Q0FDRiJ9
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import * as stream from 'stream';
|
|
2
2
|
export { stream };
|
|
3
|
+
import * as lik from '@push.rocks/lik';
|
|
3
4
|
import * as smartpromise from '@push.rocks/smartpromise';
|
|
4
5
|
import * as smartrx from '@push.rocks/smartrx';
|
|
5
|
-
export { smartpromise, smartrx };
|
|
6
|
+
export { lik, smartpromise, smartrx };
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
import * as stream from 'stream';
|
|
3
3
|
export { stream };
|
|
4
4
|
// pushrocks scope
|
|
5
|
+
import * as lik from '@push.rocks/lik';
|
|
5
6
|
import * as smartpromise from '@push.rocks/smartpromise';
|
|
6
7
|
import * as smartrx from '@push.rocks/smartrx';
|
|
7
|
-
export { smartpromise, smartrx };
|
|
8
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
8
|
+
export { lik, smartpromise, smartrx };
|
|
9
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic21hcnRzdHJlYW0ucGx1Z2lucy5qcyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3RzL3NtYXJ0c3RyZWFtLnBsdWdpbnMudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsY0FBYztBQUNkLE9BQU8sS0FBSyxNQUFNLE1BQU0sUUFBUSxDQUFDO0FBRWpDLE9BQU8sRUFBRSxNQUFNLEVBQUUsQ0FBQztBQUVsQixrQkFBa0I7QUFDbEIsT0FBTyxLQUFLLEdBQUcsTUFBTSxpQkFBaUIsQ0FBQztBQUN2QyxPQUFPLEtBQUssWUFBWSxNQUFNLDBCQUEwQixDQUFDO0FBQ3pELE9BQU8sS0FBSyxPQUFPLE1BQU0scUJBQXFCLENBQUM7QUFFL0MsT0FBTyxFQUFFLEdBQUcsRUFBRSxZQUFZLEVBQUUsT0FBTyxFQUFFLENBQUMifQ==
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@push.rocks/smartstream",
|
|
3
|
-
"version": "3.0.
|
|
3
|
+
"version": "3.0.20",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "simplifies access to node streams",
|
|
6
6
|
"main": "dist_ts/index.js",
|
|
@@ -25,11 +25,13 @@
|
|
|
25
25
|
"@git.zone/tsbuild": "^2.1.66",
|
|
26
26
|
"@git.zone/tsrun": "^1.2.44",
|
|
27
27
|
"@git.zone/tstest": "^1.0.84",
|
|
28
|
+
"@push.rocks/smartdelay": "^3.0.5",
|
|
28
29
|
"@push.rocks/smartfile": "^11.0.0",
|
|
29
30
|
"@push.rocks/tapbundle": "^5.0.15",
|
|
30
31
|
"@types/node": "^20.9.0"
|
|
31
32
|
},
|
|
32
33
|
"dependencies": {
|
|
34
|
+
"@push.rocks/lik": "^6.0.12",
|
|
33
35
|
"@push.rocks/smartpromise": "^4.0.3",
|
|
34
36
|
"@push.rocks/smartrx": "^3.0.7"
|
|
35
37
|
},
|
package/ts/00_commitinfo_data.ts
CHANGED
|
@@ -15,6 +15,8 @@ export interface IStreamFinalFunction<rT> {
|
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
export interface ISmartDuplexOptions<TInput, TOutput> extends DuplexOptions {
|
|
18
|
+
debug?: boolean;
|
|
19
|
+
name?: string;
|
|
18
20
|
handleBackpressure?: boolean;
|
|
19
21
|
readFunction?: () => Promise<void>;
|
|
20
22
|
writeFunction?: IStreamWriteFunction<TInput, TOutput>;
|
|
@@ -33,171 +35,98 @@ export class SmartDuplex<TInput = any, TOutput = any> extends Duplex {
|
|
|
33
35
|
return smartDuplex;
|
|
34
36
|
}
|
|
35
37
|
|
|
36
|
-
static fromObservable(
|
|
37
|
-
observable: plugins.smartrx.rxjs.Observable<any>,
|
|
38
|
-
options?: ISmartDuplexOptions<any, any>
|
|
39
|
-
): SmartDuplex {
|
|
40
|
-
const smartStream = new SmartDuplex(options);
|
|
41
|
-
smartStream.observableSubscription = observable.subscribe({
|
|
42
|
-
next: (data) => {
|
|
43
|
-
if (!smartStream.push(data) && smartStream.handleBackpressure) {
|
|
44
|
-
// Pause the observable if the stream buffer is full
|
|
45
|
-
smartStream.observableSubscription?.unsubscribe();
|
|
46
|
-
smartStream.once('drain', () => {
|
|
47
|
-
// Resume the observable when the stream buffer is drained
|
|
48
|
-
smartStream.observableSubscription?.unsubscribe();
|
|
49
|
-
smartStream.observableSubscription = observable.subscribe((data) => {
|
|
50
|
-
smartStream.push(data);
|
|
51
|
-
});
|
|
52
|
-
});
|
|
53
|
-
}
|
|
54
|
-
},
|
|
55
|
-
error: (err) => {
|
|
56
|
-
smartStream.emit('error', err);
|
|
57
|
-
},
|
|
58
|
-
complete: () => {
|
|
59
|
-
smartStream.push(null); // Signal the end of the data
|
|
60
|
-
},
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
return smartStream;
|
|
64
|
-
}
|
|
65
|
-
|
|
66
|
-
static fromReplaySubject(
|
|
67
|
-
replaySubject: plugins.smartrx.rxjs.ReplaySubject<any>,
|
|
68
|
-
options?: DuplexOptions
|
|
69
|
-
): SmartDuplex {
|
|
70
|
-
const smartStream = new SmartDuplex(options);
|
|
71
|
-
let isBackpressured = false;
|
|
72
|
-
|
|
73
|
-
// Subscribe to the ReplaySubject
|
|
74
|
-
const subscription = replaySubject.subscribe({
|
|
75
|
-
next: (data) => {
|
|
76
|
-
const canPush = smartStream.push(data);
|
|
77
|
-
if (!canPush) {
|
|
78
|
-
// If push returns false, pause the subscription because of backpressure
|
|
79
|
-
isBackpressured = true;
|
|
80
|
-
subscription.unsubscribe();
|
|
81
|
-
}
|
|
82
|
-
},
|
|
83
|
-
error: (err) => {
|
|
84
|
-
smartStream.emit('error', err);
|
|
85
|
-
},
|
|
86
|
-
complete: () => {
|
|
87
|
-
smartStream.push(null); // End the stream when the ReplaySubject completes
|
|
88
|
-
},
|
|
89
|
-
});
|
|
90
|
-
|
|
91
|
-
// Listen for 'drain' event to resume the subscription if it was paused
|
|
92
|
-
smartStream.on('drain', () => {
|
|
93
|
-
if (isBackpressured) {
|
|
94
|
-
isBackpressured = false;
|
|
95
|
-
// Resubscribe to the ReplaySubject since we previously paused
|
|
96
|
-
smartStream.observableSubscription = replaySubject.subscribe({
|
|
97
|
-
next: (data) => {
|
|
98
|
-
if (!smartStream.push(data)) {
|
|
99
|
-
smartStream.observableSubscription?.unsubscribe();
|
|
100
|
-
isBackpressured = true;
|
|
101
|
-
}
|
|
102
|
-
},
|
|
103
|
-
// No need to repeat error and complete handling here because it's already set up above
|
|
104
|
-
});
|
|
105
|
-
}
|
|
106
|
-
});
|
|
107
|
-
|
|
108
|
-
return smartStream;
|
|
109
|
-
}
|
|
110
|
-
|
|
111
38
|
// INSTANCE
|
|
112
|
-
private
|
|
113
|
-
|
|
114
|
-
private writeFunction?: IStreamWriteFunction<TInput, TOutput>;
|
|
115
|
-
private finalFunction?: IStreamFinalFunction<TOutput>;
|
|
39
|
+
private backpressuredArray = new plugins.lik.BackpressuredArray<TOutput>();
|
|
40
|
+
public options: ISmartDuplexOptions<TInput, TOutput>;
|
|
116
41
|
private observableSubscription?: plugins.smartrx.rxjs.Subscription;
|
|
42
|
+
private debugLog(messageArg: string) {
|
|
43
|
+
if (this.options.debug) {
|
|
44
|
+
console.log(messageArg);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
117
47
|
|
|
118
48
|
constructor(optionsArg?: ISmartDuplexOptions<TInput, TOutput>) {
|
|
119
49
|
super(optionsArg);
|
|
120
|
-
this.
|
|
121
|
-
this.writeFunction = optionsArg?.writeFunction;
|
|
122
|
-
this.finalFunction = optionsArg?.finalFunction;
|
|
123
|
-
this.handleBackpressure = optionsArg?.handleBackpressure ?? true;
|
|
50
|
+
this.options = optionsArg;
|
|
124
51
|
}
|
|
125
52
|
|
|
126
53
|
public async _read(size: number): Promise<void> {
|
|
127
|
-
|
|
128
|
-
|
|
54
|
+
await this.backpressuredArray.waitForItems();
|
|
55
|
+
this.debugLog(`${this.options.name}: read was called`);
|
|
56
|
+
if (this.options.readFunction) {
|
|
57
|
+
await this.options.readFunction();
|
|
129
58
|
}
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
public push(chunkArg?: TOutput | null): boolean {
|
|
137
|
-
const result = super.push(chunkArg);
|
|
138
|
-
if (!result && this.handleBackpressure) {
|
|
139
|
-
this.notBackpressured = false;
|
|
140
|
-
this.pause();
|
|
141
|
-
// Listen for 'drain' event to resume
|
|
142
|
-
this.once('drain', () => {
|
|
143
|
-
this.notBackpressured = true;
|
|
144
|
-
this.resume(); // Resume the source of data
|
|
145
|
-
});
|
|
59
|
+
let canPushMore = true;
|
|
60
|
+
while(this.backpressuredArray.data.length > 0 && canPushMore) {
|
|
61
|
+
const nextChunk = this.backpressuredArray.shift();
|
|
62
|
+
if (nextChunk) {
|
|
63
|
+
canPushMore = this.push(nextChunk);
|
|
64
|
+
}
|
|
146
65
|
}
|
|
147
|
-
return result;
|
|
148
66
|
}
|
|
149
67
|
|
|
68
|
+
private asyncWritePromiseObjectmap = new plugins.lik.ObjectMap<Promise<any>>();
|
|
150
69
|
// Ensure the _write method types the chunk as TInput and encodes TOutput
|
|
151
70
|
public async _write(chunk: TInput, encoding: string, callback: (error?: Error | null) => void) {
|
|
152
|
-
if (!this.writeFunction) {
|
|
71
|
+
if (!this.options.writeFunction) {
|
|
153
72
|
return callback(new Error('No stream function provided'));
|
|
154
73
|
}
|
|
155
74
|
|
|
75
|
+
let isTruncated = false;
|
|
156
76
|
const tools: IStreamTools = {
|
|
157
77
|
truncate: () => {
|
|
158
78
|
this.push(null);
|
|
79
|
+
isTruncated = true;
|
|
159
80
|
callback();
|
|
160
81
|
},
|
|
161
|
-
push: (pushArg: TOutput) =>
|
|
82
|
+
push: (pushArg: TOutput) => {
|
|
83
|
+
this.backpressuredArray.push(pushArg);
|
|
84
|
+
},
|
|
162
85
|
};
|
|
163
86
|
|
|
164
87
|
try {
|
|
165
|
-
const
|
|
88
|
+
const writeDeferred = plugins.smartpromise.defer();
|
|
89
|
+
this.asyncWritePromiseObjectmap.add(writeDeferred.promise);
|
|
90
|
+
const modifiedChunk = await this.options.writeFunction(chunk, tools);
|
|
91
|
+
if (isTruncated) {
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
166
94
|
if (modifiedChunk) {
|
|
167
|
-
this.push(modifiedChunk)
|
|
168
|
-
if (
|
|
169
|
-
this.
|
|
170
|
-
|
|
171
|
-
});
|
|
172
|
-
}
|
|
173
|
-
callback();
|
|
174
|
-
}
|
|
175
|
-
} else {
|
|
176
|
-
callback();
|
|
95
|
+
const canPushMore = this.backpressuredArray.push(modifiedChunk);
|
|
96
|
+
if (!canPushMore) {
|
|
97
|
+
this.debugLog(`${this.options.name}: cannot push more`);
|
|
98
|
+
await this.backpressuredArray.waitForSpace();
|
|
99
|
+
this.debugLog(`${this.options.name}: can push more again`);
|
|
100
|
+
}
|
|
177
101
|
}
|
|
102
|
+
callback();
|
|
103
|
+
writeDeferred.resolve();
|
|
104
|
+
writeDeferred.promise.then(() => {
|
|
105
|
+
this.asyncWritePromiseObjectmap.remove(writeDeferred.promise);
|
|
106
|
+
});
|
|
178
107
|
} catch (err) {
|
|
179
108
|
callback(err);
|
|
180
109
|
}
|
|
181
|
-
return this.notBackpressured;
|
|
182
110
|
}
|
|
183
111
|
|
|
184
112
|
public async _final(callback: (error?: Error | null) => void) {
|
|
185
|
-
|
|
113
|
+
await Promise.all(this.asyncWritePromiseObjectmap.getArray());
|
|
114
|
+
if (this.options.finalFunction) {
|
|
186
115
|
const tools: IStreamTools = {
|
|
187
116
|
truncate: () => callback(),
|
|
188
117
|
push: (pipeObject) => this.push(pipeObject),
|
|
189
118
|
};
|
|
190
119
|
|
|
191
120
|
try {
|
|
192
|
-
const finalChunk = await this.finalFunction(tools);
|
|
121
|
+
const finalChunk = await this.options.finalFunction(tools);
|
|
193
122
|
if (finalChunk) {
|
|
194
123
|
this.push(finalChunk);
|
|
195
124
|
}
|
|
196
125
|
} catch (err) {
|
|
126
|
+
this.push(null);
|
|
197
127
|
callback(err);
|
|
128
|
+
return;
|
|
198
129
|
}
|
|
199
|
-
} else {
|
|
200
|
-
// nothing here
|
|
201
130
|
}
|
|
202
131
|
this.push(null);
|
|
203
132
|
callback();
|
|
@@ -4,8 +4,9 @@ import * as stream from 'stream';
|
|
|
4
4
|
export { stream };
|
|
5
5
|
|
|
6
6
|
// pushrocks scope
|
|
7
|
+
import * as lik from '@push.rocks/lik';
|
|
7
8
|
import * as smartpromise from '@push.rocks/smartpromise';
|
|
8
9
|
import * as smartrx from '@push.rocks/smartrx';
|
|
9
10
|
|
|
10
|
-
export { smartpromise, smartrx };
|
|
11
|
+
export { lik, smartpromise, smartrx };
|
|
11
12
|
|