@mako10k/shell-server 0.2.2 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +35 -10
- package/dist/backoffice/server.js +9 -9
- package/dist/backoffice/server.js.map +1 -1
- package/dist/core/file-manager.d.ts +3 -3
- package/dist/core/file-manager.js +27 -27
- package/dist/core/file-manager.js.map +1 -1
- package/dist/core/file-storage-subscriber.d.ts +7 -7
- package/dist/core/file-storage-subscriber.js +19 -19
- package/dist/core/file-storage-subscriber.js.map +1 -1
- package/dist/core/monitoring-manager.d.ts +1 -1
- package/dist/core/monitoring-manager.js +31 -31
- package/dist/core/monitoring-manager.js.map +1 -1
- package/dist/core/process-manager.d.ts +4 -4
- package/dist/core/process-manager.js +151 -151
- package/dist/core/process-manager.js.map +1 -1
- package/dist/core/realtime-stream-subscriber.d.ts +18 -18
- package/dist/core/realtime-stream-subscriber.d.ts.map +1 -1
- package/dist/core/realtime-stream-subscriber.js +23 -23
- package/dist/core/realtime-stream-subscriber.js.map +1 -1
- package/dist/core/stream-publisher.d.ts +20 -20
- package/dist/core/stream-publisher.d.ts.map +1 -1
- package/dist/core/stream-publisher.js +15 -15
- package/dist/core/stream-publisher.js.map +1 -1
- package/dist/core/streaming-pipeline-reader.d.ts +12 -16
- package/dist/core/streaming-pipeline-reader.d.ts.map +1 -1
- package/dist/core/streaming-pipeline-reader.js +28 -28
- package/dist/core/streaming-pipeline-reader.js.map +1 -1
- package/dist/core/terminal-manager.d.ts +3 -3
- package/dist/core/terminal-manager.js +61 -61
- package/dist/core/terminal-manager.js.map +1 -1
- package/dist/executor/server.js +3 -3
- package/dist/executor/server.js.map +1 -1
- package/dist/security/chat-completion-adapter.d.ts.map +1 -1
- package/dist/security/chat-completion-adapter.js +23 -4
- package/dist/security/chat-completion-adapter.js.map +1 -1
- package/dist/security/enhanced-evaluator.js +3 -3
- package/dist/security/enhanced-evaluator.js.map +1 -1
- package/dist/security/manager.js +2 -2
- package/dist/security/manager.js.map +1 -1
- package/dist/security/security-llm-prompt-generator.d.ts +2 -2
- package/dist/security/security-llm-prompt-generator.js +4 -4
- package/dist/security/security-llm-prompt-generator.js.map +1 -1
- package/dist/tools/shell-tools.d.ts.map +1 -1
- package/dist/tools/shell-tools.js +28 -28
- package/dist/tools/shell-tools.js.map +1 -1
- package/dist/types/enhanced-security.js +29 -29
- package/dist/types/enhanced-security.js.map +1 -1
- package/dist/types/index.js +21 -21
- package/dist/types/index.js.map +1 -1
- package/dist/types/quick-schemas.js +5 -5
- package/dist/types/quick-schemas.js.map +1 -1
- package/dist/types/response-schemas.js +4 -4
- package/dist/types/response-schemas.js.map +1 -1
- package/dist/types/schemas.js +3 -3
- package/dist/types/schemas.js.map +1 -1
- package/dist/utils/errors.js +1 -1
- package/dist/utils/errors.js.map +1 -1
- package/dist/utils/helpers.js +51 -51
- package/dist/utils/helpers.js.map +1 -1
- package/dist/utils/process-utils.d.ts +6 -6
- package/dist/utils/process-utils.js +29 -29
- package/dist/utils/process-utils.js.map +1 -1
- package/package.json +2 -2
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Issue #13:
|
|
3
|
-
* Phase 1: StreamPublisher + Subscriber
|
|
2
|
+
* Issue #13: Real-time streaming feature
|
|
3
|
+
* Phase 1: StreamPublisher + Subscriber pattern implementation
|
|
4
4
|
*/
|
|
5
5
|
/**
|
|
6
|
-
*
|
|
6
|
+
* Class that manages PUB/SUB for process output
|
|
7
7
|
*/
|
|
8
8
|
export class StreamPublisher {
|
|
9
9
|
options;
|
|
@@ -19,17 +19,17 @@ export class StreamPublisher {
|
|
|
19
19
|
};
|
|
20
20
|
}
|
|
21
21
|
/**
|
|
22
|
-
*
|
|
22
|
+
* Register a subscriber
|
|
23
23
|
*/
|
|
24
24
|
subscribe(subscriber) {
|
|
25
25
|
this.subscribers.set(subscriber.id, subscriber);
|
|
26
26
|
}
|
|
27
27
|
/**
|
|
28
|
-
*
|
|
28
|
+
* Remove a subscriber
|
|
29
29
|
*/
|
|
30
30
|
unsubscribe(subscriberId) {
|
|
31
31
|
this.subscribers.delete(subscriberId);
|
|
32
|
-
//
|
|
32
|
+
// Remove this subscriber from all executions
|
|
33
33
|
for (const [executionId, subscriberIds] of this.executionSubscribers.entries()) {
|
|
34
34
|
subscriberIds.delete(subscriberId);
|
|
35
35
|
if (subscriberIds.size === 0) {
|
|
@@ -38,7 +38,7 @@ export class StreamPublisher {
|
|
|
38
38
|
}
|
|
39
39
|
}
|
|
40
40
|
/**
|
|
41
|
-
|
|
41
|
+
* Add a subscriber to a specific execution
|
|
42
42
|
*/
|
|
43
43
|
subscribeToExecution(executionId, subscriberId) {
|
|
44
44
|
if (!this.subscribers.has(subscriberId)) {
|
|
@@ -53,7 +53,7 @@ export class StreamPublisher {
|
|
|
53
53
|
}
|
|
54
54
|
}
|
|
55
55
|
/**
|
|
56
|
-
|
|
56
|
+
* Common helper for sending notifications to subscribers
|
|
57
57
|
*/
|
|
58
58
|
async notifySubscribers(executionId, callback, ...args) {
|
|
59
59
|
const subscriberIds = this.executionSubscribers.get(executionId);
|
|
@@ -73,7 +73,7 @@ export class StreamPublisher {
|
|
|
73
73
|
await Promise.allSettled(notifications);
|
|
74
74
|
}
|
|
75
75
|
/**
|
|
76
|
-
|
|
76
|
+
* Notify process start
|
|
77
77
|
*/
|
|
78
78
|
async notifyProcessStart(executionId, command) {
|
|
79
79
|
await this.notifySubscribers(executionId, (subscriber, _, executionId, command) => {
|
|
@@ -83,13 +83,13 @@ export class StreamPublisher {
|
|
|
83
83
|
}, executionId, command);
|
|
84
84
|
}
|
|
85
85
|
/**
|
|
86
|
-
|
|
86
|
+
* Notify output data
|
|
87
87
|
*/
|
|
88
88
|
async notifyOutputData(executionId, data, isStderr = false) {
|
|
89
89
|
await this.notifySubscribers(executionId, (subscriber, _, executionId, data, isStderr) => subscriber.onOutputData(executionId, data, isStderr), executionId, data, isStderr);
|
|
90
90
|
}
|
|
91
91
|
/**
|
|
92
|
-
|
|
92
|
+
* Notify process end
|
|
93
93
|
*/
|
|
94
94
|
async notifyProcessEnd(executionId, exitCode) {
|
|
95
95
|
await this.notifySubscribers(executionId, (subscriber, _, executionId, exitCode) => {
|
|
@@ -97,11 +97,11 @@ export class StreamPublisher {
|
|
|
97
97
|
return subscriber.onProcessEnd(executionId, exitCode);
|
|
98
98
|
}
|
|
99
99
|
}, executionId, exitCode);
|
|
100
|
-
//
|
|
100
|
+
// Clean up subscription data after execution completes
|
|
101
101
|
this.executionSubscribers.delete(executionId);
|
|
102
102
|
}
|
|
103
103
|
/**
|
|
104
|
-
*
|
|
104
|
+
* Notify errors
|
|
105
105
|
*/
|
|
106
106
|
async notifyError(executionId, error) {
|
|
107
107
|
await this.notifySubscribers(executionId, (subscriber, _, executionId, error) => {
|
|
@@ -111,13 +111,13 @@ export class StreamPublisher {
|
|
|
111
111
|
}, executionId, error);
|
|
112
112
|
}
|
|
113
113
|
/**
|
|
114
|
-
*
|
|
114
|
+
* Whether real-time streaming is enabled
|
|
115
115
|
*/
|
|
116
116
|
isRealtimeStreamingEnabled() {
|
|
117
117
|
return this.options.enableRealtimeStreaming === true;
|
|
118
118
|
}
|
|
119
119
|
/**
|
|
120
|
-
*
|
|
120
|
+
* Check whether subscribers are registered for a specific execution
|
|
121
121
|
*/
|
|
122
122
|
hasSubscribers(executionId) {
|
|
123
123
|
const subscriberIds = this.executionSubscribers.get(executionId);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stream-publisher.js","sourceRoot":"","sources":["../../src/core/stream-publisher.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA8BH;;GAEG;AACH,MAAM,OAAO,eAAe;IAIN;IAHZ,WAAW,GAAkC,IAAI,GAAG,EAAE,CAAC;IACvD,oBAAoB,GAA6B,IAAI,GAAG,EAAE,CAAC,CAAC,8BAA8B;IAElG,YAAoB,UAAkC,EAAE;QAApC,YAAO,GAAP,OAAO,CAA6B;QACtD,IAAI,CAAC,OAAO,GAAG;YACb,uBAAuB,EAAE,KAAK;YAC9B,UAAU,EAAE,IAAI;YAChB,oBAAoB,EAAE,GAAG;YACzB,GAAG,OAAO;SACX,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,UAA4B;QACpC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,YAAoB;QAC9B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACtC,
|
|
1
|
+
{"version":3,"file":"stream-publisher.js","sourceRoot":"","sources":["../../src/core/stream-publisher.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA8BH;;GAEG;AACH,MAAM,OAAO,eAAe;IAIN;IAHZ,WAAW,GAAkC,IAAI,GAAG,EAAE,CAAC;IACvD,oBAAoB,GAA6B,IAAI,GAAG,EAAE,CAAC,CAAC,8BAA8B;IAElG,YAAoB,UAAkC,EAAE;QAApC,YAAO,GAAP,OAAO,CAA6B;QACtD,IAAI,CAAC,OAAO,GAAG;YACb,uBAAuB,EAAE,KAAK;YAC9B,UAAU,EAAE,IAAI;YAChB,oBAAoB,EAAE,GAAG;YACzB,GAAG,OAAO;SACX,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,UAA4B;QACpC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAED;;OAEG;IACH,WAAW,CAAC,YAAoB;QAC9B,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;QACtC,6CAA6C;QAC7C,KAAK,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC,IAAI,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,EAAE,CAAC;YAC/E,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;YACnC,IAAI,aAAa,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;IACH,CAAC;IAED;;OAEG;IACH,oBAAoB,CAAC,WAAmB,EAAE,YAAoB;QAC5D,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,cAAc,YAAY,YAAY,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YAChD,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,WAAW,EAAE,IAAI,GAAG,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACjE,IAAI,aAAa,EAAE,CAAC;YAClB,aAAa,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB,CAC7B,WAAmB,EACnB,QAIyB,EACzB,GAAG,IAAO;QAEV,MAAM,aAAa,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACjE,IAAI,CAAC,aAAa;YAAE,OAAO;QAE3B,MAAM,aAAa,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE;YACzE,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YACtD,IAAI,UAAU,EAAE,CAAC;gBACf,IAAI,CAAC;oBACH,MAAM,QAAQ,CAAC,UAAU,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC,CAAC;gBACpD,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,OAAO,CAAC,KAAK,CAAC,+CAA+C,YAAY,GAAG,EAAE,KAAK,CAAC,CAAC;gBACvF,CAAC;YACH,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,MAAM,OAAO,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;IAC1C,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,kBAAkB,CAAC,WAAmB,EAAE,OAAe;QAC3D,MAAM,IAAI,CAAC,iBAAiB,CAC1B,WAAW,EACX,CAAC,UAAU,EAAE,CAAC,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE;YACtC,IAAI,UAAU,CAAC,cAAc,EAAE,CAAC;gBAC9B,OAAO,UAAU,CAAC,cAAc,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;YACzD,CAAC;QACH,CAAC,EACD,WAAW,EACX,OAAO,CACR,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CACpB,WAAmB,EACnB,IAAY,EACZ,WAAoB,KAAK;QAEzB,MAAM,IAAI,CAAC,iBAAiB,CAC1B,WAAW,EACX,CAAC,UAAU,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAC7C,UAAU,CAAC,YAAY,CAAC,WAAW,EAAE,IAAI,EAAE,QAAQ,CAAC,EACtD,WAAW,EACX,IAAI,EACJ,QAAQ,CACT,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gBAAgB,CAAC,WAAmB,EAAE,QAAuB;QACjE,MAAM,IAAI,CAAC,iBAAiB,CAC1B,WAAW,EACX,CAAC,UAAU,EAAE,CAAC,EAAE,WAAW,EAAE,QAAQ,EAAE,EAAE;YACvC,IAAI,UAAU,CAAC,YAAY,EAAE,CAAC;gBAC5B,OAAO,UAAU,CAAC,YAAY,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;YACxD,CAAC;QACH,CAAC,EACD,WAAW,EACX,QAAQ,CACT,CAAC;QAEF,uDAAuD;QACvD,IAAI,CAAC,oBAAoB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IAChD,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,WAAmB,EAAE,KAAY;QACjD,MAAM,IAAI,CAAC,iBAAiB,CAC1B,WAAW,EACX,CAAC,UAAU,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,EAAE;YACpC,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;gBACvB,OAAO,UAAU,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;YAChD,CAAC;QACH,CAAC,EACD,WAAW,EACX,KAAK,CACN,CAAC;IACJ,CAAC;IAED;;OAEG;IACH,0BAA0B;QACxB,OAAO,IAAI,CAAC,OAAO,CAAC,uBAAuB,KAAK,IAAI,CAAC;IACvD,CAAC;IAED;;OAEG;IACH,cAAc,CAAC,WAAmB;QAChC,MAAM,aAAa,GAAG,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;QACjE,OAAO,OAAO,CAAC,aAAa,IAAI,aAAa,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IAC1D,CAAC;CACF"}
|
|
@@ -1,22 +1,18 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Issue #13: Streaming Pipeline Reader
|
|
3
|
-
* 実行中プロセスの出力をリアルタイムで読み取り、次のプロセスのSTDINに流すためのStream
|
|
4
|
-
*/
|
|
5
1
|
import { Readable } from 'stream';
|
|
6
2
|
import { FileManager } from './file-manager.js';
|
|
7
3
|
import { RealtimeStreamSubscriber } from './realtime-stream-subscriber.js';
|
|
8
4
|
interface StreamingPipelineOptions {
|
|
9
|
-
/**
|
|
5
|
+
/** Read timeout (ms) */
|
|
10
6
|
readTimeout?: number;
|
|
11
|
-
/**
|
|
7
|
+
/** Buffer size (bytes) */
|
|
12
8
|
bufferSize?: number;
|
|
13
|
-
/**
|
|
9
|
+
/** Polling interval (ms) */
|
|
14
10
|
pollingInterval?: number;
|
|
15
11
|
}
|
|
16
12
|
/**
|
|
17
|
-
* input_output_id
|
|
18
|
-
* 1.
|
|
19
|
-
* 2.
|
|
13
|
+
* Streaming reader used when input_output_id refers to a running process
|
|
14
|
+
* 1. First read existing file contents
|
|
15
|
+
* 2. After reaching EOF, obtain real-time data from the RealtimeStream
|
|
20
16
|
*/
|
|
21
17
|
export declare class StreamingPipelineReader extends Readable {
|
|
22
18
|
private fileManager;
|
|
@@ -35,27 +31,27 @@ export declare class StreamingPipelineReader extends Readable {
|
|
|
35
31
|
*/
|
|
36
32
|
_read(_size: number): void;
|
|
37
33
|
/**
|
|
38
|
-
*
|
|
34
|
+
* Start the read processing
|
|
39
35
|
*/
|
|
40
36
|
private startReading;
|
|
41
37
|
/**
|
|
42
|
-
*
|
|
38
|
+
* Perform the actual read operation
|
|
43
39
|
*/
|
|
44
40
|
private performRead;
|
|
45
41
|
/**
|
|
46
|
-
* Phase 1:
|
|
42
|
+
* Phase 1: Read existing data from file
|
|
47
43
|
*/
|
|
48
44
|
private readFromFile;
|
|
49
45
|
/**
|
|
50
|
-
* Phase 2: RealtimeStream
|
|
46
|
+
* Phase 2: Read data from RealtimeStream
|
|
51
47
|
*/
|
|
52
48
|
private readFromStream;
|
|
53
49
|
/**
|
|
54
|
-
*
|
|
50
|
+
* Estimate the last sequence number that has been saved to the file
|
|
55
51
|
*/
|
|
56
52
|
private estimateLastFileSequence;
|
|
57
53
|
/**
|
|
58
|
-
*
|
|
54
|
+
* Cleanup resources
|
|
59
55
|
*/
|
|
60
56
|
private cleanup;
|
|
61
57
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streaming-pipeline-reader.d.ts","sourceRoot":"","sources":["../../src/core/streaming-pipeline-reader.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"streaming-pipeline-reader.d.ts","sourceRoot":"","sources":["../../src/core/streaming-pipeline-reader.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,wBAAwB,EAAE,MAAM,iCAAiC,CAAC;AAE3E,UAAU,wBAAwB;IAChC,wBAAwB;IACxB,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB,0BAA0B;IAC1B,UAAU,CAAC,EAAE,MAAM,CAAC;IAEpB,4BAA4B;IAC5B,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED;;;;GAIG;AACH,qBAAa,uBAAwB,SAAQ,QAAQ;IACnD,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,kBAAkB,CAA2B;IACrD,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,OAAO,CAAqC;IAGpD,OAAO,CAAC,YAAY,CAAa;IACjC,OAAO,CAAC,cAAc,CAAkB;IACxC,OAAO,CAAC,kBAAkB,CAAc;IACxC,OAAO,CAAC,SAAS,CAA+B;IAChD,OAAO,CAAC,WAAW,CAAkB;gBAGnC,WAAW,EAAE,WAAW,EACxB,kBAAkB,EAAE,wBAAwB,EAC5C,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE,wBAA6B;IAgBxC;;OAEG;IACM,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAcnC;;OAEG;IACH,OAAO,CAAC,YAAY;IAmBpB;;OAEG;YACW,WAAW;IAUzB;;OAEG;YACW,YAAY;IAkD1B;;OAEG;YACW,cAAc;IAkC5B;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAkBhC;;OAEG;IACH,OAAO,CAAC,OAAO;IAQf;;OAEG;IACM,QAAQ,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,EAAE,QAAQ,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,GAAG,IAAI,KAAK,IAAI,GAAG,IAAI;CAOvF"}
|
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
|
|
1
|
+
/*
|
|
2
2
|
* Issue #13: Streaming Pipeline Reader
|
|
3
|
-
*
|
|
3
|
+
* Read output from running processes in real time and stream into the next process's STDIN
|
|
4
4
|
*/
|
|
5
5
|
import { Readable } from 'stream';
|
|
6
6
|
/**
|
|
7
|
-
* input_output_id
|
|
8
|
-
* 1.
|
|
9
|
-
* 2.
|
|
7
|
+
* Streaming reader used when input_output_id refers to a running process
|
|
8
|
+
* 1. First read existing file contents
|
|
9
|
+
* 2. After reaching EOF, obtain real-time data from the RealtimeStream
|
|
10
10
|
*/
|
|
11
11
|
export class StreamingPipelineReader extends Readable {
|
|
12
12
|
fileManager;
|
|
@@ -27,7 +27,7 @@ export class StreamingPipelineReader extends Readable {
|
|
|
27
27
|
this.outputId = outputId;
|
|
28
28
|
this.executionId = executionId;
|
|
29
29
|
this.options = {
|
|
30
|
-
readTimeout: 30000, // 30
|
|
30
|
+
readTimeout: 30000, // 30 seconds
|
|
31
31
|
bufferSize: 8192,
|
|
32
32
|
pollingInterval: 100, // 100ms
|
|
33
33
|
...options,
|
|
@@ -41,14 +41,14 @@ export class StreamingPipelineReader extends Readable {
|
|
|
41
41
|
this.push(null);
|
|
42
42
|
return;
|
|
43
43
|
}
|
|
44
|
-
//
|
|
44
|
+
// Skip if a read timer is already running
|
|
45
45
|
if (this.readTimer) {
|
|
46
46
|
return;
|
|
47
47
|
}
|
|
48
48
|
this.startReading();
|
|
49
49
|
}
|
|
50
50
|
/**
|
|
51
|
-
*
|
|
51
|
+
* Start the read processing
|
|
52
52
|
*/
|
|
53
53
|
startReading() {
|
|
54
54
|
this.readTimer = setInterval(async () => {
|
|
@@ -60,7 +60,7 @@ export class StreamingPipelineReader extends Readable {
|
|
|
60
60
|
this.destroy(error);
|
|
61
61
|
}
|
|
62
62
|
}, this.options.pollingInterval);
|
|
63
|
-
//
|
|
63
|
+
// Set timeout
|
|
64
64
|
setTimeout(() => {
|
|
65
65
|
if (!this.isDestroyed && this.readTimer) {
|
|
66
66
|
console.error(`StreamingPipelineReader: Read timeout for ${this.outputId}`);
|
|
@@ -69,49 +69,49 @@ export class StreamingPipelineReader extends Readable {
|
|
|
69
69
|
}, this.options.readTimeout);
|
|
70
70
|
}
|
|
71
71
|
/**
|
|
72
|
-
*
|
|
72
|
+
* Perform the actual read operation
|
|
73
73
|
*/
|
|
74
74
|
async performRead() {
|
|
75
75
|
if (!this.isFileComplete) {
|
|
76
|
-
// Phase 1:
|
|
76
|
+
// Phase 1: Read existing data from file
|
|
77
77
|
await this.readFromFile();
|
|
78
78
|
}
|
|
79
79
|
else {
|
|
80
|
-
// Phase 2: RealtimeStream
|
|
80
|
+
// Phase 2: Read data from RealtimeStream
|
|
81
81
|
await this.readFromStream();
|
|
82
82
|
}
|
|
83
83
|
}
|
|
84
84
|
/**
|
|
85
|
-
* Phase 1:
|
|
85
|
+
* Phase 1: Read existing data from file
|
|
86
86
|
*/
|
|
87
87
|
async readFromFile() {
|
|
88
88
|
try {
|
|
89
89
|
const result = await this.fileManager.readFile(this.outputId, this.filePosition, this.options.bufferSize, 'utf-8');
|
|
90
90
|
if (result.content && result.content.length > 0) {
|
|
91
|
-
//
|
|
91
|
+
// If data is present, update the read position
|
|
92
92
|
this.filePosition += Buffer.byteLength(result.content, 'utf-8');
|
|
93
93
|
this.push(result.content);
|
|
94
94
|
console.error(`StreamingPipelineReader: Read ${result.content.length} chars from file position ${this.filePosition - result.content.length}`);
|
|
95
95
|
}
|
|
96
96
|
else {
|
|
97
|
-
//
|
|
97
|
+
// Reached EOF - check whether the process has ended
|
|
98
98
|
const streamState = this.realtimeSubscriber.getStreamState(this.executionId);
|
|
99
99
|
if (streamState && !streamState.isActive) {
|
|
100
|
-
//
|
|
100
|
+
// If the process has finished, reading is complete
|
|
101
101
|
console.error(`StreamingPipelineReader: Process ${this.executionId} completed, file reading finished`);
|
|
102
102
|
this.push(null); // EOF
|
|
103
103
|
this.cleanup();
|
|
104
104
|
}
|
|
105
105
|
else {
|
|
106
|
-
//
|
|
106
|
+
// If the process is still running, switch to stream mode
|
|
107
107
|
console.error(`StreamingPipelineReader: File EOF reached, switching to stream mode for ${this.outputId}`);
|
|
108
108
|
this.isFileComplete = true;
|
|
109
|
-
//
|
|
109
|
+
// Identify the last sequence number saved to the file
|
|
110
110
|
if (streamState) {
|
|
111
|
-
//
|
|
111
|
+
// Estimate the last saved sequence number from the latest buffers
|
|
112
112
|
const latestBuffers = this.realtimeSubscriber.getLatestBuffers(this.executionId, 10);
|
|
113
113
|
if (latestBuffers.length > 0) {
|
|
114
|
-
//
|
|
114
|
+
// Estimate how far data has been persisted by comparing against file size
|
|
115
115
|
this.lastSequenceNumber = this.estimateLastFileSequence(latestBuffers);
|
|
116
116
|
}
|
|
117
117
|
}
|
|
@@ -124,7 +124,7 @@ export class StreamingPipelineReader extends Readable {
|
|
|
124
124
|
}
|
|
125
125
|
}
|
|
126
126
|
/**
|
|
127
|
-
* Phase 2: RealtimeStream
|
|
127
|
+
* Phase 2: Read data from RealtimeStream
|
|
128
128
|
*/
|
|
129
129
|
async readFromStream() {
|
|
130
130
|
const streamState = this.realtimeSubscriber.getStreamState(this.executionId);
|
|
@@ -134,8 +134,8 @@ export class StreamingPipelineReader extends Readable {
|
|
|
134
134
|
this.cleanup();
|
|
135
135
|
return;
|
|
136
136
|
}
|
|
137
|
-
//
|
|
138
|
-
const newBuffers = this.realtimeSubscriber.getBuffersFromSequence(this.executionId, this.lastSequenceNumber + 1, 50 //
|
|
137
|
+
// Fetch buffers after the last read sequence number
|
|
138
|
+
const newBuffers = this.realtimeSubscriber.getBuffersFromSequence(this.executionId, this.lastSequenceNumber + 1, 50 // Max 50 buffers at a time
|
|
139
139
|
);
|
|
140
140
|
if (newBuffers.length > 0) {
|
|
141
141
|
for (const buffer of newBuffers) {
|
|
@@ -144,7 +144,7 @@ export class StreamingPipelineReader extends Readable {
|
|
|
144
144
|
console.error(`StreamingPipelineReader: Streamed buffer seq=${buffer.sequenceNumber}, ${buffer.data.length} chars`);
|
|
145
145
|
}
|
|
146
146
|
}
|
|
147
|
-
//
|
|
147
|
+
// If the process has ended and there are no new buffers, finish
|
|
148
148
|
if (!streamState.isActive && newBuffers.length === 0) {
|
|
149
149
|
console.error(`StreamingPipelineReader: Stream completed for ${this.executionId}`);
|
|
150
150
|
this.push(null); // EOF
|
|
@@ -152,11 +152,11 @@ export class StreamingPipelineReader extends Readable {
|
|
|
152
152
|
}
|
|
153
153
|
}
|
|
154
154
|
/**
|
|
155
|
-
*
|
|
155
|
+
* Estimate the last sequence number that has been saved to the file
|
|
156
156
|
*/
|
|
157
157
|
estimateLastFileSequence(latestBuffers) {
|
|
158
|
-
//
|
|
159
|
-
//
|
|
158
|
+
// Simple estimate: infer from file size and buffer sizes
|
|
159
|
+
// A more accurate implementation would integrate with FileStorageSubscriber
|
|
160
160
|
let estimatedBytes = 0;
|
|
161
161
|
for (let i = latestBuffers.length - 1; i >= 0; i--) {
|
|
162
162
|
const buffer = latestBuffers[i];
|
|
@@ -170,7 +170,7 @@ export class StreamingPipelineReader extends Readable {
|
|
|
170
170
|
return -1;
|
|
171
171
|
}
|
|
172
172
|
/**
|
|
173
|
-
*
|
|
173
|
+
* Cleanup resources
|
|
174
174
|
*/
|
|
175
175
|
cleanup() {
|
|
176
176
|
if (this.readTimer) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"streaming-pipeline-reader.js","sourceRoot":"","sources":["../../src/core/streaming-pipeline-reader.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAelC;;;;GAIG;AACH,MAAM,OAAO,uBAAwB,SAAQ,QAAQ;IAC3C,WAAW,CAAc;IACzB,kBAAkB,CAA2B;IAC7C,QAAQ,CAAS;IACjB,WAAW,CAAS;IACpB,OAAO,CAAqC;IAEpD,mBAAmB;IACX,YAAY,GAAW,CAAC,CAAC;IACzB,cAAc,GAAY,KAAK,CAAC;IAChC,kBAAkB,GAAW,CAAC,CAAC,CAAC;IAChC,SAAS,GAA0B,IAAI,CAAC;IACxC,WAAW,GAAY,KAAK,CAAC;IAErC,YACE,WAAwB,EACxB,kBAA4C,EAC5C,QAAgB,EAChB,WAAmB,EACnB,UAAoC,EAAE;QAEtC,KAAK,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;QAE7B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG;YACb,WAAW,EAAE,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"streaming-pipeline-reader.js","sourceRoot":"","sources":["../../src/core/streaming-pipeline-reader.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAC;AAelC;;;;GAIG;AACH,MAAM,OAAO,uBAAwB,SAAQ,QAAQ;IAC3C,WAAW,CAAc;IACzB,kBAAkB,CAA2B;IAC7C,QAAQ,CAAS;IACjB,WAAW,CAAS;IACpB,OAAO,CAAqC;IAEpD,mBAAmB;IACX,YAAY,GAAW,CAAC,CAAC;IACzB,cAAc,GAAY,KAAK,CAAC;IAChC,kBAAkB,GAAW,CAAC,CAAC,CAAC;IAChC,SAAS,GAA0B,IAAI,CAAC;IACxC,WAAW,GAAY,KAAK,CAAC;IAErC,YACE,WAAwB,EACxB,kBAA4C,EAC5C,QAAgB,EAChB,WAAmB,EACnB,UAAoC,EAAE;QAEtC,KAAK,CAAC,EAAE,UAAU,EAAE,KAAK,EAAE,CAAC,CAAC;QAE7B,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG;YACb,WAAW,EAAE,KAAK,EAAE,aAAa;YACjC,UAAU,EAAE,IAAI;YAChB,eAAe,EAAE,GAAG,EAAE,QAAQ;YAC9B,GAAG,OAAO;SACX,CAAC;IACJ,CAAC;IAED;;OAEG;IACM,KAAK,CAAC,KAAa;QAC1B,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChB,OAAO;QACT,CAAC;QAED,0CAA0C;QAC1C,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,YAAY,EAAE,CAAC;IACtB,CAAC;IAED;;OAEG;IACK,YAAY;QAClB,IAAI,CAAC,SAAS,GAAG,WAAW,CAAC,KAAK,IAAI,EAAE;YACtC,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;YAC3B,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,2CAA2C,IAAI,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC,CAAC;gBACpF,IAAI,CAAC,OAAO,CAAC,KAAc,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;QAEjC,cAAc;QACd,UAAU,CAAC,GAAG,EAAE;YACd,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACxC,OAAO,CAAC,KAAK,CAAC,6CAA6C,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;gBAC5E,IAAI,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IAC/B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW;QACvB,IAAI,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC;YACzB,wCAAwC;YACxC,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC5B,CAAC;aAAM,CAAC;YACN,yCAAyC;YACzC,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAC9B,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,YAAY;QACxB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAC5C,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,OAAO,CAAC,UAAU,EACvB,OAAO,CACR,CAAC;YAEF,IAAI,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChD,+CAA+C;gBAC/C,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;gBAChE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBAC1B,OAAO,CAAC,KAAK,CACX,iCAAiC,MAAM,CAAC,OAAO,CAAC,MAAM,6BAA6B,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE,CAC/H,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,oDAAoD;gBACpD,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAC7E,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC;oBACzC,mDAAmD;oBACnD,OAAO,CAAC,KAAK,CACX,oCAAoC,IAAI,CAAC,WAAW,mCAAmC,CACxF,CAAC;oBACF,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM;oBACvB,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,CAAC;qBAAM,CAAC;oBACN,yDAAyD;oBACzD,OAAO,CAAC,KAAK,CACX,2EAA2E,IAAI,CAAC,QAAQ,EAAE,CAC3F,CAAC;oBACF,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;oBAE3B,sDAAsD;oBACtD,IAAI,WAAW,EAAE,CAAC;wBAChB,kEAAkE;wBAClE,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,gBAAgB,CAAC,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;wBACrF,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;4BAC7B,0EAA0E;4BAC1E,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC,wBAAwB,CAAC,aAAa,CAAC,CAAC;wBACzE,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,6CAA6C,KAAK,EAAE,CAAC,CAAC;YACpE,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,cAAc;QAC1B,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,cAAc,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7E,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,CAAC,KAAK,CAAC,uDAAuD,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YACzF,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAChB,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,OAAO;QACT,CAAC;QAED,oDAAoD;QACpD,MAAM,UAAU,GAAG,IAAI,CAAC,kBAAkB,CAAC,sBAAsB,CAC/D,IAAI,CAAC,WAAW,EAChB,IAAI,CAAC,kBAAkB,GAAG,CAAC,EAC3B,EAAE,CAAC,2BAA2B;SAC/B,CAAC;QAEF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACvB,IAAI,CAAC,kBAAkB,GAAG,MAAM,CAAC,cAAc,CAAC;gBAChD,OAAO,CAAC,KAAK,CACX,gDAAgD,MAAM,CAAC,cAAc,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,QAAQ,CACrG,CAAC;YACJ,CAAC;QACH,CAAC;QAED,gEAAgE;QAChE,IAAI,CAAC,WAAW,CAAC,QAAQ,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrD,OAAO,CAAC,KAAK,CAAC,iDAAiD,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YACnF,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM;YACvB,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,wBAAwB,CAC9B,aAA8D;QAE9D,yDAAyD;QACzD,4EAA4E;QAC5E,IAAI,cAAc,GAAG,CAAC,CAAC;QACvB,KAAK,IAAI,CAAC,GAAG,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACnD,MAAM,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;YAChC,IAAI,MAAM,EAAE,CAAC;gBACX,cAAc,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;gBACrC,IAAI,cAAc,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBACxC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC;gBAChD,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,CAAC,CAAC,CAAC;IACZ,CAAC;IAED;;OAEG;IACK,OAAO;QACb,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAC9B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;QACD,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED;;OAEG;IACM,QAAQ,CAAC,KAAmB,EAAE,QAAwC;QAC7E,IAAI,CAAC,OAAO,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CACX,sCAAsC,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,gBAAgB,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CACrG,CAAC;QACF,QAAQ,CAAC,KAAK,CAAC,CAAC;IAClB,CAAC;CACF"}
|
|
@@ -39,7 +39,7 @@ export declare class TerminalManager {
|
|
|
39
39
|
};
|
|
40
40
|
}>;
|
|
41
41
|
/**
|
|
42
|
-
*
|
|
42
|
+
* Interpret escape sequences for control codes
|
|
43
43
|
*/
|
|
44
44
|
private parseControlCodes;
|
|
45
45
|
getOutput(terminalId: string, startLine?: number, lineCount?: number, includeAnsi?: boolean, includeForegroundProcess?: boolean): Promise<{
|
|
@@ -86,11 +86,11 @@ export declare class TerminalManager {
|
|
|
86
86
|
getIdleTerminals(idleMinutes?: number): string[];
|
|
87
87
|
cleanup(): void;
|
|
88
88
|
/**
|
|
89
|
-
|
|
89
|
+
* Update foreground process information
|
|
90
90
|
*/
|
|
91
91
|
private updateForegroundProcess;
|
|
92
92
|
/**
|
|
93
|
-
*
|
|
93
|
+
* Check program guard
|
|
94
94
|
*/
|
|
95
95
|
private checkProgramGuard;
|
|
96
96
|
}
|