@rdfc/js-runner 3.0.3 → 3.0.4-remote
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.local.json +12 -0
- package/README.md +37 -1
- package/bin/runner.js +7 -1
- package/bin/server.js +13 -0
- package/examples/echo/.idea/echo.iml +9 -0
- package/examples/echo/.idea/misc.xml +6 -0
- package/examples/echo/.idea/modules.xml +8 -0
- package/examples/echo/.idea/vcs.xml +7 -0
- package/examples/echo/.swls/config.json +1 -0
- package/examples/echo/package-lock.json +27 -29
- package/examples/echo/pipeline.ttl +0 -1
- package/examples/echo/processors.ttl +1 -1
- package/examples/echo/remote_pipeline.ttl +18 -0
- package/examples/echo/server.ttl +5 -0
- package/examples/echo/untitled:/types/MyType.ttl +0 -0
- package/file:/home/silvius/Projects/mumo-pipeline/ldes/http_3A_2F_2Fdata.mumo.be_2Fstreams_2Fnodes_2Fdefault/root/index.trig +3 -0
- package/ldes/http_3A_2F_2Fdata.mumo.be_2Fstreams_2Fnodes_2Fdefault/root/index.trig +3 -0
- package/lib/client.d.ts +2 -1
- package/lib/client.js +70 -22
- package/lib/index.d.ts +2 -0
- package/lib/index.js +3 -1
- package/lib/jsonld.d.ts +17 -0
- package/lib/jsonld.js +135 -0
- package/lib/reader.d.ts +4 -1
- package/lib/reader.js +11 -3
- package/lib/runner.d.ts +6 -1
- package/lib/runner.js +43 -15
- package/lib/server.d.ts +9 -0
- package/lib/server.js +459 -0
- package/lib/state.d.ts +32 -0
- package/lib/state.js +71 -0
- package/lib/testUtils.d.ts +24 -0
- package/lib/testUtils.js +150 -0
- package/lib/tsconfig.tsbuildinfo +1 -1
- package/lib/writer.d.ts +5 -1
- package/lib/writer.js +26 -10
- package/minimal.ttl +99 -0
- package/package.json +12 -11
- package/src/client.ts +99 -24
- package/src/index.ts +2 -0
- package/src/reader.ts +11 -1
- package/src/runner.ts +58 -11
- package/src/server.ts +545 -0
- package/src/state.ts +105 -0
- package/src/writer.ts +36 -12
package/src/writer.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { FromRunner, RunnerClient } from '@rdfc/proto'
|
|
|
2
2
|
import { promisify } from 'util'
|
|
3
3
|
import { Logger } from 'winston'
|
|
4
4
|
import { Any } from './reader'
|
|
5
|
+
import { ChannelTracker } from './state'
|
|
5
6
|
|
|
6
7
|
type Writable = (msg: FromRunner) => Promise<unknown>
|
|
7
8
|
export interface Writer {
|
|
@@ -26,12 +27,19 @@ export class WriterInstance implements Writer {
|
|
|
26
27
|
private readonly notifyOrchestrator: Writable
|
|
27
28
|
private readonly logger: Logger
|
|
28
29
|
|
|
29
|
-
private awaitingProcessed: Array<
|
|
30
|
+
private awaitingProcessed: Array<{
|
|
31
|
+
resolve: () => void
|
|
32
|
+
startMs: number
|
|
33
|
+
bytes: number
|
|
34
|
+
}> = []
|
|
30
35
|
|
|
31
36
|
private openStreams: number = 0
|
|
32
37
|
private shouldClose: Array<() => void> = []
|
|
38
|
+
private hasClosed = false
|
|
39
|
+
private remoteCloseReceived = false
|
|
33
40
|
|
|
34
41
|
private readonly runnerId: string
|
|
42
|
+
private readonly tracker: ChannelTracker | undefined
|
|
35
43
|
|
|
36
44
|
constructor(
|
|
37
45
|
uri: string,
|
|
@@ -39,16 +47,21 @@ export class WriterInstance implements Writer {
|
|
|
39
47
|
notifyOrchestrator: Writable,
|
|
40
48
|
runnerId: string,
|
|
41
49
|
logger: Logger,
|
|
50
|
+
tracker?: ChannelTracker,
|
|
42
51
|
) {
|
|
43
52
|
this.client = client
|
|
44
53
|
this.notifyOrchestrator = notifyOrchestrator
|
|
45
54
|
this.uri = uri
|
|
46
55
|
this.logger = logger
|
|
47
56
|
this.runnerId = runnerId
|
|
57
|
+
this.tracker = tracker
|
|
48
58
|
}
|
|
49
59
|
|
|
50
|
-
private awaitProcessed(): Promise<void> {
|
|
51
|
-
|
|
60
|
+
private awaitProcessed(bytes: number): Promise<void> {
|
|
61
|
+
const startMs = Date.now()
|
|
62
|
+
return new Promise<void>((resolve) => {
|
|
63
|
+
this.awaitingProcessed.push({ resolve, startMs, bytes })
|
|
64
|
+
})
|
|
52
65
|
}
|
|
53
66
|
|
|
54
67
|
async any(any: Any): Promise<void> {
|
|
@@ -66,7 +79,7 @@ export class WriterInstance implements Writer {
|
|
|
66
79
|
async buffer(buffer: Uint8Array): Promise<void> {
|
|
67
80
|
this.logger.debug(`${this.uri} sends buffer ${buffer.length} bytes`)
|
|
68
81
|
const localSequenceNumber = this.localSequenceNumber++
|
|
69
|
-
const handledPromise = this.awaitProcessed()
|
|
82
|
+
const handledPromise = this.awaitProcessed(buffer.length)
|
|
70
83
|
|
|
71
84
|
await this.notifyOrchestrator({
|
|
72
85
|
msg: { data: buffer, channel: this.uri, localSequenceNumber },
|
|
@@ -82,7 +95,7 @@ export class WriterInstance implements Writer {
|
|
|
82
95
|
const t = transform || ((x: unknown) => <Uint8Array>x)
|
|
83
96
|
const stream = this.client.sendStreamMessage()
|
|
84
97
|
|
|
85
|
-
const handledPromise = this.awaitProcessed()
|
|
98
|
+
const handledPromise = this.awaitProcessed(0) // bytes unknown for streams
|
|
86
99
|
const writeStreamMessageChunk = promisify(stream.write.bind(stream))
|
|
87
100
|
const localSequenceNumber = this.localSequenceNumber++
|
|
88
101
|
await writeStreamMessageChunk({
|
|
@@ -118,11 +131,12 @@ export class WriterInstance implements Writer {
|
|
|
118
131
|
async string(msg: string): Promise<void> {
|
|
119
132
|
this.logger.debug(`${this.uri} sends string ${msg.length} characters`)
|
|
120
133
|
const localSequenceNumber = this.localSequenceNumber++
|
|
121
|
-
const
|
|
134
|
+
const encoded = encoder.encode(msg)
|
|
135
|
+
const handledPromise = this.awaitProcessed(encoded.length)
|
|
122
136
|
|
|
123
137
|
await this.notifyOrchestrator({
|
|
124
138
|
msg: {
|
|
125
|
-
data:
|
|
139
|
+
data: encoded,
|
|
126
140
|
channel: this.uri,
|
|
127
141
|
localSequenceNumber,
|
|
128
142
|
},
|
|
@@ -138,20 +152,27 @@ export class WriterInstance implements Writer {
|
|
|
138
152
|
* - If there are still active streams, closing is deferred until they complete.
|
|
139
153
|
* - If multiple callers invoke `close()` while waiting, their Promises are queued and
|
|
140
154
|
* resolved once the channel actually closes.
|
|
141
|
-
* -
|
|
155
|
+
* - A close message is sent to the remote only if the close was locally initiated and
|
|
156
|
+
* the remote has not already sent a close.
|
|
142
157
|
*
|
|
143
158
|
* @param issued - If true, indicates the close request originated remotely
|
|
144
159
|
*/
|
|
145
160
|
async close(issued = false): Promise<void> {
|
|
146
|
-
|
|
161
|
+
if (issued) this.remoteCloseReceived = true
|
|
162
|
+
|
|
163
|
+
// Case 1: Active streams still running → defer until they finish
|
|
147
164
|
if (this.openStreams !== 0) {
|
|
148
165
|
await new Promise<void>((resolve) => this.shouldClose.push(resolve))
|
|
149
166
|
return
|
|
150
167
|
}
|
|
151
168
|
|
|
152
|
-
// Case 2:
|
|
169
|
+
// Case 2: Already closed — nothing to do
|
|
170
|
+
if (this.hasClosed) return
|
|
171
|
+
this.hasClosed = true
|
|
172
|
+
|
|
173
|
+
// Case 3: No active streams → perform actual close
|
|
153
174
|
this.logger.debug(`${this.uri} closes stream`)
|
|
154
|
-
if (!
|
|
175
|
+
if (!this.remoteCloseReceived) {
|
|
155
176
|
await this.notifyOrchestrator({
|
|
156
177
|
close: { channel: this.uri },
|
|
157
178
|
})
|
|
@@ -169,7 +190,10 @@ export class WriterInstance implements Writer {
|
|
|
169
190
|
*/
|
|
170
191
|
handled(): void {
|
|
171
192
|
if (this.awaitingProcessed.length > 0) {
|
|
172
|
-
this.awaitingProcessed.shift()!
|
|
193
|
+
const { resolve, startMs, bytes } = this.awaitingProcessed.shift()!
|
|
194
|
+
const latencyMs = Date.now() - startMs
|
|
195
|
+
this.tracker?.recordMessage(bytes, latencyMs)
|
|
196
|
+
resolve()
|
|
173
197
|
} else {
|
|
174
198
|
this.logger.error(
|
|
175
199
|
'Expected to be waiting for a message to be processed, but this is not the case ' +
|