@rdfc/js-runner 3.0.3 → 3.0.4-remote

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/.claude/settings.local.json +12 -0
  2. package/README.md +37 -1
  3. package/bin/runner.js +7 -1
  4. package/bin/server.js +13 -0
  5. package/examples/echo/.idea/echo.iml +9 -0
  6. package/examples/echo/.idea/misc.xml +6 -0
  7. package/examples/echo/.idea/modules.xml +8 -0
  8. package/examples/echo/.idea/vcs.xml +7 -0
  9. package/examples/echo/.swls/config.json +1 -0
  10. package/examples/echo/package-lock.json +27 -29
  11. package/examples/echo/pipeline.ttl +0 -1
  12. package/examples/echo/processors.ttl +1 -1
  13. package/examples/echo/remote_pipeline.ttl +18 -0
  14. package/examples/echo/server.ttl +5 -0
  15. package/examples/echo/untitled:/types/MyType.ttl +0 -0
  16. package/file:/home/silvius/Projects/mumo-pipeline/ldes/http_3A_2F_2Fdata.mumo.be_2Fstreams_2Fnodes_2Fdefault/root/index.trig +3 -0
  17. package/ldes/http_3A_2F_2Fdata.mumo.be_2Fstreams_2Fnodes_2Fdefault/root/index.trig +3 -0
  18. package/lib/client.d.ts +2 -1
  19. package/lib/client.js +70 -22
  20. package/lib/index.d.ts +2 -0
  21. package/lib/index.js +3 -1
  22. package/lib/jsonld.d.ts +17 -0
  23. package/lib/jsonld.js +135 -0
  24. package/lib/reader.d.ts +4 -1
  25. package/lib/reader.js +11 -3
  26. package/lib/runner.d.ts +6 -1
  27. package/lib/runner.js +43 -15
  28. package/lib/server.d.ts +9 -0
  29. package/lib/server.js +459 -0
  30. package/lib/state.d.ts +32 -0
  31. package/lib/state.js +71 -0
  32. package/lib/testUtils.d.ts +24 -0
  33. package/lib/testUtils.js +150 -0
  34. package/lib/tsconfig.tsbuildinfo +1 -1
  35. package/lib/writer.d.ts +5 -1
  36. package/lib/writer.js +26 -10
  37. package/minimal.ttl +99 -0
  38. package/package.json +12 -11
  39. package/src/client.ts +99 -24
  40. package/src/index.ts +2 -0
  41. package/src/reader.ts +11 -1
  42. package/src/runner.ts +58 -11
  43. package/src/server.ts +545 -0
  44. package/src/state.ts +105 -0
  45. package/src/writer.ts +36 -12
package/src/writer.ts CHANGED
@@ -2,6 +2,7 @@ import { FromRunner, RunnerClient } from '@rdfc/proto'
2
2
  import { promisify } from 'util'
3
3
  import { Logger } from 'winston'
4
4
  import { Any } from './reader'
5
+ import { ChannelTracker } from './state'
5
6
 
6
7
  type Writable = (msg: FromRunner) => Promise<unknown>
7
8
  export interface Writer {
@@ -26,12 +27,19 @@ export class WriterInstance implements Writer {
26
27
  private readonly notifyOrchestrator: Writable
27
28
  private readonly logger: Logger
28
29
 
29
- private awaitingProcessed: Array<() => void> = []
30
+ private awaitingProcessed: Array<{
31
+ resolve: () => void
32
+ startMs: number
33
+ bytes: number
34
+ }> = []
30
35
 
31
36
  private openStreams: number = 0
32
37
  private shouldClose: Array<() => void> = []
38
+ private hasClosed = false
39
+ private remoteCloseReceived = false
33
40
 
34
41
  private readonly runnerId: string
42
+ private readonly tracker: ChannelTracker | undefined
35
43
 
36
44
  constructor(
37
45
  uri: string,
@@ -39,16 +47,21 @@ export class WriterInstance implements Writer {
39
47
  notifyOrchestrator: Writable,
40
48
  runnerId: string,
41
49
  logger: Logger,
50
+ tracker?: ChannelTracker,
42
51
  ) {
43
52
  this.client = client
44
53
  this.notifyOrchestrator = notifyOrchestrator
45
54
  this.uri = uri
46
55
  this.logger = logger
47
56
  this.runnerId = runnerId
57
+ this.tracker = tracker
48
58
  }
49
59
 
50
- private awaitProcessed(): Promise<void> {
51
- return new Promise((res) => this.awaitingProcessed.push(res))
60
+ private awaitProcessed(bytes: number): Promise<void> {
61
+ const startMs = Date.now()
62
+ return new Promise<void>((resolve) => {
63
+ this.awaitingProcessed.push({ resolve, startMs, bytes })
64
+ })
52
65
  }
53
66
 
54
67
  async any(any: Any): Promise<void> {
@@ -66,7 +79,7 @@ export class WriterInstance implements Writer {
66
79
  async buffer(buffer: Uint8Array): Promise<void> {
67
80
  this.logger.debug(`${this.uri} sends buffer ${buffer.length} bytes`)
68
81
  const localSequenceNumber = this.localSequenceNumber++
69
- const handledPromise = this.awaitProcessed()
82
+ const handledPromise = this.awaitProcessed(buffer.length)
70
83
 
71
84
  await this.notifyOrchestrator({
72
85
  msg: { data: buffer, channel: this.uri, localSequenceNumber },
@@ -82,7 +95,7 @@ export class WriterInstance implements Writer {
82
95
  const t = transform || ((x: unknown) => <Uint8Array>x)
83
96
  const stream = this.client.sendStreamMessage()
84
97
 
85
- const handledPromise = this.awaitProcessed()
98
+ const handledPromise = this.awaitProcessed(0) // bytes unknown for streams
86
99
  const writeStreamMessageChunk = promisify(stream.write.bind(stream))
87
100
  const localSequenceNumber = this.localSequenceNumber++
88
101
  await writeStreamMessageChunk({
@@ -118,11 +131,12 @@ export class WriterInstance implements Writer {
118
131
  async string(msg: string): Promise<void> {
119
132
  this.logger.debug(`${this.uri} sends string ${msg.length} characters`)
120
133
  const localSequenceNumber = this.localSequenceNumber++
121
- const handledPromise = this.awaitProcessed()
134
+ const encoded = encoder.encode(msg)
135
+ const handledPromise = this.awaitProcessed(encoded.length)
122
136
 
123
137
  await this.notifyOrchestrator({
124
138
  msg: {
125
- data: encoder.encode(msg),
139
+ data: encoded,
126
140
  channel: this.uri,
127
141
  localSequenceNumber,
128
142
  },
@@ -138,20 +152,27 @@ export class WriterInstance implements Writer {
138
152
  * - If there are still active streams, closing is deferred until they complete.
139
153
  * - If multiple callers invoke `close()` while waiting, their Promises are queued and
140
154
  * resolved once the channel actually closes.
141
- * - If this side initiated the close (`issued = false`), a close message is sent to the remote.
155
+ * - A close message is sent to the remote only if the close was locally initiated and
156
+ * the remote has not already sent a close.
142
157
  *
143
158
  * @param issued - If true, indicates the close request originated remotely
144
159
  */
145
160
  async close(issued = false): Promise<void> {
146
- // Case 1: Active streams still running → wait until they finish
161
+ if (issued) this.remoteCloseReceived = true
162
+
163
+ // Case 1: Active streams still running → defer until they finish
147
164
  if (this.openStreams !== 0) {
148
165
  await new Promise<void>((resolve) => this.shouldClose.push(resolve))
149
166
  return
150
167
  }
151
168
 
152
- // Case 2: No active streams perform actual close
169
+ // Case 2: Already closed nothing to do
170
+ if (this.hasClosed) return
171
+ this.hasClosed = true
172
+
173
+ // Case 3: No active streams → perform actual close
153
174
  this.logger.debug(`${this.uri} closes stream`)
154
- if (!issued) {
175
+ if (!this.remoteCloseReceived) {
155
176
  await this.notifyOrchestrator({
156
177
  close: { channel: this.uri },
157
178
  })
@@ -169,7 +190,10 @@ export class WriterInstance implements Writer {
169
190
  */
170
191
  handled(): void {
171
192
  if (this.awaitingProcessed.length > 0) {
172
- this.awaitingProcessed.shift()!()
193
+ const { resolve, startMs, bytes } = this.awaitingProcessed.shift()!
194
+ const latencyMs = Date.now() - startMs
195
+ this.tracker?.recordMessage(bytes, latencyMs)
196
+ resolve()
173
197
  } else {
174
198
  this.logger.error(
175
199
  'Expected to be waiting for a message to be processed, but this is not the case ' +