@rdfc/js-runner 2.0.0 → 3.0.0

package/src/testUtils/index.ts

@@ -0,0 +1,430 @@
+import {
+  DataChunk,
+  FromRunner,
+  LogMessage,
+  Processor as ProcConfig,
+  ToRunner,
+} from '@rdfc/proto'
+import {} from '../reexports'
+import { extractShapes } from 'rdf-lens'
+import { NamedNode, Parser, Writer as N3Writer } from 'n3'
+import { readFile } from 'fs/promises'
+import winston, { createLogger } from 'winston'
+import { Processor } from '../processor'
+import { FullProc, Runner, Writable } from '../runner'
+import { Quad } from '@rdfjs/types'
+import { createTermNamespace } from '@treecg/types'
+import {
+  ReceivingStreamControl,
+  SendingStreamControl,
+  StreamChunk,
+  StreamIdentify,
+} from '@rdfc/proto/lib/generated/common'
+import { MockClientDuplexStream } from './duplex'
+import { promisify } from 'util'
+import { Reader } from '../reader'
+import { Writer } from '../writer'
+
+export function channel(runner: Runner, name: string): [Writer, Reader] {
+  const n = new NamedNode(name)
+  const reader = runner.createReader(n)
+  const writer = runner.createWriter(n)
+
+  return [writer, reader]
+}
+
+export class StreamMsgMock {
+  data: DataChunk[] = []
+
+  private readonly resolveId: (id: StreamIdentify) => number
+
+  constructor(resolveId: (id: StreamIdentify) => number) {
+    this.resolveId = resolveId
+  }
+
+  sendStreamMessage(): MockClientDuplexStream<
+    StreamChunk,
+    ReceivingStreamControl
+  > {
+    const sendingStream = new MockClientDuplexStream<
+      StreamChunk,
+      ReceivingStreamControl
+    >()
+
+    let at = 0
+    sendingStream.register(
+      (x) => x.data,
+      (d, send) => {
+        setTimeout(() => send({ streamSequenceNumber: ++at }), 20)
+        this.data.push(d)
+      },
+    )
+    sendingStream.register(
+      (x) => x.id,
+      (d, send) => send({ streamSequenceNumber: this.resolveId(d) }),
+    )
+    return sendingStream
+  }
+}
+
+export class OrchestratorMock {
+  connectStream: MockClientDuplexStream<FromRunner, ToRunner>
+
+  streamMsgs: {
+    [id: string]: {
+      toProducingStream: (id: ReceivingStreamControl) => void
+      receivingStream?: MockClientDuplexStream<SendingStreamControl, DataChunk>
+    }
+  } = {}
+
+  streamMsgCount = 0
+
+  connect(): MockClientDuplexStream<FromRunner, ToRunner> {
+    const connectStream = new MockClientDuplexStream<FromRunner, ToRunner>()
+    // Always bounce processed msgs back to the runner
+    connectStream.register(
+      (msg) => msg.processed,
+      ({ channel, globalSequenceNumber }, send) => {
+        send({
+          processed: { channel, localSequenceNumber: globalSequenceNumber },
+        })
+      },
+    )
+
+    // Always bounce data msgs back to the runner
+    connectStream.register(
+      (msg) => msg.msg,
+      ({ localSequenceNumber, data, channel }, send) => {
+        send({
+          msg: { globalSequenceNumber: localSequenceNumber, channel, data },
+        })
+      },
+    )
+
+    // Always bounce close msgs back to the runner
+    connectStream.register(
+      (msg) => msg.close,
+      (close, send) => {
+        send({ close })
+      },
+    )
+
+    this.connectStream = connectStream
+    return connectStream
+  }
+
+  sendStreamMessage(): MockClientDuplexStream<
+    StreamChunk,
+    ReceivingStreamControl
+  > {
+    const out = new MockClientDuplexStream<
+      StreamChunk,
+      ReceivingStreamControl
+    >()
+    const id = this.streamMsgCount
+    this.streamMsgCount++
+
+    out.registerOnce(
+      (x) => x.id,
+      ({ channel }, toProducingStream) => {
+        this.streamMsgs[id] = { toProducingStream }
+
+        // Notify stream message
+        this.connectStream.send({
+          streamMsg: { channel, globalSequenceNumber: id },
+        })
+      },
+    )
+
+    out.register(
+      (x) => x.data,
+      (data) => {
+        // bounce data to receiving stream
+        this.streamMsgs[id].receivingStream!.send(data)
+      },
+    )
+
+    out.on('end', () => {
+      // end receiving stream
+      this.streamMsgs[id].receivingStream!.end()
+    })
+
+    return out
+  }
+
+  receiveStreamMessage(): MockClientDuplexStream<
+    SendingStreamControl,
+    DataChunk
+  > {
+    const receivingStream = new MockClientDuplexStream<
+      SendingStreamControl,
+      DataChunk
+    >()
+    let streamId = 0
+
+    receivingStream.registerOnce(
+      (x) => x.globalSequenceNumber,
+      (id) => {
+        streamId = id
+        this.streamMsgs[id].receivingStream = receivingStream
+        this.streamMsgs[id].toProducingStream({ streamSequenceNumber: 0 })
+      },
+    )
+
+    receivingStream.register(
+      (x) => x.streamSequenceNumber,
+      (streamSequenceNumber) => {
+        // Bounce processed message to producing stream
+        this.streamMsgs[streamId].toProducingStream({ streamSequenceNumber })
+      },
+    )
+
+    return receivingStream
+  }
+
+  logStream(): MockClientDuplexStream<LogMessage, null> {
+    const logStream = new MockClientDuplexStream<LogMessage, null>()
+    logStream.register((x) => x, console.log)
+    return logStream
+  }
+}
+
+export function createRunner(uri = 'http://example.com/ns#') {
+  const logger = createLogger({
+    transports: new winston.transports.Console({
+      level: process.env['DEBUG'] || 'info',
+    }),
+  })
+
+  // Mock the GRPC
+  const client = new OrchestratorMock()
+
+  // Connect just like client.ts:start()
+  const stream = client.connect()
+  const writable = promisify(stream.write.bind(stream))
+
+  const runner = new Runner(
+    /* eslint-disable @typescript-eslint/no-explicit-any */
+    client as any, // Type assertion for testing
+    writable as Writable,
+    uri,
+    logger,
+  )
+
+  stream.on('data', (msg: ToRunner) => runner.handleOrchMessage(msg))
+
+  return runner
+}
+
+export async function one<T>(iter: AsyncIterable<T>): Promise<T | undefined> {
+  for await (const item of iter) {
+    return item
+  }
+}
+
+const shapeQuads = `
+@prefix rdfc: <https://w3id.org/rdf-connect#>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+[ ] a sh:NodeShape;
+  sh:targetClass <JsProcessorShape>;
+  sh:property [
+    sh:path rdfc:entrypoint;
+    sh:name "location";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+  ], [
+    sh:path rdfc:file;
+    sh:name "file";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+  ], [
+    sh:path rdfc:class;
+    sh:name "clazz";
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+  ].
+`
+export type ConfigType = {
+  location: string
+  file: string
+  clazz: string
+}
+
+const OWL = createTermNamespace('http://www.w3.org/2002/07/owl#', 'imports')
+const processorShapes = extractShapes(new Parser().parse(shapeQuads))
+const base = 'https://w3id.org/rdf-connect#'
+
+export async function importFile(
+  file: string,
+  content?: string,
+): Promise<Quad[]> {
+  const done = new Set<string>()
+
+  const quads: Quad[] = []
+  const todo: URL[] = []
+
+  const parse = (content: string, baseIRI: URL) => {
+    done.add(baseIRI.toString())
+
+    const extras = new Parser({ baseIRI: baseIRI.toString() }).parse(content)
+
+    for (const o of extras
+      .filter(
+        (x) =>
+          x.subject.value === baseIRI?.toString() &&
+          x.predicate.equals(OWL.imports),
+      )
+      .map((x) => x.object.value)) {
+      todo.push(new URL(o))
+    }
+
+    quads.push(...extras)
+  }
+
+  if (content) {
+    parse(content, new URL('file://' + file))
+  } else {
+    todo.push(new URL('file://' + file))
+  }
+
+  let item = todo.pop()
+  while (item !== undefined) {
+    if (done.has(item.toString())) {
+      item = todo.pop()
+      continue
+    }
+
+    if (item.protocol !== 'file:') {
+      throw 'No supported protocol ' + item.protocol
+    }
+    const txt = await readFile(item.pathname, { encoding: 'utf8' })
+    parse(txt, item)
+
+    item = todo.pop()
+  }
+
+  return quads
+}
+
+/**
+ * Helper class to gradually test your processors.
+ * Possible flow:
+ * - import the JsRunner index file
+ * - import your processor config file
+ * - test if the config is as you would expect (from getConfig())
+ * - import your processor definition (inline)
+ * - build your processor
+ * - test your processor
+ */
+export class ProcHelper<T extends Processor<unknown>> {
+  runner: Runner
+  quads: Quad[] = []
+  config: ConfigType
+  proc: FullProc<T>
+
+  constructor(uri?: string) {
+    this.runner = createRunner(uri)
+  }
+
+  async importInline(baseIRI: string, config: string) {
+    const configQuads = await importFile(baseIRI, config)
+    this.quads.push(...configQuads)
+  }
+
+  async importFile(file: string) {
+    const configQuads = await importFile(file)
+    this.quads.push(...configQuads)
+  }
+
+  getConfig(ty: string | NamedNode): ConfigType {
+    const id = typeof ty === 'string' ? new NamedNode(base + ty) : ty
+    const procConfig = <ConfigType>processorShapes.lenses[
+      'JsProcessorShape'
+    ].execute({
+      id,
+      quads: this.quads,
+    })
+
+    this.config = procConfig
+    return procConfig
+  }
+
+  async getProcessor(
+    uri: string = 'http://example.com/ns#processor',
+  ): Promise<FullProc<T>> {
+    await this.runner.handleOrchMessage({
+      pipeline: new N3Writer().quadsToString(this.quads),
+    })
+
+    const proc = await this.runner.addProcessor<T>({
+      config: JSON.stringify(this.config),
+      arguments: '',
+      uri,
+    })
+    return proc
+  }
+}
+
+/**
+ * @deprecated use {@link ProcHelper}
+ */
+export async function getProcInline<T extends Processor<unknown>>(
+  config: string,
+  ty: string,
+  runner: Runner,
+  baseIRI: string,
+  uri = 'http://example.com/ns#processor',
+): Promise<FullProc<T>> {
+  const configQuads = await importFile(baseIRI, config)
+  const procConfig = <ProcConfig>processorShapes.lenses[
+    'JsProcessorShape'
+  ].execute({
+    id: new NamedNode(base + ty),
+    quads: configQuads,
+  })
+
+  await runner.handleOrchMessage({
+    pipeline: new N3Writer().quadsToString(configQuads),
+  })
+
+  const proc = await runner.addProcessor<T>({
+    config: JSON.stringify(procConfig),
+    arguments: '',
+    uri,
+  })
+
+  return proc
+}
+
+/**
+ * @deprecated use {@link ProcHelper}
+ */
+export async function getProc<T extends Processor<unknown>>(
+  config: string,
+  ty: string,
+  runner: Runner,
+  configLocation: string,
+  uri = 'http://example.com/ns#processor',
+): Promise<FullProc<T>> {
+  const configQuads = await importFile(configLocation)
+  const procConfig = processorShapes.lenses['JsProcessorShape'].execute({
+    id: new NamedNode(base + ty),
+    quads: configQuads,
+  })
+
+  configQuads.push(...new Parser().parse(config))
+  await runner.handleOrchMessage({
+    pipeline: new N3Writer().quadsToString(configQuads),
+  })
+
+  const proc = await runner.addProcessor<T>({
+    config: JSON.stringify(procConfig),
+    arguments: '',
+    uri,
+  })
+
+  return proc
+}

package/src/writer.ts

@@ -1,9 +1,9 @@
-import { Id, OrchestratorMessage, RunnerClient } from '@rdfc/proto'
+import { FromRunner, RunnerClient } from '@rdfc/proto'
 import { promisify } from 'util'
 import { Logger } from 'winston'
 import { Any } from './reader'
 
-type Writable = (msg: OrchestratorMessage) => Promise<unknown>
+type Writable = (msg: FromRunner) => Promise<unknown>
 export interface Writer {
   readonly uri: string
   buffer(buffer: Uint8Array): Promise<void>
@@ -11,7 +11,7 @@ export interface Writer {
   stream(buffer: AsyncIterable<Uint8Array>): Promise<void>
   stream<T>(
     buffer: AsyncIterable<T>,
-    tranform: (x: T) => Uint8Array,
+    transform: (x: T) => Uint8Array,
   ): Promise<void>
 
   string(buffer: string): Promise<void>
@@ -21,21 +21,36 @@ export interface Writer {
 const encoder = new TextEncoder()
 export class WriterInstance implements Writer {
   readonly uri: string
+  localSequenceNumber: number = 1
   private readonly client: RunnerClient
-  private readonly write: Writable
+  private readonly notifyOrchestrator: Writable
   private readonly logger: Logger
 
+  private awaitingProcessed: Array<() => void> = []
+
+  private openStreams: number = 0
+  private shouldClose: Array<() => void> = []
+
+  private readonly runnerId: string
+
   constructor(
     uri: string,
     client: RunnerClient,
-    write: Writable,
+    notifyOrchestrator: Writable,
+    runnerId: string,
     logger: Logger,
   ) {
     this.client = client
-    this.write = write
+    this.notifyOrchestrator = notifyOrchestrator
     this.uri = uri
     this.logger = logger
+    this.runnerId = runnerId
   }
+
+  private awaitProcessed(): Promise<void> {
+    return new Promise((res) => this.awaitingProcessed.push(res))
+  }
+
   async any(any: Any): Promise<void> {
     if ('stream' in any) {
       await this.stream(any.stream)
@@ -50,41 +65,116 @@ export class WriterInstance implements Writer {
 
   async buffer(buffer: Uint8Array): Promise<void> {
     this.logger.debug(`${this.uri} sends buffer ${buffer.length} bytes`)
-    await this.write({ msg: { data: buffer, channel: this.uri } })
+    const localSequenceNumber = this.localSequenceNumber++
+    const handledPromise = this.awaitProcessed()
+
+    await this.notifyOrchestrator({
+      msg: { data: buffer, channel: this.uri, localSequenceNumber },
+    })
+    await handledPromise
   }
 
   async stream<T = Uint8Array>(
     buffer: AsyncIterable<T>,
     transform?: (x: T) => Uint8Array,
   ) {
+    this.openStreams += 1
     const t = transform || ((x: unknown) => <Uint8Array>x)
     const stream = this.client.sendStreamMessage()
-    const id: Id = await new Promise((res) => stream.once('data', res))
-    this.logger.debug(`${this.uri} streams message with id ${id.id}`)
-    await this.write({ streamMsg: { id, channel: this.uri } })
 
-    const write = promisify(stream.write.bind(stream))
+    const handledPromise = this.awaitProcessed()
+    const writeStreamMessageChunk = promisify(stream.write.bind(stream))
+    const localSequenceNumber = this.localSequenceNumber++
+    await writeStreamMessageChunk({
+      id: {
+        channel: this.uri,
+        localSequenceNumber,
+        runner: this.runnerId,
+      },
+    })
+
+    const id = await new Promise((res) => stream.once('data', res))
+
+    this.logger.debug(
+      `${this.uri} streams message with id ${JSON.stringify(id)}`,
+    )
+
     for await (const msg of buffer) {
-      await write({ data: t(msg) })
+      const processedPromise = new Promise((res) => stream.once('data', res))
+      await writeStreamMessageChunk({ data: { data: t(msg) } })
+      // Await a message on the stream, indicating that the chunk has been processed
+      await processedPromise
     }
 
-    this.logger.debug(`${this.uri} is done streaming message with id ${id.id}`)
     stream.end()
+
+    await handledPromise
+
+    this.openStreams -= 1
+
+    if (this.shouldClose.length > 0) await this.close()
   }
 
   async string(msg: string): Promise<void> {
     this.logger.debug(`${this.uri} sends string ${msg.length} characters`)
-    await this.write({
-      msg: { data: encoder.encode(msg), channel: this.uri },
+    const localSequenceNumber = this.localSequenceNumber++
+    const handledPromise = this.awaitProcessed()
+
+    await this.notifyOrchestrator({
+      msg: {
+        data: encoder.encode(msg),
+        channel: this.uri,
+        localSequenceNumber,
+      },
     })
+
+    await handledPromise
   }
 
+  /**
+   * Gracefully closes this channel.
+   *
+   * Behavior:
+   * - If there are still active streams, closing is deferred until they complete.
+   * - If multiple callers invoke `close()` while waiting, their Promises are queued and
+   *   resolved once the channel actually closes.
+   * - If this side initiated the close (`issued = false`), a close message is sent to the remote.
+   *
+   * @param issued - If true, indicates the close request originated remotely
+   */
   async close(issued = false): Promise<void> {
+    // Case 1: Active streams still running → wait until they finish
+    if (this.openStreams !== 0) {
+      await new Promise<void>((resolve) => this.shouldClose.push(resolve))
+      return
+    }
+
+    // Case 2: No active streams → perform actual close
     this.logger.debug(`${this.uri} closes stream`)
     if (!issued) {
-      await this.write({
+      await this.notifyOrchestrator({
         close: { channel: this.uri },
       })
     }
+
+    let resolve = this.shouldClose.pop()
+    while (resolve) {
+      resolve()
+      resolve = this.shouldClose.pop()
+    }
+  }
+
+  /**
+   * A message is handled, let's notify the fifo {@link awaitProcessed}
+   */
+  handled(): void {
+    if (this.awaitingProcessed.length > 0) {
+      this.awaitingProcessed.shift()!()
+    } else {
+      this.logger.error(
+        'Expected to be waiting for a message to be processed, but this is not the case ' +
+          this.uri,
+      )
+    }
   }
 }