@rdfc/js-runner 2.0.0 → 3.0.0

package/src/runner.ts

@@ -1,15 +1,18 @@
 import {
-  OrchestratorMessage,
+  Close,
+  FromRunner,
   Processor,
   RunnerClient,
-  RunnerMessage,
+  ToRunner,
+  LocalAck,
+  ReceivingMessage,
+  ReceivingStreamMessage,
 } from '@rdfc/proto'
 import { Reader, ReaderInstance } from './reader'
 import { Writer, WriterInstance } from './writer'
 import { Processor as Proc } from './processor'
-import { Logger } from 'winston'
+import { createLogger, Logger } from 'winston'
 
-import winston from 'winston'
 import { RpcTransport } from './logger'
 import { Cont, empty, extractShapes, Shapes } from 'rdf-lens'
 import { NamedNode, Parser } from 'n3'
@@ -36,7 +39,7 @@ const RDFC = createNamespace(
   'Writer',
 )
 
-export type Writable = (msg: OrchestratorMessage) => Promise<unknown>
+export type Writable = (msg: FromRunner) => Promise<unknown>
 
 type ProcessorConfig = {
   location: string
@@ -47,10 +50,10 @@ type ProcessorConfig = {
 export type FullProc<C extends Proc<unknown>> =
   C extends Proc<infer T> ? T & C : unknown
 export class Runner {
-  private readonly readers: { [uri: string]: ReaderInstance[] } = {}
-  private readonly writers: { [uri: string]: WriterInstance[] } = {}
+  private readonly readers: { [uri: string]: ReaderInstance } = {}
+  private readonly writers: { [uri: string]: WriterInstance } = {}
   private readonly client: RunnerClient
-  private readonly write: Writable
+  private readonly notifyOrchestrator: Writable
   private readonly logger: Logger
   private shapes: Shapes
   private quads: Quad[] = []
@@ -62,12 +65,12 @@ export class Runner {
 
   constructor(
     client: RunnerClient,
-    write: Writable,
+    notifyOrchestrator: Writable,
     uri: string,
     logger: Logger,
   ) {
     this.client = client
-    this.write = write
+    this.notifyOrchestrator = notifyOrchestrator
     this.uri = uri
     this.logger = logger
   }
@@ -75,7 +78,7 @@ export class Runner {
   async addProcessor<P extends Proc<unknown>>(
     proc: Processor,
   ): Promise<FullProc<P>> {
-    const procLogger = winston.createLogger({
+    const procLogger = createLogger({
       transports: [
         new RpcTransport({
           entities: [proc.uri, this.uri],
@@ -84,14 +87,13 @@ export class Runner {
       ],
     })
 
-    const ty = JSON.stringify(
-      this.quads
-        .filter(
-          (x) =>
-            x.subject.value === proc.uri && x.predicate.equals(RDF.terms.type),
-        )
-        .map((x) => x.object.value),
-    )
+    const ty = this.quads
+      .filter(
+        (x) =>
+          x.subject.value === proc.uri && x.predicate.equals(RDF.terms.type),
+      )
+      .map((x) => x.object.value)
+
     this.logger.info('parsing ' + proc.uri + ' type ' + ty)
     const args = this.shapes.lenses[RDFL.TypedExtract].execute({
       id: new NamedNode(proc.uri),
@@ -99,8 +101,6 @@ export class Runner {
     })
 
     const config: ProcessorConfig = JSON.parse(proc.config)
-    // const url = new URL(config.location)
-    // process.chdir(url.pathname)
     const jsProgram = await import(config.file)
     const clazz = jsProgram[config.clazz || 'default']
     const instance: Proc<unknown> = new clazz(args, procLogger)
@@ -111,7 +111,7 @@ export class Runner {
     this.processors.push(instance)
     this.processorTransforms.push(instance.transform())
 
-    await this.write({ init: { uri: proc.uri } })
+    await this.notifyOrchestrator({ initialized: { uri: proc.uri } })
 
     return <FullProc<P>>instance
   }
@@ -126,72 +126,135 @@ export class Runner {
   }
 
   createWriter(uri: Term): Writer {
-    const ids = uri.value
+    const id = uri.value
 
-    if (this.writers[ids] === undefined) {
-      this.writers[ids] = []
+    if (this.writers[id] !== undefined) {
+      return this.writers[id]
     }
-    const writer = new WriterInstance(ids, this.client, this.write, this.logger)
-    this.writers[ids].push(writer)
+    const writer = new WriterInstance(
+      id,
+      this.client,
+      this.notifyOrchestrator,
+      this.uri,
+      this.logger,
+    )
+    this.writers[id] = writer
     return writer
   }
 
   createReader(uri: Term): Reader {
     const ids = uri.value
 
-    if (this.readers[ids] === undefined) {
-      this.readers[ids] = []
+    if (this.readers[ids] !== undefined) {
+      return this.readers[ids]
     }
-    const reader = new ReaderInstance(ids, this.client, this.logger)
-    this.readers[ids].push(reader)
+    const reader = new ReaderInstance(
+      ids,
+      this.client,
+      this.notifyOrchestrator,
+      this.logger,
+    )
+    this.readers[ids] = reader
     return reader
   }
 
-  async handleOrchMessage(msg: RunnerMessage) {
+  async handleOrchMessage(msg: ToRunner) {
     if (msg.msg) {
-      this.logger.debug('Handling data msg for ' + msg.msg.channel)
-      for (const reader of this.readers[msg.msg.channel] || []) {
-        reader.handleMsg(msg.msg)
-      }
+      this.handleMsg(msg.msg)
     }
 
     if (msg.streamMsg) {
-      for (const reader of this.readers[msg.streamMsg.channel] || []) {
-        reader.handleStreamingMessage(msg.streamMsg)
-      }
+      await this.handleStreamMsg(msg.streamMsg)
     }
 
     if (msg.close) {
-      const uri = msg.close.channel
+      await this.handleClose(msg.close)
+    }
 
-      for (const reader of this.readers[uri] || []) {
-        reader.close()
-      }
+    if (msg.pipeline) {
+      this.handlePipeline(msg.pipeline)
+    }
 
-      for (const writer of this.writers[uri] || []) {
-        await writer.close(true)
-      }
+    if (msg.processed) {
+      this.handleProcessed(msg.processed)
     }
+  }
 
-    if (msg.pipeline) {
-      try {
-        // here
-        const quads = new Parser().parse(msg.pipeline)
-        this.shapes = extractShapes(
-          quads,
-          {
-            [RDFC.Reader]: (x: Cont) => this.createReader(x.id),
-            [RDFC.Writer]: (x: Cont) => this.createWriter(x.id),
-          },
-          {
-            [RDFC.Reader]: empty<Cont>(),
-            [RDFC.Writer]: empty<Cont>(),
-          },
-        )
-        this.quads = quads
-      } catch (ex: unknown) {
-        this.logger.error('Pipeline failed: ' + JSON.stringify(ex))
-      }
+  private async handleClose(close: Close) {
+    const uri = close.channel
+    const r = this.readers[uri]
+
+    let closed = false
+    if (r) {
+      r.close()
+      closed = true
+    }
+    const w = this.writers[uri]
+    if (w) {
+      closed = true
+      await w.close(true)
+    }
+
+    if (!closed) {
+      this.logger.error(
+        `Received a close event for channel ${uri}, but neither reader nor writer is present.`,
+      )
+    }
+  }
+
+  private handlePipeline(pipeline: string) {
+    try {
+      const quads = new Parser().parse(pipeline)
+      this.shapes = extractShapes(
+        quads,
+        {
+          [RDFC.Reader]: (x: Cont) => this.createReader(x.id),
+          [RDFC.Writer]: (x: Cont) => this.createWriter(x.id),
+        },
+        {
+          [RDFC.Reader]: empty<Cont>(),
+          [RDFC.Writer]: empty<Cont>(),
+        },
+      )
+      this.quads = quads
+    } catch (ex: unknown) {
+      this.logger.error('Pipeline failed: ' + JSON.stringify(ex))
+    }
+  }
+
+  private handleMsg(msg: ReceivingMessage) {
+    this.logger.debug('Handling data msg for ' + msg.channel)
+    const r = this.readers[msg.channel]
+
+    if (r) {
+      r.handleMsg(msg)
+    } else {
+      this.logger.error(
+        `Received message for channel ${msg.channel}, but no reader was present.`,
+      )
+    }
+  }
+
+  private async handleStreamMsg(streamMsg: ReceivingStreamMessage) {
+    const r = this.readers[streamMsg.channel]
+
+    if (r) {
+      await r.handleStreamingMessage(streamMsg)
+    } else {
+      this.logger.error(
+        `Received stream message for channel ${streamMsg.channel}, but no reader was present.`,
+      )
+    }
+  }
+
+  private handleProcessed(processed: LocalAck) {
+    const writer = this.writers[processed.channel]
+    if (writer) {
+      writer.handled()
+    } else {
+      this.logger.error(
+        `Received processed message for channel ${processed.channel}, but no writer was present.`,
+      )
     }
   }
 }

package/src/testUtils/duplex.ts

@@ -0,0 +1,112 @@
+import { Duplex } from 'stream'
+import type {
+  ClientDuplexStream,
+  InterceptingCallInterface,
+} from '@grpc/grpc-js'
+import { AuthContext } from '@grpc/grpc-js/build/src/auth-context'
+
+type Matcher<Req, T> = (req: Req) => T | undefined
+type Handler<Req, Res> = (req: Req, send: (res: Res) => void) => void
+
+type MatchObject<Req, Res, T> = {
+  matcher: Matcher<Req, T>
+  handler: Handler<T, Res>
+}
+
+let count = 0
+export class MockClientDuplexStream<Req, Res>
+  extends Duplex
+  implements ClientDuplexStream<Req, Res>
+{
+  private capabilities: Array<MatchObject<Req, Res, unknown>> = []
+  private onceCapabilities: Array<MatchObject<Req, Res, unknown>> = []
+
+  public readonly id: number
+
+  call?: InterceptingCallInterface | undefined
+  constructor() {
+    super({ objectMode: true })
+    this.id = count++
+  }
+
+  getAuthContext(): AuthContext | null {
+    return null
+  }
+
+  // ---- SurfaceCall stubs ----
+  cancel(): void {}
+  getPeer(): string {
+    return 'mock-peer'
+  }
+
+  // ---- ObjectWritable<Req> ----
+  _write(
+    chunk: Req,
+    _encoding: BufferEncoding,
+    callback: (error?: Error | null) => void,
+  ): void {
+    let handled = false
+    // check registered capabilities
+    for (const { matcher, handler } of this.capabilities) {
+      const o = matcher(chunk)
+      if (o !== undefined) {
+        handled = true
+        handler(o, (res) => this.send(res))
+      }
+    }
+
+    const newOnce: typeof this.onceCapabilities = []
+    for (const { matcher, handler } of this.onceCapabilities) {
+      const o = matcher(chunk)
+      if (o !== undefined) {
+        handled = true
+        handler(o, (res) => this.send(res))
+      } else {
+        newOnce.push({ matcher, handler })
+      }
+    }
+
+    this.onceCapabilities = newOnce
+    if (!handled) {
+      console.error('Unhandled!', Object.keys(chunk as object))
+    }
+
+    callback()
+  }
+
+  // ---- ObjectReadable<Res> ----
+  _read(): void {
+    // no-op: we push manually with send()
+  }
+
+  // ---- gRPC-style helpers ----
+  serialize(value: Req): Buffer {
+    return Buffer.from(JSON.stringify(value))
+  }
+
+  deserialize(chunk: Buffer): Res {
+    return JSON.parse(chunk.toString())
+  }
+
+  send(response: Res): void {
+    this.push(response)
+  }
+
+  end(): this {
+    this.push(null)
+    return this
+  }
+
+  // ---- Capability registration ----
+  register<T>(matcher: Matcher<Req, T>, handler: Handler<T, Res>): void {
+    this.capabilities.push({ matcher, handler })
+  }
+
+  registerOnce<T>(matcher: Matcher<Req, T>, handler: Handler<T, Res>): void {
+    this.onceCapabilities.push({ matcher, handler })
+  }
+
+  awaitMsg<T>(matcher: Matcher<Req, T>): Promise<T> {
+    return new Promise((res) => this.registerOnce(matcher, res))
+  }
+}