@rdfc/js-runner 2.0.0-alpha.9 → 3.0.0

package/minimal.ttl

@@ -0,0 +1,99 @@
+@prefix prov: <http://www.w3.org/ns/prov#>.
+@prefix sds: <https://w3id.org/sds#>.
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
+@prefix owl: <http://www.w3.org/2002/07/owl#>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+@prefix rdfc: <https://w3id.org/rdf-connect#>.
+
+############################################################
+#                    General statements                    #
+############################################################
+# sds:Activity is a prov:Activity
+sds:Activity rdfs:subClassOf prov:Activity.
+
+# rdfc:Processor too
+rdfc:Processor rdfs:subClassOf sds:Activity.
+
+# sds:implementationOf is subClassOf
+sds:implementationOf rdfs:subPropertyOf rdfs:subClassOf.
+
+############################################################
+#                   Javascript statements                  #
+############################################################
+# specialized for js too
+rdfc:jsImplementationOf rdfs:subPropertyOf sds:implementationOf.
+
+# A node runner, runs things that are rdfc:jsImplementationOf rdfc:Processor  (aka, rdfs:subClassOf prov:Activity)
+rdfc:NodeRunner a rdfc:Runner;
+  rdfc:handlesSubjectsOf rdfc:jsImplementationOf;
+  rdfc:command "npx js-runner".
+
+# This shouldn't be necessary, should work with sh:targetSubjectsOf
+# rdfc:processor_definition <JsProcessorShape>.
+# 
+# Shape that a Js Processor should fulfil;
+[ ] a sh:NodeShape;
+  # This shouldn't be necessary,should work with sh:targetSubjectsOf and this isn't a real Class
+  sh:targetClass <JsProcessorShape>;
+  # We target it with jsImplementationOf
+  sh:targetSubjectsOf rdfc:jsImplementationOf;
+  sh:property [
+    sh:path rdfc:entrypoint;
+    sh:name "location";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:datatype xsd:iri;
+  ], [
+    sh:path rdfc:file;
+    sh:name "file";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:datatype xsd:iri;
+  ], [
+    sh:path rdfc:class;
+    sh:name "clazz";
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+  ].
+
+############################################################
+#                   Processor statements                   #
+############################################################
+rdfc:FooBarProcessor a owl:Class;
+  rdfs:label "My Epic FooBar Processor";
+  rdfs:description "FooBars everything!";
+  rdfc:jsImplementationOf rdfc:Processor;
+  rdfc:entrypoint <./>;
+  rdfc:file <./lib/processors.js>;
+  rdfc:class "FooBarProcessor".
+
+[ ] a sh:NodeShape;
+  sh:targetClass rdfc:FooBarProcessor;
+  sh:property [
+    sh:path rdfc:reader;
+    sh:name "reader";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:class rdfc:Reader;
+  ], [
+    sh:path rdfc:writer;
+    sh:name "writer";
+    sh:maxCount 1;
+    sh:class rdfc:Writer;
+  ].
+
+############################################################
+#                    Pipeline statements                   #
+############################################################
+<> a rdfc:Pipeline;
+  rdfc:consistsOf [
+    rdfc:instantiates rdfc:NodeRunner;
+    rdfc:processor <foobar>;
+  ].
+
+<incomingMessages> a rdfc:Reader, rdfc:Writer.
+<foobar> a rdfc:FooBarProcessor;
+  rdfc:reader <incomingMessages>;
+  rdfc:writer <incomingMessages>.
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rdfc/js-runner",
-  "version": "2.0.0-alpha.9",
+  "version": "3.0.0",
   "main": "lib/index.js",
   "type": "module",
   "scripts": {
@@ -31,13 +31,14 @@
   "devDependencies": {
     "@eslint/js": "^9.21.0",
     "@rdfjs/types": "^2.0.1",
+    "@grpc/grpc-js": "^1.12.6",
     "@types/jest": "^29.5.14",
     "@types/jsonld": "^1.5.15",
     "@types/n3": "^1.21.1",
     "@types/node": "^22.13.5",
     "@typescript-eslint/eslint-plugin": "^8.25.0",
     "@typescript-eslint/parser": "^8.25.0",
-    "@vitest/coverage-v8": "^3.0.7",
+    "@vitest/coverage-v8": "^3.2.4",
     "eslint": "^9.21.0",
     "eslint-config-prettier": "^10.0.1",
     "eslint-plugin-prettier": "^5.2.3",
@@ -50,11 +51,10 @@
     "tsc-alias": "^1.8.10",
     "typescript": "^5.7.3",
     "typescript-eslint": "^8.25.0",
-    "vitest": "^3.0.7"
+    "vitest": "^3.2.4"
   },
   "dependencies": {
-    "@grpc/grpc-js": "^1.12.6",
-    "@rdfc/proto": "^0.1.2-alpha.1",
+    "@rdfc/proto": "^0.1.2",
     "@treecg/types": "^0.4.6",
     "jsonld": "^8.3.3",
     "jsonld-streaming-parser": "^5.0.0",

package/src/client.ts

@@ -1,14 +1,14 @@
 import * as grpc from '@grpc/grpc-js'
 import { promisify } from 'util'
-import { RunnerClient, RunnerMessage } from '@rdfc/proto'
-import winston from 'winston'
+import { RunnerClient, ToRunner } from '@rdfc/proto'
+import { createLogger } from 'winston'
 import { RpcTransport } from './logger'
 import { Runner } from './runner'
 
 export async function start(addr: string, uri: string) {
   const client = new RunnerClient(addr, grpc.credentials.createInsecure())
 
-  const logger = winston.createLogger({
+  const logger = createLogger({
     transports: [
       new RpcTransport({
         entities: [uri, 'cli'],
@@ -25,28 +25,25 @@ export async function start(addr: string, uri: string) {
 
   await writable({ identify: { uri } })
 
-  let processorsEnd!: (v: unknown) => unknown
-  const processorsEnded = new Promise((res) => (processorsEnd = res))
-  ;(async () => {
+  /* eslint-disable no-async-promise-executor */
+  await new Promise(async (res) => {
     for await (const chunk of stream) {
-      const msg: RunnerMessage = chunk
+      const msg: ToRunner = chunk
       if (msg.proc) {
         await runner.addProcessor(msg.proc)
       }
       if (msg.start) {
-        runner.start().then(processorsEnd)
+        runner.start().then(res)
       }
 
       await runner.handleOrchMessage(msg)
     }
 
     logger.error('Stream ended')
-  })()
-
-  await processorsEnded
+  })
 
   logger.info('All processors are finished')
   stream.end()
   client.close()
-  process.exit(0)
+  setTimeout(() => process.exit(0), 500)
 }

package/src/logger.ts

@@ -1,4 +1,4 @@
-import winston, { Logger } from 'winston'
+import { createLogger, LogEntry, Logger } from 'winston'
 import Transport from 'winston-transport'
 
 import * as grpc from '@grpc/grpc-js'
@@ -21,7 +21,7 @@ export class RpcTransport extends Transport {
     this.aliases = opts.aliases || []
   }
 
-  log(info: winston.LogEntry, callback: () => void) {
+  log(info: LogEntry, callback: () => void) {
     if (!this.stream.closed) {
       this.stream.write(
         {
@@ -55,7 +55,7 @@ export function extendLogger(baseLogger: Logger, newEntity: string): Logger {
     return t
   })
 
-  return winston.createLogger({
+  return createLogger({
     level: baseLogger.level,
     format: baseLogger.format,
     defaultMeta: baseLogger.defaultMeta,

package/src/reader.ts

@@ -1,6 +1,11 @@
 import { ClientReadableStream } from '@grpc/grpc-js'
-import { DataChunk, Message, RunnerClient, StreamMessage } from '@rdfc/proto'
-import winston from 'winston'
+import {
+  DataChunk,
+  ReceivingMessage,
+  ReceivingStreamMessage,
+  RunnerClient,
+} from '@rdfc/proto'
+import { Logger } from 'winston'
 import {
   AnyConvertor,
   Convertor,
@@ -8,6 +13,8 @@ import {
   StreamConvertor,
   StringConvertor,
 } from './convertor'
+import { Writable } from './runner'
+import { promisify } from 'util'
 
 export type Any =
   | {
@@ -28,41 +35,47 @@ export interface Reader {
   anys(): AsyncIterable<Any>
 }
 
+type Todo<T> = {
+  item: T
+  onComplete: () => void
+}
+
 class MyIter<T> implements AsyncIterable<T> {
   private convertor: Convertor<T>
-  private queue: (T | undefined)[] = []
+  private queue: Todo<T | undefined>[] = []
   private resolveNext: ((value: undefined) => void) | null = null
 
   constructor(convertor: Convertor<T>) {
     this.convertor = convertor
   }
 
-  push(buffer: Uint8Array) {
+  push(buffer: Uint8Array, onComplete: () => void) {
     const item = this.convertor.from(buffer)
-    this.queue.push(item)
+    this.queue.push({ item, onComplete })
     if (this.resolveNext) {
       this.resolveNext(undefined)
       this.resolveNext = null
     }
   }
 
-  close() {
-    this.queue.push(undefined)
+  close(onComplete: () => void) {
+    this.queue.push({ item: undefined, onComplete })
     if (this.resolveNext) {
       this.resolveNext(undefined)
       this.resolveNext = null
     }
   }
 
-  async pushStream(chunks: ClientReadableStream<DataChunk>) {
+  async pushStream(chunks: AsyncIterable<DataChunk>, onComplete: () => void) {
+    // This is an async generator that transforms DataChunks to Buffers
     const stream = (async function* (stream) {
-      for await (const c of stream) {
-        const chunk: DataChunk = c
+      for await (const chunk of stream) {
         yield chunk.data
       }
     })(chunks)
+
     const item = await this.convertor.fromStream(stream)
-    this.queue.push(item)
+    this.queue.push({ item, onComplete })
     if (this.resolveNext) {
       this.resolveNext(undefined)
       this.resolveNext = null
@@ -72,9 +85,15 @@ class MyIter<T> implements AsyncIterable<T> {
   async *[Symbol.asyncIterator]() {
     while (true) {
       if (this.queue.length > 0) {
-        const item = this.queue.shift()!
-        if (item === undefined) break
+        const { item, onComplete } = this.queue.shift()!
+        if (item === undefined) {
+          onComplete()
+          break
+        }
         yield item
+        // Note: execution pauses at `yield` until the consumer calls `.next()` again.
+        // We call onComplete *after* resuming, so the producer knows the item was actually consumed.
+        onComplete()
       } else {
         await new Promise<undefined>((resolve) => (this.resolveNext = resolve))
       }
@@ -85,59 +104,217 @@ class MyIter<T> implements AsyncIterable<T> {
 export class ReaderInstance implements Reader {
   private client: RunnerClient
   readonly uri: string
-  private logger: winston.Logger
+  private logger: Logger
+  private readonly notifyOrchestrator: Writable
 
-  private iterators: MyIter<unknown>[] = []
+  private consumers: MyIter<unknown>[] = []
 
-  constructor(uri: string, client: RunnerClient, logger: winston.Logger) {
+  constructor(
+    uri: string,
+    client: RunnerClient,
+    notifyOrchestrator: Writable,
+    logger: Logger,
+  ) {
     this.uri = uri
     this.client = client
     this.logger = logger
+    this.notifyOrchestrator = notifyOrchestrator
   }
 
   anys(): AsyncIterable<Any> {
     const iter = new MyIter(AnyConvertor)
-    this.iterators.push(iter)
+    this.consumers.push(iter)
     return iter
   }
 
   strings(): AsyncIterable<string> {
     const iter = new MyIter(StringConvertor)
-    this.iterators.push(iter)
+    this.consumers.push(iter)
     return iter
   }
 
   buffers(): AsyncIterable<Uint8Array> {
     const iter = new MyIter(NoConvertor)
-    this.iterators.push(iter)
+    this.consumers.push(iter)
     return iter
   }
 
   streams(): AsyncIterable<AsyncGenerator<Uint8Array>> {
     const iter = new MyIter(StreamConvertor)
-    this.iterators.push(iter)
+    this.consumers.push(iter)
     return iter
   }
 
-  handleMsg(msg: Message) {
+  handleMsg(msg: ReceivingMessage) {
     this.logger.debug(`${this.uri} handling message`)
-    console.log(`${this.uri} handling message`)
-    for (const iter of this.iterators) {
-      iter.push(msg.data)
+
+    const promises = []
+    for (const iter of this.consumers) {
+      promises.push(new Promise((res) => iter.push(msg.data, () => res(null))))
     }
+
+    Promise.all(promises).then(() =>
+      this.notifyOrchestrator({
+        processed: {
+          globalSequenceNumber: msg.globalSequenceNumber,
+          channel: this.uri,
+        },
+      }),
+    )
   }
 
   close() {
-    for (const iter of this.iterators) {
-      iter.close()
+    for (const iter of this.consumers) {
+      iter.close(() => {})
     }
   }
 
-  handleStreamingMessage(msg: StreamMessage) {
+  // There is a stream message available for this reader
+  async handleStreamingMessage({
+    channel,
+    globalSequenceNumber,
+  }: ReceivingStreamMessage) {
     this.logger.debug(`${this.uri} handling streaming message`)
-    const chunks = this.client.receiveStreamMessage(msg.id!)
-    for (const iter of this.iterators) {
-      iter.pushStream(chunks)
+
+    const chunks = this.client.receiveStreamMessage()
+    const writeControlMessage = promisify(chunks.write.bind(chunks))
+    const consumersConsumed = []
+
+    // After each chunk is handled by all consumer, emit a processed message
+    let idx = 0
+    const messageIterators = fanoutStream(
+      chunks,
+      this.consumers.length,
+      async () => {
+        await writeControlMessage({ streamSequenceNumber: idx++ })
+      },
+    )
+
+    for (const consumer of this.consumers) {
+      consumersConsumed.push(
+        new Promise((res) =>
+          consumer.pushStream(messageIterators.pop()!, () => res(null)),
+        ),
+      )
+    }
+
+    await writeControlMessage({ globalSequenceNumber })
+
+    Promise.all(consumersConsumed).then(() => {
+      console.log('Writing processed for streaming message')
+      this.notifyOrchestrator({ processed: { globalSequenceNumber, channel } })
+    })
+  }
+}
+
+/**
+ * Helper function to tee a stream `numConsumers` times
+ * When each tee'd stream has handled a chunk, call {@link onAllHandled}
+ */
+function fanoutStream<T>(
+  stream: ClientReadableStream<T>,
+  numConsumers: number,
+  onAllHandled: () => void | Promise<void>,
+): AsyncIterable<T>[] {
+  type Waiter = (value: IteratorResult<T>) => void
+
+  let ended = false
+  const buffer: T[] = []
+  const pending: Waiter[] = []
+  let activeConsumers = numConsumers
+
+  // consumer bookkeeping
+  let awaitingAck = 0
+
+  function pushChunk(chunk: T) {
+    buffer.push(chunk)
+    flush()
+  }
+
+  function flush() {
+    while (buffer.length > 0 && pending.length > 0) {
+      const chunk = buffer[0] // keep until all consumers ack
+      const waiter = pending.shift()!
+      waiter({ value: chunk, done: false })
+      awaitingAck++
+    }
+  }
+
+  function end() {
+    ended = true
+    while (pending.length > 0) {
+      const waiter = pending.shift()!
+      waiter({ value: undefined, done: true })
     }
   }
+
+  stream.on('data', (chunk: T) => {
+    pushChunk(chunk)
+  })
+
+  stream.on('end', () => {
+    end()
+  })
+
+  stream.on('error', (err) => {
+    while (pending.length > 0) {
+      const waiter = pending.shift()!
+      waiter({ value: undefined, done: true })
+    }
+    throw err
+  })
+
+  function makeIterable(): AsyncIterable<T> {
+    return {
+      [Symbol.asyncIterator]() {
+        return {
+          next(): Promise<IteratorResult<T>> {
+            if (buffer.length > 0) {
+              const chunk = buffer[0]
+              awaitingAck++
+              return Promise.resolve({ value: chunk, done: false })
+            }
+            if (ended) {
+              return Promise.resolve({ value: undefined, done: true })
+            }
+            return new Promise((resolve) => {
+              pending.push(resolve)
+            })
+          },
+          async return() {
+            activeConsumers--
+            if (activeConsumers === 0) {
+              end()
+            }
+            return { value: undefined, done: true }
+          },
+        }
+      },
+    }
+  }
+
+  async function ack() {
+    awaitingAck--
+    if (awaitingAck === 0) {
+      // all consumers done with the current chunk
+      buffer.shift() // drop it
+      await onAllHandled()
+      flush() // continue with next chunk
+    }
+  }
+
+  // wrap consumer so they *must* call ack() after processing
+  function wrap(iterable: AsyncIterable<T>): AsyncIterable<T> {
+    return {
+      async *[Symbol.asyncIterator]() {
+        for await (const item of iterable) {
+          yield item
+          await ack()
+        }
+      },
+    }
+  }
+
+  const rawIterables = Array.from({ length: numConsumers }, makeIterable)
+  return rawIterables.map(wrap)
 }