@rdfc/js-runner 2.0.0-alpha.1 → 2.0.0-alpha.11

package/minimal.ttl

@@ -0,0 +1,99 @@
+@prefix prov: <http://www.w3.org/ns/prov#>.
+@prefix sds: <https://w3id.org/sds#>.
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
+@prefix owl: <http://www.w3.org/2002/07/owl#>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+@prefix rdfc: <https://w3id.org/rdf-connect#>.
+
+############################################################
+#                    General statements                    #
+############################################################
+# sds:Activity is a prov:Activity
+sds:Activity rdfs:subClassOf prov:Activity.
+
+# rdfc:Processor too
+rdfc:Processor rdfs:subClassOf sds:Activity.
+
+# sds:implementationOf is subClassOf
+sds:implementationOf rdfs:subPropertyOf rdfs:subClassOf.
+
+############################################################
+#                   Javascript statements                  #
+############################################################
+# specialized for js too
+rdfc:jsImplementationOf rdfs:subPropertyOf sds:implementationOf.
+
+# A node runner, runs things that are rdfc:jsImplementationOf rdfc:Processor  (aka, rdfs:subClassOf prov:Activity)
+rdfc:NodeRunner a rdfc:Runner;
+  rdfc:handlesSubjectsOf rdfc:jsImplementationOf;
+  rdfc:command "npx js-runner".
+
+# This shouldn't be necessary, should work with sh:targetSubjectsOf
+# rdfc:processor_definition <JsProcessorShape>.
+# 
+# Shape that a Js Processor should fulfil;
+[ ] a sh:NodeShape;
+  # This shouldn't be necessary,should work with sh:targetSubjectsOf and this isn't a real Class
+  sh:targetClass <JsProcessorShape>;
+  # We target it with jsImplementationOf
+  sh:targetSubjectsOf rdfc:jsImplementationOf;
+  sh:property [
+    sh:path rdfc:entrypoint;
+    sh:name "location";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:datatype xsd:iri;
+  ], [
+    sh:path rdfc:file;
+    sh:name "file";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:datatype xsd:iri;
+  ], [
+    sh:path rdfc:class;
+    sh:name "clazz";
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+  ].
+
+############################################################
+#                   Processor statements                   #
+############################################################
+rdfc:FooBarProcessor a owl:Class;
+  rdfs:label "My Epic FooBar Processor";
+  rdfs:description "FooBars everything!";
+  rdfc:jsImplementationOf rdfc:Processor;
+  rdfc:entrypoint <./>;
+  rdfc:file <./lib/processors.js>;
+  rdfc:class "FooBarProcessor".
+
+[ ] a sh:NodeShape;
+  sh:targetClass rdfc:FooBarProcessor;
+  sh:property [
+    sh:path rdfc:reader;
+    sh:name "reader";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:class rdfc:Reader;
+  ], [
+    sh:path rdfc:writer;
+    sh:name "writer";
+    sh:maxCount 1;
+    sh:class rdfc:Writer;
+  ].
+
+############################################################
+#                    Pipeline statements                   #
+############################################################
+<> a rdfc:Pipeline;
+  rdfc:consistsOf [
+    rdfc:instantiates rdfc:NodeRunner;
+    rdfc:processor <foobar>;
+  ].
+
+<incomingMessages> a rdfc:Reader, rdfc:Writer.
+<foobar> a rdfc:FooBarProcessor;
+  rdfc:reader <incomingMessages>;
+  rdfc:writer <incomingMessages>.
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rdfc/js-runner",
-  "version": "2.0.0-alpha.1",
+  "version": "2.0.0-alpha.11",
   "main": "lib/index.js",
   "type": "module",
   "scripts": {
@@ -37,7 +37,7 @@
     "@types/node": "^22.13.5",
     "@typescript-eslint/eslint-plugin": "^8.25.0",
     "@typescript-eslint/parser": "^8.25.0",
-    "@vitest/coverage-v8": "^3.0.7",
+    "@vitest/coverage-v8": "^3.2.4",
     "eslint": "^9.21.0",
     "eslint-config-prettier": "^10.0.1",
     "eslint-plugin-prettier": "^5.2.3",
@@ -50,7 +50,7 @@
     "tsc-alias": "^1.8.10",
     "typescript": "^5.7.3",
     "typescript-eslint": "^8.25.0",
-    "vitest": "^3.0.7"
+    "vitest": "^3.2.4"
   },
   "dependencies": {
     "@grpc/grpc-js": "^1.12.6",

package/src/client.ts

@@ -12,7 +12,7 @@ export async function start(addr: string, uri: string) {
     transports: [
       new RpcTransport({
         entities: [uri, 'cli'],
-        stream: client.logStream(() => {}),
+        stream: client.logStream(() => { }),
       }),
     ],
   })
@@ -27,26 +27,26 @@ export async function start(addr: string, uri: string) {
 
   let processorsEnd!: (v: unknown) => unknown
   const processorsEnded = new Promise((res) => (processorsEnd = res))
-  ;(async () => {
-    for await (const chunk of stream) {
-      const msg: RunnerMessage = chunk
-      if (msg.proc) {
-        await runner.addProcessor(msg.proc)
+    ; (async () => {
+      for await (const chunk of stream) {
+        const msg: RunnerMessage = chunk
+        if (msg.proc) {
+          await runner.addProcessor(msg.proc)
+        }
+        if (msg.start) {
+          runner.start().then(processorsEnd)
+        }
+
+        await runner.handleOrchMessage(msg)
       }
-      if (msg.start) {
-        runner.start().then(processorsEnd)
-      }
-
-      await runner.handleOrchMessage(msg)
-    }
 
-    logger.error('Stream ended')
-  })()
+      logger.error('Stream ended')
+    })()
 
   await processorsEnded
 
   logger.info('All processors are finished')
   stream.end()
   client.close()
-  process.exit(0)
+  setTimeout(() => process.exit(0), 500)
 }

package/src/index.ts

@@ -2,3 +2,7 @@ export * from './client'
 export * from './writer'
 export * from './reader'
 export * from './processor'
+export * from './runner'
+export * from './logger'
+
+export * as reexports from './reexports'

package/src/jsonld.ts

@@ -30,7 +30,7 @@ const RDFL = createUriAndTermNamespace(
 
 type Writable = (msg: OrchestratorMessage) => Promise<unknown>
 const RDFC = createNamespace(
-  'https://w3id.org/rdf-connect/ontology#',
+  'https://w3id.org/rdf-connect#',
   (x) => x,
   'Reader',
   'Writer',
@@ -189,7 +189,10 @@ function expandArgs(args: Arg, cont: Cont, logger: Logger) {
             let x = 'object'
             try {
               x = JSON.stringify(v)
-            } catch (e) {}
+              // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            } catch (_e: unknown) {
+              // default is set
+            }
 
             return k + ': ' + x
           }),

package/src/logger.ts

@@ -1,12 +1,14 @@
-import winston from 'winston'
+import winston, { Logger } from 'winston'
 import Transport from 'winston-transport'
 
 import * as grpc from '@grpc/grpc-js'
 import { LogMessage } from '@rdfc/proto'
+
 export class RpcTransport extends Transport {
   private readonly stream: grpc.ClientWritableStream<LogMessage>
   private readonly entities: string[]
   private readonly aliases: string[]
+
   constructor(opts: {
     stream: grpc.ClientWritableStream<LogMessage>
     entities: string[]
@@ -35,4 +37,28 @@ export class RpcTransport extends Transport {
       callback()
     }
   }
+
+  withEntity(entity: string): RpcTransport {
+    return new RpcTransport({
+      stream: this.stream,
+      entities: [...this.entities, entity],
+      aliases: this.aliases,
+    })
+  }
+}
+
+export function extendLogger(baseLogger: Logger, newEntity: string): Logger {
+  const newTransports = baseLogger.transports.map((t) => {
+    if (t instanceof RpcTransport) {
+      return t.withEntity(newEntity)
+    }
+    return t
+  })
+
+  return winston.createLogger({
+    level: baseLogger.level,
+    format: baseLogger.format,
+    defaultMeta: baseLogger.defaultMeta,
+    transports: newTransports,
+  })
 }

package/src/reader.ts

@@ -21,6 +21,7 @@ export type Any =
     }
 
 export interface Reader {
+  readonly uri: string
   strings(): AsyncIterable<string>
   streams(): AsyncIterable<AsyncGenerator<Uint8Array>>
   buffers(): AsyncIterable<Uint8Array>
@@ -30,7 +31,7 @@ export interface Reader {
 class MyIter<T> implements AsyncIterable<T> {
   private convertor: Convertor<T>
   private queue: (T | undefined)[] = []
-  private resolveNext: ((value: T | undefined) => void) | null = null
+  private resolveNext: ((value: undefined) => void) | null = null
 
   constructor(convertor: Convertor<T>) {
     this.convertor = convertor
@@ -38,20 +39,18 @@ class MyIter<T> implements AsyncIterable<T> {
 
   push(buffer: Uint8Array) {
     const item = this.convertor.from(buffer)
+    this.queue.push(item)
     if (this.resolveNext) {
-      this.resolveNext(item)
+      this.resolveNext(undefined)
       this.resolveNext = null
-    } else {
-      this.queue.push(item)
     }
   }
 
   close() {
+    this.queue.push(undefined)
     if (this.resolveNext) {
       this.resolveNext(undefined)
       this.resolveNext = null
-    } else {
-      this.queue.push(undefined)
     }
   }
 
@@ -63,11 +62,10 @@ class MyIter<T> implements AsyncIterable<T> {
       }
     })(chunks)
     const item = await this.convertor.fromStream(stream)
+    this.queue.push(item)
     if (this.resolveNext) {
-      this.resolveNext(item)
+      this.resolveNext(undefined)
       this.resolveNext = null
-    } else {
-      this.queue.push(item)
     }
   }
 
@@ -78,11 +76,7 @@ class MyIter<T> implements AsyncIterable<T> {
         if (item === undefined) break
         yield item
       } else {
-        const item = await new Promise<T | undefined>(
-          (resolve) => (this.resolveNext = resolve),
-        )
-        if (item === undefined) break
-        yield item
+        await new Promise<undefined>((resolve) => (this.resolveNext = resolve))
       }
     }
   }
@@ -90,7 +84,7 @@ class MyIter<T> implements AsyncIterable<T> {
 
 export class ReaderInstance implements Reader {
   private client: RunnerClient
-  private uri: string
+  readonly uri: string
   private logger: winston.Logger
 
   private iterators: MyIter<unknown>[] = []

package/src/reexports.ts

@@ -0,0 +1,6 @@
+export * from '@rdfc/proto'
+export {
+  ClientReadableStreamImpl,
+  ClientReadableStream,
+} from '@grpc/grpc-js/build/src/call.js'
+export * as grpc from '@grpc/grpc-js'

package/src/runner.ts

@@ -11,7 +11,6 @@ import { Logger } from 'winston'
 
 import winston from 'winston'
 import { RpcTransport } from './logger'
-import { parse_jsonld } from './jsonld'
 import { Cont, empty, extractShapes, Shapes } from 'rdf-lens'
 import { NamedNode, Parser } from 'n3'
 import { createNamespace, createUriAndTermNamespace, RDF } from '@treecg/types'
@@ -31,13 +30,13 @@ const RDFL = createUriAndTermNamespace(
 )
 
 const RDFC = createNamespace(
-  'https://w3id.org/rdf-connect/ontology#',
+  'https://w3id.org/rdf-connect#',
   (x) => x,
   'Reader',
   'Writer',
 )
 
-type Writable = (msg: OrchestratorMessage) => Promise<unknown>
+export type Writable = (msg: OrchestratorMessage) => Promise<unknown>
 
 type ProcessorConfig = {
   location: string
@@ -45,6 +44,8 @@ type ProcessorConfig = {
   clazz?: string
 }
 
+export type FullProc<C extends Proc<unknown>> =
+  C extends Proc<infer T> ? T & C : unknown
 export class Runner {
   private readonly readers: { [uri: string]: ReaderInstance[] } = {}
   private readonly writers: { [uri: string]: WriterInstance[] } = {}
@@ -71,69 +72,72 @@ export class Runner {
     this.logger = logger
   }
 
-  async addProcessor(proc: Processor) {
+  async addProcessor<P extends Proc<unknown>>(
+    proc: Processor,
+  ): Promise<FullProc<P>> {
     const procLogger = winston.createLogger({
       transports: [
         new RpcTransport({
           entities: [proc.uri, this.uri],
-          stream: this.client.logStream(() => { }),
+          stream: this.client.logStream(() => {}),
         }),
       ],
     })
 
-    const ty = JSON.stringify(this.quads.filter(x => x.subject.value === proc.uri && x.predicate.equals(RDF.terms.type)).map(x => x.object.value))
-    this.logger.info("parsing " + proc.uri + " type " + ty)
+    const ty = JSON.stringify(
+      this.quads
+        .filter(
+          (x) =>
+            x.subject.value === proc.uri && x.predicate.equals(RDF.terms.type),
+        )
+        .map((x) => x.object.value),
+    )
+    this.logger.info('parsing ' + proc.uri + ' type ' + ty)
     const args = this.shapes.lenses[RDFL.TypedExtract].execute({
       id: new NamedNode(proc.uri),
       quads: this.quads,
     })
-    // this.logger.info('Parsed ' + JSON.stringify(args))
-
-    // const args = await parse_jsonld(proc.arguments, procLogger, {
-    //   client: this.client,
-    //   write: this.write,
-    //   readers: this.readers,
-    //   writers: this.writers,
-    // })
 
     const config: ProcessorConfig = JSON.parse(proc.config)
-    const url = new URL(config.location)
-    process.chdir(url.pathname)
+    // const url = new URL(config.location)
+    // process.chdir(url.pathname)
     const jsProgram = await import(config.file)
     const clazz = jsProgram[config.clazz || 'default']
     const instance: Proc<unknown> = new clazz(args, procLogger)
     await instance.init()
 
-    this.logger.info("inited " + proc.uri + " type " + ty)
-    await this.write({ init: { uri: proc.uri } })
+    this.logger.info('inited ' + proc.uri + ' type ' + ty)
 
     this.processors.push(instance)
     this.processorTransforms.push(instance.transform())
+
+    await this.write({ init: { uri: proc.uri } })
+
+    return <FullProc<P>>instance
   }
 
   async start() {
-    await Promise.all(this.processors.map((x) => x.produce()))
-    await Promise.all(this.processorTransforms)
+    try {
+      await Promise.all(this.processors.map((x) => x.produce()))
+      await Promise.all(this.processorTransforms)
+    } catch (ex: unknown) {
+      this.logger.error('Start failed: ' + JSON.stringify(ex))
+    }
   }
 
   createWriter(uri: Term): Writer {
-    const ids = uri.value;
+    const ids = uri.value
 
     if (this.writers[ids] === undefined) {
       this.writers[ids] = []
     }
-    const writer = new WriterInstance(
-      ids,
-      this.client,
-      this.write,
-      this.logger,
-    )
+    const writer = new WriterInstance(ids, this.client, this.write, this.logger)
     this.writers[ids].push(writer)
     return writer
   }
 
   createReader(uri: Term): Reader {
-    const ids = uri.value;
+    const ids = uri.value
 
     if (this.readers[ids] === undefined) {
       this.readers[ids] = []
@@ -170,17 +174,24 @@ export class Runner {
     }
 
     if (msg.pipeline) {
-      // here
-      const quads = new Parser().parse(msg.pipeline);
-      this.shapes = extractShapes(quads, {
-        [RDFC.Reader]: (x: Cont) => this.createReader(x.id),
-        [RDFC.Writer]: (x: Cont) => this.createWriter(x.id)
-      }, {
-        [RDFC.Reader]: empty<Cont>(),
-        [RDFC.Writer]: empty<Cont>()
-      });
-      this.quads = quads;
-      this.logger.info("extracted shapes " + JSON.stringify(Object.keys(this.shapes.lenses)));
+      try {
+        // here
+        const quads = new Parser().parse(msg.pipeline)
+        this.shapes = extractShapes(
+          quads,
+          {
+            [RDFC.Reader]: (x: Cont) => this.createReader(x.id),
+            [RDFC.Writer]: (x: Cont) => this.createWriter(x.id),
+          },
+          {
+            [RDFC.Reader]: empty<Cont>(),
+            [RDFC.Writer]: empty<Cont>(),
+          },
+        )
+        this.quads = quads
+      } catch (ex: unknown) {
+        this.logger.error('Pipeline failed: ' + JSON.stringify(ex))
+      }
     }
   }
 }

package/src/testUtils.ts

@@ -0,0 +1,196 @@
+import { DataChunk, RunnerClient } from '@rdfc/proto'
+import * as grpc from '@grpc/grpc-js'
+import { ClientReadableStream } from '@grpc/grpc-js'
+import { ClientReadableStreamImpl, OrchestratorMessage } from './reexports'
+import { extractShapes } from 'rdf-lens'
+import { NamedNode, Parser, Writer } from 'n3'
+import { readFile } from 'fs/promises'
+import winston, { createLogger } from 'winston'
+import { WriterInstance } from './writer'
+import { ReaderInstance } from './reader'
+import { Processor } from './processor'
+import { FullProc, Runner } from './runner'
+import { Quad, Term } from '@rdfjs/types'
+import { createTermNamespace } from '@treecg/types'
+import { expect } from 'vitest'
+
+export async function getProcessorShape(
+  baseIRI = process.cwd() + '/node_modules/@rdfc/js-runner/index.ttl',
+) {
+  const configFile = await readFile(baseIRI, { encoding: 'utf8' })
+  const configQuads = new Parser().parse(configFile)
+  const shapes = extractShapes(configQuads)
+
+  return shapes
+}
+
+export class TestClient extends RunnerClient {
+  next: (stream: ClientReadableStream<DataChunk>) => unknown
+
+  constructor() {
+    super('localhost:5400', grpc.credentials.createInsecure())
+  }
+
+  nextStream(): Promise<ClientReadableStream<DataChunk>> {
+    return new Promise((res) => (this.next = res))
+  }
+
+  receiveStreamMessage(): ClientReadableStream<DataChunk> {
+    const stream = new ClientReadableStreamImpl<DataChunk>((data: Buffer) => {
+      return { data }
+    })
+    this.next(stream)
+    return stream
+  }
+}
+
+export async function one<T>(iter: AsyncIterable<T>): Promise<T | undefined> {
+  for await (const item of iter) {
+    return item
+  }
+}
+
+export const client = new TestClient()
+export const uri = 'someUri'
+export const logger = createLogger({
+  transports: new winston.transports.Console({
+    level: process.env['DEBUG'] || 'info',
+  }),
+})
+
+export function createWriter(iri = uri): [WriterInstance, ReaderInstance] {
+  const reader = createReader(iri)
+  const writeStream = new WriterInstance(
+    iri,
+    client,
+    async (msg) => {
+      if (msg.msg) {
+        reader.handleMsg(msg.msg)
+      }
+      if (msg.close) {
+        reader.close()
+      }
+    },
+    logger,
+  )
+  return [writeStream, reader]
+}
+
+export function createReader(iri = uri): ReaderInstance {
+  const reader = new ReaderInstance(iri, client, logger)
+  return reader
+}
+
+const shapeQuads = `
+@prefix rdfc: <https://w3id.org/rdf-connect#>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+[ ] a sh:NodeShape;
+  sh:targetClass <JsProcessorShape>;
+  sh:property [
+    sh:path rdfc:entrypoint;
+    sh:name "location";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+  ], [
+    sh:path rdfc:file;
+    sh:name "file";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+  ], [
+    sh:path rdfc:class;
+    sh:name "clazz";
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+  ].
+`
+const OWL = createTermNamespace('http://www.w3.org/2002/07/owl#', 'imports')
+const processorShapes = extractShapes(new Parser().parse(shapeQuads))
+const base = 'https://w3id.org/rdf-connect#'
+
+export async function importFile(file: string): Promise<Quad[]> {
+  const done = new Set<string>()
+  const todo = [new URL('file://' + file)]
+  const quads: Quad[] = []
+
+  let item = todo.pop()
+  while (item !== undefined) {
+    if (done.has(item.toString())) {
+      item = todo.pop()
+      continue
+    }
+    done.add(item.toString())
+    if (item.protocol !== 'file:') {
+      throw 'No supported protocol ' + item.protocol
+    }
+
+    const txt = await readFile(item.pathname, { encoding: 'utf8' })
+    const extras = new Parser({ baseIRI: item.toString() }).parse(txt)
+
+    for (const o of extras
+      .filter(
+        (x) =>
+          x.subject.value === item?.toString() &&
+          x.predicate.equals(OWL.imports),
+      )
+      .map((x) => x.object.value)) {
+      todo.push(new URL(o))
+    }
+    quads.push(...extras)
+
+    item = todo.pop()
+  }
+
+  return quads
+}
+
+export async function getProc<T extends Processor<unknown>>(
+  config: string,
+  ty: string,
+  configLocation: string,
+  uri = 'http://example.com/ns#processor',
+): Promise<FullProc<T>> {
+  const configQuads = await importFile(configLocation)
+  const procConfig = processorShapes.lenses['JsProcessorShape'].execute({
+    id: new NamedNode(base + ty),
+    quads: configQuads,
+  })
+
+  const msgs: OrchestratorMessage[] = []
+  const write = async (x: OrchestratorMessage) => {
+    msgs.push(x)
+  }
+  const runner = new Runner(
+    new TestClient(),
+    write,
+    'http://example.com/ns#',
+    logger,
+  )
+  configQuads.push(...new Parser().parse(config))
+  await runner.handleOrchMessage({
+    pipeline: new Writer().quadsToString(configQuads),
+  })
+
+  const proc = await runner.addProcessor<T>({
+    config: JSON.stringify(procConfig),
+    arguments: '',
+    uri,
+  })
+
+  return proc
+}
+
+export async function checkProcDefinition(file: string, n: string) {
+  const quads = await importFile(file)
+  const procConfig = <{ file: Term; location: string; clazz: string }>(
+    processorShapes.lenses['JsProcessorShape'].execute({
+      id: new NamedNode(base + n),
+      quads: quads,
+    })
+  )
+  expect(procConfig.file, n + ' has file').toBeDefined()
+  expect(procConfig.location, n + ' has location').toBeDefined()
+  expect(procConfig.clazz, n + ' has clazz').toBeDefined()
+}