@rdfc/js-runner 2.0.0-alpha.1 → 2.0.0-alpha.10

package/src/logger.ts

@@ -1,12 +1,14 @@
-import winston from 'winston'
+import winston, { Logger } from 'winston'
 import Transport from 'winston-transport'
 
 import * as grpc from '@grpc/grpc-js'
 import { LogMessage } from '@rdfc/proto'
+
 export class RpcTransport extends Transport {
   private readonly stream: grpc.ClientWritableStream<LogMessage>
   private readonly entities: string[]
   private readonly aliases: string[]
+
   constructor(opts: {
     stream: grpc.ClientWritableStream<LogMessage>
     entities: string[]
@@ -35,4 +37,28 @@ export class RpcTransport extends Transport {
       callback()
     }
   }
+
+  withEntity(entity: string): RpcTransport {
+    return new RpcTransport({
+      stream: this.stream,
+      entities: [...this.entities, entity],
+      aliases: this.aliases,
+    })
+  }
+}
+
+export function extendLogger(baseLogger: Logger, newEntity: string): Logger {
+  const newTransports = baseLogger.transports.map((t) => {
+    if (t instanceof RpcTransport) {
+      return t.withEntity(newEntity)
+    }
+    return t
+  })
+
+  return winston.createLogger({
+    level: baseLogger.level,
+    format: baseLogger.format,
+    defaultMeta: baseLogger.defaultMeta,
+    transports: newTransports,
+  })
 }

package/src/reader.ts

@@ -21,6 +21,7 @@ export type Any =
     }
 
 export interface Reader {
+  readonly uri: string
   strings(): AsyncIterable<string>
   streams(): AsyncIterable<AsyncGenerator<Uint8Array>>
   buffers(): AsyncIterable<Uint8Array>
@@ -30,7 +31,7 @@ export interface Reader {
 class MyIter<T> implements AsyncIterable<T> {
   private convertor: Convertor<T>
   private queue: (T | undefined)[] = []
-  private resolveNext: ((value: T | undefined) => void) | null = null
+  private resolveNext: ((value: undefined) => void) | null = null
 
   constructor(convertor: Convertor<T>) {
     this.convertor = convertor
@@ -38,20 +39,18 @@ class MyIter<T> implements AsyncIterable<T> {
 
   push(buffer: Uint8Array) {
     const item = this.convertor.from(buffer)
+    this.queue.push(item)
     if (this.resolveNext) {
-      this.resolveNext(item)
+      this.resolveNext(undefined)
       this.resolveNext = null
-    } else {
-      this.queue.push(item)
     }
   }
 
   close() {
+    this.queue.push(undefined)
     if (this.resolveNext) {
       this.resolveNext(undefined)
       this.resolveNext = null
-    } else {
-      this.queue.push(undefined)
     }
   }
 
@@ -63,11 +62,10 @@ class MyIter<T> implements AsyncIterable<T> {
       }
     })(chunks)
     const item = await this.convertor.fromStream(stream)
+    this.queue.push(item)
     if (this.resolveNext) {
-      this.resolveNext(item)
+      this.resolveNext(undefined)
       this.resolveNext = null
-    } else {
-      this.queue.push(item)
     }
   }
 
@@ -78,11 +76,7 @@ class MyIter<T> implements AsyncIterable<T> {
         if (item === undefined) break
         yield item
       } else {
-        const item = await new Promise<T | undefined>(
-          (resolve) => (this.resolveNext = resolve),
-        )
-        if (item === undefined) break
-        yield item
+        await new Promise<undefined>((resolve) => (this.resolveNext = resolve))
       }
     }
   }
@@ -90,7 +84,7 @@ class MyIter<T> implements AsyncIterable<T> {
 
 export class ReaderInstance implements Reader {
   private client: RunnerClient
-  private uri: string
+  readonly uri: string
   private logger: winston.Logger
 
   private iterators: MyIter<unknown>[] = []

package/src/reexports.ts

@@ -0,0 +1,6 @@
+export * from '@rdfc/proto'
+export {
+  ClientReadableStreamImpl,
+  ClientReadableStream,
+} from '@grpc/grpc-js/build/src/call.js'
+export * as grpc from '@grpc/grpc-js'

package/src/runner.ts

@@ -11,7 +11,6 @@ import { Logger } from 'winston'
 
 import winston from 'winston'
 import { RpcTransport } from './logger'
-import { parse_jsonld } from './jsonld'
 import { Cont, empty, extractShapes, Shapes } from 'rdf-lens'
 import { NamedNode, Parser } from 'n3'
 import { createNamespace, createUriAndTermNamespace, RDF } from '@treecg/types'
@@ -31,13 +30,13 @@ const RDFL = createUriAndTermNamespace(
 )
 
 const RDFC = createNamespace(
-  'https://w3id.org/rdf-connect/ontology#',
+  'https://w3id.org/rdf-connect#',
   (x) => x,
   'Reader',
   'Writer',
 )
 
-type Writable = (msg: OrchestratorMessage) => Promise<unknown>
+export type Writable = (msg: OrchestratorMessage) => Promise<unknown>
 
 type ProcessorConfig = {
   location: string
@@ -45,6 +44,8 @@ type ProcessorConfig = {
   clazz?: string
 }
 
+export type FullProc<C extends Proc<unknown>> =
+  C extends Proc<infer T> ? T & C : unknown
 export class Runner {
   private readonly readers: { [uri: string]: ReaderInstance[] } = {}
   private readonly writers: { [uri: string]: WriterInstance[] } = {}
@@ -71,69 +72,72 @@ export class Runner {
     this.logger = logger
   }
 
-  async addProcessor(proc: Processor) {
+  async addProcessor<P extends Proc<unknown>>(
+    proc: Processor,
+  ): Promise<FullProc<P>> {
     const procLogger = winston.createLogger({
       transports: [
         new RpcTransport({
           entities: [proc.uri, this.uri],
-          stream: this.client.logStream(() => { }),
+          stream: this.client.logStream(() => {}),
         }),
       ],
     })
 
-    const ty = JSON.stringify(this.quads.filter(x => x.subject.value === proc.uri && x.predicate.equals(RDF.terms.type)).map(x => x.object.value))
-    this.logger.info("parsing " + proc.uri + " type " + ty)
+    const ty = JSON.stringify(
+      this.quads
+        .filter(
+          (x) =>
+            x.subject.value === proc.uri && x.predicate.equals(RDF.terms.type),
+        )
+        .map((x) => x.object.value),
+    )
+    this.logger.info('parsing ' + proc.uri + ' type ' + ty)
     const args = this.shapes.lenses[RDFL.TypedExtract].execute({
       id: new NamedNode(proc.uri),
       quads: this.quads,
     })
-    // this.logger.info('Parsed ' + JSON.stringify(args))
-
-    // const args = await parse_jsonld(proc.arguments, procLogger, {
-    //   client: this.client,
-    //   write: this.write,
-    //   readers: this.readers,
-    //   writers: this.writers,
-    // })
 
     const config: ProcessorConfig = JSON.parse(proc.config)
-    const url = new URL(config.location)
-    process.chdir(url.pathname)
+    // const url = new URL(config.location)
+    // process.chdir(url.pathname)
     const jsProgram = await import(config.file)
     const clazz = jsProgram[config.clazz || 'default']
     const instance: Proc<unknown> = new clazz(args, procLogger)
     await instance.init()
 
-    this.logger.info("inited " + proc.uri + " type " + ty)
-    await this.write({ init: { uri: proc.uri } })
+    this.logger.info('inited ' + proc.uri + ' type ' + ty)
 
     this.processors.push(instance)
     this.processorTransforms.push(instance.transform())
+
+    await this.write({ init: { uri: proc.uri } })
+
+    return <FullProc<P>>instance
   }
 
   async start() {
-    await Promise.all(this.processors.map((x) => x.produce()))
-    await Promise.all(this.processorTransforms)
+    try {
+      await Promise.all(this.processors.map((x) => x.produce()))
+      await Promise.all(this.processorTransforms)
+    } catch (ex: unknown) {
+      this.logger.error('Start failed: ' + JSON.stringify(ex))
+    }
   }
 
   createWriter(uri: Term): Writer {
-    const ids = uri.value;
+    const ids = uri.value
 
     if (this.writers[ids] === undefined) {
       this.writers[ids] = []
     }
-    const writer = new WriterInstance(
-      ids,
-      this.client,
-      this.write,
-      this.logger,
-    )
+    const writer = new WriterInstance(ids, this.client, this.write, this.logger)
     this.writers[ids].push(writer)
     return writer
   }
 
   createReader(uri: Term): Reader {
-    const ids = uri.value;
+    const ids = uri.value
 
     if (this.readers[ids] === undefined) {
       this.readers[ids] = []
@@ -170,17 +174,24 @@ export class Runner {
     }
 
     if (msg.pipeline) {
-      // here
-      const quads = new Parser().parse(msg.pipeline);
-      this.shapes = extractShapes(quads, {
-        [RDFC.Reader]: (x: Cont) => this.createReader(x.id),
-        [RDFC.Writer]: (x: Cont) => this.createWriter(x.id)
-      }, {
-        [RDFC.Reader]: empty<Cont>(),
-        [RDFC.Writer]: empty<Cont>()
-      });
-      this.quads = quads;
-      this.logger.info("extracted shapes " + JSON.stringify(Object.keys(this.shapes.lenses)));
+      try {
+        // here
+        const quads = new Parser().parse(msg.pipeline)
+        this.shapes = extractShapes(
+          quads,
+          {
+            [RDFC.Reader]: (x: Cont) => this.createReader(x.id),
+            [RDFC.Writer]: (x: Cont) => this.createWriter(x.id),
+          },
+          {
+            [RDFC.Reader]: empty<Cont>(),
+            [RDFC.Writer]: empty<Cont>(),
+          },
+        )
+        this.quads = quads
+      } catch (ex: unknown) {
+        this.logger.error('Pipeline failed: ' + JSON.stringify(ex))
+      }
     }
   }
 }

package/src/testUtils.ts

@@ -0,0 +1,196 @@
+import { DataChunk, RunnerClient } from "@rdfc/proto";
+import * as grpc from "@grpc/grpc-js";
+import { ClientReadableStream } from "@grpc/grpc-js";
+import { ClientReadableStreamImpl, OrchestratorMessage } from "./reexports";
+import { extractShapes } from "rdf-lens";
+import { NamedNode, Parser, Writer } from "n3";
+import { readFile } from "fs/promises";
+import winston, { createLogger } from "winston";
+import { WriterInstance } from "./writer";
+import { ReaderInstance } from "./reader";
+import { Processor } from "./processor";
+import { FullProc, Runner } from "./runner";
+import { Quad, Term } from "@rdfjs/types";
+import { createTermNamespace } from "@treecg/types";
+import { expect } from "vitest";
+
+export async function getProcessorShape(
+    baseIRI = process.cwd() + "/node_modules/@rdfc/js-runner/index.ttl",
+) {
+    const configFile = await readFile(baseIRI, { encoding: "utf8" });
+    const configQuads = new Parser().parse(configFile);
+    const shapes = extractShapes(configQuads);
+
+    return shapes;
+}
+
+export class TestClient extends RunnerClient {
+    next: (stream: ClientReadableStream<DataChunk>) => unknown;
+
+    constructor() {
+        super("localhost:5400", grpc.credentials.createInsecure());
+    }
+
+    nextStream(): Promise<ClientReadableStream<DataChunk>> {
+        return new Promise((res) => (this.next = res));
+    }
+
+    receiveStreamMessage(): ClientReadableStream<DataChunk> {
+        const stream = new ClientReadableStreamImpl<DataChunk>((data: Buffer) => {
+            return { data };
+        });
+        this.next(stream);
+        return stream;
+    }
+}
+
+export async function one<T>(iter: AsyncIterable<T>): Promise<T | undefined> {
+    for await (const item of iter) {
+        return item;
+    }
+}
+
+export const client = new TestClient();
+export const uri = "someUri";
+export const logger = createLogger({
+    transports: new winston.transports.Console({
+        level: process.env["DEBUG"] || "info",
+    }),
+});
+
+export function createWriter(iri = uri): [WriterInstance, ReaderInstance] {
+    const reader = createReader(iri);
+    const writeStream = new WriterInstance(
+        iri,
+        client,
+        async (msg) => {
+            if (msg.msg) {
+                reader.handleMsg(msg.msg);
+            }
+            if (msg.close) {
+                reader.close();
+            }
+        },
+        logger,
+    );
+    return [writeStream, reader];
+}
+
+export function createReader(iri = uri): ReaderInstance {
+    const reader = new ReaderInstance(iri, client, logger);
+    return reader;
+}
+
+const shapeQuads = `
+@prefix rdfc: <https://w3id.org/rdf-connect#>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+[ ] a sh:NodeShape;
+  sh:targetClass <JsProcessorShape>;
+  sh:property [
+    sh:path rdfc:entrypoint;
+    sh:name "location";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+  ], [
+    sh:path rdfc:file;
+    sh:name "file";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+  ], [
+    sh:path rdfc:class;
+    sh:name "clazz";
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+  ].
+`;
+const OWL = createTermNamespace("http://www.w3.org/2002/07/owl#", "imports");
+const processorShapes = extractShapes(new Parser().parse(shapeQuads));
+const base = "https://w3id.org/rdf-connect#";
+
+export async function importFile(file: string): Promise<Quad[]> {
+    const done = new Set<string>();
+    const todo = [new URL("file://" + file)];
+    const quads: Quad[] = [];
+
+    let item = todo.pop();
+    while (item !== undefined) {
+        if (done.has(item.toString())) {
+            item = todo.pop();
+            continue;
+        }
+        done.add(item.toString());
+        if (item.protocol !== "file:") {
+            throw "No supported protocol " + item.protocol;
+        }
+
+        const txt = await readFile(item.pathname, { encoding: "utf8" });
+        const extras = new Parser({ baseIRI: item.toString() }).parse(txt);
+
+        for (const o of extras
+            .filter(
+                (x) =>
+                    x.subject.value === item?.toString() &&
+                    x.predicate.equals(OWL.imports),
+            )
+            .map((x) => x.object.value)) {
+            todo.push(new URL(o));
+        }
+        quads.push(...extras);
+
+        item = todo.pop();
+    }
+
+    return quads;
+}
+
+export async function getProc<T extends Processor<unknown>>(
+    config: string,
+    ty: string,
+    configLocation: string,
+    uri = "http://example.com/ns#processor",
+): Promise<FullProc<T>> {
+    const configQuads = await importFile(configLocation);
+    const procConfig = processorShapes.lenses["JsProcessorShape"].execute({
+        id: new NamedNode(base + ty),
+        quads: configQuads,
+    });
+
+    const msgs: OrchestratorMessage[] = [];
+    const write = async (x: OrchestratorMessage) => {
+        msgs.push(x);
+    };
+    const runner = new Runner(
+        new TestClient(),
+        write,
+        "http://example.com/ns#",
+        logger,
+    );
+    configQuads.push(...new Parser().parse(config));
+    await runner.handleOrchMessage({
+        pipeline: new Writer().quadsToString(configQuads),
+    });
+
+    const proc = await runner.addProcessor<T>({
+        config: JSON.stringify(procConfig),
+        arguments: "",
+        uri,
+    });
+
+    return proc;
+}
+
+export async function checkProcDefinition(file: string, n: string) {
+    const quads = await importFile(file);
+    const procConfig = <{ file: Term; location: string; clazz: string }>(
+        processorShapes.lenses["JsProcessorShape"].execute({
+            id: new NamedNode(base + n),
+            quads: quads,
+        })
+    );
+    expect(procConfig.file, n + " has file").toBeDefined();
+    expect(procConfig.location, n + " has location").toBeDefined();
+    expect(procConfig.clazz, n + " has clazz").toBeDefined();
+}

package/src/writer.ts

@@ -5,6 +5,7 @@ import { Any } from './reader'
 
 type Writable = (msg: OrchestratorMessage) => Promise<unknown>
 export interface Writer {
+  readonly uri: string
   buffer(buffer: Uint8Array): Promise<void>
 
   stream(buffer: AsyncIterable<Uint8Array>): Promise<void>
@@ -19,7 +20,7 @@ export interface Writer {
 }
 const encoder = new TextEncoder()
 export class WriterInstance implements Writer {
-  private readonly uri: string
+  readonly uri: string
   private readonly client: RunnerClient
   private readonly write: Writable
   private readonly logger: Logger

package/examples/echo/test.ttl

@@ -1,14 +0,0 @@
-@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>.
-@prefix sh: <http://www.w3.org/ns/shacl#>.
-
-[] a rdf:Seq;
-  rdf:first [].
-
-<AnyJsProcessor> a sh:NodeShape;
-  sh:property [
-    sh:path sh:prop;
-    sh:node [ sh:property [
-      sh:path sh:prop;
-    ] ];
-  ].
-