@rdfc/js-runner 2.0.0-alpha.6 → 2.0.0-alpha.8

package/src/client.ts

@@ -8,11 +8,13 @@ import { Runner } from './runner'
 export async function start(addr: string, uri: string) {
   const client = new RunnerClient(addr, grpc.credentials.createInsecure())
 
+
+
   const logger = winston.createLogger({
     transports: [
       new RpcTransport({
         entities: [uri, 'cli'],
-        stream: client.logStream(() => {}),
+        stream: client.logStream(() => { }),
       }),
     ],
   })
@@ -27,21 +29,21 @@ export async function start(addr: string, uri: string) {
 
   let processorsEnd!: (v: unknown) => unknown
   const processorsEnded = new Promise((res) => (processorsEnd = res))
-  ;(async () => {
-    for await (const chunk of stream) {
-      const msg: RunnerMessage = chunk
-      if (msg.proc) {
-        await runner.addProcessor(msg.proc)
+    ; (async () => {
+      for await (const chunk of stream) {
+        const msg: RunnerMessage = chunk
+        if (msg.proc) {
+          await runner.addProcessor(msg.proc)
+        }
+        if (msg.start) {
+          runner.start().then(processorsEnd)
+        }
+
+        await runner.handleOrchMessage(msg)
       }
-      if (msg.start) {
-        runner.start().then(processorsEnd)
-      }
-
-      await runner.handleOrchMessage(msg)
-    }
 
-    logger.error('Stream ended')
-  })()
+      logger.error('Stream ended')
+    })()
 
   await processorsEnded
 

package/src/index.ts

@@ -4,4 +4,4 @@ export * from './reader'
 export * from './processor'
 export * from './runner'
 
-export * as reexports from "./reexports";
+export * as reexports from './reexports'

package/src/jsonld.ts

@@ -30,7 +30,7 @@ const RDFL = createUriAndTermNamespace(
 
 type Writable = (msg: OrchestratorMessage) => Promise<unknown>
 const RDFC = createNamespace(
-  'https://w3id.org/rdf-connect/ontology#',
+  'https://w3id.org/rdf-connect#',
   (x) => x,
   'Reader',
   'Writer',
@@ -189,7 +189,10 @@ function expandArgs(args: Arg, cont: Cont, logger: Logger) {
             let x = 'object'
             try {
               x = JSON.stringify(v)
-            } catch (e) {}
+              // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            } catch (_e: unknown) {
+              // default is set
+            }
 
             return k + ': ' + x
           }),

package/src/reader.ts

@@ -11,16 +11,17 @@ import {
 
 export type Any =
   | {
-      string: string
-    }
+    string: string
+  }
   | {
-      stream: AsyncGenerator<Uint8Array>
-    }
+    stream: AsyncGenerator<Uint8Array>
+  }
   | {
-      buffer: Uint8Array
-    }
+    buffer: Uint8Array
+  }
 
 export interface Reader {
+  readonly uri: string
   strings(): AsyncIterable<string>
   streams(): AsyncIterable<AsyncGenerator<Uint8Array>>
   buffers(): AsyncIterable<Uint8Array>
@@ -30,7 +31,7 @@ export interface Reader {
 class MyIter<T> implements AsyncIterable<T> {
   private convertor: Convertor<T>
   private queue: (T | undefined)[] = []
-  private resolveNext: ((value: T | undefined) => void) | null = null
+  private resolveNext: ((value: undefined) => void) | null = null
 
   constructor(convertor: Convertor<T>) {
     this.convertor = convertor
@@ -38,36 +39,33 @@ class MyIter<T> implements AsyncIterable<T> {
 
   push(buffer: Uint8Array) {
     const item = this.convertor.from(buffer)
+    this.queue.push(item)
     if (this.resolveNext) {
-      this.resolveNext(item)
+      this.resolveNext(undefined)
       this.resolveNext = null
-    } else {
-      this.queue.push(item)
     }
   }
 
   close() {
+    this.queue.push(undefined)
     if (this.resolveNext) {
       this.resolveNext(undefined)
       this.resolveNext = null
-    } else {
-      this.queue.push(undefined)
     }
   }
 
   async pushStream(chunks: ClientReadableStream<DataChunk>) {
-    const stream = (async function* (stream) {
+    const stream = (async function*(stream) {
       for await (const c of stream) {
         const chunk: DataChunk = c
         yield chunk.data
       }
     })(chunks)
     const item = await this.convertor.fromStream(stream)
+    this.queue.push(item)
     if (this.resolveNext) {
-      this.resolveNext(item)
+      this.resolveNext(undefined)
       this.resolveNext = null
-    } else {
-      this.queue.push(item)
     }
   }
 
@@ -78,11 +76,9 @@ class MyIter<T> implements AsyncIterable<T> {
         if (item === undefined) break
         yield item
       } else {
-        const item = await new Promise<T | undefined>(
+        await new Promise<undefined>(
           (resolve) => (this.resolveNext = resolve),
         )
-        if (item === undefined) break
-        yield item
       }
     }
   }
@@ -90,7 +86,7 @@ class MyIter<T> implements AsyncIterable<T> {
 
 export class ReaderInstance implements Reader {
   private client: RunnerClient
-  private uri: string
+  readonly uri: string
   private logger: winston.Logger
 
   private iterators: MyIter<unknown>[] = []
@@ -127,6 +123,7 @@ export class ReaderInstance implements Reader {
 
   handleMsg(msg: Message) {
     this.logger.debug(`${this.uri} handling message`)
+    console.log(`${this.uri} handling message`)
     for (const iter of this.iterators) {
       iter.push(msg.data)
     }

package/src/reexports.ts

@@ -1,3 +1,6 @@
-export * from "@rdfc/proto";
-export { ClientReadableStreamImpl, ClientReadableStream } from '@grpc/grpc-js/build/src/call.js'
+export * from '@rdfc/proto'
+export {
+  ClientReadableStreamImpl,
+  ClientReadableStream,
+} from '@grpc/grpc-js/build/src/call.js'
 export * as grpc from '@grpc/grpc-js'

package/src/runner.ts

@@ -11,7 +11,6 @@ import { Logger } from 'winston'
 
 import winston from 'winston'
 import { RpcTransport } from './logger'
-import { parse_jsonld } from './jsonld'
 import { Cont, empty, extractShapes, Shapes } from 'rdf-lens'
 import { NamedNode, Parser } from 'n3'
 import { createNamespace, createUriAndTermNamespace, RDF } from '@treecg/types'
@@ -31,7 +30,7 @@ const RDFL = createUriAndTermNamespace(
 )
 
 const RDFC = createNamespace(
-  'https://w3id.org/rdf-connect/ontology#',
+  'https://w3id.org/rdf-connect#',
   (x) => x,
   'Reader',
   'Writer',
@@ -45,7 +44,8 @@ type ProcessorConfig = {
   clazz?: string
 }
 
-export type FullProc<C extends Proc<unknown>> = C extends Proc<infer T> ? T & C : unknown;
+export type FullProc<C extends Proc<unknown>> =
+  C extends Proc<infer T> ? T & C : unknown
 export class Runner {
   private readonly readers: { [uri: string]: ReaderInstance[] } = {}
   private readonly writers: { [uri: string]: WriterInstance[] } = {}
@@ -72,7 +72,9 @@ export class Runner {
     this.logger = logger
   }
 
-  async addProcessor<P extends Proc<unknown>>(proc: Processor): Promise<FullProc<P>> {
+  async addProcessor<P extends Proc<unknown>>(
+    proc: Processor,
+  ): Promise<FullProc<P>> {
     const procLogger = winston.createLogger({
       transports: [
         new RpcTransport({
@@ -82,53 +84,60 @@ export class Runner {
       ],
     })
 
-    const ty = JSON.stringify(this.quads.filter(x => x.subject.value === proc.uri && x.predicate.equals(RDF.terms.type)).map(x => x.object.value))
-    this.logger.info("parsing " + proc.uri + " type " + ty)
+    const ty = JSON.stringify(
+      this.quads
+        .filter(
+          (x) =>
+            x.subject.value === proc.uri && x.predicate.equals(RDF.terms.type),
+        )
+        .map((x) => x.object.value),
+    )
+    this.logger.info('parsing ' + proc.uri + ' type ' + ty)
     const args = this.shapes.lenses[RDFL.TypedExtract].execute({
       id: new NamedNode(proc.uri),
       quads: this.quads,
     })
 
     const config: ProcessorConfig = JSON.parse(proc.config)
-    const url = new URL(config.location)
-    process.chdir(url.pathname)
+    // const url = new URL(config.location)
+    // process.chdir(url.pathname)
     const jsProgram = await import(config.file)
     const clazz = jsProgram[config.clazz || 'default']
     const instance: Proc<unknown> = new clazz(args, procLogger)
     await instance.init()
 
-    this.logger.info("inited " + proc.uri + " type " + ty)
-    await this.write({ init: { uri: proc.uri } })
+    this.logger.info('inited ' + proc.uri + ' type ' + ty)
 
     this.processors.push(instance)
     this.processorTransforms.push(instance.transform())
 
-    return <FullProc<P>>instance;
+    await this.write({ init: { uri: proc.uri } })
+
+    return <FullProc<P>>instance
   }
 
   async start() {
-    await Promise.all(this.processors.map((x) => x.produce()))
-    await Promise.all(this.processorTransforms)
+    try {
+      await Promise.all(this.processors.map((x) => x.produce()))
+      await Promise.all(this.processorTransforms)
+    } catch (ex: unknown) {
+      this.logger.error("Start failed: " + JSON.stringify(ex))
+    }
   }
 
   createWriter(uri: Term): Writer {
-    const ids = uri.value;
+    const ids = uri.value
 
     if (this.writers[ids] === undefined) {
       this.writers[ids] = []
     }
-    const writer = new WriterInstance(
-      ids,
-      this.client,
-      this.write,
-      this.logger,
-    )
+    const writer = new WriterInstance(ids, this.client, this.write, this.logger)
     this.writers[ids].push(writer)
     return writer
   }
 
   createReader(uri: Term): Reader {
-    const ids = uri.value;
+    const ids = uri.value
 
     if (this.readers[ids] === undefined) {
       this.readers[ids] = []
@@ -165,17 +174,26 @@ export class Runner {
     }
 
     if (msg.pipeline) {
-      // here
-      const quads = new Parser().parse(msg.pipeline);
-      this.shapes = extractShapes(quads, {
-        [RDFC.Reader]: (x: Cont) => this.createReader(x.id),
-        [RDFC.Writer]: (x: Cont) => this.createWriter(x.id)
-      }, {
-        [RDFC.Reader]: empty<Cont>(),
-        [RDFC.Writer]: empty<Cont>()
-      });
-      this.quads = quads;
-      this.logger.info("extracted shapes " + JSON.stringify(Object.keys(this.shapes.lenses)));
+      try {
+
+        // here
+        const quads = new Parser().parse(msg.pipeline)
+        this.shapes = extractShapes(
+          quads,
+          {
+            [RDFC.Reader]: (x: Cont) => this.createReader(x.id),
+            [RDFC.Writer]: (x: Cont) => this.createWriter(x.id),
+          },
+          {
+            [RDFC.Reader]: empty<Cont>(),
+            [RDFC.Writer]: empty<Cont>(),
+          },
+        )
+        this.quads = quads
+      }
+      catch (ex: unknown) {
+        this.logger.error("Pipeline failed: " + JSON.stringify(ex))
+      }
     }
   }
 }

package/src/testUtils.ts

@@ -1,20 +1,22 @@
-import { DataChunk, OrchestratorMessage, RunnerClient } from '@rdfc/proto'
+import { DataChunk, RunnerClient } from '@rdfc/proto'
 import { ClientReadableStream } from '@grpc/grpc-js'
 import * as grpc from '@grpc/grpc-js'
 import { ClientReadableStreamImpl } from './reexports'
-import { extractShapes } from 'rdf-lens';
-import { Parser } from 'n3';
-import { readFile } from 'fs/promises';
-import winston, { createLogger } from 'winston';
-import { WriterInstance } from './writer';
-import { ReaderInstance } from './reader';
+import { extractShapes } from 'rdf-lens'
+import { Parser } from 'n3'
+import { readFile } from 'fs/promises'
+import winston, { createLogger } from 'winston'
+import { WriterInstance } from './writer'
+import { ReaderInstance } from './reader'
 
-export async function getProcessorShape(baseIRI = process.cwd() + "/node_modules/@rdfc/js-runner/index.ttl") {
-  const configFile = await readFile(baseIRI, { encoding: "utf8" });
-  const configQuads = new Parser().parse(configFile);
-  const shapes = extractShapes(configQuads);
+export async function getProcessorShape(
+  baseIRI = process.cwd() + '/node_modules/@rdfc/js-runner/index.ttl',
+) {
+  const configFile = await readFile(baseIRI, { encoding: 'utf8' })
+  const configQuads = new Parser().parse(configFile)
+  const shapes = extractShapes(configQuads)
 
-  return shapes;
+  return shapes
 }
 
 export class TestClient extends RunnerClient {
@@ -46,23 +48,30 @@ export async function one<T>(iter: AsyncIterable<T>): Promise<T | undefined> {
 export const client = new TestClient()
 export const uri = 'someUri'
 export const logger = createLogger({
-  transports: new winston.transports.Console({ level: process.env['DEBUG'] || 'info' }),
+  transports: new winston.transports.Console({
+    level: process.env['DEBUG'] || 'info',
+  }),
 })
 
 export function createWriter(iri = uri): [WriterInstance, ReaderInstance] {
-  const reader = createReader(iri);
-  const writeStream = new WriterInstance(iri, client, async (msg) => {
-    if (msg.msg) {
-      reader.handleMsg(msg.msg);
-    }
-    if(msg.close) {
-      reader.close()
-    }
-  }, logger)
-  return [writeStream, reader];
+  const reader = createReader(iri)
+  const writeStream = new WriterInstance(
+    iri,
+    client,
+    async (msg) => {
+      if (msg.msg) {
+        reader.handleMsg(msg.msg)
+      }
+      if (msg.close) {
+        reader.close()
+      }
+    },
+    logger,
+  )
+  return [writeStream, reader]
 }
 
 export function createReader(iri = uri): ReaderInstance {
-  const reader = new ReaderInstance(iri, client, logger);
-  return reader;
+  const reader = new ReaderInstance(iri, client, logger)
+  return reader
 }

package/src/writer.ts

@@ -5,6 +5,7 @@ import { Any } from './reader'
 
 type Writable = (msg: OrchestratorMessage) => Promise<unknown>
 export interface Writer {
+  readonly uri: string
   buffer(buffer: Uint8Array): Promise<void>
 
   stream(buffer: AsyncIterable<Uint8Array>): Promise<void>
@@ -19,7 +20,7 @@ export interface Writer {
 }
 const encoder = new TextEncoder()
 export class WriterInstance implements Writer {
-  private readonly uri: string
+  readonly uri: string
   private readonly client: RunnerClient
   private readonly write: Writable
   private readonly logger: Logger

package/vite.config.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const config_1 = require("vitest/config");
+exports.default = (0, config_1.defineConfig)({
+    test: {
+        coverage: {
+            enabled: true,
+            include: ['src/'],
+        },
+    },
+});