@rdfc/js-runner 2.0.0-alpha.5 → 2.0.0-alpha.7

package/src/reader.ts

@@ -2,149 +2,148 @@ import { ClientReadableStream } from '@grpc/grpc-js'
 import { DataChunk, Message, RunnerClient, StreamMessage } from '@rdfc/proto'
 import winston from 'winston'
 import {
-  AnyConvertor,
-  Convertor,
-  NoConvertor,
-  StreamConvertor,
-  StringConvertor,
+    AnyConvertor,
+    Convertor,
+    NoConvertor,
+    StreamConvertor,
+    StringConvertor,
 } from './convertor'
 
 export type Any =
-  | {
-      string: string
+    | {
+        string: string
     }
-  | {
-      stream: AsyncGenerator<Uint8Array>
+    | {
+        stream: AsyncGenerator<Uint8Array>
     }
-  | {
-      buffer: Uint8Array
+    | {
+        buffer: Uint8Array
     }
 
 export interface Reader {
-  strings(): AsyncIterable<string>
-  streams(): AsyncIterable<AsyncGenerator<Uint8Array>>
-  buffers(): AsyncIterable<Uint8Array>
-  anys(): AsyncIterable<Any>
+    readonly uri: string
+    strings(): AsyncIterable<string>
+    streams(): AsyncIterable<AsyncGenerator<Uint8Array>>
+    buffers(): AsyncIterable<Uint8Array>
+    anys(): AsyncIterable<Any>
 }
 
 class MyIter<T> implements AsyncIterable<T> {
-  private convertor: Convertor<T>
-  private queue: (T | undefined)[] = []
-  private resolveNext: ((value: T | undefined) => void) | null = null
-
-  constructor(convertor: Convertor<T>) {
-    this.convertor = convertor
-  }
-
-  push(buffer: Uint8Array) {
-    const item = this.convertor.from(buffer)
-    if (this.resolveNext) {
-      this.resolveNext(item)
-      this.resolveNext = null
-    } else {
-      this.queue.push(item)
-    }
-  }
-
-  close() {
-    if (this.resolveNext) {
-      this.resolveNext(undefined)
-      this.resolveNext = null
-    } else {
-      this.queue.push(undefined)
-    }
-  }
-
-  async pushStream(chunks: ClientReadableStream<DataChunk>) {
-    const stream = (async function* (stream) {
-      for await (const c of stream) {
-        const chunk: DataChunk = c
-        yield chunk.data
-      }
-    })(chunks)
-    const item = await this.convertor.fromStream(stream)
-    if (this.resolveNext) {
-      this.resolveNext(item)
-      this.resolveNext = null
-    } else {
-      this.queue.push(item)
-    }
-  }
-
-  async *[Symbol.asyncIterator]() {
-    while (true) {
-      if (this.queue.length > 0) {
-        const item = this.queue.shift()!
-        if (item === undefined) break
-        yield item
-      } else {
-        const item = await new Promise<T | undefined>(
-          (resolve) => (this.resolveNext = resolve),
-        )
-        if (item === undefined) break
-        yield item
-      }
-    }
-  }
+    private convertor: Convertor<T>
+    private queue: (T | undefined)[] = []
+    private resolveNext: ((value: T | undefined) => void) | null = null
+
+    constructor(convertor: Convertor<T>) {
+        this.convertor = convertor
+    }
+
+    push(buffer: Uint8Array) {
+        const item = this.convertor.from(buffer)
+        if (this.resolveNext) {
+            this.resolveNext(item)
+            this.resolveNext = null
+        } else {
+            this.queue.push(item)
+        }
+    }
+
+    close() {
+        if (this.resolveNext) {
+            this.resolveNext(undefined)
+            this.resolveNext = null
+        } else {
+            this.queue.push(undefined)
+        }
+    }
+
+    async pushStream(chunks: ClientReadableStream<DataChunk>) {
+        const stream = (async function*(stream) {
+            for await (const c of stream) {
+                const chunk: DataChunk = c
+                yield chunk.data
+            }
+        })(chunks)
+        const item = await this.convertor.fromStream(stream)
+        if (this.resolveNext) {
+            this.resolveNext(item)
+            this.resolveNext = null
+        } else {
+            this.queue.push(item)
+        }
+    }
+
+    async *[Symbol.asyncIterator]() {
+        while (true) {
+            if (this.queue.length > 0) {
+                const item = this.queue.shift()!
+                if (item === undefined) break
+                yield item
+            } else {
+                const item = await new Promise<T | undefined>(
+                    (resolve) => (this.resolveNext = resolve),
+                )
+                if (item === undefined) break
+                yield item
+            }
+        }
+    }
 }
 
 export class ReaderInstance implements Reader {
-  private client: RunnerClient
-  private uri: string
-  private logger: winston.Logger
-
-  private iterators: MyIter<unknown>[] = []
-
-  constructor(uri: string, client: RunnerClient, logger: winston.Logger) {
-    this.uri = uri
-    this.client = client
-    this.logger = logger
-  }
-
-  anys(): AsyncIterable<Any> {
-    const iter = new MyIter(AnyConvertor)
-    this.iterators.push(iter)
-    return iter
-  }
-
-  strings(): AsyncIterable<string> {
-    console.log("Strings");
-    const iter = new MyIter(StringConvertor)
-    this.iterators.push(iter)
-    return iter
-  }
-
-  buffers(): AsyncIterable<Uint8Array> {
-    const iter = new MyIter(NoConvertor)
-    this.iterators.push(iter)
-    return iter
-  }
-
-  streams(): AsyncIterable<AsyncGenerator<Uint8Array>> {
-    const iter = new MyIter(StreamConvertor)
-    this.iterators.push(iter)
-    return iter
-  }
-
-  handleMsg(msg: Message) {
-    this.logger.debug(`${this.uri} handling message`)
-    console.log(this.uri, "handling message", this.iterators.length, "iterators");
-    for (const iter of this.iterators) {
-      iter.push(msg.data)
-    }
-  }
-
-  close() {
-    for (const iter of this.iterators) {
-      iter.close()
-    }
-  }
-
-  handleStreamingMessage(msg: StreamMessage) {
-    this.logger.debug(`${this.uri} handling streaming message`)
-    const chunks = this.client.receiveStreamMessage(msg.id!)
-    for (const iter of this.iterators) {
-      iter.pushStream(chunks)
-    }
-  }
+    private client: RunnerClient
+    readonly uri: string
+    private logger: winston.Logger
+
+    private iterators: MyIter<unknown>[] = []
+
+    constructor(uri: string, client: RunnerClient, logger: winston.Logger) {
+        this.uri = uri
+        this.client = client
+        this.logger = logger
+    }
+
+    anys(): AsyncIterable<Any> {
+        const iter = new MyIter(AnyConvertor)
+        this.iterators.push(iter)
+        return iter
+    }
+
+    strings(): AsyncIterable<string> {
+        const iter = new MyIter(StringConvertor)
+        this.iterators.push(iter)
+        return iter
+    }
+
+    buffers(): AsyncIterable<Uint8Array> {
+        const iter = new MyIter(NoConvertor)
+        this.iterators.push(iter)
+        return iter
+    }
+
+    streams(): AsyncIterable<AsyncGenerator<Uint8Array>> {
+        const iter = new MyIter(StreamConvertor)
+        this.iterators.push(iter)
+        return iter
+    }
+
+    handleMsg(msg: Message) {
+        this.logger.debug(`${this.uri} handling message`)
+        for (const iter of this.iterators) {
+            iter.push(msg.data)
+        }
+    }
+
+    close() {
+        for (const iter of this.iterators) {
+            iter.close()
+        }
+    }
+
+    handleStreamingMessage(msg: StreamMessage) {
+        this.logger.debug(`${this.uri} handling streaming message`)
+        const chunks = this.client.receiveStreamMessage(msg.id!)
+        for (const iter of this.iterators) {
+            iter.pushStream(chunks)
+        }
+    }
 }

package/src/runner.ts

@@ -1,8 +1,8 @@
 import {
-  OrchestratorMessage,
-  Processor,
-  RunnerClient,
-  RunnerMessage,
+    OrchestratorMessage,
+    Processor,
+    RunnerClient,
+    RunnerMessage,
 } from '@rdfc/proto'
 import { Reader, ReaderInstance } from './reader'
 import { Writer, WriterInstance } from './writer'
@@ -18,164 +18,165 @@ import { createNamespace, createUriAndTermNamespace, RDF } from '@treecg/types'
 import { Quad, Term } from '@rdfjs/types'
 
 const RDFL = createUriAndTermNamespace(
-  'https://w3id.org/rdf-lens/ontology#',
-  'CBD',
-  'Path',
-  'PathLens',
-  'Context',
-  'TypedExtract',
-  'EnvVariable',
-  'envKey',
-  'envDefault',
-  'datatype',
+    'https://w3id.org/rdf-lens/ontology#',
+    'CBD',
+    'Path',
+    'PathLens',
+    'Context',
+    'TypedExtract',
+    'EnvVariable',
+    'envKey',
+    'envDefault',
+    'datatype',
 )
 
 const RDFC = createNamespace(
-  'https://w3id.org/rdf-connect/ontology#',
-  (x) => x,
-  'Reader',
-  'Writer',
+    'https://w3id.org/rdf-connect#',
+    (x) => x,
+    'Reader',
+    'Writer',
 )
 
 export type Writable = (msg: OrchestratorMessage) => Promise<unknown>
 
 type ProcessorConfig = {
-  location: string
-  file: string
-  clazz?: string
+    location: string
+    file: string
+    clazz?: string
 }
 
-export type GetMyClassT<C extends Proc<unknown>> = C extends Proc<infer T> ? T : unknown;
+export type FullProc<C extends Proc<unknown>> = C extends Proc<infer T> ? T & C : unknown;
 export class Runner {
-  private readonly readers: { [uri: string]: ReaderInstance[] } = {}
-  private readonly writers: { [uri: string]: WriterInstance[] } = {}
-  private readonly client: RunnerClient
-  private readonly write: Writable
-  private readonly logger: Logger
-  private shapes: Shapes
-  private quads: Quad[] = []
-
-  private readonly uri: string
-
-  private readonly processors: Proc<unknown>[] = []
-  private readonly processorTransforms: Promise<unknown>[] = []
-
-  constructor(
-    client: RunnerClient,
-    write: Writable,
-    uri: string,
-    logger: Logger,
-  ) {
-    this.client = client
-    this.write = write
-    this.uri = uri
-    this.logger = logger
-  }
-
-  async addProcessor<P extends Proc<unknown>>(proc: Processor): Promise<P & GetMyClassT<P>> {
-    const procLogger = winston.createLogger({
-      transports: [
-        new RpcTransport({
-          entities: [proc.uri, this.uri],
-          stream: this.client.logStream(() => { }),
-        }),
-      ],
-    })
-
-    const ty = JSON.stringify(this.quads.filter(x => x.subject.value === proc.uri && x.predicate.equals(RDF.terms.type)).map(x => x.object.value))
-    this.logger.info("parsing " + proc.uri + " type " + ty)
-    const args = this.shapes.lenses[RDFL.TypedExtract].execute({
-      id: new NamedNode(proc.uri),
-      quads: this.quads,
-    })
-
-    const config: ProcessorConfig = JSON.parse(proc.config)
-    const url = new URL(config.location)
-    process.chdir(url.pathname)
-    const jsProgram = await import(config.file)
-    const clazz = jsProgram[config.clazz || 'default']
-    const instance: Proc<unknown> = new clazz(args, procLogger)
-    await instance.init()
-
-    this.logger.info("inited " + proc.uri + " type " + ty)
-    await this.write({ init: { uri: proc.uri } })
-
-    this.processors.push(instance)
-    this.processorTransforms.push(instance.transform())
-
-    return <P & GetMyClassT<P>>instance;
-  }
-
-  async start() {
-    await Promise.all(this.processors.map((x) => x.produce()))
-    await Promise.all(this.processorTransforms)
-  }
-
-  createWriter(uri: Term): Writer {
-    const ids = uri.value;
-
-    if (this.writers[ids] === undefined) {
-      this.writers[ids] = []
+    private readonly readers: { [uri: string]: ReaderInstance[] } = {}
+    private readonly writers: { [uri: string]: WriterInstance[] } = {}
+    private readonly client: RunnerClient
+    private readonly write: Writable
+    private readonly logger: Logger
+    private shapes: Shapes
+    private quads: Quad[] = []
+
+    private readonly uri: string
+
+    private readonly processors: Proc<unknown>[] = []
+    private readonly processorTransforms: Promise<unknown>[] = []
+
+    constructor(
+        client: RunnerClient,
+        write: Writable,
+        uri: string,
+        logger: Logger,
+    ) {
+        this.client = client
+        this.write = write
+        this.uri = uri
+        this.logger = logger
     }
-    const writer = new WriterInstance(
-      ids,
-      this.client,
-      this.write,
-      this.logger,
-    )
-    this.writers[ids].push(writer)
-    return writer
-  }
-
-  createReader(uri: Term): Reader {
-    const ids = uri.value;
-
-    if (this.readers[ids] === undefined) {
-      this.readers[ids] = []
-    }
-    const reader = new ReaderInstance(ids, this.client, this.logger)
-    this.readers[ids].push(reader)
-    return reader
-  }
-
-  async handleOrchMessage(msg: RunnerMessage) {
-    if (msg.msg) {
-      this.logger.debug('Handling data msg for ' + msg.msg.channel)
-      for (const reader of this.readers[msg.msg.channel] || []) {
-        reader.handleMsg(msg.msg)
-      }
+
+    async addProcessor<P extends Proc<unknown>>(proc: Processor): Promise<FullProc<P>> {
+        const procLogger = winston.createLogger({
+            transports: [
+                new RpcTransport({
+                    entities: [proc.uri, this.uri],
+                    stream: this.client.logStream(() => { }),
+                }),
+            ],
+        })
+
+        const ty = JSON.stringify(this.quads.filter(x => x.subject.value === proc.uri && x.predicate.equals(RDF.terms.type)).map(x => x.object.value))
+        this.logger.info("parsing " + proc.uri + " type " + ty)
+        const args = this.shapes.lenses[RDFL.TypedExtract].execute({
+            id: new NamedNode(proc.uri),
+            quads: this.quads,
+        })
+
+        const config: ProcessorConfig = JSON.parse(proc.config)
+        // const url = new URL(config.location)
+        // process.chdir(url.pathname)
+        const jsProgram = await import(config.file)
+        const clazz = jsProgram[config.clazz || 'default']
+        const instance: Proc<unknown> = new clazz(args, procLogger)
+        await instance.init()
+
+        this.logger.info("inited " + proc.uri + " type " + ty)
+
+        this.processors.push(instance)
+        this.processorTransforms.push(instance.transform())
+
+        await this.write({ init: { uri: proc.uri } })
+
+        return <FullProc<P>>instance;
     }
 
-    if (msg.streamMsg) {
-      for (const reader of this.readers[msg.streamMsg.channel] || []) {
-        reader.handleStreamingMessage(msg.streamMsg)
-      }
+    async start() {
+        await Promise.all(this.processors.map((x) => x.produce()))
+        await Promise.all(this.processorTransforms)
     }
 
-    if (msg.close) {
-      const uri = msg.close.channel
+    createWriter(uri: Term): Writer {
+        const ids = uri.value;
+
+        if (this.writers[ids] === undefined) {
+            this.writers[ids] = []
+        }
+        const writer = new WriterInstance(
+            ids,
+            this.client,
+            this.write,
+            this.logger,
+        )
+        this.writers[ids].push(writer)
+        return writer
+    }
 
-      for (const reader of this.readers[uri] || []) {
-        reader.close()
-      }
+    createReader(uri: Term): Reader {
+        const ids = uri.value;
 
-      for (const writer of this.writers[uri] || []) {
-        await writer.close(true)
-      }
+        if (this.readers[ids] === undefined) {
+            this.readers[ids] = []
+        }
+        const reader = new ReaderInstance(ids, this.client, this.logger)
+        this.readers[ids].push(reader)
+        return reader
     }
 
-    if (msg.pipeline) {
-      // here
-      const quads = new Parser().parse(msg.pipeline);
-      this.shapes = extractShapes(quads, {
-        [RDFC.Reader]: (x: Cont) => this.createReader(x.id),
-        [RDFC.Writer]: (x: Cont) => this.createWriter(x.id)
-      }, {
-        [RDFC.Reader]: empty<Cont>(),
-        [RDFC.Writer]: empty<Cont>()
-      });
-      this.quads = quads;
-      this.logger.info("extracted shapes " + JSON.stringify(Object.keys(this.shapes.lenses)));
+    async handleOrchMessage(msg: RunnerMessage) {
+        if (msg.msg) {
+            this.logger.debug('Handling data msg for ' + msg.msg.channel)
+            for (const reader of this.readers[msg.msg.channel] || []) {
+                reader.handleMsg(msg.msg)
+            }
+        }
+
+        if (msg.streamMsg) {
+            for (const reader of this.readers[msg.streamMsg.channel] || []) {
+                reader.handleStreamingMessage(msg.streamMsg)
+            }
+        }
+
+        if (msg.close) {
+            const uri = msg.close.channel
+
+            for (const reader of this.readers[uri] || []) {
+                reader.close()
+            }
+
+            for (const writer of this.writers[uri] || []) {
+                await writer.close(true)
+            }
+        }
+
+        if (msg.pipeline) {
+            // here
+            const quads = new Parser().parse(msg.pipeline);
+            this.shapes = extractShapes(quads, {
+                [RDFC.Reader]: (x: Cont) => this.createReader(x.id),
+                [RDFC.Writer]: (x: Cont) => this.createWriter(x.id)
+            }, {
+                [RDFC.Reader]: empty<Cont>(),
+                [RDFC.Writer]: empty<Cont>()
+            });
+            this.quads = quads;
+            this.logger.info("extracted shapes " + JSON.stringify(Object.keys(this.shapes.lenses)));
+        }
     }
-  }
 }

package/src/testUtils.ts

@@ -55,6 +55,9 @@ export function createWriter(iri = uri): [WriterInstance, ReaderInstance] {
     if (msg.msg) {
       reader.handleMsg(msg.msg);
     }
+    if(msg.close) {
+      reader.close()
+    }
   }, logger)
   return [writeStream, reader];
 }

package/src/util_processors.ts

@@ -16,5 +16,5 @@ export class LogProcessor extends Processor<LogArgs> {
     }
   }
 
-  async produce() {}
+  async produce() { }
 }