@rdfc/js-runner 2.0.0-alpha.7 → 2.0.0-alpha.8

package/src/reader.ts

@@ -2,148 +2,144 @@ import { ClientReadableStream } from '@grpc/grpc-js'
 import { DataChunk, Message, RunnerClient, StreamMessage } from '@rdfc/proto'
 import winston from 'winston'
 import {
-    AnyConvertor,
-    Convertor,
-    NoConvertor,
-    StreamConvertor,
-    StringConvertor,
+  AnyConvertor,
+  Convertor,
+  NoConvertor,
+  StreamConvertor,
+  StringConvertor,
 } from './convertor'
 
 export type Any =
-    | {
-        string: string
-    }
-    | {
-        stream: AsyncGenerator<Uint8Array>
-    }
-    | {
-        buffer: Uint8Array
-    }
+  | {
+    string: string
+  }
+  | {
+    stream: AsyncGenerator<Uint8Array>
+  }
+  | {
+    buffer: Uint8Array
+  }
 
 export interface Reader {
-    readonly uri: string
-    strings(): AsyncIterable<string>
-    streams(): AsyncIterable<AsyncGenerator<Uint8Array>>
-    buffers(): AsyncIterable<Uint8Array>
-    anys(): AsyncIterable<Any>
+  readonly uri: string
+  strings(): AsyncIterable<string>
+  streams(): AsyncIterable<AsyncGenerator<Uint8Array>>
+  buffers(): AsyncIterable<Uint8Array>
+  anys(): AsyncIterable<Any>
 }
 
 class MyIter<T> implements AsyncIterable<T> {
-    private convertor: Convertor<T>
-    private queue: (T | undefined)[] = []
-    private resolveNext: ((value: T | undefined) => void) | null = null
-
-    constructor(convertor: Convertor<T>) {
-        this.convertor = convertor
-    }
-
-    push(buffer: Uint8Array) {
-        const item = this.convertor.from(buffer)
-        if (this.resolveNext) {
-            this.resolveNext(item)
-            this.resolveNext = null
-        } else {
-            this.queue.push(item)
-        }
-    }
-
-    close() {
-        if (this.resolveNext) {
-            this.resolveNext(undefined)
-            this.resolveNext = null
-        } else {
-            this.queue.push(undefined)
-        }
-    }
-
-    async pushStream(chunks: ClientReadableStream<DataChunk>) {
-        const stream = (async function*(stream) {
-            for await (const c of stream) {
-                const chunk: DataChunk = c
-                yield chunk.data
-            }
-        })(chunks)
-        const item = await this.convertor.fromStream(stream)
-        if (this.resolveNext) {
-            this.resolveNext(item)
-            this.resolveNext = null
-        } else {
-            this.queue.push(item)
-        }
-    }
-
-    async *[Symbol.asyncIterator]() {
-        while (true) {
-            if (this.queue.length > 0) {
-                const item = this.queue.shift()!
-                if (item === undefined) break
-                yield item
-            } else {
-                const item = await new Promise<T | undefined>(
-                    (resolve) => (this.resolveNext = resolve),
-                )
-                if (item === undefined) break
-                yield item
-            }
-        }
-    }
+  private convertor: Convertor<T>
+  private queue: (T | undefined)[] = []
+  private resolveNext: ((value: undefined) => void) | null = null
+
+  constructor(convertor: Convertor<T>) {
+    this.convertor = convertor
+  }
+
+  push(buffer: Uint8Array) {
+    const item = this.convertor.from(buffer)
+    this.queue.push(item)
+    if (this.resolveNext) {
+      this.resolveNext(undefined)
+      this.resolveNext = null
+    }
+  }
+
+  close() {
+    this.queue.push(undefined)
+    if (this.resolveNext) {
+      this.resolveNext(undefined)
+      this.resolveNext = null
+    }
+  }
+
+  async pushStream(chunks: ClientReadableStream<DataChunk>) {
+    const stream = (async function*(stream) {
+      for await (const c of stream) {
+        const chunk: DataChunk = c
+        yield chunk.data
+      }
+    })(chunks)
+    const item = await this.convertor.fromStream(stream)
+    this.queue.push(item)
+    if (this.resolveNext) {
+      this.resolveNext(undefined)
+      this.resolveNext = null
+    }
+  }
+
+  async *[Symbol.asyncIterator]() {
+    while (true) {
+      if (this.queue.length > 0) {
+        const item = this.queue.shift()!
+        if (item === undefined) break
+        yield item
+      } else {
+        await new Promise<undefined>(
+          (resolve) => (this.resolveNext = resolve),
+        )
+      }
+    }
+  }
 }
 
 export class ReaderInstance implements Reader {
-    private client: RunnerClient
-    readonly uri: string
-    private logger: winston.Logger
-
-    private iterators: MyIter<unknown>[] = []
-
-    constructor(uri: string, client: RunnerClient, logger: winston.Logger) {
-        this.uri = uri
-        this.client = client
-        this.logger = logger
-    }
-
-    anys(): AsyncIterable<Any> {
-        const iter = new MyIter(AnyConvertor)
-        this.iterators.push(iter)
-        return iter
-    }
-
-    strings(): AsyncIterable<string> {
-        const iter = new MyIter(StringConvertor)
-        this.iterators.push(iter)
-        return iter
-    }
-
-    buffers(): AsyncIterable<Uint8Array> {
-        const iter = new MyIter(NoConvertor)
-        this.iterators.push(iter)
-        return iter
-    }
-
-    streams(): AsyncIterable<AsyncGenerator<Uint8Array>> {
-        const iter = new MyIter(StreamConvertor)
-        this.iterators.push(iter)
-        return iter
-    }
-
-    handleMsg(msg: Message) {
-        this.logger.debug(`${this.uri} handling message`)
-        for (const iter of this.iterators) {
-            iter.push(msg.data)
-        }
-    }
-
-    close() {
-        for (const iter of this.iterators) {
-            iter.close()
-        }
-    }
-
-    handleStreamingMessage(msg: StreamMessage) {
-        this.logger.debug(`${this.uri} handling streaming message`)
-        const chunks = this.client.receiveStreamMessage(msg.id!)
-        for (const iter of this.iterators) {
-            iter.pushStream(chunks)
-        }
-    }
+  private client: RunnerClient
+  readonly uri: string
+  private logger: winston.Logger
+
+  private iterators: MyIter<unknown>[] = []
+
+  constructor(uri: string, client: RunnerClient, logger: winston.Logger) {
+    this.uri = uri
+    this.client = client
+    this.logger = logger
+  }
+
+  anys(): AsyncIterable<Any> {
+    const iter = new MyIter(AnyConvertor)
+    this.iterators.push(iter)
+    return iter
+  }
+
+  strings(): AsyncIterable<string> {
+    const iter = new MyIter(StringConvertor)
+    this.iterators.push(iter)
+    return iter
+  }
+
+  buffers(): AsyncIterable<Uint8Array> {
+    const iter = new MyIter(NoConvertor)
+    this.iterators.push(iter)
+    return iter
+  }
+
+  streams(): AsyncIterable<AsyncGenerator<Uint8Array>> {
+    const iter = new MyIter(StreamConvertor)
+    this.iterators.push(iter)
+    return iter
+  }
+
+  handleMsg(msg: Message) {
+    this.logger.debug(`${this.uri} handling message`)
+    console.log(`${this.uri} handling message`)
+    for (const iter of this.iterators) {
+      iter.push(msg.data)
+    }
+  }
+
+  close() {
+    for (const iter of this.iterators) {
+      iter.close()
+    }
+  }
+
+  handleStreamingMessage(msg: StreamMessage) {
+    this.logger.debug(`${this.uri} handling streaming message`)
+    const chunks = this.client.receiveStreamMessage(msg.id!)
+    for (const iter of this.iterators) {
+      iter.pushStream(chunks)
+    }
+  }
 }

package/src/reexports.ts

@@ -1,3 +1,6 @@
-export * from "@rdfc/proto";
-export { ClientReadableStreamImpl, ClientReadableStream } from '@grpc/grpc-js/build/src/call.js'
+export * from '@rdfc/proto'
+export {
+  ClientReadableStreamImpl,
+  ClientReadableStream,
+} from '@grpc/grpc-js/build/src/call.js'
 export * as grpc from '@grpc/grpc-js'

package/src/runner.ts

@@ -1,8 +1,8 @@
 import {
-    OrchestratorMessage,
-    Processor,
-    RunnerClient,
-    RunnerMessage,
+  OrchestratorMessage,
+  Processor,
+  RunnerClient,
+  RunnerMessage,
 } from '@rdfc/proto'
 import { Reader, ReaderInstance } from './reader'
 import { Writer, WriterInstance } from './writer'
@@ -11,172 +11,189 @@ import { Logger } from 'winston'
 
 import winston from 'winston'
 import { RpcTransport } from './logger'
-import { parse_jsonld } from './jsonld'
 import { Cont, empty, extractShapes, Shapes } from 'rdf-lens'
 import { NamedNode, Parser } from 'n3'
 import { createNamespace, createUriAndTermNamespace, RDF } from '@treecg/types'
 import { Quad, Term } from '@rdfjs/types'
 
 const RDFL = createUriAndTermNamespace(
-    'https://w3id.org/rdf-lens/ontology#',
-    'CBD',
-    'Path',
-    'PathLens',
-    'Context',
-    'TypedExtract',
-    'EnvVariable',
-    'envKey',
-    'envDefault',
-    'datatype',
+  'https://w3id.org/rdf-lens/ontology#',
+  'CBD',
+  'Path',
+  'PathLens',
+  'Context',
+  'TypedExtract',
+  'EnvVariable',
+  'envKey',
+  'envDefault',
+  'datatype',
 )
 
 const RDFC = createNamespace(
-    'https://w3id.org/rdf-connect#',
-    (x) => x,
-    'Reader',
-    'Writer',
+  'https://w3id.org/rdf-connect#',
+  (x) => x,
+  'Reader',
+  'Writer',
 )
 
 export type Writable = (msg: OrchestratorMessage) => Promise<unknown>
 
 type ProcessorConfig = {
-    location: string
-    file: string
-    clazz?: string
+  location: string
+  file: string
+  clazz?: string
 }
 
-export type FullProc<C extends Proc<unknown>> = C extends Proc<infer T> ? T & C : unknown;
+export type FullProc<C extends Proc<unknown>> =
+  C extends Proc<infer T> ? T & C : unknown
 export class Runner {
-    private readonly readers: { [uri: string]: ReaderInstance[] } = {}
-    private readonly writers: { [uri: string]: WriterInstance[] } = {}
-    private readonly client: RunnerClient
-    private readonly write: Writable
-    private readonly logger: Logger
-    private shapes: Shapes
-    private quads: Quad[] = []
-
-    private readonly uri: string
-
-    private readonly processors: Proc<unknown>[] = []
-    private readonly processorTransforms: Promise<unknown>[] = []
-
-    constructor(
-        client: RunnerClient,
-        write: Writable,
-        uri: string,
-        logger: Logger,
-    ) {
-        this.client = client
-        this.write = write
-        this.uri = uri
-        this.logger = logger
+  private readonly readers: { [uri: string]: ReaderInstance[] } = {}
+  private readonly writers: { [uri: string]: WriterInstance[] } = {}
+  private readonly client: RunnerClient
+  private readonly write: Writable
+  private readonly logger: Logger
+  private shapes: Shapes
+  private quads: Quad[] = []
+
+  private readonly uri: string
+
+  private readonly processors: Proc<unknown>[] = []
+  private readonly processorTransforms: Promise<unknown>[] = []
+
+  constructor(
+    client: RunnerClient,
+    write: Writable,
+    uri: string,
+    logger: Logger,
+  ) {
+    this.client = client
+    this.write = write
+    this.uri = uri
+    this.logger = logger
+  }
+
+  async addProcessor<P extends Proc<unknown>>(
+    proc: Processor,
+  ): Promise<FullProc<P>> {
+    const procLogger = winston.createLogger({
+      transports: [
+        new RpcTransport({
+          entities: [proc.uri, this.uri],
+          stream: this.client.logStream(() => { }),
+        }),
+      ],
+    })
+
+    const ty = JSON.stringify(
+      this.quads
+        .filter(
+          (x) =>
+            x.subject.value === proc.uri && x.predicate.equals(RDF.terms.type),
+        )
+        .map((x) => x.object.value),
+    )
+    this.logger.info('parsing ' + proc.uri + ' type ' + ty)
+    const args = this.shapes.lenses[RDFL.TypedExtract].execute({
+      id: new NamedNode(proc.uri),
+      quads: this.quads,
+    })
+
+    const config: ProcessorConfig = JSON.parse(proc.config)
+    // const url = new URL(config.location)
+    // process.chdir(url.pathname)
+    const jsProgram = await import(config.file)
+    const clazz = jsProgram[config.clazz || 'default']
+    const instance: Proc<unknown> = new clazz(args, procLogger)
+    await instance.init()
+
+    this.logger.info('inited ' + proc.uri + ' type ' + ty)
+
+    this.processors.push(instance)
+    this.processorTransforms.push(instance.transform())
+
+    await this.write({ init: { uri: proc.uri } })
+
+    return <FullProc<P>>instance
+  }
+
+  async start() {
+    try {
+      await Promise.all(this.processors.map((x) => x.produce()))
+      await Promise.all(this.processorTransforms)
+    } catch (ex: unknown) {
+      this.logger.error("Start failed: " + JSON.stringify(ex))
     }
+  }
+
+  createWriter(uri: Term): Writer {
+    const ids = uri.value
 
-    async addProcessor<P extends Proc<unknown>>(proc: Processor): Promise<FullProc<P>> {
-        const procLogger = winston.createLogger({
-            transports: [
-                new RpcTransport({
-                    entities: [proc.uri, this.uri],
-                    stream: this.client.logStream(() => { }),
-                }),
-            ],
-        })
-
-        const ty = JSON.stringify(this.quads.filter(x => x.subject.value === proc.uri && x.predicate.equals(RDF.terms.type)).map(x => x.object.value))
-        this.logger.info("parsing " + proc.uri + " type " + ty)
-        const args = this.shapes.lenses[RDFL.TypedExtract].execute({
-            id: new NamedNode(proc.uri),
-            quads: this.quads,
-        })
-
-        const config: ProcessorConfig = JSON.parse(proc.config)
-        // const url = new URL(config.location)
-        // process.chdir(url.pathname)
-        const jsProgram = await import(config.file)
-        const clazz = jsProgram[config.clazz || 'default']
-        const instance: Proc<unknown> = new clazz(args, procLogger)
-        await instance.init()
-
-        this.logger.info("inited " + proc.uri + " type " + ty)
-
-        this.processors.push(instance)
-        this.processorTransforms.push(instance.transform())
-
-        await this.write({ init: { uri: proc.uri } })
-
-        return <FullProc<P>>instance;
+    if (this.writers[ids] === undefined) {
+      this.writers[ids] = []
     }
+    const writer = new WriterInstance(ids, this.client, this.write, this.logger)
+    this.writers[ids].push(writer)
+    return writer
+  }
 
-    async start() {
-        await Promise.all(this.processors.map((x) => x.produce()))
-        await Promise.all(this.processorTransforms)
+  createReader(uri: Term): Reader {
+    const ids = uri.value
+
+    if (this.readers[ids] === undefined) {
+      this.readers[ids] = []
+    }
+    const reader = new ReaderInstance(ids, this.client, this.logger)
+    this.readers[ids].push(reader)
+    return reader
+  }
+
+  async handleOrchMessage(msg: RunnerMessage) {
+    if (msg.msg) {
+      this.logger.debug('Handling data msg for ' + msg.msg.channel)
+      for (const reader of this.readers[msg.msg.channel] || []) {
+        reader.handleMsg(msg.msg)
+      }
     }
 
-    createWriter(uri: Term): Writer {
-        const ids = uri.value;
-
-        if (this.writers[ids] === undefined) {
-            this.writers[ids] = []
-        }
-        const writer = new WriterInstance(
-            ids,
-            this.client,
-            this.write,
-            this.logger,
-        )
-        this.writers[ids].push(writer)
-        return writer
+    if (msg.streamMsg) {
+      for (const reader of this.readers[msg.streamMsg.channel] || []) {
+        reader.handleStreamingMessage(msg.streamMsg)
+      }
     }
 
-    createReader(uri: Term): Reader {
-        const ids = uri.value;
+    if (msg.close) {
+      const uri = msg.close.channel
+
+      for (const reader of this.readers[uri] || []) {
+        reader.close()
+      }
 
-        if (this.readers[ids] === undefined) {
-            this.readers[ids] = []
-        }
-        const reader = new ReaderInstance(ids, this.client, this.logger)
-        this.readers[ids].push(reader)
-        return reader
+      for (const writer of this.writers[uri] || []) {
+        await writer.close(true)
+      }
     }
 
-    async handleOrchMessage(msg: RunnerMessage) {
-        if (msg.msg) {
-            this.logger.debug('Handling data msg for ' + msg.msg.channel)
-            for (const reader of this.readers[msg.msg.channel] || []) {
-                reader.handleMsg(msg.msg)
-            }
-        }
-
-        if (msg.streamMsg) {
-            for (const reader of this.readers[msg.streamMsg.channel] || []) {
-                reader.handleStreamingMessage(msg.streamMsg)
-            }
-        }
-
-        if (msg.close) {
-            const uri = msg.close.channel
-
-            for (const reader of this.readers[uri] || []) {
-                reader.close()
-            }
-
-            for (const writer of this.writers[uri] || []) {
-                await writer.close(true)
-            }
-        }
-
-        if (msg.pipeline) {
-            // here
-            const quads = new Parser().parse(msg.pipeline);
-            this.shapes = extractShapes(quads, {
-                [RDFC.Reader]: (x: Cont) => this.createReader(x.id),
-                [RDFC.Writer]: (x: Cont) => this.createWriter(x.id)
-            }, {
-                [RDFC.Reader]: empty<Cont>(),
-                [RDFC.Writer]: empty<Cont>()
-            });
-            this.quads = quads;
-            this.logger.info("extracted shapes " + JSON.stringify(Object.keys(this.shapes.lenses)));
-        }
+    if (msg.pipeline) {
+      try {
+
+        // here
+        const quads = new Parser().parse(msg.pipeline)
+        this.shapes = extractShapes(
+          quads,
+          {
+            [RDFC.Reader]: (x: Cont) => this.createReader(x.id),
+            [RDFC.Writer]: (x: Cont) => this.createWriter(x.id),
+          },
+          {
+            [RDFC.Reader]: empty<Cont>(),
+            [RDFC.Writer]: empty<Cont>(),
+          },
+        )
+        this.quads = quads
+      }
+      catch (ex: unknown) {
+        this.logger.error("Pipeline failed: " + JSON.stringify(ex))
+      }
     }
+  }
 }