@rdfc/js-runner 1.0.0-alpha.7 → 2.0.0-alpha.1

package/src/runner.ts

@@ -0,0 +1,186 @@
+import {
+  OrchestratorMessage,
+  Processor,
+  RunnerClient,
+  RunnerMessage,
+} from '@rdfc/proto'
+import { Reader, ReaderInstance } from './reader'
+import { Writer, WriterInstance } from './writer'
+import { Processor as Proc } from './processor'
+import { Logger } from 'winston'
+
+import winston from 'winston'
+import { RpcTransport } from './logger'
+import { parse_jsonld } from './jsonld'
+import { Cont, empty, extractShapes, Shapes } from 'rdf-lens'
+import { NamedNode, Parser } from 'n3'
+import { createNamespace, createUriAndTermNamespace, RDF } from '@treecg/types'
+import { Quad, Term } from '@rdfjs/types'
+
+const RDFL = createUriAndTermNamespace(
+  'https://w3id.org/rdf-lens/ontology#',
+  'CBD',
+  'Path',
+  'PathLens',
+  'Context',
+  'TypedExtract',
+  'EnvVariable',
+  'envKey',
+  'envDefault',
+  'datatype',
+)
+
+const RDFC = createNamespace(
+  'https://w3id.org/rdf-connect/ontology#',
+  (x) => x,
+  'Reader',
+  'Writer',
+)
+
+type Writable = (msg: OrchestratorMessage) => Promise<unknown>
+
+type ProcessorConfig = {
+  location: string
+  file: string
+  clazz?: string
+}
+
+export class Runner {
+  private readonly readers: { [uri: string]: ReaderInstance[] } = {}
+  private readonly writers: { [uri: string]: WriterInstance[] } = {}
+  private readonly client: RunnerClient
+  private readonly write: Writable
+  private readonly logger: Logger
+  private shapes: Shapes
+  private quads: Quad[] = []
+
+  private readonly uri: string
+
+  private readonly processors: Proc<unknown>[] = []
+  private readonly processorTransforms: Promise<unknown>[] = []
+
+  constructor(
+    client: RunnerClient,
+    write: Writable,
+    uri: string,
+    logger: Logger,
+  ) {
+    this.client = client
+    this.write = write
+    this.uri = uri
+    this.logger = logger
+  }
+
+  async addProcessor(proc: Processor) {
+    const procLogger = winston.createLogger({
+      transports: [
+        new RpcTransport({
+          entities: [proc.uri, this.uri],
+          stream: this.client.logStream(() => { }),
+        }),
+      ],
+    })
+
+    const ty = JSON.stringify(this.quads.filter(x => x.subject.value === proc.uri && x.predicate.equals(RDF.terms.type)).map(x => x.object.value))
+    this.logger.info("parsing " + proc.uri + " type " + ty)
+    const args = this.shapes.lenses[RDFL.TypedExtract].execute({
+      id: new NamedNode(proc.uri),
+      quads: this.quads,
+    })
+    // this.logger.info('Parsed ' + JSON.stringify(args))
+
+    // const args = await parse_jsonld(proc.arguments, procLogger, {
+    //   client: this.client,
+    //   write: this.write,
+    //   readers: this.readers,
+    //   writers: this.writers,
+    // })
+
+    const config: ProcessorConfig = JSON.parse(proc.config)
+    const url = new URL(config.location)
+    process.chdir(url.pathname)
+    const jsProgram = await import(config.file)
+    const clazz = jsProgram[config.clazz || 'default']
+    const instance: Proc<unknown> = new clazz(args, procLogger)
+    await instance.init()
+
+    this.logger.info("inited " + proc.uri + " type " + ty)
+    await this.write({ init: { uri: proc.uri } })
+
+    this.processors.push(instance)
+    this.processorTransforms.push(instance.transform())
+  }
+
+  async start() {
+    await Promise.all(this.processors.map((x) => x.produce()))
+    await Promise.all(this.processorTransforms)
+  }
+
+  createWriter(uri: Term): Writer {
+    const ids = uri.value;
+
+    if (this.writers[ids] === undefined) {
+      this.writers[ids] = []
+    }
+    const writer = new WriterInstance(
+      ids,
+      this.client,
+      this.write,
+      this.logger,
+    )
+    this.writers[ids].push(writer)
+    return writer
+  }
+
+  createReader(uri: Term): Reader {
+    const ids = uri.value;
+
+    if (this.readers[ids] === undefined) {
+      this.readers[ids] = []
+    }
+    const reader = new ReaderInstance(ids, this.client, this.logger)
+    this.readers[ids].push(reader)
+    return reader
+  }
+
+  async handleOrchMessage(msg: RunnerMessage) {
+    if (msg.msg) {
+      this.logger.debug('Handling data msg for ' + msg.msg.channel)
+      for (const reader of this.readers[msg.msg.channel] || []) {
+        reader.handleMsg(msg.msg)
+      }
+    }
+
+    if (msg.streamMsg) {
+      for (const reader of this.readers[msg.streamMsg.channel] || []) {
+        reader.handleStreamingMessage(msg.streamMsg)
+      }
+    }
+
+    if (msg.close) {
+      const uri = msg.close.channel
+
+      for (const reader of this.readers[uri] || []) {
+        reader.close()
+      }
+
+      for (const writer of this.writers[uri] || []) {
+        await writer.close(true)
+      }
+    }
+
+    if (msg.pipeline) {
+      // here
+      const quads = new Parser().parse(msg.pipeline);
+      this.shapes = extractShapes(quads, {
+        [RDFC.Reader]: (x: Cont) => this.createReader(x.id),
+        [RDFC.Writer]: (x: Cont) => this.createWriter(x.id)
+      }, {
+        [RDFC.Reader]: empty<Cont>(),
+        [RDFC.Writer]: empty<Cont>()
+      });
+      this.quads = quads;
+      this.logger.info("extracted shapes " + JSON.stringify(Object.keys(this.shapes.lenses)));
+    }
+  }
+}

package/src/util_processors.ts

@@ -0,0 +1,20 @@
+import { Processor } from './processor'
+import { Reader } from './reader'
+
+type LogArgs = {
+  reader: Reader
+}
+
+export class LogProcessor extends Processor<LogArgs> {
+  async init(): Promise<void> {
+    this.logger.info('Init log processor')
+  }
+
+  async transform(this: LogArgs & this): Promise<void> {
+    for await (const msg of this.args.reader.strings()) {
+      this.logger.info('Got msg' + msg)
+    }
+  }
+
+  async produce() {}
+}

package/src/writer.ts

@@ -0,0 +1,89 @@
+import { Id, OrchestratorMessage, RunnerClient } from '@rdfc/proto'
+import { promisify } from 'util'
+import { Logger } from 'winston'
+import { Any } from './reader'
+
+type Writable = (msg: OrchestratorMessage) => Promise<unknown>
+export interface Writer {
+  buffer(buffer: Uint8Array): Promise<void>
+
+  stream(buffer: AsyncIterable<Uint8Array>): Promise<void>
+  stream<T>(
+    buffer: AsyncIterable<T>,
+    tranform: (x: T) => Uint8Array,
+  ): Promise<void>
+
+  string(buffer: string): Promise<void>
+  any(any: Any): Promise<void>
+  close(): Promise<void>
+}
+const encoder = new TextEncoder()
+export class WriterInstance implements Writer {
+  private readonly uri: string
+  private readonly client: RunnerClient
+  private readonly write: Writable
+  private readonly logger: Logger
+
+  constructor(
+    uri: string,
+    client: RunnerClient,
+    write: Writable,
+    logger: Logger,
+  ) {
+    this.client = client
+    this.write = write
+    this.uri = uri
+    this.logger = logger
+  }
+  async any(any: Any): Promise<void> {
+    if ('stream' in any) {
+      await this.stream(any.stream)
+    }
+    if ('buffer' in any) {
+      await this.buffer(any.buffer)
+    }
+    if ('string' in any) {
+      await this.string(any.string)
+    }
+  }
+
+  async buffer(buffer: Uint8Array): Promise<void> {
+    this.logger.debug(`${this.uri} sends buffer ${buffer.length} bytes`)
+    await this.write({ msg: { data: buffer, channel: this.uri } })
+  }
+
+  async stream<T = Uint8Array>(
+    buffer: AsyncIterable<T>,
+    transform?: (x: T) => Uint8Array,
+  ) {
+    const t = transform || ((x: unknown) => <Uint8Array>x)
+    const stream = this.client.sendStreamMessage()
+    const id: Id = await new Promise((res) => stream.once('data', res))
+    this.logger.debug(`${this.uri} streams message with id ${id.id}`)
+    await this.write({ streamMsg: { id, channel: this.uri } })
+
+    const write = promisify(stream.write.bind(stream))
+    for await (const msg of buffer) {
+      await write({ data: t(msg) })
+    }
+
+    this.logger.debug(`${this.uri} is done streaming message with id ${id.id}`)
+    stream.end()
+  }
+
+  async string(msg: string): Promise<void> {
+    this.logger.debug(`${this.uri} sends string ${msg.length} characters`)
+    await this.write({
+      msg: { data: encoder.encode(msg), channel: this.uri },
+    })
+  }
+
+  async close(issued = false): Promise<void> {
+    this.logger.debug(`${this.uri} closes stream`)
+    if (!issued) {
+      await this.write({
+        close: { channel: this.uri },
+      })
+    }
+  }
+}

package/tsconfig.json

@@ -0,0 +1,33 @@
+{
+  "compilerOptions": {
+    "declaration": true,
+    "outDir": "lib/",
+    "lib": ["ES2022"],
+    "target": "ES2022",
+    "module": "ES2022",
+    "moduleResolution": "node",
+    "inlineSourceMap": true, // 🔴 Keeps source maps in JS files
+    "inlineSources": true, // 🔴 Includes the original TS source in maps
+    "preserveConstEnums": true,
+    "downlevelIteration": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "strictFunctionTypes": false,
+    "strictPropertyInitialization": false,
+    "experimentalDecorators": true,
+    "allowSyntheticDefaultImports": true,
+    "incremental": true,
+    "esModuleInterop": true,
+    "removeComments": true
+  },
+  "ts-node": {
+    "esm": true,
+    "experimentalSpecifierResolution": "node"
+  },
+  "tsc-alias": {
+    "resolveFullPaths": true,
+    "verbose": false
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules/**/*", "**/__tests__/**/*"]
+}

package/vite.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({
+  test: {
+    coverage: {
+      enabled: true,
+      include: ['src/'],
+    },
+  },
+})

package/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2024 Jens Pots
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

package/bin/js-runner.js

@@ -1,4 +0,0 @@
-#!/usr/bin/env node
-import { jsRunner } from "../dist/index.js";
-
-jsRunner().catch((e) => { console.error("Error:", e); console.error(e.stack); });

package/channels/file.ttl

@@ -1,37 +0,0 @@
-@prefix : <https://w3id.org/conn#>.
-@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
-@prefix sh: <http://www.w3.org/ns/shacl#>.
-@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
-
-:FileWriterChannel rdfs:subClassOf :WriterChannel.
-:FileReaderChannel rdfs:subClassOf :ReaderChannel.
-[ ] a sh:NodeShape;
-  sh:targetClass :FileReaderChannel, :FileWriterChannel;
-  sh:property [
-    sh:path :filePath;
-    sh:name "path";
-    sh:maxCount 1;
-    sh:minCount 1;
-    sh:datatype xsd:string;
-    sh:description "Path of the used file";
-  ], [
-    sh:path :fileOnReplace;
-    sh:name "onReplace";
-    sh:maxCount 1;
-    sh:minCount 1;
-    sh:datatype xsd:boolean;
-    sh:description "Boolean indicating if the files is always replaced, or appended to";
-  ], [
-    sh:path :fileReadFirstContent;
-    sh:name "readFirstContent";
-    sh:maxCount 1;
-    sh:datatype xsd:boolean;
-    sh:description "Boolean indicating whether or not the initial content is also a message";
-  ], [
-    sh:path :fileEncoding;
-    sh:name "encoding";
-    sh:maxCount 1;
-    sh:datatype xsd:string;
-    sh:description "The encoding used for the file";
-  ].
-

package/channels/http.ttl

@@ -1,59 +0,0 @@
-@prefix : <https://w3id.org/conn#>.
-@prefix sh: <http://www.w3.org/ns/shacl#>.
-@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
-@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
-
-:HttpReaderChannel rdfs:subClassOf :ReaderChannel.
-:HttpWriterChannel rdfs:subClassOf :WriterChannel.
-[ ] a sh:NodeShape;
-  sh:targetClass :HttpReaderChannel;
-  sh:property [
-    sh:path :httpEndpoint;
-    sh:name "endpoint";
-    sh:maxCount 1;
-    sh:datatype xsd:string;
-    sh:description "Used endpoint";
-  ], [
-    sh:path :httpPort;
-    sh:name "port";
-    sh:maxCount 1;
-    sh:datatype xsd:integer;
-    sh:description "Used port";
-  ], [
-    sh:path :binary;
-    sh:name "binary";
-    sh:maxCount 1;
-    sh:datatype xsd:boolean;
-    sh:description "Stream raw bytes if true";
-  ], [
-    sh:path :waitHandled;
-    sh:name "waitHandled";
-    sh:maxCount 1;
-    sh:datatype xsd:boolean;
-    sh:description "Wait for handlers to be have handled the incoming message";
-  ], [
-    sh:path :responseCode;
-    sh:name "responseCode";
-    sh:maxCount 1;
-    sh:datatype xsd:integer;
-    sh:description "Specify the expected response code (default 200)";
-  ].
-
-[ ] a sh:NodeShape;
-  sh:targetClass :HttpWriterChannel;
-  sh:property [
-    sh:path :httpEndpoint;
-    sh:name "endpoint";
-    sh:maxCount 1;
-    sh:minCount 1;
-    sh:datatype xsd:string;
-    sh:description "Used endpoint";
-  ], [
-    sh:path :httpMethod;
-    sh:name "method";
-    sh:minCount 1;
-    sh:maxCount 1;
-    sh:datatype xsd:string;
-    sh:description "Used method";
-  ].
-

package/channels/kafka.ttl

@@ -1,98 +0,0 @@
-@prefix : <https://w3id.org/conn#>.
-@prefix sh: <http://www.w3.org/ns/shacl#>.
-@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
-@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
-
-:KafkaReaderChannel rdfs:subClassOf :ReaderChannel.
-:KafkaReaderChannelShape a sh:NodeShape;
-  sh:targetClass :KafkaReaderChannel;
-  sh:property [
-    sh:path :kafkaTopic;
-    sh:name "topic";
-    sh:minCount 1;
-    sh:maxCount 1;
-    sh:class :KafkaTopic;
-    sh:description "Information about the used topic";
-  ], [
-    sh:path :kafkaConsumer;
-    sh:name "consumer";
-    sh:minCount 1;
-    sh:maxCount 1;
-    sh:class :KafkaConsumer;
-    sh:description "Information about the kafka consumer";
-  ], [
-    sh:path :kafkaBroker;
-    sh:name "broker";
-    sh:minCount 1;
-    sh:maxCount 1;
-    sh:class :KafkaBroker;
-    sh:description "Information about the kafka broker";
-  ].
-
-[ ] a sh:NodeShape;
-  sh:targetClass :KafkaTopic;
-  sh:property [
-    sh:path :topicName;
-    sh:name "name";
-    sh:minCount 1;
-    sh:maxCount 1;
-    sh:datatype xsd:string;
-    sh:description "Name of the topic";
-  ], [
-    sh:path :topicFromBeginning;
-    sh:name "fromBeginning";
-    sh:maxCount 1;
-    sh:datatype xsd:boolean;
-    sh:description "Restart the topic from the beginning";
-  ].
-
-[ ] a sh:NodeShape;
-  sh:targetClass :KafkaConsumer;
-  sh:property [
-    sh:path :groupId;
-    sh:name "groupId";
-    sh:minCount 1;
-    sh:maxCount 1;
-    sh:datatype xsd:string;
-    sh:description "Group identifier";
-  ].
-
-[ ] a sh:NodeShape;
-  sh:targetClass :KafkaBroker;
-  sh:property [
-    sh:path :brokerHost;
-    sh:name "hosts";
-    sh:minCount 1;
-    sh:datatype xsd:string;
-    sh:description "Broker host to connect to";
-  ].
-
-:KafkaWriterChannel rdfs:subClassOf :WriterChannel.
-[ ] a sh:NodeShape;
-  sh:targetClass :KafkaWriterChannel;
-  sh:property [
-    sh:path :kafkaTopic;
-    sh:name "topic";
-    sh:minCount 1;
-    sh:maxCount 1;
-    sh:class :KafkaTopic;
-    sh:description "Information about the used topic";
-  ], [
-    sh:path :kafkaBroker;
-    sh:name "broker";
-    sh:minCount 1;
-    sh:maxCount 1;
-    sh:class :KafkaBroker;
-    sh:description "Information about the kafka broker";
-  ], [
-    sh:path :kafkaProducer;
-    sh:name "producer";
-    sh:minCount 1;
-    sh:maxCount 1;
-    sh:class :KafkaProducer;
-    sh:description "Information about the kafka producer";
-  ].
-
-[ ] a sh:NodeShape;
-  sh:targetClass :KafkaProducer.
-

package/channels/ws.ttl

@@ -1,33 +0,0 @@
-@prefix : <https://w3id.org/conn#>.
-@prefix sh: <http://www.w3.org/ns/shacl#>.
-@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
-@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
-
-:WsReaderChannel rdfs:subClassOf :ReaderChannel.
-:WsWriterChannel rdfs:subClassOf :WriterChannel.
-[ ] a sh:NodeShape;
-  sh:targetClass :WsReaderChannel;
-  sh:property [
-    sh:name "url";
-    sh:path :wsUrl;
-    sh:datatype xsd:string;
-    sh:maxCount 1;
-    sh:minCount 1;
-  ].
-
-[ ] a sh:NodeShape;
-  sh:targetClass :WsWriterChannel;
-  sh:property [
-    sh:name "port";
-    sh:path :wsPort;
-    sh:datatype xsd:integer;
-    sh:maxCount 1;
-    sh:minCount 1;
-  ], [
-    sh:name "host";
-    sh:path :wsHost;
-    sh:datatype xsd:string;
-    sh:maxCount 1;
-    sh:minCount 1;
-  ].
-

package/dist/args.d.ts

@@ -1,4 +0,0 @@
-export type Args = {
-    input: string;
-};
-export declare function getArgs(): Args;

package/dist/args.js

@@ -1,59 +0,0 @@
-import commandLineArgs from "command-line-args";
-import commandLineUsage from "command-line-usage";
-const optionDefinitions = [
-    {
-        name: "input",
-        type: String,
-        defaultOption: true,
-        summary: "Specify what input file to start up",
-    },
-    {
-        name: "help",
-        alias: "h",
-        type: Boolean,
-        description: "Display this help message",
-    },
-];
-const sections = [
-    {
-        header: "Js-runner",
-        content: "JS-runner is part of the {italic connector architecture}. Starting from an input file start up all JsProcessors that are defined. Please do not use blank nodes, skolemize your data somewhere else!",
-    },
-    {
-        header: "Synopsis",
-        content: "$ js-runner <input>",
-    },
-    {
-        header: "Command List",
-        content: [
-            { name: "input", summary: "Specify what input file to start up" },
-        ],
-    },
-    {
-        optionList: [optionDefinitions[1]],
-    },
-];
-function validArgs(args) {
-    if (!args.input)
-        return false;
-    return true;
-}
-function printUsage() {
-    const usage = commandLineUsage(sections);
-    console.log(usage);
-    process.exit(0);
-}
-export function getArgs() {
-    let args;
-    try {
-        args = commandLineArgs(optionDefinitions);
-    }
-    catch (e) {
-        console.error(e);
-        printUsage();
-    }
-    if (args.help || !validArgs(args)) {
-        printUsage();
-    }
-    return args;
-}

package/dist/connectors/file.d.ts

@@ -1,15 +0,0 @@
-import { ReaderConstructor, WriterConstructor } from "../connectors.js";
-export interface FileReaderConfig {
-    path: string;
-    onReplace: boolean;
-    readFirstContent?: boolean;
-    encoding?: string;
-}
-export interface FileWriterConfig {
-    path: string;
-    onReplace: boolean;
-    readFirstContent?: boolean;
-    encoding?: string;
-}
-export declare const startFileStreamReader: ReaderConstructor<FileReaderConfig>;
-export declare const startFileStreamWriter: WriterConstructor<FileWriterConfig>;