@rdfc/js-runner 1.0.0 → 2.0.0-alpha.10

package/src/reexports.ts

@@ -0,0 +1,6 @@
+export * from '@rdfc/proto'
+export {
+  ClientReadableStreamImpl,
+  ClientReadableStream,
+} from '@grpc/grpc-js/build/src/call.js'
+export * as grpc from '@grpc/grpc-js'

package/src/runner.ts

@@ -0,0 +1,197 @@
+import {
+  OrchestratorMessage,
+  Processor,
+  RunnerClient,
+  RunnerMessage,
+} from '@rdfc/proto'
+import { Reader, ReaderInstance } from './reader'
+import { Writer, WriterInstance } from './writer'
+import { Processor as Proc } from './processor'
+import { Logger } from 'winston'
+
+import winston from 'winston'
+import { RpcTransport } from './logger'
+import { Cont, empty, extractShapes, Shapes } from 'rdf-lens'
+import { NamedNode, Parser } from 'n3'
+import { createNamespace, createUriAndTermNamespace, RDF } from '@treecg/types'
+import { Quad, Term } from '@rdfjs/types'
+
+const RDFL = createUriAndTermNamespace(
+  'https://w3id.org/rdf-lens/ontology#',
+  'CBD',
+  'Path',
+  'PathLens',
+  'Context',
+  'TypedExtract',
+  'EnvVariable',
+  'envKey',
+  'envDefault',
+  'datatype',
+)
+
+const RDFC = createNamespace(
+  'https://w3id.org/rdf-connect#',
+  (x) => x,
+  'Reader',
+  'Writer',
+)
+
+export type Writable = (msg: OrchestratorMessage) => Promise<unknown>
+
+type ProcessorConfig = {
+  location: string
+  file: string
+  clazz?: string
+}
+
+export type FullProc<C extends Proc<unknown>> =
+  C extends Proc<infer T> ? T & C : unknown
+export class Runner {
+  private readonly readers: { [uri: string]: ReaderInstance[] } = {}
+  private readonly writers: { [uri: string]: WriterInstance[] } = {}
+  private readonly client: RunnerClient
+  private readonly write: Writable
+  private readonly logger: Logger
+  private shapes: Shapes
+  private quads: Quad[] = []
+
+  private readonly uri: string
+
+  private readonly processors: Proc<unknown>[] = []
+  private readonly processorTransforms: Promise<unknown>[] = []
+
+  constructor(
+    client: RunnerClient,
+    write: Writable,
+    uri: string,
+    logger: Logger,
+  ) {
+    this.client = client
+    this.write = write
+    this.uri = uri
+    this.logger = logger
+  }
+
+  async addProcessor<P extends Proc<unknown>>(
+    proc: Processor,
+  ): Promise<FullProc<P>> {
+    const procLogger = winston.createLogger({
+      transports: [
+        new RpcTransport({
+          entities: [proc.uri, this.uri],
+          stream: this.client.logStream(() => {}),
+        }),
+      ],
+    })
+
+    const ty = JSON.stringify(
+      this.quads
+        .filter(
+          (x) =>
+            x.subject.value === proc.uri && x.predicate.equals(RDF.terms.type),
+        )
+        .map((x) => x.object.value),
+    )
+    this.logger.info('parsing ' + proc.uri + ' type ' + ty)
+    const args = this.shapes.lenses[RDFL.TypedExtract].execute({
+      id: new NamedNode(proc.uri),
+      quads: this.quads,
+    })
+
+    const config: ProcessorConfig = JSON.parse(proc.config)
+    // const url = new URL(config.location)
+    // process.chdir(url.pathname)
+    const jsProgram = await import(config.file)
+    const clazz = jsProgram[config.clazz || 'default']
+    const instance: Proc<unknown> = new clazz(args, procLogger)
+    await instance.init()
+
+    this.logger.info('inited ' + proc.uri + ' type ' + ty)
+
+    this.processors.push(instance)
+    this.processorTransforms.push(instance.transform())
+
+    await this.write({ init: { uri: proc.uri } })
+
+    return <FullProc<P>>instance
+  }
+
+  async start() {
+    try {
+      await Promise.all(this.processors.map((x) => x.produce()))
+      await Promise.all(this.processorTransforms)
+    } catch (ex: unknown) {
+      this.logger.error('Start failed: ' + JSON.stringify(ex))
+    }
+  }
+
+  createWriter(uri: Term): Writer {
+    const ids = uri.value
+
+    if (this.writers[ids] === undefined) {
+      this.writers[ids] = []
+    }
+    const writer = new WriterInstance(ids, this.client, this.write, this.logger)
+    this.writers[ids].push(writer)
+    return writer
+  }
+
+  createReader(uri: Term): Reader {
+    const ids = uri.value
+
+    if (this.readers[ids] === undefined) {
+      this.readers[ids] = []
+    }
+    const reader = new ReaderInstance(ids, this.client, this.logger)
+    this.readers[ids].push(reader)
+    return reader
+  }
+
+  async handleOrchMessage(msg: RunnerMessage) {
+    if (msg.msg) {
+      this.logger.debug('Handling data msg for ' + msg.msg.channel)
+      for (const reader of this.readers[msg.msg.channel] || []) {
+        reader.handleMsg(msg.msg)
+      }
+    }
+
+    if (msg.streamMsg) {
+      for (const reader of this.readers[msg.streamMsg.channel] || []) {
+        reader.handleStreamingMessage(msg.streamMsg)
+      }
+    }
+
+    if (msg.close) {
+      const uri = msg.close.channel
+
+      for (const reader of this.readers[uri] || []) {
+        reader.close()
+      }
+
+      for (const writer of this.writers[uri] || []) {
+        await writer.close(true)
+      }
+    }
+
+    if (msg.pipeline) {
+      try {
+        // here
+        const quads = new Parser().parse(msg.pipeline)
+        this.shapes = extractShapes(
+          quads,
+          {
+            [RDFC.Reader]: (x: Cont) => this.createReader(x.id),
+            [RDFC.Writer]: (x: Cont) => this.createWriter(x.id),
+          },
+          {
+            [RDFC.Reader]: empty<Cont>(),
+            [RDFC.Writer]: empty<Cont>(),
+          },
+        )
+        this.quads = quads
+      } catch (ex: unknown) {
+        this.logger.error('Pipeline failed: ' + JSON.stringify(ex))
+      }
+    }
+  }
+}

package/src/testUtils.ts

@@ -0,0 +1,196 @@
+import { DataChunk, RunnerClient } from "@rdfc/proto";
+import * as grpc from "@grpc/grpc-js";
+import { ClientReadableStream } from "@grpc/grpc-js";
+import { ClientReadableStreamImpl, OrchestratorMessage } from "./reexports";
+import { extractShapes } from "rdf-lens";
+import { NamedNode, Parser, Writer } from "n3";
+import { readFile } from "fs/promises";
+import winston, { createLogger } from "winston";
+import { WriterInstance } from "./writer";
+import { ReaderInstance } from "./reader";
+import { Processor } from "./processor";
+import { FullProc, Runner } from "./runner";
+import { Quad, Term } from "@rdfjs/types";
+import { createTermNamespace } from "@treecg/types";
+import { expect } from "vitest";
+
+export async function getProcessorShape(
+    baseIRI = process.cwd() + "/node_modules/@rdfc/js-runner/index.ttl",
+) {
+    const configFile = await readFile(baseIRI, { encoding: "utf8" });
+    const configQuads = new Parser().parse(configFile);
+    const shapes = extractShapes(configQuads);
+
+    return shapes;
+}
+
+export class TestClient extends RunnerClient {
+    next: (stream: ClientReadableStream<DataChunk>) => unknown;
+
+    constructor() {
+        super("localhost:5400", grpc.credentials.createInsecure());
+    }
+
+    nextStream(): Promise<ClientReadableStream<DataChunk>> {
+        return new Promise((res) => (this.next = res));
+    }
+
+    receiveStreamMessage(): ClientReadableStream<DataChunk> {
+        const stream = new ClientReadableStreamImpl<DataChunk>((data: Buffer) => {
+            return { data };
+        });
+        this.next(stream);
+        return stream;
+    }
+}
+
+export async function one<T>(iter: AsyncIterable<T>): Promise<T | undefined> {
+    for await (const item of iter) {
+        return item;
+    }
+}
+
+export const client = new TestClient();
+export const uri = "someUri";
+export const logger = createLogger({
+    transports: new winston.transports.Console({
+        level: process.env["DEBUG"] || "info",
+    }),
+});
+
+export function createWriter(iri = uri): [WriterInstance, ReaderInstance] {
+    const reader = createReader(iri);
+    const writeStream = new WriterInstance(
+        iri,
+        client,
+        async (msg) => {
+            if (msg.msg) {
+                reader.handleMsg(msg.msg);
+            }
+            if (msg.close) {
+                reader.close();
+            }
+        },
+        logger,
+    );
+    return [writeStream, reader];
+}
+
+export function createReader(iri = uri): ReaderInstance {
+    const reader = new ReaderInstance(iri, client, logger);
+    return reader;
+}
+
+const shapeQuads = `
+@prefix rdfc: <https://w3id.org/rdf-connect#>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+[ ] a sh:NodeShape;
+  sh:targetClass <JsProcessorShape>;
+  sh:property [
+    sh:path rdfc:entrypoint;
+    sh:name "location";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+  ], [
+    sh:path rdfc:file;
+    sh:name "file";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+  ], [
+    sh:path rdfc:class;
+    sh:name "clazz";
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+  ].
+`;
+const OWL = createTermNamespace("http://www.w3.org/2002/07/owl#", "imports");
+const processorShapes = extractShapes(new Parser().parse(shapeQuads));
+const base = "https://w3id.org/rdf-connect#";
+
+export async function importFile(file: string): Promise<Quad[]> {
+    const done = new Set<string>();
+    const todo = [new URL("file://" + file)];
+    const quads: Quad[] = [];
+
+    let item = todo.pop();
+    while (item !== undefined) {
+        if (done.has(item.toString())) {
+            item = todo.pop();
+            continue;
+        }
+        done.add(item.toString());
+        if (item.protocol !== "file:") {
+            throw "No supported protocol " + item.protocol;
+        }
+
+        const txt = await readFile(item.pathname, { encoding: "utf8" });
+        const extras = new Parser({ baseIRI: item.toString() }).parse(txt);
+
+        for (const o of extras
+            .filter(
+                (x) =>
+                    x.subject.value === item?.toString() &&
+                    x.predicate.equals(OWL.imports),
+            )
+            .map((x) => x.object.value)) {
+            todo.push(new URL(o));
+        }
+        quads.push(...extras);
+
+        item = todo.pop();
+    }
+
+    return quads;
+}
+
+export async function getProc<T extends Processor<unknown>>(
+    config: string,
+    ty: string,
+    configLocation: string,
+    uri = "http://example.com/ns#processor",
+): Promise<FullProc<T>> {
+    const configQuads = await importFile(configLocation);
+    const procConfig = processorShapes.lenses["JsProcessorShape"].execute({
+        id: new NamedNode(base + ty),
+        quads: configQuads,
+    });
+
+    const msgs: OrchestratorMessage[] = [];
+    const write = async (x: OrchestratorMessage) => {
+        msgs.push(x);
+    };
+    const runner = new Runner(
+        new TestClient(),
+        write,
+        "http://example.com/ns#",
+        logger,
+    );
+    configQuads.push(...new Parser().parse(config));
+    await runner.handleOrchMessage({
+        pipeline: new Writer().quadsToString(configQuads),
+    });
+
+    const proc = await runner.addProcessor<T>({
+        config: JSON.stringify(procConfig),
+        arguments: "",
+        uri,
+    });
+
+    return proc;
+}
+
+export async function checkProcDefinition(file: string, n: string) {
+    const quads = await importFile(file);
+    const procConfig = <{ file: Term; location: string; clazz: string }>(
+        processorShapes.lenses["JsProcessorShape"].execute({
+            id: new NamedNode(base + n),
+            quads: quads,
+        })
+    );
+    expect(procConfig.file, n + " has file").toBeDefined();
+    expect(procConfig.location, n + " has location").toBeDefined();
+    expect(procConfig.clazz, n + " has clazz").toBeDefined();
+}

package/src/util_processors.ts

@@ -0,0 +1,20 @@
+import { Processor } from './processor'
+import { Reader } from './reader'
+
+type LogArgs = {
+  reader: Reader
+}
+
+export class LogProcessor extends Processor<LogArgs> {
+  async init(): Promise<void> {
+    this.logger.info('Init log processor')
+  }
+
+  async transform(this: LogArgs & this): Promise<void> {
+    for await (const msg of this.args.reader.strings()) {
+      this.logger.info('Got msg' + msg)
+    }
+  }
+
+  async produce() {}
+}

package/src/writer.ts

@@ -0,0 +1,90 @@
+import { Id, OrchestratorMessage, RunnerClient } from '@rdfc/proto'
+import { promisify } from 'util'
+import { Logger } from 'winston'
+import { Any } from './reader'
+
+type Writable = (msg: OrchestratorMessage) => Promise<unknown>
+export interface Writer {
+  readonly uri: string
+  buffer(buffer: Uint8Array): Promise<void>
+
+  stream(buffer: AsyncIterable<Uint8Array>): Promise<void>
+  stream<T>(
+    buffer: AsyncIterable<T>,
+    tranform: (x: T) => Uint8Array,
+  ): Promise<void>
+
+  string(buffer: string): Promise<void>
+  any(any: Any): Promise<void>
+  close(): Promise<void>
+}
+const encoder = new TextEncoder()
+export class WriterInstance implements Writer {
+  readonly uri: string
+  private readonly client: RunnerClient
+  private readonly write: Writable
+  private readonly logger: Logger
+
+  constructor(
+    uri: string,
+    client: RunnerClient,
+    write: Writable,
+    logger: Logger,
+  ) {
+    this.client = client
+    this.write = write
+    this.uri = uri
+    this.logger = logger
+  }
+  async any(any: Any): Promise<void> {
+    if ('stream' in any) {
+      await this.stream(any.stream)
+    }
+    if ('buffer' in any) {
+      await this.buffer(any.buffer)
+    }
+    if ('string' in any) {
+      await this.string(any.string)
+    }
+  }
+
+  async buffer(buffer: Uint8Array): Promise<void> {
+    this.logger.debug(`${this.uri} sends buffer ${buffer.length} bytes`)
+    await this.write({ msg: { data: buffer, channel: this.uri } })
+  }
+
+  async stream<T = Uint8Array>(
+    buffer: AsyncIterable<T>,
+    transform?: (x: T) => Uint8Array,
+  ) {
+    const t = transform || ((x: unknown) => <Uint8Array>x)
+    const stream = this.client.sendStreamMessage()
+    const id: Id = await new Promise((res) => stream.once('data', res))
+    this.logger.debug(`${this.uri} streams message with id ${id.id}`)
+    await this.write({ streamMsg: { id, channel: this.uri } })
+
+    const write = promisify(stream.write.bind(stream))
+    for await (const msg of buffer) {
+      await write({ data: t(msg) })
+    }
+
+    this.logger.debug(`${this.uri} is done streaming message with id ${id.id}`)
+    stream.end()
+  }
+
+  async string(msg: string): Promise<void> {
+    this.logger.debug(`${this.uri} sends string ${msg.length} characters`)
+    await this.write({
+      msg: { data: encoder.encode(msg), channel: this.uri },
+    })
+  }
+
+  async close(issued = false): Promise<void> {
+    this.logger.debug(`${this.uri} closes stream`)
+    if (!issued) {
+      await this.write({
+        close: { channel: this.uri },
+      })
+    }
+  }
+}

package/tsconfig.json

@@ -0,0 +1,33 @@
+{
+  "compilerOptions": {
+    "declaration": true,
+    "outDir": "lib/",
+    "lib": ["ES2022"],
+    "target": "ES2022",
+    "module": "ES2022",
+    "moduleResolution": "node",
+    "inlineSourceMap": true, // 🔴 Keeps source maps in JS files
+    "inlineSources": true, // 🔴 Includes the original TS source in maps
+    "preserveConstEnums": true,
+    "downlevelIteration": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "strictFunctionTypes": false,
+    "strictPropertyInitialization": false,
+    "experimentalDecorators": true,
+    "allowSyntheticDefaultImports": true,
+    "incremental": true,
+    "esModuleInterop": true,
+    "removeComments": true
+  },
+  "ts-node": {
+    "esm": true,
+    "experimentalSpecifierResolution": "node"
+  },
+  "tsc-alias": {
+    "resolveFullPaths": true,
+    "verbose": false
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules/**/*", "**/__tests__/**/*"]
+}

package/vite.config.ts

@@ -0,0 +1,10 @@
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({
+  test: {
+    coverage: {
+      enabled: true,
+      include: ['src/'],
+    },
+  },
+})

package/LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2024 Jens Pots
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

package/bin/js-runner.js

@@ -1,4 +0,0 @@
-#!/usr/bin/env node
-import { jsRunner } from "../dist/index.js";
-
-jsRunner().catch((e) => { console.error("Error:", e); console.error(e.stack) });

package/channels/file.ttl

@@ -1,37 +0,0 @@
-@prefix : <https://w3id.org/conn#>.
-@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
-@prefix sh: <http://www.w3.org/ns/shacl#>.
-@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
-
-:FileWriterChannel rdfs:subClassOf :WriterChannel.
-:FileReaderChannel rdfs:subClassOf :ReaderChannel.
-[ ] a sh:NodeShape;
-  sh:targetClass :FileReaderChannel, :FileWriterChannel;
-  sh:property [
-    sh:path :filePath;
-    sh:name "path";
-    sh:maxCount 1;
-    sh:minCount 1;
-    sh:datatype xsd:string;
-    sh:description "Path of the used file";
-  ], [
-    sh:path :fileOnReplace;
-    sh:name "onReplace";
-    sh:maxCount 1;
-    sh:minCount 1;
-    sh:datatype xsd:boolean;
-    sh:description "Boolean indicating if the files is always replaced, or appended to";
-  ], [
-    sh:path :fileReadFirstContent;
-    sh:name "readFirstContent";
-    sh:maxCount 1;
-    sh:datatype xsd:boolean;
-    sh:description "Boolean indicating whether or not the initial content is also a message";
-  ], [
-    sh:path :fileEncoding;
-    sh:name "encoding";
-    sh:maxCount 1;
-    sh:datatype xsd:string;
-    sh:description "The encoding used for the file";
-  ].
-

package/channels/http.ttl

@@ -1,59 +0,0 @@
-@prefix : <https://w3id.org/conn#>.
-@prefix sh: <http://www.w3.org/ns/shacl#>.
-@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
-@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
-
-:HttpReaderChannel rdfs:subClassOf :ReaderChannel.
-:HttpWriterChannel rdfs:subClassOf :WriterChannel.
-[ ] a sh:NodeShape;
-  sh:targetClass :HttpReaderChannel;
-  sh:property [
-    sh:path :httpEndpoint;
-    sh:name "endpoint";
-    sh:maxCount 1;
-    sh:datatype xsd:string;
-    sh:description "Used endpoint";
-  ], [
-    sh:path :httpPort;
-    sh:name "port";
-    sh:maxCount 1;
-    sh:datatype xsd:integer;
-    sh:description "Used port";
-  ], [
-    sh:path :binary;
-    sh:name "binary";
-    sh:maxCount 1;
-    sh:datatype xsd:boolean;
-    sh:description "Stream raw bytes if true";
-  ], [
-    sh:path :waitHandled;
-    sh:name "waitHandled";
-    sh:maxCount 1;
-    sh:datatype xsd:boolean;
-    sh:description "Wait for handlers to be have handled the incoming message";
-  ], [
-    sh:path :responseCode;
-    sh:name "responseCode";
-    sh:maxCount 1;
-    sh:datatype xsd:integer;
-    sh:description "Specify the expected response code (default 200)";
-  ].
-
-[ ] a sh:NodeShape;
-  sh:targetClass :HttpWriterChannel;
-  sh:property [
-    sh:path :httpEndpoint;
-    sh:name "endpoint";
-    sh:maxCount 1;
-    sh:minCount 1;
-    sh:datatype xsd:string;
-    sh:description "Used endpoint";
-  ], [
-    sh:path :httpMethod;
-    sh:name "method";
-    sh:minCount 1;
-    sh:maxCount 1;
-    sh:datatype xsd:string;
-    sh:description "Used method";
-  ].
-