@rdfc/js-runner 0.2.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Jens Pots
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,42 @@
+# Js-runner
+
+[![Bun CI](https://github.com/ajuvercr/js-runner/actions/workflows/build-test.yml/badge.svg)](https://github.com/ajuvercr/js-runner/actions/workflows/build-test.yml)
+
+Executor for a javascript workbench. Starting from a turtle file describing the workbench.
+
+## Process definition
+
+Each js process must have `js:file`, `js:function` and `js:mapping` objects. 
+
+- `js:file` points to the location of the main javascript file, containing the function.
+- `js:location` points to the starting location for `js:file` relative from the current file.
+- `js:function` points to the function name in the file.
+- `js:mapping` is a `fno:Mapping` object that links properties to function arguments.
+
+
+When you declare a new js process, it is required to add a shacl shape.
+Each `sh:property` is accounted for, noting the type `sh:class` or `sh:datatype`.
+
+Example definitions are available in `processor/configs/*.ttl`.
+
+
+## Pipeline configuration
+
+In a js pipeline you can use all declared js processes, as defined in their shacl shapes.
+
+An example can be found in `input.ttl`, here a `js:Send` process and a `js:Resc` process are defined.
+`js:Send` takes in a message to send, and a channel to send it to.
+`js:Resc` only takes a channel to read from.
+
+(implementation can be found in `procossor/test.js`)
+
+Note: currently websockets are configured, but changing the configuration to use the JsReaderChannel and JsWriterChannel will work too, even without a serialization step.
+
+You can execute this pipeline with
+```bash
+$ tsc
+$ bun bin/js-runner.js input.ttl 
+```
+
+This example input configuration file uses `owl:imports` to specify additional configuration files. 
+

package/channels/file.ttl

@@ -0,0 +1,37 @@
+@prefix : <https://w3id.org/conn#>.
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+
+:FileWriterChannel rdfs:subClassOf :WriterChannel.
+:FileReaderChannel rdfs:subClassOf :ReaderChannel.
+[ ] a sh:NodeShape;
+  sh:targetClass :FileReaderChannel, :FileWriterChannel;
+  sh:property [
+    sh:path :filePath;
+    sh:name "path";
+    sh:maxCount 1;
+    sh:minCount 1;
+    sh:datatype xsd:string;
+    sh:description "Path of the used file";
+  ], [
+    sh:path :fileOnReplace;
+    sh:name "onReplace";
+    sh:maxCount 1;
+    sh:minCount 1;
+    sh:datatype xsd:boolean;
+    sh:description "Boolean indicating if the files is always replaced, or appended to";
+  ], [
+    sh:path :fileReadFirstContent;
+    sh:name "readFirstContent";
+    sh:maxCount 1;
+    sh:datatype xsd:boolean;
+    sh:description "Boolean indicating whether or not the initial content is also a message";
+  ], [
+    sh:path :fileEncoding;
+    sh:name "encoding";
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+    sh:description "The encoding used for the file";
+  ].
+

package/channels/http.ttl

@@ -0,0 +1,59 @@
+@prefix : <https://w3id.org/conn#>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
+
+:HttpReaderChannel rdfs:subClassOf :ReaderChannel.
+:HttpWriterChannel rdfs:subClassOf :WriterChannel.
+[ ] a sh:NodeShape;
+  sh:targetClass :HttpReaderChannel;
+  sh:property [
+    sh:path :httpEndpoint;
+    sh:name "endpoint";
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+    sh:description "Used endpoint";
+  ], [
+    sh:path :httpPort;
+    sh:name "port";
+    sh:maxCount 1;
+    sh:datatype xsd:integer;
+    sh:description "Used port";
+  ], [
+    sh:path :binary;
+    sh:name "binary";
+    sh:maxCount 1;
+    sh:datatype xsd:boolean;
+    sh:description "Stream raw bytes if true";
+  ], [
+    sh:path :waitHandled;
+    sh:name "waitHandled";
+    sh:maxCount 1;
+    sh:datatype xsd:boolean;
+    sh:description "Wait for handlers to be have handled the incoming message";
+  ], [
+    sh:path :responseCode;
+    sh:name "responseCode";
+    sh:maxCount 1;
+    sh:datatype xsd:integer;
+    sh:description "Specify the expected response code (default 200)";
+  ].
+
+[ ] a sh:NodeShape;
+  sh:targetClass :HttpWriterChannel;
+  sh:property [
+    sh:path :httpEndpoint;
+    sh:name "endpoint";
+    sh:maxCount 1;
+    sh:minCount 1;
+    sh:datatype xsd:string;
+    sh:description "Used endpoint";
+  ], [
+    sh:path :httpMethod;
+    sh:name "method";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+    sh:description "Used method";
+  ].
+

package/channels/kafka.ttl

@@ -0,0 +1,98 @@
+@prefix : <https://w3id.org/conn#>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
+
+:KafkaReaderChannel rdfs:subClassOf :ReaderChannel.
+:KafkaReaderChannelShape a sh:NodeShape;
+  sh:targetClass :KafkaReaderChannel;
+  sh:property [
+    sh:path :kafkaTopic;
+    sh:name "topic";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:class :KafkaTopic;
+    sh:description "Information about the used topic";
+  ], [
+    sh:path :kafkaConsumer;
+    sh:name "consumer";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:class :KafkaConsumer;
+    sh:description "Information about the kafka consumer";
+  ], [
+    sh:path :kafkaBroker;
+    sh:name "broker";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:class :KafkaBroker;
+    sh:description "Information about the kafka broker";
+  ].
+
+[ ] a sh:NodeShape;
+  sh:targetClass :KafkaTopic;
+  sh:property [
+    sh:path :topicName;
+    sh:name "name";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+    sh:description "Name of the topic";
+  ], [
+    sh:path :topicFromBeginning;
+    sh:name "fromBeginning";
+    sh:maxCount 1;
+    sh:datatype xsd:boolean;
+    sh:description "Restart the topic from the beginning";
+  ].
+
+[ ] a sh:NodeShape;
+  sh:targetClass :KafkaConsumer;
+  sh:property [
+    sh:path :groupId;
+    sh:name "groupId";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+    sh:description "Group identifier";
+  ].
+
+[ ] a sh:NodeShape;
+  sh:targetClass :KafkaBroker;
+  sh:property [
+    sh:path :brokerHost;
+    sh:name "hosts";
+    sh:minCount 1;
+    sh:datatype xsd:string;
+    sh:description "Broker host to connect to";
+  ].
+
+:KafkaWriterChannel rdfs:subClassOf :WriterChannel.
+[ ] a sh:NodeShape;
+  sh:targetClass :KafkaWriterChannel;
+  sh:property [
+    sh:path :kafkaTopic;
+    sh:name "topic";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:class :KafkaTopic;
+    sh:description "Information about the used topic";
+  ], [
+    sh:path :kafkaBroker;
+    sh:name "broker";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:class :KafkaBroker;
+    sh:description "Information about the kafka broker";
+  ], [
+    sh:path :kafkaProducer;
+    sh:name "producer";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:class :KafkaProducer;
+    sh:description "Information about the kafka producer";
+  ].
+
+[ ] a sh:NodeShape;
+  sh:targetClass :KafkaProducer.
+

package/channels/ws.ttl

@@ -0,0 +1,33 @@
+@prefix : <https://w3id.org/conn#>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
+
+:WsReaderChannel rdfs:subClassOf :ReaderChannel.
+:WsWriterChannel rdfs:subClassOf :WriterChannel.
+[ ] a sh:NodeShape;
+  sh:targetClass :WsReaderChannel;
+  sh:property [
+    sh:name "url";
+    sh:path :wsUrl;
+    sh:datatype xsd:string;
+    sh:maxCount 1;
+    sh:minCount 1;
+  ].
+
+[ ] a sh:NodeShape;
+  sh:targetClass :WsWriterChannel;
+  sh:property [
+    sh:name "port";
+    sh:path :wsPort;
+    sh:datatype xsd:integer;
+    sh:maxCount 1;
+    sh:minCount 1;
+  ], [
+    sh:name "host";
+    sh:path :wsHost;
+    sh:datatype xsd:string;
+    sh:maxCount 1;
+    sh:minCount 1;
+  ].
+

package/ontology.ttl

@@ -0,0 +1,169 @@
+@prefix js: <https://w3id.org/conn/js#>.
+@prefix fno: <https://w3id.org/function/ontology#>.
+@prefix fnom: <https://w3id.org/function/vocabulary/mapping#>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+@prefix : <https://w3id.org/conn#>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>.
+@prefix dc: <http://purl.org/dc/terms/>.
+@prefix rdfl: <https://w3id.org/rdf-lens/ontology#>.
+
+[ ] a sh:NodeShape;
+  sh:targetClass :Channel;
+  sh:property [
+    sh:path ( );
+    sh:name "id";
+    sh:maxCount 1;
+    sh:minCount 1;
+    sh:datatype xsd:iri;
+  ], [
+    sh:path :reader;
+    sh:name "reader";
+    sh:maxCount 1;
+    sh:datatype xsd:iri;
+  ], [
+    sh:path :writer;
+    sh:name "writer";
+    sh:maxCount 1;
+    sh:datatype xsd:iri;
+  ].
+
+[ ] a sh:NodeShape;
+  sh:targetClass :ReaderChannel;
+  sh:property [
+    sh:path ( );
+    sh:name "id";
+    sh:maxCount 1;
+    sh:minCount 1;
+    sh:datatype xsd:iri;
+  ], [
+    sh:path rdf:type;
+    sh:name "ty";
+    sh:maxCount 1;
+    sh:datatype xsd:iri;
+  ], [
+    sh:path ( );
+    sh:name "config";
+    sh:maxCount 1;
+    sh:minCount 1;
+    sh:class rdfl:TypedExtract;
+  ].
+
+[ ] a sh:NodeShape;
+  sh:targetClass :WriterChannel;
+  sh:property [
+    sh:path ( );
+    sh:name "id";
+    sh:maxCount 1;
+    sh:minCount 1;
+    sh:datatype xsd:iri;
+  ], [
+    sh:path rdf:type;
+    sh:name "ty";
+    sh:maxCount 1;
+    sh:minCount 1;
+    sh:datatype xsd:iri;
+  ], [
+    sh:path ( );
+    sh:name "config";
+    sh:maxCount 1;
+    sh:minCount 1;
+    sh:class rdfl:TypedExtract;
+  ].
+
+js:JsChannel rdfs:subClassOf :Channel.
+js:JsReaderChannel rdfs:subClassOf :ReaderChannel.
+[ ] a sh:NodeShape;
+  sh:targetClass js:JsReaderChannel;
+  sh:property [
+    sh:path [ sh:inversePath :reader ];
+    sh:name "channel";
+    sh:maxCount 1;
+    sh:minCount 1;
+    sh:class :Channel;
+  ].
+
+js:JsWriterChannel rdfs:subClassOf :WriterChannel.
+[ ] a sh:NodeShape;
+  sh:targetClass js:JsWriterChannel;
+  sh:property [
+    sh:path [ sh:inversePath :writer ];
+    sh:name "channel";
+    sh:maxCount 1;
+    sh:minCount 1;
+    sh:class :Channel;
+  ].
+
+#
+js:JsChannel a :Channel;
+  dc:title "Javascript in memory channel";
+  dc:description "Channel only used by the JsRunner, enabling fast in memory communication.";
+  :reader :JsReaderChannel;
+  :writer :JsWriterChannel.
+
+js:JsProcess a :ProcessClass;
+  dc:title "Javascript Runner";
+  dc:description "The JSRunner is one of the most feature rich runners of the connector architecture. It enables lifting javascript functions to connector architecture components. It allows for fast in memory communication between these processors with the js:JsChannel. More information can be found at https://the-connector-architecture.github.io/site/.";
+  :supportsChannel :WsChannel,
+    :JsChannel,
+    :FileChannel,
+    :KafkaChannel,
+    :HttpChannel.
+
+[ ] a sh:NodeShape;
+  sh:targetClass fno:Mapping;
+  sh:property [
+    sh:class fnom:PositionParameterMapping;
+    sh:path fno:parameterMapping;
+    sh:name "parameters";
+  ].
+
+[ ] a sh:NodeShape;
+  sh:targetClass fnom:PositionParameterMapping;
+  sh:property [
+    sh:datatype xsd:string;
+    sh:path fnom:functionParameter;
+    sh:name "parameter";
+    sh:maxCount 1;
+  ], [
+    sh:datatype xsd:integer;
+    sh:path fnom:implementationParameterPosition;
+    sh:name "position";
+    sh:maxCount 1;
+  ].
+
+js:JsProcessorShape a sh:NodeShape;
+  sh:targetClass js:JsProcess;
+  sh:property [
+    sh:datatype xsd:iri;
+    sh:path ( );
+    sh:name "ty";
+    sh:maxCount 1;
+    sh:minCount 1;
+  ], [
+    sh:datatype xsd:string;
+    sh:path js:file;
+    sh:name "file";
+    sh:maxCount 1;
+    sh:minCount 1;
+  ], [
+    sh:datatype xsd:string;
+    sh:path js:location;
+    sh:name "location";
+    sh:maxCount 1;
+    sh:minCount 1;
+  ], [
+    sh:datatype xsd:string;
+    sh:path js:function;
+    sh:name "func";
+    sh:maxCount 1;
+    sh:minCount 1;
+  ], [
+    sh:class fno:Mapping;
+    sh:path js:mapping;
+    sh:name "mapping";
+    sh:maxCount 1;
+    sh:minCount 1;
+  ].
+

package/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "@rdfc/js-runner",
+  "version": "0.2.0",
+  "type": "module",
+  "exports": {
+    "import": "./dist/index.js",
+    "require": "./dist/index.cjs"
+  },
+  "description": "",
+  "main": "./dist/index.js",
+  "files": [
+    "./dist/**/*",
+    "./ontology.ttl",
+    "./processor/**/*",
+    "./channels/**/*"
+  ],
+  "types": "./dist/index.d.ts",
+  "bin": {
+    "js-runner": "bin/bundle.mjs"
+  },
+  "scripts": {
+    "build": "tsc && tsc-alias && rollup ./dist/index.js --file ./dist/index.cjs --format cjs && bun build --external debug ./bin/js-runner.js --outfile bin/bundle.mjs --target node && npm run build:recompose",
+    "build:recompose": "sed -z 's/var __require = (id) => {\\n  return import.meta.require(id);\\n};/import Module from \"node:module\";\\nconst __require = Module.createRequire(import.meta.url);/' -i bin/bundle.mjs",
+    "watch": "tsc -w",
+    "test": "vitest run --coverage --coverage.include src",
+    "prepare": "husky"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "MIT",
+  "dependencies": {
+    "@rdfjs/types": "^1.1.0",
+    "@treecg/types": "^0.4.5",
+    "command-line-args": "^5.2.1",
+    "command-line-usage": "^6.1.3",
+    "debug": "^4.3.4",
+    "kafkajs": "^2.2.4",
+    "n3": "^1.17.1",
+    "rdf-lens": "^1.2.8",
+    "stream-to-array": "^2.3.0",
+    "ws": "^8.14.2"
+  },
+  "devDependencies": {
+    "@jest/globals": "^29.7.0",
+    "@knighted/duel": "^1.0.6",
+    "@types/command-line-args": "^5.2.2",
+    "@types/command-line-usage": "^5.0.3",
+    "@types/debug": "^4.1.12",
+    "@types/n3": "^1.16.3",
+    "@types/node": "^18.11.15",
+    "@types/ws": "^8.5.8",
+    "rollup": "^4.12.0",
+    "@typescript-eslint/eslint-plugin": "^7.5.0",
+    "@typescript-eslint/parser": "^7.4.0",
+    "@vitest/coverage-v8": "^1.4.0",
+    "dotenv": "^16.4.5",
+    "eslint": "^8.57.0",
+    "eslint-config-prettier": "^9.1.0",
+    "husky": "^9.0.11",
+    "lint-staged": "^15.2.2",
+    "prettier": "^3.2.5",
+    "ts-node": "^10.9.2",
+    "tsc-alias": "^1.8.8",
+    "typescript": "^5.4.3",
+    "vite-tsconfig-paths": "^4.3.2",
+    "vitest": "^1.4.0"
+  }
+}

package/processor/echo.ttl

@@ -0,0 +1,38 @@
+@prefix js: <https://w3id.org/conn/js#>.
+@prefix fno: <https://w3id.org/function/ontology#>.
+@prefix fnom: <https://w3id.org/function/vocabulary/mapping#>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+@prefix : <https://w3id.org/conn#>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+
+js:Echo a js:JsProcess;
+  js:file <./test.js>;
+  js:function "echo";
+  js:location <./>;
+  js:mapping [
+    a fno:Mapping;
+    fno:parameterMapping [
+      a fnom:PositionParameterMapping;
+      fnom:functionParameter "Input Channel";
+      fnom:implementationParameterPosition "0"^^xsd:int;
+    ], [
+      a fnom:PositionParameterMapping;
+      fnom:functionParameter "Output Channel";
+      fnom:implementationParameterPosition "1"^^xsd:int;
+    ];
+  ].
+
+[ ] a sh:NodeShape;
+  sh:targetClass js:Echo;
+  sh:property [
+    sh:class :ReaderChannel;
+    sh:path js:input;
+    sh:name "Input Channel";
+    sh:maxCount 1;
+  ], [
+    sh:class :WriterChannel;
+    sh:path js:output;
+    sh:name "Output Channel";
+    sh:maxCount 1;
+  ].
+

package/processor/resc.ttl

@@ -0,0 +1,34 @@
+@prefix js: <https://w3id.org/conn/js#>.
+@prefix fno: <https://w3id.org/function/ontology#>.
+@prefix fnom: <https://w3id.org/function/vocabulary/mapping#>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+@prefix : <https://w3id.org/conn#>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+
+<> :install [
+  a :LocalInstall;
+  :path <./>;
+].
+
+js:Resc a js:JsProcess;
+  js:file <./test.js>;
+  js:function "resc";
+  js:location <./>;
+  js:mapping [
+    a fno:Mapping;
+    fno:parameterMapping [
+      a fnom:PositionParameterMapping;
+      fnom:functionParameter "input";
+      fnom:implementationParameterPosition "0"^^xsd:int;
+    ];
+  ].
+
+[ ] a sh:NodeShape;
+  sh:targetClass js:Resc;
+  sh:property [
+    sh:class :ReaderChannel;
+    sh:path js:rescReader;
+    sh:name "input";
+    sh:maxCount 1;
+  ].
+

package/processor/send.ttl

@@ -0,0 +1,40 @@
+@prefix js: <https://w3id.org/conn/js#>.
+@prefix fno: <https://w3id.org/function/ontology#>.
+@prefix fnom: <https://w3id.org/function/vocabulary/mapping#>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+@prefix : <https://w3id.org/conn#>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+
+js:Send a js:JsProcess;
+  js:file <./test.js>;
+  js:function "send";
+  js:location <./>;
+  js:mapping [
+    a fno:Mapping;
+    fno:parameterMapping [
+      a fnom:PositionParameterMapping;
+      fnom:functionParameter "msg";
+      fnom:implementationParameterPosition "0"^^xsd:int;
+    ], [
+      a fnom:PositionParameterMapping;
+      fnom:functionParameter "output";
+      fnom:implementationParameterPosition "1"^^xsd:int;
+    ];
+  ].
+
+[ ] a sh:NodeShape;
+  sh:targetClass js:Send;
+  sh:property [
+    sh:datatype xsd:string;
+    sh:path js:msg;
+    sh:name "msg";
+    sh:maxCount 1;
+    sh:minCount 1;
+  ], [
+    sh:class :WriterChannel;
+    sh:path js:sendWriter;
+    sh:name "output";
+    sh:maxCount 1;
+    sh:minCount 1;
+  ].
+

package/processor/test.js

@@ -0,0 +1,35 @@
+import http from "http";
+
+function streamToString(ev) {
+  const datas = [];
+  return new Promise((res) => {
+    ev.on("data", (d) => datas.push(d));
+    ev.on("end", () => res(Buffer.concat(datas)));
+  });
+}
+
+export async function send(msg, writer) {
+  const host = "0.0.0.0";
+  const port = 8000;
+  const requestListener = async function (req, res) {
+    const data = await streamToString(req);
+    const ret = `${msg} ${data}`;
+    await writer.push(ret);
+    res.writeHead(200);
+    res.end(ret);
+  };
+  const server = http.createServer(requestListener);
+
+  await new Promise((res) => {
+    server.listen(port, host, () => {
+      console.log(`Server is running on http://${host}:${port} prefix ${msg}`);
+      res();
+    });
+  });
+
+  return () => writer.push("Hallo!");
+}
+
+export function resc(reader) {
+  reader.data((x) => console.log("data", x));
+}