@rdfc/js-runner 0.2.2 → 0.3.0-alpha.1

package/README.md

@@ -1,6 +1,6 @@
 # Js-runner
 
-[![Bun CI](https://github.com/rdf-connect/js-runner/actions/workflows/build-test.yml/badge.svg)](https://github.com/rdf-connect/js-runner/actions/workflows/build-test.yml) [![npm](https://img.shields.io/npm/v/@rdfc/js-runner.svg?style=popout)](https://npmjs.com/package/@rdfc/js-runner)
+[![Build and tests with Node.js](https://github.com/rdf-connect/js-runner/actions/workflows/build-test.yml/badge.svg)](https://github.com/rdf-connect/js-runner/actions/workflows/build-test.yml) [![npm](https://img.shields.io/npm/v/@rdfc/js-runner.svg?style=popout)](https://npmjs.com/package/@rdfc/js-runner)
 
 Typescript/Javascript executor for an [RDF-Connect](https://rdf-connect.github.io/rdfc.github.io/) pipeline. Starting from a declarative RDF file describing the pipeline.
 
@@ -34,7 +34,7 @@ You can execute this pipeline with
 
 ```bash
 tsc
-bun bin/js-runner.js input.ttl 
+npx bin/js-runner.js input.ttl 
 ```
 
-This example input configuration file uses `owl:imports` to specify additional configuration files.
+This example input configuration file uses `owl:imports` to specify additional configuration files.

package/bin/js-runner.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+import { jsRunner } from "../dist/index.js";
+
+jsRunner().catch((e) => { console.error("Error:", e); console.error(e.stack) });

package/dist/args.js

@@ -1,25 +1,37 @@
 import commandLineArgs from "command-line-args";
 import commandLineUsage from "command-line-usage";
 const optionDefinitions = [
-    { name: 'input', type: String, defaultOption: true, summary: "Specify what input file to start up" },
-    { name: 'help', alias: 'h', type: Boolean, description: "Display this help message" },
+    {
+        name: "input",
+        type: String,
+        defaultOption: true,
+        summary: "Specify what input file to start up",
+    },
+    {
+        name: "help",
+        alias: "h",
+        type: Boolean,
+        description: "Display this help message",
+    },
 ];
 const sections = [
     {
         header: "Js-runner",
-        content: "JS-runner is part of the {italic connector architecture}. Starting from an input file start up all JsProcessors that are defined. Please do not use blank nodes, skolemize your data somewhere else!"
+        content: "JS-runner is part of the {italic connector architecture}. Starting from an input file start up all JsProcessors that are defined. Please do not use blank nodes, skolemize your data somewhere else!",
     },
     {
         header: "Synopsis",
-        content: "$ js-runner <input>"
+        content: "$ js-runner <input>",
     },
     {
         header: "Command List",
-        content: [{ name: "input", summary: "Specify what input file to start up" }],
+        content: [
+            { name: "input", summary: "Specify what input file to start up" },
+        ],
     },
     {
-        optionList: [optionDefinitions[1]]
-    }
+        optionList: [optionDefinitions[1]],
+    },
 ];
 function validArgs(args) {
     if (!args.input)

package/dist/connectors/file.js

@@ -71,12 +71,13 @@ export const startFileStreamWriter = (config) => {
         ? config.path
         : `${process.cwd()}/${config.path}`;
     const encoding = config.encoding || "utf-8";
+    const writer = new SimpleStream();
     const init = async () => {
         if (!config.onReplace) {
             await writeFile(path, "", { encoding });
         }
     };
-    const push = async (item) => {
+    writer.push = async (item) => {
         if (config.onReplace) {
             await writeFile(path, item, { encoding });
         }
@@ -84,6 +85,5 @@ export const startFileStreamWriter = (config) => {
             await appendFile(path, item, { encoding });
         }
     };
-    const end = async () => { };
-    return { writer: { push, end }, init };
+    return { writer, init };
 };

package/dist/connectors/http.js

@@ -14,7 +14,7 @@ function streamToString(stream, binary) {
     });
 }
 export const startHttpStreamReader = (config) => {
-    let server;
+    let server = undefined;
     const stream = new SimpleStream(() => new Promise((res) => {
         if (server !== undefined) {
             server.close(() => {
@@ -57,8 +57,9 @@ export const startHttpStreamReader = (config) => {
 };
 export const startHttpStreamWriter = (config) => {
     const requestConfig = new URL(config.endpoint);
-    const push = async (item) => {
-        await new Promise(async (resolve) => {
+    const writer = new SimpleStream();
+    writer.push = async (item) => {
+        await new Promise((resolve) => {
             const options = {
                 hostname: requestConfig.hostname,
                 path: requestConfig.path,
@@ -72,10 +73,10 @@ export const startHttpStreamWriter = (config) => {
                 });
             };
             const req = http.request(options, cb);
-            await new Promise((res) => req.write(item, res));
-            await new Promise((res) => req.end(res));
+            req.write(item, () => {
+                req.end();
+            });
         });
     };
-    const end = async () => { };
-    return { writer: { push, end }, init: async () => { } };
+    return { writer, init: async () => { } };
 };

package/dist/connectors/kafka.js

@@ -10,7 +10,7 @@ export const startKafkaStreamReader = (config) => {
         Object.assign(brokerConfig, config.broker);
     }
     if (brokerConfig && brokerConfig.hosts) {
-        brokerConfig.brokers = brokerConfig.hosts;
+        brokerConfig.brokers = (brokerConfig).hosts;
     }
     const kafka = new Kafka(brokerConfig);
     const consumer = kafka.consumer(config.consumer);
@@ -29,9 +29,7 @@ export const startKafkaStreamReader = (config) => {
             async eachMessage({ topic, message, }) {
                 if (topic === config.topic.name) {
                     const element = message.value?.toString() ?? "";
-                    stream.push(element).catch((error) => {
-                        throw error;
-                    });
+                    await stream.push(element);
                 }
             },
         })
@@ -51,16 +49,16 @@ export const startKafkaStreamWriter = (config) => {
         Object.assign(brokerConfig, config.broker);
     }
     if (brokerConfig && brokerConfig.hosts) {
-        brokerConfig.brokers = brokerConfig.hosts;
+        brokerConfig.brokers = (brokerConfig).hosts;
     }
     const kafka = new Kafka(brokerConfig);
     const producer = kafka.producer(config.producer);
+    const writer = new SimpleStream(async () => {
+        await producer.disconnect();
+    });
     const init = () => producer.connect();
-    const push = async (item) => {
+    writer.push = async (item) => {
         await producer.send({ topic, messages: [{ value: item }] });
     };
-    const end = async () => {
-        await producer.disconnect();
-    };
-    return { writer: { push, end }, init };
+    return { writer, init };
 };

package/dist/connectors/ws.js

@@ -59,11 +59,11 @@ export const startWsStreamWriter = (config) => {
         ws = await connectWs(config.url);
         ws.on("open", () => console.log("open"));
     };
-    const push = async (item) => {
-        ws.send(item);
-    };
-    const end = async () => {
+    const writer = new SimpleStream(async () => {
         ws.close();
+    });
+    writer.push = async (item) => {
+        ws.send(item);
     };
-    return { writer: { push, end }, init };
+    return { writer, init };
 };

package/dist/connectors.d.ts

@@ -29,6 +29,7 @@ export declare class ChannelFactory {
 export interface Writer<T> {
     push(item: T): Promise<void>;
     end(): Promise<void>;
+    on(event: "end", listener: Handler<void>): this;
 }
 export interface Stream<T> {
     lastElement?: T;
@@ -43,6 +44,7 @@ export declare class SimpleStream<T> implements Stream<T> {
     private readonly endHandlers;
     readonly disconnect: () => Promise<void>;
     lastElement?: T | undefined;
+    private ended;
     constructor(onDisconnect?: () => Promise<void>);
     data(listener: Handler<T>): this;
     push(data: T): Promise<void>;

package/dist/connectors.js

@@ -42,13 +42,15 @@ export class ChannelFactory {
                 const id = c.channel.id.value;
                 if (c.channel.id.termType === "NamedNode") {
                     if (!this.jsChannelsNamedNodes[id]) {
-                        this.jsChannelsNamedNodes[id] = new SimpleStream();
+                        this.jsChannelsNamedNodes[id] =
+                            new SimpleStream();
                     }
                     return this.jsChannelsNamedNodes[id];
                 }
                 if (c.channel.id.termType === "BlankNode") {
                     if (!this.jsChannelsBlankNodes[id]) {
-                        this.jsChannelsBlankNodes[id] = new SimpleStream();
+                        this.jsChannelsBlankNodes[id] =
+                            new SimpleStream();
                     }
                     return this.jsChannelsBlankNodes[id];
                 }
@@ -85,13 +87,15 @@ export class ChannelFactory {
                 const id = c.channel.id.value;
                 if (c.channel.id.termType === "NamedNode") {
                     if (!this.jsChannelsNamedNodes[id]) {
-                        this.jsChannelsNamedNodes[id] = new SimpleStream();
+                        this.jsChannelsNamedNodes[id] =
+                            new SimpleStream();
                     }
                     return this.jsChannelsNamedNodes[id];
                 }
                 if (c.channel.id.termType === "BlankNode") {
                     if (!this.jsChannelsBlankNodes[id]) {
-                        this.jsChannelsBlankNodes[id] = new SimpleStream();
+                        this.jsChannelsBlankNodes[id] =
+                            new SimpleStream();
                     }
                     return this.jsChannelsBlankNodes[id];
                 }
@@ -109,6 +113,7 @@ export class SimpleStream {
     endHandlers = [];
     disconnect;
     lastElement;
+    ended = false;
     constructor(onDisconnect) {
         this.disconnect = onDisconnect || (async () => { });
     }
@@ -117,12 +122,16 @@ export class SimpleStream {
         return this;
     }
     async push(data) {
+        if (this.ended) {
+            throw new Error("Trying to push to a stream that has ended!");
+        }
         this.lastElement = data;
         await Promise.all(this.dataHandlers.map((handler) => handler(data)));
     }
     async end() {
         await this.disconnect();
         await Promise.all(this.endHandlers.map((handler) => handler()));
+        this.ended = true;
     }
     on(event, listener) {
         if (event === "data") {

package/dist/index.cjs

@@ -8,8 +8,8 @@ var http = require('http');
 var https = require('https');
 var types = require('@treecg/types');
 var debug = require('debug');
-var promises = require('fs/promises');
 var path = require('path');
+var promises = require('fs/promises');
 var node_fs = require('node:fs');
 var ws = require('ws');
 var kafkajs = require('kafkajs');
@@ -35,25 +35,37 @@ function _interopNamespaceDefault(e) {
 var http__namespace = /*#__PURE__*/_interopNamespaceDefault(http);
 
 const optionDefinitions = [
-    { name: 'input', type: String, defaultOption: true, summary: "Specify what input file to start up" },
-    { name: 'help', alias: 'h', type: Boolean, description: "Display this help message" },
+    {
+        name: "input",
+        type: String,
+        defaultOption: true,
+        summary: "Specify what input file to start up",
+    },
+    {
+        name: "help",
+        alias: "h",
+        type: Boolean,
+        description: "Display this help message",
+    },
 ];
 const sections = [
     {
         header: "Js-runner",
-        content: "JS-runner is part of the {italic connector architecture}. Starting from an input file start up all JsProcessors that are defined. Please do not use blank nodes, skolemize your data somewhere else!"
+        content: "JS-runner is part of the {italic connector architecture}. Starting from an input file start up all JsProcessors that are defined. Please do not use blank nodes, skolemize your data somewhere else!",
     },
     {
         header: "Synopsis",
-        content: "$ js-runner <input>"
+        content: "$ js-runner <input>",
     },
     {
         header: "Command List",
-        content: [{ name: "input", summary: "Specify what input file to start up" }],
+        content: [
+            { name: "input", summary: "Specify what input file to start up" },
+        ],
     },
     {
-        optionList: [optionDefinitions[1]]
-    }
+        optionList: [optionDefinitions[1]],
+    },
 ];
 function validArgs(args) {
     if (!args.input)
@@ -144,7 +156,7 @@ async function load_store(location, store, recursive = true) {
     if (recursive) {
         const loc = location.type === "remote" ? location.location : location.baseIRI;
         const other_imports = store.getObjects(namedNode(loc), OWL.terms.imports, null);
-        for (let other of other_imports) {
+        for (const other of other_imports) {
             await load_store({ location: other.value, type: "remote" }, store, true);
         }
     }
@@ -218,12 +230,13 @@ const startFileStreamWriter = (config) => {
         ? config.path
         : `${process.cwd()}/${config.path}`;
     const encoding = config.encoding || "utf-8";
+    const writer = new SimpleStream();
     const init = async () => {
         if (!config.onReplace) {
             await promises.writeFile(path$1, "", { encoding });
         }
     };
-    const push = async (item) => {
+    writer.push = async (item) => {
         if (config.onReplace) {
             await promises.writeFile(path$1, item, { encoding });
         }
@@ -231,8 +244,7 @@ const startFileStreamWriter = (config) => {
             await promises.appendFile(path$1, item, { encoding });
         }
     };
-    const end = async () => { };
-    return { writer: { push, end }, init };
+    return { writer, init };
 };
 
 function _connectWs(url, res) {
@@ -293,13 +305,13 @@ const startWsStreamWriter = (config) => {
         ws = await connectWs(config.url);
         ws.on("open", () => console.log("open"));
     };
-    const push = async (item) => {
-        ws.send(item);
-    };
-    const end = async () => {
+    const writer = new SimpleStream(async () => {
         ws.close();
+    });
+    writer.push = async (item) => {
+        ws.send(item);
     };
-    return { writer: { push, end }, init };
+    return { writer, init };
 };
 
 const startKafkaStreamReader = (config) => {
@@ -311,7 +323,7 @@ const startKafkaStreamReader = (config) => {
         Object.assign(brokerConfig, config.broker);
     }
     if (brokerConfig && brokerConfig.hosts) {
-        brokerConfig.brokers = brokerConfig.hosts;
+        brokerConfig.brokers = (brokerConfig).hosts;
     }
     const kafka = new kafkajs.Kafka(brokerConfig);
     const consumer = kafka.consumer(config.consumer);
@@ -352,18 +364,18 @@ const startKafkaStreamWriter = (config) => {
         Object.assign(brokerConfig, config.broker);
     }
     if (brokerConfig && brokerConfig.hosts) {
-        brokerConfig.brokers = brokerConfig.hosts;
+        brokerConfig.brokers = (brokerConfig).hosts;
     }
     const kafka = new kafkajs.Kafka(brokerConfig);
     const producer = kafka.producer(config.producer);
+    const writer = new SimpleStream(async () => {
+        await producer.disconnect();
+    });
     const init = () => producer.connect();
-    const push = async (item) => {
+    writer.push = async (item) => {
         await producer.send({ topic, messages: [{ value: item }] });
     };
-    const end = async () => {
-        await producer.disconnect();
-    };
-    return { writer: { push, end }, init };
+    return { writer, init };
 };
 
 function streamToString(stream, binary) {
@@ -379,7 +391,7 @@ function streamToString(stream, binary) {
     });
 }
 const startHttpStreamReader = (config) => {
-    let server;
+    let server = undefined;
     const stream = new SimpleStream(() => new Promise((res) => {
         if (server !== undefined) {
             server.close(() => {
@@ -422,8 +434,9 @@ const startHttpStreamReader = (config) => {
 };
 const startHttpStreamWriter = (config) => {
     const requestConfig = new URL(config.endpoint);
-    const push = async (item) => {
-        await new Promise(async (resolve) => {
+    const writer = new SimpleStream();
+    writer.push = async (item) => {
+        await new Promise((resolve) => {
             const options = {
                 hostname: requestConfig.hostname,
                 path: requestConfig.path,
@@ -437,12 +450,12 @@ const startHttpStreamWriter = (config) => {
                 });
             };
             const req = http__namespace.request(options, cb);
-            await new Promise((res) => req.write(item, res));
-            await new Promise((res) => req.end(res));
+            req.write(item, () => {
+                req.end();
+            });
         });
     };
-    const end = async () => { };
-    return { writer: { push, end }, init: async () => { } };
+    return { writer, init: async () => { } };
 };
 
 const Conn = types.createTermNamespace("https://w3id.org/conn#", "FileReaderChannel", "FileWriterChannel", "HttpReaderChannel", "HttpWriterChannel", "KafkaReaderChannel", "KafkaWriterChannel", "WsReaderChannel", "WsWriterChannel", "WriterChannel", "ReaderChannel");
@@ -479,13 +492,15 @@ class ChannelFactory {
                 const id = c.channel.id.value;
                 if (c.channel.id.termType === "NamedNode") {
                     if (!this.jsChannelsNamedNodes[id]) {
-                        this.jsChannelsNamedNodes[id] = new SimpleStream();
+                        this.jsChannelsNamedNodes[id] =
+                            new SimpleStream();
                     }
                     return this.jsChannelsNamedNodes[id];
                 }
                 if (c.channel.id.termType === "BlankNode") {
                     if (!this.jsChannelsBlankNodes[id]) {
-                        this.jsChannelsBlankNodes[id] = new SimpleStream();
+                        this.jsChannelsBlankNodes[id] =
+                            new SimpleStream();
                     }
                     return this.jsChannelsBlankNodes[id];
                 }
@@ -522,13 +537,15 @@ class ChannelFactory {
                 const id = c.channel.id.value;
                 if (c.channel.id.termType === "NamedNode") {
                     if (!this.jsChannelsNamedNodes[id]) {
-                        this.jsChannelsNamedNodes[id] = new SimpleStream();
+                        this.jsChannelsNamedNodes[id] =
+                            new SimpleStream();
                     }
                     return this.jsChannelsNamedNodes[id];
                 }
                 if (c.channel.id.termType === "BlankNode") {
                     if (!this.jsChannelsBlankNodes[id]) {
-                        this.jsChannelsBlankNodes[id] = new SimpleStream();
+                        this.jsChannelsBlankNodes[id] =
+                            new SimpleStream();
                     }
                     return this.jsChannelsBlankNodes[id];
                 }
@@ -546,6 +563,7 @@ class SimpleStream {
     endHandlers = [];
     disconnect;
     lastElement;
+    ended = false;
     constructor(onDisconnect) {
         this.disconnect = onDisconnect || (async () => { });
     }
@@ -554,12 +572,16 @@ class SimpleStream {
         return this;
     }
     async push(data) {
+        if (this.ended) {
+            throw new Error("Trying to push to a stream that has ended!");
+        }
         this.lastElement = data;
         await Promise.all(this.dataHandlers.map((handler) => handler(data)));
     }
     async end() {
         await this.disconnect();
         await Promise.all(this.endHandlers.map((handler) => handler()));
+        this.ended = true;
     }
     on(event, listener) {
         if (event === "data") {
@@ -598,10 +620,10 @@ function extractSteps(proc, quads, config) {
         .map((x) => x.subject);
     const processorLens = config.lenses[proc.ty.value];
     const fields = proc.mapping.parameters;
-    for (let id of subjects) {
+    for (const id of subjects) {
         const obj = processorLens.execute({ id, quads });
         const functionArgs = new Array(fields.length);
-        for (let field of fields) {
+        for (const field of fields) {
             functionArgs[field.position] = obj[field.parameter];
         }
         out.push(functionArgs);
@@ -622,16 +644,16 @@ async function jsRunner() {
     const { processors, quads, shapes: config, } = await extractProcessors(source, apply);
     LOG.main("Found %d processors", processors.length);
     const starts = [];
-    for (let proc of processors) {
+    for (const proc of processors) {
         const argss = extractSteps(proc, quads, config);
         const jsProgram = await import("file://" + proc.file);
         process.chdir(proc.location);
-        for (let args of argss) {
+        for (const args of argss) {
             starts.push(await jsProgram[proc.func](...args));
         }
     }
     await factory.init();
-    for (let s of starts) {
+    for (const s of starts) {
         if (s && typeof s === "function") {
             s();
         }

package/dist/index.d.ts

@@ -1,6 +1,6 @@
-export * from "./connectors.js";
 import { Quad, Term } from "@rdfjs/types";
 import { Shapes } from "rdf-lens";
+export * from "./connectors.js";
 type Processor = {
     ty: Term;
     file: string;

package/dist/index.js

@@ -1,11 +1,11 @@
 import { Store } from "n3";
 import { getArgs } from "./args.js";
 import { load_store, LOG } from "./util.js";
-export * from "./connectors.js";
 import path from "path";
 import { RDF } from "@treecg/types";
 import { ChannelFactory, Conn, JsOntology } from "./connectors.js";
-import { extractShapes } from "rdf-lens";
+import { envReplace, extractShapes } from "rdf-lens";
+export * from "./connectors.js";
 function safeJoin(a, b) {
     if (b.startsWith("/")) {
         return b;
@@ -23,7 +23,8 @@ export async function extractProcessors(source, apply) {
         .map((x) => x.subject);
     const processorLens = config.lenses[JsOntology.JsProcess.value];
     const processors = subjects.map((id) => processorLens.execute({ id, quads }));
-    return { processors, quads, shapes: config };
+    const newQuads = envReplace().execute(quads);
+    return { processors, quads: newQuads, shapes: config };
 }
 export function extractSteps(proc, quads, config) {
     const out = [];
@@ -32,10 +33,10 @@ export function extractSteps(proc, quads, config) {
         .map((x) => x.subject);
     const processorLens = config.lenses[proc.ty.value];
     const fields = proc.mapping.parameters;
-    for (let id of subjects) {
+    for (const id of subjects) {
         const obj = processorLens.execute({ id, quads });
         const functionArgs = new Array(fields.length);
-        for (let field of fields) {
+        for (const field of fields) {
             functionArgs[field.position] = obj[field.parameter];
         }
         out.push(functionArgs);
@@ -56,16 +57,16 @@ export async function jsRunner() {
     const { processors, quads, shapes: config, } = await extractProcessors(source, apply);
     LOG.main("Found %d processors", processors.length);
     const starts = [];
-    for (let proc of processors) {
+    for (const proc of processors) {
         const argss = extractSteps(proc, quads, config);
         const jsProgram = await import("file://" + proc.file);
         process.chdir(proc.location);
-        for (let args of argss) {
+        for (const args of argss) {
             starts.push(await jsProgram[proc.func](...args));
         }
     }
     await factory.init();
-    for (let s of starts) {
+    for (const s of starts) {
         if (s && typeof s === "function") {
             s();
         }