@rdfc/js-runner 0.2.0 → 0.2.2

package/dist/args.d.ts

@@ -0,0 +1,4 @@
+export type Args = {
+    input: string;
+};
+export declare function getArgs(): Args;

package/dist/args.js

@@ -0,0 +1,47 @@
+import commandLineArgs from "command-line-args";
+import commandLineUsage from "command-line-usage";
+const optionDefinitions = [
+    { name: 'input', type: String, defaultOption: true, summary: "Specify what input file to start up" },
+    { name: 'help', alias: 'h', type: Boolean, description: "Display this help message" },
+];
+const sections = [
+    {
+        header: "Js-runner",
+        content: "JS-runner is part of the {italic connector architecture}. Starting from an input file start up all JsProcessors that are defined. Please do not use blank nodes, skolemize your data somewhere else!"
+    },
+    {
+        header: "Synopsis",
+        content: "$ js-runner <input>"
+    },
+    {
+        header: "Command List",
+        content: [{ name: "input", summary: "Specify what input file to start up" }],
+    },
+    {
+        optionList: [optionDefinitions[1]]
+    }
+];
+function validArgs(args) {
+    if (!args.input)
+        return false;
+    return true;
+}
+function printUsage() {
+    const usage = commandLineUsage(sections);
+    console.log(usage);
+    process.exit(0);
+}
+export function getArgs() {
+    let args;
+    try {
+        args = commandLineArgs(optionDefinitions);
+    }
+    catch (e) {
+        console.error(e);
+        printUsage();
+    }
+    if (args.help || !validArgs(args)) {
+        printUsage();
+    }
+    return args;
+}

package/dist/connectors/file.d.ts

@@ -0,0 +1,15 @@
+import { ReaderConstructor, WriterConstructor } from "../connectors.js";
+export interface FileReaderConfig {
+    path: string;
+    onReplace: boolean;
+    readFirstContent?: boolean;
+    encoding?: string;
+}
+export interface FileWriterConfig {
+    path: string;
+    onReplace: boolean;
+    readFirstContent?: boolean;
+    encoding?: string;
+}
+export declare const startFileStreamReader: ReaderConstructor<FileReaderConfig>;
+export declare const startFileStreamWriter: WriterConstructor<FileWriterConfig>;

package/dist/connectors/file.js

@@ -0,0 +1,89 @@
+import { createReadStream, openSync } from "fs";
+import { appendFile, readFile, stat, writeFile } from "fs/promises";
+import { isAbsolute } from "path";
+import { watch } from "node:fs";
+import { SimpleStream, } from "../connectors.js";
+async function getFileSize(path) {
+    return (await stat(path)).size;
+}
+function readPart(path, start, end, encoding) {
+    return new Promise((res) => {
+        const stream = createReadStream(path, { encoding, start, end });
+        let buffer = "";
+        stream.on("data", (chunk) => {
+            buffer += chunk;
+        });
+        stream.on("close", () => res(buffer));
+    });
+}
+function debounce(func, timeout = 100) {
+    let timer;
+    return (...args) => {
+        clearTimeout(timer);
+        timer = setTimeout(() => {
+            func(...args);
+        }, timeout);
+    };
+}
+export const startFileStreamReader = (config) => {
+    const path = isAbsolute(config.path)
+        ? config.path
+        : `${process.cwd()}/${config.path}`;
+    openSync(path, "a+");
+    const encoding = config.encoding || "utf-8";
+    const reader = new SimpleStream();
+    const init = async () => {
+        let currentPos = await getFileSize(path);
+        const watcher = watch(path, { encoding: "utf-8" });
+        watcher.on("change", debounce(async () => {
+            try {
+                let content;
+                if (config.onReplace) {
+                    content = await readFile(path, { encoding });
+                }
+                else {
+                    const newSize = await getFileSize(path);
+                    if (newSize <= currentPos) {
+                        currentPos = newSize;
+                        return;
+                    }
+                    content = await readPart(path, currentPos, newSize, encoding);
+                    currentPos = newSize;
+                }
+                await reader.push(content);
+            }
+            catch (error) {
+                if (error.code === "ENOENT") {
+                    return;
+                }
+                throw error;
+            }
+        }));
+        if (config.onReplace && config.readFirstContent) {
+            const content = await readFile(path, { encoding });
+            await reader.push(content);
+        }
+    };
+    return { reader, init };
+};
+export const startFileStreamWriter = (config) => {
+    const path = isAbsolute(config.path)
+        ? config.path
+        : `${process.cwd()}/${config.path}`;
+    const encoding = config.encoding || "utf-8";
+    const init = async () => {
+        if (!config.onReplace) {
+            await writeFile(path, "", { encoding });
+        }
+    };
+    const push = async (item) => {
+        if (config.onReplace) {
+            await writeFile(path, item, { encoding });
+        }
+        else {
+            await appendFile(path, item, { encoding });
+        }
+    };
+    const end = async () => { };
+    return { writer: { push, end }, init };
+};

package/dist/connectors/http.d.ts

@@ -0,0 +1,14 @@
+import { ReaderConstructor, WriterConstructor } from "../connectors.js";
+export interface HttpReaderConfig {
+    endpoint: string;
+    port: number;
+    binary: boolean;
+    waitHandled?: boolean;
+    responseCode?: number;
+}
+export declare const startHttpStreamReader: ReaderConstructor<HttpReaderConfig>;
+export interface HttpWriterConfig {
+    endpoint: string;
+    method: string;
+}
+export declare const startHttpStreamWriter: WriterConstructor<HttpWriterConfig>;

package/dist/connectors/http.js

@@ -0,0 +1,81 @@
+import * as http from "http";
+import { createServer } from "http";
+import { SimpleStream, } from "../connectors.js";
+function streamToString(stream, binary) {
+    const datas = [];
+    return new Promise((res) => {
+        stream.on("data", (data) => {
+            datas.push(data);
+        });
+        stream.on("end", () => {
+            const streamData = Buffer.concat(datas);
+            res(binary ? streamData : streamData.toString());
+        });
+    });
+}
+export const startHttpStreamReader = (config) => {
+    let server;
+    const stream = new SimpleStream(() => new Promise((res) => {
+        if (server !== undefined) {
+            server.close(() => {
+                res();
+            });
+        }
+        else {
+            res();
+        }
+    }));
+    const requestListener = async function (req, res) {
+        try {
+            const content = await streamToString(req, config.binary);
+            const promise = stream.push(content).catch((error) => {
+                throw error;
+            });
+            if (config.waitHandled) {
+                await promise;
+            }
+        }
+        catch (error) {
+            console.error("Failed", error);
+        }
+        res.writeHead(config.responseCode || 200);
+        res.end("OK");
+    };
+    server = createServer(requestListener);
+    const init = () => {
+        return new Promise((res) => {
+            const cb = () => res(undefined);
+            if (server) {
+                server.listen(config.port, config.endpoint, cb);
+            }
+            else {
+                cb();
+            }
+        });
+    };
+    return { reader: stream, init };
+};
+export const startHttpStreamWriter = (config) => {
+    const requestConfig = new URL(config.endpoint);
+    const push = async (item) => {
+        await new Promise(async (resolve) => {
+            const options = {
+                hostname: requestConfig.hostname,
+                path: requestConfig.path,
+                method: config.method,
+                port: requestConfig.port,
+            };
+            const cb = (response) => {
+                response.on("data", () => { });
+                response.on("end", () => {
+                    resolve(null);
+                });
+            };
+            const req = http.request(options, cb);
+            await new Promise((res) => req.write(item, res));
+            await new Promise((res) => req.end(res));
+        });
+    };
+    const end = async () => { };
+    return { writer: { push, end }, init: async () => { } };
+};

package/dist/connectors/kafka.d.ts

@@ -0,0 +1,48 @@
+import type { ProducerConfig } from "kafkajs";
+import { ReaderConstructor, WriterConstructor } from "../connectors.js";
+export interface SASLOptions {
+    mechanism: "plain";
+    username: string;
+    password: string;
+}
+export interface BrokerConfig {
+    hosts: string[];
+    ssl?: boolean;
+    sasl?: SASLOptions;
+}
+export interface ConsumerConfig {
+    groupId: string;
+    metadataMaxAge?: number;
+    sessionTimeout?: number;
+    rebalanceTimeout?: number;
+    heartbeatInterval?: number;
+    maxBytesPerPartition?: number;
+    minBytes?: number;
+    maxBytes?: number;
+    maxWaitTimeInMs?: number;
+    allowAutoTopicCreation?: boolean;
+    maxInFlightRequests?: number;
+    readUncommitted?: boolean;
+    rackId?: string;
+}
+export interface CSTopic {
+    topic: string;
+    fromBeginning?: boolean;
+}
+export interface KafkaReaderConfig {
+    topic: {
+        name: string;
+        fromBeginning?: boolean;
+    };
+    consumer: ConsumerConfig;
+    broker: string | BrokerConfig;
+}
+export declare const startKafkaStreamReader: ReaderConstructor<KafkaReaderConfig>;
+export interface KafkaWriterConfig {
+    topic: {
+        name: string;
+    };
+    producer: ProducerConfig;
+    broker: BrokerConfig | string;
+}
+export declare const startKafkaStreamWriter: WriterConstructor<KafkaWriterConfig>;

package/dist/connectors/kafka.js

@@ -0,0 +1,66 @@
+import { readFileSync } from "node:fs";
+import { Kafka } from "kafkajs";
+import { SimpleStream, } from "../connectors.js";
+export const startKafkaStreamReader = (config) => {
+    const brokerConfig = {};
+    if (typeof config.broker === "string" || config.broker instanceof String) {
+        Object.assign(brokerConfig, JSON.parse(readFileSync(config.broker, "utf-8")));
+    }
+    else {
+        Object.assign(brokerConfig, config.broker);
+    }
+    if (brokerConfig && brokerConfig.hosts) {
+        brokerConfig.brokers = brokerConfig.hosts;
+    }
+    const kafka = new Kafka(brokerConfig);
+    const consumer = kafka.consumer(config.consumer);
+    const stream = new SimpleStream(async () => {
+        await consumer.disconnect();
+        await consumer.stop();
+    });
+    const init = async () => {
+        await consumer.connect();
+        await consumer.subscribe({
+            topic: config.topic.name,
+            fromBeginning: config.topic.fromBeginning,
+        });
+        consumer
+            .run({
+            async eachMessage({ topic, message, }) {
+                if (topic === config.topic.name) {
+                    const element = message.value?.toString() ?? "";
+                    stream.push(element).catch((error) => {
+                        throw error;
+                    });
+                }
+            },
+        })
+            .catch((error) => {
+            throw error;
+        });
+    };
+    return { reader: stream, init };
+};
+export const startKafkaStreamWriter = (config) => {
+    const topic = config.topic.name;
+    const brokerConfig = {};
+    if (typeof config.broker === "string" || config.broker instanceof String) {
+        Object.assign(brokerConfig, JSON.parse(readFileSync(config.broker, "utf-8")));
+    }
+    else {
+        Object.assign(brokerConfig, config.broker);
+    }
+    if (brokerConfig && brokerConfig.hosts) {
+        brokerConfig.brokers = brokerConfig.hosts;
+    }
+    const kafka = new Kafka(brokerConfig);
+    const producer = kafka.producer(config.producer);
+    const init = () => producer.connect();
+    const push = async (item) => {
+        await producer.send({ topic, messages: [{ value: item }] });
+    };
+    const end = async () => {
+        await producer.disconnect();
+    };
+    return { writer: { push, end }, init };
+};

package/dist/connectors/ws.d.ts

@@ -0,0 +1,10 @@
+import { ReaderConstructor, WriterConstructor } from "../connectors.js";
+export interface WsWriterConfig {
+    url: string;
+}
+export interface WsReaderConfig {
+    host: string;
+    port: number;
+}
+export declare const startWsStreamReader: ReaderConstructor<WsReaderConfig>;
+export declare const startWsStreamWriter: WriterConstructor<WsWriterConfig>;

package/dist/connectors/ws.js

@@ -0,0 +1,69 @@
+import { SimpleStream, } from "../connectors.js";
+import { WebSocket } from "ws";
+import { WebSocketServer } from "ws";
+function _connectWs(url, res) {
+    const ws = new WebSocket(url, {});
+    ws.on("error", () => {
+        setTimeout(() => _connectWs(url, res), 300);
+    });
+    ws.on("ping", () => ws.pong());
+    ws.on("open", () => {
+        res(ws);
+    });
+}
+function connectWs(url) {
+    return new Promise((res) => _connectWs(url, res));
+}
+export const startWsStreamReader = (config) => {
+    const server = new WebSocketServer(config);
+    server.on("error", (error) => {
+        console.error("Ws server error:");
+        console.error(error);
+    });
+    const connections = [];
+    const interval = setInterval(() => {
+        connections.forEach((instance, i) => {
+            if (!instance) {
+                return;
+            }
+            if (!instance.alive) {
+                instance.socket.terminate();
+                delete connections[i];
+                return;
+            }
+            instance.socket.ping();
+            instance.alive = false;
+        });
+    }, 30_000);
+    const reader = new SimpleStream(() => new Promise((res) => {
+        clearInterval(interval);
+        server.close(() => res());
+    }));
+    server.on("connection", (ws) => {
+        const instance = { socket: ws, alive: true };
+        connections.push(instance);
+        ws.on("message", async (msg) => {
+            reader.push(msg.toString()).catch((error) => {
+                throw error;
+            });
+        });
+        ws.on("pong", () => {
+            instance.alive = true;
+        });
+    });
+    return { reader, init: async () => { } };
+};
+export const startWsStreamWriter = (config) => {
+    let ws;
+    const init = async () => {
+        ws = await connectWs(config.url);
+        ws.on("open", () => console.log("open"));
+    };
+    const push = async (item) => {
+        ws.send(item);
+    };
+    const end = async () => {
+        ws.close();
+    };
+    return { writer: { push, end }, init };
+};

package/dist/connectors.d.ts

@@ -0,0 +1,52 @@
+import { NamedNode, Term } from "@rdfjs/types";
+export * from "./connectors/file.js";
+export * from "./connectors/ws.js";
+export * from "./connectors/kafka.js";
+export * from "./connectors/http.js";
+export declare const Conn: import("@treecg/types").Namespace<("FileReaderChannel" | "FileWriterChannel" | "HttpReaderChannel" | "HttpWriterChannel" | "KafkaReaderChannel" | "KafkaWriterChannel" | "WsReaderChannel" | "WsWriterChannel" | "WriterChannel" | "ReaderChannel")[], NamedNode<string>, string>;
+export interface Config<T> {
+    id: Term;
+    ty: NamedNode;
+    config: T;
+}
+export type ReaderConstructor<C> = (config: C) => {
+    reader: Stream<string | Buffer>;
+    init: () => Promise<void>;
+};
+export type WriterConstructor<C> = (config: C) => {
+    writer: Writer<string | Buffer>;
+    init: () => Promise<void>;
+};
+export declare const JsOntology: import("@treecg/types").Namespace<("JsProcess" | "JsChannel" | "JsReaderChannel" | "JsWriterChannel")[], NamedNode<string>, string>;
+export declare class ChannelFactory {
+    private inits;
+    private jsChannelsNamedNodes;
+    private jsChannelsBlankNodes;
+    createReader(config: Config<any>): Stream<string | Buffer>;
+    createWriter(config: Config<any>): Writer<string | Buffer>;
+    init(): Promise<void>;
+}
+export interface Writer<T> {
+    push(item: T): Promise<void>;
+    end(): Promise<void>;
+}
+export interface Stream<T> {
+    lastElement?: T;
+    end(): Promise<void>;
+    data(listener: (t: T) => PromiseLike<void> | void): this;
+    on(event: "data", listener: (t: T) => PromiseLike<void> | void): this;
+    on(event: "end", listener: () => PromiseLike<void> | void): this;
+}
+export type Handler<T> = (item: T) => Promise<void> | void;
+export declare class SimpleStream<T> implements Stream<T> {
+    private readonly dataHandlers;
+    private readonly endHandlers;
+    readonly disconnect: () => Promise<void>;
+    lastElement?: T | undefined;
+    constructor(onDisconnect?: () => Promise<void>);
+    data(listener: Handler<T>): this;
+    push(data: T): Promise<void>;
+    end(): Promise<void>;
+    on(event: "data", listener: Handler<T>): this;
+    on(event: "end", listener: Handler<void>): this;
+}

package/dist/connectors.js

@@ -0,0 +1,136 @@
+import { createTermNamespace } from "@treecg/types";
+import { startFileStreamReader, startFileStreamWriter, } from "./connectors/file.js";
+export * from "./connectors/file.js";
+import { startWsStreamReader, startWsStreamWriter, } from "./connectors/ws.js";
+export * from "./connectors/ws.js";
+import { startKafkaStreamReader, startKafkaStreamWriter, } from "./connectors/kafka.js";
+export * from "./connectors/kafka.js";
+import { startHttpStreamReader, startHttpStreamWriter, } from "./connectors/http.js";
+import { LOG } from "./util.js";
+export * from "./connectors/http.js";
+export const Conn = createTermNamespace("https://w3id.org/conn#", "FileReaderChannel", "FileWriterChannel", "HttpReaderChannel", "HttpWriterChannel", "KafkaReaderChannel", "KafkaWriterChannel", "WsReaderChannel", "WsWriterChannel", "WriterChannel", "ReaderChannel");
+export const JsOntology = createTermNamespace("https://w3id.org/conn/js#", "JsProcess", "JsChannel", "JsReaderChannel", "JsWriterChannel");
+export class ChannelFactory {
+    inits = [];
+    jsChannelsNamedNodes = {};
+    jsChannelsBlankNodes = {};
+    createReader(config) {
+        LOG.channel("Creating reader %s: a %s", config.id.value, config.ty.value);
+        if (config.ty.equals(Conn.FileReaderChannel)) {
+            const { reader, init } = startFileStreamReader(config.config);
+            this.inits.push(init);
+            return reader;
+        }
+        if (config.ty.equals(Conn.WsReaderChannel)) {
+            const { reader, init } = startWsStreamReader(config.config);
+            this.inits.push(init);
+            return reader;
+        }
+        if (config.ty.equals(Conn.KafkaReaderChannel)) {
+            const { reader, init } = startKafkaStreamReader(config.config);
+            this.inits.push(init);
+            return reader;
+        }
+        if (config.ty.equals(Conn.HttpReaderChannel)) {
+            const { reader, init } = startHttpStreamReader(config.config);
+            this.inits.push(init);
+            return reader;
+        }
+        if (config.ty.equals(JsOntology.JsReaderChannel)) {
+            const c = config.config;
+            if (c.channel) {
+                const id = c.channel.id.value;
+                if (c.channel.id.termType === "NamedNode") {
+                    if (!this.jsChannelsNamedNodes[id]) {
+                        this.jsChannelsNamedNodes[id] = new SimpleStream();
+                    }
+                    return this.jsChannelsNamedNodes[id];
+                }
+                if (c.channel.id.termType === "BlankNode") {
+                    if (!this.jsChannelsBlankNodes[id]) {
+                        this.jsChannelsBlankNodes[id] = new SimpleStream();
+                    }
+                    return this.jsChannelsBlankNodes[id];
+                }
+                throw "Should have found a thing";
+            }
+        }
+        throw "Unknown reader channel " + config.ty.value;
+    }
+    createWriter(config) {
+        LOG.channel("Creating writer %s: a %s", config.id.value, config.ty.value);
+        if (config.ty.equals(Conn.FileWriterChannel)) {
+            const { writer, init } = startFileStreamWriter(config.config);
+            this.inits.push(init);
+            return writer;
+        }
+        if (config.ty.equals(Conn.WsWriterChannel)) {
+            const { writer, init } = startWsStreamWriter(config.config);
+            this.inits.push(init);
+            return writer;
+        }
+        if (config.ty.equals(Conn.KafkaWriterChannel)) {
+            const { writer, init } = startKafkaStreamWriter(config.config);
+            this.inits.push(init);
+            return writer;
+        }
+        if (config.ty.equals(Conn.HttpWriterChannel)) {
+            const { writer, init } = startHttpStreamWriter(config.config);
+            this.inits.push(init);
+            return writer;
+        }
+        if (config.ty.equals(JsOntology.JsWriterChannel)) {
+            const c = config.config;
+            if (c.channel) {
+                const id = c.channel.id.value;
+                if (c.channel.id.termType === "NamedNode") {
+                    if (!this.jsChannelsNamedNodes[id]) {
+                        this.jsChannelsNamedNodes[id] = new SimpleStream();
+                    }
+                    return this.jsChannelsNamedNodes[id];
+                }
+                if (c.channel.id.termType === "BlankNode") {
+                    if (!this.jsChannelsBlankNodes[id]) {
+                        this.jsChannelsBlankNodes[id] = new SimpleStream();
+                    }
+                    return this.jsChannelsBlankNodes[id];
+                }
+                throw "Should have found a thing";
+            }
+        }
+        throw "Unknown writer channel " + config.ty.value;
+    }
+    async init() {
+        await Promise.all(this.inits.map((x) => x()));
+    }
+}
+export class SimpleStream {
+    dataHandlers = [];
+    endHandlers = [];
+    disconnect;
+    lastElement;
+    constructor(onDisconnect) {
+        this.disconnect = onDisconnect || (async () => { });
+    }
+    data(listener) {
+        this.dataHandlers.push(listener);
+        return this;
+    }
+    async push(data) {
+        this.lastElement = data;
+        await Promise.all(this.dataHandlers.map((handler) => handler(data)));
+    }
+    async end() {
+        await this.disconnect();
+        await Promise.all(this.endHandlers.map((handler) => handler()));
+    }
+    on(event, listener) {
+        if (event === "data") {
+            this.dataHandlers.push(listener);
+        }
+        if (event === "end") {
+            this.endHandlers.push(listener);
+        }
+        return this;
+    }
+}