@rdfc/js-runner 1.0.0 → 2.0.0-alpha.2

package/dist/connectors/file.js

@@ -1,89 +0,0 @@
-import { createReadStream, openSync } from "fs";
-import { appendFile, readFile, stat, writeFile } from "fs/promises";
-import { isAbsolute } from "path";
-import { watch } from "node:fs";
-import { SimpleStream, } from "../connectors.js";
-async function getFileSize(path) {
-    return (await stat(path)).size;
-}
-function readPart(path, start, end, encoding) {
-    return new Promise((res) => {
-        const stream = createReadStream(path, { encoding, start, end });
-        let buffer = "";
-        stream.on("data", (chunk) => {
-            buffer += chunk;
-        });
-        stream.on("close", () => res(buffer));
-    });
-}
-function debounce(func, timeout = 100) {
-    let timer;
-    return (...args) => {
-        clearTimeout(timer);
-        timer = setTimeout(() => {
-            func(...args);
-        }, timeout);
-    };
-}
-export const startFileStreamReader = (config) => {
-    const path = isAbsolute(config.path)
-        ? config.path
-        : `${process.cwd()}/${config.path}`;
-    openSync(path, "a+");
-    const encoding = config.encoding || "utf-8";
-    const reader = new SimpleStream();
-    const init = async () => {
-        let currentPos = await getFileSize(path);
-        const watcher = watch(path, { encoding: "utf-8" });
-        watcher.on("change", debounce(async () => {
-            try {
-                let content;
-                if (config.onReplace) {
-                    content = await readFile(path, { encoding });
-                }
-                else {
-                    const newSize = await getFileSize(path);
-                    if (newSize <= currentPos) {
-                        currentPos = newSize;
-                        return;
-                    }
-                    content = await readPart(path, currentPos, newSize, encoding);
-                    currentPos = newSize;
-                }
-                await reader.push(content);
-            }
-            catch (error) {
-                if (error.code === "ENOENT") {
-                    return;
-                }
-                throw error;
-            }
-        }));
-        if (config.onReplace && config.readFirstContent) {
-            const content = await readFile(path, { encoding });
-            await reader.push(content);
-        }
-    };
-    return { reader, init };
-};
-export const startFileStreamWriter = (config) => {
-    const path = isAbsolute(config.path)
-        ? config.path
-        : `${process.cwd()}/${config.path}`;
-    const encoding = config.encoding || "utf-8";
-    const writer = new SimpleStream();
-    const init = async () => {
-        if (!config.onReplace) {
-            await writeFile(path, "", { encoding });
-        }
-    };
-    writer.push = async (item) => {
-        if (config.onReplace) {
-            await writeFile(path, item, { encoding });
-        }
-        else {
-            await appendFile(path, item, { encoding });
-        }
-    };
-    return { writer, init };
-};

package/dist/connectors/http.d.ts

@@ -1,14 +0,0 @@
-import { ReaderConstructor, WriterConstructor } from "../connectors.js";
-export interface HttpReaderConfig {
-    endpoint: string;
-    port: number;
-    binary: boolean;
-    waitHandled?: boolean;
-    responseCode?: number;
-}
-export declare const startHttpStreamReader: ReaderConstructor<HttpReaderConfig>;
-export interface HttpWriterConfig {
-    endpoint: string;
-    method: string;
-}
-export declare const startHttpStreamWriter: WriterConstructor<HttpWriterConfig>;

package/dist/connectors/http.js

@@ -1,82 +0,0 @@
-import * as http from "http";
-import { createServer } from "http";
-import { SimpleStream, } from "../connectors.js";
-function streamToString(stream, binary) {
-    const datas = [];
-    return new Promise((res) => {
-        stream.on("data", (data) => {
-            datas.push(data);
-        });
-        stream.on("end", () => {
-            const streamData = Buffer.concat(datas);
-            res(binary ? streamData : streamData.toString());
-        });
-    });
-}
-export const startHttpStreamReader = (config) => {
-    let server = undefined;
-    const stream = new SimpleStream(() => new Promise((res) => {
-        if (server !== undefined) {
-            server.close(() => {
-                res();
-            });
-        }
-        else {
-            res();
-        }
-    }));
-    const requestListener = async function (req, res) {
-        try {
-            const content = await streamToString(req, config.binary);
-            const promise = stream.push(content).catch((error) => {
-                throw error;
-            });
-            if (config.waitHandled) {
-                await promise;
-            }
-        }
-        catch (error) {
-            console.error("Failed", error);
-        }
-        res.writeHead(config.responseCode || 200);
-        res.end("OK");
-    };
-    server = createServer(requestListener);
-    const init = () => {
-        return new Promise((res) => {
-            const cb = () => res(undefined);
-            if (server) {
-                server.listen(config.port, config.endpoint, cb);
-            }
-            else {
-                cb();
-            }
-        });
-    };
-    return { reader: stream, init };
-};
-export const startHttpStreamWriter = (config) => {
-    const requestConfig = new URL(config.endpoint);
-    const writer = new SimpleStream();
-    writer.push = async (item) => {
-        await new Promise((resolve) => {
-            const options = {
-                hostname: requestConfig.hostname,
-                path: requestConfig.path,
-                method: config.method,
-                port: requestConfig.port,
-            };
-            const cb = (response) => {
-                response.on("data", () => { });
-                response.on("end", () => {
-                    resolve(null);
-                });
-            };
-            const req = http.request(options, cb);
-            req.write(item, () => {
-                req.end();
-            });
-        });
-    };
-    return { writer, init: async () => { } };
-};

package/dist/connectors/kafka.d.ts

@@ -1,48 +0,0 @@
-import type { ProducerConfig } from "kafkajs";
-import { ReaderConstructor, WriterConstructor } from "../connectors.js";
-export interface SASLOptions {
-    mechanism: "plain";
-    username: string;
-    password: string;
-}
-export interface BrokerConfig {
-    hosts: string[];
-    ssl?: boolean;
-    sasl?: SASLOptions;
-}
-export interface ConsumerConfig {
-    groupId: string;
-    metadataMaxAge?: number;
-    sessionTimeout?: number;
-    rebalanceTimeout?: number;
-    heartbeatInterval?: number;
-    maxBytesPerPartition?: number;
-    minBytes?: number;
-    maxBytes?: number;
-    maxWaitTimeInMs?: number;
-    allowAutoTopicCreation?: boolean;
-    maxInFlightRequests?: number;
-    readUncommitted?: boolean;
-    rackId?: string;
-}
-export interface CSTopic {
-    topic: string;
-    fromBeginning?: boolean;
-}
-export interface KafkaReaderConfig {
-    topic: {
-        name: string;
-        fromBeginning?: boolean;
-    };
-    consumer: ConsumerConfig;
-    broker: string | BrokerConfig;
-}
-export declare const startKafkaStreamReader: ReaderConstructor<KafkaReaderConfig>;
-export interface KafkaWriterConfig {
-    topic: {
-        name: string;
-    };
-    producer: ProducerConfig;
-    broker: BrokerConfig | string;
-}
-export declare const startKafkaStreamWriter: WriterConstructor<KafkaWriterConfig>;

package/dist/connectors/kafka.js

@@ -1,64 +0,0 @@
-import { readFileSync } from "node:fs";
-import { Kafka } from "kafkajs";
-import { SimpleStream, } from "../connectors.js";
-export const startKafkaStreamReader = (config) => {
-    const brokerConfig = {};
-    if (typeof config.broker === "string" || config.broker instanceof String) {
-        Object.assign(brokerConfig, JSON.parse(readFileSync(config.broker, "utf-8")));
-    }
-    else {
-        Object.assign(brokerConfig, config.broker);
-    }
-    if (brokerConfig && brokerConfig.hosts) {
-        brokerConfig.brokers = (brokerConfig).hosts;
-    }
-    const kafka = new Kafka(brokerConfig);
-    const consumer = kafka.consumer(config.consumer);
-    const stream = new SimpleStream(async () => {
-        await consumer.disconnect();
-        await consumer.stop();
-    });
-    const init = async () => {
-        await consumer.connect();
-        await consumer.subscribe({
-            topic: config.topic.name,
-            fromBeginning: config.topic.fromBeginning,
-        });
-        consumer
-            .run({
-            async eachMessage({ topic, message, }) {
-                if (topic === config.topic.name) {
-                    const element = message.value?.toString() ?? "";
-                    await stream.push(element);
-                }
-            },
-        })
-            .catch((error) => {
-            throw error;
-        });
-    };
-    return { reader: stream, init };
-};
-export const startKafkaStreamWriter = (config) => {
-    const topic = config.topic.name;
-    const brokerConfig = {};
-    if (typeof config.broker === "string" || config.broker instanceof String) {
-        Object.assign(brokerConfig, JSON.parse(readFileSync(config.broker, "utf-8")));
-    }
-    else {
-        Object.assign(brokerConfig, config.broker);
-    }
-    if (brokerConfig && brokerConfig.hosts) {
-        brokerConfig.brokers = (brokerConfig).hosts;
-    }
-    const kafka = new Kafka(brokerConfig);
-    const producer = kafka.producer(config.producer);
-    const writer = new SimpleStream(async () => {
-        await producer.disconnect();
-    });
-    const init = () => producer.connect();
-    writer.push = async (item) => {
-        await producer.send({ topic, messages: [{ value: item }] });
-    };
-    return { writer, init };
-};

package/dist/connectors/ws.d.ts

@@ -1,10 +0,0 @@
-import { ReaderConstructor, WriterConstructor } from "../connectors.js";
-export interface WsWriterConfig {
-    url: string;
-}
-export interface WsReaderConfig {
-    host: string;
-    port: number;
-}
-export declare const startWsStreamReader: ReaderConstructor<WsReaderConfig>;
-export declare const startWsStreamWriter: WriterConstructor<WsWriterConfig>;

package/dist/connectors/ws.js

@@ -1,69 +0,0 @@
-import { SimpleStream, } from "../connectors.js";
-import { WebSocket } from "ws";
-import { WebSocketServer } from "ws";
-function _connectWs(url, res) {
-    const ws = new WebSocket(url, {});
-    ws.on("error", () => {
-        setTimeout(() => _connectWs(url, res), 300);
-    });
-    ws.on("ping", () => ws.pong());
-    ws.on("open", () => {
-        res(ws);
-    });
-}
-function connectWs(url) {
-    return new Promise((res) => _connectWs(url, res));
-}
-export const startWsStreamReader = (config) => {
-    const server = new WebSocketServer(config);
-    server.on("error", (error) => {
-        console.error("Ws server error:");
-        console.error(error);
-    });
-    const connections = [];
-    const interval = setInterval(() => {
-        connections.forEach((instance, i) => {
-            if (!instance) {
-                return;
-            }
-            if (!instance.alive) {
-                instance.socket.terminate();
-                delete connections[i];
-                return;
-            }
-            instance.socket.ping();
-            instance.alive = false;
-        });
-    }, 30_000);
-    const reader = new SimpleStream(() => new Promise((res) => {
-        clearInterval(interval);
-        server.close(() => res());
-    }));
-    server.on("connection", (ws) => {
-        const instance = { socket: ws, alive: true };
-        connections.push(instance);
-        ws.on("message", async (msg) => {
-            reader.push(msg.toString()).catch((error) => {
-                throw error;
-            });
-        });
-        ws.on("pong", () => {
-            instance.alive = true;
-        });
-    });
-    return { reader, init: async () => { } };
-};
-export const startWsStreamWriter = (config) => {
-    let ws;
-    const init = async () => {
-        ws = await connectWs(config.url);
-        ws.on("open", () => console.log("open"));
-    };
-    const writer = new SimpleStream(async () => {
-        ws.close();
-    });
-    writer.push = async (item) => {
-        ws.send(item);
-    };
-    return { writer, init };
-};

package/dist/connectors.d.ts

@@ -1,54 +0,0 @@
-import { NamedNode, Term } from "@rdfjs/types";
-export * from "./connectors/file.js";
-export * from "./connectors/ws.js";
-export * from "./connectors/kafka.js";
-export * from "./connectors/http.js";
-export declare const Conn: import("@treecg/types").Namespace<("FileReaderChannel" | "FileWriterChannel" | "HttpReaderChannel" | "HttpWriterChannel" | "KafkaReaderChannel" | "KafkaWriterChannel" | "WsReaderChannel" | "WsWriterChannel" | "WriterChannel" | "ReaderChannel")[], NamedNode<string>, string>;
-export interface Config<T> {
-    id: Term;
-    ty: NamedNode;
-    config: T;
-}
-export type ReaderConstructor<C> = (config: C) => {
-    reader: Stream<string | Buffer>;
-    init: () => Promise<void>;
-};
-export type WriterConstructor<C> = (config: C) => {
-    writer: Writer<string | Buffer>;
-    init: () => Promise<void>;
-};
-export declare const JsOntology: import("@treecg/types").Namespace<("JsProcess" | "JsChannel" | "JsReaderChannel" | "JsWriterChannel")[], NamedNode<string>, string>;
-export declare class ChannelFactory {
-    private inits;
-    private jsChannelsNamedNodes;
-    private jsChannelsBlankNodes;
-    createReader(config: Config<any>): Stream<string | Buffer>;
-    createWriter(config: Config<any>): Writer<string | Buffer>;
-    init(): Promise<void>;
-}
-export interface Writer<T> {
-    push(item: T): Promise<void>;
-    end(): Promise<void>;
-    on(event: "end", listener: Handler<void>): this;
-}
-export interface Stream<T> {
-    lastElement?: T;
-    end(): Promise<void>;
-    data(listener: (t: T) => PromiseLike<void> | void): this;
-    on(event: "data", listener: (t: T) => PromiseLike<void> | void): this;
-    on(event: "end", listener: () => PromiseLike<void> | void): this;
-}
-export type Handler<T> = (item: T) => Promise<void> | void;
-export declare class SimpleStream<T> implements Stream<T> {
-    private readonly dataHandlers;
-    private readonly endHandlers;
-    readonly disconnect: () => Promise<void>;
-    lastElement?: T | undefined;
-    private ended;
-    constructor(onDisconnect?: () => Promise<void>);
-    data(listener: Handler<T>): this;
-    push(data: T): Promise<void>;
-    end(): Promise<void>;
-    on(event: "data", listener: Handler<T>): this;
-    on(event: "end", listener: Handler<void>): this;
-}

package/dist/connectors.js

@@ -1,145 +0,0 @@
-import { createTermNamespace } from "@treecg/types";
-import { startFileStreamReader, startFileStreamWriter, } from "./connectors/file.js";
-export * from "./connectors/file.js";
-import { startWsStreamReader, startWsStreamWriter, } from "./connectors/ws.js";
-export * from "./connectors/ws.js";
-import { startKafkaStreamReader, startKafkaStreamWriter, } from "./connectors/kafka.js";
-export * from "./connectors/kafka.js";
-import { startHttpStreamReader, startHttpStreamWriter, } from "./connectors/http.js";
-import { LOG } from "./util.js";
-export * from "./connectors/http.js";
-export const Conn = createTermNamespace("https://w3id.org/conn#", "FileReaderChannel", "FileWriterChannel", "HttpReaderChannel", "HttpWriterChannel", "KafkaReaderChannel", "KafkaWriterChannel", "WsReaderChannel", "WsWriterChannel", "WriterChannel", "ReaderChannel");
-export const JsOntology = createTermNamespace("https://w3id.org/conn/js#", "JsProcess", "JsChannel", "JsReaderChannel", "JsWriterChannel");
-export class ChannelFactory {
-    inits = [];
-    jsChannelsNamedNodes = {};
-    jsChannelsBlankNodes = {};
-    createReader(config) {
-        LOG.channel("Creating reader %s: a %s", config.id.value, config.ty.value);
-        if (config.ty.equals(Conn.FileReaderChannel)) {
-            const { reader, init } = startFileStreamReader(config.config);
-            this.inits.push(init);
-            return reader;
-        }
-        if (config.ty.equals(Conn.WsReaderChannel)) {
-            const { reader, init } = startWsStreamReader(config.config);
-            this.inits.push(init);
-            return reader;
-        }
-        if (config.ty.equals(Conn.KafkaReaderChannel)) {
-            const { reader, init } = startKafkaStreamReader(config.config);
-            this.inits.push(init);
-            return reader;
-        }
-        if (config.ty.equals(Conn.HttpReaderChannel)) {
-            const { reader, init } = startHttpStreamReader(config.config);
-            this.inits.push(init);
-            return reader;
-        }
-        if (config.ty.equals(JsOntology.JsReaderChannel)) {
-            const c = config.config;
-            if (c.channel) {
-                const id = c.channel.id.value;
-                if (c.channel.id.termType === "NamedNode") {
-                    if (!this.jsChannelsNamedNodes[id]) {
-                        this.jsChannelsNamedNodes[id] =
-                            new SimpleStream();
-                    }
-                    return this.jsChannelsNamedNodes[id];
-                }
-                if (c.channel.id.termType === "BlankNode") {
-                    if (!this.jsChannelsBlankNodes[id]) {
-                        this.jsChannelsBlankNodes[id] =
-                            new SimpleStream();
-                    }
-                    return this.jsChannelsBlankNodes[id];
-                }
-                throw "Should have found a thing";
-            }
-        }
-        throw "Unknown reader channel " + config.ty.value;
-    }
-    createWriter(config) {
-        LOG.channel("Creating writer %s: a %s", config.id.value, config.ty.value);
-        if (config.ty.equals(Conn.FileWriterChannel)) {
-            const { writer, init } = startFileStreamWriter(config.config);
-            this.inits.push(init);
-            return writer;
-        }
-        if (config.ty.equals(Conn.WsWriterChannel)) {
-            const { writer, init } = startWsStreamWriter(config.config);
-            this.inits.push(init);
-            return writer;
-        }
-        if (config.ty.equals(Conn.KafkaWriterChannel)) {
-            const { writer, init } = startKafkaStreamWriter(config.config);
-            this.inits.push(init);
-            return writer;
-        }
-        if (config.ty.equals(Conn.HttpWriterChannel)) {
-            const { writer, init } = startHttpStreamWriter(config.config);
-            this.inits.push(init);
-            return writer;
-        }
-        if (config.ty.equals(JsOntology.JsWriterChannel)) {
-            const c = config.config;
-            if (c.channel) {
-                const id = c.channel.id.value;
-                if (c.channel.id.termType === "NamedNode") {
-                    if (!this.jsChannelsNamedNodes[id]) {
-                        this.jsChannelsNamedNodes[id] =
-                            new SimpleStream();
-                    }
-                    return this.jsChannelsNamedNodes[id];
-                }
-                if (c.channel.id.termType === "BlankNode") {
-                    if (!this.jsChannelsBlankNodes[id]) {
-                        this.jsChannelsBlankNodes[id] =
-                            new SimpleStream();
-                    }
-                    return this.jsChannelsBlankNodes[id];
-                }
-                throw "Should have found a thing";
-            }
-        }
-        throw "Unknown writer channel " + config.ty.value;
-    }
-    async init() {
-        await Promise.all(this.inits.map((x) => x()));
-    }
-}
-export class SimpleStream {
-    dataHandlers = [];
-    endHandlers = [];
-    disconnect;
-    lastElement;
-    ended = false;
-    constructor(onDisconnect) {
-        this.disconnect = onDisconnect || (async () => { });
-    }
-    data(listener) {
-        this.dataHandlers.push(listener);
-        return this;
-    }
-    async push(data) {
-        if (this.ended) {
-            throw new Error("Trying to push to a stream that has ended!");
-        }
-        this.lastElement = data;
-        await Promise.all(this.dataHandlers.map((handler) => handler(data)));
-    }
-    async end() {
-        await this.disconnect();
-        await Promise.all(this.endHandlers.map((handler) => handler()));
-        this.ended = true;
-    }
-    on(event, listener) {
-        if (event === "data") {
-            this.dataHandlers.push(listener);
-        }
-        if (event === "end") {
-            this.endHandlers.push(listener);
-        }
-        return this;
-    }
-}