@rdfc/js-runner 1.0.0 → 2.0.0-alpha.2

package/dist/index.cjs

@@ -1,677 +0,0 @@
-'use strict';
-
-var n3 = require('n3');
-var commandLineArgs = require('command-line-args');
-var commandLineUsage = require('command-line-usage');
-var fs = require('fs');
-var http = require('http');
-var https = require('https');
-var types = require('@treecg/types');
-var debug = require('debug');
-var path = require('path');
-var promises = require('fs/promises');
-var node_fs = require('node:fs');
-var ws = require('ws');
-var kafkajs = require('kafkajs');
-var rdfLens = require('rdf-lens');
-
-function _interopNamespaceDefault(e) {
-    var n = Object.create(null);
-    if (e) {
-        Object.keys(e).forEach(function (k) {
-            if (k !== 'default') {
-                var d = Object.getOwnPropertyDescriptor(e, k);
-                Object.defineProperty(n, k, d.get ? d : {
-                    enumerable: true,
-                    get: function () { return e[k]; }
-                });
-            }
-        });
-    }
-    n.default = e;
-    return Object.freeze(n);
-}
-
-var http__namespace = /*#__PURE__*/_interopNamespaceDefault(http);
-
-const optionDefinitions = [
-    {
-        name: "input",
-        type: String,
-        defaultOption: true,
-        summary: "Specify what input file to start up",
-    },
-    {
-        name: "help",
-        alias: "h",
-        type: Boolean,
-        description: "Display this help message",
-    },
-];
-const sections = [
-    {
-        header: "Js-runner",
-        content: "JS-runner is part of the {italic connector architecture}. Starting from an input file start up all JsProcessors that are defined. Please do not use blank nodes, skolemize your data somewhere else!",
-    },
-    {
-        header: "Synopsis",
-        content: "$ js-runner <input>",
-    },
-    {
-        header: "Command List",
-        content: [
-            { name: "input", summary: "Specify what input file to start up" },
-        ],
-    },
-    {
-        optionList: [optionDefinitions[1]],
-    },
-];
-function validArgs(args) {
-    if (!args.input)
-        return false;
-    return true;
-}
-function printUsage() {
-    const usage = commandLineUsage(sections);
-    console.log(usage);
-    process.exit(0);
-}
-function getArgs() {
-    let args;
-    try {
-        args = commandLineArgs(optionDefinitions);
-    }
-    catch (e) {
-        console.error(e);
-        printUsage();
-    }
-    if (args.help || !validArgs(args)) {
-        printUsage();
-    }
-    return args;
-}
-
-const LOG = (function () {
-    const main = debug("js-runner");
-    const channel = main.extend("channel");
-    const util = main.extend("util");
-    return { main, channel, util };
-})();
-function toArray(stream) {
-    const output = [];
-    return new Promise((res, rej) => {
-        stream.on("data", (x) => output.push(x));
-        stream.on("end", () => res(output));
-        stream.on("close", () => res(output));
-        stream.on("error", rej);
-    });
-}
-const OWL = types.createUriAndTermNamespace("http://www.w3.org/2002/07/owl#", "imports");
-types.createUriAndTermNamespace("https://w3id.org/conn#", "install", "build", "GitInstall", "LocalInstall", "url", "procFile", "path", "EnvVariable", "envKey", "envDefault");
-const { namedNode, literal } = n3.DataFactory;
-async function get_readstream(location) {
-    if (location.startsWith("https")) {
-        return new Promise((res) => {
-            https.get(location, res);
-        });
-    }
-    else if (location.startsWith("http")) {
-        return new Promise((res) => {
-            http.get(location, res);
-        });
-    }
-    else {
-        return fs.createReadStream(location);
-    }
-}
-async function load_quads(location, baseIRI) {
-    try {
-        LOG.util("Loading quads %s", location);
-        const parser = new n3.StreamParser({ baseIRI: baseIRI || location });
-        const rdfStream = await get_readstream(location);
-        rdfStream.pipe(parser);
-        const quads = await toArray(parser);
-        return quads;
-    }
-    catch (ex) {
-        console.error("Failed to load_quads", location, baseIRI);
-        console.error(ex);
-        return [];
-    }
-}
-function load_memory_quads(value, baseIRI) {
-    const parser = new n3.Parser({ baseIRI });
-    return parser.parse(value);
-}
-const loaded = new Set();
-async function load_store(location, store, recursive = true) {
-    if (loaded.has(location))
-        return;
-    loaded.add(location);
-    const quads = location.type === "remote"
-        ? await load_quads(location.location)
-        : load_memory_quads(location.value, location.baseIRI);
-    store.addQuads(quads);
-    if (recursive) {
-        const loc = location.type === "remote" ? location.location : location.baseIRI;
-        const other_imports = store.getObjects(namedNode(loc), OWL.terms.imports, null);
-        for (const other of other_imports) {
-            await load_store({ location: other.value, type: "remote" }, store, true);
-        }
-    }
-}
-
-async function getFileSize(path) {
-    return (await promises.stat(path)).size;
-}
-function readPart(path, start, end, encoding) {
-    return new Promise((res) => {
-        const stream = fs.createReadStream(path, { encoding, start, end });
-        let buffer = "";
-        stream.on("data", (chunk) => {
-            buffer += chunk;
-        });
-        stream.on("close", () => res(buffer));
-    });
-}
-function debounce(func, timeout = 100) {
-    let timer;
-    return (...args) => {
-        clearTimeout(timer);
-        timer = setTimeout(() => {
-            func(...args);
-        }, timeout);
-    };
-}
-const startFileStreamReader = (config) => {
-    const path$1 = path.isAbsolute(config.path)
-        ? config.path
-        : `${process.cwd()}/${config.path}`;
-    fs.openSync(path$1, "a+");
-    const encoding = config.encoding || "utf-8";
-    const reader = new SimpleStream();
-    const init = async () => {
-        let currentPos = await getFileSize(path$1);
-        const watcher = node_fs.watch(path$1, { encoding: "utf-8" });
-        watcher.on("change", debounce(async () => {
-            try {
-                let content;
-                if (config.onReplace) {
-                    content = await promises.readFile(path$1, { encoding });
-                }
-                else {
-                    const newSize = await getFileSize(path$1);
-                    if (newSize <= currentPos) {
-                        currentPos = newSize;
-                        return;
-                    }
-                    content = await readPart(path$1, currentPos, newSize, encoding);
-                    currentPos = newSize;
-                }
-                await reader.push(content);
-            }
-            catch (error) {
-                if (error.code === "ENOENT") {
-                    return;
-                }
-                throw error;
-            }
-        }));
-        if (config.onReplace && config.readFirstContent) {
-            const content = await promises.readFile(path$1, { encoding });
-            await reader.push(content);
-        }
-    };
-    return { reader, init };
-};
-const startFileStreamWriter = (config) => {
-    const path$1 = path.isAbsolute(config.path)
-        ? config.path
-        : `${process.cwd()}/${config.path}`;
-    const encoding = config.encoding || "utf-8";
-    const writer = new SimpleStream();
-    const init = async () => {
-        if (!config.onReplace) {
-            await promises.writeFile(path$1, "", { encoding });
-        }
-    };
-    writer.push = async (item) => {
-        if (config.onReplace) {
-            await promises.writeFile(path$1, item, { encoding });
-        }
-        else {
-            await promises.appendFile(path$1, item, { encoding });
-        }
-    };
-    return { writer, init };
-};
-
-function _connectWs(url, res) {
-    const ws$1 = new ws.WebSocket(url, {});
-    ws$1.on("error", () => {
-        setTimeout(() => _connectWs(url, res), 300);
-    });
-    ws$1.on("ping", () => ws$1.pong());
-    ws$1.on("open", () => {
-        res(ws$1);
-    });
-}
-function connectWs(url) {
-    return new Promise((res) => _connectWs(url, res));
-}
-const startWsStreamReader = (config) => {
-    const server = new ws.WebSocketServer(config);
-    server.on("error", (error) => {
-        console.error("Ws server error:");
-        console.error(error);
-    });
-    const connections = [];
-    const interval = setInterval(() => {
-        connections.forEach((instance, i) => {
-            if (!instance) {
-                return;
-            }
-            if (!instance.alive) {
-                instance.socket.terminate();
-                delete connections[i];
-                return;
-            }
-            instance.socket.ping();
-            instance.alive = false;
-        });
-    }, 30_000);
-    const reader = new SimpleStream(() => new Promise((res) => {
-        clearInterval(interval);
-        server.close(() => res());
-    }));
-    server.on("connection", (ws) => {
-        const instance = { socket: ws, alive: true };
-        connections.push(instance);
-        ws.on("message", async (msg) => {
-            reader.push(msg.toString()).catch((error) => {
-                throw error;
-            });
-        });
-        ws.on("pong", () => {
-            instance.alive = true;
-        });
-    });
-    return { reader, init: async () => { } };
-};
-const startWsStreamWriter = (config) => {
-    let ws;
-    const init = async () => {
-        ws = await connectWs(config.url);
-        ws.on("open", () => console.log("open"));
-    };
-    const writer = new SimpleStream(async () => {
-        ws.close();
-    });
-    writer.push = async (item) => {
-        ws.send(item);
-    };
-    return { writer, init };
-};
-
-const startKafkaStreamReader = (config) => {
-    const brokerConfig = {};
-    if (typeof config.broker === "string" || config.broker instanceof String) {
-        Object.assign(brokerConfig, JSON.parse(node_fs.readFileSync(config.broker, "utf-8")));
-    }
-    else {
-        Object.assign(brokerConfig, config.broker);
-    }
-    if (brokerConfig && brokerConfig.hosts) {
-        brokerConfig.brokers = (brokerConfig).hosts;
-    }
-    const kafka = new kafkajs.Kafka(brokerConfig);
-    const consumer = kafka.consumer(config.consumer);
-    const stream = new SimpleStream(async () => {
-        await consumer.disconnect();
-        await consumer.stop();
-    });
-    const init = async () => {
-        await consumer.connect();
-        await consumer.subscribe({
-            topic: config.topic.name,
-            fromBeginning: config.topic.fromBeginning,
-        });
-        consumer
-            .run({
-            async eachMessage({ topic, message, }) {
-                if (topic === config.topic.name) {
-                    const element = message.value?.toString() ?? "";
-                    stream.push(element).catch((error) => {
-                        throw error;
-                    });
-                }
-            },
-        })
-            .catch((error) => {
-            throw error;
-        });
-    };
-    return { reader: stream, init };
-};
-const startKafkaStreamWriter = (config) => {
-    const topic = config.topic.name;
-    const brokerConfig = {};
-    if (typeof config.broker === "string" || config.broker instanceof String) {
-        Object.assign(brokerConfig, JSON.parse(node_fs.readFileSync(config.broker, "utf-8")));
-    }
-    else {
-        Object.assign(brokerConfig, config.broker);
-    }
-    if (brokerConfig && brokerConfig.hosts) {
-        brokerConfig.brokers = (brokerConfig).hosts;
-    }
-    const kafka = new kafkajs.Kafka(brokerConfig);
-    const producer = kafka.producer(config.producer);
-    const writer = new SimpleStream(async () => {
-        await producer.disconnect();
-    });
-    const init = () => producer.connect();
-    writer.push = async (item) => {
-        await producer.send({ topic, messages: [{ value: item }] });
-    };
-    return { writer, init };
-};
-
-function streamToString(stream, binary) {
-    const datas = [];
-    return new Promise((res) => {
-        stream.on("data", (data) => {
-            datas.push(data);
-        });
-        stream.on("end", () => {
-            const streamData = Buffer.concat(datas);
-            res(binary ? streamData : streamData.toString());
-        });
-    });
-}
-const startHttpStreamReader = (config) => {
-    let server = undefined;
-    const stream = new SimpleStream(() => new Promise((res) => {
-        if (server !== undefined) {
-            server.close(() => {
-                res();
-            });
-        }
-        else {
-            res();
-        }
-    }));
-    const requestListener = async function (req, res) {
-        try {
-            const content = await streamToString(req, config.binary);
-            const promise = stream.push(content).catch((error) => {
-                throw error;
-            });
-            if (config.waitHandled) {
-                await promise;
-            }
-        }
-        catch (error) {
-            console.error("Failed", error);
-        }
-        res.writeHead(config.responseCode || 200);
-        res.end("OK");
-    };
-    server = http.createServer(requestListener);
-    const init = () => {
-        return new Promise((res) => {
-            const cb = () => res(undefined);
-            if (server) {
-                server.listen(config.port, config.endpoint, cb);
-            }
-            else {
-                cb();
-            }
-        });
-    };
-    return { reader: stream, init };
-};
-const startHttpStreamWriter = (config) => {
-    const requestConfig = new URL(config.endpoint);
-    const writer = new SimpleStream();
-    writer.push = async (item) => {
-        await new Promise((resolve) => {
-            const options = {
-                hostname: requestConfig.hostname,
-                path: requestConfig.path,
-                method: config.method,
-                port: requestConfig.port,
-            };
-            const cb = (response) => {
-                response.on("data", () => { });
-                response.on("end", () => {
-                    resolve(null);
-                });
-            };
-            const req = http__namespace.request(options, cb);
-            req.write(item, () => {
-                req.end();
-            });
-        });
-    };
-    return { writer, init: async () => { } };
-};
-
-const Conn = types.createTermNamespace("https://w3id.org/conn#", "FileReaderChannel", "FileWriterChannel", "HttpReaderChannel", "HttpWriterChannel", "KafkaReaderChannel", "KafkaWriterChannel", "WsReaderChannel", "WsWriterChannel", "WriterChannel", "ReaderChannel");
-const JsOntology = types.createTermNamespace("https://w3id.org/conn/js#", "JsProcess", "JsChannel", "JsReaderChannel", "JsWriterChannel");
-class ChannelFactory {
-    inits = [];
-    jsChannelsNamedNodes = {};
-    jsChannelsBlankNodes = {};
-    createReader(config) {
-        LOG.channel("Creating reader %s: a %s", config.id.value, config.ty.value);
-        if (config.ty.equals(Conn.FileReaderChannel)) {
-            const { reader, init } = startFileStreamReader(config.config);
-            this.inits.push(init);
-            return reader;
-        }
-        if (config.ty.equals(Conn.WsReaderChannel)) {
-            const { reader, init } = startWsStreamReader(config.config);
-            this.inits.push(init);
-            return reader;
-        }
-        if (config.ty.equals(Conn.KafkaReaderChannel)) {
-            const { reader, init } = startKafkaStreamReader(config.config);
-            this.inits.push(init);
-            return reader;
-        }
-        if (config.ty.equals(Conn.HttpReaderChannel)) {
-            const { reader, init } = startHttpStreamReader(config.config);
-            this.inits.push(init);
-            return reader;
-        }
-        if (config.ty.equals(JsOntology.JsReaderChannel)) {
-            const c = config.config;
-            if (c.channel) {
-                const id = c.channel.id.value;
-                if (c.channel.id.termType === "NamedNode") {
-                    if (!this.jsChannelsNamedNodes[id]) {
-                        this.jsChannelsNamedNodes[id] =
-                            new SimpleStream();
-                    }
-                    return this.jsChannelsNamedNodes[id];
-                }
-                if (c.channel.id.termType === "BlankNode") {
-                    if (!this.jsChannelsBlankNodes[id]) {
-                        this.jsChannelsBlankNodes[id] =
-                            new SimpleStream();
-                    }
-                    return this.jsChannelsBlankNodes[id];
-                }
-                throw "Should have found a thing";
-            }
-        }
-        throw "Unknown reader channel " + config.ty.value;
-    }
-    createWriter(config) {
-        LOG.channel("Creating writer %s: a %s", config.id.value, config.ty.value);
-        if (config.ty.equals(Conn.FileWriterChannel)) {
-            const { writer, init } = startFileStreamWriter(config.config);
-            this.inits.push(init);
-            return writer;
-        }
-        if (config.ty.equals(Conn.WsWriterChannel)) {
-            const { writer, init } = startWsStreamWriter(config.config);
-            this.inits.push(init);
-            return writer;
-        }
-        if (config.ty.equals(Conn.KafkaWriterChannel)) {
-            const { writer, init } = startKafkaStreamWriter(config.config);
-            this.inits.push(init);
-            return writer;
-        }
-        if (config.ty.equals(Conn.HttpWriterChannel)) {
-            const { writer, init } = startHttpStreamWriter(config.config);
-            this.inits.push(init);
-            return writer;
-        }
-        if (config.ty.equals(JsOntology.JsWriterChannel)) {
-            const c = config.config;
-            if (c.channel) {
-                const id = c.channel.id.value;
-                if (c.channel.id.termType === "NamedNode") {
-                    if (!this.jsChannelsNamedNodes[id]) {
-                        this.jsChannelsNamedNodes[id] =
-                            new SimpleStream();
-                    }
-                    return this.jsChannelsNamedNodes[id];
-                }
-                if (c.channel.id.termType === "BlankNode") {
-                    if (!this.jsChannelsBlankNodes[id]) {
-                        this.jsChannelsBlankNodes[id] =
-                            new SimpleStream();
-                    }
-                    return this.jsChannelsBlankNodes[id];
-                }
-                throw "Should have found a thing";
-            }
-        }
-        throw "Unknown writer channel " + config.ty.value;
-    }
-    async init() {
-        await Promise.all(this.inits.map((x) => x()));
-    }
-}
-class SimpleStream {
-    dataHandlers = [];
-    endHandlers = [];
-    disconnect;
-    lastElement;
-    ended = false;
-    constructor(onDisconnect) {
-        this.disconnect = onDisconnect || (async () => { });
-    }
-    data(listener) {
-        this.dataHandlers.push(listener);
-        return this;
-    }
-    async push(data) {
-        if (this.ended) {
-            throw new Error("Trying to push to a stream that has ended!");
-        }
-        this.lastElement = data;
-        await Promise.all(this.dataHandlers.map((handler) => handler(data)));
-    }
-    async end() {
-        await this.disconnect();
-        await Promise.all(this.endHandlers.map((handler) => handler()));
-        this.ended = true;
-    }
-    on(event, listener) {
-        if (event === "data") {
-            this.dataHandlers.push(listener);
-        }
-        if (event === "end") {
-            this.endHandlers.push(listener);
-        }
-        return this;
-    }
-}
-
-function safeJoin(a, b) {
-    if (b.startsWith("/")) {
-        return b;
-    }
-    return path.join(a, b);
-}
-async function extractProcessors(source, apply) {
-    const store = new n3.Store();
-    await load_store(source, store);
-    const quads = store.getQuads(null, null, null, null);
-    const config = rdfLens.extractShapes(quads, apply);
-    const subjects = quads
-        .filter((x) => x.predicate.equals(types.RDF.terms.type) &&
-        x.object.equals(JsOntology.JsProcess))
-        .map((x) => x.subject);
-    const processorLens = config.lenses[JsOntology.JsProcess.value];
-    const processors = subjects.map((id) => processorLens.execute({ id, quads }));
-    return { processors, quads, shapes: config };
-}
-function extractSteps(proc, quads, config) {
-    const out = [];
-    const subjects = quads
-        .filter((x) => x.predicate.equals(types.RDF.terms.type) && x.object.equals(proc.ty))
-        .map((x) => x.subject);
-    const processorLens = config.lenses[proc.ty.value];
-    const fields = proc.mapping.parameters;
-    for (const id of subjects) {
-        const obj = processorLens.execute({ id, quads });
-        const functionArgs = new Array(fields.length);
-        for (const field of fields) {
-            functionArgs[field.position] = obj[field.parameter];
-        }
-        out.push(functionArgs);
-    }
-    return out;
-}
-async function jsRunner() {
-    const args = getArgs();
-    const cwd = process.cwd();
-    const source = {
-        location: safeJoin(cwd, args.input).replaceAll("\\", "/"),
-        type: "remote",
-    };
-    const factory = new ChannelFactory();
-    const apply = {};
-    apply[Conn.ReaderChannel.value] = factory.createReader.bind(factory);
-    apply[Conn.WriterChannel.value] = factory.createWriter.bind(factory);
-    const { processors, quads, shapes: config, } = await extractProcessors(source, apply);
-    LOG.main("Found %d processors", processors.length);
-    const starts = [];
-    for (const proc of processors) {
-        const argss = extractSteps(proc, quads, config);
-        const jsProgram = await import("file://" + proc.file);
-        process.chdir(proc.location);
-        for (const args of argss) {
-            starts.push(await jsProgram[proc.func](...args));
-        }
-    }
-    await factory.init();
-    for (const s of starts) {
-        if (s && typeof s === "function") {
-            s();
-        }
-    }
-}
-
-exports.ChannelFactory = ChannelFactory;
-exports.Conn = Conn;
-exports.JsOntology = JsOntology;
-exports.SimpleStream = SimpleStream;
-exports.extractProcessors = extractProcessors;
-exports.extractSteps = extractSteps;
-exports.jsRunner = jsRunner;
-exports.startFileStreamReader = startFileStreamReader;
-exports.startFileStreamWriter = startFileStreamWriter;
-exports.startHttpStreamReader = startHttpStreamReader;
-exports.startHttpStreamWriter = startHttpStreamWriter;
-exports.startKafkaStreamReader = startKafkaStreamReader;
-exports.startKafkaStreamWriter = startKafkaStreamWriter;
-exports.startWsStreamReader = startWsStreamReader;
-exports.startWsStreamWriter = startWsStreamWriter;

package/dist/index.d.ts

@@ -1,33 +0,0 @@
-import { Quad, Term } from "@rdfjs/types";
-import { Shapes } from "rdf-lens";
-export * from "./connectors.js";
-type Processor = {
-    ty: Term;
-    file: string;
-    location: string;
-    func: string;
-    mapping: {
-        parameters: {
-            parameter: string;
-            position: number;
-        }[];
-    };
-};
-export type Source = {
-    type: "remote";
-    location: string;
-} | {
-    type: "memory";
-    value: string;
-    baseIRI: string;
-};
-export type Extracted = {
-    processors: Processor[];
-    quads: Quad[];
-    shapes: Shapes;
-};
-export declare function extractProcessors(source: Source, apply?: {
-    [label: string]: (item: any) => any;
-}): Promise<Extracted>;
-export declare function extractSteps(proc: Processor, quads: Quad[], config: Shapes): any[][];
-export declare function jsRunner(): Promise<void>;