@rdfc/js-runner 3.0.2 → 3.0.4-remote

package/dist/args.js

@@ -1,58 +0,0 @@
-import commandLineArgs from 'command-line-args'
-import commandLineUsage from 'command-line-usage'
-const optionDefinitions = [
-  {
-    name: 'input',
-    type: String,
-    defaultOption: true,
-    summary: 'Specify what input file to start up',
-  },
-  {
-    name: 'help',
-    alias: 'h',
-    type: Boolean,
-    description: 'Display this help message',
-  },
-]
-const sections = [
-  {
-    header: 'Js-runner',
-    content:
-      'JS-runner is part of the {italic connector architecture}. Starting from an input file start up all JsProcessors that are defined. Please do not use blank nodes, skolemize your data somewhere else!',
-  },
-  {
-    header: 'Synopsis',
-    content: '$ js-runner <input>',
-  },
-  {
-    header: 'Command List',
-    content: [
-      { name: 'input', summary: 'Specify what input file to start up' },
-    ],
-  },
-  {
-    optionList: [optionDefinitions[1]],
-  },
-]
-function validArgs(args) {
-  if (!args.input) return false
-  return true
-}
-function printUsage() {
-  const usage = commandLineUsage(sections)
-  console.log(usage)
-  process.exit(0)
-}
-export function getArgs() {
-  let args
-  try {
-    args = commandLineArgs(optionDefinitions)
-  } catch (e) {
-    console.error(e)
-    printUsage()
-  }
-  if (args.help || !validArgs(args)) {
-    printUsage()
-  }
-  return args
-}

package/dist/connectors/file.d.ts

@@ -1,15 +0,0 @@
-import { ReaderConstructor, WriterConstructor } from '../connectors.js'
-export interface FileReaderConfig {
-  path: string
-  onReplace: boolean
-  readFirstContent?: boolean
-  encoding?: string
-}
-export interface FileWriterConfig {
-  path: string
-  onReplace: boolean
-  readFirstContent?: boolean
-  encoding?: string
-}
-export declare const startFileStreamReader: ReaderConstructor<FileReaderConfig>
-export declare const startFileStreamWriter: WriterConstructor<FileWriterConfig>

package/dist/connectors/file.js

@@ -1,89 +0,0 @@
-import { createReadStream, openSync } from 'fs'
-import { appendFile, readFile, stat, writeFile } from 'fs/promises'
-import { isAbsolute } from 'path'
-import { watch } from 'node:fs'
-import { SimpleStream } from '../connectors.js'
-async function getFileSize(path) {
-  return (await stat(path)).size
-}
-function readPart(path, start, end, encoding) {
-  return new Promise((res) => {
-    const stream = createReadStream(path, { encoding, start, end })
-    let buffer = ''
-    stream.on('data', (chunk) => {
-      buffer += chunk
-    })
-    stream.on('close', () => res(buffer))
-  })
-}
-function debounce(func, timeout = 100) {
-  let timer
-  return (...args) => {
-    clearTimeout(timer)
-    timer = setTimeout(() => {
-      func(...args)
-    }, timeout)
-  }
-}
-export const startFileStreamReader = (config) => {
-  const path = isAbsolute(config.path)
-    ? config.path
-    : `${process.cwd()}/${config.path}`
-  openSync(path, 'a+')
-  const encoding = config.encoding || 'utf-8'
-  const reader = new SimpleStream()
-  const init = async () => {
-    let currentPos = await getFileSize(path)
-    const watcher = watch(path, { encoding: 'utf-8' })
-    watcher.on(
-      'change',
-      debounce(async () => {
-        try {
-          let content
-          if (config.onReplace) {
-            content = await readFile(path, { encoding })
-          } else {
-            const newSize = await getFileSize(path)
-            if (newSize <= currentPos) {
-              currentPos = newSize
-              return
-            }
-            content = await readPart(path, currentPos, newSize, encoding)
-            currentPos = newSize
-          }
-          await reader.push(content)
-        } catch (error) {
-          if (error.code === 'ENOENT') {
-            return
-          }
-          throw error
-        }
-      }),
-    )
-    if (config.onReplace && config.readFirstContent) {
-      const content = await readFile(path, { encoding })
-      await reader.push(content)
-    }
-  }
-  return { reader, init }
-}
-export const startFileStreamWriter = (config) => {
-  const path = isAbsolute(config.path)
-    ? config.path
-    : `${process.cwd()}/${config.path}`
-  const encoding = config.encoding || 'utf-8'
-  const writer = new SimpleStream()
-  const init = async () => {
-    if (!config.onReplace) {
-      await writeFile(path, '', { encoding })
-    }
-  }
-  writer.push = async (item) => {
-    if (config.onReplace) {
-      await writeFile(path, item, { encoding })
-    } else {
-      await appendFile(path, item, { encoding })
-    }
-  }
-  return { writer, init }
-}

package/dist/connectors/http.d.ts

@@ -1,14 +0,0 @@
-import { ReaderConstructor, WriterConstructor } from '../connectors.js'
-export interface HttpReaderConfig {
-  endpoint: string
-  port: number
-  binary: boolean
-  waitHandled?: boolean
-  responseCode?: number
-}
-export declare const startHttpStreamReader: ReaderConstructor<HttpReaderConfig>
-export interface HttpWriterConfig {
-  endpoint: string
-  method: string
-}
-export declare const startHttpStreamWriter: WriterConstructor<HttpWriterConfig>

package/dist/connectors/http.js

@@ -1,82 +0,0 @@
-import * as http from 'http'
-import { createServer } from 'http'
-import { SimpleStream } from '../connectors.js'
-function streamToString(stream, binary) {
-  const datas = []
-  return new Promise((res) => {
-    stream.on('data', (data) => {
-      datas.push(data)
-    })
-    stream.on('end', () => {
-      const streamData = Buffer.concat(datas)
-      res(binary ? streamData : streamData.toString())
-    })
-  })
-}
-export const startHttpStreamReader = (config) => {
-  let server = undefined
-  const stream = new SimpleStream(
-    () =>
-      new Promise((res) => {
-        if (server !== undefined) {
-          server.close(() => {
-            res()
-          })
-        } else {
-          res()
-        }
-      }),
-  )
-  const requestListener = async function (req, res) {
-    try {
-      const content = await streamToString(req, config.binary)
-      const promise = stream.push(content).catch((error) => {
-        throw error
-      })
-      if (config.waitHandled) {
-        await promise
-      }
-    } catch (error) {
-      console.error('Failed', error)
-    }
-    res.writeHead(config.responseCode || 200)
-    res.end('OK')
-  }
-  server = createServer(requestListener)
-  const init = () => {
-    return new Promise((res) => {
-      const cb = () => res(undefined)
-      if (server) {
-        server.listen(config.port, config.endpoint, cb)
-      } else {
-        cb()
-      }
-    })
-  }
-  return { reader: stream, init }
-}
-export const startHttpStreamWriter = (config) => {
-  const requestConfig = new URL(config.endpoint)
-  const writer = new SimpleStream()
-  writer.push = async (item) => {
-    await new Promise((resolve) => {
-      const options = {
-        hostname: requestConfig.hostname,
-        path: requestConfig.path,
-        method: config.method,
-        port: requestConfig.port,
-      }
-      const cb = (response) => {
-        response.on('data', () => {})
-        response.on('end', () => {
-          resolve(null)
-        })
-      }
-      const req = http.request(options, cb)
-      req.write(item, () => {
-        req.end()
-      })
-    })
-  }
-  return { writer, init: async () => {} }
-}

package/dist/connectors/kafka.d.ts

@@ -1,48 +0,0 @@
-import type { ProducerConfig } from 'kafkajs'
-import { ReaderConstructor, WriterConstructor } from '../connectors.js'
-export interface SASLOptions {
-  mechanism: 'plain'
-  username: string
-  password: string
-}
-export interface BrokerConfig {
-  hosts: string[]
-  ssl?: boolean
-  sasl?: SASLOptions
-}
-export interface ConsumerConfig {
-  groupId: string
-  metadataMaxAge?: number
-  sessionTimeout?: number
-  rebalanceTimeout?: number
-  heartbeatInterval?: number
-  maxBytesPerPartition?: number
-  minBytes?: number
-  maxBytes?: number
-  maxWaitTimeInMs?: number
-  allowAutoTopicCreation?: boolean
-  maxInFlightRequests?: number
-  readUncommitted?: boolean
-  rackId?: string
-}
-export interface CSTopic {
-  topic: string
-  fromBeginning?: boolean
-}
-export interface KafkaReaderConfig {
-  topic: {
-    name: string
-    fromBeginning?: boolean
-  }
-  consumer: ConsumerConfig
-  broker: string | BrokerConfig
-}
-export declare const startKafkaStreamReader: ReaderConstructor<KafkaReaderConfig>
-export interface KafkaWriterConfig {
-  topic: {
-    name: string
-  }
-  producer: ProducerConfig
-  broker: BrokerConfig | string
-}
-export declare const startKafkaStreamWriter: WriterConstructor<KafkaWriterConfig>

package/dist/connectors/kafka.js

@@ -1,68 +0,0 @@
-import { readFileSync } from 'node:fs'
-import { Kafka } from 'kafkajs'
-import { SimpleStream } from '../connectors.js'
-export const startKafkaStreamReader = (config) => {
-  const brokerConfig = {}
-  if (typeof config.broker === 'string' || config.broker instanceof String) {
-    Object.assign(
-      brokerConfig,
-      JSON.parse(readFileSync(config.broker, 'utf-8')),
-    )
-  } else {
-    Object.assign(brokerConfig, config.broker)
-  }
-  if (brokerConfig && brokerConfig.hosts) {
-    brokerConfig.brokers = brokerConfig.hosts
-  }
-  const kafka = new Kafka(brokerConfig)
-  const consumer = kafka.consumer(config.consumer)
-  const stream = new SimpleStream(async () => {
-    await consumer.disconnect()
-    await consumer.stop()
-  })
-  const init = async () => {
-    await consumer.connect()
-    await consumer.subscribe({
-      topic: config.topic.name,
-      fromBeginning: config.topic.fromBeginning,
-    })
-    consumer
-      .run({
-        async eachMessage({ topic, message }) {
-          if (topic === config.topic.name) {
-            const element = message.value?.toString() ?? ''
-            await stream.push(element)
-          }
-        },
-      })
-      .catch((error) => {
-        throw error
-      })
-  }
-  return { reader: stream, init }
-}
-export const startKafkaStreamWriter = (config) => {
-  const topic = config.topic.name
-  const brokerConfig = {}
-  if (typeof config.broker === 'string' || config.broker instanceof String) {
-    Object.assign(
-      brokerConfig,
-      JSON.parse(readFileSync(config.broker, 'utf-8')),
-    )
-  } else {
-    Object.assign(brokerConfig, config.broker)
-  }
-  if (brokerConfig && brokerConfig.hosts) {
-    brokerConfig.brokers = brokerConfig.hosts
-  }
-  const kafka = new Kafka(brokerConfig)
-  const producer = kafka.producer(config.producer)
-  const writer = new SimpleStream(async () => {
-    await producer.disconnect()
-  })
-  const init = () => producer.connect()
-  writer.push = async (item) => {
-    await producer.send({ topic, messages: [{ value: item }] })
-  }
-  return { writer, init }
-}

package/dist/connectors/ws.d.ts

@@ -1,10 +0,0 @@
-import { ReaderConstructor, WriterConstructor } from '../connectors.js'
-export interface WsWriterConfig {
-  url: string
-}
-export interface WsReaderConfig {
-  host: string
-  port: number
-}
-export declare const startWsStreamReader: ReaderConstructor<WsReaderConfig>
-export declare const startWsStreamWriter: WriterConstructor<WsWriterConfig>

package/dist/connectors/ws.js

@@ -1,72 +0,0 @@
-import { SimpleStream } from '../connectors.js'
-import { WebSocket } from 'ws'
-import { WebSocketServer } from 'ws'
-function _connectWs(url, res) {
-  const ws = new WebSocket(url, {})
-  ws.on('error', () => {
-    setTimeout(() => _connectWs(url, res), 300)
-  })
-  ws.on('ping', () => ws.pong())
-  ws.on('open', () => {
-    res(ws)
-  })
-}
-function connectWs(url) {
-  return new Promise((res) => _connectWs(url, res))
-}
-export const startWsStreamReader = (config) => {
-  const server = new WebSocketServer(config)
-  server.on('error', (error) => {
-    console.error('Ws server error:')
-    console.error(error)
-  })
-  const connections = []
-  const interval = setInterval(() => {
-    connections.forEach((instance, i) => {
-      if (!instance) {
-        return
-      }
-      if (!instance.alive) {
-        instance.socket.terminate()
-        delete connections[i]
-        return
-      }
-      instance.socket.ping()
-      instance.alive = false
-    })
-  }, 30_000)
-  const reader = new SimpleStream(
-    () =>
-      new Promise((res) => {
-        clearInterval(interval)
-        server.close(() => res())
-      }),
-  )
-  server.on('connection', (ws) => {
-    const instance = { socket: ws, alive: true }
-    connections.push(instance)
-    ws.on('message', async (msg) => {
-      reader.push(msg.toString()).catch((error) => {
-        throw error
-      })
-    })
-    ws.on('pong', () => {
-      instance.alive = true
-    })
-  })
-  return { reader, init: async () => {} }
-}
-export const startWsStreamWriter = (config) => {
-  let ws
-  const init = async () => {
-    ws = await connectWs(config.url)
-    ws.on('open', () => console.log('open'))
-  }
-  const writer = new SimpleStream(async () => {
-    ws.close()
-  })
-  writer.push = async (item) => {
-    ws.send(item)
-  }
-  return { writer, init }
-}

package/dist/connectors.d.ts

@@ -1,73 +0,0 @@
-import { NamedNode, Term } from '@rdfjs/types'
-export * from './connectors/file.js'
-export * from './connectors/ws.js'
-export * from './connectors/kafka.js'
-export * from './connectors/http.js'
-export declare const Conn: import('@treecg/types').Namespace<
-  (
-    | 'FileReaderChannel'
-    | 'FileWriterChannel'
-    | 'HttpReaderChannel'
-    | 'HttpWriterChannel'
-    | 'KafkaReaderChannel'
-    | 'KafkaWriterChannel'
-    | 'WsReaderChannel'
-    | 'WsWriterChannel'
-    | 'WriterChannel'
-    | 'ReaderChannel'
-  )[],
-  NamedNode<string>,
-  string
->
-export interface Config<T> {
-  id: Term
-  ty: NamedNode
-  config: T
-}
-export type ReaderConstructor<C> = (config: C) => {
-  reader: Stream<string | Buffer>
-  init: () => Promise<void>
-}
-export type WriterConstructor<C> = (config: C) => {
-  writer: Writer<string | Buffer>
-  init: () => Promise<void>
-}
-export declare const JsOntology: import('@treecg/types').Namespace<
-  ('JsProcess' | 'JsChannel' | 'JsReaderChannel' | 'JsWriterChannel')[],
-  NamedNode<string>,
-  string
->
-export declare class ChannelFactory {
-  private inits
-  private jsChannelsNamedNodes
-  private jsChannelsBlankNodes
-  createReader(config: Config<any>): Stream<string | Buffer>
-  createWriter(config: Config<any>): Writer<string | Buffer>
-  init(): Promise<void>
-}
-export interface Writer<T> {
-  push(item: T): Promise<void>
-  end(): Promise<void>
-  on(event: 'end', listener: Handler<void>): this
-}
-export interface Stream<T> {
-  lastElement?: T
-  end(): Promise<void>
-  data(listener: (t: T) => PromiseLike<void> | void): this
-  on(event: 'data', listener: (t: T) => PromiseLike<void> | void): this
-  on(event: 'end', listener: () => PromiseLike<void> | void): this
-}
-export type Handler<T> = (item: T) => Promise<void> | void
-export declare class SimpleStream<T> implements Stream<T> {
-  private readonly dataHandlers
-  private readonly endHandlers
-  readonly disconnect: () => Promise<void>
-  lastElement?: T | undefined
-  private ended
-  constructor(onDisconnect?: () => Promise<void>)
-  data(listener: Handler<T>): this
-  push(data: T): Promise<void>
-  end(): Promise<void>
-  on(event: 'data', listener: Handler<T>): this
-  on(event: 'end', listener: Handler<void>): this
-}

package/dist/connectors.js

@@ -1,168 +0,0 @@
-import { createTermNamespace } from '@treecg/types'
-import {
-  startFileStreamReader,
-  startFileStreamWriter,
-} from './connectors/file.js'
-export * from './connectors/file.js'
-import { startWsStreamReader, startWsStreamWriter } from './connectors/ws.js'
-export * from './connectors/ws.js'
-import {
-  startKafkaStreamReader,
-  startKafkaStreamWriter,
-} from './connectors/kafka.js'
-export * from './connectors/kafka.js'
-import {
-  startHttpStreamReader,
-  startHttpStreamWriter,
-} from './connectors/http.js'
-import { LOG } from './util.js'
-export * from './connectors/http.js'
-export const Conn = createTermNamespace(
-  'https://w3id.org/conn#',
-  'FileReaderChannel',
-  'FileWriterChannel',
-  'HttpReaderChannel',
-  'HttpWriterChannel',
-  'KafkaReaderChannel',
-  'KafkaWriterChannel',
-  'WsReaderChannel',
-  'WsWriterChannel',
-  'WriterChannel',
-  'ReaderChannel',
-)
-export const JsOntology = createTermNamespace(
-  'https://w3id.org/conn/js#',
-  'JsProcess',
-  'JsChannel',
-  'JsReaderChannel',
-  'JsWriterChannel',
-)
-export class ChannelFactory {
-  inits = []
-  jsChannelsNamedNodes = {}
-  jsChannelsBlankNodes = {}
-  createReader(config) {
-    LOG.channel('Creating reader %s: a %s', config.id.value, config.ty.value)
-    if (config.ty.equals(Conn.FileReaderChannel)) {
-      const { reader, init } = startFileStreamReader(config.config)
-      this.inits.push(init)
-      return reader
-    }
-    if (config.ty.equals(Conn.WsReaderChannel)) {
-      const { reader, init } = startWsStreamReader(config.config)
-      this.inits.push(init)
-      return reader
-    }
-    if (config.ty.equals(Conn.KafkaReaderChannel)) {
-      const { reader, init } = startKafkaStreamReader(config.config)
-      this.inits.push(init)
-      return reader
-    }
-    if (config.ty.equals(Conn.HttpReaderChannel)) {
-      const { reader, init } = startHttpStreamReader(config.config)
-      this.inits.push(init)
-      return reader
-    }
-    if (config.ty.equals(JsOntology.JsReaderChannel)) {
-      const c = config.config
-      if (c.channel) {
-        const id = c.channel.id.value
-        if (c.channel.id.termType === 'NamedNode') {
-          if (!this.jsChannelsNamedNodes[id]) {
-            this.jsChannelsNamedNodes[id] = new SimpleStream()
-          }
-          return this.jsChannelsNamedNodes[id]
-        }
-        if (c.channel.id.termType === 'BlankNode') {
-          if (!this.jsChannelsBlankNodes[id]) {
-            this.jsChannelsBlankNodes[id] = new SimpleStream()
-          }
-          return this.jsChannelsBlankNodes[id]
-        }
-        throw 'Should have found a thing'
-      }
-    }
-    throw 'Unknown reader channel ' + config.ty.value
-  }
-  createWriter(config) {
-    LOG.channel('Creating writer %s: a %s', config.id.value, config.ty.value)
-    if (config.ty.equals(Conn.FileWriterChannel)) {
-      const { writer, init } = startFileStreamWriter(config.config)
-      this.inits.push(init)
-      return writer
-    }
-    if (config.ty.equals(Conn.WsWriterChannel)) {
-      const { writer, init } = startWsStreamWriter(config.config)
-      this.inits.push(init)
-      return writer
-    }
-    if (config.ty.equals(Conn.KafkaWriterChannel)) {
-      const { writer, init } = startKafkaStreamWriter(config.config)
-      this.inits.push(init)
-      return writer
-    }
-    if (config.ty.equals(Conn.HttpWriterChannel)) {
-      const { writer, init } = startHttpStreamWriter(config.config)
-      this.inits.push(init)
-      return writer
-    }
-    if (config.ty.equals(JsOntology.JsWriterChannel)) {
-      const c = config.config
-      if (c.channel) {
-        const id = c.channel.id.value
-        if (c.channel.id.termType === 'NamedNode') {
-          if (!this.jsChannelsNamedNodes[id]) {
-            this.jsChannelsNamedNodes[id] = new SimpleStream()
-          }
-          return this.jsChannelsNamedNodes[id]
-        }
-        if (c.channel.id.termType === 'BlankNode') {
-          if (!this.jsChannelsBlankNodes[id]) {
-            this.jsChannelsBlankNodes[id] = new SimpleStream()
-          }
-          return this.jsChannelsBlankNodes[id]
-        }
-        throw 'Should have found a thing'
-      }
-    }
-    throw 'Unknown writer channel ' + config.ty.value
-  }
-  async init() {
-    await Promise.all(this.inits.map((x) => x()))
-  }
-}
-export class SimpleStream {
-  dataHandlers = []
-  endHandlers = []
-  disconnect
-  lastElement
-  ended = false
-  constructor(onDisconnect) {
-    this.disconnect = onDisconnect || (async () => {})
-  }
-  data(listener) {
-    this.dataHandlers.push(listener)
-    return this
-  }
-  async push(data) {
-    if (this.ended) {
-      throw new Error('Trying to push to a stream that has ended!')
-    }
-    this.lastElement = data
-    await Promise.all(this.dataHandlers.map((handler) => handler(data)))
-  }
-  async end() {
-    await this.disconnect()
-    await Promise.all(this.endHandlers.map((handler) => handler()))
-    this.ended = true
-  }
-  on(event, listener) {
-    if (event === 'data') {
-      this.dataHandlers.push(listener)
-    }
-    if (event === 'end') {
-      this.endHandlers.push(listener)
-    }
-    return this
-  }
-}