@rdfc/js-runner 3.0.0 → 3.0.2

package/dist/connectors/kafka.d.ts

@@ -0,0 +1,48 @@
+import type { ProducerConfig } from 'kafkajs'
+import { ReaderConstructor, WriterConstructor } from '../connectors.js'
+export interface SASLOptions {
+  mechanism: 'plain'
+  username: string
+  password: string
+}
+export interface BrokerConfig {
+  hosts: string[]
+  ssl?: boolean
+  sasl?: SASLOptions
+}
+export interface ConsumerConfig {
+  groupId: string
+  metadataMaxAge?: number
+  sessionTimeout?: number
+  rebalanceTimeout?: number
+  heartbeatInterval?: number
+  maxBytesPerPartition?: number
+  minBytes?: number
+  maxBytes?: number
+  maxWaitTimeInMs?: number
+  allowAutoTopicCreation?: boolean
+  maxInFlightRequests?: number
+  readUncommitted?: boolean
+  rackId?: string
+}
+export interface CSTopic {
+  topic: string
+  fromBeginning?: boolean
+}
+export interface KafkaReaderConfig {
+  topic: {
+    name: string
+    fromBeginning?: boolean
+  }
+  consumer: ConsumerConfig
+  broker: string | BrokerConfig
+}
+export declare const startKafkaStreamReader: ReaderConstructor<KafkaReaderConfig>
+export interface KafkaWriterConfig {
+  topic: {
+    name: string
+  }
+  producer: ProducerConfig
+  broker: BrokerConfig | string
+}
+export declare const startKafkaStreamWriter: WriterConstructor<KafkaWriterConfig>

package/dist/connectors/kafka.js

@@ -0,0 +1,68 @@
+import { readFileSync } from 'node:fs'
+import { Kafka } from 'kafkajs'
+import { SimpleStream } from '../connectors.js'
+export const startKafkaStreamReader = (config) => {
+  const brokerConfig = {}
+  if (typeof config.broker === 'string' || config.broker instanceof String) {
+    Object.assign(
+      brokerConfig,
+      JSON.parse(readFileSync(config.broker, 'utf-8')),
+    )
+  } else {
+    Object.assign(brokerConfig, config.broker)
+  }
+  if (brokerConfig && brokerConfig.hosts) {
+    brokerConfig.brokers = brokerConfig.hosts
+  }
+  const kafka = new Kafka(brokerConfig)
+  const consumer = kafka.consumer(config.consumer)
+  const stream = new SimpleStream(async () => {
+    await consumer.disconnect()
+    await consumer.stop()
+  })
+  const init = async () => {
+    await consumer.connect()
+    await consumer.subscribe({
+      topic: config.topic.name,
+      fromBeginning: config.topic.fromBeginning,
+    })
+    consumer
+      .run({
+        async eachMessage({ topic, message }) {
+          if (topic === config.topic.name) {
+            const element = message.value?.toString() ?? ''
+            await stream.push(element)
+          }
+        },
+      })
+      .catch((error) => {
+        throw error
+      })
+  }
+  return { reader: stream, init }
+}
+export const startKafkaStreamWriter = (config) => {
+  const topic = config.topic.name
+  const brokerConfig = {}
+  if (typeof config.broker === 'string' || config.broker instanceof String) {
+    Object.assign(
+      brokerConfig,
+      JSON.parse(readFileSync(config.broker, 'utf-8')),
+    )
+  } else {
+    Object.assign(brokerConfig, config.broker)
+  }
+  if (brokerConfig && brokerConfig.hosts) {
+    brokerConfig.brokers = brokerConfig.hosts
+  }
+  const kafka = new Kafka(brokerConfig)
+  const producer = kafka.producer(config.producer)
+  const writer = new SimpleStream(async () => {
+    await producer.disconnect()
+  })
+  const init = () => producer.connect()
+  writer.push = async (item) => {
+    await producer.send({ topic, messages: [{ value: item }] })
+  }
+  return { writer, init }
+}

package/dist/connectors/ws.d.ts

@@ -0,0 +1,10 @@
+import { ReaderConstructor, WriterConstructor } from '../connectors.js'
+export interface WsWriterConfig {
+  url: string
+}
+export interface WsReaderConfig {
+  host: string
+  port: number
+}
+export declare const startWsStreamReader: ReaderConstructor<WsReaderConfig>
+export declare const startWsStreamWriter: WriterConstructor<WsWriterConfig>

package/dist/connectors/ws.js

@@ -0,0 +1,72 @@
+import { SimpleStream } from '../connectors.js'
+import { WebSocket } from 'ws'
+import { WebSocketServer } from 'ws'
+function _connectWs(url, res) {
+  const ws = new WebSocket(url, {})
+  ws.on('error', () => {
+    setTimeout(() => _connectWs(url, res), 300)
+  })
+  ws.on('ping', () => ws.pong())
+  ws.on('open', () => {
+    res(ws)
+  })
+}
+function connectWs(url) {
+  return new Promise((res) => _connectWs(url, res))
+}
+export const startWsStreamReader = (config) => {
+  const server = new WebSocketServer(config)
+  server.on('error', (error) => {
+    console.error('Ws server error:')
+    console.error(error)
+  })
+  const connections = []
+  const interval = setInterval(() => {
+    connections.forEach((instance, i) => {
+      if (!instance) {
+        return
+      }
+      if (!instance.alive) {
+        instance.socket.terminate()
+        delete connections[i]
+        return
+      }
+      instance.socket.ping()
+      instance.alive = false
+    })
+  }, 30_000)
+  const reader = new SimpleStream(
+    () =>
+      new Promise((res) => {
+        clearInterval(interval)
+        server.close(() => res())
+      }),
+  )
+  server.on('connection', (ws) => {
+    const instance = { socket: ws, alive: true }
+    connections.push(instance)
+    ws.on('message', async (msg) => {
+      reader.push(msg.toString()).catch((error) => {
+        throw error
+      })
+    })
+    ws.on('pong', () => {
+      instance.alive = true
+    })
+  })
+  return { reader, init: async () => {} }
+}
+export const startWsStreamWriter = (config) => {
+  let ws
+  const init = async () => {
+    ws = await connectWs(config.url)
+    ws.on('open', () => console.log('open'))
+  }
+  const writer = new SimpleStream(async () => {
+    ws.close()
+  })
+  writer.push = async (item) => {
+    ws.send(item)
+  }
+  return { writer, init }
+}

package/dist/connectors.d.ts

@@ -0,0 +1,73 @@
+import { NamedNode, Term } from '@rdfjs/types'
+export * from './connectors/file.js'
+export * from './connectors/ws.js'
+export * from './connectors/kafka.js'
+export * from './connectors/http.js'
+export declare const Conn: import('@treecg/types').Namespace<
+  (
+    | 'FileReaderChannel'
+    | 'FileWriterChannel'
+    | 'HttpReaderChannel'
+    | 'HttpWriterChannel'
+    | 'KafkaReaderChannel'
+    | 'KafkaWriterChannel'
+    | 'WsReaderChannel'
+    | 'WsWriterChannel'
+    | 'WriterChannel'
+    | 'ReaderChannel'
+  )[],
+  NamedNode<string>,
+  string
+>
+export interface Config<T> {
+  id: Term
+  ty: NamedNode
+  config: T
+}
+export type ReaderConstructor<C> = (config: C) => {
+  reader: Stream<string | Buffer>
+  init: () => Promise<void>
+}
+export type WriterConstructor<C> = (config: C) => {
+  writer: Writer<string | Buffer>
+  init: () => Promise<void>
+}
+export declare const JsOntology: import('@treecg/types').Namespace<
+  ('JsProcess' | 'JsChannel' | 'JsReaderChannel' | 'JsWriterChannel')[],
+  NamedNode<string>,
+  string
+>
+export declare class ChannelFactory {
+  private inits
+  private jsChannelsNamedNodes
+  private jsChannelsBlankNodes
+  createReader(config: Config<any>): Stream<string | Buffer>
+  createWriter(config: Config<any>): Writer<string | Buffer>
+  init(): Promise<void>
+}
+export interface Writer<T> {
+  push(item: T): Promise<void>
+  end(): Promise<void>
+  on(event: 'end', listener: Handler<void>): this
+}
+export interface Stream<T> {
+  lastElement?: T
+  end(): Promise<void>
+  data(listener: (t: T) => PromiseLike<void> | void): this
+  on(event: 'data', listener: (t: T) => PromiseLike<void> | void): this
+  on(event: 'end', listener: () => PromiseLike<void> | void): this
+}
+export type Handler<T> = (item: T) => Promise<void> | void
+export declare class SimpleStream<T> implements Stream<T> {
+  private readonly dataHandlers
+  private readonly endHandlers
+  readonly disconnect: () => Promise<void>
+  lastElement?: T | undefined
+  private ended
+  constructor(onDisconnect?: () => Promise<void>)
+  data(listener: Handler<T>): this
+  push(data: T): Promise<void>
+  end(): Promise<void>
+  on(event: 'data', listener: Handler<T>): this
+  on(event: 'end', listener: Handler<void>): this
+}

package/dist/connectors.js

@@ -0,0 +1,168 @@
+import { createTermNamespace } from '@treecg/types'
+import {
+  startFileStreamReader,
+  startFileStreamWriter,
+} from './connectors/file.js'
+export * from './connectors/file.js'
+import { startWsStreamReader, startWsStreamWriter } from './connectors/ws.js'
+export * from './connectors/ws.js'
+import {
+  startKafkaStreamReader,
+  startKafkaStreamWriter,
+} from './connectors/kafka.js'
+export * from './connectors/kafka.js'
+import {
+  startHttpStreamReader,
+  startHttpStreamWriter,
+} from './connectors/http.js'
+import { LOG } from './util.js'
+export * from './connectors/http.js'
+export const Conn = createTermNamespace(
+  'https://w3id.org/conn#',
+  'FileReaderChannel',
+  'FileWriterChannel',
+  'HttpReaderChannel',
+  'HttpWriterChannel',
+  'KafkaReaderChannel',
+  'KafkaWriterChannel',
+  'WsReaderChannel',
+  'WsWriterChannel',
+  'WriterChannel',
+  'ReaderChannel',
+)
+export const JsOntology = createTermNamespace(
+  'https://w3id.org/conn/js#',
+  'JsProcess',
+  'JsChannel',
+  'JsReaderChannel',
+  'JsWriterChannel',
+)
+export class ChannelFactory {
+  inits = []
+  jsChannelsNamedNodes = {}
+  jsChannelsBlankNodes = {}
+  createReader(config) {
+    LOG.channel('Creating reader %s: a %s', config.id.value, config.ty.value)
+    if (config.ty.equals(Conn.FileReaderChannel)) {
+      const { reader, init } = startFileStreamReader(config.config)
+      this.inits.push(init)
+      return reader
+    }
+    if (config.ty.equals(Conn.WsReaderChannel)) {
+      const { reader, init } = startWsStreamReader(config.config)
+      this.inits.push(init)
+      return reader
+    }
+    if (config.ty.equals(Conn.KafkaReaderChannel)) {
+      const { reader, init } = startKafkaStreamReader(config.config)
+      this.inits.push(init)
+      return reader
+    }
+    if (config.ty.equals(Conn.HttpReaderChannel)) {
+      const { reader, init } = startHttpStreamReader(config.config)
+      this.inits.push(init)
+      return reader
+    }
+    if (config.ty.equals(JsOntology.JsReaderChannel)) {
+      const c = config.config
+      if (c.channel) {
+        const id = c.channel.id.value
+        if (c.channel.id.termType === 'NamedNode') {
+          if (!this.jsChannelsNamedNodes[id]) {
+            this.jsChannelsNamedNodes[id] = new SimpleStream()
+          }
+          return this.jsChannelsNamedNodes[id]
+        }
+        if (c.channel.id.termType === 'BlankNode') {
+          if (!this.jsChannelsBlankNodes[id]) {
+            this.jsChannelsBlankNodes[id] = new SimpleStream()
+          }
+          return this.jsChannelsBlankNodes[id]
+        }
+        throw 'Should have found a thing'
+      }
+    }
+    throw 'Unknown reader channel ' + config.ty.value
+  }
+  createWriter(config) {
+    LOG.channel('Creating writer %s: a %s', config.id.value, config.ty.value)
+    if (config.ty.equals(Conn.FileWriterChannel)) {
+      const { writer, init } = startFileStreamWriter(config.config)
+      this.inits.push(init)
+      return writer
+    }
+    if (config.ty.equals(Conn.WsWriterChannel)) {
+      const { writer, init } = startWsStreamWriter(config.config)
+      this.inits.push(init)
+      return writer
+    }
+    if (config.ty.equals(Conn.KafkaWriterChannel)) {
+      const { writer, init } = startKafkaStreamWriter(config.config)
+      this.inits.push(init)
+      return writer
+    }
+    if (config.ty.equals(Conn.HttpWriterChannel)) {
+      const { writer, init } = startHttpStreamWriter(config.config)
+      this.inits.push(init)
+      return writer
+    }
+    if (config.ty.equals(JsOntology.JsWriterChannel)) {
+      const c = config.config
+      if (c.channel) {
+        const id = c.channel.id.value
+        if (c.channel.id.termType === 'NamedNode') {
+          if (!this.jsChannelsNamedNodes[id]) {
+            this.jsChannelsNamedNodes[id] = new SimpleStream()
+          }
+          return this.jsChannelsNamedNodes[id]
+        }
+        if (c.channel.id.termType === 'BlankNode') {
+          if (!this.jsChannelsBlankNodes[id]) {
+            this.jsChannelsBlankNodes[id] = new SimpleStream()
+          }
+          return this.jsChannelsBlankNodes[id]
+        }
+        throw 'Should have found a thing'
+      }
+    }
+    throw 'Unknown writer channel ' + config.ty.value
+  }
+  async init() {
+    await Promise.all(this.inits.map((x) => x()))
+  }
+}
+export class SimpleStream {
+  dataHandlers = []
+  endHandlers = []
+  disconnect
+  lastElement
+  ended = false
+  constructor(onDisconnect) {
+    this.disconnect = onDisconnect || (async () => {})
+  }
+  data(listener) {
+    this.dataHandlers.push(listener)
+    return this
+  }
+  async push(data) {
+    if (this.ended) {
+      throw new Error('Trying to push to a stream that has ended!')
+    }
+    this.lastElement = data
+    await Promise.all(this.dataHandlers.map((handler) => handler(data)))
+  }
+  async end() {
+    await this.disconnect()
+    await Promise.all(this.endHandlers.map((handler) => handler()))
+    this.ended = true
+  }
+  on(event, listener) {
+    if (event === 'data') {
+      this.dataHandlers.push(listener)
+    }
+    if (event === 'end') {
+      this.endHandlers.push(listener)
+    }
+    return this
+  }
+}