@rdfc/js-runner 2.0.0-alpha.7 → 2.0.0-alpha.9

package/dist/index.cjs

@@ -0,0 +1,732 @@
+'use strict'
+
+var n3 = require('n3')
+var commandLineArgs = require('command-line-args')
+var commandLineUsage = require('command-line-usage')
+var fs = require('fs')
+var http = require('http')
+var https = require('https')
+var types = require('@treecg/types')
+var debug = require('debug')
+var path = require('path')
+var promises = require('fs/promises')
+var node_fs = require('node:fs')
+var ws = require('ws')
+var kafkajs = require('kafkajs')
+var rdfLens = require('rdf-lens')
+
+function _interopNamespaceDefault(e) {
+  var n = Object.create(null)
+  if (e) {
+    Object.keys(e).forEach(function (k) {
+      if (k !== 'default') {
+        var d = Object.getOwnPropertyDescriptor(e, k)
+        Object.defineProperty(
+          n,
+          k,
+          d.get
+            ? d
+            : {
+                enumerable: true,
+                get: function () {
+                  return e[k]
+                },
+              },
+        )
+      }
+    })
+  }
+  n.default = e
+  return Object.freeze(n)
+}
+
+var http__namespace = /*#__PURE__*/ _interopNamespaceDefault(http)
+
+const optionDefinitions = [
+  {
+    name: 'input',
+    type: String,
+    defaultOption: true,
+    summary: 'Specify what input file to start up',
+  },
+  {
+    name: 'help',
+    alias: 'h',
+    type: Boolean,
+    description: 'Display this help message',
+  },
+]
+const sections = [
+  {
+    header: 'Js-runner',
+    content:
+      'JS-runner is part of the {italic connector architecture}. Starting from an input file start up all JsProcessors that are defined. Please do not use blank nodes, skolemize your data somewhere else!',
+  },
+  {
+    header: 'Synopsis',
+    content: '$ js-runner <input>',
+  },
+  {
+    header: 'Command List',
+    content: [
+      { name: 'input', summary: 'Specify what input file to start up' },
+    ],
+  },
+  {
+    optionList: [optionDefinitions[1]],
+  },
+]
+function validArgs(args) {
+  if (!args.input) return false
+  return true
+}
+function printUsage() {
+  const usage = commandLineUsage(sections)
+  console.log(usage)
+  process.exit(0)
+}
+function getArgs() {
+  let args
+  try {
+    args = commandLineArgs(optionDefinitions)
+  } catch (e) {
+    console.error(e)
+    printUsage()
+  }
+  if (args.help || !validArgs(args)) {
+    printUsage()
+  }
+  return args
+}
+
+const LOG = (function () {
+  const main = debug('js-runner')
+  const channel = main.extend('channel')
+  const util = main.extend('util')
+  return { main, channel, util }
+})()
+function toArray(stream) {
+  const output = []
+  return new Promise((res, rej) => {
+    stream.on('data', (x) => output.push(x))
+    stream.on('end', () => res(output))
+    stream.on('close', () => res(output))
+    stream.on('error', rej)
+  })
+}
+const OWL = types.createUriAndTermNamespace(
+  'http://www.w3.org/2002/07/owl#',
+  'imports',
+)
+types.createUriAndTermNamespace(
+  'https://w3id.org/conn#',
+  'install',
+  'build',
+  'GitInstall',
+  'LocalInstall',
+  'url',
+  'procFile',
+  'path',
+  'EnvVariable',
+  'envKey',
+  'envDefault',
+)
+const { namedNode, literal } = n3.DataFactory
+async function get_readstream(location) {
+  if (location.startsWith('https')) {
+    return new Promise((res) => {
+      https.get(location, res)
+    })
+  } else if (location.startsWith('http')) {
+    return new Promise((res) => {
+      http.get(location, res)
+    })
+  } else {
+    return fs.createReadStream(location)
+  }
+}
+async function load_quads(location, baseIRI) {
+  try {
+    LOG.util('Loading quads %s', location)
+    const parser = new n3.StreamParser({ baseIRI: baseIRI || location })
+    const rdfStream = await get_readstream(location)
+    rdfStream.pipe(parser)
+    const quads = await toArray(parser)
+    return quads
+  } catch (ex) {
+    console.error('Failed to load_quads', location, baseIRI)
+    console.error(ex)
+    return []
+  }
+}
+function load_memory_quads(value, baseIRI) {
+  const parser = new n3.Parser({ baseIRI })
+  return parser.parse(value)
+}
+const loaded = new Set()
+async function load_store(location, store, recursive = true) {
+  if (loaded.has(location)) return
+  loaded.add(location)
+  const quads =
+    location.type === 'remote'
+      ? await load_quads(location.location)
+      : load_memory_quads(location.value, location.baseIRI)
+  store.addQuads(quads)
+  if (recursive) {
+    const loc =
+      location.type === 'remote' ? location.location : location.baseIRI
+    const other_imports = store.getObjects(
+      namedNode(loc),
+      OWL.terms.imports,
+      null,
+    )
+    for (const other of other_imports) {
+      await load_store({ location: other.value, type: 'remote' }, store, true)
+    }
+  }
+}
+
+async function getFileSize(path) {
+  return (await promises.stat(path)).size
+}
+function readPart(path, start, end, encoding) {
+  return new Promise((res) => {
+    const stream = fs.createReadStream(path, { encoding, start, end })
+    let buffer = ''
+    stream.on('data', (chunk) => {
+      buffer += chunk
+    })
+    stream.on('close', () => res(buffer))
+  })
+}
+function debounce(func, timeout = 100) {
+  let timer
+  return (...args) => {
+    clearTimeout(timer)
+    timer = setTimeout(() => {
+      func(...args)
+    }, timeout)
+  }
+}
+const startFileStreamReader = (config) => {
+  const path$1 = path.isAbsolute(config.path)
+    ? config.path
+    : `${process.cwd()}/${config.path}`
+  fs.openSync(path$1, 'a+')
+  const encoding = config.encoding || 'utf-8'
+  const reader = new SimpleStream()
+  const init = async () => {
+    let currentPos = await getFileSize(path$1)
+    const watcher = node_fs.watch(path$1, { encoding: 'utf-8' })
+    watcher.on(
+      'change',
+      debounce(async () => {
+        try {
+          let content
+          if (config.onReplace) {
+            content = await promises.readFile(path$1, { encoding })
+          } else {
+            const newSize = await getFileSize(path$1)
+            if (newSize <= currentPos) {
+              currentPos = newSize
+              return
+            }
+            content = await readPart(path$1, currentPos, newSize, encoding)
+            currentPos = newSize
+          }
+          await reader.push(content)
+        } catch (error) {
+          if (error.code === 'ENOENT') {
+            return
+          }
+          throw error
+        }
+      }),
+    )
+    if (config.onReplace && config.readFirstContent) {
+      const content = await promises.readFile(path$1, { encoding })
+      await reader.push(content)
+    }
+  }
+  return { reader, init }
+}
+const startFileStreamWriter = (config) => {
+  const path$1 = path.isAbsolute(config.path)
+    ? config.path
+    : `${process.cwd()}/${config.path}`
+  const encoding = config.encoding || 'utf-8'
+  const writer = new SimpleStream()
+  const init = async () => {
+    if (!config.onReplace) {
+      await promises.writeFile(path$1, '', { encoding })
+    }
+  }
+  writer.push = async (item) => {
+    if (config.onReplace) {
+      await promises.writeFile(path$1, item, { encoding })
+    } else {
+      await promises.appendFile(path$1, item, { encoding })
+    }
+  }
+  return { writer, init }
+}
+
+function _connectWs(url, res) {
+  const ws$1 = new ws.WebSocket(url, {})
+  ws$1.on('error', () => {
+    setTimeout(() => _connectWs(url, res), 300)
+  })
+  ws$1.on('ping', () => ws$1.pong())
+  ws$1.on('open', () => {
+    res(ws$1)
+  })
+}
+function connectWs(url) {
+  return new Promise((res) => _connectWs(url, res))
+}
+const startWsStreamReader = (config) => {
+  const server = new ws.WebSocketServer(config)
+  server.on('error', (error) => {
+    console.error('Ws server error:')
+    console.error(error)
+  })
+  const connections = []
+  const interval = setInterval(() => {
+    connections.forEach((instance, i) => {
+      if (!instance) {
+        return
+      }
+      if (!instance.alive) {
+        instance.socket.terminate()
+        delete connections[i]
+        return
+      }
+      instance.socket.ping()
+      instance.alive = false
+    })
+  }, 30_000)
+  const reader = new SimpleStream(
+    () =>
+      new Promise((res) => {
+        clearInterval(interval)
+        server.close(() => res())
+      }),
+  )
+  server.on('connection', (ws) => {
+    const instance = { socket: ws, alive: true }
+    connections.push(instance)
+    ws.on('message', async (msg) => {
+      reader.push(msg.toString()).catch((error) => {
+        throw error
+      })
+    })
+    ws.on('pong', () => {
+      instance.alive = true
+    })
+  })
+  return { reader, init: async () => {} }
+}
+const startWsStreamWriter = (config) => {
+  let ws
+  const init = async () => {
+    ws = await connectWs(config.url)
+    ws.on('open', () => console.log('open'))
+  }
+  const writer = new SimpleStream(async () => {
+    ws.close()
+  })
+  writer.push = async (item) => {
+    ws.send(item)
+  }
+  return { writer, init }
+}
+
+const startKafkaStreamReader = (config) => {
+  const brokerConfig = {}
+  if (typeof config.broker === 'string' || config.broker instanceof String) {
+    Object.assign(
+      brokerConfig,
+      JSON.parse(node_fs.readFileSync(config.broker, 'utf-8')),
+    )
+  } else {
+    Object.assign(brokerConfig, config.broker)
+  }
+  if (brokerConfig && brokerConfig.hosts) {
+    brokerConfig.brokers = brokerConfig.hosts
+  }
+  const kafka = new kafkajs.Kafka(brokerConfig)
+  const consumer = kafka.consumer(config.consumer)
+  const stream = new SimpleStream(async () => {
+    await consumer.disconnect()
+    await consumer.stop()
+  })
+  const init = async () => {
+    await consumer.connect()
+    await consumer.subscribe({
+      topic: config.topic.name,
+      fromBeginning: config.topic.fromBeginning,
+    })
+    consumer
+      .run({
+        async eachMessage({ topic, message }) {
+          if (topic === config.topic.name) {
+            const element = message.value?.toString() ?? ''
+            stream.push(element).catch((error) => {
+              throw error
+            })
+          }
+        },
+      })
+      .catch((error) => {
+        throw error
+      })
+  }
+  return { reader: stream, init }
+}
+const startKafkaStreamWriter = (config) => {
+  const topic = config.topic.name
+  const brokerConfig = {}
+  if (typeof config.broker === 'string' || config.broker instanceof String) {
+    Object.assign(
+      brokerConfig,
+      JSON.parse(node_fs.readFileSync(config.broker, 'utf-8')),
+    )
+  } else {
+    Object.assign(brokerConfig, config.broker)
+  }
+  if (brokerConfig && brokerConfig.hosts) {
+    brokerConfig.brokers = brokerConfig.hosts
+  }
+  const kafka = new kafkajs.Kafka(brokerConfig)
+  const producer = kafka.producer(config.producer)
+  const writer = new SimpleStream(async () => {
+    await producer.disconnect()
+  })
+  const init = () => producer.connect()
+  writer.push = async (item) => {
+    await producer.send({ topic, messages: [{ value: item }] })
+  }
+  return { writer, init }
+}
+
+function streamToString(stream, binary) {
+  const datas = []
+  return new Promise((res) => {
+    stream.on('data', (data) => {
+      datas.push(data)
+    })
+    stream.on('end', () => {
+      const streamData = Buffer.concat(datas)
+      res(binary ? streamData : streamData.toString())
+    })
+  })
+}
+const startHttpStreamReader = (config) => {
+  let server = undefined
+  const stream = new SimpleStream(
+    () =>
+      new Promise((res) => {
+        if (server !== undefined) {
+          server.close(() => {
+            res()
+          })
+        } else {
+          res()
+        }
+      }),
+  )
+  const requestListener = async function (req, res) {
+    try {
+      const content = await streamToString(req, config.binary)
+      const promise = stream.push(content).catch((error) => {
+        throw error
+      })
+      if (config.waitHandled) {
+        await promise
+      }
+    } catch (error) {
+      console.error('Failed', error)
+    }
+    res.writeHead(config.responseCode || 200)
+    res.end('OK')
+  }
+  server = http.createServer(requestListener)
+  const init = () => {
+    return new Promise((res) => {
+      const cb = () => res(undefined)
+      if (server) {
+        server.listen(config.port, config.endpoint, cb)
+      } else {
+        cb()
+      }
+    })
+  }
+  return { reader: stream, init }
+}
+const startHttpStreamWriter = (config) => {
+  const requestConfig = new URL(config.endpoint)
+  const writer = new SimpleStream()
+  writer.push = async (item) => {
+    await new Promise((resolve) => {
+      const options = {
+        hostname: requestConfig.hostname,
+        path: requestConfig.path,
+        method: config.method,
+        port: requestConfig.port,
+      }
+      const cb = (response) => {
+        response.on('data', () => {})
+        response.on('end', () => {
+          resolve(null)
+        })
+      }
+      const req = http__namespace.request(options, cb)
+      req.write(item, () => {
+        req.end()
+      })
+    })
+  }
+  return { writer, init: async () => {} }
+}
+
+const Conn = types.createTermNamespace(
+  'https://w3id.org/conn#',
+  'FileReaderChannel',
+  'FileWriterChannel',
+  'HttpReaderChannel',
+  'HttpWriterChannel',
+  'KafkaReaderChannel',
+  'KafkaWriterChannel',
+  'WsReaderChannel',
+  'WsWriterChannel',
+  'WriterChannel',
+  'ReaderChannel',
+)
+const JsOntology = types.createTermNamespace(
+  'https://w3id.org/conn/js#',
+  'JsProcess',
+  'JsChannel',
+  'JsReaderChannel',
+  'JsWriterChannel',
+)
+class ChannelFactory {
+  inits = []
+  jsChannelsNamedNodes = {}
+  jsChannelsBlankNodes = {}
+  createReader(config) {
+    LOG.channel('Creating reader %s: a %s', config.id.value, config.ty.value)
+    if (config.ty.equals(Conn.FileReaderChannel)) {
+      const { reader, init } = startFileStreamReader(config.config)
+      this.inits.push(init)
+      return reader
+    }
+    if (config.ty.equals(Conn.WsReaderChannel)) {
+      const { reader, init } = startWsStreamReader(config.config)
+      this.inits.push(init)
+      return reader
+    }
+    if (config.ty.equals(Conn.KafkaReaderChannel)) {
+      const { reader, init } = startKafkaStreamReader(config.config)
+      this.inits.push(init)
+      return reader
+    }
+    if (config.ty.equals(Conn.HttpReaderChannel)) {
+      const { reader, init } = startHttpStreamReader(config.config)
+      this.inits.push(init)
+      return reader
+    }
+    if (config.ty.equals(JsOntology.JsReaderChannel)) {
+      const c = config.config
+      if (c.channel) {
+        const id = c.channel.id.value
+        if (c.channel.id.termType === 'NamedNode') {
+          if (!this.jsChannelsNamedNodes[id]) {
+            this.jsChannelsNamedNodes[id] = new SimpleStream()
+          }
+          return this.jsChannelsNamedNodes[id]
+        }
+        if (c.channel.id.termType === 'BlankNode') {
+          if (!this.jsChannelsBlankNodes[id]) {
+            this.jsChannelsBlankNodes[id] = new SimpleStream()
+          }
+          return this.jsChannelsBlankNodes[id]
+        }
+        throw 'Should have found a thing'
+      }
+    }
+    throw 'Unknown reader channel ' + config.ty.value
+  }
+  createWriter(config) {
+    LOG.channel('Creating writer %s: a %s', config.id.value, config.ty.value)
+    if (config.ty.equals(Conn.FileWriterChannel)) {
+      const { writer, init } = startFileStreamWriter(config.config)
+      this.inits.push(init)
+      return writer
+    }
+    if (config.ty.equals(Conn.WsWriterChannel)) {
+      const { writer, init } = startWsStreamWriter(config.config)
+      this.inits.push(init)
+      return writer
+    }
+    if (config.ty.equals(Conn.KafkaWriterChannel)) {
+      const { writer, init } = startKafkaStreamWriter(config.config)
+      this.inits.push(init)
+      return writer
+    }
+    if (config.ty.equals(Conn.HttpWriterChannel)) {
+      const { writer, init } = startHttpStreamWriter(config.config)
+      this.inits.push(init)
+      return writer
+    }
+    if (config.ty.equals(JsOntology.JsWriterChannel)) {
+      const c = config.config
+      if (c.channel) {
+        const id = c.channel.id.value
+        if (c.channel.id.termType === 'NamedNode') {
+          if (!this.jsChannelsNamedNodes[id]) {
+            this.jsChannelsNamedNodes[id] = new SimpleStream()
+          }
+          return this.jsChannelsNamedNodes[id]
+        }
+        if (c.channel.id.termType === 'BlankNode') {
+          if (!this.jsChannelsBlankNodes[id]) {
+            this.jsChannelsBlankNodes[id] = new SimpleStream()
+          }
+          return this.jsChannelsBlankNodes[id]
+        }
+        throw 'Should have found a thing'
+      }
+    }
+    throw 'Unknown writer channel ' + config.ty.value
+  }
+  async init() {
+    await Promise.all(this.inits.map((x) => x()))
+  }
+}
+class SimpleStream {
+  dataHandlers = []
+  endHandlers = []
+  disconnect
+  lastElement
+  ended = false
+  constructor(onDisconnect) {
+    this.disconnect = onDisconnect || (async () => {})
+  }
+  data(listener) {
+    this.dataHandlers.push(listener)
+    return this
+  }
+  async push(data) {
+    if (this.ended) {
+      throw new Error('Trying to push to a stream that has ended!')
+    }
+    this.lastElement = data
+    await Promise.all(this.dataHandlers.map((handler) => handler(data)))
+  }
+  async end() {
+    await this.disconnect()
+    await Promise.all(this.endHandlers.map((handler) => handler()))
+    this.ended = true
+  }
+  on(event, listener) {
+    if (event === 'data') {
+      this.dataHandlers.push(listener)
+    }
+    if (event === 'end') {
+      this.endHandlers.push(listener)
+    }
+    return this
+  }
+}
+
+function safeJoin(a, b) {
+  if (b.startsWith('/')) {
+    return b
+  }
+  return path.join(a, b)
+}
+async function extractProcessors(source, apply) {
+  const store = new n3.Store()
+  await load_store(source, store)
+  const quads = store.getQuads(null, null, null, null)
+  const config = rdfLens.extractShapes(quads, apply)
+  const subjects = quads
+    .filter(
+      (x) =>
+        x.predicate.equals(types.RDF.terms.type) &&
+        x.object.equals(JsOntology.JsProcess),
+    )
+    .map((x) => x.subject)
+  const processorLens = config.lenses[JsOntology.JsProcess.value]
+  const processors = subjects.map((id) => processorLens.execute({ id, quads }))
+  return { processors, quads, shapes: config }
+}
+function extractSteps(proc, quads, config) {
+  const out = []
+  const subjects = quads
+    .filter(
+      (x) =>
+        x.predicate.equals(types.RDF.terms.type) && x.object.equals(proc.ty),
+    )
+    .map((x) => x.subject)
+  const processorLens = config.lenses[proc.ty.value]
+  const fields = proc.mapping.parameters
+  for (const id of subjects) {
+    const obj = processorLens.execute({ id, quads })
+    const functionArgs = new Array(fields.length)
+    for (const field of fields) {
+      functionArgs[field.position] = obj[field.parameter]
+    }
+    out.push(functionArgs)
+  }
+  return out
+}
+async function jsRunner() {
+  const args = getArgs()
+  const cwd = process.cwd()
+  const source = {
+    location: safeJoin(cwd, args.input).replaceAll('\\', '/'),
+    type: 'remote',
+  }
+  const factory = new ChannelFactory()
+  const apply = {}
+  apply[Conn.ReaderChannel.value] = factory.createReader.bind(factory)
+  apply[Conn.WriterChannel.value] = factory.createWriter.bind(factory)
+  const {
+    processors,
+    quads,
+    shapes: config,
+  } = await extractProcessors(source, apply)
+  LOG.main('Found %d processors', processors.length)
+  const starts = []
+  for (const proc of processors) {
+    const argss = extractSteps(proc, quads, config)
+    const jsProgram = await import('file://' + proc.file)
+    process.chdir(proc.location)
+    for (const args of argss) {
+      starts.push(await jsProgram[proc.func](...args))
+    }
+  }
+  await factory.init()
+  for (const s of starts) {
+    if (s && typeof s === 'function') {
+      s()
+    }
+  }
+}
+
+exports.ChannelFactory = ChannelFactory
+exports.Conn = Conn
+exports.JsOntology = JsOntology
+exports.SimpleStream = SimpleStream
+exports.extractProcessors = extractProcessors
+exports.extractSteps = extractSteps
+exports.jsRunner = jsRunner
+exports.startFileStreamReader = startFileStreamReader
+exports.startFileStreamWriter = startFileStreamWriter
+exports.startHttpStreamReader = startHttpStreamReader
+exports.startHttpStreamWriter = startHttpStreamWriter
+exports.startKafkaStreamReader = startKafkaStreamReader
+exports.startKafkaStreamWriter = startKafkaStreamWriter
+exports.startWsStreamReader = startWsStreamReader
+exports.startWsStreamWriter = startWsStreamWriter