@rdfc/js-runner 1.0.0 → 2.0.0-alpha.10

package/package.json

@@ -1,68 +1,66 @@
 {
   "name": "@rdfc/js-runner",
-  "version": "1.0.0",
+  "version": "2.0.0-alpha.10",
+  "main": "lib/index.js",
   "type": "module",
-  "exports": {
-    "import": "./dist/index.js",
-    "require": "./dist/index.cjs"
+  "scripts": {
+    "build": "tsc && tsc-alias",
+    "lint": "eslint . --ext .ts",
+    "format": "prettier --write .",
+    "test": "npm run build && vitest run",
+    "prepare": "husky"
   },
-  "description": "",
-  "main": "./dist/index.js",
-  "files": [
-    "./dist/**/*",
-    "./ontology.ttl",
-    "./processor/**/*",
-    "./channels/**/*"
-  ],
-  "types": "./dist/index.d.ts",
   "bin": {
-    "js-runner": "bin/js-runner.js"
+    "js-runner": "./bin/runner.js"
   },
-  "scripts": {
-    "build": "tsc && tsc-alias",
-    "watch": "tsc -w",
-    "test": "vitest run --coverage --coverage.include src",
-    "prepare": "husky",
-    "prepublishOnly": "npm run build"
+  "husky": {
+    "hooks": {
+      "pre-commit": "echo \"Husky pre-commit\""
+    }
+  },
+  "lint-staged": {
+    "*.ts": [
+      "eslint --fix",
+      "prettier --write"
+    ]
   },
   "keywords": [],
   "author": "",
-  "license": "MIT",
-  "dependencies": {
-    "@rdfjs/types": "^2.0.1",
-    "@treecg/types": "^0.4.6",
-    "command-line-args": "^6.0.1",
-    "command-line-usage": "^7.0.3",
-    "debug": "^4.4.0",
-    "kafkajs": "^2.2.4",
-    "n3": "^1.23.1",
-    "rdf-lens": "^1.3.3",
-    "stream-to-array": "^2.3.0",
-    "ws": "^8.18.0"
-  },
+  "license": "ISC",
+  "description": "",
   "devDependencies": {
-    "@jest/globals": "^29.7.0",
-    "@knighted/duel": "^2.0.0",
-    "@types/command-line-args": "^5.2.3",
-    "@types/command-line-usage": "^5.0.4",
-    "@types/debug": "^4.1.12",
+    "@eslint/js": "^9.21.0",
+    "@rdfjs/types": "^2.0.1",
+    "@types/jest": "^29.5.14",
+    "@types/jsonld": "^1.5.15",
     "@types/n3": "^1.21.1",
-    "@types/node": "^22.10.7",
-    "@types/ws": "^8.5.13",
-    "rollup": "^4.30.1",
-    "@typescript-eslint/eslint-plugin": "^8.20.0",
-    "@typescript-eslint/parser": "^8.20.0",
-    "@vitest/coverage-v8": "^3.0.2",
-    "dotenv": "^16.4.7",
-    "eslint": "^8.56.0",
+    "@types/node": "^22.13.5",
+    "@typescript-eslint/eslint-plugin": "^8.25.0",
+    "@typescript-eslint/parser": "^8.25.0",
+    "@vitest/coverage-v8": "^3.2.4",
+    "eslint": "^9.21.0",
     "eslint-config-prettier": "^10.0.1",
+    "eslint-plugin-prettier": "^5.2.3",
+    "globals": "^16.0.0",
     "husky": "^9.1.7",
-    "lint-staged": "^15.4.1",
-    "prettier": "^3.4.2",
-    "ts-node": "^10.9.2",
+    "jest": "^29.7.0",
+    "lint-staged": "^15.4.3",
+    "prettier": "^3.5.2",
+    "ts-jest": "^29.2.6",
     "tsc-alias": "^1.8.10",
     "typescript": "^5.7.3",
-    "vite-tsconfig-paths": "^5.1.4",
-    "vitest": "^3.0.2"
+    "typescript-eslint": "^8.25.0",
+    "vitest": "^3.2.4"
+  },
+  "dependencies": {
+    "@grpc/grpc-js": "^1.12.6",
+    "@rdfc/proto": "^0.1.2-alpha.1",
+    "@treecg/types": "^0.4.6",
+    "jsonld": "^8.3.3",
+    "jsonld-streaming-parser": "^5.0.0",
+    "n3": "^1.23.1",
+    "rdf-lens": "^1.3.5",
+    "winston": "^3.17.0",
+    "winston-transport": "^4.9.0"
   }
 }

package/src/client.ts

@@ -0,0 +1,52 @@
+import * as grpc from '@grpc/grpc-js'
+import { promisify } from 'util'
+import { RunnerClient, RunnerMessage } from '@rdfc/proto'
+import winston from 'winston'
+import { RpcTransport } from './logger'
+import { Runner } from './runner'
+
+export async function start(addr: string, uri: string) {
+  const client = new RunnerClient(addr, grpc.credentials.createInsecure())
+
+  const logger = winston.createLogger({
+    transports: [
+      new RpcTransport({
+        entities: [uri, 'cli'],
+        stream: client.logStream(() => {}),
+      }),
+    ],
+  })
+
+  const stream = client.connect()
+
+  logger.info('Connected with server ' + addr)
+  const writable = promisify(stream.write.bind(stream))
+  const runner = new Runner(client, writable, uri, logger)
+
+  await writable({ identify: { uri } })
+
+  let processorsEnd!: (v: unknown) => unknown
+  const processorsEnded = new Promise((res) => (processorsEnd = res))
+  ;(async () => {
+    for await (const chunk of stream) {
+      const msg: RunnerMessage = chunk
+      if (msg.proc) {
+        await runner.addProcessor(msg.proc)
+      }
+      if (msg.start) {
+        runner.start().then(processorsEnd)
+      }
+
+      await runner.handleOrchMessage(msg)
+    }
+
+    logger.error('Stream ended')
+  })()
+
+  await processorsEnded
+
+  logger.info('All processors are finished')
+  stream.end()
+  client.close()
+  process.exit(0)
+}

package/src/convertor.ts

@@ -0,0 +1,59 @@
+import { Any } from './reader'
+
+const decoder = new TextDecoder()
+export interface Convertor<T> {
+  from(buffer: Uint8Array): T
+  fromStream(stream: AsyncIterable<Uint8Array>): Promise<T>
+}
+
+export const AnyConvertor: Convertor<Any> = {
+  from: function (buffer: Uint8Array): Any {
+    return {
+      buffer,
+    }
+  },
+  fromStream: async function (inp: AsyncIterable<Uint8Array>): Promise<Any> {
+    const stream = (async function* () {
+      yield* inp
+    })()
+    return { stream }
+  },
+}
+
+export const StringConvertor: Convertor<string> = {
+  from(buffer) {
+    return decoder.decode(buffer)
+  },
+  async fromStream(stream) {
+    const chunks: Uint8Array[] = []
+    for await (const chunk of stream) {
+      chunks.push(chunk)
+    }
+    return decoder.decode(Buffer.concat(chunks))
+  },
+}
+export const StreamConvertor: Convertor<AsyncGenerator<Uint8Array>> = {
+  from(buffer) {
+    return (async function* () {
+      yield buffer
+    })()
+  },
+
+  async fromStream(stream) {
+    return (async function* () {
+      yield* stream
+    })()
+  },
+}
+export const NoConvertor: Convertor<Uint8Array> = {
+  from(buffer) {
+    return buffer
+  },
+  async fromStream(stream) {
+    const chunks: Uint8Array[] = []
+    for await (const chunk of stream) {
+      chunks.push(chunk)
+    }
+    return Buffer.concat(chunks)
+  },
+}

package/src/index.ts

@@ -0,0 +1,8 @@
+export * from './client'
+export * from './writer'
+export * from './reader'
+export * from './processor'
+export * from './runner'
+export * from './logger'
+
+export * as reexports from './reexports'

package/src/jsonld.ts

@@ -0,0 +1,220 @@
+import { OrchestratorMessage, RunnerClient } from '@rdfc/proto'
+import { createNamespace, createUriAndTermNamespace } from '@treecg/types'
+import { ReaderInstance } from './reader'
+import { WriterInstance } from './writer'
+import { Logger } from 'winston'
+import { JsonLdParser } from 'jsonld-streaming-parser'
+import { pred, ShaclPath } from 'rdf-lens'
+import { Quad, Term } from '@rdfjs/types'
+import { NamedNode } from 'n3'
+
+export type RunnerItems = {
+  readers: { [uri: string]: ReaderInstance[] }
+  writers: { [uri: string]: WriterInstance[] }
+  client: RunnerClient
+  write: Writable
+}
+
+const RDFL = createUriAndTermNamespace(
+  'https://w3id.org/rdf-lens/ontology#',
+  'CBD',
+  'Path',
+  'PathLens',
+  'Context',
+  'TypedExtract',
+  'EnvVariable',
+  'envKey',
+  'envDefault',
+  'datatype',
+)
+
+type Writable = (msg: OrchestratorMessage) => Promise<unknown>
+const RDFC = createNamespace(
+  'https://w3id.org/rdf-connect#',
+  (x) => x,
+  'Reader',
+  'Writer',
+)
+
+function as_string_array(obj: unknown): string[] {
+  const out = Array.isArray(obj) ? obj : [obj]
+  return out.filter((x) => typeof x === 'string')
+}
+
+function cbdToQuads(value: unknown) {
+  const quads: Quad[] = []
+  const parser = new JsonLdParser()
+  const promise = new Promise<{ value: Quad[] }>((res, rej) =>
+    parser
+      .on('end', () => res({ value: quads }))
+      .on('error', (e) => rej(e))
+      .on('data', (q) => {
+        quads.push(q)
+      }),
+  )
+
+  if (value instanceof Object && '@type' in value) {
+    delete value['@type']
+  }
+  parser.write(JSON.stringify(value))
+  parser.end()
+  return promise
+}
+
+async function cbdToPath(value: unknown): Promise<{ value: unknown }> {
+  const qs = (
+    await cbdToQuads({
+      '@id': 'http://example.com/ns#me1234',
+      'http://example.com/ns#innerPred': value,
+    })
+  ).value
+
+  const path = pred(new NamedNode('http://example.com/ns#innerPred'))
+    .one()
+    .then(ShaclPath)
+    .execute({
+      quads: qs,
+      id: new NamedNode('http://example.com/ns#me1234'),
+    })
+  return { value: path }
+}
+
+type Cont = { quads: Quad[]; id: Term }
+async function jsonldToQuads(value: unknown): Promise<Cont> {
+  const qs = (
+    await cbdToQuads({
+      '@id': 'http://example.com/ns#me1234',
+      'http://example.com/ns#innerPred': value,
+    })
+  ).value
+
+  const idx = qs.findIndex(
+    (x) =>
+      x.predicate.equals(new NamedNode('http://example.com/ns#innerPred')) &&
+      x.subject.equals(new NamedNode('http://example.com/ns#me1234')),
+  )
+  if (idx < 0) throw 'This cannot happen'
+  const id = qs[idx].object
+
+  qs.splice(idx, 1)
+
+  return {
+    quads: qs,
+    id,
+  }
+}
+
+/* eslint-disable  @typescript-eslint/no-explicit-any */
+function revive(
+  _key: string,
+  value: any,
+  logger: Logger,
+  promises: Promise<unknown>[],
+  runnerItems: RunnerItems,
+): any {
+  if (typeof value === 'object') {
+    const types = as_string_array(value['@type'] || [])
+    const ids = as_string_array(value['@id'] || [])[0] || ''
+
+    if (types.includes(RDFC.Reader)) {
+      if (runnerItems.readers[ids] === undefined) {
+        runnerItems.readers[ids] = []
+      }
+      const reader = new ReaderInstance(ids, runnerItems.client, logger)
+      runnerItems.readers[ids].push(reader)
+      return reader
+    }
+
+    if (types.includes(RDFC.Writer)) {
+      if (runnerItems.writers[ids] === undefined) {
+        runnerItems.writers[ids] = []
+      }
+      const writer = new WriterInstance(
+        ids,
+        runnerItems.client,
+        runnerItems.write,
+        logger,
+      )
+      runnerItems.writers[ids].push(writer)
+      return writer
+    }
+
+    if (types.includes(RDFL.CBD)) {
+      return cbdToQuads(value)
+    }
+
+    if (types.includes(RDFL.Path)) {
+      return cbdToPath(value)
+    }
+  }
+
+  Object.entries(value).forEach(([k, v]) => {
+    if (v instanceof Promise) {
+      logger.info('Found promise')
+      promises.push(
+        v.then((x) => {
+          logger.info('Setting field ' + k)
+          value[k] = x.value
+        }),
+      )
+    }
+  })
+
+  return value
+}
+
+type Arg = {
+  [id: string]: any
+}
+function expandArgs(args: Arg, cont: Cont, logger: Logger) {
+  const context = args['@context']
+  if (!context) return
+
+  for (const key of Object.keys(args)) {
+    if (key == '@context') continue
+    const ctxObj = context[key]
+    if (!ctxObj) {
+      logger.debug("Didn't find ctx things for " + key)
+      continue
+    }
+
+    logger.debug(
+      'Did find ctx things for ' +
+        key +
+        ' ' +
+        JSON.stringify(ctxObj) +
+        ': ' +
+        JSON.stringify(
+          Object.entries(args[key]).map(([k, v]) => {
+            let x = 'object'
+            try {
+              x = JSON.stringify(v)
+              // eslint-disable-next-line @typescript-eslint/no-unused-vars
+            } catch (_e: unknown) {
+              // default is set
+            }
+
+            return k + ': ' + x
+          }),
+        ),
+    )
+
+    expandArgs(args[key], cont, logger)
+  }
+}
+
+export async function parse_jsonld(
+  args: string,
+  logger: Logger,
+  items: RunnerItems,
+): Promise<unknown> {
+  const promises: Promise<unknown>[] = []
+  logger.info(args)
+  const cont = await jsonldToQuads(JSON.parse(args))
+  const out = JSON.parse(args, (k, v) => revive(k, v, logger, promises, items))
+
+  await Promise.all(promises)
+
+  expandArgs(out, cont, logger)
+  return out
+}

package/src/logger.ts

@@ -0,0 +1,64 @@
+import winston, { Logger } from 'winston'
+import Transport from 'winston-transport'
+
+import * as grpc from '@grpc/grpc-js'
+import { LogMessage } from '@rdfc/proto'
+
+export class RpcTransport extends Transport {
+  private readonly stream: grpc.ClientWritableStream<LogMessage>
+  private readonly entities: string[]
+  private readonly aliases: string[]
+
+  constructor(opts: {
+    stream: grpc.ClientWritableStream<LogMessage>
+    entities: string[]
+    aliases?: string[]
+  }) {
+    super({ level: 'debug' })
+
+    this.stream = opts.stream
+    this.entities = opts.entities
+    this.aliases = opts.aliases || []
+  }
+
+  log(info: winston.LogEntry, callback: () => void) {
+    if (!this.stream.closed) {
+      this.stream.write(
+        {
+          msg: info.message,
+          level: info.level,
+          entities: this.entities,
+          aliases: this.aliases,
+        },
+        callback,
+      )
+    } else {
+      console.log('Output stream closed')
+      callback()
+    }
+  }
+
+  withEntity(entity: string): RpcTransport {
+    return new RpcTransport({
+      stream: this.stream,
+      entities: [...this.entities, entity],
+      aliases: this.aliases,
+    })
+  }
+}
+
+export function extendLogger(baseLogger: Logger, newEntity: string): Logger {
+  const newTransports = baseLogger.transports.map((t) => {
+    if (t instanceof RpcTransport) {
+      return t.withEntity(newEntity)
+    }
+    return t
+  })
+
+  return winston.createLogger({
+    level: baseLogger.level,
+    format: baseLogger.format,
+    defaultMeta: baseLogger.defaultMeta,
+    transports: newTransports,
+  })
+}

package/src/processor.ts

@@ -0,0 +1,39 @@
+import { Logger } from 'winston'
+import { Reader } from './reader'
+import { Writer } from './writer'
+
+export type Primitive = string | number | Writer | Reader | ProcessorArgs
+
+export type ProcessorArgs = {
+  [id: string]: Primitive | Primitive[]
+}
+export type BGetter<T> = {
+  [K in keyof T]: T[K]
+}
+
+export abstract class Processor<T> {
+  protected readonly args: T // Store args safely
+  protected readonly logger: Logger
+
+  constructor(args: T, logger: Logger) {
+    Object.assign(this, args)
+    this.args = args
+    this.logger = logger
+  }
+
+  protected get<K extends keyof T>(key: K): T[K] {
+    return this.args[key]
+  }
+
+  // This is the first function that is called (and awaited), when creating a processor
+  // This is the perfect location to start things like database connections
+  abstract init(this: T & this): Promise<void>
+
+  // Function to start reading channels
+  // This function is called for each processor before _produce_ is called
+  abstract transform(this: T & this): Promise<void>
+
+  // Function to start the production of data, starting the pipeline
+  // This function is called after all processors are completely setup
+  abstract produce(this: T & this): Promise<void>
+}

package/src/reader.ts

@@ -0,0 +1,142 @@
+import { ClientReadableStream } from '@grpc/grpc-js'
+import { DataChunk, Message, RunnerClient, StreamMessage } from '@rdfc/proto'
+import winston from 'winston'
+import {
+  AnyConvertor,
+  Convertor,
+  NoConvertor,
+  StreamConvertor,
+  StringConvertor,
+} from './convertor'
+
+export type Any =
+  | {
+      string: string
+    }
+  | {
+      stream: AsyncGenerator<Uint8Array>
+    }
+  | {
+      buffer: Uint8Array
+    }
+
+export interface Reader {
+  readonly uri: string
+  strings(): AsyncIterable<string>
+  streams(): AsyncIterable<AsyncGenerator<Uint8Array>>
+  buffers(): AsyncIterable<Uint8Array>
+  anys(): AsyncIterable<Any>
+}
+
+class MyIter<T> implements AsyncIterable<T> {
+  private convertor: Convertor<T>
+  private queue: (T | undefined)[] = []
+  private resolveNext: ((value: undefined) => void) | null = null
+
+  constructor(convertor: Convertor<T>) {
+    this.convertor = convertor
+  }
+
+  push(buffer: Uint8Array) {
+    const item = this.convertor.from(buffer)
+    this.queue.push(item)
+    if (this.resolveNext) {
+      this.resolveNext(undefined)
+      this.resolveNext = null
+    }
+  }
+
+  close() {
+    this.queue.push(undefined)
+    if (this.resolveNext) {
+      this.resolveNext(undefined)
+      this.resolveNext = null
+    }
+  }
+
+  async pushStream(chunks: ClientReadableStream<DataChunk>) {
+    const stream = (async function* (stream) {
+      for await (const c of stream) {
+        const chunk: DataChunk = c
+        yield chunk.data
+      }
+    })(chunks)
+    const item = await this.convertor.fromStream(stream)
+    this.queue.push(item)
+    if (this.resolveNext) {
+      this.resolveNext(undefined)
+      this.resolveNext = null
+    }
+  }
+
+  async *[Symbol.asyncIterator]() {
+    while (true) {
+      if (this.queue.length > 0) {
+        const item = this.queue.shift()!
+        if (item === undefined) break
+        yield item
+      } else {
+        await new Promise<undefined>((resolve) => (this.resolveNext = resolve))
+      }
+    }
+  }
+}
+
+export class ReaderInstance implements Reader {
+  private client: RunnerClient
+  readonly uri: string
+  private logger: winston.Logger
+
+  private iterators: MyIter<unknown>[] = []
+
+  constructor(uri: string, client: RunnerClient, logger: winston.Logger) {
+    this.uri = uri
+    this.client = client
+    this.logger = logger
+  }
+
+  anys(): AsyncIterable<Any> {
+    const iter = new MyIter(AnyConvertor)
+    this.iterators.push(iter)
+    return iter
+  }
+
+  strings(): AsyncIterable<string> {
+    const iter = new MyIter(StringConvertor)
+    this.iterators.push(iter)
+    return iter
+  }
+
+  buffers(): AsyncIterable<Uint8Array> {
+    const iter = new MyIter(NoConvertor)
+    this.iterators.push(iter)
+    return iter
+  }
+
+  streams(): AsyncIterable<AsyncGenerator<Uint8Array>> {
+    const iter = new MyIter(StreamConvertor)
+    this.iterators.push(iter)
+    return iter
+  }
+
+  handleMsg(msg: Message) {
+    this.logger.debug(`${this.uri} handling message`)
+    for (const iter of this.iterators) {
+      iter.push(msg.data)
+    }
+  }
+
+  close() {
+    for (const iter of this.iterators) {
+      iter.close()
+    }
+  }
+
+  handleStreamingMessage(msg: StreamMessage) {
+    this.logger.debug(`${this.uri} handling streaming message`)
+    const chunks = this.client.receiveStreamMessage(msg.id!)
+    for (const iter of this.iterators) {
+      iter.pushStream(chunks)
+    }
+  }
+}