@rdfc/js-runner 2.0.0 → 3.0.1

package/src/jsonld.ts

@@ -1,220 +0,0 @@
-import { OrchestratorMessage, RunnerClient } from '@rdfc/proto'
-import { createNamespace, createUriAndTermNamespace } from '@treecg/types'
-import { ReaderInstance } from './reader'
-import { WriterInstance } from './writer'
-import { Logger } from 'winston'
-import { JsonLdParser } from 'jsonld-streaming-parser'
-import { pred, ShaclPath } from 'rdf-lens'
-import { Quad, Term } from '@rdfjs/types'
-import { NamedNode } from 'n3'
-
-export type RunnerItems = {
-  readers: { [uri: string]: ReaderInstance[] }
-  writers: { [uri: string]: WriterInstance[] }
-  client: RunnerClient
-  write: Writable
-}
-
-const RDFL = createUriAndTermNamespace(
-  'https://w3id.org/rdf-lens/ontology#',
-  'CBD',
-  'Path',
-  'PathLens',
-  'Context',
-  'TypedExtract',
-  'EnvVariable',
-  'envKey',
-  'envDefault',
-  'datatype',
-)
-
-type Writable = (msg: OrchestratorMessage) => Promise<unknown>
-const RDFC = createNamespace(
-  'https://w3id.org/rdf-connect#',
-  (x) => x,
-  'Reader',
-  'Writer',
-)
-
-function as_string_array(obj: unknown): string[] {
-  const out = Array.isArray(obj) ? obj : [obj]
-  return out.filter((x) => typeof x === 'string')
-}
-
-function cbdToQuads(value: unknown) {
-  const quads: Quad[] = []
-  const parser = new JsonLdParser()
-  const promise = new Promise<{ value: Quad[] }>((res, rej) =>
-    parser
-      .on('end', () => res({ value: quads }))
-      .on('error', (e) => rej(e))
-      .on('data', (q) => {
-        quads.push(q)
-      }),
-  )
-
-  if (value instanceof Object && '@type' in value) {
-    delete value['@type']
-  }
-  parser.write(JSON.stringify(value))
-  parser.end()
-  return promise
-}
-
-async function cbdToPath(value: unknown): Promise<{ value: unknown }> {
-  const qs = (
-    await cbdToQuads({
-      '@id': 'http://example.com/ns#me1234',
-      'http://example.com/ns#innerPred': value,
-    })
-  ).value
-
-  const path = pred(new NamedNode('http://example.com/ns#innerPred'))
-    .one()
-    .then(ShaclPath)
-    .execute({
-      quads: qs,
-      id: new NamedNode('http://example.com/ns#me1234'),
-    })
-  return { value: path }
-}
-
-type Cont = { quads: Quad[]; id: Term }
-async function jsonldToQuads(value: unknown): Promise<Cont> {
-  const qs = (
-    await cbdToQuads({
-      '@id': 'http://example.com/ns#me1234',
-      'http://example.com/ns#innerPred': value,
-    })
-  ).value
-
-  const idx = qs.findIndex(
-    (x) =>
-      x.predicate.equals(new NamedNode('http://example.com/ns#innerPred')) &&
-      x.subject.equals(new NamedNode('http://example.com/ns#me1234')),
-  )
-  if (idx < 0) throw 'This cannot happen'
-  const id = qs[idx].object
-
-  qs.splice(idx, 1)
-
-  return {
-    quads: qs,
-    id,
-  }
-}
-
-/* eslint-disable  @typescript-eslint/no-explicit-any */
-function revive(
-  _key: string,
-  value: any,
-  logger: Logger,
-  promises: Promise<unknown>[],
-  runnerItems: RunnerItems,
-): any {
-  if (typeof value === 'object') {
-    const types = as_string_array(value['@type'] || [])
-    const ids = as_string_array(value['@id'] || [])[0] || ''
-
-    if (types.includes(RDFC.Reader)) {
-      if (runnerItems.readers[ids] === undefined) {
-        runnerItems.readers[ids] = []
-      }
-      const reader = new ReaderInstance(ids, runnerItems.client, logger)
-      runnerItems.readers[ids].push(reader)
-      return reader
-    }
-
-    if (types.includes(RDFC.Writer)) {
-      if (runnerItems.writers[ids] === undefined) {
-        runnerItems.writers[ids] = []
-      }
-      const writer = new WriterInstance(
-        ids,
-        runnerItems.client,
-        runnerItems.write,
-        logger,
-      )
-      runnerItems.writers[ids].push(writer)
-      return writer
-    }
-
-    if (types.includes(RDFL.CBD)) {
-      return cbdToQuads(value)
-    }
-
-    if (types.includes(RDFL.Path)) {
-      return cbdToPath(value)
-    }
-  }
-
-  Object.entries(value).forEach(([k, v]) => {
-    if (v instanceof Promise) {
-      logger.info('Found promise')
-      promises.push(
-        v.then((x) => {
-          logger.info('Setting field ' + k)
-          value[k] = x.value
-        }),
-      )
-    }
-  })
-
-  return value
-}
-
-type Arg = {
-  [id: string]: any
-}
-function expandArgs(args: Arg, cont: Cont, logger: Logger) {
-  const context = args['@context']
-  if (!context) return
-
-  for (const key of Object.keys(args)) {
-    if (key == '@context') continue
-    const ctxObj = context[key]
-    if (!ctxObj) {
-      logger.debug("Didn't find ctx things for " + key)
-      continue
-    }
-
-    logger.debug(
-      'Did find ctx things for ' +
-        key +
-        ' ' +
-        JSON.stringify(ctxObj) +
-        ': ' +
-        JSON.stringify(
-          Object.entries(args[key]).map(([k, v]) => {
-            let x = 'object'
-            try {
-              x = JSON.stringify(v)
-              // eslint-disable-next-line @typescript-eslint/no-unused-vars
-            } catch (_e: unknown) {
-              // default is set
-            }
-
-            return k + ': ' + x
-          }),
-        ),
-    )
-
-    expandArgs(args[key], cont, logger)
-  }
-}
-
-export async function parse_jsonld(
-  args: string,
-  logger: Logger,
-  items: RunnerItems,
-): Promise<unknown> {
-  const promises: Promise<unknown>[] = []
-  logger.info(args)
-  const cont = await jsonldToQuads(JSON.parse(args))
-  const out = JSON.parse(args, (k, v) => revive(k, v, logger, promises, items))
-
-  await Promise.all(promises)
-
-  expandArgs(out, cont, logger)
-  return out
-}

package/src/testUtils.ts

@@ -1,196 +0,0 @@
-import { DataChunk, RunnerClient } from '@rdfc/proto'
-import * as grpc from '@grpc/grpc-js'
-import { ClientReadableStream } from '@grpc/grpc-js'
-import { ClientReadableStreamImpl, OrchestratorMessage } from './reexports'
-import { extractShapes } from 'rdf-lens'
-import { NamedNode, Parser, Writer } from 'n3'
-import { readFile } from 'fs/promises'
-import winston, { createLogger } from 'winston'
-import { WriterInstance } from './writer'
-import { ReaderInstance } from './reader'
-import { Processor } from './processor'
-import { FullProc, Runner } from './runner'
-import { Quad, Term } from '@rdfjs/types'
-import { createTermNamespace } from '@treecg/types'
-import { expect } from 'vitest'
-
-export async function getProcessorShape(
-  baseIRI = process.cwd() + '/node_modules/@rdfc/js-runner/index.ttl',
-) {
-  const configFile = await readFile(baseIRI, { encoding: 'utf8' })
-  const configQuads = new Parser().parse(configFile)
-  const shapes = extractShapes(configQuads)
-
-  return shapes
-}
-
-export class TestClient extends RunnerClient {
-  next: (stream: ClientReadableStream<DataChunk>) => unknown
-
-  constructor() {
-    super('localhost:5400', grpc.credentials.createInsecure())
-  }
-
-  nextStream(): Promise<ClientReadableStream<DataChunk>> {
-    return new Promise((res) => (this.next = res))
-  }
-
-  receiveStreamMessage(): ClientReadableStream<DataChunk> {
-    const stream = new ClientReadableStreamImpl<DataChunk>((data: Buffer) => {
-      return { data }
-    })
-    this.next(stream)
-    return stream
-  }
-}
-
-export async function one<T>(iter: AsyncIterable<T>): Promise<T | undefined> {
-  for await (const item of iter) {
-    return item
-  }
-}
-
-export const client = new TestClient()
-export const uri = 'someUri'
-export const logger = createLogger({
-  transports: new winston.transports.Console({
-    level: process.env['DEBUG'] || 'info',
-  }),
-})
-
-export function createWriter(iri = uri): [WriterInstance, ReaderInstance] {
-  const reader = createReader(iri)
-  const writeStream = new WriterInstance(
-    iri,
-    client,
-    async (msg) => {
-      if (msg.msg) {
-        reader.handleMsg(msg.msg)
-      }
-      if (msg.close) {
-        reader.close()
-      }
-    },
-    logger,
-  )
-  return [writeStream, reader]
-}
-
-export function createReader(iri = uri): ReaderInstance {
-  const reader = new ReaderInstance(iri, client, logger)
-  return reader
-}
-
-const shapeQuads = `
-@prefix rdfc: <https://w3id.org/rdf-connect#>.
-@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
-@prefix sh: <http://www.w3.org/ns/shacl#>.
-[ ] a sh:NodeShape;
-  sh:targetClass <JsProcessorShape>;
-  sh:property [
-    sh:path rdfc:entrypoint;
-    sh:name "location";
-    sh:minCount 1;
-    sh:maxCount 1;
-    sh:datatype xsd:string;
-  ], [
-    sh:path rdfc:file;
-    sh:name "file";
-    sh:minCount 1;
-    sh:maxCount 1;
-    sh:datatype xsd:string;
-  ], [
-    sh:path rdfc:class;
-    sh:name "clazz";
-    sh:maxCount 1;
-    sh:datatype xsd:string;
-  ].
-`
-const OWL = createTermNamespace('http://www.w3.org/2002/07/owl#', 'imports')
-const processorShapes = extractShapes(new Parser().parse(shapeQuads))
-const base = 'https://w3id.org/rdf-connect#'
-
-export async function importFile(file: string): Promise<Quad[]> {
-  const done = new Set<string>()
-  const todo = [new URL('file://' + file)]
-  const quads: Quad[] = []
-
-  let item = todo.pop()
-  while (item !== undefined) {
-    if (done.has(item.toString())) {
-      item = todo.pop()
-      continue
-    }
-    done.add(item.toString())
-    if (item.protocol !== 'file:') {
-      throw 'No supported protocol ' + item.protocol
-    }
-
-    const txt = await readFile(item.pathname, { encoding: 'utf8' })
-    const extras = new Parser({ baseIRI: item.toString() }).parse(txt)
-
-    for (const o of extras
-      .filter(
-        (x) =>
-          x.subject.value === item?.toString() &&
-          x.predicate.equals(OWL.imports),
-      )
-      .map((x) => x.object.value)) {
-      todo.push(new URL(o))
-    }
-    quads.push(...extras)
-
-    item = todo.pop()
-  }
-
-  return quads
-}
-
-export async function getProc<T extends Processor<unknown>>(
-  config: string,
-  ty: string,
-  configLocation: string,
-  uri = 'http://example.com/ns#processor',
-): Promise<FullProc<T>> {
-  const configQuads = await importFile(configLocation)
-  const procConfig = processorShapes.lenses['JsProcessorShape'].execute({
-    id: new NamedNode(base + ty),
-    quads: configQuads,
-  })
-
-  const msgs: OrchestratorMessage[] = []
-  const write = async (x: OrchestratorMessage) => {
-    msgs.push(x)
-  }
-  const runner = new Runner(
-    new TestClient(),
-    write,
-    'http://example.com/ns#',
-    logger,
-  )
-  configQuads.push(...new Parser().parse(config))
-  await runner.handleOrchMessage({
-    pipeline: new Writer().quadsToString(configQuads),
-  })
-
-  const proc = await runner.addProcessor<T>({
-    config: JSON.stringify(procConfig),
-    arguments: '',
-    uri,
-  })
-
-  return proc
-}
-
-export async function checkProcDefinition(file: string, n: string) {
-  const quads = await importFile(file)
-  const procConfig = <{ file: Term; location: string; clazz: string }>(
-    processorShapes.lenses['JsProcessorShape'].execute({
-      id: new NamedNode(base + n),
-      quads: quads,
-    })
-  )
-  expect(procConfig.file, n + ' has file').toBeDefined()
-  expect(procConfig.location, n + ' has location').toBeDefined()
-  expect(procConfig.clazz, n + ' has clazz').toBeDefined()
-}