@rdfc/js-runner 2.0.0-alpha.8 → 2.0.0

package/src/reader.ts

@@ -11,14 +11,14 @@ import {
 
 export type Any =
   | {
-    string: string
-  }
+      string: string
+    }
   | {
-    stream: AsyncGenerator<Uint8Array>
-  }
+      stream: AsyncGenerator<Uint8Array>
+    }
   | {
-    buffer: Uint8Array
-  }
+      buffer: Uint8Array
+    }
 
 export interface Reader {
   readonly uri: string
@@ -55,7 +55,7 @@ class MyIter<T> implements AsyncIterable<T> {
   }
 
   async pushStream(chunks: ClientReadableStream<DataChunk>) {
-    const stream = (async function*(stream) {
+    const stream = (async function* (stream) {
       for await (const c of stream) {
         const chunk: DataChunk = c
         yield chunk.data
@@ -76,9 +76,7 @@ class MyIter<T> implements AsyncIterable<T> {
         if (item === undefined) break
         yield item
       } else {
-        await new Promise<undefined>(
-          (resolve) => (this.resolveNext = resolve),
-        )
+        await new Promise<undefined>((resolve) => (this.resolveNext = resolve))
       }
     }
   }
@@ -123,7 +121,6 @@ export class ReaderInstance implements Reader {
 
   handleMsg(msg: Message) {
     this.logger.debug(`${this.uri} handling message`)
-    console.log(`${this.uri} handling message`)
     for (const iter of this.iterators) {
       iter.push(msg.data)
     }

package/src/runner.ts

@@ -79,7 +79,7 @@ export class Runner {
       transports: [
         new RpcTransport({
           entities: [proc.uri, this.uri],
-          stream: this.client.logStream(() => { }),
+          stream: this.client.logStream(() => {}),
         }),
       ],
     })
@@ -121,7 +121,7 @@ export class Runner {
       await Promise.all(this.processors.map((x) => x.produce()))
       await Promise.all(this.processorTransforms)
     } catch (ex: unknown) {
-      this.logger.error("Start failed: " + JSON.stringify(ex))
+      this.logger.error('Start failed: ' + JSON.stringify(ex))
     }
   }
 
@@ -175,7 +175,6 @@ export class Runner {
 
     if (msg.pipeline) {
       try {
-
         // here
         const quads = new Parser().parse(msg.pipeline)
         this.shapes = extractShapes(
@@ -190,9 +189,8 @@ export class Runner {
           },
         )
         this.quads = quads
-      }
-      catch (ex: unknown) {
-        this.logger.error("Pipeline failed: " + JSON.stringify(ex))
+      } catch (ex: unknown) {
+        this.logger.error('Pipeline failed: ' + JSON.stringify(ex))
       }
     }
   }

package/src/testUtils.ts

@@ -1,13 +1,18 @@
 import { DataChunk, RunnerClient } from '@rdfc/proto'
-import { ClientReadableStream } from '@grpc/grpc-js'
 import * as grpc from '@grpc/grpc-js'
-import { ClientReadableStreamImpl } from './reexports'
+import { ClientReadableStream } from '@grpc/grpc-js'
+import { ClientReadableStreamImpl, OrchestratorMessage } from './reexports'
 import { extractShapes } from 'rdf-lens'
-import { Parser } from 'n3'
+import { NamedNode, Parser, Writer } from 'n3'
 import { readFile } from 'fs/promises'
 import winston, { createLogger } from 'winston'
 import { WriterInstance } from './writer'
 import { ReaderInstance } from './reader'
+import { Processor } from './processor'
+import { FullProc, Runner } from './runner'
+import { Quad, Term } from '@rdfjs/types'
+import { createTermNamespace } from '@treecg/types'
+import { expect } from 'vitest'
 
 export async function getProcessorShape(
   baseIRI = process.cwd() + '/node_modules/@rdfc/js-runner/index.ttl',
@@ -75,3 +80,117 @@ export function createReader(iri = uri): ReaderInstance {
   const reader = new ReaderInstance(iri, client, logger)
   return reader
 }
+
+const shapeQuads = `
+@prefix rdfc: <https://w3id.org/rdf-connect#>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+[ ] a sh:NodeShape;
+  sh:targetClass <JsProcessorShape>;
+  sh:property [
+    sh:path rdfc:entrypoint;
+    sh:name "location";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+  ], [
+    sh:path rdfc:file;
+    sh:name "file";
+    sh:minCount 1;
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+  ], [
+    sh:path rdfc:class;
+    sh:name "clazz";
+    sh:maxCount 1;
+    sh:datatype xsd:string;
+  ].
+`
+const OWL = createTermNamespace('http://www.w3.org/2002/07/owl#', 'imports')
+const processorShapes = extractShapes(new Parser().parse(shapeQuads))
+const base = 'https://w3id.org/rdf-connect#'
+
+export async function importFile(file: string): Promise<Quad[]> {
+  const done = new Set<string>()
+  const todo = [new URL('file://' + file)]
+  const quads: Quad[] = []
+
+  let item = todo.pop()
+  while (item !== undefined) {
+    if (done.has(item.toString())) {
+      item = todo.pop()
+      continue
+    }
+    done.add(item.toString())
+    if (item.protocol !== 'file:') {
+      throw 'No supported protocol ' + item.protocol
+    }
+
+    const txt = await readFile(item.pathname, { encoding: 'utf8' })
+    const extras = new Parser({ baseIRI: item.toString() }).parse(txt)
+
+    for (const o of extras
+      .filter(
+        (x) =>
+          x.subject.value === item?.toString() &&
+          x.predicate.equals(OWL.imports),
+      )
+      .map((x) => x.object.value)) {
+      todo.push(new URL(o))
+    }
+    quads.push(...extras)
+
+    item = todo.pop()
+  }
+
+  return quads
+}
+
+export async function getProc<T extends Processor<unknown>>(
+  config: string,
+  ty: string,
+  configLocation: string,
+  uri = 'http://example.com/ns#processor',
+): Promise<FullProc<T>> {
+  const configQuads = await importFile(configLocation)
+  const procConfig = processorShapes.lenses['JsProcessorShape'].execute({
+    id: new NamedNode(base + ty),
+    quads: configQuads,
+  })
+
+  const msgs: OrchestratorMessage[] = []
+  const write = async (x: OrchestratorMessage) => {
+    msgs.push(x)
+  }
+  const runner = new Runner(
+    new TestClient(),
+    write,
+    'http://example.com/ns#',
+    logger,
+  )
+  configQuads.push(...new Parser().parse(config))
+  await runner.handleOrchMessage({
+    pipeline: new Writer().quadsToString(configQuads),
+  })
+
+  const proc = await runner.addProcessor<T>({
+    config: JSON.stringify(procConfig),
+    arguments: '',
+    uri,
+  })
+
+  return proc
+}
+
+export async function checkProcDefinition(file: string, n: string) {
+  const quads = await importFile(file)
+  const procConfig = <{ file: Term; location: string; clazz: string }>(
+    processorShapes.lenses['JsProcessorShape'].execute({
+      id: new NamedNode(base + n),
+      quads: quads,
+    })
+  )
+  expect(procConfig.file, n + ' has file').toBeDefined()
+  expect(procConfig.location, n + ' has location').toBeDefined()
+  expect(procConfig.clazz, n + ' has clazz').toBeDefined()
+}

package/.editorconfig

@@ -1,9 +0,0 @@
-# top-most EditorConfig file
-root = true
-
-# Unix-style newlines with a newline ending every file
-[*]
-end_of_line = lf
-insert_final_newline = true
-indent_style = space
-indent_size = 2

package/__tests__/channels.test.js

@@ -1,117 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __asyncValues = (this && this.__asyncValues) || function (o) {
-    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
-    var m = o[Symbol.asyncIterator], i;
-    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
-    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
-    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const vitest_1 = require("vitest");
-const reader_1 = require("../src/reader");
-const testUtils_1 = require("../src/testUtils");
-const encoder = new TextEncoder();
-(0, vitest_1.describe)('Reader', () => __awaiter(void 0, void 0, void 0, function* () {
-    (0, vitest_1.test)('handles string', () => __awaiter(void 0, void 0, void 0, function* () {
-        const reader = new reader_1.ReaderInstance(testUtils_1.uri, testUtils_1.client, testUtils_1.logger);
-        const promise = (0, testUtils_1.one)(reader.strings());
-        reader.handleMsg({ data: encoder.encode('Hello world'), channel: testUtils_1.uri });
-        const out = yield promise;
-        (0, vitest_1.expect)(out).toEqual('Hello world');
-    }));
-    (0, vitest_1.test)('handles closed string', () => __awaiter(void 0, void 0, void 0, function* () {
-        const reader = new reader_1.ReaderInstance(testUtils_1.uri, testUtils_1.client, testUtils_1.logger);
-        const promise = (0, testUtils_1.one)(reader.strings());
-        reader.close();
-        const out = yield promise;
-        (0, vitest_1.expect)(out).toEqual(undefined);
-    }));
-    (0, vitest_1.test)('handles strings', () => __awaiter(void 0, void 0, void 0, function* () {
-        const reader = new reader_1.ReaderInstance(testUtils_1.uri, testUtils_1.client, testUtils_1.logger);
-        const msgsPromise = (() => __awaiter(void 0, void 0, void 0, function* () {
-            var _a, e_1, _b, _c;
-            const msgs = [];
-            try {
-                for (var _d = true, _e = __asyncValues(reader.strings()), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
-                    _c = _f.value;
-                    _d = false;
-                    const msg = _c;
-                    msgs.push(msg);
-                }
-            }
-            catch (e_1_1) { e_1 = { error: e_1_1 }; }
-            finally {
-                try {
-                    if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
-                }
-                finally { if (e_1) throw e_1.error; }
-            }
-            return msgs;
-        }))();
-        reader.handleMsg({ data: encoder.encode('Hello'), channel: testUtils_1.uri });
-        reader.handleMsg({ data: encoder.encode('World'), channel: testUtils_1.uri });
-        reader.close();
-        const msgs = yield msgsPromise;
-        (0, vitest_1.expect)(msgs).toEqual(['Hello', 'World']);
-    }));
-    (0, vitest_1.test)('handles buffer', () => __awaiter(void 0, void 0, void 0, function* () {
-        const reader = new reader_1.ReaderInstance(testUtils_1.uri, testUtils_1.client, testUtils_1.logger);
-        const data = encoder.encode('Hello world');
-        const promise = (0, testUtils_1.one)(reader.buffers());
-        reader.handleMsg({ data, channel: testUtils_1.uri });
-        const out = yield promise;
-        (0, vitest_1.expect)(out).toEqual(data);
-    }));
-    (0, vitest_1.test)('handles buffers', () => __awaiter(void 0, void 0, void 0, function* () {
-        const reader = new reader_1.ReaderInstance(testUtils_1.uri, testUtils_1.client, testUtils_1.logger);
-        const data = encoder.encode('Hello world');
-        const promise = (0, testUtils_1.one)(reader.buffers());
-        reader.handleMsg({ data, channel: testUtils_1.uri });
-        const out = yield promise;
-        (0, vitest_1.expect)(out).toEqual(data);
-    }));
-    (0, vitest_1.test)('handles stream', () => __awaiter(void 0, void 0, void 0, function* () {
-        const reader = new reader_1.ReaderInstance(testUtils_1.uri, testUtils_1.client, testUtils_1.logger);
-        const nextStream = testUtils_1.client.nextStream();
-        const promise = (0, testUtils_1.one)(reader.streams());
-        reader.handleStreamingMessage({ id: { id: 5 }, channel: testUtils_1.uri });
-        const stream = yield nextStream;
-        stream.push('Hello');
-        stream.push('World');
-        console.log(stream.eventNames());
-        const out = yield promise;
-        const msgsPromise = (() => __awaiter(void 0, void 0, void 0, function* () {
-            var _a, e_2, _b, _c;
-            const msgs = [];
-            try {
-                for (var _d = true, _e = __asyncValues(out), _f; _f = yield _e.next(), _a = _f.done, !_a; _d = true) {
-                    _c = _f.value;
-                    _d = false;
-                    const msg = _c;
-                    msgs.push(msg);
-                }
-            }
-            catch (e_2_1) { e_2 = { error: e_2_1 }; }
-            finally {
-                try {
-                    if (!_d && !_a && (_b = _e.return)) yield _b.call(_e);
-                }
-                finally { if (e_2) throw e_2.error; }
-            }
-            return msgs;
-        }))();
-        stream.emit('end');
-        stream.emit('close');
-        const msgs = yield msgsPromise;
-        console.log(msgs);
-    }));
-}));

package/examples/echo/.idea/echo.iml

@@ -1,9 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module type="JAVA_MODULE" version="4">
-  <component name="NewModuleRootManager" inherit-compiler-output="true">
-    <exclude-output />
-    <content url="file://$MODULE_DIR$" />
-    <orderEntry type="inheritedJdk" />
-    <orderEntry type="sourceFolder" forTests="false" />
-  </component>
-</module>

package/examples/echo/.idea/misc.xml

@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="ProjectRootManager" version="2" languageLevel="JDK_23" project-jdk-name="24" project-jdk-type="JavaSDK">
-    <output url="file://$PROJECT_DIR$/out" />
-  </component>
-</project>

package/examples/echo/.idea/vcs.xml

@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="VcsDirectoryMappings">
-    <mapping directory="$PROJECT_DIR$/../.." vcs="Git" />
-    <mapping directory="$PROJECT_DIR$/node_modules/@rdfc/js-runner" vcs="Git" />
-  </component>
-</project>

package/examples/echo/.swls/config.json

@@ -1 +0,0 @@
-{ "disabled": ["shapes"] }

package/examples/echo/index.ttl

@@ -1,3 +0,0 @@
-@prefix foaf: <http://xmlns.com/foaf/0.1/>.
-<#me> a foaf:Person;
-    foaf:name "".

package/examples/echo/minimal.ttl

@@ -1,90 +0,0 @@
-@prefix rdfc: <https://w3id.org/rdf-connect#>.
-@prefix prov: <http://www.w3.org/ns/prov#>.
-@prefix sds: <https://w3id.org/sds#>.
-@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.
-@prefix owl: <http://www.w3.org/2002/07/owl#>.
-@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
-@prefix sh: <http://www.w3.org/ns/shacl#>.
-
-############################################################
-#                    General statements                    #
-############################################################
-# sds:Activity is a prov:Activity
-sds:Activity rdfs:subClassOf prov:Activity.
-
-# rdfc:Processor too
-rdfc:Processor rdfs:subClassOf sds:Activity.
-
-# sds:implementationOf is subClassOf
-sds:implementationOf rdfs:subPropertyOf rdfs:subClassOf.
-
-############################################################
-#                   Javascript statements                  #
-############################################################
-# specialized for js too
-rdfc:jsImplementationOf rdfs:subPropertyOf sds:implementationOf.
-
-# A node runner, runs things that are rdfc:jsImplementationOf rdfc:Processor  (aka, rdfs:subClassOf prov:Activity)
-rdfc:NodeRunner a rdfc:Runner;
-  rdfc:handlesSubjectsOf rdfc:jsImplementationOf;
-  rdfc:command "npx js-runner".
-
-# Shape that a Js Processor should fulfil;
-[ ] a sh:NodeShape;
-  # We target it with jsImplementationOf
-  sh:targetSubjectsOf rdfc:jsImplementationOf;
-  sh:property [
-    sh:path rdfc:entrypoint;
-    sh:name "location";
-    sh:minCount 1;
-    sh:maxCount 1;
-    sh:datatype xsd:string;
-  ], [
-    sh:path rdfc:file;
-    sh:name "file";
-    sh:minCount 1;
-    sh:maxCount 1;
-    sh:datatype xsd:string;
-  ], [
-    sh:path rdfc:class;
-    sh:name "clazz";
-    sh:maxCount 1;
-    sh:datatype xsd:string;
-  ].
-
-############################################################
-#                   Processor statements                   #
-############################################################
-rdfc:FooBarProcessor a owl:Class, rdfs:Class;
-  rdfc:jsImplementationOf rdfc:Processor;
-  rdfs:label "My Epic FooBar Processor";
-  rdfs:description "FooBars everything!";
-  rdfc:entrypoint <./>;
-  rdfc:file <./lib/processors.js>;
-  rdfc:class "FooBarProcessor".
-
-[ ] a sh:NodeShape;
-  sh:targetClass rdfc:FooBarProcessor;
-  sh:property [
-    sh:path rdfc:reader;
-    sh:name "reader";
-    sh:minCount 1;
-    sh:maxCount 1;
-    sh:class rdfc:Reader;
-  ].
-
-############################################################
-#                    Pipeline statements                   #
-############################################################
-<> a rdfc:Pipeline;
-  rdfc:consistsOf [
-    rdfc:instantiates rdfc:NodeRunner;
-    rdfc:processor <foobar>;
-  ].
-
-# <incomingMessages> a rdfc:Reader.
-<incomingMessages> a rdfc:Reader.
-
-<fooBar> a rdfc:FooBarProcessor;
-  rdfc:reader <incomingMessages>.
-

package/examples/echo/shacl.ttl

@@ -1,9 +0,0 @@
-
-@prefix foaf: <http://xmlns.com/foaf/0.1/>.
-@prefix sh: <http://www.w3.org/ns/shacl#>.
-[] a sh:NodeShape;
-  sh:targetClass foaf:Person;
-  sh:property [
-    sh:path foaf:name;
-    sh:minCount 1;
-  ].