@rdfc/js-runner 2.0.0-alpha.9 → 3.0.0

package/.editorconfig

@@ -0,0 +1,9 @@
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+indent_size = 2

package/.github/renovate.json

@@ -0,0 +1,3 @@
+{
+  "extends": ["github>smessie/renovate-presets:js"]
+}

package/README.md

@@ -1,5 +1,129 @@
-# RDFC-2 Javascript runner
+# JavaScript runner for RDF-Connect
 
-Communitcates with the rdfc orchestrator though grpc.
+## Usage
 
-Take a look at the example pipeline at `./examples/echo/` for example processors and an example pipeline.
+To use the JavaScript (Node or Bun) runner for RDF-Connect, you need to have a pipeline configuration that includes JavaScript/TypeScript processors.
+The runner can be added to your RDF-Connect pipeline as follows:
+
+```turtle
+@prefix rdfc: <https://w3id.org/rdf-connect#>.
+@prefix owl: <http://www.w3.org/2002/07/owl#>.
+
+### Import the runner
+<> owl:imports <./node_modules/@rdfc/js-runner/index.ttl>.
+
+### Define the pipeline and add the Node runner
+<> a rdfc:Pipeline;
+   rdfc:consistsOf [
+       rdfc:instantiates rdfc:NodeRunner;  # Use rdfc:BunRunner for Bun
+       rdfc:processor <log>, <send>;  # List of JavaScript processors to be used in the pipeline. You should define and configure these processors separately.
+   ].
+```
+
+This example assumes that the js-runner is installed in your project via npm or yarn in the current working directory.
+
+You can install the js-runner package using the following command:
+
+```bash
+npm install @rdfc/js-runner
+```
+
+## Logging
+
+The JavaScript runner and processors use the `winston` logging library for logging.
+The JavasScript runner initiates a logger that is passed to each processor, allowing them to log messages at various levels (info, warn, error, debug).
+You can access this logger in your processor class code on the `this.logger` property.
+Here's an example of how to use the logger in a processor:
+
+```typescript
+import { Processor } from '@rdfc/js-runner'
+
+class MyProcessor extends Processor<MyProcessorArgs> {
+  async init(this: MyProcessorArgs & this): Promise<void> {
+    this.logger.info('I am initializing my processor!')
+  }
+  // ...
+}
+```
+
+This logger is configured to forward log messages to the RDF-Connect logging system.
+This means you can view and manage these logs in the RDF-Connect logging interface, allowing for consistent log management across different components of your RDF-Connect pipeline.
+
+If you want to create a child logger for a subclass or submethod, you can do so using the `extendLogger` method.
+Here's an example:
+
+```typescript
+import { Processor, extendLogger } from '@rdfc/js-runner'
+
+class MyProcessor extends Processor<MyProcessorArgs> {
+  async init(this: MyProcessorArgs & this): Promise<void> {
+    const childLogger = extendLogger(this.logger, 'init')
+    childLogger.debug('This is a debug message from init.')
+  }
+  // ...
+}
+```
+
+## Develop a processor for this runner
+
+The simplest way to start developing a processor for the JavaScript runner, is to start from the [template-processor-ts](https://github.com/rdf-connect/template-processor-ts) template repository.
+It has everything set up to get you started quickly and let you focus on the actual processor logic.
+
+At the very least, a JavaScript/TypeScript processor should consist of a class that inherits from the `Processor` abstract base class provided by the `@rdfc/js-runner` package.
+This class should implement the `init` method, which is called when the processor is initialized.
+This method is where you can set up any necessary configuration or state for your processor like opening a database connection or reading a file.
+Additionally, you should implement the `transform` method, which is called before the `produce` method.
+In this `transform` method, you should put any logic that handles incoming data by consuming readers, possibly transforming it, and passing it to the next processor in the pipeline.
+This method should only write to writers as reply to the data it receives from readers, not produce new data, as it is important that it does not write data to channels before all readers have been initialized and are ready to consume data.
+Finally, you should implement the `produce` method, which is called after all readers have been initialized and are ready to consume data.
+It is called after the `transform` method.
+This method is where you can produce (new) output data by writing to writers to send the data to the next step in the pipeline.
+
+Next to the class, you should define a configuration for the processor in the `processor.ttl` file of your package.
+JavaScript/TypeScript processors must include the JavaScript specific configuration parameters `rdfc:file`, `rdfc:class`, and `rdfc:entrypoint`, which specify the file, class, and entrypoint path of the processor, respectively.
+
+## Development of the js-runner
+
+The JavaScript runner is implemented in TypeScript.
+The source code is contained in the `src` folder.
+The main cli entry point is the `bin/runner.js` file, which you can also run after installation of the package using `npx js-runner`.
+
+If you want to get started with the development of the js-runner, you can clone the repository and install the dependencies using the following commands.
+
+Clone the repository:
+
+```bash
+git clone git@github.com:rdf-connect/js-runner.git
+cd js-runner
+```
+
+Install the dependencies:
+
+```bash
+npm install
+```
+
+You can run the formatter and linter using:
+
+```bash
+npm run format
+npm run lint
+```
+
+You can run the tests using:
+
+```bash
+npm test
+```
+
+You can then build the project using:
+
+```bash
+npm run build
+```
+
+Lastly, you can publish the package to npm using:
+
+```bash
+npm publish --access=public
+```

package/__tests__/channels.test.ts

@@ -1,96 +1,153 @@
-import { describe, expect, test } from 'vitest'
-import { ReaderInstance } from '../src/reader'
-import { one, uri, client, logger } from '../src/testUtils'
+import { describe, expect, test, vi } from 'vitest'
+import { StreamMsgMock } from '../src/testUtils'
+import { WriterInstance } from '../src/writer'
+import { FromRunner, StreamIdentify } from '@rdfc/proto'
+import { createLogger, transports } from 'winston'
 
 const encoder = new TextEncoder()
+const decoder = new TextDecoder()
 
-describe('Reader', async () => {
-  test('handles string', async () => {
-    const reader = new ReaderInstance(uri, client, logger)
-
-    const promise = one(reader.strings())
-    reader.handleMsg({ data: encoder.encode('Hello world'), channel: uri })
-    const out = await promise
+const logger = createLogger({
+  transports: new transports.Console({
+    level: process.env['DEBUG'] || 'info',
+  }),
+})
 
-    expect(out).toEqual('Hello world')
+describe('Writer', async () => {
+  test('sends strings', async () => {
+    const uri = 'someUri'
+    const runner = 'myRunner'
+    const fn = vi.fn((id: StreamIdentify) => {
+      expect(id.channel).toBe(uri)
+      expect(id.runner).toBe(runner)
+      return 1
+    })
+    const client = new StreamMsgMock(fn)
+    const msgs: FromRunner[] = []
+    const write = async (msg: FromRunner) => msgs.push(msg)
+    const writer = new WriterInstance(uri, client as any, write, runner, logger)
+
+    const send = writer.string('hello world')
+    writer.handled()
+    await send
+
+    expect(msgs.length).toBe(1)
+    expect(msgs.map((x) => decoder.decode(x.msg!.data))).toEqual([
+      'hello world',
+    ])
+
+    expect(fn).toBeCalledTimes(0)
   })
 
-  test('handles closed string', async () => {
-    const reader = new ReaderInstance(uri, client, logger)
-
-    const promise = one(reader.strings())
-    reader.close()
-    const out = await promise
-
-    expect(out).toEqual(undefined)
+  test('sends binary', async () => {
+    const uri = 'someUri'
+    const runner = 'myRunner'
+    const fn = vi.fn((id: StreamIdentify) => {
+      expect(id.channel).toBe(uri)
+      expect(id.runner).toBe(runner)
+      return 1
+    })
+    const client = new StreamMsgMock(fn)
+    const msgs: FromRunner[] = []
+    const write = async (msg: FromRunner) => msgs.push(msg)
+    const writer = new WriterInstance(uri, client as any, write, runner, logger)
+
+    const send = writer.buffer(encoder.encode('hello world'))
+    writer.handled()
+    await send
+
+    expect(msgs.length).toBe(1)
+    expect(msgs.map((x) => decoder.decode(x.msg!.data))).toEqual([
+      'hello world',
+    ])
+
+    expect(fn).toBeCalledTimes(0)
   })
 
-  test('handles strings', async () => {
-    const reader = new ReaderInstance(uri, client, logger)
-
-    const msgsPromise = (async () => {
-      const msgs: string[] = []
-      for await (const msg of reader.strings()) {
-        msgs.push(msg)
-      }
-      return msgs
-    })()
-    reader.handleMsg({ data: encoder.encode('Hello'), channel: uri })
-    reader.handleMsg({ data: encoder.encode('World'), channel: uri })
-    reader.close()
-
-    const msgs = await msgsPromise
-
-    expect(msgs).toEqual(['Hello', 'World'])
+  test('streams data', async () => {
+    const uri = 'someUri'
+    const runner = 'myRunner'
+    const fn = vi.fn((id: StreamIdentify) => {
+      expect(id.channel).toBe(uri)
+      expect(id.runner).toBe(runner)
+      return 1
+    })
+    const client = new StreamMsgMock(fn)
+    const msgs: FromRunner[] = []
+    const write = async (msg: FromRunner) => msgs.push(msg)
+    const writer = new WriterInstance(uri, client as any, write, runner, logger)
+
+    async function* gen() {
+      yield encoder.encode('hello')
+      yield encoder.encode('world')
+
+      setTimeout(() => writer.handled(), 20)
+    }
+
+    await writer.stream(gen())
+
+    expect(client.data.length).toBe(2)
+    expect(client.data.map((x) => decoder.decode(x.data))).toEqual([
+      'hello',
+      'world',
+    ])
+    expect(fn).toBeCalled()
   })
 
-  test('handles buffer', async () => {
-    const reader = new ReaderInstance(uri, client, logger)
-
-    const data = encoder.encode('Hello world')
-    const promise = one(reader.buffers())
-    reader.handleMsg({ data, channel: uri })
-    const out = await promise
-
-    expect(out).toEqual(data)
+  test('closes', async () => {
+    const uri = 'someUri'
+    const runner = 'myRunner'
+    const fn = vi.fn((id: StreamIdentify) => {
+      expect(id.channel).toBe(uri)
+      expect(id.runner).toBe(runner)
+      return 1
+    })
+    const client = new StreamMsgMock(fn)
+    const msgs: FromRunner[] = []
+    const write = async (msg: FromRunner) => msgs.push(msg)
+    const writer = new WriterInstance(uri, client as any, write, runner, logger)
+
+    await writer.close()
+
+    expect(msgs.length).toBe(1)
+    expect(msgs.map((x) => x.close!.channel)).toEqual([uri])
+
+    expect(fn).toBeCalledTimes(0)
   })
 
-  test('handles buffers', async () => {
-    const reader = new ReaderInstance(uri, client, logger)
+  test('wait to close after stream is finished', async () => {
+    const uri = 'someUri'
+    const runner = 'myRunner'
+    const client = new StreamMsgMock(() => 1)
 
-    const data = encoder.encode('Hello world')
-    const promise = one(reader.buffers())
-    reader.handleMsg({ data, channel: uri })
-    const out = await promise
+    const msgs: FromRunner[] = []
+    const write = async (msg: FromRunner) => msgs.push(msg)
+    const writer = new WriterInstance(uri, client as any, write, runner, logger)
 
-    expect(out).toEqual(data)
-  })
+    let closingPromise: Promise<void> | undefined = undefined
+    async function* gen() {
+      yield encoder.encode('hello')
 
-  test('handles stream', async () => {
-    const reader = new ReaderInstance(uri, client, logger)
+      // initiate close but the channel cannot close yet, as it has an open stream message
+      closingPromise = writer.close()
 
-    const nextStream = client.nextStream()
-    const promise = one(reader.streams())
-    reader.handleStreamingMessage({ id: { id: 5 }, channel: uri })
-    const stream = await nextStream
+      await new Promise((res) => setTimeout(res, 20))
 
-    stream.push('Hello')
-    stream.push('World')
-    console.log(stream.eventNames())
+      expect(msgs.filter((x) => !!x.close)).toEqual([])
+      yield encoder.encode('world')
 
-    const out = await promise
-    const msgsPromise = (async () => {
-      const msgs: Uint8Array[] = []
-      for await (const msg of out!) {
-        msgs.push(msg)
-      }
-      return msgs
-    })()
+      // we 'handled' the message
+      setTimeout(() => writer.handled(), 20)
+    }
 
-    stream.emit('end')
-    stream.emit('close')
+    await writer.stream(gen())
+    await closingPromise!
+    expect(msgs.map((x) => x.close!.channel)).toEqual([uri])
 
-    const msgs = await msgsPromise
-    console.log(msgs)
+    expect(client.data.length).toBe(2)
+    expect(client.data.map((x) => decoder.decode(x.data))).toEqual([
+      'hello',
+      'world',
+    ])
   })
 })

package/__tests__/echoProcessor.test.ts

@@ -0,0 +1,131 @@
+import { describe, expect, test } from 'vitest'
+import { Reader } from '../src/reader'
+import { createRunner, channel } from '../src/testUtils'
+import { Processor } from '../src/processor'
+import { Writer } from '../src/writer'
+import { FullProc } from '../src/runner'
+import { createLogger, transports } from 'winston'
+
+const encoder = new TextEncoder()
+
+const logger = createLogger({
+  transports: new transports.Console({
+    level: process.env['DEBUG'] || 'info',
+  }),
+})
+
+class EchoProcessor extends Processor<{
+  reader: Reader
+  writer: Writer
+  streams: boolean
+}> {
+  async init(this: { reader: Reader; writer: Writer } & this): Promise<void> {
+    this.logger.info('EchoProcessor initialized')
+  }
+
+  async transform(
+    this: { reader: Reader; writer: Writer; streams: boolean } & this,
+  ): Promise<void> {
+    this.logger.info('EchoProcessor transforming')
+
+    if (this.streams) {
+      for await (const msg of this.reader.streams()) {
+        this.logger.info(`EchoProcessor received stream`)
+        await this.writer.stream(msg)
+      }
+    } else {
+      for await (const msg of this.reader.strings()) {
+        this.logger.info(`EchoProcessor received: ${msg}`)
+        await this.writer.string(msg)
+      }
+    }
+
+    this.logger.info('EchoProcessor transformed')
+    await this.writer.close()
+  }
+
+  async produce(
+    this: { reader: Reader; writer: Writer } & this,
+  ): Promise<void> {
+    // Nothing to produce, transform handles everything
+  }
+}
+
+describe('EchoProcessor', () => {
+  test('echoes string messages correctly', async () => {
+    const runner = createRunner()
+
+    const [inputWriter, inputReader] = channel(runner, 'input')
+    const [outputWriter, outputReader] = channel(runner, 'output')
+
+    const proc = <FullProc<EchoProcessor>>(
+      new EchoProcessor(
+        { reader: inputReader, writer: outputWriter, streams: false },
+        logger,
+      )
+    )
+
+    await proc.init()
+    const transformPromise = proc.transform()
+
+    const msgs: string[] = []
+
+    ;(async () => {
+      for await (const m of outputReader.strings()) {
+        msgs.push(m)
+      }
+    })()
+
+    await inputWriter.string('Hello')
+    expect(msgs).toEqual(['Hello'])
+    await inputWriter.string('world')
+
+    await inputWriter.close()
+
+    // Wait for processing to complete
+    await transformPromise
+
+    expect(msgs).toEqual(['Hello', 'world'])
+  })
+
+  test('echoes stream messages correctly', async () => {
+    const runner = createRunner()
+
+    const [inputWriter, inputReader] = channel(runner, 'input')
+    const [outputWriter, outputReader] = channel(runner, 'output')
+
+    const proc = <FullProc<EchoProcessor>>(
+      new EchoProcessor(
+        { reader: inputReader, writer: outputWriter, streams: true },
+        logger,
+      )
+    )
+
+    await proc.init()
+    const transformPromise = proc.transform()
+
+    const msgs: string[] = []
+
+    ;(async () => {
+      for await (const m of outputReader.strings()) {
+        msgs.push(m)
+      }
+    })()
+
+    const gen = async function* () {
+      yield encoder.encode('Hello')
+      yield encoder.encode('World')
+    }
+
+    await inputWriter.stream(gen())
+    expect(msgs).toEqual(['HelloWorld'])
+    await inputWriter.stream(gen())
+
+    await inputWriter.close()
+
+    // Wait for processing to complete
+    await transformPromise
+
+    expect(msgs).toEqual(['HelloWorld', 'HelloWorld'])
+  })
+})

package/__tests__/testProcessor.test.ts

@@ -0,0 +1,69 @@
+import { describe, expect, test, vi } from 'vitest'
+import { ProcHelper } from '../src/testUtils'
+import { Processor } from '../src/processor'
+import path from 'path/posix'
+
+// Example test processor
+export class TestProcessor extends Processor<{ message: string }> {
+  async init(): Promise<void> {
+    this.logger.info('Test processor initialized')
+  }
+
+  async transform(): Promise<void> {
+    // Mock transform - just wait
+    await new Promise((resolve) => setTimeout(resolve, 10))
+  }
+
+  async produce(): Promise<void> {
+    // Mock produce - do nothing
+  }
+}
+
+describe('Test processor', async () => {
+  test('Is well defined', async () => {
+    const helper = new ProcHelper<TestProcessor>()
+
+    await helper.importFile(path.resolve('./index.ttl'))
+    await helper.importInline(
+      path.resolve('config.ttl'),
+      `@prefix ex: <http://example.org/>.
+@prefix xsd: <http://www.w3.org/2001/XMLSchema#>.
+@prefix sh: <http://www.w3.org/ns/shacl#>.
+@prefix owl: <http://www.w3.org/2002/07/owl#>.
+@prefix rdfc: <https://w3id.org/rdf-connect#>.
+
+rdfc:Test a owl:Class;
+  rdfc:jsImplementationOf rdfc:Processor;
+  rdfc:file <./__tests__/testProcessor.test.ts>;
+  rdfc:class "TestProcessor";
+  rdfc:entrypoint <./>.
+
+[ ] a sh:NodeShape;
+  sh:targetClass rdfc:Test;
+  sh:property [
+    sh:datatype xsd:string;
+    sh:path rdfc:message;
+    sh:name "message";
+    sh:maxCount 1;
+    sh:minCount 1;
+  ].`,
+    )
+
+    const config = helper.getConfig('Test')
+    expect(config.clazz).toEqual('TestProcessor')
+    expect(config.location).toBeDefined()
+    expect(config.file).toBeDefined()
+
+    await helper.importInline(
+      path.resolve('pipeline.ttl'),
+      `@prefix rdfc: <https://w3id.org/rdf-connect#>.
+
+<http://example.org/proc> a rdfc:Test;
+  rdfc:message "Hello world".`,
+    )
+
+    const proc = await helper.getProcessor('http://example.org/proc')
+
+    expect(proc.message).toEqual('Hello world')
+  })
+})

package/eslint.config.mjs

@@ -11,7 +11,7 @@ export default [
   pluginJs.configs.recommended,
   ...tseslint.configs.recommended,
   {
-    ignores: ['src/generated/', '**/lib/', 'coverage/'],
+    ignores: ['src/generated/', '**/lib/', 'coverage/', '__tests__/'],
   },
   {
     languageOptions: {

package/examples/echo/.idea/echo.iml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

package/examples/echo/.idea/misc.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" languageLevel="JDK_23" project-jdk-name="24" project-jdk-type="JavaSDK">
+    <output url="file://$PROJECT_DIR$/out" />
+  </component>
+</project>

package/{.idea → examples/echo/.idea}/modules.xml

@@ -2,7 +2,7 @@
 <project version="4">
   <component name="ProjectModuleManager">
     <modules>
-      <module fileurl="file://$PROJECT_DIR$/.idea/js-runner.iml" filepath="$PROJECT_DIR$/.idea/js-runner.iml" />
+      <module fileurl="file://$PROJECT_DIR$/.idea/echo.iml" filepath="$PROJECT_DIR$/.idea/echo.iml" />
     </modules>
   </component>
 </project>

package/examples/echo/.idea/vcs.xml

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$/../.." vcs="Git" />
+    <mapping directory="$PROJECT_DIR$/node_modules/@rdfc/js-runner" vcs="Git" />
+  </component>
+</project>

package/examples/echo/.swls/config.json

@@ -0,0 +1 @@
+{ "disabled": ["shapes"] }

package/examples/echo/index.ttl

@@ -0,0 +1,3 @@
+@prefix foaf: <http://xmlns.com/foaf/0.1/>.
+<#me> a foaf:Person;
+    foaf:name "".