@rdfc/js-runner 2.0.0-alpha.7 → 2.0.0-alpha.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.editorconfig +9 -0
- package/__tests__/channels.test.js +117 -0
- package/examples/echo/src/processors.js +123 -0
- package/examples/echo/src/processors.ts +29 -31
- package/lib/client.js +1 -1
- package/lib/index.d.ts +1 -1
- package/lib/index.js +2 -2
- package/lib/jsonld.js +3 -2
- package/lib/reader.js +8 -16
- package/lib/reexports.d.ts +2 -2
- package/lib/reexports.js +3 -3
- package/lib/runner.js +27 -16
- package/lib/testUtils.js +6 -4
- package/lib/tsconfig.tsbuildinfo +1 -1
- package/lib/util_processors.js +1 -1
- package/lib/writer.js +1 -1
- package/package.json +1 -1
- package/src/client.ts +16 -14
- package/src/index.ts +1 -1
- package/src/jsonld.ts +4 -1
- package/src/reader.ts +129 -133
- package/src/reexports.ts +5 -2
- package/src/runner.ts +160 -143
- package/src/testUtils.ts +34 -25
- package/src/util_processors.ts +1 -1
- package/src/writer.ts +66 -66
- package/vite.config.js +11 -0
package/src/testUtils.ts
CHANGED
|
@@ -1,20 +1,22 @@
|
|
|
1
|
-
import { DataChunk,
|
|
1
|
+
import { DataChunk, RunnerClient } from '@rdfc/proto'
|
|
2
2
|
import { ClientReadableStream } from '@grpc/grpc-js'
|
|
3
3
|
import * as grpc from '@grpc/grpc-js'
|
|
4
4
|
import { ClientReadableStreamImpl } from './reexports'
|
|
5
|
-
import { extractShapes } from 'rdf-lens'
|
|
6
|
-
import { Parser } from 'n3'
|
|
7
|
-
import { readFile } from 'fs/promises'
|
|
8
|
-
import winston, { createLogger } from 'winston'
|
|
9
|
-
import { WriterInstance } from './writer'
|
|
10
|
-
import { ReaderInstance } from './reader'
|
|
5
|
+
import { extractShapes } from 'rdf-lens'
|
|
6
|
+
import { Parser } from 'n3'
|
|
7
|
+
import { readFile } from 'fs/promises'
|
|
8
|
+
import winston, { createLogger } from 'winston'
|
|
9
|
+
import { WriterInstance } from './writer'
|
|
10
|
+
import { ReaderInstance } from './reader'
|
|
11
11
|
|
|
12
|
-
export async function getProcessorShape(
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
const
|
|
12
|
+
export async function getProcessorShape(
|
|
13
|
+
baseIRI = process.cwd() + '/node_modules/@rdfc/js-runner/index.ttl',
|
|
14
|
+
) {
|
|
15
|
+
const configFile = await readFile(baseIRI, { encoding: 'utf8' })
|
|
16
|
+
const configQuads = new Parser().parse(configFile)
|
|
17
|
+
const shapes = extractShapes(configQuads)
|
|
16
18
|
|
|
17
|
-
return shapes
|
|
19
|
+
return shapes
|
|
18
20
|
}
|
|
19
21
|
|
|
20
22
|
export class TestClient extends RunnerClient {
|
|
@@ -46,23 +48,30 @@ export async function one<T>(iter: AsyncIterable<T>): Promise<T | undefined> {
|
|
|
46
48
|
export const client = new TestClient()
|
|
47
49
|
export const uri = 'someUri'
|
|
48
50
|
export const logger = createLogger({
|
|
49
|
-
transports: new winston.transports.Console({
|
|
51
|
+
transports: new winston.transports.Console({
|
|
52
|
+
level: process.env['DEBUG'] || 'info',
|
|
53
|
+
}),
|
|
50
54
|
})
|
|
51
55
|
|
|
52
56
|
export function createWriter(iri = uri): [WriterInstance, ReaderInstance] {
|
|
53
|
-
const reader = createReader(iri)
|
|
54
|
-
const writeStream = new WriterInstance(
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
57
|
+
const reader = createReader(iri)
|
|
58
|
+
const writeStream = new WriterInstance(
|
|
59
|
+
iri,
|
|
60
|
+
client,
|
|
61
|
+
async (msg) => {
|
|
62
|
+
if (msg.msg) {
|
|
63
|
+
reader.handleMsg(msg.msg)
|
|
64
|
+
}
|
|
65
|
+
if (msg.close) {
|
|
66
|
+
reader.close()
|
|
67
|
+
}
|
|
68
|
+
},
|
|
69
|
+
logger,
|
|
70
|
+
)
|
|
71
|
+
return [writeStream, reader]
|
|
63
72
|
}
|
|
64
73
|
|
|
65
74
|
export function createReader(iri = uri): ReaderInstance {
|
|
66
|
-
const reader = new ReaderInstance(iri, client, logger)
|
|
67
|
-
return reader
|
|
75
|
+
const reader = new ReaderInstance(iri, client, logger)
|
|
76
|
+
return reader
|
|
68
77
|
}
|
package/src/util_processors.ts
CHANGED
package/src/writer.ts
CHANGED
|
@@ -5,86 +5,86 @@ import { Any } from './reader'
|
|
|
5
5
|
|
|
6
6
|
type Writable = (msg: OrchestratorMessage) => Promise<unknown>
|
|
7
7
|
export interface Writer {
|
|
8
|
-
|
|
9
|
-
|
|
8
|
+
readonly uri: string
|
|
9
|
+
buffer(buffer: Uint8Array): Promise<void>
|
|
10
10
|
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
11
|
+
stream(buffer: AsyncIterable<Uint8Array>): Promise<void>
|
|
12
|
+
stream<T>(
|
|
13
|
+
buffer: AsyncIterable<T>,
|
|
14
|
+
tranform: (x: T) => Uint8Array,
|
|
15
|
+
): Promise<void>
|
|
16
16
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
17
|
+
string(buffer: string): Promise<void>
|
|
18
|
+
any(any: Any): Promise<void>
|
|
19
|
+
close(): Promise<void>
|
|
20
20
|
}
|
|
21
21
|
const encoder = new TextEncoder()
|
|
22
22
|
export class WriterInstance implements Writer {
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
23
|
+
readonly uri: string
|
|
24
|
+
private readonly client: RunnerClient
|
|
25
|
+
private readonly write: Writable
|
|
26
|
+
private readonly logger: Logger
|
|
27
27
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
28
|
+
constructor(
|
|
29
|
+
uri: string,
|
|
30
|
+
client: RunnerClient,
|
|
31
|
+
write: Writable,
|
|
32
|
+
logger: Logger,
|
|
33
|
+
) {
|
|
34
|
+
this.client = client
|
|
35
|
+
this.write = write
|
|
36
|
+
this.uri = uri
|
|
37
|
+
this.logger = logger
|
|
38
|
+
}
|
|
39
|
+
async any(any: Any): Promise<void> {
|
|
40
|
+
if ('stream' in any) {
|
|
41
|
+
await this.stream(any.stream)
|
|
38
42
|
}
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
await this.stream(any.stream)
|
|
42
|
-
}
|
|
43
|
-
if ('buffer' in any) {
|
|
44
|
-
await this.buffer(any.buffer)
|
|
45
|
-
}
|
|
46
|
-
if ('string' in any) {
|
|
47
|
-
await this.string(any.string)
|
|
48
|
-
}
|
|
43
|
+
if ('buffer' in any) {
|
|
44
|
+
await this.buffer(any.buffer)
|
|
49
45
|
}
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
this.logger.debug(`${this.uri} sends buffer ${buffer.length} bytes`)
|
|
53
|
-
await this.write({ msg: { data: buffer, channel: this.uri } })
|
|
46
|
+
if ('string' in any) {
|
|
47
|
+
await this.string(any.string)
|
|
54
48
|
}
|
|
49
|
+
}
|
|
55
50
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
const t = transform || ((x: unknown) => <Uint8Array>x)
|
|
61
|
-
const stream = this.client.sendStreamMessage()
|
|
62
|
-
const id: Id = await new Promise((res) => stream.once('data', res))
|
|
63
|
-
this.logger.debug(`${this.uri} streams message with id ${id.id}`)
|
|
64
|
-
await this.write({ streamMsg: { id, channel: this.uri } })
|
|
51
|
+
async buffer(buffer: Uint8Array): Promise<void> {
|
|
52
|
+
this.logger.debug(`${this.uri} sends buffer ${buffer.length} bytes`)
|
|
53
|
+
await this.write({ msg: { data: buffer, channel: this.uri } })
|
|
54
|
+
}
|
|
65
55
|
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
56
|
+
async stream<T = Uint8Array>(
|
|
57
|
+
buffer: AsyncIterable<T>,
|
|
58
|
+
transform?: (x: T) => Uint8Array,
|
|
59
|
+
) {
|
|
60
|
+
const t = transform || ((x: unknown) => <Uint8Array>x)
|
|
61
|
+
const stream = this.client.sendStreamMessage()
|
|
62
|
+
const id: Id = await new Promise((res) => stream.once('data', res))
|
|
63
|
+
this.logger.debug(`${this.uri} streams message with id ${id.id}`)
|
|
64
|
+
await this.write({ streamMsg: { id, channel: this.uri } })
|
|
70
65
|
|
|
71
|
-
|
|
72
|
-
|
|
66
|
+
const write = promisify(stream.write.bind(stream))
|
|
67
|
+
for await (const msg of buffer) {
|
|
68
|
+
await write({ data: t(msg) })
|
|
73
69
|
}
|
|
74
70
|
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
}
|
|
71
|
+
this.logger.debug(`${this.uri} is done streaming message with id ${id.id}`)
|
|
72
|
+
stream.end()
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
async string(msg: string): Promise<void> {
|
|
76
|
+
this.logger.debug(`${this.uri} sends string ${msg.length} characters`)
|
|
77
|
+
await this.write({
|
|
78
|
+
msg: { data: encoder.encode(msg), channel: this.uri },
|
|
79
|
+
})
|
|
80
|
+
}
|
|
81
81
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
}
|
|
82
|
+
async close(issued = false): Promise<void> {
|
|
83
|
+
this.logger.debug(`${this.uri} closes stream`)
|
|
84
|
+
if (!issued) {
|
|
85
|
+
await this.write({
|
|
86
|
+
close: { channel: this.uri },
|
|
87
|
+
})
|
|
89
88
|
}
|
|
89
|
+
}
|
|
90
90
|
}
|
package/vite.config.js
ADDED