@combycode/dpe-framework-ts 0.0.0-init → 2.0.0-rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +98 -1
- package/package.json +35 -5
- package/src/context.ts +121 -0
- package/src/envelope.ts +131 -0
- package/src/index.ts +28 -0
- package/src/runtime.ts +135 -0
package/README.md
CHANGED
|
@@ -1,3 +1,100 @@
|
|
|
1
1
|
# @combycode/dpe-framework-ts
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
DPE framework for building streaming pipeline tools in TypeScript / Bun.
|
|
4
|
+
|
|
5
|
+
A DPE tool is an independent program: it reads NDJSON envelopes from stdin,
|
|
6
|
+
processes each one, and writes NDJSON envelopes to stdout. The runner
|
|
7
|
+
(`dpe`) spawns it, pipes data through it, and routes its output to the
|
|
8
|
+
next stage. The framework handles the loop, the parsing, the queues, and
|
|
9
|
+
the error/trace plumbing — your code only writes the transform.
|
|
10
|
+
|
|
11
|
+
Targets Bun by default. Node 20+ should also work but isn't part of CI.
|
|
12
|
+
|
|
13
|
+
## Install
|
|
14
|
+
|
|
15
|
+
```bash
|
|
16
|
+
bun add @combycode/dpe-framework-ts
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
## Hello tool
|
|
20
|
+
|
|
21
|
+
```ts
|
|
22
|
+
// src/main.ts
|
|
23
|
+
import { run, type Context, type JSONValue } from "@combycode/dpe-framework-ts";
|
|
24
|
+
|
|
25
|
+
async function processInput(v: JSONValue, _settings: JSONValue, ctx: Context) {
|
|
26
|
+
ctx.output(v);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
await run({ input: processInput });
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Run as a tool: `bun src/main.ts '{"...settings JSON..."}'`. The runner
|
|
33
|
+
will pass settings as `argv[1]` and pipe NDJSON through stdin/stdout.
|
|
34
|
+
|
|
35
|
+
## Processor signature
|
|
36
|
+
|
|
37
|
+
```ts
|
|
38
|
+
type Processor = (
|
|
39
|
+
v: JSONValue,
|
|
40
|
+
settings: JSONValue,
|
|
41
|
+
ctx: Context,
|
|
42
|
+
) => void | Promise<void>;
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Both sync and async processors are supported. The runtime awaits each
|
|
46
|
+
call before reading the next stdin line, so you can `await` I/O without
|
|
47
|
+
losing ordering.
|
|
48
|
+
|
|
49
|
+
## Internal queues
|
|
50
|
+
|
|
51
|
+
```ts
|
|
52
|
+
import { run, type Context, type JSONValue } from "@combycode/dpe-framework-ts";
|
|
53
|
+
|
|
54
|
+
async function processInput(v: JSONValue, _s: JSONValue, ctx: Context) {
|
|
55
|
+
ctx.emit("validate", v);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
async function processValidate(v: JSONValue, _s: JSONValue, ctx: Context) {
|
|
59
|
+
if ((v as any).ok === true) ctx.output(v);
|
|
60
|
+
else ctx.error(v, "validation failed");
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
await run({
|
|
64
|
+
input: processInput,
|
|
65
|
+
queues: { validate: processValidate },
|
|
66
|
+
});
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
`ctx.emit(queue, v)` enqueues for the named handler. `await ctx.drain()`
|
|
70
|
+
flushes all queues before continuing.
|
|
71
|
+
|
|
72
|
+
## Context API (essentials)
|
|
73
|
+
|
|
74
|
+
| Call | What it does |
|
|
75
|
+
|---|---|
|
|
76
|
+
| `ctx.output(v, id?, src?)` | Emit `{t:"d",...}` to stdout. `id`/`src` default to inherit. |
|
|
77
|
+
| `ctx.emit(queue, v)` | Push to internal queue. |
|
|
78
|
+
| `ctx.drain()` | Resolve once every queue is drained. |
|
|
79
|
+
| `ctx.error(v, msg)` | Write `{type:"error",input:v,error:msg,...}` to stderr. |
|
|
80
|
+
| `ctx.log(level, msg)` | Structured stderr log. |
|
|
81
|
+
| `ctx.meta(v)` | Emit `{t:"m", v}` envelope to stdout. |
|
|
82
|
+
| `ctx.memory` | Typed accumulators (`Number`, `Average`, `MinMax`, `Set`, `Map`, …). |
|
|
83
|
+
|
|
84
|
+
See `docs/frameworks.md` in the monorepo for the full reference.
|
|
85
|
+
|
|
86
|
+
## Scaffolding
|
|
87
|
+
|
|
88
|
+
```bash
|
|
89
|
+
dpe-dev scaffold --name my-tool --runtime bun --out ./my-tool --description "what it does"
|
|
90
|
+
cd my-tool
|
|
91
|
+
dpe-dev build . # bun install
|
|
92
|
+
dpe-dev test . # bun test
|
|
93
|
+
dpe-dev verify . # spawn `bun src/main.ts`, feed input.ndjson, diff stdout vs expected.ndjson
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
## Repo & licence
|
|
97
|
+
|
|
98
|
+
- Source: <https://github.com/combycode/dpe-framework-ts>
|
|
99
|
+
- Monorepo (canonical): <https://github.com/combycode/dpe>
|
|
100
|
+
- Licence: AGPL-3.0-or-later
|
package/package.json
CHANGED
|
@@ -1,8 +1,38 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@combycode/dpe-framework-ts",
|
|
3
|
-
"version": "
|
|
4
|
-
"description": "DPE framework for building streaming pipeline tools in TypeScript
|
|
3
|
+
"version": "2.0.0-rc1",
|
|
4
|
+
"description": "DPE framework for building streaming pipeline tools in TypeScript/Bun",
|
|
5
5
|
"license": "AGPL-3.0-or-later",
|
|
6
|
-
"repository": {
|
|
7
|
-
|
|
8
|
-
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "git+https://github.com/combycode/dpe.git",
|
|
9
|
+
"directory": "frameworks/ts"
|
|
10
|
+
},
|
|
11
|
+
"homepage": "https://github.com/combycode/dpe#readme",
|
|
12
|
+
"type": "module",
|
|
13
|
+
"main": "src/index.ts",
|
|
14
|
+
"module": "src/index.ts",
|
|
15
|
+
"types": "src/index.ts",
|
|
16
|
+
"exports": {
|
|
17
|
+
".": {
|
|
18
|
+
"types": "./src/index.ts",
|
|
19
|
+
"default": "./src/index.ts"
|
|
20
|
+
}
|
|
21
|
+
},
|
|
22
|
+
"scripts": {
|
|
23
|
+
"test": "bun test"
|
|
24
|
+
},
|
|
25
|
+
"devDependencies": {
|
|
26
|
+
"@types/bun": "^1.3.13",
|
|
27
|
+
"ajv": "^8.17.1",
|
|
28
|
+
"typescript": "^5.9.0"
|
|
29
|
+
},
|
|
30
|
+
"files": [
|
|
31
|
+
"src/",
|
|
32
|
+
"README.md"
|
|
33
|
+
],
|
|
34
|
+
"publishConfig": {
|
|
35
|
+
"access": "public",
|
|
36
|
+
"provenance": true
|
|
37
|
+
}
|
|
38
|
+
}
|
package/src/context.ts
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Context object — passed to every processor invocation.
|
|
3
|
+
*
|
|
4
|
+
* Mirrors the Python and Rust Context API:
|
|
5
|
+
* ctx.output(v, {id?, src?})
|
|
6
|
+
* ctx.emit(queue, v, {id?, src?})
|
|
7
|
+
* ctx.drain()
|
|
8
|
+
* ctx.meta(v)
|
|
9
|
+
* ctx.log(msg, {level?, ...extra})
|
|
10
|
+
* ctx.error(v, err)
|
|
11
|
+
* ctx.trace(k, v)
|
|
12
|
+
* ctx.stats(data)
|
|
13
|
+
* ctx.hash(str)
|
|
14
|
+
* ctx.hashFile(path)
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import type { JSONValue } from './envelope';
|
|
18
|
+
import {
|
|
19
|
+
hashFile,
|
|
20
|
+
hashString,
|
|
21
|
+
writeData,
|
|
22
|
+
writeError,
|
|
23
|
+
writeLog,
|
|
24
|
+
writeMeta,
|
|
25
|
+
writeStats,
|
|
26
|
+
writeTrace,
|
|
27
|
+
} from './envelope';
|
|
28
|
+
|
|
29
|
+
export interface QueueItem {
|
|
30
|
+
queue: string;
|
|
31
|
+
v: JSONValue;
|
|
32
|
+
id: string;
|
|
33
|
+
src: string;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Tiny runtime surface the Context needs — avoids circular imports with Runtime. */
|
|
37
|
+
export interface RuntimeLike {
|
|
38
|
+
enqueue(item: QueueItem): void;
|
|
39
|
+
drainQueue(): Promise<void>;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export class Context {
|
|
43
|
+
readonly id: string;
|
|
44
|
+
readonly src: string;
|
|
45
|
+
readonly memory: Memory;
|
|
46
|
+
private readonly _runtime: RuntimeLike;
|
|
47
|
+
/** Labels accumulated by ctx.trace(k, v). Flushed (merged) as one
|
|
48
|
+
* {type:"trace"} stderr event before each ctx.output(), then cleared. */
|
|
49
|
+
private _labels: Record<string, JSONValue> = {};
|
|
50
|
+
|
|
51
|
+
constructor(id: string, src: string, memory: Memory, runtime: RuntimeLike) {
|
|
52
|
+
this.id = id;
|
|
53
|
+
this.src = src;
|
|
54
|
+
this.memory = memory;
|
|
55
|
+
this._runtime = runtime;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Accumulate a label on this invocation's next output envelope. */
|
|
59
|
+
trace(key: string, value: JSONValue): void {
|
|
60
|
+
this._labels[key] = value;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Emit a stats event to stderr. Example: ctx.stats({rows:1000, rps:250}). */
|
|
64
|
+
stats(data: Record<string, JSONValue>): void {
|
|
65
|
+
if (data && Object.keys(data).length > 0) {
|
|
66
|
+
writeStats(data);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
output(v: JSONValue, opts: { id?: string; src?: string } = {}): void {
|
|
71
|
+
const outId = opts.id ?? this.id;
|
|
72
|
+
const outSrc = opts.src ?? this.src;
|
|
73
|
+
// Emit merged trace first (always, even with empty labels), then data, then clear.
|
|
74
|
+
writeTrace(outId, outSrc, this._labels);
|
|
75
|
+
this._labels = {};
|
|
76
|
+
writeData(v, outId, outSrc);
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
emit(queue: string, v: JSONValue, opts: { id?: string; src?: string } = {}): void {
|
|
80
|
+
this._runtime.enqueue({
|
|
81
|
+
queue,
|
|
82
|
+
v,
|
|
83
|
+
id: opts.id ?? this.id,
|
|
84
|
+
src: opts.src ?? this.src,
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async drain(): Promise<void> {
|
|
89
|
+
await this._runtime.drainQueue();
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
meta(v: JSONValue): void {
|
|
93
|
+
writeMeta(v);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
log(msg: string, opts: { level?: string } & Record<string, JSONValue> = {}): void {
|
|
97
|
+
const { level = 'info', ...extra } = opts as { level?: string } & Record<string, JSONValue>;
|
|
98
|
+
writeLog(msg, level, Object.keys(extra).length ? extra : undefined);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
error(v: JSONValue, err: unknown): void {
|
|
102
|
+
writeError(v, err, this.id, this.src);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
hash(key: string): string {
|
|
106
|
+
return hashString(key);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
hashFile(filepath: string): Promise<string | undefined> {
|
|
110
|
+
return hashFile(filepath);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Shared accumulator memory. Tools extend this by attaching their own
|
|
116
|
+
* accumulator instances as properties. Named accumulator types will ship in a
|
|
117
|
+
* later milestone — filter tool doesn't need them.
|
|
118
|
+
*/
|
|
119
|
+
export class Memory {
|
|
120
|
+
[key: string]: unknown;
|
|
121
|
+
}
|
package/src/envelope.ts
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* NDJSON envelope types + parse / write / hash helpers.
|
|
3
|
+
*
|
|
4
|
+
* The envelope protocol matches the Python and Rust frameworks:
|
|
5
|
+
* data: {"t":"d","id":"<8B hex>","src":"<string>","v":<payload>}
|
|
6
|
+
* meta: {"t":"m","v":<payload>}
|
|
7
|
+
* log: {"type":"log","level":"...","msg":"...","...":"..."} (stderr)
|
|
8
|
+
* error: {"type":"error","error":"...","input":..,"id":..,"src":..} (stderr)
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { createHash } from 'node:crypto';
|
|
12
|
+
|
|
13
|
+
export type JSONValue = null | boolean | number | string | JSONValue[] | { [k: string]: JSONValue };
|
|
14
|
+
|
|
15
|
+
export interface DataEnvelope {
|
|
16
|
+
t: 'd';
|
|
17
|
+
id: string;
|
|
18
|
+
src: string;
|
|
19
|
+
v: JSONValue;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface MetaEnvelope {
|
|
23
|
+
t: 'm';
|
|
24
|
+
v: JSONValue;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export type Envelope = DataEnvelope | MetaEnvelope;
|
|
28
|
+
|
|
29
|
+
/** blake2b-64 hash of a string. Returns 16-char hex. Matches Python/Rust envelope ID. */
|
|
30
|
+
export function hashString(key: string): string {
|
|
31
|
+
return createHash('blake2b512').update(key, 'utf8').digest('hex').slice(0, 16);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/** Stream-hash a file by path. Returns 16 hex chars. `undefined` if the file cannot be opened.
|
|
35
|
+
*
|
|
36
|
+
* Streams chunks through the hasher rather than loading the file into a single
|
|
37
|
+
* Uint8Array — important for large inputs (> a few hundred MB).
|
|
38
|
+
*/
|
|
39
|
+
export async function hashFile(filepath: string): Promise<string | undefined> {
|
|
40
|
+
try {
|
|
41
|
+
const hasher = createHash('blake2b512');
|
|
42
|
+
const stream = Bun.file(filepath).stream();
|
|
43
|
+
for await (const chunk of stream as AsyncIterable<Uint8Array>) {
|
|
44
|
+
hasher.update(chunk);
|
|
45
|
+
}
|
|
46
|
+
return hasher.digest('hex').slice(0, 16);
|
|
47
|
+
} catch {
|
|
48
|
+
return undefined;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Deterministic id from (src | stage | sorted JSON of v). */
|
|
53
|
+
export function computeId(src: string, stage: string, v: JSONValue): string {
|
|
54
|
+
const vStr = canonicalJson(v);
|
|
55
|
+
return hashString(`${src}|${stage}|${vStr}`);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Parse one NDJSON line. Returns null on blank, invalid JSON, or any
|
|
59
|
+
* shape that isn't a well-formed envelope. Data envelopes must carry
|
|
60
|
+
* `id` and `src` strings; meta envelopes must carry a `v` payload. */
|
|
61
|
+
export function parseEnvelope(line: string): Envelope | null {
|
|
62
|
+
const trimmed = line.trim();
|
|
63
|
+
if (!trimmed) return null;
|
|
64
|
+
try {
|
|
65
|
+
const obj = JSON.parse(trimmed);
|
|
66
|
+
if (!obj || typeof obj !== 'object') return null;
|
|
67
|
+
if (obj.t === 'd') {
|
|
68
|
+
if (typeof obj.id !== 'string' || typeof obj.src !== 'string') return null;
|
|
69
|
+
if (!('v' in obj)) return null;
|
|
70
|
+
return obj as DataEnvelope;
|
|
71
|
+
}
|
|
72
|
+
if (obj.t === 'm') {
|
|
73
|
+
if (!('v' in obj)) return null;
|
|
74
|
+
return obj as MetaEnvelope;
|
|
75
|
+
}
|
|
76
|
+
return null;
|
|
77
|
+
} catch {
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function writeData(v: JSONValue, id: string, src: string): void {
|
|
83
|
+
_writeStdout({ t: 'd', id, src, v });
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export function writeMeta(v: JSONValue): void {
|
|
87
|
+
_writeStdout({ t: 'm', v });
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export function writeLog(msg: string, level = 'info', extra?: Record<string, JSONValue>): void {
|
|
91
|
+
_writeStderr({ type: 'log', level, msg, ...(extra ?? {}) });
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export function writeError(v: JSONValue, err: unknown, id: string, src: string): void {
|
|
95
|
+
_writeStderr({
|
|
96
|
+
type: 'error',
|
|
97
|
+
error: err instanceof Error ? err.message : String(err),
|
|
98
|
+
input: v,
|
|
99
|
+
id,
|
|
100
|
+
src,
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** Merged trace event flushed once per output envelope (even with empty labels). */
|
|
105
|
+
export function writeTrace(id: string, src: string, labels: Record<string, JSONValue>): void {
|
|
106
|
+
_writeStderr({ type: 'trace', id, src, labels });
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** Custom stats event. Shape: {type:"stats", ...data}. */
|
|
110
|
+
export function writeStats(data: Record<string, JSONValue>): void {
|
|
111
|
+
_writeStderr({ type: 'stats', ...data });
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Canonical JSON for hashing: sorted keys, compact, no spaces. */
|
|
115
|
+
function canonicalJson(v: JSONValue): string {
|
|
116
|
+
if (v === null || typeof v !== 'object') return JSON.stringify(v);
|
|
117
|
+
if (Array.isArray(v)) return `[${v.map(canonicalJson).join(',')}]`;
|
|
118
|
+
const keys = Object.keys(v).sort();
|
|
119
|
+
const parts = keys.map(
|
|
120
|
+
(k) => `${JSON.stringify(k)}:${canonicalJson((v as Record<string, JSONValue>)[k]!)}`,
|
|
121
|
+
);
|
|
122
|
+
return `{${parts.join(',')}}`;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function _writeStdout(obj: Record<string, JSONValue>): void {
|
|
126
|
+
process.stdout.write(`${JSON.stringify(obj)}\n`);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function _writeStderr(obj: Record<string, JSONValue>): void {
|
|
130
|
+
process.stderr.write(`${JSON.stringify(obj)}\n`);
|
|
131
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @combycode/dpe-framework-ts — DPE framework for Bun/TypeScript tools.
|
|
3
|
+
*
|
|
4
|
+
* Example:
|
|
5
|
+
* import { run } from "@combycode/dpe-framework-ts";
|
|
6
|
+
*
|
|
7
|
+
* await run({
|
|
8
|
+
* input: (v, settings, ctx) => {
|
|
9
|
+
* ctx.output({ ok: true });
|
|
10
|
+
* },
|
|
11
|
+
* });
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
export type { QueueItem, RuntimeLike } from './context';
|
|
15
|
+
export { Context, Memory } from './context';
|
|
16
|
+
export type { DataEnvelope, Envelope, JSONValue, MetaEnvelope } from './envelope';
|
|
17
|
+
export {
|
|
18
|
+
computeId,
|
|
19
|
+
hashFile,
|
|
20
|
+
hashString,
|
|
21
|
+
parseEnvelope,
|
|
22
|
+
writeData,
|
|
23
|
+
writeError,
|
|
24
|
+
writeLog,
|
|
25
|
+
writeMeta,
|
|
26
|
+
} from './envelope';
|
|
27
|
+
export type { Processor, RunOptions } from './runtime';
|
|
28
|
+
export { run } from './runtime';
|
package/src/runtime.ts
ADDED
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Runtime — stdin loop, settings parse, queue drain, processor dispatch.
|
|
3
|
+
*
|
|
4
|
+
* Tools register processors explicitly via `run({input, queues?})`.
|
|
5
|
+
* Unlike Python (which auto-discovers process_* functions), TS is structurally
|
|
6
|
+
* typed so explicit registration keeps types honest.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { Context, Memory, type QueueItem, type RuntimeLike } from './context';
|
|
10
|
+
import type { DataEnvelope, JSONValue } from './envelope';
|
|
11
|
+
import { parseEnvelope, writeLog } from './envelope';
|
|
12
|
+
|
|
13
|
+
export type Processor = (v: JSONValue, settings: JSONValue, ctx: Context) => void | Promise<void>;
|
|
14
|
+
|
|
15
|
+
export interface RunOptions {
|
|
16
|
+
/** Main input processor — called for each data envelope on stdin. */
|
|
17
|
+
input: Processor;
|
|
18
|
+
/** Optional named queue processors — triggered by ctx.emit(queue, v). */
|
|
19
|
+
queues?: Record<string, Processor>;
|
|
20
|
+
/** Called once before the stdin loop starts (e.g. parse/compile settings). */
|
|
21
|
+
onStart?: (settings: JSONValue) => void | Promise<void>;
|
|
22
|
+
/** Called once after stdin EOF / SIGTERM (e.g. flush, emit final meta). */
|
|
23
|
+
onShutdown?: () => void | Promise<void>;
|
|
24
|
+
/** Max iterations per drain cycle — safety against infinite emit loops. */
|
|
25
|
+
maxDrainIterations?: number;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
class Runtime implements RuntimeLike {
|
|
29
|
+
settings: JSONValue = {};
|
|
30
|
+
memory = new Memory();
|
|
31
|
+
private input!: Processor;
|
|
32
|
+
private queues: Record<string, Processor> = {};
|
|
33
|
+
private queue: QueueItem[] = [];
|
|
34
|
+
private shutdown = false;
|
|
35
|
+
private drainLimit = 100_000;
|
|
36
|
+
|
|
37
|
+
enqueue(item: QueueItem): void {
|
|
38
|
+
this.queue.push(item);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async drainQueue(): Promise<void> {
|
|
42
|
+
let iterations = 0;
|
|
43
|
+
while (this.queue.length > 0 && iterations < this.drainLimit) {
|
|
44
|
+
const item = this.queue.shift()!;
|
|
45
|
+
const proc = this.queues[item.queue];
|
|
46
|
+
if (!proc) {
|
|
47
|
+
writeLog(`No processor for queue '${item.queue}', dropping item`, 'warn');
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
const ctx = new Context(item.id, item.src, this.memory, this);
|
|
51
|
+
try {
|
|
52
|
+
await proc(item.v, this.settings, ctx);
|
|
53
|
+
} catch (e) {
|
|
54
|
+
ctx.error(item.v, e);
|
|
55
|
+
}
|
|
56
|
+
iterations++;
|
|
57
|
+
}
|
|
58
|
+
if (iterations >= this.drainLimit) {
|
|
59
|
+
writeLog(`Queue drain hit safety limit (${this.drainLimit})`, 'error');
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
async run(opts: RunOptions): Promise<void> {
|
|
64
|
+
this.input = opts.input;
|
|
65
|
+
this.queues = opts.queues ?? {};
|
|
66
|
+
if (opts.maxDrainIterations) this.drainLimit = opts.maxDrainIterations;
|
|
67
|
+
|
|
68
|
+
this.settings = parseSettings();
|
|
69
|
+
|
|
70
|
+
// SIGTERM / SIGINT → graceful shutdown (stop reading, drain, exit)
|
|
71
|
+
const onSignal = () => {
|
|
72
|
+
this.shutdown = true;
|
|
73
|
+
};
|
|
74
|
+
process.on('SIGTERM', onSignal);
|
|
75
|
+
process.on('SIGINT', onSignal);
|
|
76
|
+
|
|
77
|
+
if (opts.onStart) await opts.onStart(this.settings);
|
|
78
|
+
|
|
79
|
+
for await (const line of readStdinLines()) {
|
|
80
|
+
if (this.shutdown) break;
|
|
81
|
+
|
|
82
|
+
const env = parseEnvelope(line);
|
|
83
|
+
if (env === null) continue;
|
|
84
|
+
if (env.t !== 'd') continue;
|
|
85
|
+
|
|
86
|
+
const d = env as DataEnvelope;
|
|
87
|
+
const ctx = new Context(d.id ?? '', d.src ?? '', this.memory, this);
|
|
88
|
+
try {
|
|
89
|
+
await this.input(d.v, this.settings, ctx);
|
|
90
|
+
} catch (e) {
|
|
91
|
+
ctx.error(d.v, e);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
if (this.queue.length > 0) await this.drainQueue();
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
if (this.queue.length > 0) await this.drainQueue();
|
|
98
|
+
if (opts.onShutdown) await opts.onShutdown();
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export async function run(opts: RunOptions): Promise<void> {
|
|
103
|
+
const rt = new Runtime();
|
|
104
|
+
await rt.run(opts);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function parseSettings(): JSONValue {
|
|
108
|
+
if (process.argv.length < 3) return {};
|
|
109
|
+
try {
|
|
110
|
+
return JSON.parse(process.argv[2]!) as JSONValue;
|
|
111
|
+
} catch {
|
|
112
|
+
writeLog('Failed to parse settings from argv[1], using empty object', 'warn');
|
|
113
|
+
return {};
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/** Async iterator over stdin lines (splits on \n, strips \r). */
|
|
118
|
+
async function* readStdinLines(): AsyncGenerator<string> {
|
|
119
|
+
let buffer = '';
|
|
120
|
+
const decoder = new TextDecoder('utf-8');
|
|
121
|
+
for await (const chunk of process.stdin as AsyncIterable<Buffer>) {
|
|
122
|
+
buffer += decoder.decode(chunk, { stream: true });
|
|
123
|
+
while (true) {
|
|
124
|
+
const idx = buffer.indexOf('\n');
|
|
125
|
+
if (idx < 0) break;
|
|
126
|
+
let line = buffer.slice(0, idx);
|
|
127
|
+
buffer = buffer.slice(idx + 1);
|
|
128
|
+
if (line.endsWith('\r')) line = line.slice(0, -1);
|
|
129
|
+
yield line;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
// flush decoder and any residual partial line
|
|
133
|
+
buffer += decoder.decode();
|
|
134
|
+
if (buffer.length > 0) yield buffer;
|
|
135
|
+
}
|