@combycode/dpe-framework-ts 0.0.0-init → 2.0.0-rc1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -1,3 +1,100 @@
1
1
  # @combycode/dpe-framework-ts
2
2
 
3
- Placeholder for the TypeScript / Bun framework. Real source ships from `frameworks/ts/` in the monorepo on the first release.
3
+ DPE framework for building streaming pipeline tools in TypeScript / Bun.
4
+
5
+ A DPE tool is an independent program: it reads NDJSON envelopes from stdin,
6
+ processes each one, and writes NDJSON envelopes to stdout. The runner
7
+ (`dpe`) spawns it, pipes data through it, and routes its output to the
8
+ next stage. The framework handles the loop, the parsing, the queues, and
9
+ the error/trace plumbing — your code only writes the transform.
10
+
11
+ Targets Bun by default. Node 20+ should also work but isn't part of CI.
12
+
13
+ ## Install
14
+
15
+ ```bash
16
+ bun add @combycode/dpe-framework-ts
17
+ ```
18
+
19
+ ## Hello tool
20
+
21
+ ```ts
22
+ // src/main.ts
23
+ import { run, type Context, type JSONValue } from "@combycode/dpe-framework-ts";
24
+
25
+ async function processInput(v: JSONValue, _settings: JSONValue, ctx: Context) {
26
+ ctx.output(v);
27
+ }
28
+
29
+ await run({ input: processInput });
30
+ ```
31
+
32
+ Run as a tool: `bun src/main.ts '{"...settings JSON..."}'`. The runner
33
+ will pass settings as `argv[1]` and pipe NDJSON through stdin/stdout.
34
+
35
+ ## Processor signature
36
+
37
+ ```ts
38
+ type Processor = (
39
+ v: JSONValue,
40
+ settings: JSONValue,
41
+ ctx: Context,
42
+ ) => void | Promise<void>;
43
+ ```
44
+
45
+ Both sync and async processors are supported. The runtime awaits each
46
+ call before reading the next stdin line, so you can `await` I/O without
47
+ losing ordering.
48
+
49
+ ## Internal queues
50
+
51
+ ```ts
52
+ import { run, type Context, type JSONValue } from "@combycode/dpe-framework-ts";
53
+
54
+ async function processInput(v: JSONValue, _s: JSONValue, ctx: Context) {
55
+ ctx.emit("validate", v);
56
+ }
57
+
58
+ async function processValidate(v: JSONValue, _s: JSONValue, ctx: Context) {
59
+ if ((v as any).ok === true) ctx.output(v);
60
+ else ctx.error(v, "validation failed");
61
+ }
62
+
63
+ await run({
64
+ input: processInput,
65
+ queues: { validate: processValidate },
66
+ });
67
+ ```
68
+
69
+ `ctx.emit(queue, v)` enqueues for the named handler. `await ctx.drain()`
70
+ flushes all queues before continuing.
71
+
72
+ ## Context API (essentials)
73
+
74
+ | Call | What it does |
75
+ |---|---|
76
+ | `ctx.output(v, id?, src?)` | Emit `{t:"d",...}` to stdout. `id`/`src` default to inherit. |
77
+ | `ctx.emit(queue, v)` | Push to internal queue. |
78
+ | `ctx.drain()` | Resolve once every queue is drained. |
79
+ | `ctx.error(v, msg)` | Write `{type:"error",input:v,error:msg,...}` to stderr. |
80
+ | `ctx.log(level, msg)` | Structured stderr log. |
81
+ | `ctx.meta(v)` | Emit `{t:"m", v}` envelope to stdout. |
82
+ | `ctx.memory` | Typed accumulators (`Number`, `Average`, `MinMax`, `Set`, `Map`, …). |
83
+
84
+ See `docs/frameworks.md` in the monorepo for the full reference.
85
+
86
+ ## Scaffolding
87
+
88
+ ```bash
89
+ dpe-dev scaffold --name my-tool --runtime bun --out ./my-tool --description "what it does"
90
+ cd my-tool
91
+ dpe-dev build . # bun install
92
+ dpe-dev test . # bun test
93
+ dpe-dev verify . # spawn `bun src/main.ts`, feed input.ndjson, diff stdout vs expected.ndjson
94
+ ```
95
+
96
+ ## Repo & licence
97
+
98
+ - Source: <https://github.com/combycode/dpe-framework-ts>
99
+ - Monorepo (canonical): <https://github.com/combycode/dpe>
100
+ - Licence: AGPL-3.0-or-later
package/package.json CHANGED
@@ -1,8 +1,38 @@
1
1
  {
2
2
  "name": "@combycode/dpe-framework-ts",
3
- "version": "0.0.0-init",
4
- "description": "DPE framework for building streaming pipeline tools in TypeScript / Bun",
3
+ "version": "2.0.0-rc1",
4
+ "description": "DPE framework for building streaming pipeline tools in TypeScript/Bun",
5
5
  "license": "AGPL-3.0-or-later",
6
- "repository": { "type": "git", "url": "https://github.com/combycode/dpe" },
7
- "homepage": "https://github.com/combycode/dpe#readme"
8
- }
6
+ "repository": {
7
+ "type": "git",
8
+ "url": "git+https://github.com/combycode/dpe.git",
9
+ "directory": "frameworks/ts"
10
+ },
11
+ "homepage": "https://github.com/combycode/dpe#readme",
12
+ "type": "module",
13
+ "main": "src/index.ts",
14
+ "module": "src/index.ts",
15
+ "types": "src/index.ts",
16
+ "exports": {
17
+ ".": {
18
+ "types": "./src/index.ts",
19
+ "default": "./src/index.ts"
20
+ }
21
+ },
22
+ "scripts": {
23
+ "test": "bun test"
24
+ },
25
+ "devDependencies": {
26
+ "@types/bun": "^1.3.13",
27
+ "ajv": "^8.17.1",
28
+ "typescript": "^5.9.0"
29
+ },
30
+ "files": [
31
+ "src/",
32
+ "README.md"
33
+ ],
34
+ "publishConfig": {
35
+ "access": "public",
36
+ "provenance": true
37
+ }
38
+ }
package/src/context.ts ADDED
@@ -0,0 +1,121 @@
1
+ /**
2
+ * Context object — passed to every processor invocation.
3
+ *
4
+ * Mirrors the Python and Rust Context API:
5
+ * ctx.output(v, {id?, src?})
6
+ * ctx.emit(queue, v, {id?, src?})
7
+ * ctx.drain()
8
+ * ctx.meta(v)
9
+ * ctx.log(msg, {level?, ...extra})
10
+ * ctx.error(v, err)
11
+ * ctx.trace(k, v)
12
+ * ctx.stats(data)
13
+ * ctx.hash(str)
14
+ * ctx.hashFile(path)
15
+ */
16
+
17
+ import type { JSONValue } from './envelope';
18
+ import {
19
+ hashFile,
20
+ hashString,
21
+ writeData,
22
+ writeError,
23
+ writeLog,
24
+ writeMeta,
25
+ writeStats,
26
+ writeTrace,
27
+ } from './envelope';
28
+
29
+ export interface QueueItem {
30
+ queue: string;
31
+ v: JSONValue;
32
+ id: string;
33
+ src: string;
34
+ }
35
+
36
+ /** Tiny runtime surface the Context needs — avoids circular imports with Runtime. */
37
+ export interface RuntimeLike {
38
+ enqueue(item: QueueItem): void;
39
+ drainQueue(): Promise<void>;
40
+ }
41
+
42
+ export class Context {
43
+ readonly id: string;
44
+ readonly src: string;
45
+ readonly memory: Memory;
46
+ private readonly _runtime: RuntimeLike;
47
+ /** Labels accumulated by ctx.trace(k, v). Flushed (merged) as one
48
+ * {type:"trace"} stderr event before each ctx.output(), then cleared. */
49
+ private _labels: Record<string, JSONValue> = {};
50
+
51
+ constructor(id: string, src: string, memory: Memory, runtime: RuntimeLike) {
52
+ this.id = id;
53
+ this.src = src;
54
+ this.memory = memory;
55
+ this._runtime = runtime;
56
+ }
57
+
58
+ /** Accumulate a label on this invocation's next output envelope. */
59
+ trace(key: string, value: JSONValue): void {
60
+ this._labels[key] = value;
61
+ }
62
+
63
+ /** Emit a stats event to stderr. Example: ctx.stats({rows:1000, rps:250}). */
64
+ stats(data: Record<string, JSONValue>): void {
65
+ if (data && Object.keys(data).length > 0) {
66
+ writeStats(data);
67
+ }
68
+ }
69
+
70
+ output(v: JSONValue, opts: { id?: string; src?: string } = {}): void {
71
+ const outId = opts.id ?? this.id;
72
+ const outSrc = opts.src ?? this.src;
73
+ // Emit merged trace first (always, even with empty labels), then data, then clear.
74
+ writeTrace(outId, outSrc, this._labels);
75
+ this._labels = {};
76
+ writeData(v, outId, outSrc);
77
+ }
78
+
79
+ emit(queue: string, v: JSONValue, opts: { id?: string; src?: string } = {}): void {
80
+ this._runtime.enqueue({
81
+ queue,
82
+ v,
83
+ id: opts.id ?? this.id,
84
+ src: opts.src ?? this.src,
85
+ });
86
+ }
87
+
88
+ async drain(): Promise<void> {
89
+ await this._runtime.drainQueue();
90
+ }
91
+
92
+ meta(v: JSONValue): void {
93
+ writeMeta(v);
94
+ }
95
+
96
+ log(msg: string, opts: { level?: string } & Record<string, JSONValue> = {}): void {
97
+ const { level = 'info', ...extra } = opts as { level?: string } & Record<string, JSONValue>;
98
+ writeLog(msg, level, Object.keys(extra).length ? extra : undefined);
99
+ }
100
+
101
+ error(v: JSONValue, err: unknown): void {
102
+ writeError(v, err, this.id, this.src);
103
+ }
104
+
105
+ hash(key: string): string {
106
+ return hashString(key);
107
+ }
108
+
109
+ hashFile(filepath: string): Promise<string | undefined> {
110
+ return hashFile(filepath);
111
+ }
112
+ }
113
+
114
+ /**
115
+ * Shared accumulator memory. Tools extend this by attaching their own
116
+ * accumulator instances as properties. Named accumulator types will ship in a
117
+ * later milestone — filter tool doesn't need them.
118
+ */
119
+ export class Memory {
120
+ [key: string]: unknown;
121
+ }
@@ -0,0 +1,131 @@
1
+ /**
2
+ * NDJSON envelope types + parse / write / hash helpers.
3
+ *
4
+ * The envelope protocol matches the Python and Rust frameworks:
5
+ * data: {"t":"d","id":"<8B hex>","src":"<string>","v":<payload>}
6
+ * meta: {"t":"m","v":<payload>}
7
+ * log: {"type":"log","level":"...","msg":"...","...":"..."} (stderr)
8
+ * error: {"type":"error","error":"...","input":..,"id":..,"src":..} (stderr)
9
+ */
10
+
11
+ import { createHash } from 'node:crypto';
12
+
13
+ export type JSONValue = null | boolean | number | string | JSONValue[] | { [k: string]: JSONValue };
14
+
15
+ export interface DataEnvelope {
16
+ t: 'd';
17
+ id: string;
18
+ src: string;
19
+ v: JSONValue;
20
+ }
21
+
22
+ export interface MetaEnvelope {
23
+ t: 'm';
24
+ v: JSONValue;
25
+ }
26
+
27
+ export type Envelope = DataEnvelope | MetaEnvelope;
28
+
29
+ /** blake2b-64 hash of a string. Returns 16-char hex. Matches Python/Rust envelope ID. */
30
+ export function hashString(key: string): string {
31
+ return createHash('blake2b512').update(key, 'utf8').digest('hex').slice(0, 16);
32
+ }
33
+
34
+ /** Stream-hash a file by path. Returns 16 hex chars. `undefined` if the file cannot be opened.
35
+ *
36
+ * Streams chunks through the hasher rather than loading the file into a single
37
+ * Uint8Array — important for large inputs (> a few hundred MB).
38
+ */
39
+ export async function hashFile(filepath: string): Promise<string | undefined> {
40
+ try {
41
+ const hasher = createHash('blake2b512');
42
+ const stream = Bun.file(filepath).stream();
43
+ for await (const chunk of stream as AsyncIterable<Uint8Array>) {
44
+ hasher.update(chunk);
45
+ }
46
+ return hasher.digest('hex').slice(0, 16);
47
+ } catch {
48
+ return undefined;
49
+ }
50
+ }
51
+
52
+ /** Deterministic id from (src | stage | sorted JSON of v). */
53
+ export function computeId(src: string, stage: string, v: JSONValue): string {
54
+ const vStr = canonicalJson(v);
55
+ return hashString(`${src}|${stage}|${vStr}`);
56
+ }
57
+
58
+ /** Parse one NDJSON line. Returns null on blank, invalid JSON, or any
59
+ * shape that isn't a well-formed envelope. Data envelopes must carry
60
+ * `id` and `src` strings; meta envelopes must carry a `v` payload. */
61
+ export function parseEnvelope(line: string): Envelope | null {
62
+ const trimmed = line.trim();
63
+ if (!trimmed) return null;
64
+ try {
65
+ const obj = JSON.parse(trimmed);
66
+ if (!obj || typeof obj !== 'object') return null;
67
+ if (obj.t === 'd') {
68
+ if (typeof obj.id !== 'string' || typeof obj.src !== 'string') return null;
69
+ if (!('v' in obj)) return null;
70
+ return obj as DataEnvelope;
71
+ }
72
+ if (obj.t === 'm') {
73
+ if (!('v' in obj)) return null;
74
+ return obj as MetaEnvelope;
75
+ }
76
+ return null;
77
+ } catch {
78
+ return null;
79
+ }
80
+ }
81
+
82
+ export function writeData(v: JSONValue, id: string, src: string): void {
83
+ _writeStdout({ t: 'd', id, src, v });
84
+ }
85
+
86
+ export function writeMeta(v: JSONValue): void {
87
+ _writeStdout({ t: 'm', v });
88
+ }
89
+
90
+ export function writeLog(msg: string, level = 'info', extra?: Record<string, JSONValue>): void {
91
+ _writeStderr({ type: 'log', level, msg, ...(extra ?? {}) });
92
+ }
93
+
94
+ export function writeError(v: JSONValue, err: unknown, id: string, src: string): void {
95
+ _writeStderr({
96
+ type: 'error',
97
+ error: err instanceof Error ? err.message : String(err),
98
+ input: v,
99
+ id,
100
+ src,
101
+ });
102
+ }
103
+
104
+ /** Merged trace event flushed once per output envelope (even with empty labels). */
105
+ export function writeTrace(id: string, src: string, labels: Record<string, JSONValue>): void {
106
+ _writeStderr({ type: 'trace', id, src, labels });
107
+ }
108
+
109
+ /** Custom stats event. Shape: {type:"stats", ...data}. */
110
+ export function writeStats(data: Record<string, JSONValue>): void {
111
+ _writeStderr({ type: 'stats', ...data });
112
+ }
113
+
114
+ /** Canonical JSON for hashing: sorted keys, compact, no spaces. */
115
+ function canonicalJson(v: JSONValue): string {
116
+ if (v === null || typeof v !== 'object') return JSON.stringify(v);
117
+ if (Array.isArray(v)) return `[${v.map(canonicalJson).join(',')}]`;
118
+ const keys = Object.keys(v).sort();
119
+ const parts = keys.map(
120
+ (k) => `${JSON.stringify(k)}:${canonicalJson((v as Record<string, JSONValue>)[k]!)}`,
121
+ );
122
+ return `{${parts.join(',')}}`;
123
+ }
124
+
125
+ function _writeStdout(obj: Record<string, JSONValue>): void {
126
+ process.stdout.write(`${JSON.stringify(obj)}\n`);
127
+ }
128
+
129
+ function _writeStderr(obj: Record<string, JSONValue>): void {
130
+ process.stderr.write(`${JSON.stringify(obj)}\n`);
131
+ }
package/src/index.ts ADDED
@@ -0,0 +1,28 @@
1
+ /**
2
+ * @combycode/dpe-framework-ts — DPE framework for Bun/TypeScript tools.
3
+ *
4
+ * Example:
5
+ * import { run } from "@combycode/dpe-framework-ts";
6
+ *
7
+ * await run({
8
+ * input: (v, settings, ctx) => {
9
+ * ctx.output({ ok: true });
10
+ * },
11
+ * });
12
+ */
13
+
14
+ export type { QueueItem, RuntimeLike } from './context';
15
+ export { Context, Memory } from './context';
16
+ export type { DataEnvelope, Envelope, JSONValue, MetaEnvelope } from './envelope';
17
+ export {
18
+ computeId,
19
+ hashFile,
20
+ hashString,
21
+ parseEnvelope,
22
+ writeData,
23
+ writeError,
24
+ writeLog,
25
+ writeMeta,
26
+ } from './envelope';
27
+ export type { Processor, RunOptions } from './runtime';
28
+ export { run } from './runtime';
package/src/runtime.ts ADDED
@@ -0,0 +1,135 @@
1
+ /**
2
+ * Runtime — stdin loop, settings parse, queue drain, processor dispatch.
3
+ *
4
+ * Tools register processors explicitly via `run({input, queues?})`.
5
+ * Unlike Python (which auto-discovers process_* functions), TS is structurally
6
+ * typed so explicit registration keeps types honest.
7
+ */
8
+
9
+ import { Context, Memory, type QueueItem, type RuntimeLike } from './context';
10
+ import type { DataEnvelope, JSONValue } from './envelope';
11
+ import { parseEnvelope, writeLog } from './envelope';
12
+
13
+ export type Processor = (v: JSONValue, settings: JSONValue, ctx: Context) => void | Promise<void>;
14
+
15
+ export interface RunOptions {
16
+ /** Main input processor — called for each data envelope on stdin. */
17
+ input: Processor;
18
+ /** Optional named queue processors — triggered by ctx.emit(queue, v). */
19
+ queues?: Record<string, Processor>;
20
+ /** Called once before the stdin loop starts (e.g. parse/compile settings). */
21
+ onStart?: (settings: JSONValue) => void | Promise<void>;
22
+ /** Called once after stdin EOF / SIGTERM (e.g. flush, emit final meta). */
23
+ onShutdown?: () => void | Promise<void>;
24
+ /** Max iterations per drain cycle — safety against infinite emit loops. */
25
+ maxDrainIterations?: number;
26
+ }
27
+
28
+ class Runtime implements RuntimeLike {
29
+ settings: JSONValue = {};
30
+ memory = new Memory();
31
+ private input!: Processor;
32
+ private queues: Record<string, Processor> = {};
33
+ private queue: QueueItem[] = [];
34
+ private shutdown = false;
35
+ private drainLimit = 100_000;
36
+
37
+ enqueue(item: QueueItem): void {
38
+ this.queue.push(item);
39
+ }
40
+
41
+ async drainQueue(): Promise<void> {
42
+ let iterations = 0;
43
+ while (this.queue.length > 0 && iterations < this.drainLimit) {
44
+ const item = this.queue.shift()!;
45
+ const proc = this.queues[item.queue];
46
+ if (!proc) {
47
+ writeLog(`No processor for queue '${item.queue}', dropping item`, 'warn');
48
+ continue;
49
+ }
50
+ const ctx = new Context(item.id, item.src, this.memory, this);
51
+ try {
52
+ await proc(item.v, this.settings, ctx);
53
+ } catch (e) {
54
+ ctx.error(item.v, e);
55
+ }
56
+ iterations++;
57
+ }
58
+ if (iterations >= this.drainLimit) {
59
+ writeLog(`Queue drain hit safety limit (${this.drainLimit})`, 'error');
60
+ }
61
+ }
62
+
63
+ async run(opts: RunOptions): Promise<void> {
64
+ this.input = opts.input;
65
+ this.queues = opts.queues ?? {};
66
+ if (opts.maxDrainIterations) this.drainLimit = opts.maxDrainIterations;
67
+
68
+ this.settings = parseSettings();
69
+
70
+ // SIGTERM / SIGINT → graceful shutdown (stop reading, drain, exit)
71
+ const onSignal = () => {
72
+ this.shutdown = true;
73
+ };
74
+ process.on('SIGTERM', onSignal);
75
+ process.on('SIGINT', onSignal);
76
+
77
+ if (opts.onStart) await opts.onStart(this.settings);
78
+
79
+ for await (const line of readStdinLines()) {
80
+ if (this.shutdown) break;
81
+
82
+ const env = parseEnvelope(line);
83
+ if (env === null) continue;
84
+ if (env.t !== 'd') continue;
85
+
86
+ const d = env as DataEnvelope;
87
+ const ctx = new Context(d.id ?? '', d.src ?? '', this.memory, this);
88
+ try {
89
+ await this.input(d.v, this.settings, ctx);
90
+ } catch (e) {
91
+ ctx.error(d.v, e);
92
+ }
93
+
94
+ if (this.queue.length > 0) await this.drainQueue();
95
+ }
96
+
97
+ if (this.queue.length > 0) await this.drainQueue();
98
+ if (opts.onShutdown) await opts.onShutdown();
99
+ }
100
+ }
101
+
102
+ export async function run(opts: RunOptions): Promise<void> {
103
+ const rt = new Runtime();
104
+ await rt.run(opts);
105
+ }
106
+
107
+ function parseSettings(): JSONValue {
108
+ if (process.argv.length < 3) return {};
109
+ try {
110
+ return JSON.parse(process.argv[2]!) as JSONValue;
111
+ } catch {
112
+ writeLog('Failed to parse settings from argv[1], using empty object', 'warn');
113
+ return {};
114
+ }
115
+ }
116
+
117
+ /** Async iterator over stdin lines (splits on \n, strips \r). */
118
+ async function* readStdinLines(): AsyncGenerator<string> {
119
+ let buffer = '';
120
+ const decoder = new TextDecoder('utf-8');
121
+ for await (const chunk of process.stdin as AsyncIterable<Buffer>) {
122
+ buffer += decoder.decode(chunk, { stream: true });
123
+ while (true) {
124
+ const idx = buffer.indexOf('\n');
125
+ if (idx < 0) break;
126
+ let line = buffer.slice(0, idx);
127
+ buffer = buffer.slice(idx + 1);
128
+ if (line.endsWith('\r')) line = line.slice(0, -1);
129
+ yield line;
130
+ }
131
+ }
132
+ // flush decoder and any residual partial line
133
+ buffer += decoder.decode();
134
+ if (buffer.length > 0) yield buffer;
135
+ }