@combycode/dpe-framework-ts 2.0.0 → 2.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -90,9 +90,11 @@ dpe-dev scaffold --name my-tool --runtime bun --out ./my-tool --description "wha
90
90
  cd my-tool
91
91
  dpe-dev build . # bun install
92
92
  dpe-dev test . # bun test
93
- dpe-dev verify . # spawn `bun src/main.ts`, feed input.ndjson, diff stdout vs expected.ndjson
94
93
  ```
95
94
 
95
+ End-to-end stage testing happens via `dpe test <pipeline>:<stage>` once the
96
+ tool is wired into a pipeline — see `docs/testing.md`.
97
+
96
98
  ## Repo & licence
97
99
 
98
100
  - Source: <https://github.com/combycode/dpe-framework-ts>
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@combycode/dpe-framework-ts",
3
- "version": "2.0.0",
3
+ "version": "2.0.2",
4
4
  "description": "DPE framework for building streaming pipeline tools in TypeScript/Bun",
5
5
  "license": "AGPL-3.0-or-later",
6
6
  "repository": {
package/src/cache.ts ADDED
@@ -0,0 +1,100 @@
1
+ // Cache helper used by `ctx.cached(...)`. Honors the runner's
2
+ // DPE_CACHE_MODE env (use / refresh / bypass / off) and stores
3
+ // JSON-serialized values at $DPE_STORAGE/<namespace>/<key>.json.
4
+ //
5
+ // Key derivation: blake2b(canonical-JSON(key)) → 32-hex chars.
6
+ // Canonical = sorted object keys, compact, no whitespace.
7
+ //
8
+ // Failure modes (cache-disabling, NOT errors propagated to user):
9
+ // - DPE_STORAGE not set → cache disabled, every call produces
10
+ // - cache file unreadable → treat as miss, log warn
11
+ // - cache file unparseable → treat as miss, log warn
12
+ // - producer throws → re-throw to caller (no cache write)
13
+ //
14
+ // All cache I/O is sync — these are short JSON files, async wouldn't
15
+ // buy anything and would complicate the call site.
16
+
17
+ import { createHash } from 'node:crypto';
18
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
19
+ import { dirname, join } from 'node:path';
20
+ import { writeLog } from './envelope';
21
+
22
+ export type CacheMode = 'use' | 'refresh' | 'bypass' | 'off';
23
+
24
+ /** Read DPE_CACHE_MODE; default to "use". Anything unrecognized → "use". */
25
+ export function readCacheMode(): CacheMode {
26
+ const v = process.env.DPE_CACHE_MODE ?? 'use';
27
+ if (v === 'use' || v === 'refresh' || v === 'bypass' || v === 'off') return v;
28
+ return 'use';
29
+ }
30
+
31
+ /** Compute the on-disk path for a (namespace, key) pair. Returns null
32
+ * when DPE_STORAGE isn't set — caller treats that as cache disabled. */
33
+ export function cachePath(namespace: string, key: unknown): string | null {
34
+ const storage = process.env.DPE_STORAGE;
35
+ if (!storage) return null;
36
+ const k = canonicalJson(key);
37
+ const hash = createHash('blake2b512').update(k, 'utf8').digest('hex').slice(0, 32);
38
+ return join(storage, namespace, `${hash}.json`);
39
+ }
40
+
41
+ /** Helper used by Context.cached(). Implements the four-mode cache
42
+ * semantics around the produce callback. */
43
+ export async function cachedImpl<T>(
44
+ namespace: string,
45
+ key: unknown,
46
+ produce: () => T | Promise<T>,
47
+ ): Promise<T> {
48
+ const mode = readCacheMode();
49
+ const path = cachePath(namespace, key);
50
+
51
+ // Mode "off"/"bypass" or no $storage → just produce, no cache I/O.
52
+ const canRead = path !== null && (mode === 'use' || mode === 'refresh');
53
+ const canWrite = path !== null && mode !== 'bypass' && mode !== 'off';
54
+ const willRead = canRead && mode !== 'refresh';
55
+
56
+ if (willRead && existsSync(path!)) {
57
+ try {
58
+ const raw = readFileSync(path!, 'utf8');
59
+ const value = JSON.parse(raw) as T;
60
+ writeLog(`cached: hit (${namespace})`, 'debug');
61
+ return value;
62
+ } catch (e) {
63
+ writeLog(
64
+ `cached: read failed (${namespace}) — ${e instanceof Error ? e.message : String(e)}`,
65
+ 'warn',
66
+ );
67
+ // Fall through to produce.
68
+ }
69
+ }
70
+
71
+ const result = await produce();
72
+
73
+ if (canWrite && path !== null) {
74
+ try {
75
+ mkdirSync(dirname(path), { recursive: true });
76
+ writeFileSync(path, JSON.stringify(result));
77
+ } catch (e) {
78
+ writeLog(
79
+ `cached: write failed (${namespace}) — ${e instanceof Error ? e.message : String(e)}`,
80
+ 'warn',
81
+ );
82
+ // Don't fail the caller — they got their value.
83
+ }
84
+ }
85
+
86
+ return result;
87
+ }
88
+
89
+ /** Canonical JSON: stable across runs. Sorted object keys, compact,
90
+ * no whitespace. Same logic as envelope.canonicalJson but exposed
91
+ * here so cache is independent of envelope-id concerns. */
92
+ function canonicalJson(v: unknown): string {
93
+ if (v === null || typeof v !== 'object') return JSON.stringify(v);
94
+ if (Array.isArray(v)) return `[${(v as unknown[]).map(canonicalJson).join(',')}]`;
95
+ const keys = Object.keys(v as Record<string, unknown>).sort();
96
+ const parts = keys.map(
97
+ (k) => `${JSON.stringify(k)}:${canonicalJson((v as Record<string, unknown>)[k])}`,
98
+ );
99
+ return `{${parts.join(',')}}`;
100
+ }
package/src/context.ts CHANGED
@@ -14,6 +14,7 @@
14
14
  * ctx.hashFile(path)
15
15
  */
16
16
 
17
+ import { cachedImpl } from './cache';
17
18
  import type { JSONValue } from './envelope';
18
19
  import {
19
20
  hashFile,
@@ -25,6 +26,7 @@ import {
25
26
  writeStats,
26
27
  writeTrace,
27
28
  } from './envelope';
29
+ import { EnvPaths } from './paths';
28
30
 
29
31
  export interface QueueItem {
30
32
  queue: string;
@@ -44,15 +46,17 @@ export class Context {
44
46
  readonly src: string;
45
47
  readonly memory: Memory;
46
48
  private readonly _runtime: RuntimeLike;
49
+ private readonly _paths: EnvPaths;
47
50
  /** Labels accumulated by ctx.trace(k, v). Flushed (merged) as one
48
51
  * {type:"trace"} stderr event before each ctx.output(), then cleared. */
49
52
  private _labels: Record<string, JSONValue> = {};
50
53
 
51
- constructor(id: string, src: string, memory: Memory, runtime: RuntimeLike) {
54
+ constructor(id: string, src: string, memory: Memory, runtime: RuntimeLike, paths?: EnvPaths) {
52
55
  this.id = id;
53
56
  this.src = src;
54
57
  this.memory = memory;
55
58
  this._runtime = runtime;
59
+ this._paths = paths ?? new EnvPaths();
56
60
  }
57
61
 
58
62
  /** Accumulate a label on this invocation's next output envelope. */
@@ -71,9 +75,11 @@ export class Context {
71
75
  const outId = opts.id ?? this.id;
72
76
  const outSrc = opts.src ?? this.src;
73
77
  // Emit merged trace first (always, even with empty labels), then data, then clear.
74
- writeTrace(outId, outSrc, this._labels);
78
+ // channel:"data" -> runner counts as rows_out.
79
+ writeTrace(outId, outSrc, this._labels, 'data');
75
80
  this._labels = {};
76
- writeData(v, outId, outSrc);
81
+ // Tokenize absolute paths back to $token form before writing to stdout.
82
+ writeData(this._paths.tokenizeValue(v), outId, outSrc);
77
83
  }
78
84
 
79
85
  emit(queue: string, v: JSONValue, opts: { id?: string; src?: string } = {}): void {
@@ -90,7 +96,11 @@ export class Context {
90
96
  }
91
97
 
92
98
  meta(v: JSONValue): void {
93
- writeMeta(v);
99
+ // Emit a meta-channel trace before the meta envelope so the runner
100
+ // can increment its per-stage `meta` counter. Inherits ctx id/src.
101
+ // Tokenize absolute paths back to $token form.
102
+ writeTrace(this.id, this.src, {}, 'meta');
103
+ writeMeta(this._paths.tokenizeValue(v));
94
104
  }
95
105
 
96
106
  log(msg: string, opts: { level?: string } & Record<string, JSONValue> = {}): void {
@@ -109,6 +119,42 @@ export class Context {
109
119
  hashFile(filepath: string): Promise<string | undefined> {
110
120
  return hashFile(filepath);
111
121
  }
122
+
123
+ /**
124
+ * Cache the result of an expensive computation under
125
+ * `$DPE_STORAGE/<namespace>/<hash>.json`. Honors the runner's
126
+ * `DPE_CACHE_MODE` env (use / refresh / bypass / off):
127
+ * - use (default) — read cache if present, else produce + write
128
+ * - refresh — always produce, overwrite cache
129
+ * - bypass — produce, skip both read and write
130
+ * - off — same as bypass
131
+ *
132
+ * `key` is canonical-JSON-hashed (blake2b-256, first 32 hex chars).
133
+ * Compose it from whatever determines output equivalence: settings
134
+ * subset, content hash from `ctx.hashFile(...)`, page index, model
135
+ * id — anything that, if changed, should produce a different result.
136
+ *
137
+ * If `$DPE_STORAGE` isn't set (e.g. tool invoked outside a pipeline),
138
+ * cache is silently disabled — every call goes through `produce`.
139
+ *
140
+ * Returns the producer's result either way (cache hit or fresh).
141
+ * Producer errors propagate; failed runs do NOT poison the cache.
142
+ *
143
+ * @example
144
+ * const result = await ctx.cached(
145
+ * "doc-converter",
146
+ * { fileHash: await ctx.hashFile(v.path), settings, page },
147
+ * () => provider.convertPage(...),
148
+ * );
149
+ * ctx.output(result);
150
+ */
151
+ cached<T extends JSONValue>(
152
+ namespace: string,
153
+ key: unknown,
154
+ produce: () => T | Promise<T>,
155
+ ): Promise<T> {
156
+ return cachedImpl(namespace, key, produce);
157
+ }
112
158
  }
113
159
 
114
160
  /**
package/src/envelope.ts CHANGED
@@ -101,9 +101,29 @@ export function writeError(v: JSONValue, err: unknown, id: string, src: string):
101
101
  });
102
102
  }
103
103
 
104
- /** Merged trace event flushed once per output envelope (even with empty labels). */
105
- export function writeTrace(id: string, src: string, labels: Record<string, JSONValue>): void {
106
- _writeStderr({ type: 'trace', id, src, labels });
104
+ /** Merged trace event flushed once per output envelope (even with empty labels).
105
+ *
106
+ * `channel` distinguishes counter buckets at the runner side:
107
+ * - `"data"` from ctx.output → rows_out
108
+ * - `"meta"` from ctx.meta → meta
109
+ * Omit the field for back-compat (older runner treats unset as data).
110
+ */
111
+ export function writeTrace(
112
+ id: string,
113
+ src: string,
114
+ labels: Record<string, JSONValue>,
115
+ channel?: 'data' | 'meta',
116
+ ): void {
117
+ const rec: Record<string, JSONValue> = { type: 'trace', id, src, labels };
118
+ if (channel) rec.channel = channel;
119
+ _writeStderr(rec);
120
+ }
121
+
122
+ /** Input event — emitted by the runtime per envelope read from stdin,
123
+ * BEFORE the processor runs. Lets the runner count rows_in for stages
124
+ * that never call ctx.output (pass-through, terminal sinks). */
125
+ export function writeInput(id: string, src: string): void {
126
+ _writeStderr({ type: 'input', id, src });
107
127
  }
108
128
 
109
129
  /** Custom stats event. Shape: {type:"stats", ...data}. */
package/src/index.ts CHANGED
@@ -11,6 +11,7 @@
11
11
  * });
12
12
  */
13
13
 
14
+ export type { CacheMode } from './cache';
14
15
  export type { QueueItem, RuntimeLike } from './context';
15
16
  export { Context, Memory } from './context';
16
17
  export type { DataEnvelope, Envelope, JSONValue, MetaEnvelope } from './envelope';
@@ -24,5 +25,6 @@ export {
24
25
  writeLog,
25
26
  writeMeta,
26
27
  } from './envelope';
28
+ export { EnvPaths } from './paths';
27
29
  export type { Processor, RunOptions } from './runtime';
28
30
  export { run } from './runtime';
package/src/paths.ts ADDED
@@ -0,0 +1,114 @@
1
+ /**
2
+ * EnvPaths -- resolves $token path prefixes in envelope v on input;
3
+ * reverse-tokenizes absolute paths back to $token form on output.
4
+ *
5
+ * Input side (runtime, before processor): $token/subpath -> absolute path
6
+ * Output side (context.output / context.meta): absolute path -> $token/subpath
7
+ *
8
+ * Both sides are a no-op when the DPE_* env vars are not set.
9
+ */
10
+
11
+ import type { JSONValue } from './envelope';
12
+
13
+ const TOKEN_MAP: [string, string][] = [
14
+ ['input', 'DPE_INPUT'],
15
+ ['output', 'DPE_OUTPUT'],
16
+ ['configs', 'DPE_CONFIGS'],
17
+ ['storage', 'DPE_STORAGE'],
18
+ ['temp', 'DPE_TEMP'],
19
+ ['session', 'DPE_SESSION'],
20
+ ];
21
+
22
+ interface Entry {
23
+ token: string;
24
+ abs: string;
25
+ }
26
+
27
+ /** Resolved env prefix map. Build once at tool startup; cheap to clone. */
28
+ export class EnvPaths {
29
+ private entries: Entry[];
30
+
31
+ /**
32
+ * If `pairs` is provided, use those (token, abs_path) pairs directly.
33
+ * Otherwise read the standard DPE_* env vars from process.env.
34
+ *
35
+ * Prefer the named factories `EnvPaths.fromEnv()` and
36
+ * `EnvPaths.fromPairs(pairs)` -- they're consistent with the Rust SDK
37
+ * (`from_env()` / `from_pairs(&[...])`) and read more clearly at call
38
+ * sites. The polymorphic constructor is kept for backwards compat.
39
+ */
40
+ constructor(pairs?: [string, string][]) {
41
+ const raw: Entry[] =
42
+ pairs != null
43
+ ? pairs.map(([token, abs]) => ({ token, abs: abs.replace(/\\/g, '/') }))
44
+ : TOKEN_MAP.flatMap(([token, envVar]) => {
45
+ const val = process.env[envVar];
46
+ return val ? [{ token, abs: val.replace(/\\/g, '/') }] : [];
47
+ });
48
+
49
+ // Sort longest-abs-first for greedy tokenization.
50
+ this.entries = raw.sort((a, b) => b.abs.length - a.abs.length);
51
+ }
52
+
53
+ /** Read the standard DPE_* env vars; missing/empty vars are skipped. */
54
+ static fromEnv(): EnvPaths {
55
+ return new EnvPaths();
56
+ }
57
+
58
+ /** Construct from explicit (token, abs_path) pairs -- useful for tests. */
59
+ static fromPairs(pairs: [string, string][]): EnvPaths {
60
+ return new EnvPaths(pairs);
61
+ }
62
+
63
+ isEmpty(): boolean {
64
+ return this.entries.length === 0;
65
+ }
66
+
67
+ /** Walk v, resolving $token[/subpath] strings to absolute paths. */
68
+ resolveValue(v: JSONValue): JSONValue {
69
+ if (this.isEmpty()) return v;
70
+ return this.walk(v, (s) => this.resolveStr(s));
71
+ }
72
+
73
+ /** Walk v, tokenizing absolute paths back to $token[/subpath] form. */
74
+ tokenizeValue(v: JSONValue): JSONValue {
75
+ if (this.isEmpty()) return v;
76
+ return this.walk(v, (s) => this.tokenizeStr(s));
77
+ }
78
+
79
+ private walk(v: JSONValue, f: (s: string) => string): JSONValue {
80
+ if (typeof v === 'string') return f(v);
81
+ if (Array.isArray(v)) return v.map((x) => this.walk(x, f));
82
+ if (v !== null && typeof v === 'object') {
83
+ const out: Record<string, JSONValue> = {};
84
+ for (const [k, val] of Object.entries(v)) {
85
+ out[k] = this.walk(val as JSONValue, f);
86
+ }
87
+ return out;
88
+ }
89
+ return v;
90
+ }
91
+
92
+ private resolveStr(s: string): string {
93
+ if (!s.startsWith('$')) return s;
94
+ const slashIdx = s.indexOf('/');
95
+ const [name, tail] =
96
+ slashIdx < 0 ? [s.slice(1), ''] : [s.slice(1, slashIdx), s.slice(slashIdx)];
97
+ const entry = this.entries.find((e) => e.token === name);
98
+ if (!entry) return s;
99
+ return tail ? entry.abs + tail : entry.abs;
100
+ }
101
+
102
+ private tokenizeStr(s: string): string {
103
+ const normalized = s.replace(/\\/g, '/');
104
+ for (const entry of this.entries) {
105
+ if (normalized.startsWith(entry.abs)) {
106
+ const rest = normalized.slice(entry.abs.length);
107
+ if (!rest) return `$${entry.token}`;
108
+ if (rest.startsWith('/')) return `$${entry.token}${rest}`;
109
+ // rest doesn't start with '/' -- partial component match, skip
110
+ }
111
+ }
112
+ return s;
113
+ }
114
+ }
package/src/runtime.ts CHANGED
@@ -8,7 +8,8 @@
8
8
 
9
9
  import { Context, Memory, type QueueItem, type RuntimeLike } from './context';
10
10
  import type { DataEnvelope, JSONValue } from './envelope';
11
- import { parseEnvelope, writeLog } from './envelope';
11
+ import { parseEnvelope, writeInput, writeLog } from './envelope';
12
+ import { EnvPaths } from './paths';
12
13
 
13
14
  export type Processor = (v: JSONValue, settings: JSONValue, ctx: Context) => void | Promise<void>;
14
15
 
@@ -33,6 +34,7 @@ class Runtime implements RuntimeLike {
33
34
  private queue: QueueItem[] = [];
34
35
  private shutdown = false;
35
36
  private drainLimit = 100_000;
37
+ private paths = new EnvPaths();
36
38
 
37
39
  enqueue(item: QueueItem): void {
38
40
  this.queue.push(item);
@@ -47,7 +49,7 @@ class Runtime implements RuntimeLike {
47
49
  writeLog(`No processor for queue '${item.queue}', dropping item`, 'warn');
48
50
  continue;
49
51
  }
50
- const ctx = new Context(item.id, item.src, this.memory, this);
52
+ const ctx = new Context(item.id, item.src, this.memory, this, this.paths);
51
53
  try {
52
54
  await proc(item.v, this.settings, ctx);
53
55
  } catch (e) {
@@ -66,6 +68,13 @@ class Runtime implements RuntimeLike {
66
68
  if (opts.maxDrainIterations) this.drainLimit = opts.maxDrainIterations;
67
69
 
68
70
  this.settings = parseSettings();
71
+ // `accept_meta: true` in settings → the read loop dispatches t:"m"
72
+ // envelopes to the input processor alongside t:"d" (instead of
73
+ // skipping). Default false preserves historical behavior — meta
74
+ // envelopes only matter to tools that explicitly opt in (typically
75
+ // sinks like write-file-stream when used as a meta-output target).
76
+ const s = this.settings as { accept_meta?: unknown } | null;
77
+ const acceptMeta = !!(s && typeof s === 'object' && s.accept_meta === true);
69
78
 
70
79
  // SIGTERM / SIGINT → graceful shutdown (stop reading, drain, exit)
71
80
  const onSignal = () => {
@@ -81,12 +90,21 @@ class Runtime implements RuntimeLike {
81
90
 
82
91
  const env = parseEnvelope(line);
83
92
  if (env === null) continue;
84
- if (env.t !== 'd') continue;
93
+ // Default: only data envelopes. With `accept_meta: true` the
94
+ // tool ALSO receives meta envelopes through the input
95
+ // processor. Anything else still skipped.
96
+ if (env.t !== 'd' && !(acceptMeta && env.t === 'm')) continue;
85
97
 
86
98
  const d = env as DataEnvelope;
87
- const ctx = new Context(d.id ?? '', d.src ?? '', this.memory, this);
99
+ // Emit an `input` event BEFORE the processor runs so the runner
100
+ // can count rows_in for every stage that reads stdin — including
101
+ // pass-through tools and terminal sinks that never call ctx.output().
102
+ writeInput(d.id ?? '', d.src ?? '');
103
+ // Resolve $token paths in v before handing to processor.
104
+ const resolvedV = this.paths.resolveValue(d.v);
105
+ const ctx = new Context(d.id ?? '', d.src ?? '', this.memory, this, this.paths);
88
106
  try {
89
- await this.input(d.v, this.settings, ctx);
107
+ await this.input(resolvedV, this.settings, ctx);
90
108
  } catch (e) {
91
109
  ctx.error(d.v, e);
92
110
  }