@combycode/dpe-framework-ts 2.0.1 → 2.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -90,9 +90,11 @@ dpe-dev scaffold --name my-tool --runtime bun --out ./my-tool --description "wha
90
90
  cd my-tool
91
91
  dpe-dev build . # bun install
92
92
  dpe-dev test . # bun test
93
- dpe-dev verify . # spawn `bun src/main.ts`, feed input.ndjson, diff stdout vs expected.ndjson
94
93
  ```
95
94
 
95
+ End-to-end stage testing happens via `dpe test <pipeline>:<stage>` once the
96
+ tool is wired into a pipeline — see `docs/testing.md`.
97
+
96
98
  ## Repo & licence
97
99
 
98
100
  - Source: <https://github.com/combycode/dpe-framework-ts>
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@combycode/dpe-framework-ts",
3
- "version": "2.0.1",
3
+ "version": "2.0.3",
4
4
  "description": "DPE framework for building streaming pipeline tools in TypeScript/Bun",
5
5
  "license": "AGPL-3.0-or-later",
6
6
  "repository": {
package/src/context.ts CHANGED
@@ -26,6 +26,7 @@ import {
26
26
  writeStats,
27
27
  writeTrace,
28
28
  } from './envelope';
29
+ import { EnvPaths } from './paths';
29
30
 
30
31
  export interface QueueItem {
31
32
  queue: string;
@@ -45,15 +46,17 @@ export class Context {
45
46
  readonly src: string;
46
47
  readonly memory: Memory;
47
48
  private readonly _runtime: RuntimeLike;
49
+ private readonly _paths: EnvPaths;
48
50
  /** Labels accumulated by ctx.trace(k, v). Flushed (merged) as one
49
51
  * {type:"trace"} stderr event before each ctx.output(), then cleared. */
50
52
  private _labels: Record<string, JSONValue> = {};
51
53
 
52
- constructor(id: string, src: string, memory: Memory, runtime: RuntimeLike) {
54
+ constructor(id: string, src: string, memory: Memory, runtime: RuntimeLike, paths?: EnvPaths) {
53
55
  this.id = id;
54
56
  this.src = src;
55
57
  this.memory = memory;
56
58
  this._runtime = runtime;
59
+ this._paths = paths ?? new EnvPaths();
57
60
  }
58
61
 
59
62
  /** Accumulate a label on this invocation's next output envelope. */
@@ -72,10 +75,11 @@ export class Context {
72
75
  const outId = opts.id ?? this.id;
73
76
  const outSrc = opts.src ?? this.src;
74
77
  // Emit merged trace first (always, even with empty labels), then data, then clear.
75
- // channel:"data" → runner counts as rows_out.
78
+ // channel:"data" -> runner counts as rows_out.
76
79
  writeTrace(outId, outSrc, this._labels, 'data');
77
80
  this._labels = {};
78
- writeData(v, outId, outSrc);
81
+ // Tokenize absolute paths back to $token form before writing to stdout.
82
+ writeData(this._paths.tokenizeValue(v), outId, outSrc);
79
83
  }
80
84
 
81
85
  emit(queue: string, v: JSONValue, opts: { id?: string; src?: string } = {}): void {
@@ -94,8 +98,9 @@ export class Context {
94
98
  meta(v: JSONValue): void {
95
99
  // Emit a meta-channel trace before the meta envelope so the runner
96
100
  // can increment its per-stage `meta` counter. Inherits ctx id/src.
101
+ // Tokenize absolute paths back to $token form.
97
102
  writeTrace(this.id, this.src, {}, 'meta');
98
- writeMeta(v);
103
+ writeMeta(this._paths.tokenizeValue(v));
99
104
  }
100
105
 
101
106
  log(msg: string, opts: { level?: string } & Record<string, JSONValue> = {}): void {
package/src/index.ts CHANGED
@@ -25,5 +25,6 @@ export {
25
25
  writeLog,
26
26
  writeMeta,
27
27
  } from './envelope';
28
+ export { EnvPaths } from './paths';
28
29
  export type { Processor, RunOptions } from './runtime';
29
30
  export { run } from './runtime';
package/src/paths.ts ADDED
@@ -0,0 +1,114 @@
1
+ /**
2
+ * EnvPaths -- resolves $token path prefixes in envelope v on input;
3
+ * reverse-tokenizes absolute paths back to $token form on output.
4
+ *
5
+ * Input side (runtime, before processor): $token/subpath -> absolute path
6
+ * Output side (context.output / context.meta): absolute path -> $token/subpath
7
+ *
8
+ * Both sides are a no-op when the DPE_* env vars are not set.
9
+ */
10
+
11
+ import type { JSONValue } from './envelope';
12
+
13
+ const TOKEN_MAP: [string, string][] = [
14
+ ['input', 'DPE_INPUT'],
15
+ ['output', 'DPE_OUTPUT'],
16
+ ['configs', 'DPE_CONFIGS'],
17
+ ['storage', 'DPE_STORAGE'],
18
+ ['temp', 'DPE_TEMP'],
19
+ ['session', 'DPE_SESSION'],
20
+ ];
21
+
22
+ interface Entry {
23
+ token: string;
24
+ abs: string;
25
+ }
26
+
27
+ /** Resolved env prefix map. Build once at tool startup; cheap to clone. */
28
+ export class EnvPaths {
29
+ private entries: Entry[];
30
+
31
+ /**
32
+ * If `pairs` is provided, use those (token, abs_path) pairs directly.
33
+ * Otherwise read the standard DPE_* env vars from process.env.
34
+ *
35
+ * Prefer the named factories `EnvPaths.fromEnv()` and
36
+ * `EnvPaths.fromPairs(pairs)` -- they're consistent with the Rust SDK
37
+ * (`from_env()` / `from_pairs(&[...])`) and read more clearly at call
38
+ * sites. The polymorphic constructor is kept for backwards compat.
39
+ */
40
+ constructor(pairs?: [string, string][]) {
41
+ const raw: Entry[] =
42
+ pairs != null
43
+ ? pairs.map(([token, abs]) => ({ token, abs: abs.replace(/\\/g, '/') }))
44
+ : TOKEN_MAP.flatMap(([token, envVar]) => {
45
+ const val = process.env[envVar];
46
+ return val ? [{ token, abs: val.replace(/\\/g, '/') }] : [];
47
+ });
48
+
49
+ // Sort longest-abs-first for greedy tokenization.
50
+ this.entries = raw.sort((a, b) => b.abs.length - a.abs.length);
51
+ }
52
+
53
+ /** Read the standard DPE_* env vars; missing/empty vars are skipped. */
54
+ static fromEnv(): EnvPaths {
55
+ return new EnvPaths();
56
+ }
57
+
58
+ /** Construct from explicit (token, abs_path) pairs -- useful for tests. */
59
+ static fromPairs(pairs: [string, string][]): EnvPaths {
60
+ return new EnvPaths(pairs);
61
+ }
62
+
63
+ isEmpty(): boolean {
64
+ return this.entries.length === 0;
65
+ }
66
+
67
+ /** Walk v, resolving $token[/subpath] strings to absolute paths. */
68
+ resolveValue(v: JSONValue): JSONValue {
69
+ if (this.isEmpty()) return v;
70
+ return this.walk(v, (s) => this.resolveStr(s));
71
+ }
72
+
73
+ /** Walk v, tokenizing absolute paths back to $token[/subpath] form. */
74
+ tokenizeValue(v: JSONValue): JSONValue {
75
+ if (this.isEmpty()) return v;
76
+ return this.walk(v, (s) => this.tokenizeStr(s));
77
+ }
78
+
79
+ private walk(v: JSONValue, f: (s: string) => string): JSONValue {
80
+ if (typeof v === 'string') return f(v);
81
+ if (Array.isArray(v)) return v.map((x) => this.walk(x, f));
82
+ if (v !== null && typeof v === 'object') {
83
+ const out: Record<string, JSONValue> = {};
84
+ for (const [k, val] of Object.entries(v)) {
85
+ out[k] = this.walk(val as JSONValue, f);
86
+ }
87
+ return out;
88
+ }
89
+ return v;
90
+ }
91
+
92
+ private resolveStr(s: string): string {
93
+ if (!s.startsWith('$')) return s;
94
+ const slashIdx = s.indexOf('/');
95
+ const [name, tail] =
96
+ slashIdx < 0 ? [s.slice(1), ''] : [s.slice(1, slashIdx), s.slice(slashIdx)];
97
+ const entry = this.entries.find((e) => e.token === name);
98
+ if (!entry) return s;
99
+ return tail ? entry.abs + tail : entry.abs;
100
+ }
101
+
102
+ private tokenizeStr(s: string): string {
103
+ const normalized = s.replace(/\\/g, '/');
104
+ for (const entry of this.entries) {
105
+ if (normalized.startsWith(entry.abs)) {
106
+ const rest = normalized.slice(entry.abs.length);
107
+ if (!rest) return `$${entry.token}`;
108
+ if (rest.startsWith('/')) return `$${entry.token}${rest}`;
109
+ // rest doesn't start with '/' -- partial component match, skip
110
+ }
111
+ }
112
+ return s;
113
+ }
114
+ }
package/src/runtime.ts CHANGED
@@ -9,6 +9,7 @@
9
9
  import { Context, Memory, type QueueItem, type RuntimeLike } from './context';
10
10
  import type { DataEnvelope, JSONValue } from './envelope';
11
11
  import { parseEnvelope, writeInput, writeLog } from './envelope';
12
+ import { EnvPaths } from './paths';
12
13
 
13
14
  export type Processor = (v: JSONValue, settings: JSONValue, ctx: Context) => void | Promise<void>;
14
15
 
@@ -33,6 +34,7 @@ class Runtime implements RuntimeLike {
33
34
  private queue: QueueItem[] = [];
34
35
  private shutdown = false;
35
36
  private drainLimit = 100_000;
37
+ private paths = new EnvPaths();
36
38
 
37
39
  enqueue(item: QueueItem): void {
38
40
  this.queue.push(item);
@@ -47,7 +49,7 @@ class Runtime implements RuntimeLike {
47
49
  writeLog(`No processor for queue '${item.queue}', dropping item`, 'warn');
48
50
  continue;
49
51
  }
50
- const ctx = new Context(item.id, item.src, this.memory, this);
52
+ const ctx = new Context(item.id, item.src, this.memory, this, this.paths);
51
53
  try {
52
54
  await proc(item.v, this.settings, ctx);
53
55
  } catch (e) {
@@ -98,9 +100,11 @@ class Runtime implements RuntimeLike {
98
100
  // can count rows_in for every stage that reads stdin — including
99
101
  // pass-through tools and terminal sinks that never call ctx.output().
100
102
  writeInput(d.id ?? '', d.src ?? '');
101
- const ctx = new Context(d.id ?? '', d.src ?? '', this.memory, this);
103
+ // Resolve $token paths in v before handing to processor.
104
+ const resolvedV = this.paths.resolveValue(d.v);
105
+ const ctx = new Context(d.id ?? '', d.src ?? '', this.memory, this, this.paths);
102
106
  try {
103
- await this.input(d.v, this.settings, ctx);
107
+ await this.input(resolvedV, this.settings, ctx);
104
108
  } catch (e) {
105
109
  ctx.error(d.v, e);
106
110
  }
@@ -110,6 +114,19 @@ class Runtime implements RuntimeLike {
110
114
 
111
115
  if (this.queue.length > 0) await this.drainQueue();
112
116
  if (opts.onShutdown) await opts.onShutdown();
117
+
118
+ // Workaround for Bun's exit-time stdout race that drops tail-end
119
+ // envelopes when ctx.output runs back-to-back milliseconds before
120
+ // stdin EOF. The bare `process.stdout.write('', cb)` callback fires
121
+ // before the OS pipe actually settles (observed: callback alone
122
+ // reduced the bug from ~30% to ~20%; adding a 50ms grace eliminates
123
+ // it). Issue 0045 — recon-extract-col-meta → recon-normalize-cols
124
+ // hand-off losing 1 of N envelopes. Same family as inbox 0041.
125
+ await new Promise<void>((resolve) => {
126
+ process.stdout.write('', () => {
127
+ setTimeout(resolve, 50);
128
+ });
129
+ });
113
130
  }
114
131
  }
115
132