@combycode/dpe-framework-ts 2.0.1 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/package.json +1 -1
- package/src/context.ts +9 -4
- package/src/index.ts +1 -0
- package/src/paths.ts +114 -0
- package/src/runtime.ts +20 -3
package/README.md
CHANGED
|
@@ -90,9 +90,11 @@ dpe-dev scaffold --name my-tool --runtime bun --out ./my-tool --description "wha
|
|
|
90
90
|
cd my-tool
|
|
91
91
|
dpe-dev build . # bun install
|
|
92
92
|
dpe-dev test . # bun test
|
|
93
|
-
dpe-dev verify . # spawn `bun src/main.ts`, feed input.ndjson, diff stdout vs expected.ndjson
|
|
94
93
|
```
|
|
95
94
|
|
|
95
|
+
End-to-end stage testing happens via `dpe test <pipeline>:<stage>` once the
|
|
96
|
+
tool is wired into a pipeline — see `docs/testing.md`.
|
|
97
|
+
|
|
96
98
|
## Repo & licence
|
|
97
99
|
|
|
98
100
|
- Source: <https://github.com/combycode/dpe-framework-ts>
|
package/package.json
CHANGED
package/src/context.ts
CHANGED
|
@@ -26,6 +26,7 @@ import {
|
|
|
26
26
|
writeStats,
|
|
27
27
|
writeTrace,
|
|
28
28
|
} from './envelope';
|
|
29
|
+
import { EnvPaths } from './paths';
|
|
29
30
|
|
|
30
31
|
export interface QueueItem {
|
|
31
32
|
queue: string;
|
|
@@ -45,15 +46,17 @@ export class Context {
|
|
|
45
46
|
readonly src: string;
|
|
46
47
|
readonly memory: Memory;
|
|
47
48
|
private readonly _runtime: RuntimeLike;
|
|
49
|
+
private readonly _paths: EnvPaths;
|
|
48
50
|
/** Labels accumulated by ctx.trace(k, v). Flushed (merged) as one
|
|
49
51
|
* {type:"trace"} stderr event before each ctx.output(), then cleared. */
|
|
50
52
|
private _labels: Record<string, JSONValue> = {};
|
|
51
53
|
|
|
52
|
-
constructor(id: string, src: string, memory: Memory, runtime: RuntimeLike) {
|
|
54
|
+
constructor(id: string, src: string, memory: Memory, runtime: RuntimeLike, paths?: EnvPaths) {
|
|
53
55
|
this.id = id;
|
|
54
56
|
this.src = src;
|
|
55
57
|
this.memory = memory;
|
|
56
58
|
this._runtime = runtime;
|
|
59
|
+
this._paths = paths ?? new EnvPaths();
|
|
57
60
|
}
|
|
58
61
|
|
|
59
62
|
/** Accumulate a label on this invocation's next output envelope. */
|
|
@@ -72,10 +75,11 @@ export class Context {
|
|
|
72
75
|
const outId = opts.id ?? this.id;
|
|
73
76
|
const outSrc = opts.src ?? this.src;
|
|
74
77
|
// Emit merged trace first (always, even with empty labels), then data, then clear.
|
|
75
|
-
// channel:"data"
|
|
78
|
+
// channel:"data" -> runner counts as rows_out.
|
|
76
79
|
writeTrace(outId, outSrc, this._labels, 'data');
|
|
77
80
|
this._labels = {};
|
|
78
|
-
|
|
81
|
+
// Tokenize absolute paths back to $token form before writing to stdout.
|
|
82
|
+
writeData(this._paths.tokenizeValue(v), outId, outSrc);
|
|
79
83
|
}
|
|
80
84
|
|
|
81
85
|
emit(queue: string, v: JSONValue, opts: { id?: string; src?: string } = {}): void {
|
|
@@ -94,8 +98,9 @@ export class Context {
|
|
|
94
98
|
meta(v: JSONValue): void {
|
|
95
99
|
// Emit a meta-channel trace before the meta envelope so the runner
|
|
96
100
|
// can increment its per-stage `meta` counter. Inherits ctx id/src.
|
|
101
|
+
// Tokenize absolute paths back to $token form.
|
|
97
102
|
writeTrace(this.id, this.src, {}, 'meta');
|
|
98
|
-
writeMeta(v);
|
|
103
|
+
writeMeta(this._paths.tokenizeValue(v));
|
|
99
104
|
}
|
|
100
105
|
|
|
101
106
|
log(msg: string, opts: { level?: string } & Record<string, JSONValue> = {}): void {
|
package/src/index.ts
CHANGED
package/src/paths.ts
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EnvPaths -- resolves $token path prefixes in envelope v on input;
|
|
3
|
+
* reverse-tokenizes absolute paths back to $token form on output.
|
|
4
|
+
*
|
|
5
|
+
* Input side (runtime, before processor): $token/subpath -> absolute path
|
|
6
|
+
* Output side (context.output / context.meta): absolute path -> $token/subpath
|
|
7
|
+
*
|
|
8
|
+
* Both sides are a no-op when the DPE_* env vars are not set.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { JSONValue } from './envelope';
|
|
12
|
+
|
|
13
|
+
const TOKEN_MAP: [string, string][] = [
|
|
14
|
+
['input', 'DPE_INPUT'],
|
|
15
|
+
['output', 'DPE_OUTPUT'],
|
|
16
|
+
['configs', 'DPE_CONFIGS'],
|
|
17
|
+
['storage', 'DPE_STORAGE'],
|
|
18
|
+
['temp', 'DPE_TEMP'],
|
|
19
|
+
['session', 'DPE_SESSION'],
|
|
20
|
+
];
|
|
21
|
+
|
|
22
|
+
interface Entry {
|
|
23
|
+
token: string;
|
|
24
|
+
abs: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Resolved env prefix map. Build once at tool startup; cheap to clone. */
|
|
28
|
+
export class EnvPaths {
|
|
29
|
+
private entries: Entry[];
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* If `pairs` is provided, use those (token, abs_path) pairs directly.
|
|
33
|
+
* Otherwise read the standard DPE_* env vars from process.env.
|
|
34
|
+
*
|
|
35
|
+
* Prefer the named factories `EnvPaths.fromEnv()` and
|
|
36
|
+
* `EnvPaths.fromPairs(pairs)` -- they're consistent with the Rust SDK
|
|
37
|
+
* (`from_env()` / `from_pairs(&[...])`) and read more clearly at call
|
|
38
|
+
* sites. The polymorphic constructor is kept for backwards compat.
|
|
39
|
+
*/
|
|
40
|
+
constructor(pairs?: [string, string][]) {
|
|
41
|
+
const raw: Entry[] =
|
|
42
|
+
pairs != null
|
|
43
|
+
? pairs.map(([token, abs]) => ({ token, abs: abs.replace(/\\/g, '/') }))
|
|
44
|
+
: TOKEN_MAP.flatMap(([token, envVar]) => {
|
|
45
|
+
const val = process.env[envVar];
|
|
46
|
+
return val ? [{ token, abs: val.replace(/\\/g, '/') }] : [];
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
// Sort longest-abs-first for greedy tokenization.
|
|
50
|
+
this.entries = raw.sort((a, b) => b.abs.length - a.abs.length);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Read the standard DPE_* env vars; missing/empty vars are skipped. */
|
|
54
|
+
static fromEnv(): EnvPaths {
|
|
55
|
+
return new EnvPaths();
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Construct from explicit (token, abs_path) pairs -- useful for tests. */
|
|
59
|
+
static fromPairs(pairs: [string, string][]): EnvPaths {
|
|
60
|
+
return new EnvPaths(pairs);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
isEmpty(): boolean {
|
|
64
|
+
return this.entries.length === 0;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Walk v, resolving $token[/subpath] strings to absolute paths. */
|
|
68
|
+
resolveValue(v: JSONValue): JSONValue {
|
|
69
|
+
if (this.isEmpty()) return v;
|
|
70
|
+
return this.walk(v, (s) => this.resolveStr(s));
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Walk v, tokenizing absolute paths back to $token[/subpath] form. */
|
|
74
|
+
tokenizeValue(v: JSONValue): JSONValue {
|
|
75
|
+
if (this.isEmpty()) return v;
|
|
76
|
+
return this.walk(v, (s) => this.tokenizeStr(s));
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
private walk(v: JSONValue, f: (s: string) => string): JSONValue {
|
|
80
|
+
if (typeof v === 'string') return f(v);
|
|
81
|
+
if (Array.isArray(v)) return v.map((x) => this.walk(x, f));
|
|
82
|
+
if (v !== null && typeof v === 'object') {
|
|
83
|
+
const out: Record<string, JSONValue> = {};
|
|
84
|
+
for (const [k, val] of Object.entries(v)) {
|
|
85
|
+
out[k] = this.walk(val as JSONValue, f);
|
|
86
|
+
}
|
|
87
|
+
return out;
|
|
88
|
+
}
|
|
89
|
+
return v;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
private resolveStr(s: string): string {
|
|
93
|
+
if (!s.startsWith('$')) return s;
|
|
94
|
+
const slashIdx = s.indexOf('/');
|
|
95
|
+
const [name, tail] =
|
|
96
|
+
slashIdx < 0 ? [s.slice(1), ''] : [s.slice(1, slashIdx), s.slice(slashIdx)];
|
|
97
|
+
const entry = this.entries.find((e) => e.token === name);
|
|
98
|
+
if (!entry) return s;
|
|
99
|
+
return tail ? entry.abs + tail : entry.abs;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
private tokenizeStr(s: string): string {
|
|
103
|
+
const normalized = s.replace(/\\/g, '/');
|
|
104
|
+
for (const entry of this.entries) {
|
|
105
|
+
if (normalized.startsWith(entry.abs)) {
|
|
106
|
+
const rest = normalized.slice(entry.abs.length);
|
|
107
|
+
if (!rest) return `$${entry.token}`;
|
|
108
|
+
if (rest.startsWith('/')) return `$${entry.token}${rest}`;
|
|
109
|
+
// rest doesn't start with '/' -- partial component match, skip
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return s;
|
|
113
|
+
}
|
|
114
|
+
}
|
package/src/runtime.ts
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
import { Context, Memory, type QueueItem, type RuntimeLike } from './context';
|
|
10
10
|
import type { DataEnvelope, JSONValue } from './envelope';
|
|
11
11
|
import { parseEnvelope, writeInput, writeLog } from './envelope';
|
|
12
|
+
import { EnvPaths } from './paths';
|
|
12
13
|
|
|
13
14
|
export type Processor = (v: JSONValue, settings: JSONValue, ctx: Context) => void | Promise<void>;
|
|
14
15
|
|
|
@@ -33,6 +34,7 @@ class Runtime implements RuntimeLike {
|
|
|
33
34
|
private queue: QueueItem[] = [];
|
|
34
35
|
private shutdown = false;
|
|
35
36
|
private drainLimit = 100_000;
|
|
37
|
+
private paths = new EnvPaths();
|
|
36
38
|
|
|
37
39
|
enqueue(item: QueueItem): void {
|
|
38
40
|
this.queue.push(item);
|
|
@@ -47,7 +49,7 @@ class Runtime implements RuntimeLike {
|
|
|
47
49
|
writeLog(`No processor for queue '${item.queue}', dropping item`, 'warn');
|
|
48
50
|
continue;
|
|
49
51
|
}
|
|
50
|
-
const ctx = new Context(item.id, item.src, this.memory, this);
|
|
52
|
+
const ctx = new Context(item.id, item.src, this.memory, this, this.paths);
|
|
51
53
|
try {
|
|
52
54
|
await proc(item.v, this.settings, ctx);
|
|
53
55
|
} catch (e) {
|
|
@@ -98,9 +100,11 @@ class Runtime implements RuntimeLike {
|
|
|
98
100
|
// can count rows_in for every stage that reads stdin — including
|
|
99
101
|
// pass-through tools and terminal sinks that never call ctx.output().
|
|
100
102
|
writeInput(d.id ?? '', d.src ?? '');
|
|
101
|
-
|
|
103
|
+
// Resolve $token paths in v before handing to processor.
|
|
104
|
+
const resolvedV = this.paths.resolveValue(d.v);
|
|
105
|
+
const ctx = new Context(d.id ?? '', d.src ?? '', this.memory, this, this.paths);
|
|
102
106
|
try {
|
|
103
|
-
await this.input(
|
|
107
|
+
await this.input(resolvedV, this.settings, ctx);
|
|
104
108
|
} catch (e) {
|
|
105
109
|
ctx.error(d.v, e);
|
|
106
110
|
}
|
|
@@ -110,6 +114,19 @@ class Runtime implements RuntimeLike {
|
|
|
110
114
|
|
|
111
115
|
if (this.queue.length > 0) await this.drainQueue();
|
|
112
116
|
if (opts.onShutdown) await opts.onShutdown();
|
|
117
|
+
|
|
118
|
+
// Workaround for Bun's exit-time stdout race that drops tail-end
|
|
119
|
+
// envelopes when ctx.output runs back-to-back milliseconds before
|
|
120
|
+
// stdin EOF. The bare `process.stdout.write('', cb)` callback fires
|
|
121
|
+
// before the OS pipe actually settles (observed: callback alone
|
|
122
|
+
// reduced the bug from ~30% to ~20%; adding a 50ms grace eliminates
|
|
123
|
+
// it). Issue 0045 — recon-extract-col-meta → recon-normalize-cols
|
|
124
|
+
// hand-off losing 1 of N envelopes. Same family as inbox 0041.
|
|
125
|
+
await new Promise<void>((resolve) => {
|
|
126
|
+
process.stdout.write('', () => {
|
|
127
|
+
setTimeout(resolve, 50);
|
|
128
|
+
});
|
|
129
|
+
});
|
|
113
130
|
}
|
|
114
131
|
}
|
|
115
132
|
|