@combycode/dpe-framework-ts 2.0.0 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/package.json +1 -1
- package/src/cache.ts +100 -0
- package/src/context.ts +50 -4
- package/src/envelope.ts +23 -3
- package/src/index.ts +2 -0
- package/src/paths.ts +114 -0
- package/src/runtime.ts +23 -5
package/README.md
CHANGED
|
@@ -90,9 +90,11 @@ dpe-dev scaffold --name my-tool --runtime bun --out ./my-tool --description "wha
|
|
|
90
90
|
cd my-tool
|
|
91
91
|
dpe-dev build . # bun install
|
|
92
92
|
dpe-dev test . # bun test
|
|
93
|
-
dpe-dev verify . # spawn `bun src/main.ts`, feed input.ndjson, diff stdout vs expected.ndjson
|
|
94
93
|
```
|
|
95
94
|
|
|
95
|
+
End-to-end stage testing happens via `dpe test <pipeline>:<stage>` once the
|
|
96
|
+
tool is wired into a pipeline — see `docs/testing.md`.
|
|
97
|
+
|
|
96
98
|
## Repo & licence
|
|
97
99
|
|
|
98
100
|
- Source: <https://github.com/combycode/dpe-framework-ts>
|
package/package.json
CHANGED
package/src/cache.ts
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
// Cache helper used by `ctx.cached(...)`. Honors the runner's
|
|
2
|
+
// DPE_CACHE_MODE env (use / refresh / bypass / off) and stores
|
|
3
|
+
// JSON-serialized values at $DPE_STORAGE/<namespace>/<key>.json.
|
|
4
|
+
//
|
|
5
|
+
// Key derivation: blake2b(canonical-JSON(key)) → 32-hex chars.
|
|
6
|
+
// Canonical = sorted object keys, compact, no whitespace.
|
|
7
|
+
//
|
|
8
|
+
// Failure modes (cache-disabling, NOT errors propagated to user):
|
|
9
|
+
// - DPE_STORAGE not set → cache disabled, every call produces
|
|
10
|
+
// - cache file unreadable → treat as miss, log warn
|
|
11
|
+
// - cache file unparseable → treat as miss, log warn
|
|
12
|
+
// - producer throws → re-throw to caller (no cache write)
|
|
13
|
+
//
|
|
14
|
+
// All cache I/O is sync — these are short JSON files, async wouldn't
|
|
15
|
+
// buy anything and would complicate the call site.
|
|
16
|
+
|
|
17
|
+
import { createHash } from 'node:crypto';
|
|
18
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
19
|
+
import { dirname, join } from 'node:path';
|
|
20
|
+
import { writeLog } from './envelope';
|
|
21
|
+
|
|
22
|
+
export type CacheMode = 'use' | 'refresh' | 'bypass' | 'off';
|
|
23
|
+
|
|
24
|
+
/** Read DPE_CACHE_MODE; default to "use". Anything unrecognized → "use". */
|
|
25
|
+
export function readCacheMode(): CacheMode {
|
|
26
|
+
const v = process.env.DPE_CACHE_MODE ?? 'use';
|
|
27
|
+
if (v === 'use' || v === 'refresh' || v === 'bypass' || v === 'off') return v;
|
|
28
|
+
return 'use';
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Compute the on-disk path for a (namespace, key) pair. Returns null
|
|
32
|
+
* when DPE_STORAGE isn't set — caller treats that as cache disabled. */
|
|
33
|
+
export function cachePath(namespace: string, key: unknown): string | null {
|
|
34
|
+
const storage = process.env.DPE_STORAGE;
|
|
35
|
+
if (!storage) return null;
|
|
36
|
+
const k = canonicalJson(key);
|
|
37
|
+
const hash = createHash('blake2b512').update(k, 'utf8').digest('hex').slice(0, 32);
|
|
38
|
+
return join(storage, namespace, `${hash}.json`);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Helper used by Context.cached(). Implements the four-mode cache
|
|
42
|
+
* semantics around the produce callback. */
|
|
43
|
+
export async function cachedImpl<T>(
|
|
44
|
+
namespace: string,
|
|
45
|
+
key: unknown,
|
|
46
|
+
produce: () => T | Promise<T>,
|
|
47
|
+
): Promise<T> {
|
|
48
|
+
const mode = readCacheMode();
|
|
49
|
+
const path = cachePath(namespace, key);
|
|
50
|
+
|
|
51
|
+
// Mode "off"/"bypass" or no $storage → just produce, no cache I/O.
|
|
52
|
+
const canRead = path !== null && (mode === 'use' || mode === 'refresh');
|
|
53
|
+
const canWrite = path !== null && mode !== 'bypass' && mode !== 'off';
|
|
54
|
+
const willRead = canRead && mode !== 'refresh';
|
|
55
|
+
|
|
56
|
+
if (willRead && existsSync(path!)) {
|
|
57
|
+
try {
|
|
58
|
+
const raw = readFileSync(path!, 'utf8');
|
|
59
|
+
const value = JSON.parse(raw) as T;
|
|
60
|
+
writeLog(`cached: hit (${namespace})`, 'debug');
|
|
61
|
+
return value;
|
|
62
|
+
} catch (e) {
|
|
63
|
+
writeLog(
|
|
64
|
+
`cached: read failed (${namespace}) — ${e instanceof Error ? e.message : String(e)}`,
|
|
65
|
+
'warn',
|
|
66
|
+
);
|
|
67
|
+
// Fall through to produce.
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const result = await produce();
|
|
72
|
+
|
|
73
|
+
if (canWrite && path !== null) {
|
|
74
|
+
try {
|
|
75
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
76
|
+
writeFileSync(path, JSON.stringify(result));
|
|
77
|
+
} catch (e) {
|
|
78
|
+
writeLog(
|
|
79
|
+
`cached: write failed (${namespace}) — ${e instanceof Error ? e.message : String(e)}`,
|
|
80
|
+
'warn',
|
|
81
|
+
);
|
|
82
|
+
// Don't fail the caller — they got their value.
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return result;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Canonical JSON: stable across runs. Sorted object keys, compact,
|
|
90
|
+
* no whitespace. Same logic as envelope.canonicalJson but exposed
|
|
91
|
+
* here so cache is independent of envelope-id concerns. */
|
|
92
|
+
function canonicalJson(v: unknown): string {
|
|
93
|
+
if (v === null || typeof v !== 'object') return JSON.stringify(v);
|
|
94
|
+
if (Array.isArray(v)) return `[${(v as unknown[]).map(canonicalJson).join(',')}]`;
|
|
95
|
+
const keys = Object.keys(v as Record<string, unknown>).sort();
|
|
96
|
+
const parts = keys.map(
|
|
97
|
+
(k) => `${JSON.stringify(k)}:${canonicalJson((v as Record<string, unknown>)[k])}`,
|
|
98
|
+
);
|
|
99
|
+
return `{${parts.join(',')}}`;
|
|
100
|
+
}
|
package/src/context.ts
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
* ctx.hashFile(path)
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
|
+
import { cachedImpl } from './cache';
|
|
17
18
|
import type { JSONValue } from './envelope';
|
|
18
19
|
import {
|
|
19
20
|
hashFile,
|
|
@@ -25,6 +26,7 @@ import {
|
|
|
25
26
|
writeStats,
|
|
26
27
|
writeTrace,
|
|
27
28
|
} from './envelope';
|
|
29
|
+
import { EnvPaths } from './paths';
|
|
28
30
|
|
|
29
31
|
export interface QueueItem {
|
|
30
32
|
queue: string;
|
|
@@ -44,15 +46,17 @@ export class Context {
|
|
|
44
46
|
readonly src: string;
|
|
45
47
|
readonly memory: Memory;
|
|
46
48
|
private readonly _runtime: RuntimeLike;
|
|
49
|
+
private readonly _paths: EnvPaths;
|
|
47
50
|
/** Labels accumulated by ctx.trace(k, v). Flushed (merged) as one
|
|
48
51
|
* {type:"trace"} stderr event before each ctx.output(), then cleared. */
|
|
49
52
|
private _labels: Record<string, JSONValue> = {};
|
|
50
53
|
|
|
51
|
-
constructor(id: string, src: string, memory: Memory, runtime: RuntimeLike) {
|
|
54
|
+
constructor(id: string, src: string, memory: Memory, runtime: RuntimeLike, paths?: EnvPaths) {
|
|
52
55
|
this.id = id;
|
|
53
56
|
this.src = src;
|
|
54
57
|
this.memory = memory;
|
|
55
58
|
this._runtime = runtime;
|
|
59
|
+
this._paths = paths ?? new EnvPaths();
|
|
56
60
|
}
|
|
57
61
|
|
|
58
62
|
/** Accumulate a label on this invocation's next output envelope. */
|
|
@@ -71,9 +75,11 @@ export class Context {
|
|
|
71
75
|
const outId = opts.id ?? this.id;
|
|
72
76
|
const outSrc = opts.src ?? this.src;
|
|
73
77
|
// Emit merged trace first (always, even with empty labels), then data, then clear.
|
|
74
|
-
|
|
78
|
+
// channel:"data" -> runner counts as rows_out.
|
|
79
|
+
writeTrace(outId, outSrc, this._labels, 'data');
|
|
75
80
|
this._labels = {};
|
|
76
|
-
|
|
81
|
+
// Tokenize absolute paths back to $token form before writing to stdout.
|
|
82
|
+
writeData(this._paths.tokenizeValue(v), outId, outSrc);
|
|
77
83
|
}
|
|
78
84
|
|
|
79
85
|
emit(queue: string, v: JSONValue, opts: { id?: string; src?: string } = {}): void {
|
|
@@ -90,7 +96,11 @@ export class Context {
|
|
|
90
96
|
}
|
|
91
97
|
|
|
92
98
|
meta(v: JSONValue): void {
|
|
93
|
-
|
|
99
|
+
// Emit a meta-channel trace before the meta envelope so the runner
|
|
100
|
+
// can increment its per-stage `meta` counter. Inherits ctx id/src.
|
|
101
|
+
// Tokenize absolute paths back to $token form.
|
|
102
|
+
writeTrace(this.id, this.src, {}, 'meta');
|
|
103
|
+
writeMeta(this._paths.tokenizeValue(v));
|
|
94
104
|
}
|
|
95
105
|
|
|
96
106
|
log(msg: string, opts: { level?: string } & Record<string, JSONValue> = {}): void {
|
|
@@ -109,6 +119,42 @@ export class Context {
|
|
|
109
119
|
hashFile(filepath: string): Promise<string | undefined> {
|
|
110
120
|
return hashFile(filepath);
|
|
111
121
|
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Cache the result of an expensive computation under
|
|
125
|
+
* `$DPE_STORAGE/<namespace>/<hash>.json`. Honors the runner's
|
|
126
|
+
* `DPE_CACHE_MODE` env (use / refresh / bypass / off):
|
|
127
|
+
* - use (default) — read cache if present, else produce + write
|
|
128
|
+
* - refresh — always produce, overwrite cache
|
|
129
|
+
* - bypass — produce, skip both read and write
|
|
130
|
+
* - off — same as bypass
|
|
131
|
+
*
|
|
132
|
+
* `key` is canonical-JSON-hashed (blake2b-256, first 32 hex chars).
|
|
133
|
+
* Compose it from whatever determines output equivalence: settings
|
|
134
|
+
* subset, content hash from `ctx.hashFile(...)`, page index, model
|
|
135
|
+
* id — anything that, if changed, should produce a different result.
|
|
136
|
+
*
|
|
137
|
+
* If `$DPE_STORAGE` isn't set (e.g. tool invoked outside a pipeline),
|
|
138
|
+
* cache is silently disabled — every call goes through `produce`.
|
|
139
|
+
*
|
|
140
|
+
* Returns the producer's result either way (cache hit or fresh).
|
|
141
|
+
* Producer errors propagate; failed runs do NOT poison the cache.
|
|
142
|
+
*
|
|
143
|
+
* @example
|
|
144
|
+
* const result = await ctx.cached(
|
|
145
|
+
* "doc-converter",
|
|
146
|
+
* { fileHash: await ctx.hashFile(v.path), settings, page },
|
|
147
|
+
* () => provider.convertPage(...),
|
|
148
|
+
* );
|
|
149
|
+
* ctx.output(result);
|
|
150
|
+
*/
|
|
151
|
+
cached<T extends JSONValue>(
|
|
152
|
+
namespace: string,
|
|
153
|
+
key: unknown,
|
|
154
|
+
produce: () => T | Promise<T>,
|
|
155
|
+
): Promise<T> {
|
|
156
|
+
return cachedImpl(namespace, key, produce);
|
|
157
|
+
}
|
|
112
158
|
}
|
|
113
159
|
|
|
114
160
|
/**
|
package/src/envelope.ts
CHANGED
|
@@ -101,9 +101,29 @@ export function writeError(v: JSONValue, err: unknown, id: string, src: string):
|
|
|
101
101
|
});
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
-
/** Merged trace event flushed once per output envelope (even with empty labels).
|
|
105
|
-
|
|
106
|
-
|
|
104
|
+
/** Merged trace event flushed once per output envelope (even with empty labels).
|
|
105
|
+
*
|
|
106
|
+
* `channel` distinguishes counter buckets at the runner side:
|
|
107
|
+
* - `"data"` from ctx.output → rows_out
|
|
108
|
+
* - `"meta"` from ctx.meta → meta
|
|
109
|
+
* Omit the field for back-compat (older runner treats unset as data).
|
|
110
|
+
*/
|
|
111
|
+
export function writeTrace(
|
|
112
|
+
id: string,
|
|
113
|
+
src: string,
|
|
114
|
+
labels: Record<string, JSONValue>,
|
|
115
|
+
channel?: 'data' | 'meta',
|
|
116
|
+
): void {
|
|
117
|
+
const rec: Record<string, JSONValue> = { type: 'trace', id, src, labels };
|
|
118
|
+
if (channel) rec.channel = channel;
|
|
119
|
+
_writeStderr(rec);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Input event — emitted by the runtime per envelope read from stdin,
|
|
123
|
+
* BEFORE the processor runs. Lets the runner count rows_in for stages
|
|
124
|
+
* that never call ctx.output (pass-through, terminal sinks). */
|
|
125
|
+
export function writeInput(id: string, src: string): void {
|
|
126
|
+
_writeStderr({ type: 'input', id, src });
|
|
107
127
|
}
|
|
108
128
|
|
|
109
129
|
/** Custom stats event. Shape: {type:"stats", ...data}. */
|
package/src/index.ts
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* });
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
|
+
export type { CacheMode } from './cache';
|
|
14
15
|
export type { QueueItem, RuntimeLike } from './context';
|
|
15
16
|
export { Context, Memory } from './context';
|
|
16
17
|
export type { DataEnvelope, Envelope, JSONValue, MetaEnvelope } from './envelope';
|
|
@@ -24,5 +25,6 @@ export {
|
|
|
24
25
|
writeLog,
|
|
25
26
|
writeMeta,
|
|
26
27
|
} from './envelope';
|
|
28
|
+
export { EnvPaths } from './paths';
|
|
27
29
|
export type { Processor, RunOptions } from './runtime';
|
|
28
30
|
export { run } from './runtime';
|
package/src/paths.ts
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EnvPaths -- resolves $token path prefixes in envelope v on input;
|
|
3
|
+
* reverse-tokenizes absolute paths back to $token form on output.
|
|
4
|
+
*
|
|
5
|
+
* Input side (runtime, before processor): $token/subpath -> absolute path
|
|
6
|
+
* Output side (context.output / context.meta): absolute path -> $token/subpath
|
|
7
|
+
*
|
|
8
|
+
* Both sides are a no-op when the DPE_* env vars are not set.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import type { JSONValue } from './envelope';
|
|
12
|
+
|
|
13
|
+
const TOKEN_MAP: [string, string][] = [
|
|
14
|
+
['input', 'DPE_INPUT'],
|
|
15
|
+
['output', 'DPE_OUTPUT'],
|
|
16
|
+
['configs', 'DPE_CONFIGS'],
|
|
17
|
+
['storage', 'DPE_STORAGE'],
|
|
18
|
+
['temp', 'DPE_TEMP'],
|
|
19
|
+
['session', 'DPE_SESSION'],
|
|
20
|
+
];
|
|
21
|
+
|
|
22
|
+
interface Entry {
|
|
23
|
+
token: string;
|
|
24
|
+
abs: string;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Resolved env prefix map. Build once at tool startup; cheap to clone. */
|
|
28
|
+
export class EnvPaths {
|
|
29
|
+
private entries: Entry[];
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* If `pairs` is provided, use those (token, abs_path) pairs directly.
|
|
33
|
+
* Otherwise read the standard DPE_* env vars from process.env.
|
|
34
|
+
*
|
|
35
|
+
* Prefer the named factories `EnvPaths.fromEnv()` and
|
|
36
|
+
* `EnvPaths.fromPairs(pairs)` -- they're consistent with the Rust SDK
|
|
37
|
+
* (`from_env()` / `from_pairs(&[...])`) and read more clearly at call
|
|
38
|
+
* sites. The polymorphic constructor is kept for backwards compat.
|
|
39
|
+
*/
|
|
40
|
+
constructor(pairs?: [string, string][]) {
|
|
41
|
+
const raw: Entry[] =
|
|
42
|
+
pairs != null
|
|
43
|
+
? pairs.map(([token, abs]) => ({ token, abs: abs.replace(/\\/g, '/') }))
|
|
44
|
+
: TOKEN_MAP.flatMap(([token, envVar]) => {
|
|
45
|
+
const val = process.env[envVar];
|
|
46
|
+
return val ? [{ token, abs: val.replace(/\\/g, '/') }] : [];
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
// Sort longest-abs-first for greedy tokenization.
|
|
50
|
+
this.entries = raw.sort((a, b) => b.abs.length - a.abs.length);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** Read the standard DPE_* env vars; missing/empty vars are skipped. */
|
|
54
|
+
static fromEnv(): EnvPaths {
|
|
55
|
+
return new EnvPaths();
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** Construct from explicit (token, abs_path) pairs -- useful for tests. */
|
|
59
|
+
static fromPairs(pairs: [string, string][]): EnvPaths {
|
|
60
|
+
return new EnvPaths(pairs);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
isEmpty(): boolean {
|
|
64
|
+
return this.entries.length === 0;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/** Walk v, resolving $token[/subpath] strings to absolute paths. */
|
|
68
|
+
resolveValue(v: JSONValue): JSONValue {
|
|
69
|
+
if (this.isEmpty()) return v;
|
|
70
|
+
return this.walk(v, (s) => this.resolveStr(s));
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/** Walk v, tokenizing absolute paths back to $token[/subpath] form. */
|
|
74
|
+
tokenizeValue(v: JSONValue): JSONValue {
|
|
75
|
+
if (this.isEmpty()) return v;
|
|
76
|
+
return this.walk(v, (s) => this.tokenizeStr(s));
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
private walk(v: JSONValue, f: (s: string) => string): JSONValue {
|
|
80
|
+
if (typeof v === 'string') return f(v);
|
|
81
|
+
if (Array.isArray(v)) return v.map((x) => this.walk(x, f));
|
|
82
|
+
if (v !== null && typeof v === 'object') {
|
|
83
|
+
const out: Record<string, JSONValue> = {};
|
|
84
|
+
for (const [k, val] of Object.entries(v)) {
|
|
85
|
+
out[k] = this.walk(val as JSONValue, f);
|
|
86
|
+
}
|
|
87
|
+
return out;
|
|
88
|
+
}
|
|
89
|
+
return v;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
private resolveStr(s: string): string {
|
|
93
|
+
if (!s.startsWith('$')) return s;
|
|
94
|
+
const slashIdx = s.indexOf('/');
|
|
95
|
+
const [name, tail] =
|
|
96
|
+
slashIdx < 0 ? [s.slice(1), ''] : [s.slice(1, slashIdx), s.slice(slashIdx)];
|
|
97
|
+
const entry = this.entries.find((e) => e.token === name);
|
|
98
|
+
if (!entry) return s;
|
|
99
|
+
return tail ? entry.abs + tail : entry.abs;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
private tokenizeStr(s: string): string {
|
|
103
|
+
const normalized = s.replace(/\\/g, '/');
|
|
104
|
+
for (const entry of this.entries) {
|
|
105
|
+
if (normalized.startsWith(entry.abs)) {
|
|
106
|
+
const rest = normalized.slice(entry.abs.length);
|
|
107
|
+
if (!rest) return `$${entry.token}`;
|
|
108
|
+
if (rest.startsWith('/')) return `$${entry.token}${rest}`;
|
|
109
|
+
// rest doesn't start with '/' -- partial component match, skip
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
return s;
|
|
113
|
+
}
|
|
114
|
+
}
|
package/src/runtime.ts
CHANGED
|
@@ -8,7 +8,8 @@
|
|
|
8
8
|
|
|
9
9
|
import { Context, Memory, type QueueItem, type RuntimeLike } from './context';
|
|
10
10
|
import type { DataEnvelope, JSONValue } from './envelope';
|
|
11
|
-
import { parseEnvelope, writeLog } from './envelope';
|
|
11
|
+
import { parseEnvelope, writeInput, writeLog } from './envelope';
|
|
12
|
+
import { EnvPaths } from './paths';
|
|
12
13
|
|
|
13
14
|
export type Processor = (v: JSONValue, settings: JSONValue, ctx: Context) => void | Promise<void>;
|
|
14
15
|
|
|
@@ -33,6 +34,7 @@ class Runtime implements RuntimeLike {
|
|
|
33
34
|
private queue: QueueItem[] = [];
|
|
34
35
|
private shutdown = false;
|
|
35
36
|
private drainLimit = 100_000;
|
|
37
|
+
private paths = new EnvPaths();
|
|
36
38
|
|
|
37
39
|
enqueue(item: QueueItem): void {
|
|
38
40
|
this.queue.push(item);
|
|
@@ -47,7 +49,7 @@ class Runtime implements RuntimeLike {
|
|
|
47
49
|
writeLog(`No processor for queue '${item.queue}', dropping item`, 'warn');
|
|
48
50
|
continue;
|
|
49
51
|
}
|
|
50
|
-
const ctx = new Context(item.id, item.src, this.memory, this);
|
|
52
|
+
const ctx = new Context(item.id, item.src, this.memory, this, this.paths);
|
|
51
53
|
try {
|
|
52
54
|
await proc(item.v, this.settings, ctx);
|
|
53
55
|
} catch (e) {
|
|
@@ -66,6 +68,13 @@ class Runtime implements RuntimeLike {
|
|
|
66
68
|
if (opts.maxDrainIterations) this.drainLimit = opts.maxDrainIterations;
|
|
67
69
|
|
|
68
70
|
this.settings = parseSettings();
|
|
71
|
+
// `accept_meta: true` in settings → the read loop dispatches t:"m"
|
|
72
|
+
// envelopes to the input processor alongside t:"d" (instead of
|
|
73
|
+
// skipping). Default false preserves historical behavior — meta
|
|
74
|
+
// envelopes only matter to tools that explicitly opt in (typically
|
|
75
|
+
// sinks like write-file-stream when used as a meta-output target).
|
|
76
|
+
const s = this.settings as { accept_meta?: unknown } | null;
|
|
77
|
+
const acceptMeta = !!(s && typeof s === 'object' && s.accept_meta === true);
|
|
69
78
|
|
|
70
79
|
// SIGTERM / SIGINT → graceful shutdown (stop reading, drain, exit)
|
|
71
80
|
const onSignal = () => {
|
|
@@ -81,12 +90,21 @@ class Runtime implements RuntimeLike {
|
|
|
81
90
|
|
|
82
91
|
const env = parseEnvelope(line);
|
|
83
92
|
if (env === null) continue;
|
|
84
|
-
|
|
93
|
+
// Default: only data envelopes. With `accept_meta: true` the
|
|
94
|
+
// tool ALSO receives meta envelopes through the input
|
|
95
|
+
// processor. Anything else still skipped.
|
|
96
|
+
if (env.t !== 'd' && !(acceptMeta && env.t === 'm')) continue;
|
|
85
97
|
|
|
86
98
|
const d = env as DataEnvelope;
|
|
87
|
-
|
|
99
|
+
// Emit an `input` event BEFORE the processor runs so the runner
|
|
100
|
+
// can count rows_in for every stage that reads stdin — including
|
|
101
|
+
// pass-through tools and terminal sinks that never call ctx.output().
|
|
102
|
+
writeInput(d.id ?? '', d.src ?? '');
|
|
103
|
+
// Resolve $token paths in v before handing to processor.
|
|
104
|
+
const resolvedV = this.paths.resolveValue(d.v);
|
|
105
|
+
const ctx = new Context(d.id ?? '', d.src ?? '', this.memory, this, this.paths);
|
|
88
106
|
try {
|
|
89
|
-
await this.input(
|
|
107
|
+
await this.input(resolvedV, this.settings, ctx);
|
|
90
108
|
} catch (e) {
|
|
91
109
|
ctx.error(d.v, e);
|
|
92
110
|
}
|