@combycode/dpe-framework-ts 2.0.0 → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/cache.ts +100 -0
- package/src/context.ts +42 -1
- package/src/envelope.ts +23 -3
- package/src/index.ts +1 -0
- package/src/runtime.ts +16 -2
package/package.json
CHANGED
package/src/cache.ts
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
// Cache helper used by `ctx.cached(...)`. Honors the runner's
|
|
2
|
+
// DPE_CACHE_MODE env (use / refresh / bypass / off) and stores
|
|
3
|
+
// JSON-serialized values at $DPE_STORAGE/<namespace>/<key>.json.
|
|
4
|
+
//
|
|
5
|
+
// Key derivation: blake2b(canonical-JSON(key)) → 32-hex chars.
|
|
6
|
+
// Canonical = sorted object keys, compact, no whitespace.
|
|
7
|
+
//
|
|
8
|
+
// Failure modes (cache-disabling, NOT errors propagated to user):
|
|
9
|
+
// - DPE_STORAGE not set → cache disabled, every call produces
|
|
10
|
+
// - cache file unreadable → treat as miss, log warn
|
|
11
|
+
// - cache file unparseable → treat as miss, log warn
|
|
12
|
+
// - producer throws → re-throw to caller (no cache write)
|
|
13
|
+
//
|
|
14
|
+
// All cache I/O is sync — these are short JSON files, async wouldn't
|
|
15
|
+
// buy anything and would complicate the call site.
|
|
16
|
+
|
|
17
|
+
import { createHash } from 'node:crypto';
|
|
18
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
19
|
+
import { dirname, join } from 'node:path';
|
|
20
|
+
import { writeLog } from './envelope';
|
|
21
|
+
|
|
22
|
+
export type CacheMode = 'use' | 'refresh' | 'bypass' | 'off';
|
|
23
|
+
|
|
24
|
+
/** Read DPE_CACHE_MODE; default to "use". Anything unrecognized → "use". */
|
|
25
|
+
export function readCacheMode(): CacheMode {
|
|
26
|
+
const v = process.env.DPE_CACHE_MODE ?? 'use';
|
|
27
|
+
if (v === 'use' || v === 'refresh' || v === 'bypass' || v === 'off') return v;
|
|
28
|
+
return 'use';
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** Compute the on-disk path for a (namespace, key) pair. Returns null
|
|
32
|
+
* when DPE_STORAGE isn't set — caller treats that as cache disabled. */
|
|
33
|
+
export function cachePath(namespace: string, key: unknown): string | null {
|
|
34
|
+
const storage = process.env.DPE_STORAGE;
|
|
35
|
+
if (!storage) return null;
|
|
36
|
+
const k = canonicalJson(key);
|
|
37
|
+
const hash = createHash('blake2b512').update(k, 'utf8').digest('hex').slice(0, 32);
|
|
38
|
+
return join(storage, namespace, `${hash}.json`);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** Helper used by Context.cached(). Implements the four-mode cache
|
|
42
|
+
* semantics around the produce callback. */
|
|
43
|
+
export async function cachedImpl<T>(
|
|
44
|
+
namespace: string,
|
|
45
|
+
key: unknown,
|
|
46
|
+
produce: () => T | Promise<T>,
|
|
47
|
+
): Promise<T> {
|
|
48
|
+
const mode = readCacheMode();
|
|
49
|
+
const path = cachePath(namespace, key);
|
|
50
|
+
|
|
51
|
+
// Mode "off"/"bypass" or no $storage → just produce, no cache I/O.
|
|
52
|
+
const canRead = path !== null && (mode === 'use' || mode === 'refresh');
|
|
53
|
+
const canWrite = path !== null && mode !== 'bypass' && mode !== 'off';
|
|
54
|
+
const willRead = canRead && mode !== 'refresh';
|
|
55
|
+
|
|
56
|
+
if (willRead && existsSync(path!)) {
|
|
57
|
+
try {
|
|
58
|
+
const raw = readFileSync(path!, 'utf8');
|
|
59
|
+
const value = JSON.parse(raw) as T;
|
|
60
|
+
writeLog(`cached: hit (${namespace})`, 'debug');
|
|
61
|
+
return value;
|
|
62
|
+
} catch (e) {
|
|
63
|
+
writeLog(
|
|
64
|
+
`cached: read failed (${namespace}) — ${e instanceof Error ? e.message : String(e)}`,
|
|
65
|
+
'warn',
|
|
66
|
+
);
|
|
67
|
+
// Fall through to produce.
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const result = await produce();
|
|
72
|
+
|
|
73
|
+
if (canWrite && path !== null) {
|
|
74
|
+
try {
|
|
75
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
76
|
+
writeFileSync(path, JSON.stringify(result));
|
|
77
|
+
} catch (e) {
|
|
78
|
+
writeLog(
|
|
79
|
+
`cached: write failed (${namespace}) — ${e instanceof Error ? e.message : String(e)}`,
|
|
80
|
+
'warn',
|
|
81
|
+
);
|
|
82
|
+
// Don't fail the caller — they got their value.
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
return result;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** Canonical JSON: stable across runs. Sorted object keys, compact,
|
|
90
|
+
* no whitespace. Same logic as envelope.canonicalJson but exposed
|
|
91
|
+
* here so cache is independent of envelope-id concerns. */
|
|
92
|
+
function canonicalJson(v: unknown): string {
|
|
93
|
+
if (v === null || typeof v !== 'object') return JSON.stringify(v);
|
|
94
|
+
if (Array.isArray(v)) return `[${(v as unknown[]).map(canonicalJson).join(',')}]`;
|
|
95
|
+
const keys = Object.keys(v as Record<string, unknown>).sort();
|
|
96
|
+
const parts = keys.map(
|
|
97
|
+
(k) => `${JSON.stringify(k)}:${canonicalJson((v as Record<string, unknown>)[k])}`,
|
|
98
|
+
);
|
|
99
|
+
return `{${parts.join(',')}}`;
|
|
100
|
+
}
|
package/src/context.ts
CHANGED
|
@@ -14,6 +14,7 @@
|
|
|
14
14
|
* ctx.hashFile(path)
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
|
+
import { cachedImpl } from './cache';
|
|
17
18
|
import type { JSONValue } from './envelope';
|
|
18
19
|
import {
|
|
19
20
|
hashFile,
|
|
@@ -71,7 +72,8 @@ export class Context {
|
|
|
71
72
|
const outId = opts.id ?? this.id;
|
|
72
73
|
const outSrc = opts.src ?? this.src;
|
|
73
74
|
// Emit merged trace first (always, even with empty labels), then data, then clear.
|
|
74
|
-
|
|
75
|
+
// channel:"data" → runner counts as rows_out.
|
|
76
|
+
writeTrace(outId, outSrc, this._labels, 'data');
|
|
75
77
|
this._labels = {};
|
|
76
78
|
writeData(v, outId, outSrc);
|
|
77
79
|
}
|
|
@@ -90,6 +92,9 @@ export class Context {
|
|
|
90
92
|
}
|
|
91
93
|
|
|
92
94
|
meta(v: JSONValue): void {
|
|
95
|
+
// Emit a meta-channel trace before the meta envelope so the runner
|
|
96
|
+
// can increment its per-stage `meta` counter. Inherits ctx id/src.
|
|
97
|
+
writeTrace(this.id, this.src, {}, 'meta');
|
|
93
98
|
writeMeta(v);
|
|
94
99
|
}
|
|
95
100
|
|
|
@@ -109,6 +114,42 @@ export class Context {
|
|
|
109
114
|
hashFile(filepath: string): Promise<string | undefined> {
|
|
110
115
|
return hashFile(filepath);
|
|
111
116
|
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Cache the result of an expensive computation under
|
|
120
|
+
* `$DPE_STORAGE/<namespace>/<hash>.json`. Honors the runner's
|
|
121
|
+
* `DPE_CACHE_MODE` env (use / refresh / bypass / off):
|
|
122
|
+
* - use (default) — read cache if present, else produce + write
|
|
123
|
+
* - refresh — always produce, overwrite cache
|
|
124
|
+
* - bypass — produce, skip both read and write
|
|
125
|
+
* - off — same as bypass
|
|
126
|
+
*
|
|
127
|
+
* `key` is canonical-JSON-hashed (blake2b-256, first 32 hex chars).
|
|
128
|
+
* Compose it from whatever determines output equivalence: settings
|
|
129
|
+
* subset, content hash from `ctx.hashFile(...)`, page index, model
|
|
130
|
+
* id — anything that, if changed, should produce a different result.
|
|
131
|
+
*
|
|
132
|
+
* If `$DPE_STORAGE` isn't set (e.g. tool invoked outside a pipeline),
|
|
133
|
+
* cache is silently disabled — every call goes through `produce`.
|
|
134
|
+
*
|
|
135
|
+
* Returns the producer's result either way (cache hit or fresh).
|
|
136
|
+
* Producer errors propagate; failed runs do NOT poison the cache.
|
|
137
|
+
*
|
|
138
|
+
* @example
|
|
139
|
+
* const result = await ctx.cached(
|
|
140
|
+
* "doc-converter",
|
|
141
|
+
* { fileHash: await ctx.hashFile(v.path), settings, page },
|
|
142
|
+
* () => provider.convertPage(...),
|
|
143
|
+
* );
|
|
144
|
+
* ctx.output(result);
|
|
145
|
+
*/
|
|
146
|
+
cached<T extends JSONValue>(
|
|
147
|
+
namespace: string,
|
|
148
|
+
key: unknown,
|
|
149
|
+
produce: () => T | Promise<T>,
|
|
150
|
+
): Promise<T> {
|
|
151
|
+
return cachedImpl(namespace, key, produce);
|
|
152
|
+
}
|
|
112
153
|
}
|
|
113
154
|
|
|
114
155
|
/**
|
package/src/envelope.ts
CHANGED
|
@@ -101,9 +101,29 @@ export function writeError(v: JSONValue, err: unknown, id: string, src: string):
|
|
|
101
101
|
});
|
|
102
102
|
}
|
|
103
103
|
|
|
104
|
-
/** Merged trace event flushed once per output envelope (even with empty labels).
|
|
105
|
-
|
|
106
|
-
|
|
104
|
+
/** Merged trace event flushed once per output envelope (even with empty labels).
|
|
105
|
+
*
|
|
106
|
+
* `channel` distinguishes counter buckets at the runner side:
|
|
107
|
+
* - `"data"` from ctx.output → rows_out
|
|
108
|
+
* - `"meta"` from ctx.meta → meta
|
|
109
|
+
* Omit the field for back-compat (older runner treats unset as data).
|
|
110
|
+
*/
|
|
111
|
+
export function writeTrace(
|
|
112
|
+
id: string,
|
|
113
|
+
src: string,
|
|
114
|
+
labels: Record<string, JSONValue>,
|
|
115
|
+
channel?: 'data' | 'meta',
|
|
116
|
+
): void {
|
|
117
|
+
const rec: Record<string, JSONValue> = { type: 'trace', id, src, labels };
|
|
118
|
+
if (channel) rec.channel = channel;
|
|
119
|
+
_writeStderr(rec);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** Input event — emitted by the runtime per envelope read from stdin,
|
|
123
|
+
* BEFORE the processor runs. Lets the runner count rows_in for stages
|
|
124
|
+
* that never call ctx.output (pass-through, terminal sinks). */
|
|
125
|
+
export function writeInput(id: string, src: string): void {
|
|
126
|
+
_writeStderr({ type: 'input', id, src });
|
|
107
127
|
}
|
|
108
128
|
|
|
109
129
|
/** Custom stats event. Shape: {type:"stats", ...data}. */
|
package/src/index.ts
CHANGED
package/src/runtime.ts
CHANGED
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
import { Context, Memory, type QueueItem, type RuntimeLike } from './context';
|
|
10
10
|
import type { DataEnvelope, JSONValue } from './envelope';
|
|
11
|
-
import { parseEnvelope, writeLog } from './envelope';
|
|
11
|
+
import { parseEnvelope, writeInput, writeLog } from './envelope';
|
|
12
12
|
|
|
13
13
|
export type Processor = (v: JSONValue, settings: JSONValue, ctx: Context) => void | Promise<void>;
|
|
14
14
|
|
|
@@ -66,6 +66,13 @@ class Runtime implements RuntimeLike {
|
|
|
66
66
|
if (opts.maxDrainIterations) this.drainLimit = opts.maxDrainIterations;
|
|
67
67
|
|
|
68
68
|
this.settings = parseSettings();
|
|
69
|
+
// `accept_meta: true` in settings → the read loop dispatches t:"m"
|
|
70
|
+
// envelopes to the input processor alongside t:"d" (instead of
|
|
71
|
+
// skipping). Default false preserves historical behavior — meta
|
|
72
|
+
// envelopes only matter to tools that explicitly opt in (typically
|
|
73
|
+
// sinks like write-file-stream when used as a meta-output target).
|
|
74
|
+
const s = this.settings as { accept_meta?: unknown } | null;
|
|
75
|
+
const acceptMeta = !!(s && typeof s === 'object' && s.accept_meta === true);
|
|
69
76
|
|
|
70
77
|
// SIGTERM / SIGINT → graceful shutdown (stop reading, drain, exit)
|
|
71
78
|
const onSignal = () => {
|
|
@@ -81,9 +88,16 @@ class Runtime implements RuntimeLike {
|
|
|
81
88
|
|
|
82
89
|
const env = parseEnvelope(line);
|
|
83
90
|
if (env === null) continue;
|
|
84
|
-
|
|
91
|
+
// Default: only data envelopes. With `accept_meta: true` the
|
|
92
|
+
// tool ALSO receives meta envelopes through the input
|
|
93
|
+
// processor. Anything else still skipped.
|
|
94
|
+
if (env.t !== 'd' && !(acceptMeta && env.t === 'm')) continue;
|
|
85
95
|
|
|
86
96
|
const d = env as DataEnvelope;
|
|
97
|
+
// Emit an `input` event BEFORE the processor runs so the runner
|
|
98
|
+
// can count rows_in for every stage that reads stdin — including
|
|
99
|
+
// pass-through tools and terminal sinks that never call ctx.output().
|
|
100
|
+
writeInput(d.id ?? '', d.src ?? '');
|
|
87
101
|
const ctx = new Context(d.id ?? '', d.src ?? '', this.memory, this);
|
|
88
102
|
try {
|
|
89
103
|
await this.input(d.v, this.settings, ctx);
|