@tangle-network/traces 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -0
- package/dist/{chunk-HF2KGEBW.js → chunk-6TVW7PL6.js} +142 -1
- package/dist/cli.js +45 -8
- package/dist/index.d.ts +97 -2
- package/dist/index.js +13 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -18,6 +18,7 @@ It reads the transcripts your harness leaves on disk, reconstructs the run as sp
|
|
|
18
18
|
- [Supported harnesses](#supported-harnesses)
|
|
19
19
|
- [CLI reference](#cli-reference)
|
|
20
20
|
- [Upload to the Intelligence Platform](#upload-to-the-intelligence-platform)
|
|
21
|
+
- [External engines (bring your own)](#external-engines-bring-your-own)
|
|
21
22
|
- [Library (SDK)](#library-sdk)
|
|
22
23
|
- [Examples](#examples)
|
|
23
24
|
- [Develop](#develop)
|
|
@@ -129,6 +130,29 @@ It does **not** catch free-form PII — names, postal addresses, phone numbers i
|
|
|
129
130
|
|
|
130
131
|
Always `--dry-run` first to see exactly what would be sent.
|
|
131
132
|
|
|
133
|
+
## External engines (bring your own)
|
|
134
|
+
|
|
135
|
+
`traces` hosts analysis engines and PII scrubbers it does **not** bundle — you install the tool, `traces` drives it over a thin command adapter. Same pattern for any future engine or model.
|
|
136
|
+
|
|
137
|
+
**Analyzers** run over the emitted OTLP artifact as peers to the built-in analysts:
|
|
138
|
+
|
|
139
|
+
```bash
|
|
140
|
+
traces analyze --last 1 --analyzer halo # run HALO too
|
|
141
|
+
traces analyze --last 1 --analyzer halo --analyzer-prompt "find token waste"
|
|
142
|
+
traces analyze --last 1 --analyzer halo --analyzer my-engine # repeatable
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
**Redactors** scrub prompt/response prose with an external PII model (catching names/addresses the regex pass can't), running *after* the built-in redaction:
|
|
146
|
+
|
|
147
|
+
```bash
|
|
148
|
+
# the command reads a JSON array of strings on stdin, writes the scrubbed array on stdout
|
|
149
|
+
traces upload --since 24h --dry-run --redactor "my-pii-scrubber"
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
In the SDK these are the `ExternalAnalyzer` and `Redactor` interfaces (`haloAnalyzer`, `commandAnalyzer`, `commandRedactor`, `applyRedactor`, `runExternalAnalyzers`). See [`examples/external-engines.ts`](./examples/external-engines.ts).
|
|
153
|
+
|
|
154
|
+
> For the built-in agentic analysts (`--llm`), set `OPENAI_API_KEY` — or point at any OpenAI-compatible gateway with `OPENAI_BASE_URL` (e.g. an internal router) to use a non-OpenAI key.
|
|
155
|
+
|
|
132
156
|
## Library (SDK)
|
|
133
157
|
|
|
134
158
|
The CLI is a thin consumer of these exports.
|
|
@@ -143,6 +167,7 @@ The CLI is a thin consumer of these exports.
|
|
|
143
167
|
| `planUpload` / `executeUpload` | `(…, { backend? }) → …` | redact + dedup + send to any sink |
|
|
144
168
|
| `selectAdapters` / `listAdapters` / `resolveAdapter` | adapter selection + the harness registry |
|
|
145
169
|
| `HarnessTraceAdapter` | interface (`locate` + `parse`) | implement to add a harness |
|
|
170
|
+
| `ExternalAnalyzer` / `Redactor` | `haloAnalyzer` / `commandAnalyzer` / `commandRedactor` | drive engines/models you install (not bundled) |
|
|
146
171
|
|
|
147
172
|
```ts
|
|
148
173
|
import { watchSessions, analyzeSpans, AnalystRegistry, makeFinding } from '@tangle-network/traces'
|
|
@@ -172,6 +197,7 @@ Runnable, in [`examples/`](./examples):
|
|
|
172
197
|
| [`custom-analyst.ts`](./examples/custom-analyst.ts) | register and run your own analyst |
|
|
173
198
|
| [`custom-backend.ts`](./examples/custom-backend.ts) | redact + dedup + upload to your own sink |
|
|
174
199
|
| [`register-harness.ts`](./examples/register-harness.ts) | add a new harness by implementing `HarnessTraceAdapter` |
|
|
200
|
+
| [`external-engines.ts`](./examples/external-engines.ts) | drive HALO + an external PII scrubber you install yourself |
|
|
175
201
|
|
|
176
202
|
## Develop
|
|
177
203
|
|
|
@@ -70,6 +70,114 @@ async function analyzeSpans(spans, opts = {}) {
|
|
|
70
70
|
return { otlpPath, result };
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
+
// src/external.ts
|
|
74
|
+
import { spawn } from "child_process";
|
|
75
|
+
function runCommand(command, args = [], opts = {}) {
|
|
76
|
+
const timeoutMs = opts.timeoutMs ?? 12e4;
|
|
77
|
+
const maxBuffer = opts.maxBuffer ?? 32 * 1024 * 1024;
|
|
78
|
+
return new Promise((resolve, reject) => {
|
|
79
|
+
let child;
|
|
80
|
+
try {
|
|
81
|
+
child = spawn(command, [...args], { signal: opts.signal });
|
|
82
|
+
} catch (err2) {
|
|
83
|
+
reject(err2 instanceof Error ? err2 : new Error(String(err2)));
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
let out = "";
|
|
87
|
+
let err = "";
|
|
88
|
+
let killed = false;
|
|
89
|
+
const timer = setTimeout(() => {
|
|
90
|
+
killed = true;
|
|
91
|
+
child.kill("SIGKILL");
|
|
92
|
+
reject(new Error(`${command}: timed out after ${timeoutMs}ms`));
|
|
93
|
+
}, timeoutMs);
|
|
94
|
+
const cap = (add, buf) => {
|
|
95
|
+
const next = buf + add;
|
|
96
|
+
if (next.length > maxBuffer) {
|
|
97
|
+
killed = true;
|
|
98
|
+
child.kill("SIGKILL");
|
|
99
|
+
clearTimeout(timer);
|
|
100
|
+
reject(new Error(`${command}: output exceeded ${maxBuffer} bytes`));
|
|
101
|
+
}
|
|
102
|
+
return next;
|
|
103
|
+
};
|
|
104
|
+
child.stdout?.on("data", (d) => {
|
|
105
|
+
out = cap(String(d), out);
|
|
106
|
+
});
|
|
107
|
+
child.stderr?.on("data", (d) => {
|
|
108
|
+
err = cap(String(d), err);
|
|
109
|
+
});
|
|
110
|
+
child.on("error", (e) => {
|
|
111
|
+
clearTimeout(timer);
|
|
112
|
+
if (!killed) reject(e);
|
|
113
|
+
});
|
|
114
|
+
child.on("close", (code) => {
|
|
115
|
+
clearTimeout(timer);
|
|
116
|
+
if (!killed) resolve({ stdout: out, stderr: err, code });
|
|
117
|
+
});
|
|
118
|
+
if (opts.input !== void 0) {
|
|
119
|
+
child.stdin?.on("error", () => {
|
|
120
|
+
});
|
|
121
|
+
child.stdin?.end(opts.input);
|
|
122
|
+
}
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
function commandAnalyzer(spec) {
|
|
126
|
+
return {
|
|
127
|
+
name: spec.name,
|
|
128
|
+
async analyze(otlpPath, opts = {}) {
|
|
129
|
+
try {
|
|
130
|
+
const res = await runCommand(spec.command, spec.args(otlpPath, opts.prompt), {
|
|
131
|
+
signal: opts.signal,
|
|
132
|
+
timeoutMs: spec.timeoutMs
|
|
133
|
+
});
|
|
134
|
+
if (res.code !== 0) {
|
|
135
|
+
return { analyzer: spec.name, ok: false, output: res.stdout, error: res.stderr.trim() || `exit ${res.code}` };
|
|
136
|
+
}
|
|
137
|
+
return { analyzer: spec.name, ok: true, output: res.stdout.trim(), findings: spec.parse?.(res.stdout) };
|
|
138
|
+
} catch (e) {
|
|
139
|
+
return { analyzer: spec.name, ok: false, output: "", error: e instanceof Error ? e.message : String(e) };
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
function haloAnalyzer(opts = {}) {
|
|
145
|
+
const defaultPrompt = opts.defaultPrompt ?? "diagnose";
|
|
146
|
+
return commandAnalyzer({
|
|
147
|
+
name: "halo",
|
|
148
|
+
command: opts.command ?? "halo",
|
|
149
|
+
args: (otlpPath, prompt) => [otlpPath, "-p", prompt ?? defaultPrompt],
|
|
150
|
+
timeoutMs: opts.timeoutMs
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
function runExternalAnalyzers(otlpPath, analyzers, opts = {}) {
|
|
154
|
+
return Promise.all(analyzers.map((a) => a.analyze(otlpPath, opts)));
|
|
155
|
+
}
|
|
156
|
+
function commandRedactor(spec) {
|
|
157
|
+
const name = spec.name ?? spec.command;
|
|
158
|
+
return {
|
|
159
|
+
name,
|
|
160
|
+
async redactText(texts) {
|
|
161
|
+
if (texts.length === 0) return [];
|
|
162
|
+
const res = await runCommand(spec.command, spec.args ?? [], {
|
|
163
|
+
input: JSON.stringify(texts),
|
|
164
|
+
timeoutMs: spec.timeoutMs
|
|
165
|
+
});
|
|
166
|
+
if (res.code !== 0) throw new Error(`redactor ${name}: exit ${res.code}: ${res.stderr.trim()}`);
|
|
167
|
+
let parsed;
|
|
168
|
+
try {
|
|
169
|
+
parsed = JSON.parse(res.stdout);
|
|
170
|
+
} catch {
|
|
171
|
+
throw new Error(`redactor ${name}: stdout was not JSON`);
|
|
172
|
+
}
|
|
173
|
+
if (!Array.isArray(parsed) || parsed.length !== texts.length) {
|
|
174
|
+
throw new Error(`redactor ${name}: expected a JSON array of ${texts.length} strings`);
|
|
175
|
+
}
|
|
176
|
+
return parsed.map(String);
|
|
177
|
+
}
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
|
|
73
181
|
// src/time.ts
|
|
74
182
|
function parseIsoToEpochMs(ts) {
|
|
75
183
|
if (!ts) return 0;
|
|
@@ -1988,6 +2096,28 @@ function redactSpans(spans, rules = TRACES_REDACTION_RULES) {
|
|
|
1988
2096
|
});
|
|
1989
2097
|
return { spans: out, report };
|
|
1990
2098
|
}
|
|
2099
|
+
async function applyRedactor(spans, redactor) {
|
|
2100
|
+
const idx = [];
|
|
2101
|
+
const texts = [];
|
|
2102
|
+
spans.forEach((s, i) => {
|
|
2103
|
+
const c = s.attributes["content"];
|
|
2104
|
+
if (typeof c === "string" && c.length > 0) {
|
|
2105
|
+
idx.push(i);
|
|
2106
|
+
texts.push(c);
|
|
2107
|
+
}
|
|
2108
|
+
});
|
|
2109
|
+
const out = spans.map((s) => ({ ...s, attributes: { ...s.attributes } }));
|
|
2110
|
+
if (texts.length === 0) return { spans: out, changed: 0 };
|
|
2111
|
+
const scrubbed = await redactor.redactText(texts);
|
|
2112
|
+
let changed = 0;
|
|
2113
|
+
idx.forEach((spanI, k) => {
|
|
2114
|
+
if (scrubbed[k] !== texts[k]) {
|
|
2115
|
+
out[spanI].attributes["content"] = scrubbed[k];
|
|
2116
|
+
changed += 1;
|
|
2117
|
+
}
|
|
2118
|
+
});
|
|
2119
|
+
return { spans: out, changed };
|
|
2120
|
+
}
|
|
1991
2121
|
|
|
1992
2122
|
// src/upload-state.ts
|
|
1993
2123
|
import { createHash } from "crypto";
|
|
@@ -2108,7 +2238,12 @@ function stripSpanContent(spans) {
|
|
|
2108
2238
|
});
|
|
2109
2239
|
}
|
|
2110
2240
|
async function executeUpload(plan, opts = {}) {
|
|
2111
|
-
|
|
2241
|
+
let newItems = plan.items.filter((i) => i.isNew);
|
|
2242
|
+
if (opts.redactor && !opts.stripContent) {
|
|
2243
|
+
const r = opts.redactor;
|
|
2244
|
+
newItems = await Promise.all(newItems.map(async (i) => ({ ...i, spans: (await applyRedactor(i.spans, r)).spans })));
|
|
2245
|
+
}
|
|
2246
|
+
if (opts.stripContent) newItems = newItems.map((i) => ({ ...i, spans: stripSpanContent(i.spans) }));
|
|
2112
2247
|
const skipped = plan.items.length - newItems.length;
|
|
2113
2248
|
const redactionCount = newItems.reduce((n, i) => n + i.redaction.redactionCount, 0);
|
|
2114
2249
|
const uploadedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
@@ -2161,6 +2296,11 @@ export {
|
|
|
2161
2296
|
serializeSpans,
|
|
2162
2297
|
writeOtlpFile,
|
|
2163
2298
|
analyzeSpans,
|
|
2299
|
+
runCommand,
|
|
2300
|
+
commandAnalyzer,
|
|
2301
|
+
haloAnalyzer,
|
|
2302
|
+
runExternalAnalyzers,
|
|
2303
|
+
commandRedactor,
|
|
2164
2304
|
parseIsoToEpochMs,
|
|
2165
2305
|
parseSince,
|
|
2166
2306
|
toRuntimeStore,
|
|
@@ -2190,6 +2330,7 @@ export {
|
|
|
2190
2330
|
CODING_REDACTION_RULES,
|
|
2191
2331
|
TRACES_REDACTION_RULES,
|
|
2192
2332
|
redactSpans,
|
|
2333
|
+
applyRedactor,
|
|
2193
2334
|
uploadKey,
|
|
2194
2335
|
sessionHash,
|
|
2195
2336
|
loadState,
|
package/dist/cli.js
CHANGED
|
@@ -1,16 +1,20 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import {
|
|
3
3
|
analyzeSpans,
|
|
4
|
+
commandAnalyzer,
|
|
5
|
+
commandRedactor,
|
|
4
6
|
executeUpload,
|
|
7
|
+
haloAnalyzer,
|
|
5
8
|
knownHarnesses,
|
|
6
9
|
parseSince,
|
|
7
10
|
planUpload,
|
|
8
11
|
resolveAdapter,
|
|
12
|
+
runExternalAnalyzers,
|
|
9
13
|
runPipelines,
|
|
10
14
|
selectAdapters,
|
|
11
15
|
watchSessions,
|
|
12
16
|
writeOtlpFile
|
|
13
|
-
} from "./chunk-
|
|
17
|
+
} from "./chunk-6TVW7PL6.js";
|
|
14
18
|
|
|
15
19
|
// src/cli.ts
|
|
16
20
|
import { stat, writeFile } from "fs/promises";
|
|
@@ -106,7 +110,8 @@ function parseArgs(argv) {
|
|
|
106
110
|
minLoop: 3,
|
|
107
111
|
dryRun: false,
|
|
108
112
|
yes: false,
|
|
109
|
-
noContent: false
|
|
113
|
+
noContent: false,
|
|
114
|
+
analyzers: []
|
|
110
115
|
};
|
|
111
116
|
for (let i = 1; i < argv.length; i++) {
|
|
112
117
|
const arg = argv[i];
|
|
@@ -158,6 +163,17 @@ function parseArgs(argv) {
|
|
|
158
163
|
case "--no-content":
|
|
159
164
|
a.noContent = true;
|
|
160
165
|
break;
|
|
166
|
+
case "--analyzer": {
|
|
167
|
+
const v = next();
|
|
168
|
+
if (v) a.analyzers.push(v);
|
|
169
|
+
break;
|
|
170
|
+
}
|
|
171
|
+
case "--analyzer-prompt":
|
|
172
|
+
a.analyzerPrompt = next();
|
|
173
|
+
break;
|
|
174
|
+
case "--redactor":
|
|
175
|
+
a.redactorCmd = next();
|
|
176
|
+
break;
|
|
161
177
|
case "--yes":
|
|
162
178
|
case "-y":
|
|
163
179
|
a.yes = true;
|
|
@@ -197,10 +213,13 @@ async function discover(args) {
|
|
|
197
213
|
async function buildAxService() {
|
|
198
214
|
const apiKey = process.env.OPENAI_API_KEY;
|
|
199
215
|
if (!apiKey) {
|
|
200
|
-
throw new Error(
|
|
216
|
+
throw new Error(
|
|
217
|
+
"--llm needs OPENAI_API_KEY \u2014 an OpenAI key, or a router/gateway key with OPENAI_BASE_URL set to an OpenAI-compatible endpoint (e.g. the Tangle router). Deterministic analysis needs no key."
|
|
218
|
+
);
|
|
201
219
|
}
|
|
220
|
+
const apiURL = process.env.OPENAI_BASE_URL || void 0;
|
|
202
221
|
const { AxAI } = await import("@ax-llm/ax");
|
|
203
|
-
return new AxAI({ name: "openai", apiKey });
|
|
222
|
+
return new AxAI({ name: "openai", apiKey, ...apiURL ? { apiURL } : {} });
|
|
204
223
|
}
|
|
205
224
|
async function cmdList(args) {
|
|
206
225
|
const groups = await discover({ ...args, last: args.last || 20 });
|
|
@@ -257,8 +276,21 @@ async function cmdAnalyze(args) {
|
|
|
257
276
|
`)
|
|
258
277
|
});
|
|
259
278
|
const pipelines = await runPipelines(spans, { minLoopOccurrences: args.minLoop });
|
|
260
|
-
|
|
279
|
+
let report = `${renderReport(result, { harness, sessionCount, spanCount: spans.length, otlpPath })}
|
|
261
280
|
${renderPipelines(pipelines)}`;
|
|
281
|
+
if (args.analyzers.length > 0 && otlpPath) {
|
|
282
|
+
const engines = args.analyzers.map(
|
|
283
|
+
(spec) => spec === "halo" ? haloAnalyzer({ defaultPrompt: args.analyzerPrompt }) : commandAnalyzer({ name: spec, command: spec, args: (p, prompt) => prompt ? [p, prompt] : [p] })
|
|
284
|
+
);
|
|
285
|
+
const results = await runExternalAnalyzers(otlpPath, engines, { prompt: args.analyzerPrompt });
|
|
286
|
+
for (const r of results) {
|
|
287
|
+
report += `
|
|
288
|
+
|
|
289
|
+
## ${r.analyzer} (external analyzer)
|
|
290
|
+
|
|
291
|
+
${r.ok ? r.output || "(no output)" : `failed: ${r.error}`}`;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
262
294
|
if (args.out) {
|
|
263
295
|
await writeFile(args.out, report, "utf8");
|
|
264
296
|
console.log(`report \u2192 ${args.out} (${result.findings.length} findings, ${pipelines.stuckLoops.findings.length} loops, OTLP: ${otlpPath})`);
|
|
@@ -300,6 +332,8 @@ async function cmdUpload(args) {
|
|
|
300
332
|
cwd: args.cwd,
|
|
301
333
|
sinceMs
|
|
302
334
|
});
|
|
335
|
+
const redactorParts = args.redactorCmd?.split(/\s+/).filter(Boolean) ?? [];
|
|
336
|
+
const redactor = redactorParts[0] ? commandRedactor({ command: redactorParts[0], args: redactorParts.slice(1) }) : void 0;
|
|
303
337
|
const newItems = plan.items.filter((i) => i.isNew);
|
|
304
338
|
const byRule = {};
|
|
305
339
|
let totalRedactions = 0;
|
|
@@ -326,7 +360,7 @@ Window: since ${new Date(sinceMs).toISOString()}
|
|
|
326
360
|
return;
|
|
327
361
|
}
|
|
328
362
|
if (args.dryRun) {
|
|
329
|
-
const res2 = await executeUpload(plan, { dryRun: true, otlpOut: args.otlp, stripContent: args.noContent });
|
|
363
|
+
const res2 = await executeUpload(plan, { dryRun: true, otlpOut: args.otlp, stripContent: args.noContent, redactor });
|
|
330
364
|
console.log(`dry run \u2014 ${newItems.length} session(s), ${totalRedactions} redaction(s). Redacted OTLP \u2192 ${res2.otlpPath}`);
|
|
331
365
|
console.log("No upload performed. Set TANGLE_INGEST_URL / TANGLE_INGEST_API_KEY / TANGLE_TENANT_ID and drop --dry-run to send.");
|
|
332
366
|
return;
|
|
@@ -339,7 +373,7 @@ Window: since ${new Date(sinceMs).toISOString()}
|
|
|
339
373
|
}
|
|
340
374
|
}
|
|
341
375
|
const res = await executeUpload(plan, { log: (m) => w(`${m}
|
|
342
|
-
`), stripContent: args.noContent });
|
|
376
|
+
`), stripContent: args.noContent, redactor });
|
|
343
377
|
console.log(
|
|
344
378
|
`Uploaded ${res.uploadedSessions} session(s), ${res.acceptedSpans} spans accepted, ${res.redactionCount} redaction(s); ${res.skippedSessions} already-uploaded skipped.`
|
|
345
379
|
);
|
|
@@ -363,13 +397,16 @@ Options:
|
|
|
363
397
|
--since <t> upload: window \u2014 30m / 2h / 7d or an ISO date (default 24h); analyze: ISO cutoff
|
|
364
398
|
--out <path> Write report to a file
|
|
365
399
|
--otlp <path> OTLP artifact path (also the dry-run upload preview path)
|
|
366
|
-
--llm Enable agentic RLM analysts (needs OPENAI_API_KEY)
|
|
400
|
+
--llm Enable agentic RLM analysts (needs OPENAI_API_KEY / OPENAI_BASE_URL)
|
|
367
401
|
--budget <usd> USD cap for agentic analysts
|
|
402
|
+
--analyzer <cmd> analyze: also run an external engine over the OTLP (repeatable; "halo" or any command)
|
|
403
|
+
--analyzer-prompt <p> analyze: prompt passed to external analyzers (default: diagnose)
|
|
368
404
|
--interval <s> watch: poll interval seconds (default 5)
|
|
369
405
|
--window <m> watch: only sessions active in the last N minutes (default 30)
|
|
370
406
|
--min-loop <n> Min identical repeated calls to flag a loop (default 3)
|
|
371
407
|
--dry-run upload: redact + dedup + preview, write OTLP, but do NOT send
|
|
372
408
|
--no-content upload: strip prompt/response text \u2014 send metadata only
|
|
409
|
+
--redactor <cmd> upload: external PII scrubber (JSON array stdin\u2192stdout) after the regex pass
|
|
373
410
|
--yes, -y upload: skip the confirmation prompt
|
|
374
411
|
|
|
375
412
|
Upload env: TANGLE_INGEST_URL (or TANGLE_ORCHESTRATOR_URL), TANGLE_INGEST_API_KEY (or TANGLE_API_KEY), TANGLE_TENANT_ID`);
|
package/dist/index.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { ToolUseMetrics, InMemoryTraceStore } from '@tangle-network/agent-eval';
|
|
2
2
|
import { StuckLoopReport } from '@tangle-network/agent-eval/pipelines';
|
|
3
3
|
import { AxAIService } from '@ax-llm/ax';
|
|
4
|
-
import { AnalystRegistry, buildDefaultAnalystRegistry } from '@tangle-network/agent-eval/analyst';
|
|
4
|
+
import { AnalystRegistry, buildDefaultAnalystRegistry, AnalystFinding } from '@tangle-network/agent-eval/analyst';
|
|
5
5
|
export { Analyst, AnalystContext, AnalystFinding, AnalystRegistry, buildDefaultAnalystRegistry, makeFinding } from '@tangle-network/agent-eval/analyst';
|
|
6
6
|
import { RedactionRule, RedactionReport } from '@tangle-network/agent-eval/traces';
|
|
7
7
|
export { DEFAULT_REDACTION_RULES, RedactionReport, RedactionRule, redactString, redactValue } from '@tangle-network/agent-eval/traces';
|
|
@@ -553,6 +553,90 @@ interface AnalyzeResult {
|
|
|
553
553
|
}
|
|
554
554
|
declare function analyzeSpans(spans: readonly OtlpSpan[], opts?: AnalyzeOptions): Promise<AnalyzeResult>;
|
|
555
555
|
|
|
556
|
+
/**
|
|
557
|
+
* External-tool adapters: run analysis engines and PII scrubbers that `traces`
|
|
558
|
+
* does NOT bundle. Each is a thin command adapter over a shared subprocess
|
|
559
|
+
* runner — install the tool yourself (HALO, openai/privacy-filter, anything),
|
|
560
|
+
* point an adapter at it, and it composes with the built-in pipeline.
|
|
561
|
+
*
|
|
562
|
+
* - ExternalAnalyzer — run an engine over the emitted OTLP-JSONL artifact and
|
|
563
|
+
* fold its output into the report (HALO and friends).
|
|
564
|
+
* - Redactor — scrub free-form prompt/response text via an external model that
|
|
565
|
+
* catches what the regex pass can't (names, addresses).
|
|
566
|
+
*/
|
|
567
|
+
|
|
568
|
+
interface RunResult {
|
|
569
|
+
stdout: string;
|
|
570
|
+
stderr: string;
|
|
571
|
+
code: number | null;
|
|
572
|
+
}
|
|
573
|
+
/** Spawn `command args`, optionally write `input` to stdin, and collect output.
|
|
574
|
+
* Rejects on spawn error, timeout, or output exceeding `maxBuffer`. */
|
|
575
|
+
declare function runCommand(command: string, args?: readonly string[], opts?: {
|
|
576
|
+
input?: string;
|
|
577
|
+
signal?: AbortSignal;
|
|
578
|
+
timeoutMs?: number;
|
|
579
|
+
maxBuffer?: number;
|
|
580
|
+
}): Promise<RunResult>;
|
|
581
|
+
interface ExternalAnalysisResult {
|
|
582
|
+
analyzer: string;
|
|
583
|
+
ok: boolean;
|
|
584
|
+
/** Raw text the engine produced (markdown/JSON/plain — engine-specific). */
|
|
585
|
+
output: string;
|
|
586
|
+
/** Structured findings, when the engine emits parseable JSON. */
|
|
587
|
+
findings?: AnalystFinding[];
|
|
588
|
+
error?: string;
|
|
589
|
+
}
|
|
590
|
+
/** An analysis engine that runs over the emitted OTLP-JSONL artifact — a peer to
|
|
591
|
+
* the built-in analysts, so you get many analyzers beyond our own. */
|
|
592
|
+
interface ExternalAnalyzer {
|
|
593
|
+
name: string;
|
|
594
|
+
analyze(otlpPath: string, opts?: {
|
|
595
|
+
prompt?: string;
|
|
596
|
+
signal?: AbortSignal;
|
|
597
|
+
}): Promise<ExternalAnalysisResult>;
|
|
598
|
+
}
|
|
599
|
+
/** Wrap any CLI that reads the OTLP file and prints analysis to stdout. `args`
|
|
600
|
+
* builds the argv from the artifact path + optional prompt. Non-zero exit →
|
|
601
|
+
* `ok:false` with stderr in `error` (fail-soft: one engine never breaks a run). */
|
|
602
|
+
declare function commandAnalyzer(spec: {
|
|
603
|
+
name: string;
|
|
604
|
+
command: string;
|
|
605
|
+
args: (otlpPath: string, prompt?: string) => string[];
|
|
606
|
+
/** Parse stdout into structured findings; omit to keep raw output only. */
|
|
607
|
+
parse?: (stdout: string) => AnalystFinding[] | undefined;
|
|
608
|
+
timeoutMs?: number;
|
|
609
|
+
}): ExternalAnalyzer;
|
|
610
|
+
/** HALO (github.com/context-labs/halo) over the OTLP artifact: `halo <file> -p <prompt>`.
|
|
611
|
+
* Install HALO yourself; this just drives it. */
|
|
612
|
+
declare function haloAnalyzer(opts?: {
|
|
613
|
+
command?: string;
|
|
614
|
+
defaultPrompt?: string;
|
|
615
|
+
timeoutMs?: number;
|
|
616
|
+
}): ExternalAnalyzer;
|
|
617
|
+
/** Run external analyzers over one OTLP artifact, concurrently. Never throws —
|
|
618
|
+
* a failing engine returns `ok:false`. */
|
|
619
|
+
declare function runExternalAnalyzers(otlpPath: string, analyzers: readonly ExternalAnalyzer[], opts?: {
|
|
620
|
+
prompt?: string;
|
|
621
|
+
signal?: AbortSignal;
|
|
622
|
+
}): Promise<ExternalAnalysisResult[]>;
|
|
623
|
+
/** An external PII/secret scrubber for free-form text — catches what regex
|
|
624
|
+
* can't (names, addresses). Use to harden upload beyond the built-in rules. */
|
|
625
|
+
interface Redactor {
|
|
626
|
+
name: string;
|
|
627
|
+
/** Scrub a batch of texts; returns one scrubbed string per input, in order. */
|
|
628
|
+
redactText(texts: readonly string[]): Promise<string[]>;
|
|
629
|
+
}
|
|
630
|
+
/** Wrap any CLI that speaks the redaction protocol: read a JSON array of strings
|
|
631
|
+
* on stdin, write a same-length JSON array of scrubbed strings on stdout.
|
|
632
|
+
* (A 3-line wrapper adapts tools like `opf`/openai-privacy-filter.) */
|
|
633
|
+
declare function commandRedactor(spec: {
|
|
634
|
+
name?: string;
|
|
635
|
+
command: string;
|
|
636
|
+
args?: string[];
|
|
637
|
+
timeoutMs?: number;
|
|
638
|
+
}): Redactor;
|
|
639
|
+
|
|
556
640
|
/**
|
|
557
641
|
* Live trace observer — the reusable, event-driven core behind `traces watch`.
|
|
558
642
|
*
|
|
@@ -642,6 +726,14 @@ interface SpanRedaction {
|
|
|
642
726
|
/** Redact every span's attributes + status message. Returns scrubbed spans and
|
|
643
727
|
* an aggregate report (count + per-rule breakdown) over the whole set. */
|
|
644
728
|
declare function redactSpans(spans: readonly OtlpSpan[], rules?: RedactionRule[]): SpanRedaction;
|
|
729
|
+
/** Defense-in-depth: run an external {@link Redactor} (e.g. an ML PII model) over
|
|
730
|
+
* every span's captured `content` (prompt/response prose), catching free-form
|
|
731
|
+
* PII the regex pass misses. Compose AFTER `redactSpans`. Returns scrubbed spans
|
|
732
|
+
* and the number of `content` fields the model changed. */
|
|
733
|
+
declare function applyRedactor(spans: readonly OtlpSpan[], redactor: Redactor): Promise<{
|
|
734
|
+
spans: OtlpSpan[];
|
|
735
|
+
changed: number;
|
|
736
|
+
}>;
|
|
645
737
|
|
|
646
738
|
/**
|
|
647
739
|
* Batch collection seam — `collectSessions` returns redacted spans per session
|
|
@@ -750,9 +842,12 @@ interface ExecuteOptions {
|
|
|
750
842
|
* span before it leaves the machine (still keeps tool calls, tokens, timing,
|
|
751
843
|
* loop signal). The strongest privacy posture when prose can't leave. */
|
|
752
844
|
stripContent?: boolean;
|
|
845
|
+
/** External PII scrubber applied to `content` AFTER the regex pass — catches
|
|
846
|
+
* free-form PII (names, addresses) the rules miss. Ignored if stripContent. */
|
|
847
|
+
redactor?: Redactor;
|
|
753
848
|
log?: (msg: string) => void;
|
|
754
849
|
}
|
|
755
850
|
/** Send the NEW items (or, on dryRun, write the redacted OTLP that would send). */
|
|
756
851
|
declare function executeUpload(plan: UploadPlan, opts?: ExecuteOptions): Promise<UploadResult>;
|
|
757
852
|
|
|
758
|
-
export { ATTR, AmpAdapter, type AnalyzeOptions, type AnalyzeResult, CODING_REDACTION_RULES, CONTENT_CAP, ClaudeAdapter, CodexAdapter, type CollectOptions, CopilotAdapter, DEFAULT_HARNESS, type ExecuteOptions, FactoryAdapter, ForgeAdapter, GeminiAdapter, type HarnessTraceAdapter, INGEST_SOURCE_CLI, type LocateOptions, type ObservedLoop, type ObserverOptions, OpencodeAdapter, type OtlpSpan, type OtlpSpanKind, type OtlpStatusCode, PiAdapter, type PipelineOptions, type PipelineReport, type PlanOptions, QwenAdapter, type RuntimeTrace, type ScanOptions, type ScannedSession, type SessionBatch, type SessionRef, type SpanInput, type SpanRedaction, TRACES_REDACTION_RULES, type UploadBackend, type UploadItem, type UploadPlan, type UploadRecord, type UploadResult, type UploadState, alreadyUploaded, analyzeSpans, capText, collectSessions, executeUpload, knownHarnesses, listAdapters, loadState, parseIsoToEpochMs, parseSince, planUpload, redactSpans, resolveAdapter, runPipelines, saveState, scanSessions, selectAdapters, serializeSpans, sessionHash, span, toRuntimeStore, toTraceSpanEvents, uploadKey, userPromptSpan, watchSessions, writeOtlpFile };
|
|
853
|
+
export { ATTR, AmpAdapter, type AnalyzeOptions, type AnalyzeResult, CODING_REDACTION_RULES, CONTENT_CAP, ClaudeAdapter, CodexAdapter, type CollectOptions, CopilotAdapter, DEFAULT_HARNESS, type ExecuteOptions, type ExternalAnalysisResult, type ExternalAnalyzer, FactoryAdapter, ForgeAdapter, GeminiAdapter, type HarnessTraceAdapter, INGEST_SOURCE_CLI, type LocateOptions, type ObservedLoop, type ObserverOptions, OpencodeAdapter, type OtlpSpan, type OtlpSpanKind, type OtlpStatusCode, PiAdapter, type PipelineOptions, type PipelineReport, type PlanOptions, QwenAdapter, type Redactor, type RunResult, type RuntimeTrace, type ScanOptions, type ScannedSession, type SessionBatch, type SessionRef, type SpanInput, type SpanRedaction, TRACES_REDACTION_RULES, type UploadBackend, type UploadItem, type UploadPlan, type UploadRecord, type UploadResult, type UploadState, alreadyUploaded, analyzeSpans, applyRedactor, capText, collectSessions, commandAnalyzer, commandRedactor, executeUpload, haloAnalyzer, knownHarnesses, listAdapters, loadState, parseIsoToEpochMs, parseSince, planUpload, redactSpans, resolveAdapter, runCommand, runExternalAnalyzers, runPipelines, saveState, scanSessions, selectAdapters, serializeSpans, sessionHash, span, toRuntimeStore, toTraceSpanEvents, uploadKey, userPromptSpan, watchSessions, writeOtlpFile };
|
package/dist/index.js
CHANGED
|
@@ -17,8 +17,12 @@ import {
|
|
|
17
17
|
TRACES_REDACTION_RULES,
|
|
18
18
|
alreadyUploaded,
|
|
19
19
|
analyzeSpans,
|
|
20
|
+
applyRedactor,
|
|
20
21
|
capText,
|
|
22
|
+
commandAnalyzer,
|
|
23
|
+
commandRedactor,
|
|
21
24
|
executeUpload,
|
|
25
|
+
haloAnalyzer,
|
|
22
26
|
knownHarnesses,
|
|
23
27
|
listAdapters,
|
|
24
28
|
loadState,
|
|
@@ -27,6 +31,8 @@ import {
|
|
|
27
31
|
planUpload,
|
|
28
32
|
redactSpans,
|
|
29
33
|
resolveAdapter,
|
|
34
|
+
runCommand,
|
|
35
|
+
runExternalAnalyzers,
|
|
30
36
|
runPipelines,
|
|
31
37
|
saveState,
|
|
32
38
|
scanSessions,
|
|
@@ -40,7 +46,7 @@ import {
|
|
|
40
46
|
userPromptSpan,
|
|
41
47
|
watchSessions,
|
|
42
48
|
writeOtlpFile
|
|
43
|
-
} from "./chunk-
|
|
49
|
+
} from "./chunk-6TVW7PL6.js";
|
|
44
50
|
|
|
45
51
|
// src/collect.ts
|
|
46
52
|
async function collectSessions(opts = {}) {
|
|
@@ -85,11 +91,15 @@ export {
|
|
|
85
91
|
TRACES_REDACTION_RULES,
|
|
86
92
|
alreadyUploaded,
|
|
87
93
|
analyzeSpans,
|
|
94
|
+
applyRedactor,
|
|
88
95
|
buildDefaultAnalystRegistry,
|
|
89
96
|
capText,
|
|
90
97
|
collectSessions,
|
|
98
|
+
commandAnalyzer,
|
|
99
|
+
commandRedactor,
|
|
91
100
|
createHostedClient,
|
|
92
101
|
executeUpload,
|
|
102
|
+
haloAnalyzer,
|
|
93
103
|
hostedClientFromEnv,
|
|
94
104
|
knownHarnesses,
|
|
95
105
|
listAdapters,
|
|
@@ -102,6 +112,8 @@ export {
|
|
|
102
112
|
redactString,
|
|
103
113
|
redactValue,
|
|
104
114
|
resolveAdapter,
|
|
115
|
+
runCommand,
|
|
116
|
+
runExternalAnalyzers,
|
|
105
117
|
runPipelines,
|
|
106
118
|
saveState,
|
|
107
119
|
scanSessions,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/traces",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "Point it at your coding-agent session traces (Claude Code, Codex, OpenCode, Gemini, Pi, …) and get failure-mode + efficiency findings. CLI + SDK over the @tangle-network/agent-eval analyst suite — observe live sessions, run your own analysts, redact, and upload to the Tangle Intelligence Platform.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|