@tangle-network/traces 0.8.25 → 0.8.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-SV7GW5LB.js → chunk-6N2KVDOU.js} +90 -5
- package/dist/cli.js +1 -1
- package/dist/index.d.ts +5 -2
- package/dist/index.js +1 -1
- package/package.json +1 -1
|
@@ -297,7 +297,9 @@ import { computeToolUseMetrics } from "@tangle-network/agent-eval";
|
|
|
297
297
|
import { stuckLoopView } from "@tangle-network/agent-eval/pipelines";
|
|
298
298
|
async function runPipelines(spans, opts = {}) {
|
|
299
299
|
const { store, runIds } = await toRuntimeStore(spans);
|
|
300
|
-
const
|
|
300
|
+
const loopEligible = spans.filter((item) => item.attributes["traces.expected_blocking"] !== true);
|
|
301
|
+
const { store: loopStore } = await toRuntimeStore(loopEligible);
|
|
302
|
+
const stuckLoops = await stuckLoopView(loopStore, { minOccurrences: opts.minLoopOccurrences ?? 3 });
|
|
301
303
|
const toolUse = await Promise.all(runIds.map((runId) => computeToolUseMetrics(store, runId)));
|
|
302
304
|
return { stuckLoops, toolUse };
|
|
303
305
|
}
|
|
@@ -808,6 +810,33 @@ function outputIsError(output) {
|
|
|
808
810
|
const error = /"success"\s*:\s*false|exit(?:_| )code["\s:]+[1-9]|\bcommand failed\b|\bENOENT\b|\berror:/i.test(s);
|
|
809
811
|
return { error, message: error ? s.slice(0, 500) : "" };
|
|
810
812
|
}
|
|
813
|
+
function singleNestedToolName(input) {
|
|
814
|
+
if (!input) return null;
|
|
815
|
+
const names = [...input.matchAll(/\btools\.([A-Za-z][A-Za-z0-9_]*)\s*\(/g)].map((match) => match[1]);
|
|
816
|
+
const unique = [...new Set(names)];
|
|
817
|
+
return unique.length === 1 ? unique[0] : null;
|
|
818
|
+
}
|
|
819
|
+
function toolInputToString(input) {
|
|
820
|
+
if (typeof input === "string") return input;
|
|
821
|
+
if (input == null) return void 0;
|
|
822
|
+
return JSON.stringify(input);
|
|
823
|
+
}
|
|
824
|
+
var verificationCommand = /\b(?:pnpm|npm|yarn|bun)\s+(?:run\s+)?(?:test|typecheck|lint|build|check)(?::[A-Za-z0-9:_-]+)?\b|\b(?:vitest|jest|pytest|tsc|biome|eslint|sha256sum|pdfinfo|pdftotext)\b|\bgo\s+test\b|\bcargo\s+(?:test|check|clippy|build)\b|\bgit\s+(?:status|diff|show|merge-tree)\b|\bgh-drew\s+pr\s+(?:view|checks)\b/i;
|
|
825
|
+
function hasReadOnlyCurl(input) {
|
|
826
|
+
if (!/\bcurl\b/i.test(input)) return false;
|
|
827
|
+
const method = input.match(/(?:^|\s)-X(?:=|\s*)([A-Za-z]+)\b/)?.[1] ?? input.match(/(?:^|\s)--request(?:=|\s+)([A-Za-z]+)\b/i)?.[1];
|
|
828
|
+
if (method && !/^(?:GET|HEAD)$/i.test(method)) return false;
|
|
829
|
+
return !/(?:^|\s)-(?:d|F|T)(?:\S*|\s+\S+)|(?:^|\s)--(?:data(?:-ascii|-binary|-raw|-urlencode)?|form(?:-string)?|json|upload-file)(?:=|\s)/i.test(input);
|
|
830
|
+
}
|
|
831
|
+
function classifyNestedTool(name, input) {
|
|
832
|
+
return name === "exec_command" && input && (verificationCommand.test(input) || hasReadOnlyCurl(input)) ? "exec_command.verify" : name;
|
|
833
|
+
}
|
|
834
|
+
function isExpectedBlockingTool(name, input) {
|
|
835
|
+
if (!input) return false;
|
|
836
|
+
if (name === "wait") return /\bcell_id\b["']?\s*:/.test(input);
|
|
837
|
+
if (name === "write_stdin") return /\bsession_id\b["']?\s*:/.test(input);
|
|
838
|
+
return false;
|
|
839
|
+
}
|
|
811
840
|
async function* walkRollouts(root) {
|
|
812
841
|
let years;
|
|
813
842
|
try {
|
|
@@ -901,6 +930,7 @@ var CodexAdapter = class {
|
|
|
901
930
|
})
|
|
902
931
|
];
|
|
903
932
|
const toolByCallId = /* @__PURE__ */ new Map();
|
|
933
|
+
const subagentByThreadId = /* @__PURE__ */ new Map();
|
|
904
934
|
let step = 0;
|
|
905
935
|
let lastLlm = rootId;
|
|
906
936
|
let sawUserTurn = false;
|
|
@@ -929,9 +959,14 @@ var CodexAdapter = class {
|
|
|
929
959
|
lastLlm = id;
|
|
930
960
|
step += 1;
|
|
931
961
|
}
|
|
932
|
-
} else if (l.type === "response_item" && l.payload?.type === "function_call") {
|
|
933
|
-
const
|
|
962
|
+
} else if (l.type === "response_item" && (l.payload?.type === "function_call" || l.payload?.type === "custom_tool_call")) {
|
|
963
|
+
const outerName = l.payload.name ?? "tool";
|
|
934
964
|
const callId = l.payload.call_id ?? `${step}`;
|
|
965
|
+
const input = toolInputToString(
|
|
966
|
+
l.payload.type === "custom_tool_call" ? l.payload.input : l.payload.arguments
|
|
967
|
+
);
|
|
968
|
+
const nestedName = l.payload.type === "custom_tool_call" ? singleNestedToolName(input) : null;
|
|
969
|
+
const name = classifyNestedTool(nestedName ?? outerName, input);
|
|
935
970
|
const toolSpan = span({
|
|
936
971
|
traceId,
|
|
937
972
|
spanId: `tool:${callId}`,
|
|
@@ -943,18 +978,68 @@ var CodexAdapter = class {
|
|
|
943
978
|
agent: SERVICE3,
|
|
944
979
|
tool: name,
|
|
945
980
|
step,
|
|
946
|
-
content:
|
|
981
|
+
content: input ?? null,
|
|
982
|
+
extra: {
|
|
983
|
+
"traces.codex.call_type": l.payload.type,
|
|
984
|
+
...name !== outerName ? { "traces.codex.outer_tool_name": outerName } : {},
|
|
985
|
+
...nestedName ? { "traces.codex.nested_tool_name": nestedName } : {},
|
|
986
|
+
...isExpectedBlockingTool(name, input) ? { "traces.expected_blocking": true } : {}
|
|
987
|
+
}
|
|
947
988
|
});
|
|
948
989
|
spans.push(toolSpan);
|
|
949
990
|
toolByCallId.set(callId, toolSpan);
|
|
950
991
|
step += 1;
|
|
951
|
-
} else if (l.type === "response_item" && l.payload?.type === "function_call_output") {
|
|
992
|
+
} else if (l.type === "response_item" && (l.payload?.type === "function_call_output" || l.payload?.type === "custom_tool_call_output")) {
|
|
952
993
|
const t = toolByCallId.get(l.payload.call_id ?? "");
|
|
953
994
|
if (t) {
|
|
954
995
|
const { error, message } = outputIsError(l.payload.output);
|
|
955
996
|
t.end_time = ts;
|
|
956
997
|
t.status = error ? { code: "ERROR", message } : { code: "OK" };
|
|
957
998
|
}
|
|
999
|
+
} else if (l.type === "event_msg" && l.payload?.type === "sub_agent_activity") {
|
|
1000
|
+
const threadId = l.payload.agent_thread_id;
|
|
1001
|
+
const occurredAtMs = l.payload.occurred_at_ms;
|
|
1002
|
+
const eventTime = typeof occurredAtMs === "number" && Number.isFinite(occurredAtMs) ? new Date(occurredAtMs).toISOString() : ts;
|
|
1003
|
+
if (l.payload.kind === "started" && threadId && !subagentByThreadId.has(threadId)) {
|
|
1004
|
+
const agentPath = l.payload.agent_path ?? "subagent";
|
|
1005
|
+
const subagentType = agentPath.split("/").filter(Boolean).at(-1) ?? "subagent";
|
|
1006
|
+
const toolSpan = span({
|
|
1007
|
+
traceId,
|
|
1008
|
+
spanId: `subagent:${threadId}`,
|
|
1009
|
+
parentSpanId: lastLlm,
|
|
1010
|
+
name: "tool.Agent",
|
|
1011
|
+
kind: "TOOL",
|
|
1012
|
+
startTime: eventTime,
|
|
1013
|
+
service: SERVICE3,
|
|
1014
|
+
agent: SERVICE3,
|
|
1015
|
+
tool: "Agent",
|
|
1016
|
+
step,
|
|
1017
|
+
content: JSON.stringify({
|
|
1018
|
+
subagent_type: subagentType,
|
|
1019
|
+
agent_path: agentPath,
|
|
1020
|
+
agent_thread_id: threadId
|
|
1021
|
+
}),
|
|
1022
|
+
extra: {
|
|
1023
|
+
"traces.codex.subagent_path": agentPath,
|
|
1024
|
+
"traces.codex.subagent_thread_id": threadId
|
|
1025
|
+
}
|
|
1026
|
+
});
|
|
1027
|
+
spans.push(toolSpan);
|
|
1028
|
+
subagentByThreadId.set(threadId, toolSpan);
|
|
1029
|
+
step += 1;
|
|
1030
|
+
} else if (l.payload.kind === "completed" && threadId) {
|
|
1031
|
+
const toolSpan = subagentByThreadId.get(threadId);
|
|
1032
|
+
if (toolSpan) {
|
|
1033
|
+
toolSpan.end_time = eventTime;
|
|
1034
|
+
toolSpan.status = { code: "OK" };
|
|
1035
|
+
}
|
|
1036
|
+
} else if (["interrupted", "failed", "timed_out"].includes(l.payload.kind ?? "") && threadId) {
|
|
1037
|
+
const toolSpan = subagentByThreadId.get(threadId);
|
|
1038
|
+
if (toolSpan) {
|
|
1039
|
+
toolSpan.end_time = eventTime;
|
|
1040
|
+
toolSpan.status = { code: "ERROR", message: `subagent ${l.payload.kind}` };
|
|
1041
|
+
}
|
|
1042
|
+
}
|
|
958
1043
|
} else if (l.type === "response_item" && l.payload?.type === "message" && l.payload.role === "user") {
|
|
959
1044
|
const prompt = textOf3(l.payload.content);
|
|
960
1045
|
if (prompt) {
|
package/dist/cli.js
CHANGED
package/dist/index.d.ts
CHANGED
|
@@ -314,7 +314,8 @@ declare class ClaudeAdapter implements HarnessTraceAdapter {
|
|
|
314
314
|
* Line types: `session_meta` (id + cwd), `turn_context`, `event_msg`
|
|
315
315
|
* (carries `token_count` with per-turn `last_token_usage`), and
|
|
316
316
|
* `response_item` (the OpenAI Responses items: `message`, `reasoning`,
|
|
317
|
-
*
|
|
317
|
+
* function/custom tool calls, and their outputs). Current Codex builds also
|
|
318
|
+
* emit `sub_agent_activity` events for delegated agents.
|
|
318
319
|
*
|
|
319
320
|
* Token trajectory comes from the `token_count` deltas (Codex puts usage
|
|
320
321
|
* on events, not on the message). Tools come from `function_call`, with
|
|
@@ -629,7 +630,9 @@ declare const CORRECTIVE_REACTIONS: readonly Reaction[];
|
|
|
629
630
|
* error rates above, so adding it would only emit a misleading waste %.
|
|
630
631
|
*
|
|
631
632
|
* Both are cheap ($0, deterministic), so they're safe to run continuously in
|
|
632
|
-
* `watch` mode, over the OTLP spans traces already produces.
|
|
633
|
+
* `watch` mode, over the OTLP spans traces already produces. Calls explicitly
|
|
634
|
+
* marked as expected blocking stay in usage totals but are excluded from
|
|
635
|
+
* stuck-loop findings.
|
|
633
636
|
*/
|
|
634
637
|
|
|
635
638
|
interface PipelineReport {
|
package/dist/index.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tangle-network/traces",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.26",
|
|
4
4
|
"description": "Point it at your coding-agent session traces (Claude Code, Codex, OpenCode, Gemini, Pi, …) and get failure-mode + efficiency findings. CLI + SDK over the @tangle-network/agent-eval analyst suite — observe live sessions, run your own analysts, redact, and upload to the Tangle Intelligence Platform.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|