@senpi/runtime 3.0.180 → 3.0.181
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/actions/llm-decision/openclaw-llm-decision.d.ts +8 -0
- package/dist/actions/llm-decision/openclaw-llm-decision.d.ts.map +1 -1
- package/dist/actions/llm-decision/openclaw-llm-decision.js +56 -1
- package/dist/actions/llm-decision/openclaw-llm-decision.js.map +1 -1
- package/dist/approvals/approval-store.d.ts +37 -7
- package/dist/approvals/approval-store.d.ts.map +1 -1
- package/dist/approvals/approval-store.js +127 -23
- package/dist/approvals/approval-store.js.map +1 -1
- package/dist/approvals/telemetry.d.ts +27 -3
- package/dist/approvals/telemetry.d.ts.map +1 -1
- package/dist/approvals/telemetry.js +8 -4
- package/dist/approvals/telemetry.js.map +1 -1
- package/dist/approvals/tool-approval-gate.d.ts +79 -0
- package/dist/approvals/tool-approval-gate.d.ts.map +1 -1
- package/dist/approvals/tool-approval-gate.js +184 -15
- package/dist/approvals/tool-approval-gate.js.map +1 -1
- package/dist/approvals/turn-context.d.ts +271 -0
- package/dist/approvals/turn-context.d.ts.map +1 -0
- package/dist/approvals/turn-context.js +551 -0
- package/dist/approvals/turn-context.js.map +1 -0
- package/dist/cli/guide-commands.d.ts.map +1 -1
- package/dist/cli/guide-commands.js +17 -3
- package/dist/cli/guide-commands.js.map +1 -1
- package/dist/cli/senpi-commands.d.ts.map +1 -1
- package/dist/cli/senpi-commands.js +71 -4
- package/dist/cli/senpi-commands.js.map +1 -1
- package/dist/deploy/job.d.ts +32 -0
- package/dist/deploy/job.d.ts.map +1 -1
- package/dist/deploy/job.js +72 -7
- package/dist/deploy/job.js.map +1 -1
- package/dist/deploy/register.d.ts +11 -0
- package/dist/deploy/register.d.ts.map +1 -1
- package/dist/deploy/register.js +84 -2
- package/dist/deploy/register.js.map +1 -1
- package/dist/host/openclaw-host.d.ts +138 -0
- package/dist/host/openclaw-host.d.ts.map +1 -0
- package/dist/host/openclaw-host.js +177 -0
- package/dist/host/openclaw-host.js.map +1 -0
- package/dist/index.d.ts +46 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +220 -12
- package/dist/index.js.map +1 -1
- package/dist/runtime/auto-update/coordinator.d.ts +203 -0
- package/dist/runtime/auto-update/coordinator.d.ts.map +1 -1
- package/dist/runtime/auto-update/coordinator.js +621 -12
- package/dist/runtime/auto-update/coordinator.js.map +1 -1
- package/dist/runtime/auto-update/gateway-restart.d.ts +60 -1
- package/dist/runtime/auto-update/gateway-restart.d.ts.map +1 -1
- package/dist/runtime/auto-update/gateway-restart.js +95 -21
- package/dist/runtime/auto-update/gateway-restart.js.map +1 -1
- package/dist/runtime/auto-update/idle-gate.d.ts +139 -0
- package/dist/runtime/auto-update/idle-gate.d.ts.map +1 -0
- package/dist/runtime/auto-update/idle-gate.js +217 -0
- package/dist/runtime/auto-update/idle-gate.js.map +1 -0
- package/dist/runtime/auto-update/install-results.d.ts +130 -0
- package/dist/runtime/auto-update/install-results.d.ts.map +1 -0
- package/dist/runtime/auto-update/install-results.js +264 -0
- package/dist/runtime/auto-update/install-results.js.map +1 -0
- package/dist/runtime/auto-update/money-call-tracker.d.ts +36 -0
- package/dist/runtime/auto-update/money-call-tracker.d.ts.map +1 -0
- package/dist/runtime/auto-update/money-call-tracker.js +129 -0
- package/dist/runtime/auto-update/money-call-tracker.js.map +1 -0
- package/dist/runtime/auto-update/register-idle-gate.d.ts +70 -0
- package/dist/runtime/auto-update/register-idle-gate.d.ts.map +1 -0
- package/dist/runtime/auto-update/register-idle-gate.js +123 -0
- package/dist/runtime/auto-update/register-idle-gate.js.map +1 -0
- package/dist/runtime/auto-update/run-update-command.d.ts +13 -0
- package/dist/runtime/auto-update/run-update-command.d.ts.map +1 -1
- package/dist/runtime/auto-update/run-update-command.js +18 -0
- package/dist/runtime/auto-update/run-update-command.js.map +1 -1
- package/dist/runtime/auto-update/state-store.d.ts +14 -1
- package/dist/runtime/auto-update/state-store.d.ts.map +1 -1
- package/dist/runtime/auto-update/state-store.js +37 -3
- package/dist/runtime/auto-update/state-store.js.map +1 -1
- package/dist/runtime/auto-update/types.d.ts +51 -0
- package/dist/runtime/auto-update/types.d.ts.map +1 -1
- package/dist/runtime/auto-update/types.js.map +1 -1
- package/dist/runtime/auto-update/update-marker.d.ts +95 -0
- package/dist/runtime/auto-update/update-marker.d.ts.map +1 -0
- package/dist/runtime/auto-update/update-marker.js +185 -0
- package/dist/runtime/auto-update/update-marker.js.map +1 -0
- package/dist/runtime/auto-update/update-schedule.d.ts +57 -0
- package/dist/runtime/auto-update/update-schedule.d.ts.map +1 -0
- package/dist/runtime/auto-update/update-schedule.js +75 -0
- package/dist/runtime/auto-update/update-schedule.js.map +1 -0
- package/dist/runtime/decision-errors.d.ts +7 -0
- package/dist/runtime/decision-errors.d.ts.map +1 -1
- package/dist/runtime/decision-errors.js +15 -0
- package/dist/runtime/decision-errors.js.map +1 -1
- package/dist/senpi/client.d.ts +25 -0
- package/dist/senpi/client.d.ts.map +1 -1
- package/dist/senpi/client.js +56 -0
- package/dist/senpi/client.js.map +1 -1
- package/dist/telemetry/metrics.d.ts +14 -0
- package/dist/telemetry/metrics.d.ts.map +1 -1
- package/dist/telemetry/metrics.js +24 -0
- package/dist/telemetry/metrics.js.map +1 -1
- package/dist/telemetry/otel-sdk.d.ts +12 -0
- package/dist/telemetry/otel-sdk.d.ts.map +1 -1
- package/dist/telemetry/otel-sdk.js +47 -1
- package/dist/telemetry/otel-sdk.js.map +1 -1
- package/dist/telemetry/spans/span-hooks.d.ts +18 -1
- package/dist/telemetry/spans/span-hooks.d.ts.map +1 -1
- package/dist/telemetry/spans/span-hooks.js +306 -96
- package/dist/telemetry/spans/span-hooks.js.map +1 -1
- package/dist/telemetry/tool-plane.d.ts +2 -0
- package/dist/telemetry/tool-plane.d.ts.map +1 -1
- package/dist/telemetry/tool-plane.js +1 -0
- package/dist/telemetry/tool-plane.js.map +1 -1
- package/dist/tool-surface/crib-cache.d.ts +21 -0
- package/dist/tool-surface/crib-cache.d.ts.map +1 -0
- package/dist/tool-surface/crib-cache.js +54 -0
- package/dist/tool-surface/crib-cache.js.map +1 -0
- package/dist/tool-surface/crib-hook.d.ts +78 -0
- package/dist/tool-surface/crib-hook.d.ts.map +1 -0
- package/dist/tool-surface/crib-hook.js +217 -0
- package/dist/tool-surface/crib-hook.js.map +1 -0
- package/dist/tool-surface/crib-sheet.d.ts +56 -0
- package/dist/tool-surface/crib-sheet.d.ts.map +1 -0
- package/dist/tool-surface/crib-sheet.js +129 -0
- package/dist/tool-surface/crib-sheet.js.map +1 -0
- package/dist/tool-surface/mcp-tool-source.d.ts +64 -0
- package/dist/tool-surface/mcp-tool-source.d.ts.map +1 -0
- package/dist/tool-surface/mcp-tool-source.js +189 -0
- package/dist/tool-surface/mcp-tool-source.js.map +1 -0
- package/dist/utils/event-catalog.d.ts +117 -5
- package/dist/utils/event-catalog.d.ts.map +1 -1
- package/dist/utils/event-catalog.js +19 -0
- package/dist/utils/event-catalog.js.map +1 -1
- package/dist/widgets/tool.d.ts +9 -0
- package/dist/widgets/tool.d.ts.map +1 -1
- package/dist/widgets/tool.js +1 -0
- package/dist/widgets/tool.js.map +1 -1
- package/openclaw.plugin.json +11 -1
- package/package.json +5 -2
|
@@ -8,6 +8,21 @@
|
|
|
8
8
|
* - subagent_* lifecycle → `invoke_agent` (INTERNAL), parented to the model span
|
|
9
9
|
* - agent_end → ends any still-open spans for the run
|
|
10
10
|
*
|
|
11
|
+
* On OpenClaw 10.x (`modernHost`):
|
|
12
|
+
* - there is no `subagent_spawning` hook, so `subagent_spawned` opens the `invoke_agent` span;
|
|
13
|
+
* - the tool-search tools (`tool_search`, `tool_describe`, and the dispatcher `dispatch_action`,
|
|
14
|
+
* `tool_call` on 10.4) get `execute_tool` spans with `senpi.tool.bridge = true` and count on
|
|
15
|
+
* `tool.bridge.calls`, not `tool.calls`. The inner tool a dispatcher runs (id
|
|
16
|
+
* `tool_call:<sanitized outer id>:…`) nests under the dispatcher's span. Its own
|
|
17
|
+
* `after_tool_call` normally closes it (10.10 `embedded-agent-subscribe.handlers.tools.completion.ts:603-626`);
|
|
18
|
+
* any inner span still open when the dispatcher's `after_tool_call` fires is closed there with
|
|
19
|
+
* the outer outcome, and a late inner `after_tool_call` for it is then ignored, so each inner call
|
|
20
|
+
* is counted and evented once under its own name.
|
|
21
|
+
*
|
|
22
|
+
* On both hosts each `chat` span carries `senpi.llm.calls` (the run's `model_call_ended` count),
|
|
23
|
+
* and the drift alarm skips any unoffered name the host ran in the run (a direct call to a deferred
|
|
24
|
+
* tool is legal in 10.x tool-search directory mode).
|
|
25
|
+
*
|
|
11
26
|
* Open and finalize are separate callbacks, so we can't use `withSpan`; we `startSpan`
|
|
12
27
|
* and hold the handle in a session-keyed registry until the finalize hook, recording
|
|
13
28
|
* errors directly on the held span (it's never the active span, so `logger.error` won't).
|
|
@@ -21,7 +36,7 @@ import { SpanKind, context as otelContext, propagation, trace, } from "@opentele
|
|
|
21
36
|
import { getTracer } from "../../telemetry/tracing.js";
|
|
22
37
|
import { HEADER_ADDRESS, HEADER_CORRELATION_ID, HEADER_RECIPE_NAME, HEADER_RUNTIME_ID, HEADER_RUNTIME_VERSION, } from "./constants.js";
|
|
23
38
|
import { GEN_AI, errorTypeOf, firstNumber, normalizeGenAiMessages, recordSpanError, setDefinedAttributes, setGenAiContentAttribute, toFinishReasons, } from "./gen-ai.js";
|
|
24
|
-
import { elapsedSeconds, recordToolCall, recordInvokeAgentDuration, recordOperationDuration, recordTokenUsage, } from "../../telemetry/metrics.js";
|
|
39
|
+
import { elapsedSeconds, recordToolBridgeCall, recordToolCall, recordInvokeAgentDuration, recordOperationDuration, recordTokenUsage, } from "../../telemetry/metrics.js";
|
|
25
40
|
import { emitToolCalled, offeredToolNames, requestedToolNames, } from "../../telemetry/tool-plane.js";
|
|
26
41
|
import { createLogger } from "../../utils/logger.js";
|
|
27
42
|
const logger = createLogger("[senpi-core][span-hooks]");
|
|
@@ -30,6 +45,23 @@ const logger = createLogger("[senpi-core][span-hooks]");
|
|
|
30
45
|
* `skill_router.decision` event shares (`src/utils/event-catalog.ts`). Distinct from `senpi.run.id`.
|
|
31
46
|
*/
|
|
32
47
|
const OPENCLAW_RUN_ID_ATTR = "senpi.openclaw.run_id";
|
|
48
|
+
/** The tool-search bridge tools and their `senpi.tool.bridge_kind` (10.10 `src/agents/tool-search-types.ts:10-12`; `tool_call` is the 10.4 dispatcher). */
|
|
49
|
+
const BRIDGE_TOOL_KINDS = {
|
|
50
|
+
tool_search: "search",
|
|
51
|
+
tool_describe: "describe",
|
|
52
|
+
dispatch_action: "call",
|
|
53
|
+
tool_call: "call",
|
|
54
|
+
};
|
|
55
|
+
/**
|
|
56
|
+
* Port of OpenClaw's inner-call id sanitizer
|
|
57
|
+
* (`openclaw-worktrees/v2026.10.10-beta.3/src/agents/tool-search-runtime.ts:211-214`): the inner
|
|
58
|
+
* call a dispatcher runs gets the id `tool_call:${sanitizeToolCallIdPart(outerToolCallId)}:…`
|
|
59
|
+
* (the literal `tool_call:` prefix is kept on 10.10, where only the dispatcher was renamed).
|
|
60
|
+
*/
|
|
61
|
+
export function sanitizeToolCallIdPart(value) {
|
|
62
|
+
const safe = value.trim().replace(/[^A-Za-z0-9_.:-]+/g, "_");
|
|
63
|
+
return safe.slice(0, 120) || "call";
|
|
64
|
+
}
|
|
33
65
|
/**
|
|
34
66
|
* Grace window the `agent_end` sweep holds a finished model span before ending it, so a
|
|
35
67
|
* trailing `llm_output` (observed ~1 ms later in the codex/harness flow, but timed generously)
|
|
@@ -37,9 +69,20 @@ const OPENCLAW_RUN_ID_ATTR = "senpi.openclaw.run_id";
|
|
|
37
69
|
* process open; an orphan run with no `llm_output` lets the timer end the span.
|
|
38
70
|
*/
|
|
39
71
|
export const MODEL_SPAN_END_GRACE_MS = 2000;
|
|
72
|
+
/**
|
|
73
|
+
* Registers the span hooks.
|
|
74
|
+
*
|
|
75
|
+
* @returns How many hooks were registered (10 on 5.x; 9 on 10.x, which has no `subagent_spawning`).
|
|
76
|
+
*/
|
|
40
77
|
export function registerSpanHooks(options) {
|
|
41
78
|
const { api } = options;
|
|
79
|
+
const modernHost = options.modernHost === true;
|
|
42
80
|
const tracer = getTracer();
|
|
81
|
+
let hookCount = 0;
|
|
82
|
+
const on = (hookName, handler, opts) => {
|
|
83
|
+
api.on(hookName, handler, opts);
|
|
84
|
+
hookCount += 1;
|
|
85
|
+
};
|
|
43
86
|
// Cap on drift events per turn so a flood of invented tool calls can't balloon the log.
|
|
44
87
|
const MAX_DRIFT_EVENTS_PER_TURN = 8;
|
|
45
88
|
// Held model spans, keyed by (sessionKey, runId). The tool/subagent spans parent
|
|
@@ -54,13 +97,23 @@ export function registerSpanHooks(options) {
|
|
|
54
97
|
// (agentId → sessionKey/runId) from llm_input, plus the last-seen pair, so tool
|
|
55
98
|
// hooks can resolve their host session.
|
|
56
99
|
const sessionByAgentId = new Map();
|
|
100
|
+
// Latest run per session, for a 10.x subagent_spawned whose ctx names the requester session but
|
|
101
|
+
// not its run (ctx.runId is the child run there).
|
|
102
|
+
const latestRunBySession = new Map();
|
|
57
103
|
let lastActive = null;
|
|
58
104
|
const rememberSession = (agentCtx, sessionKey, runId) => {
|
|
59
105
|
const agentId = nonEmptyString(asRecord(agentCtx)?.agentId);
|
|
60
106
|
if (agentId)
|
|
61
107
|
sessionByAgentId.set(agentId, { sessionKey, runId });
|
|
108
|
+
latestRunBySession.set(sessionKey, runId);
|
|
62
109
|
lastActive = { sessionKey, runId };
|
|
63
110
|
};
|
|
111
|
+
/** Stamps the run's model-call count on a `chat` span about to end. */
|
|
112
|
+
const stampLlmCalls = (held) => {
|
|
113
|
+
if (held.llmCalls && held.llmCalls > 0) {
|
|
114
|
+
setDefinedAttributes(held.span, { "senpi.llm.calls": held.llmCalls });
|
|
115
|
+
}
|
|
116
|
+
};
|
|
64
117
|
const resolveToolSession = (ctx) => {
|
|
65
118
|
const direct = {
|
|
66
119
|
sessionKey: readSessionKey(ctx, undefined),
|
|
@@ -86,7 +139,7 @@ export function registerSpanHooks(options) {
|
|
|
86
139
|
return model ? trace.setSpan(otelContext.active(), model.span) : otelContext.active();
|
|
87
140
|
};
|
|
88
141
|
// ─── LLM (`chat` span) ───────────────────────────────────────────────────
|
|
89
|
-
|
|
142
|
+
on("llm_input", (event, agentCtx) => {
|
|
90
143
|
try {
|
|
91
144
|
const identity = readIdentity(agentCtx);
|
|
92
145
|
const sessionKey = readSessionKey(agentCtx, event);
|
|
@@ -100,8 +153,13 @@ export function registerSpanHooks(options) {
|
|
|
100
153
|
// A duplicate llm_input for the same (session, run) means a new turn opened
|
|
101
154
|
// without the previous one closing — end the prior span before replacing it.
|
|
102
155
|
const prior = modelSpans.get(key);
|
|
103
|
-
if (prior)
|
|
156
|
+
if (prior) {
|
|
157
|
+
// Its agent_end grace timer would end the replaced span a second time.
|
|
158
|
+
if (prior.endTimer)
|
|
159
|
+
clearTimeout(prior.endTimer);
|
|
160
|
+
stampLlmCalls(prior);
|
|
104
161
|
endSpan(prior.span);
|
|
162
|
+
}
|
|
105
163
|
const ev = asRecord(event) ?? {};
|
|
106
164
|
const requestModel = nonEmptyString(ev.model);
|
|
107
165
|
const provider = nonEmptyString(ev.provider);
|
|
@@ -148,7 +206,7 @@ export function registerSpanHooks(options) {
|
|
|
148
206
|
logger.warn("llm_input handler failed", { err: errMsg(err) });
|
|
149
207
|
}
|
|
150
208
|
});
|
|
151
|
-
|
|
209
|
+
on("llm_output", (event, agentCtx) => {
|
|
152
210
|
try {
|
|
153
211
|
const sessionKey = readSessionKey(agentCtx, event);
|
|
154
212
|
const runId = readRunId(agentCtx, event);
|
|
@@ -167,6 +225,7 @@ export function registerSpanHooks(options) {
|
|
|
167
225
|
// Cancel any deferred teardown the agent_end sweep scheduled — we finalize the span here.
|
|
168
226
|
if (held.endTimer)
|
|
169
227
|
clearTimeout(held.endTimer);
|
|
228
|
+
stampLlmCalls(held);
|
|
170
229
|
const ev = asRecord(event) ?? {};
|
|
171
230
|
const usage = asRecord(ev.usage);
|
|
172
231
|
// Token field naming varies by provider/wrapper; try every known spelling.
|
|
@@ -201,9 +260,14 @@ export function registerSpanHooks(options) {
|
|
|
201
260
|
// deduped + capped so a flood of invented calls can't balloon the log.
|
|
202
261
|
if (held.offeredTools) {
|
|
203
262
|
const offered = held.offeredTools;
|
|
263
|
+
// A name the host ran in this run is not invented: in 10.x tool-search directory mode a
|
|
264
|
+
// direct call to any deferred tool (a `senpi__` MCP tool or a deferred OpenClaw tool) is
|
|
265
|
+
// legal though not in the offered set (`tool-search-directory.ts:194-197` on 10.10). An
|
|
266
|
+
// unoffered name that never ran is drift on every host and in every mode.
|
|
267
|
+
const ran = held.ranTools;
|
|
204
268
|
const seen = new Set();
|
|
205
269
|
for (const name of requestedToolNames(outputMessages)) {
|
|
206
|
-
if (offered.has(name) || seen.has(name))
|
|
270
|
+
if (offered.has(name) || seen.has(name) || ran?.has(name))
|
|
207
271
|
continue;
|
|
208
272
|
if (seen.size >= MAX_DRIFT_EVENTS_PER_TURN)
|
|
209
273
|
break;
|
|
@@ -237,7 +301,7 @@ export function registerSpanHooks(options) {
|
|
|
237
301
|
// The provider-call result: `outcome`, `failureKind`, timing. This is the ONLY
|
|
238
302
|
// signal for a failed/aborted turn — `llm_output` never fires when the call
|
|
239
303
|
// errors, so without this a wedged turn is indistinguishable from a healthy one.
|
|
240
|
-
|
|
304
|
+
on("model_call_ended", (event, agentCtx) => {
|
|
241
305
|
try {
|
|
242
306
|
const sessionKey = readSessionKey(agentCtx, event);
|
|
243
307
|
const runId = readRunId(agentCtx, event);
|
|
@@ -246,6 +310,7 @@ export function registerSpanHooks(options) {
|
|
|
246
310
|
const held = modelSpans.get(sessionLlmKey(sessionKey, runId));
|
|
247
311
|
if (!held)
|
|
248
312
|
return; // span already finalized, or not one we opened — best effort.
|
|
313
|
+
held.llmCalls = (held.llmCalls ?? 0) + 1;
|
|
249
314
|
const ev = asRecord(event) ?? {};
|
|
250
315
|
const outcome = nonEmptyString(ev.outcome);
|
|
251
316
|
const failureKind = nonEmptyString(ev.failureKind);
|
|
@@ -270,7 +335,59 @@ export function registerSpanHooks(options) {
|
|
|
270
335
|
}
|
|
271
336
|
});
|
|
272
337
|
// ─── Tool (`execute_tool` span) ────────────────────────────────────────────
|
|
273
|
-
|
|
338
|
+
// Inner calls a dispatcher's after_tool_call closed, so their own late after_tool_call is not
|
|
339
|
+
// counted again; the value says whether that close already evented a failure (the outer call
|
|
340
|
+
// errored). FIFO-bounded: a host that never sends that late hook must not grow it.
|
|
341
|
+
const closedByDispatcher = new Map();
|
|
342
|
+
const CLOSED_BY_DISPATCHER_MAX = 512;
|
|
343
|
+
const innerKey = (sessionKey, runId, toolCallId) => `${sessionKey} ${runId} ${toolCallId}`;
|
|
344
|
+
const markClosedByDispatcher = (key, failureEvented) => {
|
|
345
|
+
closedByDispatcher.set(key, failureEvented);
|
|
346
|
+
while (closedByDispatcher.size > CLOSED_BY_DISPATCHER_MAX) {
|
|
347
|
+
const oldest = closedByDispatcher.keys().next().value;
|
|
348
|
+
closedByDispatcher.delete(oldest);
|
|
349
|
+
}
|
|
350
|
+
};
|
|
351
|
+
/**
|
|
352
|
+
* A 10.x dispatcher's open `execute_tool` span in this run whose inner-call prefix
|
|
353
|
+
* `tool_call:${sanitize(outer id)}:` starts `innerToolCallId`, for parenting the inner span.
|
|
354
|
+
*/
|
|
355
|
+
const findDispatcherSpanFor = (sessionKey, runId, innerToolCallId) => {
|
|
356
|
+
for (const held of toolSpans.values()) {
|
|
357
|
+
if (held.bridgeKind !== "call" || !held.toolCallId)
|
|
358
|
+
continue;
|
|
359
|
+
if (held.sessionKey !== sessionKey || held.runId !== runId)
|
|
360
|
+
continue;
|
|
361
|
+
if (innerToolCallId.startsWith(`tool_call:${sanitizeToolCallIdPart(held.toolCallId)}:`))
|
|
362
|
+
return held;
|
|
363
|
+
}
|
|
364
|
+
return undefined;
|
|
365
|
+
};
|
|
366
|
+
/** Records the outcome on a held tool span and ends it, after its args stamping settles. */
|
|
367
|
+
const finalizeToolSpan = (held, errored, error, result) => {
|
|
368
|
+
const span = held.span;
|
|
369
|
+
void (async () => {
|
|
370
|
+
try {
|
|
371
|
+
if (held.pending)
|
|
372
|
+
await held.pending;
|
|
373
|
+
if (errored) {
|
|
374
|
+
recordSpanError(span, error);
|
|
375
|
+
}
|
|
376
|
+
else {
|
|
377
|
+
await setGenAiContentAttribute(span, GEN_AI.TOOL_CALL_RESULT, result);
|
|
378
|
+
}
|
|
379
|
+
}
|
|
380
|
+
finally {
|
|
381
|
+
try {
|
|
382
|
+
span.end();
|
|
383
|
+
}
|
|
384
|
+
catch {
|
|
385
|
+
/* best effort */
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
})();
|
|
389
|
+
};
|
|
390
|
+
on("before_tool_call", (event, toolCtx) => {
|
|
274
391
|
try {
|
|
275
392
|
const identity = readIdentity(toolCtx);
|
|
276
393
|
const resolved = resolveToolSession(toolCtx);
|
|
@@ -283,14 +400,27 @@ export function registerSpanHooks(options) {
|
|
|
283
400
|
const toolCallId = nonEmptyString(ev?.toolCallId) ??
|
|
284
401
|
nonEmptyString(asRecord(toolCtx)?.toolCallId) ??
|
|
285
402
|
null;
|
|
403
|
+
const model = modelSpans.get(sessionLlmKey(resolved.sessionKey, resolved.runId));
|
|
404
|
+
if (model)
|
|
405
|
+
(model.ranTools ??= new Set()).add(toolName);
|
|
406
|
+
const bridgeKind = modernHost ? BRIDGE_TOOL_KINDS[toolName] : undefined;
|
|
407
|
+
// An inner call a 10.x dispatcher runs nests under the dispatcher's span.
|
|
408
|
+
const dispatcher = modernHost && !bridgeKind && toolCallId?.startsWith("tool_call:")
|
|
409
|
+
? findDispatcherSpanFor(resolved.sessionKey, resolved.runId, toolCallId)
|
|
410
|
+
: undefined;
|
|
286
411
|
const key = toolKey(resolved.sessionKey, resolved.runId, toolCallId, toolName, ++toolSeq);
|
|
287
|
-
const span = tracer.startSpan(`execute_tool ${toolName}`, { kind: SpanKind.INTERNAL },
|
|
412
|
+
const span = tracer.startSpan(`execute_tool ${toolName}`, { kind: SpanKind.INTERNAL }, dispatcher
|
|
413
|
+
? trace.setSpan(otelContext.active(), dispatcher.span)
|
|
414
|
+
: parentContextForModel(resolved.sessionKey, resolved.runId));
|
|
288
415
|
const held = {
|
|
289
416
|
span,
|
|
290
417
|
sessionKey: resolved.sessionKey,
|
|
291
418
|
runId: resolved.runId,
|
|
292
419
|
startMs: Date.now(),
|
|
293
420
|
toolName,
|
|
421
|
+
...(toolCallId ? { toolCallId } : {}),
|
|
422
|
+
...(bridgeKind ? { bridgeKind } : {}),
|
|
423
|
+
...(bridgeKind === "call" ? { requestedId: nonEmptyString(asRecord(ev?.params)?.id) } : {}),
|
|
294
424
|
};
|
|
295
425
|
toolSpans.set(key, held);
|
|
296
426
|
setDefinedAttributes(span, {
|
|
@@ -299,6 +429,7 @@ export function registerSpanHooks(options) {
|
|
|
299
429
|
[OPENCLAW_RUN_ID_ATTR]: resolved.runId,
|
|
300
430
|
[GEN_AI.TOOL_NAME]: toolName,
|
|
301
431
|
[GEN_AI.TOOL_CALL_ID]: toolCallId ?? undefined,
|
|
432
|
+
"senpi.tool.bridge": bridgeKind ? true : undefined,
|
|
302
433
|
// Best-effort skill attribution from a skill-file read (no first-class skill hook exists).
|
|
303
434
|
"senpi.skill.name": skillNameFromToolArgs(ev?.params ?? ev?.input),
|
|
304
435
|
...identityAttributes(identity),
|
|
@@ -311,42 +442,117 @@ export function registerSpanHooks(options) {
|
|
|
311
442
|
logger.warn("before_tool_call handler failed", { err: errMsg(err) });
|
|
312
443
|
}
|
|
313
444
|
});
|
|
314
|
-
|
|
445
|
+
on("after_tool_call", (event, toolCtx) => {
|
|
315
446
|
try {
|
|
316
447
|
const resolved = resolveToolSession(toolCtx);
|
|
317
|
-
if (!resolved) {
|
|
318
|
-
logger.debug("after_tool_call session unresolved; skipping");
|
|
319
|
-
return;
|
|
320
|
-
}
|
|
321
448
|
const ev = asRecord(event);
|
|
322
449
|
const toolName = nonEmptyString(ev?.toolName) ?? nonEmptyString(ev?.name) ?? "unknown";
|
|
323
450
|
const error = ev?.error;
|
|
324
451
|
const errored = error !== undefined && error !== null;
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
452
|
+
const result = ev?.result;
|
|
453
|
+
const toolCallId = nonEmptyString(ev?.toolCallId) ??
|
|
454
|
+
nonEmptyString(asRecord(toolCtx)?.toolCallId) ??
|
|
455
|
+
null;
|
|
456
|
+
const bridgeKind = modernHost ? BRIDGE_TOOL_KINDS[toolName] : undefined;
|
|
457
|
+
const emitFailure = (name, bridge) => {
|
|
330
458
|
emitToolCalled({
|
|
331
|
-
name
|
|
459
|
+
name,
|
|
332
460
|
origin: "agent",
|
|
333
461
|
outcome: "error",
|
|
334
462
|
errorType: errorTypeOf(error),
|
|
335
463
|
errorMessage: error instanceof Error ? error.message : String(error),
|
|
464
|
+
...(bridge ? { bridge: true } : {}),
|
|
336
465
|
});
|
|
466
|
+
};
|
|
467
|
+
const foundKey = resolved
|
|
468
|
+
? findToolKey(toolSpans, resolved.sessionKey, resolved.runId, toolCallId, toolName)
|
|
469
|
+
: undefined;
|
|
470
|
+
const held = foundKey ? toolSpans.get(foundKey) : undefined;
|
|
471
|
+
if (foundKey)
|
|
472
|
+
toolSpans.delete(foundKey);
|
|
473
|
+
if (bridgeKind) {
|
|
474
|
+
// Inner calls of this dispatcher still open: their own after_tool_call normally closed them
|
|
475
|
+
// first (10.10/10.4), so these are the ones it missed. They take the outer outcome here,
|
|
476
|
+
// count once under their own names, and a late after_tool_call for them is ignored.
|
|
477
|
+
const inner = [];
|
|
478
|
+
if (bridgeKind === "call" && resolved && toolCallId) {
|
|
479
|
+
const prefix = `tool_call:${sanitizeToolCallIdPart(toolCallId)}:`;
|
|
480
|
+
for (const [k, h] of toolSpans) {
|
|
481
|
+
if (h.sessionKey !== resolved.sessionKey || h.runId !== resolved.runId)
|
|
482
|
+
continue;
|
|
483
|
+
if (!h.toolCallId?.startsWith(prefix))
|
|
484
|
+
continue;
|
|
485
|
+
toolSpans.delete(k);
|
|
486
|
+
markClosedByDispatcher(innerKey(h.sessionKey, h.runId, h.toolCallId), errored);
|
|
487
|
+
inner.push(h);
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
if (errored) {
|
|
491
|
+
if (inner.length > 0) {
|
|
492
|
+
for (const h of inner)
|
|
493
|
+
emitFailure(h.toolName ?? "unknown");
|
|
494
|
+
}
|
|
495
|
+
else if (bridgeKind === "call" && held?.innerErrored) {
|
|
496
|
+
// The inner call's own after_tool_call already reported this failure.
|
|
497
|
+
}
|
|
498
|
+
else if (bridgeKind === "call") {
|
|
499
|
+
// No inner call, or one that succeeded: the dispatcher's own failure (a cap, a result
|
|
500
|
+
// post-processing error) is still a failure the agent saw, so it is always evented.
|
|
501
|
+
const requested = held?.requestedId ?? nonEmptyString(asRecord(ev?.params)?.id);
|
|
502
|
+
emitFailure(requested ?? toolName, requested !== undefined);
|
|
503
|
+
}
|
|
504
|
+
else {
|
|
505
|
+
emitFailure(toolName);
|
|
506
|
+
}
|
|
507
|
+
}
|
|
508
|
+
for (const h of inner) {
|
|
509
|
+
recordToolCall({
|
|
510
|
+
toolName: h.toolName,
|
|
511
|
+
origin: "agent",
|
|
512
|
+
outcome: errored ? "error" : "ok",
|
|
513
|
+
errorType: errored ? errorTypeOf(error) : undefined,
|
|
514
|
+
durationSec: elapsedSeconds(h.startMs),
|
|
515
|
+
});
|
|
516
|
+
finalizeToolSpan(h, errored, error, result);
|
|
517
|
+
}
|
|
518
|
+
recordToolBridgeCall({ kind: bridgeKind, outcome: errored ? "error" : "ok" });
|
|
519
|
+
if (held)
|
|
520
|
+
finalizeToolSpan(held, errored, error, result);
|
|
521
|
+
return;
|
|
337
522
|
}
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
523
|
+
// A late after_tool_call for an inner call its dispatcher already closed and counted. Its
|
|
524
|
+
// span and metric are done; a failure it reports is still evented unless the dispatcher's
|
|
525
|
+
// close already did (a dispatcher can return an inner error to the model as a result).
|
|
526
|
+
if (modernHost && resolved && toolCallId) {
|
|
527
|
+
const k = innerKey(resolved.sessionKey, resolved.runId, toolCallId);
|
|
528
|
+
const failureEvented = closedByDispatcher.get(k);
|
|
529
|
+
if (failureEvented !== undefined) {
|
|
530
|
+
closedByDispatcher.delete(k);
|
|
531
|
+
if (errored && !failureEvented)
|
|
532
|
+
emitFailure(toolName);
|
|
533
|
+
return;
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
// Tool-plane event (narrative): a failed tool call — the agent-blind hole, greppable instead
|
|
537
|
+
// of span-only. A successful call already rides its span + (state-changing) its domain event,
|
|
538
|
+
// so it is not re-evented here. Emitted before the span check so a failure is never lost when
|
|
539
|
+
// the matching before_tool_call span wasn't registered.
|
|
540
|
+
if (errored)
|
|
541
|
+
emitFailure(toolName);
|
|
542
|
+
// Tell the still-open dispatcher that its inner call's failure is already reported.
|
|
543
|
+
if (errored && modernHost && resolved && toolCallId?.startsWith("tool_call:")) {
|
|
544
|
+
const dispatcher = findDispatcherSpanFor(resolved.sessionKey, resolved.runId, toolCallId);
|
|
545
|
+
if (dispatcher)
|
|
546
|
+
dispatcher.innerErrored = true;
|
|
547
|
+
}
|
|
548
|
+
if (!resolved) {
|
|
549
|
+
logger.debug("after_tool_call session unresolved; skipping");
|
|
550
|
+
return;
|
|
551
|
+
}
|
|
552
|
+
if (!held) {
|
|
343
553
|
logger.debug("after_tool_call: no open tool span", { toolName, toolCallId });
|
|
344
554
|
return;
|
|
345
555
|
}
|
|
346
|
-
const held = toolSpans.get(foundKey);
|
|
347
|
-
const span = held.span;
|
|
348
|
-
toolSpans.delete(foundKey);
|
|
349
|
-
const result = ev?.result;
|
|
350
556
|
recordToolCall({
|
|
351
557
|
toolName: held.toolName,
|
|
352
558
|
origin: "agent",
|
|
@@ -354,83 +560,83 @@ export function registerSpanHooks(options) {
|
|
|
354
560
|
errorType: errored ? errorTypeOf(error) : undefined,
|
|
355
561
|
durationSec: elapsedSeconds(held.startMs),
|
|
356
562
|
});
|
|
357
|
-
|
|
358
|
-
try {
|
|
359
|
-
if (held.pending)
|
|
360
|
-
await held.pending;
|
|
361
|
-
if (errored) {
|
|
362
|
-
recordSpanError(span, error);
|
|
363
|
-
}
|
|
364
|
-
else {
|
|
365
|
-
await setGenAiContentAttribute(span, GEN_AI.TOOL_CALL_RESULT, result);
|
|
366
|
-
}
|
|
367
|
-
}
|
|
368
|
-
finally {
|
|
369
|
-
try {
|
|
370
|
-
span.end();
|
|
371
|
-
}
|
|
372
|
-
catch {
|
|
373
|
-
/* best effort */
|
|
374
|
-
}
|
|
375
|
-
}
|
|
376
|
-
})();
|
|
563
|
+
finalizeToolSpan(held, errored, error, result);
|
|
377
564
|
}
|
|
378
565
|
catch (err) {
|
|
379
566
|
logger.warn("after_tool_call handler failed", { err: errMsg(err) });
|
|
380
567
|
}
|
|
381
568
|
});
|
|
382
|
-
// ─── Subagent (`invoke_agent` span
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
569
|
+
// ─── Subagent (`invoke_agent` span) ────────────────────────────────────────
|
|
570
|
+
// 5.x: subagent_spawning opens, subagent_spawned enriches. 10.x has no subagent_spawning, so
|
|
571
|
+
// subagent_spawned opens the span when none is held for the child.
|
|
572
|
+
/** Opens and holds the `invoke_agent` span for `childSessionKey`, hosted by the given run. */
|
|
573
|
+
const openSubagentSpan = (params) => {
|
|
574
|
+
const { childSessionKey, hostSessionKey, hostRunId, ev } = params;
|
|
575
|
+
const prior = subagentSpans.get(childSessionKey);
|
|
576
|
+
if (prior)
|
|
577
|
+
endSpan(prior.span);
|
|
578
|
+
const agentName = nonEmptyString(ev?.label) ?? nonEmptyString(ev?.agentId) ?? "unknown";
|
|
579
|
+
const span = tracer.startSpan(`invoke_agent ${agentName}`, { kind: SpanKind.INTERNAL }, hostSessionKey && hostRunId
|
|
580
|
+
? parentContextForModel(hostSessionKey, hostRunId)
|
|
581
|
+
: otelContext.active());
|
|
582
|
+
// sessionKey/runId carry the HOST run so the agent_end sweep can reach this span.
|
|
583
|
+
const held = {
|
|
584
|
+
span,
|
|
585
|
+
sessionKey: hostSessionKey,
|
|
586
|
+
runId: hostRunId,
|
|
587
|
+
startMs: Date.now(),
|
|
588
|
+
agentName,
|
|
589
|
+
};
|
|
590
|
+
subagentSpans.set(childSessionKey, held);
|
|
591
|
+
setDefinedAttributes(span, {
|
|
592
|
+
[GEN_AI.OPERATION_NAME]: "invoke_agent",
|
|
593
|
+
[GEN_AI.CONVERSATION_ID]: nonEmptyString(hostSessionKey),
|
|
594
|
+
[GEN_AI.AGENT_NAME]: agentName,
|
|
595
|
+
"senpi.subagent.mode": nonEmptyString(ev?.mode),
|
|
596
|
+
"senpi.subagent.requester": nonEmptyString(ev?.requester),
|
|
597
|
+
"senpi.subagent.thread_requested": typeof ev?.threadRequested === "boolean" ? ev.threadRequested : undefined,
|
|
598
|
+
...identityAttributes(params.identity),
|
|
599
|
+
});
|
|
600
|
+
return held;
|
|
601
|
+
};
|
|
602
|
+
if (!modernHost) {
|
|
603
|
+
on("subagent_spawning", (event, subagentCtx) => {
|
|
604
|
+
try {
|
|
605
|
+
const ev = asRecord(event);
|
|
606
|
+
const ctxObj = asRecord(subagentCtx);
|
|
607
|
+
const hostSessionKey = nonEmptyString(ctxObj?.requesterSessionKey) ??
|
|
608
|
+
nonEmptyString(ctxObj?.sessionKey) ??
|
|
609
|
+
"";
|
|
610
|
+
const hostRunId = nonEmptyString(ctxObj?.runId) ?? nonEmptyString(ctxObj?.requesterRunId) ?? "";
|
|
611
|
+
const childSessionKey = nonEmptyString(ev?.childSessionKey) ?? nonEmptyString(ctxObj?.childSessionKey);
|
|
612
|
+
if (!childSessionKey) {
|
|
613
|
+
logger.warn("subagent_spawning missing childSessionKey; skipping span");
|
|
614
|
+
return;
|
|
615
|
+
}
|
|
616
|
+
openSubagentSpan({ childSessionKey, hostSessionKey, hostRunId, ev, identity: readIdentity(subagentCtx) });
|
|
396
617
|
}
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
: otelContext.active());
|
|
404
|
-
// sessionKey/runId carry the HOST run so the agent_end sweep can reach this span.
|
|
405
|
-
subagentSpans.set(childSessionKey, {
|
|
406
|
-
span,
|
|
407
|
-
sessionKey: hostSessionKey,
|
|
408
|
-
runId: hostRunId,
|
|
409
|
-
startMs: Date.now(),
|
|
410
|
-
agentName,
|
|
411
|
-
});
|
|
412
|
-
setDefinedAttributes(span, {
|
|
413
|
-
[GEN_AI.OPERATION_NAME]: "invoke_agent",
|
|
414
|
-
[GEN_AI.CONVERSATION_ID]: nonEmptyString(hostSessionKey),
|
|
415
|
-
[GEN_AI.AGENT_NAME]: agentName,
|
|
416
|
-
"senpi.subagent.mode": nonEmptyString(ev?.mode),
|
|
417
|
-
"senpi.subagent.requester": nonEmptyString(ev?.requester),
|
|
418
|
-
"senpi.subagent.thread_requested": typeof ev?.threadRequested === "boolean" ? ev.threadRequested : undefined,
|
|
419
|
-
...identityAttributes(identity),
|
|
420
|
-
});
|
|
421
|
-
}
|
|
422
|
-
catch (err) {
|
|
423
|
-
logger.warn("subagent_spawning handler failed", { err: errMsg(err) });
|
|
424
|
-
}
|
|
425
|
-
});
|
|
426
|
-
api.on("subagent_spawned", (event, subagentCtx) => {
|
|
618
|
+
catch (err) {
|
|
619
|
+
logger.warn("subagent_spawning handler failed", { err: errMsg(err) });
|
|
620
|
+
}
|
|
621
|
+
});
|
|
622
|
+
}
|
|
623
|
+
on("subagent_spawned", (event, subagentCtx) => {
|
|
427
624
|
try {
|
|
428
625
|
const ev = asRecord(event);
|
|
429
626
|
const ctxObj = asRecord(subagentCtx);
|
|
430
627
|
const childSessionKey = nonEmptyString(ev?.childSessionKey) ?? nonEmptyString(ctxObj?.childSessionKey);
|
|
431
628
|
if (!childSessionKey)
|
|
432
629
|
return;
|
|
433
|
-
|
|
630
|
+
let held = subagentSpans.get(childSessionKey);
|
|
631
|
+
if (!held && modernHost) {
|
|
632
|
+
// 10.x ctx: `runId` is the CHILD run; the host is the requester session, whose run is
|
|
633
|
+
// `requesterRunId` when given, else that session's latest run seen at llm_input.
|
|
634
|
+
const hostSessionKey = nonEmptyString(ctxObj?.requesterSessionKey) ?? nonEmptyString(ctxObj?.sessionKey) ?? "";
|
|
635
|
+
const hostRunId = nonEmptyString(ctxObj?.requesterRunId) ??
|
|
636
|
+
(hostSessionKey ? latestRunBySession.get(hostSessionKey) : undefined) ??
|
|
637
|
+
"";
|
|
638
|
+
held = openSubagentSpan({ childSessionKey, hostSessionKey, hostRunId, ev, identity: readIdentity(subagentCtx) });
|
|
639
|
+
}
|
|
434
640
|
if (!held)
|
|
435
641
|
return;
|
|
436
642
|
const span = held.span;
|
|
@@ -444,7 +650,7 @@ export function registerSpanHooks(options) {
|
|
|
444
650
|
logger.warn("subagent_spawned handler failed", { err: errMsg(err) });
|
|
445
651
|
}
|
|
446
652
|
});
|
|
447
|
-
|
|
653
|
+
on("subagent_delivery_target", (event, subagentCtx) => {
|
|
448
654
|
try {
|
|
449
655
|
const ev = asRecord(event);
|
|
450
656
|
const ctxObj = asRecord(subagentCtx);
|
|
@@ -468,7 +674,7 @@ export function registerSpanHooks(options) {
|
|
|
468
674
|
logger.warn("subagent_delivery_target handler failed", { err: errMsg(err) });
|
|
469
675
|
}
|
|
470
676
|
});
|
|
471
|
-
|
|
677
|
+
on("subagent_ended", (event, subagentCtx) => {
|
|
472
678
|
try {
|
|
473
679
|
const ev = asRecord(event);
|
|
474
680
|
const ctxObj = asRecord(subagentCtx);
|
|
@@ -509,7 +715,7 @@ export function registerSpanHooks(options) {
|
|
|
509
715
|
}
|
|
510
716
|
});
|
|
511
717
|
// ─── Agent end (orphan sweep) ──────────────────────────────────────────────
|
|
512
|
-
|
|
718
|
+
on("agent_end", (event, agentCtx) => {
|
|
513
719
|
try {
|
|
514
720
|
const sessionKey = readSessionKey(agentCtx, event);
|
|
515
721
|
const runId = readRunId(agentCtx, event);
|
|
@@ -542,6 +748,7 @@ export function registerSpanHooks(options) {
|
|
|
542
748
|
model.endTimer = setTimeout(() => {
|
|
543
749
|
if (modelSpans.get(llmKey) === model)
|
|
544
750
|
modelSpans.delete(llmKey);
|
|
751
|
+
stampLlmCalls(model);
|
|
545
752
|
endSpan(model.span);
|
|
546
753
|
}, MODEL_SPAN_END_GRACE_MS);
|
|
547
754
|
model.endTimer.unref?.();
|
|
@@ -568,11 +775,14 @@ export function registerSpanHooks(options) {
|
|
|
568
775
|
if (lastActive && lastActive.sessionKey === sessionKey && lastActive.runId === runId) {
|
|
569
776
|
lastActive = null;
|
|
570
777
|
}
|
|
778
|
+
if (latestRunBySession.get(sessionKey) === runId)
|
|
779
|
+
latestRunBySession.delete(sessionKey);
|
|
571
780
|
}
|
|
572
781
|
catch (err) {
|
|
573
782
|
logger.warn("agent_end handler failed", { err: errMsg(err) });
|
|
574
783
|
}
|
|
575
784
|
});
|
|
785
|
+
return hookCount;
|
|
576
786
|
}
|
|
577
787
|
// ─── Span key helpers ───────────────────────────────────────────────────────
|
|
578
788
|
const SEP = " ";
|