@springbrand/agent-runtime 0.2.0-alpha.15 → 0.2.0-alpha.17

package/src/runtime.ts

@@ -58,8 +58,10 @@ import {
 } from "./runtime-assembler";
 import type { RuntimeAgentHooks } from "./runtime-definition";
 import type { RuntimeTurnEventsPort } from "./kernel/bindings";
+import {
+  RuntimeTelemetryCoordinator,
+} from "./telemetry";
 import { connectConfiguredMcpServers } from "./lib/mcp";
-import { installConsoleSink } from "./lib/telemetry-dev";
 import {
   PiRuntimeAdapter,
   MODEL_STREAM_STALL_TIMEOUT_MS,
@@ -190,6 +192,8 @@ interface ActiveTurn {
 interface PlannedContinuationData {
   readonly submissionId: string;
   readonly requestId: string;
+  /** Identifies the yielded execution slice so retries dedupe without swallowing the next slice. */
+  readonly continuationId: string;
 }
 
 // 作用：把任意异常整理成可以持久化或发给客户端的文字。
@@ -339,6 +343,8 @@ function requestedCapabilitiesOf(metadata: unknown): RequestedCapability[] {
 
 interface PendingTemporaryAgentApproval {
   receipt: ApprovalReceipt;
+  submissionId?: string;
+  toolCallId: string;
   resolve(decision: TemporaryAgentApprovalDecision): void;
 }
 
@@ -375,6 +381,7 @@ export abstract class AgentRuntimeKernel<
   private runtimeGatewaySession?: RuntimeGatewaySession;
   private runtimeTurnEvents?: RuntimeTurnEventsPort;
   private runtimeHooks?: RuntimeAgentHooks;
+  private telemetryCoordinator?: RuntimeTelemetryCoordinator;
   private piAdapter?: PiRuntimeAdapter;
   private readonly transcript: PiRuntimeTranscript;
   private readonly submissions: SubmissionLifecycle<
@@ -422,6 +429,27 @@ export abstract class AgentRuntimeKernel<
     });
   }
 
+  private get telemetry(): RuntimeTelemetryCoordinator {
+    return this.telemetryCoordinator ??= new RuntimeTelemetryCoordinator({
+      outbox: () => this.db?.telemetry,
+      waitUntil: (promise) => this.ctx.waitUntil(promise),
+      scheduleRetry: async (delaySeconds, idempotent) => {
+        await this.schedule(
+          delaySeconds,
+          "_drainTelemetry",
+          undefined,
+          { idempotent },
+        );
+      },
+      onError: (error) => {
+        console.warn(
+          "[runtime-telemetry:degraded]",
+          json({ error: errorText(error) }),
+        );
+      },
+    });
+  }
+
   /**
    * 为一个 Cloudflare Durable Object 实例建立持久化与执行协作器。
    *
@@ -510,6 +538,27 @@ export abstract class AgentRuntimeKernel<
         );
       },
       onApprovalsChanged: () => this.broadcastApprovals(),
+      onTelemetryRequested: (approval) => {
+        this.telemetry.capture("approvalRequested", {
+          submissionId: approval.submissionId,
+          toolCallId: approval.toolCallId,
+          executionId: approval.executionId,
+          toolName: approval.toolName,
+          summary: approval.summary,
+          input: safeParseJson(approval.inputJson),
+          occurredAt: approval.createdAt,
+        });
+      },
+      onTelemetryDecided: (approval, decision) => {
+        this.telemetry.capture("approvalDecided", {
+          submissionId: approval.submissionId,
+          toolCallId: approval.toolCallId,
+          executionId: approval.executionId,
+          decision,
+          ...(approval.reason ? { reason: approval.reason } : {}),
+          occurredAt: approval.decidedAt ?? Date.now(),
+        });
+      },
     });
     this.interactions = new InteractionLifecycle({
       db: this.db,
@@ -526,6 +575,38 @@ export abstract class AgentRuntimeKernel<
         await this.submissions.recover(submissionId);
       },
       onInteractionsChanged: () => this.broadcastApprovals(),
+      onTelemetryRequested: (interaction) => {
+        this.telemetry.capture("interactionRequested", {
+          submissionId: interaction.submissionId,
+          toolCallId: interaction.toolCallId,
+          interactionId: interaction.interactionId,
+          toolName: interaction.toolName,
+          input: safeParseJson(interaction.inputJson),
+          occurredAt: interaction.createdAt,
+        });
+      },
+      onTelemetrySettled: (interaction) => {
+        if (interaction.status === "responded") {
+          this.telemetry.capture("interactionResponded", {
+            submissionId: interaction.submissionId,
+            toolCallId: interaction.toolCallId,
+            interactionId: interaction.interactionId,
+            ...(interaction.responseJson
+              ? { response: safeParseJson(interaction.responseJson) }
+              : {}),
+            occurredAt: interaction.respondedAt ?? Date.now(),
+          });
+          return;
+        }
+        if (interaction.status === "cancelled") {
+          this.telemetry.capture("interactionCancelled", {
+            submissionId: interaction.submissionId,
+            toolCallId: interaction.toolCallId,
+            interactionId: interaction.interactionId,
+            occurredAt: interaction.respondedAt ?? Date.now(),
+          });
+        }
+      },
     });
   }
 
@@ -671,9 +752,6 @@ export abstract class AgentRuntimeKernel<
       for (const [index, guard] of candidate.commitGuards.entries()) {
         await withRuntimeLoadTimeout(`commit-guard:${index}`, guard);
       }
-      if (candidateSnapshot.bindings.platform.telemetryConsole) {
-        installConsoleSink();
-      }
       this.pi.activate(candidateSnapshot);
       activated = true;
       repin?.commit();
@@ -681,6 +759,7 @@ export abstract class AgentRuntimeKernel<
       this.runtimeRevision = revision;
       this.runtimePi = prepared;
       this.runtimeGatewaySession = gatewaySession;
+      this.telemetry.configure(candidate.telemetry);
     } catch (error) {
       repin?.abort();
       if (activated && previous) this.pi.activate(previous);
@@ -707,6 +786,11 @@ export abstract class AgentRuntimeKernel<
     }
   }
 
+  async _drainTelemetry(): Promise<void> {
+    await this.ensureRuntimeReady();
+    await this.telemetry.drainScheduled();
+  }
+
   // 作用：为某个已准入 Submission 建立一次 Pi Turn 执行适配器。
   // 调用：新 Turn、恢复 Turn 和审批后重试 Tool 都在开始 Pi 执行前调用。
   // 原因：在这里一次性固定 revision、持久化回调与终态回调，防止各执行路径绕过同一耐久性规则。
@@ -803,12 +887,43 @@ export abstract class AgentRuntimeKernel<
         settleTool: (call) =>
           this.settleTool(submission.submissionId, call),
       },
+      onGeneration: (event) => {
+        if (event.type === "started") {
+          this.telemetry.capture("generationStarted", {
+            submissionId: submission.submissionId,
+            generationId: event.generationId,
+            provider: event.provider,
+            model: event.model,
+            input: event.input,
+            occurredAt: event.timestamp,
+          });
+          return;
+        }
+        this.telemetry.capture("generationFinished", {
+          submissionId: submission.submissionId,
+          generationId: event.generationId,
+          outcome: event.outcome === "success"
+            ? "completed"
+            : event.outcome === "cancelled"
+              ? "cancelled"
+              : "failed",
+          durationMs: event.durationMs,
+          ...(event.usage
+            ? {
+                inputTokens: event.usage.input,
+                outputTokens: event.usage.output,
+                totalTokens: event.usage.totalTokens,
+                cost: event.usage.cost.total,
+              }
+            : {}),
+          ...(event.errorName ? { error: event.errorName } : {}),
+          ...(event.output ? { output: event.output } : {}),
+          occurredAt: event.timestamp,
+        });
+      },
       ...(toolExecutors && Object.keys(toolExecutors).length > 0
         ? { toolExecutors }
         : {}),
-      onToolTelemetry: (event) => {
-        this._emit("ua:tool" as never, { ...event });
-      },
       transformContext: (messages, signal) =>
         this.transformPiContext(submission.submissionId, messages, signal),
       abortReason: () =>
@@ -911,6 +1026,7 @@ export abstract class AgentRuntimeKernel<
    */
   async onStart(): Promise<void> {
     this.runtimeLoad.reset();
+    this.telemetry.recover(() => this.ensureRuntimeReady());
     if (this.db.runtimeEvents.hasPending()) {
       this.ctx.waitUntil(
         this.ensureRuntimeReady().then(() => this.drainRuntimeEvents()),
@@ -1012,7 +1128,54 @@ export abstract class AgentRuntimeKernel<
     context: TemporaryAgentRunContext,
     execute: TemporaryAgentExecutor,
   ): Promise<string> {
-    return this.temporaryAgents.run(request, execute, context.signal);
+    return this.temporaryAgents.run(
+      request,
+      async (normalized) => {
+        const submission = this.findSubmissionByRequest(context.requestId);
+        const subagentId = `temporary:${context.toolCallId}`;
+        const startedAt = Date.now();
+        if (submission) {
+          this.telemetry.capture("subagentStarted", {
+            submissionId: submission.submissionId,
+            toolCallId: context.toolCallId,
+            subagentId,
+            agentType: normalized.subagentName,
+            prompt: {
+              instructions: normalized.instructions,
+              task: normalized.task,
+            },
+            occurredAt: startedAt,
+          });
+        }
+        try {
+          const output = await execute(normalized);
+          if (submission) {
+            this.telemetry.capture("subagentFinished", {
+              submissionId: submission.submissionId,
+              toolCallId: context.toolCallId,
+              subagentId,
+              outcome: "completed",
+              durationMs: Math.max(0, Date.now() - startedAt),
+              output,
+            });
+          }
+          return output;
+        } catch (error) {
+          if (submission) {
+            this.telemetry.capture("subagentFinished", {
+              submissionId: submission.submissionId,
+              toolCallId: context.toolCallId,
+              subagentId,
+              outcome: context.signal.aborted ? "cancelled" : "failed",
+              durationMs: Math.max(0, Date.now() - startedAt),
+              error: errorText(error),
+            });
+          }
+          throw error;
+        }
+      },
+      context.signal,
+    );
   }
 
   /**
@@ -1032,6 +1195,7 @@ export abstract class AgentRuntimeKernel<
         `temporary agent approval already pending: ${request.executionId}`,
       );
     }
+    const submission = this.findSubmissionByRequest(request.requestId);
     return new Promise((resolve, reject) => {
       this.temporaryAgentApprovals.set(request.executionId, {
         receipt: {
@@ -1044,8 +1208,20 @@ export abstract class AgentRuntimeKernel<
           inputJson: JSON.stringify(request.input ?? null),
           requestId: request.requestId,
         },
+        ...(submission ? { submissionId: submission.submissionId } : {}),
+        toolCallId: request.toolCallId,
         resolve,
       });
+      if (submission) {
+        this.telemetry.capture("approvalRequested", {
+          submissionId: submission.submissionId,
+          toolCallId: request.toolCallId,
+          executionId: request.executionId,
+          toolName: request.toolName,
+          summary: `${request.subagentName} requests ${request.toolName}`,
+          input: request.input,
+        });
+      }
       void this.broadcastApprovals().catch((error) => {
         this.temporaryAgentApprovals.delete(request.executionId);
         reject(error);
@@ -1069,6 +1245,15 @@ export abstract class AgentRuntimeKernel<
     if (!pending) return { ok: false };
     this.temporaryAgentApprovals.delete(executionId);
     pending.resolve({ approved: false, reason });
+    if (pending.submissionId) {
+      this.telemetry.capture("approvalDecided", {
+        submissionId: pending.submissionId!,
+        toolCallId: pending.toolCallId,
+        executionId,
+        decision: "cancelled",
+        reason,
+      });
+    }
     await this.broadcastApprovals();
     return { ok: true };
   }
@@ -1390,7 +1575,7 @@ export abstract class AgentRuntimeKernel<
       ),
       signal,
       submissionId,
-      onCompactionPersisted: (event) =>
+      onCompactionPersisted: (event) => {
         this.db.runtimeEvents.insert({
           eventId: event.eventId,
           body: json({
@@ -1411,7 +1596,14 @@ export abstract class AgentRuntimeKernel<
             },
           }),
           createdAt: Date.now(),
-        }),
+        });
+        this.telemetry.capture("contextCompacted", {
+          submissionId,
+          compactionId: event.eventId,
+          inputTokens: event.usage.input,
+          outputTokens: event.usage.output,
+        });
+      },
     });
     await this.drainRuntimeEvents();
     return compacted;
@@ -1631,6 +1823,14 @@ export abstract class AgentRuntimeKernel<
               createdAt,
             });
           }
+          this.telemetry.capture("turnAdmitted", {
+            submissionId,
+            requestId: admissionRequestId,
+            ...(admissionIdempotencyKey
+              ? { idempotencyKey: admissionIdempotencyKey }
+              : {}),
+            occurredAt: createdAt,
+          });
           if (hook) this.admissionsInFlight.delete(admissionKey);
           return this.readSubmission(submissionId)!;
         };
@@ -1840,12 +2040,21 @@ export abstract class AgentRuntimeKernel<
       kind: "record-tool-input",
       record: input,
     });
-    const appended = this.db.transaction(() =>
-      this.applyPiRecoveryMutations(
+    const appended = this.db.transaction(() => {
+      const inserted = this.applyPiRecoveryMutations(
         submission,
         decision.mutations,
-      ),
-    );
+      );
+      if (inserted) {
+        this.telemetry.capture("toolStarted", {
+          submissionId: submission.submissionId,
+          toolCallId: input.toolCallId,
+          toolName: input.toolName,
+          input: input.input,
+        });
+      }
+      return inserted;
+    });
     if (appended) this.announceToolStart(submission.submissionId, input);
     return appended;
   }
@@ -1934,6 +2143,26 @@ export abstract class AgentRuntimeKernel<
         body: json({ type: "tool-settlement", event }),
         createdAt,
       });
+      const details = call.result.details as {
+        kind?: string;
+        bytes?: number;
+      } | undefined;
+      this.telemetry.capture("toolFinished", {
+        submissionId,
+        toolCallId: call.toolCallId,
+        toolName: call.toolName,
+        outcome: call.isError ? "failed" : "completed",
+        outputBytes: details?.kind === "artifact_ref" &&
+            typeof details.bytes === "number"
+          ? details.bytes
+          : result.length,
+        spilled: details?.kind === "artifact_ref",
+        output: call.result,
+        ...(call.isError && call.cause
+          ? { error: errorText(call.cause) }
+          : {}),
+        occurredAt: createdAt,
+      });
     }
     const decision = this.decidePiRecovery(submission, {
       kind: "record-tool-result",
@@ -2030,6 +2259,12 @@ export abstract class AgentRuntimeKernel<
       ...(message === undefined ? {} : { message }),
     });
     this.applyPiRecoveryMutations(submission, decision.mutations);
+    if (outcome === "aborted") {
+      this.telemetry.capture("turnCancelled", {
+        submissionId: submission.submissionId,
+        ...(message ? { reason: message } : {}),
+      });
+    }
   }
 
   // 作用：把 Turn 的权威结果一次提交到 Submission、恢复里程碑和聊天续传标记。
@@ -2071,6 +2306,12 @@ export abstract class AgentRuntimeKernel<
       : durableIntent?.outcome ?? outcome;
     const effectiveMessage =
       latest.abortReason ?? durableIntent?.message ?? message;
+    const turnOutput = (await this.transcript.storedMessages())
+      .filter((entry) =>
+        entry.submissionId === latest.submissionId &&
+        entry.message.role === "assistant"
+      )
+      .at(-1)?.message;
     const terminal = this.db.transaction(() => {
       this.appendTerminalIntent(
         latest,
@@ -2126,6 +2367,21 @@ export abstract class AgentRuntimeKernel<
         latest,
         committed.mutations,
       );
+      this.telemetry.capture("turnFinished", {
+        submissionId: latest.submissionId,
+        outcome: effectiveOutcome === "succeeded"
+          ? "completed"
+          : effectiveOutcome === "aborted"
+            ? "cancelled"
+            : "failed",
+        durationMs: Math.max(
+          0,
+          (terminal.completedAt ?? Date.now()) - latest.createdAt,
+        ),
+        ...(turnOutput === undefined ? {} : { output: turnOutput }),
+        ...(effectiveMessage ? { error: effectiveMessage } : {}),
+        occurredAt: terminal.completedAt ?? Date.now(),
+      });
       return terminal;
     });
     await this.drainRuntimeEvents();
@@ -2240,12 +2496,18 @@ export abstract class AgentRuntimeKernel<
     submission: StoredSubmission,
   ): Promise<StoredSubmission> {
     if (!submission.queuedInputJson) {
-      this.db.submissions.transition(
+      const started = this.db.submissions.transition(
         submission.submissionId,
         "running",
         ["pending"],
       );
       const activated = this.readSubmission(submission.submissionId)!;
+      if (started) {
+        this.telemetry.capture("turnStarted", {
+          submissionId: activated.submissionId,
+          assemblyRevision: activated.assemblyRevision,
+        });
+      }
       await this.broadcastApprovals();
       return activated;
     }
@@ -2328,7 +2590,13 @@ export abstract class AgentRuntimeKernel<
       this.db.submissions.clearQueuedPayload(
         submission.submissionId,
       );
-      return this.readSubmission(submission.submissionId)!;
+      const started = this.readSubmission(submission.submissionId)!;
+      this.telemetry.capture("turnStarted", {
+        submissionId: started.submissionId,
+        assemblyRevision: started.assemblyRevision,
+        input: userMessage,
+      });
+      return started;
     });
     this.broadcast(json({
       type: MessageType.CF_AGENT_CHAT_MESSAGES,
@@ -2360,6 +2628,11 @@ export abstract class AgentRuntimeKernel<
       );
     }
     if (recovery) {
+      this.telemetry.capture("turnRecovering", {
+        submissionId,
+        reason: submission.recoveryReason ?? "runtime_restart",
+        attempt: Math.max(1, submission.recoveryErrorCount),
+      });
       this.db.submissions.clearRecoveryReason(submissionId);
       await this.broadcastApprovals();
       await this.ensureRuntimeReady();
@@ -2471,7 +2744,12 @@ export abstract class AgentRuntimeKernel<
       );
     }
     if (ready.status === "pending") {
-      this.db.submissions.transition(submissionId, "running", ["pending"]);
+      if (this.db.submissions.transition(submissionId, "running", ["pending"])) {
+        this.telemetry.capture("turnStarted", {
+          submissionId,
+          assemblyRevision: ready.assemblyRevision,
+        });
+      }
     }
     const storedMessages = await this.transcript.storedMessages();
     const assistantOrdinal = storedMessages.filter(
@@ -2563,10 +2841,17 @@ export abstract class AgentRuntimeKernel<
             latest.abortReason,
           );
         }
+        if (!streamId) {
+          throw new Error("Yielded SpringBrand Turn has no recoverable stream");
+        }
         await this.schedule(
           0,
           "_piPlannedContinuation",
-          { submissionId, requestId: submission.requestId },
+          {
+            submissionId,
+            requestId: submission.requestId,
+            continuationId: streamId,
+          },
           { idempotent: true },
         );
         return latest;
@@ -3249,16 +3534,24 @@ export abstract class AgentRuntimeKernel<
             message: "The current Turn is waiting for approval",
           };
         }
+        const steerId = crypto.randomUUID();
         const inserted = this.db.transaction(() => {
           if (this.db.steers.findByMessageId(message.id)) return false;
           this.db.steers.insert({
-            steerId: crypto.randomUUID(),
+            steerId,
             submissionId: target.submissionId,
             messageId: message.id,
             canonicalJson: json(userMessage),
             uiMessageJson: json(message),
             createdAt: userMessage.timestamp,
           });
+          this.telemetry.capture("turnSteered", {
+            submissionId: target.submissionId,
+            steerId,
+            messageId: message.id,
+            content: userMessage,
+            occurredAt: userMessage.timestamp,
+          });
           return true;
         });
         if (inserted) {
@@ -3376,6 +3669,7 @@ export abstract class AgentRuntimeKernel<
       };
     }
 
+    const steerId = crypto.randomUUID();
     const moved = this.db.transaction(() => {
       const latest = this.readSubmission(submissionId);
       const current = this.db.submissions.findRunning() ??
@@ -3389,13 +3683,20 @@ export abstract class AgentRuntimeKernel<
         return false;
       }
       this.db.steers.insert({
-        steerId: crypto.randomUUID(),
+        steerId,
         submissionId: target.submissionId,
         messageId: queued.userMessageId!,
         canonicalJson: queued.queuedInputJson!,
         uiMessageJson: queued.queuedUiMessageJson ?? null,
         createdAt: queued.createdAt,
       });
+      this.telemetry.capture("turnSteered", {
+        submissionId: target.submissionId,
+        steerId,
+        messageId: queued.userMessageId!,
+        content: message,
+        occurredAt: queued.createdAt,
+      });
       const reason = "Moved to active Turn";
       this.appendTerminalIntent(latest, "aborted", reason);
       this.db.submissions.updateAbortReason(submissionId, reason);
@@ -3620,6 +3921,19 @@ export abstract class AgentRuntimeKernel<
           ? { approved: true }
           : { approved: false, reason: decision.reason },
       );
+      if (temporary.submissionId) {
+        this.telemetry.capture("approvalDecided", {
+          submissionId: temporary.submissionId!,
+          toolCallId: temporary.toolCallId,
+          executionId,
+          decision: decision.decision === "allow_once"
+            ? "approved"
+            : "rejected",
+          ...(decision.decision === "deny" && decision.reason
+            ? { reason: decision.reason }
+            : {}),
+        });
+      }
       await this.broadcastApprovals();
       return { ok: true };
     }
@@ -3733,6 +4047,15 @@ export abstract class AgentRuntimeKernel<
     const metadata = runtimeSubagentRunMetadata(run.inputPreview);
     if (metadata) {
       await this.subagentLifecycle(metadata).register(run.runId);
+      this.telemetry.capture("subagentStarted", {
+        submissionId: metadata.submissionId,
+        ...(run.parentToolCallId
+          ? { toolCallId: run.parentToolCallId }
+          : {}),
+        subagentId: run.runId,
+        agentType: run.agentType,
+        occurredAt: run.startedAt,
+      });
     }
     await this.broadcastApprovals();
   }
@@ -3777,6 +4100,25 @@ export abstract class AgentRuntimeKernel<
           status,
           zeroUsage: !hasUsage,
         });
+        this.telemetry.capture("subagentFinished", {
+          submissionId: metadata.submissionId,
+          ...(run.parentToolCallId
+            ? { toolCallId: run.parentToolCallId }
+            : {}),
+          subagentId: run.runId,
+          outcome: status === "completed"
+            ? "completed"
+            : status === "aborted" || status === "skipped"
+              ? "cancelled"
+              : "failed",
+          durationMs: Math.max(
+            0,
+            (run.completedAt ?? Date.now()) - run.startedAt,
+          ),
+          ...(completion ? { output: completion.output } : {}),
+          ...(result.error ? { error: result.error } : {}),
+          occurredAt: run.completedAt ?? Date.now(),
+        });
       }
     }
     await this.broadcastApprovals();