@geminixiang/mikan 1.0.0-beta.55 → 1.0.0-beta.57

package/CHANGELOG.md

@@ -9,6 +9,26 @@ any release.
 
 ## [Unreleased]
 
+## [1.0.0-beta.57]
+
+### Changed
+
+- Export raw platform conversation, channel, session, thread, message, and user identifiers so Sentry traces can be mapped directly back to their source while continuing to exclude human-readable names and conversation or tool content.
+
+### Removed
+
+- Remove telemetry identifier hashing and the `TELEMETRY_HASH_KEY` deployment setting.
+
+## [1.0.0-beta.56]
+
+### Added
+
+- Add content-free agent diagnostics for actual model IDs, token and cost totals, first-token latency, context utilization, message and attachment counts, payload sizes, tool categories and status, and retry, compaction, and budget summaries.
+
+### Security
+
+- Replace raw conversation, session, message, thread, and user telemetry identifiers with stable opaque values, support deployment-specific HMACs through `TELEMETRY_HASH_KEY`, and stop sending usernames to Sentry.
+
 ## [1.0.0-beta.55]
 
 ### Added

package/dist/harness/presenter.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"presenter.d.ts","sourceRoot":"","sources":["../../src/harness/presenter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,KAAK,GAAG,EAAE,KAAK,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAC1E,OAAO,KAAK,EAEV,eAAe,EACf,kBAAkB,EAClB,kBAAkB,EACnB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAOtD,OAAO,KAAK,EAAE,qBAAqB,EAA4B,MAAM,eAAe,CAAC;AACrF,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,cAAc,CAAC;AAwDhE,wBAAgB,cAAc,IAAI,kBAAkB,CAsBnD;AAED,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,kBAAkB,EAC5B,OAAO,EAAE;IACP,SAAS,EAAE,qBAAqB,CAAC;IACjC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;CACxC,GACA,eAAe,CA0DjB;AAOD,wBAAgB,yBAAyB,CAAC,kBAAkB,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAEzF;AAmKD,wBAAsB,mBAAmB,CACvC,SAAS,EAAE,qBAAqB,EAChC,OAAO,EAAE,iBAAiB,EAC1B,QAAQ,EAAE,kBAAkB,EAC5B,OAAO,CAAC,EAAE;IACR,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,kBAAkB,CAAC,EAAE,MAAM,MAAM,CAAC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GACA,OAAO,CAAC,IAAI,CAAC,CAiBf;AA4DD,wBAAsB,kBAAkB,CAAC,GAAG,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CAmE/E;AA+XD,wBAAgB,0BAA0B,CAAC,MAAM,EAAE;IACjD,OAAO,EAAE,iBAAiB,CAAC;IAC3B,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IAClB,WAAW,EAAE,UAAU,CAAC,OAAO,2BAA2B,CAAC,CAAC;CAC7D,GAAG,IAAI,CAiBP"}
+{"version":3,"file":"presenter.d.ts","sourceRoot":"","sources":["../../src/harness/presenter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAe,KAAK,GAAG,EAAE,KAAK,KAAK,EAAE,MAAM,uBAAuB,CAAC;AAC1E,OAAO,KAAK,EAEV,eAAe,EACf,kBAAkB,EAClB,kBAAkB,EACnB,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAOtD,OAAO,KAAK,EAAE,qBAAqB,EAA4B,MAAM,eAAe,CAAC;AACrF,OAAO,KAAK,EAAE,2BAA2B,EAAE,MAAM,cAAc,CAAC;AA+DhE,wBAAgB,cAAc,IAAI,kBAAkB,CAkCnD;AAED,wBAAgB,uBAAuB,CACrC,QAAQ,EAAE,kBAAkB,EAC5B,OAAO,EAAE;IACP,SAAS,EAAE,qBAAqB,CAAC;IACjC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,QAAQ,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,kBAAkB,EAAE,MAAM,GAAG,SAAS,CAAC;CACxC,GACA,eAAe,CAsEjB;AAOD,wBAAgB,yBAAyB,CAAC,kBAAkB,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAEzF;AAmKD,wBAAsB,mBAAmB,CACvC,SAAS,EAAE,qBAAqB,EAChC,OAAO,EAAE,iBAAiB,EAC1B,QAAQ,EAAE,kBAAkB,EAC5B,OAAO,CAAC,EAAE;IACR,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,kBAAkB,CAAC,EAAE,MAAM,MAAM,CAAC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IACnB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GACA,OAAO,CAAC,IAAI,CAAC,CAiBf;AA4DD,wBAAsB,kBAAkB,CAAC,GAAG,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CA2F/E;AAubD,wBAAgB,0BAA0B,CAAC,MAAM,EAAE;IACjD,OAAO,EAAE,iBAAiB,CAAC;IAC3B,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IAClB,WAAW,EAAE,UAAU,CAAC,OAAO,2BAA2B,CAAC,CAAC;CAC7D,GAAG,IAAI,CAiBP"}

package/dist/harness/presenter.js

@@ -1,6 +1,6 @@
 import { contentText } from "@earendil-works/pi-ai";
 import { mergeSubagentProgress, parseSubagentProgressSnapshot, renderSubagentDashboard, settleSubagentProgress, } from "./tools/subagent.js";
-import { addLifecycleEvent, metricAttributes, recordCounter, recordDistribution, recordGauge, reportUserFacingError, startOperationSpan, } from "../observability/index.js";
+import { addLifecycleEvent, metricAttributes, recordCounter, recordDistribution, recordGauge, reportUserFacingError, startOperationSpan, updateActiveSpanAttribution, } from "../observability/index.js";
 import { appendTriggerAttribution } from "./prompt.js";
 import * as log from "../log.js";
 const operationSpans = new WeakMap();
@@ -22,8 +22,8 @@ function endOutstandingOperationSpans(runState) {
     if (!spans)
         return;
     const aborted = operationError("AbortError");
-    for (const span of spans.llm)
-        span.end({ error: aborted });
+    for (const entry of spans.llm)
+        entry.span.end({ error: aborted });
     for (const span of spans.tools.values())
         span.end({ error: aborted });
     operationSpans.delete(runState);
@@ -53,6 +53,18 @@ export function createRunState() {
         toolProgressTimer: undefined,
         totalUsage: createEmptyUsageTotals(),
         llmCallCount: 0,
+        toolCallCount: 0,
+        toolErrorCount: 0,
+        toolInputCharacters: 0,
+        toolOutputCharacters: 0,
+        assistantMessageCount: 0,
+        outputCharacters: 0,
+        reasoningTokens: 0,
+        retryCount: 0,
+        compactionCount: 0,
+        budgetExceeded: false,
+        firstTokenLatencyMs: undefined,
+        responseModel: undefined,
         stopReason: "stop",
         errorMessage: undefined,
         reportedLlmError: false,
@@ -82,6 +94,18 @@ export function activateRunPresentation(runState, context) {
     runState.toolProgressTimer = undefined;
     runState.totalUsage = createEmptyUsageTotals();
     runState.llmCallCount = 0;
+    runState.toolCallCount = 0;
+    runState.toolErrorCount = 0;
+    runState.toolInputCharacters = 0;
+    runState.toolOutputCharacters = 0;
+    runState.assistantMessageCount = 0;
+    runState.outputCharacters = 0;
+    runState.reasoningTokens = 0;
+    runState.retryCount = 0;
+    runState.compactionCount = 0;
+    runState.budgetExceeded = false;
+    runState.firstTokenLatencyMs = undefined;
+    runState.responseModel = undefined;
     runState.stopReason = "stop";
     runState.errorMessage = undefined;
     runState.reportedLlmError = false;
@@ -320,7 +344,7 @@ async function publishFinalResponse(responder, runState, finalText, options) {
     }
 }
 export async function reportUsageSummary(ctx) {
-    const { session, runState, responder, platform, model, agentConfig, sessionConversation, sessionUuid, waitForQueue, } = ctx;
+    const { session, runState, responder, platform, model, sessionConversation, sessionUuid, waitForQueue, } = ctx;
     const lastAssistantMessage = session.messages.findLast((message) => message.role === "assistant" && message.stopReason !== "aborted");
     const contextTokens = lastAssistantMessage
         ? lastAssistantMessage.usage.input +
@@ -332,7 +356,7 @@ export async function reportUsageSummary(ctx) {
     const { totalUsage } = runState;
     const runMetricAttributes = metricAttributes({
         provider: model.provider,
-        model: agentConfig.model,
+        model: model.id,
         channel_id: sessionConversation,
         session_id: sessionUuid,
         stop_reason: runState.stopReason,
@@ -353,10 +377,35 @@ export async function reportUsageSummary(ctx) {
     recordDistribution("agent.run.cost", totalUsage.cost.total, {
         attributes: runMetricAttributes,
     });
-    recordGauge("agent.context.utilization", contextTokens / contextWindow, {
+    const contextUtilization = contextTokens / contextWindow;
+    recordGauge("agent.context.utilization", contextUtilization, {
         unit: "ratio",
         attributes: runMetricAttributes,
     });
+    updateActiveSpanAttribution({
+        "gen_ai.request.model": model.id,
+        "gen_ai.response.model": runState.responseModel ?? model.id,
+        "gen_ai.usage.input_tokens": totalUsage.input + totalUsage.cacheRead + totalUsage.cacheWrite,
+        "gen_ai.usage.input_tokens.cached": totalUsage.cacheRead,
+        "gen_ai.usage.input_tokens.cache_write": totalUsage.cacheWrite,
+        "gen_ai.usage.output_tokens": totalUsage.output,
+        "gen_ai.usage.output_tokens.reasoning": runState.reasoningTokens,
+        "mikan.usage.cost_usd": totalUsage.cost.total,
+        "mikan.context.utilization": contextUtilization,
+        "mikan.llm.call_count": runState.llmCallCount,
+        "mikan.tool.call_count": runState.toolCallCount,
+        "mikan.tool.error_count": runState.toolErrorCount,
+        "mikan.tool.input.characters": runState.toolInputCharacters,
+        "mikan.tool.output.characters": runState.toolOutputCharacters,
+        "mikan.output.message_count": runState.assistantMessageCount,
+        "mikan.output.characters": runState.outputCharacters,
+        "mikan.retry.count": runState.retryCount,
+        "mikan.compaction.count": runState.compactionCount,
+        "mikan.budget.exceeded": runState.budgetExceeded,
+        ...(runState.firstTokenLatencyMs === undefined
+            ? {}
+            : { "mikan.response.first_token_ms": runState.firstTokenLatencyMs }),
+    });
     const summary = log.logUsageSummary(runState.logCtx, runState.totalUsage, contextTokens, contextWindow);
     if (platform.diagnostics?.showUsageSummary === true) {
         runState.queue.enqueue(() => responder.respondDiagnostic(summary, { style: "muted" }), "usage summary");
@@ -380,10 +429,35 @@ function extractToolResultText(result) {
         return result;
     return toolResultContentText(result) ?? JSON.stringify(result);
 }
+function serializedLength(value) {
+    if (typeof value === "string")
+        return value.length;
+    try {
+        return JSON.stringify(value)?.length ?? 0;
+    }
+    catch {
+        return 0;
+    }
+}
+function toolCategory(name) {
+    if (["read", "write", "edit", "bash"].includes(name))
+        return "sandbox";
+    if (name === "subagent")
+        return "agent";
+    if (name.startsWith("mcp__"))
+        return "mcp";
+    if (name.startsWith("github_"))
+        return "github";
+    if (name === "slack_blockkit")
+        return "platform";
+    return "function";
+}
 function handleToolStart(event, context) {
     const { runState, responder, logCtx, queue, baseAttrs } = context;
     const args = (event.args ?? {});
     const label = args.label || event.toolName;
+    runState.toolCallCount += 1;
+    runState.toolInputCharacters += serializedLength(event.args);
     runState.pendingTools.set(event.toolCallId, {
         toolName: event.toolName,
         args: event.args,
@@ -402,6 +476,9 @@ function handleToolStart(event, context) {
     spansFor(runState).tools.set(event.toolCallId, startOperationSpan(`execute_tool ${event.toolName}`, metricAttributes({
         "gen_ai.operation.name": "execute_tool",
         "gen_ai.tool.name": event.toolName,
+        "gen_ai.tool.type": "function",
+        "mikan.tool.category": toolCategory(event.toolName),
+        "mikan.tool.input.characters": serializedLength(event.args),
         "openinference.span.kind": "TOOL",
         ...baseAttrs,
     })));
@@ -437,6 +514,10 @@ function recordToolMetrics(event, durationMs, context) {
 function handleToolEnd(event, context) {
     const { runState, responder, logCtx } = context;
     const resultStr = extractToolResultText(event.result);
+    const outputCharacters = serializedLength(event.result);
+    runState.toolOutputCharacters += outputCharacters;
+    if (event.isError)
+        runState.toolErrorCount += 1;
     const pending = runState.pendingTools.get(event.toolCallId);
     const progress = runState.toolProgress.get(event.toolCallId);
     if (progress)
@@ -456,6 +537,9 @@ function handleToolEnd(event, context) {
     toolSpan?.end({
         attributes: metricAttributes({
             "gen_ai.tool.name": event.toolName,
+            "gen_ai.tool.type": "function",
+            "mikan.tool.category": toolCategory(event.toolName),
+            "mikan.tool.output.characters": outputCharacters,
             "openinference.span.kind": "TOOL",
             duration_ms: durationMs,
             ...context.baseAttrs,
@@ -476,15 +560,18 @@ function handleMessageStart(event, context) {
     if (event.message.role !== "assistant")
         return;
     context.runState.llmCallCount += 1;
-    spansFor(context.runState).llm.push(startOperationSpan(`chat ${context.agentConfig.model}`, metricAttributes({
-        "gen_ai.operation.name": "chat",
-        "gen_ai.provider.name": context.model.provider,
-        "gen_ai.request.model": context.agentConfig.model,
-        "openinference.span.kind": "LLM",
-        "llm.provider": context.model.provider,
-        "llm.model_name": context.agentConfig.model,
-        ...context.baseAttrs,
-    })));
+    spansFor(context.runState).llm.push({
+        span: startOperationSpan(`chat ${context.model.id}`, metricAttributes({
+            "gen_ai.operation.name": "chat",
+            "gen_ai.provider.name": context.model.provider,
+            "gen_ai.request.model": context.model.id,
+            "openinference.span.kind": "LLM",
+            "llm.provider": context.model.provider,
+            "llm.model_name": context.model.id,
+            ...context.baseAttrs,
+        })),
+        startedAt: Date.now(),
+    });
     addLifecycleEvent("agent.llm.call.started", {
         call_index: context.runState.llmCallCount,
         provider: context.model.provider,
@@ -497,6 +584,11 @@ function handleMessageUpdate(event, context) {
     const update = event.assistantMessageEvent;
     if (update.type !== "text_delta" || !update.delta)
         return;
+    const llmEntry = spansFor(context.runState).llm[0];
+    if (llmEntry && llmEntry.firstTokenAt === undefined) {
+        llmEntry.firstTokenAt = Date.now();
+        context.runState.firstTokenLatencyMs ??= llmEntry.firstTokenAt - llmEntry.startedAt;
+    }
     if (context.responder.appendResponseDelta && !context.runState.suppressResponseDeltas) {
         context.queue.enqueue(async () => {
             await context.responder.appendResponseDelta?.(update.delta);
@@ -516,6 +608,8 @@ function recordAssistantUsage(message, context) {
     totalUsage.cost.cacheRead += message.usage.cost.cacheRead;
     totalUsage.cost.cacheWrite += message.usage.cost.cacheWrite;
     totalUsage.cost.total += message.usage.cost.total;
+    context.runState.reasoningTokens += message.usage.reasoning ?? 0;
+    context.runState.responseModel = message.responseModel ?? message.model;
     const attributes = metricAttributes({
         provider: context.model.provider,
         model: context.agentConfig.model,
@@ -587,27 +681,36 @@ function handleMessageEnd(event, context) {
     if (event.message.role !== "assistant")
         return;
     const message = event.message;
+    context.runState.assistantMessageCount += 1;
+    context.runState.outputCharacters += message.content.reduce((total, part) => total + (part.type === "text" ? part.text.length : 0), 0);
     if (message.stopReason) {
         context.runState.stopReason = message.stopReason;
         // The settling message clears any stale error left by a recovered retry.
         context.runState.errorMessage = message.errorMessage;
     }
     recordAssistantUsage(message, context);
-    const llmSpan = spansFor(context.runState).llm.shift();
+    const llmEntry = spansFor(context.runState).llm.shift();
     const inputTokens = message.usage
         ? message.usage.input + message.usage.cacheRead + message.usage.cacheWrite
         : undefined;
-    llmSpan?.end({
+    llmEntry?.span.end({
         attributes: metricAttributes({
             "gen_ai.provider.name": context.model.provider,
-            "gen_ai.request.model": context.agentConfig.model,
+            "gen_ai.request.model": context.model.id,
+            "gen_ai.response.model": message.responseModel ?? message.model,
             "gen_ai.usage.input_tokens": inputTokens,
             "gen_ai.usage.output_tokens": message.usage?.output,
-            "gen_ai.usage.cache_read.input_tokens": message.usage?.cacheRead,
-            "gen_ai.usage.cache_write.input_tokens": message.usage?.cacheWrite,
+            "gen_ai.usage.input_tokens.cached": message.usage?.cacheRead,
+            "gen_ai.usage.input_tokens.cache_write": message.usage?.cacheWrite,
+            "gen_ai.usage.output_tokens.reasoning": message.usage?.reasoning,
+            "mikan.usage.cost_usd": message.usage?.cost.total,
+            "mikan.response.first_token_ms": llmEntry?.firstTokenAt === undefined
+                ? undefined
+                : llmEntry.firstTokenAt - llmEntry.startedAt,
+            "mikan.output.characters": message.content.reduce((total, part) => total + (part.type === "text" ? part.text.length : 0), 0),
             "openinference.span.kind": "LLM",
             "llm.provider": context.model.provider,
-            "llm.model_name": context.agentConfig.model,
+            "llm.model_name": context.model.id,
             "llm.token_count.prompt": inputTokens,
             "llm.token_count.completion": message.usage?.output,
             "llm.token_count.total": inputTokens !== undefined && message.usage ? inputTokens + message.usage.output : undefined,
@@ -622,6 +725,7 @@ function handleMessageEnd(event, context) {
 }
 function handleLifecycleEvent(event, context) {
     if (event.type === "compaction_start") {
+        context.runState.compactionCount += 1;
         const text = "_Compacting context..._";
         log.logInfo(`Auto-compaction started (reason: ${event.reason})`);
         context.queue.enqueue(() => context.responder.respond(text), "compaction start");
@@ -637,11 +741,13 @@ function handleLifecycleEvent(event, context) {
         return;
     }
     if (event.type === "auto_retry_start") {
+        context.runState.retryCount += 1;
         log.logWarning(`Retrying (${event.attempt}/${event.maxAttempts})`, event.errorMessage);
         const text = `_Retrying (${event.attempt}/${event.maxAttempts})..._`;
         context.queue.enqueue(() => context.responder.respond(text), "retry");
         return;
     }
+    context.runState.budgetExceeded = true;
     log.logWarning("Run stopped by budget circuit breaker", `${event.reason} (tokens=${event.tokens}, cost=${event.costUsd.toFixed(2)}, calls=${event.llmCalls}, ${event.durationMs}ms)`);
     const text = `_Stopped: run budget exceeded (${event.reason})_`;
     context.queue.enqueue(() => context.responder.respondDiagnostic(text, { style: "error" }), "budget exceeded");