oh-langfuse 0.1.25 → 0.1.26

package/scripts/opencode-langfuse-setup.mjs

@@ -115,10 +115,11 @@ function getPatchedLangfuseDistIndexJs() {
   // It is intentionally self-contained (no extra build steps).
   return [
     'import { LangfuseSpanProcessor } from "@langfuse/otel";',
-    'import { promises as fs } from "node:fs";',
-    'import os from "node:os";',
-    'import path from "node:path";',
-    'import { NodeSDK } from "@opentelemetry/sdk-node";',
+    'import { promises as fs } from "node:fs";',
+    'import os from "node:os";',
+    'import path from "node:path";',
+    'import { NodeSDK } from "@opentelemetry/sdk-node";',
+    'import { trace } from "@opentelemetry/api";',
     "",
     "const USER_CONFIG_PATH = path.join(",
     "  os.homedir(),",
@@ -145,26 +146,141 @@ function getPatchedLangfuseDistIndexJs() {
     "  }",
     "};",
     "",
-    "const createUserIdSpanProcessor = (userId) => ({",
-    "  onStart: (span) => {",
-    '    span.setAttribute("langfuse.user.id", userId);',
-    '    span.setAttribute("user.id", userId);',
-    '    span.setAttribute("ai.telemetry.metadata.userId", userId);',
-    '    span.setAttribute("langfuse.trace.metadata.langfuse_user_id", userId);',
-    '    span.setAttribute("langfuse.observation.metadata.langfuse_user_id", userId);',
-    '    span.setAttribute("langfuse.metadata.langfuse_user_id", userId);',
-    "  },",
-    "  onEnd: (span) => {",
-    '    span.setAttribute("langfuse.user.id", userId);',
-    '    span.setAttribute("user.id", userId);',
-    '    span.setAttribute("ai.telemetry.metadata.userId", userId);',
-    '    span.setAttribute("langfuse.trace.metadata.langfuse_user_id", userId);',
-    '    span.setAttribute("langfuse.observation.metadata.langfuse_user_id", userId);',
-    '    span.setAttribute("langfuse.metadata.langfuse_user_id", userId);',
-    "  },",
-    "  forceFlush: async () => {},",
-    "  shutdown: async () => {},",
-    "});",
+    "const createUserIdSpanProcessor = (userId) => ({",
+    "  onStart: (span) => {",
+    '    span.setAttribute("langfuse.user.id", userId);',
+    '    span.setAttribute("user.id", userId);',
+    '    span.setAttribute("ai.telemetry.metadata.userId", userId);',
+    '    span.setAttribute("oh.langfuse.source", "opencode");',
+    '    span.setAttribute("oh.langfuse.user_id", userId);',
+    '    span.setAttribute("oh.langfuse.metrics_schema_version", "1.0");',
+    '    span.setAttribute("langfuse.trace.metadata.langfuse_user_id", userId);',
+    '    span.setAttribute("langfuse.observation.metadata.langfuse_user_id", userId);',
+    '    span.setAttribute("langfuse.metadata.langfuse_user_id", userId);',
+    '    span.setAttribute("langfuse.observation.metadata.source", "opencode");',
+    '    span.setAttribute("langfuse.observation.metadata.user_id", userId);',
+    '    span.setAttribute("langfuse.observation.metadata.metrics_schema_version", "1.0");',
+    "  },",
+    "  onEnd: (span) => {",
+    '    if (typeof span.setAttribute !== "function") return;',
+    '    span.setAttribute("langfuse.user.id", userId);',
+    '    span.setAttribute("user.id", userId);',
+    '    span.setAttribute("ai.telemetry.metadata.userId", userId);',
+    '    span.setAttribute("oh.langfuse.source", "opencode");',
+    '    span.setAttribute("oh.langfuse.user_id", userId);',
+    '    span.setAttribute("oh.langfuse.metrics_schema_version", "1.0");',
+    '    span.setAttribute("langfuse.trace.metadata.langfuse_user_id", userId);',
+    '    span.setAttribute("langfuse.observation.metadata.langfuse_user_id", userId);',
+    '    span.setAttribute("langfuse.metadata.langfuse_user_id", userId);',
+    '    span.setAttribute("langfuse.observation.metadata.source", "opencode");',
+    '    span.setAttribute("langfuse.observation.metadata.user_id", userId);',
+    '    span.setAttribute("langfuse.observation.metadata.metrics_schema_version", "1.0");',
+    "  },",
+    "  forceFlush: async () => {},",
+    "  shutdown: async () => {},",
+    "});",
+    "",
+    "const metricNumber = (value) => {",
+    '  if (typeof value === "number") return Number.isFinite(value) && value >= 0 ? value : undefined;',
+    '  if (typeof value === "string" && value.trim().startsWith("{")) {',
+    "    try {",
+    "      const parsed = JSON.parse(value);",
+    "      return metricNumber(parsed.intValue ?? parsed.doubleValue ?? parsed.value);",
+    "    } catch {",
+    "      return undefined;",
+    "    }",
+    "  }",
+    "  const n = Number(value);",
+    "  return Number.isFinite(n) && n >= 0 ? n : undefined;",
+    "};",
+    "",
+    "const writeSpanAttribute = (span, key, value) => {",
+    "  if (value === undefined || value === null) return;",
+    '  if (typeof span.setAttribute === "function") {',
+    "    span.setAttribute(key, value);",
+    "    return;",
+    "  }",
+    '  if (span.attributes && typeof span.attributes === "object") {',
+    "    span.attributes[key] = value;",
+    "  }",
+    "};",
+    "",
+    "const writeMetric = (span, key, value) => writeSpanAttribute(span, `langfuse.observation.metadata.${key}`, value);",
+    "",
+    "const writeOpencodeMetricAttributes = (span, userId) => {",
+    "  const attrs = span.attributes ?? {};",
+    '  writeSpanAttribute(span, "oh.langfuse.source", "opencode");',
+    '  writeSpanAttribute(span, "oh.langfuse.metrics_schema_version", "1.0");',
+    '  if (userId) writeSpanAttribute(span, "oh.langfuse.user_id", userId);',
+    '  writeMetric(span, "source", "opencode");',
+    '  writeMetric(span, "metrics_schema_version", "1.0");',
+    '  if (userId) writeMetric(span, "user_id", userId);',
+    '  const hasModel = attrs["ai.model.id"] || attrs["ai.model.provider"] || attrs["gen_ai.request.model"];',
+    '  const isTool = attrs["ai.toolCall.name"] || attrs["ai.toolCall.id"];',
+    "  if (!hasModel || isTool) return false;",
+    '  if (typeof span.updateName === "function") span.updateName("AI Interaction");',
+    '  const sessionId = String(attrs["ai.request.headers.x-opencode-session"] || attrs["session.id"] || "unknown");',
+    '  const requestId = String(attrs["ai.request.headers.x-opencode-request"] || attrs["ai.response.id"] || attrs["ai.operationId"] || "unknown");',
+    '  const spanId = span.spanContext?.().spanId || attrs["span.id"] || "unknown";',
+    '  const provider = attrs["ai.model.provider"] || attrs["gen_ai.system"] || "";',
+    '  const modelId = attrs["ai.model.id"] || attrs["gen_ai.request.model"] || attrs["ai.response.model"] || "";',
+    '  const model = provider && modelId ? `${provider}/${modelId}` : provider || modelId || undefined;',
+    '  const inputTokens = metricNumber(attrs["ai.usage.inputTokens"] ?? attrs["ai.usage.promptTokens"] ?? attrs["gen_ai.usage.input_tokens"]);',
+    '  const outputTokens = metricNumber(attrs["ai.usage.outputTokens"] ?? attrs["ai.usage.completionTokens"] ?? attrs["gen_ai.usage.output_tokens"]);',
+    '  const totalTokens = metricNumber(attrs["ai.usage.totalTokens"]) ?? (inputTokens !== undefined && outputTokens !== undefined ? inputTokens + outputTokens : undefined);',
+    '  const cacheReadTokens = metricNumber(attrs["ai.usage.cachedInputTokens"] ?? attrs["ai.usage.inputTokenDetails.cacheReadTokens"]);',
+    '  const reasoningTokens = metricNumber(attrs["ai.usage.reasoningTokens"] ?? attrs["ai.usage.outputTokenDetails.reasoningTokens"]);',
+    "  const tokenAvailable = [inputTokens, outputTokens, totalTokens, cacheReadTokens, reasoningTokens].some((value) => value !== undefined);",
+    '  writeMetric(span, "session_id", sessionId);',
+    '  writeMetric(span, "interaction_id", `opencode:${userId || "unknown"}:${sessionId}:${requestId}:${spanId}`);',
+    '  writeMetric(span, "interaction_count", 1);',
+    '  writeMetric(span, "user_message_count", 1);',
+    '  writeMetric(span, "assistant_message_count", 1);',
+    '  writeMetric(span, "token_metrics_available", tokenAvailable);',
+    '  writeMetric(span, "tool_call_count", 0);',
+    '  writeMetric(span, "tool_result_count", 0);',
+    '  writeMetric(span, "skill_use_count", 0);',
+    '  writeMetric(span, "model", model);',
+    '  writeMetric(span, "input_tokens", inputTokens);',
+    '  writeMetric(span, "output_tokens", outputTokens);',
+    '  writeMetric(span, "total_tokens", totalTokens);',
+    '  writeMetric(span, "cache_read_tokens", cacheReadTokens);',
+    '  writeMetric(span, "reasoning_tokens", reasoningTokens);',
+    "  return true;",
+    "};",
+    "",
+    "const createMetricsSpanProcessor = (userId) => ({",
+    "  onStart: (span) => {",
+    "    writeOpencodeMetricAttributes(span, userId);",
+    "  },",
+    "  onEnd: (span) => {",
+    "    writeOpencodeMetricAttributes(span, userId);",
+    "  },",
+    "  forceFlush: async () => {},",
+    "  shutdown: async () => {},",
+    "});",
+    "",
+    "const eventPayload = (event) => event?.properties ?? event?.body ?? event ?? {};",
+    "const eventPart = (event) => {",
+    "  const payload = eventPayload(event);",
+    "  return payload.part ?? payload.properties?.part ?? payload;",
+    "};",
+    "const pickEventString = (...values) => {",
+    "  for (const value of values) {",
+    '    if (typeof value === "string" && value.trim()) return value.trim();',
+    "  }",
+    '  return "";',
+    "};",
+    "const tokenMetricsFromPart = (part) => {",
+    "  const tokens = part?.tokens ?? part?.usage ?? {};",
+    "  return {",
+    "    input: metricNumber(tokens.input ?? tokens.input_tokens ?? tokens.inputTokens),",
+    "    output: metricNumber(tokens.output ?? tokens.output_tokens ?? tokens.outputTokens),",
+    "    total: metricNumber(tokens.total ?? tokens.total_tokens ?? tokens.totalTokens),",
+    "    cacheRead: metricNumber(tokens.cache?.read ?? tokens.cacheRead ?? tokens.cache_read_tokens ?? tokens.cachedInputTokens),",
+    "    reasoning: metricNumber(tokens.reasoning ?? tokens.reasoning_tokens ?? tokens.reasoningTokens),",
+    "  };",
+    "};",
     "",
     "export const LangfusePlugin = async ({ client }) => {",
     "  const publicKey = process.env.LANGFUSE_PUBLIC_KEY;",
@@ -185,31 +301,103 @@ function getPatchedLangfuseDistIndexJs() {
     "    return {};",
     "  }",
     "",
-    "  const processor = new LangfuseSpanProcessor({ publicKey, secretKey, baseUrl, environment });",
-    "  const spanProcessors = [processor];",
-    "  if (userId) spanProcessors.push(createUserIdSpanProcessor(userId));",
+    "  const processor = new LangfuseSpanProcessor({ publicKey, secretKey, baseUrl, environment });",
+    "  const spanProcessors = [processor, createMetricsSpanProcessor(userId)];",
+    "  if (userId) spanProcessors.push(createUserIdSpanProcessor(userId));",
     "",
-    "  const sdk = new NodeSDK({ spanProcessors });",
-    "  sdk.start();",
-    "",
-    '  log("info", `OTEL tracing initialized -> ${baseUrl}`);',
-    '  if (userId) log("info", `LANGFUSE userId configured -> ${userId}`);',
-    "",
-    "  return {",
-    "    config: async (config) => {",
+    "  const sdk = new NodeSDK({ spanProcessors });",
+    "  sdk.start();",
+    "  const metricsTracer = trace.getTracer('oh-langfuse-opencode-metrics');",
+    "  const messageTextById = new Map();",
+    "  const emittedMessageIds = new Set();",
+    "",
+    '  log("info", `OTEL tracing initialized -> ${baseUrl}`);',
+    '  if (userId) log("info", `LANGFUSE userId configured -> ${userId}`);',
+    "",
+    "  let shutdownStarted = false;",
+    "  const flush = async (reason) => {",
+    "    try {",
+    '      log("info", `Flushing OTEL spans on ${reason}`);',
+    "      await processor.forceFlush();",
+    "    } catch (error) {",
+    '      log("warn", `OTEL forceFlush failed on ${reason}: ${error?.message ?? error}`);',
+    "    }",
+    "  };",
+    "  const shutdown = async (reason) => {",
+    "    if (shutdownStarted) return;",
+    "    shutdownStarted = true;",
+    "    try {",
+    '      log("info", `Shutting down OTEL SDK on ${reason}`);',
+    "      await sdk.shutdown();",
+    "    } catch (error) {",
+    '      log("warn", `OTEL shutdown failed on ${reason}: ${error?.message ?? error}`);',
+    "    }",
+    "  };",
+    "  process.once('beforeExit', async () => {",
+    "    await flush('process.beforeExit');",
+    "    await shutdown('process.beforeExit');",
+    "  });",
+    "  const originalExit = process.exit.bind(process);",
+    "  process.exit = ((code) => {",
+    "    void shutdown('process.exit').finally(() => originalExit(code));",
+    "  });",
+    "",
+    "  const recordInteractionMetric = (event) => {",
+    "    const payload = eventPayload(event);",
+    "    const part = eventPart(event);",
+    "    const partType = part?.type ?? '';",
+    "    const sessionId = pickEventString(part?.sessionID, part?.sessionId, payload?.sessionID, payload?.sessionId, payload?.session?.id);",
+    "    const messageId = pickEventString(part?.messageID, part?.messageId, payload?.messageID, payload?.messageId, payload?.message?.id);",
+    "    if (partType === 'text' && messageId && typeof part.text === 'string') {",
+    "      messageTextById.set(messageId, part.text);",
+    "      return;",
+    "    }",
+    "    if (partType !== 'step-finish' || !messageId || emittedMessageIds.has(messageId)) return;",
+    "    emittedMessageIds.add(messageId);",
+    "    const tokenMetrics = tokenMetricsFromPart(part);",
+    "    const total = tokenMetrics.total ?? (tokenMetrics.input !== undefined && tokenMetrics.output !== undefined ? tokenMetrics.input + tokenMetrics.output : undefined);",
+    "    const tokenAvailable = [tokenMetrics.input, tokenMetrics.output, total, tokenMetrics.cacheRead, tokenMetrics.reasoning].some((value) => value !== undefined);",
+    "    const span = metricsTracer.startSpan('AI Interaction');",
+    "    const text = messageTextById.get(messageId) || '';",
+    '    span.setAttribute("oh.langfuse.source", "opencode");',
+    '    span.setAttribute("oh.langfuse.user_id", userId || "");',
+    '    span.setAttribute("oh.langfuse.metrics_schema_version", "1.0");',
+    '    span.setAttribute("langfuse.observation.metadata.source", "opencode");',
+    '    span.setAttribute("langfuse.observation.metadata.user_id", userId || "");',
+    '    span.setAttribute("langfuse.observation.metadata.session_id", sessionId || "unknown");',
+    '    span.setAttribute("langfuse.observation.metadata.interaction_id", `opencode:${userId || "unknown"}:${sessionId || "unknown"}:${messageId}`);',
+    '    span.setAttribute("langfuse.observation.metadata.metrics_schema_version", "1.0");',
+    '    span.setAttribute("langfuse.observation.metadata.interaction_count", 1);',
+    '    span.setAttribute("langfuse.observation.metadata.user_message_count", 1);',
+    '    span.setAttribute("langfuse.observation.metadata.assistant_message_count", 1);',
+    '    span.setAttribute("langfuse.observation.metadata.token_metrics_available", tokenAvailable);',
+    '    span.setAttribute("langfuse.observation.metadata.tool_call_count", 0);',
+    '    span.setAttribute("langfuse.observation.metadata.tool_result_count", 0);',
+    '    span.setAttribute("langfuse.observation.metadata.skill_use_count", 0);',
+    '    if (tokenMetrics.input !== undefined) span.setAttribute("langfuse.observation.metadata.input_tokens", tokenMetrics.input);',
+    '    if (tokenMetrics.output !== undefined) span.setAttribute("langfuse.observation.metadata.output_tokens", tokenMetrics.output);',
+    '    if (total !== undefined) span.setAttribute("langfuse.observation.metadata.total_tokens", total);',
+    '    if (tokenMetrics.cacheRead !== undefined) span.setAttribute("langfuse.observation.metadata.cache_read_tokens", tokenMetrics.cacheRead);',
+    '    if (tokenMetrics.reasoning !== undefined) span.setAttribute("langfuse.observation.metadata.reasoning_tokens", tokenMetrics.reasoning);',
+    '    if (text) span.setAttribute("langfuse.observation.metadata.output_text_preview", text.slice(0, 512));',
+    "    span.end();",
+    "  };",
+    "",
+    "  return {",
+    "    config: async (config) => {",
     "      if (!config.experimental?.openTelemetry) {",
     '        log("warn", "OpenTelemetry experimental feature is disabled in Opencode config - tracing disabled");',
     "      }",
-    "    },",
-    "    event: async ({ event }) => {",
-    '      if (event.type === "session.idle") {',
-    '        log("info", "Flushing OTEL spans before idle");',
-    "        await processor.forceFlush();",
-    "      }",
-    '      if (event.type === "server.instance.disposed") {',
-    '        log("info", "Shutting down OTEL SDK before dispose");',
-    "        await sdk.shutdown();",
-    "      }",
+    "    },",
+    "    event: async ({ event }) => {",
+    "      const eventType = event?.type ?? '';",
+    '      if (eventType === "message.part.updated") recordInteractionMetric(event);',
+    '      if (eventType === "session.idle" || eventType === "message.updated" || eventType === "message.part.updated" || eventType === "session.updated") {',
+    "        await flush(eventType);",
+    "      }",
+    '      if (eventType === "server.instance.disposed") {',
+    "        await shutdown('server.instance.disposed');",
+    "      }",
     "    },",
     "  };",
     "};",

package/scripts/real-self-verify.mjs

@@ -89,8 +89,24 @@ function shellNeeded(command) {
   return process.platform === "win32" && /\.(cmd|bat)$/i.test(command);
 }
 
+function windowsShellQuote(value) {
+  return `"${String(value).replace(/"/g, '\\"')}"`;
+}
+
 function run(command, args, options = {}) {
-  const result = spawnSync(command, args, {
+  const useShell = shellNeeded(command);
+  const result = useShell
+    ? spawnSync([command, ...args].map(windowsShellQuote).join(" "), {
+        cwd: options.cwd || rootDir,
+        env: { ...process.env, ...(options.env || {}) },
+        encoding: "utf8",
+        stdio: options.stdio || "pipe",
+        timeout: options.timeoutMs,
+        maxBuffer: 20 * 1024 * 1024,
+        windowsHide: true,
+        shell: true,
+      })
+    : spawnSync(command, args, {
     cwd: options.cwd || rootDir,
     env: { ...process.env, ...(options.env || {}) },
     encoding: "utf8",
@@ -98,7 +114,6 @@ function run(command, args, options = {}) {
     timeout: options.timeoutMs,
     maxBuffer: 20 * 1024 * 1024,
     windowsHide: true,
-    shell: shellNeeded(command),
   });
   return {
     command,
@@ -163,11 +178,7 @@ function setupTarget(target, config, args) {
 
 function validationPrompt(marker, target, customPrompt) {
   if (customPrompt) return String(customPrompt).replaceAll("{marker}", marker).replaceAll("{target}", target);
-  return [
-    `This is an automated real self-verification run for ${packageJson.name}.`,
-    `Validation marker: ${marker}`,
-    "Reply with the validation marker exactly once and do not run tools.",
-  ].join("\n");
+  return `Automated real self-verification run for ${packageJson.name}. Validation marker: ${marker}. Reply with the validation marker exactly once and do not run tools.`;
 }
 
 function triggerClaude(prompt, args, env) {
@@ -281,6 +292,133 @@ function idOf(value) {
   return value?.id || value?.traceId || value?.trace_id || "";
 }
 
+function traceIdOfFound(found) {
+  return found?.item?.traceId || found?.item?.trace_id || (String(found?.kind || "").startsWith("trace") ? found.id : "");
+}
+
+function metadataValue(item, key) {
+  const metadata = item?.metadata || {};
+  if (metadata[key] !== undefined) return metadata[key];
+  const attrs = metadata.attributes || {};
+  const prefixed = `langfuse.observation.metadata.${key}`;
+  if (attrs[prefixed] !== undefined) return attrs[prefixed];
+  const ohKey = `oh.langfuse.${key}`;
+  if (attrs[ohKey] !== undefined) return attrs[ohKey];
+  if (metadata.source === "opencode" || attrs["oh.langfuse.source"] === "opencode") {
+    if (key === "interaction_id") return `opencode:${metadata.user_id || attrs["oh.langfuse.user_id"] || "unknown"}:${item?.id || item?.traceId || item?.trace_id || "unknown"}`;
+    if (key === "token_metrics_available") {
+      return [
+        "ai.usage.inputTokens",
+        "ai.usage.outputTokens",
+        "ai.usage.totalTokens",
+        "ai.usage.promptTokens",
+        "ai.usage.completionTokens",
+      ].some((attrKey) => attrs[attrKey] !== undefined);
+    }
+    if (key === "tool_call_count" || key === "skill_use_count") return 0;
+    const tokenAttrMap = {
+      input_tokens: ["ai.usage.inputTokens", "ai.usage.promptTokens"],
+      output_tokens: ["ai.usage.outputTokens", "ai.usage.completionTokens"],
+      total_tokens: ["ai.usage.totalTokens"],
+      cache_read_tokens: ["ai.usage.cachedInputTokens", "ai.usage.inputTokenDetails.cacheReadTokens"],
+      reasoning_tokens: ["ai.usage.reasoningTokens", "ai.usage.outputTokenDetails.reasoningTokens"],
+    };
+    for (const attrKey of tokenAttrMap[key] || []) {
+      if (attrs[attrKey] !== undefined) return attrs[attrKey];
+    }
+  }
+  return undefined;
+}
+
+function directMetadataValue(item, key) {
+  const metadata = item?.metadata || {};
+  return metadata[key];
+}
+
+function hasMetadataKey(item, key) {
+  return metadataValue(item, key) !== undefined;
+}
+
+function isAiInteractionObservation(item) {
+  if (item?.name === "AI Interaction" || metadataValue(item, "interaction_count") === 1 || metadataValue(item, "interaction_count") === "1") {
+    return true;
+  }
+  const metadata = item?.metadata || {};
+  const attrs = metadata.attributes || {};
+  const isOpencode = metadata.source === "opencode" || attrs["oh.langfuse.source"] === "opencode";
+  const hasModel = attrs["ai.model.id"] !== undefined || attrs["ai.model.provider"] !== undefined;
+  const isTool = attrs["ai.toolCall.name"] !== undefined || attrs["ai.toolCall.id"] !== undefined;
+  return isOpencode && hasModel && !isTool && item?.type === "GENERATION";
+}
+
+async function observationsForTrace(config, traceId, since) {
+  if (!traceId) return [];
+  const params = { limit: 100, traceId };
+  try {
+    return dataArray(await langfuseGet(config, "/observations", params));
+  } catch (error) {
+    if (error.status !== 400) throw error;
+  }
+  const fallback = dataArray(await langfuseGetLenient(config, "/observations", { limit: 100, fromTimestamp: since.toISOString() }));
+  return fallback.filter((item) => item.traceId === traceId || item.trace_id === traceId);
+}
+
+async function verifyMetricObservations(config, found, { since, target }) {
+  const traceId = traceIdOfFound(found);
+  let observations = await observationsForTrace(config, traceId, since);
+
+  if (!observations.length && Array.isArray(found?.item?.observations)) {
+    observations = found.item.observations.filter((item) => typeof item === "object");
+  }
+
+  const interactions = observations.filter(isAiInteractionObservation);
+  if (!interactions.length) {
+    throw new Error(`Metric verification failed for ${target}: AI Interaction observation was not found for trace ${traceId || found.id}.`);
+  }
+
+  const byInteractionId = new Map();
+  for (const item of interactions) {
+    const interactionId = target === "opencode"
+      ? directMetadataValue(item, "interaction_id") || metadataValue(item, "interaction_id")
+      : metadataValue(item, "interaction_id");
+    if (!interactionId) {
+      throw new Error(`Metric verification failed for ${target}: AI Interaction is missing interaction_id.`);
+    }
+    byInteractionId.set(interactionId, (byInteractionId.get(interactionId) || 0) + 1);
+    for (const key of ["user_id", "token_metrics_available", "tool_call_count", "skill_use_count", "metrics_schema_version"]) {
+      if (!hasMetadataKey(item, key)) {
+        throw new Error(`Metric verification failed for ${target}: AI Interaction is missing ${key}.`);
+      }
+    }
+    if (target === "opencode") {
+      for (const key of ["interaction_id", "interaction_count", "token_metrics_available", "tool_call_count", "skill_use_count", "input_tokens", "output_tokens", "total_tokens"]) {
+        if (directMetadataValue(item, key) === undefined) {
+          throw new Error(`Metric verification failed for ${target}: visible metadata is missing ${key}.`);
+        }
+      }
+    }
+    const tokenAvailable = metadataValue(item, "token_metrics_available");
+    for (const tokenKey of ["input_tokens", "output_tokens", "total_tokens"]) {
+      const value = metadataValue(item, tokenKey);
+      if (tokenAvailable === false || tokenAvailable === "false") {
+        if (value !== null && value !== undefined) {
+          throw new Error(`Metric verification failed for ${target}: ${tokenKey} must be null or absent when token metrics are unavailable.`);
+        }
+      } else if (value !== undefined && value !== null && Number.isNaN(Number(value))) {
+        throw new Error(`Metric verification failed for ${target}: ${tokenKey} is not numeric.`);
+      }
+    }
+  }
+
+  for (const [interactionId, count] of byInteractionId.entries()) {
+    if (count !== 1) {
+      throw new Error(`Metric verification failed for ${target}: interaction_id ${interactionId} appeared ${count} times.`);
+    }
+  }
+
+  return { traceId, interactionCount: interactions.length };
+}
+
 async function findLangfuseMarker(config, marker, { since, target }) {
   const baseParams = {
     limit: 100,
@@ -415,7 +553,9 @@ async function main() {
 
     const found = await pollLangfuse(config, marker, { ...args, since, target });
     console.log(`[OK] Langfuse marker found for ${target}: ${found.kind} ${found.id || ""}`.trim());
-    results.push({ target, marker, langfuse: { kind: found.kind, id: found.id || "" } });
+    const metrics = await verifyMetricObservations(config, found, { since, target });
+    console.log(`[OK] Langfuse metrics found for ${target}: AI Interaction x${metrics.interactionCount}`);
+    results.push({ target, marker, langfuse: { kind: found.kind, id: found.id || "", traceId: metrics.traceId }, metrics });
   }
 
   console.log("");