oh-langfuse 0.1.28 → 0.1.29

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "oh-langfuse",
-  "version": "0.1.28",
+  "version": "0.1.29",
   "private": false,
   "type": "module",
   "description": "Use npm scripts to configure Claude Code / OpenCode / Codex with Langfuse tracing.",

package/scripts/metrics-utils.mjs

@@ -51,6 +51,82 @@ function ratio(numerator, denominator) {
   return numerator / denominator;
 }
 
+function normalizeSkillNames(names) {
+  if (!Array.isArray(names)) return [];
+  const out = [];
+  const seen = new Set();
+  for (const raw of names) {
+    const name = String(raw || "").trim();
+    if (!name || seen.has(name)) continue;
+    seen.add(name);
+    out.push(name);
+  }
+  return out;
+}
+
+function escapeRegExp(value) {
+  return String(value).replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+function collectStrings(value, out = []) {
+  if (value == null) return out;
+  if (typeof value === "string") {
+    out.push(value);
+    return out;
+  }
+  if (typeof value === "number" || typeof value === "boolean") return out;
+  if (Array.isArray(value)) {
+    for (const item of value) collectStrings(item, out);
+    return out;
+  }
+  if (typeof value === "object") {
+    for (const [key, item] of Object.entries(value)) {
+      out.push(key);
+      collectStrings(item, out);
+    }
+  }
+  return out;
+}
+
+export function detectOpencodeSkillNames(source, knownSkills = []) {
+  const skills = normalizeSkillNames(knownSkills);
+  if (skills.length === 0) return [];
+  const haystack = collectStrings(source).join("\n");
+  const found = [];
+  for (const skillName of skills) {
+    const pattern = new RegExp(`(^|[^A-Za-z0-9_-])${escapeRegExp(skillName)}([^A-Za-z0-9_-]|$)`, "i");
+    if (pattern.test(haystack)) found.push(skillName);
+  }
+  return found;
+}
+
+function pickString(...values) {
+  for (const value of values) {
+    if (typeof value === "string" && value.trim()) return value.trim();
+  }
+  return "";
+}
+
+export function countOpencodeToolActivity(source) {
+  const payload = source?.properties ?? source?.body ?? source ?? {};
+  const part = payload?.part ?? payload?.properties?.part ?? payload;
+  const eventType = pickString(source?.type, payload?.type);
+  const partType = pickString(part?.type);
+  const toolName = pickString(part?.tool, part?.toolName, part?.name, payload?.tool, payload?.toolName, payload?.name);
+  const callId = pickString(part?.callID, part?.callId, payload?.callID, payload?.callId, part?.id);
+  const status = pickString(part?.state?.status, part?.status, payload?.state?.status, payload?.status).toLowerCase();
+  const hasToolInput = part?.state?.input !== undefined || part?.input !== undefined || payload?.input !== undefined;
+  const hasToolOutput = part?.state?.output !== undefined || part?.output !== undefined || payload?.output !== undefined;
+  const isToolCall = eventType === "tool_use" || eventType === "tool_result" || partType === "tool" || partType === "tool_use" || Boolean(toolName) || hasToolInput;
+  const isToolResult = eventType === "tool_result" || status === "completed" || status === "success" || hasToolOutput;
+
+  return {
+    toolCallCount: isToolCall ? 1 : 0,
+    toolResultCount: isToolCall && isToolResult ? 1 : 0,
+    toolCallId: callId || "",
+  };
+}
+
 export function buildInteractionMetadata(options = {}) {
   const source = String(options.source || "unknown");
   const sessionId = String(options.sessionId || options.session_id || "unknown");
@@ -58,9 +134,10 @@ export function buildInteractionMetadata(options = {}) {
   const tokenMetrics = normalizeTokenMetrics(options.tokenMetrics);
   const toolCallCount = Number(options.toolCallCount ?? options.tool_call_count ?? 0) || 0;
   const toolResultCount = Number(options.toolResultCount ?? options.tool_result_count ?? 0) || 0;
-  const skillUseCount = Number(options.skillUseCount ?? options.skill_use_count ?? 0) || 0;
+  const skillNames = normalizeSkillNames(options.skillNames ?? options.skill_names);
+  const skillUseCount = skillNames.length || Number(options.skillUseCount ?? options.skill_use_count ?? 0) || 0;
 
-  return {
+  const metadata = {
     source,
     user_id: String(options.userId || options.user_id || ""),
     session_id: sessionId,
@@ -83,6 +160,13 @@ export function buildInteractionMetadata(options = {}) {
       tokens_per_tool_call: ratio(tokenMetrics.total_tokens, toolCallCount),
     },
   };
+
+  if (skillNames.length) {
+    metadata.skill_names = skillNames;
+    metadata.skill_names_json = JSON.stringify(skillNames);
+  }
+
+  return metadata;
 }
 
 export function buildOpencodeMetricAttributes(options = {}) {
@@ -94,6 +178,9 @@ export function buildOpencodeMetricAttributes(options = {}) {
   const provider = attrs["ai.model.provider"] || attrs["gen_ai.system"] || "";
   const modelId = attrs["ai.model.id"] || attrs["gen_ai.request.model"] || attrs["ai.response.model"] || "";
   const model = provider && modelId ? `${provider}/${modelId}` : provider || modelId || null;
+  const skillNames = normalizeSkillNames(options.skillNames ?? options.skill_names);
+  const toolCallCount = Number(options.toolCallCount ?? options.tool_call_count ?? 0) || 0;
+  const toolResultCount = Number(options.toolResultCount ?? options.tool_result_count ?? 0) || 0;
   const tokenMetrics = normalizeTokenMetrics({
     input: attrs["ai.usage.inputTokens"] ?? attrs["ai.usage.promptTokens"] ?? attrs["gen_ai.usage.input_tokens"],
     output: attrs["ai.usage.outputTokens"] ?? attrs["ai.usage.completionTokens"] ?? attrs["gen_ai.usage.output_tokens"],
@@ -111,13 +198,18 @@ export function buildOpencodeMetricAttributes(options = {}) {
     "langfuse.observation.metadata.interaction_count": 1,
     "langfuse.observation.metadata.user_message_count": 1,
     "langfuse.observation.metadata.assistant_message_count": 1,
-    "langfuse.observation.metadata.tool_call_count": 0,
-    "langfuse.observation.metadata.tool_result_count": 0,
-    "langfuse.observation.metadata.skill_use_count": 0,
+    "langfuse.observation.metadata.tool_call_count": toolCallCount,
+    "langfuse.observation.metadata.tool_result_count": toolResultCount,
+    "langfuse.observation.metadata.skill_use_count": skillNames.length,
     "langfuse.observation.metadata.token_metrics_available": tokenMetrics.token_metrics_available,
     "langfuse.observation.metadata.model": model,
   };
 
+  if (skillNames.length) {
+    out["langfuse.observation.metadata.skill_names"] = skillNames;
+    out["langfuse.observation.metadata.skill_names_json"] = JSON.stringify(skillNames);
+  }
+
   for (const key of ["input_tokens", "output_tokens", "total_tokens", "cache_read_tokens", "reasoning_tokens"]) {
     if (tokenMetrics[key] != null) out[`langfuse.observation.metadata.${key}`] = tokenMetrics[key];
   }

package/scripts/opencode-langfuse-setup.mjs

@@ -271,6 +271,24 @@ function getPatchedLangfuseDistIndexJs() {
     "  }",
     '  return "";',
     "};",
+    "const countOpencodeToolActivity = (event) => {",
+    "  const payload = eventPayload(event);",
+    "  const part = eventPart(event);",
+    "  const eventType = pickEventString(event?.type, payload?.type);",
+    "  const partType = pickEventString(part?.type);",
+    "  const toolName = pickEventString(part?.tool, part?.toolName, part?.name, payload?.tool, payload?.toolName, payload?.name);",
+    "  const callId = pickEventString(part?.callID, part?.callId, payload?.callID, payload?.callId, part?.id);",
+    "  const status = pickEventString(part?.state?.status, part?.status, payload?.state?.status, payload?.status).toLowerCase();",
+    "  const hasToolInput = part?.state?.input !== undefined || part?.input !== undefined || payload?.input !== undefined;",
+    "  const hasToolOutput = part?.state?.output !== undefined || part?.output !== undefined || payload?.output !== undefined;",
+    "  const isToolCall = eventType === 'tool_use' || eventType === 'tool_result' || partType === 'tool' || partType === 'tool_use' || Boolean(toolName) || hasToolInput;",
+    "  const isToolResult = eventType === 'tool_result' || status === 'completed' || status === 'success' || hasToolOutput;",
+    "  return {",
+    "    toolCallCount: isToolCall ? 1 : 0,",
+    "    toolResultCount: isToolCall && isToolResult ? 1 : 0,",
+    "    toolCallId: callId || '',",
+    "  };",
+    "};",
     "const tokenMetricsFromPart = (part) => {",
     "  const tokens = part?.tokens ?? part?.usage ?? {};",
     "  return {",
@@ -281,8 +299,84 @@ function getPatchedLangfuseDistIndexJs() {
     "    reasoning: metricNumber(tokens.reasoning ?? tokens.reasoning_tokens ?? tokens.reasoningTokens),",
     "  };",
     "};",
-    "",
-    "export const LangfusePlugin = async ({ client }) => {",
+    "",
+    "const normalizeSkillNames = (names) => {",
+    "  if (!Array.isArray(names)) return [];",
+    "  const out = [];",
+    "  const seen = new Set();",
+    "  for (const raw of names) {",
+    "    const name = String(raw || '').trim();",
+    "    if (!name || seen.has(name)) continue;",
+    "    seen.add(name);",
+    "    out.push(name);",
+    "  }",
+    "  return out;",
+    "};",
+    "",
+    "const escapeRegExp = (value) => String(value).replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&');",
+    "",
+    "const collectStrings = (value, out = []) => {",
+    "  if (value == null) return out;",
+    "  if (typeof value === 'string') {",
+    "    out.push(value);",
+    "    return out;",
+    "  }",
+    "  if (typeof value === 'number' || typeof value === 'boolean') return out;",
+    "  if (Array.isArray(value)) {",
+    "    for (const item of value) collectStrings(item, out);",
+    "    return out;",
+    "  }",
+    "  if (typeof value === 'object') {",
+    "    for (const [key, item] of Object.entries(value)) {",
+    "      out.push(key);",
+    "      collectStrings(item, out);",
+    "    }",
+    "  }",
+    "  return out;",
+    "};",
+    "",
+    "const detectOpencodeSkillNames = (source, knownSkills = []) => {",
+    "  const skills = normalizeSkillNames(knownSkills);",
+    "  if (skills.length === 0) return [];",
+    "  const haystack = collectStrings(source).join('\\n');",
+    "  const found = [];",
+    "  for (const skillName of skills) {",
+    "    const pattern = new RegExp(`(^|[^A-Za-z0-9_-])${escapeRegExp(skillName)}([^A-Za-z0-9_-]|$)`, 'i');",
+    "    if (pattern.test(haystack)) found.push(skillName);",
+    "  }",
+    "  return found;",
+    "};",
+    "",
+    "const collectKnownSkillNames = async () => {",
+    "  const dirs = [",
+    "    path.join(process.cwd(), '.opencode', 'skill'),",
+    "    path.join(process.cwd(), '.opencode', 'skills'),",
+    "    path.join(os.homedir(), '.config', 'opencode', 'skill'),",
+    "    path.join(os.homedir(), '.config', 'opencode', 'skills'),",
+    "    path.join(os.homedir(), '.opencode', 'skill'),",
+    "    path.join(os.homedir(), '.opencode', 'skills'),",
+    "  ];",
+    "  const names = new Set();",
+    "  for (const dir of dirs) {",
+    "    try {",
+    "      const entries = await fs.readdir(dir, { withFileTypes: true });",
+    "      for (const entry of entries) {",
+    "        if (!entry.isDirectory() || entry.name.startsWith('.')) continue;",
+    "        try {",
+    "          await fs.access(path.join(dir, entry.name, 'SKILL.md'));",
+    "          names.add(entry.name);",
+    "        } catch {",
+    "          // Ignore helper folders that are not OpenCode skills.",
+    "        }",
+    "      }",
+    "    } catch {",
+    "      // Optional skill directories are allowed to be absent.",
+    "    }",
+    "  }",
+    "  return [...names];",
+    "};",
+    "",
+    "export const LangfusePlugin = async ({ client }) => {",
     "  const publicKey = process.env.LANGFUSE_PUBLIC_KEY;",
     "  const secretKey = process.env.LANGFUSE_SECRET_KEY;",
     '  const baseUrl = process.env.LANGFUSE_BASEURL ?? "https://cloud.langfuse.com";',
@@ -308,11 +402,19 @@ function getPatchedLangfuseDistIndexJs() {
     "  const sdk = new NodeSDK({ spanProcessors });",
     "  sdk.start();",
     "  const metricsTracer = trace.getTracer('oh-langfuse-opencode-metrics');",
+    "  const knownSkillNames = await collectKnownSkillNames();",
     "  const messageTextById = new Map();",
+    "  const skillNamesByMessageId = new Map();",
+    "  const skillNamesBySessionId = new Map();",
+    "  const toolCallIdsByMessageId = new Map();",
+    "  const toolCallIdsBySessionId = new Map();",
+    "  const toolResultIdsByMessageId = new Map();",
+    "  const toolResultIdsBySessionId = new Map();",
     "  const emittedMessageIds = new Set();",
     "",
     '  log("info", `OTEL tracing initialized -> ${baseUrl}`);',
     '  if (userId) log("info", `LANGFUSE userId configured -> ${userId}`);',
+    '  if (knownSkillNames.length) log("info", `OpenCode skills discovered -> ${knownSkillNames.length}`);',
     "",
     "  let shutdownStarted = false;",
     "  const flush = async (reason) => {",
@@ -342,14 +444,51 @@ function getPatchedLangfuseDistIndexJs() {
     "    void shutdown('process.exit').finally(() => originalExit(code));",
     "  });",
     "",
+    "  const rememberSkillNames = (messageId, names) => {",
+    "    if (!messageId || !names.length) return;",
+    "    let set = skillNamesByMessageId.get(messageId);",
+    "    if (!set) {",
+    "      set = new Set();",
+    "      skillNamesByMessageId.set(messageId, set);",
+    "    }",
+    "    for (const name of names) set.add(name);",
+    "  };",
+    "  const rememberSessionSkillNames = (sessionId, names) => {",
+    "    if (!sessionId || !names.length) return;",
+    "    let set = skillNamesBySessionId.get(sessionId);",
+    "    if (!set) {",
+    "      set = new Set();",
+    "      skillNamesBySessionId.set(sessionId, set);",
+    "    }",
+    "    for (const name of names) set.add(name);",
+    "  };",
+    "  const rememberToolActivity = (map, key, activity, kind) => {",
+    "    if (!key || !activity?.[kind]) return;",
+    "    let set = map.get(key);",
+    "    if (!set) {",
+    "      set = new Set();",
+    "      map.set(key, set);",
+    "    }",
+    "    set.add(activity.toolCallId || `${kind}:${set.size + 1}`);",
+    "  };",
+    "",
     "  const recordInteractionMetric = (event) => {",
     "    const payload = eventPayload(event);",
     "    const part = eventPart(event);",
     "    const partType = part?.type ?? '';",
     "    const sessionId = pickEventString(part?.sessionID, part?.sessionId, payload?.sessionID, payload?.sessionId, payload?.session?.id);",
     "    const messageId = pickEventString(part?.messageID, part?.messageId, payload?.messageID, payload?.messageId, payload?.message?.id);",
+    "    const eventSkillNames = detectOpencodeSkillNames([event, payload, part, messageTextById.get(messageId)], knownSkillNames);",
+    "    rememberSkillNames(messageId, eventSkillNames);",
+    "    rememberSessionSkillNames(sessionId, eventSkillNames);",
+    "    const toolActivity = countOpencodeToolActivity(event);",
+    "    rememberToolActivity(toolCallIdsByMessageId, messageId, toolActivity, 'toolCallCount');",
+    "    rememberToolActivity(toolCallIdsBySessionId, sessionId, toolActivity, 'toolCallCount');",
+    "    rememberToolActivity(toolResultIdsByMessageId, messageId, toolActivity, 'toolResultCount');",
+    "    rememberToolActivity(toolResultIdsBySessionId, sessionId, toolActivity, 'toolResultCount');",
     "    if (partType === 'text' && messageId && typeof part.text === 'string') {",
     "      messageTextById.set(messageId, part.text);",
+    "      rememberSkillNames(messageId, detectOpencodeSkillNames(part.text, knownSkillNames));",
     "      return;",
     "    }",
     "    if (partType !== 'step-finish' || !messageId || emittedMessageIds.has(messageId)) return;",
@@ -359,6 +498,9 @@ function getPatchedLangfuseDistIndexJs() {
     "    const tokenAvailable = [tokenMetrics.input, tokenMetrics.output, total, tokenMetrics.cacheRead, tokenMetrics.reasoning].some((value) => value !== undefined);",
     "    const span = metricsTracer.startSpan('AI Interaction');",
     "    const text = messageTextById.get(messageId) || '';",
+    "    const skillNames = [...new Set([...(skillNamesByMessageId.get(messageId) ?? []), ...(skillNamesBySessionId.get(sessionId) ?? [])])];",
+    "    const toolCallCount = new Set([...(toolCallIdsByMessageId.get(messageId) ?? []), ...(toolCallIdsBySessionId.get(sessionId) ?? [])]).size;",
+    "    const toolResultCount = new Set([...(toolResultIdsByMessageId.get(messageId) ?? []), ...(toolResultIdsBySessionId.get(sessionId) ?? [])]).size;",
     '    span.setAttribute("oh.langfuse.source", "opencode");',
     '    span.setAttribute("oh.langfuse.user_id", userId || "");',
     '    span.setAttribute("oh.langfuse.metrics_schema_version", "1.0");',
@@ -371,9 +513,12 @@ function getPatchedLangfuseDistIndexJs() {
     '    span.setAttribute("langfuse.observation.metadata.user_message_count", 1);',
     '    span.setAttribute("langfuse.observation.metadata.assistant_message_count", 1);',
     '    span.setAttribute("langfuse.observation.metadata.token_metrics_available", tokenAvailable);',
-    '    span.setAttribute("langfuse.observation.metadata.tool_call_count", 0);',
-    '    span.setAttribute("langfuse.observation.metadata.tool_result_count", 0);',
-    '    span.setAttribute("langfuse.observation.metadata.skill_use_count", 0);',
+    '    span.setAttribute("langfuse.observation.metadata.tool_call_count", toolCallCount);',
+    '    span.setAttribute("langfuse.observation.metadata.tool_result_count", toolResultCount);',
+    '    span.setAttribute("langfuse.observation.metadata.skill_use_count", skillNames.length);',
+    '    if (skillNames.length) span.setAttribute("langfuse.observation.metadata.skill_names", skillNames);',
+    '    if (skillNames.length) span.setAttribute("langfuse.observation.metadata.skill_names_json", JSON.stringify(skillNames));',
+    '    if (skillNames.length) span.setAttribute("langfuse.observation.metadata.skill_names_csv", skillNames.join(","));',
     '    if (tokenMetrics.input !== undefined) span.setAttribute("langfuse.observation.metadata.input_tokens", tokenMetrics.input);',
     '    if (tokenMetrics.output !== undefined) span.setAttribute("langfuse.observation.metadata.output_tokens", tokenMetrics.output);',
     '    if (total !== undefined) span.setAttribute("langfuse.observation.metadata.total_tokens", total);',
@@ -381,6 +526,13 @@ function getPatchedLangfuseDistIndexJs() {
     '    if (tokenMetrics.reasoning !== undefined) span.setAttribute("langfuse.observation.metadata.reasoning_tokens", tokenMetrics.reasoning);',
     '    if (text) span.setAttribute("langfuse.observation.metadata.output_text_preview", text.slice(0, 512));',
     "    span.end();",
+    "    messageTextById.delete(messageId);",
+    "    skillNamesByMessageId.delete(messageId);",
+    "    skillNamesBySessionId.delete(sessionId);",
+    "    toolCallIdsByMessageId.delete(messageId);",
+    "    toolCallIdsBySessionId.delete(sessionId);",
+    "    toolResultIdsByMessageId.delete(messageId);",
+    "    toolResultIdsBySessionId.delete(sessionId);",
     "  };",
     "",
     "  return {",
@@ -391,7 +543,7 @@ function getPatchedLangfuseDistIndexJs() {
     "    },",
     "    event: async ({ event }) => {",
     "      const eventType = event?.type ?? '';",
-    '      if (eventType === "message.part.updated") recordInteractionMetric(event);',
+    "      recordInteractionMetric(event);",
     '      if (eventType === "session.idle" || eventType === "message.updated" || eventType === "message.part.updated" || eventType === "session.updated") {',
     "        await flush(eventType);",
     "      }",