agentel 0.2.8 → 0.3.0

package/src/canonical-events.js

@@ -1,15 +1,19 @@
 "use strict";
 
 const crypto = require("crypto");
+const { normalizeStructuredPatch, structuredPatchFromDiffText, structuredPatchFromToolArguments } = require("./diffs");
 const { parserVersionForSource } = require("./parser-versions");
 
-const CANONICAL_EVENT_SCHEMA_VERSION = "agentlog.events.v2";
+const CANONICAL_EVENT_SCHEMA_VERSION = "agentlog.events.v5";
 
 const EVENT_KINDS = {
   SESSION_STARTED: "session.started",
   PROMPT_SUBMITTED: "prompt.submitted",
   TOOL_CALLED: "tool.called",
   TOOL_COMPLETED: "tool.completed",
+  MEMORY_READ: "memory.read",
+  MEMORY_WRITE: "memory.write",
+  MEMORY_LOADED: "memory.loaded",
   RESPONSE_GENERATED: "response.generated"
 };
 
@@ -95,7 +99,15 @@ function normalizeSessionEvents(session, messages, options = {}) {
   const parserVersion = options.parserVersion ?? session.parserVersion ?? parserVersionForSource(session.sourceType);
   const state = {
     toolCallEventIdsByKey: new Map(),
-    unmatchedToolCallEventIds: []
+    // Ordered list of tool-call event ids still awaiting a result, plus a Set
+    // for O(1) membership and a head pointer for the FIFO fallback. Consumed
+    // ids are tombstoned (left in the array, removed from the set) so we never
+    // pay O(n) splice/indexOf per result — important for sessions with tens of
+    // thousands of tool calls.
+    unmatchedToolCallEventIds: [],
+    unmatchedToolCallEventIdSet: new Set(),
+    unmatchedHead: 0,
+    memoryByToolCallEventId: new Map()
   };
   const events = [
     baseEvent(session, {
@@ -125,7 +137,7 @@ function normalizeSessionEvents(session, messages, options = {}) {
 
 function messageToCanonicalEvents(message, session, options = {}) {
   const role = String(message?.role || "").toLowerCase();
-  if (!message || role === "system" || role === "developer") return [];
+  if (!message) return [];
   const messageIndex = Number.isFinite(Number(message.index)) ? Number(message.index) : options.messageIndex || 0;
   const parserVersion = options.parserVersion ?? session.parserVersion ?? parserVersionForSource(session.sourceType);
   const occurredAt = message.timestamp || session.startedAt || new Date().toISOString();
@@ -133,7 +145,26 @@ function messageToCanonicalEvents(message, session, options = {}) {
   const metadata = message.metadata || {};
   const events = [];
 
+  if (role === "system" || role === "developer") {
+    const memory = memoryActivityFromSystemMessage(message);
+    return memory
+      ? [
+          memoryCanonicalEvent(session, {
+            messageIndex,
+            ordinal: 0,
+            occurredAt,
+            parserVersion,
+            role,
+            memory,
+            text: memory.contentPreview || memoryEventSummary(memory),
+            status: "loaded"
+          })
+        ]
+      : [];
+  }
+
   if (role === "user") {
+    const skills = skillInvocationsFromPromptText(content);
     events.push(
       baseEvent(session, {
         messageIndex,
@@ -145,7 +176,8 @@ function messageToCanonicalEvents(message, session, options = {}) {
         indexed: {
           title: "User prompt",
           summary: summarize(content),
-          status: "submitted"
+          status: "submitted",
+          ...(skills.length ? { skills } : {})
         },
         body: { text: content, toolCall: null, toolResult: null }
       })
@@ -178,6 +210,8 @@ function messageToCanonicalEvents(message, session, options = {}) {
     const toolCall = normalizeToolCall(toolCalls[index], metadata.provider || session.provider);
     if (!toolCall) continue;
     const rendered = renderToolCallText(toolCall);
+    const toolSkills = skillNamesFromToolCall(toolCall);
+    const memory = memoryActivityFromToolCall(toolCall);
     const event = baseEvent(session, {
       messageIndex,
       ordinal: index + 1,
@@ -193,42 +227,296 @@ function messageToCanonicalEvents(message, session, options = {}) {
         toolCategory: toolCall.category || "",
         toolIcon: toolCall.icon || "",
         target: toolCall.target || "",
-        status: toolCall.status || "tool_call"
+        status: toolCall.status || "tool_call",
+        ...(toolSkills.length ? { skills: toolSkills } : {})
       },
       body: { text: rendered, toolCall, toolResult: null }
     });
     rememberToolCallEvent(options.state, toolCall, event.eventId);
+    if (memory && options.state?.memoryByToolCallEventId) options.state.memoryByToolCallEventId.set(event.eventId, memory);
     events.push(event);
   }
 
   const toolResult = metadata.toolResult ? normalizeToolResult(metadata.toolResult, metadata.provider || session.provider) : role === "tool" ? normalizeToolResult(content, metadata.provider || session.provider) : null;
   if (toolResult) {
     const parentEventId = consumeToolCallEventId(options.state, toolResult);
-    events.push(
-      baseEvent(session, {
-        messageIndex,
-        ordinal: events.length + 1,
-        kind: EVENT_KINDS.TOOL_COMPLETED,
-        occurredAt,
-        parserVersion,
-        role: "tool",
-        indexed: {
-          title: toolResult.title || toolResult.kind || "Tool result",
-          summary: summarize(toolResult.summary || toolResult.output || content),
-          toolName: toolResult.name || toolResult.kind || "",
-          toolCategory: toolResult.category || "",
-          toolIcon: toolResult.icon || "",
-          status: toolResult.status || "completed"
-        },
-        parentEventId,
-        body: { text: toolResult.output || content, toolCall: null, toolResult }
-      })
-    );
+    const completedEvent = baseEvent(session, {
+      messageIndex,
+      ordinal: events.length + 1,
+      kind: EVENT_KINDS.TOOL_COMPLETED,
+      occurredAt,
+      parserVersion,
+      role: "tool",
+      indexed: {
+        title: toolResult.title || toolResult.kind || "Tool result",
+        summary: summarize(toolResult.summary || toolResult.output || content),
+        toolName: toolResult.name || toolResult.kind || "",
+        toolCategory: toolResult.category || "",
+        toolIcon: toolResult.icon || "",
+        status: toolResult.status || "completed"
+      },
+      parentEventId,
+      body: { text: toolResult.output || content, toolCall: null, toolResult }
+    });
+    events.push(completedEvent);
+    const memory = parentEventId ? options.state?.memoryByToolCallEventId?.get(parentEventId) : null;
+    if (memory && isSuccessfulToolResult(toolResult)) {
+      events.push(
+        memoryCanonicalEvent(session, {
+          messageIndex,
+          ordinal: events.length + 1,
+          occurredAt,
+          parserVersion,
+          role: "tool",
+          parentEventId: completedEvent.eventId,
+          memory,
+          text: memoryEventSummary(memory),
+          status: "completed"
+        })
+      );
+    }
   }
 
   return events;
 }
 
+function memoryCanonicalEvent(session, options = {}) {
+  const memory = options.memory || {};
+  const action = String(memory.action || "loaded").toLowerCase();
+  const kind = action === "read" ? EVENT_KINDS.MEMORY_READ : action === "write" ? EVENT_KINDS.MEMORY_WRITE : EVENT_KINDS.MEMORY_LOADED;
+  return baseEvent(session, {
+    messageIndex: options.messageIndex,
+    ordinal: options.ordinal,
+    kind,
+    occurredAt: options.occurredAt,
+    parserVersion: options.parserVersion,
+    role: options.role || "system",
+    parentEventId: options.parentEventId || "",
+    indexed: {
+      title: memoryEventTitle(memory),
+      summary: summarize(options.text || memoryEventSummary(memory)),
+      status: options.status || "completed",
+      memoryAction: action,
+      memoryPath: memory.path || "",
+      memorySource: memory.source || "",
+      memoryScope: memory.scope || ""
+    },
+    body: {
+      text: options.text || memoryEventSummary(memory),
+      toolCall: null,
+      toolResult: null,
+      memory
+    }
+  });
+}
+
+// Skill invocations appear in prompt text as Codex-style markdown markers
+// ("[$name](/path/to/SKILL.md)") or Claude-style command tags
+// ("<command-name>/recall</command-name>"). Names are normalized without the
+// "$"/"/" decorations so telemetry can aggregate across providers.
+function skillInvocationsFromPromptText(text) {
+  const value = String(text || "");
+  const names = new Set();
+  const markerPattern = /\[\$([^\]\n]+)\]\(([^)\n]*)\)/g;
+  let match;
+  while ((match = markerPattern.exec(value))) {
+    const name = normalizeSkillName(match[1]);
+    if (name) names.add(name);
+  }
+  const commandPattern = /<command-name>([^<\n]+)<\/command-name>/g;
+  while ((match = commandPattern.exec(value))) {
+    const name = normalizeSkillName(match[1]);
+    if (name) names.add(name);
+  }
+  return [...names];
+}
+
+function skillNamesFromToolCall(toolCall) {
+  if (!toolCall || toolCall.category !== "skill") return [];
+  const args = toolCall.arguments && typeof toolCall.arguments === "object" ? toolCall.arguments : {};
+  const candidates = [args.skill, args.skill_name, args.skillName, args.name, args.command, toolCall.target];
+  for (const candidate of candidates) {
+    const name = normalizeSkillName(candidate);
+    if (name) return [name];
+  }
+  return [];
+}
+
+function normalizeSkillName(value) {
+  const text = String(value || "").trim().replace(/^[$/]+/, "").trim();
+  if (!text || text.length > 120 || /\s{2,}/.test(text)) return "";
+  return text;
+}
+
+const MEMORY_PATH_BASES = new Set([
+  "agents.md",
+  "claude.md",
+  "gemini.md",
+  "memory.md",
+  ".cursorrules",
+  ".windsurfrules",
+  "rules.md"
+]);
+
+const READ_TOOL_NAMES = new Set(["read", "notebookread", "cat", "sed", "view", "open"]);
+const WRITE_TOOL_NAMES = new Set(["write", "edit", "multiedit", "notebookedit", "apply_patch", "patch", "replace", "update_file"]);
+
+function memoryActivityFromToolCall(toolCall) {
+  const info = memoryInfoFromToolCall(toolCall);
+  if (!info) return null;
+  const action = memoryToolAction(toolCall);
+  if (!action) return null;
+  return { action, ...info };
+}
+
+function memoryActivityFromSystemMessage(message) {
+  const metadata = message?.metadata || {};
+  const contextKind = String(metadata.contextKind || "").toLowerCase();
+  if (contextKind === "remote_control_nested_memory") {
+    const attachment = metadata.attachment && typeof metadata.attachment === "object" ? metadata.attachment : {};
+    const rawPath = firstString(attachment.path, attachment.contentPath, attachment.displayPath);
+    const info = memoryPathInfo(rawPath) || genericMemoryInfo(rawPath || "memory", "remote-control-memory", "remote");
+    return {
+      action: "load",
+      ...info,
+      contentPreview: firstString(attachment.contentPreview)
+    };
+  }
+  if (contextKind === "project_instructions") {
+    const path = projectInstructionsPathFromContent(message.content);
+    const info = memoryPathInfo(path);
+    if (info) return { action: "load", ...info, contentPreview: summarize(message.content, 240) };
+  }
+  return null;
+}
+
+function memoryInfoFromToolCall(toolCall) {
+  for (const candidate of memoryPathCandidates(toolCall)) {
+    const info = memoryPathInfo(candidate);
+    if (info) return info;
+  }
+  return null;
+}
+
+function memoryToolAction(toolCall) {
+  const name = normalizeToolKey(toolCall?.name || toolCall?.displayName || toolCall?.title);
+  const category = normalizeToolKey(toolCall?.category || toolCall?.rawCategory);
+  if (category === "read" || READ_TOOL_NAMES.has(name)) return "read";
+  if (category === "edit" || WRITE_TOOL_NAMES.has(name)) return "write";
+  return "";
+}
+
+function memoryPathCandidates(toolCall) {
+  const out = [];
+  appendMemoryPathCandidate(out, toolCall?.target);
+  appendMemoryPathCandidate(out, toolCall?.argument);
+  appendMemoryPathCandidate(out, toolCall?.rawInputSummary);
+  appendMemoryPathCandidate(out, toolCall?.inputPreview);
+  collectMemoryPathValues(toolCall?.arguments, "", 0, out);
+  return [...new Set(out.map(normalizeMemoryPath).filter(Boolean))];
+}
+
+function collectMemoryPathValues(value, key = "", depth = 0, out = []) {
+  if (value == null || depth > 5) return out;
+  if (typeof value === "string") {
+    if (isPathArgumentKey(key)) appendMemoryPathCandidate(out, value);
+    return out;
+  }
+  if (Array.isArray(value)) {
+    for (const item of value) collectMemoryPathValues(item, key, depth + 1, out);
+    return out;
+  }
+  if (typeof value !== "object") return out;
+  for (const [childKey, childValue] of Object.entries(value)) collectMemoryPathValues(childValue, childKey, depth + 1, out);
+  return out;
+}
+
+function appendMemoryPathCandidate(out, value) {
+  if (typeof value !== "string") return;
+  const normalized = normalizeMemoryPath(value);
+  if (normalized) out.push(normalized);
+}
+
+function memoryPathInfo(value) {
+  const normalized = normalizeMemoryPath(value);
+  if (!normalized) return null;
+  const lower = normalized.toLowerCase();
+  const base = lower.split("/").filter(Boolean).pop() || "";
+  if (/\/\.claude\/projects\/[^/]+\/memory\/[^/]+\.(md|mdx|txt|json|yaml|yml)$/i.test(normalized)) {
+    return genericMemoryInfo(normalized, "claude-code-project-memory", "project");
+  }
+  if (/\/\.claude\/(?:memory|memories)\//i.test(normalized)) {
+    return genericMemoryInfo(normalized, "claude-code-memory", "global");
+  }
+  if (/\/\.codex\/(?:memory|memories)\//i.test(normalized)) {
+    return genericMemoryInfo(normalized, "codex-memory", "global");
+  }
+  if (/\/\.cursor\/rules\//i.test(normalized)) {
+    return genericMemoryInfo(normalized, "cursor-rules", "project");
+  }
+  if (/\/\.windsurf\/rules\//i.test(normalized)) {
+    return genericMemoryInfo(normalized, "windsurf-rules", "project");
+  }
+  if (MEMORY_PATH_BASES.has(base)) {
+    const source = base === "agents.md" ? "project-instructions"
+      : base === "claude.md" ? "claude-instructions"
+      : base === "gemini.md" ? "gemini-instructions"
+      : base === ".cursorrules" ? "cursor-rules"
+      : base === ".windsurfrules" ? "windsurf-rules"
+      : "memory-file";
+    return genericMemoryInfo(normalized, source, projectMemoryScope(lower));
+  }
+  return null;
+}
+
+function genericMemoryInfo(pathValue, source, scope) {
+  const normalized = normalizeMemoryPath(pathValue);
+  const name = normalized.split("/").filter(Boolean).pop() || normalized || "memory";
+  return {
+    path: normalized,
+    name,
+    source,
+    scope
+  };
+}
+
+function projectMemoryScope(lowerPath) {
+  if (/\/\.(claude|codex|cursor|windsurf)\//.test(lowerPath)) return "project";
+  return "project";
+}
+
+function normalizeMemoryPath(value) {
+  const text = String(value || "").trim();
+  if (!text || text.length > 4096) return "";
+  return text.replace(/\\/g, "/").replace(/\/+/g, "/");
+}
+
+function projectInstructionsPathFromContent(content) {
+  const match = /^# AGENTS\.md instructions for\s+(.+)$/m.exec(String(content || ""));
+  if (!match) return "";
+  const root = normalizeMemoryPath(match[1]);
+  if (!root) return "";
+  return root.toLowerCase().endsWith("/agents.md") ? root : `${root.replace(/\/+$/, "")}/AGENTS.md`;
+}
+
+function memoryEventTitle(memory) {
+  const action = String(memory?.action || "loaded").toLowerCase();
+  if (action === "read") return "Memory read";
+  if (action === "write") return "Memory write";
+  return "Memory loaded";
+}
+
+function memoryEventSummary(memory) {
+  const label = memoryEventTitle(memory);
+  const pathValue = firstString(memory?.path, memory?.name, "memory");
+  return `${label}: ${pathValue}`;
+}
+
+function isSuccessfulToolResult(toolResult) {
+  const status = String(toolResult?.status || "completed").toLowerCase();
+  return !status.includes("error") && !status.includes("fail");
+}
+
 function rememberToolCallEvent(state, toolCall, eventId) {
   if (!state || !eventId) return;
   for (const key of toolPairKeys(toolCall)) {
@@ -236,6 +524,7 @@ function rememberToolCallEvent(state, toolCall, eventId) {
     state.toolCallEventIdsByKey.get(key).push(eventId);
   }
   state.unmatchedToolCallEventIds.push(eventId);
+  state.unmatchedToolCallEventIdSet.add(eventId);
 }
 
 function consumeToolCallEventId(state, toolResult) {
@@ -247,14 +536,19 @@ function consumeToolCallEventId(state, toolResult) {
       if (consumeUnmatchedToolEventId(state, eventId)) return eventId;
     }
   }
-  return state.unmatchedToolCallEventIds.shift() || "";
+  // FIFO fallback: advance the head pointer past tombstoned (already-consumed)
+  // ids and return the oldest one still unmatched.
+  const ids = state.unmatchedToolCallEventIds;
+  while (state.unmatchedHead < ids.length) {
+    const eventId = ids[state.unmatchedHead++];
+    if (state.unmatchedToolCallEventIdSet.delete(eventId)) return eventId;
+  }
+  return "";
 }
 
 function consumeUnmatchedToolEventId(state, eventId) {
-  const index = state.unmatchedToolCallEventIds.indexOf(eventId);
-  if (index < 0) return false;
-  state.unmatchedToolCallEventIds.splice(index, 1);
-  return true;
+  // delete() returns true only if the id was present (still unmatched).
+  return state.unmatchedToolCallEventIdSet.delete(eventId);
 }
 
 function toolPairKeys(tool) {
@@ -275,7 +569,7 @@ function baseEvent(session, options) {
     title: String(options.indexed?.title || ""),
     summary: String(options.indexed?.summary || "")
   };
-  for (const key of ["toolName", "toolCategory", "toolIcon", "model", "status", "latencyMs", "target"]) {
+  for (const key of ["toolName", "toolCategory", "toolIcon", "model", "status", "latencyMs", "target", "skills", "memoryAction", "memoryPath", "memorySource", "memoryScope"]) {
     if (options.indexed?.[key] !== undefined && options.indexed?.[key] !== "") indexed[key] = options.indexed[key];
   }
   return {
@@ -301,7 +595,8 @@ function baseEvent(session, options) {
     body: {
       text: String(body.text || ""),
       toolCall: body.toolCall || null,
-      toolResult: body.toolResult || null
+      toolResult: body.toolResult || null,
+      memory: body.memory || null
     },
     parentEventId: options.parentEventId || ""
   };
@@ -336,6 +631,9 @@ function renderEventText(event) {
     const text = [event.indexed?.title, result.summary, result.output || event.body?.text].filter(Boolean).join("\n");
     return approxTokenCount(text) > 700 ? [event.indexed?.title, result.summary].filter(Boolean).join(" ") : text;
   }
+  if (kind === EVENT_KINDS.MEMORY_READ || kind === EVENT_KINDS.MEMORY_WRITE || kind === EVENT_KINDS.MEMORY_LOADED) {
+    return event.body?.text || event.indexed?.summary || "";
+  }
   return event.body?.text || event.indexed?.summary || "";
 }
 
@@ -415,7 +713,9 @@ function normalizeToolCall(raw, provider = "") {
   const rawInputSummary = firstString(raw.rawInputSummary, argument);
   const inputPreview = firstString(raw.inputPreview, rawInputSummary, summarizeToolInput(parsedArguments));
   const target = firstString(raw.target, targetFromArguments(parsedArguments));
-  return {
+  const structuredPatch = normalizeStructuredPatch(raw.structuredPatch || raw.structured_patch);
+  const argumentPatch = structuredPatch.length ? structuredPatch : structuredPatchFromToolArguments(parsedArguments);
+  const normalized = {
     provider: firstString(raw.provider, provider),
     id: firstString(raw.id, raw.callId, raw.call_id, raw.toolCallId, raw.tool_call_id, raw.toolUseId, raw.tool_use_id) || undefined,
     name,
@@ -432,6 +732,8 @@ function normalizeToolCall(raw, provider = "") {
     target,
     arguments: parsedArguments && typeof parsedArguments === "object" && !Array.isArray(parsedArguments) ? parsedArguments : undefined
   };
+  if (argumentPatch.length) normalized.structuredPatch = argumentPatch;
+  return normalized;
 }
 
 function summarizeToolInput(value) {
@@ -467,7 +769,8 @@ function normalizeToolResult(raw, provider = "") {
     const output = raw.trim();
     if (!output) return null;
     const classification = classifyTool("tool_output");
-    return {
+    const structuredPatch = structuredPatchFromDiffText(output);
+    const normalized = {
       provider,
       id: undefined,
       kind: "Tool output",
@@ -481,6 +784,8 @@ function normalizeToolResult(raw, provider = "") {
       collapsed: output.split("\n").length > 18,
       status: "completed"
     };
+    if (structuredPatch.length) normalized.structuredPatch = structuredPatch;
+    return normalized;
   }
   if (typeof raw !== "object") return null;
   const output = firstString(raw.output, raw.text, raw.content, raw.result);
@@ -488,7 +793,9 @@ function normalizeToolResult(raw, provider = "") {
   const kind = firstString(raw.kind, raw.type, raw.toolName, "Tool output");
   const rawCategory = firstString(raw.rawCategory, raw.category, raw.kind, raw.type);
   const classification = classifyTool(firstString(raw.toolName, raw.name, kind), rawCategory);
-  return {
+  const structuredPatch = normalizeStructuredPatch(raw.structuredPatch || raw.structured_patch);
+  const outputPatch = structuredPatch.length ? structuredPatch : structuredPatchFromDiffText(output);
+  const normalized = {
     provider: firstString(raw.provider, provider),
     id: firstString(raw.id, raw.callId, raw.call_id, raw.toolCallId, raw.tool_call_id, raw.toolUseId, raw.tool_use_id) || undefined,
     name: firstString(raw.name, raw.toolName, raw.tool_name) || undefined,
@@ -504,6 +811,8 @@ function normalizeToolResult(raw, provider = "") {
     collapsed: Boolean(raw.collapsed),
     status: firstString(raw.status, "completed")
   };
+  if (outputPatch.length) normalized.structuredPatch = outputPatch;
+  return normalized;
 }
 
 function toolArgumentClauses(toolCall) {
@@ -614,11 +923,13 @@ module.exports = {
   CANONICAL_EVENT_SCHEMA_VERSION,
   EVENT_KINDS,
   isLowSignalToolCall,
+  memoryPathInfo,
   messageToCanonicalEvents,
   normalizeSessionEvents,
   normalizeToolCall,
   normalizeToolResult,
   renderEventText,
   renderToolCallText,
+  skillInvocationsFromPromptText,
   stableEventId
 };