agentel 0.2.0 → 0.2.3

package/src/archive.js

@@ -11,6 +11,16 @@ const { loadRedactionConfig, loadEnvValues, mergeSummaries, redactText, styleRed
 const { ensureBaseDirs, ensureDir, paths, readJson, writeJson } = require("./paths");
 
 const SHARED_RAW_SOURCE_CACHE = new Map();
+const ESTIMATED_TOKEN_CHARS = 4;
+const MESSAGE_TOKEN_ESTIMATE_METHOD = "chars-per-token-v1";
+const CHAT_ESTIMATED_USAGE_METHOD = "visible-message-chars-v1";
+const CURSOR_ESTIMATED_MAX_INPUT_PER_TURN = 200000;
+const CURSOR_ESTIMATED_TOKEN_RATES = {
+  opus: { inputPerAssistantTurn: 11000, outputPerAssistantTurn: 240 },
+  sonnet: { inputPerAssistantTurn: 10000, outputPerAssistantTurn: 120 },
+  unknown: { inputPerAssistantTurn: 6900, outputPerAssistantTurn: 270 },
+  default: { inputPerAssistantTurn: 8400, outputPerAssistantTurn: 230 }
+};
 
 function archiveRoot(env = process.env) {
   const cfg = loadConfig(env);
@@ -59,7 +69,7 @@ function writeSession(input, env = process.env) {
   const sessionId = input.sessionId || stableSessionId(provider, input.sourcePath || "", startedAt, messages);
   const repoCanonical = input.scopeCanonical ? "" : input.repoCanonical || repoInfo.key;
   const redactionConfig = loadRedactionConfig(env);
-  const envValues = loadEnvValues(redactionCwd, redactionConfig.envVars, env);
+  const envValues = redactionConfig.enabled ? loadEnvValues(redactionCwd, redactionConfig.envVars, env) : [];
   const redactionSummary = {};
   const redactedMessages = messages.map((message) => {
     const result = redactText(message.content, {
@@ -83,6 +93,17 @@ function writeSession(input, env = process.env) {
       })
     };
   });
+  const archivedMessages = redactedMessages.map(withMessageTokenEstimate);
+  const sessionSummary = input.sessionSummary && typeof input.sessionSummary === "object"
+    ? redactStructuredStrings(input.sessionSummary, {
+        config: redactionConfig,
+        cwd: redactionCwd,
+        env,
+        envValues,
+        repoCanonical,
+        summary: redactionSummary
+      })
+    : null;
 
   const session = {
     version: 1,
@@ -115,10 +136,17 @@ function writeSession(input, env = process.env) {
     "chatVirtualRepo",
     "chatDisplayPath",
     "conversationKind",
-    "pinned"
+    "pinned",
+    "timeStatus",
+    "composerId",
+    "parentComposerId"
   ]) {
     if (input[key] !== undefined) session[key] = input[key];
   }
+  if (sessionSummary && Object.keys(sessionSummary).length) session.sessionSummary = sessionSummary;
+  if (session.composerId === undefined && typeof input.id === "string" && /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(input.id)) {
+    session.composerId = input.id;
+  }
 
   const dir = objectPathForSession(session, env);
   ensureDir(dir);
@@ -129,7 +157,7 @@ function writeSession(input, env = process.env) {
   const viewPath = path.join(dir, `session=${sessionId}.view.json`);
   const rawPath = path.join(dir, `session=${sessionId}.raw`);
   const events = redactCanonicalEvents(
-    Array.isArray(input.events) ? normalizePrecomputedEvents(input.events, session) : normalizeSessionEvents(session, redactedMessages),
+    Array.isArray(input.events) ? normalizePrecomputedEvents(input.events, session) : normalizeSessionEvents(session, archivedMessages),
     {
       config: redactionConfig,
       cwd: redactionCwd,
@@ -147,21 +175,25 @@ function writeSession(input, env = process.env) {
     session.rawFiles = rawFiles;
     session.rawFileCount = rawFiles.length;
   }
-  fs.writeFileSync(conversationPath, renderConversationMarkdown(session, redactedMessages, events), { mode: 0o600 });
-  fs.writeFileSync(transcriptPath, redactedMessages.map((message) => `${JSON.stringify(message)}\n`).join(""), {
+  fs.writeFileSync(conversationPath, renderConversationMarkdown(session, archivedMessages, events), { mode: 0o600 });
+  fs.writeFileSync(transcriptPath, archivedMessages.map((message) => `${JSON.stringify(message)}\n`).join(""), {
     mode: 0o600
   });
   fs.writeFileSync(eventPath, events.map((event) => `${JSON.stringify(event)}\n`).join(""), { mode: 0o600 });
-  writeSessionView(viewPath, computeSessionView(session, redactedMessages, events));
+  writeSessionView(viewPath, computeSessionView(session, archivedMessages, events));
   session.conversationPath = conversationPath;
   session.transcriptPath = transcriptPath;
   session.eventPath = eventPath;
   session.viewPath = viewPath;
   session.viewSchemaVersion = VIEW_SCHEMA_VERSION;
-  const usageStats = computeSessionUsage(redactedMessages);
+  const usageStats = mergeSessionUsage(computeSessionUsage(archivedMessages), sessionSummaryUsage(session.sessionSummary));
   if (usageStats) session.usage = usageStats;
-  const models = collectSessionModels(redactedMessages);
+  const models = collectSessionModels(archivedMessages, session.sessionSummary);
+  const estimatedUsage = usageStats ? null : estimateSessionUsage(archivedMessages, { provider, sourceType, models });
+  if (estimatedUsage) session.estimatedUsage = estimatedUsage;
   if (models.length) session.models = models;
+  const commandTypeCounts = collectCursorCommandTypeCounts(archivedMessages);
+  if (commandTypeCounts) session.cursorCommandTypeCounts = commandTypeCounts;
   writeJson(metadataPath, session);
 
   if (cfg.privacy.revealCache && !input.scopeCanonical) {
@@ -578,6 +610,9 @@ function computeSessionUsage(messages) {
   let totalOutputTokens = 0;
   const seenUsageRequests = new Set();
   let any = false;
+  let estimatedUsageCount = 0;
+  let actualUsageCount = 0;
+  const estimationMethods = new Set();
   for (const message of messages || []) {
     const usage = message && message.metadata && message.metadata.usage;
     if (!usage || typeof usage !== "object") continue;
@@ -588,6 +623,12 @@ function computeSessionUsage(messages) {
     }
     const summary = usageTokenSummary(usage);
     if (!summary.hasAny) continue;
+    if (summary.estimated) {
+      estimatedUsageCount++;
+      if (summary.estimationMethod) estimationMethods.add(summary.estimationMethod);
+    } else {
+      actualUsageCount++;
+    }
     if (summary.inputTokens > 0) {
       inputTokens += summary.inputTokens;
       any = true;
@@ -607,15 +648,17 @@ function computeSessionUsage(messages) {
   }
   if (Number.isFinite(totalInputTokens) && totalInputTokens > 0) any = true;
   if (Number.isFinite(totalOutputTokens) && totalOutputTokens > 0) any = true;
+  if (reasoningOutputTokens > 0 || toolUsePromptTokens > 0) any = true;
   if (!any && maxUndifferentiatedTotalTokens > 0) {
     inputTokens = maxUndifferentiatedTotalTokens;
     any = true;
   }
   const cumulativeTokens = totalInputTokens + totalOutputTokens;
-  const finalTotalTokens = inputTokens + outputTokens || cumulativeTokens || maxUndifferentiatedTotalTokens;
+  const splitTotalTokens = inputTokens + outputTokens + reasoningOutputTokens + toolUsePromptTokens;
+  const finalTotalTokens = splitTotalTokens || cumulativeTokens || maxUndifferentiatedTotalTokens;
   if (!any && finalTotalTokens === 0) return null;
-  return {
-    inputTokens: inputTokens || totalInputTokens || (!outputTokens && finalTotalTokens ? finalTotalTokens : 0),
+  const result = {
+    inputTokens: inputTokens || totalInputTokens || (!outputTokens && !reasoningOutputTokens && !toolUsePromptTokens && maxUndifferentiatedTotalTokens ? maxUndifferentiatedTotalTokens : 0),
     outputTokens: outputTokens || totalOutputTokens,
     cacheInputTokens,
     reasoningOutputTokens,
@@ -624,10 +667,148 @@ function computeSessionUsage(messages) {
     totalInputTokens: totalInputTokens || inputTokens,
     totalOutputTokens: totalOutputTokens || outputTokens
   };
+  if (estimatedUsageCount > 0 && actualUsageCount === 0) {
+    result.estimated = true;
+    if (estimationMethods.size === 1) result.estimationMethod = Array.from(estimationMethods)[0];
+  }
+  return result;
+}
+
+function estimateSessionUsage(messages, options = {}) {
+  const provider = String(options.provider || "").toLowerCase();
+  if (provider === "cursor") return estimateCursorSessionUsage(messages, options);
+  if (isWebChatProvider(provider, options.sourceType)) return estimateVisibleMessageUsage(messages);
+  return null;
+}
+
+function estimateCursorSessionUsage(messages, options = {}) {
+  let contextTokens = 0;
+  let visibleContextInputTokens = 0;
+  let visibleOutputTokens = 0;
+  let assistantTurns = 0;
+  for (const message of messages || []) {
+    const messageTokens = estimateMessageTokens(message);
+    if (message?.role === "assistant") {
+      if (messageTokens > 0 || contextTokens > 0) {
+        visibleContextInputTokens += Math.min(contextTokens, CURSOR_ESTIMATED_MAX_INPUT_PER_TURN);
+        visibleOutputTokens += messageTokens;
+        assistantTurns++;
+      }
+    }
+    contextTokens += messageTokens;
+  }
+  if (!assistantTurns) return null;
+  const family = cursorEstimatedModelFamily(options);
+  const rates = cursorEstimatedTokenRates(family, options);
+  const empiricalInputTokens = rates.inputPerAssistantTurn * assistantTurns;
+  const empiricalOutputTokens = rates.outputPerAssistantTurn * assistantTurns;
+  const inputTokens = Math.max(empiricalInputTokens, visibleContextInputTokens);
+  const outputTokens = Math.max(empiricalOutputTokens, visibleOutputTokens);
+  if (inputTokens + outputTokens <= 0) return null;
+  return {
+    inputTokens,
+    outputTokens,
+    totalTokens: inputTokens + outputTokens,
+    estimated: true,
+    estimationMethod: "cursor-empirical-turns-v1",
+    estimatedModelFamily: family,
+    estimatedAssistantTurns: assistantTurns
+  };
+}
+
+function estimateVisibleMessageUsage(messages) {
+  let inputTokens = 0;
+  let outputTokens = 0;
+  let messageCount = 0;
+  for (const message of messages || []) {
+    const tokens = messageTokenEstimate(message);
+    if (!tokens) continue;
+    if (message?.role === "assistant") outputTokens += tokens;
+    else inputTokens += tokens;
+    messageCount++;
+  }
+  if (inputTokens + outputTokens <= 0) return null;
+  return {
+    inputTokens,
+    outputTokens,
+    totalTokens: inputTokens + outputTokens,
+    estimated: true,
+    estimationMethod: CHAT_ESTIMATED_USAGE_METHOD,
+    estimatedMessageCount: messageCount
+  };
+}
+
+function isWebChatProvider(provider, sourceType = "") {
+  if (provider === "chatgpt" || provider === "claude_web") return true;
+  return /^(web-chat-export|chatgpt-export|claude-web-export|claude-web-memory)$/.test(String(sourceType || ""));
+}
+
+function cursorEstimatedModelFamily(options = {}) {
+  const models = Array.isArray(options.models) ? options.models.join(" ").toLowerCase() : "";
+  if (models.includes("opus")) return "opus";
+  if (models.includes("sonnet")) return "sonnet";
+  if (models.includes("gpt") || models.includes("gemini") || models.includes("claude")) return "default";
+  if (String(options.sourceType || "") === "cursor-agent-transcripts") return "default";
+  return "unknown";
+}
+
+function cursorEstimatedTokenRates(family) {
+  return CURSOR_ESTIMATED_TOKEN_RATES[family] || CURSOR_ESTIMATED_TOKEN_RATES.default;
+}
+
+function estimateMessageTokens(message) {
+  const parts = [];
+  if (message?.content) parts.push(String(message.content));
+  const toolCalls = Array.isArray(message?.metadata?.toolCalls) ? message.metadata.toolCalls : [];
+  for (const call of toolCalls) {
+    const summary = {
+      name: call?.name || call?.displayName,
+      target: call?.target,
+      argument: call?.argument || call?.rawInputSummary || call?.inputPreview,
+      arguments: call?.arguments
+    };
+    parts.push(JSON.stringify(summary));
+  }
+  const text = parts.filter(Boolean).join("\n");
+  if (!text.trim()) return 0;
+  return Math.max(1, Math.ceil(text.length / ESTIMATED_TOKEN_CHARS));
+}
+
+function withMessageTokenEstimate(message) {
+  const tokens = estimateMessageTokens(message);
+  if (!tokens) return message;
+  const existing = message?.metadata?.tokenEstimate;
+  if (existing && typeof existing === "object" && positiveTokenNumber(existing.tokens)) return message;
+  return {
+    ...message,
+    metadata: {
+      ...(message.metadata || {}),
+      tokenEstimate: {
+        tokens,
+        estimated: true,
+        method: MESSAGE_TOKEN_ESTIMATE_METHOD,
+        charsPerToken: ESTIMATED_TOKEN_CHARS
+      }
+    }
+  };
+}
+
+function messageTokenEstimate(message) {
+  const estimate = message?.metadata?.tokenEstimate || message?.metadata?.token_estimate;
+  if (estimate && typeof estimate === "object") {
+    return positiveTokenNumber(firstUsageNumber(estimate.tokens, estimate.tokenCount, estimate.token_count, estimate.totalTokens, estimate.total_tokens));
+  }
+  return estimateMessageTokens(message);
 }
 
 function usageTokenSummary(usage) {
   if (!usage || typeof usage !== "object") return emptyUsageTokenSummary();
+  const estimated = usage.estimated === true || usage.estimated === "true";
+  const estimationMethod = typeof usage.estimationMethod === "string" && usage.estimationMethod.trim()
+    ? usage.estimationMethod.trim()
+    : typeof usage.estimation_method === "string" && usage.estimation_method.trim()
+      ? usage.estimation_method.trim()
+      : "";
   const inputTokens = positiveTokenNumber(firstUsageNumber(
     usage.inputTokens,
     usage.input_tokens,
@@ -716,7 +897,9 @@ function usageTokenSummary(usage) {
     hasSplitTokens: splitTokens > 0,
     hasExtraTokens: extraTokens > 0,
     hasTotalTokens: explicitTotalTokens > 0,
-    hasCumulativeTokens: cumulativeTokens > 0
+    hasCumulativeTokens: cumulativeTokens > 0,
+    estimated,
+    estimationMethod
   };
 }
 
@@ -735,7 +918,9 @@ function emptyUsageTokenSummary() {
     hasSplitTokens: false,
     hasExtraTokens: false,
     hasTotalTokens: false,
-    hasCumulativeTokens: false
+    hasCumulativeTokens: false,
+    estimated: false,
+    estimationMethod: ""
   };
 }
 
@@ -757,15 +942,76 @@ function sumPositiveTokenNumbers(...values) {
   return values.reduce((sum, value) => sum + positiveTokenNumber(value), 0);
 }
 
-function collectSessionModels(messages) {
+function sessionSummaryUsage(sessionSummary) {
+  const usage = sessionSummary?.usage;
+  if (!usage || typeof usage !== "object") return null;
+  const inputTokens = positiveTokenNumber(firstUsageNumber(usage.inputTokens, usage.input_tokens, usage.promptTokens, usage.prompt_tokens));
+  const outputTokens = positiveTokenNumber(firstUsageNumber(usage.outputTokens, usage.output_tokens, usage.completionTokens, usage.completion_tokens));
+  const cacheInputTokens = sumPositiveTokenNumbers(
+    firstUsageNumber(usage.cacheInputTokens, usage.cache_input_tokens),
+    firstUsageNumber(usage.cacheReadTokens, usage.cache_read_tokens)
+  );
+  const totalTokens = positiveTokenNumber(firstUsageNumber(usage.totalTokens, usage.total_tokens, usage.totalTokenCount, usage.total_token_count)) || inputTokens + outputTokens;
+  if (!inputTokens && !outputTokens && !cacheInputTokens && !totalTokens) return null;
+  return {
+    inputTokens,
+    outputTokens,
+    cacheInputTokens,
+    reasoningOutputTokens: 0,
+    toolUsePromptTokens: 0,
+    totalTokens,
+    totalInputTokens: inputTokens,
+    totalOutputTokens: outputTokens
+  };
+}
+
+function mergeSessionUsage(primary, secondary) {
+  if (!primary && !secondary) return null;
+  if (!primary) return secondary;
+  if (!secondary) return primary;
+  const result = {
+    inputTokens: primary.inputTokens || secondary.inputTokens || 0,
+    outputTokens: primary.outputTokens || secondary.outputTokens || 0,
+    cacheInputTokens: primary.cacheInputTokens || secondary.cacheInputTokens || 0,
+    reasoningOutputTokens: primary.reasoningOutputTokens || secondary.reasoningOutputTokens || 0,
+    toolUsePromptTokens: primary.toolUsePromptTokens || secondary.toolUsePromptTokens || 0,
+    totalTokens: Math.max(primary.totalTokens || 0, secondary.totalTokens || 0),
+    totalInputTokens: primary.totalInputTokens || secondary.totalInputTokens || primary.inputTokens || secondary.inputTokens || 0,
+    totalOutputTokens: primary.totalOutputTokens || secondary.totalOutputTokens || primary.outputTokens || secondary.outputTokens || 0
+  };
+  if (primary.estimated && secondary.estimated) {
+    result.estimated = true;
+    if (primary.estimationMethod && primary.estimationMethod === secondary.estimationMethod) {
+      result.estimationMethod = primary.estimationMethod;
+    }
+  }
+  return result;
+}
+
+function collectSessionModels(messages, sessionSummary = null) {
   const models = new Set();
   for (const message of messages || []) {
     const model = message && message.metadata && message.metadata.model;
     if (typeof model === "string" && model.trim()) models.add(model.trim());
   }
+  for (const item of sessionSummary?.modelUsage || []) {
+    if (typeof item?.model === "string" && item.model.trim()) models.add(item.model.trim());
+  }
   return Array.from(models);
 }
 
+function collectCursorCommandTypeCounts(messages) {
+  const counts = {};
+  for (const message of messages || []) {
+    if (message?.role !== "user") continue;
+    const value = message?.metadata?.commandType;
+    if (typeof value !== "string" || !value.trim()) continue;
+    const key = value.trim().toLowerCase();
+    counts[key] = (counts[key] || 0) + 1;
+  }
+  return Object.keys(counts).length ? counts : null;
+}
+
 function readTranscript(file) {
   const messages = [];
   try {
@@ -814,7 +1060,7 @@ function ensureConversationMarkdown(session, env = process.env) {
   return conversationPath;
 }
 
-const VIEW_SCHEMA_VERSION = 1;
+const VIEW_SCHEMA_VERSION = 2;
 
 function sessionViewPathFromMetadata(metadataPath) {
   if (!metadataPath) return "";
@@ -881,6 +1127,13 @@ function cursorRawSyntheticTimeParts(session) {
 }
 
 function sessionHistoryTime(session) {
+  const explicitStatus = session?.timeStatus || session?.time_status || "";
+  if (explicitStatus === "recovered-time-unknown") {
+    return { startedAt: "", endedAt: "", status: "recovered-time-unknown" };
+  }
+  if (explicitStatus === "recovered-partial-time") {
+    return { startedAt: session?.startedAt || "", endedAt: "", status: "recovered-partial-time" };
+  }
   const rawTime = cursorRawSyntheticTimeParts(session);
   if (rawTime.startSynthetic && rawTime.endSynthetic) {
     return { startedAt: "", endedAt: "", status: "recovered-time-unknown" };
@@ -900,6 +1153,7 @@ function computeSessionView(session, transcriptMessages, canonicalEvents) {
     started_at: time.startedAt || "",
     ended_at: time.endedAt || "",
     time_status: time.status || "",
+    session_summary: session?.sessionSummary || undefined,
     transcript_messages: display,
     canonical_events: Array.isArray(canonicalEvents) ? canonicalEvents : []
   };
@@ -939,9 +1193,11 @@ function ensureSessionView(session, env = process.env) {
 
 function renderConversationMarkdown(session, messages, events = []) {
   const title = session.title || `${session.provider || "agent"} session ${session.sessionId}`;
+  const time = sessionHistoryTime(session);
+  const displayMessages = displayTranscriptMessages(messages, time);
   const hasRedactions =
-    messages.some((message) => /\[REDACTED:[^\]\n]+\]/.test(JSON.stringify(message))) ||
-    events.some((event) => /\[REDACTED:[^\]\n]+\]/.test(JSON.stringify(event)));
+    displayMessages.some((message) => /\[REDACTED(?::|\s+)[^\]\n]+\]/.test(JSON.stringify(message))) ||
+    events.some((event) => /\[REDACTED(?::|\s+)[^\]\n]+\]/.test(JSON.stringify(event)));
   const frontmatter = [
     "---",
     `session_id: ${yamlString(session.sessionId)}`,
@@ -949,14 +1205,15 @@ function renderConversationMarkdown(session, messages, events = []) {
     `repo: ${yamlString(session.repoCanonical || "")}`,
     `scope: ${yamlString(session.scopeCanonical || "")}`,
     `cwd: ${yamlString(session.cwd || "")}`,
-    `started_at: ${yamlString(session.startedAt || "")}`,
-    `ended_at: ${yamlString(session.endedAt || "")}`,
+    `started_at: ${yamlString(time.startedAt || "")}`,
+    `ended_at: ${yamlString(time.endedAt || "")}`,
     `source_type: ${yamlString(session.sourceType || "")}`,
     `storage_scope: ${yamlString(session.storageScope || "")}`,
     `chat_account_id: ${yamlString(session.chatAccountId || "")}`,
     `chat_display_name: ${yamlString(session.chatDisplayName || "")}`,
     `chat_project_path: ${yamlString(session.chatProjectPath || "")}`,
     `conversation_kind: ${yamlString(session.conversationKind || "")}`,
+    `time_status: ${yamlString(time.status || session.timeStatus || "")}`,
     `archive_schema_version: ${yamlString(session.archiveSchemaVersion || 1)}`,
     `event_count: ${yamlString(events.length || session.eventCount || 0)}`,
     `parser_versions: ${yamlString(JSON.stringify(session.parserVersions || {}))}`,
@@ -974,7 +1231,7 @@ function renderConversationMarkdown(session, messages, events = []) {
       ""
     );
   }
-  for (const message of messages) {
+  for (const message of displayMessages) {
     body.push(`## ${roleLabel(message.role)} - ${message.timestamp || "unknown time"}`);
     body.push("");
     body.push(styleRedactionMarkersForMarkdown(String(message.content || "").trim()));
@@ -1108,6 +1365,8 @@ module.exports = {
   deleteSessionArchive,
   displayTranscriptMessages,
   encodeSegment,
+  estimateMessageTokens,
+  estimateSessionUsage,
   ensureConversationMarkdown,
   ensureSessionView,
   findSessionById,