agentel 0.2.6 → 0.2.8

package/src/archive.js

@@ -452,6 +452,8 @@ function hasStructuredMetadata(metadata) {
     (Array.isArray(metadata?.toolCalls) && metadata.toolCalls.length) ||
       metadata?.toolResult ||
       metadata?.eventType ||
+      (Array.isArray(metadata?.attachments) && metadata.attachments.length) ||
+      (Array.isArray(metadata?.assetPointers) && metadata.assetPointers.length) ||
       metadata?.parserVersion
   );
 }
@@ -541,15 +543,76 @@ function hydrateSessionMetadata(file, metadata) {
   return { ...metadata, conversationPath, metadataPath: file, transcriptPath, eventPath, viewPath, rawPath };
 }
 
+// TTL cache for the archive walk. The web viewer typically fires several
+// list-derived endpoints (/api/recent, /api/tree, /api/stats) within a single
+// page load; caching the snapshot for a short window lets them share one
+// walk without anyone seeing >1 second of staleness. Tests/imports/CLI paths
+// can invalidate explicitly via `invalidateSessionsSnapshotCache()`.
+const SNAPSHOT_CACHE_TTL_MS = 1000;
+const _snapshotCache = {
+  rootKey: "",
+  snapshot: null,
+  takenAtMs: 0
+};
+
 function listSessions(env = process.env) {
+  return listSessionsSnapshot(env).sessions;
+}
+
+function cachedListSessionsSnapshot(env = process.env) {
+  const rootKey = path.join(archiveRoot(env), "sessions");
+  const now = Date.now();
+  if (
+    _snapshotCache.snapshot &&
+    _snapshotCache.rootKey === rootKey &&
+    now - _snapshotCache.takenAtMs < SNAPSHOT_CACHE_TTL_MS
+  ) {
+    return _snapshotCache.snapshot;
+  }
+  const snapshot = listSessionsSnapshot(env);
+  _snapshotCache.rootKey = rootKey;
+  _snapshotCache.snapshot = snapshot;
+  _snapshotCache.takenAtMs = now;
+  return snapshot;
+}
+
+function invalidateSessionsSnapshotCache() {
+  _snapshotCache.rootKey = "";
+  _snapshotCache.snapshot = null;
+  _snapshotCache.takenAtMs = 0;
+}
+
+/**
+ * Walk the archive once and return both the hydrated session list and a
+ * fingerprint summarizing the snapshot. Callers that need an HTTP etag (recent,
+ * tree, repo-sessions, stats) can use the fingerprint without re-walking; the
+ * fingerprint changes iff some session's metadata mtime/size changed or the
+ * count changed (added/removed sessions).
+ *
+ * The fingerprint deliberately summarizes via XOR over per-file mtime/size
+ * values rather than concatenating sorted file paths — XOR is order-independent
+ * and folds the whole archive into a single 16-byte hex string in O(N) time
+ * with no allocation.
+ */
+function listSessionsSnapshot(env = process.env) {
   const root = path.join(archiveRoot(env), "sessions");
   const seen = new Set();
   const sessions = [];
+  let count = 0;
+  // 64-bit XOR accumulators kept as JS BigInt so we never overflow. Slightly
+  // slower than Number arithmetic but the cost is dwarfed by fs.statSync.
+  let mtimeXor = 0n;
+  let sizeXor = 0n;
+  let maxMtimeMs = 0;
   walk(root, (file) => {
     if (!file.endsWith(".metadata.json")) return;
     seen.add(file);
     let stat;
     try { stat = fs.statSync(file); } catch { return; }
+    count += 1;
+    mtimeXor ^= BigInt(Math.trunc(stat.mtimeMs));
+    sizeXor ^= BigInt(stat.size);
+    if (stat.mtimeMs > maxMtimeMs) maxMtimeMs = stat.mtimeMs;
     const cached = _sessionsIndex.byPath.get(file);
     if (cached && cached.mtimeMs === stat.mtimeMs && cached.size === stat.size) {
       sessions.push(cached.session);
@@ -569,7 +632,9 @@ function listSessions(env = process.env) {
       _sessionsIndex.byPath.delete(key);
     }
   }
-  return sessions.sort((a, b) => String(b.startedAt).localeCompare(String(a.startedAt)));
+  sessions.sort((a, b) => String(b.startedAt).localeCompare(String(a.startedAt)));
+  const fingerprint = `${count.toString(36)}-${mtimeXor.toString(16)}-${sizeXor.toString(16)}`;
+  return { sessions, fingerprint, count, maxMtimeMs };
 }
 
 function findSessionById(sessionId, env = process.env) {
@@ -603,7 +668,9 @@ function computeSessionUsage(messages) {
   let inputTokens = 0;
   let outputTokens = 0;
   let cacheInputTokens = 0;
+  let countedCacheInputTokens = 0;
   let reasoningOutputTokens = 0;
+  let countedReasoningOutputTokens = 0;
   let toolUsePromptTokens = 0;
   let maxUndifferentiatedTotalTokens = 0;
   let totalInputTokens = 0;
@@ -638,7 +705,9 @@ function computeSessionUsage(messages) {
       any = true;
     }
     if (summary.cacheInputTokens > 0) cacheInputTokens += summary.cacheInputTokens;
+    if (summary.countedCacheInputTokens > 0) countedCacheInputTokens += summary.countedCacheInputTokens;
     if (summary.reasoningOutputTokens > 0) reasoningOutputTokens += summary.reasoningOutputTokens;
+    if (summary.countedReasoningOutputTokens > 0) countedReasoningOutputTokens += summary.countedReasoningOutputTokens;
     if (summary.toolUsePromptTokens > 0) toolUsePromptTokens += summary.toolUsePromptTokens;
     if (!summary.hasSplitTokens && !summary.hasExtraTokens && summary.hasTotalTokens) {
       maxUndifferentiatedTotalTokens = Math.max(maxUndifferentiatedTotalTokens, summary.totalTokens);
@@ -648,17 +717,17 @@ function computeSessionUsage(messages) {
   }
   if (Number.isFinite(totalInputTokens) && totalInputTokens > 0) any = true;
   if (Number.isFinite(totalOutputTokens) && totalOutputTokens > 0) any = true;
-  if (reasoningOutputTokens > 0 || toolUsePromptTokens > 0) any = true;
+  if (cacheInputTokens > 0 || reasoningOutputTokens > 0 || toolUsePromptTokens > 0) any = true;
   if (!any && maxUndifferentiatedTotalTokens > 0) {
     inputTokens = maxUndifferentiatedTotalTokens;
     any = true;
   }
   const cumulativeTokens = totalInputTokens + totalOutputTokens;
-  const splitTotalTokens = inputTokens + outputTokens + reasoningOutputTokens + toolUsePromptTokens;
+  const splitTotalTokens = inputTokens + outputTokens + countedCacheInputTokens + countedReasoningOutputTokens + toolUsePromptTokens;
   const finalTotalTokens = splitTotalTokens || cumulativeTokens || maxUndifferentiatedTotalTokens;
   if (!any && finalTotalTokens === 0) return null;
   const result = {
-    inputTokens: inputTokens || totalInputTokens || (!outputTokens && !reasoningOutputTokens && !toolUsePromptTokens && maxUndifferentiatedTotalTokens ? maxUndifferentiatedTotalTokens : 0),
+    inputTokens: inputTokens || totalInputTokens || (!outputTokens && !cacheInputTokens && !reasoningOutputTokens && !toolUsePromptTokens && maxUndifferentiatedTotalTokens ? maxUndifferentiatedTotalTokens : 0),
     outputTokens: outputTokens || totalOutputTokens,
     cacheInputTokens,
     reasoningOutputTokens,
@@ -667,6 +736,8 @@ function computeSessionUsage(messages) {
     totalInputTokens: totalInputTokens || inputTokens,
     totalOutputTokens: totalOutputTokens || outputTokens
   };
+  if (reasoningOutputTokens > 0 && countedReasoningOutputTokens === 0) result.reasoningOutputTokensIncludedInOutput = true;
+  if (cacheInputTokens > 0 && countedCacheInputTokens === 0) result.cacheInputTokensIncludedInInput = true;
   if (estimatedUsageCount > 0 && actualUsageCount === 0) {
     result.estimated = true;
     if (estimationMethods.size === 1) result.estimationMethod = Array.from(estimationMethods)[0];
@@ -804,6 +875,7 @@ function messageTokenEstimate(message) {
 function usageTokenSummary(usage) {
   if (!usage || typeof usage !== "object") return emptyUsageTokenSummary();
   const estimated = usage.estimated === true || usage.estimated === "true";
+  const authoritativeTotalTokens = usage.authoritativeTotalTokens === true || usage.authoritative_total_tokens === true;
   const estimationMethod = typeof usage.estimationMethod === "string" && usage.estimationMethod.trim()
     ? usage.estimationMethod.trim()
     : typeof usage.estimation_method === "string" && usage.estimation_method.trim()
@@ -868,28 +940,46 @@ function usageTokenSummary(usage) {
     firstUsageNumber(usage.cachedContentTokenCount, usage.cached_content_token_count),
     firstUsageNumber(usage.cachedTokens, usage.cached_tokens, usage.cacheTokens, usage.cache_tokens, usage.cached)
   );
+  const cacheInputTokensIncludedInInput =
+    usage.cacheInputTokensIncludedInInput === true ||
+    usage.cache_input_tokens_included_in_input === true ||
+    usage.cacheReadTokensIncludedInInput === true ||
+    usage.cache_read_tokens_included_in_input === true;
+  const countedCacheInputTokens = cacheInputTokensIncludedInInput ? 0 : cacheInputTokens;
   const reasoningOutputTokens = sumPositiveTokenNumbers(
     firstUsageNumber(usage.reasoningOutputTokens, usage.reasoning_output_tokens),
     firstUsageNumber(usage.thoughtsTokens, usage.thoughts_tokens, usage.thoughtsTokenCount, usage.thoughts_token_count),
     firstUsageNumber(usage.reasoningTokens, usage.reasoning_tokens, usage.reasoningTokenCount, usage.reasoning_token_count)
   );
+  const reasoningOutputTokensIncludedInOutput =
+    usage.reasoningOutputTokensIncludedInOutput === true ||
+    usage.reasoning_output_tokens_included_in_output === true ||
+    usage.reasoningTokensIncludedInOutput === true ||
+    usage.reasoning_tokens_included_in_output === true;
+  const countedReasoningOutputTokens = reasoningOutputTokensIncludedInOutput ? 0 : reasoningOutputTokens;
   const toolUsePromptTokens = sumPositiveTokenNumbers(
     firstUsageNumber(usage.toolUsePromptTokens, usage.tool_use_prompt_tokens, usage.toolUsePromptTokenCount, usage.tool_use_prompt_token_count)
   );
-  const extraTokens = cacheInputTokens + reasoningOutputTokens + toolUsePromptTokens;
+  const extraTokens = countedCacheInputTokens + countedReasoningOutputTokens + toolUsePromptTokens;
   const splitTokens = inputTokens + outputTokens;
   const cumulativeTokens = totalInputTokens + totalOutputTokens;
   const categoryTokens = splitTokens + extraTokens;
-  const totalTokens = explicitTotalTokens || categoryTokens
-    ? Math.max(explicitTotalTokens, categoryTokens)
-    : cumulativeTokens;
+  const totalTokens = authoritativeTotalTokens && explicitTotalTokens
+    ? explicitTotalTokens
+    : explicitTotalTokens || categoryTokens
+      ? Math.max(explicitTotalTokens, categoryTokens)
+      : cumulativeTokens;
   return {
     inputTokens,
     outputTokens,
     totalInputTokens,
     totalOutputTokens,
     cacheInputTokens,
+    countedCacheInputTokens,
+    cacheInputTokensIncludedInInput,
     reasoningOutputTokens,
+    countedReasoningOutputTokens,
+    reasoningOutputTokensIncludedInOutput,
     toolUsePromptTokens,
     extraTokens,
     totalTokens,
@@ -898,6 +988,7 @@ function usageTokenSummary(usage) {
     hasExtraTokens: extraTokens > 0,
     hasTotalTokens: explicitTotalTokens > 0,
     hasCumulativeTokens: cumulativeTokens > 0,
+    authoritativeTotalTokens,
     estimated,
     estimationMethod
   };
@@ -910,7 +1001,11 @@ function emptyUsageTokenSummary() {
     totalInputTokens: 0,
     totalOutputTokens: 0,
     cacheInputTokens: 0,
+    countedCacheInputTokens: 0,
+    cacheInputTokensIncludedInInput: false,
     reasoningOutputTokens: 0,
+    countedReasoningOutputTokens: 0,
+    reasoningOutputTokensIncludedInOutput: false,
     toolUsePromptTokens: 0,
     extraTokens: 0,
     totalTokens: 0,
@@ -919,6 +1014,7 @@ function emptyUsageTokenSummary() {
     hasExtraTokens: false,
     hasTotalTokens: false,
     hasCumulativeTokens: false,
+    authoritativeTotalTokens: false,
     estimated: false,
     estimationMethod: ""
   };
@@ -951,9 +1047,15 @@ function sessionSummaryUsage(sessionSummary) {
     firstUsageNumber(usage.cacheInputTokens, usage.cache_input_tokens),
     firstUsageNumber(usage.cacheReadTokens, usage.cache_read_tokens)
   );
-  const totalTokens = positiveTokenNumber(firstUsageNumber(usage.totalTokens, usage.total_tokens, usage.totalTokenCount, usage.total_token_count)) || inputTokens + outputTokens;
+  const cacheInputTokensIncludedInInput =
+    usage.cacheInputTokensIncludedInInput === true ||
+    usage.cache_input_tokens_included_in_input === true ||
+    usage.cacheReadTokensIncludedInInput === true ||
+    usage.cache_read_tokens_included_in_input === true;
+  const countedCacheInputTokens = cacheInputTokensIncludedInInput ? 0 : cacheInputTokens;
+  const totalTokens = positiveTokenNumber(firstUsageNumber(usage.totalTokens, usage.total_tokens, usage.totalTokenCount, usage.total_token_count)) || inputTokens + outputTokens + countedCacheInputTokens;
   if (!inputTokens && !outputTokens && !cacheInputTokens && !totalTokens) return null;
-  return {
+  const result = {
     inputTokens,
     outputTokens,
     cacheInputTokens,
@@ -963,22 +1065,39 @@ function sessionSummaryUsage(sessionSummary) {
     totalInputTokens: inputTokens,
     totalOutputTokens: outputTokens
   };
+  if (usage.authoritativeTotalTokens === true || usage.authoritative_total_tokens === true) result.authoritativeTotalTokens = true;
+  if (cacheInputTokensIncludedInInput) result.cacheInputTokensIncludedInInput = true;
+  return result;
 }
 
 function mergeSessionUsage(primary, secondary) {
   if (!primary && !secondary) return null;
   if (!primary) return secondary;
   if (!secondary) return primary;
+  const totalTokens = secondary.authoritativeTotalTokens
+    ? (secondary.totalTokens || 0)
+    : primary.authoritativeTotalTokens
+      ? (primary.totalTokens || 0)
+      : Math.max(primary.totalTokens || 0, secondary.totalTokens || 0);
   const result = {
     inputTokens: primary.inputTokens || secondary.inputTokens || 0,
     outputTokens: primary.outputTokens || secondary.outputTokens || 0,
     cacheInputTokens: primary.cacheInputTokens || secondary.cacheInputTokens || 0,
     reasoningOutputTokens: primary.reasoningOutputTokens || secondary.reasoningOutputTokens || 0,
     toolUsePromptTokens: primary.toolUsePromptTokens || secondary.toolUsePromptTokens || 0,
-    totalTokens: Math.max(primary.totalTokens || 0, secondary.totalTokens || 0),
+    totalTokens,
     totalInputTokens: primary.totalInputTokens || secondary.totalInputTokens || primary.inputTokens || secondary.inputTokens || 0,
     totalOutputTokens: primary.totalOutputTokens || secondary.totalOutputTokens || primary.outputTokens || secondary.outputTokens || 0
   };
+  if (primary.authoritativeTotalTokens || secondary.authoritativeTotalTokens) {
+    result.authoritativeTotalTokens = true;
+  }
+  if (primary.cacheInputTokensIncludedInInput || secondary.cacheInputTokensIncludedInInput) {
+    result.cacheInputTokensIncludedInInput = true;
+  }
+  if (primary.reasoningOutputTokensIncludedInOutput || secondary.reasoningOutputTokensIncludedInOutput) {
+    result.reasoningOutputTokensIncludedInOutput = true;
+  }
   if (primary.estimated && secondary.estimated) {
     result.estimated = true;
     if (primary.estimationMethod && primary.estimationMethod === secondary.estimationMethod) {
@@ -1047,7 +1166,7 @@ function readEvents(sessionOrPath) {
 function ensureConversationMarkdown(session, env = process.env) {
   const conversationPath = session.conversationPath || session.metadataPath?.replace(/\.metadata\.json$/, ".conversation.md");
   if (!conversationPath) return "";
-  if (fs.existsSync(conversationPath)) return conversationPath;
+  if (fs.existsSync(conversationPath) && !conversationMarkdownNeedsRefresh(conversationPath, session)) return conversationPath;
   const messages = readTranscript(session.transcriptPath);
   if (!messages.length) return "";
   const events = readEvents(session);
@@ -1060,7 +1179,18 @@ function ensureConversationMarkdown(session, env = process.env) {
   return conversationPath;
 }
 
-const VIEW_SCHEMA_VERSION = 3;
+function conversationMarkdownNeedsRefresh(file, session) {
+  if (session?.provider !== "chatgpt" && session?.sourceType !== "chatgpt-export") return false;
+  let text = "";
+  try {
+    text = fs.readFileSync(file, "utf8");
+  } catch {
+    return false;
+  }
+  return /\uE200[A-Za-z_]*cite\uE202[^\uE201]+\uE201/.test(text);
+}
+
+const VIEW_SCHEMA_VERSION = 4;
 
 function sessionViewPathFromMetadata(metadataPath) {
   if (!metadataPath) return "";
@@ -1234,24 +1364,73 @@ function renderConversationMarkdown(session, messages, events = []) {
   for (const message of displayMessages) {
     body.push(`## ${roleLabel(message.role)} - ${message.timestamp || "unknown time"}`);
     body.push("");
-    body.push(styleRedactionMarkersForMarkdown(String(message.content || "").trim()));
-    body.push("");
+    const content = styleRedactionMarkersForMarkdown(styleChatGptCitationMarkersForMarkdown(String(message.content || "").trim()));
+    if (content) {
+      body.push(content);
+      body.push("");
+    }
+    const attachmentMarkdown = renderMessageAttachmentsMarkdown(message);
+    if (attachmentMarkdown) {
+      body.push(attachmentMarkdown);
+      body.push("");
+    }
     for (const event of events.filter((item) => item.messageIndex === message.index && item.kind === "tool.called")) {
       body.push(`### Tool Call - ${event.occurredAt || message.timestamp || "unknown time"}`);
       body.push("");
-      body.push(styleRedactionMarkersForMarkdown(String(event.body?.text || event.indexed?.summary || "").trim()));
+      body.push(styleRedactionMarkersForMarkdown(styleChatGptCitationMarkersForMarkdown(String(event.body?.text || event.indexed?.summary || "").trim())));
       body.push("");
     }
     for (const event of events.filter((item) => item.messageIndex === message.index && item.kind === "tool.completed")) {
       body.push(`### Tool Result - ${event.occurredAt || message.timestamp || "unknown time"}`);
       body.push("");
-      body.push(styleRedactionMarkersForMarkdown(String(event.indexed?.summary || event.body?.text || "").trim()));
+      body.push(styleRedactionMarkersForMarkdown(styleChatGptCitationMarkersForMarkdown(String(event.indexed?.summary || event.body?.text || "").trim())));
       body.push("");
     }
   }
   return `${frontmatter.concat(body).join("\n").trimEnd()}\n`;
 }
 
+function styleChatGptCitationMarkersForMarkdown(value) {
+  return String(value || "").replace(/\uE200([A-Za-z_]*cite)\uE202([^\uE201]+)\uE201/g, (_, kind, ref) => {
+    const label = /file/i.test(String(kind || "")) ? "file citation" : "citation";
+    const text = chatGptCitationParts(ref).join(" ");
+    return text ? `[${label}: ${text}]` : `[${label}]`;
+  });
+}
+
+function chatGptCitationParts(ref) {
+  return String(ref || "").split("\uE202").map((part) => part.trim()).filter(Boolean);
+}
+
+function renderMessageAttachmentsMarkdown(message) {
+  const attachments = Array.isArray(message?.metadata?.attachments) ? message.metadata.attachments : [];
+  const assetPointers = Array.isArray(message?.metadata?.assetPointers) ? message.metadata.assetPointers : [];
+  if (!attachments.length && !assetPointers.length) return "";
+  const lines = ["### Attachments", ""];
+  for (const attachment of attachments) {
+    const name = markdownLine(attachment?.name || attachment?.id || "attachment");
+    const details = [
+      attachment?.mimeType,
+      attachment?.width && attachment?.height ? `${attachment.width}x${attachment.height}` : "",
+      attachment?.size ? `${attachment.size} bytes` : "",
+      attachment?.assetPointer ? `asset ${attachment.assetPointer}` : ""
+    ].filter(Boolean).join(", ");
+    lines.push(`- ${name}${details ? ` (${markdownLine(details)})` : ""}`);
+  }
+  const attachedPointers = new Set(attachments.map((attachment) => attachment?.assetPointer).filter(Boolean));
+  for (const pointer of assetPointers) {
+    if (!pointer?.assetPointer || attachedPointers.has(pointer.assetPointer)) continue;
+    const details = [
+      pointer.contentType,
+      pointer.mimeType,
+      pointer.width && pointer.height ? `${pointer.width}x${pointer.height}` : "",
+      pointer.size ? `${pointer.size} bytes` : ""
+    ].filter(Boolean).join(", ");
+    lines.push(`- ${markdownLine(pointer.assetPointer)}${details ? ` (${markdownLine(details)})` : ""}`);
+  }
+  return lines.join("\n");
+}
+
 function normalizePrecomputedEvents(events, session) {
   return events
     .filter((event) => event && typeof event === "object")
@@ -1370,7 +1549,11 @@ module.exports = {
   ensureConversationMarkdown,
   ensureSessionView,
   findSessionById,
+  invalidateSessionsSnapshotCache,
+  isWebChatProvider,
   listSessions,
+  listSessionsSnapshot,
+  cachedListSessionsSnapshot,
   normalizeMessages,
   objectPathForSession,
   readEvents,