decant-core 1.12.0 → 1.14.0

package/LICENSE

@@ -1,3 +1,19 @@
+ADDITIONAL PERMISSION UNDER GNU AGPL VERSION 3 SECTION 7:
+FOSS LINKING EXCEPTION
+
+Permission is granted to link, import, or bundle decant-core into projects
+distributed under any OSI-approved open source license (including MPL-2.0, MIT,
+Apache-2.0, and BSD) and distribute the resulting work under that project's
+license, without requiring the enclosing project to be licensed under AGPLv3.
+Any modifications directly made to decant-core source files remain subject to AGPLv3.
+
+COMMERCIAL USE
+If you wish to use decant-core in closed-source, proprietary, or commercial
+software that cannot comply with the AGPLv3, a commercial license is available.
+Please contact office@covai.org for licensing terms.
+
+==============================================================================
+
                     GNU AFFERO GENERAL PUBLIC LICENSE
                        Version 3, 19 November 2007
 

package/README.md

@@ -151,9 +151,15 @@ normalized `parse()` contract. For the full extraction-strategy breakdown and ma
 
 ## License
 
-`decant-core` is licensed under the **GNU Affero General Public License v3.0 (AGPL-3.0-only)**.
+`decant-core` is licensed under the **GNU Affero General Public License v3.0 (AGPL-3.0-only)** with a **FOSS Linking Exception**, alongside a **Commercial License** option.
 
-That choice is deliberate. AI platforms change constantly, and parser fixes belong in a shared commons so the whole ecosystem benefits — not siloed in a proprietary fork. If you use `decant-core`, network-based deployments that serve modified versions must also offer the corresponding source. Please review [`LICENSE`](LICENSE) before incorporating it into your project.
+### FOSS Linking Exception (Open Source)
+
+Permission is granted to link, import, or bundle `decant-core` into projects distributed under any OSI-approved open source license (including **MPL-2.0**, **MIT**, **Apache-2.0**, and **BSD**) and distribute the resulting work under that project's license, without requiring the enclosing project to be licensed under AGPLv3. Any modifications directly made to `decant-core` source files remain subject to AGPLv3.
+
+### Commercial License
+
+If you wish to use `decant-core` in closed-source, proprietary, or commercial software that cannot comply with the AGPLv3, a commercial license is available. Please contact `office@covai.org` for licensing terms.
 
 ### Third-Party Test Fixtures Notice
 

package/ai/chatgpt.js

@@ -196,6 +196,221 @@ export function extractSharedConversationFromDom(
   return null;
 }
 
+function cleanApiPartText(partText) {
+  return partText
+    .replace(/\u{E0000}[\u{E0000}-\u{E007F}]*/gu, "")
+    .replace(/citeturn\d+\w*/g, "")
+    .trim();
+}
+
+// Linearize the newer backend-api conversation shape, which returns a
+// `messages` array instead of a `mapping` tree. Output segments match the
+// mapping-based linearize() format ({ type: "text" | "thought" | "image" })
+// so both flow through the same formatApiResult().
+export function linearizeMessagesArray(apiMessages, includeImages) {
+  const messages = [];
+  if (!Array.isArray(apiMessages)) return messages;
+
+  const pushOrMerge = (entry, isThoughtMsg) => {
+    if (
+      entry.role === "ChatGPT" &&
+      messages.length > 0 &&
+      messages[messages.length - 1].role === "ChatGPT"
+    ) {
+      const prevMsg = messages[messages.length - 1];
+      if (isThoughtMsg) {
+        prevMsg.segments.unshift(...entry.segments);
+      } else {
+        prevMsg.segments.push(...entry.segments);
+      }
+      Object.assign(prevMsg.citeMap, entry.citeMap);
+      Object.assign(prevMsg.imageGroupMap, entry.imageGroupMap);
+      if (entry.timestamp && !prevMsg.timestamp) {
+        prevMsg.timestamp = entry.timestamp;
+      }
+    } else {
+      messages.push(entry);
+    }
+  };
+
+  for (const msg of apiMessages) {
+    if (!msg) continue;
+    const role = msg?.author?.role;
+    if (role !== "user" && role !== "assistant" && role !== "tool") continue;
+    if (msg.metadata?.is_visually_hidden_from_conversation === true) continue;
+
+    const content = msg.content || {};
+    const contentType = content.content_type;
+    const segments = [];
+    // Mirror the mapping path's thought detection (content types plus
+    // author/recipient markers).
+    const isThoughtMsg =
+      msg?.author?.name === "thought" ||
+      msg?.recipient === "thought" ||
+      contentType === "thought" ||
+      contentType === "thoughts" ||
+      contentType === "reasoning_recap" ||
+      msg?.metadata?.reasoning_status === "is_reasoning";
+
+    // Reasoning summaries: thoughts = [{ summary, content }]
+    if (Array.isArray(content.thoughts) && content.thoughts.length > 0) {
+      const thoughtParts = content.thoughts
+        .map((t) =>
+          t.summary ? `**${t.summary}**\n${t.content || ""}` : t.content || "",
+        )
+        .map((t) => t.trim())
+        .filter(Boolean);
+      if (thoughtParts.length > 0) {
+        segments.push({ type: "thought", content: thoughtParts.join("\n\n") });
+      }
+    }
+
+    // Reasoning recap (e.g. "Worked for 11s")
+    if (
+      contentType === "reasoning_recap" &&
+      typeof content.content === "string" &&
+      content.content.trim()
+    ) {
+      segments.push({ type: "thought", content: content.content.trim() });
+    }
+
+    // Tool invocations (content_type "code", e.g. Deep Research args JSON)
+    // and tool-role messages are not user-visible prose — skip them.
+    if (contentType !== "code" && role !== "tool") {
+      const parts = Array.isArray(content.parts) ? content.parts : [];
+      for (const part of parts) {
+        let partText = "";
+        let isThoughtPart = isThoughtMsg;
+        if (typeof part === "string") {
+          partText = part;
+        } else if (part && typeof part === "object") {
+          if (part.content_type === "text" && typeof part.text === "string") {
+            partText = part.text;
+          } else if (
+            part.content_type === "thought" &&
+            typeof part.text === "string"
+          ) {
+            partText = part.text;
+            isThoughtPart = true;
+          } else if (
+            part.content_type === "audio_transcription" &&
+            typeof part.text === "string"
+          ) {
+            partText = part.text;
+          } else if (
+            includeImages &&
+            part?.content_type === "image_asset_pointer" &&
+            part?.asset_pointer
+          ) {
+            segments.push({
+              type: "image",
+              fileId: part.asset_pointer.split("://")[1],
+            });
+            continue;
+          }
+        }
+        const text = partText ? cleanApiPartText(partText) : "";
+        if (text) {
+          segments.push({
+            type: isThoughtPart ? "thought" : "text",
+            content: text,
+          });
+        }
+      }
+
+      // Standalone content.text without parts (plain text / execution
+      // output), mirroring the mapping path.
+      if (
+        parts.length === 0 &&
+        typeof content.text === "string" &&
+        content.text.trim()
+      ) {
+        segments.push({ type: "text", content: content.text.trim() });
+      }
+    }
+
+    // Deep Research reports (widget_state), attachments, and Canvas
+    // documents from message metadata, mirroring the mapping path.
+    if (role !== "tool") {
+      const widgetRaw =
+        msg.metadata?.chatgpt_sdk?.widget_state ||
+        msg.metadata?.tool_response_metadata?.venus_widget_state;
+      if (widgetRaw) {
+        try {
+          const widget =
+            typeof widgetRaw === "string" ? JSON.parse(widgetRaw) : widgetRaw;
+          const reportText =
+            widget.report_message?.content?.parts?.[0] || widget.markdown;
+          const steering = widget.steering_acknowledgement;
+          let researchContent = "";
+          if (steering) researchContent += `${steering}\n\n`;
+          if (reportText) researchContent += reportText;
+          if (researchContent.trim()) {
+            segments.push({ type: "text", content: researchContent.trim() });
+          }
+        } catch {
+          // Ignore widget state JSON parse errors
+        }
+      }
+
+      if (
+        Array.isArray(msg.metadata?.attachments) &&
+        msg.metadata.attachments.length > 0
+      ) {
+        const fileNames = msg.metadata.attachments
+          .map((att) => att.name)
+          .filter(Boolean);
+        if (fileNames.length > 0) {
+          segments.push({
+            type: "text",
+            content: `[Attached: ${fileNames.join(", ")}]`,
+          });
+        }
+      }
+
+      if (msg.metadata?.canvas?.title) {
+        segments.push({
+          type: "text",
+          content: `[Canvas: ${msg.metadata.canvas.title}]`,
+        });
+      }
+    }
+
+    if (segments.length === 0) continue;
+
+    const displayRole = role === "user" ? "User" : "ChatGPT";
+    const timestamp = msg?.create_time
+      ? new Date(msg.create_time * 1000).toLocaleString()
+      : null;
+    // Citation / image-group references, mirroring the mapping path.
+    const citeMap = {};
+    const imageGroupMap = {};
+    for (const ref of msg?.metadata?.content_references ?? []) {
+      if (ref.matched_text) {
+        if (ref.items?.length) citeMap[ref.matched_text] = ref.items;
+        if (
+          ref.type === "image_group" ||
+          ref.matched_text.includes("image_group")
+        ) {
+          imageGroupMap[ref.matched_text] = ref;
+        }
+      }
+    }
+    pushOrMerge(
+      {
+        role: displayRole,
+        segments,
+        citeMap,
+        imageGroupMap,
+        timestamp,
+      },
+      isThoughtMsg,
+    );
+  }
+
+  return messages;
+}
+
 export function linearize(mapping, includeImages, currentNodeId) {
   let path = [];
   const leafId = resolveActiveLeafNode(mapping, currentNodeId);
@@ -268,8 +483,20 @@ export function linearize(mapping, includeImages, currentNodeId) {
     ) {
       const segments = [];
       const parts = msg?.content?.parts ?? [];
+      // Tool-invocation payloads (content_type "code") are not user-visible
+      // prose, whether carried as standalone text or inside parts.
+      const isToolInvocation = msg?.content?.content_type === "code";
 
       for (const part of parts) {
+        if (isToolInvocation) {
+          if (!(
+            includeImages &&
+            part?.content_type === "image_asset_pointer" &&
+            part?.asset_pointer
+          )) {
+            continue;
+          }
+        }
         let partText = "";
         let isThoughtPart = isThoughtMsg;
 
@@ -316,11 +543,15 @@ export function linearize(mapping, includeImages, currentNodeId) {
         }
       }
 
-      // Handle standalone content.text (e.g. execution_output or plain text)
+      // Handle standalone content.text (e.g. execution_output or plain text).
+      // Tool-invocation payloads (content_type "code", e.g. Deep Research
+      // "/Deep Research App/start" args JSON) are not user-visible prose —
+      // the site renders a status card instead — so never dump them as text.
       if (
         typeof msg.content?.text === "string" &&
         msg.content.text.trim() &&
-        parts.length === 0
+        parts.length === 0 &&
+        msg.content?.content_type !== "code"
       ) {
         segments.push({ type: "text", content: msg.content.text.trim() });
       }
@@ -887,6 +1118,7 @@ export class ChatGPTParser extends ChatParser {
       Link: currentUrl,
       Model:
         convoData?.model_slug ||
+        convoData?.default_model_slug ||
         (typeof document !== "undefined" && document.querySelector
           ? document.querySelector('[data-testid="model-selector-dropdown"]')
               ?.innerText
@@ -961,11 +1193,22 @@ export class ChatGPTParser extends ChatParser {
           };
         }
 
-        const apiMessages = linearize(
-          result.data.mapping,
-          includeImages,
-          result.data.current_node,
-        );
+        // The backend returns either the legacy `mapping` tree or the newer
+        // `messages` array shape — support both.
+        let apiMessages = [];
+        if (result.data.mapping) {
+          apiMessages = linearize(
+            result.data.mapping,
+            includeImages,
+            result.data.current_node,
+          );
+        }
+        if (apiMessages.length === 0 && Array.isArray(result.data.messages)) {
+          apiMessages = linearizeMessagesArray(
+            result.data.messages,
+            includeImages,
+          );
+        }
         if (apiMessages.length > 0) {
           return this.formatApiResult(
             result.data,

package/ai/chatgpt_helper.js

@@ -84,16 +84,30 @@ if (!window.__chatgptHelperInjected) {
       let images = {};
       if (includeImages) {
         const fileIds = new Set();
-        for (const node of Object.values(data.mapping)) {
+        const collectImagePointer = (part) => {
+          if (
+            part &&
+            part.content_type === "image_asset_pointer" &&
+            part.asset_pointer
+          ) {
+            fileIds.add(part.asset_pointer.split("://")[1]);
+          }
+        };
+        // Legacy `mapping` tree shape…
+        for (const node of Object.values(data.mapping || {})) {
           const msg = node.message;
           if (msg && msg.content && Array.isArray(msg.content.parts)) {
             for (const part of msg.content.parts) {
-              if (
-                part &&
-                part.content_type === "image_asset_pointer" &&
-                part.asset_pointer
-              ) {
-                fileIds.add(part.asset_pointer.split("://")[1]);
+              collectImagePointer(part);
+            }
+          }
+        }
+        // …and the newer `messages` array shape.
+        if (Array.isArray(data.messages)) {
+          for (const msg of data.messages) {
+            if (msg && msg.content && Array.isArray(msg.content.parts)) {
+              for (const part of msg.content.parts) {
+                collectImagePointer(part);
               }
             }
           }

package/ai/claude.js

@@ -1,5 +1,6 @@
 import { ChatParser } from "./base.js";
 import { convertToMarkdown } from "../utils/html-to-markdown.js";
+import { pickTimestamp } from "../utils/timestamps.js";
 
 async function getOrganizationId() {
   try {
@@ -103,6 +104,124 @@ const MIME_TO_LANG = {
   "application/vnd.ant.code": "text",
 };
 
+// Binary archives cannot be inlined as text in any export format; they are
+// listed by name so they at least appear in markdown/html/json exports.
+const BINARY_ARCHIVE_MIMES = new Set([
+  "application/x-tar",
+  "application/gzip",
+  "application/zip",
+  "application/x-7z-compressed",
+  "application/x-rar-compressed",
+]);
+
+function formatFileSize(bytes) {
+  if (typeof bytes !== "number" || !Number.isFinite(bytes) || bytes < 0) {
+    return "";
+  }
+  if (bytes < 1024) return `${bytes} B`;
+  return `${(bytes / 1024).toFixed(1)} KB`;
+}
+
+function basenameOfPath(filePath) {
+  if (typeof filePath !== "string" || !filePath) return "file";
+  const base = filePath.split("/").pop();
+  return base || "file";
+}
+
+function isBinaryArchive(mimeType, filePath) {
+  if (mimeType && BINARY_ARCHIVE_MIMES.has(mimeType)) return true;
+  return /\.(tar\.gz|tgz|tar|zip|gz|7z|rar)$/i.test(filePath || "");
+}
+
+// Collect files surfaced via the `present_files` tool (File Creation
+// integration). The tool_use block carries `input.filepaths`; the matching
+// tool_result block (joined via tool_use_id) carries display names + mime
+// types as `local_resource` entries. Either side may be missing, so resolve
+// metadata when available and fall back to bare paths otherwise.
+function collectPresentedFiles(branch) {
+  const pathsByToolUseId = new Map();
+  const resourcesByToolUseId = new Map();
+  for (const msg of branch) {
+    if (!Array.isArray(msg?.content)) continue;
+    for (const block of msg.content) {
+      if (block?.type === "tool_use" && block?.name === "present_files") {
+        const filepaths = block?.input?.filepaths;
+        if (Array.isArray(filepaths) && filepaths.length > 0 && block.id) {
+          pathsByToolUseId.set(block.id, filepaths);
+        }
+      } else if (
+        block?.type === "tool_result" &&
+        Array.isArray(block.content)
+      ) {
+        const resources = block.content.filter(
+          (item) => item && item.type === "local_resource" && item.file_path,
+        );
+        if (resources.length > 0) {
+          const toolUseId = block.tool_use_id || block.id;
+          if (toolUseId) resourcesByToolUseId.set(toolUseId, resources);
+        }
+      }
+    }
+  }
+
+  const filesByToolUseId = new Map();
+  const seenPaths = new Set();
+  const toEntry = (resource, fallbackPath) => {
+    const filePath = resource?.file_path || fallbackPath || "";
+    const name = resource?.name || basenameOfPath(filePath);
+    return {
+      key: resource?.uuid || filePath || `${name}`,
+      name,
+      path: filePath,
+      mime: resource?.mime_type || "",
+    };
+  };
+
+  for (const [toolUseId, resources] of resourcesByToolUseId.entries()) {
+    const entries = [];
+    for (const resource of resources) {
+      // Skip the inline "say they are below" helper text item (type: text).
+      if (!resource.file_path) continue;
+      if (seenPaths.has(resource.file_path)) continue;
+      seenPaths.add(resource.file_path);
+      entries.push(toEntry(resource));
+    }
+    if (entries.length > 0) filesByToolUseId.set(toolUseId, entries);
+  }
+
+  for (const [toolUseId, filepaths] of pathsByToolUseId.entries()) {
+    const existing = filesByToolUseId.get(toolUseId) || [];
+    const entries = [...existing];
+    for (const filePath of filepaths) {
+      if (typeof filePath !== "string" || !filePath) continue;
+      if (seenPaths.has(filePath)) continue;
+      seenPaths.add(filePath);
+      entries.push(toEntry(null, filePath));
+    }
+    if (entries.length > 0) filesByToolUseId.set(toolUseId, entries);
+  }
+
+  return filesByToolUseId;
+}
+
+function formatGeneratedFilesSection(files) {
+  if (!Array.isArray(files) || files.length === 0) return "";
+  const lines = files.map((file) => {
+    const displayName = file.name || basenameOfPath(file.path);
+    const meta = [];
+    if (file.mime) meta.push(file.mime);
+    if (isBinaryArchive(file.mime, file.path || displayName)) {
+      meta.push("binary archive — download from Claude UI");
+    }
+    const suffix = meta.length > 0 ? ` _(${meta.join(", ")})_` : "";
+    const pathSuffix =
+      file.path && file.path !== displayName ? ` — \`${file.path}\`` : "";
+    return `- \`${displayName}\`${suffix}${pathSuffix}`;
+  });
+  const label = files.length === 1 ? "Generated file:" : "Generated files:";
+  return `\n\n**${label}**\n${lines.join("\n")}\n\n`;
+}
+
 function extractArtifactsFromText(text) {
   const artifactRegex = /<antArtifact[^>]*>([\s\S]*?)<\/antArtifact>/g;
   const artifacts = [];
@@ -267,6 +386,44 @@ function extractArtifacts(message, foldedArtifacts = new Map()) {
   return artifacts;
 }
 
+// File cards rendered for generated files (markdown docs, tarballs, …).
+// They carry no text content for convertToMarkdown, so extract their display
+// names (aria-label="View <name>") and drop the nodes to avoid button noise.
+function extractFileCards(root) {
+  if (!root || typeof root.querySelectorAll !== "function") return [];
+  const names = [];
+  const cards = root.querySelectorAll('[data-testid="file-card-open"]');
+  for (const card of cards) {
+    const label = card.getAttribute && card.getAttribute("aria-label");
+    const match = typeof label === "string" && label.match(/^View\s+(.+)$/i);
+    const name = match ? match[1].trim() : "";
+    // No name-based dedup: two cards may legitimately share a display name
+    // (same basename in different directories). Each card element is visited
+    // exactly once, so nothing is double-counted here.
+    if (name) {
+      names.push(name);
+    }
+    // Remove the whole card element so download buttons / type badges
+    // don't leak into the markdown conversion — but only when the parent
+    // is a dedicated card wrapper. If the button shares its parent with
+    // other prose, remove just the button to avoid deleting message text.
+    const parent =
+      card.parentElement && card.parentElement !== root
+        ? card.parentElement
+        : null;
+    const isDedicatedCard =
+      !!parent &&
+      parent.querySelectorAll('[data-testid="file-card-open"]').length === 1;
+    const target = isDedicatedCard ? parent : card;
+    if (target && typeof target.remove === "function") {
+      target.remove();
+    } else if (card.parentNode) {
+      card.parentNode.removeChild(card);
+    }
+  }
+  return names;
+}
+
 function unrollInteractiveElements(root, doc) {
   if (!root || !doc) return;
 
@@ -371,6 +528,8 @@ export class ClaudeParser extends ChatParser {
 
           const branch = getCurrentBranch(data);
           const foldedArtifacts = collectArtifacts(branch);
+          const presentedFilesByToolUseId = collectPresentedFiles(branch);
+          const emittedFileKeys = new Set();
 
           const toolResultMap = new Map();
           for (const msg of branch) {
@@ -473,6 +632,37 @@ export class ClaudeParser extends ChatParser {
                         : `> - ${qText}\n`;
                     }
                     contentStr += `${qStr}\n`;
+                  } else if (
+                    block.name === "present_files" &&
+                    presentedFilesByToolUseId.has(block.id)
+                  ) {
+                    // Files surfaced via the File Creation integration
+                    // (markdown docs, tarball, …). Without this they are
+                    // silently dropped from every export format.
+                    const files = presentedFilesByToolUseId.get(block.id);
+                    const fresh = files.filter(
+                      (file) => !emittedFileKeys.has(file.key),
+                    );
+                    fresh.forEach((file) => emittedFileKeys.add(file.key));
+                    if (fresh.length > 0) {
+                      contentStr += formatGeneratedFilesSection(fresh);
+                    }
+                  }
+                } else if (
+                  block.type === "tool_result" &&
+                  Array.isArray(block.content)
+                ) {
+                  // Orphan file presentation: the matching tool_use block may
+                  // sit outside the current branch, so emit unseen
+                  // local_resource entries directly from the result.
+                  const toolUseId = block.tool_use_id || block.id;
+                  const files = presentedFilesByToolUseId.get(toolUseId) || [];
+                  const fresh = files.filter(
+                    (file) => !emittedFileKeys.has(file.key),
+                  );
+                  fresh.forEach((file) => emittedFileKeys.add(file.key));
+                  if (fresh.length > 0) {
+                    contentStr += formatGeneratedFilesSection(fresh);
                   }
                 }
               }
@@ -491,8 +681,9 @@ export class ClaudeParser extends ChatParser {
                 if (attachment.file_name) {
                   let header = `### Attachment: ${attachment.file_name}`;
                   const meta = [];
-                  if (attachment.file_size) {
-                    meta.push(`${(attachment.file_size / 1024).toFixed(1)} KB`);
+                  const sizeLabel = formatFileSize(attachment.file_size);
+                  if (sizeLabel) {
+                    meta.push(sizeLabel);
                   }
                   if (attachment.file_type) {
                     meta.push(attachment.file_type);
@@ -505,7 +696,9 @@ export class ClaudeParser extends ChatParser {
                     contentStr += `\`\`\`\`\n${attachment.extracted_content}\n\`\`\`\`\n\n`;
                   }
                 } else if (attachment.extracted_content) {
-                  contentStr += `\n\n### Pasted\n\`\`\`\`\n${attachment.extracted_content}\n\`\`\`\`\n\n`;
+                  const sizeLabel = formatFileSize(attachment.file_size);
+                  const sizeSuffix = sizeLabel ? ` _(${sizeLabel})_` : "";
+                  contentStr += `\n\n### Pasted content${sizeSuffix}\n\`\`\`\`\n${attachment.extracted_content}\n\`\`\`\`\n\n`;
                 }
               }
             }
@@ -544,6 +737,13 @@ export class ClaudeParser extends ChatParser {
               if (thinkingStr) {
                 msgObj.thinking = thinkingStr;
               }
+              const timestamp = pickTimestamp(message, [
+                "created_at",
+                "updated_at",
+              ]);
+              if (timestamp) {
+                msgObj.timestamp = timestamp;
+              }
               messages.push(msgObj);
             }
 
@@ -568,6 +768,14 @@ export class ClaudeParser extends ChatParser {
               messages.push({
                 role: "Claude Artifact",
                 content: artContent.trim(),
+                ...(pickTimestamp(message, ["created_at", "updated_at"])
+                  ? {
+                      timestamp: pickTimestamp(message, [
+                        "created_at",
+                        "updated_at",
+                      ]),
+                    }
+                  : {}),
               });
             }
           }
@@ -678,8 +886,16 @@ export class ClaudeParser extends ChatParser {
       ) {
         role = "Claude";
         const clone = el.cloneNode(true);
+        // Extract file cards first: unrollInteractiveElements strips all
+        // buttons, which would destroy the card markers.
+        const fileCardNames = extractFileCards(clone);
         unrollInteractiveElements(clone, el.ownerDocument || document);
         content = convertToMarkdown(clone);
+        if (fileCardNames.length > 0) {
+          content += formatGeneratedFilesSection(
+            fileCardNames.map((name) => ({ name, path: "", mime: "" })),
+          );
+        }
       } else if (el.matches(".artifact-block-cell")) {
         role = "Claude Artifact";
 
@@ -712,7 +928,36 @@ export class ClaudeParser extends ChatParser {
       }
 
       if (content) {
-        messages.push({ role, content });
+        const msgObj = { role, content };
+        // Best-effort DOM timestamp. <time datetime> usually lives in the
+        // sibling MessageActions toolbar inside the same transcript row —
+        // not inside the message element itself. Scope to the closest row
+        // so we never borrow the previous/next turn's timestamp. Sparse in
+        // practice (hover-only on some turns) — absent stays absent.
+        try {
+          let timeEl =
+            typeof el.querySelector === "function"
+              ? el.querySelector("time[datetime]")
+              : null;
+          if (!timeEl && typeof el.closest === "function") {
+            // Narrow containers only: transcript-row holds one turn, so its
+            // time belongs to this message. Never fall back to broad
+            // containers like article (many turns) — wrong date is worse
+            // than no date.
+            const row = el.closest(
+              '[data-testid="transcript-row"], .group\\/message-row',
+            );
+            timeEl = row?.querySelector?.("time[datetime]") || null;
+          }
+          const datetime = timeEl?.getAttribute?.("datetime");
+          const timestamp = pickTimestamp({ datetime }, ["datetime"]);
+          if (timestamp) {
+            msgObj.timestamp = timestamp;
+          }
+        } catch {
+          // Ignore DOM timestamp lookup errors
+        }
+        messages.push(msgObj);
       }
     }
 

package/ai/deepseek.js

@@ -1,5 +1,6 @@
 import { ChatParser } from "./base.js";
 import { convertToMarkdown } from "../utils/html-to-markdown.js";
+import { normalizeTimestamp } from "../utils/timestamps.js";
 
 function getUserToken() {
   try {
@@ -120,6 +121,12 @@ async function fetchDeepSeekConversation(sessionId, token) {
       if (thinking) {
         msg.thinking = thinking;
       }
+      // Turn granularity: inserted_at is seconds-epoch, ~identical within a
+      // USER/ASSISTANT pair, so both sides share the turn's timestamp.
+      const timestamp = normalizeTimestamp(msgNode?.inserted_at);
+      if (timestamp) {
+        msg.timestamp = timestamp;
+      }
       return msg;
     })
     .filter((msg) => msg.content.length > 0);

package/ai/gemini.js

@@ -461,10 +461,15 @@ export class GeminiParser extends ChatParser {
       }
 
       const modelText = this.findModelTextInApiItem(item, options);
-      if (modelText) {
+      const researchExtras = this.extractDeepResearchExtras(item);
+      const combined = [modelText, researchExtras]
+        .map((t) => this.stripChipPlaceholders(t))
+        .filter((t) => t && t.trim())
+        .join("\n\n");
+      if (combined.trim()) {
         messages.push({
           role: "Model",
-          content: normalizeLatexMath(modelText.trim()),
+          content: normalizeLatexMath(combined.trim()),
           turnId,
         });
       }
@@ -473,6 +478,213 @@ export class GeminiParser extends ChatParser {
     return messages;
   }
 
+  // Deep-research turns render a short summary plus placeholder chip links
+  // (e.g. http://googleusercontent.com/immersive_entry_chip/0) whose real
+  // content — research plan, full report, citation map — lives in adjacent
+  // candidate slots. Only the known chip placeholders are stripped; every
+  // other URL (including googleusercontent subdomains hosting real images
+  // and links inside markdown) is left intact.
+  stripChipPlaceholders(text) {
+    if (typeof text !== "string" || !text) return text;
+    const chipPattern =
+      /<?https?:\/\/googleusercontent\.com\/(?:immersive_entry_chip|deep_research_confirmation_content)(?:\/\d*)?>?/;
+    const chipPatternGlobal = new RegExp(chipPattern.source, "g");
+    return text
+      .split("\n")
+      .filter((line) => {
+        if (!chipPattern.test(line)) return true;
+        return line.replace(chipPatternGlobal, "").trim() !== "";
+      })
+      .map((line) => line.replace(chipPatternGlobal, ""))
+      .join("\n")
+      .replace(/\n{3,}/g, "\n\n");
+  }
+
+  getApiCandidates(item) {
+    try {
+      if (!Array.isArray(item[3])) return [];
+      const candidates = Array.isArray(item[3][0]) ? item[3][0] : item[3];
+      return candidates.filter((cand) => Array.isArray(cand));
+    } catch {
+      return [];
+    }
+  }
+
+  buildDeepResearchCiteMap(citeGroups) {
+    const map = new Map();
+    try {
+      const groups = Array.isArray(citeGroups) ? citeGroups : [citeGroups];
+      for (const group of groups) {
+        if (!group || typeof group !== "object" || Array.isArray(group))
+          continue;
+        for (const entries of Object.values(group)) {
+          if (!Array.isArray(entries)) continue;
+          for (const entry of entries) {
+            if (!Array.isArray(entry) || !Array.isArray(entry[1])) continue;
+            for (const source of entry[1]) {
+              // Source shape: [null, null, null,
+              //   [detail, number, ...]] where detail = [favicon, url, title].
+              if (!Array.isArray(source) || !Array.isArray(source[3])) continue;
+              const detail = source[3][0];
+              const url = Array.isArray(detail) ? detail[1] : null;
+              const number = source[3][1];
+              if (
+                typeof url === "string" &&
+                url.startsWith("http") &&
+                typeof number === "number" &&
+                !map.has(number)
+              ) {
+                map.set(number, {
+                  url,
+                  title:
+                    typeof detail[2] === "string" && detail[2]
+                      ? detail[2]
+                      : url,
+                });
+              }
+            }
+          }
+        }
+      }
+    } catch {
+      // Ignore malformed citation maps
+    }
+    return map;
+  }
+
+  resolveDeepResearchCites(markdown, citeMap) {
+    if (typeof markdown !== "string" || !(citeMap instanceof Map)) {
+      return markdown;
+    }
+    return markdown.replace(/ ?\[cite: ([\d,\s]+)\]/g, (match, nums) => {
+      const numbers = [
+        ...new Set(
+          nums
+            .split(",")
+            .map((n) => parseInt(n.trim(), 10))
+            .filter((n) => Number.isFinite(n)),
+        ),
+      ];
+      if (numbers.length === 0) return "";
+      const links = numbers.map((n) => {
+        const cite = citeMap.get(n);
+        return cite ? `[[${n}]](${cite.url})` : `[${n}]`;
+      });
+      return ` ${links.join(" ")}`;
+    });
+  }
+
+  extractImmersiveDocFromCandidate(cand) {
+    try {
+      if (!Array.isArray(cand[30]) || cand[30].length === 0) return "";
+      const doc = cand[30][0];
+      if (!Array.isArray(doc)) return "";
+      // Guard: immersive research documents carry this task marker.
+      if (doc[3] !== "agency-placeholder-task-id") return "";
+      if (typeof doc[4] !== "string" || doc[4].trim().length < 100) return "";
+      const title = typeof doc[2] === "string" && doc[2] ? doc[2] : "Report";
+      const citeMap = this.buildDeepResearchCiteMap(doc[5]);
+      const markdown = this.resolveDeepResearchCites(doc[4].trim(), citeMap);
+      return `## ${title}\n\n${markdown}`;
+    } catch {
+      return "";
+    }
+  }
+
+  extractResearchPlanFromCandidate(cand) {
+    try {
+      if (!Array.isArray(cand[12])) return "";
+      for (const annotation of cand[12]) {
+        if (
+          !annotation ||
+          typeof annotation !== "object" ||
+          Array.isArray(annotation) ||
+          !Array.isArray(annotation["56"])
+        ) {
+          continue;
+        }
+        const [planTitle, steps] = annotation["56"];
+        if (!Array.isArray(steps) || steps.length === 0) continue;
+        const lines = steps.map((step, idx) => {
+          if (!Array.isArray(step)) return null;
+          const stepTitle = step[1] || `Step ${idx + 1}`;
+          const desc =
+            typeof step[2] === "string" && step[2].trim()
+              ? `: ${step[2].trim()}`
+              : "";
+          return `${idx + 1}. **${stepTitle}**${desc}`;
+        });
+        const valid = lines.filter(Boolean);
+        if (valid.length === 0) continue;
+        const heading =
+          typeof planTitle === "string" && planTitle
+            ? `### ${planTitle} — research plan`
+            : "### Research plan";
+        return `${heading}\n${valid.join("\n")}`;
+      }
+    } catch {
+      // Ignore malformed plan annotations
+    }
+    return "";
+  }
+
+  extractActivitySources(item, limit = 40) {
+    const seen = new Map();
+    try {
+      const trail = item[3]?.[4];
+      if (!Array.isArray(trail)) return [];
+      for (const entry of trail) {
+        const detail = entry?.[4]?.[2];
+        const url = Array.isArray(detail) ? detail[1] : null;
+        if (typeof url !== "string" || !url.startsWith("http")) continue;
+        if (seen.has(url)) continue;
+        const title =
+          typeof detail[2] === "string" && detail[2] ? detail[2] : url;
+        seen.set(url, title);
+      }
+    } catch {
+      // Ignore malformed activity trails
+    }
+    const all = [...seen.entries()];
+    const shown = all.slice(0, limit);
+    const lines = shown.map(([url, title]) => `- [${title}](${url})`);
+    if (all.length > shown.length) {
+      lines.push(`- …and ${all.length - shown.length} more`);
+    }
+    return lines;
+  }
+
+  extractDeepResearchExtras(item) {
+    const parts = [];
+    try {
+      // Extras must come from the same candidate that supplied the visible
+      // text (findModelTextInApiItem uses the first candidate with text),
+      // never mixed in from alternate drafts.
+      const candidates = this.getApiCandidates(item);
+      let anchorIdx = candidates.findIndex(
+        (cand) =>
+          (Array.isArray(cand[1]) && typeof cand[1][0] === "string") ||
+          typeof cand[1] === "string" ||
+          (typeof cand[0] === "string" && cand[0].length > 50),
+      );
+      if (anchorIdx === -1) anchorIdx = 0;
+      const anchor = candidates[anchorIdx];
+      if (anchor) {
+        const plan = this.extractResearchPlanFromCandidate(anchor);
+        if (plan) parts.push(plan);
+        const doc = this.extractImmersiveDocFromCandidate(anchor);
+        if (doc) parts.push(doc);
+      }
+      const sourceLines = this.extractActivitySources(item);
+      if (sourceLines.length > 0) {
+        parts.push(`**Sources consulted:**\n${sourceLines.join("\n")}`);
+      }
+    } catch {
+      // Never let research extras break the base message
+    }
+    return parts.filter(Boolean).join("\n\n");
+  }
+
   findUserTextInApiItem(item) {
     try {
       if (typeof item[2]?.[0]?.[0] === "string") return item[2][0][0];
@@ -693,20 +905,26 @@ export class GeminiParser extends ChatParser {
           if (markdownDiv) {
             const clone = markdownDiv.cloneNode(true);
 
-            // Remove UI buttons, thought overlays, and interactive toolbars
+            // Remove UI buttons, thought overlays, follow-up suggestion
+            // widgets, and interactive toolbars.
+            // Note: .hide-from-message-actions is NOT removed — it wraps
+            // deep-research plan widgets whose text must be kept (buttons
+            // inside are still stripped above).
             clone
               .querySelectorAll(
-                "button, .thoughts-container, .thoughts-wrapper, model-thoughts, .table-footer, .hide-from-message-actions, message-actions, election-info-disclaimer, finance-info-disclaimer, .sources-list",
+                "button, follow-up, .follow-up-container, .thoughts-container, .thoughts-wrapper, model-thoughts, .table-footer, message-actions, election-info-disclaimer, finance-info-disclaimer, .sources-list",
               )
               .forEach((el) => el.remove());
 
-            // Unwrap response-element wrappers
-            clone.querySelectorAll("response-element").forEach((el) => {
-              while (el.firstChild) {
-                el.parentNode.insertBefore(el.firstChild, el);
-              }
-              el.remove();
-            });
+            // Unwrap response-element wrappers and message-action guards
+            clone
+              .querySelectorAll("response-element, .hide-from-message-actions")
+              .forEach((el) => {
+                while (el.firstChild) {
+                  el.parentNode.insertBefore(el.firstChild, el);
+                }
+                el.remove();
+              });
 
             const text = convertToMarkdown(clone);
             const trimmed = text.trim();
@@ -722,7 +940,27 @@ export class GeminiParser extends ChatParser {
       });
     }
 
-    // Strategy 2: Deep Research immersive panel structure fallback
+    // Strategy 2: Deep Research immersive panel (full report document).
+    // Runs even when chat shells were found above — the panel holds the
+    // report body, which never appears in the chat transcript.
+    const immersiveSections = this.extractImmersivePanelMessages(document);
+    immersiveSections.forEach((section) => {
+      if (!section.content || seenTexts.has(section.content)) return;
+      // The panel body passes through a different conversion path than chat
+      // messages, so exact-match dedup never fires. Skip the section when a
+      // Model message already carries the report (e.g. panel content also
+      // rendered inside a chat model-response).
+      const body = section.content.replace(/^## .*\n\n/, "");
+      const probe = body.slice(0, 300);
+      const alreadyExported =
+        probe.length > 0 &&
+        messages.some((m) => m.role === "Model" && m.content.includes(probe));
+      if (!alreadyExported) {
+        seenTexts.add(section.content);
+        messages.push(section);
+      }
+    });
+
     if (messages.length === 0) {
       const deepResearchPanel = document.querySelector(
         "deep-research-immersive-panel",
@@ -838,6 +1076,67 @@ export class GeminiParser extends ChatParser {
     return sections;
   }
 
+  // Extracts the open Deep Research immersive panel (the full report
+  // document). Returns [] when no panel is rendered in the DOM.
+  extractImmersivePanelMessages(doc) {
+    const sections = [];
+    try {
+      if (!doc || typeof doc.querySelector !== "function") return sections;
+      const panel =
+        doc.querySelector("immersive-panel deep-research-immersive-panel") ||
+        doc.querySelector("deep-research-immersive-panel");
+      if (!panel) return sections;
+
+      const titleEl =
+        panel.querySelector("toolbar .title-text") ||
+        panel.querySelector(".title-text");
+      const title = (titleEl?.textContent || "").trim();
+
+      const bodyRoot =
+        panel.querySelector('[data-test-id="message-content"] .markdown') ||
+        panel.querySelector("#extended-response-markdown-content") ||
+        panel.querySelector("message-content .markdown") ||
+        panel.querySelector("message-content");
+      if (!bodyRoot) return sections;
+
+      const clone = bodyRoot.cloneNode(true);
+      // Inline citation footnotes carry only a source index — render it as
+      // text so references survive markdown conversion.
+      clone.querySelectorAll("sup[data-turn-source-index]").forEach((sup) => {
+        const idx = sup.getAttribute("data-turn-source-index");
+        if (idx && sup.parentNode) {
+          sup.parentNode.replaceChild(doc.createTextNode(`[${idx}]`), sup);
+        }
+      });
+      clone
+        .querySelectorAll(
+          "button, toolbar, toc-menu, mat-menu, message-actions, follow-up, .follow-up-container, .hide-from-message-actions button",
+        )
+        .forEach((el) => el.remove());
+      clone.querySelectorAll("response-element").forEach((el) => {
+        while (el.firstChild) {
+          el.parentNode.insertBefore(el.firstChild, el);
+        }
+        el.remove();
+      });
+
+      const body = convertToMarkdown(clone)
+        .trim()
+        // Turndown escapes the [N] citation markers inserted above;
+        // restore them (they render identically either way).
+        .replace(/\\\[(\d+)\\\]/g, "[$1]");
+      if (body && body.length > 100) {
+        sections.push({
+          role: "Model",
+          content: title ? `## ${title}\n\n${body}` : body,
+        });
+      }
+    } catch (error) {
+      console.error("[Gemini Parser] Error extracting immersive panel:", error);
+    }
+    return sections;
+  }
+
   extractDeepResearchPanelContent(panelElement) {
     const sections = [];
     try {

package/ai/meta.js

@@ -1,5 +1,6 @@
 import { ChatParser } from "./base.js";
 import { convertToMarkdown } from "../utils/html-to-markdown.js";
+import { pickTimestamp } from "../utils/timestamps.js";
 
 // Observed Relay doc_ids for the conversation message list (Sept 2026).
 // These rotate when Meta redeploys the web client. The parser tries the
@@ -139,12 +140,23 @@ export function formatMetaEdges(edges) {
   });
   const messages = [];
   for (const node of nodes) {
+    // Turn granularity: user + assistant in the same turn share createdAt,
+    // so both messages carry the same timestamp (matches Perplexity pattern).
+    const timestamp = pickTimestamp(node, ["createdAt", "userCreatedAt"]);
     if (node.__typename === "UserMessage") {
       const content = extractMetaUserText(node);
-      if (content) messages.push({ role: "User", content });
+      if (content) {
+        const msg = { role: "User", content };
+        if (timestamp) msg.timestamp = timestamp;
+        messages.push(msg);
+      }
     } else if (node.__typename === "AssistantMessage") {
       const content = extractMetaAssistantText(node);
-      if (content) messages.push({ role: "Meta AI", content });
+      if (content) {
+        const msg = { role: "Meta AI", content };
+        if (timestamp) msg.timestamp = timestamp;
+        messages.push(msg);
+      }
     }
   }
   return messages;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "decant-core",
-  "version": "1.12.0",
+  "version": "1.14.0",
   "description": "A shared web extraction layer for AI conversations and regular web pages.",
   "type": "module",
   "engines": {

package/utils/timestamps.js

@@ -0,0 +1,65 @@
+/**
+ * Shared per-message timestamp normalization (decant-core).
+ *
+ * Parsers attach `timestamp` as ISO strings, locale strings, or numeric
+ * epochs (seconds or milliseconds). Display layers (ace) format whatever
+ * string they receive, so this helper keeps ISO/locale strings as-is and
+ * converts numeric epochs to ISO.
+ *
+ * Turn-granularity sources (Perplexity entries, Meta edges, DeepSeek pairs)
+ * reuse the same timestamp for both prompt and response — callers pass the
+ * same value to both messages.
+ */
+
+/** Epoch values with abs < 1e11 are treated as seconds, else milliseconds. */
+export function normalizeEpochToMs(value) {
+  if (Math.abs(value) < 1e11) return value * 1000;
+  return value;
+}
+
+/**
+ * Normalizes a raw timestamp to a display-ready string.
+ * @param {unknown} value ISO string, locale string, epoch number, or numeric string.
+ * @returns {string|null} ISO string for epochs, trimmed input for date strings, else null.
+ */
+export function normalizeTimestamp(value) {
+  if (typeof value === "number" && Number.isFinite(value)) {
+    try {
+      return new Date(normalizeEpochToMs(value)).toISOString();
+    } catch {
+      return null;
+    }
+  }
+  if (typeof value === "string") {
+    const trimmed = value.trim();
+    if (!trimmed) return null;
+    if (/^[+-]?\d+(\.\d+)?$/.test(trimmed)) {
+      const numeric = Number(trimmed);
+      if (Number.isFinite(numeric)) {
+        try {
+          return new Date(normalizeEpochToMs(numeric)).toISOString();
+        } catch {
+          return null;
+        }
+      }
+      return null;
+    }
+    return trimmed;
+  }
+  return null;
+}
+
+/**
+ * Picks the first available timestamp from candidate fields.
+ * @param {object} source Parser payload object.
+ * @param {string[]} keys Field names to try in order.
+ * @returns {string|null}
+ */
+export function pickTimestamp(source, keys) {
+  if (!source || !Array.isArray(keys)) return null;
+  for (const key of keys) {
+    const normalized = normalizeTimestamp(source[key]);
+    if (normalized) return normalized;
+  }
+  return null;
+}