decant-core 1.11.0 → 1.13.0

package/LICENSE

@@ -1,3 +1,19 @@
+ADDITIONAL PERMISSION UNDER GNU AGPL VERSION 3 SECTION 7:
+FOSS LINKING EXCEPTION
+
+Permission is granted to link, import, or bundle decant-core into projects
+distributed under any OSI-approved open source license (including MPL-2.0, MIT,
+Apache-2.0, and BSD) and distribute the resulting work under that project's
+license, without requiring the enclosing project to be licensed under AGPLv3.
+Any modifications directly made to decant-core source files remain subject to AGPLv3.
+
+COMMERCIAL USE
+If you wish to use decant-core in closed-source, proprietary, or commercial
+software that cannot comply with the AGPLv3, a commercial license is available.
+Please contact office@covai.org for licensing terms.
+
+==============================================================================
+
                     GNU AFFERO GENERAL PUBLIC LICENSE
                        Version 3, 19 November 2007
 

package/README.md

@@ -16,7 +16,7 @@ And every time one of those platforms changes its UI, seriously re-renders a mes
 
 Existing chat exporters often suffer from two major flaws: they break whenever platform DOMs update, and many route user conversations through third-party servers.
 
-`decant-core` was created to solve both at the foundation. Originally built to power local-first extensions like [AI Chat Exporter](https://ai-chat-exporter.covai.org/) and [Decant](https://decant.covai.org/), it decouples fragile platform parsing from presentation. By sharing this engine under AGPL-3.0, any browser extension, web clipper, archiver, or research tool can rely on a maintained, local-first extraction layer instead of reverse-engineering AI platforms in isolation.
+`decant-core` was created to solve both at the foundation. Originally built to power local-first extensions like [AI Chat Exporter](https://ace.covai.org/) and [Decant](https://decant.covai.org/), it decouples fragile platform parsing from presentation. By sharing this engine under AGPL-3.0, any browser extension, web clipper, archiver, or research tool can rely on a maintained, local-first extraction layer instead of reverse-engineering AI platforms in isolation.
 
 ```bash
 npm install decant-core
@@ -119,28 +119,28 @@ import { normalizeLatexMath } from "decant-core";
 
 19 AI chat platform parsers plus generic web article extraction:
 
-| Platform                            | Parser                    | Extraction strategy                                   |
-| :---------------------------------- | :------------------------ | :---------------------------------------------------- |
-| **ChatGPT**                         | `ChatGPTParser`           | DOM + internal API                                    |
-| **Claude**                          | `ClaudeParser`            | DOM + internal API + React fiber                      |
-| **Google Gemini**                   | `GeminiParser`            | DOM + batchexecute RPC                                |
-| **Microsoft Copilot**               | `CopilotParser`           | DOM (multi-domain)                                    |
-| **Perplexity**                      | `PerplexityParser`        | Internal API + DOM fallback                           |
-| **DeepSeek**                        | `DeepSeekParser`          | DOM + internal API (`fragments[]`)                    |
-| **Qwen**                            | `QwenParser`              | DOM                                                   |
-| **Meta AI**                         | `MetaParser`              | Internal API (GraphQL) + DOM fallback                 |
-| **Mistral / Le Chat**               | `MistralParser`           | DOM                                                   |
-| **Proton Lumo**                     | `LumoParser`              | DOM (API is E2E-encrypted, not readable)              |
-| **Z.ai**                            | `ZAiParser`               | Internal API (chat + batch) + DOM fallback            |
-| **Grok**                            | `GrokParser`              | Internal API (response-node + load) + DOM             |
-| **Google AI Studio**                | `GoogleAIStudioParser`    | DOM                                                   |
-| **NotebookLM**                      | `NotebookLMParser`        | DOM                                                   |
-| **Google Search AI (AI Overviews)** | `GoogleSearchAIParser`    | DOM                                                   |
-| **Gemini Cloud Assist**             | `GeminiCloudAssistParser` | DOM                                                   |
-| **Joyland**                         | `JoylandParser`           | DOM                                                   |
-| **Chub**                            | `ChubParser`              | DOM                                                   |
-| **Duck.ai (DuckDuckGo AI)**         | `DuckAIParser`            | DOM (client-side privacy, no server chat history API) |
-| **Generic Web Article**             | `ArticleParser`           | Readability + Defuddle + Article-Extractor            |
+| Platform                            | Parser                    | Extraction strategy                        |
+| :---------------------------------- | :------------------------ | :----------------------------------------- |
+| **ChatGPT**                         | `ChatGPTParser`           | DOM + internal API                         |
+| **Claude**                          | `ClaudeParser`            | DOM + internal API + React fiber           |
+| **Google Gemini**                   | `GeminiParser`            | DOM + batchexecute RPC                     |
+| **Microsoft Copilot**               | `CopilotParser`           | DOM                                        |
+| **Perplexity**                      | `PerplexityParser`        | Internal API + DOM                         |
+| **DeepSeek**                        | `DeepSeekParser`          | DOM + internal API                         |
+| **Qwen**                            | `QwenParser`              | DOM                                        |
+| **Meta AI**                         | `MetaParser`              | Internal API + DOM                         |
+| **Mistral / Le Chat**               | `MistralParser`           | DOM                                        |
+| **Proton Lumo**                     | `LumoParser`              | DOM                                        |
+| **Z.ai**                            | `ZAiParser`               | Internal API + DOM                         |
+| **Grok**                            | `GrokParser`              | Internal API + DOM                         |
+| **Google AI Studio**                | `GoogleAIStudioParser`    | DOM                                        |
+| **NotebookLM**                      | `NotebookLMParser`        | DOM                                        |
+| **Google Search AI (AI Overviews)** | `GoogleSearchAIParser`    | DOM                                        |
+| **Gemini Cloud Assist**             | `GeminiCloudAssistParser` | DOM                                        |
+| **Joyland**                         | `JoylandParser`           | DOM                                        |
+| **Chub**                            | `ChubParser`              | DOM                                        |
+| **Duck.ai (DuckDuckGo AI)**         | `DuckAIParser`            | DOM                                        |
+| **Generic Web Article**             | `ArticleParser`           | Readability + Defuddle + Article-Extractor |
 
 All parsers extend the base [`ChatParser`](ai/base.js) interface — a consistent `isAvailable(url)` +
 normalized `parse()` contract. For the full extraction-strategy breakdown and maintenance model, see
@@ -151,9 +151,15 @@ normalized `parse()` contract. For the full extraction-strategy breakdown and ma
 
 ## License
 
-`decant-core` is licensed under the **GNU Affero General Public License v3.0 (AGPL-3.0-only)**.
+`decant-core` is licensed under the **GNU Affero General Public License v3.0 (AGPL-3.0-only)** with a **FOSS Linking Exception**, alongside a **Commercial License** option.
 
-That choice is deliberate. AI platforms change constantly, and parser fixes belong in a shared commons so the whole ecosystem benefits — not siloed in a proprietary fork. If you use `decant-core`, network-based deployments that serve modified versions must also offer the corresponding source. Please review [`LICENSE`](LICENSE) before incorporating it into your project.
+### FOSS Linking Exception (Open Source)
+
+Permission is granted to link, import, or bundle `decant-core` into projects distributed under any OSI-approved open source license (including **MPL-2.0**, **MIT**, **Apache-2.0**, and **BSD**) and distribute the resulting work under that project's license, without requiring the enclosing project to be licensed under AGPLv3. Any modifications directly made to `decant-core` source files remain subject to AGPLv3.
+
+### Commercial License
+
+If you wish to use `decant-core` in closed-source, proprietary, or commercial software that cannot comply with the AGPLv3, a commercial license is available. Please contact `office@covai.org` for licensing terms.
 
 ### Third-Party Test Fixtures Notice
 

package/ai/chatgpt.js

@@ -196,6 +196,221 @@ export function extractSharedConversationFromDom(
   return null;
 }
 
+function cleanApiPartText(partText) {
+  return partText
+    .replace(/\u{E0000}[\u{E0000}-\u{E007F}]*/gu, "")
+    .replace(/citeturn\d+\w*/g, "")
+    .trim();
+}
+
+// Linearize the newer backend-api conversation shape, which returns a
+// `messages` array instead of a `mapping` tree. Output segments match the
+// mapping-based linearize() format ({ type: "text" | "thought" | "image" })
+// so both flow through the same formatApiResult().
+export function linearizeMessagesArray(apiMessages, includeImages) {
+  const messages = [];
+  if (!Array.isArray(apiMessages)) return messages;
+
+  const pushOrMerge = (entry, isThoughtMsg) => {
+    if (
+      entry.role === "ChatGPT" &&
+      messages.length > 0 &&
+      messages[messages.length - 1].role === "ChatGPT"
+    ) {
+      const prevMsg = messages[messages.length - 1];
+      if (isThoughtMsg) {
+        prevMsg.segments.unshift(...entry.segments);
+      } else {
+        prevMsg.segments.push(...entry.segments);
+      }
+      Object.assign(prevMsg.citeMap, entry.citeMap);
+      Object.assign(prevMsg.imageGroupMap, entry.imageGroupMap);
+      if (entry.timestamp && !prevMsg.timestamp) {
+        prevMsg.timestamp = entry.timestamp;
+      }
+    } else {
+      messages.push(entry);
+    }
+  };
+
+  for (const msg of apiMessages) {
+    if (!msg) continue;
+    const role = msg?.author?.role;
+    if (role !== "user" && role !== "assistant" && role !== "tool") continue;
+    if (msg.metadata?.is_visually_hidden_from_conversation === true) continue;
+
+    const content = msg.content || {};
+    const contentType = content.content_type;
+    const segments = [];
+    // Mirror the mapping path's thought detection (content types plus
+    // author/recipient markers).
+    const isThoughtMsg =
+      msg?.author?.name === "thought" ||
+      msg?.recipient === "thought" ||
+      contentType === "thought" ||
+      contentType === "thoughts" ||
+      contentType === "reasoning_recap" ||
+      msg?.metadata?.reasoning_status === "is_reasoning";
+
+    // Reasoning summaries: thoughts = [{ summary, content }]
+    if (Array.isArray(content.thoughts) && content.thoughts.length > 0) {
+      const thoughtParts = content.thoughts
+        .map((t) =>
+          t.summary ? `**${t.summary}**\n${t.content || ""}` : t.content || "",
+        )
+        .map((t) => t.trim())
+        .filter(Boolean);
+      if (thoughtParts.length > 0) {
+        segments.push({ type: "thought", content: thoughtParts.join("\n\n") });
+      }
+    }
+
+    // Reasoning recap (e.g. "Worked for 11s")
+    if (
+      contentType === "reasoning_recap" &&
+      typeof content.content === "string" &&
+      content.content.trim()
+    ) {
+      segments.push({ type: "thought", content: content.content.trim() });
+    }
+
+    // Tool invocations (content_type "code", e.g. Deep Research args JSON)
+    // and tool-role messages are not user-visible prose — skip them.
+    if (contentType !== "code" && role !== "tool") {
+      const parts = Array.isArray(content.parts) ? content.parts : [];
+      for (const part of parts) {
+        let partText = "";
+        let isThoughtPart = isThoughtMsg;
+        if (typeof part === "string") {
+          partText = part;
+        } else if (part && typeof part === "object") {
+          if (part.content_type === "text" && typeof part.text === "string") {
+            partText = part.text;
+          } else if (
+            part.content_type === "thought" &&
+            typeof part.text === "string"
+          ) {
+            partText = part.text;
+            isThoughtPart = true;
+          } else if (
+            part.content_type === "audio_transcription" &&
+            typeof part.text === "string"
+          ) {
+            partText = part.text;
+          } else if (
+            includeImages &&
+            part?.content_type === "image_asset_pointer" &&
+            part?.asset_pointer
+          ) {
+            segments.push({
+              type: "image",
+              fileId: part.asset_pointer.split("://")[1],
+            });
+            continue;
+          }
+        }
+        const text = partText ? cleanApiPartText(partText) : "";
+        if (text) {
+          segments.push({
+            type: isThoughtPart ? "thought" : "text",
+            content: text,
+          });
+        }
+      }
+
+      // Standalone content.text without parts (plain text / execution
+      // output), mirroring the mapping path.
+      if (
+        parts.length === 0 &&
+        typeof content.text === "string" &&
+        content.text.trim()
+      ) {
+        segments.push({ type: "text", content: content.text.trim() });
+      }
+    }
+
+    // Deep Research reports (widget_state), attachments, and Canvas
+    // documents from message metadata, mirroring the mapping path.
+    if (role !== "tool") {
+      const widgetRaw =
+        msg.metadata?.chatgpt_sdk?.widget_state ||
+        msg.metadata?.tool_response_metadata?.venus_widget_state;
+      if (widgetRaw) {
+        try {
+          const widget =
+            typeof widgetRaw === "string" ? JSON.parse(widgetRaw) : widgetRaw;
+          const reportText =
+            widget.report_message?.content?.parts?.[0] || widget.markdown;
+          const steering = widget.steering_acknowledgement;
+          let researchContent = "";
+          if (steering) researchContent += `${steering}\n\n`;
+          if (reportText) researchContent += reportText;
+          if (researchContent.trim()) {
+            segments.push({ type: "text", content: researchContent.trim() });
+          }
+        } catch {
+          // Ignore widget state JSON parse errors
+        }
+      }
+
+      if (
+        Array.isArray(msg.metadata?.attachments) &&
+        msg.metadata.attachments.length > 0
+      ) {
+        const fileNames = msg.metadata.attachments
+          .map((att) => att.name)
+          .filter(Boolean);
+        if (fileNames.length > 0) {
+          segments.push({
+            type: "text",
+            content: `[Attached: ${fileNames.join(", ")}]`,
+          });
+        }
+      }
+
+      if (msg.metadata?.canvas?.title) {
+        segments.push({
+          type: "text",
+          content: `[Canvas: ${msg.metadata.canvas.title}]`,
+        });
+      }
+    }
+
+    if (segments.length === 0) continue;
+
+    const displayRole = role === "user" ? "User" : "ChatGPT";
+    const timestamp = msg?.create_time
+      ? new Date(msg.create_time * 1000).toLocaleString()
+      : null;
+    // Citation / image-group references, mirroring the mapping path.
+    const citeMap = {};
+    const imageGroupMap = {};
+    for (const ref of msg?.metadata?.content_references ?? []) {
+      if (ref.matched_text) {
+        if (ref.items?.length) citeMap[ref.matched_text] = ref.items;
+        if (
+          ref.type === "image_group" ||
+          ref.matched_text.includes("image_group")
+        ) {
+          imageGroupMap[ref.matched_text] = ref;
+        }
+      }
+    }
+    pushOrMerge(
+      {
+        role: displayRole,
+        segments,
+        citeMap,
+        imageGroupMap,
+        timestamp,
+      },
+      isThoughtMsg,
+    );
+  }
+
+  return messages;
+}
+
 export function linearize(mapping, includeImages, currentNodeId) {
   let path = [];
   const leafId = resolveActiveLeafNode(mapping, currentNodeId);
@@ -268,8 +483,20 @@ export function linearize(mapping, includeImages, currentNodeId) {
     ) {
       const segments = [];
       const parts = msg?.content?.parts ?? [];
+      // Tool-invocation payloads (content_type "code") are not user-visible
+      // prose, whether carried as standalone text or inside parts.
+      const isToolInvocation = msg?.content?.content_type === "code";
 
       for (const part of parts) {
+        if (isToolInvocation) {
+          if (!(
+            includeImages &&
+            part?.content_type === "image_asset_pointer" &&
+            part?.asset_pointer
+          )) {
+            continue;
+          }
+        }
         let partText = "";
         let isThoughtPart = isThoughtMsg;
 
@@ -316,11 +543,15 @@ export function linearize(mapping, includeImages, currentNodeId) {
         }
       }
 
-      // Handle standalone content.text (e.g. execution_output or plain text)
+      // Handle standalone content.text (e.g. execution_output or plain text).
+      // Tool-invocation payloads (content_type "code", e.g. Deep Research
+      // "/Deep Research App/start" args JSON) are not user-visible prose —
+      // the site renders a status card instead — so never dump them as text.
       if (
         typeof msg.content?.text === "string" &&
         msg.content.text.trim() &&
-        parts.length === 0
+        parts.length === 0 &&
+        msg.content?.content_type !== "code"
       ) {
         segments.push({ type: "text", content: msg.content.text.trim() });
       }
@@ -831,6 +1062,7 @@ export class ChatGPTParser extends ChatParser {
     const messages = [];
     for (const msg of apiMessages) {
       let content = "";
+      let thinking = "";
       for (const seg of msg.segments) {
         if (seg.type === "text") {
           content +=
@@ -843,7 +1075,7 @@ export class ChatGPTParser extends ChatParser {
             msg.imageGroupMap,
           );
           if (thoughtText) {
-            content += `<details><summary>Thought Process</summary>\n\n${thoughtText}\n\n</details>\n\n`;
+            thinking += (thinking ? "\n\n" : "") + thoughtText;
           }
         } else if (seg.type === "image") {
           const src = images[seg.fileId];
@@ -852,12 +1084,22 @@ export class ChatGPTParser extends ChatParser {
           }
         }
       }
-      content = content.trim();
-      if (content) {
+      let fullContent = "";
+      if (thinking) {
+        fullContent += `<think>\n${thinking}\n</think>\n\n`;
+      }
+      if (content.trim()) {
+        fullContent += content.trim();
+      }
+      fullContent = fullContent.trim();
+      if (fullContent) {
         const msgObj = {
           role: msg.role,
-          content: content,
+          content: fullContent,
         };
+        if (thinking) {
+          msgObj.thinking = thinking;
+        }
         if (msg.timestamp) {
           msgObj.timestamp = msg.timestamp;
         }
@@ -876,8 +1118,11 @@ export class ChatGPTParser extends ChatParser {
       Link: currentUrl,
       Model:
         convoData?.model_slug ||
-        document.querySelector('[data-testid="model-selector-dropdown"]')
-          ?.innerText ||
+        convoData?.default_model_slug ||
+        (typeof document !== "undefined" && document.querySelector
+          ? document.querySelector('[data-testid="model-selector-dropdown"]')
+              ?.innerText
+          : null) ||
         "ChatGPT",
       Method: method,
     };
@@ -948,11 +1193,22 @@ export class ChatGPTParser extends ChatParser {
           };
         }
 
-        const apiMessages = linearize(
-          result.data.mapping,
-          includeImages,
-          result.data.current_node,
-        );
+        // The backend returns either the legacy `mapping` tree or the newer
+        // `messages` array shape — support both.
+        let apiMessages = [];
+        if (result.data.mapping) {
+          apiMessages = linearize(
+            result.data.mapping,
+            includeImages,
+            result.data.current_node,
+          );
+        }
+        if (apiMessages.length === 0 && Array.isArray(result.data.messages)) {
+          apiMessages = linearizeMessagesArray(
+            result.data.messages,
+            includeImages,
+          );
+        }
         if (apiMessages.length > 0) {
           return this.formatApiResult(
             result.data,

package/ai/chatgpt_helper.js

@@ -84,16 +84,30 @@ if (!window.__chatgptHelperInjected) {
       let images = {};
       if (includeImages) {
         const fileIds = new Set();
-        for (const node of Object.values(data.mapping)) {
+        const collectImagePointer = (part) => {
+          if (
+            part &&
+            part.content_type === "image_asset_pointer" &&
+            part.asset_pointer
+          ) {
+            fileIds.add(part.asset_pointer.split("://")[1]);
+          }
+        };
+        // Legacy `mapping` tree shape…
+        for (const node of Object.values(data.mapping || {})) {
           const msg = node.message;
           if (msg && msg.content && Array.isArray(msg.content.parts)) {
             for (const part of msg.content.parts) {
-              if (
-                part &&
-                part.content_type === "image_asset_pointer" &&
-                part.asset_pointer
-              ) {
-                fileIds.add(part.asset_pointer.split("://")[1]);
+              collectImagePointer(part);
+            }
+          }
+        }
+        // …and the newer `messages` array shape.
+        if (Array.isArray(data.messages)) {
+          for (const msg of data.messages) {
+            if (msg && msg.content && Array.isArray(msg.content.parts)) {
+              for (const part of msg.content.parts) {
+                collectImagePointer(part);
               }
             }
           }