decant-core 1.4.0 → 1.6.0

package/ai/chatgpt.js

@@ -759,6 +759,19 @@ export class ChatGPTParser extends ChatParser {
           }
         });
 
+        clone
+          .querySelectorAll(
+            '[data-testid="webpage-citation-pill"], [data-testid*="citation-pill"]',
+          )
+          .forEach((pill) => {
+            pill.querySelectorAll("img").forEach((img) => img.remove());
+            pill.querySelectorAll("span").forEach((span) => {
+              if (/^\+\d+$/.test(span.textContent.trim())) {
+                span.remove();
+              }
+            });
+          });
+
         noiseSelectors.forEach((selector) => {
           clone.querySelectorAll(selector).forEach((node) => node.remove());
         });
@@ -783,11 +796,13 @@ export class ChatGPTParser extends ChatParser {
   }
 
   extractMountedMessages() {
-    const articles = Array.from(document.querySelectorAll("article"));
+    const turns = getConversationTurns(document);
     const containers =
-      articles.length > 0
-        ? articles
-        : Array.from(document.querySelectorAll("[data-message-author-role]"));
+      turns.length > 0
+        ? turns
+        : Array.from(
+            document.querySelectorAll("article, [data-message-author-role]"),
+          );
 
     return containers
       .map((container) => this.extractMessage(container))

package/ai/gemini.js

@@ -1,5 +1,6 @@
 import { ChatParser } from "./base.js";
 import { convertToMarkdown } from "../utils/html-to-markdown.js";
+import { normalizeLatexMath } from "../utils/latex-math.js";
 
 const GEMINI_RPC_ID = "hNvQHb";
 const DEFAULT_BARD_PATH = "/_/BardChatUi";
@@ -94,23 +95,7 @@ export class GeminiParser extends ChatParser {
 
     const mode = options.parserMode || "auto";
 
-    // 1. Prefer DOM extraction first when on a live page with conversation containers
-    if (typeof document !== "undefined" && document.querySelector) {
-      const hasDomMessages = document.querySelector(
-        ".conversation-container, user-query, model-response, deep-research-immersive-panel",
-      );
-      if (hasDomMessages && mode !== "api") {
-        const domResult = this.parseFromDom(currentUrl, options);
-        if (domResult && domResult.messages && domResult.messages.length > 0) {
-          console.log(
-            `[Gemini Parser] Successfully parsed ${domResult.messages.length} messages from DOM`,
-          );
-          return domResult;
-        }
-      }
-    }
-
-    // 2. Attempt API / RPC extraction if DOM parsing didn't find messages or mode is API
+    // 1. Attempt API / RPC extraction first when not explicitly in 'dom' mode
     if (mode !== "dom" && typeof fetch === "function") {
       try {
         const convoId = this.getConversationId(currentUrl);
@@ -149,7 +134,7 @@ export class GeminiParser extends ChatParser {
       }
     }
 
-    // Fall back to robust DOM parsing
+    // 2. Fall back to robust DOM parsing
     return this.parseFromDom(currentUrl, options);
   }
 
@@ -175,14 +160,17 @@ export class GeminiParser extends ChatParser {
       `&_reqid=${encodeURIComponent(reqId)}` +
       `&rt=c`;
 
+    const formattedConvoId = convoId.startsWith("c_")
+      ? convoId
+      : `c_${convoId}`;
     const allItems = [];
     let cursor = null;
     let pageCount = 0;
 
-    while (pageCount < 50) {
+    while (pageCount < 250) {
       pageCount++;
       const payloadArg = JSON.stringify([
-        `c_${convoId}`,
+        formattedConvoId,
         100,
         cursor,
         1,
@@ -227,11 +215,11 @@ export class GeminiParser extends ChatParser {
       const continueCursor = payload[1] || null;
 
       if (items.length > 0) {
-        // Items are in reverse chronological order from API
+        // Items are in reverse chronological order from API; reverse to maintain oldest-first order
         allItems.unshift(...items.slice().reverse());
       }
 
-      if (!continueCursor || items.length < 100) {
+      if (!continueCursor) {
         break;
       }
       cursor = continueCursor;
@@ -241,7 +229,158 @@ export class GeminiParser extends ChatParser {
       return null;
     }
 
+    return this.formatApiResult(allItems, currentUrl, options);
+  }
+
+  enrichMessagesWithDomAttachments(messages) {
+    try {
+      if (typeof document === "undefined" || !document.querySelectorAll) {
+        return messages;
+      }
+      const parentContainers = document.querySelectorAll(
+        ".conversation-container",
+      );
+      const domTurns = [];
+
+      if (parentContainers.length > 0) {
+        parentContainers.forEach((container, idx) => {
+          const rawId = (container.id || "").replace(/^r_/, "");
+          const fileElements = container.querySelectorAll(
+            "user-query-file-preview, .file-preview, .attachment-preview, [data-testid='file-preview']",
+          );
+          const fileNames = [];
+          fileElements.forEach((fe) => {
+            const name = (fe.textContent || "").trim();
+            if (name && !fileNames.includes(name)) fileNames.push(name);
+          });
+
+          const userQuery = container.querySelector("user-query") || container;
+          const clone = userQuery.cloneNode(true);
+          clone
+            .querySelectorAll(
+              "user-query-file-preview, .file-preview, .attachment-preview, button, .edit-button, model-response",
+            )
+            .forEach((el) => el.remove());
+          const queryText = (
+            clone.innerText !== undefined
+              ? clone.innerText
+              : clone.textContent || ""
+          )
+            .replace(/^You said\s*/i, "")
+            .trim();
+
+          if (fileNames.length > 0) {
+            domTurns.push({
+              turnId: rawId,
+              queryText,
+              domIndex: idx,
+              totalDom: parentContainers.length,
+              fileNames,
+            });
+          }
+        });
+      } else {
+        const userQueries = document.querySelectorAll("user-query");
+        userQueries.forEach((uq, idx) => {
+          const fileElements = uq.querySelectorAll(
+            "user-query-file-preview, .file-preview, .attachment-preview, [data-testid='file-preview']",
+          );
+          const fileNames = [];
+          fileElements.forEach((fe) => {
+            const name = (fe.textContent || "").trim();
+            if (name && !fileNames.includes(name)) fileNames.push(name);
+          });
+
+          const clone = uq.cloneNode(true);
+          clone
+            .querySelectorAll(
+              "user-query-file-preview, .file-preview, .attachment-preview, button, .edit-button",
+            )
+            .forEach((el) => el.remove());
+          const queryText = (
+            clone.innerText !== undefined
+              ? clone.innerText
+              : clone.textContent || ""
+          )
+            .replace(/^You said\s*/i, "")
+            .trim();
+
+          if (fileNames.length > 0) {
+            domTurns.push({
+              domIndex: idx,
+              queryText,
+              totalDom: userQueries.length,
+              fileNames,
+            });
+          }
+        });
+      }
+
+      if (domTurns.length === 0) return messages;
+
+      const userMessages = messages.filter((m) => m.role === "User");
+
+      for (const domTurn of domTurns) {
+        let matchedMsg = null;
+
+        // 1. Match by exact turnId if present
+        if (domTurn.turnId) {
+          matchedMsg = userMessages.find((m) => m.turnId === domTurn.turnId);
+        }
+
+        // 2. Fallback: match by queryText if present and non-empty
+        if (!matchedMsg && domTurn.queryText) {
+          matchedMsg = userMessages.find(
+            (m) =>
+              m.content &&
+              (m.content.includes(domTurn.queryText) ||
+                domTurn.queryText.includes(m.content)),
+          );
+        }
+
+        // 3. Fallback: match by trailing turn ordinal (mounted DOM elements align with latest turns)
+        if (!matchedMsg && typeof domTurn.domIndex === "number") {
+          const offset = userMessages.length - domTurn.totalDom;
+          const targetIndex =
+            offset >= 0 ? offset + domTurn.domIndex : domTurn.domIndex;
+          matchedMsg = userMessages[targetIndex];
+        }
+
+        if (matchedMsg) {
+          const attachBlock =
+            `\n\n**Attachments:**\n` +
+            domTurn.fileNames.map((fn) => `- ${fn}`).join("\n");
+          if (!matchedMsg.content.includes("**Attachments:**")) {
+            matchedMsg.content = matchedMsg.content
+              ? `${matchedMsg.content}${attachBlock}`
+              : `**Attachments:**\n` +
+                domTurn.fileNames.map((fn) => `- ${fn}`).join("\n");
+          }
+        }
+      }
+    } catch (e) {
+      console.warn(
+        "[Gemini Parser] Failed to enrich messages with DOM attachments:",
+        e,
+      );
+    } finally {
+      // Clean internal turnId tracking before returning messages
+      for (const m of messages) {
+        delete m.turnId;
+      }
+    }
+
+    return messages;
+  }
+
+  formatApiResult(allItems, currentUrl, options = {}) {
+    if (!Array.isArray(allItems) || allItems.length === 0) {
+      return null;
+    }
+
     const messages = this.convertApiItemsToMessages(allItems, options);
+    this.enrichMessagesWithDomAttachments(messages);
+
     let title = this.extractTitleFromPage();
     if (!title || title === "Gemini Conversation") {
       const firstUserMsg = messages.find((m) => m.role === "User");
@@ -296,17 +435,28 @@ export class GeminiParser extends ChatParser {
     return null;
   }
 
+  extractTurnId(item) {
+    try {
+      const raw = item[0]?.[1] || item[1]?.[1] || "";
+      return String(raw).replace(/^r_/, "");
+    } catch {
+      return "";
+    }
+  }
+
   convertApiItemsToMessages(items, options = {}) {
     const messages = [];
 
     for (const item of items) {
       if (!Array.isArray(item)) continue;
 
+      const turnId = this.extractTurnId(item);
       const userText = this.findUserTextInApiItem(item);
       if (userText) {
         messages.push({
           role: "User",
           content: userText.trim(),
+          turnId,
         });
       }
 
@@ -314,7 +464,8 @@ export class GeminiParser extends ChatParser {
       if (modelText) {
         messages.push({
           role: "Model",
-          content: modelText.trim(),
+          content: normalizeLatexMath(modelText.trim()),
+          turnId,
         });
       }
     }
@@ -324,6 +475,7 @@ export class GeminiParser extends ChatParser {
 
   findUserTextInApiItem(item) {
     try {
+      if (typeof item[2]?.[0]?.[0] === "string") return item[2][0][0];
       if (typeof item[2]?.[0] === "string") return item[2][0];
       if (typeof item[1]?.[0] === "string" && !Array.isArray(item[1][0]))
         return item[1][0];
@@ -336,6 +488,25 @@ export class GeminiParser extends ChatParser {
 
   findModelTextInApiItem(item) {
     try {
+      // 1. Candidate responses in item[3]
+      if (Array.isArray(item[3])) {
+        const candidates = Array.isArray(item[3][0]) ? item[3][0] : item[3];
+        for (const cand of candidates) {
+          if (!Array.isArray(cand)) continue;
+          // Shape: ["rc_...", ["markdown text", ...], ...]
+          if (Array.isArray(cand[1]) && typeof cand[1][0] === "string") {
+            return cand[1][0];
+          }
+          if (typeof cand[1] === "string") {
+            return cand[1];
+          }
+          if (typeof cand[0] === "string" && cand[0].length > 50) {
+            return cand[0];
+          }
+        }
+      }
+
+      // 2. Fallback candidate in item[1]
       if (Array.isArray(item[1])) {
         const candidate = item[1][0];
         if (typeof candidate === "string") return candidate;

package/ai/perplexity.js

@@ -1,5 +1,191 @@
 import { ChatParser } from "./base.js";
 import { convertToMarkdown } from "../utils/html-to-markdown.js";
+import { normalizeLatexMath } from "../utils/latex-math.js";
+
+export function getThreadSlug(url) {
+  if (!url || typeof url !== "string") return null;
+  const match = url.match(/\/search\/([a-zA-Z0-9_-]+)/);
+  if (match) return match[1];
+  const matchPage = url.match(/\/page\/([a-zA-Z0-9_-]+)/);
+  if (matchPage) return matchPage[1];
+  return null;
+}
+
+export function extractAssistantAnswer(entry) {
+  if (!entry) return "";
+  // 1. Check blocks -> plan_block.goals for markdown answer
+  if (Array.isArray(entry.blocks)) {
+    for (const b of entry.blocks) {
+      if (b.intended_usage === "plan" && b.plan_block?.goals) {
+        for (const g of b.plan_block.goals) {
+          if (g.description && g.description.length > 100) {
+            return g.description.trim();
+          }
+        }
+      }
+      if (b.answer_block?.answer) {
+        return b.answer_block.answer.trim();
+      }
+      if (b.markdown_block?.markdown) {
+        return b.markdown_block.markdown.trim();
+      }
+    }
+  }
+  // 2. Check direct answer or text fields
+  if (entry.answer && typeof entry.answer === "string") {
+    return entry.answer.trim();
+  }
+  if (entry.text && typeof entry.text === "string") {
+    try {
+      const parsed = JSON.parse(entry.text);
+      if (parsed.answer) return parsed.answer.trim();
+    } catch {
+      return entry.text.trim();
+    }
+  }
+  return "";
+}
+
+export function formatApiResult(threadData, currentUrl, fallbackTitle) {
+  const title =
+    threadData.thread_title ||
+    threadData.first_entry?.query_str ||
+    fallbackTitle ||
+    "Perplexity Search";
+
+  const allEntries = [];
+  if (threadData.first_entry) {
+    allEntries.push(threadData.first_entry);
+  }
+  if (Array.isArray(threadData.entries)) {
+    allEntries.push(...threadData.entries);
+  }
+  if (threadData.latest_entry) {
+    allEntries.push(threadData.latest_entry);
+  }
+
+  // Deduplicate entries by uuid / backend_uuid
+  const seenUuids = new Set();
+  const uniqueEntries = [];
+  for (const entry of allEntries) {
+    const id = entry.uuid || entry.backend_uuid || entry.query_str;
+    if (id && !seenUuids.has(id)) {
+      seenUuids.add(id);
+      uniqueEntries.push(entry);
+    }
+  }
+
+  const messages = [];
+  for (const entry of uniqueEntries) {
+    const prompt = (entry.query_str || "").trim();
+    const timestamp =
+      entry.entry_created_datetime || entry.updated_datetime || undefined;
+
+    if (prompt) {
+      const userMsg = { role: "User", content: prompt };
+      if (timestamp) userMsg.timestamp = timestamp;
+      messages.push(userMsg);
+    }
+
+    const rawAnswer = extractAssistantAnswer(entry);
+    if (rawAnswer) {
+      const normalizedAnswer = normalizeLatexMath(rawAnswer);
+      const assistantMsg = { role: "Perplexity", content: normalizedAnswer };
+      if (timestamp) assistantMsg.timestamp = timestamp;
+      messages.push(assistantMsg);
+    }
+  }
+
+  const effectiveUrl =
+    currentUrl ||
+    (threadData.first_entry?.thread_url_slug
+      ? `https://www.perplexity.ai/search/${threadData.first_entry.thread_url_slug}`
+      : "");
+
+  const metadata = {
+    Source: "Perplexity",
+    Date: new Date().toLocaleString(),
+    Link: effectiveUrl,
+    Method: "API",
+  };
+
+  return { title, messages, url: effectiveUrl, metadata };
+}
+
+export async function getPerplexityAccount() {
+  if (typeof window === "undefined") return null;
+
+  const isUuid = (str) =>
+    typeof str === "string" &&
+    /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(
+      str.trim(),
+    );
+
+  // 1. Check sessionStorage ('pplx-active-account')
+  try {
+    const sessionAcc = window.sessionStorage?.getItem("pplx-active-account");
+    if (sessionAcc && isUuid(sessionAcc)) return sessionAcc.trim();
+  } catch {
+    // Ignore storage errors
+  }
+
+  // 2. Check localStorage ('pplx-last-active-account')
+  try {
+    const localAcc = window.localStorage?.getItem("pplx-last-active-account");
+    if (localAcc && isUuid(localAcc)) return localAcc.trim();
+  } catch {
+    // Ignore storage errors
+  }
+
+  // 3. Scan keys in sessionStorage and localStorage for pplx account UUIDs
+  try {
+    for (const storage of [window.sessionStorage, window.localStorage]) {
+      if (!storage) continue;
+      for (let i = 0; i < storage.length; i++) {
+        const key = storage.key(i);
+        if (key && (key.includes("pplx") || key.includes("account"))) {
+          const val = storage.getItem(key);
+          if (val && isUuid(val)) return val.trim();
+        }
+      }
+    }
+  } catch {
+    // Ignore storage scanning errors
+  }
+
+  // 4. Check document.cookie if available
+  try {
+    const match = document.cookie?.match(
+      /(?:pplx-last-active-account|pplx-active-account)=([0-9a-f-]{36})/i,
+    );
+    if (match && isUuid(match[1])) return match[1].trim();
+  } catch {
+    // Ignore cookie errors
+  }
+
+  // 5. Fallback: query https://www.perplexity.ai/api/auth/linked-accounts using session cookies
+  try {
+    if (typeof fetch === "function") {
+      const res = await fetch(
+        "https://www.perplexity.ai/api/auth/linked-accounts",
+        {
+          method: "GET",
+          credentials: "include",
+          headers: { Accept: "application/json" },
+        },
+      );
+      if (res.ok) {
+        const data = await res.json();
+        const accountId = data?.accounts?.[0]?.user_id;
+        if (accountId && isUuid(accountId)) return accountId.trim();
+      }
+    }
+  } catch {
+    // Ignore linked-accounts network error
+  }
+
+  return null;
+}
 
 export class PerplexityParser extends ChatParser {
   name = "Perplexity";
@@ -7,7 +193,38 @@ export class PerplexityParser extends ChatParser {
     return url.includes("perplexity.ai");
   }
 
-  async parse() {
+  async fetchThread(slug) {
+    const accountId = await getPerplexityAccount();
+    const url = `https://www.perplexity.ai/rest/thread/${slug}?with_parent_info=true&with_schematized_response=true&version=2.18&source=default&limit=100&offset=0&from_first=true`;
+    const headers = {
+      Accept: "application/json",
+      "x-app-apiclient": "default",
+      "x-app-apiversion": "2.18",
+    };
+    if (accountId) {
+      headers["x-pplx-account"] = accountId;
+    }
+
+    const response = await fetch(url, {
+      method: "GET",
+      credentials: "include",
+      headers,
+    });
+    if (!response.ok) {
+      throw new Error(
+        `Perplexity API request failed with status ${response.status}`,
+      );
+    }
+    return response.json();
+  }
+
+  async parse(options = {}) {
+    const parserMode = options.parserMode || "prefer_api";
+    const currentUrl =
+      typeof window !== "undefined" && window.location
+        ? window.location.href || ""
+        : "";
+
     const rawTitle =
       document.querySelector(".share-title-section h1")?.textContent ||
       document.querySelector("h1")?.textContent ||
@@ -15,6 +232,25 @@ export class PerplexityParser extends ChatParser {
       "Perplexity Search";
     const title = rawTitle.trim().replace(/\s+/g, " ");
 
+    // 1. Attempt API-first extraction when in prefer_api mode
+    if (parserMode !== "prefer_dom") {
+      const slug = getThreadSlug(currentUrl);
+      if (slug) {
+        try {
+          const apiData = await this.fetchThread(slug);
+          if (apiData && (apiData.entries?.length || apiData.first_entry)) {
+            return formatApiResult(apiData, currentUrl, title);
+          }
+        } catch (err) {
+          console.warn(
+            "[AI Exporter] Perplexity API fetch failed, falling back to DOM:",
+            err,
+          );
+        }
+      }
+    }
+
+    // 2. DOM extraction fallback
     const messages = [];
 
     // Perplexity Container
@@ -23,6 +259,7 @@ export class PerplexityParser extends ChatParser {
 
     // Candidates for User Messages
     const userSelectors = [
+      ".group\\/user-bubble",
       "h1.group\\/query",
       ".group\\/query",
       ".whitespace-pre-line.text-pretty",
@@ -40,43 +277,58 @@ export class PerplexityParser extends ChatParser {
     const elements = threadContainer.querySelectorAll(selectorString);
 
     // Helper to determine role
-    const getRole = (el) => {
+    const isUser = (el) => {
       for (const s of userSelectors) {
-        if (el.matches(s)) return "User";
+        if (el.matches(s)) return true;
       }
+      return false;
+    };
+
+    const isAssistant = (el) => {
       for (const s of assistantSelectors) {
-        if (el.matches(s)) return "Perplexity";
+        if (el.matches(s)) return true;
       }
-      return "Unknown";
+      return false;
     };
 
-    const seenText = new Set();
-
     elements.forEach((el) => {
-      // Perplexity nests things. Avoid duplicates if we selected a parent and a child.
-      // Also avoid "Related" section queries if possible (usually separate container, check parents?)
-
       // Check if inside "related" or "sources"
       if (el.closest('[class*="related"], [class*="sources"]')) return;
 
-      const role = getRole(el);
-      let text = convertToMarkdown(el);
-      text = text.trim();
+      if (isUser(el)) {
+        if (el.parentElement?.closest(".group\\/user-bubble, .group\\/query")) {
+          return;
+        }
+
+        // Extract timestamp if present in bubble
+        const timeEl = el.querySelector(
+          ".text-tertiary, [class*='text-tertiary']",
+        );
+        const timestamp = timeEl ? timeEl.textContent.trim() : undefined;
 
-      if (!text || seenText.has(text)) return;
+        // Clean user text container
+        const textEl = el.querySelector(".whitespace-pre-line") || el;
+        const text = convertToMarkdown(textEl).trim();
+        if (!text) return;
 
-      // Perplexity specific cleanup
-      // Remove "Sources" label text if it gets captured?
-      // Usually .prose contains the markdown answer.
+        const msgObj = { role: "User", content: text };
+        if (timestamp) {
+          msgObj.timestamp = timestamp;
+        }
+        messages.push(msgObj);
+      } else if (isAssistant(el)) {
+        // Avoid duplicate nested assistant containers (e.g. .prose inside div[id^="markdown-content-"])
+        if (el.parentElement?.closest('div[id^="markdown-content-"], .prose')) {
+          return;
+        }
 
-      seenText.add(text);
-      messages.push({ role, content: text });
+        const text = convertToMarkdown(el).trim();
+        if (!text) return;
+
+        messages.push({ role: "Perplexity", content: text });
+      }
     });
 
-    const currentUrl =
-      typeof window !== "undefined" && window.location
-        ? window.location.href || ""
-        : "";
     const metadata = {
       Source: "Perplexity",
       Date: new Date().toLocaleString(),

package/package.json

@@ -1,8 +1,11 @@
 {
   "name": "decant-core",
-  "version": "1.4.0",
+  "version": "1.6.0",
   "description": "Shared AI chat platform parsers and detection for Covai browser extensions.",
   "type": "module",
+  "engines": {
+    "node": ">=26.0.0"
+  },
   "main": "ai/index.js",
   "exports": {
     ".": "./ai/index.js",