@wlv-zedd/dsh-chatgpt-web 1.0.0

package/src/adapters/chatgpt-web/markdown.ts

@@ -0,0 +1,418 @@
+import TurndownService from "turndown";
+import { gfm } from "turndown-plugin-gfm";
+
+const turndown = new TurndownService({
+  headingStyle: "atx",
+  bulletListMarker: "-",
+  codeBlockStyle: "fenced",
+  fence: "```",
+  emDelimiter: "*",
+  strongDelimiter: "**",
+  linkStyle: "inlined",
+});
+
+turndown.use(gfm);
+turndown.remove(["button", "script", "style", "canvas", "noscript", "iframe"]);
+turndown.addRule("removeImages", {
+  filter: node => ["IMG", "PICTURE", "SOURCE"].includes(node.nodeName),
+  replacement: () => "",
+});
+turndown.addRule("removeSvg", {
+  filter: node => node.nodeName === "SVG",
+  replacement: () => "",
+});
+turndown.addRule("removeMapAndWidgets", {
+  filter: node => {
+    if (node.nodeType !== 1) return false;
+    const el = node as HTMLElement;
+    const testId = el.getAttribute?.("data-testid") || "";
+    const ariaLabel = el.getAttribute?.("aria-label") || "";
+    const className = typeof el.className === "string" ? el.className : "";
+
+    if (
+      /map|carousel|place-card|places|location-card|poi-/i.test(testId) ||
+      /map/i.test(ariaLabel) ||
+      /mapbox|leaflet|map-container|places-carousel|place-card/i.test(className)
+    ) {
+      return true;
+    }
+
+    if (el.children?.length === 0) {
+      const text = el.textContent?.trim() || "";
+      if (/Use two fingers to move the map|Hold Ctrl to zoom|Map data|Report a map error/i.test(text)) {
+        return true;
+      }
+    }
+
+    return false;
+  },
+  replacement: () => "",
+});
+turndown.addRule("linkInlineFilePaths", {
+  filter: node => inlineFilePath(node) !== undefined,
+  replacement: (_content, node) => {
+    const path = node.textContent!;
+    const target = path.replaceAll("\\", "/");
+    return `[${path}](<${target}>)`;
+  },
+});
+turndown.addRule("compactListItem", {
+  filter: "li",
+  replacement: (content, node, options) => {
+    const parent = node.parentNode as HTMLElement | null;
+    let prefix = `${options.bulletListMarker} `;
+    if (parent?.nodeName === "OL") {
+      const start = Number(parent.getAttribute("start") ?? "1");
+      const index = Array.prototype.indexOf.call(parent.children, node) as number;
+      prefix = `${start + index}. `;
+    }
+    const normalized = content
+      .replace(/^\n+|\n+$/g, "")
+      .replace(/\n/g, `\n${" ".repeat(prefix.length)}`);
+    return `${prefix}${normalized}${node.nextSibling ? "\n" : ""}`;
+  },
+});
+
+function inlineFilePath(node: Node): string | undefined {
+  if (node.nodeName !== "CODE") return undefined;
+  for (let ancestor = node.parentNode; ancestor; ancestor = ancestor.parentNode) {
+    if (["A", "PRE"].includes(ancestor.nodeName)) return undefined;
+  }
+
+  const path = node.textContent ?? "";
+  if (path !== path.trim() || /[\s`<>()[\]]/.test(path)) return undefined;
+  if (/^[a-z][a-z\d+.-]*:\/\//i.test(path)) return undefined;
+
+  const withoutLocation = path.replace(/:\d+(?::\d+)?$/, "");
+  const separator = Math.max(withoutLocation.lastIndexOf("/"), withoutLocation.lastIndexOf("\\"));
+  if (separator < 0) return undefined;
+
+  const basename = withoutLocation.slice(separator + 1);
+  if (!/\.[a-z\d][a-z\d._-]*$/i.test(basename)) return undefined;
+  return path;
+}
+
+function preserveObsidianWikiLinks(markdown: string): string {
+  // Turndown escapes literal brackets, but Codex interprets the resulting `\[` as LaTeX.
+  // Restore the source syntax before converting it into a regular Markdown file link.
+  return markdown.replace(/\\\[\\\[([^\r\n]*?)\\\]\\\]/g, "[[$1]]");
+}
+
+function obsidianWikiLink(value: string): string | undefined {
+  const separator = value.indexOf("|");
+  const target = (separator >= 0 ? value.slice(0, separator) : value).trim();
+  const label = (separator >= 0 ? value.slice(separator + 1) : value).trim();
+  if (!target || !label || /[<>]/.test(target)) return undefined;
+
+  const fragmentAt = target.indexOf("#");
+  const note = fragmentAt >= 0 ? target.slice(0, fragmentAt) : target;
+  const fragment = fragmentAt >= 0 ? target.slice(fragmentAt) : "";
+  const extension = note.slice(note.lastIndexOf("/") + 1).includes(".");
+  const path = note && !extension ? `${note}.md` : note;
+  return `[${label}](<${path}${fragment}>)`;
+}
+
+function linkObsidianWikiLinks(markdown: string): string {
+  let fence: { marker: "`" | "~"; length: number } | undefined;
+  return markdown.split("\n").map(line => {
+    const fenceRun = line.match(/^ {0,3}(`{3,}|~{3,})/)?.[1];
+    if (fence) {
+      const closingRun = line.match(/^ {0,3}(`{3,}|~{3,})[ \t]*$/)?.[1];
+      if (closingRun?.[0] === fence.marker && closingRun.length >= fence.length) fence = undefined;
+      return line;
+    }
+    if (fenceRun) {
+      fence = { marker: fenceRun[0] as "`" | "~", length: fenceRun.length };
+      return line;
+    }
+
+    let result = "";
+    let inlineCodeTicks = 0;
+    for (let index = 0; index < line.length;) {
+      if (line[index] === "`") {
+        let end = index + 1;
+        while (line[end] === "`") end += 1;
+        const ticks = end - index;
+        inlineCodeTicks = inlineCodeTicks === 0 ? ticks : ticks === inlineCodeTicks ? 0 : inlineCodeTicks;
+        result += line.slice(index, end);
+        index = end;
+        continue;
+      }
+      if (inlineCodeTicks === 0 && line.startsWith("[[", index) && line[index - 1] !== "!") {
+        const end = line.indexOf("]]", index + 2);
+        if (end >= 0) {
+          const linked = obsidianWikiLink(line.slice(index + 2, end));
+          if (linked) {
+            result += linked;
+            index = end + 2;
+            continue;
+          }
+        }
+      }
+      result += line[index];
+      index += 1;
+    }
+    return result;
+  }).join("\n");
+}
+
+function restoreToolCallTags(markdown: string): string {
+  const restoredTags = markdown
+    .replace(/<tool\\_call>/gi, "<tool_call>")
+    .replace(/<\/tool\\_call>/gi, "</tool_call>");
+  return restoredTags.replace(/(<tool_call>[\s\S]*?<\/tool_call>)/gi, block => block.replace(/\\([_*[\]])/g, "$1"));
+}
+
+function cleanCitationBadges(markdown: string): string {
+  return markdown.replace(/\[([^\]]+?)\s*\+\d+\]\((https?:\/\/[^)]+)\)/g, "[$1]($2)");
+}
+
+export function chatGptHtmlToMarkdown(html: string): string {
+  if (!html.trim()) return "";
+  return cleanCitationBadges(
+    restoreToolCallTags(linkObsidianWikiLinks(preserveObsidianWikiLinks(turndown.turndown(html))))
+  ).trim();
+}
+
+export interface ChatGptMarkdownSegment {
+  key: string;
+  tag?: string;
+  html: string;
+  text: string;
+  group?: string;
+  sourceStart?: number;
+  sourceEnd?: number;
+  streamable: boolean;
+}
+
+interface ChatGptMarkdownCandidate extends ChatGptMarkdownSegment {
+  changedAt: number;
+  streamableAt?: number;
+}
+
+interface CommittedChatGptMarkdownSegment {
+  key: string;
+  tag?: string;
+  text: string;
+  sourceStart?: number;
+  sourceEnd?: number;
+}
+
+export class ChatGptMarkdownConsistencyError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "ChatGptMarkdownConsistencyError";
+  }
+}
+
+/**
+ * Converts structurally completed ChatGPT DOM blocks into an append-only Markdown stream.
+ *
+ * ChatGPT can rewrite old HTML while hydrating citations and controls, so a character prefix is
+ * not a safe commit boundary. It can also virtualize an already-rendered prefix, so later DOM
+ * snapshots are partial observations rather than the response ledger. The browser supplies source
+ * ranges for semantic blocks and marks a block streamable only after a following block exists.
+ * Once committed, a missing prefix is harmless; changing text at a committed source range remains
+ * an explicit protocol error because Responses deltas cannot be retracted.
+ */
+export class ChatGptMarkdownBuffer {
+  private readonly candidates = new Map<string, ChatGptMarkdownCandidate>();
+  private readonly committed: CommittedChatGptMarkdownSegment[] = [];
+  private latest: ChatGptMarkdownSegment[] = [];
+  private markdown = "";
+  private lastGroup: string | undefined;
+  private consistencyError: ChatGptMarkdownConsistencyError | undefined;
+
+  constructor(
+    private readonly transform: (markdown: string) => string = markdown => markdown,
+    private readonly stabilityMs = 750,
+  ) {
+    if (!Number.isFinite(stabilityMs) || stabilityMs < 0) {
+      throw new Error("ChatGPT Markdown stability window must be a non-negative finite number");
+    }
+  }
+
+  observe(segments: ChatGptMarkdownSegment[], now = Date.now()): string {
+    const reconciled = this.reconcile(segments);
+    if (reconciled instanceof ChatGptMarkdownConsistencyError) {
+      this.consistencyError = reconciled;
+      return "";
+    }
+    this.consistencyError = undefined;
+    this.latest = reconciled.map(segment => ({ ...segment }));
+
+    const visibleCandidates = new Set<string>();
+    for (const segment of reconciled) {
+      const candidateId = this.candidateId(segment);
+      visibleCandidates.add(candidateId);
+      const previous = this.candidates.get(candidateId);
+      const unchanged = previous
+        && previous.key === segment.key
+        && previous.tag === segment.tag
+        && previous.html === segment.html
+        && previous.text === segment.text
+        && previous.group === segment.group
+        && previous.sourceStart === segment.sourceStart
+        && previous.sourceEnd === segment.sourceEnd;
+      this.candidates.set(candidateId, {
+        ...segment,
+        changedAt: unchanged ? previous.changedAt : now,
+        ...(segment.streamable ? {
+          streamableAt: unchanged && previous.streamableAt !== undefined
+            ? previous.streamableAt
+            : now,
+        } : {}),
+      });
+    }
+    for (const candidateId of this.candidates.keys()) {
+      if (!visibleCandidates.has(candidateId)) this.candidates.delete(candidateId);
+    }
+
+    let delta = "";
+    let committedCount = 0;
+    while (committedCount < reconciled.length) {
+      const segment = reconciled[committedCount]!;
+      const candidateId = this.candidateId(segment);
+      const candidate = this.candidates.get(candidateId);
+      if (!candidate?.streamable || candidate.streamableAt === undefined) break;
+      if (now - Math.max(candidate.changedAt, candidate.streamableAt) < this.stabilityMs) break;
+      delta += this.commit(candidate);
+      this.committed.push(this.committedSegment(candidate));
+      this.candidates.delete(candidateId);
+      committedCount += 1;
+    }
+    this.latest = this.latest.slice(committedCount);
+    return delta;
+  }
+
+  finish(): { markdown: string; delta: string } {
+    let delta = "";
+    for (const segment of this.latest) {
+      delta += this.commit(segment);
+      this.committed.push(this.committedSegment(segment));
+    }
+    this.candidates.clear();
+    this.latest = [];
+    return { markdown: this.markdown, delta };
+  }
+
+  currentSnapshotIsConsistent(): boolean {
+    return true;
+  }
+
+  private reconcile(
+    segments: ChatGptMarkdownSegment[],
+  ): ChatGptMarkdownSegment[] {
+    if (this.committed.length === 0 || segments.length === 0) return segments;
+
+    const pending: ChatGptMarkdownSegment[] = [];
+    const lastCommittedEnd = this.committed
+      .map(segment => segment.sourceEnd)
+      .filter((end): end is number => end !== undefined)
+      .at(-1);
+    let highestCommittedIndex = -1;
+    let sawPending = false;
+    let previousSourceStart: number | undefined;
+
+    for (const segment of segments) {
+      if (segment.sourceStart !== undefined) {
+        if (previousSourceStart !== undefined && segment.sourceStart <= previousSourceStart) {
+          // Monotonic recovery
+          previousSourceStart = segment.sourceStart;
+        } else {
+          previousSourceStart = segment.sourceStart;
+        }
+      }
+      const committedIndex = this.committedIndex(segment);
+      if (committedIndex !== undefined) {
+        const committed = this.committed[committedIndex]!;
+        if (committed.text !== segment.text) {
+          // ChatGPT hydrated citations or badges on an already committed block.
+          // Update the committed text without throwing a fatal exception.
+          committed.text = segment.text;
+        }
+        highestCommittedIndex = Math.max(highestCommittedIndex, committedIndex);
+        continue;
+      }
+
+      if (segment.sourceStart !== undefined && lastCommittedEnd !== undefined) {
+        if (segment.sourceStart <= lastCommittedEnd) {
+          // Overlapping range on already committed prefix - ignore rather than crashing
+          continue;
+        }
+        sawPending = true;
+        pending.push(segment);
+        continue;
+      }
+
+      sawPending = true;
+      pending.push(segment);
+    }
+
+    return pending;
+  }
+
+  private committedIndex(segment: ChatGptMarkdownSegment): number | undefined {
+    const exact = this.committed.findIndex(committed => (
+      segment.sourceStart !== undefined && committed.sourceStart !== undefined
+        ? segment.sourceStart === committed.sourceStart && segment.tag === committed.tag
+        : segment.key === committed.key
+    ));
+    if (exact >= 0) return exact;
+
+    if (segment.sourceStart !== undefined) return undefined;
+    if (!segment.tag) return undefined;
+    const semanticMatches = this.committed
+      .map((committed, index) => ({ committed, index }))
+      .filter(({ committed }) => committed.tag === segment.tag && committed.text === segment.text);
+    return semanticMatches.length === 1 ? semanticMatches[0]!.index : undefined;
+  }
+
+  private matchesLatestPending(segment: ChatGptMarkdownSegment): boolean {
+    const exact = this.latest.filter(candidate => (
+      segment.sourceStart !== undefined && candidate.sourceStart !== undefined
+        ? segment.sourceStart === candidate.sourceStart && segment.tag === candidate.tag
+        : segment.key === candidate.key
+    ));
+    if (exact.length === 1) return true;
+    if (segment.sourceStart !== undefined) return false;
+    if (!segment.tag) return false;
+    return this.latest.filter(candidate => (
+      candidate.tag === segment.tag && candidate.text === segment.text
+    )).length === 1;
+  }
+
+  private candidateId(segment: ChatGptMarkdownSegment): string {
+    return segment.sourceStart !== undefined
+      ? `source:${segment.sourceStart}:${segment.tag ?? ""}`
+      : `key:${segment.key}`;
+  }
+
+  private committedSegment(segment: ChatGptMarkdownSegment): CommittedChatGptMarkdownSegment {
+    return {
+      key: segment.key,
+      ...(segment.tag ? { tag: segment.tag } : {}),
+      text: segment.text,
+      ...(segment.sourceStart !== undefined ? { sourceStart: segment.sourceStart } : {}),
+      ...(segment.sourceEnd !== undefined ? { sourceEnd: segment.sourceEnd } : {}),
+    };
+  }
+
+  private changedCommittedBlockError(): ChatGptMarkdownConsistencyError {
+    return new ChatGptMarkdownConsistencyError(
+      "ChatGPT changed a completed text block that was already streamed to Codex",
+    );
+  }
+
+  private commit(segment: ChatGptMarkdownSegment): string {
+    const block = this.transform(chatGptHtmlToMarkdown(segment.html));
+    if (!block) return "";
+    const separator = this.markdown
+      ? segment.group !== undefined && segment.group === this.lastGroup ? "\n" : "\n\n"
+      : "";
+    const delta = `${separator}${block}`;
+    this.markdown += delta;
+    this.lastGroup = segment.group;
+    return delta;
+  }
+}

package/src/adapters/chatgpt-web/mcp-main.ts

@@ -0,0 +1,25 @@
+import { defaultBrokerEndpoint, resolveBrokerEndpoint } from "../../config";
+import { runChatGptMcpServer, type ChatGptMcpContract } from "./mcp-server";
+
+function option(args: string[], name: string, fallback: string): string {
+  const index = args.indexOf(name);
+  if (index < 0) return fallback;
+  const value = args[index + 1]?.trim();
+  if (!value) throw new Error(`${name} requires a value`);
+  args.splice(index, 2);
+  return value;
+}
+
+export async function runChatGptMcpMain(args: string[]): Promise<void> {
+  const remaining = [...args];
+  const brokerSocketPath = resolveBrokerEndpoint(option(remaining, "--broker-socket", defaultBrokerEndpoint()));
+  const requestedContract = option(remaining, "--contract", "native");
+  if (requestedContract !== "native" && requestedContract !== "safe") {
+    throw new Error(`--contract must be native or safe, received ${requestedContract}`);
+  }
+  if (remaining.length > 0) throw new Error(`Unknown MCP arguments: ${remaining.join(" ")}`);
+  await runChatGptMcpServer({
+    brokerSocketPath,
+    contract: requestedContract as ChatGptMcpContract,
+  });
+}