mixdog 1.1.2 → 1.2.1

package/src/runtime/agent/orchestrator/session/compaction-archive.mjs

@@ -0,0 +1,253 @@
+/**
+ * Display-only archive of the conversation a compaction dropped.
+ *
+ * One JSONL sidecar per session next to its record
+ * (`<id>.compaction-archive.jsonl`), one line per compaction segment, oldest
+ * first. A segment holds the transcript ITEMS (the pane's display shape, never
+ * model messages) its compaction removed from the model context, followed by
+ * the "Conversation compacted" divider. Each line opens with a fixed header
+ * (`{"v":1,"seq":N,"id":"segN","prev":…,"at":…,"count":…,"bytes":…,"items":[…]}`)
+ * so a reader can pick segments by number without parsing their items.
+ *
+ * Nothing here ever reaches `session.messages` or a provider payload: the
+ * archive is read only by the stored-transcript projection and the paged
+ * history loader. Leaf module (no store.mjs, no config) so the cold catalog
+ * path can read it; the writer lives in compaction-archive-capture.mjs.
+ */
+import { unlinkSync } from 'node:fs';
+import { open } from 'node:fs/promises';
+import { join } from 'node:path';
+
+export const COMPACTION_ARCHIVE_SUFFIX = '.compaction-archive.jsonl';
+/** The one label every compaction boundary row carries (persisted and live),
+ *  translated by the desktop as the app's compaction wording. */
+export const COMPACTION_DIVIDER_LABEL = 'Conversation compacted';
+/** Per-session cap; the oldest segments are dropped past it. */
+export const COMPACTION_ARCHIVE_MAX_BYTES = 32 * 1024 * 1024;
+
+const SESSION_ID = /^[A-Za-z0-9_-]+$/;
+const SEGMENT_HEAD = /^\{"v":1,"seq":(\d+),"id":"[^"]*","prev":(null|"[^"]*"),"at":(\d+),"count":(\d+),"bytes":(\d+),/;
+
+/** The archive sidecar of `sessionId` inside a sessions directory, or null
+ *  for an id that is not a stored session id. */
+export function compactionArchivePath(storeDir, sessionId) {
+  const id = String(sessionId || '');
+  if (!SESSION_ID.test(id)) return null;
+  return join(storeDir, `${id}${COMPACTION_ARCHIVE_SUFFIX}`);
+}
+
+/** The visible archive a session record names: segments numbered in
+ *  (after, before). A clear hides everything archived before it. Null when
+ *  nothing is visible. */
+export function compactionArchiveRange(session) {
+  const archive = session?.compaction?.archive;
+  if (!archive || typeof archive !== 'object') return null;
+  const before = Number(archive.nextSeq);
+  const after = Math.max(0, Math.floor(Number(archive.clearedThrough) || 0));
+  if (!Number.isInteger(before) || before - 1 <= after) return null;
+  return { after, before };
+}
+
+/** A segment line's header fields, or null for a line that is not one. */
+export function compactionSegmentHead(line) {
+  const match = SEGMENT_HEAD.exec(String(line || ''));
+  if (!match) return null;
+  return {
+    seq: Number(match[1]),
+    id: `seg${match[1]}`,
+    prev: match[2] === 'null' ? null : match[2].slice(1, -1),
+    at: Number(match[3]),
+    count: Number(match[4]),
+    bytes: Number(match[5]),
+  };
+}
+
+/** One segment as its JSONL line (no trailing newline). */
+export function compactionSegmentLine({ seq, prev, at, items }) {
+  const body = JSON.stringify(items);
+  return `{"v":1,"seq":${seq},"id":"seg${seq}","prev":${prev == null ? 'null' : JSON.stringify(`seg${prev}`)},"at":${Math.max(0, Math.floor(Number(at) || 0))},"count":${items.length},"bytes":${Buffer.byteLength(body)},"items":${body}}`;
+}
+
+function segmentItems(text) {
+  try {
+    const items = JSON.parse(text)?.items;
+    return Array.isArray(items) ? items : [];
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * One page from a segment list (oldest → newest heads; `load(entry)` yields a
+ * segment's items): segments numbered in (after, before), newest first,
+ * skipping the `skip` newest items a reader already holds and returning at
+ * most `limit` items above them, in display order. Item counts come from
+ * the segment headers: a segment wholly inside the skipped rows or past the
+ * page is never read or parsed. `hasOlder` says whether older items remain.
+ */
+async function pageFromSegments(
+  entries,
+  load,
+  { after = 0, before = Number.POSITIVE_INFINITY, skip = 0, limit = 0 } = {}
+) {
+  const held = Math.max(0, Math.floor(Number(skip) || 0));
+  const wanted = Math.max(0, Math.floor(Number(limit) || 0));
+  const parts = [];
+  // Items newer than this entry, counted from the newest.
+  let newer = 0;
+  let more = false;
+  for (let index = entries.length - 1; index >= 0; index -= 1) {
+    const { head } = entries[index];
+    if (head.seq <= after || head.seq >= before || head.count < 1) continue;
+    if (newer >= held + wanted) {
+      more = true;
+      break;
+    }
+    const count = head.count;
+    if (newer + count > held) {
+      // This segment holds rows of the page: newest-first offsets
+      // [max(held, newer), min(held + wanted, newer + count)) of it.
+      const items = await load(entries[index]);
+      const fromEnd = Math.max(0, held - newer);
+      const take = Math.min(held + wanted, newer + count) - Math.max(held, newer);
+      const end = items.length - fromEnd;
+      parts.push(items.slice(Math.max(0, end - take), Math.max(0, end)));
+      if (newer + count > held + wanted) more = true;
+    }
+    newer += count;
+  }
+  return { items: parts.reverse().flat(), hasOlder: more };
+}
+
+/** pageFromSegments over archive text (tests and in-memory callers). */
+export function compactionArchivePageFromText(text, range = {}) {
+  const entries = [];
+  for (const line of String(text || '').split('\n')) {
+    const head = compactionSegmentHead(line);
+    if (head) entries.push({ head, line });
+  }
+  return pageFromSegments(entries, (entry) => segmentItems(entry.line), range);
+}
+
+// Per-sidecar segment index (byte offsets + headers), keyed by the file
+// identity it was built from: a page reads only the segments it reaches, and
+// the headers are indexed once per change rather than once per page.
+const INDEX_CACHE_LIMIT = 16;
+const archiveIndexes = new Map();
+
+const HEAD_PROBE_BYTES = 512;
+const ITEMS_KEY = '"items":';
+const SCAN_CHUNK_BYTES = 64 * 1024;
+
+/** Offset of the first newline at or after `from` (or `size`), scanning. */
+async function nextNewline(handle, from, size) {
+  const chunk = Buffer.alloc(SCAN_CHUNK_BYTES);
+  for (let at = from; at < size; at += SCAN_CHUNK_BYTES) {
+    const { bytesRead } = await handle.read(chunk, 0, Math.min(SCAN_CHUNK_BYTES, size - at), at);
+    const hit = chunk.subarray(0, bytesRead).indexOf(0x0a);
+    if (hit >= 0) return at + hit;
+  }
+  return size;
+}
+
+/** Segment offsets of an archive file. Each header names its items' byte
+ *  length, so the index reads only the headers and jumps over the bodies; a
+ *  line whose header does not account for its length is scanned past. */
+async function indexArchive(handle, size) {
+  const entries = [];
+  const probe = Buffer.alloc(HEAD_PROBE_BYTES);
+  let start = 0;
+  while (start < size) {
+    const { bytesRead } = await handle.read(probe, 0, Math.min(HEAD_PROBE_BYTES, size - start), start);
+    const text = probe.toString('latin1', 0, bytesRead);
+    const match = SEGMENT_HEAD.exec(text);
+    let end = -1;
+    if (match && text.startsWith(ITEMS_KEY, match[0].length)) {
+      // `…"items":<body>}` — the header is ASCII, so chars are bytes.
+      const candidate = start + match[0].length + ITEMS_KEY.length + Number(match[5]) + 1;
+      if (candidate === size) end = candidate;
+      else if (candidate < size) {
+        const tail = Buffer.alloc(1);
+        await handle.read(tail, 0, 1, candidate);
+        if (tail[0] === 0x0a) end = candidate;
+      }
+    }
+    if (end >= 0) entries.push({ head: compactionSegmentHead(text), start, length: end - start });
+    else end = await nextNewline(handle, start, size);
+    start = end + 1;
+  }
+  return entries;
+}
+
+async function archiveIndexFor(path, handle) {
+  const info = await handle.stat();
+  const stamp = `${info.dev}:${info.ino}:${info.size}:${info.mtimeMs}`;
+  const cached = archiveIndexes.get(path);
+  if (cached?.stamp === stamp) {
+    archiveIndexes.delete(path);
+    archiveIndexes.set(path, cached);
+    return cached.entries;
+  }
+  const entries = await indexArchive(handle, info.size);
+  archiveIndexes.set(path, { stamp, entries });
+  while (archiveIndexes.size > INDEX_CACHE_LIMIT) archiveIndexes.delete(archiveIndexes.keys().next().value);
+  return entries;
+}
+
+/** One page of a session's archive sidecar (see pageFromSegments). The
+ *  open handle pins the file a concurrent atomic replace may swap out, so
+ *  the index and the segment reads describe the same bytes. */
+export async function readCompactionArchivePage(storeDir, sessionId, range = {}) {
+  const path = compactionArchivePath(storeDir, sessionId);
+  if (!path) return { items: [], hasOlder: false };
+  let handle;
+  try {
+    handle = await open(path, 'r');
+  } catch {
+    // Display-only history: an absent or unreadable archive shows nothing.
+    archiveIndexes.delete(path);
+    return { items: [], hasOlder: false };
+  }
+  try {
+    const entries = await archiveIndexFor(path, handle);
+    return await pageFromSegments(
+      entries,
+      async (entry) => {
+        const buffer = Buffer.alloc(entry.length);
+        await handle.read(buffer, 0, entry.length, entry.start);
+        return segmentItems(buffer.toString('utf8'));
+      },
+      range
+    );
+  } catch {
+    return { items: [], hasOlder: false };
+  } finally {
+    await handle.close().catch(() => {});
+  }
+}
+
+// Delete fence: every deletion bumps the session's epoch. A writer captures
+// the epoch before it starts and publishes only if it is unchanged at the
+// (synchronous) rename, so a delete that lands while a segment is being
+// written is never undone by that write.
+const deleteEpochs = new Map();
+
+export function compactionArchiveDeleteEpoch(sessionId) {
+  return deleteEpochs.get(String(sessionId || '')) || 0;
+}
+
+/** Remove a session's archive (session delete) and fence every write that
+ *  started before it. Best-effort: true when the sidecar is gone afterwards. */
+export function deleteCompactionArchive(storeDir, sessionId) {
+  const path = compactionArchivePath(storeDir, sessionId);
+  if (!path) return false;
+  const id = String(sessionId);
+  deleteEpochs.set(id, compactionArchiveDeleteEpoch(id) + 1);
+  archiveIndexes.delete(path);
+  try {
+    unlinkSync(path);
+    return true;
+  } catch (error) {
+    return error?.code === 'ENOENT';
+  }
+}

package/src/runtime/agent/orchestrator/session/conversation-import/claude.mjs

@@ -0,0 +1,184 @@
+/**
+ * Claude Code project transcript (`projects/<dir>/<session>.jsonl`) → raw
+ * import turns. The active branch is followed from the last main-chain entry
+ * back through `parentUuid` (and `logicalParentUuid` across a compaction), so
+ * replayed or abandoned branches never duplicate. Kept: text blocks and
+ * `tool_use` as `[ran tool: <name>]` notes; images become an omission marker;
+ * a compaction summary only when the history it replaced is no longer linked.
+ * Dropped: sidechains, meta entries, tool results, thinking, signatures and
+ * Claude Code's own command/notification envelopes.
+ */
+import { IMAGE_OMITTED_TEXT, IMPORT_WARNING_IMAGE_OMITTED, isWholeElement, textPart, toolNote } from './turns.mjs';
+
+const CLAUDE_ENVELOPE_TAGS = [
+  'local-command-stdout',
+  'local-command-stderr',
+  'local-command-caveat',
+  'task-notification',
+  'system-reminder',
+  'user-prompt-submit-hook',
+  'bash-input',
+  'bash-stdout',
+  'bash-stderr',
+];
+const SLASH_COMMAND_RE = /^(?:\s*<(command-name|command-message|command-args)>[\s\S]*?<\/\1>)+\s*$/;
+const INTERRUPTED_RE = /^\[Request interrupted by user[^\]\n]*\]$/;
+// Claude Code's text annotation that accompanies a pasted image block.
+const IMAGE_ANNOTATION_RE = /^\[Image\b[^\n]*\]$/;
+
+export function isClaudeEnvelopeText(text) {
+  const trimmed = String(text || '').trim();
+  if (!trimmed) return true;
+  return (
+    isWholeElement(trimmed, CLAUDE_ENVELOPE_TAGS) ||
+    SLASH_COMMAND_RE.test(trimmed) ||
+    INTERRUPTED_RE.test(trimmed) ||
+    IMAGE_ANNOTATION_RE.test(trimmed)
+  );
+}
+
+const isMessageEntry = (record) => record?.type === 'user' || record?.type === 'assistant';
+const stringOrNull = (value) => (typeof value === 'string' && value.trim() ? value.trim() : null);
+const uuidOrNull = (value) => (typeof value === 'string' ? value : null);
+const timestampOrNull = (value) => (typeof value === 'string' || typeof value === 'number' ? value : null);
+// Rough fixed cost of one retained entry object, so tiny entries still count.
+const ENTRY_OVERHEAD_BYTES = 64;
+const metadataLength = (entry) =>
+  (entry.uuid?.length || 0) +
+  (entry.parentUuid?.length || 0) +
+  (entry.logicalParentUuid?.length || 0) +
+  (typeof entry.at === 'string' ? entry.at.length : 0);
+
+/**
+ * The active branch, root first: walked back from the last main-chain entry.
+ * `crossedCompaction` tells whether a compaction boundary led back into the
+ * history it replaced (Claude Code often no longer keeps that history).
+ */
+function activeChain(entries) {
+  const main = entries.filter((entry) => entry.main);
+  const leaf = main[main.length - 1];
+  if (!leaf || leaf.uuid === null) return { chain: main, crossedCompaction: false };
+  const byUuid = new Map();
+  for (const entry of entries) {
+    if (entry.uuid !== null) byUuid.set(entry.uuid, entry);
+  }
+  const chain = [];
+  const seen = new Set();
+  let crossedCompaction = false;
+  let current = leaf;
+  while (current && !seen.has(current.uuid)) {
+    seen.add(current.uuid);
+    chain.push(current);
+    const parent = current.parentUuid !== null ? byUuid.get(current.parentUuid) : null;
+    const logical = !parent && current.logicalParentUuid !== null ? byUuid.get(current.logicalParentUuid) : null;
+    if (logical) crossedCompaction = true;
+    current = parent || logical;
+  }
+  return { chain: chain.reverse().filter((entry) => entry.main), crossedCompaction };
+}
+
+function entryParts(role, content, warnings) {
+  const blocks = typeof content === 'string' ? [{ type: 'text', text: content }] : content;
+  if (!Array.isArray(blocks)) return [];
+  const parts = [];
+  for (const block of blocks) {
+    if (!block || typeof block !== 'object') continue;
+    if (block.type === 'text' && typeof block.text === 'string') {
+      if (role === 'user' && isClaudeEnvelopeText(block.text)) continue;
+      parts.push(textPart(block.text));
+    } else if (block.type === 'image') {
+      warnings.add(IMPORT_WARNING_IMAGE_OMITTED);
+      parts.push(textPart(IMAGE_OMITTED_TEXT));
+    } else if (block.type === 'tool_use' && role === 'assistant') {
+      parts.push(toolNote(block.name));
+    }
+  }
+  return parts;
+}
+
+function skippedEntry(record) {
+  // Claude Code flags its compaction summary transcript-only as well.
+  if (record.isCompactSummary === true) return false;
+  return record.isMeta === true || record.isVisibleInTranscriptOnly === true || record.isApiErrorMessage === true;
+}
+
+/**
+ * Incremental reducer: `push(record)` keeps a compact chain entry (uuid links
+ * plus the plain-text parts of a main-chain message) and drops the record
+ * itself, so tool results, thinking and image data are never retained.
+ * `chars` is the kept text across every branch so far, `entries` the chain
+ * entries retained and `metadataBytes` their link/timestamp bookkeeping.
+ */
+export function createClaudeReducer() {
+  const entries = [];
+  let recognized = false;
+  let foreignSessionId = null;
+  let cwd = null;
+  let chars = 0;
+  let metadataBytes = 0;
+  return {
+    get chars() {
+      return chars;
+    },
+    get entries() {
+      return entries.length;
+    },
+    get metadataBytes() {
+      return metadataBytes;
+    },
+    push(record) {
+      const message = isMessageEntry(record);
+      if (message) recognized = true;
+      const main = message && record.isSidechain !== true;
+      if (main) {
+        foreignSessionId ||= stringOrNull(record.sessionId);
+        cwd ||= stringOrNull(record.cwd);
+      }
+      const uuid = uuidOrNull(record?.uuid);
+      if (!main && uuid === null) return;
+      const entry = {
+        uuid,
+        parentUuid: uuidOrNull(record.parentUuid),
+        logicalParentUuid: uuidOrNull(record.logicalParentUuid),
+        main,
+        role: null,
+        parts: null,
+        warnings: null,
+        at: timestampOrNull(record.timestamp),
+      };
+      metadataBytes += ENTRY_OVERHEAD_BYTES + metadataLength(entry);
+      if (main && !skippedEntry(record)) {
+        const body = record.message && typeof record.message === 'object' ? record.message : null;
+        entry.role = body?.role === 'user' || body?.role === 'assistant' ? body.role : record.type;
+        const warnings = new Set();
+        entry.parts = entryParts(entry.role, body ? body.content : null, warnings);
+        if (record.isCompactSummary === true) {
+          entry.summary = true;
+          entry.parts = entry.parts.map((part) => ({ ...part, summary: true }));
+        }
+        if (warnings.size) entry.warnings = warnings;
+        for (const part of entry.parts) chars += part.text.length;
+      }
+      entries.push(entry);
+    },
+    result() {
+      const warnings = new Set();
+      const rawTurns = [];
+      const { chain, crossedCompaction } = activeChain(entries);
+      for (const entry of chain) {
+        if (!entry.parts?.length) continue;
+        // The summary stands in for history only when that history is gone.
+        if (entry.summary && crossedCompaction) continue;
+        rawTurns.push({ role: entry.role, parts: entry.parts, at: entry.at });
+        for (const code of entry.warnings || []) warnings.add(code);
+      }
+      return { recognized, foreignSessionId, cwd, rawTurns, warnings };
+    },
+  };
+}
+
+export function convertClaudeRecords(records) {
+  const reducer = createClaudeReducer();
+  for (const record of records) reducer.push(record);
+  return reducer.result();
+}

package/src/runtime/agent/orchestrator/session/conversation-import/codex.mjs

@@ -0,0 +1,136 @@
+/**
+ * Codex CLI rollout (`rollout-*.jsonl`) → raw import turns. Kept: the first
+ * `session_meta` (foreign id, cwd), user/assistant `message` items as text and
+ * tool calls as `[ran tool: <name>]` notes. Dropped: developer/system
+ * messages, Codex's own wrapper user messages, tool outputs, reasoning,
+ * event_msg, token counts and turn_context.
+ */
+import { IMAGE_OMITTED_TEXT, IMPORT_WARNING_IMAGE_OMITTED, isWholeElement, textPart, toolNote } from './turns.mjs';
+
+// User-role envelopes Codex injects itself (environment, instructions, abort
+// notices, plugin hints, shell-command echoes, image wrappers).
+const CODEX_ENVELOPE_TAGS = [
+  'environment_context',
+  'user_instructions',
+  'turn_aborted',
+  'recommended_plugins',
+  'user_shell_command',
+  'subagent_notification',
+  'skills_instructions',
+];
+const TOOL_CALL_TYPES = new Set(['function_call', 'custom_tool_call', 'local_shell_call']);
+
+export function isCodexEnvelopeText(text) {
+  const trimmed = String(text || '').trim();
+  if (!trimmed) return true;
+  if (/^# AGENTS\.md instructions\b/.test(trimmed) && trimmed.includes('<INSTRUCTIONS>')) return true;
+  if (isWholeElement(trimmed, CODEX_ENVELOPE_TAGS)) return true;
+  // The `<image name=…>` / `</image>` text parts that wrap an input_image.
+  return /^<image\b[^>]*>$/.test(trimmed) || trimmed === '</image>';
+}
+
+function messageParts(payload, warnings) {
+  const content = typeof payload.content === 'string' ? [{ type: 'text', text: payload.content }] : payload.content;
+  if (!Array.isArray(content)) return [];
+  const parts = [];
+  for (const item of content) {
+    if (!item || typeof item !== 'object') continue;
+    if (item.type === 'input_image' || item.type === 'image') {
+      warnings.add(IMPORT_WARNING_IMAGE_OMITTED);
+      parts.push(textPart(IMAGE_OMITTED_TEXT));
+      continue;
+    }
+    if (typeof item.text !== 'string') continue;
+    if (payload.role === 'user' && isCodexEnvelopeText(item.text)) continue;
+    parts.push(textPart(item.text));
+  }
+  return parts;
+}
+
+function toolName(payload) {
+  if (typeof payload.name === 'string' && payload.name.trim()) return payload.name;
+  return payload.type === 'local_shell_call' ? 'local_shell' : '';
+}
+
+const stringOrNull = (value) => (typeof value === 'string' && value.trim() ? value.trim() : null);
+// Rough fixed cost of one retained turn object, so tiny turns still count.
+const ENTRY_OVERHEAD_BYTES = 64;
+
+/**
+ * Incremental reducer: `push(record)` keeps only the plain-text parts of one
+ * record (the record itself is not retained), `chars` is the kept text so far,
+ * `entries` the raw turns kept and `metadataBytes` their timestamp bookkeeping.
+ */
+export function createCodexReducer() {
+  const warnings = new Set();
+  const rawTurns = [];
+  let recognized = false;
+  let foreignSessionId = null;
+  let cwd = null;
+  let metaSeen = false;
+  let chars = 0;
+  let metadataBytes = 0;
+  const keep = (role, parts, at) => {
+    for (const part of parts) chars += part.text.length;
+    const stamp = typeof at === 'string' || typeof at === 'number' ? at : null;
+    metadataBytes += ENTRY_OVERHEAD_BYTES + (typeof stamp === 'string' ? stamp.length : 0);
+    rawTurns.push({ role, parts, at: stamp });
+  };
+  return {
+    get chars() {
+      return chars;
+    },
+    get entries() {
+      return rawTurns.length;
+    },
+    get metadataBytes() {
+      return metadataBytes;
+    },
+    push(record) {
+      const payload = record?.payload && typeof record.payload === 'object' ? record.payload : null;
+      if (!payload) return;
+      if (record.type === 'session_meta') {
+        recognized = true;
+        if (!metaSeen) {
+          metaSeen = true;
+          foreignSessionId = stringOrNull(payload.id) || stringOrNull(payload.session_id);
+          cwd = stringOrNull(payload.cwd);
+        }
+        return;
+      }
+      if (record.type !== 'response_item') return;
+      recognized = true;
+      if (payload.type === 'message' && (payload.role === 'user' || payload.role === 'assistant')) {
+        const parts = messageParts(payload, warnings);
+        if (parts.length) keep(payload.role, parts, record.timestamp);
+      } else if (TOOL_CALL_TYPES.has(payload.type)) {
+        keep('assistant', [toolNote(toolName(payload))], record.timestamp);
+      }
+    },
+    result() {
+      return { recognized, foreignSessionId, cwd, rawTurns, warnings };
+    },
+  };
+}
+
+export function convertCodexRecords(records) {
+  const reducer = createCodexReducer();
+  for (const record of records) reducer.push(record);
+  return reducer.result();
+}
+
+/** id/cwd of a `session_meta` first line cut off by a bounded prefix read. */
+export function codexMetaFromTruncatedLine(line) {
+  const text = String(line || '');
+  if (!/"type"\s*:\s*"session_meta"/.test(text)) return null;
+  const field = (name) => {
+    const match = new RegExp(`"${name}"\\s*:\\s*("(?:[^"\\\\]|\\\\.)*")`).exec(text);
+    if (!match) return null;
+    try {
+      return stringOrNull(JSON.parse(match[1]));
+    } catch {
+      return null;
+    }
+  };
+  return { foreignSessionId: field('id') || field('session_id'), cwd: field('cwd') };
+}