chainlesschain 0.45.11 → 0.45.19

package/src/lib/jsonl-session-store.js

@@ -1,237 +1,21 @@
-/**
- * JSONL Session Store — append-only session persistence.
- *
- * Each session is a file: .chainlesschain/sessions/{session-id}.jsonl
- * Each line: {"type":"...", "timestamp":..., "data":{...}}
- *
- * Types: session_start, user_message, assistant_message, tool_call,
- *        tool_result, system, compact, session_end
- *
- * Feature-flag gated: JSONL_SESSION
- */
-
-import {
-  existsSync,
-  mkdirSync,
-  appendFileSync,
-  readFileSync,
-  readdirSync,
-  renameSync,
-} from "node:fs";
-import { join, basename } from "node:path";
-import { createHash } from "node:crypto";
-import { getHomeDir } from "./paths.js";
-
-function getSessionsDir() {
-  const dir = join(getHomeDir(), "sessions");
-  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
-  return dir;
-}
-
-function sessionPath(sessionId) {
-  return join(getSessionsDir(), `${sessionId}.jsonl`);
-}
-
-// ── Write operations ────────────────────────────────────────────────────
-
-/**
- * Append a single event to the session log.
- * User messages are written synchronously for crash recovery.
- */
-export function appendEvent(sessionId, type, data) {
-  const line = JSON.stringify({ type, timestamp: Date.now(), data }) + "\n";
-  appendFileSync(sessionPath(sessionId), line, "utf-8");
-}
-
-/**
- * Start a new session (writes session_start event).
- */
-export function startSession(sessionId, meta = {}) {
-  const id =
-    sessionId ||
-    `session-${Date.now()}-${createHash("sha256").update(Math.random().toString()).digest("hex").slice(0, 6)}`;
-
-  appendEvent(id, "session_start", {
-    title: meta.title || "Untitled",
-    provider: meta.provider || "",
-    model: meta.model || "",
-  });
-
-  return id;
-}
-
-/**
- * Append a user message (synchronous write for crash recovery).
- */
-export function appendUserMessage(sessionId, content) {
-  appendEvent(sessionId, "user_message", { role: "user", content });
-}
-
-/**
- * Append an assistant message.
- */
-export function appendAssistantMessage(sessionId, content) {
-  appendEvent(sessionId, "assistant_message", { role: "assistant", content });
-}
-
-/**
- * Append a tool call event.
- */
-export function appendToolCall(sessionId, toolName, args) {
-  appendEvent(sessionId, "tool_call", { tool: toolName, args });
-}
-
-/**
- * Append a tool result event.
- */
-export function appendToolResult(sessionId, toolName, result) {
-  appendEvent(sessionId, "tool_result", { tool: toolName, result });
-}
-
-/**
- * Append a compact event (context was compressed).
- */
-export function appendCompactEvent(sessionId, stats) {
-  appendEvent(sessionId, "compact", stats);
-}
-
-// ── Read operations ─────────────────────────────────────────────────────
-
-/**
- * Read all events from a session file.
- * @returns {Array<{type, timestamp, data}>}
- */
-export function readEvents(sessionId) {
-  const filePath = sessionPath(sessionId);
-  if (!existsSync(filePath)) return [];
-
-  const content = readFileSync(filePath, "utf-8");
-  const events = [];
-
-  for (const line of content.split("\n")) {
-    if (!line.trim()) continue;
-    try {
-      events.push(JSON.parse(line));
-    } catch (_e) {
-      // Skip malformed lines
-    }
-  }
-
-  return events;
-}
-
-/**
- * Reconstruct messages array from JSONL events (for API calls).
- * Skips compact events and rebuilds the current conversation state.
- */
-export function rebuildMessages(sessionId) {
-  const events = readEvents(sessionId);
-  const messages = [];
-  let lastCompactIndex = -1;
-
-  // Find the last compact event (start from there)
-  for (let i = events.length - 1; i >= 0; i--) {
-    if (events[i].type === "compact" && events[i].data.messages) {
-      lastCompactIndex = i;
-      break;
-    }
-  }
-
-  if (lastCompactIndex >= 0 && events[lastCompactIndex].data.messages) {
-    messages.push(...events[lastCompactIndex].data.messages);
-  }
-
-  // Replay events after last compact
-  const startIndex = lastCompactIndex >= 0 ? lastCompactIndex + 1 : 0;
-
-  for (let i = startIndex; i < events.length; i++) {
-    const event = events[i];
-    if (
-      event.type === "user_message" ||
-      event.type === "assistant_message" ||
-      event.type === "system"
-    ) {
-      messages.push(event.data);
-    }
-  }
-
-  return messages;
-}
-
-/**
- * List all sessions (reads session_start events from all .jsonl files).
- */
-export function listJsonlSessions(options = {}) {
-  const dir = getSessionsDir();
-  if (!existsSync(dir)) return [];
-
-  const limit = options.limit || 20;
-  const files = readdirSync(dir)
-    .filter((f) => f.endsWith(".jsonl"))
-    .map((f) => {
-      const id = basename(f, ".jsonl");
-      const events = readEvents(id);
-      const startEvent = events.find((e) => e.type === "session_start");
-      const lastEvent = events[events.length - 1];
-      const messageCount = events.filter(
-        (e) => e.type === "user_message" || e.type === "assistant_message",
-      ).length;
-
-      return {
-        id,
-        title: startEvent?.data?.title || "Untitled",
-        provider: startEvent?.data?.provider || "",
-        model: startEvent?.data?.model || "",
-        message_count: messageCount,
-        created_at: startEvent
-          ? new Date(startEvent.timestamp).toISOString()
-          : "",
-        updated_at: lastEvent
-          ? new Date(lastEvent.timestamp).toISOString()
-          : "",
-      };
-    })
-    .sort((a, b) => (b.updated_at > a.updated_at ? 1 : -1))
-    .slice(0, limit);
-
-  return files;
-}
-
-/**
- * Fork a session — copies all events to a new session ID.
- */
-export function forkSession(sourceId) {
-  const events = readEvents(sourceId);
-  if (events.length === 0) return null;
-
-  const newId = `session-${Date.now()}-${createHash("sha256").update(Math.random().toString()).digest("hex").slice(0, 6)}`;
-  const filePath = sessionPath(newId);
-
-  for (const event of events) {
-    const line = JSON.stringify(event) + "\n";
-    appendFileSync(filePath, line, "utf-8");
-  }
-
-  // Mark the fork
-  appendEvent(newId, "system", {
-    role: "system",
-    content: `[Forked from session ${sourceId}]`,
-  });
-
-  return newId;
-}
-
-/**
- * Check if a JSONL session exists.
- */
-export function sessionExists(sessionId) {
-  return existsSync(sessionPath(sessionId));
-}
-
-/**
- * Get the most recent session ID (for --continue).
- */
-export function getLastSessionId() {
-  const sessions = listJsonlSessions({ limit: 1 });
-  return sessions.length > 0 ? sessions[0].id : null;
-}
+export {
+  appendEvent,
+  startSession,
+  appendUserMessage,
+  appendAssistantMessage,
+  appendToolCall,
+  appendToolResult,
+  appendCompactEvent,
+  readEvents,
+  rebuildMessages,
+  listJsonlSessions,
+  forkSession,
+  sessionExists,
+  getLastSessionId,
+  migrateLegacySessions,
+  migrateLegacySessionsBatch,
+  migrateLegacySessionFile,
+  validateJsonlSession,
+  validateAllJsonlSessions,
+  sampleMigratedSessionsValidation,
+} from "../harness/jsonl-session-store.js";

package/src/lib/prompt-compressor.js

@@ -1,351 +1,10 @@
-/**
- * CLI Prompt Compressor — 5 strategies for context window management.
- *
- * Strategies:
- *   1. deduplication  — Remove duplicate/similar messages (Jaccard similarity)
- *   2. truncation     — Keep most recent N messages
- *   3. summarization  — LLM-generated summary of old history
- *   4. snipCompact    — Remove stale tool results and processed markers
- *   5. contextCollapse — Fold consecutive same-type messages into summaries
- *
- * Feature-flag gated: PROMPT_COMPRESSOR, CONTEXT_SNIP, CONTEXT_COLLAPSE
- */
-
-import { createHash } from "node:crypto";
-import { feature } from "./feature-flags.js";
-
-// ── Token estimation ────────────────────────────────────────────────────
-
-/**
- * Estimate token count for a string.
- * Chinese: ~1.5 chars/token, English: ~4 chars/token.
- */
-export function estimateTokens(text) {
-  if (!text) return 0;
-  const chineseChars = (text.match(/[\u4e00-\u9fa5]/g) || []).length;
-  const otherChars = text.length - chineseChars;
-  return Math.ceil(chineseChars / 1.5 + otherChars / 4);
-}
-
-/**
- * Estimate total tokens for a messages array.
- */
-export function estimateMessagesTokens(messages) {
-  return messages.reduce((sum, msg) => {
-    const content =
-      typeof msg.content === "string"
-        ? msg.content
-        : JSON.stringify(msg.content || "");
-    return sum + estimateTokens(content);
-  }, 0);
-}
-
-// ── Similarity ──────────────────────────────────────────────────────────
-
-function jaccardSimilarity(str1, str2) {
-  if (!str1 || !str2) return 0;
-  if (str1 === str2) return 1;
-  const tokens1 = new Set(str1.split(""));
-  const tokens2 = new Set(str2.split(""));
-  let intersection = 0;
-  for (const t of tokens1) {
-    if (tokens2.has(t)) intersection++;
-  }
-  return intersection / (tokens1.size + tokens2.size - intersection);
-}
-
-function getContent(msg) {
-  return typeof msg.content === "string"
-    ? msg.content
-    : JSON.stringify(msg.content || "");
-}
-
-// ── PromptCompressor class ──────────────────────────────────────────────
-
-export class PromptCompressor {
-  /**
-   * @param {object} options
-   * @param {number} [options.maxMessages=20] — Max messages before truncation
-   * @param {number} [options.maxTokens=8000] — Token threshold for auto-compact
-   * @param {number} [options.similarityThreshold=0.9] — Dedup similarity threshold
-   * @param {Function} [options.llmQuery] — async (prompt) => string, for summarization
-   */
-  constructor(options = {}) {
-    this.maxMessages = options.maxMessages || 20;
-    this.maxTokens = options.maxTokens || 8000;
-    this.similarityThreshold = options.similarityThreshold || 0.9;
-    this.llmQuery = options.llmQuery || null;
-  }
-
-  /**
-   * Run all enabled compression strategies on messages.
-   * Returns { messages, stats }.
-   */
-  async compress(messages, options = {}) {
-    if (!Array.isArray(messages) || messages.length <= 2) {
-      return {
-        messages: Array.isArray(messages) ? [...messages] : [],
-        stats: { strategy: "none", saved: 0 },
-      };
-    }
-
-    const originalTokens = estimateMessagesTokens(messages);
-    let result = [...messages];
-    const applied = [];
-
-    // Strategy 4: snipCompact (remove stale tool results)
-    if (feature("CONTEXT_SNIP")) {
-      const before = result.length;
-      result = this._snipCompact(result);
-      if (result.length < before) applied.push("snip");
-    }
-
-    // Strategy 1: deduplication
-    if (result.length > 3) {
-      const before = result.length;
-      result = this._deduplicate(result);
-      if (result.length < before) applied.push("dedup");
-    }
-
-    // Strategy 5: contextCollapse (fold consecutive tool results)
-    if (feature("CONTEXT_COLLAPSE") && result.length > 6) {
-      const before = result.length;
-      result = this._contextCollapse(result);
-      if (result.length < before) applied.push("collapse");
-    }
-
-    // Strategy 2: truncation
-    if (result.length > this.maxMessages) {
-      result = this._truncate(result);
-      applied.push("truncate");
-    }
-
-    // Strategy 3: summarization (only if still over token limit)
-    const currentTokens = estimateMessagesTokens(result);
-    if (this.llmQuery && currentTokens > this.maxTokens && result.length > 4) {
-      try {
-        result = await this._summarize(result);
-        applied.push("summarize");
-      } catch (_err) {
-        // Summarization failed — continue with what we have
-      }
-    }
-
-    const compressedTokens = estimateMessagesTokens(result);
-    return {
-      messages: result,
-      stats: {
-        strategy: applied.join("+") || "none",
-        originalMessages: messages.length,
-        compressedMessages: result.length,
-        originalTokens,
-        compressedTokens,
-        saved: originalTokens - compressedTokens,
-        ratio: originalTokens > 0 ? compressedTokens / originalTokens : 1,
-      },
-    };
-  }
-
-  /**
-   * Check if auto-compact should trigger.
-   */
-  shouldAutoCompact(messages) {
-    return (
-      messages.length > this.maxMessages ||
-      estimateMessagesTokens(messages) > this.maxTokens
-    );
-  }
-
-  // ── Strategy 1: Deduplication ───────────────────────────────────────
-
-  _deduplicate(messages) {
-    const system = messages.filter((m) => m.role === "system");
-    const last = [...messages].reverse().find((m) => m.role === "user");
-    const rest = messages.filter((m) => m.role !== "system" && m !== last);
-
-    const seen = new Map();
-    const deduped = [];
-
-    for (const msg of rest) {
-      const content = getContent(msg);
-      const hash = createHash("md5").update(content).digest("hex");
-
-      if (seen.has(hash)) continue;
-
-      let isDup = false;
-      for (const [, existing] of seen) {
-        if (
-          jaccardSimilarity(content, getContent(existing)) >=
-          this.similarityThreshold
-        ) {
-          isDup = true;
-          break;
-        }
-      }
-
-      if (!isDup) {
-        seen.set(hash, msg);
-        deduped.push(msg);
-      }
-    }
-
-    const result = [...system, ...deduped];
-    if (last && !result.includes(last)) result.push(last);
-    return result;
-  }
-
-  // ── Strategy 2: Truncation ──────────────────────────────────────────
-
-  _truncate(messages) {
-    const system = messages.filter((m) => m.role === "system");
-    const last = [...messages].reverse().find((m) => m.role === "user");
-    const rest = messages.filter((m) => m.role !== "system" && m !== last);
-
-    let slots = this.maxMessages - system.length;
-    if (last) slots -= 1;
-
-    const recent = rest.slice(-Math.max(slots, 1));
-    const result = [...system, ...recent];
-    if (last && !result.includes(last)) result.push(last);
-    return result;
-  }
-
-  // ── Strategy 3: Summarization ───────────────────────────────────────
-
-  async _summarize(messages) {
-    const system = messages.filter((m) => m.role === "system");
-    const last = [...messages].reverse().find((m) => m.role === "user");
-    const toSummarize = messages.filter(
-      (m) => m.role !== "system" && m !== last,
-    );
-
-    if (toSummarize.length < 3) return messages;
-
-    const historyText = toSummarize
-      .map((m) => `${m.role}: ${getContent(m).slice(0, 500)}`)
-      .join("\n");
-
-    const summary = await this.llmQuery(
-      `Summarize this conversation history concisely, preserving key facts and decisions:\n\n${historyText}\n\nSummary:`,
-    );
-
-    if (!summary) return messages;
-
-    const result = [
-      ...system,
-      { role: "system", content: `[Conversation Summary]\n${summary}` },
-    ];
-    if (last) result.push(last);
-    return result;
-  }
-
-  // ── Strategy 4: Snip Compact ────────────────────────────────────────
-  // Removes stale markers: processed tool results older than recent window,
-  // empty assistant messages, and system messages with [PROCESSED] tags.
-
-  _snipCompact(messages) {
-    if (messages.length <= 4) return messages;
-
-    // Keep system[0] + last 4 messages untouched
-    const head = messages.slice(0, 1);
-    const middle = messages.slice(1, -4);
-    const tail = messages.slice(-4);
-
-    const snipped = middle.filter((msg) => {
-      const content = getContent(msg);
-
-      // Remove empty messages
-      if (!content || content.trim() === "") return false;
-
-      // Remove processed markers
-      if (content.includes("[PROCESSED]") || content.includes("[STALE]"))
-        return false;
-
-      // Remove tool_result messages that are just "ok" or empty JSON
-      if (msg.role === "tool") {
-        if (
-          content === "ok" ||
-          content === "{}" ||
-          content === "null" ||
-          content.length < 3
-        )
-          return false;
-      }
-
-      // Remove very short assistant acknowledgments in middle
-      if (msg.role === "assistant" && content.length < 10) return false;
-
-      return true;
-    });
-
-    return [...head, ...snipped, ...tail];
-  }
-
-  // ── Strategy 5: Context Collapse ────────────────────────────────────
-  // Folds consecutive tool_call + tool_result pairs into a single summary.
-
-  _contextCollapse(messages) {
-    if (messages.length <= 6) return messages;
-
-    const result = [];
-    let i = 0;
-
-    while (i < messages.length) {
-      const msg = messages[i];
-
-      // Detect consecutive tool sequences in the middle (not last 3)
-      if (
-        i > 0 &&
-        i < messages.length - 3 &&
-        msg.role === "assistant" &&
-        msg.tool_calls &&
-        msg.tool_calls.length > 0
-      ) {
-        // Collect this tool call + all following tool results
-        const toolGroup = [msg];
-        let j = i + 1;
-        while (j < messages.length - 3 && messages[j].role === "tool") {
-          toolGroup.push(messages[j]);
-          j++;
-        }
-
-        // Also collect next assistant with tool_calls (chained calls)
-        while (
-          j < messages.length - 3 &&
-          messages[j].role === "assistant" &&
-          messages[j].tool_calls
-        ) {
-          toolGroup.push(messages[j]);
-          j++;
-          while (j < messages.length - 3 && messages[j].role === "tool") {
-            toolGroup.push(messages[j]);
-            j++;
-          }
-        }
-
-        // Only collapse if we collected 3+ messages
-        if (toolGroup.length >= 3) {
-          const toolNames = toolGroup
-            .filter((m) => m.tool_calls)
-            .flatMap((m) =>
-              m.tool_calls.map((tc) => tc.function?.name || "tool"),
-            )
-            .filter(Boolean);
-          const uniqueTools = [...new Set(toolNames)];
-
-          result.push({
-            role: "system",
-            content: `[Collapsed ${toolGroup.length} tool messages: ${uniqueTools.join(", ")}]`,
-          });
-          i = j;
-          continue;
-        }
-      }
-
-      result.push(msg);
-      i++;
-    }
-
-    return result;
-  }
-}
+export {
+  estimateTokens,
+  estimateMessagesTokens,
+  CONTEXT_WINDOWS,
+  getContextWindow,
+  COMPRESSION_VARIANTS,
+  getCompressionVariant,
+  adaptiveThresholds,
+  PromptCompressor,
+} from "../harness/prompt-compressor.js";