create-walle 0.9.13 → 0.9.14

package/template/wall-e/memory/ctm-session-context.js

@@ -0,0 +1,910 @@
+'use strict';
+
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+const crypto = require('node:crypto');
+const Database = require('better-sqlite3');
+
+const DEFAULT_SEARCH_LIMIT = 10;
+const MAX_SEARCH_LIMIT = 50;
+const DEFAULT_CONTEXT_LIMIT = 200;
+const MAX_CONTEXT_LIMIT = 5000;
+const DEFAULT_TOKEN_BUDGET = 12000;
+
+function resolveCtmDbPath(env = process.env) {
+  if (env.CTM_DB_PATH) return env.CTM_DB_PATH;
+  if (env.CTM_DATA_DIR) return path.join(env.CTM_DATA_DIR, 'task-manager.db');
+  if (env.WALLE_DEV_DIR) return path.join(env.WALLE_DEV_DIR, 'task-manager.db');
+  return path.join(env.HOME || os.homedir(), '.walle', 'data', 'task-manager.db');
+}
+
+function openCtmDb({ dbPath, readonly = true } = {}) {
+  const resolved = path.resolve(dbPath || resolveCtmDbPath());
+  if (!fs.existsSync(resolved)) {
+    return { db: null, dbPath: resolved, close: () => {}, unavailable: true };
+  }
+  const db = new Database(resolved, { readonly, fileMustExist: true });
+  db.pragma('busy_timeout = 2500');
+  return { db, dbPath: resolved, close: () => db.close(), unavailable: false };
+}
+
+function withCtmDb(options, fn) {
+  if (options?.db) {
+    return fn(options.db, { dbPath: options.dbPath || '', close: () => {}, owned: false });
+  }
+  const handle = openCtmDb(options);
+  if (!handle.db) {
+    return {
+      ok: false,
+      source: 'ctm-db',
+      unavailable: true,
+      db_path: handle.dbPath,
+      reason: 'ctm_db_not_found',
+    };
+  }
+  try {
+    return fn(handle.db, { dbPath: handle.dbPath, close: handle.close, owned: true });
+  } finally {
+    handle.close();
+  }
+}
+
+function searchCtmSessions({ query, limit = DEFAULT_SEARCH_LIMIT, db, dbPath } = {}) {
+  const cleanQuery = String(query || '').trim();
+  if (!cleanQuery) return { ok: false, source: 'ctm-db', results: [], reason: 'query_required' };
+  const requestedLimit = clampLimit(limit, DEFAULT_SEARCH_LIMIT, MAX_SEARCH_LIMIT);
+
+  return withCtmDb({ db, dbPath }, (conn, handle) => {
+    if (!hasAnySessionTable(conn)) {
+      return { ok: false, source: 'ctm-db', db_path: handle.dbPath, results: [], reason: 'ctm_session_tables_missing' };
+    }
+
+    const rawResults = [];
+    rawResults.push(...searchSessionMessagesFts(conn, cleanQuery, requestedLimit * 4));
+    rawResults.push(...searchSessionMessagesLike(conn, cleanQuery, requestedLimit * 4));
+    rawResults.push(...searchSessionMetadata(conn, cleanQuery, requestedLimit * 2));
+
+    const results = dedupeItems(rawResults, {
+      limit: requestedLimit,
+      contentFields: ['snippet', 'content'],
+      keyPrefix: (item) => `${item.session_id || ''}:${item.role || ''}`,
+    });
+
+    return {
+      ok: true,
+      source: 'ctm-db',
+      db_path: handle.dbPath,
+      query: cleanQuery,
+      count: results.length,
+      results,
+    };
+  });
+}
+
+function getCtmSessionContext({
+  session_id,
+  source_id,
+  session_ids,
+  ids,
+  limit = DEFAULT_CONTEXT_LIMIT,
+  cursor = 0,
+  include_raw = false,
+  dedupe = true,
+  format = 'messages',
+  db,
+  dbPath,
+} = {}) {
+  const identifiers = normalizeIdentifiers(session_ids || ids || session_id || source_id);
+  if (identifiers.length === 0) {
+    return { ok: false, source: 'ctm-db', reason: 'session_id_required', sessions: [], messages: [] };
+  }
+
+  return withCtmDb({ db, dbPath }, (conn, handle) => {
+    if (!hasAnySessionTable(conn)) {
+      return { ok: false, source: 'ctm-db', db_path: handle.dbPath, reason: 'ctm_session_tables_missing', sessions: [], messages: [] };
+    }
+
+    const bundles = identifiers.map((id) => resolveSessionBundle(conn, id)).filter(Boolean);
+    const seenBundles = new Set();
+    const uniqueBundles = bundles.filter((bundle) => {
+      const key = bundle.ctm_session_id || bundle.requested_id;
+      if (seenBundles.has(key)) return false;
+      seenBundles.add(key);
+      return true;
+    });
+
+    const sessions = [];
+    const allMessages = [];
+    for (const bundle of uniqueBundles) {
+      const conversations = getConversationRows(conn, bundle);
+      const sessionMessages = [];
+      for (const row of conversations) {
+        const messages = parseConversationMessages(row);
+        for (let i = 0; i < messages.length; i += 1) {
+          const normalized = normalizeMessage(messages[i], {
+            row,
+            bundle,
+            index: i,
+            includeRaw: include_raw,
+          });
+          if (normalized.content || include_raw) sessionMessages.push(normalized);
+        }
+      }
+
+      const dedupedSessionMessages = dedupe ? dedupeMessages(sessionMessages) : sessionMessages;
+      sessions.push(formatSessionSummary(bundle, conversations, dedupedSessionMessages));
+      allMessages.push(...dedupedSessionMessages);
+    }
+
+    const orderedMessages = orderMessages(dedupe ? dedupeMessages(allMessages) : allMessages);
+    const start = Math.max(0, Number.parseInt(cursor, 10) || 0);
+    const cap = normalizeContextLimit(limit);
+    const pagedMessages = cap === 0 ? orderedMessages.slice(start) : orderedMessages.slice(start, start + cap);
+    const nextCursor = cap !== 0 && start + cap < orderedMessages.length ? start + cap : null;
+    const result = {
+      ok: true,
+      source: 'ctm-db',
+      db_path: handle.dbPath,
+      requested_ids: identifiers,
+      sessions,
+      count: pagedMessages.length,
+      total_count: orderedMessages.length,
+      cursor: start,
+      next_cursor: nextCursor,
+      messages: pagedMessages,
+      transfer: {
+        full_context_supported: true,
+        served_from: 'session_conversations/session_messages',
+        jsonl_fallback_used: false,
+        deduped: Boolean(dedupe),
+      },
+    };
+
+    if (format === 'markdown' || format === 'compact') {
+      result.text = renderContextMarkdown(result, { compact: format === 'compact' });
+    }
+    return result;
+  });
+}
+
+function buildContextPack({
+  task,
+  query,
+  session_ids,
+  ids,
+  limit = 5,
+  token_budget = DEFAULT_TOKEN_BUDGET,
+  include_raw = false,
+  mode = 'auto',
+  db,
+  dbPath,
+} = {}) {
+  const identifiers = normalizeIdentifiers(session_ids || ids);
+  const effectiveQuery = String(query || task || '').trim();
+  const maxSessions = clampLimit(limit, 5, MAX_SEARCH_LIMIT);
+  const budgetChars = Math.max(1000, Math.min(Number(token_budget || DEFAULT_TOKEN_BUDGET), 200000) * 4);
+
+  return withCtmDb({ db, dbPath }, (conn, handle) => {
+    const selectedIds = [];
+    const search = effectiveQuery
+      ? searchCtmSessions({ query: effectiveQuery, limit: maxSessions, db: conn, dbPath: handle.dbPath })
+      : { ok: true, results: [] };
+
+    for (const id of identifiers) selectedIds.push(id);
+    for (const item of search.results || []) {
+      if (item.session_id) selectedIds.push(item.session_id);
+      if (item.agent_session_id) selectedIds.push(item.agent_session_id);
+    }
+
+    const uniqueIds = [...new Set(selectedIds)].slice(0, maxSessions);
+    const context = getCtmSessionContext({
+      session_ids: uniqueIds,
+      limit: mode === 'full' ? 0 : 1000,
+      include_raw,
+      dedupe: true,
+      db: conn,
+      dbPath: handle.dbPath,
+    });
+
+    const pack = trimContextToBudget(context, budgetChars);
+    return {
+      ok: true,
+      source: 'ctm-db',
+      db_path: handle.dbPath,
+      task: task || '',
+      query: effectiveQuery,
+      mode,
+      search_results: search.results || [],
+      selected_session_ids: uniqueIds,
+      sessions: pack.sessions,
+      messages: pack.messages,
+      text: renderContextMarkdown(pack, { compact: mode !== 'full' }),
+      transfer: {
+        full_context_supported: true,
+        served_from: 'session_conversations/session_messages',
+        jsonl_fallback_used: false,
+        deduped: true,
+        token_budget: Number(token_budget || DEFAULT_TOKEN_BUDGET),
+        truncated: pack.truncated,
+      },
+    };
+  });
+}
+
+function hasAnySessionTable(db) {
+  return hasTable(db, 'session_conversations') || hasTable(db, 'session_messages') || hasTable(db, 'ctm_sessions') || hasTable(db, 'agent_sessions');
+}
+
+function hasTable(db, name) {
+  try {
+    return Boolean(db.prepare("SELECT name FROM sqlite_master WHERE type IN ('table', 'view') AND name = ?").get(name));
+  } catch {
+    return false;
+  }
+}
+
+function tableColumns(db, table) {
+  try {
+    return db.prepare(`PRAGMA table_info(${quoteIdent(table)})`).all().map((row) => row.name);
+  } catch {
+    return [];
+  }
+}
+
+function idColumnFor(db, table) {
+  const cols = tableColumns(db, table);
+  if (cols.includes('ctm_session_id')) return 'ctm_session_id';
+  if (cols.includes('session_id')) return 'session_id';
+  return null;
+}
+
+function quoteIdent(value) {
+  return `"${String(value).replace(/"/g, '""')}"`;
+}
+
+function searchSessionMessagesFts(db, query, limit) {
+  if (!hasTable(db, 'session_messages') || !hasTable(db, 'session_messages_fts')) return [];
+  const msgSessionCol = idColumnFor(db, 'session_messages');
+  if (!msgSessionCol) return [];
+  const ftsQueries = buildFtsQueries(query);
+  if (!ftsQueries.length) return [];
+
+  try {
+    const stmt = db.prepare(`
+      SELECT sm.${quoteIdent(msgSessionCol)} AS conversation_id,
+             sm.id AS message_rowid,
+             sm.role,
+             sm.message_index,
+             sm.content,
+             sm.created_at,
+             fts.rank AS rank
+      FROM session_messages_fts fts
+      JOIN session_messages sm ON sm.id = fts.rowid
+      WHERE session_messages_fts MATCH ?
+      ORDER BY fts.rank
+      LIMIT ?
+    `);
+    let rows = [];
+    for (const ftsQuery of ftsQueries) {
+      rows = stmt.all(ftsQuery, limit);
+      if (rows.length) break;
+    }
+    return rows.map((row, index) => enrichSearchRow(db, row, {
+      source_table: 'session_messages_fts',
+      rank: index,
+      score: row.rank,
+    }));
+  } catch {
+    return [];
+  }
+}
+
+function searchSessionMessagesLike(db, query, limit) {
+  if (!hasTable(db, 'session_messages')) return [];
+  const msgSessionCol = idColumnFor(db, 'session_messages');
+  if (!msgSessionCol) return [];
+  const like = `%${escapeLike(query)}%`;
+  try {
+    const rows = db.prepare(`
+      SELECT ${quoteIdent(msgSessionCol)} AS conversation_id,
+             id AS message_rowid,
+             role,
+             message_index,
+             content,
+             created_at
+      FROM session_messages
+      WHERE content LIKE ? ESCAPE '\\'
+      ORDER BY created_at DESC, id DESC
+      LIMIT ?
+    `).all(like, limit);
+    return rows.map((row, index) => enrichSearchRow(db, row, {
+      source_table: 'session_messages',
+      rank: index + 1000,
+    }));
+  } catch {
+    return [];
+  }
+}
+
+function searchSessionMetadata(db, query, limit) {
+  const results = [];
+  const like = `%${escapeLike(query)}%`;
+  if (hasTable(db, 'session_conversations')) {
+    const convIdCol = idColumnFor(db, 'session_conversations');
+    if (convIdCol) {
+      const cols = tableColumns(db, 'session_conversations');
+      const selectCols = [
+        `${quoteIdent(convIdCol)} AS conversation_id`,
+        cols.includes('project_path') ? 'project_path' : "'' AS project_path",
+        cols.includes('title') ? 'title' : "'' AS title",
+        cols.includes('first_message') ? 'first_message' : "'' AS first_message",
+        cols.includes('git_branch') ? 'git_branch' : "'' AS git_branch",
+        cols.includes('imported_at') ? 'imported_at' : "'' AS imported_at",
+      ];
+      const predicates = ['1=0'];
+      const params = [];
+      for (const col of ['title', 'first_message', 'project_path']) {
+        if (cols.includes(col)) {
+          predicates.push(`${quoteIdent(col)} LIKE ? ESCAPE '\\'`);
+          params.push(like);
+        }
+      }
+      try {
+        const rows = db.prepare(`
+          SELECT ${selectCols.join(', ')}
+          FROM session_conversations
+          WHERE ${predicates.join(' OR ')}
+          ORDER BY imported_at DESC
+          LIMIT ?
+        `).all(...params, limit);
+        results.push(...rows.map((row, index) => enrichMetadataRow(db, row, index)));
+      } catch {}
+    }
+  }
+
+  if (hasTable(db, 'ctm_sessions')) {
+    try {
+      const rows = db.prepare(`
+        SELECT id AS ctm_session_id, provider, project_path, cwd, title, updated_at
+        FROM ctm_sessions
+        WHERE id = ? OR title LIKE ? ESCAPE '\\' OR project_path LIKE ? ESCAPE '\\' OR cwd LIKE ? ESCAPE '\\'
+        ORDER BY updated_at DESC
+        LIMIT ?
+      `).all(query, like, like, like, limit);
+      results.push(...rows.map((row, index) => ({
+        source: 'ctm-db',
+        source_table: 'ctm_sessions',
+        session_id: row.ctm_session_id,
+        ctm_session_id: row.ctm_session_id,
+        provider: row.provider || '',
+        cwd: row.cwd || row.project_path || '',
+        title: row.title || '',
+        timestamp: row.updated_at || '',
+        snippet: row.title || row.cwd || row.project_path || row.ctm_session_id,
+        rank: index + 2000,
+      })));
+    } catch {}
+  }
+
+  if (hasTable(db, 'agent_sessions')) {
+    try {
+      const rows = db.prepare(`
+        SELECT agent_session_id, ctm_session_id, provider, project_path, first_message, modified_at, model, git_branch
+        FROM agent_sessions
+        WHERE agent_session_id = ? OR ctm_session_id = ? OR first_message LIKE ? ESCAPE '\\' OR project_path LIKE ? ESCAPE '\\'
+        ORDER BY modified_at DESC, updated_at DESC
+        LIMIT ?
+      `).all(query, query, like, like, limit);
+      results.push(...rows.map((row, index) => ({
+        source: 'ctm-db',
+        source_table: 'agent_sessions',
+        session_id: row.ctm_session_id || row.agent_session_id,
+        ctm_session_id: row.ctm_session_id || '',
+        agent_session_id: row.agent_session_id || '',
+        provider: row.provider || '',
+        cwd: row.project_path || '',
+        branch: row.git_branch || '',
+        model: row.model || '',
+        timestamp: row.modified_at || '',
+        snippet: row.first_message || row.project_path || row.agent_session_id,
+        rank: index + 3000,
+      })));
+    } catch {}
+  }
+
+  return results;
+}
+
+function enrichSearchRow(db, row, extra = {}) {
+  const resolved = resolveConversationOwner(db, row.conversation_id);
+  return {
+    source: 'ctm-db',
+    source_table: extra.source_table,
+    session_id: resolved.ctm_session_id || row.conversation_id,
+    ctm_session_id: resolved.ctm_session_id || '',
+    agent_session_id: resolved.agent_session_id || '',
+    conversation_id: row.conversation_id,
+    role: row.role || '',
+    message_index: row.message_index,
+    timestamp: row.created_at || resolved.modified_at || resolved.updated_at || '',
+    cwd: resolved.cwd || resolved.project_path || '',
+    branch: resolved.git_branch || '',
+    title: resolved.title || resolved.first_message || '',
+    provider: resolved.provider || '',
+    model: resolved.model || '',
+    snippet: snippetAround(row.content || '', 800),
+    content_hash: contentFingerprint(row.content || ''),
+    rank: extra.rank || 0,
+  };
+}
+
+function enrichMetadataRow(db, row, rank) {
+  const resolved = resolveConversationOwner(db, row.conversation_id);
+  return {
+    source: 'ctm-db',
+    source_table: 'session_conversations',
+    session_id: resolved.ctm_session_id || row.conversation_id,
+    ctm_session_id: resolved.ctm_session_id || '',
+    agent_session_id: resolved.agent_session_id || '',
+    conversation_id: row.conversation_id,
+    timestamp: row.imported_at || resolved.modified_at || '',
+    cwd: resolved.cwd || row.project_path || '',
+    branch: resolved.git_branch || row.git_branch || '',
+    title: resolved.title || row.title || row.first_message || '',
+    provider: resolved.provider || '',
+    model: resolved.model || '',
+    snippet: row.title || row.first_message || row.project_path || row.conversation_id,
+    content_hash: contentFingerprint([row.title, row.first_message, row.project_path].filter(Boolean).join('\n')),
+    rank: rank + 1500,
+  };
+}
+
+function resolveConversationOwner(db, conversationId) {
+  const id = String(conversationId || '');
+  const empty = { ctm_session_id: id, agent_session_id: '', provider: '', project_path: '', cwd: '', title: '' };
+  if (!id) return empty;
+
+  let agent = null;
+  if (hasTable(db, 'agent_sessions')) {
+    try {
+      agent = db.prepare('SELECT * FROM agent_sessions WHERE agent_session_id = ?').get(id) || null;
+    } catch {}
+  }
+  const ctmId = agent?.ctm_session_id || id;
+  let ctm = null;
+  if (hasTable(db, 'ctm_sessions')) {
+    try {
+      ctm = db.prepare('SELECT * FROM ctm_sessions WHERE id = ?').get(ctmId) || null;
+    } catch {}
+  }
+  if (!agent && hasTable(db, 'agent_sessions')) {
+    try {
+      agent = db.prepare('SELECT * FROM agent_sessions WHERE ctm_session_id = ? ORDER BY modified_at DESC, created_at DESC LIMIT 1').get(ctmId) || null;
+    } catch {}
+  }
+  return {
+    ctm_session_id: ctm?.id || agent?.ctm_session_id || ctmId,
+    agent_session_id: agent?.agent_session_id || (agent?.ctm_session_id ? id : ''),
+    provider: ctm?.provider || agent?.provider || '',
+    project_path: ctm?.project_path || agent?.project_path || '',
+    cwd: ctm?.cwd || ctm?.project_path || agent?.project_path || '',
+    title: ctm?.title || agent?.first_message || '',
+    first_message: agent?.first_message || '',
+    git_branch: agent?.git_branch || '',
+    model: agent?.model || '',
+    modified_at: agent?.modified_at || '',
+    updated_at: ctm?.updated_at || '',
+  };
+}
+
+function resolveSessionBundle(db, rawId) {
+  const candidates = candidateIds(rawId);
+  for (const candidate of candidates) {
+    const bundle = resolveSessionBundleExact(db, candidate, rawId);
+    if (bundle) return bundle;
+  }
+  return null;
+}
+
+function resolveSessionBundleExact(db, id, requestedId = id) {
+  const requested = String(requestedId || id || '').trim();
+  const cleanId = String(id || '').trim();
+  if (!cleanId) return null;
+
+  let ctm = null;
+  let agent = null;
+  if (hasTable(db, 'ctm_sessions')) {
+    try { ctm = db.prepare('SELECT * FROM ctm_sessions WHERE id = ?').get(cleanId) || null; } catch {}
+  }
+  if (hasTable(db, 'agent_sessions')) {
+    try { agent = db.prepare('SELECT * FROM agent_sessions WHERE agent_session_id = ?').get(cleanId) || null; } catch {}
+  }
+
+  const ctmId = ctm?.id || agent?.ctm_session_id || cleanId;
+  if (!ctm && hasTable(db, 'ctm_sessions')) {
+    try { ctm = db.prepare('SELECT * FROM ctm_sessions WHERE id = ?').get(ctmId) || null; } catch {}
+  }
+
+  let agents = [];
+  if (hasTable(db, 'agent_sessions')) {
+    try {
+      agents = db.prepare('SELECT * FROM agent_sessions WHERE ctm_session_id = ? ORDER BY modified_at DESC, created_at DESC').all(ctmId);
+    } catch {}
+  }
+  if (agent && !agents.some((row) => row.agent_session_id === agent.agent_session_id)) agents.unshift(agent);
+
+  const conversationIds = new Set();
+  for (const idCandidate of [cleanId, ctmId, ...agents.map((row) => row.agent_session_id)]) {
+    if (idCandidate && hasConversation(db, idCandidate)) conversationIds.add(idCandidate);
+  }
+
+  if (!ctm && !agent && conversationIds.size === 0) return null;
+  return {
+    requested_id: requested,
+    ctm_session_id: ctmId,
+    agent_session_ids: agents.map((row) => row.agent_session_id).filter(Boolean),
+    conversation_ids: [...conversationIds],
+    provider: ctm?.provider || agent?.provider || '',
+    project_path: ctm?.project_path || agent?.project_path || '',
+    cwd: ctm?.cwd || ctm?.project_path || agent?.project_path || '',
+    title: ctm?.title || agent?.first_message || '',
+    starred: Boolean(ctm?.starred),
+    user_renamed: Boolean(ctm?.user_renamed),
+    created_at: ctm?.created_at || agent?.created_at || '',
+    updated_at: ctm?.updated_at || agent?.updated_at || agent?.modified_at || '',
+    agents,
+  };
+}
+
+function hasConversation(db, sessionId) {
+  if (!hasTable(db, 'session_conversations')) return false;
+  const idCol = idColumnFor(db, 'session_conversations');
+  if (!idCol) return false;
+  try {
+    return Boolean(db.prepare(`SELECT 1 FROM session_conversations WHERE ${quoteIdent(idCol)} = ? LIMIT 1`).get(sessionId));
+  } catch {
+    return false;
+  }
+}
+
+function getConversationRows(db, bundle) {
+  if (!hasTable(db, 'session_conversations')) return [];
+  const idCol = idColumnFor(db, 'session_conversations');
+  if (!idCol) return [];
+  const ids = bundle.conversation_ids && bundle.conversation_ids.length
+    ? bundle.conversation_ids
+    : [bundle.ctm_session_id, ...bundle.agent_session_ids].filter(Boolean);
+  const rows = [];
+  const seen = new Set();
+  for (const id of ids) {
+    if (seen.has(id)) continue;
+    seen.add(id);
+    try {
+      const row = db.prepare(`SELECT *, ${quoteIdent(idCol)} AS _conversation_id FROM session_conversations WHERE ${quoteIdent(idCol)} = ?`).get(id);
+      if (row) rows.push(row);
+    } catch {}
+  }
+  return rows.sort((a, b) => compareTimestamp(a.session_created_at || a.imported_at, b.session_created_at || b.imported_at));
+}
+
+function parseConversationMessages(row) {
+  try {
+    const parsed = JSON.parse(row?.messages || '[]');
+    return Array.isArray(parsed) ? parsed : [];
+  } catch {
+    return [];
+  }
+}
+
+function normalizeMessage(raw, { row, bundle, index, includeRaw }) {
+  const role = normalizeRole(raw);
+  const content = messageText(raw);
+  const conversationId = row?._conversation_id || row?.ctm_session_id || row?.session_id || bundle.ctm_session_id;
+  return {
+    source: 'ctm-db',
+    source_table: 'session_conversations',
+    session_id: bundle.ctm_session_id,
+    ctm_session_id: bundle.ctm_session_id,
+    agent_session_id: conversationId !== bundle.ctm_session_id ? conversationId : '',
+    conversation_id: conversationId,
+    message_index: Number.isInteger(raw?.message_index) ? raw.message_index : index,
+    role,
+    timestamp: raw?.timestamp || raw?.created_at || row?.session_created_at || row?.imported_at || '',
+    content,
+    content_hash: contentFingerprint(`${role}\n${content}`),
+    metadata: compactObject({
+      provider: bundle.provider || row?.model_provider || '',
+      model: row?.model_id || '',
+      title: bundle.title || row?.title || '',
+      cwd: bundle.cwd || row?.project_path || '',
+      branch: row?.git_branch || '',
+      tool_calls: raw?.tool_calls || raw?.toolCalls,
+      files: raw?.files || raw?.filesEdited,
+    }),
+    raw: includeRaw ? raw : undefined,
+  };
+}
+
+function normalizeRole(raw) {
+  const role = raw?.role || raw?.speaker || raw?.type || '';
+  if (role === 'human') return 'user';
+  if (role === 'ai') return 'assistant';
+  if (role === 'tool_result') return 'tool';
+  return String(role || 'unknown').toLowerCase();
+}
+
+function messageText(raw) {
+  if (!raw) return '';
+  if (typeof raw.content === 'string') return raw.content.trim();
+  if (Array.isArray(raw.content)) {
+    return raw.content.map((part) => {
+      if (!part) return '';
+      if (typeof part === 'string') return part;
+      if (typeof part.text === 'string') return part.text;
+      if (typeof part.content === 'string') return part.content;
+      if (typeof part.input === 'object') return JSON.stringify(part.input);
+      return '';
+    }).filter(Boolean).join('\n').trim();
+  }
+  if (typeof raw.text === 'string') return raw.text.trim();
+  if (typeof raw.message === 'string') return raw.message.trim();
+  if (raw.message && typeof raw.message.content === 'string') return raw.message.content.trim();
+  return '';
+}
+
+function formatSessionSummary(bundle, conversations, messages) {
+  const firstConversation = conversations[0] || {};
+  return {
+    session_id: bundle.ctm_session_id,
+    ctm_session_id: bundle.ctm_session_id,
+    agent_session_ids: bundle.agent_session_ids || [],
+    conversation_ids: conversations.map((row) => row._conversation_id || row.ctm_session_id || row.session_id).filter(Boolean),
+    provider: bundle.provider || firstConversation.model_provider || '',
+    cwd: bundle.cwd || firstConversation.project_path || '',
+    branch: firstConversation.git_branch || bundle.agents?.find((a) => a.git_branch)?.git_branch || '',
+    title: bundle.title || firstConversation.title || firstConversation.first_message || '',
+    created_at: bundle.created_at || firstConversation.session_created_at || '',
+    updated_at: bundle.updated_at || firstConversation.imported_at || '',
+    message_count: messages.length,
+    user_msg_count: firstConversation.user_msg_count || messages.filter((msg) => msg.role === 'user').length,
+    assistant_msg_count: firstConversation.assistant_msg_count || messages.filter((msg) => msg.role === 'assistant').length,
+  };
+}
+
+function orderMessages(messages) {
+  return [...messages].sort((a, b) => {
+    const t = compareTimestamp(a.timestamp, b.timestamp);
+    if (t !== 0) return t;
+    return (a.message_index || 0) - (b.message_index || 0);
+  });
+}
+
+function compareTimestamp(a, b) {
+  const at = Date.parse(a || '');
+  const bt = Date.parse(b || '');
+  if (Number.isFinite(at) && Number.isFinite(bt) && at !== bt) return at - bt;
+  if (Number.isFinite(at) && !Number.isFinite(bt)) return -1;
+  if (!Number.isFinite(at) && Number.isFinite(bt)) return 1;
+  return 0;
+}
+
+function dedupeMessages(messages) {
+  return dedupeItems(messages, {
+    contentFields: ['content'],
+    keyPrefix: (item) => item.role || '',
+    minContentLength: 12,
+  });
+}
+
+function dedupeItems(items, {
+  limit = 0,
+  contentFields = ['content'],
+  keyPrefix = () => '',
+  minContentLength = 0,
+} = {}) {
+  const seen = new Map();
+  for (const item of items || []) {
+    const content = contentFields.map((field) => item?.[field]).find((value) => typeof value === 'string' && value.trim()) || '';
+    const normalized = normalizeContent(content);
+    const prefix = typeof keyPrefix === 'function' ? keyPrefix(item) : String(keyPrefix || '');
+    const key = normalized.length >= minContentLength
+      ? `${prefix}:${contentFingerprint(normalized)}`
+      : `${prefix}:${item.session_id || ''}:${item.conversation_id || ''}:${item.message_index ?? item.id ?? normalized}`;
+
+    if (!seen.has(key)) {
+      seen.set(key, { ...item });
+      continue;
+    }
+    const existing = seen.get(key);
+    seen.set(key, mergeDuplicateItem(existing, item));
+  }
+  const deduped = [...seen.values()].sort((a, b) => (a.rank || 0) - (b.rank || 0));
+  return limit ? deduped.slice(0, limit) : deduped;
+}
+
+function mergeDuplicateItem(existing, next) {
+  const merged = { ...existing };
+  const provenance = [
+    ...(Array.isArray(existing.provenance) ? existing.provenance : [provenanceOf(existing)]),
+    provenanceOf(next),
+  ].filter(Boolean);
+  merged.provenance = dedupeProvenance(provenance);
+  if (!merged.agent_session_id && next.agent_session_id) merged.agent_session_id = next.agent_session_id;
+  if (!merged.ctm_session_id && next.ctm_session_id) merged.ctm_session_id = next.ctm_session_id;
+  if (!merged.conversation_id && next.conversation_id) merged.conversation_id = next.conversation_id;
+  if (!merged.timestamp && next.timestamp) merged.timestamp = next.timestamp;
+  if ((String(next.snippet || '').length > String(merged.snippet || '').length) && String(next.snippet || '').length <= 1200) {
+    merged.snippet = next.snippet;
+  }
+  return merged;
+}
+
+function provenanceOf(item) {
+  if (!item) return null;
+  return compactObject({
+    source: item.source,
+    source_table: item.source_table,
+    session_id: item.session_id,
+    ctm_session_id: item.ctm_session_id,
+    agent_session_id: item.agent_session_id,
+    conversation_id: item.conversation_id,
+    message_index: item.message_index,
+    id: item.id,
+  });
+}
+
+function dedupeProvenance(items) {
+  const seen = new Set();
+  const out = [];
+  for (const item of items) {
+    const key = JSON.stringify(item);
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(item);
+  }
+  return out;
+}
+
+function normalizeContent(value) {
+  return String(value || '')
+    .replace(/\r\n/g, '\n')
+    .replace(/\s+/g, ' ')
+    .trim()
+    .toLowerCase();
+}
+
+function contentFingerprint(value) {
+  return crypto.createHash('sha256').update(String(value || '')).digest('hex').slice(0, 16);
+}
+
+function buildFtsQueries(query) {
+  const terms = String(query || '')
+    .match(/[A-Za-z0-9_./:-]+/g)
+    ?.map((term) => term.replace(/^[-:.\\/]+|[-:.\\/]+$/g, ''))
+    .filter((term) => term.length > 1)
+    .slice(0, 12) || [];
+  if (!terms.length) return [];
+  const quoted = terms.map((term) => `"${term.replace(/"/g, '""')}"`);
+  const andQuery = quoted.join(' AND ');
+  const orQuery = quoted.join(' OR ');
+  return andQuery === orQuery ? [andQuery] : [andQuery, orQuery];
+}
+
+function escapeLike(value) {
+  return String(value || '').replace(/[\\%_]/g, (match) => `\\${match}`);
+}
+
+function snippetAround(content, max = 800) {
+  const text = String(content || '').replace(/\s+/g, ' ').trim();
+  if (text.length <= max) return text;
+  return `${text.slice(0, Math.max(0, max - 1)).trim()}...`;
+}
+
+function normalizeIdentifiers(value) {
+  if (Array.isArray(value)) return value.flatMap((item) => normalizeIdentifiers(item));
+  if (value == null) return [];
+  return String(value).split(',').map((part) => part.trim()).filter(Boolean);
+}
+
+function candidateIds(id) {
+  const raw = String(id || '').trim();
+  if (!raw) return [];
+  const out = [raw];
+  const parts = raw.split(':').filter(Boolean);
+  if (parts.length > 1) {
+    out.push(parts[parts.length - 1]);
+    if (parts.length >= 4 && parts[0] === 'diary') out.push(parts.slice(2, -1).join(':'));
+    if (parts.length >= 2) out.push(parts.slice(1).join(':'));
+  }
+  return [...new Set(out.filter(Boolean))];
+}
+
+function clampLimit(value, fallback, max) {
+  const n = Number(value || fallback);
+  if (!Number.isFinite(n)) return fallback;
+  return Math.min(Math.max(Math.trunc(n), 1), max);
+}
+
+function normalizeContextLimit(value) {
+  if (value === 0 || value === '0' || value === 'all' || value === 'full') return 0;
+  return clampLimit(value, DEFAULT_CONTEXT_LIMIT, MAX_CONTEXT_LIMIT);
+}
+
+function trimContextToBudget(context, budgetChars) {
+  const sessions = context.sessions || [];
+  const messages = [];
+  let used = JSON.stringify(sessions).length;
+  let truncated = false;
+  for (const message of context.messages || []) {
+    const size = String(message.content || '').length + 200;
+    if (used + size > budgetChars) {
+      truncated = true;
+      break;
+    }
+    messages.push(message);
+    used += size;
+  }
+  return {
+    ...context,
+    sessions,
+    messages,
+    count: messages.length,
+    truncated,
+  };
+}
+
+function renderContextMarkdown(context, { compact = false } = {}) {
+  const lines = [];
+  lines.push('# Wall-E Session Context');
+  lines.push('');
+  if (context.task) {
+    lines.push(`Task: ${context.task}`);
+    lines.push('');
+  }
+  if (context.sessions?.length) {
+    lines.push('## Sessions');
+    for (const session of context.sessions) {
+      const title = session.title ? ` - ${session.title}` : '';
+      const cwd = session.cwd ? ` (${session.cwd})` : '';
+      lines.push(`- ${session.session_id}${title}${cwd}; messages=${session.message_count}`);
+    }
+    lines.push('');
+  }
+  lines.push('## Messages');
+  const maxContent = compact ? 1200 : 8000;
+  for (const msg of context.messages || []) {
+    const label = [msg.role || 'unknown', msg.timestamp || '', msg.session_id || ''].filter(Boolean).join(' | ');
+    lines.push(`### ${label}`);
+    const content = String(msg.content || '');
+    lines.push(content.length > maxContent ? `${content.slice(0, maxContent).trim()}...` : content);
+    lines.push('');
+  }
+  if (context.transfer?.truncated || context.truncated) {
+    lines.push('_Context pack truncated to fit the requested budget._');
+  }
+  return lines.join('\n').trim();
+}
+
+function compactObject(obj) {
+  const out = {};
+  for (const [key, value] of Object.entries(obj || {})) {
+    if (value === undefined || value === null || value === '') continue;
+    if (Array.isArray(value) && value.length === 0) continue;
+    out[key] = value;
+  }
+  return out;
+}
+
+module.exports = {
+  resolveCtmDbPath,
+  openCtmDb,
+  searchCtmSessions,
+  getCtmSessionContext,
+  buildContextPack,
+  dedupeItems,
+  dedupeMessages,
+  normalizeContent,
+  contentFingerprint,
+  renderContextMarkdown,
+};