@xortex/xcode 3.0.0

package/services/autoDream/autoDream.ts

@@ -0,0 +1,324 @@
+// biome-ignore-all assist/source/organizeImports: ANT-ONLY import markers must not be reordered
+// Background memory consolidation. Fires the /dream prompt as a forked
+// subagent when time-gate passes AND enough sessions have accumulated.
+//
+// Gate order (cheapest first):
+//   1. Time: hours since lastConsolidatedAt >= minHours (one stat)
+//   2. Sessions: transcript count with mtime > lastConsolidatedAt >= minSessions
+//   3. Lock: no other process mid-consolidation
+//
+// State is closure-scoped inside initAutoDream() rather than module-level
+// (tests call initAutoDream() in beforeEach for a fresh closure).
+
+import type { REPLHookContext } from '../../utils/hooks/postSamplingHooks.js'
+import {
+  createCacheSafeParams,
+  runForkedAgent,
+} from '../../utils/forkedAgent.js'
+import {
+  createUserMessage,
+  createMemorySavedMessage,
+} from '../../utils/messages.js'
+import type { Message } from '../../types/message.js'
+import { logForDebugging } from '../../utils/debug.js'
+import type { ToolUseContext } from '../../Tool.js'
+import { logEvent } from '../analytics/index.js'
+import { getFeatureValue_CACHED_MAY_BE_STALE } from '../analytics/growthbook.js'
+import { isAutoMemoryEnabled, getAutoMemPath } from '../../memdir/paths.js'
+import { isAutoDreamEnabled } from './config.js'
+import { getProjectDir } from '../../utils/sessionStorage.js'
+import {
+  getOriginalCwd,
+  getKairosActive,
+  getIsRemoteMode,
+  getSessionId,
+} from '../../bootstrap/state.js'
+import { createAutoMemCanUseTool } from '../extractMemories/extractMemories.js'
+import { buildConsolidationPrompt } from './consolidationPrompt.js'
+import {
+  readLastConsolidatedAt,
+  listSessionsTouchedSince,
+  tryAcquireConsolidationLock,
+  rollbackConsolidationLock,
+} from './consolidationLock.js'
+import {
+  registerDreamTask,
+  addDreamTurn,
+  completeDreamTask,
+  failDreamTask,
+  isDreamTask,
+} from '../../tasks/DreamTask/DreamTask.js'
+import { FILE_EDIT_TOOL_NAME } from '../../tools/FileEditTool/constants.js'
+import { FILE_WRITE_TOOL_NAME } from '../../tools/FileWriteTool/prompt.js'
+
+// Scan throttle: when time-gate passes but session-gate doesn't, the lock
+// mtime doesn't advance, so the time-gate keeps passing every turn.
+const SESSION_SCAN_INTERVAL_MS = 10 * 60 * 1000
+
+type AutoDreamConfig = {
+  minHours: number
+  minSessions: number
+}
+
+const DEFAULTS: AutoDreamConfig = {
+  minHours: 24,
+  minSessions: 5,
+}
+
+/**
+ * Thresholds from tengu_onyx_plover. The enabled gate lives in config.ts
+ * (isAutoDreamEnabled); this returns only the scheduling knobs. Defensive
+ * per-field validation since GB cache can return stale wrong-type values.
+ */
+function getConfig(): AutoDreamConfig {
+  const raw =
+    getFeatureValue_CACHED_MAY_BE_STALE<Partial<AutoDreamConfig> | null>(
+      'tengu_onyx_plover',
+      null,
+    )
+  return {
+    minHours:
+      typeof raw?.minHours === 'number' &&
+      Number.isFinite(raw.minHours) &&
+      raw.minHours > 0
+        ? raw.minHours
+        : DEFAULTS.minHours,
+    minSessions:
+      typeof raw?.minSessions === 'number' &&
+      Number.isFinite(raw.minSessions) &&
+      raw.minSessions > 0
+        ? raw.minSessions
+        : DEFAULTS.minSessions,
+  }
+}
+
+function isGateOpen(): boolean {
+  if (getKairosActive()) return false // KAIROS mode uses disk-skill dream
+  if (getIsRemoteMode()) return false
+  if (!isAutoMemoryEnabled()) return false
+  return isAutoDreamEnabled()
+}
+
+// Ant-build-only test override. Bypasses enabled/time/session gates but NOT
+// the lock (so repeated turns don't pile up dreams) or the memory-dir
+// precondition. Still scans sessions so the prompt's session-hint is populated.
+function isForced(): boolean {
+  return false
+}
+
+type AppendSystemMessageFn = NonNullable<ToolUseContext['appendSystemMessage']>
+
+let runner:
+  | ((
+      context: REPLHookContext,
+      appendSystemMessage?: AppendSystemMessageFn,
+    ) => Promise<void>)
+  | null = null
+
+/**
+ * Call once at startup (from backgroundHousekeeping alongside
+ * initExtractMemories), or per-test in beforeEach for a fresh closure.
+ */
+export function initAutoDream(): void {
+  let lastSessionScanAt = 0
+
+  runner = async function runAutoDream(context, appendSystemMessage) {
+    const cfg = getConfig()
+    const force = isForced()
+    if (!force && !isGateOpen()) return
+
+    // --- Time gate ---
+    let lastAt: number
+    try {
+      lastAt = await readLastConsolidatedAt()
+    } catch (e: unknown) {
+      logForDebugging(
+        `[autoDream] readLastConsolidatedAt failed: ${(e as Error).message}`,
+      )
+      return
+    }
+    const hoursSince = (Date.now() - lastAt) / 3_600_000
+    if (!force && hoursSince < cfg.minHours) return
+
+    // --- Scan throttle ---
+    const sinceScanMs = Date.now() - lastSessionScanAt
+    if (!force && sinceScanMs < SESSION_SCAN_INTERVAL_MS) {
+      logForDebugging(
+        `[autoDream] scan throttle — time-gate passed but last scan was ${Math.round(sinceScanMs / 1000)}s ago`,
+      )
+      return
+    }
+    lastSessionScanAt = Date.now()
+
+    // --- Session gate ---
+    let sessionIds: string[]
+    try {
+      sessionIds = await listSessionsTouchedSince(lastAt)
+    } catch (e: unknown) {
+      logForDebugging(
+        `[autoDream] listSessionsTouchedSince failed: ${(e as Error).message}`,
+      )
+      return
+    }
+    // Exclude the current session (its mtime is always recent).
+    const currentSession = getSessionId()
+    sessionIds = sessionIds.filter(id => id !== currentSession)
+    if (!force && sessionIds.length < cfg.minSessions) {
+      logForDebugging(
+        `[autoDream] skip — ${sessionIds.length} sessions since last consolidation, need ${cfg.minSessions}`,
+      )
+      return
+    }
+
+    // --- Lock ---
+    // Under force, skip acquire entirely — use the existing mtime so
+    // kill's rollback is a no-op (rewinds to where it already is).
+    // The lock file stays untouched; next non-force turn sees it as-is.
+    let priorMtime: number | null
+    if (force) {
+      priorMtime = lastAt
+    } else {
+      try {
+        priorMtime = await tryAcquireConsolidationLock()
+      } catch (e: unknown) {
+        logForDebugging(
+          `[autoDream] lock acquire failed: ${(e as Error).message}`,
+        )
+        return
+      }
+      if (priorMtime === null) return
+    }
+
+    logForDebugging(
+      `[autoDream] firing — ${hoursSince.toFixed(1)}h since last, ${sessionIds.length} sessions to review`,
+    )
+    logEvent('tengu_auto_dream_fired', {
+      hours_since: Math.round(hoursSince),
+      sessions_since: sessionIds.length,
+    })
+
+    const setAppState =
+      context.toolUseContext.setAppStateForTasks ??
+      context.toolUseContext.setAppState
+    const abortController = new AbortController()
+    const taskId = registerDreamTask(setAppState, {
+      sessionsReviewing: sessionIds.length,
+      priorMtime,
+      abortController,
+    })
+
+    try {
+      const memoryRoot = getAutoMemPath()
+      const transcriptDir = getProjectDir(getOriginalCwd())
+      // Tool constraints note goes in `extra`, not the shared prompt body —
+      // manual /dream runs in the main loop with normal permissions and this
+      // would be misleading there.
+      const extra = `
+
+**Tool constraints for this run:** Bash is restricted to read-only commands (\`ls\`, \`find\`, \`grep\`, \`cat\`, \`stat\`, \`wc\`, \`head\`, \`tail\`, and similar). Anything that writes, redirects to a file, or modifies state will be denied. Plan your exploration with this in mind — no need to probe.
+
+Sessions since last consolidation (${sessionIds.length}):
+${sessionIds.map(id => `- ${id}`).join('\n')}`
+      const prompt = buildConsolidationPrompt(memoryRoot, transcriptDir, extra)
+
+      const result = await runForkedAgent({
+        promptMessages: [createUserMessage({ content: prompt })],
+        cacheSafeParams: createCacheSafeParams(context),
+        canUseTool: createAutoMemCanUseTool(memoryRoot),
+        querySource: 'auto_dream',
+        forkLabel: 'auto_dream',
+        skipTranscript: true,
+        overrides: { abortController },
+        onMessage: makeDreamProgressWatcher(taskId, setAppState),
+      })
+
+      completeDreamTask(taskId, setAppState)
+      // Inline completion summary in the main transcript (same surface as
+      // extractMemories's "Saved N memories" message).
+      const dreamState = context.toolUseContext.getAppState().tasks?.[taskId]
+      if (
+        appendSystemMessage &&
+        isDreamTask(dreamState) &&
+        dreamState.filesTouched.length > 0
+      ) {
+        appendSystemMessage({
+          ...createMemorySavedMessage(dreamState.filesTouched),
+          verb: 'Improved',
+        })
+      }
+      logForDebugging(
+        `[autoDream] completed — cache: read=${result.totalUsage.cache_read_input_tokens} created=${result.totalUsage.cache_creation_input_tokens}`,
+      )
+      logEvent('tengu_auto_dream_completed', {
+        cache_read: result.totalUsage.cache_read_input_tokens,
+        cache_created: result.totalUsage.cache_creation_input_tokens,
+        output: result.totalUsage.output_tokens,
+        sessions_reviewed: sessionIds.length,
+      })
+    } catch (e: unknown) {
+      // If the user killed from the bg-tasks dialog, DreamTask.kill already
+      // aborted, rolled back the lock, and set status=killed. Don't overwrite
+      // or double-rollback.
+      if (abortController.signal.aborted) {
+        logForDebugging('[autoDream] aborted by user')
+        return
+      }
+      logForDebugging(`[autoDream] fork failed: ${(e as Error).message}`)
+      logEvent('tengu_auto_dream_failed', {})
+      failDreamTask(taskId, setAppState)
+      // Rewind mtime so time-gate passes again. Scan throttle is the backoff.
+      await rollbackConsolidationLock(priorMtime)
+    }
+  }
+}
+
+/**
+ * Watch the forked agent's messages. For each assistant turn, extracts any
+ * text blocks (the agent's reasoning/summary — what the user wants to see)
+ * and collapses tool_use blocks to a count. Edit/Write file_paths are
+ * collected for phase-flip + the inline completion message.
+ */
+function makeDreamProgressWatcher(
+  taskId: string,
+  setAppState: import('../../Task.js').SetAppState,
+): (msg: Message) => void {
+  return msg => {
+    if (msg.type !== 'assistant') return
+    let text = ''
+    let toolUseCount = 0
+    const touchedPaths: string[] = []
+    for (const block of msg.message.content) {
+      if (block.type === 'text') {
+        text += block.text
+      } else if (block.type === 'tool_use') {
+        toolUseCount++
+        if (
+          block.name === FILE_EDIT_TOOL_NAME ||
+          block.name === FILE_WRITE_TOOL_NAME
+        ) {
+          const input = block.input as { file_path?: unknown }
+          if (typeof input.file_path === 'string') {
+            touchedPaths.push(input.file_path)
+          }
+        }
+      }
+    }
+    addDreamTurn(
+      taskId,
+      { text: text.trim(), toolUseCount },
+      touchedPaths,
+      setAppState,
+    )
+  }
+}
+
+/**
+ * Entry point from stopHooks. No-op until initAutoDream() has been called.
+ * Per-turn cost when enabled: one GB cache read + one stat.
+ */
+export async function executeAutoDream(
+  context: REPLHookContext,
+  appendSystemMessage?: AppendSystemMessageFn,
+): Promise<void> {
+  await runner?.(context, appendSystemMessage)
+}

package/services/autoDream/config.ts

@@ -0,0 +1,21 @@
+// Leaf config module — intentionally minimal imports so UI components
+// can read the auto-dream enabled state without dragging in the forked
+// agent / task registry / message builder chain that autoDream.ts pulls in.
+
+import { getInitialSettings } from '../../utils/settings/settings.js'
+import { getFeatureValue_CACHED_MAY_BE_STALE } from '../analytics/growthbook.js'
+
+/**
+ * Whether background memory consolidation should run. User setting
+ * (autoDreamEnabled in settings.json) overrides the GrowthBook default
+ * when explicitly set; otherwise falls through to tengu_onyx_plover.
+ */
+export function isAutoDreamEnabled(): boolean {
+  const setting = getInitialSettings().autoDreamEnabled
+  if (setting !== undefined) return setting
+  const gb = getFeatureValue_CACHED_MAY_BE_STALE<{ enabled?: unknown } | null>(
+    'tengu_onyx_plover',
+    null,
+  )
+  return gb?.enabled === true
+}

package/services/autoDream/consolidationLock.ts

@@ -0,0 +1,140 @@
+// Lock file whose mtime IS lastConsolidatedAt. Body is the holder's PID.
+//
+// Lives inside the memory dir (getAutoMemPath) so it keys on git-root
+// like memory does, and so it's writable even when the memory path comes
+// from an env/settings override whose parent may not be.
+
+import { mkdir, readFile, stat, unlink, utimes, writeFile } from 'fs/promises'
+import { join } from 'path'
+import { getOriginalCwd } from '../../bootstrap/state.js'
+import { getAutoMemPath } from '../../memdir/paths.js'
+import { logForDebugging } from '../../utils/debug.js'
+import { isProcessRunning } from '../../utils/genericProcessUtils.js'
+import { listCandidates } from '../../utils/listSessionsImpl.js'
+import { getProjectDir } from '../../utils/sessionStorage.js'
+
+const LOCK_FILE = '.consolidate-lock'
+
+// Stale past this even if the PID is live (PID reuse guard).
+const HOLDER_STALE_MS = 60 * 60 * 1000
+
+function lockPath(): string {
+  return join(getAutoMemPath(), LOCK_FILE)
+}
+
+/**
+ * mtime of the lock file = lastConsolidatedAt. 0 if absent.
+ * Per-turn cost: one stat.
+ */
+export async function readLastConsolidatedAt(): Promise<number> {
+  try {
+    const s = await stat(lockPath())
+    return s.mtimeMs
+  } catch {
+    return 0
+  }
+}
+
+/**
+ * Acquire: write PID → mtime = now. Returns the pre-acquire mtime
+ * (for rollback), or null if blocked / lost a race.
+ *
+ *   Success → do nothing. mtime stays at now.
+ *   Failure → rollbackConsolidationLock(priorMtime) rewinds mtime.
+ *   Crash   → mtime stuck, dead PID → next process reclaims.
+ */
+export async function tryAcquireConsolidationLock(): Promise<number | null> {
+  const path = lockPath()
+
+  let mtimeMs: number | undefined
+  let holderPid: number | undefined
+  try {
+    const [s, raw] = await Promise.all([stat(path), readFile(path, 'utf8')])
+    mtimeMs = s.mtimeMs
+    const parsed = parseInt(raw.trim(), 10)
+    holderPid = Number.isFinite(parsed) ? parsed : undefined
+  } catch {
+    // ENOENT — no prior lock.
+  }
+
+  if (mtimeMs !== undefined && Date.now() - mtimeMs < HOLDER_STALE_MS) {
+    if (holderPid !== undefined && isProcessRunning(holderPid)) {
+      logForDebugging(
+        `[autoDream] lock held by live PID ${holderPid} (mtime ${Math.round((Date.now() - mtimeMs) / 1000)}s ago)`,
+      )
+      return null
+    }
+    // Dead PID or unparseable body — reclaim.
+  }
+
+  // Memory dir may not exist yet.
+  await mkdir(getAutoMemPath(), { recursive: true })
+  await writeFile(path, String(process.pid))
+
+  // Two reclaimers both write → last wins the PID. Loser bails on re-read.
+  let verify: string
+  try {
+    verify = await readFile(path, 'utf8')
+  } catch {
+    return null
+  }
+  if (parseInt(verify.trim(), 10) !== process.pid) return null
+
+  return mtimeMs ?? 0
+}
+
+/**
+ * Rewind mtime to pre-acquire after a failed fork. Clears the PID body —
+ * otherwise our still-running process would look like it's holding.
+ * priorMtime 0 → unlink (restore no-file).
+ */
+export async function rollbackConsolidationLock(
+  priorMtime: number,
+): Promise<void> {
+  const path = lockPath()
+  try {
+    if (priorMtime === 0) {
+      await unlink(path)
+      return
+    }
+    await writeFile(path, '')
+    const t = priorMtime / 1000 // utimes wants seconds
+    await utimes(path, t, t)
+  } catch (e: unknown) {
+    logForDebugging(
+      `[autoDream] rollback failed: ${(e as Error).message} — next trigger delayed to minHours`,
+    )
+  }
+}
+
+/**
+ * Session IDs with mtime after sinceMs. listCandidates handles UUID
+ * validation (excludes agent-*.jsonl) and parallel stat.
+ *
+ * Uses mtime (sessions TOUCHED since), not birthtime (0 on ext4).
+ * Caller excludes the current session. Scans per-cwd transcripts — it's
+ * a skip-gate, so undercounting worktree sessions is safe.
+ */
+export async function listSessionsTouchedSince(
+  sinceMs: number,
+): Promise<string[]> {
+  const dir = getProjectDir(getOriginalCwd())
+  const candidates = await listCandidates(dir, true)
+  return candidates.filter(c => c.mtime > sinceMs).map(c => c.sessionId)
+}
+
+/**
+ * Stamp from manual /dream. Optimistic — fires at prompt-build time,
+ * no post-skill completion hook. Best-effort.
+ */
+export async function recordConsolidation(): Promise<void> {
+  try {
+    // Memory dir may not exist yet (manual /dream before any auto-trigger).
+    await mkdir(getAutoMemPath(), { recursive: true })
+    await writeFile(lockPath(), String(process.pid))
+  } catch (e: unknown) {
+    logForDebugging(
+      `[autoDream] recordConsolidation write failed: ${(e as Error).message}`,
+    )
+  }
+}

package/services/autoDream/consolidationPrompt.ts

@@ -0,0 +1,65 @@
+// Extracted from dream.ts so auto-dream ships independently of KAIROS
+// feature flags (dream.ts is behind a feature()-gated require).
+
+import {
+  DIR_EXISTS_GUIDANCE,
+  ENTRYPOINT_NAME,
+  MAX_ENTRYPOINT_LINES,
+} from '../../memdir/memdir.js'
+
+export function buildConsolidationPrompt(
+  memoryRoot: string,
+  transcriptDir: string,
+  extra: string,
+): string {
+  return `# Dream: Memory Consolidation
+
+You are performing a dream — a reflective pass over your memory files. Synthesize what you've learned recently into durable, well-organized memories so that future sessions can orient quickly.
+
+Memory directory: \`${memoryRoot}\`
+${DIR_EXISTS_GUIDANCE}
+
+Session transcripts: \`${transcriptDir}\` (large JSONL files — grep narrowly, don't read whole files)
+
+---
+
+## Phase 1 — Orient
+
+- \`ls\` the memory directory to see what already exists
+- Read \`${ENTRYPOINT_NAME}\` to understand the current index
+- Skim existing topic files so you improve them rather than creating duplicates
+- If \`logs/\` or \`sessions/\` subdirectories exist (assistant-mode layout), review recent entries there
+
+## Phase 2 — Gather recent signal
+
+Look for new information worth persisting. Sources in rough priority order:
+
+1. **Daily logs** (\`logs/YYYY/MM/YYYY-MM-DD.md\`) if present — these are the append-only stream
+2. **Existing memories that drifted** — facts that contradict something you see in the codebase now
+3. **Transcript search** — if you need specific context (e.g., "what was the error message from yesterday's build failure?"), grep the JSONL transcripts for narrow terms:
+   \`grep -rn "<narrow term>" ${transcriptDir}/ --include="*.jsonl" | tail -50\`
+
+Don't exhaustively read transcripts. Look only for things you already suspect matter.
+
+## Phase 3 — Consolidate
+
+For each thing worth remembering, write or update a memory file at the top level of the memory directory. Use the memory file format and type conventions from your system prompt's auto-memory section — it's the source of truth for what to save, how to structure it, and what NOT to save.
+
+Focus on:
+- Merging new signal into existing topic files rather than creating near-duplicates
+- Converting relative dates ("yesterday", "last week") to absolute dates so they remain interpretable after time passes
+- Deleting contradicted facts — if today's investigation disproves an old memory, fix it at the source
+
+## Phase 4 — Prune and index
+
+Update \`${ENTRYPOINT_NAME}\` so it stays under ${MAX_ENTRYPOINT_LINES} lines AND under ~25KB. It's an **index**, not a dump — each entry should be one line under ~150 characters: \`- [Title](file.md) — one-line hook\`. Never write memory content directly into it.
+
+- Remove pointers to memories that are now stale, wrong, or superseded
+- Demote verbose entries: if an index line is over ~200 chars, it's carrying content that belongs in the topic file — shorten the line, move the detail
+- Add pointers to newly important memories
+- Resolve contradictions — if two files disagree, fix the wrong one
+
+---
+
+Return a brief summary of what you consolidated, updated, or pruned. If nothing changed (memories are already tight), say so.${extra ? `\n\n## Additional context\n\n${extra}` : ''}`
+}

package/services/awaySummary.ts

@@ -0,0 +1,74 @@
+import { APIUserAbortError } from '@anthropic-ai/sdk'
+import { getEmptyToolPermissionContext } from '../Tool.js'
+import type { Message } from '../types/message.js'
+import { logForDebugging } from '../utils/debug.js'
+import {
+  createUserMessage,
+  getAssistantMessageText,
+} from '../utils/messages.js'
+import { getSmallFastModel } from '../utils/model/model.js'
+import { asSystemPrompt } from '../utils/systemPromptType.js'
+import { queryModelWithoutStreaming } from './api/claude.js'
+import { getSessionMemoryContent } from './SessionMemory/sessionMemoryUtils.js'
+
+// Recap only needs recent context — truncate to avoid "prompt too long" on
+// large sessions. 30 messages ≈ ~15 exchanges, plenty for "where we left off."
+const RECENT_MESSAGE_WINDOW = 30
+
+function buildAwaySummaryPrompt(memory: string | null): string {
+  const memoryBlock = memory
+    ? `Session memory (broader context):\n${memory}\n\n`
+    : ''
+  return `${memoryBlock}The user stepped away and is coming back. Write exactly 1-3 short sentences. Start by stating the high-level task — what they are building or debugging, not implementation details. Next: the concrete next step. Skip status reports and commit recaps.`
+}
+
+/**
+ * Generates a short session recap for the "while you were away" card.
+ * Returns null on abort, empty transcript, or error.
+ */
+export async function generateAwaySummary(
+  messages: readonly Message[],
+  signal: AbortSignal,
+): Promise<string | null> {
+  if (messages.length === 0) {
+    return null
+  }
+
+  try {
+    const memory = await getSessionMemoryContent()
+    const recent = messages.slice(-RECENT_MESSAGE_WINDOW)
+    recent.push(createUserMessage({ content: buildAwaySummaryPrompt(memory) }))
+    const response = await queryModelWithoutStreaming({
+      messages: recent,
+      systemPrompt: asSystemPrompt([]),
+      thinkingConfig: { type: 'disabled' },
+      tools: [],
+      signal,
+      options: {
+        getToolPermissionContext: async () => getEmptyToolPermissionContext(),
+        model: getSmallFastModel(),
+        toolChoice: undefined,
+        isNonInteractiveSession: false,
+        hasAppendSystemPrompt: false,
+        agents: [],
+        querySource: 'away_summary',
+        mcpTools: [],
+        skipCacheWrite: true,
+      },
+    })
+
+    if (response.isApiErrorMessage) {
+      logForDebugging(
+        `[awaySummary] API error: ${getAssistantMessageText(response)}`,
+      )
+      return null
+    }
+    return getAssistantMessageText(response)
+  } catch (err) {
+    if (err instanceof APIUserAbortError || signal.aborted) {
+      return null
+    }
+    logForDebugging(`[awaySummary] generation failed: ${err}`)
+    return null
+  }
+}