@phenx-inc/ctlsurf 0.13.6 → 0.14.0

package/src/main/observer/observer.ts

@@ -0,0 +1,566 @@
+import { randomUUID } from 'crypto'
+import path from 'path'
+
+import type { CtlsurfApi } from '../ctlsurfApi'
+import { TranscriptTailer, supportsTranscriptLogging } from '../transcripts'
+import { log as baseLog } from '../logger'
+
+import type {
+  Finding,
+  LedgerEvent,
+  ModelCaller,
+  ObserverFeedItem,
+  ObserverSettings,
+  Sensitivity,
+  Span,
+} from './types'
+import { OBSERVER_DEFAULTS, SEVERITY_ORDER, severityFloor } from './types'
+import { SessionLedger } from './ledger'
+import { tagEvent, checkTestClaim, truncateForTier1 } from './tier0'
+import { SpanTracker } from './spans'
+import { Tier1Reviewer } from './tier1'
+import { generateDigest } from './tier2'
+import { ObserverBudget } from './budget'
+import { ObserverStore } from './store'
+import { createModelCaller } from './anthropic'
+
+/**
+ * Observer — supervision layer over the session ledger (spec v0.1, Phase 1).
+ *
+ * Consumes the agent's native transcript (via its own TranscriptTailer) and
+ * pty-derived activity signals, normalizes them into a local append-only
+ * JSONL ledger, and produces two artifact streams:
+ *   - findings (Tier 0 ground-truth + Tier 1 Haiku windowed review)
+ *   - spans (deterministic time attribution + Tier 1 boundary detection)
+ * with a Tier 2 session-end digest delivered to the project's ctlsurf
+ * Agent Datastore page.
+ *
+ * Advisory only: nothing here can block or slow the acting agent, and every
+ * code path is wrapped so observation failures never break the session.
+ */
+
+function log(...args: unknown[]): void {
+  baseLog('[observer]', ...args)
+}
+
+interface TabSession {
+  tabId: string
+  agentId: string
+  agentName: string
+  cwd: string
+  sessionId: string
+  startedAt: string
+  ledger: SessionLedger
+  tailer: TranscriptTailer | null
+  spans: SpanTracker
+  tier1: Tier1Reviewer | null
+  findings: Finding[]
+  closedSpans: Span[]
+  escalations: string[]
+  userMessages: string[]
+  /** Last AgentStateMonitor state, to detect busy → awaiting transitions. */
+  lastAgentState: string
+  ended: boolean
+}
+
+export interface ObserverOptions {
+  settingsDir: string
+  api: CtlsurfApi
+  getSettings: () => ObserverSettings
+  /** Live feed hook for the in-app Observer panel. */
+  onUpdate?: (item: ObserverFeedItem) => void
+  /** Model review in-flight signal — drives the panel's processing
+   * animation. Emitted on 0→1 and →0 transitions of concurrent reviews. */
+  onActivity?: (reviewing: boolean) => void
+  /** Override transcript roots (for tests). */
+  claudeProjectsDir?: string
+  codexSessionsDir?: string
+}
+
+const FEED_LIMIT = 200
+
+export class Observer {
+  private readonly opts: ObserverOptions
+  private readonly store: ObserverStore
+  private tabs = new Map<string, TabSession>()
+  private budget: ObserverBudget | null = null
+  private budgetCapUsd = -1
+  private modelCaller: ModelCaller | null = null
+  private modelCallerKey = ''
+  /** Rolling feed for the in-app panel (survives tab close, not restart). */
+  private feed: ObserverFeedItem[] = []
+  private feedSeq = 0
+  /** Concurrent model reviews in flight (Tier 1 windows + Tier 2 digests). */
+  private activeReviews = 0
+
+  constructor(opts: ObserverOptions) {
+    this.opts = opts
+    this.store = new ObserverStore(opts.api)
+  }
+
+  isEnabled(): boolean {
+    return !!this.opts.getSettings().enabled
+  }
+
+  /** Feed + status snapshot for the in-app panel. */
+  getFeed(): {
+    items: ObserverFeedItem[]
+    enabled: boolean
+    hasApiKey: boolean
+    observing: Array<{ tabId: string; agentName: string; session: string }>
+    spentTodayUsd: number
+    reviewing: boolean
+    sensitivity: Sensitivity
+  } {
+    const settings = this.opts.getSettings()
+    return {
+      items: [...this.feed],
+      enabled: !!settings.enabled,
+      hasApiKey: !!(settings.anthropicApiKey || process.env.ANTHROPIC_API_KEY),
+      observing: [...this.tabs.values()].map((t) => ({ tabId: t.tabId, agentName: t.agentName, session: t.sessionId })),
+      spentTodayUsd: Math.round(this.ensureBudget().spent * 1000) / 1000,
+      reviewing: this.activeReviews > 0,
+      sensitivity: settings.sensitivity || OBSERVER_DEFAULTS.sensitivity,
+    }
+  }
+
+  /** Detach from all tabs without digests (used when the user flips the
+   * panel toggle off — quiet stop, not a session end). */
+  detachAll(): void {
+    for (const tabId of [...this.tabs.keys()]) this.detachSync(tabId)
+    this.pushFeed({ kind: 'status', title: 'Observer turned off' })
+  }
+
+  /** Begin observing a tab. No-op unless enabled and the agent has a native
+   * transcript (Claude Code / Codex) — Phase 1 scope. */
+  attachTab(tabId: string, agentId: string, agentName: string, cwd: string): void {
+    try {
+      if (!this.isEnabled()) return
+      if (this.tabs.has(tabId)) this.detachSync(tabId)
+      if (!supportsTranscriptLogging(agentId)) {
+        log(`Tab ${tabId}: agent "${agentId}" has no native transcript; observer idle for this tab`)
+        return
+      }
+
+      const settings = this.opts.getSettings()
+      const sessionId = `ses_${randomUUID().replace(/-/g, '').slice(0, 12)}`
+      const ledgerDir = path.join(this.opts.settingsDir, 'observer', 'ledger')
+      const ledger = new SessionLedger(ledgerDir, sessionId)
+
+      const tab: TabSession = {
+        tabId,
+        agentId,
+        agentName,
+        cwd,
+        sessionId,
+        startedAt: new Date().toISOString(),
+        ledger,
+        tailer: null,
+        spans: new SpanTracker(sessionId, (span) => this.onSpanClosed(tab, span)),
+        tier1: null,
+        findings: [],
+        closedSpans: [],
+        escalations: [],
+        userMessages: [],
+        lastAgentState: 'idle',
+        ended: false,
+      }
+
+      const caller = this.resolveModelCaller()
+      if (caller) {
+        tab.tier1 = new Tier1Reviewer({
+          session: sessionId,
+          model: settings.tier1Model || OBSERVER_DEFAULTS.tier1Model,
+          windowEvents: settings.windowEvents ?? OBSERVER_DEFAULTS.windowEvents,
+          debounceMs: (settings.debounceSec ?? OBSERVER_DEFAULTS.debounceSec) * 1000,
+          callModel: caller,
+          // Resolved lazily so a daily-cap change from the settings UI
+          // applies to sessions already running.
+          isOverBudget: () => this.ensureBudget().overCap(),
+          onUsage: (model, usage) => this.ensureBudget().addUsage(model, usage),
+          onFindings: (findings) => this.onFindings(tab, findings),
+          onSpanBoundary: (workRef, kind) => tab.spans.applyBoundary(workRef, kind),
+          onEscalate: (reason) => {
+            tab.escalations.push(reason)
+            log(`Tab ${tabId}: Tier 1 escalation — ${reason}`)
+          },
+          onHealth: (message) => this.emitHealth(tab, message),
+          onReviewState: (running) => this.noteReviewState(running),
+          getSensitivity: () => this.opts.getSettings().sensitivity || OBSERVER_DEFAULTS.sensitivity,
+          onReviewed: (eventCount, findingCount) =>
+            this.pushFeed({
+              kind: 'status',
+              tabId,
+              session: sessionId,
+              title: `Reviewed ${eventCount} event${eventCount === 1 ? '' : 's'} — ${findingCount === 0 ? 'nothing to flag' : `${findingCount} finding${findingCount === 1 ? '' : 's'}`}`,
+            }),
+          getSpanHypothesis: () => tab.spans.currentWorkRef(),
+        })
+      } else {
+        this.emitHealth(tab, 'No Anthropic API key configured (settings.observer.anthropicApiKey or ANTHROPIC_API_KEY); running Tier 0 + spans only.')
+      }
+
+      tab.spans.begin()
+      tab.tailer = new TranscriptTailer({
+        agentId: agentId as 'claude' | 'codex',
+        cwd,
+        sink: (entry) => this.onChatEntry(tab, entry.type === 'user_input' ? 'user_message' : 'agent_message', entry.content, entry.ts),
+        rawSink: (obj) => this.onRawTranscriptLine(tab, obj),
+        claudeProjectsDir: this.opts.claudeProjectsDir,
+        codexSessionsDir: this.opts.codexSessionsDir,
+      })
+      tab.tailer.start()
+      this.tabs.set(tabId, tab)
+      log(`Observing tab ${tabId} (${agentId}) → ledger ${ledger.filePath}`)
+      this.pushFeed({
+        kind: 'status',
+        tabId,
+        session: sessionId,
+        title: `Observing ${agentName} (${tab.tier1 ? 'reviews when the agent hands control back' : 'Tier 0 + spans only — no API key'})`,
+      })
+    } catch (err: any) {
+      log(`attachTab failed: ${err?.message || err}`)
+    }
+  }
+
+  /** Human keystrokes reached the tab's pty (activity attribution only —
+   * content is never recorded here; typed prompts arrive via the transcript). */
+  noteHumanInput(tabId: string): void {
+    this.tabs.get(tabId)?.spans.noteHumanInput()
+  }
+
+  /** Agent busy/awaiting/idle state changed (from AgentStateMonitor). */
+  noteAgentState(tabId: string, state: string): void {
+    const tab = this.tabs.get(tabId)
+    if (!tab || tab.ended) return
+    tab.spans.noteAgentState(state)
+    const event = tab.ledger.append({ type: 'agent_state', ts: new Date().toISOString(), state })
+    tab.spans.noteEvent(event)
+
+    // Turn boundary: the agent just handed control back to the human — the
+    // same signal that drives the dock notification. Review what's buffered
+    // now instead of waiting for the window/debounce.
+    const wasBusy = tab.lastAgentState === 'busy'
+    const awaiting = state === 'awaiting_input' || state === 'awaiting_decision'
+    tab.lastAgentState = state
+    if (wasBusy && awaiting) tab.tier1?.reviewNow()
+  }
+
+  /** Session over: flush Tier 1, close spans, generate + deliver the digest.
+   * `projectPath` overrides cwd for ctlsurf delivery (remote-bound tabs). */
+  async endTab(tabId: string, projectPath?: string): Promise<void> {
+    const tab = this.tabs.get(tabId)
+    if (!tab || tab.ended) return
+    tab.ended = true
+    this.tabs.delete(tabId)
+
+    try {
+      tab.tailer?.stop()
+      tab.tailer = null
+      await tab.tier1?.flush()
+      tab.tier1?.dispose()
+      tab.spans.end()
+
+      const settings = this.opts.getSettings()
+      const caller = this.resolveModelCaller()
+      const budget = this.ensureBudget()
+      this.noteReviewState(true)
+      let digest
+      try {
+        digest = await generateDigest(
+          {
+            session: tab.sessionId,
+            agentName: tab.agentName,
+            cwd: projectPath || tab.cwd,
+            startedAt: tab.startedAt,
+            endedAt: new Date().toISOString(),
+            findings: tab.findings,
+            spans: tab.closedSpans,
+            humanInstructionsSummary: tab.tier1?.instructionsSummary || '',
+            escalations: tab.escalations,
+            userMessages: tab.userMessages,
+          },
+          caller && !budget.overCap() ? caller : null,
+          settings.tier2Model || OBSERVER_DEFAULTS.tier2Model,
+          (model, usage) => budget.addUsage(model, usage),
+        )
+      } finally {
+        this.noteReviewState(false)
+      }
+
+      tab.ledger.append({
+        type: 'observer.digest',
+        ts: new Date().toISOString(),
+        text: digest.digest,
+        payload: { from_model: digest.fromModel, findings: tab.findings.length, spans: tab.closedSpans.length },
+      })
+      this.pushFeed({
+        kind: 'digest',
+        tabId: tab.tabId,
+        session: tab.sessionId,
+        title: `Session digest (${tab.findings.length} finding${tab.findings.length === 1 ? '' : 's'}, ${tab.closedSpans.length} span${tab.closedSpans.length === 1 ? '' : 's'})`,
+        detail: digest.digest,
+      })
+
+      const deliveryPath = projectPath || tab.cwd
+      await this.store.appendDigest(deliveryPath, digest.digest, {
+        session: tab.sessionId,
+        agent: tab.agentName,
+        findings: tab.findings.length,
+        spans: tab.closedSpans.length,
+        ledger: tab.ledger.filePath,
+      })
+      log(`Tab ${tabId}: session ${tab.sessionId} closed — ${tab.findings.length} finding(s), ${tab.closedSpans.length} span(s)`)
+    } catch (err: any) {
+      log(`endTab failed for ${tabId}: ${err?.message || err}`)
+    }
+  }
+
+  async shutdown(): Promise<void> {
+    const ids = [...this.tabs.keys()]
+    await Promise.all(ids.map((id) => this.endTab(id)))
+  }
+
+  // ─── Event intake ───────────────────────────────
+
+  /** Clean chat text from the tailer's filtered sink. */
+  private onChatEntry(tab: TabSession, type: 'user_message' | 'agent_message', text: string, ts: string): void {
+    if (tab.ended) return
+    try {
+      if (type === 'user_message') tab.userMessages.push(text)
+      this.ingest(tab, { type, ts, text })
+    } catch (err: any) {
+      log(`onChatEntry failed: ${err?.message || err}`)
+    }
+  }
+
+  /** Raw transcript objects — used only for tool_use / tool_result, which the
+   * chat sink filters out. Message text arrives via onChatEntry. */
+  private onRawTranscriptLine(tab: TabSession, obj: any): void {
+    if (tab.ended || !obj || typeof obj !== 'object') return
+    try {
+      if (tab.agentId !== 'claude') return // Codex tool events: later phase
+      if (obj.isMeta || obj.isSidechain) return
+      if (typeof obj.cwd === 'string' && stripTrailingSep(obj.cwd) !== stripTrailingSep(tab.cwd)) return
+      const content = obj.message?.content
+      if (!Array.isArray(content)) return
+      const ts = typeof obj.timestamp === 'string' ? obj.timestamp : new Date().toISOString()
+
+      if (obj.type === 'assistant') {
+        for (const block of content) {
+          if (block?.type !== 'tool_use') continue
+          this.ingest(tab, {
+            type: 'tool_use',
+            ts,
+            tool: typeof block.name === 'string' ? block.name : 'unknown',
+            input: summarizeToolInput(block.input),
+            toolUseId: typeof block.id === 'string' ? block.id : undefined,
+          })
+        }
+      } else if (obj.type === 'user') {
+        for (const block of content) {
+          if (block?.type !== 'tool_result') continue
+          this.ingest(tab, {
+            type: 'tool_result',
+            ts,
+            toolUseId: typeof block.tool_use_id === 'string' ? block.tool_use_id : undefined,
+            isError: !!block.is_error,
+            text: truncateForTier1(extractToolResultText(block.content)),
+          })
+        }
+      }
+    } catch (err: any) {
+      log(`onRawTranscriptLine failed: ${err?.message || err}`)
+    }
+  }
+
+  /** Tier 0 → ledger → spans → ground truth → Tier 1. */
+  private ingest(tab: TabSession, partial: Omit<LedgerEvent, 'id' | 'session' | 'tags'>): void {
+    const tags = tagEvent({ ...partial, id: '', session: tab.sessionId }, tab.cwd)
+    const event = tab.ledger.append({ ...partial, tags: tags.length ? tags : undefined })
+    tab.spans.noteEvent(event)
+
+    // Log-only "would have gated" (Phase 1): deterministic Tier-0 finding.
+    for (const tag of tags) {
+      if (tag.rule.startsWith('would_gate:') || tag.rule === 'secret_material') {
+        this.onFindings(tab, [{
+          type: 'observer.finding',
+          ts: event.ts,
+          category: tag.rule === 'secret_material' ? 'security' : 'destructive',
+          severity: tag.severity,
+          confidence: 0.95,
+          title: tag.rule === 'secret_material'
+            ? `Secret-shaped string in session (${tag.note})`
+            : `Would have gated: ${tag.rule.slice('would_gate:'.length)}`,
+          detail: `${tag.note || ''} (gates are log-only in Phase 1)`.trim(),
+          receipts: [event.id],
+          tier: 0,
+          status: 'open',
+          delivery: ['digest'],
+          resolution: null,
+        }])
+      }
+    }
+
+    // Ground truth: "tests pass" claims are decidable from the ledger.
+    if (event.type === 'agent_message' && tags.some((t) => t.rule === 'claim_tests_pass')) {
+      const check = checkTestClaim(tab.ledger.recent(), event.id)
+      if (!check.verified) {
+        this.onFindings(tab, [{
+          type: 'observer.finding',
+          ts: event.ts,
+          category: 'unverified_claim',
+          severity: 'high',
+          confidence: 0.9,
+          title: 'Claimed tests pass; ledger disagrees',
+          detail: check.detail,
+          receipts: check.receipts,
+          tier: 0,
+          status: 'open',
+          delivery: ['digest'],
+          resolution: null,
+        }])
+      }
+    }
+
+    tab.tier1?.push(event)
+  }
+
+  // ─── Outputs ────────────────────────────────────
+
+  private onFindings(tab: TabSession, findings: Array<Omit<Finding, 'id' | 'session'>>): void {
+    const complete: Finding[] = findings.map((f) => {
+      const event = tab.ledger.append({
+        type: 'observer.finding',
+        ts: f.ts,
+        payload: f,
+      })
+      return { ...f, id: event.id, session: tab.sessionId }
+    })
+    tab.findings.push(...complete)
+
+    // Sensitivity lever: everything is in the ledger, but only findings at
+    // or above the floor surface in the panel and ctlsurf.
+    const floor = severityFloor(this.opts.getSettings().sensitivity || OBSERVER_DEFAULTS.sensitivity)
+    const surfaced = complete.filter((f) => SEVERITY_ORDER[f.severity] >= SEVERITY_ORDER[floor])
+
+    for (const f of surfaced) {
+      this.pushFeed({
+        kind: 'finding',
+        tabId: tab.tabId,
+        session: tab.sessionId,
+        title: f.title,
+        detail: f.detail,
+        category: f.category,
+        severity: f.severity,
+        confidence: f.confidence,
+        receipts: f.receipts,
+        tier: f.tier,
+      })
+    }
+    // Deliver as they happen so the supervisor surface stays near-real-time.
+    void this.store.addFindings(tab.cwd, surfaced)
+  }
+
+  private onSpanClosed(tab: TabSession, span: Span): void {
+    tab.closedSpans.push(span)
+    tab.ledger.append({ type: 'observer.span', ts: span.end, id: span.id, payload: span })
+  }
+
+  private emitHealth(tab: TabSession, message: string): void {
+    log(`[health] ${message}`)
+    tab.ledger.append({ type: 'observer.health', ts: new Date().toISOString(), text: message })
+    this.pushFeed({ kind: 'health', tabId: tab.tabId, session: tab.sessionId, title: message })
+  }
+
+  /** Track concurrent model calls; notify the panel on busy↔idle edges. */
+  private noteReviewState(running: boolean): void {
+    const wasActive = this.activeReviews > 0
+    this.activeReviews = Math.max(0, this.activeReviews + (running ? 1 : -1))
+    const isActive = this.activeReviews > 0
+    if (wasActive !== isActive) {
+      try { this.opts.onActivity?.(isActive) } catch { /* renderer gone */ }
+    }
+  }
+
+  private pushFeed(item: Omit<ObserverFeedItem, 'id' | 'ts'>): void {
+    const complete: ObserverFeedItem = {
+      ...item,
+      id: `feed_${++this.feedSeq}`,
+      ts: new Date().toISOString(),
+    }
+    this.feed.push(complete)
+    if (this.feed.length > FEED_LIMIT) this.feed.splice(0, this.feed.length - FEED_LIMIT)
+    try { this.opts.onUpdate?.(complete) } catch { /* renderer gone — ignore */ }
+  }
+
+  // ─── Config plumbing ────────────────────────────
+
+  private resolveModelCaller(): ModelCaller | null {
+    const key = this.opts.getSettings().anthropicApiKey || process.env.ANTHROPIC_API_KEY || ''
+    if (!key) return null
+    if (!this.modelCaller || this.modelCallerKey !== key) {
+      this.modelCaller = createModelCaller(key)
+      this.modelCallerKey = key
+    }
+    return this.modelCaller
+  }
+
+  private ensureBudget(): ObserverBudget {
+    const cap = this.opts.getSettings().dailyCapUsd ?? OBSERVER_DEFAULTS.dailyCapUsd
+    if (!this.budget || this.budgetCapUsd !== cap) {
+      // Recreating re-reads today's spend from disk, so a cap change (from
+      // the settings UI) applies without losing the day's accounting.
+      this.budget = new ObserverBudget(path.join(this.opts.settingsDir, 'observer'), cap)
+      this.budgetCapUsd = cap
+    }
+    return this.budget
+  }
+
+  private detachSync(tabId: string): void {
+    const tab = this.tabs.get(tabId)
+    if (!tab) return
+    tab.ended = true
+    tab.tailer?.stop()
+    tab.tier1?.dispose()
+    this.tabs.delete(tabId)
+  }
+}
+
+// ─── Helpers ──────────────────────────────────────
+
+function stripTrailingSep(p: string): string {
+  return p.length > 1 ? p.replace(/[/\\]+$/, '') : p
+}
+
+/** Compact one-line summary of a tool_use input for the ledger. */
+export function summarizeToolInput(input: unknown): string {
+  if (!input || typeof input !== 'object') return ''
+  const obj = input as Record<string, unknown>
+  if (typeof obj.command === 'string') return obj.command.slice(0, 600)
+  const parts: string[] = []
+  if (typeof obj.file_path === 'string') parts.push(obj.file_path)
+  if (typeof obj.pattern === 'string') parts.push(`pattern=${obj.pattern}`)
+  if (typeof obj.url === 'string') parts.push(obj.url)
+  if (typeof obj.description === 'string') parts.push(obj.description)
+  if (typeof obj.new_string === 'string' || typeof obj.old_string === 'string' || typeof obj.content === 'string') {
+    const body = `${obj.old_string || ''} -> ${obj.new_string || ''}${obj.content || ''}`
+    parts.push(body.slice(0, 400))
+  }
+  if (parts.length > 0) return parts.join(' ').slice(0, 600)
+  try {
+    return JSON.stringify(obj).slice(0, 400)
+  } catch {
+    return ''
+  }
+}
+
+/** Extract text from a tool_result content value (string or block array). */
+export function extractToolResultText(content: unknown): string {
+  if (typeof content === 'string') return content
+  if (!Array.isArray(content)) return ''
+  return content
+    .filter((b: any) => b?.type === 'text' && typeof b.text === 'string')
+    .map((b: any) => b.text)
+    .join('\n')
+}