opencastle 0.31.7 → 0.32.0

package/src/cli/convoy/artifacts.ts

@@ -0,0 +1,186 @@
+import {
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  rmSync,
+  statSync,
+  writeFileSync,
+} from 'node:fs'
+import { basename, join } from 'node:path'
+
+// ── Types ─────────────────────────────────────────────────────────────────────
+
+export interface Artifact {
+  task_id: string
+  convoy_id: string
+  filename: string
+  type: 'report' | 'code' | 'data' | 'diff' | 'log' | 'other'
+  size_bytes: number
+  summary: string
+  path: string
+}
+
+export interface ArtifactRef {
+  task_id: string
+  filename: string
+  summary: string
+  path: string
+}
+
+interface ArtifactMeta {
+  type: Artifact['type']
+  summary: string
+  size_bytes: number
+  created_at: string
+}
+
+// ── Sanitization ──────────────────────────────────────────────────────────────
+
+function sanitizeSegment(input: string): string {
+  if (input.includes('..') || input.includes('/') || input.includes('\\')) {
+    throw new Error(`Invalid path segment "${input}": path traversal characters not allowed`)
+  }
+  return input.replace(/[^a-zA-Z0-9\-_.]/g, '')
+}
+
+// ── Core ──────────────────────────────────────────────────────────────────────
+
+export function getArtifactDir(convoyId: string, taskId: string): string {
+  const safeConvoyId = sanitizeSegment(convoyId)
+  const safeTaskId = sanitizeSegment(taskId)
+  return join(process.cwd(), '.opencastle', 'artifacts', safeConvoyId, safeTaskId) + '/'
+}
+
+export function writeArtifact(
+  convoyId: string,
+  taskId: string,
+  filename: string,
+  content: string,
+  type: Artifact['type'],
+): Artifact {
+  const safeFilename = sanitizeSegment(filename)
+  const dir = getArtifactDir(convoyId, taskId)
+  mkdirSync(dir, { recursive: true })
+
+  const filePath = join(dir, safeFilename)
+  writeFileSync(filePath, content, 'utf8')
+
+  const size_bytes = Buffer.byteLength(content, 'utf8')
+  const firstLine = content.split('\n')[0] ?? ''
+  const summary = firstLine.slice(0, 120)
+
+  const meta: ArtifactMeta = { type, summary, size_bytes, created_at: new Date().toISOString() }
+  writeFileSync(join(dir, safeFilename + '.meta.json'), JSON.stringify(meta, null, 2), 'utf8')
+
+  return { task_id: taskId, convoy_id: convoyId, filename: safeFilename, type, size_bytes, summary, path: filePath }
+}
+
+export function listArtifacts(convoyId: string, taskId: string): ArtifactRef[] {
+  const dir = getArtifactDir(convoyId, taskId)
+  if (!existsSync(dir)) return []
+
+  const refs: ArtifactRef[] = []
+  for (const entry of readdirSync(dir)) {
+    if (entry.endsWith('.meta.json')) continue
+
+    const filePath = join(dir, entry)
+    const metaPath = join(dir, entry + '.meta.json')
+
+    let summary = ''
+    if (existsSync(metaPath)) {
+      try {
+        const meta = JSON.parse(readFileSync(metaPath, 'utf8')) as ArtifactMeta
+        summary = meta.summary
+      } catch { /* fallback */ }
+    }
+    if (!summary) {
+      try {
+        const firstLine = readFileSync(filePath, 'utf8').split('\n')[0] ?? ''
+        summary = firstLine.slice(0, 120)
+      } catch { /* non-critical */ }
+    }
+
+    refs.push({ task_id: taskId, filename: entry, summary, path: filePath })
+  }
+
+  return refs
+}
+
+export function readArtifact(ref: ArtifactRef): string {
+  return readFileSync(ref.path, 'utf8')
+}
+
+export function extractArtifactRefs(taskId: string, convoyId: string, output: string): ArtifactRef[] {
+  const pattern = /\[ARTIFACT:\s*([^\]]+)\]\s*(.+)/g
+  const refs: ArtifactRef[] = []
+  let match: RegExpExecArray | null
+
+  while ((match = pattern.exec(output)) !== null) {
+    // Use basename to prevent path traversal from untrusted agent output
+    const filename = basename(match[1].trim())
+    const summary = match[2].trim()
+
+    if (!filename || filename === '..') {
+      process.stderr.write('[artifacts] Warning: invalid artifact filename from agent output\n')
+      continue
+    }
+
+    const dir = getArtifactDir(convoyId, taskId)
+    const filePath = join(dir, filename)
+
+    if (!existsSync(filePath)) {
+      process.stderr.write(`[artifacts] Warning: referenced artifact not found: ${filePath}\n`)
+      continue
+    }
+
+    refs.push({ task_id: taskId, filename, summary, path: filePath })
+  }
+
+  return refs
+}
+
+export function pruneArtifacts(keepCount: number): { removed: number; freed_bytes: number } {
+  const artifactsRoot = join(process.cwd(), '.opencastle', 'artifacts')
+  if (!existsSync(artifactsRoot)) return { removed: 0, freed_bytes: 0 }
+
+  const convoyDirs = readdirSync(artifactsRoot)
+    .map(name => {
+      const dirPath = join(artifactsRoot, name)
+      try {
+        return { name, path: dirPath, mtime: statSync(dirPath).mtime.getTime() }
+      } catch {
+        return { name, path: dirPath, mtime: 0 }
+      }
+    })
+    .sort((a, b) => b.mtime - a.mtime)
+
+  const toRemove = convoyDirs.slice(keepCount)
+  let removed = 0
+  let freed_bytes = 0
+
+  for (const dir of toRemove) {
+    try {
+      freed_bytes += calcDirSize(dir.path)
+      rmSync(dir.path, { recursive: true, force: true })
+      removed++
+    } catch { /* non-critical */ }
+  }
+
+  return { removed, freed_bytes }
+}
+
+function calcDirSize(dirPath: string): number {
+  let total = 0
+  try {
+    for (const entry of readdirSync(dirPath)) {
+      const p = join(dirPath, entry)
+      try {
+        const s = statSync(p)
+        if (s.isDirectory()) total += calcDirSize(p)
+        else total += s.size
+      } catch { /* skip */ }
+    }
+  } catch { /* skip */ }
+  return total
+}

package/src/cli/convoy/compaction.test.ts

@@ -0,0 +1,245 @@
+import { describe, it, expect, afterEach } from 'vitest'
+import { mkdtempSync, rmSync, existsSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import {
+  shouldCompact,
+  generateCompactionPrompt,
+  parseCompactionSummary,
+  saveCompaction,
+  loadCompaction,
+  buildContinuationPrompt,
+  canCompact,
+  getMaxCompactions,
+  getCompactionDir,
+  MODEL_CONTEXT_WINDOWS,
+  type CompactionSummary,
+} from './compaction.js'
+import type { CompactionConfig } from './types.js'
+
+const baseConfig: CompactionConfig = {
+  enabled: true,
+  token_threshold_pct: 70,
+  summary_max_tokens: 4096,
+}
+
+describe('shouldCompact', () => {
+  it('returns false when config.enabled is false', () => {
+    expect(shouldCompact(150_000, 'claude-sonnet-4-6', { ...baseConfig, enabled: false })).toBe(false)
+  })
+
+  it('returns false below threshold', () => {
+    // 69% of 200_000 = 138_000
+    expect(shouldCompact(138_000, 'claude-sonnet-4-6', baseConfig)).toBe(false)
+  })
+
+  it('returns true at exactly the threshold', () => {
+    // 70% of 200_000 = 140_000
+    expect(shouldCompact(140_000, 'claude-sonnet-4-6', baseConfig)).toBe(true)
+  })
+
+  it('returns true above threshold', () => {
+    expect(shouldCompact(180_000, 'claude-sonnet-4-6', baseConfig)).toBe(true)
+  })
+
+  it('uses DEFAULT_CONTEXT_WINDOW (128_000) for unknown model', () => {
+    // 70% of 128_000 = 89_600
+    expect(shouldCompact(90_000, 'unknown-model-xyz', baseConfig)).toBe(true)
+    expect(shouldCompact(88_000, 'unknown-model-xyz', baseConfig)).toBe(false)
+  })
+
+  it('uses correct window for gpt-5-mini (128_000)', () => {
+    expect(MODEL_CONTEXT_WINDOWS['gpt-5-mini']).toBe(128_000)
+    expect(shouldCompact(90_000, 'gpt-5-mini', baseConfig)).toBe(true)
+  })
+})
+
+describe('generateCompactionPrompt', () => {
+  it('returns a string containing COMPACTION_SUMMARY marker', () => {
+    const prompt = generateCompactionPrompt('task-42')
+    expect(prompt).toContain('COMPACTION_SUMMARY')
+    expect(prompt).toContain('Context Compaction Required')
+    expect(typeof prompt).toBe('string')
+  })
+
+  it('includes JSON structure placeholders', () => {
+    const prompt = generateCompactionPrompt('task-1')
+    expect(prompt).toContain('"phase"')
+    expect(prompt).toContain('"completed_steps"')
+    expect(prompt).toContain('"pending_steps"')
+  })
+})
+
+describe('parseCompactionSummary', () => {
+  it('parses valid summary from agent output', () => {
+    const output = [
+      'Some output text',
+      '',
+      '<!-- COMPACTION_SUMMARY',
+      '{',
+      '  "phase": "implementation",',
+      '  "completed_steps": ["created types", "wrote tests"],',
+      '  "pending_steps": ["integrate with engine"],',
+      '  "key_decisions": ["used valibot for validation"],',
+      '  "files_modified": ["src/foo.ts"],',
+      '  "artifact_refs": [".opencastle/artifacts/convoy-1/task-1/report.md"]',
+      '}',
+      '-->',
+      '',
+      'More text',
+    ].join('\n')
+    const result = parseCompactionSummary(output, 'task-1', 'convoy-1')
+    expect(result).not.toBeNull()
+    expect(result!.task_id).toBe('task-1')
+    expect(result!.convoy_id).toBe('convoy-1')
+    expect(result!.phase).toBe('implementation')
+    expect(result!.completed_steps).toEqual(['created types', 'wrote tests'])
+    expect(result!.pending_steps).toEqual(['integrate with engine'])
+    expect(result!.key_decisions).toEqual(['used valibot for validation'])
+    expect(result!.files_modified).toEqual(['src/foo.ts'])
+    expect(result!.artifact_refs).toEqual(['.opencastle/artifacts/convoy-1/task-1/report.md'])
+    expect(typeof result!.timestamp).toBe('string')
+  })
+
+  it('returns null when no COMPACTION_SUMMARY marker found', () => {
+    const result = parseCompactionSummary('just some output without the marker', 'task-1', 'convoy-1')
+    expect(result).toBeNull()
+  })
+
+  it('returns null for invalid JSON inside the marker', () => {
+    const output = '<!-- COMPACTION_SUMMARY\nnot valid json\n-->'
+    const result = parseCompactionSummary(output, 'task-1', 'convoy-1')
+    expect(result).toBeNull()
+  })
+
+  it('uses empty arrays for missing or non-array fields', () => {
+    const output = '<!-- COMPACTION_SUMMARY\n{"phase": "testing"}\n-->'
+    const result = parseCompactionSummary(output, 'task-1', 'convoy-1')
+    expect(result).not.toBeNull()
+    expect(result!.completed_steps).toEqual([])
+    expect(result!.pending_steps).toEqual([])
+    expect(result!.files_modified).toEqual([])
+  })
+})
+
+describe('saveCompaction / loadCompaction', () => {
+  let tmpDir: string
+
+  afterEach(() => {
+    if (tmpDir && existsSync(tmpDir)) {
+      rmSync(tmpDir, { recursive: true, force: true })
+    }
+  })
+
+  it('saves and restores a compaction summary', () => {
+    tmpDir = mkdtempSync(join(tmpdir(), 'compaction-test-'))
+    const summary: CompactionSummary = {
+      task_id: 'task-1',
+      convoy_id: 'convoy-abc',
+      phase: 'testing',
+      completed_steps: ['step A'],
+      pending_steps: ['step B'],
+      key_decisions: ['chose approach X'],
+      files_modified: ['src/foo.ts'],
+      artifact_refs: [],
+      timestamp: new Date().toISOString(),
+    }
+
+    const origCwd = process.cwd()
+    process.chdir(tmpDir)
+    try {
+      const savedPath = saveCompaction('convoy-abc', 'task-1', summary, 1)
+      expect(existsSync(savedPath)).toBe(true)
+      const loaded = loadCompaction(savedPath)
+      expect(loaded).not.toBeNull()
+      expect(loaded!.task_id).toBe('task-1')
+      expect(loaded!.phase).toBe('testing')
+      expect(loaded!.completed_steps).toEqual(['step A'])
+    } finally {
+      process.chdir(origCwd)
+    }
+  })
+
+  it('returns null for a non-existent path', () => {
+    const result = loadCompaction('/tmp/definitely-does-not-exist-abc123.json')
+    expect(result).toBeNull()
+  })
+})
+
+describe('buildContinuationPrompt', () => {
+  let tmpDir: string
+
+  afterEach(() => {
+    if (tmpDir && existsSync(tmpDir)) {
+      rmSync(tmpDir, { recursive: true, force: true })
+    }
+  })
+
+  it('includes original prompt, summary, and isolation preamble', () => {
+    tmpDir = mkdtempSync(join(tmpdir(), 'compaction-test-'))
+    const origCwd = process.cwd()
+    process.chdir(tmpDir)
+    try {
+      const summary: CompactionSummary = {
+        task_id: 'task-1',
+        convoy_id: 'convoy-abc',
+        phase: 'integration',
+        completed_steps: ['wrote types'],
+        pending_steps: ['update engine'],
+        key_decisions: ['decided to reuse store'],
+        files_modified: ['src/types.ts'],
+        artifact_refs: [],
+        timestamp: new Date().toISOString(),
+      }
+      const savedPath = saveCompaction('convoy-abc', 'task-1', summary, 1)
+      const result = buildContinuationPrompt('Do the remaining work', savedPath, '## Isolation preamble\n')
+      expect(result).toContain('## Isolation preamble')
+      expect(result).toContain('Do the remaining work')
+      expect(result).toContain('Continuation from Compacted Context')
+      expect(result).toContain('wrote types')
+      expect(result).toContain('update engine')
+    } finally {
+      process.chdir(origCwd)
+    }
+  })
+
+  it('handles missing summary file gracefully', () => {
+    const result = buildContinuationPrompt(
+      'Do the work',
+      '/tmp/missing-summary-def456.json',
+      '## Preamble\n',
+    )
+    expect(result).toContain('Do the work')
+    expect(result).toContain('## Preamble')
+    expect(result).not.toContain('Continuation from Compacted Context')
+  })
+})
+
+describe('canCompact', () => {
+  it('returns true below max (3)', () => {
+    expect(canCompact(0)).toBe(true)
+    expect(canCompact(1)).toBe(true)
+    expect(canCompact(2)).toBe(true)
+  })
+
+  it('returns false at max (3)', () => {
+    expect(canCompact(3)).toBe(false)
+    expect(canCompact(4)).toBe(false)
+  })
+})
+
+describe('getMaxCompactions', () => {
+  it('returns 3', () => {
+    expect(getMaxCompactions()).toBe(3)
+  })
+})
+
+describe('getCompactionDir', () => {
+  it('returns path inside .opencastle/artifacts/{convoyId}/{taskId}', () => {
+    const result = getCompactionDir('convoy-abc', 'task-1')
+    expect(result).toContain('.opencastle')
+    expect(result).toContain('artifacts')
+    expect(result).toContain('convoy-abc')
+    expect(result).toContain('task-1')
+  })
+})

package/src/cli/convoy/compaction.ts

@@ -0,0 +1,164 @@
+import { mkdirSync, readFileSync, writeFileSync } from 'node:fs'
+import { join, resolve } from 'node:path'
+import type { CompactionConfig } from './types.js'
+
+// --- Types ---
+
+export interface CompactionSummary {
+  task_id: string
+  convoy_id: string
+  phase: string
+  completed_steps: string[]
+  pending_steps: string[]
+  key_decisions: string[]
+  files_modified: string[]
+  artifact_refs: string[]
+  timestamp: string
+}
+
+export const MODEL_CONTEXT_WINDOWS: Record<string, number> = {
+  'claude-opus-4-6': 200_000,
+  'claude-sonnet-4-6': 200_000,
+  'gemini-3.1-pro': 2_000_000,
+  'gpt-5.3-codex': 200_000,
+  'gpt-5-mini': 128_000,
+}
+
+const DEFAULT_CONTEXT_WINDOW = 128_000
+const MAX_COMPACTIONS_PER_TASK = 3
+
+// --- Threshold detection ---
+
+export function shouldCompact(
+  tokensUsed: number,
+  model: string,
+  config: CompactionConfig,
+): boolean {
+  if (!config.enabled) return false
+  const contextWindow = MODEL_CONTEXT_WINDOWS[model] ?? DEFAULT_CONTEXT_WINDOW
+  return tokensUsed / contextWindow >= config.token_threshold_pct / 100
+}
+
+// --- Compaction prompt ---
+
+export function generateCompactionPrompt(taskId: string): string {
+  return [
+    '## Context Compaction Required',
+    'You are approaching your context limit. Before continuing, produce a COMPACTION_SUMMARY:',
+    '',
+    '<!-- COMPACTION_SUMMARY',
+    '{',
+    '  "phase": "current work phase",',
+    '  "completed_steps": ["step 1 done", "step 2 done"],',
+    '  "pending_steps": ["step 3 todo", "step 4 todo"],',
+    '  "key_decisions": ["chose approach A because..."],',
+    '  "files_modified": ["src/foo.ts", "src/bar.ts"],',
+    '  "artifact_refs": [".opencastle/artifacts/.../report.md"]',
+    '}',
+    '-->',
+    '',
+    'Be concise. Focus on WHAT was decided and WHAT remains, not HOW you got here.',
+  ].join('\n')
+}
+
+// --- Parse compaction summary from agent output ---
+
+export function parseCompactionSummary(
+  output: string,
+  taskId: string,
+  convoyId: string,
+): CompactionSummary | null {
+  const match = output.match(/<!--\s*COMPACTION_SUMMARY\s*\n([\s\S]*?)-->/)
+  if (!match) return null
+
+  try {
+    const parsed = JSON.parse(match[1].trim()) as Record<string, unknown>
+    return {
+      task_id: taskId,
+      convoy_id: convoyId,
+      phase: typeof parsed.phase === 'string' ? parsed.phase : 'unknown',
+      completed_steps: Array.isArray(parsed.completed_steps) ? parsed.completed_steps.filter((s): s is string => typeof s === 'string') : [],
+      pending_steps: Array.isArray(parsed.pending_steps) ? parsed.pending_steps.filter((s): s is string => typeof s === 'string') : [],
+      key_decisions: Array.isArray(parsed.key_decisions) ? parsed.key_decisions.filter((s): s is string => typeof s === 'string') : [],
+      files_modified: Array.isArray(parsed.files_modified) ? parsed.files_modified.filter((s): s is string => typeof s === 'string') : [],
+      artifact_refs: Array.isArray(parsed.artifact_refs) ? parsed.artifact_refs.filter((s): s is string => typeof s === 'string') : [],
+      timestamp: new Date().toISOString(),
+    }
+  } catch {
+    return null
+  }
+}
+
+// --- Save / restore ---
+
+export function getCompactionDir(convoyId: string, taskId: string): string {
+  return join(resolve(process.cwd()), '.opencastle', 'artifacts', convoyId, taskId)
+}
+
+export function saveCompaction(
+  convoyId: string,
+  taskId: string,
+  summary: CompactionSummary,
+  compactionCount: number,
+): string {
+  const dir = getCompactionDir(convoyId, taskId)
+  mkdirSync(dir, { recursive: true })
+  const filename = `compaction-${compactionCount}.json`
+  const filePath = join(dir, filename)
+  writeFileSync(filePath, JSON.stringify(summary, null, 2))
+  return filePath
+}
+
+export function loadCompaction(summaryPath: string): CompactionSummary | null {
+  try {
+    const content = readFileSync(summaryPath, 'utf8')
+    return JSON.parse(content) as CompactionSummary
+  } catch {
+    return null
+  }
+}
+
+// --- Build continuation prompt ---
+
+export function buildContinuationPrompt(
+  originalPrompt: string,
+  summaryPath: string,
+  isolationPreamble: string,
+): string {
+  const summary = loadCompaction(summaryPath)
+  if (!summary) {
+    return isolationPreamble + '\n\n' + originalPrompt
+  }
+
+  const summaryBlock = [
+    '## Continuation from Compacted Context',
+    'You are CONTINUING a task that was compacted. Previous progress:',
+    '',
+    '**Phase:** ' + summary.phase,
+    '**Completed steps:**',
+    ...summary.completed_steps.map(s => '- ' + s),
+    '**Pending steps:**',
+    ...summary.pending_steps.map(s => '- ' + s),
+    '**Key decisions:**',
+    ...summary.key_decisions.map(s => '- ' + s),
+    '**Files already modified:**',
+    ...summary.files_modified.map(f => '- ' + f),
+    ...(summary.artifact_refs.length > 0
+      ? ['**Artifacts:**', ...summary.artifact_refs.map(a => '- ' + a)]
+      : []),
+    '',
+    'Focus on the PENDING steps. Do NOT redo completed steps.',
+  ].join('\n')
+
+  return isolationPreamble + '\n\n' + summaryBlock + '\n\n' + originalPrompt
+}
+
+// --- Compaction count helpers ---
+
+export function canCompact(compactionCount: number): boolean {
+  return compactionCount < MAX_COMPACTIONS_PER_TASK
+}
+
+export function getMaxCompactions(): number {
+  return MAX_COMPACTIONS_PER_TASK
+}