opencastle 0.27.0 → 0.27.2

package/src/dashboard/node_modules/.vite/deps/_metadata.json

@@ -1,25 +1,25 @@
 {
-  "hash": "46302718",
+  "hash": "31be5bae",
   "configHash": "30f8ea04",
-  "lockfileHash": "0bec59b6",
-  "browserHash": "455152ca",
+  "lockfileHash": "68ecee18",
+  "browserHash": "fb400a2a",
   "optimized": {
     "astro > cssesc": {
       "src": "../../../../../node_modules/cssesc/cssesc.js",
       "file": "astro___cssesc.js",
-      "fileHash": "5409385e",
+      "fileHash": "05ee9b55",
       "needsInterop": true
     },
     "astro > aria-query": {
       "src": "../../../../../node_modules/aria-query/lib/index.js",
       "file": "astro___aria-query.js",
-      "fileHash": "5fd7c72e",
+      "fileHash": "7eac81b0",
       "needsInterop": true
     },
     "astro > axobject-query": {
       "src": "../../../../../node_modules/axobject-query/lib/index.js",
       "file": "astro___axobject-query.js",
-      "fileHash": "530306fc",
+      "fileHash": "891e021d",
       "needsInterop": true
     }
   },

package/src/dashboard/public/data/convoy-list.json

@@ -0,0 +1 @@
+[]

package/src/dashboard/public/data/overall-stats.json

@@ -0,0 +1,24 @@
+{
+  "convoyCounts": {
+    "total": 0,
+    "running": 0,
+    "done": 0,
+    "failed": 0,
+    "gate_failed": 0
+  },
+  "durationStats": {
+    "avg_sec": null,
+    "p95_sec": null,
+    "max_sec": null
+  },
+  "tokenCostTotals": {
+    "total_tokens": 0,
+    "total_cost_usd": 0
+  },
+  "topAgents": [],
+  "topModels": [],
+  "dlqSummary": {
+    "count": 0,
+    "top_failure_types": []
+  }
+}

package/src/dashboard/scripts/etl.test.ts

@@ -0,0 +1,210 @@
+import { mkdtempSync, rmSync, realpathSync, readFileSync, existsSync, mkdirSync } from 'node:fs'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { describe, it, expect, beforeEach, afterEach } from 'vitest'
+import { runEtl } from './etl.js'
+import { createConvoyStore } from '../../cli/convoy/store.js'
+
+function makeTmpDir(): string {
+  return realpathSync(mkdtempSync(join(tmpdir(), 'etl-test-')))
+}
+
+let tmpDir: string
+let outputDir: string
+
+beforeEach(() => {
+  tmpDir = makeTmpDir()
+  outputDir = join(tmpDir, 'data')
+})
+
+afterEach(() => {
+  rmSync(tmpDir, { recursive: true, force: true })
+})
+
+describe('runEtl — no database', () => {
+  it('writes empty overall-stats.json when db is missing', async () => {
+    const dbPath = join(tmpDir, 'nonexistent.db')
+    await runEtl({ dbPath, outputDir })
+    const stats = JSON.parse(readFileSync(join(outputDir, 'overall-stats.json'), 'utf8'))
+    expect(stats).toMatchObject({
+      convoyCounts: { total: 0, running: 0, done: 0, failed: 0, gate_failed: 0 },
+      durationStats: { avg_sec: null, p95_sec: null, max_sec: null },
+      tokenCostTotals: { total_tokens: 0, total_cost_usd: 0 },
+      topAgents: [],
+      topModels: [],
+      dlqSummary: { count: 0, top_failure_types: [] },
+    })
+  })
+
+  it('writes empty convoy-list.json when db is missing', async () => {
+    const dbPath = join(tmpDir, 'nonexistent.db')
+    await runEtl({ dbPath, outputDir })
+    const list = JSON.parse(readFileSync(join(outputDir, 'convoy-list.json'), 'utf8'))
+    expect(Array.isArray(list)).toBe(true)
+    expect(list).toHaveLength(0)
+  })
+
+  it('returns zero counts when db is missing', async () => {
+    const dbPath = join(tmpDir, 'nonexistent.db')
+    const result = await runEtl({ dbPath, outputDir })
+    expect(result).toEqual({ convoyCount: 0, taskCount: 0 })
+  })
+
+  it('creates the output directory structure even when db is missing', async () => {
+    const dbPath = join(tmpDir, 'nonexistent.db')
+    await runEtl({ dbPath, outputDir })
+    expect(existsSync(outputDir)).toBe(true)
+    expect(existsSync(join(outputDir, 'convoys'))).toBe(true)
+  })
+})
+
+describe('runEtl — with seeded database', () => {
+  let dbPath: string
+
+  beforeEach(() => {
+    dbPath = join(tmpDir, 'convoy.db')
+    const store = createConvoyStore(dbPath)
+    try {
+      store.insertConvoy({
+        id: 'convoy-abc',
+        name: 'Test Convoy',
+        spec_hash: 'abc123',
+        status: 'done',
+        branch: 'main',
+        created_at: '2026-03-01T10:00:00.000Z',
+        spec_yaml: 'tasks: []',
+      })
+      store.insertConvoy({
+        id: 'convoy-def',
+        name: 'Second Convoy',
+        spec_hash: 'def456',
+        status: 'failed',
+        branch: null,
+        created_at: '2026-03-02T10:00:00.000Z',
+        spec_yaml: 'tasks: []',
+      })
+      store.insertTask({
+        id: 'task-001',
+        convoy_id: 'convoy-abc',
+        phase: 1,
+        prompt: 'Do the thing',
+        agent: 'developer',
+        adapter: null,
+        model: 'claude-opus-4-6',
+        timeout_ms: 30000,
+        status: 'done',
+        retries: 0,
+        depends_on: null,
+        files: null,
+        gates: null,
+        max_retries: 3,
+      })
+      store.insertTask({
+        id: 'task-002',
+        convoy_id: 'convoy-abc',
+        phase: 2,
+        prompt: 'Do another thing',
+        agent: 'reviewer',
+        adapter: null,
+        model: 'claude-opus-4-6',
+        timeout_ms: 30000,
+        status: 'done',
+        retries: 1,
+        depends_on: null,
+        files: null,
+        gates: null,
+        max_retries: 3,
+      })
+    } finally {
+      store.close()
+    }
+  })
+
+  it('returns correct convoy and task counts', async () => {
+    const result = await runEtl({ dbPath, outputDir })
+    expect(result.convoyCount).toBe(2)
+    expect(result.taskCount).toBe(2)
+  })
+
+  it('overall-stats.json has correct convoy counts', async () => {
+    await runEtl({ dbPath, outputDir })
+    const stats = JSON.parse(readFileSync(join(outputDir, 'overall-stats.json'), 'utf8'))
+    expect(stats.convoyCounts).toMatchObject({ total: 2 })
+    expect(stats.durationStats).toHaveProperty('avg_sec')
+    expect(stats.tokenCostTotals).toHaveProperty('total_tokens')
+    expect(Array.isArray(stats.topAgents)).toBe(true)
+    expect(Array.isArray(stats.topModels)).toBe(true)
+    expect(stats.dlqSummary).toHaveProperty('count')
+  })
+
+  it('convoy-list.json contains all convoys with required fields', async () => {
+    await runEtl({ dbPath, outputDir })
+    const list = JSON.parse(readFileSync(join(outputDir, 'convoy-list.json'), 'utf8'))
+    expect(list).toHaveLength(2)
+    for (const item of list) {
+      expect(item).toHaveProperty('id')
+      expect(item).toHaveProperty('name')
+      expect(item).toHaveProperty('status')
+      expect(item).toHaveProperty('created_at')
+      expect(item).toHaveProperty('finished_at')
+      expect(item).toHaveProperty('total_tokens')
+      expect(item).toHaveProperty('total_cost_usd')
+    }
+  })
+
+  it('creates per-convoy detail JSON files', async () => {
+    await runEtl({ dbPath, outputDir })
+    const detailPath = join(outputDir, 'convoys', 'convoy-abc.json')
+    expect(existsSync(detailPath)).toBe(true)
+    const detail = JSON.parse(readFileSync(detailPath, 'utf8'))
+    expect(detail.convoy.id).toBe('convoy-abc')
+    expect(detail.convoy.name).toBe('Test Convoy')
+    expect(detail.convoy.status).toBe('done')
+    expect(detail.convoy).toHaveProperty('branch')
+    expect(detail.convoy).toHaveProperty('total_tokens')
+    expect(detail.convoy).toHaveProperty('total_cost_usd')
+    expect(detail).toHaveProperty('taskSummary')
+    expect(detail.taskSummary).toHaveProperty('total')
+    expect(Array.isArray(detail.tasks)).toBe(true)
+  })
+
+  it('detail file has correct task fields', async () => {
+    await runEtl({ dbPath, outputDir })
+    const detail = JSON.parse(
+      readFileSync(join(outputDir, 'convoys', 'convoy-abc.json'), 'utf8'),
+    )
+    expect(detail.tasks).toHaveLength(2)
+    for (const task of detail.tasks) {
+      expect(task).toHaveProperty('id')
+      expect(task).toHaveProperty('phase')
+      expect(task).toHaveProperty('agent')
+      expect(task).toHaveProperty('model')
+      expect(task).toHaveProperty('status')
+      expect(task).toHaveProperty('retries')
+      expect(task).toHaveProperty('started_at')
+      expect(task).toHaveProperty('finished_at')
+      expect(task).toHaveProperty('total_tokens')
+      expect(task).toHaveProperty('cost_usd')
+      expect(task).toHaveProperty('review_level')
+      expect(task).toHaveProperty('review_verdict')
+      expect(task).toHaveProperty('drift_score')
+    }
+  })
+
+  it('creates detail file for each convoy', async () => {
+    await runEtl({ dbPath, outputDir })
+    expect(existsSync(join(outputDir, 'convoys', 'convoy-abc.json'))).toBe(true)
+    expect(existsSync(join(outputDir, 'convoys', 'convoy-def.json'))).toBe(true)
+  })
+
+  it('detail file includes artifacts and events fields', async () => {
+    await runEtl({ dbPath, outputDir })
+    const detail = JSON.parse(
+      readFileSync(join(outputDir, 'convoys', 'convoy-abc.json'), 'utf8'),
+    )
+    expect(Array.isArray(detail.artifacts)).toBe(true)
+    expect(typeof detail.artifact_count).toBe('number')
+    expect(typeof detail.has_more_events).toBe('boolean')
+    expect(Array.isArray(detail.events)).toBe(true)
+  })
+})

package/src/dashboard/scripts/etl.ts

@@ -0,0 +1,108 @@
+import { existsSync, mkdirSync, writeFileSync } from 'node:fs'
+import { resolve, dirname } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+const __filename = fileURLToPath(import.meta.url)
+const __dirname = dirname(__filename)
+
+export interface EtlOptions {
+  dbPath: string
+  outputDir: string
+}
+
+export interface EtlResult {
+  convoyCount: number
+  taskCount: number
+}
+
+const EMPTY_OVERALL_STATS = {
+  convoyCounts: { total: 0, running: 0, done: 0, failed: 0, gate_failed: 0 },
+  durationStats: { avg_sec: null, p95_sec: null, max_sec: null },
+  tokenCostTotals: { total_tokens: 0, total_cost_usd: 0 },
+  topAgents: [] as unknown[],
+  topModels: [] as unknown[],
+  dlqSummary: { count: 0, top_failure_types: [] as unknown[] },
+}
+
+export async function runEtl(options: EtlOptions): Promise<EtlResult> {
+  const { dbPath, outputDir } = options
+
+  mkdirSync(outputDir, { recursive: true })
+  mkdirSync(resolve(outputDir, 'convoys'), { recursive: true })
+
+  if (!existsSync(dbPath)) {
+    console.warn(`  \u26a0 No convoy database found at ${dbPath}. Writing empty JSON files.`)
+    writeFileSync(
+      resolve(outputDir, 'overall-stats.json'),
+      JSON.stringify(EMPTY_OVERALL_STATS, null, 2),
+      'utf8',
+    )
+    writeFileSync(resolve(outputDir, 'convoy-list.json'), JSON.stringify([], null, 2), 'utf8')
+    return { convoyCount: 0, taskCount: 0 }
+  }
+
+  const { createConvoyStore } = await import('../../cli/convoy/store.js')
+  const store = createConvoyStore(dbPath)
+
+  try {
+    const overallStats = {
+      convoyCounts: store.getConvoyCounts(),
+      durationStats: store.getConvoyDurationStats(),
+      tokenCostTotals: store.getTokenAndCostTotals(),
+      topAgents: store.getTopAgents(5),
+      topModels: store.getTopModels(5),
+      dlqSummary: store.getDlqSummary(),
+    }
+    writeFileSync(
+      resolve(outputDir, 'overall-stats.json'),
+      JSON.stringify(overallStats, null, 2),
+      'utf8',
+    )
+
+    const allConvoys = store.getConvoyList(1000, 0)
+    const convoyList = allConvoys.map(c => ({
+      id: c.id,
+      name: c.name,
+      status: c.status,
+      created_at: c.created_at,
+      finished_at: c.finished_at,
+      total_tokens: c.total_tokens,
+      total_cost_usd: c.total_cost_usd,
+    }))
+    writeFileSync(
+      resolve(outputDir, 'convoy-list.json'),
+      JSON.stringify(convoyList, null, 2),
+      'utf8',
+    )
+
+    let totalTasks = 0
+    for (const convoy of allConvoys) {
+      const detail = store.getConvoyDetails(convoy.id)
+      if (detail) {
+        totalTasks += detail.tasks.length
+        writeFileSync(
+          resolve(outputDir, 'convoys', `${convoy.id}.json`),
+          JSON.stringify(detail, null, 2),
+          'utf8',
+        )
+      }
+    }
+
+    console.log(`ETL complete: ${allConvoys.length} convoys exported, ${totalTasks} tasks.`)
+    return { convoyCount: allConvoys.length, taskCount: totalTasks }
+  } finally {
+    store.close()
+  }
+}
+
+const isMain =
+  process.argv[1] != null &&
+  fileURLToPath(import.meta.url) === resolve(process.argv[1])
+if (isMain) {
+  const dbPath = resolve(process.cwd(), '.opencastle', 'convoy.db')
+  const outputDir = resolve(__dirname, '..', 'public', 'data')
+  runEtl({ dbPath, outputDir }).catch((err: unknown) => {
+    console.error('ETL failed:', (err as Error).message)
+    process.exit(1)
+  })
+}