opencastle 0.27.3 → 0.29.0

package/src/dashboard/scripts/etl.test.ts

@@ -1,4 +1,4 @@
-import { mkdtempSync, rmSync, realpathSync, readFileSync, existsSync, mkdirSync } from 'node:fs'
+import { mkdtempSync, rmSync, realpathSync, readFileSync, existsSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import { join } from 'node:path'
 import { describe, it, expect, beforeEach, afterEach } from 'vitest'
@@ -47,14 +47,13 @@ describe('runEtl — no database', () => {
   it('returns zero counts when db is missing', async () => {
     const dbPath = join(tmpDir, 'nonexistent.db')
     const result = await runEtl({ dbPath, outputDir })
-    expect(result).toEqual({ convoyCount: 0, taskCount: 0 })
+    expect(result).toEqual({ convoyCount: 0 })
   })
 
-  it('creates the output directory structure even when db is missing', async () => {
+  it('creates the output directory when db is missing', async () => {
     const dbPath = join(tmpDir, 'nonexistent.db')
     await runEtl({ dbPath, outputDir })
     expect(existsSync(outputDir)).toBe(true)
-    expect(existsSync(join(outputDir, 'convoys'))).toBe(true)
   })
 })
 
@@ -120,10 +119,9 @@ describe('runEtl — with seeded database', () => {
     }
   })
 
-  it('returns correct convoy and task counts', async () => {
+  it('returns correct convoy count', async () => {
     const result = await runEtl({ dbPath, outputDir })
     expect(result.convoyCount).toBe(2)
-    expect(result.taskCount).toBe(2)
   })
 
   it('overall-stats.json has correct convoy counts', async () => {
@@ -151,60 +149,4 @@ describe('runEtl — with seeded database', () => {
       expect(item).toHaveProperty('total_cost_usd')
     }
   })
-
-  it('creates per-convoy detail JSON files', async () => {
-    await runEtl({ dbPath, outputDir })
-    const detailPath = join(outputDir, 'convoys', 'convoy-abc.json')
-    expect(existsSync(detailPath)).toBe(true)
-    const detail = JSON.parse(readFileSync(detailPath, 'utf8'))
-    expect(detail.convoy.id).toBe('convoy-abc')
-    expect(detail.convoy.name).toBe('Test Convoy')
-    expect(detail.convoy.status).toBe('done')
-    expect(detail.convoy).toHaveProperty('branch')
-    expect(detail.convoy).toHaveProperty('total_tokens')
-    expect(detail.convoy).toHaveProperty('total_cost_usd')
-    expect(detail).toHaveProperty('taskSummary')
-    expect(detail.taskSummary).toHaveProperty('total')
-    expect(Array.isArray(detail.tasks)).toBe(true)
-  })
-
-  it('detail file has correct task fields', async () => {
-    await runEtl({ dbPath, outputDir })
-    const detail = JSON.parse(
-      readFileSync(join(outputDir, 'convoys', 'convoy-abc.json'), 'utf8'),
-    )
-    expect(detail.tasks).toHaveLength(2)
-    for (const task of detail.tasks) {
-      expect(task).toHaveProperty('id')
-      expect(task).toHaveProperty('phase')
-      expect(task).toHaveProperty('agent')
-      expect(task).toHaveProperty('model')
-      expect(task).toHaveProperty('status')
-      expect(task).toHaveProperty('retries')
-      expect(task).toHaveProperty('started_at')
-      expect(task).toHaveProperty('finished_at')
-      expect(task).toHaveProperty('total_tokens')
-      expect(task).toHaveProperty('cost_usd')
-      expect(task).toHaveProperty('review_level')
-      expect(task).toHaveProperty('review_verdict')
-      expect(task).toHaveProperty('drift_score')
-    }
-  })
-
-  it('creates detail file for each convoy', async () => {
-    await runEtl({ dbPath, outputDir })
-    expect(existsSync(join(outputDir, 'convoys', 'convoy-abc.json'))).toBe(true)
-    expect(existsSync(join(outputDir, 'convoys', 'convoy-def.json'))).toBe(true)
-  })
-
-  it('detail file includes artifacts and events fields', async () => {
-    await runEtl({ dbPath, outputDir })
-    const detail = JSON.parse(
-      readFileSync(join(outputDir, 'convoys', 'convoy-abc.json'), 'utf8'),
-    )
-    expect(Array.isArray(detail.artifacts)).toBe(true)
-    expect(typeof detail.artifact_count).toBe('number')
-    expect(typeof detail.has_more_events).toBe('boolean')
-    expect(Array.isArray(detail.events)).toBe(true)
-  })
 })

package/src/dashboard/scripts/etl.ts

@@ -12,7 +12,6 @@ export interface EtlOptions {
 
 export interface EtlResult {
   convoyCount: number
-  taskCount: number
 }
 
 const EMPTY_OVERALL_STATS = {
@@ -28,7 +27,6 @@ export async function runEtl(options: EtlOptions): Promise<EtlResult> {
   const { dbPath, outputDir } = options
 
   mkdirSync(outputDir, { recursive: true })
-  mkdirSync(resolve(outputDir, 'convoys'), { recursive: true })
 
   if (!existsSync(dbPath)) {
     console.warn(`  \u26a0 No convoy database found at ${dbPath}. Writing empty JSON files.`)
@@ -38,7 +36,7 @@ export async function runEtl(options: EtlOptions): Promise<EtlResult> {
       'utf8',
     )
     writeFileSync(resolve(outputDir, 'convoy-list.json'), JSON.stringify([], null, 2), 'utf8')
-    return { convoyCount: 0, taskCount: 0 }
+    return { convoyCount: 0 }
   }
 
   const { createConvoyStore } = await import('../../cli/convoy/store.js')
@@ -65,6 +63,7 @@ export async function runEtl(options: EtlOptions): Promise<EtlResult> {
       name: c.name,
       status: c.status,
       created_at: c.created_at,
+      started_at: c.started_at,
       finished_at: c.finished_at,
       total_tokens: c.total_tokens,
       total_cost_usd: c.total_cost_usd,
@@ -75,21 +74,23 @@ export async function runEtl(options: EtlOptions): Promise<EtlResult> {
       'utf8',
     )
 
-    let totalTasks = 0
-    for (const convoy of allConvoys) {
-      const detail = store.getConvoyDetails(convoy.id)
+    mkdirSync(resolve(outputDir, 'convoys'), { recursive: true })
+    let detailCount = 0
+    for (const c of allConvoys) {
+      const detail = store.getConvoyDetails(c.id)
       if (detail) {
-        totalTasks += detail.tasks.length
         writeFileSync(
-          resolve(outputDir, 'convoys', `${convoy.id}.json`),
+          resolve(outputDir, 'convoys', c.id + '.json'),
           JSON.stringify(detail, null, 2),
           'utf8',
         )
+        detailCount++
       }
     }
 
-    console.log(`ETL complete: ${allConvoys.length} convoys exported, ${totalTasks} tasks.`)
-    return { convoyCount: allConvoys.length, taskCount: totalTasks }
+    console.log(`ETL complete: ${allConvoys.length} convoys summarized, ${detailCount} detail files generated.`)
+
+    return { convoyCount: allConvoys.length }
   } finally {
     store.close()
   }