opencastle 0.27.1 → 0.27.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/convoy/dashboard-types.d.ts +146 -0
- package/dist/cli/convoy/dashboard-types.d.ts.map +1 -0
- package/dist/cli/convoy/dashboard-types.js +2 -0
- package/dist/cli/convoy/dashboard-types.js.map +1 -0
- package/dist/cli/convoy/engine.d.ts +0 -1
- package/dist/cli/convoy/engine.d.ts.map +1 -1
- package/dist/cli/convoy/engine.js +31 -99
- package/dist/cli/convoy/engine.js.map +1 -1
- package/dist/cli/convoy/engine.test.js +88 -1
- package/dist/cli/convoy/engine.test.js.map +1 -1
- package/dist/cli/convoy/event-schemas.d.ts +9 -0
- package/dist/cli/convoy/event-schemas.d.ts.map +1 -0
- package/dist/cli/convoy/event-schemas.js +185 -0
- package/dist/cli/convoy/event-schemas.js.map +1 -0
- package/dist/cli/convoy/events.d.ts +8 -0
- package/dist/cli/convoy/events.d.ts.map +1 -1
- package/dist/cli/convoy/events.js +117 -5
- package/dist/cli/convoy/events.js.map +1 -1
- package/dist/cli/convoy/events.test.js +173 -3
- package/dist/cli/convoy/events.test.js.map +1 -1
- package/dist/cli/convoy/log-merge.test.d.ts +2 -0
- package/dist/cli/convoy/log-merge.test.d.ts.map +1 -0
- package/dist/cli/convoy/log-merge.test.js +147 -0
- package/dist/cli/convoy/log-merge.test.js.map +1 -0
- package/dist/cli/convoy/store.d.ts +52 -2
- package/dist/cli/convoy/store.d.ts.map +1 -1
- package/dist/cli/convoy/store.js +244 -17
- package/dist/cli/convoy/store.js.map +1 -1
- package/dist/cli/convoy/store.test.js +481 -22
- package/dist/cli/convoy/store.test.js.map +1 -1
- package/dist/cli/convoy/types.d.ts +271 -3
- package/dist/cli/convoy/types.d.ts.map +1 -1
- package/dist/cli/convoy/types.js +42 -1
- package/dist/cli/convoy/types.js.map +1 -1
- package/dist/cli/log.d.ts +11 -0
- package/dist/cli/log.d.ts.map +1 -1
- package/dist/cli/log.js +114 -2
- package/dist/cli/log.js.map +1 -1
- package/dist/cli/run.d.ts.map +1 -1
- package/dist/cli/run.js +37 -1
- package/dist/cli/run.js.map +1 -1
- package/package.json +6 -1
- package/src/cli/convoy/TELEMETRY.md +203 -0
- package/src/cli/convoy/dashboard-types.ts +141 -0
- package/src/cli/convoy/engine.test.ts +99 -1
- package/src/cli/convoy/engine.ts +27 -96
- package/src/cli/convoy/event-schemas.ts +195 -0
- package/src/cli/convoy/events.test.ts +207 -3
- package/src/cli/convoy/events.ts +119 -5
- package/src/cli/convoy/log-merge.test.ts +179 -0
- package/src/cli/convoy/store.test.ts +545 -22
- package/src/cli/convoy/store.ts +274 -21
- package/src/cli/convoy/types.ts +108 -3
- package/src/cli/log.ts +120 -2
- package/src/cli/run.ts +37 -1
- package/src/dashboard/dist/_astro/{index.DtnyD8a5.css → index.6L3_HsPT.css} +1 -1
- package/src/dashboard/dist/data/.gitkeep +0 -0
- package/src/dashboard/dist/data/convoy-list.json +20 -0
- package/src/dashboard/dist/data/convoys/demo-convoy-1.json +111 -0
- package/src/dashboard/dist/data/convoys/demo-convoy-2.json +72 -0
- package/src/dashboard/dist/data/overall-stats.json +36 -0
- package/src/dashboard/dist/index.html +701 -3
- package/src/dashboard/node_modules/.vite/deps/_metadata.json +6 -6
- package/src/dashboard/public/data/.gitkeep +0 -0
- package/src/dashboard/public/data/convoy-list.json +20 -0
- package/src/dashboard/public/data/convoys/demo-convoy-1.json +111 -0
- package/src/dashboard/public/data/convoys/demo-convoy-2.json +72 -0
- package/src/dashboard/public/data/overall-stats.json +36 -0
- package/src/dashboard/scripts/etl.test.ts +210 -0
- package/src/dashboard/scripts/etl.ts +121 -0
- package/src/dashboard/scripts/generate-demo-db.test.ts +30 -0
- package/src/dashboard/scripts/generate-demo-db.ts +140 -0
- package/src/dashboard/scripts/integration-test.ts +504 -0
- package/src/dashboard/scripts/verify-demo-data.sh +51 -0
- package/src/dashboard/src/pages/index.astro +854 -15
- package/src/dashboard/src/styles/dashboard.css +557 -1
- package/src/orchestrator/prompts/generate-convoy.prompt.md +212 -13
package/src/cli/convoy/engine.ts
CHANGED
|
@@ -18,7 +18,7 @@ import { promisify } from 'node:util'
|
|
|
18
18
|
import type { Task, TaskSpec, AgentAdapter, ExecuteResult, ReviewHeuristics } from '../types.js'
|
|
19
19
|
import { createConvoyStore, ConvoyArtifactLimitError, type ConvoyStore } from './store.js'
|
|
20
20
|
import { acquireEngineLock } from './lock.js'
|
|
21
|
-
import { createEventEmitter, type ConvoyEventEmitter } from './events.js'
|
|
21
|
+
import { createEventEmitter, ndjsonPathForConvoy, recoverNdjson, type ConvoyEventEmitter } from './events.js'
|
|
22
22
|
import { createWorktreeManager, type WorktreeManager } from './worktree.js'
|
|
23
23
|
import { createMergeQueue, MergeConflictError, type MergeQueue } from './merge.js'
|
|
24
24
|
import { createHealthMonitor, detectDrift } from './health.js'
|
|
@@ -222,84 +222,6 @@ export async function ensureBranch(branchName: string, basePath: string): Promis
|
|
|
222
222
|
}
|
|
223
223
|
}
|
|
224
224
|
|
|
225
|
-
// ── Internal helpers ──────────────────────────────────────────────────────────
|
|
226
|
-
|
|
227
|
-
/**
|
|
228
|
-
* Truncate any trailing partial line in the NDJSON file, then replay any SQLite
|
|
229
|
-
* events for the given convoy that are missing from the file.
|
|
230
|
-
* Exported for unit testing.
|
|
231
|
-
*/
|
|
232
|
-
function safeJsonParse(raw: string): Record<string, unknown> {
|
|
233
|
-
try {
|
|
234
|
-
return JSON.parse(raw) as Record<string, unknown>
|
|
235
|
-
} catch {
|
|
236
|
-
return {}
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
export function recoverNdjson(store: ConvoyStore, convoyId: string, ndjsonPath: string): void {
|
|
241
|
-
// 1. Read the NDJSON file (if it exists)
|
|
242
|
-
let fileContent: string
|
|
243
|
-
try {
|
|
244
|
-
fileContent = readFileSync(ndjsonPath, 'utf8')
|
|
245
|
-
} catch {
|
|
246
|
-
fileContent = ''
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
// 2. Truncate any partial trailing line (no \n terminator)
|
|
250
|
-
if (fileContent.length > 0 && !fileContent.endsWith('\n')) {
|
|
251
|
-
const lastNewline = fileContent.lastIndexOf('\n')
|
|
252
|
-
if (lastNewline === -1) {
|
|
253
|
-
writeFileSync(ndjsonPath, '')
|
|
254
|
-
fileContent = ''
|
|
255
|
-
} else {
|
|
256
|
-
writeFileSync(ndjsonPath, fileContent.slice(0, lastNewline + 1))
|
|
257
|
-
fileContent = fileContent.slice(0, lastNewline + 1)
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
// 3. Count valid NDJSON event IDs for this convoy
|
|
262
|
-
const ndjsonIds = new Set<number>()
|
|
263
|
-
for (const line of fileContent.split('\n')) {
|
|
264
|
-
if (!line.trim()) continue
|
|
265
|
-
try {
|
|
266
|
-
const parsed = JSON.parse(line) as Record<string, unknown>
|
|
267
|
-
if (parsed.convoy_id === convoyId && parsed._event_id != null) {
|
|
268
|
-
ndjsonIds.add(parsed._event_id as number)
|
|
269
|
-
}
|
|
270
|
-
} catch {
|
|
271
|
-
// Skip unparseable lines
|
|
272
|
-
}
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
// 4. Get all SQLite events for this convoy
|
|
276
|
-
const sqliteEvents = store.getEvents(convoyId)
|
|
277
|
-
|
|
278
|
-
// 5. Replay missing events (those in SQLite but not in NDJSON)
|
|
279
|
-
const missing = sqliteEvents.filter(e => e.id != null && !ndjsonIds.has(e.id!))
|
|
280
|
-
if (missing.length > 0) {
|
|
281
|
-
const fd = openSync(ndjsonPath, 'a')
|
|
282
|
-
try {
|
|
283
|
-
for (const event of missing) {
|
|
284
|
-
const parsedData = event.data ? safeJsonParse(event.data) : {}
|
|
285
|
-
const record = {
|
|
286
|
-
...parsedData,
|
|
287
|
-
_event_id: event.id,
|
|
288
|
-
timestamp: event.created_at,
|
|
289
|
-
type: event.type,
|
|
290
|
-
convoy_id: event.convoy_id,
|
|
291
|
-
task_id: event.task_id,
|
|
292
|
-
worker_id: event.worker_id,
|
|
293
|
-
}
|
|
294
|
-
appendFileSync(fd, JSON.stringify(record) + '\n')
|
|
295
|
-
}
|
|
296
|
-
fsyncSync(fd)
|
|
297
|
-
} finally {
|
|
298
|
-
closeSync(fd)
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
|
|
303
225
|
// ── Convoy guard ──────────────────────────────────────────────────────────────
|
|
304
226
|
|
|
305
227
|
export interface ConvoyGuardResult {
|
|
@@ -336,12 +258,10 @@ export function runConvoyGuard(
|
|
|
336
258
|
try {
|
|
337
259
|
const content = readFileSync(ndjsonPath, 'utf8')
|
|
338
260
|
const lines = content.split('\n').filter(l => l.trim())
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
})
|
|
342
|
-
if (convoyLines.length < completedTasks.length) {
|
|
261
|
+
// Per-convoy file — all records belong to this convoy, no need to filter by convoy_id
|
|
262
|
+
if (lines.length < completedTasks.length) {
|
|
343
263
|
warnings.push(
|
|
344
|
-
`NDJSON record count (${
|
|
264
|
+
`NDJSON record count (${lines.length}) < completed tasks (${completedTasks.length})`,
|
|
345
265
|
)
|
|
346
266
|
}
|
|
347
267
|
} catch {
|
|
@@ -365,9 +285,16 @@ export function runConvoyGuard(
|
|
|
365
285
|
}
|
|
366
286
|
|
|
367
287
|
// Check 4: Gate results recorded for all gates that ran
|
|
368
|
-
const gateEvents = events.filter(e =>
|
|
369
|
-
e.type === 'built_in_gate_result'
|
|
370
|
-
|
|
288
|
+
const gateEvents = events.filter(e => {
|
|
289
|
+
if (e.type === 'built_in_gate_result') return true
|
|
290
|
+
if (e.data == null) return false
|
|
291
|
+
try {
|
|
292
|
+
const parsed = JSON.parse(e.data) as Record<string, unknown>
|
|
293
|
+
return 'gate' in parsed
|
|
294
|
+
} catch {
|
|
295
|
+
return false
|
|
296
|
+
}
|
|
297
|
+
})
|
|
371
298
|
const tasksWithGates = tasks.filter(t => t.gates)
|
|
372
299
|
if (tasksWithGates.length > 0 && gateEvents.length === 0) {
|
|
373
300
|
warnings.push('Tasks have gates configured but no gate result events found')
|
|
@@ -1065,6 +992,7 @@ async function runConvoy(
|
|
|
1065
992
|
skipTask(t.id, `on_exhausted: stop — task "${taskRecord.id}" exhausted retries`)
|
|
1066
993
|
}
|
|
1067
994
|
store.updateConvoyStatus(convoyId, 'failed')
|
|
995
|
+
events.emit('convoy_failed', { status: 'failed', reason: `on_exhausted: stop — task "${taskRecord.id}" exhausted retries` }, { convoy_id: convoyId })
|
|
1068
996
|
} else if (exhausted === 'dlq' || exhausted === 'skip') {
|
|
1069
997
|
// Default behavior: cascade failure to dependents only
|
|
1070
998
|
cascadeFailure(taskRecord.id)
|
|
@@ -2603,6 +2531,12 @@ async function runConvoy(
|
|
|
2603
2531
|
total_tokens: convoyTotalTokens,
|
|
2604
2532
|
})
|
|
2605
2533
|
|
|
2534
|
+
if (finalStatus === 'done') {
|
|
2535
|
+
events.emit('convoy_finished', { status: 'done' }, { convoy_id: convoyId })
|
|
2536
|
+
} else {
|
|
2537
|
+
events.emit('convoy_failed', { status: finalStatus, reason: finalStatus === 'gate-failed' ? 'Gate check failed' : 'One or more tasks failed' }, { convoy_id: convoyId })
|
|
2538
|
+
}
|
|
2539
|
+
|
|
2606
2540
|
// Run convoy guard checks
|
|
2607
2541
|
const guardResult = runConvoyGuard(store, convoyId, wtManager, ndjsonPath, spec.guard)
|
|
2608
2542
|
if (guardResult.warnings.length > 0) {
|
|
@@ -2690,9 +2624,8 @@ export function createConvoyEngine(options: ConvoyEngineOptions): ConvoyEngine {
|
|
|
2690
2624
|
|
|
2691
2625
|
const store = createConvoyStore(dbPath)
|
|
2692
2626
|
const ndjsonPath = options.logsDir
|
|
2693
|
-
? join(options.logsDir, '
|
|
2694
|
-
:
|
|
2695
|
-
mkdirSync(dirname(ndjsonPath), { recursive: true })
|
|
2627
|
+
? join(options.logsDir, 'convoys', `${convoyId}.ndjson`)
|
|
2628
|
+
: ndjsonPathForConvoy(convoyId, basePath)
|
|
2696
2629
|
const events = createEventEmitter(store, { ndjsonPath })
|
|
2697
2630
|
const wtManager = options._worktreeManager ?? createWorktreeManager(basePath)
|
|
2698
2631
|
const mergeQueue = options._mergeQueue ?? createMergeQueue(basePath)
|
|
@@ -2807,9 +2740,8 @@ export function createConvoyEngine(options: ConvoyEngineOptions): ConvoyEngine {
|
|
|
2807
2740
|
|
|
2808
2741
|
const store = createConvoyStore(dbPath)
|
|
2809
2742
|
const ndjsonPath = options.logsDir
|
|
2810
|
-
? join(options.logsDir, '
|
|
2811
|
-
:
|
|
2812
|
-
mkdirSync(dirname(ndjsonPath), { recursive: true })
|
|
2743
|
+
? join(options.logsDir, 'convoys', `${convoyId}.ndjson`)
|
|
2744
|
+
: ndjsonPathForConvoy(convoyId, basePath)
|
|
2813
2745
|
const events = createEventEmitter(store, { ndjsonPath })
|
|
2814
2746
|
const wtManager = options._worktreeManager ?? createWorktreeManager(basePath)
|
|
2815
2747
|
const mergeQueue = options._mergeQueue ?? createMergeQueue(basePath)
|
|
@@ -2876,9 +2808,8 @@ export function createConvoyEngine(options: ConvoyEngineOptions): ConvoyEngine {
|
|
|
2876
2808
|
mkdirSync(dirname(dbPath), { recursive: true })
|
|
2877
2809
|
const store = createConvoyStore(dbPath)
|
|
2878
2810
|
const ndjsonPath = options.logsDir
|
|
2879
|
-
? join(options.logsDir, '
|
|
2880
|
-
:
|
|
2881
|
-
mkdirSync(dirname(ndjsonPath), { recursive: true })
|
|
2811
|
+
? join(options.logsDir, 'convoys', `${convoyId}.ndjson`)
|
|
2812
|
+
: ndjsonPathForConvoy(convoyId, basePath)
|
|
2882
2813
|
const events = createEventEmitter(store, { ndjsonPath })
|
|
2883
2814
|
try {
|
|
2884
2815
|
const allTasks = store.getTasksByConvoy(convoyId)
|
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
import * as v from 'valibot'
|
|
2
|
+
|
|
3
|
+
type AnySchema = v.BaseSchema<unknown, unknown, v.BaseIssue<unknown>>
|
|
4
|
+
|
|
5
|
+
export const EVENT_DATA_SCHEMAS: Record<string, AnySchema> = {
|
|
6
|
+
convoy_started: v.looseObject({ name: v.optional(v.string()) }),
|
|
7
|
+
convoy_finished: v.looseObject({ status: v.string() }),
|
|
8
|
+
convoy_failed: v.looseObject({ status: v.string(), reason: v.optional(v.string()) }),
|
|
9
|
+
convoy_guard: v.looseObject({ checks: v.optional(v.array(v.string())) }),
|
|
10
|
+
|
|
11
|
+
task_started: v.looseObject({ worker_id: v.optional(v.string()) }),
|
|
12
|
+
task_done: v.looseObject({
|
|
13
|
+
status: v.optional(v.string()),
|
|
14
|
+
retries: v.optional(v.number()),
|
|
15
|
+
worker_id: v.optional(v.string()),
|
|
16
|
+
}),
|
|
17
|
+
task_failed: v.looseObject({
|
|
18
|
+
reason: v.string(),
|
|
19
|
+
worker_id: v.optional(v.string()),
|
|
20
|
+
gate: v.optional(v.string()),
|
|
21
|
+
hook: v.optional(v.string()),
|
|
22
|
+
}),
|
|
23
|
+
task_skipped: v.looseObject({ reason: v.string() }),
|
|
24
|
+
task_retried: v.looseObject({ previous_status: v.string() }),
|
|
25
|
+
task_waiting_input: v.looseObject({
|
|
26
|
+
task_id: v.optional(v.string()),
|
|
27
|
+
reason: v.optional(v.string()),
|
|
28
|
+
}),
|
|
29
|
+
|
|
30
|
+
review_started: v.looseObject({
|
|
31
|
+
level: v.string(),
|
|
32
|
+
task_id: v.optional(v.string()),
|
|
33
|
+
model: v.optional(v.string()),
|
|
34
|
+
}),
|
|
35
|
+
review_verdict: v.looseObject({
|
|
36
|
+
level: v.string(),
|
|
37
|
+
verdict: v.string(),
|
|
38
|
+
tokens: v.number(),
|
|
39
|
+
model: v.optional(v.string()),
|
|
40
|
+
feedback_length: v.optional(v.number()),
|
|
41
|
+
budget_exceeded: v.optional(v.boolean()),
|
|
42
|
+
budget_downgrade: v.optional(v.boolean()),
|
|
43
|
+
budget_skip: v.optional(v.boolean()),
|
|
44
|
+
passes: v.optional(v.number()),
|
|
45
|
+
blocks: v.optional(v.number()),
|
|
46
|
+
}),
|
|
47
|
+
dispute_opened: v.looseObject({
|
|
48
|
+
dispute_id: v.string(),
|
|
49
|
+
task_id: v.string(),
|
|
50
|
+
agent: v.optional(v.string()),
|
|
51
|
+
reason: v.optional(v.string()),
|
|
52
|
+
}),
|
|
53
|
+
dlq_entry_created: v.looseObject({
|
|
54
|
+
dlq_id: v.string(),
|
|
55
|
+
task_id: v.string(),
|
|
56
|
+
agent: v.optional(v.string()),
|
|
57
|
+
attempts: v.optional(v.number()),
|
|
58
|
+
}),
|
|
59
|
+
|
|
60
|
+
drift_check_result: v.looseObject({
|
|
61
|
+
score: v.optional(v.number()),
|
|
62
|
+
threshold: v.optional(v.number()),
|
|
63
|
+
passed: v.optional(v.boolean()),
|
|
64
|
+
}),
|
|
65
|
+
drift_detected: v.looseObject({
|
|
66
|
+
score: v.optional(v.number()),
|
|
67
|
+
files: v.optional(v.array(v.string())),
|
|
68
|
+
}),
|
|
69
|
+
|
|
70
|
+
circuit_breaker_tripped: v.looseObject({
|
|
71
|
+
agent: v.optional(v.string()),
|
|
72
|
+
failure_count: v.optional(v.number()),
|
|
73
|
+
threshold: v.optional(v.number()),
|
|
74
|
+
}),
|
|
75
|
+
circuit_breaker_fallback: v.looseObject({
|
|
76
|
+
original_agent: v.optional(v.string()),
|
|
77
|
+
fallback_agent: v.optional(v.string()),
|
|
78
|
+
task_id: v.optional(v.string()),
|
|
79
|
+
}),
|
|
80
|
+
circuit_breaker_blocked: v.looseObject({
|
|
81
|
+
agent: v.optional(v.string()),
|
|
82
|
+
task_id: v.optional(v.string()),
|
|
83
|
+
}),
|
|
84
|
+
|
|
85
|
+
merge_conflict_detected: v.looseObject({
|
|
86
|
+
task_id: v.optional(v.string()),
|
|
87
|
+
files: v.optional(v.array(v.string())),
|
|
88
|
+
}),
|
|
89
|
+
merge_conflict_failed: v.looseObject({
|
|
90
|
+
task_id: v.optional(v.string()),
|
|
91
|
+
error: v.optional(v.string()),
|
|
92
|
+
}),
|
|
93
|
+
|
|
94
|
+
file_injection_received: v.looseObject({
|
|
95
|
+
task_id: v.optional(v.string()),
|
|
96
|
+
from_task: v.optional(v.string()),
|
|
97
|
+
name: v.optional(v.string()),
|
|
98
|
+
}),
|
|
99
|
+
artifact_limit_reached: v.looseObject({
|
|
100
|
+
task_id: v.optional(v.string()),
|
|
101
|
+
limit: v.optional(v.number()),
|
|
102
|
+
current: v.optional(v.number()),
|
|
103
|
+
}),
|
|
104
|
+
|
|
105
|
+
agent_identity_captured: v.looseObject({
|
|
106
|
+
agent: v.optional(v.string()),
|
|
107
|
+
task_id: v.optional(v.string()),
|
|
108
|
+
}),
|
|
109
|
+
agent_identity_rejected: v.looseObject({
|
|
110
|
+
agent: v.optional(v.string()),
|
|
111
|
+
task_id: v.optional(v.string()),
|
|
112
|
+
reason: v.optional(v.string()),
|
|
113
|
+
}),
|
|
114
|
+
|
|
115
|
+
weak_area_skipped: v.looseObject({
|
|
116
|
+
agent: v.optional(v.string()),
|
|
117
|
+
weak_areas: v.optional(v.array(v.string())),
|
|
118
|
+
task_files: v.optional(v.array(v.string())),
|
|
119
|
+
}),
|
|
120
|
+
swarm_concurrency_update: v.looseObject({
|
|
121
|
+
new_concurrency: v.optional(v.number()),
|
|
122
|
+
reason: v.optional(v.string()),
|
|
123
|
+
}),
|
|
124
|
+
post_convoy_hook_failed: v.looseObject({
|
|
125
|
+
hook: v.optional(v.string()),
|
|
126
|
+
error: v.optional(v.string()),
|
|
127
|
+
}),
|
|
128
|
+
session: v.looseObject({
|
|
129
|
+
agent: v.optional(v.string()),
|
|
130
|
+
model: v.optional(v.string()),
|
|
131
|
+
task: v.optional(v.string()),
|
|
132
|
+
outcome: v.optional(v.string()),
|
|
133
|
+
duration_min: v.optional(v.number()),
|
|
134
|
+
}),
|
|
135
|
+
delegation: v.looseObject({
|
|
136
|
+
agent: v.optional(v.string()),
|
|
137
|
+
model: v.optional(v.string()),
|
|
138
|
+
tier: v.optional(v.string()),
|
|
139
|
+
mechanism: v.optional(v.string()),
|
|
140
|
+
outcome: v.optional(v.string()),
|
|
141
|
+
}),
|
|
142
|
+
secret_leak_prevented: v.looseObject({
|
|
143
|
+
original_type: v.optional(v.string()),
|
|
144
|
+
patterns: v.optional(v.array(v.string())),
|
|
145
|
+
task_id: v.optional(v.string()),
|
|
146
|
+
findings_count: v.optional(v.number()),
|
|
147
|
+
context: v.optional(v.string()),
|
|
148
|
+
}),
|
|
149
|
+
ndjson_write_failed: v.looseObject({ original_type: v.optional(v.string()) }),
|
|
150
|
+
built_in_gate_result: v.looseObject({
|
|
151
|
+
gate: v.string(),
|
|
152
|
+
passed: v.boolean(),
|
|
153
|
+
output: v.optional(v.string()),
|
|
154
|
+
level: v.optional(v.string()),
|
|
155
|
+
}),
|
|
156
|
+
watch_started: v.looseObject({
|
|
157
|
+
trigger_type: v.optional(v.string()),
|
|
158
|
+
pid: v.optional(v.number()),
|
|
159
|
+
}),
|
|
160
|
+
watch_cycle_start: v.looseObject({
|
|
161
|
+
cycle_number: v.optional(v.number()),
|
|
162
|
+
triggered_by: v.optional(v.string()),
|
|
163
|
+
}),
|
|
164
|
+
watch_cycle_end: v.looseObject({
|
|
165
|
+
cycle_number: v.optional(v.number()),
|
|
166
|
+
status: v.optional(v.string()),
|
|
167
|
+
}),
|
|
168
|
+
watch_stopped: v.looseObject({ reason: v.optional(v.string()) }),
|
|
169
|
+
worker_killed: v.looseObject({
|
|
170
|
+
reason: v.optional(v.string()),
|
|
171
|
+
worker_id: v.optional(v.string()),
|
|
172
|
+
task_id: v.optional(v.string()),
|
|
173
|
+
}),
|
|
174
|
+
discovered_issue: v.looseObject({
|
|
175
|
+
task_id: v.optional(v.string()),
|
|
176
|
+
title: v.optional(v.string()),
|
|
177
|
+
file: v.optional(v.string()),
|
|
178
|
+
description: v.optional(v.string()),
|
|
179
|
+
severity: v.optional(v.string()),
|
|
180
|
+
}),
|
|
181
|
+
}
|
|
182
|
+
export function validateEventData(
|
|
183
|
+
type: string,
|
|
184
|
+
data: unknown,
|
|
185
|
+
): { valid: boolean; issues?: string[] } {
|
|
186
|
+
const schema = EVENT_DATA_SCHEMAS[type]
|
|
187
|
+
if (schema === undefined) return { valid: true }
|
|
188
|
+
if (data === undefined || data === null) return { valid: true }
|
|
189
|
+
const result = v.safeParse(schema, data)
|
|
190
|
+
if (result.success) return { valid: true }
|
|
191
|
+
return {
|
|
192
|
+
valid: false,
|
|
193
|
+
issues: result.issues.map((i) => i.message),
|
|
194
|
+
}
|
|
195
|
+
}
|
|
@@ -2,10 +2,10 @@ import { mkdtempSync, readFileSync, writeFileSync, rmSync, existsSync, mkdirSync
|
|
|
2
2
|
import { tmpdir } from 'node:os'
|
|
3
3
|
import { join } from 'node:path'
|
|
4
4
|
import { realpathSync } from 'node:fs'
|
|
5
|
-
import { describe, it, expect, beforeEach, afterEach } from 'vitest'
|
|
5
|
+
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
|
6
6
|
import { createConvoyStore } from './store.js'
|
|
7
|
-
import { createEventEmitter } from './events.js'
|
|
8
|
-
import {
|
|
7
|
+
import { createEventEmitter, ndjsonPathForConvoy, recoverNdjson, validateEventType } from './events.js'
|
|
8
|
+
import { KNOWN_EVENT_TYPES } from './types.js'
|
|
9
9
|
import type { ConvoyStore } from './store.js'
|
|
10
10
|
|
|
11
11
|
let tmpDir: string
|
|
@@ -34,6 +34,13 @@ afterEach(() => {
|
|
|
34
34
|
})
|
|
35
35
|
|
|
36
36
|
describe('createEventEmitter', () => {
|
|
37
|
+
it('throws TypeError when options is a string', () => {
|
|
38
|
+
expect(() => createEventEmitter(store, 'bad' as unknown as any)).toThrow(TypeError)
|
|
39
|
+
expect(() => createEventEmitter(store, 'bad' as unknown as any)).toThrow(
|
|
40
|
+
'createEventEmitter options must be an object, not a string',
|
|
41
|
+
)
|
|
42
|
+
})
|
|
43
|
+
|
|
37
44
|
it('inserts the event into SQLite', () => {
|
|
38
45
|
const emitter = createEventEmitter(store)
|
|
39
46
|
emitter.emit('task_started', { msg: 'started' }, { convoy_id: 'c1' })
|
|
@@ -127,6 +134,33 @@ describe('createEventEmitter', () => {
|
|
|
127
134
|
emitter.close()
|
|
128
135
|
expect(() => emitter.close()).not.toThrow()
|
|
129
136
|
})
|
|
137
|
+
|
|
138
|
+
it('sanitizes reserved keys from caller data before NDJSON write', () => {
|
|
139
|
+
const emitter = createEventEmitter(store, { ndjsonPath })
|
|
140
|
+
emitter.emit(
|
|
141
|
+
'task_started',
|
|
142
|
+
{
|
|
143
|
+
convoy_id: 'attacker',
|
|
144
|
+
timestamp: 'fake',
|
|
145
|
+
_event_id: 999,
|
|
146
|
+
type: 'evil',
|
|
147
|
+
task_id: 'injected',
|
|
148
|
+
worker_id: 'hacker',
|
|
149
|
+
custom_field: 'ok',
|
|
150
|
+
},
|
|
151
|
+
{ convoy_id: 'c1', task_id: 't1', worker_id: 'w1' },
|
|
152
|
+
)
|
|
153
|
+
emitter.close()
|
|
154
|
+
const content = readFileSync(ndjsonPath, 'utf8')
|
|
155
|
+
const line = JSON.parse(content.trim())
|
|
156
|
+
expect(line.convoy_id).toBe('c1')
|
|
157
|
+
expect(line.task_id).toBe('t1')
|
|
158
|
+
expect(line.worker_id).toBe('w1')
|
|
159
|
+
expect(line.type).toBe('task_started')
|
|
160
|
+
expect(line._event_id).not.toBe(999)
|
|
161
|
+
expect(line.timestamp).not.toBe('fake')
|
|
162
|
+
expect(line.custom_field).toBe('ok')
|
|
163
|
+
})
|
|
130
164
|
})
|
|
131
165
|
|
|
132
166
|
describe('crash resilience', () => {
|
|
@@ -255,5 +289,175 @@ describe('crash resilience', () => {
|
|
|
255
289
|
const linesAfter = readFileSync(ndjsonPath, 'utf8').split('\n').filter(l => l.trim())
|
|
256
290
|
expect(linesAfter).toHaveLength(count)
|
|
257
291
|
})
|
|
292
|
+
|
|
293
|
+
it('6. canonical-field protection: event.data cannot override DB row fields', () => {
|
|
294
|
+
// Insert an event whose data contains attacker-controlled reserved keys
|
|
295
|
+
store.insertEvent({
|
|
296
|
+
convoy_id: 'c1',
|
|
297
|
+
task_id: 'legit-task',
|
|
298
|
+
worker_id: 'legit-worker',
|
|
299
|
+
type: 'task_done',
|
|
300
|
+
data: JSON.stringify({
|
|
301
|
+
convoy_id: 'attacker',
|
|
302
|
+
timestamp: 'fake',
|
|
303
|
+
_event_id: 9999,
|
|
304
|
+
type: 'evil',
|
|
305
|
+
task_id: 'injected',
|
|
306
|
+
worker_id: 'hacker',
|
|
307
|
+
safe_field: 'this-is-fine',
|
|
308
|
+
}),
|
|
309
|
+
created_at: new Date().toISOString(),
|
|
310
|
+
})
|
|
311
|
+
|
|
312
|
+
recoverNdjson(store, 'c1', ndjsonPath)
|
|
313
|
+
|
|
314
|
+
const lines = readFileSync(ndjsonPath, 'utf8').split('\n').filter(l => l.trim())
|
|
315
|
+
expect(lines).toHaveLength(1)
|
|
316
|
+
const record = JSON.parse(lines[0]) as Record<string, unknown>
|
|
317
|
+
|
|
318
|
+
// Canonical fields must come from the DB row, not from data
|
|
319
|
+
expect(record.convoy_id).toBe('c1')
|
|
320
|
+
expect(record.task_id).toBe('legit-task')
|
|
321
|
+
expect(record.worker_id).toBe('legit-worker')
|
|
322
|
+
expect(record.type).toBe('task_done')
|
|
323
|
+
expect(record._event_id).not.toBe(9999)
|
|
324
|
+
expect(record.timestamp).not.toBe('fake')
|
|
325
|
+
|
|
326
|
+
// Attacker values must not appear
|
|
327
|
+
expect(record.convoy_id).not.toBe('attacker')
|
|
328
|
+
expect(record.type).not.toBe('evil')
|
|
329
|
+
expect(record.task_id).not.toBe('injected')
|
|
330
|
+
expect(record.worker_id).not.toBe('hacker')
|
|
331
|
+
|
|
332
|
+
// Safe non-reserved fields are preserved
|
|
333
|
+
expect(record.safe_field).toBe('this-is-fine')
|
|
334
|
+
})
|
|
335
|
+
})
|
|
336
|
+
|
|
337
|
+
describe('KNOWN_EVENT_TYPES', () => {
|
|
338
|
+
it('contains all canonical event types', () => {
|
|
339
|
+
const canonical = [
|
|
340
|
+
'convoy_started', 'convoy_finished', 'convoy_failed', 'convoy_guard',
|
|
341
|
+
'task_started', 'task_done', 'task_failed', 'task_skipped', 'task_retried', 'task_waiting_input',
|
|
342
|
+
'review_started', 'review_verdict', 'dispute_opened', 'dlq_entry_created',
|
|
343
|
+
'drift_check_result', 'drift_detected',
|
|
344
|
+
'circuit_breaker_tripped', 'circuit_breaker_fallback', 'circuit_breaker_blocked',
|
|
345
|
+
'merge_conflict_detected', 'merge_conflict_failed',
|
|
346
|
+
'file_injection_received', 'artifact_limit_reached',
|
|
347
|
+
'agent_identity_captured', 'agent_identity_rejected',
|
|
348
|
+
'weak_area_skipped', 'swarm_concurrency_update', 'post_convoy_hook_failed',
|
|
349
|
+
'session', 'delegation',
|
|
350
|
+
'secret_leak_prevented', 'ndjson_write_failed', 'built_in_gate_result',
|
|
351
|
+
'watch_started', 'watch_cycle_start', 'watch_cycle_end', 'watch_stopped',
|
|
352
|
+
'worker_killed', 'discovered_issue',
|
|
353
|
+
]
|
|
354
|
+
for (const type of canonical) {
|
|
355
|
+
expect(KNOWN_EVENT_TYPES.has(type)).toBe(true)
|
|
356
|
+
}
|
|
357
|
+
})
|
|
358
|
+
|
|
359
|
+
it('has no duplicates (Set size matches array)', () => {
|
|
360
|
+
expect(KNOWN_EVENT_TYPES.size).toBeGreaterThanOrEqual(37)
|
|
361
|
+
})
|
|
362
|
+
})
|
|
363
|
+
|
|
364
|
+
describe('ndjsonPathForConvoy', () => {
|
|
365
|
+
it('returns correct per-convoy path with default basePath', () => {
|
|
366
|
+
const result = ndjsonPathForConvoy('abc-123')
|
|
367
|
+
expect(result).toBe(join(process.cwd(), '.opencastle', 'logs', 'convoys', 'abc-123.ndjson'))
|
|
368
|
+
})
|
|
369
|
+
|
|
370
|
+
it('returns correct per-convoy path with custom basePath', () => {
|
|
371
|
+
const result = ndjsonPathForConvoy('xyz-456', '/custom/base')
|
|
372
|
+
expect(result).toBe(join('/custom/base', '.opencastle', 'logs', 'convoys', 'xyz-456.ndjson'))
|
|
373
|
+
})
|
|
374
|
+
})
|
|
375
|
+
|
|
376
|
+
describe('createEventEmitter directory creation', () => {
|
|
377
|
+
it('creates parent directory when ndjsonPath is in a non-existent directory', () => {
|
|
378
|
+
const nestedPath = join(tmpDir, 'nested', 'dir', 'events.ndjson')
|
|
379
|
+
const emitter = createEventEmitter(store, { ndjsonPath: nestedPath })
|
|
380
|
+
emitter.emit('convoy_started', { name: 'test' }, { convoy_id: 'c1' })
|
|
381
|
+
emitter.close()
|
|
382
|
+
expect(existsSync(nestedPath)).toBe(true)
|
|
383
|
+
const content = readFileSync(nestedPath, 'utf8')
|
|
384
|
+
const line = JSON.parse(content.trim())
|
|
385
|
+
expect(line.type).toBe('convoy_started')
|
|
386
|
+
})
|
|
387
|
+
})
|
|
388
|
+
|
|
389
|
+
describe('validateEventType', () => {
|
|
390
|
+
it('returns true for known event types', () => {
|
|
391
|
+
expect(validateEventType('convoy_started')).toBe(true)
|
|
392
|
+
expect(validateEventType('task_done')).toBe(true)
|
|
393
|
+
expect(validateEventType('watch_stopped')).toBe(true)
|
|
394
|
+
})
|
|
395
|
+
|
|
396
|
+
it('returns false for unknown event types', () => {
|
|
397
|
+
expect(validateEventType('unknown_event')).toBe(false)
|
|
398
|
+
expect(validateEventType('')).toBe(false)
|
|
399
|
+
expect(validateEventType('convoy_start')).toBe(false)
|
|
400
|
+
})
|
|
401
|
+
})
|
|
402
|
+
|
|
403
|
+
describe('emit-time data validation', () => {
|
|
404
|
+
let warnSpy: ReturnType<typeof vi.spyOn>
|
|
405
|
+
|
|
406
|
+
beforeEach(() => {
|
|
407
|
+
warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {})
|
|
408
|
+
})
|
|
409
|
+
|
|
410
|
+
afterEach(() => {
|
|
411
|
+
warnSpy.mockRestore()
|
|
412
|
+
})
|
|
413
|
+
|
|
414
|
+
it('valid data passes without warning', () => {
|
|
415
|
+
const emitter = createEventEmitter(store)
|
|
416
|
+
emitter.emit('convoy_finished', { status: 'done' }, { convoy_id: 'c1' })
|
|
417
|
+
emitter.close()
|
|
418
|
+
expect(warnSpy).not.toHaveBeenCalled()
|
|
419
|
+
})
|
|
420
|
+
|
|
421
|
+
it('invalid data shape warns with correct message', () => {
|
|
422
|
+
const emitter = createEventEmitter(store)
|
|
423
|
+
emitter.emit('convoy_finished', { status: 123 } as unknown as Record<string, unknown>, { convoy_id: 'c1' })
|
|
424
|
+
emitter.close()
|
|
425
|
+
expect(warnSpy).toHaveBeenCalledWith(
|
|
426
|
+
expect.stringContaining('Invalid data for event type "convoy_finished"'),
|
|
427
|
+
)
|
|
428
|
+
})
|
|
429
|
+
|
|
430
|
+
it('missing required field warns', () => {
|
|
431
|
+
const emitter = createEventEmitter(store)
|
|
432
|
+
// task_failed requires reason: string — passing {} should fail
|
|
433
|
+
emitter.emit('task_failed', {} as Record<string, unknown>, { convoy_id: 'c1' })
|
|
434
|
+
emitter.close()
|
|
435
|
+
expect(warnSpy).toHaveBeenCalledWith(
|
|
436
|
+
expect.stringContaining('Invalid data for event type "task_failed"'),
|
|
437
|
+
)
|
|
438
|
+
})
|
|
439
|
+
|
|
440
|
+
it('extra fields are allowed (no warning)', () => {
|
|
441
|
+
const emitter = createEventEmitter(store)
|
|
442
|
+
emitter.emit('convoy_started', { name: 'test', extra: true } as Record<string, unknown>, { convoy_id: 'c1' })
|
|
443
|
+
emitter.close()
|
|
444
|
+
expect(warnSpy).not.toHaveBeenCalled()
|
|
445
|
+
})
|
|
446
|
+
|
|
447
|
+
it('undefined data is valid', () => {
|
|
448
|
+
const emitter = createEventEmitter(store)
|
|
449
|
+
emitter.emit('convoy_started', undefined, { convoy_id: 'c1' })
|
|
450
|
+
emitter.close()
|
|
451
|
+
expect(warnSpy).not.toHaveBeenCalled()
|
|
452
|
+
})
|
|
453
|
+
|
|
454
|
+
it('unknown event type bypasses data validation (only one warning)', () => {
|
|
455
|
+
const emitter = createEventEmitter(store)
|
|
456
|
+
emitter.emit('unknown_type_xyz', { any: 'data' }, { convoy_id: 'c1' })
|
|
457
|
+
emitter.close()
|
|
458
|
+
expect(warnSpy).toHaveBeenCalledTimes(1)
|
|
459
|
+
expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('Unknown event type: "unknown_type_xyz"'))
|
|
460
|
+
expect(warnSpy).not.toHaveBeenCalledWith(expect.stringContaining('Invalid data'))
|
|
461
|
+
})
|
|
258
462
|
})
|
|
259
463
|
|