opencastle 0.27.1 → 0.27.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/convoy/dashboard-types.d.ts +146 -0
- package/dist/cli/convoy/dashboard-types.d.ts.map +1 -0
- package/dist/cli/convoy/dashboard-types.js +2 -0
- package/dist/cli/convoy/dashboard-types.js.map +1 -0
- package/dist/cli/convoy/engine.d.ts +0 -1
- package/dist/cli/convoy/engine.d.ts.map +1 -1
- package/dist/cli/convoy/engine.js +31 -99
- package/dist/cli/convoy/engine.js.map +1 -1
- package/dist/cli/convoy/engine.test.js +88 -1
- package/dist/cli/convoy/engine.test.js.map +1 -1
- package/dist/cli/convoy/event-schemas.d.ts +9 -0
- package/dist/cli/convoy/event-schemas.d.ts.map +1 -0
- package/dist/cli/convoy/event-schemas.js +185 -0
- package/dist/cli/convoy/event-schemas.js.map +1 -0
- package/dist/cli/convoy/events.d.ts +8 -0
- package/dist/cli/convoy/events.d.ts.map +1 -1
- package/dist/cli/convoy/events.js +117 -5
- package/dist/cli/convoy/events.js.map +1 -1
- package/dist/cli/convoy/events.test.js +173 -3
- package/dist/cli/convoy/events.test.js.map +1 -1
- package/dist/cli/convoy/log-merge.test.d.ts +2 -0
- package/dist/cli/convoy/log-merge.test.d.ts.map +1 -0
- package/dist/cli/convoy/log-merge.test.js +147 -0
- package/dist/cli/convoy/log-merge.test.js.map +1 -0
- package/dist/cli/convoy/store.d.ts +52 -2
- package/dist/cli/convoy/store.d.ts.map +1 -1
- package/dist/cli/convoy/store.js +244 -17
- package/dist/cli/convoy/store.js.map +1 -1
- package/dist/cli/convoy/store.test.js +481 -22
- package/dist/cli/convoy/store.test.js.map +1 -1
- package/dist/cli/convoy/types.d.ts +271 -3
- package/dist/cli/convoy/types.d.ts.map +1 -1
- package/dist/cli/convoy/types.js +42 -1
- package/dist/cli/convoy/types.js.map +1 -1
- package/dist/cli/log.d.ts +11 -0
- package/dist/cli/log.d.ts.map +1 -1
- package/dist/cli/log.js +114 -2
- package/dist/cli/log.js.map +1 -1
- package/package.json +5 -1
- package/src/cli/convoy/TELEMETRY.md +203 -0
- package/src/cli/convoy/dashboard-types.ts +141 -0
- package/src/cli/convoy/engine.test.ts +99 -1
- package/src/cli/convoy/engine.ts +27 -96
- package/src/cli/convoy/event-schemas.ts +195 -0
- package/src/cli/convoy/events.test.ts +207 -3
- package/src/cli/convoy/events.ts +119 -5
- package/src/cli/convoy/log-merge.test.ts +179 -0
- package/src/cli/convoy/store.test.ts +545 -22
- package/src/cli/convoy/store.ts +274 -21
- package/src/cli/convoy/types.ts +108 -3
- package/src/cli/log.ts +120 -2
- package/src/dashboard/dist/_astro/{index.DtnyD8a5.css → index.6L3_HsPT.css} +1 -1
- package/src/dashboard/dist/data/.gitkeep +0 -0
- package/src/dashboard/dist/data/convoy-list.json +1 -0
- package/src/dashboard/dist/data/overall-stats.json +24 -0
- package/src/dashboard/dist/index.html +701 -3
- package/src/dashboard/node_modules/.vite/deps/_metadata.json +6 -6
- package/src/dashboard/public/data/.gitkeep +0 -0
- package/src/dashboard/public/data/convoy-list.json +1 -0
- package/src/dashboard/public/data/overall-stats.json +24 -0
- package/src/dashboard/scripts/etl.test.ts +210 -0
- package/src/dashboard/scripts/etl.ts +108 -0
- package/src/dashboard/scripts/integration-test.ts +504 -0
- package/src/dashboard/src/pages/index.astro +854 -15
- package/src/dashboard/src/styles/dashboard.css +557 -1
- package/src/orchestrator/prompts/generate-convoy.prompt.md +212 -13
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
# Convoy Telemetry Model
|
|
2
|
+
|
|
3
|
+
How Convoy concepts map to [OpenTelemetry](https://opentelemetry.io/) semantics.
|
|
4
|
+
|
|
5
|
+
## Conceptual Mapping
|
|
6
|
+
|
|
7
|
+
| Convoy Concept | OTel Concept | ID Field | Description |
|
|
8
|
+
|---------------|-------------|----------|-------------|
|
|
9
|
+
| **Convoy** | Trace | `convoy_id` → `trace_id` | A single execution run of a `.convoy.yml` spec |
|
|
10
|
+
| **Task** | Span | `task_id` → `span_id` | One unit of work within a convoy |
|
|
11
|
+
| **TaskStep** | Sub-span | `step_index` | Sequential steps within a multi-step task |
|
|
12
|
+
| **Event** | Log / SpanEvent | `type` | Structured occurrence during execution |
|
|
13
|
+
| **Metrics** | Derived aggregates | — | Computed from events (tokens, cost, duration) |
|
|
14
|
+
|
|
15
|
+
### ID Correlation
|
|
16
|
+
|
|
17
|
+
```
|
|
18
|
+
trace_id = convoy_id (globally unique, set at convoy creation)
|
|
19
|
+
span_id = task_id (unique within convoy, from spec)
|
|
20
|
+
worker_id = worker trace (ephemeral, tied to adapter process)
|
|
21
|
+
```
|
|
22
|
+
|
|
23
|
+
Every event carries `convoy_id`, `task_id`, and `worker_id` (all nullable) to enable correlation across the trace hierarchy.
|
|
24
|
+
|
|
25
|
+
## Storage
|
|
26
|
+
|
|
27
|
+
- **Primary**: SQLite (`convoy.db`) — durable, queryable, crash-safe
|
|
28
|
+
- **Supplementary**: NDJSON (`convoy-events.ndjson`) — append-only log for streaming/grep
|
|
29
|
+
|
|
30
|
+
SQLite is the source of truth. NDJSON is replayed from SQLite on crash recovery via `recoverNdjson()`.
|
|
31
|
+
|
|
32
|
+
### Write Strategy (v1)
|
|
33
|
+
|
|
34
|
+
NDJSON writes use synchronous `appendFileSync` + `fsyncSync` per event. This ensures crash-safety — every event is durable before the engine proceeds. Trade-offs:
|
|
35
|
+
|
|
36
|
+
- **Latency**: ~1-2ms per event (sync I/O). For convoys with <10,000 events this is negligible.
|
|
37
|
+
- **Throughput**: Not suitable for >10,000 events/second workloads.
|
|
38
|
+
- **Crash-safety**: Every event is fsynced before the engine continues, so a crash never loses the last event.
|
|
39
|
+
|
|
40
|
+
An async buffered writer is deferred as an optimization for Phase 5 if profiling shows sync writes become a bottleneck.
|
|
41
|
+
|
|
42
|
+
## Event Type Reference
|
|
43
|
+
|
|
44
|
+
All 39 canonical event types emitted by the convoy engine.
|
|
45
|
+
|
|
46
|
+
### Convoy Lifecycle
|
|
47
|
+
|
|
48
|
+
| Event Type | Source | Data Fields |
|
|
49
|
+
|-----------|--------|-------------|
|
|
50
|
+
| `convoy_started` | engine.ts | `name?: string` |
|
|
51
|
+
| `convoy_finished` | engine.ts | `status: string` |
|
|
52
|
+
| `convoy_failed` | engine.ts | `status: string; reason?: string` |
|
|
53
|
+
| `convoy_guard` | engine.ts | `checks?: string[]` |
|
|
54
|
+
|
|
55
|
+
### Task Lifecycle
|
|
56
|
+
|
|
57
|
+
| Event Type | Source | Data Fields |
|
|
58
|
+
|-----------|--------|-------------|
|
|
59
|
+
| `task_started` | engine.ts | `worker_id?: string` |
|
|
60
|
+
| `task_done` | engine.ts | `status?: string; retries?: number; worker_id?: string` |
|
|
61
|
+
| `task_failed` | engine.ts | `reason: string; worker_id?: string; gate?: string; hook?: string` |
|
|
62
|
+
| `task_skipped` | engine.ts | `reason: string` |
|
|
63
|
+
| `task_retried` | engine.ts | `previous_status: string` |
|
|
64
|
+
| `task_waiting_input` | engine.ts | `task_id?: string; reason?: string` |
|
|
65
|
+
|
|
66
|
+
### Review & Disputes
|
|
67
|
+
|
|
68
|
+
| Event Type | Source | Data Fields |
|
|
69
|
+
|-----------|--------|-------------|
|
|
70
|
+
| `review_started` | engine.ts | `level: string; task_id?: string; model?: string` |
|
|
71
|
+
| `review_verdict` | engine.ts | `level: string; verdict: string; tokens: number; model?: string; feedback_length?: number; budget_exceeded?: boolean; budget_downgrade?: boolean; budget_skip?: boolean; passes?: number; blocks?: number` |
|
|
72
|
+
| `dispute_opened` | engine.ts | `dispute_id: string; task_id: string; agent?: string; reason?: string` |
|
|
73
|
+
| `dlq_entry_created` | engine.ts | `dlq_id: string; task_id: string; agent?: string; attempts?: number` |
|
|
74
|
+
|
|
75
|
+
### Drift Detection
|
|
76
|
+
|
|
77
|
+
| Event Type | Source | Data Fields |
|
|
78
|
+
|-----------|--------|-------------|
|
|
79
|
+
| `drift_check_result` | engine.ts | `score?: number; threshold?: number; passed?: boolean` |
|
|
80
|
+
| `drift_detected` | engine.ts | `score?: number; files?: string[]` |
|
|
81
|
+
|
|
82
|
+
### Circuit Breaker
|
|
83
|
+
|
|
84
|
+
| Event Type | Source | Data Fields |
|
|
85
|
+
|-----------|--------|-------------|
|
|
86
|
+
| `circuit_breaker_tripped` | engine.ts | `agent?: string; failure_count?: number; threshold?: number` |
|
|
87
|
+
| `circuit_breaker_fallback` | engine.ts | `original_agent?: string; fallback_agent?: string; task_id?: string` |
|
|
88
|
+
| `circuit_breaker_blocked` | engine.ts | `agent?: string; task_id?: string` |
|
|
89
|
+
|
|
90
|
+
### Merge & Worktree
|
|
91
|
+
|
|
92
|
+
| Event Type | Source | Data Fields |
|
|
93
|
+
|-----------|--------|-------------|
|
|
94
|
+
| `merge_conflict_detected` | engine.ts | `task_id?: string; files?: string[]` |
|
|
95
|
+
| `merge_conflict_failed` | engine.ts | `task_id?: string; error?: string` |
|
|
96
|
+
|
|
97
|
+
### Artifacts & Injection
|
|
98
|
+
|
|
99
|
+
| Event Type | Source | Data Fields |
|
|
100
|
+
|-----------|--------|-------------|
|
|
101
|
+
| `file_injection_received` | engine.ts | `task_id?: string; from_task?: string; name?: string` |
|
|
102
|
+
| `artifact_limit_reached` | engine.ts | `task_id?: string; limit?: number; current?: number` |
|
|
103
|
+
|
|
104
|
+
### Agent Intelligence
|
|
105
|
+
|
|
106
|
+
| Event Type | Source | Data Fields |
|
|
107
|
+
|-----------|--------|-------------|
|
|
108
|
+
| `agent_identity_captured` | engine.ts | `agent?: string; task_id?: string` |
|
|
109
|
+
| `agent_identity_rejected` | engine.ts | `agent?: string; task_id?: string; reason?: string` |
|
|
110
|
+
| `weak_area_skipped` | engine.ts | `agent?: string; weak_areas?: string[]; task_files?: string[]` |
|
|
111
|
+
| `swarm_concurrency_update` | engine.ts | `new_concurrency?: number; reason?: string` |
|
|
112
|
+
|
|
113
|
+
### Hooks
|
|
114
|
+
|
|
115
|
+
| Event Type | Source | Data Fields |
|
|
116
|
+
|-----------|--------|-------------|
|
|
117
|
+
| `post_convoy_hook_failed` | engine.ts | `hook?: string; error?: string` |
|
|
118
|
+
|
|
119
|
+
### Observability / Session
|
|
120
|
+
|
|
121
|
+
| Event Type | Source | Data Fields |
|
|
122
|
+
|-----------|--------|-------------|
|
|
123
|
+
| `session` | engine.ts | `agent?: string; model?: string; task?: string; outcome?: string; duration_min?: number` |
|
|
124
|
+
| `delegation` | engine.ts | `agent?: string; model?: string; tier?: string; mechanism?: string; outcome?: string` |
|
|
125
|
+
|
|
126
|
+
### Security & Reliability
|
|
127
|
+
|
|
128
|
+
| Event Type | Source | Data Fields |
|
|
129
|
+
|-----------|--------|-------------|
|
|
130
|
+
| `secret_leak_prevented` | engine.ts, events.ts | `original_type?: string; patterns?: string[]; task_id?: string; findings_count?: number; context?: string` |
|
|
131
|
+
| `ndjson_write_failed` | events.ts | `original_type?: string` |
|
|
132
|
+
|
|
133
|
+
### Built-in Gates
|
|
134
|
+
|
|
135
|
+
| Event Type | Source | Data Fields |
|
|
136
|
+
|-----------|--------|-------------|
|
|
137
|
+
| `built_in_gate_result` | engine.ts | `gate: string; passed: boolean; output?: string; level?: string` |
|
|
138
|
+
|
|
139
|
+
### Watch Mode
|
|
140
|
+
|
|
141
|
+
| Event Type | Source | Data Fields |
|
|
142
|
+
|-----------|--------|-------------|
|
|
143
|
+
| `watch_started` | watch.ts | `trigger_type?: string; pid?: number` |
|
|
144
|
+
| `watch_cycle_start` | watch.ts | `cycle_number?: number; triggered_by?: string` |
|
|
145
|
+
| `watch_cycle_end` | watch.ts | `cycle_number?: number; status?: string` |
|
|
146
|
+
| `watch_stopped` | watch.ts | `reason?: string` |
|
|
147
|
+
|
|
148
|
+
### Worker Health
|
|
149
|
+
|
|
150
|
+
| Event Type | Source | Data Fields |
|
|
151
|
+
|-----------|--------|-------------|
|
|
152
|
+
| `worker_killed` | health.ts | `reason?: string; worker_id?: string; task_id?: string` |
|
|
153
|
+
|
|
154
|
+
### Discovered Issues
|
|
155
|
+
|
|
156
|
+
| Event Type | Source | Data Fields |
|
|
157
|
+
|-----------|--------|-------------|
|
|
158
|
+
| `discovered_issue` | issues.ts | `task_id?: string; title?: string; file?: string; description?: string; severity?: string` |
|
|
159
|
+
|
|
160
|
+
## Derived Metrics
|
|
161
|
+
|
|
162
|
+
These are computed from raw events, not emitted directly.
|
|
163
|
+
|
|
164
|
+
| Metric | Derivation |
|
|
165
|
+
|--------|-----------|
|
|
166
|
+
| Task duration | `task_done.timestamp - task_started.timestamp` |
|
|
167
|
+
| Convoy duration | `convoy_finished.timestamp - convoy_started.timestamp` |
|
|
168
|
+
| Retry rate | `COUNT(task_retried) / COUNT(task_started)` |
|
|
169
|
+
| Gate failure rate | `COUNT(built_in_gate_result WHERE !passed) / COUNT(built_in_gate_result)` |
|
|
170
|
+
| Review pass rate | `COUNT(review_verdict WHERE verdict='pass') / COUNT(review_verdict)` |
|
|
171
|
+
| Token usage | `SUM(review_verdict.tokens)` per convoy |
|
|
172
|
+
| Circuit breaker trips | `COUNT(circuit_breaker_tripped)` per agent |
|
|
173
|
+
|
|
174
|
+
## Runtime Validation
|
|
175
|
+
|
|
176
|
+
- `validateEventType(type)` — checks membership in `KNOWN_EVENT_TYPES` (a `Set<string>` exported from [`types.ts`](types.ts)). Unknown types trigger a `console.warn` but do not throw, preserving extensibility for custom event types.
|
|
177
|
+
- `validateEventData(type, data)` — validates the `data` payload shape for known event types. Defined in [`event-schemas.ts`](event-schemas.ts). Returns `{ valid: boolean; issues?: string[] }`. Invalid payloads trigger a `console.warn` but do not block emission.
|
|
178
|
+
|
|
179
|
+
Both validators are called at emit time in [`events.ts`](events.ts).
|
|
180
|
+
|
|
181
|
+
## Dashboard Build Pipeline
|
|
182
|
+
|
|
183
|
+
To build the dashboard with real convoy data:
|
|
184
|
+
|
|
185
|
+
```sh
|
|
186
|
+
# 1. Run ETL to extract data from SQLite → JSON
|
|
187
|
+
npm run dashboard:etl
|
|
188
|
+
|
|
189
|
+
# 2. Build the Astro dashboard (reads from public/data/*.json)
|
|
190
|
+
npx astro build --root src/dashboard
|
|
191
|
+
|
|
192
|
+
# 3. Serve locally (optional)
|
|
193
|
+
npx astro preview --root src/dashboard
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
In CI, add these steps after tests pass:
|
|
197
|
+
|
|
198
|
+
```yaml
|
|
199
|
+
- run: npm run dashboard:etl
|
|
200
|
+
- run: npx astro build --root src/dashboard
|
|
201
|
+
```
|
|
202
|
+
|
|
203
|
+
The ETL script gracefully handles missing databases — it produces empty JSON files so the dashboard renders an empty state instead of crashing.
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
export interface DashboardOverallStats {
|
|
2
|
+
total_convoys: number
|
|
3
|
+
running_convoys: number
|
|
4
|
+
successful_convoys: number
|
|
5
|
+
failed_convoys: number
|
|
6
|
+
avg_convoy_duration_sec: number | null
|
|
7
|
+
p95_convoy_duration_sec: number | null
|
|
8
|
+
total_tokens: number
|
|
9
|
+
total_cost_usd: number
|
|
10
|
+
top_agents: Array<{ agent: string; task_count: number; total_tokens: number }>
|
|
11
|
+
top_models: Array<{ model: string; task_count: number; total_tokens: number }>
|
|
12
|
+
retry_queue_count: number
|
|
13
|
+
disputed_tasks: number
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface DashboardConvoySummary {
|
|
17
|
+
id: string
|
|
18
|
+
name: string
|
|
19
|
+
status: string
|
|
20
|
+
branch: string | null
|
|
21
|
+
created_at: string
|
|
22
|
+
started_at: string | null
|
|
23
|
+
finished_at: string | null
|
|
24
|
+
duration_sec: number | null
|
|
25
|
+
total_tokens: number | null
|
|
26
|
+
total_cost_usd: number | null
|
|
27
|
+
tasks_total: number
|
|
28
|
+
tasks_done: number
|
|
29
|
+
tasks_running: number
|
|
30
|
+
tasks_waiting: number
|
|
31
|
+
tasks_failed: number
|
|
32
|
+
tasks_retrying: number
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface DashboardTaskSummary {
|
|
36
|
+
id: string
|
|
37
|
+
phase: number
|
|
38
|
+
agent: string
|
|
39
|
+
model: string | null
|
|
40
|
+
status: string
|
|
41
|
+
duration_sec: number | null
|
|
42
|
+
retries: number
|
|
43
|
+
files: string[]
|
|
44
|
+
total_tokens: number | null
|
|
45
|
+
cost_usd: number | null
|
|
46
|
+
review_level: string | null
|
|
47
|
+
review_verdict: string | null
|
|
48
|
+
drift_score: number | null
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface DashboardConvoyDetail {
|
|
52
|
+
convoy: {
|
|
53
|
+
id: string
|
|
54
|
+
name: string
|
|
55
|
+
status: string
|
|
56
|
+
created_at: string
|
|
57
|
+
finished_at: string | null
|
|
58
|
+
branch: string | null
|
|
59
|
+
total_tokens: number | null
|
|
60
|
+
total_cost_usd: number | null
|
|
61
|
+
}
|
|
62
|
+
taskSummary: {
|
|
63
|
+
total: number
|
|
64
|
+
done: number
|
|
65
|
+
running: number
|
|
66
|
+
failed: number
|
|
67
|
+
review_blocked: number
|
|
68
|
+
disputed: number
|
|
69
|
+
reviewed: number
|
|
70
|
+
panel_reviewed: number
|
|
71
|
+
tasks_with_drift: number
|
|
72
|
+
max_drift_score: number | null
|
|
73
|
+
drift_retried: number
|
|
74
|
+
}
|
|
75
|
+
quality: {
|
|
76
|
+
reviewed_tasks: number
|
|
77
|
+
review_blocked_tasks: number
|
|
78
|
+
disputed_tasks: number
|
|
79
|
+
panel_reviews: number
|
|
80
|
+
}
|
|
81
|
+
drift: {
|
|
82
|
+
tasks_with_drift: number
|
|
83
|
+
max_drift_score: number | null
|
|
84
|
+
drift_retried_tasks: number
|
|
85
|
+
}
|
|
86
|
+
dlq_count: number
|
|
87
|
+
dlq_entries: Array<{
|
|
88
|
+
id: string
|
|
89
|
+
task_id: string
|
|
90
|
+
agent: string
|
|
91
|
+
failure_type: string
|
|
92
|
+
attempts: number
|
|
93
|
+
resolved: number
|
|
94
|
+
}>
|
|
95
|
+
artifact_count: number
|
|
96
|
+
artifacts: Array<{
|
|
97
|
+
id: string
|
|
98
|
+
name: string
|
|
99
|
+
type: string
|
|
100
|
+
task_id: string
|
|
101
|
+
created_at: string
|
|
102
|
+
}>
|
|
103
|
+
has_more_events: boolean
|
|
104
|
+
events: Array<{
|
|
105
|
+
type: string
|
|
106
|
+
task_id: string | null
|
|
107
|
+
data: unknown
|
|
108
|
+
created_at: string
|
|
109
|
+
}>
|
|
110
|
+
tasks: Array<{
|
|
111
|
+
id: string
|
|
112
|
+
phase: number
|
|
113
|
+
agent: string
|
|
114
|
+
model: string | null
|
|
115
|
+
status: string
|
|
116
|
+
retries: number
|
|
117
|
+
started_at: string | null
|
|
118
|
+
finished_at: string | null
|
|
119
|
+
total_tokens: number | null
|
|
120
|
+
cost_usd: number | null
|
|
121
|
+
review_level: string | null
|
|
122
|
+
review_verdict: string | null
|
|
123
|
+
review_tokens: number | null
|
|
124
|
+
review_model: string | null
|
|
125
|
+
panel_attempts: number | null
|
|
126
|
+
dispute_id: string | null
|
|
127
|
+
drift_score: number | null
|
|
128
|
+
drift_retried: number | null
|
|
129
|
+
files: string[] | null
|
|
130
|
+
}>
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
export interface DashboardTimelineEvent {
|
|
134
|
+
id: number
|
|
135
|
+
timestamp: string
|
|
136
|
+
type: string
|
|
137
|
+
convoy_id: string | null
|
|
138
|
+
task_id: string | null
|
|
139
|
+
worker_id: string | null
|
|
140
|
+
summary: string
|
|
141
|
+
}
|
|
@@ -2,7 +2,8 @@ import { mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
|
|
2
2
|
import { tmpdir } from 'node:os'
|
|
3
3
|
import { join } from 'node:path'
|
|
4
4
|
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest'
|
|
5
|
-
import { createConvoyEngine, evaluateReviewLevel,
|
|
5
|
+
import { createConvoyEngine, evaluateReviewLevel, runConvoyGuard } from './engine.js'
|
|
6
|
+
import { recoverNdjson, createEventEmitter } from './events.js'
|
|
6
7
|
import type { ConvoyEngineOptions, DiffStats } from './engine.js'
|
|
7
8
|
import { createConvoyStore } from './store.js'
|
|
8
9
|
import type { AgentAdapter, Task, TaskSpec, ExecuteResult, ExecuteOptions } from '../types.js'
|
|
@@ -3658,3 +3659,100 @@ describe('circuit breaker', () => {
|
|
|
3658
3659
|
store.close()
|
|
3659
3660
|
})
|
|
3660
3661
|
})
|
|
3662
|
+
|
|
3663
|
+
describe('convoy lifecycle events', () => {
|
|
3664
|
+
it('emits convoy_finished event on successful run', async () => {
|
|
3665
|
+
const adapter = makeAdapter()
|
|
3666
|
+
const engine = makeEngine({
|
|
3667
|
+
spec: makeSpec(),
|
|
3668
|
+
specYaml: 'name: test',
|
|
3669
|
+
adapter,
|
|
3670
|
+
dbPath,
|
|
3671
|
+
_worktreeManager: makeWorktreeManager(),
|
|
3672
|
+
_mergeQueue: makeMergeQueue(),
|
|
3673
|
+
})
|
|
3674
|
+
const result = await engine.run()
|
|
3675
|
+
expect(result.status).toBe('done')
|
|
3676
|
+
|
|
3677
|
+
const store = createConvoyStore(dbPath)
|
|
3678
|
+
const events = store.getEvents(result.convoyId)
|
|
3679
|
+
store.close()
|
|
3680
|
+
|
|
3681
|
+
const finishedEvent = events.find(e => e.type === 'convoy_finished')
|
|
3682
|
+
expect(finishedEvent).toBeDefined()
|
|
3683
|
+
expect(finishedEvent!.convoy_id).toBe(result.convoyId)
|
|
3684
|
+
expect(JSON.parse(finishedEvent!.data as string).status).toBe('done')
|
|
3685
|
+
})
|
|
3686
|
+
|
|
3687
|
+
it('emits convoy_failed event when a task fails', async () => {
|
|
3688
|
+
const adapter = makeAdapter()
|
|
3689
|
+
adapter.execute.mockResolvedValue({
|
|
3690
|
+
success: false,
|
|
3691
|
+
output: 'error',
|
|
3692
|
+
exitCode: 1,
|
|
3693
|
+
})
|
|
3694
|
+
const engine = makeEngine({
|
|
3695
|
+
spec: makeSpec({}, [{ id: 'fail-task', max_retries: 0 }]),
|
|
3696
|
+
specYaml: 'name: test',
|
|
3697
|
+
adapter,
|
|
3698
|
+
dbPath,
|
|
3699
|
+
_worktreeManager: makeWorktreeManager(),
|
|
3700
|
+
_mergeQueue: makeMergeQueue(),
|
|
3701
|
+
})
|
|
3702
|
+
const result = await engine.run()
|
|
3703
|
+
expect(result.status).toBe('failed')
|
|
3704
|
+
|
|
3705
|
+
const store = createConvoyStore(dbPath)
|
|
3706
|
+
const events = store.getEvents(result.convoyId)
|
|
3707
|
+
store.close()
|
|
3708
|
+
|
|
3709
|
+
const failedEvent = events.find(e => e.type === 'convoy_failed')
|
|
3710
|
+
expect(failedEvent).toBeDefined()
|
|
3711
|
+
expect(failedEvent!.convoy_id).toBe(result.convoyId)
|
|
3712
|
+
expect(JSON.parse(failedEvent!.data as string).status).toBe('failed')
|
|
3713
|
+
})
|
|
3714
|
+
|
|
3715
|
+
it('emits convoy_failed with gate-failed status when gates fail', async () => {
|
|
3716
|
+
const adapter = makeAdapter()
|
|
3717
|
+
const engine = makeEngine({
|
|
3718
|
+
spec: makeSpec({ gates: ['false'] }),
|
|
3719
|
+
specYaml: 'name: test',
|
|
3720
|
+
adapter,
|
|
3721
|
+
dbPath,
|
|
3722
|
+
_worktreeManager: makeWorktreeManager(),
|
|
3723
|
+
_mergeQueue: makeMergeQueue(),
|
|
3724
|
+
})
|
|
3725
|
+
const result = await engine.run()
|
|
3726
|
+
expect(result.status).toBe('gate-failed')
|
|
3727
|
+
|
|
3728
|
+
const store = createConvoyStore(dbPath)
|
|
3729
|
+
const events = store.getEvents(result.convoyId)
|
|
3730
|
+
store.close()
|
|
3731
|
+
|
|
3732
|
+
const failedEvent = events.find(e => e.type === 'convoy_failed')
|
|
3733
|
+
expect(failedEvent).toBeDefined()
|
|
3734
|
+
expect(JSON.parse(failedEvent!.data as string).status).toBe('gate-failed')
|
|
3735
|
+
})
|
|
3736
|
+
})
|
|
3737
|
+
|
|
3738
|
+
describe('createEventEmitter callsite safety', () => {
|
|
3739
|
+
it('rejects a raw string argument', () => {
|
|
3740
|
+
const testStore = createConvoyStore(dbPath)
|
|
3741
|
+
expect(() => {
|
|
3742
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
3743
|
+
createEventEmitter(testStore, 'some-path' as any)
|
|
3744
|
+
}).toThrow('createEventEmitter options must be an object, not a string')
|
|
3745
|
+
testStore.close()
|
|
3746
|
+
})
|
|
3747
|
+
|
|
3748
|
+
it('accepts an options object with ndjsonPath', () => {
|
|
3749
|
+
const testStore = createConvoyStore(dbPath)
|
|
3750
|
+
const testNdjsonPath = join(tmpDir, 'callsite-test.ndjson')
|
|
3751
|
+
const emitter = createEventEmitter(testStore, { ndjsonPath: testNdjsonPath })
|
|
3752
|
+
expect(emitter).toBeDefined()
|
|
3753
|
+
expect(typeof emitter.emit).toBe('function')
|
|
3754
|
+
expect(typeof emitter.close).toBe('function')
|
|
3755
|
+
emitter.close()
|
|
3756
|
+
testStore.close()
|
|
3757
|
+
})
|
|
3758
|
+
})
|
package/src/cli/convoy/engine.ts
CHANGED
|
@@ -18,7 +18,7 @@ import { promisify } from 'node:util'
|
|
|
18
18
|
import type { Task, TaskSpec, AgentAdapter, ExecuteResult, ReviewHeuristics } from '../types.js'
|
|
19
19
|
import { createConvoyStore, ConvoyArtifactLimitError, type ConvoyStore } from './store.js'
|
|
20
20
|
import { acquireEngineLock } from './lock.js'
|
|
21
|
-
import { createEventEmitter, type ConvoyEventEmitter } from './events.js'
|
|
21
|
+
import { createEventEmitter, ndjsonPathForConvoy, recoverNdjson, type ConvoyEventEmitter } from './events.js'
|
|
22
22
|
import { createWorktreeManager, type WorktreeManager } from './worktree.js'
|
|
23
23
|
import { createMergeQueue, MergeConflictError, type MergeQueue } from './merge.js'
|
|
24
24
|
import { createHealthMonitor, detectDrift } from './health.js'
|
|
@@ -222,84 +222,6 @@ export async function ensureBranch(branchName: string, basePath: string): Promis
|
|
|
222
222
|
}
|
|
223
223
|
}
|
|
224
224
|
|
|
225
|
-
// ── Internal helpers ──────────────────────────────────────────────────────────
|
|
226
|
-
|
|
227
|
-
/**
|
|
228
|
-
* Truncate any trailing partial line in the NDJSON file, then replay any SQLite
|
|
229
|
-
* events for the given convoy that are missing from the file.
|
|
230
|
-
* Exported for unit testing.
|
|
231
|
-
*/
|
|
232
|
-
function safeJsonParse(raw: string): Record<string, unknown> {
|
|
233
|
-
try {
|
|
234
|
-
return JSON.parse(raw) as Record<string, unknown>
|
|
235
|
-
} catch {
|
|
236
|
-
return {}
|
|
237
|
-
}
|
|
238
|
-
}
|
|
239
|
-
|
|
240
|
-
export function recoverNdjson(store: ConvoyStore, convoyId: string, ndjsonPath: string): void {
|
|
241
|
-
// 1. Read the NDJSON file (if it exists)
|
|
242
|
-
let fileContent: string
|
|
243
|
-
try {
|
|
244
|
-
fileContent = readFileSync(ndjsonPath, 'utf8')
|
|
245
|
-
} catch {
|
|
246
|
-
fileContent = ''
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
// 2. Truncate any partial trailing line (no \n terminator)
|
|
250
|
-
if (fileContent.length > 0 && !fileContent.endsWith('\n')) {
|
|
251
|
-
const lastNewline = fileContent.lastIndexOf('\n')
|
|
252
|
-
if (lastNewline === -1) {
|
|
253
|
-
writeFileSync(ndjsonPath, '')
|
|
254
|
-
fileContent = ''
|
|
255
|
-
} else {
|
|
256
|
-
writeFileSync(ndjsonPath, fileContent.slice(0, lastNewline + 1))
|
|
257
|
-
fileContent = fileContent.slice(0, lastNewline + 1)
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
// 3. Count valid NDJSON event IDs for this convoy
|
|
262
|
-
const ndjsonIds = new Set<number>()
|
|
263
|
-
for (const line of fileContent.split('\n')) {
|
|
264
|
-
if (!line.trim()) continue
|
|
265
|
-
try {
|
|
266
|
-
const parsed = JSON.parse(line) as Record<string, unknown>
|
|
267
|
-
if (parsed.convoy_id === convoyId && parsed._event_id != null) {
|
|
268
|
-
ndjsonIds.add(parsed._event_id as number)
|
|
269
|
-
}
|
|
270
|
-
} catch {
|
|
271
|
-
// Skip unparseable lines
|
|
272
|
-
}
|
|
273
|
-
}
|
|
274
|
-
|
|
275
|
-
// 4. Get all SQLite events for this convoy
|
|
276
|
-
const sqliteEvents = store.getEvents(convoyId)
|
|
277
|
-
|
|
278
|
-
// 5. Replay missing events (those in SQLite but not in NDJSON)
|
|
279
|
-
const missing = sqliteEvents.filter(e => e.id != null && !ndjsonIds.has(e.id!))
|
|
280
|
-
if (missing.length > 0) {
|
|
281
|
-
const fd = openSync(ndjsonPath, 'a')
|
|
282
|
-
try {
|
|
283
|
-
for (const event of missing) {
|
|
284
|
-
const parsedData = event.data ? safeJsonParse(event.data) : {}
|
|
285
|
-
const record = {
|
|
286
|
-
...parsedData,
|
|
287
|
-
_event_id: event.id,
|
|
288
|
-
timestamp: event.created_at,
|
|
289
|
-
type: event.type,
|
|
290
|
-
convoy_id: event.convoy_id,
|
|
291
|
-
task_id: event.task_id,
|
|
292
|
-
worker_id: event.worker_id,
|
|
293
|
-
}
|
|
294
|
-
appendFileSync(fd, JSON.stringify(record) + '\n')
|
|
295
|
-
}
|
|
296
|
-
fsyncSync(fd)
|
|
297
|
-
} finally {
|
|
298
|
-
closeSync(fd)
|
|
299
|
-
}
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
|
|
303
225
|
// ── Convoy guard ──────────────────────────────────────────────────────────────
|
|
304
226
|
|
|
305
227
|
export interface ConvoyGuardResult {
|
|
@@ -336,12 +258,10 @@ export function runConvoyGuard(
|
|
|
336
258
|
try {
|
|
337
259
|
const content = readFileSync(ndjsonPath, 'utf8')
|
|
338
260
|
const lines = content.split('\n').filter(l => l.trim())
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
})
|
|
342
|
-
if (convoyLines.length < completedTasks.length) {
|
|
261
|
+
// Per-convoy file — all records belong to this convoy, no need to filter by convoy_id
|
|
262
|
+
if (lines.length < completedTasks.length) {
|
|
343
263
|
warnings.push(
|
|
344
|
-
`NDJSON record count (${
|
|
264
|
+
`NDJSON record count (${lines.length}) < completed tasks (${completedTasks.length})`,
|
|
345
265
|
)
|
|
346
266
|
}
|
|
347
267
|
} catch {
|
|
@@ -365,9 +285,16 @@ export function runConvoyGuard(
|
|
|
365
285
|
}
|
|
366
286
|
|
|
367
287
|
// Check 4: Gate results recorded for all gates that ran
|
|
368
|
-
const gateEvents = events.filter(e =>
|
|
369
|
-
e.type === 'built_in_gate_result'
|
|
370
|
-
|
|
288
|
+
const gateEvents = events.filter(e => {
|
|
289
|
+
if (e.type === 'built_in_gate_result') return true
|
|
290
|
+
if (e.data == null) return false
|
|
291
|
+
try {
|
|
292
|
+
const parsed = JSON.parse(e.data) as Record<string, unknown>
|
|
293
|
+
return 'gate' in parsed
|
|
294
|
+
} catch {
|
|
295
|
+
return false
|
|
296
|
+
}
|
|
297
|
+
})
|
|
371
298
|
const tasksWithGates = tasks.filter(t => t.gates)
|
|
372
299
|
if (tasksWithGates.length > 0 && gateEvents.length === 0) {
|
|
373
300
|
warnings.push('Tasks have gates configured but no gate result events found')
|
|
@@ -1065,6 +992,7 @@ async function runConvoy(
|
|
|
1065
992
|
skipTask(t.id, `on_exhausted: stop — task "${taskRecord.id}" exhausted retries`)
|
|
1066
993
|
}
|
|
1067
994
|
store.updateConvoyStatus(convoyId, 'failed')
|
|
995
|
+
events.emit('convoy_failed', { status: 'failed', reason: `on_exhausted: stop — task "${taskRecord.id}" exhausted retries` }, { convoy_id: convoyId })
|
|
1068
996
|
} else if (exhausted === 'dlq' || exhausted === 'skip') {
|
|
1069
997
|
// Default behavior: cascade failure to dependents only
|
|
1070
998
|
cascadeFailure(taskRecord.id)
|
|
@@ -2603,6 +2531,12 @@ async function runConvoy(
|
|
|
2603
2531
|
total_tokens: convoyTotalTokens,
|
|
2604
2532
|
})
|
|
2605
2533
|
|
|
2534
|
+
if (finalStatus === 'done') {
|
|
2535
|
+
events.emit('convoy_finished', { status: 'done' }, { convoy_id: convoyId })
|
|
2536
|
+
} else {
|
|
2537
|
+
events.emit('convoy_failed', { status: finalStatus, reason: finalStatus === 'gate-failed' ? 'Gate check failed' : 'One or more tasks failed' }, { convoy_id: convoyId })
|
|
2538
|
+
}
|
|
2539
|
+
|
|
2606
2540
|
// Run convoy guard checks
|
|
2607
2541
|
const guardResult = runConvoyGuard(store, convoyId, wtManager, ndjsonPath, spec.guard)
|
|
2608
2542
|
if (guardResult.warnings.length > 0) {
|
|
@@ -2690,9 +2624,8 @@ export function createConvoyEngine(options: ConvoyEngineOptions): ConvoyEngine {
|
|
|
2690
2624
|
|
|
2691
2625
|
const store = createConvoyStore(dbPath)
|
|
2692
2626
|
const ndjsonPath = options.logsDir
|
|
2693
|
-
? join(options.logsDir, '
|
|
2694
|
-
:
|
|
2695
|
-
mkdirSync(dirname(ndjsonPath), { recursive: true })
|
|
2627
|
+
? join(options.logsDir, 'convoys', `${convoyId}.ndjson`)
|
|
2628
|
+
: ndjsonPathForConvoy(convoyId, basePath)
|
|
2696
2629
|
const events = createEventEmitter(store, { ndjsonPath })
|
|
2697
2630
|
const wtManager = options._worktreeManager ?? createWorktreeManager(basePath)
|
|
2698
2631
|
const mergeQueue = options._mergeQueue ?? createMergeQueue(basePath)
|
|
@@ -2807,9 +2740,8 @@ export function createConvoyEngine(options: ConvoyEngineOptions): ConvoyEngine {
|
|
|
2807
2740
|
|
|
2808
2741
|
const store = createConvoyStore(dbPath)
|
|
2809
2742
|
const ndjsonPath = options.logsDir
|
|
2810
|
-
? join(options.logsDir, '
|
|
2811
|
-
:
|
|
2812
|
-
mkdirSync(dirname(ndjsonPath), { recursive: true })
|
|
2743
|
+
? join(options.logsDir, 'convoys', `${convoyId}.ndjson`)
|
|
2744
|
+
: ndjsonPathForConvoy(convoyId, basePath)
|
|
2813
2745
|
const events = createEventEmitter(store, { ndjsonPath })
|
|
2814
2746
|
const wtManager = options._worktreeManager ?? createWorktreeManager(basePath)
|
|
2815
2747
|
const mergeQueue = options._mergeQueue ?? createMergeQueue(basePath)
|
|
@@ -2876,9 +2808,8 @@ export function createConvoyEngine(options: ConvoyEngineOptions): ConvoyEngine {
|
|
|
2876
2808
|
mkdirSync(dirname(dbPath), { recursive: true })
|
|
2877
2809
|
const store = createConvoyStore(dbPath)
|
|
2878
2810
|
const ndjsonPath = options.logsDir
|
|
2879
|
-
? join(options.logsDir, '
|
|
2880
|
-
:
|
|
2881
|
-
mkdirSync(dirname(ndjsonPath), { recursive: true })
|
|
2811
|
+
? join(options.logsDir, 'convoys', `${convoyId}.ndjson`)
|
|
2812
|
+
: ndjsonPathForConvoy(convoyId, basePath)
|
|
2882
2813
|
const events = createEventEmitter(store, { ndjsonPath })
|
|
2883
2814
|
try {
|
|
2884
2815
|
const allTasks = store.getTasksByConvoy(convoyId)
|