opencode-swarm-plugin 0.39.1 → 0.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/.hive/issues.jsonl +16 -0
  2. package/CHANGELOG.md +52 -0
  3. package/bin/swarm.test.ts +406 -0
  4. package/bin/swarm.ts +303 -0
  5. package/dist/compaction-hook.d.ts +8 -1
  6. package/dist/compaction-hook.d.ts.map +1 -1
  7. package/dist/compaction-observability.d.ts +173 -0
  8. package/dist/compaction-observability.d.ts.map +1 -0
  9. package/dist/eval-capture.d.ts +93 -0
  10. package/dist/eval-capture.d.ts.map +1 -1
  11. package/dist/hive.d.ts.map +1 -1
  12. package/dist/index.d.ts +36 -1
  13. package/dist/index.d.ts.map +1 -1
  14. package/dist/index.js +15670 -580
  15. package/dist/plugin.js +15623 -557
  16. package/dist/schemas/task.d.ts +3 -3
  17. package/evals/README.md +113 -0
  18. package/evals/scorers/coordinator-discipline.evalite-test.ts +163 -0
  19. package/evals/scorers/coordinator-discipline.ts +335 -2
  20. package/evals/scorers/index.test.ts +146 -0
  21. package/evals/scorers/index.ts +104 -0
  22. package/evals/swarm-decomposition.eval.ts +9 -2
  23. package/examples/commands/swarm.md +291 -21
  24. package/package.json +1 -1
  25. package/src/compaction-hook.ts +258 -110
  26. package/src/compaction-observability.integration.test.ts +139 -0
  27. package/src/compaction-observability.test.ts +187 -0
  28. package/src/compaction-observability.ts +324 -0
  29. package/src/eval-capture.test.ts +204 -1
  30. package/src/eval-capture.ts +194 -2
  31. package/src/eval-runner.test.ts +96 -0
  32. package/src/eval-runner.ts +356 -0
  33. package/src/hive.ts +34 -0
  34. package/src/index.ts +54 -1
  35. package/src/memory.test.ts +110 -0
  36. package/src/memory.ts +34 -0
  37. package/dist/beads.d.ts +0 -386
  38. package/dist/beads.d.ts.map +0 -1
  39. package/dist/schemas/bead-events.d.ts +0 -698
  40. package/dist/schemas/bead-events.d.ts.map +0 -1
  41. package/dist/schemas/bead.d.ts +0 -255
  42. package/dist/schemas/bead.d.ts.map +0 -1
@@ -16,8 +16,8 @@ import { z } from "zod";
16
16
  */
17
17
  export declare const EffortLevelSchema: z.ZodEnum<{
18
18
  small: "small";
19
- trivial: "trivial";
20
19
  medium: "medium";
20
+ trivial: "trivial";
21
21
  large: "large";
22
22
  }>;
23
23
  export type EffortLevel = z.infer<typeof EffortLevelSchema>;
@@ -39,8 +39,8 @@ export declare const DecomposedSubtaskSchema: z.ZodObject<{
39
39
  files: z.ZodArray<z.ZodString>;
40
40
  estimated_effort: z.ZodEnum<{
41
41
  small: "small";
42
- trivial: "trivial";
43
42
  medium: "medium";
43
+ trivial: "trivial";
44
44
  large: "large";
45
45
  }>;
46
46
  risks: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
@@ -74,8 +74,8 @@ export declare const TaskDecompositionSchema: z.ZodObject<{
74
74
  files: z.ZodArray<z.ZodString>;
75
75
  estimated_effort: z.ZodEnum<{
76
76
  small: "small";
77
- trivial: "trivial";
78
77
  medium: "medium";
78
+ trivial: "trivial";
79
79
  large: "large";
80
80
  }>;
81
81
  risks: z.ZodDefault<z.ZodOptional<z.ZodArray<z.ZodString>>>;
package/evals/README.md CHANGED
@@ -167,6 +167,119 @@ coordinator-behavior
167
167
  → overallDiscipline: 0.89 ✅ PASS (bootstrap phase, collecting data)
168
168
  ```
169
169
 
170
+ #### Coordinator Session Capture (Deep Dive)
171
+
172
+ **How it works:** Session capture is fully automatic when coordinator tools are used. No manual instrumentation needed.
173
+
174
+ **Capture flow:**
175
+
176
+ ```
177
+ ┌─────────────────────────────────────────────────────────────┐
178
+ │ SESSION CAPTURE FLOW │
179
+ │ │
180
+ │ 1. Coordinator tool call detected │
181
+ │ ├─ swarm_decompose, hive_create_epic, etc. │
182
+ │ └─ Tool name + args inspected in real-time │
183
+ │ │
184
+ │ 2. Violation detection (planning-guardrails.ts) │
185
+ │ ├─ detectCoordinatorViolation() checks patterns │
186
+ │ ├─ Edit/Write tools → coordinator_edited_file │
187
+ │ ├─ bash with test patterns → coordinator_ran_tests │
188
+ │ └─ swarmmail_reserve → coordinator_reserved_files │
189
+ │ │
190
+ │ 3. Event emission (eval-capture.ts) │
191
+ │ ├─ captureCoordinatorEvent() validates via Zod │
192
+ │ ├─ Appends JSONL line to session file │
193
+ │ └─ ~/.config/swarm-tools/sessions/{session_id}.jsonl │
194
+ │ │
195
+ │ 4. Eval consumption (coordinator-session.eval.ts) │
196
+ │ ├─ loadCapturedSessions() reads all *.jsonl files │
197
+ │ ├─ Parses events, reconstructs sessions │
198
+ │ └─ Scorers analyze event sequences │
199
+ │ │
200
+ └─────────────────────────────────────────────────────────────┘
201
+ ```
202
+
203
+ **Event types:**
204
+
205
+ | Event Type | Subtypes | When Captured |
206
+ | -------------- | --------------------------------------------------------------------- | ------------------------------------ |
207
+ | `DECISION` | strategy_selected, worker_spawned, review_completed, decomposition_complete | Coordinator makes decision |
208
+ | `VIOLATION` | coordinator_edited_file, coordinator_ran_tests, coordinator_reserved_files, no_worker_spawned | Protocol violation detected |
209
+ | `OUTCOME` | subtask_success, subtask_retry, subtask_failed, epic_complete | Worker completes or epic finishes |
210
+ | `COMPACTION` | detection_complete, prompt_generated, context_injected, resumption_started, tool_call_tracked | Compaction lifecycle events |
211
+
212
+ **Violation detection patterns** (from `planning-guardrails.ts`):
213
+
214
+ ```typescript
215
+ // File modification detection
216
+ VIOLATION_PATTERNS.FILE_MODIFICATION_TOOLS = ["edit", "write"];
217
+
218
+ // Test execution detection (regex patterns in bash commands)
219
+ VIOLATION_PATTERNS.TEST_EXECUTION_PATTERNS = [
220
+ /\bbun\s+test\b/i,
221
+ /\bnpm\s+(run\s+)?test/i,
222
+ /\bjest\b/i,
223
+ /\bvitest\b/i,
224
+ // ... and 6 more patterns
225
+ ];
226
+
227
+ // File reservation detection
228
+ VIOLATION_PATTERNS.RESERVATION_TOOLS = ["swarmmail_reserve", "agentmail_reserve"];
229
+ ```
230
+
231
+ **Example session file** (`~/.config/swarm-tools/sessions/session-abc123.jsonl`):
232
+
233
+ ```jsonl
234
+ {"session_id":"session-abc123","epic_id":"mjkw81rkq4c","timestamp":"2025-01-01T12:00:00Z","event_type":"DECISION","decision_type":"strategy_selected","payload":{"strategy":"feature-based"}}
235
+ {"session_id":"session-abc123","epic_id":"mjkw81rkq4c","timestamp":"2025-01-01T12:01:00Z","event_type":"DECISION","decision_type":"decomposition_complete","payload":{"subtask_count":3}}
236
+ {"session_id":"session-abc123","epic_id":"mjkw81rkq4c","timestamp":"2025-01-01T12:02:00Z","event_type":"DECISION","decision_type":"worker_spawned","payload":{"worker_id":"SwiftFire","bead_id":"mjkw81rkq4c.1"}}
237
+ {"session_id":"session-abc123","epic_id":"mjkw81rkq4c","timestamp":"2025-01-01T12:05:00Z","event_type":"VIOLATION","violation_type":"coordinator_edited_file","payload":{"tool":"edit","file":"src/auth.ts"}}
238
+ {"session_id":"session-abc123","epic_id":"mjkw81rkq4c","timestamp":"2025-01-01T12:10:00Z","event_type":"OUTCOME","outcome_type":"subtask_success","payload":{"bead_id":"mjkw81rkq4c.1","duration_ms":480000}}
239
+ ```
240
+
241
+ **Viewing sessions:**
242
+
243
+ ```bash
244
+ # List all captured sessions (coming soon)
245
+ swarm log sessions
246
+
247
+ # View specific session events
248
+ cat ~/.config/swarm-tools/sessions/session-abc123.jsonl | jq .
249
+
250
+ # Filter to violations only
251
+ cat ~/.config/swarm-tools/sessions/*.jsonl | jq 'select(.event_type == "VIOLATION")'
252
+
253
+ # Count violations by type
254
+ cat ~/.config/swarm-tools/sessions/*.jsonl | jq -r 'select(.event_type == "VIOLATION") | .violation_type' | sort | uniq -c
255
+ ```
256
+
257
+ **Why JSONL format?**
258
+
259
+ - **Append-only**: No file locking, safe for concurrent writes
260
+ - **Streamable**: Process events one-by-one without loading full file
261
+ - **Line-oriented**: Easy to `grep`, `jq`, `tail -f` for live monitoring
262
+ - **Fault-tolerant**: Corrupted line doesn't break entire file
263
+
264
+ **Integration points:**
265
+
266
+ | Where | What Gets Captured | File |
267
+ | -------------------------- | ----------------------------------------- | ----------------------- |
268
+ | `swarm_decompose` | DECISION: strategy_selected, decomposition_complete | sessions/*.jsonl |
269
+ | `swarm_spawn_subtask` | DECISION: worker_spawned | sessions/*.jsonl |
270
+ | `swarm_review` | DECISION: review_completed | sessions/*.jsonl |
271
+ | `swarm_complete` | OUTCOME: subtask_success/failed | sessions/*.jsonl |
272
+ | Tool call inspection | VIOLATION: (real-time pattern matching) | sessions/*.jsonl |
273
+ | Compaction hook | COMPACTION: (all lifecycle stages) | sessions/*.jsonl |
274
+
275
+ **Source files:**
276
+
277
+ - **Schema**: `src/eval-capture.ts` - CoordinatorEventSchema (Zod discriminated union)
278
+ - **Violation detection**: `src/planning-guardrails.ts` - detectCoordinatorViolation()
279
+ - **Capture**: `src/eval-capture.ts` - captureCoordinatorEvent()
280
+ - **Scorers**: `evals/scorers/coordinator-discipline.ts` - violationCount, spawnEfficiency, etc.
281
+ - **Eval**: `evals/coordinator-session.eval.ts` - Real sessions + fixtures
282
+
170
283
  ### Compaction Prompt (`compaction-prompt.eval.ts`)
171
284
 
172
285
  **What it measures:** Quality of continuation prompts after context compaction
@@ -5,6 +5,7 @@ import { describe, expect, it } from "bun:test";
5
5
  import type { CoordinatorSession } from "../../src/eval-capture.js";
6
6
  import {
7
7
  overallDiscipline,
8
+ reviewEfficiency,
8
9
  reviewThoroughness,
9
10
  spawnEfficiency,
10
11
  timeToFirstSpawn,
@@ -535,3 +536,165 @@ describe("overallDiscipline", () => {
535
536
  expect(result.message).toContain("Speed:");
536
537
  });
537
538
  });
539
+
540
+ describe("reviewEfficiency", () => {
541
+ it("scores 1.0 for ideal 1:1 ratio (one review per spawn)", async () => {
542
+ const session: CoordinatorSession = {
543
+ session_id: "test-session",
544
+ epic_id: "test-epic",
545
+ start_time: "2025-01-01T00:00:00Z",
546
+ events: [
547
+ {
548
+ session_id: "test-session",
549
+ epic_id: "test-epic",
550
+ timestamp: "2025-01-01T00:00:10Z",
551
+ event_type: "DECISION",
552
+ decision_type: "worker_spawned",
553
+ payload: { bead_id: "bd-1" },
554
+ },
555
+ {
556
+ session_id: "test-session",
557
+ epic_id: "test-epic",
558
+ timestamp: "2025-01-01T00:00:20Z",
559
+ event_type: "DECISION",
560
+ decision_type: "worker_spawned",
561
+ payload: { bead_id: "bd-2" },
562
+ },
563
+ {
564
+ session_id: "test-session",
565
+ epic_id: "test-epic",
566
+ timestamp: "2025-01-01T00:10:00Z",
567
+ event_type: "DECISION",
568
+ decision_type: "review_completed",
569
+ payload: { bead_id: "bd-1" },
570
+ },
571
+ {
572
+ session_id: "test-session",
573
+ epic_id: "test-epic",
574
+ timestamp: "2025-01-01T00:10:10Z",
575
+ event_type: "DECISION",
576
+ decision_type: "review_completed",
577
+ payload: { bead_id: "bd-2" },
578
+ },
579
+ ],
580
+ };
581
+
582
+ const result = await reviewEfficiency({
583
+ output: JSON.stringify(session),
584
+ expected: {},
585
+ input: undefined,
586
+ });
587
+
588
+ expect(result.score).toBe(1.0);
589
+ expect(result.message).toContain("2 reviews / 2 spawns");
590
+ });
591
+
592
+ it("penalizes over-reviewing (>2:1 ratio)", async () => {
593
+ // 6 reviews for 2 spawns = 3:1 ratio (over-reviewing)
594
+ const session: CoordinatorSession = {
595
+ session_id: "test-session",
596
+ epic_id: "test-epic",
597
+ start_time: "2025-01-01T00:00:00Z",
598
+ events: [
599
+ {
600
+ session_id: "test-session",
601
+ epic_id: "test-epic",
602
+ timestamp: "2025-01-01T00:00:10Z",
603
+ event_type: "DECISION",
604
+ decision_type: "worker_spawned",
605
+ payload: { bead_id: "bd-1" },
606
+ },
607
+ {
608
+ session_id: "test-session",
609
+ epic_id: "test-epic",
610
+ timestamp: "2025-01-01T00:00:20Z",
611
+ event_type: "DECISION",
612
+ decision_type: "worker_spawned",
613
+ payload: { bead_id: "bd-2" },
614
+ },
615
+ ...Array.from({ length: 6 }, (_, i) => ({
616
+ session_id: "test-session",
617
+ epic_id: "test-epic",
618
+ timestamp: `2025-01-01T00:10:${String(i * 10).padStart(2, "0")}Z`,
619
+ event_type: "DECISION" as const,
620
+ decision_type: "review_completed" as const,
621
+ payload: { bead_id: `bd-${(i % 2) + 1}` },
622
+ })),
623
+ ],
624
+ };
625
+
626
+ const result = await reviewEfficiency({
627
+ output: JSON.stringify(session),
628
+ expected: {},
629
+ input: undefined,
630
+ });
631
+
632
+ // 3:1 ratio should be penalized (score < 0.5)
633
+ expect(result.score).toBeLessThan(0.5);
634
+ expect(result.message).toContain("6 reviews / 2 spawns");
635
+ });
636
+
637
+ it("handles no spawns gracefully", async () => {
638
+ const session: CoordinatorSession = {
639
+ session_id: "test-session",
640
+ epic_id: "test-epic",
641
+ start_time: "2025-01-01T00:00:00Z",
642
+ events: [
643
+ {
644
+ session_id: "test-session",
645
+ epic_id: "test-epic",
646
+ timestamp: "2025-01-01T00:00:00Z",
647
+ event_type: "DECISION",
648
+ decision_type: "strategy_selected",
649
+ payload: { strategy: "file-based" },
650
+ },
651
+ ],
652
+ };
653
+
654
+ const result = await reviewEfficiency({
655
+ output: JSON.stringify(session),
656
+ expected: {},
657
+ input: undefined,
658
+ });
659
+
660
+ expect(result.score).toBe(1.0);
661
+ expect(result.message).toContain("No workers spawned");
662
+ });
663
+
664
+ it("handles no reviews gracefully (0:N ratio)", async () => {
665
+ const session: CoordinatorSession = {
666
+ session_id: "test-session",
667
+ epic_id: "test-epic",
668
+ start_time: "2025-01-01T00:00:00Z",
669
+ events: [
670
+ {
671
+ session_id: "test-session",
672
+ epic_id: "test-epic",
673
+ timestamp: "2025-01-01T00:00:10Z",
674
+ event_type: "DECISION",
675
+ decision_type: "worker_spawned",
676
+ payload: { bead_id: "bd-1" },
677
+ },
678
+ {
679
+ session_id: "test-session",
680
+ epic_id: "test-epic",
681
+ timestamp: "2025-01-01T00:00:20Z",
682
+ event_type: "DECISION",
683
+ decision_type: "worker_spawned",
684
+ payload: { bead_id: "bd-2" },
685
+ },
686
+ ],
687
+ };
688
+
689
+ const result = await reviewEfficiency({
690
+ output: JSON.stringify(session),
691
+ expected: {},
692
+ input: undefined,
693
+ });
694
+
695
+ // No reviews is bad (should use reviewThoroughness for this)
696
+ // But this scorer focuses on over-reviewing, so no reviews = 1.0 (not over-reviewing)
697
+ expect(result.score).toBe(1.0);
698
+ expect(result.message).toContain("0 reviews / 2 spawns");
699
+ });
700
+ });
@@ -70,6 +70,9 @@ export const violationCount = createScorer({
70
70
  * Coordinators should delegate work, not do it themselves.
71
71
  *
72
72
  * Score: workers_spawned / subtasks_planned
73
+ *
74
+ * If no decomposition_complete event exists, falls back to counting spawns
75
+ * and returns 1.0 if any workers were spawned (better than nothing).
73
76
  */
74
77
  export const spawnEfficiency = createScorer({
75
78
  name: "Spawn Efficiency",
@@ -85,7 +88,20 @@ export const spawnEfficiency = createScorer({
85
88
  e.decision_type === "decomposition_complete"
86
89
  );
87
90
 
91
+ // Count worker_spawned events
92
+ const spawned = session.events.filter(
93
+ (e) =>
94
+ e.event_type === "DECISION" && e.decision_type === "worker_spawned"
95
+ ).length;
96
+
88
97
  if (!decomp) {
98
+ // Fallback: if workers were spawned but no decomp event, assume they're doing work
99
+ if (spawned > 0) {
100
+ return {
101
+ score: 1.0,
102
+ message: `${spawned} workers spawned (no decomposition event)`,
103
+ };
104
+ }
89
105
  return {
90
106
  score: 0,
91
107
  message: "No decomposition event found",
@@ -101,17 +117,81 @@ export const spawnEfficiency = createScorer({
101
117
  };
102
118
  }
103
119
 
120
+ const score = spawned / subtaskCount;
121
+
122
+ return {
123
+ score,
124
+ message: `${spawned}/${subtaskCount} workers spawned (${(score * 100).toFixed(0)}%)`,
125
+ };
126
+ } catch (error) {
127
+ return {
128
+ score: 0,
129
+ message: `Failed to parse CoordinatorSession: ${error}`,
130
+ };
131
+ }
132
+ },
133
+ });
134
+
135
+ /**
136
+ * Review Efficiency Scorer
137
+ *
138
+ * Measures review-to-spawn ratio to detect over-reviewing.
139
+ * Ideal ratio is 1:1 (one review per spawned worker).
140
+ * Penalizes >2:1 ratio (over-reviewing wastes context).
141
+ *
142
+ * Scoring:
143
+ * - 0:N or 1:1 ratio = 1.0 (perfect)
144
+ * - 2:1 ratio = 0.5 (threshold)
145
+ * - >2:1 ratio = linear penalty toward 0.0
146
+ *
147
+ * Score: normalized to 0-1 (lower ratio is better)
148
+ */
149
+ export const reviewEfficiency = createScorer({
150
+ name: "Review Efficiency",
151
+ description: "Review-to-spawn ratio (penalize over-reviewing >2:1)",
152
+ scorer: ({ output }) => {
153
+ try {
154
+ const session = JSON.parse(String(output)) as CoordinatorSession;
155
+
104
156
  // Count worker_spawned events
105
157
  const spawned = session.events.filter(
106
158
  (e) =>
107
159
  e.event_type === "DECISION" && e.decision_type === "worker_spawned"
108
160
  ).length;
109
161
 
110
- const score = spawned / subtaskCount;
162
+ if (spawned === 0) {
163
+ return {
164
+ score: 1.0,
165
+ message: "No workers spawned",
166
+ };
167
+ }
168
+
169
+ // Count review_completed events
170
+ const reviewed = session.events.filter(
171
+ (e) =>
172
+ e.event_type === "DECISION" && e.decision_type === "review_completed"
173
+ ).length;
174
+
175
+ const ratio = reviewed / spawned;
176
+
177
+ // Scoring:
178
+ // - ratio <= 1.0: perfect (1.0)
179
+ // - ratio <= 2.0: linear decay from 1.0 to 0.5
180
+ // - ratio > 2.0: linear penalty from 0.5 toward 0.0
181
+ let score: number;
182
+ if (ratio <= 1.0) {
183
+ score = 1.0;
184
+ } else if (ratio <= 2.0) {
185
+ // Linear decay: 1.0 at ratio=1.0, 0.5 at ratio=2.0
186
+ score = 1.0 - (ratio - 1.0) * 0.5;
187
+ } else {
188
+ // Penalty for extreme over-reviewing: 0.5 at ratio=2.0, 0.0 at ratio=4.0
189
+ score = Math.max(0, 0.5 - (ratio - 2.0) * 0.25);
190
+ }
111
191
 
112
192
  return {
113
193
  score,
114
- message: `${spawned}/${subtaskCount} workers spawned (${(score * 100).toFixed(0)}%)`,
194
+ message: `${reviewed} reviews / ${spawned} spawns (${ratio.toFixed(1)}:1 ratio)`,
115
195
  };
116
196
  } catch (error) {
117
197
  return {
@@ -254,6 +334,259 @@ export const timeToFirstSpawn = createScorer({
254
334
  },
255
335
  });
256
336
 
337
+ /**
338
+ * Researcher Spawn Rate Scorer
339
+ *
340
+ * Measures whether coordinator spawns researchers for unfamiliar technology.
341
+ * Coordinators should delegate research instead of calling pdf-brain/context7 directly.
342
+ *
343
+ * Score: 1.0 if researcher_spawned events exist, 0.0 otherwise
344
+ */
345
+ export const researcherSpawnRate = createScorer({
346
+ name: "Researcher Spawn Rate",
347
+ description: "Coordinator spawned researchers for unfamiliar tech",
348
+ scorer: ({ output }) => {
349
+ try {
350
+ const session = JSON.parse(String(output)) as CoordinatorSession;
351
+
352
+ // Count researcher_spawned events
353
+ const researchers = session.events.filter(
354
+ (e) =>
355
+ e.event_type === "DECISION" && e.decision_type === "researcher_spawned"
356
+ );
357
+
358
+ const count = researchers.length;
359
+
360
+ if (count === 0) {
361
+ return {
362
+ score: 0.0,
363
+ message: "No researchers spawned (may indicate coordinator queried docs directly)",
364
+ };
365
+ }
366
+
367
+ return {
368
+ score: 1.0,
369
+ message: `${count} researcher(s) spawned`,
370
+ };
371
+ } catch (error) {
372
+ return {
373
+ score: 0,
374
+ message: `Failed to parse CoordinatorSession: ${error}`,
375
+ };
376
+ }
377
+ },
378
+ });
379
+
380
+ /**
381
+ * Skill Loading Rate Scorer
382
+ *
383
+ * Measures whether coordinator loads relevant skills via skills_use().
384
+ * Shows knowledge-seeking behavior.
385
+ *
386
+ * Score: 1.0 if skill_loaded events exist, 0.5 otherwise (not critical, but helpful)
387
+ */
388
+ export const skillLoadingRate = createScorer({
389
+ name: "Skill Loading Rate",
390
+ description: "Coordinator loaded relevant skills for domain knowledge",
391
+ scorer: ({ output }) => {
392
+ try {
393
+ const session = JSON.parse(String(output)) as CoordinatorSession;
394
+
395
+ // Count skill_loaded events
396
+ const skills = session.events.filter(
397
+ (e) =>
398
+ e.event_type === "DECISION" && e.decision_type === "skill_loaded"
399
+ );
400
+
401
+ const count = skills.length;
402
+
403
+ if (count === 0) {
404
+ return {
405
+ score: 0.5,
406
+ message: "No skills loaded (not critical, but helpful)",
407
+ };
408
+ }
409
+
410
+ return {
411
+ score: 1.0,
412
+ message: `${count} skill(s) loaded`,
413
+ };
414
+ } catch (error) {
415
+ return {
416
+ score: 0,
417
+ message: `Failed to parse CoordinatorSession: ${error}`,
418
+ };
419
+ }
420
+ },
421
+ });
422
+
423
+ /**
424
+ * Inbox Monitoring Rate Scorer
425
+ *
426
+ * Measures how frequently coordinator checks inbox for worker messages.
427
+ * Regular monitoring (every ~15min or when workers finish) shows good coordination.
428
+ *
429
+ * Score based on inbox_checked events relative to worker activity:
430
+ * - 0 checks = 0.0 (coordinator not monitoring)
431
+ * - 1+ checks = 1.0 (coordinator is responsive)
432
+ */
433
+ export const inboxMonitoringRate = createScorer({
434
+ name: "Inbox Monitoring Rate",
435
+ description: "Coordinator checked inbox regularly for worker messages",
436
+ scorer: ({ output }) => {
437
+ try {
438
+ const session = JSON.parse(String(output)) as CoordinatorSession;
439
+
440
+ // Count inbox_checked events
441
+ const checks = session.events.filter(
442
+ (e) =>
443
+ e.event_type === "DECISION" && e.decision_type === "inbox_checked"
444
+ );
445
+
446
+ // Count worker activity (spawns + outcomes)
447
+ const workerActivity = session.events.filter(
448
+ (e) =>
449
+ (e.event_type === "DECISION" && e.decision_type === "worker_spawned") ||
450
+ (e.event_type === "OUTCOME" &&
451
+ ["subtask_success", "subtask_failed", "blocker_detected"].includes(
452
+ e.outcome_type
453
+ ))
454
+ );
455
+
456
+ const checkCount = checks.length;
457
+ const activityCount = workerActivity.length;
458
+
459
+ if (activityCount === 0) {
460
+ return {
461
+ score: 1.0,
462
+ message: "No worker activity to monitor",
463
+ };
464
+ }
465
+
466
+ if (checkCount === 0) {
467
+ return {
468
+ score: 0.0,
469
+ message: `${activityCount} worker events, 0 inbox checks (not monitoring)`,
470
+ };
471
+ }
472
+
473
+ return {
474
+ score: 1.0,
475
+ message: `${checkCount} inbox check(s) for ${activityCount} worker events`,
476
+ };
477
+ } catch (error) {
478
+ return {
479
+ score: 0,
480
+ message: `Failed to parse CoordinatorSession: ${error}`,
481
+ };
482
+ }
483
+ },
484
+ });
485
+
486
+ /**
487
+ * Blocker Response Time Scorer
488
+ *
489
+ * Measures how quickly coordinator responds to blocked workers.
490
+ * Time between blocker_detected (OUTCOME) and blocker_resolved (DECISION).
491
+ *
492
+ * Normalization:
493
+ * - < 5min: 1.0 (excellent)
494
+ * - 5-15min: linear decay to 0.5
495
+ * - > 15min: 0.0 (too slow, worker is idle)
496
+ *
497
+ * Score: Average response time across all blockers
498
+ */
499
+ export const blockerResponseTime = createScorer({
500
+ name: "Blocker Response Time",
501
+ description: "Coordinator unblocked workers quickly",
502
+ scorer: ({ output }) => {
503
+ try {
504
+ const session = JSON.parse(String(output)) as CoordinatorSession;
505
+
506
+ // Find blocker_detected events
507
+ const blockers = session.events.filter(
508
+ (e) =>
509
+ e.event_type === "OUTCOME" && e.outcome_type === "blocker_detected"
510
+ );
511
+
512
+ if (blockers.length === 0) {
513
+ return {
514
+ score: 1.0,
515
+ message: "No blockers detected",
516
+ };
517
+ }
518
+
519
+ // Find blocker_resolved events
520
+ const resolutions = session.events.filter(
521
+ (e) =>
522
+ e.event_type === "DECISION" && e.decision_type === "blocker_resolved"
523
+ );
524
+
525
+ if (resolutions.length === 0) {
526
+ return {
527
+ score: 0.0,
528
+ message: `${blockers.length} blocker(s) detected, 0 resolved (workers still blocked)`,
529
+ };
530
+ }
531
+
532
+ // Match blockers to resolutions by subtask_id and calculate response times
533
+ const responseTimes: number[] = [];
534
+ for (const blocker of blockers) {
535
+ const subtaskId = (blocker.payload as any).subtask_id;
536
+ const blockerTime = new Date(blocker.timestamp).getTime();
537
+
538
+ // Find resolution for this subtask
539
+ const resolution = resolutions.find(
540
+ (r) => (r.payload as any).subtask_id === subtaskId
541
+ );
542
+
543
+ if (resolution) {
544
+ const resolutionTime = new Date(resolution.timestamp).getTime();
545
+ const deltaMs = resolutionTime - blockerTime;
546
+ responseTimes.push(deltaMs);
547
+ }
548
+ }
549
+
550
+ if (responseTimes.length === 0) {
551
+ return {
552
+ score: 0.5,
553
+ message: `${blockers.length} blocker(s) detected, ${resolutions.length} resolution(s), but no matches by subtask_id`,
554
+ };
555
+ }
556
+
557
+ // Calculate average response time
558
+ const avgResponseMs =
559
+ responseTimes.reduce((sum, t) => sum + t, 0) / responseTimes.length;
560
+
561
+ // Normalize: < 5min = 1.0, > 15min = 0.0, linear in between
562
+ const EXCELLENT_MS = 5 * 60 * 1000; // 5 min
563
+ const POOR_MS = 15 * 60 * 1000; // 15 min
564
+
565
+ let score: number;
566
+ if (avgResponseMs < EXCELLENT_MS) {
567
+ score = 1.0;
568
+ } else if (avgResponseMs > POOR_MS) {
569
+ score = 0.0;
570
+ } else {
571
+ // Linear decay from 1.0 to 0.0
572
+ score = 1.0 - (avgResponseMs - EXCELLENT_MS) / (POOR_MS - EXCELLENT_MS);
573
+ }
574
+
575
+ const avgMinutes = Math.round(avgResponseMs / 1000 / 60);
576
+
577
+ return {
578
+ score,
579
+ message: `Avg response time: ${avgMinutes}min (${responseTimes.length}/${blockers.length} blockers resolved)`,
580
+ };
581
+ } catch (error) {
582
+ return {
583
+ score: 0,
584
+ message: `Failed to parse CoordinatorSession: ${error}`,
585
+ };
586
+ }
587
+ },
588
+ });
589
+
257
590
  /**
258
591
  * Overall Discipline Scorer
259
592
  *