oh-my-opencode 4.4.0 → 4.5.0

package/.agents/skills/work-with-pr-workspace/iteration-1/benchmark.json

@@ -0,0 +1,138 @@
+{
+  "skill_name": "work-with-pr",
+  "iteration": 1,
+  "summary": {
+    "with_skill": {
+      "pass_rate": 0.968,
+      "mean_duration_seconds": 340.2,
+      "stddev_duration_seconds": 169.3
+    },
+    "without_skill": {
+      "pass_rate": 0.516,
+      "mean_duration_seconds": 303.0,
+      "stddev_duration_seconds": 77.8
+    },
+    "delta": {
+      "pass_rate": 0.452,
+      "mean_duration_seconds": 37.2,
+      "stddev_duration_seconds": 91.5
+    }
+  },
+  "evals": [
+    {
+      "eval_name": "happy-path-feature-config-option",
+      "with_skill": {
+        "pass_rate": 1.0,
+        "passed": 10,
+        "total": 10,
+        "duration_seconds": 292,
+        "failed_assertions": []
+      },
+      "without_skill": {
+        "pass_rate": 0.4,
+        "passed": 4,
+        "total": 10,
+        "duration_seconds": 365,
+        "failed_assertions": [
+          {"assertion": "Plan uses git worktree in a sibling directory", "reason": "Uses git checkout -b, no worktree isolation"},
+          {"assertion": "Plan specifies multiple atomic commits for multi-file changes", "reason": "Steps listed sequentially but no atomic commit strategy mentioned"},
+          {"assertion": "Verification loop includes all 3 gates: CI, review-work, and Cubic", "reason": "Only mentions CI pipeline in step 6. No review-work or Cubic."},
+          {"assertion": "Gates are checked in order: CI first, then review-work, then Cubic", "reason": "No gate ordering - only CI mentioned"},
+          {"assertion": "Cubic check uses gh api to check cubic-dev-ai[bot] reviews", "reason": "No mention of Cubic at all"},
+          {"assertion": "Plan includes worktree cleanup after merge", "reason": "No worktree used, no cleanup needed"}
+        ]
+      }
+    },
+    {
+      "eval_name": "bugfix-atlas-null-check",
+      "with_skill": {
+        "pass_rate": 1.0,
+        "passed": 6,
+        "total": 6,
+        "duration_seconds": 506,
+        "failed_assertions": []
+      },
+      "without_skill": {
+        "pass_rate": 0.667,
+        "passed": 4,
+        "total": 6,
+        "duration_seconds": 325,
+        "failed_assertions": [
+          {"assertion": "Plan uses git worktree in a sibling directory", "reason": "No worktree. Steps go directly to creating branch and modifying files."},
+          {"assertion": "Verification loop includes all 3 gates", "reason": "Only mentions CI pipeline (step 5). No review-work or Cubic."}
+        ]
+      }
+    },
+    {
+      "eval_name": "refactor-split-constants",
+      "with_skill": {
+        "pass_rate": 1.0,
+        "passed": 5,
+        "total": 5,
+        "duration_seconds": 181,
+        "failed_assertions": []
+      },
+      "without_skill": {
+        "pass_rate": 0.4,
+        "passed": 2,
+        "total": 5,
+        "duration_seconds": 229,
+        "failed_assertions": [
+          {"assertion": "Plan uses git worktree in a sibling directory", "reason": "git checkout -b only, no worktree"},
+          {"assertion": "Uses 2+ commits for the multi-file refactor", "reason": "Single atomic commit: 'refactor: split delegate-task constants and category model requirements'"},
+          {"assertion": "Verification loop includes all 3 gates", "reason": "Only mentions typecheck/test/build. No review-work or Cubic."}
+        ]
+      }
+    },
+    {
+      "eval_name": "new-mcp-arxiv-casual",
+      "with_skill": {
+        "pass_rate": 1.0,
+        "passed": 5,
+        "total": 5,
+        "duration_seconds": 152,
+        "failed_assertions": []
+      },
+      "without_skill": {
+        "pass_rate": 0.6,
+        "passed": 3,
+        "total": 5,
+        "duration_seconds": 197,
+        "failed_assertions": [
+          {"assertion": "Verification loop includes all 3 gates", "reason": "Only mentions bun test/typecheck/build. No review-work or Cubic."}
+        ]
+      }
+    },
+    {
+      "eval_name": "regex-fix-false-positive",
+      "with_skill": {
+        "pass_rate": 0.8,
+        "passed": 4,
+        "total": 5,
+        "duration_seconds": 570,
+        "failed_assertions": [
+          {"assertion": "Only modifies regex and adds tests — no unrelated changes", "reason": "Also proposes config schema change (exclude_patterns) and Go binary update — goes beyond minimal fix"}
+        ]
+      },
+      "without_skill": {
+        "pass_rate": 0.6,
+        "passed": 3,
+        "total": 5,
+        "duration_seconds": 399,
+        "failed_assertions": [
+          {"assertion": "Plan uses git worktree in a sibling directory", "reason": "git checkout -b, no worktree"},
+          {"assertion": "Verification loop includes all 3 gates", "reason": "Only bun test and typecheck. No review-work or Cubic."}
+        ]
+      }
+    }
+  ],
+  "analyst_observations": [
+    "Three-gates assertion (CI + review-work + Cubic) is the strongest discriminator: 5/5 with-skill vs 0/5 without-skill. Without the skill, agents never know about Cubic or review-work gates.",
+    "Worktree isolation is nearly as discriminating (5/5 vs 1/5). One without-skill run (eval-4) independently chose worktree, suggesting some agents already know worktree patterns, but the skill makes it consistent.",
+    "The skill's only failure (eval-5 minimal-change) reveals a potential over-engineering tendency: the skill-guided agent proposed config schema changes and Go binary updates for what should have been a minimal regex fix. Consider adding explicit guidance for fix-type tasks to stay minimal.",
+    "Duration tradeoff: with-skill is 12% slower on average (340s vs 303s), driven mainly by eval-2 (bugfix) and eval-5 (regex fix) where the skill's thorough verification planning adds overhead. For eval-1 and eval-3-4, with-skill was actually faster.",
+    "Without-skill duration has lower variance (stddev 78s vs 169s), suggesting the skill introduces more variable execution paths depending on task complexity.",
+    "Non-discriminating assertions: 'References actual files', 'PR targets dev', 'Runs local checks' — these pass regardless of skill. They validate baseline agent competence, not skill value. Consider removing or downweighting in future iterations.",
+    "Atomic commits assertion discriminates moderately (2/2 with-skill tested vs 0/2 without-skill tested). Without the skill, agents default to single commits even for multi-file refactors."
+  ]
+}

package/.agents/skills/work-with-pr-workspace/iteration-1/benchmark.md

@@ -0,0 +1,42 @@
+# Benchmark: work-with-pr (Iteration 1)
+
+## Summary
+
+| Metric | With Skill | Without Skill | Delta |
+|--------|-----------|---------------|-------|
+| Pass Rate | 96.8% (30/31) | 51.6% (16/31) | +45.2% |
+| Mean Duration | 340.2s | 303.0s | +37.2s |
+| Duration Stddev | 169.3s | 77.8s | +91.5s |
+
+## Per-Eval Breakdown
+
+| Eval | With Skill | Without Skill | Delta |
+|------|-----------|---------------|-------|
+| happy-path-feature-config-option | 100% (10/10) | 40% (4/10) | +60% |
+| bugfix-atlas-null-check | 100% (6/6) | 67% (4/6) | +33% |
+| refactor-split-constants | 100% (5/5) | 40% (2/5) | +60% |
+| new-mcp-arxiv-casual | 100% (5/5) | 60% (3/5) | +40% |
+| regex-fix-false-positive | 80% (4/5) | 60% (3/5) | +20% |
+
+## Key Discriminators
+
+- **three-gates** (CI + review-work + Cubic): 5/5 vs 0/5 — strongest signal
+- **worktree-isolation**: 5/5 vs 1/5
+- **atomic-commits**: 2/2 vs 0/2
+- **cubic-check-method**: 1/1 vs 0/1
+
+## Non-Discriminating Assertions
+
+- References actual files: passes in both conditions
+- PR targets dev: passes in both conditions
+- Runs local checks before pushing: passes in both conditions
+
+## Only With-Skill Failure
+
+- **eval-5 minimal-change**: Skill-guided agent proposed config schema changes and Go binary update for a minimal regex fix. The skill may encourage over-engineering in fix scenarios.
+
+## Analyst Notes
+
+- The skill adds most value for procedural knowledge (verification gates, worktree workflow) that agents cannot infer from codebase alone.
+- Duration cost is modest (+12%) and acceptable given the +45% pass rate improvement.
+- Consider adding explicit "fix-type tasks: stay minimal" guidance in iteration 2.

package/.agents/skills/work-with-pr-workspace/iteration-1/eval-1/eval_metadata.json

@@ -0,0 +1,57 @@
+{
+  "eval_id": 1,
+  "eval_name": "happy-path-feature-config-option",
+  "prompt": "I need to add a `max_background_agents` config option to oh-my-opencode that limits how many background agents can run simultaneously. It should be in the plugin config schema with a default of 5. Add validation and make sure the background manager respects it. Create a PR for this.",
+  "assertions": [
+    {
+      "id": "worktree-isolation",
+      "text": "Plan uses git worktree in a sibling directory (not main working directory)",
+      "type": "manual"
+    },
+    {
+      "id": "branch-from-dev",
+      "text": "Branch is created from origin/dev (not master/main)",
+      "type": "manual"
+    },
+    {
+      "id": "atomic-commits",
+      "text": "Plan specifies multiple atomic commits for multi-file changes",
+      "type": "manual"
+    },
+    {
+      "id": "local-validation",
+      "text": "Runs bun run typecheck, bun test, and bun run build before pushing",
+      "type": "manual"
+    },
+    {
+      "id": "pr-targets-dev",
+      "text": "PR is created targeting dev branch (not master)",
+      "type": "manual"
+    },
+    {
+      "id": "three-gates",
+      "text": "Verification loop includes all 3 gates: CI, review-work, and Cubic",
+      "type": "manual"
+    },
+    {
+      "id": "gate-ordering",
+      "text": "Gates are checked in order: CI first, then review-work, then Cubic",
+      "type": "manual"
+    },
+    {
+      "id": "cubic-check-method",
+      "text": "Cubic check uses gh api to check cubic-dev-ai[bot] reviews for 'No issues found'",
+      "type": "manual"
+    },
+    {
+      "id": "worktree-cleanup",
+      "text": "Plan includes worktree cleanup after merge",
+      "type": "manual"
+    },
+    {
+      "id": "real-file-references",
+      "text": "Code changes reference actual files in the codebase (config schema, background manager)",
+      "type": "manual"
+    }
+  ]
+}

package/.agents/skills/work-with-pr-workspace/iteration-1/eval-1/with_skill/grading.json

@@ -0,0 +1,15 @@
+{
+  "run_id": "eval-1-with_skill",
+  "expectations": [
+    {"text": "Plan uses git worktree in a sibling directory", "passed": true, "evidence": "Uses ../omo-wt/feat-max-background-agents"},
+    {"text": "Branch is created from origin/dev", "passed": true, "evidence": "git checkout dev && git pull origin dev, then branch"},
+    {"text": "Plan specifies multiple atomic commits for multi-file changes", "passed": true, "evidence": "2 commits: schema+tests, then concurrency+manager"},
+    {"text": "Runs bun run typecheck, bun test, and bun run build before pushing", "passed": true, "evidence": "Explicit pre-push section with all 3 commands"},
+    {"text": "PR is created targeting dev branch", "passed": true, "evidence": "--base dev in gh pr create"},
+    {"text": "Verification loop includes all 3 gates: CI, review-work, and Cubic", "passed": true, "evidence": "Gate A (CI), Gate B (review-work 5 agents), Gate C (Cubic)"},
+    {"text": "Gates are checked in order: CI first, then review-work, then Cubic", "passed": true, "evidence": "Explicit ordering in verify loop pseudocode"},
+    {"text": "Cubic check uses gh api to check cubic-dev-ai[bot] reviews", "passed": true, "evidence": "Mentions cubic-dev-ai[bot] and 'No issues found' signal"},
+    {"text": "Plan includes worktree cleanup after merge", "passed": true, "evidence": "Phase 4: git worktree remove ../omo-wt/feat-max-background-agents"},
+    {"text": "Code changes reference actual files in the codebase", "passed": true, "evidence": "References src/config/schema/background-task.ts, src/features/background-agent/concurrency.ts, manager.ts"}
+  ]
+}