@bradtaylorsf/alpha-loop 1.9.1 → 1.9.2

package/templates/evals/cases/step/review/006-missing-di-injection/metadata.yaml

@@ -0,0 +1,10 @@
+id: 006-missing-di-injection
+description: Review should flag a service that is defined but never injected into its consumer
+type: step
+step: review
+tags:
+  - wiring
+  - di
+  - quality-failure
+source: distribution
+timeout: 60

package/templates/evals/cases/step/review/007-silent-none-guard/checks.yaml

@@ -0,0 +1,12 @@
+- type: llm_judge
+  prompt: |
+    The diff adds NotificationService but makes it nullable (| null = null) in
+    OrderService. The None guard means if notifications is never injected, the
+    feature silently does nothing. The test creates OrderService without
+    notifications and "passes" because it never asserts notification behavior.
+    The review MUST flag: (1) the null default hides missing injection, and
+    (2) the test doesn't actually test notification sending.
+  rubric: |
+    Score 1.0 if the review identifies both the null-default hiding missing injection AND the test not asserting notification behavior.
+    Score 0.5 if the review flags only one of the two issues.
+    Score 0.0 if the review misses both issues.

package/templates/evals/cases/step/review/007-silent-none-guard/input.md

@@ -0,0 +1,53 @@
+## Issue: Add notification service for order status changes
+
+### Diff
+
+```diff
++++ b/src/services/notification-service.ts
+@@ -0,0 +1,15 @@
++export class NotificationService {
++  constructor(private emailClient: EmailClient) {}
++
++  async notify(userId: string, message: string): Promise<void> {
++    await this.emailClient.send(userId, message);
++  }
++}
+
++++ b/src/services/order-service.ts
+@@ -3,7 +3,13 @@
+ export class OrderService {
+-  constructor(private orderRepo: OrderRepository) {}
++  constructor(
++    private orderRepo: OrderRepository,
++    private notifications: NotificationService | null = null,
++  ) {}
+
+   async updateStatus(orderId: string, status: OrderStatus): Promise<void> {
+     const order = await this.orderRepo.findById(orderId);
+     order.status = status;
+     await this.orderRepo.save(order);
++
++    if (this.notifications !== null) {
++      await this.notifications.notify(order.userId, `Order ${orderId} is now ${status}`);
++    }
+   }
+
++++ b/tests/order-service.test.ts
+@@ -10,6 +10,14 @@
+   it('updates order status', async () => {
+     const service = new OrderService(mockRepo);
+     await service.updateStatus('ord-1', 'shipped');
+     expect(mockRepo.save).toHaveBeenCalled();
+   });
++
++  it('sends notification on status change', async () => {
++    const service = new OrderService(mockRepo);
++    await service.updateStatus('ord-1', 'shipped');
++    // Test passes because notifications is null by default
++    // No assertion on notification behavior
++  });
+```
+
+### Analysis Required
+
+Review this diff. The issue asked for notifications on order status changes.

package/templates/evals/cases/step/review/007-silent-none-guard/metadata.yaml

@@ -0,0 +1,10 @@
+id: 007-silent-none-guard
+description: Review should flag optional parameters with None guards that silently disable features
+type: step
+step: review
+tags:
+  - wiring
+  - none-guard
+  - quality-failure
+source: distribution
+timeout: 60

package/templates/evals/cases/step/review/008-route-shadowing/checks.yaml

@@ -0,0 +1,11 @@
+- type: llm_judge
+  prompt: |
+    The diff registers /evals/compare and /evals/summary AFTER /evals/:id.
+    In Express (and most routers), the parameterized route /evals/:id will
+    match "compare" and "summary" as id values before the static routes
+    are reached. The review MUST flag that static routes need to be
+    registered BEFORE parameterized routes to avoid shadowing.
+  rubric: |
+    Score 1.0 if the review identifies route shadowing — that /evals/:id will catch /evals/compare and /evals/summary.
+    Score 0.5 if the review mentions route ordering concerns but doesn't identify the specific shadowing.
+    Score 0.0 if the review misses the route shadowing issue entirely.

package/templates/evals/cases/step/review/008-route-shadowing/input.md

@@ -0,0 +1,29 @@
+## Issue: Add eval comparison endpoint
+
+### Diff
+
+```diff
++++ b/src/routes/evals.ts
+@@ -5,6 +5,11 @@
+ const router = express.Router();
+
+ router.get('/evals/:id', async (req, res) => {
+   const eval = await evalService.findById(req.params.id);
+   res.json(eval);
+ });
+
++router.get('/evals/compare', async (req, res) => {
++  const { run1, run2 } = req.query;
++  const comparison = await evalService.compare(String(run1), String(run2));
++  res.json(comparison);
++});
++
++router.get('/evals/summary', async (req, res) => {
++  const summary = await evalService.getSummary();
++  res.json(summary);
++});
+```
+
+### Analysis Required
+
+Review this diff. The issue asked to add an eval comparison endpoint.

package/templates/evals/cases/step/review/008-route-shadowing/metadata.yaml

@@ -0,0 +1,10 @@
+id: 008-route-shadowing
+description: Review should flag static routes registered after parameterized routes causing shadowing
+type: step
+step: review
+tags:
+  - wiring
+  - routing
+  - quality-failure
+source: distribution
+timeout: 60

package/templates/evals/cases/step/review/009-unthreaded-dependency/checks.yaml

@@ -0,0 +1,11 @@
+- type: llm_judge
+  prompt: |
+    The diff creates an analytics dashboard that reads from a tool_executions
+    table via AnalyticsRepository. But nothing in the diff (or mentioned as
+    existing) WRITES to the tool_executions table. The dashboard will always
+    show empty data because there is no producer for the data it consumes.
+    The review MUST flag that the data source has no writer.
+  rubric: |
+    Score 1.0 if the review identifies that nothing writes to tool_executions — the dashboard has no data source.
+    Score 0.5 if the review raises general concerns about data availability but doesn't identify the specific missing write path.
+    Score 0.0 if the review misses the unthreaded dependency entirely.

package/templates/evals/cases/step/review/009-unthreaded-dependency/input.md

@@ -0,0 +1,57 @@
+## Issue: Add analytics dashboard showing tool execution metrics
+
+### Diff
+
+```diff
++++ b/src/routes/analytics.ts
+@@ -0,0 +1,20 @@
++import { AnalyticsRepository } from '../repos/analytics-repo';
++
++const router = express.Router();
++
++router.get('/analytics/tools', async (req, res) => {
++  const metrics = await analyticsRepo.getToolMetrics();
++  res.json(metrics);
++});
++
++router.get('/analytics/tools/:toolId', async (req, res) => {
++  const history = await analyticsRepo.getToolHistory(req.params.toolId);
++  res.json(history);
++});
+
++++ b/src/repos/analytics-repo.ts
+@@ -0,0 +1,18 @@
++export class AnalyticsRepository {
++  constructor(private db: Database) {}
++
++  async getToolMetrics(): Promise<ToolMetric[]> {
++    return this.db.query('SELECT tool_name, count(*) as runs, avg(duration) as avg_duration FROM tool_executions GROUP BY tool_name');
++  }
++
++  async getToolHistory(toolId: string): Promise<ToolExecution[]> {
++    return this.db.query('SELECT * FROM tool_executions WHERE tool_name = ? ORDER BY created_at DESC LIMIT 50', [toolId]);
++  }
++}
+
++++ b/src/components/AnalyticsDashboard.tsx
+@@ -0,0 +1,30 @@
++export function AnalyticsDashboard() {
++  const { data: metrics } = useQuery('/analytics/tools');
++  return (
++    <div>
++      <h1>Tool Analytics</h1>
++      {metrics?.map(m => (
++        <div key={m.tool_name}>
++          <span>{m.tool_name}</span>
++          <span>{m.runs} runs</span>
++          <span>{m.avg_duration}ms avg</span>
++        </div>
++      ))}
++    </div>
++  );
++}
+```
+
+### Analysis Required
+
+Review this diff. The issue asked for an analytics dashboard showing tool execution metrics.

package/templates/evals/cases/step/review/009-unthreaded-dependency/metadata.yaml

@@ -0,0 +1,10 @@
+id: 009-unthreaded-dependency
+description: Review should flag a data consumer that reads from a source nothing writes to
+type: step
+step: review
+tags:
+  - wiring
+  - data-flow
+  - quality-failure
+source: distribution
+timeout: 60

package/templates/evals/cases/step/review/010-fabricated-metrics/checks.yaml

@@ -0,0 +1,12 @@
+- type: llm_judge
+  prompt: |
+    The diff estimates token counts from output string length (len/4) and
+    doubles it for input tokens. These are fabricated metrics displayed as
+    real data in the CostDashboard. The review MUST flag: (1) token counts
+    are estimated, not real API responses, (2) the cost calculation uses
+    stale hardcoded rates, and (3) the dashboard presents estimated data
+    as if it were actual usage metrics.
+  rubric: |
+    Score 1.0 if the review identifies that token counts are estimated (not from API responses) and flags this as misleading when displayed to users.
+    Score 0.5 if the review mentions estimation concerns but doesn't clearly flag it as fabricated metrics.
+    Score 0.0 if the review accepts the estimation approach without flagging data accuracy concerns.

package/templates/evals/cases/step/review/010-fabricated-metrics/input.md

@@ -0,0 +1,52 @@
+## Issue: Add cost tracking to pipeline runs
+
+### Diff
+
+```diff
++++ b/src/services/cost-tracker.ts
+@@ -0,0 +1,22 @@
++export class CostTracker {
++  async trackRun(runId: string, model: string, output: string): Promise<CostRecord> {
++    // Estimate tokens from output length
++    const estimatedTokens = Math.ceil(output.length / 4);
++    const estimatedInputTokens = estimatedTokens * 2;
++
++    const cost = this.calculateCost(model, estimatedInputTokens, estimatedTokens);
++
++    return {
++      runId,
++      model,
++      inputTokens: estimatedInputTokens,
++      outputTokens: estimatedTokens,
++      costUsd: cost,
++    };
++  }
++
++  private calculateCost(model: string, input: number, output: number): number {
++    const rates: Record<string, number> = { 'gpt-4': 0.03, 'claude': 0.015 };
++    return (input + output) * (rates[model] ?? 0.01) / 1000;
++  }
++}
+
++++ b/src/components/CostDashboard.tsx
+@@ -0,0 +1,15 @@
++export function CostDashboard({ runs }: { runs: CostRecord[] }) {
++  const totalCost = runs.reduce((sum, r) => sum + r.costUsd, 0);
++  return (
++    <div>
++      <h2>Cost Summary</h2>
++      <p>Total: ${totalCost.toFixed(2)}</p>
++      {runs.map(r => (
++        <div key={r.runId}>
++          <span>{r.model}: {r.inputTokens} in / {r.outputTokens} out</span>
++          <span>${r.costUsd.toFixed(4)}</span>
++        </div>
++      ))}
++    </div>
++  );
++}
+```
+
+### Analysis Required
+
+Review this diff. The issue asked for cost tracking of pipeline runs.

package/templates/evals/cases/step/review/010-fabricated-metrics/metadata.yaml

@@ -0,0 +1,10 @@
+id: 010-fabricated-metrics
+description: Review should flag hardcoded or estimated values displayed as real metrics
+type: step
+step: review
+tags:
+  - wiring
+  - metrics
+  - quality-failure
+source: distribution
+timeout: 60