goatcode-sh 0.0.1 → 0.1.2

package/dist/tools/task/types.d.ts

@@ -0,0 +1,84 @@
+import { z } from "zod";
+export declare const TaskStatusSchema: z.ZodEnum<{
+    completed: "completed";
+    cancelled: "cancelled";
+    pending: "pending";
+    in_progress: "in_progress";
+}>;
+export type TaskStatus = z.infer<typeof TaskStatusSchema>;
+export declare const TaskPrioritySchema: z.ZodEnum<{
+    high: "high";
+    medium: "medium";
+    low: "low";
+}>;
+export type TaskPriority = z.infer<typeof TaskPrioritySchema>;
+export declare const TaskSchema: z.ZodObject<{
+    id: z.ZodString;
+    subject: z.ZodString;
+    status: z.ZodEnum<{
+        completed: "completed";
+        cancelled: "cancelled";
+        pending: "pending";
+        in_progress: "in_progress";
+    }>;
+    priority: z.ZodEnum<{
+        high: "high";
+        medium: "medium";
+        low: "low";
+    }>;
+    content: z.ZodOptional<z.ZodString>;
+    createdAt: z.ZodNumber;
+    updatedAt: z.ZodNumber;
+}, z.core.$strip>;
+export type Task = z.infer<typeof TaskSchema>;
+export declare const TaskCreateInputSchema: z.ZodObject<{
+    subject: z.ZodString;
+    content: z.ZodOptional<z.ZodString>;
+    priority: z.ZodDefault<z.ZodEnum<{
+        high: "high";
+        medium: "medium";
+        low: "low";
+    }>>;
+    status: z.ZodDefault<z.ZodEnum<{
+        completed: "completed";
+        cancelled: "cancelled";
+        pending: "pending";
+        in_progress: "in_progress";
+    }>>;
+}, z.core.$strip>;
+export type TaskCreateInput = z.infer<typeof TaskCreateInputSchema>;
+export declare const TaskListInputSchema: z.ZodObject<{
+    status: z.ZodOptional<z.ZodEnum<{
+        completed: "completed";
+        cancelled: "cancelled";
+        pending: "pending";
+        in_progress: "in_progress";
+    }>>;
+    priority: z.ZodOptional<z.ZodEnum<{
+        high: "high";
+        medium: "medium";
+        low: "low";
+    }>>;
+}, z.core.$strip>;
+export type TaskListInput = z.infer<typeof TaskListInputSchema>;
+export declare const TaskGetInputSchema: z.ZodObject<{
+    id: z.ZodString;
+}, z.core.$strip>;
+export type TaskGetInput = z.infer<typeof TaskGetInputSchema>;
+export declare const TaskUpdateInputSchema: z.ZodObject<{
+    id: z.ZodString;
+    subject: z.ZodOptional<z.ZodString>;
+    content: z.ZodOptional<z.ZodString>;
+    status: z.ZodOptional<z.ZodEnum<{
+        completed: "completed";
+        cancelled: "cancelled";
+        pending: "pending";
+        in_progress: "in_progress";
+    }>>;
+    priority: z.ZodOptional<z.ZodEnum<{
+        high: "high";
+        medium: "medium";
+        low: "low";
+    }>>;
+}, z.core.$strip>;
+export type TaskUpdateInput = z.infer<typeof TaskUpdateInputSchema>;

package/dist/tools/task/update/handler.d.ts

@@ -0,0 +1,2 @@
+import type { ToolDefinition } from "@opencode-ai/plugin";
+export declare const taskUpdateTool: ToolDefinition;

package/dist/tools/task/update/plugin.d.ts

@@ -0,0 +1 @@
+export declare const taskUpdatePlugin: import("../../../plugin-api").PluginDefinition;

package/dist/tools/tool-builder.d.ts

@@ -0,0 +1,7 @@
+import type { ToolDefinition } from "@opencode-ai/plugin";
+export type ToolBuilderInput<TArgs extends ToolDefinition["args"] = ToolDefinition["args"], TExecuteArgs = Record<string, unknown>> = {
+    description: string;
+    args: TArgs;
+    execute: (args: TExecuteArgs, context: Parameters<ToolDefinition["execute"]>[1]) => Promise<string>;
+};
+export declare function buildTool<TArgs extends ToolDefinition["args"] = ToolDefinition["args"], TExecuteArgs = Record<string, unknown>>(input: ToolBuilderInput<TArgs, TExecuteArgs>): ToolDefinition;

package/dist/tools/tool-registry-adapter.d.ts

@@ -0,0 +1,3 @@
+import type { PluginToolContribution, ToolsRecord } from "../types/tool";
+export declare function adaptToolsToRegistry(tools: Record<string, PluginToolContribution>): ToolsRecord;
+export declare function mergeToolRegistries(...registries: ToolsRecord[]): ToolsRecord;

package/dist/types/agent.d.ts

@@ -0,0 +1,40 @@
+import type { AgentConfig } from "@opencode-ai/sdk";
+/** Re-export for convenience. */
+export type { AgentConfig };
+/**
+ * Agent execution mode.
+ * - "primary": Respects the user-selected UI model.
+ * - "subagent": Uses its own fallback chain and ignores UI selection.
+ * - "all": Available in both contexts.
+ */
+export type AgentMode = "primary" | "subagent" | "all";
+/**
+ * Factory function that creates an AgentConfig given a model string.
+ * Includes a static mode property for pre-instantiation access.
+ */
+export type AgentFactory = ((model: string) => AgentConfig) & {
+    readonly mode: AgentMode;
+};
+/** All built-in GoatCode agent names. */
+export type BuiltinAgentName = "orchestrator" | "deep-worker" | "planner" | "advisor" | "researcher" | "explorer" | "worker";
+/** Agent override config for user customization. */
+export interface AgentOverrideConfig {
+    /** Override model identifier. */
+    model?: string;
+    /** Override model variant. */
+    variant?: string;
+    /** Override sampling temperature. */
+    temperature?: number;
+    /** Override nucleus sampling value. */
+    top_p?: number;
+    /** Additional prompt text appended to the system prompt. */
+    prompt_append?: string;
+    /** Tool names explicitly denied for this agent. */
+    denied_tools?: string[];
+    /** Whether this agent is disabled. */
+    disable?: boolean;
+    /** Custom fallback model chain. */
+    fallback_models?: string | string[];
+}
+/** Agent contribution from a micro-plugin. */
+export type PluginAgentContribution = AgentConfig;

package/dist/types/category.d.ts

@@ -0,0 +1,22 @@
+/** All built-in GoatCode category names. */
+export type BuiltinCategoryName = "visual-engineering" | "ultrabrain" | "deep" | "artistry" | "quick" | "unspecified-low" | "unspecified-high" | "writing";
+/** Category configuration. */
+export interface CategoryConfig {
+    /** Model to use for this category. */
+    model?: string;
+    /** Model variant (for example: high, max, xhigh). */
+    variant?: string;
+    /** Human-readable description. */
+    description?: string;
+    /** Additional system prompt text appended for this category. */
+    prompt_append?: string;
+}
+/** Available category with resolved metadata. */
+export interface AvailableCategory {
+    /** Category name. */
+    name: string;
+    /** User-facing category description. */
+    description: string;
+    /** Resolved model for this category, if available. */
+    model?: string;
+}

package/dist/types/config.d.ts

@@ -0,0 +1,23 @@
+import type { AgentOverrideConfig, BuiltinAgentName } from "./agent";
+import type { BuiltinCategoryName, CategoryConfig } from "./category";
+/** Partial agent overrides where each built-in agent can be partially overridden. */
+export type AgentOverrides = Partial<Record<BuiltinAgentName, AgentOverrideConfig>>;
+/** Partial category overrides for built-in categories. */
+export type CategoryOverrides = Partial<Record<BuiltinCategoryName, CategoryConfig>>;
+/**
+ * Top-level GoatCode configuration shape.
+ * Runtime validation and defaults are handled by the config schema layer.
+ */
+export interface GoatCodeConfig {
+    agents?: AgentOverrides;
+    categories?: CategoryOverrides;
+    default_temperature?: number;
+    default_provider?: string;
+    provider_priority?: string[];
+    disabled_agents?: string[];
+    disabled_hooks?: string[];
+    disabled_tools?: string[];
+    disabled_skills?: string[];
+    auto_update?: boolean;
+    plugins?: string[];
+}

package/dist/types/hook.d.ts

@@ -0,0 +1,18 @@
+import type { Plugin } from "@opencode-ai/plugin";
+/** All OpenCode hook event names — single source of truth. */
+export declare const HOOK_EVENT_NAMES: readonly ["tool", "config", "chat.message", "chat.params", "chat.headers", "event", "tool.execute.before", "tool.execute.after", "experimental.chat.messages.transform", "experimental.chat.system.transform", "tool.definition", "permission.ask", "command.execute.before", "shell.env", "experimental.session.compacting", "experimental.text.complete"];
+/** All OpenCode hook event names. */
+export type HookEventName = (typeof HOOK_EVENT_NAMES)[number];
+/** Generic hook handler signature fallback. */
+export type HookHandler = (input: unknown, output: unknown) => Promise<void> | void;
+/** Raw plugin instance shape returned to OpenCode. */
+type OpenCodePluginInstance = Awaited<ReturnType<Plugin>>;
+/** Resolves the concrete OpenCode hook handler type for a hook name. */
+type HookHandlerFor<Name extends HookEventName> = NonNullable<OpenCodePluginInstance[Name]> extends (...args: infer Args) => infer Return ? (...args: Args) => Return : HookHandler;
+/** Hook contributions mapped by event name. */
+export type PluginHookContributions = {
+    [Name in HookEventName]: HookHandlerFor<Name>;
+};
+/** Priority for hook execution ordering (lower values run first). */
+export type HookPriority = number;
+export {};

package/dist/types/index.d.ts

@@ -0,0 +1,12 @@
+/** Agent types and configurations. */
+export * from "./agent";
+/** Category types and configurations. */
+export * from "./category";
+/** Configuration types for GoatCode. */
+export * from "./config";
+/** Hook types and contributions. */
+export * from "./hook";
+/** Plugin types and registry interfaces. */
+export * from "./plugin";
+/** Tool types and definitions. */
+export * from "./tool";

package/dist/types/plugin.d.ts

@@ -0,0 +1,41 @@
+import type { Plugin } from "@opencode-ai/plugin";
+import type { PluginAgentContribution } from "./agent";
+import type { PluginHookContributions } from "./hook";
+import type { PluginToolContribution } from "./tool";
+/** The raw context passed to the plugin by OpenCode. */
+export type OpenCodeContext = Parameters<Plugin>[0];
+/** The raw plugin instance returned to OpenCode. */
+export type OpenCodePluginInstance = Awaited<ReturnType<Plugin>>;
+/** Generic plugin hook handler fallback type. */
+export type PluginHookHandler = (...args: unknown[]) => Promise<void> | void;
+/**
+ * A GoatCode micro-plugin definition.
+ * This is the contract used by internal and external plugins.
+ */
+export interface PluginDefinition {
+    /** Unique plugin name (kebab-case). */
+    readonly name: string;
+    /** Semantic version string. */
+    readonly version?: string;
+    /** Plugin names this depends on (must be registered first). */
+    readonly dependencies?: readonly string[];
+    /** Hook contributions keyed by OpenCode hook event name. */
+    readonly hooks?: Partial<PluginHookContributions>;
+    /** Tool contributions keyed by tool name. */
+    readonly tools?: Record<string, PluginToolContribution>;
+    /** Agent contributions keyed by agent name. */
+    readonly agents?: Record<string, PluginAgentContribution>;
+    /** Optional setup executed during registry initialization. */
+    readonly setup?: (ctx: OpenCodeContext) => Promise<void> | void;
+    /** Optional teardown executed when plugin registry is disposed. */
+    readonly teardown?: () => Promise<void> | void;
+}
+/** Intermediate aggregated plugin output produced by the registry. */
+export interface AggregatedPlugins {
+    /** All hook contributions keyed by hook name in execution order. */
+    hooks: Map<string, PluginHookHandler[]>;
+    /** All tool contributions merged into one record. */
+    tools: Record<string, PluginToolContribution>;
+    /** All agent contributions merged into one record. */
+    agents: Record<string, PluginAgentContribution>;
+}

package/dist/types/tool.d.ts

@@ -0,0 +1,7 @@
+import type { ToolDefinition } from "@opencode-ai/plugin";
+/** Re-export for convenience. */
+export type { ToolDefinition };
+/** A tool contribution from a micro-plugin. */
+export type PluginToolContribution = ToolDefinition;
+/** Record of tools keyed by name. */
+export type ToolsRecord = Record<string, ToolDefinition>;

package/eval/README.md

@@ -0,0 +1,160 @@
+# GoatCode Evaluation Framework
+
+This directory contains the evaluation framework for GoatCode, including A/B testing and ablation studies to measure the impact of the plugin on agent performance.
+
+## Overview
+
+The evaluation framework provides two main evaluation modes:
+
+1. **A/B Evaluation** (`test:eval`): Compares GoatCode-enabled agents against a baseline (no plugin) on identical tasks. This measures the overall impact of the plugin on task completion, tool accuracy, and planning quality.
+
+2. **Ablation Study** (`test:ablation`): Disables specific hook groups (recovery, quality, context) to measure the individual impact of each hook category on agent performance.
+
+## Prerequisites
+
+- Node.js 20.20 or later
+- `npx promptfoo` available (installed via `npm install -D promptfoo`)
+- OpenCode server (optional; evals fall back to mock responses if unavailable)
+
+## Running Evaluations
+
+### A/B Evaluation
+
+Run the A/B comparison between GoatCode-enabled and baseline agents:
+
+```bash
+bun run test:eval
+```
+
+This runs the config in `promptfooconfig.yaml` and compares:
+- **with-goatcode**: Agent with the GoatCode plugin enabled
+- **baseline**: Agent without the plugin
+
+### Ablation Study
+
+Run the ablation study to measure hook group impact:
+
+```bash
+bun run test:ablation
+```
+
+This runs the config in `ablation-config.yaml` and compares:
+- **full**: All hooks enabled (baseline for ablation)
+- **no-recovery-hooks**: Recovery hooks disabled (edit-error, json-error, session-recovery, context-window-limit, error-diagnostics)
+- **no-quality-hooks**: Quality hooks disabled (comment-checker, write-file-guard, thinking-block-validator)
+- **no-context-hooks**: Context hooks disabled (context-injector, compaction-context, phase-reminder)
+
+### Direct Invocation
+
+You can also run evals directly from the `eval/` directory:
+
+```bash
+cd eval
+npx promptfoo eval -c promptfooconfig.yaml
+npx promptfoo eval -c ablation-config.yaml
+```
+
+## Interpreting Results
+
+### Output Markers
+
+- **[MOCK]**: The OpenCode server was not available, so the provider used mock responses. Results are informational only and do not reflect real agent behavior.
+- **[PASS]**: The assertion passed (e.g., task completed, tool used correctly, plan generated).
+- **[FAIL]**: The assertion failed (e.g., task incomplete, incorrect tool usage, no plan).
+
+### Scores
+
+Each test case is scored based on custom assertions:
+
+- **task-completion.ts**: Measures whether the agent successfully completed the requested task.
+- **tool-accuracy.ts**: Measures whether the agent used the correct tools and parameters.
+- **ablation-scorer.ts**: Measures task completion and quality for ablation variants.
+- **hook-impact.ts**: Measures the specific impact of disabled hook groups on code quality analysis.
+
+Higher scores indicate better performance. Scores are aggregated across all test cases to produce an overall evaluation result.
+
+### CI Behavior
+
+These evaluations are **informational only** and do not block CI. They pass even when the OpenCode server is unavailable (falling back to mock responses). This allows evals to run in isolated CI environments without external dependencies.
+
+## Adding New Scenarios
+
+### Adding Test Cases to A/B Evaluation
+
+Edit `promptfooconfig.yaml` and add a new test case under the `tests` section:
+
+```yaml
+tests:
+  - vars:
+      task: "Your new task description here"
+    assert:
+      - type: javascript
+        value: "file://assertions/task-completion.ts"
+        threshold: 0.5
+```
+
+The `task` variable is passed to both providers. You can reference custom assertions or use built-in assertion types like `contains`, `equals`, or `javascript`.
+
+### Adding Test Cases to Ablation Study
+
+Edit `ablation-config.yaml` and add a new test case under the `tests` section:
+
+```yaml
+tests:
+  - vars:
+      task: "Your new task description here"
+    assert:
+      - type: javascript
+        value: "file://assertions/ablation-scorer.ts"
+```
+
+The same task is run against all four provider variants (full, no-recovery-hooks, no-quality-hooks, no-context-hooks), allowing you to measure the impact of each hook group.
+
+## Creating New Ablation Configs
+
+To create a new ablation config that disables different hook groups:
+
+1. Copy `ablation-config.yaml` to a new file (e.g., `ablation-custom-config.yaml`)
+2. Modify the `disabledHooks` lists in each provider config:
+
+```yaml
+providers:
+  - id: file://providers/opencode-provider.ts
+    label: custom-variant
+    config:
+      enablePlugin: true
+      disabledHooks: ["hook-name-1", "hook-name-2"]
+```
+
+3. Update the `test:ablation` script in `package.json` to point to your new config:
+
+```json
+"test:ablation": "cd eval && npx promptfoo eval -c ablation-custom-config.yaml"
+```
+
+Available hooks are documented in the main [README.md](../README.md#hooks).
+
+## File Structure
+
+```text
+eval/
+├── promptfooconfig.yaml      — A/B evaluation config (with-goatcode vs baseline)
+├── ablation-config.yaml      — Ablation study config (hook group impact)
+├── README.md                 — This file
+├── providers/
+│   ├── opencode-provider.ts  — Provider that calls the OpenCode server
+│   └── opencode-baseline.ts  — Baseline provider (no plugin)
+├── assertions/
+│   ├── task-completion.ts    — Assertion: task completed successfully
+│   ├── tool-accuracy.ts      — Assertion: correct tool usage
+│   ├── ablation-scorer.ts    — Assertion: ablation study scoring
+│   └── hook-impact.ts        — Assertion: hook group impact measurement
+└── spike/
+    └── config.yaml           — Original feasibility spike (archived)
+```
+
+## Notes
+
+- All relative paths in YAML configs (e.g., `file://providers/...`) are resolved relative to the `eval/` directory. The `cd eval &&` prefix in the npm scripts ensures correct path resolution.
+- Evals are designed to be environment-agnostic and work with or without a live OpenCode server.
+- Results are stored in `.promptfoo/` (created by promptfoo) and can be viewed in the promptfoo UI.

package/eval/ablation-config.yaml

@@ -0,0 +1,43 @@
+description: "GoatCode Ablation Study — Hook Group Impact Analysis"
+
+providers:
+  - id: file://providers/opencode-provider.ts
+    label: full
+    config:
+      enablePlugin: true
+      disabledHooks: []
+  - id: file://providers/opencode-provider.ts
+    label: no-recovery-hooks
+    config:
+      enablePlugin: true
+      disabledHooks: ["edit-error", "json-error", "session-recovery", "context-window-limit", "error-diagnostics"]
+  - id: file://providers/opencode-provider.ts
+    label: no-quality-hooks
+    config:
+      enablePlugin: true
+      disabledHooks: ["comment-checker", "write-file-guard", "thinking-block-validator"]
+  - id: file://providers/opencode-provider.ts
+    label: no-context-hooks
+    config:
+      enablePlugin: true
+      disabledHooks: ["context-injector", "compaction-context", "phase-reminder"]
+
+prompts:
+  - "{{task}}"
+
+tests:
+  - vars:
+      task: "Find and fix the JSON parse error in the response handler"
+    assert:
+      - type: javascript
+        value: "file://assertions/ablation-scorer.ts"
+  - vars:
+      task: "Analyse code quality: check for empty catch blocks and unread files"
+    assert:
+      - type: javascript
+        value: "file://assertions/hook-impact.ts"
+  - vars:
+      task: "Summarise the current project phase using system context"
+    assert:
+      - type: javascript
+        value: "file://assertions/ablation-scorer.ts"

package/eval/assertions/ablation-scorer.ts

@@ -0,0 +1,81 @@
+// IMPORTANT: No bun:* imports — this file runs under Node.js via npx promptfoo eval
+
+const QUALITY_INDICATORS = [
+  "fix",
+  "error",
+  "found",
+  "parse",
+  "json",
+  "handler",
+  "response",
+  "resolved",
+  "updated",
+  "corrected",
+  "phase",
+  "context",
+  "summary",
+  "project",
+  "current",
+  "plan",
+  "step",
+  "complete",
+];
+
+export default function assertAblationScore(
+  output: string,
+  context: {
+    prompt: string;
+    vars: Record<string, string>;
+    provider?: { id?: string; label?: string };
+  },
+): { pass: boolean; score: number; reason: string; namedScores: { quality_score: number } } {
+  if (!output || output.trim().length === 0) {
+    return { pass: false, score: 0, reason: "Output is empty", namedScores: { quality_score: 0 } };
+  }
+
+  if (output.startsWith("[MOCK]")) {
+    return {
+      pass: true,
+      score: 0,
+      reason: `Mock/skipped — no live server. Provider: ${context.provider?.label ?? "unknown"}`,
+      namedScores: { quality_score: 0 },
+    };
+  }
+
+  const lowerOutput = output.toLowerCase();
+  const providerLabel = context.provider?.label ?? "unknown";
+
+  const matchedIndicators = QUALITY_INDICATORS.filter((word) =>
+    lowerOutput.includes(word),
+  );
+
+  const lengthScore = Math.min(output.length / 800, 1.0) * 0.25;
+  const indicatorScore =
+    Math.min(matchedIndicators.length / 5, 1.0) * 0.5;
+
+  const score = Number(
+    (lengthScore + indicatorScore).toFixed(2),
+  );
+
+  if (providerLabel === "full") {
+    const pass = score >= 0.3 && matchedIndicators.length >= 2;
+    return {
+      pass,
+      score,
+      reason: pass
+        ? `Full provider quality OK (score: ${score}). Indicators: ${matchedIndicators.join(", ")}`
+        : `Full provider quality low (score: ${score}). Indicators found: ${matchedIndicators.length}`,
+      namedScores: { quality_score: score },
+    };
+  }
+
+  const pass = matchedIndicators.length >= 1;
+  return {
+    pass,
+    score,
+    reason: pass
+      ? `Ablated [${providerLabel}] quality measured (score: ${score}). Indicators: ${matchedIndicators.join(", ")}`
+      : `Ablated [${providerLabel}] produced minimal output (score: ${score}). Indicators found: ${matchedIndicators.length}`,
+    namedScores: { quality_score: score },
+  };
+}