goatcode-sh 0.0.1 → 0.1.3

package/eval/assertions/hook-impact.ts

@@ -0,0 +1,152 @@
+// IMPORTANT: No bun:* imports — this file runs under Node.js via npx promptfoo eval
+
+interface HookGroupSignals {
+  keywords: string[];
+  description: string;
+}
+
+const HOOK_GROUPS: Record<string, HookGroupSignals> = {
+  "no-recovery-hooks": {
+    keywords: [
+      "fix",
+      "retry",
+      "recover",
+      "fallback",
+      "error",
+      "catch",
+      "handle",
+      "repair",
+      "corrected",
+      "resolved",
+    ],
+    description: "error-recovery signals",
+  },
+  "no-quality-hooks": {
+    keywords: [
+      "lint",
+      "guard",
+      "validate",
+      "quality",
+      "check",
+      "comment",
+      "block",
+      "catch",
+      "empty",
+      "verified",
+    ],
+    description: "code quality signals",
+  },
+  "no-context-hooks": {
+    keywords: [
+      "context",
+      "phase",
+      "inject",
+      "reminder",
+      "current",
+      "project",
+      "state",
+      "summary",
+      "scope",
+      "awareness",
+    ],
+    description: "context injection signals",
+  },
+};
+
+const ALL_SIGNALS = [
+  ...new Set(
+    Object.values(HOOK_GROUPS).flatMap((group) => group.keywords),
+  ),
+];
+
+export default function assertHookImpact(
+  output: string,
+  context: {
+    prompt: string;
+    vars: Record<string, string>;
+    provider?: { id?: string; label?: string };
+  },
+): { pass: boolean; score: number; reason: string; namedScores: { hook_impact: number } } {
+  if (!output || output.trim().length === 0) {
+    return { pass: false, score: 0, reason: "Output is empty", namedScores: { hook_impact: 0 } };
+  }
+
+  if (output.startsWith("[MOCK]")) {
+    return {
+      pass: true,
+      score: 0,
+      reason: `Mock/skipped — no live server. Provider: ${context.provider?.label ?? "unknown"}`,
+      namedScores: { hook_impact: 0 },
+    };
+  }
+
+  const lowerOutput = output.toLowerCase();
+  const providerLabel = context.provider?.label ?? "unknown";
+
+  if (providerLabel === "full") {
+    return scoreFullProvider(lowerOutput);
+  }
+
+  const hookGroup = HOOK_GROUPS[providerLabel];
+  if (hookGroup) {
+    return scoreAblatedProvider(lowerOutput, providerLabel, hookGroup);
+  }
+
+  return scoreGeneric(lowerOutput, providerLabel);
+}
+
+function scoreFullProvider(
+  lowerOutput: string,
+): { pass: boolean; score: number; reason: string; namedScores: { hook_impact: number } } {
+  const matched = ALL_SIGNALS.filter((kw) => lowerOutput.includes(kw));
+  const score = Number(Math.min(matched.length / 8, 1.0).toFixed(2));
+  const pass = matched.length >= 2;
+
+  return {
+    pass,
+    score,
+    reason: pass
+      ? `Full provider shows broad hook signals (score: ${score}). Matched: ${matched.slice(0, 8).join(", ")}`
+      : `Full provider lacks hook signals (score: ${score}). Matched: ${matched.length}`,
+    namedScores: { hook_impact: score },
+  };
+}
+
+function scoreAblatedProvider(
+  lowerOutput: string,
+  label: string,
+  group: HookGroupSignals,
+): { pass: boolean; score: number; reason: string; namedScores: { hook_impact: number } } {
+  const matchedGroupSignals = group.keywords.filter((kw) =>
+    lowerOutput.includes(kw),
+  );
+
+  const disabledGroupPresence = matchedGroupSignals.length / group.keywords.length;
+
+  const impactScore = Number((1.0 - disabledGroupPresence).toFixed(2));
+  const pass = impactScore >= 0.5;
+
+  return {
+    pass,
+    score: impactScore,
+    reason: pass
+      ? `Ablated [${label}] shows measurable hook impact (score: ${impactScore}). ${matchedGroupSignals.length}/${group.keywords.length} ${group.description} present.`
+      : `Ablated [${label}] shows weak hook impact (score: ${impactScore}). ${matchedGroupSignals.length}/${group.keywords.length} ${group.description} present.`,
+    namedScores: { hook_impact: impactScore },
+  };
+}
+
+function scoreGeneric(
+  lowerOutput: string,
+  label: string,
+): { pass: boolean; score: number; reason: string; namedScores: { hook_impact: number } } {
+  const matched = ALL_SIGNALS.filter((kw) => lowerOutput.includes(kw));
+  const score = Number(Math.min(matched.length / 6, 1.0).toFixed(2));
+
+  return {
+    pass: true,
+    score,
+    reason: `Generic scoring for [${label}] (score: ${score}). Signals: ${matched.length}`,
+    namedScores: { hook_impact: score },
+  };
+}

package/eval/assertions/task-completion.ts

@@ -0,0 +1,65 @@
+// IMPORTANT: No bun:* imports — this file runs under Node.js via npx promptfoo eval
+
+const CONTENT_INDICATORS = [
+  "found",
+  "result",
+  "file",
+  "function",
+  "usage",
+  "reference",
+  "line",
+  "match",
+  "complete",
+  "done",
+  "created",
+  "updated",
+  "error",
+  "fixed",
+  "plan",
+  "step",
+];
+
+export default function assertTaskCompletion(
+  output: string,
+  context: {
+    prompt: string;
+    vars: Record<string, string>;
+    provider?: { id?: string };
+  },
+): { pass: boolean; score: number; reason: string; namedScores: { completion: number } } {
+  if (!output || output.trim().length === 0) {
+    return { pass: false, score: 0, reason: "Output is empty", namedScores: { completion: 0 } };
+  }
+
+  if (output.startsWith("[MOCK]")) {
+    return {
+      pass: true,
+      score: 0,
+      reason: `Mock response — assertion skipped. Output: ${output.slice(0, 100)}`,
+      namedScores: { completion: 0 },
+    };
+  }
+
+  const lowerOutput = output.toLowerCase();
+
+  const matchedIndicators = CONTENT_INDICATORS.filter((word) =>
+    lowerOutput.includes(word),
+  );
+
+  // Weighted score: content presence (0.3) + output length (0.3) + indicator matches (0.4)
+  const hasContent = output.trim().length > 20 ? 0.3 : 0.1;
+  const lengthScore = Math.min(output.length / 500, 1.0) * 0.3;
+  const indicatorScore = Math.min(matchedIndicators.length / 4, 1.0) * 0.4;
+
+  const score = Number((hasContent + lengthScore + indicatorScore).toFixed(2));
+  const pass = score >= 0.3 && matchedIndicators.length >= 1;
+
+  return {
+    pass,
+    score,
+    reason: pass
+      ? `Task completion detected (score: ${score}). Indicators: ${matchedIndicators.join(", ")}`
+      : `Insufficient task completion indicators (score: ${score}). Found: ${matchedIndicators.length} indicators`,
+    namedScores: { completion: score },
+  };
+}

package/eval/assertions/tool-accuracy.ts

@@ -0,0 +1,56 @@
+// IMPORTANT: No bun:* imports — this file runs under Node.js via npx promptfoo eval
+
+const TOOL_KEYWORDS = [
+  "lsp",
+  "goto_definition",
+  "find_references",
+  "diagnostics",
+  "symbols",
+  "grep",
+  "glob",
+  "ast_grep",
+  "search",
+  "edit",
+  "hashline_edit",
+  "rename",
+  "bash",
+];
+
+export default function assertToolAccuracy(
+  output: string,
+  context: {
+    prompt: string;
+    vars: Record<string, string>;
+    provider?: { id?: string };
+  },
+): { pass: boolean; score: number; reason: string; namedScores: { tool_accuracy: number } } {
+  if (!output || output.trim().length === 0) {
+    return { pass: false, score: 0, reason: "Output is empty", namedScores: { tool_accuracy: 0 } };
+  }
+
+  if (output.startsWith("[MOCK]") || output.startsWith("[BASELINE]")) {
+    return {
+      pass: true,
+      score: 0,
+      reason: `Mock/baseline response — assertion skipped. Output: ${output.slice(0, 100)}`,
+      namedScores: { tool_accuracy: 0 },
+    };
+  }
+
+  const lowerOutput = output.toLowerCase();
+  const matchedTools = TOOL_KEYWORDS.filter((kw) =>
+    lowerOutput.includes(kw),
+  );
+
+  const score = Math.min(matchedTools.length / 3, 1.0);
+  const pass = matchedTools.length >= 1;
+
+  return {
+    pass,
+    score,
+    reason: pass
+      ? `Found ${matchedTools.length} tool references: ${matchedTools.join(", ")}`
+      : "No tool references found in output",
+    namedScores: { tool_accuracy: score },
+  };
+}

package/eval/promptfooconfig.yaml

@@ -0,0 +1,42 @@
+description: "GoatCode A/B Evaluation — With vs Without Plugin"
+
+providers:
+  - id: file://providers/opencode-provider.ts
+    label: with-goatcode
+    config:
+      enablePlugin: true
+      model: "anthropic/claude-sonnet-4-20250514"
+  - id: file://providers/opencode-baseline.ts
+    label: baseline
+
+defaultTest:
+  assert:
+    - type: javascript
+      value: "output.length > 0"
+    - type: latency
+      threshold: 30000
+    - type: cost
+      threshold: 0.10
+
+prompts:
+  - "{{task}}"
+
+tests:
+  - vars:
+      task: "Find all usages of the function 'bootstrap'"
+    assert:
+      - type: javascript
+        value: "file://assertions/task-completion.ts"
+        threshold: 0.5
+  - vars:
+      task: "Fix the edit error and retry the write"
+    assert:
+      - type: javascript
+        value: "file://assertions/tool-accuracy.ts"
+        threshold: 0.5
+  - vars:
+      task: "Create a plan for refactoring the hooks module"
+    assert:
+      - type: javascript
+        value: "file://assertions/task-completion.ts"
+        threshold: 0.5

package/eval/providers/opencode-baseline.ts

@@ -0,0 +1,63 @@
+// IMPORTANT: No bun:* imports — this file runs under Node.js via npx promptfoo eval
+
+import {
+  callLive,
+  type OpenCodeProviderConfig,
+  type ProviderResponse,
+} from "./opencode-client.ts";
+
+/**
+ * Baseline provider for A/B comparison — simulates OpenCode WITHOUT GoatCode.
+ *
+ * promptfoo config example:
+ *   providers:
+ *     - id: file://eval/providers/opencode-baseline.ts
+ *       config:
+ *         model: "anthropic/claude-sonnet-4-20250514"
+ */
+export default class OpenCodeBaselineProvider {
+  id() {
+    return "goatcode-opencode-baseline";
+  }
+
+  async callApi(
+    prompt: string,
+    context?: { config?: { config?: { model?: string } } },
+  ): Promise<ProviderResponse> {
+    const startTime = Date.now();
+    const model = context?.config?.config?.model ?? "anthropic/claude-sonnet-4-20250514";
+    const cfg: OpenCodeProviderConfig = {
+      enablePlugin: false,
+      disabledHooks: ["*"],
+      disabledTools: ["*"],
+      model,
+    };
+
+    try {
+      return await callLive(prompt, cfg, startTime);
+    } catch (err) {
+      const errMsg = err instanceof Error ? err.message : String(err);
+
+      return {
+        output: `[BASELINE] GoatCode plugin DISABLED. No hooks, no tools, no agents. Model: ${model}. Task: ${prompt}`,
+        tokenUsage: { total: 0, prompt: 0, completion: 0 },
+        cost: 0,
+        metadata: {
+          mock: true,
+          baseline: true,
+          error: errMsg,
+          // NOTE: The OpenCode SDK does not support per-session plugin configuration.
+          // enablePlugin: false is stored in metadata only. In a production setup,
+          // this would require modifying opencode.json and restarting the server.
+          pluginEnabled: false,
+          disabledHooks: ["*"],
+          disabledTools: ["*"],
+          model,
+          toolsUsed: [],
+          hooksFired: [],
+          sessionDuration: Date.now() - startTime,
+        },
+      };
+    }
+  }
+}

package/eval/providers/opencode-client.ts

@@ -0,0 +1,112 @@
+// IMPORTANT: No bun:* imports — this file runs under Node.js via npx promptfoo eval
+
+import {
+  createOpencodeClient,
+  type Part,
+  type Session,
+  type TextPart,
+  type ToolPart,
+} from "@opencode-ai/sdk";
+
+export interface OpenCodeProviderConfig {
+  enablePlugin: boolean;
+  disabledHooks: string[];
+  disabledTools: string[];
+  model: string;
+}
+
+export interface ProviderResponse {
+  output: string;
+  tokenUsage: { total: number; prompt: number; completion: number };
+  cost: number;
+  metadata: Record<string, unknown>;
+}
+
+interface SdkResult<T> {
+  data?: T;
+  error?: unknown;
+}
+
+interface PromptInfo {
+  tokens?: { input: number; output: number; reasoning?: number };
+  cost?: number;
+  finish?: string;
+}
+
+interface PromptData {
+  parts?: Part[];
+  info?: PromptInfo;
+}
+
+export async function callLive(
+  prompt: string,
+  cfg: OpenCodeProviderConfig,
+  startTime: number,
+): Promise<ProviderResponse> {
+  const client = createOpencodeClient();
+
+  const createResult = (await client.session.create({
+    body: { title: `eval-${Date.now()}` },
+    query: { directory: process.cwd() },
+  })) as SdkResult<Session>;
+
+  const session = createResult.data;
+  if (!session?.id) {
+    throw new Error(
+      `Session creation failed: ${JSON.stringify(createResult.error ?? createResult)}`,
+    );
+  }
+
+  const sessionId = session.id;
+
+  const promptResult = (await client.session.prompt({
+    body: {
+      parts: [{ type: "text" as const, text: prompt }],
+    },
+    path: { id: sessionId },
+    query: { directory: process.cwd() },
+  })) as SdkResult<PromptData>;
+
+  const response = promptResult.data;
+  if (!response) {
+    throw new Error(
+      `Prompt failed: ${JSON.stringify(promptResult.error ?? promptResult)}`,
+    );
+  }
+
+  const parts: Part[] = response.parts ?? [];
+  const textParts = parts.filter((p): p is TextPart => p.type === "text");
+  const outputText = textParts.map((p) => p.text).join("\n") || "[No text output]";
+
+  const toolParts = parts.filter((p): p is ToolPart => p.type === "tool");
+  const toolsUsed = toolParts.map((p) => p.tool);
+
+  const info = response.info;
+  const tokens = info?.tokens ?? { input: 0, output: 0, reasoning: 0 };
+  const totalTokens = tokens.input + tokens.output + (tokens.reasoning ?? 0);
+
+  return {
+    output: outputText,
+    tokenUsage: {
+      total: totalTokens,
+      prompt: tokens.input,
+      completion: tokens.output,
+    },
+    cost: info?.cost ?? 0,
+    metadata: {
+      mock: false,
+      toolsUsed,
+      hooksFired: [],
+      sessionDuration: Date.now() - startTime,
+      sessionId,
+      // NOTE: The OpenCode SDK does not support per-session plugin configuration.
+      // enablePlugin/disabledHooks/disabledTools are stored in metadata only.
+      // In a production setup, this would require modifying opencode.json and restarting the server.
+      pluginEnabled: cfg.enablePlugin,
+      disabledHooks: cfg.disabledHooks,
+      disabledTools: cfg.disabledTools,
+      model: cfg.model,
+      finishReason: info?.finish,
+    },
+  };
+}

package/eval/providers/opencode-provider.ts

@@ -0,0 +1,66 @@
+// IMPORTANT: No bun:* imports — this file runs under Node.js via npx promptfoo eval
+
+import {
+  callLive,
+  type OpenCodeProviderConfig,
+  type ProviderResponse,
+} from "./opencode-client.ts";
+
+const DEFAULT_CONFIG: OpenCodeProviderConfig = {
+  enablePlugin: true,
+  disabledHooks: [],
+  disabledTools: [],
+  model: "anthropic/claude-sonnet-4-20250514",
+};
+
+/**
+ * promptfoo custom provider — uses @opencode-ai/sdk to spawn a real session.
+ * Falls back to a structured mock response if the server isn't running.
+ *
+ * promptfoo config example:
+ *   providers:
+ *     - id: file://eval/providers/opencode-provider.ts
+ *       config:
+ *         enablePlugin: true
+ *         disabledHooks: []
+ *         disabledTools: []
+ *         model: "anthropic/claude-sonnet-4-20250514"
+ */
+export default class OpenCodeProvider {
+  id() {
+    return "goatcode-opencode-provider";
+  }
+
+  async callApi(
+    prompt: string,
+    context?: { config?: { config?: Partial<OpenCodeProviderConfig> } },
+  ): Promise<ProviderResponse> {
+    const cfg: OpenCodeProviderConfig = {
+      ...DEFAULT_CONFIG,
+      ...(context?.config?.config ?? {}),
+    };
+
+    const startTime = Date.now();
+
+    try {
+      return await callLive(prompt, cfg, startTime);
+    } catch (err) {
+      const label = cfg.enablePlugin ? "ENABLED" : "DISABLED";
+      const errMsg = err instanceof Error ? err.message : String(err);
+      return {
+        output: `[MOCK] No OpenCode server available. GoatCode plugin would be: ${label}. Task: ${prompt}`,
+        tokenUsage: { total: 0, prompt: 0, completion: 0 },
+        cost: 0,
+        metadata: {
+          mock: true,
+          error: errMsg,
+          pluginEnabled: cfg.enablePlugin,
+          disabledHooks: cfg.disabledHooks,
+          disabledTools: cfg.disabledTools,
+          model: cfg.model,
+          sessionDuration: Date.now() - startTime,
+        },
+      };
+    }
+  }
+}

package/eval/spike/config.yaml

@@ -0,0 +1,13 @@
+description: "GoatCode promptfoo feasibility spike"
+
+providers:
+  - id: "file://provider.ts"
+    label: "spike-provider"
+
+prompts:
+  - "Hello, GoatCode!"
+
+tests:
+  - assert:
+      - type: contains
+        value: "Spike response"

package/eval/spike/provider.ts

@@ -0,0 +1,15 @@
+// IMPORTANT: No bun:* imports — this file runs under Node.js via npx promptfoo eval
+
+export default class SpikeProvider {
+  id() {
+    return "goatcode-spike-provider";
+  }
+
+  async callApi(prompt: string) {
+    return {
+      output: `Spike response for: ${prompt}`,
+      tokenUsage: { total: 10, prompt: 5, completion: 5 },
+      cost: 0,
+    };
+  }
+}

package/npm-reserve/README.md

@@ -0,0 +1,19 @@
+# goatcode
+
+Enterprise-grade OpenCode plugin with micro-plugin architecture and multi-agent orchestration.
+
+> **This package is under active development.** This is a name reservation. Full implementation coming soon.
+
+## What is goatcode?
+
+goatcode is a professionally engineered OpenCode plugin that provides:
+
+- Multi-agent orchestration with 11 specialized agents
+- Micro-plugin architecture where every feature is independently composable
+- TypeScript-native configuration (`goatcode.config.ts`)
+- Zero built-in external service connections (enterprise-secure by default)
+- Background agent parallelism with concurrency management
+
+## Links
+
+- [GitHub](https://github.com/ryanskidmore/goatcode)

package/npm-reserve/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "goatcode-sh",
+  "version": "0.0.1",
+  "description": "Enterprise-grade OpenCode plugin with micro-plugin architecture and multi-agent orchestration",
+  "main": "index.js",
+  "type": "module",
+  "keywords": [
+    "opencode",
+    "plugin",
+    "agents",
+    "orchestration",
+    "ai"
+  ],
+  "author": "ryanskidmore",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/ryanskidmore/goatcode.git"
+  },
+  "homepage": "https://github.com/ryanskidmore/goatcode#readme",
+  "bugs": {
+    "url": "https://github.com/ryanskidmore/goatcode/issues"
+  }
+}

package/opencode.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://opencode.ai/config.json",
+  "watcher": {
+    "ignore": ["deps/**"]
+  }
+}