@slowdini/slow-powers-opencode 0.1.3 → 0.1.5

package/skills/evaluating-skills/runner/plugin-shadow.test.ts

@@ -0,0 +1,228 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { homedir, tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import {
+  detectPluginShadows,
+  formatShadowBanner,
+  resolveConfigDir,
+  shadowValidityWarnings,
+} from "./plugin-shadow";
+
+const ROOT = join(tmpdir(), `slow-powers-plugin-shadow-test-${process.pid}`);
+
+beforeAll(() => mkdirSync(ROOT, { recursive: true }));
+afterAll(() => rmSync(ROOT, { recursive: true, force: true }));
+
+let seq = 0;
+function freshDirs() {
+  seq += 1;
+  const base = join(ROOT, `case-${seq}`);
+  const configDir = join(base, "config");
+  const cwd = join(base, "cwd");
+  mkdirSync(configDir, { recursive: true });
+  mkdirSync(cwd, { recursive: true });
+  return { configDir, cwd };
+}
+
+function writeFile(path: string, body: string) {
+  mkdirSync(dirname(path), { recursive: true });
+  writeFileSync(path, body);
+}
+
+/** Lay down a plugin's skill folders and return its install path. */
+function installPlugin(
+  configDir: string,
+  key: string,
+  skillNames: string[],
+): string {
+  const installPath = join(
+    configDir,
+    "plugins",
+    "cache",
+    key.replace("@", "__"),
+  );
+  for (const name of skillNames) {
+    writeFile(
+      join(installPath, "skills", name, "SKILL.md"),
+      `---\nname: ${name}\ndescription: x\n---\n`,
+    );
+  }
+  return installPath;
+}
+
+function writeInstalledManifest(
+  configDir: string,
+  entries: Record<string, string>,
+) {
+  const plugins: Record<string, Array<{ installPath: string }>> = {};
+  for (const [key, installPath] of Object.entries(entries))
+    plugins[key] = [{ installPath }];
+  writeFile(
+    join(configDir, "plugins", "installed_plugins.json"),
+    `${JSON.stringify({ version: 2, plugins }, null, 2)}\n`,
+  );
+}
+
+function writeSettings(path: string, enabledPlugins: Record<string, boolean>) {
+  writeFile(path, `${JSON.stringify({ enabledPlugins }, null, 2)}\n`);
+}
+
+describe("resolveConfigDir", () => {
+  test("honors CLAUDE_CONFIG_DIR", () => {
+    expect(
+      resolveConfigDir({
+        CLAUDE_CONFIG_DIR: "/custom/cfg",
+      } as NodeJS.ProcessEnv),
+    ).toBe("/custom/cfg");
+  });
+
+  test("defaults to ~/.claude when unset", () => {
+    expect(resolveConfigDir({} as NodeJS.ProcessEnv)).toBe(
+      join(homedir(), ".claude"),
+    );
+  });
+});
+
+describe("detectPluginShadows", () => {
+  test("flags a staged skill also provided by an enabled plugin", () => {
+    const { configDir, cwd } = freshDirs();
+    const ip = installPlugin(configDir, "slow-powers@slowdini", [
+      "verification-before-completion",
+      "writing-skills",
+    ]);
+    writeInstalledManifest(configDir, { "slow-powers@slowdini": ip });
+    writeSettings(join(configDir, "settings.json"), {
+      "slow-powers@slowdini": true,
+    });
+
+    const report = detectPluginShadows({
+      configDir,
+      cwd,
+      stagedSkillNames: ["verification-before-completion"],
+    });
+    expect(report.shadowed).toHaveLength(1);
+    expect(report.shadowed[0]).toMatchObject({
+      kind: "plugin",
+      plugin: "slow-powers@slowdini",
+      skill_name: "verification-before-completion",
+    });
+  });
+
+  test("does not flag a plugin disabled in user settings", () => {
+    const { configDir, cwd } = freshDirs();
+    const ip = installPlugin(configDir, "slow-powers@slowdini", [
+      "verification-before-completion",
+    ]);
+    writeInstalledManifest(configDir, { "slow-powers@slowdini": ip });
+    writeSettings(join(configDir, "settings.json"), {
+      "slow-powers@slowdini": false,
+    });
+
+    const report = detectPluginShadows({
+      configDir,
+      cwd,
+      stagedSkillNames: ["verification-before-completion"],
+    });
+    expect(report.shadowed).toHaveLength(0);
+  });
+
+  test("project settings disabling a user-enabled plugin suppresses the shadow", () => {
+    const { configDir, cwd } = freshDirs();
+    const ip = installPlugin(configDir, "slow-powers@slowdini", [
+      "verification-before-completion",
+    ]);
+    writeInstalledManifest(configDir, { "slow-powers@slowdini": ip });
+    writeSettings(join(configDir, "settings.json"), {
+      "slow-powers@slowdini": true,
+    });
+    // Project scope (cwd/.claude/settings.json) outranks user scope.
+    writeSettings(join(cwd, ".claude", "settings.json"), {
+      "slow-powers@slowdini": false,
+    });
+
+    const report = detectPluginShadows({
+      configDir,
+      cwd,
+      stagedSkillNames: ["verification-before-completion"],
+    });
+    expect(report.shadowed).toHaveLength(0);
+  });
+
+  test("flags a staged skill also present in the global skills dir", () => {
+    const { configDir, cwd } = freshDirs();
+    writeFile(
+      join(configDir, "skills", "my-skill", "SKILL.md"),
+      "---\nname: my-skill\n---\n",
+    );
+
+    const report = detectPluginShadows({
+      configDir,
+      cwd,
+      stagedSkillNames: ["my-skill"],
+    });
+    expect(report.shadowed).toHaveLength(1);
+    expect(report.shadowed[0]).toMatchObject({
+      kind: "global-skill",
+      skill_name: "my-skill",
+    });
+  });
+
+  test("no shadow when staged names match nothing in the environment", () => {
+    const { configDir, cwd } = freshDirs();
+    const ip = installPlugin(configDir, "p@m", ["other"]);
+    writeInstalledManifest(configDir, { "p@m": ip });
+    writeSettings(join(configDir, "settings.json"), { "p@m": true });
+
+    const report = detectPluginShadows({
+      configDir,
+      cwd,
+      stagedSkillNames: ["mine"],
+    });
+    expect(report.shadowed).toHaveLength(0);
+  });
+
+  test("is graceful when the config dir has no plugins or skills", () => {
+    const { configDir, cwd } = freshDirs();
+    const report = detectPluginShadows({
+      configDir,
+      cwd,
+      stagedSkillNames: ["x"],
+    });
+    expect(report.shadowed).toHaveLength(0);
+    expect(report.config_dir).toBe(configDir);
+  });
+});
+
+describe("warning formatting", () => {
+  const report = {
+    config_dir: "/x",
+    shadowed: [
+      {
+        kind: "plugin" as const,
+        plugin: "slow-powers@slowdini",
+        skill_name: "verification-before-completion",
+        path: "/p",
+      },
+    ],
+  };
+
+  test("shadowValidityWarnings names the skill, the plugin, and the contamination", () => {
+    const warnings = shadowValidityWarnings(report);
+    expect(warnings).toHaveLength(1);
+    expect(warnings[0]).toContain("verification-before-completion");
+    expect(warnings[0]).toContain("slow-powers@slowdini");
+    expect(warnings[0]).toMatch(/contaminat/i);
+  });
+
+  test("formatShadowBanner is empty when nothing is shadowed", () => {
+    expect(formatShadowBanner({ config_dir: "/x", shadowed: [] })).toBe("");
+  });
+
+  test("formatShadowBanner lists shadowed skills and points at isolation docs", () => {
+    const banner = formatShadowBanner(report);
+    expect(banner).toContain("verification-before-completion");
+    expect(banner).toContain("slow-powers@slowdini");
+    expect(banner).toMatch(/isolat/i);
+  });
+});

package/skills/evaluating-skills/runner/plugin-shadow.ts

@@ -0,0 +1,201 @@
+// Plugin-shadow detector (Claude Code). The runner stages eval skills into the
+// project-local `.claude/skills/` dir, but eval subagents are dispatched via the
+// Task tool and run in-process — so they ALSO inherit whatever skills the
+// orchestrator session loaded from installed plugins and the global skills dir.
+// When a staged skill name collides with one of those, both copies are
+// discoverable: the with/without comparison is contaminated and the control arm
+// is not truly skill-absent.
+//
+// The runner cannot unload a plugin from a running session (plugins load at
+// session start), so this module only *detects and reports* the overlap. It
+// reads declared settings as a best-effort proxy for what the session loaded —
+// it can't observe the live-loaded set, so a session that changed settings
+// without restarting may differ. Isolation itself is a launch-time concern; see
+// harness-details/claude.md → "Isolating from installed plugins".
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+export type ShadowSource =
+  | { kind: "plugin"; plugin: string; skill_name: string; path: string }
+  | { kind: "global-skill"; skill_name: string; path: string };
+
+export type PluginShadowReport = {
+  config_dir: string;
+  shadowed: ShadowSource[];
+};
+
+const ISOLATION_DOC =
+  'harness-details/claude.md → "Isolating from installed plugins"';
+
+/** The Claude Code config dir: `$CLAUDE_CONFIG_DIR` if set, else `~/.claude`. */
+export function resolveConfigDir(env: NodeJS.ProcessEnv = process.env): string {
+  const override = env.CLAUDE_CONFIG_DIR;
+  return override?.trim() ? override : join(homedir(), ".claude");
+}
+
+function readJsonSafe<T>(path: string): T | null {
+  if (!existsSync(path)) return null;
+  try {
+    return JSON.parse(readFileSync(path, "utf8")) as T;
+  } catch {
+    return null;
+  }
+}
+
+type Settings = { enabledPlugins?: Record<string, boolean> };
+
+/**
+ * Effective `enabledPlugins` map, honoring Claude Code's settings precedence
+ * (local > project > user). User scope lives under the config dir; project and
+ * local scope live under `<cwd>/.claude/`. Later sources override earlier keys,
+ * so a project-scope `false` correctly masks a user-scope `true`.
+ */
+export function resolveEnabledPlugins(opts: {
+  configDir: string;
+  cwd: string;
+}): Record<string, boolean> {
+  const sources = [
+    join(opts.configDir, "settings.json"),
+    join(opts.cwd, ".claude", "settings.json"),
+    join(opts.cwd, ".claude", "settings.local.json"),
+  ];
+  let merged: Record<string, boolean> = {};
+  for (const path of sources) {
+    const s = readJsonSafe<Settings>(path);
+    if (s?.enabledPlugins) merged = { ...merged, ...s.enabledPlugins };
+  }
+  return merged;
+}
+
+/** Names of skill folders (those holding a `SKILL.md`) directly under `dir`. */
+function skillFolderNames(dir: string): Array<{ name: string; path: string }> {
+  if (!existsSync(dir)) return [];
+  let entries: string[];
+  try {
+    entries = readdirSync(dir);
+  } catch {
+    return [];
+  }
+  const out: Array<{ name: string; path: string }> = [];
+  for (const name of entries) {
+    const skillDir = join(dir, name);
+    try {
+      if (!statSync(skillDir).isDirectory()) continue;
+    } catch {
+      continue;
+    }
+    if (existsSync(join(skillDir, "SKILL.md")))
+      out.push({ name, path: skillDir });
+  }
+  return out;
+}
+
+type InstalledPlugins = {
+  plugins?: Record<string, Array<{ installPath?: string }>>;
+};
+
+/** Skills exposed by currently-enabled installed plugins. */
+export function listEnabledPluginSkills(opts: {
+  configDir: string;
+  enabled: Record<string, boolean>;
+}): Array<{ plugin: string; skill_name: string; path: string }> {
+  const manifest = readJsonSafe<InstalledPlugins>(
+    join(opts.configDir, "plugins", "installed_plugins.json"),
+  );
+  const out: Array<{ plugin: string; skill_name: string; path: string }> = [];
+  if (!manifest?.plugins) return out;
+  for (const [key, installs] of Object.entries(manifest.plugins)) {
+    if (opts.enabled[key] !== true) continue; // only enabled plugins shadow
+    for (const inst of installs ?? []) {
+      if (!inst.installPath) continue;
+      for (const s of skillFolderNames(join(inst.installPath, "skills")))
+        out.push({ plugin: key, skill_name: s.name, path: s.path });
+    }
+  }
+  return out;
+}
+
+/** Skills under the global skills dir (`<configDir>/skills`). */
+export function listGlobalSkills(
+  configDir: string,
+): Array<{ skill_name: string; path: string }> {
+  return skillFolderNames(join(configDir, "skills")).map((s) => ({
+    skill_name: s.name,
+    path: s.path,
+  }));
+}
+
+/**
+ * Which of `stagedSkillNames` are also discoverable from enabled plugins or the
+ * global skills dir. Matches on the skill folder name (exact).
+ */
+export function detectPluginShadows(opts: {
+  configDir: string;
+  cwd: string;
+  stagedSkillNames: string[];
+}): PluginShadowReport {
+  const staged = new Set(opts.stagedSkillNames);
+  const enabled = resolveEnabledPlugins({
+    configDir: opts.configDir,
+    cwd: opts.cwd,
+  });
+  const shadowed: ShadowSource[] = [];
+
+  for (const s of listEnabledPluginSkills({
+    configDir: opts.configDir,
+    enabled,
+  }))
+    if (staged.has(s.skill_name))
+      shadowed.push({
+        kind: "plugin",
+        plugin: s.plugin,
+        skill_name: s.skill_name,
+        path: s.path,
+      });
+
+  for (const s of listGlobalSkills(opts.configDir))
+    if (staged.has(s.skill_name))
+      shadowed.push({
+        kind: "global-skill",
+        skill_name: s.skill_name,
+        path: s.path,
+      });
+
+  return { config_dir: opts.configDir, shadowed };
+}
+
+function sourceLabel(s: ShadowSource): string {
+  return s.kind === "plugin"
+    ? `enabled plugin '${s.plugin}'`
+    : "the global skills dir";
+}
+
+/** One `validity_warnings` line per shadowed skill (for benchmark.json). */
+export function shadowValidityWarnings(report: PluginShadowReport): string[] {
+  return report.shadowed.map(
+    (s) =>
+      `staged skill '${s.skill_name}' is also provided by ${sourceLabel(s)} — ` +
+      `eval subagents could discover both copies, so with/without results may be ` +
+      `contaminated. Re-run from an isolated session (see ${ISOLATION_DOC}).`,
+  );
+}
+
+/** Build-time banner for the runner. Empty string when nothing is shadowed. */
+export function formatShadowBanner(report: PluginShadowReport): string {
+  if (report.shadowed.length === 0) return "";
+  const lines = report.shadowed.map(
+    (s) => `  • ${s.skill_name} — ${sourceLabel(s)}`,
+  );
+  return [
+    "",
+    "⚠ Plugin-shadow warning: skills staged for this eval are ALSO discoverable",
+    "  from your live environment:",
+    ...lines,
+    "  Eval subagents (dispatched via the Task tool) inherit this session's plugins,",
+    "  so both the staged copy and the installed copy are discoverable — the",
+    "  with/without comparison may be contaminated and the control arm is not truly",
+    "  skill-absent. The runner cannot unload a plugin from a running session.",
+    `  Re-run from an isolated session — see ${ISOLATION_DOC}.`,
+  ].join("\n");
+}

package/skills/evaluating-skills/runner/profiles/claude-code/plan-mode.md

@@ -0,0 +1,11 @@
+Plan mode is active. The user wants to review an approach before any code is written, so you must NOT execute yet: do not make any edits, do not run any non-read-only tool, and do not change configs or system state. The only file you may write is the plan file. This constraint supersedes any other instruction you have received this session.
+
+You are operating inside the harness's plan-mode workflow — a fixed, multi-phase procedure. Work through the phases in order:
+
+1. **Understand.** Read the relevant code and gather context with read-only tools until you can describe the change concretely. Reuse what already exists rather than proposing new code.
+2. **Design.** Decide the implementation approach and the trade-offs.
+3. **Review.** Re-check the design against the user's request and resolve open questions with the user before finalizing.
+4. **Write the plan.** Build the plan up incrementally in the plan file — this is the one file you are permitted to write. Name the files to change and how to verify the result.
+5. **Hand off.** Call ExitPlanMode to submit the plan for the user's approval.
+
+Terminal rail: your turn must end in exactly one of two ways — by asking the user a question, or by calling ExitPlanMode to present the finished plan. Do not stop for any other reason and do not begin implementation until the user has approved the plan. The plan-mode workflow already governs how you research, design, and present the work; stay on this rail through to ExitPlanMode.