@slowdini/slow-powers-opencode 0.1.0

package/skills/evaluating-skills/runner/sandbox-policy.ts

@@ -0,0 +1,74 @@
+import { isAbsolute, resolve, sep } from "node:path";
+
+/** Tools that mutate the filesystem and carry a target path argument. */
+export const WRITE_TOOLS = new Set([
+  "Write",
+  "Edit",
+  "MultiEdit",
+  "NotebookEdit",
+]);
+
+/**
+ * Bash command patterns that mutate state outside an eval's sandbox. Heuristics
+ * — Bash is too flexible to parse exactly. `detect-stray-writes` surfaces these
+ * as warnings; the opt-in guard denies them. Each is meaningful only when the
+ * command does not reference an allowed root (see `classifyBash`).
+ */
+export const BASH_MUTATION_PATTERNS: Array<{ re: RegExp; reason: string }> = [
+  {
+    re: /\b(npm|pnpm|yarn|bun)\s+(install|add|ci|i)\b/,
+    reason: "package install/add",
+  },
+  { re: /\bpip3?\s+install\b/, reason: "pip install" },
+  { re: /\bsed\s+-i\b/, reason: "in-place file edit (sed -i)" },
+  {
+    re: /\bgit\s+(commit|add|push|checkout|reset|restore|merge|rebase)\b/,
+    reason: "git mutation",
+  },
+  { re: /(^|\s)(>>?|tee)\s/, reason: "output redirection to a file" },
+];
+
+/** Pull the target path from a write tool's arguments. */
+export function pathArg(args: unknown): string | undefined {
+  if (!args || typeof args !== "object") return undefined;
+  const a = args as Record<string, unknown>;
+  const p = a.file_path ?? a.notebook_path ?? a.path;
+  return typeof p === "string" ? p : undefined;
+}
+
+/** True when `target` resolves to `dir` or a descendant of it. */
+export function isUnder(
+  target: string,
+  dir: string,
+  repoRoot: string,
+): boolean {
+  const base = resolve(dir);
+  const abs = isAbsolute(target) ? resolve(target) : resolve(repoRoot, target);
+  return abs === base || abs.startsWith(base + sep);
+}
+
+/** True when `target` is under any of `dirs`. */
+export function isUnderAny(
+  target: string,
+  dirs: string[],
+  repoRoot: string,
+): boolean {
+  return dirs.some((d) => isUnder(target, d, repoRoot));
+}
+
+/**
+ * If a Bash command matches a mutation pattern and is not scoped to one of
+ * `allowedRoots`, return the human reason; otherwise null. A command is treated
+ * as scoped when it textually references an allowed root.
+ */
+export function classifyBash(
+  command: string,
+  allowedRoots: string[],
+): string | null {
+  if (!command) return null;
+  if (allowedRoots.some((r) => command.includes(r))) return null;
+  for (const { re, reason } of BASH_MUTATION_PATTERNS) {
+    if (re.test(command)) return reason;
+  }
+  return null;
+}

package/skills/evaluating-skills/runner/types.ts

@@ -0,0 +1,104 @@
+export type AssertionTranscriptCheck = {
+  id: string;
+  type: "transcript_check";
+  check: string;
+  pattern?: string;
+  must_precede?: "completion_claim" | "any";
+};
+
+export type AssertionLLMJudge = {
+  id: string;
+  type: "llm_judge";
+  rubric: string;
+  model?: string;
+};
+
+export type Assertion = AssertionTranscriptCheck | AssertionLLMJudge;
+
+export type Eval = {
+  id: string;
+  prompt: string;
+  expected_output: string;
+  files?: string[];
+  assertions?: Assertion[];
+  /**
+   * Whether the skill-under-test is expected to fire on this eval. Defaults to
+   * true. Set to false for negative evals where correct behavior is NOT
+   * invoking the skill (e.g. an over-trigger guard). Negative evals are
+   * excluded from the skill-invocation rate and its validity warning.
+   */
+  skill_should_trigger?: boolean;
+};
+
+export type EvalsConfig = {
+  skill_name: string;
+  evals: Eval[];
+};
+
+export type ConditionEntry = {
+  name: string;
+  skill_path: string | null;
+  staged_skill_slug?: string | null;
+};
+
+export type ConditionsRecord = {
+  mode: "new-skill" | "revision";
+  baseline?: string;
+  conditions: ConditionEntry[];
+  timestamp: string;
+  harness?: string;
+  /** Per-run nonce; namespaces dispatch descriptions so transcripts can't
+   * collide across iterations sharing one parent session's subagents dir. */
+  run_nonce?: string;
+};
+
+export type ToolInvocation = {
+  name: string;
+  args?: unknown;
+  result?: unknown;
+  ordinal: number;
+};
+
+export type RunRecord = {
+  eval_id: string;
+  condition: string;
+  skill_path: string | null;
+  prompt: string;
+  files: string[];
+  final_message: string;
+  tool_invocations: ToolInvocation[];
+  total_tokens: number | null;
+  duration_ms: number | null;
+};
+
+export type AssertionResult = {
+  id: string;
+  passed: boolean;
+  evidence: string;
+  confidence?: number;
+  grader?: "transcript_check" | "llm_judge";
+};
+
+export type GradingResult = {
+  assertion_results: AssertionResult[];
+  meta_results?: AssertionResult[];
+  summary: {
+    passed: number;
+    failed: number;
+    total: number;
+    pass_rate: number;
+  };
+  meta_summary?: {
+    passed: number;
+    failed: number;
+    total: number;
+    skill_invoked: boolean | null;
+  };
+};
+
+export const SKILL_INVOKED_META_ID = "__skill_invoked";
+
+export type TimingRecord = {
+  total_tokens?: number | null;
+  duration_ms?: number | null;
+};

package/skills/evaluating-skills/runner/validate-all.ts

@@ -0,0 +1,54 @@
+#!/usr/bin/env bun
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { join, resolve } from "node:path";
+import { validateEvalsConfig } from "./validate";
+
+function flag(argv: string[], name: string): string | undefined {
+  const i = argv.indexOf(`--${name}`);
+  if (i === -1) return undefined;
+  return argv[i + 1];
+}
+
+const skillDirRaw = flag(Bun.argv.slice(2), "skill-dir");
+if (!skillDirRaw) {
+  console.error("missing required flag --skill-dir <path>");
+  process.exit(1);
+}
+const SKILLS_DIR = resolve(skillDirRaw);
+
+if (!existsSync(SKILLS_DIR)) {
+  console.error(`skills dir not found: ${SKILLS_DIR}`);
+  process.exit(1);
+}
+
+const skills = readdirSync(SKILLS_DIR).filter((d) => {
+  const path = join(SKILLS_DIR, d);
+  return statSync(path).isDirectory();
+});
+
+let validated = 0;
+let failed = 0;
+const errors: string[] = [];
+
+for (const skill of skills) {
+  const evalsPath = join(SKILLS_DIR, skill, "evals", "evals.json");
+  if (!existsSync(evalsPath)) continue;
+
+  try {
+    const raw = JSON.parse(readFileSync(evalsPath, "utf8"));
+    validateEvalsConfig(raw, evalsPath);
+    console.log(`✓ ${skill}/evals/evals.json`);
+    validated++;
+  } catch (err) {
+    console.error(`✗ ${skill}/evals/evals.json: ${(err as Error).message}`);
+    errors.push(`${skill}: ${(err as Error).message}`);
+    failed++;
+  }
+}
+
+console.log(`\nValidated ${validated} evals.json file(s); ${failed} failed.`);
+if (failed > 0) {
+  console.error("\nFailures:");
+  for (const e of errors) console.error(`  - ${e}`);
+  process.exit(1);
+}

package/skills/evaluating-skills/runner/validate-schema.test.ts

@@ -0,0 +1,99 @@
+import { describe, expect, test } from "bun:test";
+import { validateAgainstSchema } from "./validate-schema";
+
+const validRunRecord = {
+  eval_id: "e1",
+  condition: "with_skill",
+  skill_path: null,
+  prompt: "do the thing",
+  files: [],
+  final_message: "done",
+  tool_invocations: [],
+  total_tokens: 100,
+  duration_ms: 1000,
+};
+
+describe("validateAgainstSchema", () => {
+  test("returns the data when it matches the run-record schema", () => {
+    const result = validateAgainstSchema(
+      "run-record",
+      validRunRecord,
+      "run.json",
+    );
+    expect(result).toEqual(validRunRecord);
+  });
+
+  test("accepts an empty tool_invocations array (written pre-fill)", () => {
+    expect(() =>
+      validateAgainstSchema(
+        "run-record",
+        { ...validRunRecord, tool_invocations: [] },
+        "run.json",
+      ),
+    ).not.toThrow();
+  });
+
+  test("accepts skill_path: null on the without_skill arm", () => {
+    expect(() =>
+      validateAgainstSchema(
+        "run-record",
+        { ...validRunRecord, skill_path: null },
+        "run.json",
+      ),
+    ).not.toThrow();
+  });
+
+  test("throws a source-prefixed error when a required field is missing", () => {
+    const { eval_id, ...missing } = validRunRecord;
+    expect(() =>
+      validateAgainstSchema("run-record", missing, "/tmp/run.json"),
+    ).toThrow(/\/tmp\/run\.json/);
+  });
+
+  test("requires skill_path and files (type is the contract)", () => {
+    const { skill_path, ...noSkillPath } = validRunRecord;
+    expect(() =>
+      validateAgainstSchema("run-record", noSkillPath, "run.json"),
+    ).toThrow(/skill_path/);
+
+    const { files, ...noFiles } = validRunRecord;
+    expect(() =>
+      validateAgainstSchema("run-record", noFiles, "run.json"),
+    ).toThrow(/files/);
+  });
+
+  test("rejects a run record with an unknown extra property", () => {
+    expect(() =>
+      validateAgainstSchema(
+        "run-record",
+        { ...validRunRecord, surprise: true },
+        "run.json",
+      ),
+    ).toThrow();
+  });
+
+  test("validates a tool_invocation's ordinal must be an integer", () => {
+    expect(() =>
+      validateAgainstSchema(
+        "run-record",
+        {
+          ...validRunRecord,
+          tool_invocations: [{ name: "Bash", ordinal: "zero" }],
+        },
+        "run.json",
+      ),
+    ).toThrow();
+  });
+
+  test("compiles and validates the grading schema too", () => {
+    const validGrading = {
+      assertion_results: [
+        { id: "a1", passed: true, evidence: "quote", grader: "llm_judge" },
+      ],
+      summary: { passed: 1, failed: 0, total: 1, pass_rate: 1 },
+    };
+    expect(() =>
+      validateAgainstSchema("grading", validGrading, "grading.json"),
+    ).not.toThrow();
+  });
+});

package/skills/evaluating-skills/runner/validate-schema.ts

@@ -0,0 +1,51 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { Ajv, type ValidateFunction } from "ajv";
+
+/**
+ * The four portable artifact schemas live in `../schema/<name>.schema.json` and
+ * are the single source of truth for each artifact's shape. This helper compiles
+ * them with ajv and enforces them at runtime, so the schema files are an enforced
+ * contract rather than documentation a hand-rolled validator can drift from.
+ */
+export type SchemaName = "run-record" | "evals" | "grading" | "stray-writes";
+
+const SCHEMA_DIR = join(import.meta.dir, "..", "schema");
+
+// strict: false — the schemas are plain draft-07; we don't want ajv's strict
+// metaschema checks to reject otherwise-valid schemas over stylistic keywords.
+const ajv = new Ajv({ allErrors: true, strict: false });
+const validators = new Map<SchemaName, ValidateFunction>();
+
+function getValidator(name: SchemaName): ValidateFunction {
+  let validate = validators.get(name);
+  if (!validate) {
+    const schema = JSON.parse(
+      readFileSync(join(SCHEMA_DIR, `${name}.schema.json`), "utf8"),
+    );
+    validate = ajv.compile(schema);
+    validators.set(name, validate);
+  }
+  return validate;
+}
+
+/**
+ * Validate `data` against the named schema. Returns the data typed as `T` on
+ * success; throws a `source`-prefixed Error listing every failure on mismatch.
+ */
+export function validateAgainstSchema<T>(
+  name: SchemaName,
+  data: unknown,
+  source: string,
+): T {
+  const validate = getValidator(name);
+  if (!validate(data)) {
+    const details = (validate.errors ?? [])
+      .map((e) => `  ${e.instancePath || "/"} ${e.message}`)
+      .join("\n");
+    throw new Error(
+      `${source}: does not match the ${name} schema:\n${details}`,
+    );
+  }
+  return data as T;
+}

package/skills/evaluating-skills/runner/validate.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, test } from "bun:test";
+import { validateEvalsConfig } from "./validate";
+
+const base = {
+  skill_name: "demo",
+  evals: [
+    {
+      id: "e1",
+      prompt: "do the thing",
+      expected_output: "the thing is done",
+    },
+  ],
+};
+
+describe("validateEvalsConfig skill_should_trigger", () => {
+  test("accepts a boolean skill_should_trigger", () => {
+    const cfg = {
+      ...base,
+      evals: [{ ...base.evals[0], skill_should_trigger: false }],
+    };
+    expect(() => validateEvalsConfig(cfg, "test")).not.toThrow();
+  });
+
+  test("accepts evals with no skill_should_trigger (defaults to true)", () => {
+    expect(() => validateEvalsConfig(base, "test")).not.toThrow();
+  });
+
+  test("rejects a non-boolean skill_should_trigger", () => {
+    const cfg = {
+      ...base,
+      evals: [{ ...base.evals[0], skill_should_trigger: "false" }],
+    };
+    expect(() => validateEvalsConfig(cfg, "test")).toThrow(
+      /skill_should_trigger/,
+    );
+  });
+});
+
+describe("validateEvalsConfig structural + duplicate-id", () => {
+  test("rejects a non-kebab-case id", () => {
+    const cfg = { ...base, evals: [{ ...base.evals[0], id: "Not Kebab" }] };
+    expect(() => validateEvalsConfig(cfg, "test")).toThrow();
+  });
+
+  test("rejects duplicate eval ids (not expressible in JSON Schema)", () => {
+    const cfg = {
+      ...base,
+      evals: [base.evals[0], { ...base.evals[0] }],
+    };
+    expect(() => validateEvalsConfig(cfg, "test")).toThrow(/duplicate/);
+  });
+
+  test("rejects an empty evals array", () => {
+    expect(() => validateEvalsConfig({ ...base, evals: [] }, "test")).toThrow();
+  });
+});

package/skills/evaluating-skills/runner/validate.ts

@@ -0,0 +1,21 @@
+import type { EvalsConfig } from "./types";
+import { validateAgainstSchema } from "./validate-schema";
+
+export function validateEvalsConfig(
+  config: unknown,
+  source: string,
+): EvalsConfig {
+  // Structural validation against the single source of truth.
+  const validated = validateAgainstSchema<EvalsConfig>("evals", config, source);
+
+  // Supplemental check: JSON Schema (draft-07) can't enforce uniqueness by a
+  // sub-field, so the duplicate-id guard stays hand-rolled.
+  const seenIds = new Set<string>();
+  for (const [i, ev] of validated.evals.entries()) {
+    if (seenIds.has(ev.id))
+      throw new Error(`${source}: evals[${i}].id duplicate: ${ev.id}`);
+    seenIds.add(ev.id);
+  }
+
+  return validated;
+}

package/skills/evaluating-skills/schema/evals.schema.json

@@ -0,0 +1,105 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://slow-powers.dev/schemas/evals.schema.json",
+  "title": "Skill Evaluation Definition",
+  "description": "Defines a set of test cases for evaluating a skill. Lives at <skill>/evals/evals.json.",
+  "type": "object",
+  "required": ["skill_name", "evals"],
+  "additionalProperties": false,
+  "properties": {
+    "skill_name": {
+      "type": "string",
+      "description": "Name of the skill being evaluated. Should match the skill directory name."
+    },
+    "evals": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "$ref": "#/definitions/eval" }
+    }
+  },
+  "definitions": {
+    "eval": {
+      "type": "object",
+      "required": ["id", "prompt", "expected_output"],
+      "additionalProperties": false,
+      "properties": {
+        "id": {
+          "type": "string",
+          "pattern": "^[a-z0-9][a-z0-9-]*$",
+          "description": "Stable kebab-case identifier. Used as directory name in the workspace tree."
+        },
+        "prompt": {
+          "type": "string",
+          "minLength": 1,
+          "description": "The user-facing message the subagent receives. Should read like a realistic user request."
+        },
+        "expected_output": {
+          "type": "string",
+          "minLength": 1,
+          "description": "Human-readable description of what a successful response looks like."
+        },
+        "files": {
+          "type": "array",
+          "items": { "type": "string" },
+          "description": "Fixture file paths relative to the skill's evals/ directory. Copied into the subagent's input directory before dispatch."
+        },
+        "skill_should_trigger": {
+          "type": "boolean",
+          "default": true,
+          "description": "Whether the skill-under-test is expected to fire on this eval. Defaults to true. Set false for negative evals where correct behavior is NOT invoking the skill (e.g. an over-trigger guard); such evals are excluded from the skill-invocation rate and its validity warning."
+        },
+        "assertions": {
+          "type": "array",
+          "items": { "$ref": "#/definitions/assertion" },
+          "description": "Pass/fail criteria, added after iteration 1 when you know what outputs look like."
+        }
+      }
+    },
+    "assertion": {
+      "oneOf": [
+        { "$ref": "#/definitions/transcriptCheck" },
+        { "$ref": "#/definitions/llmJudge" }
+      ]
+    },
+    "transcriptCheck": {
+      "type": "object",
+      "required": ["id", "type", "check"],
+      "additionalProperties": false,
+      "properties": {
+        "id": { "type": "string" },
+        "type": { "const": "transcript_check" },
+        "check": {
+          "type": "string",
+          "description": "Name of a transcript-check kind handled by the runner's grader (runner/grade.ts), e.g. tool_invocation_matches."
+        },
+        "pattern": {
+          "type": "string",
+          "description": "Regex (or substring) the check uses to match tool invocations."
+        },
+        "must_precede": {
+          "type": "string",
+          "enum": ["completion_claim", "any"],
+          "description": "Where in the run the matched invocation must occur. 'completion_claim' = before the final message. 'any' = anywhere in the run."
+        }
+      }
+    },
+    "llmJudge": {
+      "type": "object",
+      "required": ["id", "type", "rubric"],
+      "additionalProperties": false,
+      "properties": {
+        "id": { "type": "string" },
+        "type": { "const": "llm_judge" },
+        "rubric": {
+          "type": "string",
+          "minLength": 1,
+          "description": "The question the judge model answers. Should be answerable with PASS/FAIL + evidence."
+        },
+        "model": {
+          "type": "string",
+          "description": "Optional override. Defaults to whatever the harness operator configures for judge dispatches."
+        }
+      }
+    }
+  }
+}

package/skills/evaluating-skills/schema/grading.schema.json

@@ -0,0 +1,84 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://slow-powers.dev/schemas/grading.schema.json",
+  "title": "Grading Result",
+  "description": "Output of grading one (eval, condition) pair. Lives at <workspace>/iteration-N/eval-<id>/<condition>/grading.json.",
+  "type": "object",
+  "required": ["assertion_results", "summary"],
+  "additionalProperties": false,
+  "properties": {
+    "assertion_results": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "required": ["id", "passed", "evidence"],
+        "additionalProperties": false,
+        "properties": {
+          "id": {
+            "type": "string",
+            "description": "Matches the assertion id in evals.json."
+          },
+          "passed": { "type": "boolean" },
+          "evidence": {
+            "type": "string",
+            "description": "Direct quote or specific reference from the run record. Vague summaries are not evidence."
+          },
+          "confidence": {
+            "type": "number",
+            "minimum": 0,
+            "maximum": 1,
+            "description": "Judge confidence. Low confidence (< 0.7) flags this result for human review. Always 1.0 for transcript_check results."
+          },
+          "grader": {
+            "type": "string",
+            "enum": ["transcript_check", "llm_judge"],
+            "description": "Which grader produced this result."
+          }
+        }
+      }
+    },
+    "summary": {
+      "type": "object",
+      "required": ["passed", "failed", "total", "pass_rate"],
+      "additionalProperties": false,
+      "properties": {
+        "passed": { "type": "integer", "minimum": 0 },
+        "failed": { "type": "integer", "minimum": 0 },
+        "total": { "type": "integer", "minimum": 0 },
+        "pass_rate": { "type": "number", "minimum": 0, "maximum": 1 }
+      }
+    },
+    "meta_results": {
+      "type": "array",
+      "description": "Framework-injected meta-assertions (e.g. skill-invocation check). Reserved id prefix: __ (double underscore). Tracked separately from substantive assertion_results so they do not pollute the skill effectiveness pass_rate.",
+      "items": {
+        "type": "object",
+        "required": ["id", "passed", "evidence"],
+        "additionalProperties": false,
+        "properties": {
+          "id": { "type": "string" },
+          "passed": { "type": "boolean" },
+          "evidence": { "type": "string" },
+          "confidence": { "type": "number", "minimum": 0, "maximum": 1 },
+          "grader": {
+            "type": "string",
+            "enum": ["transcript_check", "llm_judge"]
+          }
+        }
+      }
+    },
+    "meta_summary": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "passed": { "type": "integer", "minimum": 0 },
+        "failed": { "type": "integer", "minimum": 0 },
+        "total": { "type": "integer", "minimum": 0 },
+        "skill_invoked": {
+          "description": "True when the skill-invocation meta-check passed; false when the judge found no evidence the skill influenced behavior; null when no skill was loaded for this run.",
+          "type": ["boolean", "null"]
+        }
+      }
+    }
+  }
+}