@percepta/kaizen 0.6.0 → 0.7.0

package/dist/types.d.ts

@@ -1,38 +1,11 @@
+import { DatasetItemData, DatasetItemRendererActions, DatasetItemRendererProps, TraceData, TraceRendererActions, TraceRendererContext, TraceRendererProps } from "./shared/view-types.js";
+
 //#region src/types.d.ts
 interface KaizenConfig {
   customer: {
-    slug: string;
     name: string;
   };
-  langfuse?: {
-    host?: string;
-    publicKeyEnv?: string;
-    secretKeyEnv?: string;
-  };
-  studio?: {
-    port?: number;
-  };
-}
-/** Data shape passed to custom trace renderers. */
-interface TraceData {
-  id?: string;
-  name?: string;
-  tags?: string[];
-  timestamp?: string;
-  metadata?: unknown;
-  input?: unknown;
-  output?: unknown;
-}
-/** Props contract for custom trace renderer components. */
-interface TraceRendererProps {
-  trace: TraceData;
-  datasetItem?: {
-    id: string;
-    input?: unknown;
-    expectedOutput?: unknown;
-    metadata?: Record<string, unknown> | null;
-  } | null;
 }
 //#endregion
-export { KaizenConfig, TraceData, TraceRendererProps };
+export { type DatasetItemData, type DatasetItemRendererActions, type DatasetItemRendererProps, KaizenConfig, type TraceData, type TraceRendererActions, type TraceRendererContext, type TraceRendererProps };
 //# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","names":[],"sources":["../src/types.ts"],"mappings":";UAAiB,YAAA;EACf,QAAA;IACE,IAAA;IACA,IAAA;EAAA;EAEF,QAAA;IACE,IAAA;IACA,YAAA;IACA,YAAA;EAAA;EAEF,MAAA;IACE,IAAA;EAAA;AAAA;;UAKa,SAAA;EACf,EAAA;EACA,IAAA;EACA,IAAA;EACA,SAAA;EACA,QAAA;EACA,KAAA;EACA,MAAA;AAAA;;UAIe,kBAAA;EACf,KAAA,EAAO,SAAA;EACP,WAAA;IACE,EAAA;IACA,KAAA;IACA,cAAA;IACA,QAAA,GAAW,MAAA;EAAA;AAAA"}
+{"version":3,"file":"types.d.ts","names":[],"sources":["../src/types.ts"],"mappings":";;;UAAiB,YAAA;EACf,QAAA;IACE,IAAA;EAAA;AAAA"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@percepta/kaizen",
-  "version": "0.6.0",
+  "version": "0.7.0",
   "description": "Automated AI researcher that improves AI systems",
   "keywords": [
     "ai",
@@ -14,6 +14,7 @@
     "kaizen": "./bin/kaizen.js"
   },
   "files": [
+    "agent",
     "bin",
     "dashboard/.next/standalone",
     "dist",
@@ -71,8 +72,9 @@
     "typecheck:dashboard": "tsc -p dashboard --noEmit",
     "test": "vitest run",
     "dev": "tsx src/index.ts",
-    "dev:studio": "KAIZEN_WORKSPACE=examples/legacy-workspace next dev dashboard --webpack --port 6789",
-    "dev:next": "KAIZEN_WORKSPACE=examples/legacy-workspace next dev dashboard --webpack --port 6789",
+    "dev:studio": "KAIZEN_DEMO_MODE=1 KAIZEN_WORKSPACE=$PWD/examples/demo-workspace next dev dashboard --webpack --port 6789",
+    "dev:next": "KAIZEN_DEMO_MODE=1 KAIZEN_WORKSPACE=$PWD/examples/demo-workspace next dev dashboard --webpack --port 6789",
+    "seed:demo-data": "node scripts/seed-demo-traces.mjs --workspace $PWD/examples/demo-workspace",
     "kaizen": "tsx src/index.ts"
   }
 }

package/shared/view-types.d.ts

@@ -0,0 +1,69 @@
+/** Data shape passed to custom trace renderers. */
+export interface TraceData {
+  id?: string;
+  name?: string;
+  tags?: string[];
+  timestamp?: string;
+  metadata?: unknown;
+  input?: unknown;
+  output?: unknown;
+  [key: string]: unknown;
+}
+
+/** Props contract for custom trace renderer components. */
+export interface TraceRendererProps {
+  trace: TraceData;
+  context: TraceRendererContext;
+  actions: TraceRendererActions;
+}
+
+export interface DatasetItemRendererProps {
+  datasetItem: DatasetItemData;
+  trace?: TraceData | null;
+  context: TraceRendererContext;
+  actions: DatasetItemRendererActions;
+}
+
+export interface DatasetItemData {
+  id: string;
+  input?: unknown;
+  expectedOutput?: unknown;
+  metadata?: Record<string, unknown> | null;
+  [key: string]: unknown;
+}
+
+export interface TraceRendererContext {
+  systemId: string;
+  surface: "trace" | "dataset-item" | "run-trace";
+  datasetName?: string | null;
+  runId?: string | null;
+}
+
+export interface TraceRendererActions {
+  createScore(input: {
+    traceId?: string;
+    name: string;
+    value: number | string | boolean;
+    comment?: string;
+    metadata?: Record<string, unknown>;
+  }): Promise<unknown>;
+}
+
+export interface DatasetItemRendererActions extends TraceRendererActions {
+  updateDatasetItem(input: {
+    datasetName?: string;
+    itemId?: string;
+    expectedOutput?: unknown;
+    metadata?: Record<string, unknown> | null;
+    input?: unknown;
+    sourceTraceId?: string | null;
+    status?: string | null;
+  }): Promise<unknown>;
+  createDatasetRunItem(input: {
+    datasetItemId?: string;
+    traceId?: string;
+    runName: string;
+    runDescription?: string;
+    metadata?: Record<string, unknown>;
+  }): Promise<unknown>;
+}

package/shared/view-types.js

@@ -0,0 +1 @@
+export {};

package/shared/workspace-paths.d.ts

@@ -0,0 +1,19 @@
+export declare const KAIZEN_DIR = "kaizen";
+export declare const KAIZEN_CONFIG = "config.ts";
+export declare const KAIZEN_STATE_DIR = ".kaizen";
+export declare const KAIZEN_SYSTEMS_DIR = "systems";
+
+export declare function kaizenDir(workspaceRoot: string): string;
+export declare function kaizenConfigPath(workspaceRoot: string): string;
+export declare function kaizenSystemsDir(workspaceRoot: string): string;
+export declare function kaizenSystemDir(
+  workspaceRoot: string,
+  systemId: string,
+): string;
+export declare function kaizenSystemPath(
+  workspaceRoot: string,
+  systemId: string,
+): string;
+export declare function defaultKaizenStateDir(workspaceRoot: string): string;
+export declare function resolveKaizenStateDir(workspaceRoot: string): string;
+export declare function primaryWorktreeRoot(workspaceRoot: string): string;

package/shared/workspace-paths.js

@@ -0,0 +1,84 @@
+import { execFileSync } from "node:child_process";
+import { realpathSync } from "node:fs";
+import { join, resolve } from "node:path";
+
+export const KAIZEN_DIR = "kaizen";
+export const KAIZEN_CONFIG = "config.ts";
+export const KAIZEN_STATE_DIR = ".kaizen";
+export const KAIZEN_SYSTEMS_DIR = "systems";
+
+export function kaizenDir(workspaceRoot) {
+  return join(workspaceRoot, KAIZEN_DIR);
+}
+
+export function kaizenConfigPath(workspaceRoot) {
+  return join(kaizenDir(workspaceRoot), KAIZEN_CONFIG);
+}
+
+export function kaizenSystemsDir(workspaceRoot) {
+  return join(kaizenDir(workspaceRoot), KAIZEN_SYSTEMS_DIR);
+}
+
+export function kaizenSystemDir(workspaceRoot, systemId) {
+  return join(kaizenSystemsDir(workspaceRoot), systemId);
+}
+
+export function kaizenSystemPath(workspaceRoot, systemId) {
+  return join(kaizenSystemDir(workspaceRoot, systemId), "system.md");
+}
+
+export function defaultKaizenStateDir(workspaceRoot) {
+  return join(primaryWorktreeRoot(workspaceRoot), KAIZEN_DIR, KAIZEN_STATE_DIR);
+}
+
+export function resolveKaizenStateDir(workspaceRoot) {
+  const raw = process.env.KAIZEN_STATE_DIR;
+  return raw
+    ? resolve(workspaceRoot, raw)
+    : defaultKaizenStateDir(workspaceRoot);
+}
+
+export function primaryWorktreeRoot(workspaceRoot) {
+  const normalizedWorkspace = canonicalPath(workspaceRoot);
+  const gitRoot = gitTopLevel(normalizedWorkspace);
+  if (!gitRoot || canonicalPath(gitRoot) !== normalizedWorkspace) {
+    return normalizedWorkspace;
+  }
+
+  try {
+    const out = execFileSync(
+      "git",
+      ["-C", normalizedWorkspace, "worktree", "list", "--porcelain"],
+      { encoding: "utf-8", stdio: ["ignore", "pipe", "ignore"] },
+    );
+    const first = out.split("\n").find((line) => line.startsWith("worktree "));
+    return first
+      ? canonicalPath(first.slice("worktree ".length))
+      : normalizedWorkspace;
+  } catch {
+    return normalizedWorkspace;
+  }
+}
+
+function gitTopLevel(workspaceRoot) {
+  try {
+    return execFileSync(
+      "git",
+      ["-C", workspaceRoot, "rev-parse", "--show-toplevel"],
+      {
+        encoding: "utf-8",
+        stdio: ["ignore", "pipe", "ignore"],
+      },
+    ).trim();
+  } catch {
+    return null;
+  }
+}
+
+function canonicalPath(path) {
+  try {
+    return realpathSync(path);
+  } catch {
+    return resolve(path);
+  }
+}

package/templates/system/eval.py

@@ -7,7 +7,7 @@ well-formed event stream:
     {"type": "start", "n": <int>, "eval_version": <int>, "dataset_version": "<str>"}
     {"type": "item", "id": "<str>", "score": <float in [0,1]>, "breakdown": {...}, "trace_id": "<str|null>"}
     ... one item event per dataset item ...
-    {"type": "complete", "score": <float>, "breakdown": {...}, "worst_traces": [...]}
+    {"type": "complete", "score": <float>, "n": <int>, "breakdown": {...}, "worst_traces": [...]}
 
 If something goes wrong, emit `{"type": "error", "message": "<str>"}` and exit non-zero.
 The supervisor will record the run as `crashed` if no `complete` event is seen.
@@ -19,8 +19,9 @@ For production evals backed by Langfuse, keep the NDJSON stream as the required
 Kaizen contract and also persist results back to Langfuse as a best-effort side
 effect: load the versioned dataset, run the system to create a fresh trace for
 each item, link that trace to the dataset item in a dataset run, and write the
-primary metric as a score on the trace. Include that fresh trace id in the
-Kaizen item event so failure analysis can jump directly to Langfuse.
+primary metric as a score on the trace. Treat --dataset as the Langfuse dataset
+name unless system.md says otherwise. Include that fresh trace id in the Kaizen
+item event so failure analysis can jump directly to Langfuse.
 """
 from __future__ import annotations
 
@@ -41,8 +42,14 @@ def emit(out, event: dict[str, Any]) -> None:
     out.flush()
 
 
-def iter_items(dataset_version: str, max_items: int | None) -> Iterator[dict[str, Any]]:
-    """Yield dataset items. Replace with your real loader."""
+def load_items(dataset_name: str, max_items: int | None) -> Iterator[dict[str, Any]]:
+    """Yield dataset items.
+
+    Replace this with your real dataset loader. For Langfuse-backed evals, list
+    items from dataset_name and yield stable ids, inputs, expected outputs, and
+    any metadata your scorer needs.
+    """
+    del dataset_name
     items = [
         {"id": "demo-1", "input": "hello", "expected": "hi"},
         {"id": "demo-2", "input": "world", "expected": "world"},
@@ -87,7 +94,7 @@ def main() -> int:
     else:
         out = sys.stdout
 
-    items = list(iter_items(args.dataset, args.max_items))
+    items = list(load_items(args.dataset, args.max_items))
     emit(out, {
         "type": "start",
         "n": len(items),

package/templates/system/eval.ts

@@ -8,14 +8,16 @@
  *   {"type":"start","n":<int>,"eval_version":<int>,"dataset_version":"<str>"}
  *   {"type":"item","id":"<str>","score":<float in [0,1]>,"breakdown":{},"trace_id":"<str|null>"}
  *   ... one item event per dataset item ...
- *   {"type":"complete","score":<float>,"breakdown":{},"worst_traces":[]}
+ *   {"type":"complete","score":<float>,"n":<int>,"breakdown":{},"worst_traces":[]}
  *
  * For Langfuse-backed production evals, keep the NDJSON stream as the required
  * Kaizen contract and also persist results back to Langfuse as a best-effort
  * side effect: load the versioned dataset, run the system to create a fresh
  * trace for each item, link that trace to the dataset item in a dataset run,
- * and write the primary metric as a score on the trace. Include that fresh
- * trace id in the Kaizen item event so failure analysis can jump to Langfuse.
+ * and write the primary metric as a score on the trace. Treat --dataset as the
+ * Langfuse dataset name unless your system.md says otherwise. Include each
+ * fresh trace id in the Kaizen item event so failure analysis can jump to
+ * Langfuse.
  */
 import { closeSync, writeSync } from "node:fs";
 
@@ -82,7 +84,11 @@ function parseArgs(argv: string[]): EvalArgs {
   return args;
 }
 
-function iterItems(maxItems: number | null): DemoItem[] {
+function loadItems(datasetName: string, maxItems: number | null): DemoItem[] {
+  void datasetName;
+  // Replace this with your real dataset loader. For Langfuse-backed evals,
+  // list dataset items from --dataset and return objects with stable ids,
+  // inputs, expected outputs, and any metadata your scorer needs.
   const items = [
     { id: "demo-1", input: "hello", expected: "hi" },
     { id: "demo-2", input: "world", expected: "world" },
@@ -111,7 +117,7 @@ function average(values: number[]): number {
 
 async function main(): Promise<void> {
   const args = parseArgs(process.argv.slice(2));
-  const items = iterItems(args.maxItems);
+  const items = loadItems(args.dataset, args.maxItems);
 
   emit(args.outFd, {
     type: "start",

package/templates/system/rubric.md

@@ -25,4 +25,4 @@ The judge is itself an LLM system. Calibrate it against human labels until they
 
 ## Calibration
 
-Run the judge on 30 items that also have human labels. Iterate the rubric/prompt until agreement ≥ 95%. Track in `.kaizen/runs/<system>/<judge_run_id>/`.
+Run the judge on 30 items that also have human labels. Iterate the rubric/prompt until agreement ≥ 95%. Track in `kaizen/.kaizen/runs/<system>/<judge_run_id>/`.

package/templates/system/system.md

@@ -1,13 +1,13 @@
 ---
 name: {{name}}
 description: {{description}}
-run_eval: eval/{{name}}.{{eval_ext}}
+run_eval: kaizen/systems/{{name}}/{{eval_file}}
 eval_version: 1
 dataset_version: v1
 eval_style: {{eval_style}}
 primary_metric: {{primary_metric}}
 target: {{target}}
-execution_mode: in_process
+{{rubric_frontmatter}}
 # Optional: stable Linear project URL or ID for Kaizen Ideas.
 # linear_project: https://linear.app/<workspace>/project/<project-slug>
 created_at: {{iso_now}}
@@ -21,7 +21,7 @@ created_at: {{iso_now}}
 
 ## Key files
 
-<!-- Paths in this repo that a variant-builder agent needs to read to understand the system. -->
+<!-- Paths in this repo that a coding agent needs to read to understand the system. -->
 
 - `path/to/main_workflow.py` — orchestrator
 - `path/to/prompts.py` — prompt templates
@@ -29,7 +29,7 @@ created_at: {{iso_now}}
 
 ## Setup
 
-<!-- What does the runner or variant-builder agent need before invoking the eval? E.g.:
+<!-- What does the runner or coding agent need before invoking the eval? E.g.:
 - start servers
 - install deps
 - set env vars
@@ -39,6 +39,7 @@ created_at: {{iso_now}}
 
 <!-- For Langfuse-backed production evals:
 - Load dataset items from the `dataset_version` named in frontmatter.
+- Treat `dataset_version` as the Langfuse dataset name unless this section says otherwise.
 - For each item, run the candidate system and capture the fresh Langfuse trace id.
 - Link the dataset item to that trace in a Langfuse dataset run named for the Kaizen run.
 - Write the primary metric as a Langfuse score on the fresh trace, with secondary metrics in metadata.
@@ -63,6 +64,6 @@ graph TD
 
 ## Variant candidates
 
-<!-- A scratchpad of ideas to try. The /kaizen skill reads this to seed variant generation. -->
+<!-- A scratchpad of ideas to try. Coding agents read this to seed variant generation. -->
 
 - _none yet_

package/templates/view/dataset-item.tsx

@@ -0,0 +1,63 @@
+import { useState } from "react";
+import type { DatasetItemRendererProps } from "@percepta/kaizen";
+
+export default function DatasetItemView({
+  datasetItem,
+  trace,
+  actions,
+}: DatasetItemRendererProps) {
+  const [expectedOutput, setExpectedOutput] = useState(() =>
+    JSON.stringify(datasetItem.expectedOutput ?? null, null, 2),
+  );
+  const [status, setStatus] = useState<"idle" | "saving" | "saved" | "error">(
+    "idle",
+  );
+
+  async function saveExpectedOutput() {
+    setStatus("saving");
+    try {
+      await actions.updateDatasetItem({
+        expectedOutput: JSON.parse(expectedOutput),
+      });
+      setStatus("saved");
+    } catch {
+      setStatus("error");
+    }
+  }
+
+  return (
+    <main>
+      <h2>{datasetItem.id}</h2>
+      <section>
+        <h3>Expected output</h3>
+        <textarea
+          value={expectedOutput}
+          onChange={(event) => {
+            setExpectedOutput(event.target.value);
+            setStatus("idle");
+          }}
+          rows={10}
+          style={{ width: "100%" }}
+        />
+        <button type="button" onClick={saveExpectedOutput}>
+          Save label
+        </button>
+        {status !== "idle" ? <span>{status}</span> : null}
+      </section>
+      <section>
+        <h3>Input</h3>
+        <pre>{JSON.stringify(datasetItem.input, null, 2)}</pre>
+      </section>
+      <section>
+        <h3>Metadata</h3>
+        <pre>{JSON.stringify(datasetItem.metadata, null, 2)}</pre>
+      </section>
+      {trace ? (
+        <section>
+          <h3>Source trace</h3>
+          <pre>{JSON.stringify(trace, null, 2)}</pre>
+        </section>
+      ) : null}
+    </main>
+  );
+}

package/templates/view/trace.tsx

@@ -0,0 +1,10 @@
+import type { TraceRendererProps } from "@percepta/kaizen";
+
+export default function TraceView({ trace }: TraceRendererProps) {
+  return (
+    <main>
+      <h2>{trace.name ?? trace.id ?? "Trace"}</h2>
+      <pre>{JSON.stringify(trace, null, 2)}</pre>
+    </main>
+  );
+}