@hybridlabor-api/bdb-synapse 1.0.0

package/internal/judge/cache.go

@@ -0,0 +1,180 @@
+package judge
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+
+	"github.com/hybridlabor-api/bdb-synapse/internal/model"
+)
+
+// Cache persists reports under one file per session key so re-opening a
+// session never re-runs the judge. Reports are expensive; traces are not.
+type Cache struct {
+	Dir string
+}
+
+// DefaultCacheDir is ~/.synapse/reports —.synapse's own data directory,
+// never inside ~/.claude, ~/.codex, or the inspected repository.
+func DefaultCacheDir() string {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return ""
+	}
+	return filepath.Join(home, ".synapse", "reports")
+}
+
+func (c Cache) path(sessionKey string) string {
+	return filepath.Join(c.Dir, sessionKey+".json")
+}
+
+// Path returns the on-disk location of the session's report file, for
+// callers that fingerprint reports without loading them.
+func (c Cache) Path(sessionKey string) string {
+	return c.path(sessionKey)
+}
+
+// Load returns the cached report for the session key, or nil when absent or
+// unreadable (a corrupt cache entry is treated as a miss, not an error).
+func (c Cache) Load(sessionKey string) *model.Report {
+	if c.Dir == "" || sessionKey == "" {
+		return nil
+	}
+	data, err := os.ReadFile(c.path(sessionKey))
+	if err != nil {
+		return nil
+	}
+	var report model.Report
+	if json.Unmarshal(data, &report) != nil {
+		return nil
+	}
+	// Syntactically valid but hollow payloads ("null", "{}", hand-edited
+	// files) must read as a miss: the UI dereferences dimensions and judge
+	// unconditionally.
+	if report.Version < 1 || len(report.Dimensions) == 0 || report.Judge.CLI == "" {
+		return nil
+	}
+	// A dimension's nil findings serializes as JSON null, which the panel
+	// maps over unconditionally; normalize rather than reject. Rubric
+	// criteria get the same treatment.
+	for i := range report.Dimensions {
+		if report.Dimensions[i].Findings == nil {
+			report.Dimensions[i].Findings = []model.ReportFinding{}
+		}
+	}
+	if report.Rubric != nil {
+		for i := range report.Rubric.Tasks {
+			for j := range report.Rubric.Tasks[i].Criteria {
+				if report.Rubric.Tasks[i].Criteria[j].Findings == nil {
+					report.Rubric.Tasks[i].Criteria[j].Findings = []model.ReportFinding{}
+				}
+			}
+		}
+	}
+	return &report
+}
+
+// FreshAgainstTrace reports whether a cached report still matches the trace
+// it would be regenerated from: same prompt version and the same judge input
+// digest — event counts alone miss user messages (stored as marks) and
+// content edits. The rubric layer is checked against the current task
+// evidence separately, because its input window is wider than the scoring
+// document's. The judge CLI is deliberately not part of freshness — a valid
+// report stays valid. Reports from before the digest existed are stale by
+// construction.
+func FreshAgainstTrace(report *model.Report, trace *model.Trace) bool {
+	if report == nil ||
+		report.Judge.PromptVersion != PromptVersion ||
+		report.Judge.InputDigest != InputDigest(trace) {
+		return false
+	}
+	return rubricFresh(report, trace)
+}
+
+// FreshAgainstSummary is the cheap approximation of FreshAgainstTrace for
+// callers holding only a session summary (the list view): it shares the
+// prompt-version and digest-presence preconditions but compares the
+// summary's event and user-turn counts instead of recomputing the input
+// digest, so no trace parse is needed. It may call a report fresh that
+// FreshAgainstTrace grades stale (a content edit that keeps both counts);
+// the panel's full check corrects that. User turns catch message-only
+// growth the event count is blind to.
+func FreshAgainstSummary(report *model.Report, meta model.SessionMeta) bool {
+	return report != nil &&
+		report.Judge.PromptVersion == PromptVersion &&
+		report.Judge.InputDigest != "" &&
+		report.Session.EventCount == meta.EventCount &&
+		report.Session.UserTurns == meta.UserTurns
+}
+
+// rubricFresh verifies the rubric layer against the CURRENT task evidence.
+// InputDigest reads the scoring document, whose message window is tighter
+// than the rubric phase's: a mid-window message revision can leave the
+// scoring digest untouched while the task evidence — and therefore the
+// rubric that would be regenerated — has changed.
+func rubricFresh(report *model.Report, trace *model.Trace) bool {
+	rubric := report.Rubric
+	if rubric == nil {
+		return true
+	}
+	switch rubric.Status {
+	case model.RubricStatusScored:
+		if report.Judge.RubricPromptVersion != RubricPromptVersion {
+			return false
+		}
+		// A scored rubric must name a valid source and still fingerprint the
+		// task evidence it would be regenerated from.
+		if rubric.Source != model.RubricSourceFull && rubric.Source != model.RubricSourceTask {
+			return false
+		}
+		return rubric.TaskDigest == TaskDigest(trace, rubric.Source)
+	case model.RubricStatusUnavailable:
+		// Deterministic skips stay fresh only while their condition still
+		// holds for the current task evidence. no-events needs no recheck
+		// (events appearing changes the narrative, which InputDigest covers),
+		// and generation-failed deliberately stays fresh — re-running it is
+		// the user's explicit call, and RubricSatisfied already refuses to
+		// treat it as settled.
+		switch rubric.Reason {
+		case model.RubricReasonNoTaskText:
+			return len(taskMessages(trace.Marks)) == 0
+		case model.RubricReasonWeakTaskText:
+			return len(taskMessages(trace.Marks)) > 0 && taskTextRunes(trace.Marks) < weakTaskTextRunes
+		}
+	}
+	return true
+}
+
+func (c Cache) Store(sessionKey string, report *model.Report) error {
+	if c.Dir == "" || sessionKey == "" {
+		return nil
+	}
+	if err := os.MkdirAll(c.Dir, 0o755); err != nil {
+		return err
+	}
+	data, err := json.MarshalIndent(report, "", "  ")
+	if err != nil {
+		return err
+	}
+	// A unique temp file per writer: the CLI and the server may finish
+	// evaluating the same session concurrently, and a shared name would let
+	// them truncate each other mid-write. Last rename wins, atomically.
+	tmp, err := os.CreateTemp(c.Dir, sessionKey+"-*.tmp")
+	if err != nil {
+		return err
+	}
+	if _, err := tmp.Write(data); err != nil {
+		tmp.Close()
+		os.Remove(tmp.Name())
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		os.Remove(tmp.Name())
+		return err
+	}
+	if err := os.Rename(tmp.Name(), c.path(sessionKey)); err != nil {
+		os.Remove(tmp.Name())
+		return err
+	}
+	return nil
+}

package/internal/judge/cli.go

@@ -0,0 +1,240 @@
+package judge
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/hybridlabor-api/bdb-synapse/internal/textutil"
+)
+
+// RunResult carries the judge's raw text plus which model produced it. The
+// model is recorded on the report — verdicts from different judges are only
+// comparable when the report says who judged.
+type RunResult struct {
+	Text  string
+	Model string
+}
+
+// Runner abstracts the judge CLI subprocess so tests can stub it.
+type Runner interface {
+	Run(ctx context.Context, prompt, input string) (RunResult, error)
+	Name() string
+}
+
+// SupportedCLIs lists judge CLIs in detection preference order.
+var SupportedCLIs = []string{"claude", "codex"}
+
+// DetectCLI returns the first supported judge CLI found on PATH.
+func DetectCLI() (string, error) {
+	if clis := DetectCLIs(); len(clis) > 0 {
+		return clis[0], nil
+	}
+	return "", fmt.Errorf("no judge CLI found on PATH (looked for %s)", strings.Join(SupportedCLIs, ", "))
+}
+
+// DetectCLIs returns every supported judge CLI found on PATH, in preference
+// order; the UI offers the full list so the user can pick the judge.
+func DetectCLIs() []string {
+	var clis []string
+	for _, cli := range SupportedCLIs {
+		if _, err := exec.LookPath(cli); err == nil {
+			clis = append(clis, cli)
+		}
+	}
+	return clis
+}
+
+// WorkDir returns ~/.synapse/judge, the neutral directory judge subprocesses
+// run in. It holds no repository and no project instructions, and adapters use
+// IsWorkDir to recognize sessions recorded there as.synapse's own judge runs
+// (a fallback for codex CLIs that predate --ephemeral).
+func WorkDir() string {
+	home, err := os.UserHomeDir()
+	if err != nil {
+		return ""
+	}
+	return filepath.Join(home, ".synapse", "judge")
+}
+
+// IsWorkDir reports whether path is the judge working directory.
+func IsWorkDir(path string) bool {
+	dir := WorkDir()
+	return dir != "" && path != "" && filepath.Clean(path) == dir
+}
+
+func ensureWorkDir() (string, error) {
+	dir := WorkDir()
+	if dir == "" {
+		return "", fmt.Errorf("judge workdir: cannot resolve home directory")
+	}
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return "", fmt.Errorf("judge workdir: %w", err)
+	}
+	return dir, nil
+}
+
+// CLIRunner shells out to a local agent CLI in non-interactive mode.
+//
+// The trace under evaluation is untrusted input (a prompt injection in the
+// evaluated session must not reach tools), so the judge runs sealed: no
+// tools, no MCP servers, no user or project settings, and nothing the judge
+// produces may surface as a session for.synapse itself to scan.
+type CLIRunner struct {
+	CLI string
+	// Model overrides the CLI's default model when set (an alias like
+	// "sonnet" or a full name like "gpt-5.6-sol").
+	Model string
+}
+
+func (r CLIRunner) Name() string { return r.CLI }
+
+func (r CLIRunner) Run(ctx context.Context, prompt, input string) (RunResult, error) {
+	workdir, err := ensureWorkDir()
+	if err != nil {
+		return RunResult{}, err
+	}
+	var cmd *exec.Cmd
+	switch r.CLI {
+	case "claude":
+		// --output-format json wraps the reply in a result envelope whose
+		// modelUsage names the model that actually answered; see
+		// parseClaudeEnvelope.
+		args := []string{"-p",
+			"--no-session-persistence", // never a session file for.synapse to re-scan
+			"--tools", "",
+			"--strict-mcp-config",   // with no --mcp-config: zero MCP servers
+			"--setting-sources", "", // no user/project settings, hooks, or allowlists
+			"--output-format", "json",
+		}
+		if r.Model != "" {
+			args = append(args, "--model", r.Model)
+		}
+		cmd = exec.CommandContext(ctx, "claude", append(args, prompt)...)
+		cmd.Stdin = strings.NewReader(input)
+	case "codex":
+		// "-" reads the whole prompt from stdin — as an argv it would hit
+		// per-argument size limits on big traces. codex interleaves its own
+		// logging on stdout; extractJSON in the parser copes with the noise,
+		// and the preamble's "model:" line names the model. The run is still
+		// pinned to the judge workdir, and the codex adapter marks sessions
+		// recorded there auxiliary — a belt for CLIs that predate
+		// --ephemeral and for sessions old versions already wrote.
+		//
+		// The judge must be a pure text function: the feature flags and
+		// config below strip every tool a prompt injection in the evaluated
+		// trace could reach for. Ground-truth verified (not model
+		// self-reports): with this set the judge cannot read local files,
+		// fetch URLs, apply patches, or spawn collaboration agents — each
+		// attempt fails at the tool router or sandbox.
+		args := codexExecArgs(workdir)
+		if r.Model != "" {
+			args = append(args, "-c", "model="+r.Model)
+		}
+		cmd = exec.CommandContext(ctx, "codex", append(args, "-")...)
+		cmd.Stdin = strings.NewReader(prompt + "\n\n" + input)
+	default:
+		return RunResult{}, fmt.Errorf("unsupported judge CLI %q", r.CLI)
+	}
+	cmd.Dir = workdir
+	var stdout, stderr bytes.Buffer
+	cmd.Stdout = &stdout
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		detail := strings.TrimSpace(stderr.String())
+		if detail == "" {
+			detail = strings.TrimSpace(stdout.String())
+		}
+		detail = truncateFailureDetail(detail)
+		return RunResult{}, fmt.Errorf("%s failed: %w: %s", r.CLI, err, detail)
+	}
+	if r.CLI == "claude" {
+		return parseClaudeEnvelope(stdout.String()), nil
+	}
+	// codex prints its config preamble (with the "model:" line) on stderr;
+	// older versions used stdout, so check both.
+	model := codexModel(stderr.String())
+	if model == "" {
+		model = codexModel(stdout.String())
+	}
+	return RunResult{Text: stdout.String(), Model: model}, nil
+}
+
+func truncateFailureDetail(detail string) string {
+	return textutil.TruncateRunes(detail, 500, "")
+}
+
+func codexExecArgs(workdir string) []string {
+	return []string{"exec",
+		"--ephemeral",          // no session file for.synapse to re-scan
+		"--ignore-user-config", // no user MCP servers or profiles; auth stays
+		"--ignore-rules",       // no user/project execpolicy rules
+		"--disable", "shell_tool",
+		"--disable", "browser_use",
+		"--disable", "browser_use_external",
+		"--disable", "computer_use",
+		"--disable", "in_app_browser",
+		"--disable", "apps",
+		"--disable", "plugins",
+		"--disable", "hooks",
+		"--disable", "multi_agent",
+		"--disable", "multi_agent_v2",
+		"--disable", "memories",
+		"--disable", "image_generation",
+		"-c", "include_apply_patch_tool=false",
+		"-c", "tools.view_image=false",
+		"-c", `web_search="disabled"`,
+		"--sandbox", "read-only", // defense in depth behind the tool strip
+		"--skip-git-repo-check", // the judge workdir is not a repository
+		"-C", workdir,
+	}
+}
+
+// claudeEnvelope mirrors the parts of `claude -p --output-format json` output
+//.synapse needs: the reply text and per-model usage.
+type claudeEnvelope struct {
+	Result     string `json:"result"`
+	ModelUsage map[string]struct {
+		InputTokens              int64 `json:"inputTokens"`
+		CacheReadInputTokens     int64 `json:"cacheReadInputTokens"`
+		CacheCreationInputTokens int64 `json:"cacheCreationInputTokens"`
+	} `json:"modelUsage"`
+}
+
+// parseClaudeEnvelope unwraps the result envelope. The main model is the one
+// that consumed the most input — the CLI also runs a small helper model for
+// housekeeping, but only the judge reads the full evidence document. An
+// unparseable envelope falls back to the raw text with no model recorded.
+func parseClaudeEnvelope(raw string) RunResult {
+	var envelope claudeEnvelope
+	if err := json.Unmarshal([]byte(strings.TrimSpace(raw)), &envelope); err != nil || envelope.Result == "" {
+		return RunResult{Text: raw}
+	}
+	model := ""
+	var maxInput int64 = -1
+	for name, usage := range envelope.ModelUsage {
+		total := usage.InputTokens + usage.CacheReadInputTokens + usage.CacheCreationInputTokens
+		if total > maxInput {
+			maxInput = total
+			model = name
+		}
+	}
+	return RunResult{Text: envelope.Result, Model: model}
+}
+
+var codexModelLine = regexp.MustCompile(`(?m)^model:\s+(\S+)`)
+
+// codexModel pulls the model name from the config preamble codex exec prints
+// before the reply; absent a match the model stays unrecorded.
+func codexModel(raw string) string {
+	if match := codexModelLine.FindStringSubmatch(raw); match != nil {
+		return match[1]
+	}
+	return ""
+}

package/internal/judge/cli_test.go

@@ -0,0 +1,68 @@
+package judge
+
+import (
+	"slices"
+	"strings"
+	"testing"
+	"unicode/utf8"
+)
+
+func TestParseClaudeEnvelopePicksMainModel(t *testing.T) {
+	raw := `{"type":"result","result":"{\"ok\":true}","modelUsage":{
+		"claude-haiku-4-5":{"inputTokens":523,"outputTokens":13},
+		"claude-sonnet-5":{"inputTokens":1,"cacheReadInputTokens":3289,"cacheCreationInputTokens":5563}}}`
+	got := parseClaudeEnvelope(raw)
+	if got.Text != `{"ok":true}` {
+		t.Fatalf("text = %q", got.Text)
+	}
+	// The helper model wrote more output tokens, but only the judge read the
+	// full evidence document — input volume picks the right one.
+	if got.Model != "claude-sonnet-5" {
+		t.Fatalf("model = %q, want claude-sonnet-5", got.Model)
+	}
+}
+
+func TestParseClaudeEnvelopeFallsBackToRawText(t *testing.T) {
+	raw := `plain text, not an envelope {"ok":true}`
+	got := parseClaudeEnvelope(raw)
+	if got.Text != raw || got.Model != "" {
+		t.Fatalf("fallback = %#v", got)
+	}
+}
+
+func TestCodexModelReadsPreamble(t *testing.T) {
+	raw := "OpenAI Codex v0.143.0\n--------\nworkdir: /tmp\nmodel: gpt-5.6-sol\nprovider: openai\n--------\n{\"ok\":true}\n"
+	if got := codexModel(raw); got != "gpt-5.6-sol" {
+		t.Fatalf("model = %q", got)
+	}
+	if got := codexModel("no preamble at all"); got != "" {
+		t.Fatalf("expected empty model, got %q", got)
+	}
+}
+
+func TestCodexExecArgsExcludeRemovedFeatures(t *testing.T) {
+	args := codexExecArgs("/tmp/judge")
+	if slices.Contains(args, "browser_use_full_cdp_access") {
+		t.Fatal("codex args include removed browser_use_full_cdp_access feature")
+	}
+}
+
+func TestTruncateFailureDetailPreservesUTF8(t *testing.T) {
+	detail := strings.Repeat("a", 499) + "界tail"
+	got := truncateFailureDetail(detail)
+
+	if !utf8.ValidString(got) {
+		t.Fatalf("truncated detail is not valid UTF-8: %q", got)
+	}
+	want := strings.Repeat("a", 499) + "界"
+	if got != want {
+		t.Fatalf("truncated detail = %q, want %q", got, want)
+	}
+}
+
+func TestTruncateFailureDetailRepairsInvalidUTF8(t *testing.T) {
+	got := truncateFailureDetail(string([]byte{'o', 'k', 0xff}))
+	if !utf8.ValidString(got) {
+		t.Fatalf("failure detail contains invalid UTF-8: %q", got)
+	}
+}

package/internal/judge/fresh_summary_test.go

@@ -0,0 +1,37 @@
+package judge
+
+import (
+	"testing"
+
+	"github.com/hybridlabor-api/bdb-synapse/internal/model"
+)
+
+func TestFreshAgainstSummary(t *testing.T) {
+	report := &model.Report{
+		Session: model.ReportSession{EventCount: 3, UserTurns: 2},
+		Judge:   model.ReportJudge{PromptVersion: PromptVersion, InputDigest: "d"},
+	}
+	meta := model.SessionMeta{EventCount: 3, UserTurns: 2}
+	if !FreshAgainstSummary(report, meta) {
+		t.Fatal("matching counts and current prompt graded stale")
+	}
+	if FreshAgainstSummary(report, model.SessionMeta{EventCount: 4, UserTurns: 2}) {
+		t.Fatal("event growth graded fresh")
+	}
+	if FreshAgainstSummary(report, model.SessionMeta{EventCount: 3, UserTurns: 3}) {
+		t.Fatal("message-only growth graded fresh")
+	}
+	noDigest := *report
+	noDigest.Judge.InputDigest = ""
+	if FreshAgainstSummary(&noDigest, meta) {
+		t.Fatal("pre-digest report graded fresh; the panel would disagree")
+	}
+	oldPrompt := *report
+	oldPrompt.Judge.PromptVersion = PromptVersion - 1
+	if FreshAgainstSummary(&oldPrompt, meta) {
+		t.Fatal("outdated prompt graded fresh")
+	}
+	if FreshAgainstSummary(nil, meta) {
+		t.Fatal("nil report graded fresh")
+	}
+}