@hybridlabor-api/bdb-synapse 1.0.0

package/internal/model/stats.go

@@ -0,0 +1,151 @@
+package model
+
+// ComputeStats derives session facts from a parsed trace. errorSignal is the
+// adapter's grade for its own error detection (ObservabilityExact when the
+// source log flags failures structurally, ObservabilityEstimated when they
+// are inferred from output text); an empty value falls back to estimated.
+func ComputeStats(trace *Trace, filesInRepo int, errorSignal string) Stats {
+	state := map[string]string{}
+	lastReadVersion := map[string]int{}
+	editVersion := map[string]int{}
+	readEvents := 0
+	weakReads := 0
+	repeatedReads := 0
+	errors := 0
+	unknownOutcomes := false
+	firstEdit := -1
+
+	stats := Stats{FilesInRepo: filesInRepo}
+
+	for _, event := range trace.Events {
+		countAction(&stats.Actions, event.Action)
+		if event.IsError {
+			errors++
+			countAction(&stats.Errors, event.Action)
+		} else if !event.OutcomeKnown {
+			unknownOutcomes = true
+		}
+		stats.ResultBytes += int64(event.ResultBytes)
+		switch event.Action {
+		case "verify":
+			stats.EditsAfterLastVerify = 0
+		case "edit":
+			stats.EditsAfterLastVerify++
+		}
+		for _, target := range event.Targets {
+			if target.Path == "" {
+				continue
+			}
+			prev := state[target.Path]
+			if RankTouch(target.Touch) > RankTouch(prev) {
+				state[target.Path] = target.Touch
+			}
+			if target.Touch == "edit" {
+				editVersion[target.Path]++
+			}
+			if target.Touch == "read" {
+				readEvents++
+				if target.Weak {
+					weakReads++
+				}
+				if version, ok := lastReadVersion[target.Path]; ok && version == editVersion[target.Path] {
+					repeatedReads++
+				}
+				lastReadVersion[target.Path] = editVersion[target.Path]
+			}
+			if target.Touch == "edit" && firstEdit == -1 {
+				firstEdit = event.Seq
+			}
+		}
+	}
+
+	if firstEdit >= 0 {
+		stats.EventsBeforeFirstEdit = firstEdit
+	} else {
+		stats.EventsBeforeFirstEdit = len(trace.Events)
+	}
+
+	for _, touch := range state {
+		switch touch {
+		case "edit":
+			stats.Edited++
+			stats.Fovea++
+		case "read":
+			stats.Fovea++
+		case "hit":
+			stats.Parafovea++
+		}
+	}
+	for _, count := range editVersion {
+		if count > stats.MaxEditsPerFile {
+			stats.MaxEditsPerFile = count
+		}
+		if count >= 3 {
+			stats.ChurnFiles++
+		}
+	}
+	for _, mark := range trace.Marks {
+		switch mark.Type {
+		case "user-message":
+			stats.UserTurns++
+		case "compaction":
+			stats.Compactions++
+		case "subagent":
+			stats.Subagents++
+		}
+	}
+	if readEvents > 0 {
+		stats.RegressionRate = float64(repeatedReads) / float64(readEvents)
+	}
+	if len(trace.Events) > 0 {
+		stats.ErrorRate = float64(errors) / float64(len(trace.Events))
+	}
+	// Weak read targets are inferred from command text, so any of them in the
+	// mix downgrades the re-read rate; no reads at all leaves it undefined.
+	switch {
+	case readEvents == 0:
+		stats.Observability.Reads = ObservabilityUnavailable
+	case weakReads == 0:
+		stats.Observability.Reads = ObservabilityExact
+	default:
+		stats.Observability.Reads = ObservabilityEstimated
+	}
+	if errorSignal == "" {
+		errorSignal = ObservabilityEstimated
+	}
+	if errorSignal == ObservabilityExact && unknownOutcomes {
+		errorSignal = ObservabilityEstimated
+	}
+	stats.Observability.Errors = errorSignal
+	return stats
+}
+
+func countAction(counts *ActionCounts, action string) {
+	switch action {
+	case "search":
+		counts.Search++
+	case "read":
+		counts.Read++
+	case "edit":
+		counts.Edit++
+	case "exec":
+		counts.Exec++
+	case "verify":
+		counts.Verify++
+	default:
+		counts.Other++
+	}
+}
+
+func RankTouch(touch string) int {
+	switch touch {
+	case "edit":
+		return 3
+	case "read":
+		return 2
+	case "hit":
+		return 1
+	default:
+		return 0
+	}
+}

package/internal/model/stats_test.go

@@ -0,0 +1,89 @@
+package model
+
+import "testing"
+
+func TestComputeStatsFactCounters(t *testing.T) {
+	trace := &Trace{
+		Events: []Event{
+			{Seq: 0, Action: "search", Targets: []Target{{Path: "a.go", Touch: "hit"}}, ResultBytes: 10},
+			{Seq: 1, Action: "read", Targets: []Target{{Path: "a.go", Touch: "read"}}, ResultBytes: 20},
+			{Seq: 2, Action: "edit", Targets: []Target{{Path: "a.go", Touch: "edit"}}},
+			{Seq: 3, Action: "edit", IsError: true, Targets: []Target{{Path: "a.go", Touch: "edit"}}},
+			{Seq: 4, Action: "edit", Targets: []Target{{Path: "a.go", Touch: "edit"}}},
+			{Seq: 5, Action: "verify"},
+			{Seq: 6, Action: "edit", Targets: []Target{{Path: "b.go", Touch: "edit"}}},
+			{Seq: 7, Action: "exec", IsError: true},
+		},
+		Marks: []Mark{
+			{Seq: 0, Type: "user-message"},
+			{Seq: 3, Type: "user-message"},
+			{Seq: 4, Type: "compaction"},
+			{Seq: 5, Type: "subagent"},
+		},
+	}
+
+	stats := ComputeStats(trace, 10, ObservabilityExact)
+
+	if stats.Actions != (ActionCounts{Search: 1, Read: 1, Edit: 4, Exec: 1, Verify: 1}) {
+		t.Fatalf("actions = %#v", stats.Actions)
+	}
+	if stats.Errors != (ActionCounts{Edit: 1, Exec: 1}) {
+		t.Fatalf("errors = %#v", stats.Errors)
+	}
+	if stats.MaxEditsPerFile != 3 || stats.ChurnFiles != 1 {
+		t.Fatalf("maxEditsPerFile = %d, churnFiles = %d", stats.MaxEditsPerFile, stats.ChurnFiles)
+	}
+	if stats.UserTurns != 2 || stats.Compactions != 1 || stats.Subagents != 1 {
+		t.Fatalf("marks = %d/%d/%d", stats.UserTurns, stats.Compactions, stats.Subagents)
+	}
+	if stats.ResultBytes != 30 {
+		t.Fatalf("resultBytes = %d", stats.ResultBytes)
+	}
+	if stats.EditsAfterLastVerify != 1 {
+		t.Fatalf("editsAfterLastVerify = %d", stats.EditsAfterLastVerify)
+	}
+}
+
+func TestComputeStatsEditsAfterLastVerifyWithoutVerify(t *testing.T) {
+	trace := &Trace{
+		Events: []Event{
+			{Seq: 0, Action: "edit", Targets: []Target{{Path: "a.go", Touch: "edit"}}},
+			{Seq: 1, Action: "edit", Targets: []Target{{Path: "b.go", Touch: "edit"}}},
+		},
+	}
+	if stats := ComputeStats(trace, 0, ObservabilityExact); stats.EditsAfterLastVerify != 2 {
+		t.Fatalf("editsAfterLastVerify = %d", stats.EditsAfterLastVerify)
+	}
+}
+
+func TestComputeStatsObservability(t *testing.T) {
+	strongRead := Event{Action: "read", OutcomeKnown: true, Targets: []Target{{Path: "a.go", Touch: "read"}}}
+	weakRead := Event{Action: "read", OutcomeKnown: true, Targets: []Target{{Path: "b.go", Touch: "read", Weak: true}}}
+	hitOnly := Event{Action: "search", Targets: []Target{{Path: "c.go", Touch: "hit"}}}
+	pending := Event{Action: "exec"}
+	legacyFailure := Event{Action: "exec", IsError: true}
+
+	tests := []struct {
+		name        string
+		events      []Event
+		errorSignal string
+		wantReads   string
+		wantErrors  string
+	}{
+		{"strong reads are exact", []Event{strongRead}, ObservabilityExact, ObservabilityExact, ObservabilityExact},
+		{"any weak read downgrades", []Event{strongRead, weakRead}, ObservabilityExact, ObservabilityEstimated, ObservabilityExact},
+		{"unknown outcome downgrades exact errors", []Event{strongRead, pending}, ObservabilityExact, ObservabilityExact, ObservabilityEstimated},
+		{"legacy failure remains known", []Event{legacyFailure}, ObservabilityExact, ObservabilityUnavailable, ObservabilityExact},
+		{"no reads is unavailable", []Event{hitOnly}, ObservabilityEstimated, ObservabilityUnavailable, ObservabilityEstimated},
+		{"empty error signal falls back to estimated", []Event{strongRead}, "", ObservabilityExact, ObservabilityEstimated},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			stats := ComputeStats(&Trace{Events: tt.events}, 0, tt.errorSignal)
+			if stats.Observability.Reads != tt.wantReads || stats.Observability.Errors != tt.wantErrors {
+				t.Fatalf("observability = %#v, want reads %q errors %q", stats.Observability, tt.wantReads, tt.wantErrors)
+			}
+		})
+	}
+}

package/internal/model/trace_schema_test.go

@@ -0,0 +1,67 @@
+package model
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/santhosh-tekuri/jsonschema/v6"
+)
+
+func TestTraceSchemaAcceptsOutcomeCertainty(t *testing.T) {
+	trace := Trace{
+		Version: 1,
+		Session: TraceSession{
+			ID:         "session",
+			Harness:    "codex",
+			EventCount: 3,
+		},
+		Events: []Event{
+			{
+				Seq:          0,
+				Tool:         "exec_command",
+				Action:       "verify",
+				Targets:      []Target{},
+				OutcomeKnown: true,
+				Summary:      "run tests",
+			},
+			{
+				Seq:          1,
+				Tool:         "exec_command",
+				Action:       "verify",
+				Targets:      []Target{},
+				IsError:      true,
+				OutcomeKnown: true,
+				Summary:      "tests failed",
+			},
+			{
+				Seq:     2,
+				Tool:    "exec_command",
+				Action:  "exec",
+				Targets: []Target{},
+				Summary: "still running",
+			},
+		},
+		Marks: []Mark{},
+		Stats: ComputeStats(&Trace{}, 0, ObservabilityEstimated),
+	}
+	document, err := json.Marshal(trace)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var value any
+	if err := json.Unmarshal(document, &value); err != nil {
+		t.Fatal(err)
+	}
+	events := value.(map[string]any)["events"].([]any)
+	if _, found := events[2].(map[string]any)["outcomeKnown"]; found {
+		t.Fatal("unknown outcome serialized outcomeKnown")
+	}
+	compiler := jsonschema.NewCompiler()
+	schema, err := compiler.Compile("../../schema/trace.schema.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := schema.Validate(value); err != nil {
+		t.Fatalf("trace with outcome certainty violates schema: %v\n%s", err, document)
+	}
+}

package/internal/server/analyze.go

@@ -0,0 +1,286 @@
+package server
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"slices"
+	"sync"
+
+	"github.com/hybridlabor-api/bdb-synapse/internal/judge"
+	"github.com/hybridlabor-api/bdb-synapse/internal/model"
+)
+
+// maxConcurrentJudges bounds simultaneous judge subprocesses: each one is a
+// full agent-CLI run costing tokens and about a minute, and nothing stops a
+// user from clicking evaluate across many sessions.
+const maxConcurrentJudges = 2
+
+// analyzeJob tracks one in-flight or finished judge run, keyed by session
+// key. Evaluation only ever starts from an explicit POST — never from
+// session scanning — because a judge run costs tokens and about a minute.
+type analyzeJob struct {
+	done   bool
+	report *model.Report
+	err    string
+	// config identifies what this run was asked to produce (judge CLI, model,
+	// rubric on/off). A concurrent request for the same session with a
+	// different configuration must conflict, not silently receive this run.
+	config string
+}
+
+// jobConfig renders a request's evaluation configuration as the job identity.
+func jobConfig(cli, model string, noRubric bool) string {
+	return fmt.Sprintf("cli=%s|model=%s|rubric=%t", cli, model, !noRubric)
+}
+
+type analyzeState struct {
+	mu   sync.Mutex
+	jobs map[string]*analyzeJob
+	// active counts in-flight judge subprocesses across all sessions.
+	active int
+	// runner overrides the judge subprocess in tests; nil auto-detects a CLI.
+	runner judge.Runner
+}
+
+// snapshot returns a consistent copy of the session's job state. Job fields
+// are written by the analyze goroutine under mu, so every read must happen
+// inside the lock too — callers get a copy, never the live pointer.
+func (a *analyzeState) snapshot(key string) (analyzeJob, bool) {
+	a.mu.Lock()
+	defer a.mu.Unlock()
+	job, ok := a.jobs[key]
+	if !ok {
+		return analyzeJob{}, false
+	}
+	return *job, true
+}
+
+// reportStateFor grades one session for the list view: "running" while a
+// judge job is in flight, then "done" / "stale" / "failed". The badge uses
+// the summary approximation of freshness (no trace parse, no per-session
+// disk probe); the panel's FreshAgainstTrace check stays the precise one.
+func (s *Server) reportStateFor(meta model.SessionMeta) string {
+	job, ok := s.analyze.snapshot(meta.Key)
+	var report *model.Report
+	switch {
+	case ok && !job.done:
+		return "running"
+	case ok && job.err != "":
+		return "failed"
+	case ok && job.report != nil:
+		report = job.report
+	default:
+		report = s.reportIndex.load(s.reportCache, meta.Key)
+	}
+	if report == nil {
+		return ""
+	}
+	if !judge.FreshAgainstSummary(report, meta) {
+		return "stale"
+	}
+	return "done"
+}
+
+// judgeInfo lists the judge CLIs the user can pick from, preference order
+// first. A test runner narrows the list to itself.
+func (s *Server) judgeInfo() ([]string, bool) {
+	if s.analyze.runner != nil {
+		return []string{s.analyze.runner.Name()}, true
+	}
+	clis := judge.DetectCLIs()
+	return clis, len(clis) > 0
+}
+
+type reportStatus struct {
+	State string `json:"state"` // none | running | done | failed
+	// Stale marks a done report generated from fewer events than the trace
+	// now has (or an older prompt); the UI offers re-evaluation.
+	Stale          bool          `json:"stale"`
+	Report         *model.Report `json:"report,omitempty"`
+	Error          string        `json:"error,omitempty"`
+	JudgeAvailable bool          `json:"judgeAvailable"`
+	// JudgeCLI is the default judge (first available); JudgeCLIs lists every
+	// installed CLI so the panel can offer a choice.
+	JudgeCLI  string   `json:"judgeCli,omitempty"`
+	JudgeCLIs []string `json:"judgeClis,omitempty"`
+}
+
+func (s *Server) handleSessionReport(w http.ResponseWriter, r *http.Request, selector string) {
+	if r.Method != http.MethodGet {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	meta, err := s.findSession(selector)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusNotFound)
+		return
+	}
+	trace, _, err := s.traceAndMap(selector)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusNotFound)
+		return
+	}
+
+	status := reportStatus{State: "none"}
+	status.JudgeCLIs, status.JudgeAvailable = s.judgeInfo()
+	if status.JudgeAvailable {
+		status.JudgeCLI = status.JudgeCLIs[0]
+	}
+
+	job, ok := s.analyze.snapshot(meta.Key)
+	switch {
+	case ok && !job.done:
+		status.State = "running"
+	case ok && job.err != "":
+		status.State = "failed"
+		status.Error = job.err
+	case ok && job.report != nil:
+		status.State = "done"
+		status.Report = job.report
+		status.Stale = !judge.FreshAgainstTrace(job.report, trace)
+	default:
+		if cached := s.reportIndex.load(s.reportCache, meta.Key); cached != nil {
+			status.State = "done"
+			status.Report = cached
+			status.Stale = !judge.FreshAgainstTrace(cached, trace)
+		}
+	}
+	writeJSON(w, status)
+}
+
+func (s *Server) handleSessionAnalyze(w http.ResponseWriter, r *http.Request, selector string) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+	meta, err := s.findSession(selector)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusNotFound)
+		return
+	}
+	trace, _, err := s.traceAndMap(selector)
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusNotFound)
+		return
+	}
+	clis, available := s.judgeInfo()
+	if !available {
+		http.Error(w, "no judge CLI found on PATH (looked for claude, codex)", http.StatusServiceUnavailable)
+		return
+	}
+
+	// Optional body: the panel's judge choice. An empty body keeps the
+	// default CLI and its default model; a malformed one is rejected — this
+	// request starts an expensive run, so a garbled choice must not silently
+	// fall back to defaults.
+	var req struct {
+		CLI   string `json:"cli"`
+		Model string `json:"model"`
+		// Rubric false skips the task-rubric layer; absent or true keeps it.
+		Rubric *bool `json:"rubric"`
+	}
+	if r.Body != nil {
+		decoder := json.NewDecoder(r.Body)
+		if err := decoder.Decode(&req); err != nil {
+			if !errors.Is(err, io.EOF) {
+				http.Error(w, "invalid request body: "+err.Error(), http.StatusBadRequest)
+				return
+			}
+		} else if _, err := decoder.Token(); !errors.Is(err, io.EOF) {
+			// One JSON value and nothing after it — trailing garbage means a
+			// broken client, and this request starts an expensive run.
+			http.Error(w, "invalid request body: trailing data after JSON object", http.StatusBadRequest)
+			return
+		}
+	}
+	if req.CLI != "" && !slices.Contains(clis, req.CLI) {
+		http.Error(w, fmt.Sprintf("judge CLI %q is not available (installed: %v)", req.CLI, clis), http.StatusBadRequest)
+		return
+	}
+
+	noRubric := req.Rubric != nil && !*req.Rubric
+	config := jobConfig(req.CLI, req.Model, noRubric)
+	s.analyze.mu.Lock()
+	if job := s.analyze.jobs[meta.Key]; job != nil && !job.done {
+		sameConfig := job.config == config
+		s.analyze.mu.Unlock()
+		if !sameConfig {
+			// The API must not pretend to accept a configuration it will not
+			// run: the in-flight job was asked for something else.
+			http.Error(w, "an evaluation with a different judge configuration is already running for this session; wait for it to finish", http.StatusConflict)
+			return
+		}
+		w.WriteHeader(http.StatusAccepted)
+		writeJSON(w, reportStatus{State: "running", JudgeAvailable: true})
+		return
+	}
+	if s.analyze.active >= maxConcurrentJudges {
+		s.analyze.mu.Unlock()
+		http.Error(w, fmt.Sprintf("%d evaluations already running; wait for one to finish", maxConcurrentJudges), http.StatusTooManyRequests)
+		return
+	}
+	job := &analyzeJob{config: config}
+	s.analyze.jobs[meta.Key] = job
+	s.analyze.active++
+	s.analyze.mu.Unlock()
+
+	go s.runAnalyze(meta.Key, trace, job, req.CLI, req.Model, noRubric)
+
+	w.WriteHeader(http.StatusAccepted)
+	writeJSON(w, reportStatus{State: "running", JudgeAvailable: true})
+}
+
+func (s *Server) runAnalyze(key string, trace *model.Trace, job *analyzeJob, cli, judgeModel string, noRubric bool) {
+	ctx, cancel := context.WithTimeout(context.Background(), judge.DefaultTimeout)
+	defer cancel()
+	// The cached report seeds rubric reuse: a re-evaluation with unchanged
+	// task wording keeps the same criteria — including across judge CLIs,
+	// which is exactly what makes their verdicts comparable. A rubric:false
+	// run mirrors the CLI's --no-rubric semantics and bypasses the cache in
+	// both directions: it neither mines the cache nor overwrites a richer
+	// report with a dimensions-only one — its result lives only in the job.
+	var cached *model.Report
+	if !noRubric {
+		cached = s.reportCache.Load(key)
+	}
+	report, err := judge.Analyze(ctx, trace, judge.Options{
+		Runner:       s.analyze.runner,
+		CLI:          cli,
+		Model:        judgeModel,
+		NoRubric:     noRubric,
+		CachedReport: cached,
+	})
+
+	// Persist before publishing done, and outside the lock: once the job entry
+	// is dropped, polls must be able to find the report on disk.
+	persisted := false
+	if err == nil && !noRubric && s.reportCache.Dir != "" {
+		persisted = s.reportCache.Store(key, report) == nil
+	}
+	if persisted {
+		// Polls landing between the store and the index's next directory scan
+		// must find the report once the job entry is dropped below.
+		s.reportIndex.markPresent(key)
+	}
+
+	s.analyze.mu.Lock()
+	defer s.analyze.mu.Unlock()
+	s.analyze.active--
+	job.done = true
+	if err != nil {
+		job.err = err.Error()
+		return
+	}
+	job.report = report
+	if persisted {
+		// The cache owns the report now; dropping the entry keeps the jobs map
+		// bounded. When the disk write failed the entry stays as the only copy —
+		// losing it would cost a re-run — and failed jobs stay too (small, and
+		// the UI needs the error until a re-run replaces them).
+		delete(s.analyze.jobs, key)
+	}
+}