@hybridlabor-api/bdb-synapse 1.0.0

package/internal/judge/input.go

@@ -0,0 +1,233 @@
+package judge
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/hybridlabor-api/bdb-synapse/internal/adapter"
+	"github.com/hybridlabor-api/bdb-synapse/internal/model"
+	"github.com/hybridlabor-api/bdb-synapse/internal/textutil"
+)
+
+const (
+	maxUserMessages    = 12
+	maxUserMessageLen  = 600
+	maxSummaryLen      = 160
+	maxNarrativeEvents = 2000
+	// maxTaskMessages bounds the rubric phase's task-evidence section — much
+	// wider than the scoring budget because the rubric must see every task
+	// the user raised. Anchors, the task digest, and the weak-text gate all
+	// read exactly this set; nothing rubric-related may consult a different
+	// message list.
+	maxTaskMessages = 48
+)
+
+// BuildInput renders one trace as the scoring judge's evidence document:
+// session meta, budgeted user task wording, precomputed stats, and a
+// one-line-per-event narrative. The judge reads only this — never the raw
+// session log.
+func BuildInput(trace *model.Trace) string {
+	return buildDocument(trace, renderedUserMessages(trace.Marks))
+}
+
+// BuildRubricInput renders the rubric generator's evidence document: same
+// stats and narrative, but the user-message section is the full task
+// evidence (taskMessages) rather than the scoring budget — a task raised in
+// the middle of a long session must be visible to the generator that is
+// asked to enumerate tasks.
+func BuildRubricInput(trace *model.Trace) string {
+	return buildDocument(trace, taskMessages(trace.Marks))
+}
+
+func buildDocument(trace *model.Trace, keep []userMessage) string {
+	var b strings.Builder
+	sess := trace.Session
+	b.WriteString("# Session under evaluation\n\n")
+	fmt.Fprintf(&b, "- harness: %s  model: %s\n", sess.Harness, orUnknown(sess.Model))
+	fmt.Fprintf(&b, "- cwd: %s  events: %d\n", sess.Cwd, sess.EventCount)
+	fmt.Fprintf(&b, "- started: %s  ended: %s\n\n", sess.StartedAt, sess.EndedAt)
+
+	writeMessages(&b, keep)
+	writeStats(&b, trace.Stats)
+	writeNarrative(&b, trace)
+	return b.String()
+}
+
+// InputDigest fingerprints the evidence document BuildInput renders for the
+// trace. Unlike a bare event count it moves when user messages, tool results,
+// or stats change, so freshness checks see every input the judge saw.
+func InputDigest(trace *model.Trace) string {
+	sum := sha256.Sum256([]byte(BuildInput(trace)))
+	return hex.EncodeToString(sum[:])
+}
+
+// userMessage is one user-message mark after injected-wrapper filtering.
+// Ordinal is 1-based over the filtered list and is what the evidence document
+// renders as [user #N]; seq is the mark seq it resolves to.
+type userMessage struct {
+	ordinal int
+	seq     int
+	text    string
+}
+
+// filteredUserMessages returns every user message that may appear in the
+// evidence document, before the rendering budget is applied.
+func filteredUserMessages(marks []model.Mark) []userMessage {
+	var messages []userMessage
+	for _, mark := range marks {
+		if mark.Type != "user-message" {
+			continue
+		}
+		text := strings.TrimSpace(mark.Note)
+		// Adapters already drop injected wrappers before marks exist; the
+		// re-check here keeps judge input clean even for traces built by
+		// older adapters.
+		if text == "" || adapter.InjectedUserMessage(text) {
+			continue
+		}
+		messages = append(messages, userMessage{ordinal: len(messages) + 1, seq: mark.Seq, text: text})
+	}
+	return messages
+}
+
+// budgetMessages keeps the first message (it states the task) plus the
+// newest budget-1 (they carry corrections); mid-session chatter gives way.
+func budgetMessages(messages []userMessage, budget int) []userMessage {
+	if len(messages) > budget {
+		messages = append([]userMessage{messages[0]}, messages[len(messages)-(budget-1):]...)
+	}
+	return messages
+}
+
+// renderedUserMessages is the scoring document's message budget.
+func renderedUserMessages(marks []model.Mark) []userMessage {
+	return budgetMessages(filteredUserMessages(marks), maxUserMessages)
+}
+
+// taskMessages is the single task-evidence set of the rubric phase: what the
+// generator reads, what anchors may reference, what the digest fingerprints,
+// and what the weak-text gate measures.
+func taskMessages(marks []model.Mark) []userMessage {
+	return budgetMessages(filteredUserMessages(marks), maxTaskMessages)
+}
+
+func writeMessages(b *strings.Builder, keep []userMessage) {
+	b.WriteString("## User messages (the task; later ones are follow-ups/corrections)\n\n")
+	if len(keep) == 0 {
+		b.WriteString("(no user message text available)\n\n")
+		return
+	}
+	previous := 0
+	for _, message := range keep {
+		if message.ordinal != previous+1 {
+			fmt.Fprintf(b, "…%d intermediate user messages omitted.\n\n", message.ordinal-previous-1)
+		}
+		previous = message.ordinal
+		fmt.Fprintf(b, "[user #%d] %s\n\n", message.ordinal, truncateRunes(message.text, maxUserMessageLen))
+	}
+}
+
+// taskTextRunes measures the task signal available to a rubric — over the
+// same set the generator will actually see.
+func taskTextRunes(marks []model.Mark) int {
+	total := 0
+	for _, message := range taskMessages(marks) {
+		total += len([]rune(message.text))
+	}
+	return total
+}
+
+// taskSection renders the rubric phase's user-message section.
+func taskSection(marks []model.Mark) string {
+	var b strings.Builder
+	writeMessages(&b, taskMessages(marks))
+	return b.String()
+}
+
+// TaskDigest fingerprints the exact task section the rubric generator reads.
+// Unlike InputDigest it ignores events and stats: a session that only grew
+// in activity keeps its rubric, while any change to the task wording, the
+// generation source mode, or the rubric prompt forces regeneration.
+func TaskDigest(trace *model.Trace, source string) string {
+	sum := sha256.Sum256([]byte(strings.Join([]string{
+		trace.Session.Harness,
+		taskSection(trace.Marks),
+		source,
+		strconv.Itoa(RubricPromptVersion),
+	}, "\n")))
+	return hex.EncodeToString(sum[:])
+}
+
+func writeStats(b *strings.Builder, stats model.Stats) {
+	b.WriteString("## Deterministic stats (precomputed, trust these numbers)\n\n")
+	encoded, err := json.MarshalIndent(stats, "", " ")
+	if err != nil {
+		encoded = []byte("{}")
+	}
+	b.WriteString("```json\n")
+	b.Write(encoded)
+	b.WriteString("\n```\n\n")
+}
+
+func writeNarrative(b *strings.Builder, trace *model.Trace) {
+	b.WriteString("## Event narrative (seq | action | targets | summary; ERR = tool errored)\n\n")
+	marksBySeq := map[int][]string{}
+	for _, mark := range trace.Marks {
+		marksBySeq[mark.Seq] = append(marksBySeq[mark.Seq], mark.Type)
+	}
+	seqs := make([]int, 0, len(marksBySeq))
+	for seq := range marksBySeq {
+		seqs = append(seqs, seq)
+	}
+	sort.Ints(seqs)
+
+	for i, event := range trace.Events {
+		if i >= maxNarrativeEvents {
+			fmt.Fprintf(b, "…%d later events omitted.\n", len(trace.Events)-maxNarrativeEvents)
+			break
+		}
+		for _, markType := range marksBySeq[event.Seq] {
+			fmt.Fprintf(b, "--- mark: %s ---\n", markType)
+		}
+		paths := make([]string, 0, 3)
+		for _, target := range event.Targets {
+			if len(paths) == 3 {
+				break
+			}
+			paths = append(paths, target.Path)
+		}
+		pathList := "-"
+		if len(paths) > 0 {
+			pathList = strings.Join(paths, ",")
+		}
+		errFlag := ""
+		if event.IsError {
+			errFlag = " ERR"
+		}
+		fmt.Fprintf(b, "%d | %s%s | %s | %s\n", event.Seq, event.Action, errFlag, pathList, truncateRunes(event.Summary, maxSummaryLen))
+	}
+	// Marks that point past the last event (e.g. a closing user message).
+	for _, seq := range seqs {
+		if seq >= len(trace.Events) {
+			for _, markType := range marksBySeq[seq] {
+				fmt.Fprintf(b, "--- mark: %s ---\n", markType)
+			}
+		}
+	}
+}
+
+func truncateRunes(s string, limit int) string {
+	return textutil.TruncateRunes(s, limit, " …[truncated]")
+}
+
+func orUnknown(s string) string {
+	if s == "" {
+		return "?"
+	}
+	return s
+}

package/internal/judge/judge.go

@@ -0,0 +1,416 @@
+package judge
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/hybridlabor-api/bdb-synapse/internal/model"
+)
+
+// DefaultTimeout bounds one whole evaluation. Two sealed calls (rubric, then
+// scoring) at a measured ~30-40s each leave the same headroom the single-call
+// pipeline had at five minutes.
+const DefaultTimeout = 10 * time.Minute
+
+type Options struct {
+	// Runner overrides the subprocess runner; nil selects CLIRunner{CLI, Model}.
+	Runner Runner
+	// CLI names the judge CLI ("claude" or "codex"); empty auto-detects.
+	CLI string
+	// Model overrides the CLI's default model; empty keeps the default.
+	Model string
+	// NoRubric skips the rubric layer entirely: one dimensions-only call,
+	// and the report carries no rubric block.
+	NoRubric bool
+	// CachedReport is the previous report for this session, if any; a scored
+	// rubric whose task digest still matches is reused instead of regenerated,
+	// so criteria stay stable across re-evaluations.
+	CachedReport *model.Report
+}
+
+// Analyze runs the judge over one trace and returns the evaluation report.
+// The rubric layer resolves first (skip, reuse, or generate-with-degrade);
+// scoring is a single unified call covering the four fixed dimensions plus
+// any rubric criteria. The judge only contributes findings; verdicts are
+// rolled up mechanically. Invalid judge output is retried once before failing.
+func Analyze(ctx context.Context, trace *model.Trace, opts Options) (*model.Report, error) {
+	runner := opts.Runner
+	if runner == nil {
+		cli := opts.CLI
+		if cli == "" {
+			detected, err := DetectCLI()
+			if err != nil {
+				return nil, err
+			}
+			cli = detected
+		}
+		runner = CLIRunner{CLI: cli, Model: opts.Model}
+	}
+
+	input := BuildInput(trace)
+	var rubric *model.Rubric
+	if !opts.NoRubric {
+		acquired, err := acquireRubric(ctx, runner, trace, opts.CachedReport)
+		if err != nil {
+			return nil, err
+		}
+		rubric = acquired
+	}
+	sysPrompt, scoringInput := prompt, input
+	if rubric != nil && rubric.Status == model.RubricStatusScored {
+		sysPrompt = scoringPrompt
+		scoringInput = "# RUBRIC (data)\n\n" + scoringRubricJSON(rubric) + "\n\n# SESSION\n\n" + input
+	}
+
+	var lastErr error
+	for attempt := 0; attempt < 2; attempt++ {
+		result, err := runner.Run(ctx, sysPrompt, scoringInput)
+		if err != nil {
+			return nil, err
+		}
+		report, err := parseOutput(result.Text, trace, rubric)
+		if err != nil {
+			lastErr = err
+			continue
+		}
+		// Prefer the model the CLI says it used; fall back to what was asked
+		// for so the report never silently drops the information.
+		judgeModel := result.Model
+		if judgeModel == "" {
+			judgeModel = opts.Model
+		}
+		report.Judge = model.ReportJudge{
+			CLI:            runner.Name(),
+			Model:          judgeModel,
+			RequestedModel: opts.Model,
+			PromptVersion:  PromptVersion,
+			GeneratedAt:    time.Now().UTC().Format(time.RFC3339),
+			InputDigest:    InputDigest(trace),
+		}
+		if report.Rubric != nil && report.Rubric.Status == model.RubricStatusScored {
+			report.Judge.RubricPromptVersion = RubricPromptVersion
+		}
+		return report, nil
+	}
+	return nil, fmt.Errorf("judge output invalid after retry: %w", lastErr)
+}
+
+// llmFinding and llmOutput mirror the JSON shapes the scoring prompts request.
+type llmFinding struct {
+	Claim        string `json:"claim"`
+	Severity     string `json:"severity"`
+	EvidenceSeqs []int  `json:"evidence_seqs"`
+}
+
+type llmOutput struct {
+	TaskSummary string `json:"task_summary"`
+	Dimensions  []struct {
+		Name     string       `json:"name"`
+		Findings []llmFinding `json:"findings"`
+	} `json:"dimensions"`
+	Criteria []struct {
+		ID       string       `json:"id"`
+		Coverage string       `json:"coverage"`
+		Findings []llmFinding `json:"findings"`
+	} `json:"criteria"`
+	RubricNote     string `json:"rubric_note"`
+	NotableMoments []struct {
+		Seq  int    `json:"seq"`
+		Note string `json:"note"`
+	} `json:"notable_moments"`
+	Narrative string `json:"narrative"`
+}
+
+func parseOutput(raw string, trace *model.Trace, rubric *model.Rubric) (*model.Report, error) {
+	payload, err := extractJSON(raw)
+	if err != nil {
+		return nil, err
+	}
+	var out llmOutput
+	if err := json.Unmarshal([]byte(payload), &out); err != nil {
+		return nil, fmt.Errorf("judge JSON: %w", err)
+	}
+
+	validSeqs := make(map[int]bool, len(trace.Events))
+	for _, event := range trace.Events {
+		validSeqs[event.Seq] = true
+	}
+
+	byName := map[string]*model.ReportDimension{}
+	for _, dim := range out.Dimensions {
+		if !knownDimension(dim.Name) {
+			continue
+		}
+		target, ok := byName[dim.Name]
+		if !ok {
+			target = &model.ReportDimension{Name: dim.Name, Findings: []model.ReportFinding{}}
+			byName[dim.Name] = target
+		}
+		findings, err := filterFindings(dim.Findings, validSeqs)
+		if err != nil {
+			return nil, err
+		}
+		target.Findings = append(target.Findings, findings...)
+	}
+	if len(byName) != len(model.DimensionNames) {
+		return nil, fmt.Errorf("judge output covers %d of %d dimensions", len(byName), len(model.DimensionNames))
+	}
+
+	report := &model.Report{
+		Version: 1,
+		Session: model.ReportSession{
+			ID:         trace.Session.ID,
+			Harness:    trace.Session.Harness,
+			Model:      trace.Session.Model,
+			EventCount: trace.Session.EventCount,
+			UserTurns:  trace.Stats.UserTurns,
+		},
+		TaskSummary: out.TaskSummary,
+		Narrative:   out.Narrative,
+	}
+	for _, name := range model.DimensionNames {
+		dim := byName[name]
+		if len(trace.Events) == 0 {
+			// A conversation-only trace has nothing citable: every finding was
+			// just dropped, and a good verdict here would be praise on zero
+			// evidence. No events, no signal — for all four dimensions.
+			dim.Verdict = model.VerdictInsufficientData
+		} else {
+			dim.Verdict = rollupVerdict(name, dim.Findings, trace.Stats.Observability)
+		}
+		report.Dimensions = append(report.Dimensions, *dim)
+	}
+	if rubric != nil {
+		if rubric.Status == model.RubricStatusScored {
+			scored, err := scoreRubric(rubric, &out, validSeqs)
+			if err != nil {
+				return nil, err
+			}
+			report.Rubric = scored
+		} else {
+			report.Rubric = rubric
+		}
+	}
+	for _, moment := range out.NotableMoments {
+		if validSeqs[moment.Seq] && moment.Note != "" {
+			report.NotableMoments = append(report.NotableMoments, model.ReportMoment{Seq: moment.Seq, Note: moment.Note})
+		}
+	}
+	return report, nil
+}
+
+// filterFindings applies the evidence discipline shared by dimensions and
+// rubric criteria: hallucinated seqs are stripped, a finding with no valid
+// citation left may not enter the report, and an unrecognized severity
+// invalidates the whole output — silently downgrading a misspelled "problem"
+// to info would launder a red flag into a good verdict.
+func filterFindings(raw []llmFinding, validSeqs map[int]bool) ([]model.ReportFinding, error) {
+	findings := make([]model.ReportFinding, 0, len(raw))
+	for _, finding := range raw {
+		if finding.Claim == "" {
+			continue
+		}
+		seqs := make([]int, 0, len(finding.EvidenceSeqs))
+		for _, seq := range finding.EvidenceSeqs {
+			if validSeqs[seq] {
+				seqs = append(seqs, seq)
+			}
+		}
+		if len(seqs) == 0 {
+			continue
+		}
+		severity, err := normalizeSeverity(finding.Severity)
+		if err != nil {
+			return nil, err
+		}
+		findings = append(findings, model.ReportFinding{
+			Claim:        finding.Claim,
+			Severity:     severity,
+			EvidenceSeqs: seqs,
+		})
+	}
+	return findings, nil
+}
+
+// scoreRubric merges the scoring output into a copy of the rubric. The judge
+// echoes a flat criteria list; grouping comes from the rubric itself, so a
+// criterion can never land in the wrong task. Every rubric criterion must be
+// scored — a missing one invalidates the output; unknown and duplicate ids
+// are dropped.
+func scoreRubric(rubric *model.Rubric, out *llmOutput, validSeqs map[int]bool) (*model.Rubric, error) {
+	type score struct {
+		coverage string
+		findings []model.ReportFinding
+	}
+	expected := map[string]bool{}
+	for _, task := range rubric.Tasks {
+		for _, criterion := range task.Criteria {
+			expected[criterion.ID] = true
+		}
+	}
+	scores := map[string]score{}
+	for _, criterion := range out.Criteria {
+		// Unknown ids are dropped before any validation: an invented entry is
+		// noise the contract discards, and its malformed coverage or findings
+		// must not be able to fail the whole scoring pass.
+		if !expected[criterion.ID] {
+			continue
+		}
+		if _, dup := scores[criterion.ID]; dup {
+			continue
+		}
+		coverage, err := normalizeCoverage(criterion.Coverage)
+		if err != nil {
+			return nil, err
+		}
+		findings, err := filterFindings(criterion.Findings, validSeqs)
+		if err != nil {
+			return nil, err
+		}
+		scores[criterion.ID] = score{coverage: coverage, findings: findings}
+	}
+
+	scored := &model.Rubric{
+		Status:     rubric.Status,
+		Source:     rubric.Source,
+		TaskDigest: rubric.TaskDigest,
+		Note:       truncateRunes(strings.TrimSpace(out.RubricNote), maxRubricTextRunes),
+		Tasks:      make([]model.RubricTask, len(rubric.Tasks)),
+	}
+	for i, task := range rubric.Tasks {
+		copied := task
+		copied.Criteria = make([]model.RubricCriterion, len(task.Criteria))
+		for j, criterion := range task.Criteria {
+			result, ok := scores[criterion.ID]
+			if !ok {
+				return nil, fmt.Errorf("judge output misses rubric criterion %q", criterion.ID)
+			}
+			criterion.Coverage = result.coverage
+			criterion.Findings = result.findings
+			criterion.Verdict = rollupCriterion(result.coverage, result.findings)
+			copied.Criteria[j] = criterion
+		}
+		scored.Tasks[i] = copied
+	}
+	return scored, nil
+}
+
+// rollupVerdict derives the dimension verdict from finding severities; the
+// judge never decides verdicts. Blind spots recorded by the deterministic
+// layer force insufficient-data regardless of what the judge observed.
+func rollupVerdict(name string, findings []model.ReportFinding, obs model.Observability) string {
+	if obs.Reads == model.ObservabilityUnavailable && (name == model.DimensionExploration || name == model.DimensionWandering) {
+		return model.VerdictInsufficientData
+	}
+	if obs.Errors == model.ObservabilityUnavailable && name == model.DimensionVerification {
+		return model.VerdictInsufficientData
+	}
+	return rollupSeverities(findings)
+}
+
+// rollupCriterion is the rubric-layer analogue: the coverage grade plays the
+// role observability plays for dimensions — a criterion the log cannot
+// evidence reads as insufficient data, never as a flaw.
+func rollupCriterion(coverage string, findings []model.ReportFinding) string {
+	if coverage == model.CoverageNone {
+		return model.VerdictInsufficientData
+	}
+	return rollupSeverities(findings)
+}
+
+func rollupSeverities(findings []model.ReportFinding) string {
+	verdict := model.VerdictGood
+	for _, finding := range findings {
+		switch finding.Severity {
+		case model.SeverityProblem:
+			return model.VerdictProblem
+		case model.SeverityWarning:
+			verdict = model.VerdictWarning
+		}
+	}
+	return verdict
+}
+
+func knownDimension(name string) bool {
+	for _, known := range model.DimensionNames {
+		if name == known {
+			return true
+		}
+	}
+	return false
+}
+
+// normalizeSeverity forgives casing and whitespace but nothing else: an
+// unrecognized severity is judge output we cannot trust to aggregate.
+func normalizeSeverity(severity string) (string, error) {
+	switch strings.ToLower(strings.TrimSpace(severity)) {
+	case model.SeverityInfo:
+		return model.SeverityInfo, nil
+	case model.SeverityWarning:
+		return model.SeverityWarning, nil
+	case model.SeverityProblem:
+		return model.SeverityProblem, nil
+	default:
+		return "", fmt.Errorf("judge output: unknown severity %q", severity)
+	}
+}
+
+// normalizeCoverage applies the same strictness to coverage: an unknown grade
+// could silently flip a criterion between scored and insufficient-data.
+func normalizeCoverage(coverage string) (string, error) {
+	switch strings.ToLower(strings.TrimSpace(coverage)) {
+	case model.CoverageSufficient:
+		return model.CoverageSufficient, nil
+	case model.CoveragePartial:
+		return model.CoveragePartial, nil
+	case model.CoverageNone:
+		return model.CoverageNone, nil
+	default:
+		return "", fmt.Errorf("judge output: unknown coverage %q", coverage)
+	}
+}
+
+// extractJSON returns the first balanced top-level JSON object in text,
+// tolerating judge CLIs that wrap output in logs or markdown fences.
+func extractJSON(text string) (string, error) {
+	start := -1
+	depth := 0
+	inString := false
+	escaped := false
+	for i, r := range text {
+		if start == -1 {
+			if r == '{' {
+				start = i
+				depth = 1
+			}
+			continue
+		}
+		if escaped {
+			escaped = false
+			continue
+		}
+		switch r {
+		case '\\':
+			if inString {
+				escaped = true
+			}
+		case '"':
+			inString = !inString
+		case '{':
+			if !inString {
+				depth++
+			}
+		case '}':
+			if !inString {
+				depth--
+				if depth == 0 {
+					return text[start : i+1], nil
+				}
+			}
+		}
+	}
+	return "", fmt.Errorf("no JSON object in judge output")
+}