devrites 5.11.0 → 5.13.0

package/engine/internal/overhaul/bench/run.go

@@ -0,0 +1,248 @@
+// Package bench implements the /overhaul bench tool: ratio of medians with a
+// seeded percentile-bootstrap 95% confidence interval.
+//
+// Input: {"direction": "lower_is_better"|"higher_is_better", "baseline": [...],
+// "candidate": [...], "order": ["A","B",...], "target": 2.0, "materiality": 1.05,
+// "resamples": 10000, "seed": 1, "resolution": 0}. The unit of resampling is one
+// independent invocation per sample; never pass inner iterations. A k-fold claim
+// is supported only when the CI lower bound >= k.
+package bench
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math"
+	"math/rand/v2"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+
+	"github.com/devrites/devrites/internal/overhaul/ovio"
+)
+
+const usage = "usage: devrites-engine overhaul bench <result.json>\n"
+
+// Run executes the tool with its arguments (tool name excluded).
+func Run(args []string, stdout, stderr io.Writer) int {
+	if len(args) != 1 {
+		fmt.Fprint(stderr, usage)
+		return 2
+	}
+	d, err := ovio.LoadObject(args[0])
+	var res any
+	if err == nil {
+		res, err = compare(d)
+	}
+	var out []byte
+	if err == nil {
+		out, err = ovio.MarshalIndent(res)
+	}
+	if err != nil {
+		fmt.Fprintf(stderr, "ERROR: %v\n", err)
+		return 2
+	}
+	_, _ = stdout.Write(out)
+	return 0
+}
+
+type result struct {
+	Verdict         string     `json:"verdict"`
+	Ratio           float64    `json:"ratio"`
+	CI95            [2]float64 `json:"ci95"`
+	BaselineMedian  float64    `json:"baseline_median"`
+	CandidateMedian float64    `json:"candidate_median"`
+	AbsoluteDelta   float64    `json:"absolute_delta"`
+	PercentChange   float64    `json:"percent_change"`
+	N               [2]int     `json:"n"`
+	Target          float64    `json:"target"`
+	Materiality     float64    `json:"materiality"`
+	Notes           []string   `json:"notes"`
+}
+
+type noRatio struct {
+	Verdict         string   `json:"verdict"`
+	Reason          string   `json:"reason"`
+	BaselineMedian  float64  `json:"baseline_median"`
+	CandidateMedian float64  `json:"candidate_median"`
+	Notes           []string `json:"notes"`
+}
+
+const (
+	noteFew     = "fewer than 10 invocations per arm: exploratory only"
+	noteBlocked = "arms not recorded as interleaved: contamination risk, verdict capped at inconclusive"
+)
+
+func compare(d map[string]any) (any, error) {
+	dir := d["direction"]
+	if dir != "lower_is_better" && dir != "higher_is_better" {
+		return nil, fmt.Errorf("direction must be lower_is_better or higher_is_better, got %#v", dir)
+	}
+	lower := dir == "lower_is_better"
+	a, err := samples(d, "baseline")
+	if err != nil {
+		return nil, err
+	}
+	b, err := samples(d, "candidate")
+	if err != nil {
+		return nil, err
+	}
+	if len(a) == 0 || len(b) == 0 {
+		return nil, errors.New("no median for empty data")
+	}
+	notes := []string{}
+	n := min(len(a), len(b))
+	if n < 10 {
+		notes = append(notes, noteFew)
+	}
+	order := ovio.List(d["order"])
+	switches := 0
+	for i := 1; i < len(order); i++ {
+		if !reflect.DeepEqual(order[i-1], order[i]) {
+			switches++
+		}
+	}
+	if len(order) != len(a)+len(b) || switches < n/2 {
+		notes = append(notes, noteBlocked)
+	}
+	ma, mb := median(a), median(b)
+	floor, err := number(d, "resolution", 0)
+	if err != nil {
+		return nil, err
+	}
+	if min(ma, mb) <= floor {
+		return noRatio{"NO_RATIO", "median at or below measurement resolution", ma, mb, notes}, nil
+	}
+	seed := int64(1)
+	if v, ok := d["seed"]; ok {
+		num, isNum := v.(json.Number)
+		s, err := num.Int64()
+		if !isNum || err != nil {
+			return nil, fmt.Errorf("seed must be an integer, got %v", v)
+		}
+		seed = s
+	}
+	resamples, err := number(d, "resamples", 10000)
+	if err != nil {
+		return nil, err
+	}
+	if int(resamples) < 1 {
+		return nil, fmt.Errorf("resamples must be at least 1, got %v", resamples)
+	}
+	rng := rand.New(rand.NewPCG(uint64(seed), 0)) // #nosec G404 -- bootstrap resampling needs a seeded, reproducible PRNG, not secrecy
+	boots := make([]float64, int(resamples))
+	ra, rb := make([]float64, len(a)), make([]float64, len(b))
+	for i := range boots {
+		for j := range ra {
+			ra[j] = a[rng.IntN(len(a))]
+		}
+		for j := range rb {
+			rb[j] = b[rng.IntN(len(b))]
+		}
+		boots[i] = ratio(median(ra), median(rb), lower)
+	}
+	sort.Float64s(boots)
+	hiIdx := int(0.975*float64(len(boots))) - 1
+	if hiIdx < 0 { // Python's boots[-1]
+		hiIdx += len(boots)
+	}
+	lo, hi := boots[int(0.025*float64(len(boots)))], boots[hiIdx]
+	target, err := number(d, "target", 1)
+	if err != nil {
+		return nil, err
+	}
+	mat, err := number(d, "materiality", 1)
+	if err != nil {
+		return nil, err
+	}
+	var verdict string
+	switch {
+	case hi < 1:
+		verdict = "REGRESSION"
+	case lo >= target && target > 1:
+		verdict = "SUPPORTS_TARGET"
+	case lo >= mat && lo > 1 && target > 1:
+		verdict = "IMPROVEMENT_BELOW_TARGET"
+	case lo >= mat && lo > 1:
+		verdict = "IMPROVEMENT"
+	default:
+		verdict = "INCONCLUSIVE"
+	}
+	if len(notes) > 0 && verdict != "REGRESSION" {
+		verdict = "INCONCLUSIVE"
+	}
+	delta := mb - ma
+	return result{
+		Verdict: verdict, Ratio: round(ratio(ma, mb, lower), 4), CI95: [2]float64{round(lo, 4), round(hi, 4)},
+		BaselineMedian: ma, CandidateMedian: mb, AbsoluteDelta: delta,
+		PercentChange: round(100*delta/ma, 2), N: [2]int{len(a), len(b)},
+		Target: target, Materiality: mat, Notes: notes,
+	}, nil
+}
+
+func ratio(base, cand float64, lowerBetter bool) float64 {
+	if lowerBetter {
+		return base / cand
+	}
+	return cand / base
+}
+
+func median(xs []float64) float64 {
+	s := append([]float64(nil), xs...)
+	sort.Float64s(s)
+	m := len(s) / 2
+	if len(s)%2 == 1 {
+		return s[m]
+	}
+	return (s[m-1] + s[m]) / 2
+}
+
+func round(x float64, places int) float64 {
+	p := math.Pow(10, float64(places))
+	return math.Round(x*p) / p
+}
+
+func samples(d map[string]any, key string) ([]float64, error) {
+	v, ok := d[key]
+	if !ok {
+		return nil, fmt.Errorf("missing %q", key)
+	}
+	list, ok := v.([]any)
+	if !ok {
+		return nil, fmt.Errorf("%s must be an array", key)
+	}
+	out := make([]float64, len(list))
+	for i, x := range list {
+		f, err := toFloat(x, key)
+		if err != nil {
+			return nil, err
+		}
+		out[i] = f
+	}
+	return out, nil
+}
+
+func number(d map[string]any, key string, def float64) (float64, error) {
+	v, ok := d[key]
+	if !ok {
+		return def, nil
+	}
+	return toFloat(v, key)
+}
+
+// toFloat accepts JSON numbers and numeric strings, like Python's float().
+func toFloat(v any, key string) (float64, error) {
+	switch x := v.(type) {
+	case json.Number:
+		return x.Float64()
+	case string:
+		f, err := strconv.ParseFloat(strings.TrimSpace(x), 64)
+		if err != nil {
+			return 0, fmt.Errorf("%s: could not convert %q to float", key, x)
+		}
+		return f, nil
+	}
+	return 0, fmt.Errorf("%s: expected a number, got %v", key, v)
+}

package/engine/internal/overhaul/bench/run_test.go

@@ -0,0 +1,139 @@
+package bench
+
+import (
+	"bytes"
+	"encoding/json"
+	"math/rand/v2"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func gauss(rng *rand.Rand, mean, sd float64, n int) []float64 {
+	out := make([]float64, n)
+	for i := range out {
+		out[i] = mean + rng.NormFloat64()*sd
+	}
+	return out
+}
+
+func interleaved(n int) []string {
+	var o []string
+	for range n {
+		o = append(o, "A", "B")
+	}
+	return o
+}
+
+// run writes d as the result file, runs the CLI and decodes its output.
+func run(t *testing.T, d map[string]any) (int, map[string]any, string) {
+	t.Helper()
+	b, _ := json.Marshal(d)
+	path := filepath.Join(t.TempDir(), "result.json")
+	if err := os.WriteFile(path, b, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	var out, errb bytes.Buffer
+	code := Run([]string{path}, &out, &errb)
+	var res map[string]any
+	if code == 0 {
+		if err := json.Unmarshal(out.Bytes(), &res); err != nil {
+			t.Fatalf("bad output %q: %v", out.String(), err)
+		}
+	}
+	return code, res, errb.String()
+}
+
+func with(d map[string]any, kv ...any) map[string]any {
+	c := map[string]any{}
+	for k, v := range d {
+		c[k] = v
+	}
+	for i := 0; i < len(kv); i += 2 {
+		c[kv[i].(string)] = kv[i+1]
+	}
+	return c
+}
+
+func verdict(t *testing.T, d map[string]any) (string, map[string]any) {
+	t.Helper()
+	code, res, stderr := run(t, d)
+	if code != 0 {
+		t.Fatalf("exit %d: %s", code, stderr)
+	}
+	return res["verdict"].(string), res
+}
+
+func fixture() (fast map[string]any, rng *rand.Rand) {
+	rng = rand.New(rand.NewPCG(7, 0))
+	fast = map[string]any{
+		"direction": "lower_is_better",
+		"baseline":  gauss(rng, 30, 1, 20), "candidate": gauss(rng, 10, 0.4, 20),
+		"order": interleaved(20), "target": 2.0, "materiality": 1.05,
+	}
+	return fast, rng
+}
+
+func TestBench(t *testing.T) {
+	fast, rng := fixture()
+
+	t.Run("clear 3x speedup supports target 2", func(t *testing.T) {
+		if v, res := verdict(t, fast); v != "SUPPORTS_TARGET" {
+			t.Fatalf("got %v", res)
+		}
+	})
+	t.Run("noisy data does not support target", func(t *testing.T) {
+		noisy := map[string]any{
+			"direction": "lower_is_better", "baseline": gauss(rng, 20, 9, 12), "candidate": gauss(rng, 10, 6, 12),
+			"order": interleaved(12), "target": 2.0, "materiality": 1.05,
+		}
+		v, res := verdict(t, noisy)
+		if lo := res["ci95"].([]any)[0].(float64); v == "SUPPORTS_TARGET" || lo >= 2 {
+			t.Fatalf("got %v", res)
+		}
+	})
+	t.Run("three cherry-picked samples are inconclusive", func(t *testing.T) {
+		picked := with(fast, "baseline", fast["baseline"].([]float64)[:3],
+			"candidate", fast["candidate"].([]float64)[:3], "order", interleaved(3))
+		if v, res := verdict(t, picked); v != "INCONCLUSIVE" {
+			t.Fatalf("got %v", res)
+		}
+	})
+	t.Run("blocked order is inconclusive", func(t *testing.T) {
+		order := append(strings.Split(strings.Repeat("A", 20), ""), strings.Split(strings.Repeat("B", 20), "")...)
+		if v, res := verdict(t, with(fast, "order", order)); v != "INCONCLUSIVE" {
+			t.Fatalf("got %v", res)
+		}
+	})
+	t.Run("swapped arms regress", func(t *testing.T) {
+		if v, res := verdict(t, with(fast, "baseline", fast["candidate"], "candidate", fast["baseline"])); v != "REGRESSION" {
+			t.Fatalf("got %v", res)
+		}
+	})
+	t.Run("zero baseline has no ratio", func(t *testing.T) {
+		if v, res := verdict(t, with(fast, "baseline", make([]float64, 20))); v != "NO_RATIO" {
+			t.Fatalf("got %v", res)
+		}
+	})
+	t.Run("higher is better throughput", func(t *testing.T) {
+		thr := map[string]any{
+			"direction": "higher_is_better", "baseline": gauss(rng, 100, 2, 15), "candidate": gauss(rng, 320, 5, 15),
+			"order": interleaved(15), "target": 3.0,
+		}
+		if v, res := verdict(t, thr); v != "SUPPORTS_TARGET" {
+			t.Fatalf("got %v", res)
+		}
+	})
+	t.Run("unknown or missing direction is an error", func(t *testing.T) {
+		swapped := with(fast, "baseline", fast["candidate"], "candidate", fast["baseline"])
+		missing := with(swapped)
+		delete(missing, "direction")
+		for _, d := range []map[string]any{with(swapped, "direction", "latency_ms"), missing} {
+			code, _, stderr := run(t, d)
+			if code != 2 || !strings.Contains(stderr, "direction must be lower_is_better or higher_is_better") {
+				t.Fatalf("exit %d stderr %q", code, stderr)
+			}
+		}
+	})
+}

package/engine/internal/overhaul/overhaul.go

@@ -0,0 +1,64 @@
+// Package overhaul dispatches the deterministic tools behind the standalone
+// /overhaul skill: run-record generations and validation, worker-output
+// admission, baseline snapshots, readiness scoring, benchmark statistics and
+// escaped offline views. It never calls a model or the network.
+package overhaul
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/devrites/devrites/internal/overhaul/admit"
+	"github.com/devrites/devrites/internal/overhaul/bench"
+	"github.com/devrites/devrites/internal/overhaul/records"
+	"github.com/devrites/devrites/internal/overhaul/render"
+	"github.com/devrites/devrites/internal/overhaul/score"
+	"github.com/devrites/devrites/internal/overhaul/snapshot"
+)
+
+// Usage is the command-family help text.
+const Usage = `usage: devrites-engine overhaul <tool> ...
+
+  overhaul records digest <file>
+  overhaul records stage <run>
+  overhaul records publish <run>
+  overhaul records validate <run>
+  overhaul admit receipt <run> <receipt.json> [--observed <paths.txt>]
+  overhaul admit anchor <tree> <proposals.json>
+  overhaul snapshot capture <repo> <out>
+  overhaul snapshot fingerprint <repo>
+  overhaul snapshot verify <repo> <out> [--agent-paths <file>]
+  overhaul snapshot state <repo> <out.json>
+  overhaul snapshot delta <before.json> <repo>
+  overhaul score --rubric <r.json> --results <s.json> --gates <g.json> [--out <scorecard.json>]
+  overhaul score compare --rubric <r.json> --baseline <a.json> --candidate <b.json>
+  overhaul bench <result.json>
+  overhaul render <run> <staged-generation> <review|report>
+`
+
+// Run executes one overhaul tool. Exit codes follow each tool: 0 ok, 1 rule
+// violation or rejection, 2 usage or I/O error, 3 duplicate or late receipt.
+func Run(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprint(stderr, Usage)
+		return 2
+	}
+	rest := args[1:]
+	switch args[0] {
+	case "records":
+		return records.Run(rest, stdout, stderr)
+	case "admit":
+		return admit.Run(rest, stdout, stderr)
+	case "snapshot":
+		return snapshot.Run(rest, stdout, stderr)
+	case "score":
+		return score.Run(rest, stdout, stderr)
+	case "bench":
+		return bench.Run(rest, stdout, stderr)
+	case "render":
+		return render.Run(rest, stdout, stderr)
+	default:
+		fmt.Fprintf(stderr, "devrites: unknown overhaul tool %q\n\n%s", args[0], Usage)
+		return 2
+	}
+}

package/engine/internal/overhaul/ovio/ovio.go

@@ -0,0 +1,158 @@
+// Package ovio holds the small file, digest and JSON helpers shared by the
+// /overhaul run-area tools. Run records are plain JSON documents read as
+// generic maps so the tools enforce exactly the documented fields.
+package ovio
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// SHA256Bytes returns the lowercase hex SHA-256 of b.
+func SHA256Bytes(b []byte) string {
+	sum := sha256.Sum256(b)
+	return hex.EncodeToString(sum[:])
+}
+
+// SHA256File returns the lowercase hex SHA-256 of the file's exact bytes.
+func SHA256File(path string) (string, error) {
+	b, err := os.ReadFile(path) // #nosec G304 -- callers pass operator-selected run-area paths
+	if err != nil {
+		return "", err
+	}
+	return SHA256Bytes(b), nil
+}
+
+// LoadJSON decodes a UTF-8 JSON document. Numbers stay json.Number so digests,
+// weights and ratios are never rounded through float64 by accident.
+func LoadJSON(path string) (any, error) {
+	b, err := os.ReadFile(path) // #nosec G304 -- callers pass operator-selected run-area paths
+	if err != nil {
+		return nil, err
+	}
+	dec := json.NewDecoder(bytes.NewReader(b))
+	dec.UseNumber()
+	var v any
+	if err := dec.Decode(&v); err != nil {
+		return nil, fmt.Errorf("%s: invalid JSON (%v)", path, err)
+	}
+	return v, nil
+}
+
+// LoadObject decodes a JSON document that must be an object.
+func LoadObject(path string) (map[string]any, error) {
+	v, err := LoadJSON(path)
+	if err != nil {
+		return nil, err
+	}
+	m, ok := v.(map[string]any)
+	if !ok {
+		return nil, fmt.Errorf("%s: expected a JSON object", path)
+	}
+	return m, nil
+}
+
+// MarshalIndent renders v as two-space indented JSON with sorted object keys
+// and a trailing newline.
+func MarshalIndent(v any) ([]byte, error) {
+	b, err := json.MarshalIndent(v, "", "  ")
+	if err != nil {
+		return nil, err
+	}
+	return append(b, '\n'), nil
+}
+
+// WriteJSON writes v as indented JSON.
+func WriteJSON(path string, v any, perm os.FileMode) error {
+	b, err := MarshalIndent(v)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path, b, perm)
+}
+
+// resolve follows symlinks for the longest existing prefix of path, like
+// Python's os.path.realpath on a path that may not exist yet.
+func resolve(path string) string {
+	path = filepath.Clean(path)
+	if r, err := filepath.EvalSymlinks(path); err == nil {
+		return r
+	}
+	parent := filepath.Dir(path)
+	if parent == path {
+		return path
+	}
+	return filepath.Join(resolve(parent), filepath.Base(path))
+}
+
+// Resolve is the exported form of resolve for callers that compare paths.
+func Resolve(path string) string { return resolve(path) }
+
+// Get walks a dotted path through nested objects; missing keys give nil.
+func Get(v any, dotted string) any {
+	for _, k := range strings.Split(dotted, ".") {
+		m, ok := v.(map[string]any)
+		if !ok {
+			return nil
+		}
+		v = m[k]
+	}
+	return v
+}
+
+// Str returns v as a string, or "" when it is not one.
+func Str(v any) string {
+	s, _ := v.(string)
+	return s
+}
+
+// List returns v as a slice, or nil when it is not one.
+func List(v any) []any {
+	l, _ := v.([]any)
+	return l
+}
+
+// Obj returns v as an object, or nil when it is not one.
+func Obj(v any) map[string]any {
+	m, _ := v.(map[string]any)
+	return m
+}
+
+// Strings returns the string members of a JSON array.
+func Strings(v any) []string {
+	var out []string
+	for _, x := range List(v) {
+		if s, ok := x.(string); ok {
+			out = append(out, s)
+		}
+	}
+	return out
+}
+
+// Truthy mirrors JSON truthiness used by the run records: nil, false, "",
+// 0, empty arrays and empty objects are false.
+func Truthy(v any) bool {
+	switch x := v.(type) {
+	case nil:
+		return false
+	case bool:
+		return x
+	case string:
+		return x != ""
+	case json.Number:
+		return x.String() != "0" && x.String() != "0.0"
+	case float64:
+		return x != 0
+	case []any:
+		return len(x) > 0
+	case map[string]any:
+		return len(x) > 0
+	}
+	return true
+}