devrites 5.12.0 → 5.13.1

package/engine/internal/overhaul/bench/run_test.go

@@ -0,0 +1,139 @@
+package bench
+
+import (
+	"bytes"
+	"encoding/json"
+	"math/rand/v2"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func gauss(rng *rand.Rand, mean, sd float64, n int) []float64 {
+	out := make([]float64, n)
+	for i := range out {
+		out[i] = mean + rng.NormFloat64()*sd
+	}
+	return out
+}
+
+func interleaved(n int) []string {
+	var o []string
+	for range n {
+		o = append(o, "A", "B")
+	}
+	return o
+}
+
+// run writes d as the result file, runs the CLI and decodes its output.
+func run(t *testing.T, d map[string]any) (int, map[string]any, string) {
+	t.Helper()
+	b, _ := json.Marshal(d)
+	path := filepath.Join(t.TempDir(), "result.json")
+	if err := os.WriteFile(path, b, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	var out, errb bytes.Buffer
+	code := Run([]string{path}, &out, &errb)
+	var res map[string]any
+	if code == 0 {
+		if err := json.Unmarshal(out.Bytes(), &res); err != nil {
+			t.Fatalf("bad output %q: %v", out.String(), err)
+		}
+	}
+	return code, res, errb.String()
+}
+
+func with(d map[string]any, kv ...any) map[string]any {
+	c := map[string]any{}
+	for k, v := range d {
+		c[k] = v
+	}
+	for i := 0; i < len(kv); i += 2 {
+		c[kv[i].(string)] = kv[i+1]
+	}
+	return c
+}
+
+func verdict(t *testing.T, d map[string]any) (string, map[string]any) {
+	t.Helper()
+	code, res, stderr := run(t, d)
+	if code != 0 {
+		t.Fatalf("exit %d: %s", code, stderr)
+	}
+	return res["verdict"].(string), res
+}
+
+func fixture() (fast map[string]any, rng *rand.Rand) {
+	rng = rand.New(rand.NewPCG(7, 0))
+	fast = map[string]any{
+		"direction": "lower_is_better",
+		"baseline":  gauss(rng, 30, 1, 20), "candidate": gauss(rng, 10, 0.4, 20),
+		"order": interleaved(20), "target": 2.0, "materiality": 1.05,
+	}
+	return fast, rng
+}
+
+func TestBench(t *testing.T) {
+	fast, rng := fixture()
+
+	t.Run("clear 3x speedup supports target 2", func(t *testing.T) {
+		if v, res := verdict(t, fast); v != "SUPPORTS_TARGET" {
+			t.Fatalf("got %v", res)
+		}
+	})
+	t.Run("noisy data does not support target", func(t *testing.T) {
+		noisy := map[string]any{
+			"direction": "lower_is_better", "baseline": gauss(rng, 20, 9, 12), "candidate": gauss(rng, 10, 6, 12),
+			"order": interleaved(12), "target": 2.0, "materiality": 1.05,
+		}
+		v, res := verdict(t, noisy)
+		if lo := res["ci95"].([]any)[0].(float64); v == "SUPPORTS_TARGET" || lo >= 2 {
+			t.Fatalf("got %v", res)
+		}
+	})
+	t.Run("three cherry-picked samples are inconclusive", func(t *testing.T) {
+		picked := with(fast, "baseline", fast["baseline"].([]float64)[:3],
+			"candidate", fast["candidate"].([]float64)[:3], "order", interleaved(3))
+		if v, res := verdict(t, picked); v != "INCONCLUSIVE" {
+			t.Fatalf("got %v", res)
+		}
+	})
+	t.Run("blocked order is inconclusive", func(t *testing.T) {
+		order := append(strings.Split(strings.Repeat("A", 20), ""), strings.Split(strings.Repeat("B", 20), "")...)
+		if v, res := verdict(t, with(fast, "order", order)); v != "INCONCLUSIVE" {
+			t.Fatalf("got %v", res)
+		}
+	})
+	t.Run("swapped arms regress", func(t *testing.T) {
+		if v, res := verdict(t, with(fast, "baseline", fast["candidate"], "candidate", fast["baseline"])); v != "REGRESSION" {
+			t.Fatalf("got %v", res)
+		}
+	})
+	t.Run("zero baseline has no ratio", func(t *testing.T) {
+		if v, res := verdict(t, with(fast, "baseline", make([]float64, 20))); v != "NO_RATIO" {
+			t.Fatalf("got %v", res)
+		}
+	})
+	t.Run("higher is better throughput", func(t *testing.T) {
+		thr := map[string]any{
+			"direction": "higher_is_better", "baseline": gauss(rng, 100, 2, 15), "candidate": gauss(rng, 320, 5, 15),
+			"order": interleaved(15), "target": 3.0,
+		}
+		if v, res := verdict(t, thr); v != "SUPPORTS_TARGET" {
+			t.Fatalf("got %v", res)
+		}
+	})
+	t.Run("unknown or missing direction is an error", func(t *testing.T) {
+		swapped := with(fast, "baseline", fast["candidate"], "candidate", fast["baseline"])
+		missing := with(swapped)
+		delete(missing, "direction")
+		for _, d := range []map[string]any{with(swapped, "direction", "latency_ms"), missing} {
+			code, _, stderr := run(t, d)
+			if code != 2 || !strings.Contains(stderr, "direction must be lower_is_better or higher_is_better") {
+				t.Fatalf("exit %d stderr %q", code, stderr)
+			}
+		}
+	})
+}

package/engine/internal/overhaul/overhaul.go

@@ -0,0 +1,64 @@
+// Package overhaul dispatches the deterministic tools behind the standalone
+// /overhaul skill: run-record generations and validation, worker-output
+// admission, baseline snapshots, readiness scoring, benchmark statistics and
+// escaped offline views. It never calls a model or the network.
+package overhaul
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/devrites/devrites/internal/overhaul/admit"
+	"github.com/devrites/devrites/internal/overhaul/bench"
+	"github.com/devrites/devrites/internal/overhaul/records"
+	"github.com/devrites/devrites/internal/overhaul/render"
+	"github.com/devrites/devrites/internal/overhaul/score"
+	"github.com/devrites/devrites/internal/overhaul/snapshot"
+)
+
+// Usage is the command-family help text.
+const Usage = `usage: devrites-engine overhaul <tool> ...
+
+  overhaul records digest <file>
+  overhaul records stage <run>
+  overhaul records publish <run>
+  overhaul records validate <run>
+  overhaul admit receipt <run> <receipt.json> [--observed <paths.txt>]
+  overhaul admit anchor <tree> <proposals.json>
+  overhaul snapshot capture <repo> <out>
+  overhaul snapshot fingerprint <repo>
+  overhaul snapshot verify <repo> <out> [--agent-paths <file>]
+  overhaul snapshot state <repo> <out.json>
+  overhaul snapshot delta <before.json> <repo>
+  overhaul score --rubric <r.json> --results <s.json> --gates <g.json> [--out <scorecard.json>]
+  overhaul score compare --rubric <r.json> --baseline <a.json> --candidate <b.json>
+  overhaul bench <result.json>
+  overhaul render <run> <staged-generation> <review|report>
+`
+
+// Run executes one overhaul tool. Exit codes follow each tool: 0 ok, 1 rule
+// violation or rejection, 2 usage or I/O error, 3 duplicate or late receipt.
+func Run(args []string, stdout, stderr io.Writer) int {
+	if len(args) == 0 {
+		fmt.Fprint(stderr, Usage)
+		return 2
+	}
+	rest := args[1:]
+	switch args[0] {
+	case "records":
+		return records.Run(rest, stdout, stderr)
+	case "admit":
+		return admit.Run(rest, stdout, stderr)
+	case "snapshot":
+		return snapshot.Run(rest, stdout, stderr)
+	case "score":
+		return score.Run(rest, stdout, stderr)
+	case "bench":
+		return bench.Run(rest, stdout, stderr)
+	case "render":
+		return render.Run(rest, stdout, stderr)
+	default:
+		fmt.Fprintf(stderr, "devrites: unknown overhaul tool %q\n\n%s", args[0], Usage)
+		return 2
+	}
+}

package/engine/internal/overhaul/ovio/ovio.go

@@ -0,0 +1,158 @@
+// Package ovio holds the small file, digest and JSON helpers shared by the
+// /overhaul run-area tools. Run records are plain JSON documents read as
+// generic maps so the tools enforce exactly the documented fields.
+package ovio
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+// SHA256Bytes returns the lowercase hex SHA-256 of b.
+func SHA256Bytes(b []byte) string {
+	sum := sha256.Sum256(b)
+	return hex.EncodeToString(sum[:])
+}
+
+// SHA256File returns the lowercase hex SHA-256 of the file's exact bytes.
+func SHA256File(path string) (string, error) {
+	b, err := os.ReadFile(path) // #nosec G304 -- callers pass operator-selected run-area paths
+	if err != nil {
+		return "", err
+	}
+	return SHA256Bytes(b), nil
+}
+
+// LoadJSON decodes a UTF-8 JSON document. Numbers stay json.Number so digests,
+// weights and ratios are never rounded through float64 by accident.
+func LoadJSON(path string) (any, error) {
+	b, err := os.ReadFile(path) // #nosec G304 -- callers pass operator-selected run-area paths
+	if err != nil {
+		return nil, err
+	}
+	dec := json.NewDecoder(bytes.NewReader(b))
+	dec.UseNumber()
+	var v any
+	if err := dec.Decode(&v); err != nil {
+		return nil, fmt.Errorf("%s: invalid JSON (%v)", path, err)
+	}
+	return v, nil
+}
+
+// LoadObject decodes a JSON document that must be an object.
+func LoadObject(path string) (map[string]any, error) {
+	v, err := LoadJSON(path)
+	if err != nil {
+		return nil, err
+	}
+	m, ok := v.(map[string]any)
+	if !ok {
+		return nil, fmt.Errorf("%s: expected a JSON object", path)
+	}
+	return m, nil
+}
+
+// MarshalIndent renders v as two-space indented JSON with sorted object keys
+// and a trailing newline.
+func MarshalIndent(v any) ([]byte, error) {
+	b, err := json.MarshalIndent(v, "", "  ")
+	if err != nil {
+		return nil, err
+	}
+	return append(b, '\n'), nil
+}
+
+// WriteJSON writes v as indented JSON.
+func WriteJSON(path string, v any, perm os.FileMode) error {
+	b, err := MarshalIndent(v)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path, b, perm)
+}
+
+// resolve follows symlinks for the longest existing prefix of path, like
+// Python's os.path.realpath on a path that may not exist yet.
+func resolve(path string) string {
+	path = filepath.Clean(path)
+	if r, err := filepath.EvalSymlinks(path); err == nil {
+		return r
+	}
+	parent := filepath.Dir(path)
+	if parent == path {
+		return path
+	}
+	return filepath.Join(resolve(parent), filepath.Base(path))
+}
+
+// Resolve is the exported form of resolve for callers that compare paths.
+func Resolve(path string) string { return resolve(path) }
+
+// Get walks a dotted path through nested objects; missing keys give nil.
+func Get(v any, dotted string) any {
+	for _, k := range strings.Split(dotted, ".") {
+		m, ok := v.(map[string]any)
+		if !ok {
+			return nil
+		}
+		v = m[k]
+	}
+	return v
+}
+
+// Str returns v as a string, or "" when it is not one.
+func Str(v any) string {
+	s, _ := v.(string)
+	return s
+}
+
+// List returns v as a slice, or nil when it is not one.
+func List(v any) []any {
+	l, _ := v.([]any)
+	return l
+}
+
+// Obj returns v as an object, or nil when it is not one.
+func Obj(v any) map[string]any {
+	m, _ := v.(map[string]any)
+	return m
+}
+
+// Strings returns the string members of a JSON array.
+func Strings(v any) []string {
+	var out []string
+	for _, x := range List(v) {
+		if s, ok := x.(string); ok {
+			out = append(out, s)
+		}
+	}
+	return out
+}
+
+// Truthy mirrors JSON truthiness used by the run records: nil, false, "",
+// 0, empty arrays and empty objects are false.
+func Truthy(v any) bool {
+	switch x := v.(type) {
+	case nil:
+		return false
+	case bool:
+		return x
+	case string:
+		return x != ""
+	case json.Number:
+		return x.String() != "0" && x.String() != "0.0"
+	case float64:
+		return x != 0
+	case []any:
+		return len(x) > 0
+	case map[string]any:
+		return len(x) > 0
+	}
+	return true
+}