@pi-in-go/pigpen-jev 0.1.0

package/extensions/jev/output_test.go

@@ -0,0 +1,187 @@
+package jev_test
+
+import (
+	"strings"
+	"testing"
+)
+
+// Output judge: src/index.ts tool_result, src/output.ts.
+
+func outputHost(t *testing.T, body any, extra map[string]any) (*Host, *fakeJev) {
+	t.Helper()
+	e := newEnv(t)
+	t.Setenv("TYPESAFE_API_KEY", testKey)
+	srv := newFakeJev(t, always(body))
+	e.writeGlobal(t, jevConfig(srv, extra))
+	return start(t, e, newHostState(), HostOptions{}), srv
+}
+
+func lastText(t *testing.T, patched []string) string {
+	t.Helper()
+	if len(patched) == 0 {
+		t.Fatal("result was not patched")
+	}
+	return patched[len(patched)-1]
+}
+
+func TestOutput_LeakAppendsNoticeAndWarns(t *testing.T) {
+	h, _ := outputHost(t, outBody(0.94, "no_failure", 0.9), nil)
+	patched, none := h.toolResult("bash", bash("cat .env"), text("API_KEY=abc"), false)
+	if none {
+		t.Fatal("result not patched")
+	}
+	if patched[0] != "API_KEY=abc" {
+		t.Errorf("original block changed: %q", patched)
+	}
+	want := "[pi-jev] Jev flagged this output as containing a secret (0.94). Do not repeat the value in a reply, a file, or a command; refer to it by name instead."
+	if got := lastText(t, patched); got != want {
+		t.Errorf("notice = %q", got)
+	}
+	if !h.anyNotification("warning: jev: bash output may carry a secret (0.94)") {
+		t.Errorf("notifications = %q", h.notifications())
+	}
+	if h.lastStatus() != "jev: leak (bash)" {
+		t.Errorf("status = %q", h.lastStatus())
+	}
+}
+
+func TestOutput_FailureClassAdviceTable(t *testing.T) {
+	advice := map[string]string{
+		"transient":   "retrying the same command unchanged is reasonable",
+		"environment": "fix the environment (missing tool, port, or service) before retrying",
+		"code_bug":    "fix the code or types; retrying unchanged will not help",
+		"permission":  "access was denied; change what is being accessed or ask the user",
+		"user_error":  "the invocation itself was wrong; fix the command",
+	}
+	for class, line := range advice {
+		t.Run(class, func(t *testing.T) {
+			h, _ := outputHost(t, outBody(0.01, class, 1.0), nil)
+			patched, none := h.toolResult("bash", bash("x"), text("boom"), true)
+			if none {
+				t.Fatal("no advice")
+			}
+			want := "[pi-jev] Jev read this as a " + class + " failure (confidence 1.00): " + line + "."
+			if got := lastText(t, patched); got != want {
+				t.Errorf("notice = %q, want %q", got, want)
+			}
+			if len(h.notifications()) != 0 {
+				t.Errorf("advice notified: %q", h.notifications())
+			}
+			if h.lastStatus() != "jev: advice (bash)" {
+				t.Errorf("status = %q", h.lastStatus())
+			}
+		})
+	}
+}
+
+func TestOutput_NoFailureAndUnknownClassSayNothing(t *testing.T) {
+	for _, class := range []string{"no_failure", "brand_new_class"} {
+		h, _ := outputHost(t, outBody(0.01, class, 1.0), nil)
+		if _, none := h.toolResult("bash", bash("x"), text("ok"), false); !none {
+			t.Errorf("class %q patched the result", class)
+		}
+	}
+}
+
+func TestOutput_LowClassConfidenceIsSilent(t *testing.T) {
+	h, _ := outputHost(t, outBody(0.01, "environment", 0.42), nil)
+	if _, none := h.toolResult("bash", bash("x"), text("fatal: not a git repository"), true); !none {
+		t.Fatal("advice below output.minConfidence")
+	}
+}
+
+func TestOutput_LeakThresholdIsInclusiveAndLeakWinsOverClass(t *testing.T) {
+	h, _ := outputHost(t, outBody(0.9, "transient", 1.0), nil)
+	patched, _ := h.toolResult("bash", bash("x"), text("t"), false)
+	mustContain(t, "notice", lastText(t, patched), "containing a secret (0.90)")
+}
+
+func TestOutput_OnlyConfiguredToolsAreJudged(t *testing.T) {
+	h, srv := outputHost(t, outBody(0.99, "no_failure", 1), nil)
+	h.toolResult("read", map[string]any{"path": "a"}, text("secret"), false)
+	if srv.count() != 0 {
+		t.Fatal("judged the output of read")
+	}
+}
+
+func TestOutput_EmptyOrNonTextResultIsNotJudged(t *testing.T) {
+	h, srv := outputHost(t, outBody(0.99, "no_failure", 1), nil)
+	h.toolResult("bash", bash("x"), text("  \n "), false)
+	h.toolResult("bash", bash("x"), []any{map[string]any{"type": "image", "data": "AAAA"}}, false)
+	if srv.count() != 0 {
+		t.Fatalf("requests = %d", srv.count())
+	}
+}
+
+func TestOutput_JoinsTextBlocksWithNewline(t *testing.T) {
+	h, srv := outputHost(t, outBody(0.01, "no_failure", 1), nil)
+	h.toolResult("bash", bash("x"), []any{map[string]any{"type": "text", "text": "one"}, map[string]any{"type": "text", "text": "two"}}, false)
+	st := stateOf(t, srv.first(t))
+	if st["output"] != "one\ntwo" || st["is_error"] != false || st["tool"] != "bash" {
+		t.Errorf("state = %v", st)
+	}
+}
+
+func TestOutput_RequestQuestions(t *testing.T) {
+	h, srv := outputHost(t, outBody(0.01, "no_failure", 1), nil)
+	h.toolResult("bash", bash("x"), text("hi"), false)
+	qs := srv.first(t).Body["questions"].(map[string]any)
+	leak := qs["leaks_secret"].(map[string]any)
+	if leak["type"] != "noul" || leak["instructions"] != "Does `output` contain a secret or credential that must not be written to a session transcript?" {
+		t.Errorf("leaks_secret = %v", leak)
+	}
+	fc := qs["failure_class"].(map[string]any)
+	crit := fc["criteria"].(map[string]any)
+	if fc["type"] != "choice" || len(crit) != 6 || crit["transient"] != "A network or resource hiccup that may succeed on a retry" {
+		t.Errorf("failure_class = %v", fc)
+	}
+	if got := strings.Join(questionOrder(srv.first(t).Raw), ","); got != "leaks_secret,failure_class" {
+		t.Errorf("order = %s", got)
+	}
+}
+
+func TestOutput_LongOutputIsElided(t *testing.T) {
+	h, srv := outputHost(t, outBody(0.01, "no_failure", 1), nil)
+	h.toolResult("bash", bash("x"), text(strings.Repeat("o", 2600)), false)
+	want := strings.Repeat("o", 2000) + "…[600 chars elided]"
+	if got := stateOf(t, srv.first(t))["output"]; got != want {
+		t.Errorf("output not elided: len %d", len(got.(string)))
+	}
+}
+
+func TestOutput_ArgumentsAreElidedAt400(t *testing.T) {
+	h, srv := outputHost(t, outBody(0.01, "no_failure", 1), nil)
+	h.toolResult("bash", map[string]any{"command": strings.Repeat("c", 500)}, text("hi"), false)
+	args := stateOf(t, srv.first(t))["arguments"].(map[string]any)
+	if args["command"] != strings.Repeat("c", 400)+"…[100 chars elided]" {
+		t.Errorf("command = %v", args["command"])
+	}
+}
+
+func TestOutput_IdenticalOutputIsJudgedOnce(t *testing.T) {
+	h, srv := outputHost(t, outBody(0.01, "no_failure", 1), nil)
+	h.toolResult("bash", bash("a"), text("same"), false)
+	h.toolResult("bash", bash("b"), text("same"), false)
+	if srv.count() != 1 {
+		t.Fatalf("requests = %d, want 1", srv.count())
+	}
+	h.toolResult("bash", bash("c"), text("different"), false)
+	if srv.count() != 2 {
+		t.Fatalf("requests = %d, want 2", srv.count())
+	}
+}
+
+func TestOutput_DisabledSkips(t *testing.T) {
+	h, srv := outputHost(t, outBody(0.99, "no_failure", 1), map[string]any{"output": map[string]any{"enabled": false}})
+	h.toolResult("bash", bash("x"), text("secret"), false)
+	if srv.count() != 0 {
+		t.Fatal("judged with output.enabled=false")
+	}
+}
+
+func TestOutput_LeakThresholdConfigurable(t *testing.T) {
+	h, _ := outputHost(t, outBody(0.5, "no_failure", 1), map[string]any{"output": map[string]any{"leakThreshold": 0.4}})
+	if _, none := h.toolResult("bash", bash("x"), text("t"), false); none {
+		t.Fatal("0.5 < configured 0.4 threshold not flagged")
+	}
+}

package/extensions/jev/ownmodel_test.go

@@ -0,0 +1,118 @@
+package jev_test
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+)
+
+// The default judge: the model PiG is configured with, through the shared client's
+// own-model backend (prompted JSON answers, validated strictly). No API key and no
+// endpoint of ours: the host authenticates and calls the provider.
+
+func gateAnswers(d, e, b float64, impactLevel int) string {
+	probs := map[string]float64{"0": 0, "1": 0, "2": 0, "3": 0}
+	probs[string(rune('0'+impactLevel))] = 1
+	a, _ := json.Marshal(map[string]any{"answers": map[string]any{"destructive": d, "exfiltration": e, "beyond_scope": b, "impact": probs}})
+	return string(a)
+}
+
+func outAnswers(leak float64, class string) string {
+	probs := map[string]float64{"transient": 0, "environment": 0, "code_bug": 0, "permission": 0, "user_error": 0, "no_failure": 0}
+	probs[class] = 1
+	a, _ := json.Marshal(map[string]any{"answers": map[string]any{"leaks_secret": leak, "failure_class": probs}})
+	return string(a)
+}
+
+type modelCalls struct{ models, requests []map[string]any }
+
+func modelHost(t *testing.T, reply func(request map[string]any) []map[string]any, cfg map[string]any) (*Host, *modelCalls, *env) {
+	t.Helper()
+	e := newEnv(t)
+	c := map[string]any{"enabled": true, "acknowledged": true, "display": "plain"}
+	for k, v := range cfg {
+		c[k] = v
+	}
+	e.writeGlobal(t, c)
+	mc := &modelCalls{}
+	hs := newHostState()
+	h := start(t, e, hs, HostOptions{ModelStream: func(model, request map[string]any) []map[string]any {
+		mc.models, mc.requests = append(mc.models, model), append(mc.requests, request)
+		return reply(request)
+	}})
+	return h, mc, e
+}
+
+func TestOwnModel_GateJudgesWithTheSessionModel(t *testing.T) {
+	h, mc, e := modelHost(t, func(map[string]any) []map[string]any { return ModelText(gateAnswers(0.99, 0.1, 0.1, 3)) }, nil)
+	block, _ := h.toolCall("bash", bash("rm -rf src"))
+	if block {
+		t.Fatal("shadow mode blocked")
+	}
+	if !h.anyNotification("warning: jev shadow: bash - destructive 0.99, impact 3.00/3 at confidence 1.00") {
+		t.Errorf("notifications = %q", h.notifications())
+	}
+	if len(mc.models) != 1 || mc.models[0]["provider"] != "acme" || (mc.models[0]["modelId"] != "judge-1" && mc.models[0]["id"] != "judge-1") {
+		t.Errorf("the judge is not the session model: %v", mc.models)
+	}
+	all, _ := json.Marshal(mc.requests[0])
+	mustContain(t, "model request", string(all), "rm -rf src", e.cwd, "Is this action destructive?")
+}
+
+func TestOwnModel_OutputJudgeAppendsTheNotice(t *testing.T) {
+	h, _, _ := modelHost(t, func(map[string]any) []map[string]any { return ModelText(outAnswers(0.95, "no_failure")) }, nil)
+	patched, none := h.toolResult("bash", bash("cat .env"), text("API_KEY=abc"), false)
+	if none || !strings.Contains(patched[len(patched)-1], "containing a secret (0.95)") {
+		t.Errorf("patched = %q", patched)
+	}
+}
+
+func TestOwnModel_ConfiguredModelOverridesTheSessionModel(t *testing.T) {
+	h, mc, _ := modelHost(t, func(map[string]any) []map[string]any { return ModelText(gateAnswers(0, 0, 0, 0)) }, map[string]any{"model": "other/cheap-1"})
+	h.toolCall("bash", bash("ls"))
+	if len(mc.models) != 1 || mc.models[0]["provider"] != "other" {
+		t.Errorf("models = %v", mc.models)
+	}
+}
+
+func TestOwnModel_MalformedAnswerFailsOpen(t *testing.T) {
+	h, _, _ := modelHost(t, func(map[string]any) []map[string]any { return ModelText("I think this is fine.") }, nil)
+	if block, _ := h.toolCall("bash", bash("rm -rf /")); block {
+		t.Fatal("an unusable answer blocked a call")
+	}
+	if !h.anyNotification("(failing open)") || strings.Contains(h.lastStatus(), "clear") {
+		t.Errorf("notifications %q status %q", h.notifications(), h.lastStatus())
+	}
+}
+
+func TestOwnModel_ProviderErrorFailsOpen(t *testing.T) {
+	h, _, _ := modelHost(t, func(map[string]any) []map[string]any { return ModelError("provider down") }, nil)
+	if block, _ := h.toolCall("bash", bash("ls")); block {
+		t.Fatal("blocked")
+	}
+	if !h.anyNotification("provider down") || h.lastStatus() != "jev: unavailable (failing open)" {
+		t.Errorf("notifications %q status %q", h.notifications(), h.lastStatus())
+	}
+}
+
+func TestOwnModel_JevAskUsesTheSessionModel(t *testing.T) {
+	h, mc, _ := modelHost(t, func(map[string]any) []map[string]any {
+		return ModelText(`{"answers":{"relevant":0.9}}`)
+	}, nil)
+	got, details := askResult(t, h, map[string]any{"state": "the diff", "questions": []any{map[string]any{"id": "relevant", "type": "noul", "instructions": "Is this relevant?"}}})
+	if details["ok"] != true || !strings.Contains(got, "relevant: yes 0.90") || len(mc.models) != 1 {
+		t.Errorf("got %q %v (%d model calls)", got, details, len(mc.models))
+	}
+}
+
+func TestOwnModel_NoModelNoJudge(t *testing.T) {
+	e := newEnv(t)
+	e.writeGlobal(t, map[string]any{"enabled": true, "acknowledged": true, "display": "plain"})
+	hs := newHostState()
+	hs.model = map[string]any{}
+	h := start(t, e, hs, HostOptions{})
+	h.toolCall("bash", bash("ls"))
+	if !h.anyNotification("needs a model") {
+		t.Errorf("notifications = %q", h.notifications())
+	}
+}

package/extensions/jev/render.go

@@ -0,0 +1,136 @@
+package jev
+
+import (
+	"fmt"
+	"strings"
+)
+
+// Two display styles. "plain" is the original's wording, character for character
+// (the equivalence scenarios run in it); "rich" is the default: one glyph per state,
+// a verdict card with a bar per question, and the destination named wherever content
+// is about to leave. Only wording and layout differ, never behavior.
+
+const statusKey = "jev"
+
+func (x *ext) plain() bool { return x.cfgDisplay() == "plain" }
+
+func (x *ext) cfgDisplay() string {
+	x.mu.Lock()
+	defer x.mu.Unlock()
+	return x.cfg.Display
+}
+
+// statusText renders a footer status. rest is the plain text after "jev: ".
+func statusText(plain bool, glyph, rest string) string {
+	if plain {
+		return "jev: " + rest
+	}
+	if glyph == "" {
+		return "⚖ jev: " + rest
+	}
+	return "⚖ jev " + glyph + " " + rest
+}
+
+func bar(p float64) string {
+	const width = 10
+	n := int(p*width + 0.5)
+	n = max(0, min(width, n))
+	return strings.Repeat("█", n) + strings.Repeat("·", width-n)
+}
+
+// card is the verdict card of the gate: one line per question with a bar, the
+// value, its threshold and a mark on the ones that crossed it.
+func card(v gateVerdict, c config) string {
+	t := c.Gate.BlockOn
+	line := func(label string, p, limit float64) string {
+		mark := "  "
+		cmp := "<"
+		if p >= limit {
+			mark, cmp = "⚠ ", "≥"
+		}
+		return fmt.Sprintf("  %s%-13s %s  %s  %s %s", mark, label, bar(p), toFixed2(p), cmp, toFixed2(limit))
+	}
+	lines := []string{
+		line("destructive", v.Destructive, t.Destructive),
+		line("exfiltration", v.Exfiltration, t.Exfiltration),
+		line("beyond scope", v.BeyondScope, t.BeyondScope),
+	}
+	if v.HasImpact {
+		mark, cmp := "  ", "<"
+		if v.Impact >= t.Impact && v.ImpactConfidence >= c.Gate.MinConfidence {
+			mark, cmp = "⚠ ", "≥"
+		}
+		lines = append(lines, fmt.Sprintf("  %s%-13s %s  %s/3  %s %s  (confidence %s)", mark, "impact", bar(v.Impact/3), toFixed2(v.Impact), cmp, toFixed2(t.Impact), toFixed2(v.ImpactConfidence)))
+	}
+	return strings.Join(lines, "\n")
+}
+
+// preview names what a tool call touches, for the confirmation dialog.
+func preview(tool string, input any) string {
+	m, _ := input.(map[string]any)
+	for _, k := range []string{"command", "path", "file_path", "filePath"} {
+		if s, ok := m[k].(string); ok && s != "" {
+			return cut(collapse(s), 120)
+		}
+	}
+	return ""
+}
+
+func (x *ext) shadowNote(tool string, v gateVerdict, c config) string {
+	if c.Display == "plain" {
+		return fmt.Sprintf("jev shadow: %s - %s", tool, v.summary())
+	}
+	return fmt.Sprintf("⚠ Jev flagged %s (shadow mode: reported, not blocked)\n%s", tool, card(v, c))
+}
+
+func (x *ext) headlessNote(tool string, v gateVerdict, c config) string {
+	tail := "(headless: not blocking; set gate.blockWithoutUI to block)"
+	if c.Display == "plain" {
+		return fmt.Sprintf("jev: %s - %s %s", tool, v.summary(), tail)
+	}
+	return fmt.Sprintf("⚠ Jev flagged %s: %s %s", tool, v.summary(), tail)
+}
+
+func (x *ext) confirmMessage(tool string, input any, v gateVerdict, c config, dest string) string {
+	if c.Display == "plain" {
+		return fmt.Sprintf("%s\n%s\n\nRun it anyway?", tool, v.summary())
+	}
+	head := tool
+	if p := preview(tool, input); p != "" {
+		head += "  " + p
+	}
+	return fmt.Sprintf("%s\n\n%s\n\nJudged by %s\nRun it anyway?", head, card(v, c), dest)
+}
+
+func (x *ext) leakNote(tool string, leak float64, c config) string {
+	if c.Display == "plain" {
+		return fmt.Sprintf("jev: %s output may carry a secret (%s)", tool, toFixed2(leak))
+	}
+	return fmt.Sprintf("⚠ Jev: %s output may carry a secret (%s). The model was told not to repeat it.", tool, toFixed2(leak))
+}
+
+func (x *ext) failureNote(msg string, c config) string {
+	if c.Display == "plain" {
+		return fmt.Sprintf("pi-jev: %s (failing open)", msg)
+	}
+	return fmt.Sprintf("✗ Jev could not judge: %s\nThe tool call ran unjudged (failing open).", msg)
+}
+
+// disclosure says what leaves the machine and where it goes. It is shown until the
+// user sets "acknowledged", and by /jev on before consent.
+func disclosure(c config, dest string) string {
+	var b strings.Builder
+	b.WriteString("What leaves this machine for each judgment:\n")
+	b.WriteString("  • the working directory and the tool name\n")
+	fmt.Fprintf(&b, "  • your last message (first %d characters)\n", userRequestChars)
+	fmt.Fprintf(&b, "  • the tool arguments (%s; long fields are cut at %d characters, and write/edit arguments are file content)\n", strings.Join(c.Gate.Tools, "/"), c.Gate.ArgumentChars)
+	fmt.Fprintf(&b, "  • %s output (first %d characters)\n", strings.Join(c.Output.Tools, "/"), c.Output.OutputChars)
+	fmt.Fprintf(&b, "Also sent: text the model passes to jev_ask (up to %d characters) and text you pass to /jev check.\n", c.MaxStateChars)
+	fmt.Fprintf(&b, "Destination: %s\n", dest)
+	b.WriteString("If the judge is unavailable, tool calls run unjudged (it fails open). A judgment is probabilistic advice, not a sandbox.")
+	return b.String()
+}
+
+func startupDisclosure(c config, dest string) string {
+	return fmt.Sprintf("Jev is ON (%s mode). %s\nSet \"acknowledged\": true in pi-jev.json to hide this notice.", c.Gate.Mode, disclosure(c, dest))
+}