@pi-in-go/pigpen-jev 0.1.0

package/libs/typesafe/libraries/ownmodel/canon.go

@@ -0,0 +1,190 @@
+package ownmodel
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+// The Python oracle serializes with pydantic_core.to_json: compact, no ASCII escaping, key
+// order kept. These helpers reproduce it so that prompts match the oracle byte for byte.
+
+// writeJSONString writes s like serde_json: only the quote, the backslash and the
+// control characters are escaped.
+func writeJSONString(b *strings.Builder, s string) {
+	b.WriteByte('"')
+	for _, r := range s {
+		switch r {
+		case '"':
+			b.WriteString(`\"`)
+		case '\\':
+			b.WriteString(`\\`)
+		case '\b':
+			b.WriteString(`\b`)
+		case '\f':
+			b.WriteString(`\f`)
+		case '\n':
+			b.WriteString(`\n`)
+		case '\r':
+			b.WriteString(`\r`)
+		case '\t':
+			b.WriteString(`\t`)
+		default:
+			if r < 0x20 {
+				fmt.Fprintf(b, `\u%04x`, r)
+			} else {
+				b.WriteRune(r)
+			}
+		}
+	}
+	b.WriteByte('"')
+}
+
+// canonicalJSON re-encodes JSON text the way pydantic_core.to_json encodes the parsed
+// value: compact, key order kept, strings escaped like serde_json. Number literals are
+// kept as written.
+func canonicalJSON(raw []byte) (string, error) {
+	dec := json.NewDecoder(bytes.NewReader(raw))
+	dec.UseNumber()
+	var b strings.Builder
+	if err := canonValue(dec, &b); err != nil {
+		return "", err
+	}
+	if _, err := dec.Token(); err != io.EOF {
+		return "", fmt.Errorf("unexpected data after the JSON value")
+	}
+	return b.String(), nil
+}
+
+func canonValue(dec *json.Decoder, b *strings.Builder) error {
+	tok, err := dec.Token()
+	if err != nil {
+		return err
+	}
+	switch v := tok.(type) {
+	case json.Delim:
+		switch v {
+		case '{':
+			b.WriteByte('{')
+			first := true
+			for dec.More() {
+				kt, err := dec.Token()
+				if err != nil {
+					return err
+				}
+				if !first {
+					b.WriteByte(',')
+				}
+				first = false
+				writeJSONString(b, kt.(string))
+				b.WriteByte(':')
+				if err := canonValue(dec, b); err != nil {
+					return err
+				}
+			}
+			if _, err := dec.Token(); err != nil {
+				return err
+			}
+			b.WriteByte('}')
+		case '[':
+			b.WriteByte('[')
+			first := true
+			for dec.More() {
+				if !first {
+					b.WriteByte(',')
+				}
+				first = false
+				if err := canonValue(dec, b); err != nil {
+					return err
+				}
+			}
+			if _, err := dec.Token(); err != nil {
+				return err
+			}
+			b.WriteByte(']')
+		}
+	case string:
+		writeJSONString(b, v)
+	case json.Number:
+		b.WriteString(v.String())
+	case bool:
+		b.WriteString(strconv.FormatBool(v))
+	case nil:
+		b.WriteString("null")
+	}
+	return nil
+}
+
+// kv and obj are an ordered JSON object, so schemas keep pydantic's key order.
+type kv struct {
+	k string
+	v any
+}
+
+type obj []kv
+
+func writeAny(b *strings.Builder, v any) {
+	switch x := v.(type) {
+	case obj:
+		b.WriteByte('{')
+		for i, e := range x {
+			if i > 0 {
+				b.WriteByte(',')
+			}
+			writeJSONString(b, e.k)
+			b.WriteByte(':')
+			writeAny(b, e.v)
+		}
+		b.WriteByte('}')
+	case []string:
+		b.WriteByte('[')
+		for i, e := range x {
+			if i > 0 {
+				b.WriteByte(',')
+			}
+			writeJSONString(b, e)
+		}
+		b.WriteByte(']')
+	case string:
+		writeJSONString(b, x)
+	case bool:
+		b.WriteString(strconv.FormatBool(x))
+	default:
+		panic(fmt.Sprintf("writeAny: unsupported %T", v))
+	}
+}
+
+// toMap converts an ordered object to the generic form handed to a Model.
+func toMap(v any) any {
+	switch x := v.(type) {
+	case obj:
+		m := make(map[string]any, len(x))
+		for _, e := range x {
+			m[e.k] = toMap(e.v)
+		}
+		return m
+	case []string:
+		out := make([]any, len(x))
+		for i, s := range x {
+			out[i] = s
+		}
+		return out
+	}
+	return v
+}
+
+func sortedKeys[V any](m map[string]V) []string {
+	keys := make([]string, 0, len(m))
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	return keys
+}
+
+// validUTF8 replaces invalid bytes so that a prompt is always valid UTF-8 (Python str cannot hold them).
+func validUTF8(s string) string { return strings.ToValidUTF8(s, "\uFFFD") }

package/libs/typesafe/libraries/ownmodel/convert.go

@@ -0,0 +1,199 @@
+package ownmodel
+
+import (
+	"math"
+
+	"github.com/MichaelKinsy/pigpen/components/typesafe/libraries/typesafe"
+)
+
+// probabilityTolerance is the largest deviation of a probability sum from 1 that is not
+// reported as invalid (the oracle's PROBABILITY_TOLERANCE).
+const probabilityTolerance = 1e-6
+
+// normalizeConfidence returns probs as a distribution; a zero total gives a uniform one.
+func normalizeConfidence(probs []float64) []float64 {
+	total := 0.0
+	for _, p := range probs {
+		total += p
+	}
+	out := make([]float64, len(probs))
+	for i, p := range probs {
+		if total == 0 {
+			out[i] = 1 / float64(len(probs))
+		} else {
+			out[i] = p / total
+		}
+	}
+	return out
+}
+
+// scoreConfidence measures score concentration around its modal score.
+func scoreConfidence(probs []float64) float64 {
+	if len(probs) == 1 {
+		return 1
+	}
+	n := normalizeConfidence(probs)
+	mode := 0
+	for i := range n {
+		if n[i] > n[mode] { // the first maximum, like Python's max
+			mode = i
+		}
+	}
+	distance := 0.0
+	for i, p := range n {
+		distance += p * math.Abs(float64(i-mode))
+	}
+	center := float64(len(n)-1) / 2
+	deviation := 0.0
+	for i := range n {
+		deviation += math.Abs(float64(i) - center)
+	}
+	deviation /= float64(len(n))
+	return math.Max(0, 1-distance/deviation)
+}
+
+// choiceConfidence scales peak choice probability from uniform to certainty.
+func choiceConfidence(probs []float64) float64 {
+	if len(probs) == 1 {
+		return 1
+	}
+	n := normalizeConfidence(probs)
+	peak := n[0]
+	for _, p := range n {
+		peak = math.Max(peak, p)
+	}
+	uniform := 1 / float64(len(n))
+	return (peak - uniform) / (1 - uniform)
+}
+
+// probNorm is the probability distribution of one answer, its error against a sum of 1,
+// and the original distribution when normalization changed it.
+type probNorm struct {
+	Order         []string
+	Probabilities map[string]float64
+	Error         float64
+	Original      map[string]float64
+}
+
+// rescale rescales probabilities to sum to 1, falling back to uniform for a zero total.
+func rescale(order []string, probs map[string]float64) map[string]float64 {
+	total := 0.0
+	for _, a := range order {
+		total += probs[a]
+	}
+	out := make(map[string]float64, len(order))
+	for _, a := range order {
+		if total == 0 {
+			out[a] = 1 / float64(len(order))
+		} else {
+			out[a] = probs[a] / total
+		}
+	}
+	return out
+}
+
+// normalizeProbabilities builds and optionally rescales a distribution. In discrete mode
+// selected is the chosen answer; in probabilities mode probs is the model's distribution.
+func normalizeProbabilities(answers []string, probs map[string]float64, selected string, mode AnswerMode, enabled bool) probNorm {
+	if mode == Discrete {
+		out := make(map[string]float64, len(answers))
+		for _, a := range answers {
+			out[a] = 0
+			if a == selected {
+				out[a] = 1
+			}
+		}
+		return probNorm{Order: answers, Probabilities: out}
+	}
+	original := make(map[string]float64, len(answers))
+	total := 0.0
+	for _, a := range answers {
+		original[a] = probs[a]
+		total += probs[a]
+	}
+	errAmount := math.Abs(total - 1)
+	if !enabled || errAmount <= probabilityTolerance {
+		return probNorm{Order: answers, Probabilities: original, Error: errAmount}
+	}
+	return probNorm{Order: answers, Probabilities: rescale(answers, original), Error: errAmount, Original: original}
+}
+
+// probDebug is the probability diagnostics of an evaluation.
+type probDebug struct {
+	MaxError          float64
+	InvalidProbs      int
+	ProbabilityErrors map[string]float64
+	Original          map[string]map[string]float64
+}
+
+// probabilityDebugData summarizes normalizations keyed by question name (nil for a Noul).
+func probabilityDebugData(norms map[string]*probNorm) probDebug {
+	d := probDebug{ProbabilityErrors: map[string]float64{}, Original: map[string]map[string]float64{}}
+	for name, n := range norms {
+		if n == nil {
+			continue
+		}
+		d.MaxError = math.Max(d.MaxError, n.Error)
+		if n.Error > probabilityTolerance {
+			d.ProbabilityErrors[name] = n.Error
+		}
+		if n.Original != nil {
+			d.Original[name] = n.Original
+		}
+	}
+	d.InvalidProbs = len(d.ProbabilityErrors)
+	return d
+}
+
+// convert turns a validated output into typed answers and the normalizations of the
+// Score and Choice answers.
+func (p *plan) convert(d *decoded, normalize bool) (map[string]typesafe.Answer, map[string]*probNorm, error) {
+	answers := make(map[string]typesafe.Answer, len(p.questions))
+	norms := make(map[string]*probNorm, len(p.questions))
+	for i := range p.questions {
+		q := &p.questions[i]
+		v := d.answers[q.name]
+		norms[q.name] = nil
+		switch q.kind {
+		case typesafe.TypeNoul:
+			prob := v.prob
+			if p.mode == Discrete {
+				prob = 0
+				if v.flag {
+					prob = 1
+				}
+			}
+			answers[q.name] = typesafe.NoulAnswer{Noul: prob}
+		case typesafe.TypeScore:
+			n := normalizeProbabilities(q.labels, v.probs, v.selected, p.mode, normalize)
+			norms[q.name] = &n
+			// The score is an expected value, meaningful only over a distribution summing to 1:
+			// rescale here, whatever normalize says about the reported probabilities.
+			dist := rescale(q.labels, n.Probabilities)
+			score := 0.0
+			ordered := make([]float64, len(q.labels))
+			probs := make(map[int]float64, len(q.labels))
+			legend := make(map[int]any, len(q.labels))
+			for j, label := range q.labels {
+				score += float64(j) * dist[label]
+				ordered[j] = n.Probabilities[label]
+				probs[j] = n.Probabilities[label]
+				legend[j] = q.legend[j]
+			}
+			answers[q.name] = typesafe.ScoreAnswer{Score: score, Confidence: scoreConfidence(ordered), Probabilities: probs, Legend: legend}
+		case typesafe.TypeChoice:
+			n := normalizeProbabilities(q.labels, v.probs, v.selected, p.mode, normalize)
+			norms[q.name] = &n
+			best := 0
+			ordered := make([]float64, len(q.labels))
+			for j, label := range q.labels {
+				ordered[j] = n.Probabilities[label]
+				if n.Probabilities[label] > n.Probabilities[q.labels[best]] {
+					best = j
+				}
+			}
+			answers[q.name] = typesafe.ChoiceAnswer{Choice: q.labels[best], Confidence: choiceConfidence(ordered), Probabilities: n.Probabilities}
+		}
+	}
+	return answers, norms, nil
+}

package/libs/typesafe/libraries/ownmodel/doc.go

@@ -0,0 +1,15 @@
+// Package ownmodel answers the same typed questions as the TypeSafe API with the
+// model PiG is configured with. It is a port of system-one-adapter-python
+// (https://github.com/typesafe-ai/system-one-adapter-python, MIT): the same prompts,
+// the same per-question answer schemas, probability and discrete answer modes,
+// probability normalization, confidence metrics, corrective retries on malformed
+// output, and transient retries through the typesafe retry policy.
+//
+// A [Backend] implements typesafe.Evaluator, so an extension can hold one
+// typesafe.Evaluator and pick the TypeSafe API or the own model by configuration.
+// No provider or model name is built in: the model is whatever the [Model] the
+// caller passes is; package pigmodel provides a Model backed by the PiG Go SDK's
+// ModelRegistry (the model PiG is configured with, its auth, its provider layer).
+//
+// Content is sent to the configured model's provider, not to TypeSafe.
+package ownmodel

package/libs/typesafe/libraries/ownmodel/equivalence_test.go

@@ -0,0 +1,199 @@
+package ownmodel
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"math"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/MichaelKinsy/pigpen/components/typesafe/libraries/typesafe"
+)
+
+// The differential check against the Python oracle: port/equivalence/scenarios/*.json were
+// run through the unmodified system-one-adapter 0.2.1 (e1d4cc9) by record_python.py, which
+// recorded golden/*.json. Here the same scenarios run through the Go backend with the same
+// scripted model responses, and the model calls (prompts and JSON Schema), the answers, the
+// usage counters and the diagnostics must be the same.
+
+type scenario struct {
+	Name             string            `json:"name"`
+	Mode             string            `json:"mode"`
+	Structured       bool              `json:"structured"`
+	Normalize        bool              `json:"normalize"`
+	MalformedRetries int               `json:"malformedRetries"`
+	State            json.RawMessage   `json:"state"`
+	Questions        json.RawMessage   `json:"questions"`
+	Responses        []json.RawMessage `json:"responses"`
+}
+
+type golden struct {
+	Calls []struct {
+		Messages   []Message      `json:"messages"`
+		Schema     map[string]any `json:"schema"`
+		Structured bool           `json:"structured"`
+	} `json:"calls"`
+	Answers      map[string]any `json:"answers"`
+	Usage        map[string]any `json:"usage"`
+	Debug        map[string]any `json:"debug"`
+	RetryReasons []string       `json:"retryReasons"`
+	Error        *struct {
+		Type    string `json:"type"`
+		Message string `json:"message"`
+	} `json:"error"`
+}
+
+func loadJSON(t *testing.T, path string, v any) {
+	t.Helper()
+	raw, err := os.ReadFile(path)
+	noErr(t, err)
+	noErr(t, json.Unmarshal(raw, v))
+}
+
+const (
+	correctionPrefix = "The previous response did not match the required schema: "
+	correctionSuffix = "\nReturn a single JSON object that matches the schema exactly, with no other text."
+)
+
+func TestEquivalence_GoBackendMatchesThePythonOracle(t *testing.T) {
+	dir := filepath.Join("..", "..", "port", "equivalence")
+	files, err := filepath.Glob(filepath.Join(dir, "scenarios", "*.json"))
+	noErr(t, err)
+	if len(files) < 15 {
+		t.Fatalf("expected the recorded scenarios, found %d", len(files))
+	}
+	for _, file := range files {
+		var sc scenario
+		loadJSON(t, file, &sc)
+		var g golden
+		loadJSON(t, filepath.Join(dir, "golden", filepath.Base(file)), &g)
+		t.Run(sc.Name, func(t *testing.T) {
+			qs, err := typesafe.ParseQuestions(sc.Questions)
+			noErr(t, err)
+			var steps []any
+			for _, r := range sc.Responses {
+				if bytes.HasPrefix(bytes.TrimSpace(r), []byte(`"`)) {
+					var s string
+					noErr(t, json.Unmarshal(r, &s))
+					steps = append(steps, s)
+				} else {
+					var buf bytes.Buffer
+					noErr(t, json.Compact(&buf, r))
+					steps = append(steps, buf.String())
+				}
+			}
+			model := newScripted(steps...)
+			b := mustNew(t, Options{Model: model, AnswerMode: AnswerMode(sc.Mode), StructuredOutputs: sc.Structured, NormalizeProbabilities: sc.Normalize, MalformedRetries: sc.MalformedRetries})
+			ev, err := b.Evaluate(context.Background(), typesafe.SystemOneRequest{State: typesafe.Value(sc.State), Questions: qs}, nil)
+
+			// The model calls: prompts and schema.
+			eq(t, len(model.calls), len(g.Calls))
+			for i, want := range g.Calls {
+				got := model.calls[i]
+				eq(t, len(got), len(want.Messages))
+				for j, m := range want.Messages {
+					if m.Role == RoleUser && strings.HasPrefix(m.Content, correctionPrefix) {
+						// The validation error text is this port's own; the frame around it is the oracle's.
+						if got[j].Role != RoleUser || !strings.HasPrefix(got[j].Content, correctionPrefix) || !strings.HasSuffix(got[j].Content, correctionSuffix) {
+							t.Fatalf("call %d message %d: correction frame differs:\n%q", i, j, got[j].Content)
+						}
+						continue
+					}
+					if got[j] != m {
+						t.Fatalf("call %d message %d differs\n got: %s\nwant: %s", i, j, got[j].Content, m.Content)
+					}
+				}
+				eq(t, model.structured[i], want.Structured)
+				sameJSON(t, model.schemas[i], want.Schema, "schema of call "+string(rune('0'+i)))
+			}
+
+			if g.Error != nil {
+				if err == nil {
+					t.Fatalf("the oracle failed with %s, Go succeeded", g.Error.Type)
+				}
+				eq(t, g.Error.Type, "TypeSafeAPIResponseValidationError")
+				mustAs[*MalformedOutputError](t, err)
+				de := mustAs[*DebugError](t, err)
+				eq(t, categories(de.Debug.RetryReasons), g.RetryReasons)
+				return
+			}
+			noErr(t, err)
+			raw, err := json.Marshal(ev.Result.Answers)
+			noErr(t, err)
+			var answers map[string]any
+			noErr(t, json.Unmarshal(raw, &answers))
+			sameJSON(t, answers, g.Answers, "answers")
+			usage := map[string]any{
+				"input_tokens": intOrNil(ev.Usage.InputTokens), "output_tokens": intOrNil(ev.Usage.OutputTokens),
+				"input_tokens_total": intOrNil(ev.Usage.InputTokensTotal), "output_tokens_total": intOrNil(ev.Usage.OutputTokensTotal),
+				"n_retries": float64(ev.Usage.Retries), "n_retries_malformed_structure": float64(ev.Usage.MalformedRetries),
+			}
+			sameJSON(t, usage, g.Usage, "usage")
+			eq(t, categories(ev.Debug.RetryReasons), g.RetryReasons)
+			dbg := map[string]any{"max_error": ev.Debug.MaxError, "invalid_probs": float64(ev.Debug.InvalidProbs), "probability_errors": toAny(ev.Debug.ProbabilityErrors)}
+			if len(ev.Debug.OriginalProbabilities) > 0 {
+				dbg["original_probabilities"] = toAny(ev.Debug.OriginalProbabilities)
+			}
+			sameJSON(t, dbg, g.Debug, "debug")
+		})
+	}
+}
+
+func intOrNil(p *int) any {
+	if p == nil {
+		return nil
+	}
+	return float64(*p)
+}
+
+func toAny(v any) any {
+	raw, _ := json.Marshal(v)
+	var out any
+	_ = json.Unmarshal(raw, &out)
+	return out
+}
+
+// sameJSON compares two decoded JSON values, numbers within a relative 1e-12.
+func sameJSON(t *testing.T, got, want any, what string) {
+	t.Helper()
+	if !jsonEqual(toAny(got), toAny(want)) {
+		g, _ := json.MarshalIndent(got, "", " ")
+		w, _ := json.MarshalIndent(want, "", " ")
+		t.Fatalf("%s differs\n got: %s\nwant: %s", what, g, w)
+	}
+}
+
+func jsonEqual(a, b any) bool {
+	switch x := a.(type) {
+	case float64:
+		y, ok := b.(float64)
+		return ok && math.Abs(x-y) <= 1e-12*math.Max(1, math.Abs(y))
+	case map[string]any:
+		y, ok := b.(map[string]any)
+		if !ok || len(x) != len(y) {
+			return false
+		}
+		for k, v := range x {
+			w, ok := y[k]
+			if !ok || !jsonEqual(v, w) {
+				return false
+			}
+		}
+		return true
+	case []any:
+		y, ok := b.([]any)
+		if !ok || len(x) != len(y) {
+			return false
+		}
+		for i := range x {
+			if !jsonEqual(x[i], y[i]) {
+				return false
+			}
+		}
+		return true
+	}
+	return a == b
+}