@pi-in-go/pigpen-jev 0.1.0

package/libs/typesafe/libraries/ownmodel/twins_test.go

@@ -0,0 +1,169 @@
+package ownmodel
+
+import (
+	"bufio"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// skipRules name, per upstream test file, why its cases have no Go twin. A case that is
+// not claimed by twin()/skipTwin() must match a rule; the reasons are specific to the file.
+var skipRules = []struct{ prefix, reason string }{
+	{"tests/test_client_with_live_apis.py::", "replays live OpenAI, Anthropic and Gemini calls from vcr cassettes; the Go backend has no provider clients. What these check (prompts and schema shape, answers) is checked by the differential goldens recorded from the oracle (TestEquivalence_GoBackendMatchesThePythonOracle); a live check needs an owner-supplied key"},
+	{"tests/test_openai_transports.py::", "OpenAI Responses and Chat Completions transports of the oracle's provider; PiG's model access (package pigmodel) replaces the transport"},
+	{"tests/test_gemini_transports.py::", "Gemini Interactions API transport of the oracle's provider; replaced by PiG's model access"},
+	{"tests/test_provider_lifecycle.py::", "close/aclose of the provider SDK clients the oracle constructs; the Go backend and pigmodel.Model own no client or connection"},
+	{"tests/test_provider_requests.py::", "the exact SDK request arguments of the oracle's OpenAI, Anthropic and Gemini providers; PiG's model access builds the provider request"},
+	{"tests/test_provider_retries.py::", "retry budgets inside the provider SDKs' HTTP clients; the transient retry policy here is the shared typesafe.Retry (tested through the Backend)"},
+	{"tests/test_provider_nonanswers.py::", "refusals, incomplete generations and output-limit stops in the provider-specific response shapes; PiG's model access reports them as a stop reason, mapped and tested in package pigmodel (TestComplete_StopReasons*)"},
+	{"tests/utils/test_error_handling.py::test_status_errors_map_and_preserve_status_and_body", "translation of the OpenAI/Anthropic/Gemini SDK exceptions to SDK errors; a Model returns typesafe errors directly"},
+	{"tests/utils/test_error_handling.py::test_timeout_and_connection_errors_map", "translation of provider SDK timeout and connection exceptions; a Model returns typesafe errors directly"},
+	{"tests/utils/test_error_handling.py::test_unknown_and_sdk_errors_pass_through", "pass-through rules of the provider SDK exception translators"},
+	{"tests/utils/test_error_handling.py::test_translating_context_manager_reraises_translated_error", "the translating() context manager of the provider layer"},
+}
+
+func TestEveryUpstreamCaseHasATwinOrANamedSkip(t *testing.T) {
+	f, err := os.Open(filepath.Join("..", "..", "port", "twins", "system-one-adapter-python.txt"))
+	noErr(t, err)
+	defer f.Close()
+	var cases []string
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		if line := strings.TrimSpace(sc.Text()); line != "" && !strings.HasPrefix(line, "#") {
+			cases = append(cases, line)
+		}
+	}
+	ported, skipped := scanTwinClaims(t, ".")
+	known := map[string]bool{}
+	for _, c := range cases {
+		known[c] = true
+	}
+	var missing, dup, unknown []string
+	byRule := map[string]int{}
+	nPorted, nNamed := 0, 0
+	for _, c := range cases {
+		_, isPorted := ported[c]
+		_, isSkipped := skipped[c]
+		switch {
+		case isPorted && isSkipped:
+			dup = append(dup, c)
+		case isPorted:
+			nPorted++
+		case isSkipped:
+			nNamed++
+		default:
+			matched := false
+			for _, r := range skipRules {
+				if strings.HasPrefix(c, r.prefix) {
+					byRule[r.prefix]++
+					matched = true
+					break
+				}
+			}
+			if !matched {
+				missing = append(missing, c)
+			}
+		}
+	}
+	for id := range ported {
+		if !known[id] {
+			unknown = append(unknown, id)
+		}
+	}
+	for id := range skipped {
+		if !known[id] {
+			unknown = append(unknown, id)
+		}
+	}
+	sort.Strings(unknown)
+	if len(missing)+len(dup)+len(unknown) > 0 {
+		t.Fatalf("%d cases without a twin or a skip rule:\n%s\nclaimed as both ported and skipped:\n%s\nunknown ids:\n%s",
+			len(missing), strings.Join(missing, "\n"), strings.Join(dup, "\n"), strings.Join(unknown, "\n"))
+	}
+	ruled := 0
+	for _, n := range byRule {
+		ruled += n
+	}
+	t.Logf("system-one-adapter-python: %d cases: %d ported twins, %d named skips, %d skipped by file rule", len(cases), nPorted, nNamed, ruled)
+}
+
+func scanTwinClaims(t *testing.T, dir string) (ported, skipped map[string]string) {
+	t.Helper()
+	ported, skipped = map[string]string{}, map[string]string{}
+	files, err := filepath.Glob(filepath.Join(dir, "*_test.go"))
+	noErr(t, err)
+	fset := token.NewFileSet()
+	consts := map[string]string{}
+	var parsed []*ast.File
+	for _, file := range files {
+		f, err := parser.ParseFile(fset, file, nil, 0)
+		noErr(t, err)
+		parsed = append(parsed, f)
+		ast.Inspect(f, func(n ast.Node) bool { // string constants used to build ids
+			if vs, ok := n.(*ast.ValueSpec); ok {
+				for i, name := range vs.Names {
+					if i < len(vs.Values) {
+						if lit, ok := vs.Values[i].(*ast.BasicLit); ok && lit.Kind == token.STRING {
+							s, _ := strconv.Unquote(lit.Value)
+							consts[name.Name] = s
+						}
+					}
+				}
+			}
+			return true
+		})
+	}
+	var eval func(e ast.Expr) string
+	eval = func(e ast.Expr) string {
+		switch x := e.(type) {
+		case *ast.BasicLit:
+			s, err := strconv.Unquote(x.Value)
+			noErr(t, err)
+			return s
+		case *ast.Ident:
+			s, ok := consts[x.Name]
+			if !ok {
+				t.Fatalf("unknown constant %s in a twin id", x.Name)
+			}
+			return s
+		case *ast.BinaryExpr:
+			return eval(x.X) + eval(x.Y)
+		}
+		t.Fatalf("twin ids must be string literals, constants or their concatenation, got %T", e)
+		return ""
+	}
+	for _, f := range parsed {
+		ast.Inspect(f, func(n ast.Node) bool {
+			call, ok := n.(*ast.CallExpr)
+			if !ok {
+				return true
+			}
+			id, ok := call.Fun.(*ast.Ident)
+			if !ok || (id.Name != "twin" && id.Name != "skipTwin") {
+				return true
+			}
+			args := call.Args[1:]
+			reason := ""
+			dst := ported
+			if id.Name == "skipTwin" {
+				reason = eval(args[0])
+				if strings.TrimSpace(reason) == "" {
+					t.Fatalf("%s: skipTwin needs a reason", fset.Position(call.Pos()))
+				}
+				args, dst = args[1:], skipped
+			}
+			for _, a := range args {
+				dst[eval(a)] = reason
+			}
+			return true
+		})
+	}
+	return ported, skipped
+}

package/libs/typesafe/libraries/ownmodel/utils_test.go

@@ -0,0 +1,125 @@
+package ownmodel
+
+import (
+	"math"
+	"testing"
+
+	"github.com/MichaelKinsy/pigpen/components/typesafe/libraries/typesafe"
+)
+
+func approx(t testing.TB, got, want float64) {
+	t.Helper()
+	if math.Abs(got-want) > 1e-9*math.Max(1, math.Abs(want)) {
+		t.Fatalf("got %v, want %v", got, want)
+	}
+}
+
+func TestConfidenceMetrics(t *testing.T) {
+	twin(t,
+		"tests/utils/test_confidence_metrics.py::test_confidence_metrics[score_confidence-probabilities0-0.0]",
+		"tests/utils/test_confidence_metrics.py::test_confidence_metrics[score_confidence-probabilities1-0.0]",
+		"tests/utils/test_confidence_metrics.py::test_confidence_metrics[score_confidence-probabilities2-0.55]",
+		"tests/utils/test_confidence_metrics.py::test_confidence_metrics[choice_confidence-probabilities3-0.0]",
+		"tests/utils/test_confidence_metrics.py::test_confidence_metrics[choice_confidence-probabilities4-0.0]",
+		"tests/utils/test_confidence_metrics.py::test_confidence_metrics[choice_confidence-probabilities5-0.64]",
+		"tests/utils/test_confidence_metrics.py::test_confidence_metrics[score_confidence-probabilities6-1.0]",
+		"tests/utils/test_confidence_metrics.py::test_confidence_metrics[choice_confidence-probabilities7-1.0]")
+	for _, tc := range []struct {
+		metric func([]float64) float64
+		probs  []float64
+		want   float64
+	}{
+		{scoreConfidence, []float64{0.2, 0.2, 0.2, 0.2, 0.2}, 0.0},
+		{scoreConfidence, []float64{0.04, 0.04, 0.04, 0.04, 0.04}, 0.0},
+		{scoreConfidence, []float64{0.01, 0.02, 0.07, 0.3, 0.6}, 0.55},
+		{choiceConfidence, []float64{0.5, 0.5}, 0.0},
+		{choiceConfidence, []float64{0.2, 0.2}, 0.0},
+		{choiceConfidence, []float64{0.82, 0.18}, 0.64},
+		{scoreConfidence, []float64{1.0}, 1.0},
+		{choiceConfidence, []float64{1.0}, 1.0},
+	} {
+		approx(t, tc.metric(tc.probs), tc.want)
+	}
+	// A zero total falls back to a uniform distribution.
+	approx(t, choiceConfidence([]float64{0, 0}), 0)
+	approx(t, scoreConfidence([]float64{0, 0, 0}), 0)
+}
+
+func TestProbabilityNormalizationAndDebugData(t *testing.T) {
+	twin(t,
+		"tests/utils/test_probability_normalization.py::test_probability_normalization_and_debug_data[False-0.2-0.2-None-0.6-expected_probability_errors0]",
+		"tests/utils/test_probability_normalization.py::test_probability_normalization_and_debug_data[True-0.2-0.5-expected_originals1-0.6-expected_probability_errors1]",
+		"tests/utils/test_probability_normalization.py::test_probability_normalization_and_debug_data[False-0.50000025-0.50000025-None-5e-07-expected_probability_errors2]")
+	for _, tc := range []struct {
+		enabled       bool
+		raw, want     float64
+		originals     bool
+		maxError      float64
+		invalidCounts int
+	}{{false, 0.2, 0.2, false, 0.6, 2}, {true, 0.2, 0.5, true, 0.6, 2}, {false, 0.50000025, 0.50000025, false, 5e-7, 0}} {
+		score := normalizeProbabilities([]string{"0", "1"}, map[string]float64{"0": tc.raw, "1": tc.raw}, "", Probabilities, tc.enabled)
+		genre := normalizeProbabilities([]string{"fiction", "nonfiction"}, map[string]float64{"fiction": tc.raw, "nonfiction": tc.raw}, "", Probabilities, tc.enabled)
+		approx(t, score.Probabilities["0"], tc.want)
+		approx(t, score.Probabilities["1"], tc.want)
+		approx(t, genre.Probabilities["fiction"], tc.want)
+		d := probabilityDebugData(map[string]*probNorm{"positive": nil, "stars": &score, "genre": &genre})
+		approx(t, d.MaxError, tc.maxError)
+		eq(t, d.InvalidProbs, tc.invalidCounts)
+		eq(t, len(d.ProbabilityErrors), tc.invalidCounts)
+		for _, e := range d.ProbabilityErrors {
+			approx(t, e, tc.maxError)
+		}
+		if tc.originals {
+			eq(t, d.Original, map[string]map[string]float64{"stars": {"0": 0.2, "1": 0.2}, "genre": {"fiction": 0.2, "nonfiction": 0.2}})
+		} else if len(d.Original) != 0 {
+			t.Fatalf("no originals expected, got %v", d.Original)
+		}
+	}
+}
+
+func TestDiscreteNormalizationSelectsOneAnswer(t *testing.T) {
+	n := normalizeProbabilities([]string{"a", "b", "c"}, nil, "b", Discrete, false)
+	eq(t, n.Probabilities, map[string]float64{"a": 0, "b": 1, "c": 0})
+	eq(t, n.Error, 0.0)
+}
+
+func TestRetriesSucceedAfterTransientError(t *testing.T) {
+	twin(t, "tests/utils/test_error_handling.py::test_retries_succeed_after_transient_error")
+	calls := 0
+	res, n, err := retryTransient(t, typesafe.DefaultRetryPolicy(), func() (string, error) {
+		calls++
+		if calls == 1 {
+			return "", providerError(503)
+		}
+		return "success", nil
+	})
+	noErr(t, err)
+	eq(t, res, "success")
+	eq(t, calls, 2)
+	eq(t, n, 1)
+}
+
+func TestNonRetryableErrorIsNotRetried(t *testing.T) {
+	twin(t, "tests/utils/test_error_handling.py::test_non_retryable_error_is_not_retried")
+	calls := 0
+	_, _, err := retryTransient(t, typesafe.DefaultRetryPolicy(), func() (string, error) { calls++; return "", providerError(400) })
+	mustAs[*typesafe.BadRequestError](t, err)
+	eq(t, calls, 1)
+}
+
+func TestRetriesAreExhaustedAndReasonsRecorded(t *testing.T) {
+	twin(t, "tests/utils/test_error_handling.py::test_retries_are_exhausted_and_reasons_recorded")
+	model := newScripted(providerError(503))
+	b := mustNew(t, Options{Model: model, Retry: fastRetry(2)})
+	_, err := evaluate(t, b, "s", answerNoul("q"), nil)
+	mustAs[*typesafe.InternalServerError](t, err)
+	de := mustAs[*DebugError](t, err)
+	eq(t, categories(de.Debug.RetryReasons), []string{"provider_error", "provider_error"})
+}
+
+// retryTransient runs fn through a Backend-like transient retry (a helper on the Backend's loop).
+func retryTransient(t testing.TB, p typesafe.RetryPolicy, fn func() (string, error)) (string, int, error) {
+	t.Helper()
+	p.BackoffInitial, p.BackoffJitter = durMS(1), 0
+	return runTransient(p, fn)
+}

package/libs/typesafe/libraries/pigmodel/pigmodel.go

@@ -0,0 +1,264 @@
+// Package pigmodel is an ownmodel.Model backed by the PiG Go SDK's model access: the
+// model PiG is configured with, authenticated and called through the host's provider
+// layer (Context.ModelRegistry: Find, GetApiKeyAndHeaders, Complete). No provider or
+// model name is built in.
+//
+// The SDK's sdk.ModelRegistry value (what ctx.ModelRegistry() returns) satisfies [Registry] as is:
+//
+//	info, err := ctx.GetModelInfo()                       // the active model; nil when none is set
+//	m, err := pigmodel.New(ctx.ModelRegistry(), pigmodel.Ref{Provider: info.Provider, ID: info.ID})
+//	backend, err := ownmodel.New(ownmodel.Options{Model: m})
+//
+// PiG's model access has no native structured-output mode, so this Model serves
+// prompted mode only: a request with Structured set fails with a *typesafe.TypeSafeError.
+package pigmodel
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/MichaelKinsy/pigpen/components/typesafe/libraries/ownmodel"
+	"github.com/MichaelKinsy/pigpen/components/typesafe/libraries/typesafe"
+)
+
+// Registry is the part of the PiG Go SDK's ModelRegistry this package uses.
+type Registry interface {
+	Find(providerID, modelID string) map[string]any
+	GetApiKeyAndHeaders(model map[string]any) (map[string]any, error)
+	Complete(model, request, options map[string]any) map[string]any
+}
+
+// Ref names a model of the host's registry.
+type Ref struct {
+	Provider string
+	ID       string
+}
+
+// Model is an ownmodel.Model on a [Registry].
+type Model struct {
+	reg Registry
+	ref Ref
+
+	mu    sync.Mutex
+	model map[string]any
+}
+
+var _ ownmodel.Model = (*Model)(nil)
+
+// New returns a Model for ref. The registry is asked for the model and its auth on
+// the first request, not here.
+func New(reg Registry, ref Ref) (*Model, error) {
+	if reg == nil {
+		return nil, &typesafe.TypeSafeError{Message: "A model registry is required."}
+	}
+	if ref.ID == "" {
+		return nil, &typesafe.TypeSafeError{Message: "A model ID is required."}
+	}
+	return &Model{reg: reg, ref: ref}, nil
+}
+
+// Name returns the model ID.
+func (m *Model) Name() string { return m.ref.ID }
+
+// Complete sends the conversation through the registry. The system message becomes the
+// request's system prompt. A stop reason of "error" is a *typesafe.APIError when the
+// provider's message starts with an HTTP status (so the retry policy classifies it), else a
+// retryable *typesafe.APIConnectionError carrying the provider's message; "aborted" is an
+// *typesafe.APIUserAbortError; "length" (output limit), "toolUse" and any other non-"stop"
+// reason are plain *typesafe.TypeSafeError values that name the reason. A cancelled
+// ctx returns an *typesafe.APIUserAbortError without waiting for the host.
+func (m *Model) Complete(ctx context.Context, req ownmodel.Request) (ownmodel.Result, error) {
+	if req.Structured {
+		return ownmodel.Result{}, &typesafe.TypeSafeError{Message: "PiG's model access has no native structured-output mode; use prompted mode (StructuredOutputs false)."}
+	}
+	if ctx.Err() != nil {
+		return ownmodel.Result{}, typesafe.NewAbortError(context.Cause(ctx))
+	}
+	type outcome struct {
+		res ownmodel.Result
+		err error
+	}
+	done := make(chan outcome, 1)
+	go func() {
+		res, err := m.complete(req)
+		done <- outcome{res, err}
+	}()
+	select {
+	case o := <-done:
+		return o.res, o.err
+	case <-ctx.Done():
+		// The host call cannot be interrupted from here; its result is dropped when it arrives.
+		return ownmodel.Result{}, typesafe.NewAbortError(context.Cause(ctx))
+	}
+}
+
+func (m *Model) resolve() (map[string]any, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	if m.model != nil {
+		return m.model, nil
+	}
+	found := m.reg.Find(m.ref.Provider, m.ref.ID)
+	if found == nil {
+		return nil, &typesafe.TypeSafeError{Message: fmt.Sprintf("The model %s/%s was not found in the model registry.", m.ref.Provider, m.ref.ID)}
+	}
+	m.model = found
+	return found, nil
+}
+
+func str(v any) string {
+	s, _ := v.(string)
+	return s
+}
+
+// providerAndID reads a registry model the way the SDK does: the provider is a string or an
+// object with an id, the model ID is "modelId" or "id".
+func providerAndID(model map[string]any) (provider, id string) {
+	provider = str(model["provider"])
+	if provider == "" {
+		if p, ok := model["provider"].(map[string]any); ok {
+			provider = str(p["id"])
+		}
+	}
+	id = str(model["modelId"])
+	if id == "" {
+		id = str(model["id"])
+	}
+	return provider, id
+}
+
+func textBlocks(text string) []any {
+	return []any{map[string]any{"type": "text", "text": text}}
+}
+
+func zeroUsage() map[string]any {
+	return map[string]any{
+		"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0, "totalTokens": 0,
+		"cost": map[string]any{"input": 0, "output": 0, "cacheRead": 0, "cacheWrite": 0, "total": 0},
+	}
+}
+
+func buildRequest(model map[string]any, msgs []ownmodel.Message) map[string]any {
+	provider, id := providerAndID(model)
+	now := time.Now().UnixMilli()
+	var system []string
+	var out []any
+	for _, msg := range msgs {
+		switch msg.Role {
+		case ownmodel.RoleSystem:
+			system = append(system, msg.Content)
+		case ownmodel.RoleAssistant:
+			out = append(out, map[string]any{
+				"role": "assistant", "content": textBlocks(msg.Content),
+				"api": str(model["api"]), "provider": provider, "model": id,
+				"usage": zeroUsage(), "stopReason": "stop", "timestamp": now,
+			})
+		default:
+			out = append(out, map[string]any{"role": "user", "content": textBlocks(msg.Content), "timestamp": now})
+		}
+	}
+	req := map[string]any{"messages": out}
+	if len(system) > 0 {
+		req["systemPrompt"] = strings.Join(system, "\n\n")
+	}
+	return req
+}
+
+func (m *Model) complete(req ownmodel.Request) (ownmodel.Result, error) {
+	model, err := m.resolve()
+	if err != nil {
+		return ownmodel.Result{}, err
+	}
+	auth, err := m.reg.GetApiKeyAndHeaders(model)
+	if err != nil {
+		return ownmodel.Result{}, &typesafe.TypeSafeError{Message: "Could not resolve the model's credentials: " + err.Error(), Cause: err}
+	}
+	options := map[string]any{}
+	if auth != nil {
+		if ok, present := auth["ok"].(bool); present && !ok {
+			msg := str(auth["error"])
+			if msg == "" {
+				msg = "the registry reported no credentials"
+			}
+			return ownmodel.Result{}, &typesafe.TypeSafeError{Message: "Could not resolve the model's credentials: " + msg}
+		}
+		if key := str(auth["apiKey"]); key != "" {
+			options["apiKey"] = key
+		}
+		if headers, ok := auth["headers"]; ok && headers != nil {
+			options["headers"] = headers
+		}
+	}
+	reply := m.reg.Complete(model, buildRequest(model, req.Messages), options)
+	if reply == nil {
+		return ownmodel.Result{}, &typesafe.TypeSafeError{Message: "The host returned no model result."}
+	}
+	return mapReply(reply)
+}
+
+var leadingStatus = regexp.MustCompile(`^\s*([1-5]\d\d)\b\s*(.*)$`)
+
+func toInt(v any) *int {
+	var n int
+	switch x := v.(type) {
+	case float64:
+		n = int(x)
+	case int:
+		n = x
+	case int64:
+		n = int(x)
+	default:
+		return nil
+	}
+	return &n
+}
+
+// mapReply converts the terminal assistant message of the host to a Result or an error.
+func mapReply(reply map[string]any) (ownmodel.Result, error) {
+	reason := str(reply["stopReason"])
+	switch reason {
+	case "", "stop":
+	case "error":
+		msg := strings.TrimSpace(str(reply["errorMessage"]))
+		if match := leadingStatus.FindStringSubmatch(msg); match != nil {
+			if status, _ := strconv.Atoi(match[1]); status >= 400 {
+				var body any
+				if match[2] != "" {
+					body = match[2]
+				}
+				return ownmodel.Result{}, typesafe.NewAPIError(status, body, nil)
+			}
+		}
+		if msg == "" {
+			msg = "the model reported an error"
+		}
+		return ownmodel.Result{}, typesafe.NewConnectionError(errors.New(msg))
+	case "aborted":
+		return ownmodel.Result{}, typesafe.NewAbortError(context.Canceled)
+	case "length":
+		return ownmodel.Result{}, &typesafe.TypeSafeError{Message: "The model stopped at its output token limit before finishing the answer. Ask fewer questions or raise the model's maximum output tokens."}
+	case "toolUse":
+		return ownmodel.Result{}, &typesafe.TypeSafeError{Message: "The model tried to call a tool instead of answering."}
+	default:
+		return ownmodel.Result{}, &typesafe.TypeSafeError{Message: fmt.Sprintf("The model stopped with reason %q instead of answering.", reason)}
+	}
+	var text strings.Builder
+	if blocks, ok := reply["content"].([]any); ok {
+		for _, b := range blocks {
+			if block, ok := b.(map[string]any); ok && str(block["type"]) == "text" {
+				text.WriteString(str(block["text"]))
+			}
+		}
+	}
+	res := ownmodel.Result{Text: text.String()}
+	if usage, ok := reply["usage"].(map[string]any); ok {
+		res.InputTokens, res.OutputTokens = toInt(usage["input"]), toInt(usage["output"])
+	}
+	return res, nil
+}