@pi-in-go/pigpen-jev 0.1.0

package/libs/typesafe/libraries/ownmodel/helpers_test.go

@@ -0,0 +1,155 @@
+package ownmodel
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"reflect"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/MichaelKinsy/pigpen/components/typesafe/libraries/typesafe"
+)
+
+// twin marks a test as the twin of upstream cases: ids are pytest node ids from
+// port/twins/system-one-adapter-python.txt. skipTwin records cases with no Go counterpart.
+func twin(t testing.TB, ids ...string) { t.Helper() }
+
+func skipTwin(t testing.TB, reason string, ids ...string) {
+	t.Helper()
+	t.Skip(reason)
+}
+
+func ptr[T any](v T) *T { return &v }
+
+// scripted is the analog of the oracle's _ScriptedProvider: it answers with a scripted
+// sequence of payloads (encoded to JSON), raw strings, Results and errors; the last step
+// repeats once the script is exhausted.
+type scripted struct {
+	mu         sync.Mutex
+	steps      []any
+	usage      [2]int
+	calls      [][]Message
+	structured []bool
+	schemas    []map[string]any
+}
+
+func newScripted(steps ...any) *scripted { return &scripted{steps: steps, usage: [2]int{11, 7}} }
+
+func (s *scripted) Name() string { return "fake-model" }
+
+func (s *scripted) Complete(ctx context.Context, req Request) (Result, error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.calls = append(s.calls, append([]Message(nil), req.Messages...))
+	s.structured = append(s.structured, req.Structured)
+	s.schemas = append(s.schemas, req.Schema)
+	i := len(s.calls) - 1
+	if i >= len(s.steps) {
+		i = len(s.steps) - 1
+	}
+	switch step := s.steps[i].(type) {
+	case error:
+		return Result{}, step
+	case Result:
+		return step, nil
+	case string:
+		return Result{Text: step, InputTokens: ptr(s.usage[0]), OutputTokens: ptr(s.usage[1])}, nil
+	default:
+		raw, err := json.Marshal(step)
+		if err != nil {
+			panic(err)
+		}
+		return Result{Text: string(raw), InputTokens: ptr(s.usage[0]), OutputTokens: ptr(s.usage[1])}, nil
+	}
+}
+
+func (s *scripted) callCount() int {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return len(s.calls)
+}
+
+func (s *scripted) lastCall() []Message {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.calls[len(s.calls)-1]
+}
+
+// providerError is the error a real model source returns after translating an HTTP failure.
+func providerError(status int) error {
+	return typesafe.NewAPIError(status, map[string]any{"message": "unavailable"}, http.Header{})
+}
+
+func mustNew(t testing.TB, opts Options) *Backend {
+	t.Helper()
+	b, err := New(opts)
+	if err != nil {
+		t.Fatalf("New: %v", err)
+	}
+	return b
+}
+
+func evaluate(t testing.TB, b *Backend, state any, qs typesafe.Questions, opts *typesafe.RequestOptions) (*Evaluation, error) {
+	t.Helper()
+	return b.Evaluate(context.Background(), typesafe.SystemOneRequest{State: typesafe.EntryOf(state), Questions: qs}, opts)
+}
+
+func eq[T any](t testing.TB, got, want T) {
+	t.Helper()
+	if !reflect.DeepEqual(got, want) {
+		t.Fatalf("got %#v, want %#v", got, want)
+	}
+}
+
+func noErr(t testing.TB, err error) {
+	t.Helper()
+	if err != nil {
+		t.Fatalf("unexpected error %T: %v", err, err)
+	}
+}
+
+func contains(t testing.TB, s, sub string) {
+	t.Helper()
+	if !strings.Contains(s, sub) {
+		t.Fatalf("%q does not contain %q", s, sub)
+	}
+}
+
+func mustAs[T any](t testing.TB, err error) T {
+	t.Helper()
+	var target T
+	if !errors.As(err, &target) {
+		t.Fatalf("error %T (%v) is not %T", err, err, target)
+	}
+	return target
+}
+
+func questionSet() typesafe.Questions {
+	return typesafe.Questions{
+		typesafe.Ask("positive", typesafe.Noul("The review is positive.")),
+		typesafe.Ask("stars", typesafe.Score("Rating.", "Bad.", "Good.")),
+		typesafe.Ask("genre", typesafe.Choice("Genre.", typesafe.Opt("fiction", "A story."), typesafe.Opt("nonfiction", "Facts."))),
+	}
+}
+
+func answerNoul(name string) typesafe.Questions {
+	return typesafe.Questions{typesafe.Ask(name, typesafe.Noul("The review is positive."))}
+}
+
+func categories(reasons []RetryReason) []string {
+	out := []string{}
+	for _, r := range reasons {
+		out = append(out, r.Category)
+	}
+	return out
+}
+
+func fastRetry(max int) typesafe.RetryOverrides {
+	return typesafe.RetryOverrides{MaxRetries: typesafe.Ptr(max), BackoffInitial: typesafe.Ptr(durMS(1)), BackoffJitter: typesafe.Ptr(0.0)}
+}
+
+func durMS(n int) time.Duration { return time.Duration(n) * time.Millisecond }

package/libs/typesafe/libraries/ownmodel/mutation_test.go

@@ -0,0 +1,31 @@
+package ownmodel
+
+import "testing"
+
+// Added by the mutation check (port/mutations.json: tolerance-loosened): the oracle's tolerance
+// is 1e-6, so a deviation just above it is normalized and just below it is left alone.
+func TestNormalizationToleranceIsOneMillionth(t *testing.T) {
+	answers := []string{"a", "b"}
+	over := normalizeProbabilities(answers, map[string]float64{"a": 0.500005, "b": 0.500005}, "", Probabilities, true)
+	approx(t, over.Probabilities["a"], 0.5)
+	if over.Original == nil {
+		t.Fatal("a deviation of 1e-5 must be normalized and its original kept")
+	}
+	under := normalizeProbabilities(answers, map[string]float64{"a": 0.5000004, "b": 0.5000004}, "", Probabilities, true)
+	approx(t, under.Probabilities["a"], 0.5000004)
+	if under.Original != nil {
+		t.Fatal("a deviation of 8e-7 is within the tolerance")
+	}
+}
+
+// Added by the mutation check (default-transient-retries-2): the oracle's default is no
+// transient retries, so a failing provider is called once unless the caller opts in.
+func TestNoTransientRetriesByDefault(t *testing.T) {
+	model := newScripted(providerError(500))
+	b := mustNew(t, Options{Model: model})
+	_, err := evaluate(t, b, "state", answerNoul("positive"), nil)
+	if err == nil {
+		t.Fatal("expected the provider error")
+	}
+	eq(t, model.callCount(), 1)
+}

package/libs/typesafe/libraries/ownmodel/ownmodel.go

@@ -0,0 +1,225 @@
+package ownmodel
+
+import (
+	"context"
+	"time"
+
+	"github.com/MichaelKinsy/pigpen/components/typesafe/libraries/typesafe"
+)
+
+// Role is the author of a message.
+type Role string
+
+// Message roles.
+const (
+	RoleSystem    Role = "system"
+	RoleUser      Role = "user"
+	RoleAssistant Role = "assistant"
+)
+
+// Message is one chat message in provider-neutral form.
+type Message struct {
+	Role    Role   `json:"role"`
+	Content string `json:"content"`
+}
+
+// Request is one model request: the conversation and the JSON Schema of the answer.
+// When Structured is false the schema is also in the system prompt and the model is
+// only asked for JSON text (prompted mode).
+type Request struct {
+	Messages   []Message
+	Schema     map[string]any
+	Structured bool
+}
+
+// Result is the raw text the model produced and its token counts; nil counts are
+// unreported.
+type Result struct {
+	Text         string `json:"text"`
+	InputTokens  *int   `json:"input_tokens"`
+	OutputTokens *int   `json:"output_tokens"`
+}
+
+// Model performs one model request. It is the seam between this package and a model
+// source; implementations return typesafe errors so the retry policy can classify
+// them: *typesafe.APIError (with a status) and *typesafe.APIConnectionError are
+// retried per policy, anything else is final. A refusal, an incomplete generation or
+// an output-limit stop must be a plain *typesafe.TypeSafeError (never retried and
+// never treated as malformed output).
+type Model interface {
+	// Name is the model name reported in the result.
+	Name() string
+	Complete(ctx context.Context, req Request) (Result, error)
+}
+
+// AnswerMode selects what the model is asked to return.
+type AnswerMode string
+
+// Answer modes.
+const (
+	// Probabilities asks for a probability per label (a probability for a Noul).
+	Probabilities AnswerMode = "probabilities"
+	// Discrete asks for exactly one value per question.
+	Discrete AnswerMode = "discrete"
+)
+
+// Options configure a [Backend].
+type Options struct {
+	// Model answers the questions. Required unless Resolve is set.
+	Model Model
+	// Resolve, when set, picks the Model for a request that names one
+	// (SystemOneRequest.Model); a request that names none uses Model.
+	Resolve func(name string) (Model, error)
+	// AnswerMode is Probabilities (the default when empty) or Discrete.
+	AnswerMode AnswerMode
+	// StructuredOutputs asks the Model for its native structured-output mode; when false
+	// the schema goes in the prompt and the JSON is validated here.
+	StructuredOutputs bool
+	// NormalizeProbabilities rescales invalid probability distributions to sum to 1.
+	NormalizeProbabilities bool
+	// MalformedRetries is the number of corrective retries when the output fails
+	// validation. Default 0.
+	MalformedRetries int
+	// Retry overrides the transient-error policy, applied over a policy with no retries
+	// (the default is no retries, as in the Python adapter).
+	Retry typesafe.RetryOverrides
+	// Logger receives request summaries; nil discards.
+	Logger typesafe.Logger
+}
+
+// Backend answers questions with a Model. It is safe for concurrent use.
+type Backend struct {
+	opts   Options
+	mode   AnswerMode
+	policy typesafe.RetryPolicy
+	logger typesafe.Logger
+}
+
+var _ typesafe.Evaluator = (*Backend)(nil)
+
+// New validates the options. It returns a *typesafe.TypeSafeError for an unknown
+// answer mode, a negative MalformedRetries, or no Model and no Resolve.
+func New(opts Options) (*Backend, error) {
+	mode := opts.AnswerMode
+	if mode == "" {
+		mode = Probabilities
+	}
+	if mode != Probabilities && mode != Discrete {
+		return nil, &typesafe.TypeSafeError{Message: "AnswerMode must be 'probabilities' or 'discrete'."}
+	}
+	if opts.MalformedRetries < 0 {
+		return nil, &typesafe.TypeSafeError{Message: "MalformedRetries must be >= 0."}
+	}
+	if opts.Model == nil && opts.Resolve == nil {
+		return nil, &typesafe.TypeSafeError{Message: "An LLM model is required: set Options.Model (or Options.Resolve)."}
+	}
+	base := typesafe.DefaultRetryPolicy()
+	base.MaxRetries = 0 // the oracle's default: no transient retries
+	policy, err := base.Resolve(opts.Retry)
+	if err != nil {
+		return nil, err
+	}
+	logger := opts.Logger
+	if logger == nil {
+		logger = discardLogger{}
+	}
+	return &Backend{opts: opts, mode: mode, policy: policy, logger: logger}, nil
+}
+
+type discardLogger struct{}
+
+func (discardLogger) Debug(string, ...any) {}
+func (discardLogger) Info(string, ...any)  {}
+func (discardLogger) Warn(string, ...any)  {}
+func (discardLogger) Error(string, ...any) {}
+
+// SystemOne implements [typesafe.Evaluator]: the questions are validated, one prompt is
+// built from the state, the model is called (with corrective and transient retries),
+// and its output is converted to typed answers. Usage token counts the model did not
+// report are zero here; Evaluate keeps them unknown.
+func (b *Backend) SystemOne(ctx context.Context, req typesafe.SystemOneRequest, opts *typesafe.RequestOptions) (*typesafe.SystemOneResult, error) {
+	ev, err := b.Evaluate(ctx, req, opts)
+	if err != nil {
+		return nil, err
+	}
+	return ev.Result, nil
+}
+
+// Usage is the token and retry accounting of one evaluation. A nil count is unknown;
+// a total is nil if any attempt did not report that count.
+type Usage struct {
+	InputTokens       *int
+	OutputTokens      *int
+	InputTokensTotal  *int
+	OutputTokensTotal *int
+	// Retries counts transient-error retries, MalformedRetries corrective retries.
+	Retries          int
+	MalformedRetries int
+	Latency          time.Duration
+}
+
+// RetryReason records why one retry happened.
+type RetryReason struct {
+	// Category is "provider_error" or "malformed_structure".
+	Category string
+	Message  string
+}
+
+// Attempt records one model call, including failed ones.
+type Attempt struct {
+	Messages   []Message
+	Schema     map[string]any
+	Structured bool
+	// Response is nil when the call failed before returning.
+	Response  *Result
+	ModelName string
+	// Error and ErrorType describe a failed call.
+	Error     string
+	ErrorType string
+}
+
+// Debug holds the diagnostics of one evaluation.
+type Debug struct {
+	LLMAttempts  []Attempt
+	RetryReasons []RetryReason
+	// MaxError is the largest deviation of a probability sum from 1; InvalidProbs
+	// counts the questions beyond the 1e-6 tolerance, ProbabilityErrors names them.
+	MaxError          float64
+	InvalidProbs      int
+	ProbabilityErrors map[string]float64
+	// OriginalProbabilities holds distributions that normalization changed.
+	OriginalProbabilities map[string]map[string]float64
+}
+
+// Evaluation is a result with usage and diagnostics.
+type Evaluation struct {
+	Result *typesafe.SystemOneResult
+	Usage  Usage
+	Debug  Debug
+}
+
+// Evaluate is [Backend.SystemOne] with usage and diagnostics. An input error (no
+// questions, a bad question, a missing model) is a plain *typesafe.TypeSafeError; a
+// failure after the model was called is a *DebugError carrying the attempt history.
+func (b *Backend) Evaluate(ctx context.Context, req typesafe.SystemOneRequest, opts *typesafe.RequestOptions) (*Evaluation, error) {
+	return b.evaluate(ctx, req, opts)
+}
+
+// MalformedOutputError is returned (inside a *DebugError) when the model's output still
+// fails validation after the last corrective retry. Message is
+// "Model output did not match the schema: <details>" and Cause the validation error.
+type MalformedOutputError struct{ *typesafe.TypeSafeError }
+
+func (e *MalformedOutputError) Unwrap() error { return e.TypeSafeError }
+
+// DebugError carries the attempt history of a failed evaluation and unwraps to the
+// cause: a *typesafe.TypeSafeError for malformed output that is still invalid after the
+// last corrective retry (message "Model output did not match the schema: ..."), or the
+// error the Model returned.
+type DebugError struct {
+	Err   error
+	Debug Debug
+}
+
+func (e *DebugError) Error() string { return e.Err.Error() }
+func (e *DebugError) Unwrap() error { return e.Err }