@pi-in-go/pigpen-pi-typesafe-api 0.1.0

package/evaluation.go

@@ -0,0 +1,86 @@
+package pitypesafe
+
+import (
+	"encoding/json"
+	"sort"
+)
+
+// MarshalJSON encodes the evaluation the way the original's JSON.stringify(result) does: model, answers
+// (in the order the questions were asked), usage, then elapsedMs.
+func (e *Evaluation) MarshalJSON() ([]byte, error) {
+	raw, err := json.Marshal(e.SystemOneResult)
+	if err != nil {
+		return nil, err
+	}
+	tree, err := ParseJSON(raw)
+	if err != nil {
+		return nil, err
+	}
+	source, ok := tree.(*Object)
+	if !ok {
+		return nil, newError(CodeResponse, "TypeSafe returned an unreadable or unexpected response.")
+	}
+	out := NewObject()
+	if v, ok := source.Get("model"); ok {
+		out.Set("model", v)
+	}
+	if v, ok := source.Get("answers"); ok {
+		out.Set("answers", e.orderAnswers(v))
+	}
+	if v, ok := source.Get("usage"); ok {
+		out.Set("usage", v)
+	}
+	for _, k := range source.Keys() {
+		if !out.Has(k) {
+			out.Set(k, source.vals[k])
+		}
+	}
+	out.Set("elapsedMs", float64(e.ElapsedMs))
+	return EncodeJSON(out)
+}
+
+// orderAnswers reorders an answers object to the question order; unknown ids follow, sorted.
+func (e *Evaluation) orderAnswers(v any) any {
+	answers, ok := v.(*Object)
+	if !ok {
+		return v
+	}
+	ordered := NewObject()
+	for _, id := range e.Order {
+		if a, ok := answers.Get(id); ok {
+			ordered.Set(id, a)
+		}
+	}
+	rest := []string{}
+	for _, id := range answers.Keys() {
+		if !ordered.Has(id) {
+			rest = append(rest, id)
+		}
+	}
+	sort.Strings(rest)
+	for _, id := range rest {
+		a, _ := answers.Get(id)
+		ordered.Set(id, a)
+	}
+	return ordered
+}
+
+// DetailsJSON is MarshalJSON plus an "order" array naming the answer ids in question order. A host that decodes
+// tool details into maps loses key order, so a renderer reads the order from here.
+func (e *Evaluation) DetailsJSON() ([]byte, error) {
+	raw, err := e.MarshalJSON()
+	if err != nil {
+		return nil, err
+	}
+	tree, err := ParseJSON(raw)
+	if err != nil {
+		return nil, err
+	}
+	obj := tree.(*Object)
+	order := make([]any, len(e.Order))
+	for i, id := range e.Order {
+		order[i] = id
+	}
+	obj.Set("order", order)
+	return EncodeJSON(obj)
+}

package/evaluation_schema.json

@@ -0,0 +1,264 @@
+{
+ "type": "object",
+ "required": [
+  "state",
+  "questions"
+ ],
+ "properties": {
+  "state": {
+   "anyOf": [
+    {
+     "type": "string"
+    },
+    {
+     "type": "null"
+    },
+    {
+     "type": "array",
+     "items": {}
+    },
+    {
+     "type": "object",
+     "patternProperties": {
+      "^.*$": {}
+     }
+    }
+   ],
+   "description": "What to judge: text, or an object whose fields the questions name."
+  },
+  "questions": {
+   "type": "object",
+   "patternProperties": {
+    "^.*$": {
+     "anyOf": [
+      {
+       "type": "object",
+       "required": [
+        "type"
+       ],
+       "properties": {
+        "type": {
+         "type": "string",
+         "const": "noul",
+         "description": "Yes or no: the probability the instructions hold."
+        },
+        "instructions": {
+         "anyOf": [
+          {
+           "type": "string"
+          },
+          {
+           "type": "null"
+          },
+          {
+           "type": "array",
+           "items": {}
+          },
+          {
+           "type": "object",
+           "patternProperties": {
+            "^.*$": {}
+           }
+          }
+         ],
+         "description": "One judgment about the whole state, phrased as a question or a statement."
+        },
+        "criteria": {
+         "anyOf": [
+          {
+           "type": "null"
+          },
+          {
+           "type": "object",
+           "properties": {
+            "true": {
+             "anyOf": [
+              {
+               "type": "string"
+              },
+              {
+               "type": "null"
+              },
+              {
+               "type": "array",
+               "items": {}
+              },
+              {
+               "type": "object",
+               "patternProperties": {
+                "^.*$": {}
+               }
+              }
+             ]
+            },
+            "false": {
+             "anyOf": [
+              {
+               "type": "string"
+              },
+              {
+               "type": "null"
+              },
+              {
+               "type": "array",
+               "items": {}
+              },
+              {
+               "type": "object",
+               "patternProperties": {
+                "^.*$": {}
+               }
+              }
+             ]
+            }
+           },
+           "additionalProperties": false
+          }
+         ],
+         "description": "Optional: what counts as yes and what counts as no, { true, false }."
+        }
+       },
+       "additionalProperties": false
+      },
+      {
+       "type": "object",
+       "required": [
+        "type",
+        "criteria"
+       ],
+       "properties": {
+        "type": {
+         "type": "string",
+         "const": "choice",
+         "description": "Pick one criteria label."
+        },
+        "instructions": {
+         "anyOf": [
+          {
+           "type": "string"
+          },
+          {
+           "type": "null"
+          },
+          {
+           "type": "array",
+           "items": {}
+          },
+          {
+           "type": "object",
+           "patternProperties": {
+            "^.*$": {}
+           }
+          }
+         ],
+         "description": "One judgment about the whole state, phrased as a question or a statement."
+        },
+        "criteria": {
+         "type": "object",
+         "patternProperties": {
+          "^.*$": {
+           "anyOf": [
+            {
+             "type": "string"
+            },
+            {
+             "type": "null"
+            },
+            {
+             "type": "array",
+             "items": {}
+            },
+            {
+             "type": "object",
+             "patternProperties": {
+              "^.*$": {}
+             }
+            }
+           ]
+          }
+         },
+         "minProperties": 1,
+         "maxProperties": 64,
+         "description": "The options as a map from label to when it applies: { billing: \"Charges and payments\", other: null }. 1–64 entries."
+        }
+       },
+       "additionalProperties": false
+      },
+      {
+       "type": "object",
+       "required": [
+        "type",
+        "criteria"
+       ],
+       "properties": {
+        "type": {
+         "type": "string",
+         "const": "score",
+         "description": "Rate against the ordered criteria levels."
+        },
+        "instructions": {
+         "anyOf": [
+          {
+           "type": "string"
+          },
+          {
+           "type": "null"
+          },
+          {
+           "type": "array",
+           "items": {}
+          },
+          {
+           "type": "object",
+           "patternProperties": {
+            "^.*$": {}
+           }
+          }
+         ],
+         "description": "One judgment about the whole state, phrased as a question or a statement."
+        },
+        "criteria": {
+         "type": "array",
+         "items": {
+          "anyOf": [
+           {
+            "type": "string"
+           },
+           {
+            "type": "null"
+           },
+           {
+            "type": "array",
+            "items": {}
+           },
+           {
+            "type": "object",
+            "patternProperties": {
+             "^.*$": {}
+            }
+           }
+          ]
+         },
+         "minItems": 2,
+         "maxItems": 32,
+         "description": "Ordered rubric levels, lowest first: [\"neutral\", \"angry\"]. 2–32 levels."
+        }
+       },
+       "additionalProperties": false
+      }
+     ]
+    }
+   },
+   "minProperties": 1,
+   "maxProperties": 32,
+   "description": "Questions keyed by a short id, as an object map, not an array: { \"urgent\": { type: \"noul\", instructions: ... } }."
+  },
+  "model": {
+   "type": "string",
+   "minLength": 1,
+   "maxLength": 100,
+   "description": "Jev model id, e.g. jev-latest. Omit for the default."
+  }
+ },
+ "additionalProperties": false
+}

package/gaps_test.go

@@ -0,0 +1,77 @@
+package pitypesafe
+
+import (
+	"context"
+	"os"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/MichaelKinsy/pigpen/components/typesafe/libraries/typesafe"
+)
+
+// Cases the mutation check showed unproven by the ported twins.
+
+func TestGroupReadableKeyFileIsRefused(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("POSIX permission bits")
+	}
+	isolate(t)
+	_, _ = StoreAPIKey(validKey)
+	_ = os.Chmod(CredentialsPath(), 0o640)
+	if s, _ := KeySituationFor(nil); s.Kind != KeyUnusable {
+		t.Fatalf("a group-readable key file must be unusable: %+v", s)
+	}
+}
+
+func TestAskDefaultDeadlineIsFifteenSeconds(t *testing.T) {
+	var remaining time.Duration
+	judge := judgeFunc(func(ctx context.Context, _ typesafe.SystemOneRequest) (*Evaluation, error) {
+		deadline, ok := ctx.Deadline()
+		if !ok {
+			t.Error("Ask must set a deadline")
+		}
+		remaining = time.Until(deadline)
+		return &Evaluation{SystemOneResult: &typesafe.SystemOneResult{}}, nil
+	})
+	Ask(context.Background(), judge, sampleRequest(), AskOptions{})
+	if remaining < 14*time.Second || remaining > 15*time.Second {
+		t.Fatalf("deadline in %v", remaining)
+	}
+}
+
+func TestAbortedFailureStopsTheBatchToo(t *testing.T) {
+	calls := 0
+	judge := batchJudge(func(ctx context.Context, _ typesafe.SystemOneRequest) (*Evaluation, error) {
+		calls++
+		return nil, newError(CodeAborted, "cancelled")
+	})
+	batch := EvaluateMany(context.Background(), judge, []typesafe.SystemOneRequest{sampleRequest(), sampleRequest(), sampleRequest()}, BatchOptions{Concurrency: 1})
+	if calls != 1 || batch.Skipped != 2 {
+		t.Fatalf("calls=%d skipped=%d", calls, batch.Skipped)
+	}
+}
+
+type batchJudge func(context.Context, typesafe.SystemOneRequest) (*Evaluation, error)
+
+func (f batchJudge) Evaluate(ctx context.Context, r typesafe.SystemOneRequest) (*Evaluation, error) {
+	return f(ctx, r)
+}
+
+func TestOnlyHTTPRejectionsDegradeTheKey(t *testing.T) {
+	isolate(t)
+	t.Setenv("TYPESAFE_API_KEY", "env-key-0123456789abcdef")
+	// A 401 status on a non-HTTP failure code is not a rejection.
+	RecordAuthFailure(&IntegrationError{Code: CodeResponse, Message: "odd", Status: 401}, time.Now())
+	if !mustAuth(t, nil).Usable {
+		t.Fatal("only an HTTP 401 or 403 rejects the key")
+	}
+	RecordAuthFailure(&IntegrationError{Code: CodeHTTP, Message: "forbidden", Status: 403}, time.Now())
+	if mustAuth(t, nil).Usable {
+		t.Fatal("a 403 rejects the key")
+	}
+	RecordAuthFailure(&IntegrationError{Code: CodeHTTP, Message: "gone", Status: 404}, time.Now())
+	if !mustAuth(t, nil).Usable {
+		t.Fatal("a 404 does not")
+	}
+}

package/go.mod

@@ -0,0 +1,9 @@
+module github.com/MichaelKinsy/pigpen/components/pi-typesafe-api
+
+go 1.26
+
+// PiG resolves this requirement to the version-matched staged SDK at build time.
+require github.com/MichaelKinsy/PiG/extensions/sdk v0.4.1
+
+// The shared TypeSafe client (components/typesafe), found through go.work.
+require github.com/MichaelKinsy/pigpen/components/typesafe v0.0.0

package/go.sum

@@ -0,0 +1,2 @@
+github.com/MichaelKinsy/PiG/extensions/sdk v0.4.1/go.mod h1:4j0XhCW0EPOBWvqYE7XKjTvIAahI7K1VyyW3KpjTKj8=
+github.com/MichaelKinsy/PiG/extensions/sdk v0.4.1 h1:Ql9sTkc6dvAXaYQLq5nxqmSZgqu8S0L53ZTs8MkOuxw=

package/helpers_test.go

@@ -0,0 +1,169 @@
+package pitypesafe
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/MichaelKinsy/pigpen/components/typesafe/libraries/typesafe"
+)
+
+// isolate points every path and every key variable at a temporary directory (rule 17): a test never
+// reads or writes the real agent directory, and never sees a key from the environment.
+func isolate(t *testing.T) string {
+	t.Helper()
+	dir := t.TempDir()
+	t.Setenv("HOME", dir)
+	t.Setenv("PIG_HOME", filepath.Join(dir, "pighome"))
+	t.Setenv("PIG_CODING_AGENT_DIR", filepath.Join(dir, "agent"))
+	t.Setenv("PI_CODING_AGENT_DIR", filepath.Join(dir, "pi-agent"))
+	t.Setenv("PIG_USE_PI_DIRS", "")
+	for _, name := range []string{"TYPESAFE_API_KEY", "OPENROUTER_API_KEY", "COMMANDCODE_API_KEY", "GATEWAY_JEV_KEY",
+		"PI_TYPESAFE_ENABLED", "PI_TYPESAFE_MAX_REQUESTS_PER_DAY", "PI_TYPESAFE_MAX_INPUT_TOKENS_PER_DAY", "PI_TYPESAFE_MAX_USD_PER_DAY",
+		"TYPESAFE_BASE_URL", "TYPESAFE_LOG_LEVEL"} {
+		t.Setenv(name, "")
+	}
+	return filepath.Join(dir, "agent")
+}
+
+type doerFunc func(*http.Request) (*http.Response, error)
+
+func (f doerFunc) Do(r *http.Request) (*http.Response, error) { return f(r) }
+
+func jsonResponse(status int, body any, header http.Header) *http.Response {
+	data, _ := json.Marshal(body)
+	if header == nil {
+		header = http.Header{}
+	}
+	header.Set("Content-Type", "application/json")
+	return &http.Response{StatusCode: status, Status: http.StatusText(status), Header: header, Body: io.NopCloser(bytes.NewReader(data)), ContentLength: int64(len(data))}
+}
+
+func rawResponse(status int, body string) *http.Response {
+	return &http.Response{StatusCode: status, Header: http.Header{}, Body: io.NopCloser(bytes.NewReader([]byte(body)))}
+}
+
+// answersFor builds the answers a server returns for questions: the first option with certainty.
+func answersFor(questions typesafe.Questions) map[string]any {
+	answers := map[string]any{}
+	for _, nq := range questions {
+		switch q := nq.Question.(type) {
+		case typesafe.NoulQuestion:
+			answers[nq.Name] = map[string]any{"type": "noul", "noul": 0.9}
+		case typesafe.ChoiceQuestion:
+			probs := map[string]any{}
+			for i, o := range q.Criteria {
+				probs[o.Label] = map[bool]float64{true: 1, false: 0}[i == 0]
+			}
+			answers[nq.Name] = map[string]any{"type": "choice", "choice": q.Criteria[0].Label, "confidence": 1, "probabilities": probs}
+		case typesafe.ScoreQuestion:
+			probs, legend := map[string]any{}, map[string]any{}
+			for i, level := range q.Criteria {
+				probs[itoa(i)] = map[bool]float64{true: 1, false: 0}[i == 0]
+				legend[itoa(i)] = level.Data()
+			}
+			answers[nq.Name] = map[string]any{"type": "score", "score": 0, "confidence": 1, "probabilities": probs, "legend": legend}
+		}
+	}
+	return answers
+}
+
+func itoa(i int) string { b, _ := json.Marshal(i); return string(b) }
+
+// responseFor is the successful reply of the fake TypeSafe server to a request body.
+func responseFor(body []byte) *http.Response {
+	var req struct {
+		Questions json.RawMessage `json:"questions"`
+	}
+	_ = json.Unmarshal(body, &req)
+	questions, err := typesafe.ParseQuestions(req.Questions)
+	if err != nil {
+		return jsonResponse(400, map[string]any{"error": "bad questions"}, nil)
+	}
+	return jsonResponse(200, map[string]any{"model": "jev-test", "answers": answersFor(questions), "usage": map[string]any{"input_tokens": 42, "output_tokens": 0}}, nil)
+}
+
+func requestBody(t *testing.T, r *http.Request) []byte {
+	t.Helper()
+	if r.Body == nil {
+		return nil
+	}
+	data, err := io.ReadAll(r.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return data
+}
+
+// answering is a fake server that answers every questions request and counts the calls.
+type answering struct {
+	mu    sync.Mutex
+	calls int
+	urls  []string
+	seen  []*http.Request
+}
+
+func (a *answering) Do(r *http.Request) (*http.Response, error) {
+	body, _ := io.ReadAll(r.Body)
+	a.mu.Lock()
+	a.calls++
+	a.urls = append(a.urls, r.URL.String())
+	a.seen = append(a.seen, r)
+	a.mu.Unlock()
+	return responseFor(body), nil
+}
+
+func (a *answering) count() int { a.mu.Lock(); defer a.mu.Unlock(); return a.calls }
+
+func sampleRequest() typesafe.SystemOneRequest {
+	return typesafe.SystemOneRequest{State: typesafe.Text("synthetic"), Questions: typesafe.Questions{typesafe.Ask("yes", typesafe.Noul("Is this synthetic?"))}}
+}
+
+func manyQuestions(n int) typesafe.Questions {
+	qs := make(typesafe.Questions, n)
+	for i := range qs {
+		qs[i] = typesafe.Ask("q"+itoa(i), typesafe.Noul("Question "+itoa(i)+"?"))
+	}
+	return qs
+}
+
+func hasCode(err error, code ErrorCode) bool {
+	ie, ok := err.(*IntegrationError)
+	return ok && ie.Code == code
+}
+
+func mustTree(t *testing.T, text string) any {
+	t.Helper()
+	v, err := ParseJSON([]byte(text))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return v
+}
+
+func writeFile(t *testing.T, path, content string, mode os.FileMode) {
+	t.Helper()
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(path, []byte(content), mode); err != nil {
+		t.Fatal(err)
+	}
+	_ = os.Chmod(path, mode)
+}
+
+var _ = context.Background
+
+func regexpMatch(pattern, s string) bool { return regexp.MustCompile(pattern).MatchString(s) }
+
+func contains(s, sub string) bool { return strings.Contains(s, sub) }
+
+func apiError(status int) error { return typesafe.NewAPIError(status, nil, http.Header{}) }

package/hostmodel/hostmodel.go

@@ -0,0 +1,87 @@
+// Package hostmodel is the own-model backend on the PiG Go SDK: it answers TypeSafe's typed questions with the
+// model PiG is configured with, chosen at call time from the request's Context, so a model switch takes effect
+// on the next call without rebuilding the client. Nothing here names a provider or a model.
+//
+// Pass the Evaluator to pitypesafe.New as Options.Evaluator with Backend "ownmodel", and put the tool call's
+// Context in the request context with [WithContext].
+package hostmodel
+
+import (
+	"context"
+
+	sdk "github.com/MichaelKinsy/PiG/extensions/sdk"
+	pitypesafe "github.com/MichaelKinsy/pigpen/components/pi-typesafe-api"
+	"github.com/MichaelKinsy/pigpen/components/typesafe/libraries/ownmodel"
+	"github.com/MichaelKinsy/pigpen/components/typesafe/libraries/pigmodel"
+	"github.com/MichaelKinsy/pigpen/components/typesafe/libraries/typesafe"
+)
+
+type contextKey struct{}
+
+// WithContext returns a context that carries the PiG extension Context the model is read from.
+func WithContext(ctx context.Context, host sdk.Context) context.Context {
+	return context.WithValue(ctx, contextKey{}, host)
+}
+
+// Options configure the backend built for each call.
+type Options struct {
+	// AnswerMode is ownmodel.Probabilities (the default) or ownmodel.Discrete.
+	AnswerMode ownmodel.AnswerMode
+	// NormalizeProbabilities rescales invalid probability distributions to sum to 1.
+	NormalizeProbabilities bool
+	// MalformedRetries is the number of corrective retries when the output fails validation.
+	MalformedRetries int
+	// Logger receives request summaries; nil discards.
+	Logger typesafe.Logger
+}
+
+// Evaluator answers questions with the host's active model. It is safe for concurrent use.
+type Evaluator struct{ opts Options }
+
+// New returns an Evaluator.
+func New(opts Options) *Evaluator { return &Evaluator{opts: opts} }
+
+var _ typesafe.Evaluator = (*Evaluator)(nil)
+
+// ModelRef names the model a Context is configured with.
+type ModelRef struct{ Provider, ID string }
+
+// Active returns the model PiG is configured with, or an error when none is selected. The setup errors here
+// are configuration errors (*pitypesafe.IntegrationError) so their reason reaches the operator unchanged.
+func Active(host sdk.Context) (ModelRef, error) {
+	info, err := host.GetModelInfo()
+	if err != nil {
+		return ModelRef{}, err
+	}
+	if info == nil || info.ID == "" {
+		return ModelRef{}, &pitypesafe.IntegrationError{Code: pitypesafe.CodeConfiguration, Message: "No model is configured in PiG; select one with /model first."}
+	}
+	return ModelRef{Provider: info.Provider, ID: info.ID}, nil
+}
+
+// SystemOne implements typesafe.Evaluator on the model of the Context in ctx.
+func (e *Evaluator) SystemOne(ctx context.Context, req typesafe.SystemOneRequest, opts *typesafe.RequestOptions) (*typesafe.SystemOneResult, error) {
+	host, ok := ctx.Value(contextKey{}).(sdk.Context)
+	if !ok {
+		return nil, &pitypesafe.IntegrationError{Code: pitypesafe.CodeConfiguration, Message: "The own-model backend needs the PiG Context in the request context (hostmodel.WithContext)."}
+	}
+	ref, err := Active(host)
+	if err != nil {
+		return nil, err
+	}
+	model, err := pigmodel.New(host.ModelRegistry(), pigmodel.Ref{Provider: ref.Provider, ID: ref.ID})
+	if err != nil {
+		return nil, err
+	}
+	backend, err := ownmodel.New(ownmodel.Options{
+		Model:                  model,
+		AnswerMode:             e.opts.AnswerMode,
+		NormalizeProbabilities: e.opts.NormalizeProbabilities,
+		MalformedRetries:       e.opts.MalformedRetries,
+		Logger:                 e.opts.Logger,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return backend.SystemOne(ctx, req, opts)
+}