@effect-agent/ai-typesafe 0.1.0-beta.100

package/src/TypeSafeSchema.ts

@@ -0,0 +1,275 @@
+/**
+ * Request and response schemas for TypeSafe's System One HTTP API.
+ *
+ * @since 0.1.0
+ */
+import * as Schema from "effect/Schema";
+
+/**
+ * Text or JSON objects and arrays used as state and question instructions.
+ *
+ * @category schemas
+ * @since 0.1.0
+ */
+export const Content = Schema.Union([Schema.String, Schema.JsonObject, Schema.Array(Schema.Json)]);
+
+/** @category models
+ * @since 0.1.0
+ */
+export type Content = typeof Content.Type;
+
+/**
+ * A choice between named options. A null rubric uses the option name alone.
+ *
+ * @category schemas
+ * @since 0.1.0
+ */
+export const ChoiceQuestion = Schema.Struct({
+  type: Schema.Literal("choice"),
+  instructions: Content,
+  criteria: Schema.Record(Schema.String, Schema.NullOr(Schema.String)).check(
+    Schema.isMinProperties(1),
+  ),
+});
+
+/** @category models
+ * @since 0.1.0
+ */
+export type ChoiceQuestion = typeof ChoiceQuestion.Type;
+
+/**
+ * A rating along at least two ordered, zero-indexed level descriptions.
+ *
+ * @category schemas
+ * @since 0.1.0
+ */
+export const ScoreQuestion = Schema.Struct({
+  type: Schema.Literal("score"),
+  instructions: Content,
+  criteria: Schema.Array(Schema.String).check(Schema.isMinLength(2)),
+});
+
+/** @category models
+ * @since 0.1.0
+ */
+export type ScoreQuestion = typeof ScoreQuestion.Type;
+
+/**
+ * A yes/no judgment, with optional descriptions of either outcome.
+ *
+ * @category schemas
+ * @since 0.1.0
+ */
+export const NoulQuestion = Schema.Struct({
+  type: Schema.Literal("noul"),
+  instructions: Content,
+  criteria: Schema.optionalKey(
+    Schema.Struct({
+      true: Schema.optionalKey(Schema.String),
+      false: Schema.optionalKey(Schema.String),
+    }),
+  ),
+});
+
+/** @category models
+ * @since 0.1.0
+ */
+export type NoulQuestion = typeof NoulQuestion.Type;
+
+/** @category schemas
+ * @since 0.1.0
+ */
+export const Question = Schema.Union([ChoiceQuestion, ScoreQuestion, NoulQuestion]);
+
+/** @category models
+ * @since 0.1.0
+ */
+export type Question = typeof Question.Type;
+
+/** @category schemas
+ * @since 0.1.0
+ */
+export const Questions = Schema.Record(Schema.String, Question);
+
+/** @category models
+ * @since 0.1.0
+ */
+export type Questions = typeof Questions.Type;
+
+/** @category schemas
+ * @since 0.1.0
+ */
+export const EvaluateRequest = Schema.Struct({
+  model: Schema.String,
+  state: Content,
+  questions: Questions,
+});
+
+/**
+ * Evaluation input. Keep question literals with `satisfies Questions` when
+ * storing questions separately from the call to `evaluate`.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type EvaluateRequest<Q extends Questions = Questions> = Omit<
+  typeof EvaluateRequest.Type,
+  "questions"
+> & { readonly questions: Q };
+
+/**
+ * A finite probability or confidence in the inclusive range [0, 1].
+ *
+ * @category schemas
+ * @since 0.1.0
+ */
+export const Probability = Schema.Finite.check(Schema.isBetween({ minimum: 0, maximum: 1 }));
+
+/**
+ * A selected option and its full distribution. Confidence summarizes the
+ * distribution; it does not guarantee correctness.
+ *
+ * @category schemas
+ * @since 0.1.0
+ */
+export const ChoiceAnswer = Schema.Struct({
+  type: Schema.Literal("choice"),
+  choice: Schema.String,
+  probabilities: Schema.Record(Schema.String, Probability),
+  confidence: Probability,
+});
+
+/**
+ * Known option unions infer literal choices and required probability keys.
+ * Open string or template-pattern option types allow absent dictionary entries.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type ChoiceAnswer<Choice extends string = string> = Omit<
+  typeof ChoiceAnswer.Type,
+  "choice" | "probabilities"
+> & {
+  readonly choice: Choice;
+  readonly probabilities: {
+    readonly [K in Choice]: {} extends Pick<Record<Choice, unknown>, K>
+      ? number | undefined
+      : number;
+  };
+};
+
+/**
+ * A fractional, probability-weighted level, with the supplied rubric as legend.
+ *
+ * @category schemas
+ * @since 0.1.0
+ */
+export const ScoreAnswer = Schema.Struct({
+  type: Schema.Literal("score"),
+  score: Schema.Finite.check(Schema.isGreaterThanOrEqualTo(0)),
+  legend: Schema.Record(Schema.String, Schema.String),
+  probabilities: Schema.Record(Schema.String, Probability),
+  confidence: Probability,
+});
+
+/**
+ * Score levels are keyed at runtime; an arbitrary legend or probability lookup
+ * may be absent.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type ScoreAnswer = Omit<typeof ScoreAnswer.Type, "legend" | "probabilities"> & {
+  readonly legend: { readonly [level: string]: string | undefined };
+  readonly probabilities: { readonly [level: string]: number | undefined };
+};
+
+/**
+ * The probability of yes. Noul has no separate confidence field.
+ *
+ * @category schemas
+ * @since 0.1.0
+ */
+export const NoulAnswer = Schema.Struct({ type: Schema.Literal("noul"), noul: Probability });
+
+/** @category models
+ * @since 0.1.0
+ */
+export type NoulAnswer = typeof NoulAnswer.Type;
+
+/** @category schemas
+ * @since 0.1.0
+ */
+export const Answer = Schema.Union([ChoiceAnswer, ScoreAnswer, NoulAnswer]);
+
+/** @category models
+ * @since 0.1.0
+ */
+export type Answer = ChoiceAnswer | ScoreAnswer | NoulAnswer;
+
+/** @category schemas
+ * @since 0.1.0
+ */
+export const Usage = Schema.Struct({ input_tokens: Schema.Natural, output_tokens: Schema.Natural });
+
+/** @category models
+ * @since 0.1.0
+ */
+export type Usage = typeof Usage.Type;
+
+/**
+ * The wire response shape. `TypeSafeClient.evaluate` additionally validates
+ * answer IDs, question kinds, criteria, distributions, and score correlations.
+ *
+ * @category schemas
+ * @since 0.1.0
+ */
+export const EvaluateResponse = Schema.Struct({
+  model: Schema.String,
+  answers: Schema.Record(Schema.String, Answer),
+  usage: Usage,
+});
+
+type ChoiceAnswerFor<Criteria> = Criteria extends unknown
+  ? Omit<typeof ChoiceAnswer.Type, "choice" | "probabilities"> & {
+      readonly choice: `${Extract<keyof Criteria, string | number>}`;
+      readonly probabilities: {
+        readonly [
+          K in keyof Criteria as K extends string | number ? `${K}` : never
+        ]: {} extends Pick<Criteria, K> ? number | undefined : number;
+      };
+    }
+  : never;
+
+/**
+ * Infer a question's answer, preserving optional choice criteria in its probabilities.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type AnswerFor<Q extends Question> = Q extends ChoiceQuestion
+  ? ChoiceAnswerFor<Q["criteria"]>
+  : Q extends ScoreQuestion
+    ? ScoreAnswer
+    : NoulAnswer;
+
+/**
+ * One answer for each required question. Optional properties and open string,
+ * numeric, or template-pattern indexes require checking an entry for absence.
+ *
+ * @category models
+ * @since 0.1.0
+ */
+export type Answers<Q extends Questions> = {
+  readonly [K in keyof Q as K extends string | number ? `${K}` : never]: {} extends Pick<Q, K>
+    ? AnswerFor<NonNullable<Q[K]>> | undefined
+    : AnswerFor<Q[K]>;
+};
+
+/** @category models
+ * @since 0.1.0
+ */
+export type EvaluateResponse<Q extends Questions = Questions> = Omit<
+  typeof EvaluateResponse.Type,
+  "answers"
+> & { readonly answers: Answers<Q> };

package/src/index.ts

@@ -0,0 +1,2 @@
+export * as TypeSafeClient from "./TypeSafeClient.ts";
+export * as TypeSafeSchema from "./TypeSafeSchema.ts";

package/src/internal/errors.ts

@@ -0,0 +1,103 @@
+import * as Effect from "effect/Effect";
+import * as Option from "effect/Option";
+import * as Record from "effect/Record";
+import * as Redacted from "effect/Redacted";
+import type * as Schema from "effect/Schema";
+import * as AiError from "effect/unstable/ai/AiError";
+import * as Headers from "effect/unstable/http/Headers";
+import type * as HttpClientError from "effect/unstable/http/HttpClientError";
+
+export const make = (reason: AiError.AiErrorReason): AiError.AiError =>
+  AiError.make({ module: "TypeSafeClient", method: "evaluate", reason });
+
+export const mapSchemaError = (
+  error: Schema.SchemaError,
+  redact: (text: string) => string,
+): AiError.AiError => {
+  const reason = AiError.InvalidOutputError.fromSchemaError(error);
+
+  return make(
+    new AiError.InvalidOutputError({ ...reason, description: redact(reason.description) }),
+  );
+};
+
+const redactRequest = (
+  request: typeof AiError.HttpRequestDetails.Type,
+  redact: (text: string) => string,
+): typeof AiError.HttpRequestDetails.Type => ({
+  ...request,
+  url: redact(request.url),
+  urlParams: request.urlParams.map(([key, value]) => [key, redact(value)]),
+  hash: request.hash === undefined ? undefined : redact(request.hash),
+  headers: Record.map(request.headers, (value) =>
+    Redacted.isRedacted(value) ? "<redacted>" : redact(value),
+  ),
+});
+
+export const mapHttpClientError = Effect.fnUntraced(function* (
+  error: HttpClientError.HttpClientError,
+  redact: (text: string) => string,
+): Effect.fn.Return<never, AiError.AiError> {
+  const reason = error.reason;
+
+  switch (reason._tag) {
+    case "TransportError":
+    case "EncodeError":
+    case "InvalidUrlError": {
+      const network = AiError.NetworkError.fromRequestError(reason);
+
+      return yield* make(
+        new AiError.NetworkError({
+          ...network,
+          request: redactRequest(network.request, redact),
+          description: network.description === undefined ? undefined : redact(network.description),
+        }),
+      );
+    }
+    case "DecodeError":
+    case "EmptyBodyError":
+      return yield* make(
+        new AiError.InvalidOutputError({
+          description: redact(reason.description ?? "Could not decode the TypeSafe response body"),
+        }),
+      );
+    case "StatusCodeError": {
+      const { request, response } = reason;
+      const redactedNames = yield* Headers.CurrentRedactedNames;
+
+      const headers = (value: Headers.Headers) =>
+        Record.map(Headers.redact(value, redactedNames), (value) =>
+          Redacted.isRedacted(value) ? "<redacted>" : redact(value),
+        );
+
+      const text = yield* Effect.option(response.text);
+      const body = Option.isSome(text) ? redact(text.value) : undefined;
+
+      const http: typeof AiError.HttpContext.Type = {
+        request: {
+          method: request.method,
+          url: redact(request.url),
+          urlParams: Array.from(request.urlParams, ([key, value]) => [key, redact(value)]),
+          hash: Option.getOrUndefined(Option.map(request.hash, redact)),
+          headers: headers(request.headers),
+        },
+        response: { status: response.status, headers: headers(response.headers) },
+        body,
+      };
+
+      const description = AiError.buildErrorDescription({
+        status: response.status,
+        method: request.method,
+        url: http.request.url,
+        message: undefined,
+        body,
+      });
+
+      return yield* make(
+        response.status === 422
+          ? new AiError.InvalidRequestError({ description, http })
+          : AiError.reasonFromHttpStatus({ status: response.status, description, http }),
+      );
+    }
+  }
+});

package/src/internal/schema.ts

@@ -0,0 +1,86 @@
+import * as Schema from "effect/Schema";
+
+import * as TypeSafeSchema from "../TypeSafeSchema.ts";
+
+// Permit floating-point serialization error without changing provider values.
+const tolerance = 1e-6;
+
+const distribution = (keys: ReadonlyArray<string>) =>
+  Schema.Record(Schema.Literals(keys), TypeSafeSchema.Probability).check(
+    Schema.makeFilter(
+      (probabilities) =>
+        Math.abs(Object.values(probabilities).reduce((sum, value) => sum + value, 0) - 1) <=
+        tolerance,
+      { expected: "probabilities summing to 1 (within 1e-6)" },
+    ),
+  );
+
+const answerFor = (question: TypeSafeSchema.Question) => {
+  switch (question.type) {
+    case "choice": {
+      const keys = Object.keys(question.criteria);
+
+      return Schema.Struct({
+        ...TypeSafeSchema.ChoiceAnswer.fields,
+        choice: Schema.Literals(keys),
+        probabilities: distribution(keys),
+      }).check(
+        Schema.makeFilter(
+          ({ choice, probabilities }) =>
+            Object.values(probabilities).every((value) => value <= probabilities[choice]),
+          { expected: "a highest-probability choice" },
+        ),
+      );
+    }
+    case "score": {
+      const maxLevel = question.criteria.length - 1;
+
+      const levels = question.criteria.map(
+        (description, index) => [String(index), description] as const,
+      );
+
+      return Schema.Struct({
+        ...TypeSafeSchema.ScoreAnswer.fields,
+        score: Schema.Finite.check(Schema.isBetween({ minimum: 0, maximum: maxLevel })),
+        legend: Schema.Struct(
+          Object.fromEntries(
+            levels.map(([key, description]) => [key, Schema.Literal(description)]),
+          ),
+        ),
+        probabilities: distribution(levels.map(([key]) => key)),
+      }).check(
+        Schema.makeFilter(
+          ({ score, probabilities }) =>
+            Math.abs(
+              score -
+                Object.entries(probabilities).reduce(
+                  (sum, [level, probability]) => sum + Number(level) * probability,
+                  0,
+                ),
+            ) <=
+            tolerance * Math.max(1, maxLevel),
+          { expected: "the probability-weighted score (within 1e-6 per level)" },
+        ),
+      );
+    }
+    case "noul":
+      return TypeSafeSchema.NoulAnswer;
+  }
+};
+
+// The overload describes the dependent type enforced by the literal keys and
+// per-question schemas below. No JSON value is asserted to have that type.
+export function responseFor<const Q extends TypeSafeSchema.Questions>(
+  questions: Q,
+): Schema.Codec<TypeSafeSchema.EvaluateResponse<Q>>;
+
+export function responseFor(questions: TypeSafeSchema.Questions): Schema.Top {
+  return Schema.Struct({
+    ...TypeSafeSchema.EvaluateResponse.fields,
+    answers: Schema.Struct(
+      Object.fromEntries(
+        Object.entries(questions).map(([id, question]) => [id, answerFor(question)]),
+      ),
+    ),
+  });
+}