modaic 0.1.0

package/src/modaic/signatures.ts

@@ -0,0 +1,398 @@
+import { z } from "zod";
+
+/**
+ * A Signature defines the interface for a DSPy-style predictor.
+ * It includes optional instructions, an input schema, and an output schema.
+ * Both input and output are defined using Zod schemas.
+ *
+ * This is a port of the `@modaic/dsts` Signature with tool support removed —
+ * Arbiters are pure judges and never call tools.
+ */
+export class Signature<
+  I extends z.ZodObject<any> = z.ZodObject<any>,
+  O extends z.ZodObject<any> = z.ZodObject<any>,
+> {
+  public instructions?: string;
+  public input: I;
+  public output: O;
+  public prefixes: Record<string, string> = {};
+  /**
+   * Type-only property for input inference.
+   * Use with `typeof signature.InferInput` or `SignatureInstance["InferInput"]`.
+   */
+  declare readonly InferInput: z.infer<I>;
+
+  /**
+   * Type-only property for output inference.
+   * Use with `typeof signature.InferOutput` or `SignatureInstance["InferOutput"]`.
+   */
+  declare readonly InferOutput: z.infer<O>;
+
+  // `instructions?: string | undefined` (not just `?: string`) so callers may pass
+  // a `string | undefined` through under `exactOptionalPropertyTypes`.
+  constructor(sig: { instructions?: string | undefined; input: I; output: O }) {
+    this.instructions =
+      sig.instructions ||
+      this.generateDefaultInstructions(sig.input, sig.output);
+    this.input = sig.input;
+    this.output = sig.output;
+
+    // Initialize default prefixes
+    const allFields = { ...sig.input.shape, ...sig.output.shape };
+    for (const name in allFields) {
+      this.prefixes[name] = infer_prefix(name) + ":";
+    }
+  }
+
+  private generateDefaultInstructions(input: I, output: O): string {
+    const inputFields = Object.keys(input.shape);
+    const outputFields = Object.keys(output.shape);
+    const inputStr = inputFields.map((f) => `\`${f}\``).join(", ");
+    const outputStr = outputFields.map((f) => `\`${f}\``).join(", ");
+    return `Given the fields ${inputStr}, produce the fields ${outputStr}.`;
+  }
+
+  get input_fields(): Record<string, any> {
+    return this.input.shape;
+  }
+
+  get output_fields(): Record<string, any> {
+    return this.output.shape;
+  }
+
+  /**
+   * Static helper to create a Signature from a string definition.
+   * e.g. Signature.parse("question -> answer", "Answer the question.")
+   */
+  static parse(
+    sigStr: string,
+    instructions?: string,
+  ): Signature<z.ZodObject<any>, z.ZodObject<any>> {
+    const parsed = parseStringSignature(sigStr);
+    return new Signature({
+      instructions,
+      input: parsed.input,
+      output: parsed.output,
+    });
+  }
+
+  /**
+   * Returns a new Signature with the specified field removed from either input or output.
+   */
+  delete(
+    inputOrOutput: "input" | "output",
+    name: string,
+  ): Signature<z.ZodObject<any>, z.ZodObject<any>> {
+    const inputShape = this.input.shape;
+    const outputShape = this.output.shape;
+
+    let newInput: z.ZodObject<any> = this.input;
+    let newOutput: z.ZodObject<any> = this.output;
+
+    // Remove from input if present and requested
+    if (name in inputShape && inputOrOutput === "input") {
+      newInput = (this.input as z.ZodObject<any>).omit({ [name]: true } as any);
+    }
+
+    // Remove from output if present and requested
+    if (name in outputShape && inputOrOutput === "output") {
+      newOutput = (this.output as z.ZodObject<any>).omit({
+        [name]: true,
+      } as any);
+    }
+
+    return new Signature({
+      instructions: this.instructions,
+      input: newInput,
+      output: newOutput,
+    });
+  }
+
+  /**
+   * Returns a new Signature with updated instructions.
+   */
+  withInstructions(instructions: string): Signature<I, O> {
+    return new Signature({
+      instructions,
+      input: this.input,
+      output: this.output,
+    });
+  }
+
+  dump_state(): any {
+    const fields = Object.entries({
+      ...this.input.shape,
+      ...this.output.shape,
+    }).map(([name, field]) => ({
+      prefix: this.prefixes[name],
+      description:
+        (field as any).meta?.()?.description ||
+        (field as any).meta?.()?.desc ||
+        (field as any)._def?.description ||
+        "",
+    }));
+
+    return {
+      instructions: this.instructions,
+      fields,
+    };
+  }
+
+  load_state(state: any): Signature<z.ZodObject<any>, z.ZodObject<any>> {
+    const prefixToName: Record<string, string> = {};
+    for (const [name, prefix] of Object.entries(this.prefixes)) {
+      prefixToName[prefix] = name;
+    }
+
+    const inputPatch: Record<string, z.ZodType> = {};
+    const outputPatch: Record<string, z.ZodType> = {};
+
+    for (const fieldState of state.fields) {
+      const name = prefixToName[fieldState.prefix];
+      if (!name) continue;
+
+      const isInput = name in this.input.shape;
+      const field = (
+        isInput ? this.input.shape[name] : this.output.shape[name]
+      ) as z.ZodType;
+
+      let updatedField = field;
+      const description = fieldState.description;
+
+      updatedField = updatedField.describe(description);
+
+      if (typeof (updatedField as any).meta === "function") {
+        const currentMeta = (updatedField as any).meta() || {};
+        updatedField = (updatedField as any).meta({
+          ...currentMeta,
+          description,
+          desc: description,
+        });
+      }
+
+      if (isInput) {
+        inputPatch[name] = updatedField;
+      } else {
+        outputPatch[name] = updatedField;
+      }
+    }
+
+    return new Signature({
+      instructions: state.instructions,
+      input: this.input.extend(inputPatch),
+      output: this.output.extend(outputPatch),
+    });
+  }
+}
+
+/**
+ * Helper to infer the input type of a signature.
+ */
+export type InferInput<S extends Signature<any, any>> = z.infer<S["input"]>;
+
+/**
+ * Helper to infer the output type of a signature.
+ */
+export type InferOutput<S extends Signature<any, any>> = z.infer<S["output"]>;
+
+/**
+ * Maps common string type names to Zod schemas.
+ */
+const mapTypeToZod = (typeStr?: string): z.ZodType => {
+  switch (typeStr?.toLowerCase()) {
+    case "number":
+      return z.number();
+    case "boolean":
+      return z.boolean();
+    case "string":
+    case undefined:
+    case "":
+      return z.string();
+    default:
+      // Default to string for unknown types, similar to DSPy behavior
+      return z.string();
+  }
+};
+
+/**
+ * Parses a string-based signature like "input1: string, input2: number -> output: string"
+ */
+const parseStringSignature = (
+  sigStr: string,
+): { input: z.ZodObject<any>; output: z.ZodObject<any> } => {
+  const parts = sigStr.split("->");
+  if (parts.length !== 2) {
+    throw new Error(
+      `Invalid signature string: "${sigStr}". Must contain exactly one "->" separator.`,
+    );
+  }
+
+  const [inputsPart, outputsPart] = parts.map((s) => s.trim());
+
+  const parsePart = (part: string) => {
+    const shape: Record<string, z.ZodType> = {};
+    const fields = part
+      .split(",")
+      .map((f) => f.trim())
+      .filter(Boolean);
+
+    for (const field of fields) {
+      const [name, type] = field.split(":").map((s) => s.trim());
+      if (!name) continue;
+      shape[name] = mapTypeToZod(type);
+    }
+
+    return z.object(shape);
+  };
+
+  return {
+    input: parsePart(inputsPart!),
+    output: parsePart(outputsPart!),
+  };
+};
+
+/**
+ * Infer a prefix from an attribute name by converting it to a human-readable format.
+ * Matches the logic in DSPy's infer_prefix.
+ *
+ * Examples:
+ *   "camelCaseText" -> "Camel Case Text"
+ *   "snake_case_text" -> "Snake Case Text"
+ *   "text2number" -> "Text 2 Number"
+ *   "HTMLParser" -> "HTML Parser"
+ */
+export function infer_prefix(attributeName: string): string {
+  // Step 1: Convert camelCase to snake_case
+  // Example: "camelCase" -> "camel_Case"
+  const s1 = attributeName.replace(/(.)([A-Z][a-z]+)/g, "$1_$2");
+
+  // Handle consecutive capitals
+  // Example: "camel_Case" -> "camel_case"
+  const intermediateName = s1.replace(/([a-z0-9])([A-Z])/g, "$1_$2");
+
+  // Step 2: Handle numbers by adding underscores around them
+  // Example: "text2number" -> "text_2_number"
+  let withUnderscoresAroundNumbers = intermediateName.replace(
+    /([a-zA-Z])(\d)/g,
+    "$1_$2",
+  );
+  // Example: "2text" -> "2_text"
+  withUnderscoresAroundNumbers = withUnderscoresAroundNumbers.replace(
+    /(\d)([a-zA-Z])/g,
+    "$1_$2",
+  );
+
+  // Step 3: Convert to Title Case while preserving acronyms
+  const words = withUnderscoresAroundNumbers.split("_");
+  const titleCasedWords = words.map((word) => {
+    if (word === word.toUpperCase() && word !== word.toLowerCase()) {
+      // Preserve acronyms like 'HTML', 'API' as-is
+      return word;
+    }
+    // Capitalize first letter: 'text' -> 'Text'
+    return word.charAt(0).toUpperCase() + word.slice(1).toLowerCase();
+  });
+
+  // Join words with spaces
+  // Example: ["Text", "2", "Number"] -> "Text 2 Number"
+  return titleCasedWords.join(" ");
+}
+
+/* -------------------------------------------------------------------------- */
+/*  Special signature field types                                              */
+/* -------------------------------------------------------------------------- */
+
+/**
+ * Special signature field types — the modaic-ts analogs of dspy's custom types.
+ *
+ * These are how a Signature declares a field that is more than a plain primitive.
+ * Each one is detected by the serializer (`serialization.ts`) via a `__modaic_type`
+ * marker on the field's Zod `.meta()`, and is serialized into the exact JSON the
+ * Python SDK produces — so an Arbiter authored here round-trips into the right
+ * Python type with **no Python-side changes**:
+ *
+ *   - `Image` / `Audio`  -> `$defs["Image"] = {"type":"dspy.Image"}` + a `$ref` field,
+ *                           which the Python SDK reconstructs as a real `dspy.Image` /
+ *                           `dspy.Audio` (its adapters already render them multimodally).
+ *   - `Scale(lo, hi)`    -> a plain integer enum `{"enum":[lo..hi], "type":"integer"}`,
+ *                           matching Python `modaic.Scale[lo, hi]`.
+ *   - `Enum(...values)`  -> a plain string enum `{"enum":[...], "type":"string"}`,
+ *                           matching Python `modaic.Enum[...]`.
+ */
+
+/** Marker value the serializer reads from a field's `.meta().__modaic_type`. */
+export type ModaicTypeTag = "image" | "audio" | "scale";
+
+/**
+ * `modaic.Image` — a multimodal image field, the TS analog of `dspy.Image`.
+ *
+ * Use {@link Image.field} inside a `Signature`'s input/output schema, and construct
+ * `new Image({ url })` to carry a value when calling `predict`.
+ */
+export class Image {
+  url: string;
+
+  constructor(init: { url: string }) {
+    this.url = init.url;
+  }
+
+  /**
+   * The Zod schema to place in a `Signature`. Serializes to a `dspy.Image` `$ref`.
+   * Chain `.describe(...)` to set the field description.
+   */
+  static field(): z.ZodType {
+    return z.custom<Image>().meta({ __modaic_type: "image" });
+  }
+}
+
+/**
+ * `modaic.Audio` — a multimodal audio field, the TS analog of `dspy.Audio`.
+ *
+ * Use {@link Audio.field} inside a `Signature`'s input/output schema, and construct
+ * `new Audio({ url })` to carry a value when calling `predict`.
+ */
+export class Audio {
+  url: string;
+
+  constructor(init: { url: string }) {
+    this.url = init.url;
+  }
+
+  /**
+   * The Zod schema to place in a `Signature`. Serializes to a `dspy.Audio` `$ref`.
+   * Chain `.describe(...)` to set the field description.
+   */
+  static field(): z.ZodType {
+    return z.custom<Audio>().meta({ __modaic_type: "audio" });
+  }
+}
+
+/**
+ * `modaic.Scale(lo, hi)` — an integer rating in `[lo, hi]` inclusive.
+ *
+ * Serializes to a plain integer enum `{"enum":[lo, lo+1, …, hi], "type":"integer"}`,
+ * matching Python `modaic.Scale[lo, hi]` (which presents to the model as a Literal of ints).
+ */
+export function Scale(lo: number, hi: number): z.ZodType {
+  if (!Number.isInteger(lo) || !Number.isInteger(hi)) {
+    throw new Error("Scale values must be integers");
+  }
+  if (lo > hi) {
+    throw new Error(`Scale lo (${lo}) must be <= hi (${hi})`);
+  }
+  return z
+    .int()
+    .gte(lo)
+    .lte(hi)
+    .meta({ __modaic_type: "scale", lo, hi });
+}
+
+/**
+ * `modaic.Enum(...values)` — a string choice from a fixed set.
+ *
+ * Serializes to `{"enum":[...], "type":"string"}` (a single value serializes as
+ * `{"const": v, "type":"string"}`), matching Python `modaic.Enum[...]`.
+ */
+export function Enum<T extends string>(...values: [T, ...T[]]): z.ZodType {
+  return z.enum(values);
+}

package/src/models/annotate-example-request.ts

@@ -0,0 +1,38 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v4-mini";
+import {
+  PredictionAnnotation,
+  PredictionAnnotation$Outbound,
+  PredictionAnnotation$outboundSchema,
+} from "./prediction-annotation.js";
+
+/**
+ * Annotate an example with one or more prediction annotations.
+ */
+export type AnnotateExampleRequest = {
+  annotations: Array<PredictionAnnotation>;
+};
+
+/** @internal */
+export type AnnotateExampleRequest$Outbound = {
+  annotations: Array<PredictionAnnotation$Outbound>;
+};
+
+/** @internal */
+export const AnnotateExampleRequest$outboundSchema: z.ZodMiniType<
+  AnnotateExampleRequest$Outbound,
+  AnnotateExampleRequest
+> = z.object({
+  annotations: z.array(PredictionAnnotation$outboundSchema),
+});
+
+export function annotateExampleRequestToJSON(
+  annotateExampleRequest: AnnotateExampleRequest,
+): string {
+  return JSON.stringify(
+    AnnotateExampleRequest$outboundSchema.parse(annotateExampleRequest),
+  );
+}

package/src/models/arbiter-request.ts

@@ -0,0 +1,46 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v4-mini";
+import { remap as remap$ } from "../lib/primitives.js";
+
+export type ArbiterRequest = {
+  arbiterRepo: string;
+  arbiterRevision?: string | undefined;
+  groundTruth?: string | null | undefined;
+  groundReasoning?: string | undefined;
+};
+
+/** @internal */
+export type ArbiterRequest$Outbound = {
+  arbiter_repo: string;
+  arbiter_revision: string;
+  ground_truth?: string | null | undefined;
+  ground_reasoning: string;
+};
+
+/** @internal */
+export const ArbiterRequest$outboundSchema: z.ZodMiniType<
+  ArbiterRequest$Outbound,
+  ArbiterRequest
+> = z.pipe(
+  z.object({
+    arbiterRepo: z.string(),
+    arbiterRevision: z._default(z.string(), "main"),
+    groundTruth: z.optional(z.nullable(z.string())),
+    groundReasoning: z._default(z.string(), ""),
+  }),
+  z.transform((v) => {
+    return remap$(v, {
+      arbiterRepo: "arbiter_repo",
+      arbiterRevision: "arbiter_revision",
+      groundTruth: "ground_truth",
+      groundReasoning: "ground_reasoning",
+    });
+  }),
+);
+
+export function arbiterRequestToJSON(arbiterRequest: ArbiterRequest): string {
+  return JSON.stringify(ArbiterRequest$outboundSchema.parse(arbiterRequest));
+}

package/src/models/batch-arbiter.ts

@@ -0,0 +1,38 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v4-mini";
+import { remap as remap$ } from "../lib/primitives.js";
+
+export type BatchArbiter = {
+  arbiterRepo: string;
+  arbiterRevision?: string | undefined;
+};
+
+/** @internal */
+export type BatchArbiter$Outbound = {
+  arbiter_repo: string;
+  arbiter_revision: string;
+};
+
+/** @internal */
+export const BatchArbiter$outboundSchema: z.ZodMiniType<
+  BatchArbiter$Outbound,
+  BatchArbiter
+> = z.pipe(
+  z.object({
+    arbiterRepo: z.string(),
+    arbiterRevision: z._default(z.string(), "main"),
+  }),
+  z.transform((v) => {
+    return remap$(v, {
+      arbiterRepo: "arbiter_repo",
+      arbiterRevision: "arbiter_revision",
+    });
+  }),
+);
+
+export function batchArbiterToJSON(batchArbiter: BatchArbiter): string {
+  return JSON.stringify(BatchArbiter$outboundSchema.parse(batchArbiter));
+}

package/src/models/batch-example.ts

@@ -0,0 +1,75 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v4-mini";
+import { remap as remap$ } from "../lib/primitives.js";
+import { ClosedEnum } from "../types/enums.js";
+
+export const BatchExampleSplit = {
+  Train: "train",
+  Test: "test",
+  None: "none",
+} as const;
+export type BatchExampleSplit = ClosedEnum<typeof BatchExampleSplit>;
+
+/**
+ * One row in a batch predictions request.
+ *
+ * @remarks
+ *
+ * Split contract (mirrors `examples/schemas.py::PredictedExample`):
+ * only `"train"` and `"test"` are valid downstream — `"none"` exists
+ * purely as the storage marker for unannotated rows. Callers MUST NOT
+ * send `"none"` explicitly; omit `split` instead and let `get_split`
+ * randomly assign train/test (80/20) when `ground_truth` is provided.
+ * Without that auto-assignment, annotated batch rows would land in
+ * ClickHouse with `split='none'` and be skipped by every downstream
+ * consumer (GEPA, scoring, dataset export).
+ */
+export type BatchExample = {
+  input: { [k: string]: any };
+  altId?: string | null | undefined;
+  groundTruth?: { [k: string]: any } | null | undefined;
+  groundReasoning?: string | undefined;
+  split?: BatchExampleSplit | null | undefined;
+};
+
+/** @internal */
+export const BatchExampleSplit$outboundSchema: z.ZodMiniEnum<
+  typeof BatchExampleSplit
+> = z.enum(BatchExampleSplit);
+
+/** @internal */
+export type BatchExample$Outbound = {
+  input: { [k: string]: any };
+  alt_id?: string | null | undefined;
+  ground_truth?: { [k: string]: any } | null | undefined;
+  ground_reasoning: string;
+  split?: string | null | undefined;
+};
+
+/** @internal */
+export const BatchExample$outboundSchema: z.ZodMiniType<
+  BatchExample$Outbound,
+  BatchExample
+> = z.pipe(
+  z.object({
+    input: z.record(z.string(), z.any()),
+    altId: z.optional(z.nullable(z.string())),
+    groundTruth: z.optional(z.nullable(z.record(z.string(), z.any()))),
+    groundReasoning: z._default(z.string(), ""),
+    split: z.optional(z.nullable(BatchExampleSplit$outboundSchema)),
+  }),
+  z.transform((v) => {
+    return remap$(v, {
+      altId: "alt_id",
+      groundTruth: "ground_truth",
+      groundReasoning: "ground_reasoning",
+    });
+  }),
+);
+
+export function batchExampleToJSON(batchExample: BatchExample): string {
+  return JSON.stringify(BatchExample$outboundSchema.parse(batchExample));
+}

package/src/models/batch-predictions-job-request.ts

@@ -0,0 +1,68 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as z from "zod/v4-mini";
+import { remap as remap$ } from "../lib/primitives.js";
+import {
+  BatchArbiter,
+  BatchArbiter$Outbound,
+  BatchArbiter$outboundSchema,
+} from "./batch-arbiter.js";
+import {
+  BatchExample,
+  BatchExample$Outbound,
+  BatchExample$outboundSchema,
+} from "./batch-example.js";
+
+/**
+ * Request body for ``POST /api/v1/jobs/batch/predictions``.
+ *
+ * @remarks
+ *
+ * Either ``examples`` or ``example_ids`` must be set, not both. The
+ * ``example_ids`` path re-predicts existing examples (by their ClickHouse
+ * ids) instead of ingesting brand-new ones; predictions are inserted as new
+ * versions on the existing example rows.
+ */
+export type BatchPredictionsJobRequest = {
+  arbiters: Array<BatchArbiter>;
+  examples?: Array<BatchExample> | null | undefined;
+  exampleIds?: Array<string> | null | undefined;
+  computeConfidence?: boolean | undefined;
+};
+
+/** @internal */
+export type BatchPredictionsJobRequest$Outbound = {
+  arbiters: Array<BatchArbiter$Outbound>;
+  examples?: Array<BatchExample$Outbound> | null | undefined;
+  example_ids?: Array<string> | null | undefined;
+  compute_confidence: boolean;
+};
+
+/** @internal */
+export const BatchPredictionsJobRequest$outboundSchema: z.ZodMiniType<
+  BatchPredictionsJobRequest$Outbound,
+  BatchPredictionsJobRequest
+> = z.pipe(
+  z.object({
+    arbiters: z.array(BatchArbiter$outboundSchema),
+    examples: z.optional(z.nullable(z.array(BatchExample$outboundSchema))),
+    exampleIds: z.optional(z.nullable(z.array(z.string()))),
+    computeConfidence: z._default(z.boolean(), false),
+  }),
+  z.transform((v) => {
+    return remap$(v, {
+      exampleIds: "example_ids",
+      computeConfidence: "compute_confidence",
+    });
+  }),
+);
+
+export function batchPredictionsJobRequestToJSON(
+  batchPredictionsJobRequest: BatchPredictionsJobRequest,
+): string {
+  return JSON.stringify(
+    BatchPredictionsJobRequest$outboundSchema.parse(batchPredictionsJobRequest),
+  );
+}