modaic 0.1.0

package/src/lib/security.ts

@@ -0,0 +1,275 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+import * as models from "../models/index.js";
+import { env } from "./env.js";
+
+type OAuth2PasswordFlow = {
+  username: string;
+  password: string;
+  clientID?: string | undefined;
+  clientSecret?: string | undefined;
+  tokenURL: string;
+};
+
+export enum SecurityErrorCode {
+  Incomplete = "incomplete",
+  UnrecognisedSecurityType = "unrecognized_security_type",
+}
+
+export class SecurityError extends Error {
+  constructor(
+    public code: SecurityErrorCode,
+    message: string,
+  ) {
+    super(message);
+    this.name = "SecurityError";
+  }
+
+  static incomplete(): SecurityError {
+    return new SecurityError(
+      SecurityErrorCode.Incomplete,
+      "Security requirements not met in order to perform the operation",
+    );
+  }
+  static unrecognizedType(type: string): SecurityError {
+    return new SecurityError(
+      SecurityErrorCode.UnrecognisedSecurityType,
+      `Unrecognised security type: ${type}`,
+    );
+  }
+}
+
+export type SecurityState = {
+  basic: { username?: string | undefined; password?: string | undefined };
+  headers: Record<string, string>;
+  queryParams: Record<string, string>;
+  cookies: Record<string, string>;
+  oauth2: ({ type: "password" } & OAuth2PasswordFlow) | { type: "none" };
+};
+
+type SecurityInputBasic = {
+  type: "http:basic";
+  value:
+    | { username?: string | undefined; password?: string | undefined }
+    | null
+    | undefined;
+};
+
+type SecurityInputBearer = {
+  type: "http:bearer";
+  value: string | null | undefined;
+  fieldName: string;
+};
+
+type SecurityInputAPIKey = {
+  type: "apiKey:header" | "apiKey:query" | "apiKey:cookie";
+  value: string | null | undefined;
+  fieldName: string;
+};
+
+type SecurityInputOIDC = {
+  type: "openIdConnect";
+  value: string | null | undefined;
+  fieldName: string;
+};
+
+type SecurityInputOAuth2 = {
+  type: "oauth2";
+  value: string | null | undefined;
+  fieldName: string;
+};
+
+type SecurityInputOAuth2ClientCredentials = {
+  type: "oauth2:client_credentials";
+  value:
+    | {
+      clientID?: string | undefined;
+      clientSecret?: string | undefined;
+    }
+    | null
+    | string
+    | undefined;
+  fieldName?: string;
+};
+
+type SecurityInputOAuth2PasswordCredentials = {
+  type: "oauth2:password";
+  value:
+    | string
+    | null
+    | undefined;
+  fieldName?: string;
+};
+
+type SecurityInputCustom = {
+  type: "http:custom";
+  value: any | null | undefined;
+  fieldName?: string;
+};
+
+export type SecurityInput =
+  | SecurityInputBasic
+  | SecurityInputBearer
+  | SecurityInputAPIKey
+  | SecurityInputOAuth2
+  | SecurityInputOAuth2ClientCredentials
+  | SecurityInputOAuth2PasswordCredentials
+  | SecurityInputOIDC
+  | SecurityInputCustom;
+
+export function resolveSecurity(
+  ...options: SecurityInput[][]
+): SecurityState | null {
+  const state: SecurityState = {
+    basic: {},
+    headers: {},
+    queryParams: {},
+    cookies: {},
+    oauth2: { type: "none" },
+  };
+
+  const option = options.find((opts) => {
+    return opts.every((o) => {
+      if (o.value == null) {
+        return false;
+      } else if (o.type === "http:basic") {
+        return o.value.username != null || o.value.password != null;
+      } else if (o.type === "http:custom") {
+        return null;
+      } else if (o.type === "oauth2:password") {
+        return (
+          typeof o.value === "string" && !!o.value
+        );
+      } else if (o.type === "oauth2:client_credentials") {
+        if (typeof o.value == "string") {
+          return !!o.value;
+        }
+        return o.value.clientID != null || o.value.clientSecret != null;
+      } else if (typeof o.value === "string") {
+        return !!o.value;
+      } else {
+        throw new Error(
+          `Unrecognized security type: ${o.type} (value type: ${typeof o
+            .value})`,
+        );
+      }
+    });
+  });
+  if (option == null) {
+    return null;
+  }
+
+  option.forEach((spec) => {
+    if (spec.value == null) {
+      return;
+    }
+
+    const { type } = spec;
+
+    switch (type) {
+      case "apiKey:header":
+        state.headers[spec.fieldName] = spec.value;
+        break;
+      case "apiKey:query":
+        state.queryParams[spec.fieldName] = spec.value;
+        break;
+      case "apiKey:cookie":
+        state.cookies[spec.fieldName] = spec.value;
+        break;
+      case "http:basic":
+        applyBasic(state, spec);
+        break;
+      case "http:custom":
+        break;
+      case "http:bearer":
+        applyBearer(state, spec);
+        break;
+      case "oauth2":
+        applyBearer(state, spec);
+        break;
+      case "oauth2:password":
+        applyBearer(state, spec);
+        break;
+      case "oauth2:client_credentials":
+        break;
+      case "openIdConnect":
+        applyBearer(state, spec);
+        break;
+      default:
+        throw SecurityError.unrecognizedType((spec satisfies never, type));
+    }
+  });
+
+  return state;
+}
+
+function applyBasic(
+  state: SecurityState,
+  spec: SecurityInputBasic,
+) {
+  if (spec.value == null) {
+    return;
+  }
+
+  state.basic = spec.value;
+}
+
+function applyBearer(
+  state: SecurityState,
+  spec:
+    | SecurityInputBearer
+    | SecurityInputOAuth2
+    | SecurityInputOIDC
+    | SecurityInputOAuth2PasswordCredentials,
+) {
+  if (typeof spec.value !== "string" || !spec.value) {
+    return;
+  }
+
+  let value = spec.value;
+  if (value.slice(0, 7).toLowerCase() !== "bearer ") {
+    value = `Bearer ${value}`;
+  }
+
+  if (spec.fieldName !== undefined) {
+    state.headers[spec.fieldName] = value;
+  }
+}
+
+export function resolveGlobalSecurity(
+  security: Partial<models.Security> | null | undefined,
+  allowedFields?: number[],
+): SecurityState | null {
+  let inputs: SecurityInput[][] = [
+    [
+      {
+        fieldName: "Authorization",
+        type: "http:bearer",
+        value: security?.token ?? env().MODAIC_TOKEN,
+      },
+    ],
+  ];
+
+  if (allowedFields) {
+    inputs = allowedFields.map((i) => {
+      if (i < 0 || i >= inputs.length) {
+        throw new RangeError(`invalid allowedFields index ${i}`);
+      }
+      return inputs[i]!;
+    });
+  }
+
+  return resolveSecurity(...inputs);
+}
+
+export async function extractSecurity<
+  T extends string | Record<string, unknown>,
+>(sec: T | (() => Promise<T>) | undefined): Promise<T | undefined> {
+  if (sec == null) {
+    return;
+  }
+
+  return typeof sec === "function" ? sec() : sec;
+}

package/src/lib/url.ts

@@ -0,0 +1,35 @@
+/*
+ * Code generated by Speakeasy (https://speakeasy.com). DO NOT EDIT.
+ */
+
+const hasOwn = Object.prototype.hasOwnProperty;
+
+export type Params = Partial<Record<string, string | number>>;
+
+export function pathToFunc(
+  pathPattern: string,
+  options?: { charEncoding?: "percent" | "none" },
+): (params?: Params) => string {
+  const paramRE = /\{([a-zA-Z0-9_][a-zA-Z0-9_-]*?)\}/g;
+
+  return function buildURLPath(params: Record<string, unknown> = {}): string {
+    return pathPattern
+      .replace(paramRE, function (_, placeholder) {
+        if (!hasOwn.call(params, placeholder)) {
+          throw new Error(`Parameter '${placeholder}' is required`);
+        }
+
+        const value = params[placeholder];
+        if (typeof value !== "string" && typeof value !== "number") {
+          throw new Error(
+            `Parameter '${placeholder}' must be a string or number`,
+          );
+        }
+
+        return options?.charEncoding === "percent"
+          ? encodeURIComponent(`${value}`)
+          : `${value}`;
+      })
+      .replace(/^\/+/, "");
+  };
+}

package/src/modaic/arbiter.ts

@@ -0,0 +1,252 @@
+import { z } from "zod";
+import { Signature } from "./signatures.js";
+import {
+  serializeSignatureToConfig,
+  buildProgramJson,
+  repoNameToTitle,
+} from "./serialization.js";
+import { resolveToken, getApiUrl } from "./runtime.js";
+import {
+  syncAndPush,
+  createRepo,
+  type Commit,
+  type PushFiles,
+} from "./git.js";
+import { ModaicClient } from "../sdk/sdk.js";
+import type { PredictExampleResponse } from "../models/index.js";
+
+/**
+ * The fixed description given to an Arbiter's injected `reasoning` output field.
+ * Matches the Python SDK (`make_arbiter`) verbatim — original wording included —
+ * so judges authored here stay byte-compatible with Python ones.
+ */
+const REASONING_DESC =
+  "Your reasoning for your answer. Inlude any uncertainties about your answer or ambiguity in the task.";
+
+/**
+ * Return a Signature guaranteed to have a `reasoning` output field — the TS analog
+ * of Python `make_arbiter`. Arbiters must expose a plain `reasoning: string` output
+ * because the server reads `prediction.reasoning` on every prediction; without it,
+ * predictions fail server-side.
+ *
+ * Mirrors `make_arbiter`'s `signature.insert(-2, "reasoning", …)`: reasoning is
+ * inserted within the output section just before the last output field (so a
+ * single-output judge gets reasoning first). Idempotent — if a `reasoning` output
+ * already exists, the signature is returned unchanged.
+ */
+export function ensure_reasoning_field(
+  signature: Signature,
+): Signature<z.ZodObject<any>, z.ZodObject<any>> {
+  const outShape = signature.output.shape as Record<string, z.ZodType>;
+  if ("reasoning" in outShape) {
+    return new Signature({
+      instructions: signature.instructions,
+      input: signature.input,
+      output: signature.output,
+    });
+  }
+
+  // A plain string field. Set the description on both `.describe()` and `.meta()`
+  // so it surfaces in program.json (dump_state) and config.json (serializeField).
+  let reasoningField: z.ZodType = z.string().describe(REASONING_DESC);
+  if (typeof (reasoningField as any).meta === "function") {
+    reasoningField = (reasoningField as any).meta({
+      description: REASONING_DESC,
+      desc: REASONING_DESC,
+    });
+  }
+
+  const entries = Object.entries(outShape);
+  const insertIdx = Math.max(0, entries.length - 1);
+  const newShape: Record<string, z.ZodType> = {};
+  entries.forEach(([k, field], i) => {
+    if (i === insertIdx) newShape["reasoning"] = reasoningField;
+    newShape[k] = field;
+  });
+  if (entries.length === 0) newShape["reasoning"] = reasoningField;
+
+  return new Signature({
+    instructions: signature.instructions,
+    input: signature.input,
+    output: z.object(newShape),
+  });
+}
+
+/** Options for constructing an Arbiter handle to an existing repo. */
+export interface ArbiterOptions {
+  /** Git branch used for create/update operations. Default "main". */
+  branch?: string;
+  /** Revision (branch/tag/commit) used when running predictions. Default = branch. */
+  rev?: string;
+}
+
+/** Options for `Arbiter.create`. */
+export interface CreateOptions {
+  repo: string;
+  signature: Signature;
+  /**
+   * The LiteLLM model string the server runs this judge with, e.g.
+   * "gpt-oss-120b" or "openai/gpt-4o". Required — an arbiter with no model is
+   * not runnable: `predict()` would fail server-side.
+   */
+  model: string;
+  branch?: string;
+  tag?: string;
+  access_token?: string;
+  commit_message?: string;
+  private?: boolean;
+  metadata?: Record<string, unknown> | null;
+  extra_files?: string[] | null;
+}
+
+/** Options for `arbiter.update`. */
+export interface UpdateOptions {
+  /** Optional new signature; when provided, config.json and program.json are rewritten. */
+  signature?: Signature;
+  /**
+   * LiteLLM model string. Required when `signature` is provided, since rewriting
+   * `program.json` needs the model written into its `lm` block.
+   */
+  model?: string;
+  metadata?: Record<string, unknown> | null;
+  extra_files?: string[] | null;
+  commit_message?: string;
+  tag?: string;
+  access_token?: string;
+}
+
+/** Options for `arbiter.predict`. */
+export interface PredictOptions {
+  ground_truth?: Record<string, unknown> | null;
+  ground_reasoning?: string;
+  compute_confidence?: boolean;
+}
+
+/**
+ * A handle to a Modaic Arbiter (an LLM judge) stored on Modaic Hub.
+ *
+ * Like `modaic_client.Arbiter`, this is a thin wrapper over the Modaic REST API
+ * and git — it never runs an LLM locally. `predict()` calls the API; `create()`
+ * and `update()` write the judge's `config.json` / `program.json` and push them
+ * to the hub via git, mirroring `modaic.Predict.push_to_hub`.
+ */
+export class Arbiter {
+  repo: string;
+  branch: string;
+  rev: string;
+
+  constructor(repo: string, opts: ArbiterOptions = {}) {
+    this.repo = repo;
+    this.branch = opts.branch ?? "main";
+    this.rev = opts.rev ?? this.branch;
+  }
+
+  /**
+   * Create a new Arbiter repo on the hub and push its signature.
+   *
+   * Builds `config.json` (signature schema) and `program.json` (stored prompt)
+   * the same way the Python SDK does, creates the remote repo (private by
+   * default), then commits & pushes on `branch`.
+   */
+  static async create(opts: CreateOptions): Promise<Arbiter> {
+    const branch = opts.branch ?? "main";
+    const token = resolveToken(opts.access_token);
+
+    // Build files first so a bad signature fails before we create a repo.
+    // Inject the `reasoning` output field so the judge is a valid Arbiter.
+    const signature = ensure_reasoning_field(opts.signature);
+    const config = serializeSignatureToConfig(
+      signature,
+      repoNameToTitle(opts.repo),
+    );
+    const program = buildProgramJson(signature, opts.model);
+
+    await createRepo(opts.repo, {
+      private: opts.private ?? true,
+      existOk: true,
+      token,
+    });
+
+    await syncAndPush({
+      repo: opts.repo,
+      branch,
+      token,
+      files: { config, program },
+      metadata: opts.metadata ?? null,
+      extraFiles: opts.extra_files ?? null,
+      commitMessage: opts.commit_message ?? "(no commit message)",
+      tag: opts.tag,
+    });
+
+    return new Arbiter(opts.repo, { branch, rev: opts.tag ?? branch });
+  }
+
+  /**
+   * Pull the latest commit of this Arbiter's branch, write whatever is provided
+   * (a new signature rewrites config.json + program.json; metadata updates the
+   * README; extra_files are copied in), then commit & push.
+   */
+  async update(opts: UpdateOptions = {}): Promise<Commit> {
+    const token = resolveToken(opts.access_token);
+
+    const files: PushFiles = {};
+    if (opts.signature) {
+      if (!opts.model) {
+        throw new Error(
+          "update() requires `model` when `signature` is provided, so the " +
+            "rewritten program.json records the inference model.",
+        );
+      }
+      const signature = ensure_reasoning_field(opts.signature);
+      files.config = serializeSignatureToConfig(
+        signature,
+        repoNameToTitle(this.repo),
+      );
+      files.program = buildProgramJson(signature, opts.model);
+    }
+
+    return syncAndPush({
+      repo: this.repo,
+      branch: this.branch,
+      token,
+      files,
+      metadata: opts.metadata ?? null,
+      extraFiles: opts.extra_files ?? null,
+      commitMessage: opts.commit_message ?? "(no commit message)",
+      tag: opts.tag,
+    });
+  }
+
+  /**
+   * Run this Arbiter against a single input via the Modaic API. The server runs
+   * the LLM; this is a pure HTTP call through the generated SDK
+   * (`POST /api/v2/arbiters/predictions`).
+   */
+  async predict(
+    input: Record<string, unknown>,
+    opts: PredictOptions = {},
+  ): Promise<PredictExampleResponse> {
+    // Resolve the token up front so a missing token throws AuthenticationError
+    // before any client is constructed.
+    const token = resolveToken();
+    const client = new ModaicClient({ token, serverURL: getApiUrl() });
+    return client.createPredictionApiV2ArbitersPredictionsPost({
+      input,
+      arbiterRepo: this.repo,
+      arbiterRevision: this.rev,
+      groundTruth: opts.ground_truth ?? null,
+      groundReasoning: opts.ground_reasoning ?? "",
+      computeConfidence: opts.compute_confidence ?? false,
+    });
+  }
+
+  /**
+   * Alias for {@link predict}. (Python's `__call__` is not idiomatic in TS.)
+   */
+  call(
+    input: Record<string, unknown>,
+    opts: PredictOptions = {},
+  ): Promise<PredictExampleResponse> {
+    return this.predict(input, opts);
+  }
+}