@orq-ai/node 4.15.0-rc.9 → 4.16.0-rc.0

package/src/models/operations/createeval.ts

@@ -3,18 +3,640 @@
  */
 
 import * as z from "zod/v3";
+import { remap as remap$ } from "../../lib/primitives.js";
+import { safeParse } from "../../lib/schemas.js";
+import { ClosedEnum } from "../../types/enums.js";
+import { Result as SafeParseResult } from "../../types/fp.js";
+import * as components from "../components/index.js";
+import { SDKValidationError } from "../errors/sdkvalidationerror.js";
 
-export type CreateEvalRequestBody = {};
+export const OutputType = {
+  Boolean: "boolean",
+  Number: "number",
+} as const;
+export type OutputType = ClosedEnum<typeof OutputType>;
+
+export const CreateEvalRequestBodyType = {
+  PythonEval: "python_eval",
+} as const;
+export type CreateEvalRequestBodyType = ClosedEnum<
+  typeof CreateEvalRequestBodyType
+>;
+
+export type Python = {
+  guardrailConfig?: any | undefined;
+  outputType?: OutputType | undefined;
+  code: string;
+  type: CreateEvalRequestBodyType;
+  /**
+   * Legacy alternative to `project_id`. Storage path whose first segment names the project that owns the evaluator. Mutually exclusive with `project_id`.
+   */
+  path?: string | undefined;
+  /**
+   * Unique identifier of the project that owns the evaluator, as returned by `GET /v2/projects`. Mutually exclusive with `path`.
+   */
+  projectId?: string | undefined;
+  description?: string | undefined;
+  key: string;
+};
+
+/**
+ * The type of output expected from the evaluator
+ */
+export const OneOutputType = {
+  Boolean: "boolean",
+  Categorical: "categorical",
+  Number: "number",
+  String: "string",
+} as const;
+/**
+ * The type of output expected from the evaluator
+ */
+export type OneOutputType = ClosedEnum<typeof OneOutputType>;
+
+export const CreateEval1Type = {
+  LlmEval: "llm_eval",
+} as const;
+export type CreateEval1Type = ClosedEnum<typeof CreateEval1Type>;
+
+export type OneCategoricalLabels = {
+  value: string;
+  description?: string | undefined;
+};
+
+export type OneRetry = {
+  count?: number | undefined;
+  onCodes?: Array<number> | undefined;
+};
+
+export type OneFallbacks = {
+  model: string;
+};
+
+export type OneJudges = {
+  model: string;
+  retry?: OneRetry | undefined;
+  fallbacks?: Array<OneFallbacks> | undefined;
+};
+
+export type CreateEval1Retry = {
+  count?: number | undefined;
+  onCodes?: Array<number> | undefined;
+};
+
+export type CreateEval1Fallbacks = {
+  model: string;
+};
+
+export type OneReplacementJudges = {
+  model: string;
+  retry?: CreateEval1Retry | undefined;
+  fallbacks?: Array<CreateEval1Fallbacks> | undefined;
+};
+
+export const OneTieValue = {
+  Tie: "Tie",
+} as const;
+export type OneTieValue = ClosedEnum<typeof OneTieValue>;
+
+export type OneJury = {
+  judges: Array<OneJudges>;
+  replacementJudges?: Array<OneReplacementJudges> | undefined;
+  minSuccessfulJudges?: number | undefined;
+  tieValue?: OneTieValue | undefined;
+};
+
+export type LLMJury = {
+  guardrailConfig?: any | undefined;
+  /**
+   * The type of output expected from the evaluator
+   */
+  outputType?: OneOutputType | undefined;
+  type: CreateEval1Type;
+  repetitions?: number | null | undefined;
+  prompt: string;
+  categories?: Array<string> | null | undefined;
+  categoricalLabels?: Array<OneCategoricalLabels> | null | undefined;
+  datasetId?: string | null | undefined;
+  /**
+   * Legacy alternative to `project_id`. Storage path whose first segment names the project that owns the evaluator. Mutually exclusive with `project_id`.
+   */
+  path?: string | undefined;
+  /**
+   * Unique identifier of the project that owns the evaluator, as returned by `GET /v2/projects`. Mutually exclusive with `path`.
+   */
+  projectId?: string | undefined;
+  description?: string | undefined;
+  key: string;
+  mode: "jury";
+  jury: OneJury;
+};
+
+/**
+ * The type of output expected from the evaluator
+ */
+export const CreateEval1OutputType = {
+  Boolean: "boolean",
+  Categorical: "categorical",
+  Number: "number",
+  String: "string",
+} as const;
+/**
+ * The type of output expected from the evaluator
+ */
+export type CreateEval1OutputType = ClosedEnum<typeof CreateEval1OutputType>;
+
+export const OneType = {
+  LlmEval: "llm_eval",
+} as const;
+export type OneType = ClosedEnum<typeof OneType>;
+
+export type CreateEval1CategoricalLabels = {
+  value: string;
+  description?: string | undefined;
+};
+
+export type Llm = {
+  guardrailConfig?: any | undefined;
+  /**
+   * The type of output expected from the evaluator
+   */
+  outputType?: CreateEval1OutputType | undefined;
+  type: OneType;
+  repetitions?: number | null | undefined;
+  prompt: string;
+  categories?: Array<string> | null | undefined;
+  categoricalLabels?: Array<CreateEval1CategoricalLabels> | null | undefined;
+  datasetId?: string | null | undefined;
+  /**
+   * Legacy alternative to `project_id`. Storage path whose first segment names the project that owns the evaluator. Mutually exclusive with `project_id`.
+   */
+  path?: string | undefined;
+  /**
+   * Unique identifier of the project that owns the evaluator, as returned by `GET /v2/projects`. Mutually exclusive with `path`.
+   */
+  projectId?: string | undefined;
+  description?: string | undefined;
+  key: string;
+  mode: "single";
+  model: string;
+};
+
+export type CreateEvalRequestBody1 = Llm | LLMJury;
+
+export type CreateEvalRequestBody = Python | Llm | LLMJury;
+
+/**
+ * Successfully created an evaluator
+ */
+export type CreateEvalResponseBody =
+  | components.EvaluatorResponseLlm
+  | components.EvaluatorResponseJsonSchema
+  | components.EvaluatorResponseHttp
+  | components.EvaluatorResponsePython
+  | components.EvaluatorResponseFunction
+  | components.EvaluatorResponseRagas
+  | components.EvaluatorResponseTypescript;
+
+/** @internal */
+export const OutputType$outboundSchema: z.ZodNativeEnum<typeof OutputType> = z
+  .nativeEnum(OutputType);
+
+/** @internal */
+export const CreateEvalRequestBodyType$outboundSchema: z.ZodNativeEnum<
+  typeof CreateEvalRequestBodyType
+> = z.nativeEnum(CreateEvalRequestBodyType);
+
+/** @internal */
+export type Python$Outbound = {
+  guardrail_config?: any | undefined;
+  output_type?: string | undefined;
+  code: string;
+  type: string;
+  path?: string | undefined;
+  project_id?: string | undefined;
+  description: string;
+  key: string;
+};
+
+/** @internal */
+export const Python$outboundSchema: z.ZodType<
+  Python$Outbound,
+  z.ZodTypeDef,
+  Python
+> = z.object({
+  guardrailConfig: z.any().optional(),
+  outputType: OutputType$outboundSchema.optional(),
+  code: z.string(),
+  type: CreateEvalRequestBodyType$outboundSchema,
+  path: z.string().optional(),
+  projectId: z.string().optional(),
+  description: z.string().default(""),
+  key: z.string(),
+}).transform((v) => {
+  return remap$(v, {
+    guardrailConfig: "guardrail_config",
+    outputType: "output_type",
+    projectId: "project_id",
+  });
+});
+
+export function pythonToJSON(python: Python): string {
+  return JSON.stringify(Python$outboundSchema.parse(python));
+}
+
+/** @internal */
+export const OneOutputType$outboundSchema: z.ZodNativeEnum<
+  typeof OneOutputType
+> = z.nativeEnum(OneOutputType);
+
+/** @internal */
+export const CreateEval1Type$outboundSchema: z.ZodNativeEnum<
+  typeof CreateEval1Type
+> = z.nativeEnum(CreateEval1Type);
+
+/** @internal */
+export type OneCategoricalLabels$Outbound = {
+  value: string;
+  description?: string | undefined;
+};
+
+/** @internal */
+export const OneCategoricalLabels$outboundSchema: z.ZodType<
+  OneCategoricalLabels$Outbound,
+  z.ZodTypeDef,
+  OneCategoricalLabels
+> = z.object({
+  value: z.string(),
+  description: z.string().optional(),
+});
+
+export function oneCategoricalLabelsToJSON(
+  oneCategoricalLabels: OneCategoricalLabels,
+): string {
+  return JSON.stringify(
+    OneCategoricalLabels$outboundSchema.parse(oneCategoricalLabels),
+  );
+}
+
+/** @internal */
+export type OneRetry$Outbound = {
+  count: number;
+  on_codes?: Array<number> | undefined;
+};
+
+/** @internal */
+export const OneRetry$outboundSchema: z.ZodType<
+  OneRetry$Outbound,
+  z.ZodTypeDef,
+  OneRetry
+> = z.object({
+  count: z.number().int().default(2),
+  onCodes: z.array(z.number().int()).optional(),
+}).transform((v) => {
+  return remap$(v, {
+    onCodes: "on_codes",
+  });
+});
+
+export function oneRetryToJSON(oneRetry: OneRetry): string {
+  return JSON.stringify(OneRetry$outboundSchema.parse(oneRetry));
+}
+
+/** @internal */
+export type OneFallbacks$Outbound = {
+  model: string;
+};
+
+/** @internal */
+export const OneFallbacks$outboundSchema: z.ZodType<
+  OneFallbacks$Outbound,
+  z.ZodTypeDef,
+  OneFallbacks
+> = z.object({
+  model: z.string(),
+});
+
+export function oneFallbacksToJSON(oneFallbacks: OneFallbacks): string {
+  return JSON.stringify(OneFallbacks$outboundSchema.parse(oneFallbacks));
+}
+
+/** @internal */
+export type OneJudges$Outbound = {
+  model: string;
+  retry?: OneRetry$Outbound | undefined;
+  fallbacks?: Array<OneFallbacks$Outbound> | undefined;
+};
+
+/** @internal */
+export const OneJudges$outboundSchema: z.ZodType<
+  OneJudges$Outbound,
+  z.ZodTypeDef,
+  OneJudges
+> = z.object({
+  model: z.string(),
+  retry: z.lazy(() => OneRetry$outboundSchema).optional(),
+  fallbacks: z.array(z.lazy(() => OneFallbacks$outboundSchema)).optional(),
+});
+
+export function oneJudgesToJSON(oneJudges: OneJudges): string {
+  return JSON.stringify(OneJudges$outboundSchema.parse(oneJudges));
+}
+
+/** @internal */
+export type CreateEval1Retry$Outbound = {
+  count: number;
+  on_codes?: Array<number> | undefined;
+};
+
+/** @internal */
+export const CreateEval1Retry$outboundSchema: z.ZodType<
+  CreateEval1Retry$Outbound,
+  z.ZodTypeDef,
+  CreateEval1Retry
+> = z.object({
+  count: z.number().int().default(2),
+  onCodes: z.array(z.number().int()).optional(),
+}).transform((v) => {
+  return remap$(v, {
+    onCodes: "on_codes",
+  });
+});
+
+export function createEval1RetryToJSON(
+  createEval1Retry: CreateEval1Retry,
+): string {
+  return JSON.stringify(
+    CreateEval1Retry$outboundSchema.parse(createEval1Retry),
+  );
+}
+
+/** @internal */
+export type CreateEval1Fallbacks$Outbound = {
+  model: string;
+};
+
+/** @internal */
+export const CreateEval1Fallbacks$outboundSchema: z.ZodType<
+  CreateEval1Fallbacks$Outbound,
+  z.ZodTypeDef,
+  CreateEval1Fallbacks
+> = z.object({
+  model: z.string(),
+});
+
+export function createEval1FallbacksToJSON(
+  createEval1Fallbacks: CreateEval1Fallbacks,
+): string {
+  return JSON.stringify(
+    CreateEval1Fallbacks$outboundSchema.parse(createEval1Fallbacks),
+  );
+}
 
 /** @internal */
-export type CreateEvalRequestBody$Outbound = {};
+export type OneReplacementJudges$Outbound = {
+  model: string;
+  retry?: CreateEval1Retry$Outbound | undefined;
+  fallbacks?: Array<CreateEval1Fallbacks$Outbound> | undefined;
+};
+
+/** @internal */
+export const OneReplacementJudges$outboundSchema: z.ZodType<
+  OneReplacementJudges$Outbound,
+  z.ZodTypeDef,
+  OneReplacementJudges
+> = z.object({
+  model: z.string(),
+  retry: z.lazy(() => CreateEval1Retry$outboundSchema).optional(),
+  fallbacks: z.array(z.lazy(() => CreateEval1Fallbacks$outboundSchema))
+    .optional(),
+});
+
+export function oneReplacementJudgesToJSON(
+  oneReplacementJudges: OneReplacementJudges,
+): string {
+  return JSON.stringify(
+    OneReplacementJudges$outboundSchema.parse(oneReplacementJudges),
+  );
+}
+
+/** @internal */
+export const OneTieValue$outboundSchema: z.ZodNativeEnum<typeof OneTieValue> = z
+  .nativeEnum(OneTieValue);
+
+/** @internal */
+export type OneJury$Outbound = {
+  judges: Array<OneJudges$Outbound>;
+  replacement_judges?: Array<OneReplacementJudges$Outbound> | undefined;
+  min_successful_judges: number;
+  tie_value: string;
+};
+
+/** @internal */
+export const OneJury$outboundSchema: z.ZodType<
+  OneJury$Outbound,
+  z.ZodTypeDef,
+  OneJury
+> = z.object({
+  judges: z.array(z.lazy(() => OneJudges$outboundSchema)),
+  replacementJudges: z.array(z.lazy(() => OneReplacementJudges$outboundSchema))
+    .optional(),
+  minSuccessfulJudges: z.number().int().default(2),
+  tieValue: OneTieValue$outboundSchema.default("Tie"),
+}).transform((v) => {
+  return remap$(v, {
+    replacementJudges: "replacement_judges",
+    minSuccessfulJudges: "min_successful_judges",
+    tieValue: "tie_value",
+  });
+});
+
+export function oneJuryToJSON(oneJury: OneJury): string {
+  return JSON.stringify(OneJury$outboundSchema.parse(oneJury));
+}
+
+/** @internal */
+export type LLMJury$Outbound = {
+  guardrail_config?: any | undefined;
+  output_type?: string | undefined;
+  type: string;
+  repetitions?: number | null | undefined;
+  prompt: string;
+  categories?: Array<string> | null | undefined;
+  categorical_labels?: Array<OneCategoricalLabels$Outbound> | null | undefined;
+  dataset_id?: string | null | undefined;
+  path?: string | undefined;
+  project_id?: string | undefined;
+  description: string;
+  key: string;
+  mode: "jury";
+  jury: OneJury$Outbound;
+};
+
+/** @internal */
+export const LLMJury$outboundSchema: z.ZodType<
+  LLMJury$Outbound,
+  z.ZodTypeDef,
+  LLMJury
+> = z.object({
+  guardrailConfig: z.any().optional(),
+  outputType: OneOutputType$outboundSchema.optional(),
+  type: CreateEval1Type$outboundSchema,
+  repetitions: z.nullable(z.number().int()).optional(),
+  prompt: z.string(),
+  categories: z.nullable(z.array(z.string())).optional(),
+  categoricalLabels: z.nullable(
+    z.array(z.lazy(() => OneCategoricalLabels$outboundSchema)),
+  ).optional(),
+  datasetId: z.nullable(z.string()).optional(),
+  path: z.string().optional(),
+  projectId: z.string().optional(),
+  description: z.string().default(""),
+  key: z.string(),
+  mode: z.literal("jury"),
+  jury: z.lazy(() => OneJury$outboundSchema),
+}).transform((v) => {
+  return remap$(v, {
+    guardrailConfig: "guardrail_config",
+    outputType: "output_type",
+    categoricalLabels: "categorical_labels",
+    datasetId: "dataset_id",
+    projectId: "project_id",
+  });
+});
+
+export function llmJuryToJSON(llmJury: LLMJury): string {
+  return JSON.stringify(LLMJury$outboundSchema.parse(llmJury));
+}
+
+/** @internal */
+export const CreateEval1OutputType$outboundSchema: z.ZodNativeEnum<
+  typeof CreateEval1OutputType
+> = z.nativeEnum(CreateEval1OutputType);
+
+/** @internal */
+export const OneType$outboundSchema: z.ZodNativeEnum<typeof OneType> = z
+  .nativeEnum(OneType);
+
+/** @internal */
+export type CreateEval1CategoricalLabels$Outbound = {
+  value: string;
+  description?: string | undefined;
+};
+
+/** @internal */
+export const CreateEval1CategoricalLabels$outboundSchema: z.ZodType<
+  CreateEval1CategoricalLabels$Outbound,
+  z.ZodTypeDef,
+  CreateEval1CategoricalLabels
+> = z.object({
+  value: z.string(),
+  description: z.string().optional(),
+});
+
+export function createEval1CategoricalLabelsToJSON(
+  createEval1CategoricalLabels: CreateEval1CategoricalLabels,
+): string {
+  return JSON.stringify(
+    CreateEval1CategoricalLabels$outboundSchema.parse(
+      createEval1CategoricalLabels,
+    ),
+  );
+}
+
+/** @internal */
+export type Llm$Outbound = {
+  guardrail_config?: any | undefined;
+  output_type?: string | undefined;
+  type: string;
+  repetitions?: number | null | undefined;
+  prompt: string;
+  categories?: Array<string> | null | undefined;
+  categorical_labels?:
+    | Array<CreateEval1CategoricalLabels$Outbound>
+    | null
+    | undefined;
+  dataset_id?: string | null | undefined;
+  path?: string | undefined;
+  project_id?: string | undefined;
+  description: string;
+  key: string;
+  mode: "single";
+  model: string;
+};
+
+/** @internal */
+export const Llm$outboundSchema: z.ZodType<Llm$Outbound, z.ZodTypeDef, Llm> = z
+  .object({
+    guardrailConfig: z.any().optional(),
+    outputType: CreateEval1OutputType$outboundSchema.optional(),
+    type: OneType$outboundSchema,
+    repetitions: z.nullable(z.number().int()).optional(),
+    prompt: z.string(),
+    categories: z.nullable(z.array(z.string())).optional(),
+    categoricalLabels: z.nullable(
+      z.array(z.lazy(() => CreateEval1CategoricalLabels$outboundSchema)),
+    ).optional(),
+    datasetId: z.nullable(z.string()).optional(),
+    path: z.string().optional(),
+    projectId: z.string().optional(),
+    description: z.string().default(""),
+    key: z.string(),
+    mode: z.literal("single"),
+    model: z.string(),
+  }).transform((v) => {
+    return remap$(v, {
+      guardrailConfig: "guardrail_config",
+      outputType: "output_type",
+      categoricalLabels: "categorical_labels",
+      datasetId: "dataset_id",
+      projectId: "project_id",
+    });
+  });
+
+export function llmToJSON(llm: Llm): string {
+  return JSON.stringify(Llm$outboundSchema.parse(llm));
+}
+
+/** @internal */
+export type CreateEvalRequestBody1$Outbound = Llm$Outbound | LLMJury$Outbound;
+
+/** @internal */
+export const CreateEvalRequestBody1$outboundSchema: z.ZodType<
+  CreateEvalRequestBody1$Outbound,
+  z.ZodTypeDef,
+  CreateEvalRequestBody1
+> = z.union([
+  z.lazy(() => Llm$outboundSchema),
+  z.lazy(() => LLMJury$outboundSchema),
+]);
+
+export function createEvalRequestBody1ToJSON(
+  createEvalRequestBody1: CreateEvalRequestBody1,
+): string {
+  return JSON.stringify(
+    CreateEvalRequestBody1$outboundSchema.parse(createEvalRequestBody1),
+  );
+}
+
+/** @internal */
+export type CreateEvalRequestBody$Outbound =
+  | Python$Outbound
+  | Llm$Outbound
+  | LLMJury$Outbound;
 
 /** @internal */
 export const CreateEvalRequestBody$outboundSchema: z.ZodType<
   CreateEvalRequestBody$Outbound,
   z.ZodTypeDef,
   CreateEvalRequestBody
-> = z.object({});
+> = z.union([
+  z.lazy(() => Python$outboundSchema),
+  z.union([
+    z.lazy(() => Llm$outboundSchema),
+    z.lazy(() => LLMJury$outboundSchema),
+  ]),
+]);
 
 export function createEvalRequestBodyToJSON(
   createEvalRequestBody: CreateEvalRequestBody,
@@ -23,3 +645,28 @@ export function createEvalRequestBodyToJSON(
     CreateEvalRequestBody$outboundSchema.parse(createEvalRequestBody),
   );
 }
+
+/** @internal */
+export const CreateEvalResponseBody$inboundSchema: z.ZodType<
+  CreateEvalResponseBody,
+  z.ZodTypeDef,
+  unknown
+> = z.union([
+  components.EvaluatorResponseLlm$inboundSchema,
+  components.EvaluatorResponseJsonSchema$inboundSchema,
+  components.EvaluatorResponseHttp$inboundSchema,
+  components.EvaluatorResponsePython$inboundSchema,
+  components.EvaluatorResponseFunction$inboundSchema,
+  components.EvaluatorResponseRagas$inboundSchema,
+  components.EvaluatorResponseTypescript$inboundSchema,
+]);
+
+export function createEvalResponseBodyFromJSON(
+  jsonString: string,
+): SafeParseResult<CreateEvalResponseBody, SDKValidationError> {
+  return safeParse(
+    jsonString,
+    (x) => CreateEvalResponseBody$inboundSchema.parse(JSON.parse(x)),
+    `Failed to parse 'CreateEvalResponseBody' from JSON`,
+  );
+}