@keystrokehq/openai 0.1.0 → 0.1.3

package/dist/actions/run-grader.cjs

@@ -15,20 +15,20 @@ const OpenaiRunGraderInput = zod.z.object({
 				"ilike"
 			]).describe("The comparison operation to perform: 'eq' for equals, 'ne' for not equals, 'like' for SQL LIKE, 'ilike' for case-insensitive LIKE"),
 			reference: zod.z.string().describe("The reference value to compare against")
-		}).describe("String check grader configuration."),
+		}).passthrough().describe("String check grader configuration."),
 		zod.z.object({
 			name: zod.z.string().describe("Name identifier for this grader"),
 			type: zod.z.string().describe("Type of grader, must be 'text_similarity'"),
 			input: zod.z.string().describe("The input field to evaluate"),
 			reference: zod.z.string().describe("The reference text to compare against"),
 			evaluation_metric: zod.z.string().describe("The metric to use for evaluating text similarity")
-		}).describe("Text similarity grader configuration."),
+		}).passthrough().describe("Text similarity grader configuration."),
 		zod.z.object({
 			name: zod.z.string().describe("Name identifier for this grader"),
 			type: zod.z.string().describe("Type of grader, must be 'python'"),
 			source: zod.z.string().describe("Python source code to execute for grading"),
 			image_tag: zod.z.string().nullable().optional()
-		}).describe("Python grader configuration."),
+		}).passthrough().describe("Python grader configuration."),
 		zod.z.object({
 			name: zod.z.string().describe("Name identifier for this grader"),
 			type: zod.z.string().describe("Type of grader, must be 'score_model'"),
@@ -36,59 +36,59 @@ const OpenaiRunGraderInput = zod.z.object({
 			model: zod.z.string().describe("The model to use for scoring"),
 			range: zod.z.array(zod.z.union([zod.z.number().int(), zod.z.number()])).nullable().optional(),
 			sampling_params: zod.z.record(zod.z.string(), zod.z.unknown()).nullable().optional()
-		}).describe("Score model grader configuration."),
+		}).passthrough().describe("Score model grader configuration."),
 		zod.z.object({
 			name: zod.z.string().describe("Name identifier for this grader"),
 			type: zod.z.string().describe("Type of grader, must be 'multi'"),
 			graders: zod.z.array(zod.z.record(zod.z.string(), zod.z.unknown())).describe("List of grader configurations to combine"),
 			calculate_output: zod.z.string().describe("Formula to calculate the final output from sub-grader results")
-		}).describe("Multi grader configuration that combines multiple graders."),
+		}).passthrough().describe("Multi grader configuration that combines multiple graders."),
 		zod.z.record(zod.z.string(), zod.z.unknown())
 	]),
 	model_sample: zod.z.string().describe("The model sample output to be evaluated. This value will be used to populate the 'sample' namespace. The 'output_json' variable will be populated if the model sample is a valid JSON string")
 }).describe("Request model for running a grader to evaluate model performance.");
 const OpenaiRunGrader_GraderErrorsSchema = zod.z.object({
-	other_error: zod.z.boolean().describe("Whether another type of error occurred"),
-	sample_parse_error: zod.z.boolean().describe("Whether a sample parsing error occurred"),
-	formula_parse_error: zod.z.boolean().describe("Whether a formula parsing error occurred"),
-	invalid_variable_error: zod.z.boolean().describe("Whether an invalid variable error occurred"),
-	model_grader_parse_error: zod.z.boolean().describe("Whether a model grader parsing error occurred"),
-	model_grader_server_error: zod.z.boolean().describe("Whether a model grader server error occurred"),
-	unresponsive_reward_error: zod.z.boolean().describe("Whether an unresponsive reward error occurred"),
-	model_grader_refusal_error: zod.z.boolean().describe("Whether a model grader refusal error occurred"),
-	python_grader_server_error: zod.z.boolean().describe("Whether a Python grader server error occurred"),
+	other_error: zod.z.boolean().describe("Whether another type of error occurred").nullable(),
+	sample_parse_error: zod.z.boolean().describe("Whether a sample parsing error occurred").nullable(),
+	formula_parse_error: zod.z.boolean().describe("Whether a formula parsing error occurred").nullable(),
+	invalid_variable_error: zod.z.boolean().describe("Whether an invalid variable error occurred").nullable(),
+	model_grader_parse_error: zod.z.boolean().describe("Whether a model grader parsing error occurred").nullable(),
+	model_grader_server_error: zod.z.boolean().describe("Whether a model grader server error occurred").nullable(),
+	unresponsive_reward_error: zod.z.boolean().describe("Whether an unresponsive reward error occurred").nullable(),
+	model_grader_refusal_error: zod.z.boolean().describe("Whether a model grader refusal error occurred").nullable(),
+	python_grader_server_error: zod.z.boolean().describe("Whether a Python grader server error occurred").nullable(),
 	sample_parse_error_details: zod.z.record(zod.z.string(), zod.z.unknown()).describe("Details about the sample parsing error").nullable().optional(),
-	python_grader_runtime_error: zod.z.boolean().describe("Whether a Python grader runtime error occurred"),
-	truncated_observation_error: zod.z.boolean().describe("Whether an observation truncation error occurred"),
-	endpoint_grader_server_error: zod.z.boolean().describe("Whether an endpoint grader server error occurred"),
-	endpoint_grader_internal_error: zod.z.boolean().describe("Whether an endpoint grader internal error occurred"),
+	python_grader_runtime_error: zod.z.boolean().describe("Whether a Python grader runtime error occurred").nullable(),
+	truncated_observation_error: zod.z.boolean().describe("Whether an observation truncation error occurred").nullable(),
+	endpoint_grader_server_error: zod.z.boolean().describe("Whether an endpoint grader server error occurred").nullable(),
+	endpoint_grader_internal_error: zod.z.boolean().describe("Whether an endpoint grader internal error occurred").nullable(),
 	invalid_variable_error_details: zod.z.record(zod.z.string(), zod.z.unknown()).describe("Details about the invalid variable error").nullable().optional(),
 	python_grader_server_error_type: zod.z.record(zod.z.string(), zod.z.unknown()).describe("Details about the Python grader server error type").nullable().optional(),
 	model_grader_parse_error_details: zod.z.record(zod.z.string(), zod.z.unknown()).describe("Details about the model grader parsing error").nullable().optional(),
 	model_grader_server_error_details: zod.z.record(zod.z.string(), zod.z.unknown()).describe("Details about the model grader server error").nullable().optional(),
-	endpoint_grader_safety_check_error: zod.z.boolean().describe("Whether an endpoint grader safety check error occurred"),
+	endpoint_grader_safety_check_error: zod.z.boolean().describe("Whether an endpoint grader safety check error occurred").nullable(),
 	model_grader_refusal_error_details: zod.z.record(zod.z.string(), zod.z.unknown()).describe("Details about the model grader refusal error").nullable().optional(),
 	python_grader_runtime_error_details: zod.z.record(zod.z.string(), zod.z.unknown()).describe("Details about the Python grader runtime error").nullable().optional(),
 	endpoint_grader_server_error_details: zod.z.record(zod.z.string(), zod.z.unknown()).describe("Details about the endpoint grader server error").nullable().optional(),
 	endpoint_grader_internal_error_details: zod.z.record(zod.z.string(), zod.z.unknown()).describe("Details about the endpoint grader internal error").nullable().optional(),
-	model_grader_exceeded_max_tokens_error: zod.z.boolean().describe("Whether a model grader exceeded max tokens error occurred")
-}).describe("Detailed error information for grader execution.");
+	model_grader_exceeded_max_tokens_error: zod.z.boolean().describe("Whether a model grader exceeded max tokens error occurred").nullable()
+}).passthrough().describe("Detailed error information for grader execution.");
 const OpenaiRunGrader_GraderMetadataSchema = zod.z.object({
-	name: zod.z.string().describe("The name of the grader that was executed"),
-	type: zod.z.string().describe("The type of grader that was executed"),
+	name: zod.z.string().describe("The name of the grader that was executed").nullable(),
+	type: zod.z.string().describe("The type of grader that was executed").nullable(),
 	errors: OpenaiRunGrader_GraderErrorsSchema.nullable(),
 	scores: zod.z.record(zod.z.string(), zod.z.unknown()).describe("Individual scores computed by the grader"),
 	metadata: zod.z.record(zod.z.string(), zod.z.unknown()).describe("Additional metadata from the grader execution").nullable().optional(),
 	token_usage: zod.z.number().int().describe("Number of tokens used during grader execution, if applicable").nullable().optional(),
-	execution_time: zod.z.number().describe("Time taken to execute the grader in seconds"),
+	execution_time: zod.z.number().describe("Time taken to execute the grader in seconds").nullable(),
 	sampled_model_name: zod.z.string().describe("Name of the model that was sampled, if applicable").nullable().optional()
-}).describe("Metadata about the grader execution.");
+}).passthrough().describe("Metadata about the grader execution.");
 const OpenaiRunGraderOutput = zod.z.object({
-	reward: zod.z.number().describe("The final reward score computed by the grader. Higher values indicate better performance"),
+	reward: zod.z.number().describe("The final reward score computed by the grader. Higher values indicate better performance").nullable(),
 	metadata: OpenaiRunGrader_GraderMetadataSchema.nullable(),
 	sub_rewards: zod.z.record(zod.z.string(), zod.z.unknown()).describe("Individual sub-rewards from component graders, if using a multi-grader"),
 	model_grader_token_usage_per_model: zod.z.record(zod.z.string(), zod.z.unknown()).describe("Token usage breakdown per model used in grading")
-}).describe("Response schema for running a grader.");
+}).passthrough().describe("Response schema for running a grader.");
 const openaiRunGrader = require_action.action("OPENAI_RUN_GRADER", {
 	slug: "openai-run-grader",
 	name: "Run grader",

package/dist/actions/run-grader.cjs.map

@@ -1 +1 @@
-{"version":3,"file":"run-grader.cjs","names":["z","action"],"sources":["../../src/actions/run-grader.ts"],"sourcesContent":["import { z } from \"zod\";\n\nimport { action } from \"../action\";\n\nexport const OpenaiRunGraderInput: z.ZodTypeAny = z.object({\n  item: z.record(z.string(), z.unknown()).describe(\"The dataset item provided to the grader. This will be used to populate the 'item' namespace. See the guide for more details\").optional(),\n  grader: z.union([z.object({\n  name: z.string().describe(\"Name identifier for this grader\"),\n  type: z.string().describe(\"Type of grader, must be 'string_check'\"),\n  input: z.string().describe(\"The input field to check. Use 'sample.output' to reference the model sample output\"),\n  operation: z.enum([\"eq\", \"ne\", \"like\", \"ilike\"]).describe(\"The comparison operation to perform: 'eq' for equals, 'ne' for not equals, 'like' for SQL LIKE, 'ilike' for case-insensitive LIKE\"),\n  reference: z.string().describe(\"The reference value to compare against\"),\n}).describe(\"String check grader configuration.\"), z.object({\n  name: z.string().describe(\"Name identifier for this grader\"),\n  type: z.string().describe(\"Type of grader, must be 'text_similarity'\"),\n  input: z.string().describe(\"The input field to evaluate\"),\n  reference: z.string().describe(\"The reference text to compare against\"),\n  evaluation_metric: z.string().describe(\"The metric to use for evaluating text similarity\"),\n}).describe(\"Text similarity grader configuration.\"), z.object({\n  name: z.string().describe(\"Name identifier for this grader\"),\n  type: z.string().describe(\"Type of grader, must be 'python'\"),\n  source: z.string().describe(\"Python source code to execute for grading\"),\n  image_tag: z.string().nullable().optional(),\n}).describe(\"Python grader configuration.\"), z.object({\n  name: z.string().describe(\"Name identifier for this grader\"),\n  type: z.string().describe(\"Type of grader, must be 'score_model'\"),\n  input: z.array(z.record(z.string(), z.unknown())).describe(\"Input data for the scoring model\"),\n  model: z.string().describe(\"The model to use for scoring\"),\n  range: z.array(z.union([z.number().int(), z.number()])).nullable().optional(),\n  sampling_params: z.record(z.string(), z.unknown()).nullable().optional(),\n}).describe(\"Score model grader configuration.\"), z.object({\n  name: z.string().describe(\"Name identifier for this grader\"),\n  type: z.string().describe(\"Type of grader, must be 'multi'\"),\n  graders: z.array(z.record(z.string(), z.unknown())).describe(\"List of grader configurations to combine\"),\n  calculate_output: z.string().describe(\"Formula to calculate the final output from sub-grader results\"),\n}).describe(\"Multi grader configuration that combines multiple graders.\"), z.record(z.string(), z.unknown())]),\n  model_sample: z.string().describe(\"The model sample output to be evaluated. This value will be used to populate the 'sample' namespace. The 'output_json' variable will be populated if the model sample is a valid JSON string\"),\n}).describe(\"Request model for running a grader to evaluate model performance.\");\nconst OpenaiRunGrader_GraderErrorsSchema: z.ZodTypeAny = z.object({\n  other_error: z.boolean().describe(\"Whether another type of error occurred\"),\n  sample_parse_error: z.boolean().describe(\"Whether a sample parsing error occurred\"),\n  formula_parse_error: z.boolean().describe(\"Whether a formula parsing error occurred\"),\n  invalid_variable_error: z.boolean().describe(\"Whether an invalid variable error occurred\"),\n  model_grader_parse_error: z.boolean().describe(\"Whether a model grader parsing error occurred\"),\n  model_grader_server_error: z.boolean().describe(\"Whether a model grader server error occurred\"),\n  unresponsive_reward_error: z.boolean().describe(\"Whether an unresponsive reward error occurred\"),\n  model_grader_refusal_error: z.boolean().describe(\"Whether a model grader refusal error occurred\"),\n  python_grader_server_error: z.boolean().describe(\"Whether a Python grader server error occurred\"),\n  sample_parse_error_details: z.record(z.string(), z.unknown()).describe(\"Details about the sample parsing error\").nullable().optional(),\n  python_grader_runtime_error: z.boolean().describe(\"Whether a Python grader runtime error occurred\"),\n  truncated_observation_error: z.boolean().describe(\"Whether an observation truncation error occurred\"),\n  endpoint_grader_server_error: z.boolean().describe(\"Whether an endpoint grader server error occurred\"),\n  endpoint_grader_internal_error: z.boolean().describe(\"Whether an endpoint grader internal error occurred\"),\n  invalid_variable_error_details: z.record(z.string(), z.unknown()).describe(\"Details about the invalid variable error\").nullable().optional(),\n  python_grader_server_error_type: z.record(z.string(), z.unknown()).describe(\"Details about the Python grader server error type\").nullable().optional(),\n  model_grader_parse_error_details: z.record(z.string(), z.unknown()).describe(\"Details about the model grader parsing error\").nullable().optional(),\n  model_grader_server_error_details: z.record(z.string(), z.unknown()).describe(\"Details about the model grader server error\").nullable().optional(),\n  endpoint_grader_safety_check_error: z.boolean().describe(\"Whether an endpoint grader safety check error occurred\"),\n  model_grader_refusal_error_details: z.record(z.string(), z.unknown()).describe(\"Details about the model grader refusal error\").nullable().optional(),\n  python_grader_runtime_error_details: z.record(z.string(), z.unknown()).describe(\"Details about the Python grader runtime error\").nullable().optional(),\n  endpoint_grader_server_error_details: z.record(z.string(), z.unknown()).describe(\"Details about the endpoint grader server error\").nullable().optional(),\n  endpoint_grader_internal_error_details: z.record(z.string(), z.unknown()).describe(\"Details about the endpoint grader internal error\").nullable().optional(),\n  model_grader_exceeded_max_tokens_error: z.boolean().describe(\"Whether a model grader exceeded max tokens error occurred\"),\n}).describe(\"Detailed error information for grader execution.\");\nconst OpenaiRunGrader_GraderMetadataSchema: z.ZodTypeAny = z.object({\n  name: z.string().describe(\"The name of the grader that was executed\"),\n  type: z.string().describe(\"The type of grader that was executed\"),\n  errors: OpenaiRunGrader_GraderErrorsSchema.nullable(),\n  scores: z.record(z.string(), z.unknown()).describe(\"Individual scores computed by the grader\"),\n  metadata: z.record(z.string(), z.unknown()).describe(\"Additional metadata from the grader execution\").nullable().optional(),\n  token_usage: z.number().int().describe(\"Number of tokens used during grader execution, if applicable\").nullable().optional(),\n  execution_time: z.number().describe(\"Time taken to execute the grader in seconds\"),\n  sampled_model_name: z.string().describe(\"Name of the model that was sampled, if applicable\").nullable().optional(),\n}).describe(\"Metadata about the grader execution.\");\nexport const OpenaiRunGraderOutput: z.ZodTypeAny = z.object({\n  reward: z.number().describe(\"The final reward score computed by the grader. Higher values indicate better performance\"),\n  metadata: OpenaiRunGrader_GraderMetadataSchema.nullable(),\n  sub_rewards: z.record(z.string(), z.unknown()).describe(\"Individual sub-rewards from component graders, if using a multi-grader\"),\n  model_grader_token_usage_per_model: z.record(z.string(), z.unknown()).describe(\"Token usage breakdown per model used in grading\"),\n}).describe(\"Response schema for running a grader.\");\n\nexport const openaiRunGrader = action(\"OPENAI_RUN_GRADER\", {\n  slug: \"openai-run-grader\",\n  name: \"Run grader\",\n  description: \"Tool to run a grader to evaluate model performance on a given sample. Use when you need to evaluate model outputs against expected results using various grading strategies like string matching, text similarity, custom Python code, or model-based scoring.\",\n  input: OpenaiRunGraderInput,\n  output: OpenaiRunGraderOutput,\n});\n"],"mappings":";;;AAIA,MAAa,uBAAqCA,IAAAA,EAAE,OAAO;CACzD,MAAMA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,6HAA6H,CAAC,CAAC,SAAS;CACzL,QAAQA,IAAAA,EAAE,MAAM;EAACA,IAAAA,EAAE,OAAO;GAC1B,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,iCAAiC;GAC3D,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,wCAAwC;GAClE,OAAOA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,oFAAoF;GAC/G,WAAWA,IAAAA,EAAE,KAAK;IAAC;IAAM;IAAM;IAAQ;GAAO,CAAC,CAAC,CAAC,SAAS,mIAAmI;GAC7L,WAAWA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,wCAAwC;EACzE,CAAC,CAAC,CAAC,SAAS,oCAAoC;EAAGA,IAAAA,EAAE,OAAO;GAC1D,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,iCAAiC;GAC3D,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,2CAA2C;GACrE,OAAOA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,6BAA6B;GACxD,WAAWA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,uCAAuC;GACtE,mBAAmBA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,kDAAkD;EAC3F,CAAC,CAAC,CAAC,SAAS,uCAAuC;EAAGA,IAAAA,EAAE,OAAO;GAC7D,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,iCAAiC;GAC3D,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,kCAAkC;GAC5D,QAAQA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,2CAA2C;GACvE,WAAWA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;EAC5C,CAAC,CAAC,CAAC,SAAS,8BAA8B;EAAGA,IAAAA,EAAE,OAAO;GACpD,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,iCAAiC;GAC3D,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,uCAAuC;GACjE,OAAOA,IAAAA,EAAE,MAAMA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,kCAAkC;GAC7F,OAAOA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,8BAA8B;GACzD,OAAOA,IAAAA,EAAE,MAAMA,IAAAA,EAAE,MAAM,CAACA,IAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,GAAGA,IAAAA,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;GAC5E,iBAAiBA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;EACzE,CAAC,CAAC,CAAC,SAAS,mCAAmC;EAAGA,IAAAA,EAAE,OAAO;GACzD,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,iCAAiC;GAC3D,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,iCAAiC;GAC3D,SAASA,IAAAA,EAAE,MAAMA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,0CAA0C;GACvG,kBAAkBA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,+DAA+D;EACvG,CAAC,CAAC,CAAC,SAAS,4DAA4D;EAAGA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC;CAAC,CAAC;CAC3G,cAAcA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,8LAA8L;AAClO,CAAC,CAAC,CAAC,SAAS,mEAAmE;AAC/E,MAAM,qCAAmDA,IAAAA,EAAE,OAAO;CAChE,aAAaA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,wCAAwC;CAC1E,oBAAoBA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,yCAAyC;CAClF,qBAAqBA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,0CAA0C;CACpF,wBAAwBA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,4CAA4C;CACzF,0BAA0BA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,+CAA+C;CAC9F,2BAA2BA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,8CAA8C;CAC9F,2BAA2BA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,+CAA+C;CAC/F,4BAA4BA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,+CAA+C;CAChG,4BAA4BA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,+CAA+C;CAChG,4BAA4BA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,wCAAwC,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CACrI,6BAA6BA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,gDAAgD;CAClG,6BAA6BA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,kDAAkD;CACpG,8BAA8BA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,kDAAkD;CACrG,gCAAgCA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,oDAAoD;CACzG,gCAAgCA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,0CAA0C,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CAC3I,iCAAiCA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,mDAAmD,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CACrJ,kCAAkCA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,8CAA8C,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CACjJ,mCAAmCA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,6CAA6C,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CACjJ,oCAAoCA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,wDAAwD;CACjH,oCAAoCA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,8CAA8C,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CACnJ,qCAAqCA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,+CAA+C,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CACrJ,sCAAsCA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,gDAAgD,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CACvJ,wCAAwCA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,kDAAkD,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CAC3J,wCAAwCA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,2DAA2D;AAC1H,CAAC,CAAC,CAAC,SAAS,kDAAkD;AAC9D,MAAM,uCAAqDA,IAAAA,EAAE,OAAO;CAClE,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,0CAA0C;CACpE,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,sCAAsC;CAChE,QAAQ,mCAAmC,SAAS;CACpD,QAAQA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,0CAA0C;CAC7F,UAAUA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,+CAA+C,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CAC1H,aAAaA,IAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,8DAA8D,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CAC3H,gBAAgBA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,6CAA6C;CACjF,oBAAoBA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,mDAAmD,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;AACnH,CAAC,CAAC,CAAC,SAAS,sCAAsC;AAClD,MAAa,wBAAsCA,IAAAA,EAAE,OAAO;CAC1D,QAAQA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,0FAA0F;CACtH,UAAU,qCAAqC,SAAS;CACxD,aAAaA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,wEAAwE;CAChI,oCAAoCA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,iDAAiD;AAClI,CAAC,CAAC,CAAC,SAAS,uCAAuC;AAEnD,MAAa,kBAAkBC,eAAAA,OAAO,qBAAqB;CACzD,MAAM;CACN,MAAM;CACN,aAAa;CACb,OAAO;CACP,QAAQ;AACV,CAAC"}
+{"version":3,"file":"run-grader.cjs","names":["z","action"],"sources":["../../src/actions/run-grader.ts"],"sourcesContent":["import { z } from \"zod\";\n\nimport { action } from \"../action\";\n\nexport const OpenaiRunGraderInput = z.object({\n  item: z.record(z.string(), z.unknown()).describe(\"The dataset item provided to the grader. This will be used to populate the 'item' namespace. See the guide for more details\").optional(),\n  grader: z.union([z.object({\n  name: z.string().describe(\"Name identifier for this grader\"),\n  type: z.string().describe(\"Type of grader, must be 'string_check'\"),\n  input: z.string().describe(\"The input field to check. Use 'sample.output' to reference the model sample output\"),\n  operation: z.enum([\"eq\", \"ne\", \"like\", \"ilike\"]).describe(\"The comparison operation to perform: 'eq' for equals, 'ne' for not equals, 'like' for SQL LIKE, 'ilike' for case-insensitive LIKE\"),\n  reference: z.string().describe(\"The reference value to compare against\"),\n}).passthrough().describe(\"String check grader configuration.\"), z.object({\n  name: z.string().describe(\"Name identifier for this grader\"),\n  type: z.string().describe(\"Type of grader, must be 'text_similarity'\"),\n  input: z.string().describe(\"The input field to evaluate\"),\n  reference: z.string().describe(\"The reference text to compare against\"),\n  evaluation_metric: z.string().describe(\"The metric to use for evaluating text similarity\"),\n}).passthrough().describe(\"Text similarity grader configuration.\"), z.object({\n  name: z.string().describe(\"Name identifier for this grader\"),\n  type: z.string().describe(\"Type of grader, must be 'python'\"),\n  source: z.string().describe(\"Python source code to execute for grading\"),\n  image_tag: z.string().nullable().optional(),\n}).passthrough().describe(\"Python grader configuration.\"), z.object({\n  name: z.string().describe(\"Name identifier for this grader\"),\n  type: z.string().describe(\"Type of grader, must be 'score_model'\"),\n  input: z.array(z.record(z.string(), z.unknown())).describe(\"Input data for the scoring model\"),\n  model: z.string().describe(\"The model to use for scoring\"),\n  range: z.array(z.union([z.number().int(), z.number()])).nullable().optional(),\n  sampling_params: z.record(z.string(), z.unknown()).nullable().optional(),\n}).passthrough().describe(\"Score model grader configuration.\"), z.object({\n  name: z.string().describe(\"Name identifier for this grader\"),\n  type: z.string().describe(\"Type of grader, must be 'multi'\"),\n  graders: z.array(z.record(z.string(), z.unknown())).describe(\"List of grader configurations to combine\"),\n  calculate_output: z.string().describe(\"Formula to calculate the final output from sub-grader results\"),\n}).passthrough().describe(\"Multi grader configuration that combines multiple graders.\"), z.record(z.string(), z.unknown())]),\n  model_sample: z.string().describe(\"The model sample output to be evaluated. This value will be used to populate the 'sample' namespace. The 'output_json' variable will be populated if the model sample is a valid JSON string\"),\n}).describe(\"Request model for running a grader to evaluate model performance.\");\nconst OpenaiRunGrader_GraderErrorsSchema = z.object({\n  other_error: z.boolean().describe(\"Whether another type of error occurred\").nullable(),\n  sample_parse_error: z.boolean().describe(\"Whether a sample parsing error occurred\").nullable(),\n  formula_parse_error: z.boolean().describe(\"Whether a formula parsing error occurred\").nullable(),\n  invalid_variable_error: z.boolean().describe(\"Whether an invalid variable error occurred\").nullable(),\n  model_grader_parse_error: z.boolean().describe(\"Whether a model grader parsing error occurred\").nullable(),\n  model_grader_server_error: z.boolean().describe(\"Whether a model grader server error occurred\").nullable(),\n  unresponsive_reward_error: z.boolean().describe(\"Whether an unresponsive reward error occurred\").nullable(),\n  model_grader_refusal_error: z.boolean().describe(\"Whether a model grader refusal error occurred\").nullable(),\n  python_grader_server_error: z.boolean().describe(\"Whether a Python grader server error occurred\").nullable(),\n  sample_parse_error_details: z.record(z.string(), z.unknown()).describe(\"Details about the sample parsing error\").nullable().optional(),\n  python_grader_runtime_error: z.boolean().describe(\"Whether a Python grader runtime error occurred\").nullable(),\n  truncated_observation_error: z.boolean().describe(\"Whether an observation truncation error occurred\").nullable(),\n  endpoint_grader_server_error: z.boolean().describe(\"Whether an endpoint grader server error occurred\").nullable(),\n  endpoint_grader_internal_error: z.boolean().describe(\"Whether an endpoint grader internal error occurred\").nullable(),\n  invalid_variable_error_details: z.record(z.string(), z.unknown()).describe(\"Details about the invalid variable error\").nullable().optional(),\n  python_grader_server_error_type: z.record(z.string(), z.unknown()).describe(\"Details about the Python grader server error type\").nullable().optional(),\n  model_grader_parse_error_details: z.record(z.string(), z.unknown()).describe(\"Details about the model grader parsing error\").nullable().optional(),\n  model_grader_server_error_details: z.record(z.string(), z.unknown()).describe(\"Details about the model grader server error\").nullable().optional(),\n  endpoint_grader_safety_check_error: z.boolean().describe(\"Whether an endpoint grader safety check error occurred\").nullable(),\n  model_grader_refusal_error_details: z.record(z.string(), z.unknown()).describe(\"Details about the model grader refusal error\").nullable().optional(),\n  python_grader_runtime_error_details: z.record(z.string(), z.unknown()).describe(\"Details about the Python grader runtime error\").nullable().optional(),\n  endpoint_grader_server_error_details: z.record(z.string(), z.unknown()).describe(\"Details about the endpoint grader server error\").nullable().optional(),\n  endpoint_grader_internal_error_details: z.record(z.string(), z.unknown()).describe(\"Details about the endpoint grader internal error\").nullable().optional(),\n  model_grader_exceeded_max_tokens_error: z.boolean().describe(\"Whether a model grader exceeded max tokens error occurred\").nullable(),\n}).passthrough().describe(\"Detailed error information for grader execution.\");\nconst OpenaiRunGrader_GraderMetadataSchema = z.object({\n  name: z.string().describe(\"The name of the grader that was executed\").nullable(),\n  type: z.string().describe(\"The type of grader that was executed\").nullable(),\n  errors: OpenaiRunGrader_GraderErrorsSchema.nullable(),\n  scores: z.record(z.string(), z.unknown()).describe(\"Individual scores computed by the grader\"),\n  metadata: z.record(z.string(), z.unknown()).describe(\"Additional metadata from the grader execution\").nullable().optional(),\n  token_usage: z.number().int().describe(\"Number of tokens used during grader execution, if applicable\").nullable().optional(),\n  execution_time: z.number().describe(\"Time taken to execute the grader in seconds\").nullable(),\n  sampled_model_name: z.string().describe(\"Name of the model that was sampled, if applicable\").nullable().optional(),\n}).passthrough().describe(\"Metadata about the grader execution.\");\nexport const OpenaiRunGraderOutput = z.object({\n  reward: z.number().describe(\"The final reward score computed by the grader. Higher values indicate better performance\").nullable(),\n  metadata: OpenaiRunGrader_GraderMetadataSchema.nullable(),\n  sub_rewards: z.record(z.string(), z.unknown()).describe(\"Individual sub-rewards from component graders, if using a multi-grader\"),\n  model_grader_token_usage_per_model: z.record(z.string(), z.unknown()).describe(\"Token usage breakdown per model used in grading\"),\n}).passthrough().describe(\"Response schema for running a grader.\");\n\nexport const openaiRunGrader = action(\"OPENAI_RUN_GRADER\", {\n  slug: \"openai-run-grader\",\n  name: \"Run grader\",\n  description: \"Tool to run a grader to evaluate model performance on a given sample. Use when you need to evaluate model outputs against expected results using various grading strategies like string matching, text similarity, custom Python code, or model-based scoring.\",\n  input: OpenaiRunGraderInput,\n  output: OpenaiRunGraderOutput,\n});\n"],"mappings":";;;AAIA,MAAa,uBAAuBA,IAAAA,EAAE,OAAO;CAC3C,MAAMA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,6HAA6H,CAAC,CAAC,SAAS;CACzL,QAAQA,IAAAA,EAAE,MAAM;EAACA,IAAAA,EAAE,OAAO;GAC1B,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,iCAAiC;GAC3D,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,wCAAwC;GAClE,OAAOA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,oFAAoF;GAC/G,WAAWA,IAAAA,EAAE,KAAK;IAAC;IAAM;IAAM;IAAQ;GAAO,CAAC,CAAC,CAAC,SAAS,mIAAmI;GAC7L,WAAWA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,wCAAwC;EACzE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,SAAS,oCAAoC;EAAGA,IAAAA,EAAE,OAAO;GACxE,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,iCAAiC;GAC3D,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,2CAA2C;GACrE,OAAOA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,6BAA6B;GACxD,WAAWA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,uCAAuC;GACtE,mBAAmBA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,kDAAkD;EAC3F,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,SAAS,uCAAuC;EAAGA,IAAAA,EAAE,OAAO;GAC3E,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,iCAAiC;GAC3D,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,kCAAkC;GAC5D,QAAQA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,2CAA2C;GACvE,WAAWA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;EAC5C,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,SAAS,8BAA8B;EAAGA,IAAAA,EAAE,OAAO;GAClE,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,iCAAiC;GAC3D,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,uCAAuC;GACjE,OAAOA,IAAAA,EAAE,MAAMA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,kCAAkC;GAC7F,OAAOA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,8BAA8B;GACzD,OAAOA,IAAAA,EAAE,MAAMA,IAAAA,EAAE,MAAM,CAACA,IAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,GAAGA,IAAAA,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;GAC5E,iBAAiBA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;EACzE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,SAAS,mCAAmC;EAAGA,IAAAA,EAAE,OAAO;GACvE,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,iCAAiC;GAC3D,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,iCAAiC;GAC3D,SAASA,IAAAA,EAAE,MAAMA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,SAAS,0CAA0C;GACvG,kBAAkBA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,+DAA+D;EACvG,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,SAAS,4DAA4D;EAAGA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC;CAAC,CAAC;CACzH,cAAcA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,8LAA8L;AAClO,CAAC,CAAC,CAAC,SAAS,mEAAmE;AAC/E,MAAM,qCAAqCA,IAAAA,EAAE,OAAO;CAClD,aAAaA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,wCAAwC,CAAC,CAAC,SAAS;CACrF,oBAAoBA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,yCAAyC,CAAC,CAAC,SAAS;CAC7F,qBAAqBA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,0CAA0C,CAAC,CAAC,SAAS;CAC/F,wBAAwBA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,4CAA4C,CAAC,CAAC,SAAS;CACpG,0BAA0BA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,+CAA+C,CAAC,CAAC,SAAS;CACzG,2BAA2BA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,8CAA8C,CAAC,CAAC,SAAS;CACzG,2BAA2BA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,+CAA+C,CAAC,CAAC,SAAS;CAC1G,4BAA4BA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,+CAA+C,CAAC,CAAC,SAAS;CAC3G,4BAA4BA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,+CAA+C,CAAC,CAAC,SAAS;CAC3G,4BAA4BA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,wCAAwC,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CACrI,6BAA6BA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,gDAAgD,CAAC,CAAC,SAAS;CAC7G,6BAA6BA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,kDAAkD,CAAC,CAAC,SAAS;CAC/G,8BAA8BA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,kDAAkD,CAAC,CAAC,SAAS;CAChH,gCAAgCA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,oDAAoD,CAAC,CAAC,SAAS;CACpH,gCAAgCA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,0CAA0C,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CAC3I,iCAAiCA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,mDAAmD,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CACrJ,kCAAkCA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,8CAA8C,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CACjJ,mCAAmCA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,6CAA6C,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CACjJ,oCAAoCA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,wDAAwD,CAAC,CAAC,SAAS;CAC5H,oCAAoCA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,8CAA8C,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CACnJ,qCAAqCA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,+CAA+C,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CACrJ,sCAAsCA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,gDAAgD,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CACvJ,wCAAwCA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,kDAAkD,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CAC3J,wCAAwCA,IAAAA,EAAE,QAAQ,CAAC,CAAC,SAAS,2DAA2D,CAAC,CAAC,SAAS;AACrI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,SAAS,kDAAkD;AAC5E,MAAM,uCAAuCA,IAAAA,EAAE,OAAO;CACpD,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,0CAA0C,CAAC,CAAC,SAAS;CAC/E,MAAMA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,sCAAsC,CAAC,CAAC,SAAS;CAC3E,QAAQ,mCAAmC,SAAS;CACpD,QAAQA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,0CAA0C;CAC7F,UAAUA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,+CAA+C,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CAC1H,aAAaA,IAAAA,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,8DAA8D,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;CAC3H,gBAAgBA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,6CAA6C,CAAC,CAAC,SAAS;CAC5F,oBAAoBA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,mDAAmD,CAAC,CAAC,SAAS,CAAC,CAAC,SAAS;AACnH,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,SAAS,sCAAsC;AAChE,MAAa,wBAAwBA,IAAAA,EAAE,OAAO;CAC5C,QAAQA,IAAAA,EAAE,OAAO,CAAC,CAAC,SAAS,0FAA0F,CAAC,CAAC,SAAS;CACjI,UAAU,qCAAqC,SAAS;CACxD,aAAaA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,wEAAwE;CAChI,oCAAoCA,IAAAA,EAAE,OAAOA,IAAAA,EAAE,OAAO,GAAGA,IAAAA,EAAE,QAAQ,CAAC,CAAC,CAAC,SAAS,iDAAiD;AAClI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,SAAS,uCAAuC;AAEjE,MAAa,kBAAkBC,eAAAA,OAAO,qBAAqB;CACzD,MAAM;CACN,MAAM;CACN,aAAa;CACb,OAAO;CACP,QAAQ;AACV,CAAC"}

package/dist/actions/run-grader.d.cts

@@ -1,9 +1,124 @@
 import { z } from "zod";
 
 //#region src/actions/run-grader.d.ts
-declare const OpenaiRunGraderInput: z.ZodTypeAny;
-declare const OpenaiRunGraderOutput: z.ZodTypeAny;
-declare const openaiRunGrader: import("@keystrokehq/action").WorkflowActionDefinition<unknown, unknown, import("@keystrokehq/shared").ResolvedCredentials<readonly [import("@keystrokehq/shared").Credential]>, readonly [import("@keystrokehq/shared").Credential]>;
+declare const OpenaiRunGraderInput: z.ZodObject<{
+  item: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+  grader: z.ZodUnion<readonly [z.ZodObject<{
+    name: z.ZodString;
+    type: z.ZodString;
+    input: z.ZodString;
+    operation: z.ZodEnum<{
+      eq: "eq";
+      ne: "ne";
+      like: "like";
+      ilike: "ilike";
+    }>;
+    reference: z.ZodString;
+  }, z.core.$loose>, z.ZodObject<{
+    name: z.ZodString;
+    type: z.ZodString;
+    input: z.ZodString;
+    reference: z.ZodString;
+    evaluation_metric: z.ZodString;
+  }, z.core.$loose>, z.ZodObject<{
+    name: z.ZodString;
+    type: z.ZodString;
+    source: z.ZodString;
+    image_tag: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+  }, z.core.$loose>, z.ZodObject<{
+    name: z.ZodString;
+    type: z.ZodString;
+    input: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    model: z.ZodString;
+    range: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodUnion<readonly [z.ZodNumber, z.ZodNumber]>>>>;
+    sampling_params: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+  }, z.core.$loose>, z.ZodObject<{
+    name: z.ZodString;
+    type: z.ZodString;
+    graders: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    calculate_output: z.ZodString;
+  }, z.core.$loose>, z.ZodRecord<z.ZodString, z.ZodUnknown>]>;
+  model_sample: z.ZodString;
+}, z.core.$strip>;
+declare const OpenaiRunGraderOutput: z.ZodObject<{
+  reward: z.ZodNullable<z.ZodNumber>;
+  metadata: z.ZodNullable<z.ZodObject<{
+    name: z.ZodNullable<z.ZodString>;
+    type: z.ZodNullable<z.ZodString>;
+    errors: z.ZodNullable<z.ZodObject<{
+      other_error: z.ZodNullable<z.ZodBoolean>;
+      sample_parse_error: z.ZodNullable<z.ZodBoolean>;
+      formula_parse_error: z.ZodNullable<z.ZodBoolean>;
+      invalid_variable_error: z.ZodNullable<z.ZodBoolean>;
+      model_grader_parse_error: z.ZodNullable<z.ZodBoolean>;
+      model_grader_server_error: z.ZodNullable<z.ZodBoolean>;
+      unresponsive_reward_error: z.ZodNullable<z.ZodBoolean>;
+      model_grader_refusal_error: z.ZodNullable<z.ZodBoolean>;
+      python_grader_server_error: z.ZodNullable<z.ZodBoolean>;
+      sample_parse_error_details: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+      python_grader_runtime_error: z.ZodNullable<z.ZodBoolean>;
+      truncated_observation_error: z.ZodNullable<z.ZodBoolean>;
+      endpoint_grader_server_error: z.ZodNullable<z.ZodBoolean>;
+      endpoint_grader_internal_error: z.ZodNullable<z.ZodBoolean>;
+      invalid_variable_error_details: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+      python_grader_server_error_type: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+      model_grader_parse_error_details: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+      model_grader_server_error_details: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+      endpoint_grader_safety_check_error: z.ZodNullable<z.ZodBoolean>;
+      model_grader_refusal_error_details: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+      python_grader_runtime_error_details: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+      endpoint_grader_server_error_details: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+      endpoint_grader_internal_error_details: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+      model_grader_exceeded_max_tokens_error: z.ZodNullable<z.ZodBoolean>;
+    }, z.core.$loose>>;
+    scores: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    metadata: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+    token_usage: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+    execution_time: z.ZodNullable<z.ZodNumber>;
+    sampled_model_name: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+  }, z.core.$loose>>;
+  sub_rewards: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+  model_grader_token_usage_per_model: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+}, z.core.$loose>;
+declare const openaiRunGrader: import("@keystrokehq/action").WorkflowActionDefinition<{
+  grader: Record<string, unknown> | {
+    [x: string]: unknown;
+    name: string;
+    type: string;
+    input: string;
+    operation: "eq" | "ne" | "like" | "ilike";
+    reference: string;
+  } | {
+    [x: string]: unknown;
+    name: string;
+    type: string;
+    input: string;
+    reference: string;
+    evaluation_metric: string;
+  } | {
+    [x: string]: unknown;
+    name: string;
+    type: string;
+    source: string;
+    image_tag?: string | null | undefined;
+  } | {
+    [x: string]: unknown;
+    name: string;
+    type: string;
+    input: Record<string, unknown>[];
+    model: string;
+    range?: number[] | null | undefined;
+    sampling_params?: Record<string, unknown> | null | undefined;
+  } | {
+    [x: string]: unknown;
+    name: string;
+    type: string;
+    graders: Record<string, unknown>[];
+    calculate_output: string;
+  };
+  model_sample: string;
+  item?: Record<string, unknown> | undefined;
+}, unknown, import("@keystrokehq/shared").ResolvedCredentials<readonly [import("@keystrokehq/shared").Credential]>, readonly [import("@keystrokehq/shared").Credential]>;
 //#endregion
 export { openaiRunGrader };
 //# sourceMappingURL=run-grader.d.cts.map

package/dist/actions/run-grader.d.cts.map

@@ -1 +1 @@
-{"version":3,"file":"run-grader.d.cts","names":[],"sources":["../../src/actions/run-grader.ts"],"mappings":";;;cAIa,oBAAA,EAAsB,CAAA,CAAE,UAiC2C;AAAA,cAqCnE,qBAAA,EAAuB,CAAA,CAAE,UAKc;AAAA,cAEvC,eAAA,gCAAe,wBAAA,iDAAA,mBAAA,yCAAA,UAAA,4CAAA,UAAA"}
+{"version":3,"file":"run-grader.d.cts","names":[],"sources":["../../src/actions/run-grader.ts"],"mappings":";;;cAIa,oBAAA,EAAoB,CAAA,CAAA,SAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAsEpB,qBAAA,EAAqB,CAAA,CAAA,SAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAOrB,eAAA,gCAAe,wBAAA"}

package/dist/actions/run-grader.d.mts

@@ -1,9 +1,124 @@
 import { z } from "zod";
 
 //#region src/actions/run-grader.d.ts
-declare const OpenaiRunGraderInput: z.ZodTypeAny;
-declare const OpenaiRunGraderOutput: z.ZodTypeAny;
-declare const openaiRunGrader: import("@keystrokehq/action").WorkflowActionDefinition<unknown, unknown, import("@keystrokehq/shared").ResolvedCredentials<readonly [import("@keystrokehq/shared").Credential]>, readonly [import("@keystrokehq/shared").Credential]>;
+declare const OpenaiRunGraderInput: z.ZodObject<{
+  item: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+  grader: z.ZodUnion<readonly [z.ZodObject<{
+    name: z.ZodString;
+    type: z.ZodString;
+    input: z.ZodString;
+    operation: z.ZodEnum<{
+      eq: "eq";
+      ne: "ne";
+      like: "like";
+      ilike: "ilike";
+    }>;
+    reference: z.ZodString;
+  }, z.core.$loose>, z.ZodObject<{
+    name: z.ZodString;
+    type: z.ZodString;
+    input: z.ZodString;
+    reference: z.ZodString;
+    evaluation_metric: z.ZodString;
+  }, z.core.$loose>, z.ZodObject<{
+    name: z.ZodString;
+    type: z.ZodString;
+    source: z.ZodString;
+    image_tag: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+  }, z.core.$loose>, z.ZodObject<{
+    name: z.ZodString;
+    type: z.ZodString;
+    input: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    model: z.ZodString;
+    range: z.ZodOptional<z.ZodNullable<z.ZodArray<z.ZodUnion<readonly [z.ZodNumber, z.ZodNumber]>>>>;
+    sampling_params: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+  }, z.core.$loose>, z.ZodObject<{
+    name: z.ZodString;
+    type: z.ZodString;
+    graders: z.ZodArray<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    calculate_output: z.ZodString;
+  }, z.core.$loose>, z.ZodRecord<z.ZodString, z.ZodUnknown>]>;
+  model_sample: z.ZodString;
+}, z.core.$strip>;
+declare const OpenaiRunGraderOutput: z.ZodObject<{
+  reward: z.ZodNullable<z.ZodNumber>;
+  metadata: z.ZodNullable<z.ZodObject<{
+    name: z.ZodNullable<z.ZodString>;
+    type: z.ZodNullable<z.ZodString>;
+    errors: z.ZodNullable<z.ZodObject<{
+      other_error: z.ZodNullable<z.ZodBoolean>;
+      sample_parse_error: z.ZodNullable<z.ZodBoolean>;
+      formula_parse_error: z.ZodNullable<z.ZodBoolean>;
+      invalid_variable_error: z.ZodNullable<z.ZodBoolean>;
+      model_grader_parse_error: z.ZodNullable<z.ZodBoolean>;
+      model_grader_server_error: z.ZodNullable<z.ZodBoolean>;
+      unresponsive_reward_error: z.ZodNullable<z.ZodBoolean>;
+      model_grader_refusal_error: z.ZodNullable<z.ZodBoolean>;
+      python_grader_server_error: z.ZodNullable<z.ZodBoolean>;
+      sample_parse_error_details: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+      python_grader_runtime_error: z.ZodNullable<z.ZodBoolean>;
+      truncated_observation_error: z.ZodNullable<z.ZodBoolean>;
+      endpoint_grader_server_error: z.ZodNullable<z.ZodBoolean>;
+      endpoint_grader_internal_error: z.ZodNullable<z.ZodBoolean>;
+      invalid_variable_error_details: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+      python_grader_server_error_type: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+      model_grader_parse_error_details: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+      model_grader_server_error_details: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+      endpoint_grader_safety_check_error: z.ZodNullable<z.ZodBoolean>;
+      model_grader_refusal_error_details: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+      python_grader_runtime_error_details: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+      endpoint_grader_server_error_details: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+      endpoint_grader_internal_error_details: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+      model_grader_exceeded_max_tokens_error: z.ZodNullable<z.ZodBoolean>;
+    }, z.core.$loose>>;
+    scores: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    metadata: z.ZodOptional<z.ZodNullable<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+    token_usage: z.ZodOptional<z.ZodNullable<z.ZodNumber>>;
+    execution_time: z.ZodNullable<z.ZodNumber>;
+    sampled_model_name: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+  }, z.core.$loose>>;
+  sub_rewards: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+  model_grader_token_usage_per_model: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+}, z.core.$loose>;
+declare const openaiRunGrader: import("@keystrokehq/action").WorkflowActionDefinition<{
+  grader: Record<string, unknown> | {
+    [x: string]: unknown;
+    name: string;
+    type: string;
+    input: string;
+    operation: "eq" | "ne" | "like" | "ilike";
+    reference: string;
+  } | {
+    [x: string]: unknown;
+    name: string;
+    type: string;
+    input: string;
+    reference: string;
+    evaluation_metric: string;
+  } | {
+    [x: string]: unknown;
+    name: string;
+    type: string;
+    source: string;
+    image_tag?: string | null | undefined;
+  } | {
+    [x: string]: unknown;
+    name: string;
+    type: string;
+    input: Record<string, unknown>[];
+    model: string;
+    range?: number[] | null | undefined;
+    sampling_params?: Record<string, unknown> | null | undefined;
+  } | {
+    [x: string]: unknown;
+    name: string;
+    type: string;
+    graders: Record<string, unknown>[];
+    calculate_output: string;
+  };
+  model_sample: string;
+  item?: Record<string, unknown> | undefined;
+}, unknown, import("@keystrokehq/shared").ResolvedCredentials<readonly [import("@keystrokehq/shared").Credential]>, readonly [import("@keystrokehq/shared").Credential]>;
 //#endregion
 export { openaiRunGrader };
 //# sourceMappingURL=run-grader.d.mts.map

package/dist/actions/run-grader.d.mts.map

@@ -1 +1 @@
-{"version":3,"file":"run-grader.d.mts","names":[],"sources":["../../src/actions/run-grader.ts"],"mappings":";;;cAIa,oBAAA,EAAsB,CAAA,CAAE,UAiC2C;AAAA,cAqCnE,qBAAA,EAAuB,CAAA,CAAE,UAKc;AAAA,cAEvC,eAAA,gCAAe,wBAAA,iDAAA,mBAAA,yCAAA,UAAA,4CAAA,UAAA"}
+{"version":3,"file":"run-grader.d.mts","names":[],"sources":["../../src/actions/run-grader.ts"],"mappings":";;;cAIa,oBAAA,EAAoB,CAAA,CAAA,SAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAsEpB,qBAAA,EAAqB,CAAA,CAAA,SAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAOrB,eAAA,gCAAe,wBAAA"}

package/dist/actions/run-grader.mjs

@@ -15,20 +15,20 @@ const OpenaiRunGraderInput = z.object({
 				"ilike"
 			]).describe("The comparison operation to perform: 'eq' for equals, 'ne' for not equals, 'like' for SQL LIKE, 'ilike' for case-insensitive LIKE"),
 			reference: z.string().describe("The reference value to compare against")
-		}).describe("String check grader configuration."),
+		}).passthrough().describe("String check grader configuration."),
 		z.object({
 			name: z.string().describe("Name identifier for this grader"),
 			type: z.string().describe("Type of grader, must be 'text_similarity'"),
 			input: z.string().describe("The input field to evaluate"),
 			reference: z.string().describe("The reference text to compare against"),
 			evaluation_metric: z.string().describe("The metric to use for evaluating text similarity")
-		}).describe("Text similarity grader configuration."),
+		}).passthrough().describe("Text similarity grader configuration."),
 		z.object({
 			name: z.string().describe("Name identifier for this grader"),
 			type: z.string().describe("Type of grader, must be 'python'"),
 			source: z.string().describe("Python source code to execute for grading"),
 			image_tag: z.string().nullable().optional()
-		}).describe("Python grader configuration."),
+		}).passthrough().describe("Python grader configuration."),
 		z.object({
 			name: z.string().describe("Name identifier for this grader"),
 			type: z.string().describe("Type of grader, must be 'score_model'"),
@@ -36,64 +36,64 @@ const OpenaiRunGraderInput = z.object({
 			model: z.string().describe("The model to use for scoring"),
 			range: z.array(z.union([z.number().int(), z.number()])).nullable().optional(),
 			sampling_params: z.record(z.string(), z.unknown()).nullable().optional()
-		}).describe("Score model grader configuration."),
+		}).passthrough().describe("Score model grader configuration."),
 		z.object({
 			name: z.string().describe("Name identifier for this grader"),
 			type: z.string().describe("Type of grader, must be 'multi'"),
 			graders: z.array(z.record(z.string(), z.unknown())).describe("List of grader configurations to combine"),
 			calculate_output: z.string().describe("Formula to calculate the final output from sub-grader results")
-		}).describe("Multi grader configuration that combines multiple graders."),
+		}).passthrough().describe("Multi grader configuration that combines multiple graders."),
 		z.record(z.string(), z.unknown())
 	]),
 	model_sample: z.string().describe("The model sample output to be evaluated. This value will be used to populate the 'sample' namespace. The 'output_json' variable will be populated if the model sample is a valid JSON string")
 }).describe("Request model for running a grader to evaluate model performance.");
 const OpenaiRunGrader_GraderErrorsSchema = z.object({
-	other_error: z.boolean().describe("Whether another type of error occurred"),
-	sample_parse_error: z.boolean().describe("Whether a sample parsing error occurred"),
-	formula_parse_error: z.boolean().describe("Whether a formula parsing error occurred"),
-	invalid_variable_error: z.boolean().describe("Whether an invalid variable error occurred"),
-	model_grader_parse_error: z.boolean().describe("Whether a model grader parsing error occurred"),
-	model_grader_server_error: z.boolean().describe("Whether a model grader server error occurred"),
-	unresponsive_reward_error: z.boolean().describe("Whether an unresponsive reward error occurred"),
-	model_grader_refusal_error: z.boolean().describe("Whether a model grader refusal error occurred"),
-	python_grader_server_error: z.boolean().describe("Whether a Python grader server error occurred"),
+	other_error: z.boolean().describe("Whether another type of error occurred").nullable(),
+	sample_parse_error: z.boolean().describe("Whether a sample parsing error occurred").nullable(),
+	formula_parse_error: z.boolean().describe("Whether a formula parsing error occurred").nullable(),
+	invalid_variable_error: z.boolean().describe("Whether an invalid variable error occurred").nullable(),
+	model_grader_parse_error: z.boolean().describe("Whether a model grader parsing error occurred").nullable(),
+	model_grader_server_error: z.boolean().describe("Whether a model grader server error occurred").nullable(),
+	unresponsive_reward_error: z.boolean().describe("Whether an unresponsive reward error occurred").nullable(),
+	model_grader_refusal_error: z.boolean().describe("Whether a model grader refusal error occurred").nullable(),
+	python_grader_server_error: z.boolean().describe("Whether a Python grader server error occurred").nullable(),
 	sample_parse_error_details: z.record(z.string(), z.unknown()).describe("Details about the sample parsing error").nullable().optional(),
-	python_grader_runtime_error: z.boolean().describe("Whether a Python grader runtime error occurred"),
-	truncated_observation_error: z.boolean().describe("Whether an observation truncation error occurred"),
-	endpoint_grader_server_error: z.boolean().describe("Whether an endpoint grader server error occurred"),
-	endpoint_grader_internal_error: z.boolean().describe("Whether an endpoint grader internal error occurred"),
+	python_grader_runtime_error: z.boolean().describe("Whether a Python grader runtime error occurred").nullable(),
+	truncated_observation_error: z.boolean().describe("Whether an observation truncation error occurred").nullable(),
+	endpoint_grader_server_error: z.boolean().describe("Whether an endpoint grader server error occurred").nullable(),
+	endpoint_grader_internal_error: z.boolean().describe("Whether an endpoint grader internal error occurred").nullable(),
 	invalid_variable_error_details: z.record(z.string(), z.unknown()).describe("Details about the invalid variable error").nullable().optional(),
 	python_grader_server_error_type: z.record(z.string(), z.unknown()).describe("Details about the Python grader server error type").nullable().optional(),
 	model_grader_parse_error_details: z.record(z.string(), z.unknown()).describe("Details about the model grader parsing error").nullable().optional(),
 	model_grader_server_error_details: z.record(z.string(), z.unknown()).describe("Details about the model grader server error").nullable().optional(),
-	endpoint_grader_safety_check_error: z.boolean().describe("Whether an endpoint grader safety check error occurred"),
+	endpoint_grader_safety_check_error: z.boolean().describe("Whether an endpoint grader safety check error occurred").nullable(),
 	model_grader_refusal_error_details: z.record(z.string(), z.unknown()).describe("Details about the model grader refusal error").nullable().optional(),
 	python_grader_runtime_error_details: z.record(z.string(), z.unknown()).describe("Details about the Python grader runtime error").nullable().optional(),
 	endpoint_grader_server_error_details: z.record(z.string(), z.unknown()).describe("Details about the endpoint grader server error").nullable().optional(),
 	endpoint_grader_internal_error_details: z.record(z.string(), z.unknown()).describe("Details about the endpoint grader internal error").nullable().optional(),
-	model_grader_exceeded_max_tokens_error: z.boolean().describe("Whether a model grader exceeded max tokens error occurred")
-}).describe("Detailed error information for grader execution.");
+	model_grader_exceeded_max_tokens_error: z.boolean().describe("Whether a model grader exceeded max tokens error occurred").nullable()
+}).passthrough().describe("Detailed error information for grader execution.");
 const OpenaiRunGrader_GraderMetadataSchema = z.object({
-	name: z.string().describe("The name of the grader that was executed"),
-	type: z.string().describe("The type of grader that was executed"),
+	name: z.string().describe("The name of the grader that was executed").nullable(),
+	type: z.string().describe("The type of grader that was executed").nullable(),
 	errors: OpenaiRunGrader_GraderErrorsSchema.nullable(),
 	scores: z.record(z.string(), z.unknown()).describe("Individual scores computed by the grader"),
 	metadata: z.record(z.string(), z.unknown()).describe("Additional metadata from the grader execution").nullable().optional(),
 	token_usage: z.number().int().describe("Number of tokens used during grader execution, if applicable").nullable().optional(),
-	execution_time: z.number().describe("Time taken to execute the grader in seconds"),
+	execution_time: z.number().describe("Time taken to execute the grader in seconds").nullable(),
 	sampled_model_name: z.string().describe("Name of the model that was sampled, if applicable").nullable().optional()
-}).describe("Metadata about the grader execution.");
+}).passthrough().describe("Metadata about the grader execution.");
 const openaiRunGrader = action("OPENAI_RUN_GRADER", {
 	slug: "openai-run-grader",
 	name: "Run grader",
 	description: "Tool to run a grader to evaluate model performance on a given sample. Use when you need to evaluate model outputs against expected results using various grading strategies like string matching, text similarity, custom Python code, or model-based scoring.",
 	input: OpenaiRunGraderInput,
 	output: z.object({
-		reward: z.number().describe("The final reward score computed by the grader. Higher values indicate better performance"),
+		reward: z.number().describe("The final reward score computed by the grader. Higher values indicate better performance").nullable(),
 		metadata: OpenaiRunGrader_GraderMetadataSchema.nullable(),
 		sub_rewards: z.record(z.string(), z.unknown()).describe("Individual sub-rewards from component graders, if using a multi-grader"),
 		model_grader_token_usage_per_model: z.record(z.string(), z.unknown()).describe("Token usage breakdown per model used in grading")
-	}).describe("Response schema for running a grader.")
+	}).passthrough().describe("Response schema for running a grader.")
 });
 //#endregion
 export { openaiRunGrader };