orchestrated 0.1.11 → 0.1.13

package/index.d.ts

@@ -969,16 +969,60 @@ declare interface ScorerMetadata {
 export declare type ScorerName = keyof typeof SCORERS;
 
 /**
- * Scorer reference definition (from definitions.json)
+ * Scorer reference or full definition
+ * Can be either a reference (internal or custom_scorer by slug/fingerprint)
+ * or a complete definition (prompt or custom_scorer with all fields)
  */
-declare type ScorerReferenceDefinition = {
-    type: "internal";
-    name: string;
-} | {
-    type: "custom_scorer";
-    slug: string;
-    fingerprint?: string;
-};
+declare type ScorerReferenceDefinition = ScorerReferenceOrDefinition;
+
+declare type ScorerReferenceOrDefinition = z.infer<typeof ScorerReferenceOrDefinitionSchema>;
+
+/**
+ * Union schema for scorer references or full definitions
+ */
+declare const ScorerReferenceOrDefinitionSchema: z.ZodUnion<readonly [z.ZodDiscriminatedUnion<[z.ZodObject<{
+    type: z.ZodLiteral<"internal">;
+    name: z.ZodString;
+}, z.core.$strip>, z.ZodObject<{
+    type: z.ZodLiteral<"custom_scorer">;
+    slug: z.ZodString;
+    fingerprint: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>], "type">, z.ZodDiscriminatedUnion<[z.ZodObject<{
+    name: z.ZodString;
+    slug: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    schema: z.ZodObject<{
+        type: z.ZodLiteral<"zod">;
+        definition: z.ZodString;
+    }, z.core.$strip>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+    fingerprint: z.ZodOptional<z.ZodString>;
+    type: z.ZodLiteral<"prompt">;
+    promptTemplate: z.ZodString;
+    choiceScores: z.ZodRecord<z.ZodString, z.ZodNumber>;
+    model: z.ZodOptional<z.ZodString>;
+    useCoT: z.ZodOptional<z.ZodBoolean>;
+    temperature: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>, z.ZodObject<{
+    name: z.ZodString;
+    slug: z.ZodOptional<z.ZodString>;
+    description: z.ZodOptional<z.ZodString>;
+    schema: z.ZodObject<{
+        type: z.ZodLiteral<"zod">;
+        definition: z.ZodString;
+    }, z.core.$strip>;
+    metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+    fingerprint: z.ZodOptional<z.ZodString>;
+    type: z.ZodLiteral<"custom_scorer">;
+    handler: z.ZodObject<{
+        type: z.ZodLiteral<"custom_function">;
+        reference: z.ZodString;
+        bundle: z.ZodOptional<z.ZodObject<{
+            file: z.ZodString;
+            fingerprint: z.ZodString;
+        }, z.core.$strip>>;
+    }, z.core.$strip>;
+}, z.core.$strip>], "type">]>;
 
 export declare class ScorerRegistry {
     /**
@@ -1081,14 +1125,49 @@ declare const SerializableEvaluationSchema: z.ZodObject<{
             fingerprint: z.ZodString;
         }, z.core.$strip>>;
     }, z.core.$strip>>;
-    scores: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
+    scores: z.ZodArray<z.ZodUnion<readonly [z.ZodDiscriminatedUnion<[z.ZodObject<{
         type: z.ZodLiteral<"internal">;
         name: z.ZodString;
     }, z.core.$strip>, z.ZodObject<{
         type: z.ZodLiteral<"custom_scorer">;
         slug: z.ZodString;
         fingerprint: z.ZodOptional<z.ZodString>;
-    }, z.core.$strip>], "type">>;
+    }, z.core.$strip>], "type">, z.ZodDiscriminatedUnion<[z.ZodObject<{
+        name: z.ZodString;
+        slug: z.ZodOptional<z.ZodString>;
+        description: z.ZodOptional<z.ZodString>;
+        schema: z.ZodObject<{
+            type: z.ZodLiteral<"zod">;
+            definition: z.ZodString;
+        }, z.core.$strip>;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+        fingerprint: z.ZodOptional<z.ZodString>;
+        type: z.ZodLiteral<"prompt">;
+        promptTemplate: z.ZodString;
+        choiceScores: z.ZodRecord<z.ZodString, z.ZodNumber>;
+        model: z.ZodOptional<z.ZodString>;
+        useCoT: z.ZodOptional<z.ZodBoolean>;
+        temperature: z.ZodOptional<z.ZodNumber>;
+    }, z.core.$strip>, z.ZodObject<{
+        name: z.ZodString;
+        slug: z.ZodOptional<z.ZodString>;
+        description: z.ZodOptional<z.ZodString>;
+        schema: z.ZodObject<{
+            type: z.ZodLiteral<"zod">;
+            definition: z.ZodString;
+        }, z.core.$strip>;
+        metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
+        fingerprint: z.ZodOptional<z.ZodString>;
+        type: z.ZodLiteral<"custom_scorer">;
+        handler: z.ZodObject<{
+            type: z.ZodLiteral<"custom_function">;
+            reference: z.ZodString;
+            bundle: z.ZodOptional<z.ZodObject<{
+                file: z.ZodString;
+                fingerprint: z.ZodString;
+            }, z.core.$strip>>;
+        }, z.core.$strip>;
+    }, z.core.$strip>], "type">]>>;
     metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;
     fingerprint: z.ZodOptional<z.ZodString>;
     options: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodAny>>;

package/index.js

@@ -88621,7 +88621,7 @@ function trackEvaluationPromise(promise3) {
   const registry2 = getRegistry2();
   registry2.pendingEvaluations.push(promise3);
 }
-var BundleReferenceSchema, SerializableSchemaSchema, HandlerReferenceSchema, TaskReferenceSchema, ScorerReferenceSchema, SerializableDataSourceSchema, SerializableScorerBaseSchema, SerializableScorerDefinitionSchema, SerializableCustomScorerSchema, SerializableScorerSchema, SerializableEvaluationSchema;
+var BundleReferenceSchema, SerializableSchemaSchema, HandlerReferenceSchema, TaskReferenceSchema, ScorerReferenceSchema, SerializableDataSourceSchema, SerializableScorerBaseSchema, SerializableScorerDefinitionSchema, SerializableCustomScorerSchema, SerializableScorerSchema, ScorerReferenceOrDefinitionSchema, SerializableEvaluationSchema;
 var init_types4 = __esm(() => {
   init_zod();
   BundleReferenceSchema = exports_external.object({
@@ -88698,12 +88698,16 @@ var init_types4 = __esm(() => {
     SerializableScorerDefinitionSchema,
     SerializableCustomScorerSchema
   ]);
+  ScorerReferenceOrDefinitionSchema = exports_external.union([
+    ScorerReferenceSchema,
+    SerializableScorerSchema
+  ]);
   SerializableEvaluationSchema = exports_external.object({
     name: exports_external.string(),
     slug: exports_external.string(),
     data: SerializableDataSourceSchema,
     task: TaskReferenceSchema.optional(),
-    scores: exports_external.array(ScorerReferenceSchema).min(1),
+    scores: exports_external.array(ScorerReferenceOrDefinitionSchema).min(1),
     metadata: exports_external.record(exports_external.string(), exports_external.any()).optional(),
     fingerprint: exports_external.string().optional(),
     options: exports_external.record(exports_external.string(), exports_external.any()).optional()
@@ -88807,15 +88811,53 @@ function createScorerWrapper(scorerDef, handlerFn) {
   });
   return scorerWrapper;
 }
-async function resolveScorer(scorerRef) {
+async function resolveScorer(scorerRefOrDef) {
   try {
-    ScorerReferenceSchema.parse(scorerRef);
+    ScorerReferenceOrDefinitionSchema.parse(scorerRefOrDef);
   } catch (error48) {
     if (error48 instanceof ZodError) {
-      throw new ValidationError("Invalid scorer reference", error48, "resolveScorer");
+      throw new ValidationError("Invalid scorer reference or definition", error48, "resolveScorer");
     }
     throw error48;
   }
+  const isFullDefinition = "name" in scorerRefOrDef && scorerRefOrDef.type !== "internal";
+  if (isFullDefinition) {
+    const scorerDef2 = scorerRefOrDef;
+    if (scorerDef2.type === "prompt") {
+      const schemaObj = JSON.parse(scorerDef2.schema.definition);
+      const zodSchema = zodFromJSONSchema(schemaObj);
+      return buildPromptScorer({
+        name: scorerDef2.name,
+        promptTemplate: scorerDef2.promptTemplate,
+        choiceScores: scorerDef2.choiceScores,
+        model: scorerDef2.model,
+        useCoT: scorerDef2.useCoT,
+        temperature: scorerDef2.temperature,
+        parameters: zodSchema
+      });
+    }
+    if (scorerDef2.type === "custom_scorer") {
+      const handlerName2 = scorerDef2.handler.reference;
+      const localHandler = getLocalHandler(handlerName2);
+      if (localHandler) {
+        console.log(`Found handler "${handlerName2}" in local registry`);
+        return createScorerWrapper(scorerDef2, localHandler);
+      }
+      if (scorerDef2.handler.bundle) {
+        console.log(`Downloading bundle for handler "${handlerName2}"`);
+        const bundlePath2 = await downloadBundle(scorerDef2.handler.bundle.file, scorerDef2.handler.bundle.fingerprint);
+        const handlers2 = await loadHandlers(bundlePath2);
+        const handlerFn2 = handlers2[handlerName2]?.handler;
+        if (!handlerFn2) {
+          throw new Error(`Handler function not found in bundle: ${handlerName2}
+Available handlers: ${Object.keys(handlers2).join(", ")}`);
+        }
+        return createScorerWrapper(scorerDef2, handlerFn2);
+      }
+      throw new Error(`Handler "${handlerName2}" not found in local registry and no bundle specified`);
+    }
+  }
+  const scorerRef = scorerRefOrDef;
   if (scorerRef.type === "internal") {
     throw new Error(`Internal scorer "${scorerRef.name}" should be resolved by the evaluator, not the resolver`);
   }
@@ -118434,7 +118476,7 @@ async function resolveEvaluatorDefinitions(name, evaluator) {
   }
   const resolvedScores = [];
   for (const scorer2 of evaluator.scores) {
-    if (typeof scorer2 === "object" && scorer2 !== null && "type" in scorer2 && (scorer2.type === "internal" || scorer2.type === "custom_scorer")) {
+    if (typeof scorer2 === "object" && scorer2 !== null && "type" in scorer2 && (scorer2.type === "internal" || scorer2.type === "custom_scorer" || scorer2.type === "prompt")) {
       const scorerDef = scorer2;
       if (scorerDef.type === "internal") {
         resolvedScores.push(scorerDef.name);
@@ -118781,4 +118823,4 @@ export {
   Behavioral
 };
 
-//# debugId=E27B15039D5A105264756E2164756E21
+//# debugId=E93BEB0C27F3EB7E64756E2164756E21