orchestrated 0.1.8 → 0.1.9

package/index.d.ts

@@ -72,8 +72,14 @@ declare class BatchClient {
     /**
      * Initialize the batch client by checking for existing batches
      * Must be called after constructor
+     *
+     * @param name - Evaluation name
+     * @param checksum - Dataset checksum
+     * @param dataSourceType - Type of data source ("static" or "dataset")
+     *   - "static": Filter by both name AND checksum (fixed dataset)
+     *   - "dataset": Filter by name only (dynamic dataset)
      */
-    initialize(name: string, checksum: string): Promise<void>;
+    initialize(name: string, checksum: string, dataSourceType?: "static" | "dataset"): Promise<void>;
     generateRequestId(body: BatchRequest["body"]): string;
     /**
      * Adds a request to the batch
@@ -933,6 +939,12 @@ declare interface ResolvedDataCtx {
      * - "synthetic.my-group" - synthetic data with custom grouping
      */
     sourceType: string;
+    /**
+     * Data source type - used for batch naming strategy
+     * - "static": use checksum in batch name (data is fixed)
+     * - "dataset": use only eval name (data changes over time)
+     */
+    dataSourceType?: "static" | "dataset";
     checksum: string;
     caseCount: number;
 }

package/index.js

@@ -21158,6 +21158,7 @@ function getState() {
 function resetState() {
   globalState = null;
   isInitialized = false;
+  globalThis.__ORCHESTRATED_LAZY_LOAD__ = undefined;
 }
 function isStateInitialized() {
   return isInitialized || globalThis.__ORCHESTRATED_SHARED_STATE__ !== undefined;
@@ -88555,6 +88556,48 @@ var init_scorer = __esm(() => {
   init_jsdist();
 });
 
+// src/serialization/types.ts
+function initRegistry() {
+  if (!globalThis.__ORCHESTRATED_REGISTRY__) {
+    globalThis.__ORCHESTRATED_REGISTRY__ = {
+      scorers: [],
+      handlers: {},
+      evaluations: [],
+      pendingEvaluations: []
+    };
+  }
+  return globalThis.__ORCHESTRATED_REGISTRY__;
+}
+function getRegistry2() {
+  return initRegistry();
+}
+function registerHandler(name, handler, usageRef) {
+  const registry2 = getRegistry2();
+  if (!registry2.handlers[name]) {
+    registry2.handlers[name] = {
+      handler,
+      source: handler.toString(),
+      usedIn: []
+    };
+  }
+  if (usageRef) {
+    registry2.handlers[name].usedIn.push(usageRef);
+  }
+  return name;
+}
+function registerScorer(scorer) {
+  const registry2 = getRegistry2();
+  registry2.scorers.push(scorer);
+}
+function registerEvaluation2(evaluation) {
+  const registry2 = getRegistry2();
+  registry2.evaluations.push(evaluation);
+}
+function trackEvaluationPromise(promise3) {
+  const registry2 = getRegistry2();
+  registry2.pendingEvaluations.push(promise3);
+}
+
 // src/serialization/resolver.ts
 var exports_resolver = {};
 __export(exports_resolver, {
@@ -88567,6 +88610,14 @@ __export(exports_resolver, {
 });
 import { existsSync as existsSync4, mkdirSync as mkdirSync3, writeFileSync as writeFileSync4 } from "node:fs";
 import { resolve } from "node:path";
+function getLocalScorer(slug) {
+  const registry2 = getRegistry2();
+  return registry2.scorers.find((s) => (s.slug || s.name.toLowerCase()) === slug) || null;
+}
+function getLocalHandler(name) {
+  const registry2 = getRegistry2();
+  return registry2.handlers[name]?.handler || null;
+}
 function getCacheBaseDir() {
   const isLambda = process.env.AWS_LAMBDA_FUNCTION_NAME !== undefined;
   if (isLambda) {
@@ -88647,6 +88698,33 @@ async function resolveScorer(scorerRef) {
   if (scorerRef.type === "internal") {
     throw new Error(`Internal scorer "${scorerRef.name}" should be resolved by the evaluator, not the resolver`);
   }
+  const localScorer = getLocalScorer(scorerRef.slug);
+  if (localScorer) {
+    console.log(`Found scorer "${scorerRef.slug}" in local registry`);
+    if (localScorer.type === "prompt") {
+      const schemaObj = JSON.parse(localScorer.schema.definition);
+      const zodSchema = zodFromJSONSchema(schemaObj);
+      return buildPromptScorer({
+        name: localScorer.name,
+        promptTemplate: localScorer.promptTemplate,
+        choiceScores: localScorer.choiceScores,
+        model: localScorer.model,
+        useCoT: localScorer.useCoT,
+        temperature: localScorer.temperature,
+        parameters: zodSchema
+      });
+    }
+    if (localScorer.type === "custom_scorer") {
+      const handlerName2 = localScorer.handler.reference;
+      const localHandler = getLocalHandler(handlerName2);
+      if (localHandler) {
+        console.log(`Found handler "${handlerName2}" in local registry`);
+        return createScorerWrapper(localScorer, localHandler);
+      }
+      console.log(`Handler "${handlerName2}" not found locally, falling back to S3`);
+    }
+  }
+  console.log(`Scorer "${scorerRef.slug}" not found locally, fetching from API`);
   const version2 = scorerRef.fingerprint || "LATEST";
   const scorerRecord = await getDefinition("scorer", scorerRef.slug, version2);
   const scorerDef = scorerRecord.definition;
@@ -88681,6 +88759,12 @@ Available handlers: ${Object.keys(handlers).join(", ")}`);
 }
 async function resolveTask(taskRef) {
   const handlerName = taskRef.reference;
+  const localHandler = getLocalHandler(handlerName);
+  if (localHandler) {
+    console.log(`Found task handler "${handlerName}" in local registry`);
+    return localHandler;
+  }
+  console.log(`Task handler "${handlerName}" not found locally, fetching from API`);
   const handlerVersion = taskRef.bundle?.fingerprint || "LATEST";
   const handlerRecord = await getDefinition("handler", handlerName, handlerVersion);
   const handlerDef = handlerRecord.definition;
@@ -116717,10 +116801,17 @@ class BatchClient {
       fs.mkdirSync(this.tempDir, { recursive: true });
     }
   }
-  async initialize(name, checksum) {
+  async initialize(name, checksum, dataSourceType) {
     this.metadata = { name, checksum };
     const batches = await this.listBatches(100);
-    const matchingBatches = batches.filter((batch) => batch.metadata?.name === this.metadata?.name);
+    const matchingBatches = batches.filter((batch) => {
+      const nameMatches = batch.metadata?.name === this.metadata?.name;
+      if (dataSourceType === "static") {
+        return nameMatches && batch.metadata?.checksum === this.metadata?.checksum;
+      } else {
+        return nameMatches;
+      }
+    });
     if (matchingBatches.length === 0) {
       return;
     }
@@ -116943,49 +117034,6 @@ init_data_source();
 init_resolver();
 init_schema_serializer();
 init_scorer();
-
-// src/serialization/types.ts
-function initRegistry() {
-  if (!globalThis.__ORCHESTRATED_REGISTRY__) {
-    globalThis.__ORCHESTRATED_REGISTRY__ = {
-      scorers: [],
-      handlers: {},
-      evaluations: [],
-      pendingEvaluations: []
-    };
-  }
-  return globalThis.__ORCHESTRATED_REGISTRY__;
-}
-function getRegistry2() {
-  return initRegistry();
-}
-function registerHandler(name, handler, usageRef) {
-  const registry2 = getRegistry2();
-  if (!registry2.handlers[name]) {
-    registry2.handlers[name] = {
-      handler,
-      source: handler.toString(),
-      usedIn: []
-    };
-  }
-  if (usageRef) {
-    registry2.handlers[name].usedIn.push(usageRef);
-  }
-  return name;
-}
-function registerScorer(scorer) {
-  const registry2 = getRegistry2();
-  registry2.scorers.push(scorer);
-}
-function registerEvaluation2(evaluation) {
-  const registry2 = getRegistry2();
-  registry2.evaluations.push(evaluation);
-}
-function trackEvaluationPromise(promise3) {
-  const registry2 = getRegistry2();
-  registry2.pendingEvaluations.push(promise3);
-}
-
 // src/serialization/serializer.ts
 async function serializeDataSource(data) {
   if (typeof data === "object" && data !== null && "type" in data && (data.type === "dataset" || data.type === "static")) {
@@ -117649,6 +117697,7 @@ async function resolveDataSet(dataset, datasetMetadata) {
       ctx: {
         isLiveData,
         sourceType,
+        dataSourceType: datasetMetadata?.type,
         checksum,
         caseCount: dataset.length
       }
@@ -117974,7 +118023,7 @@ async function runEval(name, evaluator, options) {
       [ATTR_DATASET_SOURCE_TYPE]: data.ctx.sourceType,
       [ATTR_EVAL_EXECUTION_METADATA_TEST_CASE_COUNT]: data.ctx.caseCount
     });
-    await batchClient.initialize(name, data.ctx.checksum);
+    await batchClient.initialize(name, data.ctx.checksum, data.ctx.dataSourceType);
     if (batchClient.hasPendingBatch) {
       return batchClient.getPending();
     }
@@ -118083,12 +118132,14 @@ async function resolveEvaluatorDefinitions(name, evaluator) {
     if (dataSourceDef.type === "dataset") {
       datasetMetadata = {
         name: dataSourceDef.config.name,
-        group_name: dataSourceDef.config.group_name
+        group_name: dataSourceDef.config.group_name,
+        type: "dataset"
       };
     } else if (dataSourceDef.type === "static") {
       datasetMetadata = {
         name: "static",
-        group_name: dataSourceDef.group_name
+        group_name: dataSourceDef.group_name,
+        type: "static"
       };
     }
     const dataSourceFn = await resolveDataSource(dataSourceDef);
@@ -118379,4 +118430,4 @@ export {
   Behavioral
 };
 
-//# debugId=1A3AD42D6FAE354F64756E2164756E21
+//# debugId=7CBAC3512DE449E364756E2164756E21