orchestrated 0.1.16 → 0.1.18

package/index.js

@@ -117068,7 +117068,9 @@ class S3FileManager {
       body: content
     });
     if (!uploadResponse.ok) {
-      throw new Error(`Failed to upload batch file to S3: ${uploadResponse.statusText}`);
+      const errorBody = await uploadResponse.text();
+      throw new Error(`Failed to upload batch file to S3 (${uploadResponse.status}): ${uploadResponse.statusText}
+${errorBody}`);
     }
     return `s3://${result.bucket}/${asset.s3Key}`;
   }
@@ -118172,11 +118174,11 @@ function getEvaluationOptions(name, options, state) {
   let batchClient;
   let evalClient;
   let openaiClient;
+  const batchProcessor = merged.batchProcessor ?? "OPENAI";
   if (execute === "batch") {
     if (merged.batchClient) {
       batchClient = merged.batchClient;
     } else {
-      const batchProcessor = merged.batchProcessor ?? "OPENAI";
       if (batchProcessor === "BEDROCK_OPENAI") {
         if (!state.bedrockRegion || !state.bedrockApiKey || !state.bedrockServiceRoleArn || !state.bedrockModelId || !state.bedrockS3Bucket) {
           throw new Error(`Bedrock configuration is incomplete. Required: bedrockRegion, bedrockApiKey, bedrockServiceRoleArn, bedrockModelId, bedrockS3Bucket.
@@ -118192,6 +118194,15 @@ function getEvaluationOptions(name, options, state) {
             s3Bucket: state.bedrockS3Bucket
           }
         });
+        const OpenAI4 = __require("openai").default;
+        openaiClient = new OpenAI4({
+          baseURL: `https://bedrock-runtime.${state.bedrockRegion}.amazonaws.com/openai/v1`,
+          apiKey: state.bedrockApiKey,
+          defaultHeaders: {
+            "X-Amzn-Bedrock-RoleArn": state.bedrockServiceRoleArn,
+            "X-Amzn-Bedrock-ModelId": state.bedrockModelId
+          }
+        });
       } else {
         batchClient = new BatchClient;
       }
@@ -118215,6 +118226,7 @@ function getEvaluationOptions(name, options, state) {
     createProgressTracker: createProgressTracker || (() => new NullProgressTracker),
     onPendingBatch,
     execute,
+    batchProcessor,
     batchClient,
     evalClient,
     openaiClient,
@@ -118245,7 +118257,8 @@ async function evaluateDataCase(dataCase, evaluator, _ctx, options) {
       expected: dataCase.expected,
       tags: dataCase.tags,
       id: caseId,
-      client: client2
+      client: client2,
+      model: options.execute === "sync" && options.batchProcessor === "BEDROCK_OPENAI" ? _ctx.state.bedrockModelId : undefined
     };
     const fieldsToExclude = new Set(["state", "ctx", "tags", "id", "client"]);
     const argsForStorage = Object.fromEntries(Object.entries(scorerArgs).filter(([key]) => !fieldsToExclude.has(key)));
@@ -118494,9 +118507,10 @@ async function runEval(name, evaluator, options) {
     jsonl,
     returnResults,
     progress,
-    execute,
+    batchProcessor,
     batchClient
   } = options;
+  let { execute } = options;
   const ctx = init({
     ...evaluator.ctx ?? {},
     state,
@@ -118508,6 +118522,11 @@ async function runEval(name, evaluator, options) {
       [ATTR_DATASET_SOURCE_TYPE]: data.ctx.sourceType,
       [ATTR_EVAL_EXECUTION_METADATA_TEST_CASE_COUNT]: data.ctx.caseCount
     });
+    const totalBatchRequests = data.dataset.length * evaluator.scores.length;
+    if (execute === "batch" && batchProcessor === "BEDROCK_OPENAI" && totalBatchRequests < 100) {
+      execute = "sync";
+      iso.writeln(`ℹ ${totalBatchRequests} batch request(s) (${data.dataset.length} records × ${evaluator.scores.length} scorers) is below the 100 minimum for Bedrock batch. Falling back to sync mode.`);
+    }
     if (execute === "batch" && batchClient) {
       await batchClient.initialize(name, data.ctx.checksum, data.ctx.dataSourceType);
       if (batchClient.hasPendingBatch) {
@@ -118517,10 +118536,11 @@ async function runEval(name, evaluator, options) {
     const results = [];
     const errors6 = new Map;
     progress.start(data.dataset.length);
+    const effectiveOptions = execute !== options.execute ? { ...options, execute } : options;
     for (const dataCase of data.dataset) {
       const caseResult = await evaluateDataCase(dataCase, evaluator, ctx.mutate({
         dataset: data.ctx
-      }), options);
+      }), effectiveOptions);
       results.push(caseResult.result);
       aggregateScorerErrors(errors6, caseResult.errors);
       if (execute === "sync" && options.onResult) {
@@ -118995,4 +119015,4 @@ export {
   Behavioral
 };
 
-//# debugId=B986FC088997E94964756E2164756E21
+//# debugId=31E51C8DF22CA45F64756E2164756E21