orchestrated 0.1.31 → 0.1.32

package/index.js

@@ -58842,6 +58842,7 @@ async function getDataset(datasetName, source, options) {
       cursor = hasExplicitLimit ? undefined : data.nextCursor;
       latestHasMore = data.hasMore ?? Boolean(data.nextCursor);
       latestLastEvalTimestamp = data.lastEvalTimestamp ?? null;
+      console.log(`[dataset:${datasetName}] page fetched: pageCount=${data.data.length}, runningTotal=${allResults.length}, hasMore=${latestHasMore}, lastEvalTimestamp=${latestLastEvalTimestamp}`);
     } catch (error48) {
       clearTimeout(timeoutId);
       if (error48 instanceof Error && error48.name === "AbortError") {
@@ -58856,6 +58857,7 @@ async function getDataset(datasetName, source, options) {
     hasMore: latestHasMore,
     lastEvalTimestamp: latestLastEvalTimestamp
   };
+  console.log(`[dataset:${datasetName}] pull complete: total=${allResults.length}, hasMore=${latestHasMore}, lastEvalTimestamp=${latestLastEvalTimestamp}`);
   return allResults;
 }
 var getApiUrl = () => {
@@ -114213,30 +114215,30 @@ class BatchClient {
     }
     console.log(`BatchClient.initialize: ${matchingBatches.length} matching batch(es): ${JSON.stringify(matchingBatches.map((b) => ({ id: b.id, status: b.status })))}`);
     const pendingBatches = matchingBatches.filter((batch) => !this.isBatchComplete(batch));
-    const successfulBatches = matchingBatches.filter((batch) => batch.status === "completed");
+    const resultBearingBatches = matchingBatches.filter((batch) => batch.status === "completed" || batch.status === "expired" || batch.status === "cancelled");
     if (pendingBatches.length > 0) {
       console.log(`BatchClient.initialize: ${pendingBatches.length} pending batch(es), waiting`);
       this.hasPendingBatch = true;
       this.pendingBatches = pendingBatches;
       return;
     }
-    if (successfulBatches.length > 0) {
-      console.log(`BatchClient.initialize: loading results from ${successfulBatches.length} completed batch(es)`);
+    if (resultBearingBatches.length > 0) {
+      console.log(`BatchClient.initialize: loading results from ${resultBearingBatches.length} batch(es): ${JSON.stringify(resultBearingBatches.map((b) => ({ id: b.id, status: b.status })))}`);
       this.hasCompletedBatch = true;
-      this.completedBatches = successfulBatches;
+      this.completedBatches = resultBearingBatches;
       const allResponses = [];
-      for (const batch of successfulBatches) {
+      for (const batch of resultBearingBatches) {
         try {
           const results = await this.getBatchResults(batch.id);
-          console.log(`BatchClient.initialize: loaded ${results.length} results from batch ${batch.id}`);
+          console.log(`BatchClient.initialize: loaded ${results.length} result(s) from batch ${batch.id} (status=${batch.status})`);
           allResponses.push(...results);
         } catch (error48) {
           console.warn(`Failed to load results from batch ${batch.id}:`, error48);
         }
       }
       this.responses = allResponses;
-      if (successfulBatches.length > 1) {
-        iso.writeln(`${colors.gray}Loaded ${allResponses.length} total results from ${successfulBatches.length} batches`);
+      if (resultBearingBatches.length > 1) {
+        iso.writeln(`${colors.gray}Loaded ${allResponses.length} total results from ${resultBearingBatches.length} batches`);
       }
     }
   }
@@ -115079,6 +115081,7 @@ var traced = {
 };
 
 // src/evaluator/core.ts
+var BATCH_MIN_RECORDS = 100;
 function deterministicStringify(obj) {
   if (obj === null)
     return "null";
@@ -115232,9 +115235,28 @@ function getEvaluationOptions(name, options, state) {
   let evalClient;
   let openaiClient;
   const batchProcessor = merged.batchProcessor ?? "OPENAI";
+  const createSyncFallbackClient = () => {
+    if (batchProcessor === "BEDROCK_OPENAI") {
+      if (!state.bedrockRegion || !state.bedrockApiKey || !state.bedrockServiceRoleArn || !state.bedrockModelId) {
+        return;
+      }
+      const { BedrockOpenAI: BedrockOpenAI2 } = (init_bedrock_openai_client(), __toCommonJS(exports_bedrock_openai_client));
+      return new BedrockOpenAI2({
+        baseURL: `https://bedrock-mantle.${state.bedrockRegion}.api.aws/v1`,
+        apiKey: state.bedrockApiKey,
+        defaultHeaders: {
+          "X-Amzn-Bedrock-RoleArn": state.bedrockServiceRoleArn,
+          "X-Amzn-Bedrock-ModelId": state.bedrockModelId
+        }
+      });
+    }
+    const OpenAI5 = __require("openai").default;
+    return new OpenAI5;
+  };
   if (execute === "batch") {
     if (merged.batchClient) {
       batchClient = merged.batchClient;
+      openaiClient = merged.openaiClient ?? createSyncFallbackClient();
     } else {
       if (batchProcessor === "BEDROCK_OPENAI") {
         if (!state.bedrockRegion || !state.bedrockApiKey || !state.bedrockServiceRoleArn || !state.bedrockModelId || !state.bedrockS3Bucket) {
@@ -115251,19 +115273,10 @@ function getEvaluationOptions(name, options, state) {
             s3Bucket: state.bedrockS3Bucket
           }
         });
-        const { BedrockOpenAI: BedrockOpenAI2 } = (init_bedrock_openai_client(), __toCommonJS(exports_bedrock_openai_client));
-        openaiClient = new BedrockOpenAI2({
-          baseURL: `https://bedrock-mantle.${state.bedrockRegion}.api.aws/v1`,
-          apiKey: state.bedrockApiKey,
-          defaultHeaders: {
-            "X-Amzn-Bedrock-RoleArn": state.bedrockServiceRoleArn,
-            "X-Amzn-Bedrock-ModelId": state.bedrockModelId
-          }
-        });
+        openaiClient = createSyncFallbackClient();
       } else {
         batchClient = new BatchClient;
-        const OpenAI5 = __require("openai").default;
-        openaiClient = new OpenAI5;
+        openaiClient = createSyncFallbackClient();
       }
     }
     evalClient = merged.evalClient || new EvalClient({
@@ -115584,13 +115597,12 @@ async function runEval(name, evaluator, options) {
       [ATTR_DATASET_SOURCE_TYPE]: data.ctx.sourceType,
       [ATTR_EVAL_EXECUTION_METADATA_TEST_CASE_COUNT]: data.ctx.caseCount
     });
-    if (execute === "batch" && data.dataset.length < 100) {
-      execute = "sync";
-      iso.writeln(`ℹ ${data.dataset.length} record(s) is below the 100-record threshold for batch mode. Falling back to sync (individual requests).`);
-    }
+    iso.writeln(`ℹ [batch-trace] dataset resolved: ${data.dataset.length} case(s), sourceType="${data.ctx.sourceType}", dataSourceType="${data.ctx.dataSourceType}", checksum="${data.ctx.checksum}", execute="${execute}", batchProcessor="${batchProcessor}", scorers=${evaluator.scores.length}.`);
     if (execute === "batch" && batchClient) {
       await batchClient.initialize(name, data.ctx.checksum, data.ctx.dataSourceType);
+      iso.writeln(`ℹ [batch-trace] after initialize: ${batchClient.responses.length} cached response(s) loaded from prior batch(es), hasPendingBatch=${batchClient.hasPendingBatch}, hasCompletedBatch=${batchClient.hasCompletedBatch}.`);
       if (batchClient.hasPendingBatch) {
+        iso.writeln("ℹ [batch-trace] a prior batch is still pending — returning early to wait/retry (no new work this run).");
         return batchClient.getPending();
       }
     }
@@ -115615,11 +115627,37 @@ async function runEval(name, evaluator, options) {
     if (!jsonl) {
       progress.stop();
     }
+    if (execute === "batch" && batchClient) {
+      const pendingSoFar = results.filter((r) => r.hasPendingBatch).length;
+      const erroredSoFar = results.filter((r) => r.error).length;
+      const resolvedSoFar = results.length - pendingSoFar - erroredSoFar;
+      iso.writeln(`ℹ [batch-trace] after scoring loop: ${results.length} case(s) processed → ${resolvedSoFar} resolved from cache, ${pendingSoFar} pending (new/changed), ${erroredSoFar} errored. Accumulated ${batchClient.requests.length} batch record(s) across ${evaluator.scores.length} scorer(s)/case, ${batchClient.duplicateCount} duplicate request(s) skipped.`);
+    }
     let batch;
     if (execute === "batch" && batchClient) {
-      batch = await batchClient.submit();
-      if (verbose) {
-        console.log(batch);
+      const pendingRequestCount = batchClient.requests.length;
+      const pendingCaseCount = results.filter((r) => r.hasPendingBatch).length;
+      iso.writeln(`ℹ Batch submit decision: ${pendingCaseCount} pending case(s) → ${pendingRequestCount} batch record(s) (minimum ${BATCH_MIN_RECORDS}, sync-fallback client ${options.openaiClient ? "available" : "MISSING"}).`);
+      if (pendingRequestCount > 0 && pendingRequestCount < BATCH_MIN_RECORDS && options.openaiClient) {
+        iso.writeln(`ℹ ${pendingRequestCount} pending request(s) is below the ${BATCH_MIN_RECORDS}-record batch minimum. Running pending case(s) synchronously.`);
+        const syncOptions = { ...effectiveOptions, execute: "sync" };
+        let reranCount = 0;
+        for (let i2 = 0;i2 < results.length; i2++) {
+          if (!results[i2].hasPendingBatch) {
+            continue;
+          }
+          const caseResult = await evaluateDataCase(data.dataset[i2], evaluator, ctx.mutate({ dataset: data.ctx }), syncOptions);
+          results[i2] = caseResult.result;
+          aggregateScorerErrors(errors6, caseResult.errors);
+          reranCount++;
+        }
+        iso.writeln(`ℹ [batch-trace] SYNC-FALLBACK path: re-ran ${reranCount} pending case(s) synchronously; no batch submitted.`);
+      } else {
+        iso.writeln(`ℹ [batch-trace] SUBMIT path: sending ${pendingRequestCount} record(s) to a new batch${options.openaiClient ? "" : " (sync-fallback unavailable, could not run synchronously)"}.`);
+        batch = await batchClient.submit();
+        if (verbose) {
+          console.log(batch);
+        }
       }
     }
     const summary = createEvaluationSummary(name, results, errors6);
@@ -116080,4 +116118,4 @@ export {
   Behavioral
 };
 
-//# debugId=26267E7384DF594064756E2164756E21
+//# debugId=E69C9539C2C08B5A64756E2164756E21