orchestrated 0.1.30 → 0.1.32

package/index.d.ts

@@ -815,6 +815,7 @@ declare const InteractionsDatasetOptionsSchema: z.ZodObject<{
     month: z.ZodOptional<z.ZodString>;
     startDate: z.ZodOptional<z.ZodString>;
     endDate: z.ZodOptional<z.ZodString>;
+    limit: z.ZodOptional<z.ZodNumber>;
     timeout: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strip>;
 
@@ -1287,6 +1288,7 @@ declare const SessionsDatasetOptionsSchema: z.ZodObject<{
     month: z.ZodOptional<z.ZodString>;
     startDate: z.ZodOptional<z.ZodString>;
     endDate: z.ZodOptional<z.ZodString>;
+    limit: z.ZodOptional<z.ZodNumber>;
     timeout: z.ZodOptional<z.ZodNumber>;
 }, z.core.$strip>;
 

package/index.js

@@ -21230,6 +21230,7 @@ function resetState() {
   globalState = null;
   isInitialized = false;
   globalThis.__ORCHESTRATED_LAZY_LOAD__ = undefined;
+  if (false) {}
 }
 function isStateInitialized() {
   return isInitialized || globalThis.__ORCHESTRATED_SHARED_STATE__ !== undefined;
@@ -58802,6 +58803,9 @@ async function getDataset(datasetName, source, options) {
   const timeout = options.timeout ?? 30000;
   const allResults = [];
   let cursor;
+  let latestHasMore = false;
+  let latestLastEvalTimestamp = null;
+  const hasExplicitLimit = typeof options.limit === "number" && options.limit > 0;
   do {
     const controller = new AbortController;
     const timeoutId = setTimeout(() => controller.abort(), timeout);
@@ -58835,7 +58839,10 @@ async function getDataset(datasetName, source, options) {
       const rawData = await response.json();
       const data = DatasetResponseSchema.parse(rawData);
       allResults.push(...data.data);
-      cursor = data.nextCursor;
+      cursor = hasExplicitLimit ? undefined : data.nextCursor;
+      latestHasMore = data.hasMore ?? Boolean(data.nextCursor);
+      latestLastEvalTimestamp = data.lastEvalTimestamp ?? null;
+      console.log(`[dataset:${datasetName}] page fetched: pageCount=${data.data.length}, runningTotal=${allResults.length}, hasMore=${latestHasMore}, lastEvalTimestamp=${latestLastEvalTimestamp}`);
     } catch (error48) {
       clearTimeout(timeoutId);
       if (error48 instanceof Error && error48.name === "AbortError") {
@@ -58844,11 +58851,18 @@ async function getDataset(datasetName, source, options) {
       throw error48;
     }
   } while (cursor);
+  globalThis[DATASET_METADATA_KEY] = {
+    datasetName,
+    count: allResults.length,
+    hasMore: latestHasMore,
+    lastEvalTimestamp: latestLastEvalTimestamp
+  };
+  console.log(`[dataset:${datasetName}] pull complete: total=${allResults.length}, hasMore=${latestHasMore}, lastEvalTimestamp=${latestLastEvalTimestamp}`);
   return allResults;
 }
 var getApiUrl = () => {
   return getState().apiUrl;
-}, DatasetResponseSchema;
+}, DatasetResponseSchema, DATASET_METADATA_KEY = "__ORCHESTRATED_DATASET_METADATA__";
 var init_data_source = __esm(() => {
   init_zod();
   DatasetResponseSchema = exports_external.object({
@@ -58860,7 +58874,9 @@ var init_data_source = __esm(() => {
       input: exports_external.string(),
       output: exports_external.string()
     }).loose()),
-    nextCursor: exports_external.string().optional()
+    nextCursor: exports_external.string().optional(),
+    hasMore: exports_external.boolean().optional(),
+    lastEvalTimestamp: exports_external.string().nullable().optional()
   }).passthrough();
 });
 
@@ -113999,8 +114015,27 @@ class S3FileManager {
   async upload(content, key) {
     const { getState: getState2 } = await Promise.resolve().then(() => exports_state);
     const state = getState2();
+    if (state.awsAccessKeyId && state.awsSecretAccessKey) {
+      const { PutObjectCommand, S3Client } = require_dist_cjs79();
+      const s3Key = state.tenantId && state.serviceName ? `${state.tenantId}/${state.serviceName}/${key}` : key;
+      const s3Client = new S3Client({
+        region: this.region,
+        credentials: {
+          accessKeyId: state.awsAccessKeyId,
+          secretAccessKey: state.awsSecretAccessKey,
+          sessionToken: state.awsSessionToken || undefined
+        }
+      });
+      await s3Client.send(new PutObjectCommand({
+        Bucket: this.bucket,
+        Key: s3Key,
+        Body: content,
+        ContentType: "application/jsonl"
+      }));
+      return `s3://${this.bucket}/${s3Key}`;
+    }
     if (!state.accessToken) {
-      throw new Error("No access token found. Please run 'orcha login' to authenticate before using Bedrock batch mode.");
+      throw new Error("No AWS credentials or access token found. Please run 'orcha login' to authenticate before using Bedrock batch mode.");
     }
     if (!state.tenantId || !state.serviceName) {
       throw new Error("tenantId and serviceName are required for Bedrock batch uploads.");
@@ -114155,13 +114190,16 @@ class BatchClient {
       fs.mkdirSync(this.tempDir, { recursive: true });
     }
   }
-  async initialize(name, checksum, dataSourceType, maxAgeDays = 3) {
+  async initialize(name, checksum, dataSourceType, maxAgeDays = 2) {
     this.metadata = { name, checksum };
-    const batches = await this.listBatches(2);
+    console.log(`BatchClient.initialize: searching for name="${name}", checksum="${checksum}", mode=${this.mode}, dataSourceType=${dataSourceType ?? "unknown"}`);
+    const batches = await this.listBatches(100);
+    console.log(`BatchClient.initialize: listBatches returned ${batches.length} batch(es): ${JSON.stringify(batches.map((b) => ({ id: b.id, status: b.status, created_at: b.created_at, metadata: b.metadata })))}`);
     const maxAgeMs = maxAgeDays * 24 * 60 * 60 * 1000;
     const cutoff = Date.now() - maxAgeMs;
     const matchingBatches = batches.filter((batch) => {
-      if (batch.created_at * 1000 < cutoff) {
+      const createdAtMs = this.mode === "OPENAI" ? batch.created_at * 1000 : batch.created_at;
+      if (createdAtMs < cutoff) {
         return false;
       }
       const nameMatches = batch.metadata?.name === this.metadata?.name;
@@ -114172,30 +114210,35 @@ class BatchClient {
       }
     });
     if (matchingBatches.length === 0) {
+      console.log("BatchClient.initialize: no matching batches found");
       return;
     }
+    console.log(`BatchClient.initialize: ${matchingBatches.length} matching batch(es): ${JSON.stringify(matchingBatches.map((b) => ({ id: b.id, status: b.status })))}`);
     const pendingBatches = matchingBatches.filter((batch) => !this.isBatchComplete(batch));
-    const successfulBatches = matchingBatches.filter((batch) => batch.status === "completed");
+    const resultBearingBatches = matchingBatches.filter((batch) => batch.status === "completed" || batch.status === "expired" || batch.status === "cancelled");
     if (pendingBatches.length > 0) {
+      console.log(`BatchClient.initialize: ${pendingBatches.length} pending batch(es), waiting`);
       this.hasPendingBatch = true;
       this.pendingBatches = pendingBatches;
       return;
     }
-    if (successfulBatches.length > 0) {
+    if (resultBearingBatches.length > 0) {
+      console.log(`BatchClient.initialize: loading results from ${resultBearingBatches.length} batch(es): ${JSON.stringify(resultBearingBatches.map((b) => ({ id: b.id, status: b.status })))}`);
       this.hasCompletedBatch = true;
-      this.completedBatches = successfulBatches;
+      this.completedBatches = resultBearingBatches;
       const allResponses = [];
-      for (const batch of successfulBatches) {
+      for (const batch of resultBearingBatches) {
         try {
           const results = await this.getBatchResults(batch.id);
+          console.log(`BatchClient.initialize: loaded ${results.length} result(s) from batch ${batch.id} (status=${batch.status})`);
           allResponses.push(...results);
         } catch (error48) {
           console.warn(`Failed to load results from batch ${batch.id}:`, error48);
         }
       }
       this.responses = allResponses;
-      if (successfulBatches.length > 1) {
-        iso.writeln(`${colors.gray}Loaded ${allResponses.length} total results from ${successfulBatches.length} batches`);
+      if (resultBearingBatches.length > 1) {
+        iso.writeln(`${colors.gray}Loaded ${allResponses.length} total results from ${resultBearingBatches.length} batches`);
       }
     }
   }
@@ -115038,6 +115081,7 @@ var traced = {
 };
 
 // src/evaluator/core.ts
+var BATCH_MIN_RECORDS = 100;
 function deterministicStringify(obj) {
   if (obj === null)
     return "null";
@@ -115191,9 +115235,28 @@ function getEvaluationOptions(name, options, state) {
   let evalClient;
   let openaiClient;
   const batchProcessor = merged.batchProcessor ?? "OPENAI";
+  const createSyncFallbackClient = () => {
+    if (batchProcessor === "BEDROCK_OPENAI") {
+      if (!state.bedrockRegion || !state.bedrockApiKey || !state.bedrockServiceRoleArn || !state.bedrockModelId) {
+        return;
+      }
+      const { BedrockOpenAI: BedrockOpenAI2 } = (init_bedrock_openai_client(), __toCommonJS(exports_bedrock_openai_client));
+      return new BedrockOpenAI2({
+        baseURL: `https://bedrock-mantle.${state.bedrockRegion}.api.aws/v1`,
+        apiKey: state.bedrockApiKey,
+        defaultHeaders: {
+          "X-Amzn-Bedrock-RoleArn": state.bedrockServiceRoleArn,
+          "X-Amzn-Bedrock-ModelId": state.bedrockModelId
+        }
+      });
+    }
+    const OpenAI5 = __require("openai").default;
+    return new OpenAI5;
+  };
   if (execute === "batch") {
     if (merged.batchClient) {
       batchClient = merged.batchClient;
+      openaiClient = merged.openaiClient ?? createSyncFallbackClient();
     } else {
       if (batchProcessor === "BEDROCK_OPENAI") {
         if (!state.bedrockRegion || !state.bedrockApiKey || !state.bedrockServiceRoleArn || !state.bedrockModelId || !state.bedrockS3Bucket) {
@@ -115210,17 +115273,10 @@ function getEvaluationOptions(name, options, state) {
             s3Bucket: state.bedrockS3Bucket
           }
         });
-        const { BedrockOpenAI: BedrockOpenAI2 } = (init_bedrock_openai_client(), __toCommonJS(exports_bedrock_openai_client));
-        openaiClient = new BedrockOpenAI2({
-          baseURL: `https://bedrock-mantle.${state.bedrockRegion}.api.aws/v1`,
-          apiKey: state.bedrockApiKey,
-          defaultHeaders: {
-            "X-Amzn-Bedrock-RoleArn": state.bedrockServiceRoleArn,
-            "X-Amzn-Bedrock-ModelId": state.bedrockModelId
-          }
-        });
+        openaiClient = createSyncFallbackClient();
       } else {
         batchClient = new BatchClient;
+        openaiClient = createSyncFallbackClient();
       }
     }
     evalClient = merged.evalClient || new EvalClient({
@@ -115541,14 +115597,12 @@ async function runEval(name, evaluator, options) {
       [ATTR_DATASET_SOURCE_TYPE]: data.ctx.sourceType,
       [ATTR_EVAL_EXECUTION_METADATA_TEST_CASE_COUNT]: data.ctx.caseCount
     });
-    const totalBatchRequests = data.dataset.length * evaluator.scores.length;
-    if (execute === "batch" && batchProcessor === "BEDROCK_OPENAI" && totalBatchRequests < 100) {
-      execute = "sync";
-      iso.writeln(`ℹ ${totalBatchRequests} batch request(s) (${data.dataset.length} records × ${evaluator.scores.length} scorers) is below the 100 minimum for Bedrock batch. Falling back to sync mode.`);
-    }
+    iso.writeln(`ℹ [batch-trace] dataset resolved: ${data.dataset.length} case(s), sourceType="${data.ctx.sourceType}", dataSourceType="${data.ctx.dataSourceType}", checksum="${data.ctx.checksum}", execute="${execute}", batchProcessor="${batchProcessor}", scorers=${evaluator.scores.length}.`);
     if (execute === "batch" && batchClient) {
       await batchClient.initialize(name, data.ctx.checksum, data.ctx.dataSourceType);
+      iso.writeln(`ℹ [batch-trace] after initialize: ${batchClient.responses.length} cached response(s) loaded from prior batch(es), hasPendingBatch=${batchClient.hasPendingBatch}, hasCompletedBatch=${batchClient.hasCompletedBatch}.`);
       if (batchClient.hasPendingBatch) {
+        iso.writeln("ℹ [batch-trace] a prior batch is still pending — returning early to wait/retry (no new work this run).");
         return batchClient.getPending();
       }
     }
@@ -115573,11 +115627,37 @@ async function runEval(name, evaluator, options) {
     if (!jsonl) {
       progress.stop();
     }
+    if (execute === "batch" && batchClient) {
+      const pendingSoFar = results.filter((r) => r.hasPendingBatch).length;
+      const erroredSoFar = results.filter((r) => r.error).length;
+      const resolvedSoFar = results.length - pendingSoFar - erroredSoFar;
+      iso.writeln(`ℹ [batch-trace] after scoring loop: ${results.length} case(s) processed → ${resolvedSoFar} resolved from cache, ${pendingSoFar} pending (new/changed), ${erroredSoFar} errored. Accumulated ${batchClient.requests.length} batch record(s) across ${evaluator.scores.length} scorer(s)/case, ${batchClient.duplicateCount} duplicate request(s) skipped.`);
+    }
     let batch;
     if (execute === "batch" && batchClient) {
-      batch = await batchClient.submit();
-      if (verbose) {
-        console.log(batch);
+      const pendingRequestCount = batchClient.requests.length;
+      const pendingCaseCount = results.filter((r) => r.hasPendingBatch).length;
+      iso.writeln(`ℹ Batch submit decision: ${pendingCaseCount} pending case(s) → ${pendingRequestCount} batch record(s) (minimum ${BATCH_MIN_RECORDS}, sync-fallback client ${options.openaiClient ? "available" : "MISSING"}).`);
+      if (pendingRequestCount > 0 && pendingRequestCount < BATCH_MIN_RECORDS && options.openaiClient) {
+        iso.writeln(`ℹ ${pendingRequestCount} pending request(s) is below the ${BATCH_MIN_RECORDS}-record batch minimum. Running pending case(s) synchronously.`);
+        const syncOptions = { ...effectiveOptions, execute: "sync" };
+        let reranCount = 0;
+        for (let i2 = 0;i2 < results.length; i2++) {
+          if (!results[i2].hasPendingBatch) {
+            continue;
+          }
+          const caseResult = await evaluateDataCase(data.dataset[i2], evaluator, ctx.mutate({ dataset: data.ctx }), syncOptions);
+          results[i2] = caseResult.result;
+          aggregateScorerErrors(errors6, caseResult.errors);
+          reranCount++;
+        }
+        iso.writeln(`ℹ [batch-trace] SYNC-FALLBACK path: re-ran ${reranCount} pending case(s) synchronously; no batch submitted.`);
+      } else {
+        iso.writeln(`ℹ [batch-trace] SUBMIT path: sending ${pendingRequestCount} record(s) to a new batch${options.openaiClient ? "" : " (sync-fallback unavailable, could not run synchronously)"}.`);
+        batch = await batchClient.submit();
+        if (verbose) {
+          console.log(batch);
+        }
       }
     }
     const summary = createEvaluationSummary(name, results, errors6);
@@ -115762,6 +115842,7 @@ var InteractionsDatasetOptionsSchema = exports_external.object({
   month: exports_external.string().optional(),
   startDate: exports_external.string().optional(),
   endDate: exports_external.string().optional(),
+  limit: exports_external.number().optional(),
   timeout: exports_external.number().optional()
 });
 function interactions(options = {}) {
@@ -115784,6 +115865,7 @@ function interactions(options = {}) {
         month: options.month,
         startDate: options.startDate,
         endDate: options.endDate,
+        limit: options.limit,
         timeout: options.timeout
       },
       args: argsJsonSchema
@@ -115798,6 +115880,7 @@ var SessionsDatasetOptionsSchema = exports_external.object({
   month: exports_external.string().optional(),
   startDate: exports_external.string().optional(),
   endDate: exports_external.string().optional(),
+  limit: exports_external.number().optional(),
   timeout: exports_external.number().optional()
 });
 function sessions(options = {}) {
@@ -115820,6 +115903,7 @@ function sessions(options = {}) {
         month: options.month,
         startDate: options.startDate,
         endDate: options.endDate,
+        limit: options.limit,
         timeout: options.timeout
       },
       args: argsJsonSchema
@@ -116034,4 +116118,4 @@ export {
   Behavioral
 };
 
-//# debugId=05DB886F5F4CE9D864756E2164756E21
+//# debugId=E69C9539C2C08B5A64756E2164756E21