orchestrated 0.1.6 → 0.1.8

package/index.d.ts

@@ -247,6 +247,12 @@ declare type DataSourceDefinition = {
     type: "dataset";
     config: {
         name: string;
+        /**
+         * Optional group name for organizing datasets in the same namespace
+         * If not provided, the name will be used for grouping
+         * This is used to build the source_type namespace (e.g., "live.interactions")
+         */
+        group_name?: string;
         source: {
             baseUrl: string;
             path: string;
@@ -257,6 +263,11 @@ declare type DataSourceDefinition = {
 } | {
     type: "static";
     data: unknown[];
+    /**
+     * Optional group name for static datasets
+     * If not provided, defaults to "static"
+     */
+    group_name?: string;
 };
 
 /**
@@ -447,7 +458,10 @@ declare interface EvalScorerContext {
     scorerName: string;
     caseId: string;
     datasetId: string;
-    datasetSourceType: "live" | "synthetic";
+    /**
+     * Namespaced source type (e.g., "live.interactions", "synthetic.my-group")
+     */
+    datasetSourceType: string;
     dataCase: EvalCase<any, any, any, any>;
 }
 
@@ -755,6 +769,13 @@ export declare interface InteractionsDatasetOptions extends HTTPDataSourceOption
      * @example "production", "staging", "development"
      */
     environment: string;
+    /**
+     * Optional group name for organizing datasets in the same namespace
+     * If not provided, defaults to "interactions"
+     * This is used to build the source_type namespace (e.g., "live.interactions")
+     * @example "sessions", "user-queries"
+     */
+    group_name?: string;
     /**
      * Optional month filter (date string)
      * @example "01/2026"
@@ -905,7 +926,13 @@ export declare function resetState(): void;
 
 declare interface ResolvedDataCtx {
     isLiveData: boolean;
-    sourceType: "live" | "synthetic";
+    /**
+     * Source type with namespace pattern:
+     * - "live.interactions" - live data from interactions dataset
+     * - "live.sessions" - live data from sessions dataset
+     * - "synthetic.my-group" - synthetic data with custom grouping
+     */
+    sourceType: string;
     checksum: string;
     caseCount: number;
 }
@@ -1133,6 +1160,69 @@ export declare interface SerializableScorerDefinition extends SerializableScorer
     temperature?: number;
 }
 
+/**
+ * Creates a dataset definition for the sessions dataset
+ * Fetches sessions with their last interaction's input/output messages
+ * and session-level metadata (usage_by_model, segments, etc.)
+ *
+ * @example
+ * ```ts
+ * import { Eval } from "./evaluator";
+ * import { sessions } from "./data-source/apigateway";
+ *
+ * await Eval("Session Quality Eval", {
+ *   data: sessions({
+ *     tenantId: "shortcuts",
+ *     serviceName: "shortie",
+ *     environment: "production",
+ *   }),
+ *   scores: ["Effectiveness", "GuardrailAdherence"]
+ * });
+ * ```
+ */
+export declare function sessions(options?: Partial<Omit<SessionsDatasetOptions, "baseUrl">>): DataSourceDefinition;
+
+/**
+ * Options for fetching the sessions dataset
+ */
+export declare interface SessionsDatasetOptions extends HTTPDataSourceOptions {
+    /**
+     * Tenant ID to filter sessions
+     */
+    tenantId: string;
+    /**
+     * Service name to filter sessions
+     */
+    serviceName: string;
+    /**
+     * Environment to filter sessions
+     * @example "production", "staging", "development"
+     */
+    environment: string;
+    /**
+     * Optional group name for organizing datasets in the same namespace
+     * If not provided, defaults to "sessions"
+     * This is used to build the source_type namespace (e.g., "live.sessions")
+     * @example "user-sessions", "support-sessions"
+     */
+    group_name?: string;
+    /**
+     * Optional month filter (date string)
+     * @example "01/2026"
+     */
+    month?: string;
+    /**
+     * Optional period filter (date string)
+     * @example "01/01/2026"
+     */
+    startDate?: string;
+    /**
+     * Optional period filter (date string)
+     * @example "01/01/2026"
+     */
+    endDate?: string;
+}
+
 /**
  * An interface that represents a span. A span represents a single operation
  * within a trace. Examples of span might include remote procedure calls or a

package/index.js

@@ -21185,6 +21185,7 @@ async function getDataset(datasetName, source, options) {
     path2 = `me/${path2}`;
   }
   const params = {
+    ...options,
     name: datasetName,
     tenantId: options.tenantId || state.tenantId,
     serviceName: options.serviceName || state.serviceName,
@@ -117326,10 +117327,6 @@ async function initTelemetry(state) {
   } else {
     throw new Error("Authentication is required. Either provide accessToken, apiKey option, or set ORCHESTRATED_ACCESS_TOKEN/ORCHESTRATED_API_KEY environment variable. Run 'orcha login' to authenticate.");
   }
-  console.log({
-    headers,
-    url: `${state.otelEndpoint}${isAccessToken ? "/me" : ""}/v1/traces`
-  });
   try {
     const exporter = new import_exporter_trace_otlp_http.OTLPTraceExporter({
       url: `${state.otelEndpoint}${isAccessToken ? "/me" : ""}/v1/traces`,
@@ -117510,7 +117507,7 @@ var traced = {
         [ATTR_DATASET_CASE_ID]: ctx.caseId,
         [ATTR_DATASET_SOURCE_TYPE]: ctx.dataset.sourceType
       });
-      if (ctx.dataset.sourceType === "live" && ctx.traceId && ctx.spanId) {
+      if (ctx.dataset.sourceType.startsWith("live") && ctx.traceId && ctx.spanId) {
         evalSpan.addLink({
           context: {
             traceId: ctx.traceId,
@@ -117550,7 +117547,7 @@ var traced = {
         [ATTR_DATASET_CASE_METADATA]: JSON.stringify(ctx.dataCase.ctx)
       });
       const dataCaseCtx = ctx.dataCase.ctx;
-      if (ctx.datasetSourceType === "live" && dataCaseCtx && dataCaseCtx.traceId && dataCaseCtx.spanId) {
+      if (ctx.datasetSourceType.startsWith("live") && dataCaseCtx && dataCaseCtx.traceId && dataCaseCtx.spanId) {
         scorerSpan.addLink({
           context: {
             traceId: dataCaseCtx.traceId,
@@ -117629,10 +117626,23 @@ function calculatePercentile(sortedValues, percentile) {
   const upperValue = sortedValues[upperIndex];
   return lowerValue + (upperValue - lowerValue) * fraction;
 }
-async function resolveDataSet(dataset) {
+function buildSourceType(isLiveData, datasetName, groupName) {
+  const prefix = isLiveData ? "live" : "synthetic";
+  let namespace;
+  if (groupName) {
+    namespace = groupName;
+  } else if (datasetName) {
+    const parts = datasetName.split(".");
+    namespace = parts[parts.length - 1];
+  } else {
+    namespace = isLiveData ? "data" : "data";
+  }
+  return `${prefix}.${namespace}`;
+}
+async function resolveDataSet(dataset, datasetMetadata) {
   return traced.dataSource(async () => {
     const isLiveData = dataset.some((c) => c.ctx?.traceId && c.ctx?.spanId);
-    const sourceType = isLiveData ? "live" : "synthetic";
+    const sourceType = buildSourceType(isLiveData, datasetMetadata?.name, datasetMetadata?.group_name);
     const checksum = generateTestCasesChecksum(dataset);
     return {
       dataset,
@@ -117959,7 +117969,7 @@ async function runEval(name, evaluator, options) {
     evaluatorName: name
   });
   return traced.execution(ctx, async (rootSpan) => {
-    const data = await resolveDataSet(evaluator.data);
+    const data = await resolveDataSet(evaluator.data, evaluator.datasetMetadata);
     rootSpan.setAttributes({
       [ATTR_DATASET_SOURCE_TYPE]: data.ctx.sourceType,
       [ATTR_EVAL_EXECUTION_METADATA_TEST_CASE_COUNT]: data.ctx.caseCount
@@ -118040,8 +118050,10 @@ function Eval(name, evaluator, options) {
       }
       if (pendingInfo.batches.length > 0) {
         await resolvedOptions.onPendingBatch(pendingInfo.evalName, pendingInfo.batches);
+        await shutdownTelemetry();
         return evalResult;
       }
+      await shutdownTelemetry();
       return evalResult;
     }
     if (evalResult) {
@@ -118059,13 +118071,27 @@ function Eval(name, evaluator, options) {
   registerEvaluation(promise3);
   promise3.catch((error48) => {
     console.error(`Error in Eval "${name}":`, error48);
+    return shutdownTelemetry();
   });
   return promise3;
 }
 async function resolveEvaluatorDefinitions(name, evaluator) {
   let resolvedData;
+  let datasetMetadata;
   if (typeof evaluator.data === "object" && evaluator.data !== null && !Array.isArray(evaluator.data) && "type" in evaluator.data && (evaluator.data.type === "dataset" || evaluator.data.type === "static")) {
-    const dataSourceFn = await resolveDataSource(evaluator.data);
+    const dataSourceDef = evaluator.data;
+    if (dataSourceDef.type === "dataset") {
+      datasetMetadata = {
+        name: dataSourceDef.config.name,
+        group_name: dataSourceDef.config.group_name
+      };
+    } else if (dataSourceDef.type === "static") {
+      datasetMetadata = {
+        name: "static",
+        group_name: dataSourceDef.group_name
+      };
+    }
+    const dataSourceFn = await resolveDataSource(dataSourceDef);
     resolvedData = await dataSourceFn();
   } else {
     resolvedData = evaluator.data;
@@ -118095,6 +118121,7 @@ async function resolveEvaluatorDefinitions(name, evaluator) {
     name,
     ctx: evaluator.ctx,
     data: resolvedData,
+    datasetMetadata,
     task: resolvedTask,
     scores: resolvedScores
   };
@@ -118106,6 +118133,31 @@ function interactions(options = {}) {
     type: "dataset",
     config: {
       name: "orchestrated.dataset.interactions",
+      group_name: options.group_name,
+      source: {
+        baseUrl: getApiUrl(),
+        path: "/dataset",
+        method: "POST"
+      },
+      options: {
+        tenantId: options.tenantId || state.tenantId,
+        serviceName: options.serviceName || state.serviceName,
+        environment: options.environment || state.environment,
+        month: options.month,
+        startDate: options.startDate,
+        endDate: options.endDate,
+        timeout: options.timeout
+      }
+    }
+  };
+}
+function sessions(options = {}) {
+  const state = getState();
+  return {
+    type: "dataset",
+    config: {
+      name: "orchestrated.dataset.sessions",
+      group_name: options.group_name,
       source: {
         baseUrl: getApiUrl(),
         path: "/dataset",
@@ -118305,6 +118357,7 @@ var legacyReporter = {
 export {
   waitForEvaluations,
   traced,
+  sessions,
   resetState,
   registerEvaluation,
   projects,
@@ -118326,4 +118379,4 @@ export {
   Behavioral
 };
 
-//# debugId=AD8F1FF6A824721564756E2164756E21
+//# debugId=1A3AD42D6FAE354F64756E2164756E21