observability-toolkit 2.0.0 → 2.1.1

package/dist/tools/query-evaluations.d.ts

@@ -5,13 +5,25 @@
  * https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-events/
  */
 import { z } from 'zod';
-import type { EvaluationResult, EvaluationAggregation, EvaluationGroupByField, EvaluatorType, TelemetryBackend } from '../backends/index.js';
+import { type EvaluationResult, type EvaluationAggregation, type EvaluationGroupByField, type TelemetryBackend } from '../backends/index.js';
 export declare const queryEvaluationsSchema: z.ZodObject<{
-    aggregation: z.ZodOptional<z.ZodEnum<["avg", "min", "max", "count", "p50", "p95", "p99"]>>;
-    groupBy: z.ZodOptional<z.ZodArray<z.ZodEnum<["evaluationName", "scoreLabel", "evaluator"]>, "many">>;
+    aggregation: z.ZodOptional<z.ZodEnum<{
+        avg: "avg";
+        min: "min";
+        max: "max";
+        count: "count";
+        p50: "p50";
+        p95: "p95";
+        p99: "p99";
+    }>>;
+    groupBy: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+        evaluationName: "evaluationName";
+        scoreLabel: "scoreLabel";
+        evaluator: "evaluator";
+    }>>>;
     limit: z.ZodDefault<z.ZodNumber>;
-    startDate: z.ZodOptional<z.ZodString>;
-    endDate: z.ZodOptional<z.ZodString>;
+    startDate: z.ZodOptional<z.ZodISODate>;
+    endDate: z.ZodOptional<z.ZodISODate>;
     sessionId: z.ZodOptional<z.ZodString>;
     traceId: z.ZodOptional<z.ZodString>;
     evaluationName: z.ZodOptional<z.ZodString>;
@@ -19,80 +31,55 @@ export declare const queryEvaluationsSchema: z.ZodObject<{
     scoreMax: z.ZodOptional<z.ZodNumber>;
     scoreLabel: z.ZodOptional<z.ZodString>;
     evaluator: z.ZodOptional<z.ZodString>;
-    evaluatorType: z.ZodOptional<z.ZodEnum<["llm", "human", "rule", "classifier"]>>;
+    evaluatorType: z.ZodOptional<z.ZodEnum<{
+        llm: "llm";
+        human: "human";
+        rule: "rule";
+        classifier: "classifier";
+        seed: "seed";
+        canary: "canary";
+    }>>;
     responseId: z.ZodOptional<z.ZodString>;
     agentId: z.ZodOptional<z.ZodString>;
     agentName: z.ZodOptional<z.ZodString>;
-}, "strip", z.ZodTypeAny, {
-    limit: number;
-    traceId?: string | undefined;
-    sessionId?: string | undefined;
-    agentId?: string | undefined;
-    agentName?: string | undefined;
-    aggregation?: "avg" | "min" | "max" | "count" | "p50" | "p95" | "p99" | undefined;
-    groupBy?: ("evaluationName" | "scoreLabel" | "evaluator")[] | undefined;
-    evaluationName?: string | undefined;
-    scoreLabel?: string | undefined;
-    evaluator?: string | undefined;
-    scoreMin?: number | undefined;
-    scoreMax?: number | undefined;
-    evaluatorType?: "llm" | "human" | "rule" | "classifier" | undefined;
-    responseId?: string | undefined;
-    endDate?: string | undefined;
-    startDate?: string | undefined;
-}, {
-    limit?: number | undefined;
-    traceId?: string | undefined;
-    sessionId?: string | undefined;
-    agentId?: string | undefined;
-    agentName?: string | undefined;
-    aggregation?: "avg" | "min" | "max" | "count" | "p50" | "p95" | "p99" | undefined;
-    groupBy?: ("evaluationName" | "scoreLabel" | "evaluator")[] | undefined;
-    evaluationName?: string | undefined;
-    scoreLabel?: string | undefined;
-    evaluator?: string | undefined;
-    scoreMin?: number | undefined;
-    scoreMax?: number | undefined;
-    evaluatorType?: "llm" | "human" | "rule" | "classifier" | undefined;
-    responseId?: string | undefined;
-    endDate?: string | undefined;
-    startDate?: string | undefined;
-}>;
+    datasetId: z.ZodOptional<z.ZodString>;
+    backend: z.ZodDefault<z.ZodEnum<{
+        cloud: "cloud";
+        local: "local";
+    }>>;
+}, z.core.$strip>;
 export type QueryEvaluationsInput = z.input<typeof queryEvaluationsSchema>;
 export interface QueryEvaluationsOptions {
     localBackend?: TelemetryBackend;
 }
 /** Evaluation result response */
-export interface EvaluationResponse {
-    /** ISO timestamp of the evaluation */
-    timestamp: string;
-    /** Evaluation metric name (gen_ai.evaluation.name) */
-    evaluationName: string;
-    /** Numeric score (gen_ai.evaluation.score.value) */
-    scoreValue?: number;
-    /** Human-readable score label (gen_ai.evaluation.score.label) */
-    scoreLabel?: string;
-    /** Score unit/scale (e.g., "ratio_0_1", "percentage") */
-    scoreUnit?: string;
-    /** Free-form explanation (gen_ai.evaluation.explanation) */
-    explanation?: string;
-    /** Identity of the evaluator (model, human, system) */
-    evaluator?: string;
-    /** Type of evaluator (llm, human, rule, classifier) */
-    evaluatorType?: EvaluatorType;
-    /** Response ID for correlation (gen_ai.response.id) */
-    responseId?: string;
-    /** Trace ID for correlation */
-    traceId?: string;
-    /** Session ID for session-scoped queries */
-    sessionId?: string;
-    /** Subject agent ID being evaluated (gen_ai.agent.id) */
-    agentId?: string;
-    /** Subject agent name being evaluated (gen_ai.agent.name) */
-    agentName?: string;
-    /** Number of steps in agent trajectory */
-    trajectoryLength?: number;
-}
+export declare const evaluationResponseSchema: z.ZodObject<{
+    timestamp: z.ZodString;
+    evaluationName: z.ZodString;
+    scoreValue: z.ZodOptional<z.ZodNumber>;
+    scoreLabel: z.ZodOptional<z.ZodString>;
+    scoreUnit: z.ZodOptional<z.ZodString>;
+    explanation: z.ZodOptional<z.ZodString>;
+    evaluator: z.ZodOptional<z.ZodString>;
+    evaluatorType: z.ZodOptional<z.ZodEnum<{
+        llm: "llm";
+        human: "human";
+        rule: "rule";
+        classifier: "classifier";
+        seed: "seed";
+        canary: "canary";
+    }>>;
+    responseId: z.ZodOptional<z.ZodString>;
+    traceId: z.ZodOptional<z.ZodString>;
+    sessionId: z.ZodOptional<z.ZodString>;
+    agentId: z.ZodOptional<z.ZodString>;
+    agentName: z.ZodOptional<z.ZodString>;
+    trajectoryLength: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strip>;
+export type EvaluationResponse = z.infer<typeof evaluationResponseSchema>;
+export declare const MAX_RETURNED_EVALUATIONS = 20;
+/** Maximum aggregation groups to prevent memory exhaustion from combinatorial explosion */
+export declare const MAX_AGGREGATION_GROUPS = 10000;
 /** Summary data structure */
 interface EvaluationSummary {
     byEvaluationName: Record<string, number>;
@@ -118,7 +105,26 @@ export declare function buildSummary(evaluations: EvaluationResult[]): Evaluatio
  * Groups evaluations by specified fields and calculates the requested aggregation.
  */
 export declare function buildAggregations(evaluations: EvaluationResult[], aggregation: EvaluationAggregation, groupBy: EvaluationGroupByField[]): AggregationResult[];
-export declare function queryEvaluations(rawInput: QueryEvaluationsInput, backendOptions?: QueryEvaluationsOptions): Promise<{
+export declare const queryEvaluations: (rawInput: {
+    aggregation?: "avg" | "min" | "max" | "count" | "p50" | "p95" | "p99" | undefined;
+    groupBy?: ("evaluationName" | "scoreLabel" | "evaluator")[] | undefined;
+    limit?: number | undefined;
+    startDate?: string | undefined;
+    endDate?: string | undefined;
+    sessionId?: string | undefined;
+    traceId?: string | undefined;
+    evaluationName?: string | undefined;
+    scoreMin?: number | undefined;
+    scoreMax?: number | undefined;
+    scoreLabel?: string | undefined;
+    evaluator?: string | undefined;
+    evaluatorType?: "llm" | "human" | "rule" | "classifier" | "seed" | "canary" | undefined;
+    responseId?: string | undefined;
+    agentId?: string | undefined;
+    agentName?: string | undefined;
+    datasetId?: string | undefined;
+    backend?: "cloud" | "local" | undefined;
+}, backendOptions?: QueryEvaluationsOptions | undefined) => Promise<{
     summary: {
         aggregations?: AggregationResult[] | undefined;
         byEvaluationName: Record<string, number>;
@@ -134,7 +140,7 @@ export declare function queryEvaluations(rawInput: QueryEvaluationsInput, backen
         scoreUnit: string | undefined;
         explanation: string | undefined;
         evaluator: string | undefined;
-        evaluatorType: EvaluatorType | undefined;
+        evaluatorType: "llm" | "human" | "rule" | "classifier" | "seed" | "canary" | undefined;
         responseId: string | undefined;
         traceId: string | undefined;
         sessionId: string | undefined;
@@ -150,11 +156,23 @@ export declare const queryEvaluationsTool: {
     name: string;
     description: string;
     inputSchema: z.ZodObject<{
-        aggregation: z.ZodOptional<z.ZodEnum<["avg", "min", "max", "count", "p50", "p95", "p99"]>>;
-        groupBy: z.ZodOptional<z.ZodArray<z.ZodEnum<["evaluationName", "scoreLabel", "evaluator"]>, "many">>;
+        aggregation: z.ZodOptional<z.ZodEnum<{
+            avg: "avg";
+            min: "min";
+            max: "max";
+            count: "count";
+            p50: "p50";
+            p95: "p95";
+            p99: "p99";
+        }>>;
+        groupBy: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+            evaluationName: "evaluationName";
+            scoreLabel: "scoreLabel";
+            evaluator: "evaluator";
+        }>>>;
         limit: z.ZodDefault<z.ZodNumber>;
-        startDate: z.ZodOptional<z.ZodString>;
-        endDate: z.ZodOptional<z.ZodString>;
+        startDate: z.ZodOptional<z.ZodISODate>;
+        endDate: z.ZodOptional<z.ZodISODate>;
         sessionId: z.ZodOptional<z.ZodString>;
         traceId: z.ZodOptional<z.ZodString>;
         evaluationName: z.ZodOptional<z.ZodString>;
@@ -162,46 +180,70 @@ export declare const queryEvaluationsTool: {
         scoreMax: z.ZodOptional<z.ZodNumber>;
         scoreLabel: z.ZodOptional<z.ZodString>;
         evaluator: z.ZodOptional<z.ZodString>;
-        evaluatorType: z.ZodOptional<z.ZodEnum<["llm", "human", "rule", "classifier"]>>;
+        evaluatorType: z.ZodOptional<z.ZodEnum<{
+            llm: "llm";
+            human: "human";
+            rule: "rule";
+            classifier: "classifier";
+            seed: "seed";
+            canary: "canary";
+        }>>;
         responseId: z.ZodOptional<z.ZodString>;
         agentId: z.ZodOptional<z.ZodString>;
         agentName: z.ZodOptional<z.ZodString>;
-    }, "strip", z.ZodTypeAny, {
-        limit: number;
-        traceId?: string | undefined;
-        sessionId?: string | undefined;
-        agentId?: string | undefined;
-        agentName?: string | undefined;
+        datasetId: z.ZodOptional<z.ZodString>;
+        backend: z.ZodDefault<z.ZodEnum<{
+            cloud: "cloud";
+            local: "local";
+        }>>;
+    }, z.core.$strip>;
+    handler: (rawInput: {
         aggregation?: "avg" | "min" | "max" | "count" | "p50" | "p95" | "p99" | undefined;
         groupBy?: ("evaluationName" | "scoreLabel" | "evaluator")[] | undefined;
-        evaluationName?: string | undefined;
-        scoreLabel?: string | undefined;
-        evaluator?: string | undefined;
-        scoreMin?: number | undefined;
-        scoreMax?: number | undefined;
-        evaluatorType?: "llm" | "human" | "rule" | "classifier" | undefined;
-        responseId?: string | undefined;
-        endDate?: string | undefined;
-        startDate?: string | undefined;
-    }, {
         limit?: number | undefined;
-        traceId?: string | undefined;
+        startDate?: string | undefined;
+        endDate?: string | undefined;
         sessionId?: string | undefined;
-        agentId?: string | undefined;
-        agentName?: string | undefined;
-        aggregation?: "avg" | "min" | "max" | "count" | "p50" | "p95" | "p99" | undefined;
-        groupBy?: ("evaluationName" | "scoreLabel" | "evaluator")[] | undefined;
+        traceId?: string | undefined;
         evaluationName?: string | undefined;
-        scoreLabel?: string | undefined;
-        evaluator?: string | undefined;
         scoreMin?: number | undefined;
         scoreMax?: number | undefined;
-        evaluatorType?: "llm" | "human" | "rule" | "classifier" | undefined;
+        scoreLabel?: string | undefined;
+        evaluator?: string | undefined;
+        evaluatorType?: "llm" | "human" | "rule" | "classifier" | "seed" | "canary" | undefined;
         responseId?: string | undefined;
-        endDate?: string | undefined;
-        startDate?: string | undefined;
+        agentId?: string | undefined;
+        agentName?: string | undefined;
+        datasetId?: string | undefined;
+        backend?: "cloud" | "local" | undefined;
+    }, backendOptions?: QueryEvaluationsOptions | undefined) => Promise<{
+        summary: {
+            aggregations?: AggregationResult[] | undefined;
+            byEvaluationName: Record<string, number>;
+            byScoreLabel: Record<string, number>;
+            byEvaluator: Record<string, number>;
+            averageScore?: number;
+        };
+        evaluations: {
+            timestamp: string;
+            evaluationName: string;
+            scoreValue: number | undefined;
+            scoreLabel: string | undefined;
+            scoreUnit: string | undefined;
+            explanation: string | undefined;
+            evaluator: string | undefined;
+            evaluatorType: "llm" | "human" | "rule" | "classifier" | "seed" | "canary" | undefined;
+            responseId: string | undefined;
+            traceId: string | undefined;
+            sessionId: string | undefined;
+            agentId: string | undefined;
+            agentName: string | undefined;
+            trajectoryLength: number | undefined;
+        }[];
+        truncated?: boolean | undefined;
+        originalCount?: number | undefined;
+        count: number;
     }>;
-    handler: typeof queryEvaluations;
 };
 export {};
 //# sourceMappingURL=query-evaluations.d.ts.map

package/dist/tools/query-evaluations.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"query-evaluations.d.ts","sourceRoot":"","sources":["../../src/tools/query-evaluations.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAIxB,OAAO,KAAK,EAAE,gBAAgB,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAc7I,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAiBjC,CAAC;AAEH,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAG3E,MAAM,WAAW,uBAAuB;IACtC,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC;AAED,iCAAiC;AACjC,MAAM,WAAW,kBAAkB;IACjC,sCAAsC;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,sDAAsD;IACtD,cAAc,EAAE,MAAM,CAAC;IACvB,oDAAoD;IACpD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,iEAAiE;IACjE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,yDAAyD;IACzD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,4DAA4D;IAC5D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uDAAuD;IACvD,aAAa,CAAC,EAAE,aAAa,CAAC;IAC9B,uDAAuD;IACvD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,+BAA+B;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,4CAA4C;IAC5C,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB,yDAAyD;IACzD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,0CAA0C;IAC1C,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAuHD,6BAA6B;AAC7B,UAAU,iBAAiB;IACzB,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,8BAA8B;AAC9B,UAAU,iBAAiB;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,WAAW,EAAE,gBAAgB,EAAE,GAAG,iBAAiB,CA6B/E;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,WAAW,EAAE,gBAAgB,EAAE,EAC/B,WAAW,EAAE,qBAAqB,EAClC,OAAO,EAAE,sBAAsB,EAAE,GAChC,iBAAiB,EAAE,CAiDrB;AAED,wBAAsB,gBAAgB,CAAC,QAAQ,EAAE,qBAAqB,EAAE,cAAc,CAAC,EAAE,uBAAuB;;;0BA7G5F,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;sBAC1B,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;qBACvB,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;uBACpB,MAAM;;;;;;;;;;;;;;;;;;;;;GAoRtB;AAED,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAKhC,CAAC"}
+{"version":3,"file":"query-evaluations.d.ts","sourceRoot":"","sources":["../../src/tools/query-evaluations.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,CAAC,EAAE,MAAM,KAAK,CAAC;AAYjB,OAAO,EAAuB,KAAK,gBAAgB,EAAE,KAAK,qBAAqB,EAAE,KAAK,sBAAsB,EAAsB,KAAK,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAsBtL,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBAoBjC,CAAC;AAEH,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAG3E,MAAM,WAAW,uBAAuB;IACtC,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACjC;AAED,iCAAiC;AACjC,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;iBA8BnC,CAAC;AAEH,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAgC1E,eAAO,MAAM,wBAAwB,KAAK,CAAC;AAC3C,2FAA2F;AAC3F,eAAO,MAAM,sBAAsB,QAAQ,CAAC;AAmG5C,6BAA6B;AAC7B,UAAU,iBAAiB;IACzB,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACzC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,8BAA8B;AAC9B,UAAU,iBAAiB;IACzB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,WAAW,EAAE,gBAAgB,EAAE,GAAG,iBAAiB,CA6B/E;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAC/B,WAAW,EAAE,gBAAgB,EAAE,EAC/B,WAAW,EAAE,qBAAqB,EAClC,OAAO,EAAE,sBAAsB,EAAE,GAChC,iBAAiB,EAAE,CA6CrB;AAED,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;;;;;0BAzGT,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;sBAC1B,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;qBACvB,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;uBACpB,MAAM;;;;;;;;;;;;;;;;;;;;;EAgQrB,CAAC;AAEH,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;8BArQb,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;0BAC1B,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;yBACvB,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC;2BACpB,MAAM;;;;;;;;;;;;;;;;;;;;;;CAuQtB,CAAC"}