@agnt5/sdk 0.3.5 → 0.3.7

package/dist/scorer.d.ts

@@ -27,6 +27,18 @@ export interface TraceEvent {
     data: Record<string, any>;
     name?: string;
 }
+/** Tool call extracted from journal events or normalized session payloads */
+export interface ToolCall {
+    name: string;
+    arguments?: any;
+    callId?: string;
+    spanId?: string;
+    timestampNs?: number;
+    startedAt?: number;
+    endedAt?: number;
+    status?: string;
+    metadata: Record<string, any>;
+}
 /** Result of a scorer evaluation */
 export declare class ScorerResult {
     /** Score between 0.0 and 1.0 */
@@ -70,11 +82,13 @@ export interface ScorerContext {
 }
 /** Scorer handler function signature */
 export type ScorerHandler = (ctx: ScorerContext, request: ScorerRequest) => ScorerResult | Promise<ScorerResult>;
+export type ScorerScope = 'item' | 'run' | 'trace' | 'span' | 'session' | 'fleet_run';
 /** Configuration for a registered scorer */
 export interface ScorerConfig {
     name: string;
     handler: ScorerHandler;
     description: string;
+    scope: ScorerScope;
     isAsync: boolean;
     inputSchema?: Record<string, any>;
 }
@@ -91,7 +105,7 @@ export interface ScorerConfig {
  * );
  * ```
  */
-export declare function scorer(name?: string, description?: string): <F extends ScorerHandler>(handler: F) => F;
+export declare function scorer(name?: string, description?: string, scope?: ScorerScope): <F extends ScorerHandler>(handler: F) => F;
 /** Check if a function is a scorer */
 export declare function isScorer(fn: any): boolean;
 /** Get scorer config from a decorated function */
@@ -173,6 +187,8 @@ export declare function levenshtein(request: ScorerRequest): ScorerResult;
  * parsing rules, same parse_error label on bad output.
  */
 export declare function llmJudge(request: ScorerRequest, ctx?: ScorerContext): Promise<ScorerResult>;
+export declare function correctness(request: ScorerRequest, ctx?: ScorerContext): Promise<ScorerResult>;
+export declare function faithfulness(request: ScorerRequest, ctx?: ScorerContext): Promise<ScorerResult>;
 /**
  * Run a scorer by name against a request.
  */
@@ -183,4 +199,20 @@ export declare function getRequestConfig(request: ScorerRequest, key: string, de
 export declare function getTraceEvents(request: ScorerRequest, eventType: string): TraceEvent[];
 /** Get total tokens from trace events */
 export declare function getTotalTokens(request: ScorerRequest): number;
+/** Extract typed tool calls from ScorerRequest journal events */
+export declare function getToolCalls(request: ScorerRequest): ToolCall[];
+/** Extract typed tool calls from journal events */
+export declare function extractToolCallsFromEvents(events?: TraceEvent[]): ToolCall[];
+/** Return tool names in observed call order */
+export declare function getToolCallNames(request: ScorerRequest): string[];
+/** Return tool-call names from typed tool calls */
+export declare function toolCallNames(calls: ToolCall[]): string[];
+/** Return true when the observed trajectory exactly matches expected */
+export declare function toolTrajectoryExact(actual: string[], expected: string[]): boolean;
+/** Return true when expected appears as an ordered subsequence */
+export declare function toolTrajectoryInOrder(actual: string[], expected: string[]): boolean;
+/** Return true when actual contains expected names with matching counts */
+export declare function toolTrajectoryAnyOrder(actual: string[], expected: string[]): boolean;
+/** Compare a tool trajectory using exact, in_order, or any_order semantics */
+export declare function toolTrajectoryMatches(actual: string[], expected: string[], mode?: 'exact' | 'in_order' | 'any_order'): boolean;
 //# sourceMappingURL=scorer.d.ts.map

package/dist/scorer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scorer.d.ts","sourceRoot":"","sources":["../src/scorer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH,mEAAmE;AACnE,MAAM,WAAW,aAAa;IAC5B,kCAAkC;IAClC,MAAM,EAAE,GAAG,CAAC;IACZ,iCAAiC;IACjC,QAAQ,CAAC,EAAE,GAAG,CAAC;IACf,gCAAgC;IAChC,KAAK,CAAC,EAAE,GAAG,CAAC;IACZ,6CAA6C;IAC7C,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC;IACrB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC9B;AAED,6CAA6C;AAC7C,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,oCAAoC;AACpC,qBAAa,YAAY;IACvB,gCAAgC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,oCAAoC;IACpC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,2BAA2B;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,0CAA0C;IAC1C,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,0BAA0B;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAE5B,IAAI,EAAE;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAChC;IAQD,8BAA8B;IAC9B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,YAAY;IAI/C,8BAA8B;IAC9B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,YAAY;CAGhD;AAED,wDAAwD;AACxD,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED,2CAA2C;AAC3C,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,CAAC;CAC7D;AAED,wCAAwC;AACxC,MAAM,MAAM,aAAa,GAAG,CAC1B,GAAG,EAAE,aAAa,EAClB,OAAO,EAAE,aAAa,KACnB,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;AAE1C,4CAA4C;AAC5C,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,aAAa,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACnC;AAMD;;;;;;;;;;;;GAYG;AACH,wBAAgB,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,IACvC,CAAC,SAAS,aAAa,EAAE,SAAS,CAAC,KAAG,CAAC,CAazD;AAED,sCAAsC;AACtC,wBAAgB,QAAQ,CAAC,EAAE,EAAE,GAAG,GAAG,OAAO,CAEzC;AAED,kDAAkD;AAClD,wBAAgB,eAAe,CAAC,EAAE,EAAE,GAAG,GAAG,YAAY,GAAG,SAAS,CAEjE;AAID;;GAEG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAmC;IAE1D,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAI3C,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS;IAIlD,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC;IAIvC,MAAM,CAAC,SAAS,IAAI,MAAM,EAAE;IAI5B,MAAM,CAAC,KAAK,IAAI,IAAI;CAGrB;AAID,uCAAuC;AACvC,wBAAgB,UAAU,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAS/D;AAED;;;;;;;GAOG;AACH,wBAAgB,QAAQ,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAc7D;AAED;;;;;;GAMG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAY9D;AAED;;;;;GAKG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAoB/D;AAED;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAkE/D;AAED;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAgDjE;AAED,wDAAwD;AACxD,wBAAgB,WAAW,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAuChE;AAkBD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,QAAQ,CAC5B,OAAO,EAAE,aAAa,EACtB,GAAG,CAAC,EAAE,aAAa,GAClB,OAAO,CAAC,YAAY,CAAC,CAyEvB;AA0GD;;GAEG;AACH,wBAAsB,SAAS,CAC7B,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,aAAa,EACtB,GAAG,CAAC,EAAE,aAAa,GAClB,OAAO,CAAC,YAAY,CAAC,CAcvB;AAID,uCAAuC;AACvC,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,GAAG,GAAG,GAAG,CAE7F;AAED,oDAAoD;AACpD,wBAAgB,cAAc,CAAC,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,GAAG,UAAU,EAAE,CAEtF;AAED,yCAAyC;AACzC,wBAAgB,cAAc,CAAC,OAAO,EAAE,aAAa,GAAG,MAAM,CAI7D"}
+{"version":3,"file":"scorer.d.ts","sourceRoot":"","sources":["../src/scorer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH,mEAAmE;AACnE,MAAM,WAAW,aAAa;IAC5B,kCAAkC;IAClC,MAAM,EAAE,GAAG,CAAC;IACZ,iCAAiC;IACjC,QAAQ,CAAC,EAAE,GAAG,CAAC;IACf,gCAAgC;IAChC,KAAK,CAAC,EAAE,GAAG,CAAC;IACZ,6CAA6C;IAC7C,KAAK,CAAC,EAAE,UAAU,EAAE,CAAC;IACrB,+CAA+C;IAC/C,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC9B;AAED,6CAA6C;AAC7C,MAAM,WAAW,UAAU;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;IAChB,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,6EAA6E;AAC7E,MAAM,WAAW,QAAQ;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,CAAC,EAAE,GAAG,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAC/B;AAED,oCAAoC;AACpC,qBAAa,YAAY;IACvB,gCAAgC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,oCAAoC;IACpC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,2BAA2B;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,0CAA0C;IAC1C,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,0BAA0B;IAC1B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBAE5B,IAAI,EAAE;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;KAChC;IAQD,8BAA8B;IAC9B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,YAAY;IAI/C,8BAA8B;IAC9B,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,MAAM,GAAG,YAAY;CAGhD;AAED,wDAAwD;AACxD,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CAChC;AAED,2CAA2C;AAC3C,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,GAAG,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,IAAI,CAAC;CAC7D;AAED,wCAAwC;AACxC,MAAM,MAAM,aAAa,GAAG,CAC1B,GAAG,EAAE,aAAa,EAClB,OAAO,EAAE,aAAa,KACnB,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;AAE1C,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,KAAK,GAAG,OAAO,GAAG,MAAM,GAAG,SAAS,GAAG,WAAW,CAAC;AAEtF,4CAA4C;AAC5C,MAAM,WAAW,YAAY;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,aAAa,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,WAAW,CAAC;IACnB,OAAO,EAAE,OAAO,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;CACnC;AAMD;;;;;;;;;;;;GAYG;AACH,wBAAgB,MAAM,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,EAAE,KAAK,GAAE,WAAoB,IACpE,CAAC,SAAS,aAAa,EAAE,SAAS,CAAC,KAAG,CAAC,CAczD;AAED,sCAAsC;AACtC,wBAAgB,QAAQ,CAAC,EAAE,EAAE,GAAG,GAAG,OAAO,CAEzC;AAED,kDAAkD;AAClD,wBAAgB,eAAe,CAAC,EAAE,EAAE,GAAG,GAAG,YAAY,GAAG,SAAS,CAEjE;AAID;;GAEG;AACH,qBAAa,cAAc;IACzB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAmC;IAE1D,MAAM,CAAC,QAAQ,CAAC,MAAM,EAAE,YAAY,GAAG,IAAI;IAI3C,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,GAAG,SAAS;IAIlD,MAAM,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,EAAE,YAAY,CAAC;IAIvC,MAAM,CAAC,SAAS,IAAI,MAAM,EAAE;IAI5B,MAAM,CAAC,KAAK,IAAI,IAAI;CAGrB;AAID,uCAAuC;AACvC,wBAAgB,UAAU,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAS/D;AAED;;;;;;;GAOG;AACH,wBAAgB,QAAQ,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAc7D;AAED;;;;;;GAMG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAY9D;AAED;;;;;GAKG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAoB/D;AAED;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAkE/D;AAED;;;;;;;GAOG;AACH,wBAAgB,YAAY,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAgDjE;AAED,wDAAwD;AACxD,wBAAgB,WAAW,CAAC,OAAO,EAAE,aAAa,GAAG,YAAY,CAuChE;AAwBD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,QAAQ,CAC5B,OAAO,EAAE,aAAa,EACtB,GAAG,CAAC,EAAE,aAAa,GAClB,OAAO,CAAC,YAAY,CAAC,CA8GvB;AA2LD,wBAAsB,WAAW,CAC/B,OAAO,EAAE,aAAa,EACtB,GAAG,CAAC,EAAE,aAAa,GAClB,OAAO,CAAC,YAAY,CAAC,CA4BvB;AAED,wBAAsB,YAAY,CAChC,OAAO,EAAE,aAAa,EACtB,GAAG,CAAC,EAAE,aAAa,GAClB,OAAO,CAAC,YAAY,CAAC,CAgCvB;AAwFD;;GAEG;AACH,wBAAsB,SAAS,CAC7B,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,aAAa,EACtB,GAAG,CAAC,EAAE,aAAa,GAClB,OAAO,CAAC,YAAY,CAAC,CAwBvB;AAkID,uCAAuC;AACvC,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,aAAa,EAAE,GAAG,EAAE,MAAM,EAAE,YAAY,CAAC,EAAE,GAAG,GAAG,GAAG,CAE7F;AAED,oDAAoD;AACpD,wBAAgB,cAAc,CAAC,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,GAAG,UAAU,EAAE,CAEtF;AAED,yCAAyC;AACzC,wBAAgB,cAAc,CAAC,OAAO,EAAE,aAAa,GAAG,MAAM,CAI7D;AAED,iEAAiE;AACjE,wBAAgB,YAAY,CAAC,OAAO,EAAE,aAAa,GAAG,QAAQ,EAAE,CAE/D;AAED,mDAAmD;AACnD,wBAAgB,0BAA0B,CAAC,MAAM,GAAE,UAAU,EAAO,GAAG,QAAQ,EAAE,CA0BhF;AAED,+CAA+C;AAC/C,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,aAAa,GAAG,MAAM,EAAE,CAEjE;AAED,mDAAmD;AACnD,wBAAgB,aAAa,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,MAAM,EAAE,CAEzD;AAED,wEAAwE;AACxE,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAEjF;AAED,kEAAkE;AAClE,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAUnF;AAED,2EAA2E;AAC3E,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CASpF;AAED,8EAA8E;AAC9E,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,MAAM,EAAE,EAChB,QAAQ,EAAE,MAAM,EAAE,EAClB,IAAI,GAAE,OAAO,GAAG,UAAU,GAAG,WAAqB,GACjD,OAAO,CAIT"}