orchestrated 0.1.0 → 0.1.2

package/index.d.ts

@@ -0,0 +1,243 @@
+/**
+ * Orchestrated - LLM Evaluation Framework
+ *
+ * A comprehensive evaluation framework for LLM applications with batch processing,
+ * data sources, and multi-backend export capabilities.
+ *
+ * @packageDocumentation
+ */
+
+import type { Score, Scorer } from "autoevals";
+
+// Core Evaluation API
+export declare function Eval(
+	name: string,
+	config: EvalConfig,
+	options?: EvalOptions
+): Promise<EvalSummary>;
+
+export interface EvalConfig {
+	data: EvalData;
+	task?: TaskFunction;
+	scores: (string | Scorer)[];
+	ctx?: any;
+}
+
+export type EvalData =
+	| Array<Record<string, any>>
+	| DataSourceDefinition
+	| (() => Promise<Array<Record<string, any>>>);
+
+export interface DataSourceDefinition {
+	type: string;
+	config?: any;
+}
+
+export interface EvalOptions {
+	reporters?: Reporter[];
+	exporters?: any[];
+}
+
+export interface EvalResult {
+	input: any;
+	output?: any;
+	expected?: any;
+	scores: Record<string, Score>;
+	error?: Error;
+	tags?: string[];
+}
+
+export interface EvalSummary {
+	name: string;
+	results: EvalResult[];
+	summary: {
+		total: number;
+		passed: number;
+		failed: number;
+		scores: Record<string, ScoreSummary>;
+	};
+	hasPendingBatch?: boolean;
+}
+
+export interface ScoreSummary {
+	name: string;
+	mean: number;
+	median: number;
+	min: number;
+	max: number;
+	p10: number;
+	p25: number;
+	p75: number;
+	p90: number;
+	stddev: number;
+	count: number;
+}
+
+export type TaskFunction = (
+	input: any,
+	ctx?: any
+) => Promise<any> | any;
+
+// Evaluation Registry & Management
+export declare function registerEvaluation(promise: Promise<any>): void;
+export declare function clearEvaluations(): void;
+export declare function getRunningEvaluationCount(): number;
+export declare function waitForEvaluations(): Promise<void>;
+
+// Data Sources
+export declare function interactions(
+	options?: InteractionsDatasetOptions
+): DataSourceDefinition;
+
+export interface InteractionsDatasetOptions {
+	tenantId?: string;
+	serviceName?: string;
+	environment?: string;
+	month?: string;
+	startDate?: string;
+	endDate?: string;
+}
+
+// Project API (Scorers)
+export declare const projects: {
+	create(options?: ProjectOptions): Project;
+};
+
+export interface ProjectOptions {
+	tenantId?: string;
+	serviceName?: string;
+	environment?: string;
+}
+
+export declare class Project {
+	scorers: ScorerRegistry;
+}
+
+export declare class ScorerRegistry {
+	create(config: ScorerConfig): Scorer;
+}
+
+export interface BaseScorerConfig {
+	name: string;
+	slug: string;
+	description: string;
+	parameters: any; // Zod schema
+	metadata?: Record<string, any>;
+}
+
+export interface TypedScorerConfig extends BaseScorerConfig {
+	promptTemplate: string;
+	choiceScores: Record<string, number>;
+	model?: string;
+	useCoT?: boolean;
+	temperature?: number;
+}
+
+export interface CustomScorerConfig extends BaseScorerConfig {
+	handler: (args: any) => Promise<Score>;
+}
+
+export type ScorerConfig = TypedScorerConfig | CustomScorerConfig;
+
+// State Management
+export declare function initState(
+	partial?: PartialEvalState,
+	skipAuth?: boolean
+): Promise<void>;
+
+export declare function getState(): Readonly<EvalState>;
+export declare function resetState(): void;
+export declare function isStateInitialized(): boolean;
+
+export interface EvalState {
+	apiUrl: string;
+	tenantId: string;
+	serviceName: string;
+	loggedInUser: string | null;
+	accessToken: string | null;
+	apiKey: string | null;
+	environment: string;
+	appUrl: string;
+	appClientId: string;
+	otelEndpoint: string | null;
+	sendNoLogs: boolean;
+	lazyLoad: boolean;
+	awsAccessKeyId: string | null;
+	awsSecretAccessKey: string | null;
+	awsSessionToken: string | null;
+	disableBundleCache: boolean;
+}
+
+export type PartialEvalState = Partial<EvalState>;
+
+// Reporters
+export declare const legacyReporter: Reporter;
+
+export interface Reporter {
+	onStart?: (ctx: any) => void | Promise<void>;
+	onResult?: (ctx: any, result: EvalResult) => void | Promise<void>;
+	onComplete?: (ctx: any, summary: EvalSummary) => void | Promise<void>;
+}
+
+// Utilities
+export declare const colors: {
+	gray: (text: string) => string;
+	green: (text: string) => string;
+	red: (text: string) => string;
+	yellow: (text: string) => string;
+	blue: (text: string) => string;
+	magenta: (text: string) => string;
+	cyan: (text: string) => string;
+	white: (text: string) => string;
+	bold: (text: string) => string;
+};
+
+export declare const iso: {
+	now: () => string;
+};
+
+// Tracing
+export declare function traced<T extends (...args: any[]) => any>(
+	fn: T,
+	options?: { name?: string }
+): T;
+
+// Serialization (for advanced users)
+export interface SerializableScorerDefinition {
+	type: "prompt";
+	name: string;
+	slug?: string;
+	description?: string;
+	schema: any;
+	promptTemplate: string;
+	choiceScores: Record<string, number>;
+	model?: string;
+	useCoT?: boolean;
+	temperature?: number;
+	metadata?: Record<string, any>;
+	fingerprint?: string;
+}
+
+export interface SerializableCustomScorer {
+	type: "custom_scorer";
+	name: string;
+	slug: string;
+	description: string;
+	schema: any;
+	handler: any;
+	metadata?: Record<string, any>;
+	fingerprint?: string;
+}
+
+export interface SerializableEvaluation {
+	slug: string;
+	name: string;
+	data: any;
+	task?: any;
+	scorers: any[];
+	fingerprint?: string;
+	options?: any;
+}
+
+// Re-export common types from autoevals for convenience
+export type { Score, Scorer } from "autoevals";