ai-experiments 0.1.0 → 2.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,198 @@
1
+ /**
2
+ * Core types for AI experimentation
3
+ */
4
+ /**
5
+ * A variant within an experiment
6
+ */
7
+ export interface ExperimentVariant<TConfig = unknown> {
8
+ /** Unique identifier for the variant */
9
+ id: string;
10
+ /** Human-readable name */
11
+ name: string;
12
+ /** Variant configuration */
13
+ config: TConfig;
14
+ /** Weight for weighted random selection (default: 1) */
15
+ weight?: number;
16
+ /** Optional description */
17
+ description?: string;
18
+ }
19
+ /**
20
+ * Configuration for an experiment
21
+ */
22
+ export interface ExperimentConfig<TConfig = unknown, TResult = unknown> {
23
+ /** Unique experiment identifier */
24
+ id: string;
25
+ /** Human-readable name */
26
+ name: string;
27
+ /** Experiment description */
28
+ description?: string;
29
+ /** List of variants to test */
30
+ variants: ExperimentVariant<TConfig>[];
31
+ /** Function to execute for each variant */
32
+ execute: (config: TConfig, context?: ExperimentContext) => Promise<TResult> | TResult;
33
+ /** Optional success metric function */
34
+ metric?: (result: TResult) => number | Promise<number>;
35
+ /** Metadata for the experiment */
36
+ metadata?: Record<string, unknown>;
37
+ }
38
+ /**
39
+ * Context passed to experiment execution
40
+ */
41
+ export interface ExperimentContext {
42
+ /** Experiment ID */
43
+ experimentId: string;
44
+ /** Variant ID */
45
+ variantId: string;
46
+ /** Run ID (unique per execution) */
47
+ runId: string;
48
+ /** Timestamp when execution started */
49
+ startedAt: Date;
50
+ /** Additional context data */
51
+ data?: Record<string, unknown>;
52
+ }
53
+ /**
54
+ * Result of executing an experiment variant
55
+ */
56
+ export interface ExperimentResult<TResult = unknown> {
57
+ /** Experiment ID */
58
+ experimentId: string;
59
+ /** Variant ID */
60
+ variantId: string;
61
+ /** Variant name */
62
+ variantName: string;
63
+ /** Run ID */
64
+ runId: string;
65
+ /** Execution result */
66
+ result: TResult;
67
+ /** Computed metric value (if metric function provided) */
68
+ metricValue?: number;
69
+ /** Execution duration in milliseconds */
70
+ duration: number;
71
+ /** Timestamp when execution started */
72
+ startedAt: Date;
73
+ /** Timestamp when execution completed */
74
+ completedAt: Date;
75
+ /** Error if execution failed */
76
+ error?: Error;
77
+ /** Success flag */
78
+ success: boolean;
79
+ /** Additional metadata */
80
+ metadata?: Record<string, unknown>;
81
+ }
82
+ /**
83
+ * Summary of experiment results across all variants
84
+ */
85
+ export interface ExperimentSummary<TResult = unknown> {
86
+ /** Experiment ID */
87
+ experimentId: string;
88
+ /** Experiment name */
89
+ experimentName: string;
90
+ /** Results for each variant */
91
+ results: ExperimentResult<TResult>[];
92
+ /** Best performing variant (by metric) */
93
+ bestVariant?: {
94
+ variantId: string;
95
+ variantName: string;
96
+ metricValue: number;
97
+ };
98
+ /** Total execution duration */
99
+ totalDuration: number;
100
+ /** Number of successful runs */
101
+ successCount: number;
102
+ /** Number of failed runs */
103
+ failureCount: number;
104
+ /** Timestamp when experiment started */
105
+ startedAt: Date;
106
+ /** Timestamp when experiment completed */
107
+ completedAt: Date;
108
+ }
109
+ /**
110
+ * Options for running an experiment
111
+ */
112
+ export interface RunExperimentOptions {
113
+ /** Run variants in parallel (default: true) */
114
+ parallel?: boolean;
115
+ /** Maximum concurrent executions (default: unlimited) */
116
+ maxConcurrency?: number;
117
+ /** Stop on first error (default: false) */
118
+ stopOnError?: boolean;
119
+ /** Custom context data */
120
+ context?: Record<string, unknown>;
121
+ /** Event callbacks */
122
+ onVariantStart?: (variantId: string, variantName: string) => void;
123
+ onVariantComplete?: (result: ExperimentResult) => void;
124
+ onVariantError?: (variantId: string, error: Error) => void;
125
+ }
126
+ /**
127
+ * Parameters for cartesian product generation
128
+ */
129
+ export type CartesianParams = Record<string, unknown[]>;
130
+ /**
131
+ * Result of cartesian product - array of parameter combinations
132
+ */
133
+ export type CartesianResult<T extends CartesianParams> = Array<{
134
+ [K in keyof T]: T[K][number];
135
+ }>;
136
+ /**
137
+ * Decision options
138
+ */
139
+ export interface DecideOptions<T> {
140
+ /** Options to choose from */
141
+ options: T[];
142
+ /** Scoring function for each option */
143
+ score: (option: T) => number | Promise<number>;
144
+ /** Context or prompt for decision making */
145
+ context?: string;
146
+ /** Whether to return all options sorted by score (default: false) */
147
+ returnAll?: boolean;
148
+ }
149
+ /**
150
+ * Result of a decision
151
+ */
152
+ export interface DecisionResult<T> {
153
+ /** The selected option */
154
+ selected: T;
155
+ /** Score of the selected option */
156
+ score: number;
157
+ /** All options with their scores (if returnAll was true) */
158
+ allOptions?: Array<{
159
+ option: T;
160
+ score: number;
161
+ }>;
162
+ }
163
+ /**
164
+ * Tracking event types
165
+ */
166
+ export type TrackingEventType = 'experiment.start' | 'experiment.complete' | 'variant.start' | 'variant.complete' | 'variant.error' | 'metric.computed' | 'decision.made';
167
+ /**
168
+ * Tracking event
169
+ */
170
+ export interface TrackingEvent {
171
+ /** Event type */
172
+ type: TrackingEventType;
173
+ /** Timestamp */
174
+ timestamp: Date;
175
+ /** Event data */
176
+ data: Record<string, unknown>;
177
+ }
178
+ /**
179
+ * Tracking backend interface
180
+ */
181
+ export interface TrackingBackend {
182
+ /** Track an event */
183
+ track(event: TrackingEvent): void | Promise<void>;
184
+ /** Flush pending events */
185
+ flush?(): void | Promise<void>;
186
+ }
187
+ /**
188
+ * Options for tracking configuration
189
+ */
190
+ export interface TrackingOptions {
191
+ /** Custom tracking backend */
192
+ backend?: TrackingBackend;
193
+ /** Whether tracking is enabled (default: true) */
194
+ enabled?: boolean;
195
+ /** Additional metadata to include with all events */
196
+ metadata?: Record<string, unknown>;
197
+ }
198
+ //# sourceMappingURL=types.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,WAAW,iBAAiB,CAAC,OAAO,GAAG,OAAO;IAClD,wCAAwC;IACxC,EAAE,EAAE,MAAM,CAAA;IACV,0BAA0B;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,4BAA4B;IAC5B,MAAM,EAAE,OAAO,CAAA;IACf,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,2BAA2B;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,GAAG,OAAO;IACpE,mCAAmC;IACnC,EAAE,EAAE,MAAM,CAAA;IACV,0BAA0B;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,6BAA6B;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,+BAA+B;IAC/B,QAAQ,EAAE,iBAAiB,CAAC,OAAO,CAAC,EAAE,CAAA;IACtC,2CAA2C;IAC3C,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,iBAAiB,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;IACrF,uCAAuC;IACvC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IACtD,kCAAkC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,oBAAoB;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAA;IACb,uCAAuC;IACvC,SAAS,EAAE,IAAI,CAAA;IACf,8BAA8B;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB,CAAC,OAAO,GAAG,OAAO;IACjD,oBAAoB;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,mBAAmB;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa;IACb,KAAK,EAAE,MAAM,CAAA;IACb,uBAAuB;IACvB,MAAM,EAAE,OAAO,CAAA;IACf,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,yCAAyC;IACzC,QAAQ,EAAE,MAAM,CAAA;IAChB,uCAAuC;IACvC,SAAS,EAAE,IAAI,CAAA;IACf,yCAAyC;IACzC,WAAW,EAAE,IAAI,CAAA;IACjB,gCAAgC;IAChC,KAAK,CAAC,EAAE,KAAK,CAAA;IACb,mBAAmB;IACnB,OAAO,EAAE,OAAO,CAAA;IAChB,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB,CAAC,OAAO,GAAG,OAAO;IAClD,oBAAoB;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,sBAAsB;IACtB,cAAc,EAAE,MAAM,CAAA;IACtB,+BAA+B;IAC/B,OAAO,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAA;IACpC,0CAA0C;IAC1C,WAAW,CAAC,EAAE;QACZ,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;QACnB,WAAW,EAAE,MAAM,CAAA;KACpB,CAAA;IACD,+BAA+B;IAC/B,aAAa,EAAE,MAAM,CAAA;IACrB,gCAAgC;IAChC,YAAY,EAAE,MAAM,CAAA;IACpB,4BAA4B;IAC5B,YAAY,EAAE,MAAM,CAAA;IACpB,wCAAwC;IACxC,SAAS,EAAE,IAAI,CAAA;IACf,0CAA0C;IAC1C,WAAW,EAAE,IAAI,CAAA;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,yDAAyD;IACzD,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,2CAA2C;IAC3C,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,0BAA0B;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACjC,sBAAsB;IACtB,cAAc,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,KAAK,IAAI,CAAA;IACjE,iBAAiB,CAAC,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,IAAI,CAAA;IACtD,cAAc,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,KAAK,IAAI,CAAA;CAC3D;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAA;AAEvD;;GAEG;AACH,MAAM,MAAM,eAAe,CAAC,CAAC,SAAS,eAAe,IAAI,KAAK,CAAC;KAC5D,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;CAC7B,CAAC,CAAA;AAEF;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,6BAA6B;IAC7B,OAAO,EAAE,CAAC,EAAE,CAAA;IACZ,uCAAuC;IACvC,KAAK,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IAC9C,4CAA4C;IAC5C,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,qEAAqE;IACrE,SAAS,CAAC,EAAE,OAAO,CAAA;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,0BAA0B;IAC1B,QAAQ,EAAE,CAAC,CAAA;IACX,mCAAmC;IACnC,KAAK,EAAE,MAAM,CAAA;IACb,4DAA4D;IAC5D,UAAU,CAAC,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,CAAC,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CACjD;AAED;;GAEG;AACH,MAAM,MAAM,iBAAiB,GACzB,kBAAkB,GAClB,qBAAqB,GACrB,eAAe,GACf,kBAAkB,GAClB,eAAe,GACf,iBAAiB,GACjB,eAAe,CAAA;AAEnB;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,iBAAiB;IACjB,IAAI,EAAE,iBAAiB,CAAA;IACvB,gBAAgB;IAChB,SAAS,EAAE,IAAI,CAAA;IACf,iBAAiB;IACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qBAAqB;IACrB,KAAK,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACjD,2BAA2B;IAC3B,KAAK,CAAC,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,8BAA8B;IAC9B,OAAO,CAAC,EAAE,eAAe,CAAA;IACzB,kDAAkD;IAClD,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,qDAAqD;IACrD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC"}
package/dist/types.js ADDED
@@ -0,0 +1,5 @@
1
+ /**
2
+ * Core types for AI experimentation
3
+ */
4
+ export {};
5
+ //# sourceMappingURL=types.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG"}
package/examples.ts ADDED
@@ -0,0 +1,261 @@
1
+ /**
2
+ * Usage examples for ai-experiments
3
+ *
4
+ * These examples demonstrate the core APIs and patterns.
5
+ */
6
+
7
+ import {
8
+ Experiment,
9
+ cartesian,
10
+ decide,
11
+ track,
12
+ configureTracking,
13
+ createMemoryBackend,
14
+ type ExperimentConfig,
15
+ } from './src/index.js'
16
+
17
+ // ============================================================================
18
+ // Example 1: Simple A/B Experiment
19
+ // ============================================================================
20
+
21
+ async function example1_simpleExperiment() {
22
+ console.log('\n=== Example 1: Simple A/B Experiment ===\n')
23
+
24
+ const results = await Experiment({
25
+ id: 'greeting-test',
26
+ name: 'Greeting Message Test',
27
+ description: 'Test different greeting messages',
28
+ variants: [
29
+ {
30
+ id: 'formal',
31
+ name: 'Formal Greeting',
32
+ config: { message: 'Good day, sir.' },
33
+ },
34
+ {
35
+ id: 'casual',
36
+ name: 'Casual Greeting',
37
+ config: { message: 'Hey there!' },
38
+ },
39
+ {
40
+ id: 'friendly',
41
+ name: 'Friendly Greeting',
42
+ config: { message: 'Hello friend!' },
43
+ },
44
+ ],
45
+ execute: async (config) => {
46
+ // Simulate some work
47
+ await new Promise((resolve) => setTimeout(resolve, 100))
48
+ return {
49
+ message: config.message,
50
+ wordCount: config.message.split(' ').length,
51
+ charCount: config.message.length,
52
+ }
53
+ },
54
+ metric: (result) => {
55
+ // Score based on message length (prefer shorter)
56
+ return 1 / result.charCount
57
+ },
58
+ })
59
+
60
+ console.log('Experiment Results:')
61
+ console.log('- Best Variant:', results.bestVariant)
62
+ console.log('- Success Rate:', `${results.successCount}/${results.results.length}`)
63
+ console.log('- Total Duration:', `${results.totalDuration}ms`)
64
+ console.log('\nAll Results:')
65
+ results.results.forEach((r) => {
66
+ console.log(
67
+ ` ${r.variantName}: metric=${r.metricValue?.toFixed(4)}, duration=${r.duration}ms`
68
+ )
69
+ })
70
+ }
71
+
72
+ // ============================================================================
73
+ // Example 2: Parameter Grid with Cartesian Product
74
+ // ============================================================================
75
+
76
+ async function example2_parameterGrid() {
77
+ console.log('\n=== Example 2: Parameter Grid Experiment ===\n')
78
+
79
+ // Generate all combinations of parameters
80
+ const parameterGrid = cartesian({
81
+ temperature: [0.3, 0.7, 1.0],
82
+ maxTokens: [50, 100],
83
+ topP: [0.9, 1.0],
84
+ })
85
+
86
+ console.log(`Generated ${parameterGrid.length} parameter combinations:`)
87
+ parameterGrid.slice(0, 5).forEach((params, i) => {
88
+ console.log(` ${i + 1}.`, params)
89
+ })
90
+ console.log(' ...')
91
+
92
+ // Create variants from parameter combinations
93
+ const variants = parameterGrid.map((params, i) => ({
94
+ id: `variant-${i}`,
95
+ name: `T=${params.temperature} max=${params.maxTokens} topP=${params.topP}`,
96
+ config: params,
97
+ }))
98
+
99
+ const results = await Experiment({
100
+ id: 'parameter-sweep',
101
+ name: 'Parameter Sweep Experiment',
102
+ variants,
103
+ execute: async (config) => {
104
+ // Simulate AI generation with different parameters
105
+ await new Promise((resolve) => setTimeout(resolve, 50))
106
+ const score = Math.random() * config.temperature + config.maxTokens / 1000
107
+ return { score, config }
108
+ },
109
+ metric: (result) => result.score,
110
+ })
111
+
112
+ console.log(`\nBest parameters: ${results.bestVariant?.variantName}`)
113
+ console.log(`Best score: ${results.bestVariant?.metricValue.toFixed(4)}`)
114
+ }
115
+
116
+ // ============================================================================
117
+ // Example 3: Decision Making
118
+ // ============================================================================
119
+
120
+ async function example3_decisionMaking() {
121
+ console.log('\n=== Example 3: Decision Making ===\n')
122
+
123
+ // Simple decision with sync scoring
124
+ const result1 = await decide({
125
+ options: ['apple', 'banana', 'orange'],
126
+ score: (fruit) => {
127
+ const prices: Record<string, number> = { apple: 1.5, banana: 0.5, orange: 2.0 }
128
+ return 1 / prices[fruit] // Lower price = higher score
129
+ },
130
+ context: 'Choosing fruit based on value',
131
+ })
132
+
133
+ console.log('Decision 1 (best value):')
134
+ console.log(` Selected: ${result1.selected}`)
135
+ console.log(` Score: ${result1.score.toFixed(4)}`)
136
+
137
+ // Decision with all options returned
138
+ const result2 = await decide({
139
+ options: ['fast', 'accurate', 'balanced'],
140
+ score: async (approach) => {
141
+ const scores: Record<string, number> = { fast: 0.7, accurate: 0.85, balanced: 0.9 }
142
+ return scores[approach]
143
+ },
144
+ returnAll: true,
145
+ })
146
+
147
+ console.log('\nDecision 2 (with all options):')
148
+ console.log(` Selected: ${result2.selected}`)
149
+ result2.allOptions?.forEach((opt) => {
150
+ console.log(` - ${opt.option}: ${opt.score}`)
151
+ })
152
+ }
153
+
154
+ // ============================================================================
155
+ // Example 4: Event Tracking
156
+ // ============================================================================
157
+
158
+ async function example4_eventTracking() {
159
+ console.log('\n=== Example 4: Event Tracking ===\n')
160
+
161
+ // Create a memory backend to collect events
162
+ const backend = createMemoryBackend()
163
+ configureTracking({ backend })
164
+
165
+ // Run a small experiment
166
+ await Experiment({
167
+ id: 'tracked-experiment',
168
+ name: 'Tracked Experiment',
169
+ variants: [
170
+ { id: 'v1', name: 'Variant 1', config: { value: 1 } },
171
+ { id: 'v2', name: 'Variant 2', config: { value: 2 } },
172
+ ],
173
+ execute: async (config) => ({ result: config.value * 2 }),
174
+ metric: (result) => result.result,
175
+ })
176
+
177
+ // Get all tracked events
178
+ const events = backend.getEvents()
179
+ console.log(`Tracked ${events.length} events:`)
180
+ events.forEach((event) => {
181
+ console.log(` [${event.type}]`, JSON.stringify(event.data).slice(0, 80))
182
+ })
183
+
184
+ // Reset tracking to default
185
+ configureTracking({ enabled: true })
186
+ }
187
+
188
+ // ============================================================================
189
+ // Example 5: Advanced - Sequential vs Parallel Execution
190
+ // ============================================================================
191
+
192
+ async function example5_executionModes() {
193
+ console.log('\n=== Example 5: Sequential vs Parallel Execution ===\n')
194
+
195
+ const variants = [
196
+ { id: 'v1', name: 'Variant 1', config: { delay: 100 } },
197
+ { id: 'v2', name: 'Variant 2', config: { delay: 150 } },
198
+ { id: 'v3', name: 'Variant 3', config: { delay: 200 } },
199
+ ]
200
+
201
+ const execute = async (config: { delay: number }) => {
202
+ await new Promise((resolve) => setTimeout(resolve, config.delay))
203
+ return { completed: true }
204
+ }
205
+
206
+ // Parallel execution (default)
207
+ const start1 = Date.now()
208
+ await Experiment({
209
+ id: 'parallel-test',
210
+ name: 'Parallel Execution',
211
+ variants,
212
+ execute,
213
+ })
214
+ const parallel_duration = Date.now() - start1
215
+
216
+ // Sequential execution
217
+ const start2 = Date.now()
218
+ await Experiment(
219
+ {
220
+ id: 'sequential-test',
221
+ name: 'Sequential Execution',
222
+ variants,
223
+ execute,
224
+ },
225
+ { parallel: false }
226
+ )
227
+ const sequential_duration = Date.now() - start2
228
+
229
+ console.log(`Parallel execution: ${parallel_duration}ms`)
230
+ console.log(`Sequential execution: ${sequential_duration}ms`)
231
+ console.log(
232
+ `Speedup: ${(sequential_duration / parallel_duration).toFixed(2)}x`
233
+ )
234
+ }
235
+
236
+ // ============================================================================
237
+ // Run all examples
238
+ // ============================================================================
239
+
240
+ async function main() {
241
+ // Disable verbose tracking for examples
242
+ configureTracking({ enabled: false })
243
+
244
+ try {
245
+ await example1_simpleExperiment()
246
+ await example2_parameterGrid()
247
+ await example3_decisionMaking()
248
+ await example4_eventTracking()
249
+ await example5_executionModes()
250
+
251
+ console.log('\n✅ All examples completed successfully!\n')
252
+ } catch (error) {
253
+ console.error('\n❌ Error running examples:', error)
254
+ process.exit(1)
255
+ }
256
+ }
257
+
258
+ // Run if this file is executed directly
259
+ if (import.meta.url === `file://${process.argv[1]}`) {
260
+ main()
261
+ }
package/package.json CHANGED
@@ -1,50 +1,32 @@
1
1
  {
2
2
  "name": "ai-experiments",
3
- "version": "0.1.0",
4
- "description": "A minimalistic experiment runner for AI tasks",
5
- "main": "./dist/index.js",
6
- "module": "./dist/index.mjs",
7
- "types": "./dist/index.d.ts",
3
+ "version": "2.0.2",
4
+ "description": "AI-powered experimentation primitives for testing and evaluating models",
5
+ "type": "module",
6
+ "main": "dist/index.js",
7
+ "types": "dist/index.d.ts",
8
8
  "exports": {
9
9
  ".": {
10
- "require": "./dist/index.js",
11
- "import": "./dist/index.mjs",
10
+ "import": "./dist/index.js",
12
11
  "types": "./dist/index.d.ts"
13
12
  }
14
13
  },
15
- "files": [
16
- "dist"
17
- ],
14
+ "scripts": {
15
+ "build": "tsc",
16
+ "dev": "tsc --watch",
17
+ "test": "vitest",
18
+ "typecheck": "tsc --noEmit",
19
+ "lint": "eslint .",
20
+ "clean": "rm -rf dist"
21
+ },
22
+ "dependencies": {
23
+ "ai-functions": "2.0.2",
24
+ "rpc.do": "^0.1.0"
25
+ },
18
26
  "keywords": [
19
27
  "ai",
20
28
  "experiments",
21
- "testing",
22
- "vitest"
29
+ "primitives"
23
30
  ],
24
- "dependencies": {
25
- "vitest": "^1.1.0"
26
- },
27
- "devDependencies": {
28
- "tsup": "^8.0.1",
29
- "typescript": "^5.2.2",
30
- "prettier": "^3.4.2",
31
- "@types/node": "^20.17.0"
32
- },
33
- "repository": {
34
- "type": "git",
35
- "url": "https://github.com/drivly/primitives.org.ai.git"
36
- },
37
- "bugs": {
38
- "url": "https://github.com/drivly/primitives.org.ai/issues"
39
- },
40
- "homepage": "https://primitives.org.ai",
41
- "license": "MIT",
42
- "scripts": {
43
- "build": "tsup",
44
- "dev": "tsup --watch",
45
- "test": "vitest run",
46
- "test:watch": "vitest",
47
- "typecheck": "tsc --noEmit",
48
- "format": "prettier --write 'src/**/*.{ts,tsx,js,jsx}'"
49
- }
50
- }
31
+ "license": "MIT"
32
+ }