ai-experiments 0.1.0 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +5 -0
- package/CHANGELOG.md +15 -0
- package/README.md +306 -91
- package/dist/cartesian.d.ts +140 -0
- package/dist/cartesian.d.ts.map +1 -0
- package/dist/cartesian.js +216 -0
- package/dist/cartesian.js.map +1 -0
- package/dist/decide.d.ts +152 -0
- package/dist/decide.d.ts.map +1 -0
- package/dist/decide.js +329 -0
- package/dist/decide.js.map +1 -0
- package/dist/experiment.d.ts +53 -0
- package/dist/experiment.d.ts.map +1 -0
- package/dist/experiment.js +292 -0
- package/dist/experiment.js.map +1 -0
- package/dist/index.d.ts +14 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +19 -0
- package/dist/index.js.map +1 -0
- package/dist/tracking.d.ts +159 -0
- package/dist/tracking.d.ts.map +1 -0
- package/dist/tracking.js +310 -0
- package/dist/tracking.js.map +1 -0
- package/dist/types.d.ts +198 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +5 -0
- package/dist/types.js.map +1 -0
- package/examples.ts +261 -0
- package/package.json +21 -39
- package/src/cartesian.ts +259 -0
- package/src/decide.ts +398 -0
- package/src/experiment.ts +358 -0
- package/src/index.ts +44 -0
- package/src/tracking.ts +339 -0
- package/src/types.ts +215 -0
- package/tsconfig.json +9 -0
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,198 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Core types for AI experimentation
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* A variant within an experiment
|
|
6
|
+
*/
|
|
7
|
+
export interface ExperimentVariant<TConfig = unknown> {
|
|
8
|
+
/** Unique identifier for the variant */
|
|
9
|
+
id: string;
|
|
10
|
+
/** Human-readable name */
|
|
11
|
+
name: string;
|
|
12
|
+
/** Variant configuration */
|
|
13
|
+
config: TConfig;
|
|
14
|
+
/** Weight for weighted random selection (default: 1) */
|
|
15
|
+
weight?: number;
|
|
16
|
+
/** Optional description */
|
|
17
|
+
description?: string;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Configuration for an experiment
|
|
21
|
+
*/
|
|
22
|
+
export interface ExperimentConfig<TConfig = unknown, TResult = unknown> {
|
|
23
|
+
/** Unique experiment identifier */
|
|
24
|
+
id: string;
|
|
25
|
+
/** Human-readable name */
|
|
26
|
+
name: string;
|
|
27
|
+
/** Experiment description */
|
|
28
|
+
description?: string;
|
|
29
|
+
/** List of variants to test */
|
|
30
|
+
variants: ExperimentVariant<TConfig>[];
|
|
31
|
+
/** Function to execute for each variant */
|
|
32
|
+
execute: (config: TConfig, context?: ExperimentContext) => Promise<TResult> | TResult;
|
|
33
|
+
/** Optional success metric function */
|
|
34
|
+
metric?: (result: TResult) => number | Promise<number>;
|
|
35
|
+
/** Metadata for the experiment */
|
|
36
|
+
metadata?: Record<string, unknown>;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Context passed to experiment execution
|
|
40
|
+
*/
|
|
41
|
+
export interface ExperimentContext {
|
|
42
|
+
/** Experiment ID */
|
|
43
|
+
experimentId: string;
|
|
44
|
+
/** Variant ID */
|
|
45
|
+
variantId: string;
|
|
46
|
+
/** Run ID (unique per execution) */
|
|
47
|
+
runId: string;
|
|
48
|
+
/** Timestamp when execution started */
|
|
49
|
+
startedAt: Date;
|
|
50
|
+
/** Additional context data */
|
|
51
|
+
data?: Record<string, unknown>;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Result of executing an experiment variant
|
|
55
|
+
*/
|
|
56
|
+
export interface ExperimentResult<TResult = unknown> {
|
|
57
|
+
/** Experiment ID */
|
|
58
|
+
experimentId: string;
|
|
59
|
+
/** Variant ID */
|
|
60
|
+
variantId: string;
|
|
61
|
+
/** Variant name */
|
|
62
|
+
variantName: string;
|
|
63
|
+
/** Run ID */
|
|
64
|
+
runId: string;
|
|
65
|
+
/** Execution result */
|
|
66
|
+
result: TResult;
|
|
67
|
+
/** Computed metric value (if metric function provided) */
|
|
68
|
+
metricValue?: number;
|
|
69
|
+
/** Execution duration in milliseconds */
|
|
70
|
+
duration: number;
|
|
71
|
+
/** Timestamp when execution started */
|
|
72
|
+
startedAt: Date;
|
|
73
|
+
/** Timestamp when execution completed */
|
|
74
|
+
completedAt: Date;
|
|
75
|
+
/** Error if execution failed */
|
|
76
|
+
error?: Error;
|
|
77
|
+
/** Success flag */
|
|
78
|
+
success: boolean;
|
|
79
|
+
/** Additional metadata */
|
|
80
|
+
metadata?: Record<string, unknown>;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Summary of experiment results across all variants
|
|
84
|
+
*/
|
|
85
|
+
export interface ExperimentSummary<TResult = unknown> {
|
|
86
|
+
/** Experiment ID */
|
|
87
|
+
experimentId: string;
|
|
88
|
+
/** Experiment name */
|
|
89
|
+
experimentName: string;
|
|
90
|
+
/** Results for each variant */
|
|
91
|
+
results: ExperimentResult<TResult>[];
|
|
92
|
+
/** Best performing variant (by metric) */
|
|
93
|
+
bestVariant?: {
|
|
94
|
+
variantId: string;
|
|
95
|
+
variantName: string;
|
|
96
|
+
metricValue: number;
|
|
97
|
+
};
|
|
98
|
+
/** Total execution duration */
|
|
99
|
+
totalDuration: number;
|
|
100
|
+
/** Number of successful runs */
|
|
101
|
+
successCount: number;
|
|
102
|
+
/** Number of failed runs */
|
|
103
|
+
failureCount: number;
|
|
104
|
+
/** Timestamp when experiment started */
|
|
105
|
+
startedAt: Date;
|
|
106
|
+
/** Timestamp when experiment completed */
|
|
107
|
+
completedAt: Date;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Options for running an experiment
|
|
111
|
+
*/
|
|
112
|
+
export interface RunExperimentOptions {
|
|
113
|
+
/** Run variants in parallel (default: true) */
|
|
114
|
+
parallel?: boolean;
|
|
115
|
+
/** Maximum concurrent executions (default: unlimited) */
|
|
116
|
+
maxConcurrency?: number;
|
|
117
|
+
/** Stop on first error (default: false) */
|
|
118
|
+
stopOnError?: boolean;
|
|
119
|
+
/** Custom context data */
|
|
120
|
+
context?: Record<string, unknown>;
|
|
121
|
+
/** Event callbacks */
|
|
122
|
+
onVariantStart?: (variantId: string, variantName: string) => void;
|
|
123
|
+
onVariantComplete?: (result: ExperimentResult) => void;
|
|
124
|
+
onVariantError?: (variantId: string, error: Error) => void;
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Parameters for cartesian product generation
|
|
128
|
+
*/
|
|
129
|
+
export type CartesianParams = Record<string, unknown[]>;
|
|
130
|
+
/**
|
|
131
|
+
* Result of cartesian product - array of parameter combinations
|
|
132
|
+
*/
|
|
133
|
+
export type CartesianResult<T extends CartesianParams> = Array<{
|
|
134
|
+
[K in keyof T]: T[K][number];
|
|
135
|
+
}>;
|
|
136
|
+
/**
|
|
137
|
+
* Decision options
|
|
138
|
+
*/
|
|
139
|
+
export interface DecideOptions<T> {
|
|
140
|
+
/** Options to choose from */
|
|
141
|
+
options: T[];
|
|
142
|
+
/** Scoring function for each option */
|
|
143
|
+
score: (option: T) => number | Promise<number>;
|
|
144
|
+
/** Context or prompt for decision making */
|
|
145
|
+
context?: string;
|
|
146
|
+
/** Whether to return all options sorted by score (default: false) */
|
|
147
|
+
returnAll?: boolean;
|
|
148
|
+
}
|
|
149
|
+
/**
|
|
150
|
+
* Result of a decision
|
|
151
|
+
*/
|
|
152
|
+
export interface DecisionResult<T> {
|
|
153
|
+
/** The selected option */
|
|
154
|
+
selected: T;
|
|
155
|
+
/** Score of the selected option */
|
|
156
|
+
score: number;
|
|
157
|
+
/** All options with their scores (if returnAll was true) */
|
|
158
|
+
allOptions?: Array<{
|
|
159
|
+
option: T;
|
|
160
|
+
score: number;
|
|
161
|
+
}>;
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* Tracking event types
|
|
165
|
+
*/
|
|
166
|
+
export type TrackingEventType = 'experiment.start' | 'experiment.complete' | 'variant.start' | 'variant.complete' | 'variant.error' | 'metric.computed' | 'decision.made';
|
|
167
|
+
/**
|
|
168
|
+
* Tracking event
|
|
169
|
+
*/
|
|
170
|
+
export interface TrackingEvent {
|
|
171
|
+
/** Event type */
|
|
172
|
+
type: TrackingEventType;
|
|
173
|
+
/** Timestamp */
|
|
174
|
+
timestamp: Date;
|
|
175
|
+
/** Event data */
|
|
176
|
+
data: Record<string, unknown>;
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* Tracking backend interface
|
|
180
|
+
*/
|
|
181
|
+
export interface TrackingBackend {
|
|
182
|
+
/** Track an event */
|
|
183
|
+
track(event: TrackingEvent): void | Promise<void>;
|
|
184
|
+
/** Flush pending events */
|
|
185
|
+
flush?(): void | Promise<void>;
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Options for tracking configuration
|
|
189
|
+
*/
|
|
190
|
+
export interface TrackingOptions {
|
|
191
|
+
/** Custom tracking backend */
|
|
192
|
+
backend?: TrackingBackend;
|
|
193
|
+
/** Whether tracking is enabled (default: true) */
|
|
194
|
+
enabled?: boolean;
|
|
195
|
+
/** Additional metadata to include with all events */
|
|
196
|
+
metadata?: Record<string, unknown>;
|
|
197
|
+
}
|
|
198
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;GAEG;AACH,MAAM,WAAW,iBAAiB,CAAC,OAAO,GAAG,OAAO;IAClD,wCAAwC;IACxC,EAAE,EAAE,MAAM,CAAA;IACV,0BAA0B;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,4BAA4B;IAC5B,MAAM,EAAE,OAAO,CAAA;IACf,wDAAwD;IACxD,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,2BAA2B;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,GAAG,OAAO;IACpE,mCAAmC;IACnC,EAAE,EAAE,MAAM,CAAA;IACV,0BAA0B;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,6BAA6B;IAC7B,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,+BAA+B;IAC/B,QAAQ,EAAE,iBAAiB,CAAC,OAAO,CAAC,EAAE,CAAA;IACtC,2CAA2C;IAC3C,OAAO,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,iBAAiB,KAAK,OAAO,CAAC,OAAO,CAAC,GAAG,OAAO,CAAA;IACrF,uCAAuC;IACvC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,OAAO,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IACtD,kCAAkC;IAClC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,oBAAoB;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,oCAAoC;IACpC,KAAK,EAAE,MAAM,CAAA;IACb,uCAAuC;IACvC,SAAS,EAAE,IAAI,CAAA;IACf,8BAA8B;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB,CAAC,OAAO,GAAG,OAAO;IACjD,oBAAoB;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,mBAAmB;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,aAAa;IACb,KAAK,EAAE,MAAM,CAAA;IACb,uBAAuB;IACvB,MAAM,EAAE,OAAO,CAAA;IACf,0DAA0D;IAC1D,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,yCAAyC;IACzC,QAAQ,EAAE,MAAM,CAAA;IAChB,uCAAuC;IACvC,SAAS,EAAE,IAAI,CAAA;IACf,yCAAyC;IACzC,WAAW,EAAE,IAAI,CAAA;IACjB,gCAAgC;IAChC,KAAK,CAAC,EAAE,KAAK,CAAA;IACb,mBAAmB;IACnB,OAAO,EAAE,OAAO,CAAA;IAChB,0BAA0B;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB,CAAC,OAAO,GAAG,OAAO;IAClD,oBAAoB;IACpB,YAAY,EAAE,MAAM,CAAA;IACpB,sBAAsB;IACtB,cAAc,EAAE,MAAM,CAAA;IACtB,+BAA+B;IAC/B,OAAO,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAA;IACpC,0CAA0C;IAC1C,WAAW,CAAC,EAAE;QACZ,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;QACnB,WAAW,EAAE,MAAM,CAAA;KACpB,CAAA;IACD,+BAA+B;IAC/B,aAAa,EAAE,MAAM,CAAA;IACrB,gCAAgC;IAChC,YAAY,EAAE,MAAM,CAAA;IACpB,4BAA4B;IAC5B,YAAY,EAAE,MAAM,CAAA;IACpB,wCAAwC;IACxC,SAAS,EAAE,IAAI,CAAA;IACf,0CAA0C;IAC1C,WAAW,EAAE,IAAI,CAAA;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACnC,+CAA+C;IAC/C,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,yDAAyD;IACzD,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,2CAA2C;IAC3C,WAAW,CAAC,EAAE,OAAO,CAAA;IACrB,0BAA0B;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACjC,sBAAsB;IACtB,cAAc,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,KAAK,IAAI,CAAA;IACjE,iBAAiB,CAAC,EAAE,CAAC,MAAM,EAAE,gBAAgB,KAAK,IAAI,CAAA;IACtD,cAAc,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,KAAK,IAAI,CAAA;CAC3D;AAED;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAA;AAEvD;;GAEG;AACH,MAAM,MAAM,eAAe,CAAC,CAAC,SAAS,eAAe,IAAI,KAAK,CAAC;KAC5D,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;CAC7B,CAAC,CAAA;AAEF;;GAEG;AACH,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,6BAA6B;IAC7B,OAAO,EAAE,CAAC,EAAE,CAAA;IACZ,uCAAuC;IACvC,KAAK,EAAE,CAAC,MAAM,EAAE,CAAC,KAAK,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IAC9C,4CAA4C;IAC5C,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,qEAAqE;IACrE,SAAS,CAAC,EAAE,OAAO,CAAA;CACpB;AAED;;GAEG;AACH,MAAM,WAAW,cAAc,CAAC,CAAC;IAC/B,0BAA0B;IAC1B,QAAQ,EAAE,CAAC,CAAA;IACX,mCAAmC;IACnC,KAAK,EAAE,MAAM,CAAA;IACb,4DAA4D;IAC5D,UAAU,CAAC,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,CAAC,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAA;CACjD;AAED;;GAEG;AACH,MAAM,MAAM,iBAAiB,GACzB,kBAAkB,GAClB,qBAAqB,GACrB,eAAe,GACf,kBAAkB,GAClB,eAAe,GACf,iBAAiB,GACjB,eAAe,CAAA;AAEnB;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,iBAAiB;IACjB,IAAI,EAAE,iBAAiB,CAAA;IACvB,gBAAgB;IAChB,SAAS,EAAE,IAAI,CAAA;IACf,iBAAiB;IACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,qBAAqB;IACrB,KAAK,CAAC,KAAK,EAAE,aAAa,GAAG,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACjD,2BAA2B;IAC3B,KAAK,CAAC,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,8BAA8B;IAC9B,OAAO,CAAC,EAAE,eAAe,CAAA;IACzB,kDAAkD;IAClD,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,qDAAqD;IACrD,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC"}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;GAEG"}
|
package/examples.ts
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Usage examples for ai-experiments
|
|
3
|
+
*
|
|
4
|
+
* These examples demonstrate the core APIs and patterns.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import {
|
|
8
|
+
Experiment,
|
|
9
|
+
cartesian,
|
|
10
|
+
decide,
|
|
11
|
+
track,
|
|
12
|
+
configureTracking,
|
|
13
|
+
createMemoryBackend,
|
|
14
|
+
type ExperimentConfig,
|
|
15
|
+
} from './src/index.js'
|
|
16
|
+
|
|
17
|
+
// ============================================================================
|
|
18
|
+
// Example 1: Simple A/B Experiment
|
|
19
|
+
// ============================================================================
|
|
20
|
+
|
|
21
|
+
async function example1_simpleExperiment() {
|
|
22
|
+
console.log('\n=== Example 1: Simple A/B Experiment ===\n')
|
|
23
|
+
|
|
24
|
+
const results = await Experiment({
|
|
25
|
+
id: 'greeting-test',
|
|
26
|
+
name: 'Greeting Message Test',
|
|
27
|
+
description: 'Test different greeting messages',
|
|
28
|
+
variants: [
|
|
29
|
+
{
|
|
30
|
+
id: 'formal',
|
|
31
|
+
name: 'Formal Greeting',
|
|
32
|
+
config: { message: 'Good day, sir.' },
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
id: 'casual',
|
|
36
|
+
name: 'Casual Greeting',
|
|
37
|
+
config: { message: 'Hey there!' },
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
id: 'friendly',
|
|
41
|
+
name: 'Friendly Greeting',
|
|
42
|
+
config: { message: 'Hello friend!' },
|
|
43
|
+
},
|
|
44
|
+
],
|
|
45
|
+
execute: async (config) => {
|
|
46
|
+
// Simulate some work
|
|
47
|
+
await new Promise((resolve) => setTimeout(resolve, 100))
|
|
48
|
+
return {
|
|
49
|
+
message: config.message,
|
|
50
|
+
wordCount: config.message.split(' ').length,
|
|
51
|
+
charCount: config.message.length,
|
|
52
|
+
}
|
|
53
|
+
},
|
|
54
|
+
metric: (result) => {
|
|
55
|
+
// Score based on message length (prefer shorter)
|
|
56
|
+
return 1 / result.charCount
|
|
57
|
+
},
|
|
58
|
+
})
|
|
59
|
+
|
|
60
|
+
console.log('Experiment Results:')
|
|
61
|
+
console.log('- Best Variant:', results.bestVariant)
|
|
62
|
+
console.log('- Success Rate:', `${results.successCount}/${results.results.length}`)
|
|
63
|
+
console.log('- Total Duration:', `${results.totalDuration}ms`)
|
|
64
|
+
console.log('\nAll Results:')
|
|
65
|
+
results.results.forEach((r) => {
|
|
66
|
+
console.log(
|
|
67
|
+
` ${r.variantName}: metric=${r.metricValue?.toFixed(4)}, duration=${r.duration}ms`
|
|
68
|
+
)
|
|
69
|
+
})
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// ============================================================================
|
|
73
|
+
// Example 2: Parameter Grid with Cartesian Product
|
|
74
|
+
// ============================================================================
|
|
75
|
+
|
|
76
|
+
async function example2_parameterGrid() {
|
|
77
|
+
console.log('\n=== Example 2: Parameter Grid Experiment ===\n')
|
|
78
|
+
|
|
79
|
+
// Generate all combinations of parameters
|
|
80
|
+
const parameterGrid = cartesian({
|
|
81
|
+
temperature: [0.3, 0.7, 1.0],
|
|
82
|
+
maxTokens: [50, 100],
|
|
83
|
+
topP: [0.9, 1.0],
|
|
84
|
+
})
|
|
85
|
+
|
|
86
|
+
console.log(`Generated ${parameterGrid.length} parameter combinations:`)
|
|
87
|
+
parameterGrid.slice(0, 5).forEach((params, i) => {
|
|
88
|
+
console.log(` ${i + 1}.`, params)
|
|
89
|
+
})
|
|
90
|
+
console.log(' ...')
|
|
91
|
+
|
|
92
|
+
// Create variants from parameter combinations
|
|
93
|
+
const variants = parameterGrid.map((params, i) => ({
|
|
94
|
+
id: `variant-${i}`,
|
|
95
|
+
name: `T=${params.temperature} max=${params.maxTokens} topP=${params.topP}`,
|
|
96
|
+
config: params,
|
|
97
|
+
}))
|
|
98
|
+
|
|
99
|
+
const results = await Experiment({
|
|
100
|
+
id: 'parameter-sweep',
|
|
101
|
+
name: 'Parameter Sweep Experiment',
|
|
102
|
+
variants,
|
|
103
|
+
execute: async (config) => {
|
|
104
|
+
// Simulate AI generation with different parameters
|
|
105
|
+
await new Promise((resolve) => setTimeout(resolve, 50))
|
|
106
|
+
const score = Math.random() * config.temperature + config.maxTokens / 1000
|
|
107
|
+
return { score, config }
|
|
108
|
+
},
|
|
109
|
+
metric: (result) => result.score,
|
|
110
|
+
})
|
|
111
|
+
|
|
112
|
+
console.log(`\nBest parameters: ${results.bestVariant?.variantName}`)
|
|
113
|
+
console.log(`Best score: ${results.bestVariant?.metricValue.toFixed(4)}`)
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// ============================================================================
|
|
117
|
+
// Example 3: Decision Making
|
|
118
|
+
// ============================================================================
|
|
119
|
+
|
|
120
|
+
async function example3_decisionMaking() {
|
|
121
|
+
console.log('\n=== Example 3: Decision Making ===\n')
|
|
122
|
+
|
|
123
|
+
// Simple decision with sync scoring
|
|
124
|
+
const result1 = await decide({
|
|
125
|
+
options: ['apple', 'banana', 'orange'],
|
|
126
|
+
score: (fruit) => {
|
|
127
|
+
const prices: Record<string, number> = { apple: 1.5, banana: 0.5, orange: 2.0 }
|
|
128
|
+
return 1 / prices[fruit] // Lower price = higher score
|
|
129
|
+
},
|
|
130
|
+
context: 'Choosing fruit based on value',
|
|
131
|
+
})
|
|
132
|
+
|
|
133
|
+
console.log('Decision 1 (best value):')
|
|
134
|
+
console.log(` Selected: ${result1.selected}`)
|
|
135
|
+
console.log(` Score: ${result1.score.toFixed(4)}`)
|
|
136
|
+
|
|
137
|
+
// Decision with all options returned
|
|
138
|
+
const result2 = await decide({
|
|
139
|
+
options: ['fast', 'accurate', 'balanced'],
|
|
140
|
+
score: async (approach) => {
|
|
141
|
+
const scores: Record<string, number> = { fast: 0.7, accurate: 0.85, balanced: 0.9 }
|
|
142
|
+
return scores[approach]
|
|
143
|
+
},
|
|
144
|
+
returnAll: true,
|
|
145
|
+
})
|
|
146
|
+
|
|
147
|
+
console.log('\nDecision 2 (with all options):')
|
|
148
|
+
console.log(` Selected: ${result2.selected}`)
|
|
149
|
+
result2.allOptions?.forEach((opt) => {
|
|
150
|
+
console.log(` - ${opt.option}: ${opt.score}`)
|
|
151
|
+
})
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// ============================================================================
|
|
155
|
+
// Example 4: Event Tracking
|
|
156
|
+
// ============================================================================
|
|
157
|
+
|
|
158
|
+
async function example4_eventTracking() {
|
|
159
|
+
console.log('\n=== Example 4: Event Tracking ===\n')
|
|
160
|
+
|
|
161
|
+
// Create a memory backend to collect events
|
|
162
|
+
const backend = createMemoryBackend()
|
|
163
|
+
configureTracking({ backend })
|
|
164
|
+
|
|
165
|
+
// Run a small experiment
|
|
166
|
+
await Experiment({
|
|
167
|
+
id: 'tracked-experiment',
|
|
168
|
+
name: 'Tracked Experiment',
|
|
169
|
+
variants: [
|
|
170
|
+
{ id: 'v1', name: 'Variant 1', config: { value: 1 } },
|
|
171
|
+
{ id: 'v2', name: 'Variant 2', config: { value: 2 } },
|
|
172
|
+
],
|
|
173
|
+
execute: async (config) => ({ result: config.value * 2 }),
|
|
174
|
+
metric: (result) => result.result,
|
|
175
|
+
})
|
|
176
|
+
|
|
177
|
+
// Get all tracked events
|
|
178
|
+
const events = backend.getEvents()
|
|
179
|
+
console.log(`Tracked ${events.length} events:`)
|
|
180
|
+
events.forEach((event) => {
|
|
181
|
+
console.log(` [${event.type}]`, JSON.stringify(event.data).slice(0, 80))
|
|
182
|
+
})
|
|
183
|
+
|
|
184
|
+
// Reset tracking to default
|
|
185
|
+
configureTracking({ enabled: true })
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// ============================================================================
|
|
189
|
+
// Example 5: Advanced - Sequential vs Parallel Execution
|
|
190
|
+
// ============================================================================
|
|
191
|
+
|
|
192
|
+
async function example5_executionModes() {
|
|
193
|
+
console.log('\n=== Example 5: Sequential vs Parallel Execution ===\n')
|
|
194
|
+
|
|
195
|
+
const variants = [
|
|
196
|
+
{ id: 'v1', name: 'Variant 1', config: { delay: 100 } },
|
|
197
|
+
{ id: 'v2', name: 'Variant 2', config: { delay: 150 } },
|
|
198
|
+
{ id: 'v3', name: 'Variant 3', config: { delay: 200 } },
|
|
199
|
+
]
|
|
200
|
+
|
|
201
|
+
const execute = async (config: { delay: number }) => {
|
|
202
|
+
await new Promise((resolve) => setTimeout(resolve, config.delay))
|
|
203
|
+
return { completed: true }
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
// Parallel execution (default)
|
|
207
|
+
const start1 = Date.now()
|
|
208
|
+
await Experiment({
|
|
209
|
+
id: 'parallel-test',
|
|
210
|
+
name: 'Parallel Execution',
|
|
211
|
+
variants,
|
|
212
|
+
execute,
|
|
213
|
+
})
|
|
214
|
+
const parallel_duration = Date.now() - start1
|
|
215
|
+
|
|
216
|
+
// Sequential execution
|
|
217
|
+
const start2 = Date.now()
|
|
218
|
+
await Experiment(
|
|
219
|
+
{
|
|
220
|
+
id: 'sequential-test',
|
|
221
|
+
name: 'Sequential Execution',
|
|
222
|
+
variants,
|
|
223
|
+
execute,
|
|
224
|
+
},
|
|
225
|
+
{ parallel: false }
|
|
226
|
+
)
|
|
227
|
+
const sequential_duration = Date.now() - start2
|
|
228
|
+
|
|
229
|
+
console.log(`Parallel execution: ${parallel_duration}ms`)
|
|
230
|
+
console.log(`Sequential execution: ${sequential_duration}ms`)
|
|
231
|
+
console.log(
|
|
232
|
+
`Speedup: ${(sequential_duration / parallel_duration).toFixed(2)}x`
|
|
233
|
+
)
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
// ============================================================================
|
|
237
|
+
// Run all examples
|
|
238
|
+
// ============================================================================
|
|
239
|
+
|
|
240
|
+
async function main() {
|
|
241
|
+
// Disable verbose tracking for examples
|
|
242
|
+
configureTracking({ enabled: false })
|
|
243
|
+
|
|
244
|
+
try {
|
|
245
|
+
await example1_simpleExperiment()
|
|
246
|
+
await example2_parameterGrid()
|
|
247
|
+
await example3_decisionMaking()
|
|
248
|
+
await example4_eventTracking()
|
|
249
|
+
await example5_executionModes()
|
|
250
|
+
|
|
251
|
+
console.log('\n✅ All examples completed successfully!\n')
|
|
252
|
+
} catch (error) {
|
|
253
|
+
console.error('\n❌ Error running examples:', error)
|
|
254
|
+
process.exit(1)
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
// Run if this file is executed directly
|
|
259
|
+
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
260
|
+
main()
|
|
261
|
+
}
|
package/package.json
CHANGED
|
@@ -1,50 +1,32 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ai-experiments",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "
|
|
5
|
-
"
|
|
6
|
-
"
|
|
7
|
-
"types": "
|
|
3
|
+
"version": "2.0.2",
|
|
4
|
+
"description": "AI-powered experimentation primitives for testing and evaluating models",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
8
|
"exports": {
|
|
9
9
|
".": {
|
|
10
|
-
"
|
|
11
|
-
"import": "./dist/index.mjs",
|
|
10
|
+
"import": "./dist/index.js",
|
|
12
11
|
"types": "./dist/index.d.ts"
|
|
13
12
|
}
|
|
14
13
|
},
|
|
15
|
-
"
|
|
16
|
-
"
|
|
17
|
-
|
|
14
|
+
"scripts": {
|
|
15
|
+
"build": "tsc",
|
|
16
|
+
"dev": "tsc --watch",
|
|
17
|
+
"test": "vitest",
|
|
18
|
+
"typecheck": "tsc --noEmit",
|
|
19
|
+
"lint": "eslint .",
|
|
20
|
+
"clean": "rm -rf dist"
|
|
21
|
+
},
|
|
22
|
+
"dependencies": {
|
|
23
|
+
"ai-functions": "2.0.2",
|
|
24
|
+
"rpc.do": "^0.1.0"
|
|
25
|
+
},
|
|
18
26
|
"keywords": [
|
|
19
27
|
"ai",
|
|
20
28
|
"experiments",
|
|
21
|
-
"
|
|
22
|
-
"vitest"
|
|
29
|
+
"primitives"
|
|
23
30
|
],
|
|
24
|
-
"
|
|
25
|
-
|
|
26
|
-
},
|
|
27
|
-
"devDependencies": {
|
|
28
|
-
"tsup": "^8.0.1",
|
|
29
|
-
"typescript": "^5.2.2",
|
|
30
|
-
"prettier": "^3.4.2",
|
|
31
|
-
"@types/node": "^20.17.0"
|
|
32
|
-
},
|
|
33
|
-
"repository": {
|
|
34
|
-
"type": "git",
|
|
35
|
-
"url": "https://github.com/drivly/primitives.org.ai.git"
|
|
36
|
-
},
|
|
37
|
-
"bugs": {
|
|
38
|
-
"url": "https://github.com/drivly/primitives.org.ai/issues"
|
|
39
|
-
},
|
|
40
|
-
"homepage": "https://primitives.org.ai",
|
|
41
|
-
"license": "MIT",
|
|
42
|
-
"scripts": {
|
|
43
|
-
"build": "tsup",
|
|
44
|
-
"dev": "tsup --watch",
|
|
45
|
-
"test": "vitest run",
|
|
46
|
-
"test:watch": "vitest",
|
|
47
|
-
"typecheck": "tsc --noEmit",
|
|
48
|
-
"format": "prettier --write 'src/**/*.{ts,tsx,js,jsx}'"
|
|
49
|
-
}
|
|
50
|
-
}
|
|
31
|
+
"license": "MIT"
|
|
32
|
+
}
|