ai-experiments 2.1.3 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.turbo/turbo-build.log +1 -1
- package/CHANGELOG.md +17 -0
- package/README.md +13 -1
- package/dist/clickhouse-storage.d.ts +227 -0
- package/dist/clickhouse-storage.d.ts.map +1 -0
- package/dist/clickhouse-storage.js +522 -0
- package/dist/clickhouse-storage.js.map +1 -0
- package/dist/decide.d.ts.map +1 -1
- package/dist/decide.js +6 -2
- package/dist/decide.js.map +1 -1
- package/dist/experiment.js +9 -9
- package/dist/experiment.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -2
- package/dist/index.js.map +1 -1
- package/dist/tracking.js +3 -3
- package/dist/tracking.js.map +1 -1
- package/package.json +16 -16
- package/src/clickhouse-storage.ts +698 -0
- package/src/decide.ts +8 -10
- package/src/experiment.ts +12 -12
- package/src/index.ts +13 -3
- package/src/tracking.ts +3 -3
- package/test/cartesian.test.ts +287 -0
- package/test/clickhouse-storage.test.ts +470 -0
- package/test/decide.test.ts +414 -0
- package/test/experiment.test.ts +473 -0
- package/test/tracking.test.ts +347 -0
- package/vitest.config.ts +34 -0
- package/LICENSE +0 -21
- package/dist/chdb-storage.d.ts +0 -135
- package/dist/chdb-storage.d.ts.map +0 -1
- package/dist/chdb-storage.js +0 -468
- package/dist/chdb-storage.js.map +0 -1
- package/src/cartesian.js +0 -215
- package/src/chdb-storage.js +0 -464
- package/src/chdb-storage.ts +0 -567
- package/src/decide.js +0 -328
- package/src/experiment.js +0 -291
- package/src/index.js +0 -20
- package/src/tracking.js +0 -309
- package/src/types.js +0 -4
package/src/tracking.js
DELETED
|
@@ -1,309 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Event tracking for experiments
|
|
3
|
-
*/
|
|
4
|
-
/**
|
|
5
|
-
* Default tracking configuration
|
|
6
|
-
*/
|
|
7
|
-
let trackingConfig = {
|
|
8
|
-
backend: createConsoleBackend(),
|
|
9
|
-
enabled: true,
|
|
10
|
-
metadata: {},
|
|
11
|
-
};
|
|
12
|
-
/**
|
|
13
|
-
* Configure tracking
|
|
14
|
-
*
|
|
15
|
-
* @example
|
|
16
|
-
* ```ts
|
|
17
|
-
* import { configureTracking } from 'ai-experiments'
|
|
18
|
-
*
|
|
19
|
-
* // Use console backend (default)
|
|
20
|
-
* configureTracking({
|
|
21
|
-
* enabled: true,
|
|
22
|
-
* metadata: { projectId: 'my-project' },
|
|
23
|
-
* })
|
|
24
|
-
*
|
|
25
|
-
* // Use custom backend
|
|
26
|
-
* configureTracking({
|
|
27
|
-
* backend: {
|
|
28
|
-
* track: async (event) => {
|
|
29
|
-
* await fetch('/api/analytics', {
|
|
30
|
-
* method: 'POST',
|
|
31
|
-
* body: JSON.stringify(event),
|
|
32
|
-
* })
|
|
33
|
-
* },
|
|
34
|
-
* },
|
|
35
|
-
* })
|
|
36
|
-
*
|
|
37
|
-
* // Disable tracking
|
|
38
|
-
* configureTracking({ enabled: false })
|
|
39
|
-
* ```
|
|
40
|
-
*/
|
|
41
|
-
export function configureTracking(options) {
|
|
42
|
-
trackingConfig = {
|
|
43
|
-
backend: options.backend ?? trackingConfig.backend,
|
|
44
|
-
enabled: options.enabled ?? trackingConfig.enabled,
|
|
45
|
-
metadata: options.metadata ?? trackingConfig.metadata,
|
|
46
|
-
};
|
|
47
|
-
}
|
|
48
|
-
/**
|
|
49
|
-
* Track an event
|
|
50
|
-
*
|
|
51
|
-
* @example
|
|
52
|
-
* ```ts
|
|
53
|
-
* import { track } from 'ai-experiments'
|
|
54
|
-
*
|
|
55
|
-
* track({
|
|
56
|
-
* type: 'experiment.start',
|
|
57
|
-
* timestamp: new Date(),
|
|
58
|
-
* data: {
|
|
59
|
-
* experimentId: 'my-experiment',
|
|
60
|
-
* variantCount: 3,
|
|
61
|
-
* },
|
|
62
|
-
* })
|
|
63
|
-
* ```
|
|
64
|
-
*/
|
|
65
|
-
export function track(event) {
|
|
66
|
-
if (!trackingConfig.enabled) {
|
|
67
|
-
return;
|
|
68
|
-
}
|
|
69
|
-
// Merge global metadata
|
|
70
|
-
const enrichedEvent = {
|
|
71
|
-
...event,
|
|
72
|
-
data: {
|
|
73
|
-
...event.data,
|
|
74
|
-
...trackingConfig.metadata,
|
|
75
|
-
},
|
|
76
|
-
};
|
|
77
|
-
// Track via backend (handle both sync and async)
|
|
78
|
-
const result = trackingConfig.backend.track(enrichedEvent);
|
|
79
|
-
if (result instanceof Promise) {
|
|
80
|
-
// Don't await - fire and forget
|
|
81
|
-
result.catch((error) => {
|
|
82
|
-
console.error('Error tracking event:', error);
|
|
83
|
-
});
|
|
84
|
-
}
|
|
85
|
-
}
|
|
86
|
-
/**
|
|
87
|
-
* Flush pending events
|
|
88
|
-
*
|
|
89
|
-
* Call this before the process exits to ensure all events are sent.
|
|
90
|
-
*
|
|
91
|
-
* @example
|
|
92
|
-
* ```ts
|
|
93
|
-
* import { flush } from 'ai-experiments'
|
|
94
|
-
*
|
|
95
|
-
* process.on('SIGINT', async () => {
|
|
96
|
-
* await flush()
|
|
97
|
-
* process.exit(0)
|
|
98
|
-
* })
|
|
99
|
-
* ```
|
|
100
|
-
*/
|
|
101
|
-
export async function flush() {
|
|
102
|
-
if (trackingConfig.backend.flush) {
|
|
103
|
-
await trackingConfig.backend.flush();
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
/**
|
|
107
|
-
* Create a console-based tracking backend
|
|
108
|
-
*
|
|
109
|
-
* Logs events to console.log in a human-readable format.
|
|
110
|
-
*/
|
|
111
|
-
export function createConsoleBackend(options) {
|
|
112
|
-
const { verbose = false } = options ?? {};
|
|
113
|
-
return {
|
|
114
|
-
track: (event) => {
|
|
115
|
-
const timestamp = event.timestamp.toISOString();
|
|
116
|
-
if (verbose) {
|
|
117
|
-
console.log(`[${timestamp}] ${event.type}`, event.data);
|
|
118
|
-
}
|
|
119
|
-
else {
|
|
120
|
-
// Condensed format
|
|
121
|
-
const key = extractKey(event);
|
|
122
|
-
console.log(`[${timestamp}] ${event.type} ${key}`);
|
|
123
|
-
}
|
|
124
|
-
},
|
|
125
|
-
};
|
|
126
|
-
}
|
|
127
|
-
/**
|
|
128
|
-
* Create an in-memory tracking backend that stores events
|
|
129
|
-
*
|
|
130
|
-
* Useful for testing or collecting events for batch processing.
|
|
131
|
-
*
|
|
132
|
-
* @example
|
|
133
|
-
* ```ts
|
|
134
|
-
* import { createMemoryBackend } from 'ai-experiments'
|
|
135
|
-
*
|
|
136
|
-
* const backend = createMemoryBackend()
|
|
137
|
-
* configureTracking({ backend })
|
|
138
|
-
*
|
|
139
|
-
* // Run experiments...
|
|
140
|
-
*
|
|
141
|
-
* // Get all events
|
|
142
|
-
* const events = backend.getEvents()
|
|
143
|
-
* console.log(`Tracked ${events.length} events`)
|
|
144
|
-
*
|
|
145
|
-
* // Clear events
|
|
146
|
-
* backend.clear()
|
|
147
|
-
* ```
|
|
148
|
-
*/
|
|
149
|
-
export function createMemoryBackend() {
|
|
150
|
-
const events = [];
|
|
151
|
-
return {
|
|
152
|
-
track: (event) => {
|
|
153
|
-
events.push(event);
|
|
154
|
-
},
|
|
155
|
-
getEvents: () => [...events],
|
|
156
|
-
clear: () => {
|
|
157
|
-
events.length = 0;
|
|
158
|
-
},
|
|
159
|
-
};
|
|
160
|
-
}
|
|
161
|
-
/**
|
|
162
|
-
* Create a batching tracking backend
|
|
163
|
-
*
|
|
164
|
-
* Batches events and sends them in groups to reduce network overhead.
|
|
165
|
-
*
|
|
166
|
-
* @example
|
|
167
|
-
* ```ts
|
|
168
|
-
* import { createBatchBackend } from 'ai-experiments'
|
|
169
|
-
*
|
|
170
|
-
* const backend = createBatchBackend({
|
|
171
|
-
* batchSize: 10,
|
|
172
|
-
* flushInterval: 5000, // 5 seconds
|
|
173
|
-
* send: async (events) => {
|
|
174
|
-
* await fetch('/api/analytics/batch', {
|
|
175
|
-
* method: 'POST',
|
|
176
|
-
* body: JSON.stringify({ events }),
|
|
177
|
-
* })
|
|
178
|
-
* },
|
|
179
|
-
* })
|
|
180
|
-
*
|
|
181
|
-
* configureTracking({ backend })
|
|
182
|
-
* ```
|
|
183
|
-
*/
|
|
184
|
-
export function createBatchBackend(options) {
|
|
185
|
-
const { batchSize, flushInterval, send } = options;
|
|
186
|
-
const batch = [];
|
|
187
|
-
let flushTimer = null;
|
|
188
|
-
const flush = async () => {
|
|
189
|
-
if (batch.length === 0)
|
|
190
|
-
return;
|
|
191
|
-
const eventsToSend = [...batch];
|
|
192
|
-
batch.length = 0;
|
|
193
|
-
try {
|
|
194
|
-
await send(eventsToSend);
|
|
195
|
-
}
|
|
196
|
-
catch (error) {
|
|
197
|
-
console.error('Error sending batch:', error);
|
|
198
|
-
// Re-add failed events to batch (simple retry strategy)
|
|
199
|
-
batch.unshift(...eventsToSend);
|
|
200
|
-
}
|
|
201
|
-
};
|
|
202
|
-
const scheduleFlush = () => {
|
|
203
|
-
if (flushTimer) {
|
|
204
|
-
clearTimeout(flushTimer);
|
|
205
|
-
}
|
|
206
|
-
if (flushInterval) {
|
|
207
|
-
flushTimer = setTimeout(() => {
|
|
208
|
-
flush().catch(console.error);
|
|
209
|
-
}, flushInterval);
|
|
210
|
-
}
|
|
211
|
-
};
|
|
212
|
-
return {
|
|
213
|
-
track: (event) => {
|
|
214
|
-
batch.push(event);
|
|
215
|
-
// Auto-flush if batch is full
|
|
216
|
-
if (batch.length >= batchSize) {
|
|
217
|
-
flush().catch(console.error);
|
|
218
|
-
}
|
|
219
|
-
else {
|
|
220
|
-
scheduleFlush();
|
|
221
|
-
}
|
|
222
|
-
},
|
|
223
|
-
flush: async () => {
|
|
224
|
-
if (flushTimer) {
|
|
225
|
-
clearTimeout(flushTimer);
|
|
226
|
-
flushTimer = null;
|
|
227
|
-
}
|
|
228
|
-
await flush();
|
|
229
|
-
},
|
|
230
|
-
};
|
|
231
|
-
}
|
|
232
|
-
/**
|
|
233
|
-
* Create a file-based tracking backend
|
|
234
|
-
*
|
|
235
|
-
* Writes events to a file (JSONL format).
|
|
236
|
-
*
|
|
237
|
-
* @example
|
|
238
|
-
* ```ts
|
|
239
|
-
* import { createFileBackend } from 'ai-experiments'
|
|
240
|
-
*
|
|
241
|
-
* const backend = createFileBackend({
|
|
242
|
-
* path: './experiments.jsonl',
|
|
243
|
-
* })
|
|
244
|
-
*
|
|
245
|
-
* configureTracking({ backend })
|
|
246
|
-
* ```
|
|
247
|
-
*/
|
|
248
|
-
export function createFileBackend(options) {
|
|
249
|
-
// Note: This requires Node.js fs module
|
|
250
|
-
// Import dynamically to avoid breaking in non-Node environments
|
|
251
|
-
let fs = null;
|
|
252
|
-
let writeStream = null;
|
|
253
|
-
const ensureStream = async () => {
|
|
254
|
-
if (!writeStream) {
|
|
255
|
-
try {
|
|
256
|
-
fs = await import('fs');
|
|
257
|
-
writeStream = fs.createWriteStream(options.path, { flags: 'a' });
|
|
258
|
-
}
|
|
259
|
-
catch (error) {
|
|
260
|
-
console.error('Failed to create file stream:', error);
|
|
261
|
-
throw error;
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
return writeStream;
|
|
265
|
-
};
|
|
266
|
-
return {
|
|
267
|
-
track: async (event) => {
|
|
268
|
-
try {
|
|
269
|
-
const stream = await ensureStream();
|
|
270
|
-
const line = JSON.stringify(event) + '\n';
|
|
271
|
-
stream.write(line);
|
|
272
|
-
}
|
|
273
|
-
catch (error) {
|
|
274
|
-
console.error('Failed to write event to file:', error);
|
|
275
|
-
}
|
|
276
|
-
},
|
|
277
|
-
flush: async () => {
|
|
278
|
-
if (writeStream) {
|
|
279
|
-
return new Promise((resolve, reject) => {
|
|
280
|
-
writeStream.end((error) => {
|
|
281
|
-
if (error)
|
|
282
|
-
reject(error);
|
|
283
|
-
else
|
|
284
|
-
resolve();
|
|
285
|
-
});
|
|
286
|
-
});
|
|
287
|
-
}
|
|
288
|
-
},
|
|
289
|
-
};
|
|
290
|
-
}
|
|
291
|
-
/**
|
|
292
|
-
* Extract a key identifier from event data for logging
|
|
293
|
-
*/
|
|
294
|
-
function extractKey(event) {
|
|
295
|
-
const data = event.data;
|
|
296
|
-
if ('experimentId' in data)
|
|
297
|
-
return `exp=${data.experimentId}`;
|
|
298
|
-
if ('variantId' in data)
|
|
299
|
-
return `variant=${data.variantId}`;
|
|
300
|
-
if ('runId' in data)
|
|
301
|
-
return `run=${data.runId}`;
|
|
302
|
-
return '';
|
|
303
|
-
}
|
|
304
|
-
/**
|
|
305
|
-
* Get current tracking configuration
|
|
306
|
-
*/
|
|
307
|
-
export function getTrackingConfig() {
|
|
308
|
-
return { ...trackingConfig };
|
|
309
|
-
}
|
package/src/types.js
DELETED