ai-experiments 2.1.3 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/README.md +13 -1
- package/package.json +16 -16
- package/src/clickhouse-storage.ts +698 -0
- package/src/decide.ts +8 -10
- package/src/experiment.ts +12 -12
- package/src/index.ts +13 -3
- package/src/tracking.ts +3 -3
- package/test/cartesian.test.ts +287 -0
- package/test/clickhouse-storage.test.ts +470 -0
- package/test/decide.test.ts +414 -0
- package/test/experiment.test.ts +473 -0
- package/test/tracking.test.ts +347 -0
- package/vitest.config.ts +34 -0
- package/.turbo/turbo-build.log +0 -4
- package/LICENSE +0 -21
- package/dist/cartesian.d.ts +0 -140
- package/dist/cartesian.d.ts.map +0 -1
- package/dist/cartesian.js +0 -216
- package/dist/cartesian.js.map +0 -1
- package/dist/chdb-storage.d.ts +0 -135
- package/dist/chdb-storage.d.ts.map +0 -1
- package/dist/chdb-storage.js +0 -468
- package/dist/chdb-storage.js.map +0 -1
- package/dist/decide.d.ts +0 -152
- package/dist/decide.d.ts.map +0 -1
- package/dist/decide.js +0 -329
- package/dist/decide.js.map +0 -1
- package/dist/experiment.d.ts +0 -53
- package/dist/experiment.d.ts.map +0 -1
- package/dist/experiment.js +0 -292
- package/dist/experiment.js.map +0 -1
- package/dist/index.d.ts +0 -16
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -21
- package/dist/index.js.map +0 -1
- package/dist/tracking.d.ts +0 -159
- package/dist/tracking.d.ts.map +0 -1
- package/dist/tracking.js +0 -310
- package/dist/tracking.js.map +0 -1
- package/dist/types.d.ts +0 -198
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js +0 -5
- package/dist/types.js.map +0 -1
- package/src/cartesian.js +0 -215
- package/src/chdb-storage.js +0 -464
- package/src/chdb-storage.ts +0 -567
- package/src/decide.js +0 -328
- package/src/experiment.js +0 -291
- package/src/index.js +0 -20
- package/src/tracking.js +0 -309
- package/src/types.js +0 -4
package/dist/cartesian.js
DELETED
|
@@ -1,216 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Cartesian product utilities for parameter exploration
|
|
3
|
-
*/
|
|
4
|
-
/**
|
|
5
|
-
* Generate cartesian product of parameter sets
|
|
6
|
-
*
|
|
7
|
-
* Takes an object where each key maps to an array of possible values,
|
|
8
|
-
* and returns all possible combinations as an array of objects.
|
|
9
|
-
*
|
|
10
|
-
* @example
|
|
11
|
-
* ```ts
|
|
12
|
-
* import { cartesian } from 'ai-experiments'
|
|
13
|
-
*
|
|
14
|
-
* const combinations = cartesian({
|
|
15
|
-
* model: ['sonnet', 'opus', 'gpt-4o'],
|
|
16
|
-
* temperature: [0.3, 0.7, 1.0],
|
|
17
|
-
* maxTokens: [100, 500, 1000],
|
|
18
|
-
* })
|
|
19
|
-
*
|
|
20
|
-
* // Returns 27 combinations (3 * 3 * 3):
|
|
21
|
-
* // [
|
|
22
|
-
* // { model: 'sonnet', temperature: 0.3, maxTokens: 100 },
|
|
23
|
-
* // { model: 'sonnet', temperature: 0.3, maxTokens: 500 },
|
|
24
|
-
* // { model: 'sonnet', temperature: 0.3, maxTokens: 1000 },
|
|
25
|
-
* // { model: 'sonnet', temperature: 0.7, maxTokens: 100 },
|
|
26
|
-
* // ...
|
|
27
|
-
* // ]
|
|
28
|
-
*
|
|
29
|
-
* // Use with experiments:
|
|
30
|
-
* const variants = combinations.map((config, i) => ({
|
|
31
|
-
* id: `variant-${i}`,
|
|
32
|
-
* name: `${config.model} T=${config.temperature} max=${config.maxTokens}`,
|
|
33
|
-
* config,
|
|
34
|
-
* }))
|
|
35
|
-
* ```
|
|
36
|
-
*/
|
|
37
|
-
export function cartesian(params) {
|
|
38
|
-
const keys = Object.keys(params);
|
|
39
|
-
const values = keys.map((k) => params[k]);
|
|
40
|
-
// Handle empty input
|
|
41
|
-
if (keys.length === 0) {
|
|
42
|
-
return [];
|
|
43
|
-
}
|
|
44
|
-
// Handle single parameter
|
|
45
|
-
if (keys.length === 1) {
|
|
46
|
-
const key = keys[0];
|
|
47
|
-
return values[0].map((val) => ({ [key]: val }));
|
|
48
|
-
}
|
|
49
|
-
// Generate all combinations using recursive helper
|
|
50
|
-
const combinations = cartesianProduct(values);
|
|
51
|
-
// Map back to objects
|
|
52
|
-
return combinations.map((combo) => {
|
|
53
|
-
const obj = {};
|
|
54
|
-
keys.forEach((key, i) => {
|
|
55
|
-
obj[key] = combo[i];
|
|
56
|
-
});
|
|
57
|
-
return obj;
|
|
58
|
-
});
|
|
59
|
-
}
|
|
60
|
-
/**
|
|
61
|
-
* Recursive cartesian product implementation
|
|
62
|
-
*/
|
|
63
|
-
function cartesianProduct(arrays) {
|
|
64
|
-
if (arrays.length === 0)
|
|
65
|
-
return [[]];
|
|
66
|
-
const [first, ...rest] = arrays;
|
|
67
|
-
// Base case: single array
|
|
68
|
-
if (rest.length === 0) {
|
|
69
|
-
return first.map((x) => [x]);
|
|
70
|
-
}
|
|
71
|
-
// Recursive case
|
|
72
|
-
const restProduct = cartesianProduct(rest);
|
|
73
|
-
const result = [];
|
|
74
|
-
for (const x of first) {
|
|
75
|
-
for (const restCombo of restProduct) {
|
|
76
|
-
result.push([x, ...restCombo]);
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
return result;
|
|
80
|
-
}
|
|
81
|
-
/**
|
|
82
|
-
* Generate a grid of parameter combinations with filtering
|
|
83
|
-
*
|
|
84
|
-
* Similar to cartesian(), but allows filtering out invalid combinations.
|
|
85
|
-
*
|
|
86
|
-
* @example
|
|
87
|
-
* ```ts
|
|
88
|
-
* import { cartesianFilter } from 'ai-experiments'
|
|
89
|
-
*
|
|
90
|
-
* const combinations = cartesianFilter(
|
|
91
|
-
* {
|
|
92
|
-
* model: ['sonnet', 'opus'],
|
|
93
|
-
* temperature: [0.3, 0.7, 1.0],
|
|
94
|
-
* maxTokens: [100, 500],
|
|
95
|
-
* },
|
|
96
|
-
* // Filter out combinations where opus uses high temperature
|
|
97
|
-
* (combo) => !(combo.model === 'opus' && combo.temperature > 0.7)
|
|
98
|
-
* )
|
|
99
|
-
* ```
|
|
100
|
-
*/
|
|
101
|
-
export function cartesianFilter(params, filter) {
|
|
102
|
-
const allCombinations = cartesian(params);
|
|
103
|
-
return allCombinations.filter(filter);
|
|
104
|
-
}
|
|
105
|
-
/**
|
|
106
|
-
* Generate a random sample from the cartesian product
|
|
107
|
-
*
|
|
108
|
-
* Useful when the full cartesian product is too large to test all combinations.
|
|
109
|
-
*
|
|
110
|
-
* @example
|
|
111
|
-
* ```ts
|
|
112
|
-
* import { cartesianSample } from 'ai-experiments'
|
|
113
|
-
*
|
|
114
|
-
* // Full product would be 1000 combinations (10 * 10 * 10)
|
|
115
|
-
* // Sample just 20 random combinations
|
|
116
|
-
* const sample = cartesianSample(
|
|
117
|
-
* {
|
|
118
|
-
* param1: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
|
|
119
|
-
* param2: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
|
|
120
|
-
* param3: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
|
|
121
|
-
* },
|
|
122
|
-
* 20
|
|
123
|
-
* )
|
|
124
|
-
* ```
|
|
125
|
-
*/
|
|
126
|
-
export function cartesianSample(params, sampleSize, options = {}) {
|
|
127
|
-
const { unique = true } = options;
|
|
128
|
-
// Generate all combinations first
|
|
129
|
-
const allCombinations = cartesian(params);
|
|
130
|
-
// If sample size is larger than available combinations, return all
|
|
131
|
-
if (sampleSize >= allCombinations.length) {
|
|
132
|
-
return allCombinations;
|
|
133
|
-
}
|
|
134
|
-
// Shuffle and take first n items
|
|
135
|
-
const shuffled = [...allCombinations];
|
|
136
|
-
// Simple Fisher-Yates shuffle
|
|
137
|
-
for (let i = shuffled.length - 1; i > 0; i--) {
|
|
138
|
-
const j = Math.floor(Math.random() * (i + 1));
|
|
139
|
-
[shuffled[i], shuffled[j]] = [shuffled[j], shuffled[i]];
|
|
140
|
-
}
|
|
141
|
-
return shuffled.slice(0, sampleSize);
|
|
142
|
-
}
|
|
143
|
-
/**
|
|
144
|
-
* Count the total number of combinations without generating them
|
|
145
|
-
*
|
|
146
|
-
* Useful for checking if cartesian product is feasible before generating.
|
|
147
|
-
*
|
|
148
|
-
* @example
|
|
149
|
-
* ```ts
|
|
150
|
-
* import { cartesianCount } from 'ai-experiments'
|
|
151
|
-
*
|
|
152
|
-
* const count = cartesianCount({
|
|
153
|
-
* model: ['sonnet', 'opus', 'gpt-4o'],
|
|
154
|
-
* temperature: [0.3, 0.5, 0.7, 0.9],
|
|
155
|
-
* maxTokens: [100, 500, 1000, 2000],
|
|
156
|
-
* })
|
|
157
|
-
* // Returns 48 (3 * 4 * 4)
|
|
158
|
-
*
|
|
159
|
-
* if (count > 100) {
|
|
160
|
-
* console.log('Too many combinations, use cartesianSample instead')
|
|
161
|
-
* }
|
|
162
|
-
* ```
|
|
163
|
-
*/
|
|
164
|
-
export function cartesianCount(params) {
|
|
165
|
-
const keys = Object.keys(params);
|
|
166
|
-
if (keys.length === 0)
|
|
167
|
-
return 0;
|
|
168
|
-
return keys.reduce((total, key) => {
|
|
169
|
-
const arr = params[key];
|
|
170
|
-
return total * (arr?.length ?? 0);
|
|
171
|
-
}, 1);
|
|
172
|
-
}
|
|
173
|
-
/**
|
|
174
|
-
* Generate cartesian product with labels for each dimension
|
|
175
|
-
*
|
|
176
|
-
* Returns combinations with additional metadata about which dimension each value came from.
|
|
177
|
-
*
|
|
178
|
-
* @example
|
|
179
|
-
* ```ts
|
|
180
|
-
* import { cartesianWithLabels } from 'ai-experiments'
|
|
181
|
-
*
|
|
182
|
-
* const labeled = cartesianWithLabels({
|
|
183
|
-
* model: ['sonnet', 'opus'],
|
|
184
|
-
* temperature: [0.3, 0.7],
|
|
185
|
-
* })
|
|
186
|
-
* // [
|
|
187
|
-
* // { values: { model: 'sonnet', temperature: 0.3 }, labels: { model: 0, temperature: 0 } },
|
|
188
|
-
* // { values: { model: 'sonnet', temperature: 0.7 }, labels: { model: 0, temperature: 1 } },
|
|
189
|
-
* // { values: { model: 'opus', temperature: 0.3 }, labels: { model: 1, temperature: 0 } },
|
|
190
|
-
* // { values: { model: 'opus', temperature: 0.7 }, labels: { model: 1, temperature: 1 } },
|
|
191
|
-
* // ]
|
|
192
|
-
* ```
|
|
193
|
-
*/
|
|
194
|
-
export function cartesianWithLabels(params) {
|
|
195
|
-
const keys = Object.keys(params);
|
|
196
|
-
const values = keys.map((k) => params[k]);
|
|
197
|
-
if (keys.length === 0) {
|
|
198
|
-
return [];
|
|
199
|
-
}
|
|
200
|
-
const combinations = cartesianProduct(values);
|
|
201
|
-
return combinations.map((combo) => {
|
|
202
|
-
const valuesObj = {};
|
|
203
|
-
const labelsObj = {};
|
|
204
|
-
keys.forEach((key, i) => {
|
|
205
|
-
const value = combo[i];
|
|
206
|
-
valuesObj[key] = value;
|
|
207
|
-
const arr = params[key];
|
|
208
|
-
labelsObj[key] = arr ? arr.indexOf(value) : -1;
|
|
209
|
-
});
|
|
210
|
-
return {
|
|
211
|
-
values: valuesObj,
|
|
212
|
-
labels: labelsObj,
|
|
213
|
-
};
|
|
214
|
-
});
|
|
215
|
-
}
|
|
216
|
-
//# sourceMappingURL=cartesian.js.map
|
package/dist/cartesian.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cartesian.js","sourceRoot":"","sources":["../src/cartesian.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AACH,MAAM,UAAU,SAAS,CAA4B,MAAS;IAC5D,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAgB,CAAA;IAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAEzC,qBAAqB;IACrB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,EAAwB,CAAA;IACjC,CAAC;IAED,0BAA0B;IAC1B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAE,CAAA;QACpB,OAAO,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,EAAuC,CAAA,CAAC,CAAA;IACtF,CAAC;IAED,mDAAmD;IACnD,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAA;IAE7C,sBAAsB;IACtB,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAChC,MAAM,GAAG,GAAG,EAAsC,CAAA;QAClD,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;YACtB,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,CAA0B,CAAA;QAC9C,CAAC,CAAC,CAAA;QACF,OAAO,GAAG,CAAA;IACZ,CAAC,CAAC,CAAA;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAI,MAAa;IACxC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,EAAE,CAAC,CAAA;IAEpC,MAAM,CAAC,KAAK,EAAE,GAAG,IAAI,CAAC,GAAG,MAAM,CAAA;IAE/B,0BAA0B;IAC1B,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,KAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAC/B,CAAC;IAED,iBAAiB;IACjB,MAAM,WAAW,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;IAC1C,MAAM,MAAM,GAAU,EAAE,CAAA;IAExB,KAAK,MAAM,CAAC,IAAI,KAAM,EAAE,CAAC;QACvB,KAAK,MAAM,SAAS,IAAI,WAAW,EAAE,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,GAAG,SAAS,CAAC,CAAC,CAAA;QAChC,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAA;AACf,CAAC;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,UAAU,eAAe,CAC7B,MAAS,EACT,MAA4D;IAE5D,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC,CAAA;IACzC,OAAO,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;AACvC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,UAAU,eAAe,CAC7B,MAAS,EACT,UAAkB,EAClB,UAKI,EAAE;IAEN,MAAM,EAAE,MAAM,GAAG,IAAI,EAAE,GAAG,OAAO,CAAA;IAEjC,kCAAkC;IAClC,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC,CAAA;IAEzC,mEAAmE;IACnE,IAAI,UAAU,IAAI,eAAe,CAAC,MAAM,EAAE,CAAC;QACzC,OAAO,eAAe,CAAA;IACxB,CAAC;IAED,iCAAiC;IACjC,MAAM,QAAQ,GAAG,CAAC,GAAG,eAAe,CAAC,CAAA;IAErC,8BAA8B;IAC9B,KAAK,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAC5C;QAAA,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAE,EAAE,QAAQ,CAAC,CAAC,CAAE,CAAC,CAAA;IAC5D,CAAC;IAED,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAA;AACtC,CAAC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,UAAU,cAAc,CAA4B,MAAS;IACjE,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAgB,CAAA;IAE/C,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAA;IAE/B,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;QAChC,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAA;QACvB,OAAO,KAAK,GAAG,CAAC,GAAG,EAAE,MAAM,IAAI,CAAC,CAAC,CAAA;IACnC,CAAC,EAAE,CAAC,CAAC,CAAA;AACP,CAAC;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,UAAU,mBAAmB,CACjC,MAAS;IAKT,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAgB,CAAA;IAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;IAEzC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,EAAE,CAAA;IACX,CAAC;IAED,MAAM,YAAY,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAA;IAE7C,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAChC,MAAM,SAAS,GAAG,EAAsC,CAAA;QACxD,MAAM,SAAS,GAAG,EAAgC,CAAA;QAElD,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE;YACtB,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAA0B,CAAA;YAC/C,SAAS,CAAC,GAAG,CAAC,GAAG,KAAK,CAAA;YACtB,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAA;YACvB,SAAS,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAChD,CAAC,CAAC,CAAA;QAEF,OAAO;YACL,MAAM,EAAE,SAAS;YACjB,MAAM,EAAE,SAAS;SAClB,CAAA;IACH,CAAC,CAAC,CAAA;AACJ,CAAC"}
|
package/dist/chdb-storage.d.ts
DELETED
|
@@ -1,135 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* ClickHouse storage backend for AI experiments using chdb (embedded ClickHouse)
|
|
3
|
-
*
|
|
4
|
-
* Design: Single flat table where experiment events are the atomic unit.
|
|
5
|
-
* Cartesian product results, variant comparisons, and winner analysis
|
|
6
|
-
* are all derived via aggregation queries.
|
|
7
|
-
*
|
|
8
|
-
* This maximizes compression and enables fast analytical queries across
|
|
9
|
-
* thousands of experiment runs.
|
|
10
|
-
*/
|
|
11
|
-
import type { TrackingBackend, TrackingEvent, ExperimentResult } from './types.js';
|
|
12
|
-
export interface ChdbStorageOptions {
|
|
13
|
-
/** Path to store the chdb database (default: ./.experiments-chdb) */
|
|
14
|
-
dataPath?: string;
|
|
15
|
-
/** Whether to create tables on init (default: true) */
|
|
16
|
-
autoInit?: boolean;
|
|
17
|
-
}
|
|
18
|
-
export declare class ChdbStorage implements TrackingBackend {
|
|
19
|
-
private options;
|
|
20
|
-
private session;
|
|
21
|
-
private initialized;
|
|
22
|
-
private eventCounter;
|
|
23
|
-
constructor(options?: ChdbStorageOptions);
|
|
24
|
-
private init;
|
|
25
|
-
private escapeString;
|
|
26
|
-
private toJson;
|
|
27
|
-
private parseJson;
|
|
28
|
-
private nextEventId;
|
|
29
|
-
track(event: TrackingEvent): Promise<void>;
|
|
30
|
-
flush(): Promise<void>;
|
|
31
|
-
storeResult(result: ExperimentResult): Promise<void>;
|
|
32
|
-
storeResults(results: ExperimentResult[]): Promise<void>;
|
|
33
|
-
/**
|
|
34
|
-
* Get all experiments
|
|
35
|
-
*/
|
|
36
|
-
getExperiments(): Promise<Array<{
|
|
37
|
-
experimentId: string;
|
|
38
|
-
experimentName: string;
|
|
39
|
-
variantCount: number;
|
|
40
|
-
runCount: number;
|
|
41
|
-
firstRun: string;
|
|
42
|
-
lastRun: string;
|
|
43
|
-
}>>;
|
|
44
|
-
/**
|
|
45
|
-
* Get variant performance for an experiment
|
|
46
|
-
*/
|
|
47
|
-
getVariantStats(experimentId: string): Promise<Array<{
|
|
48
|
-
variantId: string;
|
|
49
|
-
variantName: string;
|
|
50
|
-
runCount: number;
|
|
51
|
-
successCount: number;
|
|
52
|
-
successRate: number;
|
|
53
|
-
avgDuration: number;
|
|
54
|
-
avgMetric: number;
|
|
55
|
-
minMetric: number;
|
|
56
|
-
maxMetric: number;
|
|
57
|
-
}>>;
|
|
58
|
-
/**
|
|
59
|
-
* Get the best performing variant for an experiment
|
|
60
|
-
*/
|
|
61
|
-
getBestVariant(experimentId: string, options?: {
|
|
62
|
-
metric?: 'avgMetric' | 'successRate' | 'avgDuration';
|
|
63
|
-
minimumRuns?: number;
|
|
64
|
-
}): Promise<{
|
|
65
|
-
variantId: string;
|
|
66
|
-
variantName: string;
|
|
67
|
-
metricValue: number;
|
|
68
|
-
runCount: number;
|
|
69
|
-
} | null>;
|
|
70
|
-
/**
|
|
71
|
-
* Get cartesian product analysis - performance by dimension values
|
|
72
|
-
*/
|
|
73
|
-
getCartesianAnalysis(experimentId: string, dimension: string): Promise<Array<{
|
|
74
|
-
dimensionValue: string;
|
|
75
|
-
runCount: number;
|
|
76
|
-
avgMetric: number;
|
|
77
|
-
successRate: number;
|
|
78
|
-
}>>;
|
|
79
|
-
/**
|
|
80
|
-
* Get multi-dimensional cartesian analysis
|
|
81
|
-
*/
|
|
82
|
-
getCartesianGrid(experimentId: string, dimensions: string[]): Promise<Array<{
|
|
83
|
-
dimensions: Record<string, string>;
|
|
84
|
-
runCount: number;
|
|
85
|
-
avgMetric: number;
|
|
86
|
-
successRate: number;
|
|
87
|
-
}>>;
|
|
88
|
-
/**
|
|
89
|
-
* Get time series of experiment metrics
|
|
90
|
-
*/
|
|
91
|
-
getTimeSeries(experimentId: string, options?: {
|
|
92
|
-
interval?: 'hour' | 'day' | 'week';
|
|
93
|
-
variantId?: string;
|
|
94
|
-
}): Promise<Array<{
|
|
95
|
-
period: string;
|
|
96
|
-
runCount: number;
|
|
97
|
-
avgMetric: number;
|
|
98
|
-
successRate: number;
|
|
99
|
-
}>>;
|
|
100
|
-
/**
|
|
101
|
-
* Get raw events for an experiment
|
|
102
|
-
*/
|
|
103
|
-
getEvents(experimentId: string, options?: {
|
|
104
|
-
eventType?: string;
|
|
105
|
-
variantId?: string;
|
|
106
|
-
limit?: number;
|
|
107
|
-
}): Promise<TrackingEvent[]>;
|
|
108
|
-
/**
|
|
109
|
-
* Raw SQL query access for custom analytics
|
|
110
|
-
*/
|
|
111
|
-
query(sql: string, format?: string): string;
|
|
112
|
-
/**
|
|
113
|
-
* Close the storage (cleanup)
|
|
114
|
-
*/
|
|
115
|
-
close(): void;
|
|
116
|
-
}
|
|
117
|
-
/**
|
|
118
|
-
* Create a chdb storage backend for experiments
|
|
119
|
-
*
|
|
120
|
-
* @example
|
|
121
|
-
* ```ts
|
|
122
|
-
* import { configureTracking } from 'ai-experiments'
|
|
123
|
-
* import { createChdbBackend } from 'ai-experiments/storage'
|
|
124
|
-
*
|
|
125
|
-
* const storage = createChdbBackend({ dataPath: './.my-experiments' })
|
|
126
|
-
*
|
|
127
|
-
* configureTracking({ backend: storage })
|
|
128
|
-
*
|
|
129
|
-
* // Later: analyze results
|
|
130
|
-
* const best = await storage.getBestVariant('my-experiment')
|
|
131
|
-
* console.log(`Best variant: ${best.variantName} (${best.metricValue})`)
|
|
132
|
-
* ```
|
|
133
|
-
*/
|
|
134
|
-
export declare function createChdbBackend(options?: ChdbStorageOptions): ChdbStorage;
|
|
135
|
-
//# sourceMappingURL=chdb-storage.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"chdb-storage.d.ts","sourceRoot":"","sources":["../src/chdb-storage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AA0ClF,MAAM,WAAW,kBAAkB;IACjC,qEAAqE;IACrE,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,uDAAuD;IACvD,QAAQ,CAAC,EAAE,OAAO,CAAA;CACnB;AAED,qBAAa,WAAY,YAAW,eAAe;IAKrC,OAAO,CAAC,OAAO;IAJ3B,OAAO,CAAC,OAAO,CAAS;IACxB,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,YAAY,CAAI;gBAEJ,OAAO,GAAE,kBAAuB;YAItC,IAAI;IAQlB,OAAO,CAAC,YAAY;IAIpB,OAAO,CAAC,MAAM;IAId,OAAO,CAAC,SAAS;IAQjB,OAAO,CAAC,WAAW;IAQb,KAAK,CAAC,KAAK,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC;IA+C1C,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAQtB,WAAW,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBpD,YAAY,CAAC,OAAO,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAU9D;;OAEG;IACG,cAAc,IAAI,OAAO,CAAC,KAAK,CAAC;QACpC,YAAY,EAAE,MAAM,CAAA;QACpB,cAAc,EAAE,MAAM,CAAA;QACtB,YAAY,EAAE,MAAM,CAAA;QACpB,QAAQ,EAAE,MAAM,CAAA;QAChB,QAAQ,EAAE,MAAM,CAAA;QAChB,OAAO,EAAE,MAAM,CAAA;KAChB,CAAC,CAAC;IAwBH;;OAEG;IACG,eAAe,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;QACzD,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,MAAM,CAAA;QAChB,YAAY,EAAE,MAAM,CAAA;QACpB,WAAW,EAAE,MAAM,CAAA;QACnB,WAAW,EAAE,MAAM,CAAA;QACnB,SAAS,EAAE,MAAM,CAAA;QACjB,SAAS,EAAE,MAAM,CAAA;QACjB,SAAS,EAAE,MAAM,CAAA;KAClB,CAAC,CAAC;IAwCH;;OAEG;IACG,cAAc,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,GAAE;QAClD,MAAM,CAAC,EAAE,WAAW,GAAG,aAAa,GAAG,aAAa,CAAA;QACpD,WAAW,CAAC,EAAE,MAAM,CAAA;KAChB,GAAG,OAAO,CAAC;QACf,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;QACnB,WAAW,EAAE,MAAM,CAAA;QACnB,QAAQ,EAAE,MAAM,CAAA;KACjB,GAAG,IAAI,CAAC;IAuCT;;OAEG;IACG,oBAAoB,CAAC,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;QACjF,cAAc,EAAE,MAAM,CAAA;QACtB,QAAQ,EAAE,MAAM,CAAA;QAChB,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;KACpB,CAAC,CAAC;IAgCH;;OAEG;IACG,gBAAgB,CAAC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC;QAChF,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAClC,QAAQ,EAAE,MAAM,CAAA;QAChB,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;KACpB,CAAC,CAAC;IAyCH;;OAEG;IACG,aAAa,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,GAAE;QACjD,QAAQ,CAAC,EAAE,MAAM,GAAG,KAAK,GAAG,MAAM,CAAA;QAClC,SAAS,CAAC,EAAE,MAAM,CAAA;KACd,GAAG,OAAO,CAAC,KAAK,CAAC;QACrB,MAAM,EAAE,MAAM,CAAA;QACd,QAAQ,EAAE,MAAM,CAAA;QAChB,SAAS,EAAE,MAAM,CAAA;QACjB,WAAW,EAAE,MAAM,CAAA;KACpB,CAAC,CAAC;IA2CH;;OAEG;IACG,SAAS,CAAC,YAAY,EAAE,MAAM,EAAE,OAAO,GAAE;QAC7C,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,SAAS,CAAC,EAAE,MAAM,CAAA;QAClB,KAAK,CAAC,EAAE,MAAM,CAAA;KACV,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IA2DjC;;OAEG;IACH,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,GAAE,MAAsB,GAAG,MAAM;IAI1D;;OAEG;IACH,KAAK,IAAI,IAAI;CAGd;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,iBAAiB,CAAC,OAAO,CAAC,EAAE,kBAAkB,GAAG,WAAW,CAE3E"}
|