ai-experiments 2.1.3 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/README.md +13 -1
- package/package.json +16 -16
- package/src/clickhouse-storage.ts +698 -0
- package/src/decide.ts +8 -10
- package/src/experiment.ts +12 -12
- package/src/index.ts +13 -3
- package/src/tracking.ts +3 -3
- package/test/cartesian.test.ts +287 -0
- package/test/clickhouse-storage.test.ts +470 -0
- package/test/decide.test.ts +414 -0
- package/test/experiment.test.ts +473 -0
- package/test/tracking.test.ts +347 -0
- package/vitest.config.ts +34 -0
- package/.turbo/turbo-build.log +0 -4
- package/LICENSE +0 -21
- package/dist/cartesian.d.ts +0 -140
- package/dist/cartesian.d.ts.map +0 -1
- package/dist/cartesian.js +0 -216
- package/dist/cartesian.js.map +0 -1
- package/dist/chdb-storage.d.ts +0 -135
- package/dist/chdb-storage.d.ts.map +0 -1
- package/dist/chdb-storage.js +0 -468
- package/dist/chdb-storage.js.map +0 -1
- package/dist/decide.d.ts +0 -152
- package/dist/decide.d.ts.map +0 -1
- package/dist/decide.js +0 -329
- package/dist/decide.js.map +0 -1
- package/dist/experiment.d.ts +0 -53
- package/dist/experiment.d.ts.map +0 -1
- package/dist/experiment.js +0 -292
- package/dist/experiment.js.map +0 -1
- package/dist/index.d.ts +0 -16
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -21
- package/dist/index.js.map +0 -1
- package/dist/tracking.d.ts +0 -159
- package/dist/tracking.d.ts.map +0 -1
- package/dist/tracking.js +0 -310
- package/dist/tracking.js.map +0 -1
- package/dist/types.d.ts +0 -198
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js +0 -5
- package/dist/types.js.map +0 -1
- package/src/cartesian.js +0 -215
- package/src/chdb-storage.js +0 -464
- package/src/chdb-storage.ts +0 -567
- package/src/decide.js +0 -328
- package/src/experiment.js +0 -291
- package/src/index.js +0 -20
- package/src/tracking.js +0 -309
- package/src/types.js +0 -4
|
@@ -0,0 +1,698 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ClickHouse storage backend for AI experiments.
|
|
3
|
+
*
|
|
4
|
+
* Migrated (per [ADR-0003](../../../docs/adr/0003-storage-strategy-pg-clickhouse-default.md))
|
|
5
|
+
* from the standalone embedded `chdb` adapter to the canonical
|
|
6
|
+
* `ClickHouseProvider` from `ai-database`. Experiment runs are recorded as
|
|
7
|
+
* SVO Actions on the canonical `actions` table:
|
|
8
|
+
*
|
|
9
|
+
* - `verb` — event type (e.g. `variant.complete`)
|
|
10
|
+
* - `subject` — `experimentId`
|
|
11
|
+
* - `object` — `variantId`
|
|
12
|
+
* - `roles` — `{ runId }` (other named roles can be carried here)
|
|
13
|
+
* - `data` — JSON payload (`experimentName`, `variantName`, `dimensions`,
|
|
14
|
+
* `runId`, `success`, `durationMs`, `result`, `metricName`,
|
|
15
|
+
* `metricValue`, `errorMessage`, `errorStack`, `metadata`)
|
|
16
|
+
* - `status` — `'completed'` for successful runs, `'failed'` otherwise
|
|
17
|
+
*
|
|
18
|
+
* Writes go through {@link ClickHouseProvider.recordAction} (and
|
|
19
|
+
* {@link ClickHouseProvider.commitBatch} when bulk-storing). Aggregations
|
|
20
|
+
* use {@link ClickHouseProvider.analyticsQuery} against the `actions` table
|
|
21
|
+
* with `JSONExtract*` against the `data` column. This is the **option (c)
|
|
22
|
+
* hybrid** approach from the migration plan: SVO surface for writes, raw
|
|
23
|
+
* analytical SQL for the experiment-specific aggregations the SVO surface
|
|
24
|
+
* doesn't expose directly (cartesian grids, time-series rollups,
|
|
25
|
+
* best-variant selection by metric).
|
|
26
|
+
*
|
|
27
|
+
* Deployment shift: the previous `chdb` adapter used embedded ClickHouse
|
|
28
|
+
* (file-system path). This adapter is HTTP-based — callers wire up a
|
|
29
|
+
* ClickHouse server (self-hosted or ClickHouse Cloud) via
|
|
30
|
+
* {@link createClickHouseHttpFetcher}. Tests use a fake fetcher.
|
|
31
|
+
*
|
|
32
|
+
* @packageDocumentation
|
|
33
|
+
*/
|
|
34
|
+
|
|
35
|
+
import {
|
|
36
|
+
type ClickHouseHttpFetcher,
|
|
37
|
+
type ClickHouseProvider,
|
|
38
|
+
createClickHouseProvider,
|
|
39
|
+
bootstrapClickHouseSchema,
|
|
40
|
+
} from 'ai-database'
|
|
41
|
+
import type { TrackingBackend, TrackingEvent, ExperimentResult } from './types.js'
|
|
42
|
+
|
|
43
|
+
/** Options for {@link ClickHouseExperimentStorage}. */
|
|
44
|
+
export interface ClickHouseExperimentStorageOptions {
|
|
45
|
+
/** HTTP fetcher (typically constructed via `createClickHouseHttpFetcher`). */
|
|
46
|
+
fetcher: ClickHouseHttpFetcher
|
|
47
|
+
/**
|
|
48
|
+
* ClickHouse database. Must already exist (or call
|
|
49
|
+
* {@link bootstrapExperimentsSchema}).
|
|
50
|
+
*
|
|
51
|
+
* @default 'aidb'
|
|
52
|
+
*/
|
|
53
|
+
database?: string
|
|
54
|
+
/**
|
|
55
|
+
* Namespace partition for experiment events.
|
|
56
|
+
*
|
|
57
|
+
* @default 'experiments'
|
|
58
|
+
*/
|
|
59
|
+
namespace?: string
|
|
60
|
+
/**
|
|
61
|
+
* Whether to bootstrap the canonical schema (`things`/`actions`/`verbs`)
|
|
62
|
+
* before the first write.
|
|
63
|
+
*
|
|
64
|
+
* @default false
|
|
65
|
+
*/
|
|
66
|
+
autoBootstrap?: boolean
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Backwards-compat alias for the prior `ChdbStorageOptions`. The
|
|
71
|
+
* `dataPath` and `autoInit` fields are retained as no-ops to ease the
|
|
72
|
+
* upgrade — they are ignored in favor of HTTP fetcher options.
|
|
73
|
+
*/
|
|
74
|
+
export interface ChdbStorageOptions extends Partial<ClickHouseExperimentStorageOptions> {
|
|
75
|
+
/** @deprecated Embedded chdb is no longer used; supply `fetcher`. */
|
|
76
|
+
dataPath?: string
|
|
77
|
+
/** @deprecated Replaced by `autoBootstrap`. */
|
|
78
|
+
autoInit?: boolean
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const DEFAULT_NAMESPACE = 'experiments'
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Generate a stable event id. Timestamp + monotonic counter inside the
|
|
85
|
+
* process — ClickHouse de-dupes on `actions.id` only via explicit DELETE,
|
|
86
|
+
* so collisions across processes are tolerated (each becomes a separate
|
|
87
|
+
* row).
|
|
88
|
+
*/
|
|
89
|
+
function makeEventIdFactory(): () => string {
|
|
90
|
+
let counter = 0
|
|
91
|
+
return () => `evt-${Date.now()}-${++counter}`
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
/**
|
|
95
|
+
* Map a {@link TrackingEvent} to the SVO Action shape.
|
|
96
|
+
*
|
|
97
|
+
* `verb` carries the event type (`variant.complete`, etc.).
|
|
98
|
+
* `subject` = `experimentId`, `object` = `variantId`. Everything else
|
|
99
|
+
* lands in `data` so analytical queries can `JSONExtract*` whatever they
|
|
100
|
+
* need without changing the schema.
|
|
101
|
+
*/
|
|
102
|
+
function eventToActionInput(
|
|
103
|
+
event: TrackingEvent,
|
|
104
|
+
eventId: string
|
|
105
|
+
): {
|
|
106
|
+
id: string
|
|
107
|
+
verb: string
|
|
108
|
+
subject: string
|
|
109
|
+
object: string
|
|
110
|
+
roles: Record<string, string>
|
|
111
|
+
data: Record<string, unknown>
|
|
112
|
+
status: 'completed' | 'failed' | 'pending'
|
|
113
|
+
} {
|
|
114
|
+
const data = event.data
|
|
115
|
+
const experimentId = String(data['experimentId'] ?? '')
|
|
116
|
+
const variantId = String(data['variantId'] ?? '')
|
|
117
|
+
const runId = String(data['runId'] ?? '')
|
|
118
|
+
const success = data['success'] !== false
|
|
119
|
+
const status: 'completed' | 'failed' | 'pending' =
|
|
120
|
+
event.type === 'variant.error' || !success ? 'failed' : 'completed'
|
|
121
|
+
|
|
122
|
+
// Capture every payload field — aggregation SQL uses JSONExtract* against
|
|
123
|
+
// the `data` JSON blob.
|
|
124
|
+
const errorMessage =
|
|
125
|
+
data['error'] instanceof Error
|
|
126
|
+
? data['error'].message
|
|
127
|
+
: data['errorMessage'] !== undefined
|
|
128
|
+
? String(data['errorMessage'])
|
|
129
|
+
: ''
|
|
130
|
+
const errorStack = data['error'] instanceof Error ? data['error'].stack ?? '' : ''
|
|
131
|
+
const durationMs = Number(data['duration'] ?? data['durationMs'] ?? 0)
|
|
132
|
+
|
|
133
|
+
const payload: Record<string, unknown> = {
|
|
134
|
+
eventId,
|
|
135
|
+
eventType: event.type,
|
|
136
|
+
timestamp: event.timestamp.toISOString(),
|
|
137
|
+
experimentId,
|
|
138
|
+
experimentName: String(data['experimentName'] ?? ''),
|
|
139
|
+
variantId,
|
|
140
|
+
variantName: String(data['variantName'] ?? ''),
|
|
141
|
+
runId,
|
|
142
|
+
success: success ? 1 : 0,
|
|
143
|
+
durationMs,
|
|
144
|
+
dimensions: data['dimensions'] ?? data['config'] ?? {},
|
|
145
|
+
result: data['result'] ?? {},
|
|
146
|
+
metricName: String(data['metricName'] ?? ''),
|
|
147
|
+
metricValue: Number(data['metricValue'] ?? 0),
|
|
148
|
+
errorMessage,
|
|
149
|
+
errorStack,
|
|
150
|
+
metadata: data['metadata'] ?? {},
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
return {
|
|
154
|
+
id: eventId,
|
|
155
|
+
verb: event.type,
|
|
156
|
+
subject: experimentId,
|
|
157
|
+
object: variantId,
|
|
158
|
+
roles: runId ? { runId } : {},
|
|
159
|
+
data: payload,
|
|
160
|
+
status,
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* ClickHouse-backed experiment storage built on the canonical
|
|
166
|
+
* {@link ClickHouseProvider} adapter. Implements
|
|
167
|
+
* {@link TrackingBackend}.
|
|
168
|
+
*
|
|
169
|
+
* @example
|
|
170
|
+
* ```ts
|
|
171
|
+
* import { configureTracking } from 'ai-experiments'
|
|
172
|
+
* import { createClickHouseExperimentStorage } from 'ai-experiments/storage'
|
|
173
|
+
* import { createClickHouseHttpFetcher } from 'ai-database'
|
|
174
|
+
*
|
|
175
|
+
* const fetcher = createClickHouseHttpFetcher({
|
|
176
|
+
* url: process.env.CLICKHOUSE_URL!,
|
|
177
|
+
* username: process.env.CLICKHOUSE_USER,
|
|
178
|
+
* password: process.env.CLICKHOUSE_PASSWORD,
|
|
179
|
+
* database: 'aidb',
|
|
180
|
+
* })
|
|
181
|
+
* const storage = createClickHouseExperimentStorage({ fetcher })
|
|
182
|
+
*
|
|
183
|
+
* configureTracking({ backend: storage })
|
|
184
|
+
*
|
|
185
|
+
* // Later: analyse results
|
|
186
|
+
* const best = await storage.getBestVariant('my-experiment')
|
|
187
|
+
* ```
|
|
188
|
+
*/
|
|
189
|
+
export class ClickHouseExperimentStorage implements TrackingBackend {
|
|
190
|
+
private readonly provider: ClickHouseProvider
|
|
191
|
+
private readonly fetcher: ClickHouseHttpFetcher
|
|
192
|
+
private readonly database: string
|
|
193
|
+
private readonly namespace: string
|
|
194
|
+
private readonly autoBootstrap: boolean
|
|
195
|
+
private bootstrapped = false
|
|
196
|
+
private readonly nextEventId = makeEventIdFactory()
|
|
197
|
+
|
|
198
|
+
constructor(options: ClickHouseExperimentStorageOptions) {
|
|
199
|
+
this.fetcher = options.fetcher
|
|
200
|
+
this.database = options.database ?? 'aidb'
|
|
201
|
+
this.namespace = options.namespace ?? DEFAULT_NAMESPACE
|
|
202
|
+
this.autoBootstrap = options.autoBootstrap ?? false
|
|
203
|
+
this.provider = createClickHouseProvider({
|
|
204
|
+
fetcher: options.fetcher,
|
|
205
|
+
database: this.database,
|
|
206
|
+
namespace: this.namespace,
|
|
207
|
+
})
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
/** Underlying canonical CH provider (escape hatch for advanced callers). */
|
|
211
|
+
getProvider(): ClickHouseProvider {
|
|
212
|
+
return this.provider
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
private async ensureBootstrap(): Promise<void> {
|
|
216
|
+
if (this.bootstrapped) return
|
|
217
|
+
if (this.autoBootstrap) {
|
|
218
|
+
await bootstrapClickHouseSchema(this.fetcher, { database: this.database })
|
|
219
|
+
}
|
|
220
|
+
this.bootstrapped = true
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
// ───────────────────────────────────────────────────────────────────────────
|
|
224
|
+
// TrackingBackend implementation
|
|
225
|
+
// ───────────────────────────────────────────────────────────────────────────
|
|
226
|
+
|
|
227
|
+
async track(event: TrackingEvent): Promise<void> {
|
|
228
|
+
await this.ensureBootstrap()
|
|
229
|
+
const eventId = this.nextEventId()
|
|
230
|
+
const action = eventToActionInput(event, eventId)
|
|
231
|
+
await this.provider.recordAction({
|
|
232
|
+
verb: action.verb,
|
|
233
|
+
subject: action.subject,
|
|
234
|
+
object: action.object,
|
|
235
|
+
roles: action.roles,
|
|
236
|
+
data: action.data,
|
|
237
|
+
status: action.status,
|
|
238
|
+
})
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
async flush(): Promise<void> {
|
|
242
|
+
// Writes go straight through HTTP — nothing to flush.
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// ───────────────────────────────────────────────────────────────────────────
|
|
246
|
+
// Direct experiment-result API
|
|
247
|
+
// ───────────────────────────────────────────────────────────────────────────
|
|
248
|
+
|
|
249
|
+
async storeResult(result: ExperimentResult): Promise<void> {
|
|
250
|
+
await this.track({
|
|
251
|
+
type: 'variant.complete',
|
|
252
|
+
timestamp: result.completedAt,
|
|
253
|
+
data: {
|
|
254
|
+
experimentId: result.experimentId,
|
|
255
|
+
variantId: result.variantId,
|
|
256
|
+
variantName: result.variantName,
|
|
257
|
+
runId: result.runId,
|
|
258
|
+
success: result.success,
|
|
259
|
+
duration: result.duration,
|
|
260
|
+
result: result.result,
|
|
261
|
+
metricValue: result.metricValue,
|
|
262
|
+
error: result.error,
|
|
263
|
+
metadata: result.metadata,
|
|
264
|
+
},
|
|
265
|
+
})
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Bulk-store results via {@link ClickHouseProvider.commitBatch}. One
|
|
270
|
+
* HTTP POST per call carries the entire batch as `JSONEachRow`.
|
|
271
|
+
*/
|
|
272
|
+
async storeResults(results: ExperimentResult[]): Promise<void> {
|
|
273
|
+
if (results.length === 0) return
|
|
274
|
+
await this.ensureBootstrap()
|
|
275
|
+
const actions = results.map((r) => {
|
|
276
|
+
const action = eventToActionInput(
|
|
277
|
+
{
|
|
278
|
+
type: 'variant.complete',
|
|
279
|
+
timestamp: r.completedAt,
|
|
280
|
+
data: {
|
|
281
|
+
experimentId: r.experimentId,
|
|
282
|
+
variantId: r.variantId,
|
|
283
|
+
variantName: r.variantName,
|
|
284
|
+
runId: r.runId,
|
|
285
|
+
success: r.success,
|
|
286
|
+
duration: r.duration,
|
|
287
|
+
result: r.result,
|
|
288
|
+
metricValue: r.metricValue,
|
|
289
|
+
error: r.error,
|
|
290
|
+
metadata: r.metadata,
|
|
291
|
+
},
|
|
292
|
+
},
|
|
293
|
+
this.nextEventId()
|
|
294
|
+
)
|
|
295
|
+
return action
|
|
296
|
+
})
|
|
297
|
+
await this.provider.commitBatch({ actions })
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// ───────────────────────────────────────────────────────────────────────────
|
|
301
|
+
// Analytics queries (raw SQL via canonical adapter's analyticsQuery)
|
|
302
|
+
// ───────────────────────────────────────────────────────────────────────────
|
|
303
|
+
|
|
304
|
+
/** Quote a SQL string literal — same shape as the canonical adapter. */
|
|
305
|
+
private quote(value: string): string {
|
|
306
|
+
return `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* Get all experiments seen in the actions table (variant.complete events).
|
|
311
|
+
*/
|
|
312
|
+
async getExperiments(): Promise<
|
|
313
|
+
Array<{
|
|
314
|
+
experimentId: string
|
|
315
|
+
experimentName: string
|
|
316
|
+
variantCount: number
|
|
317
|
+
runCount: number
|
|
318
|
+
firstRun: string
|
|
319
|
+
lastRun: string
|
|
320
|
+
}>
|
|
321
|
+
> {
|
|
322
|
+
await this.ensureBootstrap()
|
|
323
|
+
const ns = this.quote(this.namespace)
|
|
324
|
+
const sql = `
|
|
325
|
+
SELECT
|
|
326
|
+
subject AS experimentId,
|
|
327
|
+
any(JSONExtractString(data, 'experimentName')) AS experimentName,
|
|
328
|
+
uniq(object) AS variantCount,
|
|
329
|
+
count() AS runCount,
|
|
330
|
+
toString(min(created_at)) AS firstRun,
|
|
331
|
+
toString(max(created_at)) AS lastRun
|
|
332
|
+
FROM ${this.database}.actions
|
|
333
|
+
WHERE ns = ${ns} AND verb = 'variant.complete' AND subject != ''
|
|
334
|
+
GROUP BY subject
|
|
335
|
+
ORDER BY lastRun DESC
|
|
336
|
+
`
|
|
337
|
+
const rows = await this.provider.analyticsQuery(sql)
|
|
338
|
+
return rows.map((row) => ({
|
|
339
|
+
experimentId: String(row['experimentId'] ?? ''),
|
|
340
|
+
experimentName: String(row['experimentName'] ?? ''),
|
|
341
|
+
variantCount: Number(row['variantCount'] ?? 0),
|
|
342
|
+
runCount: Number(row['runCount'] ?? 0),
|
|
343
|
+
firstRun: String(row['firstRun'] ?? ''),
|
|
344
|
+
lastRun: String(row['lastRun'] ?? ''),
|
|
345
|
+
}))
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
async getVariantStats(experimentId: string): Promise<
|
|
349
|
+
Array<{
|
|
350
|
+
variantId: string
|
|
351
|
+
variantName: string
|
|
352
|
+
runCount: number
|
|
353
|
+
successCount: number
|
|
354
|
+
successRate: number
|
|
355
|
+
avgDuration: number
|
|
356
|
+
avgMetric: number
|
|
357
|
+
minMetric: number
|
|
358
|
+
maxMetric: number
|
|
359
|
+
}>
|
|
360
|
+
> {
|
|
361
|
+
await this.ensureBootstrap()
|
|
362
|
+
const ns = this.quote(this.namespace)
|
|
363
|
+
const exp = this.quote(experimentId)
|
|
364
|
+
const sql = `
|
|
365
|
+
SELECT
|
|
366
|
+
object AS variantId,
|
|
367
|
+
any(JSONExtractString(data, 'variantName')) AS variantName,
|
|
368
|
+
count() AS runCount,
|
|
369
|
+
countIf(JSONExtractInt(data, 'success') = 1) AS successCount,
|
|
370
|
+
countIf(JSONExtractInt(data, 'success') = 1) / count() AS successRate,
|
|
371
|
+
avg(JSONExtractFloat(data, 'durationMs')) AS avgDuration,
|
|
372
|
+
avg(JSONExtractFloat(data, 'metricValue')) AS avgMetric,
|
|
373
|
+
min(JSONExtractFloat(data, 'metricValue')) AS minMetric,
|
|
374
|
+
max(JSONExtractFloat(data, 'metricValue')) AS maxMetric
|
|
375
|
+
FROM ${this.database}.actions
|
|
376
|
+
WHERE ns = ${ns} AND verb = 'variant.complete' AND subject = ${exp}
|
|
377
|
+
GROUP BY object
|
|
378
|
+
ORDER BY avgMetric DESC
|
|
379
|
+
`
|
|
380
|
+
const rows = await this.provider.analyticsQuery(sql)
|
|
381
|
+
return rows.map((row) => ({
|
|
382
|
+
variantId: String(row['variantId'] ?? ''),
|
|
383
|
+
variantName: String(row['variantName'] ?? ''),
|
|
384
|
+
runCount: Number(row['runCount'] ?? 0),
|
|
385
|
+
successCount: Number(row['successCount'] ?? 0),
|
|
386
|
+
successRate: Number(row['successRate'] ?? 0),
|
|
387
|
+
avgDuration: Number(row['avgDuration'] ?? 0),
|
|
388
|
+
avgMetric: Number(row['avgMetric'] ?? 0),
|
|
389
|
+
minMetric: Number(row['minMetric'] ?? 0),
|
|
390
|
+
maxMetric: Number(row['maxMetric'] ?? 0),
|
|
391
|
+
}))
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
async getBestVariant(
|
|
395
|
+
experimentId: string,
|
|
396
|
+
options: {
|
|
397
|
+
metric?: 'avgMetric' | 'successRate' | 'avgDuration'
|
|
398
|
+
minimumRuns?: number
|
|
399
|
+
} = {}
|
|
400
|
+
): Promise<{
|
|
401
|
+
variantId: string
|
|
402
|
+
variantName: string
|
|
403
|
+
metricValue: number
|
|
404
|
+
runCount: number
|
|
405
|
+
} | null> {
|
|
406
|
+
await this.ensureBootstrap()
|
|
407
|
+
const { metric = 'avgMetric', minimumRuns = 1 } = options
|
|
408
|
+
const ns = this.quote(this.namespace)
|
|
409
|
+
const exp = this.quote(experimentId)
|
|
410
|
+
const orderBy = metric === 'avgDuration' ? 'ASC' : 'DESC'
|
|
411
|
+
const metricExpr =
|
|
412
|
+
metric === 'successRate'
|
|
413
|
+
? "countIf(JSONExtractInt(data, 'success') = 1) / count()"
|
|
414
|
+
: metric === 'avgDuration'
|
|
415
|
+
? "avg(JSONExtractFloat(data, 'durationMs'))"
|
|
416
|
+
: "avg(JSONExtractFloat(data, 'metricValue'))"
|
|
417
|
+
|
|
418
|
+
const sql = `
|
|
419
|
+
SELECT
|
|
420
|
+
object AS variantId,
|
|
421
|
+
any(JSONExtractString(data, 'variantName')) AS variantName,
|
|
422
|
+
${metricExpr} AS metricValue,
|
|
423
|
+
count() AS runCount
|
|
424
|
+
FROM ${this.database}.actions
|
|
425
|
+
WHERE ns = ${ns} AND verb = 'variant.complete' AND subject = ${exp}
|
|
426
|
+
GROUP BY object
|
|
427
|
+
HAVING runCount >= ${Number(minimumRuns)}
|
|
428
|
+
ORDER BY metricValue ${orderBy}
|
|
429
|
+
LIMIT 1
|
|
430
|
+
`
|
|
431
|
+
const rows = await this.provider.analyticsQuery(sql)
|
|
432
|
+
if (rows.length === 0) return null
|
|
433
|
+
const row = rows[0]!
|
|
434
|
+
return {
|
|
435
|
+
variantId: String(row['variantId'] ?? ''),
|
|
436
|
+
variantName: String(row['variantName'] ?? ''),
|
|
437
|
+
metricValue: Number(row['metricValue'] ?? 0),
|
|
438
|
+
runCount: Number(row['runCount'] ?? 0),
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
async getCartesianAnalysis(
|
|
443
|
+
experimentId: string,
|
|
444
|
+
dimension: string
|
|
445
|
+
): Promise<
|
|
446
|
+
Array<{
|
|
447
|
+
dimensionValue: string
|
|
448
|
+
runCount: number
|
|
449
|
+
avgMetric: number
|
|
450
|
+
successRate: number
|
|
451
|
+
}>
|
|
452
|
+
> {
|
|
453
|
+
await this.ensureBootstrap()
|
|
454
|
+
const ns = this.quote(this.namespace)
|
|
455
|
+
const exp = this.quote(experimentId)
|
|
456
|
+
const dimQuoted = this.quote(dimension)
|
|
457
|
+
const sql = `
|
|
458
|
+
SELECT
|
|
459
|
+
JSONExtractString(JSONExtractRaw(data, 'dimensions'), ${dimQuoted}) AS dimensionValue,
|
|
460
|
+
count() AS runCount,
|
|
461
|
+
avg(JSONExtractFloat(data, 'metricValue')) AS avgMetric,
|
|
462
|
+
countIf(JSONExtractInt(data, 'success') = 1) / count() AS successRate
|
|
463
|
+
FROM ${this.database}.actions
|
|
464
|
+
WHERE ns = ${ns} AND verb = 'variant.complete' AND subject = ${exp}
|
|
465
|
+
GROUP BY dimensionValue
|
|
466
|
+
HAVING dimensionValue != ''
|
|
467
|
+
ORDER BY avgMetric DESC
|
|
468
|
+
`
|
|
469
|
+
const rows = await this.provider.analyticsQuery(sql)
|
|
470
|
+
return rows.map((row) => ({
|
|
471
|
+
dimensionValue: String(row['dimensionValue'] ?? ''),
|
|
472
|
+
runCount: Number(row['runCount'] ?? 0),
|
|
473
|
+
avgMetric: Number(row['avgMetric'] ?? 0),
|
|
474
|
+
successRate: Number(row['successRate'] ?? 0),
|
|
475
|
+
}))
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
async getCartesianGrid(
|
|
479
|
+
experimentId: string,
|
|
480
|
+
dimensions: string[]
|
|
481
|
+
): Promise<
|
|
482
|
+
Array<{
|
|
483
|
+
dimensions: Record<string, string>
|
|
484
|
+
runCount: number
|
|
485
|
+
avgMetric: number
|
|
486
|
+
successRate: number
|
|
487
|
+
}>
|
|
488
|
+
> {
|
|
489
|
+
await this.ensureBootstrap()
|
|
490
|
+
const ns = this.quote(this.namespace)
|
|
491
|
+
const exp = this.quote(experimentId)
|
|
492
|
+
const dimExtracts = dimensions
|
|
493
|
+
.map(
|
|
494
|
+
(d) => `JSONExtractString(JSONExtractRaw(data, 'dimensions'), ${this.quote(d)}) AS dim_${d}`
|
|
495
|
+
)
|
|
496
|
+
.join(', ')
|
|
497
|
+
const dimGroupBy = dimensions.map((d) => `dim_${d}`).join(', ')
|
|
498
|
+
|
|
499
|
+
const sql = `
|
|
500
|
+
SELECT
|
|
501
|
+
${dimExtracts},
|
|
502
|
+
count() AS runCount,
|
|
503
|
+
avg(JSONExtractFloat(data, 'metricValue')) AS avgMetric,
|
|
504
|
+
countIf(JSONExtractInt(data, 'success') = 1) / count() AS successRate
|
|
505
|
+
FROM ${this.database}.actions
|
|
506
|
+
WHERE ns = ${ns} AND verb = 'variant.complete' AND subject = ${exp}
|
|
507
|
+
GROUP BY ${dimGroupBy}
|
|
508
|
+
ORDER BY avgMetric DESC
|
|
509
|
+
`
|
|
510
|
+
const rows = await this.provider.analyticsQuery(sql)
|
|
511
|
+
return rows.map((row) => {
|
|
512
|
+
const dims: Record<string, string> = {}
|
|
513
|
+
for (const d of dimensions) {
|
|
514
|
+
dims[d] = String(row[`dim_${d}`] ?? '')
|
|
515
|
+
}
|
|
516
|
+
return {
|
|
517
|
+
dimensions: dims,
|
|
518
|
+
runCount: Number(row['runCount'] ?? 0),
|
|
519
|
+
avgMetric: Number(row['avgMetric'] ?? 0),
|
|
520
|
+
successRate: Number(row['successRate'] ?? 0),
|
|
521
|
+
}
|
|
522
|
+
})
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
async getTimeSeries(
|
|
526
|
+
experimentId: string,
|
|
527
|
+
options: {
|
|
528
|
+
interval?: 'hour' | 'day' | 'week'
|
|
529
|
+
variantId?: string
|
|
530
|
+
} = {}
|
|
531
|
+
): Promise<
|
|
532
|
+
Array<{
|
|
533
|
+
period: string
|
|
534
|
+
runCount: number
|
|
535
|
+
avgMetric: number
|
|
536
|
+
successRate: number
|
|
537
|
+
}>
|
|
538
|
+
> {
|
|
539
|
+
await this.ensureBootstrap()
|
|
540
|
+
const { interval = 'day', variantId } = options
|
|
541
|
+
const ns = this.quote(this.namespace)
|
|
542
|
+
const exp = this.quote(experimentId)
|
|
543
|
+
const dateFunc =
|
|
544
|
+
interval === 'hour'
|
|
545
|
+
? 'toStartOfHour(created_at)'
|
|
546
|
+
: interval === 'week'
|
|
547
|
+
? 'toStartOfWeek(created_at)'
|
|
548
|
+
: 'toStartOfDay(created_at)'
|
|
549
|
+
const variantFilter = variantId ? `AND object = ${this.quote(variantId)}` : ''
|
|
550
|
+
|
|
551
|
+
const sql = `
|
|
552
|
+
SELECT
|
|
553
|
+
toString(${dateFunc}) AS period,
|
|
554
|
+
count() AS runCount,
|
|
555
|
+
avg(JSONExtractFloat(data, 'metricValue')) AS avgMetric,
|
|
556
|
+
countIf(JSONExtractInt(data, 'success') = 1) / count() AS successRate
|
|
557
|
+
FROM ${this.database}.actions
|
|
558
|
+
WHERE ns = ${ns} AND verb = 'variant.complete' AND subject = ${exp}
|
|
559
|
+
${variantFilter}
|
|
560
|
+
GROUP BY period
|
|
561
|
+
ORDER BY period ASC
|
|
562
|
+
`
|
|
563
|
+
const rows = await this.provider.analyticsQuery(sql)
|
|
564
|
+
return rows.map((row) => ({
|
|
565
|
+
period: String(row['period'] ?? ''),
|
|
566
|
+
runCount: Number(row['runCount'] ?? 0),
|
|
567
|
+
avgMetric: Number(row['avgMetric'] ?? 0),
|
|
568
|
+
successRate: Number(row['successRate'] ?? 0),
|
|
569
|
+
}))
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
async getEvents(
|
|
573
|
+
experimentId: string,
|
|
574
|
+
options: {
|
|
575
|
+
eventType?: string
|
|
576
|
+
variantId?: string
|
|
577
|
+
limit?: number
|
|
578
|
+
} = {}
|
|
579
|
+
): Promise<TrackingEvent[]> {
|
|
580
|
+
await this.ensureBootstrap()
|
|
581
|
+
const { eventType, variantId, limit = 100 } = options
|
|
582
|
+
const ns = this.quote(this.namespace)
|
|
583
|
+
const exp = this.quote(experimentId)
|
|
584
|
+
const filters = [`ns = ${ns}`, `subject = ${exp}`]
|
|
585
|
+
if (eventType) filters.push(`verb = ${this.quote(eventType)}`)
|
|
586
|
+
if (variantId) filters.push(`object = ${this.quote(variantId)}`)
|
|
587
|
+
|
|
588
|
+
const sql = `
|
|
589
|
+
SELECT
|
|
590
|
+
verb AS eventType,
|
|
591
|
+
toString(created_at) AS createdAt,
|
|
592
|
+
data
|
|
593
|
+
FROM ${this.database}.actions
|
|
594
|
+
WHERE ${filters.join(' AND ')}
|
|
595
|
+
ORDER BY created_at DESC
|
|
596
|
+
LIMIT ${Number(limit)}
|
|
597
|
+
`
|
|
598
|
+
const rows = await this.provider.analyticsQuery(sql)
|
|
599
|
+
return rows.map((row) => {
|
|
600
|
+
const dataStr = row['data']
|
|
601
|
+
let payload: Record<string, unknown> = {}
|
|
602
|
+
if (typeof dataStr === 'string' && dataStr.length > 0) {
|
|
603
|
+
try {
|
|
604
|
+
payload = JSON.parse(dataStr) as Record<string, unknown>
|
|
605
|
+
} catch {
|
|
606
|
+
payload = {}
|
|
607
|
+
}
|
|
608
|
+
} else if (dataStr && typeof dataStr === 'object') {
|
|
609
|
+
payload = dataStr as Record<string, unknown>
|
|
610
|
+
}
|
|
611
|
+
const createdAt = String(row['createdAt'] ?? '')
|
|
612
|
+
const ts = createdAt ? new Date(createdAt) : new Date()
|
|
613
|
+
return {
|
|
614
|
+
type: String(row['eventType'] ?? '') as TrackingEvent['type'],
|
|
615
|
+
timestamp: Number.isNaN(ts.getTime()) ? new Date() : ts,
|
|
616
|
+
data: payload,
|
|
617
|
+
}
|
|
618
|
+
})
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
/**
|
|
622
|
+
* Raw analytical SQL pass-through. Routes to
|
|
623
|
+
* {@link ClickHouseProvider.analyticsQuery}.
|
|
624
|
+
*/
|
|
625
|
+
async query(sql: string): Promise<Array<Record<string, unknown>>> {
|
|
626
|
+
return this.provider.analyticsQuery(sql)
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
/** No-op (kept for backwards compat with the chdb-era API). */
|
|
630
|
+
close(): void {
|
|
631
|
+
// HTTP fetcher has nothing to release.
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
/**
|
|
636
|
+
* Bootstrap the canonical CH schema (`things`, `actions`, `verbs`).
|
|
637
|
+
* Convenience wrapper around the canonical
|
|
638
|
+
* {@link bootstrapClickHouseSchema}.
|
|
639
|
+
*/
|
|
640
|
+
export async function bootstrapExperimentsSchema(
|
|
641
|
+
fetcher: ClickHouseHttpFetcher,
|
|
642
|
+
options: { database?: string } = {}
|
|
643
|
+
): Promise<void> {
|
|
644
|
+
await bootstrapClickHouseSchema(
|
|
645
|
+
fetcher,
|
|
646
|
+
options.database !== undefined ? { database: options.database } : {}
|
|
647
|
+
)
|
|
648
|
+
}
|
|
649
|
+
|
|
650
|
+
/**
|
|
651
|
+
* Create a ClickHouse-backed experiment storage backend.
|
|
652
|
+
*
|
|
653
|
+
* @example
|
|
654
|
+
* ```ts
|
|
655
|
+
* import { configureTracking } from 'ai-experiments'
|
|
656
|
+
* import { createClickHouseExperimentStorage } from 'ai-experiments/storage'
|
|
657
|
+
* import { createClickHouseHttpFetcher } from 'ai-database'
|
|
658
|
+
*
|
|
659
|
+
* const fetcher = createClickHouseHttpFetcher({
|
|
660
|
+
* url: process.env.CLICKHOUSE_URL!,
|
|
661
|
+
* username: process.env.CLICKHOUSE_USER,
|
|
662
|
+
* password: process.env.CLICKHOUSE_PASSWORD,
|
|
663
|
+
* database: 'aidb',
|
|
664
|
+
* })
|
|
665
|
+
* const storage = createClickHouseExperimentStorage({ fetcher })
|
|
666
|
+
*
|
|
667
|
+
* configureTracking({ backend: storage })
|
|
668
|
+
*
|
|
669
|
+
* const best = await storage.getBestVariant('my-experiment')
|
|
670
|
+
* ```
|
|
671
|
+
*/
|
|
672
|
+
export function createClickHouseExperimentStorage(
|
|
673
|
+
options: ClickHouseExperimentStorageOptions
|
|
674
|
+
): ClickHouseExperimentStorage {
|
|
675
|
+
return new ClickHouseExperimentStorage(options)
|
|
676
|
+
}
|
|
677
|
+
|
|
678
|
+
// =============================================================================
|
|
679
|
+
// Backwards-compat aliases (chdb-era names)
|
|
680
|
+
// =============================================================================
|
|
681
|
+
|
|
682
|
+
/**
|
|
683
|
+
* @deprecated Renamed to {@link ClickHouseExperimentStorage}. The class
|
|
684
|
+
* now wraps the canonical `ClickHouseProvider` from `ai-database` instead
|
|
685
|
+
* of embedded chdb. The old `dataPath` constructor option is ignored —
|
|
686
|
+
* supply a `fetcher` instead.
|
|
687
|
+
*/
|
|
688
|
+
export const ChdbStorage = ClickHouseExperimentStorage
|
|
689
|
+
export type ChdbStorage = ClickHouseExperimentStorage
|
|
690
|
+
|
|
691
|
+
/**
|
|
692
|
+
* @deprecated Renamed to {@link createClickHouseExperimentStorage}.
|
|
693
|
+
*/
|
|
694
|
+
export function createChdbBackend(
|
|
695
|
+
options: ClickHouseExperimentStorageOptions
|
|
696
|
+
): ClickHouseExperimentStorage {
|
|
697
|
+
return createClickHouseExperimentStorage(options)
|
|
698
|
+
}
|