ai-experiments 2.1.1 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/CHANGELOG.md +16 -0
  2. package/README.md +13 -1
  3. package/package.json +7 -7
  4. package/src/clickhouse-storage.ts +698 -0
  5. package/src/decide.ts +8 -10
  6. package/src/experiment.ts +12 -12
  7. package/src/index.ts +13 -3
  8. package/src/tracking.ts +3 -3
  9. package/test/cartesian.test.ts +287 -0
  10. package/test/clickhouse-storage.test.ts +470 -0
  11. package/test/decide.test.ts +414 -0
  12. package/test/experiment.test.ts +473 -0
  13. package/test/tracking.test.ts +347 -0
  14. package/vitest.config.ts +34 -0
  15. package/.turbo/turbo-build.log +0 -5
  16. package/dist/cartesian.d.ts +0 -140
  17. package/dist/cartesian.d.ts.map +0 -1
  18. package/dist/cartesian.js +0 -216
  19. package/dist/cartesian.js.map +0 -1
  20. package/dist/decide.d.ts +0 -152
  21. package/dist/decide.d.ts.map +0 -1
  22. package/dist/decide.js +0 -329
  23. package/dist/decide.js.map +0 -1
  24. package/dist/experiment.d.ts +0 -53
  25. package/dist/experiment.d.ts.map +0 -1
  26. package/dist/experiment.js +0 -292
  27. package/dist/experiment.js.map +0 -1
  28. package/dist/index.d.ts +0 -14
  29. package/dist/index.d.ts.map +0 -1
  30. package/dist/index.js +0 -19
  31. package/dist/index.js.map +0 -1
  32. package/dist/tracking.d.ts +0 -159
  33. package/dist/tracking.d.ts.map +0 -1
  34. package/dist/tracking.js +0 -310
  35. package/dist/tracking.js.map +0 -1
  36. package/dist/types.d.ts +0 -198
  37. package/dist/types.d.ts.map +0 -1
  38. package/dist/types.js +0 -5
  39. package/dist/types.js.map +0 -1
  40. package/src/cartesian.js +0 -215
  41. package/src/chdb-storage.js +0 -464
  42. package/src/chdb-storage.ts +0 -565
  43. package/src/decide.js +0 -328
  44. package/src/experiment.js +0 -291
  45. package/src/index.js +0 -20
  46. package/src/tracking.js +0 -309
  47. package/src/types.js +0 -4
@@ -0,0 +1,698 @@
1
+ /**
2
+ * ClickHouse storage backend for AI experiments.
3
+ *
4
+ * Migrated (per [ADR-0003](../../../docs/adr/0003-storage-strategy-pg-clickhouse-default.md))
5
+ * from the standalone embedded `chdb` adapter to the canonical
6
+ * `ClickHouseProvider` from `ai-database`. Experiment runs are recorded as
7
+ * SVO Actions on the canonical `actions` table:
8
+ *
9
+ * - `verb` — event type (e.g. `variant.complete`)
10
+ * - `subject` — `experimentId`
11
+ * - `object` — `variantId`
12
+ * - `roles` — `{ runId }` (other named roles can be carried here)
13
+ * - `data` — JSON payload (`experimentName`, `variantName`, `dimensions`,
14
+ * `runId`, `success`, `durationMs`, `result`, `metricName`,
15
+ * `metricValue`, `errorMessage`, `errorStack`, `metadata`)
16
+ * - `status` — `'completed'` for successful runs, `'failed'` otherwise
17
+ *
18
+ * Writes go through {@link ClickHouseProvider.recordAction} (and
19
+ * {@link ClickHouseProvider.commitBatch} when bulk-storing). Aggregations
20
+ * use {@link ClickHouseProvider.analyticsQuery} against the `actions` table
21
+ * with `JSONExtract*` against the `data` column. This is the **option (c)
22
+ * hybrid** approach from the migration plan: SVO surface for writes, raw
23
+ * analytical SQL for the experiment-specific aggregations the SVO surface
24
+ * doesn't expose directly (cartesian grids, time-series rollups,
25
+ * best-variant selection by metric).
26
+ *
27
+ * Deployment shift: the previous `chdb` adapter used embedded ClickHouse
28
+ * (file-system path). This adapter is HTTP-based — callers wire up a
29
+ * ClickHouse server (self-hosted or ClickHouse Cloud) via
30
+ * {@link createClickHouseHttpFetcher}. Tests use a fake fetcher.
31
+ *
32
+ * @packageDocumentation
33
+ */
34
+
35
+ import {
36
+ type ClickHouseHttpFetcher,
37
+ type ClickHouseProvider,
38
+ createClickHouseProvider,
39
+ bootstrapClickHouseSchema,
40
+ } from 'ai-database'
41
+ import type { TrackingBackend, TrackingEvent, ExperimentResult } from './types.js'
42
+
43
+ /** Options for {@link ClickHouseExperimentStorage}. */
44
+ export interface ClickHouseExperimentStorageOptions {
45
+ /** HTTP fetcher (typically constructed via `createClickHouseHttpFetcher`). */
46
+ fetcher: ClickHouseHttpFetcher
47
+ /**
48
+ * ClickHouse database. Must already exist (or call
49
+ * {@link bootstrapExperimentsSchema}).
50
+ *
51
+ * @default 'aidb'
52
+ */
53
+ database?: string
54
+ /**
55
+ * Namespace partition for experiment events.
56
+ *
57
+ * @default 'experiments'
58
+ */
59
+ namespace?: string
60
+ /**
61
+ * Whether to bootstrap the canonical schema (`things`/`actions`/`verbs`)
62
+ * before the first write.
63
+ *
64
+ * @default false
65
+ */
66
+ autoBootstrap?: boolean
67
+ }
68
+
69
+ /**
70
+ * Backwards-compat alias for the prior `ChdbStorageOptions`. The
71
+ * `dataPath` and `autoInit` fields are retained as no-ops to ease the
72
+ * upgrade — they are ignored in favor of HTTP fetcher options.
73
+ */
74
+ export interface ChdbStorageOptions extends Partial<ClickHouseExperimentStorageOptions> {
75
+ /** @deprecated Embedded chdb is no longer used; supply `fetcher`. */
76
+ dataPath?: string
77
+ /** @deprecated Replaced by `autoBootstrap`. */
78
+ autoInit?: boolean
79
+ }
80
+
81
+ const DEFAULT_NAMESPACE = 'experiments'
82
+
83
+ /**
84
+ * Generate a stable event id. Timestamp + monotonic counter inside the
85
+ * process — ClickHouse de-dupes on `actions.id` only via explicit DELETE,
86
+ * so collisions across processes are tolerated (each becomes a separate
87
+ * row).
88
+ */
89
+ function makeEventIdFactory(): () => string {
90
+ let counter = 0
91
+ return () => `evt-${Date.now()}-${++counter}`
92
+ }
93
+
94
+ /**
95
+ * Map a {@link TrackingEvent} to the SVO Action shape.
96
+ *
97
+ * `verb` carries the event type (`variant.complete`, etc.).
98
+ * `subject` = `experimentId`, `object` = `variantId`. Everything else
99
+ * lands in `data` so analytical queries can `JSONExtract*` whatever they
100
+ * need without changing the schema.
101
+ */
102
+ function eventToActionInput(
103
+ event: TrackingEvent,
104
+ eventId: string
105
+ ): {
106
+ id: string
107
+ verb: string
108
+ subject: string
109
+ object: string
110
+ roles: Record<string, string>
111
+ data: Record<string, unknown>
112
+ status: 'completed' | 'failed' | 'pending'
113
+ } {
114
+ const data = event.data
115
+ const experimentId = String(data['experimentId'] ?? '')
116
+ const variantId = String(data['variantId'] ?? '')
117
+ const runId = String(data['runId'] ?? '')
118
+ const success = data['success'] !== false
119
+ const status: 'completed' | 'failed' | 'pending' =
120
+ event.type === 'variant.error' || !success ? 'failed' : 'completed'
121
+
122
+ // Capture every payload field — aggregation SQL uses JSONExtract* against
123
+ // the `data` JSON blob.
124
+ const errorMessage =
125
+ data['error'] instanceof Error
126
+ ? data['error'].message
127
+ : data['errorMessage'] !== undefined
128
+ ? String(data['errorMessage'])
129
+ : ''
130
+ const errorStack = data['error'] instanceof Error ? data['error'].stack ?? '' : ''
131
+ const durationMs = Number(data['duration'] ?? data['durationMs'] ?? 0)
132
+
133
+ const payload: Record<string, unknown> = {
134
+ eventId,
135
+ eventType: event.type,
136
+ timestamp: event.timestamp.toISOString(),
137
+ experimentId,
138
+ experimentName: String(data['experimentName'] ?? ''),
139
+ variantId,
140
+ variantName: String(data['variantName'] ?? ''),
141
+ runId,
142
+ success: success ? 1 : 0,
143
+ durationMs,
144
+ dimensions: data['dimensions'] ?? data['config'] ?? {},
145
+ result: data['result'] ?? {},
146
+ metricName: String(data['metricName'] ?? ''),
147
+ metricValue: Number(data['metricValue'] ?? 0),
148
+ errorMessage,
149
+ errorStack,
150
+ metadata: data['metadata'] ?? {},
151
+ }
152
+
153
+ return {
154
+ id: eventId,
155
+ verb: event.type,
156
+ subject: experimentId,
157
+ object: variantId,
158
+ roles: runId ? { runId } : {},
159
+ data: payload,
160
+ status,
161
+ }
162
+ }
163
+
164
+ /**
165
+ * ClickHouse-backed experiment storage built on the canonical
166
+ * {@link ClickHouseProvider} adapter. Implements
167
+ * {@link TrackingBackend}.
168
+ *
169
+ * @example
170
+ * ```ts
171
+ * import { configureTracking } from 'ai-experiments'
172
+ * import { createClickHouseExperimentStorage } from 'ai-experiments/storage'
173
+ * import { createClickHouseHttpFetcher } from 'ai-database'
174
+ *
175
+ * const fetcher = createClickHouseHttpFetcher({
176
+ * url: process.env.CLICKHOUSE_URL!,
177
+ * username: process.env.CLICKHOUSE_USER,
178
+ * password: process.env.CLICKHOUSE_PASSWORD,
179
+ * database: 'aidb',
180
+ * })
181
+ * const storage = createClickHouseExperimentStorage({ fetcher })
182
+ *
183
+ * configureTracking({ backend: storage })
184
+ *
185
+ * // Later: analyse results
186
+ * const best = await storage.getBestVariant('my-experiment')
187
+ * ```
188
+ */
189
+ export class ClickHouseExperimentStorage implements TrackingBackend {
190
+ private readonly provider: ClickHouseProvider
191
+ private readonly fetcher: ClickHouseHttpFetcher
192
+ private readonly database: string
193
+ private readonly namespace: string
194
+ private readonly autoBootstrap: boolean
195
+ private bootstrapped = false
196
+ private readonly nextEventId = makeEventIdFactory()
197
+
198
+ constructor(options: ClickHouseExperimentStorageOptions) {
199
+ this.fetcher = options.fetcher
200
+ this.database = options.database ?? 'aidb'
201
+ this.namespace = options.namespace ?? DEFAULT_NAMESPACE
202
+ this.autoBootstrap = options.autoBootstrap ?? false
203
+ this.provider = createClickHouseProvider({
204
+ fetcher: options.fetcher,
205
+ database: this.database,
206
+ namespace: this.namespace,
207
+ })
208
+ }
209
+
210
+ /** Underlying canonical CH provider (escape hatch for advanced callers). */
211
+ getProvider(): ClickHouseProvider {
212
+ return this.provider
213
+ }
214
+
215
+ private async ensureBootstrap(): Promise<void> {
216
+ if (this.bootstrapped) return
217
+ if (this.autoBootstrap) {
218
+ await bootstrapClickHouseSchema(this.fetcher, { database: this.database })
219
+ }
220
+ this.bootstrapped = true
221
+ }
222
+
223
+ // ───────────────────────────────────────────────────────────────────────────
224
+ // TrackingBackend implementation
225
+ // ───────────────────────────────────────────────────────────────────────────
226
+
227
+ async track(event: TrackingEvent): Promise<void> {
228
+ await this.ensureBootstrap()
229
+ const eventId = this.nextEventId()
230
+ const action = eventToActionInput(event, eventId)
231
+ await this.provider.recordAction({
232
+ verb: action.verb,
233
+ subject: action.subject,
234
+ object: action.object,
235
+ roles: action.roles,
236
+ data: action.data,
237
+ status: action.status,
238
+ })
239
+ }
240
+
241
+ async flush(): Promise<void> {
242
+ // Writes go straight through HTTP — nothing to flush.
243
+ }
244
+
245
+ // ───────────────────────────────────────────────────────────────────────────
246
+ // Direct experiment-result API
247
+ // ───────────────────────────────────────────────────────────────────────────
248
+
249
+ async storeResult(result: ExperimentResult): Promise<void> {
250
+ await this.track({
251
+ type: 'variant.complete',
252
+ timestamp: result.completedAt,
253
+ data: {
254
+ experimentId: result.experimentId,
255
+ variantId: result.variantId,
256
+ variantName: result.variantName,
257
+ runId: result.runId,
258
+ success: result.success,
259
+ duration: result.duration,
260
+ result: result.result,
261
+ metricValue: result.metricValue,
262
+ error: result.error,
263
+ metadata: result.metadata,
264
+ },
265
+ })
266
+ }
267
+
268
+ /**
269
+ * Bulk-store results via {@link ClickHouseProvider.commitBatch}. One
270
+ * HTTP POST per call carries the entire batch as `JSONEachRow`.
271
+ */
272
+ async storeResults(results: ExperimentResult[]): Promise<void> {
273
+ if (results.length === 0) return
274
+ await this.ensureBootstrap()
275
+ const actions = results.map((r) => {
276
+ const action = eventToActionInput(
277
+ {
278
+ type: 'variant.complete',
279
+ timestamp: r.completedAt,
280
+ data: {
281
+ experimentId: r.experimentId,
282
+ variantId: r.variantId,
283
+ variantName: r.variantName,
284
+ runId: r.runId,
285
+ success: r.success,
286
+ duration: r.duration,
287
+ result: r.result,
288
+ metricValue: r.metricValue,
289
+ error: r.error,
290
+ metadata: r.metadata,
291
+ },
292
+ },
293
+ this.nextEventId()
294
+ )
295
+ return action
296
+ })
297
+ await this.provider.commitBatch({ actions })
298
+ }
299
+
300
+ // ───────────────────────────────────────────────────────────────────────────
301
+ // Analytics queries (raw SQL via canonical adapter's analyticsQuery)
302
+ // ───────────────────────────────────────────────────────────────────────────
303
+
304
+ /** Quote a SQL string literal — same shape as the canonical adapter. */
305
+ private quote(value: string): string {
306
+ return `'${value.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`
307
+ }
308
+
309
+ /**
310
+ * Get all experiments seen in the actions table (variant.complete events).
311
+ */
312
+ async getExperiments(): Promise<
313
+ Array<{
314
+ experimentId: string
315
+ experimentName: string
316
+ variantCount: number
317
+ runCount: number
318
+ firstRun: string
319
+ lastRun: string
320
+ }>
321
+ > {
322
+ await this.ensureBootstrap()
323
+ const ns = this.quote(this.namespace)
324
+ const sql = `
325
+ SELECT
326
+ subject AS experimentId,
327
+ any(JSONExtractString(data, 'experimentName')) AS experimentName,
328
+ uniq(object) AS variantCount,
329
+ count() AS runCount,
330
+ toString(min(created_at)) AS firstRun,
331
+ toString(max(created_at)) AS lastRun
332
+ FROM ${this.database}.actions
333
+ WHERE ns = ${ns} AND verb = 'variant.complete' AND subject != ''
334
+ GROUP BY subject
335
+ ORDER BY lastRun DESC
336
+ `
337
+ const rows = await this.provider.analyticsQuery(sql)
338
+ return rows.map((row) => ({
339
+ experimentId: String(row['experimentId'] ?? ''),
340
+ experimentName: String(row['experimentName'] ?? ''),
341
+ variantCount: Number(row['variantCount'] ?? 0),
342
+ runCount: Number(row['runCount'] ?? 0),
343
+ firstRun: String(row['firstRun'] ?? ''),
344
+ lastRun: String(row['lastRun'] ?? ''),
345
+ }))
346
+ }
347
+
348
+ async getVariantStats(experimentId: string): Promise<
349
+ Array<{
350
+ variantId: string
351
+ variantName: string
352
+ runCount: number
353
+ successCount: number
354
+ successRate: number
355
+ avgDuration: number
356
+ avgMetric: number
357
+ minMetric: number
358
+ maxMetric: number
359
+ }>
360
+ > {
361
+ await this.ensureBootstrap()
362
+ const ns = this.quote(this.namespace)
363
+ const exp = this.quote(experimentId)
364
+ const sql = `
365
+ SELECT
366
+ object AS variantId,
367
+ any(JSONExtractString(data, 'variantName')) AS variantName,
368
+ count() AS runCount,
369
+ countIf(JSONExtractInt(data, 'success') = 1) AS successCount,
370
+ countIf(JSONExtractInt(data, 'success') = 1) / count() AS successRate,
371
+ avg(JSONExtractFloat(data, 'durationMs')) AS avgDuration,
372
+ avg(JSONExtractFloat(data, 'metricValue')) AS avgMetric,
373
+ min(JSONExtractFloat(data, 'metricValue')) AS minMetric,
374
+ max(JSONExtractFloat(data, 'metricValue')) AS maxMetric
375
+ FROM ${this.database}.actions
376
+ WHERE ns = ${ns} AND verb = 'variant.complete' AND subject = ${exp}
377
+ GROUP BY object
378
+ ORDER BY avgMetric DESC
379
+ `
380
+ const rows = await this.provider.analyticsQuery(sql)
381
+ return rows.map((row) => ({
382
+ variantId: String(row['variantId'] ?? ''),
383
+ variantName: String(row['variantName'] ?? ''),
384
+ runCount: Number(row['runCount'] ?? 0),
385
+ successCount: Number(row['successCount'] ?? 0),
386
+ successRate: Number(row['successRate'] ?? 0),
387
+ avgDuration: Number(row['avgDuration'] ?? 0),
388
+ avgMetric: Number(row['avgMetric'] ?? 0),
389
+ minMetric: Number(row['minMetric'] ?? 0),
390
+ maxMetric: Number(row['maxMetric'] ?? 0),
391
+ }))
392
+ }
393
+
394
+ async getBestVariant(
395
+ experimentId: string,
396
+ options: {
397
+ metric?: 'avgMetric' | 'successRate' | 'avgDuration'
398
+ minimumRuns?: number
399
+ } = {}
400
+ ): Promise<{
401
+ variantId: string
402
+ variantName: string
403
+ metricValue: number
404
+ runCount: number
405
+ } | null> {
406
+ await this.ensureBootstrap()
407
+ const { metric = 'avgMetric', minimumRuns = 1 } = options
408
+ const ns = this.quote(this.namespace)
409
+ const exp = this.quote(experimentId)
410
+ const orderBy = metric === 'avgDuration' ? 'ASC' : 'DESC'
411
+ const metricExpr =
412
+ metric === 'successRate'
413
+ ? "countIf(JSONExtractInt(data, 'success') = 1) / count()"
414
+ : metric === 'avgDuration'
415
+ ? "avg(JSONExtractFloat(data, 'durationMs'))"
416
+ : "avg(JSONExtractFloat(data, 'metricValue'))"
417
+
418
+ const sql = `
419
+ SELECT
420
+ object AS variantId,
421
+ any(JSONExtractString(data, 'variantName')) AS variantName,
422
+ ${metricExpr} AS metricValue,
423
+ count() AS runCount
424
+ FROM ${this.database}.actions
425
+ WHERE ns = ${ns} AND verb = 'variant.complete' AND subject = ${exp}
426
+ GROUP BY object
427
+ HAVING runCount >= ${Number(minimumRuns)}
428
+ ORDER BY metricValue ${orderBy}
429
+ LIMIT 1
430
+ `
431
+ const rows = await this.provider.analyticsQuery(sql)
432
+ if (rows.length === 0) return null
433
+ const row = rows[0]!
434
+ return {
435
+ variantId: String(row['variantId'] ?? ''),
436
+ variantName: String(row['variantName'] ?? ''),
437
+ metricValue: Number(row['metricValue'] ?? 0),
438
+ runCount: Number(row['runCount'] ?? 0),
439
+ }
440
+ }
441
+
442
+ async getCartesianAnalysis(
443
+ experimentId: string,
444
+ dimension: string
445
+ ): Promise<
446
+ Array<{
447
+ dimensionValue: string
448
+ runCount: number
449
+ avgMetric: number
450
+ successRate: number
451
+ }>
452
+ > {
453
+ await this.ensureBootstrap()
454
+ const ns = this.quote(this.namespace)
455
+ const exp = this.quote(experimentId)
456
+ const dimQuoted = this.quote(dimension)
457
+ const sql = `
458
+ SELECT
459
+ JSONExtractString(JSONExtractRaw(data, 'dimensions'), ${dimQuoted}) AS dimensionValue,
460
+ count() AS runCount,
461
+ avg(JSONExtractFloat(data, 'metricValue')) AS avgMetric,
462
+ countIf(JSONExtractInt(data, 'success') = 1) / count() AS successRate
463
+ FROM ${this.database}.actions
464
+ WHERE ns = ${ns} AND verb = 'variant.complete' AND subject = ${exp}
465
+ GROUP BY dimensionValue
466
+ HAVING dimensionValue != ''
467
+ ORDER BY avgMetric DESC
468
+ `
469
+ const rows = await this.provider.analyticsQuery(sql)
470
+ return rows.map((row) => ({
471
+ dimensionValue: String(row['dimensionValue'] ?? ''),
472
+ runCount: Number(row['runCount'] ?? 0),
473
+ avgMetric: Number(row['avgMetric'] ?? 0),
474
+ successRate: Number(row['successRate'] ?? 0),
475
+ }))
476
+ }
477
+
478
+ async getCartesianGrid(
479
+ experimentId: string,
480
+ dimensions: string[]
481
+ ): Promise<
482
+ Array<{
483
+ dimensions: Record<string, string>
484
+ runCount: number
485
+ avgMetric: number
486
+ successRate: number
487
+ }>
488
+ > {
489
+ await this.ensureBootstrap()
490
+ const ns = this.quote(this.namespace)
491
+ const exp = this.quote(experimentId)
492
+ const dimExtracts = dimensions
493
+ .map(
494
+ (d) => `JSONExtractString(JSONExtractRaw(data, 'dimensions'), ${this.quote(d)}) AS dim_${d}`
495
+ )
496
+ .join(', ')
497
+ const dimGroupBy = dimensions.map((d) => `dim_${d}`).join(', ')
498
+
499
+ const sql = `
500
+ SELECT
501
+ ${dimExtracts},
502
+ count() AS runCount,
503
+ avg(JSONExtractFloat(data, 'metricValue')) AS avgMetric,
504
+ countIf(JSONExtractInt(data, 'success') = 1) / count() AS successRate
505
+ FROM ${this.database}.actions
506
+ WHERE ns = ${ns} AND verb = 'variant.complete' AND subject = ${exp}
507
+ GROUP BY ${dimGroupBy}
508
+ ORDER BY avgMetric DESC
509
+ `
510
+ const rows = await this.provider.analyticsQuery(sql)
511
+ return rows.map((row) => {
512
+ const dims: Record<string, string> = {}
513
+ for (const d of dimensions) {
514
+ dims[d] = String(row[`dim_${d}`] ?? '')
515
+ }
516
+ return {
517
+ dimensions: dims,
518
+ runCount: Number(row['runCount'] ?? 0),
519
+ avgMetric: Number(row['avgMetric'] ?? 0),
520
+ successRate: Number(row['successRate'] ?? 0),
521
+ }
522
+ })
523
+ }
524
+
525
+ async getTimeSeries(
526
+ experimentId: string,
527
+ options: {
528
+ interval?: 'hour' | 'day' | 'week'
529
+ variantId?: string
530
+ } = {}
531
+ ): Promise<
532
+ Array<{
533
+ period: string
534
+ runCount: number
535
+ avgMetric: number
536
+ successRate: number
537
+ }>
538
+ > {
539
+ await this.ensureBootstrap()
540
+ const { interval = 'day', variantId } = options
541
+ const ns = this.quote(this.namespace)
542
+ const exp = this.quote(experimentId)
543
+ const dateFunc =
544
+ interval === 'hour'
545
+ ? 'toStartOfHour(created_at)'
546
+ : interval === 'week'
547
+ ? 'toStartOfWeek(created_at)'
548
+ : 'toStartOfDay(created_at)'
549
+ const variantFilter = variantId ? `AND object = ${this.quote(variantId)}` : ''
550
+
551
+ const sql = `
552
+ SELECT
553
+ toString(${dateFunc}) AS period,
554
+ count() AS runCount,
555
+ avg(JSONExtractFloat(data, 'metricValue')) AS avgMetric,
556
+ countIf(JSONExtractInt(data, 'success') = 1) / count() AS successRate
557
+ FROM ${this.database}.actions
558
+ WHERE ns = ${ns} AND verb = 'variant.complete' AND subject = ${exp}
559
+ ${variantFilter}
560
+ GROUP BY period
561
+ ORDER BY period ASC
562
+ `
563
+ const rows = await this.provider.analyticsQuery(sql)
564
+ return rows.map((row) => ({
565
+ period: String(row['period'] ?? ''),
566
+ runCount: Number(row['runCount'] ?? 0),
567
+ avgMetric: Number(row['avgMetric'] ?? 0),
568
+ successRate: Number(row['successRate'] ?? 0),
569
+ }))
570
+ }
571
+
572
+ async getEvents(
573
+ experimentId: string,
574
+ options: {
575
+ eventType?: string
576
+ variantId?: string
577
+ limit?: number
578
+ } = {}
579
+ ): Promise<TrackingEvent[]> {
580
+ await this.ensureBootstrap()
581
+ const { eventType, variantId, limit = 100 } = options
582
+ const ns = this.quote(this.namespace)
583
+ const exp = this.quote(experimentId)
584
+ const filters = [`ns = ${ns}`, `subject = ${exp}`]
585
+ if (eventType) filters.push(`verb = ${this.quote(eventType)}`)
586
+ if (variantId) filters.push(`object = ${this.quote(variantId)}`)
587
+
588
+ const sql = `
589
+ SELECT
590
+ verb AS eventType,
591
+ toString(created_at) AS createdAt,
592
+ data
593
+ FROM ${this.database}.actions
594
+ WHERE ${filters.join(' AND ')}
595
+ ORDER BY created_at DESC
596
+ LIMIT ${Number(limit)}
597
+ `
598
+ const rows = await this.provider.analyticsQuery(sql)
599
+ return rows.map((row) => {
600
+ const dataStr = row['data']
601
+ let payload: Record<string, unknown> = {}
602
+ if (typeof dataStr === 'string' && dataStr.length > 0) {
603
+ try {
604
+ payload = JSON.parse(dataStr) as Record<string, unknown>
605
+ } catch {
606
+ payload = {}
607
+ }
608
+ } else if (dataStr && typeof dataStr === 'object') {
609
+ payload = dataStr as Record<string, unknown>
610
+ }
611
+ const createdAt = String(row['createdAt'] ?? '')
612
+ const ts = createdAt ? new Date(createdAt) : new Date()
613
+ return {
614
+ type: String(row['eventType'] ?? '') as TrackingEvent['type'],
615
+ timestamp: Number.isNaN(ts.getTime()) ? new Date() : ts,
616
+ data: payload,
617
+ }
618
+ })
619
+ }
620
+
621
+ /**
622
+ * Raw analytical SQL pass-through. Routes to
623
+ * {@link ClickHouseProvider.analyticsQuery}.
624
+ */
625
+ async query(sql: string): Promise<Array<Record<string, unknown>>> {
626
+ return this.provider.analyticsQuery(sql)
627
+ }
628
+
629
+ /** No-op (kept for backwards compat with the chdb-era API). */
630
+ close(): void {
631
+ // HTTP fetcher has nothing to release.
632
+ }
633
+ }
634
+
635
+ /**
636
+ * Bootstrap the canonical CH schema (`things`, `actions`, `verbs`).
637
+ * Convenience wrapper around the canonical
638
+ * {@link bootstrapClickHouseSchema}.
639
+ */
640
+ export async function bootstrapExperimentsSchema(
641
+ fetcher: ClickHouseHttpFetcher,
642
+ options: { database?: string } = {}
643
+ ): Promise<void> {
644
+ await bootstrapClickHouseSchema(
645
+ fetcher,
646
+ options.database !== undefined ? { database: options.database } : {}
647
+ )
648
+ }
649
+
650
+ /**
651
+ * Create a ClickHouse-backed experiment storage backend.
652
+ *
653
+ * @example
654
+ * ```ts
655
+ * import { configureTracking } from 'ai-experiments'
656
+ * import { createClickHouseExperimentStorage } from 'ai-experiments/storage'
657
+ * import { createClickHouseHttpFetcher } from 'ai-database'
658
+ *
659
+ * const fetcher = createClickHouseHttpFetcher({
660
+ * url: process.env.CLICKHOUSE_URL!,
661
+ * username: process.env.CLICKHOUSE_USER,
662
+ * password: process.env.CLICKHOUSE_PASSWORD,
663
+ * database: 'aidb',
664
+ * })
665
+ * const storage = createClickHouseExperimentStorage({ fetcher })
666
+ *
667
+ * configureTracking({ backend: storage })
668
+ *
669
+ * const best = await storage.getBestVariant('my-experiment')
670
+ * ```
671
+ */
672
+ export function createClickHouseExperimentStorage(
673
+ options: ClickHouseExperimentStorageOptions
674
+ ): ClickHouseExperimentStorage {
675
+ return new ClickHouseExperimentStorage(options)
676
+ }
677
+
678
+ // =============================================================================
679
+ // Backwards-compat aliases (chdb-era names)
680
+ // =============================================================================
681
+
682
+ /**
683
+ * @deprecated Renamed to {@link ClickHouseExperimentStorage}. The class
684
+ * now wraps the canonical `ClickHouseProvider` from `ai-database` instead
685
+ * of embedded chdb. The old `dataPath` constructor option is ignored —
686
+ * supply a `fetcher` instead.
687
+ */
688
+ export const ChdbStorage = ClickHouseExperimentStorage
689
+ export type ChdbStorage = ClickHouseExperimentStorage
690
+
691
+ /**
692
+ * @deprecated Renamed to {@link createClickHouseExperimentStorage}.
693
+ */
694
+ export function createChdbBackend(
695
+ options: ClickHouseExperimentStorageOptions
696
+ ): ClickHouseExperimentStorage {
697
+ return createClickHouseExperimentStorage(options)
698
+ }