ai-experiments 2.1.3 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/README.md +13 -1
- package/package.json +16 -16
- package/src/clickhouse-storage.ts +698 -0
- package/src/decide.ts +8 -10
- package/src/experiment.ts +12 -12
- package/src/index.ts +13 -3
- package/src/tracking.ts +3 -3
- package/test/cartesian.test.ts +287 -0
- package/test/clickhouse-storage.test.ts +470 -0
- package/test/decide.test.ts +414 -0
- package/test/experiment.test.ts +473 -0
- package/test/tracking.test.ts +347 -0
- package/vitest.config.ts +34 -0
- package/.turbo/turbo-build.log +0 -4
- package/LICENSE +0 -21
- package/dist/cartesian.d.ts +0 -140
- package/dist/cartesian.d.ts.map +0 -1
- package/dist/cartesian.js +0 -216
- package/dist/cartesian.js.map +0 -1
- package/dist/chdb-storage.d.ts +0 -135
- package/dist/chdb-storage.d.ts.map +0 -1
- package/dist/chdb-storage.js +0 -468
- package/dist/chdb-storage.js.map +0 -1
- package/dist/decide.d.ts +0 -152
- package/dist/decide.d.ts.map +0 -1
- package/dist/decide.js +0 -329
- package/dist/decide.js.map +0 -1
- package/dist/experiment.d.ts +0 -53
- package/dist/experiment.d.ts.map +0 -1
- package/dist/experiment.js +0 -292
- package/dist/experiment.js.map +0 -1
- package/dist/index.d.ts +0 -16
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js +0 -21
- package/dist/index.js.map +0 -1
- package/dist/tracking.d.ts +0 -159
- package/dist/tracking.d.ts.map +0 -1
- package/dist/tracking.js +0 -310
- package/dist/tracking.js.map +0 -1
- package/dist/types.d.ts +0 -198
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js +0 -5
- package/dist/types.js.map +0 -1
- package/src/cartesian.js +0 -215
- package/src/chdb-storage.js +0 -464
- package/src/chdb-storage.ts +0 -567
- package/src/decide.js +0 -328
- package/src/experiment.js +0 -291
- package/src/index.js +0 -20
- package/src/tracking.js +0 -309
- package/src/types.js +0 -4
package/CHANGELOG.md
CHANGED
package/README.md
CHANGED
|
@@ -1,6 +1,18 @@
|
|
|
1
1
|
# ai-experiments
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+

|
|
4
|
+
|
|
5
|
+
Runtime A/B testing for AI variants in production. Traffic splitting, multi-armed bandits, parameter sweeps, and ClickHouse-backed outcome tracking via the canonical `ai-database` adapter.
|
|
6
|
+
|
|
7
|
+
## Lifecycle
|
|
8
|
+
|
|
9
|
+
**Runtime (production).** `ai-experiments` is for splitting live traffic across model/prompt/parameter variants and measuring outcomes against real users. Reach for it when you need to learn which variant wins under production conditions — not in a test suite.
|
|
10
|
+
|
|
11
|
+
## Not for
|
|
12
|
+
|
|
13
|
+
- **Unit testing.** If you want to assert that a specific input produces a specific output during development or CI, use [`ai-tests`](../ai-tests) plus `vitest`. `ai-experiments` exists to *measure* outcomes across users, not to *assert* them against fixtures.
|
|
14
|
+
- **Sandboxing untrusted code.** That's [`ai-evaluate`](../ai-evaluate)'s job.
|
|
15
|
+
- **Offline-only experiments.** While `Experiment()` works offline for parameter sweeps, the package's value comes from production tracking — if you're never deploying the variants, a plain `for` loop is simpler.
|
|
4
16
|
|
|
5
17
|
## Overview
|
|
6
18
|
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ai-experiments",
|
|
3
|
-
"version": "2.
|
|
4
|
-
"description": "AI-
|
|
3
|
+
"version": "2.3.0",
|
|
4
|
+
"description": "Runtime A/B testing for AI variants in production - traffic splitting, decision strategies, and ClickHouse-backed outcome tracking",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"types": "dist/index.d.ts",
|
|
@@ -11,13 +11,21 @@
|
|
|
11
11
|
"types": "./dist/index.d.ts"
|
|
12
12
|
},
|
|
13
13
|
"./storage": {
|
|
14
|
-
"import": "./dist/
|
|
15
|
-
"types": "./dist/
|
|
14
|
+
"import": "./dist/clickhouse-storage.js",
|
|
15
|
+
"types": "./dist/clickhouse-storage.d.ts"
|
|
16
16
|
}
|
|
17
17
|
},
|
|
18
|
+
"scripts": {
|
|
19
|
+
"build": "tsc",
|
|
20
|
+
"dev": "tsc --watch",
|
|
21
|
+
"test": "vitest run",
|
|
22
|
+
"typecheck": "tsc --noEmit",
|
|
23
|
+
"lint": "eslint .",
|
|
24
|
+
"clean": "rm -rf dist"
|
|
25
|
+
},
|
|
18
26
|
"dependencies": {
|
|
19
|
-
"
|
|
20
|
-
"ai-functions": "2.
|
|
27
|
+
"ai-database": "2.3.0",
|
|
28
|
+
"ai-functions": "2.3.0"
|
|
21
29
|
},
|
|
22
30
|
"keywords": [
|
|
23
31
|
"ai",
|
|
@@ -26,13 +34,5 @@
|
|
|
26
34
|
"analytics",
|
|
27
35
|
"primitives"
|
|
28
36
|
],
|
|
29
|
-
"license": "MIT"
|
|
30
|
-
|
|
31
|
-
"build": "tsc",
|
|
32
|
-
"dev": "tsc --watch",
|
|
33
|
-
"test": "vitest",
|
|
34
|
-
"typecheck": "tsc --noEmit",
|
|
35
|
-
"lint": "eslint .",
|
|
36
|
-
"clean": "rm -rf dist"
|
|
37
|
-
}
|
|
38
|
-
}
|
|
37
|
+
"license": "MIT"
|
|
38
|
+
}
|