@use-lattice/litmus 0.121.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (199) hide show
  1. package/LICENSE +19 -0
  2. package/dist/src/accounts-Bt1oJb1Z.cjs +219 -0
  3. package/dist/src/accounts-DjOU8Rm3.js +178 -0
  4. package/dist/src/agentic-utils-D03IiXQc.js +153 -0
  5. package/dist/src/agentic-utils-Dh7xaMQM.cjs +180 -0
  6. package/dist/src/agents-C6BIMlZa.js +231 -0
  7. package/dist/src/agents-DvIpNX1L.cjs +666 -0
  8. package/dist/src/agents-ZP0RP9vV.cjs +231 -0
  9. package/dist/src/agents-maJXdjbR.js +665 -0
  10. package/dist/src/aimlapi-BTbQjG2E.cjs +30 -0
  11. package/dist/src/aimlapi-CwMxqfXP.js +30 -0
  12. package/dist/src/audio-BBUdvsde.cjs +97 -0
  13. package/dist/src/audio-D5DPZ7I-.js +97 -0
  14. package/dist/src/base-BEysXrkq.cjs +222 -0
  15. package/dist/src/base-C451JQfq.js +193 -0
  16. package/dist/src/blobs-BY8MDmpo.js +230 -0
  17. package/dist/src/blobs-BgcNn97m.cjs +256 -0
  18. package/dist/src/cache-BBE_lsTA.cjs +4 -0
  19. package/dist/src/cache-BkrqU5Ba.js +237 -0
  20. package/dist/src/cache-DsCxFlsZ.cjs +297 -0
  21. package/dist/src/chat-CPJWDP6a.cjs +289 -0
  22. package/dist/src/chat-CXX3xzkk.cjs +811 -0
  23. package/dist/src/chat-CcDgZFJ4.js +787 -0
  24. package/dist/src/chat-Dz5ZeGO2.js +289 -0
  25. package/dist/src/chatkit-Dw0mKkML.cjs +1158 -0
  26. package/dist/src/chatkit-swAIVuea.js +1157 -0
  27. package/dist/src/chunk-DEq-mXcV.js +15 -0
  28. package/dist/src/claude-agent-sdk-BXZJtOg6.js +379 -0
  29. package/dist/src/claude-agent-sdk-CkfyjDoG.cjs +383 -0
  30. package/dist/src/cloudflare-ai-BzpJcqUH.js +161 -0
  31. package/dist/src/cloudflare-ai-Cmy_R1y2.cjs +161 -0
  32. package/dist/src/cloudflare-gateway-B9tVQKok.cjs +272 -0
  33. package/dist/src/cloudflare-gateway-DrD3ew3H.js +272 -0
  34. package/dist/src/codex-sdk-Dezj9Nwm.js +1056 -0
  35. package/dist/src/codex-sdk-Dl9D4k5B.cjs +1060 -0
  36. package/dist/src/cometapi-C-9YvCHC.js +54 -0
  37. package/dist/src/cometapi-DHgDKoO2.cjs +54 -0
  38. package/dist/src/completion-B8Ctyxpr.js +120 -0
  39. package/dist/src/completion-Cxrt08sj.cjs +131 -0
  40. package/dist/src/createHash-BwgE13yv.cjs +27 -0
  41. package/dist/src/createHash-DmPQkvBh.js +15 -0
  42. package/dist/src/docker-BiqcTwLv.js +80 -0
  43. package/dist/src/docker-C7tEJnP-.cjs +80 -0
  44. package/dist/src/esm-C62Zofr1.cjs +409 -0
  45. package/dist/src/esm-DMVc93eh.js +379 -0
  46. package/dist/src/evalResult-C3NJPQOo.cjs +301 -0
  47. package/dist/src/evalResult-C7JJAPBb.js +295 -0
  48. package/dist/src/evalResult-DoVTZZWI.cjs +2 -0
  49. package/dist/src/extractor-DnMD3fwt.cjs +391 -0
  50. package/dist/src/extractor-DtlL28vL.js +374 -0
  51. package/dist/src/fetch-BTxakTSg.cjs +1133 -0
  52. package/dist/src/fetch-DQckpUFz.js +928 -0
  53. package/dist/src/fileExtensions-DnqA1y9x.js +85 -0
  54. package/dist/src/fileExtensions-bYh77CN8.cjs +114 -0
  55. package/dist/src/genaiTracer-CyZrmaK0.cjs +268 -0
  56. package/dist/src/genaiTracer-D3fD9dNV.js +256 -0
  57. package/dist/src/graders-BNscxFrU.js +13644 -0
  58. package/dist/src/graders-D2oE9Msq.js +2 -0
  59. package/dist/src/graders-c0Ez_w-9.cjs +2 -0
  60. package/dist/src/graders-d0F2M3e9.cjs +14056 -0
  61. package/dist/src/image-0ZhE0VlR.cjs +280 -0
  62. package/dist/src/image-CWE1pdNv.js +257 -0
  63. package/dist/src/image-D9ZK6hwL.js +163 -0
  64. package/dist/src/image-DKZgZITg.cjs +163 -0
  65. package/dist/src/index.cjs +11366 -0
  66. package/dist/src/index.d.cts +19640 -0
  67. package/dist/src/index.d.ts +19641 -0
  68. package/dist/src/index.js +11306 -0
  69. package/dist/src/invariant-Ddh24eXh.js +25 -0
  70. package/dist/src/invariant-kfQ8Bu82.cjs +30 -0
  71. package/dist/src/knowledgeBase-BgPyGFUd.cjs +122 -0
  72. package/dist/src/knowledgeBase-DyHilYaP.js +122 -0
  73. package/dist/src/litellm-CyMeneHS.js +135 -0
  74. package/dist/src/litellm-DWDF73yF.cjs +135 -0
  75. package/dist/src/logger-C40ZGil9.js +717 -0
  76. package/dist/src/logger-DyfK9PBt.cjs +917 -0
  77. package/dist/src/luma-ray-BAU9X_ep.cjs +315 -0
  78. package/dist/src/luma-ray-nwVseBbv.js +313 -0
  79. package/dist/src/messages-B5ADWTTv.js +245 -0
  80. package/dist/src/messages-BCnZfqrS.cjs +257 -0
  81. package/dist/src/meteor-DLZZ3osF.cjs +134 -0
  82. package/dist/src/meteor-DUiCJRC-.js +134 -0
  83. package/dist/src/modelslab-00cveB8L.cjs +163 -0
  84. package/dist/src/modelslab-D9sCU_L7.js +163 -0
  85. package/dist/src/nova-reel-CTapvqYH.js +276 -0
  86. package/dist/src/nova-reel-DlWuuroF.cjs +278 -0
  87. package/dist/src/nova-sonic-5UPWfeMv.cjs +363 -0
  88. package/dist/src/nova-sonic-BhSwQNym.js +363 -0
  89. package/dist/src/openai-BWrJK9d8.cjs +52 -0
  90. package/dist/src/openai-DumO8WQn.js +47 -0
  91. package/dist/src/openclaw-B8brrjC_.cjs +577 -0
  92. package/dist/src/openclaw-Bkayww9q.js +571 -0
  93. package/dist/src/opencode-sdk-7xjoDNiM.cjs +562 -0
  94. package/dist/src/opencode-sdk-SGwAPxht.js +558 -0
  95. package/dist/src/otlpReceiver-CoAHfAN9.cjs +15 -0
  96. package/dist/src/otlpReceiver-oO3EQwI9.js +14 -0
  97. package/dist/src/providerRegistry-4yjhaEM8.js +45 -0
  98. package/dist/src/providerRegistry-DhV4rJIc.cjs +50 -0
  99. package/dist/src/providers-B5RJVG-7.cjs +33609 -0
  100. package/dist/src/providers-BdmZCLzV.js +33262 -0
  101. package/dist/src/providers-CxtRxn8e.js +2 -0
  102. package/dist/src/providers-DnQLNbx1.cjs +3 -0
  103. package/dist/src/pythonUtils-BD0druiM.cjs +275 -0
  104. package/dist/src/pythonUtils-IBhn5YGR.js +249 -0
  105. package/dist/src/quiverai-BDOwZBsM.cjs +213 -0
  106. package/dist/src/quiverai-D3JTF5lD.js +213 -0
  107. package/dist/src/responses-B2LCDCXZ.js +667 -0
  108. package/dist/src/responses-BvNm4Xv9.cjs +685 -0
  109. package/dist/src/rubyUtils-B0NwnfpY.cjs +245 -0
  110. package/dist/src/rubyUtils-BroxzZ7c.cjs +2 -0
  111. package/dist/src/rubyUtils-hqVw5UvJ.js +222 -0
  112. package/dist/src/sagemaker-Cno2V-Sx.js +689 -0
  113. package/dist/src/sagemaker-fV_KUgs5.cjs +691 -0
  114. package/dist/src/server-BOuAXb06.cjs +238 -0
  115. package/dist/src/server-CtI-EWzm.cjs +2 -0
  116. package/dist/src/server-Cy3DZymt.js +189 -0
  117. package/dist/src/slack-CP8xBePa.js +135 -0
  118. package/dist/src/slack-DSQ1yXVb.cjs +135 -0
  119. package/dist/src/store-BwDDaBjb.cjs +246 -0
  120. package/dist/src/store-DcbLC593.cjs +2 -0
  121. package/dist/src/store-IGpqMIkv.js +240 -0
  122. package/dist/src/tables-3Q2cL7So.cjs +373 -0
  123. package/dist/src/tables-Bi2fjr4W.js +288 -0
  124. package/dist/src/telemetry-Bg2WqF79.js +161 -0
  125. package/dist/src/telemetry-D0x6u5kX.cjs +166 -0
  126. package/dist/src/telemetry-DXNimrI0.cjs +2 -0
  127. package/dist/src/text-B_UCRPp2.js +22 -0
  128. package/dist/src/text-CW1cyrwj.cjs +33 -0
  129. package/dist/src/tokenUsageUtils-NYT-WKS6.js +138 -0
  130. package/dist/src/tokenUsageUtils-bVa1ga6f.cjs +173 -0
  131. package/dist/src/transcription-Cl_W16Pr.js +122 -0
  132. package/dist/src/transcription-yt1EecY8.cjs +124 -0
  133. package/dist/src/transform-BCtGrl_W.cjs +228 -0
  134. package/dist/src/transform-Bv6gG2MJ.cjs +1688 -0
  135. package/dist/src/transform-CY1wbpRy.js +1507 -0
  136. package/dist/src/transform-DU8rUL9P.cjs +2 -0
  137. package/dist/src/transform-yWaShiKr.js +216 -0
  138. package/dist/src/transformersAvailability-BGkzavwb.js +35 -0
  139. package/dist/src/transformersAvailability-DKoRtQLy.cjs +35 -0
  140. package/dist/src/types-5aqHpBwE.cjs +3769 -0
  141. package/dist/src/types-Bn6D9c4U.js +3300 -0
  142. package/dist/src/util-BkKlTkI2.js +293 -0
  143. package/dist/src/util-CTh0bfOm.cjs +1119 -0
  144. package/dist/src/util-D17oBwo7.cjs +328 -0
  145. package/dist/src/util-DsS_-v4p.js +613 -0
  146. package/dist/src/util-DuntT1Ga.js +951 -0
  147. package/dist/src/util-aWjdCYMI.cjs +667 -0
  148. package/dist/src/utils-CisQwpjA.js +94 -0
  149. package/dist/src/utils-yWamDvmz.cjs +123 -0
  150. package/dist/tsconfig.tsbuildinfo +1 -0
  151. package/drizzle/0000_lush_hellion.sql +36 -0
  152. package/drizzle/0001_wide_calypso.sql +3 -0
  153. package/drizzle/0002_tidy_juggernaut.sql +1 -0
  154. package/drizzle/0003_lively_naoko.sql +8 -0
  155. package/drizzle/0004_minor_peter_quill.sql +19 -0
  156. package/drizzle/0005_silky_millenium_guard.sql +2 -0
  157. package/drizzle/0006_harsh_caretaker.sql +42 -0
  158. package/drizzle/0007_cloudy_wong.sql +1 -0
  159. package/drizzle/0008_broad_boomer.sql +2 -0
  160. package/drizzle/0009_strong_marten_broadcloak.sql +19 -0
  161. package/drizzle/0010_needy_bishop.sql +11 -0
  162. package/drizzle/0011_moaning_millenium_guard.sql +1 -0
  163. package/drizzle/0012_late_marten_broadcloak.sql +2 -0
  164. package/drizzle/0013_previous_dormammu.sql +9 -0
  165. package/drizzle/0014_lazy_captain_universe.sql +2 -0
  166. package/drizzle/0015_zippy_wallop.sql +29 -0
  167. package/drizzle/0016_jazzy_zemo.sql +2 -0
  168. package/drizzle/0017_reflective_praxagora.sql +4 -0
  169. package/drizzle/0018_fat_vanisher.sql +22 -0
  170. package/drizzle/0019_new_clint_barton.sql +8 -0
  171. package/drizzle/0020_skinny_maverick.sql +1 -0
  172. package/drizzle/0021_mysterious_madelyne_pryor.sql +13 -0
  173. package/drizzle/0022_sleepy_ultimo.sql +25 -0
  174. package/drizzle/0023_wooden_mandrill.sql +2 -0
  175. package/drizzle/AGENTS.md +68 -0
  176. package/drizzle/CLAUDE.md +1 -0
  177. package/drizzle/meta/0000_snapshot.json +221 -0
  178. package/drizzle/meta/0001_snapshot.json +214 -0
  179. package/drizzle/meta/0002_snapshot.json +221 -0
  180. package/drizzle/meta/0005_snapshot.json +369 -0
  181. package/drizzle/meta/0006_snapshot.json +638 -0
  182. package/drizzle/meta/0007_snapshot.json +640 -0
  183. package/drizzle/meta/0008_snapshot.json +649 -0
  184. package/drizzle/meta/0009_snapshot.json +554 -0
  185. package/drizzle/meta/0010_snapshot.json +619 -0
  186. package/drizzle/meta/0011_snapshot.json +627 -0
  187. package/drizzle/meta/0012_snapshot.json +639 -0
  188. package/drizzle/meta/0013_snapshot.json +717 -0
  189. package/drizzle/meta/0014_snapshot.json +717 -0
  190. package/drizzle/meta/0015_snapshot.json +897 -0
  191. package/drizzle/meta/0016_snapshot.json +1031 -0
  192. package/drizzle/meta/0018_snapshot.json +1210 -0
  193. package/drizzle/meta/0019_snapshot.json +1165 -0
  194. package/drizzle/meta/0020_snapshot.json +1232 -0
  195. package/drizzle/meta/0021_snapshot.json +1311 -0
  196. package/drizzle/meta/0022_snapshot.json +1481 -0
  197. package/drizzle/meta/0023_snapshot.json +1496 -0
  198. package/drizzle/meta/_journal.json +174 -0
  199. package/package.json +240 -0
@@ -0,0 +1,36 @@
1
+ CREATE TABLE `datasets` (
2
+ `id` text PRIMARY KEY NOT NULL,
3
+ `test_case_id` text NOT NULL,
4
+ `created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL
5
+ );
6
+ --> statement-breakpoint
7
+ CREATE TABLE `evals` (
8
+ `id` text PRIMARY KEY NOT NULL,
9
+ `created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
10
+ `description` text,
11
+ `results` text NOT NULL,
12
+ `config` text NOT NULL
13
+ );
14
+ --> statement-breakpoint
15
+ CREATE TABLE `evals_to_datasets` (
16
+ `eval_id` text NOT NULL,
17
+ `dataset_id` text NOT NULL,
18
+ PRIMARY KEY(`dataset_id`, `eval_id`),
19
+ FOREIGN KEY (`eval_id`) REFERENCES `evals`(`id`) ON UPDATE no action ON DELETE no action,
20
+ FOREIGN KEY (`dataset_id`) REFERENCES `datasets`(`id`) ON UPDATE no action ON DELETE no action
21
+ );
22
+ --> statement-breakpoint
23
+ CREATE TABLE `evals_to_prompts` (
24
+ `eval_id` text NOT NULL,
25
+ `prompt_id` text NOT NULL,
26
+ PRIMARY KEY(`eval_id`, `prompt_id`),
27
+ FOREIGN KEY (`eval_id`) REFERENCES `evals`(`id`) ON UPDATE no action ON DELETE no action,
28
+ FOREIGN KEY (`prompt_id`) REFERENCES `prompts`(`id`) ON UPDATE no action ON DELETE no action
29
+ );
30
+ --> statement-breakpoint
31
+ CREATE TABLE `prompts` (
32
+ `id` text PRIMARY KEY NOT NULL,
33
+ `created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
34
+ `prompt` text NOT NULL,
35
+ `hash` text NOT NULL
36
+ );
@@ -0,0 +1,3 @@
1
+ ALTER TABLE datasets ADD `tests` text;--> statement-breakpoint
2
+ ALTER TABLE `datasets` DROP COLUMN `test_case_id`;--> statement-breakpoint
3
+ ALTER TABLE `prompts` DROP COLUMN `hash`;
@@ -0,0 +1 @@
1
+ ALTER TABLE evals ADD `author` text;
@@ -0,0 +1,8 @@
1
+ CREATE INDEX `datasets_created_at_idx` ON `datasets` (`created_at`);--> statement-breakpoint
2
+ CREATE INDEX `evals_created_at_idx` ON `evals` (`created_at`);--> statement-breakpoint
3
+ CREATE INDEX `evals_author_idx` ON `evals` (`author`);--> statement-breakpoint
4
+ CREATE INDEX `evals_to_datasets_eval_id_idx` ON `evals_to_datasets` (`eval_id`);--> statement-breakpoint
5
+ CREATE INDEX `evals_to_datasets_dataset_id_idx` ON `evals_to_datasets` (`dataset_id`);--> statement-breakpoint
6
+ CREATE INDEX `evals_to_prompts_eval_id_idx` ON `evals_to_prompts` (`eval_id`);--> statement-breakpoint
7
+ CREATE INDEX `evals_to_prompts_prompt_id_idx` ON `evals_to_prompts` (`prompt_id`);--> statement-breakpoint
8
+ CREATE INDEX `prompts_created_at_idx` ON `prompts` (`created_at`);
@@ -0,0 +1,19 @@
1
+ CREATE TABLE `evals_to_tags` (
2
+ `eval_id` text NOT NULL,
3
+ `tag_id` text NOT NULL,
4
+ PRIMARY KEY(`eval_id`, `tag_id`),
5
+ FOREIGN KEY (`eval_id`) REFERENCES `evals`(`id`) ON UPDATE no action ON DELETE no action,
6
+ FOREIGN KEY (`tag_id`) REFERENCES `tags`(`id`) ON UPDATE no action ON DELETE no action
7
+ );
8
+ --> statement-breakpoint
9
+ CREATE TABLE `tags` (
10
+ `id` text PRIMARY KEY NOT NULL,
11
+ `name` text NOT NULL,
12
+ `value` text NOT NULL
13
+ );
14
+ --> statement-breakpoint
15
+ DROP INDEX IF EXISTS `evals_tags_idx`;--> statement-breakpoint
16
+ CREATE INDEX `evals_to_tags_eval_id_idx` ON `evals_to_tags` (`eval_id`);--> statement-breakpoint
17
+ CREATE INDEX `evals_to_tags_tag_id_idx` ON `evals_to_tags` (`tag_id`);--> statement-breakpoint
18
+ CREATE UNIQUE INDEX `tags_name_unique` ON `tags` (`name`);--> statement-breakpoint
19
+ CREATE INDEX `tags_name_idx` ON `tags` (`name`);
@@ -0,0 +1,2 @@
1
+ DROP INDEX IF EXISTS `tags_name_unique`;--> statement-breakpoint
2
+ CREATE UNIQUE INDEX `tags_name_value_unique` ON `tags` (`name`,`value`);
@@ -0,0 +1,42 @@
1
+ CREATE TABLE `eval_results` (
2
+ `id` text PRIMARY KEY NOT NULL,
3
+ `created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
4
+ `updated_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
5
+ `eval_id` text NOT NULL,
6
+ `prompt_idx` integer NOT NULL,
7
+ `test_case_idx` integer NOT NULL,
8
+ `test_case` text NOT NULL,
9
+ `prompt` text NOT NULL,
10
+ `prompt_id` text,
11
+ `provider` text NOT NULL,
12
+ `provider_id` text,
13
+ `latency_ms` integer,
14
+ `cost` real,
15
+ `response` text,
16
+ `error` text,
17
+ `success` integer NOT NULL,
18
+ `score` real NOT NULL,
19
+ `grading_result` text,
20
+ `named_scores` text,
21
+ `metadata` text,
22
+ FOREIGN KEY (`eval_id`) REFERENCES `evals`(`id`) ON UPDATE no action ON DELETE no action,
23
+ FOREIGN KEY (`prompt_id`) REFERENCES `prompts`(`id`) ON UPDATE no action ON DELETE no action,
24
+ FOREIGN KEY (`provider_id`) REFERENCES `providers`(`id`) ON UPDATE no action ON DELETE no action
25
+ );
26
+ --> statement-breakpoint
27
+ CREATE TABLE `evals_to_providers` (
28
+ `provider_id` text NOT NULL,
29
+ `eval_id` text NOT NULL,
30
+ PRIMARY KEY(`provider_id`, `eval_id`),
31
+ FOREIGN KEY (`provider_id`) REFERENCES `providers`(`id`) ON UPDATE no action ON DELETE no action,
32
+ FOREIGN KEY (`eval_id`) REFERENCES `evals`(`id`) ON UPDATE no action ON DELETE no action
33
+ );
34
+ --> statement-breakpoint
35
+ CREATE TABLE `providers` (
36
+ `id` text PRIMARY KEY NOT NULL,
37
+ `provider_id` text NOT NULL,
38
+ `options` text NOT NULL
39
+ );
40
+ --> statement-breakpoint
41
+ ALTER TABLE `evals` ADD `prompts` text;--> statement-breakpoint
42
+ CREATE INDEX `eval_result_eval_id_idx` ON `eval_results` (`eval_id`);
@@ -0,0 +1 @@
1
+ ALTER TABLE `eval_results` RENAME COLUMN `test_case_idx` TO `test_idx`;
@@ -0,0 +1,2 @@
1
+ ALTER TABLE `providers` ADD `label` text;--> statement-breakpoint
2
+ ALTER TABLE `providers` DROP COLUMN `options`;
@@ -0,0 +1,19 @@
1
+ PRAGMA foreign_keys=off;--> statement-breakpoint
2
+
3
+ -- Create a new table without the provider_id column
4
+ CREATE TABLE `new_eval_results` AS SELECT * FROM `eval_results`;--> statement-breakpoint
5
+
6
+ -- Drop the provider_id column from the new table
7
+ ALTER TABLE `new_eval_results` DROP COLUMN `provider_id`;--> statement-breakpoint
8
+
9
+ -- Drop the old table
10
+ DROP TABLE `eval_results`;--> statement-breakpoint
11
+
12
+ -- Rename the new table to the original name
13
+ ALTER TABLE `new_eval_results` RENAME TO `eval_results`;--> statement-breakpoint
14
+
15
+ -- The rest of your migration remains the same
16
+ DROP TABLE `evals_to_providers`;--> statement-breakpoint
17
+ DROP TABLE `providers`;--> statement-breakpoint
18
+
19
+ PRAGMA foreign_keys=on;
@@ -0,0 +1,11 @@
1
+ CREATE TABLE `configs` (
2
+ `id` text PRIMARY KEY NOT NULL,
3
+ `created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
4
+ `updated_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
5
+ `name` text NOT NULL,
6
+ `type` text NOT NULL,
7
+ `config` text NOT NULL
8
+ );
9
+ --> statement-breakpoint
10
+ CREATE INDEX `configs_created_at_idx` ON `configs` (`created_at`);--> statement-breakpoint
11
+ CREATE INDEX `configs_type_idx` ON `configs` (`type`);
@@ -0,0 +1 @@
1
+ ALTER TABLE `eval_results` ADD `failure_reason` integer DEFAULT 0 NOT NULL;
@@ -0,0 +1,2 @@
1
+ ALTER TABLE `evals` ADD `vars` text;--> statement-breakpoint
2
+ CREATE INDEX `eval_result_eval_id_test_idx_idx` ON `eval_results` (`eval_id`,`test_idx`);
@@ -0,0 +1,9 @@
1
+ DROP INDEX `eval_result_eval_id_test_idx_idx`;--> statement-breakpoint
2
+ CREATE INDEX `eval_result_test_idx_idx` ON `eval_results` (`test_idx`);--> statement-breakpoint
3
+ CREATE INDEX `eval_result_response_idx` ON `eval_results` (`response`);--> statement-breakpoint
4
+ CREATE INDEX `eval_result_grading_result_reason_idx` ON `eval_results` (json_extract(grading_result, '$.reason'));--> statement-breakpoint
5
+ CREATE INDEX `eval_result_grading_result_comment_idx` ON `eval_results` (json_extract(grading_result, '$.comment'));--> statement-breakpoint
6
+ CREATE INDEX `eval_result_test_case_vars_idx` ON `eval_results` (json_extract(test_case, '$.vars'));--> statement-breakpoint
7
+ CREATE INDEX `eval_result_test_case_metadata_idx` ON `eval_results` (json_extract(metadata, '$'));--> statement-breakpoint
8
+ CREATE INDEX `eval_result_named_scores_idx` ON `eval_results` (json_extract(named_scores, '$'));--> statement-breakpoint
9
+ CREATE INDEX `eval_result_metadata_idx` ON `eval_results` (json_extract(metadata, '$'));
@@ -0,0 +1,2 @@
1
+ DROP INDEX `eval_result_test_idx_idx`;--> statement-breakpoint
2
+ CREATE INDEX `eval_result_test_idx` ON `eval_results` (`test_idx`);
@@ -0,0 +1,29 @@
1
+ CREATE TABLE `spans` (
2
+ `id` text PRIMARY KEY NOT NULL,
3
+ `trace_id` text NOT NULL,
4
+ `span_id` text NOT NULL,
5
+ `parent_span_id` text,
6
+ `name` text NOT NULL,
7
+ `start_time` integer NOT NULL,
8
+ `end_time` integer,
9
+ `attributes` text,
10
+ `status_code` integer,
11
+ `status_message` text,
12
+ FOREIGN KEY (`trace_id`) REFERENCES `traces`(`trace_id`) ON UPDATE no action ON DELETE no action
13
+ );
14
+ --> statement-breakpoint
15
+ CREATE INDEX `spans_trace_id_idx` ON `spans` (`trace_id`);--> statement-breakpoint
16
+ CREATE INDEX `spans_span_id_idx` ON `spans` (`span_id`);--> statement-breakpoint
17
+ CREATE TABLE `traces` (
18
+ `id` text PRIMARY KEY NOT NULL,
19
+ `trace_id` text NOT NULL,
20
+ `evaluation_id` text NOT NULL,
21
+ `test_case_id` text NOT NULL,
22
+ `created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
23
+ `metadata` text,
24
+ FOREIGN KEY (`evaluation_id`) REFERENCES `evals`(`id`) ON UPDATE no action ON DELETE no action
25
+ );
26
+ --> statement-breakpoint
27
+ CREATE UNIQUE INDEX `traces_trace_id_unique` ON `traces` (`trace_id`);--> statement-breakpoint
28
+ CREATE INDEX `traces_evaluation_idx` ON `traces` (`evaluation_id`);--> statement-breakpoint
29
+ CREATE INDEX `traces_trace_id_idx` ON `traces` (`trace_id`);
@@ -0,0 +1,2 @@
1
+ CREATE INDEX `eval_result_metadata_plugin_id_idx` ON `eval_results` (json_extract(metadata, '$.pluginId'));--> statement-breakpoint
2
+ CREATE INDEX `eval_result_metadata_strategy_id_idx` ON `eval_results` (json_extract(metadata, '$.strategyId'));
@@ -0,0 +1,4 @@
1
+ CREATE INDEX `eval_result_eval_test_idx` ON `eval_results` (`eval_id`,`test_idx`);--> statement-breakpoint
2
+ CREATE INDEX `eval_result_eval_success_idx` ON `eval_results` (`eval_id`,`success`);--> statement-breakpoint
3
+ CREATE INDEX `eval_result_eval_failure_idx` ON `eval_results` (`eval_id`,`failure_reason`);--> statement-breakpoint
4
+ CREATE INDEX `eval_result_eval_test_success_idx` ON `eval_results` (`eval_id`,`test_idx`,`success`);
@@ -0,0 +1,22 @@
1
+ CREATE TABLE `model_audits` (
2
+ `id` text PRIMARY KEY NOT NULL,
3
+ `created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
4
+ `updated_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
5
+ `name` text,
6
+ `author` text,
7
+ `model_path` text NOT NULL,
8
+ `model_type` text,
9
+ `results` text NOT NULL,
10
+ `checks` text,
11
+ `issues` text,
12
+ `has_errors` integer NOT NULL,
13
+ `total_checks` integer,
14
+ `passed_checks` integer,
15
+ `failed_checks` integer,
16
+ `metadata` text
17
+ );
18
+ --> statement-breakpoint
19
+ CREATE INDEX `model_audits_created_at_idx` ON `model_audits` (`created_at`);--> statement-breakpoint
20
+ CREATE INDEX `model_audits_model_path_idx` ON `model_audits` (`model_path`);--> statement-breakpoint
21
+ CREATE INDEX `model_audits_has_errors_idx` ON `model_audits` (`has_errors`);--> statement-breakpoint
22
+ CREATE INDEX `model_audits_model_type_idx` ON `model_audits` (`model_type`);
@@ -0,0 +1,8 @@
1
+ ALTER TABLE `evals` ADD `is_redteam` integer DEFAULT false NOT NULL;--> statement-breakpoint
2
+ CREATE INDEX `evals_is_redteam_idx` ON `evals` (`is_redteam`);--> statement-breakpoint
3
+
4
+
5
+ UPDATE `evals` set `is_redteam` = CASE
6
+ WHEN json_valid(config) AND json_extract(config, '$.redteam') IS NOT NULL THEN 1
7
+ ELSE 0
8
+ END;
@@ -0,0 +1 @@
1
+ ALTER TABLE `evals` ADD `runtime_options` text;
@@ -0,0 +1,13 @@
1
+ ALTER TABLE `model_audits` ADD `model_id` text;--> statement-breakpoint
2
+ ALTER TABLE `model_audits` ADD `revision_sha` text;--> statement-breakpoint
3
+ ALTER TABLE `model_audits` ADD `content_hash` text;--> statement-breakpoint
4
+ ALTER TABLE `model_audits` ADD `model_source` text;--> statement-breakpoint
5
+ ALTER TABLE `model_audits` ADD `source_last_modified` integer;--> statement-breakpoint
6
+ ALTER TABLE `model_audits` ADD `scanner_version` text;--> statement-breakpoint
7
+ CREATE INDEX `model_audits_model_id_idx` ON `model_audits` (`model_id`);--> statement-breakpoint
8
+ CREATE INDEX `model_audits_revision_sha_idx` ON `model_audits` (`revision_sha`);--> statement-breakpoint
9
+ CREATE INDEX `model_audits_content_hash_idx` ON `model_audits` (`content_hash`);--> statement-breakpoint
10
+ CREATE INDEX `model_audits_model_revision_idx` ON `model_audits` (`model_id`,`revision_sha`);--> statement-breakpoint
11
+ CREATE INDEX `model_audits_model_content_idx` ON `model_audits` (`model_id`,`content_hash`);--> statement-breakpoint
12
+ CREATE UNIQUE INDEX `idx_model_audits_unique_revision` ON `model_audits` (`model_id`, `revision_sha`) WHERE `revision_sha` IS NOT NULL;--> statement-breakpoint
13
+ CREATE UNIQUE INDEX `idx_model_audits_unique_content` ON `model_audits` (`model_id`, `content_hash`) WHERE `revision_sha` IS NULL AND `content_hash` IS NOT NULL;
@@ -0,0 +1,25 @@
1
+ CREATE TABLE `blob_assets` (
2
+ `hash` text PRIMARY KEY NOT NULL,
3
+ `size_bytes` integer NOT NULL,
4
+ `mime_type` text NOT NULL,
5
+ `provider` text NOT NULL,
6
+ `created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL
7
+ );
8
+ --> statement-breakpoint
9
+ CREATE INDEX `blob_assets_provider_idx` ON `blob_assets` (`provider`);--> statement-breakpoint
10
+ CREATE INDEX `blob_assets_created_at_idx` ON `blob_assets` (`created_at`);--> statement-breakpoint
11
+ CREATE TABLE `blob_references` (
12
+ `id` text PRIMARY KEY NOT NULL,
13
+ `blob_hash` text NOT NULL,
14
+ `eval_id` text NOT NULL,
15
+ `test_idx` integer,
16
+ `prompt_idx` integer,
17
+ `location` text,
18
+ `kind` text,
19
+ `created_at` integer DEFAULT CURRENT_TIMESTAMP NOT NULL,
20
+ FOREIGN KEY (`blob_hash`) REFERENCES `blob_assets`(`hash`) ON UPDATE no action ON DELETE cascade,
21
+ FOREIGN KEY (`eval_id`) REFERENCES `evals`(`id`) ON UPDATE no action ON DELETE cascade
22
+ );
23
+ --> statement-breakpoint
24
+ CREATE INDEX `blob_references_blob_idx` ON `blob_references` (`blob_hash`);--> statement-breakpoint
25
+ CREATE INDEX `blob_references_eval_idx` ON `blob_references` (`eval_id`);
@@ -0,0 +1,2 @@
1
+ CREATE INDEX `blob_assets_mime_type_idx` ON `blob_assets` (`mime_type`);--> statement-breakpoint
2
+ CREATE INDEX `blob_references_blob_created_at_idx` ON `blob_references` (`blob_hash`,`created_at`);
@@ -0,0 +1,68 @@
1
+ # Database Migrations
2
+
3
+ **What this is:** Drizzle ORM migration files for SQLite database. Auto-generated SQL that applies schema changes.
4
+
5
+ ## Schema is NOT Here
6
+
7
+ **Schema definitions are in `src/database/schema.ts`, NOT in this directory.**
8
+
9
+ This directory only contains generated SQL migration files.
10
+
11
+ ## Migration Workflow
12
+
13
+ 1. **Modify schema** in `src/database/schema.ts`
14
+ 2. **Generate migration:** `npm run db:generate`
15
+ 3. **Review SQL** in new `drizzle/NNNN_name.sql` file
16
+ 4. **Apply migration:** `npm run db:migrate`
17
+
18
+ ## File Naming
19
+
20
+ Drizzle auto-generates names: `NNNN_adjective_noun.sql`
21
+
22
+ - `0000_lush_hellion.sql`
23
+ - `0001_wide_calypso.sql`
24
+ - Sequential numbering ensures order
25
+
26
+ ## Key Commands
27
+
28
+ ```bash
29
+ npm run db:generate # Generate migration from schema changes
30
+ npm run db:migrate # Apply pending migrations
31
+ npm run db:studio # Open GUI at localhost:4983
32
+ ```
33
+
34
+ ## Database Location
35
+
36
+ Default: `~/.promptfoo/promptfoo.db` (SQLite file)
37
+
38
+ Override with `PROMPTFOO_DB_PATH` env var.
39
+
40
+ **NEVER delete the database.** You may read from it but deletion destroys user data.
41
+
42
+ ## SQLite Limitations
43
+
44
+ **Cannot drop columns** - SQLite doesn't support `ALTER TABLE DROP COLUMN`.
45
+
46
+ Workaround: Create new table, copy data, drop old table, rename new table.
47
+
48
+ **Cannot rename columns** (older SQLite) - Same workaround as drop column.
49
+
50
+ ## Migration Best Practices
51
+
52
+ 1. **Always review generated SQL** before applying
53
+ 2. **Never edit applied migrations** - create new migration instead
54
+ 3. **Backup before major changes:** `sqlite3 ~/.promptfoo/promptfoo.db ".backup backup.db"`
55
+ 4. **Test with in-memory DB** in tests (see `test/` for patterns)
56
+
57
+ ## What's Stored
58
+
59
+ - Evaluation results and test configurations
60
+ - Execution traces for debugging
61
+ - Analytics data and telemetry
62
+ - User settings and preferences
63
+
64
+ ## Tech Stack
65
+
66
+ - **Drizzle ORM** - Type-safe schema and migrations
67
+ - **Better SQLite3** - Fast synchronous driver
68
+ - **SQLite** - Embedded database
@@ -0,0 +1 @@
1
+ @AGENTS.md
@@ -0,0 +1,221 @@
1
+ {
2
+ "version": "6",
3
+ "dialect": "sqlite",
4
+ "tables": {
5
+ "datasets": {
6
+ "name": "datasets",
7
+ "columns": {
8
+ "id": {
9
+ "name": "id",
10
+ "type": "text",
11
+ "primaryKey": true,
12
+ "notNull": true,
13
+ "autoincrement": false
14
+ },
15
+ "test_case_id": {
16
+ "name": "test_case_id",
17
+ "type": "text",
18
+ "primaryKey": false,
19
+ "notNull": true,
20
+ "autoincrement": false
21
+ },
22
+ "created_at": {
23
+ "name": "created_at",
24
+ "type": "integer",
25
+ "primaryKey": false,
26
+ "notNull": true,
27
+ "autoincrement": false,
28
+ "default": "CURRENT_TIMESTAMP"
29
+ }
30
+ },
31
+ "indexes": {},
32
+ "foreignKeys": {},
33
+ "compositePrimaryKeys": {},
34
+ "uniqueConstraints": {}
35
+ },
36
+ "evals": {
37
+ "name": "evals",
38
+ "columns": {
39
+ "id": {
40
+ "name": "id",
41
+ "type": "text",
42
+ "primaryKey": true,
43
+ "notNull": true,
44
+ "autoincrement": false
45
+ },
46
+ "created_at": {
47
+ "name": "created_at",
48
+ "type": "integer",
49
+ "primaryKey": false,
50
+ "notNull": true,
51
+ "autoincrement": false,
52
+ "default": "CURRENT_TIMESTAMP"
53
+ },
54
+ "description": {
55
+ "name": "description",
56
+ "type": "text",
57
+ "primaryKey": false,
58
+ "notNull": false,
59
+ "autoincrement": false
60
+ },
61
+ "results": {
62
+ "name": "results",
63
+ "type": "text",
64
+ "primaryKey": false,
65
+ "notNull": true,
66
+ "autoincrement": false
67
+ },
68
+ "config": {
69
+ "name": "config",
70
+ "type": "text",
71
+ "primaryKey": false,
72
+ "notNull": true,
73
+ "autoincrement": false
74
+ }
75
+ },
76
+ "indexes": {},
77
+ "foreignKeys": {},
78
+ "compositePrimaryKeys": {},
79
+ "uniqueConstraints": {}
80
+ },
81
+ "evals_to_datasets": {
82
+ "name": "evals_to_datasets",
83
+ "columns": {
84
+ "eval_id": {
85
+ "name": "eval_id",
86
+ "type": "text",
87
+ "primaryKey": false,
88
+ "notNull": true,
89
+ "autoincrement": false
90
+ },
91
+ "dataset_id": {
92
+ "name": "dataset_id",
93
+ "type": "text",
94
+ "primaryKey": false,
95
+ "notNull": true,
96
+ "autoincrement": false
97
+ }
98
+ },
99
+ "indexes": {},
100
+ "foreignKeys": {
101
+ "evals_to_datasets_eval_id_evals_id_fk": {
102
+ "name": "evals_to_datasets_eval_id_evals_id_fk",
103
+ "tableFrom": "evals_to_datasets",
104
+ "columnsFrom": ["eval_id"],
105
+ "tableTo": "evals",
106
+ "columnsTo": ["id"],
107
+ "onUpdate": "no action",
108
+ "onDelete": "no action"
109
+ },
110
+ "evals_to_datasets_dataset_id_datasets_id_fk": {
111
+ "name": "evals_to_datasets_dataset_id_datasets_id_fk",
112
+ "tableFrom": "evals_to_datasets",
113
+ "columnsFrom": ["dataset_id"],
114
+ "tableTo": "datasets",
115
+ "columnsTo": ["id"],
116
+ "onUpdate": "no action",
117
+ "onDelete": "no action"
118
+ }
119
+ },
120
+ "compositePrimaryKeys": {
121
+ "evals_to_datasets_eval_id_dataset_id_pk": {
122
+ "columns": ["dataset_id", "eval_id"],
123
+ "name": "evals_to_datasets_eval_id_dataset_id_pk"
124
+ }
125
+ },
126
+ "uniqueConstraints": {}
127
+ },
128
+ "evals_to_prompts": {
129
+ "name": "evals_to_prompts",
130
+ "columns": {
131
+ "eval_id": {
132
+ "name": "eval_id",
133
+ "type": "text",
134
+ "primaryKey": false,
135
+ "notNull": true,
136
+ "autoincrement": false
137
+ },
138
+ "prompt_id": {
139
+ "name": "prompt_id",
140
+ "type": "text",
141
+ "primaryKey": false,
142
+ "notNull": true,
143
+ "autoincrement": false
144
+ }
145
+ },
146
+ "indexes": {},
147
+ "foreignKeys": {
148
+ "evals_to_prompts_eval_id_evals_id_fk": {
149
+ "name": "evals_to_prompts_eval_id_evals_id_fk",
150
+ "tableFrom": "evals_to_prompts",
151
+ "columnsFrom": ["eval_id"],
152
+ "tableTo": "evals",
153
+ "columnsTo": ["id"],
154
+ "onUpdate": "no action",
155
+ "onDelete": "no action"
156
+ },
157
+ "evals_to_prompts_prompt_id_prompts_id_fk": {
158
+ "name": "evals_to_prompts_prompt_id_prompts_id_fk",
159
+ "tableFrom": "evals_to_prompts",
160
+ "columnsFrom": ["prompt_id"],
161
+ "tableTo": "prompts",
162
+ "columnsTo": ["id"],
163
+ "onUpdate": "no action",
164
+ "onDelete": "no action"
165
+ }
166
+ },
167
+ "compositePrimaryKeys": {
168
+ "evals_to_prompts_eval_id_prompt_id_pk": {
169
+ "columns": ["eval_id", "prompt_id"],
170
+ "name": "evals_to_prompts_eval_id_prompt_id_pk"
171
+ }
172
+ },
173
+ "uniqueConstraints": {}
174
+ },
175
+ "prompts": {
176
+ "name": "prompts",
177
+ "columns": {
178
+ "id": {
179
+ "name": "id",
180
+ "type": "text",
181
+ "primaryKey": true,
182
+ "notNull": true,
183
+ "autoincrement": false
184
+ },
185
+ "created_at": {
186
+ "name": "created_at",
187
+ "type": "integer",
188
+ "primaryKey": false,
189
+ "notNull": true,
190
+ "autoincrement": false,
191
+ "default": "CURRENT_TIMESTAMP"
192
+ },
193
+ "prompt": {
194
+ "name": "prompt",
195
+ "type": "text",
196
+ "primaryKey": false,
197
+ "notNull": true,
198
+ "autoincrement": false
199
+ },
200
+ "hash": {
201
+ "name": "hash",
202
+ "type": "text",
203
+ "primaryKey": false,
204
+ "notNull": true,
205
+ "autoincrement": false
206
+ }
207
+ },
208
+ "indexes": {},
209
+ "foreignKeys": {},
210
+ "compositePrimaryKeys": {},
211
+ "uniqueConstraints": {}
212
+ }
213
+ },
214
+ "enums": {},
215
+ "_meta": {
216
+ "tables": {},
217
+ "columns": {}
218
+ },
219
+ "id": "8b53403f-5b6f-436a-862e-9fd17a52204e",
220
+ "prevId": "00000000-0000-0000-0000-000000000000"
221
+ }