@use-lattice/litmus 0.121.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (199) hide show
  1. package/LICENSE +19 -0
  2. package/dist/src/accounts-Bt1oJb1Z.cjs +219 -0
  3. package/dist/src/accounts-DjOU8Rm3.js +178 -0
  4. package/dist/src/agentic-utils-D03IiXQc.js +153 -0
  5. package/dist/src/agentic-utils-Dh7xaMQM.cjs +180 -0
  6. package/dist/src/agents-C6BIMlZa.js +231 -0
  7. package/dist/src/agents-DvIpNX1L.cjs +666 -0
  8. package/dist/src/agents-ZP0RP9vV.cjs +231 -0
  9. package/dist/src/agents-maJXdjbR.js +665 -0
  10. package/dist/src/aimlapi-BTbQjG2E.cjs +30 -0
  11. package/dist/src/aimlapi-CwMxqfXP.js +30 -0
  12. package/dist/src/audio-BBUdvsde.cjs +97 -0
  13. package/dist/src/audio-D5DPZ7I-.js +97 -0
  14. package/dist/src/base-BEysXrkq.cjs +222 -0
  15. package/dist/src/base-C451JQfq.js +193 -0
  16. package/dist/src/blobs-BY8MDmpo.js +230 -0
  17. package/dist/src/blobs-BgcNn97m.cjs +256 -0
  18. package/dist/src/cache-BBE_lsTA.cjs +4 -0
  19. package/dist/src/cache-BkrqU5Ba.js +237 -0
  20. package/dist/src/cache-DsCxFlsZ.cjs +297 -0
  21. package/dist/src/chat-CPJWDP6a.cjs +289 -0
  22. package/dist/src/chat-CXX3xzkk.cjs +811 -0
  23. package/dist/src/chat-CcDgZFJ4.js +787 -0
  24. package/dist/src/chat-Dz5ZeGO2.js +289 -0
  25. package/dist/src/chatkit-Dw0mKkML.cjs +1158 -0
  26. package/dist/src/chatkit-swAIVuea.js +1157 -0
  27. package/dist/src/chunk-DEq-mXcV.js +15 -0
  28. package/dist/src/claude-agent-sdk-BXZJtOg6.js +379 -0
  29. package/dist/src/claude-agent-sdk-CkfyjDoG.cjs +383 -0
  30. package/dist/src/cloudflare-ai-BzpJcqUH.js +161 -0
  31. package/dist/src/cloudflare-ai-Cmy_R1y2.cjs +161 -0
  32. package/dist/src/cloudflare-gateway-B9tVQKok.cjs +272 -0
  33. package/dist/src/cloudflare-gateway-DrD3ew3H.js +272 -0
  34. package/dist/src/codex-sdk-Dezj9Nwm.js +1056 -0
  35. package/dist/src/codex-sdk-Dl9D4k5B.cjs +1060 -0
  36. package/dist/src/cometapi-C-9YvCHC.js +54 -0
  37. package/dist/src/cometapi-DHgDKoO2.cjs +54 -0
  38. package/dist/src/completion-B8Ctyxpr.js +120 -0
  39. package/dist/src/completion-Cxrt08sj.cjs +131 -0
  40. package/dist/src/createHash-BwgE13yv.cjs +27 -0
  41. package/dist/src/createHash-DmPQkvBh.js +15 -0
  42. package/dist/src/docker-BiqcTwLv.js +80 -0
  43. package/dist/src/docker-C7tEJnP-.cjs +80 -0
  44. package/dist/src/esm-C62Zofr1.cjs +409 -0
  45. package/dist/src/esm-DMVc93eh.js +379 -0
  46. package/dist/src/evalResult-C3NJPQOo.cjs +301 -0
  47. package/dist/src/evalResult-C7JJAPBb.js +295 -0
  48. package/dist/src/evalResult-DoVTZZWI.cjs +2 -0
  49. package/dist/src/extractor-DnMD3fwt.cjs +391 -0
  50. package/dist/src/extractor-DtlL28vL.js +374 -0
  51. package/dist/src/fetch-BTxakTSg.cjs +1133 -0
  52. package/dist/src/fetch-DQckpUFz.js +928 -0
  53. package/dist/src/fileExtensions-DnqA1y9x.js +85 -0
  54. package/dist/src/fileExtensions-bYh77CN8.cjs +114 -0
  55. package/dist/src/genaiTracer-CyZrmaK0.cjs +268 -0
  56. package/dist/src/genaiTracer-D3fD9dNV.js +256 -0
  57. package/dist/src/graders-BNscxFrU.js +13644 -0
  58. package/dist/src/graders-D2oE9Msq.js +2 -0
  59. package/dist/src/graders-c0Ez_w-9.cjs +2 -0
  60. package/dist/src/graders-d0F2M3e9.cjs +14056 -0
  61. package/dist/src/image-0ZhE0VlR.cjs +280 -0
  62. package/dist/src/image-CWE1pdNv.js +257 -0
  63. package/dist/src/image-D9ZK6hwL.js +163 -0
  64. package/dist/src/image-DKZgZITg.cjs +163 -0
  65. package/dist/src/index.cjs +11366 -0
  66. package/dist/src/index.d.cts +19640 -0
  67. package/dist/src/index.d.ts +19641 -0
  68. package/dist/src/index.js +11306 -0
  69. package/dist/src/invariant-Ddh24eXh.js +25 -0
  70. package/dist/src/invariant-kfQ8Bu82.cjs +30 -0
  71. package/dist/src/knowledgeBase-BgPyGFUd.cjs +122 -0
  72. package/dist/src/knowledgeBase-DyHilYaP.js +122 -0
  73. package/dist/src/litellm-CyMeneHS.js +135 -0
  74. package/dist/src/litellm-DWDF73yF.cjs +135 -0
  75. package/dist/src/logger-C40ZGil9.js +717 -0
  76. package/dist/src/logger-DyfK9PBt.cjs +917 -0
  77. package/dist/src/luma-ray-BAU9X_ep.cjs +315 -0
  78. package/dist/src/luma-ray-nwVseBbv.js +313 -0
  79. package/dist/src/messages-B5ADWTTv.js +245 -0
  80. package/dist/src/messages-BCnZfqrS.cjs +257 -0
  81. package/dist/src/meteor-DLZZ3osF.cjs +134 -0
  82. package/dist/src/meteor-DUiCJRC-.js +134 -0
  83. package/dist/src/modelslab-00cveB8L.cjs +163 -0
  84. package/dist/src/modelslab-D9sCU_L7.js +163 -0
  85. package/dist/src/nova-reel-CTapvqYH.js +276 -0
  86. package/dist/src/nova-reel-DlWuuroF.cjs +278 -0
  87. package/dist/src/nova-sonic-5UPWfeMv.cjs +363 -0
  88. package/dist/src/nova-sonic-BhSwQNym.js +363 -0
  89. package/dist/src/openai-BWrJK9d8.cjs +52 -0
  90. package/dist/src/openai-DumO8WQn.js +47 -0
  91. package/dist/src/openclaw-B8brrjC_.cjs +577 -0
  92. package/dist/src/openclaw-Bkayww9q.js +571 -0
  93. package/dist/src/opencode-sdk-7xjoDNiM.cjs +562 -0
  94. package/dist/src/opencode-sdk-SGwAPxht.js +558 -0
  95. package/dist/src/otlpReceiver-CoAHfAN9.cjs +15 -0
  96. package/dist/src/otlpReceiver-oO3EQwI9.js +14 -0
  97. package/dist/src/providerRegistry-4yjhaEM8.js +45 -0
  98. package/dist/src/providerRegistry-DhV4rJIc.cjs +50 -0
  99. package/dist/src/providers-B5RJVG-7.cjs +33609 -0
  100. package/dist/src/providers-BdmZCLzV.js +33262 -0
  101. package/dist/src/providers-CxtRxn8e.js +2 -0
  102. package/dist/src/providers-DnQLNbx1.cjs +3 -0
  103. package/dist/src/pythonUtils-BD0druiM.cjs +275 -0
  104. package/dist/src/pythonUtils-IBhn5YGR.js +249 -0
  105. package/dist/src/quiverai-BDOwZBsM.cjs +213 -0
  106. package/dist/src/quiverai-D3JTF5lD.js +213 -0
  107. package/dist/src/responses-B2LCDCXZ.js +667 -0
  108. package/dist/src/responses-BvNm4Xv9.cjs +685 -0
  109. package/dist/src/rubyUtils-B0NwnfpY.cjs +245 -0
  110. package/dist/src/rubyUtils-BroxzZ7c.cjs +2 -0
  111. package/dist/src/rubyUtils-hqVw5UvJ.js +222 -0
  112. package/dist/src/sagemaker-Cno2V-Sx.js +689 -0
  113. package/dist/src/sagemaker-fV_KUgs5.cjs +691 -0
  114. package/dist/src/server-BOuAXb06.cjs +238 -0
  115. package/dist/src/server-CtI-EWzm.cjs +2 -0
  116. package/dist/src/server-Cy3DZymt.js +189 -0
  117. package/dist/src/slack-CP8xBePa.js +135 -0
  118. package/dist/src/slack-DSQ1yXVb.cjs +135 -0
  119. package/dist/src/store-BwDDaBjb.cjs +246 -0
  120. package/dist/src/store-DcbLC593.cjs +2 -0
  121. package/dist/src/store-IGpqMIkv.js +240 -0
  122. package/dist/src/tables-3Q2cL7So.cjs +373 -0
  123. package/dist/src/tables-Bi2fjr4W.js +288 -0
  124. package/dist/src/telemetry-Bg2WqF79.js +161 -0
  125. package/dist/src/telemetry-D0x6u5kX.cjs +166 -0
  126. package/dist/src/telemetry-DXNimrI0.cjs +2 -0
  127. package/dist/src/text-B_UCRPp2.js +22 -0
  128. package/dist/src/text-CW1cyrwj.cjs +33 -0
  129. package/dist/src/tokenUsageUtils-NYT-WKS6.js +138 -0
  130. package/dist/src/tokenUsageUtils-bVa1ga6f.cjs +173 -0
  131. package/dist/src/transcription-Cl_W16Pr.js +122 -0
  132. package/dist/src/transcription-yt1EecY8.cjs +124 -0
  133. package/dist/src/transform-BCtGrl_W.cjs +228 -0
  134. package/dist/src/transform-Bv6gG2MJ.cjs +1688 -0
  135. package/dist/src/transform-CY1wbpRy.js +1507 -0
  136. package/dist/src/transform-DU8rUL9P.cjs +2 -0
  137. package/dist/src/transform-yWaShiKr.js +216 -0
  138. package/dist/src/transformersAvailability-BGkzavwb.js +35 -0
  139. package/dist/src/transformersAvailability-DKoRtQLy.cjs +35 -0
  140. package/dist/src/types-5aqHpBwE.cjs +3769 -0
  141. package/dist/src/types-Bn6D9c4U.js +3300 -0
  142. package/dist/src/util-BkKlTkI2.js +293 -0
  143. package/dist/src/util-CTh0bfOm.cjs +1119 -0
  144. package/dist/src/util-D17oBwo7.cjs +328 -0
  145. package/dist/src/util-DsS_-v4p.js +613 -0
  146. package/dist/src/util-DuntT1Ga.js +951 -0
  147. package/dist/src/util-aWjdCYMI.cjs +667 -0
  148. package/dist/src/utils-CisQwpjA.js +94 -0
  149. package/dist/src/utils-yWamDvmz.cjs +123 -0
  150. package/dist/tsconfig.tsbuildinfo +1 -0
  151. package/drizzle/0000_lush_hellion.sql +36 -0
  152. package/drizzle/0001_wide_calypso.sql +3 -0
  153. package/drizzle/0002_tidy_juggernaut.sql +1 -0
  154. package/drizzle/0003_lively_naoko.sql +8 -0
  155. package/drizzle/0004_minor_peter_quill.sql +19 -0
  156. package/drizzle/0005_silky_millenium_guard.sql +2 -0
  157. package/drizzle/0006_harsh_caretaker.sql +42 -0
  158. package/drizzle/0007_cloudy_wong.sql +1 -0
  159. package/drizzle/0008_broad_boomer.sql +2 -0
  160. package/drizzle/0009_strong_marten_broadcloak.sql +19 -0
  161. package/drizzle/0010_needy_bishop.sql +11 -0
  162. package/drizzle/0011_moaning_millenium_guard.sql +1 -0
  163. package/drizzle/0012_late_marten_broadcloak.sql +2 -0
  164. package/drizzle/0013_previous_dormammu.sql +9 -0
  165. package/drizzle/0014_lazy_captain_universe.sql +2 -0
  166. package/drizzle/0015_zippy_wallop.sql +29 -0
  167. package/drizzle/0016_jazzy_zemo.sql +2 -0
  168. package/drizzle/0017_reflective_praxagora.sql +4 -0
  169. package/drizzle/0018_fat_vanisher.sql +22 -0
  170. package/drizzle/0019_new_clint_barton.sql +8 -0
  171. package/drizzle/0020_skinny_maverick.sql +1 -0
  172. package/drizzle/0021_mysterious_madelyne_pryor.sql +13 -0
  173. package/drizzle/0022_sleepy_ultimo.sql +25 -0
  174. package/drizzle/0023_wooden_mandrill.sql +2 -0
  175. package/drizzle/AGENTS.md +68 -0
  176. package/drizzle/CLAUDE.md +1 -0
  177. package/drizzle/meta/0000_snapshot.json +221 -0
  178. package/drizzle/meta/0001_snapshot.json +214 -0
  179. package/drizzle/meta/0002_snapshot.json +221 -0
  180. package/drizzle/meta/0005_snapshot.json +369 -0
  181. package/drizzle/meta/0006_snapshot.json +638 -0
  182. package/drizzle/meta/0007_snapshot.json +640 -0
  183. package/drizzle/meta/0008_snapshot.json +649 -0
  184. package/drizzle/meta/0009_snapshot.json +554 -0
  185. package/drizzle/meta/0010_snapshot.json +619 -0
  186. package/drizzle/meta/0011_snapshot.json +627 -0
  187. package/drizzle/meta/0012_snapshot.json +639 -0
  188. package/drizzle/meta/0013_snapshot.json +717 -0
  189. package/drizzle/meta/0014_snapshot.json +717 -0
  190. package/drizzle/meta/0015_snapshot.json +897 -0
  191. package/drizzle/meta/0016_snapshot.json +1031 -0
  192. package/drizzle/meta/0018_snapshot.json +1210 -0
  193. package/drizzle/meta/0019_snapshot.json +1165 -0
  194. package/drizzle/meta/0020_snapshot.json +1232 -0
  195. package/drizzle/meta/0021_snapshot.json +1311 -0
  196. package/drizzle/meta/0022_snapshot.json +1481 -0
  197. package/drizzle/meta/0023_snapshot.json +1496 -0
  198. package/drizzle/meta/_journal.json +174 -0
  199. package/package.json +240 -0
@@ -0,0 +1,295 @@
1
+ import { t as __exportAll } from "./chunk-DEq-mXcV.js";
2
+ import { h as safeJsonStringify, r as logger, v as getEnvBool } from "./logger-C40ZGil9.js";
3
+ import { A as isApiProvider, g as ResultFailureReason, j as isProviderOptions, k as isResultFailureReason } from "./types-Bn6D9c4U.js";
4
+ import { u as getCurrentTimestamp } from "./fetch-DQckpUFz.js";
5
+ import { t as hashPrompt } from "./utils-CisQwpjA.js";
6
+ import { i as evalResultsTable, p as getDb } from "./tables-Bi2fjr4W.js";
7
+ import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-DtlL28vL.js";
8
+ import { and, eq, gte, inArray, lt, ne } from "drizzle-orm";
9
+ //#region src/models/evalResult.ts
10
+ var evalResult_exports = /* @__PURE__ */ __exportAll({
11
+ default: () => EvalResult,
12
+ sanitizeProvider: () => sanitizeProvider
13
+ });
14
+ function sanitizeProvider(provider) {
15
+ try {
16
+ if (isApiProvider(provider)) return {
17
+ id: provider.id(),
18
+ label: provider.label,
19
+ ...provider.config && { config: JSON.parse(safeJsonStringify(provider.config)) }
20
+ };
21
+ if (isProviderOptions(provider)) return {
22
+ id: provider.id,
23
+ label: provider.label,
24
+ ...provider.config && { config: JSON.parse(safeJsonStringify(provider.config)) }
25
+ };
26
+ if (typeof provider === "object" && provider) {
27
+ const providerObj = provider;
28
+ return {
29
+ id: typeof providerObj.id === "function" ? providerObj.id() : providerObj.id,
30
+ label: providerObj.label,
31
+ ...providerObj.config && { config: JSON.parse(safeJsonStringify(providerObj.config)) }
32
+ };
33
+ }
34
+ } catch {}
35
+ return JSON.parse(safeJsonStringify(provider));
36
+ }
37
+ /**
38
+ * Sanitize an object for database storage by removing circular references
39
+ * and non-serializable values (functions, Timeout objects, etc.).
40
+ * Uses safeJsonStringify which handles circular references gracefully.
41
+ *
42
+ * This prevents "Converting circular structure to JSON" errors that can occur
43
+ * when Node.js Timeout objects or other non-serializable data leaks into results.
44
+ * See: https://github.com/promptfoo/promptfoo/issues/7266
45
+ */
46
+ function sanitizeForDb(obj) {
47
+ if (obj === null || obj === void 0) return obj;
48
+ try {
49
+ const serialized = safeJsonStringify(obj);
50
+ if (serialized === void 0) {
51
+ logger.debug("sanitizeForDb: Failed to serialize object, using fallback", {
52
+ valueType: typeof obj,
53
+ isArray: Array.isArray(obj)
54
+ });
55
+ return Array.isArray(obj) ? [] : null;
56
+ }
57
+ return JSON.parse(serialized);
58
+ } catch (error) {
59
+ logger.debug("sanitizeForDb: Parse error, using fallback", { error });
60
+ return Array.isArray(obj) ? [] : null;
61
+ }
62
+ }
63
+ var EvalResult = class EvalResult {
64
+ static async createFromEvaluateResult(evalId, result, opts) {
65
+ const persist = opts?.persist == null ? true : opts.persist;
66
+ const { prompt, error, score, latencyMs, success, provider, gradingResult, namedScores, cost, metadata, failureReason, testCase } = result;
67
+ const preSanitizeTestCase = {
68
+ ...testCase,
69
+ ...testCase.provider && { provider: sanitizeProvider(testCase.provider) }
70
+ };
71
+ const processedResponse = await extractAndStoreBinaryData(result.response, {
72
+ evalId,
73
+ testIdx: result.testIdx,
74
+ promptIdx: result.promptIdx
75
+ });
76
+ const args = {
77
+ id: crypto.randomUUID(),
78
+ evalId,
79
+ testCase: sanitizeForDb(preSanitizeTestCase),
80
+ promptIdx: result.promptIdx,
81
+ testIdx: result.testIdx,
82
+ prompt: sanitizeForDb(prompt),
83
+ promptId: hashPrompt(prompt),
84
+ error: error?.toString(),
85
+ success,
86
+ score: score == null ? 0 : score,
87
+ response: sanitizeForDb(processedResponse || null),
88
+ gradingResult: sanitizeForDb(gradingResult || null),
89
+ namedScores: sanitizeForDb(namedScores),
90
+ provider: sanitizeProvider(provider),
91
+ latencyMs,
92
+ cost,
93
+ metadata: sanitizeForDb(metadata),
94
+ failureReason
95
+ };
96
+ if (persist) return new EvalResult({
97
+ ...(await getDb().insert(evalResultsTable).values(args).returning())[0],
98
+ persisted: true
99
+ });
100
+ return new EvalResult(args);
101
+ }
102
+ static async createManyFromEvaluateResult(results, evalId) {
103
+ const db = getDb();
104
+ const returnResults = [];
105
+ const processedResults = [];
106
+ for (const result of results) {
107
+ const processedResponse = isBlobStorageEnabled() ? await extractAndStoreBinaryData(result.response, {
108
+ evalId,
109
+ testIdx: result.testIdx,
110
+ promptIdx: result.promptIdx
111
+ }) : result.response;
112
+ processedResults.push({
113
+ ...result,
114
+ response: processedResponse ?? void 0
115
+ });
116
+ }
117
+ db.transaction(() => {
118
+ for (const result of processedResults) {
119
+ const sanitizedResult = {
120
+ ...result,
121
+ testCase: sanitizeForDb(result.testCase),
122
+ prompt: sanitizeForDb(result.prompt),
123
+ response: sanitizeForDb(result.response),
124
+ gradingResult: sanitizeForDb(result.gradingResult),
125
+ namedScores: sanitizeForDb(result.namedScores),
126
+ metadata: sanitizeForDb(result.metadata),
127
+ provider: result.provider ? sanitizeProvider(result.provider) : result.provider
128
+ };
129
+ const dbResult = db.insert(evalResultsTable).values({
130
+ ...sanitizedResult,
131
+ evalId,
132
+ id: crypto.randomUUID()
133
+ }).returning().get();
134
+ returnResults.push(new EvalResult({
135
+ ...dbResult,
136
+ persisted: true
137
+ }));
138
+ }
139
+ });
140
+ return returnResults;
141
+ }
142
+ static async findById(id) {
143
+ const result = await getDb().select().from(evalResultsTable).where(eq(evalResultsTable.id, id));
144
+ return result.length > 0 ? new EvalResult({
145
+ ...result[0],
146
+ persisted: true
147
+ }) : null;
148
+ }
149
+ static async findManyByEvalId(evalId, opts) {
150
+ return (await getDb().select().from(evalResultsTable).where(and(eq(evalResultsTable.evalId, evalId), opts?.testIdx == null ? void 0 : eq(evalResultsTable.testIdx, opts.testIdx)))).map((result) => new EvalResult({
151
+ ...result,
152
+ persisted: true
153
+ }));
154
+ }
155
+ static async findManyByEvalIdAndTestIndices(evalId, testIndices) {
156
+ if (!testIndices.length) return [];
157
+ return (await getDb().select().from(evalResultsTable).where(and(eq(evalResultsTable.evalId, evalId), testIndices.length === 1 ? eq(evalResultsTable.testIdx, testIndices[0]) : inArray(evalResultsTable.testIdx, testIndices)))).map((result) => new EvalResult({
158
+ ...result,
159
+ persisted: true
160
+ }));
161
+ }
162
+ /**
163
+ * Returns a set of completed (testIdx,promptIdx) pairs for a given eval.
164
+ * Key format: `${testIdx}:${promptIdx}`
165
+ *
166
+ * @param evalId - The evaluation ID to query
167
+ * @param opts.excludeErrors - If true, excludes results with ERROR failureReason (used in retry mode)
168
+ */
169
+ static async getCompletedIndexPairs(evalId, opts) {
170
+ const db = getDb();
171
+ const whereClause = opts?.excludeErrors ? and(eq(evalResultsTable.evalId, evalId), ne(evalResultsTable.failureReason, ResultFailureReason.ERROR)) : eq(evalResultsTable.evalId, evalId);
172
+ const rows = await db.select({
173
+ testIdx: evalResultsTable.testIdx,
174
+ promptIdx: evalResultsTable.promptIdx
175
+ }).from(evalResultsTable).where(whereClause);
176
+ const ret = /* @__PURE__ */ new Set();
177
+ for (const r of rows) ret.add(`${r.testIdx}:${r.promptIdx}`);
178
+ return ret;
179
+ }
180
+ static async *findManyByEvalIdBatched(evalId, opts) {
181
+ const db = getDb();
182
+ const batchSize = opts?.batchSize || 100;
183
+ let offset = 0;
184
+ while (true) {
185
+ const results = await db.select().from(evalResultsTable).where(and(eq(evalResultsTable.evalId, evalId), gte(evalResultsTable.testIdx, offset), lt(evalResultsTable.testIdx, offset + batchSize))).all();
186
+ if (results.length === 0) break;
187
+ yield results.map((result) => new EvalResult({
188
+ ...result,
189
+ persisted: true
190
+ }));
191
+ offset += batchSize;
192
+ }
193
+ }
194
+ id;
195
+ evalId;
196
+ description;
197
+ promptIdx;
198
+ testIdx;
199
+ testCase;
200
+ prompt;
201
+ promptId;
202
+ error;
203
+ success;
204
+ score;
205
+ response;
206
+ gradingResult;
207
+ namedScores;
208
+ provider;
209
+ latencyMs;
210
+ cost;
211
+ metadata;
212
+ failureReason;
213
+ persisted;
214
+ pluginId;
215
+ constructor(opts) {
216
+ this.id = opts.id;
217
+ this.evalId = opts.evalId;
218
+ this.promptIdx = opts.promptIdx;
219
+ this.testIdx = opts.testIdx;
220
+ this.testCase = opts.testCase;
221
+ this.prompt = opts.prompt;
222
+ this.promptId = opts.promptId || hashPrompt(opts.prompt);
223
+ this.error = opts.error;
224
+ this.score = opts.score;
225
+ this.success = opts.success;
226
+ this.response = opts.response || void 0;
227
+ this.gradingResult = opts.gradingResult;
228
+ this.namedScores = opts.namedScores || {};
229
+ this.provider = opts.provider;
230
+ this.latencyMs = opts.latencyMs || 0;
231
+ this.cost = opts.cost || 0;
232
+ this.metadata = opts.metadata || {};
233
+ this.failureReason = isResultFailureReason(opts.failureReason) ? opts.failureReason : ResultFailureReason.NONE;
234
+ this.persisted = opts.persisted || false;
235
+ this.pluginId = opts.testCase.metadata?.pluginId;
236
+ }
237
+ async save() {
238
+ const db = getDb();
239
+ if (this.persisted) await db.update(evalResultsTable).set({
240
+ ...this,
241
+ updatedAt: getCurrentTimestamp()
242
+ }).where(eq(evalResultsTable.id, this.id));
243
+ else {
244
+ this.id = (await db.insert(evalResultsTable).values(this).returning())[0].id;
245
+ this.persisted = true;
246
+ }
247
+ }
248
+ toEvaluateResult() {
249
+ const shouldStripPromptText = getEnvBool("PROMPTFOO_STRIP_PROMPT_TEXT", false);
250
+ const shouldStripResponseOutput = getEnvBool("PROMPTFOO_STRIP_RESPONSE_OUTPUT", false);
251
+ const shouldStripTestVars = getEnvBool("PROMPTFOO_STRIP_TEST_VARS", false);
252
+ const shouldStripGradingResult = getEnvBool("PROMPTFOO_STRIP_GRADING_RESULT", false);
253
+ const shouldStripMetadata = getEnvBool("PROMPTFOO_STRIP_METADATA", false);
254
+ const response = shouldStripResponseOutput && this.response ? {
255
+ ...this.response,
256
+ output: "[output stripped]"
257
+ } : this.response;
258
+ const prompt = shouldStripPromptText ? {
259
+ ...this.prompt,
260
+ raw: "[prompt stripped]"
261
+ } : this.prompt;
262
+ const testCase = shouldStripTestVars ? {
263
+ ...this.testCase,
264
+ vars: void 0
265
+ } : this.testCase;
266
+ return {
267
+ cost: this.cost,
268
+ description: this.description || void 0,
269
+ error: this.error || void 0,
270
+ gradingResult: shouldStripGradingResult ? null : this.gradingResult,
271
+ id: this.id,
272
+ latencyMs: this.latencyMs,
273
+ namedScores: this.namedScores,
274
+ prompt,
275
+ promptId: this.promptId,
276
+ promptIdx: this.promptIdx,
277
+ provider: {
278
+ id: this.provider.id,
279
+ label: this.provider.label
280
+ },
281
+ response,
282
+ score: this.score,
283
+ success: this.success,
284
+ testCase,
285
+ testIdx: this.testIdx,
286
+ vars: shouldStripTestVars ? {} : this.testCase.vars || {},
287
+ metadata: shouldStripMetadata ? {} : this.metadata,
288
+ failureReason: this.failureReason
289
+ };
290
+ }
291
+ };
292
+ //#endregion
293
+ export { evalResult_exports as n, EvalResult as t };
294
+
295
+ //# sourceMappingURL=evalResult-C7JJAPBb.js.map
@@ -0,0 +1,2 @@
1
+ const require_evalResult = require("./evalResult-C3NJPQOo.cjs");
2
+ exports.default = require_evalResult.EvalResult;
@@ -0,0 +1,391 @@
1
+ const require_logger = require("./logger-DyfK9PBt.cjs");
2
+ const require_fetch = require("./fetch-BTxakTSg.cjs");
3
+ const require_accounts = require("./accounts-Bt1oJb1Z.cjs");
4
+ const require_blobs = require("./blobs-BgcNn97m.cjs");
5
+ //#region src/blobs/remoteUpload.ts
6
+ function buildRemoteUrl() {
7
+ const baseUrl = require_fetch.cloudConfig.getApiHost();
8
+ const apiKey = require_fetch.cloudConfig.getApiKey();
9
+ if (!baseUrl || !apiKey || !require_accounts.isLoggedIntoCloud()) return null;
10
+ try {
11
+ return new URL("/api/blobs", baseUrl).toString();
12
+ } catch (error) {
13
+ require_logger.logger.debug("[RemoteBlob] Invalid remote blob URL", {
14
+ error: error instanceof Error ? error.message : String(error),
15
+ baseUrl
16
+ });
17
+ return null;
18
+ }
19
+ }
20
+ function shouldAttemptRemoteBlobUpload() {
21
+ return buildRemoteUrl() !== null;
22
+ }
23
+ async function uploadBlobRemote(buffer, mimeType, context) {
24
+ const url = buildRemoteUrl();
25
+ const apiKey = require_fetch.cloudConfig.getApiKey();
26
+ if (!url || !apiKey) return null;
27
+ try {
28
+ const { fetchWithProxy } = await Promise.resolve().then(() => require("./fetch-BTxakTSg.cjs")).then((n) => n.fetch_exports);
29
+ const response = await fetchWithProxy(url, {
30
+ method: "POST",
31
+ headers: {
32
+ "Content-Type": "application/json",
33
+ Authorization: `Bearer ${apiKey}`
34
+ },
35
+ body: JSON.stringify({
36
+ data: buffer.toString("base64"),
37
+ mimeType,
38
+ context
39
+ })
40
+ });
41
+ if (response.status === 404 || response.status === 400) {
42
+ require_logger.logger.debug("[RemoteBlob] Remote blob upload unavailable", { status: response.status });
43
+ return null;
44
+ }
45
+ if (!response.ok) {
46
+ const text = await response.text();
47
+ require_logger.logger.debug("[RemoteBlob] Failed to upload blob", {
48
+ status: response.status,
49
+ statusText: response.statusText,
50
+ body: text
51
+ });
52
+ return null;
53
+ }
54
+ const data = await response.json();
55
+ if (!data?.ref?.hash) {
56
+ require_logger.logger.debug("[RemoteBlob] Remote upload returned malformed response");
57
+ return null;
58
+ }
59
+ return data;
60
+ } catch (error) {
61
+ require_logger.logger.debug("[RemoteBlob] Error uploading blob", { error: error instanceof Error ? error.message : String(error) });
62
+ return null;
63
+ }
64
+ }
65
+ //#endregion
66
+ //#region src/blobs/extractor.ts
67
+ const BLOB_URI_REGEX = /^promptfoo:\/\/blob\/([a-f0-9]{64})$/i;
68
+ const BLOB_HASH_REGEX = /^[a-f0-9]{64}$/i;
69
+ function isDataUrl(value) {
70
+ return /^data:(audio|image)\/[^;]+;base64,/.test(value);
71
+ }
72
+ function extractBase64(value) {
73
+ const match = value.match(/^data:([^;]+);base64,(.+)$/);
74
+ if (!match) return null;
75
+ const mimeType = match[1];
76
+ try {
77
+ return {
78
+ buffer: Buffer.from(match[2], "base64"),
79
+ mimeType
80
+ };
81
+ } catch (error) {
82
+ require_logger.logger.warn("[BlobExtractor] Failed to parse base64 data URL", { error });
83
+ return null;
84
+ }
85
+ }
86
+ function shouldExternalize(buffer) {
87
+ const size = buffer.length;
88
+ return size >= 1024 && size <= 52428800;
89
+ }
90
+ function getKindFromMimeType(mimeType) {
91
+ return mimeType.startsWith("audio/") ? "audio" : "image";
92
+ }
93
+ /**
94
+ * Normalize audio format to proper MIME type.
95
+ * Some providers return just 'wav' instead of 'audio/wav'.
96
+ * @internal Exported for testing
97
+ */
98
+ function normalizeAudioMimeType(format) {
99
+ if (!format) return "audio/wav";
100
+ const trimmedFormat = format.trim();
101
+ if (/^audio\/[a-z0-9_+-]+$/i.test(trimmedFormat)) return trimmedFormat;
102
+ const formatLower = trimmedFormat.toLowerCase();
103
+ const mimeMap = {
104
+ wav: "audio/wav",
105
+ mp3: "audio/mpeg",
106
+ ogg: "audio/ogg",
107
+ flac: "audio/flac",
108
+ aac: "audio/aac",
109
+ m4a: "audio/mp4",
110
+ webm: "audio/webm"
111
+ };
112
+ if (mimeMap[formatLower]) return mimeMap[formatLower];
113
+ if (!/^[a-z0-9_-]+$/i.test(formatLower)) {
114
+ require_logger.logger.warn("[BlobExtractor] Invalid audio format, using default", { format });
115
+ return "audio/wav";
116
+ }
117
+ return `audio/${formatLower}`;
118
+ }
119
+ function parseBinary(base64OrDataUrl, defaultMimeType) {
120
+ if (isDataUrl(base64OrDataUrl)) {
121
+ const parsed = extractBase64(base64OrDataUrl);
122
+ if (!parsed) return null;
123
+ return parsed;
124
+ }
125
+ try {
126
+ return {
127
+ buffer: Buffer.from(base64OrDataUrl, "base64"),
128
+ mimeType: defaultMimeType
129
+ };
130
+ } catch (error) {
131
+ require_logger.logger.warn("[BlobExtractor] Failed to parse base64 data", { error });
132
+ return null;
133
+ }
134
+ }
135
+ async function maybeStore(base64OrDataUrl, defaultMimeType, context, location, kind) {
136
+ const parsed = parseBinary(base64OrDataUrl, defaultMimeType);
137
+ if (!parsed || !shouldExternalize(parsed.buffer)) return null;
138
+ if (!isBlobStorageEnabled()) return null;
139
+ const mimeType = parsed.mimeType || "application/octet-stream";
140
+ const { ref } = await require_blobs.storeBlob(parsed.buffer, mimeType, {
141
+ ...context,
142
+ location,
143
+ kind
144
+ });
145
+ if (shouldAttemptRemoteBlobUpload()) uploadBlobRemote(parsed.buffer, mimeType, {
146
+ evalId: context.evalId,
147
+ testIdx: context.testIdx,
148
+ promptIdx: context.promptIdx,
149
+ location,
150
+ kind
151
+ }).catch((error) => {
152
+ require_logger.logger.debug("[BlobExtractor] Cloud upload failed (non-fatal)", {
153
+ error: error instanceof Error ? error.message : String(error),
154
+ hash: ref.hash
155
+ });
156
+ });
157
+ return ref;
158
+ }
159
+ async function externalizeDataUrls(value, context, location) {
160
+ if (typeof value === "string") {
161
+ if (!isDataUrl(value)) return {
162
+ value,
163
+ mutated: false
164
+ };
165
+ const parsed = extractBase64(value);
166
+ if (!parsed || !shouldExternalize(parsed.buffer)) return {
167
+ value,
168
+ mutated: false
169
+ };
170
+ const storedRef = await maybeStore(parsed.buffer.toString("base64"), parsed.mimeType, context, location, getKindFromMimeType(parsed.mimeType)) || null;
171
+ if (!storedRef) return {
172
+ value,
173
+ mutated: false
174
+ };
175
+ return {
176
+ value: storedRef.uri,
177
+ mutated: true
178
+ };
179
+ }
180
+ if (Array.isArray(value)) {
181
+ let mutated = false;
182
+ const nextValues = await Promise.all(value.map(async (item, idx) => {
183
+ const { value: nextValue, mutated: childMutated } = await externalizeDataUrls(item, context, `${location}[${idx}]`);
184
+ mutated ||= childMutated;
185
+ return nextValue;
186
+ }));
187
+ return mutated ? {
188
+ value: nextValues,
189
+ mutated
190
+ } : {
191
+ value,
192
+ mutated: false
193
+ };
194
+ }
195
+ if (value && typeof value === "object") {
196
+ let mutated = false;
197
+ const nextObject = { ...value };
198
+ for (const [key, child] of Object.entries(value)) {
199
+ const { value: nextValue, mutated: childMutated } = await externalizeDataUrls(child, context, location ? `${location}.${key}` : key);
200
+ if (childMutated) {
201
+ nextObject[key] = nextValue;
202
+ mutated = true;
203
+ }
204
+ }
205
+ return mutated ? {
206
+ value: nextObject,
207
+ mutated: true
208
+ } : {
209
+ value,
210
+ mutated: false
211
+ };
212
+ }
213
+ return {
214
+ value,
215
+ mutated: false
216
+ };
217
+ }
218
+ /**
219
+ * Best-effort extraction of binary data from provider responses.
220
+ * Currently focuses on audio.data fields and data URL outputs.
221
+ */
222
+ async function extractAndStoreBinaryData(response, context) {
223
+ if (!response) return response;
224
+ let mutated = false;
225
+ const next = { ...response };
226
+ const blobContext = context || {};
227
+ if (response.audio?.data && typeof response.audio.data === "string") {
228
+ const stored = await maybeStore(response.audio.data, normalizeAudioMimeType(response.audio.format), blobContext, "response.audio.data", "audio");
229
+ if (stored) {
230
+ next.audio = {
231
+ ...response.audio,
232
+ data: void 0,
233
+ blobRef: stored
234
+ };
235
+ mutated = true;
236
+ require_logger.logger.debug("[BlobExtractor] Stored audio blob", {
237
+ ...context,
238
+ hash: stored.hash
239
+ });
240
+ }
241
+ }
242
+ if (response.images?.length) next.images = await Promise.all(response.images.map(async (img, idx) => {
243
+ if (!img.data || typeof img.data !== "string" || !isDataUrl(img.data)) return img;
244
+ const stored = await maybeStore(img.data, img.mimeType || "image/png", blobContext, `response.images[${idx}].data`, "image");
245
+ if (stored) {
246
+ mutated = true;
247
+ require_logger.logger.debug("[BlobExtractor] Stored image blob", {
248
+ ...context,
249
+ hash: stored.hash
250
+ });
251
+ return {
252
+ ...img,
253
+ data: void 0,
254
+ blobRef: stored
255
+ };
256
+ }
257
+ return img;
258
+ }));
259
+ const turns = response.turns;
260
+ if (Array.isArray(turns)) next.turns = await Promise.all(turns.map(async (turn, idx) => {
261
+ if (turn?.audio?.data && typeof turn.audio.data === "string") {
262
+ const stored = await maybeStore(turn.audio.data, normalizeAudioMimeType(turn.audio.format), blobContext, `response.turns[${idx}].audio.data`, "audio");
263
+ if (stored) {
264
+ mutated = true;
265
+ return {
266
+ ...turn,
267
+ audio: {
268
+ ...turn.audio,
269
+ data: void 0,
270
+ blobRef: stored
271
+ }
272
+ };
273
+ }
274
+ }
275
+ return turn;
276
+ }));
277
+ if (typeof response.output === "string" && isDataUrl(response.output)) {
278
+ const parsed = extractBase64(response.output);
279
+ if (parsed && shouldExternalize(parsed.buffer)) {
280
+ const stored = await maybeStore(parsed.buffer.toString("base64"), parsed.mimeType, blobContext, "response.output", getKindFromMimeType(parsed.mimeType));
281
+ if (stored) {
282
+ next.output = stored.uri;
283
+ mutated = true;
284
+ require_logger.logger.debug("[BlobExtractor] Stored output blob", {
285
+ ...context,
286
+ hash: stored.hash
287
+ });
288
+ }
289
+ }
290
+ }
291
+ if (typeof response.output === "string" && response.output.trim().startsWith("{") && (response.isBase64 && response.format === "json" || response.output.includes("\"b64_json\"") || response.output.includes("b64_json"))) try {
292
+ const parsed = JSON.parse(response.output);
293
+ if (Array.isArray(parsed.data)) {
294
+ let jsonMutated = false;
295
+ const storedUris = [];
296
+ for (const item of parsed.data) if (item?.b64_json && typeof item.b64_json === "string") {
297
+ const stored = await maybeStore(item.b64_json, "image/png", blobContext, "response.output.data[].b64_json", "image");
298
+ if (stored) {
299
+ item.b64_json = stored.uri;
300
+ storedUris.push(stored.uri);
301
+ jsonMutated = true;
302
+ mutated = true;
303
+ require_logger.logger.debug("[BlobExtractor] Stored image blob from b64_json", {
304
+ ...context,
305
+ hash: stored.hash
306
+ });
307
+ }
308
+ }
309
+ if (jsonMutated) {
310
+ if (storedUris.length === 1) next.output = storedUris[0];
311
+ else if (storedUris.length > 1) next.output = JSON.stringify(storedUris);
312
+ else next.output = JSON.stringify(parsed);
313
+ next.metadata = {
314
+ ...response.metadata || {},
315
+ blobUris: storedUris,
316
+ originalFormat: response.format
317
+ };
318
+ }
319
+ }
320
+ } catch (err) {
321
+ require_logger.logger.debug("[BlobExtractor] Failed to parse base64 JSON output", {
322
+ error: err instanceof Error ? err.message : String(err),
323
+ location: "response.output"
324
+ });
325
+ }
326
+ if (response.metadata) {
327
+ const { value, mutated: metadataMutated } = await externalizeDataUrls(response.metadata, blobContext, "response.metadata");
328
+ if (metadataMutated) {
329
+ next.metadata = value;
330
+ mutated = true;
331
+ }
332
+ }
333
+ const finalResponse = mutated ? next : response;
334
+ if (blobContext.evalId) await recordExistingBlobReferences(finalResponse, blobContext, "response");
335
+ return finalResponse;
336
+ }
337
+ function isBlobStorageEnabled() {
338
+ return !require_logger.getEnvBool("PROMPTFOO_INLINE_MEDIA", false);
339
+ }
340
+ function parseBlobHashFromValue(value) {
341
+ if (!value) return null;
342
+ if (typeof value === "string") {
343
+ const match = value.match(BLOB_URI_REGEX);
344
+ return match ? match[1] : null;
345
+ }
346
+ if (typeof value === "object") {
347
+ const candidate = value;
348
+ if (candidate.hash && BLOB_HASH_REGEX.test(candidate.hash)) return candidate.hash;
349
+ if (candidate.uri && typeof candidate.uri === "string") {
350
+ const match = candidate.uri.match(BLOB_URI_REGEX);
351
+ if (match) return match[1];
352
+ }
353
+ }
354
+ return null;
355
+ }
356
+ async function recordExistingBlobReferences(value, context, location) {
357
+ const hash = parseBlobHashFromValue(value);
358
+ if (hash) {
359
+ await require_blobs.recordBlobReference(hash, {
360
+ ...context,
361
+ location
362
+ });
363
+ return;
364
+ }
365
+ if (Array.isArray(value)) {
366
+ await Promise.all(value.map((child, idx) => recordExistingBlobReferences(child, context, `${location}[${idx}]`)));
367
+ return;
368
+ }
369
+ if (value && typeof value === "object") for (const [key, child] of Object.entries(value)) await recordExistingBlobReferences(child, context, location ? `${location}.${key}` : key);
370
+ }
371
+ //#endregion
372
+ Object.defineProperty(exports, "extractAndStoreBinaryData", {
373
+ enumerable: true,
374
+ get: function() {
375
+ return extractAndStoreBinaryData;
376
+ }
377
+ });
378
+ Object.defineProperty(exports, "isBlobStorageEnabled", {
379
+ enumerable: true,
380
+ get: function() {
381
+ return isBlobStorageEnabled;
382
+ }
383
+ });
384
+ Object.defineProperty(exports, "shouldAttemptRemoteBlobUpload", {
385
+ enumerable: true,
386
+ get: function() {
387
+ return shouldAttemptRemoteBlobUpload;
388
+ }
389
+ });
390
+
391
+ //# sourceMappingURL=extractor-DnMD3fwt.cjs.map