@use-lattice/litmus 0.121.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +19 -0
- package/dist/src/accounts-Bt1oJb1Z.cjs +219 -0
- package/dist/src/accounts-DjOU8Rm3.js +178 -0
- package/dist/src/agentic-utils-D03IiXQc.js +153 -0
- package/dist/src/agentic-utils-Dh7xaMQM.cjs +180 -0
- package/dist/src/agents-C6BIMlZa.js +231 -0
- package/dist/src/agents-DvIpNX1L.cjs +666 -0
- package/dist/src/agents-ZP0RP9vV.cjs +231 -0
- package/dist/src/agents-maJXdjbR.js +665 -0
- package/dist/src/aimlapi-BTbQjG2E.cjs +30 -0
- package/dist/src/aimlapi-CwMxqfXP.js +30 -0
- package/dist/src/audio-BBUdvsde.cjs +97 -0
- package/dist/src/audio-D5DPZ7I-.js +97 -0
- package/dist/src/base-BEysXrkq.cjs +222 -0
- package/dist/src/base-C451JQfq.js +193 -0
- package/dist/src/blobs-BY8MDmpo.js +230 -0
- package/dist/src/blobs-BgcNn97m.cjs +256 -0
- package/dist/src/cache-BBE_lsTA.cjs +4 -0
- package/dist/src/cache-BkrqU5Ba.js +237 -0
- package/dist/src/cache-DsCxFlsZ.cjs +297 -0
- package/dist/src/chat-CPJWDP6a.cjs +289 -0
- package/dist/src/chat-CXX3xzkk.cjs +811 -0
- package/dist/src/chat-CcDgZFJ4.js +787 -0
- package/dist/src/chat-Dz5ZeGO2.js +289 -0
- package/dist/src/chatkit-Dw0mKkML.cjs +1158 -0
- package/dist/src/chatkit-swAIVuea.js +1157 -0
- package/dist/src/chunk-DEq-mXcV.js +15 -0
- package/dist/src/claude-agent-sdk-BXZJtOg6.js +379 -0
- package/dist/src/claude-agent-sdk-CkfyjDoG.cjs +383 -0
- package/dist/src/cloudflare-ai-BzpJcqUH.js +161 -0
- package/dist/src/cloudflare-ai-Cmy_R1y2.cjs +161 -0
- package/dist/src/cloudflare-gateway-B9tVQKok.cjs +272 -0
- package/dist/src/cloudflare-gateway-DrD3ew3H.js +272 -0
- package/dist/src/codex-sdk-Dezj9Nwm.js +1056 -0
- package/dist/src/codex-sdk-Dl9D4k5B.cjs +1060 -0
- package/dist/src/cometapi-C-9YvCHC.js +54 -0
- package/dist/src/cometapi-DHgDKoO2.cjs +54 -0
- package/dist/src/completion-B8Ctyxpr.js +120 -0
- package/dist/src/completion-Cxrt08sj.cjs +131 -0
- package/dist/src/createHash-BwgE13yv.cjs +27 -0
- package/dist/src/createHash-DmPQkvBh.js +15 -0
- package/dist/src/docker-BiqcTwLv.js +80 -0
- package/dist/src/docker-C7tEJnP-.cjs +80 -0
- package/dist/src/esm-C62Zofr1.cjs +409 -0
- package/dist/src/esm-DMVc93eh.js +379 -0
- package/dist/src/evalResult-C3NJPQOo.cjs +301 -0
- package/dist/src/evalResult-C7JJAPBb.js +295 -0
- package/dist/src/evalResult-DoVTZZWI.cjs +2 -0
- package/dist/src/extractor-DnMD3fwt.cjs +391 -0
- package/dist/src/extractor-DtlL28vL.js +374 -0
- package/dist/src/fetch-BTxakTSg.cjs +1133 -0
- package/dist/src/fetch-DQckpUFz.js +928 -0
- package/dist/src/fileExtensions-DnqA1y9x.js +85 -0
- package/dist/src/fileExtensions-bYh77CN8.cjs +114 -0
- package/dist/src/genaiTracer-CyZrmaK0.cjs +268 -0
- package/dist/src/genaiTracer-D3fD9dNV.js +256 -0
- package/dist/src/graders-BNscxFrU.js +13644 -0
- package/dist/src/graders-D2oE9Msq.js +2 -0
- package/dist/src/graders-c0Ez_w-9.cjs +2 -0
- package/dist/src/graders-d0F2M3e9.cjs +14056 -0
- package/dist/src/image-0ZhE0VlR.cjs +280 -0
- package/dist/src/image-CWE1pdNv.js +257 -0
- package/dist/src/image-D9ZK6hwL.js +163 -0
- package/dist/src/image-DKZgZITg.cjs +163 -0
- package/dist/src/index.cjs +11366 -0
- package/dist/src/index.d.cts +19640 -0
- package/dist/src/index.d.ts +19641 -0
- package/dist/src/index.js +11306 -0
- package/dist/src/invariant-Ddh24eXh.js +25 -0
- package/dist/src/invariant-kfQ8Bu82.cjs +30 -0
- package/dist/src/knowledgeBase-BgPyGFUd.cjs +122 -0
- package/dist/src/knowledgeBase-DyHilYaP.js +122 -0
- package/dist/src/litellm-CyMeneHS.js +135 -0
- package/dist/src/litellm-DWDF73yF.cjs +135 -0
- package/dist/src/logger-C40ZGil9.js +717 -0
- package/dist/src/logger-DyfK9PBt.cjs +917 -0
- package/dist/src/luma-ray-BAU9X_ep.cjs +315 -0
- package/dist/src/luma-ray-nwVseBbv.js +313 -0
- package/dist/src/messages-B5ADWTTv.js +245 -0
- package/dist/src/messages-BCnZfqrS.cjs +257 -0
- package/dist/src/meteor-DLZZ3osF.cjs +134 -0
- package/dist/src/meteor-DUiCJRC-.js +134 -0
- package/dist/src/modelslab-00cveB8L.cjs +163 -0
- package/dist/src/modelslab-D9sCU_L7.js +163 -0
- package/dist/src/nova-reel-CTapvqYH.js +276 -0
- package/dist/src/nova-reel-DlWuuroF.cjs +278 -0
- package/dist/src/nova-sonic-5UPWfeMv.cjs +363 -0
- package/dist/src/nova-sonic-BhSwQNym.js +363 -0
- package/dist/src/openai-BWrJK9d8.cjs +52 -0
- package/dist/src/openai-DumO8WQn.js +47 -0
- package/dist/src/openclaw-B8brrjC_.cjs +577 -0
- package/dist/src/openclaw-Bkayww9q.js +571 -0
- package/dist/src/opencode-sdk-7xjoDNiM.cjs +562 -0
- package/dist/src/opencode-sdk-SGwAPxht.js +558 -0
- package/dist/src/otlpReceiver-CoAHfAN9.cjs +15 -0
- package/dist/src/otlpReceiver-oO3EQwI9.js +14 -0
- package/dist/src/providerRegistry-4yjhaEM8.js +45 -0
- package/dist/src/providerRegistry-DhV4rJIc.cjs +50 -0
- package/dist/src/providers-B5RJVG-7.cjs +33609 -0
- package/dist/src/providers-BdmZCLzV.js +33262 -0
- package/dist/src/providers-CxtRxn8e.js +2 -0
- package/dist/src/providers-DnQLNbx1.cjs +3 -0
- package/dist/src/pythonUtils-BD0druiM.cjs +275 -0
- package/dist/src/pythonUtils-IBhn5YGR.js +249 -0
- package/dist/src/quiverai-BDOwZBsM.cjs +213 -0
- package/dist/src/quiverai-D3JTF5lD.js +213 -0
- package/dist/src/responses-B2LCDCXZ.js +667 -0
- package/dist/src/responses-BvNm4Xv9.cjs +685 -0
- package/dist/src/rubyUtils-B0NwnfpY.cjs +245 -0
- package/dist/src/rubyUtils-BroxzZ7c.cjs +2 -0
- package/dist/src/rubyUtils-hqVw5UvJ.js +222 -0
- package/dist/src/sagemaker-Cno2V-Sx.js +689 -0
- package/dist/src/sagemaker-fV_KUgs5.cjs +691 -0
- package/dist/src/server-BOuAXb06.cjs +238 -0
- package/dist/src/server-CtI-EWzm.cjs +2 -0
- package/dist/src/server-Cy3DZymt.js +189 -0
- package/dist/src/slack-CP8xBePa.js +135 -0
- package/dist/src/slack-DSQ1yXVb.cjs +135 -0
- package/dist/src/store-BwDDaBjb.cjs +246 -0
- package/dist/src/store-DcbLC593.cjs +2 -0
- package/dist/src/store-IGpqMIkv.js +240 -0
- package/dist/src/tables-3Q2cL7So.cjs +373 -0
- package/dist/src/tables-Bi2fjr4W.js +288 -0
- package/dist/src/telemetry-Bg2WqF79.js +161 -0
- package/dist/src/telemetry-D0x6u5kX.cjs +166 -0
- package/dist/src/telemetry-DXNimrI0.cjs +2 -0
- package/dist/src/text-B_UCRPp2.js +22 -0
- package/dist/src/text-CW1cyrwj.cjs +33 -0
- package/dist/src/tokenUsageUtils-NYT-WKS6.js +138 -0
- package/dist/src/tokenUsageUtils-bVa1ga6f.cjs +173 -0
- package/dist/src/transcription-Cl_W16Pr.js +122 -0
- package/dist/src/transcription-yt1EecY8.cjs +124 -0
- package/dist/src/transform-BCtGrl_W.cjs +228 -0
- package/dist/src/transform-Bv6gG2MJ.cjs +1688 -0
- package/dist/src/transform-CY1wbpRy.js +1507 -0
- package/dist/src/transform-DU8rUL9P.cjs +2 -0
- package/dist/src/transform-yWaShiKr.js +216 -0
- package/dist/src/transformersAvailability-BGkzavwb.js +35 -0
- package/dist/src/transformersAvailability-DKoRtQLy.cjs +35 -0
- package/dist/src/types-5aqHpBwE.cjs +3769 -0
- package/dist/src/types-Bn6D9c4U.js +3300 -0
- package/dist/src/util-BkKlTkI2.js +293 -0
- package/dist/src/util-CTh0bfOm.cjs +1119 -0
- package/dist/src/util-D17oBwo7.cjs +328 -0
- package/dist/src/util-DsS_-v4p.js +613 -0
- package/dist/src/util-DuntT1Ga.js +951 -0
- package/dist/src/util-aWjdCYMI.cjs +667 -0
- package/dist/src/utils-CisQwpjA.js +94 -0
- package/dist/src/utils-yWamDvmz.cjs +123 -0
- package/dist/tsconfig.tsbuildinfo +1 -0
- package/drizzle/0000_lush_hellion.sql +36 -0
- package/drizzle/0001_wide_calypso.sql +3 -0
- package/drizzle/0002_tidy_juggernaut.sql +1 -0
- package/drizzle/0003_lively_naoko.sql +8 -0
- package/drizzle/0004_minor_peter_quill.sql +19 -0
- package/drizzle/0005_silky_millenium_guard.sql +2 -0
- package/drizzle/0006_harsh_caretaker.sql +42 -0
- package/drizzle/0007_cloudy_wong.sql +1 -0
- package/drizzle/0008_broad_boomer.sql +2 -0
- package/drizzle/0009_strong_marten_broadcloak.sql +19 -0
- package/drizzle/0010_needy_bishop.sql +11 -0
- package/drizzle/0011_moaning_millenium_guard.sql +1 -0
- package/drizzle/0012_late_marten_broadcloak.sql +2 -0
- package/drizzle/0013_previous_dormammu.sql +9 -0
- package/drizzle/0014_lazy_captain_universe.sql +2 -0
- package/drizzle/0015_zippy_wallop.sql +29 -0
- package/drizzle/0016_jazzy_zemo.sql +2 -0
- package/drizzle/0017_reflective_praxagora.sql +4 -0
- package/drizzle/0018_fat_vanisher.sql +22 -0
- package/drizzle/0019_new_clint_barton.sql +8 -0
- package/drizzle/0020_skinny_maverick.sql +1 -0
- package/drizzle/0021_mysterious_madelyne_pryor.sql +13 -0
- package/drizzle/0022_sleepy_ultimo.sql +25 -0
- package/drizzle/0023_wooden_mandrill.sql +2 -0
- package/drizzle/AGENTS.md +68 -0
- package/drizzle/CLAUDE.md +1 -0
- package/drizzle/meta/0000_snapshot.json +221 -0
- package/drizzle/meta/0001_snapshot.json +214 -0
- package/drizzle/meta/0002_snapshot.json +221 -0
- package/drizzle/meta/0005_snapshot.json +369 -0
- package/drizzle/meta/0006_snapshot.json +638 -0
- package/drizzle/meta/0007_snapshot.json +640 -0
- package/drizzle/meta/0008_snapshot.json +649 -0
- package/drizzle/meta/0009_snapshot.json +554 -0
- package/drizzle/meta/0010_snapshot.json +619 -0
- package/drizzle/meta/0011_snapshot.json +627 -0
- package/drizzle/meta/0012_snapshot.json +639 -0
- package/drizzle/meta/0013_snapshot.json +717 -0
- package/drizzle/meta/0014_snapshot.json +717 -0
- package/drizzle/meta/0015_snapshot.json +897 -0
- package/drizzle/meta/0016_snapshot.json +1031 -0
- package/drizzle/meta/0018_snapshot.json +1210 -0
- package/drizzle/meta/0019_snapshot.json +1165 -0
- package/drizzle/meta/0020_snapshot.json +1232 -0
- package/drizzle/meta/0021_snapshot.json +1311 -0
- package/drizzle/meta/0022_snapshot.json +1481 -0
- package/drizzle/meta/0023_snapshot.json +1496 -0
- package/drizzle/meta/_journal.json +174 -0
- package/package.json +240 -0
|
@@ -0,0 +1,295 @@
|
|
|
1
|
+
import { t as __exportAll } from "./chunk-DEq-mXcV.js";
|
|
2
|
+
import { h as safeJsonStringify, r as logger, v as getEnvBool } from "./logger-C40ZGil9.js";
|
|
3
|
+
import { A as isApiProvider, g as ResultFailureReason, j as isProviderOptions, k as isResultFailureReason } from "./types-Bn6D9c4U.js";
|
|
4
|
+
import { u as getCurrentTimestamp } from "./fetch-DQckpUFz.js";
|
|
5
|
+
import { t as hashPrompt } from "./utils-CisQwpjA.js";
|
|
6
|
+
import { i as evalResultsTable, p as getDb } from "./tables-Bi2fjr4W.js";
|
|
7
|
+
import { n as isBlobStorageEnabled, t as extractAndStoreBinaryData } from "./extractor-DtlL28vL.js";
|
|
8
|
+
import { and, eq, gte, inArray, lt, ne } from "drizzle-orm";
|
|
9
|
+
//#region src/models/evalResult.ts
|
|
10
|
+
var evalResult_exports = /* @__PURE__ */ __exportAll({
|
|
11
|
+
default: () => EvalResult,
|
|
12
|
+
sanitizeProvider: () => sanitizeProvider
|
|
13
|
+
});
|
|
14
|
+
function sanitizeProvider(provider) {
|
|
15
|
+
try {
|
|
16
|
+
if (isApiProvider(provider)) return {
|
|
17
|
+
id: provider.id(),
|
|
18
|
+
label: provider.label,
|
|
19
|
+
...provider.config && { config: JSON.parse(safeJsonStringify(provider.config)) }
|
|
20
|
+
};
|
|
21
|
+
if (isProviderOptions(provider)) return {
|
|
22
|
+
id: provider.id,
|
|
23
|
+
label: provider.label,
|
|
24
|
+
...provider.config && { config: JSON.parse(safeJsonStringify(provider.config)) }
|
|
25
|
+
};
|
|
26
|
+
if (typeof provider === "object" && provider) {
|
|
27
|
+
const providerObj = provider;
|
|
28
|
+
return {
|
|
29
|
+
id: typeof providerObj.id === "function" ? providerObj.id() : providerObj.id,
|
|
30
|
+
label: providerObj.label,
|
|
31
|
+
...providerObj.config && { config: JSON.parse(safeJsonStringify(providerObj.config)) }
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
} catch {}
|
|
35
|
+
return JSON.parse(safeJsonStringify(provider));
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Sanitize an object for database storage by removing circular references
|
|
39
|
+
* and non-serializable values (functions, Timeout objects, etc.).
|
|
40
|
+
* Uses safeJsonStringify which handles circular references gracefully.
|
|
41
|
+
*
|
|
42
|
+
* This prevents "Converting circular structure to JSON" errors that can occur
|
|
43
|
+
* when Node.js Timeout objects or other non-serializable data leaks into results.
|
|
44
|
+
* See: https://github.com/promptfoo/promptfoo/issues/7266
|
|
45
|
+
*/
|
|
46
|
+
function sanitizeForDb(obj) {
|
|
47
|
+
if (obj === null || obj === void 0) return obj;
|
|
48
|
+
try {
|
|
49
|
+
const serialized = safeJsonStringify(obj);
|
|
50
|
+
if (serialized === void 0) {
|
|
51
|
+
logger.debug("sanitizeForDb: Failed to serialize object, using fallback", {
|
|
52
|
+
valueType: typeof obj,
|
|
53
|
+
isArray: Array.isArray(obj)
|
|
54
|
+
});
|
|
55
|
+
return Array.isArray(obj) ? [] : null;
|
|
56
|
+
}
|
|
57
|
+
return JSON.parse(serialized);
|
|
58
|
+
} catch (error) {
|
|
59
|
+
logger.debug("sanitizeForDb: Parse error, using fallback", { error });
|
|
60
|
+
return Array.isArray(obj) ? [] : null;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
var EvalResult = class EvalResult {
|
|
64
|
+
static async createFromEvaluateResult(evalId, result, opts) {
|
|
65
|
+
const persist = opts?.persist == null ? true : opts.persist;
|
|
66
|
+
const { prompt, error, score, latencyMs, success, provider, gradingResult, namedScores, cost, metadata, failureReason, testCase } = result;
|
|
67
|
+
const preSanitizeTestCase = {
|
|
68
|
+
...testCase,
|
|
69
|
+
...testCase.provider && { provider: sanitizeProvider(testCase.provider) }
|
|
70
|
+
};
|
|
71
|
+
const processedResponse = await extractAndStoreBinaryData(result.response, {
|
|
72
|
+
evalId,
|
|
73
|
+
testIdx: result.testIdx,
|
|
74
|
+
promptIdx: result.promptIdx
|
|
75
|
+
});
|
|
76
|
+
const args = {
|
|
77
|
+
id: crypto.randomUUID(),
|
|
78
|
+
evalId,
|
|
79
|
+
testCase: sanitizeForDb(preSanitizeTestCase),
|
|
80
|
+
promptIdx: result.promptIdx,
|
|
81
|
+
testIdx: result.testIdx,
|
|
82
|
+
prompt: sanitizeForDb(prompt),
|
|
83
|
+
promptId: hashPrompt(prompt),
|
|
84
|
+
error: error?.toString(),
|
|
85
|
+
success,
|
|
86
|
+
score: score == null ? 0 : score,
|
|
87
|
+
response: sanitizeForDb(processedResponse || null),
|
|
88
|
+
gradingResult: sanitizeForDb(gradingResult || null),
|
|
89
|
+
namedScores: sanitizeForDb(namedScores),
|
|
90
|
+
provider: sanitizeProvider(provider),
|
|
91
|
+
latencyMs,
|
|
92
|
+
cost,
|
|
93
|
+
metadata: sanitizeForDb(metadata),
|
|
94
|
+
failureReason
|
|
95
|
+
};
|
|
96
|
+
if (persist) return new EvalResult({
|
|
97
|
+
...(await getDb().insert(evalResultsTable).values(args).returning())[0],
|
|
98
|
+
persisted: true
|
|
99
|
+
});
|
|
100
|
+
return new EvalResult(args);
|
|
101
|
+
}
|
|
102
|
+
static async createManyFromEvaluateResult(results, evalId) {
|
|
103
|
+
const db = getDb();
|
|
104
|
+
const returnResults = [];
|
|
105
|
+
const processedResults = [];
|
|
106
|
+
for (const result of results) {
|
|
107
|
+
const processedResponse = isBlobStorageEnabled() ? await extractAndStoreBinaryData(result.response, {
|
|
108
|
+
evalId,
|
|
109
|
+
testIdx: result.testIdx,
|
|
110
|
+
promptIdx: result.promptIdx
|
|
111
|
+
}) : result.response;
|
|
112
|
+
processedResults.push({
|
|
113
|
+
...result,
|
|
114
|
+
response: processedResponse ?? void 0
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
db.transaction(() => {
|
|
118
|
+
for (const result of processedResults) {
|
|
119
|
+
const sanitizedResult = {
|
|
120
|
+
...result,
|
|
121
|
+
testCase: sanitizeForDb(result.testCase),
|
|
122
|
+
prompt: sanitizeForDb(result.prompt),
|
|
123
|
+
response: sanitizeForDb(result.response),
|
|
124
|
+
gradingResult: sanitizeForDb(result.gradingResult),
|
|
125
|
+
namedScores: sanitizeForDb(result.namedScores),
|
|
126
|
+
metadata: sanitizeForDb(result.metadata),
|
|
127
|
+
provider: result.provider ? sanitizeProvider(result.provider) : result.provider
|
|
128
|
+
};
|
|
129
|
+
const dbResult = db.insert(evalResultsTable).values({
|
|
130
|
+
...sanitizedResult,
|
|
131
|
+
evalId,
|
|
132
|
+
id: crypto.randomUUID()
|
|
133
|
+
}).returning().get();
|
|
134
|
+
returnResults.push(new EvalResult({
|
|
135
|
+
...dbResult,
|
|
136
|
+
persisted: true
|
|
137
|
+
}));
|
|
138
|
+
}
|
|
139
|
+
});
|
|
140
|
+
return returnResults;
|
|
141
|
+
}
|
|
142
|
+
static async findById(id) {
|
|
143
|
+
const result = await getDb().select().from(evalResultsTable).where(eq(evalResultsTable.id, id));
|
|
144
|
+
return result.length > 0 ? new EvalResult({
|
|
145
|
+
...result[0],
|
|
146
|
+
persisted: true
|
|
147
|
+
}) : null;
|
|
148
|
+
}
|
|
149
|
+
static async findManyByEvalId(evalId, opts) {
|
|
150
|
+
return (await getDb().select().from(evalResultsTable).where(and(eq(evalResultsTable.evalId, evalId), opts?.testIdx == null ? void 0 : eq(evalResultsTable.testIdx, opts.testIdx)))).map((result) => new EvalResult({
|
|
151
|
+
...result,
|
|
152
|
+
persisted: true
|
|
153
|
+
}));
|
|
154
|
+
}
|
|
155
|
+
static async findManyByEvalIdAndTestIndices(evalId, testIndices) {
|
|
156
|
+
if (!testIndices.length) return [];
|
|
157
|
+
return (await getDb().select().from(evalResultsTable).where(and(eq(evalResultsTable.evalId, evalId), testIndices.length === 1 ? eq(evalResultsTable.testIdx, testIndices[0]) : inArray(evalResultsTable.testIdx, testIndices)))).map((result) => new EvalResult({
|
|
158
|
+
...result,
|
|
159
|
+
persisted: true
|
|
160
|
+
}));
|
|
161
|
+
}
|
|
162
|
+
/**
|
|
163
|
+
* Returns a set of completed (testIdx,promptIdx) pairs for a given eval.
|
|
164
|
+
* Key format: `${testIdx}:${promptIdx}`
|
|
165
|
+
*
|
|
166
|
+
* @param evalId - The evaluation ID to query
|
|
167
|
+
* @param opts.excludeErrors - If true, excludes results with ERROR failureReason (used in retry mode)
|
|
168
|
+
*/
|
|
169
|
+
static async getCompletedIndexPairs(evalId, opts) {
|
|
170
|
+
const db = getDb();
|
|
171
|
+
const whereClause = opts?.excludeErrors ? and(eq(evalResultsTable.evalId, evalId), ne(evalResultsTable.failureReason, ResultFailureReason.ERROR)) : eq(evalResultsTable.evalId, evalId);
|
|
172
|
+
const rows = await db.select({
|
|
173
|
+
testIdx: evalResultsTable.testIdx,
|
|
174
|
+
promptIdx: evalResultsTable.promptIdx
|
|
175
|
+
}).from(evalResultsTable).where(whereClause);
|
|
176
|
+
const ret = /* @__PURE__ */ new Set();
|
|
177
|
+
for (const r of rows) ret.add(`${r.testIdx}:${r.promptIdx}`);
|
|
178
|
+
return ret;
|
|
179
|
+
}
|
|
180
|
+
static async *findManyByEvalIdBatched(evalId, opts) {
|
|
181
|
+
const db = getDb();
|
|
182
|
+
const batchSize = opts?.batchSize || 100;
|
|
183
|
+
let offset = 0;
|
|
184
|
+
while (true) {
|
|
185
|
+
const results = await db.select().from(evalResultsTable).where(and(eq(evalResultsTable.evalId, evalId), gte(evalResultsTable.testIdx, offset), lt(evalResultsTable.testIdx, offset + batchSize))).all();
|
|
186
|
+
if (results.length === 0) break;
|
|
187
|
+
yield results.map((result) => new EvalResult({
|
|
188
|
+
...result,
|
|
189
|
+
persisted: true
|
|
190
|
+
}));
|
|
191
|
+
offset += batchSize;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
id;
|
|
195
|
+
evalId;
|
|
196
|
+
description;
|
|
197
|
+
promptIdx;
|
|
198
|
+
testIdx;
|
|
199
|
+
testCase;
|
|
200
|
+
prompt;
|
|
201
|
+
promptId;
|
|
202
|
+
error;
|
|
203
|
+
success;
|
|
204
|
+
score;
|
|
205
|
+
response;
|
|
206
|
+
gradingResult;
|
|
207
|
+
namedScores;
|
|
208
|
+
provider;
|
|
209
|
+
latencyMs;
|
|
210
|
+
cost;
|
|
211
|
+
metadata;
|
|
212
|
+
failureReason;
|
|
213
|
+
persisted;
|
|
214
|
+
pluginId;
|
|
215
|
+
constructor(opts) {
|
|
216
|
+
this.id = opts.id;
|
|
217
|
+
this.evalId = opts.evalId;
|
|
218
|
+
this.promptIdx = opts.promptIdx;
|
|
219
|
+
this.testIdx = opts.testIdx;
|
|
220
|
+
this.testCase = opts.testCase;
|
|
221
|
+
this.prompt = opts.prompt;
|
|
222
|
+
this.promptId = opts.promptId || hashPrompt(opts.prompt);
|
|
223
|
+
this.error = opts.error;
|
|
224
|
+
this.score = opts.score;
|
|
225
|
+
this.success = opts.success;
|
|
226
|
+
this.response = opts.response || void 0;
|
|
227
|
+
this.gradingResult = opts.gradingResult;
|
|
228
|
+
this.namedScores = opts.namedScores || {};
|
|
229
|
+
this.provider = opts.provider;
|
|
230
|
+
this.latencyMs = opts.latencyMs || 0;
|
|
231
|
+
this.cost = opts.cost || 0;
|
|
232
|
+
this.metadata = opts.metadata || {};
|
|
233
|
+
this.failureReason = isResultFailureReason(opts.failureReason) ? opts.failureReason : ResultFailureReason.NONE;
|
|
234
|
+
this.persisted = opts.persisted || false;
|
|
235
|
+
this.pluginId = opts.testCase.metadata?.pluginId;
|
|
236
|
+
}
|
|
237
|
+
async save() {
|
|
238
|
+
const db = getDb();
|
|
239
|
+
if (this.persisted) await db.update(evalResultsTable).set({
|
|
240
|
+
...this,
|
|
241
|
+
updatedAt: getCurrentTimestamp()
|
|
242
|
+
}).where(eq(evalResultsTable.id, this.id));
|
|
243
|
+
else {
|
|
244
|
+
this.id = (await db.insert(evalResultsTable).values(this).returning())[0].id;
|
|
245
|
+
this.persisted = true;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
toEvaluateResult() {
|
|
249
|
+
const shouldStripPromptText = getEnvBool("PROMPTFOO_STRIP_PROMPT_TEXT", false);
|
|
250
|
+
const shouldStripResponseOutput = getEnvBool("PROMPTFOO_STRIP_RESPONSE_OUTPUT", false);
|
|
251
|
+
const shouldStripTestVars = getEnvBool("PROMPTFOO_STRIP_TEST_VARS", false);
|
|
252
|
+
const shouldStripGradingResult = getEnvBool("PROMPTFOO_STRIP_GRADING_RESULT", false);
|
|
253
|
+
const shouldStripMetadata = getEnvBool("PROMPTFOO_STRIP_METADATA", false);
|
|
254
|
+
const response = shouldStripResponseOutput && this.response ? {
|
|
255
|
+
...this.response,
|
|
256
|
+
output: "[output stripped]"
|
|
257
|
+
} : this.response;
|
|
258
|
+
const prompt = shouldStripPromptText ? {
|
|
259
|
+
...this.prompt,
|
|
260
|
+
raw: "[prompt stripped]"
|
|
261
|
+
} : this.prompt;
|
|
262
|
+
const testCase = shouldStripTestVars ? {
|
|
263
|
+
...this.testCase,
|
|
264
|
+
vars: void 0
|
|
265
|
+
} : this.testCase;
|
|
266
|
+
return {
|
|
267
|
+
cost: this.cost,
|
|
268
|
+
description: this.description || void 0,
|
|
269
|
+
error: this.error || void 0,
|
|
270
|
+
gradingResult: shouldStripGradingResult ? null : this.gradingResult,
|
|
271
|
+
id: this.id,
|
|
272
|
+
latencyMs: this.latencyMs,
|
|
273
|
+
namedScores: this.namedScores,
|
|
274
|
+
prompt,
|
|
275
|
+
promptId: this.promptId,
|
|
276
|
+
promptIdx: this.promptIdx,
|
|
277
|
+
provider: {
|
|
278
|
+
id: this.provider.id,
|
|
279
|
+
label: this.provider.label
|
|
280
|
+
},
|
|
281
|
+
response,
|
|
282
|
+
score: this.score,
|
|
283
|
+
success: this.success,
|
|
284
|
+
testCase,
|
|
285
|
+
testIdx: this.testIdx,
|
|
286
|
+
vars: shouldStripTestVars ? {} : this.testCase.vars || {},
|
|
287
|
+
metadata: shouldStripMetadata ? {} : this.metadata,
|
|
288
|
+
failureReason: this.failureReason
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
};
|
|
292
|
+
//#endregion
|
|
293
|
+
export { evalResult_exports as n, EvalResult as t };
|
|
294
|
+
|
|
295
|
+
//# sourceMappingURL=evalResult-C7JJAPBb.js.map
|
|
@@ -0,0 +1,391 @@
|
|
|
1
|
+
const require_logger = require("./logger-DyfK9PBt.cjs");
|
|
2
|
+
const require_fetch = require("./fetch-BTxakTSg.cjs");
|
|
3
|
+
const require_accounts = require("./accounts-Bt1oJb1Z.cjs");
|
|
4
|
+
const require_blobs = require("./blobs-BgcNn97m.cjs");
|
|
5
|
+
//#region src/blobs/remoteUpload.ts
|
|
6
|
+
function buildRemoteUrl() {
|
|
7
|
+
const baseUrl = require_fetch.cloudConfig.getApiHost();
|
|
8
|
+
const apiKey = require_fetch.cloudConfig.getApiKey();
|
|
9
|
+
if (!baseUrl || !apiKey || !require_accounts.isLoggedIntoCloud()) return null;
|
|
10
|
+
try {
|
|
11
|
+
return new URL("/api/blobs", baseUrl).toString();
|
|
12
|
+
} catch (error) {
|
|
13
|
+
require_logger.logger.debug("[RemoteBlob] Invalid remote blob URL", {
|
|
14
|
+
error: error instanceof Error ? error.message : String(error),
|
|
15
|
+
baseUrl
|
|
16
|
+
});
|
|
17
|
+
return null;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
function shouldAttemptRemoteBlobUpload() {
|
|
21
|
+
return buildRemoteUrl() !== null;
|
|
22
|
+
}
|
|
23
|
+
async function uploadBlobRemote(buffer, mimeType, context) {
|
|
24
|
+
const url = buildRemoteUrl();
|
|
25
|
+
const apiKey = require_fetch.cloudConfig.getApiKey();
|
|
26
|
+
if (!url || !apiKey) return null;
|
|
27
|
+
try {
|
|
28
|
+
const { fetchWithProxy } = await Promise.resolve().then(() => require("./fetch-BTxakTSg.cjs")).then((n) => n.fetch_exports);
|
|
29
|
+
const response = await fetchWithProxy(url, {
|
|
30
|
+
method: "POST",
|
|
31
|
+
headers: {
|
|
32
|
+
"Content-Type": "application/json",
|
|
33
|
+
Authorization: `Bearer ${apiKey}`
|
|
34
|
+
},
|
|
35
|
+
body: JSON.stringify({
|
|
36
|
+
data: buffer.toString("base64"),
|
|
37
|
+
mimeType,
|
|
38
|
+
context
|
|
39
|
+
})
|
|
40
|
+
});
|
|
41
|
+
if (response.status === 404 || response.status === 400) {
|
|
42
|
+
require_logger.logger.debug("[RemoteBlob] Remote blob upload unavailable", { status: response.status });
|
|
43
|
+
return null;
|
|
44
|
+
}
|
|
45
|
+
if (!response.ok) {
|
|
46
|
+
const text = await response.text();
|
|
47
|
+
require_logger.logger.debug("[RemoteBlob] Failed to upload blob", {
|
|
48
|
+
status: response.status,
|
|
49
|
+
statusText: response.statusText,
|
|
50
|
+
body: text
|
|
51
|
+
});
|
|
52
|
+
return null;
|
|
53
|
+
}
|
|
54
|
+
const data = await response.json();
|
|
55
|
+
if (!data?.ref?.hash) {
|
|
56
|
+
require_logger.logger.debug("[RemoteBlob] Remote upload returned malformed response");
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
return data;
|
|
60
|
+
} catch (error) {
|
|
61
|
+
require_logger.logger.debug("[RemoteBlob] Error uploading blob", { error: error instanceof Error ? error.message : String(error) });
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
//#endregion
|
|
66
|
+
//#region src/blobs/extractor.ts
|
|
67
|
+
const BLOB_URI_REGEX = /^promptfoo:\/\/blob\/([a-f0-9]{64})$/i;
|
|
68
|
+
const BLOB_HASH_REGEX = /^[a-f0-9]{64}$/i;
|
|
69
|
+
function isDataUrl(value) {
|
|
70
|
+
return /^data:(audio|image)\/[^;]+;base64,/.test(value);
|
|
71
|
+
}
|
|
72
|
+
function extractBase64(value) {
|
|
73
|
+
const match = value.match(/^data:([^;]+);base64,(.+)$/);
|
|
74
|
+
if (!match) return null;
|
|
75
|
+
const mimeType = match[1];
|
|
76
|
+
try {
|
|
77
|
+
return {
|
|
78
|
+
buffer: Buffer.from(match[2], "base64"),
|
|
79
|
+
mimeType
|
|
80
|
+
};
|
|
81
|
+
} catch (error) {
|
|
82
|
+
require_logger.logger.warn("[BlobExtractor] Failed to parse base64 data URL", { error });
|
|
83
|
+
return null;
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
function shouldExternalize(buffer) {
|
|
87
|
+
const size = buffer.length;
|
|
88
|
+
return size >= 1024 && size <= 52428800;
|
|
89
|
+
}
|
|
90
|
+
function getKindFromMimeType(mimeType) {
|
|
91
|
+
return mimeType.startsWith("audio/") ? "audio" : "image";
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Normalize audio format to proper MIME type.
|
|
95
|
+
* Some providers return just 'wav' instead of 'audio/wav'.
|
|
96
|
+
* @internal Exported for testing
|
|
97
|
+
*/
|
|
98
|
+
function normalizeAudioMimeType(format) {
|
|
99
|
+
if (!format) return "audio/wav";
|
|
100
|
+
const trimmedFormat = format.trim();
|
|
101
|
+
if (/^audio\/[a-z0-9_+-]+$/i.test(trimmedFormat)) return trimmedFormat;
|
|
102
|
+
const formatLower = trimmedFormat.toLowerCase();
|
|
103
|
+
const mimeMap = {
|
|
104
|
+
wav: "audio/wav",
|
|
105
|
+
mp3: "audio/mpeg",
|
|
106
|
+
ogg: "audio/ogg",
|
|
107
|
+
flac: "audio/flac",
|
|
108
|
+
aac: "audio/aac",
|
|
109
|
+
m4a: "audio/mp4",
|
|
110
|
+
webm: "audio/webm"
|
|
111
|
+
};
|
|
112
|
+
if (mimeMap[formatLower]) return mimeMap[formatLower];
|
|
113
|
+
if (!/^[a-z0-9_-]+$/i.test(formatLower)) {
|
|
114
|
+
require_logger.logger.warn("[BlobExtractor] Invalid audio format, using default", { format });
|
|
115
|
+
return "audio/wav";
|
|
116
|
+
}
|
|
117
|
+
return `audio/${formatLower}`;
|
|
118
|
+
}
|
|
119
|
+
function parseBinary(base64OrDataUrl, defaultMimeType) {
|
|
120
|
+
if (isDataUrl(base64OrDataUrl)) {
|
|
121
|
+
const parsed = extractBase64(base64OrDataUrl);
|
|
122
|
+
if (!parsed) return null;
|
|
123
|
+
return parsed;
|
|
124
|
+
}
|
|
125
|
+
try {
|
|
126
|
+
return {
|
|
127
|
+
buffer: Buffer.from(base64OrDataUrl, "base64"),
|
|
128
|
+
mimeType: defaultMimeType
|
|
129
|
+
};
|
|
130
|
+
} catch (error) {
|
|
131
|
+
require_logger.logger.warn("[BlobExtractor] Failed to parse base64 data", { error });
|
|
132
|
+
return null;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
async function maybeStore(base64OrDataUrl, defaultMimeType, context, location, kind) {
|
|
136
|
+
const parsed = parseBinary(base64OrDataUrl, defaultMimeType);
|
|
137
|
+
if (!parsed || !shouldExternalize(parsed.buffer)) return null;
|
|
138
|
+
if (!isBlobStorageEnabled()) return null;
|
|
139
|
+
const mimeType = parsed.mimeType || "application/octet-stream";
|
|
140
|
+
const { ref } = await require_blobs.storeBlob(parsed.buffer, mimeType, {
|
|
141
|
+
...context,
|
|
142
|
+
location,
|
|
143
|
+
kind
|
|
144
|
+
});
|
|
145
|
+
if (shouldAttemptRemoteBlobUpload()) uploadBlobRemote(parsed.buffer, mimeType, {
|
|
146
|
+
evalId: context.evalId,
|
|
147
|
+
testIdx: context.testIdx,
|
|
148
|
+
promptIdx: context.promptIdx,
|
|
149
|
+
location,
|
|
150
|
+
kind
|
|
151
|
+
}).catch((error) => {
|
|
152
|
+
require_logger.logger.debug("[BlobExtractor] Cloud upload failed (non-fatal)", {
|
|
153
|
+
error: error instanceof Error ? error.message : String(error),
|
|
154
|
+
hash: ref.hash
|
|
155
|
+
});
|
|
156
|
+
});
|
|
157
|
+
return ref;
|
|
158
|
+
}
|
|
159
|
+
async function externalizeDataUrls(value, context, location) {
|
|
160
|
+
if (typeof value === "string") {
|
|
161
|
+
if (!isDataUrl(value)) return {
|
|
162
|
+
value,
|
|
163
|
+
mutated: false
|
|
164
|
+
};
|
|
165
|
+
const parsed = extractBase64(value);
|
|
166
|
+
if (!parsed || !shouldExternalize(parsed.buffer)) return {
|
|
167
|
+
value,
|
|
168
|
+
mutated: false
|
|
169
|
+
};
|
|
170
|
+
const storedRef = await maybeStore(parsed.buffer.toString("base64"), parsed.mimeType, context, location, getKindFromMimeType(parsed.mimeType)) || null;
|
|
171
|
+
if (!storedRef) return {
|
|
172
|
+
value,
|
|
173
|
+
mutated: false
|
|
174
|
+
};
|
|
175
|
+
return {
|
|
176
|
+
value: storedRef.uri,
|
|
177
|
+
mutated: true
|
|
178
|
+
};
|
|
179
|
+
}
|
|
180
|
+
if (Array.isArray(value)) {
|
|
181
|
+
let mutated = false;
|
|
182
|
+
const nextValues = await Promise.all(value.map(async (item, idx) => {
|
|
183
|
+
const { value: nextValue, mutated: childMutated } = await externalizeDataUrls(item, context, `${location}[${idx}]`);
|
|
184
|
+
mutated ||= childMutated;
|
|
185
|
+
return nextValue;
|
|
186
|
+
}));
|
|
187
|
+
return mutated ? {
|
|
188
|
+
value: nextValues,
|
|
189
|
+
mutated
|
|
190
|
+
} : {
|
|
191
|
+
value,
|
|
192
|
+
mutated: false
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
if (value && typeof value === "object") {
|
|
196
|
+
let mutated = false;
|
|
197
|
+
const nextObject = { ...value };
|
|
198
|
+
for (const [key, child] of Object.entries(value)) {
|
|
199
|
+
const { value: nextValue, mutated: childMutated } = await externalizeDataUrls(child, context, location ? `${location}.${key}` : key);
|
|
200
|
+
if (childMutated) {
|
|
201
|
+
nextObject[key] = nextValue;
|
|
202
|
+
mutated = true;
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
return mutated ? {
|
|
206
|
+
value: nextObject,
|
|
207
|
+
mutated: true
|
|
208
|
+
} : {
|
|
209
|
+
value,
|
|
210
|
+
mutated: false
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
return {
|
|
214
|
+
value,
|
|
215
|
+
mutated: false
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Best-effort extraction of binary data from provider responses.
|
|
220
|
+
* Currently focuses on audio.data fields and data URL outputs.
|
|
221
|
+
*/
|
|
222
|
+
async function extractAndStoreBinaryData(response, context) {
|
|
223
|
+
if (!response) return response;
|
|
224
|
+
let mutated = false;
|
|
225
|
+
const next = { ...response };
|
|
226
|
+
const blobContext = context || {};
|
|
227
|
+
if (response.audio?.data && typeof response.audio.data === "string") {
|
|
228
|
+
const stored = await maybeStore(response.audio.data, normalizeAudioMimeType(response.audio.format), blobContext, "response.audio.data", "audio");
|
|
229
|
+
if (stored) {
|
|
230
|
+
next.audio = {
|
|
231
|
+
...response.audio,
|
|
232
|
+
data: void 0,
|
|
233
|
+
blobRef: stored
|
|
234
|
+
};
|
|
235
|
+
mutated = true;
|
|
236
|
+
require_logger.logger.debug("[BlobExtractor] Stored audio blob", {
|
|
237
|
+
...context,
|
|
238
|
+
hash: stored.hash
|
|
239
|
+
});
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
if (response.images?.length) next.images = await Promise.all(response.images.map(async (img, idx) => {
|
|
243
|
+
if (!img.data || typeof img.data !== "string" || !isDataUrl(img.data)) return img;
|
|
244
|
+
const stored = await maybeStore(img.data, img.mimeType || "image/png", blobContext, `response.images[${idx}].data`, "image");
|
|
245
|
+
if (stored) {
|
|
246
|
+
mutated = true;
|
|
247
|
+
require_logger.logger.debug("[BlobExtractor] Stored image blob", {
|
|
248
|
+
...context,
|
|
249
|
+
hash: stored.hash
|
|
250
|
+
});
|
|
251
|
+
return {
|
|
252
|
+
...img,
|
|
253
|
+
data: void 0,
|
|
254
|
+
blobRef: stored
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
return img;
|
|
258
|
+
}));
|
|
259
|
+
const turns = response.turns;
|
|
260
|
+
if (Array.isArray(turns)) next.turns = await Promise.all(turns.map(async (turn, idx) => {
|
|
261
|
+
if (turn?.audio?.data && typeof turn.audio.data === "string") {
|
|
262
|
+
const stored = await maybeStore(turn.audio.data, normalizeAudioMimeType(turn.audio.format), blobContext, `response.turns[${idx}].audio.data`, "audio");
|
|
263
|
+
if (stored) {
|
|
264
|
+
mutated = true;
|
|
265
|
+
return {
|
|
266
|
+
...turn,
|
|
267
|
+
audio: {
|
|
268
|
+
...turn.audio,
|
|
269
|
+
data: void 0,
|
|
270
|
+
blobRef: stored
|
|
271
|
+
}
|
|
272
|
+
};
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
return turn;
|
|
276
|
+
}));
|
|
277
|
+
if (typeof response.output === "string" && isDataUrl(response.output)) {
|
|
278
|
+
const parsed = extractBase64(response.output);
|
|
279
|
+
if (parsed && shouldExternalize(parsed.buffer)) {
|
|
280
|
+
const stored = await maybeStore(parsed.buffer.toString("base64"), parsed.mimeType, blobContext, "response.output", getKindFromMimeType(parsed.mimeType));
|
|
281
|
+
if (stored) {
|
|
282
|
+
next.output = stored.uri;
|
|
283
|
+
mutated = true;
|
|
284
|
+
require_logger.logger.debug("[BlobExtractor] Stored output blob", {
|
|
285
|
+
...context,
|
|
286
|
+
hash: stored.hash
|
|
287
|
+
});
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
if (typeof response.output === "string" && response.output.trim().startsWith("{") && (response.isBase64 && response.format === "json" || response.output.includes("\"b64_json\"") || response.output.includes("b64_json"))) try {
|
|
292
|
+
const parsed = JSON.parse(response.output);
|
|
293
|
+
if (Array.isArray(parsed.data)) {
|
|
294
|
+
let jsonMutated = false;
|
|
295
|
+
const storedUris = [];
|
|
296
|
+
for (const item of parsed.data) if (item?.b64_json && typeof item.b64_json === "string") {
|
|
297
|
+
const stored = await maybeStore(item.b64_json, "image/png", blobContext, "response.output.data[].b64_json", "image");
|
|
298
|
+
if (stored) {
|
|
299
|
+
item.b64_json = stored.uri;
|
|
300
|
+
storedUris.push(stored.uri);
|
|
301
|
+
jsonMutated = true;
|
|
302
|
+
mutated = true;
|
|
303
|
+
require_logger.logger.debug("[BlobExtractor] Stored image blob from b64_json", {
|
|
304
|
+
...context,
|
|
305
|
+
hash: stored.hash
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
if (jsonMutated) {
|
|
310
|
+
if (storedUris.length === 1) next.output = storedUris[0];
|
|
311
|
+
else if (storedUris.length > 1) next.output = JSON.stringify(storedUris);
|
|
312
|
+
else next.output = JSON.stringify(parsed);
|
|
313
|
+
next.metadata = {
|
|
314
|
+
...response.metadata || {},
|
|
315
|
+
blobUris: storedUris,
|
|
316
|
+
originalFormat: response.format
|
|
317
|
+
};
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
} catch (err) {
|
|
321
|
+
require_logger.logger.debug("[BlobExtractor] Failed to parse base64 JSON output", {
|
|
322
|
+
error: err instanceof Error ? err.message : String(err),
|
|
323
|
+
location: "response.output"
|
|
324
|
+
});
|
|
325
|
+
}
|
|
326
|
+
if (response.metadata) {
|
|
327
|
+
const { value, mutated: metadataMutated } = await externalizeDataUrls(response.metadata, blobContext, "response.metadata");
|
|
328
|
+
if (metadataMutated) {
|
|
329
|
+
next.metadata = value;
|
|
330
|
+
mutated = true;
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
const finalResponse = mutated ? next : response;
|
|
334
|
+
if (blobContext.evalId) await recordExistingBlobReferences(finalResponse, blobContext, "response");
|
|
335
|
+
return finalResponse;
|
|
336
|
+
}
|
|
337
|
+
function isBlobStorageEnabled() {
|
|
338
|
+
return !require_logger.getEnvBool("PROMPTFOO_INLINE_MEDIA", false);
|
|
339
|
+
}
|
|
340
|
+
function parseBlobHashFromValue(value) {
|
|
341
|
+
if (!value) return null;
|
|
342
|
+
if (typeof value === "string") {
|
|
343
|
+
const match = value.match(BLOB_URI_REGEX);
|
|
344
|
+
return match ? match[1] : null;
|
|
345
|
+
}
|
|
346
|
+
if (typeof value === "object") {
|
|
347
|
+
const candidate = value;
|
|
348
|
+
if (candidate.hash && BLOB_HASH_REGEX.test(candidate.hash)) return candidate.hash;
|
|
349
|
+
if (candidate.uri && typeof candidate.uri === "string") {
|
|
350
|
+
const match = candidate.uri.match(BLOB_URI_REGEX);
|
|
351
|
+
if (match) return match[1];
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
return null;
|
|
355
|
+
}
|
|
356
|
+
async function recordExistingBlobReferences(value, context, location) {
|
|
357
|
+
const hash = parseBlobHashFromValue(value);
|
|
358
|
+
if (hash) {
|
|
359
|
+
await require_blobs.recordBlobReference(hash, {
|
|
360
|
+
...context,
|
|
361
|
+
location
|
|
362
|
+
});
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
if (Array.isArray(value)) {
|
|
366
|
+
await Promise.all(value.map((child, idx) => recordExistingBlobReferences(child, context, `${location}[${idx}]`)));
|
|
367
|
+
return;
|
|
368
|
+
}
|
|
369
|
+
if (value && typeof value === "object") for (const [key, child] of Object.entries(value)) await recordExistingBlobReferences(child, context, location ? `${location}.${key}` : key);
|
|
370
|
+
}
|
|
371
|
+
//#endregion
|
|
372
|
+
Object.defineProperty(exports, "extractAndStoreBinaryData", {
|
|
373
|
+
enumerable: true,
|
|
374
|
+
get: function() {
|
|
375
|
+
return extractAndStoreBinaryData;
|
|
376
|
+
}
|
|
377
|
+
});
|
|
378
|
+
Object.defineProperty(exports, "isBlobStorageEnabled", {
|
|
379
|
+
enumerable: true,
|
|
380
|
+
get: function() {
|
|
381
|
+
return isBlobStorageEnabled;
|
|
382
|
+
}
|
|
383
|
+
});
|
|
384
|
+
Object.defineProperty(exports, "shouldAttemptRemoteBlobUpload", {
|
|
385
|
+
enumerable: true,
|
|
386
|
+
get: function() {
|
|
387
|
+
return shouldAttemptRemoteBlobUpload;
|
|
388
|
+
}
|
|
389
|
+
});
|
|
390
|
+
|
|
391
|
+
//# sourceMappingURL=extractor-DnMD3fwt.cjs.map
|