@use-lattice/litmus 0.121.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (199) hide show
  1. package/LICENSE +19 -0
  2. package/dist/src/accounts-Bt1oJb1Z.cjs +219 -0
  3. package/dist/src/accounts-DjOU8Rm3.js +178 -0
  4. package/dist/src/agentic-utils-D03IiXQc.js +153 -0
  5. package/dist/src/agentic-utils-Dh7xaMQM.cjs +180 -0
  6. package/dist/src/agents-C6BIMlZa.js +231 -0
  7. package/dist/src/agents-DvIpNX1L.cjs +666 -0
  8. package/dist/src/agents-ZP0RP9vV.cjs +231 -0
  9. package/dist/src/agents-maJXdjbR.js +665 -0
  10. package/dist/src/aimlapi-BTbQjG2E.cjs +30 -0
  11. package/dist/src/aimlapi-CwMxqfXP.js +30 -0
  12. package/dist/src/audio-BBUdvsde.cjs +97 -0
  13. package/dist/src/audio-D5DPZ7I-.js +97 -0
  14. package/dist/src/base-BEysXrkq.cjs +222 -0
  15. package/dist/src/base-C451JQfq.js +193 -0
  16. package/dist/src/blobs-BY8MDmpo.js +230 -0
  17. package/dist/src/blobs-BgcNn97m.cjs +256 -0
  18. package/dist/src/cache-BBE_lsTA.cjs +4 -0
  19. package/dist/src/cache-BkrqU5Ba.js +237 -0
  20. package/dist/src/cache-DsCxFlsZ.cjs +297 -0
  21. package/dist/src/chat-CPJWDP6a.cjs +289 -0
  22. package/dist/src/chat-CXX3xzkk.cjs +811 -0
  23. package/dist/src/chat-CcDgZFJ4.js +787 -0
  24. package/dist/src/chat-Dz5ZeGO2.js +289 -0
  25. package/dist/src/chatkit-Dw0mKkML.cjs +1158 -0
  26. package/dist/src/chatkit-swAIVuea.js +1157 -0
  27. package/dist/src/chunk-DEq-mXcV.js +15 -0
  28. package/dist/src/claude-agent-sdk-BXZJtOg6.js +379 -0
  29. package/dist/src/claude-agent-sdk-CkfyjDoG.cjs +383 -0
  30. package/dist/src/cloudflare-ai-BzpJcqUH.js +161 -0
  31. package/dist/src/cloudflare-ai-Cmy_R1y2.cjs +161 -0
  32. package/dist/src/cloudflare-gateway-B9tVQKok.cjs +272 -0
  33. package/dist/src/cloudflare-gateway-DrD3ew3H.js +272 -0
  34. package/dist/src/codex-sdk-Dezj9Nwm.js +1056 -0
  35. package/dist/src/codex-sdk-Dl9D4k5B.cjs +1060 -0
  36. package/dist/src/cometapi-C-9YvCHC.js +54 -0
  37. package/dist/src/cometapi-DHgDKoO2.cjs +54 -0
  38. package/dist/src/completion-B8Ctyxpr.js +120 -0
  39. package/dist/src/completion-Cxrt08sj.cjs +131 -0
  40. package/dist/src/createHash-BwgE13yv.cjs +27 -0
  41. package/dist/src/createHash-DmPQkvBh.js +15 -0
  42. package/dist/src/docker-BiqcTwLv.js +80 -0
  43. package/dist/src/docker-C7tEJnP-.cjs +80 -0
  44. package/dist/src/esm-C62Zofr1.cjs +409 -0
  45. package/dist/src/esm-DMVc93eh.js +379 -0
  46. package/dist/src/evalResult-C3NJPQOo.cjs +301 -0
  47. package/dist/src/evalResult-C7JJAPBb.js +295 -0
  48. package/dist/src/evalResult-DoVTZZWI.cjs +2 -0
  49. package/dist/src/extractor-DnMD3fwt.cjs +391 -0
  50. package/dist/src/extractor-DtlL28vL.js +374 -0
  51. package/dist/src/fetch-BTxakTSg.cjs +1133 -0
  52. package/dist/src/fetch-DQckpUFz.js +928 -0
  53. package/dist/src/fileExtensions-DnqA1y9x.js +85 -0
  54. package/dist/src/fileExtensions-bYh77CN8.cjs +114 -0
  55. package/dist/src/genaiTracer-CyZrmaK0.cjs +268 -0
  56. package/dist/src/genaiTracer-D3fD9dNV.js +256 -0
  57. package/dist/src/graders-BNscxFrU.js +13644 -0
  58. package/dist/src/graders-D2oE9Msq.js +2 -0
  59. package/dist/src/graders-c0Ez_w-9.cjs +2 -0
  60. package/dist/src/graders-d0F2M3e9.cjs +14056 -0
  61. package/dist/src/image-0ZhE0VlR.cjs +280 -0
  62. package/dist/src/image-CWE1pdNv.js +257 -0
  63. package/dist/src/image-D9ZK6hwL.js +163 -0
  64. package/dist/src/image-DKZgZITg.cjs +163 -0
  65. package/dist/src/index.cjs +11366 -0
  66. package/dist/src/index.d.cts +19640 -0
  67. package/dist/src/index.d.ts +19641 -0
  68. package/dist/src/index.js +11306 -0
  69. package/dist/src/invariant-Ddh24eXh.js +25 -0
  70. package/dist/src/invariant-kfQ8Bu82.cjs +30 -0
  71. package/dist/src/knowledgeBase-BgPyGFUd.cjs +122 -0
  72. package/dist/src/knowledgeBase-DyHilYaP.js +122 -0
  73. package/dist/src/litellm-CyMeneHS.js +135 -0
  74. package/dist/src/litellm-DWDF73yF.cjs +135 -0
  75. package/dist/src/logger-C40ZGil9.js +717 -0
  76. package/dist/src/logger-DyfK9PBt.cjs +917 -0
  77. package/dist/src/luma-ray-BAU9X_ep.cjs +315 -0
  78. package/dist/src/luma-ray-nwVseBbv.js +313 -0
  79. package/dist/src/messages-B5ADWTTv.js +245 -0
  80. package/dist/src/messages-BCnZfqrS.cjs +257 -0
  81. package/dist/src/meteor-DLZZ3osF.cjs +134 -0
  82. package/dist/src/meteor-DUiCJRC-.js +134 -0
  83. package/dist/src/modelslab-00cveB8L.cjs +163 -0
  84. package/dist/src/modelslab-D9sCU_L7.js +163 -0
  85. package/dist/src/nova-reel-CTapvqYH.js +276 -0
  86. package/dist/src/nova-reel-DlWuuroF.cjs +278 -0
  87. package/dist/src/nova-sonic-5UPWfeMv.cjs +363 -0
  88. package/dist/src/nova-sonic-BhSwQNym.js +363 -0
  89. package/dist/src/openai-BWrJK9d8.cjs +52 -0
  90. package/dist/src/openai-DumO8WQn.js +47 -0
  91. package/dist/src/openclaw-B8brrjC_.cjs +577 -0
  92. package/dist/src/openclaw-Bkayww9q.js +571 -0
  93. package/dist/src/opencode-sdk-7xjoDNiM.cjs +562 -0
  94. package/dist/src/opencode-sdk-SGwAPxht.js +558 -0
  95. package/dist/src/otlpReceiver-CoAHfAN9.cjs +15 -0
  96. package/dist/src/otlpReceiver-oO3EQwI9.js +14 -0
  97. package/dist/src/providerRegistry-4yjhaEM8.js +45 -0
  98. package/dist/src/providerRegistry-DhV4rJIc.cjs +50 -0
  99. package/dist/src/providers-B5RJVG-7.cjs +33609 -0
  100. package/dist/src/providers-BdmZCLzV.js +33262 -0
  101. package/dist/src/providers-CxtRxn8e.js +2 -0
  102. package/dist/src/providers-DnQLNbx1.cjs +3 -0
  103. package/dist/src/pythonUtils-BD0druiM.cjs +275 -0
  104. package/dist/src/pythonUtils-IBhn5YGR.js +249 -0
  105. package/dist/src/quiverai-BDOwZBsM.cjs +213 -0
  106. package/dist/src/quiverai-D3JTF5lD.js +213 -0
  107. package/dist/src/responses-B2LCDCXZ.js +667 -0
  108. package/dist/src/responses-BvNm4Xv9.cjs +685 -0
  109. package/dist/src/rubyUtils-B0NwnfpY.cjs +245 -0
  110. package/dist/src/rubyUtils-BroxzZ7c.cjs +2 -0
  111. package/dist/src/rubyUtils-hqVw5UvJ.js +222 -0
  112. package/dist/src/sagemaker-Cno2V-Sx.js +689 -0
  113. package/dist/src/sagemaker-fV_KUgs5.cjs +691 -0
  114. package/dist/src/server-BOuAXb06.cjs +238 -0
  115. package/dist/src/server-CtI-EWzm.cjs +2 -0
  116. package/dist/src/server-Cy3DZymt.js +189 -0
  117. package/dist/src/slack-CP8xBePa.js +135 -0
  118. package/dist/src/slack-DSQ1yXVb.cjs +135 -0
  119. package/dist/src/store-BwDDaBjb.cjs +246 -0
  120. package/dist/src/store-DcbLC593.cjs +2 -0
  121. package/dist/src/store-IGpqMIkv.js +240 -0
  122. package/dist/src/tables-3Q2cL7So.cjs +373 -0
  123. package/dist/src/tables-Bi2fjr4W.js +288 -0
  124. package/dist/src/telemetry-Bg2WqF79.js +161 -0
  125. package/dist/src/telemetry-D0x6u5kX.cjs +166 -0
  126. package/dist/src/telemetry-DXNimrI0.cjs +2 -0
  127. package/dist/src/text-B_UCRPp2.js +22 -0
  128. package/dist/src/text-CW1cyrwj.cjs +33 -0
  129. package/dist/src/tokenUsageUtils-NYT-WKS6.js +138 -0
  130. package/dist/src/tokenUsageUtils-bVa1ga6f.cjs +173 -0
  131. package/dist/src/transcription-Cl_W16Pr.js +122 -0
  132. package/dist/src/transcription-yt1EecY8.cjs +124 -0
  133. package/dist/src/transform-BCtGrl_W.cjs +228 -0
  134. package/dist/src/transform-Bv6gG2MJ.cjs +1688 -0
  135. package/dist/src/transform-CY1wbpRy.js +1507 -0
  136. package/dist/src/transform-DU8rUL9P.cjs +2 -0
  137. package/dist/src/transform-yWaShiKr.js +216 -0
  138. package/dist/src/transformersAvailability-BGkzavwb.js +35 -0
  139. package/dist/src/transformersAvailability-DKoRtQLy.cjs +35 -0
  140. package/dist/src/types-5aqHpBwE.cjs +3769 -0
  141. package/dist/src/types-Bn6D9c4U.js +3300 -0
  142. package/dist/src/util-BkKlTkI2.js +293 -0
  143. package/dist/src/util-CTh0bfOm.cjs +1119 -0
  144. package/dist/src/util-D17oBwo7.cjs +328 -0
  145. package/dist/src/util-DsS_-v4p.js +613 -0
  146. package/dist/src/util-DuntT1Ga.js +951 -0
  147. package/dist/src/util-aWjdCYMI.cjs +667 -0
  148. package/dist/src/utils-CisQwpjA.js +94 -0
  149. package/dist/src/utils-yWamDvmz.cjs +123 -0
  150. package/dist/tsconfig.tsbuildinfo +1 -0
  151. package/drizzle/0000_lush_hellion.sql +36 -0
  152. package/drizzle/0001_wide_calypso.sql +3 -0
  153. package/drizzle/0002_tidy_juggernaut.sql +1 -0
  154. package/drizzle/0003_lively_naoko.sql +8 -0
  155. package/drizzle/0004_minor_peter_quill.sql +19 -0
  156. package/drizzle/0005_silky_millenium_guard.sql +2 -0
  157. package/drizzle/0006_harsh_caretaker.sql +42 -0
  158. package/drizzle/0007_cloudy_wong.sql +1 -0
  159. package/drizzle/0008_broad_boomer.sql +2 -0
  160. package/drizzle/0009_strong_marten_broadcloak.sql +19 -0
  161. package/drizzle/0010_needy_bishop.sql +11 -0
  162. package/drizzle/0011_moaning_millenium_guard.sql +1 -0
  163. package/drizzle/0012_late_marten_broadcloak.sql +2 -0
  164. package/drizzle/0013_previous_dormammu.sql +9 -0
  165. package/drizzle/0014_lazy_captain_universe.sql +2 -0
  166. package/drizzle/0015_zippy_wallop.sql +29 -0
  167. package/drizzle/0016_jazzy_zemo.sql +2 -0
  168. package/drizzle/0017_reflective_praxagora.sql +4 -0
  169. package/drizzle/0018_fat_vanisher.sql +22 -0
  170. package/drizzle/0019_new_clint_barton.sql +8 -0
  171. package/drizzle/0020_skinny_maverick.sql +1 -0
  172. package/drizzle/0021_mysterious_madelyne_pryor.sql +13 -0
  173. package/drizzle/0022_sleepy_ultimo.sql +25 -0
  174. package/drizzle/0023_wooden_mandrill.sql +2 -0
  175. package/drizzle/AGENTS.md +68 -0
  176. package/drizzle/CLAUDE.md +1 -0
  177. package/drizzle/meta/0000_snapshot.json +221 -0
  178. package/drizzle/meta/0001_snapshot.json +214 -0
  179. package/drizzle/meta/0002_snapshot.json +221 -0
  180. package/drizzle/meta/0005_snapshot.json +369 -0
  181. package/drizzle/meta/0006_snapshot.json +638 -0
  182. package/drizzle/meta/0007_snapshot.json +640 -0
  183. package/drizzle/meta/0008_snapshot.json +649 -0
  184. package/drizzle/meta/0009_snapshot.json +554 -0
  185. package/drizzle/meta/0010_snapshot.json +619 -0
  186. package/drizzle/meta/0011_snapshot.json +627 -0
  187. package/drizzle/meta/0012_snapshot.json +639 -0
  188. package/drizzle/meta/0013_snapshot.json +717 -0
  189. package/drizzle/meta/0014_snapshot.json +717 -0
  190. package/drizzle/meta/0015_snapshot.json +897 -0
  191. package/drizzle/meta/0016_snapshot.json +1031 -0
  192. package/drizzle/meta/0018_snapshot.json +1210 -0
  193. package/drizzle/meta/0019_snapshot.json +1165 -0
  194. package/drizzle/meta/0020_snapshot.json +1232 -0
  195. package/drizzle/meta/0021_snapshot.json +1311 -0
  196. package/drizzle/meta/0022_snapshot.json +1481 -0
  197. package/drizzle/meta/0023_snapshot.json +1496 -0
  198. package/drizzle/meta/_journal.json +174 -0
  199. package/package.json +240 -0
@@ -0,0 +1,717 @@
1
+ import { t as invariant } from "./invariant-Ddh24eXh.js";
2
+ import fs from "fs";
3
+ import * as path$1 from "path";
4
+ import path from "path";
5
+ import yaml from "js-yaml";
6
+ import dotenv from "dotenv";
7
+ import winston from "winston";
8
+ import * as os$1 from "os";
9
+ import Ajv from "ajv";
10
+ import addFormats from "ajv-formats";
11
+ import safeStringify from "fast-safe-stringify";
12
+ //#region src/cliState.ts
13
+ const state = {};
14
+ //#endregion
15
+ //#region src/envars.ts
16
+ dotenv.config({ quiet: true });
17
+ function getEnvString(key, defaultValue) {
18
+ if (state.config?.env && typeof state.config.env === "object") {
19
+ const envValue = state.config.env[key];
20
+ if (envValue !== void 0) return String(envValue);
21
+ }
22
+ const value = process.env[key];
23
+ if (value === void 0) return defaultValue;
24
+ return value;
25
+ }
26
+ /**
27
+ * Get a boolean environment variable.
28
+ * @param key The name of the environment variable.
29
+ * @param defaultValue Optional default value if the environment variable is not set.
30
+ * @returns The boolean value of the environment variable, or the default value if provided.
31
+ */
32
+ function getEnvBool(key, defaultValue) {
33
+ const value = getEnvString(key) || defaultValue;
34
+ if (typeof value === "boolean") return value;
35
+ if (typeof value === "string") return [
36
+ "1",
37
+ "true",
38
+ "yes",
39
+ "yup",
40
+ "yeppers"
41
+ ].includes(value.toLowerCase());
42
+ return Boolean(defaultValue);
43
+ }
44
+ function getEnvInt(key, defaultValue) {
45
+ const str = getEnvString(key);
46
+ if (str === void 0 || str === "") return defaultValue;
47
+ const parsedValue = Number.parseInt(str, 10);
48
+ if (!Number.isNaN(parsedValue)) return parsedValue;
49
+ return defaultValue;
50
+ }
51
+ function getEnvFloat(key, defaultValue) {
52
+ const str = getEnvString(key);
53
+ if (str === void 0 || str === "") return defaultValue;
54
+ const parsedValue = Number.parseFloat(str);
55
+ if (!Number.isNaN(parsedValue)) return parsedValue;
56
+ return defaultValue;
57
+ }
58
+ /**
59
+ * Get the timeout in milliseconds for each individual test case/provider API call.
60
+ * When this timeout is reached, that specific test is marked as an error.
61
+ * @param defaultValue Optional default value if the environment variable is not set. Defaults to 0 (no timeout).
62
+ * @returns The timeout value in milliseconds, or the default value if not set.
63
+ */
64
+ function getEvalTimeoutMs(defaultValue = 0) {
65
+ return getEnvInt("PROMPTFOO_EVAL_TIMEOUT_MS", defaultValue);
66
+ }
67
+ /**
68
+ * Get the maximum total runtime in milliseconds for the entire evaluation process.
69
+ * When this timeout is reached, all remaining tests are marked as errors and the evaluation ends.
70
+ * @param defaultValue Optional default value if the environment variable is not set. Defaults to 0 (no limit).
71
+ * @returns The max duration in milliseconds, or the default value if not set.
72
+ */
73
+ function getMaxEvalTimeMs(defaultValue = 0) {
74
+ return getEnvInt("PROMPTFOO_MAX_EVAL_TIME_MS", defaultValue);
75
+ }
76
+ /**
77
+ * Check if the application is running in a CI environment.
78
+ * @returns True if running in a CI environment, false otherwise.
79
+ */
80
+ function isCI() {
81
+ return getEnvBool("CI") || getEnvBool("GITHUB_ACTIONS") || getEnvBool("TRAVIS") || getEnvBool("CIRCLECI") || getEnvBool("JENKINS") || getEnvBool("GITLAB_CI") || getEnvBool("APPVEYOR") || getEnvBool("CODEBUILD_BUILD_ID") || getEnvBool("TF_BUILD") || getEnvBool("BITBUCKET_COMMIT") || getEnvBool("BUDDY") || getEnvBool("BUILDKITE") || getEnvBool("TEAMCITY_VERSION");
82
+ }
83
+ //#endregion
84
+ //#region src/util/config/manage.ts
85
+ let configDirectoryPath = getEnvString("PROMPTFOO_CONFIG_DIR");
86
+ const isNodeEnvironment = typeof process !== "undefined" && process.versions && process.versions.node;
87
+ function getConfigDirectoryPath(createIfNotExists = false) {
88
+ const p = configDirectoryPath || path$1.join(os$1.homedir(), ".promptfoo");
89
+ if (createIfNotExists && isNodeEnvironment) try {
90
+ fs.mkdirSync(p, { recursive: true });
91
+ } catch {}
92
+ return p;
93
+ }
94
+ //#endregion
95
+ //#region src/util/json.ts
96
+ let ajvInstance = null;
97
+ function getAjv() {
98
+ if (!ajvInstance) {
99
+ ajvInstance = new Ajv({ strictSchema: !getEnvBool("PROMPTFOO_DISABLE_AJV_STRICT_MODE") });
100
+ addFormats(ajvInstance);
101
+ }
102
+ return ajvInstance;
103
+ }
104
+ function isValidJson(str) {
105
+ try {
106
+ JSON.parse(str);
107
+ return true;
108
+ } catch {
109
+ return false;
110
+ }
111
+ }
112
+ /**
113
+ * Creates a truncated version of an object for safe JSON stringification.
114
+ * Prevents memory issues by limiting string, array, and object sizes.
115
+ *
116
+ * @param value - The value to truncate and stringify
117
+ * @param prettyPrint - Whether to format the JSON with indentation
118
+ * @returns A JSON string representation of the truncated value
119
+ */
120
+ function safeJsonStringifyTruncated(value, prettyPrint = false) {
121
+ const cache = /* @__PURE__ */ new Set();
122
+ const space = prettyPrint ? 2 : void 0;
123
+ const truncateValue = (val) => {
124
+ if (typeof val === "string") return val.length > 1e3 ? val.substring(0, 1e3) + "...[truncated]" : val;
125
+ if (Array.isArray(val)) {
126
+ const truncated = val.slice(0, 10).map(truncateValue);
127
+ if (val.length > 10) truncated.push(`...[${val.length - 10} more items]`);
128
+ return truncated;
129
+ }
130
+ if (typeof val === "object" && val !== null) {
131
+ if (cache.has(val)) return "[Circular Reference]";
132
+ cache.add(val);
133
+ const truncated = {};
134
+ let count = 0;
135
+ for (const [k, v] of Object.entries(val)) {
136
+ if (count >= 20) {
137
+ truncated["...[truncated]"] = `${Object.keys(val).length - count} more keys`;
138
+ break;
139
+ }
140
+ truncated[k] = truncateValue(v);
141
+ count++;
142
+ }
143
+ cache.delete(val);
144
+ return truncated;
145
+ }
146
+ return val;
147
+ };
148
+ try {
149
+ return JSON.stringify(truncateValue(value), null, space) || "{}";
150
+ } catch {
151
+ return `{"error": "Failed to stringify even truncated data", "type": "${typeof value}", "constructor": "${value?.constructor?.name || "unknown"}"}`;
152
+ }
153
+ }
154
+ /**
155
+ * Safely stringify a value to JSON, handling circular references and large objects.
156
+ *
157
+ * @param value - The value to stringify
158
+ * @param prettyPrint - Whether to format the JSON with indentation
159
+ * @returns JSON string representation, or undefined if serialization fails
160
+ */
161
+ function safeJsonStringify(value, prettyPrint = false) {
162
+ const ancestors = [];
163
+ const space = prettyPrint ? 2 : void 0;
164
+ try {
165
+ return JSON.stringify(value, function(_key, val) {
166
+ if (typeof val === "object" && val !== null) {
167
+ while (ancestors.length > 0 && ancestors[ancestors.length - 1] !== this) ancestors.pop();
168
+ if (ancestors.includes(val)) return;
169
+ ancestors.push(val);
170
+ }
171
+ return val;
172
+ }, space) || void 0;
173
+ } catch (error) {
174
+ if (error instanceof RangeError && error.message.includes("Invalid string length")) return safeJsonStringifyTruncated(value, prettyPrint);
175
+ return;
176
+ }
177
+ }
178
+ function convertSlashCommentsToHash(str) {
179
+ return str.split("\n").map((line) => {
180
+ let state = "normal";
181
+ let result = "";
182
+ let i = 0;
183
+ while (i < line.length) {
184
+ const char = line[i];
185
+ const nextChar = line[i + 1];
186
+ const prevChar = i > 0 ? line[i - 1] : "";
187
+ switch (state) {
188
+ case "normal":
189
+ if (char === "'" && !/[a-zA-Z]/.test(prevChar)) {
190
+ state = "singleQuote";
191
+ result += char;
192
+ } else if (char === "\"") {
193
+ state = "doubleQuote";
194
+ result += char;
195
+ } else if (char === "/" && nextChar === "/") {
196
+ let tokenStart = 0;
197
+ for (let j = i - 1; j >= 0; j--) if (/\s/.test(line[j])) {
198
+ tokenStart = j + 1;
199
+ break;
200
+ }
201
+ if (line.slice(tokenStart, i + 2).includes("://")) {
202
+ result += char;
203
+ break;
204
+ }
205
+ let slashCount = 2;
206
+ while (i + slashCount < line.length && line[i + slashCount] === "/") slashCount++;
207
+ const hashes = "#".repeat(Math.floor(slashCount / 2));
208
+ return result + hashes + line.slice(i + slashCount);
209
+ } else result += char;
210
+ break;
211
+ case "singleQuote":
212
+ result += char;
213
+ if (char === "'" && prevChar !== "\\" && !/[a-zA-Z]/.test(nextChar)) state = "normal";
214
+ break;
215
+ case "doubleQuote":
216
+ result += char;
217
+ if (char === "\"" && prevChar !== "\\") state = "normal";
218
+ break;
219
+ }
220
+ i++;
221
+ }
222
+ return result;
223
+ }).join("\n");
224
+ }
225
+ function extractJsonObjects(str) {
226
+ const jsonObjects = [];
227
+ const maxJsonLength = 1e5;
228
+ for (let i = 0; i < str.length; i++) if (str[i] === "{") {
229
+ let openBraces = 1;
230
+ let closeBraces = 0;
231
+ let j = i + 1;
232
+ while (j < Math.min(i + maxJsonLength, str.length) && openBraces > closeBraces) {
233
+ if (str[j] === "{") openBraces++;
234
+ if (str[j] === "}") closeBraces++;
235
+ j++;
236
+ if (openBraces === closeBraces || j === str.length || j === i + maxJsonLength) try {
237
+ let potentialJson = str.slice(i, j);
238
+ if (openBraces > closeBraces) potentialJson += "}".repeat(openBraces - closeBraces);
239
+ const processedJson = convertSlashCommentsToHash(potentialJson);
240
+ const parsedObj = yaml.load(processedJson, { json: true });
241
+ if (typeof parsedObj === "object" && parsedObj !== null) {
242
+ jsonObjects.push(parsedObj);
243
+ i = j - 1;
244
+ break;
245
+ }
246
+ } catch {
247
+ if (openBraces === closeBraces) break;
248
+ }
249
+ }
250
+ }
251
+ return jsonObjects;
252
+ }
253
+ function extractFirstJsonObject(str) {
254
+ const jsonObjects = extractJsonObjects(str);
255
+ invariant(jsonObjects.length >= 1, `Expected a JSON object, but got ${JSON.stringify(str)}`);
256
+ return jsonObjects[0];
257
+ }
258
+ /**
259
+ * Reorders the keys of an object based on a specified order, preserving any unspecified keys.
260
+ * Symbol keys are preserved and added at the end.
261
+ *
262
+ * @param obj - The object whose keys need to be reordered.
263
+ * @param order - An array specifying the desired order of keys.
264
+ * @returns A new object with keys reordered according to the specified order.
265
+ *
266
+ * @example
267
+ * const obj = { c: 3, a: 1, b: 2 };
268
+ * const orderedObj = orderKeys(obj, ['a', 'b']);
269
+ * // Result: { a: 1, b: 2, c: 3 }
270
+ */
271
+ function orderKeys(obj, order) {
272
+ const result = {};
273
+ for (const key of order) if (key in obj && obj[key] !== void 0) result[key] = obj[key];
274
+ for (const key in obj) if (!(key in result) && obj[key] !== void 0) result[key] = obj[key];
275
+ const symbolKeys = Object.getOwnPropertySymbols(obj);
276
+ for (const sym of symbolKeys) if (obj[sym] !== void 0) result[sym] = obj[sym];
277
+ return result;
278
+ }
279
+ /**
280
+ * Creates a summary of an EvaluateResult for logging purposes, avoiding RangeError
281
+ * when stringifying large evaluation results.
282
+ *
283
+ * Extracts key information while truncating potentially large fields like response
284
+ * outputs and metadata values.
285
+ *
286
+ * @param result - The evaluation result to summarize
287
+ * @param maxOutputLength - Maximum length for response output before truncation. Default: 500
288
+ * @param includeMetadataKeys - Whether to include metadata keys in the summary. Default: true
289
+ * @returns A summarized version safe for JSON stringification
290
+ * @throws {TypeError} If result is null or undefined
291
+ */
292
+ function summarizeEvaluateResultForLogging(result, maxOutputLength = 500, includeMetadataKeys = true) {
293
+ if (!result) throw new TypeError("EvaluateResult cannot be null or undefined");
294
+ const summary = {
295
+ id: result.id,
296
+ testIdx: result.testIdx,
297
+ promptIdx: result.promptIdx,
298
+ success: result.success,
299
+ score: result.score,
300
+ error: result.error,
301
+ failureReason: result.failureReason
302
+ };
303
+ if (result.provider) summary.provider = {
304
+ id: result.provider.id || "",
305
+ label: result.provider.label
306
+ };
307
+ if (result.response) {
308
+ summary.response = {
309
+ error: result.response.error,
310
+ cached: result.response.cached,
311
+ cost: result.response.cost,
312
+ tokenUsage: result.response.tokenUsage
313
+ };
314
+ if (result.response.output != null) {
315
+ const output = String(result.response.output);
316
+ summary.response.output = output.length > maxOutputLength ? output.substring(0, maxOutputLength) + "...[truncated]" : output;
317
+ }
318
+ if (result.response.metadata && includeMetadataKeys) summary.response.metadata = {
319
+ keys: Object.keys(result.response.metadata),
320
+ keyCount: Object.keys(result.response.metadata).length
321
+ };
322
+ }
323
+ if (result.testCase) summary.testCase = {
324
+ description: result.testCase.description,
325
+ vars: result.testCase.vars ? Object.keys(result.testCase.vars) : void 0
326
+ };
327
+ return summary;
328
+ }
329
+ //#endregion
330
+ //#region src/util/sanitizer.ts
331
+ /**
332
+ * Generic utility functions for sanitizing objects to prevent logging of secrets and credentials
333
+ * Uses a custom recursive approach for reliable deep object sanitization.
334
+ */
335
+ const MAX_DEPTH = 4;
336
+ const DUMMY_BASE = "http://placeholder";
337
+ const REDACTED = "[REDACTED]";
338
+ /**
339
+ * Set of field names that should be redacted (case-insensitive, with hyphens/underscores normalized)
340
+ * Note: Keys are stored in their normalized form (lowercase, no hyphens/underscores)
341
+ */
342
+ const SECRET_FIELD_NAMES = new Set([
343
+ "password",
344
+ "passwd",
345
+ "pwd",
346
+ "secret",
347
+ "secrets",
348
+ "secretkey",
349
+ "credentials",
350
+ "apikey",
351
+ "apisecret",
352
+ "token",
353
+ "accesstoken",
354
+ "refreshtoken",
355
+ "idtoken",
356
+ "bearertoken",
357
+ "authtoken",
358
+ "clientsecret",
359
+ "webhooksecret",
360
+ "anthropicapikey",
361
+ "awsbearertokenbedrock",
362
+ "authorization",
363
+ "auth",
364
+ "bearer",
365
+ "apikeyenvar",
366
+ "xapikey",
367
+ "xauthtoken",
368
+ "xaccesstoken",
369
+ "xauth",
370
+ "xsecret",
371
+ "xcsrftoken",
372
+ "xsessiondata",
373
+ "csrftoken",
374
+ "sessionid",
375
+ "session",
376
+ "cookie",
377
+ "setcookie",
378
+ "certificatepassword",
379
+ "keystorepassword",
380
+ "pfxpassword",
381
+ "privatekey",
382
+ "certkey",
383
+ "encryptionkey",
384
+ "signingkey",
385
+ "signature",
386
+ "sig",
387
+ "passphrase",
388
+ "certificatecontent",
389
+ "keystorecontent",
390
+ "pfx",
391
+ "pfxcontent",
392
+ "keycontent",
393
+ "certcontent"
394
+ ]);
395
+ /**
396
+ * Normalize field names for comparison (lowercase, no hyphens/underscores)
397
+ */
398
+ function normalizeFieldName(fieldName) {
399
+ return fieldName.toLowerCase().replace(/[-_]/g, "");
400
+ }
401
+ /**
402
+ * Check if a field name should be redacted
403
+ */
404
+ function isSecretField(fieldName) {
405
+ return SECRET_FIELD_NAMES.has(normalizeFieldName(fieldName));
406
+ }
407
+ /**
408
+ * Check if a value looks like a secret based on common patterns.
409
+ * Detects API keys, tokens, and other credential patterns.
410
+ */
411
+ function looksLikeSecret(value) {
412
+ if (typeof value !== "string") return false;
413
+ if (/^sk-[a-zA-Z0-9-_]{20,}/.test(value)) return true;
414
+ if (/^sk-proj-[a-zA-Z0-9-_]{20,}/.test(value)) return true;
415
+ if (/^sk-ant-[a-zA-Z0-9-_]{20,}/.test(value)) return true;
416
+ if (/^key-[a-zA-Z0-9]{20,}/.test(value)) return true;
417
+ if (/^Bearer\s+.{20,}/i.test(value)) return true;
418
+ if (/^Basic\s+.{20,}/i.test(value)) return true;
419
+ if (/^[a-zA-Z0-9+/=_-]{64,}$/.test(value)) return true;
420
+ if (/^AKIA[A-Z0-9]{16}/.test(value)) return true;
421
+ if (/^AIza[a-zA-Z0-9_-]{35}/.test(value)) return true;
422
+ return false;
423
+ }
424
+ /**
425
+ * Detect class instances (objects with custom prototypes and methods)
426
+ */
427
+ function isClassInstance(obj) {
428
+ const proto = Object.getPrototypeOf(obj);
429
+ if (!proto || proto === Object.prototype) return false;
430
+ return Object.getOwnPropertyNames(proto).some((prop) => prop !== "constructor" && typeof proto[prop] === "function");
431
+ }
432
+ /**
433
+ * Parse and sanitize JSON strings, also check if the string looks like a secret
434
+ */
435
+ function sanitizeJsonString(str, depth, maxDepth) {
436
+ try {
437
+ const parsed = JSON.parse(str);
438
+ if (parsed && typeof parsed === "object") {
439
+ const sanitized = recursiveSanitize(parsed, depth, maxDepth);
440
+ return JSON.stringify(sanitized);
441
+ }
442
+ } catch {
443
+ if (looksLikeSecret(str)) return REDACTED;
444
+ }
445
+ return str;
446
+ }
447
+ /**
448
+ * Sanitize plain object fields
449
+ */
450
+ function sanitizePlainObject(obj, depth, maxDepth) {
451
+ const sanitized = {};
452
+ for (const [key, value] of Object.entries(obj)) if (key === "url" && typeof value === "string") sanitized[key] = sanitizeUrl(value);
453
+ else if (isSecretField(key)) sanitized[key] = REDACTED;
454
+ else if (typeof value === "string" && looksLikeSecret(value)) sanitized[key] = REDACTED;
455
+ else sanitized[key] = recursiveSanitize(value, depth + 1, maxDepth);
456
+ return sanitized;
457
+ }
458
+ /**
459
+ * Recursively sanitize an object, redacting secret fields at any depth
460
+ */
461
+ function recursiveSanitize(obj, depth = 0, maxDepth = MAX_DEPTH) {
462
+ if (typeof obj === "function") return `[Function] ${obj.name}`;
463
+ if (typeof obj === "string") return sanitizeJsonString(obj, depth, maxDepth);
464
+ if (obj === null || obj === void 0 || typeof obj !== "object") return obj;
465
+ if (depth > maxDepth) return "[...]";
466
+ if (Array.isArray(obj)) return obj.map((item) => recursiveSanitize(item, depth + 1, maxDepth));
467
+ if (isClassInstance(obj)) return `[${obj.constructor?.name || "Object"} Instance]`;
468
+ return sanitizePlainObject(obj, depth, maxDepth);
469
+ }
470
+ /**
471
+ * Generic function to sanitize any object by removing or redacting sensitive information
472
+ * @param obj - The object to sanitize
473
+ * @param options - Optional configuration
474
+ * @returns A sanitized copy of the object with secrets redacted
475
+ */
476
+ function sanitizeObject(obj, options = {}) {
477
+ const { context = "object", throwOnError = false, maxDepth = MAX_DEPTH } = options;
478
+ try {
479
+ if (obj === null || obj === void 0) return obj;
480
+ if (typeof obj === "string") return sanitizeJsonString(obj, 0, maxDepth);
481
+ if (typeof obj !== "object") return obj;
482
+ return recursiveSanitize(JSON.parse(safeStringify(obj, (_key, val) => {
483
+ if (val instanceof Error) return {
484
+ name: val.name,
485
+ message: val.message
486
+ };
487
+ return val;
488
+ }, void 0, {
489
+ depthLimit: Number.MAX_SAFE_INTEGER,
490
+ edgesLimit: Number.MAX_SAFE_INTEGER
491
+ })), 0, maxDepth);
492
+ } catch (error) {
493
+ if (throwOnError) throw error;
494
+ console.error(`Error sanitizing ${context}:`, error);
495
+ return obj;
496
+ }
497
+ }
498
+ function sanitizeUrl(url) {
499
+ try {
500
+ if (typeof url !== "string" || !url.trim()) return url;
501
+ if (url.includes("{{") && url.includes("}}")) return url;
502
+ const isPathOnly = url.startsWith("/") && !url.startsWith("//");
503
+ const parsedUrl = isPathOnly ? new URL(url, DUMMY_BASE) : new URL(url);
504
+ const sanitizedUrl = new URL(parsedUrl.href);
505
+ if (sanitizedUrl.username || sanitizedUrl.password) {
506
+ sanitizedUrl.username = "***";
507
+ sanitizedUrl.password = "***";
508
+ }
509
+ const sensitiveParams = /(api[_-]?key|token|password|secret|signature|sig|access[_-]?token|refresh[_-]?token|id[_-]?token|client[_-]?secret|authorization)/i;
510
+ try {
511
+ for (const key of Array.from(sanitizedUrl.searchParams.keys())) if (sensitiveParams.test(key)) sanitizedUrl.searchParams.set(key, "[REDACTED]");
512
+ } catch (paramError) {
513
+ console.warn(`Failed to sanitize URL parameters ${url}: ${paramError}`);
514
+ }
515
+ if (isPathOnly) return sanitizedUrl.pathname + sanitizedUrl.search + sanitizedUrl.hash;
516
+ return sanitizedUrl.toString();
517
+ } catch (error) {
518
+ console.warn(`Failed to sanitize URL ${url}: ${error}`);
519
+ return url;
520
+ }
521
+ }
522
+ //#endregion
523
+ //#region src/logger.ts
524
+ let globalLogCallback = null;
525
+ function setLogCallback(callback) {
526
+ globalLogCallback = callback;
527
+ }
528
+ const LOG_LEVELS = {
529
+ error: 0,
530
+ warn: 1,
531
+ info: 2,
532
+ debug: 3
533
+ };
534
+ let sourceMapSupportInitialized = false;
535
+ let sourceMapSupportInitializationPromise = null;
536
+ async function initializeSourceMapSupport() {
537
+ if (sourceMapSupportInitialized) return;
538
+ sourceMapSupportInitializationPromise ??= (async () => {
539
+ try {
540
+ (await import("source-map-support")).install();
541
+ sourceMapSupportInitialized = true;
542
+ } catch {} finally {
543
+ sourceMapSupportInitializationPromise = null;
544
+ }
545
+ })();
546
+ await sourceMapSupportInitializationPromise;
547
+ }
548
+ /**
549
+ * Gets the caller location (filename and line number)
550
+ * @returns String with file location information
551
+ */
552
+ function getCallerLocation() {
553
+ try {
554
+ const callerLine = ((/* @__PURE__ */ new Error("stack trace capture")).stack?.split("\n") || [])[3];
555
+ if (callerLine) {
556
+ const matchParens = callerLine.match(/at (?:.*) \((.+):(\d+):(\d+)\)/);
557
+ const matchNormal = callerLine.match(/at (.+):(\d+):(\d+)/);
558
+ const match = matchParens || matchNormal;
559
+ if (match) {
560
+ const filePath = match[1];
561
+ const line = match[2];
562
+ return `[${path.basename(filePath)}:${line}]`;
563
+ }
564
+ }
565
+ } catch {}
566
+ return "";
567
+ }
568
+ /**
569
+ * Extracts the actual message string from potentially nested info objects
570
+ */
571
+ function extractMessage(info) {
572
+ if (typeof info.message === "object" && info.message !== null && "message" in info.message) return typeof info.message.message === "string" ? info.message.message : String(info.message.message);
573
+ return typeof info.message === "string" ? info.message : JSON.stringify(info.message);
574
+ }
575
+ const consoleFormatter = winston.format.printf((info) => {
576
+ const message = extractMessage(info);
577
+ if (globalLogCallback) globalLogCallback(message);
578
+ const location = info.location ? `${info.location} ` : "";
579
+ if (info.level === "error") return `${location}${message}`;
580
+ else if (info.level === "warn") return `${location}${message}`;
581
+ else if (info.level === "info") return `${location}${message}`;
582
+ else if (info.level === "debug") return `${location}${message}`;
583
+ throw new Error(`Invalid log level: ${info.level}`);
584
+ });
585
+ winston.format.printf((info) => {
586
+ const timestamp = (/* @__PURE__ */ new Date()).toISOString();
587
+ const location = info.location ? ` ${info.location}` : "";
588
+ const message = extractMessage(info);
589
+ return `${timestamp} [${info.level.toUpperCase()}]${location}: ${message}`;
590
+ });
591
+ const winstonLogger = winston.createLogger({
592
+ levels: LOG_LEVELS,
593
+ transports: [new winston.transports.Console({
594
+ level: getEnvString("LOG_LEVEL", "info"),
595
+ format: winston.format.combine(winston.format.simple(), consoleFormatter)
596
+ })]
597
+ });
598
+ function getLogLevel() {
599
+ return winstonLogger.transports[0].level;
600
+ }
601
+ function setLogLevel(level) {
602
+ if (level in LOG_LEVELS) {
603
+ winstonLogger.transports[0].level = level;
604
+ if (level === "debug") initializeSourceMapSupport();
605
+ } else throw new Error(`Invalid log level: ${level}`);
606
+ }
607
+ function isDebugEnabled() {
608
+ return getLogLevel() === "debug";
609
+ }
610
+ /**
611
+ * Creates a logger method for the specified log level.
612
+ * Accepts either a string message or a structured object with a message field.
613
+ */
614
+ function createLogMethod(level) {
615
+ return (input) => {
616
+ const location = level === "debug" ? getCallerLocation() : isDebugEnabled() ? getCallerLocation() : "";
617
+ if (level === "debug") initializeSourceMapSupport();
618
+ const message = typeof input === "string" ? input : input.message;
619
+ return winstonLogger[level]({
620
+ message,
621
+ location
622
+ });
623
+ };
624
+ }
625
+ let internalLogger = Object.assign({}, winstonLogger, {
626
+ error: createLogMethod("error"),
627
+ warn: createLogMethod("warn"),
628
+ info: createLogMethod("info"),
629
+ debug: createLogMethod("debug"),
630
+ add: winstonLogger.add.bind(winstonLogger),
631
+ remove: winstonLogger.remove.bind(winstonLogger),
632
+ transports: winstonLogger.transports
633
+ });
634
+ /**
635
+ * Sanitizes context object for logging using generic sanitization
636
+ */
637
+ function sanitizeContext(context) {
638
+ const contextWithSanitizedUrls = {};
639
+ for (const [key, value] of Object.entries(context)) if (key === "url" && typeof value === "string") contextWithSanitizedUrls[key] = sanitizeUrl(value);
640
+ else contextWithSanitizedUrls[key] = value;
641
+ return sanitizeObject(contextWithSanitizedUrls, { context: "log context" });
642
+ }
643
+ /**
644
+ * Creates a log method that accepts an optional context parameter.
645
+ * If context is provided, it will be sanitized and formatted.
646
+ *
647
+ * When structured logging is enabled (via setStructuredLogging(true)):
648
+ * - Passes { message, ...context } object to the logger
649
+ * - Ideal for cloud logging integrations that expect structured data
650
+ *
651
+ * When structured logging is disabled (default):
652
+ * - Formats context as JSON string appended to message
653
+ * - Suitable for CLI/console output
654
+ */
655
+ function createLogMethodWithContext(level) {
656
+ return (message, context) => {
657
+ if (!context) {
658
+ internalLogger[level](message);
659
+ return;
660
+ }
661
+ const sanitized = sanitizeContext(context);
662
+ {
663
+ const contextStr = safeJsonStringify(sanitized, true);
664
+ internalLogger[level](`${message}\n${contextStr}`);
665
+ }
666
+ };
667
+ }
668
+ const logger = {
669
+ error: createLogMethodWithContext("error"),
670
+ warn: createLogMethodWithContext("warn"),
671
+ info: createLogMethodWithContext("info"),
672
+ debug: createLogMethodWithContext("debug"),
673
+ add: (transport) => internalLogger.add ? internalLogger.add(transport) : void 0,
674
+ remove: (transport) => internalLogger.remove ? internalLogger.remove(transport) : void 0,
675
+ get transports() {
676
+ return internalLogger.transports || [];
677
+ },
678
+ get level() {
679
+ return internalLogger.transports?.[0]?.level || "info";
680
+ },
681
+ set level(newLevel) {
682
+ if (internalLogger.transports?.[0]) internalLogger.transports[0].level = newLevel;
683
+ }
684
+ };
685
+ /**
686
+ * Logs request/response details in a formatted way
687
+ * @param url - Request URL
688
+ * @param requestBody - Request body object
689
+ * @param response - Response object (optional)
690
+ * @param error - Whether to log as error (true) or debug (false)
691
+ */
692
+ async function logRequestResponse(options) {
693
+ const { url, requestBody, requestMethod, response, error } = options;
694
+ const logMethod = error ? logger.error : logger.debug;
695
+ let responseText = "";
696
+ if (response) try {
697
+ responseText = await response.clone().text();
698
+ } catch {
699
+ responseText = "Unable to read response";
700
+ }
701
+ logMethod("Api Request", {
702
+ message: "API request",
703
+ url: sanitizeUrl(url),
704
+ method: requestMethod,
705
+ requestBody: sanitizeObject(requestBody, { context: "request body" }),
706
+ ...response && {
707
+ status: response.status,
708
+ statusText: response.statusText
709
+ },
710
+ ...responseText && { response: responseText }
711
+ });
712
+ }
713
+ if (getEnvString("LOG_LEVEL", "info") === "debug") initializeSourceMapSupport();
714
+ //#endregion
715
+ export { getMaxEvalTimeMs as C, getEvalTimeoutMs as S, state as T, getConfigDirectoryPath as _, setLogLevel as a, getEnvInt as b, sanitizeObject as c, extractJsonObjects as d, getAjv as f, summarizeEvaluateResultForLogging as g, safeJsonStringify as h, setLogCallback as i, sanitizeUrl as l, orderKeys as m, logRequestResponse as n, REDACTED as o, isValidJson as p, logger as r, normalizeFieldName as s, getLogLevel as t, extractFirstJsonObject as u, getEnvBool as v, isCI as w, getEnvString as x, getEnvFloat as y };
716
+
717
+ //# sourceMappingURL=logger-C40ZGil9.js.map