anpord 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -78
- package/dist/bin.cjs +211 -204
- package/dist/bin.mjs +211 -204
- package/dist/cli-runtime.cjs +2 -2
- package/dist/cli-runtime.d.cts +1 -1
- package/dist/cli-runtime.d.mts +1 -1
- package/dist/cli-runtime.mjs +2 -2
- package/dist/client-BfdjFtlV.mjs +305 -0
- package/dist/{client-C9m0uLCz.d.cts → client-BgRs8JvW.d.mts} +2150 -174
- package/dist/{client-TFCLuMv8.d.mts → client-BirFO3jz.d.cts} +2150 -174
- package/dist/client-C-P2I00z.cjs +346 -0
- package/dist/{compiler-DXlpU_In.mjs → compiler-D96wI7UP.mjs} +108 -75
- package/dist/{compiler-Ap8DM-JU.cjs → compiler-uwb1pkdK.cjs} +116 -83
- package/dist/config.cjs +1 -1
- package/dist/config.d.cts +1 -1
- package/dist/config.d.mts +1 -1
- package/dist/config.mjs +1 -1
- package/dist/{errors-C9bQUcA3.mjs → errors-B0YknR5V.mjs} +3 -19
- package/dist/{errors-Boum34f5.cjs → errors-BX1wry8K.cjs} +3 -19
- package/dist/{errors-BK3f7Rdr.d.cts → errors-aGrd21jy.d.cts} +0 -3
- package/dist/{errors-BK3f7Rdr.d.mts → errors-aGrd21jy.d.mts} +0 -3
- package/dist/eval-judges-BkmiVLLq.cjs +59 -0
- package/dist/eval-judges-CwTvTq7N.mjs +42 -0
- package/dist/eval-judges-DPPUftbh.d.cts +48 -0
- package/dist/eval-judges-DPPUftbh.d.mts +48 -0
- package/dist/eval-validations-BRdoZrTQ.mjs +113 -0
- package/dist/eval-validations-_MlnQ_uv.cjs +148 -0
- package/dist/eval.cjs +1 -1
- package/dist/eval.d.cts +60 -6
- package/dist/eval.d.mts +60 -6
- package/dist/eval.mjs +1 -1
- package/dist/evals-B6aT3xgH.d.cts +1364 -0
- package/dist/evals-B6aT3xgH.d.mts +1364 -0
- package/dist/evals-Dcc_x6jW.mjs +582 -0
- package/dist/evals-DouMsrsM.cjs +737 -0
- package/dist/index.cjs +23 -99
- package/dist/index.d.cts +138 -34
- package/dist/index.d.mts +138 -34
- package/dist/index.mjs +23 -99
- package/dist/source.d.cts +1 -1
- package/dist/source.d.mts +1 -1
- package/dist/{types-oaMF6-E1.d.mts → types-Codjaq-o.d.mts} +146 -45
- package/dist/{types-H9L450Iu.d.cts → types-ISsteq95.d.cts} +146 -45
- package/dist/validator-runtime.cjs +238 -0
- package/dist/validator-runtime.d.cts +9 -0
- package/dist/validator-runtime.d.mts +9 -0
- package/dist/validator-runtime.mjs +237 -0
- package/dist/validators.cjs +10 -0
- package/dist/validators.d.cts +8 -0
- package/dist/validators.d.mts +8 -0
- package/dist/validators.mjs +9 -0
- package/package.json +11 -1
- package/dist/client-5K3dMI5Q.mjs +0 -849
- package/dist/client-Ci0woWLW.cjs +0 -890
- package/dist/evals-BEgEmfiO.d.cts +0 -603
- package/dist/evals-BEgEmfiO.d.mts +0 -603
- package/dist/harness-profile-BMjsN460.mjs +0 -43
- package/dist/harness-profile-CDQpPLPd.cjs +0 -78
|
@@ -0,0 +1,737 @@
|
|
|
1
|
+
const require_eval_judges = require("./eval-judges-BkmiVLLq.cjs");
|
|
2
|
+
const require_eval_validations = require("./eval-validations-_MlnQ_uv.cjs");
|
|
3
|
+
let effect = require("effect");
|
|
4
|
+
//#region ../schema/src/domain/eval-limits.ts
|
|
5
|
+
const PROMPT_LIMIT = 16384;
|
|
6
|
+
const VERIFY_LIMIT = 8192;
|
|
7
|
+
const NAME_LIMIT = 200;
|
|
8
|
+
const VARIABLE_VALUE_LIMIT = 2048;
|
|
9
|
+
const EvalPrompt = effect.Schema.String.pipe(effect.Schema.maxLength(PROMPT_LIMIT), effect.Schema.annotations({
|
|
10
|
+
description: "What the agent is asked to do.",
|
|
11
|
+
identifier: "EvalPrompt",
|
|
12
|
+
message: () => `A prompt must be at most ${PROMPT_LIMIT} characters, because it is quoted into one sandbox command line.`
|
|
13
|
+
}));
|
|
14
|
+
const EvalVerify = effect.Schema.String.pipe(effect.Schema.maxLength(VERIFY_LIMIT), effect.Schema.annotations({
|
|
15
|
+
description: "The shell command that decides whether a trial passed.",
|
|
16
|
+
identifier: "EvalVerify",
|
|
17
|
+
message: () => `A verifier must be at most ${VERIFY_LIMIT} characters, because it is quoted into one sandbox command line.`
|
|
18
|
+
}));
|
|
19
|
+
const EvalCaseName = effect.Schema.String.pipe(effect.Schema.maxLength(NAME_LIMIT), effect.Schema.annotations({
|
|
20
|
+
description: "What a case is called.",
|
|
21
|
+
identifier: "EvalCaseName",
|
|
22
|
+
message: () => `A case name must be at most ${NAME_LIMIT} characters.`
|
|
23
|
+
}));
|
|
24
|
+
const EvalVariableValue = effect.Schema.String.pipe(effect.Schema.maxLength(VARIABLE_VALUE_LIMIT), effect.Schema.annotations({
|
|
25
|
+
identifier: "EvalVariableValue",
|
|
26
|
+
message: () => `A variable value must be at most ${VARIABLE_VALUE_LIMIT} characters, because it is substituted into the prompt before it is quoted.`
|
|
27
|
+
}));
|
|
28
|
+
//#endregion
|
|
29
|
+
//#region ../schema/src/domain/eval-trigger.ts
|
|
30
|
+
const EvalTrigger = effect.Schema.Struct({
|
|
31
|
+
source: effect.Schema.Literal("dashboard", "api", "cli", "ci", "mcp"),
|
|
32
|
+
url: effect.Schema.optional(effect.Schema.String.pipe(effect.Schema.maxLength(2048), effect.Schema.filter((value) => {
|
|
33
|
+
if (!URL.canParse(value)) return false;
|
|
34
|
+
const url = new URL(value);
|
|
35
|
+
return ["https:", "http:"].includes(url.protocol) && !url.username && !url.password;
|
|
36
|
+
}, { message: () => "Use an HTTP or HTTPS URL without credentials." })))
|
|
37
|
+
}).annotations({
|
|
38
|
+
identifier: "EvalTrigger",
|
|
39
|
+
description: "What started this run. Client-reported context, not authorization."
|
|
40
|
+
});
|
|
41
|
+
//#endregion
|
|
42
|
+
//#region ../schema/src/domain/credentials.ts
|
|
43
|
+
const CredentialScope = effect.Schema.Literal("organization", "personal");
|
|
44
|
+
const CredentialStatus = effect.Schema.Literal("active", "invalid");
|
|
45
|
+
const CredentialField = effect.Schema.Struct({
|
|
46
|
+
hint: effect.Schema.optional(effect.Schema.String),
|
|
47
|
+
label: effect.Schema.String,
|
|
48
|
+
name: effect.Schema.String,
|
|
49
|
+
required: effect.Schema.Boolean,
|
|
50
|
+
secret: effect.Schema.Boolean
|
|
51
|
+
});
|
|
52
|
+
const CredentialAuthMethod = effect.Schema.Struct({
|
|
53
|
+
fields: effect.Schema.Array(CredentialField),
|
|
54
|
+
id: effect.Schema.String,
|
|
55
|
+
kind: effect.Schema.Literal("secret", "device", "env"),
|
|
56
|
+
label: effect.Schema.String
|
|
57
|
+
});
|
|
58
|
+
effect.Schema.Struct({
|
|
59
|
+
authMethods: effect.Schema.Array(CredentialAuthMethod),
|
|
60
|
+
category: effect.Schema.Literal("harness", "sandbox"),
|
|
61
|
+
id: effect.Schema.String,
|
|
62
|
+
label: effect.Schema.String
|
|
63
|
+
});
|
|
64
|
+
effect.Schema.Struct({
|
|
65
|
+
integrationId: effect.Schema.String,
|
|
66
|
+
owners: effect.Schema.Array(effect.Schema.String)
|
|
67
|
+
}).annotations({
|
|
68
|
+
description: "Who else in the organization has an integration connected.",
|
|
69
|
+
identifier: "IntegrationAwareness"
|
|
70
|
+
});
|
|
71
|
+
effect.Schema.Struct({
|
|
72
|
+
authMethodId: effect.Schema.String,
|
|
73
|
+
createdAt: effect.Schema.DateTimeUtc,
|
|
74
|
+
id: effect.Schema.String,
|
|
75
|
+
integrationId: effect.Schema.String,
|
|
76
|
+
isDefault: effect.Schema.Boolean,
|
|
77
|
+
lastUsedAt: effect.Schema.NullOr(effect.Schema.DateTimeUtc),
|
|
78
|
+
name: effect.Schema.String,
|
|
79
|
+
scope: CredentialScope,
|
|
80
|
+
status: CredentialStatus
|
|
81
|
+
});
|
|
82
|
+
const CredentialValues = effect.Schema.Record({
|
|
83
|
+
key: effect.Schema.String,
|
|
84
|
+
value: effect.Schema.String
|
|
85
|
+
});
|
|
86
|
+
effect.Schema.Struct({
|
|
87
|
+
authMethodId: effect.Schema.String,
|
|
88
|
+
connectionId: effect.Schema.String,
|
|
89
|
+
integrationId: effect.Schema.String,
|
|
90
|
+
revision: effect.Schema.Int,
|
|
91
|
+
values: CredentialValues
|
|
92
|
+
});
|
|
93
|
+
effect.Schema.Struct({
|
|
94
|
+
authMethodId: effect.Schema.String,
|
|
95
|
+
integrationId: effect.Schema.String,
|
|
96
|
+
isDefault: effect.Schema.optionalWith(effect.Schema.Boolean, { default: () => false }),
|
|
97
|
+
name: effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.maxLength(80)),
|
|
98
|
+
scope: CredentialScope,
|
|
99
|
+
values: CredentialValues
|
|
100
|
+
});
|
|
101
|
+
const CredentialBindings = effect.Schema.Struct({
|
|
102
|
+
harnessConnectionId: effect.Schema.optional(effect.Schema.String),
|
|
103
|
+
sandboxConnectionId: effect.Schema.optional(effect.Schema.String)
|
|
104
|
+
});
|
|
105
|
+
const CredentialSelections = effect.Schema.Record({
|
|
106
|
+
key: effect.Schema.String,
|
|
107
|
+
value: effect.Schema.String
|
|
108
|
+
});
|
|
109
|
+
effect.Schema.Struct({ values: CredentialValues });
|
|
110
|
+
effect.Schema.Struct({
|
|
111
|
+
integrationId: effect.Schema.Literal("codex"),
|
|
112
|
+
name: effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.maxLength(80)),
|
|
113
|
+
scope: CredentialScope
|
|
114
|
+
});
|
|
115
|
+
effect.Schema.Struct({
|
|
116
|
+
attemptId: effect.Schema.String,
|
|
117
|
+
code: effect.Schema.String,
|
|
118
|
+
expiresAt: effect.Schema.DateTimeUtc,
|
|
119
|
+
verificationUrl: effect.Schema.String
|
|
120
|
+
});
|
|
121
|
+
effect.Schema.Struct({
|
|
122
|
+
connectionId: effect.Schema.NullOr(effect.Schema.String),
|
|
123
|
+
status: effect.Schema.Literal("pending", "complete", "failed", "expired")
|
|
124
|
+
});
|
|
125
|
+
//#endregion
|
|
126
|
+
//#region ../schema/src/domain/eval-source-files.ts
|
|
127
|
+
const EvalSourceFile = effect.Schema.Struct({
|
|
128
|
+
path: effect.Schema.String.pipe(effect.Schema.maxLength(512), effect.Schema.pattern(/^(?!\.{1,2}(?:\/|$))[^/\\:\0]+(?:\/(?!\.{1,2}(?:\/|$))[^/\\:\0]+)*$/)),
|
|
129
|
+
content: effect.Schema.String.pipe(effect.Schema.maxLength(1e6))
|
|
130
|
+
});
|
|
131
|
+
const EvalSourceFiles = effect.Schema.Array(EvalSourceFile).pipe(effect.Schema.maxItems(100), effect.Schema.filter((files) => new Set(files.map(({ path }) => path)).size === files.length), effect.Schema.filter((files) => files.reduce((size, file) => size + file.content.length, 0) <= 1e6));
|
|
132
|
+
//#endregion
|
|
133
|
+
//#region ../schema/src/domain/harness-profile.ts
|
|
134
|
+
const PROFILE_LIMITS = {
|
|
135
|
+
fileChars: 96e3,
|
|
136
|
+
files: 256,
|
|
137
|
+
totalChars: 2e6
|
|
138
|
+
};
|
|
139
|
+
const ProfileName = effect.Schema.String.pipe(effect.Schema.pattern(/^[a-z0-9][a-z0-9-]{0,63}$/), effect.Schema.annotations({
|
|
140
|
+
description: "Lowercase letters, digits and hyphens, at most 64 characters, starting with a letter or digit.",
|
|
141
|
+
identifier: "ProfileName"
|
|
142
|
+
}));
|
|
143
|
+
const ProfilePath = effect.Schema.String.pipe(effect.Schema.pattern(/^(home|workspace)(?:\/(?!\.\.?(?:\/|$))[^/\0]+)+$/), effect.Schema.maxLength(512), effect.Schema.annotations({
|
|
144
|
+
description: "A relative path under home/ or workspace/, without a .. segment.",
|
|
145
|
+
identifier: "ProfilePath"
|
|
146
|
+
}));
|
|
147
|
+
const EnvName = effect.Schema.String.pipe(effect.Schema.pattern(/^[A-Z_][A-Z0-9_]*$/), effect.Schema.annotations({
|
|
148
|
+
description: "An environment variable name.",
|
|
149
|
+
identifier: "EnvName"
|
|
150
|
+
}));
|
|
151
|
+
const ProfileFiles = effect.Schema.Record({
|
|
152
|
+
key: ProfilePath,
|
|
153
|
+
value: effect.Schema.String.pipe(effect.Schema.maxLength(PROFILE_LIMITS.fileChars))
|
|
154
|
+
}).pipe(effect.Schema.filter((files) => Object.keys(files).length <= PROFILE_LIMITS.files, { message: () => `A profile holds at most ${PROFILE_LIMITS.files} files.` }), effect.Schema.filter((files) => Object.values(files).reduce((sum, content) => sum + content.length, 0) <= PROFILE_LIMITS.totalChars, { message: () => `A profile's files hold at most ${PROFILE_LIMITS.totalChars} characters in total.` }));
|
|
155
|
+
const HarnessProfile = effect.Schema.Struct({
|
|
156
|
+
env: effect.Schema.optional(effect.Schema.Record({
|
|
157
|
+
key: EnvName,
|
|
158
|
+
value: effect.Schema.String
|
|
159
|
+
})),
|
|
160
|
+
files: ProfileFiles,
|
|
161
|
+
install: effect.Schema.optional(effect.Schema.String),
|
|
162
|
+
name: ProfileName,
|
|
163
|
+
run: effect.Schema.optional(effect.Schema.String),
|
|
164
|
+
systemPrompt: effect.Schema.optional(effect.Schema.String)
|
|
165
|
+
}).annotations({
|
|
166
|
+
description: "Configuration layered on a harness: files written under the sandbox home and workspace, a system prompt, environment, an install command run before the harness, and for the command harness the run command.",
|
|
167
|
+
identifier: "HarnessProfile"
|
|
168
|
+
});
|
|
169
|
+
const profileFitsHarness = (task) => task.harness === "command" ? task.profile?.run !== void 0 : task.profile?.run === void 0;
|
|
170
|
+
const PROFILE_HARNESS_RULE = "The command harness needs a profile with a run command; other harnesses take no run. Any harness may take an install command.";
|
|
171
|
+
//#endregion
|
|
172
|
+
//#region ../schema/src/domain/evals.ts
|
|
173
|
+
const EvalHarness = require_eval_judges.EvalHarness;
|
|
174
|
+
const EvalSandbox = effect.Schema.Literal("daytona", "e2b", "upstash", "modal", "cloudflare", "vercel");
|
|
175
|
+
EvalSandbox.literals;
|
|
176
|
+
const EvalTrialStatus = effect.Schema.Literal("queued", "running", "passed", "failed", "void");
|
|
177
|
+
const EvalRunStatus = effect.Schema.Literal("running", "finished", "failed");
|
|
178
|
+
const EvalSource = effect.Schema.Union(effect.Schema.Struct({ kind: effect.Schema.Literal("empty") }), effect.Schema.Struct({
|
|
179
|
+
kind: effect.Schema.Literal("repo"),
|
|
180
|
+
ref: effect.Schema.NullOr(effect.Schema.String),
|
|
181
|
+
url: effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.annotations({ message: () => "Give the repository a URL." }))
|
|
182
|
+
}), effect.Schema.Struct({
|
|
183
|
+
files: effect.Schema.Record({
|
|
184
|
+
key: effect.Schema.String,
|
|
185
|
+
value: effect.Schema.String
|
|
186
|
+
}),
|
|
187
|
+
kind: effect.Schema.Literal("files")
|
|
188
|
+
})).annotations({
|
|
189
|
+
description: "The workspace available to the harness before setup runs.",
|
|
190
|
+
identifier: "EvalSource"
|
|
191
|
+
});
|
|
192
|
+
const EvalCodeValidator = effect.Schema.Struct({
|
|
193
|
+
capture: effect.Schema.optional(effect.Schema.Boolean),
|
|
194
|
+
manifest: effect.Schema.optional(effect.Schema.Array(effect.Schema.Struct({
|
|
195
|
+
index: effect.Schema.NonNegativeInt,
|
|
196
|
+
name: effect.Schema.String.pipe(effect.Schema.maxLength(200))
|
|
197
|
+
})).pipe(effect.Schema.minItems(1), effect.Schema.maxItems(20), effect.Schema.filter((checks) => new Set(checks.map((check) => check.index)).size === checks.length, { message: () => "Validator indices must be unique" }))),
|
|
198
|
+
name: effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.maxLength(100)),
|
|
199
|
+
source: effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.maxLength(1e6))
|
|
200
|
+
});
|
|
201
|
+
const EvalValidator = effect.Schema.Union(EvalCodeValidator.pipe(effect.Schema.extend(effect.Schema.Struct({ sourceFiles: effect.Schema.optional(EvalSourceFiles) }))), effect.Schema.Struct({
|
|
202
|
+
kind: effect.Schema.Literal("judged"),
|
|
203
|
+
capture: effect.Schema.optional(effect.Schema.Boolean),
|
|
204
|
+
name: effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.maxLength(100)),
|
|
205
|
+
checks: effect.Schema.Array(EvalCodeValidator).pipe(effect.Schema.maxItems(20)),
|
|
206
|
+
judges: effect.Schema.Array(require_eval_judges.EvalJudge).pipe(effect.Schema.minItems(1), effect.Schema.maxItems(20)),
|
|
207
|
+
sourceFiles: effect.Schema.optional(EvalSourceFiles)
|
|
208
|
+
}).pipe(effect.Schema.filter((validator) => validator.checks.reduce((total, check) => total + (check.manifest?.length ?? 1), 0) <= 20, { message: () => "At most 20 code validators are supported" }))).annotations({
|
|
209
|
+
description: "A bundled TypeScript validator and its exported function name.",
|
|
210
|
+
identifier: "EvalValidator"
|
|
211
|
+
});
|
|
212
|
+
const EvalPrepare = effect.Schema.Struct({
|
|
213
|
+
name: effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.maxLength(100)),
|
|
214
|
+
source: effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.maxLength(1e6))
|
|
215
|
+
}).annotations({
|
|
216
|
+
description: "A bundled TypeScript workspace setup and its exported function name.",
|
|
217
|
+
identifier: "EvalPrepare"
|
|
218
|
+
});
|
|
219
|
+
const EvalPrepareValue = effect.Schema.Record({
|
|
220
|
+
key: effect.Schema.String,
|
|
221
|
+
value: effect.Schema.Unknown
|
|
222
|
+
}).annotations({
|
|
223
|
+
description: "What a workspace setup returned, handed to the validator.",
|
|
224
|
+
identifier: "EvalPrepareValue"
|
|
225
|
+
});
|
|
226
|
+
const EvalVariables = effect.Schema.Record({
|
|
227
|
+
key: effect.Schema.String,
|
|
228
|
+
value: EvalVariableValue
|
|
229
|
+
}).annotations({
|
|
230
|
+
description: "Values for the placeholders the run prompt names, such as {{task}}.",
|
|
231
|
+
identifier: "EvalVariables"
|
|
232
|
+
});
|
|
233
|
+
const CaseCache = effect.Schema.Struct({
|
|
234
|
+
key: effect.Schema.String.pipe(effect.Schema.minLength(1)),
|
|
235
|
+
path: effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.filter((value) => !(value.startsWith("/") || value.split("/").includes("..")), { message: () => "a cache path must stay inside the workspace" }))
|
|
236
|
+
}).annotations({ identifier: "CaseCache" });
|
|
237
|
+
const EvalCase = effect.Schema.Struct({
|
|
238
|
+
cache: effect.Schema.optional(CaseCache),
|
|
239
|
+
name: EvalCaseName,
|
|
240
|
+
prepare: effect.Schema.NullOr(EvalPrepare),
|
|
241
|
+
source: EvalSource,
|
|
242
|
+
variables: effect.Schema.optionalWith(EvalVariables, { default: () => ({}) }),
|
|
243
|
+
validator: effect.Schema.optionalWith(effect.Schema.NullOr(EvalValidator), { default: () => null }),
|
|
244
|
+
verify: effect.Schema.NullOr(EvalVerify)
|
|
245
|
+
});
|
|
246
|
+
const EvalTaskProfile = effect.Schema.Struct({
|
|
247
|
+
name: effect.Schema.String,
|
|
248
|
+
version: effect.Schema.String
|
|
249
|
+
}).annotations({
|
|
250
|
+
description: "The profile a cell's harness ran under, by name and version.",
|
|
251
|
+
identifier: "EvalTaskProfile"
|
|
252
|
+
});
|
|
253
|
+
const EvalTask = effect.Schema.Struct({
|
|
254
|
+
harness: EvalHarness,
|
|
255
|
+
harnessVersion: effect.Schema.String,
|
|
256
|
+
model: effect.Schema.String,
|
|
257
|
+
profile: effect.Schema.optional(effect.Schema.NullOr(EvalTaskProfile)),
|
|
258
|
+
sandbox: EvalSandbox
|
|
259
|
+
}).annotations({
|
|
260
|
+
description: "The harness, installed version, profile, model, and sandbox for a cell.",
|
|
261
|
+
identifier: "EvalTask"
|
|
262
|
+
});
|
|
263
|
+
const EvalTaskRequest = effect.Schema.Struct({
|
|
264
|
+
credentials: effect.Schema.optional(CredentialBindings),
|
|
265
|
+
harness: EvalHarness,
|
|
266
|
+
model: effect.Schema.String,
|
|
267
|
+
profile: effect.Schema.optional(HarnessProfile),
|
|
268
|
+
sandbox: effect.Schema.optional(EvalSandbox)
|
|
269
|
+
}).pipe(effect.Schema.filter(profileFitsHarness, { message: () => PROFILE_HARNESS_RULE }));
|
|
270
|
+
const EvalName = effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.maxLength(100));
|
|
271
|
+
effect.Schema.Struct({
|
|
272
|
+
cases: effect.Schema.Array(EvalCase).pipe(effect.Schema.minItems(1), effect.Schema.maxItems(100)),
|
|
273
|
+
name: effect.Schema.optional(EvalName),
|
|
274
|
+
prompt: EvalPrompt,
|
|
275
|
+
tasks: effect.Schema.Array(EvalTaskRequest).pipe(effect.Schema.minItems(1), effect.Schema.maxItems(20)),
|
|
276
|
+
trials: effect.Schema.Int.pipe(effect.Schema.between(1, 10))
|
|
277
|
+
});
|
|
278
|
+
const OccurredAtMillis = effect.Schema.NullOr(effect.Schema.Number);
|
|
279
|
+
const EvalUsage = effect.Schema.Struct({
|
|
280
|
+
cacheReadTokens: effect.Schema.Int,
|
|
281
|
+
cacheWriteTokens: effect.Schema.Int,
|
|
282
|
+
costUsd: effect.Schema.optional(effect.Schema.NullOr(effect.Schema.Number)),
|
|
283
|
+
inputTokens: effect.Schema.Int,
|
|
284
|
+
outputTokens: effect.Schema.Int,
|
|
285
|
+
totalTokens: effect.Schema.Int
|
|
286
|
+
}).annotations({
|
|
287
|
+
description: "Token usage reported by the harness.",
|
|
288
|
+
identifier: "EvalUsage"
|
|
289
|
+
});
|
|
290
|
+
const EvalJournalEntry = effect.Schema.Union(effect.Schema.Struct({
|
|
291
|
+
_tag: effect.Schema.Literal("command"),
|
|
292
|
+
command: effect.Schema.String,
|
|
293
|
+
exitCode: effect.Schema.NullOr(effect.Schema.Int),
|
|
294
|
+
finishedAtMillis: OccurredAtMillis,
|
|
295
|
+
output: effect.Schema.String,
|
|
296
|
+
outputTruncated: effect.Schema.optional(effect.Schema.Boolean),
|
|
297
|
+
startedAtMillis: OccurredAtMillis
|
|
298
|
+
}), effect.Schema.Struct({
|
|
299
|
+
_tag: effect.Schema.Literal("message"),
|
|
300
|
+
finishedAtMillis: OccurredAtMillis,
|
|
301
|
+
text: effect.Schema.String,
|
|
302
|
+
usage: effect.Schema.optional(effect.Schema.NullOr(EvalUsage))
|
|
303
|
+
}), effect.Schema.Struct({
|
|
304
|
+
_tag: effect.Schema.Literal("toolCall"),
|
|
305
|
+
finishedAtMillis: OccurredAtMillis,
|
|
306
|
+
input: effect.Schema.optional(effect.Schema.String),
|
|
307
|
+
name: effect.Schema.String,
|
|
308
|
+
output: effect.Schema.optional(effect.Schema.String),
|
|
309
|
+
error: effect.Schema.optional(effect.Schema.String),
|
|
310
|
+
outputTruncated: effect.Schema.optional(effect.Schema.Boolean),
|
|
311
|
+
inputTruncated: effect.Schema.optional(effect.Schema.Boolean),
|
|
312
|
+
errorTruncated: effect.Schema.optional(effect.Schema.Boolean),
|
|
313
|
+
startedAtMillis: effect.Schema.optional(OccurredAtMillis),
|
|
314
|
+
status: effect.Schema.NullOr(effect.Schema.String)
|
|
315
|
+
}), effect.Schema.Struct({
|
|
316
|
+
_tag: effect.Schema.Literal("fileChange"),
|
|
317
|
+
finishedAtMillis: OccurredAtMillis,
|
|
318
|
+
paths: effect.Schema.Array(effect.Schema.String)
|
|
319
|
+
})).annotations({
|
|
320
|
+
description: "A normalized event recorded from the harness trajectory.",
|
|
321
|
+
identifier: "EvalJournalEntry"
|
|
322
|
+
});
|
|
323
|
+
const EvalVerifyStep = effect.Schema.Struct({
|
|
324
|
+
command: effect.Schema.String,
|
|
325
|
+
exitCode: effect.Schema.Int
|
|
326
|
+
}).annotations({
|
|
327
|
+
description: "One condition of the verifier, and how it exited. Only the steps that ran are listed: the script stops at the first failure.",
|
|
328
|
+
identifier: "EvalVerifyStep"
|
|
329
|
+
});
|
|
330
|
+
const CostClassification = effect.Schema.Literal("actual", "allocated", "estimate", "included", "managed", "unknown");
|
|
331
|
+
const CostComponentName = effect.Schema.Literal("judge", "harness", "model", "platform", "sandbox");
|
|
332
|
+
const EvalCostComponent = effect.Schema.Struct({
|
|
333
|
+
classification: CostClassification,
|
|
334
|
+
component: CostComponentName,
|
|
335
|
+
detail: effect.Schema.Record({
|
|
336
|
+
key: effect.Schema.String,
|
|
337
|
+
value: effect.Schema.Unknown
|
|
338
|
+
}),
|
|
339
|
+
explanation: effect.Schema.String,
|
|
340
|
+
source: effect.Schema.String,
|
|
341
|
+
usd: effect.Schema.NullOr(effect.Schema.Number)
|
|
342
|
+
}).annotations({
|
|
343
|
+
description: "What one layer of a trial cost, and how far that is known.",
|
|
344
|
+
identifier: "EvalCostComponent"
|
|
345
|
+
});
|
|
346
|
+
const EvalCosts = effect.Schema.Struct({
|
|
347
|
+
allocatedUsd: effect.Schema.Number,
|
|
348
|
+
components: effect.Schema.Array(EvalCostComponent),
|
|
349
|
+
estimatedEquivalentUsd: effect.Schema.Number,
|
|
350
|
+
incomplete: effect.Schema.Boolean,
|
|
351
|
+
knownActualUsd: effect.Schema.Number
|
|
352
|
+
}).annotations({
|
|
353
|
+
description: "Cost by component, kept apart by classification.",
|
|
354
|
+
identifier: "EvalCosts"
|
|
355
|
+
});
|
|
356
|
+
const EvalTrial = effect.Schema.Struct({
|
|
357
|
+
validations: effect.Schema.optional(require_eval_validations.EvalValidations),
|
|
358
|
+
judgments: effect.Schema.optional(effect.Schema.Array(require_eval_judges.EvalJudgment)),
|
|
359
|
+
commands: effect.Schema.Int,
|
|
360
|
+
costs: effect.Schema.NullOr(EvalCosts),
|
|
361
|
+
prepared: effect.Schema.NullOr(EvalPrepareValue),
|
|
362
|
+
exitCode: effect.Schema.Int,
|
|
363
|
+
failedCommands: effect.Schema.Int,
|
|
364
|
+
filesChanged: effect.Schema.Array(effect.Schema.String),
|
|
365
|
+
modelMs: effect.Schema.Int,
|
|
366
|
+
ordinal: effect.Schema.Int,
|
|
367
|
+
passed: effect.Schema.Boolean,
|
|
368
|
+
sandboxId: effect.Schema.NullOr(effect.Schema.String),
|
|
369
|
+
sandboxMs: effect.Schema.Int,
|
|
370
|
+
status: EvalTrialStatus,
|
|
371
|
+
timed: effect.Schema.Boolean,
|
|
372
|
+
trajectory: effect.Schema.Array(EvalJournalEntry),
|
|
373
|
+
usage: effect.Schema.NullOr(EvalUsage),
|
|
374
|
+
verifySteps: effect.Schema.Array(EvalVerifyStep),
|
|
375
|
+
voidFields: effect.Schema.Array(effect.Schema.String)
|
|
376
|
+
}).annotations({
|
|
377
|
+
description: "One sandbox attempt for a grid cell.",
|
|
378
|
+
identifier: "EvalTrial"
|
|
379
|
+
});
|
|
380
|
+
const EvalDistribution = effect.Schema.Struct({
|
|
381
|
+
commandMax: effect.Schema.Int,
|
|
382
|
+
commandMedian: effect.Schema.Number,
|
|
383
|
+
commandMin: effect.Schema.Int,
|
|
384
|
+
deterministic: effect.Schema.Boolean,
|
|
385
|
+
failed: effect.Schema.Int,
|
|
386
|
+
passRate: effect.Schema.Number,
|
|
387
|
+
passed: effect.Schema.Int,
|
|
388
|
+
scored: effect.Schema.Int,
|
|
389
|
+
trials: effect.Schema.Int,
|
|
390
|
+
voided: effect.Schema.Int
|
|
391
|
+
}).annotations({
|
|
392
|
+
description: "The scored outcome across all trials in a cell.",
|
|
393
|
+
identifier: "EvalDistribution"
|
|
394
|
+
});
|
|
395
|
+
const EvalVerdict = effect.Schema.Literal("improved", "incomparable", "regressed", "unchanged");
|
|
396
|
+
const EvalComparison = effect.Schema.Struct({
|
|
397
|
+
baselineHarnessVersion: effect.Schema.String,
|
|
398
|
+
baselinePassRate: effect.Schema.Number,
|
|
399
|
+
baselineProfileVersion: effect.Schema.NullOr(effect.Schema.String),
|
|
400
|
+
candidateHarnessVersion: effect.Schema.String,
|
|
401
|
+
candidatePassRate: effect.Schema.Number,
|
|
402
|
+
candidateProfileVersion: effect.Schema.NullOr(effect.Schema.String),
|
|
403
|
+
delta: effect.Schema.Number,
|
|
404
|
+
determinismLost: effect.Schema.Boolean,
|
|
405
|
+
reason: effect.Schema.NullOr(effect.Schema.String),
|
|
406
|
+
verdict: EvalVerdict
|
|
407
|
+
}).annotations({
|
|
408
|
+
description: "The cell result compared with its most recent baseline.",
|
|
409
|
+
identifier: "EvalComparison"
|
|
410
|
+
});
|
|
411
|
+
const EvalSetup = effect.Schema.Struct({
|
|
412
|
+
prompt: effect.Schema.String,
|
|
413
|
+
repoRef: effect.Schema.NullOr(effect.Schema.String),
|
|
414
|
+
repoUrl: effect.Schema.NullOr(effect.Schema.String),
|
|
415
|
+
prepareName: effect.Schema.NullOr(effect.Schema.String),
|
|
416
|
+
validatorName: effect.Schema.NullOr(effect.Schema.String),
|
|
417
|
+
validatorFiles: effect.Schema.optional(EvalSourceFiles),
|
|
418
|
+
verifyCommand: effect.Schema.NullOr(effect.Schema.String),
|
|
419
|
+
workspace: effect.Schema.String
|
|
420
|
+
}).annotations({
|
|
421
|
+
description: "The prompt, workspace, setup, and verifier used by a cell.",
|
|
422
|
+
identifier: "EvalSetup"
|
|
423
|
+
});
|
|
424
|
+
const EvalCell = effect.Schema.Struct({
|
|
425
|
+
caseName: effect.Schema.String,
|
|
426
|
+
cellKey: effect.Schema.NullOr(effect.Schema.String),
|
|
427
|
+
costs: effect.Schema.NullOr(EvalCosts),
|
|
428
|
+
comparison: effect.Schema.NullOr(EvalComparison),
|
|
429
|
+
distribution: effect.Schema.NullOr(EvalDistribution),
|
|
430
|
+
internalId: effect.Schema.NullOr(effect.Schema.String),
|
|
431
|
+
setup: effect.Schema.NullOr(EvalSetup),
|
|
432
|
+
status: EvalRunStatus,
|
|
433
|
+
taskIndex: effect.Schema.Int,
|
|
434
|
+
trials: effect.Schema.Array(EvalTrial)
|
|
435
|
+
}).annotations({
|
|
436
|
+
description: "One case and task combination in an eval grid.",
|
|
437
|
+
identifier: "EvalCell"
|
|
438
|
+
});
|
|
439
|
+
const EvalTimestamp = effect.Schema.DateTimeUtc.annotations({
|
|
440
|
+
description: "An ISO-8601 timestamp in UTC.",
|
|
441
|
+
identifier: "EvalTimestamp",
|
|
442
|
+
jsonSchema: { format: "date-time" }
|
|
443
|
+
});
|
|
444
|
+
const EvalRun = effect.Schema.Struct({
|
|
445
|
+
trigger: effect.Schema.optionalWith(effect.Schema.NullOr(EvalTrigger), { default: () => null }),
|
|
446
|
+
cases: effect.Schema.Array(effect.Schema.String),
|
|
447
|
+
cells: effect.Schema.Array(EvalCell),
|
|
448
|
+
costs: effect.Schema.NullOr(EvalCosts),
|
|
449
|
+
failure: effect.Schema.NullOr(effect.Schema.String),
|
|
450
|
+
finishedAt: effect.Schema.NullOr(EvalTimestamp),
|
|
451
|
+
id: effect.Schema.String,
|
|
452
|
+
name: effect.Schema.NullOr(EvalName),
|
|
453
|
+
startedAt: EvalTimestamp,
|
|
454
|
+
status: EvalRunStatus,
|
|
455
|
+
tasks: effect.Schema.Array(EvalTask)
|
|
456
|
+
}).annotations({
|
|
457
|
+
description: "A complete eval run with its cells and trials.",
|
|
458
|
+
identifier: "EvalRun"
|
|
459
|
+
});
|
|
460
|
+
const EvalRunSummary = effect.Schema.Struct({
|
|
461
|
+
trigger: effect.Schema.optionalWith(effect.Schema.NullOr(EvalTrigger), { default: () => null }),
|
|
462
|
+
caseCount: effect.Schema.Int,
|
|
463
|
+
columns: effect.Schema.Array(EvalTask),
|
|
464
|
+
commandMax: effect.Schema.NullOr(effect.Schema.Int),
|
|
465
|
+
commandMin: effect.Schema.NullOr(effect.Schema.Int),
|
|
466
|
+
failure: effect.Schema.NullOr(effect.Schema.String),
|
|
467
|
+
finishedAt: effect.Schema.NullOr(EvalTimestamp),
|
|
468
|
+
firstCaseName: effect.Schema.NullOr(effect.Schema.String),
|
|
469
|
+
id: effect.Schema.String,
|
|
470
|
+
name: effect.Schema.NullOr(effect.Schema.String),
|
|
471
|
+
passed: effect.Schema.Int,
|
|
472
|
+
scored: effect.Schema.Int,
|
|
473
|
+
startedAt: EvalTimestamp,
|
|
474
|
+
status: EvalRunStatus,
|
|
475
|
+
taskCount: effect.Schema.Int,
|
|
476
|
+
voided: effect.Schema.Int
|
|
477
|
+
}).annotations({
|
|
478
|
+
description: "A compact eval run returned by the run list.",
|
|
479
|
+
identifier: "EvalRunSummary"
|
|
480
|
+
});
|
|
481
|
+
const EvalPageCursor = effect.Schema.Struct({
|
|
482
|
+
id: effect.Schema.String,
|
|
483
|
+
startedAtMillis: effect.Schema.Int
|
|
484
|
+
});
|
|
485
|
+
const EvalRunPage = effect.Schema.Struct({
|
|
486
|
+
next: effect.Schema.NullOr(EvalPageCursor),
|
|
487
|
+
runs: effect.Schema.Array(EvalRunSummary),
|
|
488
|
+
total: effect.Schema.Int
|
|
489
|
+
});
|
|
490
|
+
const EvalCellHistoryEntry = effect.Schema.Struct({
|
|
491
|
+
trigger: effect.Schema.optionalWith(effect.Schema.NullOr(EvalTrigger), { default: () => null }),
|
|
492
|
+
distribution: EvalDistribution,
|
|
493
|
+
finishedAt: effect.Schema.NullOr(EvalTimestamp),
|
|
494
|
+
harnessVersion: effect.Schema.String,
|
|
495
|
+
internalId: effect.Schema.String,
|
|
496
|
+
profileVersion: effect.Schema.NullOr(effect.Schema.String),
|
|
497
|
+
runId: effect.Schema.String,
|
|
498
|
+
trials: effect.Schema.Array(EvalTrial)
|
|
499
|
+
}).annotations({
|
|
500
|
+
description: "A previous scored result for the same cell identity.",
|
|
501
|
+
identifier: "EvalCellHistoryEntry"
|
|
502
|
+
});
|
|
503
|
+
const PlaygroundCaseView = effect.Schema.Struct({
|
|
504
|
+
name: effect.Schema.String,
|
|
505
|
+
setup: effect.Schema.NullOr(effect.Schema.String),
|
|
506
|
+
source: EvalSource,
|
|
507
|
+
variables: effect.Schema.optionalWith(EvalVariables, { default: () => ({}) }),
|
|
508
|
+
verify: effect.Schema.NullOr(effect.Schema.String)
|
|
509
|
+
});
|
|
510
|
+
const PlaygroundColumnView = effect.Schema.Struct({
|
|
511
|
+
harness: EvalHarness,
|
|
512
|
+
model: effect.Schema.String,
|
|
513
|
+
sandbox: EvalSandbox
|
|
514
|
+
});
|
|
515
|
+
const PlaygroundConfigView = effect.Schema.Struct({
|
|
516
|
+
cases: effect.Schema.Array(PlaygroundCaseView),
|
|
517
|
+
columns: effect.Schema.Array(PlaygroundColumnView),
|
|
518
|
+
connections: effect.Schema.optionalWith(CredentialSelections, { default: () => ({}) }),
|
|
519
|
+
prompt: effect.Schema.String,
|
|
520
|
+
trials: effect.Schema.Int.pipe(effect.Schema.between(1, 10))
|
|
521
|
+
});
|
|
522
|
+
const EvalDraftCase = effect.Schema.Struct({
|
|
523
|
+
name: effect.Schema.String.pipe(effect.Schema.minLength(1), effect.Schema.annotations({ message: () => "Name this case." })),
|
|
524
|
+
setup: effect.Schema.NullOr(effect.Schema.String),
|
|
525
|
+
source: EvalSource,
|
|
526
|
+
variables: EvalVariables.pipe(effect.Schema.filter((values) => Object.values(values).some((value) => value.trim() !== ""), { message: () => "Say what the agent should do." })),
|
|
527
|
+
verify: effect.Schema.NullOr(effect.Schema.String)
|
|
528
|
+
});
|
|
529
|
+
const EvalAgent = effect.Schema.Struct({
|
|
530
|
+
harness: EvalHarness,
|
|
531
|
+
model: effect.Schema.String
|
|
532
|
+
});
|
|
533
|
+
effect.Schema.Struct({
|
|
534
|
+
agents: effect.Schema.mutable(effect.Schema.Array(EvalAgent)).pipe(effect.Schema.minItems(1), effect.Schema.annotations({ message: () => "Choose at least one agent." })),
|
|
535
|
+
cases: effect.Schema.mutable(effect.Schema.Array(EvalDraftCase)).pipe(effect.Schema.minItems(1), effect.Schema.annotations({ message: () => "Add at least one case." })),
|
|
536
|
+
connections: CredentialSelections,
|
|
537
|
+
name: effect.Schema.String,
|
|
538
|
+
prompt: effect.Schema.String,
|
|
539
|
+
sandboxes: effect.Schema.mutable(effect.Schema.Array(EvalSandbox)),
|
|
540
|
+
trials: effect.Schema.Int.pipe(effect.Schema.between(1, 10), effect.Schema.annotations({ message: () => "Run between 1 and 10 trials." }))
|
|
541
|
+
});
|
|
542
|
+
effect.Schema.Struct({
|
|
543
|
+
config: PlaygroundConfigView,
|
|
544
|
+
id: effect.Schema.String,
|
|
545
|
+
lastRunId: effect.Schema.NullOr(effect.Schema.String),
|
|
546
|
+
name: effect.Schema.String,
|
|
547
|
+
problems: effect.Schema.Array(effect.Schema.String),
|
|
548
|
+
ungated: effect.Schema.Array(effect.Schema.String),
|
|
549
|
+
updatedAt: effect.Schema.DateTimeUtc
|
|
550
|
+
});
|
|
551
|
+
effect.Schema.Struct({ name: effect.Schema.String.pipe(effect.Schema.minLength(1)) });
|
|
552
|
+
effect.Schema.Struct({
|
|
553
|
+
config: PlaygroundConfigView,
|
|
554
|
+
name: effect.Schema.String.pipe(effect.Schema.minLength(1))
|
|
555
|
+
});
|
|
556
|
+
const StartedEval = effect.Schema.Struct({ id: effect.Schema.String }).annotations({
|
|
557
|
+
description: "The id of an eval run accepted for background execution.",
|
|
558
|
+
identifier: "StartedEval"
|
|
559
|
+
});
|
|
560
|
+
const RerunCellRequest = effect.Schema.Struct({
|
|
561
|
+
trigger: effect.Schema.optional(EvalTrigger),
|
|
562
|
+
trials: effect.Schema.Int.pipe(effect.Schema.between(1, 10))
|
|
563
|
+
});
|
|
564
|
+
const CatalogueModel = effect.Schema.Struct({
|
|
565
|
+
displayName: effect.Schema.String,
|
|
566
|
+
id: effect.Schema.String,
|
|
567
|
+
summary: effect.Schema.NullOr(effect.Schema.String),
|
|
568
|
+
vendor: effect.Schema.NullOr(effect.Schema.String)
|
|
569
|
+
}).annotations({
|
|
570
|
+
description: "A model available to the installed harness.",
|
|
571
|
+
identifier: "CatalogueModel"
|
|
572
|
+
});
|
|
573
|
+
const ModelCatalogue = effect.Schema.Struct({
|
|
574
|
+
harness: EvalHarness,
|
|
575
|
+
models: effect.Schema.Array(CatalogueModel),
|
|
576
|
+
total: effect.Schema.Int
|
|
577
|
+
}).annotations({
|
|
578
|
+
description: "Models available to the installed harness.",
|
|
579
|
+
identifier: "ModelCatalogue"
|
|
580
|
+
});
|
|
581
|
+
//#endregion
|
|
582
|
+
Object.defineProperty(exports, "CaseCache", {
|
|
583
|
+
enumerable: true,
|
|
584
|
+
get: function() {
|
|
585
|
+
return CaseCache;
|
|
586
|
+
}
|
|
587
|
+
});
|
|
588
|
+
Object.defineProperty(exports, "EnvName", {
|
|
589
|
+
enumerable: true,
|
|
590
|
+
get: function() {
|
|
591
|
+
return EnvName;
|
|
592
|
+
}
|
|
593
|
+
});
|
|
594
|
+
Object.defineProperty(exports, "EvalCaseName", {
|
|
595
|
+
enumerable: true,
|
|
596
|
+
get: function() {
|
|
597
|
+
return EvalCaseName;
|
|
598
|
+
}
|
|
599
|
+
});
|
|
600
|
+
Object.defineProperty(exports, "EvalCellHistoryEntry", {
|
|
601
|
+
enumerable: true,
|
|
602
|
+
get: function() {
|
|
603
|
+
return EvalCellHistoryEntry;
|
|
604
|
+
}
|
|
605
|
+
});
|
|
606
|
+
Object.defineProperty(exports, "EvalHarness", {
|
|
607
|
+
enumerable: true,
|
|
608
|
+
get: function() {
|
|
609
|
+
return EvalHarness;
|
|
610
|
+
}
|
|
611
|
+
});
|
|
612
|
+
Object.defineProperty(exports, "EvalName", {
|
|
613
|
+
enumerable: true,
|
|
614
|
+
get: function() {
|
|
615
|
+
return EvalName;
|
|
616
|
+
}
|
|
617
|
+
});
|
|
618
|
+
Object.defineProperty(exports, "EvalPageCursor", {
|
|
619
|
+
enumerable: true,
|
|
620
|
+
get: function() {
|
|
621
|
+
return EvalPageCursor;
|
|
622
|
+
}
|
|
623
|
+
});
|
|
624
|
+
Object.defineProperty(exports, "EvalPrepare", {
|
|
625
|
+
enumerable: true,
|
|
626
|
+
get: function() {
|
|
627
|
+
return EvalPrepare;
|
|
628
|
+
}
|
|
629
|
+
});
|
|
630
|
+
Object.defineProperty(exports, "EvalPrompt", {
|
|
631
|
+
enumerable: true,
|
|
632
|
+
get: function() {
|
|
633
|
+
return EvalPrompt;
|
|
634
|
+
}
|
|
635
|
+
});
|
|
636
|
+
Object.defineProperty(exports, "EvalRun", {
|
|
637
|
+
enumerable: true,
|
|
638
|
+
get: function() {
|
|
639
|
+
return EvalRun;
|
|
640
|
+
}
|
|
641
|
+
});
|
|
642
|
+
Object.defineProperty(exports, "EvalRunPage", {
|
|
643
|
+
enumerable: true,
|
|
644
|
+
get: function() {
|
|
645
|
+
return EvalRunPage;
|
|
646
|
+
}
|
|
647
|
+
});
|
|
648
|
+
Object.defineProperty(exports, "EvalSandbox", {
|
|
649
|
+
enumerable: true,
|
|
650
|
+
get: function() {
|
|
651
|
+
return EvalSandbox;
|
|
652
|
+
}
|
|
653
|
+
});
|
|
654
|
+
Object.defineProperty(exports, "EvalSource", {
|
|
655
|
+
enumerable: true,
|
|
656
|
+
get: function() {
|
|
657
|
+
return EvalSource;
|
|
658
|
+
}
|
|
659
|
+
});
|
|
660
|
+
Object.defineProperty(exports, "EvalSourceFiles", {
|
|
661
|
+
enumerable: true,
|
|
662
|
+
get: function() {
|
|
663
|
+
return EvalSourceFiles;
|
|
664
|
+
}
|
|
665
|
+
});
|
|
666
|
+
Object.defineProperty(exports, "EvalTrigger", {
|
|
667
|
+
enumerable: true,
|
|
668
|
+
get: function() {
|
|
669
|
+
return EvalTrigger;
|
|
670
|
+
}
|
|
671
|
+
});
|
|
672
|
+
Object.defineProperty(exports, "EvalValidator", {
|
|
673
|
+
enumerable: true,
|
|
674
|
+
get: function() {
|
|
675
|
+
return EvalValidator;
|
|
676
|
+
}
|
|
677
|
+
});
|
|
678
|
+
Object.defineProperty(exports, "EvalVariables", {
|
|
679
|
+
enumerable: true,
|
|
680
|
+
get: function() {
|
|
681
|
+
return EvalVariables;
|
|
682
|
+
}
|
|
683
|
+
});
|
|
684
|
+
Object.defineProperty(exports, "EvalVerify", {
|
|
685
|
+
enumerable: true,
|
|
686
|
+
get: function() {
|
|
687
|
+
return EvalVerify;
|
|
688
|
+
}
|
|
689
|
+
});
|
|
690
|
+
Object.defineProperty(exports, "HarnessProfile", {
|
|
691
|
+
enumerable: true,
|
|
692
|
+
get: function() {
|
|
693
|
+
return HarnessProfile;
|
|
694
|
+
}
|
|
695
|
+
});
|
|
696
|
+
Object.defineProperty(exports, "ModelCatalogue", {
|
|
697
|
+
enumerable: true,
|
|
698
|
+
get: function() {
|
|
699
|
+
return ModelCatalogue;
|
|
700
|
+
}
|
|
701
|
+
});
|
|
702
|
+
Object.defineProperty(exports, "PROFILE_HARNESS_RULE", {
|
|
703
|
+
enumerable: true,
|
|
704
|
+
get: function() {
|
|
705
|
+
return PROFILE_HARNESS_RULE;
|
|
706
|
+
}
|
|
707
|
+
});
|
|
708
|
+
Object.defineProperty(exports, "PROFILE_LIMITS", {
|
|
709
|
+
enumerable: true,
|
|
710
|
+
get: function() {
|
|
711
|
+
return PROFILE_LIMITS;
|
|
712
|
+
}
|
|
713
|
+
});
|
|
714
|
+
Object.defineProperty(exports, "ProfilePath", {
|
|
715
|
+
enumerable: true,
|
|
716
|
+
get: function() {
|
|
717
|
+
return ProfilePath;
|
|
718
|
+
}
|
|
719
|
+
});
|
|
720
|
+
Object.defineProperty(exports, "RerunCellRequest", {
|
|
721
|
+
enumerable: true,
|
|
722
|
+
get: function() {
|
|
723
|
+
return RerunCellRequest;
|
|
724
|
+
}
|
|
725
|
+
});
|
|
726
|
+
Object.defineProperty(exports, "StartedEval", {
|
|
727
|
+
enumerable: true,
|
|
728
|
+
get: function() {
|
|
729
|
+
return StartedEval;
|
|
730
|
+
}
|
|
731
|
+
});
|
|
732
|
+
Object.defineProperty(exports, "profileFitsHarness", {
|
|
733
|
+
enumerable: true,
|
|
734
|
+
get: function() {
|
|
735
|
+
return profileFitsHarness;
|
|
736
|
+
}
|
|
737
|
+
});
|