anpord 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -78
- package/dist/bin.cjs +211 -204
- package/dist/bin.mjs +211 -204
- package/dist/cli-runtime.cjs +2 -2
- package/dist/cli-runtime.d.cts +1 -1
- package/dist/cli-runtime.d.mts +1 -1
- package/dist/cli-runtime.mjs +2 -2
- package/dist/client-BfdjFtlV.mjs +305 -0
- package/dist/{client-C9m0uLCz.d.cts → client-BgRs8JvW.d.mts} +2150 -174
- package/dist/{client-TFCLuMv8.d.mts → client-BirFO3jz.d.cts} +2150 -174
- package/dist/client-C-P2I00z.cjs +346 -0
- package/dist/{compiler-DXlpU_In.mjs → compiler-D96wI7UP.mjs} +108 -75
- package/dist/{compiler-Ap8DM-JU.cjs → compiler-uwb1pkdK.cjs} +116 -83
- package/dist/config.cjs +1 -1
- package/dist/config.d.cts +1 -1
- package/dist/config.d.mts +1 -1
- package/dist/config.mjs +1 -1
- package/dist/{errors-C9bQUcA3.mjs → errors-B0YknR5V.mjs} +3 -19
- package/dist/{errors-Boum34f5.cjs → errors-BX1wry8K.cjs} +3 -19
- package/dist/{errors-BK3f7Rdr.d.cts → errors-aGrd21jy.d.cts} +0 -3
- package/dist/{errors-BK3f7Rdr.d.mts → errors-aGrd21jy.d.mts} +0 -3
- package/dist/eval-judges-BkmiVLLq.cjs +59 -0
- package/dist/eval-judges-CwTvTq7N.mjs +42 -0
- package/dist/eval-judges-DPPUftbh.d.cts +48 -0
- package/dist/eval-judges-DPPUftbh.d.mts +48 -0
- package/dist/eval-validations-BRdoZrTQ.mjs +113 -0
- package/dist/eval-validations-_MlnQ_uv.cjs +148 -0
- package/dist/eval.cjs +1 -1
- package/dist/eval.d.cts +60 -6
- package/dist/eval.d.mts +60 -6
- package/dist/eval.mjs +1 -1
- package/dist/evals-B6aT3xgH.d.cts +1364 -0
- package/dist/evals-B6aT3xgH.d.mts +1364 -0
- package/dist/evals-Dcc_x6jW.mjs +582 -0
- package/dist/evals-DouMsrsM.cjs +737 -0
- package/dist/index.cjs +23 -99
- package/dist/index.d.cts +138 -34
- package/dist/index.d.mts +138 -34
- package/dist/index.mjs +23 -99
- package/dist/source.d.cts +1 -1
- package/dist/source.d.mts +1 -1
- package/dist/{types-oaMF6-E1.d.mts → types-Codjaq-o.d.mts} +146 -45
- package/dist/{types-H9L450Iu.d.cts → types-ISsteq95.d.cts} +146 -45
- package/dist/validator-runtime.cjs +238 -0
- package/dist/validator-runtime.d.cts +9 -0
- package/dist/validator-runtime.d.mts +9 -0
- package/dist/validator-runtime.mjs +237 -0
- package/dist/validators.cjs +10 -0
- package/dist/validators.d.cts +8 -0
- package/dist/validators.d.mts +8 -0
- package/dist/validators.mjs +9 -0
- package/package.json +11 -1
- package/dist/client-5K3dMI5Q.mjs +0 -849
- package/dist/client-Ci0woWLW.cjs +0 -890
- package/dist/evals-BEgEmfiO.d.cts +0 -603
- package/dist/evals-BEgEmfiO.d.mts +0 -603
- package/dist/harness-profile-BMjsN460.mjs +0 -43
- package/dist/harness-profile-CDQpPLPd.cjs +0 -78
package/dist/index.cjs
CHANGED
|
@@ -1,17 +1,12 @@
|
|
|
1
1
|
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
-
const require_client = require("./client-
|
|
3
|
-
const require_errors = require("./errors-
|
|
2
|
+
const require_client = require("./client-C-P2I00z.cjs");
|
|
3
|
+
const require_errors = require("./errors-BX1wry8K.cjs");
|
|
4
4
|
const require_source = require("./source-DvlGawEw.cjs");
|
|
5
5
|
let _effect_platform = require("@effect/platform");
|
|
6
6
|
let effect = require("effect");
|
|
7
7
|
//#region ../template/src/errors.ts
|
|
8
|
-
/**
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
* A plain `Error` rather than a tagged one: this package has no dependencies,
|
|
12
|
-
* and it is thrown across the published SDK boundary where callers reach for
|
|
13
|
-
* `instanceof`.
|
|
14
|
-
*/
|
|
8
|
+
/** Plain `Error`: crosses the published SDK boundary, where callers use
|
|
9
|
+
* `instanceof`. */
|
|
15
10
|
var MissingVariables = class extends Error {
|
|
16
11
|
missing;
|
|
17
12
|
constructor(missing) {
|
|
@@ -23,20 +18,12 @@ var MissingVariables = class extends Error {
|
|
|
23
18
|
};
|
|
24
19
|
//#endregion
|
|
25
20
|
//#region ../template/src/render.ts
|
|
26
|
-
/** Null and undefined read as absent
|
|
27
|
-
* "undefined", because that is what `{ name: user.name }` produces when the
|
|
28
|
-
* field is not there. */
|
|
21
|
+
/** Null and undefined read as absent, not as the strings "null"/"undefined". */
|
|
29
22
|
const suppliedValue = (values, name) => {
|
|
30
23
|
const value = values[name];
|
|
31
24
|
return value === null || value === void 0 ? void 0 : String(value);
|
|
32
25
|
};
|
|
33
|
-
/**
|
|
34
|
-
* Fills `{{name}}` from a flat map.
|
|
35
|
-
*
|
|
36
|
-
* Dots and hyphens are part of the name rather than a path: `{{user.name}}`
|
|
37
|
-
* reads the key `"user.name"`. One rule beats two, and a caller who wants
|
|
38
|
-
* nesting writes the flat key themselves.
|
|
39
|
-
*/
|
|
26
|
+
/** Flat map only: `{{user.name}}` reads the key `"user.name"`, not a path. */
|
|
40
27
|
function render(template, values, options = {}) {
|
|
41
28
|
const onMissing = options.onMissing ?? "throw";
|
|
42
29
|
const missing = [];
|
|
@@ -57,9 +44,6 @@ function render(template, values, options = {}) {
|
|
|
57
44
|
}
|
|
58
45
|
//#endregion
|
|
59
46
|
//#region src/client/cache/cache-key.ts
|
|
60
|
-
/** Structural rather than referential, so two identical selectors are one
|
|
61
|
-
* entry and one in-flight request rather than two. A plain object literal
|
|
62
|
-
* hashes by reference and every call would miss. */
|
|
63
47
|
const cacheKey = (selector) => effect.Data.struct({
|
|
64
48
|
channel: selector.channel,
|
|
65
49
|
id: selector.id,
|
|
@@ -69,25 +53,14 @@ const cacheKey = (selector) => effect.Data.struct({
|
|
|
69
53
|
//#endregion
|
|
70
54
|
//#region src/client/cache/keys.ts
|
|
71
55
|
const NAMESPACE = "prompt";
|
|
72
|
-
/**
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
* Asking for no channel is its own selector rather than a named one: the
|
|
76
|
-
* organisation decides which channel answers, it can rename or drop that
|
|
77
|
-
* channel, and naming a guess here would key the answer under a channel the
|
|
78
|
-
* server never resolved.
|
|
79
|
-
*/
|
|
56
|
+
/** No channel is its own segment, never a guessed name: the server decides
|
|
57
|
+
* which channel answers and may rename or drop it. */
|
|
80
58
|
const selectorSegment = (selector) => {
|
|
81
59
|
if (selector.version !== void 0) return `v:${selector.version}`;
|
|
82
60
|
return selector.channel === void 0 ? "default" : `c:${selector.channel}`;
|
|
83
61
|
};
|
|
84
|
-
/**
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
* `includeVersions` belongs here even though the server's key omits it: this
|
|
88
|
-
* cache holds the response rather than the resolution, and a response asked
|
|
89
|
-
* for without history has no `versions` to give to a caller who wanted it.
|
|
90
|
-
*/
|
|
62
|
+
/** `includeVersions` is part of the key even though the server's is not: this
|
|
63
|
+
* caches the response, and a bare one has no `versions` to hand back. */
|
|
91
64
|
const promptKey = (selector) => [
|
|
92
65
|
NAMESPACE,
|
|
93
66
|
encodeURIComponent(selector.id),
|
|
@@ -105,12 +78,8 @@ const make$1 = (settings, fetch) => effect.Effect.gen(function* () {
|
|
|
105
78
|
const cache = yield* effect.Cache.makeWith({
|
|
106
79
|
capacity: settings.capacity,
|
|
107
80
|
lookup: (selector) => fetch(selector),
|
|
108
|
-
/** A failure is not an answer, so it is not kept: the next call tries
|
|
109
|
-
* again rather than waiting out a whole window on one bad response. */
|
|
110
81
|
timeToLive: (exit) => effect.Exit.isFailure(exit) ? effect.Duration.zero : effect.Duration.infinity
|
|
111
82
|
});
|
|
112
|
-
/** A pinned version cannot change, so only capacity ever evicts it. A
|
|
113
|
-
* channel can be repointed at any moment. */
|
|
114
83
|
const ttlFor = (selector) => selector.version === void 0 ? settings.ttlMs : Number.POSITIVE_INFINITY;
|
|
115
84
|
const ageOf = (key) => effect.Effect.gen(function* () {
|
|
116
85
|
const stats = yield* cache.entryStats(key);
|
|
@@ -118,9 +87,6 @@ const make$1 = (settings, fetch) => effect.Effect.gen(function* () {
|
|
|
118
87
|
const now = yield* effect.Clock.currentTimeMillis;
|
|
119
88
|
return effect.Option.some(Math.max(0, now - stats.value.loadedMillis));
|
|
120
89
|
});
|
|
121
|
-
/** Forked into the cache's own scope rather than as a daemon, so a client
|
|
122
|
-
* that is disposed takes its refreshes with it. A refresh that cannot get
|
|
123
|
-
* a permit is dropped: the next call will ask again. */
|
|
124
90
|
const revalidate = (key) => permits.withPermitsIfAvailable(1)(cache.refresh(key)).pipe(effect.Effect.catchAllCause(() => effect.Effect.void), effect.Effect.forkIn(scope), effect.Effect.asVoid);
|
|
125
91
|
const heldWithin = (selector, limitMs) => effect.Effect.gen(function* () {
|
|
126
92
|
const key = cacheKey(selector);
|
|
@@ -154,11 +120,7 @@ const make$1 = (settings, fetch) => effect.Effect.gen(function* () {
|
|
|
154
120
|
const layer$1 = (settings, fetch) => effect.Layer.scoped(PromptCache, make$1(settings, fetch));
|
|
155
121
|
//#endregion
|
|
156
122
|
//#region src/client/cache/noop.ts
|
|
157
|
-
/** Disabled caching is a cache that holds nothing, so the one path through
|
|
158
|
-
* `resolvePrompt` still runs rather than being wrapped in a condition. */
|
|
159
123
|
const noopLayer = (fetch) => effect.Layer.succeed(PromptCache, {
|
|
160
|
-
/** Effects rather than bare values: yielding an Option would make an empty
|
|
161
|
-
* one a failure, and every read would end in NoSuchElementException. */
|
|
162
124
|
held: () => effect.Effect.succeedNone,
|
|
163
125
|
invalidate: () => effect.Effect.void,
|
|
164
126
|
load: fetch,
|
|
@@ -166,8 +128,8 @@ const noopLayer = (fetch) => effect.Layer.succeed(PromptCache, {
|
|
|
166
128
|
});
|
|
167
129
|
//#endregion
|
|
168
130
|
//#region src/client/cache/failure.ts
|
|
169
|
-
/**
|
|
170
|
-
*
|
|
131
|
+
/** Statuses the server chose deliberately, about the request rather than its
|
|
132
|
+
* own health. */
|
|
171
133
|
const DEFINITIVE = /* @__PURE__ */ new Set([
|
|
172
134
|
400,
|
|
173
135
|
401,
|
|
@@ -176,32 +138,16 @@ const DEFINITIVE = /* @__PURE__ */ new Set([
|
|
|
176
138
|
409,
|
|
177
139
|
422
|
|
178
140
|
]);
|
|
179
|
-
/**
|
|
180
|
-
*
|
|
181
|
-
*
|
|
182
|
-
* A 404 is a successful reply carrying real information: this prompt does not
|
|
183
|
-
* exist. Covering it with a fallback turns a typo in a prompt id into a silent
|
|
184
|
-
* permanent substitution that ships and is found weeks later. Serving stale
|
|
185
|
-
* content covers the case where a prompt someone was using is removed, and it
|
|
186
|
-
* sits ahead of the fallback anyway.
|
|
187
|
-
*
|
|
188
|
-
* An unrecognised failure counts as unavailable: a transport error carries no
|
|
189
|
-
* status at all, and guessing the other way would turn a flaky network into an
|
|
190
|
-
* outage.
|
|
191
|
-
*/
|
|
141
|
+
/** Unavailable means fall back; a definitive no must not, or a typo'd prompt
|
|
142
|
+
* id becomes a silent permanent substitution. Unrecognised counts as
|
|
143
|
+
* unavailable, since a transport error carries no status. */
|
|
192
144
|
const isAvailabilityFailure = (error) => error.status === void 0 || !DEFINITIVE.has(error.status);
|
|
193
145
|
//#endregion
|
|
194
146
|
//#region src/client/cache/fallback.ts
|
|
195
|
-
/** No version the API
|
|
196
|
-
* without being told that they are reading a fallback rather than a prompt. */
|
|
147
|
+
/** No version the API returns is zero, so a logged 0 reads as a fallback. */
|
|
197
148
|
const FALLBACK_VERSION = 0;
|
|
198
|
-
/**
|
|
199
|
-
*
|
|
200
|
-
*
|
|
201
|
-
* The branded fields are asserted rather than decoded: this value never came
|
|
202
|
-
* from the API and never goes back to it, and a fallback that could fail to
|
|
203
|
-
* parse would defeat the reason it exists.
|
|
204
|
-
*/
|
|
149
|
+
/** Branded fields are asserted, not decoded: this value never touches the API,
|
|
150
|
+
* and a fallback that could fail to parse would defeat its purpose. */
|
|
205
151
|
const fallbackPrompt = (selector, fallback, now) => {
|
|
206
152
|
const given = typeof fallback === "string" ? { content: fallback } : fallback;
|
|
207
153
|
return {
|
|
@@ -217,15 +163,8 @@ const fallbackPrompt = (selector, fallback, now) => {
|
|
|
217
163
|
};
|
|
218
164
|
//#endregion
|
|
219
165
|
//#region src/client/cache/resolve.ts
|
|
220
|
-
/**
|
|
221
|
-
*
|
|
222
|
-
*
|
|
223
|
-
* Fresh cache first, because that is the whole point. Then the API, since it
|
|
224
|
-
* is the only source that can be right. Then whatever is held however old,
|
|
225
|
-
* because a prompt someone wrote and promoted beats a string written at deploy
|
|
226
|
-
* time. Then the fallback, which the caller asked for. Then the failure, since
|
|
227
|
-
* with nothing cached and no fallback there is no honest answer to give.
|
|
228
|
-
*/
|
|
166
|
+
/** Ordered by how good the answer is: fresh cache, API, stale cache, caller's
|
|
167
|
+
* fallback, then the failure. */
|
|
229
168
|
const resolvePrompt = (options) => effect.Effect.gen(function* () {
|
|
230
169
|
const cache = yield* PromptCache;
|
|
231
170
|
const key = promptKey(options);
|
|
@@ -233,9 +172,6 @@ const resolvePrompt = (options) => effect.Effect.gen(function* () {
|
|
|
233
172
|
if (effect.Option.isSome(held)) return {
|
|
234
173
|
metadata: {
|
|
235
174
|
ageMs: held.value.ageMs,
|
|
236
|
-
/** Where the answer came from, not how old it is. An entry read in
|
|
237
|
-
* the same millisecond it was stored is still an entry, and calling
|
|
238
|
-
* it fresh made a cache hit indistinguishable from a network read. */
|
|
239
175
|
freshness: "cached",
|
|
240
176
|
key
|
|
241
177
|
},
|
|
@@ -276,23 +212,11 @@ const resolvePrompt = (options) => effect.Effect.gen(function* () {
|
|
|
276
212
|
}).pipe(effect.Effect.withSpan("Prompts.resolve", { attributes: { key: promptKey(options) } }));
|
|
277
213
|
//#endregion
|
|
278
214
|
//#region src/client/cache/settings.ts
|
|
279
|
-
/**
|
|
280
|
-
* How long a channel's answer is trusted without asking again.
|
|
281
|
-
*
|
|
282
|
-
* This is the delay between a promotion and callers seeing it, so it is short:
|
|
283
|
-
* the dashboard tells whoever promotes that callers receive the new version
|
|
284
|
-
* immediately, and a minute of drift makes that untrue exactly when someone is
|
|
285
|
-
* rolling back something bad. Serving stale while refreshing means the number
|
|
286
|
-
* costs no latency, only freshness.
|
|
287
|
-
*/
|
|
215
|
+
/** The delay between a promotion and callers seeing it, so it stays short. */
|
|
288
216
|
const TTL_MS = 15e3;
|
|
289
|
-
/**
|
|
290
|
-
*
|
|
291
|
-
* it cannot be reached at all there is no bound, since a day-old real prompt
|
|
292
|
-
* beats no prompt. */
|
|
217
|
+
/** Only bounds staleness while the API answers; an unreachable API has no
|
|
218
|
+
* bound, since a day-old real prompt beats no prompt. */
|
|
293
219
|
const MAX_STALE_MS = 864e5;
|
|
294
|
-
/** Least recently used past this. A thousand prompts is far more than one
|
|
295
|
-
* process serves and still bounds the memory a long-running server holds. */
|
|
296
220
|
const CAPACITY = 1e3;
|
|
297
221
|
/** A cold cache going stale at once must not fork a refresh per prompt. */
|
|
298
222
|
const MAX_CONCURRENT_REFRESH = 8;
|
package/dist/index.d.cts
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import { a as layer, i as DEFAULT_BASE_URL, n as AnpordClient, o as make, r as ClientOptions, t as AnpordApi } from "./client-
|
|
2
|
-
import { _ as EvalVerdict, a as EvalDistribution, b as StartedEval, c as
|
|
3
|
-
import { a as EvalTaskDefinition, c as Prepare, d as ProfileRef, f as Validator, i as EvalDefinition, l as PrepareContext, m as ValidatorResult, n as CommandResult, o as ExecOptions, p as ValidatorContext, r as EvalCaseDefinition, s as HarnessRef, t as CaseCache, u as PrepareValue } from "./types-
|
|
1
|
+
import { a as layer, i as DEFAULT_BASE_URL, n as AnpordClient, o as make, r as ClientOptions, t as AnpordApi } from "./client-BirFO3jz.cjs";
|
|
2
|
+
import { _ as EvalVerdict, a as EvalDistribution, b as StartedEval, c as EvalRun, d as EvalSandbox, f as EvalSource, g as EvalUsage, h as EvalTrialStatus, i as EvalComparison, l as EvalRunStatus, m as EvalTrial, n as EvalCell, o as EvalHarness, p as EvalTask, r as EvalCellHistoryEntry, s as EvalJournalEntry, t as CatalogueModel, u as EvalRunSummary, v as ModelCatalogue, y as RerunCellRequest } from "./evals-B6aT3xgH.cjs";
|
|
3
|
+
import { a as EvalTaskDefinition, c as Prepare, d as ProfileRef, f as Validator, i as EvalDefinition, l as PrepareContext, m as ValidatorResult, n as CommandResult, o as ExecOptions, p as ValidatorContext, r as EvalCaseDefinition, s as HarnessRef, t as CaseCache, u as PrepareValue } from "./types-ISsteq95.cjs";
|
|
4
4
|
import { t as McpCall } from "./calls-DYqBi5FA.cjs";
|
|
5
5
|
import { t as CliCall } from "./calls-Drj_dZpu.cjs";
|
|
6
6
|
import { empty, files, repo } from "./source.cjs";
|
|
7
7
|
import { Brand, Effect, Schema } from "effect";
|
|
8
8
|
import { HttpClientResponse } from "@effect/platform";
|
|
9
9
|
//#region ../template/src/render.d.ts
|
|
10
|
-
/** Objects are rejected
|
|
11
|
-
*
|
|
10
|
+
/** Objects are rejected: `[object Object]` is a bug and `JSON.stringify` a
|
|
11
|
+
* guess, so the caller decides. */
|
|
12
12
|
type VariableValue = string | number | boolean | null | undefined;
|
|
13
13
|
type Variables = Readonly<Record<string, VariableValue>>;
|
|
14
14
|
//#endregion
|
|
@@ -27,9 +27,7 @@ interface PromptSelector {
|
|
|
27
27
|
readonly includeVersions?: boolean;
|
|
28
28
|
readonly version?: number;
|
|
29
29
|
}
|
|
30
|
-
/** A string
|
|
31
|
-
* on too. Never a whole prompt: inventing a version and a timestamp for a
|
|
32
|
-
* hardcoded string is work the caller should not have to do. */
|
|
30
|
+
/** A string, or an object when there is config to fall back on too. */
|
|
33
31
|
type PromptFallback = string | {
|
|
34
32
|
readonly config?: Record<string, unknown>;
|
|
35
33
|
readonly content: string;
|
|
@@ -40,7 +38,6 @@ interface GetPromptOptions extends PromptSelector {
|
|
|
40
38
|
}
|
|
41
39
|
interface PromptMetadata {
|
|
42
40
|
readonly ageMs: number;
|
|
43
|
-
/** Why the caller received the value they received. */
|
|
44
41
|
readonly freshness: "fresh" | "cached" | "stale" | "fallback";
|
|
45
42
|
readonly key: string;
|
|
46
43
|
/** Present when the answer is stale or a fallback, naming what went wrong. */
|
|
@@ -51,43 +48,26 @@ interface PromptMetadata {
|
|
|
51
48
|
type Payload<Method> = Method extends ((request: {
|
|
52
49
|
readonly payload: infer P;
|
|
53
50
|
}) => unknown) ? P : never;
|
|
54
|
-
/** Brands are the server's business. A caller passes the string they have
|
|
55
|
-
* rather than one the schema has blessed. */
|
|
56
51
|
type Unbranded<A> = A extends string & Brand.Brand<string> ? string : A extends number & Brand.Brand<string> ? number : A extends readonly (infer Item)[] ? readonly Unbranded<Item>[] : A extends Date ? A : A extends object ? { readonly [K in keyof A]: Unbranded<A[K]>; } : A;
|
|
57
52
|
type WithoutResponse<A> = A extends readonly [unknown, HttpClientResponse.HttpClientResponse] ? never : A;
|
|
58
53
|
type Result<Method> = Method extends ((request: never) => Effect.Effect<infer A, unknown, never>) ? WithoutResponse<A> : never;
|
|
59
54
|
type Request<Method> = Unbranded<Payload<Method>>;
|
|
60
55
|
/** The Effect client as promises, so a caller who does not use Effect never
|
|
61
|
-
* meets it.
|
|
56
|
+
* meets it. */
|
|
62
57
|
type Promised<Group> = { readonly [Method in keyof Group]: Payload<Group[Method]> extends Record<string, never> ? (request?: Request<Group[Method]>) => Promise<Result<Group[Method]>> : (request: Request<Group[Method]>) => Promise<Result<Group[Method]>>; };
|
|
63
58
|
//#endregion
|
|
64
59
|
//#region src/client/variables.d.ts
|
|
65
|
-
/**
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
* Empty here on purpose. A caller who never generates passes whatever they
|
|
69
|
-
* like and everything works; a caller who does gets their own prompts named,
|
|
70
|
-
* and a misspelling becomes a compile error rather than a brace reaching a
|
|
71
|
-
* model.
|
|
72
|
-
*/
|
|
60
|
+
/** Filled in by `anpord generate`; empty here so callers who never generate
|
|
61
|
+
* stay unconstrained. */
|
|
73
62
|
interface AnpordPromptVariables {}
|
|
74
63
|
type Known = keyof AnpordPromptVariables & string;
|
|
75
|
-
/** A widened id cannot be looked up, so a
|
|
76
|
-
*
|
|
64
|
+
/** A widened id cannot be looked up, so a runtime-built one keeps the loose
|
|
65
|
+
* shape rather than being refused. */
|
|
77
66
|
type IsLiteral<Id extends string> = string extends Id ? false : true;
|
|
78
67
|
type Declared<Id extends string> = Id extends Known ? AnpordPromptVariables[Id] : never;
|
|
79
|
-
/**
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
* A prompt that had none when the file was written is not a prompt that can
|
|
83
|
-
* never have any: somebody adds one in the dashboard and a build that never
|
|
84
|
-
* touched it stops compiling. Declaring none is read as not knowing rather
|
|
85
|
-
* than as knowing there are none, so only a prompt with named variables holds
|
|
86
|
-
* a caller to a shape.
|
|
87
|
-
*/
|
|
68
|
+
/** No declared variables reads as "unknown", not "none", so a variable added
|
|
69
|
+
* in the dashboard cannot break a build that never touched the prompt. */
|
|
88
70
|
type Names<Id extends string> = Id extends Known ? Declared<Id> extends Record<string, never> ? false : true : false;
|
|
89
|
-
/** Names the template does not use, so passing one is a compile error that
|
|
90
|
-
* says which. */
|
|
91
71
|
type Unexpected<Id extends string, Given> = Exclude<keyof Given, keyof Declared<Id>>;
|
|
92
72
|
type VariablesFor<Id extends string, Given> = IsLiteral<Id> extends true ? Names<Id> extends true ? [Unexpected<Id, Given>] extends [never] ? Declared<Id> : Declared<Id> & Record<Unexpected<Id, Given>, never> : Variables : Variables;
|
|
93
73
|
//#endregion
|
|
@@ -148,6 +128,130 @@ declare class Anpord {
|
|
|
148
128
|
private resolve;
|
|
149
129
|
}
|
|
150
130
|
//#endregion
|
|
131
|
+
//#region ../schema/src/domain/eval-trigger.d.ts
|
|
132
|
+
declare const EvalTrigger: Schema.Struct<{
|
|
133
|
+
source: Schema.Literal<["dashboard", "api", "cli", "ci", "mcp"]>;
|
|
134
|
+
url: Schema.optional<Schema.filter<Schema.filter<typeof Schema.String>>>;
|
|
135
|
+
}>;
|
|
136
|
+
type EvalTrigger = typeof EvalTrigger.Type;
|
|
137
|
+
//#endregion
|
|
138
|
+
//#region ../schema/src/domain/eval-validations.d.ts
|
|
139
|
+
declare const ValidationValue: Schema.Struct<{
|
|
140
|
+
text: Schema.filter<typeof Schema.String>;
|
|
141
|
+
format: Schema.Literal<["text", "json"]>;
|
|
142
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
143
|
+
truncated: typeof Schema.Boolean;
|
|
144
|
+
}>;
|
|
145
|
+
type ValidationValue = typeof ValidationValue.Type;
|
|
146
|
+
declare const ValidationCall: Schema.Struct<{
|
|
147
|
+
index: Schema.refine<number, typeof Schema.NonNegative>;
|
|
148
|
+
method: Schema.Literal<["answer", "transcript", "readText", "exists", "exec", "cli.calls", "mcp.calls"]>;
|
|
149
|
+
startedAt: typeof Schema.Number;
|
|
150
|
+
durationMs: Schema.NullOr<Schema.refine<number, typeof Schema.NonNegative>>;
|
|
151
|
+
input: Schema.Struct<{
|
|
152
|
+
text: Schema.filter<typeof Schema.String>;
|
|
153
|
+
format: Schema.Literal<["text", "json"]>;
|
|
154
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
155
|
+
truncated: typeof Schema.Boolean;
|
|
156
|
+
}>;
|
|
157
|
+
output: Schema.Struct<{
|
|
158
|
+
text: Schema.filter<typeof Schema.String>;
|
|
159
|
+
format: Schema.Literal<["text", "json"]>;
|
|
160
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
161
|
+
truncated: typeof Schema.Boolean;
|
|
162
|
+
}>;
|
|
163
|
+
error: Schema.NullOr<Schema.Struct<{
|
|
164
|
+
text: Schema.filter<typeof Schema.String>;
|
|
165
|
+
format: Schema.Literal<["text", "json"]>;
|
|
166
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
167
|
+
truncated: typeof Schema.Boolean;
|
|
168
|
+
}>>;
|
|
169
|
+
}>;
|
|
170
|
+
type ValidationCall = typeof ValidationCall.Type;
|
|
171
|
+
declare const EvalValidation: Schema.Struct<{
|
|
172
|
+
id: Schema.filter<Schema.filter<typeof Schema.String>>;
|
|
173
|
+
index: Schema.refine<number, typeof Schema.NonNegative>;
|
|
174
|
+
name: Schema.filter<typeof Schema.String>;
|
|
175
|
+
kind: Schema.Literal<["code", "judge", "command"]>;
|
|
176
|
+
status: Schema.Literal<["queued", "running", "passed", "failed", "error", "skipped"]>;
|
|
177
|
+
startedAt: Schema.NullOr<typeof Schema.Number>;
|
|
178
|
+
durationMs: Schema.NullOr<Schema.refine<number, typeof Schema.NonNegative>>;
|
|
179
|
+
message: Schema.filter<typeof Schema.String>;
|
|
180
|
+
exitCode: Schema.NullOr<typeof Schema.Int>;
|
|
181
|
+
truncated: typeof Schema.Boolean;
|
|
182
|
+
judgment: Schema.optional<Schema.Struct<{
|
|
183
|
+
name: typeof Schema.String;
|
|
184
|
+
model: typeof Schema.String;
|
|
185
|
+
evaluator: typeof Schema.String;
|
|
186
|
+
score: Schema.NullOr<Schema.filter<typeof Schema.Number>>;
|
|
187
|
+
choice: Schema.NullOr<typeof Schema.String>;
|
|
188
|
+
reason: typeof Schema.String;
|
|
189
|
+
threshold: Schema.filter<typeof Schema.Number>;
|
|
190
|
+
durationMs: Schema.refine<number, typeof Schema.NonNegative>;
|
|
191
|
+
error: Schema.NullOr<typeof Schema.String>;
|
|
192
|
+
}>>;
|
|
193
|
+
input: Schema.Struct<{
|
|
194
|
+
text: Schema.filter<typeof Schema.String>;
|
|
195
|
+
format: Schema.Literal<["text", "json"]>;
|
|
196
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
197
|
+
truncated: typeof Schema.Boolean;
|
|
198
|
+
}>;
|
|
199
|
+
output: Schema.Struct<{
|
|
200
|
+
text: Schema.filter<typeof Schema.String>;
|
|
201
|
+
format: Schema.Literal<["text", "json"]>;
|
|
202
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
203
|
+
truncated: typeof Schema.Boolean;
|
|
204
|
+
}>;
|
|
205
|
+
error: Schema.NullOr<Schema.Struct<{
|
|
206
|
+
text: Schema.filter<typeof Schema.String>;
|
|
207
|
+
format: Schema.Literal<["text", "json"]>;
|
|
208
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
209
|
+
truncated: typeof Schema.Boolean;
|
|
210
|
+
}>>;
|
|
211
|
+
metadata: Schema.optional<Schema.Struct<{
|
|
212
|
+
text: Schema.filter<typeof Schema.String>;
|
|
213
|
+
format: Schema.Literal<["text", "json"]>;
|
|
214
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
215
|
+
truncated: typeof Schema.Boolean;
|
|
216
|
+
}>>;
|
|
217
|
+
calls: Schema.filter<Schema.Array$<Schema.Struct<{
|
|
218
|
+
index: Schema.refine<number, typeof Schema.NonNegative>;
|
|
219
|
+
method: Schema.Literal<["answer", "transcript", "readText", "exists", "exec", "cli.calls", "mcp.calls"]>;
|
|
220
|
+
startedAt: typeof Schema.Number;
|
|
221
|
+
durationMs: Schema.NullOr<Schema.refine<number, typeof Schema.NonNegative>>;
|
|
222
|
+
input: Schema.Struct<{
|
|
223
|
+
text: Schema.filter<typeof Schema.String>;
|
|
224
|
+
format: Schema.Literal<["text", "json"]>;
|
|
225
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
226
|
+
truncated: typeof Schema.Boolean;
|
|
227
|
+
}>;
|
|
228
|
+
output: Schema.Struct<{
|
|
229
|
+
text: Schema.filter<typeof Schema.String>;
|
|
230
|
+
format: Schema.Literal<["text", "json"]>;
|
|
231
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
232
|
+
truncated: typeof Schema.Boolean;
|
|
233
|
+
}>;
|
|
234
|
+
error: Schema.NullOr<Schema.Struct<{
|
|
235
|
+
text: Schema.filter<typeof Schema.String>;
|
|
236
|
+
format: Schema.Literal<["text", "json"]>;
|
|
237
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
238
|
+
truncated: typeof Schema.Boolean;
|
|
239
|
+
}>>;
|
|
240
|
+
}>>>;
|
|
241
|
+
logs: Schema.filter<Schema.Array$<Schema.Struct<{
|
|
242
|
+
index: Schema.refine<number, typeof Schema.NonNegative>;
|
|
243
|
+
at: typeof Schema.Number;
|
|
244
|
+
level: Schema.Literal<["stdout", "stderr"]>;
|
|
245
|
+
value: Schema.Struct<{
|
|
246
|
+
text: Schema.filter<typeof Schema.String>;
|
|
247
|
+
format: Schema.Literal<["text", "json"]>;
|
|
248
|
+
state: Schema.Literal<["captured", "disabled", "unavailable"]>;
|
|
249
|
+
truncated: typeof Schema.Boolean;
|
|
250
|
+
}>;
|
|
251
|
+
}>>>;
|
|
252
|
+
}>;
|
|
253
|
+
type EvalValidation = typeof EvalValidation.Type;
|
|
254
|
+
//#endregion
|
|
151
255
|
//#region ../schema/src/public/shapes.d.ts
|
|
152
256
|
declare const PublicPrompt: Schema.Struct<{
|
|
153
257
|
channel: Schema.NullOr<Schema.brand<Schema.filter<Schema.filter<Schema.filter<typeof Schema.String>>>, "ChannelName">>;
|
|
@@ -215,4 +319,4 @@ type StartEvalRequest = Parameters<EvalsSurface["start"]>[0];
|
|
|
215
319
|
type EvalCase = StartEvalRequest["cases"][number];
|
|
216
320
|
type EvalTaskRequest = StartEvalRequest["tasks"][number];
|
|
217
321
|
//#endregion
|
|
218
|
-
export { Anpord, AnpordApi, type AnpordClient, AnpordError, type AnpordOptions, type AnpordPromptVariables, type CacheOptions, type CaseCache, type CatalogueModel, type CliCall, type ClientOptions, type CommandResult, DEFAULT_BASE_URL, EvalAborted, EvalCase, type EvalCaseDefinition, type EvalCell, type EvalCellHistoryEntry, type EvalComparison, type EvalDefinition, type EvalDistribution, type EvalHarness, type EvalJournalEntry, type
|
|
322
|
+
export { Anpord, AnpordApi, type AnpordClient, AnpordError, type AnpordOptions, type AnpordPromptVariables, type CacheOptions, type CaseCache, type CatalogueModel, type CliCall, type ClientOptions, type CommandResult, DEFAULT_BASE_URL, EvalAborted, EvalCase, type EvalCaseDefinition, type EvalCell, type EvalCellHistoryEntry, type EvalComparison, type EvalDefinition, type EvalDistribution, type EvalHarness, type EvalJournalEntry, type EvalRun, type EvalRunStatus, type EvalRunSummary, type EvalSandbox, type EvalSource, type EvalTask, type EvalTaskDefinition, EvalTaskRequest, EvalTimeout, type EvalTrial, type EvalTrialStatus, type EvalTrigger, type EvalUsage, type EvalValidation, type EvalVerdict, type EvalsSurface, type ExecOptions, type GetPromptOptions, type HarnessRef, type McpCall, MissingApiKey, type ModelCatalogue, type Prepare, type PrepareContext, type PrepareValue, type ProfileRef, type PromptFallback, type PromptMetadata, type PromptResult, type PromptsSurface, type PublicPrompt, type PublicPromptSummary, type PublicPromptWithVersions, type PublicVersion, type RerunCellRequest, StartEvalRequest, type StartedEval, type ValidationCall, type ValidationValue, type Validator, type ValidatorContext, type ValidatorResult, type WaitOptions, defineEval, empty, files, layer, make, repo };
|