evals-lab 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +88 -1
- package/README.md +12 -6
- package/bin/evals-lab.js +9 -1
- package/bin/lab-file.js +97 -0
- package/bin/run.js +16 -16
- package/lab/VERSION +1 -1
- package/lab/demo/CREDITS.md +37 -3
- package/lab/demo/datasets/demo-1.json +2 -2
- package/lab/demo/datasets/demo-2.json +2 -2
- package/lab/demo/datasets/sentiment.json +383 -0
- package/lab/demo/datasets/weather-alerts.json +61 -0
- package/lab/demo/datasets/wordings.json +39 -0
- package/lab/demo/manifest.json +3 -3
- package/lab/demo/pipelines/demo-1.json +8 -6
- package/lab/demo/pipelines/demo-2.json +8 -6
- package/lab/demo/pipelines/sentiment.json +65 -0
- package/lab/demo/pipelines/weather-alerts.json +66 -0
- package/lab/demo/pipelines/wordings.json +104 -0
- package/lab/demo/sources/Demo 1/source.json +3 -0
- package/lab/demo/sources/Demo 2/source.json +3 -0
- package/lab/demo/sources/sentiment/bakery-croissant.txt +1 -0
- package/lab/demo/sources/sentiment/bus-timetable.txt +1 -0
- package/lab/demo/sources/sentiment/cafe-coffee.txt +1 -0
- package/lab/demo/sources/sentiment/concert-night.txt +1 -0
- package/lab/demo/sources/sentiment/film-sequel.txt +1 -0
- package/lab/demo/sources/sentiment/garden-tomatoes.txt +1 -0
- package/lab/demo/sources/sentiment/gym-contract.txt +1 -0
- package/lab/demo/sources/sentiment/headphones-review.txt +1 -0
- package/lab/demo/sources/sentiment/hotel-room.txt +1 -0
- package/lab/demo/sources/sentiment/library-hours.txt +1 -0
- package/lab/demo/sources/sentiment/meeting-minutes.txt +1 -0
- package/lab/demo/sources/sentiment/parcel-lost.txt +1 -0
- package/lab/demo/sources/sentiment/phone-update.txt +1 -0
- package/lab/demo/sources/sentiment/puppy-home.txt +1 -0
- package/lab/demo/sources/sentiment/recipe-note.txt +1 -0
- package/lab/demo/sources/sentiment/road-closure.txt +1 -0
- package/lab/demo/sources/sentiment/source.json +3 -0
- package/lab/demo/sources/sentiment/teacher-thanks.txt +1 -0
- package/lab/demo/sources/sentiment/train-delay.txt +1 -0
- package/lab/demo/sources/sentiment/volunteer-day.txt +1 -0
- package/lab/demo/sources/sentiment/weather-report.txt +1 -0
- package/lab/demo/sources/weather-alerts/https-api.weather.gov-alerts-types.txt +116 -0
- package/lab/demo/sources/weather-alerts/source.json +8 -0
- package/lab/evals-core.mjs +584 -167
- package/lab/flows/record.mjs +16 -4
- package/lab/kinds/list.mjs +32 -23
- package/lab/run-evals.js +3 -3
- package/lab/server.py +913 -119
- package/lab/web/dist/assets/gallery-CFntjQFG.js +5 -0
- package/lab/web/dist/assets/main-BlTG2i20.css +1 -0
- package/lab/web/dist/assets/main-D6MNWqak.js +17 -0
- package/lab/web/dist/assets/tokens-DwzzdSIF.css +1 -0
- package/lab/web/dist/assets/tokens-bvuNFuYg.js +60 -0
- package/lab/web/dist/gallery.html +3 -3
- package/lab/web/dist/index.html +4 -4
- package/package.json +10 -2
- package/lab/web/dist/assets/gallery-BcvmOPqb.js +0 -3
- package/lab/web/dist/assets/main-BBUxHZ-D.js +0 -16
- package/lab/web/dist/assets/main-CMoyjImZ.css +0 -1
- package/lab/web/dist/assets/tokens-B7N4NvmM.css +0 -1
- package/lab/web/dist/assets/tokens-CiJkwD3V.js +0 -60
package/lab/evals-core.mjs
CHANGED
|
@@ -146,8 +146,7 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
|
|
|
146
146
|
held inline, run at the step's `stage` (#432: a set has no stage of its
|
|
147
147
|
own; the step runs its filters of that stage). A job may hold one `filters` step of
|
|
148
148
|
each stage -- a `text` one run on the raw reply before it is read into
|
|
149
|
-
items (
|
|
150
|
-
as). */
|
|
149
|
+
items (Pre-format), an `items` one run on the items after (Post-format). */
|
|
151
150
|
|
|
152
151
|
|
|
153
152
|
|
|
@@ -167,13 +166,11 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
|
|
|
167
166
|
/** A target step that sends a Power Automate step's request
|
|
168
167
|
(docs/power-automate.md § The HTTP Request step): the flow's own template
|
|
169
168
|
-- method, path, query and body, Workflow Definition Language expressions
|
|
170
|
-
and all -- with the target's words and fields placed in the body by
|
|
171
|
-
HTTP API, evaluated against each item's record.
|
|
172
|
-
address and the key. */
|
|
169
|
+
and all -- with the target's words and fields placed in the body by the
|
|
170
|
+
job's HTTP API (its flow step's), evaluated against each item's record.
|
|
171
|
+
The profile gives the address and the key. */
|
|
173
172
|
|
|
174
173
|
|
|
175
|
-
|
|
176
|
-
|
|
177
174
|
|
|
178
175
|
|
|
179
176
|
|
|
@@ -221,9 +218,11 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
|
|
|
221
218
|
|
|
222
219
|
|
|
223
220
|
/** What a target's step says beside its type: what to ask in that job,
|
|
224
|
-
and whom, where it names
|
|
221
|
+
and whom, where it names an AI Connection of its own -- and, optionally,
|
|
222
|
+
a Connection Profile over it (version 16). */
|
|
225
223
|
|
|
226
224
|
|
|
225
|
+
|
|
227
226
|
|
|
228
227
|
|
|
229
228
|
|
|
@@ -239,7 +238,7 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
|
|
|
239
238
|
|
|
240
239
|
|
|
241
240
|
/** What a target sends in one job: one of the target step types. */
|
|
242
|
-
|
|
241
|
+
|
|
243
242
|
|
|
244
243
|
/** A target (pipeline-model §16): one column through every job, compared
|
|
245
244
|
side by side in Results -- what version 9 called a scenario. Its step in
|
|
@@ -249,8 +248,11 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
|
|
|
249
248
|
|
|
250
249
|
|
|
251
250
|
|
|
252
|
-
|
|
253
|
-
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
|
|
254
256
|
|
|
255
257
|
|
|
256
258
|
|
|
@@ -345,7 +347,7 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
|
|
|
345
347
|
* at. `filters` are tried in order -- each drop-item narrows the items the
|
|
346
348
|
* next reads, and the first reject-reply that fires gates the answer.
|
|
347
349
|
*/
|
|
348
|
-
/** Which side of
|
|
350
|
+
/** Which side of Response Format a filter runs on (#405): `text` the raw reply before
|
|
349
351
|
it is read into items (rewrite-text / reject-text), `items` the items after
|
|
350
352
|
(drop-item, rewrite-item, keep-first, reject-reply). A filter's type decides
|
|
351
353
|
its stage; a set belongs to none (#432), and runs its share of filters in
|
|
@@ -356,6 +358,9 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
|
|
|
356
358
|
reply; item, list and answer filters read the items it is read into. */
|
|
357
359
|
export const stageOfScope = (scope ) => (scope === "text" ? "text" : "items");
|
|
358
360
|
|
|
361
|
+
/** A filter stage as the page names it: before the reply is formatted, or after. */
|
|
362
|
+
export const FILTER_STAGE_LABEL = { text: "Pre-format", items: "Post-format" };
|
|
363
|
+
|
|
359
364
|
|
|
360
365
|
|
|
361
366
|
|
|
@@ -401,11 +406,16 @@ export const stageOfScope = (scope ) => (scope === "text" ?
|
|
|
401
406
|
|
|
402
407
|
|
|
403
408
|
|
|
404
|
-
/**
|
|
409
|
+
/** What a target asked, as a run carries it: the AI Connection and its
|
|
410
|
+
Connection Profile's settings resolved into one -- request settings,
|
|
411
|
+
never the key. */
|
|
405
412
|
|
|
406
413
|
|
|
407
|
-
|
|
414
|
+
|
|
408
415
|
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
|
|
409
419
|
|
|
410
420
|
|
|
411
421
|
|
|
@@ -420,6 +430,8 @@ export const stageOfScope = (scope ) => (scope === "text" ?
|
|
|
420
430
|
/** A pipeline resolved at submit (§5): what the queue runs and History
|
|
421
431
|
reads. The additions, and nothing else. */
|
|
422
432
|
|
|
433
|
+
|
|
434
|
+
|
|
423
435
|
|
|
424
436
|
|
|
425
437
|
|
|
@@ -455,6 +467,38 @@ export const stageOfScope = (scope ) => (scope === "text" ?
|
|
|
455
467
|
|
|
456
468
|
|
|
457
469
|
|
|
470
|
+
/**
|
|
471
|
+
* An AI Connection as Setup stores it (version 16): where the lab sends a
|
|
472
|
+
* prompt -- its type, address, key and model, and any of its type's settings
|
|
473
|
+
* that say how it is reached (`scope: "connection"`). Its key is one typed
|
|
474
|
+
* in this browser, or KEY_HELD and its id, or empty.
|
|
475
|
+
*/
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
/**
|
|
488
|
+
* A Connection Profile as Setup stores it: settings a target may carry over
|
|
489
|
+
* its connection -- model and image settings, each optional, an absent one
|
|
490
|
+
* the lab's default -- and, optionally, the connection it answers on in
|
|
491
|
+
* place of the target's. One split from a Target profile is stamped with
|
|
492
|
+
* that profile's id (`fromProfile`), which is what an upgrade finds it by.
|
|
493
|
+
*/
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
|
|
458
502
|
/** A connection's settings in one flat set, the shape a type reads. */
|
|
459
503
|
|
|
460
504
|
|
|
@@ -468,18 +512,22 @@ export const stageOfScope = (scope ) => (scope === "text" ?
|
|
|
468
512
|
/** A pipeline read from YAML, references remapped to this lab. */
|
|
469
513
|
|
|
470
514
|
|
|
471
|
-
|
|
515
|
+
|
|
472
516
|
|
|
473
517
|
|
|
474
518
|
|
|
475
519
|
|
|
476
|
-
/**
|
|
520
|
+
/** An AI Connection as an import or export matches it: its slug, where it
|
|
477
521
|
has one -- withSlugs gives the rest theirs. */
|
|
478
522
|
|
|
479
523
|
|
|
480
524
|
/** What an import matches references against: { id, name } each. */
|
|
481
525
|
|
|
526
|
+
|
|
527
|
+
|
|
482
528
|
|
|
529
|
+
|
|
530
|
+
|
|
483
531
|
|
|
484
532
|
|
|
485
533
|
|
|
@@ -487,6 +535,8 @@ export const stageOfScope = (scope ) => (scope === "text" ?
|
|
|
487
535
|
|
|
488
536
|
|
|
489
537
|
|
|
538
|
+
|
|
539
|
+
|
|
490
540
|
|
|
491
541
|
|
|
492
542
|
/** An import's outcome: the remapped pipeline, or one sentence refusing it. */
|
|
@@ -501,7 +551,7 @@ export const stageOfScope = (scope ) => (scope === "text" ?
|
|
|
501
551
|
|
|
502
552
|
|
|
503
553
|
|
|
504
|
-
|
|
554
|
+
|
|
505
555
|
|
|
506
556
|
|
|
507
557
|
|
|
@@ -576,10 +626,24 @@ export function kept(text , max = KEPT_CHARS) {
|
|
|
576
626
|
|
|
577
627
|
|
|
578
628
|
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
|
|
632
|
+
|
|
579
633
|
|
|
580
634
|
|
|
581
635
|
|
|
582
636
|
|
|
637
|
+
/** Which filter discarded a job's reply, or emptied it, and where (#435). */
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
|
|
583
647
|
/** What runPipeline is told besides the stages. */
|
|
584
648
|
|
|
585
649
|
|
|
@@ -943,6 +1007,9 @@ export function kept(text , max = KEPT_CHARS) {
|
|
|
943
1007
|
|
|
944
1008
|
|
|
945
1009
|
|
|
1010
|
+
|
|
1011
|
+
|
|
1012
|
+
|
|
946
1013
|
|
|
947
1014
|
|
|
948
1015
|
/**
|
|
@@ -1000,7 +1067,12 @@ export function kept(text , max = KEPT_CHARS) {
|
|
|
1000
1067
|
the lab holds. Each is optional, and a
|
|
1001
1068
|
reference nothing is given to look up is not checked. */
|
|
1002
1069
|
|
|
1003
|
-
|
|
1070
|
+
|
|
1071
|
+
|
|
1072
|
+
|
|
1073
|
+
|
|
1074
|
+
|
|
1075
|
+
|
|
1004
1076
|
|
|
1005
1077
|
|
|
1006
1078
|
|
|
@@ -1299,7 +1371,7 @@ const SLOTS = ["content", "target", "responses"];
|
|
|
1299
1371
|
|
|
1300
1372
|
|
|
1301
1373
|
|
|
1302
|
-
|
|
1374
|
+
|
|
1303
1375
|
|
|
1304
1376
|
|
|
1305
1377
|
|
|
@@ -1578,6 +1650,10 @@ function datasetRules(body ) {
|
|
|
1578
1650
|
|
|
1579
1651
|
|
|
1580
1652
|
|
|
1653
|
+
|
|
1654
|
+
|
|
1655
|
+
|
|
1656
|
+
|
|
1581
1657
|
|
|
1582
1658
|
|
|
1583
1659
|
|
|
@@ -1641,6 +1717,10 @@ function datasetRules(body ) {
|
|
|
1641
1717
|
|
|
1642
1718
|
|
|
1643
1719
|
|
|
1720
|
+
|
|
1721
|
+
|
|
1722
|
+
|
|
1723
|
+
|
|
1644
1724
|
|
|
1645
1725
|
|
|
1646
1726
|
|
|
@@ -2092,6 +2172,7 @@ CONNECTION_TYPES.anthropic = {
|
|
|
2092
2172
|
// Asked words (a Prompt step, a grader) or sent a flow's whole request (an
|
|
2093
2173
|
// HTTP Request step): one profile and one key do both.
|
|
2094
2174
|
answers: ["prompt", "request"],
|
|
2175
|
+
speaks: () => "anthropic",
|
|
2095
2176
|
// The Messages API: the x-api-key and anthropic-version headers, base64
|
|
2096
2177
|
// image content blocks, and /v1/models for the list.
|
|
2097
2178
|
request(conn, prompt, dataUrl, base, key) {
|
|
@@ -2417,14 +2498,23 @@ CONNECTION_TYPES.http = {
|
|
|
2417
2498
|
id: "http", label: "HTTP endpoint",
|
|
2418
2499
|
description: "Any API a Power Automate step calls: an address, a key and how the key is sent.",
|
|
2419
2500
|
settings: [
|
|
2420
|
-
{ key: "auth", label: "Key header", control: "select", default: "bearer", hint: "",
|
|
2501
|
+
{ key: "auth", label: "Key header", control: "select", default: "bearer", hint: "", scope: "connection",
|
|
2421
2502
|
choices: [{ value: "bearer", label: "Authorization: Bearer" }, { value: "x-api-key", label: "x-api-key" },
|
|
2422
2503
|
{ value: "api-key", label: "api-key" }, { value: "none", label: "none" }] },
|
|
2423
|
-
{ key: "headers", label: "Headers", control: "text", default: "", hint: "" },
|
|
2504
|
+
{ key: "headers", label: "Headers", control: "text", default: "", hint: "", scope: "connection" },
|
|
2424
2505
|
// What Test connection asks of it: a GET, keyed as a run is.
|
|
2425
|
-
{ key: "testPath", label: "Test path", control: "text", default: "", hint: "" },
|
|
2506
|
+
{ key: "testPath", label: "Test path", control: "text", default: "", hint: "", scope: "connection" },
|
|
2507
|
+
// Read off the address (httpApiFor); set by hand for one the lab cannot
|
|
2508
|
+
// tell, such as a gateway in front of a provider.
|
|
2509
|
+
{ key: "requestFormat", label: "Request format", control: "select", default: "", hint: "", scope: "connection",
|
|
2510
|
+
choices: [{ value: "", label: "From address" },
|
|
2511
|
+
...Object.entries(HTTP_APIS).map(([value, a]) => ({ value, label: a.label }))] },
|
|
2426
2512
|
],
|
|
2427
2513
|
answers: ["request"],
|
|
2514
|
+
speaks(conn) {
|
|
2515
|
+
const api = String(conn.requestFormat || "") || httpApiFor(originBase(conn.url));
|
|
2516
|
+
return HTTP_APIS[api] && api !== "raw" ? api : null;
|
|
2517
|
+
},
|
|
2428
2518
|
lists: false,
|
|
2429
2519
|
baseOf: originBase,
|
|
2430
2520
|
send(conn, req, base, key) {
|
|
@@ -3100,22 +3190,30 @@ async function runPipeline(stages , dataUrl , cal
|
|
|
3100
3190
|
// and that answer is graded as though the job had worked.
|
|
3101
3191
|
const handsOn = i < stages.length - 1;
|
|
3102
3192
|
const against = res.readAgainst ?? echo;
|
|
3103
|
-
// The
|
|
3193
|
+
// The Pre-format (text) filters shape the raw reply before the kind
|
|
3104
3194
|
// reads it (#405): a Rewrite text reshapes it, a Reject text ends the job.
|
|
3105
3195
|
const pre = applyFilterSet(st.before || [], { body: res.raw ?? "", items: [], instruction: against }, "text");
|
|
3106
|
-
|
|
3196
|
+
const by = (filter , stage , rejected ) => ({ filter, stage, job: i + 1, rejected });
|
|
3197
|
+
if (pre.reject != null) return result({ error: named(`discarded: ${pre.reject}`), filteredBy: by(pre.reject, "text", true) });
|
|
3107
3198
|
const read = reader.read(pre.body, { echo: against, parsing: opts.parsing, settings: st.settings });
|
|
3108
3199
|
if (read.error) return result({ error: named(read.error) });
|
|
3109
3200
|
const mod = applyModifiers(mods[i], kind, read.value, { instruction: against, body: read.body ?? res.raw ?? "" });
|
|
3110
|
-
if (mod.reject)
|
|
3201
|
+
if (mod.reject) {
|
|
3202
|
+
return result({ error: named(`discarded: ${mod.reject}`), ...(mod.by ? { filteredBy: by(mod.by, "items", true) } : {}) });
|
|
3203
|
+
}
|
|
3204
|
+
// A filter that dropped the last item: what a zero result is explained by.
|
|
3205
|
+
const emptied = mod.by && reader.terms && !reader.terms(mod.value).length ? { filteredBy: by(mod.by, "items", false) } : {};
|
|
3111
3206
|
const nothing = handsOn ? reader.handsOn?.(mod.value) : null;
|
|
3112
|
-
if (nothing) return result({ error: named(nothing) });
|
|
3207
|
+
if (nothing) return result({ error: named(nothing), ...emptied });
|
|
3113
3208
|
const value = mod.value;
|
|
3114
3209
|
held = { kind, value };
|
|
3115
3210
|
const gone = [...(read.dropped || []), ...mod.dropped];
|
|
3116
3211
|
echoed = (read.echoed || 0) + mod.dropped.filter(d => d.by === ECHO).length;
|
|
3117
3212
|
done.push({ reply: res.raw ?? "", rendered: reader.render?.(value) || {} });
|
|
3118
|
-
if (!handsOn)
|
|
3213
|
+
if (!handsOn) {
|
|
3214
|
+
dropped = gone;
|
|
3215
|
+
return result({ echoed, dropped, ...emptied });
|
|
3216
|
+
}
|
|
3119
3217
|
}
|
|
3120
3218
|
return result({ echoed, dropped });
|
|
3121
3219
|
}
|
|
@@ -3132,7 +3230,7 @@ function readList(reply , out , instruction =
|
|
|
3132
3230
|
const reader = OUTPUT_KINDS[out.kind];
|
|
3133
3231
|
if (!reader) return { items: [], dropped: [], echoed: 0, error: `"${out.kind}" is not an output kind this lab has` };
|
|
3134
3232
|
const { kind, modifiers, before, ...settings } = out;
|
|
3135
|
-
// The
|
|
3233
|
+
// The Pre-format (text) filters shape the raw reply before the kind
|
|
3136
3234
|
// reads it (#405); a Reject text ends the job.
|
|
3137
3235
|
const pre = applyFilterSet(before || [], { body: reply, items: [], instruction }, "text");
|
|
3138
3236
|
if (pre.reject != null) return { items: [], dropped: [], echoed: 0, error: `discarded: ${pre.reject}` };
|
|
@@ -3153,7 +3251,7 @@ function readList(reply , out , instruction =
|
|
|
3153
3251
|
was thrown away. */
|
|
3154
3252
|
function applyModifiers (list , kind , value ,
|
|
3155
3253
|
ctx = { instruction: "", body: "" }) {
|
|
3156
|
-
let v = value;
|
|
3254
|
+
let v = value, by ;
|
|
3157
3255
|
const dropped = [];
|
|
3158
3256
|
for (const m of list || []) {
|
|
3159
3257
|
const entry = MODIFIERS[m.type];
|
|
@@ -3164,9 +3262,10 @@ function applyModifiers (list , kind ,
|
|
|
3164
3262
|
if (!outcome) { v = out ; continue; }
|
|
3165
3263
|
v = outcome.value;
|
|
3166
3264
|
dropped.push(...(outcome.dropped || []));
|
|
3167
|
-
if (outcome.reject) return { value: v, dropped, reject: outcome.reject };
|
|
3265
|
+
if (outcome.reject) return { value: v, dropped, reject: outcome.reject, ...(outcome.by ? { by: outcome.by } : {}) };
|
|
3266
|
+
by = outcome.by ?? by;
|
|
3168
3267
|
}
|
|
3169
|
-
return { value: v, dropped };
|
|
3268
|
+
return { value: v, dropped, ...(by ? { by } : {}) };
|
|
3170
3269
|
}
|
|
3171
3270
|
|
|
3172
3271
|
// ---- The pipeline model ----------------------------------------------------
|
|
@@ -3205,7 +3304,12 @@ function applyModifiers (list , kind ,
|
|
|
3205
3304
|
// 12: a Contains metric's Ignore case holds item by item too.
|
|
3206
3305
|
// 13: an eval is a link to an eval group, or a group of the pipeline's own,
|
|
3207
3306
|
// and the document has an overall pass rule (pipeline-model §17).
|
|
3208
|
-
|
|
3307
|
+
// 14: a job's Responses stage links filter sets on a `filters` step (§18).
|
|
3308
|
+
// 15: every cleaning step is a filter, Pre- or Post-format (#405).
|
|
3309
|
+
// 16: a target names an AI Connection and, optionally, a Connection Profile,
|
|
3310
|
+
// where it named one Target profile; a run's table is keyed by the pair
|
|
3311
|
+
// (answerKey) -- the Target profile split in the store as it opens (#445).
|
|
3312
|
+
const PIPELINE_VERSION = 17 ;
|
|
3209
3313
|
|
|
3210
3314
|
// Plain objects, so an entry is added by assignment and a reader never needs
|
|
3211
3315
|
// to know which registered it.
|
|
@@ -3338,7 +3442,7 @@ OUTPUT_KINDS.text = {
|
|
|
3338
3442
|
// name is only a label and two profiles can share one. #46: the profile
|
|
3339
3443
|
// carries `type`, and the run's resolved copy carries it too
|
|
3340
3444
|
// (docs/pipeline-model.md §7).
|
|
3341
|
-
const CONNECTION_FIELDS = ["name", "slug", "url", "model", "type", "temperature",
|
|
3445
|
+
const CONNECTION_FIELDS = ["name", "slug", "connection", "url", "model", "type", "temperature",
|
|
3342
3446
|
"px", "format", "quality", "options"];
|
|
3343
3447
|
const SETTING_KEYS = [...new Set(Object.values(CONNECTION_TYPES).flatMap(t => t.settings.map(s => s.key)))];
|
|
3344
3448
|
const OPTION_FIELDS = ["seed", "nPredict", ...DECODING_KEYS];
|
|
@@ -3346,6 +3450,9 @@ const IMAGE_FORMATS = ["image/jpeg", "image/webp", "image/png
|
|
|
3346
3450
|
// Ids are what a key variable is spelled from, so they stay spellable: a
|
|
3347
3451
|
// letter or digit run, with at most one hyphen.
|
|
3348
3452
|
const PROFILE_ID = /^[A-Za-z0-9]+(?:-[A-Za-z0-9]+)?$/;
|
|
3453
|
+
// A run's table key (answerKey): a connection's id, then a dot and its
|
|
3454
|
+
// profile's id -- or nothing, for none -- or a grader's bare id.
|
|
3455
|
+
const ANSWER_KEY = /^[A-Za-z0-9]+(?:-[A-Za-z0-9]+)?(?:\.(?:[A-Za-z0-9]+(?:-[A-Za-z0-9]+)?)?)?$/;
|
|
3349
3456
|
const LOOKS_LIKE_A_KEY = /^(?:api[-_]?)?key$|^(?:authorization|bearer|secret|password|token)$/i;
|
|
3350
3457
|
|
|
3351
3458
|
/** The variable a profile's key is read from: its slug's, when it has one --
|
|
@@ -3441,6 +3548,138 @@ function connectionSettings(c ) {
|
|
|
3441
3548
|
return { ...rest, ...(options || {}) };
|
|
3442
3549
|
}
|
|
3443
3550
|
|
|
3551
|
+
// ---- AI Connections and Connection Profiles (version 16) ---------------------
|
|
3552
|
+
// A Target profile was one stored thing: where a prompt goes and the settings
|
|
3553
|
+
// it goes with. It is two now -- an AI Connection (type, address, key, model)
|
|
3554
|
+
// and an optional Connection Profile (settings, and optionally a connection of
|
|
3555
|
+
// its own in place of the target's) -- and a target names both. A store from
|
|
3556
|
+
// before is split once as it opens (server.py `split_profiles`): each profile
|
|
3557
|
+
// becomes a connection keeping its id and, where it set settings, a profile
|
|
3558
|
+
// stamped with that id (`fromProfile`). What a run carries of a pair is the
|
|
3559
|
+
// two resolved into one Connection, as a profile resolved before.
|
|
3560
|
+
|
|
3561
|
+
/** What an AI Connection keeps of a Target profile, beside its type's
|
|
3562
|
+
connection-scoped settings. A legacy `profile` (#46) stays with the
|
|
3563
|
+
connection: it says the type until the page converts it. */
|
|
3564
|
+
const AI_CONNECTION_FIELDS = ["id", "name", "slug", "type", "url", "key", "model", "profile"];
|
|
3565
|
+
/** What a Connection Profile holds that is not a setting. */
|
|
3566
|
+
const CONNECTION_PROFILE_FIELDS = ["id", "name", "connection", "fromProfile"];
|
|
3567
|
+
|
|
3568
|
+
/** The settings of [type] that say how a connection is reached. */
|
|
3569
|
+
const connectionScoped = (type ) =>
|
|
3570
|
+
(CONNECTION_TYPES[type]?.settings ?? []).filter(s => s.scope === "connection").map(s => s.key);
|
|
3571
|
+
|
|
3572
|
+
/** Whether a stored setting is set: blank is the lab's default. */
|
|
3573
|
+
const setValue = (v ) => v != null && v !== "";
|
|
3574
|
+
|
|
3575
|
+
/**
|
|
3576
|
+
* A Target profile as an AI Connection and, where it set any setting, the
|
|
3577
|
+
* Connection Profile holding them -- the same id, stamped with its origin.
|
|
3578
|
+
* Resolved together (`mergeAnswer`) they send what the profile sent.
|
|
3579
|
+
* server.py's `split_profile` is its twin, held to it by proxy-check.py.
|
|
3580
|
+
*/
|
|
3581
|
+
function splitProfile(p ) {
|
|
3582
|
+
const scoped = connectionScoped(typeOf(p ));
|
|
3583
|
+
const connection = {}, settings = {};
|
|
3584
|
+
for (const [k, v] of Object.entries(p)) {
|
|
3585
|
+
if (AI_CONNECTION_FIELDS.includes(k) || scoped.includes(k)) connection[k] = v;
|
|
3586
|
+
else if (setValue(v)) settings[k] = v;
|
|
3587
|
+
}
|
|
3588
|
+
const profile = Object.keys(settings).length
|
|
3589
|
+
? { id: String(p.id ?? ""), name: String(p.name ?? ""), fromProfile: String(p.id ?? ""), ...settings } : null;
|
|
3590
|
+
return { connection, profile };
|
|
3591
|
+
}
|
|
3592
|
+
|
|
3593
|
+
/** [conn] with [profile]'s settings over it: the one set of settings a
|
|
3594
|
+
request is built from, the shape a Target profile had. A profile sets
|
|
3595
|
+
no connection field. */
|
|
3596
|
+
function mergeAnswer(conn , profile ) {
|
|
3597
|
+
const out = { ...conn };
|
|
3598
|
+
const scoped = connectionScoped(typeOf(conn ));
|
|
3599
|
+
for (const [k, v] of Object.entries(profile ?? {})) {
|
|
3600
|
+
if (!CONNECTION_PROFILE_FIELDS.includes(k) && !AI_CONNECTION_FIELDS.includes(k) && !scoped.includes(k) && setValue(v)) out[k] = v;
|
|
3601
|
+
}
|
|
3602
|
+
return out;
|
|
3603
|
+
}
|
|
3604
|
+
|
|
3605
|
+
/** Whom a target or one of its steps asks: an AI Connection, and the
|
|
3606
|
+
Connection Profile over it, or none. */
|
|
3607
|
+
|
|
3608
|
+
|
|
3609
|
+
|
|
3610
|
+
|
|
3611
|
+
|
|
3612
|
+
/** The key a run's table holds [connection] under, with [profile] -- or with
|
|
3613
|
+
none, the lab's defaults. A grader names its connection bare, its id alone,
|
|
3614
|
+
so it never meets a target's pair (see `answersOf`). */
|
|
3615
|
+
const answerKey = (connection , profile ) => `${connection}.${profile ?? ""}`;
|
|
3616
|
+
|
|
3617
|
+
/** [key] read back: a pair's connection and profile, or a bare id. */
|
|
3618
|
+
function answerOfKey(key ) {
|
|
3619
|
+
const at = key.indexOf(".");
|
|
3620
|
+
if (at < 0) return { connection: key, profile: null, bare: true };
|
|
3621
|
+
return { connection: key.slice(0, at), profile: key.slice(at + 1) || null, bare: false };
|
|
3622
|
+
}
|
|
3623
|
+
|
|
3624
|
+
/** Whom target [i] asks in job [k]: its step's own connection or profile,
|
|
3625
|
+
where it names either, else the target's. */
|
|
3626
|
+
function targetAnswerOf(doc , i , k ) {
|
|
3627
|
+
const t = targetsOf(doc)[i];
|
|
3628
|
+
const st = targetStepOf(doc, i, k);
|
|
3629
|
+
const own = st?.connection != null || st?.profile != null;
|
|
3630
|
+
const connection = (own ? st?.connection ?? t?.connection : t?.connection) ?? null;
|
|
3631
|
+
if (!isRef(connection)) return undefined;
|
|
3632
|
+
const profile = (own ? st?.profile : t?.profile) ?? null;
|
|
3633
|
+
return { connection, profile: isRef(profile) ? profile : null };
|
|
3634
|
+
}
|
|
3635
|
+
|
|
3636
|
+
/** The answer key of target [i] in job [k], or undefined where it asks none. */
|
|
3637
|
+
function targetAnswerKey(doc , i , k ) {
|
|
3638
|
+
const a = targetAnswerOf(doc, i, k);
|
|
3639
|
+
return a ? answerKey(a.connection.id, a.profile?.id) : undefined;
|
|
3640
|
+
}
|
|
3641
|
+
|
|
3642
|
+
/**
|
|
3643
|
+
* What the lab's two stores answer: each by id, the profile split from a
|
|
3644
|
+
* Target profile's id (what an upgrade and a grader read it by), and an
|
|
3645
|
+
* answer key resolved to the settings it sends -- the profile's connection
|
|
3646
|
+
* where it names one, else the key's, with the profile's settings over it.
|
|
3647
|
+
* A grader's bare id resolves as the Target profile it named did: its
|
|
3648
|
+
* connection under the profile split from it. Null where any part is gone.
|
|
3649
|
+
*/
|
|
3650
|
+
function answersOf(connections , profiles ) {
|
|
3651
|
+
const connection = (id ) => connections.find(c => c?.id === id) ?? null;
|
|
3652
|
+
const profile = (id ) => profiles.find(p => p?.id === id) ?? null;
|
|
3653
|
+
const split = (id ) => {
|
|
3654
|
+
const p = profiles.find(x => x?.fromProfile === id);
|
|
3655
|
+
return p ? { id: p.id, name: p.name } : null;
|
|
3656
|
+
};
|
|
3657
|
+
const resolve = (key ) => {
|
|
3658
|
+
const at = answerOfKey(key);
|
|
3659
|
+
const pid = at.bare ? split(at.connection)?.id ?? null : at.profile;
|
|
3660
|
+
const p = pid ? profile(pid) : null;
|
|
3661
|
+
if (pid && !p) return null;
|
|
3662
|
+
const c = connection(isRef(p?.connection) ? p .connection .id : at.connection);
|
|
3663
|
+
return c ? mergeAnswer(c, p) : null;
|
|
3664
|
+
};
|
|
3665
|
+
return { connection, profile, split, resolve };
|
|
3666
|
+
}
|
|
3667
|
+
|
|
3668
|
+
/** The connection a run carries for [key]: its own entry, or -- a run from
|
|
3669
|
+
before version 16, whose table is keyed by Target profile id, and names
|
|
3670
|
+
the connection alone once upgraded -- the one under the connection's id. */
|
|
3671
|
+
function runAnswer(run , key ) {
|
|
3672
|
+
const table = run?.profiles ;
|
|
3673
|
+
if (!isObj(table)) return undefined;
|
|
3674
|
+
if (Object.hasOwn(table, key)) return table[key];
|
|
3675
|
+
const at = answerOfKey(key);
|
|
3676
|
+
return !at.bare && at.profile == null && Object.hasOwn(table, at.connection) ? table[at.connection] : undefined;
|
|
3677
|
+
}
|
|
3678
|
+
|
|
3679
|
+
/** The id of the AI Connection whose key a run's entry [key] is sent with. */
|
|
3680
|
+
const answerConnectionId = (key , conn ) =>
|
|
3681
|
+
(isStr(conn?.connection) && conn .connection) || answerOfKey(key).connection;
|
|
3682
|
+
|
|
3444
3683
|
// The rest of what a connection's fields mean lives in the connection-type
|
|
3445
3684
|
// registry above (#46): the runner asks a type how to build a request and how
|
|
3446
3685
|
// to read a reply, and validation asks connectionProblems (below, with the run
|
|
@@ -3721,7 +3960,7 @@ DYNAMIC_PROVIDERS["command"] = { // provider: the Command adapter's own id
|
|
|
3721
3960
|
holds, so a tick costs a predicate and not a request. A reader names a field
|
|
3722
3961
|
it needs; a wizard that reads more declares it here. */
|
|
3723
3962
|
|
|
3724
|
-
|
|
3963
|
+
|
|
3725
3964
|
|
|
3726
3965
|
|
|
3727
3966
|
|
|
@@ -4091,14 +4330,14 @@ LEGACY_TESTS.metrics = {
|
|
|
4091
4330
|
if (t.grader != null && !isRef(t.grader)) bad.push("the Metrics name their grader as { id, name }");
|
|
4092
4331
|
// The eval's own grader, or the lab's.
|
|
4093
4332
|
const grader = isRef(t.grader) ? t.grader : isRef(ctx.grader) ? ctx.grader : null;
|
|
4094
|
-
if (gradedIn(t.metrics) && !grader) bad.push("a model-graded metric needs a grader: pick one on the eval, or make a
|
|
4333
|
+
if (gradedIn(t.metrics) && !grader) bad.push("a model-graded metric needs a grader: pick one on the eval, or make a connection the lab's grader");
|
|
4095
4334
|
// A grader is asked words, and needs a model to ask.
|
|
4096
4335
|
const p = grader && ctx.profiles ? ctx.profiles(grader.id) : null;
|
|
4097
|
-
if (grader && ctx.profiles && !p) bad.push(`
|
|
4336
|
+
if (grader && ctx.profiles && !p) bad.push(`Connection ${grader.name || grader.id} not found`);
|
|
4098
4337
|
else if (p) {
|
|
4099
4338
|
const type = CONNECTION_TYPES[typeOf(p )];
|
|
4100
4339
|
if (type && !(type.answers ?? ["prompt"]).includes("prompt")) bad.push(`the grader has to be asked words, and ${type.label} is not`);
|
|
4101
|
-
else if (!type?.local && !String((p ).model || "").trim()) bad.push("no model on the grader's
|
|
4340
|
+
else if (!type?.local && !String((p ).model || "").trim()) bad.push("no model on the grader's connection");
|
|
4102
4341
|
}
|
|
4103
4342
|
},
|
|
4104
4343
|
rules: t => (Array.isArray(t.metrics) ? t.metrics : []).map((m , x ) => ({
|
|
@@ -4226,14 +4465,14 @@ EVAL_TYPES.group = {
|
|
|
4226
4465
|
if (own.run.length && !wholeRunGroup(t)) bad.push("the group's own body reads each item or the whole run, not both");
|
|
4227
4466
|
// The group's own grader, or the lab's.
|
|
4228
4467
|
const grader = isRef(own.grader) ? own.grader : isRef(ctx.grader) ? ctx.grader : null;
|
|
4229
|
-
if (gradedIn(own.every) && !grader) bad.push("a model-graded metric needs a grader: pick one on the eval, or make a
|
|
4468
|
+
if (gradedIn(own.every) && !grader) bad.push("a model-graded metric needs a grader: pick one on the eval, or make a connection the lab's grader");
|
|
4230
4469
|
// A grader is asked words, and needs a model to ask.
|
|
4231
4470
|
const p = grader && ctx.profiles ? ctx.profiles(grader.id) : null;
|
|
4232
|
-
if (grader && ctx.profiles && !p) bad.push(`
|
|
4471
|
+
if (grader && ctx.profiles && !p) bad.push(`Connection ${grader.name || grader.id} not found`);
|
|
4233
4472
|
else if (p) {
|
|
4234
4473
|
const type = CONNECTION_TYPES[typeOf(p )];
|
|
4235
4474
|
if (type && !(type.answers ?? ["prompt"]).includes("prompt")) bad.push(`the grader has to be asked words, and ${type.label} is not`);
|
|
4236
|
-
else if (!type?.local && !String((p ).model || "").trim()) bad.push("no model on the grader's
|
|
4475
|
+
else if (!type?.local && !String((p ).model || "").trim()) bad.push("no model on the grader's connection");
|
|
4237
4476
|
}
|
|
4238
4477
|
},
|
|
4239
4478
|
rules: t => ownList(t).map((m , x ) => ({
|
|
@@ -4565,7 +4804,7 @@ STEP_TYPES.prompt = {
|
|
|
4565
4804
|
description: "Asks each target's model its prompt about the item.",
|
|
4566
4805
|
in: "item", out: "text",
|
|
4567
4806
|
apply: "runPipeline",
|
|
4568
|
-
fields: ["type", "prompt", "profile", "from"],
|
|
4807
|
+
fields: ["type", "prompt", "connection", "profile", "from"],
|
|
4569
4808
|
validate(step, _ctx, bad, at){
|
|
4570
4809
|
if (!isStr(step.prompt)) bad.push(`${at}: prompt has to be text`);
|
|
4571
4810
|
},
|
|
@@ -4591,16 +4830,18 @@ STEP_TYPES.httpRequest = {
|
|
|
4591
4830
|
label: "HTTP Request", slot: "target", asks: "request", verbatim: true, modelFrom: "step",
|
|
4592
4831
|
description: "Sends a Power Automate step's own request, rebuilt from each record, with each target's words in it.",
|
|
4593
4832
|
firstJobOnly: "an HTTP Request is sent from its item's record, so it is job 1's",
|
|
4594
|
-
// A new target starts from the flow's own words and fields
|
|
4595
|
-
|
|
4833
|
+
// A new target starts from the flow's own words and fields, read by the
|
|
4834
|
+
// job's HTTP API ([step] carries it: requestStepOf).
|
|
4835
|
+
newStep: (step) => ({ type: "httpRequest", method: step.method, path: step.path,
|
|
4596
4836
|
query: clone(step.query), body: clone(step.body),
|
|
4597
4837
|
...(HTTP_APIS[step.api] ?? HTTP_APIS.raw ).cellOf(step.body) }),
|
|
4598
4838
|
in: "item", out: "text",
|
|
4599
4839
|
apply: "runPipeline",
|
|
4600
|
-
fields: ["type", "
|
|
4840
|
+
fields: ["type", "method", "path", "query", "body", "prompt", "connection", "profile", "from", "system", "model", "settings", "prefill"],
|
|
4841
|
+
// [step] carries its job's HTTP API, where the job has a flow step to say
|
|
4842
|
+
// it (validatePipeline); the flow step's own check names an unknown one.
|
|
4601
4843
|
validate(step, _ctx, bad, at){
|
|
4602
4844
|
const api = HTTP_APIS[step.api];
|
|
4603
|
-
if (!api) bad.push(`${at}: "${step.api}" is not an HTTP API this lab has`);
|
|
4604
4845
|
if (!HTTP_METHODS.includes(step.method)) bad.push(`${at}: the method is one of ${HTTP_METHODS.join(", ")}`);
|
|
4605
4846
|
if (!isStr(step.path) || !/^[/@]/.test(step.path)) bad.push(`${at}: the path starts with /`);
|
|
4606
4847
|
if (!isObj(step.query) || !Object.values(step.query).every(isStr)) bad.push(`${at}: the query is names to text`);
|
|
@@ -4608,7 +4849,7 @@ STEP_TYPES.httpRequest = {
|
|
|
4608
4849
|
// A key is the Setup profile's, sent by its Key header; one in the
|
|
4609
4850
|
// template would be kept in the pipeline and every run of it.
|
|
4610
4851
|
const hit = keyField(step.body) ?? keyField(step.query, "query");
|
|
4611
|
-
if (hit) bad.push(`${at} carries a key at ${hit}, and a key never goes in a pipeline -- the
|
|
4852
|
+
if (hit) bad.push(`${at} carries a key at ${hit}, and a key never goes in a pipeline -- the connection sends it`);
|
|
4612
4853
|
if (!api) return;
|
|
4613
4854
|
// Its fields, where its HTTP API has a place for them.
|
|
4614
4855
|
for (const f of ["system", "model", "prefill"] ) {
|
|
@@ -4723,7 +4964,7 @@ STEP_TYPES.modifier = {
|
|
|
4723
4964
|
|
|
4724
4965
|
// A filters step: the filter sets a job cleans and gates its reply by, each a
|
|
4725
4966
|
// link or a private set (#339, #405), at the step's `stage`. A job may hold
|
|
4726
|
-
// one of each stage -- a `text` one
|
|
4967
|
+
// one of each stage -- a `text` one Pre-format, an `items` one Post-format -- so
|
|
4727
4968
|
// the type is `many`; two of a stage is the job's to refuse. A set belongs to
|
|
4728
4969
|
// no stage (#432): linked here, it runs the filters whose type fits the step's
|
|
4729
4970
|
// stage and skips the rest. A private set's body is the core's to judge
|
|
@@ -4824,11 +5065,6 @@ function jobTargetType(doc
|
|
|
4824
5065
|
return targetStepOf(doc, 0, k)?.type ?? "prompt";
|
|
4825
5066
|
}
|
|
4826
5067
|
|
|
4827
|
-
/** The Setup profile target [i] asks in job [k]: its step's own, or the target's. */
|
|
4828
|
-
function targetProfileOf(doc , i , k ) {
|
|
4829
|
-
return targetStepOf(doc, i, k)?.profile ?? targetsOf(doc)[i]?.profile ?? undefined;
|
|
4830
|
-
}
|
|
4831
|
-
|
|
4832
5068
|
/** Whether target [i] is a Recorded target: it replays each call's recorded
|
|
4833
5069
|
reply rather than sending anything (its job-1 step answers locally with a
|
|
4834
5070
|
connection that `replays`). A recorded-reply metric compares the recorded
|
|
@@ -4882,7 +5118,7 @@ function requestStepOf(doc
|
|
|
4882
5118
|
|
|
4883
5119
|
/** What httpRequestOf builds a request from: a target's template, and the
|
|
4884
5120
|
loop its flow step sits in. */
|
|
4885
|
-
|
|
5121
|
+
|
|
4886
5122
|
/** What httpReplyOf reads a reply with: the flow step, its API, its reading. */
|
|
4887
5123
|
|
|
4888
5124
|
|
|
@@ -4958,8 +5194,8 @@ function withOut(job , out ) {
|
|
|
4958
5194
|
}
|
|
4959
5195
|
|
|
4960
5196
|
/** [job] reading a reply as `legacyTagsOut` builds it, on the current model
|
|
4961
|
-
(#405): its output a List, a
|
|
4962
|
-
the old `preamble` did, and
|
|
5197
|
+
(#405): its output a List, a Pre-format text set that reads the reply as
|
|
5198
|
+
the old `preamble` did, and a Post-format items set that cleans and gates
|
|
4963
5199
|
it. The filters are `legacyTagsOut`'s, placed as private sets -- what the
|
|
4964
5200
|
version-15 upgrade builds for a legacy list job, for a test to build
|
|
4965
5201
|
directly. */
|
|
@@ -4978,7 +5214,7 @@ const filterStepOf = (job , stage )
|
|
|
4978
5214
|
|
|
4979
5215
|
/** The filter-set links a job's Responses stage carries for [stage], in
|
|
4980
5216
|
document order (#339, #405): read from that stage's filters step, or none.
|
|
4981
|
-
Defaults to the
|
|
5217
|
+
Defaults to the Post-format (items) stage, the one that was there before. */
|
|
4982
5218
|
function filterSetsOf(job , stage = "items") {
|
|
4983
5219
|
return (filterStepOf(job, stage)?.filterSets ?? []).slice();
|
|
4984
5220
|
}
|
|
@@ -4998,10 +5234,10 @@ function withFilterSets(job , links , stage
|
|
|
4998
5234
|
const linkFilters = (link , resolve ) =>
|
|
4999
5235
|
(link.own ?? (link.set && resolve ? resolve(link.set) : null))?.filters || [];
|
|
5000
5236
|
|
|
5001
|
-
/** A job's
|
|
5237
|
+
/** A job's Post-format (items) Responses as an ordered modifier list for a
|
|
5002
5238
|
run (#339): each modifier step as it is, and each items filter-set link as a
|
|
5003
5239
|
synthetic modifier that applies the set's filters where the link sits. The
|
|
5004
|
-
|
|
5240
|
+
Pre-format (text) filters are not modifiers -- `beforeFilters` gives
|
|
5005
5241
|
them, and a run applies them to the raw reply before it is read. [resolve]
|
|
5006
5242
|
gives a Library link's body. */
|
|
5007
5243
|
function stageModifiers(job , resolve ) {
|
|
@@ -5016,7 +5252,7 @@ function stageModifiers(job , resolve
|
|
|
5016
5252
|
return out;
|
|
5017
5253
|
}
|
|
5018
5254
|
|
|
5019
|
-
/** A job's
|
|
5255
|
+
/** A job's Pre-format (text) filters for a run (#405), in document order:
|
|
5020
5256
|
the filters of every link on its `text` filters step, which a run applies to
|
|
5021
5257
|
the raw reply before the kind reads it into items. [resolve] gives a Library
|
|
5022
5258
|
link's body, as `stageModifiers` does. */
|
|
@@ -5067,11 +5303,11 @@ STEP_TYPES.job = {
|
|
|
5067
5303
|
}
|
|
5068
5304
|
const twice = steps.find((st, x) => !STEP_TYPES[st.type] .many && steps.findIndex(o => o.type === st.type) !== x);
|
|
5069
5305
|
if (twice) return void bad.push(`${at} has two ${STEP_TYPES[twice.type] .label ?? twice.type} steps`);
|
|
5070
|
-
// A `filters` step is `many`, but only one per stage: a
|
|
5071
|
-
//
|
|
5306
|
+
// A `filters` step is `many`, but only one per stage: a Pre-format and
|
|
5307
|
+
// a Post-format set at most (#405).
|
|
5072
5308
|
const twoFilters = steps.find((st, x) => st.type === "filters"
|
|
5073
5309
|
&& steps.findIndex(o => o.type === "filters" && o.stage === st.stage) !== x);
|
|
5074
|
-
if (twoFilters) return void bad.push(`${at} has two ${twoFilters.stage === "text" ? "
|
|
5310
|
+
if (twoFilters) return void bad.push(`${at} has two ${twoFilters.stage === "text" ? "Pre-format" : "Post-format"} filter steps`);
|
|
5075
5311
|
if (steps.some((st, x) => x > 0 && placeOf(st) < placeOf(steps[x - 1]))) {
|
|
5076
5312
|
return void bad.push(`${at}: steps are its Content, then its Responses, each in order`);
|
|
5077
5313
|
}
|
|
@@ -5173,10 +5409,16 @@ function versionProblem(doc ) {
|
|
|
5173
5409
|
rules, as a run with no graded eval parsed. */
|
|
5174
5410
|
|
|
5175
5411
|
|
|
5176
|
-
|
|
5412
|
+
|
|
5177
5413
|
|
|
5178
|
-
|
|
5414
|
+
|
|
5415
|
+
|
|
5179
5416
|
|
|
5417
|
+
|
|
5418
|
+
|
|
5419
|
+
|
|
5420
|
+
|
|
5421
|
+
|
|
5180
5422
|
|
|
5181
5423
|
|
|
5182
5424
|
/**
|
|
@@ -5227,8 +5469,10 @@ function columnsOf(doc )
|
|
|
5227
5469
|
function profileRefs(doc ) {
|
|
5228
5470
|
const cut = (r ) => (isObj(r) && isStr(r.id) ? { id: r.id, name: isStr(r.name) ? r.name : "" } : r);
|
|
5229
5471
|
for (const { column, steps } of columnsOf(doc)) {
|
|
5230
|
-
|
|
5231
|
-
|
|
5472
|
+
for (const f of ["connection", "profile"]) {
|
|
5473
|
+
if (f in column) column[f] = cut(column[f]);
|
|
5474
|
+
for (const st of steps) if (st[f] != null) st[f] = cut(st[f]);
|
|
5475
|
+
}
|
|
5232
5476
|
}
|
|
5233
5477
|
}
|
|
5234
5478
|
|
|
@@ -5238,7 +5482,40 @@ const cutRefs = (doc ) => { profileRefs(doc); return doc; }
|
|
|
5238
5482
|
/** Whether any profile reference in a pipeline holds more than { id, name }. */
|
|
5239
5483
|
function fatProfileRef(doc ) {
|
|
5240
5484
|
const fat = (r ) => isObj(r) && Object.keys(r).some(k => k !== "id" && k !== "name");
|
|
5241
|
-
return isObj(doc) && columnsOf(doc).some(({ column, steps }) => fat(column.profile) ||
|
|
5485
|
+
return isObj(doc) && columnsOf(doc).some(({ column, steps }) => fat(column.profile) || fat(column.connection)
|
|
5486
|
+
|| steps.some(st => fat(st.profile) || fat(st.connection)));
|
|
5487
|
+
}
|
|
5488
|
+
|
|
5489
|
+
/**
|
|
5490
|
+
* Version 15 to 16: a Target profile is an AI Connection now, keeping the
|
|
5491
|
+
* profile's id, and -- where the profile set settings -- the Connection
|
|
5492
|
+
* Profile split from it ([ctx]'s `connectionSplit`), so each target and step
|
|
5493
|
+
* asks what it asked. A run document carries each profile resolved, under
|
|
5494
|
+
* its id (`runAnswer` reads it so), so it names the connection alone, as
|
|
5495
|
+
* does a reader that cannot say what was split. Idempotent: a target that
|
|
5496
|
+
* names a connection already is left as it is.
|
|
5497
|
+
*/
|
|
5498
|
+
function answersFromProfiles(doc , ctx ) {
|
|
5499
|
+
doc.version = PIPELINE_VERSION;
|
|
5500
|
+
const split = (ref ) => (isObj(doc.profiles) || !isObj(ref) || !isStr(ref.id) ? null
|
|
5501
|
+
: ctx.connectionSplit?.(ref.id) ?? null);
|
|
5502
|
+
if (!Array.isArray(doc.targets)) return doc;
|
|
5503
|
+
doc.targets = doc.targets.map((t ) => {
|
|
5504
|
+
if (!isObj(t) || "connection" in t) return t;
|
|
5505
|
+
const { id, name, profile = null, steps, ...rest } = t;
|
|
5506
|
+
const p = split(profile);
|
|
5507
|
+
const out = { id, name, connection: profile, profile: p ? { id: p.id, name: p.name ?? "" } : null, steps, ...rest };
|
|
5508
|
+
if (Array.isArray(steps)) {
|
|
5509
|
+
out.steps = steps.map((st ) => {
|
|
5510
|
+
if (!isObj(st) || st.profile == null || "connection" in st) return st;
|
|
5511
|
+
const { profile: own, ...more } = st;
|
|
5512
|
+
const sp = split(own);
|
|
5513
|
+
return { ...more, connection: own, ...(sp ? { profile: { id: sp.id, name: sp.name ?? "" } } : {}) };
|
|
5514
|
+
});
|
|
5515
|
+
}
|
|
5516
|
+
return out;
|
|
5517
|
+
});
|
|
5518
|
+
return doc;
|
|
5242
5519
|
}
|
|
5243
5520
|
|
|
5244
5521
|
/** Version 5 to 6: the one test, or none, as a list. Its id is `t1`, so
|
|
@@ -5443,32 +5720,38 @@ function upgradePipeline (doc , ctx = {}) {
|
|
|
5443
5720
|
}
|
|
5444
5721
|
return localSteps(ownBodiesNow(out), ctx) ;
|
|
5445
5722
|
}
|
|
5446
|
-
if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14].includes(doc.version )) return doc;
|
|
5723
|
+
if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16].includes(doc.version )) return doc;
|
|
5724
|
+
// Every version reads as 16 first, then as 17: its requests read by their
|
|
5725
|
+
// job's HTTP API (requestsOnJobApi).
|
|
5726
|
+
const seventeen = (d ) => localSteps(ownBodiesNow(requestsOnJobApi(d)), ctx) ;
|
|
5727
|
+
if (doc.version === 16) return seventeen(cutRefs(clone(doc) ));
|
|
5728
|
+
// Every version reads as 15 first, then as 16: its Target profiles split
|
|
5729
|
+
// into AI Connections and Connection Profiles (answersFromProfiles).
|
|
5730
|
+
const sixteen = (fifteen ) => seventeen(answersFromProfiles(fifteen, ctx));
|
|
5731
|
+
if (doc.version === 15) return sixteen(cutRefs(clone(doc) ));
|
|
5447
5732
|
// Versions 13 and 14 read as 15 by migrating a job's cleaning modifier steps
|
|
5448
|
-
// and its List `preamble` setting into
|
|
5733
|
+
// and its List `preamble` setting into Pre-/Post-format filter sets
|
|
5449
5734
|
// (#339, #405). A version-14 document already moved its Drop items / Reject
|
|
5450
5735
|
// into a (stageless) filters step; filtersFromCleaning stages it and folds in
|
|
5451
5736
|
// the remaining Tidy / Remove duplicates / Keep first / Case.
|
|
5452
|
-
if (doc.version === 13 || doc.version === 14)
|
|
5453
|
-
return localSteps(ownBodiesNow(filtersFromCleaning(clone(doc) )), ctx) ;
|
|
5454
|
-
}
|
|
5737
|
+
if (doc.version === 13 || doc.version === 14) return sixteen(filtersFromCleaning(clone(doc) ));
|
|
5455
5738
|
if (doc.version === 11 || doc.version === 12) {
|
|
5456
|
-
return
|
|
5739
|
+
return sixteen(filtersFromCleaning(groupsOf(withoutCaseMetric(caseAsWritten(cutRefs(clone(doc) ))))));
|
|
5457
5740
|
}
|
|
5458
5741
|
// Every version before 10 reads as version 9 first, then as 10, then up
|
|
5459
5742
|
// through 15. A version-10 document is cut as a current one was
|
|
5460
5743
|
// (profileRefs); the earlier ones are cut on their way through nineOf.
|
|
5461
5744
|
const ten = doc.version === 10 ? cutRefs(clone(doc) ) : targetsFromScenarios(nineOf(clone(doc) , ctx));
|
|
5462
|
-
return
|
|
5745
|
+
return sixteen(filtersFromCleaning(groupsOf(withoutCaseMetric(caseAsWritten(evalsKey(ten))))));
|
|
5463
5746
|
}
|
|
5464
5747
|
|
|
5465
5748
|
/** [doc] with every job's cleaning modifier steps and its List `preamble`
|
|
5466
5749
|
setting migrated into filter sets on `filters` steps, up to version 15
|
|
5467
|
-
(#339, #405). The
|
|
5750
|
+
(#339, #405). The Post-format (items) filters are built by walking the
|
|
5468
5751
|
Responses steps in document order -- each cleaning modifier its filters
|
|
5469
5752
|
(`cleaningFilters`), each existing filters step's links in place, a private
|
|
5470
5753
|
one's filters coalescing with adjacent modifier filters into one own set --
|
|
5471
|
-
and the `preamble: true` setting becomes a
|
|
5754
|
+
and the `preamble: true` setting becomes a Pre-format (text) set of
|
|
5472
5755
|
Remove thinking then Remove preamble. The steps take the place of the first
|
|
5473
5756
|
of them, and the preamble setting is stripped.
|
|
5474
5757
|
|
|
@@ -5556,12 +5839,20 @@ function withoutCaseMetric(doc ) {
|
|
|
5556
5839
|
step then asks none, or names its own, keeps no profile either. The same
|
|
5557
5840
|
document, untouched, when there is nothing to read so. */
|
|
5558
5841
|
function localSteps(doc , ctx ) {
|
|
5559
|
-
const
|
|
5560
|
-
|
|
5561
|
-
|
|
5842
|
+
const run = isObj(doc.profiles);
|
|
5843
|
+
if (!ctx.profileType && !run) return doc;
|
|
5844
|
+
// A connection's type: [ctx]'s, by its id, or what a run carries for it.
|
|
5845
|
+
const typeOfAnswer = (connection , profile ) => {
|
|
5846
|
+
if (!isObj(connection) || !isStr(connection.id)) return undefined;
|
|
5847
|
+
if (ctx.profileType) return ctx.profileType(connection.id);
|
|
5848
|
+
const conn = runAnswer(doc, answerKey(connection.id, isObj(profile) && isStr(profile.id) ? profile.id : null));
|
|
5849
|
+
return isObj(conn) ? conn.type : undefined;
|
|
5850
|
+
};
|
|
5562
5851
|
const stands = new Map(Object.entries(STEP_TYPES).filter(([, e]) => e.slot === "target" && e.replaces).map(([type, e]) => [e.replaces , type]));
|
|
5563
|
-
const as = (
|
|
5564
|
-
const
|
|
5852
|
+
const as = (connection , profile ) => stands.get(typeOfAnswer(connection, profile) ?? "");
|
|
5853
|
+
const ownsAnswer = (st ) => st.connection != null || st.profile != null;
|
|
5854
|
+
const turns = (t , st ) => isObj(st) && STEP_TYPES[st.type]?.asks === "prompt"
|
|
5855
|
+
&& (ownsAnswer(st) ? as(st.connection ?? t.connection, st.profile) : as(t.connection, t.profile));
|
|
5565
5856
|
const targets = Array.isArray(doc.targets) ? doc.targets : [];
|
|
5566
5857
|
if (!targets.some(t => isObj(t) && Array.isArray(t.steps) && t.steps.some((st ) => turns(t, st)))) return doc;
|
|
5567
5858
|
const next = clone(doc) ;
|
|
@@ -5571,9 +5862,43 @@ function localSteps(doc , ctx ) {
|
|
|
5571
5862
|
const type = turns(t, st);
|
|
5572
5863
|
return type ? { type, prompt: st.prompt, ...(st.from ? { from: st.from } : {}) } : st;
|
|
5573
5864
|
});
|
|
5574
|
-
// A run keeps the
|
|
5575
|
-
if (!
|
|
5576
|
-
&& t.steps.every((st ) => isObj(st) && (STEP_TYPES[st.type]?.local || st
|
|
5865
|
+
// A run keeps the connection it ran as, so its report names it as it did.
|
|
5866
|
+
if (!run && as(t.connection, t.profile)
|
|
5867
|
+
&& t.steps.every((st ) => isObj(st) && (STEP_TYPES[st.type]?.local || ownsAnswer(st)))) {
|
|
5868
|
+
t.connection = null;
|
|
5869
|
+
t.profile = null;
|
|
5870
|
+
}
|
|
5871
|
+
}
|
|
5872
|
+
return next;
|
|
5873
|
+
}
|
|
5874
|
+
|
|
5875
|
+
/**
|
|
5876
|
+
* Version 16 to 17 (#477): a request step has no HTTP API of its own; its
|
|
5877
|
+
* job's flow step says it. A step that named another -- Any endpoint over an
|
|
5878
|
+
* Anthropic flow, say -- keeps its body as it would have sent it, read again
|
|
5879
|
+
* by the job's (its words and fields as that API holds them). A request in a
|
|
5880
|
+
* job with no flow step, which could never run, is a Prompt keeping its words.
|
|
5881
|
+
*/
|
|
5882
|
+
function requestsOnJobApi(next ) {
|
|
5883
|
+
next.version = PIPELINE_VERSION;
|
|
5884
|
+
const jobs = Array.isArray(next.jobs) ? next.jobs : [];
|
|
5885
|
+
const step = (st , k ) => {
|
|
5886
|
+
if (!isObj(st) || st.type !== "httpRequest") return st;
|
|
5887
|
+
const { api: own, ...rest } = st;
|
|
5888
|
+
const flow = isObj(jobs[k]) ? flowStepOf(jobs[k] ) : null;
|
|
5889
|
+
if (!flow) {
|
|
5890
|
+
return { type: "prompt", prompt: rest.prompt,
|
|
5891
|
+
...Object.fromEntries(["connection", "profile", "from"].filter(f => rest[f] != null).map(f => [f, rest[f]])) };
|
|
5892
|
+
}
|
|
5893
|
+
const was = HTTP_APIS[own ], now = HTTP_APIS[flow.api];
|
|
5894
|
+
if (!was || !now || own === flow.api) return rest;
|
|
5895
|
+
let body = rest.body;
|
|
5896
|
+
try { body = was.place(rest.body, rest ); } catch { /* words that never placed: the template stands */ }
|
|
5897
|
+
const { system: _s, model: _m, settings: _t, prefill: _p, ...kept } = rest;
|
|
5898
|
+
return { ...kept, body, ...now.cellOf(body) };
|
|
5899
|
+
};
|
|
5900
|
+
if (Array.isArray(next.targets)) {
|
|
5901
|
+
next.targets = next.targets.map((t ) => (isObj(t) && Array.isArray(t.steps) ? { ...t, steps: t.steps.map(step) } : t));
|
|
5577
5902
|
}
|
|
5578
5903
|
return next;
|
|
5579
5904
|
}
|
|
@@ -5633,6 +5958,16 @@ function blankPipeline(opts
|
|
|
5633
5958
|
jobs: [job], targets: [], evals: [], pass: { mode: "all" } };
|
|
5634
5959
|
}
|
|
5635
5960
|
|
|
5961
|
+
/** What is wrong with how a target or a step [at] names whom it asks: each
|
|
5962
|
+
reference as { id, name }, and a profile only over a connection. */
|
|
5963
|
+
function answerRefsProblems(o , at , bad ) {
|
|
5964
|
+
for (const [f, what] of [["connection", "AI Connection"], ["profile", "Connection Profile"]] ) {
|
|
5965
|
+
if (o[f] == null) continue;
|
|
5966
|
+
if (!isRef(o[f])) bad.push(`${at} has to name its ${what} as { id, name }`);
|
|
5967
|
+
else onlyFields(o[f], `${at}'s ${f}`, ["id", "name"], bad);
|
|
5968
|
+
}
|
|
5969
|
+
}
|
|
5970
|
+
|
|
5636
5971
|
/**
|
|
5637
5972
|
* Every reason [doc] cannot run, as sentences -- none when it can. The
|
|
5638
5973
|
* reference lookups are [ctx]'s: `profiles(id)` and `sources(id)` answer with
|
|
@@ -5645,9 +5980,7 @@ function validatePipeline(input , ctx = {}) {
|
|
|
5645
5980
|
const bad = [];
|
|
5646
5981
|
const run = doc.profiles !== undefined;
|
|
5647
5982
|
const c = { ...ctx, run };
|
|
5648
|
-
if (run && !ctx.profiles)
|
|
5649
|
-
c.profiles = (id ) => isObj(doc.profiles) && Object.hasOwn(doc.profiles, id) ? doc.profiles[id] : null;
|
|
5650
|
-
}
|
|
5983
|
+
if (run && !ctx.profiles) c.profiles = (key ) => (runAnswer(doc, key) ) ?? null;
|
|
5651
5984
|
onlyFields(doc, "the pipeline", run ? RUN_FIELDS : PIPELINE_FIELDS, bad);
|
|
5652
5985
|
if (!isStr(doc.id) || !doc.id.trim()) bad.push("the pipeline has no id");
|
|
5653
5986
|
if (doc.name != null && !isStr(doc.name)) bad.push("name has to be text");
|
|
@@ -5668,32 +6001,33 @@ function validatePipeline(input , ctx = {}) {
|
|
|
5668
6001
|
targets.forEach((t, i) => {
|
|
5669
6002
|
const at = targetLabel(doc, i);
|
|
5670
6003
|
if (!isObj(t)) return void bad.push(`${at} has to be an object`);
|
|
5671
|
-
onlyFields(t, at, ["id", "name", "profile", "steps"], bad);
|
|
6004
|
+
onlyFields(t, at, ["id", "name", "connection", "profile", "steps"], bad);
|
|
5672
6005
|
if (!isStr(t.id) || !t.id.trim()) bad.push(`${at} has no id`);
|
|
5673
6006
|
else if (targets.findIndex((o) => isObj(o) && o.id === t.id) !== i) bad.push(`${at} has the id of another target`);
|
|
5674
6007
|
if (t.name != null && !isStr(t.name)) bad.push(`${at}: name has to be text`);
|
|
5675
|
-
|
|
5676
|
-
else if (t.profile != null) onlyFields(t.profile, `${at}'s profile`, ["id", "name"], bad);
|
|
6008
|
+
answerRefsProblems(t, at, bad);
|
|
5677
6009
|
if (!Array.isArray(t.steps) || t.steps.length !== n) {
|
|
5678
6010
|
bad.push(`${at} has ${Array.isArray(t.steps) ? t.steps.length : "no"} prompts for ${n} job${n === 1 ? "" : "s"}`);
|
|
5679
6011
|
return;
|
|
5680
6012
|
}
|
|
5681
|
-
// A target with no
|
|
5682
|
-
// each name theirs.
|
|
5683
|
-
|
|
5684
|
-
|
|
6013
|
+
// A target with no connection of its own is one whose steps need none,
|
|
6014
|
+
// or each name theirs; a profile needs a connection to be over.
|
|
6015
|
+
const ownsOne = (st ) => isObj(st) && st.connection != null;
|
|
6016
|
+
if (t.connection == null && t.steps.some((st ) => !(isObj(st) && STEP_TYPES[st.type]?.local) && !ownsOne(st))) {
|
|
6017
|
+
bad.push(`${at} has to name its AI Connection as { id, name }`);
|
|
5685
6018
|
}
|
|
5686
6019
|
t.steps.forEach((st , k ) => {
|
|
5687
6020
|
const where = `${at}, ${jobLabel(doc, k)}`;
|
|
5688
6021
|
if (!isObj(st)) return void bad.push(`${where} has to be an object`);
|
|
5689
6022
|
const entry = STEP_TYPES[st.type];
|
|
5690
6023
|
if (entry?.slot !== "target") return void bad.push(`${where}: "${st.type}" is not something a target can send`);
|
|
5691
|
-
if (entry.local && st.profile != null) return void bad.push(`${where}: ${entry.label} asks no
|
|
6024
|
+
if (entry.local && (st.profile != null || st.connection != null)) return void bad.push(`${where}: ${entry.label} asks no connection`);
|
|
5692
6025
|
if (k > 0 && entry.firstJobOnly) return void bad.push(`${where}: ${entry.firstJobOnly}`);
|
|
5693
6026
|
onlyFields(st, where, entry.fields ?? [], bad);
|
|
5694
|
-
|
|
5695
|
-
|
|
5696
|
-
|
|
6027
|
+
// A request is read and placed by its job's HTTP API.
|
|
6028
|
+
const flow = entry.asks === "request" ? flowStepOf(doc.jobs[k]) : null;
|
|
6029
|
+
entry.validate(flow ? { ...st, api: flow.api } : st, c, bad, where);
|
|
6030
|
+
answerRefsProblems(st, where, bad);
|
|
5697
6031
|
if (st.from != null && !isRef(st.from)) bad.push(`${where} has to name the prompt it was picked from as { id, name }`);
|
|
5698
6032
|
// A request is a flow step's, rebuilt from its records, and its
|
|
5699
6033
|
// template has no place for an image.
|
|
@@ -5703,13 +6037,16 @@ function validatePipeline(input , ctx = {}) {
|
|
|
5703
6037
|
});
|
|
5704
6038
|
if (bad.length) return bad;
|
|
5705
6039
|
|
|
5706
|
-
// A reference that resolves to nothing, once per reference
|
|
6040
|
+
// A reference that resolves to nothing, once per reference: the part
|
|
6041
|
+
// that is gone, where the lookups can say which.
|
|
5707
6042
|
const missing = new Set ();
|
|
5708
|
-
const lookup = (
|
|
5709
|
-
const
|
|
5710
|
-
|
|
5711
|
-
|
|
5712
|
-
|
|
6043
|
+
const lookup = (a ) => {
|
|
6044
|
+
const key = answerKey(a.connection.id, a.profile?.id);
|
|
6045
|
+
const p = c.profiles ? c.profiles(key) : {};
|
|
6046
|
+
if (!p && !missing.has(key)) {
|
|
6047
|
+
missing.add(key);
|
|
6048
|
+
const gone = a.profile && c.connections?.(a.connection.id) ? a.profile : a.connection;
|
|
6049
|
+
bad.push(`${gone === a.profile ? "Connection Profile" : "Connection"} ${gone.name || gone.id} not found`);
|
|
5713
6050
|
}
|
|
5714
6051
|
return p;
|
|
5715
6052
|
};
|
|
@@ -5725,7 +6062,7 @@ function validatePipeline(input , ctx = {}) {
|
|
|
5725
6062
|
// What target [i] asks in job [k]: Echo's own connection for a step
|
|
5726
6063
|
// answered here, else its profile as the lookups hold it.
|
|
5727
6064
|
const askedOf = (i , k ) => (targetEntryOf(doc, i, k)?.local ? localConnOf(targetEntryOf(doc, i, k))
|
|
5728
|
-
: c.profiles ? c.profiles(
|
|
6065
|
+
: c.profiles ? c.profiles(targetAnswerKey(doc, i, k) ) : null);
|
|
5729
6066
|
const itemsHaveText = !!content && !CONTENT_TYPES[content.type]?.bare;
|
|
5730
6067
|
for (let k = 0; k < n; k++) {
|
|
5731
6068
|
const i = targets.findIndex((_, i) => {
|
|
@@ -5746,7 +6083,7 @@ function validatePipeline(input , ctx = {}) {
|
|
|
5746
6083
|
const nonText = (contentFiles || []).map(f => String((f )?.name ?? f))
|
|
5747
6084
|
.find(n => !/\.(?:txt|md|csv)$/i.test(n));
|
|
5748
6085
|
targets.forEach((_, i) => {
|
|
5749
|
-
const conns = doc.jobs.map((_ , k ) => (targetEntryOf(doc, i, k)?.local ? localConnOf(targetEntryOf(doc, i, k)) : lookup(
|
|
6086
|
+
const conns = doc.jobs.map((_ , k ) => (targetEntryOf(doc, i, k)?.local ? localConnOf(targetEntryOf(doc, i, k)) : lookup(targetAnswerOf(doc, i, k) )));
|
|
5750
6087
|
conns.forEach((p , k ) => {
|
|
5751
6088
|
if (!local(p)) return;
|
|
5752
6089
|
const entry = CONNECTION_TYPES[typeOf(p )] ;
|
|
@@ -5764,6 +6101,15 @@ function validatePipeline(input , ctx = {}) {
|
|
|
5764
6101
|
const type = CONNECTION_TYPES[typeOf(p )];
|
|
5765
6102
|
if (type && !(type.answers ?? ["prompt"]).includes(step.asks ?? "prompt")) {
|
|
5766
6103
|
bad.push(`${n > 1 ? `${jobLabel(doc, k)}, ` : ""}${targetLabel(doc, i)}: ${step.label ?? "this step"} needs a profile that answers it, and ${type.label} does not`);
|
|
6104
|
+
return;
|
|
6105
|
+
}
|
|
6106
|
+
// A request goes as the flow wrote it, so its connection has to speak
|
|
6107
|
+
// the job's HTTP API; one the lab cannot tell (Any endpoint) passes.
|
|
6108
|
+
const api = step.asks === "request" ? flowStepOf(doc.jobs[k])?.api : undefined;
|
|
6109
|
+
const speaks = api && api !== "raw" && type?.speaks ? type.speaks(p ) : null;
|
|
6110
|
+
if (api && speaks && speaks !== api) {
|
|
6111
|
+
const name = targetAnswerOf(doc, i, k)?.connection.name || type .label;
|
|
6112
|
+
bad.push(`${n > 1 ? `${jobLabel(doc, k)}, ` : ""}${targetLabel(doc, i)}: ${name} can't send an ${HTTP_APIS[api]?.label ?? api} request`);
|
|
5767
6113
|
}
|
|
5768
6114
|
});
|
|
5769
6115
|
// The model is the profile's unless the target's step carries it.
|
|
@@ -5771,8 +6117,8 @@ function validatePipeline(input , ctx = {}) {
|
|
|
5771
6117
|
&& targetEntryOf(doc, i, k)?.modelFrom !== "step" && !String(p.model || "").trim());
|
|
5772
6118
|
if (bare >= 0) {
|
|
5773
6119
|
bad.push(n > 1
|
|
5774
|
-
? `no model on the
|
|
5775
|
-
: `no model on ${targetLabel(doc, i)}'s
|
|
6120
|
+
? `no model on the connection ${jobLabel(doc, bare)} of ${targetLabel(doc, i)} uses — manage connections on the Setup tab`
|
|
6121
|
+
: `no model on ${targetLabel(doc, i)}'s connection — manage connections on the Setup tab`);
|
|
5776
6122
|
}
|
|
5777
6123
|
});
|
|
5778
6124
|
STEP_TYPES.evals .validate(doc.evals, c, bad, outOf(doc.jobs.at(-1)).kind, doc);
|
|
@@ -5841,15 +6187,22 @@ function pipelineWarnings(doc , ctx = {}) {
|
|
|
5841
6187
|
/** A run document's profiles table: id → connection, keyless, spellable. */
|
|
5842
6188
|
function profilesProblems(profiles , bad ) {
|
|
5843
6189
|
if (!isObj(profiles)) return void bad.push("profiles has to be an object of id → connection");
|
|
6190
|
+
// Two entries may share one connection, and so one key; two connections
|
|
6191
|
+
// spelling one variable would hand one the other's key.
|
|
5844
6192
|
const spelt = new Map ();
|
|
5845
6193
|
for (const [id, conn] of Object.entries(profiles)) {
|
|
5846
|
-
if (!
|
|
6194
|
+
if (!ANSWER_KEY.test(id)) {
|
|
5847
6195
|
bad.push(`profile id "${id}" has to be letters and digits, with at most one hyphen: its key comes from EVALSLAB_API_KEY_<ID>`);
|
|
5848
6196
|
continue;
|
|
5849
6197
|
}
|
|
5850
|
-
|
|
5851
|
-
|
|
5852
|
-
|
|
6198
|
+
if (isObj(conn) && conn.connection != null && !(isStr(conn.connection) && PROFILE_ID.test(conn.connection))) {
|
|
6199
|
+
bad.push(`profile ${id}: connection has to be an AI Connection's id`);
|
|
6200
|
+
continue;
|
|
6201
|
+
}
|
|
6202
|
+
const cid = answerConnectionId(id, isObj(conn) ? conn : null);
|
|
6203
|
+
const from = keyVar(cid, isObj(conn) && isStr(conn.slug) ? conn.slug : null);
|
|
6204
|
+
if (spelt.has(from) && spelt.get(from) !== cid) bad.push(`profiles ${spelt.get(from)} and ${cid} both take their key from $${from}`);
|
|
6205
|
+
spelt.set(from, cid);
|
|
5853
6206
|
const at = `profile ${id}`;
|
|
5854
6207
|
if (!isObj(conn)) { bad.push(`${at} has to be an object`); continue; }
|
|
5855
6208
|
connectionProblems(conn, at, from, bad);
|
|
@@ -5915,24 +6268,34 @@ function connectionProblems(conn , at , from
|
|
|
5915
6268
|
for (const why of CONNECTION_TYPES[type]?.settingsProblems?.(connectionSettings(conn)) ?? []) bad.push(`${at} ${why}`);
|
|
5916
6269
|
}
|
|
5917
6270
|
|
|
5918
|
-
/** The
|
|
6271
|
+
/** The AI Connection ids a pipeline asks -- each target's and step's, then
|
|
6272
|
+
each grader's -- in order: the Target profile ids, before version 16. */
|
|
5919
6273
|
function profileIds(doc ) {
|
|
5920
|
-
|
|
5921
|
-
|
|
5922
|
-
|
|
5923
|
-
|
|
5924
|
-
|
|
5925
|
-
|
|
6274
|
+
return [...new Set(answerKeys(doc).map(k => answerOfKey(k).connection))];
|
|
6275
|
+
}
|
|
6276
|
+
|
|
6277
|
+
/** Every answer key a pipeline asks, targets first, in order: each target's
|
|
6278
|
+
and step's connection and profile, then each grader's bare id. What a run
|
|
6279
|
+
carries one connection for. */
|
|
6280
|
+
function answerKeys(doc ) {
|
|
6281
|
+
const keys = [];
|
|
6282
|
+
const add = (key ) => { if (key && !keys.includes(key)) keys.push(key); };
|
|
6283
|
+
targetsOf(doc).forEach((t, i) => {
|
|
6284
|
+
// The target's own, which History reads it by, then each step's.
|
|
6285
|
+
if (isRef(t.connection)) add(answerKey(t.connection.id, isRef(t.profile) ? t.profile.id : null));
|
|
6286
|
+
(Array.isArray(t.steps) ? t.steps : []).forEach((_, k) => add(targetAnswerKey(doc, i, k)));
|
|
6287
|
+
});
|
|
5926
6288
|
// Then the ones an eval asks (a grader), so a run carries them too.
|
|
5927
6289
|
for (const t of evalsOf(doc)) {
|
|
5928
|
-
for (const ref of EVAL_TYPES[t.type]?.profiles?.(t) ?? []) if (ref?.id && !
|
|
6290
|
+
for (const ref of EVAL_TYPES[t.type]?.profiles?.(t) ?? []) if (ref?.id && !keys.includes(ref.id)) keys.push(ref.id);
|
|
5929
6291
|
}
|
|
5930
|
-
return
|
|
6292
|
+
return keys;
|
|
5931
6293
|
}
|
|
5932
6294
|
|
|
5933
6295
|
/**
|
|
5934
6296
|
* The run document: [doc] plus what its references resolved to, once, at
|
|
5935
|
-
* submit. `ctx.profiles(
|
|
6297
|
+
* submit. `ctx.profiles(key)` answers an answer key with what it resolves to
|
|
6298
|
+
* (`answersOf`'s `resolve`), `ctx.files` is
|
|
5936
6299
|
* the Source's file list as it reads now, `ctx.comment` is the run's own.
|
|
5937
6300
|
*/
|
|
5938
6301
|
function resolvePipeline(doc ,
|
|
@@ -5942,10 +6305,16 @@ function resolvePipeline(doc ,
|
|
|
5942
6305
|
// What the lab supplies an eval -- its grader -- before the profiles it
|
|
5943
6306
|
// asks are carried.
|
|
5944
6307
|
run.evals = (Array.isArray(run.evals) ? run.evals : []).map((t ) => EVAL_TYPES[t?.type]?.resolve?.(t, ctx) ?? t);
|
|
6308
|
+
// Each pair asked, resolved and frozen on its own: two targets sharing a
|
|
6309
|
+
// connection under different profiles send different settings. The entry
|
|
6310
|
+
// names the connection whose key it is sent with.
|
|
5945
6311
|
run.profiles = {};
|
|
5946
|
-
for (const
|
|
5947
|
-
const p = ctx.profiles?.(
|
|
5948
|
-
if (p)
|
|
6312
|
+
for (const key of answerKeys(run)) {
|
|
6313
|
+
const p = ctx.profiles?.(key);
|
|
6314
|
+
if (!p) continue;
|
|
6315
|
+
const conn = connectionOf(p );
|
|
6316
|
+
const cid = isStr(p.id) && p.id ? p.id : answerOfKey(key).connection;
|
|
6317
|
+
run.profiles[key] = { ...conn, connection: cid };
|
|
5949
6318
|
}
|
|
5950
6319
|
const content = contentOf(run) ;
|
|
5951
6320
|
if (content?.type === "source") content.files = CONTENT_TYPES.source .expand (content, ctx.files);
|
|
@@ -6008,13 +6377,14 @@ function pipelineToYaml(doc , ctx
|
|
|
6008
6377
|
return yaml.dump(out, { lineWidth: -1, noRefs: true });
|
|
6009
6378
|
}
|
|
6010
6379
|
|
|
6011
|
-
/** Every
|
|
6012
|
-
steps', each eval's grader -- replaced, in place, by [f] of it.
|
|
6380
|
+
/** Every AI Connection reference in [doc] -- each target's, each of its
|
|
6381
|
+
steps', each eval's grader -- replaced, in place, by [f] of it. A
|
|
6382
|
+
Connection Profile is named by id and name, and left as it is. */
|
|
6013
6383
|
function eachProfileRef(doc , f ) {
|
|
6014
6384
|
for (const t of targetsOf(doc)) {
|
|
6015
|
-
if (t.
|
|
6385
|
+
if (t.connection) t.connection = f(t.connection);
|
|
6016
6386
|
for (const st of Array.isArray(t.steps) ? t.steps : []) {
|
|
6017
|
-
if (st?.
|
|
6387
|
+
if (st?.connection) st.connection = f(st.connection);
|
|
6018
6388
|
}
|
|
6019
6389
|
}
|
|
6020
6390
|
for (const t of Array.isArray(doc.evals) ? doc.evals : []) {
|
|
@@ -6035,6 +6405,9 @@ function eachProfileRef(doc , f )
|
|
|
6035
6405
|
* stands.
|
|
6036
6406
|
*/
|
|
6037
6407
|
function importPipeline(input , ctx = {}) {
|
|
6408
|
+
// A version-15 file names its profiles by slug, which the upgrade cannot
|
|
6409
|
+
// split (it needs this lab's id), so the split is asked again once remapped.
|
|
6410
|
+
const preSplit = isObj(input) && typeof input.version === "number" && input.version < 16 && !isObj(input.profiles);
|
|
6038
6411
|
// A file exported before version 2 still imports: it is upgraded, not refused.
|
|
6039
6412
|
const doc = upgradePipeline(input, ctx);
|
|
6040
6413
|
const v = versionProblem(doc);
|
|
@@ -6042,9 +6415,10 @@ function importPipeline(input , ctx = {}) {
|
|
|
6042
6415
|
const profiles = ctx.profiles ?? [], sources = ctx.sources ?? [], groups = ctx.groups ?? [];
|
|
6043
6416
|
const next = clone(doc) ;
|
|
6044
6417
|
const missing = [], seen = new Set ();
|
|
6045
|
-
const lists = { profile: profiles, source: sources, group: groups };
|
|
6418
|
+
const lists = { profile: profiles, connectionProfile: ctx.connectionProfiles ?? [], source: sources, group: groups };
|
|
6046
6419
|
const words = {
|
|
6047
|
-
profile: (ref ) => `
|
|
6420
|
+
profile: (ref ) => `Connection ${ref.slug || ref.name || ref.id} not found`,
|
|
6421
|
+
connectionProfile: (ref ) => `Connection Profile ${ref.name || ref.id} not found`,
|
|
6048
6422
|
source: (ref ) => `Source ${ref.name || ref.id} not found`,
|
|
6049
6423
|
group: (ref ) => `Eval group ${ref.name || ref.id} not found`,
|
|
6050
6424
|
};
|
|
@@ -6070,10 +6444,22 @@ function importPipeline(input , ctx = {}) {
|
|
|
6070
6444
|
};
|
|
6071
6445
|
const content = contentOf(next) ;
|
|
6072
6446
|
if (content?.type === "source") content.ref = remap(content.ref, "source");
|
|
6447
|
+
const split = (connection ) => {
|
|
6448
|
+
const p = preSplit && isRef(connection) && profiles.some(x => x.id === connection.id) ? ctx.connectionSplit?.(connection.id) : null;
|
|
6449
|
+
return p ? { id: p.id, name: p.name ?? "" } : null;
|
|
6450
|
+
};
|
|
6073
6451
|
for (const t of targetsOf(next)) {
|
|
6074
|
-
|
|
6452
|
+
// A profile is matched where the lab says which it holds.
|
|
6453
|
+
if (t.connection) t.connection = remapProfile(t.connection);
|
|
6454
|
+
if (isRef(t.profile) && ctx.connectionProfiles) t.profile = remap(t.profile, "connectionProfile");
|
|
6455
|
+
else if (t.profile == null) t.profile = split(t.connection);
|
|
6075
6456
|
for (const st of Array.isArray(t.steps) ? t.steps : []) {
|
|
6076
|
-
if (st?.
|
|
6457
|
+
if (st?.connection) st.connection = remapProfile(st.connection);
|
|
6458
|
+
if (isRef(st?.profile) && ctx.connectionProfiles) st.profile = remap(st.profile, "connectionProfile");
|
|
6459
|
+
else if (isObj(st) && st.connection && st.profile == null) {
|
|
6460
|
+
const sp = split(st.connection);
|
|
6461
|
+
if (sp) st.profile = sp;
|
|
6462
|
+
}
|
|
6077
6463
|
}
|
|
6078
6464
|
}
|
|
6079
6465
|
for (const t of Array.isArray(next.evals) ? next.evals : []) {
|
|
@@ -6173,7 +6559,7 @@ function junitXml(suites , runner ) {
|
|
|
6173
6559
|
//
|
|
6174
6560
|
// Export for CI writes a pipeline as files a repository keeps and
|
|
6175
6561
|
// `evals-lab run` runs with no lab around it (docs/pipeline-yaml.md, "Export
|
|
6176
|
-
// for CI"): the pipeline's YAML, its
|
|
6562
|
+
// for CI"): the pipeline's YAML, its connections resolved and keyed by slug, and each
|
|
6177
6563
|
// dataset it grades against as the Library's export of it. The server adds
|
|
6178
6564
|
// the plugins and, when asked, the Source's items, and zips the lot. Read
|
|
6179
6565
|
// back, the files are the run document the page would have submitted, with
|
|
@@ -6196,28 +6582,38 @@ const BUNDLE_DATASET = /^datasets\/([a-z0-9]+(?:-[a-z0-9]+)*)\.json$/;
|
|
|
6196
6582
|
|
|
6197
6583
|
/**
|
|
6198
6584
|
* [doc] as a bundle's text files: `pipeline.yaml`, `profiles.yaml` and
|
|
6199
|
-
* `datasets/<slug>.json`. [ctx.
|
|
6200
|
-
* none of which is written --
|
|
6201
|
-
*
|
|
6202
|
-
*
|
|
6203
|
-
*
|
|
6585
|
+
* `datasets/<slug>.json`. [ctx.connections] and [ctx.connectionProfiles]
|
|
6586
|
+
* are Setup's -- keys and all, none of which is written -- or
|
|
6587
|
+
* [ctx.profiles], Target profiles to split as the store was; [ctx.grader]
|
|
6588
|
+
* the lab's grader, written into each eval that would have asked it, and
|
|
6589
|
+
* [ctx.datasets] each graded dataset by id, as `{ name, body }`. Each
|
|
6590
|
+
* connection asked is written resolved, under its answer key with the
|
|
6591
|
+
* connection's slug for its id. `{ error }` when a dataset or a connection
|
|
6592
|
+
* the pipeline names is not given, since a bundle without it runs
|
|
6204
6593
|
* something else.
|
|
6205
6594
|
*/
|
|
6206
|
-
function exportBundle(doc , ctx
|
|
6595
|
+
function exportBundle(doc , ctx
|
|
6596
|
+
|
|
6207
6597
|
= {})
|
|
6208
6598
|
{
|
|
6209
|
-
const
|
|
6599
|
+
const split = (ctx.profiles ?? []).map(splitProfile);
|
|
6600
|
+
const profiles = withSlugs([...(ctx.connections ?? []), ...split.map(x => x.connection)] );
|
|
6601
|
+
const answers = answersOf(profiles, [...(ctx.connectionProfiles ?? []), ...split.flatMap(x => (x.profile ? [x.profile] : []))]);
|
|
6210
6602
|
// What the lab would supply at submit, written down: no lab supplies it later.
|
|
6211
6603
|
const out = clone(doc) ;
|
|
6212
6604
|
out.evals = (Array.isArray(out.evals) ? out.evals : [])
|
|
6213
6605
|
.map((t ) => EVAL_TYPES[t?.type]?.resolve?.(t, { grader: ctx.grader ?? null,
|
|
6214
6606
|
groups: Object.entries(ctx.datasets ?? {}).map(([id, d]) => ({ id, name: d.name, grader: d.body.grader ?? null })) }) ?? t);
|
|
6215
6607
|
const table = {};
|
|
6216
|
-
for (const
|
|
6217
|
-
const
|
|
6218
|
-
|
|
6608
|
+
for (const key of answerKeys(out)) {
|
|
6609
|
+
const at = answerOfKey(key);
|
|
6610
|
+
const named = answers.connection(at.connection);
|
|
6611
|
+
const p = answers.resolve(key);
|
|
6612
|
+
if (!named) return { error: `Connection ${at.connection} not found` };
|
|
6613
|
+
if (!p) return { error: `Connection Profile ${at.profile} not found` };
|
|
6219
6614
|
const { slug, ...flat } = connectionSettings(connectionOf(p) );
|
|
6220
|
-
|
|
6615
|
+
// A profile answering on a connection of its own takes that one's key.
|
|
6616
|
+
table[at.bare ? named.slug : answerKey(named.slug , at.profile)] = slug === named.slug ? flat : { ...flat, slug };
|
|
6221
6617
|
}
|
|
6222
6618
|
const files = {};
|
|
6223
6619
|
const taken = new Set (), written = new Set ();
|
|
@@ -6264,7 +6660,7 @@ function bundleProblems(files , keys = [])
|
|
|
6264
6660
|
for (const [path, text] of Object.entries(files)) {
|
|
6265
6661
|
for (const key of keys) {
|
|
6266
6662
|
if (isStr(key) && key.trim().length >= 4 && text.includes(key.trim())) {
|
|
6267
|
-
bad.push(`${path} holds a
|
|
6663
|
+
bad.push(`${path} holds a connection's key, and a key never leaves the lab`);
|
|
6268
6664
|
break;
|
|
6269
6665
|
}
|
|
6270
6666
|
}
|
|
@@ -6294,14 +6690,23 @@ function readBundle(files , ctx = {})
|
|
|
6294
6690
|
if (!isObj(table)) return { error: `${BUNDLE_PROFILES} maps each profile's slug to its settings` };
|
|
6295
6691
|
const keyed = bundleProblems(files);
|
|
6296
6692
|
if (keyed.length) return { error: keyed[0] };
|
|
6297
|
-
|
|
6298
|
-
|
|
6299
|
-
|
|
6300
|
-
|
|
6693
|
+
// Each entry under its answer key, the connection's slug for its id: a
|
|
6694
|
+
// bundle from before version 16 keys each by its Target profile's slug.
|
|
6695
|
+
const profiles = [];
|
|
6696
|
+
const entries = {};
|
|
6697
|
+
for (const [key, p] of Object.entries(table)) {
|
|
6698
|
+
const at = answerOfKey(key);
|
|
6699
|
+
if (!isSlug(at.connection) || (at.profile != null && !PROFILE_ID.test(at.profile))) {
|
|
6700
|
+
return { error: `${BUNDLE_PROFILES}: ${JSON.stringify(key)} is not a slug` };
|
|
6701
|
+
}
|
|
6702
|
+
if (!isObj(p)) return { error: `${BUNDLE_PROFILES}: ${key} has to hold its settings` };
|
|
6301
6703
|
if (!CONNECTION_TYPES[typeOf(p )]) {
|
|
6302
|
-
return { error: `${BUNDLE_PROFILES}: ${
|
|
6704
|
+
return { error: `${BUNDLE_PROFILES}: ${key} is of type ${p.type}, which this lab does not have` };
|
|
6303
6705
|
}
|
|
6304
|
-
|
|
6706
|
+
const slug = isSlug(p.slug) ? p.slug : at.connection;
|
|
6707
|
+
const name = isStr(p.name) && p.name ? p.name : slug;
|
|
6708
|
+
entries[key] = { ...p, id: slug, slug, name };
|
|
6709
|
+
if (!profiles.some(x => x.id === at.connection)) profiles.push({ id: at.connection, slug: at.connection, name: slug === at.connection ? name : at.connection });
|
|
6305
6710
|
}
|
|
6306
6711
|
// Each dataset by its name, which is what joins it to the eval naming it.
|
|
6307
6712
|
const datasets = [];
|
|
@@ -6334,6 +6739,8 @@ function readBundle(files , ctx = {})
|
|
|
6334
6739
|
if (!ref) continue;
|
|
6335
6740
|
if (datasets.some(d => d.name === ref.name)) refs.push({ id: ref.id, name: ref.name });
|
|
6336
6741
|
}
|
|
6742
|
+
// A profile is named by the id the key holds it under; one the table
|
|
6743
|
+
// lacks is a reference the run does not carry, and validation says so.
|
|
6337
6744
|
const imported = importPipeline(raw, { profiles, groups: refs,
|
|
6338
6745
|
sources: content?.type === "source" && isRef(content.ref) ? [content.ref] : [] });
|
|
6339
6746
|
if ("error" in imported) return imported;
|
|
@@ -6342,7 +6749,7 @@ function readBundle(files , ctx = {})
|
|
|
6342
6749
|
for (const r of refs) byId[r.id] = datasets.find(d => d.name === r.name) .body;
|
|
6343
6750
|
const caseItems = [...new Set(Object.values(byId).flatMap(b => gradedSetFrom(b ).filter(c => !c.todo).map(caseItem)).filter(Boolean))];
|
|
6344
6751
|
const run = resolvePipeline(imported.doc, {
|
|
6345
|
-
profiles:
|
|
6752
|
+
profiles: key => runAnswer({ profiles: entries }, key) ?? null,
|
|
6346
6753
|
files: ctx.items ? [...ctx.items].sort() : caseItems,
|
|
6347
6754
|
});
|
|
6348
6755
|
const bad = validatePipeline(run, { groups: refs });
|
|
@@ -6361,24 +6768,28 @@ function stagesFor(run , i , resolveFilters
|
|
|
6361
6768
|
{
|
|
6362
6769
|
const stages = run.jobs.map((ch, k) => {
|
|
6363
6770
|
const { kind, modifiers: _mods, ...settings } = outOf(ch);
|
|
6364
|
-
// The modifiers a run applies, with each
|
|
6365
|
-
// synthetic modifier where its link sits (#339), and the
|
|
6771
|
+
// The modifiers a run applies, with each Post-format filter set a
|
|
6772
|
+
// synthetic modifier where its link sits (#339), and the Pre-format
|
|
6366
6773
|
// (text) filters applied to the raw reply before it is read (#405);
|
|
6367
6774
|
// without a resolver, a Library link reads nothing, a private one its own.
|
|
6368
6775
|
return { text: targetStepOf(run, i, k) .prompt, kind, withImage: sendsImage(ch),
|
|
6369
6776
|
verbatim: !!targetEntryOf(run, i, k)?.verbatim, before: beforeFilters(ch, resolveFilters),
|
|
6370
6777
|
modifiers: stageModifiers(ch, resolveFilters), settings };
|
|
6371
6778
|
});
|
|
6779
|
+
// Each named by the connection whose key it is sent with.
|
|
6780
|
+
const asked = (key , held ) => {
|
|
6781
|
+
const { connection: _c, ...conn } = held || {} ;
|
|
6782
|
+
return { id: answerConnectionId(key, held), ...conn };
|
|
6783
|
+
};
|
|
6372
6784
|
const connections = run.jobs.map((_, k) => {
|
|
6373
6785
|
// A step answered here asks nothing: the profile a run made before it
|
|
6374
6786
|
// kept is what it ran as, else Echo itself.
|
|
6787
|
+
const key = targetAnswerKey(run, i, k);
|
|
6375
6788
|
if (targetEntryOf(run, i, k)?.local) {
|
|
6376
|
-
const
|
|
6377
|
-
|
|
6378
|
-
return held && CONNECTION_TYPES[held.type]?.local ? { id: id , ...held } : { id: "", ...localConnOf(targetEntryOf(run, i, k)) };
|
|
6789
|
+
const held = key != null ? runAnswer(run, key) : undefined;
|
|
6790
|
+
return held && CONNECTION_TYPES[held.type]?.local ? asked(key , held) : { id: "", ...localConnOf(targetEntryOf(run, i, k)) };
|
|
6379
6791
|
}
|
|
6380
|
-
|
|
6381
|
-
return { id, ...(run.profiles?.[id] || {}) };
|
|
6792
|
+
return key != null ? asked(key, runAnswer(run, key)) : { id: "" };
|
|
6382
6793
|
});
|
|
6383
6794
|
// What a transport that builds its own request (an HTTP Request) builds
|
|
6384
6795
|
// it from in each job, and this target's step there.
|
|
@@ -6388,11 +6799,16 @@ function stagesFor(run , i , resolveFilters
|
|
|
6388
6799
|
|
|
6389
6800
|
/** The Setup profile scenario [i] of a run ran under, as History shows it: keyless. */
|
|
6390
6801
|
function scenarioProfile(run , i ) {
|
|
6391
|
-
|
|
6802
|
+
// Read by the target: its own connection and profile, frozen at submit
|
|
6803
|
+
// apart from any other target's that shares the connection.
|
|
6804
|
+
const t = targetsOf(run)[i];
|
|
6805
|
+
const key = isRef(t?.connection) ? answerKey(t.connection.id, isRef(t.profile) ? t.profile.id : null) : undefined;
|
|
6392
6806
|
// A target that asks nothing ran as Echo.
|
|
6393
|
-
if (
|
|
6394
|
-
const
|
|
6395
|
-
|
|
6807
|
+
if (key == null && targetEntryOf(run, i, 0)?.local) { const lc = localConnOf(targetEntryOf(run, i, 0)); return { id: "", name: lc.name, settings: { ...lc } }; }
|
|
6808
|
+
const held = key != null ? runAnswer(run, key) : undefined;
|
|
6809
|
+
if (!held) return null;
|
|
6810
|
+
const { connection: _c, ...conn } = held;
|
|
6811
|
+
return { id: answerConnectionId(key , held), name: conn.name, settings: connectionSettings(conn ) };
|
|
6396
6812
|
}
|
|
6397
6813
|
|
|
6398
6814
|
/** The kind a run's result is read as: its last job's. */
|
|
@@ -6758,13 +7174,14 @@ export {
|
|
|
6758
7174
|
PIPELINE_VERSION, DATASET_BODY_VERSION, STEP_TYPES, CONTENT_TYPES, OUTPUT_KINDS, MODIFIERS, EVAL_TYPES, LEGACY_TESTS, METRICS, readMetrics, metricSummary, metricCheck, itemScoresAsync, recordedReplyOf,
|
|
6759
7175
|
SOURCE_TYPES, DEFAULT_SOURCE_TYPE, sourceTypeOf, WORKFLOW_PLATFORMS, platformOf, DYNAMIC_PROVIDERS, providerOf, WIZARDS, wizardProgress,
|
|
6760
7176
|
registerKinds, defaultKind, jobDefaults, applyModifiers, keyVar, slugOf, uniqueSlug, isSlug, withSlugs, SLUG_MAX, connectionOf, addressOf, connectionSettings,
|
|
7177
|
+
splitProfile, mergeAnswer, answerKey, answerOfKey, answersOf, runAnswer, answerConnectionId, connectionScoped,
|
|
6761
7178
|
connectionRequest, connectionBase, connectionSend, connectionReply, connectionReplyProblem, connectionImage, connectionProblems,
|
|
6762
7179
|
CONNECTION_FIELDS, SETTING_KEYS, OPTION_FIELDS, jobLabel, scenarioLabel, jobWords, versionProblem,
|
|
6763
7180
|
contentOf, withContent, replyOf,
|
|
6764
|
-
targetsOf, targetLetter, targetLabel, targetStepOf, targetEntryOf, jobTargetType,
|
|
6765
|
-
tokensOf, withTokens, sendsImage, withImage, flowStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
|
|
7181
|
+
targetsOf, targetLetter, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetAnswerOf, targetAnswerKey, recordedTargetOf, withTarget,
|
|
7182
|
+
tokensOf, withTokens, sendsImage, withImage, flowStepOf, requestStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
|
|
6766
7183
|
filterSetsOf, withFilterSets, stageModifiers, beforeFilters,
|
|
6767
|
-
blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, newLink, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds,
|
|
7184
|
+
blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, newLink, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds, answerKeys,
|
|
6768
7185
|
scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioEvals, scenarioPasses, groupPass, overallVerdict, passVerdict, isSkipped, failedScore,
|
|
6769
7186
|
evalLabel, evalsOf, evalsDataset, casesRef, groupOf, groupOfMetrics, pipelineWarnings, itemsFound, previewOf, isWholeRun, EVAL_FIELDS,
|
|
6770
7187
|
pipelineToYaml, importPipeline, yamlToPipeline, exportBundle, readBundle, bundleProblems, junitXml,
|