evals-lab 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/CHANGELOG.md +88 -1
  2. package/README.md +12 -6
  3. package/bin/evals-lab.js +9 -1
  4. package/bin/lab-file.js +97 -0
  5. package/bin/run.js +16 -16
  6. package/lab/VERSION +1 -1
  7. package/lab/demo/CREDITS.md +37 -3
  8. package/lab/demo/datasets/demo-1.json +2 -2
  9. package/lab/demo/datasets/demo-2.json +2 -2
  10. package/lab/demo/datasets/sentiment.json +383 -0
  11. package/lab/demo/datasets/weather-alerts.json +61 -0
  12. package/lab/demo/datasets/wordings.json +39 -0
  13. package/lab/demo/manifest.json +3 -3
  14. package/lab/demo/pipelines/demo-1.json +8 -6
  15. package/lab/demo/pipelines/demo-2.json +8 -6
  16. package/lab/demo/pipelines/sentiment.json +65 -0
  17. package/lab/demo/pipelines/weather-alerts.json +66 -0
  18. package/lab/demo/pipelines/wordings.json +104 -0
  19. package/lab/demo/sources/Demo 1/source.json +3 -0
  20. package/lab/demo/sources/Demo 2/source.json +3 -0
  21. package/lab/demo/sources/sentiment/bakery-croissant.txt +1 -0
  22. package/lab/demo/sources/sentiment/bus-timetable.txt +1 -0
  23. package/lab/demo/sources/sentiment/cafe-coffee.txt +1 -0
  24. package/lab/demo/sources/sentiment/concert-night.txt +1 -0
  25. package/lab/demo/sources/sentiment/film-sequel.txt +1 -0
  26. package/lab/demo/sources/sentiment/garden-tomatoes.txt +1 -0
  27. package/lab/demo/sources/sentiment/gym-contract.txt +1 -0
  28. package/lab/demo/sources/sentiment/headphones-review.txt +1 -0
  29. package/lab/demo/sources/sentiment/hotel-room.txt +1 -0
  30. package/lab/demo/sources/sentiment/library-hours.txt +1 -0
  31. package/lab/demo/sources/sentiment/meeting-minutes.txt +1 -0
  32. package/lab/demo/sources/sentiment/parcel-lost.txt +1 -0
  33. package/lab/demo/sources/sentiment/phone-update.txt +1 -0
  34. package/lab/demo/sources/sentiment/puppy-home.txt +1 -0
  35. package/lab/demo/sources/sentiment/recipe-note.txt +1 -0
  36. package/lab/demo/sources/sentiment/road-closure.txt +1 -0
  37. package/lab/demo/sources/sentiment/source.json +3 -0
  38. package/lab/demo/sources/sentiment/teacher-thanks.txt +1 -0
  39. package/lab/demo/sources/sentiment/train-delay.txt +1 -0
  40. package/lab/demo/sources/sentiment/volunteer-day.txt +1 -0
  41. package/lab/demo/sources/sentiment/weather-report.txt +1 -0
  42. package/lab/demo/sources/weather-alerts/https-api.weather.gov-alerts-types.txt +116 -0
  43. package/lab/demo/sources/weather-alerts/source.json +8 -0
  44. package/lab/evals-core.mjs +584 -167
  45. package/lab/flows/record.mjs +16 -4
  46. package/lab/kinds/list.mjs +32 -23
  47. package/lab/run-evals.js +3 -3
  48. package/lab/server.py +913 -119
  49. package/lab/web/dist/assets/gallery-CFntjQFG.js +5 -0
  50. package/lab/web/dist/assets/main-BlTG2i20.css +1 -0
  51. package/lab/web/dist/assets/main-D6MNWqak.js +17 -0
  52. package/lab/web/dist/assets/tokens-DwzzdSIF.css +1 -0
  53. package/lab/web/dist/assets/tokens-bvuNFuYg.js +60 -0
  54. package/lab/web/dist/gallery.html +3 -3
  55. package/lab/web/dist/index.html +4 -4
  56. package/package.json +10 -2
  57. package/lab/web/dist/assets/gallery-BcvmOPqb.js +0 -3
  58. package/lab/web/dist/assets/main-BBUxHZ-D.js +0 -16
  59. package/lab/web/dist/assets/main-CMoyjImZ.css +0 -1
  60. package/lab/web/dist/assets/tokens-B7N4NvmM.css +0 -1
  61. package/lab/web/dist/assets/tokens-CiJkwD3V.js +0 -60
@@ -146,8 +146,7 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
146
146
  held inline, run at the step's `stage` (#432: a set has no stage of its
147
147
  own; the step runs its filters of that stage). A job may hold one `filters` step of
148
148
  each stage -- a `text` one run on the raw reply before it is read into
149
- items (before Read as), an `items` one run on the items after (after Read
150
- as). */
149
+ items (Pre-format), an `items` one run on the items after (Post-format). */
151
150
 
152
151
 
153
152
 
@@ -167,13 +166,11 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
167
166
  /** A target step that sends a Power Automate step's request
168
167
  (docs/power-automate.md § The HTTP Request step): the flow's own template
169
168
  -- method, path, query and body, Workflow Definition Language expressions
170
- and all -- with the target's words and fields placed in the body by its
171
- HTTP API, evaluated against each item's record. The profile gives the
172
- address and the key. */
169
+ and all -- with the target's words and fields placed in the body by the
170
+ job's HTTP API (its flow step's), evaluated against each item's record.
171
+ The profile gives the address and the key. */
173
172
 
174
173
 
175
-
176
-
177
174
 
178
175
 
179
176
 
@@ -221,9 +218,11 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
221
218
 
222
219
 
223
220
  /** What a target's step says beside its type: what to ask in that job,
224
- and whom, where it names a profile of its own. */
221
+ and whom, where it names an AI Connection of its own -- and, optionally,
222
+ a Connection Profile over it (version 16). */
225
223
 
226
224
 
225
+
227
226
 
228
227
 
229
228
 
@@ -239,7 +238,7 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
239
238
 
240
239
 
241
240
  /** What a target sends in one job: one of the target step types. */
242
-
241
+
243
242
 
244
243
  /** A target (pipeline-model §16): one column through every job, compared
245
244
  side by side in Results -- what version 9 called a scenario. Its step in
@@ -249,8 +248,11 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
249
248
 
250
249
 
251
250
 
252
-
253
-
251
+
252
+
253
+
254
+
255
+
254
256
 
255
257
 
256
258
 
@@ -345,7 +347,7 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
345
347
  * at. `filters` are tried in order -- each drop-item narrows the items the
346
348
  * next reads, and the first reject-reply that fires gates the answer.
347
349
  */
348
- /** Which side of Read as a filter runs on (#405): `text` the raw reply before
350
+ /** Which side of Response Format a filter runs on (#405): `text` the raw reply before
349
351
  it is read into items (rewrite-text / reject-text), `items` the items after
350
352
  (drop-item, rewrite-item, keep-first, reject-reply). A filter's type decides
351
353
  its stage; a set belongs to none (#432), and runs its share of filters in
@@ -356,6 +358,9 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
356
358
  reply; item, list and answer filters read the items it is read into. */
357
359
  export const stageOfScope = (scope ) => (scope === "text" ? "text" : "items");
358
360
 
361
+ /** A filter stage as the page names it: before the reply is formatted, or after. */
362
+ export const FILTER_STAGE_LABEL = { text: "Pre-format", items: "Post-format" };
363
+
359
364
 
360
365
 
361
366
 
@@ -401,11 +406,16 @@ export const stageOfScope = (scope ) => (scope === "text" ?
401
406
 
402
407
 
403
408
 
404
- /** A Setup profile as a run carries it: request settings, never the key. */
409
+ /** What a target asked, as a run carries it: the AI Connection and its
410
+ Connection Profile's settings resolved into one -- request settings,
411
+ never the key. */
405
412
 
406
413
 
407
-
414
+
408
415
 
416
+
417
+
418
+
409
419
 
410
420
 
411
421
 
@@ -420,6 +430,8 @@ export const stageOfScope = (scope ) => (scope === "text" ?
420
430
  /** A pipeline resolved at submit (§5): what the queue runs and History
421
431
  reads. The additions, and nothing else. */
422
432
 
433
+
434
+
423
435
 
424
436
 
425
437
 
@@ -455,6 +467,38 @@ export const stageOfScope = (scope ) => (scope === "text" ?
455
467
 
456
468
 
457
469
 
470
+ /**
471
+ * An AI Connection as Setup stores it (version 16): where the lab sends a
472
+ * prompt -- its type, address, key and model, and any of its type's settings
473
+ * that say how it is reached (`scope: "connection"`). Its key is one typed
474
+ * in this browser, or KEY_HELD and its id, or empty.
475
+ */
476
+
477
+
478
+
479
+
480
+
481
+
482
+
483
+
484
+
485
+
486
+
487
+ /**
488
+ * A Connection Profile as Setup stores it: settings a target may carry over
489
+ * its connection -- model and image settings, each optional, an absent one
490
+ * the lab's default -- and, optionally, the connection it answers on in
491
+ * place of the target's. One split from a Target profile is stamped with
492
+ * that profile's id (`fromProfile`), which is what an upgrade finds it by.
493
+ */
494
+
495
+
496
+
497
+
498
+
499
+
500
+
501
+
458
502
  /** A connection's settings in one flat set, the shape a type reads. */
459
503
 
460
504
 
@@ -468,18 +512,22 @@ export const stageOfScope = (scope ) => (scope === "text" ?
468
512
  /** A pipeline read from YAML, references remapped to this lab. */
469
513
 
470
514
 
471
-
515
+
472
516
 
473
517
 
474
518
 
475
519
 
476
- /** A Target profile as an import or export matches it: its slug, where it
520
+ /** An AI Connection as an import or export matches it: its slug, where it
477
521
  has one -- withSlugs gives the rest theirs. */
478
522
 
479
523
 
480
524
  /** What an import matches references against: { id, name } each. */
481
525
 
526
+
527
+
482
528
 
529
+
530
+
483
531
 
484
532
 
485
533
 
@@ -487,6 +535,8 @@ export const stageOfScope = (scope ) => (scope === "text" ?
487
535
 
488
536
 
489
537
 
538
+
539
+
490
540
 
491
541
 
492
542
  /** An import's outcome: the remapped pipeline, or one sentence refusing it. */
@@ -501,7 +551,7 @@ export const stageOfScope = (scope ) => (scope === "text" ?
501
551
 
502
552
 
503
553
 
504
-
554
+
505
555
 
506
556
 
507
557
 
@@ -576,10 +626,24 @@ export function kept(text , max = KEPT_CHARS) {
576
626
 
577
627
 
578
628
 
629
+
630
+
631
+
632
+
579
633
 
580
634
 
581
635
 
582
636
 
637
+ /** Which filter discarded a job's reply, or emptied it, and where (#435). */
638
+
639
+
640
+
641
+
642
+
643
+
644
+
645
+
646
+
583
647
  /** What runPipeline is told besides the stages. */
584
648
 
585
649
 
@@ -943,6 +1007,9 @@ export function kept(text , max = KEPT_CHARS) {
943
1007
 
944
1008
 
945
1009
 
1010
+
1011
+
1012
+
946
1013
 
947
1014
 
948
1015
  /**
@@ -1000,7 +1067,12 @@ export function kept(text , max = KEPT_CHARS) {
1000
1067
  the lab holds. Each is optional, and a
1001
1068
  reference nothing is given to look up is not checked. */
1002
1069
 
1003
-
1070
+
1071
+
1072
+
1073
+
1074
+
1075
+
1004
1076
 
1005
1077
 
1006
1078
 
@@ -1299,7 +1371,7 @@ const SLOTS = ["content", "target", "responses"];
1299
1371
 
1300
1372
 
1301
1373
 
1302
-
1374
+
1303
1375
 
1304
1376
 
1305
1377
 
@@ -1578,6 +1650,10 @@ function datasetRules(body ) {
1578
1650
 
1579
1651
 
1580
1652
 
1653
+
1654
+
1655
+
1656
+
1581
1657
 
1582
1658
 
1583
1659
 
@@ -1641,6 +1717,10 @@ function datasetRules(body ) {
1641
1717
 
1642
1718
 
1643
1719
 
1720
+
1721
+
1722
+
1723
+
1644
1724
 
1645
1725
 
1646
1726
 
@@ -2092,6 +2172,7 @@ CONNECTION_TYPES.anthropic = {
2092
2172
  // Asked words (a Prompt step, a grader) or sent a flow's whole request (an
2093
2173
  // HTTP Request step): one profile and one key do both.
2094
2174
  answers: ["prompt", "request"],
2175
+ speaks: () => "anthropic",
2095
2176
  // The Messages API: the x-api-key and anthropic-version headers, base64
2096
2177
  // image content blocks, and /v1/models for the list.
2097
2178
  request(conn, prompt, dataUrl, base, key) {
@@ -2417,14 +2498,23 @@ CONNECTION_TYPES.http = {
2417
2498
  id: "http", label: "HTTP endpoint",
2418
2499
  description: "Any API a Power Automate step calls: an address, a key and how the key is sent.",
2419
2500
  settings: [
2420
- { key: "auth", label: "Key header", control: "select", default: "bearer", hint: "",
2501
+ { key: "auth", label: "Key header", control: "select", default: "bearer", hint: "", scope: "connection",
2421
2502
  choices: [{ value: "bearer", label: "Authorization: Bearer" }, { value: "x-api-key", label: "x-api-key" },
2422
2503
  { value: "api-key", label: "api-key" }, { value: "none", label: "none" }] },
2423
- { key: "headers", label: "Headers", control: "text", default: "", hint: "" },
2504
+ { key: "headers", label: "Headers", control: "text", default: "", hint: "", scope: "connection" },
2424
2505
  // What Test connection asks of it: a GET, keyed as a run is.
2425
- { key: "testPath", label: "Test path", control: "text", default: "", hint: "" },
2506
+ { key: "testPath", label: "Test path", control: "text", default: "", hint: "", scope: "connection" },
2507
+ // Read off the address (httpApiFor); set by hand for one the lab cannot
2508
+ // tell, such as a gateway in front of a provider.
2509
+ { key: "requestFormat", label: "Request format", control: "select", default: "", hint: "", scope: "connection",
2510
+ choices: [{ value: "", label: "From address" },
2511
+ ...Object.entries(HTTP_APIS).map(([value, a]) => ({ value, label: a.label }))] },
2426
2512
  ],
2427
2513
  answers: ["request"],
2514
+ speaks(conn) {
2515
+ const api = String(conn.requestFormat || "") || httpApiFor(originBase(conn.url));
2516
+ return HTTP_APIS[api] && api !== "raw" ? api : null;
2517
+ },
2428
2518
  lists: false,
2429
2519
  baseOf: originBase,
2430
2520
  send(conn, req, base, key) {
@@ -3100,22 +3190,30 @@ async function runPipeline(stages , dataUrl , cal
3100
3190
  // and that answer is graded as though the job had worked.
3101
3191
  const handsOn = i < stages.length - 1;
3102
3192
  const against = res.readAgainst ?? echo;
3103
- // The before-Read-as (text) filters shape the raw reply before the kind
3193
+ // The Pre-format (text) filters shape the raw reply before the kind
3104
3194
  // reads it (#405): a Rewrite text reshapes it, a Reject text ends the job.
3105
3195
  const pre = applyFilterSet(st.before || [], { body: res.raw ?? "", items: [], instruction: against }, "text");
3106
- if (pre.reject != null) return result({ error: named(`discarded: ${pre.reject}`) });
3196
+ const by = (filter , stage , rejected ) => ({ filter, stage, job: i + 1, rejected });
3197
+ if (pre.reject != null) return result({ error: named(`discarded: ${pre.reject}`), filteredBy: by(pre.reject, "text", true) });
3107
3198
  const read = reader.read(pre.body, { echo: against, parsing: opts.parsing, settings: st.settings });
3108
3199
  if (read.error) return result({ error: named(read.error) });
3109
3200
  const mod = applyModifiers(mods[i], kind, read.value, { instruction: against, body: read.body ?? res.raw ?? "" });
3110
- if (mod.reject) return result({ error: named(`discarded: ${mod.reject}`) });
3201
+ if (mod.reject) {
3202
+ return result({ error: named(`discarded: ${mod.reject}`), ...(mod.by ? { filteredBy: by(mod.by, "items", true) } : {}) });
3203
+ }
3204
+ // A filter that dropped the last item: what a zero result is explained by.
3205
+ const emptied = mod.by && reader.terms && !reader.terms(mod.value).length ? { filteredBy: by(mod.by, "items", false) } : {};
3111
3206
  const nothing = handsOn ? reader.handsOn?.(mod.value) : null;
3112
- if (nothing) return result({ error: named(nothing) });
3207
+ if (nothing) return result({ error: named(nothing), ...emptied });
3113
3208
  const value = mod.value;
3114
3209
  held = { kind, value };
3115
3210
  const gone = [...(read.dropped || []), ...mod.dropped];
3116
3211
  echoed = (read.echoed || 0) + mod.dropped.filter(d => d.by === ECHO).length;
3117
3212
  done.push({ reply: res.raw ?? "", rendered: reader.render?.(value) || {} });
3118
- if (!handsOn) dropped = gone;
3213
+ if (!handsOn) {
3214
+ dropped = gone;
3215
+ return result({ echoed, dropped, ...emptied });
3216
+ }
3119
3217
  }
3120
3218
  return result({ echoed, dropped });
3121
3219
  }
@@ -3132,7 +3230,7 @@ function readList(reply , out , instruction =
3132
3230
  const reader = OUTPUT_KINDS[out.kind];
3133
3231
  if (!reader) return { items: [], dropped: [], echoed: 0, error: `"${out.kind}" is not an output kind this lab has` };
3134
3232
  const { kind, modifiers, before, ...settings } = out;
3135
- // The before-Read-as (text) filters shape the raw reply before the kind
3233
+ // The Pre-format (text) filters shape the raw reply before the kind
3136
3234
  // reads it (#405); a Reject text ends the job.
3137
3235
  const pre = applyFilterSet(before || [], { body: reply, items: [], instruction }, "text");
3138
3236
  if (pre.reject != null) return { items: [], dropped: [], echoed: 0, error: `discarded: ${pre.reject}` };
@@ -3153,7 +3251,7 @@ function readList(reply , out , instruction =
3153
3251
  was thrown away. */
3154
3252
  function applyModifiers (list , kind , value ,
3155
3253
  ctx = { instruction: "", body: "" }) {
3156
- let v = value;
3254
+ let v = value, by ;
3157
3255
  const dropped = [];
3158
3256
  for (const m of list || []) {
3159
3257
  const entry = MODIFIERS[m.type];
@@ -3164,9 +3262,10 @@ function applyModifiers (list , kind ,
3164
3262
  if (!outcome) { v = out ; continue; }
3165
3263
  v = outcome.value;
3166
3264
  dropped.push(...(outcome.dropped || []));
3167
- if (outcome.reject) return { value: v, dropped, reject: outcome.reject };
3265
+ if (outcome.reject) return { value: v, dropped, reject: outcome.reject, ...(outcome.by ? { by: outcome.by } : {}) };
3266
+ by = outcome.by ?? by;
3168
3267
  }
3169
- return { value: v, dropped };
3268
+ return { value: v, dropped, ...(by ? { by } : {}) };
3170
3269
  }
3171
3270
 
3172
3271
  // ---- The pipeline model ----------------------------------------------------
@@ -3205,7 +3304,12 @@ function applyModifiers (list , kind ,
3205
3304
  // 12: a Contains metric's Ignore case holds item by item too.
3206
3305
  // 13: an eval is a link to an eval group, or a group of the pipeline's own,
3207
3306
  // and the document has an overall pass rule (pipeline-model §17).
3208
- const PIPELINE_VERSION = 15 ;
3307
+ // 14: a job's Responses stage links filter sets on a `filters` step (§18).
3308
+ // 15: every cleaning step is a filter, Pre- or Post-format (#405).
3309
+ // 16: a target names an AI Connection and, optionally, a Connection Profile,
3310
+ // where it named one Target profile; a run's table is keyed by the pair
3311
+ // (answerKey) -- the Target profile split in the store as it opens (#445).
3312
+ const PIPELINE_VERSION = 17 ;
3209
3313
 
3210
3314
  // Plain objects, so an entry is added by assignment and a reader never needs
3211
3315
  // to know which registered it.
@@ -3338,7 +3442,7 @@ OUTPUT_KINDS.text = {
3338
3442
  // name is only a label and two profiles can share one. #46: the profile
3339
3443
  // carries `type`, and the run's resolved copy carries it too
3340
3444
  // (docs/pipeline-model.md §7).
3341
- const CONNECTION_FIELDS = ["name", "slug", "url", "model", "type", "temperature",
3445
+ const CONNECTION_FIELDS = ["name", "slug", "connection", "url", "model", "type", "temperature",
3342
3446
  "px", "format", "quality", "options"];
3343
3447
  const SETTING_KEYS = [...new Set(Object.values(CONNECTION_TYPES).flatMap(t => t.settings.map(s => s.key)))];
3344
3448
  const OPTION_FIELDS = ["seed", "nPredict", ...DECODING_KEYS];
@@ -3346,6 +3450,9 @@ const IMAGE_FORMATS = ["image/jpeg", "image/webp", "image/png
3346
3450
  // Ids are what a key variable is spelled from, so they stay spellable: a
3347
3451
  // letter or digit run, with at most one hyphen.
3348
3452
  const PROFILE_ID = /^[A-Za-z0-9]+(?:-[A-Za-z0-9]+)?$/;
3453
+ // A run's table key (answerKey): a connection's id, then a dot and its
3454
+ // profile's id -- or nothing, for none -- or a grader's bare id.
3455
+ const ANSWER_KEY = /^[A-Za-z0-9]+(?:-[A-Za-z0-9]+)?(?:\.(?:[A-Za-z0-9]+(?:-[A-Za-z0-9]+)?)?)?$/;
3349
3456
  const LOOKS_LIKE_A_KEY = /^(?:api[-_]?)?key$|^(?:authorization|bearer|secret|password|token)$/i;
3350
3457
 
3351
3458
  /** The variable a profile's key is read from: its slug's, when it has one --
@@ -3441,6 +3548,138 @@ function connectionSettings(c ) {
3441
3548
  return { ...rest, ...(options || {}) };
3442
3549
  }
3443
3550
 
3551
+ // ---- AI Connections and Connection Profiles (version 16) ---------------------
3552
+ // A Target profile was one stored thing: where a prompt goes and the settings
3553
+ // it goes with. It is two now -- an AI Connection (type, address, key, model)
3554
+ // and an optional Connection Profile (settings, and optionally a connection of
3555
+ // its own in place of the target's) -- and a target names both. A store from
3556
+ // before is split once as it opens (server.py `split_profiles`): each profile
3557
+ // becomes a connection keeping its id and, where it set settings, a profile
3558
+ // stamped with that id (`fromProfile`). What a run carries of a pair is the
3559
+ // two resolved into one Connection, as a profile resolved before.
3560
+
3561
+ /** What an AI Connection keeps of a Target profile, beside its type's
3562
+ connection-scoped settings. A legacy `profile` (#46) stays with the
3563
+ connection: it says the type until the page converts it. */
3564
+ const AI_CONNECTION_FIELDS = ["id", "name", "slug", "type", "url", "key", "model", "profile"];
3565
+ /** What a Connection Profile holds that is not a setting. */
3566
+ const CONNECTION_PROFILE_FIELDS = ["id", "name", "connection", "fromProfile"];
3567
+
3568
+ /** The settings of [type] that say how a connection is reached. */
3569
+ const connectionScoped = (type ) =>
3570
+ (CONNECTION_TYPES[type]?.settings ?? []).filter(s => s.scope === "connection").map(s => s.key);
3571
+
3572
+ /** Whether a stored setting is set: blank is the lab's default. */
3573
+ const setValue = (v ) => v != null && v !== "";
3574
+
3575
+ /**
3576
+ * A Target profile as an AI Connection and, where it set any setting, the
3577
+ * Connection Profile holding them -- the same id, stamped with its origin.
3578
+ * Resolved together (`mergeAnswer`) they send what the profile sent.
3579
+ * server.py's `split_profile` is its twin, held to it by proxy-check.py.
3580
+ */
3581
+ function splitProfile(p ) {
3582
+ const scoped = connectionScoped(typeOf(p ));
3583
+ const connection = {}, settings = {};
3584
+ for (const [k, v] of Object.entries(p)) {
3585
+ if (AI_CONNECTION_FIELDS.includes(k) || scoped.includes(k)) connection[k] = v;
3586
+ else if (setValue(v)) settings[k] = v;
3587
+ }
3588
+ const profile = Object.keys(settings).length
3589
+ ? { id: String(p.id ?? ""), name: String(p.name ?? ""), fromProfile: String(p.id ?? ""), ...settings } : null;
3590
+ return { connection, profile };
3591
+ }
3592
+
3593
+ /** [conn] with [profile]'s settings over it: the one set of settings a
3594
+ request is built from, the shape a Target profile had. A profile sets
3595
+ no connection field. */
3596
+ function mergeAnswer(conn , profile ) {
3597
+ const out = { ...conn };
3598
+ const scoped = connectionScoped(typeOf(conn ));
3599
+ for (const [k, v] of Object.entries(profile ?? {})) {
3600
+ if (!CONNECTION_PROFILE_FIELDS.includes(k) && !AI_CONNECTION_FIELDS.includes(k) && !scoped.includes(k) && setValue(v)) out[k] = v;
3601
+ }
3602
+ return out;
3603
+ }
3604
+
3605
+ /** Whom a target or one of its steps asks: an AI Connection, and the
3606
+ Connection Profile over it, or none. */
3607
+
3608
+
3609
+
3610
+
3611
+
3612
+ /** The key a run's table holds [connection] under, with [profile] -- or with
3613
+ none, the lab's defaults. A grader names its connection bare, its id alone,
3614
+ so it never meets a target's pair (see `answersOf`). */
3615
+ const answerKey = (connection , profile ) => `${connection}.${profile ?? ""}`;
3616
+
3617
+ /** [key] read back: a pair's connection and profile, or a bare id. */
3618
+ function answerOfKey(key ) {
3619
+ const at = key.indexOf(".");
3620
+ if (at < 0) return { connection: key, profile: null, bare: true };
3621
+ return { connection: key.slice(0, at), profile: key.slice(at + 1) || null, bare: false };
3622
+ }
3623
+
3624
+ /** Whom target [i] asks in job [k]: its step's own connection or profile,
3625
+ where it names either, else the target's. */
3626
+ function targetAnswerOf(doc , i , k ) {
3627
+ const t = targetsOf(doc)[i];
3628
+ const st = targetStepOf(doc, i, k);
3629
+ const own = st?.connection != null || st?.profile != null;
3630
+ const connection = (own ? st?.connection ?? t?.connection : t?.connection) ?? null;
3631
+ if (!isRef(connection)) return undefined;
3632
+ const profile = (own ? st?.profile : t?.profile) ?? null;
3633
+ return { connection, profile: isRef(profile) ? profile : null };
3634
+ }
3635
+
3636
+ /** The answer key of target [i] in job [k], or undefined where it asks none. */
3637
+ function targetAnswerKey(doc , i , k ) {
3638
+ const a = targetAnswerOf(doc, i, k);
3639
+ return a ? answerKey(a.connection.id, a.profile?.id) : undefined;
3640
+ }
3641
+
3642
+ /**
3643
+ * What the lab's two stores answer: each by id, the profile split from a
3644
+ * Target profile's id (what an upgrade and a grader read it by), and an
3645
+ * answer key resolved to the settings it sends -- the profile's connection
3646
+ * where it names one, else the key's, with the profile's settings over it.
3647
+ * A grader's bare id resolves as the Target profile it named did: its
3648
+ * connection under the profile split from it. Null where any part is gone.
3649
+ */
3650
+ function answersOf(connections , profiles ) {
3651
+ const connection = (id ) => connections.find(c => c?.id === id) ?? null;
3652
+ const profile = (id ) => profiles.find(p => p?.id === id) ?? null;
3653
+ const split = (id ) => {
3654
+ const p = profiles.find(x => x?.fromProfile === id);
3655
+ return p ? { id: p.id, name: p.name } : null;
3656
+ };
3657
+ const resolve = (key ) => {
3658
+ const at = answerOfKey(key);
3659
+ const pid = at.bare ? split(at.connection)?.id ?? null : at.profile;
3660
+ const p = pid ? profile(pid) : null;
3661
+ if (pid && !p) return null;
3662
+ const c = connection(isRef(p?.connection) ? p .connection .id : at.connection);
3663
+ return c ? mergeAnswer(c, p) : null;
3664
+ };
3665
+ return { connection, profile, split, resolve };
3666
+ }
3667
+
3668
+ /** The connection a run carries for [key]: its own entry, or -- a run from
3669
+ before version 16, whose table is keyed by Target profile id, and names
3670
+ the connection alone once upgraded -- the one under the connection's id. */
3671
+ function runAnswer(run , key ) {
3672
+ const table = run?.profiles ;
3673
+ if (!isObj(table)) return undefined;
3674
+ if (Object.hasOwn(table, key)) return table[key];
3675
+ const at = answerOfKey(key);
3676
+ return !at.bare && at.profile == null && Object.hasOwn(table, at.connection) ? table[at.connection] : undefined;
3677
+ }
3678
+
3679
+ /** The id of the AI Connection whose key a run's entry [key] is sent with. */
3680
+ const answerConnectionId = (key , conn ) =>
3681
+ (isStr(conn?.connection) && conn .connection) || answerOfKey(key).connection;
3682
+
3444
3683
  // The rest of what a connection's fields mean lives in the connection-type
3445
3684
  // registry above (#46): the runner asks a type how to build a request and how
3446
3685
  // to read a reply, and validation asks connectionProblems (below, with the run
@@ -3721,7 +3960,7 @@ DYNAMIC_PROVIDERS["command"] = { // provider: the Command adapter's own id
3721
3960
  holds, so a tick costs a predicate and not a request. A reader names a field
3722
3961
  it needs; a wizard that reads more declares it here. */
3723
3962
 
3724
-
3963
+
3725
3964
 
3726
3965
 
3727
3966
 
@@ -4091,14 +4330,14 @@ LEGACY_TESTS.metrics = {
4091
4330
  if (t.grader != null && !isRef(t.grader)) bad.push("the Metrics name their grader as { id, name }");
4092
4331
  // The eval's own grader, or the lab's.
4093
4332
  const grader = isRef(t.grader) ? t.grader : isRef(ctx.grader) ? ctx.grader : null;
4094
- if (gradedIn(t.metrics) && !grader) bad.push("a model-graded metric needs a grader: pick one on the eval, or make a Target profile the lab's grader");
4333
+ if (gradedIn(t.metrics) && !grader) bad.push("a model-graded metric needs a grader: pick one on the eval, or make a connection the lab's grader");
4095
4334
  // A grader is asked words, and needs a model to ask.
4096
4335
  const p = grader && ctx.profiles ? ctx.profiles(grader.id) : null;
4097
- if (grader && ctx.profiles && !p) bad.push(`Target profile ${grader.name || grader.id} not found`);
4336
+ if (grader && ctx.profiles && !p) bad.push(`Connection ${grader.name || grader.id} not found`);
4098
4337
  else if (p) {
4099
4338
  const type = CONNECTION_TYPES[typeOf(p )];
4100
4339
  if (type && !(type.answers ?? ["prompt"]).includes("prompt")) bad.push(`the grader has to be asked words, and ${type.label} is not`);
4101
- else if (!type?.local && !String((p ).model || "").trim()) bad.push("no model on the grader's Target profile");
4340
+ else if (!type?.local && !String((p ).model || "").trim()) bad.push("no model on the grader's connection");
4102
4341
  }
4103
4342
  },
4104
4343
  rules: t => (Array.isArray(t.metrics) ? t.metrics : []).map((m , x ) => ({
@@ -4226,14 +4465,14 @@ EVAL_TYPES.group = {
4226
4465
  if (own.run.length && !wholeRunGroup(t)) bad.push("the group's own body reads each item or the whole run, not both");
4227
4466
  // The group's own grader, or the lab's.
4228
4467
  const grader = isRef(own.grader) ? own.grader : isRef(ctx.grader) ? ctx.grader : null;
4229
- if (gradedIn(own.every) && !grader) bad.push("a model-graded metric needs a grader: pick one on the eval, or make a Target profile the lab's grader");
4468
+ if (gradedIn(own.every) && !grader) bad.push("a model-graded metric needs a grader: pick one on the eval, or make a connection the lab's grader");
4230
4469
  // A grader is asked words, and needs a model to ask.
4231
4470
  const p = grader && ctx.profiles ? ctx.profiles(grader.id) : null;
4232
- if (grader && ctx.profiles && !p) bad.push(`Target profile ${grader.name || grader.id} not found`);
4471
+ if (grader && ctx.profiles && !p) bad.push(`Connection ${grader.name || grader.id} not found`);
4233
4472
  else if (p) {
4234
4473
  const type = CONNECTION_TYPES[typeOf(p )];
4235
4474
  if (type && !(type.answers ?? ["prompt"]).includes("prompt")) bad.push(`the grader has to be asked words, and ${type.label} is not`);
4236
- else if (!type?.local && !String((p ).model || "").trim()) bad.push("no model on the grader's Target profile");
4475
+ else if (!type?.local && !String((p ).model || "").trim()) bad.push("no model on the grader's connection");
4237
4476
  }
4238
4477
  },
4239
4478
  rules: t => ownList(t).map((m , x ) => ({
@@ -4565,7 +4804,7 @@ STEP_TYPES.prompt = {
4565
4804
  description: "Asks each target's model its prompt about the item.",
4566
4805
  in: "item", out: "text",
4567
4806
  apply: "runPipeline",
4568
- fields: ["type", "prompt", "profile", "from"],
4807
+ fields: ["type", "prompt", "connection", "profile", "from"],
4569
4808
  validate(step, _ctx, bad, at){
4570
4809
  if (!isStr(step.prompt)) bad.push(`${at}: prompt has to be text`);
4571
4810
  },
@@ -4591,16 +4830,18 @@ STEP_TYPES.httpRequest = {
4591
4830
  label: "HTTP Request", slot: "target", asks: "request", verbatim: true, modelFrom: "step",
4592
4831
  description: "Sends a Power Automate step's own request, rebuilt from each record, with each target's words in it.",
4593
4832
  firstJobOnly: "an HTTP Request is sent from its item's record, so it is job 1's",
4594
- // A new target starts from the flow's own words and fields.
4595
- newStep: (step) => ({ type: "httpRequest", api: step.api, method: step.method, path: step.path,
4833
+ // A new target starts from the flow's own words and fields, read by the
4834
+ // job's HTTP API ([step] carries it: requestStepOf).
4835
+ newStep: (step) => ({ type: "httpRequest", method: step.method, path: step.path,
4596
4836
  query: clone(step.query), body: clone(step.body),
4597
4837
  ...(HTTP_APIS[step.api] ?? HTTP_APIS.raw ).cellOf(step.body) }),
4598
4838
  in: "item", out: "text",
4599
4839
  apply: "runPipeline",
4600
- fields: ["type", "api", "method", "path", "query", "body", "prompt", "profile", "from", "system", "model", "settings", "prefill"],
4840
+ fields: ["type", "method", "path", "query", "body", "prompt", "connection", "profile", "from", "system", "model", "settings", "prefill"],
4841
+ // [step] carries its job's HTTP API, where the job has a flow step to say
4842
+ // it (validatePipeline); the flow step's own check names an unknown one.
4601
4843
  validate(step, _ctx, bad, at){
4602
4844
  const api = HTTP_APIS[step.api];
4603
- if (!api) bad.push(`${at}: "${step.api}" is not an HTTP API this lab has`);
4604
4845
  if (!HTTP_METHODS.includes(step.method)) bad.push(`${at}: the method is one of ${HTTP_METHODS.join(", ")}`);
4605
4846
  if (!isStr(step.path) || !/^[/@]/.test(step.path)) bad.push(`${at}: the path starts with /`);
4606
4847
  if (!isObj(step.query) || !Object.values(step.query).every(isStr)) bad.push(`${at}: the query is names to text`);
@@ -4608,7 +4849,7 @@ STEP_TYPES.httpRequest = {
4608
4849
  // A key is the Setup profile's, sent by its Key header; one in the
4609
4850
  // template would be kept in the pipeline and every run of it.
4610
4851
  const hit = keyField(step.body) ?? keyField(step.query, "query");
4611
- if (hit) bad.push(`${at} carries a key at ${hit}, and a key never goes in a pipeline -- the Target profile sends it`);
4852
+ if (hit) bad.push(`${at} carries a key at ${hit}, and a key never goes in a pipeline -- the connection sends it`);
4612
4853
  if (!api) return;
4613
4854
  // Its fields, where its HTTP API has a place for them.
4614
4855
  for (const f of ["system", "model", "prefill"] ) {
@@ -4723,7 +4964,7 @@ STEP_TYPES.modifier = {
4723
4964
 
4724
4965
  // A filters step: the filter sets a job cleans and gates its reply by, each a
4725
4966
  // link or a private set (#339, #405), at the step's `stage`. A job may hold
4726
- // one of each stage -- a `text` one before Read as, an `items` one after -- so
4967
+ // one of each stage -- a `text` one Pre-format, an `items` one Post-format -- so
4727
4968
  // the type is `many`; two of a stage is the job's to refuse. A set belongs to
4728
4969
  // no stage (#432): linked here, it runs the filters whose type fits the step's
4729
4970
  // stage and skips the rest. A private set's body is the core's to judge
@@ -4824,11 +5065,6 @@ function jobTargetType(doc
4824
5065
  return targetStepOf(doc, 0, k)?.type ?? "prompt";
4825
5066
  }
4826
5067
 
4827
- /** The Setup profile target [i] asks in job [k]: its step's own, or the target's. */
4828
- function targetProfileOf(doc , i , k ) {
4829
- return targetStepOf(doc, i, k)?.profile ?? targetsOf(doc)[i]?.profile ?? undefined;
4830
- }
4831
-
4832
5068
  /** Whether target [i] is a Recorded target: it replays each call's recorded
4833
5069
  reply rather than sending anything (its job-1 step answers locally with a
4834
5070
  connection that `replays`). A recorded-reply metric compares the recorded
@@ -4882,7 +5118,7 @@ function requestStepOf(doc
4882
5118
 
4883
5119
  /** What httpRequestOf builds a request from: a target's template, and the
4884
5120
  loop its flow step sits in. */
4885
-
5121
+
4886
5122
  /** What httpReplyOf reads a reply with: the flow step, its API, its reading. */
4887
5123
 
4888
5124
 
@@ -4958,8 +5194,8 @@ function withOut(job , out ) {
4958
5194
  }
4959
5195
 
4960
5196
  /** [job] reading a reply as `legacyTagsOut` builds it, on the current model
4961
- (#405): its output a List, a before-Read-as text set that reads the reply as
4962
- the old `preamble` did, and an after-Read-as items set that cleans and gates
5197
+ (#405): its output a List, a Pre-format text set that reads the reply as
5198
+ the old `preamble` did, and a Post-format items set that cleans and gates
4963
5199
  it. The filters are `legacyTagsOut`'s, placed as private sets -- what the
4964
5200
  version-15 upgrade builds for a legacy list job, for a test to build
4965
5201
  directly. */
@@ -4978,7 +5214,7 @@ const filterStepOf = (job , stage )
4978
5214
 
4979
5215
  /** The filter-set links a job's Responses stage carries for [stage], in
4980
5216
  document order (#339, #405): read from that stage's filters step, or none.
4981
- Defaults to the after-Read-as (items) stage, the one that was there before. */
5217
+ Defaults to the Post-format (items) stage, the one that was there before. */
4982
5218
  function filterSetsOf(job , stage = "items") {
4983
5219
  return (filterStepOf(job, stage)?.filterSets ?? []).slice();
4984
5220
  }
@@ -4998,10 +5234,10 @@ function withFilterSets(job , links , stage
4998
5234
  const linkFilters = (link , resolve ) =>
4999
5235
  (link.own ?? (link.set && resolve ? resolve(link.set) : null))?.filters || [];
5000
5236
 
5001
- /** A job's after-Read-as (items) Responses as an ordered modifier list for a
5237
+ /** A job's Post-format (items) Responses as an ordered modifier list for a
5002
5238
  run (#339): each modifier step as it is, and each items filter-set link as a
5003
5239
  synthetic modifier that applies the set's filters where the link sits. The
5004
- before-Read-as (text) filters are not modifiers -- `beforeFilters` gives
5240
+ Pre-format (text) filters are not modifiers -- `beforeFilters` gives
5005
5241
  them, and a run applies them to the raw reply before it is read. [resolve]
5006
5242
  gives a Library link's body. */
5007
5243
  function stageModifiers(job , resolve ) {
@@ -5016,7 +5252,7 @@ function stageModifiers(job , resolve
5016
5252
  return out;
5017
5253
  }
5018
5254
 
5019
- /** A job's before-Read-as (text) filters for a run (#405), in document order:
5255
+ /** A job's Pre-format (text) filters for a run (#405), in document order:
5020
5256
  the filters of every link on its `text` filters step, which a run applies to
5021
5257
  the raw reply before the kind reads it into items. [resolve] gives a Library
5022
5258
  link's body, as `stageModifiers` does. */
@@ -5067,11 +5303,11 @@ STEP_TYPES.job = {
5067
5303
  }
5068
5304
  const twice = steps.find((st, x) => !STEP_TYPES[st.type] .many && steps.findIndex(o => o.type === st.type) !== x);
5069
5305
  if (twice) return void bad.push(`${at} has two ${STEP_TYPES[twice.type] .label ?? twice.type} steps`);
5070
- // A `filters` step is `many`, but only one per stage: a before-Read-as and
5071
- // an after-Read-as set at most (#405).
5306
+ // A `filters` step is `many`, but only one per stage: a Pre-format and
5307
+ // a Post-format set at most (#405).
5072
5308
  const twoFilters = steps.find((st, x) => st.type === "filters"
5073
5309
  && steps.findIndex(o => o.type === "filters" && o.stage === st.stage) !== x);
5074
- if (twoFilters) return void bad.push(`${at} has two ${twoFilters.stage === "text" ? "before Read as" : "after Read as"} filter steps`);
5310
+ if (twoFilters) return void bad.push(`${at} has two ${twoFilters.stage === "text" ? "Pre-format" : "Post-format"} filter steps`);
5075
5311
  if (steps.some((st, x) => x > 0 && placeOf(st) < placeOf(steps[x - 1]))) {
5076
5312
  return void bad.push(`${at}: steps are its Content, then its Responses, each in order`);
5077
5313
  }
@@ -5173,10 +5409,16 @@ function versionProblem(doc ) {
5173
5409
  rules, as a run with no graded eval parsed. */
5174
5410
 
5175
5411
 
5176
-
5412
+
5177
5413
 
5178
-
5414
+
5415
+
5179
5416
 
5417
+
5418
+
5419
+
5420
+
5421
+
5180
5422
 
5181
5423
 
5182
5424
  /**
@@ -5227,8 +5469,10 @@ function columnsOf(doc )
5227
5469
  function profileRefs(doc ) {
5228
5470
  const cut = (r ) => (isObj(r) && isStr(r.id) ? { id: r.id, name: isStr(r.name) ? r.name : "" } : r);
5229
5471
  for (const { column, steps } of columnsOf(doc)) {
5230
- if ("profile" in column) column.profile = cut(column.profile);
5231
- for (const st of steps) if (st.profile != null) st.profile = cut(st.profile);
5472
+ for (const f of ["connection", "profile"]) {
5473
+ if (f in column) column[f] = cut(column[f]);
5474
+ for (const st of steps) if (st[f] != null) st[f] = cut(st[f]);
5475
+ }
5232
5476
  }
5233
5477
  }
5234
5478
 
@@ -5238,7 +5482,40 @@ const cutRefs = (doc ) => { profileRefs(doc); return doc; }
5238
5482
  /** Whether any profile reference in a pipeline holds more than { id, name }. */
5239
5483
  function fatProfileRef(doc ) {
5240
5484
  const fat = (r ) => isObj(r) && Object.keys(r).some(k => k !== "id" && k !== "name");
5241
- return isObj(doc) && columnsOf(doc).some(({ column, steps }) => fat(column.profile) || steps.some(st => fat(st.profile)));
5485
+ return isObj(doc) && columnsOf(doc).some(({ column, steps }) => fat(column.profile) || fat(column.connection)
5486
+ || steps.some(st => fat(st.profile) || fat(st.connection)));
5487
+ }
5488
+
5489
+ /**
5490
+ * Version 15 to 16: a Target profile is an AI Connection now, keeping the
5491
+ * profile's id, and -- where the profile set settings -- the Connection
5492
+ * Profile split from it ([ctx]'s `connectionSplit`), so each target and step
5493
+ * asks what it asked. A run document carries each profile resolved, under
5494
+ * its id (`runAnswer` reads it so), so it names the connection alone, as
5495
+ * does a reader that cannot say what was split. Idempotent: a target that
5496
+ * names a connection already is left as it is.
5497
+ */
5498
+ function answersFromProfiles(doc , ctx ) {
5499
+ doc.version = PIPELINE_VERSION;
5500
+ const split = (ref ) => (isObj(doc.profiles) || !isObj(ref) || !isStr(ref.id) ? null
5501
+ : ctx.connectionSplit?.(ref.id) ?? null);
5502
+ if (!Array.isArray(doc.targets)) return doc;
5503
+ doc.targets = doc.targets.map((t ) => {
5504
+ if (!isObj(t) || "connection" in t) return t;
5505
+ const { id, name, profile = null, steps, ...rest } = t;
5506
+ const p = split(profile);
5507
+ const out = { id, name, connection: profile, profile: p ? { id: p.id, name: p.name ?? "" } : null, steps, ...rest };
5508
+ if (Array.isArray(steps)) {
5509
+ out.steps = steps.map((st ) => {
5510
+ if (!isObj(st) || st.profile == null || "connection" in st) return st;
5511
+ const { profile: own, ...more } = st;
5512
+ const sp = split(own);
5513
+ return { ...more, connection: own, ...(sp ? { profile: { id: sp.id, name: sp.name ?? "" } } : {}) };
5514
+ });
5515
+ }
5516
+ return out;
5517
+ });
5518
+ return doc;
5242
5519
  }
5243
5520
 
5244
5521
  /** Version 5 to 6: the one test, or none, as a list. Its id is `t1`, so
@@ -5443,32 +5720,38 @@ function upgradePipeline (doc , ctx = {}) {
5443
5720
  }
5444
5721
  return localSteps(ownBodiesNow(out), ctx) ;
5445
5722
  }
5446
- if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14].includes(doc.version )) return doc;
5723
+ if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16].includes(doc.version )) return doc;
5724
+ // Every version reads as 16 first, then as 17: its requests read by their
5725
+ // job's HTTP API (requestsOnJobApi).
5726
+ const seventeen = (d ) => localSteps(ownBodiesNow(requestsOnJobApi(d)), ctx) ;
5727
+ if (doc.version === 16) return seventeen(cutRefs(clone(doc) ));
5728
+ // Every version reads as 15 first, then as 16: its Target profiles split
5729
+ // into AI Connections and Connection Profiles (answersFromProfiles).
5730
+ const sixteen = (fifteen ) => seventeen(answersFromProfiles(fifteen, ctx));
5731
+ if (doc.version === 15) return sixteen(cutRefs(clone(doc) ));
5447
5732
  // Versions 13 and 14 read as 15 by migrating a job's cleaning modifier steps
5448
- // and its List `preamble` setting into before/after-Read-as filter sets
5733
+ // and its List `preamble` setting into Pre-/Post-format filter sets
5449
5734
  // (#339, #405). A version-14 document already moved its Drop items / Reject
5450
5735
  // into a (stageless) filters step; filtersFromCleaning stages it and folds in
5451
5736
  // the remaining Tidy / Remove duplicates / Keep first / Case.
5452
- if (doc.version === 13 || doc.version === 14) {
5453
- return localSteps(ownBodiesNow(filtersFromCleaning(clone(doc) )), ctx) ;
5454
- }
5737
+ if (doc.version === 13 || doc.version === 14) return sixteen(filtersFromCleaning(clone(doc) ));
5455
5738
  if (doc.version === 11 || doc.version === 12) {
5456
- return localSteps(ownBodiesNow(filtersFromCleaning(groupsOf(withoutCaseMetric(caseAsWritten(cutRefs(clone(doc) )))))), ctx) ;
5739
+ return sixteen(filtersFromCleaning(groupsOf(withoutCaseMetric(caseAsWritten(cutRefs(clone(doc) ))))));
5457
5740
  }
5458
5741
  // Every version before 10 reads as version 9 first, then as 10, then up
5459
5742
  // through 15. A version-10 document is cut as a current one was
5460
5743
  // (profileRefs); the earlier ones are cut on their way through nineOf.
5461
5744
  const ten = doc.version === 10 ? cutRefs(clone(doc) ) : targetsFromScenarios(nineOf(clone(doc) , ctx));
5462
- return localSteps(ownBodiesNow(filtersFromCleaning(groupsOf(withoutCaseMetric(caseAsWritten(evalsKey(ten)))))), ctx) ;
5745
+ return sixteen(filtersFromCleaning(groupsOf(withoutCaseMetric(caseAsWritten(evalsKey(ten))))));
5463
5746
  }
5464
5747
 
5465
5748
  /** [doc] with every job's cleaning modifier steps and its List `preamble`
5466
5749
  setting migrated into filter sets on `filters` steps, up to version 15
5467
- (#339, #405). The after-Read-as (items) filters are built by walking the
5750
+ (#339, #405). The Post-format (items) filters are built by walking the
5468
5751
  Responses steps in document order -- each cleaning modifier its filters
5469
5752
  (`cleaningFilters`), each existing filters step's links in place, a private
5470
5753
  one's filters coalescing with adjacent modifier filters into one own set --
5471
- and the `preamble: true` setting becomes a before-Read-as (text) set of
5754
+ and the `preamble: true` setting becomes a Pre-format (text) set of
5472
5755
  Remove thinking then Remove preamble. The steps take the place of the first
5473
5756
  of them, and the preamble setting is stripped.
5474
5757
 
@@ -5556,12 +5839,20 @@ function withoutCaseMetric(doc ) {
5556
5839
  step then asks none, or names its own, keeps no profile either. The same
5557
5840
  document, untouched, when there is nothing to read so. */
5558
5841
  function localSteps(doc , ctx ) {
5559
- const typeOfProfile = ctx.profileType
5560
- ?? (isObj(doc.profiles) ? (id ) => (isObj(doc.profiles[id]) ? doc.profiles[id].type : undefined) : null);
5561
- if (!typeOfProfile) return doc;
5842
+ const run = isObj(doc.profiles);
5843
+ if (!ctx.profileType && !run) return doc;
5844
+ // A connection's type: [ctx]'s, by its id, or what a run carries for it.
5845
+ const typeOfAnswer = (connection , profile ) => {
5846
+ if (!isObj(connection) || !isStr(connection.id)) return undefined;
5847
+ if (ctx.profileType) return ctx.profileType(connection.id);
5848
+ const conn = runAnswer(doc, answerKey(connection.id, isObj(profile) && isStr(profile.id) ? profile.id : null));
5849
+ return isObj(conn) ? conn.type : undefined;
5850
+ };
5562
5851
  const stands = new Map(Object.entries(STEP_TYPES).filter(([, e]) => e.slot === "target" && e.replaces).map(([type, e]) => [e.replaces , type]));
5563
- const as = (ref ) => (isObj(ref) && isStr(ref.id) ? stands.get(typeOfProfile(ref.id) ?? "") : undefined);
5564
- const turns = (t , st ) => isObj(st) && STEP_TYPES[st.type]?.asks === "prompt" && as(st.profile ?? t.profile);
5852
+ const as = (connection , profile ) => stands.get(typeOfAnswer(connection, profile) ?? "");
5853
+ const ownsAnswer = (st ) => st.connection != null || st.profile != null;
5854
+ const turns = (t , st ) => isObj(st) && STEP_TYPES[st.type]?.asks === "prompt"
5855
+ && (ownsAnswer(st) ? as(st.connection ?? t.connection, st.profile) : as(t.connection, t.profile));
5565
5856
  const targets = Array.isArray(doc.targets) ? doc.targets : [];
5566
5857
  if (!targets.some(t => isObj(t) && Array.isArray(t.steps) && t.steps.some((st ) => turns(t, st)))) return doc;
5567
5858
  const next = clone(doc) ;
@@ -5571,9 +5862,43 @@ function localSteps(doc , ctx ) {
5571
5862
  const type = turns(t, st);
5572
5863
  return type ? { type, prompt: st.prompt, ...(st.from ? { from: st.from } : {}) } : st;
5573
5864
  });
5574
- // A run keeps the profile it ran as, so its report names it as it did.
5575
- if (!isObj(doc.profiles) && as(t.profile)
5576
- && t.steps.every((st ) => isObj(st) && (STEP_TYPES[st.type]?.local || st.profile != null))) t.profile = null;
5865
+ // A run keeps the connection it ran as, so its report names it as it did.
5866
+ if (!run && as(t.connection, t.profile)
5867
+ && t.steps.every((st ) => isObj(st) && (STEP_TYPES[st.type]?.local || ownsAnswer(st)))) {
5868
+ t.connection = null;
5869
+ t.profile = null;
5870
+ }
5871
+ }
5872
+ return next;
5873
+ }
5874
+
5875
+ /**
5876
+ * Version 16 to 17 (#477): a request step has no HTTP API of its own; its
5877
+ * job's flow step says it. A step that named another -- Any endpoint over an
5878
+ * Anthropic flow, say -- keeps its body as it would have sent it, read again
5879
+ * by the job's (its words and fields as that API holds them). A request in a
5880
+ * job with no flow step, which could never run, is a Prompt keeping its words.
5881
+ */
5882
+ function requestsOnJobApi(next ) {
5883
+ next.version = PIPELINE_VERSION;
5884
+ const jobs = Array.isArray(next.jobs) ? next.jobs : [];
5885
+ const step = (st , k ) => {
5886
+ if (!isObj(st) || st.type !== "httpRequest") return st;
5887
+ const { api: own, ...rest } = st;
5888
+ const flow = isObj(jobs[k]) ? flowStepOf(jobs[k] ) : null;
5889
+ if (!flow) {
5890
+ return { type: "prompt", prompt: rest.prompt,
5891
+ ...Object.fromEntries(["connection", "profile", "from"].filter(f => rest[f] != null).map(f => [f, rest[f]])) };
5892
+ }
5893
+ const was = HTTP_APIS[own ], now = HTTP_APIS[flow.api];
5894
+ if (!was || !now || own === flow.api) return rest;
5895
+ let body = rest.body;
5896
+ try { body = was.place(rest.body, rest ); } catch { /* words that never placed: the template stands */ }
5897
+ const { system: _s, model: _m, settings: _t, prefill: _p, ...kept } = rest;
5898
+ return { ...kept, body, ...now.cellOf(body) };
5899
+ };
5900
+ if (Array.isArray(next.targets)) {
5901
+ next.targets = next.targets.map((t ) => (isObj(t) && Array.isArray(t.steps) ? { ...t, steps: t.steps.map(step) } : t));
5577
5902
  }
5578
5903
  return next;
5579
5904
  }
@@ -5633,6 +5958,16 @@ function blankPipeline(opts
5633
5958
  jobs: [job], targets: [], evals: [], pass: { mode: "all" } };
5634
5959
  }
5635
5960
 
5961
+ /** What is wrong with how a target or a step [at] names whom it asks: each
5962
+ reference as { id, name }, and a profile only over a connection. */
5963
+ function answerRefsProblems(o , at , bad ) {
5964
+ for (const [f, what] of [["connection", "AI Connection"], ["profile", "Connection Profile"]] ) {
5965
+ if (o[f] == null) continue;
5966
+ if (!isRef(o[f])) bad.push(`${at} has to name its ${what} as { id, name }`);
5967
+ else onlyFields(o[f], `${at}'s ${f}`, ["id", "name"], bad);
5968
+ }
5969
+ }
5970
+
5636
5971
  /**
5637
5972
  * Every reason [doc] cannot run, as sentences -- none when it can. The
5638
5973
  * reference lookups are [ctx]'s: `profiles(id)` and `sources(id)` answer with
@@ -5645,9 +5980,7 @@ function validatePipeline(input , ctx = {}) {
5645
5980
  const bad = [];
5646
5981
  const run = doc.profiles !== undefined;
5647
5982
  const c = { ...ctx, run };
5648
- if (run && !ctx.profiles) {
5649
- c.profiles = (id ) => isObj(doc.profiles) && Object.hasOwn(doc.profiles, id) ? doc.profiles[id] : null;
5650
- }
5983
+ if (run && !ctx.profiles) c.profiles = (key ) => (runAnswer(doc, key) ) ?? null;
5651
5984
  onlyFields(doc, "the pipeline", run ? RUN_FIELDS : PIPELINE_FIELDS, bad);
5652
5985
  if (!isStr(doc.id) || !doc.id.trim()) bad.push("the pipeline has no id");
5653
5986
  if (doc.name != null && !isStr(doc.name)) bad.push("name has to be text");
@@ -5668,32 +6001,33 @@ function validatePipeline(input , ctx = {}) {
5668
6001
  targets.forEach((t, i) => {
5669
6002
  const at = targetLabel(doc, i);
5670
6003
  if (!isObj(t)) return void bad.push(`${at} has to be an object`);
5671
- onlyFields(t, at, ["id", "name", "profile", "steps"], bad);
6004
+ onlyFields(t, at, ["id", "name", "connection", "profile", "steps"], bad);
5672
6005
  if (!isStr(t.id) || !t.id.trim()) bad.push(`${at} has no id`);
5673
6006
  else if (targets.findIndex((o) => isObj(o) && o.id === t.id) !== i) bad.push(`${at} has the id of another target`);
5674
6007
  if (t.name != null && !isStr(t.name)) bad.push(`${at}: name has to be text`);
5675
- if (t.profile != null && !isRef(t.profile)) bad.push(`${at} has to name its Target profile as { id, name }`);
5676
- else if (t.profile != null) onlyFields(t.profile, `${at}'s profile`, ["id", "name"], bad);
6008
+ answerRefsProblems(t, at, bad);
5677
6009
  if (!Array.isArray(t.steps) || t.steps.length !== n) {
5678
6010
  bad.push(`${at} has ${Array.isArray(t.steps) ? t.steps.length : "no"} prompts for ${n} job${n === 1 ? "" : "s"}`);
5679
6011
  return;
5680
6012
  }
5681
- // A target with no profile of its own is one whose steps need none, or
5682
- // each name theirs.
5683
- if (t.profile == null && t.steps.some((st ) => !(isObj(st) && STEP_TYPES[st.type]?.local) && !(isObj(st) && st.profile != null))) {
5684
- bad.push(`${at} has to name its Target profile as { id, name }`);
6013
+ // A target with no connection of its own is one whose steps need none,
6014
+ // or each name theirs; a profile needs a connection to be over.
6015
+ const ownsOne = (st ) => isObj(st) && st.connection != null;
6016
+ if (t.connection == null && t.steps.some((st ) => !(isObj(st) && STEP_TYPES[st.type]?.local) && !ownsOne(st))) {
6017
+ bad.push(`${at} has to name its AI Connection as { id, name }`);
5685
6018
  }
5686
6019
  t.steps.forEach((st , k ) => {
5687
6020
  const where = `${at}, ${jobLabel(doc, k)}`;
5688
6021
  if (!isObj(st)) return void bad.push(`${where} has to be an object`);
5689
6022
  const entry = STEP_TYPES[st.type];
5690
6023
  if (entry?.slot !== "target") return void bad.push(`${where}: "${st.type}" is not something a target can send`);
5691
- if (entry.local && st.profile != null) return void bad.push(`${where}: ${entry.label} asks no profile`);
6024
+ if (entry.local && (st.profile != null || st.connection != null)) return void bad.push(`${where}: ${entry.label} asks no connection`);
5692
6025
  if (k > 0 && entry.firstJobOnly) return void bad.push(`${where}: ${entry.firstJobOnly}`);
5693
6026
  onlyFields(st, where, entry.fields ?? [], bad);
5694
- entry.validate(st, c, bad, where);
5695
- if (st.profile != null && !isRef(st.profile)) bad.push(`${where} has to name its Target profile as { id, name }`);
5696
- else if (st.profile != null) onlyFields(st.profile, `${where}'s profile`, ["id", "name"], bad);
6027
+ // A request is read and placed by its job's HTTP API.
6028
+ const flow = entry.asks === "request" ? flowStepOf(doc.jobs[k]) : null;
6029
+ entry.validate(flow ? { ...st, api: flow.api } : st, c, bad, where);
6030
+ answerRefsProblems(st, where, bad);
5697
6031
  if (st.from != null && !isRef(st.from)) bad.push(`${where} has to name the prompt it was picked from as { id, name }`);
5698
6032
  // A request is a flow step's, rebuilt from its records, and its
5699
6033
  // template has no place for an image.
@@ -5703,13 +6037,16 @@ function validatePipeline(input , ctx = {}) {
5703
6037
  });
5704
6038
  if (bad.length) return bad;
5705
6039
 
5706
- // A reference that resolves to nothing, once per reference.
6040
+ // A reference that resolves to nothing, once per reference: the part
6041
+ // that is gone, where the lookups can say which.
5707
6042
  const missing = new Set ();
5708
- const lookup = (ref ) => {
5709
- const p = c.profiles ? c.profiles(ref.id) : {};
5710
- if (!p && !missing.has(ref.id)) {
5711
- missing.add(ref.id);
5712
- bad.push(`Target profile ${ref.name || ref.id} not found`);
6043
+ const lookup = (a ) => {
6044
+ const key = answerKey(a.connection.id, a.profile?.id);
6045
+ const p = c.profiles ? c.profiles(key) : {};
6046
+ if (!p && !missing.has(key)) {
6047
+ missing.add(key);
6048
+ const gone = a.profile && c.connections?.(a.connection.id) ? a.profile : a.connection;
6049
+ bad.push(`${gone === a.profile ? "Connection Profile" : "Connection"} ${gone.name || gone.id} not found`);
5713
6050
  }
5714
6051
  return p;
5715
6052
  };
@@ -5725,7 +6062,7 @@ function validatePipeline(input , ctx = {}) {
5725
6062
  // What target [i] asks in job [k]: Echo's own connection for a step
5726
6063
  // answered here, else its profile as the lookups hold it.
5727
6064
  const askedOf = (i , k ) => (targetEntryOf(doc, i, k)?.local ? localConnOf(targetEntryOf(doc, i, k))
5728
- : c.profiles ? c.profiles(targetProfileOf(doc, i, k) .id) : null);
6065
+ : c.profiles ? c.profiles(targetAnswerKey(doc, i, k) ) : null);
5729
6066
  const itemsHaveText = !!content && !CONTENT_TYPES[content.type]?.bare;
5730
6067
  for (let k = 0; k < n; k++) {
5731
6068
  const i = targets.findIndex((_, i) => {
@@ -5746,7 +6083,7 @@ function validatePipeline(input , ctx = {}) {
5746
6083
  const nonText = (contentFiles || []).map(f => String((f )?.name ?? f))
5747
6084
  .find(n => !/\.(?:txt|md|csv)$/i.test(n));
5748
6085
  targets.forEach((_, i) => {
5749
- const conns = doc.jobs.map((_ , k ) => (targetEntryOf(doc, i, k)?.local ? localConnOf(targetEntryOf(doc, i, k)) : lookup(targetProfileOf(doc, i, k) )));
6086
+ const conns = doc.jobs.map((_ , k ) => (targetEntryOf(doc, i, k)?.local ? localConnOf(targetEntryOf(doc, i, k)) : lookup(targetAnswerOf(doc, i, k) )));
5750
6087
  conns.forEach((p , k ) => {
5751
6088
  if (!local(p)) return;
5752
6089
  const entry = CONNECTION_TYPES[typeOf(p )] ;
@@ -5764,6 +6101,15 @@ function validatePipeline(input , ctx = {}) {
5764
6101
  const type = CONNECTION_TYPES[typeOf(p )];
5765
6102
  if (type && !(type.answers ?? ["prompt"]).includes(step.asks ?? "prompt")) {
5766
6103
  bad.push(`${n > 1 ? `${jobLabel(doc, k)}, ` : ""}${targetLabel(doc, i)}: ${step.label ?? "this step"} needs a profile that answers it, and ${type.label} does not`);
6104
+ return;
6105
+ }
6106
+ // A request goes as the flow wrote it, so its connection has to speak
6107
+ // the job's HTTP API; one the lab cannot tell (Any endpoint) passes.
6108
+ const api = step.asks === "request" ? flowStepOf(doc.jobs[k])?.api : undefined;
6109
+ const speaks = api && api !== "raw" && type?.speaks ? type.speaks(p ) : null;
6110
+ if (api && speaks && speaks !== api) {
6111
+ const name = targetAnswerOf(doc, i, k)?.connection.name || type .label;
6112
+ bad.push(`${n > 1 ? `${jobLabel(doc, k)}, ` : ""}${targetLabel(doc, i)}: ${name} can't send an ${HTTP_APIS[api]?.label ?? api} request`);
5767
6113
  }
5768
6114
  });
5769
6115
  // The model is the profile's unless the target's step carries it.
@@ -5771,8 +6117,8 @@ function validatePipeline(input , ctx = {}) {
5771
6117
  && targetEntryOf(doc, i, k)?.modelFrom !== "step" && !String(p.model || "").trim());
5772
6118
  if (bare >= 0) {
5773
6119
  bad.push(n > 1
5774
- ? `no model on the profile ${jobLabel(doc, bare)} of ${targetLabel(doc, i)} uses — manage profiles on the Setup tab`
5775
- : `no model on ${targetLabel(doc, i)}'s Target profile — manage profiles on the Setup tab`);
6120
+ ? `no model on the connection ${jobLabel(doc, bare)} of ${targetLabel(doc, i)} uses — manage connections on the Setup tab`
6121
+ : `no model on ${targetLabel(doc, i)}'s connection — manage connections on the Setup tab`);
5776
6122
  }
5777
6123
  });
5778
6124
  STEP_TYPES.evals .validate(doc.evals, c, bad, outOf(doc.jobs.at(-1)).kind, doc);
@@ -5841,15 +6187,22 @@ function pipelineWarnings(doc , ctx = {}) {
5841
6187
  /** A run document's profiles table: id → connection, keyless, spellable. */
5842
6188
  function profilesProblems(profiles , bad ) {
5843
6189
  if (!isObj(profiles)) return void bad.push("profiles has to be an object of id → connection");
6190
+ // Two entries may share one connection, and so one key; two connections
6191
+ // spelling one variable would hand one the other's key.
5844
6192
  const spelt = new Map ();
5845
6193
  for (const [id, conn] of Object.entries(profiles)) {
5846
- if (!PROFILE_ID.test(id)) {
6194
+ if (!ANSWER_KEY.test(id)) {
5847
6195
  bad.push(`profile id "${id}" has to be letters and digits, with at most one hyphen: its key comes from EVALSLAB_API_KEY_<ID>`);
5848
6196
  continue;
5849
6197
  }
5850
- const from = keyVar(id, isObj(conn) && isStr(conn.slug) ? conn.slug : null);
5851
- if (spelt.has(from)) bad.push(`profiles ${spelt.get(from)} and ${id} both take their key from $${from}`);
5852
- spelt.set(from, id);
6198
+ if (isObj(conn) && conn.connection != null && !(isStr(conn.connection) && PROFILE_ID.test(conn.connection))) {
6199
+ bad.push(`profile ${id}: connection has to be an AI Connection's id`);
6200
+ continue;
6201
+ }
6202
+ const cid = answerConnectionId(id, isObj(conn) ? conn : null);
6203
+ const from = keyVar(cid, isObj(conn) && isStr(conn.slug) ? conn.slug : null);
6204
+ if (spelt.has(from) && spelt.get(from) !== cid) bad.push(`profiles ${spelt.get(from)} and ${cid} both take their key from $${from}`);
6205
+ spelt.set(from, cid);
5853
6206
  const at = `profile ${id}`;
5854
6207
  if (!isObj(conn)) { bad.push(`${at} has to be an object`); continue; }
5855
6208
  connectionProblems(conn, at, from, bad);
@@ -5915,24 +6268,34 @@ function connectionProblems(conn , at , from
5915
6268
  for (const why of CONNECTION_TYPES[type]?.settingsProblems?.(connectionSettings(conn)) ?? []) bad.push(`${at} ${why}`);
5916
6269
  }
5917
6270
 
5918
- /** The profile ids a pipeline references, scenarios first, in order. */
6271
+ /** The AI Connection ids a pipeline asks -- each target's and step's, then
6272
+ each grader's -- in order: the Target profile ids, before version 16. */
5919
6273
  function profileIds(doc ) {
5920
- const ids = [];
5921
- for (const t of targetsOf(doc)) {
5922
- for (const ref of [t.profile, ...(Array.isArray(t.steps) ? t.steps : []).map(st => st?.profile)] ) {
5923
- if (ref?.id && !ids.includes(ref.id)) ids.push(ref.id);
5924
- }
5925
- }
6274
+ return [...new Set(answerKeys(doc).map(k => answerOfKey(k).connection))];
6275
+ }
6276
+
6277
+ /** Every answer key a pipeline asks, targets first, in order: each target's
6278
+ and step's connection and profile, then each grader's bare id. What a run
6279
+ carries one connection for. */
6280
+ function answerKeys(doc ) {
6281
+ const keys = [];
6282
+ const add = (key ) => { if (key && !keys.includes(key)) keys.push(key); };
6283
+ targetsOf(doc).forEach((t, i) => {
6284
+ // The target's own, which History reads it by, then each step's.
6285
+ if (isRef(t.connection)) add(answerKey(t.connection.id, isRef(t.profile) ? t.profile.id : null));
6286
+ (Array.isArray(t.steps) ? t.steps : []).forEach((_, k) => add(targetAnswerKey(doc, i, k)));
6287
+ });
5926
6288
  // Then the ones an eval asks (a grader), so a run carries them too.
5927
6289
  for (const t of evalsOf(doc)) {
5928
- for (const ref of EVAL_TYPES[t.type]?.profiles?.(t) ?? []) if (ref?.id && !ids.includes(ref.id)) ids.push(ref.id);
6290
+ for (const ref of EVAL_TYPES[t.type]?.profiles?.(t) ?? []) if (ref?.id && !keys.includes(ref.id)) keys.push(ref.id);
5929
6291
  }
5930
- return ids;
6292
+ return keys;
5931
6293
  }
5932
6294
 
5933
6295
  /**
5934
6296
  * The run document: [doc] plus what its references resolved to, once, at
5935
- * submit. `ctx.profiles(id)` answers with a stored profile, `ctx.files` is
6297
+ * submit. `ctx.profiles(key)` answers an answer key with what it resolves to
6298
+ * (`answersOf`'s `resolve`), `ctx.files` is
5936
6299
  * the Source's file list as it reads now, `ctx.comment` is the run's own.
5937
6300
  */
5938
6301
  function resolvePipeline(doc ,
@@ -5942,10 +6305,16 @@ function resolvePipeline(doc ,
5942
6305
  // What the lab supplies an eval -- its grader -- before the profiles it
5943
6306
  // asks are carried.
5944
6307
  run.evals = (Array.isArray(run.evals) ? run.evals : []).map((t ) => EVAL_TYPES[t?.type]?.resolve?.(t, ctx) ?? t);
6308
+ // Each pair asked, resolved and frozen on its own: two targets sharing a
6309
+ // connection under different profiles send different settings. The entry
6310
+ // names the connection whose key it is sent with.
5945
6311
  run.profiles = {};
5946
- for (const id of profileIds(run)) {
5947
- const p = ctx.profiles?.(id);
5948
- if (p) run.profiles[id] = connectionOf(p );
6312
+ for (const key of answerKeys(run)) {
6313
+ const p = ctx.profiles?.(key);
6314
+ if (!p) continue;
6315
+ const conn = connectionOf(p );
6316
+ const cid = isStr(p.id) && p.id ? p.id : answerOfKey(key).connection;
6317
+ run.profiles[key] = { ...conn, connection: cid };
5949
6318
  }
5950
6319
  const content = contentOf(run) ;
5951
6320
  if (content?.type === "source") content.files = CONTENT_TYPES.source .expand (content, ctx.files);
@@ -6008,13 +6377,14 @@ function pipelineToYaml(doc , ctx
6008
6377
  return yaml.dump(out, { lineWidth: -1, noRefs: true });
6009
6378
  }
6010
6379
 
6011
- /** Every Target profile reference in [doc] -- each target's, each of its
6012
- steps', each eval's grader -- replaced, in place, by [f] of it. */
6380
+ /** Every AI Connection reference in [doc] -- each target's, each of its
6381
+ steps', each eval's grader -- replaced, in place, by [f] of it. A
6382
+ Connection Profile is named by id and name, and left as it is. */
6013
6383
  function eachProfileRef(doc , f ) {
6014
6384
  for (const t of targetsOf(doc)) {
6015
- if (t.profile) t.profile = f(t.profile);
6385
+ if (t.connection) t.connection = f(t.connection);
6016
6386
  for (const st of Array.isArray(t.steps) ? t.steps : []) {
6017
- if (st?.profile) st.profile = f(st.profile);
6387
+ if (st?.connection) st.connection = f(st.connection);
6018
6388
  }
6019
6389
  }
6020
6390
  for (const t of Array.isArray(doc.evals) ? doc.evals : []) {
@@ -6035,6 +6405,9 @@ function eachProfileRef(doc , f )
6035
6405
  * stands.
6036
6406
  */
6037
6407
  function importPipeline(input , ctx = {}) {
6408
+ // A version-15 file names its profiles by slug, which the upgrade cannot
6409
+ // split (it needs this lab's id), so the split is asked again once remapped.
6410
+ const preSplit = isObj(input) && typeof input.version === "number" && input.version < 16 && !isObj(input.profiles);
6038
6411
  // A file exported before version 2 still imports: it is upgraded, not refused.
6039
6412
  const doc = upgradePipeline(input, ctx);
6040
6413
  const v = versionProblem(doc);
@@ -6042,9 +6415,10 @@ function importPipeline(input , ctx = {}) {
6042
6415
  const profiles = ctx.profiles ?? [], sources = ctx.sources ?? [], groups = ctx.groups ?? [];
6043
6416
  const next = clone(doc) ;
6044
6417
  const missing = [], seen = new Set ();
6045
- const lists = { profile: profiles, source: sources, group: groups };
6418
+ const lists = { profile: profiles, connectionProfile: ctx.connectionProfiles ?? [], source: sources, group: groups };
6046
6419
  const words = {
6047
- profile: (ref ) => `Target profile ${ref.slug || ref.name || ref.id} not found`,
6420
+ profile: (ref ) => `Connection ${ref.slug || ref.name || ref.id} not found`,
6421
+ connectionProfile: (ref ) => `Connection Profile ${ref.name || ref.id} not found`,
6048
6422
  source: (ref ) => `Source ${ref.name || ref.id} not found`,
6049
6423
  group: (ref ) => `Eval group ${ref.name || ref.id} not found`,
6050
6424
  };
@@ -6070,10 +6444,22 @@ function importPipeline(input , ctx = {}) {
6070
6444
  };
6071
6445
  const content = contentOf(next) ;
6072
6446
  if (content?.type === "source") content.ref = remap(content.ref, "source");
6447
+ const split = (connection ) => {
6448
+ const p = preSplit && isRef(connection) && profiles.some(x => x.id === connection.id) ? ctx.connectionSplit?.(connection.id) : null;
6449
+ return p ? { id: p.id, name: p.name ?? "" } : null;
6450
+ };
6073
6451
  for (const t of targetsOf(next)) {
6074
- if (t.profile) t.profile = remapProfile(t.profile);
6452
+ // A profile is matched where the lab says which it holds.
6453
+ if (t.connection) t.connection = remapProfile(t.connection);
6454
+ if (isRef(t.profile) && ctx.connectionProfiles) t.profile = remap(t.profile, "connectionProfile");
6455
+ else if (t.profile == null) t.profile = split(t.connection);
6075
6456
  for (const st of Array.isArray(t.steps) ? t.steps : []) {
6076
- if (st?.profile) st.profile = remapProfile(st.profile);
6457
+ if (st?.connection) st.connection = remapProfile(st.connection);
6458
+ if (isRef(st?.profile) && ctx.connectionProfiles) st.profile = remap(st.profile, "connectionProfile");
6459
+ else if (isObj(st) && st.connection && st.profile == null) {
6460
+ const sp = split(st.connection);
6461
+ if (sp) st.profile = sp;
6462
+ }
6077
6463
  }
6078
6464
  }
6079
6465
  for (const t of Array.isArray(next.evals) ? next.evals : []) {
@@ -6173,7 +6559,7 @@ function junitXml(suites , runner ) {
6173
6559
  //
6174
6560
  // Export for CI writes a pipeline as files a repository keeps and
6175
6561
  // `evals-lab run` runs with no lab around it (docs/pipeline-yaml.md, "Export
6176
- // for CI"): the pipeline's YAML, its Target profiles by slug, and each
6562
+ // for CI"): the pipeline's YAML, its connections resolved and keyed by slug, and each
6177
6563
  // dataset it grades against as the Library's export of it. The server adds
6178
6564
  // the plugins and, when asked, the Source's items, and zips the lot. Read
6179
6565
  // back, the files are the run document the page would have submitted, with
@@ -6196,28 +6582,38 @@ const BUNDLE_DATASET = /^datasets\/([a-z0-9]+(?:-[a-z0-9]+)*)\.json$/;
6196
6582
 
6197
6583
  /**
6198
6584
  * [doc] as a bundle's text files: `pipeline.yaml`, `profiles.yaml` and
6199
- * `datasets/<slug>.json`. [ctx.profiles] is Setup's list -- keys and all,
6200
- * none of which is written -- [ctx.grader] the lab's grader, written into
6201
- * each eval that would have asked it, and [ctx.datasets] each graded
6202
- * dataset by id, as `{ name, body }`. `{ error }` when a dataset or a
6203
- * profile the pipeline names is not given, since a bundle without it runs
6585
+ * `datasets/<slug>.json`. [ctx.connections] and [ctx.connectionProfiles]
6586
+ * are Setup's -- keys and all, none of which is written -- or
6587
+ * [ctx.profiles], Target profiles to split as the store was; [ctx.grader]
6588
+ * the lab's grader, written into each eval that would have asked it, and
6589
+ * [ctx.datasets] each graded dataset by id, as `{ name, body }`. Each
6590
+ * connection asked is written resolved, under its answer key with the
6591
+ * connection's slug for its id. `{ error }` when a dataset or a connection
6592
+ * the pipeline names is not given, since a bundle without it runs
6204
6593
  * something else.
6205
6594
  */
6206
- function exportBundle(doc , ctx
6595
+ function exportBundle(doc , ctx
6596
+
6207
6597
  = {})
6208
6598
  {
6209
- const profiles = withSlugs((ctx.profiles ?? []) );
6599
+ const split = (ctx.profiles ?? []).map(splitProfile);
6600
+ const profiles = withSlugs([...(ctx.connections ?? []), ...split.map(x => x.connection)] );
6601
+ const answers = answersOf(profiles, [...(ctx.connectionProfiles ?? []), ...split.flatMap(x => (x.profile ? [x.profile] : []))]);
6210
6602
  // What the lab would supply at submit, written down: no lab supplies it later.
6211
6603
  const out = clone(doc) ;
6212
6604
  out.evals = (Array.isArray(out.evals) ? out.evals : [])
6213
6605
  .map((t ) => EVAL_TYPES[t?.type]?.resolve?.(t, { grader: ctx.grader ?? null,
6214
6606
  groups: Object.entries(ctx.datasets ?? {}).map(([id, d]) => ({ id, name: d.name, grader: d.body.grader ?? null })) }) ?? t);
6215
6607
  const table = {};
6216
- for (const id of profileIds(out)) {
6217
- const p = profiles.find(x => x.id === id);
6218
- if (!p) return { error: `Target profile ${id} not found` };
6608
+ for (const key of answerKeys(out)) {
6609
+ const at = answerOfKey(key);
6610
+ const named = answers.connection(at.connection);
6611
+ const p = answers.resolve(key);
6612
+ if (!named) return { error: `Connection ${at.connection} not found` };
6613
+ if (!p) return { error: `Connection Profile ${at.profile} not found` };
6219
6614
  const { slug, ...flat } = connectionSettings(connectionOf(p) );
6220
- table[slug ] = flat;
6615
+ // A profile answering on a connection of its own takes that one's key.
6616
+ table[at.bare ? named.slug : answerKey(named.slug , at.profile)] = slug === named.slug ? flat : { ...flat, slug };
6221
6617
  }
6222
6618
  const files = {};
6223
6619
  const taken = new Set (), written = new Set ();
@@ -6264,7 +6660,7 @@ function bundleProblems(files , keys = [])
6264
6660
  for (const [path, text] of Object.entries(files)) {
6265
6661
  for (const key of keys) {
6266
6662
  if (isStr(key) && key.trim().length >= 4 && text.includes(key.trim())) {
6267
- bad.push(`${path} holds a Target profile's key, and a key never leaves the lab`);
6663
+ bad.push(`${path} holds a connection's key, and a key never leaves the lab`);
6268
6664
  break;
6269
6665
  }
6270
6666
  }
@@ -6294,14 +6690,23 @@ function readBundle(files , ctx = {})
6294
6690
  if (!isObj(table)) return { error: `${BUNDLE_PROFILES} maps each profile's slug to its settings` };
6295
6691
  const keyed = bundleProblems(files);
6296
6692
  if (keyed.length) return { error: keyed[0] };
6297
- const profiles = [];
6298
- for (const [slug, p] of Object.entries(table)) {
6299
- if (!isSlug(slug)) return { error: `${BUNDLE_PROFILES}: ${JSON.stringify(slug)} is not a slug` };
6300
- if (!isObj(p)) return { error: `${BUNDLE_PROFILES}: ${slug} has to hold its settings` };
6693
+ // Each entry under its answer key, the connection's slug for its id: a
6694
+ // bundle from before version 16 keys each by its Target profile's slug.
6695
+ const profiles = [];
6696
+ const entries = {};
6697
+ for (const [key, p] of Object.entries(table)) {
6698
+ const at = answerOfKey(key);
6699
+ if (!isSlug(at.connection) || (at.profile != null && !PROFILE_ID.test(at.profile))) {
6700
+ return { error: `${BUNDLE_PROFILES}: ${JSON.stringify(key)} is not a slug` };
6701
+ }
6702
+ if (!isObj(p)) return { error: `${BUNDLE_PROFILES}: ${key} has to hold its settings` };
6301
6703
  if (!CONNECTION_TYPES[typeOf(p )]) {
6302
- return { error: `${BUNDLE_PROFILES}: ${slug} is of type ${p.type}, which this lab does not have` };
6704
+ return { error: `${BUNDLE_PROFILES}: ${key} is of type ${p.type}, which this lab does not have` };
6303
6705
  }
6304
- profiles.push({ ...p, id: slug, slug, name: isStr(p.name) && p.name ? p.name : slug });
6706
+ const slug = isSlug(p.slug) ? p.slug : at.connection;
6707
+ const name = isStr(p.name) && p.name ? p.name : slug;
6708
+ entries[key] = { ...p, id: slug, slug, name };
6709
+ if (!profiles.some(x => x.id === at.connection)) profiles.push({ id: at.connection, slug: at.connection, name: slug === at.connection ? name : at.connection });
6305
6710
  }
6306
6711
  // Each dataset by its name, which is what joins it to the eval naming it.
6307
6712
  const datasets = [];
@@ -6334,6 +6739,8 @@ function readBundle(files , ctx = {})
6334
6739
  if (!ref) continue;
6335
6740
  if (datasets.some(d => d.name === ref.name)) refs.push({ id: ref.id, name: ref.name });
6336
6741
  }
6742
+ // A profile is named by the id the key holds it under; one the table
6743
+ // lacks is a reference the run does not carry, and validation says so.
6337
6744
  const imported = importPipeline(raw, { profiles, groups: refs,
6338
6745
  sources: content?.type === "source" && isRef(content.ref) ? [content.ref] : [] });
6339
6746
  if ("error" in imported) return imported;
@@ -6342,7 +6749,7 @@ function readBundle(files , ctx = {})
6342
6749
  for (const r of refs) byId[r.id] = datasets.find(d => d.name === r.name) .body;
6343
6750
  const caseItems = [...new Set(Object.values(byId).flatMap(b => gradedSetFrom(b ).filter(c => !c.todo).map(caseItem)).filter(Boolean))];
6344
6751
  const run = resolvePipeline(imported.doc, {
6345
- profiles: id => profiles.find(p => p.id === id) ?? null,
6752
+ profiles: key => runAnswer({ profiles: entries }, key) ?? null,
6346
6753
  files: ctx.items ? [...ctx.items].sort() : caseItems,
6347
6754
  });
6348
6755
  const bad = validatePipeline(run, { groups: refs });
@@ -6361,24 +6768,28 @@ function stagesFor(run , i , resolveFilters
6361
6768
  {
6362
6769
  const stages = run.jobs.map((ch, k) => {
6363
6770
  const { kind, modifiers: _mods, ...settings } = outOf(ch);
6364
- // The modifiers a run applies, with each after-Read-as filter set a
6365
- // synthetic modifier where its link sits (#339), and the before-Read-as
6771
+ // The modifiers a run applies, with each Post-format filter set a
6772
+ // synthetic modifier where its link sits (#339), and the Pre-format
6366
6773
  // (text) filters applied to the raw reply before it is read (#405);
6367
6774
  // without a resolver, a Library link reads nothing, a private one its own.
6368
6775
  return { text: targetStepOf(run, i, k) .prompt, kind, withImage: sendsImage(ch),
6369
6776
  verbatim: !!targetEntryOf(run, i, k)?.verbatim, before: beforeFilters(ch, resolveFilters),
6370
6777
  modifiers: stageModifiers(ch, resolveFilters), settings };
6371
6778
  });
6779
+ // Each named by the connection whose key it is sent with.
6780
+ const asked = (key , held ) => {
6781
+ const { connection: _c, ...conn } = held || {} ;
6782
+ return { id: answerConnectionId(key, held), ...conn };
6783
+ };
6372
6784
  const connections = run.jobs.map((_, k) => {
6373
6785
  // A step answered here asks nothing: the profile a run made before it
6374
6786
  // kept is what it ran as, else Echo itself.
6787
+ const key = targetAnswerKey(run, i, k);
6375
6788
  if (targetEntryOf(run, i, k)?.local) {
6376
- const id = targetProfileOf(run, i, k)?.id;
6377
- const held = id != null ? run.profiles?.[id] : undefined;
6378
- return held && CONNECTION_TYPES[held.type]?.local ? { id: id , ...held } : { id: "", ...localConnOf(targetEntryOf(run, i, k)) };
6789
+ const held = key != null ? runAnswer(run, key) : undefined;
6790
+ return held && CONNECTION_TYPES[held.type]?.local ? asked(key , held) : { id: "", ...localConnOf(targetEntryOf(run, i, k)) };
6379
6791
  }
6380
- const id = targetProfileOf(run, i, k) .id;
6381
- return { id, ...(run.profiles?.[id] || {}) };
6792
+ return key != null ? asked(key, runAnswer(run, key)) : { id: "" };
6382
6793
  });
6383
6794
  // What a transport that builds its own request (an HTTP Request) builds
6384
6795
  // it from in each job, and this target's step there.
@@ -6388,11 +6799,16 @@ function stagesFor(run , i , resolveFilters
6388
6799
 
6389
6800
  /** The Setup profile scenario [i] of a run ran under, as History shows it: keyless. */
6390
6801
  function scenarioProfile(run , i ) {
6391
- const id = targetsOf(run)[i]?.profile?.id;
6802
+ // Read by the target: its own connection and profile, frozen at submit
6803
+ // apart from any other target's that shares the connection.
6804
+ const t = targetsOf(run)[i];
6805
+ const key = isRef(t?.connection) ? answerKey(t.connection.id, isRef(t.profile) ? t.profile.id : null) : undefined;
6392
6806
  // A target that asks nothing ran as Echo.
6393
- if (id == null && targetEntryOf(run, i, 0)?.local) { const lc = localConnOf(targetEntryOf(run, i, 0)); return { id: "", name: lc.name, settings: { ...lc } }; }
6394
- const conn = id != null ? run.profiles?.[id] : null;
6395
- return conn ? { id: id , name: conn.name, settings: connectionSettings(conn ) } : null;
6807
+ if (key == null && targetEntryOf(run, i, 0)?.local) { const lc = localConnOf(targetEntryOf(run, i, 0)); return { id: "", name: lc.name, settings: { ...lc } }; }
6808
+ const held = key != null ? runAnswer(run, key) : undefined;
6809
+ if (!held) return null;
6810
+ const { connection: _c, ...conn } = held;
6811
+ return { id: answerConnectionId(key , held), name: conn.name, settings: connectionSettings(conn ) };
6396
6812
  }
6397
6813
 
6398
6814
  /** The kind a run's result is read as: its last job's. */
@@ -6758,13 +7174,14 @@ export {
6758
7174
  PIPELINE_VERSION, DATASET_BODY_VERSION, STEP_TYPES, CONTENT_TYPES, OUTPUT_KINDS, MODIFIERS, EVAL_TYPES, LEGACY_TESTS, METRICS, readMetrics, metricSummary, metricCheck, itemScoresAsync, recordedReplyOf,
6759
7175
  SOURCE_TYPES, DEFAULT_SOURCE_TYPE, sourceTypeOf, WORKFLOW_PLATFORMS, platformOf, DYNAMIC_PROVIDERS, providerOf, WIZARDS, wizardProgress,
6760
7176
  registerKinds, defaultKind, jobDefaults, applyModifiers, keyVar, slugOf, uniqueSlug, isSlug, withSlugs, SLUG_MAX, connectionOf, addressOf, connectionSettings,
7177
+ splitProfile, mergeAnswer, answerKey, answerOfKey, answersOf, runAnswer, answerConnectionId, connectionScoped,
6761
7178
  connectionRequest, connectionBase, connectionSend, connectionReply, connectionReplyProblem, connectionImage, connectionProblems,
6762
7179
  CONNECTION_FIELDS, SETTING_KEYS, OPTION_FIELDS, jobLabel, scenarioLabel, jobWords, versionProblem,
6763
7180
  contentOf, withContent, replyOf,
6764
- targetsOf, targetLetter, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetProfileOf, recordedTargetOf, withTarget,
6765
- tokensOf, withTokens, sendsImage, withImage, flowStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
7181
+ targetsOf, targetLetter, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetAnswerOf, targetAnswerKey, recordedTargetOf, withTarget,
7182
+ tokensOf, withTokens, sendsImage, withImage, flowStepOf, requestStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
6766
7183
  filterSetsOf, withFilterSets, stageModifiers, beforeFilters,
6767
- blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, newLink, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds,
7184
+ blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, newLink, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds, answerKeys,
6768
7185
  scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioEvals, scenarioPasses, groupPass, overallVerdict, passVerdict, isSkipped, failedScore,
6769
7186
  evalLabel, evalsOf, evalsDataset, casesRef, groupOf, groupOfMetrics, pipelineWarnings, itemsFound, previewOf, isWholeRun, EVAL_FIELDS,
6770
7187
  pipelineToYaml, importPipeline, yamlToPipeline, exportBundle, readBundle, bundleProblems, junitXml,