evals-lab 0.5.0 → 0.7.0

package/lab/evals-core.mjs

@@ -35,6 +35,8 @@
 import yaml from "./js-yaml.mjs";
                                                 
 import { evaluate, asText, WdlError,                 } from "./flows/wdl.mjs";
+import { stepsOf as flowStepsOf, actionsByName, readerFor, readsOf,                          } from "./flows/record.mjs";
+import { flowApi,              } from "./flows/flowApi.mjs";
 import registerMetrics from "./metrics/builtin.mjs";
 import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProblems, dropBy, rejectBy, matcher } from "./kinds/list.mjs";
 
@@ -543,6 +545,12 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                                             
                 
                      
+                                                                         
+                                                                           
+                                                                           
+                                                                         
+                                         
+                           
                          
  
 
@@ -646,9 +654,9 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                                      
                        
              
-                                                                         
-                                              
-                            
+                                                                              
+                                                              
+                          
                     
                                                                        
                     
@@ -662,6 +670,10 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                                                                            
                         
                                             
+                                                                            
+                                                                            
+                                                          
+                           
  
 
 /** A metric's reading: pass or fail, a score -- 0 to 1, or a count -- and why. */
@@ -775,16 +787,18 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
 
 // ---- the registries ----
 
-/** An option a modifier or an eval type exposes for editing. */
-                        
-                                                                                               
-                                                
+/** An option a modifier or an eval type exposes for editing. `hint` says
+    what it does in one short line, under its control, where the label
+    cannot: a switch's effect, not why it exists. */
+                                                                          
+                                                                   
+                    
                                                           
-                                                    
-                                                  
-                                                    
+                        
+                      
+                        
                                                                              
-                                                                            
+                                                 
 
 /** What an output kind made of a reply. */
                                 
@@ -878,11 +892,15 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
  
 
 /** A kind of Source: what one of its items is, and what the page and the
-    runner do with it. A reader asks the entry; it never branches on an id. */
+    runner do with it. A reader asks the entry; it never branches on an id.
+    The row's `type` is the kind; a workflow kind's platform -- which flow
+    engine a row speaks to -- is `config.platform`, a WORKFLOW_PLATFORMS id. */
                                   
                 
                                                                  
                        
+                                                                             
+                
                                                                
                
                                                                            
@@ -891,10 +909,68 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                                     
                                                                          
                       
+                                                                          
+                                                                          
+                  
+                      
                                                                 
                                   
+                                                                         
+                                            
+                                
  
 
+/** What a workflow of one platform -- a flow engine -- answers: the pieces
+    that were Power-Automate-specific, lifted behind the platform id so
+    generic code asks the entry and never names a platform. Power Automate is
+    the one built; a second (Logic App, n8n) is one more entry. */
+                                        
+                                                                                      
+                                                                                
+                                                                    
+                                              
+                                                                            
+                                                                     
+                        
+                                                                           
+                                                             
+                                            
+                                                                       
+                    
+                           
+                                                                           
+                                                                     
+                                                 
+                                                           
+                                    
+                                                                        
+                                               
+                                                   
+                                                                
+                                                                                                      
+                                                                         
+                                                             
+                                                        
+                                                                             
+                             
+                                                                         
+                                                          
+                                                                        
+                                                                            
+                                
+ 
+/** A platform's management API reader (flowApi.ts for Power Automate); its
+    shape is the platform's own, so generic code holds it opaquely. */
+                                  
+
+const WORKFLOW_PLATFORMS                                        = Object.create(null);
+/** The platform a workflow Source speaks to, from its `config.platform`;
+    undefined for a row that is not a workflow or names an unknown platform. */
+const platformOf = (src                                                               )                                    => {
+  const id = src?.config?.platform;
+  return isStr(id) ? WORKFLOW_PLATFORMS[id] : undefined;
+};
+
                                    
                 
                                                                        
@@ -959,6 +1035,9 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
     the item, and a grader, where the run has them. */
                            
                              
+                                                                             
+                                                               
+                           
                                                          
                   
                                                            
@@ -966,6 +1045,10 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                                                                            
                        
                                                 
+                                                                              
+                                                                             
+                                                                         
+                       
  
 
 /** A job's stages, in the order they run (pipeline-model §16). A target's
@@ -1053,16 +1136,43 @@ const SLOTS                      = ["content", "target", "responses"];
                            
  
 
-/** The dataset body's version: 7 is an eval group (§17); 6 marks itself; 5
-    named its Source; 4 and earlier, neither. */
-const DATASET_BODY_VERSION = 7         ;
+/** The dataset body's version: 8 reads the recorded-reply metric ids under
+    their new names (#299); 7 is an eval group (§17); 6 marks itself; 5 named
+    its Source; 4 and earlier, neither. */
+const DATASET_BODY_VERSION = 8         ;
+
+/** The recorded-reply metric type ids renamed at version 8 (#299): the family
+    read "production" before the Recorded target gave it a home. The ids are
+    stored tokens inside eval group bodies and a pipeline's private group, so
+    they are mapped wherever a body is read rather than rewritten in place. */
+const RECORDED_IDS                         = {
+  "equals-production": "same-as-recorded",
+  "fields-equal-production": "fields-equal-recorded",
+  "same-parse-outcome": "same-parse-as-recorded",
+};
+const renameMetric = (m         )          =>
+  isObj(m) && isStr(m.type) && RECORDED_IDS[m.type] ? { ...m, type: RECORDED_IDS[m.type] } : m;
+const renameMetrics = (list         )          =>
+  Array.isArray(list) ? list.map(renameMetric) : list;
+/** A version-7 body (or a freshly made version-8 one) with its recorded-reply
+    metric ids read under their version-8 names, at version 8. */
+function recordedIdsV7(b           )              {
+  return {
+    ...b,
+    version: DATASET_BODY_VERSION,
+    ...(Array.isArray(b.every) ? { every: renameMetrics(b.every) } : {}),
+    ...(Array.isArray(b.run) ? { run: renameMetrics(b.run) } : {}),
+    ...(Array.isArray(b.cases) ? { cases: b.cases.map((c           ) =>
+      isObj(c) && Array.isArray(c.metrics) ? { ...c, metrics: renameMetrics(c.metrics) } : c) } : {}),
+  }               ;
+}
 
 /** The metrics whose Ignore case version 6 made mean what it says for a
     reply read as a list, as well as one read as text. */
 const CASE_FOLDING = ["contains", "contains-all", "contains-any"];
 
 /**
- * [body] as this version of a dataset (7), from any earlier one. Version 1
+ * [body] as this version of a dataset (8), from any earlier one. Version 1
  * held `imageCases`, each with `minTags`/`maxTags`, and the parser's
  * `replays` and `conformance` (now fixtures/replays.json beside the checks).
  * Version 2 held `rules`, which clean a job's answer and so belong to the
@@ -1075,8 +1185,12 @@ const CASE_FOLDING = ["contains", "contains-all", "contains-any"];
  * a list's items ignoring case whatever a Contains metric's Ignore case
  * said, so each of its Contains metrics says Ignore case (`caseOfV5`), and
  * the body says its version. Version 6 is a group's cases alone: version 7
- * adds its scoring, grader, Every item and Whole run (`groupOfV6`), and a
- * stored row is read that way rather than rewritten. Every reader of a
+ * adds its scoring, grader, Every item and Whole run (`groupOfV6`). Version 8
+ * reads the recorded-reply metric ids (`equals-production` and its two
+ * siblings) under their new names (`RECORDED_IDS`, #299), in a group's own
+ * metrics and its cases' -- a pipeline's private group is a group body, so
+ * this covers it too. A stored row is read that way rather than rewritten.
+ * Every reader of a
  * body calls this: the runner, the page, a run's kept copy. A reader that
  * needs a version-2 body's rules -- to upgrade a pipeline graded against
  * it -- takes them first (`datasetRules`). Pure: the same body gives the
@@ -1087,19 +1201,25 @@ function upgradeDatasetBody   (body   )                  {
   if (!isObj(body)) return body;
   if (body.version === DATASET_BODY_VERSION) return body;
   const b = body             ;
-  // A body saying any other version is one this lab does not read.
-  if ("version" in b) return b.version === 6 ? groupOfV6(b) : body;
+  if ("version" in b) {
+    // Version 7 is an eval group whose recorded-reply metrics are read under
+    // their version-8 ids; version 6 is its cases alone. Any other version is
+    // one this lab does not read.
+    if (b.version === 7) return recordedIdsV7(b);
+    if (b.version === 6) return recordedIdsV7(groupOfV6(b));
+    return body;
+  }
   // A body that names its Source, even as null, is version 5.
   if ("source" in b) {
-    return groupOfV6({ ...b, ...(Array.isArray(b.cases) ? { cases: b.cases.map(caseOfV5) } : {}) });
+    return recordedIdsV7(groupOfV6({ ...b, ...(Array.isArray(b.cases) ? { cases: b.cases.map(caseOfV5) } : {}) }));
   }
   const raw = Array.isArray(b.cases) ? b.cases : Array.isArray(b.imageCases) ? b.imageCases : null;
   // Not a body of any version -- a copy kept while a dataset was an overlay.
   if (!raw) return body;
-  return groupOfV6({ source: null, cases: raw.map(caseOfV4).map(caseOfV5) });
+  return recordedIdsV7(groupOfV6({ source: null, cases: raw.map(caseOfV4).map(caseOfV5) }));
 }
 
-/** A version-6 body as a version-7 eval group: scored All, the lab's
+/** A version-6 body as a version-8 eval group: scored All, the lab's
     grader, and no metrics of its own for every item or the whole run --
     what a Metrics eval naming the dataset with none of its own graded, which
     is what every Graded set converted to. */
@@ -1243,7 +1363,12 @@ function datasetRules(body         )                  {
                                                                         
                                                                              
                                                                              
-                                                              
+                                                                                
+                                                                            
+                                                                            
+                                                                            
+                                    
+                    
                                                                            
                                                                       
                                                                      
@@ -1614,12 +1739,33 @@ CONNECTION_TYPES.echo = {
   request: sendsNothing, parseReply: sendsNothing, listModels: sendsNothing, parseModels: () => [],
 };
 
+/** A call's recorded result as transport-level text -- what a model asked the
+    same way would have come back with, before the flow reads it. The runner's
+    replay branch reads it as the flow reads a reply (`httpReplyOf`); this is
+    the fallback where there is no Read as to read it through. */
+function recordedRaw(record         )         {
+  const result = isObj(record) && isObj(record.result) ? record.result : null;
+  if (!result) return "";
+  return isStr(result.body) ? result.body : JSON.stringify(result.body ?? "");
+}
+
+// Recorded (#299): a target that replays each call's recorded reply, sending
+// nothing and holding no key or model. The runner reads the recorded result as
+// the flow reads a reply; `local` is the no-Read-as fallback.
+CONNECTION_TYPES.recorded = {
+  id: "recorded", label: "Recorded", settings: [], keyless: true, picker: false, replays: true,
+  description: "Replays the reply production recorded for each call. Sends nothing.",
+  local: (_item, _sent, record) => recordedRaw(record),
+  request: sendsNothing, parseReply: sendsNothing, listModels: sendsNothing, parseModels: () => [],
+};
+
 /** A local type's answer to one stage, as a model's would come back. An item
     with no text of its own (Prompt only) is answered from the prompt, and
     that reply is read against nothing -- the prompt is the reply, not an
-    instruction it could repeat. */
-function localAnswer(conn          , text               , sent        )        {
-  const raw = CONNECTION_TYPES[typeOf(conn)] .local ({ text }, sent);
+    instruction it could repeat. A replaying type (Recorded) is handed the
+    item's record to answer from. */
+function localAnswer(conn          , text               , sent        , record          )        {
+  const raw = CONNECTION_TYPES[typeOf(conn)] .local ({ text }, sent, record);
   return { raw, ms: 0, conn: (conn.name                      ) ?? null, ...(text == null ? { readAgainst: "" } : {}) };
 }
 
@@ -2991,24 +3137,54 @@ const FILE_LIBRARY_TEXT = [".txt", ".md", ".csv"]         ;
 SOURCE_TYPES.files = {
   label: "File Library",
   description: "A folder of images and text files you upload.",
+  group: "Files",
   noun: "file",
   uploads: [".jpg", ".jpeg", ".png", ".heic", ".heif", ".dng", ".tif", ".tiff", ...FILE_LIBRARY_TEXT],
   sendsImage: true,
+  hasActions: false,
   itemText: name => FILE_LIBRARY_TEXT.some(ext => name.toLowerCase().endsWith(ext)),
 };
 
-// A Power Automate cloud flow (docs/power-automate.md): its items are
-// records -- one call of a Power Automate step each: what its template read,
-// the request production sent and what came back (flows/record.ts) -- and it
-// keeps the flow's definition beside them. Records are imported from run
-// history, written by hand or uploaded; nothing it holds is an image.
-SOURCE_TYPES["power-automate"] = {
-  label: "Power Automate workflow",
-  description: "A cloud flow read from Microsoft 365, with records of its steps' calls from its run history.",
+// A workflow Source: a cloud flow, read through a platform (WORKFLOW_PLATFORMS,
+// below) its row names in `config.platform`. Its items are records -- one call
+// of one of the flow's steps each: what the step's template read, the request
+// production sent and what came back (flows/record.ts) -- and it keeps the
+// flow's definition beside them. The kind is generic; the platform owns what
+// is engine-specific, and the type's shown label is the platform's.
+SOURCE_TYPES.workflow = {
+  label: "Workflow",
+  description: "A cloud flow, with records of its steps' calls from its run history.",
+  group: "Workflows",
   noun: "record",
   uploads: [".json"],
   sendsImage: false,
+  hasActions: true,
   itemText: () => false,
+  platforms: ["power-automate"],   // platform: the WORKFLOW_PLATFORMS a workflow offers
+};
+
+// Power Automate (docs/power-automate.md): the one workflow platform built.
+// Its pieces were the lab's Power-Automate-specific code -- flows/record.ts,
+// flows/wdl.ts and flowApi.ts -- gathered here so generic code calls the
+// entry, never a platform id.
+WORKFLOW_PLATFORMS["power-automate"] = {   // platform: the Power Automate adapter's own id
+  id: "power-automate",                    // platform: its id, the row's config.platform
+  label: "Power Automate flow",
+  nouns: { workflow: "flow", action: "action" },
+  signIn: "microsoft",
+  uploads: { ".json": "application/json; charset=utf-8" },
+  keepsDefinition: true,
+  api: token => flowApi(token),
+  stepsOf: flowStepsOf,
+  actionsOf: actionsByName,
+  readerFor,
+  evaluate,
+  asText,
+  readsOf,
+  // Copy request… writes the body back as a flow's HTTP action holds it: JSON
+  // whose strings keep Power Automate's own `@{…}` expressions untouched, so a
+  // rebuilt body round-trips into the flow (docs/power-automate.md).
+  render: body => JSON.stringify(body, null, 2),
 };
 
 CONTENT_TYPES.source = {
@@ -3188,7 +3364,11 @@ function metricInput(res                    , kase             , more          )
   const last = res.transcript?.at(-1);
   const text = last?.got ?? res.raw ?? "";
   return { text, said: last?.said ?? null, terms: res.terms ?? [], replies: [text], plain: !!more.plain,
-           error: res.error ?? null, ms: res.ms ?? 0, production: more.production ?? null, kase };
+           // A case that has been given its own expected reply ("Take B as
+           // expected") is compared with that; otherwise the Source's recorded
+           // reply for the item.
+           error: res.error ?? null, ms: res.ms ?? 0,
+           recorded: typeof kase?.recorded === "string" ? kase.recorded : (more.production ?? null), kase };
 }
 
 /** A whole run's replies as one input: every reply's text together, and
@@ -3202,7 +3382,7 @@ function runInput(ress                                       , plain         )
     terms.push(...(r.terms || []));
     ms += r.ms ?? 0;
   }
-  return { text: replies.join("\n"), said: null, terms, replies, plain, error: null, ms, production: null, kase: null };
+  return { text: replies.join("\n"), said: null, terms, replies, plain, error: null, ms, recorded: null, kase: null };
 }
 
 /** One metric's reading of [input]: `of`, `not` and a failure all applied,
@@ -3487,10 +3667,21 @@ EVAL_TYPES.group = {
   read: async (t, kase, res, more) => {
     const group = groupOf(t, more);
     const grader = isRef(group.grader) ? group.grader : isRef(t.grader) ? t.grader : null;
-    return readGroup({ ...group, grader }               , casesRef(t) ? kase : null, res, more);
+    const ref = casesRef(t);
+    // The item's case in this eval's own group: resolved from the group whose
+    // cases it reads where the runner names the item (grading several groups
+    // at once, --groups), else the case handed in (a single group).
+    const own = ref && isStr(more.item) ? caseIn(more.group?.(ref), more.item) : kase;
+    return readGroup({ ...group, grader }               , ref ? own : null, res, more);
   },
 };
 
+/** The case for an item in a group's body, by the name its file has, or null:
+    a non-todo case whose item matches, as a Library group reads one. */
+function caseIn(body                         , item        )              {
+  return (body?.cases ?? []).find(c => !c.todo && caseItem(c) === item) ?? null;
+}
+
 // ---- the lab's own scorers, as metrics ----------------------------------------
 // What the Single Test checks, one metric each, so an eval of it converts to
 // Metrics that read a run exactly as it did (metrics-parity-check.js). Here
@@ -3612,10 +3803,38 @@ function readGroupRun(group             , ress
   return group.run.length ? wholeRunVerdict(group.run, ress, kind, group.scoring.mode, group.scoring.threshold) : null;
 }
 
+/**
+ * [body] with the cases named in [replies] expecting the reply given for
+ * their item -- Results' "Take B as expected" (docs/workflow-sources.md §
+ * Calls are graded cases). Each such case keeps everything else and gains
+ * `recorded` (the reply the recorded-reply metrics compare against, read in
+ * place of the Source's) and `todo: false`, so a later run grades the item
+ * against that reply. A selected item the group has no case for, or one whose
+ * reply is missing (Target B errored, or is the Recorded target), is left
+ * untouched and named in `missing`; `taken` names the items whose expected was
+ * written. Pure: the same body and replies give the same body, so Undo is the
+ * body as it was.
+ */
+function takeAsExpected(body             , replies                                                     ) 
+                                                              {
+  const byItem = new Map                ();
+  body.cases.forEach((c, i) => { const item = caseItem(c); if (item) byItem.set(item, i); });
+  const cases = body.cases.slice();
+  const taken           = [], missing           = [];
+  for (const item of Object.keys(replies)) {
+    const reply = replies[item];
+    const at = byItem.get(item);
+    if (at == null || typeof reply !== "string") { missing.push(item); continue; }
+    cases[at] = { ...cases[at], recorded: reply, todo: false };
+    taken.push(item);
+  }
+  return { body: { ...body, cases }, taken, missing };
+}
+
 /** The grader a Metrics eval names, reached through what the runner hands it. */
 function graderCtx(t           , more          )            {
   const ask = isRef(t.grader) ? more.grader?.(t.grader) : undefined;
-  return ask ? { ask } : {};
+  return { ...(ask ? { ask } : {}), ...(more.recordedTarget ? { recordedTarget: true } : {}) };
 }
 
 /** A metric's options in a few words -- "watering", "^\\{" -- or nothing
@@ -3791,9 +4010,30 @@ STEP_TYPES.echo = {
   },
 };
 
-/** What a local target step is answered as: Echo's own connection, with no
+// Recorded (#299): a target that replays each call's recorded reply, read as
+// the flow reads one. No profile, no prompt -- the record is the reply.
+STEP_TYPES.recorded = {
+  label: "Recorded", slot: "target", asks: "nothing", local: true, replaces: "recorded",
+  description: "Replays the reply production recorded for each call. Sends nothing.",
+  firstJobOnly: "a call's recorded reply is job 1's",
+  in: "item", out: "text",
+  apply: "runPipeline",
+  // `prompt` only so the step satisfies the target-step shape; it is never
+  // sent (the record is the reply), and the prompt rule exempts it below.
+  fields: ["type", "prompt"],
+  validate(){},
+};
+
+/** What a local target step is answered as, by the connection type it stands
+    in for (`replaces`): Echo's own connection, or Recorded's -- each with no
     address, key or model. */
 const ECHO_CONNECTION = { name: "Echo", url: "", model: "", type: "echo" };
+const RECORDED_CONNECTION = { name: "Recorded", url: "", model: "", type: "recorded" };
+const LOCAL_CONNECTIONS                                                                             =
+  { echo: ECHO_CONNECTION, recorded: RECORDED_CONNECTION };
+/** The connection a local target step stands in for, by its entry. */
+const localConnOf = (entry                           )                                                             =>
+  LOCAL_CONNECTIONS[entry?.replaces ?? ""] ?? ECHO_CONNECTION;
 
 // Responses: how a job's reply is read, in order -- possibly not at all.
 
@@ -3901,9 +4141,15 @@ function targetsOf(doc
   return Array.isArray(doc?.targets) ? doc .targets             : [];
 }
 
-/** Target [i]'s name, or the number it has always shown. */
+/** Target [i]'s letter, A for the first: the same in every job, so a
+    Target reads as one column through them. A number past Z. */
+function targetLetter(i        )         {
+  return i < 26 ? String.fromCharCode(65 + i) : String(i + 1);
+}
+
+/** Target [i]'s name, or its letter's: "Target A". */
 function targetLabel(doc                                                           , i        )         {
-  return (targetsOf(doc)[i]?.name || "").trim() || `Target ${i + 1}`;
+  return (targetsOf(doc)[i]?.name || "").trim() || `Target ${targetLetter(i)}`;
 }
 
 /** What target [i] sends in job [k]: its step there. */
@@ -3927,6 +4173,16 @@ function targetProfileOf(doc
   return targetStepOf(doc, i, k)?.profile ?? targetsOf(doc)[i]?.profile ?? undefined;
 }
 
+/** Whether target [i] is a Recorded target: it replays each call's recorded
+    reply rather than sending anything (its job-1 step answers locally with a
+    connection that `replays`). A recorded-reply metric compares the recorded
+    reply with itself there, so it says nothing (n/a). Asked through the
+    registry, never a step id. */
+function recordedTargetOf(doc                                                           , i        )          {
+  const entry = targetEntryOf(doc, i, 0);
+  return !!(entry?.local && CONNECTION_TYPES[localConnOf(entry).type]?.replays);
+}
+
 /** [doc] with target [i]'s step in job [k] changed: [change] merged into
     it, or the step [change] makes of it. */
 function withTarget                                 (doc   , i        , k        ,
@@ -4150,11 +4406,8 @@ STEP_TYPES.evals = {
                + `and the last job answers with ${OUTPUT_KINDS[lastKind] .noun || lastKind}`);
       }
     });
-    // The worker is handed one Library group's body to grade against
-    // (server-side-runs §4) until it reads the body the queue keeps per
-    // group (#233).
-    const named = new Set(evals.map(casesRef).filter(Boolean).map(r => r .id));
-    if (named.size > 1) bad.push("the evals grade against one Library eval group at a time");
+    // The worker reads a body per group now (#233), so the evals may grade
+    // against several Library groups at once; the queue keeps each body.
   },
 };
 
@@ -4621,11 +4874,16 @@ function validatePipeline(input         , ctx          = {})           {
   // what it is read against. Over Prompt only the prompt is the reply.
   // What target [i] asks in job [k]: Echo's own connection for a step
   // answered here, else its profile as the lookups hold it.
-  const askedOf = (i        , k        ) => (targetEntryOf(doc, i, k)?.local ? ECHO_CONNECTION
+  const askedOf = (i        , k        ) => (targetEntryOf(doc, i, k)?.local ? localConnOf(targetEntryOf(doc, i, k))
     : c.profiles ? c.profiles(targetProfileOf(doc, i, k) .id) : null);
   const itemsHaveText = !!content && !CONTENT_TYPES[content.type]?.bare;
   for (let k = 0; k < n; k++) {
     const i = targets.findIndex((_, i) => {
+      const entry = targetEntryOf(doc, i, k);
+      // A step with no prompt of its own, or a replaying target (Recorded,
+      // which answers from the record), is never asked for one.
+      if (!entry?.fields?.includes("prompt")) return false;
+      if (entry.local && CONNECTION_TYPES[localConnOf(entry).type]?.replays) return false;
       const words = targetStepOf(doc, i, k)?.prompt;
       return (!isStr(words) || !words.trim())
         && !(k === 0 && itemsHaveText && local(askedOf(i, 0)));
@@ -4638,12 +4896,14 @@ function validatePipeline(input         , ctx          = {})           {
   const nonText = (contentFiles || []).map(f => String((f                             )?.name ?? f))
     .find(n => !/\.(?:txt|md|csv)$/i.test(n));
   targets.forEach((_, i) => {
-    const conns = doc.jobs.map((_     , k        ) => (targetEntryOf(doc, i, k)?.local ? ECHO_CONNECTION : lookup(targetProfileOf(doc, i, k) )));
+    const conns = doc.jobs.map((_     , k        ) => (targetEntryOf(doc, i, k)?.local ? localConnOf(targetEntryOf(doc, i, k)) : lookup(targetProfileOf(doc, i, k) )));
     conns.forEach((p         , k        ) => {
       if (!local(p)) return;
-      const label = CONNECTION_TYPES[typeOf(p            )] .label;
-      if (k > 0) bad.push(`${jobLabel(doc, k)}, ${targetLabel(doc, i)}: ${label} answers only job 1, with the item's own text`);
-      else if (nonText) bad.push(`${targetLabel(doc, i)}: ${label} answers each text item with its own text, and ${nonText} is not text`);
+      const entry = CONNECTION_TYPES[typeOf(p            )] ;
+      // A replaying target (Recorded) answers from the record, not the item
+      // text, so the text rules do not apply -- only the job-1 one does.
+      if (k > 0) bad.push(`${jobLabel(doc, k)}, ${targetLabel(doc, i)}: ${entry.label} answers only job 1${entry.replays ? "" : ", with the item's own text"}`);
+      else if (nonText && !entry.replays) bad.push(`${targetLabel(doc, i)}: ${entry.label} answers each text item with its own text, and ${nonText} is not text`);
     });
     // A connection answers what its target's step asks: words, or a whole request.
     conns.forEach((p         , k        ) => {
@@ -5241,7 +5501,7 @@ function stagesFor(run             , i        )
     if (targetEntryOf(run, i, k)?.local) {
       const id = targetProfileOf(run, i, k)?.id;
       const held = id != null ? run.profiles?.[id] : undefined;
-      return held && CONNECTION_TYPES[held.type]?.local ? { id: id , ...held } : { id: "", ...ECHO_CONNECTION };
+      return held && CONNECTION_TYPES[held.type]?.local ? { id: id , ...held } : { id: "", ...localConnOf(targetEntryOf(run, i, k)) };
     }
     const id = targetProfileOf(run, i, k) .id;
     return { id, ...(run.profiles?.[id] || {}) };
@@ -5256,7 +5516,7 @@ function stagesFor(run             , i        )
 function scenarioProfile(run             , i        )                                                          {
   const id = targetsOf(run)[i]?.profile?.id;
   // A target that asks nothing ran as Echo.
-  if (id == null && targetEntryOf(run, i, 0)?.local) return { id: "", name: ECHO_CONNECTION.name, settings: { ...ECHO_CONNECTION } };
+  if (id == null && targetEntryOf(run, i, 0)?.local) { const lc = localConnOf(targetEntryOf(run, i, 0)); return { id: "", name: lc.name, settings: { ...lc } }; }
   const conn = id != null ? run.profiles?.[id] : null;
   return conn ? { id: id , name: conn.name, settings: connectionSettings(conn                       ) } : null;
 }
@@ -5317,9 +5577,9 @@ function itemScores(run             , kase                         , res
   return Object.keys(out).length ? out : null;
 }
 
-/** What production replied to an item, for a run's metrics: its last job's
-    call says, from the item's record, or nobody does. */
-function productionOf(run                          , record         )                {
+/** What production recorded as the reply to an item, for a run's metrics: its
+    last job's call says, from the item's record, or nobody does. */
+function recordedReplyOf(run                          , record         )                {
   const last = run.jobs.at(-1), flow = flowStepOf(last);
   return record && flow ? STEP_TYPES.flowStep .production ({ ...flow, readAs: readAsOf(last) }, record) : null;
 }
@@ -5390,22 +5650,40 @@ function scenarioEvals(run             , i        , items
   });
 }
 
+/** One eval group's verdict over a scenario (§17): a whole-run group's own
+    verdict, or a per-item group's "every item it read passed"; null where it
+    was skipped, or had nothing to read yet. */
+function groupPass(o             )                 {
+  if (o.skipped) return null;
+  if (o.whole) return o.verdict?.ran ? o.verdict.pass : null;
+  return o.ran ? o.passed === o.ran : null;
+}
+
 /** A scenario's pass or fail over every eval that read it: null where none
     has anything to say yet. */
 function scenarioPasses(outcomes                        )                 {
-  let said = false;
-  for (const o of outcomes) {
-    if (o.skipped) continue;
-    if (o.whole) {
-      if (!o.verdict?.ran) continue;
-      said = true;
-      if (!o.verdict.pass) return false;
-    } else if (o.ran) {
-      said = true;
-      if (o.passed < o.ran) return false;
-    }
-  }
-  return said ? true : null;
+  const passes = outcomes.map(groupPass);
+  if (!passes.some((p) => p != null)) return null;
+  return !passes.includes(false);
+}
+
+/** A scenario's overall verdict under a run's pass rule (§17): each linked
+    group's verdict folded together the way `pass` says -- every group passes,
+    or at least a number of them. Null where no group has a verdict yet, so a
+    run with no evals, or one still grading, reads as it does without a rule. */
+function overallVerdict(outcomes                        , pass                             )                 {
+  const passes = outcomes.map(groupPass);
+  if (!passes.some((p) => p != null)) return null;
+  return passVerdict(pass, passes);
+}
+
+/** Whether a run passes a Target under its overall pass rule (§17): `all`
+    passes when no linked group fails, `atLeast` when at least `count` pass.
+    [passes] is each group's pass (true), fail (false), or neither (null: it
+    graded nothing, or an earlier group skipped it). */
+function passVerdict(pass                             , passes                             )          {
+  if (pass?.mode === "atLeast") return passes.filter(p => p === true).length >= pass.count;
+  return !passes.includes(false);
 }
 
 /**
@@ -5597,23 +5875,23 @@ registerMetrics({ registerKinds });
 export {
     TOKEN_DEFAULTS, TOKEN_TYPES, tokenMapping, tokenNames, resolvePrompt, tokenSet, textPrompt, words,
     loopReplyError, preparedSize,
-    termIn, forbiddenIn, readCase, gradedSetFrom, caseItem, caseMetrics, metricLines, upgradeDatasetBody, readGroup, readGroupRun, datasetRules, upgradeResults, emptyTally, addToTally, isHostedUrl,
+    termIn, forbiddenIn, readCase, gradedSetFrom, caseItem, caseMetrics, metricLines, upgradeDatasetBody, takeAsExpected, readGroup, readGroupRun, datasetRules, upgradeResults, emptyTally, addToTally, isHostedUrl,
     SEED, REPLY_TOKENS_BEFORE, ANTHROPIC_MAX_TOKENS, asNumber, pinReplyTokens, mappingsFor,
     CONNECTION_TYPES, HTTP_APIS, httpApiFor, httpRequestOf, httpReplyOf, readFlowReply, WdlError, localAnswer, typeOf, profileType, convertProfile, splitOllama, DECODING_KEYS, apiBase,
     EDGE_448, budgetLabel, IMAGE_FORMATS, encoderQuality,
     tallyPercent, runPipeline, validateEvals, evalsWarnings, evalsJson, readList,
     scoreSingle, singleRules, parseCount, lengthHolds, countHolds, COUNT_OPS, LENGTH_OPS,
     COUNT_OP_LABEL, LENGTH_OP_LABEL,
-    PIPELINE_VERSION, DATASET_BODY_VERSION, STEP_TYPES, CONTENT_TYPES, OUTPUT_KINDS, MODIFIERS, EVAL_TYPES, LEGACY_TESTS, METRICS, readMetrics, metricSummary, itemScoresAsync, productionOf,
-    SOURCE_TYPES, DEFAULT_SOURCE_TYPE, sourceTypeOf,
+    PIPELINE_VERSION, DATASET_BODY_VERSION, STEP_TYPES, CONTENT_TYPES, OUTPUT_KINDS, MODIFIERS, EVAL_TYPES, LEGACY_TESTS, METRICS, readMetrics, metricSummary, itemScoresAsync, recordedReplyOf,
+    SOURCE_TYPES, DEFAULT_SOURCE_TYPE, sourceTypeOf, WORKFLOW_PLATFORMS, platformOf,
     registerKinds, defaultKind, jobDefaults, applyModifiers, keyVar, slugOf, uniqueSlug, isSlug, withSlugs, SLUG_MAX, connectionOf, connectionSettings,
     connectionRequest, connectionBase, connectionSend, connectionReply, connectionReplyProblem, connectionImage, connectionProblems,
 CONNECTION_FIELDS, SETTING_KEYS, OPTION_FIELDS, jobLabel, scenarioLabel, jobWords, versionProblem,
     contentOf, withContent, replyOf,
-    targetsOf, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetProfileOf, withTarget,
+    targetsOf, targetLetter, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetProfileOf, recordedTargetOf, withTarget,
     tokensOf, withTokens, sendsImage, withImage, flowStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
     blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, newLink, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds,
-    scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioEvals, scenarioPasses, isSkipped, failedScore,
+    scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioEvals, scenarioPasses, groupPass, overallVerdict, passVerdict, isSkipped, failedScore,
     evalLabel, evalsOf, evalsDataset, casesRef, groupOf, groupOfMetrics, pipelineWarnings, isWholeRun, EVAL_FIELDS,
     pipelineToYaml, importPipeline, yamlToPipeline, exportBundle, readBundle, bundleProblems, junitXml,
     pluginHost, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProblems, dropBy, rejectBy, matcher,