evals-lab 0.6.0 → 0.8.0

package/lab/evals-core.mjs

@@ -35,6 +35,8 @@
 import yaml from "./js-yaml.mjs";
                                                 
 import { evaluate, asText, WdlError,                 } from "./flows/wdl.mjs";
+import { stepsOf as flowStepsOf, actionsByName, readerFor, readsOf,                          } from "./flows/record.mjs";
+import { flowApi,              } from "./flows/flowApi.mjs";
 import registerMetrics from "./metrics/builtin.mjs";
 import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProblems, dropBy, rejectBy, matcher } from "./kinds/list.mjs";
 
@@ -543,6 +545,12 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                                             
                 
                      
+                                                                         
+                                                                           
+                                                                           
+                                                                         
+                                         
+                           
                          
  
 
@@ -646,9 +654,12 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                                      
                        
              
-                                                                         
-                                              
-                            
+                                                                              
+                                                              
+                          
+                                                                              
+                                                          
+                       
                     
                                                                        
                     
@@ -659,9 +670,14 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
 
 /** What a metric needs besides the reply: a model to grade with. */
                             
-                                                                           
-                        
-                                            
+                                                                        
+                                                                        
+                               
+                                                                              
+                                                                            
+                                                                            
+                                                          
+                           
  
 
 /** A metric's reading: pass or fail, a score -- 0 to 1, or a count -- and why. */
@@ -711,6 +727,13 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                     
                                                                                     
                        
+                                                                             
+                                                                           
+                 
+                       
+                                                                             
+                                                  
+                    
  
 
 /** What an earlier eval that failed and does not continue leaves a later one. */
@@ -785,6 +808,9 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                         
                       
                         
+                                                                    
+                                                
+                                                                  
                                                                              
                                                  
 
@@ -869,9 +895,9 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                       
                                                                                                    
                                                                        
+                                                                            
                                                                           
-                                                                
-                                                                                            
+                                                                                                                 
  
 
 /** Lookups, and whether it is a run document being validated. */
@@ -880,11 +906,15 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
  
 
 /** A kind of Source: what one of its items is, and what the page and the
-    runner do with it. A reader asks the entry; it never branches on an id. */
+    runner do with it. A reader asks the entry; it never branches on an id.
+    The row's `type` is the kind; a workflow kind's platform -- which flow
+    engine a row speaks to -- is `config.platform`, a WORKFLOW_PLATFORMS id. */
                                   
                 
                                                                  
                        
+                                                                             
+                
                                                                
                
                                                                            
@@ -893,10 +923,68 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                                     
                                                                          
                       
+                                                                          
+                                                                          
+                  
+                      
                                                                 
                                   
+                                                                         
+                                            
+                                
  
 
+/** What a workflow of one platform -- a flow engine -- answers: the pieces
+    that were Power-Automate-specific, lifted behind the platform id so
+    generic code asks the entry and never names a platform. Power Automate is
+    the one built; a second (Logic App, n8n) is one more entry. */
+                                        
+                                                                                      
+                                                                                
+                                                                    
+                                              
+                                                                            
+                                                                     
+                        
+                                                                           
+                                                             
+                                            
+                                                                       
+                    
+                           
+                                                                           
+                                                                     
+                                                 
+                                                           
+                                    
+                                                                        
+                                               
+                                                   
+                                                                
+                                                                                                      
+                                                                         
+                                                             
+                                                        
+                                                                             
+                             
+                                                                         
+                                                          
+                                                                        
+                                                                            
+                                
+ 
+/** A platform's management API reader (flowApi.ts for Power Automate); its
+    shape is the platform's own, so generic code holds it opaquely. */
+                                  
+
+const WORKFLOW_PLATFORMS                                        = Object.create(null);
+/** The platform a workflow Source speaks to, from its `config.platform`;
+    undefined for a row that is not a workflow or names an unknown platform. */
+const platformOf = (src                                                               )                                    => {
+  const id = src?.config?.platform;
+  return isStr(id) ? WORKFLOW_PLATFORMS[id] : undefined;
+};
+
                                    
                 
                                                                        
@@ -961,10 +1049,13 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
     the item, and a grader, where the run has them. */
                            
                              
+                                                                             
+                                                               
+                           
                                                          
                   
                                                            
-                                                                           
+                                                                                                             
                                                                            
                        
                                                 
@@ -1059,16 +1150,43 @@ const SLOTS                      = ["content", "target", "responses"];
                            
  
 
-/** The dataset body's version: 7 is an eval group (§17); 6 marks itself; 5
-    named its Source; 4 and earlier, neither. */
-const DATASET_BODY_VERSION = 7         ;
+/** The dataset body's version: 8 reads the recorded-reply metric ids under
+    their new names (#299); 7 is an eval group (§17); 6 marks itself; 5 named
+    its Source; 4 and earlier, neither. */
+const DATASET_BODY_VERSION = 8         ;
+
+/** The recorded-reply metric type ids renamed at version 8 (#299): the family
+    read "production" before the Recorded target gave it a home. The ids are
+    stored tokens inside eval group bodies and a pipeline's private group, so
+    they are mapped wherever a body is read rather than rewritten in place. */
+const RECORDED_IDS                         = {
+  "equals-production": "same-as-recorded",
+  "fields-equal-production": "fields-equal-recorded",
+  "same-parse-outcome": "same-parse-as-recorded",
+};
+const renameMetric = (m         )          =>
+  isObj(m) && isStr(m.type) && RECORDED_IDS[m.type] ? { ...m, type: RECORDED_IDS[m.type] } : m;
+const renameMetrics = (list         )          =>
+  Array.isArray(list) ? list.map(renameMetric) : list;
+/** A version-7 body (or a freshly made version-8 one) with its recorded-reply
+    metric ids read under their version-8 names, at version 8. */
+function recordedIdsV7(b           )              {
+  return {
+    ...b,
+    version: DATASET_BODY_VERSION,
+    ...(Array.isArray(b.every) ? { every: renameMetrics(b.every) } : {}),
+    ...(Array.isArray(b.run) ? { run: renameMetrics(b.run) } : {}),
+    ...(Array.isArray(b.cases) ? { cases: b.cases.map((c           ) =>
+      isObj(c) && Array.isArray(c.metrics) ? { ...c, metrics: renameMetrics(c.metrics) } : c) } : {}),
+  }               ;
+}
 
 /** The metrics whose Ignore case version 6 made mean what it says for a
     reply read as a list, as well as one read as text. */
 const CASE_FOLDING = ["contains", "contains-all", "contains-any"];
 
 /**
- * [body] as this version of a dataset (7), from any earlier one. Version 1
+ * [body] as this version of a dataset (8), from any earlier one. Version 1
  * held `imageCases`, each with `minTags`/`maxTags`, and the parser's
  * `replays` and `conformance` (now fixtures/replays.json beside the checks).
  * Version 2 held `rules`, which clean a job's answer and so belong to the
@@ -1081,8 +1199,12 @@ const CASE_FOLDING = ["contains", "contains-all", "contains-any"];
  * a list's items ignoring case whatever a Contains metric's Ignore case
  * said, so each of its Contains metrics says Ignore case (`caseOfV5`), and
  * the body says its version. Version 6 is a group's cases alone: version 7
- * adds its scoring, grader, Every item and Whole run (`groupOfV6`), and a
- * stored row is read that way rather than rewritten. Every reader of a
+ * adds its scoring, grader, Every item and Whole run (`groupOfV6`). Version 8
+ * reads the recorded-reply metric ids (`equals-production` and its two
+ * siblings) under their new names (`RECORDED_IDS`, #299), in a group's own
+ * metrics and its cases' -- a pipeline's private group is a group body, so
+ * this covers it too. A stored row is read that way rather than rewritten.
+ * Every reader of a
  * body calls this: the runner, the page, a run's kept copy. A reader that
  * needs a version-2 body's rules -- to upgrade a pipeline graded against
  * it -- takes them first (`datasetRules`). Pure: the same body gives the
@@ -1093,19 +1215,25 @@ function upgradeDatasetBody   (body   )                  {
   if (!isObj(body)) return body;
   if (body.version === DATASET_BODY_VERSION) return body;
   const b = body             ;
-  // A body saying any other version is one this lab does not read.
-  if ("version" in b) return b.version === 6 ? groupOfV6(b) : body;
+  if ("version" in b) {
+    // Version 7 is an eval group whose recorded-reply metrics are read under
+    // their version-8 ids; version 6 is its cases alone. Any other version is
+    // one this lab does not read.
+    if (b.version === 7) return recordedIdsV7(b);
+    if (b.version === 6) return recordedIdsV7(groupOfV6(b));
+    return body;
+  }
   // A body that names its Source, even as null, is version 5.
   if ("source" in b) {
-    return groupOfV6({ ...b, ...(Array.isArray(b.cases) ? { cases: b.cases.map(caseOfV5) } : {}) });
+    return recordedIdsV7(groupOfV6({ ...b, ...(Array.isArray(b.cases) ? { cases: b.cases.map(caseOfV5) } : {}) }));
   }
   const raw = Array.isArray(b.cases) ? b.cases : Array.isArray(b.imageCases) ? b.imageCases : null;
   // Not a body of any version -- a copy kept while a dataset was an overlay.
   if (!raw) return body;
-  return groupOfV6({ source: null, cases: raw.map(caseOfV4).map(caseOfV5) });
+  return recordedIdsV7(groupOfV6({ source: null, cases: raw.map(caseOfV4).map(caseOfV5) }));
 }
 
-/** A version-6 body as a version-7 eval group: scored All, the lab's
+/** A version-6 body as a version-8 eval group: scored All, the lab's
     grader, and no metrics of its own for every item or the whole run --
     what a Metrics eval naming the dataset with none of its own graded, which
     is what every Graded set converted to. */
@@ -1209,6 +1337,10 @@ function datasetRules(body         )                  {
                   
                
                                                
+                                                                            
+                                                                             
+                                                
+                                      
  
 
 /** An HTTP request as a connection type builds it, for the relay to send. */
@@ -1232,6 +1364,10 @@ function datasetRules(body         )                  {
                        
                                          
                                                                                                                   
+                                                                             
+                                                                            
+                                          
+                                                                                                       
                                                                        
                                                                                          
                                                                                                   
@@ -1240,8 +1376,9 @@ function datasetRules(body         )                  {
                                                                            
                                  
                     
-                                                                           
-                                                                 
+                                                                          
+                                                                               
+                                                                         
                
                                                                              
                                                                          
@@ -1249,7 +1386,12 @@ function datasetRules(body         )                  {
                                                                         
                                                                              
                                                                              
-                                                              
+                                                                                
+                                                                            
+                                                                            
+                                                                            
+                                    
+                    
                                                                            
                                                                       
                                                                      
@@ -1589,6 +1731,8 @@ function ollamaType(id        , label        , cloud         )                 {
         body: { model: conn.model, messages: [message], options, stream: false },
       };
     },
+    // Ollama's own: the schema is the format.
+    structured: (body, schema) => ({ ...body, format: schema }),
     parseReply(j) {
       const r = j                                                                   ;
       return { raw: r?.message?.content ?? "", finishReason: r?.done_reason ?? null };
@@ -1620,28 +1764,55 @@ CONNECTION_TYPES.echo = {
   request: sendsNothing, parseReply: sendsNothing, listModels: sendsNothing, parseModels: () => [],
 };
 
+/** A call's recorded result as transport-level text -- what a model asked the
+    same way would have come back with, before the flow reads it. The runner's
+    replay branch reads it as the flow reads a reply (`httpReplyOf`); this is
+    the fallback where there is no Read as to read it through. */
+function recordedRaw(record         )         {
+  const result = isObj(record) && isObj(record.result) ? record.result : null;
+  if (!result) return "";
+  return isStr(result.body) ? result.body : JSON.stringify(result.body ?? "");
+}
+
+// Recorded (#299): a target that replays each call's recorded reply, sending
+// nothing and holding no key or model. The runner reads the recorded result as
+// the flow reads a reply; `local` is the no-Read-as fallback.
+CONNECTION_TYPES.recorded = {
+  id: "recorded", label: "Recorded", settings: [], keyless: true, picker: false, replays: true,
+  description: "Answers each call with the reply it got when it was recorded. Sends nothing.",
+  local: (_item, _sent, record) => recordedRaw(record),
+  request: sendsNothing, parseReply: sendsNothing, listModels: sendsNothing, parseModels: () => [],
+};
+
 /** A local type's answer to one stage, as a model's would come back. An item
     with no text of its own (Prompt only) is answered from the prompt, and
     that reply is read against nothing -- the prompt is the reply, not an
-    instruction it could repeat. */
-function localAnswer(conn          , text               , sent        )        {
-  const raw = CONNECTION_TYPES[typeOf(conn)] .local ({ text }, sent);
+    instruction it could repeat. A replaying type (Recorded) is handed the
+    item's record to answer from. */
+function localAnswer(conn          , text               , sent        , record          )        {
+  const raw = CONNECTION_TYPES[typeOf(conn)] .local ({ text }, sent, record);
   return { raw, ms: 0, conn: (conn.name                      ) ?? null, ...(text == null ? { readAgainst: "" } : {}) };
 }
 
+/** A chat completions body asking for a reply held to [schema]: OpenAI's
+    response_format, which llama-server and most compatible servers take too. */
+const chatSchema = (body                         , schema                         )                          =>
+  ({ ...body, response_format: { type: "json_schema", json_schema: { name: "reply", strict: true, schema } } });
+
 CONNECTION_TYPES["openai-compatible"] = {
   id: "openai-compatible", label: "OpenAI-compatible",
   description: "OpenAI, OpenRouter, vLLM, LM Studio: any chat completions endpoint.",
   settings: [
-    { key: "temperature", label: "Temperature", control: "text", default: "", hint: "" },
+    // A reasoning model refuses a temperature.
+    { key: "temperature", label: "Temperature", control: "text", default: "", hint: "",
+      appliesTo: (conn) => !/^(gpt-5|o\d)/i.test(String(conn.model ?? "")) },
     { key: "nPredict", label: "Reply tokens", control: "text", default: "", hint: "" },
   ],
-  // /v1/chat/completions, covering OpenAI, OpenRouter, vLLM, LM Studio. The
-  // reasoning-model temperature rule and max_completion_tokens stay with this
-  // type, as the hosted providers reject the fields a llama.cpp server takes.
+  // /v1/chat/completions, covering OpenAI, OpenRouter, vLLM, LM Studio.
+  // max_completion_tokens stays with this type, as the hosted providers
+  // reject the fields a llama.cpp server takes.
   request(conn, prompt, dataUrl, base, key) {
     const hosted = isHostedUrl(base);
-    const isReasoning = /^(gpt-5|o\d)/i.test(String(conn.model ?? ""));
     const temp = asNumber(conn.temperature);
     const predict = asNumber(conn.nPredict);
     return {
@@ -1649,7 +1820,7 @@ CONNECTION_TYPES["openai-compatible"] = {
       headers: bearer(key),
       body: {
         model: conn.model,
-        ...(isReasoning || temp == null ? {} : { temperature: temp }),
+        ...(temp == null ? {} : { temperature: temp }),
         ...(predict == null ? {} : hosted ? { max_completion_tokens: predict } : { max_tokens: predict }),
         messages: [{ role: "user", content: [
           { type: "text", text: prompt },
@@ -1658,15 +1829,23 @@ CONNECTION_TYPES["openai-compatible"] = {
       },
     };
   },
+  structured: chatSchema,
   parseReply: chatReply,
   listModels, parseModels: idsFrom,
 };
 
+/** The Claude models that refuse a sampling setting: Opus from 4.7, and
+    every Opus, Sonnet, Fable and Mythos from 5. Older ones (Opus 4.6, Sonnet
+    4.6, Haiku 4.5) take one. */
+const CLAUDE_FIXED_SAMPLING = /claude-(opus-4-[7-9]|(opus|sonnet|fable|mythos)-[5-9])/i;
+
 CONNECTION_TYPES.anthropic = {
   id: "anthropic", label: "Anthropic",
   description: "Claude, through Anthropic's Messages API.",
+  url: "https://api.anthropic.com",
   settings: [
-    { key: "temperature", label: "Temperature", control: "text", default: "", hint: "" },
+    { key: "temperature", label: "Temperature", control: "text", default: "", hint: "",
+      appliesTo: (conn) => !CLAUDE_FIXED_SAMPLING.test(String(conn.model ?? "")) },
     { key: "maxTokens", label: "Reply tokens", control: "text", default: "", hint: "" },
   ],
   // The Messages API: the x-api-key and anthropic-version headers, base64
@@ -1691,6 +1870,9 @@ CONNECTION_TYPES.anthropic = {
       },
     };
   },
+  // The Messages API's structured outputs. A model that cannot hold one
+  // refuses the request, and the grader asks again without it.
+  structured: (body, schema) => ({ ...body, output_config: { ...(body.output_config                      ), format: { type: "json_schema", schema } } }),
   parseReply(j) {
     const r = j                                                                                ;
     const raw = (r?.content || []).filter(b => b.type === "text").map(b => b.text).join("");
@@ -1763,6 +1945,8 @@ CONNECTION_TYPES["llama.cpp"] = {
       },
     };
   },
+  // llama-server takes OpenAI's response_format, a schema and all.
+  structured: chatSchema,
   parseReply: chatReply,
   listModels, parseModels: idsFrom,
   // A reply that arrived is still a failure when this profile asked for the
@@ -1958,12 +2142,14 @@ function extraHeaders(raw         )                         {
   }
   return out;
 }
-/** An address as a whole-request type hangs paths off it: its origin, no /v1. */
+/** An address as a whole-request type hangs paths off it: its origin, no
+    path. A step's path is the whole path production called, so an address
+    entered with one (`…/v1/messages`) would send it twice. */
 function originBase(raw         )         {
-  let s = String(raw || "").trim().replace(/\/+$/, "");
+  let s = String(raw || "").trim();
   if (!s) return "";
   if (!/^https?:\/\//.test(s)) s = "https://" + s;
-  return s;
+  try { return new URL(s).origin; } catch { return s.replace(/\/+$/, ""); }
 }
 const wholeRequestsOnly = ()        => { throw new Error("an HTTP endpoint is sent an HTTP Request step's request, not a prompt"); };
 CONNECTION_TYPES.http = {
@@ -2090,9 +2276,14 @@ function splitOllama(p         )          {
  * connection carries its type's settings under `options`, so they are merged
  * up before the type reads them.
  */
-function connectionRequest(conn          , prompt        , dataUrl               , base        , key                )              {
+function connectionRequest(conn          , prompt        , dataUrl               , base        , key                ,
+                           schema                                 )              {
   const flat           = { ...conn, ...(conn?.options || {}) };
-  return typeEntry(flat).request(flat, prompt, dataUrl, base, key);
+  const type = typeEntry(flat);
+  // A setting the model refuses is not sent, whatever the profile holds.
+  for (const s of type.settings) if (s.appliesTo && !s.appliesTo(flat)) delete flat[s.key];
+  const req = type.request(flat, prompt, dataUrl, base, key);
+  return schema && type.structured ? { ...req, body: type.structured(req.body                           , schema) } : req;
 }
 
 /** Where [conn]'s requests hang off: its type's own reading of its address,
@@ -2794,6 +2985,15 @@ function registerKinds(k       )       {
                                                                          
                                                      
                                                   
+                                                                              
+                                                                             
+                                                                           
+                                                                           
+                                                                       
+                                                           
+                                                                            
+                                                    
+                                       
  
 
 function pluginHost(pluginId        )             {
@@ -2821,6 +3021,14 @@ function pluginHost(pluginId        )             {
       taken(CONNECTION_TYPES, "connection type", [t.id]);
       CONNECTION_TYPES[t.id] = t;
     },
+    registerWorkflowPlatform(t) {
+      taken(WORKFLOW_PLATFORMS, "workflow platform", [t.id]);
+      WORKFLOW_PLATFORMS[t.id] = t;
+    },
+    registerWizard(w) {
+      taken(WIZARDS, "wizard", [w.id]);
+      WIZARDS[w.id] = w;
+    },
   };
 }
 
@@ -2918,6 +3126,13 @@ function withSlugs                                              (list
 /** Whether [s] is a slug as a profile may hold one. */
 const isSlug = (s         )              => isStr(s) && SLUG.test(s) && s.length <= SLUG_MAX;
 
+/** Where [p] connects: its address, or blank, its type's own -- an Anthropic
+    profile left blank reaches Anthropic. Blank for a type with none means the
+    lab's own Ollama, which the runner and the relay fill in. */
+function addressOf(p                                   )         {
+  return String(p.url ?? "").trim() || CONNECTION_TYPES[typeOf(p            )]?.url || "";
+}
+
 /** A stored profile as a run carries it: its request settings, no key. */
 function connectionOf(p         )             {
   const type = typeOf(p);
@@ -2926,7 +3141,7 @@ function connectionOf(p         )             {
   // temperature is a common field, carried at the top like px/format/quality;
   // the options bag holds the type's own settings only.
   for (const k of settings) if (k !== "temperature" && String(p[k] ?? "").trim()) options[k] = p[k];
-  const out             = { name: p.name || "unnamed", url: p.url || "", model: p.model || "", type };
+  const out             = { name: p.name || "unnamed", url: addressOf(p), model: p.model || "", type };
   if (isSlug(p.slug)) out.slug = p.slug;
   for (const k of ["px", "format", "quality"]         ) if (p[k] != null && p[k] !== "") out[k] = p[k]             ;
   if (String(p.temperature ?? "").trim()) out.temperature = p.temperature                   ;
@@ -2997,26 +3212,120 @@ const FILE_LIBRARY_TEXT = [".txt", ".md", ".csv"]         ;
 SOURCE_TYPES.files = {
   label: "File Library",
   description: "A folder of images and text files you upload.",
+  group: "Files",
   noun: "file",
   uploads: [".jpg", ".jpeg", ".png", ".heic", ".heif", ".dng", ".tif", ".tiff", ...FILE_LIBRARY_TEXT],
   sendsImage: true,
+  hasActions: false,
   itemText: name => FILE_LIBRARY_TEXT.some(ext => name.toLowerCase().endsWith(ext)),
 };
 
-// A Power Automate cloud flow (docs/power-automate.md): its items are
-// records -- one call of a Power Automate step each: what its template read,
-// the request production sent and what came back (flows/record.ts) -- and it
-// keeps the flow's definition beside them. Records are imported from run
-// history, written by hand or uploaded; nothing it holds is an image.
-SOURCE_TYPES["power-automate"] = {
-  label: "Power Automate workflow",
-  description: "A cloud flow read from Microsoft 365, with records of its steps' calls from its run history.",
+// A workflow Source: a cloud flow, read through a platform (WORKFLOW_PLATFORMS,
+// below) its row names in `config.platform`. Its items are records -- one call
+// of one of the flow's steps each: what the step's template read, the request
+// production sent and what came back (flows/record.ts) -- and it keeps the
+// flow's definition beside them. The kind is generic; the platform owns what
+// is engine-specific, and the type's shown label is the platform's.
+SOURCE_TYPES.workflow = {
+  label: "Workflow",
+  description: "A cloud flow, with records of its steps' calls from its run history.",
+  group: "Workflows",
   noun: "record",
   uploads: [".json"],
   sendsImage: false,
+  hasActions: true,
   itemText: () => false,
+  platforms: ["power-automate"],   // platform: the WORKFLOW_PLATFORMS a workflow offers
+};
+
+// Power Automate (docs/power-automate.md): the one workflow platform built.
+// Its pieces were the lab's Power-Automate-specific code -- flows/record.ts,
+// flows/wdl.ts and flowApi.ts -- gathered here so generic code calls the
+// entry, never a platform id.
+WORKFLOW_PLATFORMS["power-automate"] = {   // platform: the Power Automate adapter's own id
+  id: "power-automate",                    // platform: its id, the row's config.platform
+  label: "Power Automate flow",
+  nouns: { workflow: "flow", action: "action" },
+  signIn: "microsoft",
+  uploads: { ".json": "application/json; charset=utf-8" },
+  keepsDefinition: true,
+  api: token => flowApi(token),
+  stepsOf: flowStepsOf,
+  actionsOf: actionsByName,
+  readerFor,
+  evaluate,
+  asText,
+  readsOf,
+  // Copy request… writes the body back as a flow's HTTP action holds it: JSON
+  // whose strings keep Power Automate's own `@{…}` expressions untouched, so a
+  // rebuilt body round-trips into the flow (docs/power-automate.md).
+  render: body => JSON.stringify(body, null, 2),
 };
 
+// ---- Setup Wizards (docs/workflow-sources.md § Wizards) ---------------------
+//
+// A wizard is a registry entry whose steps each say where they happen and a
+// done-when predicate on lab state. Steps tick automatically: `done(lab)` reads
+// the cheap state the page already holds (a workflow Source exists, a pipeline
+// with a Recorded target exists, a run of it is in History), never a poll. One
+// wizard runs at a time, held in the browser-only `promptlab.wizard` store; a
+// header pill shows its progress and the Setup › Wizards section draws the same
+// walk-through. The registry lives here so a plugin registers a wizard through
+// the same host as every other kind (PluginHost.registerWizard, phase 7); the
+// page ships the built-in "Test a workflow" and owns its routes and selectors
+// (web/src/app/wizards.ts). The predicates run with the page's privileges and
+// only read lab state, so a plugin wizard is no new trust (docs/packs.md).
+
+/** The slice of lab state a wizard step's `done` reads: what the page already
+    holds, so a tick costs a predicate and not a request. A reader names a field
+    it needs; a wizard that reads more declares it here. */
+                           
+                                               
+                                                         
+                                                                     
+                                           
+                                                                    
+                                                           
+                                                                     
+                                                           
+ 
+
+                             
+             
+                
+                                                                            
+                 
+                                                                   
+                                                                             
+                
+                                                                             
+                                                                       
+                 
+                                                                    
+                               
+ 
+
+                              
+             
+                
+                                                                      
+                
+                               
+ 
+
+const WIZARDS                              = Object.create(null);
+
+/** How far a wizard has got on [lab]: each step's tick, how many are done, and
+    the first step not yet done -- the one the checklist marks current and Go
+    to step heads for. `current` is -1 once every step is done. */
+function wizardProgress(entry             , lab          )   
+                                                                 
+  {
+  const ticks = entry.steps.map((s) => s.done(lab));
+  const done = ticks.filter(Boolean).length;
+  return { ticks, done, total: entry.steps.length, current: ticks.findIndex((t) => !t) };
+}
+
 CONTENT_TYPES.source = {
   label: "Source",
   description: "Every file or record in a Source from the Library.",
@@ -3194,7 +3503,12 @@ function metricInput(res                    , kase             , more          )
   const last = res.transcript?.at(-1);
   const text = last?.got ?? res.raw ?? "";
   return { text, said: last?.said ?? null, terms: res.terms ?? [], replies: [text], plain: !!more.plain,
-           error: res.error ?? null, ms: res.ms ?? 0, production: more.production ?? null, kase };
+           asked: last?.sent ?? null,
+           // A case that has been given its own expected reply ("Take B as
+           // expected") is compared with that; otherwise the Source's recorded
+           // reply for the item.
+           error: res.error ?? null, ms: res.ms ?? 0,
+           recorded: typeof kase?.recorded === "string" ? kase.recorded : (more.production ?? null), kase };
 }
 
 /** A whole run's replies as one input: every reply's text together, and
@@ -3208,7 +3522,7 @@ function runInput(ress                                       , plain         )
     terms.push(...(r.terms || []));
     ms += r.ms ?? 0;
   }
-  return { text: replies.join("\n"), said: null, terms, replies, plain, error: null, ms, production: null, kase: null };
+  return { text: replies.join("\n"), said: null, terms, replies, plain, error: null, ms, recorded: null, asked: null, kase: null };
 }
 
 /** One metric's reading of [input]: `of`, `not` and a failure all applied,
@@ -3469,16 +3783,21 @@ EVAL_TYPES.group = {
     want: [m.not ? "not" : "", metricSummary(m), typeof m.weight === "number" && m.weight !== 1 ? `×${m.weight}` : ""]
       .filter(Boolean).join(" ") })),
   profiles: t => (isObj(t.own) && isRef(t.own.grader) ? [t.own.grader] : isRef(t.grader) ? [t.grader] : []),
-  // A run carries the lab's grader where the group names none and may ask
-  // one: a model-graded metric of its own, or a case's. A link's group is
-  // the Library's, so the run's copy of the link carries it.
+  // A run carries the grader each group asks, so its profile goes with the
+  // run: a link's is the Library group's own, or the lab's where it names
+  // none -- the run's copy of the link carries it. A private group takes the
+  // lab's where it names none and may ask one: a model-graded metric of its
+  // own, or a case's.
   resolve(t, ctx){
-    if (!isRef(ctx.grader)) return t;
-    const grader = { id: ctx.grader.id, name: ctx.grader.name };
+    const ref = (g         )             => (isRef(g) ? { id: g.id, name: g.name } : null);
+    const lab = ref(ctx.grader);
     if (isObj(t.own)) {
-      return !isRef(t.own.grader) && (gradedIn(t.own.every) || isRef(t.own.casesFrom)) ? { ...t, own: { ...t.own, grader } } : t;
+      return lab && !isRef(t.own.grader) && (gradedIn(t.own.every) || isRef(t.own.casesFrom))
+        ? { ...t, own: { ...t.own, grader: lab } } : t;
     }
-    return isRef(t.group) && !isRef(t.grader) ? { ...t, grader } : t;
+    if (!isRef(t.group) || isRef(t.grader)) return t;
+    const grader = ref(ctx.groups?.find(g => g.id === (t.group       ).id)?.grader) ?? lab;
+    return grader ? { ...t, grader } : t;
   },
   wholeRun: wholeRunGroup,
   verdict: (t, ress, kind) => {
@@ -3629,10 +3948,38 @@ function readGroupRun(group             , ress
   return group.run.length ? wholeRunVerdict(group.run, ress, kind, group.scoring.mode, group.scoring.threshold) : null;
 }
 
+/**
+ * [body] with the cases named in [replies] expecting the reply given for
+ * their item -- Results' "Take B as expected" (docs/workflow-sources.md §
+ * Calls are graded cases). Each such case keeps everything else and gains
+ * `recorded` (the reply the recorded-reply metrics compare against, read in
+ * place of the Source's) and `todo: false`, so a later run grades the item
+ * against that reply. A selected item the group has no case for, or one whose
+ * reply is missing (Target B errored, or is the Recorded target), is left
+ * untouched and named in `missing`; `taken` names the items whose expected was
+ * written. Pure: the same body and replies give the same body, so Undo is the
+ * body as it was.
+ */
+function takeAsExpected(body             , replies                                                     ) 
+                                                              {
+  const byItem = new Map                ();
+  body.cases.forEach((c, i) => { const item = caseItem(c); if (item) byItem.set(item, i); });
+  const cases = body.cases.slice();
+  const taken           = [], missing           = [];
+  for (const item of Object.keys(replies)) {
+    const reply = replies[item];
+    const at = byItem.get(item);
+    if (at == null || typeof reply !== "string") { missing.push(item); continue; }
+    cases[at] = { ...cases[at], recorded: reply, todo: false };
+    taken.push(item);
+  }
+  return { body: { ...body, cases }, taken, missing };
+}
+
 /** The grader a Metrics eval names, reached through what the runner hands it. */
 function graderCtx(t           , more          )            {
   const ask = isRef(t.grader) ? more.grader?.(t.grader) : undefined;
-  return ask ? { ask } : {};
+  return { ...(ask ? { ask } : {}), ...(more.recordedTarget ? { recordedTarget: true } : {}) };
 }
 
 /** A metric's options in a few words -- "watering", "^\\{" -- or nothing
@@ -3647,6 +3994,9 @@ function metricSummary(m        )         {
     const v = m[o.key];
     if (o.type === "select") return o.choices.find(c => c.value === v)?.label ?? "";
     if (o.type === "checkbox") return !!v === !!fresh[o.key] ? "" : v ? o.label.toLowerCase() : `${o.label.toLowerCase()} off`;
+    // The choices ticked, by their labels, where they are not a new one's.
+    if (o.type === "multi") return !Array.isArray(v) || JSON.stringify(v) === JSON.stringify(fresh[o.key]) ? ""
+      : o.choices.filter(c => v.includes(c.value)).map(c => c.label).join(" or ");
     // Text that runs to lines (a schema) reads as its first words, run together.
     return v == null || isObj(v) ? "" : String(v).replace(/\s+/g, " ").trim().slice(0, 40);
   }).filter(Boolean);
@@ -3808,9 +4158,30 @@ STEP_TYPES.echo = {
   },
 };
 
-/** What a local target step is answered as: Echo's own connection, with no
+// Recorded (#299): a target that replays each call's recorded reply, read as
+// the flow reads one. No profile, no prompt -- the record is the reply.
+STEP_TYPES.recorded = {
+  label: "Recorded", slot: "target", asks: "nothing", local: true, replaces: "recorded",
+  description: "Answers each call with the reply it got when it was recorded. Sends nothing.",
+  firstJobOnly: "a call's recorded reply is job 1's",
+  in: "item", out: "text",
+  apply: "runPipeline",
+  // `prompt` only so the step satisfies the target-step shape; it is never
+  // sent (the record is the reply), and the prompt rule exempts it below.
+  fields: ["type", "prompt"],
+  validate(){},
+};
+
+/** What a local target step is answered as, by the connection type it stands
+    in for (`replaces`): Echo's own connection, or Recorded's -- each with no
     address, key or model. */
 const ECHO_CONNECTION = { name: "Echo", url: "", model: "", type: "echo" };
+const RECORDED_CONNECTION = { name: "Recorded", url: "", model: "", type: "recorded" };
+const LOCAL_CONNECTIONS                                                                             =
+  { echo: ECHO_CONNECTION, recorded: RECORDED_CONNECTION };
+/** The connection a local target step stands in for, by its entry. */
+const localConnOf = (entry                           )                                                             =>
+  LOCAL_CONNECTIONS[entry?.replaces ?? ""] ?? ECHO_CONNECTION;
 
 // Responses: how a job's reply is read, in order -- possibly not at all.
 
@@ -3950,6 +4321,16 @@ function targetProfileOf(doc
   return targetStepOf(doc, i, k)?.profile ?? targetsOf(doc)[i]?.profile ?? undefined;
 }
 
+/** Whether target [i] is a Recorded target: it replays each call's recorded
+    reply rather than sending anything (its job-1 step answers locally with a
+    connection that `replays`). A recorded-reply metric compares the recorded
+    reply with itself there, so it says nothing (n/a). Asked through the
+    registry, never a step id. */
+function recordedTargetOf(doc                                                           , i        )          {
+  const entry = targetEntryOf(doc, i, 0);
+  return !!(entry?.local && CONNECTION_TYPES[localConnOf(entry).type]?.replays);
+}
+
 /** [doc] with target [i]'s step in job [k] changed: [change] merged into
     it, or the step [change] makes of it. */
 function withTarget                                 (doc   , i        , k        ,
@@ -4641,11 +5022,16 @@ function validatePipeline(input         , ctx          = {})           {
   // what it is read against. Over Prompt only the prompt is the reply.
   // What target [i] asks in job [k]: Echo's own connection for a step
   // answered here, else its profile as the lookups hold it.
-  const askedOf = (i        , k        ) => (targetEntryOf(doc, i, k)?.local ? ECHO_CONNECTION
+  const askedOf = (i        , k        ) => (targetEntryOf(doc, i, k)?.local ? localConnOf(targetEntryOf(doc, i, k))
     : c.profiles ? c.profiles(targetProfileOf(doc, i, k) .id) : null);
   const itemsHaveText = !!content && !CONTENT_TYPES[content.type]?.bare;
   for (let k = 0; k < n; k++) {
     const i = targets.findIndex((_, i) => {
+      const entry = targetEntryOf(doc, i, k);
+      // A step with no prompt of its own, or a replaying target (Recorded,
+      // which answers from the record), is never asked for one.
+      if (!entry?.fields?.includes("prompt")) return false;
+      if (entry.local && CONNECTION_TYPES[localConnOf(entry).type]?.replays) return false;
       const words = targetStepOf(doc, i, k)?.prompt;
       return (!isStr(words) || !words.trim())
         && !(k === 0 && itemsHaveText && local(askedOf(i, 0)));
@@ -4658,12 +5044,14 @@ function validatePipeline(input         , ctx          = {})           {
   const nonText = (contentFiles || []).map(f => String((f                             )?.name ?? f))
     .find(n => !/\.(?:txt|md|csv)$/i.test(n));
   targets.forEach((_, i) => {
-    const conns = doc.jobs.map((_     , k        ) => (targetEntryOf(doc, i, k)?.local ? ECHO_CONNECTION : lookup(targetProfileOf(doc, i, k) )));
+    const conns = doc.jobs.map((_     , k        ) => (targetEntryOf(doc, i, k)?.local ? localConnOf(targetEntryOf(doc, i, k)) : lookup(targetProfileOf(doc, i, k) )));
     conns.forEach((p         , k        ) => {
       if (!local(p)) return;
-      const label = CONNECTION_TYPES[typeOf(p            )] .label;
-      if (k > 0) bad.push(`${jobLabel(doc, k)}, ${targetLabel(doc, i)}: ${label} answers only job 1, with the item's own text`);
-      else if (nonText) bad.push(`${targetLabel(doc, i)}: ${label} answers each text item with its own text, and ${nonText} is not text`);
+      const entry = CONNECTION_TYPES[typeOf(p            )] ;
+      // A replaying target (Recorded) answers from the record, not the item
+      // text, so the text rules do not apply -- only the job-1 one does.
+      if (k > 0) bad.push(`${jobLabel(doc, k)}, ${targetLabel(doc, i)}: ${entry.label} answers only job 1${entry.replays ? "" : ", with the item's own text"}`);
+      else if (nonText && !entry.replays) bad.push(`${targetLabel(doc, i)}: ${entry.label} answers each text item with its own text, and ${nonText} is not text`);
     });
     // A connection answers what its target's step asks: words, or a whole request.
     conns.forEach((p         , k        ) => {
@@ -5102,7 +5490,8 @@ function exportBundle(doc          , ctx
   // What the lab would supply at submit, written down: no lab supplies it later.
   const out = clone(doc)             ;
   out.evals = (Array.isArray(out.evals) ? out.evals : [])
-    .map((t           ) => EVAL_TYPES[t?.type]?.resolve?.(t, { grader: ctx.grader ?? null }) ?? t);
+    .map((t           ) => EVAL_TYPES[t?.type]?.resolve?.(t, { grader: ctx.grader ?? null,
+      groups: Object.entries(ctx.datasets ?? {}).map(([id, d]) => ({ id, name: d.name, grader: d.body.grader ?? null })) }) ?? t);
   const table                           = {};
   for (const id of profileIds(out)) {
     const p = profiles.find(x => x.id === id);
@@ -5261,7 +5650,7 @@ function stagesFor(run             , i        )
     if (targetEntryOf(run, i, k)?.local) {
       const id = targetProfileOf(run, i, k)?.id;
       const held = id != null ? run.profiles?.[id] : undefined;
-      return held && CONNECTION_TYPES[held.type]?.local ? { id: id , ...held } : { id: "", ...ECHO_CONNECTION };
+      return held && CONNECTION_TYPES[held.type]?.local ? { id: id , ...held } : { id: "", ...localConnOf(targetEntryOf(run, i, k)) };
     }
     const id = targetProfileOf(run, i, k) .id;
     return { id, ...(run.profiles?.[id] || {}) };
@@ -5276,7 +5665,7 @@ function stagesFor(run             , i        )
 function scenarioProfile(run             , i        )                                                          {
   const id = targetsOf(run)[i]?.profile?.id;
   // A target that asks nothing ran as Echo.
-  if (id == null && targetEntryOf(run, i, 0)?.local) return { id: "", name: ECHO_CONNECTION.name, settings: { ...ECHO_CONNECTION } };
+  if (id == null && targetEntryOf(run, i, 0)?.local) { const lc = localConnOf(targetEntryOf(run, i, 0)); return { id: "", name: lc.name, settings: { ...lc } }; }
   const conn = id != null ? run.profiles?.[id] : null;
   return conn ? { id: id , name: conn.name, settings: connectionSettings(conn                       ) } : null;
 }
@@ -5337,9 +5726,9 @@ function itemScores(run             , kase                         , res
   return Object.keys(out).length ? out : null;
 }
 
-/** What production replied to an item, for a run's metrics: its last job's
-    call says, from the item's record, or nobody does. */
-function productionOf(run                          , record         )                {
+/** What production recorded as the reply to an item, for a run's metrics: its
+    last job's call says, from the item's record, or nobody does. */
+function recordedReplyOf(run                          , record         )                {
   const last = run.jobs.at(-1), flow = flowStepOf(last);
   return record && flow ? STEP_TYPES.flowStep .production ({ ...flow, readAs: readAsOf(last) }, record) : null;
 }
@@ -5635,20 +6024,20 @@ registerMetrics({ registerKinds });
 export {
     TOKEN_DEFAULTS, TOKEN_TYPES, tokenMapping, tokenNames, resolvePrompt, tokenSet, textPrompt, words,
     loopReplyError, preparedSize,
-    termIn, forbiddenIn, readCase, gradedSetFrom, caseItem, caseMetrics, metricLines, upgradeDatasetBody, readGroup, readGroupRun, datasetRules, upgradeResults, emptyTally, addToTally, isHostedUrl,
+    termIn, forbiddenIn, readCase, gradedSetFrom, caseItem, caseMetrics, metricLines, upgradeDatasetBody, takeAsExpected, readGroup, readGroupRun, datasetRules, upgradeResults, emptyTally, addToTally, isHostedUrl,
     SEED, REPLY_TOKENS_BEFORE, ANTHROPIC_MAX_TOKENS, asNumber, pinReplyTokens, mappingsFor,
     CONNECTION_TYPES, HTTP_APIS, httpApiFor, httpRequestOf, httpReplyOf, readFlowReply, WdlError, localAnswer, typeOf, profileType, convertProfile, splitOllama, DECODING_KEYS, apiBase,
     EDGE_448, budgetLabel, IMAGE_FORMATS, encoderQuality,
     tallyPercent, runPipeline, validateEvals, evalsWarnings, evalsJson, readList,
     scoreSingle, singleRules, parseCount, lengthHolds, countHolds, COUNT_OPS, LENGTH_OPS,
     COUNT_OP_LABEL, LENGTH_OP_LABEL,
-    PIPELINE_VERSION, DATASET_BODY_VERSION, STEP_TYPES, CONTENT_TYPES, OUTPUT_KINDS, MODIFIERS, EVAL_TYPES, LEGACY_TESTS, METRICS, readMetrics, metricSummary, itemScoresAsync, productionOf,
-    SOURCE_TYPES, DEFAULT_SOURCE_TYPE, sourceTypeOf,
-    registerKinds, defaultKind, jobDefaults, applyModifiers, keyVar, slugOf, uniqueSlug, isSlug, withSlugs, SLUG_MAX, connectionOf, connectionSettings,
+    PIPELINE_VERSION, DATASET_BODY_VERSION, STEP_TYPES, CONTENT_TYPES, OUTPUT_KINDS, MODIFIERS, EVAL_TYPES, LEGACY_TESTS, METRICS, readMetrics, metricSummary, itemScoresAsync, recordedReplyOf,
+    SOURCE_TYPES, DEFAULT_SOURCE_TYPE, sourceTypeOf, WORKFLOW_PLATFORMS, platformOf, WIZARDS, wizardProgress,
+    registerKinds, defaultKind, jobDefaults, applyModifiers, keyVar, slugOf, uniqueSlug, isSlug, withSlugs, SLUG_MAX, connectionOf, addressOf, connectionSettings,
     connectionRequest, connectionBase, connectionSend, connectionReply, connectionReplyProblem, connectionImage, connectionProblems,
 CONNECTION_FIELDS, SETTING_KEYS, OPTION_FIELDS, jobLabel, scenarioLabel, jobWords, versionProblem,
     contentOf, withContent, replyOf,
-    targetsOf, targetLetter, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetProfileOf, withTarget,
+    targetsOf, targetLetter, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetProfileOf, recordedTargetOf, withTarget,
     tokensOf, withTokens, sendsImage, withImage, flowStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
     blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, newLink, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds,
     scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioEvals, scenarioPasses, groupPass, overallVerdict, passVerdict, isSkipped, failedScore,