evals-lab 0.13.0 → 0.14.0

package/lab/evals-core.mjs

@@ -199,12 +199,23 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
                                                                                 
                                                                 
                  
+                                                                           
+                                                                         
+                                                                          
+                                                                           
+                           
                                                                             
                                                          
                    
                                 
  
 
+/** One kept response of a dynamic Source's item (#596). */
+                              
+               
+            
+ 
+
                               
                
                
@@ -347,17 +358,26 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
  * at. `filters` are tried in order -- each drop-item narrows the items the
  * next reads, and the first reject-reply that fires gates the answer.
  */
-/** Which side of Response Format a filter runs on (#405): `text` the raw reply before
-    it is read into items (rewrite-text / reject-text), `items` the items after
-    (drop-item, rewrite-item, keep-first, reject-reply). A filter's type decides
-    its stage; a set belongs to none (#432), and runs its share of filters in
-    each stage it is linked in. */
+/** Which side of Response Format a filter step sits on (#405): `text`
+    before it, on the raw reply (Pre-format), `items` after it (Post-format).
+    It also names what a filter reads: the reply's text (rewrite-text /
+    reject-text) or the items (drop-item, rewrite-item, keep-first,
+    reject-reply). A text filter runs at either stage, an item filter only
+    Post-format after a kind that reads items (#557, `stageReads`); a set
+    belongs to no stage (#432), and runs its share in each it is linked in. */
                                            
 
 /** The stage a filter type's scope belongs to: text filters read the raw
     reply; item, list and answer filters read the items it is read into. */
 export const stageOfScope = (scope        )              => (scope === "text" ? "text" : "items");
 
+/** What job's [stage] filters read when it answers as [kind] (#557):
+    Pre-format the text; Post-format the items after a kind that reads
+    items, and the text after any other. A filter runs there when its
+    `stageOfScope` is this. */
+export const stageReads = (stage             , kind                           )              =>
+  stage === "items" && OUTPUT_KINDS[kind ?? ""]?.offers?.includes("items") ? "items" : "text";
+
 /** A filter stage as the page names it: before the reply is formatted, or after. */
 export const FILTER_STAGE_LABEL                              = { text: "Pre-format", items: "Post-format" };
 
@@ -555,6 +575,10 @@ export const FILTER_STAGE_LABEL                              = { text: "Pre-form
                                                                            
                   
                     
+                                                                          
+                                                                          
+                                                                          
+                   
                          
                                                                                    
                                      
@@ -586,7 +610,15 @@ export const FILTER_STAGE_LABEL                              = { text: "Pre-form
                                                                         
                                                        
                 
+                                                                          
+                                                                       
+                  
+                      
  
+
+/** A reply the reply-token limit stopped: the limit the request sent, or
+    null where it sent none and the server's own applied. */
+                                                   
 
 /** The transport: a stage's prompt and image in, its reply out. */
                                                                               
@@ -607,9 +639,15 @@ export const FILTER_STAGE_LABEL                              = { text: "Pre-form
                                                                             
                                               
                 
-                                                                            
                                                                          
+                                                                         
+                                             
                                            
+                                                          
+                  
+                                                                           
+                                                                              
+                      
  
 
 /** How much of one text a run keeps -- a reply's body, a grader's prompt or
@@ -675,6 +713,7 @@ export function kept(text        , max = KEPT_CHARS)         {
                
                      
                    
+                  
                         
                                     
  
@@ -1204,10 +1243,14 @@ const platformOf = (src
                                                                                 
                                                                          
                 
-                                                                            
-                                                                       
-                                                       
+                                                                         
+                                                                             
+                                                                   
                
+                                                                       
+                                                                        
+                                                                     
+                        
                                                                                
                                                 
                                       
@@ -1697,6 +1740,10 @@ function datasetRules(body         )                  {
                                           
                                                                                                        
                                                                        
+                                                                         
+                                                                           
+                                                   
+                                                                                   
                                                                                          
                                                                                                   
                                     
@@ -2106,6 +2153,7 @@ function ollamaType(id        , label        , cloud         )                 {
       const r = j                                                                   ;
       return { raw: r?.message?.content ?? "", finishReason: r?.done_reason ?? null };
     },
+    cutOff: { reason: "length", limit: (body) => asNumber((body.options                                       )?.num_predict) },
     listModels: (base, key) => listModels(base, cloud ? key : null),
     parseModels: idsFrom,
     ...(cloud ? { url: "https://ollama.com" } : { keyless: true }),
@@ -2220,6 +2268,7 @@ CONNECTION_TYPES["openai-compatible"] = {
   documents: true,
   structured: chatSchema,
   parseReply: chatReply,
+  cutOff: { reason: "length", limit: (body) => asNumber(body.max_completion_tokens ?? body.max_tokens) },
   listModels, parseModels: idsFrom,
 };
 
@@ -2271,6 +2320,7 @@ CONNECTION_TYPES.anthropic = {
     const raw = (r?.content || []).filter(b => b.type === "text").map(b => b.text).join("");
     return { raw, finishReason: r?.stop_reason ?? null };
   },
+  cutOff: { reason: "max_tokens", limit: (body) => asNumber(body.max_tokens) },
   // A flow's request, keyed as the Messages API is -- what the HTTP endpoint
   // type's Anthropic template sets. [base] is the /v1 base words hang off;
   // the step's path is the whole path production called, so it goes to the
@@ -2354,6 +2404,7 @@ CONNECTION_TYPES["llama.cpp"] = {
   // llama-server takes OpenAI's response_format, a schema and all.
   structured: chatSchema,
   parseReply: chatReply,
+  cutOff: { reason: "length", limit: (body) => asNumber(body.n_predict) },
   listModels, parseModels: idsFrom,
   // A reply that arrived is still a failure when this profile asked for the
   // Speed looping check (today's speedReplyError, now an opt-in setting).
@@ -2728,6 +2779,18 @@ function connectionReply(conn          , j         )
 }
 
 /** Why a reply [conn] received is a failure though it arrived, or null. */
+/** How a reply finished, as a stage keeps it (#597): the reason [conn]'s
+    API gave, and -- where that reason is its type's reply-token limit -- the
+    limit [body], the request sent, carried. A type that says no reason
+    keeps nothing. */
+function replyFinish(conn          , body         , finishReason               )                                    {
+  if (!finishReason) return {};
+  const cut = typeEntry(conn).cutOff;
+  if (!cut || finishReason !== cut.reason) return { finish: finishReason };
+  const limit = body && typeof body === "object" ? cut.limit(body                           ) : null;
+  return { finish: finishReason, stopped: { limit } };
+}
+
 function connectionReplyProblem(conn          , raw        , finishReason               )                {
   return typeEntry(conn).replyProblem?.(conn, raw, finishReason) ?? null;
 }
@@ -3253,7 +3316,8 @@ async function runPipeline(stages                  , dataUrl               , cal
     total += res.ms;
     transcript.push({ n: i + 1, kind, conn: res.conn ?? null, withImage, sent: res.sent ?? sent,
                       got: res.raw ?? "", ms: res.ms, error: res.error, ...(res.said != null ? { said: res.said } : {}),
-                      ...(res.body != null ? { body: kept(res.body) } : {}) });
+                      ...(res.body != null ? { body: kept(res.body) } : {}),
+                      ...(res.finish ? { finish: res.finish } : {}), ...(res.stopped ? { stopped: res.stopped } : {}) });
     if (res.error) return result({ error: named(res.error) });
     // A stage that failed is the end of the job: nothing after it is asked.
     // Handed nothing, the next stage still answers -- about an empty list --
@@ -3272,6 +3336,16 @@ async function runPipeline(stages                  , dataUrl               , cal
     if (mod.reject) {
       return result({ error: named(`discarded: ${mod.reject}`), ...(mod.by ? { filteredBy: by(mod.by, "items", true) } : {}) });
     }
+    // A kind that reads no items runs its Post-format filters on its text,
+    // once read (#557); what they leave is what is graded.
+    if (st.after?.length && typeof mod.value === "string") {
+      const post = applyFilterSet(st.after, { body: mod.value, items: [], instruction: against }, "text");
+      if (post.reject != null) return result({ error: named(`discarded: ${post.reject}`), filteredBy: by(post.reject, "items", true) });
+      if (post.rewrote) {
+        transcript[transcript.length - 1] .filtered = { got: post.body, by: [...(pre.rewrote || []), ...post.rewrote] };
+        mod.value = post.body;
+      }
+    }
     // A filter that dropped the last item: what a zero result is explained by.
     const emptied = mod.by && reader.terms && !reader.terms(mod.value).length ? { filteredBy: by(mod.by, "items", false) } : {};
     const nothing = handsOn ? reader.handsOn?.(mod.value) : null;
@@ -3280,7 +3354,9 @@ async function runPipeline(stages                  , dataUrl               , cal
     held = { kind, value };
     const gone = [...(read.dropped || []), ...mod.dropped];
     echoed = (read.echoed || 0) + mod.dropped.filter(d => d.by === ECHO).length;
-    done.push({ reply: res.raw ?? "", rendered: reader.render?.(value) || {} });
+    // {reply} and {stageN} hand on the text as its filters left it -- what
+    // was graded -- never the raw reply a filter cleaned (#557).
+    done.push({ reply: transcript[transcript.length - 1] .filtered?.got ?? res.raw ?? "", rendered: reader.render?.(value) || {} });
     if (!handsOn) {
       dropped = gone;
       return result({ echoed, dropped, ...emptied });
@@ -3380,7 +3456,7 @@ function applyModifiers   (list                                 , kind        ,
 // 16: a target names an AI Connection and, optionally, a Connection Profile,
 // where it named one Target profile; a run's table is keyed by the pair
 // (answerKey) -- the Target profile split in the store as it opens (#445).
-const PIPELINE_VERSION = 17         ;
+const PIPELINE_VERSION = 18         ;
 
 // Plain objects, so an entry is added by assignment and a reader never needs
 // to know which registered it.
@@ -3662,9 +3738,10 @@ function splitProfile(p         )
 }
 
 /** [conn] with [profile]'s settings over it: the one set of settings a
-    request is built from, the shape a Target profile had. A profile sets
-    no connection field. */
-function mergeAnswer(conn                        , profile                           )          {
+    request is built from, the shape a Target profile had. A profile -- or
+    (Default)'s settings, which are one without its id and name -- sets no
+    connection field. */
+function mergeAnswer(conn                        , profile                                      )          {
   const out          = { ...conn };
   const scoped = connectionScoped(typeOf(conn            ));
   for (const [k, v] of Object.entries(profile ?? {})) {
@@ -3681,7 +3758,7 @@ function mergeAnswer(conn                        , profile
  
 
 /** The key a run's table holds [connection] under, with [profile] -- or with
-    none, the lab's defaults. A grader names its connection bare, its id alone,
+    none, (Default)'s settings. A grader names its connection bare, its id alone,
     so it never meets a target's pair (see `answersOf`). */
 const answerKey = (connection        , profile                )         => `${connection}.${profile ?? ""}`;
 
@@ -3716,9 +3793,13 @@ function targetAnswerKey(doc
  * answer key resolved to the settings it sends -- the profile's connection
  * where it names one, else the key's, with the profile's settings over it.
  * A grader's bare id resolves as the Target profile it named did: its
- * connection under the profile split from it. Null where any part is gone.
+ * connection under the profile split from it. One that names no profile, or
+ * a grader with none split beside it, takes [fallback] instead -- (Default)'s
+ * settings, which sit under no named profile, so a profile is still what it
+ * sets (Setup's Used by counts the same, `defaultUses`). Null where any part
+ * is gone.
  */
-function answersOf(connections                         , profiles                              ) {
+function answersOf(connections                         , profiles                              , fallback                  ) {
   const connection = (id        ) => connections.find(c => c?.id === id) ?? null;
   const profile = (id        ) => profiles.find(p => p?.id === id) ?? null;
   const split = (id        )             => {
@@ -3731,7 +3812,7 @@ function answersOf(connections                         , profiles
     const p = pid ? profile(pid) : null;
     if (pid && !p) return null;
     const c = connection(isRef(p?.connection) ? p .connection .id : at.connection);
-    return c ? mergeAnswer(c, p) : null;
+    return c ? mergeAnswer(c, p ?? fallback) : null;
   };
   return { connection, profile, split, resolve };
 }
@@ -3884,15 +3965,15 @@ SOURCE_TYPES.dynamic = {
 };
 
 // HTTP GET (docs/dynamic-sources.md § 1): the safest pull provider. It fetches
-// a URL through the relay server-side -- Snapshot now freezes one Call into the
-// Source, Live produces one fresh at submit -- so the page drives only the
+// a URL through the relay server-side -- Fetch now keeps a response, Live
+// data fetches it again at submit -- so the page drives only the
 // create dialog and the Call labels; the fetch, redaction and token-refusal are
 // the server's (its SOURCE_PROVIDERS mirror in server.py).
 DYNAMIC_PROVIDERS["http-get"] = {   // provider: the HTTP GET adapter's own id
   id: "http-get",                   // provider: its id, the row's config.provider
   label: "HTTP GET",
   icon: "square-code",
-  noun: "call",
+  noun: "response",
   tier: "pull",
   config: ["url"],
   configProblems(config){
@@ -3921,7 +4002,7 @@ DYNAMIC_PROVIDERS["api"] = {      // provider: the API call adapter's own id
   id: "api",                      // provider: its id, the row's config.provider
   label: "API call",
   icon: "arrow-left-right",
-  noun: "call",
+  noun: "response",
   tier: "pull",
   config: ["method", "url", "reqHeaders", "body", "connection"],
   configProblems(config){
@@ -3950,7 +4031,7 @@ DYNAMIC_PROVIDERS["api"] = {      // provider: the API call adapter's own id
 // or an Upload… by hand, formalised as a provider. The lab fetches and executes
 // nothing: a Supplied item is named by the id the caller gives it (its file
 // name), which is what joins it to a case. It has no config and no server-side
-// production, so no Snapshot now and no live; its Calls are the items handed in,
+// production, so no Fetch now and no live data; its Calls are the items handed in,
 // redacted on the way in like any other (server.py's take_dynamic). A hosted run
 // grades those Calls; a solely-Supplied Source with none is run only via the CLI.
 DYNAMIC_PROVIDERS["supplied"] = {   // provider: the Supplied adapter's own id
@@ -3980,6 +4061,7 @@ DYNAMIC_PROVIDERS["file"] = {   // provider: the File on disk adapter's own id
   label: "File on disk",
   icon: "file",
   noun: "file",
+  produceLabel: "Read now",
   tier: "local",
   config: ["path"],
   configProblems(config){
@@ -4003,6 +4085,7 @@ DYNAMIC_PROVIDERS["command"] = {   // provider: the Command adapter's own id
   label: "Command",
   icon: "terminal",
   noun: "call",
+  produceLabel: "Run now",
   tier: "local",
   config: ["command", "args"],
   configProblems(config){
@@ -4085,11 +4168,14 @@ CONTENT_TYPES.source = {
   description: "Every file or record in a Source from the Library.",
   // `live` is a dynamic Source's per-run snapshot-vs-live override
   // (docs/dynamic-sources.md); meaningless for another kind, ignored there.
-  fields: ["type", "ref", "first", "loops", "live"],
+  fields: ["type", "ref", "first", "loops", "live", "pin"],
   runFields: ["files", "revs"],
   validate(c, ctx, bad){
     if (!isRef(c.ref)) return void bad.push("a Source content has to name its Source as { id, name }");
     if (c.live != null && typeof c.live !== "boolean") bad.push("a Source content's live is true or false");
+    if (c.pin != null && !(isObj(c.pin) && isStr(c.pin.name) && c.pin.name && Number.isInteger(c.pin.n) && c.pin.n >= 0)) {
+      bad.push("a Source content's pin names a response as { name, n }");
+    }
     if (c.first != null && !(Number.isInteger(c.first) && c.first > 0)) {
       bad.push("Run first wants a positive number, or empty for all");
     }
@@ -4269,7 +4355,7 @@ function erroredScore                              (s   , error
 }
 
 /** The reply a metric grades and Results shows (#546): the last stage's
-    reply after its Pre-format filters, or the raw reply. */
+    reply after its filters (#557), or the raw reply. */
 function gradedText(res                                                      )         {
   const last = res?.transcript?.at(-1);
   return last?.filtered?.got ?? last?.got ?? res?.raw ?? "";
@@ -5147,6 +5233,22 @@ function withContent                           (doc   , content                )
   return { ...doc, jobs: [withStep(first, "attachContent", content ? { type: "attachContent", content } : null), ...rest] };
 }
 
+/**
+ * Whether target [i]'s words in job [k] have the item's text joined after
+ * them (#597): job 1, a target that asks a prompt, text content -- a Source
+ * of text items, or inline text, as its CONTENT_TYPES entry says -- and no
+ * {text} in the words to place it (textPrompt). A later job's reply goes
+ * only where a token places it, and an image or a document is sent as a part
+ * of its own, so neither is joined. [ctx] reads a Source as validation does.
+ */
+function contentJoined(doc                                                                                  , i        , k        ,
+                       ctx                = {})          {
+  if (k !== 0 || targetEntryOf(doc, i, k)?.asks !== "prompt") return false;
+  if (String(targetStepOf(doc, i, k)?.prompt ?? "").includes("{text}")) return false;
+  const content = contentOf(doc);
+  return !!content && (CONTENT_TYPES[content.type]?.text?.(content, ctx) ?? null) != null;
+}
+
 /** A document's targets, in column order. */
 function targetsOf(doc                                                           )           {
   return Array.isArray(doc?.targets) ? doc .targets             : [];
@@ -5366,6 +5468,15 @@ function stageModifiers(job         , resolve
   return out;
 }
 
+/** A job's Post-format filters for a run when its kind reads no items
+    (#557), in document order: every link's on its `items` filters step, which
+    a run applies to the text after the kind reads it. A kind that reads items
+    has none here -- `stageModifiers` runs its Post-format links. */
+function afterFilters(job         , resolve                                                          )           {
+  if (stageReads("items", outOf(job)?.kind) !== "text") return [];
+  return (filterStepOf(job, "items")?.filterSets || []).flatMap(link => linkFilters(link, resolve));
+}
+
 /** A job's Pre-format (text) filters for a run (#405), in document order:
     the filters of every link on its `text` filters step, which a run applies to
     the raw reply before the kind reads it into items. [resolve] gives a Library
@@ -5533,6 +5644,12 @@ function versionProblem(doc         )                {
                                                                             
                                              
                                                            
+                                                                           
+                                                                           
+                                                                          
+                                                                         
+                           
+                                                                                           
  
 
 /**
@@ -5824,7 +5941,8 @@ function ownBodiesNow(doc           )            {
 
 function upgradePipeline   (doc   , ctx             = {})    {
   // A current document is read as it is, but for a profile reference the Runs
-  // tab saved whole (see profileRefs), which is cut back, and a step on a
+  // tab saved whole (see profileRefs), which is cut back, a Read Reply's
+  // leftover `preamble: false` (withoutLeftoverPreamble), and a step on a
   // profile a target step stands in for (localSteps).
   if (isObj(doc) && doc.version === PIPELINE_VERSION) {
     let out = doc             ;
@@ -5832,12 +5950,16 @@ function upgradePipeline   (doc   , ctx             = {})    {
       out = clone(out)             ;
       profileRefs(out);
     }
-    return localSteps(ownBodiesNow(out), ctx)     ;
+    return localSteps(ownBodiesNow(withoutLeftoverPreamble(out)), ctx)     ;
   }
-  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16].includes(doc.version          )) return doc;
+  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17].includes(doc.version          )) return doc;
+  // Every version reads as 17 first, then as 18: a saved-mode Source content
+  // pins one response (savedPinned).
+  const eighteen = (d           )    => localSteps(ownBodiesNow(savedPinned(d, ctx)), ctx)     ;
+  if (doc.version === 17) return eighteen(cutRefs(clone(doc)             ));
   // Every version reads as 16 first, then as 17: its requests read by their
   // job's HTTP API (requestsOnJobApi).
-  const seventeen = (d           )    => localSteps(ownBodiesNow(requestsOnJobApi(d)), ctx)     ;
+  const seventeen = (d           )    => eighteen(requestsOnJobApi(d));
   if (doc.version === 16) return seventeen(cutRefs(clone(doc)             ));
   // Every version reads as 15 first, then as 16: its Target profiles split
   // into AI Connections and Connection Profiles (answersFromProfiles).
@@ -5883,11 +6005,13 @@ function filtersFromCleaning(next           )            {
   const isFilters = (st           ) => isObj(st) && st.type === "filters";
   next.jobs = next.jobs.map((job           ) => {
     if (!isObj(job) || !Array.isArray(job.steps)) return job;
-    const steps = job.steps               ;
-    const readReply = steps.find(st => isObj(st) && st.type === "readReply")                         ;
+    const readReply = job.steps.find((st           ) => isObj(st) && st.type === "readReply")                         ;
     const preamble = !!(readReply && isObj(readReply.out) && readReply.out.preamble === true);
+    // Stripped from every Read Reply, `false` included, before the job is
+    // asked whether it has anything to migrate (#604).
+    const steps = (job.steps               ).map(withoutPreamble);
     const stale = steps.some(st => isFilters(st) && (st             ).stage == null);
-    if (!steps.some(isCleaning) && !preamble && !stale) return job;
+    if (!steps.some(isCleaning) && !preamble && !stale) return { ...job, steps };
     const jid = isStr(job.id) ? job.id : "job";
 
     // Walk the Responses steps, turning each into a chunk of items filters
@@ -5916,11 +6040,6 @@ function filtersFromCleaning(next           )            {
     const out              = [];
     let placed = false;
     for (const st of steps) {
-      if (isObj(st) && st.type === "readReply" && isObj(st.out) && "preamble" in st.out) {
-        const { preamble: _p, ...rest } = st.out                           ;
-        out.push({ ...st, out: rest });
-        continue;
-      }
       if (isCleaning(st) || isFilters(st)) {
         if (!placed) { if (textStep) out.push(textStep); if (itemsStep) out.push(itemsStep); placed = true; }
         continue;
@@ -5933,6 +6052,30 @@ function filtersFromCleaning(next           )            {
   return next;
 }
 
+/** [st] without its List `preamble` setting, when it is a Read Reply holding
+    one; [st] itself otherwise. */
+function withoutPreamble(st           )            {
+  if (!isObj(st) || st.type !== "readReply" || !isObj(st.out) || !("preamble" in st.out)) return st;
+  const { preamble: _p, ...out } = st.out                           ;
+  return { ...st, out };
+}
+
+/** Whether a current document still carries a `preamble: false` that an
+    upgrade before #604 left behind. It did nothing, so it is cut as the
+    document is read; a `preamble: true` the lab never wrote at this version is
+    left for validation to refuse. */
+function leftoverPreamble(doc           )          {
+  return Array.isArray(doc.jobs) && doc.jobs.some((job           ) => isObj(job) && Array.isArray(job.steps)
+    && job.steps.some((st           ) => isObj(st) && st.type === "readReply" && isObj(st.out) && st.out.preamble === false));
+}
+
+/** [doc] with each Read Reply's leftover `preamble: false` cut (leftoverPreamble). */
+function withoutLeftoverPreamble(doc           )            {
+  if (!leftoverPreamble(doc)) return doc;
+  return { ...doc, jobs: doc.jobs.map((job           ) => (!isObj(job) || !Array.isArray(job.steps) ? job
+    : { ...job, steps: job.steps.map((st           ) => (isObj(st) && isObj(st.out) && st.out.preamble === false ? withoutPreamble(st) : st)) })) };
+}
+
 /** [doc] with no Metrics eval holding the `case` metric: a dataset's cases
     are metrics now (dataset body version 5), which an eval naming the
     dataset adds for each item, so the case's own metrics carry what that
@@ -5986,6 +6129,24 @@ function localSteps(doc           , ctx            )            {
   return next;
 }
 
+/**
+ * Version 17 to 18 (#596): Saved response pins one response. A Source content
+ * graded as saved -- `live` false, or unset over a Source whose default is
+ * saved, which [ctx] says -- pins the Source's newest successful response
+ * (owner), so a Source holding several grades one where it graded all. Live
+ * data, a content that is not a Source's, or one [ctx] cannot say of, pins
+ * none; a run document carries its files, and needs none.
+ */
+function savedPinned(next           , ctx            )            {
+  next.version = PIPELINE_VERSION;
+  const content = contentOf(next            )             ;
+  if (!isObj(content) || content.type !== "source" || content.live === true || content.pin != null
+      || isObj(next.profiles) || !isRef(content.ref) || !ctx.pinFor) return next;
+  const pin = ctx.pinFor(content.ref.id, typeof content.live === "boolean" ? content.live : undefined);
+  if (pin) content.pin = { name: pin.name, n: pin.n };
+  return next;
+}
+
 /**
  * Version 16 to 17 (#477): a request step has no HTTP API of its own; its
  * job's flow step says it. A step that named another -- Any endpoint over an
@@ -6439,7 +6600,11 @@ function resolvePipeline(doc          ,
     run.profiles[key] = { ...conn, connection: cid };
   }
   const content = contentOf(run)             ;
-  if (content?.type === "source") content.files = CONTENT_TYPES.source .expand (content, ctx.files);
+  // Saved response grades its one pinned response (#596), whatever else the
+  // Source holds; the server keeps that response's bytes with the run.
+  if (content?.type === "source") {
+    content.files = CONTENT_TYPES.source .expand (content, isObj(content.pin) && content.live !== true ? [content.pin.name] : ctx.files);
+  }
   // Each Library group the evals read, as the version it grades with: the
   // body's fingerprint and, where the lab numbers them, `n` -- a pinned
   // link's pin.
@@ -6706,7 +6871,8 @@ const BUNDLE_DATASET = /^datasets\/([a-z0-9]+(?:-[a-z0-9]+)*)\.json$/;
  * [doc] as a bundle's text files: `pipeline.yaml`, `profiles.yaml` and
  * `datasets/<slug>.json`. [ctx.connections] and [ctx.connectionProfiles]
  * are Setup's -- keys and all, none of which is written -- or
- * [ctx.profiles], Target profiles to split as the store was; [ctx.grader]
+ * [ctx.profiles], Target profiles to split as the store was, and
+ * [ctx.profileDefault] (Default)'s settings; [ctx.grader]
  * the lab's grader, written into each eval that would have asked it, and
  * [ctx.datasets] each graded dataset by id, as `{ name, body }`. Each
  * connection asked is written resolved, under its answer key with the
@@ -6715,12 +6881,13 @@ const BUNDLE_DATASET = /^datasets\/([a-z0-9]+(?:-[a-z0-9]+)*)\.json$/;
  * something else.
  */
 function exportBundle(doc          , ctx                                                                         
-                                                                                                                  
-                                                                                                            = {}) 
+                                                                                                                               
+                                                                                                                                 = {}) 
                                                {
   const split = (ctx.profiles ?? []).map(splitProfile);
   const profiles = withSlugs([...(ctx.connections ?? []), ...split.map(x => x.connection)]                          );
-  const answers = answersOf(profiles, [...(ctx.connectionProfiles ?? []), ...split.flatMap(x => (x.profile ? [x.profile] : []))]);
+  const answers = answersOf(profiles, [...(ctx.connectionProfiles ?? []), ...split.flatMap(x => (x.profile ? [x.profile] : []))],
+                           ctx.profileDefault);
   // What the lab would supply at submit, written down: no lab supplies it later.
   const out = clone(doc)             ;
   out.evals = (Array.isArray(out.evals) ? out.evals : [])
@@ -6896,7 +7063,7 @@ function stagesFor(run             , i        , resolveFilters
     // without a resolver, a Library link reads nothing, a private one its own.
     return { text: targetStepOf(run, i, k) .prompt, kind, withImage: sendsImage(ch),
              verbatim: !!targetEntryOf(run, i, k)?.verbatim, before: beforeFilters(ch, resolveFilters),
-             modifiers: stageModifiers(ch, resolveFilters), settings };
+             after: afterFilters(ch, resolveFilters), modifiers: stageModifiers(ch, resolveFilters), settings };
   });
   // Each named by the connection whose key it is sent with.
   const asked = (key        , held                        ) => {
@@ -7178,8 +7345,10 @@ function validateEvals(ev           , files
       continue;
     }
     if (scored.some(m => m.type === "refused" && !m.not)) {
-      // Nor does a call that was refused: there is no answer.
-      if (scored.length > 1) say("expects its call refused, and states something the answer should hold too");
+      // Nor does a call that was refused: there is no answer. Scored in
+      // points, the other metrics are alternatives rather than more to hold
+      // (#560: refused, or the reply says it can't read the file).
+      if (scored.length > 1 && ev.scoring?.mode !== "weighted") say("expects its call refused, and states something the answer should hold too");
       continue;
     }
     const wants = scored.filter(m => !m.not && (m.type === "contains-all" || m.type === "contains-any"));
@@ -7304,12 +7473,12 @@ export {
     SOURCE_TYPES, DEFAULT_SOURCE_TYPE, sourceTypeOf, WORKFLOW_PLATFORMS, platformOf, DYNAMIC_PROVIDERS, providerOf, WIZARDS, wizardProgress,
     registerKinds, defaultKind, jobDefaults, applyModifiers, keyVar, slugOf, uniqueSlug, isSlug, withSlugs, SLUG_MAX, connectionOf, addressOf, connectionSettings,
     splitProfile, mergeAnswer, answerKey, answerOfKey, answersOf, runAnswer, answerConnectionId, connectionScoped,
-    connectionRequest, connectionBase, connectionSend, connectionReply, connectionReplyProblem, connectionImage, connectionProblems,
+    connectionRequest, connectionBase, connectionSend, connectionReply, connectionReplyProblem, replyFinish, contentJoined, connectionImage, connectionProblems,
 CONNECTION_FIELDS, SETTING_KEYS, OPTION_FIELDS, jobLabel, scenarioLabel, jobWords, versionProblem,
     contentOf, withContent, replyOf,
     targetsOf, targetLetter, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetAnswerOf, targetAnswerKey, recordedTargetOf, withTarget,
     tokensOf, withTokens, sendsImage, withImage, flowStepOf, requestStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
-    filterSetsOf, withFilterSets, stageModifiers, beforeFilters, gradedText,
+    filterSetsOf, withFilterSets, stageModifiers, beforeFilters, afterFilters, gradedText,
     blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, newLink, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds, answerKeys,
     scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioEvals, scenarioPasses, groupPass, overallVerdict, passVerdict, isSkipped, failedScore,
     evalLabel, evalsOf, evalsDataset, casesRef, groupOf, groupOfMetrics, pipelineWarnings, itemsFound, previewOf, isWholeRun, EVAL_FIELDS,

package/lab/flows/record.mjs

@@ -153,11 +153,12 @@ export function skeletonFor(step        , reads                 )             {
     call of the step at that iteration: imported twice, it is skipped the
     second time, while another run's call -- the same request, sent again,
     answered afresh -- is its own record, since a flow that sends the same
-    words every time is exactly where production's replies differ. One
-    written by hand or uploaded is the same as another that read and sent the
-    same things. */
+    words every time is exactly where production's replies differ. An edited
+    call keeps its run's key, so Import runs skips it too (#587). One written
+    by hand or uploaded is the same as another that read and sent the same
+    things. Mirrored by server.py's record_key. */
 export function recordKey(r                                                           )         {
-  if (r.origin?.kind === "run" && r.origin.run) {
+  if (r.origin?.run) {
     return JSON.stringify({ step: r.step, run: r.origin.run, iteration: r.origin.iteration ?? "" });
   }
   const sorted = (v         )          => Array.isArray(v) ? v.map(sorted)