evals-lab 0.8.0 → 0.9.0

package/CHANGELOG.md

@@ -5,6 +5,48 @@ Newest first. Read a release's **Upgrade notes** before installing it: an
 upgrade can rewrite what the lab keeps in your data directory, and an older
 version cannot always read it back.
 
+## 0.9.0
+
+### Upgrade notes
+
+- **Copy your data directory before upgrading** (see "User data" in the
+  README). A pipeline 0.9.0 saves is pipeline version 14, which 0.8.x
+  cannot read. To go back, reinstall 0.8.0 and restore the copy.
+- A job's Drop items and Reject the answer rules become one private filter
+  set in that job, which keeps and rejects exactly what they did. Pipelines
+  read as version 14, and are saved as it at their next edit.
+- On first start, 0.9.0 adds the tables filter sets are kept in, and each
+  run keeps the filter sets it ran with.
+
+### Added
+
+- Library › Filter sets: named, versioned lists of filters that clean a list
+  reply. A Drop item filter drops the items its conditions match; a Reject
+  reply filter rejects the whole reply. Filters run in their list's order.
+- A job's Responses stage links filter sets, following each one's latest
+  version or pinned to one, or holds a private set of its own.
+- Each Target profile shows its connection status: Not tested, Connected,
+  Key needed or Unreachable, checked quietly when Setup opens. The dot opens
+  the Test connection result.
+- Target profiles list each profile's Model and what uses it, with quick
+  filters: All, Connected, Needs attention and Grader. A profile's edit
+  sheet links the pipelines and eval groups that use it.
+- Selected profiles can be tested, or have their workspaces changed, all
+  at once.
+- An HTTP Request target's prompt can be edited. Insert field… adds one of
+  the item's fields, and the preview highlights the words changed from the
+  recorded prompt.
+- The Recorded target shows the flow's original prompt.
+
+### Changed
+
+- Target profiles no longer show Type, Address and Key columns, or the
+  Type, Key and Sort selects.
+- The Edited target's request template is in Flow's request, folded until
+  opened.
+- Drop items and Reject the answer are no longer offered under a job's
+  Responses: link a filter set instead.
+
 ## 0.8.0
 
 ### Upgrade notes

package/README.md

@@ -11,6 +11,7 @@ Runs are kept in history which can be exported.
 Currently in early release. Runs locally and doesn't send your data anywhere else.
 
 Example use-cases:
+- Keywording images using a model's vision capability, and running evals on the tags it returns
 - Providing images to an LLM and comparing responses for accuracy
 - Checking which model can meet evals in a Power Automate workflow
 - Determining which prompt gets the highest score in metrics

package/lab/VERSION

@@ -1 +1 @@
-0.8.0 (2026.10.07-484)
+0.9.0 (2026.10.07-516)

package/lab/demo/pipelines/demo-1.json

@@ -1,5 +1,5 @@
 {
-  "version": 13,
+  "version": 14,
   "id": "demo-1",
   "name": "Demo 1",
   "jobs": [

package/lab/demo/pipelines/demo-2.json

@@ -1,5 +1,5 @@
 {
-  "version": 13,
+  "version": 14,
   "id": "demo-2",
   "name": "Demo 2",
   "jobs": [

package/lab/evals-core.mjs

@@ -33,12 +33,14 @@
 // and that is a measurement.
 
 import yaml from "./js-yaml.mjs";
-                                                
+                                                                                                                                                 
 import { evaluate, asText, WdlError,                 } from "./flows/wdl.mjs";
 import { stepsOf as flowStepsOf, actionsByName, readerFor, readsOf,                          } from "./flows/record.mjs";
 import { flowApi,              } from "./flows/flowApi.mjs";
 import registerMetrics from "./metrics/builtin.mjs";
-import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProblems, dropBy, rejectBy, matcher } from "./kinds/list.mjs";
+import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProblems, dropBy, rejectBy, matcher,
+  FILTER_TYPES, FILTER_SEPARATORS, FILTER_MODIFIER, applyFilter, applyFilterSet, tryFilter, splitSample, filterProblems, filterFromItemRule,
+                             } from "./kinds/list.mjs";
 
 // ---- The types -------------------------------------------------------------
 // docs/pipeline-model.md as types, and the shapes of the registries every
@@ -138,9 +140,19 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                      
  
 
+/** The filter sets a job cleans and gates its reply by (#339): each a link to
+    a Library filter set (followed latest or pinned) or a private one held
+    inline. A Responses step beside the modifiers, at the same place, so its
+    sets' filters apply in document order with them -- what the Drop items and
+    Reject the answer modifiers did inline, lifted into linked sets. */
+                             
+                  
+                              
+ 
+
 /** A job's steps: its Content stage, then its Responses (§16). */
                                                                                          
-                                              
+                                                           
 
 /** A target step that prompts a model: its words, resolved under the job's
     token mappings, and a profile of its own where it names one. */
@@ -316,6 +328,45 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
 /** A pipeline's evals, in the order they read a run. */
                            
 
+/**
+ * A filter set (issue #335): a versioned Library body holding the filters
+ * that shape a list reply, mirroring DatasetBody / the eval group. The lab
+ * will keep each as a row and a run grade against the body kept with it, as a
+ * dataset does (#336); for now it is the format a filter set reads and writes
+ * at. `filters` are tried in order -- each drop-item narrows the items the
+ * next reads, and the first reject-reply that fires gates the answer.
+ */
+                                
+                                     
+                    
+ 
+
+/** A Library filter set by reference; a run's copy says the version it used:
+    the body's fingerprint, and its number. Mirrors GroupRef. */
+                                                                  
+
+/** A private filter set: a filter-set body held in the pipeline rather than
+    the Library -- what Runs makes when a link is not to a Library set (#339),
+    as a private eval group (OwnGroup) is. A filter set has no cases, so this
+    is the body itself. */
+                                         
+
+/**
+ * A pipeline's link to a filter set in Responses (issue #335, mirrors
+ * GroupEval / OwnGroup): `set` names a Library set, followed at its newest
+ * version (`pin: null`) or pinned at one (`pin: n`); a private set is `own`,
+ * with `set: null`. From version 14 a job's Responses stage carries these on
+ * a `filters` step (#339); the upgrade from 13 migrates the inline Drop items
+ * / Reject rules into a private one, and the worker cleans and gates a run's
+ * replies against the kept bodies.
+ */
+                                
+             
+                           
+                     
+                     
+ 
+
                            
                                    
                                                                              
@@ -874,6 +925,11 @@ import list, { ECHO, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProb
                  
                                                                      
                        
+                                                                              
+                                                                           
+                                                                              
+                                                                              
+                   
                                               
                               
                             
@@ -1233,6 +1289,57 @@ function upgradeDatasetBody   (body   )                  {
   return recordedIdsV7(groupOfV6({ source: null, cases: raw.map(caseOfV4).map(caseOfV5) }));
 }
 
+/** The filter-set body's version (issue #335). Version 1 is the first: the
+    `{ version, filters }` body #334 introduces. A later format adds a step to
+    `upgradeFilterSetBody`, as `upgradeDatasetBody` grew, with server.py's
+    `upgrade_filter_set_body` its twin. */
+const FILTER_SET_VERSION = 1         ;
+
+/** [body] as this version of a filter set (1), from any earlier one. There is
+    no earlier version yet, so a version-1 body reads as it is and anything
+    else comes back untouched for `filterSetBodyProblems` to refuse. Pure: the
+    same body gives the same answer, and server.py's upgrade_filter_set_body is
+    its twin. */
+function upgradeFilterSetBody   (body   )                    {
+  if (!isObj(body)) return body;
+  if (body.version === FILTER_SET_VERSION) return body;
+  return body;
+}
+
+/** A filter set with no filters, at this version: what New filter set starts
+    from, and what a lab with none holds. */
+function blankFilterSetBody()                {
+  return { version: FILTER_SET_VERSION, filters: [] };
+}
+
+/** Why [body] is not a filter set the lab reads, one sentence each: an object
+    at this version whose filters are a valid filter list (filterProblems). For
+    the store (#336) and the pipeline link (#339). */
+function filterSetBodyProblems(body         , at = "the filter set")           {
+  if (!isObj(body)) return [`${at} has to be a JSON object`];
+  if (body.version !== FILTER_SET_VERSION) {
+    return [`${at} is version ${JSON.stringify(body.version)}, and the lab reads version ${FILTER_SET_VERSION}`];
+  }
+  return filterProblems(body.filters, at);
+}
+
+/** Why [link] is not a filter-set link a run's Responses step may carry, one
+    sentence each (#339): an id, exactly one of a Library `set` reference or a
+    private `own` body (whose shape is the core's to judge), and a `pin` that
+    is null or a whole number. A Library set is the server's to resolve, so its
+    body is not checked here. */
+function filterSetLinkProblems(link         , at = "a filter set")           {
+  if (!isObj(link)) return [`${at}: a filter-set link has to be a JSON object`];
+  const bad           = [];
+  if (!isStr(link.id) || !link.id.trim()) bad.push(`${at}: a filter-set link has no id`);
+  const hasSet = isObj(link.set) && isStr(link.set.id);
+  const hasOwn = link.own != null;
+  if (hasSet === hasOwn) bad.push(`${at}: a filter-set link names a Library set or holds a private one, not both`);
+  if (hasOwn) bad.push(...filterSetBodyProblems(link.own, `${at}: the private filter set`));
+  if (link.pin != null && !Number.isInteger(link.pin)) bad.push(`${at}: a pin is a whole number or none`);
+  return bad;
+}
+
 /** A version-6 body as a version-8 eval group: scored All, the lab's
     grader, and no metrics of its own for every item or the whole run --
     what a Metrics eval naming the dataset with none of its own graded, which
@@ -2936,7 +3043,7 @@ function applyModifiers   (list                                 , kind        ,
 // 12: a Contains metric's Ignore case holds item by item too.
 // 13: an eval is a link to an eval group, or a group of the pipeline's own,
 // and the document has an overall pass rule (pipeline-model §17).
-const PIPELINE_VERSION = 13         ;
+const PIPELINE_VERSION = 14         ;
 
 // Plain objects, so an entry is added by assignment and a reader never needs
 // to know which registered it.
@@ -4234,6 +4341,23 @@ STEP_TYPES.modifier = {
   },
 };
 
+// A filters step: the filter sets a job cleans and gates its reply by, each a
+// link or a private set (#339). It sits with the modifiers (rank 2), so its
+// sets apply in document order among them. A private set's body is the core's
+// to judge (filterSetBodyProblems); a link to a Library set is the server's to
+// resolve, so only its shape is checked here.
+STEP_TYPES.filters = {
+  label: "Filter sets", slot: "responses", rank: 2, many: true,
+  description: "Cleans and gates the reply with linked or private filter sets.",
+  in: "value", out: "value",
+  apply: "runPipeline",
+  fields: ["type", "filterSets"],
+  validate(step, _ctx, bad, at){
+    if (!Array.isArray(step.filterSets)) return void bad.push(`${at}: filterSets has to be a list`);
+    for (const link of step.filterSets               ) bad.push(...filterSetLinkProblems(link, at));
+  },
+};
+
 // ---- reading a job's steps, and a target's -----------------------------------
 // Every reader asks these, never a step's position or the document's shape:
 // which steps a job holds is its own, and a version that moves a field
@@ -4429,12 +4553,58 @@ function outOf(job         )      {
   return { ...read, modifiers }       ;
 }
 
-/** [job] reading its reply as [out]: its Format Validation and modifiers. */
+/** [job] reading its reply as [out]: its Format Validation and modifiers.
+    The modifiers replace the job's modifier steps in place, so a modifier step
+    that interleaves with a filters step (#339) keeps its place among them --
+    an edit to one modifier never reorders another step past it. Extra
+    modifiers (one added) come after the last, and a step with no modifier left
+    (one removed) drops out. */
 function withOut(job     , out     )      {
   const { modifiers, ...read } = out;
-  const steps = job.steps.filter(st => st.type !== "readReply" && st.type !== "modifier");
-  return { ...job, steps: ordered([...steps, { type: "readReply", out: read },
-    ...(modifiers || []).map(modifier => ({ type: "modifier", modifier })        )]) };
+  const mods = modifiers || [];
+  let mi = 0;
+  const steps         = [];
+  for (const st of job.steps) {
+    if (st.type === "readReply") continue;        // re-added below, in its place
+    if (st.type === "modifier") { if (mi < mods.length) steps.push({ type: "modifier", modifier: mods[mi++]  }); }
+    else steps.push(st);                           // readAs, filters, content -- kept where they are
+  }
+  for (; mi < mods.length; mi++) steps.push({ type: "modifier", modifier: mods[mi]  });
+  return { ...job, steps: ordered([...steps, { type: "readReply", out: read }]) };
+}
+
+/** The filter-set links a job's Responses stage carries, in document order
+    (#339): read from its one filters step, or none. */
+function filterSetsOf(job         )                  {
+  return (stepOf            (job, "filters")?.filterSets ?? []).slice();
+}
+
+/** [job] with its filter-set links set to [links]: its filters step added,
+    replaced, or taken away when none are left. */
+function withFilterSets(job     , links                          )      {
+  return withStep(job, "filters", links.length ? { type: "filters", filterSets: [...links] } : null);
+}
+
+/** A job's Responses as an ordered modifier list for a run (#339): each
+    modifier step as it is, and each filter-set link as a synthetic modifier
+    that applies the set's filters where the link sits -- so a linked or
+    private set cleans and gates a list reply exactly as the Drop items /
+    Reject modifiers it replaced did, in the same order. [resolve] gives a
+    Library link's body (the worker's kept bodies); a private (`own`) link
+    carries its own, and a Library link with no body resolves to nothing. */
+function stageModifiers(job         , resolve                                                          )             {
+  const out             = [];
+  for (const st of job?.steps || []) {
+    if (!isObj(st)) continue;
+    if (st.type === "modifier") out.push(st.modifier);
+    else if (st.type === "filters") {
+      for (const link of (st.filterSets                   ) || []) {
+        const body = link.own ?? (link.set && resolve ? resolve(link.set) : null);
+        out.push({ type: FILTER_MODIFIER, filters: body?.filters || [] });
+      }
+    }
+  }
+  return out;
 }
 
 /** A new id: random, so one minted in one browser never collides with one
@@ -4676,6 +4846,13 @@ function newLink(group     , name = "")            {
   return { type: "group", id: newId(), name, continueOnFailure: true, group: { id: group.id, name: group.name }, pin: null };
 }
 
+/** A new pipeline link to the Library filter set [set], followed at its newest
+    version. Mirrors newLink; for Runs to use when it links a set in Responses
+    (#339). */
+function newFilterLink(set     )                {
+  return { id: newId(), set: { id: set.id, name: set.name }, pin: null };
+}
+
 function jobsOf(next           )            {
   next.version = PIPELINE_VERSION;
   if (Array.isArray(next.chains)) {
@@ -4822,16 +4999,62 @@ function upgradePipeline   (doc   , ctx             = {})    {
     }
     return localSteps(out, ctx)     ;
   }
-  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12].includes(doc.version          )) return doc;
+  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13].includes(doc.version          )) return doc;
+  // Version 13 reads as 14 by migrating a job's inline Drop items / Reject
+  // rules into a private filter set (#339); nothing else about it changes.
+  if (doc.version === 13) {
+    return localSteps(filtersFromRules(clone(doc)             ), ctx)     ;
+  }
   if (doc.version === 11 || doc.version === 12) {
-    return localSteps(groupsOf(withoutCaseMetric(caseAsWritten(cutRefs(clone(doc)             )))), ctx)     ;
+    return localSteps(filtersFromRules(groupsOf(withoutCaseMetric(caseAsWritten(cutRefs(clone(doc)             ))))), ctx)     ;
   }
   // Every version before 10 reads as version 9 first, then as 10, then as 11,
-  // 12 and 13.
+  // 12, 13 and 14.
   // A version-10 document is cut as a current one was (profileRefs); the
   // earlier ones are cut on their way through nineOf.
   const ten = doc.version === 10 ? cutRefs(clone(doc)             ) : targetsFromScenarios(nineOf(clone(doc)             , ctx));
-  return localSteps(groupsOf(withoutCaseMetric(caseAsWritten(evalsKey(ten)))), ctx)     ;
+  return localSteps(filtersFromRules(groupsOf(withoutCaseMetric(caseAsWritten(evalsKey(ten))))), ctx)     ;
+}
+
+/** [doc] with each job's inline Drop items / Reject modifier steps migrated
+    into one private filter set on a `filters` step, version 13 to 14 (#339).
+    Every drop/reject modifier's rules become filters in document order -- a
+    `drop` modifier's rules drop-item filters, a `reject` modifier's
+    reject-reply filters (filterFromItemRule, which carries the matcher
+    unchanged) -- in one `own` set, and the step takes the place of the first
+    of those modifiers, so the reply is cleaned and gated as it was. A job with
+    no such modifier, or only empty ones, is left as it is; this is a no-op on
+    every path that reaches version 14, so it is safe to run on all of them. */
+function filtersFromRules(next           )            {
+  next.version = PIPELINE_VERSION;
+  if (!Array.isArray(next.jobs)) return next;
+  const TYPE                         = { drop: "drop-item", reject: "reject-reply" };
+  const isRule = (st           )          =>
+    isObj(st) && st.type === "modifier" && isObj(st.modifier) && (st.modifier.type          ) in TYPE;
+  next.jobs = next.jobs.map((job           ) => {
+    if (!isObj(job) || !Array.isArray(job.steps) || !job.steps.some(isRule)) return job;
+    const filters           = [];
+    for (const st of job.steps               ) {
+      if (!isRule(st)) continue;
+      const m = (st             ).modifier             ;
+      for (const r of Array.isArray(m.rules) ? m.rules               : []) {
+        filters.push(filterFromItemRule(r, TYPE[m.type          ] ));
+      }
+    }
+    // Empty Drop items / Reject modifiers held nothing, so they simply drop.
+    // The link's id is the job's, so re-upgrading a document mints the same
+    // one (the migration is idempotent) and a run keyed by it reads the same.
+    const link                       = filters.length
+      ? { id: `filters-${isStr(job.id) ? job.id : "job"}`, set: null, pin: null, own: { version: FILTER_SET_VERSION, filters } } : null;
+    let placed = false;
+    const steps              = [];
+    for (const st of job.steps               ) {
+      if (!isRule(st)) { steps.push(st); continue; }
+      if (link && !placed) { steps.push({ type: "filters", filterSets: [link] }); placed = true; }
+    }
+    return { ...job, steps };
+  });
+  return next;
 }
 
 /** [doc] with no Metrics eval holding the `case` metric: a dataset's cases
@@ -5636,13 +5859,16 @@ function readBundle(files             , ctx                                = {})
  * the job it sits in, and the connection it asks -- its own profile, or
  * its scenario's.
  */
-function stagesFor(run             , i        ) 
+function stagesFor(run             , i        , resolveFilters                                                          ) 
                                                                                                         
                                                                                {
   const stages = run.jobs.map((ch, k) => {
-    const { kind, modifiers, ...settings } = outOf(ch);
+    const { kind, modifiers: _mods, ...settings } = outOf(ch);
+    // The modifiers a run applies, with each linked or private filter set a
+    // synthetic modifier where its link sits (#339); without a resolver, a
+    // Library link reads nothing, a private one its own body.
     return { text: targetStepOf(run, i, k) .prompt, kind, withImage: sendsImage(ch),
-             verbatim: !!targetEntryOf(run, i, k)?.verbatim, modifiers: modifiers || [], settings };
+             verbatim: !!targetEntryOf(run, i, k)?.verbatim, modifiers: stageModifiers(ch, resolveFilters), settings };
   });
   const connections = run.jobs.map((_, k) => {
     // A step answered here asks nothing: the profile a run made before it
@@ -6039,9 +6265,13 @@ CONNECTION_FIELDS, SETTING_KEYS, OPTION_FIELDS, jobLabel, scenarioLabel, jobWord
     contentOf, withContent, replyOf,
     targetsOf, targetLetter, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetProfileOf, recordedTargetOf, withTarget,
     tokensOf, withTokens, sendsImage, withImage, flowStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
+    filterSetsOf, withFilterSets, stageModifiers,
     blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, newLink, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds,
     scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioEvals, scenarioPasses, groupPass, overallVerdict, passVerdict, isSkipped, failedScore,
     evalLabel, evalsOf, evalsDataset, casesRef, groupOf, groupOfMetrics, pipelineWarnings, isWholeRun, EVAL_FIELDS,
     pipelineToYaml, importPipeline, yamlToPipeline, exportBundle, readBundle, bundleProblems, junitXml,
     pluginHost, legacyTagsOut, splitItems, tidy, tryExamples, itemRulesProblems, dropBy, rejectBy, matcher,
+    FILTER_SET_VERSION, FILTER_TYPES, FILTER_SEPARATORS, applyFilter, applyFilterSet, tryFilter, splitSample,
+    filterProblems, filterFromItemRule, upgradeFilterSetBody, blankFilterSetBody, filterSetBodyProblems, filterSetLinkProblems,
+    newFilterLink, FILTER_MODIFIER,
 };