evals-lab 0.16.0 → 0.17.0

package/CHANGELOG.md

@@ -5,6 +5,35 @@ Newest first. Read a release's **Upgrade notes** before installing it: an
 upgrade can rewrite what the lab keeps in your data directory, and an older
 version cannot always read it back.
 
+## 0.17.0
+
+### Added
+
+- Home's footer says when a newer version is out. The lab asks npm at most
+  once a day; `LAB_NO_UPDATE_CHECK=1` turns it off.
+- A demo that grades with no AI Connection: "Demo: recorded replies", whose
+  Variants answer with saved replies.
+- An experiment answered only by Echo or Recorded steps runs in a lab with no
+  AI Connection.
+
+### Changed
+
+- The package is about 10 MB, not 46: the demo's images are 1024 px on the
+  long side.
+
+### Fixed
+
+- Trim with bullets drops a list's stray square brackets, as in
+  `[first, second]`, and a bracketed label before the first item.
+- Results: a Variant's connection wraps in its own column. Experiments ›
+  Variants: a long Prompt wraps instead of being cut to one line.
+
+## 0.16.1
+
+### Fixed
+
+- History's Overview shows its Items readably on a phone.
+
 ## 0.16.0
 
 ### Upgrade notes

package/README.md

@@ -2,8 +2,9 @@
 
 Test what a model, an HTTP endpoint or an API answers, side by side, and score
 it with evals you define in advance. Runs locally in a friendly UI, and
-doesn't send your data anywhere you haven't pointed it.
-
+doesn't send your data anywhere you haven't pointed it. The one request of its
+own is a daily look at npm for a newer version, which sends nothing about you or
+your lab ([Options](#options) turns it off).
 An experiment takes items from a **Source** and sends each one to one or more
 **Variants**, then scores the **Responses**.
 
@@ -18,7 +19,7 @@ An experiment takes items from a **Source** and sends each one to one or more
   a fresh one at run time.
 
 Evals include parsing checks, item counts, comparison with production
-responses, grading by a model, and evals (evals) shared between
+responses, grading by a model, and evals shared between
 experiments. Run history is kept and can be exported.
 
 The same evals run in CI: `evals-lab run` fails the build when an eval
@@ -29,8 +30,6 @@ Currently in early release.
 Example uses:
 - Checking that an HTTP endpoint or internal API still answers correctly
   after a change, in the lab or on every commit in CI
-- Keywording images with a model's vision, and running evals on the
-  keywords it returns
 - Comparing two models, or two prompts, on the same items and seeing which
   scores higher
 - Grading what an API returns against an eval of expected answers
@@ -45,8 +44,8 @@ npm install -g evals-lab
 evals-lab
 ```
 
-Open <http://localhost:8080>. A new lab starts with six demos, each an eval
-and its experiment, which run once you add an AI Connection in Setup. Note the first start takes longer while it installs them. Without
+Open <http://localhost:8080>. A new lab starts with seven demos, each an eval
+and its experiment. "Demo: recorded replies" runs as it is; the rest run once you add an AI Connection in Setup. Note the first start takes longer while it installs them. Without
 installing: `npx evals-lab`.
 
 | Needs | Windows | Linux |
@@ -84,6 +83,7 @@ evals-lab [--port <n>] [--host <address>] [--data-dir <dir>]
 | | `LAB_PASSWORD` | none |
 | | `OLLAMA_URL` | `http://127.0.0.1:11434` |
 | | `PYTHON` | `py -3`, `python` or `python3`, whichever is found |
+| | `LAB_NO_UPDATE_CHECK` | unset; `1` stops the daily look at npm for a newer version |
 
 `--host 0.0.0.0` makes the lab reachable from your network and needs `LAB_PASSWORD` to be set (otherwise it refuses connections). 
 
@@ -144,7 +144,7 @@ jobs:
       # Only when the items are images:
       # - run: sudo apt-get update && sudo apt-get install -y imagemagick
       - run: >
-          npx --yes evals-lab@0.5 run evals/
+          npx --yes evals-lab@0.17 run evals/
           --junit evals.xml --json evals.json --summary "$GITHUB_STEP_SUMMARY"
         env:
           EVALSLAB_API_KEY_ANTHROPIC: ${{ secrets.ANTHROPIC_API_KEY }}
@@ -205,9 +205,10 @@ each preset's request fields:
 
 ## Getting started
 
-First set up a Setup profile: **Setup › Add profile**, then pick its type —
-Ollama (local or cloud), OpenAI-compatible, Anthropic, llama.cpp or an HTTP
-endpoint — and give its address, model and key. NB: An Ollama profile without an address uses `OLLAMA_URL`.
+First add an AI Connection: **Setup › Connections › Add connection**, then
+pick its type — Ollama (local or cloud), OpenAI-compatible, Anthropic,
+llama.cpp or an HTTP endpoint — and give its address, model and key. An
+Ollama connection without an address uses `OLLAMA_URL`.
 
 Set up your data source(s) in Library.
 

package/bin/evals-lab.js

@@ -21,6 +21,7 @@ const http = require("http");
 const net = require("net");
 const os = require("os");
 const path = require("path");
+const updates = require("./update.js");
 
 const LAB = path.join(__dirname, "..", "lab");
 // The oldest Python the package's workflow starts the lab on.
@@ -50,7 +51,8 @@ SOURCE_ALLOW_LOCAL_FILES=1 and SOURCE_ALLOW_LOCAL_COMMANDS=1 let a Source read a
 file or run a command on this machine; both are off unless set.
 SOURCE_LOCAL_ROOT=<dir> lets a file Source read only under that directory.
 LAB_IMPORT=<file> imports a lab file each time the lab starts, each
-connection's key from $EVALSLAB_API_KEY_<SLUG>.`;
+connection's key from $EVALSLAB_API_KEY_<SLUG>.
+LAB_NO_UPDATE_CHECK=1 stops the daily look at npm for a newer version.`;
 
 // Where a lab keeps its store when nobody said: the platform's per-user data
 // directory, so it survives the package being upgraded or reinstalled.
@@ -210,6 +212,8 @@ async function main() {
   // else the PATH holds. Windows spells the variable `Path`.
   const pathKey = Object.keys(env).find(k => k.toUpperCase() === "PATH") || "PATH";
   env[pathKey] = path.dirname(process.execPath) + path.delimiter + (env[pathKey] || "");
+  const updateFile = updates.start(o.dataDir, require("../package.json").version);
+  if (updateFile) env.LAB_UPDATE_FILE = updateFile;
   // -X utf8: the server reads and writes its files as UTF-8 by default,
   // which Windows' Python otherwise does in the machine's code page.
   const child = spawn(python.cmd[0], [...python.cmd.slice(1), "-X", "utf8", "-u", path.join(LAB, "server.py")],

package/bin/update.js

@@ -0,0 +1,65 @@
+// Whether a newer evals-lab is out, for Home's footer (docs/distribution.md §
+// Which build a package is). The launcher starts this, so the image and
+// `evals-lab run` never do. It keeps its answer in a file in the data
+// directory, which server.py reads through LAB_UPDATE_FILE: the file is
+// rewritten at every start as well as after a check, so an upgrade clears what
+// the version before it was told.
+const fs = require("fs");
+const path = require("path");
+
+const LATEST = "https://registry.npmjs.org/evals-lab/latest";
+const VERSION = /^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/;
+const HOUR = 60 * 60 * 1000;
+const DAY = 24 * HOUR;
+
+// Whatever the registry answers ends up on the page, so it is held to VERSION.
+async function latestVersion() {
+  const r = await fetch(LATEST, { redirect: "error", signal: AbortSignal.timeout(5000) });
+  const { version } = await r.json();
+  return VERSION.test(version) ? version : null;
+}
+
+// A prerelease comes before its release; two prereleases of one release are
+// not told apart, since `latest` only names a prerelease by mistake.
+function isNewer(latest, current) {
+  const [a, b] = [latest, current].map(v => v.split("-")[0].split(".").map(Number));
+  for (let i = 0; i < 3; i++) if (a[i] !== b[i]) return a[i] > b[i];
+  return current.includes("-") && !latest.includes("-");
+}
+
+function read(file) {
+  try { return JSON.parse(fs.readFileSync(file, "utf8")) || {}; } catch { return {}; }
+}
+
+function write(file, checked, latest, current) {
+  const newer = latest && isNewer(latest, current) ? latest : null;
+  try { fs.writeFileSync(file, JSON.stringify({ checked, latest, newer })); } catch { /* it says nothing */ }
+}
+
+// A failed check counts as the day's: offline, it is not retried each hour.
+async function check(file, current, now = Date.now(), get = latestVersion) {
+  const was = read(file);
+  const checked = Number(was.checked) || 0;
+  const latest = VERSION.test(was.latest) ? was.latest : null;
+  write(file, checked, latest, current);
+  if (now - checked < DAY) return;
+  write(file, now, await get().catch(() => null) || latest, current);
+}
+
+function off(env) {
+  return /^(1|true|yes|on)$/i.test(env.LAB_NO_UPDATE_CHECK || "");
+}
+
+// Starts checking, and returns the file the answer lands in; null when turned
+// off. Not awaited: nothing about starting the lab waits on the registry. A
+// lab left running stays told, by looking each hour for a day having passed.
+function start(dataDir, current, env = process.env, get = latestVersion) {
+  if (off(env)) return null;
+  const file = path.join(dataDir, "update.json");
+  const tick = () => check(file, current, Date.now(), get);
+  tick();
+  setInterval(tick, HOUR).unref();
+  return file;
+}
+
+module.exports = { start, check, isNewer, off, LATEST };

package/lab/VERSION

@@ -1 +1 @@
-0.16.0 (2026.10.16-944)
+0.17.0 (2026.11.11-1029)

package/lab/demo/CREDITS.md

@@ -5,9 +5,10 @@ need no attribution; the sources are listed so each one can be traced.
 
 ## Images
 
-The images of `Demo 1/` and `Demo 2/`. Files were resized to about 3
-megapixels and their metadata stripped; the panorama is a crop, and the PNG
-and TIFF are conversions.
+The images of `Demo 1/` and `Demo 2/`. Files were resized to 1024 pixels on
+the long side (JPEG quality 75), which keeps the npm package small, and their
+metadata stripped; the panorama is a crop, and the PNG and TIFF (JPEG
+compressed) are conversions.
 
 | File | Source | Author | Licence |
 |---|---|---|---|

package/lab/demo/datasets/recorded-replies.json

@@ -0,0 +1,39 @@
+{
+  "format": "evals-lab/eval-group",
+  "version": 9,
+  "group": {
+    "name": "Demo: recorded replies evals",
+    "body": {
+      "version": 9,
+      "scoring": {
+        "mode": "all",
+        "threshold": null
+      },
+      "grader": null,
+      "every": [
+        {
+          "type": "contains",
+          "value": "Mercury",
+          "ignoreCase": true
+        },
+        {
+          "type": "contains",
+          "value": "Venus",
+          "not": true,
+          "ignoreCase": true
+        },
+        {
+          "type": "regex",
+          "pattern": "^\\W*Mercury\\W*$",
+          "flags": "i"
+        },
+        {
+          "type": "latency",
+          "max": 10000
+        }
+      ],
+      "run": [],
+      "cases": []
+    }
+  }
+}

package/lab/demo/manifest.json

@@ -3,10 +3,10 @@
   "version": 1,
   "id": "demo",
   "name": "Demo",
-  "packVersion": "2026.10.14.3",
+  "packVersion": "2026.11.10",
   "requires": {
     "lab": ">=19",
     "plugins": []
   },
-  "description": "Six demos, each a dataset with a starter pipeline: two sets of public-domain images, short texts to classify, one question in four wordings, a dynamic Source's snapshot, and PDFs to check and sort."
+  "description": "Seven demos, each a dataset with a starter pipeline: two sets of public-domain images, short texts to classify, one question in four wordings, the same question's replies recorded earlier (runs with no AI Connection), a dynamic Source's snapshot, and PDFs to check and sort."
 }

package/lab/demo/pipelines/recorded-replies.json

@@ -0,0 +1,89 @@
+{
+  "version": 20,
+  "id": "recorded-replies",
+  "name": "Demo: recorded replies pipeline",
+  "steps": [
+    {
+      "type": "getItems",
+      "id": "items-j1",
+      "content": {
+        "type": "prompt"
+      }
+    },
+    {
+      "type": "request",
+      "id": "j1",
+      "name": "",
+      "cell": {
+        "type": "echo",
+        "prompt": "Mercury"
+      }
+    },
+    {
+      "type": "readResponse",
+      "id": "read-j1",
+      "request": "j1",
+      "out": {
+        "kind": "text"
+      }
+    }
+  ],
+  "variants": [
+    {
+      "id": "s1",
+      "name": "",
+      "connection": null,
+      "profile": null,
+      "overrides": {}
+    },
+    {
+      "id": "s2",
+      "name": "",
+      "connection": null,
+      "profile": null,
+      "overrides": {
+        "j1": {
+          "prompt": "Mercury is closest to the Sun."
+        }
+      }
+    },
+    {
+      "id": "s3",
+      "name": "",
+      "connection": null,
+      "profile": null,
+      "overrides": {
+        "j1": {
+          "prompt": "Venus"
+        }
+      }
+    },
+    {
+      "id": "s4",
+      "name": "",
+      "connection": null,
+      "profile": null,
+      "overrides": {
+        "j1": {
+          "prompt": "The innermost planet is Mercury."
+        }
+      }
+    }
+  ],
+  "evals": [
+    {
+      "id": "t1",
+      "name": "",
+      "type": "group",
+      "continueOnFailure": true,
+      "group": {
+        "id": "recorded-replies",
+        "name": "Demo: recorded replies evals"
+      },
+      "pin": null
+    }
+  ],
+  "pass": {
+    "mode": "all"
+  }
+}

package/lab/kinds/list.mjs

@@ -766,16 +766,24 @@ export function filterV4(f        )         {
 // ---- the modifiers -------------------------------------------------------------
 
 /** What an item's numbering looks like: "[1] suv", "(2) car", "3. dirt",
-    "4) trees", "5: sky" -- and not "1950s car" or "5.1 surround". */
-const NUMBERING = /^(?:\[\d+\]|\(\d+\)|\d+[.):](?!\d))\s*/;
+    "4) trees", "5: sky" -- and not "1950s car" or "5.1 surround". A
+    bracketed label is numbering too when a space follows it ("[x] done",
+    "[items] suv"), but not a markdown link's "[text](url)". */
+const NUMBERING = /^(?:\[\d+\]|\[[^[\]]*\](?=\s)|\(\d+\)|\d+[.):](?!\d))\s*/;
 
 /** One item as Tidy items leaves it: a leading bullet, its numbering, stray
-    quotes and stops at the ends, and runs of whitespace gone. */
+    quotes, stops and brackets at the ends, and runs of whitespace gone. A
+    stray bracket is one whose pair is not in the item: a list written as an
+    array, "[a, b, c]", split on its commas leaves "[a" and "c]", while
+    "[text](url)" keeps its own. */
 export function tidy(item        )         {
   let t = item.trim();
   for (const p of ["-", "*", "•"]) if (t.startsWith(p)) t = t.slice(p.length);
   t = t.trim().replace(NUMBERING, "");
   t = trimEnds(t.trim(), ".\"'`");
+  if (t.startsWith("[") && !t.includes("]")) t = t.slice(1);
+  if (t.endsWith("]") && !t.includes("[")) t = t.slice(0, -1);
+  t = trimEnds(t.trim(), ".\"'`");
   return t.replace(/\s+/g, " ").trim();
 }