evals-lab 0.11.0 → 0.12.0

package/lab/server.py

@@ -42,6 +42,7 @@ import urllib.error
 import urllib.parse
 import urllib.request
 import zipfile
+from concurrent.futures import ThreadPoolExecutor
 from contextlib import closing
 from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
 from pathlib import Path
@@ -53,6 +54,9 @@ OLLAMA = os.environ.get("OLLAMA_URL", "http://127.0.0.1:11434")
 # as a lab run on a workstation always has been; the deployed one sets it,
 # because it keeps setups and API keys and can spend GPU time.
 PASSWORD = os.environ.get("LAB_PASSWORD") or None
+# A lab file imported on every start (docs/lab-export.md): a demo lab that
+# keeps no data starts set up, its keys from $EVALSLAB_API_KEY_<SLUG>.
+LAB_IMPORT = os.environ.get("LAB_IMPORT") or None
 # Where what follows the user between browsers is kept (#463). Unset means
 # nothing is, and the page stays with its browser as it always has.
 DATA_DIR = os.environ.get("DATA_DIR") or None
@@ -170,6 +174,21 @@ def auth_headers(ctype: str, key: str) -> dict:
     return {"Authorization": "Bearer " + key}
 
 
+def http_key_header(how, key: str) -> dict:
+    """The header an HTTP endpoint connection sends its key in: its Key header
+    setting, kept flat on the connection (evals-core.ts's KEY_HEADERS)."""
+    if not key:
+        return {}
+    how = str(how or "bearer")
+    if how == "x-api-key":
+        return {"x-api-key": key}
+    if how == "api-key":
+        return {"api-key": key}
+    if how == "none":
+        return {}
+    return {"Authorization": "Bearer " + key}
+
+
 def connection_url(base: str, ctype: str) -> str:
     path = CONNECTION_CHAT_PATHS.get(ctype, "/chat/completions")
     if ctype in ("ollama", "ollama-cloud"):
@@ -348,7 +367,8 @@ SOURCE_PROVIDERS = {
     # The API call provider (docs/dynamic-sources.md § 2): HTTP GET plus a verb,
     # headers and a body, keyed from a connection -- the same relay fetch, one
     # layer up. `produce` and `take` are attached below, beside http-get's.
-    "api": {"label": "API call", "noun": "call", "tier": "pull", "flag": None,
+    # `keyed`: it sends a connection's key, so a pack may not carry one (read_pack).
+    "api": {"label": "API call", "noun": "call", "tier": "pull", "flag": None, "keyed": True,
             "uploads": dict(_CALL_UPLOADS), "longConfig": {"body": 8000}},
     # Supplied has no `produce`: the caller hands items in (the CLI's --items, or
     # an Upload… by hand), so the server fetches and executes nothing -- it only
@@ -774,7 +794,11 @@ def connection_auth(pid):
     connection's type says (an HTTP endpoint's Key header, else the type's own,
     auth_headers) -- it never leaves the server or reaches a Call (#258).
     Returns (headers, key, None); ({}, "", None) for no connection or no key;
-    or (None, None, (code, why))."""
+    or (None, None, (code, why)). This is the third seam a connection resolves
+    to a key, beside worker_destinations and the relay's _unshared, so it
+    refuses a connection not shared with the bound workspace the same way --
+    the request thread's for a snapshot, the submitting request's for a live
+    run (docs/workspaces.md)."""
     if not pid:
         return {}, "", None
     doc = (STORE.all().get("promptlab.connections") or {}) if STORE is not None else {}
@@ -782,19 +806,16 @@ def connection_auth(pid):
     profile = next((p for p in profiles if isinstance(p, dict) and p.get("id") == pid), None)
     if profile is None:
         return None, None, (400, "that Source's auth connection is not one the lab has")
+    ws = workspace_of(STORE)
+    if not STORE.shared("profile", pid, ws):
+        return None, None, (403, not_shared_sentence([profile.get("name") or pid], STORE.workspace_name(ws)))
     key = str(profile.get("key") or "").strip()
     if not key:
         return {}, "", None
     if not header_safe(key):
         return None, None, (400, "that connection's key cannot be sent in a header")
     ctype = profile.get("type")
-    if ctype == "http":
-        options = profile.get("options") if isinstance(profile.get("options"), dict) else {}
-        how = str(options.get("auth") or "bearer")
-        head = ({"x-api-key": key} if how == "x-api-key" else {"api-key": key} if how == "api-key"
-                else {} if how == "none" else {"Authorization": "Bearer " + key})
-    else:
-        head = auth_headers(ctype, key)
+    head = http_key_header(profile.get("auth"), key) if ctype == "http" else auth_headers(ctype, key)
     return head, key, None
 
 
@@ -809,6 +830,10 @@ def produce_api(config):
     method = (cfg.get("method").upper() if isinstance(cfg.get("method"), str) and cfg.get("method").strip()
               else "GET")
     auth, key, why = connection_auth(cfg.get("connection"))
+    if why and why[0] == 403:
+        # Refused, not failed: no request went out, so the caller answers the
+        # refusal rather than keeping a failed Call.
+        return {"error": why[1], "refused": why, "request": {"url": url, "method": method}, "status": None}
     if why:
         return {"error": why[1], "request": {"url": url, "method": method}, "status": None}
     headers = {**_header_pairs(cfg.get("reqHeaders")), **auth}
@@ -2488,6 +2513,8 @@ class Sources:
         if produce is None:
             return None, (400, "that provider cannot be produced from on the server")
         result = produce(cfg)
+        if result.get("refused"):
+            return None, result["refused"]
         request = result.get("request")
         status = result.get("status")
         content = result.get("content")
@@ -5024,32 +5051,47 @@ class Pipelines:
         self.store._put(db, self.VERSIONS, target_ws, tv + 1, tbody, self._now())
 
 
-def make_thumbs(sid, name):
-    """JPEGs a browser can render, beside a stored file: a tile-size one
-    for every image type, and a full-size render for the formats a browser
-    cannot decode, whose zoom view has no original to show. Non-fatal by
-    design: a missing thumb is a blank tile, not a failed upload, and the
-    startup sweep clears an orphan on a later start. #33 #527"""
+def thumb_jobs(sid, name):
+    """The `magick` runs that make a stored file's thumbnails, each (argv,
+    out): a tile-size JPEG for every image type, and a full-size render for
+    the formats a browser cannot decode, whose zoom view has no original to
+    show. The paths are looked up here, in the request's own thread, since a
+    Source is found through the workspace bound to it. #33 #527"""
     if CONVERT is None or Path(name).suffix.lower() not in IMAGE_TYPES:
-        return
+        return []
+    jobs = []
     for full in (False, True):
         if full and Path(name).suffix.lower() not in THUMB_TYPES:
             continue
         path = SOURCES.file_path(sid, name)
         out = SOURCES.thumb_path(sid, name, full)
         if path is None or out is None:
-            return
+            return jobs
         size = FULL_SIZE if full else TILE_SIZE
-        out.parent.mkdir(parents=True, exist_ok=True)
-        try:
-            subprocess.run([CONVERT, str(path), "-auto-orient",
-                            "-thumbnail", f"{size}x{size}>",
-                            "-quality", str(FULL_QUALITY if full else TILE_QUALITY),
-                            str(out)],
-                           timeout=30, stdout=subprocess.DEVNULL,
-                           stderr=subprocess.DEVNULL, check=True)
-        except (subprocess.SubprocessError, OSError):
-            out.unlink(missing_ok=True)
+        jobs.append(([CONVERT, str(path), "-auto-orient",
+                      "-thumbnail", f"{size}x{size}>",
+                      "-quality", str(FULL_QUALITY if full else TILE_QUALITY),
+                      str(out)], out))
+    return jobs
+
+
+def run_thumb(job):
+    """One thumbnail. Non-fatal by design: a missing thumb is a blank tile,
+    not a failed upload, and the startup sweep clears an orphan on a later
+    start."""
+    argv, out = job
+    out.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        subprocess.run(argv, timeout=30, stdout=subprocess.DEVNULL,
+                       stderr=subprocess.DEVNULL, check=True)
+    except (subprocess.SubprocessError, OSError):
+        out.unlink(missing_ok=True)
+
+
+def make_thumbs(sid, name):
+    """A stored file's thumbnails, made now (thumb_jobs)."""
+    for job in thumb_jobs(sid, name):
+        run_thumb(job)
 
 
 # ---- Packs ------------------------------------------------------------------
@@ -5120,7 +5162,9 @@ def pack_requires_problem(manifest: dict, plugins: set) -> str:
 def read_pack(data: bytes):
     """A pack's zip read and checked whole: (pack, None) or (None, why). The
     pack is {manifest, datasets: [(key, name, body, raw)], pipelines:
-    [(key, doc)], presets: [...], sources: {name: [(file, bytes)]}}."""
+    [(key, doc)], presets: [...], sources: {name: [(file, bytes)]}, source_meta:
+    {name: {name, type, config}}} -- a Source's meta from its folder's
+    `source.json`, or the folder's name as a File Library."""
     try:
         zf = zipfile.ZipFile(io.BytesIO(data))
     except zipfile.BadZipFile:
@@ -5149,7 +5193,8 @@ def read_pack(data: bytes):
     for k in ("name", "packVersion"):
         if not isinstance(manifest.get(k), str) or not manifest[k].strip():
             return None, f"a pack's manifest needs a {k}"
-    pack = {"manifest": manifest, "datasets": [], "pipelines": [], "presets": [], "sources": {}}
+    pack = {"manifest": manifest, "datasets": [], "pipelines": [], "presets": [], "sources": {}, "source_meta": {}}
+    declared = {}
     for name in sorted(names):
         parts = name.split("/")
         if name == "manifest.json":
@@ -5192,21 +5237,86 @@ def read_pack(data: bytes):
                     or not (isinstance(doc.get("jobs"), list) or isinstance(doc.get("chains"), list))):
                 return None, f"{name} is not a pipeline document the lab reads"
             pack["pipelines"].append((parts[1][:-5], doc))
+        elif len(parts) == 3 and parts[0] == "sources" and parts[2] == PACK_SOURCE_META:
+            src = source_name(parts[1])
+            meta, why = load(name)
+            if why:
+                return None, why
+            if src is None or not isinstance(meta, dict):
+                return None, f"{name} is not a Source's {PACK_SOURCE_META}"
+            declared[src] = meta
         elif len(parts) == 3 and parts[0] == "sources":
             src, file = source_name(parts[1]), source_name(parts[2])
-            if src is None or file is None or Path(file).suffix.lower() not in FILE_TYPES:
+            if src is None or file is None:
                 return None, f"{name} is not a file a Source takes"
             if names[name].file_size > MAX_FILE:
                 return None, f"{name} is over the cap for a file"
             pack["sources"].setdefault(src, []).append((file, zf.read(name)))
         else:
             return None, f"{name} is not something a pack holds"
-    for src, files in pack["sources"].items():
+    for src in sorted(set(pack["sources"]) | set(declared)):
+        meta, why = pack_source_meta(src, declared.get(src))
+        if why:
+            return None, why
+        pack["source_meta"][src] = meta
+        files = pack["sources"].setdefault(src, [])
+        entry = source_entry(meta["type"], meta["config"]) or {}
+        takes, take = entry.get("uploads") or {}, entry.get("take")
+        for i, (file, blob) in enumerate(files):
+            if Path(file).suffix.lower() not in takes:
+                return None, f"sources/{src}/{file} is not a file a Source takes"
+            # A shipped Call is checked as any other is: a token refuses the
+            # pack, and a secret's shape is redacted before it is kept.
+            if take is not None:
+                blob, err = take(file, blob)
+                if err:
+                    return None, f"sources/{src}/{file}: {err[1]}"
+                files[i] = (file, blob)
         if sum(len(b) for _, b in files) > MAX_SOURCE:
             return None, f"the Source {src} is over the cap for a Source"
     return pack, None
 
 
+# A pack Source's own meta, beside its files: {name?, type?, config?}.
+PACK_SOURCE_META = "source.json"
+
+
+def pack_source_meta(folder, meta):
+    """A pack Source's {name, type, config} from its folder's source.json (or
+    None: a File Library named after the folder), or (None, why). A pack may
+    carry a File Library or a dynamic Source whose provider pulls with no key
+    and is enrolled here -- the line a plugin's provider is held to
+    (docs/packs.md); anything else refuses the pack whole."""
+    meta = meta or {}
+    at = f"sources/{folder}/{PACK_SOURCE_META}"
+    unknown = set(meta) - {"name", "type", "config"}
+    if unknown:
+        return None, f"{at} holds {sorted(unknown)[0]}, which a pack Source does not"
+    name, err = Sources._name(meta.get("name", folder))
+    if err:
+        return None, f"{at}: {err[1]}"
+    kind = meta.get("type", DEFAULT_SOURCE_TYPE)
+    config = meta.get("config")
+    if kind == DEFAULT_SOURCE_TYPE:
+        if config is not None:
+            return None, f"{at}: a File Library takes no config"
+        return {"name": name, "type": kind, "config": None}, None
+    if not (SOURCE_TYPES.get(kind) or {}).get("providers"):
+        return None, f"{at}: a pack carries a File Library or a dynamic Source, not {kind!r}"
+    provider = config.get("provider") if isinstance(config, dict) else None
+    entry = SOURCE_PROVIDERS.get(provider)
+    if entry is None:
+        return None, f"{at}: the lab has no provider {provider!r}"
+    if entry.get("tier") != "pull" or entry.get("keyed") or config.get("connection"):
+        return None, f"{at}: a pack's dynamic Source pulls with no key, and {entry.get('label', provider)} does not"
+    if not provider_enrolled(entry):
+        return None, f"{at}: this lab does not enrol {entry.get('label', provider)}"
+    _, err = Sources._checked(kind, config)
+    if err:
+        return None, f"{at}: {err[1]}"
+    return {"name": name, "type": kind, "config": config}, None
+
+
 class Packs:
     """What each installed pack made, so it can be updated and removed."""
 
@@ -5341,22 +5451,41 @@ class Packs:
             made = {}
             # Sources first, then datasets: a pipeline names both.
             src_ids = {}
+            thumbs = []
             for name, files in pack["sources"].items():
+                meta = pack["source_meta"][name]
                 sid = owned.get(("source", name))
-                if sid and SOURCES.get(sid):
-                    old = [f["name"] for f in SOURCES.get(sid)["files"]]
+                have = SOURCES.get(sid) if sid else None
+                # An update keeps the Source's name, so one renamed here, or
+                # by a later pack, is not renamed back. Another kind is
+                # another Source.
+                if have and have.get("type") == meta["type"]:
+                    old = [f["name"] for f in have["files"]]
                     if old:
                         SOURCES.remove_files(sid, old)
+                    if meta["config"] is not None:
+                        SOURCES.update(sid, {"config": meta["config"]})
                 else:
                     taken = {s["name"] for s in SOURCES.list()}
-                    src, _ = SOURCES.create(unique_dataset_name(name, taken))
+                    src, _ = SOURCES.create(unique_dataset_name(meta["name"], taken), meta["type"], meta["config"])
                     sid = src["id"]
                 for file, blob in files:
                     stored, err = SOURCES.add_file(sid, file, blob)
-                    if not err:
-                        make_thumbs(sid, stored)
+                    if err:
+                        continue
+                    thumbs += thumb_jobs(sid, stored)
+                    # A dynamic Source's files are Calls: the pack's are its
+                    # snapshot, fetched when the pack was made.
+                    if meta["type"] != DEFAULT_SOURCE_TYPE:
+                        cfg = meta["config"]
+                        request = {"url": cfg["url"], "method": cfg.get("method") or "GET"} if cfg.get("url") else None
+                        SOURCES._record_call(sid, stored, "snapshot", cfg.get("provider"), request, None, None, stored)
                 src_ids[name] = sid
                 made[("source", name)] = sid
+            # The demo pack installs as the server starts, before it answers,
+            # and one `magick` at a time was most of that wait.
+            with ThreadPoolExecutor(max_workers=min(4, os.cpu_count() or 1)) as pool:
+                list(pool.map(run_thumb, thumbs))
             ds_ids = {}
             cuts = []   # (kind, id, the document as it was), kept before it changes
             for key, name, body, raw, preview in pack["datasets"]:
@@ -5413,7 +5542,8 @@ class Packs:
                     entry = next((w for w in body["list"] if w.get("id") == wid), None) if wid else None
                     if entry:
                         cuts.append(("pipeline", wid, entry.get("work")))
-                        entry["work"] = doc
+                        # Its name is the lab's now, as a dataset's is.
+                        entry["work"] = {**doc, "name": entry.get("name") or doc.get("name")}
                     else:
                         wid = "w" + secrets.token_hex(6)
                         name = unique_dataset_name(doc.get("name") or key, {w.get("name") for w in body["list"]})
@@ -7280,11 +7410,19 @@ def worker_destinations(run: dict):
     if STORE is not None:
         doc = STORE.all().get("promptlab.connections") or {}
         stored = (doc.get("body") or {}).get("list") or []
-    # One variable a connection, however many targets or profiles share it.
-    env = {}
+    # One variable a connection, however many targets or profiles share it --
+    # and never two connections one variable, whichever path the run came by:
+    # a stored snapshot is not checked again as a submit is (run_problems),
+    # and the runner reads each connection's key from its slug's variable, so
+    # the second would send the first one's key to its own address.
+    env, spelt = {}, {}
     for key, conn in table.items():
         pid = answer_connection(key, conn)
         name = conn.get("name") or pid
+        var = key_var(pid, conn.get("slug") if isinstance(conn.get("slug"), str) else None)
+        if var in spelt and spelt[var][0] != pid:
+            return None, f"Connections {spelt[var][1]} and {name} both take their key from ${var}"
+        spelt[var] = (pid, name)
         if isinstance(conn, dict) and conn.get("type") in LOCAL_CONNECTIONS:
             continue
         base = api_base(str(conn.get("url") or "")) or api_base(OLLAMA)
@@ -7301,7 +7439,7 @@ def worker_destinations(run: dict):
             why = allowed(base, key)
             if why:
                 return None, f"Connection {name}: {why}"
-            env[key_var(pid, conn.get("slug"))] = key
+            env[var] = key
     return env, None
 
 
@@ -7333,10 +7471,12 @@ def worker_destinations(run: dict):
 # 16: a target names an AI Connection and, optionally, a Connection Profile,
 # where it named one Target profile; the run's table is keyed by the pair
 # (answer_key), each entry naming the connection whose key it takes (#445).
-PIPELINE_VERSION = 16
+# 17: an HTTP Request target has no `api` of its own; its job's flow step's
+# is the one its body is read by (#477).
+PIPELINE_VERSION = 17
 # What a stored run may be: the current version, and the ones evals-core.ts's
 # upgradePipeline reads. A new submission is upgraded to the current one.
-READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16)
+READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17)
 TARGET_CAP = 4
 # Target steps whose words the Prompt library does not record as a use: they
 # ask no model (evals-core.ts's STEP_TYPES.echo).
@@ -7369,7 +7509,7 @@ CONNECTION_TYPES = {
     "echo": (),
     # An HTTP endpoint (evals-core.ts): an address and how its key travels; an
     # HTTP Request step builds the whole request.
-    "http": ("auth", "headers", "testPath"),
+    "http": ("auth", "headers", "testPath", "requestFormat"),
     "ollama": ("seed", "nPredict", "temperature", "repeatPenalty"),
     "ollama-cloud": ("seed", "nPredict", "temperature", "repeatPenalty"),
     "openai-compatible": ("temperature", "nPredict"),
@@ -7424,7 +7564,7 @@ def run_answer(table, key):
 # other setting set is a Connection Profile's. Mirrored, and held to the
 # core's splitProfile, by proxy-check.py.
 AI_CONNECTION_FIELDS = ("id", "name", "slug", "type", "url", "key", "model", "profile")
-CONNECTION_SCOPED = {"http": ("auth", "headers", "testPath")}
+CONNECTION_SCOPED = {"http": ("auth", "headers", "testPath", "requestFormat")}
 # evals-core.ts REPLY_TOKENS_BEFORE, and the setting of each type that sent it.
 REPLY_TOKENS_BEFORE = "160"
 REPLY_TOKENS_SETTING = {"openai-compatible": "nPredict", "llama.cpp": "nPredict", "anthropic": "maxTokens"}
@@ -7526,14 +7666,7 @@ def http_endpoint_test(payload, key):
             name, value = part.split(":", 1)
             if name.strip():
                 headers[name.strip()] = value.strip()
-    if key:
-        how = str(settings.get("auth") or "bearer")
-        if how == "x-api-key":
-            headers["x-api-key"] = key
-        elif how == "api-key":
-            headers["api-key"] = key
-        elif how != "none":
-            headers["Authorization"] = "Bearer " + key
+    headers.update(http_key_header(settings.get("auth"), key))
     if not all(header_safe(v) for v in headers.values()):
         return "", {}, "A header of this profile has characters that cannot be sent."
     return origin + (path if path.startswith("/") else "/" + path), headers, ""
@@ -8119,6 +8252,16 @@ class Handler(BaseHTTPRequestHandler):
                 return self._send(404, b"not found", "text/plain")
             ctype = TYPES[".json"] if f.suffix in (".json", ".map") else TYPES[".js"]
             return self._send(200, f.read_bytes(), ctype, (("Cache-Control", "no-store"),))
+        if path == "/api/lab-export":
+            # The lab's settings as a file (docs/lab-export.md): every
+            # workspace's entry, or ?workspace=<slug>'s alone. No key.
+            if STORE is None or WORKSPACES is None:
+                return self._send(404, b"not found", "text/plain")
+            only = (urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query).get("workspace") or [None])[0]
+            out = lab_export(only)
+            if out is None:
+                return self._json(404, {"error": f"no workspace at /w/{only}"})
+            return self._json(200, out)
         if path == "/api/packs":
             if PACKS is None:
                 return self._send(404, b"not found", "text/plain")
@@ -8471,6 +8614,22 @@ class Handler(BaseHTTPRequestHandler):
             return self._filter_sets_post(path)
         if path == "/api/copy-move" or path.startswith("/api/copy-move/"):
             return self._copy_move_post(path)
+        if path == "/api/lab-import":
+            # A lab file into this lab (docs/lab-export.md), adding what it
+            # lacks. `keys` is what `evals-lab import` read from its own
+            # $EVALSLAB_API_KEY_<SLUG> variables; the page sends none.
+            if STORE is None or WORKSPACES is None:
+                return self._send(404, b"not found", "text/plain")
+            payload = self._payload()
+            if payload is None:
+                return self._json(400, {"error": "an import is {lab, keys?} as JSON"})
+            keys = payload.get("keys") if isinstance(payload.get("keys"), dict) else {}
+            keys = {k: v for k, v in keys.items()
+                    if isinstance(k, str) and k.startswith("EVALSLAB_API_KEY_") and isinstance(v, str)}
+            said, err = lab_import(payload.get("lab"), keys)
+            if err:
+                return self._json(err[0], {"error": err[1]})
+            return self._json(200, {"said": said})
         if path == "/api/packs" or path.startswith("/api/packs/"):
             return self._packs_post(path)
         if path == "/api/workspaces" or path.startswith("/api/workspaces/"):
@@ -8629,8 +8788,8 @@ class Handler(BaseHTTPRequestHandler):
         else None. Only a held key (KEY_HELD<id>) names a stored profile; a key
         typed into the Setup form is a connection not yet saved, so nothing
         gates it. This is the relay/Test seam, beside worker_destinations' run
-        seam -- the two places a connection resolves to a key, so the two
-        places sharing is enforced. The caller writes the 403: a helper that
+        seam and connection_auth's API Source seam -- the places a connection
+        resolves to a key, so the places sharing is enforced. The caller writes the 403: a helper that
         answers here would send the body and still fall through to the proxy."""
         if STORE is None:
             return None
@@ -8897,6 +9056,8 @@ class Handler(BaseHTTPRequestHandler):
         if produce is None:
             return None, None, (400, "that provider cannot be produced from on the server")
         result = produce(cfg)
+        if result.get("refused"):
+            return None, None, result["refused"]
         content = result.get("content")
         if content is None:
             return None, None, (502, "the live fetch produced nothing: " + (result.get("error") or "unknown"))
@@ -9514,6 +9675,252 @@ class Handler(BaseHTTPRequestHandler):
             return self._json(502, {"error": f"{url}: {e.reason}"})
 
 
+# ---- Lab export and import (docs/lab-export.md) -----------------------------
+#
+# A lab's settings as one file: its AI Connections, Connection Profiles and
+# grader, and its workspaces, so a new lab can start as another one is set up
+# -- by Setup › Workspaces › Data, by `evals-lab export`/`import`, or on every
+# start with LAB_IMPORT (a demo lab that keeps no data). An import only adds:
+# what the lab already has, by a connection's or profile's id and a
+# workspace's address, is left as it is, so importing twice changes nothing
+# and a lab that keeps its data keeps its edits. The file never holds a key
+# (#258): each comes from $EVALSLAB_API_KEY_<SLUG>, as a CI run's does, and a
+# key that comes is the connection's from then on, a rotated one included.
+# A connection carries its share set (`shares`, workspace slugs and the
+# sentinels; absent is All) from version 2: version 1 carried none, so an
+# import reads its connections as All.
+
+LAB_FORMAT = "evals-lab/lab"
+LAB_VERSIONS = (1, 2)
+LAB_FIELDS = ("format", "version", "exportedAt", "connections", "connectionProfiles", "grader", "workspaces")
+
+
+def lab_export(only=None) -> dict:
+    """The lab's settings as a file, every workspace's entry or the one at
+    /w/[only]; None when no workspace is at that address. One workspace's file
+    carries only what that workspace can use: the connections shared with it,
+    the profiles over them (or over none) and the grader if it is one of them,
+    each share set narrowed to that workspace, the one the file names, and
+    *new kept where the connection has it. A connection shared only with
+    workspaces in the trash is usable nowhere, and an empty share set would
+    import as All, so it is left out, and out as the grader too."""
+    body = (STORE.all().get("promptlab.connections") or {}).get("body") or {}
+    profiles = (STORE.all().get("promptlab.connectionProfiles") or {}).get("body") or {}
+    every = WORKSPACES.list()
+    spaces = [w for w in every if not only or w["slug"] == only]
+    if only and not spaces:
+        return None
+    slug_of = {w["id"]: w["slug"] for w in (spaces if only else every)}
+    rows = STORE.shares()
+    conns, nowhere = [], set()
+    for c in body.get("list") or []:
+        if not isinstance(c, dict) or (only and not STORE.shared("profile", str(c.get("id") or ""), spaces[0]["id"])):
+            continue
+        entry = {k: v for k, v in c.items() if k not in ("key", "shares")}
+        held = rows.get(f"profile:{c.get('id')}")
+        if held and SHARE_ALL not in held:
+            entry["shares"] = [w if w == SHARE_NEW else slug_of[w] for w in held if w == SHARE_NEW or w in slug_of]
+            if not entry["shares"]:
+                nowhere.add(c.get("id"))
+                continue
+        conns.append(entry)
+    kept = {c["id"] for c in conns}
+
+    def over_kept(p):
+        ref = p.get("connection")
+        return not ref or not only or (isinstance(ref, dict) and ref.get("id") in kept)
+    out = {"format": LAB_FORMAT, "version": LAB_VERSIONS[-1],
+           "exportedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+           "connections": conns,
+           "connectionProfiles": [p for p in profiles.get("list") or [] if isinstance(p, dict) and over_kept(p)],
+           "workspaces": [{"slug": w["slug"], "name": w["name"], **({"archived": True} if w["archived"] else {})}
+                          for w in spaces]}
+    if (isinstance(body.get("grader"), str) and body["grader"] and body["grader"] not in nowhere
+            and (not only or body["grader"] in kept)):
+        out["grader"] = body["grader"]
+    return out
+
+
+def _keyless(entry, what):
+    """A sentence when [entry] holds a key, which a lab file never does."""
+    for k, v in entry.items():
+        if LOOKS_LIKE_A_KEY.match(k) and v not in ("", None):
+            return f"{what} holds a {k}; a lab file holds no key -- set ${key_var(entry['id'], entry.get('slug'))}"
+    return None
+
+
+def lab_problems(doc) -> str:
+    """Why [doc] is not a lab file this lab imports, or None. Checked whole
+    before anything is written, as a pack is."""
+    if not isinstance(doc, dict) or doc.get("format") != LAB_FORMAT:
+        return f"not a lab file: its format is not {LAB_FORMAT}"
+    if doc.get("version") not in LAB_VERSIONS:
+        return f"a lab file of version {doc.get('version')!r}; this lab reads {', '.join(map(str, LAB_VERSIONS))}"
+    extra = sorted(set(doc) - set(LAB_FIELDS))
+    if extra:
+        return f"a lab file holds no {', '.join(extra)}"
+    seen = set()
+    for i, c in enumerate(doc.get("connections") or []):
+        what = f"connection {i + 1}"
+        if not isinstance(c, dict) or not isinstance(c.get("id"), str) or not PROFILE_ID.fullmatch(c["id"]):
+            return f"{what} has no id of letters and digits"
+        what = f"connection {c['id']}"
+        if c["id"] in seen:
+            return f"{what} is listed twice"
+        seen.add(c["id"])
+        if not isinstance(c.get("name"), str) or not c["name"].strip():
+            return f"{what} has no name"
+        if c.get("type") not in CONNECTION_TYPES:
+            return f"{what} is of type {c.get('type')!r}, which this lab has no connection type for"
+        bad = next((k for k, v in c.items() if not isinstance(v, str) and k != "shares"), None)
+        if "shares" in c and doc["version"] < 2:
+            bad = "shares"
+        if bad:
+            return f"{what}'s {bad} is not text"
+        why = _keyless(c, what)
+        if why:
+            return why
+    seen = set()
+    for i, p in enumerate(doc.get("connectionProfiles") or []):
+        what = f"connection profile {i + 1}"
+        if not isinstance(p, dict) or not isinstance(p.get("id"), str) or not PROFILE_ID.fullmatch(p["id"]):
+            return f"{what} has no id of letters and digits"
+        what = f"connection profile {p['id']}"
+        if p["id"] in seen:
+            return f"{what} is listed twice"
+        seen.add(p["id"])
+        if not isinstance(p.get("name"), str) or not p["name"].strip():
+            return f"{what} has no name"
+        why = _keyless(p, what)
+        if why:
+            return why
+    if "grader" in doc and not isinstance(doc["grader"], str):
+        return "the grader is not a connection's id"
+    seen = set()
+    for i, w in enumerate(doc.get("workspaces") or []):
+        what = f"workspace {i + 1}"
+        if not isinstance(w, dict) or not isinstance(w.get("slug"), str) or not w["slug"]:
+            return f"{what} has no address"
+        if w["slug"] in seen:
+            return f"workspace {w['slug']} is listed twice"
+        seen.add(w["slug"])
+        if not isinstance(w.get("name"), str) or not w["name"].strip():
+            return f"workspace {w['slug']} has no name"
+        if Workspaces._slugify(w["slug"]) != w["slug"]:
+            return f"workspace {w['slug']}'s address is not one a lab makes: it would be {Workspaces._slugify(w['slug'])}"
+        fields = {"slug", "name"} | ({"archived"} if doc["version"] >= 2 else set())
+        if set(w) - fields:
+            return f"workspace {w['slug']} holds no {', '.join(sorted(set(w) - fields))}"
+        if "archived" in w and not isinstance(w["archived"], bool):
+            return f"workspace {w['slug']}'s archived is not true or false"
+    for k in ("connections", "connectionProfiles", "workspaces"):
+        if k in doc and not isinstance(doc[k], list):
+            return f"a lab file's {k} is a list"
+    # Workspaces before shares: a share names the file's workspaces or the lab's.
+    known = seen | ({w["slug"] for w in WORKSPACES.list()} if WORKSPACES is not None else set())
+    for c in doc.get("connections") or []:
+        if "shares" not in c:
+            continue
+        if not isinstance(c["shares"], list) or not all(isinstance(s, str) for s in c["shares"]):
+            return f"connection {c['id']}'s shares is a list of workspace addresses"
+        # No rows is All in the store, so an empty list would grant everywhere.
+        if not c["shares"]:
+            return (f"connection {c['id']}'s shares is empty; leave shares out for All, "
+                    "or name a workspace or *new")
+        lost = [s for s in c["shares"] if s not in (SHARE_ALL, SHARE_NEW) and s not in known]
+        if lost:
+            return f"connection {c['id']} is shared with {', '.join(lost)}, which neither the file nor this lab has"
+    return None
+
+
+def lab_import(doc, keys) -> tuple:
+    """[doc] into this lab, adding what it lacks; [keys] is where each
+    connection's key is looked up by its variable's name ($EVALSLAB_API_KEY_
+    <SLUG>) -- the server's environment as it starts, or what `evals-lab
+    import` sent. Returns (lines saying what happened, None) or (None, error).
+    No line holds a key."""
+    why = lab_problems(doc)
+    if why:
+        return None, (400, why)
+    said = []
+    # A file's address to this lab's workspace id; one added where the trash
+    # holds its address lands at the next free one, and its shares follow it.
+    wid_of = {w["slug"]: w["id"] for w in WORKSPACES.list()}
+    for w in doc.get("workspaces") or []:
+        if w["slug"] in wid_of:
+            continue
+        wid, err = WORKSPACES.create(w["name"], w["slug"])
+        wid_of[w["slug"]] = wid
+        if not err and w.get("archived"):
+            _, err = WORKSPACES.archive(wid)
+        if err:
+            return None, err
+        said.append(f"workspace {w['name']} added at /w/{WORKSPACES.get(wid)['slug']}"
+                    + (", archived" if w.get("archived") else ""))
+    for _ in range(3):
+        docs = STORE.all()
+        cur = docs.get("promptlab.connections") or {"version": 0, "body": None}
+        pcur = docs.get("promptlab.connectionProfiles") or {"version": 0, "body": None}
+        body = json.loads(json.dumps(cur["body"] or {"list": [], "at": ""}))
+        pbody = json.loads(json.dumps(pcur["body"] or {"list": []}))
+        conns = body.setdefault("list", [])
+        lines, changed, pchanged, added = [], False, False, []
+        for c in doc.get("connections") or []:
+            mine = next((x for x in conns if isinstance(x, dict) and x.get("id") == c["id"]), None)
+            if mine is None:
+                mine = {**{k: v for k, v in c.items() if k != "shares"}, "key": ""}
+                added.append(c)
+                conns.append(mine)
+                changed = True
+                lines.append(f"connection {c['name']} added")
+            key = str(keys.get(key_var(c["id"], c.get("slug"))) or "").strip()
+            if key and mine.get("key") != key:
+                if not header_safe(key):
+                    return None, (400, f"${key_var(c['id'], c.get('slug'))} cannot go in a header")
+                mine["key"] = key
+                changed = True
+                lines.append(f"connection {mine.get('name') or c['name']} takes its key from "
+                             f"${key_var(c['id'], c.get('slug'))}")
+        profs = pbody.setdefault("list", [])
+        for p in doc.get("connectionProfiles") or []:
+            if not any(isinstance(x, dict) and x.get("id") == p["id"] for x in profs):
+                profs.append(dict(p))
+                pchanged = True
+                lines.append(f"connection profile {p['name']} added")
+        if doc.get("grader") and not body.get("grader"):
+            body["grader"] = doc["grader"]
+            changed = True
+            lines.append(f"grader set to {doc['grader']}")
+        # Only an added connection takes the file's share set: one the lab had
+        # keeps its own, as everything else it had does. Set before the write,
+        # so the connection is never there and shared with All in between; a
+        # failed write puts back what the rows were, since its ids never landed.
+        before = STORE.shares()
+        for c in added:
+            if "shares" in c:
+                STORE.set_shares("profile", c["id"], [s if s in (SHARE_ALL, SHARE_NEW) else wid_of[s]
+                                                      for s in c["shares"]])
+        writes = {}
+        if changed:
+            body["at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+            writes["promptlab.connections"] = {"version": cur["version"], "body": body}
+        if pchanged:
+            writes["promptlab.connectionProfiles"] = {"version": pcur["version"], "body": pbody}
+        if not writes:
+            break
+        # Global documents, so the default workspace's write reaches them.
+        ok, _ = STORE.write(writes, STORE.default_ws)
+        if ok:
+            break
+        for c in added:
+            if "shares" in c:
+                STORE.set_shares("profile", c["id"], before.get(f"profile:{c['id']}") or [])
+    else:
+        return None, (409, "the lab's connections kept changing while they were imported; import again")
+    said += lines
+    return said or ["nothing to add"], None
+
+
 def main():
     if not (WEB_DIST / "index.html").is_file():
         raise SystemExit(f"no {WEB_DIST / 'index.html'} -- the page is built from web/: "
@@ -9525,6 +9932,11 @@ def main():
     # package, not one without the app: said at once, not at Sign in.
     if PUBLIC_GOOGLE_PROBLEM:
         raise SystemExit(PUBLIC_GOOGLE_PROBLEM)
+    # A lab file named and not there is a broken deployment, said at once.
+    if LAB_IMPORT and not Path(LAB_IMPORT).is_file():
+        raise SystemExit(f"LAB_IMPORT names {LAB_IMPORT}, which is not a file")
+    if LAB_IMPORT and STORE is None:
+        raise SystemExit("LAB_IMPORT needs DATA_DIR: a lab file's settings are kept in its store")
     # A root that is not there would refuse every File Source -- said at once.
     if LOCAL_ROOT and not Path(LOCAL_ROOT).is_dir():
         raise SystemExit(f"SOURCE_LOCAL_ROOT names {LOCAL_ROOT}, which is not a directory")
@@ -9593,6 +10005,24 @@ def main():
         # The drop-in folder: a pack put there is installed as the lab starts.
         for line in PACKS.install_dropins(Path(DATA_DIR) / "packs"):
             print(f"pack:      {line}", flush=True)
+    if LAB_IMPORT:
+        # After the demo: a lab file adds settings, never datasets, so the
+        # demo still finds the new lab it installs into.
+        try:
+            doc = json.loads(Path(LAB_IMPORT).read_text("utf-8"))
+        except (OSError, ValueError) as e:
+            raise SystemExit(f"LAB_IMPORT: {LAB_IMPORT} is not JSON ({e})") from None
+        said, err = lab_import(doc, os.environ)
+        if err:
+            raise SystemExit(f"LAB_IMPORT: {err[1]}")
+        for line in said:
+            print(f"import:    {line}", flush=True)
+        # A held key goes wherever a submitted run points it, so a lab open to
+        # its network with keys in it is as loud as one with a Local provider on.
+        if local_exposed() and any(os.environ.get(key_var(c["id"], c.get("slug")))
+                                   for c in doc.get("connections") or []):
+            print(f"WARNING:   LAB_IMPORT gave a connection its key, and {HOST} is beyond loopback with no "
+                  "LAB_PASSWORD -- anyone who can reach this port can run against it", flush=True)
     if QUEUE is not None:
         QUEUE.start()
     ThreadingHTTPServer((HOST, PORT), Handler).serve_forever()