evals-lab 0.10.0 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/CHANGELOG.md +88 -1
  2. package/README.md +12 -6
  3. package/bin/evals-lab.js +9 -1
  4. package/bin/lab-file.js +97 -0
  5. package/bin/run.js +16 -16
  6. package/lab/VERSION +1 -1
  7. package/lab/demo/CREDITS.md +37 -3
  8. package/lab/demo/datasets/demo-1.json +2 -2
  9. package/lab/demo/datasets/demo-2.json +2 -2
  10. package/lab/demo/datasets/sentiment.json +383 -0
  11. package/lab/demo/datasets/weather-alerts.json +61 -0
  12. package/lab/demo/datasets/wordings.json +39 -0
  13. package/lab/demo/manifest.json +3 -3
  14. package/lab/demo/pipelines/demo-1.json +8 -6
  15. package/lab/demo/pipelines/demo-2.json +8 -6
  16. package/lab/demo/pipelines/sentiment.json +65 -0
  17. package/lab/demo/pipelines/weather-alerts.json +66 -0
  18. package/lab/demo/pipelines/wordings.json +104 -0
  19. package/lab/demo/sources/Demo 1/source.json +3 -0
  20. package/lab/demo/sources/Demo 2/source.json +3 -0
  21. package/lab/demo/sources/sentiment/bakery-croissant.txt +1 -0
  22. package/lab/demo/sources/sentiment/bus-timetable.txt +1 -0
  23. package/lab/demo/sources/sentiment/cafe-coffee.txt +1 -0
  24. package/lab/demo/sources/sentiment/concert-night.txt +1 -0
  25. package/lab/demo/sources/sentiment/film-sequel.txt +1 -0
  26. package/lab/demo/sources/sentiment/garden-tomatoes.txt +1 -0
  27. package/lab/demo/sources/sentiment/gym-contract.txt +1 -0
  28. package/lab/demo/sources/sentiment/headphones-review.txt +1 -0
  29. package/lab/demo/sources/sentiment/hotel-room.txt +1 -0
  30. package/lab/demo/sources/sentiment/library-hours.txt +1 -0
  31. package/lab/demo/sources/sentiment/meeting-minutes.txt +1 -0
  32. package/lab/demo/sources/sentiment/parcel-lost.txt +1 -0
  33. package/lab/demo/sources/sentiment/phone-update.txt +1 -0
  34. package/lab/demo/sources/sentiment/puppy-home.txt +1 -0
  35. package/lab/demo/sources/sentiment/recipe-note.txt +1 -0
  36. package/lab/demo/sources/sentiment/road-closure.txt +1 -0
  37. package/lab/demo/sources/sentiment/source.json +3 -0
  38. package/lab/demo/sources/sentiment/teacher-thanks.txt +1 -0
  39. package/lab/demo/sources/sentiment/train-delay.txt +1 -0
  40. package/lab/demo/sources/sentiment/volunteer-day.txt +1 -0
  41. package/lab/demo/sources/sentiment/weather-report.txt +1 -0
  42. package/lab/demo/sources/weather-alerts/https-api.weather.gov-alerts-types.txt +116 -0
  43. package/lab/demo/sources/weather-alerts/source.json +8 -0
  44. package/lab/evals-core.mjs +584 -167
  45. package/lab/flows/record.mjs +16 -4
  46. package/lab/kinds/list.mjs +32 -23
  47. package/lab/run-evals.js +3 -3
  48. package/lab/server.py +913 -119
  49. package/lab/web/dist/assets/gallery-CFntjQFG.js +5 -0
  50. package/lab/web/dist/assets/main-BlTG2i20.css +1 -0
  51. package/lab/web/dist/assets/main-D6MNWqak.js +17 -0
  52. package/lab/web/dist/assets/tokens-DwzzdSIF.css +1 -0
  53. package/lab/web/dist/assets/tokens-bvuNFuYg.js +60 -0
  54. package/lab/web/dist/gallery.html +3 -3
  55. package/lab/web/dist/index.html +4 -4
  56. package/package.json +10 -2
  57. package/lab/web/dist/assets/gallery-BcvmOPqb.js +0 -3
  58. package/lab/web/dist/assets/main-BBUxHZ-D.js +0 -16
  59. package/lab/web/dist/assets/main-CMoyjImZ.css +0 -1
  60. package/lab/web/dist/assets/tokens-B7N4NvmM.css +0 -1
  61. package/lab/web/dist/assets/tokens-CiJkwD3V.js +0 -60
package/lab/server.py CHANGED
@@ -42,6 +42,7 @@ import urllib.error
42
42
  import urllib.parse
43
43
  import urllib.request
44
44
  import zipfile
45
+ from concurrent.futures import ThreadPoolExecutor
45
46
  from contextlib import closing
46
47
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
47
48
  from pathlib import Path
@@ -53,6 +54,9 @@ OLLAMA = os.environ.get("OLLAMA_URL", "http://127.0.0.1:11434")
53
54
  # as a lab run on a workstation always has been; the deployed one sets it,
54
55
  # because it keeps setups and API keys and can spend GPU time.
55
56
  PASSWORD = os.environ.get("LAB_PASSWORD") or None
57
+ # A lab file imported on every start (docs/lab-export.md): a demo lab that
58
+ # keeps no data starts set up, its keys from $EVALSLAB_API_KEY_<SLUG>.
59
+ LAB_IMPORT = os.environ.get("LAB_IMPORT") or None
56
60
  # Where what follows the user between browsers is kept (#463). Unset means
57
61
  # nothing is, and the page stays with its browser as it always has.
58
62
  DATA_DIR = os.environ.get("DATA_DIR") or None
@@ -170,6 +174,21 @@ def auth_headers(ctype: str, key: str) -> dict:
170
174
  return {"Authorization": "Bearer " + key}
171
175
 
172
176
 
177
+ def http_key_header(how, key: str) -> dict:
178
+ """The header an HTTP endpoint connection sends its key in: its Key header
179
+ setting, kept flat on the connection (evals-core.ts's KEY_HEADERS)."""
180
+ if not key:
181
+ return {}
182
+ how = str(how or "bearer")
183
+ if how == "x-api-key":
184
+ return {"x-api-key": key}
185
+ if how == "api-key":
186
+ return {"api-key": key}
187
+ if how == "none":
188
+ return {}
189
+ return {"Authorization": "Bearer " + key}
190
+
191
+
173
192
  def connection_url(base: str, ctype: str) -> str:
174
193
  path = CONNECTION_CHAT_PATHS.get(ctype, "/chat/completions")
175
194
  if ctype in ("ollama", "ollama-cloud"):
@@ -348,7 +367,8 @@ SOURCE_PROVIDERS = {
348
367
  # The API call provider (docs/dynamic-sources.md § 2): HTTP GET plus a verb,
349
368
  # headers and a body, keyed from a connection -- the same relay fetch, one
350
369
  # layer up. `produce` and `take` are attached below, beside http-get's.
351
- "api": {"label": "API call", "noun": "call", "tier": "pull", "flag": None,
370
+ # `keyed`: it sends a connection's key, so a pack may not carry one (read_pack).
371
+ "api": {"label": "API call", "noun": "call", "tier": "pull", "flag": None, "keyed": True,
352
372
  "uploads": dict(_CALL_UPLOADS), "longConfig": {"body": 8000}},
353
373
  # Supplied has no `produce`: the caller hands items in (the CLI's --items, or
354
374
  # an Upload… by hand), so the server fetches and executes nothing -- it only
@@ -565,7 +585,20 @@ def redact_flow(value):
565
585
  # again, so a page that got it wrong cannot put a key or a malformed record on
566
586
  # disk.
567
587
  RECORD_FORMAT = "evals-lab/record"
568
- RECORD_VERSION = 1
588
+ RECORD_VERSION = 2
589
+ # The origins a record names. Version 2 added `edited`: a call changed in the
590
+ # Call dialog, keeping the `run` and `iteration` it came from.
591
+ RECORD_ORIGINS = ("run", "hand", "upload", "edited")
592
+
593
+
594
+ def upgrade_record(value):
595
+ """A version-1 record as today's: flows/record.ts's upgradeRecord. Version
596
+ 2 only widened the origins, so the upgrade is the number. Anything else is
597
+ returned as it is, for record_problems to judge."""
598
+ if isinstance(value, dict) and value.get("format") == RECORD_FORMAT and value.get("version") == 1 \
599
+ and not isinstance(value.get("version"), bool):
600
+ return {**value, "version": RECORD_VERSION}
601
+ return value
569
602
 
570
603
 
571
604
  def record_problems(value):
@@ -600,8 +633,8 @@ def record_problems(value):
600
633
  or isinstance(res.get("error"), dict))):
601
634
  bad.append("a record's result is { status, body }, { error } or null")
602
635
  origin = value.get("origin")
603
- if not isinstance(origin, dict) or origin.get("kind") not in ("run", "hand", "upload"):
604
- bad.append("a record's origin is run, hand or upload")
636
+ if not isinstance(origin, dict) or origin.get("kind") not in RECORD_ORIGINS:
637
+ bad.append("a record's origin is run, hand, upload or edited")
605
638
  return bad
606
639
 
607
640
 
@@ -612,6 +645,7 @@ def take_record(name, data):
612
645
  value = json.loads(data.decode("utf-8"))
613
646
  except (UnicodeDecodeError, ValueError):
614
647
  return None, (400, f"{name} is not JSON")
648
+ value = upgrade_record(value)
615
649
  bad = record_problems(value)
616
650
  if bad:
617
651
  return None, (400, f"{name}: {bad[0]}")
@@ -689,6 +723,12 @@ def take_dynamic(name, data):
689
723
  return None, (400, f"{name} is not text")
690
724
  if TOKEN_SHAPE.search(text):
691
725
  return None, (400, f"{name} holds a token, and a token never reaches the server")
726
+ # The lab's own secrets have no shape to redact by, and a command can read
727
+ # them from its parent's environment, so they are matched by value -- a
728
+ # verbatim read only; an encoded one gets past. A value under 8 characters is too likely to
729
+ # be ordinary text to refuse a Call for.
730
+ if any(len(v) >= 8 and v in text for v in (os.environ.get(k) or "" for k in _LAB_SECRET_ENV)):
731
+ return None, (400, f"{name} holds one of the lab's own secrets")
692
732
  for pattern, to in SECRET_TEXT:
693
733
  text = pattern.sub(to, text)
694
734
  return text.encode("utf-8"), None
@@ -750,31 +790,32 @@ def _header_pairs(spec):
750
790
  def connection_auth(pid):
751
791
  """The one auth header a dynamic API Source borrows from the connection
752
792
  [pid] it names, and the key it carries so the relay's https-or-loopback
753
- rule can bind. The key is the profiles store's, by id, sent the way the
793
+ rule can bind. The key is the connections store's, by id, sent the way the
754
794
  connection's type says (an HTTP endpoint's Key header, else the type's own,
755
795
  auth_headers) -- it never leaves the server or reaches a Call (#258).
756
796
  Returns (headers, key, None); ({}, "", None) for no connection or no key;
757
- or (None, None, (code, why))."""
797
+ or (None, None, (code, why)). This is the third seam a connection resolves
798
+ to a key, beside worker_destinations and the relay's _unshared, so it
799
+ refuses a connection not shared with the bound workspace the same way --
800
+ the request thread's for a snapshot, the submitting request's for a live
801
+ run (docs/workspaces.md)."""
758
802
  if not pid:
759
803
  return {}, "", None
760
- doc = (STORE.all().get("promptlab.profiles") or {}) if STORE is not None else {}
804
+ doc = (STORE.all().get("promptlab.connections") or {}) if STORE is not None else {}
761
805
  profiles = (doc.get("body") or {}).get("list") or []
762
806
  profile = next((p for p in profiles if isinstance(p, dict) and p.get("id") == pid), None)
763
807
  if profile is None:
764
808
  return None, None, (400, "that Source's auth connection is not one the lab has")
809
+ ws = workspace_of(STORE)
810
+ if not STORE.shared("profile", pid, ws):
811
+ return None, None, (403, not_shared_sentence([profile.get("name") or pid], STORE.workspace_name(ws)))
765
812
  key = str(profile.get("key") or "").strip()
766
813
  if not key:
767
814
  return {}, "", None
768
815
  if not header_safe(key):
769
816
  return None, None, (400, "that connection's key cannot be sent in a header")
770
817
  ctype = profile.get("type")
771
- if ctype == "http":
772
- options = profile.get("options") if isinstance(profile.get("options"), dict) else {}
773
- how = str(options.get("auth") or "bearer")
774
- head = ({"x-api-key": key} if how == "x-api-key" else {"api-key": key} if how == "api-key"
775
- else {} if how == "none" else {"Authorization": "Bearer " + key})
776
- else:
777
- head = auth_headers(ctype, key)
818
+ head = http_key_header(profile.get("auth"), key) if ctype == "http" else auth_headers(ctype, key)
778
819
  return head, key, None
779
820
 
780
821
 
@@ -789,6 +830,10 @@ def produce_api(config):
789
830
  method = (cfg.get("method").upper() if isinstance(cfg.get("method"), str) and cfg.get("method").strip()
790
831
  else "GET")
791
832
  auth, key, why = connection_auth(cfg.get("connection"))
833
+ if why and why[0] == 403:
834
+ # Refused, not failed: no request went out, so the caller answers the
835
+ # refusal rather than keeping a failed Call.
836
+ return {"error": why[1], "refused": why, "request": {"url": url, "method": method}, "status": None}
792
837
  if why:
793
838
  return {"error": why[1], "request": {"url": url, "method": method}, "status": None}
794
839
  headers = {**_header_pairs(cfg.get("reqHeaders")), **auth}
@@ -858,7 +903,8 @@ def _outside_root(real):
858
903
  root = Path(LOCAL_ROOT).resolve()
859
904
  return real != root and root not in real.parents
860
905
 
861
- # The lab's own secrets in its environment, never handed to a command it runs.
906
+ # The lab's own secrets in its environment, never handed to a command it runs
907
+ # nor kept in a Call (take_dynamic).
862
908
  _LAB_SECRET_ENV = ("LAB_PASSWORD", "GOOGLE_CLIENT_SECRET", "GOOGLE_API_KEY")
863
909
 
864
910
 
@@ -895,7 +941,8 @@ def command_config_problems(config):
895
941
  def produce_file(config):
896
942
  """Read a File on disk Source's one Call now: the path its config names,
897
943
  capped at the file cap. The lab's own data directory is never read -- it
898
- holds the store, keys included. Returns the shape _relay_produce does."""
944
+ holds the store, keys included -- nor /proc, which holds the server's
945
+ environment. Returns the shape _relay_produce does."""
899
946
  path = config.get("path") if isinstance(config, dict) else None
900
947
  request = {"path": path}
901
948
  if not isinstance(path, str) or not path.strip():
@@ -910,6 +957,10 @@ def produce_file(config):
910
957
  for own in {Path(d).resolve() for d in (DATA_DIR, STORE and Path(STORE.path).parent) if d}:
911
958
  if real == own or own in real.parents:
912
959
  return {"error": "the lab's own data is not a file it reads", "request": request, "status": None}
960
+ # /proc holds the server's own environment (every secret it was started
961
+ # with, not only the lab's) and its memory: the process, not a file.
962
+ if real == Path("/proc") or Path("/proc") in real.parents:
963
+ return {"error": "the server's own process is not a file it reads", "request": request, "status": None}
913
964
  if not real.is_file():
914
965
  return {"error": "that path is not a file", "request": request, "status": None}
915
966
  try:
@@ -1024,8 +1075,10 @@ SIGN_INS = {"microsoft": lambda: microsoft_config() is not None}
1024
1075
  # in the store: a document is not migrated or deleted because nothing reads it.
1025
1076
  # promptlab.cases, .rules and their .base copies went that way when a dataset
1026
1077
  # became a row of its own (#86), and promptlab.mappings once a dataset named
1027
- # the Source it grades (#199).
1028
- SYNCED = ("promptlab.workflows", "promptlab.profiles",
1078
+ # the Source it grades (#199). promptlab.profiles went once a Target profile
1079
+ # was split into an AI Connection and a Connection Profile (#445), each kept
1080
+ # in a document of its own; its rows are what the split read.
1081
+ SYNCED = ("promptlab.workflows", "promptlab.connections", "promptlab.connectionProfiles",
1029
1082
  "promptlab.versions", "promptlab.tokens")
1030
1083
  MAX_DOC = 8 * 1024 * 1024
1031
1084
  RUNS_PAGE = 25
@@ -1088,14 +1141,16 @@ QUEUE_WAIT = 1.0
1088
1141
  WATCH_EVERY = 5.0
1089
1142
 
1090
1143
 
1091
- # A Target profile's key is written from the page and never read back by it
1144
+ # An AI Connection's key is written from the page and never read back by it
1092
1145
  # (#258): a client holding the lab's password -- CI's runners and their logs
1093
- # among them -- is handed no key. Where a profile holds one, what is served
1146
+ # among them -- is handed no key. Where a connection holds one, what is served
1094
1147
  # (/api/state, the page's carried copy, a refused write's current copy) holds
1095
- # KEY_HELD and the profile's id instead, and a write that brings that back
1096
- # keeps the key the store holds for the id: the profile's own, or the one it
1097
- # was cloned or restored from. KEY_HELD starts with a character no key can
1098
- # (header_safe), so a pasted key is never taken for it.
1148
+ # KEY_HELD and the connection's id instead, and a write that brings that back
1149
+ # keeps the key the store holds for the id: the connection's own, or the one
1150
+ # it was cloned or restored from. KEY_HELD starts with a character no key can
1151
+ # (header_safe), so a pasted key is never taken for it. A Target profile held
1152
+ # the key before the split (#445); a connection keeps its id, so a KEY_HELD
1153
+ # the page carried then still names it.
1099
1154
  KEY_HELD = "\u2022held:"
1100
1155
 
1101
1156
 
@@ -1104,11 +1159,12 @@ def held(key) -> bool:
1104
1159
 
1105
1160
 
1106
1161
  def profiles_in(name: str, body):
1107
- """Every profile a synced document holds: the profiles store's list, and
1108
- each profile version's copy."""
1162
+ """Everything keyed a synced document holds: the connections store's
1163
+ list, and each profile version's copy -- a Target profile's settings and
1164
+ key, under its connection's id."""
1109
1165
  if not isinstance(body, dict):
1110
1166
  return
1111
- if name == "promptlab.profiles":
1167
+ if name == "promptlab.connections":
1112
1168
  for p in body.get("list") or []:
1113
1169
  if isinstance(p, dict):
1114
1170
  yield p
@@ -1165,7 +1221,9 @@ SHARE_NEW = "*new"
1165
1221
  # its pipeline/dataset slices per-workspace, its profile slice global, because
1166
1222
  # profiles are global so their Restore must be too. served()/write() route
1167
1223
  # each key by these; any other key is per-workspace.
1168
- DOC_GLOBAL = ("promptlab.profiles", "promptlab.tokens")
1224
+ # promptlab.profiles stays global though no longer served: its row is what
1225
+ # the split on open reads (split_profiles).
1226
+ DOC_GLOBAL = ("promptlab.connections", "promptlab.connectionProfiles", "promptlab.profiles", "promptlab.tokens")
1169
1227
  DOC_SPLIT = "promptlab.versions"
1170
1228
 
1171
1229
 
@@ -1232,6 +1290,30 @@ class Store:
1232
1290
  db.execute("UPDATE runs SET workspace = ? WHERE workspace IS NULL", (self.default_ws,))
1233
1291
  if cols and "workspace" not in cols:
1234
1292
  self._rebuild_docs(db)
1293
+ self._split_profiles(db)
1294
+
1295
+ def _split_profiles(self, db):
1296
+ """A store from before the split (#445) opened once: its Target
1297
+ profiles become AI Connections, each keeping the profile's id, its key
1298
+ and its Setup choices (`at`, the lab's grader), and -- where a profile
1299
+ set settings of its own -- Connection Profiles (split_profiles). The
1300
+ old document stays, unserved. Idempotent: a store holding the
1301
+ connections document already, even an emptied one, is left as it is;
1302
+ a store with no profiles has nothing to split."""
1303
+ if db.execute("SELECT 1 FROM docs WHERE name = 'promptlab.connections' AND workspace IS NULL").fetchone():
1304
+ return
1305
+ row = db.execute("SELECT body FROM docs WHERE name = 'promptlab.profiles' AND workspace IS NULL").fetchone()
1306
+ try:
1307
+ body = json.loads(row[0]) if row and row[0] else None
1308
+ except ValueError:
1309
+ body = None
1310
+ if not isinstance(body, dict):
1311
+ return
1312
+ connections, profiles = split_profiles(body.get("list"))
1313
+ kept = {k: body[k] for k in ("at", "grader") if k in body}
1314
+ at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
1315
+ self._put(db, "promptlab.connections", None, 1, {"list": connections, "at": "", **kept}, at)
1316
+ self._put(db, "promptlab.connectionProfiles", None, 1, {"list": profiles}, at)
1235
1317
 
1236
1318
  def _ensure_default(self, db) -> str:
1237
1319
  """The flagged default workspace's id, seeding "Default" (/w/default,
@@ -1363,8 +1445,8 @@ class Store:
1363
1445
  (name, ws, version, text, at))
1364
1446
 
1365
1447
  def _keyring(self, db, now: dict) -> dict:
1366
- """Each profile id's key as the store holds it: its profile's, else a
1367
- dropped one's, else its newest version's."""
1448
+ """Each connection id's key as the store holds it: its connection's,
1449
+ else a dropped one's, else its newest version's."""
1368
1450
  ring = {}
1369
1451
  for p in profiles_in("promptlab.versions", now.get("promptlab.versions", {}).get("body")):
1370
1452
  pid, key = str(p.get("id") or ""), p.get("key")
@@ -1372,7 +1454,7 @@ class Store:
1372
1454
  ring[pid] = key
1373
1455
  db.execute("DELETE FROM dropped_keys WHERE at < ?", (time.time() - TRASH_SECONDS,))
1374
1456
  ring.update(db.execute("SELECT id, key FROM dropped_keys").fetchall())
1375
- for p in profiles_in("promptlab.profiles", now.get("promptlab.profiles", {}).get("body")):
1457
+ for p in profiles_in("promptlab.connections", now.get("promptlab.connections", {}).get("body")):
1376
1458
  key = p.get("key")
1377
1459
  if isinstance(key, str) and key and not held(key):
1378
1460
  ring[str(p.get("id") or "")] = key
@@ -1408,14 +1490,14 @@ class Store:
1408
1490
  for p in profiles_in(n, d["body"]):
1409
1491
  if held(p.get("key")):
1410
1492
  p["key"] = ring.get(p["key"][len(KEY_HELD):], "")
1411
- if "promptlab.profiles" in docs:
1493
+ if "promptlab.connections" in docs:
1412
1494
  kept = {str(p.get("id") or "") for p in profiles_in(
1413
- "promptlab.profiles", docs["promptlab.profiles"]["body"])}
1495
+ "promptlab.connections", docs["promptlab.connections"]["body"])}
1414
1496
  db.executemany("DELETE FROM dropped_keys WHERE id = ?", [(i,) for i in kept])
1415
1497
  db.executemany(
1416
1498
  "INSERT OR REPLACE INTO dropped_keys (id, key, at) VALUES (?, ?, ?)",
1417
1499
  [(str(p.get("id") or ""), p["key"], time.time())
1418
- for p in profiles_in("promptlab.profiles", have("promptlab.profiles")["body"])
1500
+ for p in profiles_in("promptlab.connections", have("promptlab.connections")["body"])
1419
1501
  if str(p.get("id") or "") not in kept and isinstance(p.get("key"), str) and p["key"]])
1420
1502
  for n, d in docs.items():
1421
1503
  version, body = have(n)["version"] + 1, d["body"]
@@ -1434,9 +1516,10 @@ class Store:
1434
1516
  return {n: have(n)["version"] + 1 for n in docs}, None
1435
1517
 
1436
1518
  # ---- shared-by-choice connections (docs/workspaces.md) ------------------
1437
- # A Target profile or an Account (kind "profile"/"account", id the
1438
- # profile's or the account's) is usable in a workspace by its share set in
1439
- # `connection_shares`: a row for that workspace, or the SHARE_ALL sentinel.
1519
+ # An AI Connection or an Account (kind "profile"/"account", id the
1520
+ # connection's -- the Target profile's it kept -- or the account's) is
1521
+ # usable in a workspace by its share set in `connection_shares`: a row for
1522
+ # that workspace, or the SHARE_ALL sentinel.
1440
1523
  # No row at all is the migration and new-connection default -- shared with
1441
1524
  # every workspace, so nothing stops running the moment workspaces exist
1442
1525
  # (open question 2, resolved All); narrowing a connection is adding the
@@ -1498,10 +1581,10 @@ class Store:
1498
1581
  return row[0] if row else "this workspace"
1499
1582
 
1500
1583
  def profile_name(self, pid: str) -> str:
1501
- """A Target profile's display name, by id, from the global profiles
1584
+ """An AI Connection's display name, by id, from the global connections
1502
1585
  document -- for the refusal sentence. The id itself when none is held."""
1503
1586
  with self.lock, closing(sqlite3.connect(self.path)) as db:
1504
- row = db.execute("SELECT body FROM docs WHERE name = 'promptlab.profiles' "
1587
+ row = db.execute("SELECT body FROM docs WHERE name = 'promptlab.connections' "
1505
1588
  "AND workspace IS NULL").fetchone()
1506
1589
  try:
1507
1590
  for p in (json.loads(row[0]) or {}).get("list", []) if row and row[0] else []:
@@ -1827,6 +1910,15 @@ CONFIG_KEYS_MAX = 10
1827
1910
  CONFIG_VALUE_MAX = 400
1828
1911
 
1829
1912
 
1913
+ class _Refused(Exception):
1914
+ """A refusal found inside a transaction: rolls it back, carrying the
1915
+ (code, sentence) the caller answers with."""
1916
+
1917
+ def __init__(self, err):
1918
+ super().__init__(err[1])
1919
+ self.err = err
1920
+
1921
+
1830
1922
  class Sources:
1831
1923
  """
1832
1924
  A named bag of files a run might be over (#526).
@@ -2421,16 +2513,19 @@ class Sources:
2421
2513
  if produce is None:
2422
2514
  return None, (400, "that provider cannot be produced from on the server")
2423
2515
  result = produce(cfg)
2516
+ if result.get("refused"):
2517
+ return None, result["refused"]
2424
2518
  request = result.get("request")
2425
2519
  status = result.get("status")
2426
2520
  content = result.get("content")
2427
2521
  if content is None:
2428
2522
  # A failed production: kept as a Call with its error, no file.
2429
- name = dynamic_item_name(cfg, (request or {}).get("url", "") or secrets.token_hex(6))
2523
+ name = self._snapshot_name(sid, dynamic_item_name(cfg, (request or {}).get("url", "")
2524
+ or secrets.token_hex(6)))
2430
2525
  self._record_call(sid, name, "snapshot", provider, request, status, result.get("error"), None)
2431
2526
  return {"name": name, "origin": "snapshot", "at": None, "provider": provider,
2432
2527
  "request": request, "status": status, "error": result.get("error")}, None
2433
- name = dynamic_item_name(cfg, content)
2528
+ name = self._snapshot_name(sid, dynamic_item_name(cfg, content))
2434
2529
  kept, take_err = entry["take"](name, content)
2435
2530
  if take_err:
2436
2531
  code, message = take_err
@@ -2438,7 +2533,8 @@ class Sources:
2438
2533
  return {"name": name, "origin": "snapshot", "at": None, "provider": provider,
2439
2534
  "request": request, "status": status, "error": message}, None
2440
2535
  # The name is stable (the URL's), so a re-snapshot overwrites the item
2441
- # rather than adding a second: remove the old file first, then add.
2536
+ # rather than adding a second: remove the old file first, then add. An
2537
+ # Edited Call is never the one overwritten (_snapshot_name).
2442
2538
  self.remove_files(sid, [name])
2443
2539
  stored, add_err = self.add_file(sid, name, kept)
2444
2540
  if add_err:
@@ -2447,6 +2543,162 @@ class Sources:
2447
2543
  return {"name": stored, "origin": "snapshot", "at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
2448
2544
  "provider": provider, "request": request, "status": status, "bytes": len(kept)}, None
2449
2545
 
2546
+ def _snapshot_name(self, sid, name):
2547
+ """The name a snapshot is kept under: [name], unless an Edited Call
2548
+ holds it -- a snapshot never overwrites an edit -- and then the first
2549
+ of `<stem>-2`, `<stem>-3`... that none does. One that is not Edited is
2550
+ overwritten, as the stable name always was."""
2551
+ with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
2552
+ edited = {r[0] for r in db.execute(
2553
+ "SELECT name FROM source_calls WHERE source = ? AND origin = 'edited'", (sid,))}
2554
+ if name not in edited:
2555
+ return name
2556
+ stem, ext = os.path.splitext(name)
2557
+ n = 2
2558
+ while f"{stem}-{n}{ext}" in edited:
2559
+ n += 1
2560
+ return f"{stem}-{n}{ext}"
2561
+
2562
+ # ---- a Call opened in the Call dialog: edited, renamed, cloned (#438) ---
2563
+ #
2564
+ # Both kinds of Call -- a dynamic Source's and a Power Automate record --
2565
+ # are files of a Source whose kind has a `take` hook, so a saved edit goes
2566
+ # through the same check as content arriving any other way: a key refused,
2567
+ # the rest redacted. A File Library has no hook and no Calls.
2568
+
2569
+ # The origins a dynamic Call's provenance row may be put back to (Undo).
2570
+ CALL_ORIGINS = ("snapshot", "run", "upload", "hand", "edited")
2571
+
2572
+ def _calls_entry(self, sid):
2573
+ """The enforcement entry of a user Source whose files are Calls, or
2574
+ (None, error)."""
2575
+ with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
2576
+ r = db.execute("SELECT system FROM sources WHERE id = ? AND (workspace = ? OR system = 1)",
2577
+ (sid, workspace_of(self.store))).fetchone()
2578
+ if r is None:
2579
+ return None, (404, "no such source")
2580
+ if r[0]:
2581
+ return None, (403, "the sample library is read-only")
2582
+ entry = self.type_of(sid) or {}
2583
+ if entry.get("take") is None:
2584
+ return None, (400, "that Source keeps no Calls")
2585
+ return entry, None
2586
+
2587
+ def _names_held(self, db, sid):
2588
+ """Every name a Source's files and Calls hold: what a new name may not be."""
2589
+ return ({r[0] for r in db.execute("SELECT name FROM source_files WHERE source = ?", (sid,))}
2590
+ | {r[0] for r in db.execute("SELECT name FROM source_calls WHERE source = ?", (sid,))})
2591
+
2592
+ def edit_call(self, sid, name, to=None, content=None, origin=None, named_by=None):
2593
+ """One Call as the Call dialog saves it: renamed to [to], its text
2594
+ replaced by [content], or both, in one write. The text goes through
2595
+ the Source's `take` hook as an upload does. A rename keeps the
2596
+ extension and refuses a name in use, or a Call that a dataset's cases
2597
+ name ([named_by] answers which datasets do), since a case finds its
2598
+ item by name. A dynamic Call's origin becomes `edited` when its text
2599
+ changed -- a rename alone is no edit -- unless [origin] puts back the
2600
+ one it had (Undo). Nothing is written unless all of it passes.
2601
+ Returns ({name, origin, bytes}, None), or (None, (code, sentence)) --
2602
+ (code, sentence, "name") when the Name field is what is wrong."""
2603
+ entry, err = self._calls_entry(sid)
2604
+ if err:
2605
+ return None, err
2606
+ path = self.file_path(sid, name)
2607
+ if path is None or not path.is_file():
2608
+ return None, (404, f"{name} is not a Call of that Source")
2609
+ if origin is not None and origin not in self.CALL_ORIGINS:
2610
+ return None, (400, "that is not an origin a Call has")
2611
+ new = name
2612
+ if to is not None and to != name:
2613
+ new = source_name(to)
2614
+ if new is None or new != to:
2615
+ return None, (400, "that name is not accepted here", "name")
2616
+ if Path(new).suffix.lower() != Path(name).suffix.lower():
2617
+ return None, (400, "a Call keeps its extension", "name")
2618
+ with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
2619
+ taken = new in self._names_held(db, sid)
2620
+ if taken:
2621
+ return None, (409, f"{new} is taken.", "name")
2622
+ users = named_by(name) if named_by else []
2623
+ if users:
2624
+ return None, (409, f"{users[0]} names {name} in its cases.", "name")
2625
+ old = path.read_bytes()
2626
+ data = old
2627
+ if content is not None:
2628
+ if not isinstance(content, str):
2629
+ return None, (400, "a Call's content is text")
2630
+ raw = content.encode("utf-8")
2631
+ if not raw:
2632
+ return None, (400, "a Call needs some content")
2633
+ if len(raw) > MAX_FILE:
2634
+ return None, (413, "that is over the file cap")
2635
+ data, take_err = entry["take"](new, raw)
2636
+ if take_err:
2637
+ return None, take_err
2638
+ changed = data != old
2639
+ folder = self.dir / sid
2640
+ tmp = None
2641
+ if changed or new != name:
2642
+ with tempfile.NamedTemporaryFile("wb", dir=folder, delete=False) as f:
2643
+ f.write(data)
2644
+ tmp = f.name
2645
+ try:
2646
+ with self.store.lock, closing(sqlite3.connect(self.store.path)) as db, db:
2647
+ if new != name and new in self._names_held(db, sid):
2648
+ raise _Refused((409, f"{new} is taken.", "name"))
2649
+ total = db.execute("SELECT bytes FROM sources WHERE id = ?", (sid,)).fetchone()[0]
2650
+ if total + len(data) - len(old) > MAX_SOURCE:
2651
+ raise _Refused((413, "that would take the source past its cap"))
2652
+ db.execute("UPDATE source_files SET name = ?, bytes = ? WHERE source = ? AND name = ?",
2653
+ (new, len(data), sid, name))
2654
+ db.execute("UPDATE sources SET bytes = bytes + ? WHERE id = ?", (len(data) - len(old), sid))
2655
+ row = db.execute("SELECT origin FROM source_calls WHERE source = ? AND name = ?",
2656
+ (sid, name)).fetchone()
2657
+ now = None
2658
+ if row is not None:
2659
+ now = origin or ("edited" if changed else row[0])
2660
+ db.execute("UPDATE source_calls SET name = ?, file = ?, origin = ? "
2661
+ "WHERE source = ? AND name = ?", (new, new, now, sid, name))
2662
+ except _Refused as no:
2663
+ if tmp:
2664
+ os.unlink(tmp)
2665
+ return None, no.err
2666
+ if tmp:
2667
+ os.replace(tmp, folder / new)
2668
+ if new != name:
2669
+ (folder / name).unlink(missing_ok=True)
2670
+ return {"name": new, "origin": now, "bytes": len(data)}, None
2671
+
2672
+ def clone_call(self, sid, name):
2673
+ """A copy of one Call under `<stem>-copy` (then `-copy-2`, ...), its
2674
+ file and, for a dynamic Call, its provenance -- origin, request,
2675
+ status and when it was produced -- as they are. Returns ({name}, None),
2676
+ or (None, error)."""
2677
+ entry, err = self._calls_entry(sid)
2678
+ if err:
2679
+ return None, err
2680
+ path = self.file_path(sid, name)
2681
+ if path is None or not path.is_file():
2682
+ return None, (404, f"{name} is not a Call of that Source")
2683
+ stem, ext = os.path.splitext(name)
2684
+ with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
2685
+ held = self._names_held(db, sid)
2686
+ n = 1
2687
+ while True:
2688
+ tail = "-copy" if n == 1 else f"-copy-{n}"
2689
+ copy = stem[:120 - len(tail) - len(ext)] + tail + ext
2690
+ if copy not in held:
2691
+ break
2692
+ n += 1
2693
+ stored, err = self.add_file(sid, copy, path.read_bytes())
2694
+ if err:
2695
+ return None, err
2696
+ with self.store.lock, closing(sqlite3.connect(self.store.path)) as db, db:
2697
+ db.execute("INSERT INTO source_calls (source, name, origin, at, provider, request, status, error, file) "
2698
+ "SELECT source, ?, origin, at, provider, request, status, error, ? "
2699
+ "FROM source_calls WHERE source = ? AND name = ?", (stored, stored, sid, name))
2700
+ return {"name": stored}, None
2701
+
2450
2702
  def remove_calls(self, sid, names):
2451
2703
  """Calls out by name: a file-backed Call to the trash (Undo restores
2452
2704
  it), a failed Call's row dropped outright (it has no file). Returns
@@ -3871,6 +4123,24 @@ class Datasets:
3871
4123
  "SELECT name FROM datasets WHERE trash IS NULL AND id IS NOT ? AND workspace = ?",
3872
4124
  (but, workspace_of(self.store)))}
3873
4125
 
4126
+ def naming_item(self, sid, item) -> list:
4127
+ """The names of this workspace's groups previewed against Source
4128
+ [sid] whose cases name [item]: what renaming that item would break."""
4129
+ out = []
4130
+ with self.store.lock, self._connect() as db:
4131
+ for r in db.execute("SELECT name, body, preview FROM datasets WHERE trash IS NULL "
4132
+ "AND workspace = ? AND preview IS NOT NULL ORDER BY name COLLATE NOCASE",
4133
+ (workspace_of(self.store),)):
4134
+ try:
4135
+ ref, body = json.loads(r["preview"]), json.loads(r["body"])
4136
+ except ValueError:
4137
+ continue
4138
+ cases = body.get("cases") if isinstance(body, dict) else None
4139
+ if isinstance(ref, dict) and ref.get("id") == sid and isinstance(cases, list) \
4140
+ and any(isinstance(c, dict) and c.get("item") == item for c in cases):
4141
+ out.append(r["name"])
4142
+ return out
4143
+
3874
4144
  def list(self) -> list:
3875
4145
  with self.store.lock, self._connect() as db:
3876
4146
  counts = self._counts(db)
@@ -4781,32 +5051,47 @@ class Pipelines:
4781
5051
  self.store._put(db, self.VERSIONS, target_ws, tv + 1, tbody, self._now())
4782
5052
 
4783
5053
 
4784
- def make_thumbs(sid, name):
4785
- """JPEGs a browser can render, beside a stored file: a tile-size one
4786
- for every image type, and a full-size render for the formats a browser
4787
- cannot decode, whose zoom view has no original to show. Non-fatal by
4788
- design: a missing thumb is a blank tile, not a failed upload, and the
4789
- startup sweep clears an orphan on a later start. #33 #527"""
5054
+ def thumb_jobs(sid, name):
5055
+ """The `magick` runs that make a stored file's thumbnails, each (argv,
5056
+ out): a tile-size JPEG for every image type, and a full-size render for
5057
+ the formats a browser cannot decode, whose zoom view has no original to
5058
+ show. The paths are looked up here, in the request's own thread, since a
5059
+ Source is found through the workspace bound to it. #33 #527"""
4790
5060
  if CONVERT is None or Path(name).suffix.lower() not in IMAGE_TYPES:
4791
- return
5061
+ return []
5062
+ jobs = []
4792
5063
  for full in (False, True):
4793
5064
  if full and Path(name).suffix.lower() not in THUMB_TYPES:
4794
5065
  continue
4795
5066
  path = SOURCES.file_path(sid, name)
4796
5067
  out = SOURCES.thumb_path(sid, name, full)
4797
5068
  if path is None or out is None:
4798
- return
5069
+ return jobs
4799
5070
  size = FULL_SIZE if full else TILE_SIZE
4800
- out.parent.mkdir(parents=True, exist_ok=True)
4801
- try:
4802
- subprocess.run([CONVERT, str(path), "-auto-orient",
4803
- "-thumbnail", f"{size}x{size}>",
4804
- "-quality", str(FULL_QUALITY if full else TILE_QUALITY),
4805
- str(out)],
4806
- timeout=30, stdout=subprocess.DEVNULL,
4807
- stderr=subprocess.DEVNULL, check=True)
4808
- except (subprocess.SubprocessError, OSError):
4809
- out.unlink(missing_ok=True)
5071
+ jobs.append(([CONVERT, str(path), "-auto-orient",
5072
+ "-thumbnail", f"{size}x{size}>",
5073
+ "-quality", str(FULL_QUALITY if full else TILE_QUALITY),
5074
+ str(out)], out))
5075
+ return jobs
5076
+
5077
+
5078
+ def run_thumb(job):
5079
+ """One thumbnail. Non-fatal by design: a missing thumb is a blank tile,
5080
+ not a failed upload, and the startup sweep clears an orphan on a later
5081
+ start."""
5082
+ argv, out = job
5083
+ out.parent.mkdir(parents=True, exist_ok=True)
5084
+ try:
5085
+ subprocess.run(argv, timeout=30, stdout=subprocess.DEVNULL,
5086
+ stderr=subprocess.DEVNULL, check=True)
5087
+ except (subprocess.SubprocessError, OSError):
5088
+ out.unlink(missing_ok=True)
5089
+
5090
+
5091
+ def make_thumbs(sid, name):
5092
+ """A stored file's thumbnails, made now (thumb_jobs)."""
5093
+ for job in thumb_jobs(sid, name):
5094
+ run_thumb(job)
4810
5095
 
4811
5096
 
4812
5097
  # ---- Packs ------------------------------------------------------------------
@@ -4877,7 +5162,9 @@ def pack_requires_problem(manifest: dict, plugins: set) -> str:
4877
5162
  def read_pack(data: bytes):
4878
5163
  """A pack's zip read and checked whole: (pack, None) or (None, why). The
4879
5164
  pack is {manifest, datasets: [(key, name, body, raw)], pipelines:
4880
- [(key, doc)], presets: [...], sources: {name: [(file, bytes)]}}."""
5165
+ [(key, doc)], presets: [...], sources: {name: [(file, bytes)]}, source_meta:
5166
+ {name: {name, type, config}}} -- a Source's meta from its folder's
5167
+ `source.json`, or the folder's name as a File Library."""
4881
5168
  try:
4882
5169
  zf = zipfile.ZipFile(io.BytesIO(data))
4883
5170
  except zipfile.BadZipFile:
@@ -4906,7 +5193,8 @@ def read_pack(data: bytes):
4906
5193
  for k in ("name", "packVersion"):
4907
5194
  if not isinstance(manifest.get(k), str) or not manifest[k].strip():
4908
5195
  return None, f"a pack's manifest needs a {k}"
4909
- pack = {"manifest": manifest, "datasets": [], "pipelines": [], "presets": [], "sources": {}}
5196
+ pack = {"manifest": manifest, "datasets": [], "pipelines": [], "presets": [], "sources": {}, "source_meta": {}}
5197
+ declared = {}
4910
5198
  for name in sorted(names):
4911
5199
  parts = name.split("/")
4912
5200
  if name == "manifest.json":
@@ -4949,21 +5237,86 @@ def read_pack(data: bytes):
4949
5237
  or not (isinstance(doc.get("jobs"), list) or isinstance(doc.get("chains"), list))):
4950
5238
  return None, f"{name} is not a pipeline document the lab reads"
4951
5239
  pack["pipelines"].append((parts[1][:-5], doc))
5240
+ elif len(parts) == 3 and parts[0] == "sources" and parts[2] == PACK_SOURCE_META:
5241
+ src = source_name(parts[1])
5242
+ meta, why = load(name)
5243
+ if why:
5244
+ return None, why
5245
+ if src is None or not isinstance(meta, dict):
5246
+ return None, f"{name} is not a Source's {PACK_SOURCE_META}"
5247
+ declared[src] = meta
4952
5248
  elif len(parts) == 3 and parts[0] == "sources":
4953
5249
  src, file = source_name(parts[1]), source_name(parts[2])
4954
- if src is None or file is None or Path(file).suffix.lower() not in FILE_TYPES:
5250
+ if src is None or file is None:
4955
5251
  return None, f"{name} is not a file a Source takes"
4956
5252
  if names[name].file_size > MAX_FILE:
4957
5253
  return None, f"{name} is over the cap for a file"
4958
5254
  pack["sources"].setdefault(src, []).append((file, zf.read(name)))
4959
5255
  else:
4960
5256
  return None, f"{name} is not something a pack holds"
4961
- for src, files in pack["sources"].items():
5257
+ for src in sorted(set(pack["sources"]) | set(declared)):
5258
+ meta, why = pack_source_meta(src, declared.get(src))
5259
+ if why:
5260
+ return None, why
5261
+ pack["source_meta"][src] = meta
5262
+ files = pack["sources"].setdefault(src, [])
5263
+ entry = source_entry(meta["type"], meta["config"]) or {}
5264
+ takes, take = entry.get("uploads") or {}, entry.get("take")
5265
+ for i, (file, blob) in enumerate(files):
5266
+ if Path(file).suffix.lower() not in takes:
5267
+ return None, f"sources/{src}/{file} is not a file a Source takes"
5268
+ # A shipped Call is checked as any other is: a token refuses the
5269
+ # pack, and a secret's shape is redacted before it is kept.
5270
+ if take is not None:
5271
+ blob, err = take(file, blob)
5272
+ if err:
5273
+ return None, f"sources/{src}/{file}: {err[1]}"
5274
+ files[i] = (file, blob)
4962
5275
  if sum(len(b) for _, b in files) > MAX_SOURCE:
4963
5276
  return None, f"the Source {src} is over the cap for a Source"
4964
5277
  return pack, None
4965
5278
 
4966
5279
 
5280
+ # A pack Source's own meta, beside its files: {name?, type?, config?}.
5281
+ PACK_SOURCE_META = "source.json"
5282
+
5283
+
5284
+ def pack_source_meta(folder, meta):
5285
+ """A pack Source's {name, type, config} from its folder's source.json (or
5286
+ None: a File Library named after the folder), or (None, why). A pack may
5287
+ carry a File Library or a dynamic Source whose provider pulls with no key
5288
+ and is enrolled here -- the line a plugin's provider is held to
5289
+ (docs/packs.md); anything else refuses the pack whole."""
5290
+ meta = meta or {}
5291
+ at = f"sources/{folder}/{PACK_SOURCE_META}"
5292
+ unknown = set(meta) - {"name", "type", "config"}
5293
+ if unknown:
5294
+ return None, f"{at} holds {sorted(unknown)[0]}, which a pack Source does not"
5295
+ name, err = Sources._name(meta.get("name", folder))
5296
+ if err:
5297
+ return None, f"{at}: {err[1]}"
5298
+ kind = meta.get("type", DEFAULT_SOURCE_TYPE)
5299
+ config = meta.get("config")
5300
+ if kind == DEFAULT_SOURCE_TYPE:
5301
+ if config is not None:
5302
+ return None, f"{at}: a File Library takes no config"
5303
+ return {"name": name, "type": kind, "config": None}, None
5304
+ if not (SOURCE_TYPES.get(kind) or {}).get("providers"):
5305
+ return None, f"{at}: a pack carries a File Library or a dynamic Source, not {kind!r}"
5306
+ provider = config.get("provider") if isinstance(config, dict) else None
5307
+ entry = SOURCE_PROVIDERS.get(provider)
5308
+ if entry is None:
5309
+ return None, f"{at}: the lab has no provider {provider!r}"
5310
+ if entry.get("tier") != "pull" or entry.get("keyed") or config.get("connection"):
5311
+ return None, f"{at}: a pack's dynamic Source pulls with no key, and {entry.get('label', provider)} does not"
5312
+ if not provider_enrolled(entry):
5313
+ return None, f"{at}: this lab does not enrol {entry.get('label', provider)}"
5314
+ _, err = Sources._checked(kind, config)
5315
+ if err:
5316
+ return None, f"{at}: {err[1]}"
5317
+ return {"name": name, "type": kind, "config": config}, None
5318
+
5319
+
4967
5320
  class Packs:
4968
5321
  """What each installed pack made, so it can be updated and removed."""
4969
5322
 
@@ -5098,22 +5451,41 @@ class Packs:
5098
5451
  made = {}
5099
5452
  # Sources first, then datasets: a pipeline names both.
5100
5453
  src_ids = {}
5454
+ thumbs = []
5101
5455
  for name, files in pack["sources"].items():
5456
+ meta = pack["source_meta"][name]
5102
5457
  sid = owned.get(("source", name))
5103
- if sid and SOURCES.get(sid):
5104
- old = [f["name"] for f in SOURCES.get(sid)["files"]]
5458
+ have = SOURCES.get(sid) if sid else None
5459
+ # An update keeps the Source's name, so one renamed here, or
5460
+ # by a later pack, is not renamed back. Another kind is
5461
+ # another Source.
5462
+ if have and have.get("type") == meta["type"]:
5463
+ old = [f["name"] for f in have["files"]]
5105
5464
  if old:
5106
5465
  SOURCES.remove_files(sid, old)
5466
+ if meta["config"] is not None:
5467
+ SOURCES.update(sid, {"config": meta["config"]})
5107
5468
  else:
5108
5469
  taken = {s["name"] for s in SOURCES.list()}
5109
- src, _ = SOURCES.create(unique_dataset_name(name, taken))
5470
+ src, _ = SOURCES.create(unique_dataset_name(meta["name"], taken), meta["type"], meta["config"])
5110
5471
  sid = src["id"]
5111
5472
  for file, blob in files:
5112
5473
  stored, err = SOURCES.add_file(sid, file, blob)
5113
- if not err:
5114
- make_thumbs(sid, stored)
5474
+ if err:
5475
+ continue
5476
+ thumbs += thumb_jobs(sid, stored)
5477
+ # A dynamic Source's files are Calls: the pack's are its
5478
+ # snapshot, fetched when the pack was made.
5479
+ if meta["type"] != DEFAULT_SOURCE_TYPE:
5480
+ cfg = meta["config"]
5481
+ request = {"url": cfg["url"], "method": cfg.get("method") or "GET"} if cfg.get("url") else None
5482
+ SOURCES._record_call(sid, stored, "snapshot", cfg.get("provider"), request, None, None, stored)
5115
5483
  src_ids[name] = sid
5116
5484
  made[("source", name)] = sid
5485
+ # The demo pack installs as the server starts, before it answers,
5486
+ # and one `magick` at a time was most of that wait.
5487
+ with ThreadPoolExecutor(max_workers=min(4, os.cpu_count() or 1)) as pool:
5488
+ list(pool.map(run_thumb, thumbs))
5117
5489
  ds_ids = {}
5118
5490
  cuts = [] # (kind, id, the document as it was), kept before it changes
5119
5491
  for key, name, body, raw, preview in pack["datasets"]:
@@ -5170,7 +5542,8 @@ class Packs:
5170
5542
  entry = next((w for w in body["list"] if w.get("id") == wid), None) if wid else None
5171
5543
  if entry:
5172
5544
  cuts.append(("pipeline", wid, entry.get("work")))
5173
- entry["work"] = doc
5545
+ # Its name is the lab's now, as a dataset's is.
5546
+ entry["work"] = {**doc, "name": entry.get("name") or doc.get("name")}
5174
5547
  else:
5175
5548
  wid = "w" + secrets.token_hex(6)
5176
5549
  name = unique_dataset_name(doc.get("name") or key, {w.get("name") for w in body["list"]})
@@ -6942,7 +7315,7 @@ def bundle_problems(files, keys=()):
6942
7315
  for key in keys:
6943
7316
  key = str(key or "").strip()
6944
7317
  if len(key) >= 4 and key in text:
6945
- return f"{path} holds a Target profile's key, and a key never leaves the lab"
7318
+ return f"{path} holds a connection's key, and a key never leaves the lab"
6946
7319
  if TOKEN_SHAPE.search(text):
6947
7320
  return f"{path} holds a token, and a key never leaves the lab"
6948
7321
  m = BUNDLE_KEY_FIELD.search(text)
@@ -6965,7 +7338,7 @@ def build_bundle(payload):
6965
7338
  return None, (400, "a bundle holds pipeline.yaml and profiles.yaml")
6966
7339
  stored = []
6967
7340
  if STORE is not None:
6968
- stored = ((STORE.all().get("promptlab.profiles") or {}).get("body") or {}).get("list") or []
7341
+ stored = ((STORE.all().get("promptlab.connections") or {}).get("body") or {}).get("list") or []
6969
7342
  why = bundle_problems(files, [p.get("key") for p in stored if isinstance(p, dict)])
6970
7343
  if why:
6971
7344
  return None, (400, why)
@@ -6994,10 +7367,12 @@ def build_bundle(payload):
6994
7367
 
6995
7368
 
6996
7369
  # Shared-by-choice enforcement (docs/workspaces.md): the one sentence the Run
6997
- # bar turns into a Share link, and the profiles that earn it -- the run's
6998
- # Target profiles not shared with workspace `ws`. A local profile reaches
6999
- # nothing, so sharing does not gate it. Keyed by the run's profiles table,
7000
- # whose keys are the profile ids the share set is written against.
7370
+ # bar turns into a Share link, and the connections that earn it -- the AI
7371
+ # Connections the run resolved to that are not shared with workspace `ws`,
7372
+ # a profile's own connection included where it answers on one. A local
7373
+ # connection reaches nothing, so sharing does not gate it. A share set is
7374
+ # written against the id a Target profile had, which its connection keeps
7375
+ # (#445), so a "profile" row names the connection.
7001
7376
  def not_shared_sentence(names, wsname: str) -> str:
7002
7377
  verb = "is" if len(names) == 1 else "are"
7003
7378
  return f"{', '.join(names)} {verb} not shared with {wsname}."
@@ -7007,12 +7382,16 @@ def unshared_profiles(run: dict, ws):
7007
7382
  table = run.get("profiles") if isinstance(run, dict) else None
7008
7383
  if not isinstance(table, dict) or STORE is None:
7009
7384
  return []
7010
- out = []
7011
- for pid, conn in table.items():
7385
+ out, seen = [], set()
7386
+ for key, conn in table.items():
7012
7387
  if isinstance(conn, dict) and conn.get("type") in LOCAL_CONNECTIONS:
7013
7388
  continue
7014
- if not STORE.shared("profile", pid, ws):
7015
- out.append({"id": pid, "name": (isinstance(conn, dict) and conn.get("name")) or pid})
7389
+ cid = answer_connection(key, conn)
7390
+ if cid in seen:
7391
+ continue
7392
+ seen.add(cid)
7393
+ if not STORE.shared("profile", cid, ws):
7394
+ out.append({"id": cid, "name": (isinstance(conn, dict) and conn.get("name")) or cid})
7016
7395
  return out
7017
7396
 
7018
7397
 
@@ -7029,28 +7408,38 @@ def worker_destinations(run: dict):
7029
7408
  return None, not_shared_sentence([b["name"] for b in bad], STORE.workspace_name(ws))
7030
7409
  stored = []
7031
7410
  if STORE is not None:
7032
- doc = STORE.all().get("promptlab.profiles") or {}
7411
+ doc = STORE.all().get("promptlab.connections") or {}
7033
7412
  stored = (doc.get("body") or {}).get("list") or []
7034
- env = {}
7035
- for pid, conn in table.items():
7413
+ # One variable a connection, however many targets or profiles share it --
7414
+ # and never two connections one variable, whichever path the run came by:
7415
+ # a stored snapshot is not checked again as a submit is (run_problems),
7416
+ # and the runner reads each connection's key from its slug's variable, so
7417
+ # the second would send the first one's key to its own address.
7418
+ env, spelt = {}, {}
7419
+ for key, conn in table.items():
7420
+ pid = answer_connection(key, conn)
7036
7421
  name = conn.get("name") or pid
7422
+ var = key_var(pid, conn.get("slug") if isinstance(conn.get("slug"), str) else None)
7423
+ if var in spelt and spelt[var][0] != pid:
7424
+ return None, f"Connections {spelt[var][1]} and {name} both take their key from ${var}"
7425
+ spelt[var] = (pid, name)
7037
7426
  if isinstance(conn, dict) and conn.get("type") in LOCAL_CONNECTIONS:
7038
7427
  continue
7039
7428
  base = api_base(str(conn.get("url") or "")) or api_base(OLLAMA)
7040
7429
  why = allowed(base, "")
7041
7430
  if why:
7042
- return None, f"Target profile {name}: {why}"
7431
+ return None, f"Connection {name}: {why}"
7043
7432
  profile = next((p for p in stored if isinstance(p, dict) and p.get("id") == pid), None)
7044
7433
  if profile is None:
7045
- return None, f"Target profile {name} not found"
7434
+ return None, f"Connection {name} not found"
7046
7435
  key = str(profile.get("key") or "").strip()
7047
7436
  if key:
7048
7437
  if not header_safe(key):
7049
- return None, f"Target profile {name} has a key that cannot go in a header"
7438
+ return None, f"Connection {name} has a key that cannot go in a header"
7050
7439
  why = allowed(base, key)
7051
7440
  if why:
7052
- return None, f"Target profile {name}: {why}"
7053
- env[key_var(pid, conn.get("slug"))] = key
7441
+ return None, f"Connection {name}: {why}"
7442
+ env[var] = key
7054
7443
  return env, None
7055
7444
 
7056
7445
 
@@ -7078,11 +7467,16 @@ def worker_destinations(run: dict):
7078
7467
  # inline Drop items / Reject rules migrate into a private one (§18, #339).
7079
7468
  # 15: every cleaning step is a filter -- a job's Tidy / Drop prompt echoes /
7080
7469
  # Remove duplicates / Keep first / Case modifiers and its List `preamble`
7081
- # setting migrate into before/after-Read-as filter sets (§18, #405).
7082
- PIPELINE_VERSION = 15
7470
+ # setting migrate into Pre-/Post-format filter sets (§18, #405).
7471
+ # 16: a target names an AI Connection and, optionally, a Connection Profile,
7472
+ # where it named one Target profile; the run's table is keyed by the pair
7473
+ # (answer_key), each entry naming the connection whose key it takes (#445).
7474
+ # 17: an HTTP Request target has no `api` of its own; its job's flow step's
7475
+ # is the one its body is read by (#477).
7476
+ PIPELINE_VERSION = 17
7083
7477
  # What a stored run may be: the current version, and the ones evals-core.ts's
7084
7478
  # upgradePipeline reads. A new submission is upgraded to the current one.
7085
- READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)
7479
+ READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17)
7086
7480
  TARGET_CAP = 4
7087
7481
  # Target steps whose words the Prompt library does not record as a use: they
7088
7482
  # ask no model (evals-core.ts's STEP_TYPES.echo).
@@ -7115,7 +7509,7 @@ CONNECTION_TYPES = {
7115
7509
  "echo": (),
7116
7510
  # An HTTP endpoint (evals-core.ts): an address and how its key travels; an
7117
7511
  # HTTP Request step builds the whole request.
7118
- "http": ("auth", "headers", "testPath"),
7512
+ "http": ("auth", "headers", "testPath", "requestFormat"),
7119
7513
  "ollama": ("seed", "nPredict", "temperature", "repeatPenalty"),
7120
7514
  "ollama-cloud": ("seed", "nPredict", "temperature", "repeatPenalty"),
7121
7515
  "openai-compatible": ("temperature", "nPredict"),
@@ -7133,11 +7527,88 @@ BUILTIN_CONNECTION_TYPES = dict(CONNECTION_TYPES)
7133
7527
  BUILTIN_LOCAL = set(LOCAL_CONNECTIONS)
7134
7528
  BUILTIN_CHAT_PATHS = dict(CONNECTION_CHAT_PATHS)
7135
7529
  BUILTIN_AUTH = dict(CONNECTION_AUTH)
7136
- CONNECTION_FIELDS = ("name", "slug", "url", "model", "type", "temperature", "px", "format",
7530
+ CONNECTION_FIELDS = ("name", "slug", "connection", "url", "model", "type", "temperature", "px", "format",
7137
7531
  "quality", "options")
7138
7532
  LOOKS_LIKE_A_KEY = re.compile(r"^(?:api[-_]?)?key$|^(?:authorization|bearer|secret|password|token)$",
7139
7533
  re.IGNORECASE)
7140
7534
  PROFILE_ID = re.compile(r"[A-Za-z0-9]+(?:-[A-Za-z0-9]+)?")
7535
+ # A run's table key (evals-core.ts answerKey): a connection's id, then a dot
7536
+ # and its profile's id -- or nothing, for none -- or a grader's bare id.
7537
+ ANSWER_KEY = re.compile(r"[A-Za-z0-9]+(?:-[A-Za-z0-9]+)?(?:\.(?:[A-Za-z0-9]+(?:-[A-Za-z0-9]+)?)?)?")
7538
+
7539
+
7540
+ def answer_key(connection: str, profile=None) -> str:
7541
+ return f"{connection}.{profile or ''}"
7542
+
7543
+
7544
+ def answer_connection(key, conn) -> str:
7545
+ """The AI Connection a run's entry [key] takes its key from: the one it
7546
+ names, or -- a run from before version 16 -- the key's own id."""
7547
+ cid = conn.get("connection") if isinstance(conn, dict) else None
7548
+ return cid if isinstance(cid, str) and cid else str(key).split(".", 1)[0]
7549
+
7550
+
7551
+ def run_answer(table, key):
7552
+ """What a run's table holds for [key] (evals-core.ts runAnswer): its own
7553
+ entry, or -- a run from before version 16, keyed by Target profile id --
7554
+ the connection's, where the key names no profile."""
7555
+ if key in table:
7556
+ return table[key]
7557
+ head, dot, tail = key.partition(".")
7558
+ return table.get(head) if dot and not tail else None
7559
+
7560
+
7561
+ # What an AI Connection keeps of a Target profile (evals-core.ts
7562
+ # AI_CONNECTION_FIELDS), beside the settings of its type that say how it is
7563
+ # reached (CONNECTION_SCOPED, the registry's `scope: "connection"`); every
7564
+ # other setting set is a Connection Profile's. Mirrored, and held to the
7565
+ # core's splitProfile, by proxy-check.py.
7566
+ AI_CONNECTION_FIELDS = ("id", "name", "slug", "type", "url", "key", "model", "profile")
7567
+ CONNECTION_SCOPED = {"http": ("auth", "headers", "testPath", "requestFormat")}
7568
+ # evals-core.ts REPLY_TOKENS_BEFORE, and the setting of each type that sent it.
7569
+ REPLY_TOKENS_BEFORE = "160"
7570
+ REPLY_TOKENS_SETTING = {"openai-compatible": "nPredict", "llama.cpp": "nPredict", "anthropic": "maxTokens"}
7571
+
7572
+
7573
+ def pin_reply_tokens(p):
7574
+ """evals-core.ts pinReplyTokens: a profile with no Reply tokens setting, of
7575
+ a type that sent the cap, has the cap it was sending written in. The split
7576
+ keeps no blank setting, so after it nothing could tell such a profile from
7577
+ one made since; it is pinned here, before."""
7578
+ key = REPLY_TOKENS_SETTING.get(str(p.get("type")))
7579
+ return p if not key or key in p else {**p, key: REPLY_TOKENS_BEFORE}
7580
+
7581
+
7582
+ def split_profile(p):
7583
+ """A Target profile as (AI Connection, Connection Profile or None): the
7584
+ connection keeps its id, key and address; a profile, stamped with the id
7585
+ it came from, holds every other setting it set (its reply-token cap
7586
+ pinned, pin_reply_tokens)."""
7587
+ p = pin_reply_tokens(p)
7588
+ scoped = CONNECTION_SCOPED.get(p.get("type"), ())
7589
+ conn, settings = {}, {}
7590
+ for k, v in p.items():
7591
+ if k in AI_CONNECTION_FIELDS or k in scoped:
7592
+ conn[k] = v
7593
+ elif v is not None and v != "":
7594
+ settings[k] = v
7595
+ if not settings:
7596
+ return conn, None
7597
+ pid = str(p.get("id") or "")
7598
+ return conn, {"id": pid, "name": str(p.get("name") or ""), "fromProfile": pid, **settings}
7599
+
7600
+
7601
+ def split_profiles(listed):
7602
+ """A profiles store's list as (connections, connection profiles)."""
7603
+ connections, profiles = [], []
7604
+ for p in listed if isinstance(listed, list) else []:
7605
+ if not isinstance(p, dict):
7606
+ continue
7607
+ conn, prof = split_profile(p)
7608
+ connections.append(conn)
7609
+ if prof is not None:
7610
+ profiles.append(prof)
7611
+ return connections, profiles
7141
7612
  # evals-core.ts's SLUG and SLUG_MAX: what a key's variable is spelt from.
7142
7613
  PROFILE_SLUG = re.compile(r"[a-z0-9]+(?:-[a-z0-9]+)*")
7143
7614
  SLUG_MAX = 32
@@ -7195,14 +7666,7 @@ def http_endpoint_test(payload, key):
7195
7666
  name, value = part.split(":", 1)
7196
7667
  if name.strip():
7197
7668
  headers[name.strip()] = value.strip()
7198
- if key:
7199
- how = str(settings.get("auth") or "bearer")
7200
- if how == "x-api-key":
7201
- headers["x-api-key"] = key
7202
- elif how == "api-key":
7203
- headers["api-key"] = key
7204
- elif how != "none":
7205
- headers["Authorization"] = "Bearer " + key
7669
+ headers.update(http_key_header(settings.get("auth"), key))
7206
7670
  if not all(header_safe(v) for v in headers.values()):
7207
7671
  return "", {}, "A header of this profile has characters that cannot be sent."
7208
7672
  return origin + (path if path.startswith("/") else "/" + path), headers, ""
@@ -7218,7 +7682,7 @@ def connection_problems(pid, conn, at, bad):
7218
7682
  if slug is not None and not (isinstance(slug, str) and PROFILE_SLUG.fullmatch(slug) and len(slug) <= SLUG_MAX):
7219
7683
  bad.append(f"{at}: slug has to be lowercase letters and digits, with hyphens between, at most {SLUG_MAX}")
7220
7684
  slug = None
7221
- var = key_var(pid, slug)
7685
+ var = key_var(answer_connection(pid, conn), slug)
7222
7686
  _fields(conn, at, CONNECTION_FIELDS, bad, var)
7223
7687
  ctype = conn.get("type")
7224
7688
  if not isinstance(ctype, str) or ctype not in CONNECTION_TYPES:
@@ -7364,17 +7828,23 @@ def run_problems(run):
7364
7828
  spelt = {}
7365
7829
  for pid, conn in table.items():
7366
7830
  at = f"profile {pid}"
7367
- if not PROFILE_ID.fullmatch(str(pid)):
7831
+ if not ANSWER_KEY.fullmatch(str(pid)):
7368
7832
  bad.append(f"{at}: an id is letters and digits, with at most one hyphen")
7369
7833
  continue
7370
7834
  if not isinstance(conn, dict):
7371
7835
  bad.append(f"{at} has to be an object")
7372
7836
  continue
7373
- # Two profiles spelling one variable would hand one the other's key.
7374
- var = key_var(pid, conn.get("slug") if isinstance(conn.get("slug"), str) else None)
7375
- if var in spelt:
7376
- bad.append(f"profiles {spelt[var]} and {pid} both take their key from ${var}")
7377
- spelt[var] = pid
7837
+ if conn.get("connection") is not None and not (
7838
+ isinstance(conn["connection"], str) and PROFILE_ID.fullmatch(conn["connection"])):
7839
+ bad.append(f"{at}: connection has to be an AI Connection's id")
7840
+ continue
7841
+ # Entries sharing a connection share its key; two connections
7842
+ # spelling one variable would hand one the other's.
7843
+ cid = answer_connection(pid, conn)
7844
+ var = key_var(cid, conn.get("slug") if isinstance(conn.get("slug"), str) else None)
7845
+ if var in spelt and spelt[var] != cid:
7846
+ bad.append(f"profiles {spelt[var]} and {cid} both take their key from ${var}")
7847
+ spelt[var] = cid
7378
7848
  connection_problems(pid, conn, at, bad)
7379
7849
  targets = run.get("targets")
7380
7850
  if not isinstance(targets, list) or not 1 <= len(targets) <= TARGET_CAP:
@@ -7394,10 +7864,17 @@ def run_problems(run):
7394
7864
  if not isinstance(steps, list) or len(steps) != len(jobs):
7395
7865
  bad.append(f"{at} has to have one step per job")
7396
7866
  continue
7397
- # A target with no profile is one whose steps ask none (Echo) or each
7398
- # name their own; which steps need one is the core's to judge.
7399
- refs = ([t["profile"]] if t.get("profile") is not None else []) + [
7400
- st.get("profile") for st in steps if isinstance(st, dict) and st.get("profile") is not None]
7867
+ # A target with no connection is one whose steps ask none (Echo) or
7868
+ # each name their own; which steps need one is the core's to judge.
7869
+ # Each pair named -- the target's, a step's own -- is one the run
7870
+ # carries (evals-core.ts targetAnswerOf).
7871
+ def pair(o, base=None):
7872
+ c = o.get("connection") if o.get("connection") is not None else (base or {}).get("connection")
7873
+ p = o.get("profile")
7874
+ return c, p
7875
+ refs = [pair(t)] if t.get("connection") is not None or t.get("profile") is not None else []
7876
+ refs += [pair(st, t) for st in steps
7877
+ if isinstance(st, dict) and (st.get("connection") is not None or st.get("profile") is not None)]
7401
7878
  # Whether the words may be blank -- Echo answering from the item --
7402
7879
  # is the core's to judge, and the worker refuses the run in its words.
7403
7880
  for k, step in enumerate(steps):
@@ -7408,9 +7885,11 @@ def run_problems(run):
7408
7885
  elif step.get("from") is not None and not (
7409
7886
  isinstance(step["from"], dict) and isinstance(step["from"].get("id"), str)):
7410
7887
  bad.append(f"{at}, job {k + 1} names the prompt it was picked from without an id")
7411
- for ref in refs:
7412
- if not isinstance(ref, dict) or ref.get("id") not in table:
7413
- bad.append(f"{at} names a Target profile the run does not carry")
7888
+ for c, p in refs:
7889
+ if (not isinstance(c, dict) or not isinstance(c.get("id"), str)
7890
+ or (p is not None and not (isinstance(p, dict) and isinstance(p.get("id"), str)))
7891
+ or run_answer(table, answer_key(c["id"], p["id"] if p else None)) is None):
7892
+ bad.append(f"{at} names a connection the run does not carry")
7414
7893
  content = content_of(run)
7415
7894
  kind = content.get("type") if isinstance(content, dict) else None
7416
7895
  if kind == "source":
@@ -7747,7 +8226,7 @@ class Handler(BaseHTTPRequestHandler):
7747
8226
  return self._send(404, b"not found", "text/plain")
7748
8227
  return self._json(200, {"plugins": PLUGINS.list(), "cap": PLUGIN_CAP})
7749
8228
  if path == "/api/connections/shares":
7750
- # Which workspaces may use each Target profile and Account
8229
+ # Which workspaces may use each AI Connection and Account
7751
8230
  # (docs/workspaces.md): the share sets the Connections menus read.
7752
8231
  # Keyed by "<kind>:<id>", workspace ids and the sentinels, no key.
7753
8232
  if STORE is None:
@@ -7773,6 +8252,16 @@ class Handler(BaseHTTPRequestHandler):
7773
8252
  return self._send(404, b"not found", "text/plain")
7774
8253
  ctype = TYPES[".json"] if f.suffix in (".json", ".map") else TYPES[".js"]
7775
8254
  return self._send(200, f.read_bytes(), ctype, (("Cache-Control", "no-store"),))
8255
+ if path == "/api/lab-export":
8256
+ # The lab's settings as a file (docs/lab-export.md): every
8257
+ # workspace's entry, or ?workspace=<slug>'s alone. No key.
8258
+ if STORE is None or WORKSPACES is None:
8259
+ return self._send(404, b"not found", "text/plain")
8260
+ only = (urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query).get("workspace") or [None])[0]
8261
+ out = lab_export(only)
8262
+ if out is None:
8263
+ return self._json(404, {"error": f"no workspace at /w/{only}"})
8264
+ return self._json(200, out)
7776
8265
  if path == "/api/packs":
7777
8266
  if PACKS is None:
7778
8267
  return self._send(404, b"not found", "text/plain")
@@ -8125,6 +8614,22 @@ class Handler(BaseHTTPRequestHandler):
8125
8614
  return self._filter_sets_post(path)
8126
8615
  if path == "/api/copy-move" or path.startswith("/api/copy-move/"):
8127
8616
  return self._copy_move_post(path)
8617
+ if path == "/api/lab-import":
8618
+ # A lab file into this lab (docs/lab-export.md), adding what it
8619
+ # lacks. `keys` is what `evals-lab import` read from its own
8620
+ # $EVALSLAB_API_KEY_<SLUG> variables; the page sends none.
8621
+ if STORE is None or WORKSPACES is None:
8622
+ return self._send(404, b"not found", "text/plain")
8623
+ payload = self._payload()
8624
+ if payload is None:
8625
+ return self._json(400, {"error": "an import is {lab, keys?} as JSON"})
8626
+ keys = payload.get("keys") if isinstance(payload.get("keys"), dict) else {}
8627
+ keys = {k: v for k, v in keys.items()
8628
+ if isinstance(k, str) and k.startswith("EVALSLAB_API_KEY_") and isinstance(v, str)}
8629
+ said, err = lab_import(payload.get("lab"), keys)
8630
+ if err:
8631
+ return self._json(err[0], {"error": err[1]})
8632
+ return self._json(200, {"said": said})
8128
8633
  if path == "/api/packs" or path.startswith("/api/packs/"):
8129
8634
  return self._packs_post(path)
8130
8635
  if path == "/api/workspaces" or path.startswith("/api/workspaces/"):
@@ -8283,8 +8788,8 @@ class Handler(BaseHTTPRequestHandler):
8283
8788
  else None. Only a held key (KEY_HELD<id>) names a stored profile; a key
8284
8789
  typed into the Setup form is a connection not yet saved, so nothing
8285
8790
  gates it. This is the relay/Test seam, beside worker_destinations' run
8286
- seam -- the two places a connection resolves to a key, so the two
8287
- places sharing is enforced. The caller writes the 403: a helper that
8791
+ seam and connection_auth's API Source seam -- the places a connection
8792
+ resolves to a key, so the places sharing is enforced. The caller writes the 403: a helper that
8288
8793
  answers here would send the body and still fall through to the proxy."""
8289
8794
  if STORE is None:
8290
8795
  return None
@@ -8480,7 +8985,7 @@ class Handler(BaseHTTPRequestHandler):
8480
8985
  if revs[name] is None:
8481
8986
  return self._json(400, {"error": f"{name!r} is not in that Source"})
8482
8987
  content["revs"] = revs
8483
- # A profile not shared with this workspace blocks the run with the
8988
+ # A connection not shared with this workspace blocks the run with the
8484
8989
  # sentence the Run bar shows and the connections it names, so its Share
8485
8990
  # link can grant them at once (docs/workspaces.md).
8486
8991
  ws = workspace_of(STORE) if STORE is not None else None
@@ -8551,6 +9056,8 @@ class Handler(BaseHTTPRequestHandler):
8551
9056
  if produce is None:
8552
9057
  return None, None, (400, "that provider cannot be produced from on the server")
8553
9058
  result = produce(cfg)
9059
+ if result.get("refused"):
9060
+ return None, None, result["refused"]
8554
9061
  content = result.get("content")
8555
9062
  if content is None:
8556
9063
  return None, None, (502, "the live fetch produced nothing: " + (result.get("error") or "unknown"))
@@ -9001,6 +9508,24 @@ class Handler(BaseHTTPRequestHandler):
9001
9508
  code, message = err
9002
9509
  return self._json(code, {"error": message})
9003
9510
  return self._json(200, {"trash": token})
9511
+ if len(parts) == 6 and parts[4] == "calls" and parts[5] in ("edit", "clone"):
9512
+ payload = self._payload() or {}
9513
+ name = source_name(payload.get("name"))
9514
+ if name is None:
9515
+ return self._json(400, {"error": "a call name is not accepted here"})
9516
+ if parts[5] == "clone":
9517
+ call, err = SOURCES.clone_call(parts[3], name)
9518
+ else:
9519
+ to = payload.get("to")
9520
+ if to is not None and not isinstance(to, str):
9521
+ return self._json(400, {"error": "that name is not accepted here", "field": "name"})
9522
+ sid = parts[3]
9523
+ call, err = SOURCES.edit_call(
9524
+ sid, name, to, payload.get("content"), payload.get("origin"),
9525
+ named_by=(lambda item: DATASETS.naming_item(sid, item)) if DATASETS else None)
9526
+ if err:
9527
+ return self._json(err[0], {"error": err[1], **({"field": err[2]} if len(err) > 2 else {})})
9528
+ return self._json(201 if parts[5] == "clone" else 200, {"call": call})
9004
9529
  if len(parts) == 5 and parts[4] == "copy":
9005
9530
  payload = self._payload() or {}
9006
9531
  source, err = SOURCES.copy_files(parts[3], payload.get("from"),
@@ -9150,6 +9675,252 @@ class Handler(BaseHTTPRequestHandler):
9150
9675
  return self._json(502, {"error": f"{url}: {e.reason}"})
9151
9676
 
9152
9677
 
9678
+ # ---- Lab export and import (docs/lab-export.md) -----------------------------
9679
+ #
9680
+ # A lab's settings as one file: its AI Connections, Connection Profiles and
9681
+ # grader, and its workspaces, so a new lab can start as another one is set up
9682
+ # -- by Setup › Workspaces › Data, by `evals-lab export`/`import`, or on every
9683
+ # start with LAB_IMPORT (a demo lab that keeps no data). An import only adds:
9684
+ # what the lab already has, by a connection's or profile's id and a
9685
+ # workspace's address, is left as it is, so importing twice changes nothing
9686
+ # and a lab that keeps its data keeps its edits. The file never holds a key
9687
+ # (#258): each comes from $EVALSLAB_API_KEY_<SLUG>, as a CI run's does, and a
9688
+ # key that comes is the connection's from then on, a rotated one included.
9689
+ # A connection carries its share set (`shares`, workspace slugs and the
9690
+ # sentinels; absent is All) from version 2: version 1 carried none, so an
9691
+ # import reads its connections as All.
9692
+
9693
+ LAB_FORMAT = "evals-lab/lab"
9694
+ LAB_VERSIONS = (1, 2)
9695
+ LAB_FIELDS = ("format", "version", "exportedAt", "connections", "connectionProfiles", "grader", "workspaces")
9696
+
9697
+
9698
+ def lab_export(only=None) -> dict:
9699
+ """The lab's settings as a file, every workspace's entry or the one at
9700
+ /w/[only]; None when no workspace is at that address. One workspace's file
9701
+ carries only what that workspace can use: the connections shared with it,
9702
+ the profiles over them (or over none) and the grader if it is one of them,
9703
+ each share set narrowed to that workspace, the one the file names, and
9704
+ *new kept where the connection has it. A connection shared only with
9705
+ workspaces in the trash is usable nowhere, and an empty share set would
9706
+ import as All, so it is left out, and out as the grader too."""
9707
+ body = (STORE.all().get("promptlab.connections") or {}).get("body") or {}
9708
+ profiles = (STORE.all().get("promptlab.connectionProfiles") or {}).get("body") or {}
9709
+ every = WORKSPACES.list()
9710
+ spaces = [w for w in every if not only or w["slug"] == only]
9711
+ if only and not spaces:
9712
+ return None
9713
+ slug_of = {w["id"]: w["slug"] for w in (spaces if only else every)}
9714
+ rows = STORE.shares()
9715
+ conns, nowhere = [], set()
9716
+ for c in body.get("list") or []:
9717
+ if not isinstance(c, dict) or (only and not STORE.shared("profile", str(c.get("id") or ""), spaces[0]["id"])):
9718
+ continue
9719
+ entry = {k: v for k, v in c.items() if k not in ("key", "shares")}
9720
+ held = rows.get(f"profile:{c.get('id')}")
9721
+ if held and SHARE_ALL not in held:
9722
+ entry["shares"] = [w if w == SHARE_NEW else slug_of[w] for w in held if w == SHARE_NEW or w in slug_of]
9723
+ if not entry["shares"]:
9724
+ nowhere.add(c.get("id"))
9725
+ continue
9726
+ conns.append(entry)
9727
+ kept = {c["id"] for c in conns}
9728
+
9729
+ def over_kept(p):
9730
+ ref = p.get("connection")
9731
+ return not ref or not only or (isinstance(ref, dict) and ref.get("id") in kept)
9732
+ out = {"format": LAB_FORMAT, "version": LAB_VERSIONS[-1],
9733
+ "exportedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
9734
+ "connections": conns,
9735
+ "connectionProfiles": [p for p in profiles.get("list") or [] if isinstance(p, dict) and over_kept(p)],
9736
+ "workspaces": [{"slug": w["slug"], "name": w["name"], **({"archived": True} if w["archived"] else {})}
9737
+ for w in spaces]}
9738
+ if (isinstance(body.get("grader"), str) and body["grader"] and body["grader"] not in nowhere
9739
+ and (not only or body["grader"] in kept)):
9740
+ out["grader"] = body["grader"]
9741
+ return out
9742
+
9743
+
9744
+ def _keyless(entry, what):
9745
+ """A sentence when [entry] holds a key, which a lab file never does."""
9746
+ for k, v in entry.items():
9747
+ if LOOKS_LIKE_A_KEY.match(k) and v not in ("", None):
9748
+ return f"{what} holds a {k}; a lab file holds no key -- set ${key_var(entry['id'], entry.get('slug'))}"
9749
+ return None
9750
+
9751
+
9752
+ def lab_problems(doc) -> str:
9753
+ """Why [doc] is not a lab file this lab imports, or None. Checked whole
9754
+ before anything is written, as a pack is."""
9755
+ if not isinstance(doc, dict) or doc.get("format") != LAB_FORMAT:
9756
+ return f"not a lab file: its format is not {LAB_FORMAT}"
9757
+ if doc.get("version") not in LAB_VERSIONS:
9758
+ return f"a lab file of version {doc.get('version')!r}; this lab reads {', '.join(map(str, LAB_VERSIONS))}"
9759
+ extra = sorted(set(doc) - set(LAB_FIELDS))
9760
+ if extra:
9761
+ return f"a lab file holds no {', '.join(extra)}"
9762
+ seen = set()
9763
+ for i, c in enumerate(doc.get("connections") or []):
9764
+ what = f"connection {i + 1}"
9765
+ if not isinstance(c, dict) or not isinstance(c.get("id"), str) or not PROFILE_ID.fullmatch(c["id"]):
9766
+ return f"{what} has no id of letters and digits"
9767
+ what = f"connection {c['id']}"
9768
+ if c["id"] in seen:
9769
+ return f"{what} is listed twice"
9770
+ seen.add(c["id"])
9771
+ if not isinstance(c.get("name"), str) or not c["name"].strip():
9772
+ return f"{what} has no name"
9773
+ if c.get("type") not in CONNECTION_TYPES:
9774
+ return f"{what} is of type {c.get('type')!r}, which this lab has no connection type for"
9775
+ bad = next((k for k, v in c.items() if not isinstance(v, str) and k != "shares"), None)
9776
+ if "shares" in c and doc["version"] < 2:
9777
+ bad = "shares"
9778
+ if bad:
9779
+ return f"{what}'s {bad} is not text"
9780
+ why = _keyless(c, what)
9781
+ if why:
9782
+ return why
9783
+ seen = set()
9784
+ for i, p in enumerate(doc.get("connectionProfiles") or []):
9785
+ what = f"connection profile {i + 1}"
9786
+ if not isinstance(p, dict) or not isinstance(p.get("id"), str) or not PROFILE_ID.fullmatch(p["id"]):
9787
+ return f"{what} has no id of letters and digits"
9788
+ what = f"connection profile {p['id']}"
9789
+ if p["id"] in seen:
9790
+ return f"{what} is listed twice"
9791
+ seen.add(p["id"])
9792
+ if not isinstance(p.get("name"), str) or not p["name"].strip():
9793
+ return f"{what} has no name"
9794
+ why = _keyless(p, what)
9795
+ if why:
9796
+ return why
9797
+ if "grader" in doc and not isinstance(doc["grader"], str):
9798
+ return "the grader is not a connection's id"
9799
+ seen = set()
9800
+ for i, w in enumerate(doc.get("workspaces") or []):
9801
+ what = f"workspace {i + 1}"
9802
+ if not isinstance(w, dict) or not isinstance(w.get("slug"), str) or not w["slug"]:
9803
+ return f"{what} has no address"
9804
+ if w["slug"] in seen:
9805
+ return f"workspace {w['slug']} is listed twice"
9806
+ seen.add(w["slug"])
9807
+ if not isinstance(w.get("name"), str) or not w["name"].strip():
9808
+ return f"workspace {w['slug']} has no name"
9809
+ if Workspaces._slugify(w["slug"]) != w["slug"]:
9810
+ return f"workspace {w['slug']}'s address is not one a lab makes: it would be {Workspaces._slugify(w['slug'])}"
9811
+ fields = {"slug", "name"} | ({"archived"} if doc["version"] >= 2 else set())
9812
+ if set(w) - fields:
9813
+ return f"workspace {w['slug']} holds no {', '.join(sorted(set(w) - fields))}"
9814
+ if "archived" in w and not isinstance(w["archived"], bool):
9815
+ return f"workspace {w['slug']}'s archived is not true or false"
9816
+ for k in ("connections", "connectionProfiles", "workspaces"):
9817
+ if k in doc and not isinstance(doc[k], list):
9818
+ return f"a lab file's {k} is a list"
9819
+ # Workspaces before shares: a share names the file's workspaces or the lab's.
9820
+ known = seen | ({w["slug"] for w in WORKSPACES.list()} if WORKSPACES is not None else set())
9821
+ for c in doc.get("connections") or []:
9822
+ if "shares" not in c:
9823
+ continue
9824
+ if not isinstance(c["shares"], list) or not all(isinstance(s, str) for s in c["shares"]):
9825
+ return f"connection {c['id']}'s shares is a list of workspace addresses"
9826
+ # No rows is All in the store, so an empty list would grant everywhere.
9827
+ if not c["shares"]:
9828
+ return (f"connection {c['id']}'s shares is empty; leave shares out for All, "
9829
+ "or name a workspace or *new")
9830
+ lost = [s for s in c["shares"] if s not in (SHARE_ALL, SHARE_NEW) and s not in known]
9831
+ if lost:
9832
+ return f"connection {c['id']} is shared with {', '.join(lost)}, which neither the file nor this lab has"
9833
+ return None
9834
+
9835
+
9836
+ def lab_import(doc, keys) -> tuple:
9837
+ """[doc] into this lab, adding what it lacks; [keys] is where each
9838
+ connection's key is looked up by its variable's name ($EVALSLAB_API_KEY_
9839
+ <SLUG>) -- the server's environment as it starts, or what `evals-lab
9840
+ import` sent. Returns (lines saying what happened, None) or (None, error).
9841
+ No line holds a key."""
9842
+ why = lab_problems(doc)
9843
+ if why:
9844
+ return None, (400, why)
9845
+ said = []
9846
+ # A file's address to this lab's workspace id; one added where the trash
9847
+ # holds its address lands at the next free one, and its shares follow it.
9848
+ wid_of = {w["slug"]: w["id"] for w in WORKSPACES.list()}
9849
+ for w in doc.get("workspaces") or []:
9850
+ if w["slug"] in wid_of:
9851
+ continue
9852
+ wid, err = WORKSPACES.create(w["name"], w["slug"])
9853
+ wid_of[w["slug"]] = wid
9854
+ if not err and w.get("archived"):
9855
+ _, err = WORKSPACES.archive(wid)
9856
+ if err:
9857
+ return None, err
9858
+ said.append(f"workspace {w['name']} added at /w/{WORKSPACES.get(wid)['slug']}"
9859
+ + (", archived" if w.get("archived") else ""))
9860
+ for _ in range(3):
9861
+ docs = STORE.all()
9862
+ cur = docs.get("promptlab.connections") or {"version": 0, "body": None}
9863
+ pcur = docs.get("promptlab.connectionProfiles") or {"version": 0, "body": None}
9864
+ body = json.loads(json.dumps(cur["body"] or {"list": [], "at": ""}))
9865
+ pbody = json.loads(json.dumps(pcur["body"] or {"list": []}))
9866
+ conns = body.setdefault("list", [])
9867
+ lines, changed, pchanged, added = [], False, False, []
9868
+ for c in doc.get("connections") or []:
9869
+ mine = next((x for x in conns if isinstance(x, dict) and x.get("id") == c["id"]), None)
9870
+ if mine is None:
9871
+ mine = {**{k: v for k, v in c.items() if k != "shares"}, "key": ""}
9872
+ added.append(c)
9873
+ conns.append(mine)
9874
+ changed = True
9875
+ lines.append(f"connection {c['name']} added")
9876
+ key = str(keys.get(key_var(c["id"], c.get("slug"))) or "").strip()
9877
+ if key and mine.get("key") != key:
9878
+ if not header_safe(key):
9879
+ return None, (400, f"${key_var(c['id'], c.get('slug'))} cannot go in a header")
9880
+ mine["key"] = key
9881
+ changed = True
9882
+ lines.append(f"connection {mine.get('name') or c['name']} takes its key from "
9883
+ f"${key_var(c['id'], c.get('slug'))}")
9884
+ profs = pbody.setdefault("list", [])
9885
+ for p in doc.get("connectionProfiles") or []:
9886
+ if not any(isinstance(x, dict) and x.get("id") == p["id"] for x in profs):
9887
+ profs.append(dict(p))
9888
+ pchanged = True
9889
+ lines.append(f"connection profile {p['name']} added")
9890
+ if doc.get("grader") and not body.get("grader"):
9891
+ body["grader"] = doc["grader"]
9892
+ changed = True
9893
+ lines.append(f"grader set to {doc['grader']}")
9894
+ # Only an added connection takes the file's share set: one the lab had
9895
+ # keeps its own, as everything else it had does. Set before the write,
9896
+ # so the connection is never there and shared with All in between; a
9897
+ # failed write puts back what the rows were, since its ids never landed.
9898
+ before = STORE.shares()
9899
+ for c in added:
9900
+ if "shares" in c:
9901
+ STORE.set_shares("profile", c["id"], [s if s in (SHARE_ALL, SHARE_NEW) else wid_of[s]
9902
+ for s in c["shares"]])
9903
+ writes = {}
9904
+ if changed:
9905
+ body["at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
9906
+ writes["promptlab.connections"] = {"version": cur["version"], "body": body}
9907
+ if pchanged:
9908
+ writes["promptlab.connectionProfiles"] = {"version": pcur["version"], "body": pbody}
9909
+ if not writes:
9910
+ break
9911
+ # Global documents, so the default workspace's write reaches them.
9912
+ ok, _ = STORE.write(writes, STORE.default_ws)
9913
+ if ok:
9914
+ break
9915
+ for c in added:
9916
+ if "shares" in c:
9917
+ STORE.set_shares("profile", c["id"], before.get(f"profile:{c['id']}") or [])
9918
+ else:
9919
+ return None, (409, "the lab's connections kept changing while they were imported; import again")
9920
+ said += lines
9921
+ return said or ["nothing to add"], None
9922
+
9923
+
9153
9924
  def main():
9154
9925
  if not (WEB_DIST / "index.html").is_file():
9155
9926
  raise SystemExit(f"no {WEB_DIST / 'index.html'} -- the page is built from web/: "
@@ -9161,6 +9932,11 @@ def main():
9161
9932
  # package, not one without the app: said at once, not at Sign in.
9162
9933
  if PUBLIC_GOOGLE_PROBLEM:
9163
9934
  raise SystemExit(PUBLIC_GOOGLE_PROBLEM)
9935
+ # A lab file named and not there is a broken deployment, said at once.
9936
+ if LAB_IMPORT and not Path(LAB_IMPORT).is_file():
9937
+ raise SystemExit(f"LAB_IMPORT names {LAB_IMPORT}, which is not a file")
9938
+ if LAB_IMPORT and STORE is None:
9939
+ raise SystemExit("LAB_IMPORT needs DATA_DIR: a lab file's settings are kept in its store")
9164
9940
  # A root that is not there would refuse every File Source -- said at once.
9165
9941
  if LOCAL_ROOT and not Path(LOCAL_ROOT).is_dir():
9166
9942
  raise SystemExit(f"SOURCE_LOCAL_ROOT names {LOCAL_ROOT}, which is not a directory")
@@ -9229,6 +10005,24 @@ def main():
9229
10005
  # The drop-in folder: a pack put there is installed as the lab starts.
9230
10006
  for line in PACKS.install_dropins(Path(DATA_DIR) / "packs"):
9231
10007
  print(f"pack: {line}", flush=True)
10008
+ if LAB_IMPORT:
10009
+ # After the demo: a lab file adds settings, never datasets, so the
10010
+ # demo still finds the new lab it installs into.
10011
+ try:
10012
+ doc = json.loads(Path(LAB_IMPORT).read_text("utf-8"))
10013
+ except (OSError, ValueError) as e:
10014
+ raise SystemExit(f"LAB_IMPORT: {LAB_IMPORT} is not JSON ({e})") from None
10015
+ said, err = lab_import(doc, os.environ)
10016
+ if err:
10017
+ raise SystemExit(f"LAB_IMPORT: {err[1]}")
10018
+ for line in said:
10019
+ print(f"import: {line}", flush=True)
10020
+ # A held key goes wherever a submitted run points it, so a lab open to
10021
+ # its network with keys in it is as loud as one with a Local provider on.
10022
+ if local_exposed() and any(os.environ.get(key_var(c["id"], c.get("slug")))
10023
+ for c in doc.get("connections") or []):
10024
+ print(f"WARNING: LAB_IMPORT gave a connection its key, and {HOST} is beyond loopback with no "
10025
+ "LAB_PASSWORD -- anyone who can reach this port can run against it", flush=True)
9232
10026
  if QUEUE is not None:
9233
10027
  QUEUE.start()
9234
10028
  ThreadingHTTPServer((HOST, PORT), Handler).serve_forever()