evals-lab 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +88 -1
- package/README.md +12 -6
- package/bin/evals-lab.js +9 -1
- package/bin/lab-file.js +97 -0
- package/bin/run.js +16 -16
- package/lab/VERSION +1 -1
- package/lab/demo/CREDITS.md +37 -3
- package/lab/demo/datasets/demo-1.json +2 -2
- package/lab/demo/datasets/demo-2.json +2 -2
- package/lab/demo/datasets/sentiment.json +383 -0
- package/lab/demo/datasets/weather-alerts.json +61 -0
- package/lab/demo/datasets/wordings.json +39 -0
- package/lab/demo/manifest.json +3 -3
- package/lab/demo/pipelines/demo-1.json +8 -6
- package/lab/demo/pipelines/demo-2.json +8 -6
- package/lab/demo/pipelines/sentiment.json +65 -0
- package/lab/demo/pipelines/weather-alerts.json +66 -0
- package/lab/demo/pipelines/wordings.json +104 -0
- package/lab/demo/sources/Demo 1/source.json +3 -0
- package/lab/demo/sources/Demo 2/source.json +3 -0
- package/lab/demo/sources/sentiment/bakery-croissant.txt +1 -0
- package/lab/demo/sources/sentiment/bus-timetable.txt +1 -0
- package/lab/demo/sources/sentiment/cafe-coffee.txt +1 -0
- package/lab/demo/sources/sentiment/concert-night.txt +1 -0
- package/lab/demo/sources/sentiment/film-sequel.txt +1 -0
- package/lab/demo/sources/sentiment/garden-tomatoes.txt +1 -0
- package/lab/demo/sources/sentiment/gym-contract.txt +1 -0
- package/lab/demo/sources/sentiment/headphones-review.txt +1 -0
- package/lab/demo/sources/sentiment/hotel-room.txt +1 -0
- package/lab/demo/sources/sentiment/library-hours.txt +1 -0
- package/lab/demo/sources/sentiment/meeting-minutes.txt +1 -0
- package/lab/demo/sources/sentiment/parcel-lost.txt +1 -0
- package/lab/demo/sources/sentiment/phone-update.txt +1 -0
- package/lab/demo/sources/sentiment/puppy-home.txt +1 -0
- package/lab/demo/sources/sentiment/recipe-note.txt +1 -0
- package/lab/demo/sources/sentiment/road-closure.txt +1 -0
- package/lab/demo/sources/sentiment/source.json +3 -0
- package/lab/demo/sources/sentiment/teacher-thanks.txt +1 -0
- package/lab/demo/sources/sentiment/train-delay.txt +1 -0
- package/lab/demo/sources/sentiment/volunteer-day.txt +1 -0
- package/lab/demo/sources/sentiment/weather-report.txt +1 -0
- package/lab/demo/sources/weather-alerts/https-api.weather.gov-alerts-types.txt +116 -0
- package/lab/demo/sources/weather-alerts/source.json +8 -0
- package/lab/evals-core.mjs +584 -167
- package/lab/flows/record.mjs +16 -4
- package/lab/kinds/list.mjs +32 -23
- package/lab/run-evals.js +3 -3
- package/lab/server.py +913 -119
- package/lab/web/dist/assets/gallery-CFntjQFG.js +5 -0
- package/lab/web/dist/assets/main-BlTG2i20.css +1 -0
- package/lab/web/dist/assets/main-D6MNWqak.js +17 -0
- package/lab/web/dist/assets/tokens-DwzzdSIF.css +1 -0
- package/lab/web/dist/assets/tokens-bvuNFuYg.js +60 -0
- package/lab/web/dist/gallery.html +3 -3
- package/lab/web/dist/index.html +4 -4
- package/package.json +10 -2
- package/lab/web/dist/assets/gallery-BcvmOPqb.js +0 -3
- package/lab/web/dist/assets/main-BBUxHZ-D.js +0 -16
- package/lab/web/dist/assets/main-CMoyjImZ.css +0 -1
- package/lab/web/dist/assets/tokens-B7N4NvmM.css +0 -1
- package/lab/web/dist/assets/tokens-CiJkwD3V.js +0 -60
package/lab/server.py
CHANGED
|
@@ -42,6 +42,7 @@ import urllib.error
|
|
|
42
42
|
import urllib.parse
|
|
43
43
|
import urllib.request
|
|
44
44
|
import zipfile
|
|
45
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
45
46
|
from contextlib import closing
|
|
46
47
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
47
48
|
from pathlib import Path
|
|
@@ -53,6 +54,9 @@ OLLAMA = os.environ.get("OLLAMA_URL", "http://127.0.0.1:11434")
|
|
|
53
54
|
# as a lab run on a workstation always has been; the deployed one sets it,
|
|
54
55
|
# because it keeps setups and API keys and can spend GPU time.
|
|
55
56
|
PASSWORD = os.environ.get("LAB_PASSWORD") or None
|
|
57
|
+
# A lab file imported on every start (docs/lab-export.md): a demo lab that
|
|
58
|
+
# keeps no data starts set up, its keys from $EVALSLAB_API_KEY_<SLUG>.
|
|
59
|
+
LAB_IMPORT = os.environ.get("LAB_IMPORT") or None
|
|
56
60
|
# Where what follows the user between browsers is kept (#463). Unset means
|
|
57
61
|
# nothing is, and the page stays with its browser as it always has.
|
|
58
62
|
DATA_DIR = os.environ.get("DATA_DIR") or None
|
|
@@ -170,6 +174,21 @@ def auth_headers(ctype: str, key: str) -> dict:
|
|
|
170
174
|
return {"Authorization": "Bearer " + key}
|
|
171
175
|
|
|
172
176
|
|
|
177
|
+
def http_key_header(how, key: str) -> dict:
|
|
178
|
+
"""The header an HTTP endpoint connection sends its key in: its Key header
|
|
179
|
+
setting, kept flat on the connection (evals-core.ts's KEY_HEADERS)."""
|
|
180
|
+
if not key:
|
|
181
|
+
return {}
|
|
182
|
+
how = str(how or "bearer")
|
|
183
|
+
if how == "x-api-key":
|
|
184
|
+
return {"x-api-key": key}
|
|
185
|
+
if how == "api-key":
|
|
186
|
+
return {"api-key": key}
|
|
187
|
+
if how == "none":
|
|
188
|
+
return {}
|
|
189
|
+
return {"Authorization": "Bearer " + key}
|
|
190
|
+
|
|
191
|
+
|
|
173
192
|
def connection_url(base: str, ctype: str) -> str:
|
|
174
193
|
path = CONNECTION_CHAT_PATHS.get(ctype, "/chat/completions")
|
|
175
194
|
if ctype in ("ollama", "ollama-cloud"):
|
|
@@ -348,7 +367,8 @@ SOURCE_PROVIDERS = {
|
|
|
348
367
|
# The API call provider (docs/dynamic-sources.md § 2): HTTP GET plus a verb,
|
|
349
368
|
# headers and a body, keyed from a connection -- the same relay fetch, one
|
|
350
369
|
# layer up. `produce` and `take` are attached below, beside http-get's.
|
|
351
|
-
|
|
370
|
+
# `keyed`: it sends a connection's key, so a pack may not carry one (read_pack).
|
|
371
|
+
"api": {"label": "API call", "noun": "call", "tier": "pull", "flag": None, "keyed": True,
|
|
352
372
|
"uploads": dict(_CALL_UPLOADS), "longConfig": {"body": 8000}},
|
|
353
373
|
# Supplied has no `produce`: the caller hands items in (the CLI's --items, or
|
|
354
374
|
# an Upload… by hand), so the server fetches and executes nothing -- it only
|
|
@@ -565,7 +585,20 @@ def redact_flow(value):
|
|
|
565
585
|
# again, so a page that got it wrong cannot put a key or a malformed record on
|
|
566
586
|
# disk.
|
|
567
587
|
RECORD_FORMAT = "evals-lab/record"
|
|
568
|
-
RECORD_VERSION =
|
|
588
|
+
RECORD_VERSION = 2
|
|
589
|
+
# The origins a record names. Version 2 added `edited`: a call changed in the
|
|
590
|
+
# Call dialog, keeping the `run` and `iteration` it came from.
|
|
591
|
+
RECORD_ORIGINS = ("run", "hand", "upload", "edited")
|
|
592
|
+
|
|
593
|
+
|
|
594
|
+
def upgrade_record(value):
|
|
595
|
+
"""A version-1 record as today's: flows/record.ts's upgradeRecord. Version
|
|
596
|
+
2 only widened the origins, so the upgrade is the number. Anything else is
|
|
597
|
+
returned as it is, for record_problems to judge."""
|
|
598
|
+
if isinstance(value, dict) and value.get("format") == RECORD_FORMAT and value.get("version") == 1 \
|
|
599
|
+
and not isinstance(value.get("version"), bool):
|
|
600
|
+
return {**value, "version": RECORD_VERSION}
|
|
601
|
+
return value
|
|
569
602
|
|
|
570
603
|
|
|
571
604
|
def record_problems(value):
|
|
@@ -600,8 +633,8 @@ def record_problems(value):
|
|
|
600
633
|
or isinstance(res.get("error"), dict))):
|
|
601
634
|
bad.append("a record's result is { status, body }, { error } or null")
|
|
602
635
|
origin = value.get("origin")
|
|
603
|
-
if not isinstance(origin, dict) or origin.get("kind") not in
|
|
604
|
-
bad.append("a record's origin is run, hand or
|
|
636
|
+
if not isinstance(origin, dict) or origin.get("kind") not in RECORD_ORIGINS:
|
|
637
|
+
bad.append("a record's origin is run, hand, upload or edited")
|
|
605
638
|
return bad
|
|
606
639
|
|
|
607
640
|
|
|
@@ -612,6 +645,7 @@ def take_record(name, data):
|
|
|
612
645
|
value = json.loads(data.decode("utf-8"))
|
|
613
646
|
except (UnicodeDecodeError, ValueError):
|
|
614
647
|
return None, (400, f"{name} is not JSON")
|
|
648
|
+
value = upgrade_record(value)
|
|
615
649
|
bad = record_problems(value)
|
|
616
650
|
if bad:
|
|
617
651
|
return None, (400, f"{name}: {bad[0]}")
|
|
@@ -689,6 +723,12 @@ def take_dynamic(name, data):
|
|
|
689
723
|
return None, (400, f"{name} is not text")
|
|
690
724
|
if TOKEN_SHAPE.search(text):
|
|
691
725
|
return None, (400, f"{name} holds a token, and a token never reaches the server")
|
|
726
|
+
# The lab's own secrets have no shape to redact by, and a command can read
|
|
727
|
+
# them from its parent's environment, so they are matched by value -- a
|
|
728
|
+
# verbatim read only; an encoded one gets past. A value under 8 characters is too likely to
|
|
729
|
+
# be ordinary text to refuse a Call for.
|
|
730
|
+
if any(len(v) >= 8 and v in text for v in (os.environ.get(k) or "" for k in _LAB_SECRET_ENV)):
|
|
731
|
+
return None, (400, f"{name} holds one of the lab's own secrets")
|
|
692
732
|
for pattern, to in SECRET_TEXT:
|
|
693
733
|
text = pattern.sub(to, text)
|
|
694
734
|
return text.encode("utf-8"), None
|
|
@@ -750,31 +790,32 @@ def _header_pairs(spec):
|
|
|
750
790
|
def connection_auth(pid):
|
|
751
791
|
"""The one auth header a dynamic API Source borrows from the connection
|
|
752
792
|
[pid] it names, and the key it carries so the relay's https-or-loopback
|
|
753
|
-
rule can bind. The key is the
|
|
793
|
+
rule can bind. The key is the connections store's, by id, sent the way the
|
|
754
794
|
connection's type says (an HTTP endpoint's Key header, else the type's own,
|
|
755
795
|
auth_headers) -- it never leaves the server or reaches a Call (#258).
|
|
756
796
|
Returns (headers, key, None); ({}, "", None) for no connection or no key;
|
|
757
|
-
or (None, None, (code, why)).
|
|
797
|
+
or (None, None, (code, why)). This is the third seam a connection resolves
|
|
798
|
+
to a key, beside worker_destinations and the relay's _unshared, so it
|
|
799
|
+
refuses a connection not shared with the bound workspace the same way --
|
|
800
|
+
the request thread's for a snapshot, the submitting request's for a live
|
|
801
|
+
run (docs/workspaces.md)."""
|
|
758
802
|
if not pid:
|
|
759
803
|
return {}, "", None
|
|
760
|
-
doc = (STORE.all().get("promptlab.
|
|
804
|
+
doc = (STORE.all().get("promptlab.connections") or {}) if STORE is not None else {}
|
|
761
805
|
profiles = (doc.get("body") or {}).get("list") or []
|
|
762
806
|
profile = next((p for p in profiles if isinstance(p, dict) and p.get("id") == pid), None)
|
|
763
807
|
if profile is None:
|
|
764
808
|
return None, None, (400, "that Source's auth connection is not one the lab has")
|
|
809
|
+
ws = workspace_of(STORE)
|
|
810
|
+
if not STORE.shared("profile", pid, ws):
|
|
811
|
+
return None, None, (403, not_shared_sentence([profile.get("name") or pid], STORE.workspace_name(ws)))
|
|
765
812
|
key = str(profile.get("key") or "").strip()
|
|
766
813
|
if not key:
|
|
767
814
|
return {}, "", None
|
|
768
815
|
if not header_safe(key):
|
|
769
816
|
return None, None, (400, "that connection's key cannot be sent in a header")
|
|
770
817
|
ctype = profile.get("type")
|
|
771
|
-
if ctype == "http"
|
|
772
|
-
options = profile.get("options") if isinstance(profile.get("options"), dict) else {}
|
|
773
|
-
how = str(options.get("auth") or "bearer")
|
|
774
|
-
head = ({"x-api-key": key} if how == "x-api-key" else {"api-key": key} if how == "api-key"
|
|
775
|
-
else {} if how == "none" else {"Authorization": "Bearer " + key})
|
|
776
|
-
else:
|
|
777
|
-
head = auth_headers(ctype, key)
|
|
818
|
+
head = http_key_header(profile.get("auth"), key) if ctype == "http" else auth_headers(ctype, key)
|
|
778
819
|
return head, key, None
|
|
779
820
|
|
|
780
821
|
|
|
@@ -789,6 +830,10 @@ def produce_api(config):
|
|
|
789
830
|
method = (cfg.get("method").upper() if isinstance(cfg.get("method"), str) and cfg.get("method").strip()
|
|
790
831
|
else "GET")
|
|
791
832
|
auth, key, why = connection_auth(cfg.get("connection"))
|
|
833
|
+
if why and why[0] == 403:
|
|
834
|
+
# Refused, not failed: no request went out, so the caller answers the
|
|
835
|
+
# refusal rather than keeping a failed Call.
|
|
836
|
+
return {"error": why[1], "refused": why, "request": {"url": url, "method": method}, "status": None}
|
|
792
837
|
if why:
|
|
793
838
|
return {"error": why[1], "request": {"url": url, "method": method}, "status": None}
|
|
794
839
|
headers = {**_header_pairs(cfg.get("reqHeaders")), **auth}
|
|
@@ -858,7 +903,8 @@ def _outside_root(real):
|
|
|
858
903
|
root = Path(LOCAL_ROOT).resolve()
|
|
859
904
|
return real != root and root not in real.parents
|
|
860
905
|
|
|
861
|
-
# The lab's own secrets in its environment, never handed to a command it runs
|
|
906
|
+
# The lab's own secrets in its environment, never handed to a command it runs
|
|
907
|
+
# nor kept in a Call (take_dynamic).
|
|
862
908
|
_LAB_SECRET_ENV = ("LAB_PASSWORD", "GOOGLE_CLIENT_SECRET", "GOOGLE_API_KEY")
|
|
863
909
|
|
|
864
910
|
|
|
@@ -895,7 +941,8 @@ def command_config_problems(config):
|
|
|
895
941
|
def produce_file(config):
|
|
896
942
|
"""Read a File on disk Source's one Call now: the path its config names,
|
|
897
943
|
capped at the file cap. The lab's own data directory is never read -- it
|
|
898
|
-
holds the store, keys included
|
|
944
|
+
holds the store, keys included -- nor /proc, which holds the server's
|
|
945
|
+
environment. Returns the shape _relay_produce does."""
|
|
899
946
|
path = config.get("path") if isinstance(config, dict) else None
|
|
900
947
|
request = {"path": path}
|
|
901
948
|
if not isinstance(path, str) or not path.strip():
|
|
@@ -910,6 +957,10 @@ def produce_file(config):
|
|
|
910
957
|
for own in {Path(d).resolve() for d in (DATA_DIR, STORE and Path(STORE.path).parent) if d}:
|
|
911
958
|
if real == own or own in real.parents:
|
|
912
959
|
return {"error": "the lab's own data is not a file it reads", "request": request, "status": None}
|
|
960
|
+
# /proc holds the server's own environment (every secret it was started
|
|
961
|
+
# with, not only the lab's) and its memory: the process, not a file.
|
|
962
|
+
if real == Path("/proc") or Path("/proc") in real.parents:
|
|
963
|
+
return {"error": "the server's own process is not a file it reads", "request": request, "status": None}
|
|
913
964
|
if not real.is_file():
|
|
914
965
|
return {"error": "that path is not a file", "request": request, "status": None}
|
|
915
966
|
try:
|
|
@@ -1024,8 +1075,10 @@ SIGN_INS = {"microsoft": lambda: microsoft_config() is not None}
|
|
|
1024
1075
|
# in the store: a document is not migrated or deleted because nothing reads it.
|
|
1025
1076
|
# promptlab.cases, .rules and their .base copies went that way when a dataset
|
|
1026
1077
|
# became a row of its own (#86), and promptlab.mappings once a dataset named
|
|
1027
|
-
# the Source it grades (#199).
|
|
1028
|
-
|
|
1078
|
+
# the Source it grades (#199). promptlab.profiles went once a Target profile
|
|
1079
|
+
# was split into an AI Connection and a Connection Profile (#445), each kept
|
|
1080
|
+
# in a document of its own; its rows are what the split read.
|
|
1081
|
+
SYNCED = ("promptlab.workflows", "promptlab.connections", "promptlab.connectionProfiles",
|
|
1029
1082
|
"promptlab.versions", "promptlab.tokens")
|
|
1030
1083
|
MAX_DOC = 8 * 1024 * 1024
|
|
1031
1084
|
RUNS_PAGE = 25
|
|
@@ -1088,14 +1141,16 @@ QUEUE_WAIT = 1.0
|
|
|
1088
1141
|
WATCH_EVERY = 5.0
|
|
1089
1142
|
|
|
1090
1143
|
|
|
1091
|
-
#
|
|
1144
|
+
# An AI Connection's key is written from the page and never read back by it
|
|
1092
1145
|
# (#258): a client holding the lab's password -- CI's runners and their logs
|
|
1093
|
-
# among them -- is handed no key. Where a
|
|
1146
|
+
# among them -- is handed no key. Where a connection holds one, what is served
|
|
1094
1147
|
# (/api/state, the page's carried copy, a refused write's current copy) holds
|
|
1095
|
-
# KEY_HELD and the
|
|
1096
|
-
# keeps the key the store holds for the id: the
|
|
1097
|
-
# was cloned or restored from. KEY_HELD starts with a character no key can
|
|
1098
|
-
# (header_safe), so a pasted key is never taken for it.
|
|
1148
|
+
# KEY_HELD and the connection's id instead, and a write that brings that back
|
|
1149
|
+
# keeps the key the store holds for the id: the connection's own, or the one
|
|
1150
|
+
# it was cloned or restored from. KEY_HELD starts with a character no key can
|
|
1151
|
+
# (header_safe), so a pasted key is never taken for it. A Target profile held
|
|
1152
|
+
# the key before the split (#445); a connection keeps its id, so a KEY_HELD
|
|
1153
|
+
# the page carried then still names it.
|
|
1099
1154
|
KEY_HELD = "\u2022held:"
|
|
1100
1155
|
|
|
1101
1156
|
|
|
@@ -1104,11 +1159,12 @@ def held(key) -> bool:
|
|
|
1104
1159
|
|
|
1105
1160
|
|
|
1106
1161
|
def profiles_in(name: str, body):
|
|
1107
|
-
"""
|
|
1108
|
-
each profile version's copy
|
|
1162
|
+
"""Everything keyed a synced document holds: the connections store's
|
|
1163
|
+
list, and each profile version's copy -- a Target profile's settings and
|
|
1164
|
+
key, under its connection's id."""
|
|
1109
1165
|
if not isinstance(body, dict):
|
|
1110
1166
|
return
|
|
1111
|
-
if name == "promptlab.
|
|
1167
|
+
if name == "promptlab.connections":
|
|
1112
1168
|
for p in body.get("list") or []:
|
|
1113
1169
|
if isinstance(p, dict):
|
|
1114
1170
|
yield p
|
|
@@ -1165,7 +1221,9 @@ SHARE_NEW = "*new"
|
|
|
1165
1221
|
# its pipeline/dataset slices per-workspace, its profile slice global, because
|
|
1166
1222
|
# profiles are global so their Restore must be too. served()/write() route
|
|
1167
1223
|
# each key by these; any other key is per-workspace.
|
|
1168
|
-
|
|
1224
|
+
# promptlab.profiles stays global though no longer served: its row is what
|
|
1225
|
+
# the split on open reads (split_profiles).
|
|
1226
|
+
DOC_GLOBAL = ("promptlab.connections", "promptlab.connectionProfiles", "promptlab.profiles", "promptlab.tokens")
|
|
1169
1227
|
DOC_SPLIT = "promptlab.versions"
|
|
1170
1228
|
|
|
1171
1229
|
|
|
@@ -1232,6 +1290,30 @@ class Store:
|
|
|
1232
1290
|
db.execute("UPDATE runs SET workspace = ? WHERE workspace IS NULL", (self.default_ws,))
|
|
1233
1291
|
if cols and "workspace" not in cols:
|
|
1234
1292
|
self._rebuild_docs(db)
|
|
1293
|
+
self._split_profiles(db)
|
|
1294
|
+
|
|
1295
|
+
def _split_profiles(self, db):
|
|
1296
|
+
"""A store from before the split (#445) opened once: its Target
|
|
1297
|
+
profiles become AI Connections, each keeping the profile's id, its key
|
|
1298
|
+
and its Setup choices (`at`, the lab's grader), and -- where a profile
|
|
1299
|
+
set settings of its own -- Connection Profiles (split_profiles). The
|
|
1300
|
+
old document stays, unserved. Idempotent: a store holding the
|
|
1301
|
+
connections document already, even an emptied one, is left as it is;
|
|
1302
|
+
a store with no profiles has nothing to split."""
|
|
1303
|
+
if db.execute("SELECT 1 FROM docs WHERE name = 'promptlab.connections' AND workspace IS NULL").fetchone():
|
|
1304
|
+
return
|
|
1305
|
+
row = db.execute("SELECT body FROM docs WHERE name = 'promptlab.profiles' AND workspace IS NULL").fetchone()
|
|
1306
|
+
try:
|
|
1307
|
+
body = json.loads(row[0]) if row and row[0] else None
|
|
1308
|
+
except ValueError:
|
|
1309
|
+
body = None
|
|
1310
|
+
if not isinstance(body, dict):
|
|
1311
|
+
return
|
|
1312
|
+
connections, profiles = split_profiles(body.get("list"))
|
|
1313
|
+
kept = {k: body[k] for k in ("at", "grader") if k in body}
|
|
1314
|
+
at = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
1315
|
+
self._put(db, "promptlab.connections", None, 1, {"list": connections, "at": "", **kept}, at)
|
|
1316
|
+
self._put(db, "promptlab.connectionProfiles", None, 1, {"list": profiles}, at)
|
|
1235
1317
|
|
|
1236
1318
|
def _ensure_default(self, db) -> str:
|
|
1237
1319
|
"""The flagged default workspace's id, seeding "Default" (/w/default,
|
|
@@ -1363,8 +1445,8 @@ class Store:
|
|
|
1363
1445
|
(name, ws, version, text, at))
|
|
1364
1446
|
|
|
1365
1447
|
def _keyring(self, db, now: dict) -> dict:
|
|
1366
|
-
"""Each
|
|
1367
|
-
dropped one's, else its newest version's."""
|
|
1448
|
+
"""Each connection id's key as the store holds it: its connection's,
|
|
1449
|
+
else a dropped one's, else its newest version's."""
|
|
1368
1450
|
ring = {}
|
|
1369
1451
|
for p in profiles_in("promptlab.versions", now.get("promptlab.versions", {}).get("body")):
|
|
1370
1452
|
pid, key = str(p.get("id") or ""), p.get("key")
|
|
@@ -1372,7 +1454,7 @@ class Store:
|
|
|
1372
1454
|
ring[pid] = key
|
|
1373
1455
|
db.execute("DELETE FROM dropped_keys WHERE at < ?", (time.time() - TRASH_SECONDS,))
|
|
1374
1456
|
ring.update(db.execute("SELECT id, key FROM dropped_keys").fetchall())
|
|
1375
|
-
for p in profiles_in("promptlab.
|
|
1457
|
+
for p in profiles_in("promptlab.connections", now.get("promptlab.connections", {}).get("body")):
|
|
1376
1458
|
key = p.get("key")
|
|
1377
1459
|
if isinstance(key, str) and key and not held(key):
|
|
1378
1460
|
ring[str(p.get("id") or "")] = key
|
|
@@ -1408,14 +1490,14 @@ class Store:
|
|
|
1408
1490
|
for p in profiles_in(n, d["body"]):
|
|
1409
1491
|
if held(p.get("key")):
|
|
1410
1492
|
p["key"] = ring.get(p["key"][len(KEY_HELD):], "")
|
|
1411
|
-
if "promptlab.
|
|
1493
|
+
if "promptlab.connections" in docs:
|
|
1412
1494
|
kept = {str(p.get("id") or "") for p in profiles_in(
|
|
1413
|
-
"promptlab.
|
|
1495
|
+
"promptlab.connections", docs["promptlab.connections"]["body"])}
|
|
1414
1496
|
db.executemany("DELETE FROM dropped_keys WHERE id = ?", [(i,) for i in kept])
|
|
1415
1497
|
db.executemany(
|
|
1416
1498
|
"INSERT OR REPLACE INTO dropped_keys (id, key, at) VALUES (?, ?, ?)",
|
|
1417
1499
|
[(str(p.get("id") or ""), p["key"], time.time())
|
|
1418
|
-
for p in profiles_in("promptlab.
|
|
1500
|
+
for p in profiles_in("promptlab.connections", have("promptlab.connections")["body"])
|
|
1419
1501
|
if str(p.get("id") or "") not in kept and isinstance(p.get("key"), str) and p["key"]])
|
|
1420
1502
|
for n, d in docs.items():
|
|
1421
1503
|
version, body = have(n)["version"] + 1, d["body"]
|
|
@@ -1434,9 +1516,10 @@ class Store:
|
|
|
1434
1516
|
return {n: have(n)["version"] + 1 for n in docs}, None
|
|
1435
1517
|
|
|
1436
1518
|
# ---- shared-by-choice connections (docs/workspaces.md) ------------------
|
|
1437
|
-
#
|
|
1438
|
-
#
|
|
1439
|
-
#
|
|
1519
|
+
# An AI Connection or an Account (kind "profile"/"account", id the
|
|
1520
|
+
# connection's -- the Target profile's it kept -- or the account's) is
|
|
1521
|
+
# usable in a workspace by its share set in `connection_shares`: a row for
|
|
1522
|
+
# that workspace, or the SHARE_ALL sentinel.
|
|
1440
1523
|
# No row at all is the migration and new-connection default -- shared with
|
|
1441
1524
|
# every workspace, so nothing stops running the moment workspaces exist
|
|
1442
1525
|
# (open question 2, resolved All); narrowing a connection is adding the
|
|
@@ -1498,10 +1581,10 @@ class Store:
|
|
|
1498
1581
|
return row[0] if row else "this workspace"
|
|
1499
1582
|
|
|
1500
1583
|
def profile_name(self, pid: str) -> str:
|
|
1501
|
-
"""
|
|
1584
|
+
"""An AI Connection's display name, by id, from the global connections
|
|
1502
1585
|
document -- for the refusal sentence. The id itself when none is held."""
|
|
1503
1586
|
with self.lock, closing(sqlite3.connect(self.path)) as db:
|
|
1504
|
-
row = db.execute("SELECT body FROM docs WHERE name = 'promptlab.
|
|
1587
|
+
row = db.execute("SELECT body FROM docs WHERE name = 'promptlab.connections' "
|
|
1505
1588
|
"AND workspace IS NULL").fetchone()
|
|
1506
1589
|
try:
|
|
1507
1590
|
for p in (json.loads(row[0]) or {}).get("list", []) if row and row[0] else []:
|
|
@@ -1827,6 +1910,15 @@ CONFIG_KEYS_MAX = 10
|
|
|
1827
1910
|
CONFIG_VALUE_MAX = 400
|
|
1828
1911
|
|
|
1829
1912
|
|
|
1913
|
+
class _Refused(Exception):
|
|
1914
|
+
"""A refusal found inside a transaction: rolls it back, carrying the
|
|
1915
|
+
(code, sentence) the caller answers with."""
|
|
1916
|
+
|
|
1917
|
+
def __init__(self, err):
|
|
1918
|
+
super().__init__(err[1])
|
|
1919
|
+
self.err = err
|
|
1920
|
+
|
|
1921
|
+
|
|
1830
1922
|
class Sources:
|
|
1831
1923
|
"""
|
|
1832
1924
|
A named bag of files a run might be over (#526).
|
|
@@ -2421,16 +2513,19 @@ class Sources:
|
|
|
2421
2513
|
if produce is None:
|
|
2422
2514
|
return None, (400, "that provider cannot be produced from on the server")
|
|
2423
2515
|
result = produce(cfg)
|
|
2516
|
+
if result.get("refused"):
|
|
2517
|
+
return None, result["refused"]
|
|
2424
2518
|
request = result.get("request")
|
|
2425
2519
|
status = result.get("status")
|
|
2426
2520
|
content = result.get("content")
|
|
2427
2521
|
if content is None:
|
|
2428
2522
|
# A failed production: kept as a Call with its error, no file.
|
|
2429
|
-
name = dynamic_item_name(cfg, (request or {}).get("url", "")
|
|
2523
|
+
name = self._snapshot_name(sid, dynamic_item_name(cfg, (request or {}).get("url", "")
|
|
2524
|
+
or secrets.token_hex(6)))
|
|
2430
2525
|
self._record_call(sid, name, "snapshot", provider, request, status, result.get("error"), None)
|
|
2431
2526
|
return {"name": name, "origin": "snapshot", "at": None, "provider": provider,
|
|
2432
2527
|
"request": request, "status": status, "error": result.get("error")}, None
|
|
2433
|
-
name = dynamic_item_name(cfg, content)
|
|
2528
|
+
name = self._snapshot_name(sid, dynamic_item_name(cfg, content))
|
|
2434
2529
|
kept, take_err = entry["take"](name, content)
|
|
2435
2530
|
if take_err:
|
|
2436
2531
|
code, message = take_err
|
|
@@ -2438,7 +2533,8 @@ class Sources:
|
|
|
2438
2533
|
return {"name": name, "origin": "snapshot", "at": None, "provider": provider,
|
|
2439
2534
|
"request": request, "status": status, "error": message}, None
|
|
2440
2535
|
# The name is stable (the URL's), so a re-snapshot overwrites the item
|
|
2441
|
-
# rather than adding a second: remove the old file first, then add.
|
|
2536
|
+
# rather than adding a second: remove the old file first, then add. An
|
|
2537
|
+
# Edited Call is never the one overwritten (_snapshot_name).
|
|
2442
2538
|
self.remove_files(sid, [name])
|
|
2443
2539
|
stored, add_err = self.add_file(sid, name, kept)
|
|
2444
2540
|
if add_err:
|
|
@@ -2447,6 +2543,162 @@ class Sources:
|
|
|
2447
2543
|
return {"name": stored, "origin": "snapshot", "at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
2448
2544
|
"provider": provider, "request": request, "status": status, "bytes": len(kept)}, None
|
|
2449
2545
|
|
|
2546
|
+
def _snapshot_name(self, sid, name):
|
|
2547
|
+
"""The name a snapshot is kept under: [name], unless an Edited Call
|
|
2548
|
+
holds it -- a snapshot never overwrites an edit -- and then the first
|
|
2549
|
+
of `<stem>-2`, `<stem>-3`... that none does. One that is not Edited is
|
|
2550
|
+
overwritten, as the stable name always was."""
|
|
2551
|
+
with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
|
|
2552
|
+
edited = {r[0] for r in db.execute(
|
|
2553
|
+
"SELECT name FROM source_calls WHERE source = ? AND origin = 'edited'", (sid,))}
|
|
2554
|
+
if name not in edited:
|
|
2555
|
+
return name
|
|
2556
|
+
stem, ext = os.path.splitext(name)
|
|
2557
|
+
n = 2
|
|
2558
|
+
while f"{stem}-{n}{ext}" in edited:
|
|
2559
|
+
n += 1
|
|
2560
|
+
return f"{stem}-{n}{ext}"
|
|
2561
|
+
|
|
2562
|
+
# ---- a Call opened in the Call dialog: edited, renamed, cloned (#438) ---
|
|
2563
|
+
#
|
|
2564
|
+
# Both kinds of Call -- a dynamic Source's and a Power Automate record --
|
|
2565
|
+
# are files of a Source whose kind has a `take` hook, so a saved edit goes
|
|
2566
|
+
# through the same check as content arriving any other way: a key refused,
|
|
2567
|
+
# the rest redacted. A File Library has no hook and no Calls.
|
|
2568
|
+
|
|
2569
|
+
# The origins a dynamic Call's provenance row may be put back to (Undo).
|
|
2570
|
+
CALL_ORIGINS = ("snapshot", "run", "upload", "hand", "edited")
|
|
2571
|
+
|
|
2572
|
+
def _calls_entry(self, sid):
|
|
2573
|
+
"""The enforcement entry of a user Source whose files are Calls, or
|
|
2574
|
+
(None, error)."""
|
|
2575
|
+
with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
|
|
2576
|
+
r = db.execute("SELECT system FROM sources WHERE id = ? AND (workspace = ? OR system = 1)",
|
|
2577
|
+
(sid, workspace_of(self.store))).fetchone()
|
|
2578
|
+
if r is None:
|
|
2579
|
+
return None, (404, "no such source")
|
|
2580
|
+
if r[0]:
|
|
2581
|
+
return None, (403, "the sample library is read-only")
|
|
2582
|
+
entry = self.type_of(sid) or {}
|
|
2583
|
+
if entry.get("take") is None:
|
|
2584
|
+
return None, (400, "that Source keeps no Calls")
|
|
2585
|
+
return entry, None
|
|
2586
|
+
|
|
2587
|
+
def _names_held(self, db, sid):
|
|
2588
|
+
"""Every name a Source's files and Calls hold: what a new name may not be."""
|
|
2589
|
+
return ({r[0] for r in db.execute("SELECT name FROM source_files WHERE source = ?", (sid,))}
|
|
2590
|
+
| {r[0] for r in db.execute("SELECT name FROM source_calls WHERE source = ?", (sid,))})
|
|
2591
|
+
|
|
2592
|
+
def edit_call(self, sid, name, to=None, content=None, origin=None, named_by=None):
|
|
2593
|
+
"""One Call as the Call dialog saves it: renamed to [to], its text
|
|
2594
|
+
replaced by [content], or both, in one write. The text goes through
|
|
2595
|
+
the Source's `take` hook as an upload does. A rename keeps the
|
|
2596
|
+
extension and refuses a name in use, or a Call that a dataset's cases
|
|
2597
|
+
name ([named_by] answers which datasets do), since a case finds its
|
|
2598
|
+
item by name. A dynamic Call's origin becomes `edited` when its text
|
|
2599
|
+
changed -- a rename alone is no edit -- unless [origin] puts back the
|
|
2600
|
+
one it had (Undo). Nothing is written unless all of it passes.
|
|
2601
|
+
Returns ({name, origin, bytes}, None), or (None, (code, sentence)) --
|
|
2602
|
+
(code, sentence, "name") when the Name field is what is wrong."""
|
|
2603
|
+
entry, err = self._calls_entry(sid)
|
|
2604
|
+
if err:
|
|
2605
|
+
return None, err
|
|
2606
|
+
path = self.file_path(sid, name)
|
|
2607
|
+
if path is None or not path.is_file():
|
|
2608
|
+
return None, (404, f"{name} is not a Call of that Source")
|
|
2609
|
+
if origin is not None and origin not in self.CALL_ORIGINS:
|
|
2610
|
+
return None, (400, "that is not an origin a Call has")
|
|
2611
|
+
new = name
|
|
2612
|
+
if to is not None and to != name:
|
|
2613
|
+
new = source_name(to)
|
|
2614
|
+
if new is None or new != to:
|
|
2615
|
+
return None, (400, "that name is not accepted here", "name")
|
|
2616
|
+
if Path(new).suffix.lower() != Path(name).suffix.lower():
|
|
2617
|
+
return None, (400, "a Call keeps its extension", "name")
|
|
2618
|
+
with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
|
|
2619
|
+
taken = new in self._names_held(db, sid)
|
|
2620
|
+
if taken:
|
|
2621
|
+
return None, (409, f"{new} is taken.", "name")
|
|
2622
|
+
users = named_by(name) if named_by else []
|
|
2623
|
+
if users:
|
|
2624
|
+
return None, (409, f"{users[0]} names {name} in its cases.", "name")
|
|
2625
|
+
old = path.read_bytes()
|
|
2626
|
+
data = old
|
|
2627
|
+
if content is not None:
|
|
2628
|
+
if not isinstance(content, str):
|
|
2629
|
+
return None, (400, "a Call's content is text")
|
|
2630
|
+
raw = content.encode("utf-8")
|
|
2631
|
+
if not raw:
|
|
2632
|
+
return None, (400, "a Call needs some content")
|
|
2633
|
+
if len(raw) > MAX_FILE:
|
|
2634
|
+
return None, (413, "that is over the file cap")
|
|
2635
|
+
data, take_err = entry["take"](new, raw)
|
|
2636
|
+
if take_err:
|
|
2637
|
+
return None, take_err
|
|
2638
|
+
changed = data != old
|
|
2639
|
+
folder = self.dir / sid
|
|
2640
|
+
tmp = None
|
|
2641
|
+
if changed or new != name:
|
|
2642
|
+
with tempfile.NamedTemporaryFile("wb", dir=folder, delete=False) as f:
|
|
2643
|
+
f.write(data)
|
|
2644
|
+
tmp = f.name
|
|
2645
|
+
try:
|
|
2646
|
+
with self.store.lock, closing(sqlite3.connect(self.store.path)) as db, db:
|
|
2647
|
+
if new != name and new in self._names_held(db, sid):
|
|
2648
|
+
raise _Refused((409, f"{new} is taken.", "name"))
|
|
2649
|
+
total = db.execute("SELECT bytes FROM sources WHERE id = ?", (sid,)).fetchone()[0]
|
|
2650
|
+
if total + len(data) - len(old) > MAX_SOURCE:
|
|
2651
|
+
raise _Refused((413, "that would take the source past its cap"))
|
|
2652
|
+
db.execute("UPDATE source_files SET name = ?, bytes = ? WHERE source = ? AND name = ?",
|
|
2653
|
+
(new, len(data), sid, name))
|
|
2654
|
+
db.execute("UPDATE sources SET bytes = bytes + ? WHERE id = ?", (len(data) - len(old), sid))
|
|
2655
|
+
row = db.execute("SELECT origin FROM source_calls WHERE source = ? AND name = ?",
|
|
2656
|
+
(sid, name)).fetchone()
|
|
2657
|
+
now = None
|
|
2658
|
+
if row is not None:
|
|
2659
|
+
now = origin or ("edited" if changed else row[0])
|
|
2660
|
+
db.execute("UPDATE source_calls SET name = ?, file = ?, origin = ? "
|
|
2661
|
+
"WHERE source = ? AND name = ?", (new, new, now, sid, name))
|
|
2662
|
+
except _Refused as no:
|
|
2663
|
+
if tmp:
|
|
2664
|
+
os.unlink(tmp)
|
|
2665
|
+
return None, no.err
|
|
2666
|
+
if tmp:
|
|
2667
|
+
os.replace(tmp, folder / new)
|
|
2668
|
+
if new != name:
|
|
2669
|
+
(folder / name).unlink(missing_ok=True)
|
|
2670
|
+
return {"name": new, "origin": now, "bytes": len(data)}, None
|
|
2671
|
+
|
|
2672
|
+
def clone_call(self, sid, name):
|
|
2673
|
+
"""A copy of one Call under `<stem>-copy` (then `-copy-2`, ...), its
|
|
2674
|
+
file and, for a dynamic Call, its provenance -- origin, request,
|
|
2675
|
+
status and when it was produced -- as they are. Returns ({name}, None),
|
|
2676
|
+
or (None, error)."""
|
|
2677
|
+
entry, err = self._calls_entry(sid)
|
|
2678
|
+
if err:
|
|
2679
|
+
return None, err
|
|
2680
|
+
path = self.file_path(sid, name)
|
|
2681
|
+
if path is None or not path.is_file():
|
|
2682
|
+
return None, (404, f"{name} is not a Call of that Source")
|
|
2683
|
+
stem, ext = os.path.splitext(name)
|
|
2684
|
+
with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
|
|
2685
|
+
held = self._names_held(db, sid)
|
|
2686
|
+
n = 1
|
|
2687
|
+
while True:
|
|
2688
|
+
tail = "-copy" if n == 1 else f"-copy-{n}"
|
|
2689
|
+
copy = stem[:120 - len(tail) - len(ext)] + tail + ext
|
|
2690
|
+
if copy not in held:
|
|
2691
|
+
break
|
|
2692
|
+
n += 1
|
|
2693
|
+
stored, err = self.add_file(sid, copy, path.read_bytes())
|
|
2694
|
+
if err:
|
|
2695
|
+
return None, err
|
|
2696
|
+
with self.store.lock, closing(sqlite3.connect(self.store.path)) as db, db:
|
|
2697
|
+
db.execute("INSERT INTO source_calls (source, name, origin, at, provider, request, status, error, file) "
|
|
2698
|
+
"SELECT source, ?, origin, at, provider, request, status, error, ? "
|
|
2699
|
+
"FROM source_calls WHERE source = ? AND name = ?", (stored, stored, sid, name))
|
|
2700
|
+
return {"name": stored}, None
|
|
2701
|
+
|
|
2450
2702
|
def remove_calls(self, sid, names):
|
|
2451
2703
|
"""Calls out by name: a file-backed Call to the trash (Undo restores
|
|
2452
2704
|
it), a failed Call's row dropped outright (it has no file). Returns
|
|
@@ -3871,6 +4123,24 @@ class Datasets:
|
|
|
3871
4123
|
"SELECT name FROM datasets WHERE trash IS NULL AND id IS NOT ? AND workspace = ?",
|
|
3872
4124
|
(but, workspace_of(self.store)))}
|
|
3873
4125
|
|
|
4126
|
+
def naming_item(self, sid, item) -> list:
|
|
4127
|
+
"""The names of this workspace's groups previewed against Source
|
|
4128
|
+
[sid] whose cases name [item]: what renaming that item would break."""
|
|
4129
|
+
out = []
|
|
4130
|
+
with self.store.lock, self._connect() as db:
|
|
4131
|
+
for r in db.execute("SELECT name, body, preview FROM datasets WHERE trash IS NULL "
|
|
4132
|
+
"AND workspace = ? AND preview IS NOT NULL ORDER BY name COLLATE NOCASE",
|
|
4133
|
+
(workspace_of(self.store),)):
|
|
4134
|
+
try:
|
|
4135
|
+
ref, body = json.loads(r["preview"]), json.loads(r["body"])
|
|
4136
|
+
except ValueError:
|
|
4137
|
+
continue
|
|
4138
|
+
cases = body.get("cases") if isinstance(body, dict) else None
|
|
4139
|
+
if isinstance(ref, dict) and ref.get("id") == sid and isinstance(cases, list) \
|
|
4140
|
+
and any(isinstance(c, dict) and c.get("item") == item for c in cases):
|
|
4141
|
+
out.append(r["name"])
|
|
4142
|
+
return out
|
|
4143
|
+
|
|
3874
4144
|
def list(self) -> list:
|
|
3875
4145
|
with self.store.lock, self._connect() as db:
|
|
3876
4146
|
counts = self._counts(db)
|
|
@@ -4781,32 +5051,47 @@ class Pipelines:
|
|
|
4781
5051
|
self.store._put(db, self.VERSIONS, target_ws, tv + 1, tbody, self._now())
|
|
4782
5052
|
|
|
4783
5053
|
|
|
4784
|
-
def
|
|
4785
|
-
"""
|
|
4786
|
-
for every image type, and a full-size render for
|
|
4787
|
-
cannot decode, whose zoom view has no original to
|
|
4788
|
-
|
|
4789
|
-
|
|
5054
|
+
def thumb_jobs(sid, name):
|
|
5055
|
+
"""The `magick` runs that make a stored file's thumbnails, each (argv,
|
|
5056
|
+
out): a tile-size JPEG for every image type, and a full-size render for
|
|
5057
|
+
the formats a browser cannot decode, whose zoom view has no original to
|
|
5058
|
+
show. The paths are looked up here, in the request's own thread, since a
|
|
5059
|
+
Source is found through the workspace bound to it. #33 #527"""
|
|
4790
5060
|
if CONVERT is None or Path(name).suffix.lower() not in IMAGE_TYPES:
|
|
4791
|
-
return
|
|
5061
|
+
return []
|
|
5062
|
+
jobs = []
|
|
4792
5063
|
for full in (False, True):
|
|
4793
5064
|
if full and Path(name).suffix.lower() not in THUMB_TYPES:
|
|
4794
5065
|
continue
|
|
4795
5066
|
path = SOURCES.file_path(sid, name)
|
|
4796
5067
|
out = SOURCES.thumb_path(sid, name, full)
|
|
4797
5068
|
if path is None or out is None:
|
|
4798
|
-
return
|
|
5069
|
+
return jobs
|
|
4799
5070
|
size = FULL_SIZE if full else TILE_SIZE
|
|
4800
|
-
|
|
4801
|
-
|
|
4802
|
-
|
|
4803
|
-
|
|
4804
|
-
|
|
4805
|
-
|
|
4806
|
-
|
|
4807
|
-
|
|
4808
|
-
|
|
4809
|
-
|
|
5071
|
+
jobs.append(([CONVERT, str(path), "-auto-orient",
|
|
5072
|
+
"-thumbnail", f"{size}x{size}>",
|
|
5073
|
+
"-quality", str(FULL_QUALITY if full else TILE_QUALITY),
|
|
5074
|
+
str(out)], out))
|
|
5075
|
+
return jobs
|
|
5076
|
+
|
|
5077
|
+
|
|
5078
|
+
def run_thumb(job):
|
|
5079
|
+
"""One thumbnail. Non-fatal by design: a missing thumb is a blank tile,
|
|
5080
|
+
not a failed upload, and the startup sweep clears an orphan on a later
|
|
5081
|
+
start."""
|
|
5082
|
+
argv, out = job
|
|
5083
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
5084
|
+
try:
|
|
5085
|
+
subprocess.run(argv, timeout=30, stdout=subprocess.DEVNULL,
|
|
5086
|
+
stderr=subprocess.DEVNULL, check=True)
|
|
5087
|
+
except (subprocess.SubprocessError, OSError):
|
|
5088
|
+
out.unlink(missing_ok=True)
|
|
5089
|
+
|
|
5090
|
+
|
|
5091
|
+
def make_thumbs(sid, name):
|
|
5092
|
+
"""A stored file's thumbnails, made now (thumb_jobs)."""
|
|
5093
|
+
for job in thumb_jobs(sid, name):
|
|
5094
|
+
run_thumb(job)
|
|
4810
5095
|
|
|
4811
5096
|
|
|
4812
5097
|
# ---- Packs ------------------------------------------------------------------
|
|
@@ -4877,7 +5162,9 @@ def pack_requires_problem(manifest: dict, plugins: set) -> str:
|
|
|
4877
5162
|
def read_pack(data: bytes):
|
|
4878
5163
|
"""A pack's zip read and checked whole: (pack, None) or (None, why). The
|
|
4879
5164
|
pack is {manifest, datasets: [(key, name, body, raw)], pipelines:
|
|
4880
|
-
[(key, doc)], presets: [...], sources: {name: [(file, bytes)]}
|
|
5165
|
+
[(key, doc)], presets: [...], sources: {name: [(file, bytes)]}, source_meta:
|
|
5166
|
+
{name: {name, type, config}}} -- a Source's meta from its folder's
|
|
5167
|
+
`source.json`, or the folder's name as a File Library."""
|
|
4881
5168
|
try:
|
|
4882
5169
|
zf = zipfile.ZipFile(io.BytesIO(data))
|
|
4883
5170
|
except zipfile.BadZipFile:
|
|
@@ -4906,7 +5193,8 @@ def read_pack(data: bytes):
|
|
|
4906
5193
|
for k in ("name", "packVersion"):
|
|
4907
5194
|
if not isinstance(manifest.get(k), str) or not manifest[k].strip():
|
|
4908
5195
|
return None, f"a pack's manifest needs a {k}"
|
|
4909
|
-
pack = {"manifest": manifest, "datasets": [], "pipelines": [], "presets": [], "sources": {}}
|
|
5196
|
+
pack = {"manifest": manifest, "datasets": [], "pipelines": [], "presets": [], "sources": {}, "source_meta": {}}
|
|
5197
|
+
declared = {}
|
|
4910
5198
|
for name in sorted(names):
|
|
4911
5199
|
parts = name.split("/")
|
|
4912
5200
|
if name == "manifest.json":
|
|
@@ -4949,21 +5237,86 @@ def read_pack(data: bytes):
|
|
|
4949
5237
|
or not (isinstance(doc.get("jobs"), list) or isinstance(doc.get("chains"), list))):
|
|
4950
5238
|
return None, f"{name} is not a pipeline document the lab reads"
|
|
4951
5239
|
pack["pipelines"].append((parts[1][:-5], doc))
|
|
5240
|
+
elif len(parts) == 3 and parts[0] == "sources" and parts[2] == PACK_SOURCE_META:
|
|
5241
|
+
src = source_name(parts[1])
|
|
5242
|
+
meta, why = load(name)
|
|
5243
|
+
if why:
|
|
5244
|
+
return None, why
|
|
5245
|
+
if src is None or not isinstance(meta, dict):
|
|
5246
|
+
return None, f"{name} is not a Source's {PACK_SOURCE_META}"
|
|
5247
|
+
declared[src] = meta
|
|
4952
5248
|
elif len(parts) == 3 and parts[0] == "sources":
|
|
4953
5249
|
src, file = source_name(parts[1]), source_name(parts[2])
|
|
4954
|
-
if src is None or file is None
|
|
5250
|
+
if src is None or file is None:
|
|
4955
5251
|
return None, f"{name} is not a file a Source takes"
|
|
4956
5252
|
if names[name].file_size > MAX_FILE:
|
|
4957
5253
|
return None, f"{name} is over the cap for a file"
|
|
4958
5254
|
pack["sources"].setdefault(src, []).append((file, zf.read(name)))
|
|
4959
5255
|
else:
|
|
4960
5256
|
return None, f"{name} is not something a pack holds"
|
|
4961
|
-
for src
|
|
5257
|
+
for src in sorted(set(pack["sources"]) | set(declared)):
|
|
5258
|
+
meta, why = pack_source_meta(src, declared.get(src))
|
|
5259
|
+
if why:
|
|
5260
|
+
return None, why
|
|
5261
|
+
pack["source_meta"][src] = meta
|
|
5262
|
+
files = pack["sources"].setdefault(src, [])
|
|
5263
|
+
entry = source_entry(meta["type"], meta["config"]) or {}
|
|
5264
|
+
takes, take = entry.get("uploads") or {}, entry.get("take")
|
|
5265
|
+
for i, (file, blob) in enumerate(files):
|
|
5266
|
+
if Path(file).suffix.lower() not in takes:
|
|
5267
|
+
return None, f"sources/{src}/{file} is not a file a Source takes"
|
|
5268
|
+
# A shipped Call is checked as any other is: a token refuses the
|
|
5269
|
+
# pack, and a secret's shape is redacted before it is kept.
|
|
5270
|
+
if take is not None:
|
|
5271
|
+
blob, err = take(file, blob)
|
|
5272
|
+
if err:
|
|
5273
|
+
return None, f"sources/{src}/{file}: {err[1]}"
|
|
5274
|
+
files[i] = (file, blob)
|
|
4962
5275
|
if sum(len(b) for _, b in files) > MAX_SOURCE:
|
|
4963
5276
|
return None, f"the Source {src} is over the cap for a Source"
|
|
4964
5277
|
return pack, None
|
|
4965
5278
|
|
|
4966
5279
|
|
|
5280
|
+
# A pack Source's own meta, beside its files: {name?, type?, config?}.
|
|
5281
|
+
PACK_SOURCE_META = "source.json"
|
|
5282
|
+
|
|
5283
|
+
|
|
5284
|
+
def pack_source_meta(folder, meta):
|
|
5285
|
+
"""A pack Source's {name, type, config} from its folder's source.json (or
|
|
5286
|
+
None: a File Library named after the folder), or (None, why). A pack may
|
|
5287
|
+
carry a File Library or a dynamic Source whose provider pulls with no key
|
|
5288
|
+
and is enrolled here -- the line a plugin's provider is held to
|
|
5289
|
+
(docs/packs.md); anything else refuses the pack whole."""
|
|
5290
|
+
meta = meta or {}
|
|
5291
|
+
at = f"sources/{folder}/{PACK_SOURCE_META}"
|
|
5292
|
+
unknown = set(meta) - {"name", "type", "config"}
|
|
5293
|
+
if unknown:
|
|
5294
|
+
return None, f"{at} holds {sorted(unknown)[0]}, which a pack Source does not"
|
|
5295
|
+
name, err = Sources._name(meta.get("name", folder))
|
|
5296
|
+
if err:
|
|
5297
|
+
return None, f"{at}: {err[1]}"
|
|
5298
|
+
kind = meta.get("type", DEFAULT_SOURCE_TYPE)
|
|
5299
|
+
config = meta.get("config")
|
|
5300
|
+
if kind == DEFAULT_SOURCE_TYPE:
|
|
5301
|
+
if config is not None:
|
|
5302
|
+
return None, f"{at}: a File Library takes no config"
|
|
5303
|
+
return {"name": name, "type": kind, "config": None}, None
|
|
5304
|
+
if not (SOURCE_TYPES.get(kind) or {}).get("providers"):
|
|
5305
|
+
return None, f"{at}: a pack carries a File Library or a dynamic Source, not {kind!r}"
|
|
5306
|
+
provider = config.get("provider") if isinstance(config, dict) else None
|
|
5307
|
+
entry = SOURCE_PROVIDERS.get(provider)
|
|
5308
|
+
if entry is None:
|
|
5309
|
+
return None, f"{at}: the lab has no provider {provider!r}"
|
|
5310
|
+
if entry.get("tier") != "pull" or entry.get("keyed") or config.get("connection"):
|
|
5311
|
+
return None, f"{at}: a pack's dynamic Source pulls with no key, and {entry.get('label', provider)} does not"
|
|
5312
|
+
if not provider_enrolled(entry):
|
|
5313
|
+
return None, f"{at}: this lab does not enrol {entry.get('label', provider)}"
|
|
5314
|
+
_, err = Sources._checked(kind, config)
|
|
5315
|
+
if err:
|
|
5316
|
+
return None, f"{at}: {err[1]}"
|
|
5317
|
+
return {"name": name, "type": kind, "config": config}, None
|
|
5318
|
+
|
|
5319
|
+
|
|
4967
5320
|
class Packs:
|
|
4968
5321
|
"""What each installed pack made, so it can be updated and removed."""
|
|
4969
5322
|
|
|
@@ -5098,22 +5451,41 @@ class Packs:
|
|
|
5098
5451
|
made = {}
|
|
5099
5452
|
# Sources first, then datasets: a pipeline names both.
|
|
5100
5453
|
src_ids = {}
|
|
5454
|
+
thumbs = []
|
|
5101
5455
|
for name, files in pack["sources"].items():
|
|
5456
|
+
meta = pack["source_meta"][name]
|
|
5102
5457
|
sid = owned.get(("source", name))
|
|
5103
|
-
|
|
5104
|
-
|
|
5458
|
+
have = SOURCES.get(sid) if sid else None
|
|
5459
|
+
# An update keeps the Source's name, so one renamed here, or
|
|
5460
|
+
# by a later pack, is not renamed back. Another kind is
|
|
5461
|
+
# another Source.
|
|
5462
|
+
if have and have.get("type") == meta["type"]:
|
|
5463
|
+
old = [f["name"] for f in have["files"]]
|
|
5105
5464
|
if old:
|
|
5106
5465
|
SOURCES.remove_files(sid, old)
|
|
5466
|
+
if meta["config"] is not None:
|
|
5467
|
+
SOURCES.update(sid, {"config": meta["config"]})
|
|
5107
5468
|
else:
|
|
5108
5469
|
taken = {s["name"] for s in SOURCES.list()}
|
|
5109
|
-
src, _ = SOURCES.create(unique_dataset_name(name, taken))
|
|
5470
|
+
src, _ = SOURCES.create(unique_dataset_name(meta["name"], taken), meta["type"], meta["config"])
|
|
5110
5471
|
sid = src["id"]
|
|
5111
5472
|
for file, blob in files:
|
|
5112
5473
|
stored, err = SOURCES.add_file(sid, file, blob)
|
|
5113
|
-
if
|
|
5114
|
-
|
|
5474
|
+
if err:
|
|
5475
|
+
continue
|
|
5476
|
+
thumbs += thumb_jobs(sid, stored)
|
|
5477
|
+
# A dynamic Source's files are Calls: the pack's are its
|
|
5478
|
+
# snapshot, fetched when the pack was made.
|
|
5479
|
+
if meta["type"] != DEFAULT_SOURCE_TYPE:
|
|
5480
|
+
cfg = meta["config"]
|
|
5481
|
+
request = {"url": cfg["url"], "method": cfg.get("method") or "GET"} if cfg.get("url") else None
|
|
5482
|
+
SOURCES._record_call(sid, stored, "snapshot", cfg.get("provider"), request, None, None, stored)
|
|
5115
5483
|
src_ids[name] = sid
|
|
5116
5484
|
made[("source", name)] = sid
|
|
5485
|
+
# The demo pack installs as the server starts, before it answers,
|
|
5486
|
+
# and one `magick` at a time was most of that wait.
|
|
5487
|
+
with ThreadPoolExecutor(max_workers=min(4, os.cpu_count() or 1)) as pool:
|
|
5488
|
+
list(pool.map(run_thumb, thumbs))
|
|
5117
5489
|
ds_ids = {}
|
|
5118
5490
|
cuts = [] # (kind, id, the document as it was), kept before it changes
|
|
5119
5491
|
for key, name, body, raw, preview in pack["datasets"]:
|
|
@@ -5170,7 +5542,8 @@ class Packs:
|
|
|
5170
5542
|
entry = next((w for w in body["list"] if w.get("id") == wid), None) if wid else None
|
|
5171
5543
|
if entry:
|
|
5172
5544
|
cuts.append(("pipeline", wid, entry.get("work")))
|
|
5173
|
-
|
|
5545
|
+
# Its name is the lab's now, as a dataset's is.
|
|
5546
|
+
entry["work"] = {**doc, "name": entry.get("name") or doc.get("name")}
|
|
5174
5547
|
else:
|
|
5175
5548
|
wid = "w" + secrets.token_hex(6)
|
|
5176
5549
|
name = unique_dataset_name(doc.get("name") or key, {w.get("name") for w in body["list"]})
|
|
@@ -6942,7 +7315,7 @@ def bundle_problems(files, keys=()):
|
|
|
6942
7315
|
for key in keys:
|
|
6943
7316
|
key = str(key or "").strip()
|
|
6944
7317
|
if len(key) >= 4 and key in text:
|
|
6945
|
-
return f"{path} holds a
|
|
7318
|
+
return f"{path} holds a connection's key, and a key never leaves the lab"
|
|
6946
7319
|
if TOKEN_SHAPE.search(text):
|
|
6947
7320
|
return f"{path} holds a token, and a key never leaves the lab"
|
|
6948
7321
|
m = BUNDLE_KEY_FIELD.search(text)
|
|
@@ -6965,7 +7338,7 @@ def build_bundle(payload):
|
|
|
6965
7338
|
return None, (400, "a bundle holds pipeline.yaml and profiles.yaml")
|
|
6966
7339
|
stored = []
|
|
6967
7340
|
if STORE is not None:
|
|
6968
|
-
stored = ((STORE.all().get("promptlab.
|
|
7341
|
+
stored = ((STORE.all().get("promptlab.connections") or {}).get("body") or {}).get("list") or []
|
|
6969
7342
|
why = bundle_problems(files, [p.get("key") for p in stored if isinstance(p, dict)])
|
|
6970
7343
|
if why:
|
|
6971
7344
|
return None, (400, why)
|
|
@@ -6994,10 +7367,12 @@ def build_bundle(payload):
|
|
|
6994
7367
|
|
|
6995
7368
|
|
|
6996
7369
|
# Shared-by-choice enforcement (docs/workspaces.md): the one sentence the Run
|
|
6997
|
-
# bar turns into a Share link, and the
|
|
6998
|
-
#
|
|
6999
|
-
#
|
|
7000
|
-
#
|
|
7370
|
+
# bar turns into a Share link, and the connections that earn it -- the AI
|
|
7371
|
+
# Connections the run resolved to that are not shared with workspace `ws`,
|
|
7372
|
+
# a profile's own connection included where it answers on one. A local
|
|
7373
|
+
# connection reaches nothing, so sharing does not gate it. A share set is
|
|
7374
|
+
# written against the id a Target profile had, which its connection keeps
|
|
7375
|
+
# (#445), so a "profile" row names the connection.
|
|
7001
7376
|
def not_shared_sentence(names, wsname: str) -> str:
|
|
7002
7377
|
verb = "is" if len(names) == 1 else "are"
|
|
7003
7378
|
return f"{', '.join(names)} {verb} not shared with {wsname}."
|
|
@@ -7007,12 +7382,16 @@ def unshared_profiles(run: dict, ws):
|
|
|
7007
7382
|
table = run.get("profiles") if isinstance(run, dict) else None
|
|
7008
7383
|
if not isinstance(table, dict) or STORE is None:
|
|
7009
7384
|
return []
|
|
7010
|
-
out = []
|
|
7011
|
-
for
|
|
7385
|
+
out, seen = [], set()
|
|
7386
|
+
for key, conn in table.items():
|
|
7012
7387
|
if isinstance(conn, dict) and conn.get("type") in LOCAL_CONNECTIONS:
|
|
7013
7388
|
continue
|
|
7014
|
-
|
|
7015
|
-
|
|
7389
|
+
cid = answer_connection(key, conn)
|
|
7390
|
+
if cid in seen:
|
|
7391
|
+
continue
|
|
7392
|
+
seen.add(cid)
|
|
7393
|
+
if not STORE.shared("profile", cid, ws):
|
|
7394
|
+
out.append({"id": cid, "name": (isinstance(conn, dict) and conn.get("name")) or cid})
|
|
7016
7395
|
return out
|
|
7017
7396
|
|
|
7018
7397
|
|
|
@@ -7029,28 +7408,38 @@ def worker_destinations(run: dict):
|
|
|
7029
7408
|
return None, not_shared_sentence([b["name"] for b in bad], STORE.workspace_name(ws))
|
|
7030
7409
|
stored = []
|
|
7031
7410
|
if STORE is not None:
|
|
7032
|
-
doc = STORE.all().get("promptlab.
|
|
7411
|
+
doc = STORE.all().get("promptlab.connections") or {}
|
|
7033
7412
|
stored = (doc.get("body") or {}).get("list") or []
|
|
7034
|
-
|
|
7035
|
-
|
|
7413
|
+
# One variable a connection, however many targets or profiles share it --
|
|
7414
|
+
# and never two connections one variable, whichever path the run came by:
|
|
7415
|
+
# a stored snapshot is not checked again as a submit is (run_problems),
|
|
7416
|
+
# and the runner reads each connection's key from its slug's variable, so
|
|
7417
|
+
# the second would send the first one's key to its own address.
|
|
7418
|
+
env, spelt = {}, {}
|
|
7419
|
+
for key, conn in table.items():
|
|
7420
|
+
pid = answer_connection(key, conn)
|
|
7036
7421
|
name = conn.get("name") or pid
|
|
7422
|
+
var = key_var(pid, conn.get("slug") if isinstance(conn.get("slug"), str) else None)
|
|
7423
|
+
if var in spelt and spelt[var][0] != pid:
|
|
7424
|
+
return None, f"Connections {spelt[var][1]} and {name} both take their key from ${var}"
|
|
7425
|
+
spelt[var] = (pid, name)
|
|
7037
7426
|
if isinstance(conn, dict) and conn.get("type") in LOCAL_CONNECTIONS:
|
|
7038
7427
|
continue
|
|
7039
7428
|
base = api_base(str(conn.get("url") or "")) or api_base(OLLAMA)
|
|
7040
7429
|
why = allowed(base, "")
|
|
7041
7430
|
if why:
|
|
7042
|
-
return None, f"
|
|
7431
|
+
return None, f"Connection {name}: {why}"
|
|
7043
7432
|
profile = next((p for p in stored if isinstance(p, dict) and p.get("id") == pid), None)
|
|
7044
7433
|
if profile is None:
|
|
7045
|
-
return None, f"
|
|
7434
|
+
return None, f"Connection {name} not found"
|
|
7046
7435
|
key = str(profile.get("key") or "").strip()
|
|
7047
7436
|
if key:
|
|
7048
7437
|
if not header_safe(key):
|
|
7049
|
-
return None, f"
|
|
7438
|
+
return None, f"Connection {name} has a key that cannot go in a header"
|
|
7050
7439
|
why = allowed(base, key)
|
|
7051
7440
|
if why:
|
|
7052
|
-
return None, f"
|
|
7053
|
-
env[
|
|
7441
|
+
return None, f"Connection {name}: {why}"
|
|
7442
|
+
env[var] = key
|
|
7054
7443
|
return env, None
|
|
7055
7444
|
|
|
7056
7445
|
|
|
@@ -7078,11 +7467,16 @@ def worker_destinations(run: dict):
|
|
|
7078
7467
|
# inline Drop items / Reject rules migrate into a private one (§18, #339).
|
|
7079
7468
|
# 15: every cleaning step is a filter -- a job's Tidy / Drop prompt echoes /
|
|
7080
7469
|
# Remove duplicates / Keep first / Case modifiers and its List `preamble`
|
|
7081
|
-
# setting migrate into
|
|
7082
|
-
|
|
7470
|
+
# setting migrate into Pre-/Post-format filter sets (§18, #405).
|
|
7471
|
+
# 16: a target names an AI Connection and, optionally, a Connection Profile,
|
|
7472
|
+
# where it named one Target profile; the run's table is keyed by the pair
|
|
7473
|
+
# (answer_key), each entry naming the connection whose key it takes (#445).
|
|
7474
|
+
# 17: an HTTP Request target has no `api` of its own; its job's flow step's
|
|
7475
|
+
# is the one its body is read by (#477).
|
|
7476
|
+
PIPELINE_VERSION = 17
|
|
7083
7477
|
# What a stored run may be: the current version, and the ones evals-core.ts's
|
|
7084
7478
|
# upgradePipeline reads. A new submission is upgraded to the current one.
|
|
7085
|
-
READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15)
|
|
7479
|
+
READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17)
|
|
7086
7480
|
TARGET_CAP = 4
|
|
7087
7481
|
# Target steps whose words the Prompt library does not record as a use: they
|
|
7088
7482
|
# ask no model (evals-core.ts's STEP_TYPES.echo).
|
|
@@ -7115,7 +7509,7 @@ CONNECTION_TYPES = {
|
|
|
7115
7509
|
"echo": (),
|
|
7116
7510
|
# An HTTP endpoint (evals-core.ts): an address and how its key travels; an
|
|
7117
7511
|
# HTTP Request step builds the whole request.
|
|
7118
|
-
"http": ("auth", "headers", "testPath"),
|
|
7512
|
+
"http": ("auth", "headers", "testPath", "requestFormat"),
|
|
7119
7513
|
"ollama": ("seed", "nPredict", "temperature", "repeatPenalty"),
|
|
7120
7514
|
"ollama-cloud": ("seed", "nPredict", "temperature", "repeatPenalty"),
|
|
7121
7515
|
"openai-compatible": ("temperature", "nPredict"),
|
|
@@ -7133,11 +7527,88 @@ BUILTIN_CONNECTION_TYPES = dict(CONNECTION_TYPES)
|
|
|
7133
7527
|
BUILTIN_LOCAL = set(LOCAL_CONNECTIONS)
|
|
7134
7528
|
BUILTIN_CHAT_PATHS = dict(CONNECTION_CHAT_PATHS)
|
|
7135
7529
|
BUILTIN_AUTH = dict(CONNECTION_AUTH)
|
|
7136
|
-
CONNECTION_FIELDS = ("name", "slug", "url", "model", "type", "temperature", "px", "format",
|
|
7530
|
+
CONNECTION_FIELDS = ("name", "slug", "connection", "url", "model", "type", "temperature", "px", "format",
|
|
7137
7531
|
"quality", "options")
|
|
7138
7532
|
LOOKS_LIKE_A_KEY = re.compile(r"^(?:api[-_]?)?key$|^(?:authorization|bearer|secret|password|token)$",
|
|
7139
7533
|
re.IGNORECASE)
|
|
7140
7534
|
PROFILE_ID = re.compile(r"[A-Za-z0-9]+(?:-[A-Za-z0-9]+)?")
|
|
7535
|
+
# A run's table key (evals-core.ts answerKey): a connection's id, then a dot
|
|
7536
|
+
# and its profile's id -- or nothing, for none -- or a grader's bare id.
|
|
7537
|
+
ANSWER_KEY = re.compile(r"[A-Za-z0-9]+(?:-[A-Za-z0-9]+)?(?:\.(?:[A-Za-z0-9]+(?:-[A-Za-z0-9]+)?)?)?")
|
|
7538
|
+
|
|
7539
|
+
|
|
7540
|
+
def answer_key(connection: str, profile=None) -> str:
|
|
7541
|
+
return f"{connection}.{profile or ''}"
|
|
7542
|
+
|
|
7543
|
+
|
|
7544
|
+
def answer_connection(key, conn) -> str:
|
|
7545
|
+
"""The AI Connection a run's entry [key] takes its key from: the one it
|
|
7546
|
+
names, or -- a run from before version 16 -- the key's own id."""
|
|
7547
|
+
cid = conn.get("connection") if isinstance(conn, dict) else None
|
|
7548
|
+
return cid if isinstance(cid, str) and cid else str(key).split(".", 1)[0]
|
|
7549
|
+
|
|
7550
|
+
|
|
7551
|
+
def run_answer(table, key):
|
|
7552
|
+
"""What a run's table holds for [key] (evals-core.ts runAnswer): its own
|
|
7553
|
+
entry, or -- a run from before version 16, keyed by Target profile id --
|
|
7554
|
+
the connection's, where the key names no profile."""
|
|
7555
|
+
if key in table:
|
|
7556
|
+
return table[key]
|
|
7557
|
+
head, dot, tail = key.partition(".")
|
|
7558
|
+
return table.get(head) if dot and not tail else None
|
|
7559
|
+
|
|
7560
|
+
|
|
7561
|
+
# What an AI Connection keeps of a Target profile (evals-core.ts
|
|
7562
|
+
# AI_CONNECTION_FIELDS), beside the settings of its type that say how it is
|
|
7563
|
+
# reached (CONNECTION_SCOPED, the registry's `scope: "connection"`); every
|
|
7564
|
+
# other setting set is a Connection Profile's. Mirrored, and held to the
|
|
7565
|
+
# core's splitProfile, by proxy-check.py.
|
|
7566
|
+
AI_CONNECTION_FIELDS = ("id", "name", "slug", "type", "url", "key", "model", "profile")
|
|
7567
|
+
CONNECTION_SCOPED = {"http": ("auth", "headers", "testPath", "requestFormat")}
|
|
7568
|
+
# evals-core.ts REPLY_TOKENS_BEFORE, and the setting of each type that sent it.
|
|
7569
|
+
REPLY_TOKENS_BEFORE = "160"
|
|
7570
|
+
REPLY_TOKENS_SETTING = {"openai-compatible": "nPredict", "llama.cpp": "nPredict", "anthropic": "maxTokens"}
|
|
7571
|
+
|
|
7572
|
+
|
|
7573
|
+
def pin_reply_tokens(p):
|
|
7574
|
+
"""evals-core.ts pinReplyTokens: a profile with no Reply tokens setting, of
|
|
7575
|
+
a type that sent the cap, has the cap it was sending written in. The split
|
|
7576
|
+
keeps no blank setting, so after it nothing could tell such a profile from
|
|
7577
|
+
one made since; it is pinned here, before."""
|
|
7578
|
+
key = REPLY_TOKENS_SETTING.get(str(p.get("type")))
|
|
7579
|
+
return p if not key or key in p else {**p, key: REPLY_TOKENS_BEFORE}
|
|
7580
|
+
|
|
7581
|
+
|
|
7582
|
+
def split_profile(p):
|
|
7583
|
+
"""A Target profile as (AI Connection, Connection Profile or None): the
|
|
7584
|
+
connection keeps its id, key and address; a profile, stamped with the id
|
|
7585
|
+
it came from, holds every other setting it set (its reply-token cap
|
|
7586
|
+
pinned, pin_reply_tokens)."""
|
|
7587
|
+
p = pin_reply_tokens(p)
|
|
7588
|
+
scoped = CONNECTION_SCOPED.get(p.get("type"), ())
|
|
7589
|
+
conn, settings = {}, {}
|
|
7590
|
+
for k, v in p.items():
|
|
7591
|
+
if k in AI_CONNECTION_FIELDS or k in scoped:
|
|
7592
|
+
conn[k] = v
|
|
7593
|
+
elif v is not None and v != "":
|
|
7594
|
+
settings[k] = v
|
|
7595
|
+
if not settings:
|
|
7596
|
+
return conn, None
|
|
7597
|
+
pid = str(p.get("id") or "")
|
|
7598
|
+
return conn, {"id": pid, "name": str(p.get("name") or ""), "fromProfile": pid, **settings}
|
|
7599
|
+
|
|
7600
|
+
|
|
7601
|
+
def split_profiles(listed):
|
|
7602
|
+
"""A profiles store's list as (connections, connection profiles)."""
|
|
7603
|
+
connections, profiles = [], []
|
|
7604
|
+
for p in listed if isinstance(listed, list) else []:
|
|
7605
|
+
if not isinstance(p, dict):
|
|
7606
|
+
continue
|
|
7607
|
+
conn, prof = split_profile(p)
|
|
7608
|
+
connections.append(conn)
|
|
7609
|
+
if prof is not None:
|
|
7610
|
+
profiles.append(prof)
|
|
7611
|
+
return connections, profiles
|
|
7141
7612
|
# evals-core.ts's SLUG and SLUG_MAX: what a key's variable is spelt from.
|
|
7142
7613
|
PROFILE_SLUG = re.compile(r"[a-z0-9]+(?:-[a-z0-9]+)*")
|
|
7143
7614
|
SLUG_MAX = 32
|
|
@@ -7195,14 +7666,7 @@ def http_endpoint_test(payload, key):
|
|
|
7195
7666
|
name, value = part.split(":", 1)
|
|
7196
7667
|
if name.strip():
|
|
7197
7668
|
headers[name.strip()] = value.strip()
|
|
7198
|
-
|
|
7199
|
-
how = str(settings.get("auth") or "bearer")
|
|
7200
|
-
if how == "x-api-key":
|
|
7201
|
-
headers["x-api-key"] = key
|
|
7202
|
-
elif how == "api-key":
|
|
7203
|
-
headers["api-key"] = key
|
|
7204
|
-
elif how != "none":
|
|
7205
|
-
headers["Authorization"] = "Bearer " + key
|
|
7669
|
+
headers.update(http_key_header(settings.get("auth"), key))
|
|
7206
7670
|
if not all(header_safe(v) for v in headers.values()):
|
|
7207
7671
|
return "", {}, "A header of this profile has characters that cannot be sent."
|
|
7208
7672
|
return origin + (path if path.startswith("/") else "/" + path), headers, ""
|
|
@@ -7218,7 +7682,7 @@ def connection_problems(pid, conn, at, bad):
|
|
|
7218
7682
|
if slug is not None and not (isinstance(slug, str) and PROFILE_SLUG.fullmatch(slug) and len(slug) <= SLUG_MAX):
|
|
7219
7683
|
bad.append(f"{at}: slug has to be lowercase letters and digits, with hyphens between, at most {SLUG_MAX}")
|
|
7220
7684
|
slug = None
|
|
7221
|
-
var = key_var(pid, slug)
|
|
7685
|
+
var = key_var(answer_connection(pid, conn), slug)
|
|
7222
7686
|
_fields(conn, at, CONNECTION_FIELDS, bad, var)
|
|
7223
7687
|
ctype = conn.get("type")
|
|
7224
7688
|
if not isinstance(ctype, str) or ctype not in CONNECTION_TYPES:
|
|
@@ -7364,17 +7828,23 @@ def run_problems(run):
|
|
|
7364
7828
|
spelt = {}
|
|
7365
7829
|
for pid, conn in table.items():
|
|
7366
7830
|
at = f"profile {pid}"
|
|
7367
|
-
if not
|
|
7831
|
+
if not ANSWER_KEY.fullmatch(str(pid)):
|
|
7368
7832
|
bad.append(f"{at}: an id is letters and digits, with at most one hyphen")
|
|
7369
7833
|
continue
|
|
7370
7834
|
if not isinstance(conn, dict):
|
|
7371
7835
|
bad.append(f"{at} has to be an object")
|
|
7372
7836
|
continue
|
|
7373
|
-
|
|
7374
|
-
|
|
7375
|
-
|
|
7376
|
-
|
|
7377
|
-
|
|
7837
|
+
if conn.get("connection") is not None and not (
|
|
7838
|
+
isinstance(conn["connection"], str) and PROFILE_ID.fullmatch(conn["connection"])):
|
|
7839
|
+
bad.append(f"{at}: connection has to be an AI Connection's id")
|
|
7840
|
+
continue
|
|
7841
|
+
# Entries sharing a connection share its key; two connections
|
|
7842
|
+
# spelling one variable would hand one the other's.
|
|
7843
|
+
cid = answer_connection(pid, conn)
|
|
7844
|
+
var = key_var(cid, conn.get("slug") if isinstance(conn.get("slug"), str) else None)
|
|
7845
|
+
if var in spelt and spelt[var] != cid:
|
|
7846
|
+
bad.append(f"profiles {spelt[var]} and {cid} both take their key from ${var}")
|
|
7847
|
+
spelt[var] = cid
|
|
7378
7848
|
connection_problems(pid, conn, at, bad)
|
|
7379
7849
|
targets = run.get("targets")
|
|
7380
7850
|
if not isinstance(targets, list) or not 1 <= len(targets) <= TARGET_CAP:
|
|
@@ -7394,10 +7864,17 @@ def run_problems(run):
|
|
|
7394
7864
|
if not isinstance(steps, list) or len(steps) != len(jobs):
|
|
7395
7865
|
bad.append(f"{at} has to have one step per job")
|
|
7396
7866
|
continue
|
|
7397
|
-
# A target with no
|
|
7398
|
-
# name their own; which steps need one is the core's to judge.
|
|
7399
|
-
|
|
7400
|
-
|
|
7867
|
+
# A target with no connection is one whose steps ask none (Echo) or
|
|
7868
|
+
# each name their own; which steps need one is the core's to judge.
|
|
7869
|
+
# Each pair named -- the target's, a step's own -- is one the run
|
|
7870
|
+
# carries (evals-core.ts targetAnswerOf).
|
|
7871
|
+
def pair(o, base=None):
|
|
7872
|
+
c = o.get("connection") if o.get("connection") is not None else (base or {}).get("connection")
|
|
7873
|
+
p = o.get("profile")
|
|
7874
|
+
return c, p
|
|
7875
|
+
refs = [pair(t)] if t.get("connection") is not None or t.get("profile") is not None else []
|
|
7876
|
+
refs += [pair(st, t) for st in steps
|
|
7877
|
+
if isinstance(st, dict) and (st.get("connection") is not None or st.get("profile") is not None)]
|
|
7401
7878
|
# Whether the words may be blank -- Echo answering from the item --
|
|
7402
7879
|
# is the core's to judge, and the worker refuses the run in its words.
|
|
7403
7880
|
for k, step in enumerate(steps):
|
|
@@ -7408,9 +7885,11 @@ def run_problems(run):
|
|
|
7408
7885
|
elif step.get("from") is not None and not (
|
|
7409
7886
|
isinstance(step["from"], dict) and isinstance(step["from"].get("id"), str)):
|
|
7410
7887
|
bad.append(f"{at}, job {k + 1} names the prompt it was picked from without an id")
|
|
7411
|
-
for
|
|
7412
|
-
if not isinstance(
|
|
7413
|
-
|
|
7888
|
+
for c, p in refs:
|
|
7889
|
+
if (not isinstance(c, dict) or not isinstance(c.get("id"), str)
|
|
7890
|
+
or (p is not None and not (isinstance(p, dict) and isinstance(p.get("id"), str)))
|
|
7891
|
+
or run_answer(table, answer_key(c["id"], p["id"] if p else None)) is None):
|
|
7892
|
+
bad.append(f"{at} names a connection the run does not carry")
|
|
7414
7893
|
content = content_of(run)
|
|
7415
7894
|
kind = content.get("type") if isinstance(content, dict) else None
|
|
7416
7895
|
if kind == "source":
|
|
@@ -7747,7 +8226,7 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
7747
8226
|
return self._send(404, b"not found", "text/plain")
|
|
7748
8227
|
return self._json(200, {"plugins": PLUGINS.list(), "cap": PLUGIN_CAP})
|
|
7749
8228
|
if path == "/api/connections/shares":
|
|
7750
|
-
# Which workspaces may use each
|
|
8229
|
+
# Which workspaces may use each AI Connection and Account
|
|
7751
8230
|
# (docs/workspaces.md): the share sets the Connections menus read.
|
|
7752
8231
|
# Keyed by "<kind>:<id>", workspace ids and the sentinels, no key.
|
|
7753
8232
|
if STORE is None:
|
|
@@ -7773,6 +8252,16 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
7773
8252
|
return self._send(404, b"not found", "text/plain")
|
|
7774
8253
|
ctype = TYPES[".json"] if f.suffix in (".json", ".map") else TYPES[".js"]
|
|
7775
8254
|
return self._send(200, f.read_bytes(), ctype, (("Cache-Control", "no-store"),))
|
|
8255
|
+
if path == "/api/lab-export":
|
|
8256
|
+
# The lab's settings as a file (docs/lab-export.md): every
|
|
8257
|
+
# workspace's entry, or ?workspace=<slug>'s alone. No key.
|
|
8258
|
+
if STORE is None or WORKSPACES is None:
|
|
8259
|
+
return self._send(404, b"not found", "text/plain")
|
|
8260
|
+
only = (urllib.parse.parse_qs(urllib.parse.urlsplit(self.path).query).get("workspace") or [None])[0]
|
|
8261
|
+
out = lab_export(only)
|
|
8262
|
+
if out is None:
|
|
8263
|
+
return self._json(404, {"error": f"no workspace at /w/{only}"})
|
|
8264
|
+
return self._json(200, out)
|
|
7776
8265
|
if path == "/api/packs":
|
|
7777
8266
|
if PACKS is None:
|
|
7778
8267
|
return self._send(404, b"not found", "text/plain")
|
|
@@ -8125,6 +8614,22 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
8125
8614
|
return self._filter_sets_post(path)
|
|
8126
8615
|
if path == "/api/copy-move" or path.startswith("/api/copy-move/"):
|
|
8127
8616
|
return self._copy_move_post(path)
|
|
8617
|
+
if path == "/api/lab-import":
|
|
8618
|
+
# A lab file into this lab (docs/lab-export.md), adding what it
|
|
8619
|
+
# lacks. `keys` is what `evals-lab import` read from its own
|
|
8620
|
+
# $EVALSLAB_API_KEY_<SLUG> variables; the page sends none.
|
|
8621
|
+
if STORE is None or WORKSPACES is None:
|
|
8622
|
+
return self._send(404, b"not found", "text/plain")
|
|
8623
|
+
payload = self._payload()
|
|
8624
|
+
if payload is None:
|
|
8625
|
+
return self._json(400, {"error": "an import is {lab, keys?} as JSON"})
|
|
8626
|
+
keys = payload.get("keys") if isinstance(payload.get("keys"), dict) else {}
|
|
8627
|
+
keys = {k: v for k, v in keys.items()
|
|
8628
|
+
if isinstance(k, str) and k.startswith("EVALSLAB_API_KEY_") and isinstance(v, str)}
|
|
8629
|
+
said, err = lab_import(payload.get("lab"), keys)
|
|
8630
|
+
if err:
|
|
8631
|
+
return self._json(err[0], {"error": err[1]})
|
|
8632
|
+
return self._json(200, {"said": said})
|
|
8128
8633
|
if path == "/api/packs" or path.startswith("/api/packs/"):
|
|
8129
8634
|
return self._packs_post(path)
|
|
8130
8635
|
if path == "/api/workspaces" or path.startswith("/api/workspaces/"):
|
|
@@ -8283,8 +8788,8 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
8283
8788
|
else None. Only a held key (KEY_HELD<id>) names a stored profile; a key
|
|
8284
8789
|
typed into the Setup form is a connection not yet saved, so nothing
|
|
8285
8790
|
gates it. This is the relay/Test seam, beside worker_destinations' run
|
|
8286
|
-
seam
|
|
8287
|
-
places sharing is enforced. The caller writes the 403: a helper that
|
|
8791
|
+
seam and connection_auth's API Source seam -- the places a connection
|
|
8792
|
+
resolves to a key, so the places sharing is enforced. The caller writes the 403: a helper that
|
|
8288
8793
|
answers here would send the body and still fall through to the proxy."""
|
|
8289
8794
|
if STORE is None:
|
|
8290
8795
|
return None
|
|
@@ -8480,7 +8985,7 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
8480
8985
|
if revs[name] is None:
|
|
8481
8986
|
return self._json(400, {"error": f"{name!r} is not in that Source"})
|
|
8482
8987
|
content["revs"] = revs
|
|
8483
|
-
# A
|
|
8988
|
+
# A connection not shared with this workspace blocks the run with the
|
|
8484
8989
|
# sentence the Run bar shows and the connections it names, so its Share
|
|
8485
8990
|
# link can grant them at once (docs/workspaces.md).
|
|
8486
8991
|
ws = workspace_of(STORE) if STORE is not None else None
|
|
@@ -8551,6 +9056,8 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
8551
9056
|
if produce is None:
|
|
8552
9057
|
return None, None, (400, "that provider cannot be produced from on the server")
|
|
8553
9058
|
result = produce(cfg)
|
|
9059
|
+
if result.get("refused"):
|
|
9060
|
+
return None, None, result["refused"]
|
|
8554
9061
|
content = result.get("content")
|
|
8555
9062
|
if content is None:
|
|
8556
9063
|
return None, None, (502, "the live fetch produced nothing: " + (result.get("error") or "unknown"))
|
|
@@ -9001,6 +9508,24 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
9001
9508
|
code, message = err
|
|
9002
9509
|
return self._json(code, {"error": message})
|
|
9003
9510
|
return self._json(200, {"trash": token})
|
|
9511
|
+
if len(parts) == 6 and parts[4] == "calls" and parts[5] in ("edit", "clone"):
|
|
9512
|
+
payload = self._payload() or {}
|
|
9513
|
+
name = source_name(payload.get("name"))
|
|
9514
|
+
if name is None:
|
|
9515
|
+
return self._json(400, {"error": "a call name is not accepted here"})
|
|
9516
|
+
if parts[5] == "clone":
|
|
9517
|
+
call, err = SOURCES.clone_call(parts[3], name)
|
|
9518
|
+
else:
|
|
9519
|
+
to = payload.get("to")
|
|
9520
|
+
if to is not None and not isinstance(to, str):
|
|
9521
|
+
return self._json(400, {"error": "that name is not accepted here", "field": "name"})
|
|
9522
|
+
sid = parts[3]
|
|
9523
|
+
call, err = SOURCES.edit_call(
|
|
9524
|
+
sid, name, to, payload.get("content"), payload.get("origin"),
|
|
9525
|
+
named_by=(lambda item: DATASETS.naming_item(sid, item)) if DATASETS else None)
|
|
9526
|
+
if err:
|
|
9527
|
+
return self._json(err[0], {"error": err[1], **({"field": err[2]} if len(err) > 2 else {})})
|
|
9528
|
+
return self._json(201 if parts[5] == "clone" else 200, {"call": call})
|
|
9004
9529
|
if len(parts) == 5 and parts[4] == "copy":
|
|
9005
9530
|
payload = self._payload() or {}
|
|
9006
9531
|
source, err = SOURCES.copy_files(parts[3], payload.get("from"),
|
|
@@ -9150,6 +9675,252 @@ class Handler(BaseHTTPRequestHandler):
|
|
|
9150
9675
|
return self._json(502, {"error": f"{url}: {e.reason}"})
|
|
9151
9676
|
|
|
9152
9677
|
|
|
9678
|
+
# ---- Lab export and import (docs/lab-export.md) -----------------------------
|
|
9679
|
+
#
|
|
9680
|
+
# A lab's settings as one file: its AI Connections, Connection Profiles and
|
|
9681
|
+
# grader, and its workspaces, so a new lab can start as another one is set up
|
|
9682
|
+
# -- by Setup › Workspaces › Data, by `evals-lab export`/`import`, or on every
|
|
9683
|
+
# start with LAB_IMPORT (a demo lab that keeps no data). An import only adds:
|
|
9684
|
+
# what the lab already has, by a connection's or profile's id and a
|
|
9685
|
+
# workspace's address, is left as it is, so importing twice changes nothing
|
|
9686
|
+
# and a lab that keeps its data keeps its edits. The file never holds a key
|
|
9687
|
+
# (#258): each comes from $EVALSLAB_API_KEY_<SLUG>, as a CI run's does, and a
|
|
9688
|
+
# key that comes is the connection's from then on, a rotated one included.
|
|
9689
|
+
# A connection carries its share set (`shares`, workspace slugs and the
|
|
9690
|
+
# sentinels; absent is All) from version 2: version 1 carried none, so an
|
|
9691
|
+
# import reads its connections as All.
|
|
9692
|
+
|
|
9693
|
+
LAB_FORMAT = "evals-lab/lab"
|
|
9694
|
+
LAB_VERSIONS = (1, 2)
|
|
9695
|
+
LAB_FIELDS = ("format", "version", "exportedAt", "connections", "connectionProfiles", "grader", "workspaces")
|
|
9696
|
+
|
|
9697
|
+
|
|
9698
|
+
def lab_export(only=None) -> dict:
|
|
9699
|
+
"""The lab's settings as a file, every workspace's entry or the one at
|
|
9700
|
+
/w/[only]; None when no workspace is at that address. One workspace's file
|
|
9701
|
+
carries only what that workspace can use: the connections shared with it,
|
|
9702
|
+
the profiles over them (or over none) and the grader if it is one of them,
|
|
9703
|
+
each share set narrowed to that workspace, the one the file names, and
|
|
9704
|
+
*new kept where the connection has it. A connection shared only with
|
|
9705
|
+
workspaces in the trash is usable nowhere, and an empty share set would
|
|
9706
|
+
import as All, so it is left out, and out as the grader too."""
|
|
9707
|
+
body = (STORE.all().get("promptlab.connections") or {}).get("body") or {}
|
|
9708
|
+
profiles = (STORE.all().get("promptlab.connectionProfiles") or {}).get("body") or {}
|
|
9709
|
+
every = WORKSPACES.list()
|
|
9710
|
+
spaces = [w for w in every if not only or w["slug"] == only]
|
|
9711
|
+
if only and not spaces:
|
|
9712
|
+
return None
|
|
9713
|
+
slug_of = {w["id"]: w["slug"] for w in (spaces if only else every)}
|
|
9714
|
+
rows = STORE.shares()
|
|
9715
|
+
conns, nowhere = [], set()
|
|
9716
|
+
for c in body.get("list") or []:
|
|
9717
|
+
if not isinstance(c, dict) or (only and not STORE.shared("profile", str(c.get("id") or ""), spaces[0]["id"])):
|
|
9718
|
+
continue
|
|
9719
|
+
entry = {k: v for k, v in c.items() if k not in ("key", "shares")}
|
|
9720
|
+
held = rows.get(f"profile:{c.get('id')}")
|
|
9721
|
+
if held and SHARE_ALL not in held:
|
|
9722
|
+
entry["shares"] = [w if w == SHARE_NEW else slug_of[w] for w in held if w == SHARE_NEW or w in slug_of]
|
|
9723
|
+
if not entry["shares"]:
|
|
9724
|
+
nowhere.add(c.get("id"))
|
|
9725
|
+
continue
|
|
9726
|
+
conns.append(entry)
|
|
9727
|
+
kept = {c["id"] for c in conns}
|
|
9728
|
+
|
|
9729
|
+
def over_kept(p):
|
|
9730
|
+
ref = p.get("connection")
|
|
9731
|
+
return not ref or not only or (isinstance(ref, dict) and ref.get("id") in kept)
|
|
9732
|
+
out = {"format": LAB_FORMAT, "version": LAB_VERSIONS[-1],
|
|
9733
|
+
"exportedAt": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
|
9734
|
+
"connections": conns,
|
|
9735
|
+
"connectionProfiles": [p for p in profiles.get("list") or [] if isinstance(p, dict) and over_kept(p)],
|
|
9736
|
+
"workspaces": [{"slug": w["slug"], "name": w["name"], **({"archived": True} if w["archived"] else {})}
|
|
9737
|
+
for w in spaces]}
|
|
9738
|
+
if (isinstance(body.get("grader"), str) and body["grader"] and body["grader"] not in nowhere
|
|
9739
|
+
and (not only or body["grader"] in kept)):
|
|
9740
|
+
out["grader"] = body["grader"]
|
|
9741
|
+
return out
|
|
9742
|
+
|
|
9743
|
+
|
|
9744
|
+
def _keyless(entry, what):
|
|
9745
|
+
"""A sentence when [entry] holds a key, which a lab file never does."""
|
|
9746
|
+
for k, v in entry.items():
|
|
9747
|
+
if LOOKS_LIKE_A_KEY.match(k) and v not in ("", None):
|
|
9748
|
+
return f"{what} holds a {k}; a lab file holds no key -- set ${key_var(entry['id'], entry.get('slug'))}"
|
|
9749
|
+
return None
|
|
9750
|
+
|
|
9751
|
+
|
|
9752
|
+
def lab_problems(doc) -> str:
|
|
9753
|
+
"""Why [doc] is not a lab file this lab imports, or None. Checked whole
|
|
9754
|
+
before anything is written, as a pack is."""
|
|
9755
|
+
if not isinstance(doc, dict) or doc.get("format") != LAB_FORMAT:
|
|
9756
|
+
return f"not a lab file: its format is not {LAB_FORMAT}"
|
|
9757
|
+
if doc.get("version") not in LAB_VERSIONS:
|
|
9758
|
+
return f"a lab file of version {doc.get('version')!r}; this lab reads {', '.join(map(str, LAB_VERSIONS))}"
|
|
9759
|
+
extra = sorted(set(doc) - set(LAB_FIELDS))
|
|
9760
|
+
if extra:
|
|
9761
|
+
return f"a lab file holds no {', '.join(extra)}"
|
|
9762
|
+
seen = set()
|
|
9763
|
+
for i, c in enumerate(doc.get("connections") or []):
|
|
9764
|
+
what = f"connection {i + 1}"
|
|
9765
|
+
if not isinstance(c, dict) or not isinstance(c.get("id"), str) or not PROFILE_ID.fullmatch(c["id"]):
|
|
9766
|
+
return f"{what} has no id of letters and digits"
|
|
9767
|
+
what = f"connection {c['id']}"
|
|
9768
|
+
if c["id"] in seen:
|
|
9769
|
+
return f"{what} is listed twice"
|
|
9770
|
+
seen.add(c["id"])
|
|
9771
|
+
if not isinstance(c.get("name"), str) or not c["name"].strip():
|
|
9772
|
+
return f"{what} has no name"
|
|
9773
|
+
if c.get("type") not in CONNECTION_TYPES:
|
|
9774
|
+
return f"{what} is of type {c.get('type')!r}, which this lab has no connection type for"
|
|
9775
|
+
bad = next((k for k, v in c.items() if not isinstance(v, str) and k != "shares"), None)
|
|
9776
|
+
if "shares" in c and doc["version"] < 2:
|
|
9777
|
+
bad = "shares"
|
|
9778
|
+
if bad:
|
|
9779
|
+
return f"{what}'s {bad} is not text"
|
|
9780
|
+
why = _keyless(c, what)
|
|
9781
|
+
if why:
|
|
9782
|
+
return why
|
|
9783
|
+
seen = set()
|
|
9784
|
+
for i, p in enumerate(doc.get("connectionProfiles") or []):
|
|
9785
|
+
what = f"connection profile {i + 1}"
|
|
9786
|
+
if not isinstance(p, dict) or not isinstance(p.get("id"), str) or not PROFILE_ID.fullmatch(p["id"]):
|
|
9787
|
+
return f"{what} has no id of letters and digits"
|
|
9788
|
+
what = f"connection profile {p['id']}"
|
|
9789
|
+
if p["id"] in seen:
|
|
9790
|
+
return f"{what} is listed twice"
|
|
9791
|
+
seen.add(p["id"])
|
|
9792
|
+
if not isinstance(p.get("name"), str) or not p["name"].strip():
|
|
9793
|
+
return f"{what} has no name"
|
|
9794
|
+
why = _keyless(p, what)
|
|
9795
|
+
if why:
|
|
9796
|
+
return why
|
|
9797
|
+
if "grader" in doc and not isinstance(doc["grader"], str):
|
|
9798
|
+
return "the grader is not a connection's id"
|
|
9799
|
+
seen = set()
|
|
9800
|
+
for i, w in enumerate(doc.get("workspaces") or []):
|
|
9801
|
+
what = f"workspace {i + 1}"
|
|
9802
|
+
if not isinstance(w, dict) or not isinstance(w.get("slug"), str) or not w["slug"]:
|
|
9803
|
+
return f"{what} has no address"
|
|
9804
|
+
if w["slug"] in seen:
|
|
9805
|
+
return f"workspace {w['slug']} is listed twice"
|
|
9806
|
+
seen.add(w["slug"])
|
|
9807
|
+
if not isinstance(w.get("name"), str) or not w["name"].strip():
|
|
9808
|
+
return f"workspace {w['slug']} has no name"
|
|
9809
|
+
if Workspaces._slugify(w["slug"]) != w["slug"]:
|
|
9810
|
+
return f"workspace {w['slug']}'s address is not one a lab makes: it would be {Workspaces._slugify(w['slug'])}"
|
|
9811
|
+
fields = {"slug", "name"} | ({"archived"} if doc["version"] >= 2 else set())
|
|
9812
|
+
if set(w) - fields:
|
|
9813
|
+
return f"workspace {w['slug']} holds no {', '.join(sorted(set(w) - fields))}"
|
|
9814
|
+
if "archived" in w and not isinstance(w["archived"], bool):
|
|
9815
|
+
return f"workspace {w['slug']}'s archived is not true or false"
|
|
9816
|
+
for k in ("connections", "connectionProfiles", "workspaces"):
|
|
9817
|
+
if k in doc and not isinstance(doc[k], list):
|
|
9818
|
+
return f"a lab file's {k} is a list"
|
|
9819
|
+
# Workspaces before shares: a share names the file's workspaces or the lab's.
|
|
9820
|
+
known = seen | ({w["slug"] for w in WORKSPACES.list()} if WORKSPACES is not None else set())
|
|
9821
|
+
for c in doc.get("connections") or []:
|
|
9822
|
+
if "shares" not in c:
|
|
9823
|
+
continue
|
|
9824
|
+
if not isinstance(c["shares"], list) or not all(isinstance(s, str) for s in c["shares"]):
|
|
9825
|
+
return f"connection {c['id']}'s shares is a list of workspace addresses"
|
|
9826
|
+
# No rows is All in the store, so an empty list would grant everywhere.
|
|
9827
|
+
if not c["shares"]:
|
|
9828
|
+
return (f"connection {c['id']}'s shares is empty; leave shares out for All, "
|
|
9829
|
+
"or name a workspace or *new")
|
|
9830
|
+
lost = [s for s in c["shares"] if s not in (SHARE_ALL, SHARE_NEW) and s not in known]
|
|
9831
|
+
if lost:
|
|
9832
|
+
return f"connection {c['id']} is shared with {', '.join(lost)}, which neither the file nor this lab has"
|
|
9833
|
+
return None
|
|
9834
|
+
|
|
9835
|
+
|
|
9836
|
+
def lab_import(doc, keys) -> tuple:
|
|
9837
|
+
"""[doc] into this lab, adding what it lacks; [keys] is where each
|
|
9838
|
+
connection's key is looked up by its variable's name ($EVALSLAB_API_KEY_
|
|
9839
|
+
<SLUG>) -- the server's environment as it starts, or what `evals-lab
|
|
9840
|
+
import` sent. Returns (lines saying what happened, None) or (None, error).
|
|
9841
|
+
No line holds a key."""
|
|
9842
|
+
why = lab_problems(doc)
|
|
9843
|
+
if why:
|
|
9844
|
+
return None, (400, why)
|
|
9845
|
+
said = []
|
|
9846
|
+
# A file's address to this lab's workspace id; one added where the trash
|
|
9847
|
+
# holds its address lands at the next free one, and its shares follow it.
|
|
9848
|
+
wid_of = {w["slug"]: w["id"] for w in WORKSPACES.list()}
|
|
9849
|
+
for w in doc.get("workspaces") or []:
|
|
9850
|
+
if w["slug"] in wid_of:
|
|
9851
|
+
continue
|
|
9852
|
+
wid, err = WORKSPACES.create(w["name"], w["slug"])
|
|
9853
|
+
wid_of[w["slug"]] = wid
|
|
9854
|
+
if not err and w.get("archived"):
|
|
9855
|
+
_, err = WORKSPACES.archive(wid)
|
|
9856
|
+
if err:
|
|
9857
|
+
return None, err
|
|
9858
|
+
said.append(f"workspace {w['name']} added at /w/{WORKSPACES.get(wid)['slug']}"
|
|
9859
|
+
+ (", archived" if w.get("archived") else ""))
|
|
9860
|
+
for _ in range(3):
|
|
9861
|
+
docs = STORE.all()
|
|
9862
|
+
cur = docs.get("promptlab.connections") or {"version": 0, "body": None}
|
|
9863
|
+
pcur = docs.get("promptlab.connectionProfiles") or {"version": 0, "body": None}
|
|
9864
|
+
body = json.loads(json.dumps(cur["body"] or {"list": [], "at": ""}))
|
|
9865
|
+
pbody = json.loads(json.dumps(pcur["body"] or {"list": []}))
|
|
9866
|
+
conns = body.setdefault("list", [])
|
|
9867
|
+
lines, changed, pchanged, added = [], False, False, []
|
|
9868
|
+
for c in doc.get("connections") or []:
|
|
9869
|
+
mine = next((x for x in conns if isinstance(x, dict) and x.get("id") == c["id"]), None)
|
|
9870
|
+
if mine is None:
|
|
9871
|
+
mine = {**{k: v for k, v in c.items() if k != "shares"}, "key": ""}
|
|
9872
|
+
added.append(c)
|
|
9873
|
+
conns.append(mine)
|
|
9874
|
+
changed = True
|
|
9875
|
+
lines.append(f"connection {c['name']} added")
|
|
9876
|
+
key = str(keys.get(key_var(c["id"], c.get("slug"))) or "").strip()
|
|
9877
|
+
if key and mine.get("key") != key:
|
|
9878
|
+
if not header_safe(key):
|
|
9879
|
+
return None, (400, f"${key_var(c['id'], c.get('slug'))} cannot go in a header")
|
|
9880
|
+
mine["key"] = key
|
|
9881
|
+
changed = True
|
|
9882
|
+
lines.append(f"connection {mine.get('name') or c['name']} takes its key from "
|
|
9883
|
+
f"${key_var(c['id'], c.get('slug'))}")
|
|
9884
|
+
profs = pbody.setdefault("list", [])
|
|
9885
|
+
for p in doc.get("connectionProfiles") or []:
|
|
9886
|
+
if not any(isinstance(x, dict) and x.get("id") == p["id"] for x in profs):
|
|
9887
|
+
profs.append(dict(p))
|
|
9888
|
+
pchanged = True
|
|
9889
|
+
lines.append(f"connection profile {p['name']} added")
|
|
9890
|
+
if doc.get("grader") and not body.get("grader"):
|
|
9891
|
+
body["grader"] = doc["grader"]
|
|
9892
|
+
changed = True
|
|
9893
|
+
lines.append(f"grader set to {doc['grader']}")
|
|
9894
|
+
# Only an added connection takes the file's share set: one the lab had
|
|
9895
|
+
# keeps its own, as everything else it had does. Set before the write,
|
|
9896
|
+
# so the connection is never there and shared with All in between; a
|
|
9897
|
+
# failed write puts back what the rows were, since its ids never landed.
|
|
9898
|
+
before = STORE.shares()
|
|
9899
|
+
for c in added:
|
|
9900
|
+
if "shares" in c:
|
|
9901
|
+
STORE.set_shares("profile", c["id"], [s if s in (SHARE_ALL, SHARE_NEW) else wid_of[s]
|
|
9902
|
+
for s in c["shares"]])
|
|
9903
|
+
writes = {}
|
|
9904
|
+
if changed:
|
|
9905
|
+
body["at"] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
|
9906
|
+
writes["promptlab.connections"] = {"version": cur["version"], "body": body}
|
|
9907
|
+
if pchanged:
|
|
9908
|
+
writes["promptlab.connectionProfiles"] = {"version": pcur["version"], "body": pbody}
|
|
9909
|
+
if not writes:
|
|
9910
|
+
break
|
|
9911
|
+
# Global documents, so the default workspace's write reaches them.
|
|
9912
|
+
ok, _ = STORE.write(writes, STORE.default_ws)
|
|
9913
|
+
if ok:
|
|
9914
|
+
break
|
|
9915
|
+
for c in added:
|
|
9916
|
+
if "shares" in c:
|
|
9917
|
+
STORE.set_shares("profile", c["id"], before.get(f"profile:{c['id']}") or [])
|
|
9918
|
+
else:
|
|
9919
|
+
return None, (409, "the lab's connections kept changing while they were imported; import again")
|
|
9920
|
+
said += lines
|
|
9921
|
+
return said or ["nothing to add"], None
|
|
9922
|
+
|
|
9923
|
+
|
|
9153
9924
|
def main():
|
|
9154
9925
|
if not (WEB_DIST / "index.html").is_file():
|
|
9155
9926
|
raise SystemExit(f"no {WEB_DIST / 'index.html'} -- the page is built from web/: "
|
|
@@ -9161,6 +9932,11 @@ def main():
|
|
|
9161
9932
|
# package, not one without the app: said at once, not at Sign in.
|
|
9162
9933
|
if PUBLIC_GOOGLE_PROBLEM:
|
|
9163
9934
|
raise SystemExit(PUBLIC_GOOGLE_PROBLEM)
|
|
9935
|
+
# A lab file named and not there is a broken deployment, said at once.
|
|
9936
|
+
if LAB_IMPORT and not Path(LAB_IMPORT).is_file():
|
|
9937
|
+
raise SystemExit(f"LAB_IMPORT names {LAB_IMPORT}, which is not a file")
|
|
9938
|
+
if LAB_IMPORT and STORE is None:
|
|
9939
|
+
raise SystemExit("LAB_IMPORT needs DATA_DIR: a lab file's settings are kept in its store")
|
|
9164
9940
|
# A root that is not there would refuse every File Source -- said at once.
|
|
9165
9941
|
if LOCAL_ROOT and not Path(LOCAL_ROOT).is_dir():
|
|
9166
9942
|
raise SystemExit(f"SOURCE_LOCAL_ROOT names {LOCAL_ROOT}, which is not a directory")
|
|
@@ -9229,6 +10005,24 @@ def main():
|
|
|
9229
10005
|
# The drop-in folder: a pack put there is installed as the lab starts.
|
|
9230
10006
|
for line in PACKS.install_dropins(Path(DATA_DIR) / "packs"):
|
|
9231
10007
|
print(f"pack: {line}", flush=True)
|
|
10008
|
+
if LAB_IMPORT:
|
|
10009
|
+
# After the demo: a lab file adds settings, never datasets, so the
|
|
10010
|
+
# demo still finds the new lab it installs into.
|
|
10011
|
+
try:
|
|
10012
|
+
doc = json.loads(Path(LAB_IMPORT).read_text("utf-8"))
|
|
10013
|
+
except (OSError, ValueError) as e:
|
|
10014
|
+
raise SystemExit(f"LAB_IMPORT: {LAB_IMPORT} is not JSON ({e})") from None
|
|
10015
|
+
said, err = lab_import(doc, os.environ)
|
|
10016
|
+
if err:
|
|
10017
|
+
raise SystemExit(f"LAB_IMPORT: {err[1]}")
|
|
10018
|
+
for line in said:
|
|
10019
|
+
print(f"import: {line}", flush=True)
|
|
10020
|
+
# A held key goes wherever a submitted run points it, so a lab open to
|
|
10021
|
+
# its network with keys in it is as loud as one with a Local provider on.
|
|
10022
|
+
if local_exposed() and any(os.environ.get(key_var(c["id"], c.get("slug")))
|
|
10023
|
+
for c in doc.get("connections") or []):
|
|
10024
|
+
print(f"WARNING: LAB_IMPORT gave a connection its key, and {HOST} is beyond loopback with no "
|
|
10025
|
+
"LAB_PASSWORD -- anyone who can reach this port can run against it", flush=True)
|
|
9232
10026
|
if QUEUE is not None:
|
|
9233
10027
|
QUEUE.start()
|
|
9234
10028
|
ThreadingHTTPServer((HOST, PORT), Handler).serve_forever()
|