evals-lab 0.1.4 → 0.2.0

package/lab/run-evals.js

@@ -453,21 +453,47 @@ let seconds = REQUEST_CAP; // the run's per-request cap, set in main()
 // or, over Prompt only, the prompt -- and the stage reads it as it would a
 // model's.
 //
-// A call that asks for a whole request (an HTTP Request, #flows) builds it
-// from its step, the scenario's cell and the item's record, and is handed
-// only its words: the transport is where the three meet.
+// A target step that asks for a whole request (an HTTP Request, #flows)
+// builds it from the step, the job's flow step and the item's record, and is
+// handed only its words: the transport is where the three meet.
+//
+// Any other step's reply, in a job that reads the flow's step through a Read
+// as, is read the same way: put in the body the flow's API would have sent
+// it in, then read as the flow reads it -- so a model asked in words and
+// production are compared alike (docs/pipeline-model.md §16 › Targets).
 function callsFor(connections, links, text, plan, record) {
   return connections.map((c, k) => {
     const step = plan?.calls?.[k];
     if (core.STEP_TYPES[step?.type]?.asks === "request") {
       return () => send(links[k], c, step, plan.cells[k], record ?? textRecord(text));
     }
-    return core.CONNECTION_TYPES[core.typeOf(c)]?.local
+    const call = core.CONNECTION_TYPES[core.typeOf(c)]?.local
       ? async sent => core.localAnswer(c, text, sent)
       : (sent, url) => ask(links[k], c, sent, url);
+    if (!step?.readAs || !step?.step) return call;
+    return async (sent, url) => {
+      const r = await call(sent, url);
+      if (r.error || r.raw == null) return r;
+      try {
+        return { ...r, ...core.readFlowReply(step, r.raw, record ?? textRecord(text)) };
+      } catch (e) {
+        return { ...r, error: `the reply could not be read as the flow reads it -- ${e.message}` };
+      }
+    };
   });
 }
 
+// What an expression token reads: the item's record, or a text item read as
+// one, and the loop job 1's flow step sits in.
+const tokenRecord = (run, record, text) => {
+  const r = record ?? (text != null ? textRecord(text) : null);
+  return r && { scope: r.scope || {}, at: r.at ?? null, loop: core.flowStepOf(run.jobs[0])?.loop ?? null };
+};
+
+// A prompt as a report restates it: an expression token reads an item's
+// record, and a report has none to name, so its words stay as written.
+const shown = (text, tokens) => { try { return core.resolvePrompt(text, tokens); } catch { return text; } };
+
 /** A Setup profile the run carries, asked as a grader: its reply's text, or
     the failure as an error the metric reports. One transport per profile. */
 const graders = new WeakMap();
@@ -590,7 +616,7 @@ function readUtf8(file) {
 // the page reads it: a whole-run test's verdict, settled once every item is
 // in, and a per-item test's counts. [settled] is false for a run that
 // stopped short, whose whole-run tests have not settled.
-const verdictsOf = (run, items, settled) => run.scenarios.map((_, i) =>
+const verdictsOf = (run, items, settled) => core.targetsOf(run).map((_, i) =>
   Object.fromEntries(core.scenarioTests(run, i, items, settled).map(o => [o.id, {
     name: o.label, skipped: o.skipped,
     ...(o.whole
@@ -600,7 +626,7 @@ const verdictsOf = (run, items, settled) => run.scenarios.map((_, i) =>
 
 // The run as the report restates it: each scenario's stages with the
 // connection each one asked.
-const scenariosOf = run => run.scenarios.map((sc, i) => {
+const scenariosOf = run => core.targetsOf(run).map((sc, i) => {
   const { stages, connections } = core.stagesFor(run, i);
   return {
     n: i + 1, name: sc.name || null,
@@ -649,7 +675,7 @@ async function runSnapshot(o) {
   // Each scenario once: its stages, the jobs' token sets, and a transport
   // per stage to the profile that stage resolves to, keyed by that
   // profile's id.
-  const plans = run.scenarios.map((_, i) => {
+  const plans = core.targetsOf(run).map((_, i) => {
     const { stages, tokens, connections, calls, cells } = core.stagesFor(run, i);
     const links = connections.map(c => reach(c, core.keyVar(c.id)));
     return { stages, tokens, connections, links, calls, cells };
@@ -704,7 +730,7 @@ async function runSnapshot(o) {
       const textOf = item.kind === "text" && !item.bare
         ? (item.text != null ? item.text : readUtf8(path.join(o.source, item.name))) : null;
       const res = await core.runPipeline(plan.stages, dataUrl, callsFor(plan.connections, plan.links, textOf, plan, record),
-                                         { tokens: plan.tokens, text: textOf });
+                                         { tokens: plan.tokens, text: textOf, record: tokenRecord(run, record, textOf) });
       // A case is found by its file's name, whatever kind of file it is: a
       // text item a dataset grades is graded like an image.
       const kase = item.name ? graded.get(item.name) ?? null : null;
@@ -822,14 +848,14 @@ function cliRun(o, dataset) {
   doc.jobs[0] = core.withOut(doc.jobs[0], core.legacyTagsOut(rulesOf(dataset), "LOWER"));
   if (o.tokens) {
     try {
-      doc.jobs[0] = core.withCall(doc.jobs[0], { tokenMappings: JSON.parse(fs.readFileSync(o.tokens, "utf8")) });
+      doc.jobs[0] = core.withTokens(doc.jobs[0], JSON.parse(fs.readFileSync(o.tokens, "utf8")));
     } catch (e) {
       broken(`${o.tokens}: ${e.message}`);
     }
   }
   doc = core.withContent(doc, { type: "source", ref: { id: "cli", name: "" }, first: null, loops: 1, files: [] });
-  doc.scenarios = [{ id: "cli", name: "", profile: { id: "cli", name: "" },
-                     stages: [{ prompt: o.prompt ?? "" }] }];
+  doc.targets = [{ id: "cli", name: "", profile: { id: "cli", name: "" },
+                   steps: [{ type: "prompt", prompt: o.prompt ?? "" }] }];
   doc.tests = [{ id: "cli", type: "metrics", name: "", continueOnFailure: true, dataset: { id: "cli", name: "" },
                  mode: "all", threshold: null, grader: null, over: "item", metrics: [{ type: "case" }] }];
   doc.profiles = { cli: { name: null, url: o.url, model: o.model, type: "openai-compatible" } };
@@ -881,7 +907,7 @@ async function main() {
   }
   if (o.run) {
     if (o.pipeline || o.prompt != null || o.tokens || o.model || o.url || o.replies || o.samples || o.files) {
-      broken("--run names everything a run takes -- scenarios, files and text -- so it "
+      broken("--run names everything a run takes -- targets, files and text -- so it "
            + "cannot be given with --pipeline, --prompt, --tokens, --model, --url, --replies, --samples or --files.");
     }
     const n = Number(o.timeout ?? REQUEST_CAP);
@@ -920,8 +946,8 @@ async function main() {
   }
   const run = o.pipeline ? readRun(o.pipeline, dataset) : cliRun(o, dataset);
   if (o.pipeline) {
-    if (run.scenarios.length !== 1) {
-      broken(`${o.pipeline} has ${run.scenarios.length} scenarios, and --pipeline grades one `
+    if (core.targetsOf(run).length !== 1) {
+      broken(`${o.pipeline} has ${core.targetsOf(run).length} targets, and --pipeline grades one `
            + "against the set -- --run runs them all.");
     }
     if (!core.testsDataset(run)) {
@@ -934,7 +960,7 @@ async function main() {
   // Case by case: each item against its case, as the case metric of the
   // test that names the dataset reads it (scoreCase).
   const { stages, tokens, connections } = core.stagesFor(run, 0);
-  const prompt = core.resolvePrompt(stages[0].text, core.tokenSet(tokens, 0));
+  const prompt = shown(stages[0].text, core.tokenSet(tokens, 0));
 
   // The graded half, through the function the tab builds its own list with.
   const set = core.gradedSetFrom(dataset);
@@ -1170,7 +1196,7 @@ async function main() {
       stages: stages.map((s, i) => {
         const { id, ...connection } = connections[i];
         return { n: i + 1, kind: s.kind, withImage: s.withImage,
-                 prompt: core.resolvePrompt(s.text, core.tokenSet(tokens, i)), profile: id, connection };
+                 prompt: shown(s.text, core.tokenSet(tokens, i)), profile: id, connection };
       }),
     } } : {}),
     // Which variable a key came from, never the key.

package/lab/server.py

@@ -1476,7 +1476,7 @@ def scenario_ref(sc, i):
     what version 4's upgrade gives one, by position."""
     sid = sc.get("id") if isinstance(sc, dict) else None
     name = (sc.get("name") or "").strip() if isinstance(sc, dict) and isinstance(sc.get("name"), str) else ""
-    return (sid if isinstance(sid, str) and sid else f"s{i + 1}"), (name or f"Scenario {i + 1}")
+    return (sid if isinstance(sid, str) and sid else f"s{i + 1}"), (name or f"Target {i + 1}")
 
 
 class Prompts:
@@ -1632,27 +1632,28 @@ class Prompts:
         recorded once: a second call for the same run changes nothing."""
         # The caller's connection: a Row still reads by index, as it does.
         db.row_factory = sqlite3.Row
-        for i, sc in enumerate(run.get("scenarios") or []):
-            if not isinstance(sc, dict):
-                continue
+        for i, sc, k, cell in cells_of(run):
             sid, sname = scenario_ref(sc, i)
-            for k, cell in enumerate(sc.get("stages") or []):
-                text = cell.get("prompt") if isinstance(cell, dict) else None
-                if not isinstance(text, str) or not text.strip():
-                    continue
-                pid = version = None
-                ref = cell.get("from")
-                if isinstance(ref, dict) and isinstance(ref.get("id"), str) and self._live(db, ref["id"]):
-                    pid = ref["id"]
-                    hit = db.execute("SELECT version FROM prompt_versions WHERE prompt_id = ? AND text = ? "
-                                     "ORDER BY version DESC LIMIT 1", (pid, text)).fetchone()
-                    version = hit[0] if hit else self._cut(db, pid, text)
-                if pid is None:
-                    hit = self._matching(db, text)
-                    pid, version = (hit[0], hit[1]) if hit else (self._insert(db, "", text), 1)
-                db.execute("INSERT OR IGNORE INTO prompt_uses (prompt_id, version, run_id, scenario_id, "
-                           "scenario_name, job, at) VALUES (?, ?, ?, ?, ?, ?, ?)",
-                           (pid, version, rid, sid, sname, k, at))
+            # Echo's words are no wording under test: the item is its reply,
+            # and over Prompt only the words are the reply itself.
+            if isinstance(cell, dict) and cell.get("type") in UNRECORDED_STEPS:
+                continue
+            text = cell.get("prompt") if isinstance(cell, dict) else None
+            if not isinstance(text, str) or not text.strip():
+                continue
+            pid = version = None
+            ref = cell.get("from")
+            if isinstance(ref, dict) and isinstance(ref.get("id"), str) and self._live(db, ref["id"]):
+                pid = ref["id"]
+                hit = db.execute("SELECT version FROM prompt_versions WHERE prompt_id = ? AND text = ? "
+                                 "ORDER BY version DESC LIMIT 1", (pid, text)).fetchone()
+                version = hit[0] if hit else self._cut(db, pid, text)
+            if pid is None:
+                hit = self._matching(db, text)
+                pid, version = (hit[0], hit[1]) if hit else (self._insert(db, "", text), 1)
+            db.execute("INSERT OR IGNORE INTO prompt_uses (prompt_id, version, run_id, scenario_id, "
+                       "scenario_name, job, at) VALUES (?, ?, ?, ?, ?, ?, ?)",
+                       (pid, version, rid, sid, sname, k, at))
 
     def backfill(self, runs):
         """The runs from before the library, read into it once. [runs] is
@@ -2577,16 +2578,56 @@ def registered_ids(manifest: dict) -> set:
 # ---- Connections: the lab's grants to outside services (#127) --------------
 #
 # One Google grant per lab, for Sources that read a Drive folder. The OAuth
-# client is the deployment's (env vars), never the repository's or the
-# store's; the refresh token the grant yields is the store's, in a table of
-# its own, so /api/state -- which serves the synced documents to the page --
-# can never carry it. The page is told only whether there is a grant and
-# whose. The hosts are fixed here, not chosen by anything a request carries,
-# and reached through OPENER: no redirects, no proxy from the environment.
+# client it signs in with is the deployment's (GOOGLE_* below), then the one
+# entered in Setup (GoogleApp), then the published app (PUBLIC_GOOGLE_APP),
+# as Microsoft's is (docs/power-automate.md). The refresh token the grant
+# yields is the store's, in a table of its own, so /api/state -- which serves
+# the synced documents to the page -- can never carry it, and nor can the
+# client's secret. The page is told only whether there is a grant and whose.
+# The hosts are fixed here, not chosen by anything a request carries, and
+# reached through OPENER: no redirects, no proxy from the environment.
 GOOGLE_CLIENT_ID = os.environ.get("GOOGLE_CLIENT_ID") or None
 GOOGLE_CLIENT_SECRET = os.environ.get("GOOGLE_CLIENT_SECRET") or None
 GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY") or None
-GOOGLE_APP_ID = os.environ.get("GOOGLE_APP_ID") or None
+# A deployment reached at its own address signs in with a Web application
+# client, whose redirect it registers; "installed" is Google's Desktop app.
+GOOGLE_CLIENT_TYPE = os.environ.get("GOOGLE_CLIENT_TYPE") or "web"
+GOOGLE_CLIENT_TYPES = ("installed", "web")
+# An OAuth client's id is its project's number, a dash, and Google's own
+# suffix: the number is the app id the Picker is told, so it is never asked.
+GOOGLE_CLIENT = re.compile(r"(\d+)-[0-9a-z]+\.apps\.googleusercontent\.com")
+GOOGLE_KEY = re.compile(r"[A-Za-z0-9_-]{30,60}")
+
+
+def read_published_google(path):
+    """The published app from the file the npm package's build writes
+    beside server.py: (app, None), (None, None) when there is no file, or
+    (None, why) for a file that is not one."""
+    if not path.is_file():
+        return None, None
+    try:
+        got = json.loads(path.read_text("utf-8"))
+    except (OSError, ValueError):
+        return None, f"{path} is not JSON"
+    if not (isinstance(got, dict) and isinstance(got.get("clientId"), str)
+            and GOOGLE_CLIENT.fullmatch(got["clientId"]) and isinstance(got.get("clientSecret"), str)
+            and got["clientSecret"] and isinstance(got.get("apiKey"), str) and GOOGLE_KEY.fullmatch(got["apiKey"])):
+        return None, f"{path} is not a Google app: {{ clientId, clientSecret, apiKey }}"
+    return {"clientId": got["clientId"], "clientSecret": got["clientSecret"],
+            "apiKey": got["apiKey"], "clientType": "installed"}, None
+
+
+# The Desktop-app client published for every lab, so a lab installed from
+# npm signs in with nothing to set up. A Desktop client signs in back to any
+# port on this machine with no redirect registered, and Google does not hold
+# its secret to be one: it is in every copy of the package by design. It
+# serves only a lab reached on this machine -- one reached at its own address
+# brings a Web application client of its own. Never in the repository: the
+# package's build writes it beside server.py from the Package workflow's
+# PUBLIC_GOOGLE_APP secret (docs/google-drive.md § The published app), and
+# a checkout or the image has none.
+GOOGLE_APP_FILE = HERE / "google-app.json"
+PUBLIC_GOOGLE_APP, PUBLIC_GOOGLE_PROBLEM = read_published_google(GOOGLE_APP_FILE)
 # drive.file: only what the user picks in Google's Picker, and non-sensitive,
 # so the app can be published without Google's verification (#127).
 GOOGLE_SCOPE = "https://www.googleapis.com/auth/drive.file"
@@ -2600,6 +2641,94 @@ GOOGLE_CALLBACK = "/api/connections/google/callback"
 GOOGLE_STATE_SECONDS = 600
 
 
+def google_app():
+    """The Google app this lab signs in with -- secret included, for the
+    server's own use only -- and where it came from; None with none."""
+    if GOOGLE_CLIENT_ID:
+        return {"clientId": GOOGLE_CLIENT_ID, "clientSecret": GOOGLE_CLIENT_SECRET,
+                "apiKey": GOOGLE_API_KEY, "clientType": GOOGLE_CLIENT_TYPE, "from": "env"}
+    kept = GOOGLE.get() if GOOGLE is not None else None
+    if kept:
+        return {**kept, "from": "lab"}
+    if PUBLIC_GOOGLE_APP:
+        return {**PUBLIC_GOOGLE_APP, "from": "default"}
+    return None
+
+
+def google_public(app):
+    """What the page is told of an app: everything but its secret."""
+    if app is None:
+        return None
+    m = GOOGLE_CLIENT.fullmatch(app.get("clientId") or "")
+    return {"clientId": app.get("clientId"), "clientType": app.get("clientType"),
+            "apiKey": app.get("apiKey"), "appId": m.group(1) if m else None,
+            "hasSecret": bool(app.get("clientSecret")), "from": app["from"]}
+
+
+def loopback_origin(origin) -> bool:
+    """Whether a page's origin is this machine: where a Desktop client may
+    send the browser back to."""
+    host = urllib.parse.urlsplit(origin).hostname or ""
+    return host in ("localhost", "::1") or host.startswith("127.")
+
+
+class GoogleApp:
+    """The Google app entered in Setup: a client and its secret, from the
+    client file Google hands out, and the Picker's key. Kept so a lab with no
+    deployment around it needs no environment variable; the secret is
+    written here and never read back out to the page."""
+
+    KEYS = ("google.clientId", "google.clientSecret", "google.apiKey", "google.clientType")
+
+    def __init__(self, store: Store):
+        self.store = store
+        with store.lock, closing(sqlite3.connect(store.path)) as db, db:
+            db.execute("CREATE TABLE IF NOT EXISTS settings (key TEXT PRIMARY KEY, value TEXT NOT NULL)")
+
+    def _read(self, db):
+        rows = dict(db.execute("SELECT key, value FROM settings WHERE key IN (?, ?, ?, ?)", self.KEYS))
+        return dict(zip(("clientId", "clientSecret", "apiKey", "clientType"),
+                        (rows.get(k) or None for k in self.KEYS)))
+
+    def get(self):
+        with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
+            got = self._read(db)
+        return {**got, "clientType": got["clientType"] or "installed"} if got["clientId"] else None
+
+    def set(self, payload):
+        """Keeps what is sent, or with no client id forgets it all: (kept,
+        error, whether the client changed). A secret not sent is kept while
+        the client is the same one, and forgotten when it is not: a secret
+        belongs to its client."""
+        if not isinstance(payload, dict):
+            return None, (400, "a Google app is { clientId, clientSecret, apiKey, clientType }"), False
+        client = payload.get("clientId")
+        secret, key = payload.get("clientSecret"), payload.get("apiKey")
+        ctype = payload.get("clientType") or "installed"
+        if not isinstance(client, str) or not all(v is None or isinstance(v, str) for v in (secret, key)):
+            return None, (400, "a Google app is { clientId, clientSecret, apiKey, clientType }"), False
+        client, key = client.strip(), (key or "").strip()
+        with self.store.lock, closing(sqlite3.connect(self.store.path)) as db, db:
+            had = self._read(db)
+            changed = (had["clientId"] or "") != client
+            if not client:
+                db.execute("DELETE FROM settings WHERE key IN (?, ?, ?, ?)", self.KEYS)
+                return None, None, changed
+            if not GOOGLE_CLIENT.fullmatch(client):
+                return None, (400, "a client ID ends .apps.googleusercontent.com"), False
+            if key and not GOOGLE_KEY.fullmatch(key):
+                return None, (400, "that is not an API key"), False
+            if ctype not in GOOGLE_CLIENT_TYPES:
+                return None, (400, "a client is a Desktop app or a Web application"), False
+            if secret is None:
+                secret = None if changed else had["clientSecret"]
+            secret = (secret or "").strip()
+            db.execute("DELETE FROM settings WHERE key IN (?, ?, ?, ?)", self.KEYS)
+            db.executemany("INSERT INTO settings VALUES (?, ?)",
+                           [(k, v) for k, v in zip(self.KEYS, (client, secret, key, ctype)) if v])
+        return self.get(), None, changed
+
+
 class Connections:
     """The lab's grants, held server-side; the page sees their state only."""
 
@@ -2619,7 +2748,8 @@ class Connections:
 
     @staticmethod
     def configured() -> bool:
-        return all((GOOGLE_CLIENT_ID, GOOGLE_CLIENT_SECRET, GOOGLE_API_KEY, GOOGLE_APP_ID))
+        app = google_app()
+        return bool(app and app.get("clientId") and app.get("clientSecret"))
 
     def _row(self):
         with self.store.lock, closing(sqlite3.connect(self.store.path)) as db:
@@ -2633,15 +2763,21 @@ class Connections:
 
     def start(self, origin: str):
         """The consent address the page sends the browser to."""
+        app = google_app()
         if not self.configured():
             return None, (409, "Google is not configured for this lab")
+        if app["clientType"] == "installed" and not loopback_origin(origin):
+            return None, (409, "a Desktop app client signs in only on this machine: "
+                               "give this lab a Web application client in Configure…")
         state = secrets.token_urlsafe(24)
         now = time.time()
         with self.lock:
             self.states = {s: v for s, v in self.states.items() if v[0] > now}
-            self.states[state] = (now + GOOGLE_STATE_SECONDS, origin)
+            # The app is kept with the state: the code Google sends back is
+            # exchanged with the client that asked for it, whatever changes.
+            self.states[state] = (now + GOOGLE_STATE_SECONDS, origin, app)
         return GOOGLE_CONSENT_URL + "?" + urllib.parse.urlencode({
-            "client_id": GOOGLE_CLIENT_ID, "redirect_uri": origin + GOOGLE_CALLBACK,
+            "client_id": app["clientId"], "redirect_uri": origin + GOOGLE_CALLBACK,
             "response_type": "code", "scope": GOOGLE_SCOPE, "state": state,
             "access_type": "offline", "prompt": "consent", "include_granted_scopes": "true",
         }), None
@@ -2661,7 +2797,7 @@ class Connections:
         which never include the code or a token."""
         state = (query.get("state") or [""])[0]
         with self.lock:
-            lapses, origin = self.states.pop(state, (0, ""))
+            lapses, origin, app = self.states.pop(state, (0, "", None))
         if lapses <= time.time():
             return "that sign-in had lapsed or was not this lab's; sign in again"
         if query.get("error"):
@@ -2671,7 +2807,7 @@ class Connections:
             return "Google sent no code back"
         try:
             got = self._call(GOOGLE_TOKEN_URL, {
-                "code": code, "client_id": GOOGLE_CLIENT_ID, "client_secret": GOOGLE_CLIENT_SECRET,
+                "code": code, "client_id": app["clientId"], "client_secret": app["clientSecret"],
                 "redirect_uri": origin + GOOGLE_CALLBACK, "grant_type": "authorization_code"})
         except (OSError, ValueError) as e:
             return f"the code could not be exchanged ({type(e).__name__})"
@@ -2689,6 +2825,12 @@ class Connections:
                         time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())))
         return None
 
+    def forget(self):
+        """Drops the grant without a word to Google: the client it was made
+        with is no longer this lab's, so the grant is no use to it."""
+        with self.store.lock, closing(sqlite3.connect(self.store.path)) as db, db:
+            db.execute("DELETE FROM connections WHERE id = 'google'")
+
     def sign_out(self):
         """Forgets the grant, and asks Google to revoke it; a revoke that
         fails still forgets it here, which is what signing out means."""
@@ -3589,17 +3731,17 @@ def worker_destinations(run: dict):
         base = api_base(str(conn.get("url") or "")) or api_base(OLLAMA)
         why = allowed(base, "")
         if why:
-            return None, f"Setup profile {name}: {why}"
+            return None, f"Target profile {name}: {why}"
         profile = next((p for p in stored if isinstance(p, dict) and p.get("id") == pid), None)
         if profile is None:
-            return None, f"Setup profile {name} not found"
+            return None, f"Target profile {name} not found"
         key = str(profile.get("key") or "").strip()
         if key:
             if not header_safe(key):
-                return None, f"Setup profile {name} has a key that cannot go in a header"
+                return None, f"Target profile {name} has a key that cannot go in a header"
             why = allowed(base, key)
             if why:
-                return None, f"Setup profile {name}: {why}"
+                return None, f"Target profile {name}: {why}"
             env[key_var(pid)] = key
     return env, None
 
@@ -3618,12 +3760,17 @@ def worker_destinations(run: dict):
 # 7: chains are jobs: the field is `jobs` and each job's type is "job".
 # 8: a job is its steps; the content is job 1's Attach Content step.
 # 9: a test is Metrics; a Single Test or a Graded set is read converted.
-PIPELINE_VERSION = 9
+# 10: a job's steps are its stages, and each scenario is a target whose own
+# step in each job is what it sends there (docs/pipeline-model.md §16).
+PIPELINE_VERSION = 10
 # What a stored run may be: the current version, and the ones evals-core.ts's
-# upgradePipeline reads. A new submission is always the current one.
-READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9)
-SCENARIO_CAP = 4
-RUN_FIELDS = ("version", "id", "name", "jobs", "scenarios", "tests", "profiles", "comment", "plugins")
+# upgradePipeline reads. A new submission is upgraded to the current one.
+READABLE_VERSIONS = (1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
+TARGET_CAP = 4
+# Target steps whose words the Prompt library does not record as a use: they
+# ask no model (evals-core.ts's STEP_TYPES.echo).
+UNRECORDED_STEPS = {"echo"}
+RUN_FIELDS = ("version", "id", "name", "jobs", "targets", "tests", "profiles", "comment", "plugins")
 
 
 def content_of(doc):
@@ -3823,16 +3970,20 @@ def upgrade_run(run):
     return up, None
 
 
-def answers_from_item(run, sc, cell):
-    """Whether job 1 of a scenario is answered from its item's own text --
-    a local type (Echo) over a Source or Text -- so its prompt may be blank:
-    evals-core.ts's rule, mirrored. Over Prompt only the prompt is the reply."""
-    content = content_of(run)
-    if not isinstance(content, dict) or content.get("type") not in ("source", "text"):
-        return False
-    ref = cell.get("profile") or sc.get("profile")
-    conn = (run.get("profiles") or {}).get(ref.get("id")) if isinstance(ref, dict) else None
-    return isinstance(conn, dict) and conn.get("type") in LOCAL_CONNECTIONS
+def cells_of(run):
+    """Every target's step in every job of a run, as (i, target, k, step):
+    version 10's targets and their steps, or an earlier version's scenarios
+    and their cells -- what the Prompt library records a use from, whichever
+    version a stored run is."""
+    if not isinstance(run, dict):
+        return
+    targets = run.get("targets") if isinstance(run.get("targets"), list) else run.get("scenarios")
+    for i, t in enumerate(targets if isinstance(targets, list) else []):
+        if not isinstance(t, dict):
+            continue
+        steps = t.get("steps") if isinstance(t.get("steps"), list) else t.get("stages")
+        for k, step in enumerate(steps if isinstance(steps, list) else []):
+            yield i, t, k, step
 
 
 def run_problems(run):
@@ -3877,36 +4028,41 @@ def run_problems(run):
             bad.append(f"{at} has to be an object")
             continue
         connection_problems(pid, conn, at, bad)
-    scenarios = run.get("scenarios")
-    if not isinstance(scenarios, list) or not 1 <= len(scenarios) <= SCENARIO_CAP:
-        return bad + [f"a run needs between one and {SCENARIO_CAP} scenarios"]
-    ids = [sc.get("id") for sc in scenarios if isinstance(sc, dict)]
-    for i, sc in enumerate(scenarios):
-        at = f"scenario {i + 1}"
-        if not isinstance(sc, dict):
+    targets = run.get("targets")
+    if not isinstance(targets, list) or not 1 <= len(targets) <= TARGET_CAP:
+        return bad + [f"a run needs between one and {TARGET_CAP} targets"]
+    ids = [t.get("id") for t in targets if isinstance(t, dict)]
+    for i, t in enumerate(targets):
+        at = f"target {i + 1}"
+        if not isinstance(t, dict):
             bad.append(f"{at} has to be an object")
             continue
         # The id the Prompt library records a use under (version 4).
-        if not isinstance(sc.get("id"), str) or not sc["id"].strip():
+        if not isinstance(t.get("id"), str) or not t["id"].strip():
             bad.append(f"{at} has no id")
-        elif ids.count(sc["id"]) > 1:
-            bad.append(f"{at} has the id of another scenario")
-        stages = sc.get("stages")
-        if not isinstance(stages, list) or len(stages) != len(jobs):
-            bad.append(f"{at} has to have one prompt per job")
+        elif ids.count(t["id"]) > 1:
+            bad.append(f"{at} has the id of another target")
+        steps = t.get("steps")
+        if not isinstance(steps, list) or len(steps) != len(jobs):
+            bad.append(f"{at} has to have one step per job")
             continue
-        refs = [sc.get("profile")] + [c.get("profile") for c in stages
-                                      if isinstance(c, dict) and c.get("profile") is not None]
-        for k, cell in enumerate(stages):
-            if not isinstance(cell, dict) or not isinstance(cell.get("prompt"), str) or (
-                    not cell["prompt"].strip() and not (k == 0 and answers_from_item(run, sc, cell))):
+        # A target with no profile is one whose steps ask none (Echo) or each
+        # name their own; which steps need one is the core's to judge.
+        refs = ([t["profile"]] if t.get("profile") is not None else []) + [
+            st.get("profile") for st in steps if isinstance(st, dict) and st.get("profile") is not None]
+        # Whether the words may be blank -- Echo answering from the item --
+        # is the core's to judge, and the worker refuses the run in its words.
+        for k, step in enumerate(steps):
+            if not isinstance(step, dict) or not isinstance(step.get("type"), str):
+                bad.append(f"{at}, job {k + 1} has to name what it sends")
+            elif not isinstance(step.get("prompt"), str):
                 bad.append(f"{at}, job {k + 1} has no prompt")
-            elif cell.get("from") is not None and not (
-                    isinstance(cell["from"], dict) and isinstance(cell["from"].get("id"), str)):
+            elif step.get("from") is not None and not (
+                    isinstance(step["from"], dict) and isinstance(step["from"].get("id"), str)):
                 bad.append(f"{at}, job {k + 1} names the prompt it was picked from without an id")
         for ref in refs:
             if not isinstance(ref, dict) or ref.get("id") not in table:
-                bad.append(f"{at} names a Setup profile the run does not carry")
+                bad.append(f"{at} names a Target profile the run does not carry")
     content = content_of(run)
     kind = content.get("type") if isinstance(content, dict) else None
     if kind == "source":
@@ -3946,10 +4102,11 @@ if DATA_DIR:
     PLUGINS = Plugins(STORE)
     CONNECTIONS = Connections(STORE)
     MICROSOFT = MicrosoftApp(STORE)
+    GOOGLE = GoogleApp(STORE)
     QUEUE = Queue(STORE)
     QUEUE.prompts = PROMPTS
 else:
-    STORE = SOURCES = PROMPTS = DATASETS = PACKS = PLUGINS = CONNECTIONS = MICROSOFT = QUEUE = None
+    STORE = SOURCES = PROMPTS = DATASETS = PACKS = PLUGINS = CONNECTIONS = MICROSOFT = GOOGLE = QUEUE = None
 
 
 class NoRedirects(urllib.request.HTTPRedirectHandler):
@@ -4105,7 +4262,7 @@ class Handler(BaseHTTPRequestHandler):
             # And the Microsoft app a flow is read with, when there is one.
             # The app may be entered in Setup unless the deployment names
             # its own, and only in a lab with a store to keep it in.
-            return self._json(200, {"scenarioCap": SCENARIO_CAP, "microsoft": microsoft_config(),
+            return self._json(200, {"targetCap": TARGET_CAP, "microsoft": microsoft_config(),
                                     "microsoftEditable": MICROSOFT is not None and not M365_CLIENT_ID})
         if path == "/api/state":
             if STORE is None:
@@ -4161,6 +4318,10 @@ class Handler(BaseHTTPRequestHandler):
             if CONNECTIONS is None:
                 return self._send(404, b"not found", "text/plain")
             return self._json(200, {"connections": CONNECTIONS.list()})
+        if path == "/api/google":
+            if GOOGLE is None:
+                return self._send(404, b"not found", "text/plain")
+            return self._json(200, {"google": google_public(google_app()), "editable": not GOOGLE_CLIENT_ID})
         if path == GOOGLE_CALLBACK:
             return self._google_callback()
         if path.startswith("/plugins/"):
@@ -4381,6 +4542,17 @@ class Handler(BaseHTTPRequestHandler):
         parts = self.path.split("?", 1)[0].split("/")
         if len(parts) == 4 and parts[:3] == ["", "api", "prompts"] and PROMPTS is not None:
             return self._prompts_put(parts[3])
+        if parts == ["", "api", "google"] and GOOGLE is not None:
+            if GOOGLE_CLIENT_ID:
+                return self._json(409, {"error": "this lab's Google app is its deployment's (GOOGLE_CLIENT_ID)"})
+            _, err, changed = GOOGLE.set(self._payload())
+            if err:
+                return self._json(err[0], {"error": err[1]})
+            # A grant is its client's: one made with another client cannot be
+            # refreshed with this one, so it goes rather than failing later.
+            if changed and CONNECTIONS is not None:
+                CONNECTIONS.forget()
+            return self._json(200, {"google": google_public(google_app()), "editable": True})
         if parts == ["", "api", "microsoft"] and MICROSOFT is not None:
             if M365_CLIENT_ID:
                 return self._json(409, {"error": "this lab's Microsoft app is its deployment's (M365_CLIENT_ID)"})
@@ -5065,6 +5237,10 @@ def main():
     if not (DEMO / "manifest.json").is_file():
         raise SystemExit(f"no {DEMO / 'manifest.json'} -- the demo pack sits beside server.py, "
                          "and the image copies it there")
+    # A published app the build wrote and that cannot be read is a broken
+    # package, not one without the app: said at once, not at Sign in.
+    if PUBLIC_GOOGLE_PROBLEM:
+        raise SystemExit(PUBLIC_GOOGLE_PROBLEM)
     print(f"prompt-lab on {HOST}:{PORT}  ->  {OLLAMA} by default", flush=True)
     print(f"samples: {SAMPLES}", flush=True)
     print(f"page:    {WEB_DIST}", flush=True)