evals-lab 0.11.0 → 0.12.0

package/lab/demo/pipelines/wordings.json

@@ -0,0 +1,104 @@
+{
+  "version": 17,
+  "id": "wordings",
+  "name": "Demo: compare wordings",
+  "jobs": [
+    {
+      "type": "job",
+      "id": "j1",
+      "name": "",
+      "steps": [
+        {
+          "type": "attachContent",
+          "content": {
+            "type": "prompt"
+          }
+        },
+        {
+          "type": "readReply",
+          "out": {
+            "kind": "text"
+          }
+        }
+      ]
+    }
+  ],
+  "targets": [
+    {
+      "id": "s1",
+      "name": "",
+      "connection": {
+        "id": "model",
+        "name": "Model"
+      },
+      "profile": null,
+      "steps": [
+        {
+          "type": "prompt",
+          "prompt": "Which planet is closest to the Sun?"
+        }
+      ]
+    },
+    {
+      "id": "s2",
+      "name": "",
+      "connection": {
+        "id": "model",
+        "name": "Model"
+      },
+      "profile": null,
+      "steps": [
+        {
+          "type": "prompt",
+          "prompt": "Name the planet nearest the Sun. Reply with its name only."
+        }
+      ]
+    },
+    {
+      "id": "s3",
+      "name": "",
+      "connection": {
+        "id": "model",
+        "name": "Model"
+      },
+      "profile": null,
+      "steps": [
+        {
+          "type": "prompt",
+          "prompt": "Of the eight planets in our solar system, which one orbits closest to the Sun? Answer in one word."
+        }
+      ]
+    },
+    {
+      "id": "s4",
+      "name": "",
+      "connection": {
+        "id": "model",
+        "name": "Model"
+      },
+      "profile": null,
+      "steps": [
+        {
+          "type": "prompt",
+          "prompt": "Quick question: what's the innermost planet?"
+        }
+      ]
+    }
+  ],
+  "evals": [
+    {
+      "id": "t1",
+      "name": "",
+      "type": "group",
+      "continueOnFailure": true,
+      "group": {
+        "id": "wordings",
+        "name": "Demo: compare wordings"
+      },
+      "pin": null
+    }
+  ],
+  "pass": {
+    "mode": "all"
+  }
+}

package/lab/demo/sources/Demo 1/source.json

@@ -0,0 +1,3 @@
+{
+  "name": "Demo: describe images"
+}

package/lab/demo/sources/Demo 2/source.json

@@ -0,0 +1,3 @@
+{
+  "name": "Demo: describe images 2"
+}

package/lab/demo/sources/sentiment/bakery-croissant.txt

@@ -0,0 +1 @@
+Stopped at the new bakery on Elm Street this morning. The croissant was still warm, flaky all the way through, and the woman at the till remembered my name from last week. I'll be back tomorrow.

package/lab/demo/sources/sentiment/bus-timetable.txt

@@ -0,0 +1 @@
+Route 12 now leaves the station every twenty minutes between seven and seven, and every forty minutes in the evening. The last bus is at eleven fifteen.

package/lab/demo/sources/sentiment/cafe-coffee.txt

@@ -0,0 +1 @@
+Waited twenty-five minutes for a lukewarm flat white that tasted burnt. The staff were chatting the whole time and nobody apologised.

package/lab/demo/sources/sentiment/concert-night.txt

@@ -0,0 +1 @@
+The band played for nearly three hours and the whole crowd sang the last song back to them. My voice is gone today and I would do it all again.

package/lab/demo/sources/sentiment/film-sequel.txt

@@ -0,0 +1 @@
+The sequel takes everything that made the first film charming and buries it under two and a half hours of noise. I checked my watch more than the screen.

package/lab/demo/sources/sentiment/garden-tomatoes.txt

@@ -0,0 +1 @@
+Picked the first tomatoes of the summer today, and they were worth every evening of watering. Sweet, a little sharp, and still warm from the sun.

package/lab/demo/sources/sentiment/gym-contract.txt

@@ -0,0 +1 @@
+Cancelling my gym membership took four phone calls, a letter by recorded post and a fee nobody mentioned when I signed up. Avoid.

package/lab/demo/sources/sentiment/headphones-review.txt

@@ -0,0 +1 @@
+Six months in and these headphones still last two full days on a charge. The noise cancelling turns my open-plan office into a quiet room. Best thing I bought this year.

package/lab/demo/sources/sentiment/hotel-room.txt

@@ -0,0 +1 @@
+The website promised a sea view. Our window faced a car park and the air conditioner rattled all night. When we asked to move, the desk said nothing else was free.

package/lab/demo/sources/sentiment/library-hours.txt

@@ -0,0 +1 @@
+From the first of next month, the library will open at ten on weekdays instead of nine. Weekend hours stay the same. Returns can still be left in the box by the door at any time.

package/lab/demo/sources/sentiment/meeting-minutes.txt

@@ -0,0 +1 @@
+The committee met on Tuesday. The budget for the spring fair was approved as proposed. The next meeting is on the fourteenth, in the community hall.

package/lab/demo/sources/sentiment/parcel-lost.txt

@@ -0,0 +1 @@
+The tracking says delivered, but nothing came. Customer service closed my complaint without reply, twice. Forty pounds gone and nobody will even answer.

package/lab/demo/sources/sentiment/phone-update.txt

@@ -0,0 +1 @@
+Since last night's update my phone drains from full to empty by lunchtime, and half my apps crash on opening. I wish I had never pressed install.

package/lab/demo/sources/sentiment/puppy-home.txt

@@ -0,0 +1 @@
+We brought our puppy home on Saturday. She has already learned to sit, chewed one slipper, and fallen asleep on my feet every evening. We adore her.

package/lab/demo/sources/sentiment/recipe-note.txt

@@ -0,0 +1 @@
+Preheat the oven to 180 degrees. Mix the flour, sugar and butter until it looks like breadcrumbs, then add the egg and press into the tin.

package/lab/demo/sources/sentiment/road-closure.txt

@@ -0,0 +1 @@
+Mill Lane will be closed between the bridge and the school from Monday for two weeks while the water main is replaced. A diversion will be signed.

package/lab/demo/sources/sentiment/source.json

@@ -0,0 +1,3 @@
+{
+  "name": "Demo: classify sentiment"
+}

package/lab/demo/sources/sentiment/teacher-thanks.txt

@@ -0,0 +1 @@
+Mr Okafor stayed after class every Thursday for a term to help me with algebra. I passed the exam last week with a grade I never thought I'd get. Thank you, sir.

package/lab/demo/sources/sentiment/train-delay.txt

@@ -0,0 +1 @@
+Third cancelled train this week. No announcement, no replacement bus, just a blank board and a crowd of us standing in the rain. I was an hour late for work again.

package/lab/demo/sources/sentiment/volunteer-day.txt

@@ -0,0 +1 @@
+Thirty of us turned out to clear the riverbank on Sunday. We filled forty bags, met neighbours we'd never spoken to, and finished with tea and cake. A lovely day.

package/lab/demo/sources/sentiment/weather-report.txt

@@ -0,0 +1 @@
+Tomorrow will be cloudy in the morning with sunny spells later. Highs of seventeen degrees, with a light westerly wind.

package/lab/demo/sources/weather-alerts/https-api.weather.gov-alerts-types.txt

@@ -0,0 +1,116 @@
+{
+    "@context": [],
+    "eventTypes": [
+        "911 Telephone Outage",
+        "Administrative Message",
+        "Air Quality Alert",
+        "Air Stagnation Advisory",
+        "Ashfall Advisory",
+        "Ashfall Warning",
+        "Avalanche Advisory",
+        "Avalanche Warning",
+        "Avalanche Watch",
+        "Beach Hazards Statement",
+        "Blizzard Warning",
+        "Blowing Dust Advisory",
+        "Blowing Dust Warning",
+        "Blue Alert",
+        "Brisk Wind Advisory",
+        "Child Abduction Emergency",
+        "Civil Danger Warning",
+        "Civil Emergency Message",
+        "Coastal Flood Advisory",
+        "Coastal Flood Statement",
+        "Coastal Flood Warning",
+        "Coastal Flood Watch",
+        "Cold Weather Advisory",
+        "Dense Fog Advisory",
+        "Dense Smoke Advisory",
+        "Dust Advisory",
+        "Dust Storm Warning",
+        "Earthquake Warning",
+        "Evacuation Immediate",
+        "Extreme Heat Warning",
+        "Extreme Heat Watch",
+        "Extreme Cold Warning",
+        "Extreme Cold Watch",
+        "Extreme Fire Danger",
+        "Extreme Wind Warning",
+        "Fire Warning",
+        "Fire Weather Watch",
+        "Flash Flood Statement",
+        "Flash Flood Warning",
+        "Flash Flood Watch",
+        "Flood Advisory",
+        "Flood Statement",
+        "Flood Warning",
+        "Flood Watch",
+        "Freeze Warning",
+        "Freeze Watch",
+        "Freezing Fog Advisory",
+        "Freezing Spray Advisory",
+        "Frost Advisory",
+        "Gale Warning",
+        "Gale Watch",
+        "Hazardous Materials Warning",
+        "Hazardous Seas Warning",
+        "Hazardous Seas Watch",
+        "Hazardous Weather Outlook",
+        "Heat Advisory",
+        "Heavy Freezing Spray Warning",
+        "Heavy Freezing Spray Watch",
+        "High Surf Advisory",
+        "High Surf Warning",
+        "High Wind Warning",
+        "High Wind Watch",
+        "Hurricane Force Wind Warning",
+        "Hurricane Force Wind Watch",
+        "Hurricane Warning",
+        "Hurricane Watch",
+        "Hydrologic Outlook",
+        "Ice Storm Warning",
+        "Lake Effect Snow Warning",
+        "Lake Wind Advisory",
+        "Lakeshore Flood Advisory",
+        "Lakeshore Flood Statement",
+        "Lakeshore Flood Warning",
+        "Lakeshore Flood Watch",
+        "Law Enforcement Warning",
+        "Local Area Emergency",
+        "Low Water Advisory",
+        "Marine Weather Statement",
+        "Nuclear Power Plant Warning",
+        "Radiological Hazard Warning",
+        "Red Flag Warning",
+        "Rip Current Statement",
+        "Severe Thunderstorm Warning",
+        "Severe Thunderstorm Watch",
+        "Severe Weather Statement",
+        "Shelter In Place Warning",
+        "Short Term Forecast",
+        "Small Craft Advisory",
+        "Snow Squall Warning",
+        "Special Marine Warning",
+        "Special Weather Statement",
+        "Storm Surge Warning",
+        "Storm Surge Watch",
+        "Storm Warning",
+        "Storm Watch",
+        "Test",
+        "Tornado Warning",
+        "Tornado Watch",
+        "Tropical Cyclone Local Statement",
+        "Tropical Storm Warning",
+        "Tropical Storm Watch",
+        "Tsunami Advisory",
+        "Tsunami Warning",
+        "Tsunami Watch",
+        "Typhoon Warning",
+        "Typhoon Watch",
+        "Volcano Warning",
+        "Wind Advisory",
+        "Winter Storm Warning",
+        "Winter Storm Watch",
+        "Winter Weather Advisory"
+    ]
+}

package/lab/demo/sources/weather-alerts/source.json

@@ -0,0 +1,8 @@
+{
+  "name": "Demo: sort weather alerts",
+  "type": "dynamic",
+  "config": {
+    "provider": "http-get",
+    "url": "https://api.weather.gov/alerts/types"
+  }
+}

package/lab/evals-core.mjs

@@ -166,13 +166,11 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
 /** A target step that sends a Power Automate step's request
     (docs/power-automate.md § The HTTP Request step): the flow's own template
     -- method, path, query and body, Workflow Definition Language expressions
-    and all -- with the target's words and fields placed in the body by its
-    HTTP API, evaluated against each item's record. The profile gives the
-    address and the key. */
+    and all -- with the target's words and fields placed in the body by the
+    job's HTTP API (its flow step's), evaluated against each item's record.
+    The profile gives the address and the key. */
                                                
                       
-                                                    
-              
                  
                
                                 
@@ -240,7 +238,7 @@ import list, { ECHO, CASE_MODES, legacyTagsOut, legacyTagsModifiers, cleaningFil
  
 
 /** What a target sends in one job: one of the target step types. */
-                                                                                                                                
+                                                                                                                        
 
 /** A target (pipeline-model §16): one column through every job, compared
     side by side in Results -- what version 9 called a scenario. Its step in
@@ -1719,6 +1717,10 @@ function datasetRules(body         )                  {
                                                     
                                                                                                             
                                                         
+                                                                             
+                                                                          
+                                                                   
+                                         
                                                                                    
                                 
                                                                             
@@ -2170,6 +2172,7 @@ CONNECTION_TYPES.anthropic = {
   // Asked words (a Prompt step, a grader) or sent a flow's whole request (an
   // HTTP Request step): one profile and one key do both.
   answers: ["prompt", "request"],
+  speaks: () => "anthropic",
   // The Messages API: the x-api-key and anthropic-version headers, base64
   // image content blocks, and /v1/models for the list.
   request(conn, prompt, dataUrl, base, key) {
@@ -2501,8 +2504,17 @@ CONNECTION_TYPES.http = {
     { key: "headers", label: "Headers", control: "text", default: "", hint: "", scope: "connection" },
     // What Test connection asks of it: a GET, keyed as a run is.
     { key: "testPath", label: "Test path", control: "text", default: "", hint: "", scope: "connection" },
+    // Read off the address (httpApiFor); set by hand for one the lab cannot
+    // tell, such as a gateway in front of a provider.
+    { key: "requestFormat", label: "Request format", control: "select", default: "", hint: "", scope: "connection",
+      choices: [{ value: "", label: "From address" },
+                ...Object.entries(HTTP_APIS).map(([value, a]) => ({ value, label: a.label }))] },
   ],
   answers: ["request"],
+  speaks(conn) {
+    const api = String(conn.requestFormat || "") || httpApiFor(originBase(conn.url));
+    return HTTP_APIS[api] && api !== "raw" ? api : null;
+  },
   lists: false,
   baseOf: originBase,
   send(conn, req, base, key) {
@@ -3297,7 +3309,7 @@ function applyModifiers   (list                                 , kind        ,
 // 16: a target names an AI Connection and, optionally, a Connection Profile,
 // where it named one Target profile; a run's table is keyed by the pair
 // (answerKey) -- the Target profile split in the store as it opens (#445).
-const PIPELINE_VERSION = 16         ;
+const PIPELINE_VERSION = 17         ;
 
 // Plain objects, so an entry is added by assignment and a reader never needs
 // to know which registered it.
@@ -4818,16 +4830,18 @@ STEP_TYPES.httpRequest = {
   label: "HTTP Request", slot: "target", asks: "request", verbatim: true, modelFrom: "step",
   description: "Sends a Power Automate step's own request, rebuilt from each record, with each target's words in it.",
   firstJobOnly: "an HTTP Request is sent from its item's record, so it is job 1's",
-  // A new target starts from the flow's own words and fields.
-  newStep: (step) => ({ type: "httpRequest", api: step.api, method: step.method, path: step.path,
+  // A new target starts from the flow's own words and fields, read by the
+  // job's HTTP API ([step] carries it: requestStepOf).
+  newStep: (step) => ({ type: "httpRequest", method: step.method, path: step.path,
                         query: clone(step.query), body: clone(step.body),
                         ...(HTTP_APIS[step.api] ?? HTTP_APIS.raw ).cellOf(step.body) }),
   in: "item", out: "text",
   apply: "runPipeline",
-  fields: ["type", "api", "method", "path", "query", "body", "prompt", "connection", "profile", "from", "system", "model", "settings", "prefill"],
+  fields: ["type", "method", "path", "query", "body", "prompt", "connection", "profile", "from", "system", "model", "settings", "prefill"],
+  // [step] carries its job's HTTP API, where the job has a flow step to say
+  // it (validatePipeline); the flow step's own check names an unknown one.
   validate(step, _ctx, bad, at){
     const api = HTTP_APIS[step.api];
-    if (!api) bad.push(`${at}: "${step.api}" is not an HTTP API this lab has`);
     if (!HTTP_METHODS.includes(step.method)) bad.push(`${at}: the method is one of ${HTTP_METHODS.join(", ")}`);
     if (!isStr(step.path) || !/^[/@]/.test(step.path)) bad.push(`${at}: the path starts with /`);
     if (!isObj(step.query) || !Object.values(step.query).every(isStr)) bad.push(`${at}: the query is names to text`);
@@ -5104,7 +5118,7 @@ function requestStepOf(doc
 
 /** What httpRequestOf builds a request from: a target's template, and the
     loop its flow step sits in. */
-                                                                                                                          
+                                                                                                                               
 /** What httpReplyOf reads a reply with: the flow step, its API, its reading. */
                                                                                                      
 
@@ -5706,10 +5720,14 @@ function upgradePipeline   (doc   , ctx             = {})    {
     }
     return localSteps(ownBodiesNow(out), ctx)     ;
   }
-  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15].includes(doc.version          )) return doc;
+  if (!isObj(doc) || ![1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16].includes(doc.version          )) return doc;
+  // Every version reads as 16 first, then as 17: its requests read by their
+  // job's HTTP API (requestsOnJobApi).
+  const seventeen = (d           )    => localSteps(ownBodiesNow(requestsOnJobApi(d)), ctx)     ;
+  if (doc.version === 16) return seventeen(cutRefs(clone(doc)             ));
   // Every version reads as 15 first, then as 16: its Target profiles split
   // into AI Connections and Connection Profiles (answersFromProfiles).
-  const sixteen = (fifteen           )    => localSteps(ownBodiesNow(answersFromProfiles(fifteen, ctx)), ctx)     ;
+  const sixteen = (fifteen           )    => seventeen(answersFromProfiles(fifteen, ctx));
   if (doc.version === 15) return sixteen(cutRefs(clone(doc)             ));
   // Versions 13 and 14 read as 15 by migrating a job's cleaning modifier steps
   // and its List `preamble` setting into Pre-/Post-format filter sets
@@ -5854,6 +5872,37 @@ function localSteps(doc           , ctx            )            {
   return next;
 }
 
+/**
+ * Version 16 to 17 (#477): a request step has no HTTP API of its own; its
+ * job's flow step says it. A step that named another -- Any endpoint over an
+ * Anthropic flow, say -- keeps its body as it would have sent it, read again
+ * by the job's (its words and fields as that API holds them). A request in a
+ * job with no flow step, which could never run, is a Prompt keeping its words.
+ */
+function requestsOnJobApi(next           )            {
+  next.version = PIPELINE_VERSION;
+  const jobs            = Array.isArray(next.jobs) ? next.jobs : [];
+  const step = (st           , k        )            => {
+    if (!isObj(st) || st.type !== "httpRequest") return st;
+    const { api: own, ...rest } = st;
+    const flow = isObj(jobs[k]) ? flowStepOf(jobs[k]       ) : null;
+    if (!flow) {
+      return { type: "prompt", prompt: rest.prompt,
+               ...Object.fromEntries(["connection", "profile", "from"].filter(f => rest[f] != null).map(f => [f, rest[f]])) };
+    }
+    const was = HTTP_APIS[own          ], now = HTTP_APIS[flow.api];
+    if (!was || !now || own === flow.api) return rest;
+    let body = rest.body;
+    try { body = was.place(rest.body, rest        ); } catch { /* words that never placed: the template stands */ }
+    const { system: _s, model: _m, settings: _t, prefill: _p, ...kept } = rest;
+    return { ...kept, body, ...now.cellOf(body) };
+  };
+  if (Array.isArray(next.targets)) {
+    next.targets = next.targets.map((t           ) => (isObj(t) && Array.isArray(t.steps) ? { ...t, steps: t.steps.map(step) } : t));
+  }
+  return next;
+}
+
 /** [next] as version 9 held it, from any version before 10. */
 function nineOf(next           , ctx            )            {
   if (next.version === 9) { profileRefs(next); return next; }
@@ -5975,7 +6024,9 @@ function validatePipeline(input         , ctx          = {})           {
       if (entry.local && (st.profile != null || st.connection != null)) return void bad.push(`${where}: ${entry.label} asks no connection`);
       if (k > 0 && entry.firstJobOnly) return void bad.push(`${where}: ${entry.firstJobOnly}`);
       onlyFields(st, where, entry.fields ?? [], bad);
-      entry.validate(st, c, bad, where);
+      // A request is read and placed by its job's HTTP API.
+      const flow = entry.asks === "request" ? flowStepOf(doc.jobs[k]) : null;
+      entry.validate(flow ? { ...st, api: flow.api } : st, c, bad, where);
       answerRefsProblems(st, where, bad);
       if (st.from != null && !isRef(st.from)) bad.push(`${where} has to name the prompt it was picked from as { id, name }`);
       // A request is a flow step's, rebuilt from its records, and its
@@ -6050,6 +6101,15 @@ function validatePipeline(input         , ctx          = {})           {
       const type = CONNECTION_TYPES[typeOf(p            )];
       if (type && !(type.answers ?? ["prompt"]).includes(step.asks ?? "prompt")) {
         bad.push(`${n > 1 ? `${jobLabel(doc, k)}, ` : ""}${targetLabel(doc, i)}: ${step.label ?? "this step"} needs a profile that answers it, and ${type.label} does not`);
+        return;
+      }
+      // A request goes as the flow wrote it, so its connection has to speak
+      // the job's HTTP API; one the lab cannot tell (Any endpoint) passes.
+      const api = step.asks === "request" ? flowStepOf(doc.jobs[k])?.api : undefined;
+      const speaks = api && api !== "raw" && type?.speaks ? type.speaks(p            ) : null;
+      if (api && speaks && speaks !== api) {
+        const name = targetAnswerOf(doc, i, k)?.connection.name || type .label;
+        bad.push(`${n > 1 ? `${jobLabel(doc, k)}, ` : ""}${targetLabel(doc, i)}: ${name} can't send an ${HTTP_APIS[api]?.label ?? api} request`);
       }
     });
     // The model is the profile's unless the target's step carries it.
@@ -7119,7 +7179,7 @@ export {
 CONNECTION_FIELDS, SETTING_KEYS, OPTION_FIELDS, jobLabel, scenarioLabel, jobWords, versionProblem,
     contentOf, withContent, replyOf,
     targetsOf, targetLetter, targetLabel, targetStepOf, targetEntryOf, jobTargetType, targetAnswerOf, targetAnswerKey, recordedTargetOf, withTarget,
-    tokensOf, withTokens, sendsImage, withImage, flowStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
+    tokensOf, withTokens, sendsImage, withImage, flowStepOf, requestStepOf, readAsOf, outOf, withOut, slotEntry, SLOTS,
     filterSetsOf, withFilterSets, stageModifiers, beforeFilters,
     blankPipeline, upgradePipeline, fatProfileRef, newId, newEval, newLink, validatePipeline, resolvePipeline, pipelineOfRun, stagesFor, profileIds, answerKeys,
     scenarioProfile, lastKind, lastModifier, modifierSummary, itemScores, scenarioEvals, scenarioPasses, groupPass, overallVerdict, passVerdict, isSkipped, failedScore,