evals-lab 0.10.0 → 0.12.0

package/lab/demo/pipelines/weather-alerts.json

@@ -0,0 +1,66 @@
+{
+  "version": 17,
+  "id": "weather-alerts",
+  "name": "Demo: sort weather alerts",
+  "jobs": [
+    {
+      "type": "job",
+      "id": "j1",
+      "name": "",
+      "steps": [
+        {
+          "type": "attachContent",
+          "content": {
+            "type": "source",
+            "ref": {
+              "id": "weather-alerts",
+              "name": "Demo: sort weather alerts"
+            },
+            "first": null,
+            "loops": 1,
+            "live": false
+          }
+        },
+        {
+          "type": "readReply",
+          "out": {
+            "kind": "text"
+          }
+        }
+      ]
+    }
+  ],
+  "targets": [
+    {
+      "id": "s1",
+      "name": "",
+      "connection": {
+        "id": "model",
+        "name": "Model"
+      },
+      "profile": null,
+      "steps": [
+        {
+          "type": "prompt",
+          "prompt": "Below is the US National Weather Service's list of alert types, as JSON.\n\nSort every alert type on it by hazard into three groups. Reply with exactly three lines and nothing else:\nHigh: the types that warn of a threat to life\nMedium: the types that warn of damage or disruption\nLow: the rest\nSeparate the types on a line with commas, and write each exactly as the list does.\n\n{text}"
+        }
+      ]
+    }
+  ],
+  "evals": [
+    {
+      "id": "t1",
+      "name": "",
+      "type": "group",
+      "continueOnFailure": true,
+      "group": {
+        "id": "weather-alerts",
+        "name": "Demo: sort weather alerts"
+      },
+      "pin": null
+    }
+  ],
+  "pass": {
+    "mode": "all"
+  }
+}

package/lab/demo/pipelines/wordings.json

@@ -0,0 +1,104 @@
+{
+  "version": 17,
+  "id": "wordings",
+  "name": "Demo: compare wordings",
+  "jobs": [
+    {
+      "type": "job",
+      "id": "j1",
+      "name": "",
+      "steps": [
+        {
+          "type": "attachContent",
+          "content": {
+            "type": "prompt"
+          }
+        },
+        {
+          "type": "readReply",
+          "out": {
+            "kind": "text"
+          }
+        }
+      ]
+    }
+  ],
+  "targets": [
+    {
+      "id": "s1",
+      "name": "",
+      "connection": {
+        "id": "model",
+        "name": "Model"
+      },
+      "profile": null,
+      "steps": [
+        {
+          "type": "prompt",
+          "prompt": "Which planet is closest to the Sun?"
+        }
+      ]
+    },
+    {
+      "id": "s2",
+      "name": "",
+      "connection": {
+        "id": "model",
+        "name": "Model"
+      },
+      "profile": null,
+      "steps": [
+        {
+          "type": "prompt",
+          "prompt": "Name the planet nearest the Sun. Reply with its name only."
+        }
+      ]
+    },
+    {
+      "id": "s3",
+      "name": "",
+      "connection": {
+        "id": "model",
+        "name": "Model"
+      },
+      "profile": null,
+      "steps": [
+        {
+          "type": "prompt",
+          "prompt": "Of the eight planets in our solar system, which one orbits closest to the Sun? Answer in one word."
+        }
+      ]
+    },
+    {
+      "id": "s4",
+      "name": "",
+      "connection": {
+        "id": "model",
+        "name": "Model"
+      },
+      "profile": null,
+      "steps": [
+        {
+          "type": "prompt",
+          "prompt": "Quick question: what's the innermost planet?"
+        }
+      ]
+    }
+  ],
+  "evals": [
+    {
+      "id": "t1",
+      "name": "",
+      "type": "group",
+      "continueOnFailure": true,
+      "group": {
+        "id": "wordings",
+        "name": "Demo: compare wordings"
+      },
+      "pin": null
+    }
+  ],
+  "pass": {
+    "mode": "all"
+  }
+}

package/lab/demo/sources/Demo 1/source.json

@@ -0,0 +1,3 @@
+{
+  "name": "Demo: describe images"
+}

package/lab/demo/sources/Demo 2/source.json

@@ -0,0 +1,3 @@
+{
+  "name": "Demo: describe images 2"
+}

package/lab/demo/sources/sentiment/bakery-croissant.txt

@@ -0,0 +1 @@
+Stopped at the new bakery on Elm Street this morning. The croissant was still warm, flaky all the way through, and the woman at the till remembered my name from last week. I'll be back tomorrow.

package/lab/demo/sources/sentiment/bus-timetable.txt

@@ -0,0 +1 @@
+Route 12 now leaves the station every twenty minutes between seven and seven, and every forty minutes in the evening. The last bus is at eleven fifteen.

package/lab/demo/sources/sentiment/cafe-coffee.txt

@@ -0,0 +1 @@
+Waited twenty-five minutes for a lukewarm flat white that tasted burnt. The staff were chatting the whole time and nobody apologised.

package/lab/demo/sources/sentiment/concert-night.txt

@@ -0,0 +1 @@
+The band played for nearly three hours and the whole crowd sang the last song back to them. My voice is gone today and I would do it all again.

package/lab/demo/sources/sentiment/film-sequel.txt

@@ -0,0 +1 @@
+The sequel takes everything that made the first film charming and buries it under two and a half hours of noise. I checked my watch more than the screen.

package/lab/demo/sources/sentiment/garden-tomatoes.txt

@@ -0,0 +1 @@
+Picked the first tomatoes of the summer today, and they were worth every evening of watering. Sweet, a little sharp, and still warm from the sun.

package/lab/demo/sources/sentiment/gym-contract.txt

@@ -0,0 +1 @@
+Cancelling my gym membership took four phone calls, a letter by recorded post and a fee nobody mentioned when I signed up. Avoid.

package/lab/demo/sources/sentiment/headphones-review.txt

@@ -0,0 +1 @@
+Six months in and these headphones still last two full days on a charge. The noise cancelling turns my open-plan office into a quiet room. Best thing I bought this year.

package/lab/demo/sources/sentiment/hotel-room.txt

@@ -0,0 +1 @@
+The website promised a sea view. Our window faced a car park and the air conditioner rattled all night. When we asked to move, the desk said nothing else was free.

package/lab/demo/sources/sentiment/library-hours.txt

@@ -0,0 +1 @@
+From the first of next month, the library will open at ten on weekdays instead of nine. Weekend hours stay the same. Returns can still be left in the box by the door at any time.

package/lab/demo/sources/sentiment/meeting-minutes.txt

@@ -0,0 +1 @@
+The committee met on Tuesday. The budget for the spring fair was approved as proposed. The next meeting is on the fourteenth, in the community hall.

package/lab/demo/sources/sentiment/parcel-lost.txt

@@ -0,0 +1 @@
+The tracking says delivered, but nothing came. Customer service closed my complaint without reply, twice. Forty pounds gone and nobody will even answer.

package/lab/demo/sources/sentiment/phone-update.txt

@@ -0,0 +1 @@
+Since last night's update my phone drains from full to empty by lunchtime, and half my apps crash on opening. I wish I had never pressed install.

package/lab/demo/sources/sentiment/puppy-home.txt

@@ -0,0 +1 @@
+We brought our puppy home on Saturday. She has already learned to sit, chewed one slipper, and fallen asleep on my feet every evening. We adore her.

package/lab/demo/sources/sentiment/recipe-note.txt

@@ -0,0 +1 @@
+Preheat the oven to 180 degrees. Mix the flour, sugar and butter until it looks like breadcrumbs, then add the egg and press into the tin.

package/lab/demo/sources/sentiment/road-closure.txt

@@ -0,0 +1 @@
+Mill Lane will be closed between the bridge and the school from Monday for two weeks while the water main is replaced. A diversion will be signed.

package/lab/demo/sources/sentiment/source.json

@@ -0,0 +1,3 @@
+{
+  "name": "Demo: classify sentiment"
+}

package/lab/demo/sources/sentiment/teacher-thanks.txt

@@ -0,0 +1 @@
+Mr Okafor stayed after class every Thursday for a term to help me with algebra. I passed the exam last week with a grade I never thought I'd get. Thank you, sir.

package/lab/demo/sources/sentiment/train-delay.txt

@@ -0,0 +1 @@
+Third cancelled train this week. No announcement, no replacement bus, just a blank board and a crowd of us standing in the rain. I was an hour late for work again.

package/lab/demo/sources/sentiment/volunteer-day.txt

@@ -0,0 +1 @@
+Thirty of us turned out to clear the riverbank on Sunday. We filled forty bags, met neighbours we'd never spoken to, and finished with tea and cake. A lovely day.

package/lab/demo/sources/sentiment/weather-report.txt

@@ -0,0 +1 @@
+Tomorrow will be cloudy in the morning with sunny spells later. Highs of seventeen degrees, with a light westerly wind.

package/lab/demo/sources/weather-alerts/https-api.weather.gov-alerts-types.txt

@@ -0,0 +1,116 @@
+{
+    "@context": [],
+    "eventTypes": [
+        "911 Telephone Outage",
+        "Administrative Message",
+        "Air Quality Alert",
+        "Air Stagnation Advisory",
+        "Ashfall Advisory",
+        "Ashfall Warning",
+        "Avalanche Advisory",
+        "Avalanche Warning",
+        "Avalanche Watch",
+        "Beach Hazards Statement",
+        "Blizzard Warning",
+        "Blowing Dust Advisory",
+        "Blowing Dust Warning",
+        "Blue Alert",
+        "Brisk Wind Advisory",
+        "Child Abduction Emergency",
+        "Civil Danger Warning",
+        "Civil Emergency Message",
+        "Coastal Flood Advisory",
+        "Coastal Flood Statement",
+        "Coastal Flood Warning",
+        "Coastal Flood Watch",
+        "Cold Weather Advisory",
+        "Dense Fog Advisory",
+        "Dense Smoke Advisory",
+        "Dust Advisory",
+        "Dust Storm Warning",
+        "Earthquake Warning",
+        "Evacuation Immediate",
+        "Extreme Heat Warning",
+        "Extreme Heat Watch",
+        "Extreme Cold Warning",
+        "Extreme Cold Watch",
+        "Extreme Fire Danger",
+        "Extreme Wind Warning",
+        "Fire Warning",
+        "Fire Weather Watch",
+        "Flash Flood Statement",
+        "Flash Flood Warning",
+        "Flash Flood Watch",
+        "Flood Advisory",
+        "Flood Statement",
+        "Flood Warning",
+        "Flood Watch",
+        "Freeze Warning",
+        "Freeze Watch",
+        "Freezing Fog Advisory",
+        "Freezing Spray Advisory",
+        "Frost Advisory",
+        "Gale Warning",
+        "Gale Watch",
+        "Hazardous Materials Warning",
+        "Hazardous Seas Warning",
+        "Hazardous Seas Watch",
+        "Hazardous Weather Outlook",
+        "Heat Advisory",
+        "Heavy Freezing Spray Warning",
+        "Heavy Freezing Spray Watch",
+        "High Surf Advisory",
+        "High Surf Warning",
+        "High Wind Warning",
+        "High Wind Watch",
+        "Hurricane Force Wind Warning",
+        "Hurricane Force Wind Watch",
+        "Hurricane Warning",
+        "Hurricane Watch",
+        "Hydrologic Outlook",
+        "Ice Storm Warning",
+        "Lake Effect Snow Warning",
+        "Lake Wind Advisory",
+        "Lakeshore Flood Advisory",
+        "Lakeshore Flood Statement",
+        "Lakeshore Flood Warning",
+        "Lakeshore Flood Watch",
+        "Law Enforcement Warning",
+        "Local Area Emergency",
+        "Low Water Advisory",
+        "Marine Weather Statement",
+        "Nuclear Power Plant Warning",
+        "Radiological Hazard Warning",
+        "Red Flag Warning",
+        "Rip Current Statement",
+        "Severe Thunderstorm Warning",
+        "Severe Thunderstorm Watch",
+        "Severe Weather Statement",
+        "Shelter In Place Warning",
+        "Short Term Forecast",
+        "Small Craft Advisory",
+        "Snow Squall Warning",
+        "Special Marine Warning",
+        "Special Weather Statement",
+        "Storm Surge Warning",
+        "Storm Surge Watch",
+        "Storm Warning",
+        "Storm Watch",
+        "Test",
+        "Tornado Warning",
+        "Tornado Watch",
+        "Tropical Cyclone Local Statement",
+        "Tropical Storm Warning",
+        "Tropical Storm Watch",
+        "Tsunami Advisory",
+        "Tsunami Warning",
+        "Tsunami Watch",
+        "Typhoon Warning",
+        "Typhoon Watch",
+        "Volcano Warning",
+        "Wind Advisory",
+        "Winter Storm Warning",
+        "Winter Storm Watch",
+        "Winter Weather Advisory"
+    ]
+}

package/lab/demo/sources/weather-alerts/source.json

@@ -0,0 +1,8 @@
+{
+  "name": "Demo: sort weather alerts",
+  "type": "dynamic",
+  "config": {
+    "provider": "http-get",
+    "url": "https://api.weather.gov/alerts/types"
+  }
+}