evals-lab 0.11.0 → 0.13.0

package/lab/demo/datasets/sentiment.json

@@ -0,0 +1,383 @@
+{
+  "format": "evals-lab/eval-group",
+  "version": 9,
+  "group": {
+    "name": "Demo: classify sentiment",
+    "body": {
+      "version": 9,
+      "scoring": {
+        "mode": "all",
+        "threshold": null
+      },
+      "grader": null,
+      "every": [],
+      "run": [],
+      "cases": [
+        {
+          "id": "bakery-croissant",
+          "item": "bakery-croissant.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*positive\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is positive. The writer enjoyed the warm, flaky croissant and the friendly service, and plans to return.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "train-delay",
+          "item": "train-delay.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*negative\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is negative. The writer is frustrated by repeated cancellations with no notice or alternative, which made them late.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "library-hours",
+          "item": "library-hours.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*neutral\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is neutral. It is a factual notice of changed opening hours with no opinion either way.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "headphones-review",
+          "item": "headphones-review.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*positive\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is positive. The writer praises the battery life and noise cancelling and calls them their best purchase of the year.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "hotel-room",
+          "item": "hotel-room.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*negative\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is negative. The room did not match the advertised view, was noisy, and the hotel would not move them.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "meeting-minutes",
+          "item": "meeting-minutes.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*neutral\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is neutral. It reports what the committee decided and when it meets next, without expressing a feeling.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "garden-tomatoes",
+          "item": "garden-tomatoes.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*positive\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is positive. The writer is delighted with the taste of their first home-grown tomatoes and feels the effort paid off.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "phone-update",
+          "item": "phone-update.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*negative\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is negative. The update made the battery drain fast and apps crash, and the writer regrets installing it.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "bus-timetable",
+          "item": "bus-timetable.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*neutral\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is neutral. It states the bus frequency and last departure, a plain timetable fact.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "teacher-thanks",
+          "item": "teacher-thanks.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*positive\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is positive. The writer is grateful to a teacher whose extra help led to an exam result they are proud of.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "parcel-lost",
+          "item": "parcel-lost.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*negative\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is negative. The parcel never arrived and the complaints were closed without a reply, leaving the writer out of pocket.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "weather-report",
+          "item": "weather-report.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*neutral\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is neutral. It is a forecast of the weather, giving conditions without any opinion.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "concert-night",
+          "item": "concert-night.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*positive\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is positive. The writer loved the long concert and the shared singing, and would happily go again.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "gym-contract",
+          "item": "gym-contract.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*negative\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is negative. Cancelling was a long ordeal with a hidden fee, and the writer warns others away.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "recipe-note",
+          "item": "recipe-note.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*neutral\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is neutral. It gives cooking instructions and expresses no feeling.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "cafe-coffee",
+          "item": "cafe-coffee.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*negative\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is negative. The coffee was slow, lukewarm and burnt, and the staff ignored the writer without apology.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "puppy-home",
+          "item": "puppy-home.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*positive\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is positive. The writer is clearly fond of their new puppy and describes her with affection.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "road-closure",
+          "item": "road-closure.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*neutral\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is neutral. It announces a road closure and diversion, a factual notice.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "film-sequel",
+          "item": "film-sequel.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*negative\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is negative. The writer found the sequel long, loud and boring compared with the first film.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "volunteer-day",
+          "item": "volunteer-day.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*positive\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is positive. The writer enjoyed the clean-up, the sense of community and the day as a whole.",
+              "threshold": 0.5
+            }
+          ]
+        }
+      ]
+    },
+    "preview": {
+      "id": "sentiment",
+      "name": "Demo: classify sentiment"
+    }
+  }
+}

package/lab/demo/datasets/weather-alerts.json

@@ -0,0 +1,61 @@
+{
+  "format": "evals-lab/eval-group",
+  "version": 9,
+  "group": {
+    "name": "Demo: sort weather alerts",
+    "body": {
+      "version": 9,
+      "scoring": {
+        "mode": "all",
+        "threshold": null
+      },
+      "grader": null,
+      "every": [],
+      "run": [],
+      "cases": [
+        {
+          "id": "alert-types",
+          "item": "https-api.weather.gov-alerts-types.txt",
+          "todo": false,
+          "note": "The National Weather Service's alert types. The plain dangers belong in High and the notices with no hazard in Low.",
+          "metrics": [
+            {
+              "type": "contains",
+              "value": "Tornado Warning",
+              "ignoreCase": true
+            },
+            {
+              "type": "regex",
+              "pattern": "^\\W*High:.*\\bTornado Warning\\b",
+              "flags": "im"
+            },
+            {
+              "type": "regex",
+              "pattern": "^\\W*High:.*\\bTsunami Warning\\b",
+              "flags": "im"
+            },
+            {
+              "type": "regex",
+              "pattern": "^\\W*High:.*\\bHurricane Warning\\b",
+              "flags": "im"
+            },
+            {
+              "type": "regex",
+              "pattern": "^\\W*Low:.*\\bAdministrative Message\\b",
+              "flags": "im"
+            },
+            {
+              "type": "regex",
+              "pattern": "^\\W*Low:.*\\bTest\\b",
+              "flags": "im"
+            }
+          ]
+        }
+      ]
+    },
+    "preview": {
+      "id": "weather-alerts",
+      "name": "Demo: sort weather alerts"
+    }
+  }
+}

package/lab/demo/datasets/wordings.json

@@ -0,0 +1,39 @@
+{
+  "format": "evals-lab/eval-group",
+  "version": 9,
+  "group": {
+    "name": "Demo: compare wordings",
+    "body": {
+      "version": 9,
+      "scoring": {
+        "mode": "all",
+        "threshold": null
+      },
+      "grader": null,
+      "every": [
+        {
+          "type": "contains",
+          "value": "Mercury",
+          "ignoreCase": true
+        },
+        {
+          "type": "contains",
+          "value": "Venus",
+          "not": true,
+          "ignoreCase": true
+        },
+        {
+          "type": "regex",
+          "pattern": "^\\W*Mercury\\W*$",
+          "flags": "i"
+        },
+        {
+          "type": "latency",
+          "max": 10000
+        }
+      ],
+      "run": [],
+      "cases": []
+    }
+  }
+}

package/lab/demo/manifest.json

@@ -3,10 +3,10 @@
   "version": 1,
   "id": "demo",
   "name": "Demo",
-  "packVersion": "2026.10.02",
+  "packVersion": "2026.10.11",
   "requires": {
-    "lab": ">=12",
+    "lab": ">=17",
     "plugins": []
   },
-  "description": "Two graded sets of public-domain images, each with a starter pipeline."
+  "description": "Five demos, each a dataset with a starter pipeline: two sets of public-domain images, short texts to classify, one question in four wordings, and a dynamic Source's snapshot."
 }

package/lab/demo/pipelines/demo-1.json

@@ -1,7 +1,7 @@
 {
-  "version": 16,
+  "version": 17,
   "id": "demo-1",
-  "name": "Demo 1",
+  "name": "Demo: describe images",
   "jobs": [
     {
       "type": "job",
@@ -14,7 +14,7 @@
             "type": "source",
             "ref": {
               "id": "Demo 1",
-              "name": "Demo 1"
+              "name": "Demo: describe images"
             },
             "first": null,
             "loops": 1
@@ -141,7 +141,7 @@
       "continueOnFailure": true,
       "group": {
         "id": "demo-1",
-        "name": "Demo 1"
+        "name": "Demo: describe images"
       },
       "pin": null
     }

package/lab/demo/pipelines/demo-2.json

@@ -1,7 +1,7 @@
 {
-  "version": 16,
+  "version": 17,
   "id": "demo-2",
-  "name": "Demo 2",
+  "name": "Demo: describe images 2",
   "jobs": [
     {
       "type": "job",
@@ -14,7 +14,7 @@
             "type": "source",
             "ref": {
               "id": "Demo 2",
-              "name": "Demo 2"
+              "name": "Demo: describe images 2"
             },
             "first": null,
             "loops": 1
@@ -141,7 +141,7 @@
       "continueOnFailure": true,
       "group": {
         "id": "demo-2",
-        "name": "Demo 2"
+        "name": "Demo: describe images 2"
       },
       "pin": null
     }

package/lab/demo/pipelines/sentiment.json

@@ -0,0 +1,65 @@
+{
+  "version": 17,
+  "id": "sentiment",
+  "name": "Demo: classify sentiment",
+  "jobs": [
+    {
+      "type": "job",
+      "id": "j1",
+      "name": "",
+      "steps": [
+        {
+          "type": "attachContent",
+          "content": {
+            "type": "source",
+            "ref": {
+              "id": "sentiment",
+              "name": "Demo: classify sentiment"
+            },
+            "first": null,
+            "loops": 1
+          }
+        },
+        {
+          "type": "readReply",
+          "out": {
+            "kind": "text"
+          }
+        }
+      ]
+    }
+  ],
+  "targets": [
+    {
+      "id": "s1",
+      "name": "",
+      "connection": {
+        "id": "model",
+        "name": "Model"
+      },
+      "profile": null,
+      "steps": [
+        {
+          "type": "prompt",
+          "prompt": "Read the text below and say how its writer feels about what they describe.\n\nReply with two lines and nothing else:\nLabel: positive, negative or neutral\nReason: one sentence saying why\n\nText:\n{text}"
+        }
+      ]
+    }
+  ],
+  "evals": [
+    {
+      "id": "t1",
+      "name": "",
+      "type": "group",
+      "continueOnFailure": true,
+      "group": {
+        "id": "sentiment",
+        "name": "Demo: classify sentiment"
+      },
+      "pin": null
+    }
+  ],
+  "pass": {
+    "mode": "all"
+  }
+}

package/lab/demo/pipelines/weather-alerts.json

@@ -0,0 +1,66 @@
+{
+  "version": 17,
+  "id": "weather-alerts",
+  "name": "Demo: sort weather alerts",
+  "jobs": [
+    {
+      "type": "job",
+      "id": "j1",
+      "name": "",
+      "steps": [
+        {
+          "type": "attachContent",
+          "content": {
+            "type": "source",
+            "ref": {
+              "id": "weather-alerts",
+              "name": "Demo: sort weather alerts"
+            },
+            "first": null,
+            "loops": 1,
+            "live": false
+          }
+        },
+        {
+          "type": "readReply",
+          "out": {
+            "kind": "text"
+          }
+        }
+      ]
+    }
+  ],
+  "targets": [
+    {
+      "id": "s1",
+      "name": "",
+      "connection": {
+        "id": "model",
+        "name": "Model"
+      },
+      "profile": null,
+      "steps": [
+        {
+          "type": "prompt",
+          "prompt": "Below is the US National Weather Service's list of alert types, as JSON.\n\nSort every alert type on it by hazard into three groups. Reply with exactly three lines and nothing else:\nHigh: the types that warn of a threat to life\nMedium: the types that warn of damage or disruption\nLow: the rest\nSeparate the types on a line with commas, and write each exactly as the list does.\n\n{text}"
+        }
+      ]
+    }
+  ],
+  "evals": [
+    {
+      "id": "t1",
+      "name": "",
+      "type": "group",
+      "continueOnFailure": true,
+      "group": {
+        "id": "weather-alerts",
+        "name": "Demo: sort weather alerts"
+      },
+      "pin": null
+    }
+  ],
+  "pass": {
+    "mode": "all"
+  }
+}