evals-lab 0.10.0 → 0.12.0

package/lab/demo/datasets/sentiment.json

@@ -0,0 +1,383 @@
+{
+  "format": "evals-lab/eval-group",
+  "version": 9,
+  "group": {
+    "name": "Demo: classify sentiment",
+    "body": {
+      "version": 9,
+      "scoring": {
+        "mode": "all",
+        "threshold": null
+      },
+      "grader": null,
+      "every": [],
+      "run": [],
+      "cases": [
+        {
+          "id": "bakery-croissant",
+          "item": "bakery-croissant.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*positive\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is positive. The writer enjoyed the warm, flaky croissant and the friendly service, and plans to return.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "train-delay",
+          "item": "train-delay.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*negative\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is negative. The writer is frustrated by repeated cancellations with no notice or alternative, which made them late.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "library-hours",
+          "item": "library-hours.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*neutral\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is neutral. It is a factual notice of changed opening hours with no opinion either way.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "headphones-review",
+          "item": "headphones-review.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*positive\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is positive. The writer praises the battery life and noise cancelling and calls them their best purchase of the year.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "hotel-room",
+          "item": "hotel-room.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*negative\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is negative. The room did not match the advertised view, was noisy, and the hotel would not move them.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "meeting-minutes",
+          "item": "meeting-minutes.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*neutral\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is neutral. It reports what the committee decided and when it meets next, without expressing a feeling.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "garden-tomatoes",
+          "item": "garden-tomatoes.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*positive\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is positive. The writer is delighted with the taste of their first home-grown tomatoes and feels the effort paid off.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "phone-update",
+          "item": "phone-update.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*negative\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is negative. The update made the battery drain fast and apps crash, and the writer regrets installing it.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "bus-timetable",
+          "item": "bus-timetable.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*neutral\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is neutral. It states the bus frequency and last departure, a plain timetable fact.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "teacher-thanks",
+          "item": "teacher-thanks.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*positive\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is positive. The writer is grateful to a teacher whose extra help led to an exam result they are proud of.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "parcel-lost",
+          "item": "parcel-lost.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*negative\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is negative. The parcel never arrived and the complaints were closed without a reply, leaving the writer out of pocket.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "weather-report",
+          "item": "weather-report.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*neutral\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is neutral. It is a forecast of the weather, giving conditions without any opinion.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "concert-night",
+          "item": "concert-night.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*positive\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is positive. The writer loved the long concert and the shared singing, and would happily go again.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "gym-contract",
+          "item": "gym-contract.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*negative\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is negative. Cancelling was a long ordeal with a hidden fee, and the writer warns others away.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "recipe-note",
+          "item": "recipe-note.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*neutral\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is neutral. It gives cooking instructions and expresses no feeling.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "cafe-coffee",
+          "item": "cafe-coffee.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*negative\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is negative. The coffee was slow, lukewarm and burnt, and the staff ignored the writer without apology.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "puppy-home",
+          "item": "puppy-home.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*positive\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is positive. The writer is clearly fond of their new puppy and describes her with affection.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "road-closure",
+          "item": "road-closure.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*neutral\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is neutral. It announces a road closure and diversion, a factual notice.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "film-sequel",
+          "item": "film-sequel.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*negative\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is negative. The writer found the sequel long, loud and boring compared with the first film.",
+              "threshold": 0.5
+            }
+          ]
+        },
+        {
+          "id": "volunteer-day",
+          "item": "volunteer-day.txt",
+          "todo": false,
+          "note": "",
+          "metrics": [
+            {
+              "type": "regex",
+              "pattern": "^\\s*Label:\\s*positive\\s*\\.?\\s*$",
+              "flags": "im"
+            },
+            {
+              "type": "factuality",
+              "reference": "The text is positive. The writer enjoyed the clean-up, the sense of community and the day as a whole.",
+              "threshold": 0.5
+            }
+          ]
+        }
+      ]
+    },
+    "preview": {
+      "id": "sentiment",
+      "name": "Demo: classify sentiment"
+    }
+  }
+}

package/lab/demo/datasets/weather-alerts.json

@@ -0,0 +1,61 @@
+{
+  "format": "evals-lab/eval-group",
+  "version": 9,
+  "group": {
+    "name": "Demo: sort weather alerts",
+    "body": {
+      "version": 9,
+      "scoring": {
+        "mode": "all",
+        "threshold": null
+      },
+      "grader": null,
+      "every": [],
+      "run": [],
+      "cases": [
+        {
+          "id": "alert-types",
+          "item": "https-api.weather.gov-alerts-types.txt",
+          "todo": false,
+          "note": "The National Weather Service's alert types. The plain dangers belong in High and the notices with no hazard in Low.",
+          "metrics": [
+            {
+              "type": "contains",
+              "value": "Tornado Warning",
+              "ignoreCase": true
+            },
+            {
+              "type": "regex",
+              "pattern": "^\\W*High:.*\\bTornado Warning\\b",
+              "flags": "im"
+            },
+            {
+              "type": "regex",
+              "pattern": "^\\W*High:.*\\bTsunami Warning\\b",
+              "flags": "im"
+            },
+            {
+              "type": "regex",
+              "pattern": "^\\W*High:.*\\bHurricane Warning\\b",
+              "flags": "im"
+            },
+            {
+              "type": "regex",
+              "pattern": "^\\W*Low:.*\\bAdministrative Message\\b",
+              "flags": "im"
+            },
+            {
+              "type": "regex",
+              "pattern": "^\\W*Low:.*\\bTest\\b",
+              "flags": "im"
+            }
+          ]
+        }
+      ]
+    },
+    "preview": {
+      "id": "weather-alerts",
+      "name": "Demo: sort weather alerts"
+    }
+  }
+}

package/lab/demo/datasets/wordings.json

@@ -0,0 +1,39 @@
+{
+  "format": "evals-lab/eval-group",
+  "version": 9,
+  "group": {
+    "name": "Demo: compare wordings",
+    "body": {
+      "version": 9,
+      "scoring": {
+        "mode": "all",
+        "threshold": null
+      },
+      "grader": null,
+      "every": [
+        {
+          "type": "contains",
+          "value": "Mercury",
+          "ignoreCase": true
+        },
+        {
+          "type": "contains",
+          "value": "Venus",
+          "not": true,
+          "ignoreCase": true
+        },
+        {
+          "type": "regex",
+          "pattern": "^\\W*Mercury\\W*$",
+          "flags": "i"
+        },
+        {
+          "type": "latency",
+          "max": 10000
+        }
+      ],
+      "run": [],
+      "cases": []
+    }
+  }
+}

package/lab/demo/manifest.json

@@ -3,10 +3,10 @@
   "version": 1,
   "id": "demo",
   "name": "Demo",
-  "packVersion": "2026.10.02",
+  "packVersion": "2026.10.11",
   "requires": {
-    "lab": ">=12",
+    "lab": ">=17",
     "plugins": []
   },
-  "description": "Two graded sets of public-domain images, each with a starter pipeline."
+  "description": "Five demos, each a dataset with a starter pipeline: two sets of public-domain images, short texts to classify, one question in four wordings, and a dynamic Source's snapshot."
 }

package/lab/demo/pipelines/demo-1.json

@@ -1,7 +1,7 @@
 {
-  "version": 15,
+  "version": 17,
   "id": "demo-1",
-  "name": "Demo 1",
+  "name": "Demo: describe images",
   "jobs": [
     {
       "type": "job",
@@ -14,7 +14,7 @@
             "type": "source",
             "ref": {
               "id": "Demo 1",
-              "name": "Demo 1"
+              "name": "Demo: describe images"
             },
             "first": null,
             "loops": 1
@@ -105,10 +105,11 @@
     {
       "id": "s1",
       "name": "",
-      "profile": {
+      "connection": {
         "id": "vision",
         "name": "Vision"
       },
+      "profile": null,
       "steps": [
         {
           "type": "prompt",
@@ -119,10 +120,11 @@
     {
       "id": "s2",
       "name": "",
-      "profile": {
+      "connection": {
         "id": "vision",
         "name": "Vision"
       },
+      "profile": null,
       "steps": [
         {
           "type": "prompt",
@@ -139,7 +141,7 @@
       "continueOnFailure": true,
       "group": {
         "id": "demo-1",
-        "name": "Demo 1"
+        "name": "Demo: describe images"
       },
       "pin": null
     }

package/lab/demo/pipelines/demo-2.json

@@ -1,7 +1,7 @@
 {
-  "version": 15,
+  "version": 17,
   "id": "demo-2",
-  "name": "Demo 2",
+  "name": "Demo: describe images 2",
   "jobs": [
     {
       "type": "job",
@@ -14,7 +14,7 @@
             "type": "source",
             "ref": {
               "id": "Demo 2",
-              "name": "Demo 2"
+              "name": "Demo: describe images 2"
             },
             "first": null,
             "loops": 1
@@ -105,10 +105,11 @@
     {
       "id": "s1",
       "name": "",
-      "profile": {
+      "connection": {
         "id": "vision",
         "name": "Vision"
       },
+      "profile": null,
       "steps": [
         {
           "type": "prompt",
@@ -119,10 +120,11 @@
     {
       "id": "s2",
       "name": "",
-      "profile": {
+      "connection": {
         "id": "vision",
         "name": "Vision"
       },
+      "profile": null,
       "steps": [
         {
           "type": "prompt",
@@ -139,7 +141,7 @@
       "continueOnFailure": true,
       "group": {
         "id": "demo-2",
-        "name": "Demo 2"
+        "name": "Demo: describe images 2"
       },
       "pin": null
     }

package/lab/demo/pipelines/sentiment.json

@@ -0,0 +1,65 @@
+{
+  "version": 17,
+  "id": "sentiment",
+  "name": "Demo: classify sentiment",
+  "jobs": [
+    {
+      "type": "job",
+      "id": "j1",
+      "name": "",
+      "steps": [
+        {
+          "type": "attachContent",
+          "content": {
+            "type": "source",
+            "ref": {
+              "id": "sentiment",
+              "name": "Demo: classify sentiment"
+            },
+            "first": null,
+            "loops": 1
+          }
+        },
+        {
+          "type": "readReply",
+          "out": {
+            "kind": "text"
+          }
+        }
+      ]
+    }
+  ],
+  "targets": [
+    {
+      "id": "s1",
+      "name": "",
+      "connection": {
+        "id": "model",
+        "name": "Model"
+      },
+      "profile": null,
+      "steps": [
+        {
+          "type": "prompt",
+          "prompt": "Read the text below and say how its writer feels about what they describe.\n\nReply with two lines and nothing else:\nLabel: positive, negative or neutral\nReason: one sentence saying why\n\nText:\n{text}"
+        }
+      ]
+    }
+  ],
+  "evals": [
+    {
+      "id": "t1",
+      "name": "",
+      "type": "group",
+      "continueOnFailure": true,
+      "group": {
+        "id": "sentiment",
+        "name": "Demo: classify sentiment"
+      },
+      "pin": null
+    }
+  ],
+  "pass": {
+    "mode": "all"
+  }
+}