evals-lab 0.1.2

package/lab/demo/manifest.json

@@ -0,0 +1,12 @@
+{
+  "format": "evals-lab/pack",
+  "version": 1,
+  "id": "demo",
+  "name": "Demo",
+  "packVersion": "2026.09.29",
+  "requires": {
+    "lab": ">=8",
+    "plugins": []
+  },
+  "description": "Two graded sets of public-domain images, each with a starter pipeline."
+}

package/lab/demo/pipelines/demo-1.json

@@ -0,0 +1,104 @@
+{
+  "version": 9,
+  "id": "demo-1",
+  "name": "Demo 1",
+  "jobs": [
+    {
+      "type": "job",
+      "id": "j1",
+      "name": "",
+      "steps": [
+        {
+          "type": "attachContent",
+          "content": {
+            "type": "source",
+            "ref": {
+              "id": "Demo 1",
+              "name": "Demo 1"
+            },
+            "first": null,
+            "loops": 1
+          }
+        },
+        {
+          "type": "prompt",
+          "withImage": true,
+          "tokenMappings": []
+        },
+        {
+          "type": "readReply",
+          "out": {
+            "kind": "list",
+            "parse": "csv",
+            "preamble": false,
+            "modifiers": [
+              {
+                "type": "tidy"
+              },
+              {
+                "type": "dedupe"
+              },
+              {
+                "type": "keep-first",
+                "n": 12
+              },
+              {
+                "type": "case",
+                "mode": "LOWER"
+              }
+            ]
+          }
+        }
+      ]
+    }
+  ],
+  "scenarios": [
+    {
+      "id": "s1",
+      "name": "",
+      "profile": {
+        "id": "vision",
+        "name": "Vision"
+      },
+      "stages": [
+        {
+          "prompt": "List the things you can see in this image, as a comma-separated list of short items (one to three words each), most important first. Up to 12 items."
+        }
+      ]
+    },
+    {
+      "id": "s2",
+      "name": "",
+      "profile": {
+        "id": "vision",
+        "name": "Vision"
+      },
+      "stages": [
+        {
+          "prompt": "Name what is in this image. Reply with a comma-separated list only: up to 12 short items, most important first."
+        }
+      ]
+    }
+  ],
+  "tests": [
+    {
+      "id": "t1",
+      "name": "",
+      "continueOnFailure": true,
+      "type": "metrics",
+      "mode": "all",
+      "threshold": null,
+      "grader": null,
+      "dataset": {
+        "id": "demo-1",
+        "name": "Demo 1"
+      },
+      "over": "item",
+      "metrics": [
+        {
+          "type": "case"
+        }
+      ]
+    }
+  ]
+}

package/lab/demo/pipelines/demo-2.json

@@ -0,0 +1,104 @@
+{
+  "version": 9,
+  "id": "demo-2",
+  "name": "Demo 2",
+  "jobs": [
+    {
+      "type": "job",
+      "id": "j1",
+      "name": "",
+      "steps": [
+        {
+          "type": "attachContent",
+          "content": {
+            "type": "source",
+            "ref": {
+              "id": "Demo 2",
+              "name": "Demo 2"
+            },
+            "first": null,
+            "loops": 1
+          }
+        },
+        {
+          "type": "prompt",
+          "withImage": true,
+          "tokenMappings": []
+        },
+        {
+          "type": "readReply",
+          "out": {
+            "kind": "list",
+            "parse": "csv",
+            "preamble": false,
+            "modifiers": [
+              {
+                "type": "tidy"
+              },
+              {
+                "type": "dedupe"
+              },
+              {
+                "type": "keep-first",
+                "n": 12
+              },
+              {
+                "type": "case",
+                "mode": "LOWER"
+              }
+            ]
+          }
+        }
+      ]
+    }
+  ],
+  "scenarios": [
+    {
+      "id": "s1",
+      "name": "",
+      "profile": {
+        "id": "vision",
+        "name": "Vision"
+      },
+      "stages": [
+        {
+          "prompt": "List the things you can see in this image, as a comma-separated list of short items (one to three words each), most important first. Up to 12 items."
+        }
+      ]
+    },
+    {
+      "id": "s2",
+      "name": "",
+      "profile": {
+        "id": "vision",
+        "name": "Vision"
+      },
+      "stages": [
+        {
+          "prompt": "Name what is in this image. Reply with a comma-separated list only: up to 12 short items, most important first."
+        }
+      ]
+    }
+  ],
+  "tests": [
+    {
+      "id": "t1",
+      "name": "",
+      "continueOnFailure": true,
+      "type": "metrics",
+      "mode": "all",
+      "threshold": null,
+      "grader": null,
+      "dataset": {
+        "id": "demo-2",
+        "name": "Demo 2"
+      },
+      "over": "item",
+      "metrics": [
+        {
+          "type": "case"
+        }
+      ]
+    }
+  ]
+}