evals-lab 0.14.0 → 0.16.0

package/lab/demo/pipelines/sentiment.json

@@ -1,35 +1,40 @@
 {
-  "version": 18,
+  "version": 20,
   "id": "sentiment",
   "name": "Demo: sentiment pipeline",
-  "jobs": [
+  "steps": [
     {
-      "type": "job",
+      "type": "getItems",
+      "id": "items-j1",
+      "content": {
+        "type": "source",
+        "ref": {
+          "id": "sentiment",
+          "name": "Demo: sentiment source"
+        },
+        "first": null,
+        "loops": 1
+      }
+    },
+    {
+      "type": "request",
       "id": "j1",
       "name": "",
-      "steps": [
-        {
-          "type": "attachContent",
-          "content": {
-            "type": "source",
-            "ref": {
-              "id": "sentiment",
-              "name": "Demo: sentiment source"
-            },
-            "first": null,
-            "loops": 1
-          }
-        },
-        {
-          "type": "readReply",
-          "out": {
-            "kind": "text"
-          }
-        }
-      ]
+      "cell": {
+        "type": "prompt",
+        "prompt": "Read the text below and say how its writer feels about what they describe.\n\nReply with two lines and nothing else:\nLabel: positive, negative or neutral\nReason: one sentence saying why\n\nText:\n{text}"
+      }
+    },
+    {
+      "type": "readResponse",
+      "id": "read-j1",
+      "request": "j1",
+      "out": {
+        "kind": "text"
+      }
     }
   ],
-  "targets": [
+  "variants": [
     {
       "id": "s1",
       "name": "",
@@ -38,12 +43,7 @@
         "name": "Model"
       },
       "profile": null,
-      "steps": [
-        {
-          "type": "prompt",
-          "prompt": "Read the text below and say how its writer feels about what they describe.\n\nReply with two lines and nothing else:\nLabel: positive, negative or neutral\nReason: one sentence saying why\n\nText:\n{text}"
-        }
-      ]
+      "overrides": {}
     }
   ],
   "evals": [

package/lab/demo/pipelines/weather-alerts.json

@@ -1,40 +1,45 @@
 {
-  "version": 18,
+  "version": 20,
   "id": "weather-alerts",
   "name": "Demo: weather pipeline",
-  "jobs": [
+  "steps": [
     {
-      "type": "job",
-      "id": "j1",
-      "name": "",
-      "steps": [
-        {
-          "type": "attachContent",
-          "content": {
-            "type": "source",
-            "ref": {
-              "id": "weather-alerts",
-              "name": "Demo: weather source"
-            },
-            "first": null,
-            "loops": 1,
-            "live": false,
-            "pin": {
-              "name": "https-api.weather.gov-alerts-types.txt",
-              "n": 0
-            }
-          }
+      "type": "getItems",
+      "id": "items-j1",
+      "content": {
+        "type": "source",
+        "ref": {
+          "id": "weather-alerts",
+          "name": "Demo: weather source"
         },
-        {
-          "type": "readReply",
-          "out": {
-            "kind": "text"
-          }
+        "first": null,
+        "loops": 1,
+        "live": false,
+        "pin": {
+          "name": "https-api.weather.gov-alerts-types.txt",
+          "n": 0
         }
-      ]
+      }
+    },
+    {
+      "type": "request",
+      "id": "j1",
+      "name": "",
+      "cell": {
+        "type": "prompt",
+        "prompt": "Below is the US National Weather Service's list of alert types, as JSON.\n\nSort every alert type on it by hazard into three groups. Reply with exactly three lines and nothing else:\nHigh: the types that warn of a threat to life\nMedium: the types that warn of damage or disruption\nLow: the rest\nSeparate the types on a line with commas, and write each exactly as the list does.\n\n{text}"
+      }
+    },
+    {
+      "type": "readResponse",
+      "id": "read-j1",
+      "request": "j1",
+      "out": {
+        "kind": "text"
+      }
     }
   ],
-  "targets": [
+  "variants": [
     {
       "id": "s1",
       "name": "",
@@ -43,12 +48,7 @@
         "name": "Model"
       },
       "profile": null,
-      "steps": [
-        {
-          "type": "prompt",
-          "prompt": "Below is the US National Weather Service's list of alert types, as JSON.\n\nSort every alert type on it by hazard into three groups. Reply with exactly three lines and nothing else:\nHigh: the types that warn of a threat to life\nMedium: the types that warn of damage or disruption\nLow: the rest\nSeparate the types on a line with commas, and write each exactly as the list does.\n\n{text}"
-        }
-      ]
+      "overrides": {}
     }
   ],
   "evals": [

package/lab/demo/pipelines/wordings.json

@@ -1,29 +1,34 @@
 {
-  "version": 18,
+  "version": 20,
   "id": "wordings",
   "name": "Demo: wordings pipeline",
-  "jobs": [
+  "steps": [
     {
-      "type": "job",
+      "type": "getItems",
+      "id": "items-j1",
+      "content": {
+        "type": "prompt"
+      }
+    },
+    {
+      "type": "request",
       "id": "j1",
       "name": "",
-      "steps": [
-        {
-          "type": "attachContent",
-          "content": {
-            "type": "prompt"
-          }
-        },
-        {
-          "type": "readReply",
-          "out": {
-            "kind": "text"
-          }
-        }
-      ]
+      "cell": {
+        "type": "prompt",
+        "prompt": "Which planet is closest to the Sun?"
+      }
+    },
+    {
+      "type": "readResponse",
+      "id": "read-j1",
+      "request": "j1",
+      "out": {
+        "kind": "text"
+      }
     }
   ],
-  "targets": [
+  "variants": [
     {
       "id": "s1",
       "name": "",
@@ -32,12 +37,7 @@
         "name": "Model"
       },
       "profile": null,
-      "steps": [
-        {
-          "type": "prompt",
-          "prompt": "Which planet is closest to the Sun?"
-        }
-      ]
+      "overrides": {}
     },
     {
       "id": "s2",
@@ -47,12 +47,11 @@
         "name": "Model"
       },
       "profile": null,
-      "steps": [
-        {
-          "type": "prompt",
+      "overrides": {
+        "j1": {
           "prompt": "Name the planet nearest the Sun. Reply with its name only."
         }
-      ]
+      }
     },
     {
       "id": "s3",
@@ -62,12 +61,11 @@
         "name": "Model"
       },
       "profile": null,
-      "steps": [
-        {
-          "type": "prompt",
+      "overrides": {
+        "j1": {
           "prompt": "Of the eight planets in our solar system, which one orbits closest to the Sun? Answer in one word."
         }
-      ]
+      }
     },
     {
       "id": "s4",
@@ -77,12 +75,11 @@
         "name": "Model"
       },
       "profile": null,
-      "steps": [
-        {
-          "type": "prompt",
+      "overrides": {
+        "j1": {
           "prompt": "Quick question: what's the innermost planet?"
         }
-      ]
+      }
     }
   ],
   "evals": [