evals-lab 0.13.0 → 0.15.0

package/lab/demo/pipelines/documents.json

@@ -0,0 +1,101 @@
+{
+  "version": 19,
+  "id": "documents",
+  "name": "Demo: documents pipeline",
+  "jobs": [
+    {
+      "type": "job",
+      "id": "j1",
+      "name": "",
+      "steps": [
+        {
+          "type": "attachContent",
+          "content": {
+            "type": "source",
+            "ref": {
+              "id": "documents",
+              "name": "Demo: documents source"
+            },
+            "first": null,
+            "loops": 1
+          }
+        },
+        {
+          "type": "attachImage"
+        },
+        {
+          "type": "filters",
+          "filterSets": [
+            {
+              "id": "filters-j1",
+              "set": null,
+              "pin": null,
+              "own": {
+                "version": 4,
+                "filters": [
+                  {
+                    "id": "unfence",
+                    "type": "remove",
+                    "match": {},
+                    "options": {
+                      "ignoreFailures": false,
+                      "repeat": 1
+                    },
+                    "tests": [],
+                    "op": {
+                      "kind": "fences"
+                    }
+                  }
+                ]
+              }
+            }
+          ]
+        }
+      ]
+    }
+  ],
+  "targets": [
+    {
+      "id": "s1",
+      "name": "",
+      "connection": {
+        "id": "model",
+        "name": "Model"
+      },
+      "profile": null,
+      "steps": [
+        {
+          "type": "prompt",
+          "prompt": "Read the attached document. Reply with JSON only: {\"valid\": true or false (false if you can't read it as a document), \"type\": \"invoice\" | \"letter\" | \"receipt\" | \"report\" | \"other\"}"
+        }
+      ]
+    }
+  ],
+  "evals": [
+    {
+      "id": "t1",
+      "name": "",
+      "type": "group",
+      "continueOnFailure": true,
+      "group": {
+        "id": "documents-valid",
+        "name": "PDF is valid"
+      },
+      "pin": null
+    },
+    {
+      "id": "t2",
+      "name": "",
+      "type": "group",
+      "continueOnFailure": true,
+      "group": {
+        "id": "documents-type",
+        "name": "Document type"
+      },
+      "pin": null
+    }
+  ],
+  "pass": {
+    "mode": "all"
+  }
+}

package/lab/demo/pipelines/sentiment.json

@@ -1,7 +1,7 @@
 {
-  "version": 17,
+  "version": 19,
   "id": "sentiment",
-  "name": "Demo: classify sentiment",
+  "name": "Demo: sentiment pipeline",
   "jobs": [
     {
       "type": "job",
@@ -14,7 +14,7 @@
             "type": "source",
             "ref": {
               "id": "sentiment",
-              "name": "Demo: classify sentiment"
+              "name": "Demo: sentiment source"
             },
             "first": null,
             "loops": 1
@@ -54,7 +54,7 @@
       "continueOnFailure": true,
       "group": {
         "id": "sentiment",
-        "name": "Demo: classify sentiment"
+        "name": "Demo: sentiment evals"
       },
       "pin": null
     }

package/lab/demo/pipelines/weather-alerts.json

@@ -1,7 +1,7 @@
 {
-  "version": 17,
+  "version": 19,
   "id": "weather-alerts",
-  "name": "Demo: sort weather alerts",
+  "name": "Demo: weather pipeline",
   "jobs": [
     {
       "type": "job",
@@ -14,11 +14,15 @@
             "type": "source",
             "ref": {
               "id": "weather-alerts",
-              "name": "Demo: sort weather alerts"
+              "name": "Demo: weather source"
             },
             "first": null,
             "loops": 1,
-            "live": false
+            "live": false,
+            "pin": {
+              "name": "https-api.weather.gov-alerts-types.txt",
+              "n": 0
+            }
           }
         },
         {
@@ -55,7 +59,7 @@
       "continueOnFailure": true,
       "group": {
         "id": "weather-alerts",
-        "name": "Demo: sort weather alerts"
+        "name": "Demo: weather evals"
       },
       "pin": null
     }

package/lab/demo/pipelines/wordings.json

@@ -1,7 +1,7 @@
 {
-  "version": 17,
+  "version": 19,
   "id": "wordings",
-  "name": "Demo: compare wordings",
+  "name": "Demo: wordings pipeline",
   "jobs": [
     {
       "type": "job",
@@ -93,7 +93,7 @@
       "continueOnFailure": true,
       "group": {
         "id": "wordings",
-        "name": "Demo: compare wordings"
+        "name": "Demo: wordings evals"
       },
       "pin": null
     }

package/lab/demo/sources/Demo 1/source.json

@@ -1,3 +1,3 @@
 {
-  "name": "Demo: describe images"
+  "name": "Demo: images source"
 }

package/lab/demo/sources/Demo 2/source.json

@@ -1,3 +1,3 @@
 {
-  "name": "Demo: describe images 2"
+  "name": "Demo: images 2 source"
 }

package/lab/demo/sources/documents/invoice.pdf

@@ -0,0 +1,65 @@
+%PDF-1.4
+%����
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+3 0 obj
+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R /F2 5 0 R >> >> /Contents 6 0 R >>
+endobj
+4 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>
+endobj
+5 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold >>
+endobj
+6 0 obj
+<< /Length 688 >>
+stream
+BT
+/F2 18 Tf
+72 760 Td
+(INVOICE) Tj
+/F1 11 Tf
+0 -30 Td
+14 TL
+(Invoice number: INV-2026-0142) '
+(Date: 3 October 2026) '
+(Due: 2 November 2026) '
+() '
+(From: Hartley Joinery Ltd, 14 Mill Lane, Ashford) '
+(Bill to: Northgate Community Centre, 2 Station Road) '
+() '
+(Description                              Qty     Unit      Amount) '
+(Oak shelving units, fitted                 4    120.00      480.00) '
+(Replacement door hinges                    6      8.50       51.00) '
+(Labour \(hours\)                             5     35.00      175.00) '
+() '
+(Subtotal: 706.00) '
+(VAT \(20%\): 141.20) '
+(Total due: 847.20) '
+() '
+(Please pay by bank transfer quoting the invoice number.) '
+ET
+endstream
+endobj
+7 0 obj
+<< /Title (INVOICE) /Producer (Evals Lab demo pack) >>
+endobj
+xref
+0 8
+0000000000 65535 f 
+0000000015 00000 n 
+0000000064 00000 n 
+0000000121 00000 n 
+0000000257 00000 n 
+0000000327 00000 n 
+0000000402 00000 n 
+0000001141 00000 n 
+trailer
+<< /Size 8 /Root 1 0 R /Info 7 0 R >>
+startxref
+1211
+%%EOF

package/lab/demo/sources/documents/letter.pdf

@@ -0,0 +1,63 @@
+%PDF-1.4
+%����
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+3 0 obj
+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R /F2 5 0 R >> >> /Contents 6 0 R >>
+endobj
+4 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>
+endobj
+5 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold >>
+endobj
+6 0 obj
+<< /Length 471 >>
+stream
+BT
+/F2 18 Tf
+72 760 Td
+() Tj
+/F1 11 Tf
+0 -30 Td
+14 TL
+(12 Willow Crescent) '
+(Bramley) '
+(5 October 2026) '
+() '
+(Dear Ms Okafor,) '
+() '
+(Thank you for your help at the library's book sale last Saturday. Thanks to) '
+(volunteers like you we raised enough to replace the children's reading chairs.) '
+() '
+(We hope to hold another sale in the spring, and would be glad to see you there.) '
+() '
+(Yours sincerely,) '
+() '
+(Daniel Price) '
+(Friends of Bramley Library) '
+ET
+endstream
+endobj
+7 0 obj
+<< /Title () /Producer (Evals Lab demo pack) >>
+endobj
+xref
+0 8
+0000000000 65535 f 
+0000000015 00000 n 
+0000000064 00000 n 
+0000000121 00000 n 
+0000000257 00000 n 
+0000000327 00000 n 
+0000000402 00000 n 
+0000000924 00000 n 
+trailer
+<< /Size 8 /Root 1 0 R /Info 7 0 R >>
+startxref
+987
+%%EOF

package/lab/demo/sources/documents/not-a-pdf.pdf

@@ -0,0 +1,2 @@
+This is a plain text file with a .pdf name. It holds no PDF at all:
+no header, no pages, no fonts. A reader expecting a PDF should not be able to open it.

package/lab/demo/sources/documents/receipt.pdf

@@ -0,0 +1,62 @@
+%PDF-1.4
+%����
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+3 0 obj
+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R /F2 5 0 R >> >> /Contents 6 0 R >>
+endobj
+4 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>
+endobj
+5 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold >>
+endobj
+6 0 obj
+<< /Length 490 >>
+stream
+BT
+/F2 18 Tf
+72 760 Td
+(RECEIPT) Tj
+/F1 11 Tf
+0 -30 Td
+14 TL
+(Corner Grocer, 88 High Street) '
+(Receipt no. 004417) '
+(Date: 7 October 2026  14:32) '
+() '
+(Bread, wholemeal                1.45) '
+(Milk, 2 litres                  1.60) '
+(Apples, 1 kg                    2.20) '
+(Cheddar, 400 g                  3.75) '
+() '
+(Total                           9.00) '
+(Paid by card                    9.00) '
+(Change                          0.00) '
+() '
+(Thank you for shopping with us.) '
+ET
+endstream
+endobj
+7 0 obj
+<< /Title (RECEIPT) /Producer (Evals Lab demo pack) >>
+endobj
+xref
+0 8
+0000000000 65535 f 
+0000000015 00000 n 
+0000000064 00000 n 
+0000000121 00000 n 
+0000000257 00000 n 
+0000000327 00000 n 
+0000000402 00000 n 
+0000000943 00000 n 
+trailer
+<< /Size 8 /Root 1 0 R /Info 7 0 R >>
+startxref
+1013
+%%EOF

package/lab/demo/sources/documents/report.pdf

@@ -0,0 +1,59 @@
+%PDF-1.4
+%����
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+3 0 obj
+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R /F2 5 0 R >> >> /Contents 6 0 R >>
+endobj
+4 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>
+endobj
+5 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold >>
+endobj
+6 0 obj
+<< /Length 459 >>
+stream
+BT
+/F2 18 Tf
+72 760 Td
+(Quarterly Report: Q3 2026) Tj
+/F1 11 Tf
+0 -30 Td
+14 TL
+(Riverside Allotment Society) '
+() '
+(1. Summary) '
+(Membership rose from 64 to 71 plots this quarter. Water use fell by 12 per cent) '
+(after the new butts were fitted.) '
+() '
+(2. Finances) '
+(Income from rents: 2,130.00. Spending on repairs and tools: 1,480.00.) '
+() '
+(3. Recommendations) '
+(Replace the north gate before winter, and review the waiting list in January.) '
+ET
+endstream
+endobj
+7 0 obj
+<< /Title (Quarterly Report: Q3 2026) /Producer (Evals Lab demo pack) >>
+endobj
+xref
+0 8
+0000000000 65535 f 
+0000000015 00000 n 
+0000000064 00000 n 
+0000000121 00000 n 
+0000000257 00000 n 
+0000000327 00000 n 
+0000000402 00000 n 
+0000000912 00000 n 
+trailer
+<< /Size 8 /Root 1 0 R /Info 7 0 R >>
+startxref
+1000
+%%EOF

package/lab/demo/sources/documents/source.json

@@ -0,0 +1,3 @@
+{
+  "name": "Demo: documents source"
+}

package/lab/demo/sources/documents/truncated.pdf

@@ -0,0 +1,19 @@
+%PDF-1.4
+%����
+1 0 obj
+<< /Type /Catalog /Pages 2 0 R >>
+endobj
+2 0 obj
+<< /Type /Pages /Kids [3 0 R] /Count 1 >>
+endobj
+3 0 obj
+<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 4 0 R /F2 5 0 R >> >> /Contents 6 0 R >>
+endobj
+4 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>
+endobj
+5 0 obj
+<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold >>
+endobj
+6 0 obj
+<< /Le

package/lab/demo/sources/sentiment/source.json

@@ -1,3 +1,3 @@
 {
-  "name": "Demo: classify sentiment"
+  "name": "Demo: sentiment source"
 }

package/lab/demo/sources/weather-alerts/source.json

@@ -1,5 +1,5 @@
 {
-  "name": "Demo: sort weather alerts",
+  "name": "Demo: weather source",
   "type": "dynamic",
   "config": {
     "provider": "http-get",