evals-lab 0.13.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/CHANGELOG.md +83 -0
  2. package/README.md +2 -2
  3. package/bin/run.js +4 -2
  4. package/lab/VERSION +1 -1
  5. package/lab/demo/CREDITS.md +14 -0
  6. package/lab/demo/datasets/demo-1.json +2 -2
  7. package/lab/demo/datasets/demo-2.json +2 -2
  8. package/lab/demo/datasets/documents-type.json +79 -0
  9. package/lab/demo/datasets/documents-valid.json +117 -0
  10. package/lab/demo/datasets/sentiment.json +2 -2
  11. package/lab/demo/datasets/weather-alerts.json +2 -2
  12. package/lab/demo/datasets/wordings.json +1 -1
  13. package/lab/demo/manifest.json +3 -3
  14. package/lab/demo/pipelines/demo-1.json +17 -14
  15. package/lab/demo/pipelines/demo-2.json +17 -14
  16. package/lab/demo/pipelines/documents.json +101 -0
  17. package/lab/demo/pipelines/sentiment.json +4 -4
  18. package/lab/demo/pipelines/weather-alerts.json +9 -5
  19. package/lab/demo/pipelines/wordings.json +3 -3
  20. package/lab/demo/sources/Demo 1/source.json +1 -1
  21. package/lab/demo/sources/Demo 2/source.json +1 -1
  22. package/lab/demo/sources/documents/invoice.pdf +65 -0
  23. package/lab/demo/sources/documents/letter.pdf +63 -0
  24. package/lab/demo/sources/documents/not-a-pdf.pdf +2 -0
  25. package/lab/demo/sources/documents/receipt.pdf +62 -0
  26. package/lab/demo/sources/documents/report.pdf +59 -0
  27. package/lab/demo/sources/documents/source.json +3 -0
  28. package/lab/demo/sources/documents/truncated.pdf +19 -0
  29. package/lab/demo/sources/sentiment/source.json +1 -1
  30. package/lab/demo/sources/weather-alerts/source.json +1 -1
  31. package/lab/evals-core.mjs +408 -160
  32. package/lab/flows/record.mjs +5 -4
  33. package/lab/kinds/list.mjs +337 -182
  34. package/lab/run-evals.js +3 -1
  35. package/lab/server.py +661 -108
  36. package/lab/web/dist/assets/gallery-Cy8oycht.js +5 -0
  37. package/lab/web/dist/assets/gallery-eVJoaK-6.css +1 -0
  38. package/lab/web/dist/assets/main-B-LGmhPu.css +1 -0
  39. package/lab/web/dist/assets/main-wqNvXpWp.js +15 -0
  40. package/lab/web/dist/assets/tokens-CWVzmtid.css +1 -0
  41. package/lab/web/dist/assets/tokens-mCS6IZGh.js +64 -0
  42. package/lab/web/dist/gallery.html +4 -4
  43. package/lab/web/dist/index.html +4 -4
  44. package/package.json +1 -1
  45. package/lab/web/dist/assets/gallery-B_-TH0F-.css +0 -1
  46. package/lab/web/dist/assets/gallery-DpaqOg9Z.js +0 -5
  47. package/lab/web/dist/assets/main-BSw-jyYD.css +0 -1
  48. package/lab/web/dist/assets/main-CrB8wPZn.js +0 -17
  49. package/lab/web/dist/assets/tokens-CrjDMCbz.js +0 -60
  50. package/lab/web/dist/assets/tokens-DDOONQon.css +0 -1
@@ -153,11 +153,12 @@ export function skeletonFor(step , reads ) {
153
153
  call of the step at that iteration: imported twice, it is skipped the
154
154
  second time, while another run's call -- the same request, sent again,
155
155
  answered afresh -- is its own record, since a flow that sends the same
156
- words every time is exactly where production's replies differ. One
157
- written by hand or uploaded is the same as another that read and sent the
158
- same things. */
156
+ words every time is exactly where production's replies differ. An edited
157
+ call keeps its run's key, so Import runs skips it too (#587). One written
158
+ by hand or uploaded is the same as another that read and sent the same
159
+ things. Mirrored by server.py's record_key. */
159
160
  export function recordKey(r ) {
160
- if (r.origin?.kind === "run" && r.origin.run) {
161
+ if (r.origin?.run) {
161
162
  return JSON.stringify({ step: r.step, run: r.origin.run, iteration: r.origin.iteration ?? "" });
162
163
  }
163
164
  const sorted = (v ) => Array.isArray(v) ? v.map(sorted)