evals-lab 0.13.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. package/CHANGELOG.md +83 -0
  2. package/README.md +2 -2
  3. package/bin/run.js +4 -2
  4. package/lab/VERSION +1 -1
  5. package/lab/demo/CREDITS.md +14 -0
  6. package/lab/demo/datasets/demo-1.json +2 -2
  7. package/lab/demo/datasets/demo-2.json +2 -2
  8. package/lab/demo/datasets/documents-type.json +79 -0
  9. package/lab/demo/datasets/documents-valid.json +117 -0
  10. package/lab/demo/datasets/sentiment.json +2 -2
  11. package/lab/demo/datasets/weather-alerts.json +2 -2
  12. package/lab/demo/datasets/wordings.json +1 -1
  13. package/lab/demo/manifest.json +3 -3
  14. package/lab/demo/pipelines/demo-1.json +17 -14
  15. package/lab/demo/pipelines/demo-2.json +17 -14
  16. package/lab/demo/pipelines/documents.json +101 -0
  17. package/lab/demo/pipelines/sentiment.json +4 -4
  18. package/lab/demo/pipelines/weather-alerts.json +9 -5
  19. package/lab/demo/pipelines/wordings.json +3 -3
  20. package/lab/demo/sources/Demo 1/source.json +1 -1
  21. package/lab/demo/sources/Demo 2/source.json +1 -1
  22. package/lab/demo/sources/documents/invoice.pdf +65 -0
  23. package/lab/demo/sources/documents/letter.pdf +63 -0
  24. package/lab/demo/sources/documents/not-a-pdf.pdf +2 -0
  25. package/lab/demo/sources/documents/receipt.pdf +62 -0
  26. package/lab/demo/sources/documents/report.pdf +59 -0
  27. package/lab/demo/sources/documents/source.json +3 -0
  28. package/lab/demo/sources/documents/truncated.pdf +19 -0
  29. package/lab/demo/sources/sentiment/source.json +1 -1
  30. package/lab/demo/sources/weather-alerts/source.json +1 -1
  31. package/lab/evals-core.mjs +408 -160
  32. package/lab/flows/record.mjs +5 -4
  33. package/lab/kinds/list.mjs +337 -182
  34. package/lab/run-evals.js +3 -1
  35. package/lab/server.py +661 -108
  36. package/lab/web/dist/assets/gallery-Cy8oycht.js +5 -0
  37. package/lab/web/dist/assets/gallery-eVJoaK-6.css +1 -0
  38. package/lab/web/dist/assets/main-B-LGmhPu.css +1 -0
  39. package/lab/web/dist/assets/main-wqNvXpWp.js +15 -0
  40. package/lab/web/dist/assets/tokens-CWVzmtid.css +1 -0
  41. package/lab/web/dist/assets/tokens-mCS6IZGh.js +64 -0
  42. package/lab/web/dist/gallery.html +4 -4
  43. package/lab/web/dist/index.html +4 -4
  44. package/package.json +1 -1
  45. package/lab/web/dist/assets/gallery-B_-TH0F-.css +0 -1
  46. package/lab/web/dist/assets/gallery-DpaqOg9Z.js +0 -5
  47. package/lab/web/dist/assets/main-BSw-jyYD.css +0 -1
  48. package/lab/web/dist/assets/main-CrB8wPZn.js +0 -17
  49. package/lab/web/dist/assets/tokens-CrjDMCbz.js +0 -60
  50. package/lab/web/dist/assets/tokens-DDOONQon.css +0 -1
package/lab/run-evals.js CHANGED
@@ -724,7 +724,9 @@ async function ask(to, c, prompt, dataUrl, schema = null) {
724
724
  if (!r.ok) return said({ error: j.error?.message || j.error || `HTTP ${r.status}` });
725
725
  const { raw, finishReason } = core.connectionReply(c, j);
726
726
  const error = core.connectionReplyProblem(c, raw, finishReason);
727
- return said(error ? { raw, error } : { raw });
727
+ // Why it stopped, and the limit sent where the limit stopped it (#597).
728
+ const finish = core.replyFinish(c, req.body, finishReason);
729
+ return said(error ? { raw, error, ...finish } : { raw, ...finish });
728
730
  } catch (e) {
729
731
  if (e?.name === "TimeoutError" || e?.name === "AbortError") {
730
732
  return said({ error: `the model gave no reply within ${seconds}s — timed out` });