evals-lab 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +57 -0
- package/README.md +98 -0
- package/bin/evals-lab.js +196 -0
- package/lab/demo/CREDITS.md +92 -0
- package/lab/demo/datasets/demo-1.json +1839 -0
- package/lab/demo/datasets/demo-2.json +1683 -0
- package/lab/demo/manifest.json +12 -0
- package/lab/demo/pipelines/demo-1.json +104 -0
- package/lab/demo/pipelines/demo-2.json +104 -0
- package/lab/demo/sources/Demo 1/basketball-hoop.jpg +0 -0
- package/lab/demo/sources/Demo 1/blue-boardwalk.jpg +0 -0
- package/lab/demo/sources/Demo 1/busy-beach.jpg +0 -0
- package/lab/demo/sources/Demo 1/butcher-sign.jpg +0 -0
- package/lab/demo/sources/Demo 1/cactus-flower.jpg +0 -0
- package/lab/demo/sources/Demo 1/corner-shop.jpg +0 -0
- package/lab/demo/sources/Demo 1/crosswalk-cyclist-panorama.jpg +0 -0
- package/lab/demo/sources/Demo 1/cyanotype-room.jpg +0 -0
- package/lab/demo/sources/Demo 1/dead-end-sign.jpg +0 -0
- package/lab/demo/sources/Demo 1/dog-in-snow.jpg +0 -0
- package/lab/demo/sources/Demo 1/empty-bedroom.jpg +0 -0
- package/lab/demo/sources/Demo 1/farmers-market-stall.jpg +0 -0
- package/lab/demo/sources/Demo 1/four-jets.jpg +0 -0
- package/lab/demo/sources/Demo 1/german-shepherd.jpg +0 -0
- package/lab/demo/sources/Demo 1/harbour-bridge-dusk.jpg +0 -0
- package/lab/demo/sources/Demo 1/harrow-on-the-hill-sign.jpg +0 -0
- package/lab/demo/sources/Demo 1/hong-kong-street.jpg +0 -0
- package/lab/demo/sources/Demo 1/house-salisbury-street.jpg +0 -0
- package/lab/demo/sources/Demo 1/infrared-orchard.jpg +0 -0
- package/lab/demo/sources/Demo 1/keyboard-desk.jpg +0 -0
- package/lab/demo/sources/Demo 1/lighthouse-dunes.jpg +0 -0
- package/lab/demo/sources/Demo 1/mountain-lake.jpg +0 -0
- package/lab/demo/sources/Demo 1/museum-skeleton.jpg +0 -0
- package/lab/demo/sources/Demo 1/music-room.jpg +0 -0
- package/lab/demo/sources/Demo 1/old-red-car.jpg +0 -0
- package/lab/demo/sources/Demo 1/parked-car-plate.jpg +0 -0
- package/lab/demo/sources/Demo 1/phone-and-wallet.png +0 -0
- package/lab/demo/sources/Demo 1/phone-box.jpg +0 -0
- package/lab/demo/sources/Demo 1/pink-flamingo.jpg +0 -0
- package/lab/demo/sources/Demo 1/postcards.jpg +0 -0
- package/lab/demo/sources/Demo 1/red-roof-church.jpg +0 -0
- package/lab/demo/sources/Demo 1/roadside-mailboxes.jpg +0 -0
- package/lab/demo/sources/Demo 1/rodeo.jpg +0 -0
- package/lab/demo/sources/Demo 1/running-tap.tif +0 -0
- package/lab/demo/sources/Demo 1/snail-on-stem.jpg +0 -0
- package/lab/demo/sources/Demo 1/two-horses-field.jpg +0 -0
- package/lab/demo/sources/Demo 1/university-sign.jpg +0 -0
- package/lab/demo/sources/Demo 1/vegetable-crates.jpg +0 -0
- package/lab/demo/sources/Demo 1/watermarked-car.jpg +0 -0
- package/lab/demo/sources/Demo 1/watermarked-pills.jpg +0 -0
- package/lab/demo/sources/Demo 1/whiteboard-delegate.jpg +0 -0
- package/lab/demo/sources/Demo 1/whiteboard-germ-layers.jpg +0 -0
- package/lab/demo/sources/Demo 2/aerial-city.jpg +0 -0
- package/lab/demo/sources/Demo 2/alligator-pen.jpg +0 -0
- package/lab/demo/sources/Demo 2/arm-tattoo.jpg +0 -0
- package/lab/demo/sources/Demo 2/bed-and-plant.jpg +0 -0
- package/lab/demo/sources/Demo 2/bright-bedroom.jpg +0 -0
- package/lab/demo/sources/Demo 2/car-headlight.jpg +0 -0
- package/lab/demo/sources/Demo 2/child-in-surf.jpg +0 -0
- package/lab/demo/sources/Demo 2/city-highway.jpg +0 -0
- package/lab/demo/sources/Demo 2/corner-bakery.jpg +0 -0
- package/lab/demo/sources/Demo 2/cyclist-yellow-jacket.jpg +0 -0
- package/lab/demo/sources/Demo 2/excavator-street-signs.jpg +0 -0
- package/lab/demo/sources/Demo 2/globe-closeup.jpg +0 -0
- package/lab/demo/sources/Demo 2/harbour-village.jpg +0 -0
- package/lab/demo/sources/Demo 2/high-street-walkers.jpg +0 -0
- package/lab/demo/sources/Demo 2/hillside-rooftops.jpg +0 -0
- package/lab/demo/sources/Demo 2/house-at-night.jpg +0 -0
- package/lab/demo/sources/Demo 2/ivy-leaves.jpg +0 -0
- package/lab/demo/sources/Demo 2/man-with-alligator.jpg +0 -0
- package/lab/demo/sources/Demo 2/orange-mailbox-hedge.jpg +0 -0
- package/lab/demo/sources/Demo 2/pedal-boats.jpg +0 -0
- package/lab/demo/sources/Demo 2/pink-trees-vignette.jpg +0 -0
- package/lab/demo/sources/Demo 2/please-leave-quietly-sign.jpg +0 -0
- package/lab/demo/sources/Demo 2/railroad-crossing-sign.jpg +0 -0
- package/lab/demo/sources/Demo 2/roadworks-sign.jpg +0 -0
- package/lab/demo/sources/Demo 2/roundabout-sign.jpg +0 -0
- package/lab/demo/sources/Demo 2/sea-cave.jpg +0 -0
- package/lab/demo/sources/Demo 2/shadow-on-sand.jpg +0 -0
- package/lab/demo/sources/Demo 2/signpost-a404.jpg +0 -0
- package/lab/demo/sources/Demo 2/street-fruit-cart.jpg +0 -0
- package/lab/demo/sources/Demo 2/street-sign-parnassusweg.jpg +0 -0
- package/lab/demo/sources/Demo 2/two-horses-close.jpg +0 -0
- package/lab/demo/sources/Demo 2/victorian-house.jpg +0 -0
- package/lab/demo/sources/Demo 2/vintage-dashboard.jpg +0 -0
- package/lab/demo/sources/Demo 2/volcano-at-dusk.jpg +0 -0
- package/lab/demo/sources/Demo 2/wall-camera.jpg +0 -0
- package/lab/demo/sources/Demo 2/watch-for-rocks-sign.jpg +0 -0
- package/lab/demo/sources/Demo 2/waterfall.jpg +0 -0
- package/lab/demo/sources/Demo 2/white-domes.jpg +0 -0
- package/lab/demo/sources/Demo 2/whiteboard-meeting-notes.jpg +0 -0
- package/lab/demo/sources/Demo 2/whiteboard-messages.jpg +0 -0
- package/lab/demo/sources/Demo 2/wooden-house-fence.jpg +0 -0
- package/lab/evals-core.mjs +4526 -0
- package/lab/flows/wdl.mjs +373 -0
- package/lab/js-yaml.mjs +3851 -0
- package/lab/kinds/list.mjs +411 -0
- package/lab/metrics/builtin.mjs +353 -0
- package/lab/presets.json +62 -0
- package/lab/run-evals.js +1262 -0
- package/lab/server.py +5044 -0
- package/lab/web/dist/assets/dist-DI3ewZYj.js +1 -0
- package/lab/web/dist/assets/gallery-DipkRvqJ.js +3 -0
- package/lab/web/dist/assets/gallery-o7c4lfpn.css +1 -0
- package/lab/web/dist/assets/main-CpVssvrb.js +18 -0
- package/lab/web/dist/assets/main-DjQQums6.css +1 -0
- package/lab/web/dist/assets/tokens-B9intIuT.js +51 -0
- package/lab/web/dist/assets/tokens-s6I-RMVq.css +1 -0
- package/lab/web/dist/gallery.html +18 -0
- package/lab/web/dist/index.html +23 -0
- package/package.json +20 -0
package/LICENSE.md
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
Required Notice: Copyright 2026 Grainbox Limited
|
|
2
|
+
|
|
3
|
+
# PolyForm Internal Use License 1.0.0
|
|
4
|
+
|
|
5
|
+
<https://polyformproject.org/licenses/internal-use/1.0.0>
|
|
6
|
+
|
|
7
|
+
## Acceptance
|
|
8
|
+
|
|
9
|
+
In order to get any license under these terms, you must agree to them as both strict obligations and conditions to all your licenses.
|
|
10
|
+
|
|
11
|
+
## Copyright License
|
|
12
|
+
|
|
13
|
+
The licensor grants you a copyright license for the software to do everything you might do with the software that would otherwise infringe the licensor's copyright in it for any permitted purpose. However, you may only make changes or new works based on the software according to [Changes and New Works License](#changes-and-new-works-license), and you may not distribute the software.
|
|
14
|
+
|
|
15
|
+
## Changes and New Works License
|
|
16
|
+
|
|
17
|
+
The licensor grants you an additional copyright license to make changes and new works based on the software for any permitted purpose.
|
|
18
|
+
|
|
19
|
+
## Patent License
|
|
20
|
+
|
|
21
|
+
The licensor grants you a patent license for the software that covers patent claims the licensor can license, or becomes able to license, that you would infringe by using the software.
|
|
22
|
+
|
|
23
|
+
## Fair Use
|
|
24
|
+
|
|
25
|
+
You may have "fair use" rights for the software under the law. These terms do not limit them.
|
|
26
|
+
|
|
27
|
+
## Internal Business Use
|
|
28
|
+
|
|
29
|
+
Use of the software for the internal business operations of you and your company is use for a permitted purpose.
|
|
30
|
+
|
|
31
|
+
## No Other Rights
|
|
32
|
+
|
|
33
|
+
These terms do not allow you to sublicense or transfer any of your licenses to anyone else, or prevent the licensor from granting licenses to anyone else. These terms do not imply any other licenses.
|
|
34
|
+
|
|
35
|
+
## Patent Defense
|
|
36
|
+
|
|
37
|
+
If you make any written claim that the software infringes or contributes to infringement of any patent, your patent license for the software granted under these terms ends immediately. If your company makes such a claim, your patent license ends immediately for work on behalf of your company.
|
|
38
|
+
|
|
39
|
+
## Violations
|
|
40
|
+
|
|
41
|
+
The first time you are notified in writing that you have violated any of these terms, or done anything with the software not covered by your licenses, your licenses can nonetheless continue if you come into full compliance with these terms, and take practical steps to correct past violations, within 32 days of receiving notice. Otherwise, all your licenses end immediately.
|
|
42
|
+
|
|
43
|
+
## No Liability
|
|
44
|
+
|
|
45
|
+
***As far as the law allows, the software comes as is, without any warranty or condition, and the licensor will not be liable to you for any damages arising out of these terms or the use or nature of the software, under any kind of legal claim.***
|
|
46
|
+
|
|
47
|
+
## Definitions
|
|
48
|
+
|
|
49
|
+
The **licensor** is the individual or entity offering these terms, and the **software** is the software the licensor makes available under these terms.
|
|
50
|
+
|
|
51
|
+
**You** refers to the individual or entity agreeing to these terms.
|
|
52
|
+
|
|
53
|
+
**Your company** is any legal entity, sole proprietorship, or other kind of organization that you work for, plus all organizations that have control over, are under the control of, or are under common control with that organization. **Control** means ownership of substantially all the assets of an entity, or the power to direct its management and policies by vote, contract, or otherwise. Control can be direct or indirect.
|
|
54
|
+
|
|
55
|
+
**Your licenses** are all the licenses granted to you for the software under these terms.
|
|
56
|
+
|
|
57
|
+
**Use** means anything you do with the software requiring one of your licenses.
|
package/README.md
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# evals-lab
|
|
2
|
+
|
|
3
|
+
A browser bench for grading what a model answers: two wordings side by side
|
|
4
|
+
over a set of files, scored against answers fixed in advance.
|
|
5
|
+
|
|
6
|
+
This is an early test release. It runs on your own machine and talks to the
|
|
7
|
+
models you point it at.
|
|
8
|
+
|
|
9
|
+
## Install and run
|
|
10
|
+
|
|
11
|
+
```bash
|
|
12
|
+
npm install -g evals-lab
|
|
13
|
+
evals-lab
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
Then open <http://127.0.0.1:8080>. A new lab opens on two demo datasets, each
|
|
17
|
+
with its images and a starter pipeline. The first start takes a little longer
|
|
18
|
+
than later ones while it installs them.
|
|
19
|
+
|
|
20
|
+
Without installing: `npx evals-lab`.
|
|
21
|
+
|
|
22
|
+
## What it needs
|
|
23
|
+
|
|
24
|
+
| | | Windows | Linux |
|
|
25
|
+
|---|---|---|---|
|
|
26
|
+
| **Node.js** 22.18 or newer | runs the lab's worker | you have it if you have `npm` | same |
|
|
27
|
+
| **Python** 3.10 or newer | runs the lab's server | `winget install Python.Python.3.13` | your distribution's `python3` |
|
|
28
|
+
| **ImageMagick** | sizes images before a model is sent them, and makes thumbnails | `winget install ImageMagick.ImageMagick` | your distribution's `imagemagick` |
|
|
29
|
+
|
|
30
|
+
`evals-lab` says which of these it could not find. It starts without
|
|
31
|
+
ImageMagick, but a run that sends images needs it. Open a new terminal after
|
|
32
|
+
installing one, so it is on your `PATH`.
|
|
33
|
+
|
|
34
|
+
macOS should work the same way and has not been tried.
|
|
35
|
+
|
|
36
|
+
## Where your data is kept
|
|
37
|
+
|
|
38
|
+
Datasets, pipelines, Setup profiles (API keys among them), prompts, uploaded
|
|
39
|
+
files and runs are kept on your machine, in one directory, so the lab opens
|
|
40
|
+
as you left it:
|
|
41
|
+
|
|
42
|
+
| | |
|
|
43
|
+
|---|---|
|
|
44
|
+
| Windows | `%LOCALAPPDATA%\evals-lab` |
|
|
45
|
+
| Linux | `~/.local/share/evals-lab` (or `$XDG_DATA_HOME/evals-lab`) |
|
|
46
|
+
| macOS | `~/Library/Application Support/evals-lab` |
|
|
47
|
+
|
|
48
|
+
Upgrading or reinstalling the package leaves it alone. Delete the directory
|
|
49
|
+
to start again from a new lab.
|
|
50
|
+
|
|
51
|
+
## Options
|
|
52
|
+
|
|
53
|
+
```text
|
|
54
|
+
evals-lab [--port <n>] [--host <address>] [--data-dir <dir>]
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
| Option | Environment | Default |
|
|
58
|
+
|---|---|---|
|
|
59
|
+
| `--port` | `PORT` | `8080` |
|
|
60
|
+
| `--host` | `LAB_HOST` | `127.0.0.1` |
|
|
61
|
+
| `--data-dir` | `DATA_DIR` | the directory above |
|
|
62
|
+
| | `LAB_PASSWORD` | none |
|
|
63
|
+
| | `OLLAMA_URL` | `http://127.0.0.1:11434` |
|
|
64
|
+
| | `PYTHON` | `py -3`, `python` or `python3`, whichever is found |
|
|
65
|
+
|
|
66
|
+
`OLLAMA_URL` is only the connection a new lab starts with. Every other
|
|
67
|
+
connection — an OpenAI-compatible endpoint, Anthropic, llama.cpp — is a Setup
|
|
68
|
+
profile you add in the page.
|
|
69
|
+
|
|
70
|
+
## Reaching it from another machine
|
|
71
|
+
|
|
72
|
+
By default only your own machine can reach the lab. That is deliberate: the
|
|
73
|
+
lab's server forwards requests to whatever address a Setup profile names, so
|
|
74
|
+
anyone who can reach the lab can reach those addresses through it, and the
|
|
75
|
+
lab holds your API keys.
|
|
76
|
+
|
|
77
|
+
`--host 0.0.0.0` makes it reachable from your network, and is refused unless
|
|
78
|
+
`LAB_PASSWORD` is set. Do not put it on the open internet.
|
|
79
|
+
|
|
80
|
+
## Upgrading and removing
|
|
81
|
+
|
|
82
|
+
```bash
|
|
83
|
+
npm install -g evals-lab@latest # upgrade
|
|
84
|
+
npm uninstall -g evals-lab # remove; your data directory stays
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
## Licence
|
|
88
|
+
|
|
89
|
+
Copyright 2026 Grainbox Limited. Licensed under the
|
|
90
|
+
[PolyForm Internal Use License 1.0.0](https://polyformproject.org/licenses/internal-use/1.0.0)
|
|
91
|
+
(`LICENSE.md` in the package): you and your company may use it, and change it,
|
|
92
|
+
for your own internal business. You may not give it, or a changed copy, to
|
|
93
|
+
anyone else, sell it, or offer it as a service.
|
|
94
|
+
|
|
95
|
+
## The demo images
|
|
96
|
+
|
|
97
|
+
Every image in the demo datasets is in the public domain. Each has its line in
|
|
98
|
+
`lab/demo/CREDITS.md` inside the package.
|
package/bin/evals-lab.js
ADDED
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// The npm package's launcher: `evals-lab` starts the lab's server and says
|
|
3
|
+
// where it is.
|
|
4
|
+
//
|
|
5
|
+
// The lab itself is server.py, a stdlib Python server, staged beside this file
|
|
6
|
+
// in ../lab by build.js. What this adds is what a lab run by someone who only
|
|
7
|
+
// installed it needs and the container was given by its deploy:
|
|
8
|
+
//
|
|
9
|
+
// A PLACE TO KEEP THINGS. DATA_DIR defaults to a per-user directory, so a
|
|
10
|
+
// second start on the same machine opens on what the first one left.
|
|
11
|
+
// LOOPBACK. The relay forwards to any address (AGENTS.md § Security), so a
|
|
12
|
+
// lab on a workstation listens on 127.0.0.1 unless it is told otherwise,
|
|
13
|
+
// and is only told otherwise with a password in front of it.
|
|
14
|
+
// A PYTHON. Found by name, since Windows and Linux spell it differently.
|
|
15
|
+
const { spawn, spawnSync } = require("child_process");
|
|
16
|
+
const fs = require("fs");
|
|
17
|
+
const http = require("http");
|
|
18
|
+
const net = require("net");
|
|
19
|
+
const os = require("os");
|
|
20
|
+
const path = require("path");
|
|
21
|
+
|
|
22
|
+
const LAB = path.join(__dirname, "..", "lab");
|
|
23
|
+
// The oldest Python the package's workflow starts the lab on.
|
|
24
|
+
const PYTHON_FLOOR = [3, 10];
|
|
25
|
+
// Ollama's own default, on this machine: the server's is the owner's.
|
|
26
|
+
const LOCAL_OLLAMA = "http://127.0.0.1:11434";
|
|
27
|
+
|
|
28
|
+
const USAGE = `Usage: evals-lab [options]
|
|
29
|
+
|
|
30
|
+
--port <n> the port to listen on. Default: $PORT, else 8080.
|
|
31
|
+
--host <address> the address to listen on. Default: $LAB_HOST, else
|
|
32
|
+
127.0.0.1. Anything but loopback needs LAB_PASSWORD set.
|
|
33
|
+
--data-dir <dir> where the lab keeps its datasets, pipelines, profiles and
|
|
34
|
+
runs. Default: $DATA_DIR, else a per-user directory.
|
|
35
|
+
--version the package's version.
|
|
36
|
+
--help this.
|
|
37
|
+
|
|
38
|
+
Environment: LAB_PASSWORD puts a password in front of the lab, OLLAMA_URL is
|
|
39
|
+
the connection a new lab starts with, and PYTHON names the Python to run.`;
|
|
40
|
+
|
|
41
|
+
// Where a lab keeps its store when nobody said: the platform's per-user data
|
|
42
|
+
// directory, so it survives the package being upgraded or reinstalled.
|
|
43
|
+
function dataDir(env = process.env, platform = process.platform, home = os.homedir()) {
|
|
44
|
+
if (env.DATA_DIR) return env.DATA_DIR;
|
|
45
|
+
if (platform === "win32") {
|
|
46
|
+
return path.win32.join(env.LOCALAPPDATA || path.win32.join(home, "AppData", "Local"), "evals-lab");
|
|
47
|
+
}
|
|
48
|
+
if (platform === "darwin") return path.posix.join(home, "Library", "Application Support", "evals-lab");
|
|
49
|
+
return path.posix.join(env.XDG_DATA_HOME || path.posix.join(home, ".local", "share"), "evals-lab");
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function isLoopback(host) {
|
|
53
|
+
return host === "localhost" || /^127(?:\.\d{1,3}){3}$/.test(host);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// What the command line and the environment ask for, or an Error saying what
|
|
57
|
+
// is wrong with it.
|
|
58
|
+
function options(argv, env = process.env, platform = process.platform, home = os.homedir()) {
|
|
59
|
+
const o = { port: env.PORT || "8080", host: env.LAB_HOST || "127.0.0.1", dataDir: dataDir(env, platform, home) };
|
|
60
|
+
const value = { "--port": "port", "--host": "host", "--data-dir": "dataDir" };
|
|
61
|
+
for (let i = 0; i < argv.length; i++) {
|
|
62
|
+
const a = argv[i];
|
|
63
|
+
if (a === "--help" || a === "-h") o.help = true;
|
|
64
|
+
else if (a === "--version" || a === "-v") o.version = true;
|
|
65
|
+
else if (value[a]) {
|
|
66
|
+
if (argv[i + 1] === undefined) throw new Error(`${a} needs a value`);
|
|
67
|
+
o[value[a]] = argv[++i];
|
|
68
|
+
} else throw new Error(`unknown option ${a}`);
|
|
69
|
+
}
|
|
70
|
+
if (!/^\d+$/.test(o.port) || +o.port < 1 || +o.port > 65535) throw new Error(`${o.port} is not a port`);
|
|
71
|
+
o.port = +o.port;
|
|
72
|
+
if (!isLoopback(o.host) && !env.LAB_PASSWORD) {
|
|
73
|
+
throw new Error(`${o.host} is reachable from other machines, and the lab forwards requests to any address `
|
|
74
|
+
+ "it is given: set LAB_PASSWORD to put a password in front of it");
|
|
75
|
+
}
|
|
76
|
+
return o;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function pythonCandidates(env = process.env, platform = process.platform) {
|
|
80
|
+
const named = env.PYTHON ? [[env.PYTHON]] : [];
|
|
81
|
+
// `py` is the launcher python.org's Windows installer adds; a bare `python`
|
|
82
|
+
// there may be the Store's stub, which answers with no version.
|
|
83
|
+
return [...named, ...(platform === "win32" ? [["py", "-3"], ["python"], ["python3"]] : [["python3"], ["python"]])];
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function atLeast(version, floor) {
|
|
87
|
+
return version[0] > floor[0] || (version[0] === floor[0] && version[1] >= floor[1]);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
// The first Python new enough, and the ones passed over on the way to it.
|
|
91
|
+
function findPython(candidates, probe = pythonVersion) {
|
|
92
|
+
const old = [];
|
|
93
|
+
for (const cmd of candidates) {
|
|
94
|
+
const version = probe(cmd);
|
|
95
|
+
if (!version) continue;
|
|
96
|
+
if (atLeast(version, PYTHON_FLOOR)) return { cmd, version, old };
|
|
97
|
+
old.push(`${cmd.join(" ")} is ${version.join(".")}`);
|
|
98
|
+
}
|
|
99
|
+
return { cmd: null, old };
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
function pythonVersion(cmd) {
|
|
103
|
+
const r = spawnSync(cmd[0], [...cmd.slice(1), "-c", "import sys; print('%d.%d' % sys.version_info[:2])"],
|
|
104
|
+
{ encoding: "utf8", windowsHide: true });
|
|
105
|
+
const m = r.status === 0 && /^(\d+)\.(\d+)\s*$/.exec(r.stdout || "");
|
|
106
|
+
return m ? [+m[1], +m[2]] : null;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function hasImageMagick(platform = process.platform) {
|
|
110
|
+
// As server.py looks for it: `convert` is another program on Windows.
|
|
111
|
+
return (platform === "win32" ? ["magick"] : ["magick", "convert"])
|
|
112
|
+
.some(bin => spawnSync(bin, ["-version"], { stdio: "ignore", windowsHide: true }).status === 0);
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function portFree(host, port) {
|
|
116
|
+
return new Promise(resolve => {
|
|
117
|
+
const probe = net.createServer();
|
|
118
|
+
probe.once("error", e => resolve(e.code !== "EADDRINUSE"));
|
|
119
|
+
probe.once("listening", () => probe.close(() => resolve(true)));
|
|
120
|
+
probe.listen(port, host);
|
|
121
|
+
});
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function answers(url) {
|
|
125
|
+
return new Promise(resolve => {
|
|
126
|
+
const req = http.get(url, res => { res.resume(); resolve(res.statusCode === 200); });
|
|
127
|
+
req.on("error", () => resolve(false));
|
|
128
|
+
req.setTimeout(2000, () => { req.destroy(); resolve(false); });
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function fail(message) {
|
|
133
|
+
console.error(`evals-lab: ${message}`);
|
|
134
|
+
process.exit(1);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
async function main() {
|
|
138
|
+
let o;
|
|
139
|
+
try { o = options(process.argv.slice(2)); } catch (e) { fail(`${e.message}\n\n${USAGE}`); }
|
|
140
|
+
if (o.help) return console.log(USAGE);
|
|
141
|
+
if (o.version) return console.log(require("../package.json").version);
|
|
142
|
+
if (!fs.existsSync(path.join(LAB, "server.py"))) {
|
|
143
|
+
fail(`no ${path.join(LAB, "server.py")} -- in a checkout the package is staged by \`node npm/build.js\``);
|
|
144
|
+
}
|
|
145
|
+
const python = findPython(pythonCandidates());
|
|
146
|
+
if (!python.cmd) {
|
|
147
|
+
const floor = PYTHON_FLOOR.join(".");
|
|
148
|
+
fail(`the lab's server needs Python ${floor} or newer, and none was found`
|
|
149
|
+
+ (python.old.length ? ` (${python.old.join(", ")})` : "")
|
|
150
|
+
+ `.\n Windows: winget install Python.Python.3.13\n Linux: your distribution's python3 package\n`
|
|
151
|
+
+ " Or name one with the PYTHON environment variable.");
|
|
152
|
+
}
|
|
153
|
+
if (!(await portFree(o.host, o.port))) {
|
|
154
|
+
fail(`port ${o.port} is in use -- name another with --port`);
|
|
155
|
+
}
|
|
156
|
+
fs.mkdirSync(o.dataDir, { recursive: true });
|
|
157
|
+
if (!hasImageMagick()) {
|
|
158
|
+
console.error("evals-lab: no ImageMagick (`magick`) on the PATH, so images cannot be sent to a model "
|
|
159
|
+
+ "and Sources show no thumbnails.\n"
|
|
160
|
+
+ " Windows: winget install ImageMagick.ImageMagick\n Linux: your distribution's imagemagick package");
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const env = { ...process.env, PORT: String(o.port), LAB_HOST: o.host, DATA_DIR: o.dataDir,
|
|
164
|
+
OLLAMA_URL: process.env.OLLAMA_URL || LOCAL_OLLAMA };
|
|
165
|
+
// The node that ran this is the node the server's worker runs on, whatever
|
|
166
|
+
// else the PATH holds. Windows spells the variable `Path`.
|
|
167
|
+
const pathKey = Object.keys(env).find(k => k.toUpperCase() === "PATH") || "PATH";
|
|
168
|
+
env[pathKey] = path.dirname(process.execPath) + path.delimiter + (env[pathKey] || "");
|
|
169
|
+
// -X utf8: the server reads and writes its files as UTF-8 by default,
|
|
170
|
+
// which Windows' Python otherwise does in the machine's code page.
|
|
171
|
+
const child = spawn(python.cmd[0], [...python.cmd.slice(1), "-X", "utf8", "-u", path.join(LAB, "server.py")],
|
|
172
|
+
{ stdio: "inherit", env });
|
|
173
|
+
let stopping = false;
|
|
174
|
+
child.on("error", e => fail(`${python.cmd.join(" ")} did not start: ${e.message}`));
|
|
175
|
+
child.on("exit", code => process.exit(stopping ? 0 : code ?? 1));
|
|
176
|
+
// Ctrl+C reaches the server too, from the terminal, and it stops itself;
|
|
177
|
+
// it is sent on only to a server that was not told, since a second one
|
|
178
|
+
// lands in the middle of its stopping.
|
|
179
|
+
process.on("SIGINT", () => { stopping = true; setTimeout(() => child.kill("SIGINT"), 1000).unref(); });
|
|
180
|
+
process.on("SIGTERM", () => { stopping = true; child.kill("SIGTERM"); });
|
|
181
|
+
|
|
182
|
+
const address = `http://${o.host}:${o.port}`;
|
|
183
|
+
// A first start installs the demo pack before it listens, which takes longer
|
|
184
|
+
// than any later one.
|
|
185
|
+
for (let i = 0; i < 1200; i++) {
|
|
186
|
+
if (await answers(`${address}/healthz`)) {
|
|
187
|
+
console.log(`\nevals-lab is running at ${address}\n data: ${o.dataDir}\n stop: Ctrl+C`);
|
|
188
|
+
return;
|
|
189
|
+
}
|
|
190
|
+
await new Promise(r => setTimeout(r, 250));
|
|
191
|
+
}
|
|
192
|
+
console.error(`evals-lab: the server has not answered at ${address} after five minutes`);
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
if (require.main === module) main();
|
|
196
|
+
module.exports = { dataDir, isLoopback, options, pythonCandidates, findPython, atLeast, PYTHON_FLOOR };
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# Credits
|
|
2
|
+
|
|
3
|
+
Every file in `images/`, where it came from and under what terms. All
|
|
4
|
+
need no attribution; the sources are listed so each one can be traced.
|
|
5
|
+
Files were resized to about 3 megapixels and their metadata stripped;
|
|
6
|
+
the panorama is a crop, and the PNG and TIFF are conversions.
|
|
7
|
+
|
|
8
|
+
| File | Source | Author | Licence |
|
|
9
|
+
|---|---|---|---|
|
|
10
|
+
| `aerial-city.jpg` | https://unsplash.com/photos/wVmdwjvF2JM | Georgi Kalaydzhiev | Unsplash License |
|
|
11
|
+
| `alligator-pen.jpg` | Public Domain Image Archive | | Public domain |
|
|
12
|
+
| `arm-tattoo.jpg` | https://unsplash.com/photos/ATtkp9zRToI | Christopher Stites | Unsplash License |
|
|
13
|
+
| `basketball-hoop.jpg` | https://unsplash.com/photos/GXG4p81JPBk | Jason Zeis | Unsplash License |
|
|
14
|
+
| `bed-and-plant.jpg` | https://unsplash.com/photos/RIkwCyeweSA | Bennie Bates | Unsplash License |
|
|
15
|
+
| `blue-boardwalk.jpg` | https://www.flickr.com/photo.gne?id=1653048445 | | Public domain |
|
|
16
|
+
| `bright-bedroom.jpg` | https://unsplash.com/photos/JjqzgWk36NM | Clay Banks | Unsplash License |
|
|
17
|
+
| `busy-beach.jpg` | https://unsplash.com/photos/3Gz52Og8Vtw | Ben Elliott | Unsplash License |
|
|
18
|
+
| `butcher-sign.jpg` | https://www.flickr.com/photo.gne?id=46338644535 | | Public domain |
|
|
19
|
+
| `cactus-flower.jpg` | https://unsplash.com/photos/9a8GgZPWnqU | Arnaud Padalle | Unsplash License |
|
|
20
|
+
| `car-headlight.jpg` | https://unsplash.com/photos/pbmCuwa4vvQ | Linus Belanger | Unsplash License |
|
|
21
|
+
| `child-in-surf.jpg` | https://unsplash.com/photos/Bv0FAF9kJZU | Adam Kring | Unsplash License |
|
|
22
|
+
| `city-highway.jpg` | https://unsplash.com/photos/fNfCjzJzSHA | Katelyn Warner | Unsplash License |
|
|
23
|
+
| `corner-bakery.jpg` | https://unsplash.com/photos/qqL1y15xQ9w | Ruben Mavarez | Unsplash License |
|
|
24
|
+
| `corner-shop.jpg` | https://unsplash.com/photos/tnL9CSDgiJE | Danny Greenberg | Unsplash License |
|
|
25
|
+
| `crosswalk-cyclist-panorama.jpg` | https://unsplash.com/photos/GQyn7iDAjNM | Ryoji Iwata | Unsplash License |
|
|
26
|
+
| `cyanotype-room.jpg` | Public Domain Image Archive | | Public domain |
|
|
27
|
+
| `cyclist-yellow-jacket.jpg` | https://unsplash.com/photos/nlY-pr6Fr1w | Alexander Lunyov | Unsplash License |
|
|
28
|
+
| `dead-end-sign.jpg` | https://www.flickr.com/photo.gne?id=29782083985 | | Public domain |
|
|
29
|
+
| `dog-in-snow.jpg` | https://www.flickr.com/photo.gne?id=46945240391 | | Public domain |
|
|
30
|
+
| `empty-bedroom.jpg` | https://unsplash.com/photos/7yFSdK-UwGE | Taylor Friehl | Unsplash License |
|
|
31
|
+
| `excavator-street-signs.jpg` | https://www.flickr.com/photo.gne?id=45952765524 | | Public domain |
|
|
32
|
+
| `farmers-market-stall.jpg` | https://www.flickr.com/photo.gne?id=47942372931 | | Public domain |
|
|
33
|
+
| `four-jets.jpg` | https://unsplash.com/photos/kU5Hnqpr4eA | Joel Ambass | Unsplash License |
|
|
34
|
+
| `german-shepherd.jpg` | https://unsplash.com/photos/tIdH8sFgKrs | Grace Anne Bobadilla | Unsplash License |
|
|
35
|
+
| `globe-closeup.jpg` | https://unsplash.com/photos/dIZBL-VZJlQ | Grace Anne Bobadilla | Unsplash License |
|
|
36
|
+
| `harbour-bridge-dusk.jpg` | https://unsplash.com/photos/efDfUFITwvw | Jeremy Kwok | Unsplash License |
|
|
37
|
+
| `harbour-village.jpg` | https://unsplash.com/photos/25Bu6uO9xLw | Danny Rienecker | Unsplash License |
|
|
38
|
+
| `harrow-on-the-hill-sign.jpg` | https://www.flickr.com/photo.gne?id=3176414431 | | Public domain |
|
|
39
|
+
| `high-street-walkers.jpg` | https://unsplash.com/photos/Qfw7Ig02W14 | T | Unsplash License |
|
|
40
|
+
| `hillside-rooftops.jpg` | https://www.flickr.com/photo.gne?id=26666962857 | | Public domain |
|
|
41
|
+
| `hong-kong-street.jpg` | https://unsplash.com/photos/CELpBy3F3cQ | Quan Jing | Unsplash License |
|
|
42
|
+
| `house-at-night.jpg` | https://www.flickr.com/photo.gne?id=55534582361 | | Public domain |
|
|
43
|
+
| `house-salisbury-street.jpg` | https://www.flickr.com/photo.gne?id=15726668073 | | Public domain |
|
|
44
|
+
| `infrared-orchard.jpg` | https://www.flickr.com/photo.gne?id=48490080332 | | Public domain |
|
|
45
|
+
| `ivy-leaves.jpg` | https://unsplash.com/photos/U6_j6Yz2OPo | Pawel Czerwinski | Unsplash License |
|
|
46
|
+
| `keyboard-desk.jpg` | https://unsplash.com/photos/rW2KRSx2vJs | Jeremy Rorimpandey | Unsplash License |
|
|
47
|
+
| `lighthouse-dunes.jpg` | https://unsplash.com/photos/TpzL7Fdw5zs | Max Bohme | Unsplash License |
|
|
48
|
+
| `man-with-alligator.jpg` | Public Domain Image Archive | | Public domain |
|
|
49
|
+
| `mountain-lake.jpg` | https://unsplash.com/photos/aw6kFpvhxys | Rafael Peier | Unsplash License |
|
|
50
|
+
| `museum-skeleton.jpg` | Public Domain Image Archive | | Public domain |
|
|
51
|
+
| `music-room.jpg` | https://unsplash.com/photos/ChXKPHornJQ | Barney Goodman | Unsplash License |
|
|
52
|
+
| `old-red-car.jpg` | https://unsplash.com/photos/BaaxuvB-Dcc | Jake Johnson | Unsplash License |
|
|
53
|
+
| `orange-mailbox-hedge.jpg` | https://www.flickr.com/photo.gne?id=24372563229 | | Public domain |
|
|
54
|
+
| `parked-car-plate.jpg` | https://www.flickr.com/photo.gne?id=52171146783 | | Public domain |
|
|
55
|
+
| `pedal-boats.jpg` | https://unsplash.com/photos/pXYgH0rtKIM | Aldo Vukaj | Unsplash License |
|
|
56
|
+
| `phone-and-wallet.png` | https://unsplash.com/photos/8L3uJzSOyOY | Adrian Regeci | Unsplash License |
|
|
57
|
+
| `phone-box.jpg` | https://unsplash.com/photos/AIahhQvov88 | Dan Begel | Unsplash License |
|
|
58
|
+
| `pink-flamingo.jpg` | Public Domain Image Archive | | Public domain |
|
|
59
|
+
| `pink-trees-vignette.jpg` | https://www.flickr.com/photo.gne?id=54285731451 | | Public domain |
|
|
60
|
+
| `please-leave-quietly-sign.jpg` | https://unsplash.com/photos/27yACkIPN8Y | T | Unsplash License |
|
|
61
|
+
| `postcards.jpg` | https://unsplash.com/photos/dZZxUjWUx_o | Fiona Murray Degraaff | Unsplash License |
|
|
62
|
+
| `railroad-crossing-sign.jpg` | https://www.flickr.com/photo.gne?id=14938289668 | | Public domain |
|
|
63
|
+
| `red-roof-church.jpg` | https://unsplash.com/photos/DwkgQvGF6Ys | Jean Carlo Emer | Unsplash License |
|
|
64
|
+
| `roadside-mailboxes.jpg` | https://www.flickr.com/photo.gne?id=52161450764 | | Public domain |
|
|
65
|
+
| `roadworks-sign.jpg` | https://www.flickr.com/photo.gne?id=42075543145 | | Public domain |
|
|
66
|
+
| `rodeo.jpg` | https://unsplash.com/photos/qrizVWecf6M | Tim Mossholder | Unsplash License |
|
|
67
|
+
| `roundabout-sign.jpg` | https://www.flickr.com/photo.gne?id=3100530868 | | Public domain |
|
|
68
|
+
| `running-tap.tif` | https://unsplash.com/photos/lQ6U_GSziPQ | Clay Banks | Unsplash License |
|
|
69
|
+
| `sea-cave.jpg` | https://unsplash.com/photos/SL5zdSLugHs | Evgeni Tcherkasski | Unsplash License |
|
|
70
|
+
| `shadow-on-sand.jpg` | https://unsplash.com/photos/BbgjCY5Mt54 | Polina Kuzovkova | Unsplash License |
|
|
71
|
+
| `signpost-a404.jpg` | https://www.flickr.com/photo.gne?id=3100256191 | | Public domain |
|
|
72
|
+
| `snail-on-stem.jpg` | https://www.flickr.com/photo.gne?id=55534667446 | | Public domain |
|
|
73
|
+
| `street-fruit-cart.jpg` | https://unsplash.com/photos/ccOH2l8f3K0 | Celine Cao | Unsplash License |
|
|
74
|
+
| `street-sign-parnassusweg.jpg` | https://www.flickr.com/photo.gne?id=26070508123 | | Public domain |
|
|
75
|
+
| `two-horses-close.jpg` | https://unsplash.com/photos/E7HZMn2Gb5o | Alexander Mass | Unsplash License |
|
|
76
|
+
| `two-horses-field.jpg` | https://unsplash.com/photos/Y4VMzlU2OqQ | Alexander Mass | Unsplash License |
|
|
77
|
+
| `university-sign.jpg` | https://www.flickr.com/photo.gne?id=30173257360 | | Public domain |
|
|
78
|
+
| `vegetable-crates.jpg` | https://www.flickr.com/photo.gne?id=49727488612 | | Public domain |
|
|
79
|
+
| `victorian-house.jpg` | https://www.flickr.com/photo.gne?id=39718690633 | | Public domain |
|
|
80
|
+
| `vintage-dashboard.jpg` | https://unsplash.com/photos/PcqCSsaRZak | Linus Belanger | Unsplash License |
|
|
81
|
+
| `volcano-at-dusk.jpg` | https://unsplash.com/photos/zHRlfpTGXx8 | Jayesh Patel | Unsplash License |
|
|
82
|
+
| `wall-camera.jpg` | https://unsplash.com/photos/L8sCDG5MO5Y | Cai Fang | Unsplash License |
|
|
83
|
+
| `watch-for-rocks-sign.jpg` | https://www.flickr.com/photo.gne?id=34296692233 | | Public domain |
|
|
84
|
+
| `waterfall.jpg` | https://unsplash.com/photos/Go6GUEEbHgI | Rafael Peier | Unsplash License |
|
|
85
|
+
| `watermarked-car.jpg` | https://www.flickr.com/photo.gne?id=35773714735 | | Public domain |
|
|
86
|
+
| `watermarked-pills.jpg` | https://www.flickr.com/photo.gne?id=51126868965 | | Public domain |
|
|
87
|
+
| `white-domes.jpg` | https://unsplash.com/photos/YNiXuanD2iY | Aladin Habboubi | Unsplash License |
|
|
88
|
+
| `whiteboard-delegate.jpg` | https://www.flickr.com/photo.gne?id=6061228692 | | Public domain |
|
|
89
|
+
| `whiteboard-germ-layers.jpg` | https://www.flickr.com/photo.gne?id=16321196867 | | Public domain |
|
|
90
|
+
| `whiteboard-meeting-notes.jpg` | https://www.flickr.com/photo.gne?id=186873704 | | Public domain |
|
|
91
|
+
| `whiteboard-messages.jpg` | https://www.flickr.com/photo.gne?id=23162308061 | | Public domain |
|
|
92
|
+
| `wooden-house-fence.jpg` | https://www.flickr.com/photo.gne?id=26595748497 | | Public domain |
|