@wendellresearch-ai/benchwarmers 0.3.0 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +11 -8
- package/GATEWAY.md +9 -1
- package/PRIVACY.md +7 -5
- package/README.md +122 -51
- package/SECURITY.md +5 -4
- package/USAGE.md +19 -2
- package/VERSION +1 -1
- package/gateway.py +111 -2
- package/mcp_server.py +133 -36
- package/metrics/client.py +9 -8
- package/metrics/guard.py +8 -1
- package/package.json +1 -1
- package/scripts/test_mcp.py +207 -2
- package/scripts/test_serve_edit.py +2 -2
- package/scripts/test_verdict.py +18 -1
package/scripts/test_verdict.py
CHANGED
|
@@ -96,7 +96,24 @@ def bench_field() -> None:
|
|
|
96
96
|
srv.shutdown()
|
|
97
97
|
|
|
98
98
|
|
|
99
|
+
def records() -> None:
|
|
100
|
+
"""GET /bench/record's counts: one collection's judged seats, a task's by its notes, never an unjudged one."""
|
|
101
|
+
for call, model, coll, useful, note in [("r1.m1", "m1", "bench/vision", True, "thicken slab faces: agreed"),
|
|
102
|
+
("r1.m2", "m2", "bench/vision", False, "thicken slab faces: no answer"),
|
|
103
|
+
("r2.m1", "m1", "bench/vision", True, "thicken what it is: agreed"),
|
|
104
|
+
("r3.m1", "m1", "bench/core", False, "a review")]:
|
|
105
|
+
offload.add(model, f"look at this ({coll}, vision)", None, "member; ok", call)
|
|
106
|
+
offload.verdict(call, useful, note)
|
|
107
|
+
offload.add("m3", "look at this (bench/vision, vision)", None, "member; ok", "r4.m3") # never judged
|
|
108
|
+
assert gateway.record("bench/vision", "thicken slab faces") == {
|
|
109
|
+
"m1": {"useful": 1, "notUseful": 0, "judged": 1, "lastAt": offload.entries()[-1]["date"]},
|
|
110
|
+
"m2": {"useful": 0, "notUseful": 1, "judged": 1, "lastAt": offload.entries()[-1]["date"]}}
|
|
111
|
+
assert gateway.record("bench/vision")["m1"]["judged"] == 2 and "m3" not in gateway.record("bench/vision")
|
|
112
|
+
assert gateway.record("bench/core")["m1"]["notUseful"] == 1 and gateway.record("bench/none") == {}
|
|
113
|
+
|
|
114
|
+
|
|
99
115
|
if __name__ == "__main__":
|
|
100
116
|
verdicts()
|
|
101
117
|
bench_field()
|
|
102
|
-
|
|
118
|
+
records()
|
|
119
|
+
print("verdict, bench-field and record checks pass")
|