humanish 0.65.0 → 0.66.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -1
- package/dist/actor-contract.d.ts +2 -1
- package/dist/actor-contract.js.map +1 -1
- package/dist/cua-actor-lab.d.ts +23 -0
- package/dist/cua-actor-lab.js +59 -13
- package/dist/cua-actor-lab.js.map +1 -1
- package/dist/e2b-desktop-launch.d.ts +11 -0
- package/dist/e2b-desktop-launch.js.map +1 -1
- package/dist/e2b-terminal-lab.js +15 -0
- package/dist/e2b-terminal-lab.js.map +1 -1
- package/dist/lab-config.d.ts +17 -0
- package/dist/lab-config.js +14 -0
- package/dist/lab-config.js.map +1 -1
- package/dist/program.d.ts +3 -0
- package/dist/program.js +44 -6
- package/dist/program.js.map +1 -1
- package/dist/telemetry.d.ts +14 -0
- package/dist/telemetry.js +104 -1
- package/dist/telemetry.js.map +1 -1
- package/docs/contracts/schemas.md +1 -1
- package/docs/goals/current.md +1 -1
- package/docs/ramp/README.md +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -56,7 +56,11 @@ npx humanish run try-live # a REAL study on a hosted desktop, capped at $2
|
|
|
56
56
|
`try-live` studies a real public app so that your first live run works without
|
|
57
57
|
configuring anything; point its `subject` at your own app once you have seen it
|
|
58
58
|
run. It needs `E2B_API_KEY`, and a model: either a provider key or a coding
|
|
59
|
-
agent you are already signed in to (see below).
|
|
59
|
+
agent you are already signed in to (see below). Measured on 2026-09-01 from
|
|
60
|
+
three fresh directories against the published 0.65.0: 3 of 3 reached the goal,
|
|
61
|
+
108 to 111 s each, about $0.16 each, and all three reported the same two
|
|
62
|
+
frictions in the demo app
|
|
63
|
+
([receipt](docs/goals/computer-use-actor/receipts/cold-install-try-live-2026-09-01.md)).
|
|
60
64
|
|
|
61
65
|
**Install it, do not one-shot it.** A live run needs the optional peer
|
|
62
66
|
`@e2b/desktop`, and Node resolves that relative to humanish itself, so a
|
package/dist/actor-contract.d.ts
CHANGED
|
@@ -14,7 +14,8 @@ export declare const ACTOR_TRACE_SCHEMA = "humanish.actor-trace.v1";
|
|
|
14
14
|
* - `passed` the participant reached the goal
|
|
15
15
|
* - `abandoned` the participant stopped trying. A FINDING, not a malfunction
|
|
16
16
|
* - `incomplete` the session ended (time or budget) before the goal was reached
|
|
17
|
-
* - `blocked` the participant
|
|
17
|
+
* - `blocked` the participant could not proceed: an approval the run could not give, or a
|
|
18
|
+
* blocker they described in their own final words (#476)
|
|
18
19
|
* - `timed_out` the session hit its deadline with no productive activity at all
|
|
19
20
|
* - `failed` the HARNESS failed: a dead sandbox, a provider error, a broken artifact
|
|
20
21
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"actor-contract.js","sourceRoot":"","sources":["../src/actor-contract.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAE5C,oEAAoE;AACpE,0EAA0E;AAC1E,oFAAoF;AACpF,EAAE;AACF,iFAAiF;AACjF,8EAA8E;AAC9E,gFAAgF;AAChF,uBAAuB;AAEvB,MAAM,CAAC,MAAM,kBAAkB,GAAG,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"actor-contract.js","sourceRoot":"","sources":["../src/actor-contract.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAE5C,oEAAoE;AACpE,0EAA0E;AAC1E,oFAAoF;AACpF,EAAE;AACF,iFAAiF;AACjF,8EAA8E;AAC9E,gFAAgF;AAChF,uBAAuB;AAEvB,MAAM,CAAC,MAAM,kBAAkB,GAAG,yBAAyB,CAAC;AAoB5D;;iCAEiC;AACjC,MAAM,CAAC,MAAM,4BAA4B,GAA2B,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;AAyLhG,MAAM,CAAC,MAAM,6BAA6B,GAAsB;IAC9D,QAAQ,EAAE,IAAI;IACd,eAAe,EAAE,IAAI;IACrB,KAAK,EAAE,CAAC,MAAM,CAAC;IACf,mBAAmB,EAAE,KAAK;IAC1B,QAAQ,EAAE,KAAK;IACf,qBAAqB,EAAE,IAAI;IAC3B,cAAc,EAAE,KAAK;IACrB,OAAO,EAAE,MAAM;CAChB,CAAC;AAEF,kFAAkF;AAClF,gFAAgF;AAChF,yEAAyE;AACzE,MAAM,CAAC,MAAM,0BAA0B,GAAsB;IAC3D,QAAQ,EAAE,IAAI;IACd,eAAe,EAAE,IAAI;IACrB,KAAK,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC;IACtB,mBAAmB,EAAE,KAAK;IAC1B,QAAQ,EAAE,IAAI;IACd,qBAAqB,EAAE,KAAK;IAC5B,cAAc,EAAE,IAAI;IACpB,OAAO,EAAE,MAAM;CAChB,CAAC;AAEF,+EAA+E;AAC/E,gFAAgF;AAChF,uEAAuE;AACvE,MAAM,CAAC,MAAM,6BAA6B,GAAsB;IAC9D,QAAQ,EAAE,IAAI;IACd,eAAe,EAAE,IAAI;IACrB,KAAK,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC;IACtB,mBAAmB,EAAE,KAAK;IAC1B,QAAQ,EAAE,KAAK;IACf,qBAAqB,EAAE,IAAI;IAC3B,cAAc,EAAE,IAAI;IACpB,OAAO,EAAE,MAAM;CAChB,CAAC;AAEF,yFAAyF;AACzF,6FAA6F;AAC7F,8FAA8F;AAC9F,MAAM,CAAC,MAAM,6BAA6B,GAAsB;IAC9D,QAAQ,EAAE,IAAI;IACd,eAAe,EAAE,IAAI;IACrB,KAAK,EAAE,CAAC,kBAAkB,CAAC;IAC3B,mBAAmB,EAAE,IAAI;IACzB,QAAQ,EAAE,KAAK;IACf,qBAAqB,EAAE,KAAK;IAC5B,cAAc,EAAE,KAAK;IACrB,OAAO,EAAE,MAAM,CAAC,8EAA8E;CAC/F,CAAC;AAEF,+FAA+F;AAC/F,iGAAiG;AACjG,+FAA+F;AAC/F,8FAA8F;AAC9F,kGAAkG;AAClG,6FAA6F;AAC7F,4FAA4F;AAC5F,6EAA6E;AAC7E,MAAM,CAAC,MAAM,2BAA2B,GAAsB;IAC5D,QAAQ,EAAE,IAAI;IACd,eAAe,EAAE,IAAI;IACrB,KAAK,EAAE,CAAC,UAAU,CAAC;IACnB,mBAAmB,EAAE,KAAK;IAC1B,QAAQ,EAAE,KAAK;IACf,qBAAqB,EAAE,KAAK;IAC5B,cAAc,EAAE,KAAK;IACrB,OAAO,EAAE,MAAM,EAAE,gFAAgF;IACjG,YAAY,EAAE,2BAA2B;CAC1C,CAAC;AAEF,6EAA6E;AAC7E,gFAAgF;AAChF,gFAAgF;AAChF,gFAAgF;AAChF,uEAAuE;AACvE,MAAM,UAAU,6BAA6B,CAAC,MAA4B;IACxE,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,QAAQ;YACX,OAAO,gBAAgB,CAAC;QAC1B,KAAK,WAAW;YACd,OAAO,WAAW,CAAC;QACrB,KAAK,SAAS;YACZ,OAAO,kBAAkB,CAAC;QAC5B,KAAK,QAAQ;YACX,OAAO,aAAa,CAAC;IACzB,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,kBAAkB;YACrB,OAAO,SAAS,CAAC;QACnB,KAAK,YAAY;YACf,OAAO,aAAa,CAAC;QACvB,KAAK,aAAa,CAAC;QACnB,KAAK,iBAAiB;YACpB,OAAO,WAAW,CAAC;QACrB,KAAK,cAAc;YACjB,OAAO,SAAS,CAAC;QACnB,KAAK,WAAW;YACd,OAAO,WAAW,CAAC;QACrB,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;YACnC,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAAE,OAAO,SAAS,CAAC;YAClD,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;gBAAE,OAAO,WAAW,CAAC;YACnD,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAAE,OAAO,SAAS,CAAC;YAClD,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,OAAO,aAAa,CAAC;YACnD,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,OAAO,WAAW,CAAC;YACjD,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,OAAO,MAAM,CAAC;YAC5C,OAAO,QAAQ,CAAC;QAClB,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,cAAc,CAAC,GAAsC;IAC5D,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QACjE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,MAAM,GAAG,GAA8B,CAAC;IAC9C,MAAM,UAAU,GAAG,CAAC,IAAc,EAAsB,EAAE;QACxD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;YAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;gBACxD,OAAO,KAAK,CAAC;YACf,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,eAAe,CAAC,CAAC,CAAC;IACpF,MAAM,MAAM,GAAG,UAAU,CAAC,CAAC,QAAQ,EAAE,eAAe,EAAE,cAAc,EAAE,mBAAmB,CAAC,CAAC,CAAC;IAC5F,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,OAAO,EAAE,cAAc,EAAE,aAAa,CAAC,CAAC,CAAC;IACnE,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,SAAS,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC;IAC5D,MAAM,KAAK,GAAoB;QAC7B,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;QACzC,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;QAC3C,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;QACzC,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;KAC9C,CAAC;IACF,OAAO,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;AAC3D,CAAC;AAED,6EAA6E;AAC7E,gFAAgF;AAChF,+EAA+E;AAC/E,mCAAmC;AACnC,SAAS,sBAAsB,CAAC,KAA0B;IACxD,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1F,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3E,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1F,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;IACvF,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;IAE7F,MAAM,KAAK,GAAqB,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACvD,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,IAAI,GAAmB;YAC3B,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,IAAI;YACJ,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;SAC9D,CAAC;QACF,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC3C,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO;oBACL,GAAG,IAAI;oBACP,OAAO,EAAE;wBACP,GAAG,CAAC,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,CAAC;wBACnE,GAAG,CAAC,OAAO,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;wBAC1D,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;wBACzE,GAAG,CAAC,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC;qBAChF;iBACF,CAAC;YACJ,CAAC;QACH,CAAC;QACD,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YACzB,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACrC,IAAI,IAAI,EAAE,CAAC;gBACT,OAAO;oBACL,GAAG,IAAI;oBACP,IAAI,EAAE;wBACJ,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC;wBAC7D,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;qBACxD;iBACF,CAAC;YACJ,CAAC;QACH,CAAC;QACD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,MAAM,OAAO,GAAG,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC3C,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;YACzC,CAAC;QACH,CAAC;QACD,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YACzB,MAAM,SAAS,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC/C,IAAI,SAAS,EAAE,CAAC;gBACd,OAAO,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC;YAC3C,CAAC;QACH,CAAC;QACD,IAAI,IAAI,KAAK,aAAa,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC7C,IAAI,MAAM,EAAE,UAAU,KAAK,SAAS,EAAE,CAAC;gBACrC,OAAO,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,UAAU,EAAE,CAAC;YAC9C,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;QACvC,KAAK,CAAC,IAAI,CAAC;YACT,EAAE,EAAE,YAAY,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE;YACrC,IAAI,EAAE,UAAU;YAChB,SAAS,EAAE,WAAW;YACtB,MAAM,EAAE,QAAQ,CAAC,QAAQ;YACzB,KAAK,EAAE,GAAG,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,QAAQ,GAAG;YAClD,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACtD,CAAC,CAAC;IACL,CAAC;IACD,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAClC,KAAK,CAAC,IAAI,CAAC;YACT,EAAE,EAAE,QAAQ,KAAK,GAAG,CAAC,EAAE;YACvB,IAAI,EAAE,MAAM;YACZ,SAAS,EAAE,WAAW;YACtB,KAAK,EAAE,IAAI,CAAC,WAAW,IAAI,aAAa;YACxC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClE,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,CAAC,GAAG,KAAK,CAAC,QAAQ,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;QAC7D,KAAK,CAAC,IAAI,CAAC;YACT,EAAE,EAAE,UAAU,KAAK,GAAG,CAAC,EAAE;YACzB,IAAI,EAAE,QAAQ;YACd,SAAS,EAAE,WAAW;YACtB,KAAK,EAAE,MAAM,CAAC,MAAM;YACpB,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpD,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,uBAAuB,CAAC,MAA+B,EAAE,OAAwB;IAC/F,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAC3B,MAAM,UAAU,GAAG,cAAc,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACpD,OAAO;QACL,MAAM,EAAE,kBAAkB;QAC1B,QAAQ,EAAE,kBAAkB;QAC5B,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,KAAK,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;QACxG,QAAQ,EAAE,UAAU;QACpB,IAAI,EAAE,MAAM;QACZ,OAAO;QACP,SAAS,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE;QACjF,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,gBAAgB,EAAE,6BAA6B,CAAC,KAAK,CAAC,MAAM,CAAC;QAC7D,8EAA8E;QAC9E,8EAA8E;QAC9E,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC;QACjC,GAAG,EAAE;YACH,GAAG,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,SAAS,EAAE,CAAC;YAC1E,GAAG,CAAC,MAAM,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;YACvE,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC;YACjE,GAAG,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;SAC/D;QACD,MAAM,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,EAAE;QAC3B,KAAK,EAAE,sBAAsB,CAAC,KAAK,CAAC;QACpC,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;QACnD,YAAY,EAAE,6BAA6B;KAC5C,CAAC;AACJ,CAAC"}
|
package/dist/cua-actor-lab.d.ts
CHANGED
|
@@ -918,6 +918,18 @@ export declare function buildCuaBundle(args: {
|
|
|
918
918
|
captureRedaction?: "raw" | "blurred";
|
|
919
919
|
session?: CuaLoopResult;
|
|
920
920
|
sessionError?: string;
|
|
921
|
+
/**
|
|
922
|
+
* The lane's own credibility read of a goal_satisfied session (#476). The actor's status is
|
|
923
|
+
* evidence of what it CLAIMED; whether the harness counts the claim is decided by the lane
|
|
924
|
+
* (zero engagement, a final message that describes a blocker). The review has to say the same
|
|
925
|
+
* thing the lane's exit code says, or the durable bundle reports a participant reaching the
|
|
926
|
+
* goal on a run the harness refused to count.
|
|
927
|
+
*/
|
|
928
|
+
credibility?: {
|
|
929
|
+
noEngagement: boolean;
|
|
930
|
+
selfReportedBlocker: boolean;
|
|
931
|
+
reportedFriction: boolean;
|
|
932
|
+
};
|
|
921
933
|
source: RunBundle["source"];
|
|
922
934
|
/** Provisioned-route provenance (clone or local-tree): what the actor actually drove (names
|
|
923
935
|
* + digests only, never values or command text), including the subject's state story. */
|
|
@@ -972,3 +984,14 @@ export declare function buildCuaFanoutBundle(args: {
|
|
|
972
984
|
subjectEnvNames: string[];
|
|
973
985
|
inProgress?: boolean;
|
|
974
986
|
}): RunBundle;
|
|
987
|
+
/**
|
|
988
|
+
* The status a participant is TALLIED under, given what the lane made of the session. A
|
|
989
|
+
* goal_satisfied claim with zero engagement is a session that ran out before anything happened;
|
|
990
|
+
* one whose final message describes a blocker is a participant who could not proceed and said so.
|
|
991
|
+
* Both keep their trace status (the claim is evidence); neither is a participant who reached the
|
|
992
|
+
* goal. One rule for the single lane and the fan-out roll-up (#476).
|
|
993
|
+
*/
|
|
994
|
+
export declare function participantStatusForCredibility(status: ActorStatus, credibility: {
|
|
995
|
+
noEngagement: boolean;
|
|
996
|
+
selfReportedBlocker: boolean;
|
|
997
|
+
} | undefined): ActorStatus;
|
package/dist/cua-actor-lab.js
CHANGED
|
@@ -2963,13 +2963,17 @@ async function runCuaActorLabInScope(options) {
|
|
|
2963
2963
|
message: outcome?.sessionError
|
|
2964
2964
|
?? (outcome?.noEngagement
|
|
2965
2965
|
? "Actor took no actions and produced no message (likely a blank/still-loading screen); not a credible goal_satisfied."
|
|
2966
|
-
|
|
2967
|
-
|
|
2968
|
-
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
2966
|
+
// The lane result (toLaneResult) named this refusal; the N=1 envelope fell through to
|
|
2967
|
+
// "did not produce a terminal session", which is false — it produced one and refused it.
|
|
2968
|
+
: outcome?.selfReportedBlocker
|
|
2969
|
+
? "Actor reported goal_satisfied while its final message described a blocker or asked for missing instructions; not a credible pass."
|
|
2970
|
+
: observer.ok
|
|
2971
|
+
? outcome?.session?.completionReason === "harness_error"
|
|
2972
|
+
? `Computer-use session ended with a harness error: ${outcome.session.reason}`
|
|
2973
|
+
: outcome?.session?.status !== "passed"
|
|
2974
|
+
? `Computer-use session ended with ${outcome?.session?.status ?? "unknown"}: ${outcome?.session?.reason ?? "no terminal reason"}`
|
|
2975
|
+
: "Computer-use lab did not produce a terminal session."
|
|
2976
|
+
: observer.error?.message ?? "Observer failed for the computer-use lab run.")
|
|
2973
2977
|
};
|
|
2974
2978
|
}
|
|
2975
2979
|
const failingLane = (outcomes ?? []).find((outcome) => !laneOk(outcome));
|
|
@@ -3117,6 +3121,15 @@ function buildSingleLaneBundle(args) {
|
|
|
3117
3121
|
captureRedaction: args.redactScreenshots ? "blurred" : "raw",
|
|
3118
3122
|
...(outcome?.session ? { session: outcome.session } : {}),
|
|
3119
3123
|
...(outcome?.sessionError ? { sessionError: outcome.sessionError } : {}),
|
|
3124
|
+
...(outcome === undefined
|
|
3125
|
+
? {}
|
|
3126
|
+
: {
|
|
3127
|
+
credibility: {
|
|
3128
|
+
noEngagement: outcome.noEngagement === true,
|
|
3129
|
+
selfReportedBlocker: outcome.selfReportedBlocker === true,
|
|
3130
|
+
reportedFriction: outcome.reportedFriction === true
|
|
3131
|
+
}
|
|
3132
|
+
}),
|
|
3120
3133
|
source: args.source,
|
|
3121
3134
|
...(args.inProgress === undefined ? {} : { inProgress: args.inProgress }),
|
|
3122
3135
|
...(args.subjectProvenance === undefined ? {} : { subjectProvenance: args.subjectProvenance }),
|
|
@@ -3871,22 +3884,36 @@ export function buildCuaBundle(args) {
|
|
|
3871
3884
|
const singleStudyTasks = args.inProgress !== true && args.session?.trace.taskFunnel !== undefined
|
|
3872
3885
|
? aggregateTaskFunnels([args.session.trace.taskFunnel])
|
|
3873
3886
|
: undefined;
|
|
3887
|
+
// What happened to the participant, as the LANE judged it — the same rule the fan-out roll-up
|
|
3888
|
+
// applies (participantStatusForOutcome). Before #476 this read the actor's own status, so a
|
|
3889
|
+
// run the lane refused as "not a credible pass" was written up as verdict pass, 1/1 reached
|
|
3890
|
+
// the goal, and every projection of the bundle (Observer tally, `runs`, the status index)
|
|
3891
|
+
// repeated it. Found on a real drawDB run whose participant wrote "Blocked after partial
|
|
3892
|
+
// completion".
|
|
3893
|
+
const participantStatus = args.session === undefined
|
|
3894
|
+
? undefined
|
|
3895
|
+
: participantStatusForCredibility(args.session.status, args.credibility);
|
|
3896
|
+
const credibilityNote = args.session === undefined || participantStatus === args.session.status
|
|
3897
|
+
? undefined
|
|
3898
|
+
: args.credibility?.noEngagement === true
|
|
3899
|
+
? "Not counted as a pass: the participant took no actions and said nothing."
|
|
3900
|
+
: "Not counted as a pass: the participant's final message described a blocker.";
|
|
3874
3901
|
const review = {
|
|
3875
3902
|
schema: REVIEW_SCHEMA,
|
|
3876
3903
|
verdict: args.inProgress === true
|
|
3877
3904
|
? "contract_proof_only"
|
|
3878
|
-
:
|
|
3879
|
-
? verdictForStatus(
|
|
3905
|
+
: participantStatus !== undefined
|
|
3906
|
+
? verdictForStatus(participantStatus)
|
|
3880
3907
|
: args.sessionError
|
|
3881
3908
|
? "fail"
|
|
3882
3909
|
: "contract_proof_only",
|
|
3883
3910
|
// One lane is still a study with a denominator of one, and saying so keeps a single-lane
|
|
3884
3911
|
// result from being read as though it generalized.
|
|
3885
|
-
...(
|
|
3886
|
-
? { participants: tallyParticipantOutcomes([args.
|
|
3912
|
+
...(participantStatus !== undefined && args.inProgress !== true
|
|
3913
|
+
? { participants: tallyParticipantOutcomes([participantStatus], [args.credibility?.reportedFriction === true]) }
|
|
3887
3914
|
: {}),
|
|
3888
3915
|
...(singleStudyTasks === undefined ? {} : { tasks: singleStudyTasks }),
|
|
3889
|
-
summary: reason
|
|
3916
|
+
summary: credibilityNote === undefined ? reason : `${credibilityNote} ${reason}`,
|
|
3890
3917
|
gaps: args.session || args.sessionError
|
|
3891
3918
|
? []
|
|
3892
3919
|
: args.inProgress === true
|
|
@@ -4309,7 +4336,10 @@ export function buildCuaFanoutBundle(args) {
|
|
|
4309
4336
|
// it — but `reachedGoal` was reading the trace status directly, so one run could be both
|
|
4310
4337
|
// "not a passed lane" AND "1/1 reached the goal". The headline number a researcher reads
|
|
4311
4338
|
// first was the dishonest one. Found by a provider bug that ended a study on turn one.
|
|
4312
|
-
terminalOutcomes.map((outcome) => (outcome.
|
|
4339
|
+
terminalOutcomes.map((outcome) => participantStatusForCredibility(outcome.session.status, {
|
|
4340
|
+
noEngagement: outcome.noEngagement === true,
|
|
4341
|
+
selfReportedBlocker: outcome.selfReportedBlocker === true
|
|
4342
|
+
})),
|
|
4313
4343
|
// A participant who reached the goal AND told you the road there was broken is the most
|
|
4314
4344
|
// useful result a study produces; reporting only the outcome would bury it.
|
|
4315
4345
|
terminalOutcomes.map((outcome) => outcome.reportedFriction === true))
|
|
@@ -4486,6 +4516,22 @@ function providerResourcesForOutcome(args) {
|
|
|
4486
4516
|
}
|
|
4487
4517
|
}];
|
|
4488
4518
|
}
|
|
4519
|
+
/**
|
|
4520
|
+
* The status a participant is TALLIED under, given what the lane made of the session. A
|
|
4521
|
+
* goal_satisfied claim with zero engagement is a session that ran out before anything happened;
|
|
4522
|
+
* one whose final message describes a blocker is a participant who could not proceed and said so.
|
|
4523
|
+
* Both keep their trace status (the claim is evidence); neither is a participant who reached the
|
|
4524
|
+
* goal. One rule for the single lane and the fan-out roll-up (#476).
|
|
4525
|
+
*/
|
|
4526
|
+
export function participantStatusForCredibility(status, credibility) {
|
|
4527
|
+
if (status !== "passed" || credibility === undefined)
|
|
4528
|
+
return status;
|
|
4529
|
+
if (credibility.noEngagement)
|
|
4530
|
+
return "incomplete";
|
|
4531
|
+
if (credibility.selfReportedBlocker)
|
|
4532
|
+
return "blocked";
|
|
4533
|
+
return status;
|
|
4534
|
+
}
|
|
4489
4535
|
function verdictForStatus(status) {
|
|
4490
4536
|
switch (status) {
|
|
4491
4537
|
case "passed":
|