humanish 0.89.1 → 0.91.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -1
- package/dist/automatic-analysis-completion.d.ts +25 -0
- package/dist/automatic-analysis-completion.js +56 -0
- package/dist/automatic-analysis-completion.js.map +1 -0
- package/dist/automatic-analysis-config.d.ts +25 -0
- package/dist/automatic-analysis-config.js +44 -0
- package/dist/automatic-analysis-config.js.map +1 -0
- package/dist/automatic-study-analysis.d.ts +15 -0
- package/dist/automatic-study-analysis.js +187 -0
- package/dist/automatic-study-analysis.js.map +1 -0
- package/dist/concurrent-shared-world-lab.d.ts +4 -2
- package/dist/concurrent-shared-world-lab.js +10 -3
- package/dist/concurrent-shared-world-lab.js.map +1 -1
- package/dist/cua-actor-lab.d.ts +4 -2
- package/dist/cua-actor-lab.js +10 -5
- package/dist/cua-actor-lab.js.map +1 -1
- package/dist/e2b-terminal-lab.d.ts +4 -2
- package/dist/e2b-terminal-lab.js +15 -3
- package/dist/e2b-terminal-lab.js.map +1 -1
- package/dist/export-bundle.js +2 -0
- package/dist/export-bundle.js.map +1 -1
- package/dist/index.d.ts +6 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/lab-config.d.ts +5 -0
- package/dist/lab-config.js +21 -2
- package/dist/lab-config.js.map +1 -1
- package/dist/lab-engine.d.ts +2 -0
- package/dist/lab-engine.js +12 -3
- package/dist/lab-engine.js.map +1 -1
- package/dist/lab-preflight.d.ts +3 -0
- package/dist/lab-preflight.js +3 -0
- package/dist/lab-preflight.js.map +1 -1
- package/dist/lab-summary.d.ts +4 -0
- package/dist/lab-summary.js +6 -1
- package/dist/lab-summary.js.map +1 -1
- package/dist/observer-app.html +8 -8
- package/dist/observer.d.ts +3 -0
- package/dist/observer.js +23 -6
- package/dist/observer.js.map +1 -1
- package/dist/oss-lab.d.ts +1 -1
- package/dist/oss-lab.js.map +1 -1
- package/dist/oss-meta-lab.d.ts +1 -1
- package/dist/oss-meta-lab.js.map +1 -1
- package/dist/program.d.ts +5 -0
- package/dist/program.js +49 -8
- package/dist/program.js.map +1 -1
- package/dist/run-detail.d.ts +2 -0
- package/dist/run-detail.js +3 -0
- package/dist/run-detail.js.map +1 -1
- package/dist/run.d.ts +1 -1
- package/dist/run.js.map +1 -1
- package/dist/scripted-browser-lab.d.ts +4 -2
- package/dist/scripted-browser-lab.js +14 -9
- package/dist/scripted-browser-lab.js.map +1 -1
- package/dist/shared-world-lab.d.ts +4 -2
- package/dist/shared-world-lab.js +10 -3
- package/dist/shared-world-lab.js.map +1 -1
- package/dist/study-analysis-engine.d.ts +7 -2
- package/dist/study-analysis-engine.js +26 -6
- package/dist/study-analysis-engine.js.map +1 -1
- package/dist/study-analysis-evidence.js +273 -59
- package/dist/study-analysis-evidence.js.map +1 -1
- package/dist/study-analysis-job.d.ts +88 -0
- package/dist/study-analysis-job.js +197 -0
- package/dist/study-analysis-job.js.map +1 -0
- package/dist/study-analysis-service.d.ts +10 -1
- package/dist/study-analysis-service.js +31 -8
- package/dist/study-analysis-service.js.map +1 -1
- package/dist/study-analysis-sharing.js +13 -5
- package/dist/study-analysis-sharing.js.map +1 -1
- package/dist/study-analysis-store.d.ts +4 -0
- package/dist/study-analysis-store.js +40 -1
- package/dist/study-analysis-store.js.map +1 -1
- package/dist/study-analysis-validation.d.ts +135 -1
- package/dist/study-analysis-validation.js +62 -26
- package/dist/study-analysis-validation.js.map +1 -1
- package/dist/study-analysis.d.ts +14 -0
- package/dist/study-analysis.js.map +1 -1
- package/dist/terminal-participant-activity.d.ts +6 -0
- package/dist/terminal-participant-activity.js +34 -0
- package/dist/terminal-participant-activity.js.map +1 -0
- package/dist/tui-actions.d.ts +1 -1
- package/dist/tui-actions.js +15 -1
- package/dist/tui-actions.js.map +1 -1
- package/dist/tui-app.js +116 -116
- package/dist/tui-contract.d.ts +3 -2
- package/dist/tui-contract.js.map +1 -1
- package/docs/architecture/examples/state-driven-local-app/README.md +5 -0
- package/docs/architecture/examples/state-driven-local-app/runner.mjs +1 -0
- package/docs/contracts/schemas.md +1 -1
- package/docs/contracts/study-analysis.md +47 -3
- package/docs/goals/current.md +4 -4
- package/docs/principles/invariants-and-defaults.md +7 -3
- package/docs/product/automatic-analysis.md +81 -0
- package/docs/ramp/README.md +11 -2
- package/docs/release/0.90.0-automatic-analysis.md +21 -0
- package/docs/release/0.91.0-analysis-quality-and-defaults.md +33 -0
- package/package.json +1 -1
package/dist/program.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type AutomaticAnalysisHooks } from "./automatic-analysis-completion.js";
|
|
1
2
|
import { Command } from "commander";
|
|
2
3
|
import { discoverProviderKeys } from "./key-resolution.js";
|
|
3
4
|
import { type TelemetryProperties } from "./telemetry.js";
|
|
@@ -84,4 +85,8 @@ export declare function shouldServeOssMetaLabObserver(output: OssMetaLabResult,
|
|
|
84
85
|
};
|
|
85
86
|
};
|
|
86
87
|
export declare function exitCodeForOssMetaLab(output: OssMetaLabResult): number;
|
|
88
|
+
/** Listeners exist only while post-run analysis is active; actor signal behavior is unchanged. */
|
|
89
|
+
export declare function cliAutomaticAnalysisHooks(io: Pick<CliIo, "writeErr">): AutomaticAnalysisHooks;
|
|
90
|
+
/** Preserve the run's own result while making requested post-processing failures machine-visible. */
|
|
91
|
+
export declare function automaticAnalysisEnvelope<T>(result: T): T;
|
|
87
92
|
export {};
|
package/dist/program.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { automaticAnalysisBudget, formatAutomaticAnalysisBudget } from "./automatic-analysis-config.js";
|
|
2
|
+
import { automaticAnalysisSucceeded } from "./automatic-analysis-completion.js";
|
|
1
3
|
import { formatCuaDiagnostics, formatCuaStopCause } from "./cua-diagnostics.js";
|
|
2
4
|
import { existsSync, readFileSync } from "node:fs";
|
|
3
5
|
import { formatOrientationHuman, readOrientation } from "./orientation.js";
|
|
@@ -965,7 +967,7 @@ function analysisSelection(options, command) {
|
|
|
965
967
|
function registerAnalyzeCommand(parent, io) {
|
|
966
968
|
const analyze = parent.command("analyze")
|
|
967
969
|
.enablePositionalOptions()
|
|
968
|
-
.description("Analyze retained participant evidence into versioned findings.
|
|
970
|
+
.description("Analyze retained participant evidence into versioned findings. This request sends selected text and captures to OpenAI. Opening Observer never starts analysis.")
|
|
969
971
|
.summary("Generate evidence-linked study findings.")
|
|
970
972
|
.option("--run <id>", "Completed run id or latest pointer.", "latest")
|
|
971
973
|
.option("--cwd <path>", "Target project directory.", ".")
|
|
@@ -2419,6 +2421,10 @@ async function runLabCommand(args) {
|
|
|
2419
2421
|
// visibly so the invoker (who may not be the manifest author) knows executable code just ran.
|
|
2420
2422
|
args.io.writeErr(`warning: review scorer ${scorer.provenance.ref} (${scorer.provenance.source}) is executable host code loaded and run in-process — review it as code, not config.\n`);
|
|
2421
2423
|
}
|
|
2424
|
+
const analysisBudget = automaticAnalysisBudget(config.review?.analysis, backend);
|
|
2425
|
+
if (analysisBudget && resolveLabDryRun(config, args.options.dryRun, true) === false) {
|
|
2426
|
+
args.io.writeErr(`${formatAutomaticAnalysisBudget(analysisBudget)}\n`);
|
|
2427
|
+
}
|
|
2422
2428
|
switch (backend) {
|
|
2423
2429
|
case "synthetic":
|
|
2424
2430
|
await runSyntheticBackend({ ...args, config, labProvenance: lab });
|
|
@@ -2597,6 +2603,7 @@ async function runCuaBackend(args) {
|
|
|
2597
2603
|
let outcome;
|
|
2598
2604
|
try {
|
|
2599
2605
|
outcome = await runLab(args.config, {
|
|
2606
|
+
automaticAnalysis: cliAutomaticAnalysisHooks(args.io),
|
|
2600
2607
|
cwd: args.options.cwd,
|
|
2601
2608
|
...(args.labProvenance === undefined ? {} : { lab: args.labProvenance }),
|
|
2602
2609
|
// Watch mode opens the served Observer below (or prints the phone target under --expose)
|
|
@@ -2695,7 +2702,7 @@ async function runCuaBackend(args) {
|
|
|
2695
2702
|
};
|
|
2696
2703
|
}
|
|
2697
2704
|
writeResult(args.command, args.io, output, formatCuaLabHuman);
|
|
2698
|
-
args.io.setExitCode(result.ok ? 0 : 2);
|
|
2705
|
+
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
2699
2706
|
if (server && (result.observer?.ok || attachedObserver)) {
|
|
2700
2707
|
const activeServer = server;
|
|
2701
2708
|
const followResult = output.observer?.ok ? output.observer : withObserverServer(attachedObserver, activeServer);
|
|
@@ -2733,6 +2740,7 @@ async function runScriptedBackend(args) {
|
|
|
2733
2740
|
wantsMachine
|
|
2734
2741
|
});
|
|
2735
2742
|
const outcome = await runLab(args.config, {
|
|
2743
|
+
automaticAnalysis: cliAutomaticAnalysisHooks(args.io),
|
|
2736
2744
|
cwd: args.options.cwd,
|
|
2737
2745
|
...(args.labProvenance === undefined ? {} : { lab: args.labProvenance }),
|
|
2738
2746
|
// Watch mode opens the served Observer below instead of the static render.
|
|
@@ -2745,7 +2753,7 @@ async function runScriptedBackend(args) {
|
|
|
2745
2753
|
}
|
|
2746
2754
|
const result = outcome.result;
|
|
2747
2755
|
writeResult(args.command, args.io, result, formatScriptedLabHuman);
|
|
2748
|
-
args.io.setExitCode(result.ok ? 0 : 2);
|
|
2756
|
+
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
2749
2757
|
// Watch mode serves the freshly rendered Observer (and opens it unless told not to).
|
|
2750
2758
|
if (args.mode === "watch" && result.ok && !wantsMachine) {
|
|
2751
2759
|
await renderAndMaybeFollowObserver({
|
|
@@ -2770,6 +2778,7 @@ async function runTerminalBackend(args) {
|
|
|
2770
2778
|
wantsMachine
|
|
2771
2779
|
});
|
|
2772
2780
|
const outcome = await runLab(args.config, {
|
|
2781
|
+
automaticAnalysis: cliAutomaticAnalysisHooks(args.io),
|
|
2773
2782
|
cwd: args.options.cwd,
|
|
2774
2783
|
...(args.labProvenance === undefined ? {} : { lab: args.labProvenance }),
|
|
2775
2784
|
open: args.mode === "watch" ? false : shouldOpen,
|
|
@@ -2782,7 +2791,7 @@ async function runTerminalBackend(args) {
|
|
|
2782
2791
|
}
|
|
2783
2792
|
const result = outcome.result;
|
|
2784
2793
|
writeResult(args.command, args.io, result, formatTerminalLabHuman);
|
|
2785
|
-
args.io.setExitCode(result.ok ? 0 : 2);
|
|
2794
|
+
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
2786
2795
|
if (args.mode === "watch" && result.ok && !wantsMachine) {
|
|
2787
2796
|
await renderAndMaybeFollowObserver({
|
|
2788
2797
|
command: args.command,
|
|
@@ -2804,6 +2813,7 @@ async function runSharedWorldBackend(args) {
|
|
|
2804
2813
|
wantsMachine
|
|
2805
2814
|
});
|
|
2806
2815
|
const outcome = await runLab(args.config, {
|
|
2816
|
+
automaticAnalysis: cliAutomaticAnalysisHooks(args.io),
|
|
2807
2817
|
cwd: args.options.cwd,
|
|
2808
2818
|
...(args.labProvenance === undefined ? {} : { lab: args.labProvenance }),
|
|
2809
2819
|
open: args.mode === "watch" ? false : shouldOpen,
|
|
@@ -2816,7 +2826,7 @@ async function runSharedWorldBackend(args) {
|
|
|
2816
2826
|
}
|
|
2817
2827
|
const result = outcome.result;
|
|
2818
2828
|
writeResult(args.command, args.io, result, formatSharedWorldLabHuman);
|
|
2819
|
-
args.io.setExitCode(result.ok ? 0 : 2);
|
|
2829
|
+
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
2820
2830
|
if (args.mode === "watch" && result.ok && !wantsMachine) {
|
|
2821
2831
|
await renderAndMaybeFollowObserver({
|
|
2822
2832
|
command: args.command,
|
|
@@ -2886,6 +2896,7 @@ async function runConcurrentSharedWorldBackend(args) {
|
|
|
2886
2896
|
let outcome;
|
|
2887
2897
|
try {
|
|
2888
2898
|
outcome = await runLab(args.config, {
|
|
2899
|
+
automaticAnalysis: cliAutomaticAnalysisHooks(args.io),
|
|
2889
2900
|
cwd: args.options.cwd,
|
|
2890
2901
|
...(args.labProvenance === undefined ? {} : { lab: args.labProvenance }),
|
|
2891
2902
|
open: wantsFollow ? false : shouldOpen,
|
|
@@ -2932,7 +2943,7 @@ async function runConcurrentSharedWorldBackend(args) {
|
|
|
2932
2943
|
};
|
|
2933
2944
|
}
|
|
2934
2945
|
writeResult(args.command, args.io, output, formatConcurrentSharedWorldLabHuman);
|
|
2935
|
-
args.io.setExitCode(result.ok ? 0 : 2);
|
|
2946
|
+
args.io.setExitCode(result.ok && automaticAnalysisSucceeded(result) ? 0 : 2);
|
|
2936
2947
|
if (server && output.observer?.ok) {
|
|
2937
2948
|
await followObserver(args.io, output.observer, server);
|
|
2938
2949
|
}
|
|
@@ -3306,11 +3317,17 @@ function withOssMetaLabServer(result, server) {
|
|
|
3306
3317
|
// funnel every real command uses, without duplicating its stdout-vs-formatHuman
|
|
3307
3318
|
// branching. Not re-exported from src/index.ts; this stays an internal seam.
|
|
3308
3319
|
export function writeResult(command, io, result, formatHuman) {
|
|
3320
|
+
const output = automaticAnalysisEnvelope(result);
|
|
3309
3321
|
if (wantsJson(command)) {
|
|
3310
|
-
io.writeOut(`${JSON.stringify(
|
|
3322
|
+
io.writeOut(`${JSON.stringify(output, null, 2)}\n`);
|
|
3311
3323
|
}
|
|
3312
3324
|
else {
|
|
3313
|
-
io.writeOut(formatHuman(
|
|
3325
|
+
io.writeOut(formatHuman(output));
|
|
3326
|
+
if (output !== null && typeof output === "object" && "automaticAnalysis" in output) {
|
|
3327
|
+
const analysis = output.automaticAnalysis;
|
|
3328
|
+
if (analysis)
|
|
3329
|
+
io.writeOut(`analysis: ${analysis.state}${analysis.reason ? ` (${analysis.reason})` : ""}\n`);
|
|
3330
|
+
}
|
|
3314
3331
|
}
|
|
3315
3332
|
markInvocationEnvelopeWritten(command);
|
|
3316
3333
|
// Every backend's result passes through here, so this is where a study's facts get read for
|
|
@@ -3466,6 +3483,7 @@ function formatLabPreflightHuman(result) {
|
|
|
3466
3483
|
`targets: ${checkedTargets.length ? `${reachableTargets.length}/${checkedTargets.length} reachable` : `${result.targets.length} declared, not checked`}`,
|
|
3467
3484
|
...(blockedTargets.length ? [`blocked-targets: ${blockedTargets.length}`] : []),
|
|
3468
3485
|
`spend: ${result.spend.e2bDesktop ? "one e2b desktop, no model calls" : "none"}`,
|
|
3486
|
+
...(result.analysis ? [formatAutomaticAnalysisBudget(result.analysis)] : []),
|
|
3469
3487
|
...(result.sandbox.created
|
|
3470
3488
|
? [`sandbox: created=yes killed=${result.sandbox.killed === true ? "yes" : "no"}`]
|
|
3471
3489
|
: []),
|
|
@@ -3757,4 +3775,27 @@ function wantsJson(command) {
|
|
|
3757
3775
|
}
|
|
3758
3776
|
return false;
|
|
3759
3777
|
}
|
|
3778
|
+
/** Listeners exist only while post-run analysis is active; actor signal behavior is unchanged. */
|
|
3779
|
+
export function cliAutomaticAnalysisHooks(io) {
|
|
3780
|
+
const controller = new AbortController();
|
|
3781
|
+
return {
|
|
3782
|
+
deps: { signal: controller.signal },
|
|
3783
|
+
onStart: () => {
|
|
3784
|
+
const cancel = () => { controller.abort(); };
|
|
3785
|
+
const signals = ["SIGINT", "SIGTERM", "SIGHUP"];
|
|
3786
|
+
io.writeErr("Participants finished; preparing analysis…\n");
|
|
3787
|
+
for (const signal of signals)
|
|
3788
|
+
process.on(signal, cancel);
|
|
3789
|
+
return () => { for (const signal of signals)
|
|
3790
|
+
process.off(signal, cancel); };
|
|
3791
|
+
}
|
|
3792
|
+
};
|
|
3793
|
+
}
|
|
3794
|
+
/** Preserve the run's own result while making requested post-processing failures machine-visible. */
|
|
3795
|
+
export function automaticAnalysisEnvelope(result) {
|
|
3796
|
+
if (result === null || typeof result !== "object" || !("automaticAnalysis" in result) || !("ok" in result))
|
|
3797
|
+
return result;
|
|
3798
|
+
const run = result;
|
|
3799
|
+
return { ...run, runOk: run.ok, ok: run.ok && automaticAnalysisSucceeded(run) };
|
|
3800
|
+
}
|
|
3760
3801
|
//# sourceMappingURL=program.js.map
|