humanish 0.146.0 → 0.147.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/dist/actors/computer-use/loop/spend.js +5 -4
- package/dist/actors/computer-use/loop/trace.js +3 -0
- package/dist/actors/computer-use/loop/types.d.ts +5 -0
- package/dist/actors/computer-use/loop/usage.js +10 -2
- package/dist/actors/computer-use/openai-conversation.d.ts +7 -0
- package/dist/actors/computer-use/openai-conversation.js +19 -2
- package/dist/actors/computer-use/openai-provider.d.ts +2 -0
- package/dist/actors/computer-use/openai-provider.js +2 -0
- package/dist/actors/computer-use/openai-wire.d.ts +4 -0
- package/dist/actors/computer-use/openai-wire.js +9 -1
- package/dist/actors/contract.d.ts +5 -0
- package/dist/analysis/evidence.js +8 -4
- package/dist/analysis/image-tokens.js +9 -3
- package/dist/evidence/image.d.ts +2 -0
- package/dist/evidence/image.js +24 -3
- package/dist/evidence/webp-encode-worker.d.ts +1 -0
- package/dist/evidence/webp-encode-worker.js +31 -0
- package/dist/evidence/webp-transcoder.d.ts +15 -0
- package/dist/evidence/webp-transcoder.js +98 -0
- package/dist/evidence/webp.d.ts +31 -0
- package/dist/evidence/webp.js +94 -0
- package/dist/feedback/derivative-text.d.ts +7 -2
- package/dist/feedback/derivative-text.js +20 -1
- package/dist/feedback/export-bundle.js +79 -25
- package/dist/remote/launch.js +9 -4
- package/dist/routes/computer-use/participant-execution.d.ts +0 -7
- package/dist/routes/computer-use/participant-execution.js +6 -28
- package/dist/routes/computer-use/participant-frames.d.ts +32 -0
- package/dist/routes/computer-use/participant-frames.js +93 -0
- package/dist/routes/computer-use/participant-model.js +3 -0
- package/dist/routes/computer-use/participant-runs.js +4 -0
- package/dist/routes/computer-use/plan.js +5 -1
- package/dist/routes/computer-use/result.js +6 -0
- package/dist/routes/computer-use/types.d.ts +4 -0
- package/dist/routes/shared-world/plan.js +4 -1
- package/dist/run/pricing.d.ts +7 -0
- package/dist/run/pricing.js +58 -15
- package/dist/study/config.js +4 -1
- package/dist/study/keys.d.ts +1 -0
- package/dist/study/keys.js +1 -0
- package/dist/study/parse/actors.js +5 -0
- package/dist/study/plan-participants.d.ts +2 -0
- package/dist/study/plan-participants.js +2 -0
- package/dist/study/types.d.ts +7 -0
- package/dist/study/validation.d.ts +5 -0
- package/dist/study/validation.js +23 -0
- package/dist/verify/artifacts.js +2 -1
- package/docs/contracts/run-bundle.md +29 -11
- package/docs/contracts/schemas.md +19 -11
- package/docs/contracts/study-analysis.md +2 -2
- package/package.json +2 -1
|
@@ -13,7 +13,7 @@ import { brainOf, callerBrainOf, planCaps, desktopRequirements, planBase, provis
|
|
|
13
13
|
import { computerUseParticipants, declaredTargets, } from "../../study/plan-participants.js";
|
|
14
14
|
import { planStarts, windowDeadlineRefusal } from "../../study/arrivals.js";
|
|
15
15
|
import { computerUseParticipantCount, participantCountReason } from "../../study/routing.js";
|
|
16
|
-
import { cloneTargetValidationReason, computerUseValidationReason, desktopMediaValidationReason, networkCaptureValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "../../study/validation.js";
|
|
16
|
+
import { cloneTargetValidationReason, computerUseValidationReason, desktopMediaValidationReason, networkCaptureValidationReason, outputTokenLimitValidationReason, serviceTierValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "../../study/validation.js";
|
|
17
17
|
import { desktopCliProductReason } from "../../study/composition-rules.js";
|
|
18
18
|
import { isLocalBrowserStudy } from "../../substrates/local/runtime-config.js";
|
|
19
19
|
import { boundedConcurrency, defaultSessionTimeoutMs, resolveParticipantSandboxMs, } from "./participant-runs.js";
|
|
@@ -89,12 +89,16 @@ function unsupportedDeclarationReason(config, hasRunSession, driving, { inProces
|
|
|
89
89
|
const reason = desktopMediaValidationReason(config) ||
|
|
90
90
|
networkCaptureValidationReason(config, "computer-use") ||
|
|
91
91
|
outputTokenLimitValidationReason(config) ||
|
|
92
|
+
serviceTierValidationReason(config) ||
|
|
92
93
|
waitLimitValidationReason(config);
|
|
93
94
|
if (reason)
|
|
94
95
|
return invalid(reason);
|
|
95
96
|
if (config.actor?.maxOutputTokens !== undefined &&
|
|
96
97
|
(hasRunSession || driving.createProvider || driving.inProcess))
|
|
97
98
|
return invalid("maxOutputTokens cannot be enforced by a custom runSession/provider/executor route.");
|
|
99
|
+
if (config.actor?.serviceTier !== undefined &&
|
|
100
|
+
(hasRunSession || driving.createProvider || driving.inProcess))
|
|
101
|
+
return invalid("serviceTier is not sent by a custom runSession/provider/executor route.");
|
|
98
102
|
const receivingReason = receivingEmailValidationReason(config);
|
|
99
103
|
if (receivingReason)
|
|
100
104
|
return invalid(receivingReason);
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { removeContainedOutputFile } from "../../run/contained-output.js";
|
|
1
2
|
import { adapterScoreFailureMessage, applyBrowserScorer } from "../../study/adapter-extension.js";
|
|
2
3
|
import { redactText } from "../../evidence/redaction.js";
|
|
3
4
|
import { foldScorerFailures, judgeExecution, judgeParticipantRecords, OUTCOME_POLICIES, participantHarnessFailed, resultOk, } from "../../run/judge.js";
|
|
@@ -260,6 +261,11 @@ export async function finishCuaRun(setup, finish, ran) {
|
|
|
260
261
|
execution,
|
|
261
262
|
policy,
|
|
262
263
|
});
|
|
264
|
+
// The final bundle names each replaced frame's WebP, so its PNG goes before the Observer
|
|
265
|
+
// verifies the run directory.
|
|
266
|
+
for (const outcome of outcomes ?? [])
|
|
267
|
+
for (const png of outcome.replacedFrames ?? [])
|
|
268
|
+
await removeContainedOutputFile(finished.paths, png);
|
|
263
269
|
const observer = await finished.renderObserver();
|
|
264
270
|
// A caller's renderer runs with this process's file access, so after one that reports success
|
|
265
271
|
// the run directory is checked again before the result points anyone at it. A failed render is
|
|
@@ -112,6 +112,8 @@ export interface CuaParticipantPlanEntry {
|
|
|
112
112
|
* you read before spending money, so a declared per-participant difference has to be visible there. */
|
|
113
113
|
reasoningEffort?: string;
|
|
114
114
|
maxOutputTokens?: number;
|
|
115
|
+
/** The declared OpenAI service tier, which bills at its own rate. */
|
|
116
|
+
serviceTier?: string;
|
|
115
117
|
/** Present only when a participant overrides subject.appUrl; digest avoids leaking preview hosts in plan logs. */
|
|
116
118
|
targetDigest?: string;
|
|
117
119
|
/** The participant's declared start, in ms after the run starts its participants. */
|
|
@@ -432,6 +434,8 @@ export interface SandboxReleaseFact {
|
|
|
432
434
|
/** One participant's end-to-end run outcome (internal; projected into CuaParticipantResult + the bundle). */
|
|
433
435
|
export interface ParticipantRunOutcome {
|
|
434
436
|
spec: DesktopParticipantRun;
|
|
437
|
+
/** The PNG frames its WebP replaced, which the run removes once its final bundle is written. */
|
|
438
|
+
replacedFrames?: string[];
|
|
435
439
|
session?: CuaLoopResult;
|
|
436
440
|
/** The harness failed before the session reached a terminal status. */
|
|
437
441
|
sessionError?: string;
|
|
@@ -12,7 +12,7 @@ import { DESKTOP_FILES_ROUTE_REASON } from "../../study/desktop-files.js";
|
|
|
12
12
|
import { participantList } from "../../study/study-fields.js";
|
|
13
13
|
import { PUBLIC_TARGET_OWNER_PATTERN } from "../../study/parse/subject.js";
|
|
14
14
|
import { REPO_SLUG_PATTERN } from "../../study/parse/values.js";
|
|
15
|
-
import { concurrentSharedWorldValidationReason, desktopMediaValidationReason, networkCaptureValidationReason, externalPublicSharedWorldValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "../../study/validation.js";
|
|
15
|
+
import { concurrentSharedWorldValidationReason, desktopMediaValidationReason, networkCaptureValidationReason, externalPublicSharedWorldValidationReason, outputTokenLimitValidationReason, serviceTierValidationReason, receivingEmailValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "../../study/validation.js";
|
|
16
16
|
import { unpricedCapCheck } from "../../study/requirements.js";
|
|
17
17
|
import { participantsAtOnce, sandboxDeadlineRefusal, sandboxHeadroomMs, } from "../../substrates/e2b/lifetime.js";
|
|
18
18
|
import { boundedConcurrency } from "../computer-use/participant-runs.js";
|
|
@@ -62,6 +62,7 @@ export function planSharedWorldStudy(config, input) {
|
|
|
62
62
|
// A library caller's config skips the parser, which refuses these first.
|
|
63
63
|
studyUrlCredentialReason(config) ??
|
|
64
64
|
outputTokenLimitValidationReason(config) ??
|
|
65
|
+
serviceTierValidationReason(config) ??
|
|
65
66
|
waitLimitValidationReason(config) ??
|
|
66
67
|
(externalPublic
|
|
67
68
|
? externalPublicSharedWorldValidationReason(config)
|
|
@@ -70,6 +71,8 @@ export function planSharedWorldStudy(config, input) {
|
|
|
70
71
|
return refuse(invalid, invalidReason, actor);
|
|
71
72
|
if (config.actor?.maxOutputTokens !== undefined && input.hasRunSession === true)
|
|
72
73
|
return refuse(invalid, "maxOutputTokens cannot be enforced by a custom runSession.", actor);
|
|
74
|
+
if (config.actor?.serviceTier !== undefined && input.hasRunSession === true)
|
|
75
|
+
return refuse(invalid, "serviceTier is not sent by a custom runSession.", actor);
|
|
73
76
|
// A live cap is priced here, at plan time, so its refusal wins over the route's key checks.
|
|
74
77
|
const unpriced = input.dryRun
|
|
75
78
|
? undefined
|
package/dist/run/pricing.d.ts
CHANGED
|
@@ -30,6 +30,11 @@ export interface ModelRate {
|
|
|
30
30
|
until?: string;
|
|
31
31
|
/** The rates the sheet publishes from the day after `until`. */
|
|
32
32
|
after?: Pick<ModelRate, "inputUsdPerToken" | "outputUsdPerToken" | "cachedInputUsdPerToken">;
|
|
33
|
+
/** Fast mode rates (OpenAI `service_tier: "fast"`, served as `fast` or `priority`), when the
|
|
34
|
+
* sheet publishes them. The long-context multipliers apply on top. */
|
|
35
|
+
fast?: Pick<ModelRate, "inputUsdPerToken" | "outputUsdPerToken" | "cachedInputUsdPerToken" | "cacheWriteUsdPerToken"> & {
|
|
36
|
+
source?: string;
|
|
37
|
+
};
|
|
33
38
|
/** The `YYYY-MM-DD` date the entry was last checked against `source`. */
|
|
34
39
|
asOf: string;
|
|
35
40
|
/** Public pricing page the number came from (a comment/URL, never a secret). */
|
|
@@ -101,6 +106,8 @@ export interface ActorEstimatedCost {
|
|
|
101
106
|
/** How many requests crossed the long-context threshold and were re-tiered. Present only
|
|
102
107
|
* when per-request `turns` records made exact tiering possible. */
|
|
103
108
|
longContextTurns?: number;
|
|
109
|
+
/** How many requests the provider served in Fast mode and were priced at its rates. */
|
|
110
|
+
fastTurns?: number;
|
|
104
111
|
};
|
|
105
112
|
}
|
|
106
113
|
/** The desktop-minute cost estimate (host-side create->teardown span * a per-minute rate). Same
|
package/dist/run/pricing.js
CHANGED
|
@@ -54,6 +54,26 @@ function geminiRate(inPer1M, cachedPer1M, outPer1M, change = {}) {
|
|
|
54
54
|
source: "ai.google.dev/gemini-api/docs/pricing (paid tier, standard)",
|
|
55
55
|
};
|
|
56
56
|
}
|
|
57
|
+
// Fast mode: OpenAI's `service_tier: "fast"` (formerly priority processing), billed per token at
|
|
58
|
+
// its own rates with the cached-input discount and the long-context tier. gpt-5.6-sol's fast rates
|
|
59
|
+
// are twice every standard rate ("For GPT-5.6 Sol, Fast mode costs twice the corresponding Standard
|
|
60
|
+
// rate", developers.openai.com/api/docs/guides/fast-mode, read 2026-10-24); the 6-series rates are
|
|
61
|
+
// the Fast tab of developers.openai.com/api/docs/pricing?latest-pricing=fast, read 2026-10-24.
|
|
62
|
+
const FAST_MODE_GUIDE = "developers.openai.com/api/docs/guides/fast-mode (gpt-5.6-sol, 2x standard)";
|
|
63
|
+
const FAST_MODE_SHEET = "developers.openai.com/api/docs/pricing?latest-pricing=fast (fast tier)";
|
|
64
|
+
/** A rate with Fast mode rates per 1M tokens: input, cached input, cache write and output. */
|
|
65
|
+
function withFast(rate, inPer1M, cachedPer1M, writePer1M, outPer1M, source) {
|
|
66
|
+
return {
|
|
67
|
+
...rate,
|
|
68
|
+
fast: {
|
|
69
|
+
inputUsdPerToken: inPer1M * 1e-6,
|
|
70
|
+
cachedInputUsdPerToken: cachedPer1M * 1e-6,
|
|
71
|
+
cacheWriteUsdPerToken: writePer1M * 1e-6,
|
|
72
|
+
outputUsdPerToken: outPer1M * 1e-6,
|
|
73
|
+
source,
|
|
74
|
+
},
|
|
75
|
+
};
|
|
76
|
+
}
|
|
57
77
|
// gpt-5.6-sol promotional rates (live sheet 2026-09-03: $4 / $0.40 cached / $5 write / $20 out,
|
|
58
78
|
// "available at least through November 21, 2026"). Re-verify against the sheet after Nov 21.
|
|
59
79
|
const GPT56_SOL_PROMO_AS_OF = "2026-09-03";
|
|
@@ -84,10 +104,10 @@ export const MODEL_RATES = {
|
|
|
84
104
|
// the longContext block prices the >272K re-tier when per-request turns are recorded).
|
|
85
105
|
// sol = flagship (the shipped CUA default), terra = cost-balanced, luna = high-volume,
|
|
86
106
|
// cyber = the Daybreak frontier tier.
|
|
87
|
-
"gpt-5.6-sol": rateWithLongContextTier(4, 0.4, 5, 20, GPT56_SOL_PROMO_AS_OF),
|
|
107
|
+
"gpt-5.6-sol": withFast(rateWithLongContextTier(4, 0.4, 5, 20, GPT56_SOL_PROMO_AS_OF), 8, 0.8, 10, 40, FAST_MODE_GUIDE),
|
|
88
108
|
// "gpt-5.6" is OpenAI's own alias for gpt-5.6-sol (models index); priced identically so a
|
|
89
109
|
// study configured with the alias never reads as unpriced.
|
|
90
|
-
"gpt-5.6": rateWithLongContextTier(4, 0.4, 5, 20, GPT56_SOL_PROMO_AS_OF),
|
|
110
|
+
"gpt-5.6": withFast(rateWithLongContextTier(4, 0.4, 5, 20, GPT56_SOL_PROMO_AS_OF), 8, 0.8, 10, 40, FAST_MODE_GUIDE),
|
|
91
111
|
"gpt-5.6-terra": rateWithLongContextTier(2, 0.2, 2.5, 12),
|
|
92
112
|
"gpt-5.6-luna": rateWithLongContextTier(0.2, 0.02, 0.25, 1.2),
|
|
93
113
|
"gpt-5.6-cyber": rateWithLongContextTier(12.5, 1.25, 15.625, 75),
|
|
@@ -100,11 +120,11 @@ export const MODEL_RATES = {
|
|
|
100
120
|
// as the 5.6 family on the sheet: writes at 1.25x, >272K re-tiers at 2x input-side / 1.5x
|
|
101
121
|
// output ($20 / $2 / $25 / $75 long-context columns). These rates cover explicit study
|
|
102
122
|
// model choices and the study-analysis default; the computer-use default is separate.
|
|
103
|
-
"gpt-6-astra": rateWithLongContextTier(10, 1, 12.5, 50, GPT6_ASTRA_AS_OF, GPT6_ASTRA_SOURCE),
|
|
123
|
+
"gpt-6-astra": withFast(rateWithLongContextTier(10, 1, 12.5, 50, GPT6_ASTRA_AS_OF, GPT6_ASTRA_SOURCE), 20, 2, 25, 100, FAST_MODE_SHEET),
|
|
104
124
|
// gpt-6.1-sol and gpt-6-luna: the same write and long-context mechanics on the sheet and each
|
|
105
125
|
// model page. 6.1 Sol bills cache reads at 5% of input ($0.10), where the others bill 10%.
|
|
106
|
-
"gpt-6.1-sol": rateWithLongContextTier(2, 0.1, 2.5, 10, GPT6_SOL_LUNA_AS_OF, GPT6_SOL_LUNA_SOURCE),
|
|
107
|
-
"gpt-6-luna": rateWithLongContextTier(0.1, 0.01, 0.125, 0.5, GPT6_SOL_LUNA_AS_OF, GPT6_SOL_LUNA_SOURCE),
|
|
126
|
+
"gpt-6.1-sol": withFast(rateWithLongContextTier(2, 0.1, 2.5, 10, GPT6_SOL_LUNA_AS_OF, GPT6_SOL_LUNA_SOURCE), 4, 0.2, 5, 20, FAST_MODE_SHEET),
|
|
127
|
+
"gpt-6-luna": withFast(rateWithLongContextTier(0.1, 0.01, 0.125, 0.5, GPT6_SOL_LUNA_AS_OF, GPT6_SOL_LUNA_SOURCE), 0.2, 0.02, 0.25, 1, FAST_MODE_SHEET),
|
|
108
128
|
// Gemini analysis models, paid standard tier. Thinking tokens bill as output. No long-context
|
|
109
129
|
// tier is published for them. Cached input is the implicit context-cache rate. On the free tier
|
|
110
130
|
// these requests bill nothing, so the estimate is the paid-tier cost.
|
|
@@ -283,24 +303,41 @@ export function estimateActorCost(tokenUsage, modelId, rates = MODEL_RATES, at =
|
|
|
283
303
|
// price every token at the full rate while the session totals sit ignored, overstating spend
|
|
284
304
|
// 3.5-5x and tripping caps early. Inconsistent evidence falls back to the totals path, which
|
|
285
305
|
// honors the declared splits on the base tier.
|
|
286
|
-
const tiered = rate.longContext !== undefined &&
|
|
306
|
+
const tiered = (rate.longContext !== undefined || turns.some((turn) => isFastServiceTier(turn.serviceTier))) &&
|
|
287
307
|
turns.length > 0 &&
|
|
288
308
|
sumOf("input") === inTok &&
|
|
289
309
|
sumOf("output") === outTok &&
|
|
290
310
|
sumOf("cachedInput") === (tokenUsage.cachedInput ?? 0) &&
|
|
291
311
|
sumOf("cacheWriteInput") === (tokenUsage.cacheWriteInput ?? 0);
|
|
312
|
+
// A request served in Fast mode bills at the fast rates. Totals that the ledger does not
|
|
313
|
+
// decompose price at the fast rate when any request was served fast: the over-estimate direction.
|
|
314
|
+
const fastTotals = !tiered && turns.some((turn) => isFastServiceTier(turn.serviceTier));
|
|
315
|
+
// A request served fast on a model with no Fast mode rate has no rate in this table.
|
|
316
|
+
if (rate.fast === undefined && turns.some((turn) => isFastServiceTier(turn.serviceTier))) {
|
|
317
|
+
return {
|
|
318
|
+
schema: ACTOR_ESTIMATED_COST_SCHEMA,
|
|
319
|
+
estimatedCostUsd: null,
|
|
320
|
+
reason: "no_rate_for_model",
|
|
321
|
+
ratesAsOf: null,
|
|
322
|
+
...(modelId ? { modelId } : {}),
|
|
323
|
+
};
|
|
324
|
+
}
|
|
292
325
|
let inputUsd = 0;
|
|
293
326
|
let outputUsd = 0;
|
|
294
327
|
let cachedTotal = 0;
|
|
295
328
|
let writeTotal = 0;
|
|
296
329
|
let longContextTurns = 0;
|
|
330
|
+
let fastTurns = 0;
|
|
297
331
|
// Price one request's usage at one tier. Cached input is billed at a fraction of the full rate,
|
|
298
332
|
// and on a session that threads provider state it is the majority of input, and pricing it at the
|
|
299
333
|
// full rate overstates real spend by up to ~10x. Cache writes bill at their own rate (1.25x on
|
|
300
334
|
// OpenAI 5.6+) as the total rate for those tokens; a sheet without a write rate prices writes as
|
|
301
335
|
// plain input (pre-5.6: writes are free-of-extra-fee, i.e. plain input). An unreported piece
|
|
302
336
|
// stays absent: no reported split means no discount and no surcharge assumed.
|
|
303
|
-
const priceRequest = (usage, tierable) => {
|
|
337
|
+
const priceRequest = (usage, tierable, fast) => {
|
|
338
|
+
const sheet = fast && rate.fast !== undefined ? { ...rate, ...rate.fast } : rate;
|
|
339
|
+
if (fast)
|
|
340
|
+
fastTurns += 1;
|
|
304
341
|
const reqIn = usage.input ?? 0;
|
|
305
342
|
const reqOut = usage.output ?? 0;
|
|
306
343
|
// Tiering applies only to a real per-request record: session totals crossing the threshold
|
|
@@ -310,7 +347,7 @@ export function estimateActorCost(tokenUsage, modelId, rates = MODEL_RATES, at =
|
|
|
310
347
|
const outMul = long ? rate.longContext.outputMultiplier : 1;
|
|
311
348
|
if (long)
|
|
312
349
|
longContextTurns += 1;
|
|
313
|
-
const cached =
|
|
350
|
+
const cached = sheet.cachedInputUsdPerToken === undefined
|
|
314
351
|
? 0
|
|
315
352
|
: Math.min(reqIn, Math.max(0, usage.cachedInput ?? 0));
|
|
316
353
|
const writes = Math.min(reqIn - cached, Math.max(0, usage.cacheWriteInput ?? 0));
|
|
@@ -318,14 +355,14 @@ export function estimateActorCost(tokenUsage, modelId, rates = MODEL_RATES, at =
|
|
|
318
355
|
cachedTotal += cached;
|
|
319
356
|
writeTotal += writes;
|
|
320
357
|
inputUsd +=
|
|
321
|
-
full *
|
|
322
|
-
cached * (
|
|
323
|
-
writes * (
|
|
324
|
-
outputUsd += reqOut *
|
|
358
|
+
full * sheet.inputUsdPerToken * inMul +
|
|
359
|
+
cached * (sheet.cachedInputUsdPerToken ?? 0) * inMul +
|
|
360
|
+
writes * (sheet.cacheWriteUsdPerToken ?? sheet.inputUsdPerToken) * inMul;
|
|
361
|
+
outputUsd += reqOut * sheet.outputUsdPerToken * outMul;
|
|
325
362
|
};
|
|
326
363
|
if (tiered) {
|
|
327
364
|
for (const turn of turns)
|
|
328
|
-
priceRequest(turn, true);
|
|
365
|
+
priceRequest(turn, true, isFastServiceTier(turn.serviceTier));
|
|
329
366
|
}
|
|
330
367
|
else {
|
|
331
368
|
priceRequest({
|
|
@@ -335,7 +372,7 @@ export function estimateActorCost(tokenUsage, modelId, rates = MODEL_RATES, at =
|
|
|
335
372
|
...(tokenUsage.cacheWriteInput === undefined
|
|
336
373
|
? {}
|
|
337
374
|
: { cacheWriteInput: tokenUsage.cacheWriteInput }),
|
|
338
|
-
}, false);
|
|
375
|
+
}, false, fastTotals);
|
|
339
376
|
}
|
|
340
377
|
inputUsd = round6(inputUsd);
|
|
341
378
|
outputUsd = round6(outputUsd);
|
|
@@ -343,7 +380,7 @@ export function estimateActorCost(tokenUsage, modelId, rates = MODEL_RATES, at =
|
|
|
343
380
|
schema: ACTOR_ESTIMATED_COST_SCHEMA,
|
|
344
381
|
estimatedCostUsd: round6(inputUsd + outputUsd),
|
|
345
382
|
ratesAsOf: rate.asOf,
|
|
346
|
-
source: rate.source,
|
|
383
|
+
source: fastTurns > 0 && rate.fast?.source !== undefined ? rate.fast.source : rate.source,
|
|
347
384
|
...(modelId ? { modelId } : {}),
|
|
348
385
|
...(rate.placeholder ? { placeholder: true } : {}),
|
|
349
386
|
breakdown: {
|
|
@@ -354,9 +391,15 @@ export function estimateActorCost(tokenUsage, modelId, rates = MODEL_RATES, at =
|
|
|
354
391
|
...(cachedTotal > 0 ? { cachedInputTokens: cachedTotal } : {}),
|
|
355
392
|
...(writeTotal > 0 ? { cacheWriteInputTokens: writeTotal } : {}),
|
|
356
393
|
...(longContextTurns > 0 ? { longContextTurns } : {}),
|
|
394
|
+
...(fastTurns > 0 ? { fastTurns } : {}),
|
|
357
395
|
},
|
|
358
396
|
};
|
|
359
397
|
}
|
|
398
|
+
/** Whether a provider-reported tier bills at Fast mode rates. GPT-5.6 reports `priority` for a
|
|
399
|
+
* request that asked for `fast`. */
|
|
400
|
+
function isFastServiceTier(tier) {
|
|
401
|
+
return tier === "fast" || tier === "priority";
|
|
402
|
+
}
|
|
360
403
|
/**
|
|
361
404
|
* Estimate the E2B desktop-minute cost from a host-side create->teardown span (minutes). The rate
|
|
362
405
|
* is injectable. Returns a declared-absent estimate (null + "no_duration") when no duration was
|
package/dist/study/config.js
CHANGED
|
@@ -39,7 +39,7 @@ import { invalid, optionalStr, str } from "./parse/values.js";
|
|
|
39
39
|
import { isComputerUseComposition, routeOf } from "./routing.js";
|
|
40
40
|
import { declaredParticipantIds } from "./plan-participants.js";
|
|
41
41
|
import { ID_PATTERN, STUDY_SCHEMA, } from "./types.js";
|
|
42
|
-
import { automaticAnalysisRouteReason, desktopMediaValidationReason, networkCaptureValidationReason, outputTokenLimitValidationReason, receivingEmailValidationReason, smtpValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "./validation.js";
|
|
42
|
+
import { automaticAnalysisRouteReason, desktopMediaValidationReason, networkCaptureValidationReason, outputTokenLimitValidationReason, serviceTierValidationReason, receivingEmailValidationReason, smtpValidationReason, taskProtocolValidationReason, waitLimitValidationReason, } from "./validation.js";
|
|
43
43
|
import { forwardDeclaredWarnings, inertFieldLabels, inertFieldPaths } from "./warnings.js";
|
|
44
44
|
import { plural } from "../run/text.js";
|
|
45
45
|
/**
|
|
@@ -234,6 +234,9 @@ function checkStudyConfig(config) {
|
|
|
234
234
|
const outputLimitReason = outputTokenLimitValidationReason(config);
|
|
235
235
|
if (outputLimitReason)
|
|
236
236
|
return invalid(outputLimitReason);
|
|
237
|
+
const serviceTierReason = serviceTierValidationReason(config);
|
|
238
|
+
if (serviceTierReason)
|
|
239
|
+
return invalid(serviceTierReason);
|
|
237
240
|
const waitLimitReason = waitLimitValidationReason(config);
|
|
238
241
|
if (waitLimitReason)
|
|
239
242
|
return invalid(waitLimitReason);
|
package/dist/study/keys.d.ts
CHANGED
package/dist/study/keys.js
CHANGED
|
@@ -112,6 +112,11 @@ export function parseActorFields(entry, path) {
|
|
|
112
112
|
}
|
|
113
113
|
actor.reasoningEffort = entry.reasoningEffort;
|
|
114
114
|
}
|
|
115
|
+
if (entry.serviceTier !== undefined) {
|
|
116
|
+
if (entry.serviceTier !== "fast")
|
|
117
|
+
return invalid(`${path}.serviceTier must be "fast" (OpenAI Fast mode), or left out for the standard tier.`);
|
|
118
|
+
actor.serviceTier = entry.serviceTier;
|
|
119
|
+
}
|
|
115
120
|
const stopWhenResult = parseStopWhen(entry.stopWhen, `${path}.stopWhen`);
|
|
116
121
|
if (!stopWhenResult.ok)
|
|
117
122
|
return stopWhenResult;
|
|
@@ -37,6 +37,8 @@ export interface DesktopParticipant extends Participant {
|
|
|
37
37
|
readonly dwell?: DwellWindow;
|
|
38
38
|
readonly reasoningEffort?: ReasoningEffort;
|
|
39
39
|
readonly maxOutputTokens?: number;
|
|
40
|
+
/** actor.serviceTier: OpenAI Fast mode on every request. */
|
|
41
|
+
readonly serviceTier?: "fast";
|
|
40
42
|
/** actor.maxWaitMs: the longest one wait or one turn of only waits lasts. */
|
|
41
43
|
readonly maxWaitMs?: number;
|
|
42
44
|
/**
|
|
@@ -13,6 +13,7 @@ function desktopParticipant(config, entry, index, kind, focus) {
|
|
|
13
13
|
const dwell = entry?.dwell ?? actor?.dwell;
|
|
14
14
|
const reasoningEffort = entry?.reasoningEffort ?? actor?.reasoningEffort;
|
|
15
15
|
const maxOutputTokens = actor?.maxOutputTokens;
|
|
16
|
+
const serviceTier = actor?.serviceTier;
|
|
16
17
|
const maxWaitMs = actor?.maxWaitMs;
|
|
17
18
|
const idleWaitMs = entry?.idleWaitMs ?? actor?.idleWaitMs;
|
|
18
19
|
const timeZone = entry?.clock?.timeZone ?? actor?.clock?.timeZone;
|
|
@@ -35,6 +36,7 @@ function desktopParticipant(config, entry, index, kind, focus) {
|
|
|
35
36
|
...(dwell === undefined ? {} : { dwell }),
|
|
36
37
|
...(reasoningEffort === undefined ? {} : { reasoningEffort }),
|
|
37
38
|
...(maxOutputTokens === undefined ? {} : { maxOutputTokens }),
|
|
39
|
+
...(serviceTier === undefined ? {} : { serviceTier }),
|
|
38
40
|
...(maxWaitMs === undefined ? {} : { maxWaitMs }),
|
|
39
41
|
...(idleWaitMs === undefined ? {} : { idleWaitMs }),
|
|
40
42
|
...(timeZone === undefined ? {} : { timeZone }),
|
package/dist/study/types.d.ts
CHANGED
|
@@ -453,6 +453,13 @@ export interface StudyActor {
|
|
|
453
453
|
* docs/principles/actor-fidelity.md.
|
|
454
454
|
*/
|
|
455
455
|
reasoningEffort?: ReasoningEffort;
|
|
456
|
+
/**
|
|
457
|
+
* `fast` sends every participant request in OpenAI Fast mode (`service_tier: "fast"`), which
|
|
458
|
+
* bills at its own per-token rates. Absent sends no tier. `openai-computer-use` only, and only for
|
|
459
|
+
* a model whose Fast mode rate src/run/pricing.ts carries, so the spend estimate and caps price it.
|
|
460
|
+
* Each request's trace usage records the tier that served it.
|
|
461
|
+
*/
|
|
462
|
+
serviceTier?: "fast";
|
|
456
463
|
/**
|
|
457
464
|
* Deterministic completion guard used as the default for CUA participants. A `participants` entry's stopWhen
|
|
458
465
|
* overrides this value.
|
|
@@ -44,6 +44,11 @@ export declare function taskProtocolValidationReason(config: StudyConfig, suppor
|
|
|
44
44
|
export declare function cloneTargetValidationReason(config: StudyConfig): string | null;
|
|
45
45
|
/** Refuse a claimed output bound when the route cannot pass it to the first-party provider. */
|
|
46
46
|
export declare function outputTokenLimitValidationReason(config: StudyConfig): string | null;
|
|
47
|
+
/**
|
|
48
|
+
* Refuse a Fast mode request humanish cannot send or price: the first-party OpenAI computer-use
|
|
49
|
+
* route sends it, and the spend estimate needs the model's Fast mode rate.
|
|
50
|
+
*/
|
|
51
|
+
export declare function serviceTierValidationReason(config: StudyConfig): string | null;
|
|
47
52
|
/**
|
|
48
53
|
* Refuse a wait field out of range, or an idle wait longer than the longest wait: the actor's, and
|
|
49
54
|
* each participant's own idleWaitMs under actor.maxWaitMs. On a route whose participants take no
|
package/dist/study/validation.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { isMaxOutputTokens } from "../actors/output-token-limit.js";
|
|
2
|
+
import { DEFAULT_OPENAI_CU_MODEL } from "../actors/computer-use/openai-provider.js";
|
|
3
|
+
import { MODEL_RATES, participantRate } from "../run/pricing.js";
|
|
2
4
|
import { MAX_RECEIVING_INBOXES } from "../comms/receiving-types.js";
|
|
3
5
|
import { waitFieldsReason } from "../actors/computer-use/wait.js";
|
|
4
6
|
import { PARTICIPANT_ID_MAX_CHARS, PARTICIPANT_ID_PATTERN } from "./parse/actors.js";
|
|
@@ -288,6 +290,27 @@ export function outputTokenLimitValidationReason(config) {
|
|
|
288
290
|
}
|
|
289
291
|
return null;
|
|
290
292
|
}
|
|
293
|
+
/**
|
|
294
|
+
* Refuse a Fast mode request humanish cannot send or price: the first-party OpenAI computer-use
|
|
295
|
+
* route sends it, and the spend estimate needs the model's Fast mode rate.
|
|
296
|
+
*/
|
|
297
|
+
export function serviceTierValidationReason(config) {
|
|
298
|
+
const actor = config.actor;
|
|
299
|
+
if (actor?.serviceTier === undefined)
|
|
300
|
+
return null;
|
|
301
|
+
if (actor.type !== "openai-computer-use" ||
|
|
302
|
+
!isComputerUseComposition(config) ||
|
|
303
|
+
config.subject.source === "local-app")
|
|
304
|
+
return "actor.serviceTier is supported only by first-party OpenAI computer-use routes; terminal, local-agent, scripted and custom in-process routes do not send it.";
|
|
305
|
+
const model = actor.model ?? DEFAULT_OPENAI_CU_MODEL;
|
|
306
|
+
if (participantRate(model)?.fast === undefined) {
|
|
307
|
+
const priced = Object.keys(MODEL_RATES)
|
|
308
|
+
.filter((id) => participantRate(id)?.fast !== undefined)
|
|
309
|
+
.sort();
|
|
310
|
+
return `actor.serviceTier: fast needs a Fast mode rate for model "${model}", and humanish has none, so it could not price the run. Models with a Fast mode rate: ${priced.join(", ")}. Use one of them, or remove serviceTier.`;
|
|
311
|
+
}
|
|
312
|
+
return null;
|
|
313
|
+
}
|
|
291
314
|
/**
|
|
292
315
|
* Refuse a wait field out of range, or an idle wait longer than the longest wait: the actor's, and
|
|
293
316
|
* each participant's own idleWaitMs under actor.maxWaitMs. On a route whose participants take no
|
package/dist/verify/artifacts.js
CHANGED
|
@@ -192,7 +192,8 @@ export function streamScreenshotPaths(bundle) {
|
|
|
192
192
|
return paths;
|
|
193
193
|
}
|
|
194
194
|
function isHarnessScreenshotPath(normalized) {
|
|
195
|
-
|
|
195
|
+
const lower = normalized.toLowerCase();
|
|
196
|
+
return (normalized.startsWith("screenshots/") && (lower.endsWith(".png") || lower.endsWith(".webp")));
|
|
196
197
|
}
|
|
197
198
|
function isRunRootEvidenceReference(value) {
|
|
198
199
|
return (typeof value === "string" &&
|
|
@@ -1092,11 +1092,23 @@ an unclaimed frame on a raw, silent or live-only trace, and `ocr_scrubbed`, whic
|
|
|
1092
1092
|
the trace contract reserves but no writer produces. An aggregate raw declaration
|
|
1093
1093
|
wins over every frame claim.
|
|
1094
1094
|
|
|
1095
|
-
A screenshot is a
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1095
|
+
A screenshot is a PNG or, at a path ending `.webp`, a lossless WebP. A PNG
|
|
1096
|
+
screenshot is non-interlaced, its first chunk is a 13-byte IHDR, its IEND is
|
|
1097
|
+
empty, and it holds image data only. A chunk outside IHDR, PLTE, IDAT, IEND,
|
|
1098
|
+
tRNS, cHRM, gAMA, sBIT, sRGB, pHYs and bKGD (text chunks, ICC profiles, Exif,
|
|
1099
|
+
timestamps, private chunks) fails verification. The harness writers drop those
|
|
1100
|
+
chunks before writing a frame. A WebP screenshot is the simple lossless form: a
|
|
1101
|
+
RIFF container holding one VP8L chunk that ends the file, with version 0 and
|
|
1102
|
+
dimensions under the PNG pixel cap. The extended form (VP8X), which carries Exif,
|
|
1103
|
+
XMP, ICC profiles and animation, fails verification. Verify reads a WebP's
|
|
1104
|
+
structure without decoding its pixels, so a WebP frame is always full-fidelity
|
|
1105
|
+
evidence: only a PNG in the redactor's shape counts as blurred. Earlier humanish
|
|
1106
|
+
versions do not accept WebP screenshots. A computer-use participant writes each
|
|
1107
|
+
frame as PNG during its session. When the session ends, its trace references
|
|
1108
|
+
the lossless WebP of each frame encoded by then in place of its PNG, and the run
|
|
1109
|
+
removes those PNGs once its final bundle is written. A study that blurs its
|
|
1110
|
+
screenshots keeps PNG frames, and an interrupted run can hold both files of a
|
|
1111
|
+
frame, with run.json's live trace naming the PNG.
|
|
1100
1112
|
|
|
1101
1113
|
The public-safety scan classifies a run file by its bytes, not its name. It scans
|
|
1102
1114
|
a file for secret and path patterns only when the bytes are strict UTF-8 with no
|
|
@@ -1175,7 +1187,8 @@ Its `derivation.json` uses schema `humanish.redacted-derivation.v1` and records:
|
|
|
1175
1187
|
rows, sorted by path using JavaScript string order;
|
|
1176
1188
|
- `files[]`: source-relative `path`, `sourceSha256`, action (`copied`, `updated`,
|
|
1177
1189
|
`blurred`, or `omitted`), and `outputSha256` for retained files or a reason for
|
|
1178
|
-
omission;
|
|
1190
|
+
omission; `outputPath` when the copy holds the file at another path, as it does a
|
|
1191
|
+
WebP frame;
|
|
1179
1192
|
- `generated[]`: paths and SHA256s of rebuilt Observer projections.
|
|
1180
1193
|
|
|
1181
1194
|
The receipt describes the transformation; it does not attest that a participant's
|
|
@@ -1195,11 +1208,16 @@ An id that appears only in free text, in a run with no receipts, is not known to
|
|
|
1195
1208
|
export. `sandbox-receipts.ndjson` is omitted, and the source keeps its raw ids,
|
|
1196
1209
|
which reclaim reads.
|
|
1197
1210
|
|
|
1198
|
-
PNG files that actor traces reference as stream screenshots are
|
|
1199
|
-
blurred thumbnails. A
|
|
1200
|
-
|
|
1201
|
-
|
|
1202
|
-
|
|
1211
|
+
PNG and WebP files that actor traces reference as stream screenshots are
|
|
1212
|
+
re-encoded as blurred PNG thumbnails. A WebP frame's thumbnail is written at the
|
|
1213
|
+
same path ending `.png`, and the copy's actor traces, stream artifacts, feedback
|
|
1214
|
+
evidence and adapter artifacts name that path. Export refuses a run that already
|
|
1215
|
+
holds a file at that path, and a run whose other copied text names the WebP frame,
|
|
1216
|
+
since the copy could not resolve that reference. An image that nothing in
|
|
1217
|
+
`run.json` cites is omitted and inventoried. Export refuses a run whose feedback
|
|
1218
|
+
candidates, adapter artifacts or stream artifact entries cite an image that is not
|
|
1219
|
+
a stream screenshot, since verify cannot read that image and dropping it would
|
|
1220
|
+
change the evidence.
|
|
1203
1221
|
Known actor screenshot declarations describe export-time blur while retaining the
|
|
1204
1222
|
original redaction notes. Observer is rebuilt; old feedback outputs, local process
|
|
1205
1223
|
status and the operational sandbox journal are omitted and inventoried. A derivative
|
|
@@ -76,17 +76,17 @@ map each one to its v3 key.
|
|
|
76
76
|
A study declares its route and composes code primitives; it is not a hardcoded kind. The
|
|
77
77
|
top-level keys:
|
|
78
78
|
|
|
79
|
-
| Key | Meaning
|
|
80
|
-
| ----------------------------------------------------------------- |
|
|
81
|
-
| `schema`, `id`, `title`, `description` | the document's identity
|
|
82
|
-
| `route` | required: `preview`, `computer-use`, `shared-world`, `terminal` or `scripted`
|
|
83
|
-
| `mode` | `dry-run` (the default) or `live`
|
|
84
|
-
| `actor` | who drives it, one object: `type`, `model`, `localAgent`, `persona`, `mission`, `tasks`, `maxOutputTokens`, `maxWaitMs`, `idleWaitMs`, `reasoningEffort`, `stopWhen`, `dwell` |
|
|
85
|
-
| `participants` | a count, a homogeneous `{ count, instruction }`, or a list of entries (below)
|
|
86
|
-
| `surfaces` | scripted only: `[desktop]` or `[desktop, mobile]`
|
|
87
|
-
| `caps` | the route's budget (below)
|
|
88
|
-
| `scenario` | scripted only: a committed scenario id or path
|
|
89
|
-
| `subject`, `execution`, `policies`, `review`, `defaults`, `comms` | as below
|
|
79
|
+
| Key | Meaning |
|
|
80
|
+
| ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
81
|
+
| `schema`, `id`, `title`, `description` | the document's identity |
|
|
82
|
+
| `route` | required: `preview`, `computer-use`, `shared-world`, `terminal` or `scripted` |
|
|
83
|
+
| `mode` | `dry-run` (the default) or `live` |
|
|
84
|
+
| `actor` | who drives it, one object: `type`, `model`, `localAgent`, `persona`, `mission`, `tasks`, `maxOutputTokens`, `maxWaitMs`, `idleWaitMs`, `reasoningEffort`, `serviceTier`, `stopWhen`, `dwell` |
|
|
85
|
+
| `participants` | a count, a homogeneous `{ count, instruction }`, or a list of entries (below) |
|
|
86
|
+
| `surfaces` | scripted only: `[desktop]` or `[desktop, mobile]` |
|
|
87
|
+
| `caps` | the route's budget (below) |
|
|
88
|
+
| `scenario` | scripted only: a committed scenario id or path |
|
|
89
|
+
| `subject`, `execution`, `policies`, `review`, `defaults`, `comms` | as below |
|
|
90
90
|
|
|
91
91
|
The parser reads only the declared route's keys. A key that route never reads is an error that
|
|
92
92
|
names the route, such as "route: computer-use does not read scenario. Remove it.", and an
|
|
@@ -1148,6 +1148,14 @@ Core-owned fields:
|
|
|
1148
1148
|
is part of who the participant was rather than of how the instrument was tuned
|
|
1149
1149
|
(docs/principles/actor-fidelity.md), so a trace without it is a result missing
|
|
1150
1150
|
half its sample description
|
|
1151
|
+
- optional `modelSettings.serviceTier`: `fast` when the study declared
|
|
1152
|
+
`actor.serviceTier: fast`, so every first-party OpenAI CUA request carried
|
|
1153
|
+
`service_tier: "fast"` (OpenAI Fast mode). Absent when undeclared. Each
|
|
1154
|
+
`tokenUsage.turns[]` record of such a trace has `serviceTier`, the tier the
|
|
1155
|
+
response says served it: `priority` or `fast` bill at the model's Fast mode rate
|
|
1156
|
+
in `src/run/pricing.ts`, and `default`, which OpenAI serves when Fast mode
|
|
1157
|
+
traffic ramps too fast, bills at the standard rate. `estimatedCost.breakdown.fastTurns`
|
|
1158
|
+
counts the requests priced at the fast rate.
|
|
1151
1159
|
- optional `modelSettings.maxOutputTokens`: the declared positive integer
|
|
1152
1160
|
`actor.maxOutputTokens`, passed to every first-party OpenAI CUA response
|
|
1153
1161
|
request, including reasoning output. Absent when undeclared. The first request
|
|
@@ -314,8 +314,8 @@ stopped, an operator can remove the empty lock directory and retry.
|
|
|
314
314
|
|
|
315
315
|
## Evidence and findings
|
|
316
316
|
|
|
317
|
-
Each request's packet admits up to 16 participants, 800 evidence items, 40
|
|
318
|
-
captures, 160 KiB of text and 20 MiB of images. Individual source files, image
|
|
317
|
+
Each request's packet admits up to 16 participants, 800 evidence items, 40
|
|
318
|
+
captures (PNG or lossless WebP), 160 KiB of text and 20 MiB of images. Individual source files, image
|
|
319
319
|
dimensions and result sizes have separate limits. Analysis reads run.json up to
|
|
320
320
|
16 MiB and each participant's trace up to 32 MiB, as verify and the Observer do.
|
|
321
321
|
It reads the traces one at a time and keeps of each only what selection can
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "humanish",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.147.0",
|
|
4
4
|
"description": "Synthetic user research for apps, CLIs, and agent-facing product flows.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"cli",
|
|
@@ -105,6 +105,7 @@
|
|
|
105
105
|
"site:hero:proof": "node scripts/site-hero-wrap-proof.mjs"
|
|
106
106
|
},
|
|
107
107
|
"dependencies": {
|
|
108
|
+
"@jsquash/webp": "1.5.0",
|
|
108
109
|
"commander": "^15.0.0",
|
|
109
110
|
"parse5": "^8.0.1",
|
|
110
111
|
"playwright-core": "^1.63.0",
|