@e2edev/e2e 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +104 -0
- package/dist/agent/act-validation.d.ts +27 -0
- package/dist/agent/act-validation.d.ts.map +1 -0
- package/dist/agent/act-validation.js +121 -0
- package/dist/agent/act-validation.js.map +1 -0
- package/dist/agent/act.d.ts +27 -0
- package/dist/agent/act.d.ts.map +1 -0
- package/dist/agent/act.js +775 -0
- package/dist/agent/act.js.map +1 -0
- package/dist/agent/actions.d.ts +66 -0
- package/dist/agent/actions.d.ts.map +1 -0
- package/dist/agent/actions.js +82 -0
- package/dist/agent/actions.js.map +1 -0
- package/dist/agent/ai-sdk.d.ts +16 -0
- package/dist/agent/ai-sdk.d.ts.map +1 -0
- package/dist/agent/ai-sdk.js +59 -0
- package/dist/agent/ai-sdk.js.map +1 -0
- package/dist/agent/call-options.d.ts +10 -0
- package/dist/agent/call-options.d.ts.map +1 -0
- package/dist/agent/call-options.js +28 -0
- package/dist/agent/call-options.js.map +1 -0
- package/dist/agent/default-agent.d.ts +22 -0
- package/dist/agent/default-agent.d.ts.map +1 -0
- package/dist/agent/default-agent.js +354 -0
- package/dist/agent/default-agent.js.map +1 -0
- package/dist/agent/error.d.ts +53 -0
- package/dist/agent/error.d.ts.map +1 -0
- package/dist/agent/error.js +94 -0
- package/dist/agent/error.js.map +1 -0
- package/dist/agent/executor.d.ts +238 -0
- package/dist/agent/executor.d.ts.map +1 -0
- package/dist/agent/executor.js +56 -0
- package/dist/agent/executor.js.map +1 -0
- package/dist/agent/index.d.ts +15 -0
- package/dist/agent/index.d.ts.map +1 -0
- package/dist/agent/index.js +304 -0
- package/dist/agent/index.js.map +1 -0
- package/dist/agent/invocation.d.ts +194 -0
- package/dist/agent/invocation.d.ts.map +1 -0
- package/dist/agent/invocation.js +389 -0
- package/dist/agent/invocation.js.map +1 -0
- package/dist/agent/ledger.d.ts +24 -0
- package/dist/agent/ledger.d.ts.map +1 -0
- package/dist/agent/ledger.js +48 -0
- package/dist/agent/ledger.js.map +1 -0
- package/dist/agent/loop-guards.d.ts +35 -0
- package/dist/agent/loop-guards.d.ts.map +1 -0
- package/dist/agent/loop-guards.js +110 -0
- package/dist/agent/loop-guards.js.map +1 -0
- package/dist/agent/model/adapter.d.ts +97 -0
- package/dist/agent/model/adapter.d.ts.map +1 -0
- package/dist/agent/model/adapter.js +40 -0
- package/dist/agent/model/adapter.js.map +1 -0
- package/dist/agent/model/router.d.ts +36 -0
- package/dist/agent/model/router.d.ts.map +1 -0
- package/dist/agent/model/router.js +45 -0
- package/dist/agent/model/router.js.map +1 -0
- package/dist/agent/model/schema.d.ts +16 -0
- package/dist/agent/model/schema.d.ts.map +1 -0
- package/dist/agent/model/schema.js +37 -0
- package/dist/agent/model/schema.js.map +1 -0
- package/dist/agent/model/sdk.d.ts +28 -0
- package/dist/agent/model/sdk.d.ts.map +1 -0
- package/dist/agent/model/sdk.js +258 -0
- package/dist/agent/model/sdk.js.map +1 -0
- package/dist/agent/observation.d.ts +70 -0
- package/dist/agent/observation.d.ts.map +1 -0
- package/dist/agent/observation.js +174 -0
- package/dist/agent/observation.js.map +1 -0
- package/dist/agent/phases.d.ts +71 -0
- package/dist/agent/phases.d.ts.map +1 -0
- package/dist/agent/phases.js +124 -0
- package/dist/agent/phases.js.map +1 -0
- package/dist/agent/prompts.d.ts +40 -0
- package/dist/agent/prompts.d.ts.map +1 -0
- package/dist/agent/prompts.js +104 -0
- package/dist/agent/prompts.js.map +1 -0
- package/dist/agent/protocol.d.ts +31 -0
- package/dist/agent/protocol.d.ts.map +1 -0
- package/dist/agent/protocol.js +67 -0
- package/dist/agent/protocol.js.map +1 -0
- package/dist/agent/public.d.ts +12 -0
- package/dist/agent/public.d.ts.map +1 -0
- package/dist/agent/public.js +12 -0
- package/dist/agent/public.js.map +1 -0
- package/dist/agent/replay.d.ts +61 -0
- package/dist/agent/replay.d.ts.map +1 -0
- package/dist/agent/replay.js +185 -0
- package/dist/agent/replay.js.map +1 -0
- package/dist/agent/secrets.d.ts +22 -0
- package/dist/agent/secrets.d.ts.map +1 -0
- package/dist/agent/secrets.js +63 -0
- package/dist/agent/secrets.js.map +1 -0
- package/dist/agent/step-cache.d.ts +156 -0
- package/dist/agent/step-cache.d.ts.map +1 -0
- package/dist/agent/step-cache.js +301 -0
- package/dist/agent/step-cache.js.map +1 -0
- package/dist/agent/tool-loop.d.ts +51 -0
- package/dist/agent/tool-loop.d.ts.map +1 -0
- package/dist/agent/tool-loop.js +331 -0
- package/dist/agent/tool-loop.js.map +1 -0
- package/dist/agent/tool.d.ts +39 -0
- package/dist/agent/tool.d.ts.map +1 -0
- package/dist/agent/tool.js +52 -0
- package/dist/agent/tool.js.map +1 -0
- package/dist/backend/contract.d.ts +186 -0
- package/dist/backend/contract.d.ts.map +1 -0
- package/dist/backend/contract.js +65 -0
- package/dist/backend/contract.js.map +1 -0
- package/dist/backend/index.d.ts +316 -0
- package/dist/backend/index.d.ts.map +1 -0
- package/dist/backend/index.js +219 -0
- package/dist/backend/index.js.map +1 -0
- package/dist/backend/session.d.ts +23 -0
- package/dist/backend/session.d.ts.map +1 -0
- package/dist/backend/session.js +213 -0
- package/dist/backend/session.js.map +1 -0
- package/dist/backend/surface.d.ts +83 -0
- package/dist/backend/surface.d.ts.map +1 -0
- package/dist/backend/surface.js +11 -0
- package/dist/backend/surface.js.map +1 -0
- package/dist/cache/anchors.d.ts +37 -0
- package/dist/cache/anchors.d.ts.map +1 -0
- package/dist/cache/anchors.js +89 -0
- package/dist/cache/anchors.js.map +1 -0
- package/dist/cache/context.d.ts +74 -0
- package/dist/cache/context.d.ts.map +1 -0
- package/dist/cache/context.js +82 -0
- package/dist/cache/context.js.map +1 -0
- package/dist/cache/decide.d.ts +42 -0
- package/dist/cache/decide.d.ts.map +1 -0
- package/dist/cache/decide.js +47 -0
- package/dist/cache/decide.js.map +1 -0
- package/dist/cache/identity.d.ts +113 -0
- package/dist/cache/identity.d.ts.map +1 -0
- package/dist/cache/identity.js +109 -0
- package/dist/cache/identity.js.map +1 -0
- package/dist/cache/recorder.d.ts +76 -0
- package/dist/cache/recorder.d.ts.map +1 -0
- package/dist/cache/recorder.js +143 -0
- package/dist/cache/recorder.js.map +1 -0
- package/dist/cache/relocate.d.ts +69 -0
- package/dist/cache/relocate.d.ts.map +1 -0
- package/dist/cache/relocate.js +125 -0
- package/dist/cache/relocate.js.map +1 -0
- package/dist/cache/store.d.ts +68 -0
- package/dist/cache/store.d.ts.map +1 -0
- package/dist/cache/store.js +106 -0
- package/dist/cache/store.js.map +1 -0
- package/dist/cache/trace.d.ts +146 -0
- package/dist/cache/trace.d.ts.map +1 -0
- package/dist/cache/trace.js +249 -0
- package/dist/cache/trace.js.map +1 -0
- package/dist/cli/bin.d.ts +3 -0
- package/dist/cli/bin.d.ts.map +1 -0
- package/dist/cli/bin.js +4 -0
- package/dist/cli/bin.js.map +1 -0
- package/dist/cli/index.d.ts +4 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +108 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/init.d.ts +6 -0
- package/dist/cli/init.d.ts.map +1 -0
- package/dist/cli/init.js +101 -0
- package/dist/cli/init.js.map +1 -0
- package/dist/collect/collect.d.ts +44 -0
- package/dist/collect/collect.d.ts.map +1 -0
- package/dist/collect/collect.js +89 -0
- package/dist/collect/collect.js.map +1 -0
- package/dist/collect/registry.d.ts +47 -0
- package/dist/collect/registry.d.ts.map +1 -0
- package/dist/collect/registry.js +262 -0
- package/dist/collect/registry.js.map +1 -0
- package/dist/collect/select.d.ts +54 -0
- package/dist/collect/select.d.ts.map +1 -0
- package/dist/collect/select.js +230 -0
- package/dist/collect/select.js.map +1 -0
- package/dist/config/agent.d.ts +91 -0
- package/dist/config/agent.d.ts.map +1 -0
- package/dist/config/agent.js +232 -0
- package/dist/config/agent.js.map +1 -0
- package/dist/config/load.d.ts +23 -0
- package/dist/config/load.d.ts.map +1 -0
- package/dist/config/load.js +75 -0
- package/dist/config/load.js.map +1 -0
- package/dist/config/resolve.d.ts +94 -0
- package/dist/config/resolve.d.ts.map +1 -0
- package/dist/config/resolve.js +500 -0
- package/dist/config/resolve.js.map +1 -0
- package/dist/config/validate.d.ts +6 -0
- package/dist/config/validate.d.ts.map +1 -0
- package/dist/config/validate.js +21 -0
- package/dist/config/validate.js.map +1 -0
- package/dist/credentials.d.ts +7 -0
- package/dist/credentials.d.ts.map +1 -0
- package/dist/credentials.js +42 -0
- package/dist/credentials.js.map +1 -0
- package/dist/expect/async.d.ts +5 -0
- package/dist/expect/async.d.ts.map +1 -0
- package/dist/expect/async.js +186 -0
- package/dist/expect/async.js.map +1 -0
- package/dist/expect/index.d.ts +7 -0
- package/dist/expect/index.d.ts.map +1 -0
- package/dist/expect/index.js +23 -0
- package/dist/expect/index.js.map +1 -0
- package/dist/expect/values.d.ts +4 -0
- package/dist/expect/values.d.ts.map +1 -0
- package/dist/expect/values.js +108 -0
- package/dist/expect/values.js.map +1 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +15 -0
- package/dist/index.js.map +1 -0
- package/dist/internal/ai-trace.d.ts +151 -0
- package/dist/internal/ai-trace.d.ts.map +1 -0
- package/dist/internal/ai-trace.js +360 -0
- package/dist/internal/ai-trace.js.map +1 -0
- package/dist/internal/brands.d.ts +9 -0
- package/dist/internal/brands.d.ts.map +1 -0
- package/dist/internal/brands.js +9 -0
- package/dist/internal/brands.js.map +1 -0
- package/dist/internal/debug.d.ts +43 -0
- package/dist/internal/debug.d.ts.map +1 -0
- package/dist/internal/debug.js +102 -0
- package/dist/internal/debug.js.map +1 -0
- package/dist/internal/errors.d.ts +98 -0
- package/dist/internal/errors.d.ts.map +1 -0
- package/dist/internal/errors.js +212 -0
- package/dist/internal/errors.js.map +1 -0
- package/dist/internal/globs.d.ts +29 -0
- package/dist/internal/globs.d.ts.map +1 -0
- package/dist/internal/globs.js +126 -0
- package/dist/internal/globs.js.map +1 -0
- package/dist/internal/ids.d.ts +32 -0
- package/dist/internal/ids.d.ts.map +1 -0
- package/dist/internal/ids.js +124 -0
- package/dist/internal/ids.js.map +1 -0
- package/dist/internal/json-value.d.ts +15 -0
- package/dist/internal/json-value.d.ts.map +1 -0
- package/dist/internal/json-value.js +61 -0
- package/dist/internal/json-value.js.map +1 -0
- package/dist/internal/package-version.d.ts +6 -0
- package/dist/internal/package-version.d.ts.map +1 -0
- package/dist/internal/package-version.js +14 -0
- package/dist/internal/package-version.js.map +1 -0
- package/dist/internal/realm-slot.d.ts +13 -0
- package/dist/internal/realm-slot.d.ts.map +1 -0
- package/dist/internal/realm-slot.js +23 -0
- package/dist/internal/realm-slot.js.map +1 -0
- package/dist/internal/redact.d.ts +33 -0
- package/dist/internal/redact.d.ts.map +1 -0
- package/dist/internal/redact.js +56 -0
- package/dist/internal/redact.js.map +1 -0
- package/dist/internal/regexp.d.ts +10 -0
- package/dist/internal/regexp.d.ts.map +1 -0
- package/dist/internal/regexp.js +27 -0
- package/dist/internal/regexp.js.map +1 -0
- package/dist/internal/text.d.ts +39 -0
- package/dist/internal/text.d.ts.map +1 -0
- package/dist/internal/text.js +66 -0
- package/dist/internal/text.js.map +1 -0
- package/dist/internal/time.d.ts +37 -0
- package/dist/internal/time.d.ts.map +1 -0
- package/dist/internal/time.js +91 -0
- package/dist/internal/time.js.map +1 -0
- package/dist/internal/trace.d.ts +18 -0
- package/dist/internal/trace.d.ts.map +1 -0
- package/dist/internal/trace.js +46 -0
- package/dist/internal/trace.js.map +1 -0
- package/dist/internal/urls.d.ts +31 -0
- package/dist/internal/urls.d.ts.map +1 -0
- package/dist/internal/urls.js +95 -0
- package/dist/internal/urls.js.map +1 -0
- package/dist/locator/engine.d.ts +65 -0
- package/dist/locator/engine.d.ts.map +1 -0
- package/dist/locator/engine.js +164 -0
- package/dist/locator/engine.js.map +1 -0
- package/dist/locator/expression.d.ts +19 -0
- package/dist/locator/expression.d.ts.map +1 -0
- package/dist/locator/expression.js +85 -0
- package/dist/locator/expression.js.map +1 -0
- package/dist/locator/screen.d.ts +36 -0
- package/dist/locator/screen.d.ts.map +1 -0
- package/dist/locator/screen.js +281 -0
- package/dist/locator/screen.js.map +1 -0
- package/dist/report/build.d.ts +214 -0
- package/dist/report/build.d.ts.map +1 -0
- package/dist/report/build.js +346 -0
- package/dist/report/build.js.map +1 -0
- package/dist/report/code-frame.d.ts +20 -0
- package/dist/report/code-frame.d.ts.map +1 -0
- package/dist/report/code-frame.js +73 -0
- package/dist/report/code-frame.js.map +1 -0
- package/dist/report/debug-steps.d.ts +14 -0
- package/dist/report/debug-steps.d.ts.map +1 -0
- package/dist/report/debug-steps.js +85 -0
- package/dist/report/debug-steps.js.map +1 -0
- package/dist/report/list.d.ts +80 -0
- package/dist/report/list.d.ts.map +1 -0
- package/dist/report/list.js +372 -0
- package/dist/report/list.js.map +1 -0
- package/dist/report/live-status.d.ts +51 -0
- package/dist/report/live-status.d.ts.map +1 -0
- package/dist/report/live-status.js +162 -0
- package/dist/report/live-status.js.map +1 -0
- package/dist/report/write.d.ts +4 -0
- package/dist/report/write.d.ts.map +1 -0
- package/dist/report/write.js +13 -0
- package/dist/report/write.js.map +1 -0
- package/dist/run/app-process.d.ts +18 -0
- package/dist/run/app-process.d.ts.map +1 -0
- package/dist/run/app-process.js +112 -0
- package/dist/run/app-process.js.map +1 -0
- package/dist/run/artifacts.d.ts +51 -0
- package/dist/run/artifacts.d.ts.map +1 -0
- package/dist/run/artifacts.js +114 -0
- package/dist/run/artifacts.js.map +1 -0
- package/dist/run/events.d.ts +100 -0
- package/dist/run/events.d.ts.map +1 -0
- package/dist/run/events.js +59 -0
- package/dist/run/events.js.map +1 -0
- package/dist/run/execute.d.ts +119 -0
- package/dist/run/execute.d.ts.map +1 -0
- package/dist/run/execute.js +555 -0
- package/dist/run/execute.js.map +1 -0
- package/dist/run/fixtures.d.ts +40 -0
- package/dist/run/fixtures.d.ts.map +1 -0
- package/dist/run/fixtures.js +408 -0
- package/dist/run/fixtures.js.map +1 -0
- package/dist/run/in-process.d.ts +24 -0
- package/dist/run/in-process.d.ts.map +1 -0
- package/dist/run/in-process.js +120 -0
- package/dist/run/in-process.js.map +1 -0
- package/dist/run/public.d.ts +19 -0
- package/dist/run/public.d.ts.map +1 -0
- package/dist/run/public.js +11 -0
- package/dist/run/public.js.map +1 -0
- package/dist/run/realm.d.ts +52 -0
- package/dist/run/realm.d.ts.map +1 -0
- package/dist/run/realm.js +101 -0
- package/dist/run/realm.js.map +1 -0
- package/dist/run/records.d.ts +87 -0
- package/dist/run/records.d.ts.map +1 -0
- package/dist/run/records.js +3 -0
- package/dist/run/records.js.map +1 -0
- package/dist/run/retry.d.ts +18 -0
- package/dist/run/retry.d.ts.map +1 -0
- package/dist/run/retry.js +33 -0
- package/dist/run/retry.js.map +1 -0
- package/dist/run/runner.d.ts +49 -0
- package/dist/run/runner.d.ts.map +1 -0
- package/dist/run/runner.js +375 -0
- package/dist/run/runner.js.map +1 -0
- package/dist/run/scheduler.d.ts +45 -0
- package/dist/run/scheduler.d.ts.map +1 -0
- package/dist/run/scheduler.js +512 -0
- package/dist/run/scheduler.js.map +1 -0
- package/dist/run/serial.d.ts +52 -0
- package/dist/run/serial.d.ts.map +1 -0
- package/dist/run/serial.js +247 -0
- package/dist/run/serial.js.map +1 -0
- package/dist/run/sessions.d.ts +54 -0
- package/dist/run/sessions.d.ts.map +1 -0
- package/dist/run/sessions.js +221 -0
- package/dist/run/sessions.js.map +1 -0
- package/dist/run/steps.d.ts +196 -0
- package/dist/run/steps.d.ts.map +1 -0
- package/dist/run/steps.js +151 -0
- package/dist/run/steps.js.map +1 -0
- package/dist/run/unit-runner.d.ts +30 -0
- package/dist/run/unit-runner.d.ts.map +1 -0
- package/dist/run/unit-runner.js +11 -0
- package/dist/run/unit-runner.js.map +1 -0
- package/dist/run/units.d.ts +50 -0
- package/dist/run/units.d.ts.map +1 -0
- package/dist/run/units.js +103 -0
- package/dist/run/units.js.map +1 -0
- package/dist/run/worker/entry.d.ts +8 -0
- package/dist/run/worker/entry.d.ts.map +1 -0
- package/dist/run/worker/entry.js +137 -0
- package/dist/run/worker/entry.js.map +1 -0
- package/dist/run/worker/handle.d.ts +10 -0
- package/dist/run/worker/handle.d.ts.map +1 -0
- package/dist/run/worker/handle.js +74 -0
- package/dist/run/worker/handle.js.map +1 -0
- package/dist/run/worker/protocol.d.ts +129 -0
- package/dist/run/worker/protocol.d.ts.map +1 -0
- package/dist/run/worker/protocol.js +18 -0
- package/dist/run/worker/protocol.js.map +1 -0
- package/dist/run/worker/session.d.ts +69 -0
- package/dist/run/worker/session.d.ts.map +1 -0
- package/dist/run/worker/session.js +123 -0
- package/dist/run/worker/session.js.map +1 -0
- package/dist/types.d.ts +630 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +6 -0
- package/dist/types.js.map +1 -0
- package/package.json +91 -0
|
@@ -0,0 +1,775 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Harness-owned dispatch of one `agent.act()` step (RFC0001, layer 3).
|
|
3
|
+
*
|
|
4
|
+
* The harness opens the step, owns the deadline, the action budget, origin
|
|
5
|
+
* policy, observation redaction, and recording — then hands the step to the
|
|
6
|
+
* configured executor and maps its verdict back onto the runner's error
|
|
7
|
+
* taxonomy. The executor never touches the backend: everything bottoms out in
|
|
8
|
+
* the context built here, on the same accounting core (phases.ts) the
|
|
9
|
+
* locate/judgment tier runs on. Trace-cache participation — replay, live
|
|
10
|
+
* recording, staging — lives beside the dispatch in `StepTraceSession`.
|
|
11
|
+
*/
|
|
12
|
+
import { writeFileSync } from 'node:fs';
|
|
13
|
+
import { join } from 'node:path';
|
|
14
|
+
import { BackendError } from '../backend/surface.js';
|
|
15
|
+
import { ConfigurationError, TestError } from '../internal/errors.js';
|
|
16
|
+
import { timestamp } from '../internal/ids.js';
|
|
17
|
+
import { resolveNavigationUrl } from '../internal/urls.js';
|
|
18
|
+
import { AgentError, CATEGORY_BY_CODE, isAgentError, toAgentError } from './error.js';
|
|
19
|
+
import { rejectUnsupportedActOptions, validateInstruction, validateParams, validateVerdict, } from './act-validation.js';
|
|
20
|
+
import { resolveBoundedBudget, resolveTimeout } from './call-options.js';
|
|
21
|
+
import { RUNTIME_CODES, } from './executor.js';
|
|
22
|
+
import { serializeLedger } from './ledger.js';
|
|
23
|
+
import { instantiateLanguageModel } from './model/sdk.js';
|
|
24
|
+
import { observationShape, prepareObservation, settleObservation, } from './observation.js';
|
|
25
|
+
import { boundedOperation, checkStepClock, instrumentPhase, recordPolicyEvent, retryingObserve } from './phases.js';
|
|
26
|
+
import { describeAction } from './actions.js';
|
|
27
|
+
import { authorizeSecretFill } from './secrets.js';
|
|
28
|
+
import { StepTraceSession } from './step-cache.js';
|
|
29
|
+
/** Runs one `agent.act()` call as a harness-dispatched executor step. */
|
|
30
|
+
export async function runActStep(runtime, instruction, params, options) {
|
|
31
|
+
const normalized = validateInstruction(instruction, 'agent.act');
|
|
32
|
+
rejectUnsupportedActOptions(options);
|
|
33
|
+
const { projected, secrets } = validateParams(params);
|
|
34
|
+
await dispatchAgentStep(runtime, {
|
|
35
|
+
api: 'agent.act',
|
|
36
|
+
kind: 'act',
|
|
37
|
+
instruction: normalized,
|
|
38
|
+
params: projected,
|
|
39
|
+
secrets,
|
|
40
|
+
defaultFailureCode: 'ACTION_FAILED',
|
|
41
|
+
timeout: options?.timeout,
|
|
42
|
+
maxSteps: options?.maxSteps,
|
|
43
|
+
maxModelCalls: options?.maxModelCalls,
|
|
44
|
+
});
|
|
45
|
+
return { ok: true };
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Runs one `agent.assert()` call through the executor socket. Used when a
|
|
49
|
+
* custom executor is configured: the brain that plans flows also judges
|
|
50
|
+
* assertions, so swapping brains swaps all the thinking. The built-in
|
|
51
|
+
* single-judgment tier remains the default-path implementation.
|
|
52
|
+
*/
|
|
53
|
+
export async function runAssertStep(runtime, assertion, options) {
|
|
54
|
+
const normalized = validateInstruction(assertion, 'agent.assert');
|
|
55
|
+
const unsupported = (name) => {
|
|
56
|
+
throw new ConfigurationError('UNSUPPORTED_CAPABILITY', `agent.assert ${name} is not supported with a custom executor`);
|
|
57
|
+
};
|
|
58
|
+
if (options?.vision !== undefined)
|
|
59
|
+
unsupported('vision evidence (options.vision)');
|
|
60
|
+
if (options?.screenshot !== undefined)
|
|
61
|
+
unsupported('screenshot evidence (options.screenshot)');
|
|
62
|
+
await dispatchAgentStep(runtime, {
|
|
63
|
+
api: 'agent.assert',
|
|
64
|
+
kind: 'assert',
|
|
65
|
+
instruction: normalized,
|
|
66
|
+
params: undefined,
|
|
67
|
+
secrets: new Map(),
|
|
68
|
+
defaultFailureCode: 'ASSERTION_FAILED',
|
|
69
|
+
timeout: options?.timeout,
|
|
70
|
+
maxSteps: undefined,
|
|
71
|
+
maxModelCalls: undefined,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Opens the step, tries a cached replay, delegates to the executor when the
|
|
76
|
+
* cache misses or diverges, settles the verdict, and hands the outcome to the
|
|
77
|
+
* cache session, which owns the stage-or-evict decision
|
|
78
|
+
* (`StepTraceSession.conclude`). An assert is a verification step: its
|
|
79
|
+
* passing is what confirms the traces staged before it at attempt end
|
|
80
|
+
* (cache/context.ts).
|
|
81
|
+
*/
|
|
82
|
+
async function dispatchAgentStep(runtime, spec) {
|
|
83
|
+
await runtime.steps.run('agent', spec.api, spec.instruction, async () => {
|
|
84
|
+
const dispatch = new ActDispatch(runtime, spec);
|
|
85
|
+
try {
|
|
86
|
+
let verdict;
|
|
87
|
+
try {
|
|
88
|
+
verdict = await dispatch.run();
|
|
89
|
+
}
|
|
90
|
+
catch (cause) {
|
|
91
|
+
throw dispatch.settleThrown(toAgentError(cause));
|
|
92
|
+
}
|
|
93
|
+
dispatch.settle(validateVerdict(verdict, runtime.executor.name));
|
|
94
|
+
await dispatch.conclude('passed');
|
|
95
|
+
}
|
|
96
|
+
catch (cause) {
|
|
97
|
+
await dispatch.conclude(isAgentError(cause) && cause.code === 'CANCELLED' ? 'cancelled' : 'failed');
|
|
98
|
+
throw cause;
|
|
99
|
+
}
|
|
100
|
+
finally {
|
|
101
|
+
dispatch.finish();
|
|
102
|
+
}
|
|
103
|
+
}, { verifies: spec.kind === 'assert' });
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* One step's harness-side state: budgets, the newest observation, and the
|
|
107
|
+
* fatal error (if any) that must outrank whatever the executor reports.
|
|
108
|
+
*/
|
|
109
|
+
class ActDispatch {
|
|
110
|
+
runtime;
|
|
111
|
+
spec;
|
|
112
|
+
timeoutMs;
|
|
113
|
+
deadline;
|
|
114
|
+
maxActions;
|
|
115
|
+
maxModelCalls;
|
|
116
|
+
metrics = {
|
|
117
|
+
modelCalls: 0,
|
|
118
|
+
actionSteps: 0,
|
|
119
|
+
observationBytes: 0,
|
|
120
|
+
contextBytes: 0,
|
|
121
|
+
ledgerBytes: 0,
|
|
122
|
+
};
|
|
123
|
+
latest;
|
|
124
|
+
explanation;
|
|
125
|
+
/** First budget/timeout/cancel failure; runtime truth outranks the verdict. */
|
|
126
|
+
hardStop;
|
|
127
|
+
/** Aborts the executor on any hard stop, so a step never outlives its clock. */
|
|
128
|
+
stepAbort = new AbortController();
|
|
129
|
+
/**
|
|
130
|
+
* The one signal everything inside the step aborts with: the attempt's
|
|
131
|
+
* cancellation or the step's own hard stop. Built once in the constructor
|
|
132
|
+
* so the executor context, the replay host, and the settle clock can never
|
|
133
|
+
* disagree about what "the step's signal" means.
|
|
134
|
+
*/
|
|
135
|
+
stepSignal;
|
|
136
|
+
/**
|
|
137
|
+
* Serializes observations and actions in call order. An executor (or an AI
|
|
138
|
+
* SDK loop running parallel tool calls) that issues a second action before
|
|
139
|
+
* the first settles would otherwise resolve both targets against the same
|
|
140
|
+
* pre-action observation — exactly the wrong-node hazard the staleness rule
|
|
141
|
+
* exists to prevent. Queued, the second call sees the newest observation
|
|
142
|
+
* and a stale id fails loud instead of acting on the wrong node.
|
|
143
|
+
*
|
|
144
|
+
* Invariant: a serialized body must never call `observe()` or `runAction()`
|
|
145
|
+
* itself — the inner call would queue behind its own caller and deadlock.
|
|
146
|
+
* Grammar bodies call the session directly, and secret authorization
|
|
147
|
+
* never observes; keep it that way.
|
|
148
|
+
*/
|
|
149
|
+
grammarChain = Promise.resolve();
|
|
150
|
+
sdkModel;
|
|
151
|
+
sdkModelResolved = false;
|
|
152
|
+
transcript;
|
|
153
|
+
/** Set by `finish()`: late executor accounting must not land on the next step. */
|
|
154
|
+
closed = false;
|
|
155
|
+
inputTokens = 0;
|
|
156
|
+
outputTokens = 0;
|
|
157
|
+
estimatedCostUsd;
|
|
158
|
+
peakTokensPerCall = 0;
|
|
159
|
+
providerReportedUsage = false;
|
|
160
|
+
modelProvider;
|
|
161
|
+
modelId;
|
|
162
|
+
redact;
|
|
163
|
+
/** The step's trace-cache session; undefined when caching is off or the kind is not cacheable. */
|
|
164
|
+
stepCache;
|
|
165
|
+
constructor(runtime, spec) {
|
|
166
|
+
this.runtime = runtime;
|
|
167
|
+
this.spec = spec;
|
|
168
|
+
this.stepSignal = AbortSignal.any([runtime.signal, this.stepAbort.signal]);
|
|
169
|
+
this.timeoutMs = resolveTimeout(spec.timeout, runtime.config.timeout);
|
|
170
|
+
this.deadline = runtime.engine.deadline(this.timeoutMs);
|
|
171
|
+
this.maxActions = resolveBoundedBudget(spec.maxSteps, runtime.config.agent.maxSteps, 'maxSteps');
|
|
172
|
+
this.maxModelCalls = resolveBoundedBudget(spec.maxModelCalls, runtime.config.agent.maxModelCalls, 'maxModelCalls');
|
|
173
|
+
this.metrics.contextBytes = new TextEncoder().encode(runtime.agentContext ?? '').byteLength;
|
|
174
|
+
this.redact = runtime.redact;
|
|
175
|
+
// Only act steps are cacheable: an assert must not change state, so its
|
|
176
|
+
// trace would be empty — nothing to replay, nothing worth a read. The
|
|
177
|
+
// dispatch always runs inside a recorded step; a missing index would mean
|
|
178
|
+
// that invariant broke, and withholding the whole session is the safe
|
|
179
|
+
// answer.
|
|
180
|
+
const cache = spec.kind === 'act' ? runtime.cache : undefined;
|
|
181
|
+
const stepIndex = runtime.steps.currentStepIndex;
|
|
182
|
+
this.stepCache =
|
|
183
|
+
cache === undefined || stepIndex === undefined
|
|
184
|
+
? undefined
|
|
185
|
+
: new StepTraceSession(this.cacheHost(), {
|
|
186
|
+
cache,
|
|
187
|
+
instruction: spec.instruction,
|
|
188
|
+
params: spec.params,
|
|
189
|
+
executor: {
|
|
190
|
+
name: runtime.executor.name,
|
|
191
|
+
...(runtime.executor.version === undefined
|
|
192
|
+
? {}
|
|
193
|
+
: { version: runtime.executor.version }),
|
|
194
|
+
},
|
|
195
|
+
redact: this.redact,
|
|
196
|
+
testIdAttribute: runtime.config.testIdAttribute,
|
|
197
|
+
maxActions: this.maxActions,
|
|
198
|
+
stepIndex,
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
/** Builds the executor-facing context. */
|
|
202
|
+
context() {
|
|
203
|
+
// The `model` getter below runs with the context object as `this`.
|
|
204
|
+
// oxlint-disable-next-line typescript/no-this-alias
|
|
205
|
+
const dispatch = this;
|
|
206
|
+
const ledger = serializeLedger(this.runtime.priorSteps(), this.runtime.config.limits.maxLedgerBytes);
|
|
207
|
+
this.metrics.ledgerBytes = ledger.bytes;
|
|
208
|
+
const replayedPrefix = this.stepCache?.replayedPrefix;
|
|
209
|
+
return {
|
|
210
|
+
step: {
|
|
211
|
+
kind: this.spec.kind,
|
|
212
|
+
instruction: this.spec.instruction,
|
|
213
|
+
params: this.spec.params,
|
|
214
|
+
secrets: [...this.spec.secrets.values()].map((secret) => ({
|
|
215
|
+
name: secret.name,
|
|
216
|
+
purpose: secret.purpose,
|
|
217
|
+
})),
|
|
218
|
+
},
|
|
219
|
+
target: this.runtime.target,
|
|
220
|
+
...(replayedPrefix === undefined ? {} : { replayedPrefix }),
|
|
221
|
+
signal: this.stepSignal,
|
|
222
|
+
// Resolved on first read, so executors that bring their own model (or
|
|
223
|
+
// none) never pay for — or fail on — config model resolution.
|
|
224
|
+
get model() {
|
|
225
|
+
return dispatch.resolveModel();
|
|
226
|
+
},
|
|
227
|
+
ledger: ledger.text,
|
|
228
|
+
agentContext: this.runtime.agentContext,
|
|
229
|
+
budgets: {
|
|
230
|
+
maxActions: this.maxActions,
|
|
231
|
+
maxModelCalls: this.maxModelCalls,
|
|
232
|
+
actionsUsed: () => this.metrics.actionSteps,
|
|
233
|
+
remainingMs: () => this.deadline.remaining(),
|
|
234
|
+
recordModelCall: (usage) => this.recordModelCall(usage),
|
|
235
|
+
recordToolCall: (call) => this.recordToolCall(call),
|
|
236
|
+
},
|
|
237
|
+
observe: () => this.observe(),
|
|
238
|
+
attachTranscript: (text) => {
|
|
239
|
+
// Debug detail only: transcripts are model prose and can be large.
|
|
240
|
+
if (this.runtime.debug?.enabled === true && typeof text === 'string' && text !== '') {
|
|
241
|
+
this.transcript = text;
|
|
242
|
+
}
|
|
243
|
+
},
|
|
244
|
+
actions: this.buildActions(),
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
/**
|
|
248
|
+
* The action grammar, shared verbatim by the executor context and the
|
|
249
|
+
* replay engine: a replayed action runs under exactly the same deadline,
|
|
250
|
+
* budget, policy, and recording as a live one. Committed actions are
|
|
251
|
+
* recorded into the step trace with the node they actually acted on —
|
|
252
|
+
* capture at commit time is what makes the descriptor durable evidence
|
|
253
|
+
* rather than a guess.
|
|
254
|
+
*/
|
|
255
|
+
buildActions() {
|
|
256
|
+
return {
|
|
257
|
+
tap: (target) => this.commitTargeted('tap', target, async (node) => {
|
|
258
|
+
await this.session.perform(node.ref, { kind: 'tap' }, this.operation());
|
|
259
|
+
return { name: 'tap', node };
|
|
260
|
+
}),
|
|
261
|
+
type: (target, value) => {
|
|
262
|
+
if (typeof value !== 'string') {
|
|
263
|
+
throw new TestError('INVALID_ARGUMENT', 'type value must be a string');
|
|
264
|
+
}
|
|
265
|
+
return this.commitTargeted('type', target, async (node) => {
|
|
266
|
+
await this.session.perform(node.ref, { kind: 'fill', value, sensitive: false }, this.operation());
|
|
267
|
+
return { name: 'type', node, value };
|
|
268
|
+
});
|
|
269
|
+
},
|
|
270
|
+
typeSecret: (target, name) => this.typeSecret(target, name),
|
|
271
|
+
press: (target, key) => {
|
|
272
|
+
if (typeof key !== 'string' || key.trim() === '' || key.length > 64) {
|
|
273
|
+
throw new TestError('INVALID_ARGUMENT', 'press key must be a short non-empty string');
|
|
274
|
+
}
|
|
275
|
+
return this.commitTargeted('press', target, async (node) => {
|
|
276
|
+
await this.session.perform(node.ref, { kind: 'press', key }, this.operation());
|
|
277
|
+
return { name: 'press', node, key };
|
|
278
|
+
});
|
|
279
|
+
},
|
|
280
|
+
select: (target, value) => {
|
|
281
|
+
if (typeof value !== 'string' || value === '') {
|
|
282
|
+
throw new TestError('INVALID_ARGUMENT', 'select value must be a non-empty option label');
|
|
283
|
+
}
|
|
284
|
+
return this.commitTargeted('selectOption', target, async (node) => {
|
|
285
|
+
await this.session.perform(node.ref, { kind: 'selectOption', value }, this.operation());
|
|
286
|
+
return { name: 'select', node, value };
|
|
287
|
+
});
|
|
288
|
+
},
|
|
289
|
+
scroll: (direction, target) => this.scroll(direction, target),
|
|
290
|
+
navigate: (url) => this.navigate(url),
|
|
291
|
+
};
|
|
292
|
+
}
|
|
293
|
+
/**
|
|
294
|
+
* Races the step body — cached replay first, then the executor on a miss or
|
|
295
|
+
* divergence — against the step deadline. An executor that ignores every
|
|
296
|
+
* context call still cannot outlive the clock: the timer records the
|
|
297
|
+
* timeout as the hard stop, aborts the step signal, and settles the step —
|
|
298
|
+
* the promise is abandoned, never awaited past the deadline.
|
|
299
|
+
*/
|
|
300
|
+
async run() {
|
|
301
|
+
const timer = setTimeout(() => {
|
|
302
|
+
this.fatalize(new AgentError('STEP_TIMEOUT', `${this.spec.api} exceeded its ${this.timeoutMs} ms timeout`));
|
|
303
|
+
}, Math.max(1, this.deadline.remaining()));
|
|
304
|
+
try {
|
|
305
|
+
return await Promise.race([
|
|
306
|
+
this.dispatchStep(),
|
|
307
|
+
new Promise((_, reject) => {
|
|
308
|
+
const signal = this.stepAbort.signal;
|
|
309
|
+
if (signal.aborted) {
|
|
310
|
+
reject(signal.reason);
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
signal.addEventListener('abort', () => reject(signal.reason), { once: true });
|
|
314
|
+
}),
|
|
315
|
+
]);
|
|
316
|
+
}
|
|
317
|
+
finally {
|
|
318
|
+
clearTimeout(timer);
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
async dispatchStep() {
|
|
322
|
+
const replayed = await this.stepCache?.begin();
|
|
323
|
+
if (replayed !== undefined)
|
|
324
|
+
return replayed;
|
|
325
|
+
return this.runtime.executor.runStep(this.context());
|
|
326
|
+
}
|
|
327
|
+
/** The cache session's narrow view of this dispatch. */
|
|
328
|
+
cacheHost() {
|
|
329
|
+
return {
|
|
330
|
+
observe: async () => (await this.observeLatest()).nodes,
|
|
331
|
+
observeSettled: async () => (await this.observeSettled()).nodes,
|
|
332
|
+
actions: this.buildActions(),
|
|
333
|
+
signal: this.stepSignal,
|
|
334
|
+
remainingMs: () => this.deadline.remaining(),
|
|
335
|
+
redact: this.redact,
|
|
336
|
+
testIdAttribute: this.runtime.config.testIdAttribute,
|
|
337
|
+
currentPath: () => this.currentPath(),
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
/** Best-effort current location path + query, for trace preconditions. */
|
|
341
|
+
async currentPath() {
|
|
342
|
+
const currentUrl = this.session.url;
|
|
343
|
+
if (currentUrl === undefined)
|
|
344
|
+
return undefined;
|
|
345
|
+
try {
|
|
346
|
+
const url = new URL(await currentUrl(this.operation()));
|
|
347
|
+
return `${url.pathname}${url.search}`;
|
|
348
|
+
}
|
|
349
|
+
catch {
|
|
350
|
+
return undefined;
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
/** Hands the settled outcome to the cache session, which stages, evicts, or does nothing. */
|
|
354
|
+
async conclude(outcome) {
|
|
355
|
+
await this.stepCache?.conclude(outcome, this.explanation);
|
|
356
|
+
}
|
|
357
|
+
/** Maps the executor's verdict onto the runner outcome. Fail-closed on hard stops. */
|
|
358
|
+
settle(verdict) {
|
|
359
|
+
let settled = verdict;
|
|
360
|
+
if (this.hardStop !== undefined) {
|
|
361
|
+
if (this.hardStop.code === 'CANCELLED')
|
|
362
|
+
throw this.hardStop;
|
|
363
|
+
if (settled.status === 'passed') {
|
|
364
|
+
// The step ran out of budget or time mid-flight; an executor cannot
|
|
365
|
+
// declare success over the runtime's own accounting.
|
|
366
|
+
settled = {
|
|
367
|
+
status: 'blocked',
|
|
368
|
+
summary: this.hardStop.message,
|
|
369
|
+
errorCode: this.hardStop.code,
|
|
370
|
+
};
|
|
371
|
+
}
|
|
372
|
+
else if (settled.errorCode === undefined) {
|
|
373
|
+
// The executor's analysis stands, but runtime exhaustion is never
|
|
374
|
+
// hidden from the report: a code-less failure inherits the hard stop.
|
|
375
|
+
settled = { ...settled, errorCode: this.hardStop.code };
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
if (settled.errorCode !== undefined &&
|
|
379
|
+
RUNTIME_CODES.has(settled.errorCode) &&
|
|
380
|
+
!this.vouches(settled.errorCode)) {
|
|
381
|
+
throw this.invented(settled.errorCode);
|
|
382
|
+
}
|
|
383
|
+
this.explanation = settled.summary;
|
|
384
|
+
if (settled.status === 'passed')
|
|
385
|
+
return;
|
|
386
|
+
const code = settled.errorCode !== undefined && settled.errorCode in CATEGORY_BY_CODE
|
|
387
|
+
? settled.errorCode
|
|
388
|
+
: this.spec.defaultFailureCode;
|
|
389
|
+
throw new AgentError(code, `${this.spec.api} ${settled.status}: ${settled.summary}`, {
|
|
390
|
+
blocked: settled.status === 'blocked',
|
|
391
|
+
});
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Classifies an executor throw. The recorded hard stop wins over any
|
|
395
|
+
* derived failure, and a runtime code the runtime cannot corroborate is an
|
|
396
|
+
* invalid verdict, not a runtime failure.
|
|
397
|
+
*/
|
|
398
|
+
settleThrown(error) {
|
|
399
|
+
if (!RUNTIME_CODES.has(error.code))
|
|
400
|
+
return error;
|
|
401
|
+
if (this.hardStop !== undefined)
|
|
402
|
+
return this.hardStop;
|
|
403
|
+
if (this.vouches(error.code))
|
|
404
|
+
return error;
|
|
405
|
+
return this.invented(error.code, error);
|
|
406
|
+
}
|
|
407
|
+
/**
|
|
408
|
+
* Whether the runtime's own accounting corroborates a runtime code. Codes
|
|
409
|
+
* are runtime-assigned: the recorded hard stop vouches directly, and the
|
|
410
|
+
* clock, the budgets, and the abort signal vouch for an executor that
|
|
411
|
+
* observed exhaustion before the context machinery did.
|
|
412
|
+
*/
|
|
413
|
+
vouches(code) {
|
|
414
|
+
if (this.hardStop?.code === code)
|
|
415
|
+
return true;
|
|
416
|
+
switch (code) {
|
|
417
|
+
case 'CANCELLED':
|
|
418
|
+
return this.runtime.signal.aborted;
|
|
419
|
+
case 'STEP_TIMEOUT':
|
|
420
|
+
return this.deadline.expired();
|
|
421
|
+
case 'STEP_BUDGET_EXHAUSTED':
|
|
422
|
+
return (this.metrics.actionSteps >= this.maxActions ||
|
|
423
|
+
this.metrics.modelCalls >= this.maxModelCalls);
|
|
424
|
+
default:
|
|
425
|
+
return false;
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
invented(code, cause) {
|
|
429
|
+
return new AgentError('MODEL_OUTPUT_INVALID', `executor "${this.runtime.executor.name}" reported runtime code ${code}, which the runtime never assigned`, cause === undefined ? {} : { cause });
|
|
430
|
+
}
|
|
431
|
+
/**
|
|
432
|
+
* Records the first hard stop. Timeout and cancellation also abort the step
|
|
433
|
+
* signal — there is nothing left for the executor to say. Budget exhaustion
|
|
434
|
+
* does not: the executor may still catch it and conclude with its own
|
|
435
|
+
* analysis (which `settle` stamps with the runtime code), bounded by the
|
|
436
|
+
* deadline either way.
|
|
437
|
+
*/
|
|
438
|
+
fatalize(error) {
|
|
439
|
+
this.hardStop ??= error;
|
|
440
|
+
if (error.code !== 'STEP_BUDGET_EXHAUSTED' && !this.stepAbort.signal.aborted) {
|
|
441
|
+
this.stepAbort.abort(this.hardStop);
|
|
442
|
+
}
|
|
443
|
+
return error;
|
|
444
|
+
}
|
|
445
|
+
/**
|
|
446
|
+
* Counts and accounts one executor-made model call. The budget is enforced:
|
|
447
|
+
* the call past the limit records, then hard-stops the step.
|
|
448
|
+
*/
|
|
449
|
+
recordModelCall(usage) {
|
|
450
|
+
// An executor abandoned by a hard stop can still report late; the step it
|
|
451
|
+
// belonged to is closed, and the recorder's active step is now another.
|
|
452
|
+
if (this.closed)
|
|
453
|
+
return;
|
|
454
|
+
this.metrics.modelCalls += 1;
|
|
455
|
+
const inputTokens = usage?.inputTokens ?? 0;
|
|
456
|
+
const outputTokens = usage?.outputTokens ?? 0;
|
|
457
|
+
if (usage?.inputTokens !== undefined || usage?.outputTokens !== undefined) {
|
|
458
|
+
this.providerReportedUsage = true;
|
|
459
|
+
this.inputTokens += inputTokens;
|
|
460
|
+
this.outputTokens += outputTokens;
|
|
461
|
+
this.peakTokensPerCall = Math.max(this.peakTokensPerCall, inputTokens + outputTokens);
|
|
462
|
+
}
|
|
463
|
+
if (usage?.provider !== undefined)
|
|
464
|
+
this.modelProvider = usage.provider;
|
|
465
|
+
if (usage?.modelId !== undefined)
|
|
466
|
+
this.modelId = usage.modelId;
|
|
467
|
+
// Executors are trusted, but the report schema requires a finite,
|
|
468
|
+
// non-negative cost; a bogus value must not invalidate the whole report.
|
|
469
|
+
if (usage?.estimatedCostUsd !== undefined &&
|
|
470
|
+
Number.isFinite(usage.estimatedCostUsd) &&
|
|
471
|
+
usage.estimatedCostUsd >= 0) {
|
|
472
|
+
this.estimatedCostUsd = (this.estimatedCostUsd ?? 0) + usage.estimatedCostUsd;
|
|
473
|
+
}
|
|
474
|
+
this.runtime.steps.recordEvent({
|
|
475
|
+
kind: 'model',
|
|
476
|
+
startedAt: timestamp(),
|
|
477
|
+
durationMs: Math.max(0, Math.round(usage?.durationMs ?? 0)),
|
|
478
|
+
status: 'passed',
|
|
479
|
+
name: 'executor',
|
|
480
|
+
count: inputTokens + outputTokens,
|
|
481
|
+
});
|
|
482
|
+
this.runtime.debug?.record('agent.model', Math.max(0, Math.round(usage?.durationMs ?? 0)));
|
|
483
|
+
if (this.metrics.modelCalls > this.maxModelCalls) {
|
|
484
|
+
throw this.fatalize(new AgentError('STEP_BUDGET_EXHAUSTED', `${this.spec.api} exhausted its model-call budget of ${this.maxModelCalls}`));
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
/**
|
|
488
|
+
* Records one executor tool call that bypassed the grammar. Mutating tools
|
|
489
|
+
* consume an action-budget slot, so a project tool cannot spend past the
|
|
490
|
+
* ceiling the grammar enforces.
|
|
491
|
+
*/
|
|
492
|
+
recordToolCall(call) {
|
|
493
|
+
if (this.closed)
|
|
494
|
+
return;
|
|
495
|
+
this.checkpoint();
|
|
496
|
+
this.runtime.steps.recordEvent({
|
|
497
|
+
kind: 'backend',
|
|
498
|
+
startedAt: timestamp(),
|
|
499
|
+
durationMs: Math.max(0, Math.round(call.durationMs ?? 0)),
|
|
500
|
+
status: 'passed',
|
|
501
|
+
name: `tool:${call.name}`,
|
|
502
|
+
});
|
|
503
|
+
if (!call.mutates)
|
|
504
|
+
return;
|
|
505
|
+
this.metrics.actionSteps += 1;
|
|
506
|
+
// A project-tool mutation is a gap: the grammar cannot reproduce it, so a
|
|
507
|
+
// replay of this step's trace ends here rather than skipping the change.
|
|
508
|
+
this.stepCache?.recordGap(call.name);
|
|
509
|
+
if (this.metrics.actionSteps > this.maxActions) {
|
|
510
|
+
throw this.fatalize(new AgentError('STEP_BUDGET_EXHAUSTED', `${this.spec.api} exhausted its action budget of ${this.maxActions}`));
|
|
511
|
+
}
|
|
512
|
+
}
|
|
513
|
+
/** Attaches metrics, model provenance, and the verdict explanation to the step. */
|
|
514
|
+
finish() {
|
|
515
|
+
this.closed = true;
|
|
516
|
+
const cacheInfo = this.stepCache?.cacheInfo;
|
|
517
|
+
this.runtime.steps.attachAgentDetails({
|
|
518
|
+
metrics: { ...this.metrics },
|
|
519
|
+
...(this.metrics.modelCalls > 0 ? { model: this.modelInfo() } : {}),
|
|
520
|
+
...(cacheInfo === undefined ? {} : { cache: cacheInfo }),
|
|
521
|
+
...(this.explanation !== undefined ? { explanation: this.explanation } : {}),
|
|
522
|
+
...(this.latest !== undefined ? { observationRevision: this.latest.revision } : {}),
|
|
523
|
+
});
|
|
524
|
+
this.writeTranscript();
|
|
525
|
+
}
|
|
526
|
+
/** Persists the executor transcript as a step-attributed `log` artifact. */
|
|
527
|
+
writeTranscript() {
|
|
528
|
+
if (this.transcript === undefined)
|
|
529
|
+
return;
|
|
530
|
+
const stepId = this.runtime.steps.currentStepId ?? 'act';
|
|
531
|
+
const name = `transcript-${stepId.replace(/[^A-Za-z0-9_-]+/g, '-')}.txt`;
|
|
532
|
+
try {
|
|
533
|
+
// Model prose and project-tool output are not model input, but they
|
|
534
|
+
// are a log: the same redactor that guards the tree guards the file.
|
|
535
|
+
writeFileSync(join(this.runtime.artifacts.dir, name), this.redact(this.transcript), 'utf8');
|
|
536
|
+
this.runtime.steps.attachArtifact(this.runtime.artifacts.register('log', name));
|
|
537
|
+
}
|
|
538
|
+
catch {
|
|
539
|
+
// The transcript is best-effort debug detail; never fail the step for it.
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
/**
|
|
543
|
+
* Model provenance for the report. The executor is the authority on which
|
|
544
|
+
* model answered; an executor that reports nothing is still identified, so a
|
|
545
|
+
* step's model calls are never attributed to the wrong tier.
|
|
546
|
+
*/
|
|
547
|
+
modelInfo() {
|
|
548
|
+
const executor = this.runtime.executor;
|
|
549
|
+
const executorVersion = executor.version ?? '0';
|
|
550
|
+
return {
|
|
551
|
+
provider: this.modelProvider ?? executor.name,
|
|
552
|
+
model: this.modelId ?? executor.name,
|
|
553
|
+
endpoint: 'provider-default',
|
|
554
|
+
adapterVersion: `executor/${executor.name}@${executorVersion}`,
|
|
555
|
+
policyVersion: `${executor.name}/${executorVersion}`,
|
|
556
|
+
calls: this.metrics.modelCalls,
|
|
557
|
+
tokenAccounting: this.providerReportedUsage ? 'provider' : 'adapter-upper-bound',
|
|
558
|
+
peakTokensPerCall: this.peakTokensPerCall,
|
|
559
|
+
inputTokens: this.inputTokens,
|
|
560
|
+
outputTokens: this.outputTokens,
|
|
561
|
+
...(this.estimatedCostUsd === undefined ? {} : { estimatedCostUsd: this.estimatedCostUsd }),
|
|
562
|
+
};
|
|
563
|
+
}
|
|
564
|
+
get session() {
|
|
565
|
+
return this.runtime.engine.session;
|
|
566
|
+
}
|
|
567
|
+
operation() {
|
|
568
|
+
return boundedOperation(this.runtime.engine, this.runtime.config.actionTimeout, this.deadline);
|
|
569
|
+
}
|
|
570
|
+
/** Resolves the configured model once; executors that never read it never pay. */
|
|
571
|
+
resolveModel() {
|
|
572
|
+
if (!this.sdkModelResolved) {
|
|
573
|
+
const resolved = this.runtime.config.agent.model;
|
|
574
|
+
this.sdkModel = resolved === undefined ? undefined : instantiateLanguageModel(resolved);
|
|
575
|
+
this.sdkModelResolved = true;
|
|
576
|
+
}
|
|
577
|
+
return this.sdkModel;
|
|
578
|
+
}
|
|
579
|
+
/** Fails when the step is cancelled or out of time; records the hard stop. */
|
|
580
|
+
checkpoint(cause) {
|
|
581
|
+
try {
|
|
582
|
+
checkStepClock({
|
|
583
|
+
signal: this.runtime.signal,
|
|
584
|
+
deadline: this.deadline,
|
|
585
|
+
api: this.spec.api,
|
|
586
|
+
timeoutMs: this.timeoutMs,
|
|
587
|
+
...(cause === undefined ? {} : { cause }),
|
|
588
|
+
});
|
|
589
|
+
}
|
|
590
|
+
catch (error) {
|
|
591
|
+
if (isAgentError(error))
|
|
592
|
+
this.fatalize(error);
|
|
593
|
+
throw error;
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
/** Chains one grammar operation behind every earlier one, in call order. */
|
|
597
|
+
serialized(body) {
|
|
598
|
+
// The chain is always already settled-to-undefined, so failures propagate
|
|
599
|
+
// to their own caller and never poison the queue.
|
|
600
|
+
const run = this.grammarChain.then(body);
|
|
601
|
+
this.grammarChain = run.then(() => undefined, () => undefined);
|
|
602
|
+
return run;
|
|
603
|
+
}
|
|
604
|
+
/**
|
|
605
|
+
* The executor-facing observe: always settled. An executor observation is
|
|
606
|
+
* followed by a model call measured in seconds, so the bounded settle wait
|
|
607
|
+
* is noise there — and it guarantees the model never reads a snapshot the
|
|
608
|
+
* app is still reacting to, which a fast model turns into a repeated action
|
|
609
|
+
* (double-committing a toggle) and a verdict judged on pre-render state.
|
|
610
|
+
* Replay's pre-action looks and the cache session's probes settle through
|
|
611
|
+
* the same path (`observeSettled`); only replay's polls between retries
|
|
612
|
+
* read raw (`observeLatest`).
|
|
613
|
+
*/
|
|
614
|
+
async observe() {
|
|
615
|
+
const observation = await this.observeSettled();
|
|
616
|
+
return {
|
|
617
|
+
revision: observation.revision,
|
|
618
|
+
text: observation.text,
|
|
619
|
+
truncated: observation.truncated,
|
|
620
|
+
viewport: observation.viewport,
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
observeLatest() {
|
|
624
|
+
return this.serialized(() => this.observeNow(false));
|
|
625
|
+
}
|
|
626
|
+
observeSettled() {
|
|
627
|
+
return this.serialized(() => this.observeNow(true));
|
|
628
|
+
}
|
|
629
|
+
/** One recorded observation; when `settle`, the captures loop inside it. */
|
|
630
|
+
async observeNow(settle) {
|
|
631
|
+
this.checkpoint();
|
|
632
|
+
const observation = await instrumentPhase(this.runtime, { api: this.spec.api, kind: 'observation', phase: 'agent.observe' }, () => settle
|
|
633
|
+
? settleObservation(() => this.captureObservation(), observationShape, {
|
|
634
|
+
remainingMs: () => this.deadline.remaining(),
|
|
635
|
+
// The step's own hard stop must interrupt a settle sleep too —
|
|
636
|
+
// the attempt signal alone would let settling outlive the step
|
|
637
|
+
// by one poll interval.
|
|
638
|
+
signal: this.stepSignal,
|
|
639
|
+
})
|
|
640
|
+
: this.captureObservation(), (prepared) => ({ count: prepared.nodes.size, bytes: prepared.bytes }));
|
|
641
|
+
this.latest = observation;
|
|
642
|
+
this.metrics.observationBytes = Math.max(this.metrics.observationBytes, observation.bytes);
|
|
643
|
+
return observation;
|
|
644
|
+
}
|
|
645
|
+
/** One raw observation capture: retried at the backend, then redacted and bounded. */
|
|
646
|
+
async captureObservation() {
|
|
647
|
+
const raw = await retryingObserve({
|
|
648
|
+
observe: (operation) => this.session.observe(operation, { pixels: false }),
|
|
649
|
+
operation: () => this.operation(),
|
|
650
|
+
guard: (cause) => this.checkpoint(cause),
|
|
651
|
+
signal: this.runtime.signal,
|
|
652
|
+
api: this.spec.api,
|
|
653
|
+
});
|
|
654
|
+
return prepareObservation(raw, {
|
|
655
|
+
redact: this.runtime.redact,
|
|
656
|
+
maxBytes: this.runtime.config.agent.maxObservationBytes,
|
|
657
|
+
testIdAttribute: this.runtime.config.testIdAttribute,
|
|
658
|
+
});
|
|
659
|
+
}
|
|
660
|
+
/** Resolves an executor target against the newest observation. */
|
|
661
|
+
resolveTarget(target) {
|
|
662
|
+
if (typeof target?.id !== 'string' || target.id === '') {
|
|
663
|
+
throw new TestError('INVALID_ARGUMENT', 'action target must be { id: string }');
|
|
664
|
+
}
|
|
665
|
+
const id = target.id.replace(/^#/, '');
|
|
666
|
+
const latest = this.latest;
|
|
667
|
+
if (latest === undefined) {
|
|
668
|
+
throw new AgentError('LOCATOR_NOT_FOUND', 'no observation has been captured yet; observe before acting');
|
|
669
|
+
}
|
|
670
|
+
const node = latest.nodes.get(id);
|
|
671
|
+
if (node === undefined) {
|
|
672
|
+
throw new AgentError('LOCATOR_NOT_FOUND', `node #${id} is not part of observation ${latest.revision}; re-observe and use a current id`);
|
|
673
|
+
}
|
|
674
|
+
return node;
|
|
675
|
+
}
|
|
676
|
+
/**
|
|
677
|
+
* Runs one grammar action against the action budget, recorded as a backend
|
|
678
|
+
* event. The body performs the backend call and returns the committed
|
|
679
|
+
* action's recordable descriptor — one value carries both concerns: the
|
|
680
|
+
* event's `detail` prose derives from it in a pure hook, and the dispatch
|
|
681
|
+
* writes it to the trace cache after the phase settles.
|
|
682
|
+
*/
|
|
683
|
+
runAction(name, body) {
|
|
684
|
+
return this.serialized(() => this.runActionNow(name, body));
|
|
685
|
+
}
|
|
686
|
+
async runActionNow(name, body) {
|
|
687
|
+
this.checkpoint();
|
|
688
|
+
if (this.metrics.actionSteps >= this.maxActions) {
|
|
689
|
+
throw this.fatalize(new AgentError('STEP_BUDGET_EXHAUSTED', `${this.spec.api} exhausted its action budget of ${this.maxActions}`));
|
|
690
|
+
}
|
|
691
|
+
// The budget slot is consumed either way: a failed dispatch was an attempt.
|
|
692
|
+
this.metrics.actionSteps += 1;
|
|
693
|
+
let action;
|
|
694
|
+
try {
|
|
695
|
+
action = await instrumentPhase(this.runtime, { api: this.spec.api, kind: 'backend', phase: 'agent.action', name }, body, (committed) => ({
|
|
696
|
+
detail: describeAction(committed, this.redact, this.runtime.config.testIdAttribute).summary,
|
|
697
|
+
}));
|
|
698
|
+
}
|
|
699
|
+
catch (cause) {
|
|
700
|
+
this.checkpoint(cause);
|
|
701
|
+
throw cause;
|
|
702
|
+
}
|
|
703
|
+
this.stepCache?.record(action);
|
|
704
|
+
}
|
|
705
|
+
/** One action against a resolved node; a stale ref asks for a re-observe. */
|
|
706
|
+
commitTargeted(name, target, perform) {
|
|
707
|
+
return this.runAction(name, async () => {
|
|
708
|
+
const node = this.resolveTarget(target);
|
|
709
|
+
try {
|
|
710
|
+
return await perform(node);
|
|
711
|
+
}
|
|
712
|
+
catch (cause) {
|
|
713
|
+
if (cause instanceof BackendError && cause.code === 'NODE_STALE') {
|
|
714
|
+
throw new AgentError('LOCATOR_NOT_FOUND', 'the target node is stale; re-observe and use a current id', { cause });
|
|
715
|
+
}
|
|
716
|
+
throw cause;
|
|
717
|
+
}
|
|
718
|
+
});
|
|
719
|
+
}
|
|
720
|
+
async scroll(direction, target) {
|
|
721
|
+
if (!['up', 'down', 'left', 'right'].includes(direction)) {
|
|
722
|
+
throw new TestError('INVALID_ARGUMENT', `invalid scroll direction "${String(direction)}"`);
|
|
723
|
+
}
|
|
724
|
+
if (target === undefined) {
|
|
725
|
+
await this.runAction('scroll', async () => {
|
|
726
|
+
await this.session.swipe(direction, undefined, this.operation());
|
|
727
|
+
return { name: 'scroll', direction };
|
|
728
|
+
});
|
|
729
|
+
return;
|
|
730
|
+
}
|
|
731
|
+
await this.commitTargeted('scroll', target, async (node) => {
|
|
732
|
+
await this.session.perform(node.ref, { kind: 'swipe', direction }, this.operation());
|
|
733
|
+
return { name: 'scroll', direction, node };
|
|
734
|
+
});
|
|
735
|
+
}
|
|
736
|
+
/**
|
|
737
|
+
* Fills one declared secret. The name must come from the step's own params
|
|
738
|
+
* — an executor can never fill a credential the test did not hand it — and
|
|
739
|
+
* the fill itself runs the full secret authorization policy: registered credential, origin allowlists, and an editable sink
|
|
740
|
+
* whose purpose matches. Pixel evidence is tainted from here on.
|
|
741
|
+
*/
|
|
742
|
+
async typeSecret(target, name) {
|
|
743
|
+
const secret = this.spec.secrets.get(name);
|
|
744
|
+
if (secret === undefined) {
|
|
745
|
+
throw new AgentError('POLICY_DENIED', `secret "${name}" was not declared in this step's params; only declared secrets can be filled`);
|
|
746
|
+
}
|
|
747
|
+
await this.commitTargeted('typeSecret', target, async (node) => {
|
|
748
|
+
const plaintext = await authorizeSecretFill({
|
|
749
|
+
session: this.session,
|
|
750
|
+
operation: () => this.operation(),
|
|
751
|
+
recordPolicy: (policy, decision, code) => this.recordPolicy(policy, decision, code),
|
|
752
|
+
}, this.runtime, secret, node);
|
|
753
|
+
await this.session.perform(node.ref, { kind: 'fill', value: plaintext, sensitive: true }, this.operation());
|
|
754
|
+
this.runtime.taint.value = true;
|
|
755
|
+
// Recorded by stable name only; replay re-runs the full authorization.
|
|
756
|
+
return { name: 'typeSecret', node, secret: name };
|
|
757
|
+
});
|
|
758
|
+
}
|
|
759
|
+
recordPolicy(name, decision, code) {
|
|
760
|
+
recordPolicyEvent(this.runtime.steps, name, decision, code);
|
|
761
|
+
}
|
|
762
|
+
async navigate(url) {
|
|
763
|
+
if (typeof url !== 'string' || url.trim() === '') {
|
|
764
|
+
throw new TestError('INVALID_ARGUMENT', 'navigate requires a URL');
|
|
765
|
+
}
|
|
766
|
+
const resolved = resolveNavigationUrl(url, this.runtime.config.app.base, this.runtime.config.app.allowedOrigins).url;
|
|
767
|
+
// The raw argument is recorded, not the resolved URL: replay re-resolves
|
|
768
|
+
// through the same base and origin policy this call just passed.
|
|
769
|
+
await this.runAction('navigate', async () => {
|
|
770
|
+
await this.session.app.open(resolved, this.operation());
|
|
771
|
+
return { name: 'navigate', url };
|
|
772
|
+
});
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
//# sourceMappingURL=act.js.map
|