firedrill-run 2.2.4 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +178 -22
- package/bin/firedrill.mjs +29 -5
- package/dist/assembly.json +13 -0
- package/dist/cli/runtime/candidate_copy.py +395 -0
- package/dist/cli/runtime/source_snapshot.py +391 -0
- package/dist/cli/src/administration.js +295 -0
- package/dist/cli/src/agent-changes.js +496 -0
- package/dist/cli/src/assessment-requests.js +67 -0
- package/dist/cli/src/auth.js +465 -0
- package/dist/cli/src/billing-commerce-contracts.js +1 -0
- package/dist/cli/src/billing-usage.js +227 -0
- package/dist/cli/src/billing-wallet-contracts.js +1 -0
- package/dist/cli/src/billing.js +170 -0
- package/dist/cli/src/bootstrap-process-lock.js +136 -0
- package/dist/cli/src/browser-links.js +14 -0
- package/dist/cli/src/campaign-controls.js +122 -0
- package/dist/cli/src/campaign-diagnoses.js +118 -0
- package/dist/cli/src/campaign-reports.js +1215 -0
- package/dist/cli/src/campaign-world.js +156 -0
- package/dist/cli/src/candidate-copy.js +376 -0
- package/dist/cli/src/catalog-contracts.js +1 -0
- package/dist/cli/src/ci-access-contracts.js +1 -0
- package/dist/cli/src/ci-configuration-contracts.js +1 -0
- package/dist/cli/src/ci-coverage.js +64 -0
- package/dist/cli/src/ci-delivery-contracts.js +1 -0
- package/dist/cli/src/ci-result-contracts.js +1 -0
- package/dist/cli/src/ci-run-contracts.js +1 -0
- package/dist/cli/src/ci-setup-proposals.js +52 -0
- package/dist/cli/src/company-policies.js +229 -0
- package/dist/cli/src/completion.js +79 -0
- package/dist/cli/src/configuration-intents.js +277 -0
- package/dist/cli/src/configuration.js +369 -0
- package/dist/cli/src/continuous-simulation.js +277 -0
- package/dist/cli/src/coordinated-checkpoints.js +133 -0
- package/dist/cli/src/credential-mode.js +139 -0
- package/dist/cli/src/credentials.js +331 -0
- package/dist/cli/src/credit-format.js +8 -0
- package/dist/cli/src/dataset-contracts.js +1 -0
- package/dist/cli/src/dataset-export.js +154 -0
- package/dist/cli/src/dataset-reads.js +37 -0
- package/dist/cli/src/dependency-preparation.js +239 -0
- package/dist/cli/src/director-tool-setup.js +104 -0
- package/dist/cli/src/driver-review.js +225 -0
- package/dist/cli/src/evaluation-authoring.js +315 -0
- package/dist/cli/src/evidence-bundles.js +367 -0
- package/dist/cli/src/execution-proposals.js +36 -0
- package/dist/cli/src/http.js +302 -0
- package/dist/cli/src/local-container-boundary.js +38 -0
- package/dist/cli/src/local-container-policy.js +201 -0
- package/dist/cli/src/local-execution-setup.js +81 -0
- package/dist/cli/src/local-source-basis.js +73 -0
- package/dist/cli/src/local-source.js +66 -0
- package/dist/cli/src/local-value-store.js +91 -0
- package/dist/cli/src/local-values.js +163 -0
- package/dist/cli/src/login-organizations.js +70 -0
- package/dist/cli/src/main.js +1622 -0
- package/dist/cli/src/mcp-connection.js +109 -0
- package/dist/cli/src/native/bindings.js +137 -0
- package/dist/cli/src/native/canonical.js +370 -0
- package/dist/cli/src/native/cloud-resources.js +942 -0
- package/dist/cli/src/native/companion-channel.js +421 -0
- package/dist/cli/src/native/companion-ipc.js +473 -0
- package/dist/cli/src/native/contracts.js +218 -0
- package/dist/cli/src/native/control-plane.js +1354 -0
- package/dist/cli/src/native/credit-approval.js +15 -0
- package/dist/cli/src/native/dependency-proxy.js +1006 -0
- package/dist/cli/src/native/drill-cleanup.js +115 -0
- package/dist/cli/src/native/drill-runner.js +1562 -0
- package/dist/cli/src/native/drivers/action-trace.js +210 -0
- package/dist/cli/src/native/drivers/function-wrapper.js +152 -0
- package/dist/cli/src/native/drivers/http.js +442 -0
- package/dist/cli/src/native/drivers/index.js +47 -0
- package/dist/cli/src/native/drivers/jsonl-process.js +538 -0
- package/dist/cli/src/native/drivers/playwright.js +600 -0
- package/dist/cli/src/native/drivers/signed-event.js +244 -0
- package/dist/cli/src/native/drivers/types.js +578 -0
- package/dist/cli/src/native/evidence-filter.js +255 -0
- package/dist/cli/src/native/isolated-access.js +241 -0
- package/dist/cli/src/native/library-sharing.js +220 -0
- package/dist/cli/src/native/local-run.js +667 -0
- package/dist/cli/src/native/local-secrets.js +19 -0
- package/dist/cli/src/native/mcp-config.js +123 -0
- package/dist/cli/src/native/metadata-requests.js +120 -0
- package/dist/cli/src/native/observations.js +1159 -0
- package/dist/cli/src/native/owned-groups.js +46 -0
- package/dist/cli/src/native/plan.js +594 -0
- package/dist/cli/src/native/png.js +159 -0
- package/dist/cli/src/native/ports.js +43 -0
- package/dist/cli/src/native/readiness.js +139 -0
- package/dist/cli/src/native/reassessments.js +206 -0
- package/dist/cli/src/native/redaction.js +235 -0
- package/dist/cli/src/native/registration.js +155 -0
- package/dist/cli/src/native/roots.js +110 -0
- package/dist/cli/src/native/run-cleanup.js +199 -0
- package/dist/cli/src/native/scratch.js +117 -0
- package/dist/cli/src/native/screenshots.js +60 -0
- package/dist/cli/src/native/session.js +556 -0
- package/dist/cli/src/native/standalone-proxy.js +580 -0
- package/dist/cli/src/native/standalone-renewals.js +183 -0
- package/dist/cli/src/native/standalone-state.js +193 -0
- package/dist/cli/src/native/standalone-tools.js +187 -0
- package/dist/cli/src/native/supervisor.js +831 -0
- package/dist/cli/src/native/target-auth.js +161 -0
- package/dist/cli/src/native/target-state.js +330 -0
- package/dist/cli/src/native/tool-reuse.js +364 -0
- package/dist/cli/src/native/tool-setups.js +336 -0
- package/dist/cli/src/native/world-reuse.js +288 -0
- package/dist/cli/src/onboarding.js +23 -0
- package/dist/cli/src/priority-requests.js +70 -0
- package/dist/cli/src/priority.js +223 -0
- package/dist/cli/src/prompts.js +184 -0
- package/dist/cli/src/real-operations.js +221 -0
- package/dist/cli/src/recipe-intents.js +58 -0
- package/dist/cli/src/recorded-context-intents.js +59 -0
- package/dist/cli/src/recorded-context.js +248 -0
- package/dist/cli/src/recordings.js +69 -0
- package/dist/cli/src/registration-guide.js +39 -0
- package/dist/cli/src/report-exports.js +213 -0
- package/dist/cli/src/request-input.js +47 -0
- package/dist/cli/src/resource-lifecycle.js +141 -0
- package/dist/cli/src/revision-requests.js +55 -0
- package/dist/cli/src/runner-abandon.js +53 -0
- package/dist/cli/src/runner-campaign-controls.js +262 -0
- package/dist/cli/src/runner-claim-reads.js +36 -0
- package/dist/cli/src/runner-completion.js +65 -0
- package/dist/cli/src/runner-configuration.js +546 -0
- package/dist/cli/src/runner-container.js +794 -0
- package/dist/cli/src/runner-evidence-erasure.js +351 -0
- package/dist/cli/src/runner-evidence-privacy.js +37 -0
- package/dist/cli/src/runner-evidence-retention.js +161 -0
- package/dist/cli/src/runner-evidence.js +326 -0
- package/dist/cli/src/runner-execute.js +92 -0
- package/dist/cli/src/runner-execution-deadline.js +25 -0
- package/dist/cli/src/runner-grants.js +307 -0
- package/dist/cli/src/runner-identity.js +615 -0
- package/dist/cli/src/runner-inputs.js +308 -0
- package/dist/cli/src/runner-jobs.js +49 -0
- package/dist/cli/src/runner-model-transport.js +392 -0
- package/dist/cli/src/runner-ownership.js +332 -0
- package/dist/cli/src/runner-process-lock.js +175 -0
- package/dist/cli/src/runner-runtime.js +396 -0
- package/dist/cli/src/runner-session-channel.js +377 -0
- package/dist/cli/src/runner-trial-completion.js +34 -0
- package/dist/cli/src/runner-trial-journal.js +358 -0
- package/dist/cli/src/runner-trial-live.js +393 -0
- package/dist/cli/src/runner-trial-protocol.js +289 -0
- package/dist/cli/src/runner-trial-sync.js +110 -0
- package/dist/cli/src/runner-trial-transfer.js +107 -0
- package/dist/cli/src/runner-voice.js +40 -0
- package/dist/cli/src/runner-world-claim.js +21 -0
- package/dist/cli/src/runner-world-collector.js +159 -0
- package/dist/cli/src/runner-world-command-evidence.js +487 -0
- package/dist/cli/src/runner-world-commands.js +268 -0
- package/dist/cli/src/runner-world-completion.js +76 -0
- package/dist/cli/src/runner-world-erasure.js +95 -0
- package/dist/cli/src/runner-world-evidence-sync.js +91 -0
- package/dist/cli/src/runner-world-journal.js +510 -0
- package/dist/cli/src/runner-world-native-filters.js +134 -0
- package/dist/cli/src/runner-world-observations.js +50 -0
- package/dist/cli/src/runner-world-session-protocol.js +134 -0
- package/dist/cli/src/saved-records.js +40 -0
- package/dist/cli/src/scenario-authoring.js +212 -0
- package/dist/cli/src/scenario-discovery.js +216 -0
- package/dist/cli/src/scenario-paths.js +68 -0
- package/dist/cli/src/scenario-requests.js +4 -0
- package/dist/cli/src/service-receipts.js +105 -0
- package/dist/cli/src/simple/authoring.js +270 -0
- package/dist/cli/src/simple/credit-approval.js +16 -0
- package/dist/cli/src/simple/help.js +98 -0
- package/dist/cli/src/simple/init.js +61 -0
- package/dist/cli/src/simple/library-sharing.js +414 -0
- package/dist/cli/src/simple/metadata-requests.js +46 -0
- package/dist/cli/src/simple/native-report.js +841 -0
- package/dist/cli/src/simple/native-tools.js +780 -0
- package/dist/cli/src/simple/plan-apply.js +506 -0
- package/dist/cli/src/simple/reassessment.js +323 -0
- package/dist/cli/src/simple/register.js +123 -0
- package/dist/cli/src/simple/run.js +122 -0
- package/dist/cli/src/simple/status.js +49 -0
- package/dist/cli/src/simple/tool-renewal.js +308 -0
- package/dist/cli/src/simple/tool-reuse.js +526 -0
- package/dist/cli/src/simple/tool-start-catalog.js +100 -0
- package/dist/cli/src/simple/tool-state.js +426 -0
- package/dist/cli/src/simple/tools-author.js +222 -0
- package/dist/cli/src/simple/watch.js +28 -0
- package/dist/cli/src/simple/world-reuse.js +586 -0
- package/dist/cli/src/simulation-intents.js +406 -0
- package/dist/cli/src/simulation-run.js +1058 -0
- package/dist/cli/src/source-batch.js +322 -0
- package/dist/cli/src/source-connection.js +451 -0
- package/dist/cli/src/source-runtime.js +346 -0
- package/dist/cli/src/source-selection-intents.js +75 -0
- package/dist/cli/src/source-sets.js +214 -0
- package/dist/cli/src/target-models.js +109 -0
- package/dist/cli/src/tool-catalog.js +224 -0
- package/dist/cli/src/tool-command-contracts.js +1 -0
- package/dist/cli/src/tool-command-export.js +245 -0
- package/dist/cli/src/tool-connection-contracts.js +1 -0
- package/dist/cli/src/tool-connection-reads.js +23 -0
- package/dist/cli/src/tool-contributions.js +58 -0
- package/dist/cli/src/tool-data.js +116 -0
- package/dist/cli/src/tool-datasets.js +176 -0
- package/dist/cli/src/tool-group-contracts.js +1 -0
- package/dist/cli/src/tool-group-reads.js +26 -0
- package/dist/cli/src/tool-group-state.js +147 -0
- package/dist/cli/src/tool-groups.js +311 -0
- package/dist/cli/src/tool-instances.js +186 -0
- package/dist/cli/src/tool-lifecycle-contracts.js +1 -0
- package/dist/cli/src/tool-lifecycle-reads.js +30 -0
- package/dist/cli/src/tool-lifecycle.js +323 -0
- package/dist/cli/src/tool-package-contracts.js +1 -0
- package/dist/cli/src/tool-package-reads.js +41 -0
- package/dist/cli/src/tool-records.js +53 -0
- package/dist/cli/src/tool-runtime.js +96 -0
- package/dist/cli/src/tool-state-contracts.js +1 -0
- package/dist/cli/src/tool-state-reads.js +31 -0
- package/dist/cli/src/tools.js +306 -0
- package/dist/cli/src/trial-actions.js +93 -0
- package/dist/cli/src/trial-checkpoints.js +133 -0
- package/dist/cli/src/trial-checks.js +70 -0
- package/dist/cli/src/trial-state.js +197 -0
- package/dist/cli/src/vault.js +266 -0
- package/dist/cli/src/workspace-registration.js +383 -0
- package/dist/cli/src/workspace-team-contracts.js +1 -0
- package/dist/cli/src/world-access.js +231 -0
- package/dist/cli/src/world-agent.js +4 -0
- package/dist/cli/src/world-checkpoint-scopes.js +15 -0
- package/dist/cli/src/world-contracts.js +6 -0
- package/dist/cli/src/world-creation.js +201 -0
- package/dist/cli/src/world-datasets.js +207 -0
- package/dist/cli/src/world-inspection.js +257 -0
- package/dist/cli/src/world-recipes.js +103 -0
- package/dist/cli/src/world-requests.js +241 -0
- package/dist/cli/src/world-session-checkpoints.js +196 -0
- package/dist/cli/src/world-session-lifecycle.js +207 -0
- package/dist/cli/src/world-session-requests.js +215 -0
- package/dist/cli/src/world-session-state.js +145 -0
- package/dist/cli/src/world-session-voice.js +66 -0
- package/dist/cli/src/world-sessions.js +288 -0
- package/dist/cli/src/world-stream.js +99 -0
- package/dist/cli/src/world-time-report.js +58 -0
- package/dist/contracts/agent-change-contracts.js +75 -0
- package/dist/contracts/agent-execution-contracts.js +38 -0
- package/dist/contracts/application-state-contracts.js +26 -0
- package/dist/contracts/assessment-aggregate-contracts.js +15 -0
- package/dist/contracts/assessment-contracts.js +74 -0
- package/dist/contracts/billing-commerce-contracts.js +107 -0
- package/dist/contracts/billing-wallet-contracts.js +43 -0
- package/dist/contracts/campaign-lifecycle.js +1 -0
- package/dist/contracts/campaign-world-contracts.js +80 -0
- package/dist/contracts/catalog-contracts.js +73 -0
- package/dist/contracts/ci-access-contracts.js +68 -0
- package/dist/contracts/ci-configuration-contracts.js +88 -0
- package/dist/contracts/ci-coverage-contracts.js +65 -0
- package/dist/contracts/ci-coverage-labels.js +25 -0
- package/dist/contracts/ci-delivery-contracts.js +61 -0
- package/dist/contracts/ci-result-contracts.js +103 -0
- package/dist/contracts/ci-run-contracts.js +51 -0
- package/dist/contracts/ci-setup-contracts.js +88 -0
- package/dist/contracts/clip-contracts.js +32 -0
- package/dist/contracts/comparison-contracts.js +53 -0
- package/dist/contracts/composite-contracts.js +34 -0
- package/dist/contracts/context-contracts.js +76 -0
- package/dist/contracts/continuation-report-contracts.js +1 -0
- package/dist/contracts/continuity-requirements.js +8 -0
- package/dist/contracts/coordinated-checkpoint-contracts.js +34 -0
- package/dist/contracts/dataset-contracts.js +88 -0
- package/dist/contracts/dependency-decision-contracts.js +36 -0
- package/dist/contracts/dependency-protocol-contracts.js +43 -0
- package/dist/contracts/director-execution.js +51 -0
- package/dist/contracts/driver-review-contracts.js +99 -0
- package/dist/contracts/evidence-contracts.js +65 -0
- package/dist/contracts/evidence-coverage-contracts.js +10 -0
- package/dist/contracts/export-contracts.js +44 -0
- package/dist/contracts/github-directory-contracts.js +96 -0
- package/dist/contracts/insight-contracts.js +11 -0
- package/dist/contracts/inspection-contracts.js +1 -0
- package/dist/contracts/local-claim-contracts.js +39 -0
- package/dist/contracts/local-configuration-contracts.js +7 -0
- package/dist/contracts/local-evidence-contracts.js +32 -0
- package/dist/contracts/local-grant-contracts.js +83 -0
- package/dist/contracts/local-input-contracts.js +19 -0
- package/dist/contracts/local-runner-contracts.js +42 -0
- package/dist/contracts/local-trial-live-contracts.js +16 -0
- package/dist/contracts/local-value-source-contracts.js +92 -0
- package/dist/contracts/local-world-native-contracts.js +8 -0
- package/dist/contracts/mcp-contracts.js +42 -0
- package/dist/contracts/media-contracts.js +28 -0
- package/dist/contracts/native-check-contracts.js +31 -0
- package/dist/contracts/native-identity-contracts.js +52 -0
- package/dist/contracts/onboarding-contracts.js +32 -0
- package/dist/contracts/performance-contracts.js +85 -0
- package/dist/contracts/planning-contracts.js +81 -0
- package/dist/contracts/real-operation-contracts.js +30 -0
- package/dist/contracts/recommendation-contracts.js +54 -0
- package/dist/contracts/report-contracts.js +1 -0
- package/dist/contracts/resource-lifecycle-contracts.js +57 -0
- package/dist/contracts/revision-contracts.js +12 -0
- package/dist/contracts/revision-request-contracts.js +39 -0
- package/dist/contracts/saved-record-contracts.js +76 -0
- package/dist/contracts/scenario-draft-comparison.js +91 -0
- package/dist/contracts/scenario-world-contracts.js +14 -0
- package/dist/contracts/service-contracts.js +1 -0
- package/dist/contracts/session-command-contracts.js +56 -0
- package/dist/contracts/session-contracts.js +65 -0
- package/dist/contracts/session-state-contracts.js +67 -0
- package/dist/contracts/session-voice-contracts.js +45 -0
- package/dist/contracts/source-set-contracts.js +26 -0
- package/dist/contracts/source-upload-contracts.js +22 -0
- package/dist/contracts/tool-command-contracts.js +57 -0
- package/dist/contracts/tool-connection-contracts.js +87 -0
- package/dist/contracts/tool-contribution-contracts.js +35 -0
- package/dist/contracts/tool-group-contracts.js +53 -0
- package/dist/contracts/tool-group-state-contracts.js +63 -0
- package/dist/contracts/tool-lifecycle-contracts.js +94 -0
- package/dist/contracts/tool-package-contracts.js +91 -0
- package/dist/contracts/tool-protocol-contracts.js +71 -0
- package/dist/contracts/tool-runtime-billing-contracts.js +28 -0
- package/dist/contracts/tool-setup-proposal.js +59 -0
- package/dist/contracts/tool-state-contracts.js +79 -0
- package/dist/contracts/trial-state-contracts.js +30 -0
- package/dist/contracts/typescript/agent-change.js +1 -0
- package/dist/contracts/typescript/assessment-aggregate.js +1 -0
- package/dist/contracts/typescript/assessment.js +1 -0
- package/dist/contracts/typescript/billing-commerce.js +1 -0
- package/dist/contracts/typescript/campaigns.js +1 -0
- package/dist/contracts/typescript/catalog.js +1 -0
- package/dist/contracts/typescript/ci-access.js +1 -0
- package/dist/contracts/typescript/ci-configuration.js +1 -0
- package/dist/contracts/typescript/ci-delivery.js +1 -0
- package/dist/contracts/typescript/ci-result.js +1 -0
- package/dist/contracts/typescript/ci-runs.js +1 -0
- package/dist/contracts/typescript/clips.js +1 -0
- package/dist/contracts/typescript/comparison.js +1 -0
- package/dist/contracts/typescript/composite.js +1 -0
- package/dist/contracts/typescript/connections.js +1 -0
- package/dist/contracts/typescript/context.js +1 -0
- package/dist/contracts/typescript/datasets.js +1 -0
- package/dist/contracts/typescript/evidence-coverage.js +1 -0
- package/dist/contracts/typescript/export.js +1 -0
- package/dist/contracts/typescript/github-directory.js +1 -0
- package/dist/contracts/typescript/insight.js +1 -0
- package/dist/contracts/typescript/inspection.js +1 -0
- package/dist/contracts/typescript/intake.js +1 -0
- package/dist/contracts/typescript/local-claim.js +1 -0
- package/dist/contracts/typescript/local-configuration.js +1 -0
- package/dist/contracts/typescript/local-evidence.js +1 -0
- package/dist/contracts/typescript/local-grant.js +1 -0
- package/dist/contracts/typescript/local-input.js +1 -0
- package/dist/contracts/typescript/local-runner.js +1 -0
- package/dist/contracts/typescript/local-trial-live.js +1 -0
- package/dist/contracts/typescript/local-value-source.js +1 -0
- package/dist/contracts/typescript/local-world-native.js +1 -0
- package/dist/contracts/typescript/mcp.js +1 -0
- package/dist/contracts/typescript/media.js +1 -0
- package/dist/contracts/typescript/native-check.js +1 -0
- package/dist/contracts/typescript/native-identity.js +1 -0
- package/dist/contracts/typescript/onboarding.js +1 -0
- package/dist/contracts/typescript/performance.js +1 -0
- package/dist/contracts/typescript/planning.js +1 -0
- package/dist/contracts/typescript/preparation.js +1 -0
- package/dist/contracts/typescript/real-operation.js +1 -0
- package/dist/contracts/typescript/recommendation.js +1 -0
- package/dist/contracts/typescript/reports.js +1 -0
- package/dist/contracts/typescript/resource-lifecycle.js +1 -0
- package/dist/contracts/typescript/saved-record.js +1 -0
- package/dist/contracts/typescript/scenarios.js +1 -0
- package/dist/contracts/typescript/services.js +1 -0
- package/dist/contracts/typescript/session-command.js +1 -0
- package/dist/contracts/typescript/session-state.js +1 -0
- package/dist/contracts/typescript/session-voice.js +1 -0
- package/dist/contracts/typescript/session.js +1 -0
- package/dist/contracts/typescript/tool-command.js +1 -0
- package/dist/contracts/typescript/tool-connection.js +1 -0
- package/dist/contracts/typescript/tool-contribution.js +1 -0
- package/dist/contracts/typescript/tool-groups.js +1 -0
- package/dist/contracts/typescript/tool-lifecycle.js +1 -0
- package/dist/contracts/typescript/tool-package.js +1 -0
- package/dist/contracts/typescript/tool-state.js +1 -0
- package/dist/contracts/typescript/trial-state.js +1 -0
- package/dist/contracts/typescript/vault.js +1 -0
- package/dist/contracts/typescript/workspace-team.js +1 -0
- package/dist/contracts/typescript/world-stream.js +1 -0
- package/dist/contracts/workspace-team-contracts.js +55 -0
- package/dist/contracts/world-command.js +65 -0
- package/dist/contracts/world-lifetime-contracts.js +74 -0
- package/dist/contracts/world-recipe-contracts.js +27 -0
- package/dist/contracts/world-stream-contracts.js +27 -0
- package/dist/package.json +1 -0
- package/package.json +16 -16
- package/LICENSE.md +0 -23
- package/PACKAGING.json +0 -50
|
@@ -0,0 +1,1215 @@
|
|
|
1
|
+
import { APP_ORIGIN } from './http.js';
|
|
2
|
+
import { workspaceLink } from './browser-links.js';
|
|
3
|
+
import { api, openBrowser } from './auth.js';
|
|
4
|
+
import { CliError } from './http.js';
|
|
5
|
+
import { selectOne, terminalText } from './prompts.js';
|
|
6
|
+
import { selectedWorkspace } from './world-access.js';
|
|
7
|
+
import { identifier } from './world-contracts.js';
|
|
8
|
+
import { reportExport, validateExport } from './report-exports.js';
|
|
9
|
+
import { worldTimeReport } from './world-time-report.js';
|
|
10
|
+
import { evidenceCommand, timelineCommand } from './evidence-bundles.js';
|
|
11
|
+
import { serviceReceipts, validateReceiptOptions } from './service-receipts.js';
|
|
12
|
+
import { trialActions, validateActionOptions } from './trial-actions.js';
|
|
13
|
+
import { trialRecordedChecks, validateCheckOptions } from './trial-checks.js';
|
|
14
|
+
import { trialStateReport, validateTrialStateOptions } from './trial-state.js';
|
|
15
|
+
import { campaignWorldCommand } from './campaign-world.js';
|
|
16
|
+
import { captureMatchesTrial, fileMatchesTrial } from '../../contracts/media-contracts.js';
|
|
17
|
+
import { captureClipMatches } from '../../contracts/clip-contracts.js';
|
|
18
|
+
import { coveragePageMatches } from '../../contracts/evidence-coverage-contracts.js';
|
|
19
|
+
import { assessmentMatches, assessmentPageMatches, assessmentEvidenceMatches } from '../../contracts/assessment-contracts.js';
|
|
20
|
+
import { checkpointLabel } from '../../contracts/trial-state-contracts.js';
|
|
21
|
+
import { scoreCohortMatches, checkpointMatches, styleCohortMatches } from '../../contracts/assessment-aggregate-contracts.js';
|
|
22
|
+
import { comparisonPageMatches, comparisonObservationsMatch } from '../../contracts/comparison-contracts.js';
|
|
23
|
+
import { checkGroupsMatch, checkObservationsMatch, nativeCheckObservationMatches } from '../../contracts/native-check-contracts.js';
|
|
24
|
+
import { compositeMatches, compositePageMatches, compositeGroupsMatch, compositeModelsMatch, compositeObservationsMatch } from '../../contracts/composite-contracts.js';
|
|
25
|
+
import { targetUsageMatches, targetCompletionNotes, performanceGroupsMatch, performanceObservationsMatch } from '../../contracts/performance-contracts.js';
|
|
26
|
+
import { findingsMatch, insightsMatch } from '../../contracts/insight-contracts.js';
|
|
27
|
+
const scoreStatuses = { assessed: 'succeeded', 'not-assessable': 'not_assessable', 'not-applicable': 'not_applicable' };
|
|
28
|
+
const assessmentLabels = { queued: 'Queued', preparing: 'Preparing evidence', submitted: 'Evaluating',
|
|
29
|
+
succeeded: 'Assessed', not_assessable: 'Not assessable', not_applicable: 'Not applicable', failed: 'Evaluation error', unknown: 'Reply unconfirmed', unavailable: 'Evidence unavailable' };
|
|
30
|
+
const assessmentStates = new Set(['queued', 'preparing', 'submitted', 'succeeded', 'not_assessable', 'not_applicable', 'failed', 'unknown', 'unavailable']);
|
|
31
|
+
const taskModes = new Set(['single_request', 'browser_workflow', 'adaptive_browser', 'desktop', 'process', 'event']);
|
|
32
|
+
const trialStates = new Set(['planned', 'claimed', 'dispatching', 'running', 'reconciling', 'completed', 'failed',
|
|
33
|
+
'timed_out', 'canceled', 'platform_failed', 'platform_failed_before_start', 'canceled_before_start']);
|
|
34
|
+
const trialChecks = { met: 'All checks met', unmet: 'Unmet checks', unassessed: 'Not assessed',
|
|
35
|
+
unavailable: 'Check result unavailable', missing: 'No available result' };
|
|
36
|
+
const executionPhase = (value) => ({ 'Running trial': 'Running drill', 'Preparing trial': 'Preparing drill' }[value] ?? value);
|
|
37
|
+
const activityLabel = (kind) => (kind.startsWith('trial.') ? `Drill ${kind.slice(6)}` : kind).replaceAll('.', ' ').replaceAll('_', ' ');
|
|
38
|
+
const memoryPointLabel = (point) => point.kind === 'turn' ? `Turn ${point.turn}` : `Event ${point.event_key}`;
|
|
39
|
+
const memoryCheckpointLabel = (probe) => probe.checkpoint ? memoryPointLabel(probe.checkpoint) : `Turn ${probe.checkpoint_turn}`;
|
|
40
|
+
const memoryExposureLabels = (probe) => probe.exposures ? probe.exposures.map(memoryPointLabel).join(', ')
|
|
41
|
+
: probe.exposure_turns.map(turn => `Turn ${turn}`).join(', ');
|
|
42
|
+
function trialListPath(prefix, options) {
|
|
43
|
+
const query = new URLSearchParams();
|
|
44
|
+
if (options.scenario)
|
|
45
|
+
query.set('scenario', identifier(options.scenario));
|
|
46
|
+
if (options.status)
|
|
47
|
+
query.set('status', options.status);
|
|
48
|
+
if (options.checks)
|
|
49
|
+
query.set('checks', options.checks);
|
|
50
|
+
if (options.cohort)
|
|
51
|
+
query.set('cohort', identifier(options.cohort));
|
|
52
|
+
if (options.assessmentStatus)
|
|
53
|
+
query.set('assessment_status', scoreStatuses[options.assessmentStatus]);
|
|
54
|
+
if (options.scoreBucket !== undefined)
|
|
55
|
+
query.set('score_bucket', options.scoreBucket);
|
|
56
|
+
return `${prefix}/trials${query.size ? `?${query}` : ''}`;
|
|
57
|
+
}
|
|
58
|
+
async function choosePage(title, path, scope, options, describe, confirm) {
|
|
59
|
+
let page = options.page ?? '1';
|
|
60
|
+
while (true) {
|
|
61
|
+
const data = await api(`${path}${path.includes('?') ? '&' : '?'}page=${page}&q=${encodeURIComponent(options.search ?? '')}`, { actor: scope.owner, signal: options.signal });
|
|
62
|
+
if (confirm && !confirm(data))
|
|
63
|
+
throw new CliError('report_unavailable', 'This saved report selection could not be confirmed. Reopen the report.');
|
|
64
|
+
const choices = data.results.map(value => ({ value: { kind: 'item', value }, ...describe(value) }));
|
|
65
|
+
if (Number(page) > 1)
|
|
66
|
+
choices.unshift({ value: { kind: 'previous' }, label: 'Previous page', detail: '' });
|
|
67
|
+
if (data.next)
|
|
68
|
+
choices.push({ value: { kind: 'next' }, label: 'Next page', detail: '' });
|
|
69
|
+
if (!data.count)
|
|
70
|
+
throw new CliError('report_empty', 'No matching saved reports are available.');
|
|
71
|
+
const choice = await selectOne(title, choices, options.signal);
|
|
72
|
+
if (choice.kind === 'item')
|
|
73
|
+
return choice.value;
|
|
74
|
+
page = String(Number(page) + (choice.kind === 'previous' ? -1 : 1));
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
function duration(value) {
|
|
78
|
+
if (value === null)
|
|
79
|
+
return 'Not recorded';
|
|
80
|
+
if (!/^[0-9]{1,40}$/.test(value))
|
|
81
|
+
throw new CliError('report_unavailable', 'The recorded timing could not be read.');
|
|
82
|
+
const milliseconds = BigInt(value), seconds = milliseconds / 1000n;
|
|
83
|
+
if (milliseconds < 1000n)
|
|
84
|
+
return `${milliseconds}ms`;
|
|
85
|
+
return `${seconds / 3600n}h ${seconds / 60n % 60n}m ${seconds % 60n}s`;
|
|
86
|
+
}
|
|
87
|
+
function entryLabel(row, agent) {
|
|
88
|
+
if (row.task_phase === 'submission')
|
|
89
|
+
return `${agent} submission response`;
|
|
90
|
+
if (row.task_phase === 'poll')
|
|
91
|
+
return `${agent} status response`;
|
|
92
|
+
return taskModes.has(row.interaction_mode) ? row.role === 'user' ? 'Task input' : `${agent} output`
|
|
93
|
+
: row.role === 'user' ? 'Synthetic user' : agent;
|
|
94
|
+
}
|
|
95
|
+
function completionLines(value) {
|
|
96
|
+
if (!value)
|
|
97
|
+
return [];
|
|
98
|
+
const names = { pending: 'Pending', completed: 'Completed', failed: 'Failed', unconfirmed: 'Completion not confirmed', rejected: 'Submission rejected' };
|
|
99
|
+
return [`Source job status: ${names[value.state]}${'status_value' in value && value.status_value ? ` (${value.status_value})` : ''}`,
|
|
100
|
+
`${value.poll_requests.toLocaleString()} status read attempts; ${value.retained_responses.toLocaleString()} of ${value.responses.toLocaleString()} response bodies recorded.`,
|
|
101
|
+
...(value.omitted_responses ? ['Some response bodies could not be retained. Missing content is not an empty or successful result.'] : []),
|
|
102
|
+
...(value.reason === 'evidence_unavailable' ? ['Response recording could not finish. Available observations are preserved.'] : []),
|
|
103
|
+
'Job status is not an evaluation score. Status reads do not submit the task again.'];
|
|
104
|
+
}
|
|
105
|
+
function streamLines(streams) {
|
|
106
|
+
return streams.flatMap(stream => [`Attempt ${stream.attempt_number}: ${stream.available_responses.toLocaleString()} response entries available${stream.recorded_responses === null ? '.' : ` of ${stream.recorded_responses.toLocaleString()} recorded.`}`,
|
|
107
|
+
...(!stream.consistent ? ['The response summary does not match the retained entries. Completeness cannot be confirmed.'] : []),
|
|
108
|
+
...(!stream.input_available ? ['The original task input is unavailable here.'] : []),
|
|
109
|
+
...(!stream.final_summary_available ? ['The final response summary has not been retained.'] : []),
|
|
110
|
+
...(stream.unavailable_responses !== null && stream.unavailable_responses > 0 ? ['Some recorded responses are unavailable in this report.'] : [])]);
|
|
111
|
+
}
|
|
112
|
+
function conversationLines(streams) {
|
|
113
|
+
return streams.flatMap(stream => [
|
|
114
|
+
`Conversation attempt ${stream.attempt_number}: ${stream.available_inputs.toLocaleString()}${stream.progress ? ` of ${stream.progress.inputs.toLocaleString()} submitted` : ''} inputs and ${stream.available_responses.toLocaleString()}${stream.progress ? ` of ${stream.progress.responses.toLocaleString()} observed` : ''} replies available.`,
|
|
115
|
+
...(!stream.consistent ? ['The conversation summary could not be reconciled. Completeness is unknown.'] : []),
|
|
116
|
+
...(!stream.final_summary_available ? ['The final conversation summary is unavailable.'] : []),
|
|
117
|
+
...(stream.progress?.state === 'interrupted' ? ['The conversation did not finish; retained messages are preserved.'] : []),
|
|
118
|
+
...(stream.progress && (stream.progress.omitted_inputs + stream.progress.omitted_responses > 0
|
|
119
|
+
|| stream.available_inputs < stream.progress.recorded_inputs || stream.available_responses < stream.progress.recorded_responses)
|
|
120
|
+
? ['Some conversation content is unavailable, not an empty or successful answer.'] : [])
|
|
121
|
+
]);
|
|
122
|
+
}
|
|
123
|
+
function captureLines(value) {
|
|
124
|
+
if (!value)
|
|
125
|
+
return [];
|
|
126
|
+
if (!value.window_observed)
|
|
127
|
+
return ['The task request did not reach the capture window. Tool usage is unknown.'];
|
|
128
|
+
return [`Mapped task requests: ${value.observed.toLocaleString()} observed; ${value.captured.toLocaleString()} payloads captured; ${value.unavailable.toLocaleString()} unavailable; ${value.unfinished.toLocaleString()} unfinished at task return.`,
|
|
129
|
+
...(value.pending_untracked ? ['Additional unfinished requests could not be tracked individually.'] : []),
|
|
130
|
+
'Only mapped service traffic during the task. Private or unsupported payloads are withheld; captured files may still be awaiting cloud retention.'];
|
|
131
|
+
}
|
|
132
|
+
function operationLines(streams) {
|
|
133
|
+
return streams.flatMap(stream => [
|
|
134
|
+
`${stream.channel === 'http' ? 'Mapped HTTP/MCP' : stream.channel === 'source' ? 'Agent-reported calls' : 'Handler'} attempt ${stream.attempt_number}: ${(stream.coverage?.observed ?? stream.starts_available).toLocaleString()} ${stream.coverage ? stream.channel === 'source' ? 'calls reported' : 'requests observed' : 'entries available'}; ${stream.receipts_available.toLocaleString()} payloads available${stream.coverage ? `; ${stream.coverage.withheld.toLocaleString()} withheld; ${stream.coverage.unfinished.toLocaleString()} ${stream.channel === 'http' ? 'responses unknown' : stream.channel === 'source' ? 'reports incomplete' : 'completions unknown'}` : ''}.`,
|
|
135
|
+
...(!stream.consistent ? ['The request summary could not be reconciled. Completeness is unknown.'] : []),
|
|
136
|
+
...(!stream.final_summary_available ? ['The final request summary is unavailable.'] : []),
|
|
137
|
+
...(stream.coverage && (stream.receipts_available < stream.coverage.retained || stream.starts_available < stream.coverage.observed)
|
|
138
|
+
? ['Some recorded operation evidence is unavailable here.'] : [])
|
|
139
|
+
]);
|
|
140
|
+
}
|
|
141
|
+
function speechLines(value) {
|
|
142
|
+
if (!value)
|
|
143
|
+
return [];
|
|
144
|
+
return [
|
|
145
|
+
...(value.started_at === null || value.ended_at === null ? [] : [`${value.basis === 'session_audio' ? 'Audio timing' : 'Transcription delivery'}: ${new Date(value.started_at * 1000).toISOString()} to ${new Date(value.ended_at * 1000).toISOString()}`]),
|
|
146
|
+
...(value.duration_ms === null ? [] : [`${value.basis === 'session_audio' ? 'Observed audio interval' : 'Transcription delivery span'}: ${value.duration_ms}ms`]),
|
|
147
|
+
...(value.reported_latency_ms === null ? [] : [`Reported turn latency: ${value.reported_latency_ms.toLocaleString()}ms`]),
|
|
148
|
+
...(value.interrupted === null ? [] : [`Reported interrupted: ${value.interrupted ? 'yes' : 'no'}`])
|
|
149
|
+
];
|
|
150
|
+
}
|
|
151
|
+
function speechSummaryLines(groups, agent) {
|
|
152
|
+
return groups.flatMap(row => row.audio_messages || row.delivery_messages || row.latency_observations || row.interruption_observations ? [
|
|
153
|
+
`Speech measurements: ${row.role === 'assistant' ? agent : 'Synthetic user'}, attempt ${row.attempt_number}`,
|
|
154
|
+
...(row.audio_duration_ms === null ? [] : [`Observed audio intervals: ${row.audio_duration_ms}ms across ${row.audio_messages} of ${row.messages} messages`]),
|
|
155
|
+
...(row.delivery_duration_ms === null ? [] : [`Transcription delivery spans: ${row.delivery_duration_ms}ms across ${row.delivery_messages} of ${row.messages} messages; not speaking time`]),
|
|
156
|
+
...(row.mean_reported_latency_ms === null ? [] : [`Reported turn latency: mean ${row.mean_reported_latency_ms}ms, range ${row.min_reported_latency_ms}ms to ${row.max_reported_latency_ms}ms from ${row.latency_observations} observations`]),
|
|
157
|
+
...(row.interruption_observations ? [`Reported interruptions: ${row.interrupted} of ${row.interruption_observations} observed messages`] : [])
|
|
158
|
+
] : []);
|
|
159
|
+
}
|
|
160
|
+
async function trialCoverageReport(path, scope, interactive, options, output, supplied, agent = 'Agent') {
|
|
161
|
+
if (supplied !== undefined && !['messages', 'actions'].includes(supplied)
|
|
162
|
+
|| options.search !== undefined && (!/^[1-9][0-9]{0,9}$/.test(options.search) || Number(options.search) > 2 ** 31 - 1))
|
|
163
|
+
throw new CliError('arguments_invalid', 'Choose coverage messages|actions. --search filters one positive attempt number.', 2);
|
|
164
|
+
const kind = supplied ?? (interactive ? await selectOne('Evidence coverage', [
|
|
165
|
+
{ value: 'messages', label: 'Messages and speech' }, { value: 'actions', label: 'Tool evidence' },
|
|
166
|
+
], options.signal) : 'messages');
|
|
167
|
+
let page = Number(options.page ?? 1);
|
|
168
|
+
while (true) {
|
|
169
|
+
const params = new URLSearchParams({ kind, page: String(page), q: options.search ?? '' });
|
|
170
|
+
const data = await api(`${path}/coverage?${params}`, {
|
|
171
|
+
actor: scope.owner, workspace: scope.workspace, signal: options.signal
|
|
172
|
+
});
|
|
173
|
+
if (!coveragePageMatches(data, kind, options.search))
|
|
174
|
+
throw new CliError('trial_coverage_unavailable', 'The selected evidence coverage could not be confirmed.');
|
|
175
|
+
const parts = path.split('/');
|
|
176
|
+
output(data, [...data.results.flatMap(row => [`Attempt ${row.attempt_number.toLocaleString()}`,
|
|
177
|
+
...operationLines(row.operation_streams), ...streamLines(row.task_message_streams), ...conversationLines(row.message_streams),
|
|
178
|
+
...speechSummaryLines(row.speech_measurements, agent), ...row.voice_streams.flatMap(stream => [
|
|
179
|
+
`Speech transcript: ${stream.available_messages.toLocaleString()} messages retained`,
|
|
180
|
+
...(!stream.consistent ? ['Transcript coverage could not be reconciled.'] : []),
|
|
181
|
+
...(!stream.canonical_transcript_available ? ['The final transcript is unavailable; retained speech observations may be incomplete.'] : []),
|
|
182
|
+
...((stream.progress?.omitted_messages ?? 0) > 0 ? ['Some speech observations could not be retained.'] : [])
|
|
183
|
+
])]),
|
|
184
|
+
...(!data.count ? [options.search ? 'No evidence matches this attempt number.' : 'No coverage evidence is available.'] : []),
|
|
185
|
+
`Page ${data.page}; ${data.count.toLocaleString()} attempts with retained ${kind === 'actions' ? 'tool' : 'message'} evidence.`,
|
|
186
|
+
...(data.next ? [`Continue: firedrill report ${parts.at(-3)} drill ${parts.at(-1)} coverage ${kind} --workspace ${scope.workspace} --page ${page + 1}${options.search ? ' --search ' + options.search : ''}${options.json ? ' --json' : ''}`] : [])].join('\n'));
|
|
187
|
+
if (!interactive)
|
|
188
|
+
return;
|
|
189
|
+
const choice = await selectOne('Coverage page', [...(page > 1 ? [{ value: 'previous', label: 'Previous page' }] : []),
|
|
190
|
+
...(data.next ? [{ value: 'next', label: 'Next page' }] : []), { value: 'done', label: 'Done' }], options.signal);
|
|
191
|
+
if (choice === 'done')
|
|
192
|
+
return;
|
|
193
|
+
page += choice === 'next' ? 1 : -1;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
async function trialReport(args, prefix, scope, interactive, options, output) {
|
|
197
|
+
if (options.checkpoint || args[3] && !['messages', 'actions', 'requests', 'checks', 'check-observation', 'coverage', 'state', 'captures', 'desktop-captures', 'clips', 'recordings', 'schedule'].includes(args[3]) || args[4] && !args[2]
|
|
198
|
+
|| (args[3] === 'captures' || args[3] === 'desktop-captures' || args[3] === 'recordings') && (args[4] || options.search)
|
|
199
|
+
|| args[4] && !['schedule', 'coverage', 'state'].includes(args[3]) && (options.page || options.search)
|
|
200
|
+
|| args[3] === 'schedule' && (args.length > 6 || args[4] && !/^[a-f0-9]{64}$/.test(args[4]) || args[5] && !['changes', 'conditions'].includes(args[5]))) {
|
|
201
|
+
throw new CliError('arguments_invalid', 'Use report <campaign> drill [drill] [messages|actions|requests [entry]|checks [position]|check-observation <id>|coverage [messages|actions]|captures|desktop-captures|clips [clip]|recordings|schedule [event] [changes|conditions]]. Paging applies to lists.', 2);
|
|
202
|
+
}
|
|
203
|
+
if (args[3] === 'clips' && (options.search || args.length > 5))
|
|
204
|
+
throw new CliError('arguments_invalid', 'Use clips [clip-id] with --page for its history. Preparation, playback and downloads open in the browser.', 2);
|
|
205
|
+
const auth = { actor: scope.owner, workspace: scope.workspace, signal: options.signal };
|
|
206
|
+
let execution = args[2] ? identifier(args[2]) : undefined;
|
|
207
|
+
const listPath = trialListPath(prefix, options);
|
|
208
|
+
if (!execution && !interactive) {
|
|
209
|
+
const data = await api(`${listPath}${listPath.includes('?') ? '&' : '?'}page=${options.page ?? '1'}&q=${encodeURIComponent(options.search ?? '')}`, auth);
|
|
210
|
+
const quote = (value) => `'${value.replaceAll("'", "'\\''")}'`;
|
|
211
|
+
output(data, [`${data.count.toLocaleString()} matching drills`, ...data.results.map(row => `Drill ${row.position + 1}: ${row.scenario_name} · ${row.state}\n${trialChecks[row.checks] ?? 'Unavailable'}\n${row.id}`),
|
|
212
|
+
...(data.next ? [`Continue: firedrill report ${prefix.split('/').at(-1)} drill --workspace ${scope.workspace} --page ${data.page + 1}${options.search ? ' --search ' + quote(options.search) : ''}${options.status ? ' --status ' + options.status : ''}${options.checks ? ' --checks ' + options.checks : ''}${options.scenario ? ' --scenario ' + identifier(options.scenario) : ''}${options.cohort ? ' --cohort ' + identifier(options.cohort) : ''}${options.assessmentStatus ? ' --assessment-status ' + options.assessmentStatus : ''}${options.scoreBucket !== undefined ? ' --score-bucket ' + options.scoreBucket : ''}${options.json ? ' --json' : ''}`] : [])].join('\n\n'));
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
const chosen = !execution;
|
|
216
|
+
if (!execution)
|
|
217
|
+
execution = identifier((await choosePage('Drill', listPath, scope, options, row => ({ label: `Drill ${row.position + 1}: ${row.scenario_name}`, detail: `${row.state}; ${trialChecks[row.checks] ?? 'Unavailable'}` }))).id);
|
|
218
|
+
const path = `${prefix}/trials/${execution}`;
|
|
219
|
+
if (args[3] === 'check-observation') {
|
|
220
|
+
if (args.length !== 5 || !args[4] || options.page !== undefined || options.search !== undefined || options.status !== undefined
|
|
221
|
+
|| options.evidenceKind !== undefined || options.checks !== undefined || options.scenario !== undefined)
|
|
222
|
+
throw new CliError('arguments_invalid', 'Choose one original check-observation ID without list filters.', 2);
|
|
223
|
+
const observation = identifier(args[4]), campaign = identifier(prefix.split('/').at(-1));
|
|
224
|
+
const row = await api(`${path}/check-observations/${observation}`, auth);
|
|
225
|
+
if (!nativeCheckObservationMatches(row, { workspace: scope.workspace, campaign, execution, observation }))
|
|
226
|
+
throw new CliError('trial_check_unavailable', 'The selected check observation could not be confirmed.');
|
|
227
|
+
output(row, [`${terminalText(row.name)}: ${!row.reported ? 'Not reported' : row.status === 'met' ? 'Met' : row.status === 'unmet' ? 'Unmet' : 'Unconfirmed'}`,
|
|
228
|
+
nativeCheckMethods[row.method], ...(row.reason !== null ? [terminalText(row.reason)] : row.reason_truncated
|
|
229
|
+
? ['The explanation is too large to show inline. Read the saved result.']
|
|
230
|
+
: [row.reported ? 'No reason recorded.' : 'This check was declared but was not included in the retained result.']),
|
|
231
|
+
...(row.reason_truncated ? [`Saved result: firedrill report ${campaign} drill ${execution} evidence ${row.evidence_id} --workspace ${scope.workspace}`] : []),
|
|
232
|
+
workspaceLink(scope.workspace, `/campaigns/${campaign}/trials/${execution}?view=assessment&check_observation=${observation}`)].join('\n'));
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
if (args[3] === 'state') {
|
|
236
|
+
await trialStateReport(args.slice(4), path, scope, interactive, options, output);
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
if (args[3] === 'checks') {
|
|
240
|
+
await trialRecordedChecks(path, scope, interactive, options, output, args[4]);
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
243
|
+
if (args[3] === 'coverage') {
|
|
244
|
+
await trialCoverageReport(path, scope, interactive, options, output, args[4]);
|
|
245
|
+
return;
|
|
246
|
+
}
|
|
247
|
+
const data = await api(path, auth);
|
|
248
|
+
if (data.id !== execution || data.campaign_id !== prefix.split('/').at(-1))
|
|
249
|
+
throw new CliError('trial_report_unavailable', 'This retained drill report could not be confirmed. Reopen its campaign.');
|
|
250
|
+
let inspect = args[3];
|
|
251
|
+
if (!inspect) {
|
|
252
|
+
if (args[2] && (options.page || options.search))
|
|
253
|
+
throw new CliError('arguments_invalid', 'Search and paging apply to drill, input/output or action lists.', 2);
|
|
254
|
+
const task = data.conversation.interaction_modes.length > 0 && data.conversation.interaction_modes.every(mode => taskModes.has(mode));
|
|
255
|
+
output(data, [`Drill ${data.number}: ${data.scenario.name} · ${data.execution_state.replaceAll('_', ' ')}`,
|
|
256
|
+
data.agent.name, data.scenario.instruction, ...(data.problem ? [data.problem] : []), ...(data.evidence_notice ? [data.evidence_notice] : []),
|
|
257
|
+
data.execution_location === 'local' ? 'Local device execution. Evidence is device-reported, not independently observed by a managed sandbox.' : 'Managed sandbox execution.',
|
|
258
|
+
...(data.starting_data ? [`Starting data: ${data.starting_data.edit_count} saved edits selected.`,
|
|
259
|
+
data.starting_data.application ? `Applied before scenario setup at ${data.starting_data.application.applied_at}; ${data.starting_data.application.rows_affected} rows directly affected. This is not the final state.`
|
|
260
|
+
: data.starting_data.problem ?? (data.starting_data.application_state === 'pending'
|
|
261
|
+
? 'Starting-data application evidence is pending. Read this same drill again; selection alone does not confirm application.'
|
|
262
|
+
: 'The original starting-data application record is unavailable.')] : []),
|
|
263
|
+
...(data.application_continuation ? [`Saved starting state: ${data.application_continuation.selection.scope === 'supported_application_and_native_state'
|
|
264
|
+
? 'application and supported synthetic service data' : 'application with fresh synthetic services'}.`,
|
|
265
|
+
data.application_continuation.state === 'observed' && data.application_continuation.observed_at
|
|
266
|
+
? `Restoration reported at ${data.application_continuation.observed_at}${data.application_continuation.provenance === 'customer_device' ? ' by the local runner' : ' by the managed runtime'}.`
|
|
267
|
+
: 'Restoration evidence is not available. Selection alone does not confirm application.',
|
|
268
|
+
'New process only. External accounts, live process memory and elapsed wall time are not restored.'] : []),
|
|
269
|
+
`${data.conversation.message_count.toLocaleString()} ${task ? 'inputs and outputs' : 'messages'} retained; ${data.actions.count.toLocaleString()} tool actions; ${data.service_requests.toLocaleString()} mapped service requests`,
|
|
270
|
+
`${data.state_observations.toLocaleString()} state observations`,
|
|
271
|
+
...(data.voice_capture.enabled || data.voice_capture.retained ? [`${data.voice_capture.recordings.toLocaleString()} audio recordings; ${data.voice_capture.segments.toLocaleString()} partial clips retained`] : []),
|
|
272
|
+
...(data.browser_capture.enabled || data.browser_capture.retained ? [
|
|
273
|
+
`${data.browser_capture.images.toLocaleString()} browser images available from ${data.browser_capture.retained.toLocaleString()} retained capture records`,
|
|
274
|
+
...(data.browser_capture.latest ? [`Latest capture: ${data.browser_capture.latest.captured_at}; ${data.browser_capture.latest.state}`] : [])
|
|
275
|
+
] : []),
|
|
276
|
+
...(data.desktop_capture.enabled || data.desktop_capture.retained ? [
|
|
277
|
+
`${data.desktop_capture.images.toLocaleString()} desktop images available from ${data.desktop_capture.retained.toLocaleString()} retained capture records`
|
|
278
|
+
] : []),
|
|
279
|
+
...captureLines(data.actions.capture),
|
|
280
|
+
...completionLines(data.conversation.task_completion ?? data.conversation.task_progress),
|
|
281
|
+
...(data.coverage.message_attempts || data.coverage.action_attempts ? [`Evidence coverage: ${data.coverage.message_attempts.toLocaleString()} attempts with messages; ${data.coverage.action_attempts.toLocaleString()} with tool evidence.`,
|
|
282
|
+
`Inspect: firedrill report ${prefix.split('/').at(-1)} drill ${execution} coverage --workspace ${scope.workspace}`] : []),
|
|
283
|
+
...(data.conversation.duration_ms === null ? [] : [`${task ? 'Task' : 'Conversation'} duration: ${data.conversation.duration_ms.toLocaleString()}ms`]),
|
|
284
|
+
!data.recorded_checks.result_retained ? 'The original check result is not available.' : data.recorded_checks.available
|
|
285
|
+
? `${data.recorded_checks.count.toLocaleString()} checks recorded during execution.` : 'Recorded checks could not be read. Inspect their original result in Files.',
|
|
286
|
+
workspaceLink(scope.workspace, `/campaigns/${args[0] ?? prefix.split('/').at(-1)}/trials/${execution}`)].join('\n'));
|
|
287
|
+
if (!interactive)
|
|
288
|
+
return;
|
|
289
|
+
inspect = await selectOne('Inspect drill', [...(data.conversation.available ? [{ value: 'messages', label: task ? 'Input and output' : 'Conversation' }] : []),
|
|
290
|
+
...(data.actions.count ? [{ value: 'actions', label: 'Tool actions' }] : []),
|
|
291
|
+
...(data.service_requests ? [{ value: 'requests', label: 'Mapped service receipts' }] : []),
|
|
292
|
+
...(data.state_observations ? [{ value: 'state', label: 'Observed state changes' }] : []),
|
|
293
|
+
...((data.recorded_checks.count ?? 0) > 0 || data.recorded_checks.result_retained && !data.recorded_checks.available ? [{ value: 'checks', label: 'Recorded checks' }] : []),
|
|
294
|
+
...(data.coverage.message_attempts || data.coverage.action_attempts ? [{ value: 'coverage', label: 'Evidence coverage' }] : []),
|
|
295
|
+
...(data.browser_capture.enabled || data.browser_capture.retained ? [{ value: 'captures', label: 'Browser captures' }] : []),
|
|
296
|
+
...(data.desktop_capture.enabled || data.desktop_capture.retained ? [{ value: 'desktop-captures', label: 'Desktop captures' }] : []),
|
|
297
|
+
...(data.browser_capture.retained || data.desktop_capture.retained ? [{ value: 'clips', label: 'Video clips from retained captures' }] : []),
|
|
298
|
+
...(data.voice_capture.retained ? [{ value: 'recordings', label: 'Audio recordings' }] : []),
|
|
299
|
+
{ value: 'schedule', label: 'World schedule' }, { value: 'done', label: 'Done' }], options.signal);
|
|
300
|
+
}
|
|
301
|
+
const listOptions = chosen ? { ...options, page: undefined, search: undefined } : options;
|
|
302
|
+
if (inspect === 'state') {
|
|
303
|
+
await trialStateReport([], path, scope, interactive, listOptions, output);
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
if (inspect === 'coverage') {
|
|
307
|
+
await trialCoverageReport(path, scope, interactive, listOptions, output, undefined, data.agent.name);
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
310
|
+
if (inspect === 'checks') {
|
|
311
|
+
await trialRecordedChecks(path, scope, interactive, listOptions, output);
|
|
312
|
+
return;
|
|
313
|
+
}
|
|
314
|
+
if (inspect === 'requests') {
|
|
315
|
+
await serviceReceipts(`${path}/requests`, scope, listOptions, interactive, output, `firedrill report ${identifier(prefix.split('/').at(-1))} drill ${execution} requests`, args[4]);
|
|
316
|
+
return;
|
|
317
|
+
}
|
|
318
|
+
if (inspect === 'schedule') {
|
|
319
|
+
await worldTimeReport(path, scope, interactive, listOptions, output, args[4], args[5]);
|
|
320
|
+
return;
|
|
321
|
+
}
|
|
322
|
+
if (inspect === 'recordings') {
|
|
323
|
+
const page = await api(`${path}/files?kind=audio&page=${listOptions.page ?? '1'}`, auth);
|
|
324
|
+
if (page.results.some(row => row.audio === null || !fileMatchesTrial(row, path)))
|
|
325
|
+
throw new CliError('report_unavailable', 'The selected drill recordings could not be confirmed. Reopen the report.');
|
|
326
|
+
output(page, [...page.results.map(row => `${row.label ?? 'Audio recording'}${row.audio ? ` · ${row.audio.duration_ms.toLocaleString()}ms · ${row.audio.partial && row.audio.segment ? `Partial clip at ${(row.audio.segment.start_frame / row.audio.segment.sample_rate).toLocaleString(undefined, { maximumFractionDigits: 1 })}s` : row.audio.channels === 2 ? 'Stereo' : 'Mono'}` : ''}\n${row.id}`),
|
|
327
|
+
`Page ${page.page} of ${Math.max(1, Math.ceil(page.count / page.page_size))}`,
|
|
328
|
+
`Listen at 1×, 2× or 4×, or download: ${workspaceLink(scope.workspace, `/campaigns/${prefix.split('/').at(-1)}/trials/${execution}?view=files`)}`].join('\n\n'));
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
if (inspect === 'clips') {
|
|
332
|
+
const browser = (clip) => {
|
|
333
|
+
const url = new URL(`/campaigns/${prefix.split('/').at(-1)}/trials/${execution}`, APP_ORIGIN);
|
|
334
|
+
url.searchParams.set('workspace', scope.workspace);
|
|
335
|
+
url.searchParams.set('view', 'session');
|
|
336
|
+
if (clip) {
|
|
337
|
+
url.searchParams.set('clip', clip.id);
|
|
338
|
+
url.searchParams.set('capture', clip.source_kind);
|
|
339
|
+
}
|
|
340
|
+
url.hash = 'capture-clips';
|
|
341
|
+
return url.href;
|
|
342
|
+
};
|
|
343
|
+
if (args[4]) {
|
|
344
|
+
const row = await api(`${path}/capture-clips/${identifier(args[4])}`, auth);
|
|
345
|
+
if (!captureClipMatches(row, `${path}/capture-clips`, identifier(args[4])))
|
|
346
|
+
throw new CliError('report_unavailable', 'The selected capture clip could not be confirmed. Reopen the report.');
|
|
347
|
+
output(row, [`Capture clip ${row.id}: ${row.state}`, `${row.frames.toLocaleString()} ${row.source_kind} captures`,
|
|
348
|
+
`${row.started_at} to ${row.ended_at}`, 'Browser-created playback of retained images, not continuous footage or a new target run.',
|
|
349
|
+
...(row.problem ? [row.problem] : []), ...(row.file ? [`MP4: ${row.file.size.toLocaleString()} bytes`] : []), browser(row)].join('\n'));
|
|
350
|
+
if (interactive && options.openBrowser && await selectOne('Open capture clip?', [{ value: true, label: 'Open in browser' }, { value: false, label: 'Stay here' }], options.signal))
|
|
351
|
+
await openBrowser(browser(row));
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
const clips = await api(`${path}/capture-clips?page=${listOptions.page ?? '1'}`, auth);
|
|
355
|
+
if (clips.results.some(row => !captureClipMatches(row, `${path}/capture-clips`)))
|
|
356
|
+
throw new CliError('report_unavailable', 'The capture clip history could not be confirmed. Reopen the report.');
|
|
357
|
+
output(clips, [...clips.results.map(row => `${row.started_at} · ${row.source_kind} · ${row.frames.toLocaleString()} captures · ${row.state}\n${row.id}`),
|
|
358
|
+
...(!clips.count ? ['No capture clips have been saved.'] : []), `Page ${clips.page}; ${clips.count.toLocaleString()} clips.`,
|
|
359
|
+
...(clips.next ? [`Continue with --page ${clips.page + 1}.`] : []), 'Create, resume, cancel, play or download a clip in the existing capture inspector.', browser()].join('\n\n'));
|
|
360
|
+
if (interactive && options.openBrowser && await selectOne('Capture clips', [{ value: true, label: 'Open capture inspector' }, { value: false, label: 'Stay here' }], options.signal))
|
|
361
|
+
await openBrowser(browser());
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
if (inspect === 'captures' || inspect === 'desktop-captures') {
|
|
365
|
+
const desktop = inspect === 'desktop-captures';
|
|
366
|
+
const page = await api(`${path}/${inspect}?page=${listOptions.page ?? '1'}`, auth);
|
|
367
|
+
if (page.results.some(row => !captureMatchesTrial(row, path, desktop)))
|
|
368
|
+
throw new CliError('report_unavailable', 'The selected drill captures could not be confirmed. Reopen the report.');
|
|
369
|
+
output(page, [...page.results.map(row => `${row.captured_at} · ${desktop ? 'Agent desktop' : row.phase === 'session' ? 'Agent browser' : `Turn ${row.turn_index + 1} · ${row.phase.replaceAll('_', ' ')}`} · ${row.state}\n${row.id}`),
|
|
370
|
+
`Page ${page.page} of ${Math.max(1, Math.ceil(page.count / page.page_size))}`,
|
|
371
|
+
`View or replay captures at 1×, 2× or 4×: ${workspaceLink(scope.workspace, `/campaigns/${prefix.split('/').at(-1)}/trials/${execution}?view=session${desktop ? '&capture=desktop' : ''}`)}`].join('\n\n'));
|
|
372
|
+
return;
|
|
373
|
+
}
|
|
374
|
+
if (inspect === 'actions') {
|
|
375
|
+
await trialActions(path, scope, interactive, listOptions, output, args[4]);
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
if (inspect !== 'messages')
|
|
379
|
+
return;
|
|
380
|
+
if (args[4]) {
|
|
381
|
+
const row = await api(`${path}/messages/${identifier(args[4])}`, { ...auth, maximumResponseBytes: 16 * 1024 ** 2 });
|
|
382
|
+
output(row, [`${entryLabel(row, data.agent.name)}${row.http_status === null ? '' : ` · HTTP ${row.http_status}`}\n${row.content}`, ...speechLines(row.speech)].join('\n'));
|
|
383
|
+
return;
|
|
384
|
+
}
|
|
385
|
+
if (!interactive) {
|
|
386
|
+
const page = await api(`${path}/messages?page=${listOptions.page ?? '1'}&q=${encodeURIComponent(listOptions.search ?? '')}`, auth);
|
|
387
|
+
output(page, [...page.results.map(row => [`${entryLabel(row, data.agent.name)}${row.http_status === null ? '' : ` · HTTP ${row.http_status}`}\n${row.content}${row.truncated ? '\nFull content: request this entry by ID.' : ''}`, ...speechLines(row.speech), row.id].join('\n')),
|
|
388
|
+
`Page ${page.page} of ${Math.max(1, Math.ceil(page.count / page.page_size))}`].join('\n\n'));
|
|
389
|
+
return;
|
|
390
|
+
}
|
|
391
|
+
const selected = await choosePage('Recorded input or output', `${path}/messages`, scope, listOptions, row => ({ label: `${row.position + 1}. ${entryLabel(row, data.agent.name)}`, detail: row.http_status === null ? '' : `HTTP ${row.http_status}` }));
|
|
392
|
+
const row = await api(`${path}/messages/${identifier(selected.id)}`, { ...auth, maximumResponseBytes: 16 * 1024 ** 2 });
|
|
393
|
+
output(row, [`${entryLabel(row, data.agent.name)}${row.http_status === null ? '' : ` · HTTP ${row.http_status}`}\n${row.content}`, ...speechLines(row.speech)].join('\n'));
|
|
394
|
+
}
|
|
395
|
+
async function findingsReport(supplied, prefix, scope, interactive, options, output) {
|
|
396
|
+
if (options.checkpoint !== undefined)
|
|
397
|
+
throw new CliError('arguments_invalid', 'Checkpoint selection applies to memory reports, not scenario findings.', 2);
|
|
398
|
+
if (supplied && (options.page !== undefined || options.search !== undefined))
|
|
399
|
+
throw new CliError('arguments_invalid', 'Paging and search apply to the findings directory, not an already selected scenario.', 2);
|
|
400
|
+
const campaign = identifier(prefix.split('/').at(-1));
|
|
401
|
+
const chosen = supplied ?? (interactive ? (await choosePage('Scenario findings', `${prefix}/scenario-findings`, scope, options, row => ({ label: row.name, detail: `Revision ${row.revision}; ${row.retained_trials.toLocaleString()} retained drills` }), value => findingsMatch(value, { campaign, query: options.search ?? '' }))).id : undefined);
|
|
402
|
+
const params = new URLSearchParams({ page: chosen ? '1' : options.page ?? '1', q: chosen ? '' : options.search ?? '' });
|
|
403
|
+
if (chosen)
|
|
404
|
+
params.set('scenario', identifier(chosen));
|
|
405
|
+
const data = await api(`${prefix}/scenario-findings?${params}`, { actor: scope.owner, workspace: scope.workspace, signal: options.signal });
|
|
406
|
+
if (!findingsMatch(data, { campaign, query: params.get('q'), scenario: params.get('scenario') ?? undefined }))
|
|
407
|
+
throw new CliError('report_unavailable', 'These scenario findings could not be confirmed. Reopen the report.');
|
|
408
|
+
const quote = (value) => "'" + value.replaceAll("'", "'\\''") + "'";
|
|
409
|
+
const reasons = { unmet_checks: 'Unmet checks', agent_error: 'Agent error', timed_out: 'Timed out', execution_error: 'Execution error', checks_met: 'All checks met', assessment_record: 'Assessment available' };
|
|
410
|
+
output(data, [...data.results.flatMap(row => [row.name, `Revision ${row.revision}; ${row.retained_trials.toLocaleString()} retained drills`,
|
|
411
|
+
...Object.entries(row.execution_states).map(([state, count]) => `${state.replaceAll('_', ' ')}: ${count.toLocaleString()}`),
|
|
412
|
+
...(row.unclassified_execution ? [`Unrecognized execution status: ${row.unclassified_execution.toLocaleString()} drills`] : []),
|
|
413
|
+
...Object.entries(row.checks).map(([state, count]) => `${trialChecks[state] ?? state}: ${count.toLocaleString()}`),
|
|
414
|
+
`${row.semantic.scored_trials.toLocaleString()} drills with at least one score; ${row.semantic.scored.toLocaleString()} scored assessments; ${row.semantic.not_assessable.toLocaleString()} not assessable; ${row.semantic.not_applicable.toLocaleString()} not applicable`,
|
|
415
|
+
...row.examples.map(sample => `Example drill ${sample.trial_position + 1}: ${sample.selected_for.map(kind => reasons[kind] ?? 'Recorded outcome').join(', ')}\n${workspaceLink(scope.workspace, `/campaigns/${prefix.split('/').at(-1)}/trials/${sample.execution_id}${sample.assessment_id ? `?view=assessment&assessment=${sample.assessment_id}` : ''}`)}`),
|
|
416
|
+
`Scenario revision: ${row.id}`]), `Page ${data.page} of ${Math.max(1, Math.ceil(data.count / data.page_size))}`,
|
|
417
|
+
...(data.pending ? ['More assessments are pending.'] : []),
|
|
418
|
+
...(data.next && !chosen ? [`Continue: firedrill report ${campaign} findings --workspace ${scope.workspace} --page ${data.page + 1}${options.search ? ' --search ' + quote(options.search) : ''}${options.json ? ' --json' : ''}`] : []),
|
|
419
|
+
'Examples are the lowest-numbered available drill for each outcome or assessment, not a random sample. Execution and checks are not semantic scores.'].join('\n'));
|
|
420
|
+
}
|
|
421
|
+
async function scoreReport(supplied, prefix, scope, interactive, options, output) {
|
|
422
|
+
if (options.checkpoint !== undefined)
|
|
423
|
+
throw new CliError('arguments_invalid', 'Checkpoint selection applies to memory reports, not score groups.', 2);
|
|
424
|
+
const auth = { actor: scope.owner, workspace: scope.workspace, signal: options.signal };
|
|
425
|
+
const number = (value) => value === null ? 'Not assessed' : value.toLocaleString(undefined, { maximumFractionDigits: 2 });
|
|
426
|
+
const summary = (value, cohort) => [
|
|
427
|
+
`${value.assessed.toLocaleString()} assessed of ${value.observed.toLocaleString()} retained assessments; ${value.not_assessable.toLocaleString()} not assessable; ${value.not_applicable.toLocaleString()} not applicable`,
|
|
428
|
+
`Mean: ${number(value.mean)}${value.mean === null ? '' : ` / ${cohort.levels.length - 1}`}; range ${number(value.minimum)} to ${number(value.maximum)}`
|
|
429
|
+
];
|
|
430
|
+
if (!supplied && !interactive) {
|
|
431
|
+
const data = await api(`${prefix}/assessment-groups?page=${options.page ?? '1'}&q=${encodeURIComponent(options.search ?? '')}`, auth);
|
|
432
|
+
output(data, [...data.results.flatMap(row => [`${row.cohort.scenario_name}: ${row.cohort.criterion}`, `Evaluator: ${row.cohort.model} (requested ${row.cohort.requested_model})`,
|
|
433
|
+
...summary(row.summary, row.cohort), `Cohort assessment: ${row.cohort.assessment_id}`]),
|
|
434
|
+
`Page ${data.page} of ${Math.max(1, Math.ceil(data.count / data.page_size))}`].join('\n'));
|
|
435
|
+
return;
|
|
436
|
+
}
|
|
437
|
+
const selected = supplied ?? (await choosePage('Score group', `${prefix}/assessment-groups`, scope, options, row => ({ label: `${row.cohort.scenario_name}: ${row.cohort.criterion}`, detail: `${row.cohort.model}; ${row.summary.assessed.toLocaleString()} assessed` }))).cohort.assessment_id;
|
|
438
|
+
const params = new URLSearchParams({ page: supplied ? options.page ?? '1' : '1' });
|
|
439
|
+
if (options.assessmentStatus)
|
|
440
|
+
params.set('status', scoreStatuses[options.assessmentStatus]);
|
|
441
|
+
if (options.scoreBucket !== undefined)
|
|
442
|
+
params.set('bucket', options.scoreBucket);
|
|
443
|
+
const path = `${prefix}/assessment-groups/${identifier(selected)}/observations`;
|
|
444
|
+
const readScores = async () => {
|
|
445
|
+
const value = await api(`${path}?${params}`, auth);
|
|
446
|
+
if (!scoreCohortMatches(value.cohort, { assessment_id: selected }) || value.filters.status !== (params.get('status') ?? '')
|
|
447
|
+
|| value.filters.bucket !== (params.has('bucket') ? Number(params.get('bucket')) : null))
|
|
448
|
+
throw new CliError('assessment_cohort_changed', 'The selected score group could not be confirmed.');
|
|
449
|
+
return value;
|
|
450
|
+
};
|
|
451
|
+
let data = await readScores();
|
|
452
|
+
if (interactive && options.assessmentStatus === undefined && options.scoreBucket === undefined) {
|
|
453
|
+
const filter = await selectOne('Inspect assessments', [
|
|
454
|
+
{ value: '', label: 'All retained assessments' },
|
|
455
|
+
...Object.values(scoreStatuses).map(value => ({ value: `status:${value}`, label: assessmentLabels[value] })),
|
|
456
|
+
...data.distribution.map(row => ({ value: `bucket:${row.index}`, label: `Score ${row.minimum} to ${row.upper_inclusive ? '' : 'below '}${row.maximum}`, detail: `${row.count.toLocaleString()} assessed drills` })),
|
|
457
|
+
], options.signal);
|
|
458
|
+
if (filter) {
|
|
459
|
+
const [key, value] = filter.split(':');
|
|
460
|
+
params.set(key, value);
|
|
461
|
+
params.set('page', '1');
|
|
462
|
+
data = await readScores();
|
|
463
|
+
}
|
|
464
|
+
}
|
|
465
|
+
output(data, [data.cohort.scenario_name, data.cohort.criterion,
|
|
466
|
+
`Evaluator: ${data.cohort.model} (requested ${data.cohort.requested_model})`,
|
|
467
|
+
data.cohort.direction === 'lower' ? 'Lower scores are better.' : 'Higher scores are better.',
|
|
468
|
+
...summary(data.summary, data.cohort),
|
|
469
|
+
...data.cohort.levels.map((label, index) => `${index}. ${label}`),
|
|
470
|
+
...data.distribution.map(row => `Interval ${row.index}: ${row.minimum} to ${row.upper_inclusive ? '' : 'below '}${row.maximum}: ${row.count.toLocaleString()} assessed drills`),
|
|
471
|
+
`${data.count.toLocaleString()} matching assessments`,
|
|
472
|
+
`Matching drills: firedrill report ${prefix.split('/').at(-1)} drill --workspace ${scope.workspace} --cohort ${identifier(selected)}${params.get('status') ? ' --assessment-status ' + Object.entries(scoreStatuses).find(([, value]) => value === params.get('status'))[0] : ''}${params.has('bucket') ? ' --score-bucket ' + params.get('bucket') : ''} --json`,
|
|
473
|
+
...data.results.map(row => `Drill ${row.trial_position + 1}: ${assessmentLabels[row.status] ?? 'Unavailable'}; ${number(row.score)}\n${workspaceLink(scope.workspace, `/campaigns/${prefix.split('/').at(-1)}/trials/${row.execution_id}?view=assessment&assessment=${row.assessment_id}`)}`),
|
|
474
|
+
`Page ${data.page} of ${Math.max(1, Math.ceil(data.count / data.page_size))}`,
|
|
475
|
+
...(data.pending ? ['More assessments are pending.'] : []),
|
|
476
|
+
'Intervals describe retained scores, not pass/fail thresholds. Missing evidence is not a zero score.'].join('\n'));
|
|
477
|
+
}
|
|
478
|
+
async function assessmentPreparationReport(trial, prefix, scope, interactive, options, output) {
|
|
479
|
+
const labels = { queued: 'Queued', preparing: 'Preparing', retrying: 'Retrying', stopped: 'Stopped', finished: 'Preparation finished' };
|
|
480
|
+
const states = new Set(['issues', ...Object.keys(labels)]);
|
|
481
|
+
if (options.checkpoint || options.scenario || options.checks || options.origin || options.evidenceKind || options.status && !states.has(options.status))
|
|
482
|
+
throw new CliError('arguments_invalid', 'Use --status issues|queued|preparing|retrying|stopped|finished with assessment-status. Search finds scenario names.', 2);
|
|
483
|
+
const path = trial ? `${prefix}/trials/${identifier(trial)}` : prefix;
|
|
484
|
+
const status = options.status ?? (interactive ? await selectOne('Automatic assessment preparation', [
|
|
485
|
+
{ value: 'issues', label: 'Needs review' }, { value: '', label: 'All preparation' },
|
|
486
|
+
...Object.entries(labels).map(([value, label]) => ({ value, label })),
|
|
487
|
+
], options.signal) : 'issues');
|
|
488
|
+
const params = new URLSearchParams({ page: options.page ?? '1', status, q: options.search ?? '' });
|
|
489
|
+
const data = await api(`${path}/assessment-preparation?${params}`, {
|
|
490
|
+
actor: scope.owner, workspace: scope.workspace, signal: options.signal
|
|
491
|
+
});
|
|
492
|
+
const quote = (value) => `'${value.replaceAll("'", "'\\''")}'`;
|
|
493
|
+
output(data, [data.results.map(row => `Drill ${row.trial_position + 1}: ${row.scenario_name} · ${labels[row.state] ?? 'Status unavailable'}\n${row.created_at}${row.problem ? '\n' + row.problem : ''}\n${workspaceLink(scope.workspace, `/campaigns/${prefix.split('/').at(-1)}/trials/${row.execution_id}?view=assessment`)}`).join('\n\n') || 'No matching automatic assessment preparation records.',
|
|
494
|
+
`Page ${data.page}, ${data.count} records.`, ...(data.pending ? ['Automatic assessment preparation is still in progress.'] : []),
|
|
495
|
+
...(data.next ? [`Continue: firedrill report ${prefix.split('/').at(-1)} assessment-status${trial ? ' ' + trial : ''} --workspace ${scope.workspace} --page ${data.page + 1}${status ? ' --status ' + status : ''}${options.search ? ' --search ' + quote(options.search) : ''}${options.json ? ' --json' : ''}`] : []),
|
|
496
|
+
'Preparation status is not an agent result or an evaluator score.'].join('\n'));
|
|
497
|
+
}
|
|
498
|
+
async function assessmentReport(args, prefix, scope, interactive, options, output) {
|
|
499
|
+
if (options.checkpoint || args[4] && !['policy', 'observation', 'evidence'].includes(args[4])
|
|
500
|
+
|| args[3] && (options.search || options.status || options.origin || options.page && args[4] !== 'evidence')
|
|
501
|
+
|| options.evidenceKind && args[4] !== 'evidence') {
|
|
502
|
+
throw new CliError('arguments_invalid', 'Use report <campaign> assessments [drill] [assessment] [policy|observation|evidence]. Search/status/origin apply to assessment lists; evidence has its own page and --kind.', 2);
|
|
503
|
+
}
|
|
504
|
+
const auth = { actor: scope.owner, workspace: scope.workspace, signal: options.signal };
|
|
505
|
+
let trial = args[2] ? identifier(args[2]) : undefined;
|
|
506
|
+
if (!trial && !interactive && (options.status || options.origin))
|
|
507
|
+
throw new CliError('trial_required', 'Choose a drill ID before filtering its assessment history. No assessment was requested.', 2);
|
|
508
|
+
if (!trial && !interactive) {
|
|
509
|
+
const data = await api(`${prefix}/trials?page=${options.page ?? '1'}&q=${encodeURIComponent(options.search ?? '')}`, auth);
|
|
510
|
+
output(data, data.results.map(row => `Drill ${row.position + 1}: ${row.scenario_name} · ${row.state}\n${row.id}`).join('\n\n') || 'No retained drills.');
|
|
511
|
+
return;
|
|
512
|
+
}
|
|
513
|
+
const chosenTrial = !trial;
|
|
514
|
+
if (!trial)
|
|
515
|
+
trial = identifier((await choosePage('Drill assessments', `${prefix}/trials`, scope, options, row => ({ label: `Drill ${row.position + 1}: ${row.scenario_name}`, detail: row.state }))).id);
|
|
516
|
+
const path = `${prefix}/trials/${trial}`;
|
|
517
|
+
const search = chosenTrial ? '' : options.search ?? '';
|
|
518
|
+
const list = new URLSearchParams({ q: search, status: options.status ?? '', origin: options.origin ?? '' });
|
|
519
|
+
let assessment = args[3] ? identifier(args[3]) : undefined;
|
|
520
|
+
if (!assessment && !interactive) {
|
|
521
|
+
const data = await api(`${path}/assessments?${list}&page=${options.page ?? '1'}`, auth);
|
|
522
|
+
if (!assessmentPageMatches(data, prefix.split('/').at(-1), trial))
|
|
523
|
+
throw new CliError('assessment_report_changed', 'The selected assessment history could not be confirmed.');
|
|
524
|
+
const quote = (value) => `'${value.replaceAll("'", "'\\''")}'`;
|
|
525
|
+
output(data, (data.results.map(row => `${row.name}: ${assessmentLabels[row.state] ?? row.state.replaceAll('_', ' ')}${typeof row.score === 'number' ? ` (${row.score} / ${row.maximum})${row.band ? ' · ' + row.band.label : ''}` : ''}${row.checkpoint ? `\nCheckpoint at turn ${row.checkpoint.turn}. Provisional, excluded from final scores and CI.` : row.partial_execution ? '\nPartial evidence, excluded from final scores and CI.' : ''}\n${row.id}`).join('\n\n') || 'No matching assessments.')
|
|
526
|
+
+ `\nPage ${data.page}, ${data.count} assessments.${data.next ? `\nContinue: firedrill report ${prefix.split('/').at(-1)} assessments ${trial} --workspace ${scope.workspace} --page ${data.page + 1}${search ? ' --search ' + quote(search) : ''}${options.status ? ' --status ' + options.status : ''}${options.origin ? ' --origin ' + options.origin : ''}${options.json ? ' --json' : ''}` : ''}`
|
|
527
|
+
+ (data.automatic_preparation.issue_trials ? `\nAutomatic assessment preparation needs review. Inspect: firedrill report ${prefix.split('/').at(-1)} assessment-status ${trial} --workspace ${scope.workspace}` : ''));
|
|
528
|
+
return;
|
|
529
|
+
}
|
|
530
|
+
if (!assessment)
|
|
531
|
+
assessment = identifier((await choosePage('Assessment', `${path}/assessments?status=${options.status ?? ''}&origin=${options.origin ?? ''}`, scope, { ...options, page: chosenTrial ? undefined : options.page, search }, row => ({ label: row.name,
|
|
532
|
+
detail: `${assessmentLabels[row.state] ?? row.state.replaceAll('_', ' ')}${row.score === null ? '' : ` · ${row.score} / ${row.maximum}`}${row.checkpoint ? ' · Provisional checkpoint' : row.partial_execution ? ' · Partial evidence' : ''}` }))).id);
|
|
533
|
+
const row = await api(`${path}/assessments/${assessment}`, auth);
|
|
534
|
+
if (!assessmentMatches(row, prefix.split('/').at(-1), trial, assessment))
|
|
535
|
+
throw new CliError('assessment_report_changed', 'The selected assessment could not be confirmed. No evaluation was repeated.');
|
|
536
|
+
output(row, [`${row.name} · ${row.state.replaceAll('_', ' ')}`,
|
|
537
|
+
typeof row.result?.score === 'number' ? `Score: ${row.result.score} / ${row.result.levels.length - 1}. ${row.rubric.direction === 'lower' ? 'Lower' : 'Higher'} is better.` : 'No numeric score recorded.',
|
|
538
|
+
`${row.provider} · ${row.model ?? 'Requested ' + row.requested_model}`,
|
|
539
|
+
...(row.checkpoint ? [`Checkpoint at turn ${row.checkpoint.turn}, ${row.checkpoint.observed_at}. Provisional, excluded from final scores and CI.`]
|
|
540
|
+
: row.partial_execution ? ['Assessed from partial evidence, excluded from final scores and CI.'] : []),
|
|
541
|
+
...(row.problem ? [row.problem] : []), ...(row.result?.company_policies ?? []).map(policy => `${policy.name} · Version ${policy.version}`),
|
|
542
|
+
...(row.result?.explanation ? [`Evidence explanation: ${row.result.explanation.state.replaceAll('_', ' ')}`,
|
|
543
|
+
...(row.result.explanation.state === 'succeeded' ? [`Explanation, linked to the saved evidence.`,
|
|
544
|
+
...row.result.explanation.observations.flatMap(item => [item.text,
|
|
545
|
+
...item.evidence.map(ref => `${ref.kind}${ref.kind === 'state' || ref.position === null ? '' : ` · Position ${ref.position + 1}`} · ${ref.id}\n${ref.quote}\nInspect: firedrill report ${prefix.split('/').at(-1)} drill ${trial} ${ref.kind === 'state' ? 'state checkpoints' : ref.kind === 'message' ? 'messages' : 'actions'} ${ref.id} --workspace ${scope.workspace}`),
|
|
546
|
+
...item.policies.map(ref => `Policy ${ref.revision_id} · ${ref.paragraph_id}\n${ref.quote}`)]), row.result.explanation.limitation]
|
|
547
|
+
: [row.result.explanation.problem || 'The cited explanation is pending. Reading this report does not repeat evaluation.'])] : []),
|
|
548
|
+
row.rubric.instructions ?? '', ...((row.rubric.levels ?? []).map((level, index) => `${index}. ${level}`)),
|
|
549
|
+
workspaceLink(scope.workspace, `/campaigns/${args[0] ?? prefix.split('/').at(-1)}/trials/${trial}?view=assessment&assessment=${assessment}`)].filter(Boolean).join('\n'));
|
|
550
|
+
const support = row.result?.policy_support;
|
|
551
|
+
let inspect = args[4];
|
|
552
|
+
if (!inspect && interactive && (support?.policy || support?.observation || row.evidence_count))
|
|
553
|
+
inspect = await selectOne('Inspect supporting evidence', [
|
|
554
|
+
{ value: 'done', label: 'Done' }, ...(support?.policy ? [{ value: 'policy', label: 'Policy passage' }] : []),
|
|
555
|
+
...(support?.observation ? [{ value: 'observation', label: support.observation.kind === 'state' ? 'Observed service state' : 'Recorded message or tool action' }] : []),
|
|
556
|
+
...(row.evidence_count ? [{ value: 'evidence', label: 'Evidence files' }] : [])
|
|
557
|
+
], options.signal);
|
|
558
|
+
if (inspect === 'policy') {
|
|
559
|
+
const selected = support?.policy;
|
|
560
|
+
if (!selected || !/^p[0-9]{1,11}$/.test(selected.paragraph_id))
|
|
561
|
+
throw new CliError('assessment_anchor_unavailable', 'No available supporting policy passage was retained.');
|
|
562
|
+
const data = await api(`/api/workspaces/${scope.workspace}/agents/${identifier(row.agent_id)}/policies/${identifier(selected.revision_id)}/paragraphs/${selected.paragraph_id}`, auth);
|
|
563
|
+
if (data.policy.id !== selected.revision_id || data.policy.agent_id !== row.agent_id || data.paragraph.id !== selected.paragraph_id)
|
|
564
|
+
throw new CliError('assessment_anchor_changed', 'The original policy passage could not be confirmed.');
|
|
565
|
+
output(data, `${data.policy.name} · Version ${data.policy.version}\n${data.paragraph.id}${data.paragraph.page ? ` · Page ${data.paragraph.page}` : ''}\n${data.paragraph.text}`);
|
|
566
|
+
}
|
|
567
|
+
else if (inspect === 'observation') {
|
|
568
|
+
const selected = support?.observation;
|
|
569
|
+
if (!selected || !['message', 'action', 'state'].includes(selected.kind))
|
|
570
|
+
throw new CliError('assessment_anchor_unavailable', 'No available supporting drill observation was retained.');
|
|
571
|
+
const data = await api(`${path}/${selected.kind === 'state' ? 'state-comparisons/checkpoints' : selected.kind === 'message' ? 'messages' : 'actions'}/${identifier(selected.id)}`, auth);
|
|
572
|
+
if (!assessmentEvidenceMatches(data, selected))
|
|
573
|
+
throw new CliError('assessment_anchor_changed', 'The original supporting observation could not be confirmed.');
|
|
574
|
+
output(data, 'sequence' in data ? `${checkpointLabel(data)} · ${data.status}\nInspect: firedrill report ${prefix.split('/').at(-1)} drill ${trial} state checkpoints ${data.id} --workspace ${scope.workspace}` : JSON.stringify(data, null, 2));
|
|
575
|
+
}
|
|
576
|
+
else if (inspect === 'evidence') {
|
|
577
|
+
const data = await api(`${path}/assessments/${assessment}/evidence?page=${options.page ?? '1'}&kind=${options.evidenceKind ?? ''}`, auth);
|
|
578
|
+
output(data, (data.results.map(file => `${file.kind === 'transcript' ? 'Conversation' : file.kind === 'state_snapshot' ? 'Observed service state' : 'Tool actions'} · ${file.size} bytes · ${file.created_at}\n${file.id}\n${APP_ORIGIN}${path}/files/${file.id}/download`).join('\n\n') || 'No matching assessment evidence files.')
|
|
579
|
+
+ `\nPage ${data.page}, ${data.count} files.${data.next ? `\nContinue: firedrill report ${prefix.split('/').at(-1)} assessments ${trial} ${assessment} evidence --workspace ${scope.workspace} --page ${data.page + 1}${options.evidenceKind ? ' --kind ' + options.evidenceKind : ''}${options.json ? ' --json' : ''}` : ''}`);
|
|
580
|
+
}
|
|
581
|
+
}
|
|
582
|
+
export async function reportCommand(args, options, output) {
|
|
583
|
+
// Keep retained API/type identities stable while the terminal uses Drill.
|
|
584
|
+
args = args.map((value, index) => index === 1 && value === 'drill' ? 'trial'
|
|
585
|
+
: index === 5 && value === 'drills' ? 'trials' : value);
|
|
586
|
+
const interactive = !options.json && !!process.stdin.isTTY && !!process.stderr.isTTY;
|
|
587
|
+
if ([options.evidenceAttempt, options.evidenceStep, options.evidenceFrom, options.evidenceTo].some(value => value !== undefined)
|
|
588
|
+
&& !(args[1] === 'trial' && args[2] && ['evidence', 'timeline'].includes(args[3])))
|
|
589
|
+
throw new CliError('arguments_invalid', 'Use evidence filters only with report <campaign> drill <drill> evidence or timeline.', 2);
|
|
590
|
+
if (args[1] === 'trial' && args[2] && args[3] === 'world-events') {
|
|
591
|
+
if ([options.checkpoint, options.scenario, options.checks, options.cohort, options.origin,
|
|
592
|
+
options.serviceRequest, options.scoreBucket, options.assessmentStatus, options.evaluator,
|
|
593
|
+
options.checkMethod, options.side, options.evidenceKind, options.into].some(value => value !== undefined) || options.approveDownload)
|
|
594
|
+
throw new CliError('arguments_invalid', 'Use report <campaign> drill <drill> world-events with page, search or event status only.', 2);
|
|
595
|
+
const scope = await selectedWorkspace({ workspace: options.workspace, interactive, openBrowser: options.openBrowser, signal: options.signal });
|
|
596
|
+
await campaignWorldCommand(args.slice(4), `/api/workspaces/${scope.workspace}/campaigns/${identifier(args[0])}/trials/${identifier(args[2])}`, scope, options, output);
|
|
597
|
+
return;
|
|
598
|
+
}
|
|
599
|
+
if (options.file !== undefined || options.change !== undefined)
|
|
600
|
+
throw new CliError('arguments_invalid', 'Use --file and --approve-change only with drill world-events controls.', 2);
|
|
601
|
+
if (args[1] === 'trial' && args[2] && ['evidence', 'timeline'].includes(args[3])) {
|
|
602
|
+
if (args.length > 5 || !args[0] || options.search !== undefined || options.checkpoint !== undefined || options.status !== undefined
|
|
603
|
+
|| options.scenario !== undefined || options.checks !== undefined || options.cohort !== undefined || options.origin !== undefined
|
|
604
|
+
|| options.requestKey !== undefined || options.serviceRequest !== undefined || options.scoreBucket !== undefined || options.assessmentStatus !== undefined
|
|
605
|
+
|| options.evaluator !== undefined || options.checkMethod !== undefined || options.side !== undefined
|
|
606
|
+
|| options.page !== undefined && !/^[1-9][0-9]{0,8}$/.test(options.page))
|
|
607
|
+
throw new CliError('arguments_invalid', 'Use report <campaign> drill <drill> evidence [artifact] or timeline [driver-step]. --kind filters evidence or observation type; --into and --approve-download apply to evidence.', 2);
|
|
608
|
+
const scope = await selectedWorkspace({ workspace: options.workspace, interactive, openBrowser: options.openBrowser, signal: options.signal });
|
|
609
|
+
const execution = identifier(args[2]);
|
|
610
|
+
const path = `/api/workspaces/${scope.workspace}/campaigns/${identifier(args[0])}/trials/${execution}`;
|
|
611
|
+
if (args[3] === 'evidence')
|
|
612
|
+
await evidenceCommand(path, execution, args[4], scope, interactive, options, output);
|
|
613
|
+
else
|
|
614
|
+
await timelineCommand(path, execution, args[4], scope, options, output);
|
|
615
|
+
return;
|
|
616
|
+
}
|
|
617
|
+
const stateInspection = args[1] === 'trial' && Boolean(args[2]) && args[3] === 'state';
|
|
618
|
+
const nativeCheckView = args[1] === 'checks' || args[1] === 'compare' && args[3] === 'checks';
|
|
619
|
+
const compositeView = args[1] === 'compare' && args[3] === 'composite';
|
|
620
|
+
if (options.evaluator !== undefined && (!compositeView || !options.evaluator.length || options.evaluator.length > 128 || /[\u0000-\u001f\u007f]/.test(options.evaluator)))
|
|
621
|
+
throw new CliError('arguments_invalid', 'Use --evaluator <version|unavailable> with composite comparison drills.', 2);
|
|
622
|
+
if (options.checkMethod !== undefined && (!nativeCheckView || !['', 'exact', 'semantic', 'both', 'unknown'].includes(options.checkMethod)))
|
|
623
|
+
throw new CliError('arguments_invalid', 'Use --method exact|semantic|both|unknown with a recorded-check group list.', 2);
|
|
624
|
+
if (options.side !== undefined && (!['compare', 'checks'].includes(args[1]) || !['current', 'baseline', 'both'].includes(options.side)))
|
|
625
|
+
throw new CliError('arguments_invalid', 'Use --side current|baseline|both with selected check or comparison observations.', 2);
|
|
626
|
+
if (stateInspection)
|
|
627
|
+
validateTrialStateOptions(args.slice(4), options);
|
|
628
|
+
if (args[1] === 'trial' && args[3] && !['messages', 'actions', 'requests', 'checks', 'check-observation', 'coverage', 'state', 'schedule', 'captures', 'desktop-captures', 'recordings', 'clips'].includes(args[3]))
|
|
629
|
+
throw new CliError('arguments_invalid', 'Choose messages, actions, requests, checks, check-observation, coverage, state, schedule, captures, desktop-captures, recordings or clips from the drill report.', 2);
|
|
630
|
+
if (options.serviceRequest !== undefined) {
|
|
631
|
+
if (args[1] !== 'trial' || !args[2] || args[3] !== 'actions' || args[4] || options.checkpoint !== undefined
|
|
632
|
+
|| options.scenario !== undefined || options.checks !== undefined)
|
|
633
|
+
throw new CliError('arguments_invalid', 'Use --service-request <receipt-id> with report <campaign> drill <drill> actions.', 2);
|
|
634
|
+
validateActionOptions(options);
|
|
635
|
+
}
|
|
636
|
+
if (args[1] === 'export') {
|
|
637
|
+
validateExport(args.slice(2), options);
|
|
638
|
+
if (options.checkpoint !== undefined)
|
|
639
|
+
throw new CliError('arguments_invalid', 'Checkpoint selection does not apply to report exports.', 2);
|
|
640
|
+
}
|
|
641
|
+
else if (options.requestKey !== undefined || options.into !== undefined || options.approveDownload !== undefined) {
|
|
642
|
+
throw new CliError('arguments_invalid', 'Use export request and download options with report <campaign> export.', 2);
|
|
643
|
+
}
|
|
644
|
+
if (options.cohort !== undefined) {
|
|
645
|
+
if (!(args[1] === 'trial' && !args[2] || args[1] === 'export' && args[2] === 'create') || options.checkpoint !== undefined)
|
|
646
|
+
throw new CliError('arguments_invalid', 'Use --cohort <assessment> with a drill list or export create.', 2);
|
|
647
|
+
identifier(options.cohort);
|
|
648
|
+
}
|
|
649
|
+
if (options.scoreBucket !== undefined || options.assessmentStatus !== undefined) {
|
|
650
|
+
if (!(args[1] === 'scores' && args[2] || (args[1] === 'trial' && !args[2] || args[1] === 'export' && args[2] === 'create') && options.cohort) || options.checkpoint !== undefined
|
|
651
|
+
|| options.scoreBucket !== undefined && !/^[0-8]$/.test(options.scoreBucket)
|
|
652
|
+
|| options.assessmentStatus !== undefined && !Object.hasOwn(scoreStatuses, options.assessmentStatus)
|
|
653
|
+
|| options.scoreBucket !== undefined && options.assessmentStatus !== undefined && options.assessmentStatus !== 'assessed') {
|
|
654
|
+
throw new CliError('arguments_invalid', 'Use score filters with scores <cohort-assessment>, drill --cohort <assessment>, or export create --cohort <assessment>. A score interval applies only to assessed drills.', 2);
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
if (options.scenario !== undefined || options.status !== undefined || options.checks !== undefined) {
|
|
658
|
+
const requestList = args[1] === 'trial' && !!args[2] && args[3] === 'requests' && !args[4]
|
|
659
|
+
&& options.scenario === undefined && options.checks === undefined;
|
|
660
|
+
const actionList = args[1] === 'trial' && !!args[2] && args[3] === 'actions' && !args[4]
|
|
661
|
+
&& options.scenario === undefined && options.checks === undefined;
|
|
662
|
+
const checkList = args[1] === 'trial' && !!args[2] && args[3] === 'checks' && !args[4]
|
|
663
|
+
&& options.scenario === undefined && options.checks === undefined;
|
|
664
|
+
const preparationList = args[1] === 'assessment-status' && args.length <= 3 && options.scenario === undefined && options.checks === undefined;
|
|
665
|
+
const assessmentList = args[1] === 'assessments' && !args[3] && options.scenario === undefined && options.checks === undefined;
|
|
666
|
+
const stateEvents = stateInspection && args[5] === 'events' && options.scenario === undefined && options.checks === undefined;
|
|
667
|
+
const nativeCheckFilter = nativeCheckView && options.scenario === undefined && options.checks === undefined;
|
|
668
|
+
const compositeFilter = compositeView && options.scenario === undefined && options.checks === undefined;
|
|
669
|
+
if (!(actionList || requestList || checkList || nativeCheckFilter || compositeFilter || stateEvents || assessmentList || preparationList || args[1] === 'assignments' && !args[2] || args[1] === 'trial' && !args[2] || args[1] === 'export' && args[2] === 'create') || options.checkpoint !== undefined)
|
|
670
|
+
throw new CliError('arguments_invalid', 'Use status with report observation lists, or execution/scenario/check filters with drill lists, assignments and export create.', 2);
|
|
671
|
+
if (options.scenario)
|
|
672
|
+
identifier(options.scenario);
|
|
673
|
+
if (requestList)
|
|
674
|
+
validateReceiptOptions(options);
|
|
675
|
+
if (actionList)
|
|
676
|
+
validateActionOptions(options);
|
|
677
|
+
if (checkList)
|
|
678
|
+
validateCheckOptions(options);
|
|
679
|
+
if (nativeCheckFilter && options.status && !['met', 'unmet', 'unconfirmed'].includes(options.status))
|
|
680
|
+
throw new CliError('arguments_invalid', 'Choose met, unmet or unconfirmed recorded checks.', 2);
|
|
681
|
+
if (compositeFilter && options.status !== undefined && !['all', 'available', 'unavailable'].includes(options.status))
|
|
682
|
+
throw new CliError('arguments_invalid', 'Choose all, available or unavailable composite drill indices.', 2);
|
|
683
|
+
if (options.status && !actionList && !requestList && !checkList && !nativeCheckFilter && !compositeFilter && !stateEvents && !preparationList && !(assessmentList ? assessmentStates : trialStates).has(options.status) || options.checks && !Object.hasOwn(trialChecks, options.checks)) {
|
|
684
|
+
throw new CliError('arguments_invalid', 'Choose a recorded execution status and met, unmet, unassessed, unavailable or missing checks.', 2);
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
if (options.origin !== undefined && (args[1] !== 'assessments' || args[3] || !['manual', 'automatic'].includes(options.origin)))
|
|
688
|
+
throw new CliError('arguments_invalid', 'Use --origin manual|automatic with an assessment list.', 2);
|
|
689
|
+
if (options.evidenceKind !== undefined) {
|
|
690
|
+
if (stateInspection)
|
|
691
|
+
validateTrialStateOptions(args.slice(4), options);
|
|
692
|
+
else if (args[1] === 'trial' && args[2] && args[3] === 'checks' && !args[4])
|
|
693
|
+
validateCheckOptions(options);
|
|
694
|
+
else if (args[1] !== 'assessments' || args[4] !== 'evidence' || !['transcript', 'tool_trace', 'state_snapshot'].includes(options.evidenceKind))
|
|
695
|
+
throw new CliError('arguments_invalid', 'Use --kind with assessment evidence, a drill recorded-check list or observed state changes.', 2);
|
|
696
|
+
}
|
|
697
|
+
if (args.length > (['export', 'activity'].includes(args[1]) ? 4 : args[1] === 'compare' ? args[3] === 'composite' ? 6 : 5 : stateInspection ? 8 : args[1] === 'trial' && args[3] === 'schedule' ? 6 : ['assessments', 'trial'].includes(args[1]) ? 5 : 3) || args[1] && !['memory', 'composite', 'compare', 'style', 'usage', 'checks', 'assessments', 'assessment-status', 'trial', 'scores', 'findings', 'export', 'assignments', 'activity', 'insights'].includes(args[1]) || args[2] && !['memory', 'composite', 'compare', 'style', 'usage', 'checks', 'assessments', 'assessment-status', 'trial', 'scores', 'findings', 'export', 'activity'].includes(args[1])
|
|
698
|
+
|| options.page !== undefined && !/^[1-9][0-9]{0,8}$/.test(options.page) || (options.search?.length ?? 0) > 200
|
|
699
|
+
|| options.checkpoint !== undefined && !/^[a-z][a-z0-9_]{0,47}$/.test(options.checkpoint)) {
|
|
700
|
+
throw new CliError('arguments_invalid', 'Choose a report: drill, assignments, activity, insights, findings, checks, scores, assessments, memory, style, usage, composite, compare or export. Run firedrill report interactively for guided selection.', 2);
|
|
701
|
+
}
|
|
702
|
+
if (args[1] === 'activity' && (options.page !== undefined || options.checkpoint !== undefined
|
|
703
|
+
|| /[\u0000-\u001f\u007f-\u009f\u202a-\u202e\u2066-\u2069]/.test(options.search ?? '')
|
|
704
|
+
|| !/^[0-9]{1,19}$/.test(args[2] ?? '0') || BigInt(args[2] ?? '0') > 9223372036854775807n
|
|
705
|
+
|| args[3] !== undefined && !/^[a-z][a-z0-9_.]{0,63}$/.test(args[3]))) {
|
|
706
|
+
throw new CliError('arguments_invalid', 'Use report <campaign> activity [before-sequence] [activity-type] with optional --search for recorded services or operations. Omit the cursor for latest activity.', 2);
|
|
707
|
+
}
|
|
708
|
+
if (args[1] === 'scores') {
|
|
709
|
+
if (options.checkpoint !== undefined || args[2] && options.search)
|
|
710
|
+
throw new CliError('arguments_invalid', 'Search applies to score groups. Use a cohort assessment ID to inspect its observations, without a checkpoint.', 2);
|
|
711
|
+
if (args[2])
|
|
712
|
+
identifier(args[2]);
|
|
713
|
+
}
|
|
714
|
+
if (args[1] === 'findings') {
|
|
715
|
+
if (options.checkpoint !== undefined || args[2] && (options.search || options.page))
|
|
716
|
+
throw new CliError('arguments_invalid', 'Search and paging apply to the scenario directory, not one selected scenario. Checkpoints do not apply to findings.', 2);
|
|
717
|
+
if (args[2])
|
|
718
|
+
identifier(args[2]);
|
|
719
|
+
}
|
|
720
|
+
let campaign = args[0] ? identifier(args[0]) : undefined;
|
|
721
|
+
if (!campaign && !interactive)
|
|
722
|
+
throw new CliError('campaign_required', 'Pass a saved campaign ID. Use firedrill simulations --json to find it.', 2);
|
|
723
|
+
const scope = await selectedWorkspace({ workspace: options.workspace, interactive, openBrowser: options.openBrowser, signal: options.signal });
|
|
724
|
+
if (!campaign) {
|
|
725
|
+
const selected = await choosePage('Simulation report', `/api/workspaces/${scope.workspace}/simulations`, scope, options, row => ({ label: row.name, detail: row.status }));
|
|
726
|
+
campaign = identifier(selected.campaign_id);
|
|
727
|
+
}
|
|
728
|
+
const prefix = `/api/workspaces/${scope.workspace}/campaigns/${campaign}`;
|
|
729
|
+
const mode = args[1] ?? (interactive ? await selectOne('Report', [{ value: 'summary', label: 'Summary' },
|
|
730
|
+
{ value: 'insights', label: 'Retained recommendations' },
|
|
731
|
+
{ value: 'trial', label: 'Drill details and input/output' },
|
|
732
|
+
{ value: 'assignments', label: 'Current execution slots' },
|
|
733
|
+
{ value: 'activity', label: 'Latest activity and completions' },
|
|
734
|
+
{ value: 'assessments', label: 'Evidence-based assessments' },
|
|
735
|
+
{ value: 'assessment-status', label: 'Automatic assessment preparation' },
|
|
736
|
+
{ value: 'scores', label: 'Score distributions and drills' },
|
|
737
|
+
{ value: 'findings', label: 'Scenario findings and example drills' },
|
|
738
|
+
{ value: 'checks', label: 'Recorded checks' }, { value: 'memory', label: 'Memory checkpoints' }, { value: 'style', label: 'Style observations' }, { value: 'usage', label: 'Target model usage' },
|
|
739
|
+
{ value: 'composite', label: 'Composite indices' }, { value: 'compare', label: 'Compare saved simulations' },
|
|
740
|
+
{ value: 'export', label: 'Saved report exports' }], options.signal) : 'summary');
|
|
741
|
+
if (mode === 'export') {
|
|
742
|
+
await reportExport(args.slice(2), prefix, scope, interactive, options, output);
|
|
743
|
+
return;
|
|
744
|
+
}
|
|
745
|
+
if (mode === 'trial') {
|
|
746
|
+
await trialReport(args, prefix, scope, interactive, options, output);
|
|
747
|
+
return;
|
|
748
|
+
}
|
|
749
|
+
if (mode === 'insights') {
|
|
750
|
+
if (options.page !== undefined || options.search !== undefined || options.checkpoint !== undefined)
|
|
751
|
+
throw new CliError('arguments_invalid', 'Insights reads the retained review only. Paging and checkpoint options do not apply.', 2);
|
|
752
|
+
const value = await api(`${prefix}/insights`, { actor: scope.owner, workspace: scope.workspace, signal: options.signal });
|
|
753
|
+
if (!insightsMatch(value, campaign))
|
|
754
|
+
throw new CliError('report_unavailable', 'These retained recommendations could not be confirmed. Reopen the report.');
|
|
755
|
+
output(value, [`Review: ${value.state.replaceAll('_', ' ')}`, value.content, value.problem,
|
|
756
|
+
value.state === 'unavailable' && !value.problem ? 'No retained recommendations are available. Execution results remain in the summary.' : '',
|
|
757
|
+
value.proposal_count ? `${value.proposal_count} retained recommendations.` : '',
|
|
758
|
+
workspaceLink(scope.workspace, `/campaigns/${campaign}?view=insights`)].filter(Boolean).join('\n\n'));
|
|
759
|
+
return;
|
|
760
|
+
}
|
|
761
|
+
if (mode === 'assignments') {
|
|
762
|
+
if (options.checkpoint !== undefined)
|
|
763
|
+
throw new CliError('arguments_invalid', 'Checkpoint selection does not apply to current execution slots.', 2);
|
|
764
|
+
if (/[\u0000-\u001f\u007f-\u009f\u202a-\u202e\u2066-\u2069]/.test(options.search ?? ''))
|
|
765
|
+
throw new CliError('arguments_invalid', 'Use search text without terminal control characters.', 2);
|
|
766
|
+
const quote = (value) => `'${value.replaceAll("'", "'\\''")}'`;
|
|
767
|
+
const filters = new URLSearchParams({ page: options.page ?? '1', q: options.search ?? '' });
|
|
768
|
+
if (options.status)
|
|
769
|
+
filters.set('status', options.status);
|
|
770
|
+
if (options.checks)
|
|
771
|
+
filters.set('checks', options.checks);
|
|
772
|
+
if (options.scenario)
|
|
773
|
+
filters.set('scenario', identifier(options.scenario));
|
|
774
|
+
const value = await api(`${prefix}/assignments?${filters}`, { actor: scope.owner, signal: options.signal });
|
|
775
|
+
if (value.campaign_id !== campaign)
|
|
776
|
+
throw new CliError('campaign_changed', 'The simulation could not be confirmed. Open it again.', 2);
|
|
777
|
+
output(value, [(value.results.map(row => `Drill ${(row.position + 1).toLocaleString()}: ${row.scenario_name}\n${executionPhase(row.phase)}${row.attempt_number !== null && row.attempt_number > 1 ? ` · Attempt ${row.attempt_number}` : ''}\n${workspaceLink(scope.workspace, `/campaigns/${campaign}/trials/${row.execution_id}`)}`).join('\n\n') || 'No execution slots are currently assigned for these filters.'),
|
|
778
|
+
`Page ${value.page}, ${value.count.toLocaleString()} matching assignments. Assignment is not proof of agent invocation or a successful outcome.`,
|
|
779
|
+
...(value.next ? [`Continue: firedrill report ${campaign} assignments --workspace ${scope.workspace} --page ${value.page + 1}${options.search ? ' --search ' + quote(options.search) : ''}${options.status ? ' --status ' + options.status : ''}${options.checks ? ' --checks ' + options.checks : ''}${options.scenario ? ' --scenario ' + identifier(options.scenario) : ''} --json`] : [])].join('\n\n'));
|
|
780
|
+
return;
|
|
781
|
+
}
|
|
782
|
+
if (mode === 'activity') {
|
|
783
|
+
if (options.page !== undefined || options.checkpoint !== undefined)
|
|
784
|
+
throw new CliError('arguments_invalid', 'Activity uses the retained sequence cursor, not page or checkpoint options.', 2);
|
|
785
|
+
const kind = args[3] ?? '', cursor = args[2] ?? '0', search = (options.search ?? '').trim();
|
|
786
|
+
const value = await api(`${prefix}/events?${new URLSearchParams({ before: cursor, kind, q: search })}`, { actor: scope.owner, signal: options.signal });
|
|
787
|
+
if (value.campaign_id !== campaign || value.direction !== 'older' || value.kind !== kind || value.q !== search || !/^[0-9]{1,19}$/.test(value.next_cursor))
|
|
788
|
+
throw new CliError('campaign_activity_changed', 'The saved activity could not be confirmed. Open it again.', 2);
|
|
789
|
+
const quote = (input) => `'${input.replaceAll("'", "'\\''")}'`;
|
|
790
|
+
output(value, [(value.results.map(row => `${row.created_at}: ${activityLabel(row.kind)}${typeof row.content.service_key === 'string' && row.content.service_key ? ` · ${row.content.service_key}` : ''}${typeof row.content.operation === 'string' && row.content.operation ? ` · ${row.content.operation}` : ''}${typeof row.content.http_status === 'number' ? ` · HTTP ${row.content.http_status}` : ''}${row.execution_id ? `\n${workspaceLink(scope.workspace, `/campaigns/${campaign}/trials/${row.execution_id}${row.kind.startsWith('dependency.') ? '?view=requests' : ''}`)}` : ''}`).join('\n\n') || 'No matching activity retained.'),
|
|
791
|
+
...(value.has_more ? [`Older: firedrill report ${campaign} activity ${value.next_cursor}${kind ? ' ' + kind : ''} --workspace ${scope.workspace}${search ? ' --search ' + quote(search) : ''} --json`] : [])].join('\n\n'));
|
|
792
|
+
return;
|
|
793
|
+
}
|
|
794
|
+
if (mode === 'assessments') {
|
|
795
|
+
await assessmentReport(args, prefix, scope, interactive, options, output);
|
|
796
|
+
return;
|
|
797
|
+
}
|
|
798
|
+
if (mode === 'assessment-status') {
|
|
799
|
+
await assessmentPreparationReport(args[2], prefix, scope, interactive, options, output);
|
|
800
|
+
return;
|
|
801
|
+
}
|
|
802
|
+
if (mode === 'scores') {
|
|
803
|
+
await scoreReport(args[2], prefix, scope, interactive, options, output);
|
|
804
|
+
return;
|
|
805
|
+
}
|
|
806
|
+
if (mode === 'findings') {
|
|
807
|
+
await findingsReport(args[2], prefix, scope, interactive, options, output);
|
|
808
|
+
return;
|
|
809
|
+
}
|
|
810
|
+
if (mode === 'compare') {
|
|
811
|
+
await comparisonCommand(args, prefix, scope, interactive, options, output);
|
|
812
|
+
return;
|
|
813
|
+
}
|
|
814
|
+
if (mode === 'checks') {
|
|
815
|
+
await nativeCheckCommand(prefix, scope, interactive, options, output, args[2]);
|
|
816
|
+
return;
|
|
817
|
+
}
|
|
818
|
+
if (mode === 'style' || mode === 'usage') {
|
|
819
|
+
if (options.checkpoint)
|
|
820
|
+
throw new CliError('arguments_invalid', 'Checkpoint selection applies only to memory reports.', 2);
|
|
821
|
+
if (mode === 'usage') {
|
|
822
|
+
if ((options.search?.length ?? 0) > 128)
|
|
823
|
+
throw new CliError('arguments_invalid', 'Use a model search of up to 128 characters.', 2);
|
|
824
|
+
const execution = args[2] ? identifier(args[2]) : undefined;
|
|
825
|
+
const query = new URLSearchParams({ page: options.page ?? '1', model: options.search ?? '' });
|
|
826
|
+
const data = await api(`${prefix}${execution ? `/trials/${execution}` : ''}/target-usage?${query}`, { actor: scope.owner, workspace: scope.workspace, signal: options.signal });
|
|
827
|
+
if (!targetUsageMatches(data, campaign, execution, options.search ?? ''))
|
|
828
|
+
throw new CliError('report_unavailable', 'These target-model records could not be confirmed. Reopen the report.');
|
|
829
|
+
const count = (value) => typeof value === 'string' && /^[0-9]+$/.test(value) ? BigInt(value).toLocaleString() : 'Not reported';
|
|
830
|
+
const quote = (value) => "'" + value.replaceAll("'", "'\\''") + "'";
|
|
831
|
+
output(data, [`${data.summary.requests.toLocaleString()} target-model request records`,
|
|
832
|
+
`Completed: ${data.summary.completed}; failed: ${data.summary.failed}; interrupted: ${data.summary.interrupted}; unfinished: ${data.summary.pending}`,
|
|
833
|
+
...['input_tokens', 'output_tokens', 'cache_creation_input_tokens', 'cache_read_input_tokens'].map(key => `${key.replaceAll('_', ' ')}: ${count(data.summary[key])} (${data.summary[`${key}_coverage`]} requests)`),
|
|
834
|
+
`Mean relay elapsed: ${data.summary.mean_elapsed_ms === null ? 'Not measured' : `${data.summary.mean_elapsed_ms}ms`}`,
|
|
835
|
+
...data.results.map(row => `Drill ${row.trial_position + 1}: ${row.requested_model}; ${row.provider}; ${row.operation}; ${row.state.replaceAll('_', ' ')}; input ${count(row.usage.input_tokens)}; output ${count(row.usage.output_tokens)}\n${targetCompletionNotes(row).join(' ')}\n${workspaceLink(scope.workspace, `/campaigns/${campaign}/trials/${row.execution_id}`)}`),
|
|
836
|
+
`Page ${data.page} of ${Math.max(1, Math.ceil(data.count / data.page_size))}`,
|
|
837
|
+
...(data.next ? [`Continue: firedrill report ${campaign} usage${execution ? ' ' + execution : ''} --workspace ${scope.workspace} --page ${data.page + 1}${options.search ? ' --search ' + quote(options.search) : ''}${options.json ? ' --json' : ''}`] : []),
|
|
838
|
+
'Provider-reported target usage only, not Firedrill preparation/evaluation spend or billed dollar cost. Relay elapsed includes network handling.'].join('\n'));
|
|
839
|
+
return;
|
|
840
|
+
}
|
|
841
|
+
let assessment = args[2] ? identifier(args[2]) : undefined;
|
|
842
|
+
if (!assessment && !interactive) {
|
|
843
|
+
const data = await api(`${prefix}/style-groups?page=${options.page ?? '1'}&q=${encodeURIComponent(options.search ?? '')}`, { actor: scope.owner, workspace: scope.workspace, signal: options.signal });
|
|
844
|
+
output(data, [...data.results.flatMap(row => [`${row.cohort.scenario_name}: ${row.cohort.criterion} (${row.cohort.assessment_id})`,
|
|
845
|
+
`${row.summary.occurrences === null ? 'Unmeasured' : row.summary.occurrences.toLocaleString()} occurrences; ${row.summary.assessed} observed of ${row.summary.observed} retained drills; ${row.summary.partial} partial`]),
|
|
846
|
+
'Selected style measurements, not semantic quality or an AI-authorship probability.'].join('\n'));
|
|
847
|
+
return;
|
|
848
|
+
}
|
|
849
|
+
const picked = !assessment;
|
|
850
|
+
if (!assessment)
|
|
851
|
+
assessment = identifier((await choosePage('Style observations', `${prefix}/style-groups`, scope, options, row => ({ label: row.cohort.scenario_name, detail: row.cohort.criterion }))).cohort.assessment_id);
|
|
852
|
+
else if (options.search)
|
|
853
|
+
throw new CliError('arguments_invalid', 'Search applies to style groups, not an already selected group.', 2);
|
|
854
|
+
const data = await api(`${prefix}/style-groups/${assessment}/observations?page=${picked ? '1' : options.page ?? '1'}`, { actor: scope.owner, workspace: scope.workspace, signal: options.signal });
|
|
855
|
+
if (!styleCohortMatches(data.cohort, { assessment_id: assessment }))
|
|
856
|
+
throw new CliError('assessment_cohort_changed', 'The selected style group could not be confirmed.');
|
|
857
|
+
output(data, [...data.results.map(row => `Drill ${row.trial_position + 1}: ${row.status.replaceAll('_', ' ')}; ${row.count === null ? 'Unmeasured' : `${row.count.toLocaleString()} occurrences`}\n${workspaceLink(scope.workspace, `/campaigns/${campaign}/trials/${row.execution_id}?view=assessment&assessment=${row.assessment_id}`)}`),
|
|
858
|
+
`Page ${data.page} of ${Math.max(1, Math.ceil(data.count / data.page_size))}`].join('\n'));
|
|
859
|
+
return;
|
|
860
|
+
}
|
|
861
|
+
if (mode === 'summary') {
|
|
862
|
+
if (options.checkpoint || args[0] && (options.page || options.search))
|
|
863
|
+
throw new CliError('arguments_invalid', 'Choose a detailed report before using report paging. Checkpoints apply to memory reports.', 2);
|
|
864
|
+
const data = await api(`${prefix}/report`, { actor: scope.owner, signal: options.signal });
|
|
865
|
+
if (data.campaign_id !== campaign)
|
|
866
|
+
throw new CliError('campaign_report_unavailable', 'This retained report could not be confirmed. Reopen the campaign.');
|
|
867
|
+
output(data, [`Simulation ${data.campaign_id}`, data.state.replaceAll('_', ' '),
|
|
868
|
+
...(data.external_effects ? [`${data.external_effects.real_connections_count} real-account connections. External state is shared across drills and is not reset; other drills and account activity can affect results.`] : []),
|
|
869
|
+
...(data.starting_data ? [`${data.starting_data.edit_count} saved edits selected for each independent drill. Inspect each drill for actual application.`] : []),
|
|
870
|
+
...(data.application_continuation ? [`Saved starting state: ${data.application_continuation.scope === 'supported_application_and_native_state'
|
|
871
|
+
? 'application and supported synthetic service data' : 'application state with fresh synthetic services'}. ${data.application_continuation.component_count} application components. New process, current wall time; inspect each drill for restoration evidence.`] : []),
|
|
872
|
+
`${data.retained_trials.toLocaleString()} of ${data.planned_trials.toLocaleString()} drills retained`,
|
|
873
|
+
...Object.entries(data.execution_states).map(([state, count]) => `${state.replaceAll('_', ' ')}: ${count.toLocaleString()}`),
|
|
874
|
+
`Drills with all checks met: ${data.recorded_checks.met.toLocaleString()}; with unmet checks: ${data.recorded_checks.unmet.toLocaleString()}`,
|
|
875
|
+
`Drill check outcomes unassessed: ${data.recorded_checks.unassessed.toLocaleString()}; unavailable: ${data.recorded_checks.unavailable.toLocaleString()}`,
|
|
876
|
+
`Elapsed: ${duration(data.timing.wall_elapsed_ms)}`, `Measured drill work: ${duration(data.timing.trial_work_ms)}`,
|
|
877
|
+
`Report: ${workspaceLink(scope.workspace, `/campaigns/${campaign}`)}`].join('\n'));
|
|
878
|
+
return;
|
|
879
|
+
}
|
|
880
|
+
if (mode === 'composite') {
|
|
881
|
+
if (options.checkpoint !== undefined)
|
|
882
|
+
throw new CliError('arguments_invalid', 'Checkpoint selection applies to memory reports.', 2);
|
|
883
|
+
let execution = args[2] ? identifier(args[2]) : undefined;
|
|
884
|
+
if (execution && (options.page || options.search))
|
|
885
|
+
throw new CliError('arguments_invalid', 'Paging and search apply to the drill list, not one composite.', 2);
|
|
886
|
+
if (!execution && !interactive) {
|
|
887
|
+
const data = await api(`${prefix}/composites?page=${options.page ?? '1'}&q=${encodeURIComponent(options.search ?? '')}`, { actor: scope.owner, workspace: scope.workspace, signal: options.signal });
|
|
888
|
+
if (!compositePageMatches(data, campaign))
|
|
889
|
+
throw new CliError('report_unavailable', 'These saved drill indices could not be confirmed. Reopen the report.');
|
|
890
|
+
const quote = (value) => "'" + value.replaceAll("'", "'\\''") + "'";
|
|
891
|
+
output(data, [data.results.map(row => `Drill ${row.trial_position + 1}: ${row.value === null ? 'Unavailable' : `${row.value.toLocaleString(undefined, { maximumFractionDigits: 2 })} / 100`}\n${row.execution_id} ${row.scenario_name}`).join('\n') || 'No matching composite drills.',
|
|
892
|
+
`Page ${data.page} of ${Math.max(1, Math.ceil(data.count / data.page_size))}`,
|
|
893
|
+
...(data.next ? [`Continue: firedrill report ${campaign} composite --workspace ${scope.workspace} --page ${data.page + 1}${options.search ? ' --search ' + quote(options.search) : ''}${options.json ? ' --json' : ''}`] : [])].join('\n'));
|
|
894
|
+
return;
|
|
895
|
+
}
|
|
896
|
+
if (!execution)
|
|
897
|
+
execution = identifier((await choosePage('Composite drill', `${prefix}/composites`, scope, options, row => ({ label: `Drill ${row.trial_position + 1}: ${row.scenario_name}`, detail: row.value === null ? 'Index unavailable' : `Index ${row.value.toLocaleString(undefined, { maximumFractionDigits: 2 })} / 100` }), value => compositePageMatches(value, campaign))).execution_id);
|
|
898
|
+
const data = await api(`${prefix}/trials/${execution}/composite`, { actor: scope.owner, workspace: scope.workspace, signal: options.signal });
|
|
899
|
+
if (!compositeMatches(data, campaign, execution))
|
|
900
|
+
throw new CliError('report_unavailable', 'This drill index could not be confirmed. Reopen the report.');
|
|
901
|
+
output(data, [`Drill ${data.trial_position + 1}: ${data.scenario_name}`,
|
|
902
|
+
`Composite index: ${data.value === null ? 'Unavailable' : `${data.value.toLocaleString(undefined, { maximumFractionDigits: 2 })} / 100`}`,
|
|
903
|
+
data.state.replaceAll('_', ' '), ...data.dimensions.map(row => `${row.name}: ${row.status.replaceAll('_', ' ')}; weight ${row.weight}; mapped value ${row.mapped_value === null ? 'Not assessed' : row.mapped_value.toLocaleString(undefined, { maximumFractionDigits: 2 })}`),
|
|
904
|
+
...data.required_checks.map(row => `${row.name}: ${row.state}`), 'Weighted index, not a probability or a CI result.',
|
|
905
|
+
`Report: ${workspaceLink(scope.workspace, `/campaigns/${campaign}/trials/${execution}?view=assessment`)}`].join('\n'));
|
|
906
|
+
return;
|
|
907
|
+
}
|
|
908
|
+
let assessment = args[2] ? identifier(args[2]) : undefined;
|
|
909
|
+
const selectedInteractively = !assessment && interactive;
|
|
910
|
+
if (!assessment && !interactive) {
|
|
911
|
+
if (options.checkpoint)
|
|
912
|
+
throw new CliError('assessment_required', 'Choose an assessment_id from the memory groups before choosing a checkpoint.', 2);
|
|
913
|
+
const data = await api(`${prefix}/memory-groups?page=${options.page ?? '1'}&q=${encodeURIComponent(options.search ?? '')}`, { actor: scope.owner, workspace: scope.workspace, signal: options.signal });
|
|
914
|
+
output(data, data.results.map(row => `${row.assessment_id} ${row.scenario_name} ${row.criterion}`).join('\n') || 'No matching memory assessments.');
|
|
915
|
+
return;
|
|
916
|
+
}
|
|
917
|
+
if (!assessment)
|
|
918
|
+
assessment = identifier((await choosePage('Memory assessment', `${prefix}/memory-groups`, scope, options, row => ({ label: row.scenario_name, detail: `${row.criterion} · ${row.model}` }))).assessment_id);
|
|
919
|
+
else if (options.search)
|
|
920
|
+
throw new CliError('arguments_invalid', 'Search applies to the memory group list, not an already selected group.', 2);
|
|
921
|
+
const page = selectedInteractively ? '1' : options.page ?? '1';
|
|
922
|
+
const path = `${prefix}/memory-groups/${assessment}/checkpoints`;
|
|
923
|
+
let index = 0;
|
|
924
|
+
const readCheckpoint = async (index) => {
|
|
925
|
+
const value = await api(`${path}?probe=${index}&page=${page}`, { actor: scope.owner, workspace: scope.workspace, signal: options.signal });
|
|
926
|
+
if (!checkpointMatches(value, assessment, index))
|
|
927
|
+
throw new CliError('assessment_checkpoint_changed', 'The selected memory checkpoint could not be confirmed.');
|
|
928
|
+
return value;
|
|
929
|
+
};
|
|
930
|
+
let data = await readCheckpoint(0);
|
|
931
|
+
if (options.checkpoint) {
|
|
932
|
+
index = data.cohort.memory.probes.findIndex(probe => probe.key === options.checkpoint);
|
|
933
|
+
if (index < 0)
|
|
934
|
+
throw new CliError('checkpoint_unavailable', 'This frozen criterion does not contain that checkpoint.', 2);
|
|
935
|
+
}
|
|
936
|
+
else if (interactive)
|
|
937
|
+
index = await selectOne('Checkpoint', data.cohort.memory.probes.map((probe, value) => ({ value,
|
|
938
|
+
label: `${memoryCheckpointLabel(probe)}: ${probe.key.replaceAll('_', ' ')}`, detail: probe.expectation })), options.signal);
|
|
939
|
+
if (index !== 0)
|
|
940
|
+
data = await readCheckpoint(index);
|
|
941
|
+
output(data, [data.cohort.scenario_name, data.cohort.criterion, `Checkpoint ${data.probe.key}: ${memoryCheckpointLabel(data.probe)}`,
|
|
942
|
+
`Earlier context: ${memoryExposureLabels(data.probe)}`,
|
|
943
|
+
`${data.summary.assessed.toLocaleString()} of ${data.summary.observed_trials.toLocaleString()} observed drills assessed`,
|
|
944
|
+
`Not assessable: ${data.summary.not_assessable.toLocaleString()}`, `Not applicable: ${data.summary.not_applicable.toLocaleString()}`,
|
|
945
|
+
`Mean score: ${data.summary.mean === null ? 'Not assessed' : `${data.summary.mean.toLocaleString(undefined, { maximumFractionDigits: 2 })} / ${data.cohort.levels.length - 1}`}. ${data.cohort.direction === 'lower' ? 'Lower' : 'Higher'} is better.`,
|
|
946
|
+
...data.results.map(row => `Drill ${row.trial_position + 1}: ${row.status.replaceAll('_', ' ')}${row.score === null ? '' : ` (${row.score.toLocaleString(undefined, { maximumFractionDigits: 2 })} / ${data.cohort.levels.length - 1})`}\n${workspaceLink(scope.workspace, `/campaigns/${campaign}/trials/${row.execution_id}?view=assessment&assessment=${row.assessment_id}`)}`),
|
|
947
|
+
`Page ${data.page} of ${Math.max(1, Math.ceil(data.count / data.page_size))}`,
|
|
948
|
+
...(data.next ? [`Next page: firedrill report ${campaign} memory ${assessment} --checkpoint ${data.probe.key} --page ${data.page + 1}`] : [])].join('\n'));
|
|
949
|
+
}
|
|
950
|
+
async function comparisonCommand(args, prefix, scope, interactive, options, output) {
|
|
951
|
+
let baseline = args[2] ? identifier(args[2]) : undefined;
|
|
952
|
+
if (!baseline && !interactive)
|
|
953
|
+
throw new CliError('baseline_required', 'Pass a saved baseline simulation ID after compare. Interactive mode provides a selection.', 2);
|
|
954
|
+
if (!baseline)
|
|
955
|
+
baseline = identifier((await choosePage('Baseline simulation', `${prefix}/comparison-candidates`, scope, { ...options, search: undefined, page: undefined }, row => ({ label: row.name, detail: `${row.created_at} · Source ${row.source_revision.slice(0, 12)}` }))).id);
|
|
956
|
+
const supplied = args[3];
|
|
957
|
+
if (supplied && !['semantic', 'memory', 'style', 'checks', 'performance', 'usage', 'composite'].includes(supplied))
|
|
958
|
+
throw new CliError('arguments_invalid', 'Choose semantic, memory, style, checks, performance, usage or composite after the baseline ID.', 2);
|
|
959
|
+
const kind = (supplied ?? (interactive ? await selectOne('Compare', [{ value: 'semantic', label: 'Semantic scores' },
|
|
960
|
+
{ value: 'memory', label: 'Memory checkpoints' }, { value: 'style', label: 'Style observations' }, { value: 'checks', label: 'Recorded checks' },
|
|
961
|
+
{ value: 'performance', label: 'Conversation timing' }, { value: 'usage', label: 'Target model usage' }, { value: 'composite', label: 'Composite indices' }], options.signal) : 'semantic'));
|
|
962
|
+
if (options.side !== undefined && (!['semantic', 'memory', 'style', 'checks', 'composite', 'performance', 'usage'].includes(kind) || !args[4] && !interactive))
|
|
963
|
+
throw new CliError('arguments_invalid', 'Choose a comparison group before applying --side.', 2);
|
|
964
|
+
if (kind === 'composite') {
|
|
965
|
+
await compositeComparisonCommand(args, prefix, baseline, scope, interactive, options, output);
|
|
966
|
+
return;
|
|
967
|
+
}
|
|
968
|
+
if (kind === 'checks') {
|
|
969
|
+
await nativeCheckCommand(prefix, scope, interactive, options, output, args[4], baseline);
|
|
970
|
+
return;
|
|
971
|
+
}
|
|
972
|
+
if (kind === 'performance' || kind === 'usage') {
|
|
973
|
+
await performanceCommand(prefix, baseline, kind === 'usage' ? 'model' : 'conversation', scope, interactive, options, output, args[4]);
|
|
974
|
+
return;
|
|
975
|
+
}
|
|
976
|
+
let assessment = args[4] ? identifier(args[4]) : undefined;
|
|
977
|
+
if (options.checkpoint && (kind !== 'memory' || !assessment && !interactive))
|
|
978
|
+
throw new CliError('arguments_invalid', 'Choose a memory cohort assessment before selecting its checkpoint.', 2);
|
|
979
|
+
const base = new URLSearchParams({ baseline, kind });
|
|
980
|
+
const number = (value) => value === null ? 'Not assessed' : value.toLocaleString(undefined, { maximumFractionDigits: 2 });
|
|
981
|
+
const summary = (side, value, cohort) => {
|
|
982
|
+
const schedule = value.scenario;
|
|
983
|
+
const lines = [`${side}: ${number(value.mean)}${value.mean === null ? '' : kind === 'style' ? ' occurrences per observed drill' : ` / ${cohort.levels.length - 1}`}; ${value.assessed.toLocaleString()} assessed of ${value.observed.toLocaleString()} retained in this assessment group; ${value.not_assessable.toLocaleString()} not assessable; ${value.not_applicable.toLocaleString()} not applicable; ${value.missing.toLocaleString()} unavailable`,
|
|
984
|
+
`Scenario schedule: ${schedule.planned.toLocaleString()} planned; ${schedule.retained.toLocaleString()} retained; ${schedule.invoked.toLocaleString()} invocations observed; ${schedule.completed.toLocaleString()} completed; ${schedule.target_failed.toLocaleString()} agent failures; ${schedule.platform_failed.toLocaleString()} platform failures; ${schedule.timed_out.toLocaleString()} timed out; ${schedule.canceled.toLocaleString()} canceled; ${schedule.unfinished.toLocaleString()} unfinished.`,
|
|
985
|
+
`${value.without_assessment.toLocaleString()} retained drills outside this assessment group; ${schedule.changed_world.toLocaleString()} changed-world drills and ${schedule.unconfirmed_world.toLocaleString()} unconfirmed-world drills excluded from comparison scores.`];
|
|
986
|
+
if ('distribution' in value)
|
|
987
|
+
lines.push(`Score distribution, retained counts: ${value.distribution.map(bin => `${number(bin.minimum)} ${bin.upper_inclusive ? 'through' : 'to below'} ${number(bin.maximum)}: ${bin.count.toLocaleString()}`).join('; ')}.`);
|
|
988
|
+
return lines.join('\n');
|
|
989
|
+
};
|
|
990
|
+
if (!assessment && !interactive) {
|
|
991
|
+
const data = await api(`${prefix}/comparison?${base}&page=${options.page ?? '1'}&q=${encodeURIComponent(options.search ?? '')}`, { actor: scope.owner, signal: options.signal });
|
|
992
|
+
if (!comparisonPageMatches(data, { current: prefix.split('/').at(-1), baseline, kind }))
|
|
993
|
+
throw new CliError('assessment_comparison_changed', 'The selected simulation comparison could not be confirmed.');
|
|
994
|
+
output(data, [`Current: ${data.current.name} (${data.current.id})`, `Baseline: ${data.baseline.name} (${data.baseline.id})`,
|
|
995
|
+
...Object.entries(data.context).map(([key, equal]) => `${key.replaceAll('_', ' ')}: ${equal === null ? 'Unknown' : equal ? 'Same recorded input' : 'Different'}`),
|
|
996
|
+
...data.results.flatMap(row => [`${row.cohort.scenario_name}: ${row.cohort.criterion} (${row.cohort.assessment_id})`, summary('Current', row.current, row.cohort), summary('Baseline', row.baseline, row.cohort),
|
|
997
|
+
`Mean difference (current minus baseline): ${number(row.difference)}`]),
|
|
998
|
+
`Page ${data.page} of ${Math.max(1, Math.ceil(data.count / data.page_size))}`,
|
|
999
|
+
'Only compatible retained observations are grouped. This is not a causal winner or CI decision.'].join('\n'));
|
|
1000
|
+
return;
|
|
1001
|
+
}
|
|
1002
|
+
const picked = !assessment;
|
|
1003
|
+
if (!assessment)
|
|
1004
|
+
assessment = identifier((await choosePage('Assessment group', `${prefix}/comparison?${base}`, scope, options, row => ({ label: `${row.cohort.scenario_name}: ${row.cohort.criterion}`, detail: `${row.cohort.model} · Current ${number(row.current.mean)} · Baseline ${number(row.baseline.mean)}` }))).cohort.assessment_id);
|
|
1005
|
+
else if (options.search)
|
|
1006
|
+
throw new CliError('arguments_invalid', 'Search applies to the group list, not one selected assessment.', 2);
|
|
1007
|
+
const side = options.side ?? (interactive ? await selectOne('Show observations from', [
|
|
1008
|
+
{ value: 'both', label: 'Both simulations' }, { value: 'current', label: 'Current simulation' },
|
|
1009
|
+
{ value: 'baseline', label: 'Baseline simulation' }
|
|
1010
|
+
], options.signal) : 'both');
|
|
1011
|
+
const params = new URLSearchParams({ ...Object.fromEntries(base), page: picked ? '1' : options.page ?? '1', ...(kind === 'memory' ? { probe: '0' } : {}) });
|
|
1012
|
+
params.set('side', side);
|
|
1013
|
+
const path = `${prefix}/comparison/${assessment}/observations`;
|
|
1014
|
+
const readObservations = async () => {
|
|
1015
|
+
const value = await api(`${path}?${params}`, { actor: scope.owner, signal: options.signal });
|
|
1016
|
+
if (!comparisonObservationsMatch(value, { current: prefix.split('/').at(-1), baseline: baseline, kind,
|
|
1017
|
+
assessment: assessment, side, ...(kind === 'memory' ? { probe: Number(params.get('probe') ?? '0') } : {}) }))
|
|
1018
|
+
throw new CliError('assessment_comparison_changed', 'The selected comparison observations could not be confirmed.');
|
|
1019
|
+
return value;
|
|
1020
|
+
};
|
|
1021
|
+
let data = await readObservations();
|
|
1022
|
+
if (kind === 'memory' && data.kind === 'memory') {
|
|
1023
|
+
const probes = data.cohort.memory.probes;
|
|
1024
|
+
const index = options.checkpoint ? probes.findIndex(row => row.key === options.checkpoint) : interactive
|
|
1025
|
+
? await selectOne('Checkpoint', probes.map((row, value) => ({ value, label: `${memoryCheckpointLabel(row)}: ${row.key.replaceAll('_', ' ')}`, detail: row.expectation })), options.signal) : 0;
|
|
1026
|
+
if (index < 0)
|
|
1027
|
+
throw new CliError('checkpoint_unavailable', 'That checkpoint is not in this frozen criterion.', 2);
|
|
1028
|
+
if (index !== 0) {
|
|
1029
|
+
params.set('probe', String(index));
|
|
1030
|
+
data = await readObservations();
|
|
1031
|
+
}
|
|
1032
|
+
}
|
|
1033
|
+
output(data, [data.cohort.scenario_name, data.cohort.criterion, `Evaluator: ${data.cohort.model}`,
|
|
1034
|
+
...(kind === 'style' ? ['Selected prose occurrences, not a quality score.'] : [data.cohort.direction === 'lower' ? 'Lower scores are better.' : 'Higher scores are better.']),
|
|
1035
|
+
...(data.probe ? [`${memoryCheckpointLabel(data.probe)}: ${data.probe.expectation}`, `Earlier context: ${memoryExposureLabels(data.probe)}`] : []),
|
|
1036
|
+
summary('Current', data.summary.current, data.cohort), summary('Baseline', data.summary.baseline, data.cohort),
|
|
1037
|
+
`Mean difference (current minus baseline): ${number(data.summary.difference)}`,
|
|
1038
|
+
...data.results.map(row => `${row.side === 'current' ? 'Current' : 'Baseline'} drill ${row.trial_position + 1}: ${row.status.replaceAll('_', ' ')} (${number(row.score)})\n${workspaceLink(scope.workspace, `/campaigns/${row.campaign_id}/trials/${row.execution_id}?view=assessment&assessment=${row.assessment_id}`)}`),
|
|
1039
|
+
`Page ${data.page} of ${Math.max(1, Math.ceil(data.count / data.page_size))}`,
|
|
1040
|
+
...(data.next ? [`Continue: firedrill report ${prefix.split('/').at(-1)} compare ${baseline} ${kind} ${assessment} --workspace ${identifier(prefix.split('/')[3])} --page ${data.page + 1} --side ${side}${data.probe ? ' --checkpoint ' + data.probe.key : ''}${options.json ? ' --json' : ''}`] : []),
|
|
1041
|
+
'Missing observations are not zero. Independent drills are not one memory session.'].join('\n'));
|
|
1042
|
+
}
|
|
1043
|
+
const nativeCheckMethods = { exact: 'Objective', semantic: 'Judged', both: 'Objective and judged', unknown: 'Definition unavailable' };
|
|
1044
|
+
const checkCounts = (label, value) => `${label}: ${value.met.toLocaleString()} met; ${value.unmet.toLocaleString()} unmet; ${value.unconfirmed.toLocaleString()} unconfirmed; ${value.reported.toLocaleString()} reported of ${value.records.toLocaleString()} retained drill results`;
|
|
1045
|
+
async function nativeCheckCommand(prefix, scope, interactive, options, output, supplied, baseline) {
|
|
1046
|
+
if (options.checkpoint)
|
|
1047
|
+
throw new CliError('arguments_invalid', 'Checkpoint selection applies only to memory reports.', 2);
|
|
1048
|
+
let check = supplied ? identifier(supplied) : undefined;
|
|
1049
|
+
if (check && options.search)
|
|
1050
|
+
throw new CliError('arguments_invalid', 'Search applies to check groups, not an already selected check.', 2);
|
|
1051
|
+
if (check && options.checkMethod !== undefined)
|
|
1052
|
+
throw new CliError('arguments_invalid', 'Check type filters apply to group lists, not a selected original check.', 2);
|
|
1053
|
+
if (!check && !interactive && (options.status !== undefined || options.side !== undefined))
|
|
1054
|
+
throw new CliError('check_required', 'Choose a check ID before filtering its observations by status or side.', 2);
|
|
1055
|
+
if (!baseline && options.side === 'baseline')
|
|
1056
|
+
throw new CliError('baseline_required', 'Choose a saved baseline before requesting its check observations.', 2);
|
|
1057
|
+
const method = options.checkMethod ?? '';
|
|
1058
|
+
const groups = new URLSearchParams({ method, ...(baseline ? { baseline } : {}) });
|
|
1059
|
+
const path = `${prefix}/check-groups`;
|
|
1060
|
+
if (!check && !interactive) {
|
|
1061
|
+
groups.set('page', options.page ?? '1');
|
|
1062
|
+
groups.set('q', options.search ?? '');
|
|
1063
|
+
const data = await api(`${path}?${groups}`, { actor: scope.owner, signal: options.signal });
|
|
1064
|
+
if (!checkGroupsMatch(data, prefix.split('/').at(-1), baseline, method))
|
|
1065
|
+
throw new CliError('native_check_report_changed', 'The selected check groups could not be confirmed.');
|
|
1066
|
+
output(data, [...(data.baseline ? [`Current: ${data.current.name} (${data.current.id})`, `Baseline: ${data.baseline.name} (${data.baseline.id})`,
|
|
1067
|
+
...Object.entries(data.context ?? {}).map(([key, equal]) => `${key.replaceAll('_', ' ')}: ${equal === null ? 'Unknown' : equal ? 'Same recorded input' : 'Different'}`)] : []),
|
|
1068
|
+
...data.results.flatMap(row => [`${row.check.scenario_name}: ${row.check.name} (${row.check.id})`, nativeCheckMethods[row.check.method] ?? nativeCheckMethods.unknown,
|
|
1069
|
+
checkCounts('Current', row.current), ...(row.baseline ? [checkCounts('Baseline', row.baseline)] : []),
|
|
1070
|
+
...(!row.check.definition_known ? ['Original definition unavailable; not matched across simulations.'] : [])]),
|
|
1071
|
+
...(!data.count ? ['No matching retained check results.'] : []),
|
|
1072
|
+
`Page ${data.page} of ${Math.max(1, Math.ceil(data.count / data.page_size))}`,
|
|
1073
|
+
...(data.next ? [`Continue: firedrill report ${prefix.split('/').at(-1)} ${baseline ? 'compare ' + baseline + ' checks' : 'checks'} --workspace ${identifier(prefix.split('/')[3])} --page ${data.page + 1}${method ? ' --method ' + method : ''}${options.search ? " --search '" + options.search.replaceAll("'", "'\\''") + "'" : ''}${options.json ? ' --json' : ''}`] : []),
|
|
1074
|
+
'Counts cover retained drill results. An unconfirmed or missing check is not a failure or a semantic score.'].join('\n'));
|
|
1075
|
+
return;
|
|
1076
|
+
}
|
|
1077
|
+
const picked = !check;
|
|
1078
|
+
if (!check)
|
|
1079
|
+
check = identifier((await choosePage('Recorded check', `${path}?${groups}`, scope, options, row => ({ label: `${row.check.scenario_name}: ${row.check.name}`, detail: `${nativeCheckMethods[row.check.method] ?? nativeCheckMethods.unknown} · ${row.current.unmet} current unmet${row.baseline ? ` · ${row.baseline.unmet} baseline unmet` : ''}` }))).check.id);
|
|
1080
|
+
const side = options.side ?? (baseline && interactive ? await selectOne('Simulation', [{ value: 'both', label: 'Both simulations' }, { value: 'current', label: 'Current' }, { value: 'baseline', label: 'Baseline' }], options.signal) : 'both');
|
|
1081
|
+
const status = options.status ?? (interactive ? await selectOne('Check results', [{ value: '', label: 'All results' }, { value: 'unmet', label: 'Unmet' }, { value: 'met', label: 'Met' }, { value: 'unconfirmed', label: 'Unconfirmed' }], options.signal) : '');
|
|
1082
|
+
const query = new URLSearchParams({ page: picked ? '1' : options.page ?? '1', side, status, ...(baseline ? { baseline } : {}) });
|
|
1083
|
+
const data = await api(`${path}/${check}/observations?${query}`, { actor: scope.owner, signal: options.signal });
|
|
1084
|
+
if (!checkObservationsMatch(data, { current: prefix.split('/').at(-1), baseline, check, side, status }))
|
|
1085
|
+
throw new CliError('native_check_report_changed', 'The selected check observations could not be confirmed.');
|
|
1086
|
+
output(data, [`${data.check.scenario_name}: ${data.check.name}`, nativeCheckMethods[data.check.method] ?? nativeCheckMethods.unknown,
|
|
1087
|
+
checkCounts('Current', data.summary.current), ...(data.summary.baseline ? [checkCounts('Baseline', data.summary.baseline)] : []),
|
|
1088
|
+
...(!data.check.definition_known ? ['Original definition unavailable; not matched across simulations.'] : []),
|
|
1089
|
+
...data.results.map(row => `${baseline ? row.side === 'baseline' ? 'Baseline ' : 'Current ' : ''}drill ${row.trial_position + 1}: ${row.reported ? row.status : 'Not reported'}${row.reason_excerpt ? `\n${row.reason_excerpt}${row.reason_truncated ? '\nOpen the original check for the full explanation.' : ''}` : ''}\nInspect: firedrill report ${row.campaign_id} drill ${row.execution_id} check-observation ${row.id} --workspace ${scope.workspace}\n${workspaceLink(scope.workspace, `/campaigns/${row.campaign_id}/trials/${row.execution_id}?view=assessment&check_observation=${row.id}`)}`),
|
|
1090
|
+
...(!data.count ? ['No retained check results match these filters.'] : []), `Page ${data.page} of ${Math.max(1, Math.ceil(data.count / data.page_size))}`,
|
|
1091
|
+
...(data.next ? [`Continue: firedrill report ${prefix.split('/').at(-1)} ${baseline ? 'compare ' + baseline + ' checks' : 'checks'} ${check} --workspace ${identifier(prefix.split('/')[3])} --page ${data.page + 1} --side ${side}${status ? ' --status ' + status : ''}${options.json ? ' --json' : ''}`] : [])].join('\n'));
|
|
1092
|
+
}
|
|
1093
|
+
async function performanceCommand(prefix, baseline, kind, scope, interactive, options, output, supplied) {
|
|
1094
|
+
if (options.checkpoint)
|
|
1095
|
+
throw new CliError('arguments_invalid', 'Checkpoints apply only to memory reports.', 2);
|
|
1096
|
+
let group = supplied ? identifier(supplied) : undefined;
|
|
1097
|
+
if (group && options.search)
|
|
1098
|
+
throw new CliError('arguments_invalid', 'Search applies to measurement groups, not an already selected group.', 2);
|
|
1099
|
+
const current = identifier(prefix.split('/').at(-1));
|
|
1100
|
+
const auth = { actor: scope.owner, workspace: scope.workspace, signal: options.signal };
|
|
1101
|
+
const quote = (value) => "'" + value.replaceAll("'", "'\\''") + "'";
|
|
1102
|
+
const command = `firedrill report ${current} compare ${baseline} ${kind === 'model' ? 'usage' : 'performance'}`;
|
|
1103
|
+
const continuation = (data, suffix, filters = '') => data.next
|
|
1104
|
+
? [`Continue: ${command}${suffix} --workspace ${scope.workspace} --page ${data.page + 1}${filters}${options.json ? ' --json' : ''}`] : [];
|
|
1105
|
+
const values = new URLSearchParams({ baseline, kind });
|
|
1106
|
+
const path = `${prefix}/performance-groups`;
|
|
1107
|
+
const measured = (value) => value === null ? 'Not measured' : value;
|
|
1108
|
+
const timing = (value) => value === null ? 'Not measured' : `${value}ms`;
|
|
1109
|
+
const describe = (label, value) => [`${label}: ${value.records.toLocaleString()} ${kind === 'model' ? 'request records' : 'retained drill results'}`,
|
|
1110
|
+
`${kind === 'model' ? 'Mean request elapsed' : 'Mean conversation duration'}: ${timing(value.elapsed_ms.mean)}; ${value.timed_records.toLocaleString()} timed records; range ${timing(value.elapsed_ms.minimum)} to ${timing(value.elapsed_ms.maximum)}`,
|
|
1111
|
+
...(value.turns ? [`Mean turns: ${measured(value.turns.mean)}; ${value.turns.coverage.toLocaleString()} measured results`] : []),
|
|
1112
|
+
...Object.entries(value.tokens ?? {}).map(([name, entry]) => `${name.replaceAll('_', ' ')}: ${measured(entry.total)} total; ${measured(entry.mean)} mean per measured request; ${entry.coverage.toLocaleString()} measured of ${value.records.toLocaleString()}`),
|
|
1113
|
+
...(value.requests ? [`Completed ${value.requests.completed}; failed ${value.requests.failed}; interrupted ${value.requests.interrupted}; unfinished ${value.requests.pending}`] : [])];
|
|
1114
|
+
const identity = (value) => `${value.scenario_name}${kind === 'model' ? ` · ${value.requested_model} · ${value.provider ?? 'Provider unrecorded'}${value.operation ? ' · ' + value.operation : ''}` : ''}`;
|
|
1115
|
+
const limitation = kind === 'model' ? 'Target-model request usage only, including available partial fields. Elapsed includes relay/network handling. Requested models are not resolved model versions; no dollar costs or platform evaluator spend are estimated.'
|
|
1116
|
+
: 'Conversation measurements from original drill results, not startup/cleanup, model inference latency or total drill work.';
|
|
1117
|
+
if (!group && !interactive) {
|
|
1118
|
+
const data = await api(`${path}?${values}&page=${options.page ?? '1'}&q=${encodeURIComponent(options.search ?? '')}`, auth);
|
|
1119
|
+
if (!performanceGroupsMatch(data, current, baseline, kind))
|
|
1120
|
+
throw new CliError('report_unavailable', 'This measurement comparison could not be confirmed. Reopen the report.');
|
|
1121
|
+
output(data, [`Current: ${data.current.name} (${data.current.id})`, `Baseline: ${data.baseline.name} (${data.baseline.id})`,
|
|
1122
|
+
...Object.entries(data.context).map(([key, equal]) => `${key.replaceAll('_', ' ')}: ${equal === null ? 'Unknown' : equal ? 'Same recorded input' : 'Different'}`),
|
|
1123
|
+
...data.results.flatMap(row => [`${identity(row.cohort)} (${row.cohort.id})`, ...describe('Current', row.current), ...describe('Baseline', row.baseline)]),
|
|
1124
|
+
...(!data.count ? ['No matching retained measurements.'] : []), `Page ${data.page} of ${Math.max(1, Math.ceil(data.count / data.page_size))}`,
|
|
1125
|
+
...continuation(data, '', options.search ? ' --search ' + quote(options.search) : ''), limitation,
|
|
1126
|
+
'Missing measurements are not zero. Exact scenario/provider/model groups and setup differences do not establish a causal winner.'].join('\n'));
|
|
1127
|
+
return;
|
|
1128
|
+
}
|
|
1129
|
+
const picked = !group;
|
|
1130
|
+
if (!group)
|
|
1131
|
+
group = identifier((await choosePage('Measurement group', `${path}?${values}`, scope, options, row => ({ label: identity(row.cohort), detail: `Current ${row.current.records} retained · Baseline ${row.baseline.records} retained` }), value => performanceGroupsMatch(value, current, baseline, kind))).cohort.id);
|
|
1132
|
+
const side = options.side ?? (interactive ? await selectOne('Simulation', [{ value: 'both', label: 'Both simulations' }, { value: 'current', label: 'Current' }, { value: 'baseline', label: 'Baseline' }], options.signal) : 'both');
|
|
1133
|
+
values.set('side', side);
|
|
1134
|
+
values.set('page', picked ? '1' : options.page ?? '1');
|
|
1135
|
+
const data = await api(`${path}/${group}/observations?${values}`, auth);
|
|
1136
|
+
if (!performanceObservationsMatch(data, { current, baseline, kind, cohort: group, side }))
|
|
1137
|
+
throw new CliError('report_unavailable', 'These retained measurements could not be confirmed. Reopen the comparison.');
|
|
1138
|
+
output(data, [identity(data.cohort), ...describe('Current', data.summary.current), ...describe('Baseline', data.summary.baseline),
|
|
1139
|
+
...data.results.flatMap(row => [`${row.side === 'baseline' ? 'Baseline' : 'Current'} drill ${row.trial_position + 1}: ${timing(row.elapsed_ms)}${kind === 'model' ? `; ${row.state?.replaceAll('_', ' ') ?? 'Unconfirmed'}` : `; ${measured(row.turns)} turns`}`,
|
|
1140
|
+
...Object.entries(row.tokens ?? {}).map(([name, value]) => `${name.replaceAll('_', ' ')}: ${measured(value)}`),
|
|
1141
|
+
workspaceLink(scope.workspace, `/campaigns/${row.campaign_id}/trials/${row.execution_id}`)]),
|
|
1142
|
+
...(!data.count ? ['No retained measurements on this side.'] : []), `Page ${data.page} of ${Math.max(1, Math.ceil(data.count / data.page_size))}`,
|
|
1143
|
+
...continuation(data, ` ${group}`, ` --side ${side}`), limitation].join('\n'));
|
|
1144
|
+
}
|
|
1145
|
+
async function compositeComparisonCommand(args, prefix, baseline, scope, interactive, options, output) {
|
|
1146
|
+
if (options.checkpoint)
|
|
1147
|
+
throw new CliError('arguments_invalid', 'Checkpoints apply only to memory reports.', 2);
|
|
1148
|
+
let group = args[4];
|
|
1149
|
+
if (group && !/^[a-f0-9]{64}$/.test(group))
|
|
1150
|
+
throw new CliError('comparison_group_required', 'Choose a composite cohort ID from this comparison.', 2);
|
|
1151
|
+
if (group && options.search)
|
|
1152
|
+
throw new CliError('arguments_invalid', 'Search applies to composite groups, not an already selected group.', 2);
|
|
1153
|
+
if (args[5] && !['models', 'trials'].includes(args[5]))
|
|
1154
|
+
throw new CliError('arguments_invalid', 'Choose models or drills after the composite cohort ID.', 2);
|
|
1155
|
+
const filtered = options.side !== undefined || options.status !== undefined || options.evaluator !== undefined;
|
|
1156
|
+
if (filtered && (args[5] === 'models' || !interactive && (!group || args[5] !== 'trials')))
|
|
1157
|
+
throw new CliError('arguments_invalid', 'Choose a composite cohort and drills before applying --side, --status or --evaluator.', 2);
|
|
1158
|
+
const current = identifier(prefix.split('/').at(-1));
|
|
1159
|
+
const auth = { actor: scope.owner, workspace: scope.workspace, signal: options.signal };
|
|
1160
|
+
const quote = (value) => "'" + value.replaceAll("'", "'\\''") + "'";
|
|
1161
|
+
const command = `firedrill report ${current} compare ${baseline} composite`;
|
|
1162
|
+
const continuation = (data, suffix, filters = '') => data.next
|
|
1163
|
+
? [`Continue: ${command}${suffix} --workspace ${scope.workspace} --page ${data.page + 1}${filters}${options.json ? ' --json' : ''}`] : [];
|
|
1164
|
+
const base = new URLSearchParams({ baseline });
|
|
1165
|
+
const path = `${prefix}/composite-comparison`;
|
|
1166
|
+
const number = (value) => value === null ? 'Unavailable' : value.toLocaleString(undefined, { maximumFractionDigits: 2 });
|
|
1167
|
+
const summary = (label, value) => `${label}: ${number(value.mean)}${value.mean === null ? '' : ' / 100'}; ${value.assessed.toLocaleString()} indexed of ${value.records.toLocaleString()} retained drills; ${value.unavailable.toLocaleString()} unavailable; range ${number(value.minimum)} to ${number(value.maximum)}; ${value.checks_unmet.toLocaleString()} with unmet checks, ${value.checks_unconfirmed.toLocaleString()} with unconfirmed checks, ${value.execution_incomplete.toLocaleString()} incomplete`;
|
|
1168
|
+
const schedule = (cohort) => `Current scenario schedule: ${cohort.current.retained.toLocaleString()} retained of ${cohort.current.planned.toLocaleString()} planned; ${cohort.current.changed_world.toLocaleString()} changed-world and ${cohort.current.unconfirmed_world.toLocaleString()} unconfirmed-world drills excluded. Baseline: ${cohort.baseline.retained.toLocaleString()} retained of ${cohort.baseline.planned.toLocaleString()} planned; ${cohort.baseline.changed_world.toLocaleString()} changed-world and ${cohort.baseline.unconfirmed_world.toLocaleString()} unconfirmed-world drills excluded.`;
|
|
1169
|
+
const limitation = 'Only exact frozen scenario/world/criteria/weights/mappings/check/calculation cohorts are grouped, with evaluator versions separate. Missing inputs are not zero or redistributed weight. This is not a probability, CI result or causal winner.';
|
|
1170
|
+
if (!group && !interactive) {
|
|
1171
|
+
const data = await api(`${path}?${base}&page=${options.page ?? '1'}&q=${encodeURIComponent(options.search ?? '')}`, auth);
|
|
1172
|
+
if (!compositeGroupsMatch(data, current, baseline))
|
|
1173
|
+
throw new CliError('report_unavailable', 'This composite comparison could not be confirmed. Reopen the report.');
|
|
1174
|
+
output(data, [`Current: ${data.current.name} (${data.current.id})`, `Baseline: ${data.baseline.name} (${data.baseline.id})`,
|
|
1175
|
+
...Object.entries(data.context).map(([key, equal]) => `${key.replaceAll('_', ' ')}: ${equal === null ? 'Unknown' : equal ? 'Same recorded input' : 'Different'}`),
|
|
1176
|
+
...data.results.flatMap(row => [`${row.scenario_name} (${row.id})`, `${row.dimensions.length} weighted criteria.`, schedule(row),
|
|
1177
|
+
...(!row.check_definitions_known ? ['Original check definitions unconfirmed; this group is not matched across simulations.'] : []),
|
|
1178
|
+
...(!row.calculation_supported ? ['This retained calculation version is not summarized; open original drills.'] : [])]),
|
|
1179
|
+
...(!data.count ? ['No matching frozen composite definitions.'] : []), `Page ${data.page} of ${Math.max(1, Math.ceil(data.count / data.page_size))}`,
|
|
1180
|
+
...continuation(data, '', options.search ? ' --search ' + quote(options.search) : ''), limitation].join('\n'));
|
|
1181
|
+
return;
|
|
1182
|
+
}
|
|
1183
|
+
const picked = !group;
|
|
1184
|
+
if (!group)
|
|
1185
|
+
group = (await choosePage('Composite definition', `${path}?${base}`, scope, options, row => ({ label: row.scenario_name, detail: `${row.dimensions.length} weighted criteria · Current ${row.current.retained} retained · Baseline ${row.baseline.retained} retained` }), value => compositeGroupsMatch(value, current, baseline))).id;
|
|
1186
|
+
if (!/^[a-f0-9]{64}$/.test(group))
|
|
1187
|
+
throw new CliError('report_unavailable', 'The saved composite comparison could not be read.');
|
|
1188
|
+
const action = args[5] ?? (filtered ? 'trials' : interactive ? await selectOne('Inspect', [{ value: 'models', label: 'Evaluator summaries' }, { value: 'trials', label: 'Drill indices' }], options.signal) : 'models');
|
|
1189
|
+
const params = new URLSearchParams({ baseline, page: picked ? '1' : options.page ?? '1' });
|
|
1190
|
+
if (action === 'models') {
|
|
1191
|
+
const data = await api(`${path}/${group}/models?${params}`, auth);
|
|
1192
|
+
if (!compositeModelsMatch(data, current, baseline, group))
|
|
1193
|
+
throw new CliError('report_unavailable', 'This evaluator summary could not be confirmed. Reopen the comparison.');
|
|
1194
|
+
output(data, [data.cohort.scenario_name, schedule(data.cohort), ...data.cohort.dimensions.map(row => `${row.name}: relative weight ${row.weight}; numeric mapping ${row.values.join(', ')}`),
|
|
1195
|
+
...data.results.flatMap(row => [row.model ?? 'Evaluator unavailable', summary('Current', row.current), summary('Baseline', row.baseline),
|
|
1196
|
+
`Mean difference (current minus baseline): ${row.difference === null ? 'Not comparable' : `${row.difference > 0 ? '+' : ''}${number(row.difference)}`}`]),
|
|
1197
|
+
...(!data.count ? ['No retained drill slots for this definition yet.'] : []), `Page ${data.page} of ${Math.max(1, Math.ceil(data.count / data.page_size))}`,
|
|
1198
|
+
...continuation(data, ` ${group} models`), limitation].join('\n'));
|
|
1199
|
+
return;
|
|
1200
|
+
}
|
|
1201
|
+
const side = options.side ?? (interactive ? await selectOne('Simulation', [{ value: 'both', label: 'Both simulations' }, { value: 'current', label: 'Current' }, { value: 'baseline', label: 'Baseline' }], options.signal) : 'both');
|
|
1202
|
+
const status = options.status ?? (interactive ? await selectOne('Index', [{ value: 'all', label: 'All drills' }, { value: 'available', label: 'Available' }, { value: 'unavailable', label: 'Unavailable' }], options.signal) : 'all');
|
|
1203
|
+
params.set('side', side);
|
|
1204
|
+
params.set('status', status);
|
|
1205
|
+
const model = options.evaluator === undefined ? undefined : options.evaluator === 'unavailable' ? null : options.evaluator;
|
|
1206
|
+
params.set('model_scope', model === undefined ? 'all' : model === null ? 'unavailable' : 'known');
|
|
1207
|
+
if (model !== undefined && model !== null)
|
|
1208
|
+
params.set('model', model);
|
|
1209
|
+
const data = await api(`${path}/${group}/observations?${params}`, auth);
|
|
1210
|
+
if (!compositeObservationsMatch(data, { current, baseline, cohort: group, side, status, model }))
|
|
1211
|
+
throw new CliError('report_unavailable', 'These drill indices could not be confirmed. Reopen the comparison.');
|
|
1212
|
+
output(data, [data.cohort.scenario_name, schedule(data.cohort), ...data.results.map(row => `${row.side === 'current' ? 'Current' : 'Baseline'} drill ${row.trial_position + 1}: ${row.value === null ? 'Unavailable' : `${number(row.value)} / 100`}; ${row.state.replaceAll('_', ' ')}; ${row.model ?? 'Evaluator unavailable'}\n${workspaceLink(scope.workspace, `/campaigns/${row.campaign_id}/trials/${row.execution_id}?view=assessment`)}`),
|
|
1213
|
+
...(!data.count ? ['No retained drills match these filters.'] : []), `Page ${data.page} of ${Math.max(1, Math.ceil(data.count / data.page_size))}`,
|
|
1214
|
+
...continuation(data, ` ${group} drills`, ` --side ${side} --status ${status}${options.evaluator === undefined ? '' : ' --evaluator ' + quote(options.evaluator)}`), limitation].join('\n'));
|
|
1215
|
+
}
|