@automatalabs/workflows 0.40.3 → 0.40.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -11
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/validate-internal.d.ts +1 -0
- package/dist/validate-internal.d.ts.map +1 -1
- package/dist/validate.d.ts +7 -2
- package/dist/validate.d.ts.map +1 -1
- package/dist/validate.js +327 -26
- package/package.json +3 -3
package/README.md
CHANGED
|
@@ -637,20 +637,33 @@ Three passes: a **static parse** (the `meta` literal, syntax, and direct nondete
|
|
|
637
637
|
expressions), then a **dry run** — the script executes in the real engine realm while every
|
|
638
638
|
`agent()` call is served by an in-process mock `AgentRunner` that fabricates schema-conforming
|
|
639
639
|
results. The dry run catches what a parse can't: thunk-vs-promise mistakes, reference errors,
|
|
640
|
-
broken plumbing between calls. Finally, validation opens one no-prompt session
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
640
|
+
broken plumbing between calls. Finally, validation opens one no-prompt session for every distinct
|
|
641
|
+
routed `{ backend, model }` pair, selects that call's model verbatim when one was authored, surfaces
|
|
642
|
+
the echoed model-specific config-option catalog, and checks every authored `configOptions` bag
|
|
643
|
+
against it. This probe uses zero tokens. A pair that cannot spawn, authenticate, select its model,
|
|
644
|
+
or open a session contributes one warning and `probed:false`; only that pair's option checks are
|
|
645
|
+
skipped, so probe failure alone never invalidates the script. There is no cached catalog or opt-out
|
|
646
|
+
flag.
|
|
645
647
|
A mock live confirm answers checkpoints with `default ?? true`, so `headless: "pause"`
|
|
646
648
|
dry-runs cleanly; `headless: "abort"` warns because a truly unattended run would abort.
|
|
647
649
|
Script-declared `meta.backends` are treated as approved (with a warning that real runs require
|
|
648
650
|
approval). The report lists every agent call with its backend attribution and `configOptions`
|
|
649
|
-
echo, every checkpoint, the full option table for every routed
|
|
651
|
+
echo, every checkpoint, the full option table for every routed backend/model pair (even when no call authors
|
|
650
652
|
options), and warnings. Unknown ids, invalid select values, non-boolean boolean values, and the
|
|
651
653
|
reserved `"model"` id make the report invalid with exit code `2`; each diagnostic names the call,
|
|
652
|
-
authored value, and advertised alternatives.
|
|
653
|
-
|
|
654
|
+
authored value, and advertised alternatives. For select options carrying
|
|
655
|
+
`_meta["@automatalabs/agentprism"].recognizedValues`, a supported value passes, an unsupported but
|
|
656
|
+
recognized value passes with a warning naming the effective clamp target, and an unrecognized value
|
|
657
|
+
is invalid. Pi's `thinkingLevel` option publishes this metadata from Pi's own ordered domain and
|
|
658
|
+
therefore needs no extra probes. For an ordered built-in that omits the metadata (Claude or Codex),
|
|
659
|
+
validation reads its advertised model picker, probes each selectable model through the same
|
|
660
|
+
per-`{ backend, model }` cache, merges consistent per-model thought-level orders, and applies the
|
|
661
|
+
same recognized-value/clamp path. Claude models that omit `effort` do not inherit another model's
|
|
662
|
+
option, and its `default` sentinel is recognized but excluded from ceiling ordering. Enumeration is
|
|
663
|
+
skipped, with a warning, when a picker advertises more than 32 models or the orders cannot be merged.
|
|
664
|
+
OpenCode and custom/unknown backends are exact-set: an unadvertised thought-level value is invalid
|
|
665
|
+
instead of clamping. Exit codes are `0` valid, `1` parse failure, `2` dry-run or config-option failure,
|
|
666
|
+
`3` usage error.
|
|
654
667
|
|
|
655
668
|
Flags: `--args <json>` / `--args-file <path>`, `--workflows-dir <dir>` (repeatable — validate by
|
|
656
669
|
NAME and resolve nested `workflow("<name>")` calls from your folder), `--parse-only`,
|
|
@@ -705,7 +718,7 @@ const report = await validateWorkflowScript(script, { args: { target: "src/" },
|
|
|
705
718
|
report.ok; // parse ok AND dry run completed
|
|
706
719
|
report.dryRun?.agentCalls; // calls include mockAnswer: { glob, sequenceIndex?, sequenceLength? }
|
|
707
720
|
report.dryRun?.harnessOptions;
|
|
708
|
-
// [{ backendId, probed, options?: SessionConfigOption[], error?: string }]
|
|
721
|
+
// [{ backendId, model?, probed, options?: SessionConfigOption[], error?: string }]
|
|
709
722
|
report.dryRun?.mockAnswers;// normalized rule counters + item-level unused records
|
|
710
723
|
report.warnings; // approval reminders, phase mismatches, headless-abort checkpoints, …
|
|
711
724
|
```
|
|
@@ -740,7 +753,7 @@ import { probeHarnessConfig, formatHarnessConfigReport } from "@automatalabs/wor
|
|
|
740
753
|
|
|
741
754
|
const report = await probeHarnessConfig({ harnesses: ["codex"] });
|
|
742
755
|
report.ok; // every requested harness probed
|
|
743
|
-
report.harnessOptions; // [{ backendId, probed, options?: SessionConfigOption[], error?: string }]
|
|
756
|
+
report.harnessOptions; // [{ backendId, model?, probed, options?: SessionConfigOption[], error?: string }]
|
|
744
757
|
formatHarnessConfigReport(report); // the CLI's human table
|
|
745
758
|
```
|
|
746
759
|
|
|
@@ -820,6 +833,12 @@ opens because the dedicated `model` field is its only channel. Run `validate` an
|
|
|
820
833
|
harness's advertised-options table before choosing ids or select values. A live harness rejection
|
|
821
834
|
otherwise follows the existing agent-error path.
|
|
822
835
|
|
|
836
|
+
Pi exposes `thinkingLevel` through that same surface. The validator selects each call's Pi model
|
|
837
|
+
before reading the option, so a call can safely differ from the session default. Values shown in the
|
|
838
|
+
model-specific choices are applied unchanged. A Pi-recognized value above the model ceiling or in a
|
|
839
|
+
model gap validates with a warning and clamps in Pi's order; a value outside Pi's recognized domain
|
|
840
|
+
fails validation with exit code `2`.
|
|
841
|
+
|
|
823
842
|
---
|
|
824
843
|
|
|
825
844
|
## Exports
|
|
@@ -881,7 +900,7 @@ WorkflowReplayFirstNonReplay, WorkflowReplayEligibility,
|
|
|
881
900
|
WorkflowPathOptions, RunPersistence, RunPersistenceOptions,
|
|
882
901
|
AcpPoolOptions, AcpRunnerOptions, AgentRunner, RunOptions, AgentResult, AgentUsage, JournalEntry,
|
|
883
902
|
AgentSessionRef, AgentSessionRecord, WorkflowBackendConfig, WorkflowCallRecord, WorkflowRecordedError,
|
|
884
|
-
InteractiveSessionOptions, InteractiveTurn, ProbedConfigOptions, SessionConfigOption,
|
|
903
|
+
InteractiveSessionOptions, InteractiveTurn, ProbeConfigOptionsOptions, ProbedConfigOptions, SessionConfigOption,
|
|
885
904
|
PermissionResolver,
|
|
886
905
|
AuthResolver, AuthContext, AuthResolution, AuthMethodDescriptor, AuthCapableRunner,
|
|
887
906
|
ProviderCapableRunner, // duck-type gate for the MCP provider tools (providers/list|set|disable)
|
package/dist/index.d.ts
CHANGED
|
@@ -7,7 +7,7 @@ export { runWorkflow, parseWorkflowScript, hashCheckpointInputs, resolveAgentTim
|
|
|
7
7
|
export { runIsolation, createReplayRunner, type RunIsolationSdkOptions } from "./isolation.js";
|
|
8
8
|
export type { RunIsolationOptions, IsolationRunResult, ReplayRunnerOptions, ResolvedIsolationTarget, IsolationTarget, ReplayRunner, ReplayObservation, ReplayReport, ReplayCallReport, ReplayDivergenceEvent, CheckpointCallContext, WorkflowCallRecord, WorkflowRecordedError, } from "./isolation.js";
|
|
9
9
|
export { openWorkflowDir, type WorkflowDir, type WorkflowDirEntry, type OpenWorkflowDirOptions, } from "@automatalabs/workflow-engine";
|
|
10
|
-
export { validateWorkflowScript, fabricateFromSchema, formatValidateReport, MOCK_TOKENS_PER_AGENT } from "./validate.js";
|
|
10
|
+
export { validateWorkflowScript, fabricateFromSchema, formatValidateReport, MOCK_TOKENS_PER_AGENT, ORDERED_THOUGHT_LEVEL_ENUMERATION_MODEL_LIMIT, } from "./validate.js";
|
|
11
11
|
export type { MockAnswerJson, MockAnswerRule, MockAnswers, MockAnswerSequence, UnusedMockAnswer, ValidateWorkflowOptions, ValidateWorkflowReport, ValidateHarnessOptions, ValidatedAgentCall, ValidatedCheckpoint, ValidatedMockAnswerRule, ValidatedMockAnswers, ValidatedMockAnswerUse, } from "./validate.js";
|
|
12
12
|
export { probeHarnessConfig, formatHarnessConfigReport } from "./config.js";
|
|
13
13
|
export type { ProbeHarnessConfigOptions, HarnessConfigReport } from "./config.js";
|
|
@@ -16,7 +16,7 @@ export { AGENTPRISM_PERSISTENCE_ROOT_ENV, WorkflowError, WorkflowErrorCode, isWo
|
|
|
16
16
|
export { withRunEvents, RunEventLogError, RUN_EVENT_READ_LIMIT_DEFAULT, RUN_EVENT_READ_LIMIT_MAX, RUN_EVENT_MAX_RECORD_BYTES, } from "@automatalabs/workflow-engine";
|
|
17
17
|
export type { RunEventPersistence, RunEventStream, AppendRunEventInput, ReadRunEventsOptions, ReadRunEventsResult, WatchRunEventsOptions, RunEventLogErrorCode, } from "@automatalabs/workflow-engine";
|
|
18
18
|
export { createAcpRunner, AcpAgentRunner, InteractiveSession, selectBackend, ClaudeBackend, CodexBackend, CustomAcpBackend, AGENT_METHODS, CLIENT_METHODS, AGENT_METHOD_COVERAGE, CLIENT_METHOD_COVERAGE, ACP_AUTH_REQUIRED_ERROR_CODE, clientCapabilitiesFor, adaptPromptContent, resolveBackendRegistry, BACKENDS_ENV, toJsonSchema, toStrictJsonSchema, } from "@automatalabs/acp-agents";
|
|
19
|
-
export type { AcpPoolOptions, AcpRunnerOptions, AuthenticateOptions, AuthMethodsOptions, DisableProviderOptions, DeleteSessionOptions, InteractiveSessionOptions, InteractiveTurn, ListProvidersOptions, ListSessionsOptions, LogoutOptions, ProbedConfigOptions, ReattachSessionOptions, SetProviderOptions, BackendRegistry, CustomBackendConfig, RegisteredBackend, ClientCapabilityOptions, ClientHandlers, FsHandlers, McpHandlers, TerminalHandlers, AcpSessionContext, NegotiatedCapabilities, PermissionResolver, AgentAuthCapabilities, AgentRequestMethod, AgentRequestParamsByMethod, AgentRequestResponsesByMethod, AuthCapabilities, AuthEnvVar, AuthenticateRequest, AuthenticateResponse, AuthMethod, AuthMethodAgent, AuthMethodEnvVar, AuthMethodId, AuthMethodTerminal, ConnectMcpRequest, ConnectMcpResponse, DeleteSessionRequest, DeleteSessionResponse, DisableProviderRequest, DisableProviderResponse, DisconnectMcpRequest, DisconnectMcpResponse, ListProvidersRequest, ListProvidersResponse, ListSessionsRequest, ListSessionsResponse, LlmProtocol, LoadSessionRequest, LoadSessionResponse, LogoutCapabilities, LogoutRequest, LogoutResponse, McpConnectionId, McpServerAcp, McpServerAcpId, MessageMcpNotification, MessageMcpRequest, MessageMcpResponse, ProviderCurrentConfig, ProviderId, ProviderInfo, ProvidersCapabilities, ResumeSessionRequest, ResumeSessionResponse, SetProviderRequest, SetProviderResponse, AgentNotificationMethod, AgentNotificationParamsByMethod, CompleteElicitationNotification, CreateElicitationRequest, CreateElicitationResponse, ElicitationAcceptAction, ElicitationCapabilities, ElicitationContentValue, ElicitationFormCapabilities, ElicitationFormMode, ElicitationId, ElicitationPropertySchema, ElicitationRequestScope, ElicitationResolver, ElicitationSchema, ElicitationSchemaType, ElicitationSessionScope, ElicitationUrlCapabilities, ElicitationUrlMode, SessionMode, SessionModeState, SessionConfigOption, SessionInfo, SendRequestOptions, AgentMethodCoverage, ClientMethodCoverage, } from "@automatalabs/acp-agents";
|
|
19
|
+
export type { AcpPoolOptions, AcpRunnerOptions, AuthenticateOptions, AuthMethodsOptions, DisableProviderOptions, DeleteSessionOptions, InteractiveSessionOptions, InteractiveTurn, ListProvidersOptions, ListSessionsOptions, LogoutOptions, ProbeConfigOptionsOptions, ProbedConfigOptions, ReattachSessionOptions, SetProviderOptions, BackendRegistry, CustomBackendConfig, RegisteredBackend, ClientCapabilityOptions, ClientHandlers, FsHandlers, McpHandlers, TerminalHandlers, AcpSessionContext, NegotiatedCapabilities, PermissionResolver, AgentAuthCapabilities, AgentRequestMethod, AgentRequestParamsByMethod, AgentRequestResponsesByMethod, AuthCapabilities, AuthEnvVar, AuthenticateRequest, AuthenticateResponse, AuthMethod, AuthMethodAgent, AuthMethodEnvVar, AuthMethodId, AuthMethodTerminal, ConnectMcpRequest, ConnectMcpResponse, DeleteSessionRequest, DeleteSessionResponse, DisableProviderRequest, DisableProviderResponse, DisconnectMcpRequest, DisconnectMcpResponse, ListProvidersRequest, ListProvidersResponse, ListSessionsRequest, ListSessionsResponse, LlmProtocol, LoadSessionRequest, LoadSessionResponse, LogoutCapabilities, LogoutRequest, LogoutResponse, McpConnectionId, McpServerAcp, McpServerAcpId, MessageMcpNotification, MessageMcpRequest, MessageMcpResponse, ProviderCurrentConfig, ProviderId, ProviderInfo, ProvidersCapabilities, ResumeSessionRequest, ResumeSessionResponse, SetProviderRequest, SetProviderResponse, AgentNotificationMethod, AgentNotificationParamsByMethod, CompleteElicitationNotification, CreateElicitationRequest, CreateElicitationResponse, ElicitationAcceptAction, ElicitationCapabilities, ElicitationContentValue, ElicitationFormCapabilities, ElicitationFormMode, ElicitationId, ElicitationPropertySchema, ElicitationRequestScope, ElicitationResolver, ElicitationSchema, ElicitationSchemaType, ElicitationSessionScope, ElicitationUrlCapabilities, ElicitationUrlMode, SessionMode, SessionModeState, SessionConfigOption, SessionInfo, SendRequestOptions, AgentMethodCoverage, ClientMethodCoverage, } from "@automatalabs/acp-agents";
|
|
20
20
|
export type { AuthResolver, AuthContext, AuthResolution, AuthMethodDescriptor, CompleteAuthOptions, AuthOutcome, AuthController, AuthStatusSnapshot, AuthCapableRunner, ProviderCapableRunner, } from "@automatalabs/acp-agents";
|
|
21
21
|
export type { ProviderUsageLimitContext, AuthErrorContext, CheckpointContext, } from "@automatalabs/shared-types";
|
|
22
22
|
export { TypedEventEmitter } from "@automatalabs/acp-agents";
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAaA,OAAO,EAKL,eAAe,IAAI,qBAAqB,EACzC,MAAM,+BAA+B,CAAC;AACvC,OAAO,KAAK,EAAoB,YAAY,EAAE,iBAAiB,EAAiB,MAAM,0BAA0B,CAAC;AACjH,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACtG,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAyB,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAClH,OAAO,EAAyB,KAAK,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAMzF,OAAO,EACL,WAAW,EACX,mBAAmB,EACnB,oBAAoB,EACpB,qBAAqB,EACrB,wBAAwB,EACxB,UAAU,EACV,YAAY,EACZ,gBAAgB,EAChB,kBAAkB,EAClB,wBAAwB,EACxB,uBAAuB,EACvB,uBAAuB,EACvB,wBAAwB,EACxB,0BAA0B,GAC3B,MAAM,+BAA+B,CAAC;AAIvC,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,KAAK,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAC/F,YAAY,EACV,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,EACf,YAAY,EACZ,iBAAiB,EACjB,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EACrB,qBAAqB,EACrB,kBAAkB,EAClB,qBAAqB,GACtB,MAAM,gBAAgB,CAAC;AAOxB,OAAO,EACL,eAAe,EACf,KAAK,WAAW,EAChB,KAAK,gBAAgB,EACrB,KAAK,sBAAsB,GAC5B,MAAM,+BAA+B,CAAC;AAIvC,OAAO,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAaA,OAAO,EAKL,eAAe,IAAI,qBAAqB,EACzC,MAAM,+BAA+B,CAAC;AACvC,OAAO,KAAK,EAAoB,YAAY,EAAE,iBAAiB,EAAiB,MAAM,0BAA0B,CAAC;AACjH,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,sBAAsB,EAAE,MAAM,+BAA+B,CAAC;AACtG,OAAO,KAAK,EAAE,WAAW,EAAE,QAAQ,EAAyB,iBAAiB,EAAE,MAAM,4BAA4B,CAAC;AAClH,OAAO,EAAyB,KAAK,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAMzF,OAAO,EACL,WAAW,EACX,mBAAmB,EACnB,oBAAoB,EACpB,qBAAqB,EACrB,wBAAwB,EACxB,UAAU,EACV,YAAY,EACZ,gBAAgB,EAChB,kBAAkB,EAClB,wBAAwB,EACxB,uBAAuB,EACvB,uBAAuB,EACvB,wBAAwB,EACxB,0BAA0B,GAC3B,MAAM,+BAA+B,CAAC;AAIvC,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,KAAK,sBAAsB,EAAE,MAAM,gBAAgB,CAAC;AAC/F,YAAY,EACV,mBAAmB,EACnB,kBAAkB,EAClB,mBAAmB,EACnB,uBAAuB,EACvB,eAAe,EACf,YAAY,EACZ,iBAAiB,EACjB,YAAY,EACZ,gBAAgB,EAChB,qBAAqB,EACrB,qBAAqB,EACrB,kBAAkB,EAClB,qBAAqB,GACtB,MAAM,gBAAgB,CAAC;AAOxB,OAAO,EACL,eAAe,EACf,KAAK,WAAW,EAChB,KAAK,gBAAgB,EACrB,KAAK,sBAAsB,GAC5B,MAAM,+BAA+B,CAAC;AAIvC,OAAO,EACL,sBAAsB,EACtB,mBAAmB,EACnB,oBAAoB,EACpB,qBAAqB,EACrB,6CAA6C,GAC9C,MAAM,eAAe,CAAC;AACvB,YAAY,EACV,cAAc,EACd,cAAc,EACd,WAAW,EACX,kBAAkB,EAClB,gBAAgB,EAChB,uBAAuB,EACvB,sBAAsB,EACtB,sBAAsB,EACtB,kBAAkB,EAClB,mBAAmB,EACnB,uBAAuB,EACvB,oBAAoB,EACpB,sBAAsB,GACvB,MAAM,eAAe,CAAC;AAKvB,OAAO,EAAE,kBAAkB,EAAE,yBAAyB,EAAE,MAAM,aAAa,CAAC;AAC5E,YAAY,EAAE,yBAAyB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AAClF,YAAY,EACV,kBAAkB,EAClB,uBAAuB,EACvB,2BAA2B,EAC3B,6BAA6B,EAC7B,YAAY,EACZ,WAAW,EACX,sBAAsB,EACtB,iBAAiB,EACjB,iBAAiB,EACjB,gBAAgB,EAChB,mBAAmB,EACnB,cAAc,EACd,qBAAqB,EACrB,iBAAiB,EACjB,mBAAmB,EACnB,qBAAqB,EACrB,wBAAwB,EACxB,0BAA0B,EAC1B,4BAA4B,EAC5B,mBAAmB,EACnB,4BAA4B,EAC5B,cAAc,EACd,eAAe,EACf,qBAAqB,EACrB,4BAA4B,EAC5B,iBAAiB,EACjB,iBAAiB,EACjB,2BAA2B,EAC3B,mBAAmB,EACnB,wBAAwB,EACxB,uBAAuB,EACvB,YAAY,EACZ,sBAAsB,EACtB,mBAAmB,EACnB,4BAA4B,EAC5B,4BAA4B,EAC5B,4BAA4B,EAC5B,8BAA8B,EAC9B,oBAAoB,EACpB,4BAA4B,EAC5B,0BAA0B,EAC1B,oBAAoB,EACpB,+BAA+B,EAC/B,+BAA+B,EAC/B,6BAA6B,EAC7B,8BAA8B,EAC9B,4BAA4B,EAC5B,yBAAyB,GAC1B,MAAM,+BAA+B,CAAC;AACvC,OAAO,EACL,+BAA+B,EAC/B,aAAa,EACb,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,cAAc,GACf,MAAM,+BAA+B,CAAC;AAMvC,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,4BAA4B,EAC5B,wBAAwB,EACxB,0BAA0B,GAC3B,MAAM,+BAA+B,CAAC;AACvC,YAAY,EACV,mBAAmB,EACnB,cAAc,EACd,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,EACnB,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,+BAA+B,CAAC;AAQvC,OAAO,EACL,eAAe,EACf,cAAc,EACd,kBAAkB,EAClB,aAAa,EACb,aAAa,EACb,YAAY,EACZ,gBAAgB,EAChB,aAAa,EACb,cAAc,EACd,qBAAqB,EACrB,sBAAsB,EACtB,4BAA4B,EAC5B,qBAAqB,EACrB,kBAAkB,EAClB,sBAAsB,EACtB,YAAY,EACZ,YAAY,EACZ,kBAAkB,GACnB,MAAM,0BAA0B,CAAC;AAClC,YAAY,EACV,cAAc,EACd,gBAAgB,EAChB,mBAAmB,EACnB,kBAAkB,EAClB,sBAAsB,EACtB,oBAAoB,EACpB,yBAAyB,EACzB,eAAe,EACf,oBAAoB,EACpB,mBAAmB,EACnB,aAAa,EACb,yBAAyB,EACzB,mBAAmB,EACnB,sBAAsB,EACtB,kBAAkB,EAClB,eAAe,EACf,mBAAmB,EACnB,iBAAiB,EACjB,uBAAuB,EACvB,cAAc,EACd,UAAU,EACV,WAAW,EACX,gBAAgB,EAChB,iBAAiB,EACjB,sBAAsB,EACtB,kBAAkB,EAClB,qBAAqB,EACrB,kBAAkB,EAClB,0BAA0B,EAC1B,6BAA6B,EAC7B,gBAAgB,EAChB,UAAU,EACV,mBAAmB,EACnB,oBAAoB,EACpB,UAAU,EACV,eAAe,EACf,gBAAgB,EAChB,YAAY,EACZ,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EACpB,qBAAqB,EACrB,sBAAsB,EACtB,uBAAuB,EACvB,oBAAoB,EACpB,qBAAqB,EACrB,oBAAoB,EACpB,qBAAqB,EACrB,mBAAmB,EACnB,oBAAoB,EACpB,WAAW,EACX,kBAAkB,EAClB,mBAAmB,EACnB,kBAAkB,EAClB,aAAa,EACb,cAAc,EACd,eAAe,EACf,YAAY,EACZ,cAAc,EACd,sBAAsB,EACtB,iBAAiB,EACjB,kBAAkB,EAClB,qBAAqB,EACrB,UAAU,EACV,YAAY,EACZ,qBAAqB,EACrB,oBAAoB,EACpB,qBAAqB,EACrB,kBAAkB,EAClB,mBAAmB,EACnB,uBAAuB,EACvB,+BAA+B,EAC/B,+BAA+B,EAC/B,wBAAwB,EACxB,yBAAyB,EACzB,uBAAuB,EACvB,uBAAuB,EACvB,uBAAuB,EACvB,2BAA2B,EAC3B,mBAAmB,EACnB,aAAa,EACb,yBAAyB,EACzB,uBAAuB,EACvB,mBAAmB,EACnB,iBAAiB,EACjB,qBAAqB,EACrB,uBAAuB,EACvB,0BAA0B,EAC1B,kBAAkB,EAClB,WAAW,EACX,gBAAgB,EAChB,mBAAmB,EACnB,WAAW,EACX,kBAAkB,EAClB,mBAAmB,EACnB,oBAAoB,GACrB,MAAM,0BAA0B,CAAC;AAOlC,YAAY,EACV,YAAY,EACZ,WAAW,EACX,cAAc,EACd,oBAAoB,EACpB,mBAAmB,EACnB,WAAW,EACX,cAAc,EACd,kBAAkB,EAClB,iBAAiB,EACjB,qBAAqB,GACtB,MAAM,0BAA0B,CAAC;AAClC,YAAY,EACV,yBAAyB,EACzB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,4BAA4B,CAAC;AAMpC,OAAO,EAAE,iBAAiB,EAAE,MAAM,0BAA0B,CAAC;AAC7D,YAAY,EACV,iBAAiB,EACjB,YAAY,EACZ,gBAAgB,EAChB,eAAe,EACf,gBAAgB,EAChB,aAAa,EACb,2BAA2B,EAC3B,mBAAmB,EACnB,0BAA0B,EAC1B,yBAAyB,EACzB,kBAAkB,EAClB,kBAAkB,EAClB,oBAAoB,GACrB,MAAM,0BAA0B,CAAC;AAIlC,YAAY,EACV,WAAW,EACX,kBAAkB,EAClB,eAAe,EACf,UAAU,EACV,WAAW,EACX,UAAU,GACX,MAAM,4BAA4B,CAAC;AACpC,YAAY,EACV,kBAAkB,EAClB,eAAe,EACf,mBAAmB,EACnB,YAAY,EACZ,qBAAqB,EACrB,YAAY,GACb,MAAM,4BAA4B,CAAC;AAMpC,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AACnE,YAAY,EACV,QAAQ,EACR,cAAc,EACd,kBAAkB,EAClB,wBAAwB,EACxB,yBAAyB,EACzB,iBAAiB,EACjB,iBAAiB,EACjB,uBAAuB,EACvB,uBAAuB,EACvB,4BAA4B,EAC5B,2BAA2B,GAC5B,MAAM,4BAA4B,CAAC;AAwBpC,KAAK,eAAe,CAAC,CAAC,EAAE,GAAG,SAAS,WAAW,IAAI,GAAG,SAAS,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;AACxF,KAAK,uBAAuB,CAAC,CAAC,EAAE,GAAG,SAAS,WAAW,IACrD,GAAG,SAAS,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;AAE3C,6FAA6F;AAC7F,MAAM,MAAM,sBAAsB,GAAG,OAAO,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC;AAE7E;;;;GAIG;AACH,KAAK,6BAA6B,GAAG;KAClC,IAAI,IAAI,YAAY,GAAG;QACtB,IAAI,EAAE,IAAI,CAAC;QACX,KAAK,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAC/B,SAAS,EAAE,eAAe,CAAC,iBAAiB,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC,CAAC;KAClE,GAAG,CAAC,WAAW,SAAS,MAAM,iBAAiB,CAAC,IAAI,CAAC,GAClD;QAAE,SAAS,EAAE,eAAe,CAAC,iBAAiB,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC,CAAA;KAAE,GACpE;QAAE,SAAS,CAAC,EAAE,SAAS,CAAA;KAAE,CAAC,GAAG;QAC7B,KAAK,CAAC,EAAE,uBAAuB,CAAC,iBAAiB,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;QAClE,KAAK,CAAC,EAAE,uBAAuB,CAAC,iBAAiB,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;QAClE,KAAK,CAAC,EAAE,uBAAuB,CAAC,iBAAiB,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,CAAC;QAClE,SAAS,CAAC,EAAE,uBAAuB,CAAC,iBAAiB,CAAC,IAAI,CAAC,EAAE,WAAW,CAAC,CAAC;KAC3E;CACJ,CAAC;AAEF,MAAM,MAAM,4BAA4B,GAAG,IAAI,CAC7C,6BAA6B,EAC7B,sBAAsB,CACvB,CAAC;AAEF,MAAM,MAAM,yBAAyB,CACnC,IAAI,SAAS,sBAAsB,GAAG,sBAAsB,IAC1D,4BAA4B,CAAC,IAAI,CAAC,CAAC;AAEvC,MAAM,MAAM,kBAAkB,GAAG;KAC9B,IAAI,IAAI,sBAAsB,GAC7B;QAAE,IAAI,EAAE,YAAY,CAAA;KAAE,GAAG,yBAAyB,CAAC,IAAI,CAAC;CAC3D,CAAC,sBAAsB,CAAC,CAAC;AAE1B,MAAM,MAAM,gBAAgB,GAAG,QAAQ,CAAC,kBAAkB,CAAC,CAAC;AAE5D;;;;GAIG;AACH,MAAM,MAAM,iBAAiB,CAC3B,IAAI,SAAS,YAAY,GAAG,YAAY,IACtC,6BAA6B,CAAC,IAAI,CAAC,CAAC;AAExC,MAAM,WAAW,eAAe;IAC9B,WAAW,CAAC,SAAS,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,GAAG,IAAI,CAAC;IACnG,EAAE,CAAC,SAAS,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,GAAG,IAAI,CAAC;IAC1F,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,GAAG,IAAI,CAAC;IAC5F,cAAc,CAAC,SAAS,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,GAAG,IAAI,CAAC;IACtG,GAAG,CAAC,SAAS,EAAE,YAAY,EAAE,QAAQ,EAAE,CAAC,OAAO,EAAE,yBAAyB,KAAK,IAAI,GAAG,IAAI,CAAC;IAC3F,IAAI,CAAC,SAAS,EAAE,YAAY,EAAE,OAAO,EAAE,yBAAyB,GAAG,OAAO,CAAC;IAC3E,WAAW,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,GAAG,IAAI,CAAC;IAClF,EAAE,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,GAAG,IAAI,CAAC;IACzE,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,GAAG,IAAI,CAAC;IAC3E,cAAc,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,GAAG,IAAI,CAAC;IACrF,GAAG,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,IAAI,GAAG,IAAI,CAAC;IAC1E,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;CAC3D;AAED;;;;;;;;;GASG;AACH,qBAAa,eAAgB,SAAQ,qBAAqB;IACxD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAgD;gBAE/D,OAAO,GAAE,sBAA2B;IAKvC,iBAAiB,CACxB,MAAM,EAAE,MAAM,EACd,IAAI,CAAC,EAAE,OAAO,EACd,IAAI,GAAE,WAAgB,GACrB;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAA;KAAE;IAY1C,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,GAAE,WAAgB,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAS3F,kBAAkB,CAC/B,KAAK,EAAE,MAAM,EACb,IAAI,GAAE,WAAgB,GACrB,OAAO,CACN;QAAE,QAAQ,EAAE,KAAK,CAAC;QAAC,OAAO,CAAC,EAAE,SAAS,CAAA;KAAE,GACxC;QAAE,QAAQ,EAAE,IAAI,CAAC;QAAC,OAAO,EAAE,OAAO,CAAC,iBAAiB,CAAC,CAAA;KAAE,CAC1D;IAgBc,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,GAAE,WAAgB,GAAG,OAAO,CAAC,OAAO,CAAC;IAK9E;8FAC0F;IAC1F,OAAO,IAAI,IAAI;IAOf,8EAA8E;IAC9E,KAAK,IAAI,IAAI;IAIb,OAAO,CAAC,sBAAsB;CAiC/B;AA2CD;;;;;;;;GAQG;AACH,YAAY,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAElE,8CAA8C;AAC9C,MAAM,WAAW,yBAAyB;IACxC;;;;OAIG;IACH,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB;;;;OAIG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,+EAA+E;IAC/E,IAAI,CAAC,EAAE,OAAO,CAAC;IACf,kGAAkG;IAClG,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,+FAA+F;IAC/F,mBAAmB,CAAC,EAAE,qBAAqB,CAAC;IAC5C;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,WAAW,CAAC;CAC7C;AAED;;;;;;;;;GASG;AACH,wBAAsB,kBAAkB,CACtC,MAAM,EAAE,MAAM,EACd,IAAI,GAAE,yBAA8B,GACnC,OAAO,CAAC,iBAAiB,CAAC,CAsC5B"}
|
package/dist/index.js
CHANGED
|
@@ -26,7 +26,7 @@ export { runIsolation, createReplayRunner } from "./isolation.js";
|
|
|
26
26
|
export { openWorkflowDir, } from "@automatalabs/workflow-engine";
|
|
27
27
|
// ── Token-free script validation: static parse + mock-runner dry run. Also the core of
|
|
28
28
|
// the `agentprism-workflows validate` CLI (./cli.ts). ──
|
|
29
|
-
export { validateWorkflowScript, fabricateFromSchema, formatValidateReport, MOCK_TOKENS_PER_AGENT } from "./validate.js";
|
|
29
|
+
export { validateWorkflowScript, fabricateFromSchema, formatValidateReport, MOCK_TOKENS_PER_AGENT, ORDERED_THOUGHT_LEVEL_ENUMERATION_MODEL_LIMIT, } from "./validate.js";
|
|
30
30
|
// ── Harness config discovery: validate's sibling (`agentprism-workflows config`) — probe
|
|
31
31
|
// any routable ACP harness's advertised config-option catalog (model ids, effort levels,
|
|
32
32
|
// modes, …) without authoring a script. ──
|
|
@@ -2,6 +2,7 @@ import { type CustomBackendConfig, type ProbedConfigOptions } from "@automatalab
|
|
|
2
2
|
export interface ValidateProbeRunner {
|
|
3
3
|
probeConfigOptions(spec?: string, opts?: {
|
|
4
4
|
cwd?: string;
|
|
5
|
+
selectModel?: boolean;
|
|
5
6
|
}): Promise<ProbedConfigOptions>;
|
|
6
7
|
dispose(): Promise<void>;
|
|
7
8
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate-internal.d.ts","sourceRoot":"","sources":["../src/validate-internal.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACzB,MAAM,0BAA0B,CAAC;AAElC,MAAM,WAAW,mBAAmB;IAClC,kBAAkB,
|
|
1
|
+
{"version":3,"file":"validate-internal.d.ts","sourceRoot":"","sources":["../src/validate-internal.ts"],"names":[],"mappings":"AAAA,OAAO,EAEL,KAAK,mBAAmB,EACxB,KAAK,mBAAmB,EACzB,MAAM,0BAA0B,CAAC;AAElC,MAAM,WAAW,mBAAmB;IAClC,kBAAkB,CAChB,IAAI,CAAC,EAAE,MAAM,EACb,IAAI,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,OAAO,CAAA;KAAE,GAC7C,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAChC,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CAC1B;AAED,MAAM,MAAM,oBAAoB,GAAG,CACjC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,GAAG,SAAS,KACtD,mBAAmB,CAAC;AAIzB,wBAAgB,yBAAyB,CACvC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,mBAAmB,CAAC,GAAG,SAAS,GACxD,mBAAmB,CAErB;AAED,6FAA6F;AAC7F,wBAAgB,+BAA+B,CAAC,OAAO,EAAE,oBAAoB,GAAG,MAAM,IAAI,CAMzF"}
|
package/dist/validate.d.ts
CHANGED
|
@@ -76,6 +76,8 @@ export interface ValidatedAgentCall {
|
|
|
76
76
|
}
|
|
77
77
|
export interface ValidateHarnessOptions {
|
|
78
78
|
backendId: string;
|
|
79
|
+
/** The call's verbatim selected model; absent means the harness/session default. */
|
|
80
|
+
model?: string;
|
|
79
81
|
probed: boolean;
|
|
80
82
|
/** Present when probed=false: the harness's spawn/auth/session error. */
|
|
81
83
|
error?: string;
|
|
@@ -108,7 +110,7 @@ export interface ValidateWorkflowReport {
|
|
|
108
110
|
phasesVisited: string[];
|
|
109
111
|
logs: string[];
|
|
110
112
|
durationMs: number;
|
|
111
|
-
/** Fresh, per-run advertised config-option catalogs for every routed
|
|
113
|
+
/** Fresh, per-run advertised config-option catalogs for every routed backend/model pair. */
|
|
112
114
|
harnessOptions?: ValidateHarnessOptions[];
|
|
113
115
|
/** The script's return value, composed from fabricated agent results. */
|
|
114
116
|
result?: unknown;
|
|
@@ -125,9 +127,12 @@ export declare function fabricateFromSchema(schema: unknown, hint?: string, dept
|
|
|
125
127
|
/** Tokens the mock runner reports per agent call, so `--token-budget` exercises
|
|
126
128
|
* budget-guarded script paths deterministically. */
|
|
127
129
|
export declare const MOCK_TOKENS_PER_AGENT = 1000;
|
|
130
|
+
/** Maximum advertised model count for client-side ordered thought-domain enumeration.
|
|
131
|
+
* Larger catalogs stay on exact advertised-value validation so zero-token validation remains bounded. */
|
|
132
|
+
export declare const ORDERED_THOUGHT_LEVEL_ENUMERATION_MODEL_LIMIT = 32;
|
|
128
133
|
/**
|
|
129
134
|
* Validate a workflow script: parse it, dry-run against a mock AgentRunner, then probe
|
|
130
|
-
* each routed
|
|
135
|
+
* each routed backend/model pair's advertised config options. Never throws for an invalid script —
|
|
131
136
|
* read `report.ok` / `report.exitCode`.
|
|
132
137
|
*/
|
|
133
138
|
export declare function validateWorkflowScript(script: string, options?: ValidateWorkflowOptions): Promise<ValidateWorkflowReport>;
|
package/dist/validate.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../src/validate.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../src/validate.ts"],"names":[],"mappings":"AA2BA,OAAO,KAAK,EAGV,mBAAmB,EAEpB,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AACjE,OAAO,KAAK,EAA2B,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAIxF,MAAM,MAAM,cAAc,GACtB,IAAI,GACJ,OAAO,GACP,MAAM,GACN,MAAM,GACN,cAAc,EAAE,GAChB;IAAE,CAAC,GAAG,EAAE,MAAM,GAAG,cAAc,CAAA;CAAE,CAAC;AAEtC,MAAM,WAAW,kBAAkB;IACjC,QAAQ,CAAC,SAAS,EAAE,SAAS,cAAc,EAAE,CAAC;CAC/C;AAED,MAAM,MAAM,cAAc,GAAG,cAAc,GAAG,kBAAkB,CAAC;AAEjE,uEAAuE;AACvE,MAAM,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;AAEnE,MAAM,WAAW,uBAAuB;IACtC,iEAAiE;IACjE,IAAI,CAAC,EAAE,OAAO,CAAC;IACf;2FACuF;IACvF,SAAS,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,WAAW,CAAC;IAC5C;6EACyE;IACzE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,4DAA4D;IAC5D,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB;8CAC0C;IAC1C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,qEAAqE;IACrE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,mDAAmD;IACnD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,4DAA4D;IAC5D,WAAW,CAAC,EAAE,WAAW,CAAC;CAC3B;AAED,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,6EAA6E;IAC7E,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,uBAAuB;IACtC,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,QAAQ,GAAG,UAAU,CAAC;IAC5B,yFAAyF;IACzF,aAAa,EAAE,MAAM,CAAC;IACtB,mGAAmG;IACnG,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,qEAAqE;IACrE,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,UAAU,GAAG,UAAU,GAAG,aAAa,CAAC;CACjD;AAED,MAAM,WAAW,oBAAoB;IACnC,kFAAkF;IAClF,KAAK,EAAE,uBAAuB,EAAE,CAAC;IACjC,MAAM,EAAE,gBAAgB,EAAE,CAAC;CAC5B;AAED,kFAAkF;AAClF,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wFAAwF;IACxF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kEAAkE;IAClE,aAAa,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC;IACjD;kEAC8D;IAC9D,OAAO,EAAE,MAAM,CAAC;IAChB,sDAAsD;IACtD,MAAM,EAAE,OAAO,CAAC;IAChB,UAAU,CAAC,EAAE,sBAAsB,CAAC;CACrC;AAED,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,oFAAoF;IACpF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,CAAC;IAChB,yEAAyE;IACzE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,mBAAmB,EAAE,CAAC;CACjC;AAED,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,8FAA8F;IAC9F,KAAK,EAAE,OAAO,CAAC;CAChB;AAED,MAAM,WAAW,sBAAsB;IACrC,0FAA0F;IAC1F,EAAE,EAAE,OAAO,CAAC;IACZ,iFAAiF;IACjF,QAAQ,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACpB,KAAK,EAAE;QACL,EAAE,EAAE,OAAO,CAAC;QACZ,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,IAAI,CAAC,EAAE,YAAY,CAAC;KACrB,CAAC;IACF,MAAM,CAAC,EAAE;QACP,EAAE,EAAE,OAAO,CAAC;QACZ,MAAM,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,0EAA0E;QAC1E,QAAQ,EAAE,OAAO,CAAC;QAClB,UAAU,EAAE,kBAAkB,EAAE,CAAC;QACjC,WAAW,EAAE,mBAAmB,EAAE,CAAC;QACnC,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,IAAI,EAAE,MAAM,EAAE,CAAC;QACf,UAAU,EAAE,MAAM,CAAC;QACnB,4FAA4F;QAC5F,cAAc,CAAC,EAAE,sBAAsB,EAAE,CAAC;QAC1C,yEAAyE;QACzE,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,WAAW,CAAC,EAAE,oBAAoB,CAAC;KACpC,CAAC;IACF,QAAQ,EAAE,MAAM,EAAE,CAAC;CACpB;AAyfD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,SAAU,EAAE,KAAK,SAAI,GAAG,OAAO,CA6DvF;AAeD;qDACqD;AACrD,eAAO,MAAM,qBAAqB,OAAO,CAAC;AAE1C;0GAC0G;AAC1G,eAAO,MAAM,6CAA6C,KAAK,CAAC;AA+gBhE;;;;GAIG;AACH,wBAAsB,sBAAsB,CAC1C,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,uBAA4B,GACpC,OAAO,CAAC,sBAAsB,CAAC,CA6QjC;AAMD;;qEAEqE;AACrE,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,SAAS,sBAAsB,EAAE,EAC5C,MAAM,EAAE,MAAM,GACb,MAAM,EAAE,CAwBV;AAED,wEAAwE;AACxE,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,sBAAsB,GAAG,MAAM,CAyC3E"}
|
package/dist/validate.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// direct nondeterministic call expressions) followed by an optional DRY RUN — the script
|
|
3
3
|
// executes for real in the engine's deterministic realm, but every agent() call is served by
|
|
4
4
|
// an in-process mock AgentRunner that fabricates schema-conforming results. Afterward, each
|
|
5
|
-
// routed ACP
|
|
5
|
+
// routed ACP backend/model pair is opened once without a prompt to read its advertised config options.
|
|
6
6
|
// No tokens are spent, a mock live confirm resolves checkpoints to their declared defaults,
|
|
7
7
|
// and run state is journaled nowhere (journaling off + a throwaway persistence root for the run lease).
|
|
8
8
|
//
|
|
@@ -11,7 +11,7 @@ import { mkdtempSync, rmSync } from "node:fs";
|
|
|
11
11
|
import { tmpdir } from "node:os";
|
|
12
12
|
import { join } from "node:path";
|
|
13
13
|
import { openWorkflowDir, WorkflowError, WorkflowErrorCode, WorkflowManager, parseWorkflowScript, redactText, } from "@automatalabs/workflow-engine";
|
|
14
|
-
import { registryWithRunBackends, resolveBackendRegistry, selectBackend, } from "@automatalabs/acp-agents";
|
|
14
|
+
import { builtinThoughtLevelDomainSemantics, registryWithRunBackends, resolveBackendRegistry, selectBackend, } from "@automatalabs/acp-agents";
|
|
15
15
|
import { Check, Errors } from "typebox/value";
|
|
16
16
|
import { createValidateProbeRunner } from "./validate-internal.js";
|
|
17
17
|
const MAX_MOCK_ANSWERS_BYTES = 256 * 1024;
|
|
@@ -20,6 +20,7 @@ const MAX_MOCK_ANSWER_GLOB_LENGTH = 256;
|
|
|
20
20
|
const MAX_MOCK_ANSWER_SEQUENCE_LENGTH = 256;
|
|
21
21
|
const MAX_MOCK_ANSWER_DEPTH = 32;
|
|
22
22
|
const MAX_FIXTURE_REASON_LENGTH = 1024;
|
|
23
|
+
const CONFIG_OPTION_META_NAMESPACE = "@automatalabs/agentprism";
|
|
23
24
|
function defineDataProperty(target, key, value) {
|
|
24
25
|
Object.defineProperty(target, key, {
|
|
25
26
|
value,
|
|
@@ -552,13 +553,22 @@ export function fabricateFromSchema(schema, hint = "value", depth = 0) {
|
|
|
552
553
|
/** Tokens the mock runner reports per agent call, so `--token-budget` exercises
|
|
553
554
|
* budget-guarded script paths deterministically. */
|
|
554
555
|
export const MOCK_TOKENS_PER_AGENT = 1000;
|
|
556
|
+
/** Maximum advertised model count for client-side ordered thought-domain enumeration.
|
|
557
|
+
* Larger catalogs stay on exact advertised-value validation so zero-token validation remains bounded. */
|
|
558
|
+
export const ORDERED_THOUGHT_LEVEL_ENUMERATION_MODEL_LIMIT = 32;
|
|
555
559
|
function routeBackend(model, tier, registry, hostRegistry, declared) {
|
|
556
|
-
const
|
|
560
|
+
const backend = selectBackend({ model, tier }, registry);
|
|
561
|
+
const backendId = backend.id;
|
|
557
562
|
const scriptDeclared = !hostRegistry.has(backendId) &&
|
|
558
563
|
Object.keys(declared ?? {}).some((name) => name.toLowerCase() === backendId.toLowerCase());
|
|
559
564
|
return {
|
|
560
565
|
backendId,
|
|
561
566
|
display: scriptDeclared ? `${backendId} (script-declared)` : backendId,
|
|
567
|
+
// Auth profiles are a built-in-only contract. A custom backend that shadows a built-in id
|
|
568
|
+
// therefore stays exact-set instead of inheriting that built-in's ordering declaration.
|
|
569
|
+
thoughtLevelDomainSemantics: (backend.authProfile === undefined
|
|
570
|
+
? undefined
|
|
571
|
+
: builtinThoughtLevelDomainSemantics(backendId)) ?? "exact-set",
|
|
562
572
|
};
|
|
563
573
|
}
|
|
564
574
|
function registryOptions(registry) {
|
|
@@ -569,13 +579,35 @@ function registryOptions(registry) {
|
|
|
569
579
|
return [name, config];
|
|
570
580
|
}));
|
|
571
581
|
}
|
|
582
|
+
function configCatalogKey(backendId, model) {
|
|
583
|
+
return JSON.stringify([backendId, model ?? null]);
|
|
584
|
+
}
|
|
585
|
+
function configProbeTargets(calls, registry, hostRegistry, declared) {
|
|
586
|
+
const targets = new Map();
|
|
587
|
+
for (const call of calls) {
|
|
588
|
+
const routed = routeBackend(call.model, call.tier, registry, hostRegistry, declared);
|
|
589
|
+
const backendId = routed.backendId;
|
|
590
|
+
const target = call.model === undefined
|
|
591
|
+
? { backendId, thoughtLevelDomainSemantics: routed.thoughtLevelDomainSemantics }
|
|
592
|
+
: {
|
|
593
|
+
backendId,
|
|
594
|
+
model: call.model,
|
|
595
|
+
thoughtLevelDomainSemantics: routed.thoughtLevelDomainSemantics,
|
|
596
|
+
};
|
|
597
|
+
targets.set(configCatalogKey(backendId, call.model), target);
|
|
598
|
+
}
|
|
599
|
+
return [...targets.values()].sort((left, right) => left.backendId.localeCompare(right.backendId) || (left.model ?? "").localeCompare(right.model ?? ""));
|
|
600
|
+
}
|
|
601
|
+
function probeTargetName(target) {
|
|
602
|
+
return target.model === undefined
|
|
603
|
+
? `${target.backendId} (session default model)`
|
|
604
|
+
: `${target.backendId} model ${JSON.stringify(target.model)}`;
|
|
605
|
+
}
|
|
572
606
|
async function probeHarnessConfigOptions(calls, cwd, registry, hostRegistry, declared, warnings) {
|
|
573
|
-
const
|
|
574
|
-
...new Set(calls.map((call) => routeBackend(call.model, call.tier, registry, hostRegistry, declared).backendId)),
|
|
575
|
-
].sort();
|
|
607
|
+
const targets = configProbeTargets(calls, registry, hostRegistry, declared);
|
|
576
608
|
const harnessOptions = [];
|
|
577
609
|
const catalogs = new Map();
|
|
578
|
-
if (
|
|
610
|
+
if (targets.length === 0)
|
|
579
611
|
return { harnessOptions, catalogs };
|
|
580
612
|
let runner;
|
|
581
613
|
try {
|
|
@@ -583,28 +615,73 @@ async function probeHarnessConfigOptions(calls, cwd, registry, hostRegistry, dec
|
|
|
583
615
|
}
|
|
584
616
|
catch (error) {
|
|
585
617
|
const reason = errorMessage(error);
|
|
586
|
-
for (const
|
|
587
|
-
warnings.push(`could not probe ${
|
|
588
|
-
harnessOptions.push({
|
|
618
|
+
for (const target of targets) {
|
|
619
|
+
warnings.push(`could not probe ${probeTargetName(target)} — configOptions on its calls are unverified: ${reason}`);
|
|
620
|
+
harnessOptions.push({
|
|
621
|
+
backendId: target.backendId,
|
|
622
|
+
...(target.model === undefined ? {} : { model: target.model }),
|
|
623
|
+
probed: false,
|
|
624
|
+
error: reason,
|
|
625
|
+
});
|
|
589
626
|
}
|
|
590
627
|
return { harnessOptions, catalogs };
|
|
591
628
|
}
|
|
592
629
|
try {
|
|
593
|
-
|
|
630
|
+
const failures = new Map();
|
|
631
|
+
const probeTarget = async (target, reportHarness) => {
|
|
632
|
+
const key = configCatalogKey(target.backendId, target.model);
|
|
633
|
+
const cached = catalogs.get(key);
|
|
634
|
+
if (cached)
|
|
635
|
+
return cached;
|
|
636
|
+
if (failures.has(key))
|
|
637
|
+
return undefined;
|
|
594
638
|
try {
|
|
595
|
-
const result = await runner.probeConfigOptions(backendId, {
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
backendId: result.backendId,
|
|
599
|
-
probed: true,
|
|
600
|
-
options: result.options,
|
|
639
|
+
const result = await runner.probeConfigOptions(target.model ?? target.backendId, {
|
|
640
|
+
cwd,
|
|
641
|
+
selectModel: target.model !== undefined,
|
|
601
642
|
});
|
|
643
|
+
catalogs.set(key, result.options);
|
|
644
|
+
if (reportHarness) {
|
|
645
|
+
harnessOptions.push({
|
|
646
|
+
backendId: result.backendId,
|
|
647
|
+
...(target.model === undefined ? {} : { model: target.model }),
|
|
648
|
+
probed: true,
|
|
649
|
+
options: result.options,
|
|
650
|
+
});
|
|
651
|
+
}
|
|
652
|
+
return result.options;
|
|
602
653
|
}
|
|
603
654
|
catch (error) {
|
|
604
655
|
const reason = errorMessage(error);
|
|
605
|
-
|
|
606
|
-
|
|
656
|
+
failures.set(key, reason);
|
|
657
|
+
if (reportHarness) {
|
|
658
|
+
warnings.push(`could not probe ${probeTargetName(target)} — configOptions on its calls are unverified: ${reason}`);
|
|
659
|
+
harnessOptions.push({
|
|
660
|
+
backendId: target.backendId,
|
|
661
|
+
...(target.model === undefined ? {} : { model: target.model }),
|
|
662
|
+
probed: false,
|
|
663
|
+
error: reason,
|
|
664
|
+
});
|
|
665
|
+
}
|
|
666
|
+
return undefined;
|
|
607
667
|
}
|
|
668
|
+
};
|
|
669
|
+
for (const target of targets)
|
|
670
|
+
await probeTarget(target, true);
|
|
671
|
+
const orderedTargets = new Map();
|
|
672
|
+
for (const target of targets) {
|
|
673
|
+
if (target.thoughtLevelDomainSemantics !== "ordered")
|
|
674
|
+
continue;
|
|
675
|
+
const catalog = catalogs.get(configCatalogKey(target.backendId, target.model));
|
|
676
|
+
if (!catalog?.some((option) => isThoughtLevelSelect(option) &&
|
|
677
|
+
recognizedSelectValues(option, selectChoiceValues(option)) === undefined))
|
|
678
|
+
continue;
|
|
679
|
+
const backendTargets = orderedTargets.get(target.backendId) ?? [];
|
|
680
|
+
backendTargets.push(target);
|
|
681
|
+
orderedTargets.set(target.backendId, backendTargets);
|
|
682
|
+
}
|
|
683
|
+
for (const [backendId, backendTargets] of orderedTargets) {
|
|
684
|
+
await enumerateOrderedThoughtLevelDomains(backendId, backendTargets, catalogs, probeTarget, warnings);
|
|
608
685
|
}
|
|
609
686
|
}
|
|
610
687
|
finally {
|
|
@@ -617,16 +694,169 @@ async function probeHarnessConfigOptions(calls, cwd, registry, hostRegistry, dec
|
|
|
617
694
|
}
|
|
618
695
|
return { harnessOptions, catalogs };
|
|
619
696
|
}
|
|
697
|
+
function isThoughtLevelSelect(option) {
|
|
698
|
+
return option.type === "select" &&
|
|
699
|
+
(option.category === "thought_level" || option.id === "thinkingLevel");
|
|
700
|
+
}
|
|
701
|
+
const NON_ORDERED_THOUGHT_LEVEL_SENTINELS = new Set(["default"]);
|
|
702
|
+
async function enumerateOrderedThoughtLevelDomains(backendId, targets, catalogs, probeTarget, warnings) {
|
|
703
|
+
const seedCatalogs = targets.flatMap((target) => {
|
|
704
|
+
const catalog = catalogs.get(configCatalogKey(target.backendId, target.model));
|
|
705
|
+
return catalog ? [catalog] : [];
|
|
706
|
+
});
|
|
707
|
+
const modelOption = seedCatalogs
|
|
708
|
+
.flatMap((catalog) => catalog)
|
|
709
|
+
.find((option) => option.type === "select" && option.id === "model");
|
|
710
|
+
if (!modelOption) {
|
|
711
|
+
warnings.push(`skipped ordered thought-level domain enumeration for ${backendId}: no advertised model select option; ` +
|
|
712
|
+
"using exact advertised-value validation");
|
|
713
|
+
return;
|
|
714
|
+
}
|
|
715
|
+
const models = [...new Set(selectChoiceValues(modelOption))];
|
|
716
|
+
if (models.length > ORDERED_THOUGHT_LEVEL_ENUMERATION_MODEL_LIMIT) {
|
|
717
|
+
warnings.push(`skipped ordered thought-level domain enumeration for ${backendId}: advertised ${models.length} models, ` +
|
|
718
|
+
`above the limit of ${ORDERED_THOUGHT_LEVEL_ENUMERATION_MODEL_LIMIT}; using exact advertised-value validation`);
|
|
719
|
+
return;
|
|
720
|
+
}
|
|
721
|
+
if (models.length === 0) {
|
|
722
|
+
warnings.push(`skipped ordered thought-level domain enumeration for ${backendId}: the advertised model select has no values; ` +
|
|
723
|
+
"using exact advertised-value validation");
|
|
724
|
+
return;
|
|
725
|
+
}
|
|
726
|
+
const orderedSubsets = new Map();
|
|
727
|
+
const sentinels = new Map();
|
|
728
|
+
for (const model of models) {
|
|
729
|
+
const modelSpec = `${backendId}/${model}`;
|
|
730
|
+
const catalog = await probeTarget({
|
|
731
|
+
backendId,
|
|
732
|
+
model: modelSpec,
|
|
733
|
+
thoughtLevelDomainSemantics: "ordered",
|
|
734
|
+
}, false);
|
|
735
|
+
if (!catalog) {
|
|
736
|
+
warnings.push(`could not complete ordered thought-level domain enumeration for ${backendId}: ` +
|
|
737
|
+
`model ${JSON.stringify(model)} could not be probed; using exact advertised-value validation`);
|
|
738
|
+
return;
|
|
739
|
+
}
|
|
740
|
+
for (const option of catalog) {
|
|
741
|
+
if (!isThoughtLevelSelect(option))
|
|
742
|
+
continue;
|
|
743
|
+
const values = selectChoiceValues(option);
|
|
744
|
+
const ordered = values.filter((value) => !NON_ORDERED_THOUGHT_LEVEL_SENTINELS.has(value));
|
|
745
|
+
const optionSentinels = values.filter((value) => NON_ORDERED_THOUGHT_LEVEL_SENTINELS.has(value));
|
|
746
|
+
const subsets = orderedSubsets.get(option.id) ?? [];
|
|
747
|
+
subsets.push(ordered);
|
|
748
|
+
orderedSubsets.set(option.id, subsets);
|
|
749
|
+
const knownSentinels = sentinels.get(option.id) ?? [];
|
|
750
|
+
for (const sentinel of optionSentinels) {
|
|
751
|
+
if (!knownSentinels.includes(sentinel))
|
|
752
|
+
knownSentinels.push(sentinel);
|
|
753
|
+
}
|
|
754
|
+
sentinels.set(option.id, knownSentinels);
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
const recognizedByOption = new Map();
|
|
758
|
+
for (const [optionId, subsets] of orderedSubsets) {
|
|
759
|
+
const ordered = mergeOrderedSubsets(subsets);
|
|
760
|
+
if (!ordered) {
|
|
761
|
+
warnings.push(`could not merge advertised thought-level orders for ${backendId} option ${JSON.stringify(optionId)}; ` +
|
|
762
|
+
"using exact advertised-value validation");
|
|
763
|
+
return;
|
|
764
|
+
}
|
|
765
|
+
const recognized = [...ordered, ...(sentinels.get(optionId) ?? [])];
|
|
766
|
+
if (recognized.length > 0)
|
|
767
|
+
recognizedByOption.set(optionId, recognized);
|
|
768
|
+
}
|
|
769
|
+
attachClientRecognizedDomains(catalogs, backendId, recognizedByOption);
|
|
770
|
+
}
|
|
771
|
+
function mergeOrderedSubsets(subsets) {
|
|
772
|
+
const firstSeen = new Map();
|
|
773
|
+
const outgoing = new Map();
|
|
774
|
+
const indegree = new Map();
|
|
775
|
+
for (const subset of subsets) {
|
|
776
|
+
const unique = [...new Set(subset)];
|
|
777
|
+
for (const value of unique) {
|
|
778
|
+
if (!firstSeen.has(value))
|
|
779
|
+
firstSeen.set(value, firstSeen.size);
|
|
780
|
+
if (!outgoing.has(value))
|
|
781
|
+
outgoing.set(value, new Set());
|
|
782
|
+
if (!indegree.has(value))
|
|
783
|
+
indegree.set(value, 0);
|
|
784
|
+
}
|
|
785
|
+
for (let index = 1; index < unique.length; index++) {
|
|
786
|
+
const before = unique[index - 1];
|
|
787
|
+
const after = unique[index];
|
|
788
|
+
if (before === undefined || after === undefined)
|
|
789
|
+
continue;
|
|
790
|
+
const edges = outgoing.get(before);
|
|
791
|
+
if (edges.has(after))
|
|
792
|
+
continue;
|
|
793
|
+
edges.add(after);
|
|
794
|
+
indegree.set(after, (indegree.get(after) ?? 0) + 1);
|
|
795
|
+
}
|
|
796
|
+
}
|
|
797
|
+
const ready = [...indegree]
|
|
798
|
+
.filter(([, count]) => count === 0)
|
|
799
|
+
.map(([value]) => value);
|
|
800
|
+
const compare = (left, right) => (firstSeen.get(left) ?? 0) - (firstSeen.get(right) ?? 0) || left.localeCompare(right);
|
|
801
|
+
ready.sort(compare);
|
|
802
|
+
const merged = [];
|
|
803
|
+
while (ready.length > 0) {
|
|
804
|
+
const value = ready.shift();
|
|
805
|
+
merged.push(value);
|
|
806
|
+
for (const after of outgoing.get(value) ?? []) {
|
|
807
|
+
const remaining = (indegree.get(after) ?? 0) - 1;
|
|
808
|
+
indegree.set(after, remaining);
|
|
809
|
+
if (remaining === 0) {
|
|
810
|
+
ready.push(after);
|
|
811
|
+
ready.sort(compare);
|
|
812
|
+
}
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
return merged.length === indegree.size ? merged : undefined;
|
|
816
|
+
}
|
|
817
|
+
function attachClientRecognizedDomains(catalogs, backendId, recognizedByOption) {
|
|
818
|
+
for (const [key, catalog] of catalogs) {
|
|
819
|
+
const [catalogBackend] = JSON.parse(key);
|
|
820
|
+
if (catalogBackend !== backendId)
|
|
821
|
+
continue;
|
|
822
|
+
for (const option of catalog) {
|
|
823
|
+
if (!isThoughtLevelSelect(option))
|
|
824
|
+
continue;
|
|
825
|
+
const supported = selectChoiceValues(option);
|
|
826
|
+
if (recognizedSelectValues(option, supported))
|
|
827
|
+
continue;
|
|
828
|
+
const recognized = recognizedByOption.get(option.id);
|
|
829
|
+
if (!recognized || !supported.every((value) => recognized.includes(value)))
|
|
830
|
+
continue;
|
|
831
|
+
const meta = option._meta && typeof option._meta === "object" && !Array.isArray(option._meta)
|
|
832
|
+
? option._meta
|
|
833
|
+
: {};
|
|
834
|
+
const existingNamespace = meta[CONFIG_OPTION_META_NAMESPACE];
|
|
835
|
+
const namespace = existingNamespace &&
|
|
836
|
+
typeof existingNamespace === "object" &&
|
|
837
|
+
!Array.isArray(existingNamespace)
|
|
838
|
+
? existingNamespace
|
|
839
|
+
: {};
|
|
840
|
+
option._meta = {
|
|
841
|
+
...meta,
|
|
842
|
+
[CONFIG_OPTION_META_NAMESPACE]: {
|
|
843
|
+
...namespace,
|
|
844
|
+
recognizedValues: [...recognized],
|
|
845
|
+
},
|
|
846
|
+
};
|
|
847
|
+
}
|
|
848
|
+
}
|
|
849
|
+
}
|
|
620
850
|
function errorMessage(error) {
|
|
621
851
|
return redactText(error instanceof Error ? error.message : String(error)).value;
|
|
622
852
|
}
|
|
623
|
-
function configOptionErrors(calls, catalogs, registry, hostRegistry, declared) {
|
|
853
|
+
function configOptionErrors(calls, catalogs, registry, hostRegistry, declared, warnings) {
|
|
624
854
|
const errors = [];
|
|
625
855
|
for (const call of calls) {
|
|
626
856
|
if (!call.configOptions || Object.keys(call.configOptions).length === 0)
|
|
627
857
|
continue;
|
|
628
858
|
const backendId = routeBackend(call.model, call.tier, registry, hostRegistry, declared).backendId;
|
|
629
|
-
const advertised = catalogs.get(backendId);
|
|
859
|
+
const advertised = catalogs.get(configCatalogKey(backendId, call.model));
|
|
630
860
|
if (!advertised)
|
|
631
861
|
continue;
|
|
632
862
|
const optionIds = advertised.map((option) => option.id);
|
|
@@ -645,7 +875,37 @@ function configOptionErrors(calls, catalogs, registry, hostRegistry, declared) {
|
|
|
645
875
|
}
|
|
646
876
|
if (option.type === "select") {
|
|
647
877
|
const choices = selectChoiceValues(option);
|
|
648
|
-
if (typeof value !== "string"
|
|
878
|
+
if (typeof value !== "string") {
|
|
879
|
+
errors.push(`agent "${call.label}" configOptions option ${JSON.stringify(id)} authored value ${authored} is not an advertised select value; ` +
|
|
880
|
+
`advertised alternatives: ${displayAlternatives(choices)}`);
|
|
881
|
+
continue;
|
|
882
|
+
}
|
|
883
|
+
if (choices.includes(value))
|
|
884
|
+
continue;
|
|
885
|
+
const recognized = recognizedSelectValues(option, choices);
|
|
886
|
+
if (recognized) {
|
|
887
|
+
if (!recognized.includes(value)) {
|
|
888
|
+
errors.push(`agent "${call.label}" configOptions option ${JSON.stringify(id)} authored value ${authored} is not recognized for ` +
|
|
889
|
+
`${callModelName(backendId, call.model)}; valid values: ${displayAlternatives(recognized)}`);
|
|
890
|
+
continue;
|
|
891
|
+
}
|
|
892
|
+
const effective = clampSelectValue(value, choices, recognized);
|
|
893
|
+
if (effective !== undefined) {
|
|
894
|
+
warnings.push(`agent "${call.label}" configOptions option ${JSON.stringify(id)} value ${authored} is unsupported by ` +
|
|
895
|
+
`${callModelName(backendId, call.model)} and will clamp to ${JSON.stringify(effective)}; ` +
|
|
896
|
+
`supported values: ${displayAlternatives(choices)}`);
|
|
897
|
+
continue;
|
|
898
|
+
}
|
|
899
|
+
errors.push(`agent "${call.label}" configOptions option ${JSON.stringify(id)} authored value ${authored} cannot clamp for ` +
|
|
900
|
+
`${callModelName(backendId, call.model)} because it advertises no supported values`);
|
|
901
|
+
continue;
|
|
902
|
+
}
|
|
903
|
+
if (isThoughtLevelSelect(option)) {
|
|
904
|
+
errors.push(`agent "${call.label}" configOptions option ${JSON.stringify(id)} authored value ${authored} is not advertised by ` +
|
|
905
|
+
`${callModelName(backendId, call.model)}; advertised alternatives: ${displayAlternatives(choices)}; ` +
|
|
906
|
+
"no recognized ordered domain is available, so the value must match exactly");
|
|
907
|
+
}
|
|
908
|
+
else {
|
|
649
909
|
errors.push(`agent "${call.label}" configOptions option ${JSON.stringify(id)} authored value ${authored} is not an advertised select value; ` +
|
|
650
910
|
`advertised alternatives: ${displayAlternatives(choices)}`);
|
|
651
911
|
}
|
|
@@ -659,6 +919,44 @@ function configOptionErrors(calls, catalogs, registry, hostRegistry, declared) {
|
|
|
659
919
|
}
|
|
660
920
|
return errors;
|
|
661
921
|
}
|
|
922
|
+
function callModelName(backendId, model) {
|
|
923
|
+
return model === undefined
|
|
924
|
+
? `${backendId} session default model`
|
|
925
|
+
: `${backendId} model ${JSON.stringify(model)}`;
|
|
926
|
+
}
|
|
927
|
+
function recognizedSelectValues(option, supported) {
|
|
928
|
+
const namespace = option._meta?.[CONFIG_OPTION_META_NAMESPACE];
|
|
929
|
+
if (namespace === null || typeof namespace !== "object" || Array.isArray(namespace))
|
|
930
|
+
return undefined;
|
|
931
|
+
const values = namespace.recognizedValues;
|
|
932
|
+
if (!Array.isArray(values) || values.length === 0 || !values.every((value) => typeof value === "string")) {
|
|
933
|
+
return undefined;
|
|
934
|
+
}
|
|
935
|
+
const recognized = values;
|
|
936
|
+
if (new Set(recognized).size !== recognized.length || !supported.every((value) => recognized.includes(value))) {
|
|
937
|
+
return undefined;
|
|
938
|
+
}
|
|
939
|
+
return recognized;
|
|
940
|
+
}
|
|
941
|
+
function clampSelectValue(requested, supported, recognized) {
|
|
942
|
+
if (supported.includes(requested))
|
|
943
|
+
return requested;
|
|
944
|
+
const orderedRecognized = recognized.filter((value) => !NON_ORDERED_THOUGHT_LEVEL_SENTINELS.has(value));
|
|
945
|
+
const requestedIndex = orderedRecognized.indexOf(requested);
|
|
946
|
+
if (requestedIndex < 0)
|
|
947
|
+
return undefined;
|
|
948
|
+
for (let index = requestedIndex; index < orderedRecognized.length; index++) {
|
|
949
|
+
const candidate = orderedRecognized[index];
|
|
950
|
+
if (candidate !== undefined && supported.includes(candidate))
|
|
951
|
+
return candidate;
|
|
952
|
+
}
|
|
953
|
+
for (let index = requestedIndex - 1; index >= 0; index--) {
|
|
954
|
+
const candidate = orderedRecognized[index];
|
|
955
|
+
if (candidate !== undefined && supported.includes(candidate))
|
|
956
|
+
return candidate;
|
|
957
|
+
}
|
|
958
|
+
return undefined;
|
|
959
|
+
}
|
|
662
960
|
function selectChoiceValues(option) {
|
|
663
961
|
return option.options.flatMap((entry) => ("options" in entry ? entry.options : [entry])).map((entry) => entry.value);
|
|
664
962
|
}
|
|
@@ -670,7 +968,7 @@ function displayAlternatives(values) {
|
|
|
670
968
|
}
|
|
671
969
|
/**
|
|
672
970
|
* Validate a workflow script: parse it, dry-run against a mock AgentRunner, then probe
|
|
673
|
-
* each routed
|
|
971
|
+
* each routed backend/model pair's advertised config options. Never throws for an invalid script —
|
|
674
972
|
* read `report.ok` / `report.exitCode`.
|
|
675
973
|
*/
|
|
676
974
|
export async function validateWorkflowScript(script, options = {}) {
|
|
@@ -845,7 +1143,7 @@ export async function validateWorkflowScript(script, options = {}) {
|
|
|
845
1143
|
if (mockAnswerState && mockAnswers)
|
|
846
1144
|
appendMockAnswerWarnings(mockAnswerState, mockAnswers, warnings);
|
|
847
1145
|
const probed = await probeHarnessConfigOptions(agentCalls, baseCwd, backendRegistry, hostRegistry, declaredBackends, warnings);
|
|
848
|
-
const optionErrors = configOptionErrors(agentCalls, probed.catalogs, backendRegistry, hostRegistry, declaredBackends);
|
|
1146
|
+
const optionErrors = configOptionErrors(agentCalls, probed.catalogs, backendRegistry, hostRegistry, declaredBackends, warnings);
|
|
849
1147
|
const ok = runOk && optionErrors.length === 0;
|
|
850
1148
|
const runReason = timedOut ? `dry run exceeded ${timeoutMs}ms and was aborted` : run.reason;
|
|
851
1149
|
const reason = optionErrors.length === 0
|
|
@@ -901,11 +1199,14 @@ function truncate(text, max) {
|
|
|
901
1199
|
export function renderHarnessOptionLines(harnesses, indent) {
|
|
902
1200
|
const lines = [];
|
|
903
1201
|
for (const harness of harnesses) {
|
|
1202
|
+
const target = harness.model === undefined
|
|
1203
|
+
? harness.backendId
|
|
1204
|
+
: `${harness.backendId} (model ${JSON.stringify(harness.model)})`;
|
|
904
1205
|
if (!harness.probed) {
|
|
905
|
-
lines.push(`${indent}${
|
|
1206
|
+
lines.push(`${indent}${target}: probe failed — ${harness.error ?? "unknown error"}`);
|
|
906
1207
|
continue;
|
|
907
1208
|
}
|
|
908
|
-
lines.push(`${indent}${
|
|
1209
|
+
lines.push(`${indent}${target}:`);
|
|
909
1210
|
lines.push(`${indent} id | type | current | choices`);
|
|
910
1211
|
if ((harness.options ?? []).length === 0) {
|
|
911
1212
|
lines.push(`${indent} (none advertised)`);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@automatalabs/workflows",
|
|
3
|
-
"version": "0.40.
|
|
3
|
+
"version": "0.40.5",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
5
|
"engines": {
|
|
6
6
|
"node": ">=22"
|
|
@@ -32,8 +32,8 @@
|
|
|
32
32
|
"dependencies": {
|
|
33
33
|
"typebox": "1.3.2",
|
|
34
34
|
"@automatalabs/shared-types": "0.26.2",
|
|
35
|
-
"@automatalabs/
|
|
36
|
-
"@automatalabs/
|
|
35
|
+
"@automatalabs/acp-agents": "0.34.4",
|
|
36
|
+
"@automatalabs/workflow-engine": "0.29.2"
|
|
37
37
|
},
|
|
38
38
|
"scripts": {
|
|
39
39
|
"build": "tsc -b",
|