@uns-kit/assistant-workflow 2.0.77 → 2.0.79
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -0
- package/dist/tool-selection-evaluation.d.ts +13 -3
- package/dist/tool-selection-evaluation.d.ts.map +1 -1
- package/dist/tool-selection-evaluation.js +128 -27
- package/dist/tool-selection-evaluation.js.map +1 -1
- package/dist/tool-selection-json.js +1 -0
- package/dist/tool-selection-json.js.map +1 -1
- package/dist/tool-selection-policy.d.ts +3 -1
- package/dist/tool-selection-policy.d.ts.map +1 -1
- package/dist/tool-selection-policy.js +10 -0
- package/dist/tool-selection-policy.js.map +1 -1
- package/dist/tool-selection-trace-export.js +3 -0
- package/dist/tool-selection-trace-export.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -30,6 +30,24 @@ The package deliberately does not contain provider credentials, database access,
|
|
|
30
30
|
HTTP transport, UNS-specific tool implementations, or application-specific
|
|
31
31
|
workflow definitions.
|
|
32
32
|
|
|
33
|
+
## Tool-Selection Evaluation
|
|
34
|
+
|
|
35
|
+
Tool-selection replay reports distinguish a workflow failure from an expected
|
|
36
|
+
clarification boundary. A `workflow_blocked` decision with
|
|
37
|
+
`workflowStatus: "needs-clarification"` is emitted as the informational
|
|
38
|
+
`workflow_expected_clarification` signal. It remains visible in readiness and
|
|
39
|
+
migration artifacts, but it neither increments warning counts nor asks to
|
|
40
|
+
broaden runtime authority. Other blocked workflow states remain warnings and
|
|
41
|
+
continue to block a migration proposal.
|
|
42
|
+
|
|
43
|
+
When an application deliberately exposes its full legacy tool catalog
|
|
44
|
+
(`pruningEnabled: false`), replay reports record
|
|
45
|
+
`workflow_selection_not_exercised` rather than interpreting the larger catalog
|
|
46
|
+
as a workflow mismatch. These rows remain visible as
|
|
47
|
+
`selection_not_exercised_only` migration candidates and require a pruned shadow
|
|
48
|
+
selection before authority can be broadened; they never hide a genuine mismatch
|
|
49
|
+
observed while selection was exercised.
|
|
50
|
+
|
|
33
51
|
## Exact Tool Handoffs
|
|
34
52
|
|
|
35
53
|
When a host has rebuilt a workflow plan and wants to delegate exactly one
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { AssistantWorkflowJsonValue } from "./run-report-json.js";
|
|
2
2
|
import type { AssistantWorkflowSerializedToolSelectionDecision } from "./tool-selection-json.js";
|
|
3
3
|
export type AssistantWorkflowToolSelectionEvaluationSeverity = "info" | "warning";
|
|
4
|
-
export type AssistantWorkflowToolSelectionEvaluationSignalName = "legacy_authority" | "workflow_unavailable" | "workflow_authority_not_enabled" | "workflow_blocked" | "workflow_differs" | "workflow_candidate_not_selected" | "selected_outside_workflow_candidate" | "selected_outside_workflow_suggestions" | "selected_policy_excluded_optional_tool";
|
|
4
|
+
export type AssistantWorkflowToolSelectionEvaluationSignalName = "legacy_authority" | "workflow_unavailable" | "workflow_authority_not_enabled" | "workflow_blocked" | "workflow_expected_clarification" | "workflow_selection_not_exercised" | "workflow_differs" | "workflow_candidate_not_selected" | "selected_outside_workflow_candidate" | "selected_outside_workflow_suggestions" | "selected_policy_excluded_optional_tool";
|
|
5
5
|
export type AssistantWorkflowToolSelectionEvaluationSignal = {
|
|
6
6
|
name: AssistantWorkflowToolSelectionEvaluationSignalName;
|
|
7
7
|
severity: AssistantWorkflowToolSelectionEvaluationSeverity;
|
|
@@ -25,6 +25,8 @@ export type AssistantWorkflowToolSelectionEvaluation = {
|
|
|
25
25
|
workflowSelectionActiveProfileIds: string[];
|
|
26
26
|
workflowSelectionProfileToolNames: string[];
|
|
27
27
|
workflowSelectionProfileToolCount: number;
|
|
28
|
+
expectedClarificationBlock: boolean;
|
|
29
|
+
selectionNotExercised: boolean;
|
|
28
30
|
signalCount: number;
|
|
29
31
|
warningCount: number;
|
|
30
32
|
signals: AssistantWorkflowToolSelectionEvaluationSignal[];
|
|
@@ -48,6 +50,8 @@ export type AssistantWorkflowToolSelectionEvaluationIntentReadiness = {
|
|
|
48
50
|
shadowEquivalentCount: number;
|
|
49
51
|
workflowDiffersCount: number;
|
|
50
52
|
workflowBlockedCount: number;
|
|
53
|
+
expectedClarificationBlockCount: number;
|
|
54
|
+
selectionNotExercisedCount: number;
|
|
51
55
|
workflowUnavailableCount: number;
|
|
52
56
|
warningCount: number;
|
|
53
57
|
workflowAuthorityConfirmed: boolean;
|
|
@@ -67,8 +71,8 @@ export type AssistantWorkflowToolSelectionEvaluationReadinessSummary = {
|
|
|
67
71
|
warningSegmentCount: number;
|
|
68
72
|
};
|
|
69
73
|
export type AssistantWorkflowToolSelectionMigrationCandidateKind = "intent" | "segment";
|
|
70
|
-
export type AssistantWorkflowToolSelectionMigrationCandidateStatus = "ready_for_workflow_authority" | "workflow_authority_confirmed" | "needs_review";
|
|
71
|
-
export type AssistantWorkflowToolSelectionMigrationRecommendedAction = "add_intent_to_authority_allow_list" | "keep_runtime_authority" | "review_segment_before_authority_change" | "review_blockers_before_migration";
|
|
74
|
+
export type AssistantWorkflowToolSelectionMigrationCandidateStatus = "ready_for_workflow_authority" | "workflow_authority_confirmed" | "expected_clarification_only" | "selection_not_exercised_only" | "needs_review";
|
|
75
|
+
export type AssistantWorkflowToolSelectionMigrationRecommendedAction = "add_intent_to_authority_allow_list" | "keep_runtime_authority" | "keep_controller_clarification" | "exercise_pruned_selection" | "review_segment_before_authority_change" | "review_blockers_before_migration";
|
|
72
76
|
export type AssistantWorkflowToolSelectionMigrationCandidate = {
|
|
73
77
|
kind: AssistantWorkflowToolSelectionMigrationCandidateKind;
|
|
74
78
|
key: string;
|
|
@@ -81,6 +85,8 @@ export type AssistantWorkflowToolSelectionMigrationCandidate = {
|
|
|
81
85
|
shadowEquivalentCount: number;
|
|
82
86
|
workflowDiffersCount: number;
|
|
83
87
|
workflowBlockedCount: number;
|
|
88
|
+
expectedClarificationBlockCount: number;
|
|
89
|
+
selectionNotExercisedCount: number;
|
|
84
90
|
workflowUnavailableCount: number;
|
|
85
91
|
warningCount: number;
|
|
86
92
|
status: AssistantWorkflowToolSelectionMigrationCandidateStatus;
|
|
@@ -103,6 +109,8 @@ export type AssistantWorkflowToolSelectionMigrationReport = {
|
|
|
103
109
|
readySegmentCandidates: AssistantWorkflowToolSelectionMigrationCandidate[];
|
|
104
110
|
confirmedIntentCandidates: AssistantWorkflowToolSelectionMigrationCandidate[];
|
|
105
111
|
confirmedSegmentCandidates: AssistantWorkflowToolSelectionMigrationCandidate[];
|
|
112
|
+
expectedClarificationCandidates: AssistantWorkflowToolSelectionMigrationCandidate[];
|
|
113
|
+
selectionNotExercisedCandidates: AssistantWorkflowToolSelectionMigrationCandidate[];
|
|
106
114
|
reviewCandidates: AssistantWorkflowToolSelectionMigrationCandidate[];
|
|
107
115
|
};
|
|
108
116
|
export type AssistantWorkflowToolSelectionAuthorityAllowListProposalOptions = {
|
|
@@ -167,6 +175,8 @@ export type AssistantWorkflowToolSelectionMigrationReviewArtifact = {
|
|
|
167
175
|
readySegmentCandidateKeys: string[];
|
|
168
176
|
confirmedIntentCandidateKeys: string[];
|
|
169
177
|
confirmedSegmentCandidateKeys: string[];
|
|
178
|
+
expectedClarificationCandidateKeys: string[];
|
|
179
|
+
selectionNotExercisedCandidateKeys: string[];
|
|
170
180
|
};
|
|
171
181
|
review: {
|
|
172
182
|
reviewIntentIds: string[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tool-selection-evaluation.d.ts","sourceRoot":"","sources":["../src/tool-selection-evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,KAAK,EAAE,gDAAgD,EAAE,MAAM,0BAA0B,CAAC;AAQjG,MAAM,MAAM,gDAAgD,GAAG,MAAM,GAAG,SAAS,CAAC;AAElF,MAAM,MAAM,kDAAkD,GAC1D,kBAAkB,GAClB,sBAAsB,GACtB,gCAAgC,GAChC,kBAAkB,GAClB,kBAAkB,GAClB,iCAAiC,GACjC,qCAAqC,GACrC,uCAAuC,GACvC,wCAAwC,CAAC;AAE7C,MAAM,MAAM,8CAA8C,GAAG;IAC3D,IAAI,EAAE,kDAAkD,CAAC;IACzD,QAAQ,EAAE,gDAAgD,CAAC;IAC3D,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,wCAAwC,GAAG;IACrD,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,wBAAwB,EAAE,OAAO,GAAG,IAAI,CAAC;IACzC,2BAA2B,EAAE,MAAM,CAAC;IACpC,+BAA+B,EAAE,OAAO,GAAG,IAAI,CAAC;IAChD,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,0BAA0B,EAAE,MAAM,CAAC;IACnC,mCAAmC,EAAE,MAAM,CAAC;IAC5C,iCAAiC,EAAE,MAAM,EAAE,CAAC;IAC5C,iCAAiC,EAAE,MAAM,EAAE,CAAC;IAC5C,iCAAiC,EAAE,MAAM,CAAC;IAC1C,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,8CAA8C,EAAE,CAAC;CAC3D,CAAC;AAEF,MAAM,MAAM,oDAAoD,GAAG;IACjE,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,kDAAkD,GAAG;IAC/D,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,wDAAwD,GAAG;IACrE,IAAI,EAAE,kDAAkD,CAAC;IACzD,QAAQ,EAAE,gDAAgD,CAAC;IAC3D,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,uDAAuD,GAAG;IACpE,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,wBAAwB,EAAE,MAAM,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B,EAAE,OAAO,CAAC;IACpC,yBAAyB,EAAE,OAAO,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,wDAAwD,GAClE,uDAAuD,GAAG;IACxD,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B,CAAC;AAEJ,MAAM,MAAM,wDAAwD,GAAG;IACrE,gBAAgB,EAAE,MAAM,CAAC;IACzB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mBAAmB,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,oDAAoD,GAAG,QAAQ,GAAG,SAAS,CAAC;AAExF,MAAM,MAAM,sDAAsD,GAC9D,8BAA8B,GAC9B,8BAA8B,GAC9B,cAAc,CAAC;AAEnB,MAAM,MAAM,wDAAwD,GAChE,oCAAoC,GACpC,wBAAwB,GACxB,wCAAwC,GACxC,kCAAkC,CAAC;AAEvC,MAAM,MAAM,gDAAgD,GAAG;IAC7D,IAAI,EAAE,oDAAoD,CAAC;IAC3D,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,wBAAwB,EAAE,MAAM,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,sDAAsD,CAAC;IAC/D,iBAAiB,EAAE,wDAAwD,CAAC;IAC5E,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oDAAoD,GAAG;IACjE,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,6CAA6C,GAAG;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,2BAA2B,EAAE,MAAM,EAAE,CAAC;IACtC,6BAA6B,EAAE,MAAM,EAAE,CAAC;IACxC,cAAc,EAAE,MAAM,CAAC;IACvB,qBAAqB,EAAE,gDAAgD,EAAE,CAAC;IAC1E,sBAAsB,EAAE,gDAAgD,EAAE,CAAC;IAC3E,yBAAyB,EAAE,gDAAgD,EAAE,CAAC;IAC9E,0BAA0B,EAAE,gDAAgD,EAAE,CAAC;IAC/E,gBAAgB,EAAE,gDAAgD,EAAE,CAAC;CACtE,CAAC;AAEF,MAAM,MAAM,+DAA+D,GAAG;IAC5E,yBAAyB,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAC;IACrD,2BAA2B,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAC;CACxD,CAAC;AAEF,MAAM,MAAM,wDAAwD,GAAG;IACrE,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,yBAAyB,EAAE,MAAM,EAAE,CAAC;IACpC,2BAA2B,EAAE,MAAM,EAAE,CAAC;IACtC,2BAA2B,EAAE,MAAM,EAAE,CAAC;IACtC,6BAA6B,EAAE,MAAM,EAAE,CAAC;IACxC,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,0BAA0B,EAAE,MAAM,EAAE,CAAC;IACrC,4BAA4B,EAAE,MAAM,EAAE,CAAC;IACvC,yBAAyB,EAAE,MAAM,CAAC;IAClC,0BAA0B,EAAE,MAAM,CAAC;IACnC,6BAA6B,EAAE,MAAM,CAAC;IACtC,8BAA8B,EAAE,MAAM,CAAC;IACvC,0BAA0B,EAAE,MAAM,CAAC;IACnC,2BAA2B,EAAE,MAAM,CAAC;IACpC,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,kCAAkC,EAAE,MAAM,EAAE,CAAC;IAC7C,oCAAoC,EAAE,MAAM,EAAE,CAAC;IAC/C,iBAAiB,EAAE,OAAO,CAAC;IAC3B,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,2DAA2D,GACnE,0BAA0B,GAC1B,mBAAmB,CAAC;AAExB,MAAM,MAAM,sEAAsE,GAC9E,mCAAmC,GACnC,wBAAwB,CAAC;AAE7B,MAAM,MAAM,4DAA4D,GAAG;IACzE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAC;IACxC,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAC;CAC5C,CAAC;AAEF,MAAM,MAAM,qDAAqD,GAAG;IAClE,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,2DAA2D,CAAC;IACpE,iBAAiB,EAAE,sEAAsE,CAAC;IAC1F,gBAAgB,EAAE,MAAM,CAAC;IACzB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE;QACb,YAAY,EAAE,MAAM,EAAE,CAAC;QACvB,cAAc,EAAE,MAAM,EAAE,CAAC;QACzB,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,eAAe,EAAE,MAAM,EAAE,CAAC;QAC1B,0BAA0B,EAAE,MAAM,EAAE,CAAC;QACrC,4BAA4B,EAAE,MAAM,EAAE,CAAC;KACxC,CAAC;IACF,QAAQ,EAAE;QACR,wBAAwB,EAAE,MAAM,EAAE,CAAC;QACnC,yBAAyB,EAAE,MAAM,EAAE,CAAC;QACpC,4BAA4B,EAAE,MAAM,EAAE,CAAC;QACvC,6BAA6B,EAAE,MAAM,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"tool-selection-evaluation.d.ts","sourceRoot":"","sources":["../src/tool-selection-evaluation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AACvE,OAAO,KAAK,EAAE,gDAAgD,EAAE,MAAM,0BAA0B,CAAC;AAQjG,MAAM,MAAM,gDAAgD,GAAG,MAAM,GAAG,SAAS,CAAC;AAElF,MAAM,MAAM,kDAAkD,GAC1D,kBAAkB,GAClB,sBAAsB,GACtB,gCAAgC,GAChC,kBAAkB,GAClB,iCAAiC,GACjC,kCAAkC,GAClC,kBAAkB,GAClB,iCAAiC,GACjC,qCAAqC,GACrC,uCAAuC,GACvC,wCAAwC,CAAC;AAE7C,MAAM,MAAM,8CAA8C,GAAG;IAC3D,IAAI,EAAE,kDAAkD,CAAC;IACzD,QAAQ,EAAE,gDAAgD,CAAC;IAC3D,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;CACvB,CAAC;AAEF,MAAM,MAAM,wCAAwC,GAAG;IACrD,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,wBAAwB,EAAE,OAAO,GAAG,IAAI,CAAC;IACzC,2BAA2B,EAAE,MAAM,CAAC;IACpC,+BAA+B,EAAE,OAAO,GAAG,IAAI,CAAC;IAChD,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,0BAA0B,EAAE,MAAM,CAAC;IACnC,mCAAmC,EAAE,MAAM,CAAC;IAC5C,iCAAiC,EAAE,MAAM,EAAE,CAAC;IAC5C,iCAAiC,EAAE,MAAM,EAAE,CAAC;IAC5C,iCAAiC,EAAE,MAAM,CAAC;IAC1C,0BAA0B,EAAE,OAAO,CAAC;IACpC,qBAAqB,EAAE,OAAO,CAAC;IAC/B,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,8CAA8C,EAAE,CAAC;CAC3D,CAAC;AAEF,MAAM,MAAM,oDAAoD,GAAG;IACjE,eAAe,CAAC,EAAE,OAAO,CAAC;CAC3B,CAAC;AAEF,MAAM,MAAM,kDAAkD,GAAG;IAC/D,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,wDAAwD,GAAG;IACrE,IAAI,EAAE,kDAAkD,CAAC;IACzD,QAAQ,EAAE,gDAAgD,CAAC;IAC3D,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,MAAM,MAAM,uDAAuD,GAAG;IACpE,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,+BAA+B,EAAE,MAAM,CAAC;IACxC,0BAA0B,EAAE,MAAM,CAAC;IACnC,wBAAwB,EAAE,MAAM,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,0BAA0B,EAAE,OAAO,CAAC;IACpC,yBAAyB,EAAE,OAAO,CAAC;CACpC,CAAC;AAEF,MAAM,MAAM,wDAAwD,GAClE,uDAAuD,GAAG;IACxD,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B,CAAC;AAEJ,MAAM,MAAM,wDAAwD,GAAG;IACrE,gBAAgB,EAAE,MAAM,CAAC;IACzB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,mBAAmB,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,oDAAoD,GAAG,QAAQ,GAAG,SAAS,CAAC;AAExF,MAAM,MAAM,sDAAsD,GAC9D,8BAA8B,GAC9B,8BAA8B,GAC9B,6BAA6B,GAC7B,8BAA8B,GAC9B,cAAc,CAAC;AAEnB,MAAM,MAAM,wDAAwD,GAChE,oCAAoC,GACpC,wBAAwB,GACxB,+BAA+B,GAC/B,2BAA2B,GAC3B,wCAAwC,GACxC,kCAAkC,CAAC;AAEvC,MAAM,MAAM,gDAAgD,GAAG;IAC7D,IAAI,EAAE,oDAAoD,CAAC;IAC3D,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IAC1B,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;IACnB,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,aAAa,EAAE,MAAM,CAAC;IACtB,sBAAsB,EAAE,MAAM,CAAC;IAC/B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,+BAA+B,EAAE,MAAM,CAAC;IACxC,0BAA0B,EAAE,MAAM,CAAC;IACnC,wBAAwB,EAAE,MAAM,CAAC;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,sDAAsD,CAAC;IAC/D,iBAAiB,EAAE,wDAAwD,CAAC;IAC5E,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,SAAS,EAAE,MAAM,CAAC;CACnB,CAAC;AAEF,MAAM,MAAM,oDAAoD,GAAG;IACjE,gBAAgB,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,6CAA6C,GAAG;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,2BAA2B,EAAE,MAAM,EAAE,CAAC;IACtC,6BAA6B,EAAE,MAAM,EAAE,CAAC;IACxC,cAAc,EAAE,MAAM,CAAC;IACvB,qBAAqB,EAAE,gDAAgD,EAAE,CAAC;IAC1E,sBAAsB,EAAE,gDAAgD,EAAE,CAAC;IAC3E,yBAAyB,EAAE,gDAAgD,EAAE,CAAC;IAC9E,0BAA0B,EAAE,gDAAgD,EAAE,CAAC;IAC/E,+BAA+B,EAAE,gDAAgD,EAAE,CAAC;IACpF,+BAA+B,EAAE,gDAAgD,EAAE,CAAC;IACpF,gBAAgB,EAAE,gDAAgD,EAAE,CAAC;CACtE,CAAC;AAEF,MAAM,MAAM,+DAA+D,GAAG;IAC5E,yBAAyB,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAC;IACrD,2BAA2B,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAC;CACxD,CAAC;AAEF,MAAM,MAAM,wDAAwD,GAAG;IACrE,WAAW,EAAE,MAAM,CAAC;IACpB,gBAAgB,EAAE,MAAM,CAAC;IACzB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,yBAAyB,EAAE,MAAM,EAAE,CAAC;IACpC,2BAA2B,EAAE,MAAM,EAAE,CAAC;IACtC,2BAA2B,EAAE,MAAM,EAAE,CAAC;IACtC,6BAA6B,EAAE,MAAM,EAAE,CAAC;IACxC,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,cAAc,EAAE,MAAM,EAAE,CAAC;IACzB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,0BAA0B,EAAE,MAAM,EAAE,CAAC;IACrC,4BAA4B,EAAE,MAAM,EAAE,CAAC;IACvC,yBAAyB,EAAE,MAAM,CAAC;IAClC,0BAA0B,EAAE,MAAM,CAAC;IACnC,6BAA6B,EAAE,MAAM,CAAC;IACtC,8BAA8B,EAAE,MAAM,CAAC;IACvC,0BAA0B,EAAE,MAAM,CAAC;IACnC,2BAA2B,EAAE,MAAM,CAAC;IACpC,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,iBAAiB,EAAE,MAAM,EAAE,CAAC;IAC5B,kCAAkC,EAAE,MAAM,EAAE,CAAC;IAC7C,oCAAoC,EAAE,MAAM,EAAE,CAAC;IAC/C,iBAAiB,EAAE,OAAO,CAAC;IAC3B,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,2DAA2D,GACnE,0BAA0B,GAC1B,mBAAmB,CAAC;AAExB,MAAM,MAAM,sEAAsE,GAC9E,mCAAmC,GACnC,wBAAwB,CAAC;AAE7B,MAAM,MAAM,4DAA4D,GAAG;IACzE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAC;IACxC,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI,CAAC;CAC5C,CAAC;AAEF,MAAM,MAAM,qDAAqD,GAAG;IAClE,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,2DAA2D,CAAC;IACpE,iBAAiB,EAAE,sEAAsE,CAAC;IAC1F,gBAAgB,EAAE,MAAM,CAAC;IACzB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,YAAY,EAAE,MAAM,EAAE,CAAC;IACvB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,aAAa,EAAE;QACb,YAAY,EAAE,MAAM,EAAE,CAAC;QACvB,cAAc,EAAE,MAAM,EAAE,CAAC;QACzB,aAAa,EAAE,MAAM,EAAE,CAAC;QACxB,eAAe,EAAE,MAAM,EAAE,CAAC;QAC1B,0BAA0B,EAAE,MAAM,EAAE,CAAC;QACrC,4BAA4B,EAAE,MAAM,EAAE,CAAC;KACxC,CAAC;IACF,QAAQ,EAAE;QACR,wBAAwB,EAAE,MAAM,EAAE,CAAC;QACnC,yBAAyB,EAAE,MAAM,EAAE,CAAC;QACpC,4BAA4B,EAAE,MAAM,EAAE,CAAC;QACvC,6BAA6B,EAAE,MAAM,EAAE,CAAC;QACxC,kCAAkC,EAAE,MAAM,EAAE,CAAC;QAC7C,kCAAkC,EAAE,MAAM,EAAE,CAAC;KAC9C,CAAC;IACF,MAAM,EAAE;QACN,eAAe,EAAE,MAAM,EAAE,CAAC;QAC1B,iBAAiB,EAAE,MAAM,EAAE,CAAC;QAC5B,kCAAkC,EAAE,MAAM,EAAE,CAAC;QAC7C,oCAAoC,EAAE,MAAM,EAAE,CAAC;QAC/C,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,gDAAgD,EAAE,CAAC;KAC9D,CAAC;IACF,SAAS,EAAE,MAAM,EAAE,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,6CAA6C,GAAG;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,eAAe,EAAE,MAAM,CAAC;IACxB,YAAY,EAAE,kDAAkD,EAAE,CAAC;IACnE,eAAe,EAAE,uDAAuD,EAAE,CAAC;IAC3E,gBAAgB,EAAE,wDAAwD,EAAE,CAAC;IAC7E,gBAAgB,EAAE,wDAAwD,CAAC;IAC3E,qBAAqB,EAAE,kDAAkD,EAAE,CAAC;IAC5E,qBAAqB,EAAE,kDAAkD,EAAE,CAAC;IAC5E,oBAAoB,EAAE,kDAAkD,EAAE,CAAC;IAC3E,qBAAqB,EAAE,kDAAkD,EAAE,CAAC;IAC5E,8BAA8B,EAAE,kDAAkD,EAAE,CAAC;IACrF,kCAAkC,EAAE,kDAAkD,EAAE,CAAC;IACzF,YAAY,EAAE,wDAAwD,EAAE,CAAC;IACzE,IAAI,EAAE,wCAAwC,EAAE,CAAC;CAClD,CAAC;AAEF,wBAAgB,8CAA8C,CAC5D,QAAQ,EAAE,gDAAgD,GACzD,wCAAwC,CAuD1C;AAED,wBAAgB,sDAAsD,CACpE,QAAQ,EAAE,gDAAgD,GACzD,8CAA8C,EAAE,CAwGlD;AAED,wBAAgB,kDAAkD,CAChE,SAAS,EAAE,SAAS,gDAAgD,EAAE,EACtE,OAAO,GAAE,oDAAyD,GACjE,6CAA6C,CAyB/C;AAED,wBAAgB,yDAAyD,CACvE,UAAU,EAAE,wCAAwC,GACnD,MAAM,CAAC,MAAM,EAAE,0BAA0B,CAAC,CA6B5C;AAED,wBAAgB,8DAA8D,CAC5E,KAAK,EAAE,6CAA6C,GACnD,MAAM,CAAC,MAAM,EAAE,0BAA0B,CAAC,CAuD5C;AAED,wBAAgB,kDAAkD,CAChE,KAAK,EAAE,6CAA6C,EACpD,OAAO,GAAE,oDAAyD,GACjE,6CAA6C,CA8C/C;AAED,wBAAgB,8DAA8D,CAC5E,MAAM,EAAE,6CAA6C,GACpD,MAAM,CAAC,MAAM,EAAE,0BAA0B,CAAC,CAiB5C;AAED,wBAAgB,6DAA6D,CAC3E,MAAM,EAAE,6CAA6C,EACrD,OAAO,GAAE,+DAAoE,GAC5E,wDAAwD,CA0E1D;AAED,wBAAgB,yEAAyE,CACvF,QAAQ,EAAE,wDAAwD,GACjE,MAAM,CAAC,MAAM,EAAE,0BAA0B,CAAC,CA6B5C;AAED,wBAAgB,0DAA0D,CACxE,KAAK,EAAE;IACL,MAAM,EAAE,6CAA6C,CAAC;IACtD,QAAQ,EAAE,wDAAwD,CAAC;CACpE,EACD,OAAO,GAAE,4DAAiE,GACzE,qDAAqD,CA6CvD;AAED,wBAAgB,sEAAsE,CACpF,QAAQ,EAAE,qDAAqD,GAC9D,MAAM,CAAC,MAAM,EAAE,0BAA0B,CAAC,CAqC5C"}
|
|
@@ -14,12 +14,14 @@ export function evaluateAssistantWorkflowToolSelectionDecision(decision) {
|
|
|
14
14
|
hop: readNumber(decision.comparisonPayload?.["hop"]),
|
|
15
15
|
selectedReason: readString(decision.comparisonPayload?.["selectedReason"]),
|
|
16
16
|
});
|
|
17
|
+
const expectedClarificationBlock = isExpectedClarificationBlock(decision);
|
|
18
|
+
const selectionNotExercised = isWorkflowSelectionNotExercised(decision);
|
|
17
19
|
return {
|
|
18
20
|
intentId,
|
|
19
21
|
hop: readNumber(decision.comparisonPayload?.["hop"]),
|
|
20
22
|
selectedReason: readString(decision.comparisonPayload?.["selectedReason"]),
|
|
21
23
|
authoritySource: decision.authority.source,
|
|
22
|
-
authorityReason: decision
|
|
24
|
+
authorityReason: getEvaluationAuthorityReason(decision),
|
|
23
25
|
workflowStatus: decision.authority.workflowStatus,
|
|
24
26
|
workflowAuthorityEnabled: intentId && workflowAuthorityIntentIds.length > 0
|
|
25
27
|
? workflowAuthorityIntentIds.includes(intentId)
|
|
@@ -36,6 +38,8 @@ export function evaluateAssistantWorkflowToolSelectionDecision(decision) {
|
|
|
36
38
|
workflowSelectionActiveProfileIds,
|
|
37
39
|
workflowSelectionProfileToolNames: workflowSelectionProfileToolNames,
|
|
38
40
|
workflowSelectionProfileToolCount: workflowSelectionProfileToolNames.length,
|
|
41
|
+
expectedClarificationBlock,
|
|
42
|
+
selectionNotExercised,
|
|
39
43
|
signalCount: signals.length,
|
|
40
44
|
warningCount: signals.filter((signal) => signal.severity === "warning").length,
|
|
41
45
|
signals,
|
|
@@ -43,6 +47,9 @@ export function evaluateAssistantWorkflowToolSelectionDecision(decision) {
|
|
|
43
47
|
}
|
|
44
48
|
export function collectAssistantWorkflowToolSelectionEvaluationSignals(decision) {
|
|
45
49
|
const signals = [];
|
|
50
|
+
const expectedClarificationBlock = isExpectedClarificationBlock(decision);
|
|
51
|
+
const selectionNotExercised = isWorkflowSelectionNotExercised(decision);
|
|
52
|
+
const authorityReason = getEvaluationAuthorityReason(decision);
|
|
46
53
|
const missingCandidateTools = readStringArray(decision.comparisonPayload?.["missingWorkflowSelectionCandidateTools"]);
|
|
47
54
|
const selectedOutsideCandidate = readStringArray(decision.comparisonPayload?.["selectedOutsideWorkflowSelectionCandidate"]);
|
|
48
55
|
const selectedOutsideSuggestions = readStringArray(decision.comparisonPayload?.["selectedOutsideWorkflowSuggestions"]);
|
|
@@ -52,59 +59,68 @@ export function collectAssistantWorkflowToolSelectionEvaluationSignals(decision)
|
|
|
52
59
|
signals.push({
|
|
53
60
|
name: "legacy_authority",
|
|
54
61
|
severity: "info",
|
|
55
|
-
detail:
|
|
62
|
+
detail: authorityReason,
|
|
56
63
|
});
|
|
57
64
|
}
|
|
58
|
-
if (
|
|
65
|
+
if (authorityReason === "workflow_unavailable") {
|
|
59
66
|
signals.push({
|
|
60
67
|
name: "workflow_unavailable",
|
|
61
68
|
severity: "warning",
|
|
62
69
|
detail: "tool selection ran without a workflow comparison payload",
|
|
63
70
|
});
|
|
64
71
|
}
|
|
65
|
-
if (
|
|
72
|
+
if (authorityReason === "workflow_authority_not_enabled") {
|
|
66
73
|
signals.push({
|
|
67
74
|
name: "workflow_authority_not_enabled",
|
|
68
75
|
severity: "info",
|
|
69
76
|
detail: readString(decision.comparisonPayload?.["intent"]),
|
|
70
77
|
});
|
|
71
78
|
}
|
|
72
|
-
if (
|
|
79
|
+
if (authorityReason === "workflow_blocked") {
|
|
73
80
|
signals.push({
|
|
74
|
-
name:
|
|
75
|
-
|
|
81
|
+
name: expectedClarificationBlock
|
|
82
|
+
? "workflow_expected_clarification"
|
|
83
|
+
: "workflow_blocked",
|
|
84
|
+
severity: expectedClarificationBlock ? "info" : "warning",
|
|
76
85
|
detail: decision.authority.workflowStatus,
|
|
77
86
|
});
|
|
78
87
|
}
|
|
79
|
-
if (
|
|
88
|
+
if (authorityReason === "workflow_selection_not_exercised") {
|
|
89
|
+
signals.push({
|
|
90
|
+
name: "workflow_selection_not_exercised",
|
|
91
|
+
severity: "info",
|
|
92
|
+
detail: "pruning_disabled",
|
|
93
|
+
});
|
|
94
|
+
}
|
|
95
|
+
if (authorityReason === "workflow_differs") {
|
|
80
96
|
signals.push({
|
|
81
97
|
name: "workflow_differs",
|
|
82
98
|
severity: "info",
|
|
83
99
|
detail: summarizeDifference(missingCandidateTools, selectedOutsideCandidate),
|
|
84
100
|
});
|
|
85
101
|
}
|
|
86
|
-
if (missingCandidateTools.length > 0) {
|
|
102
|
+
if (missingCandidateTools.length > 0 && !expectedClarificationBlock && !selectionNotExercised) {
|
|
87
103
|
signals.push({
|
|
88
104
|
name: "workflow_candidate_not_selected",
|
|
89
105
|
severity: "warning",
|
|
90
106
|
detail: missingCandidateTools.join(", "),
|
|
91
107
|
});
|
|
92
108
|
}
|
|
93
|
-
if (selectedOutsideCandidate.length > 0) {
|
|
109
|
+
if (selectedOutsideCandidate.length > 0 && !selectionNotExercised) {
|
|
94
110
|
signals.push({
|
|
95
111
|
name: "selected_outside_workflow_candidate",
|
|
96
112
|
severity: "info",
|
|
97
113
|
detail: selectedOutsideCandidate.join(", "),
|
|
98
114
|
});
|
|
99
115
|
}
|
|
100
|
-
if (selectedOutsideSuggestions.length > 0) {
|
|
116
|
+
if (selectedOutsideSuggestions.length > 0 && !selectionNotExercised) {
|
|
101
117
|
signals.push({
|
|
102
118
|
name: "selected_outside_workflow_suggestions",
|
|
103
119
|
severity: "info",
|
|
104
120
|
detail: selectedOutsideSuggestions.join(", "),
|
|
105
121
|
});
|
|
106
122
|
}
|
|
107
|
-
if (selectedPolicyExcludedOptionalTools.length > 0) {
|
|
123
|
+
if (selectedPolicyExcludedOptionalTools.length > 0 && !selectionNotExercised) {
|
|
108
124
|
signals.push({
|
|
109
125
|
name: "selected_policy_excluded_optional_tool",
|
|
110
126
|
severity: "info",
|
|
@@ -157,6 +173,8 @@ export function buildAssistantWorkflowToolSelectionEvaluationTracePayload(evalua
|
|
|
157
173
|
workflowSelectionActiveProfileIds: evaluation.workflowSelectionActiveProfileIds,
|
|
158
174
|
workflowSelectionProfileToolNames: evaluation.workflowSelectionProfileToolNames,
|
|
159
175
|
workflowSelectionProfileToolCount: evaluation.workflowSelectionProfileToolCount,
|
|
176
|
+
expectedClarificationBlock: evaluation.expectedClarificationBlock,
|
|
177
|
+
selectionNotExercised: evaluation.selectionNotExercised,
|
|
160
178
|
signalCount: evaluation.signalCount,
|
|
161
179
|
warningCount: evaluation.warningCount,
|
|
162
180
|
signals: evaluation.signals.map((signal) => ({
|
|
@@ -189,6 +207,8 @@ export function buildAssistantWorkflowToolSelectionEvaluationBatchTracePayload(b
|
|
|
189
207
|
shadowEquivalentCount: intent.shadowEquivalentCount,
|
|
190
208
|
workflowDiffersCount: intent.workflowDiffersCount,
|
|
191
209
|
workflowBlockedCount: intent.workflowBlockedCount,
|
|
210
|
+
expectedClarificationBlockCount: intent.expectedClarificationBlockCount,
|
|
211
|
+
selectionNotExercisedCount: intent.selectionNotExercisedCount,
|
|
192
212
|
workflowUnavailableCount: intent.workflowUnavailableCount,
|
|
193
213
|
warningCount: intent.warningCount,
|
|
194
214
|
workflowAuthorityConfirmed: intent.workflowAuthorityConfirmed,
|
|
@@ -204,6 +224,8 @@ export function buildAssistantWorkflowToolSelectionEvaluationBatchTracePayload(b
|
|
|
204
224
|
shadowEquivalentCount: segment.shadowEquivalentCount,
|
|
205
225
|
workflowDiffersCount: segment.workflowDiffersCount,
|
|
206
226
|
workflowBlockedCount: segment.workflowBlockedCount,
|
|
227
|
+
expectedClarificationBlockCount: segment.expectedClarificationBlockCount,
|
|
228
|
+
selectionNotExercisedCount: segment.selectionNotExercisedCount,
|
|
207
229
|
workflowUnavailableCount: segment.workflowUnavailableCount,
|
|
208
230
|
warningCount: segment.warningCount,
|
|
209
231
|
workflowAuthorityConfirmed: segment.workflowAuthorityConfirmed,
|
|
@@ -226,6 +248,8 @@ export function buildAssistantWorkflowToolSelectionMigrationReport(batch, option
|
|
|
226
248
|
const readySegmentCandidates = segmentCandidates.filter((candidate) => candidate.status === "ready_for_workflow_authority");
|
|
227
249
|
const confirmedIntentCandidates = intentCandidates.filter((candidate) => candidate.status === "workflow_authority_confirmed");
|
|
228
250
|
const confirmedSegmentCandidates = segmentCandidates.filter((candidate) => candidate.status === "workflow_authority_confirmed");
|
|
251
|
+
const expectedClarificationCandidates = [...intentCandidates, ...segmentCandidates].filter((candidate) => candidate.status === "expected_clarification_only");
|
|
252
|
+
const selectionNotExercisedCandidates = [...intentCandidates, ...segmentCandidates].filter((candidate) => candidate.status === "selection_not_exercised_only");
|
|
229
253
|
const reviewCandidates = [...intentCandidates, ...segmentCandidates].filter((candidate) => candidate.status === "needs_review");
|
|
230
254
|
return {
|
|
231
255
|
generatedAt: batch.generatedAt,
|
|
@@ -239,6 +263,8 @@ export function buildAssistantWorkflowToolSelectionMigrationReport(batch, option
|
|
|
239
263
|
readySegmentCandidates,
|
|
240
264
|
confirmedIntentCandidates,
|
|
241
265
|
confirmedSegmentCandidates,
|
|
266
|
+
expectedClarificationCandidates,
|
|
267
|
+
selectionNotExercisedCandidates,
|
|
242
268
|
reviewCandidates,
|
|
243
269
|
};
|
|
244
270
|
}
|
|
@@ -255,6 +281,8 @@ export function buildAssistantWorkflowToolSelectionMigrationReportTracePayload(r
|
|
|
255
281
|
readySegmentCandidates: report.readySegmentCandidates.map(toMigrationCandidateTracePayload),
|
|
256
282
|
confirmedIntentCandidates: report.confirmedIntentCandidates.map(toMigrationCandidateTracePayload),
|
|
257
283
|
confirmedSegmentCandidates: report.confirmedSegmentCandidates.map(toMigrationCandidateTracePayload),
|
|
284
|
+
expectedClarificationCandidates: report.expectedClarificationCandidates.map(toMigrationCandidateTracePayload),
|
|
285
|
+
selectionNotExercisedCandidates: report.selectionNotExercisedCandidates.map(toMigrationCandidateTracePayload),
|
|
258
286
|
reviewCandidates: report.reviewCandidates.map(toMigrationCandidateTracePayload),
|
|
259
287
|
};
|
|
260
288
|
}
|
|
@@ -377,6 +405,8 @@ export function buildAssistantWorkflowToolSelectionMigrationReviewArtifact(input
|
|
|
377
405
|
readySegmentCandidateKeys: input.report.readySegmentCandidates.map((candidate) => candidate.key),
|
|
378
406
|
confirmedIntentCandidateKeys: input.report.confirmedIntentCandidates.map((candidate) => candidate.key),
|
|
379
407
|
confirmedSegmentCandidateKeys: input.report.confirmedSegmentCandidates.map((candidate) => candidate.key),
|
|
408
|
+
expectedClarificationCandidateKeys: input.report.expectedClarificationCandidates.map((candidate) => candidate.key),
|
|
409
|
+
selectionNotExercisedCandidateKeys: input.report.selectionNotExercisedCandidates.map((candidate) => candidate.key),
|
|
380
410
|
},
|
|
381
411
|
review: {
|
|
382
412
|
reviewIntentIds: input.proposal.reviewIntentIds,
|
|
@@ -413,6 +443,8 @@ export function buildAssistantWorkflowToolSelectionMigrationReviewArtifactTraceP
|
|
|
413
443
|
readySegmentCandidateKeys: artifact.evidence.readySegmentCandidateKeys,
|
|
414
444
|
confirmedIntentCandidateKeys: artifact.evidence.confirmedIntentCandidateKeys,
|
|
415
445
|
confirmedSegmentCandidateKeys: artifact.evidence.confirmedSegmentCandidateKeys,
|
|
446
|
+
expectedClarificationCandidateKeys: artifact.evidence.expectedClarificationCandidateKeys,
|
|
447
|
+
selectionNotExercisedCandidateKeys: artifact.evidence.selectionNotExercisedCandidateKeys,
|
|
416
448
|
},
|
|
417
449
|
review: {
|
|
418
450
|
reviewIntentIds: artifact.review.reviewIntentIds,
|
|
@@ -438,14 +470,22 @@ function buildReadinessSummary(intentReadiness, segmentReadiness) {
|
|
|
438
470
|
function buildMigrationCandidate(kind, readiness, minDecisionCount) {
|
|
439
471
|
const isSegment = isSegmentReadiness(readiness);
|
|
440
472
|
const key = isSegment ? readiness.segmentKey : readiness.intentId;
|
|
441
|
-
const
|
|
442
|
-
const
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
473
|
+
const expectedClarificationOnly = isExpectedClarificationOnly(readiness);
|
|
474
|
+
const selectionNotExercisedOnly = isSelectionNotExercisedOnly(readiness);
|
|
475
|
+
const blockingReasons = expectedClarificationOnly || selectionNotExercisedOnly
|
|
476
|
+
? []
|
|
477
|
+
: collectMigrationBlockingReasons(readiness, minDecisionCount);
|
|
478
|
+
const status = expectedClarificationOnly
|
|
479
|
+
? "expected_clarification_only"
|
|
480
|
+
: selectionNotExercisedOnly
|
|
481
|
+
? "selection_not_exercised_only"
|
|
482
|
+
: blockingReasons.length > 0
|
|
483
|
+
? "needs_review"
|
|
484
|
+
: readiness.workflowAuthorityConfirmed
|
|
485
|
+
? "workflow_authority_confirmed"
|
|
486
|
+
: readiness.readyForWorkflowAuthority
|
|
487
|
+
? "ready_for_workflow_authority"
|
|
488
|
+
: "needs_review";
|
|
449
489
|
const recommendedAction = buildMigrationRecommendedAction(kind, status);
|
|
450
490
|
return {
|
|
451
491
|
kind,
|
|
@@ -459,6 +499,8 @@ function buildMigrationCandidate(kind, readiness, minDecisionCount) {
|
|
|
459
499
|
shadowEquivalentCount: readiness.shadowEquivalentCount,
|
|
460
500
|
workflowDiffersCount: readiness.workflowDiffersCount,
|
|
461
501
|
workflowBlockedCount: readiness.workflowBlockedCount,
|
|
502
|
+
expectedClarificationBlockCount: readiness.expectedClarificationBlockCount,
|
|
503
|
+
selectionNotExercisedCount: readiness.selectionNotExercisedCount,
|
|
462
504
|
workflowUnavailableCount: readiness.workflowUnavailableCount,
|
|
463
505
|
warningCount: readiness.warningCount,
|
|
464
506
|
status,
|
|
@@ -467,6 +509,27 @@ function buildMigrationCandidate(kind, readiness, minDecisionCount) {
|
|
|
467
509
|
rationale: buildMigrationRationale(kind, status, readiness, blockingReasons, minDecisionCount),
|
|
468
510
|
};
|
|
469
511
|
}
|
|
512
|
+
function isExpectedClarificationOnly(readiness) {
|
|
513
|
+
return readiness.expectedClarificationBlockCount > 0 &&
|
|
514
|
+
readiness.workflowAuthorityCount === 0 &&
|
|
515
|
+
readiness.shadowEquivalentCount === 0 &&
|
|
516
|
+
readiness.workflowDiffersCount === 0 &&
|
|
517
|
+
unexpectedWorkflowBlockedCount(readiness) === 0 &&
|
|
518
|
+
readiness.workflowUnavailableCount === 0 &&
|
|
519
|
+
readiness.warningCount === 0;
|
|
520
|
+
}
|
|
521
|
+
function isSelectionNotExercisedOnly(readiness) {
|
|
522
|
+
return readiness.selectionNotExercisedCount > 0 &&
|
|
523
|
+
readiness.workflowAuthorityCount === 0 &&
|
|
524
|
+
readiness.shadowEquivalentCount === 0 &&
|
|
525
|
+
readiness.workflowDiffersCount === 0 &&
|
|
526
|
+
unexpectedWorkflowBlockedCount(readiness) === 0 &&
|
|
527
|
+
readiness.workflowUnavailableCount === 0 &&
|
|
528
|
+
readiness.warningCount === 0;
|
|
529
|
+
}
|
|
530
|
+
function unexpectedWorkflowBlockedCount(readiness) {
|
|
531
|
+
return Math.max(0, readiness.workflowBlockedCount - readiness.expectedClarificationBlockCount);
|
|
532
|
+
}
|
|
470
533
|
function collectMigrationBlockingReasons(readiness, minDecisionCount) {
|
|
471
534
|
const reasons = [];
|
|
472
535
|
if (readiness.intentId === "unknown")
|
|
@@ -477,21 +540,21 @@ function collectMigrationBlockingReasons(readiness, minDecisionCount) {
|
|
|
477
540
|
reasons.push("warnings_present");
|
|
478
541
|
if (readiness.workflowDiffersCount > 0)
|
|
479
542
|
reasons.push("workflow_differs");
|
|
480
|
-
if (readiness
|
|
543
|
+
if (unexpectedWorkflowBlockedCount(readiness) > 0)
|
|
481
544
|
reasons.push("workflow_blocked");
|
|
482
545
|
if (readiness.workflowUnavailableCount > 0)
|
|
483
546
|
reasons.push("workflow_unavailable");
|
|
484
547
|
if (readiness.workflowAuthorityCount > 0 &&
|
|
485
548
|
readiness.shadowEquivalentCount > 0 &&
|
|
486
549
|
readiness.workflowDiffersCount === 0 &&
|
|
487
|
-
readiness
|
|
550
|
+
unexpectedWorkflowBlockedCount(readiness) === 0 &&
|
|
488
551
|
readiness.workflowUnavailableCount === 0) {
|
|
489
552
|
reasons.push("mixed_authority_evidence");
|
|
490
553
|
}
|
|
491
554
|
if (readiness.workflowAuthorityCount === 0 &&
|
|
492
555
|
readiness.shadowEquivalentCount === 0 &&
|
|
493
556
|
readiness.workflowDiffersCount === 0 &&
|
|
494
|
-
readiness
|
|
557
|
+
unexpectedWorkflowBlockedCount(readiness) === 0 &&
|
|
495
558
|
readiness.workflowUnavailableCount === 0) {
|
|
496
559
|
reasons.push("no_workflow_equivalence_evidence");
|
|
497
560
|
}
|
|
@@ -500,6 +563,10 @@ function collectMigrationBlockingReasons(readiness, minDecisionCount) {
|
|
|
500
563
|
function buildMigrationRecommendedAction(kind, status) {
|
|
501
564
|
if (status === "workflow_authority_confirmed")
|
|
502
565
|
return "keep_runtime_authority";
|
|
566
|
+
if (status === "expected_clarification_only")
|
|
567
|
+
return "keep_controller_clarification";
|
|
568
|
+
if (status === "selection_not_exercised_only")
|
|
569
|
+
return "exercise_pruned_selection";
|
|
503
570
|
if (status === "ready_for_workflow_authority") {
|
|
504
571
|
return kind === "intent"
|
|
505
572
|
? "add_intent_to_authority_allow_list"
|
|
@@ -509,11 +576,23 @@ function buildMigrationRecommendedAction(kind, status) {
|
|
|
509
576
|
}
|
|
510
577
|
function buildMigrationRationale(kind, status, readiness, blockingReasons, minDecisionCount) {
|
|
511
578
|
const scope = kind === "intent" ? `intent ${readiness.intentId}` : `segment ${getReadinessKey(readiness)}`;
|
|
579
|
+
const clarificationSuffix = readiness.expectedClarificationBlockCount > 0
|
|
580
|
+
? ` with ${readiness.expectedClarificationBlockCount} expected clarification block(s)`
|
|
581
|
+
: "";
|
|
582
|
+
const selectionNotExercisedSuffix = readiness.selectionNotExercisedCount > 0
|
|
583
|
+
? ` with ${readiness.selectionNotExercisedCount} selection-not-exercised decision(s)`
|
|
584
|
+
: "";
|
|
512
585
|
if (status === "workflow_authority_confirmed") {
|
|
513
|
-
return `${scope} is already workflow-owned across ${readiness.
|
|
586
|
+
return `${scope} is already workflow-owned across ${readiness.workflowAuthorityCount} decision(s)${clarificationSuffix} and without warnings.`;
|
|
587
|
+
}
|
|
588
|
+
if (status === "expected_clarification_only") {
|
|
589
|
+
return `${scope} has ${readiness.expectedClarificationBlockCount} expected clarification block(s) and does not require an authority change.`;
|
|
590
|
+
}
|
|
591
|
+
if (status === "selection_not_exercised_only") {
|
|
592
|
+
return `${scope} was observed with pruning disabled and needs a pruned shadow selection before an authority change.`;
|
|
514
593
|
}
|
|
515
594
|
if (status === "ready_for_workflow_authority") {
|
|
516
|
-
return `${scope} has ${readiness.shadowEquivalentCount} shadow-equivalent decision(s), no warnings, and meets minDecisionCount=${minDecisionCount}.`;
|
|
595
|
+
return `${scope} has ${readiness.shadowEquivalentCount} shadow-equivalent decision(s)${clarificationSuffix}${selectionNotExercisedSuffix}, no warnings, and meets minDecisionCount=${minDecisionCount}.`;
|
|
517
596
|
}
|
|
518
597
|
return `${scope} is not ready for runtime migration: ${blockingReasons.join(", ") || "unknown_reason"}.`;
|
|
519
598
|
}
|
|
@@ -536,6 +615,8 @@ function toMigrationCandidateTracePayload(candidate) {
|
|
|
536
615
|
shadowEquivalentCount: candidate.shadowEquivalentCount,
|
|
537
616
|
workflowDiffersCount: candidate.workflowDiffersCount,
|
|
538
617
|
workflowBlockedCount: candidate.workflowBlockedCount,
|
|
618
|
+
expectedClarificationBlockCount: candidate.expectedClarificationBlockCount,
|
|
619
|
+
selectionNotExercisedCount: candidate.selectionNotExercisedCount,
|
|
539
620
|
workflowUnavailableCount: candidate.workflowUnavailableCount,
|
|
540
621
|
warningCount: candidate.warningCount,
|
|
541
622
|
status: candidate.status,
|
|
@@ -596,6 +677,20 @@ function normalizeMinDecisionCount(value) {
|
|
|
596
677
|
function isInterestingRow(row) {
|
|
597
678
|
return row.authoritySource !== "workflow";
|
|
598
679
|
}
|
|
680
|
+
function isExpectedClarificationBlock(decision) {
|
|
681
|
+
return decision.authority.reason === "workflow_blocked" &&
|
|
682
|
+
decision.authority.workflowStatus === "needs-clarification";
|
|
683
|
+
}
|
|
684
|
+
function isWorkflowSelectionNotExercised(decision) {
|
|
685
|
+
return decision.authority.source === "legacy-pruner" && (decision.authority.reason === "workflow_selection_not_exercised" ||
|
|
686
|
+
(decision.authority.reason === "workflow_differs" &&
|
|
687
|
+
decision.comparisonPayload?.["pruningEnabled"] === false));
|
|
688
|
+
}
|
|
689
|
+
function getEvaluationAuthorityReason(decision) {
|
|
690
|
+
return isWorkflowSelectionNotExercised(decision)
|
|
691
|
+
? "workflow_selection_not_exercised"
|
|
692
|
+
: decision.authority.reason;
|
|
693
|
+
}
|
|
599
694
|
function summarizeDifference(missingCandidateTools, selectedOutsideCandidate) {
|
|
600
695
|
const parts = [
|
|
601
696
|
missingCandidateTools.length > 0 ? `missing=${missingCandidateTools.join(",")}` : "",
|
|
@@ -617,6 +712,8 @@ function buildIntentReadiness(rows) {
|
|
|
617
712
|
const shadowEquivalentCount = intentRows.filter((row) => row.authorityReason === "workflow_authority_not_enabled").length;
|
|
618
713
|
const workflowDiffersCount = intentRows.filter((row) => row.authorityReason === "workflow_differs").length;
|
|
619
714
|
const workflowBlockedCount = intentRows.filter((row) => row.authorityReason === "workflow_blocked").length;
|
|
715
|
+
const expectedClarificationBlockCount = intentRows.filter((row) => row.expectedClarificationBlock).length;
|
|
716
|
+
const selectionNotExercisedCount = intentRows.filter((row) => row.selectionNotExercised).length;
|
|
620
717
|
const workflowUnavailableCount = intentRows.filter((row) => row.authorityReason === "workflow_unavailable").length;
|
|
621
718
|
const warningCount = intentRows.reduce((total, row) => total + row.warningCount, 0);
|
|
622
719
|
return {
|
|
@@ -626,20 +723,22 @@ function buildIntentReadiness(rows) {
|
|
|
626
723
|
shadowEquivalentCount,
|
|
627
724
|
workflowDiffersCount,
|
|
628
725
|
workflowBlockedCount,
|
|
726
|
+
expectedClarificationBlockCount,
|
|
727
|
+
selectionNotExercisedCount,
|
|
629
728
|
workflowUnavailableCount,
|
|
630
729
|
warningCount,
|
|
631
730
|
workflowAuthorityConfirmed: (intentId !== "unknown" &&
|
|
632
731
|
workflowAuthorityCount > 0 &&
|
|
633
732
|
shadowEquivalentCount === 0 &&
|
|
634
733
|
workflowDiffersCount === 0 &&
|
|
635
|
-
workflowBlockedCount === 0 &&
|
|
734
|
+
unexpectedWorkflowBlockedCount({ workflowBlockedCount, expectedClarificationBlockCount }) === 0 &&
|
|
636
735
|
workflowUnavailableCount === 0 &&
|
|
637
736
|
warningCount === 0),
|
|
638
737
|
readyForWorkflowAuthority: (intentId !== "unknown" &&
|
|
639
738
|
workflowAuthorityCount === 0 &&
|
|
640
739
|
shadowEquivalentCount > 0 &&
|
|
641
740
|
workflowDiffersCount === 0 &&
|
|
642
|
-
workflowBlockedCount === 0 &&
|
|
741
|
+
unexpectedWorkflowBlockedCount({ workflowBlockedCount, expectedClarificationBlockCount }) === 0 &&
|
|
643
742
|
workflowUnavailableCount === 0 &&
|
|
644
743
|
warningCount === 0),
|
|
645
744
|
};
|
|
@@ -677,6 +776,8 @@ function buildSegmentReadiness(rows) {
|
|
|
677
776
|
shadowEquivalentCount: 0,
|
|
678
777
|
workflowDiffersCount: 0,
|
|
679
778
|
workflowBlockedCount: 0,
|
|
779
|
+
expectedClarificationBlockCount: 0,
|
|
780
|
+
selectionNotExercisedCount: 0,
|
|
680
781
|
workflowUnavailableCount: 0,
|
|
681
782
|
warningCount: 0,
|
|
682
783
|
workflowAuthorityConfirmed: false,
|