noteconnection 1.7.0 → 1.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +674 -21
- package/README.md +242 -62
- package/dist/src/agent_workspace.contract.parity.test.js +475 -0
- package/dist/src/agent_workspace.frontend.test.js +9989 -0
- package/dist/src/agent_workspace.locale.contract.test.js +95 -0
- package/dist/src/agent_workspace.runtime.behavior.test.js +5072 -0
- package/dist/src/copy.assets.contract.test.js +130 -0
- package/dist/src/core/PathBridge.js +28 -3
- package/dist/src/export/WorkspaceExportBundle.js +1130 -0
- package/dist/src/export/WorkspaceExportBundle.test.js +1480 -0
- package/dist/src/export/index.js +18 -0
- package/dist/src/export/types.js +2 -0
- package/dist/src/fixrisk.issue.verifier.contract.test.js +3 -1
- package/dist/src/foundation.ann.runtime.contract.test.js +81 -0
- package/dist/src/foundation.release.evidence.contract.test.js +406 -0
- package/dist/src/foundation.sqlite.runtime.contract.test.js +82 -0
- package/dist/src/frontend/README.md +242 -62
- package/dist/src/frontend/agent_workspace.js +5353 -0
- package/dist/src/frontend/agent_workspace_runtime.js +4434 -0
- package/dist/src/frontend/analysis.js +31 -4
- package/dist/src/frontend/app.js +2178 -62
- package/dist/src/frontend/focus_mode_interactions.js +33 -0
- package/dist/src/frontend/godot_future_path_renderer.js +540 -0
- package/dist/src/frontend/godot_tree_interactions.js +178 -0
- package/dist/src/frontend/graph_state.mjs +105 -0
- package/dist/src/frontend/hosted_future_path_runtime.js +157 -0
- package/dist/src/frontend/i18n.mjs +186 -0
- package/dist/src/frontend/index.html +641 -266
- package/dist/src/frontend/layout_gpu.js +12 -4
- package/dist/src/frontend/locales/en.json +721 -5
- package/dist/src/frontend/locales/zh.json +721 -5
- package/dist/src/frontend/main.mjs +60 -0
- package/dist/src/frontend/markdown_runtime.js +827 -0
- package/dist/src/frontend/notemd.css +49 -0
- package/dist/src/frontend/notemd.html +7 -1
- package/dist/src/frontend/notemd.js +64 -0
- package/dist/src/frontend/path.html +107 -0
- package/dist/src/frontend/path_app.js +2108 -130
- package/dist/src/frontend/path_layout.mjs +143 -0
- package/dist/src/frontend/path_mermaid_utils.mjs +108 -0
- package/dist/src/frontend/path_modules_bridge.js +486 -0
- package/dist/src/frontend/path_state.mjs +118 -0
- package/dist/src/frontend/path_styles.css +146 -0
- package/dist/src/frontend/path_worker_bridge.mjs +85 -0
- package/dist/src/frontend/reader.js +393 -33
- package/dist/src/frontend/runtime_bridge.js +67 -54
- package/dist/src/frontend/runtime_bridge.mjs +279 -0
- package/dist/src/frontend/settings.js +130 -12
- package/dist/src/frontend/source_manager.js +190 -21
- package/dist/src/frontend/styles.css +2720 -72
- package/dist/src/frontend/workbench_state.mjs +101 -0
- package/dist/src/frontend/workspace_panes.js +10168 -0
- package/dist/src/frontend.locale.contract.test.js +62 -0
- package/dist/src/godot.sidecar.bootstrap.contract.test.js +244 -0
- package/dist/src/indexing/IndexLifecycle.js +195 -0
- package/dist/src/indexing/IndexLifecycle.test.js +49 -0
- package/dist/src/indexing/SegmentBuilder.js +64 -0
- package/dist/src/indexing/UnitBuilder.js +48 -0
- package/dist/src/indexing/types.js +2 -0
- package/dist/src/knowledge.api.contract.test.js +170 -0
- package/dist/src/learning/KnowledgeLearningPlatform.js +10386 -0
- package/dist/src/learning/KnowledgeLearningPlatform.persistence.test.js +327 -0
- package/dist/src/learning/KnowledgeLearningPlatform.program-f.test.js +99 -0
- package/dist/src/learning/KnowledgeLearningPlatform.test.js +2971 -0
- package/dist/src/learning/KnowledgeWorkspaceConversationRegression.js +2974 -0
- package/dist/src/learning/KnowledgeWorkspaceConversationRegression.test.js +3928 -0
- package/dist/src/learning/answerReleaseReview.js +4319 -0
- package/dist/src/learning/answerReleaseReview.test.js +2888 -0
- package/dist/src/learning/api.js +2 -0
- package/dist/src/learning/conversationComposer.js +1480 -0
- package/dist/src/learning/conversationComposer.test.js +1817 -0
- package/dist/src/learning/domains/ConversationManager.js +53 -0
- package/dist/src/learning/domains/KnowledgeIngestor.js +238 -0
- package/dist/src/learning/domains/KnowledgeQuerier.js +187 -0
- package/dist/src/learning/domains/MasteryEngine.js +387 -0
- package/dist/src/learning/domains/MemoryPolicyManager.js +408 -0
- package/dist/src/learning/domains/QualityEvaluator.js +307 -0
- package/dist/src/learning/domains/TutorRouter.js +313 -0
- package/dist/src/learning/domains/index.js +33 -0
- package/dist/src/learning/domains/types.js +7 -0
- package/dist/src/learning/errors.js +29 -0
- package/dist/src/learning/evidenceContextAssembler.js +1176 -0
- package/dist/src/learning/evidenceContextAssembler.test.js +6332 -0
- package/dist/src/learning/graphContextAssembler.js +870 -0
- package/dist/src/learning/graphContextAssembler.test.js +1033 -0
- package/dist/src/learning/index.js +28 -0
- package/dist/src/learning/queryBackend.js +1898 -0
- package/dist/src/learning/queryBackend.test.js +955 -0
- package/dist/src/learning/ragContextPack.js +257 -0
- package/dist/src/learning/ragContextPack.test.js +160 -0
- package/dist/src/learning/ragPublicText.js +38 -0
- package/dist/src/learning/ragSufficiencyJudge.js +161 -0
- package/dist/src/learning/ragSufficiencyJudge.test.js +177 -0
- package/dist/src/learning/ragSufficiencyProviderJudge.js +227 -0
- package/dist/src/learning/ragSufficiencyProviderJudge.test.js +156 -0
- package/dist/src/learning/requestNormalization.js +198 -0
- package/dist/src/learning/runtimeCapability.js +4677 -0
- package/dist/src/learning/runtimeCapability.test.js +3635 -0
- package/dist/src/learning/store.js +1240 -0
- package/dist/src/learning/store.test.js +1126 -0
- package/dist/src/learning/tutorAdapter.js +2 -0
- package/dist/src/learning/types.js +2 -0
- package/dist/src/learning/vectorAccelerationAdapter.js +942 -0
- package/dist/src/learning/vectorAccelerationAdapter.test.js +382 -0
- package/dist/src/lfs.asset.policy.contract.test.js +153 -0
- package/dist/src/license.policy.contract.test.js +66 -0
- package/dist/src/memory/MemoryGovernance.js +74 -0
- package/dist/src/memory/MemoryGovernance.test.js +46 -0
- package/dist/src/memory/types.js +2 -0
- package/dist/src/mermaid.frontend.guard.contract.test.js +77 -0
- package/dist/src/middleware/auth.js +17 -0
- package/dist/src/middleware/body-parser.js +45 -0
- package/dist/src/middleware/cors.js +44 -0
- package/dist/src/middleware/index.js +21 -0
- package/dist/src/middleware/request-trace.js +96 -0
- package/dist/src/notemd/AppConfigToml.js +6 -4
- package/dist/src/notemd/MermaidProcessor.js +400 -50
- package/dist/src/notemd/NotemdService.js +498 -13
- package/dist/src/notemd/PromptManager.js +15 -0
- package/dist/src/notemd/cli/commands.js +357 -0
- package/dist/src/notemd/cli/dispatcher.js +225 -0
- package/dist/src/notemd/cli/index.js +169 -0
- package/dist/src/notemd/cli/parser.js +68 -0
- package/dist/src/notemd/cli/types.js +2 -0
- package/dist/src/notemd/constants.js +43 -0
- package/dist/src/notemd/diagram/diagramGenerationService.js +78 -0
- package/dist/src/notemd/diagram/diagramSpec.js +79 -0
- package/dist/src/notemd/diagram/diagramSpecResponseParser.js +131 -0
- package/dist/src/notemd/diagram/intent.js +95 -0
- package/dist/src/notemd/diagram/planner.js +71 -0
- package/dist/src/notemd/diagram/prompts/diagramSpecPrompt.js +42 -0
- package/dist/src/notemd/diagram/types.js +18 -0
- package/dist/src/notemd/index.js +26 -0
- package/dist/src/notemd/operations/capabilityManifest.js +23 -0
- package/dist/src/notemd/operations/cliContracts.js +17 -0
- package/dist/src/notemd/operations/configProfileCommands.js +85 -0
- package/dist/src/notemd/operations/registry.contract.test.js +95 -0
- package/dist/src/notemd/operations/registry.js +991 -0
- package/dist/src/notemd/operations/types.js +2 -0
- package/dist/src/notemd/providerDiagnostics.js +220 -0
- package/dist/src/notemd/providerProfiles.js +42 -0
- package/dist/src/notemd/providerTemplates.js +231 -0
- package/dist/src/notemd/search/DuckDuckGoProvider.js +39 -0
- package/dist/src/notemd/search/SearchManager.js +13 -0
- package/dist/src/notemd/search/SearchProvider.js +2 -0
- package/dist/src/notemd/search/TavilyProvider.js +44 -0
- package/dist/src/notemd.agent.manifest.test.js +85 -0
- package/dist/src/notemd.api.contract.test.js +14 -1
- package/dist/src/notemd.app_config_toml.test.js +2 -1
- package/dist/src/notemd.batch.workflow.test.js +117 -0
- package/dist/src/notemd.cli.e2e.test.js +136 -0
- package/dist/src/notemd.core.test.js +51 -0
- package/dist/src/notemd.diagram.pipeline.test.js +233 -0
- package/dist/src/notemd.providerTemplates.test.js +34 -0
- package/dist/src/notemd.server.integration.test.js +143 -35
- package/dist/src/notemd.workflow.pipeline.test.js +162 -0
- package/dist/src/pathbridge.handshake.contract.test.js +16 -2
- package/dist/src/pathmode.background.contract.test.js +69 -0
- package/dist/src/pathmode.settings.api.contract.test.js +9 -0
- package/dist/src/pkg.sidecar.contract.test.js +9 -3
- package/dist/src/platform/ExportProfile.js +58 -0
- package/dist/src/platform/PlatformCapabilities.js +45 -0
- package/dist/src/platform/PlatformCapabilities.test.js +30 -0
- package/dist/src/platform/RenderMaterializer.js +33 -0
- package/dist/src/platform/RenderMaterializer.test.js +32 -0
- package/dist/src/query_backend.external_http.integration.test.js +410 -0
- package/dist/src/reader_renderer.js +404 -3
- package/dist/src/reader_renderer.test.js +87 -0
- package/dist/src/release.godot.mirror.contract.test.js +73 -0
- package/dist/src/resources/ResourceRegistry.js +223 -0
- package/dist/src/resources/ResourceRegistry.test.js +61 -0
- package/dist/src/resources/types.js +2 -0
- package/dist/src/routes/agentWorkspaceDiagnostics.js +173 -0
- package/dist/src/routes/data.js +267 -0
- package/dist/src/routes/diagnostics.js +51 -0
- package/dist/src/routes/index.js +23 -0
- package/dist/src/routes/knowledge.js +968 -0
- package/dist/src/routes/markdown.js +287 -0
- package/dist/src/routes/notemd.js +565 -0
- package/dist/src/routes/registry.contract.test.js +130 -0
- package/dist/src/routes/render.js +285 -0
- package/dist/src/routes/runtimeRunbookRouteOps.js +149 -0
- package/dist/src/routes/runtimeRunbookRouteOps.test.js +194 -0
- package/dist/src/routes/settings.js +6 -0
- package/dist/src/routes/staticFiles.js +94 -0
- package/dist/src/routes/types.js +2 -0
- package/dist/src/runtime.transport.adapter.contract.test.js +81 -0
- package/dist/src/server.js +11090 -1549
- package/dist/src/server.migration.test.js +193 -21
- package/dist/src/server.port.fallback.contract.test.js +63 -0
- package/dist/src/session/SessionStateStore.js +81 -0
- package/dist/src/session/SessionStateStore.test.js +58 -0
- package/dist/src/session/types.js +2 -0
- package/dist/src/settings.runtime.contract.test.js +50 -0
- package/dist/src/shared/types.contract.test.js +107 -0
- package/dist/src/shared/types.js +22 -0
- package/dist/src/sidecar.replacement.boundary.contract.test.js +128 -0
- package/dist/src/sidecar.supply.readiness.contract.test.js +144 -0
- package/dist/src/source_manager.loadflow.test.js +46 -0
- package/dist/src/startup.layout.snapshot.contract.test.js +57 -0
- package/dist/src/tauri.frontend.build.contract.test.js +60 -0
- package/dist/src/tauri.sidecar.cleanup.contract.test.js +21 -0
- package/dist/src/utils/RuntimePaths.js +4 -13
- package/dist/src/utils/platform.js +153 -0
- package/dist/src/workflows/WorkflowArtifactStore.js +96 -0
- package/dist/src/workflows/WorkflowArtifactStore.test.js +80 -0
- package/dist/src/workflows/types.js +2 -0
- package/dist/src/workspace/WorkspaceRegistry.js +122 -0
- package/dist/src/workspace/WorkspaceRegistry.test.js +29 -0
- package/dist/src/workspace/types.js +2 -0
- package/package.json +53 -10
|
@@ -0,0 +1,4677 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS = void 0;
|
|
4
|
+
exports.normalizeRuntimeCapabilityThresholds = normalizeRuntimeCapabilityThresholds;
|
|
5
|
+
exports.resolveRuntimeCapabilityThresholdsFromEnv = resolveRuntimeCapabilityThresholdsFromEnv;
|
|
6
|
+
exports.buildRuntimeCapabilityMatrix = buildRuntimeCapabilityMatrix;
|
|
7
|
+
exports.buildRuntimeCapabilityRunbook = buildRuntimeCapabilityRunbook;
|
|
8
|
+
exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS = {
|
|
9
|
+
minQuerySampleSize: 5,
|
|
10
|
+
queryFallbackWarnRatioPct: 10,
|
|
11
|
+
queryFallbackFailRatioPct: 20,
|
|
12
|
+
queryEvidenceCoverageWarnRatioPct: 90,
|
|
13
|
+
queryEvidenceCoverageFailRatioPct: 75,
|
|
14
|
+
queryTemporalValidityWarnRatioPct: 90,
|
|
15
|
+
queryTemporalValidityFailRatioPct: 75,
|
|
16
|
+
queryBackendExplainabilityGapWarnRatioPct: 20,
|
|
17
|
+
queryBackendExplainabilityGapFailRatioPct: 35,
|
|
18
|
+
queryBackendTrendWarnConfidenceRatioPct: 40,
|
|
19
|
+
queryBackendTrendFailConfidenceRatioPct: 70,
|
|
20
|
+
sessionPlanQualityWarnFailureStreak: 1,
|
|
21
|
+
sessionPlanQualityFailFailureStreak: 2,
|
|
22
|
+
apiInvalidRequestMinErrorSample: 3,
|
|
23
|
+
apiInvalidRequestWarnRatioPct: 30,
|
|
24
|
+
apiInvalidRequestFailRatioPct: 60,
|
|
25
|
+
apiInvalidRequestHotspotWarnCount: 3,
|
|
26
|
+
apiInvalidRequestHotspotFailCount: 8,
|
|
27
|
+
apiServerErrorMinRequestSample: 10,
|
|
28
|
+
apiServerErrorWarnRatioPct: 5,
|
|
29
|
+
apiServerErrorFailRatioPct: 15,
|
|
30
|
+
apiServerErrorHotspotWarnCount: 2,
|
|
31
|
+
apiServerErrorHotspotFailCount: 5,
|
|
32
|
+
apiTransientErrorMinRequestSample: 10,
|
|
33
|
+
apiTransientErrorWarnRatioPct: 8,
|
|
34
|
+
apiTransientErrorFailRatioPct: 18,
|
|
35
|
+
apiTransientErrorHotspotWarnCount: 2,
|
|
36
|
+
apiTransientErrorHotspotFailCount: 6,
|
|
37
|
+
apiLatencyMinRequestSample: 10,
|
|
38
|
+
apiLatencyP95WarnMs: 1200,
|
|
39
|
+
apiLatencyP95FailMs: 2500,
|
|
40
|
+
apiLatencyHotspotWarnMs: 1800,
|
|
41
|
+
apiLatencyHotspotFailMs: 3500,
|
|
42
|
+
queryVectorAccelerationShortCircuitWarnCount: 1,
|
|
43
|
+
queryVectorAccelerationShortCircuitFailCount: 4,
|
|
44
|
+
queryVectorAccelerationShortCircuitWarnRatioPct: 5,
|
|
45
|
+
queryVectorAccelerationShortCircuitFailRatioPct: 20,
|
|
46
|
+
queryVectorAccelerationConsecutiveFailuresWarnCount: 1,
|
|
47
|
+
queryVectorAccelerationConsecutiveFailuresFailCount: 3,
|
|
48
|
+
queryVectorAccelerationHalfOpenSuccessWarnRatioPct: 80,
|
|
49
|
+
queryVectorAccelerationHalfOpenSuccessFailRatioPct: 50,
|
|
50
|
+
queryVectorAccelerationPrefilterMinRequestSample: 5,
|
|
51
|
+
queryVectorAccelerationPrefilterWarnCandidateRatioPct: 95,
|
|
52
|
+
queryVectorAccelerationPrefilterFailCandidateRatioPct: 99,
|
|
53
|
+
storeGraphDbConnectorMinRequestSample: 5,
|
|
54
|
+
storeGraphDbConnectorFailureWarnRatioPct: 20,
|
|
55
|
+
storeGraphDbConnectorFailureFailRatioPct: 40,
|
|
56
|
+
storeGraphDbConnectorShortCircuitWarnRatioPct: 5,
|
|
57
|
+
storeGraphDbConnectorShortCircuitFailRatioPct: 20,
|
|
58
|
+
storeGraphDbConnectorConsecutiveFailuresWarnCount: 1,
|
|
59
|
+
storeGraphDbConnectorConsecutiveFailuresFailCount: 3,
|
|
60
|
+
};
|
|
61
|
+
function clamp(value, minValue, maxValue) {
|
|
62
|
+
return Math.max(minValue, Math.min(maxValue, value));
|
|
63
|
+
}
|
|
64
|
+
function normalizeInteger(value, fallback, minValue, maxValue) {
|
|
65
|
+
const numeric = Number(value);
|
|
66
|
+
if (!Number.isFinite(numeric)) {
|
|
67
|
+
return fallback;
|
|
68
|
+
}
|
|
69
|
+
return Math.floor(clamp(numeric, minValue, maxValue));
|
|
70
|
+
}
|
|
71
|
+
function normalizeRatio(value, fallback) {
|
|
72
|
+
const numeric = Number(value);
|
|
73
|
+
if (!Number.isFinite(numeric)) {
|
|
74
|
+
return Number(clamp(fallback, 0, 100).toFixed(4));
|
|
75
|
+
}
|
|
76
|
+
return Number(clamp(numeric, 0, 100).toFixed(4));
|
|
77
|
+
}
|
|
78
|
+
function normalizeRuntimeCapabilityHttpMethod(value) {
|
|
79
|
+
const normalized = String(value || '').trim().toUpperCase();
|
|
80
|
+
return (/^[A-Z]+$/).test(normalized) ? normalized : '';
|
|
81
|
+
}
|
|
82
|
+
function parseRuntimeCapabilityApiTraceRoute(value) {
|
|
83
|
+
const normalized = String(value || '').trim().slice(0, 256);
|
|
84
|
+
if (!normalized) {
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
const matched = normalized.match(/^([A-Z]+)\s+(.+)$/);
|
|
88
|
+
if (matched) {
|
|
89
|
+
const method = normalizeRuntimeCapabilityHttpMethod(matched[1]);
|
|
90
|
+
const pathPrefix = String(matched[2] || '').trim().slice(0, 256);
|
|
91
|
+
if (!pathPrefix) {
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
return {
|
|
95
|
+
method,
|
|
96
|
+
pathPrefix,
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
return {
|
|
100
|
+
method: '',
|
|
101
|
+
pathPrefix: normalized,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
function normalizeRuntimeCapabilityDebugTraceHint(hint) {
|
|
105
|
+
if (!hint) {
|
|
106
|
+
return undefined;
|
|
107
|
+
}
|
|
108
|
+
const pathPrefix = String(hint.pathPrefix || '').trim().slice(0, 256);
|
|
109
|
+
const method = normalizeRuntimeCapabilityHttpMethod(hint.method || '');
|
|
110
|
+
const errorCode = String(hint.errorCode || '')
|
|
111
|
+
.trim()
|
|
112
|
+
.toLowerCase()
|
|
113
|
+
.replace(/[^a-z0-9_.:-]+/g, '_')
|
|
114
|
+
.slice(0, 64);
|
|
115
|
+
const statusAtLeastRaw = Number(hint.statusAtLeast || 0);
|
|
116
|
+
const statusAtLeast = Number.isFinite(statusAtLeastRaw)
|
|
117
|
+
? Math.max(0, Math.floor(statusAtLeastRaw))
|
|
118
|
+
: 0;
|
|
119
|
+
if (!pathPrefix && !method && !errorCode && statusAtLeast <= 0) {
|
|
120
|
+
return undefined;
|
|
121
|
+
}
|
|
122
|
+
return {
|
|
123
|
+
pathPrefix,
|
|
124
|
+
statusAtLeast,
|
|
125
|
+
method,
|
|
126
|
+
errorCode,
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
function normalizeRuntimeCapabilityRecommendedActions(actionsRaw) {
|
|
130
|
+
const actions = actionsRaw
|
|
131
|
+
.map((item) => String(item || '').replace(/\s+/g, ' ').trim().slice(0, 220))
|
|
132
|
+
.filter(Boolean)
|
|
133
|
+
.slice(0, 5);
|
|
134
|
+
return actions.length > 0 ? actions : undefined;
|
|
135
|
+
}
|
|
136
|
+
function formatRuntimeCapabilityRouteLabel(route, fallbackPathPrefix, fallbackMethod) {
|
|
137
|
+
const method = normalizeRuntimeCapabilityHttpMethod(route?.method || fallbackMethod || '');
|
|
138
|
+
const pathPrefix = String(route?.pathPrefix || fallbackPathPrefix || '/api/knowledge').trim() || '/api/knowledge';
|
|
139
|
+
return [method, pathPrefix].filter(Boolean).join(' ');
|
|
140
|
+
}
|
|
141
|
+
function normalizeTutorRoutingDynamicPreferredModeToken(rawValue) {
|
|
142
|
+
const normalized = String(rawValue || '').trim().toLowerCase();
|
|
143
|
+
if (normalized === 'local' || normalized === 'cloud') {
|
|
144
|
+
return normalized;
|
|
145
|
+
}
|
|
146
|
+
return 'auto';
|
|
147
|
+
}
|
|
148
|
+
function extractTutorRoutingDynamicSignalsFromReason(routingReasonRaw) {
|
|
149
|
+
const routingReason = String(routingReasonRaw || '').trim();
|
|
150
|
+
if (!routingReason) {
|
|
151
|
+
return {
|
|
152
|
+
preferredMode: 'auto',
|
|
153
|
+
modeReason: '',
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
const tokens = routingReason
|
|
157
|
+
.split(',')
|
|
158
|
+
.map((item) => String(item || '').trim())
|
|
159
|
+
.filter(Boolean);
|
|
160
|
+
const preferredModeToken = tokens.find((item) => item.startsWith('dynamicPreferredMode='));
|
|
161
|
+
const modeReasonToken = tokens.find((item) => item.startsWith('dynamicModeReason='));
|
|
162
|
+
const preferredMode = normalizeTutorRoutingDynamicPreferredModeToken(preferredModeToken?.slice('dynamicPreferredMode='.length));
|
|
163
|
+
const modeReason = modeReasonToken
|
|
164
|
+
? modeReasonToken.slice('dynamicModeReason='.length).trim().slice(0, 220)
|
|
165
|
+
: '';
|
|
166
|
+
return {
|
|
167
|
+
preferredMode,
|
|
168
|
+
modeReason,
|
|
169
|
+
};
|
|
170
|
+
}
|
|
171
|
+
function buildRuntimeCapabilityRecommendedActions(check, context) {
|
|
172
|
+
const checkId = String(check?.checkId || '').trim().toLowerCase();
|
|
173
|
+
const status = String(check?.status || '').trim().toLowerCase();
|
|
174
|
+
if (!checkId || status === 'pass') {
|
|
175
|
+
return undefined;
|
|
176
|
+
}
|
|
177
|
+
const invalidRouteLabel = formatRuntimeCapabilityRouteLabel(context.apiInvalidRequestTopRoute, context.apiScopePathPrefix, context.apiScopeMethod);
|
|
178
|
+
const serverRouteLabel = formatRuntimeCapabilityRouteLabel(context.apiServerErrorTopRoute, context.apiScopePathPrefix, context.apiScopeMethod);
|
|
179
|
+
const transientRouteLabel = formatRuntimeCapabilityRouteLabel(context.apiTransientErrorTopRoute, context.apiScopePathPrefix, context.apiScopeMethod);
|
|
180
|
+
const latencyRouteLabel = formatRuntimeCapabilityRouteLabel(context.apiLatencyTopRoute, context.apiScopePathPrefix, context.apiScopeMethod);
|
|
181
|
+
const failOrWarnTargetSuffix = status === 'fail'
|
|
182
|
+
? 'before shipping the next release.'
|
|
183
|
+
: 'in the next stabilization pass.';
|
|
184
|
+
if (checkId === 'api_invalid_request_ratio' || checkId === 'api_invalid_request_hotspots') {
|
|
185
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
186
|
+
`Apply trace filter for ${invalidRouteLabel} with status>=400 and errorCode=invalid_request.`,
|
|
187
|
+
'Verify request normalization + required-field validation before any downstream call.',
|
|
188
|
+
`Backfill API contract tests for malformed payloads and boundary values ${failOrWarnTargetSuffix}`,
|
|
189
|
+
]);
|
|
190
|
+
}
|
|
191
|
+
if (checkId === 'api_server_error_ratio' || checkId === 'api_server_error_hotspots') {
|
|
192
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
193
|
+
`Apply trace filter for ${serverRouteLabel} with status>=500 and capture requestId-linked logs.`,
|
|
194
|
+
'Classify each 5xx root cause (validation gap, dependency failure, timeout, unknown) and patch the dominant bucket first.',
|
|
195
|
+
`Gate rollout on server/total <= ${context.thresholds.apiServerErrorWarnRatioPct}% with endpoint-level alerting.`,
|
|
196
|
+
]);
|
|
197
|
+
}
|
|
198
|
+
if (checkId === 'api_transient_error_ratio' || checkId === 'api_transient_error_hotspots') {
|
|
199
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
200
|
+
`Apply trace filter for ${transientRouteLabel} with status>=400 and focus on transient codes (408/425/429/502/503/504).`,
|
|
201
|
+
'Tune retry/backoff behavior against Retry-After and cap retry storms on hotspot endpoints.',
|
|
202
|
+
`Gate rollout on transient/total <= ${context.thresholds.apiTransientErrorWarnRatioPct}% and review provider timeout budgets.`,
|
|
203
|
+
]);
|
|
204
|
+
}
|
|
205
|
+
if (checkId === 'api_latency_p95' || checkId === 'api_latency_hotspots') {
|
|
206
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
207
|
+
`Apply trace filter for ${latencyRouteLabel} with status>=0 to keep successful slow requests visible.`,
|
|
208
|
+
`Current p95=${context.apiTraceP95DurationMs}ms (hotspot p95=${context.apiLatencyHotspotPeakP95Ms}ms); split heavy handlers and defer non-critical work.`,
|
|
209
|
+
`Gate rollout on p95 <= ${context.thresholds.apiLatencyP95WarnMs}ms and hotspot p95 <= ${context.thresholds.apiLatencyHotspotWarnMs}ms.`,
|
|
210
|
+
]);
|
|
211
|
+
}
|
|
212
|
+
if (checkId === 'query_backend_trend_config') {
|
|
213
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
214
|
+
'Align trend config so limit >= windowSize + minSamples and keep dual-window comparison valid.',
|
|
215
|
+
'Add trend-window sample budget checks in CI to prevent stale regression signals.',
|
|
216
|
+
'Refresh runtime capability matrix after updating trend config and compare confidence deltas.',
|
|
217
|
+
]);
|
|
218
|
+
}
|
|
219
|
+
if (checkId === 'query_backend_last_error') {
|
|
220
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
221
|
+
'Apply trace filter on /api/knowledge/query with status>=400 and capture requestId-linked payload diagnostics.',
|
|
222
|
+
'Compare local_hybrid / local_vector vs keyword_only query responses to isolate backend-specific regression surface.',
|
|
223
|
+
'Harden query fallback path and add regression tests around the latest backend error signature.',
|
|
224
|
+
]);
|
|
225
|
+
}
|
|
226
|
+
if (checkId === 'query_backend_runtime_health') {
|
|
227
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
228
|
+
'Inspect /api/knowledge/query-backend-diagnostics and verify runtime.ready plus backend id alignment.',
|
|
229
|
+
'If runtime backend is not ready, switch to local_hybrid via /api/knowledge/query-backend-config while investigating backend initialization failures.',
|
|
230
|
+
'Re-run a controlled /api/knowledge/query sample and confirm runtime diagnostics recover before re-enabling strict rollout gates.',
|
|
231
|
+
]);
|
|
232
|
+
}
|
|
233
|
+
if (checkId === 'store_graphdb_connector_health') {
|
|
234
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
235
|
+
'Inspect /api/knowledge/store-diagnostics and verify store.connector.healthStatus/healthMessage/circuitState.',
|
|
236
|
+
'Validate graphdb HTTP endpoint availability, then tune NOTE_CONNECTION_KNOWLEDGE_GRAPHDB_HTTP_TIMEOUT_MS / NOTE_CONNECTION_KNOWLEDGE_GRAPHDB_HTTP_MAX_RETRIES / NOTE_CONNECTION_KNOWLEDGE_GRAPHDB_HTTP_RETRY_DELAY_MS for current workload.',
|
|
237
|
+
'Run /api/knowledge/store/reload and confirm connector telemetry converges to health=ready, circuit=closed with stable request/retry counters.',
|
|
238
|
+
]);
|
|
239
|
+
}
|
|
240
|
+
if (checkId === 'store_graphdb_connector_budget') {
|
|
241
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
242
|
+
'Inspect /api/knowledge/store-diagnostics and verify requestCount/failureCount/shortCircuitCount/consecutiveFailures counters against rollout traffic.',
|
|
243
|
+
'Tune NOTE_CONNECTION_RUNTIME_STORE_GRAPHDB_CONNECTOR_* budget thresholds to match current load profile and alert sensitivity.',
|
|
244
|
+
'If failure/short-circuit budgets stay elevated, tune NOTE_CONNECTION_KNOWLEDGE_GRAPHDB_HTTP_TIMEOUT_MS / NOTE_CONNECTION_KNOWLEDGE_GRAPHDB_HTTP_MAX_RETRIES / NOTE_CONNECTION_KNOWLEDGE_GRAPHDB_HTTP_RETRY_DELAY_MS and circuit thresholds (NOTE_CONNECTION_KNOWLEDGE_GRAPHDB_HTTP_CIRCUIT_FAILURE_THRESHOLD / NOTE_CONNECTION_KNOWLEDGE_GRAPHDB_HTTP_CIRCUIT_COOLDOWN_MS).',
|
|
245
|
+
'Run /api/knowledge/store/reload and replay representative graphdb-backed query traffic before tightening strict rollout gates.',
|
|
246
|
+
]);
|
|
247
|
+
}
|
|
248
|
+
if (checkId === 'query_vector_index_status' || checkId === 'query_vector_index_persistence') {
|
|
249
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
250
|
+
'Inspect /api/knowledge/query-backend-diagnostics and verify diagnostics.runtime.vectorIndex.status moves to ready.',
|
|
251
|
+
'Trigger local_vector query traffic and confirm vector index atom count/signature update after ingest changes.',
|
|
252
|
+
'If persistence remains disabled, enable NOTE_CONNECTION_QUERY_VECTOR_INDEX_PERSIST and re-check diagnostics runtime fields.',
|
|
253
|
+
]);
|
|
254
|
+
}
|
|
255
|
+
if (checkId === 'query_vector_acceleration_mode') {
|
|
256
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
257
|
+
'Inspect /api/knowledge/query-backend-diagnostics and verify diagnostics.runtime.vectorIndex.acceleration fields are populated.',
|
|
258
|
+
'Issue representative /api/knowledge/query traffic and confirm ann_prefilter appears in trace.retrievalModes when acceleration is enabled.',
|
|
259
|
+
'If acceleration is intentionally disabled, set NOTE_CONNECTION_QUERY_VECTOR_ANN_PREFILTER=true before performance-sensitive validation runs.',
|
|
260
|
+
]);
|
|
261
|
+
}
|
|
262
|
+
if (checkId === 'query_vector_acceleration_representation_consistency') {
|
|
263
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
264
|
+
'Inspect /api/knowledge/query-backend-diagnostics and verify diagnostics.runtime.vectorIndex.acceleration representation metadata fields are populated (representationVersion, embeddingModelId, embeddingDimension, indexSignature).',
|
|
265
|
+
`Confirm diagnostics.runtime.vectorIndex.acceleration.representationStatus converges to aligned under representative query traffic (requestCount=${context.queryVectorIndexAccelerationRequestCount}).`,
|
|
266
|
+
'If representationStatus remains mismatch, align vector acceleration adapter embedding/model/index semantics with local index generation and re-run controlled /api/knowledge/query verification.',
|
|
267
|
+
'When rollout requires hard guarantees, enable NOTE_CONNECTION_QUERY_VECTOR_ACCELERATION_REPRESENTATION_STRICT=true and monitor mismatch failures via diagnostics/runtime runbook.',
|
|
268
|
+
]);
|
|
269
|
+
}
|
|
270
|
+
if (checkId === 'query_vector_acceleration_prefilter_effectiveness') {
|
|
271
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
272
|
+
'Inspect /api/knowledge/query-backend-diagnostics and confirm acceleration.lastSelectionMode plus acceleration.lastCandidateCount are populated under ANN prefilter traffic.',
|
|
273
|
+
`Issue representative /api/knowledge/query traffic and verify lastSelectionMode converges to token_prefilter|token_signature_prefilter (requestCount=${context.queryVectorIndexAccelerationRequestCount}).`,
|
|
274
|
+
`Compare lastCandidateCount against atomCount and keep candidateRatio below ${context.thresholds.queryVectorAccelerationPrefilterWarnCandidateRatioPct}% (hard fail at ${context.thresholds.queryVectorAccelerationPrefilterFailCandidateRatioPct}%) before tightening ANN rollout gates (atomCount=${context.queryVectorIndexAtomCount}, lastCandidateCount=${context.queryVectorIndexAccelerationLastCandidateCount}).`,
|
|
275
|
+
'Tune NOTE_CONNECTION_RUNTIME_QUERY_VECTOR_ACCELERATION_PREFILTER_MIN_REQUEST_SAMPLE / NOTE_CONNECTION_RUNTIME_QUERY_VECTOR_ACCELERATION_PREFILTER_WARN_CANDIDATE_RATIO_PCT / NOTE_CONNECTION_RUNTIME_QUERY_VECTOR_ACCELERATION_PREFILTER_FAIL_CANDIDATE_RATIO_PCT to match corpus shape and rollout phase.',
|
|
276
|
+
]);
|
|
277
|
+
}
|
|
278
|
+
if (checkId === 'query_vector_acceleration_calibration_readiness') {
|
|
279
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
280
|
+
`Calibration readiness depends on representative ANN telemetry: syncStatus=${context.queryVectorIndexAccelerationIndexSyncStatus}, requestCount=${context.queryVectorIndexAccelerationRequestCount}, selectionMode=${context.queryVectorIndexAccelerationLastSelectionMode}, candidateCount=${context.queryVectorIndexAccelerationLastCandidateCount}, circuitState=${context.queryVectorIndexAccelerationCircuitState}, healthStatus=${context.queryVectorIndexAccelerationHealthStatus}.`,
|
|
281
|
+
'Drive ingest + representative /api/knowledge/query traffic until index-sync telemetry, prefilter selection telemetry, and connector correlation fields are all populated for the same runtime window.',
|
|
282
|
+
`Do not start release-grade ANN threshold tuning until circuit budget status is ok and prefilter sample size reaches ${context.thresholds.queryVectorAccelerationPrefilterMinRequestSample} with evaluable candidate telemetry.`,
|
|
283
|
+
'Once readiness prerequisites are satisfied, tighten ANN workload thresholds and re-run runtime-capability-runbook verify/history/checks before promoting the gate.',
|
|
284
|
+
]);
|
|
285
|
+
}
|
|
286
|
+
if (checkId === 'query_vector_acceleration_health') {
|
|
287
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
288
|
+
'Inspect /api/knowledge/query-backend-diagnostics and verify diagnostics.runtime.vectorIndex.acceleration.healthStatus/healthMessage fields.',
|
|
289
|
+
'For external_http provider, validate NOTE_CONNECTION_QUERY_VECTOR_ACCELERATION_HTTP_ENDPOINT reachability and timeout budget.',
|
|
290
|
+
'Tune NOTE_CONNECTION_QUERY_VECTOR_ACCELERATION_HTTP_MAX_RETRIES / NOTE_CONNECTION_QUERY_VECTOR_ACCELERATION_HTTP_RETRY_DELAY_MS and circuit settings (NOTE_CONNECTION_QUERY_VECTOR_ACCELERATION_HTTP_CIRCUIT_FAILURE_THRESHOLD / NOTE_CONNECTION_QUERY_VECTOR_ACCELERATION_HTTP_CIRCUIT_COOLDOWN_MS).',
|
|
291
|
+
'Issue representative /api/knowledge/query traffic and confirm acceleration health recovers to ready|unknown while fallback ratio remains controlled.',
|
|
292
|
+
]);
|
|
293
|
+
}
|
|
294
|
+
if (checkId === 'query_vector_acceleration_index_sync_health') {
|
|
295
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
296
|
+
'Inspect /api/knowledge/query-backend-diagnostics and verify diagnostics.runtime.vectorIndex.acceleration.indexSyncStatus/indexSyncMessage/lastSyncAt fields.',
|
|
297
|
+
'Issue ingest + local_vector query traffic and confirm syncRequestCount/syncSuccessCount increment with syncedIndexSignature and syncedAtomCount populated.',
|
|
298
|
+
'If sync remains degraded/unavailable, validate /sync-index endpoint reachability and connector-side index refresh/persistence behavior before tightening ANN rollout gates.',
|
|
299
|
+
]);
|
|
300
|
+
}
|
|
301
|
+
if (checkId === 'query_vector_acceleration_traceability') {
|
|
302
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
303
|
+
'Inspect /api/knowledge/query-backend-diagnostics and verify diagnostics.runtime.vectorIndex.acceleration.lastRequestId/lastErrorCode/lastRetryAfterMs are populated for external connectors.',
|
|
304
|
+
'Issue representative /api/knowledge/query traffic while connector is degraded/open and confirm correlation fields are emitted for incident drilldown.',
|
|
305
|
+
'If correlation fields remain empty, verify connector response headers/body mapping and keep fallback/circuit telemetry enabled before stricter rollout gates.',
|
|
306
|
+
]);
|
|
307
|
+
}
|
|
308
|
+
if (checkId === 'query_vector_acceleration_circuit_state') {
|
|
309
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
310
|
+
'Inspect /api/knowledge/query-backend-diagnostics and verify diagnostics.runtime.vectorIndex.acceleration.circuitState plus retry/shortCircuit counters.',
|
|
311
|
+
'If circuitState=open, verify connector availability first, then tune NOTE_CONNECTION_QUERY_VECTOR_ACCELERATION_HTTP_CIRCUIT_FAILURE_THRESHOLD / NOTE_CONNECTION_QUERY_VECTOR_ACCELERATION_HTTP_CIRCUIT_COOLDOWN_MS for workload volatility.',
|
|
312
|
+
'Tune runtime governance thresholds with NOTE_CONNECTION_RUNTIME_QUERY_VECTOR_ACCELERATION_SHORT_CIRCUIT_* / NOTE_CONNECTION_RUNTIME_QUERY_VECTOR_ACCELERATION_CONSECUTIVE_FAILURES_* / NOTE_CONNECTION_RUNTIME_QUERY_VECTOR_ACCELERATION_HALF_OPEN_SUCCESS_* env keys.',
|
|
313
|
+
'Track half-open probe outcomes and keep halfOpenSuccessRate above configured warn/fail floors before tightening retry thresholds.',
|
|
314
|
+
]);
|
|
315
|
+
}
|
|
316
|
+
if (checkId === 'quality_trend_direction') {
|
|
317
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
318
|
+
`Trend status=${context.qualityTrendStatus}, confidence=${context.qualityTrendConfidence}; run targeted retest sessions on low-mastery atoms first.`,
|
|
319
|
+
'Review recent misconception tags and align next session plan toward top recurring error families.',
|
|
320
|
+
'Gate release on stable/improving quality trend for at least one complete trend window.',
|
|
321
|
+
]);
|
|
322
|
+
}
|
|
323
|
+
if (checkId === 'session_plan_quality_gate') {
|
|
324
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
325
|
+
`Current failureStreak=${context.sessionPlanQualityFailureStreak}, passRate=${context.sessionPlanQualityPassRatePct}%; prioritize fixing the most frequent failed gates.`,
|
|
326
|
+
'Re-run plan quality evaluation after threshold tuning and ensure evidence/coverage gates pass together.',
|
|
327
|
+
`Keep failureStreak below ${context.thresholds.sessionPlanQualityWarnFailureStreak} before enabling aggressive orchestration modes.`,
|
|
328
|
+
]);
|
|
329
|
+
}
|
|
330
|
+
if (checkId === 'session_plan_quality_trend') {
|
|
331
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
332
|
+
`Session plan trend is ${context.sessionPlanTrendStatus}; compare last two windows for gate-level drift.`,
|
|
333
|
+
'Promote stable plan templates from successful sessions and demote low-quality templates.',
|
|
334
|
+
'Run controlled A/B on plan strategy (balanced vs mastery_recovery) and retain the higher pass-rate policy.',
|
|
335
|
+
]);
|
|
336
|
+
}
|
|
337
|
+
if (checkId === 'orchestration_path_strategy_alignment') {
|
|
338
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
339
|
+
`Strategy alignment telemetry: strategyRecords=${context.sessionStrategyStrategyRecords}, trendAutoShare=${context.sessionStrategyTrendAutoSelectionSharePct}%, trendAutoAvgDelta=${context.sessionStrategyTrendAutoAverageMasteryDeltaPct}%, trendAutoNegative=${context.sessionStrategyTrendAutoNegativeRatioPct}%, topAverage=${context.sessionStrategyTopAverageStrategy}@${context.sessionStrategyTopAverageMasteryDeltaPct}%.`,
|
|
340
|
+
'Inspect /api/knowledge/session/history to compare strategy_trend decisions against explicit_request and mode_fallback outcomes.',
|
|
341
|
+
'If trend-auto outcomes remain negative, tighten strategy auto-selection confidence gate and re-run quality/session-plan trends before rollout.',
|
|
342
|
+
]);
|
|
343
|
+
}
|
|
344
|
+
if (checkId === 'memory_policy_health' || checkId === 'memory_policy_trend') {
|
|
345
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
346
|
+
`Memory status=${context.memoryPolicyStatus}, healthScore=${context.memoryPolicyHealthScore}; prioritize cleanup of expired/stale/low-confidence entries.`,
|
|
347
|
+
'Run memory diagnostics + retraining cycle, then verify trend status improves on the next window.',
|
|
348
|
+
'Enforce memory write/read TTL and confidence floors to prevent degraded memories from re-entering active planning.',
|
|
349
|
+
]);
|
|
350
|
+
}
|
|
351
|
+
if (checkId === 'knowledge_staleness_data' || checkId === 'knowledge_staleness_health') {
|
|
352
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
353
|
+
`Staleness snapshot: evaluated=${context.knowledgeStalenessEvaluatedDocuments}, stale=${context.knowledgeStalenessStaleDocuments}, freshness=${context.knowledgeStalenessFreshnessRatioPct}%.`,
|
|
354
|
+
`Run incremental rebuild for stale/hash-mismatch/missing docs (mismatch=${context.knowledgeStalenessHashMismatchDocuments}, missing=${context.knowledgeStalenessMissingSourceDocuments}, readError=${context.knowledgeStalenessReadErrorDocuments}).`,
|
|
355
|
+
'Block learning-path updates when source read errors persist, and recover source availability first.',
|
|
356
|
+
]);
|
|
357
|
+
}
|
|
358
|
+
if (checkId === 'session_memory_promotion_coverage') {
|
|
359
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
360
|
+
`Session memory promotion coverage is ${context.sessionMemoryPromotionCoveragePct}%; raise promotion for high-value session outcomes.`,
|
|
361
|
+
'Audit memory write policy so verified tutor actions and high-confidence mastery updates are persistable.',
|
|
362
|
+
'Re-run one controlled session and verify promotedActions plus promotedEntries both increase.',
|
|
363
|
+
]);
|
|
364
|
+
}
|
|
365
|
+
if (checkId === 'tutor_adapter_inventory') {
|
|
366
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
367
|
+
`Adapter inventory=${context.tutorAdaptersActive}/${context.tutorAdaptersTotal}, requests=${context.tutorRequests}; ensure at least one healthy adapter remains active.`,
|
|
368
|
+
'Validate adapter registry and provider bindings, then probe each adapter endpoint before rerouting traffic.',
|
|
369
|
+
'Fallback to deterministic local tutor mode if adapter inventory remains unstable.',
|
|
370
|
+
]);
|
|
371
|
+
}
|
|
372
|
+
if (checkId === 'tutor_adapter_timeout_budget') {
|
|
373
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
374
|
+
`Current adapterTimeoutMs=${context.tutorRoutingAdapterTimeoutMs}; keep timeout within governance budget.`,
|
|
375
|
+
'Tune adapter timeout against observed p95 adapter latency and provider-specific response behavior.',
|
|
376
|
+
'Align timeout policy with retry limits to avoid cascading queue buildup under load.',
|
|
377
|
+
]);
|
|
378
|
+
}
|
|
379
|
+
if (checkId === 'tutor_routing_health_budget') {
|
|
380
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
381
|
+
`Routing enabled=${context.tutorRoutingEnabled}, requests=${context.tutorRequests}/${context.tutorRoutingMinSamples}, fail=${context.tutorFailedRatioPct}%, downgraded=${context.tutorDowngradedRatioPct}%, fallback=${context.tutorProviderFallbackRatioPct}%, avgAttempts=${context.tutorAverageProviderAttemptCount}, conf=${context.tutorAverageConfidence}, dynamicMode=${context.tutorRoutingDynamicPreferredMode}${context.tutorRoutingDynamicModeReason ? `(${context.tutorRoutingDynamicModeReason})` : ''}.`,
|
|
382
|
+
`Keep fail<=${context.tutorRoutingMaxFailedRatioPct}% and downgraded<=${context.tutorRoutingMaxDowngradedRatioPct}% while fallback stays controlled and avgConfidence>=${context.tutorRoutingMinAverageConfidence}.`,
|
|
383
|
+
'If budgets keep failing, temporarily pin preferred mode (local or cloud) and resume adaptive routing after stabilization.',
|
|
384
|
+
]);
|
|
385
|
+
}
|
|
386
|
+
if (checkId === 'tutor_routing_traceability') {
|
|
387
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
388
|
+
`Last routing strategy=${context.tutorLastRoutingStrategy || 'unknown'}, dynamicMode=${context.tutorRoutingDynamicPreferredMode}, dynamicSuggested=${context.tutorRoutingDynamicModeSuggestionActive}; require explicit reason + score on each routed tutor response.`,
|
|
389
|
+
'Treat repeated fallback_default routing as degraded mode and trigger adapter inventory + health checks.',
|
|
390
|
+
'Persist routing traces for postmortem comparison between accepted and downgraded tutor responses.',
|
|
391
|
+
]);
|
|
392
|
+
}
|
|
393
|
+
if (checkId === 'tutor_routing_dynamic_mode_alignment') {
|
|
394
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
395
|
+
`Routing preference=${context.tutorRoutingPreferredMode}, dynamicMode=${context.tutorRoutingDynamicPreferredMode}, dynamicSuggested=${context.tutorRoutingDynamicModeSuggestionActive}${context.tutorRoutingDynamicModeReason ? `, reason=${context.tutorRoutingDynamicModeReason}` : ''}.`,
|
|
396
|
+
'When dynamic suggestion remains stable, switch preferredMode to auto (or to the suggested mode) and re-measure failure/fallback budgets.',
|
|
397
|
+
'Re-run tutor trace diagnostics history to verify mode conflict and trend penalty signals converge.',
|
|
398
|
+
]);
|
|
399
|
+
}
|
|
400
|
+
if (checkId === 'tutor_provider_concentration') {
|
|
401
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
402
|
+
`Provider concentration: providers=${context.tutorProviderCount}, dominant=${context.tutorDominantProviderName || 'unknown'}(${context.tutorDominantProviderSharePct}%), fallbackDominant=${context.tutorDominantFallbackProviderName || 'none'}(${context.tutorDominantFallbackProviderSharePct}%).`,
|
|
403
|
+
'Distribute fallback routing across at least two healthy providers and re-check concentration after stabilization.',
|
|
404
|
+
'If a single provider dominates fallback traffic, probe alternate providers and lower their warmup threshold to avoid SPOF.',
|
|
405
|
+
]);
|
|
406
|
+
}
|
|
407
|
+
if (checkId === 'tutor_provider_trend_regression') {
|
|
408
|
+
return normalizeRuntimeCapabilityRecommendedActions([
|
|
409
|
+
`Provider trend: regressing=${context.tutorProviderTrendRegressingCount}, improving=${context.tutorProviderTrendImprovingCount}, insufficient=${context.tutorProviderTrendInsufficientDataCount}, history(regressing=${context.tutorProviderTrendHistoryRegressingRecords}/${context.tutorProviderTrendHistoryRecords},stable=${context.tutorProviderTrendHistoryStableRecords},improving=${context.tutorProviderTrendHistoryImprovingRecords},insufficient=${context.tutorProviderTrendHistoryInsufficientDataRecords}), topRegressing=${context.tutorProviderTrendTopRegressingProvider || 'none'}@${context.tutorProviderTrendTopRegressingScore.toFixed(2)}(${context.tutorProviderTrendTopRegressingConfidence.toFixed(2)}).`,
|
|
410
|
+
`Focus provider=${context.tutorProviderTrendRecommendedFocusProviderName || context.tutorProviderTrendTopRegressingProvider || 'n/a'} and validate fallback/failure deltas against the previous window.`,
|
|
411
|
+
'Pin degraded provider to reduced traffic, run targeted probe sessions, then reopen adaptive routing after trend recovers.',
|
|
412
|
+
]);
|
|
413
|
+
}
|
|
414
|
+
return undefined;
|
|
415
|
+
}
|
|
416
|
+
function extractRuntimeCapabilityHotspotPeakFromObserved(observedRaw) {
|
|
417
|
+
const observed = String(observedRaw || '');
|
|
418
|
+
if (!observed) {
|
|
419
|
+
return 0;
|
|
420
|
+
}
|
|
421
|
+
const tokens = observed.match(/:(\d+)(?=\||$)/g) || [];
|
|
422
|
+
let maxCount = 0;
|
|
423
|
+
tokens.forEach((token) => {
|
|
424
|
+
const numeric = Number(String(token || '').replace(':', ''));
|
|
425
|
+
if (Number.isFinite(numeric) && numeric > maxCount) {
|
|
426
|
+
maxCount = numeric;
|
|
427
|
+
}
|
|
428
|
+
});
|
|
429
|
+
return maxCount;
|
|
430
|
+
}
|
|
431
|
+
function computeRuntimeCapabilityPriorityScore(check) {
|
|
432
|
+
const status = String(check?.status || 'warn').trim().toLowerCase();
|
|
433
|
+
const checkId = String(check?.checkId || '').trim().toLowerCase();
|
|
434
|
+
const hotspotPeakCount = extractRuntimeCapabilityHotspotPeakFromObserved(check?.observed);
|
|
435
|
+
let score = status === 'fail'
|
|
436
|
+
? 300
|
|
437
|
+
: (status === 'warn' ? 200 : 100);
|
|
438
|
+
if (checkId.includes('hotspots')) {
|
|
439
|
+
score += 40;
|
|
440
|
+
}
|
|
441
|
+
if (checkId.includes('server_error')) {
|
|
442
|
+
score += 20;
|
|
443
|
+
}
|
|
444
|
+
if (checkId.includes('invalid_request')) {
|
|
445
|
+
score += 15;
|
|
446
|
+
}
|
|
447
|
+
if (checkId.includes('ratio')) {
|
|
448
|
+
score += 5;
|
|
449
|
+
}
|
|
450
|
+
score += Math.min(20, Math.max(0, Math.floor(Number(hotspotPeakCount || 0))));
|
|
451
|
+
return Math.max(1, Math.floor(score));
|
|
452
|
+
}
|
|
453
|
+
function sortRuntimeCapabilityChecksByPriority(checks) {
|
|
454
|
+
return checks.slice().sort((left, right) => {
|
|
455
|
+
const leftScore = Number(left?.priorityScore || 0);
|
|
456
|
+
const rightScore = Number(right?.priorityScore || 0);
|
|
457
|
+
if (rightScore !== leftScore) {
|
|
458
|
+
return rightScore - leftScore;
|
|
459
|
+
}
|
|
460
|
+
const leftId = String(left?.checkId || '').trim().toLowerCase();
|
|
461
|
+
const rightId = String(right?.checkId || '').trim().toLowerCase();
|
|
462
|
+
return leftId.localeCompare(rightId);
|
|
463
|
+
});
|
|
464
|
+
}
|
|
465
|
+
function normalizeRuntimeCapabilityThresholds(input = {}) {
|
|
466
|
+
const minQuerySampleSize = normalizeInteger(input.minQuerySampleSize, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.minQuerySampleSize, 1, 100000);
|
|
467
|
+
const queryFallbackWarnRatioPct = normalizeRatio(input.queryFallbackWarnRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryFallbackWarnRatioPct);
|
|
468
|
+
let queryFallbackFailRatioPct = normalizeRatio(input.queryFallbackFailRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryFallbackFailRatioPct);
|
|
469
|
+
if (queryFallbackFailRatioPct < queryFallbackWarnRatioPct) {
|
|
470
|
+
queryFallbackFailRatioPct = queryFallbackWarnRatioPct;
|
|
471
|
+
}
|
|
472
|
+
const queryEvidenceCoverageWarnRatioPct = normalizeRatio(input.queryEvidenceCoverageWarnRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryEvidenceCoverageWarnRatioPct);
|
|
473
|
+
let queryEvidenceCoverageFailRatioPct = normalizeRatio(input.queryEvidenceCoverageFailRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryEvidenceCoverageFailRatioPct);
|
|
474
|
+
if (queryEvidenceCoverageFailRatioPct > queryEvidenceCoverageWarnRatioPct) {
|
|
475
|
+
queryEvidenceCoverageFailRatioPct = queryEvidenceCoverageWarnRatioPct;
|
|
476
|
+
}
|
|
477
|
+
const queryTemporalValidityWarnRatioPct = normalizeRatio(input.queryTemporalValidityWarnRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryTemporalValidityWarnRatioPct);
|
|
478
|
+
let queryTemporalValidityFailRatioPct = normalizeRatio(input.queryTemporalValidityFailRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryTemporalValidityFailRatioPct);
|
|
479
|
+
if (queryTemporalValidityFailRatioPct > queryTemporalValidityWarnRatioPct) {
|
|
480
|
+
queryTemporalValidityFailRatioPct = queryTemporalValidityWarnRatioPct;
|
|
481
|
+
}
|
|
482
|
+
const queryBackendExplainabilityGapWarnRatioPct = normalizeRatio(input.queryBackendExplainabilityGapWarnRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryBackendExplainabilityGapWarnRatioPct);
|
|
483
|
+
let queryBackendExplainabilityGapFailRatioPct = normalizeRatio(input.queryBackendExplainabilityGapFailRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryBackendExplainabilityGapFailRatioPct);
|
|
484
|
+
if (queryBackendExplainabilityGapFailRatioPct < queryBackendExplainabilityGapWarnRatioPct) {
|
|
485
|
+
queryBackendExplainabilityGapFailRatioPct = queryBackendExplainabilityGapWarnRatioPct;
|
|
486
|
+
}
|
|
487
|
+
const queryBackendTrendWarnConfidenceRatioPct = normalizeRatio(input.queryBackendTrendWarnConfidenceRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryBackendTrendWarnConfidenceRatioPct);
|
|
488
|
+
let queryBackendTrendFailConfidenceRatioPct = normalizeRatio(input.queryBackendTrendFailConfidenceRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryBackendTrendFailConfidenceRatioPct);
|
|
489
|
+
if (queryBackendTrendFailConfidenceRatioPct < queryBackendTrendWarnConfidenceRatioPct) {
|
|
490
|
+
queryBackendTrendFailConfidenceRatioPct = queryBackendTrendWarnConfidenceRatioPct;
|
|
491
|
+
}
|
|
492
|
+
const sessionPlanQualityWarnFailureStreak = normalizeInteger(input.sessionPlanQualityWarnFailureStreak, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.sessionPlanQualityWarnFailureStreak, 1, 100);
|
|
493
|
+
let sessionPlanQualityFailFailureStreak = normalizeInteger(input.sessionPlanQualityFailFailureStreak, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.sessionPlanQualityFailFailureStreak, 1, 100);
|
|
494
|
+
if (sessionPlanQualityFailFailureStreak < sessionPlanQualityWarnFailureStreak) {
|
|
495
|
+
sessionPlanQualityFailFailureStreak = sessionPlanQualityWarnFailureStreak;
|
|
496
|
+
}
|
|
497
|
+
const apiInvalidRequestMinErrorSample = normalizeInteger(input.apiInvalidRequestMinErrorSample, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiInvalidRequestMinErrorSample, 1, 100000);
|
|
498
|
+
const apiInvalidRequestWarnRatioPct = normalizeRatio(input.apiInvalidRequestWarnRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiInvalidRequestWarnRatioPct);
|
|
499
|
+
let apiInvalidRequestFailRatioPct = normalizeRatio(input.apiInvalidRequestFailRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiInvalidRequestFailRatioPct);
|
|
500
|
+
if (apiInvalidRequestFailRatioPct < apiInvalidRequestWarnRatioPct) {
|
|
501
|
+
apiInvalidRequestFailRatioPct = apiInvalidRequestWarnRatioPct;
|
|
502
|
+
}
|
|
503
|
+
const apiInvalidRequestHotspotWarnCount = normalizeInteger(input.apiInvalidRequestHotspotWarnCount, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiInvalidRequestHotspotWarnCount, 1, 100000);
|
|
504
|
+
let apiInvalidRequestHotspotFailCount = normalizeInteger(input.apiInvalidRequestHotspotFailCount, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiInvalidRequestHotspotFailCount, 1, 100000);
|
|
505
|
+
if (apiInvalidRequestHotspotFailCount < apiInvalidRequestHotspotWarnCount) {
|
|
506
|
+
apiInvalidRequestHotspotFailCount = apiInvalidRequestHotspotWarnCount;
|
|
507
|
+
}
|
|
508
|
+
const apiServerErrorMinRequestSample = normalizeInteger(input.apiServerErrorMinRequestSample, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiServerErrorMinRequestSample, 1, 100000);
|
|
509
|
+
const apiServerErrorWarnRatioPct = normalizeRatio(input.apiServerErrorWarnRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiServerErrorWarnRatioPct);
|
|
510
|
+
let apiServerErrorFailRatioPct = normalizeRatio(input.apiServerErrorFailRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiServerErrorFailRatioPct);
|
|
511
|
+
if (apiServerErrorFailRatioPct < apiServerErrorWarnRatioPct) {
|
|
512
|
+
apiServerErrorFailRatioPct = apiServerErrorWarnRatioPct;
|
|
513
|
+
}
|
|
514
|
+
const apiServerErrorHotspotWarnCount = normalizeInteger(input.apiServerErrorHotspotWarnCount, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiServerErrorHotspotWarnCount, 1, 100000);
|
|
515
|
+
let apiServerErrorHotspotFailCount = normalizeInteger(input.apiServerErrorHotspotFailCount, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiServerErrorHotspotFailCount, 1, 100000);
|
|
516
|
+
if (apiServerErrorHotspotFailCount < apiServerErrorHotspotWarnCount) {
|
|
517
|
+
apiServerErrorHotspotFailCount = apiServerErrorHotspotWarnCount;
|
|
518
|
+
}
|
|
519
|
+
const apiTransientErrorMinRequestSample = normalizeInteger(input.apiTransientErrorMinRequestSample, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiTransientErrorMinRequestSample, 1, 100000);
|
|
520
|
+
const apiTransientErrorWarnRatioPct = normalizeRatio(input.apiTransientErrorWarnRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiTransientErrorWarnRatioPct);
|
|
521
|
+
let apiTransientErrorFailRatioPct = normalizeRatio(input.apiTransientErrorFailRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiTransientErrorFailRatioPct);
|
|
522
|
+
if (apiTransientErrorFailRatioPct < apiTransientErrorWarnRatioPct) {
|
|
523
|
+
apiTransientErrorFailRatioPct = apiTransientErrorWarnRatioPct;
|
|
524
|
+
}
|
|
525
|
+
const apiTransientErrorHotspotWarnCount = normalizeInteger(input.apiTransientErrorHotspotWarnCount, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiTransientErrorHotspotWarnCount, 1, 100000);
|
|
526
|
+
let apiTransientErrorHotspotFailCount = normalizeInteger(input.apiTransientErrorHotspotFailCount, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiTransientErrorHotspotFailCount, 1, 100000);
|
|
527
|
+
if (apiTransientErrorHotspotFailCount < apiTransientErrorHotspotWarnCount) {
|
|
528
|
+
apiTransientErrorHotspotFailCount = apiTransientErrorHotspotWarnCount;
|
|
529
|
+
}
|
|
530
|
+
const apiLatencyMinRequestSample = normalizeInteger(input.apiLatencyMinRequestSample, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiLatencyMinRequestSample, 1, 100000);
|
|
531
|
+
const apiLatencyP95WarnMs = normalizeInteger(input.apiLatencyP95WarnMs, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiLatencyP95WarnMs, 1, 600000);
|
|
532
|
+
let apiLatencyP95FailMs = normalizeInteger(input.apiLatencyP95FailMs, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiLatencyP95FailMs, 1, 600000);
|
|
533
|
+
if (apiLatencyP95FailMs < apiLatencyP95WarnMs) {
|
|
534
|
+
apiLatencyP95FailMs = apiLatencyP95WarnMs;
|
|
535
|
+
}
|
|
536
|
+
const apiLatencyHotspotWarnMs = normalizeInteger(input.apiLatencyHotspotWarnMs, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiLatencyHotspotWarnMs, 1, 600000);
|
|
537
|
+
let apiLatencyHotspotFailMs = normalizeInteger(input.apiLatencyHotspotFailMs, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.apiLatencyHotspotFailMs, 1, 600000);
|
|
538
|
+
if (apiLatencyHotspotFailMs < apiLatencyHotspotWarnMs) {
|
|
539
|
+
apiLatencyHotspotFailMs = apiLatencyHotspotWarnMs;
|
|
540
|
+
}
|
|
541
|
+
const queryVectorAccelerationShortCircuitWarnCount = normalizeInteger(input.queryVectorAccelerationShortCircuitWarnCount, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryVectorAccelerationShortCircuitWarnCount, 0, 100000);
|
|
542
|
+
let queryVectorAccelerationShortCircuitFailCount = normalizeInteger(input.queryVectorAccelerationShortCircuitFailCount, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryVectorAccelerationShortCircuitFailCount, 0, 100000);
|
|
543
|
+
if (queryVectorAccelerationShortCircuitFailCount < queryVectorAccelerationShortCircuitWarnCount) {
|
|
544
|
+
queryVectorAccelerationShortCircuitFailCount = queryVectorAccelerationShortCircuitWarnCount;
|
|
545
|
+
}
|
|
546
|
+
const queryVectorAccelerationShortCircuitWarnRatioPct = normalizeRatio(input.queryVectorAccelerationShortCircuitWarnRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryVectorAccelerationShortCircuitWarnRatioPct);
|
|
547
|
+
let queryVectorAccelerationShortCircuitFailRatioPct = normalizeRatio(input.queryVectorAccelerationShortCircuitFailRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryVectorAccelerationShortCircuitFailRatioPct);
|
|
548
|
+
if (queryVectorAccelerationShortCircuitFailRatioPct < queryVectorAccelerationShortCircuitWarnRatioPct) {
|
|
549
|
+
queryVectorAccelerationShortCircuitFailRatioPct = queryVectorAccelerationShortCircuitWarnRatioPct;
|
|
550
|
+
}
|
|
551
|
+
const queryVectorAccelerationConsecutiveFailuresWarnCount = normalizeInteger(input.queryVectorAccelerationConsecutiveFailuresWarnCount, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryVectorAccelerationConsecutiveFailuresWarnCount, 0, 100000);
|
|
552
|
+
let queryVectorAccelerationConsecutiveFailuresFailCount = normalizeInteger(input.queryVectorAccelerationConsecutiveFailuresFailCount, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryVectorAccelerationConsecutiveFailuresFailCount, 0, 100000);
|
|
553
|
+
if (queryVectorAccelerationConsecutiveFailuresFailCount < queryVectorAccelerationConsecutiveFailuresWarnCount) {
|
|
554
|
+
queryVectorAccelerationConsecutiveFailuresFailCount = queryVectorAccelerationConsecutiveFailuresWarnCount;
|
|
555
|
+
}
|
|
556
|
+
const queryVectorAccelerationHalfOpenSuccessWarnRatioPct = normalizeRatio(input.queryVectorAccelerationHalfOpenSuccessWarnRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryVectorAccelerationHalfOpenSuccessWarnRatioPct);
|
|
557
|
+
let queryVectorAccelerationHalfOpenSuccessFailRatioPct = normalizeRatio(input.queryVectorAccelerationHalfOpenSuccessFailRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryVectorAccelerationHalfOpenSuccessFailRatioPct);
|
|
558
|
+
if (queryVectorAccelerationHalfOpenSuccessFailRatioPct > queryVectorAccelerationHalfOpenSuccessWarnRatioPct) {
|
|
559
|
+
queryVectorAccelerationHalfOpenSuccessFailRatioPct = queryVectorAccelerationHalfOpenSuccessWarnRatioPct;
|
|
560
|
+
}
|
|
561
|
+
const queryVectorAccelerationPrefilterMinRequestSample = normalizeInteger(input.queryVectorAccelerationPrefilterMinRequestSample, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryVectorAccelerationPrefilterMinRequestSample, 1, 100000);
|
|
562
|
+
const queryVectorAccelerationPrefilterWarnCandidateRatioPct = normalizeRatio(input.queryVectorAccelerationPrefilterWarnCandidateRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryVectorAccelerationPrefilterWarnCandidateRatioPct);
|
|
563
|
+
let queryVectorAccelerationPrefilterFailCandidateRatioPct = normalizeRatio(input.queryVectorAccelerationPrefilterFailCandidateRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.queryVectorAccelerationPrefilterFailCandidateRatioPct);
|
|
564
|
+
if (queryVectorAccelerationPrefilterFailCandidateRatioPct < queryVectorAccelerationPrefilterWarnCandidateRatioPct) {
|
|
565
|
+
queryVectorAccelerationPrefilterFailCandidateRatioPct = queryVectorAccelerationPrefilterWarnCandidateRatioPct;
|
|
566
|
+
}
|
|
567
|
+
const storeGraphDbConnectorMinRequestSample = normalizeInteger(input.storeGraphDbConnectorMinRequestSample, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.storeGraphDbConnectorMinRequestSample, 1, 100000);
|
|
568
|
+
const storeGraphDbConnectorFailureWarnRatioPct = normalizeRatio(input.storeGraphDbConnectorFailureWarnRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.storeGraphDbConnectorFailureWarnRatioPct);
|
|
569
|
+
let storeGraphDbConnectorFailureFailRatioPct = normalizeRatio(input.storeGraphDbConnectorFailureFailRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.storeGraphDbConnectorFailureFailRatioPct);
|
|
570
|
+
if (storeGraphDbConnectorFailureFailRatioPct < storeGraphDbConnectorFailureWarnRatioPct) {
|
|
571
|
+
storeGraphDbConnectorFailureFailRatioPct = storeGraphDbConnectorFailureWarnRatioPct;
|
|
572
|
+
}
|
|
573
|
+
const storeGraphDbConnectorShortCircuitWarnRatioPct = normalizeRatio(input.storeGraphDbConnectorShortCircuitWarnRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.storeGraphDbConnectorShortCircuitWarnRatioPct);
|
|
574
|
+
let storeGraphDbConnectorShortCircuitFailRatioPct = normalizeRatio(input.storeGraphDbConnectorShortCircuitFailRatioPct, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.storeGraphDbConnectorShortCircuitFailRatioPct);
|
|
575
|
+
if (storeGraphDbConnectorShortCircuitFailRatioPct < storeGraphDbConnectorShortCircuitWarnRatioPct) {
|
|
576
|
+
storeGraphDbConnectorShortCircuitFailRatioPct = storeGraphDbConnectorShortCircuitWarnRatioPct;
|
|
577
|
+
}
|
|
578
|
+
const storeGraphDbConnectorConsecutiveFailuresWarnCount = normalizeInteger(input.storeGraphDbConnectorConsecutiveFailuresWarnCount, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.storeGraphDbConnectorConsecutiveFailuresWarnCount, 0, 100000);
|
|
579
|
+
let storeGraphDbConnectorConsecutiveFailuresFailCount = normalizeInteger(input.storeGraphDbConnectorConsecutiveFailuresFailCount, exports.DEFAULT_RUNTIME_CAPABILITY_THRESHOLDS.storeGraphDbConnectorConsecutiveFailuresFailCount, 0, 100000);
|
|
580
|
+
if (storeGraphDbConnectorConsecutiveFailuresFailCount < storeGraphDbConnectorConsecutiveFailuresWarnCount) {
|
|
581
|
+
storeGraphDbConnectorConsecutiveFailuresFailCount = storeGraphDbConnectorConsecutiveFailuresWarnCount;
|
|
582
|
+
}
|
|
583
|
+
return {
|
|
584
|
+
minQuerySampleSize,
|
|
585
|
+
queryFallbackWarnRatioPct,
|
|
586
|
+
queryFallbackFailRatioPct,
|
|
587
|
+
queryEvidenceCoverageWarnRatioPct,
|
|
588
|
+
queryEvidenceCoverageFailRatioPct,
|
|
589
|
+
queryTemporalValidityWarnRatioPct,
|
|
590
|
+
queryTemporalValidityFailRatioPct,
|
|
591
|
+
queryBackendExplainabilityGapWarnRatioPct,
|
|
592
|
+
queryBackendExplainabilityGapFailRatioPct,
|
|
593
|
+
queryBackendTrendWarnConfidenceRatioPct,
|
|
594
|
+
queryBackendTrendFailConfidenceRatioPct,
|
|
595
|
+
sessionPlanQualityWarnFailureStreak,
|
|
596
|
+
sessionPlanQualityFailFailureStreak,
|
|
597
|
+
apiInvalidRequestMinErrorSample,
|
|
598
|
+
apiInvalidRequestWarnRatioPct,
|
|
599
|
+
apiInvalidRequestFailRatioPct,
|
|
600
|
+
apiInvalidRequestHotspotWarnCount,
|
|
601
|
+
apiInvalidRequestHotspotFailCount,
|
|
602
|
+
apiServerErrorMinRequestSample,
|
|
603
|
+
apiServerErrorWarnRatioPct,
|
|
604
|
+
apiServerErrorFailRatioPct,
|
|
605
|
+
apiServerErrorHotspotWarnCount,
|
|
606
|
+
apiServerErrorHotspotFailCount,
|
|
607
|
+
apiTransientErrorMinRequestSample,
|
|
608
|
+
apiTransientErrorWarnRatioPct,
|
|
609
|
+
apiTransientErrorFailRatioPct,
|
|
610
|
+
apiTransientErrorHotspotWarnCount,
|
|
611
|
+
apiTransientErrorHotspotFailCount,
|
|
612
|
+
apiLatencyMinRequestSample,
|
|
613
|
+
apiLatencyP95WarnMs,
|
|
614
|
+
apiLatencyP95FailMs,
|
|
615
|
+
apiLatencyHotspotWarnMs,
|
|
616
|
+
apiLatencyHotspotFailMs,
|
|
617
|
+
queryVectorAccelerationShortCircuitWarnCount,
|
|
618
|
+
queryVectorAccelerationShortCircuitFailCount,
|
|
619
|
+
queryVectorAccelerationShortCircuitWarnRatioPct,
|
|
620
|
+
queryVectorAccelerationShortCircuitFailRatioPct,
|
|
621
|
+
queryVectorAccelerationConsecutiveFailuresWarnCount,
|
|
622
|
+
queryVectorAccelerationConsecutiveFailuresFailCount,
|
|
623
|
+
queryVectorAccelerationHalfOpenSuccessWarnRatioPct,
|
|
624
|
+
queryVectorAccelerationHalfOpenSuccessFailRatioPct,
|
|
625
|
+
queryVectorAccelerationPrefilterMinRequestSample,
|
|
626
|
+
queryVectorAccelerationPrefilterWarnCandidateRatioPct,
|
|
627
|
+
queryVectorAccelerationPrefilterFailCandidateRatioPct,
|
|
628
|
+
storeGraphDbConnectorMinRequestSample,
|
|
629
|
+
storeGraphDbConnectorFailureWarnRatioPct,
|
|
630
|
+
storeGraphDbConnectorFailureFailRatioPct,
|
|
631
|
+
storeGraphDbConnectorShortCircuitWarnRatioPct,
|
|
632
|
+
storeGraphDbConnectorShortCircuitFailRatioPct,
|
|
633
|
+
storeGraphDbConnectorConsecutiveFailuresWarnCount,
|
|
634
|
+
storeGraphDbConnectorConsecutiveFailuresFailCount,
|
|
635
|
+
};
|
|
636
|
+
}
|
|
637
|
+
function resolveRuntimeCapabilityThresholdsFromEnv(env) {
|
|
638
|
+
return normalizeRuntimeCapabilityThresholds({
|
|
639
|
+
minQuerySampleSize: env.NOTE_CONNECTION_RUNTIME_QUERY_MIN_SAMPLE,
|
|
640
|
+
queryFallbackWarnRatioPct: env.NOTE_CONNECTION_RUNTIME_QUERY_FALLBACK_WARN_RATIO_PCT,
|
|
641
|
+
queryFallbackFailRatioPct: env.NOTE_CONNECTION_RUNTIME_QUERY_FALLBACK_FAIL_RATIO_PCT,
|
|
642
|
+
queryEvidenceCoverageWarnRatioPct: env.NOTE_CONNECTION_RUNTIME_QUERY_EVIDENCE_COVERAGE_WARN_RATIO_PCT,
|
|
643
|
+
queryEvidenceCoverageFailRatioPct: env.NOTE_CONNECTION_RUNTIME_QUERY_EVIDENCE_COVERAGE_FAIL_RATIO_PCT,
|
|
644
|
+
queryTemporalValidityWarnRatioPct: env.NOTE_CONNECTION_RUNTIME_QUERY_TEMPORAL_VALIDITY_WARN_RATIO_PCT,
|
|
645
|
+
queryTemporalValidityFailRatioPct: env.NOTE_CONNECTION_RUNTIME_QUERY_TEMPORAL_VALIDITY_FAIL_RATIO_PCT,
|
|
646
|
+
queryBackendExplainabilityGapWarnRatioPct: env.NOTE_CONNECTION_RUNTIME_QUERY_BACKEND_EXPLAINABILITY_GAP_WARN_RATIO_PCT,
|
|
647
|
+
queryBackendExplainabilityGapFailRatioPct: env.NOTE_CONNECTION_RUNTIME_QUERY_BACKEND_EXPLAINABILITY_GAP_FAIL_RATIO_PCT,
|
|
648
|
+
queryBackendTrendWarnConfidenceRatioPct: env.NOTE_CONNECTION_RUNTIME_QUERY_BACKEND_TREND_WARN_CONFIDENCE_PCT,
|
|
649
|
+
queryBackendTrendFailConfidenceRatioPct: env.NOTE_CONNECTION_RUNTIME_QUERY_BACKEND_TREND_FAIL_CONFIDENCE_PCT,
|
|
650
|
+
sessionPlanQualityWarnFailureStreak: env.NOTE_CONNECTION_RUNTIME_SESSION_PLAN_QUALITY_WARN_FAILURE_STREAK,
|
|
651
|
+
sessionPlanQualityFailFailureStreak: env.NOTE_CONNECTION_RUNTIME_SESSION_PLAN_QUALITY_FAIL_FAILURE_STREAK,
|
|
652
|
+
apiInvalidRequestMinErrorSample: env.NOTE_CONNECTION_RUNTIME_API_INVALID_REQUEST_MIN_ERROR_SAMPLE,
|
|
653
|
+
apiInvalidRequestWarnRatioPct: env.NOTE_CONNECTION_RUNTIME_API_INVALID_REQUEST_WARN_RATIO_PCT,
|
|
654
|
+
apiInvalidRequestFailRatioPct: env.NOTE_CONNECTION_RUNTIME_API_INVALID_REQUEST_FAIL_RATIO_PCT,
|
|
655
|
+
apiInvalidRequestHotspotWarnCount: env.NOTE_CONNECTION_RUNTIME_API_INVALID_REQUEST_HOTSPOT_WARN_COUNT,
|
|
656
|
+
apiInvalidRequestHotspotFailCount: env.NOTE_CONNECTION_RUNTIME_API_INVALID_REQUEST_HOTSPOT_FAIL_COUNT,
|
|
657
|
+
apiServerErrorMinRequestSample: env.NOTE_CONNECTION_RUNTIME_API_SERVER_ERROR_MIN_REQUEST_SAMPLE,
|
|
658
|
+
apiServerErrorWarnRatioPct: env.NOTE_CONNECTION_RUNTIME_API_SERVER_ERROR_WARN_RATIO_PCT,
|
|
659
|
+
apiServerErrorFailRatioPct: env.NOTE_CONNECTION_RUNTIME_API_SERVER_ERROR_FAIL_RATIO_PCT,
|
|
660
|
+
apiServerErrorHotspotWarnCount: env.NOTE_CONNECTION_RUNTIME_API_SERVER_ERROR_HOTSPOT_WARN_COUNT,
|
|
661
|
+
apiServerErrorHotspotFailCount: env.NOTE_CONNECTION_RUNTIME_API_SERVER_ERROR_HOTSPOT_FAIL_COUNT,
|
|
662
|
+
apiTransientErrorMinRequestSample: env.NOTE_CONNECTION_RUNTIME_API_TRANSIENT_ERROR_MIN_REQUEST_SAMPLE,
|
|
663
|
+
apiTransientErrorWarnRatioPct: env.NOTE_CONNECTION_RUNTIME_API_TRANSIENT_ERROR_WARN_RATIO_PCT,
|
|
664
|
+
apiTransientErrorFailRatioPct: env.NOTE_CONNECTION_RUNTIME_API_TRANSIENT_ERROR_FAIL_RATIO_PCT,
|
|
665
|
+
apiTransientErrorHotspotWarnCount: env.NOTE_CONNECTION_RUNTIME_API_TRANSIENT_ERROR_HOTSPOT_WARN_COUNT,
|
|
666
|
+
apiTransientErrorHotspotFailCount: env.NOTE_CONNECTION_RUNTIME_API_TRANSIENT_ERROR_HOTSPOT_FAIL_COUNT,
|
|
667
|
+
apiLatencyMinRequestSample: env.NOTE_CONNECTION_RUNTIME_API_LATENCY_MIN_REQUEST_SAMPLE,
|
|
668
|
+
apiLatencyP95WarnMs: env.NOTE_CONNECTION_RUNTIME_API_LATENCY_P95_WARN_MS,
|
|
669
|
+
apiLatencyP95FailMs: env.NOTE_CONNECTION_RUNTIME_API_LATENCY_P95_FAIL_MS,
|
|
670
|
+
apiLatencyHotspotWarnMs: env.NOTE_CONNECTION_RUNTIME_API_LATENCY_HOTSPOT_WARN_MS,
|
|
671
|
+
apiLatencyHotspotFailMs: env.NOTE_CONNECTION_RUNTIME_API_LATENCY_HOTSPOT_FAIL_MS,
|
|
672
|
+
queryVectorAccelerationShortCircuitWarnCount: env.NOTE_CONNECTION_RUNTIME_QUERY_VECTOR_ACCELERATION_SHORT_CIRCUIT_WARN_COUNT,
|
|
673
|
+
queryVectorAccelerationShortCircuitFailCount: env.NOTE_CONNECTION_RUNTIME_QUERY_VECTOR_ACCELERATION_SHORT_CIRCUIT_FAIL_COUNT,
|
|
674
|
+
queryVectorAccelerationShortCircuitWarnRatioPct: env.NOTE_CONNECTION_RUNTIME_QUERY_VECTOR_ACCELERATION_SHORT_CIRCUIT_WARN_RATIO_PCT,
|
|
675
|
+
queryVectorAccelerationShortCircuitFailRatioPct: env.NOTE_CONNECTION_RUNTIME_QUERY_VECTOR_ACCELERATION_SHORT_CIRCUIT_FAIL_RATIO_PCT,
|
|
676
|
+
queryVectorAccelerationConsecutiveFailuresWarnCount: env.NOTE_CONNECTION_RUNTIME_QUERY_VECTOR_ACCELERATION_CONSECUTIVE_FAILURES_WARN_COUNT,
|
|
677
|
+
queryVectorAccelerationConsecutiveFailuresFailCount: env.NOTE_CONNECTION_RUNTIME_QUERY_VECTOR_ACCELERATION_CONSECUTIVE_FAILURES_FAIL_COUNT,
|
|
678
|
+
queryVectorAccelerationHalfOpenSuccessWarnRatioPct: env.NOTE_CONNECTION_RUNTIME_QUERY_VECTOR_ACCELERATION_HALF_OPEN_SUCCESS_WARN_RATIO_PCT,
|
|
679
|
+
queryVectorAccelerationHalfOpenSuccessFailRatioPct: env.NOTE_CONNECTION_RUNTIME_QUERY_VECTOR_ACCELERATION_HALF_OPEN_SUCCESS_FAIL_RATIO_PCT,
|
|
680
|
+
queryVectorAccelerationPrefilterMinRequestSample: env.NOTE_CONNECTION_RUNTIME_QUERY_VECTOR_ACCELERATION_PREFILTER_MIN_REQUEST_SAMPLE,
|
|
681
|
+
queryVectorAccelerationPrefilterWarnCandidateRatioPct: env.NOTE_CONNECTION_RUNTIME_QUERY_VECTOR_ACCELERATION_PREFILTER_WARN_CANDIDATE_RATIO_PCT,
|
|
682
|
+
queryVectorAccelerationPrefilterFailCandidateRatioPct: env.NOTE_CONNECTION_RUNTIME_QUERY_VECTOR_ACCELERATION_PREFILTER_FAIL_CANDIDATE_RATIO_PCT,
|
|
683
|
+
storeGraphDbConnectorMinRequestSample: env.NOTE_CONNECTION_RUNTIME_STORE_GRAPHDB_CONNECTOR_MIN_REQUEST_SAMPLE,
|
|
684
|
+
storeGraphDbConnectorFailureWarnRatioPct: env.NOTE_CONNECTION_RUNTIME_STORE_GRAPHDB_CONNECTOR_FAILURE_WARN_RATIO_PCT,
|
|
685
|
+
storeGraphDbConnectorFailureFailRatioPct: env.NOTE_CONNECTION_RUNTIME_STORE_GRAPHDB_CONNECTOR_FAILURE_FAIL_RATIO_PCT,
|
|
686
|
+
storeGraphDbConnectorShortCircuitWarnRatioPct: env.NOTE_CONNECTION_RUNTIME_STORE_GRAPHDB_CONNECTOR_SHORT_CIRCUIT_WARN_RATIO_PCT,
|
|
687
|
+
storeGraphDbConnectorShortCircuitFailRatioPct: env.NOTE_CONNECTION_RUNTIME_STORE_GRAPHDB_CONNECTOR_SHORT_CIRCUIT_FAIL_RATIO_PCT,
|
|
688
|
+
storeGraphDbConnectorConsecutiveFailuresWarnCount: env.NOTE_CONNECTION_RUNTIME_STORE_GRAPHDB_CONNECTOR_CONSECUTIVE_FAILURES_WARN_COUNT,
|
|
689
|
+
storeGraphDbConnectorConsecutiveFailuresFailCount: env.NOTE_CONNECTION_RUNTIME_STORE_GRAPHDB_CONNECTOR_CONSECUTIVE_FAILURES_FAIL_COUNT,
|
|
690
|
+
});
|
|
691
|
+
}
|
|
692
|
+
function buildRuntimeCapabilityMatrix(params) {
|
|
693
|
+
const thresholds = normalizeRuntimeCapabilityThresholds(params.thresholds || {});
|
|
694
|
+
const checks = [];
|
|
695
|
+
const safeQueryCount = Math.max(0, Math.floor(Number(params.queryCount || 0)));
|
|
696
|
+
const safeFallbackCount = Math.max(0, Math.floor(Number(params.queryDiagnostics.fallbackCount || 0)));
|
|
697
|
+
const queryFallbackRatioPct = Number(((safeFallbackCount / Math.max(1, safeQueryCount)) * 100).toFixed(4));
|
|
698
|
+
const graphDbConnectorHealthStatusRaw = String(params.store.connector?.healthStatus || '')
|
|
699
|
+
.trim()
|
|
700
|
+
.toLowerCase();
|
|
701
|
+
const graphDbConnectorHealthStatus = (graphDbConnectorHealthStatusRaw === 'ready'
|
|
702
|
+
|| graphDbConnectorHealthStatusRaw === 'degraded'
|
|
703
|
+
|| graphDbConnectorHealthStatusRaw === 'unavailable') ? graphDbConnectorHealthStatusRaw : 'unknown';
|
|
704
|
+
const graphDbConnectorHealthMessage = String(params.store.connector?.healthMessage || '')
|
|
705
|
+
.trim()
|
|
706
|
+
.slice(0, 240);
|
|
707
|
+
const graphDbConnectorCircuitStateRaw = String(params.store.connector?.circuitState || '')
|
|
708
|
+
.trim()
|
|
709
|
+
.toLowerCase();
|
|
710
|
+
const graphDbConnectorCircuitState = (graphDbConnectorCircuitStateRaw === 'closed'
|
|
711
|
+
|| graphDbConnectorCircuitStateRaw === 'open'
|
|
712
|
+
|| graphDbConnectorCircuitStateRaw === 'half_open') ? graphDbConnectorCircuitStateRaw : 'unknown';
|
|
713
|
+
const graphDbConnectorRequestCount = Math.max(0, Math.floor(Number(params.store.connector?.requestCount || 0)));
|
|
714
|
+
const graphDbConnectorRetryCount = Math.max(0, Math.floor(Number(params.store.connector?.retryCount || 0)));
|
|
715
|
+
const graphDbConnectorShortCircuitCount = Math.max(0, Math.floor(Number(params.store.connector?.shortCircuitCount || 0)));
|
|
716
|
+
const graphDbConnectorSuccessCount = Math.max(0, Math.floor(Number(params.store.connector?.successCount || 0)));
|
|
717
|
+
const graphDbConnectorFailureCount = Math.max(0, Math.floor(Number(params.store.connector?.failureCount || 0)));
|
|
718
|
+
const graphDbConnectorConsecutiveFailures = Math.max(0, Math.floor(Number(params.store.connector?.consecutiveFailures || 0)));
|
|
719
|
+
const graphDbConnectorFailureRatioPct = Number((graphDbConnectorRequestCount > 0
|
|
720
|
+
? (graphDbConnectorFailureCount
|
|
721
|
+
/ graphDbConnectorRequestCount) * 100
|
|
722
|
+
: 0).toFixed(4));
|
|
723
|
+
const graphDbConnectorShortCircuitRatioPct = Number((graphDbConnectorRequestCount > 0
|
|
724
|
+
? (graphDbConnectorShortCircuitCount
|
|
725
|
+
/ graphDbConnectorRequestCount) * 100
|
|
726
|
+
: 0).toFixed(4));
|
|
727
|
+
const shouldEvaluateGraphDbConnectorBudget = (params.configuredStoreBackend === 'graphdb'
|
|
728
|
+
&& params.store.storeType === 'graphdb'
|
|
729
|
+
&& params.store.backendReady !== false
|
|
730
|
+
&& params.store.usingFallback !== true);
|
|
731
|
+
const graphDbConnectorBudgetMeetsMinSample = graphDbConnectorRequestCount >= thresholds.storeGraphDbConnectorMinRequestSample;
|
|
732
|
+
const graphDbConnectorFailureWarnBudgetExceeded = (graphDbConnectorBudgetMeetsMinSample
|
|
733
|
+
&& graphDbConnectorFailureRatioPct >= thresholds.storeGraphDbConnectorFailureWarnRatioPct);
|
|
734
|
+
const graphDbConnectorFailureFailBudgetExceeded = (graphDbConnectorBudgetMeetsMinSample
|
|
735
|
+
&& graphDbConnectorFailureRatioPct >= thresholds.storeGraphDbConnectorFailureFailRatioPct);
|
|
736
|
+
const graphDbConnectorShortCircuitWarnBudgetExceeded = (graphDbConnectorBudgetMeetsMinSample
|
|
737
|
+
&& graphDbConnectorShortCircuitRatioPct >= thresholds.storeGraphDbConnectorShortCircuitWarnRatioPct);
|
|
738
|
+
const graphDbConnectorShortCircuitFailBudgetExceeded = (graphDbConnectorBudgetMeetsMinSample
|
|
739
|
+
&& graphDbConnectorShortCircuitRatioPct >= thresholds.storeGraphDbConnectorShortCircuitFailRatioPct);
|
|
740
|
+
const graphDbConnectorConsecutiveFailuresWarnBudgetExceeded = (graphDbConnectorBudgetMeetsMinSample
|
|
741
|
+
&& graphDbConnectorConsecutiveFailures >= thresholds.storeGraphDbConnectorConsecutiveFailuresWarnCount);
|
|
742
|
+
const graphDbConnectorConsecutiveFailuresFailBudgetExceeded = (graphDbConnectorBudgetMeetsMinSample
|
|
743
|
+
&& graphDbConnectorConsecutiveFailures >= thresholds.storeGraphDbConnectorConsecutiveFailuresFailCount);
|
|
744
|
+
const graphDbConnectorFailBudgetExceeded = (shouldEvaluateGraphDbConnectorBudget
|
|
745
|
+
&& (graphDbConnectorCircuitState === 'open'
|
|
746
|
+
|| graphDbConnectorFailureFailBudgetExceeded
|
|
747
|
+
|| graphDbConnectorShortCircuitFailBudgetExceeded
|
|
748
|
+
|| graphDbConnectorConsecutiveFailuresFailBudgetExceeded));
|
|
749
|
+
const graphDbConnectorWarnBudgetExceeded = (shouldEvaluateGraphDbConnectorBudget
|
|
750
|
+
&& (graphDbConnectorFailBudgetExceeded
|
|
751
|
+
|| graphDbConnectorCircuitState === 'half_open'
|
|
752
|
+
|| graphDbConnectorFailureWarnBudgetExceeded
|
|
753
|
+
|| graphDbConnectorShortCircuitWarnBudgetExceeded
|
|
754
|
+
|| graphDbConnectorConsecutiveFailuresWarnBudgetExceeded));
|
|
755
|
+
const graphDbConnectorBudgetStatus = graphDbConnectorFailBudgetExceeded
|
|
756
|
+
? 'fail'
|
|
757
|
+
: (graphDbConnectorWarnBudgetExceeded ? 'warn' : 'ok');
|
|
758
|
+
const graphDbConnectorLastRequestId = String(params.store.connector?.lastRequestId || '')
|
|
759
|
+
.trim()
|
|
760
|
+
.replace(/[^a-zA-Z0-9._:-]+/g, '')
|
|
761
|
+
.slice(0, 96);
|
|
762
|
+
const graphDbConnectorLastErrorCode = String(params.store.connector?.lastErrorCode || '')
|
|
763
|
+
.trim()
|
|
764
|
+
.toLowerCase()
|
|
765
|
+
.replace(/[^a-z0-9_:-]+/g, '_')
|
|
766
|
+
.slice(0, 96);
|
|
767
|
+
const graphDbConnectorLastStatusCode = Math.max(0, Math.floor(Number(params.store.connector?.lastStatusCode || 0)));
|
|
768
|
+
const graphDbConnectorLastRetryAfterMs = Math.max(0, Math.floor(Number(params.store.connector?.lastRetryAfterMs || 0)));
|
|
769
|
+
const queryBackendRuntimeId = String(params.queryDiagnostics.runtime?.backendId || '')
|
|
770
|
+
.trim()
|
|
771
|
+
.slice(0, 128);
|
|
772
|
+
const queryBackendRuntimeReady = params.queryDiagnostics.runtime?.ready !== false;
|
|
773
|
+
const queryVectorIndexEnabled = params.queryDiagnostics.runtime?.vectorIndex?.enabled === true;
|
|
774
|
+
const queryVectorIndexStatusRaw = String(params.queryDiagnostics.runtime?.vectorIndex?.status || '').trim().toLowerCase();
|
|
775
|
+
const queryVectorIndexStatus = (queryVectorIndexStatusRaw === 'ready'
|
|
776
|
+
|| queryVectorIndexStatusRaw === 'stale'
|
|
777
|
+
|| queryVectorIndexStatusRaw === 'unavailable') ? queryVectorIndexStatusRaw : 'unknown';
|
|
778
|
+
const queryVectorIndexPersisted = params.queryDiagnostics.runtime?.vectorIndex?.persisted === true;
|
|
779
|
+
const queryVectorIndexLoadedFromDisk = params.queryDiagnostics.runtime?.vectorIndex?.loadedFromDisk === true;
|
|
780
|
+
const queryVectorIndexAtomCount = Math.max(0, Math.floor(Number(params.queryDiagnostics.runtime?.vectorIndex?.atomCount || 0)));
|
|
781
|
+
const queryVectorIndexLocation = String(params.queryDiagnostics.runtime?.vectorIndex?.location || '')
|
|
782
|
+
.trim()
|
|
783
|
+
.slice(0, 256);
|
|
784
|
+
const queryVectorIndexAccelerationEnabled = params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.enabled === true;
|
|
785
|
+
const queryVectorIndexAccelerationModeRaw = String(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.mode || '').trim().toLowerCase();
|
|
786
|
+
const queryVectorIndexAccelerationMode = (queryVectorIndexAccelerationModeRaw === 'ann_prefilter'
|
|
787
|
+
|| queryVectorIndexAccelerationModeRaw === 'full_scan') ? queryVectorIndexAccelerationModeRaw : 'unknown';
|
|
788
|
+
const queryVectorIndexAccelerationLastSelectionModeRaw = String(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.lastSelectionMode || '').trim().toLowerCase();
|
|
789
|
+
const queryVectorIndexAccelerationLastSelectionMode = (queryVectorIndexAccelerationLastSelectionModeRaw === 'full_scan'
|
|
790
|
+
|| queryVectorIndexAccelerationLastSelectionModeRaw === 'token_prefilter'
|
|
791
|
+
|| queryVectorIndexAccelerationLastSelectionModeRaw === 'token_signature_prefilter') ? queryVectorIndexAccelerationLastSelectionModeRaw : 'unknown';
|
|
792
|
+
const queryVectorIndexAccelerationLastCandidateCount = Math.max(0, Math.floor(Number(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.lastCandidateCount || 0)));
|
|
793
|
+
const queryVectorIndexAccelerationAdapterId = String(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.adapterId || '').trim().slice(0, 160);
|
|
794
|
+
const queryVectorIndexAccelerationAdapterError = String(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.adapterError || '').trim().slice(0, 240);
|
|
795
|
+
const queryVectorIndexAccelerationHealthStatusRaw = String(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.healthStatus || '').trim().toLowerCase();
|
|
796
|
+
const queryVectorIndexAccelerationHealthStatus = (queryVectorIndexAccelerationHealthStatusRaw === 'ready'
|
|
797
|
+
|| queryVectorIndexAccelerationHealthStatusRaw === 'degraded'
|
|
798
|
+
|| queryVectorIndexAccelerationHealthStatusRaw === 'unavailable'
|
|
799
|
+
|| queryVectorIndexAccelerationHealthStatusRaw === 'unknown') ? queryVectorIndexAccelerationHealthStatusRaw : 'unknown';
|
|
800
|
+
const queryVectorIndexAccelerationHealthMessage = String(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.healthMessage || '').trim().slice(0, 240);
|
|
801
|
+
const queryVectorIndexAccelerationIndexSyncStatusRaw = String(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.indexSyncStatus || '').trim().toLowerCase();
|
|
802
|
+
const queryVectorIndexAccelerationIndexSyncStatus = (queryVectorIndexAccelerationIndexSyncStatusRaw === 'ready'
|
|
803
|
+
|| queryVectorIndexAccelerationIndexSyncStatusRaw === 'degraded'
|
|
804
|
+
|| queryVectorIndexAccelerationIndexSyncStatusRaw === 'unavailable'
|
|
805
|
+
|| queryVectorIndexAccelerationIndexSyncStatusRaw === 'unknown') ? queryVectorIndexAccelerationIndexSyncStatusRaw : 'unknown';
|
|
806
|
+
const queryVectorIndexAccelerationIndexSyncMessage = String(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.indexSyncMessage || '').trim().slice(0, 240);
|
|
807
|
+
const queryVectorIndexAccelerationLastSyncAt = String(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.lastSyncAt || '').trim().slice(0, 64);
|
|
808
|
+
const queryVectorIndexAccelerationSyncRequestCount = Math.max(0, Math.floor(Number(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.syncRequestCount || 0)));
|
|
809
|
+
const queryVectorIndexAccelerationSyncSuccessCount = Math.max(0, Math.floor(Number(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.syncSuccessCount || 0)));
|
|
810
|
+
const queryVectorIndexAccelerationSyncFailureCount = Math.max(0, Math.floor(Number(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.syncFailureCount || 0)));
|
|
811
|
+
const queryVectorIndexAccelerationSyncedIndexSignature = String(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.syncedIndexSignature || '').trim().replace(/[^a-zA-Z0-9:_-]+/g, '').slice(0, 200);
|
|
812
|
+
const queryVectorIndexAccelerationSyncedAtomCount = Math.max(0, Math.floor(Number(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.syncedAtomCount || 0)));
|
|
813
|
+
const queryVectorIndexAccelerationRepresentationVersion = String(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.representationVersion || '').trim().slice(0, 160);
|
|
814
|
+
const queryVectorIndexAccelerationEmbeddingModelId = String(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.embeddingModelId || '').trim().slice(0, 160);
|
|
815
|
+
const queryVectorIndexAccelerationEmbeddingDimension = Math.max(0, Math.floor(Number(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.embeddingDimension || 0)));
|
|
816
|
+
const queryVectorIndexAccelerationIndexSignature = String(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.indexSignature || '').trim().replace(/[^a-zA-Z0-9:_-]+/g, '').slice(0, 200);
|
|
817
|
+
const queryVectorIndexAccelerationRepresentationStatusRaw = String(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.representationStatus || '').trim().toLowerCase();
|
|
818
|
+
const queryVectorIndexAccelerationRepresentationStatus = (queryVectorIndexAccelerationRepresentationStatusRaw === 'aligned'
|
|
819
|
+
|| queryVectorIndexAccelerationRepresentationStatusRaw === 'mismatch'
|
|
820
|
+
|| queryVectorIndexAccelerationRepresentationStatusRaw === 'unknown') ? queryVectorIndexAccelerationRepresentationStatusRaw : 'unknown';
|
|
821
|
+
const queryVectorIndexAccelerationRepresentationStatusReason = String(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.representationStatusReason || '').trim().replace(/\s+/g, ' ').slice(0, 240);
|
|
822
|
+
const queryVectorIndexAccelerationRepresentationStrictMode = params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.representationStrictMode === true;
|
|
823
|
+
const queryVectorIndexAccelerationLastRequestId = String(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.lastRequestId || '').trim().replace(/[^a-zA-Z0-9._:-]+/g, '').slice(0, 96);
|
|
824
|
+
const queryVectorIndexAccelerationLastErrorCode = String(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.lastErrorCode || '').trim().toLowerCase().replace(/[^a-z0-9_:-]+/g, '_').slice(0, 96);
|
|
825
|
+
const queryVectorIndexAccelerationLastRetryAfterMs = Math.max(0, Math.floor(Number(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.lastRetryAfterMs || 0)));
|
|
826
|
+
const queryVectorIndexAccelerationCircuitStateRaw = String(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.circuitState || '').trim().toLowerCase();
|
|
827
|
+
const queryVectorIndexAccelerationCircuitState = (queryVectorIndexAccelerationCircuitStateRaw === 'closed'
|
|
828
|
+
|| queryVectorIndexAccelerationCircuitStateRaw === 'open'
|
|
829
|
+
|| queryVectorIndexAccelerationCircuitStateRaw === 'half_open'
|
|
830
|
+
|| queryVectorIndexAccelerationCircuitStateRaw === 'unknown') ? queryVectorIndexAccelerationCircuitStateRaw : 'unknown';
|
|
831
|
+
const queryVectorIndexAccelerationConsecutiveFailures = Math.max(0, Math.floor(Number(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.consecutiveFailures || 0)));
|
|
832
|
+
const queryVectorIndexAccelerationRequestCount = Math.max(0, Math.floor(Number(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.requestCount || 0)));
|
|
833
|
+
const queryVectorIndexAccelerationRetryCount = Math.max(0, Math.floor(Number(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.retryCount || 0)));
|
|
834
|
+
const queryVectorIndexAccelerationShortCircuitCount = Math.max(0, Math.floor(Number(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.shortCircuitCount || 0)));
|
|
835
|
+
const queryVectorIndexAccelerationSuccessCount = Math.max(0, Math.floor(Number(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.successCount || 0)));
|
|
836
|
+
const queryVectorIndexAccelerationFailureCount = Math.max(0, Math.floor(Number(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.failureCount || 0)));
|
|
837
|
+
const queryVectorIndexAccelerationHalfOpenProbeSuccessCount = Math.max(0, Math.floor(Number(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.halfOpenProbeSuccessCount || 0)));
|
|
838
|
+
const queryVectorIndexAccelerationHalfOpenProbeFailureCount = Math.max(0, Math.floor(Number(params.queryDiagnostics.runtime?.vectorIndex?.acceleration?.halfOpenProbeFailureCount || 0)));
|
|
839
|
+
const queryVectorIndexAccelerationHalfOpenProbeCount = queryVectorIndexAccelerationHalfOpenProbeSuccessCount
|
|
840
|
+
+ queryVectorIndexAccelerationHalfOpenProbeFailureCount;
|
|
841
|
+
const queryVectorIndexAccelerationHalfOpenSuccessRatePct = Number((queryVectorIndexAccelerationHalfOpenProbeCount > 0
|
|
842
|
+
? (queryVectorIndexAccelerationHalfOpenProbeSuccessCount
|
|
843
|
+
/ queryVectorIndexAccelerationHalfOpenProbeCount) * 100
|
|
844
|
+
: 0).toFixed(4));
|
|
845
|
+
const queryVectorIndexAccelerationShortCircuitRatioPct = Number((queryVectorIndexAccelerationRequestCount > 0
|
|
846
|
+
? (queryVectorIndexAccelerationShortCircuitCount
|
|
847
|
+
/ queryVectorIndexAccelerationRequestCount) * 100
|
|
848
|
+
: 0).toFixed(4));
|
|
849
|
+
let queryVectorIndexAccelerationCircuitWarnBudgetExceeded = false;
|
|
850
|
+
let queryVectorIndexAccelerationCircuitFailBudgetExceeded = false;
|
|
851
|
+
let queryVectorIndexAccelerationCircuitBudgetStatus = 'ok';
|
|
852
|
+
const queryExplainabilitySampleCount = Math.max(0, Math.floor(Number(params.queryExplainabilityTelemetry?.sampleCount || 0)));
|
|
853
|
+
const queryEvidenceCoverageRatioPct = Number(clamp(Number(params.queryExplainabilityTelemetry?.evidenceCoverageRatioPct || 0), 0, 100).toFixed(4));
|
|
854
|
+
const queryRelationPathCoverageRatioPct = Number(clamp(Number(params.queryExplainabilityTelemetry?.relationPathCoverageRatioPct || 0), 0, 100).toFixed(4));
|
|
855
|
+
const queryTemporalValidityPassRatioPct = Number(clamp(Number(params.queryExplainabilityTelemetry?.temporalValidityPassRatioPct || 0), 0, 100).toFixed(4));
|
|
856
|
+
const queryAverageEvidenceSpanCount = Number(clamp(Number(params.queryExplainabilityTelemetry?.averageEvidenceSpanCount || 0), 0, 1000).toFixed(4));
|
|
857
|
+
const queryAverageRelationPathLength = Number(clamp(Number(params.queryExplainabilityTelemetry?.averageRelationPathLength || 0), 0, 1000).toFixed(4));
|
|
858
|
+
const queryBackendComparisonSampleCount = Math.max(0, Math.floor(Number(params.queryBackendComparisonTelemetry?.summary?.returnedRecords || 0)));
|
|
859
|
+
const queryBackendComparisonEvidenceGapRatioPct = Number((Math.abs(Number(params.queryBackendComparisonTelemetry?.summary?.averageLeftEvidenceCoverageRatio || 0)
|
|
860
|
+
- Number(params.queryBackendComparisonTelemetry?.summary?.averageRightEvidenceCoverageRatio || 0)) * 100).toFixed(4));
|
|
861
|
+
const queryBackendComparisonRelationGapRatioPct = Number((Math.abs(Number(params.queryBackendComparisonTelemetry?.summary?.averageLeftRelationPathCoverageRatio || 0)
|
|
862
|
+
- Number(params.queryBackendComparisonTelemetry?.summary?.averageRightRelationPathCoverageRatio || 0)) * 100).toFixed(4));
|
|
863
|
+
const queryBackendComparisonTemporalGapRatioPct = Number((Math.abs(Number(params.queryBackendComparisonTelemetry?.summary?.averageLeftTemporalValidityPassRatio || 0)
|
|
864
|
+
- Number(params.queryBackendComparisonTelemetry?.summary?.averageRightTemporalValidityPassRatio || 0)) * 100).toFixed(4));
|
|
865
|
+
const queryBackendComparisonMaxExplainabilityGapRatioPct = Number(Math.max(queryBackendComparisonEvidenceGapRatioPct, queryBackendComparisonRelationGapRatioPct, queryBackendComparisonTemporalGapRatioPct).toFixed(4));
|
|
866
|
+
const queryBackendTrendStatusRaw = String(params.queryBackendComparisonTrend?.status || '').trim().toLowerCase();
|
|
867
|
+
const hasQueryBackendComparisonTrendSignal = Boolean(params.queryBackendComparisonTrend);
|
|
868
|
+
const queryBackendComparisonTrendStatus = (queryBackendTrendStatusRaw === 'improving'
|
|
869
|
+
|| queryBackendTrendStatusRaw === 'stable'
|
|
870
|
+
|| queryBackendTrendStatusRaw === 'regressing'
|
|
871
|
+
|| queryBackendTrendStatusRaw === 'insufficient_data') ? queryBackendTrendStatusRaw : 'unknown';
|
|
872
|
+
const queryBackendComparisonTrendScore = Number((Number.isFinite(Number(params.queryBackendComparisonTrend?.score))
|
|
873
|
+
? Number(params.queryBackendComparisonTrend?.score)
|
|
874
|
+
: 0).toFixed(4));
|
|
875
|
+
const queryBackendComparisonTrendConfidence = Number((Number.isFinite(Number(params.queryBackendComparisonTrend?.confidence))
|
|
876
|
+
? clamp(Number(params.queryBackendComparisonTrend?.confidence), 0, 1)
|
|
877
|
+
: 0).toFixed(4));
|
|
878
|
+
const queryBackendComparisonTrendConfidenceRatioPct = Number((queryBackendComparisonTrendConfidence * 100).toFixed(4));
|
|
879
|
+
const queryBackendComparisonTrendReason = String(params.queryBackendComparisonTrend?.summary?.reason || '').trim();
|
|
880
|
+
const hasQueryBackendComparisonTrendConfig = Boolean(params.queryBackendComparisonTrendConfig);
|
|
881
|
+
const queryBackendComparisonTrendLimit = hasQueryBackendComparisonTrendConfig
|
|
882
|
+
? Math.max(2, Math.floor(Number(params.queryBackendComparisonTrendConfig?.limit || 0)))
|
|
883
|
+
: 0;
|
|
884
|
+
const queryBackendComparisonTrendWindowSize = hasQueryBackendComparisonTrendConfig
|
|
885
|
+
? Math.max(1, Math.floor(Number(params.queryBackendComparisonTrendConfig?.windowSize || 0)))
|
|
886
|
+
: 0;
|
|
887
|
+
const queryBackendComparisonTrendMinSamples = hasQueryBackendComparisonTrendConfig
|
|
888
|
+
? clamp(Math.floor(Number(params.queryBackendComparisonTrendConfig?.minSamples || 0)), 1, Math.max(1, queryBackendComparisonTrendWindowSize))
|
|
889
|
+
: 0;
|
|
890
|
+
const queryBackendComparisonTrendRequiredRecords = hasQueryBackendComparisonTrendConfig
|
|
891
|
+
? queryBackendComparisonTrendWindowSize + queryBackendComparisonTrendMinSamples
|
|
892
|
+
: 0;
|
|
893
|
+
const queryBackendComparisonTrendObserved = [
|
|
894
|
+
`status=${queryBackendComparisonTrendStatus}`,
|
|
895
|
+
`score=${queryBackendComparisonTrendScore}`,
|
|
896
|
+
`confidence=${queryBackendComparisonTrendConfidence}`,
|
|
897
|
+
hasQueryBackendComparisonTrendConfig
|
|
898
|
+
? `config(limit=${queryBackendComparisonTrendLimit},windowSize=${queryBackendComparisonTrendWindowSize},minSamples=${queryBackendComparisonTrendMinSamples})`
|
|
899
|
+
: '',
|
|
900
|
+
queryBackendComparisonTrendReason
|
|
901
|
+
? `reason=${queryBackendComparisonTrendReason}`
|
|
902
|
+
: '',
|
|
903
|
+
].filter(Boolean).join(', ');
|
|
904
|
+
const hasApiRequestErrorTelemetry = Boolean(params.apiRequestErrorTelemetry);
|
|
905
|
+
const apiTraceWindowRequests = Math.max(0, Math.floor(Number(params.apiRequestErrorTelemetry?.totalRequests || 0)));
|
|
906
|
+
const apiTraceWindowErrors = Math.max(0, Math.floor(Number(params.apiRequestErrorTelemetry?.errorRequests || 0)));
|
|
907
|
+
const apiTraceWindowInvalidRequests = Math.max(0, Math.floor(Number(params.apiRequestErrorTelemetry?.invalidRequestErrors || 0)));
|
|
908
|
+
const apiTraceWindowServerErrors = Math.max(0, Math.floor(Number(params.apiRequestErrorTelemetry?.serverErrorRequests || 0)));
|
|
909
|
+
const apiTraceWindowTransientErrors = Math.max(0, Math.floor(Number(params.apiRequestErrorTelemetry?.transientErrorRequests || 0)));
|
|
910
|
+
const apiTraceAverageDurationMs = Number(clamp(Number(params.apiRequestErrorTelemetry?.averageDurationMs || 0), 0, 600000).toFixed(4));
|
|
911
|
+
const apiTraceP95DurationMs = Number(clamp(Number(params.apiRequestErrorTelemetry?.p95DurationMs || 0), 0, 600000).toFixed(4));
|
|
912
|
+
const apiTraceWindowInvalidRequestRatioPct = Number(((apiTraceWindowInvalidRequests / Math.max(1, apiTraceWindowErrors)) * 100).toFixed(4));
|
|
913
|
+
const apiTraceWindowInvalidRequestToTotalRatioPct = Number(((apiTraceWindowInvalidRequests / Math.max(1, apiTraceWindowRequests)) * 100).toFixed(4));
|
|
914
|
+
const apiTraceWindowServerErrorRatioPct = Number(((apiTraceWindowServerErrors / Math.max(1, apiTraceWindowRequests)) * 100).toFixed(4));
|
|
915
|
+
const apiTraceWindowTransientErrorRatioPct = Number(((apiTraceWindowTransientErrors / Math.max(1, apiTraceWindowRequests)) * 100).toFixed(4));
|
|
916
|
+
const apiTraceScopePathPrefix = String(params.apiRequestErrorTelemetry?.scopePathPrefix || '').trim().slice(0, 128);
|
|
917
|
+
const apiTraceScopeMethod = normalizeRuntimeCapabilityHttpMethod(params.apiRequestErrorTelemetry?.scopeMethod || '');
|
|
918
|
+
const apiTraceWindowInvalidRequestTopPaths = Array.isArray(params.apiRequestErrorTelemetry?.invalidRequestTopPaths)
|
|
919
|
+
? params.apiRequestErrorTelemetry?.invalidRequestTopPaths
|
|
920
|
+
.map((item) => {
|
|
921
|
+
const path = String(item?.path || '').trim().slice(0, 256);
|
|
922
|
+
const count = Math.max(0, Math.floor(Number(item?.count || 0)));
|
|
923
|
+
if (!path || count <= 0) {
|
|
924
|
+
return null;
|
|
925
|
+
}
|
|
926
|
+
return { path, count };
|
|
927
|
+
})
|
|
928
|
+
.filter((item) => Boolean(item))
|
|
929
|
+
.slice(0, 5)
|
|
930
|
+
: [];
|
|
931
|
+
const apiTraceWindowServerErrorTopPaths = Array.isArray(params.apiRequestErrorTelemetry?.serverErrorTopPaths)
|
|
932
|
+
? params.apiRequestErrorTelemetry?.serverErrorTopPaths
|
|
933
|
+
.map((item) => {
|
|
934
|
+
const path = String(item?.path || '').trim().slice(0, 256);
|
|
935
|
+
const count = Math.max(0, Math.floor(Number(item?.count || 0)));
|
|
936
|
+
if (!path || count <= 0) {
|
|
937
|
+
return null;
|
|
938
|
+
}
|
|
939
|
+
return { path, count };
|
|
940
|
+
})
|
|
941
|
+
.filter((item) => Boolean(item))
|
|
942
|
+
.slice(0, 5)
|
|
943
|
+
: [];
|
|
944
|
+
const apiTraceWindowTransientErrorTopPaths = Array.isArray(params.apiRequestErrorTelemetry?.transientErrorTopPaths)
|
|
945
|
+
? params.apiRequestErrorTelemetry?.transientErrorTopPaths
|
|
946
|
+
.map((item) => {
|
|
947
|
+
const path = String(item?.path || '').trim().slice(0, 256);
|
|
948
|
+
const count = Math.max(0, Math.floor(Number(item?.count || 0)));
|
|
949
|
+
if (!path || count <= 0) {
|
|
950
|
+
return null;
|
|
951
|
+
}
|
|
952
|
+
return { path, count };
|
|
953
|
+
})
|
|
954
|
+
.filter((item) => Boolean(item))
|
|
955
|
+
.slice(0, 5)
|
|
956
|
+
: [];
|
|
957
|
+
const apiTraceSlowTopPaths = Array.isArray(params.apiRequestErrorTelemetry?.slowTopPaths)
|
|
958
|
+
? params.apiRequestErrorTelemetry?.slowTopPaths
|
|
959
|
+
.map((item) => {
|
|
960
|
+
const path = String(item?.path || '').trim().slice(0, 256);
|
|
961
|
+
const count = Math.max(0, Math.floor(Number(item?.count || 0)));
|
|
962
|
+
const p95DurationMs = Number(clamp(Number(item?.p95DurationMs || 0), 0, 600000).toFixed(4));
|
|
963
|
+
if (!path || count <= 0) {
|
|
964
|
+
return null;
|
|
965
|
+
}
|
|
966
|
+
return { path, count, p95DurationMs };
|
|
967
|
+
})
|
|
968
|
+
.filter((item) => Boolean(item))
|
|
969
|
+
.slice(0, 5)
|
|
970
|
+
: [];
|
|
971
|
+
const apiServerErrorHotspotPeakCount = apiTraceWindowServerErrorTopPaths.length > 0
|
|
972
|
+
? Number(apiTraceWindowServerErrorTopPaths[0]?.count || 0)
|
|
973
|
+
: 0;
|
|
974
|
+
const apiTransientErrorHotspotPeakCount = apiTraceWindowTransientErrorTopPaths.length > 0
|
|
975
|
+
? Number(apiTraceWindowTransientErrorTopPaths[0]?.count || 0)
|
|
976
|
+
: 0;
|
|
977
|
+
const apiInvalidRequestHotspotPeakCount = apiTraceWindowInvalidRequestTopPaths.length > 0
|
|
978
|
+
? Number(apiTraceWindowInvalidRequestTopPaths[0]?.count || 0)
|
|
979
|
+
: 0;
|
|
980
|
+
const apiInvalidRequestTopRoute = parseRuntimeCapabilityApiTraceRoute(apiTraceWindowInvalidRequestTopPaths[0]?.path || '');
|
|
981
|
+
const apiServerErrorTopRoute = parseRuntimeCapabilityApiTraceRoute(apiTraceWindowServerErrorTopPaths[0]?.path || '');
|
|
982
|
+
const apiTransientErrorTopRoute = parseRuntimeCapabilityApiTraceRoute(apiTraceWindowTransientErrorTopPaths[0]?.path || '');
|
|
983
|
+
const apiLatencyTopRoute = parseRuntimeCapabilityApiTraceRoute(apiTraceSlowTopPaths[0]?.path || '');
|
|
984
|
+
const apiInvalidRequestHotspotsSummary = apiTraceWindowInvalidRequestTopPaths.length > 0
|
|
985
|
+
? apiTraceWindowInvalidRequestTopPaths
|
|
986
|
+
.map((item) => `${item.path}:${item.count}`)
|
|
987
|
+
.join('|')
|
|
988
|
+
: 'none';
|
|
989
|
+
const apiServerErrorTopPathsSummary = apiTraceWindowServerErrorTopPaths.length > 0
|
|
990
|
+
? apiTraceWindowServerErrorTopPaths
|
|
991
|
+
.map((item) => `${item.path}:${item.count}`)
|
|
992
|
+
.join('|')
|
|
993
|
+
: 'none';
|
|
994
|
+
const apiTransientErrorTopPathsSummary = apiTraceWindowTransientErrorTopPaths.length > 0
|
|
995
|
+
? apiTraceWindowTransientErrorTopPaths
|
|
996
|
+
.map((item) => `${item.path}:${item.count}`)
|
|
997
|
+
.join('|')
|
|
998
|
+
: 'none';
|
|
999
|
+
const apiLatencyTopPathsSummary = apiTraceSlowTopPaths.length > 0
|
|
1000
|
+
? apiTraceSlowTopPaths
|
|
1001
|
+
.map((item) => `${item.path}:${item.count}@${Number(item.p95DurationMs || 0).toFixed(2)}ms`)
|
|
1002
|
+
.join('|')
|
|
1003
|
+
: 'none';
|
|
1004
|
+
const defaultApiTraceHintPathPrefix = apiTraceScopePathPrefix || '/api/knowledge';
|
|
1005
|
+
const defaultApiTraceHintMethod = apiTraceScopeMethod || '';
|
|
1006
|
+
const apiInvalidRequestRatioDebugTraceHint = normalizeRuntimeCapabilityDebugTraceHint({
|
|
1007
|
+
pathPrefix: defaultApiTraceHintPathPrefix,
|
|
1008
|
+
statusAtLeast: 400,
|
|
1009
|
+
method: defaultApiTraceHintMethod,
|
|
1010
|
+
errorCode: 'invalid_request',
|
|
1011
|
+
});
|
|
1012
|
+
const apiInvalidRequestHotspotsDebugTraceHint = normalizeRuntimeCapabilityDebugTraceHint({
|
|
1013
|
+
pathPrefix: apiInvalidRequestTopRoute?.pathPrefix || defaultApiTraceHintPathPrefix,
|
|
1014
|
+
statusAtLeast: 400,
|
|
1015
|
+
method: apiInvalidRequestTopRoute?.method || defaultApiTraceHintMethod,
|
|
1016
|
+
errorCode: 'invalid_request',
|
|
1017
|
+
});
|
|
1018
|
+
const apiServerErrorRatioDebugTraceHint = normalizeRuntimeCapabilityDebugTraceHint({
|
|
1019
|
+
pathPrefix: apiServerErrorTopRoute?.pathPrefix || defaultApiTraceHintPathPrefix,
|
|
1020
|
+
statusAtLeast: 500,
|
|
1021
|
+
method: apiServerErrorTopRoute?.method || defaultApiTraceHintMethod,
|
|
1022
|
+
errorCode: '',
|
|
1023
|
+
});
|
|
1024
|
+
const apiServerErrorHotspotsDebugTraceHint = normalizeRuntimeCapabilityDebugTraceHint({
|
|
1025
|
+
pathPrefix: apiServerErrorTopRoute?.pathPrefix || defaultApiTraceHintPathPrefix,
|
|
1026
|
+
statusAtLeast: 500,
|
|
1027
|
+
method: apiServerErrorTopRoute?.method || defaultApiTraceHintMethod,
|
|
1028
|
+
errorCode: '',
|
|
1029
|
+
});
|
|
1030
|
+
const apiTransientErrorRatioDebugTraceHint = normalizeRuntimeCapabilityDebugTraceHint({
|
|
1031
|
+
pathPrefix: apiTransientErrorTopRoute?.pathPrefix || defaultApiTraceHintPathPrefix,
|
|
1032
|
+
statusAtLeast: 400,
|
|
1033
|
+
method: apiTransientErrorTopRoute?.method || defaultApiTraceHintMethod,
|
|
1034
|
+
errorCode: '',
|
|
1035
|
+
});
|
|
1036
|
+
const apiTransientErrorHotspotsDebugTraceHint = normalizeRuntimeCapabilityDebugTraceHint({
|
|
1037
|
+
pathPrefix: apiTransientErrorTopRoute?.pathPrefix || defaultApiTraceHintPathPrefix,
|
|
1038
|
+
statusAtLeast: 400,
|
|
1039
|
+
method: apiTransientErrorTopRoute?.method || defaultApiTraceHintMethod,
|
|
1040
|
+
errorCode: '',
|
|
1041
|
+
});
|
|
1042
|
+
const apiLatencyRatioDebugTraceHint = normalizeRuntimeCapabilityDebugTraceHint({
|
|
1043
|
+
pathPrefix: apiLatencyTopRoute?.pathPrefix || defaultApiTraceHintPathPrefix,
|
|
1044
|
+
statusAtLeast: 0,
|
|
1045
|
+
method: apiLatencyTopRoute?.method || defaultApiTraceHintMethod,
|
|
1046
|
+
errorCode: '',
|
|
1047
|
+
});
|
|
1048
|
+
const apiLatencyHotspotsDebugTraceHint = normalizeRuntimeCapabilityDebugTraceHint({
|
|
1049
|
+
pathPrefix: apiLatencyTopRoute?.pathPrefix || defaultApiTraceHintPathPrefix,
|
|
1050
|
+
statusAtLeast: 0,
|
|
1051
|
+
method: apiLatencyTopRoute?.method || defaultApiTraceHintMethod,
|
|
1052
|
+
errorCode: '',
|
|
1053
|
+
});
|
|
1054
|
+
const apiInvalidRequestObserved = [
|
|
1055
|
+
`scopePathPrefix=${apiTraceScopePathPrefix || '<all>'}`,
|
|
1056
|
+
`scopeMethod=${apiTraceScopeMethod || '<all>'}`,
|
|
1057
|
+
`requests=${apiTraceWindowRequests}`,
|
|
1058
|
+
`errors=${apiTraceWindowErrors}`,
|
|
1059
|
+
`invalidRequest=${apiTraceWindowInvalidRequests}`,
|
|
1060
|
+
`invalid/error=${apiTraceWindowInvalidRequestRatioPct}%`,
|
|
1061
|
+
`invalid/total=${apiTraceWindowInvalidRequestToTotalRatioPct}%`,
|
|
1062
|
+
`invalidTopPaths=${apiInvalidRequestHotspotsSummary}`,
|
|
1063
|
+
].join(', ');
|
|
1064
|
+
const apiServerErrorObserved = [
|
|
1065
|
+
`scopePathPrefix=${apiTraceScopePathPrefix || '<all>'}`,
|
|
1066
|
+
`scopeMethod=${apiTraceScopeMethod || '<all>'}`,
|
|
1067
|
+
`requests=${apiTraceWindowRequests}`,
|
|
1068
|
+
`serverErrors=${apiTraceWindowServerErrors}`,
|
|
1069
|
+
`server/total=${apiTraceWindowServerErrorRatioPct}%`,
|
|
1070
|
+
`serverTopPaths=${apiServerErrorTopPathsSummary}`,
|
|
1071
|
+
].join(', ');
|
|
1072
|
+
const apiTransientErrorObserved = [
|
|
1073
|
+
`scopePathPrefix=${apiTraceScopePathPrefix || '<all>'}`,
|
|
1074
|
+
`scopeMethod=${apiTraceScopeMethod || '<all>'}`,
|
|
1075
|
+
`requests=${apiTraceWindowRequests}`,
|
|
1076
|
+
`transientErrors=${apiTraceWindowTransientErrors}`,
|
|
1077
|
+
`transient/total=${apiTraceWindowTransientErrorRatioPct}%`,
|
|
1078
|
+
`transientTopPaths=${apiTransientErrorTopPathsSummary}`,
|
|
1079
|
+
].join(', ');
|
|
1080
|
+
const apiLatencyObserved = [
|
|
1081
|
+
`scopePathPrefix=${apiTraceScopePathPrefix || '<all>'}`,
|
|
1082
|
+
`scopeMethod=${apiTraceScopeMethod || '<all>'}`,
|
|
1083
|
+
`requests=${apiTraceWindowRequests}`,
|
|
1084
|
+
`avg=${apiTraceAverageDurationMs}ms`,
|
|
1085
|
+
`p95=${apiTraceP95DurationMs}ms`,
|
|
1086
|
+
`slowTopPaths=${apiLatencyTopPathsSummary}`,
|
|
1087
|
+
].join(', ');
|
|
1088
|
+
const apiLatencyHotspotPeakP95Ms = apiTraceSlowTopPaths.length > 0
|
|
1089
|
+
? Number(apiTraceSlowTopPaths[0]?.p95DurationMs || 0)
|
|
1090
|
+
: 0;
|
|
1091
|
+
const trendStatusRaw = String(params.learningQualityTrend?.status || '').trim().toLowerCase();
|
|
1092
|
+
const trendStatus = (trendStatusRaw === 'improving'
|
|
1093
|
+
|| trendStatusRaw === 'stable'
|
|
1094
|
+
|| trendStatusRaw === 'regressing'
|
|
1095
|
+
|| trendStatusRaw === 'insufficient_data') ? trendStatusRaw : 'unknown';
|
|
1096
|
+
const trendScoreRaw = Number(params.learningQualityTrend?.score || 0);
|
|
1097
|
+
const trendScore = Number((Number.isFinite(trendScoreRaw) ? trendScoreRaw : 0).toFixed(4));
|
|
1098
|
+
const trendConfidenceRaw = Number(params.learningQualityTrend?.confidence || 0);
|
|
1099
|
+
const trendConfidence = Number((Number.isFinite(trendConfidenceRaw)
|
|
1100
|
+
? clamp(trendConfidenceRaw, 0, 1)
|
|
1101
|
+
: 0).toFixed(4));
|
|
1102
|
+
const trendReason = String(params.learningQualityTrend?.reason || '').trim();
|
|
1103
|
+
const trendObserved = [
|
|
1104
|
+
`status=${trendStatus}`,
|
|
1105
|
+
`score=${trendScore}`,
|
|
1106
|
+
`confidence=${trendConfidence}`,
|
|
1107
|
+
trendReason ? `reason=${trendReason}` : '',
|
|
1108
|
+
].filter(Boolean).join(', ');
|
|
1109
|
+
const sessionPlanQualityRecords = Math.max(0, Math.floor(Number(params.sessionPlanQualityHistory?.summary?.totalRecords || 0)));
|
|
1110
|
+
const sessionPlanQualityPassRatePct = Number(clamp(Number(params.sessionPlanQualityHistory?.summary?.overallPassRatePct || 0), 0, 100).toFixed(4));
|
|
1111
|
+
const sessionPlanQualityFailureStreak = Math.max(0, Math.floor(Number(params.sessionPlanQualityHistory?.summary?.consecutiveFailureCount || 0)));
|
|
1112
|
+
const sessionPlanQualityCommonFailedGates = Array.isArray(params.sessionPlanQualityHistory?.summary?.commonFailedGates)
|
|
1113
|
+
? params.sessionPlanQualityHistory?.summary?.commonFailedGates
|
|
1114
|
+
.map((item) => {
|
|
1115
|
+
const gateId = String(item?.gateId || '').trim();
|
|
1116
|
+
const count = Math.max(0, Math.floor(Number(item?.count || 0)));
|
|
1117
|
+
if (!gateId || count <= 0) {
|
|
1118
|
+
return '';
|
|
1119
|
+
}
|
|
1120
|
+
return `${gateId}:${count}`;
|
|
1121
|
+
})
|
|
1122
|
+
.filter((item) => item.length > 0)
|
|
1123
|
+
.slice(0, 3)
|
|
1124
|
+
.join('|')
|
|
1125
|
+
: '';
|
|
1126
|
+
const sessionPlanTrendStatusRaw = String(params.sessionPlanQualityTrend?.status || '').trim().toLowerCase();
|
|
1127
|
+
const sessionPlanTrendStatus = (sessionPlanTrendStatusRaw === 'improving'
|
|
1128
|
+
|| sessionPlanTrendStatusRaw === 'stable'
|
|
1129
|
+
|| sessionPlanTrendStatusRaw === 'regressing'
|
|
1130
|
+
|| sessionPlanTrendStatusRaw === 'insufficient_data') ? sessionPlanTrendStatusRaw : 'unknown';
|
|
1131
|
+
const sessionPlanTrendScore = Number((Number.isFinite(Number(params.sessionPlanQualityTrend?.score))
|
|
1132
|
+
? Number(params.sessionPlanQualityTrend?.score)
|
|
1133
|
+
: 0).toFixed(4));
|
|
1134
|
+
const sessionPlanTrendConfidence = Number((Number.isFinite(Number(params.sessionPlanQualityTrend?.confidence))
|
|
1135
|
+
? clamp(Number(params.sessionPlanQualityTrend?.confidence), 0, 1)
|
|
1136
|
+
: 0).toFixed(4));
|
|
1137
|
+
const sessionPlanTrendReason = String(params.sessionPlanQualityTrend?.reason || '').trim();
|
|
1138
|
+
const sessionPlanTrendObserved = [
|
|
1139
|
+
`status=${sessionPlanTrendStatus}`,
|
|
1140
|
+
`score=${sessionPlanTrendScore}`,
|
|
1141
|
+
`confidence=${sessionPlanTrendConfidence}`,
|
|
1142
|
+
sessionPlanTrendReason ? `reason=${sessionPlanTrendReason}` : '',
|
|
1143
|
+
].filter(Boolean).join(', ');
|
|
1144
|
+
const memoryPolicyStatusRaw = String(params.memoryPolicyDiagnostics?.summary?.status || '').trim().toLowerCase();
|
|
1145
|
+
const memoryPolicyStatus = (memoryPolicyStatusRaw === 'healthy'
|
|
1146
|
+
|| memoryPolicyStatusRaw === 'watch'
|
|
1147
|
+
|| memoryPolicyStatusRaw === 'risk'
|
|
1148
|
+
|| memoryPolicyStatusRaw === 'insufficient_data') ? memoryPolicyStatusRaw : 'unknown';
|
|
1149
|
+
const memoryPolicyHealthScore = Number((Number.isFinite(Number(params.memoryPolicyDiagnostics?.summary?.healthScore))
|
|
1150
|
+
? clamp(Number(params.memoryPolicyDiagnostics?.summary?.healthScore), 0, 100)
|
|
1151
|
+
: 0).toFixed(4));
|
|
1152
|
+
const memoryPolicyTotalEntries = Math.max(0, Math.floor(Number(params.memoryPolicyDiagnostics?.summary?.totalEntries || 0)));
|
|
1153
|
+
const memoryPolicyExpiredEntries = Math.max(0, Math.floor(Number(params.memoryPolicyDiagnostics?.summary?.expiredEntries || 0)));
|
|
1154
|
+
const memoryPolicyStaleEntries = Math.max(0, Math.floor(Number(params.memoryPolicyDiagnostics?.summary?.staleEntries || 0)));
|
|
1155
|
+
const memoryPolicyLowConfidenceEntries = Math.max(0, Math.floor(Number(params.memoryPolicyDiagnostics?.summary?.lowConfidenceEntries || 0)));
|
|
1156
|
+
const memoryPolicyReason = String(params.memoryPolicyDiagnostics?.summary?.reason || '').trim();
|
|
1157
|
+
const memoryPolicyObserved = [
|
|
1158
|
+
`status=${memoryPolicyStatus}`,
|
|
1159
|
+
`score=${memoryPolicyHealthScore}`,
|
|
1160
|
+
`entries=${memoryPolicyTotalEntries}`,
|
|
1161
|
+
`expired=${memoryPolicyExpiredEntries}`,
|
|
1162
|
+
`stale=${memoryPolicyStaleEntries}`,
|
|
1163
|
+
`lowConfidence=${memoryPolicyLowConfidenceEntries}`,
|
|
1164
|
+
memoryPolicyReason ? `reason=${memoryPolicyReason}` : '',
|
|
1165
|
+
].filter(Boolean).join(', ');
|
|
1166
|
+
const memoryPolicyTrendStatusRaw = String(params.memoryPolicyTrend?.status || '').trim().toLowerCase();
|
|
1167
|
+
const memoryPolicyTrendStatus = (memoryPolicyTrendStatusRaw === 'improving'
|
|
1168
|
+
|| memoryPolicyTrendStatusRaw === 'stable'
|
|
1169
|
+
|| memoryPolicyTrendStatusRaw === 'regressing'
|
|
1170
|
+
|| memoryPolicyTrendStatusRaw === 'insufficient_data') ? memoryPolicyTrendStatusRaw : 'unknown';
|
|
1171
|
+
const memoryPolicyTrendScore = Number((Number.isFinite(Number(params.memoryPolicyTrend?.score))
|
|
1172
|
+
? Number(params.memoryPolicyTrend?.score)
|
|
1173
|
+
: 0).toFixed(4));
|
|
1174
|
+
const memoryPolicyTrendConfidence = Number((Number.isFinite(Number(params.memoryPolicyTrend?.confidence))
|
|
1175
|
+
? clamp(Number(params.memoryPolicyTrend?.confidence), 0, 1)
|
|
1176
|
+
: 0).toFixed(4));
|
|
1177
|
+
const memoryPolicyTrendReason = String(params.memoryPolicyTrend?.reason || '').trim();
|
|
1178
|
+
const memoryPolicyTrendObserved = [
|
|
1179
|
+
`status=${memoryPolicyTrendStatus}`,
|
|
1180
|
+
`score=${memoryPolicyTrendScore}`,
|
|
1181
|
+
`confidence=${memoryPolicyTrendConfidence}`,
|
|
1182
|
+
memoryPolicyTrendReason ? `reason=${memoryPolicyTrendReason}` : '',
|
|
1183
|
+
].filter(Boolean).join(', ');
|
|
1184
|
+
const knowledgeStalenessSummary = params.knowledgeStalenessDiagnostics?.summary || null;
|
|
1185
|
+
const knowledgeStalenessEvaluatedDocuments = Math.max(0, Math.floor(Number(knowledgeStalenessSummary?.evaluatedDocuments || 0)));
|
|
1186
|
+
const knowledgeStalenessStaleDocuments = Math.max(0, Math.floor(Number(knowledgeStalenessSummary?.staleDocuments || 0)));
|
|
1187
|
+
const knowledgeStalenessFreshnessRatioPct = Number(clamp(Number(knowledgeStalenessSummary?.freshnessRatioPct || 0), 0, 100).toFixed(4));
|
|
1188
|
+
const knowledgeStalenessHashMismatchDocuments = Math.max(0, Math.floor(Number(knowledgeStalenessSummary?.hashMismatchDocuments || 0)));
|
|
1189
|
+
const knowledgeStalenessMissingSourceDocuments = Math.max(0, Math.floor(Number(knowledgeStalenessSummary?.missingSourceDocuments || 0)));
|
|
1190
|
+
const knowledgeStalenessReadErrorDocuments = Math.max(0, Math.floor(Number(knowledgeStalenessSummary?.readErrorDocuments || 0)));
|
|
1191
|
+
const knowledgeStalenessObserved = [
|
|
1192
|
+
`evaluated=${knowledgeStalenessEvaluatedDocuments}`,
|
|
1193
|
+
`stale=${knowledgeStalenessStaleDocuments}`,
|
|
1194
|
+
`freshness=${knowledgeStalenessFreshnessRatioPct}%`,
|
|
1195
|
+
`mismatch=${knowledgeStalenessHashMismatchDocuments}`,
|
|
1196
|
+
`missing=${knowledgeStalenessMissingSourceDocuments}`,
|
|
1197
|
+
`readError=${knowledgeStalenessReadErrorDocuments}`,
|
|
1198
|
+
].join(', ');
|
|
1199
|
+
const sessionActionExecutionCount = Math.max(0, Math.floor(Number(params.sessionActionTelemetry?.executionCount || 0)));
|
|
1200
|
+
const sessionMemoryPersistedCount = Math.max(0, Math.floor(Number(params.sessionActionTelemetry?.memoryPersistedCount || 0)));
|
|
1201
|
+
const sessionMemoryPromotionAppliedCount = Math.max(0, Math.floor(Number(params.sessionActionTelemetry?.memoryPromotionAppliedCount || 0)));
|
|
1202
|
+
const sessionMemoryPromotionCount = Math.max(0, Math.floor(Number(params.sessionActionTelemetry?.memoryPromotionCount || 0)));
|
|
1203
|
+
const sessionMemoryPromotionCoveragePct = Number(((sessionMemoryPromotionAppliedCount / Math.max(1, sessionMemoryPersistedCount)) * 100).toFixed(4));
|
|
1204
|
+
const sessionStrategyTotalRecords = Math.max(0, Math.floor(Number(params.sessionStrategyTelemetry?.totalRecords || 0)));
|
|
1205
|
+
const sessionStrategyStrategyRecords = Math.max(0, Math.floor(Number(params.sessionStrategyTelemetry?.strategyRecords || 0)));
|
|
1206
|
+
const sessionStrategyTrendAutoSelectionSharePct = Number(clamp(Number(params.sessionStrategyTelemetry?.trendAutoSelectionSharePct || 0), 0, 100).toFixed(4));
|
|
1207
|
+
const sessionStrategyTrendAutoAverageMasteryDeltaPct = Number(clamp(Number(params.sessionStrategyTelemetry?.trendAutoAverageMasteryDeltaPct || 0), -100, 100).toFixed(4));
|
|
1208
|
+
const sessionStrategyTrendAutoNegativeRatioPct = Number(clamp(Number(params.sessionStrategyTelemetry?.trendAutoNegativeRatioPct || 0), 0, 100).toFixed(4));
|
|
1209
|
+
const sessionStrategyModeFallbackSelectionSharePct = Number(clamp(Number(params.sessionStrategyTelemetry?.modeFallbackSelectionSharePct || 0), 0, 100).toFixed(4));
|
|
1210
|
+
const sessionStrategySelectionSourceExplicitCount = Math.max(0, Math.floor(Number(params.sessionStrategyTelemetry?.selectionSourceCounts?.explicit_request || 0)));
|
|
1211
|
+
const sessionStrategySelectionSourceTrendCount = Math.max(0, Math.floor(Number(params.sessionStrategyTelemetry?.selectionSourceCounts?.strategy_trend || 0)));
|
|
1212
|
+
const sessionStrategySelectionSourceFallbackCount = Math.max(0, Math.floor(Number(params.sessionStrategyTelemetry?.selectionSourceCounts?.mode_fallback || 0)));
|
|
1213
|
+
const sessionStrategySelectionSourceUnknownCount = Math.max(0, Math.floor(Number(params.sessionStrategyTelemetry?.selectionSourceCounts?.unknown || 0)));
|
|
1214
|
+
const sessionStrategyBreakdown = Array.isArray(params.sessionStrategyTelemetry?.strategyBreakdown)
|
|
1215
|
+
? params.sessionStrategyTelemetry.strategyBreakdown.reduce((entries, item) => {
|
|
1216
|
+
const strategyRaw = String(item?.strategy || '').trim().toLowerCase();
|
|
1217
|
+
const strategy = (strategyRaw === 'balanced'
|
|
1218
|
+
|| strategyRaw === 'mastery_recovery'
|
|
1219
|
+
|| strategyRaw === 'exploration_boost') ? strategyRaw : null;
|
|
1220
|
+
if (!strategy) {
|
|
1221
|
+
return entries;
|
|
1222
|
+
}
|
|
1223
|
+
entries.push({
|
|
1224
|
+
strategy,
|
|
1225
|
+
executions: Math.max(0, Math.floor(Number(item?.executions || 0))),
|
|
1226
|
+
averageMasteryDeltaPct: Number(clamp(Number(item?.averageMasteryDeltaPct || 0), -100, 100).toFixed(4)),
|
|
1227
|
+
positiveRatioPct: Number(clamp(Number(item?.positiveRatioPct || 0), 0, 100).toFixed(4)),
|
|
1228
|
+
negativeRatioPct: Number(clamp(Number(item?.negativeRatioPct || 0), 0, 100).toFixed(4)),
|
|
1229
|
+
});
|
|
1230
|
+
return entries;
|
|
1231
|
+
}, [])
|
|
1232
|
+
: [];
|
|
1233
|
+
const sessionStrategyTopAverageEntry = sessionStrategyBreakdown
|
|
1234
|
+
.slice()
|
|
1235
|
+
.sort((left, right) => {
|
|
1236
|
+
if (right.averageMasteryDeltaPct !== left.averageMasteryDeltaPct) {
|
|
1237
|
+
return right.averageMasteryDeltaPct - left.averageMasteryDeltaPct;
|
|
1238
|
+
}
|
|
1239
|
+
if (right.executions !== left.executions) {
|
|
1240
|
+
return right.executions - left.executions;
|
|
1241
|
+
}
|
|
1242
|
+
return left.strategy.localeCompare(right.strategy);
|
|
1243
|
+
})[0] || null;
|
|
1244
|
+
const sessionStrategyTopAverageStrategy = sessionStrategyTopAverageEntry
|
|
1245
|
+
? sessionStrategyTopAverageEntry.strategy
|
|
1246
|
+
: 'unknown';
|
|
1247
|
+
const sessionStrategyTopAverageMasteryDeltaPct = Number(clamp(Number(sessionStrategyTopAverageEntry?.averageMasteryDeltaPct || 0), -100, 100).toFixed(4));
|
|
1248
|
+
const sessionStrategyTopAverageNegativeRatioPct = Number(clamp(Number(sessionStrategyTopAverageEntry?.negativeRatioPct || 0), 0, 100).toFixed(4));
|
|
1249
|
+
const sessionStrategyObserved = [
|
|
1250
|
+
`records=${sessionStrategyTotalRecords}`,
|
|
1251
|
+
`strategyRecords=${sessionStrategyStrategyRecords}`,
|
|
1252
|
+
`source(explicit/trend/fallback/unknown)=`
|
|
1253
|
+
+ `${sessionStrategySelectionSourceExplicitCount}/${sessionStrategySelectionSourceTrendCount}/`
|
|
1254
|
+
+ `${sessionStrategySelectionSourceFallbackCount}/${sessionStrategySelectionSourceUnknownCount}`,
|
|
1255
|
+
`trendAutoShare=${sessionStrategyTrendAutoSelectionSharePct}%`,
|
|
1256
|
+
`trendAutoAvgDelta=${sessionStrategyTrendAutoAverageMasteryDeltaPct}%`,
|
|
1257
|
+
`trendAutoNegative=${sessionStrategyTrendAutoNegativeRatioPct}%`,
|
|
1258
|
+
`modeFallbackShare=${sessionStrategyModeFallbackSelectionSharePct}%`,
|
|
1259
|
+
`topAverage=${sessionStrategyTopAverageStrategy}@${sessionStrategyTopAverageMasteryDeltaPct}%`
|
|
1260
|
+
+ ` (negative=${sessionStrategyTopAverageNegativeRatioPct}%)`,
|
|
1261
|
+
].join(', ');
|
|
1262
|
+
const tutorAdaptersTotal = Math.max(0, Math.floor(Number(params.tutorAdapterTelemetry?.summary?.totalAdapters || 0)));
|
|
1263
|
+
const tutorAdaptersActive = Math.max(0, Math.floor(Number(params.tutorAdapterTelemetry?.summary?.activeAdapters || 0)));
|
|
1264
|
+
const tutorRequests = Math.max(0, Math.floor(Number(params.tutorAdapterTelemetry?.summary?.totalRequests || 0)));
|
|
1265
|
+
const tutorAcceptedResponses = Math.max(0, Math.floor(Number(params.tutorAdapterTelemetry?.summary?.acceptedResponses || 0)));
|
|
1266
|
+
const tutorDowngradedResponses = Math.max(0, Math.floor(Number(params.tutorAdapterTelemetry?.summary?.downgradedResponses || 0)));
|
|
1267
|
+
const tutorFailedResponses = Math.max(0, Math.floor(Number(params.tutorAdapterTelemetry?.summary?.failedResponses || 0)));
|
|
1268
|
+
const tutorProviderFallbackResponses = Math.max(0, Math.floor(Number(params.tutorAdapterTelemetry?.summary?.providerFallbackResponses || 0)));
|
|
1269
|
+
const tutorProviderFallbackRatioPct = Number((Number.isFinite(Number(params.tutorAdapterTelemetry?.summary?.providerFallbackRatioPct))
|
|
1270
|
+
? clamp(Number(params.tutorAdapterTelemetry?.summary?.providerFallbackRatioPct), 0, 100)
|
|
1271
|
+
: clamp((tutorProviderFallbackResponses / Math.max(1, tutorRequests)) * 100, 0, 100)).toFixed(4));
|
|
1272
|
+
const tutorAverageProviderAttemptCount = Number((Number.isFinite(Number(params.tutorAdapterTelemetry?.summary?.averageProviderAttemptCount))
|
|
1273
|
+
? clamp(Number(params.tutorAdapterTelemetry?.summary?.averageProviderAttemptCount), 1, 20)
|
|
1274
|
+
: clamp(1 + (tutorProviderFallbackResponses / Math.max(1, tutorRequests)), 1, 20)).toFixed(4));
|
|
1275
|
+
const tutorProviderBreakdown = Array.isArray(params.tutorTraceDiagnostics?.providerBreakdown)
|
|
1276
|
+
? params.tutorTraceDiagnostics?.providerBreakdown
|
|
1277
|
+
.map((item) => ({
|
|
1278
|
+
providerName: String(item?.providerName || '').trim() || 'unknown',
|
|
1279
|
+
traces: Math.max(0, Math.floor(Number(item?.traces || 0))),
|
|
1280
|
+
fallbackTraces: Math.max(0, Math.floor(Number(item?.fallbackTraces || 0))),
|
|
1281
|
+
}))
|
|
1282
|
+
.filter((item) => item.traces > 0)
|
|
1283
|
+
: [];
|
|
1284
|
+
const tutorProviderCount = tutorProviderBreakdown.length;
|
|
1285
|
+
const tutorProviderTotalTraces = tutorProviderBreakdown.reduce((sum, item) => sum + item.traces, 0);
|
|
1286
|
+
const tutorFallbackTraceCountFromSummary = Math.max(0, Math.floor(Number(params.tutorTraceDiagnostics?.summary?.fallbackTraces || 0)));
|
|
1287
|
+
const tutorFallbackTraceCountFromBreakdown = tutorProviderBreakdown.reduce((sum, item) => sum + item.fallbackTraces, 0);
|
|
1288
|
+
const tutorFallbackTraceCount = Math.max(tutorFallbackTraceCountFromSummary, tutorFallbackTraceCountFromBreakdown);
|
|
1289
|
+
const tutorDominantProvider = tutorProviderBreakdown
|
|
1290
|
+
.slice()
|
|
1291
|
+
.sort((left, right) => {
|
|
1292
|
+
if (right.traces !== left.traces) {
|
|
1293
|
+
return right.traces - left.traces;
|
|
1294
|
+
}
|
|
1295
|
+
return left.providerName.localeCompare(right.providerName);
|
|
1296
|
+
})[0];
|
|
1297
|
+
const tutorDominantProviderName = tutorDominantProvider?.providerName || '';
|
|
1298
|
+
const tutorDominantProviderSharePct = Number(clamp(((Number(tutorDominantProvider?.traces || 0) / Math.max(1, tutorProviderTotalTraces)) * 100), 0, 100).toFixed(4));
|
|
1299
|
+
const tutorDominantFallbackProvider = tutorProviderBreakdown
|
|
1300
|
+
.filter((item) => item.fallbackTraces > 0)
|
|
1301
|
+
.sort((left, right) => {
|
|
1302
|
+
if (right.fallbackTraces !== left.fallbackTraces) {
|
|
1303
|
+
return right.fallbackTraces - left.fallbackTraces;
|
|
1304
|
+
}
|
|
1305
|
+
return left.providerName.localeCompare(right.providerName);
|
|
1306
|
+
})[0];
|
|
1307
|
+
const tutorDominantFallbackProviderName = tutorDominantFallbackProvider?.providerName || '';
|
|
1308
|
+
const tutorDominantFallbackProviderSharePct = Number(clamp(((Number(tutorDominantFallbackProvider?.fallbackTraces || 0) / Math.max(1, tutorFallbackTraceCount)) * 100), 0, 100).toFixed(4));
|
|
1309
|
+
const tutorProviderTrendProviders = Array.isArray(params.tutorProviderTrendDiagnostics?.providers)
|
|
1310
|
+
? params.tutorProviderTrendDiagnostics?.providers
|
|
1311
|
+
.map((item) => {
|
|
1312
|
+
const providerName = String(item?.providerName || '').trim() || 'unknown';
|
|
1313
|
+
const trendStatusRaw = String(item?.trendStatus || '').trim().toLowerCase();
|
|
1314
|
+
const trendStatus = (trendStatusRaw === 'improving'
|
|
1315
|
+
|| trendStatusRaw === 'stable'
|
|
1316
|
+
|| trendStatusRaw === 'regressing')
|
|
1317
|
+
? trendStatusRaw
|
|
1318
|
+
: 'insufficient_data';
|
|
1319
|
+
return {
|
|
1320
|
+
providerName,
|
|
1321
|
+
trendStatus,
|
|
1322
|
+
trendScore: Number(clamp(Number(item?.trendScore || 0), -100, 100).toFixed(4)),
|
|
1323
|
+
trendConfidence: Number(clamp(Number(item?.trendConfidence || 0), 0, 100).toFixed(4)),
|
|
1324
|
+
fallbackRatioPct: Number(clamp(Number(item?.fallbackRatioPct || 0), 0, 100).toFixed(4)),
|
|
1325
|
+
};
|
|
1326
|
+
})
|
|
1327
|
+
.filter((item) => item.providerName)
|
|
1328
|
+
: [];
|
|
1329
|
+
const tutorProviderTrendRegressingProviders = tutorProviderTrendProviders
|
|
1330
|
+
.filter((item) => item.trendStatus === 'regressing')
|
|
1331
|
+
.sort((left, right) => {
|
|
1332
|
+
if (right.trendScore !== left.trendScore) {
|
|
1333
|
+
return right.trendScore - left.trendScore;
|
|
1334
|
+
}
|
|
1335
|
+
if (right.trendConfidence !== left.trendConfidence) {
|
|
1336
|
+
return right.trendConfidence - left.trendConfidence;
|
|
1337
|
+
}
|
|
1338
|
+
return left.providerName.localeCompare(right.providerName);
|
|
1339
|
+
});
|
|
1340
|
+
const tutorProviderTrendRegressingCount = tutorProviderTrendRegressingProviders.length;
|
|
1341
|
+
const tutorProviderTrendImprovingCount = tutorProviderTrendProviders.filter((item) => item.trendStatus === 'improving').length;
|
|
1342
|
+
const tutorProviderTrendInsufficientDataCount = tutorProviderTrendProviders
|
|
1343
|
+
.filter((item) => item.trendStatus === 'insufficient_data')
|
|
1344
|
+
.length;
|
|
1345
|
+
const tutorProviderTrendTopRegressingProvider = tutorProviderTrendRegressingProviders[0] || null;
|
|
1346
|
+
const tutorProviderTrendTopRegressingProviderName = tutorProviderTrendTopRegressingProvider?.providerName || '';
|
|
1347
|
+
const tutorProviderTrendTopRegressingScore = Number(clamp(Number(tutorProviderTrendTopRegressingProvider?.trendScore || 0), -100, 100).toFixed(4));
|
|
1348
|
+
const tutorProviderTrendTopRegressingConfidence = Number(clamp(Number(tutorProviderTrendTopRegressingProvider?.trendConfidence || 0), 0, 100).toFixed(4));
|
|
1349
|
+
const tutorProviderTrendRecommendedFocusProviderName = String(params.tutorProviderTrendDiagnostics?.summary?.recommendedFocusProviderName
|
|
1350
|
+
|| tutorProviderTrendTopRegressingProviderName
|
|
1351
|
+
|| '').trim();
|
|
1352
|
+
const tutorProviderTrendHistoryRecords = Math.max(Math.floor(Number(params.tutorProviderTrendHistory?.summary?.totalRecords || 0)), Array.isArray(params.tutorProviderTrendHistory?.records)
|
|
1353
|
+
? params.tutorProviderTrendHistory?.records.length
|
|
1354
|
+
: 0);
|
|
1355
|
+
const tutorProviderTrendHistoryRegressingRecords = Math.max(Math.floor(Number(params.tutorProviderTrendHistory?.summary?.regressingRecords || 0)), Array.isArray(params.tutorProviderTrendHistory?.records)
|
|
1356
|
+
? params.tutorProviderTrendHistory.records
|
|
1357
|
+
.filter((item) => String(item?.trendStatus || '').trim().toLowerCase() === 'regressing')
|
|
1358
|
+
.length
|
|
1359
|
+
: 0);
|
|
1360
|
+
const tutorProviderTrendHistoryStableRecords = Math.max(Math.floor(Number(params.tutorProviderTrendHistory?.summary?.stableRecords || 0)), Array.isArray(params.tutorProviderTrendHistory?.records)
|
|
1361
|
+
? params.tutorProviderTrendHistory.records
|
|
1362
|
+
.filter((item) => String(item?.trendStatus || '').trim().toLowerCase() === 'stable')
|
|
1363
|
+
.length
|
|
1364
|
+
: 0);
|
|
1365
|
+
const tutorProviderTrendHistoryImprovingRecords = Math.max(Math.floor(Number(params.tutorProviderTrendHistory?.summary?.improvingRecords || 0)), Array.isArray(params.tutorProviderTrendHistory?.records)
|
|
1366
|
+
? params.tutorProviderTrendHistory.records
|
|
1367
|
+
.filter((item) => String(item?.trendStatus || '').trim().toLowerCase() === 'improving')
|
|
1368
|
+
.length
|
|
1369
|
+
: 0);
|
|
1370
|
+
const tutorProviderTrendHistoryInsufficientDataRecords = Math.max(Math.floor(Number(params.tutorProviderTrendHistory?.summary?.insufficientDataRecords || 0)), Array.isArray(params.tutorProviderTrendHistory?.records)
|
|
1371
|
+
? params.tutorProviderTrendHistory.records
|
|
1372
|
+
.filter((item) => String(item?.trendStatus || '').trim().toLowerCase() === 'insufficient_data')
|
|
1373
|
+
.length
|
|
1374
|
+
: 0);
|
|
1375
|
+
const tutorFailedRatioPct = Number(((tutorFailedResponses / Math.max(1, tutorRequests)) * 100).toFixed(4));
|
|
1376
|
+
const tutorDowngradedRatioPct = Number(((tutorDowngradedResponses / Math.max(1, tutorRequests)) * 100).toFixed(4));
|
|
1377
|
+
const tutorAverageConfidence = Number((Number.isFinite(Number(params.tutorAdapterTelemetry?.summary?.averageConfidence))
|
|
1378
|
+
? clamp(Number(params.tutorAdapterTelemetry?.summary?.averageConfidence), 0, 1)
|
|
1379
|
+
: 0).toFixed(4));
|
|
1380
|
+
const tutorRoutingEnabled = params.tutorRoutingConfig?.enabled !== false;
|
|
1381
|
+
const tutorRoutingMinSamples = Math.max(1, Math.floor(Number(params.tutorRoutingConfig?.minSamples || 1)));
|
|
1382
|
+
const tutorRoutingMaxFailedRatioPct = Number(clamp(Number(params.tutorRoutingConfig?.maxFailedRatioPct || 100), 0, 100).toFixed(4));
|
|
1383
|
+
const tutorRoutingMaxDowngradedRatioPct = Number(clamp(Number(params.tutorRoutingConfig?.maxDowngradedRatioPct || 100), 0, 100).toFixed(4));
|
|
1384
|
+
const tutorRoutingMinAverageConfidence = Number(clamp(Number(params.tutorRoutingConfig?.minAverageConfidence || 0), 0, 1).toFixed(4));
|
|
1385
|
+
const tutorRoutingPreferredModeRaw = String(params.tutorRoutingConfig?.preferredMode || '').trim().toLowerCase();
|
|
1386
|
+
const tutorRoutingPreferredMode = (tutorRoutingPreferredModeRaw === 'local'
|
|
1387
|
+
|| tutorRoutingPreferredModeRaw === 'cloud') ? tutorRoutingPreferredModeRaw : 'auto';
|
|
1388
|
+
const tutorRoutingAdapterTimeoutRaw = Number(params.tutorRoutingConfig?.adapterTimeoutMs);
|
|
1389
|
+
const tutorRoutingAdapterTimeoutMs = Math.max(100, Math.min(120000, Math.floor(Number.isFinite(tutorRoutingAdapterTimeoutRaw) ? tutorRoutingAdapterTimeoutRaw : 15000)));
|
|
1390
|
+
const tutorLastRoutingStrategyRaw = String(params.tutorAdapterTelemetry?.summary?.lastRoutingStrategy || '').trim();
|
|
1391
|
+
const tutorLastRoutingStrategy = (tutorLastRoutingStrategyRaw === 'explicit_adapter_id'
|
|
1392
|
+
|| tutorLastRoutingStrategyRaw === 'explicit_provider_mode'
|
|
1393
|
+
|| tutorLastRoutingStrategyRaw === 'adaptive_health_routing'
|
|
1394
|
+
|| tutorLastRoutingStrategyRaw === 'fallback_default') ? tutorLastRoutingStrategyRaw : 'unknown';
|
|
1395
|
+
const tutorLastRoutingReason = String(params.tutorAdapterTelemetry?.summary?.lastRoutingReason || '').trim().slice(0, 160);
|
|
1396
|
+
const tutorLastRoutingScore = Number((Number.isFinite(Number(params.tutorAdapterTelemetry?.summary?.lastRoutingScore))
|
|
1397
|
+
? clamp(Number(params.tutorAdapterTelemetry?.summary?.lastRoutingScore), 0, 1)
|
|
1398
|
+
: 0).toFixed(4));
|
|
1399
|
+
const tutorRoutingDynamicPreferredModeRaw = String(params.tutorAdapterTelemetry?.summary?.lastRoutingDynamicPreferredMode || '').trim();
|
|
1400
|
+
const hasStructuredTutorRoutingDynamicPreferredMode = tutorRoutingDynamicPreferredModeRaw.length > 0;
|
|
1401
|
+
const tutorRoutingDynamicModeReasonRaw = String(params.tutorAdapterTelemetry?.summary?.lastRoutingDynamicModeReason || '').trim().slice(0, 220);
|
|
1402
|
+
const parsedTutorRoutingDynamicSignals = extractTutorRoutingDynamicSignalsFromReason(params.tutorAdapterTelemetry?.summary?.lastRoutingReason);
|
|
1403
|
+
const tutorRoutingDynamicPreferredMode = hasStructuredTutorRoutingDynamicPreferredMode
|
|
1404
|
+
? normalizeTutorRoutingDynamicPreferredModeToken(tutorRoutingDynamicPreferredModeRaw)
|
|
1405
|
+
: parsedTutorRoutingDynamicSignals.preferredMode;
|
|
1406
|
+
const tutorRoutingDynamicModeReason = tutorRoutingDynamicModeReasonRaw
|
|
1407
|
+
|| parsedTutorRoutingDynamicSignals.modeReason;
|
|
1408
|
+
const tutorRoutingDynamicModeSuggestionActive = (tutorRoutingDynamicPreferredMode === 'local'
|
|
1409
|
+
|| tutorRoutingDynamicPreferredMode === 'cloud');
|
|
1410
|
+
if (params.configuredStoreBackend === 'graphdb') {
|
|
1411
|
+
if (params.store.storeType !== 'graphdb') {
|
|
1412
|
+
checks.push({
|
|
1413
|
+
checkId: 'store_backend_type',
|
|
1414
|
+
status: 'fail',
|
|
1415
|
+
message: 'Configured graphdb backend is not active at runtime.',
|
|
1416
|
+
observed: `storeType=${params.store.storeType}`,
|
|
1417
|
+
expected: 'storeType=graphdb',
|
|
1418
|
+
});
|
|
1419
|
+
}
|
|
1420
|
+
else if (params.store.usingFallback === true) {
|
|
1421
|
+
checks.push({
|
|
1422
|
+
checkId: 'store_graphdb_fallback',
|
|
1423
|
+
status: 'warn',
|
|
1424
|
+
message: 'Graphdb backend is running with fallback store.',
|
|
1425
|
+
observed: `usingFallback=true, fallbackStoreType=${String(params.store.fallbackStoreType || 'unknown')}`,
|
|
1426
|
+
expected: 'usingFallback=false',
|
|
1427
|
+
});
|
|
1428
|
+
}
|
|
1429
|
+
else if (params.store.backendReady === false) {
|
|
1430
|
+
checks.push({
|
|
1431
|
+
checkId: 'store_graphdb_readiness',
|
|
1432
|
+
status: 'warn',
|
|
1433
|
+
message: 'Graphdb backend is selected but adapter readiness is false.',
|
|
1434
|
+
observed: 'backendReady=false',
|
|
1435
|
+
expected: 'backendReady=true',
|
|
1436
|
+
});
|
|
1437
|
+
}
|
|
1438
|
+
else {
|
|
1439
|
+
checks.push({
|
|
1440
|
+
checkId: 'store_graphdb_readiness',
|
|
1441
|
+
status: 'pass',
|
|
1442
|
+
message: 'Graphdb backend is active and ready.',
|
|
1443
|
+
observed: `storeType=${params.store.storeType}, adapterId=${String(params.store.adapterId || 'n/a')}`,
|
|
1444
|
+
});
|
|
1445
|
+
}
|
|
1446
|
+
if (params.store.storeType === 'graphdb'
|
|
1447
|
+
&& params.store.backendReady !== false
|
|
1448
|
+
&& params.store.usingFallback !== true) {
|
|
1449
|
+
if (graphDbConnectorHealthStatus === 'unavailable' || graphDbConnectorCircuitState === 'open') {
|
|
1450
|
+
checks.push({
|
|
1451
|
+
checkId: 'store_graphdb_connector_health',
|
|
1452
|
+
status: 'fail',
|
|
1453
|
+
message: 'Graphdb connector health is unavailable or circuit-open in strict runtime path.',
|
|
1454
|
+
observed: [
|
|
1455
|
+
`health=${graphDbConnectorHealthStatus}`,
|
|
1456
|
+
`circuit=${graphDbConnectorCircuitState}`,
|
|
1457
|
+
`errorCode=${graphDbConnectorLastErrorCode || 'none'}`,
|
|
1458
|
+
`requestCount=${graphDbConnectorRequestCount}`,
|
|
1459
|
+
].join(', '),
|
|
1460
|
+
expected: 'health in {ready,unknown} and circuit!=open',
|
|
1461
|
+
});
|
|
1462
|
+
}
|
|
1463
|
+
else if (graphDbConnectorHealthStatus === 'degraded'
|
|
1464
|
+
|| graphDbConnectorCircuitState === 'half_open'
|
|
1465
|
+
|| graphDbConnectorConsecutiveFailures > 0) {
|
|
1466
|
+
checks.push({
|
|
1467
|
+
checkId: 'store_graphdb_connector_health',
|
|
1468
|
+
status: 'warn',
|
|
1469
|
+
message: 'Graphdb connector health is degraded; stabilize before widening strict rollout.',
|
|
1470
|
+
observed: [
|
|
1471
|
+
`health=${graphDbConnectorHealthStatus}`,
|
|
1472
|
+
`circuit=${graphDbConnectorCircuitState}`,
|
|
1473
|
+
`consecutiveFailures=${graphDbConnectorConsecutiveFailures}`,
|
|
1474
|
+
`retryCount=${graphDbConnectorRetryCount}`,
|
|
1475
|
+
].join(', '),
|
|
1476
|
+
expected: 'health=ready with circuit=closed under sustained traffic',
|
|
1477
|
+
});
|
|
1478
|
+
}
|
|
1479
|
+
else {
|
|
1480
|
+
checks.push({
|
|
1481
|
+
checkId: 'store_graphdb_connector_health',
|
|
1482
|
+
status: 'pass',
|
|
1483
|
+
message: 'Graphdb connector health telemetry is stable.',
|
|
1484
|
+
observed: [
|
|
1485
|
+
`health=${graphDbConnectorHealthStatus}`,
|
|
1486
|
+
`circuit=${graphDbConnectorCircuitState}`,
|
|
1487
|
+
`requestCount=${graphDbConnectorRequestCount}`,
|
|
1488
|
+
`retryCount=${graphDbConnectorRetryCount}`,
|
|
1489
|
+
].join(', '),
|
|
1490
|
+
expected: 'health=ready with circuit=closed',
|
|
1491
|
+
});
|
|
1492
|
+
}
|
|
1493
|
+
if (!graphDbConnectorBudgetMeetsMinSample) {
|
|
1494
|
+
checks.push({
|
|
1495
|
+
checkId: 'store_graphdb_connector_budget',
|
|
1496
|
+
status: 'pass',
|
|
1497
|
+
message: 'Graphdb connector budget governance is deferred until minimum request sample is reached.',
|
|
1498
|
+
observed: [
|
|
1499
|
+
`requestCount=${graphDbConnectorRequestCount}`,
|
|
1500
|
+
`minSample=${thresholds.storeGraphDbConnectorMinRequestSample}`,
|
|
1501
|
+
`failureRatio=${graphDbConnectorFailureRatioPct}%`,
|
|
1502
|
+
`shortCircuitRatio=${graphDbConnectorShortCircuitRatioPct}%`,
|
|
1503
|
+
].join(', '),
|
|
1504
|
+
expected: `requestCount>=${thresholds.storeGraphDbConnectorMinRequestSample}`,
|
|
1505
|
+
});
|
|
1506
|
+
}
|
|
1507
|
+
else if (graphDbConnectorFailBudgetExceeded) {
|
|
1508
|
+
checks.push({
|
|
1509
|
+
checkId: 'store_graphdb_connector_budget',
|
|
1510
|
+
status: 'fail',
|
|
1511
|
+
message: 'Graphdb connector runtime budgets exceed fail thresholds.',
|
|
1512
|
+
observed: [
|
|
1513
|
+
`circuit=${graphDbConnectorCircuitState}`,
|
|
1514
|
+
`requestCount=${graphDbConnectorRequestCount}`,
|
|
1515
|
+
`failureRatio=${graphDbConnectorFailureRatioPct}%`,
|
|
1516
|
+
`shortCircuitRatio=${graphDbConnectorShortCircuitRatioPct}%`,
|
|
1517
|
+
`consecutiveFailures=${graphDbConnectorConsecutiveFailures}`,
|
|
1518
|
+
].join(', '),
|
|
1519
|
+
expected: [
|
|
1520
|
+
`failureRatio<${thresholds.storeGraphDbConnectorFailureFailRatioPct}%`,
|
|
1521
|
+
`shortCircuitRatio<${thresholds.storeGraphDbConnectorShortCircuitFailRatioPct}%`,
|
|
1522
|
+
`consecutiveFailures<${thresholds.storeGraphDbConnectorConsecutiveFailuresFailCount}`,
|
|
1523
|
+
'circuit!=open',
|
|
1524
|
+
].join(', '),
|
|
1525
|
+
});
|
|
1526
|
+
}
|
|
1527
|
+
else if (graphDbConnectorWarnBudgetExceeded) {
|
|
1528
|
+
checks.push({
|
|
1529
|
+
checkId: 'store_graphdb_connector_budget',
|
|
1530
|
+
status: 'warn',
|
|
1531
|
+
message: 'Graphdb connector runtime budgets are approaching warning thresholds.',
|
|
1532
|
+
observed: [
|
|
1533
|
+
`circuit=${graphDbConnectorCircuitState}`,
|
|
1534
|
+
`requestCount=${graphDbConnectorRequestCount}`,
|
|
1535
|
+
`failureRatio=${graphDbConnectorFailureRatioPct}%`,
|
|
1536
|
+
`shortCircuitRatio=${graphDbConnectorShortCircuitRatioPct}%`,
|
|
1537
|
+
`consecutiveFailures=${graphDbConnectorConsecutiveFailures}`,
|
|
1538
|
+
].join(', '),
|
|
1539
|
+
expected: [
|
|
1540
|
+
`failureRatio<${thresholds.storeGraphDbConnectorFailureWarnRatioPct}%`,
|
|
1541
|
+
`shortCircuitRatio<${thresholds.storeGraphDbConnectorShortCircuitWarnRatioPct}%`,
|
|
1542
|
+
`consecutiveFailures<${thresholds.storeGraphDbConnectorConsecutiveFailuresWarnCount}`,
|
|
1543
|
+
'circuit=closed|unknown',
|
|
1544
|
+
].join(', '),
|
|
1545
|
+
});
|
|
1546
|
+
}
|
|
1547
|
+
else {
|
|
1548
|
+
checks.push({
|
|
1549
|
+
checkId: 'store_graphdb_connector_budget',
|
|
1550
|
+
status: 'pass',
|
|
1551
|
+
message: 'Graphdb connector runtime budgets are within configured thresholds.',
|
|
1552
|
+
observed: [
|
|
1553
|
+
`circuit=${graphDbConnectorCircuitState}`,
|
|
1554
|
+
`requestCount=${graphDbConnectorRequestCount}`,
|
|
1555
|
+
`failureRatio=${graphDbConnectorFailureRatioPct}%`,
|
|
1556
|
+
`shortCircuitRatio=${graphDbConnectorShortCircuitRatioPct}%`,
|
|
1557
|
+
`consecutiveFailures=${graphDbConnectorConsecutiveFailures}`,
|
|
1558
|
+
].join(', '),
|
|
1559
|
+
expected: [
|
|
1560
|
+
`failureRatio<${thresholds.storeGraphDbConnectorFailureWarnRatioPct}%`,
|
|
1561
|
+
`shortCircuitRatio<${thresholds.storeGraphDbConnectorShortCircuitWarnRatioPct}%`,
|
|
1562
|
+
`consecutiveFailures<${thresholds.storeGraphDbConnectorConsecutiveFailuresWarnCount}`,
|
|
1563
|
+
].join(', '),
|
|
1564
|
+
});
|
|
1565
|
+
}
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
1568
|
+
else {
|
|
1569
|
+
const matched = params.store.storeType === params.configuredStoreBackend;
|
|
1570
|
+
checks.push({
|
|
1571
|
+
checkId: 'store_backend_type',
|
|
1572
|
+
status: matched ? 'pass' : 'warn',
|
|
1573
|
+
message: matched
|
|
1574
|
+
? 'Store backend matches runtime configuration.'
|
|
1575
|
+
: 'Store backend differs from configured backend.',
|
|
1576
|
+
observed: `storeType=${params.store.storeType}`,
|
|
1577
|
+
expected: `storeType=${params.configuredStoreBackend}`,
|
|
1578
|
+
});
|
|
1579
|
+
}
|
|
1580
|
+
checks.push({
|
|
1581
|
+
checkId: 'query_backend_identity',
|
|
1582
|
+
status: String(params.queryDiagnostics.backendId || '').trim().length > 0 ? 'pass' : 'fail',
|
|
1583
|
+
message: String(params.queryDiagnostics.backendId || '').trim().length > 0
|
|
1584
|
+
? 'Query backend is registered.'
|
|
1585
|
+
: 'Query backend id is missing.',
|
|
1586
|
+
observed: `backendId=${params.queryDiagnostics.backendId || 'unknown'}`,
|
|
1587
|
+
});
|
|
1588
|
+
if (!queryBackendRuntimeReady) {
|
|
1589
|
+
checks.push({
|
|
1590
|
+
checkId: 'query_backend_runtime_health',
|
|
1591
|
+
status: 'fail',
|
|
1592
|
+
message: 'Runtime diagnostics report query backend as not ready.',
|
|
1593
|
+
observed: `runtimeReady=false, backendId=${queryBackendRuntimeId || 'unknown'}`,
|
|
1594
|
+
expected: 'runtimeReady=true',
|
|
1595
|
+
});
|
|
1596
|
+
}
|
|
1597
|
+
else if (queryBackendRuntimeId && queryBackendRuntimeId !== String(params.queryDiagnostics.backendId || '').trim()) {
|
|
1598
|
+
checks.push({
|
|
1599
|
+
checkId: 'query_backend_runtime_health',
|
|
1600
|
+
status: 'warn',
|
|
1601
|
+
message: 'Query backend runtime id does not match diagnostics backend id.',
|
|
1602
|
+
observed: `runtimeBackendId=${queryBackendRuntimeId}, diagnosticsBackendId=${String(params.queryDiagnostics.backendId || '').trim() || 'unknown'}`,
|
|
1603
|
+
expected: 'runtimeBackendId==diagnosticsBackendId',
|
|
1604
|
+
});
|
|
1605
|
+
}
|
|
1606
|
+
else {
|
|
1607
|
+
checks.push({
|
|
1608
|
+
checkId: 'query_backend_runtime_health',
|
|
1609
|
+
status: 'pass',
|
|
1610
|
+
message: 'Query backend runtime diagnostics are healthy.',
|
|
1611
|
+
observed: `runtimeReady=${queryBackendRuntimeReady}, backendId=${queryBackendRuntimeId || params.queryDiagnostics.backendId || 'unknown'}`,
|
|
1612
|
+
expected: 'runtimeReady=true',
|
|
1613
|
+
});
|
|
1614
|
+
}
|
|
1615
|
+
const localVectorConfigured = params.configuredQueryBackend === 'local_vector';
|
|
1616
|
+
if (localVectorConfigured) {
|
|
1617
|
+
if (!queryVectorIndexEnabled) {
|
|
1618
|
+
checks.push({
|
|
1619
|
+
checkId: 'query_vector_index_status',
|
|
1620
|
+
status: 'fail',
|
|
1621
|
+
message: 'Local vector backend is active but vector index runtime is disabled.',
|
|
1622
|
+
observed: `configuredQueryBackend=${params.configuredQueryBackend}, vectorIndexEnabled=${queryVectorIndexEnabled}`,
|
|
1623
|
+
expected: 'vectorIndexEnabled=true',
|
|
1624
|
+
});
|
|
1625
|
+
}
|
|
1626
|
+
else if (queryVectorIndexStatus === 'unavailable') {
|
|
1627
|
+
checks.push({
|
|
1628
|
+
checkId: 'query_vector_index_status',
|
|
1629
|
+
status: 'fail',
|
|
1630
|
+
message: 'Local vector index runtime is unavailable.',
|
|
1631
|
+
observed: `vectorIndexStatus=${queryVectorIndexStatus}, runtimeReady=${queryBackendRuntimeReady}`,
|
|
1632
|
+
expected: 'vectorIndexStatus=ready',
|
|
1633
|
+
});
|
|
1634
|
+
}
|
|
1635
|
+
else if (queryVectorIndexStatus === 'stale' || queryVectorIndexStatus === 'unknown') {
|
|
1636
|
+
checks.push({
|
|
1637
|
+
checkId: 'query_vector_index_status',
|
|
1638
|
+
status: 'warn',
|
|
1639
|
+
message: 'Local vector index requires refresh before stable retrieval governance.',
|
|
1640
|
+
observed: `vectorIndexStatus=${queryVectorIndexStatus}, atomCount=${queryVectorIndexAtomCount}`,
|
|
1641
|
+
expected: 'vectorIndexStatus=ready',
|
|
1642
|
+
});
|
|
1643
|
+
}
|
|
1644
|
+
else {
|
|
1645
|
+
checks.push({
|
|
1646
|
+
checkId: 'query_vector_index_status',
|
|
1647
|
+
status: 'pass',
|
|
1648
|
+
message: 'Local vector index runtime is ready.',
|
|
1649
|
+
observed: `vectorIndexStatus=${queryVectorIndexStatus}, atomCount=${queryVectorIndexAtomCount}`,
|
|
1650
|
+
expected: 'vectorIndexStatus=ready',
|
|
1651
|
+
});
|
|
1652
|
+
}
|
|
1653
|
+
if (queryVectorIndexEnabled && queryVectorIndexStatus === 'ready' && queryVectorIndexPersisted) {
|
|
1654
|
+
checks.push({
|
|
1655
|
+
checkId: 'query_vector_index_persistence',
|
|
1656
|
+
status: 'pass',
|
|
1657
|
+
message: 'Local vector index persistence is active.',
|
|
1658
|
+
observed: `persisted=${queryVectorIndexPersisted}, loadedFromDisk=${queryVectorIndexLoadedFromDisk}, location=${queryVectorIndexLocation || 'memory-only'}`,
|
|
1659
|
+
expected: 'persisted=true',
|
|
1660
|
+
});
|
|
1661
|
+
}
|
|
1662
|
+
else if (queryVectorIndexEnabled && queryVectorIndexStatus === 'ready') {
|
|
1663
|
+
checks.push({
|
|
1664
|
+
checkId: 'query_vector_index_persistence',
|
|
1665
|
+
status: 'warn',
|
|
1666
|
+
message: 'Local vector index is running in memory-only mode; persistence safeguards are disabled.',
|
|
1667
|
+
observed: `persisted=${queryVectorIndexPersisted}, location=${queryVectorIndexLocation || 'memory-only'}`,
|
|
1668
|
+
expected: 'persisted=true',
|
|
1669
|
+
});
|
|
1670
|
+
}
|
|
1671
|
+
else if (queryVectorIndexEnabled) {
|
|
1672
|
+
checks.push({
|
|
1673
|
+
checkId: 'query_vector_index_persistence',
|
|
1674
|
+
status: 'warn',
|
|
1675
|
+
message: 'Local vector index persistence cannot be validated before index reaches ready state.',
|
|
1676
|
+
observed: `vectorIndexStatus=${queryVectorIndexStatus}, persisted=${queryVectorIndexPersisted}`,
|
|
1677
|
+
expected: 'vectorIndexStatus=ready and persisted=true',
|
|
1678
|
+
});
|
|
1679
|
+
}
|
|
1680
|
+
else {
|
|
1681
|
+
checks.push({
|
|
1682
|
+
checkId: 'query_vector_index_persistence',
|
|
1683
|
+
status: 'fail',
|
|
1684
|
+
message: 'Local vector backend is active but vector index persistence is unavailable because index runtime is disabled.',
|
|
1685
|
+
observed: `vectorIndexEnabled=${queryVectorIndexEnabled}, persisted=${queryVectorIndexPersisted}`,
|
|
1686
|
+
expected: 'vectorIndexEnabled=true and persisted=true',
|
|
1687
|
+
});
|
|
1688
|
+
}
|
|
1689
|
+
if (!queryVectorIndexEnabled || queryVectorIndexStatus !== 'ready') {
|
|
1690
|
+
checks.push({
|
|
1691
|
+
checkId: 'query_vector_acceleration_mode',
|
|
1692
|
+
status: 'warn',
|
|
1693
|
+
message: 'Vector acceleration mode cannot be validated before vector index reaches ready state.',
|
|
1694
|
+
observed: `vectorIndexEnabled=${queryVectorIndexEnabled}, vectorIndexStatus=${queryVectorIndexStatus}`,
|
|
1695
|
+
expected: 'vectorIndexEnabled=true and vectorIndexStatus=ready',
|
|
1696
|
+
});
|
|
1697
|
+
}
|
|
1698
|
+
else if (!queryVectorIndexAccelerationEnabled) {
|
|
1699
|
+
checks.push({
|
|
1700
|
+
checkId: 'query_vector_acceleration_mode',
|
|
1701
|
+
status: 'warn',
|
|
1702
|
+
message: 'Vector acceleration is disabled; retrieval falls back to full scan.',
|
|
1703
|
+
observed: `accelerationEnabled=${queryVectorIndexAccelerationEnabled}, mode=${queryVectorIndexAccelerationMode}`,
|
|
1704
|
+
expected: 'accelerationEnabled=true with mode=ann_prefilter',
|
|
1705
|
+
});
|
|
1706
|
+
}
|
|
1707
|
+
else if (queryVectorIndexAccelerationMode === 'ann_prefilter') {
|
|
1708
|
+
checks.push({
|
|
1709
|
+
checkId: 'query_vector_acceleration_mode',
|
|
1710
|
+
status: 'pass',
|
|
1711
|
+
message: 'Vector acceleration is active with ANN prefilter mode.',
|
|
1712
|
+
observed: `mode=${queryVectorIndexAccelerationMode}, adapterId=${queryVectorIndexAccelerationAdapterId || 'unknown'}`,
|
|
1713
|
+
expected: 'mode=ann_prefilter',
|
|
1714
|
+
});
|
|
1715
|
+
}
|
|
1716
|
+
else {
|
|
1717
|
+
checks.push({
|
|
1718
|
+
checkId: 'query_vector_acceleration_mode',
|
|
1719
|
+
status: 'warn',
|
|
1720
|
+
message: 'Vector acceleration is enabled but ANN prefilter is not active in the latest diagnostic cycle.',
|
|
1721
|
+
observed: `mode=${queryVectorIndexAccelerationMode}, adapterId=${queryVectorIndexAccelerationAdapterId || 'unknown'}, adapterError=${queryVectorIndexAccelerationAdapterError || 'none'}`,
|
|
1722
|
+
expected: 'mode=ann_prefilter',
|
|
1723
|
+
});
|
|
1724
|
+
}
|
|
1725
|
+
const queryVectorAccelerationRepresentationExternalConnector = (queryVectorIndexAccelerationAdapterId.length > 0
|
|
1726
|
+
&& queryVectorIndexAccelerationAdapterId.toLowerCase().includes('external'));
|
|
1727
|
+
const queryVectorAccelerationRepresentationObserved = (`representationStatus=${queryVectorIndexAccelerationRepresentationStatus},`
|
|
1728
|
+
+ ` strictMode=${queryVectorIndexAccelerationRepresentationStrictMode},`
|
|
1729
|
+
+ ` representationVersion=${queryVectorIndexAccelerationRepresentationVersion || '<none>'},`
|
|
1730
|
+
+ ` embeddingModelId=${queryVectorIndexAccelerationEmbeddingModelId || '<none>'},`
|
|
1731
|
+
+ ` embeddingDimension=${queryVectorIndexAccelerationEmbeddingDimension},`
|
|
1732
|
+
+ ` indexSignature=${queryVectorIndexAccelerationIndexSignature || '<none>'},`
|
|
1733
|
+
+ ` reason=${queryVectorIndexAccelerationRepresentationStatusReason || '<none>'},`
|
|
1734
|
+
+ ` adapterId=${queryVectorIndexAccelerationAdapterId || 'unknown'},`
|
|
1735
|
+
+ ` requestCount=${queryVectorIndexAccelerationRequestCount}`);
|
|
1736
|
+
if (!queryVectorIndexEnabled || queryVectorIndexStatus !== 'ready') {
|
|
1737
|
+
checks.push({
|
|
1738
|
+
checkId: 'query_vector_acceleration_representation_consistency',
|
|
1739
|
+
status: 'warn',
|
|
1740
|
+
message: 'Vector acceleration representation consistency cannot be validated before vector index reaches ready state.',
|
|
1741
|
+
observed: `vectorIndexEnabled=${queryVectorIndexEnabled}, vectorIndexStatus=${queryVectorIndexStatus}`,
|
|
1742
|
+
expected: 'vectorIndexEnabled=true and vectorIndexStatus=ready',
|
|
1743
|
+
});
|
|
1744
|
+
}
|
|
1745
|
+
else if (!queryVectorIndexAccelerationEnabled) {
|
|
1746
|
+
checks.push({
|
|
1747
|
+
checkId: 'query_vector_acceleration_representation_consistency',
|
|
1748
|
+
status: 'pass',
|
|
1749
|
+
message: 'Vector acceleration representation consistency check skipped because acceleration is disabled.',
|
|
1750
|
+
observed: queryVectorAccelerationRepresentationObserved,
|
|
1751
|
+
expected: 'accelerationEnabled=true and representationStatus=aligned',
|
|
1752
|
+
});
|
|
1753
|
+
}
|
|
1754
|
+
else if (queryVectorIndexAccelerationRepresentationStatus === 'mismatch') {
|
|
1755
|
+
checks.push({
|
|
1756
|
+
checkId: 'query_vector_acceleration_representation_consistency',
|
|
1757
|
+
status: 'fail',
|
|
1758
|
+
message: queryVectorIndexAccelerationRepresentationStrictMode
|
|
1759
|
+
? 'Vector acceleration representation metadata mismatches index semantics while strict mode is enabled.'
|
|
1760
|
+
: 'Vector acceleration representation metadata mismatches index semantics.',
|
|
1761
|
+
observed: queryVectorAccelerationRepresentationObserved,
|
|
1762
|
+
expected: 'representationStatus=aligned',
|
|
1763
|
+
});
|
|
1764
|
+
}
|
|
1765
|
+
else if (queryVectorIndexAccelerationRepresentationStatus === 'aligned') {
|
|
1766
|
+
checks.push({
|
|
1767
|
+
checkId: 'query_vector_acceleration_representation_consistency',
|
|
1768
|
+
status: 'pass',
|
|
1769
|
+
message: 'Vector acceleration representation metadata is aligned with local index semantics.',
|
|
1770
|
+
observed: queryVectorAccelerationRepresentationObserved,
|
|
1771
|
+
expected: 'representationStatus=aligned',
|
|
1772
|
+
});
|
|
1773
|
+
}
|
|
1774
|
+
else if (queryVectorAccelerationRepresentationExternalConnector
|
|
1775
|
+
&& queryVectorIndexAccelerationRequestCount >= thresholds.minQuerySampleSize) {
|
|
1776
|
+
checks.push({
|
|
1777
|
+
checkId: 'query_vector_acceleration_representation_consistency',
|
|
1778
|
+
status: 'warn',
|
|
1779
|
+
message: 'External vector acceleration connector has representative traffic but representation metadata is still unknown.',
|
|
1780
|
+
observed: queryVectorAccelerationRepresentationObserved,
|
|
1781
|
+
expected: 'representationStatus=aligned with representationVersion/embeddingModelId/indexSignature',
|
|
1782
|
+
});
|
|
1783
|
+
}
|
|
1784
|
+
else if (queryVectorAccelerationRepresentationExternalConnector) {
|
|
1785
|
+
checks.push({
|
|
1786
|
+
checkId: 'query_vector_acceleration_representation_consistency',
|
|
1787
|
+
status: 'pass',
|
|
1788
|
+
message: 'External vector acceleration connector representation metadata is pending; insufficient traffic for strict consistency evaluation.',
|
|
1789
|
+
observed: queryVectorAccelerationRepresentationObserved,
|
|
1790
|
+
expected: 'representationStatus=aligned after representative traffic',
|
|
1791
|
+
});
|
|
1792
|
+
}
|
|
1793
|
+
else {
|
|
1794
|
+
checks.push({
|
|
1795
|
+
checkId: 'query_vector_acceleration_representation_consistency',
|
|
1796
|
+
status: 'pass',
|
|
1797
|
+
message: 'Vector acceleration representation metadata is operating in local-verified mode.',
|
|
1798
|
+
observed: queryVectorAccelerationRepresentationObserved,
|
|
1799
|
+
expected: 'representationStatus=aligned',
|
|
1800
|
+
});
|
|
1801
|
+
}
|
|
1802
|
+
const queryVectorAccelerationPrefilterSelectionActive = (queryVectorIndexAccelerationLastSelectionMode === 'token_prefilter'
|
|
1803
|
+
|| queryVectorIndexAccelerationLastSelectionMode === 'token_signature_prefilter');
|
|
1804
|
+
const queryVectorAccelerationPrefilterSampleReady = (queryVectorIndexAccelerationRequestCount >= thresholds.queryVectorAccelerationPrefilterMinRequestSample);
|
|
1805
|
+
const queryVectorAccelerationPrefilterCandidateRatioPct = (queryVectorIndexAtomCount > 0
|
|
1806
|
+
&& queryVectorIndexAccelerationLastCandidateCount > 0)
|
|
1807
|
+
? Number(((queryVectorIndexAccelerationLastCandidateCount / Math.max(1, queryVectorIndexAtomCount))
|
|
1808
|
+
* 100).toFixed(4))
|
|
1809
|
+
: 0;
|
|
1810
|
+
const queryVectorAccelerationPrefilterObserved = (`mode=${queryVectorIndexAccelerationMode},`
|
|
1811
|
+
+ ` lastSelectionMode=${queryVectorIndexAccelerationLastSelectionMode},`
|
|
1812
|
+
+ ` lastCandidateCount=${queryVectorIndexAccelerationLastCandidateCount},`
|
|
1813
|
+
+ ` atomCount=${queryVectorIndexAtomCount},`
|
|
1814
|
+
+ ` candidateRatio=${queryVectorAccelerationPrefilterCandidateRatioPct}%,`
|
|
1815
|
+
+ ` requestCount=${queryVectorIndexAccelerationRequestCount},`
|
|
1816
|
+
+ ` healthStatus=${queryVectorIndexAccelerationHealthStatus},`
|
|
1817
|
+
+ ` circuitState=${queryVectorIndexAccelerationCircuitState}`);
|
|
1818
|
+
const queryVectorAccelerationStableConnector = (queryVectorIndexAccelerationHealthStatus === 'ready'
|
|
1819
|
+
&& (queryVectorIndexAccelerationCircuitState === 'closed'
|
|
1820
|
+
|| queryVectorIndexAccelerationCircuitState === 'unknown'));
|
|
1821
|
+
const queryVectorAccelerationCanEvaluatePrefilterReduction = (queryVectorIndexAtomCount > 0
|
|
1822
|
+
&& queryVectorIndexAccelerationLastCandidateCount > 0
|
|
1823
|
+
&& queryVectorIndexAccelerationLastCandidateCount <= queryVectorIndexAtomCount);
|
|
1824
|
+
const queryVectorAccelerationPrefilterReductionSevere = (queryVectorAccelerationCanEvaluatePrefilterReduction
|
|
1825
|
+
&& queryVectorAccelerationPrefilterCandidateRatioPct
|
|
1826
|
+
>= thresholds.queryVectorAccelerationPrefilterFailCandidateRatioPct);
|
|
1827
|
+
const queryVectorAccelerationPrefilterReductionWeak = (queryVectorAccelerationCanEvaluatePrefilterReduction
|
|
1828
|
+
&& queryVectorAccelerationPrefilterCandidateRatioPct
|
|
1829
|
+
>= thresholds.queryVectorAccelerationPrefilterWarnCandidateRatioPct);
|
|
1830
|
+
if (!queryVectorIndexEnabled || queryVectorIndexStatus !== 'ready') {
|
|
1831
|
+
checks.push({
|
|
1832
|
+
checkId: 'query_vector_acceleration_prefilter_effectiveness',
|
|
1833
|
+
status: 'warn',
|
|
1834
|
+
message: 'Vector acceleration prefilter effectiveness cannot be validated before vector index reaches ready state.',
|
|
1835
|
+
observed: `vectorIndexEnabled=${queryVectorIndexEnabled}, vectorIndexStatus=${queryVectorIndexStatus}`,
|
|
1836
|
+
expected: 'vectorIndexEnabled=true and vectorIndexStatus=ready',
|
|
1837
|
+
});
|
|
1838
|
+
}
|
|
1839
|
+
else if (!queryVectorIndexAccelerationEnabled) {
|
|
1840
|
+
checks.push({
|
|
1841
|
+
checkId: 'query_vector_acceleration_prefilter_effectiveness',
|
|
1842
|
+
status: 'pass',
|
|
1843
|
+
message: 'Vector acceleration prefilter effectiveness check skipped because acceleration is disabled.',
|
|
1844
|
+
observed: `accelerationEnabled=${queryVectorIndexAccelerationEnabled}, mode=${queryVectorIndexAccelerationMode}`,
|
|
1845
|
+
expected: 'accelerationEnabled=true and mode=ann_prefilter',
|
|
1846
|
+
});
|
|
1847
|
+
}
|
|
1848
|
+
else if (queryVectorIndexAccelerationMode !== 'ann_prefilter') {
|
|
1849
|
+
checks.push({
|
|
1850
|
+
checkId: 'query_vector_acceleration_prefilter_effectiveness',
|
|
1851
|
+
status: 'pass',
|
|
1852
|
+
message: 'Vector acceleration prefilter effectiveness check deferred because ANN prefilter mode is not active.',
|
|
1853
|
+
observed: queryVectorAccelerationPrefilterObserved,
|
|
1854
|
+
expected: 'mode=ann_prefilter',
|
|
1855
|
+
});
|
|
1856
|
+
}
|
|
1857
|
+
else if (queryVectorIndexAccelerationLastSelectionMode === 'full_scan'
|
|
1858
|
+
&& queryVectorAccelerationPrefilterSampleReady
|
|
1859
|
+
&& queryVectorAccelerationStableConnector) {
|
|
1860
|
+
checks.push({
|
|
1861
|
+
checkId: 'query_vector_acceleration_prefilter_effectiveness',
|
|
1862
|
+
status: 'fail',
|
|
1863
|
+
message: 'ANN prefilter mode is enabled but selection repeatedly falls back to full scan under stable connector conditions.',
|
|
1864
|
+
observed: queryVectorAccelerationPrefilterObserved,
|
|
1865
|
+
expected: 'lastSelectionMode=token_prefilter|token_signature_prefilter under representative traffic',
|
|
1866
|
+
});
|
|
1867
|
+
}
|
|
1868
|
+
else if (!queryVectorAccelerationPrefilterSelectionActive) {
|
|
1869
|
+
checks.push({
|
|
1870
|
+
checkId: 'query_vector_acceleration_prefilter_effectiveness',
|
|
1871
|
+
status: queryVectorAccelerationPrefilterSampleReady ? 'warn' : 'pass',
|
|
1872
|
+
message: queryVectorAccelerationPrefilterSampleReady
|
|
1873
|
+
? 'ANN prefilter effectiveness is inconclusive because selection telemetry is not yet prefilter-driven.'
|
|
1874
|
+
: 'ANN prefilter effectiveness check deferred until representative acceleration traffic is collected.',
|
|
1875
|
+
observed: queryVectorAccelerationPrefilterObserved,
|
|
1876
|
+
expected: 'lastSelectionMode=token_prefilter|token_signature_prefilter with candidate telemetry',
|
|
1877
|
+
});
|
|
1878
|
+
}
|
|
1879
|
+
else if (queryVectorAccelerationPrefilterSampleReady
|
|
1880
|
+
&& queryVectorIndexAccelerationLastCandidateCount <= 0) {
|
|
1881
|
+
checks.push({
|
|
1882
|
+
checkId: 'query_vector_acceleration_prefilter_effectiveness',
|
|
1883
|
+
status: 'warn',
|
|
1884
|
+
message: 'ANN prefilter selection is active but candidate count telemetry is missing.',
|
|
1885
|
+
observed: queryVectorAccelerationPrefilterObserved,
|
|
1886
|
+
expected: 'lastCandidateCount>0 when prefilter selection is active',
|
|
1887
|
+
});
|
|
1888
|
+
}
|
|
1889
|
+
else if (queryVectorAccelerationPrefilterSampleReady
|
|
1890
|
+
&& queryVectorAccelerationPrefilterReductionSevere) {
|
|
1891
|
+
checks.push({
|
|
1892
|
+
checkId: 'query_vector_acceleration_prefilter_effectiveness',
|
|
1893
|
+
status: 'fail',
|
|
1894
|
+
message: 'ANN prefilter is active but candidate reduction breaches the configured fail threshold.',
|
|
1895
|
+
observed: queryVectorAccelerationPrefilterObserved,
|
|
1896
|
+
expected: `candidateRatio<${thresholds.queryVectorAccelerationPrefilterFailCandidateRatioPct}% under representative ANN prefilter traffic`,
|
|
1897
|
+
});
|
|
1898
|
+
}
|
|
1899
|
+
else if (queryVectorAccelerationPrefilterSampleReady
|
|
1900
|
+
&& queryVectorAccelerationPrefilterReductionWeak) {
|
|
1901
|
+
checks.push({
|
|
1902
|
+
checkId: 'query_vector_acceleration_prefilter_effectiveness',
|
|
1903
|
+
status: 'warn',
|
|
1904
|
+
message: 'ANN prefilter is active but candidate reduction is weak in the latest diagnostic cycle.',
|
|
1905
|
+
observed: queryVectorAccelerationPrefilterObserved,
|
|
1906
|
+
expected: `candidateRatio<${thresholds.queryVectorAccelerationPrefilterWarnCandidateRatioPct}% under representative ANN prefilter traffic`,
|
|
1907
|
+
});
|
|
1908
|
+
}
|
|
1909
|
+
else {
|
|
1910
|
+
checks.push({
|
|
1911
|
+
checkId: 'query_vector_acceleration_prefilter_effectiveness',
|
|
1912
|
+
status: 'pass',
|
|
1913
|
+
message: 'ANN prefilter effectiveness telemetry is healthy for the latest diagnostic cycle.',
|
|
1914
|
+
observed: queryVectorAccelerationPrefilterObserved,
|
|
1915
|
+
expected: 'prefilter selection active with representative candidate telemetry',
|
|
1916
|
+
});
|
|
1917
|
+
}
|
|
1918
|
+
if (!queryVectorIndexEnabled || queryVectorIndexStatus !== 'ready') {
|
|
1919
|
+
checks.push({
|
|
1920
|
+
checkId: 'query_vector_acceleration_health',
|
|
1921
|
+
status: 'warn',
|
|
1922
|
+
message: 'Vector acceleration health cannot be validated before vector index reaches ready state.',
|
|
1923
|
+
observed: `vectorIndexEnabled=${queryVectorIndexEnabled}, vectorIndexStatus=${queryVectorIndexStatus}`,
|
|
1924
|
+
expected: 'vectorIndexEnabled=true and vectorIndexStatus=ready',
|
|
1925
|
+
});
|
|
1926
|
+
}
|
|
1927
|
+
else if (!queryVectorIndexAccelerationEnabled) {
|
|
1928
|
+
checks.push({
|
|
1929
|
+
checkId: 'query_vector_acceleration_health',
|
|
1930
|
+
status: 'warn',
|
|
1931
|
+
message: 'Vector acceleration health check is running in fallback mode because acceleration is disabled.',
|
|
1932
|
+
observed: `accelerationEnabled=${queryVectorIndexAccelerationEnabled}, healthStatus=${queryVectorIndexAccelerationHealthStatus}`,
|
|
1933
|
+
expected: 'accelerationEnabled=true and healthStatus=ready|unknown',
|
|
1934
|
+
});
|
|
1935
|
+
}
|
|
1936
|
+
else if (queryVectorIndexAccelerationHealthStatus === 'unavailable') {
|
|
1937
|
+
checks.push({
|
|
1938
|
+
checkId: 'query_vector_acceleration_health',
|
|
1939
|
+
status: 'fail',
|
|
1940
|
+
message: 'Vector acceleration adapter health is unavailable and requires immediate connector recovery.',
|
|
1941
|
+
observed: `healthStatus=${queryVectorIndexAccelerationHealthStatus}, adapterId=${queryVectorIndexAccelerationAdapterId || 'unknown'}, message=${queryVectorIndexAccelerationHealthMessage || queryVectorIndexAccelerationAdapterError || 'none'}`,
|
|
1942
|
+
expected: 'healthStatus=ready|unknown',
|
|
1943
|
+
});
|
|
1944
|
+
}
|
|
1945
|
+
else if (queryVectorIndexAccelerationHealthStatus === 'degraded') {
|
|
1946
|
+
checks.push({
|
|
1947
|
+
checkId: 'query_vector_acceleration_health',
|
|
1948
|
+
status: 'warn',
|
|
1949
|
+
message: 'Vector acceleration adapter health is degraded; monitor connector stability and fallback ratio.',
|
|
1950
|
+
observed: `healthStatus=${queryVectorIndexAccelerationHealthStatus}, adapterId=${queryVectorIndexAccelerationAdapterId || 'unknown'}, message=${queryVectorIndexAccelerationHealthMessage || queryVectorIndexAccelerationAdapterError || 'none'}`,
|
|
1951
|
+
expected: 'healthStatus=ready|unknown',
|
|
1952
|
+
});
|
|
1953
|
+
}
|
|
1954
|
+
else if (queryVectorIndexAccelerationHealthStatus === 'ready') {
|
|
1955
|
+
checks.push({
|
|
1956
|
+
checkId: 'query_vector_acceleration_health',
|
|
1957
|
+
status: 'pass',
|
|
1958
|
+
message: 'Vector acceleration adapter reports ready health status.',
|
|
1959
|
+
observed: `healthStatus=${queryVectorIndexAccelerationHealthStatus}, adapterId=${queryVectorIndexAccelerationAdapterId || 'unknown'}`,
|
|
1960
|
+
expected: 'healthStatus=ready|unknown',
|
|
1961
|
+
});
|
|
1962
|
+
}
|
|
1963
|
+
else {
|
|
1964
|
+
checks.push({
|
|
1965
|
+
checkId: 'query_vector_acceleration_health',
|
|
1966
|
+
status: 'pass',
|
|
1967
|
+
message: 'Vector acceleration adapter health telemetry is not explicitly reported.',
|
|
1968
|
+
observed: `healthStatus=${queryVectorIndexAccelerationHealthStatus}, adapterId=${queryVectorIndexAccelerationAdapterId || 'unknown'}`,
|
|
1969
|
+
expected: 'healthStatus=ready|unknown',
|
|
1970
|
+
});
|
|
1971
|
+
}
|
|
1972
|
+
const queryVectorAccelerationExternalConnector = (queryVectorIndexAccelerationAdapterId.length > 0
|
|
1973
|
+
&& queryVectorIndexAccelerationAdapterId.toLowerCase().includes('external'));
|
|
1974
|
+
const queryVectorAccelerationIndexSyncObserved = (`adapterId=${queryVectorIndexAccelerationAdapterId || 'unknown'},`
|
|
1975
|
+
+ ` externalConnector=${queryVectorAccelerationExternalConnector},`
|
|
1976
|
+
+ ` indexSyncStatus=${queryVectorIndexAccelerationIndexSyncStatus},`
|
|
1977
|
+
+ ` syncRequestCount=${queryVectorIndexAccelerationSyncRequestCount},`
|
|
1978
|
+
+ ` syncSuccessCount=${queryVectorIndexAccelerationSyncSuccessCount},`
|
|
1979
|
+
+ ` syncFailureCount=${queryVectorIndexAccelerationSyncFailureCount},`
|
|
1980
|
+
+ ` syncedAtomCount=${queryVectorIndexAccelerationSyncedAtomCount},`
|
|
1981
|
+
+ ` syncedIndexSignature=${queryVectorIndexAccelerationSyncedIndexSignature || '<none>'},`
|
|
1982
|
+
+ ` lastSyncAt=${queryVectorIndexAccelerationLastSyncAt || '<none>'}`);
|
|
1983
|
+
if (!queryVectorIndexEnabled || queryVectorIndexStatus !== 'ready') {
|
|
1984
|
+
checks.push({
|
|
1985
|
+
checkId: 'query_vector_acceleration_index_sync_health',
|
|
1986
|
+
status: 'warn',
|
|
1987
|
+
message: 'Vector acceleration index sync health cannot be validated before vector index reaches ready state.',
|
|
1988
|
+
observed: `vectorIndexEnabled=${queryVectorIndexEnabled}, vectorIndexStatus=${queryVectorIndexStatus}`,
|
|
1989
|
+
expected: 'vectorIndexEnabled=true and vectorIndexStatus=ready',
|
|
1990
|
+
});
|
|
1991
|
+
}
|
|
1992
|
+
else if (!queryVectorIndexAccelerationEnabled) {
|
|
1993
|
+
checks.push({
|
|
1994
|
+
checkId: 'query_vector_acceleration_index_sync_health',
|
|
1995
|
+
status: 'warn',
|
|
1996
|
+
message: 'Vector acceleration index sync health is running in fallback mode because acceleration is disabled.',
|
|
1997
|
+
observed: `accelerationEnabled=${queryVectorIndexAccelerationEnabled}, adapterId=${queryVectorIndexAccelerationAdapterId || 'unknown'}`,
|
|
1998
|
+
expected: 'accelerationEnabled=true for sync-backed connector validation',
|
|
1999
|
+
});
|
|
2000
|
+
}
|
|
2001
|
+
else if (!queryVectorAccelerationExternalConnector) {
|
|
2002
|
+
checks.push({
|
|
2003
|
+
checkId: 'query_vector_acceleration_index_sync_health',
|
|
2004
|
+
status: 'pass',
|
|
2005
|
+
message: 'Vector acceleration is using local adapter path; external index sync is not required.',
|
|
2006
|
+
observed: queryVectorAccelerationIndexSyncObserved,
|
|
2007
|
+
expected: 'external connector adapters should report sync telemetry',
|
|
2008
|
+
});
|
|
2009
|
+
}
|
|
2010
|
+
else if (queryVectorIndexAccelerationIndexSyncStatus === 'unavailable') {
|
|
2011
|
+
checks.push({
|
|
2012
|
+
checkId: 'query_vector_acceleration_index_sync_health',
|
|
2013
|
+
status: 'fail',
|
|
2014
|
+
message: 'External vector acceleration index sync is unavailable.',
|
|
2015
|
+
observed: queryVectorAccelerationIndexSyncObserved,
|
|
2016
|
+
expected: 'indexSyncStatus=ready with syncedIndexSignature and syncedAtomCount populated',
|
|
2017
|
+
});
|
|
2018
|
+
}
|
|
2019
|
+
else if (queryVectorIndexAccelerationIndexSyncStatus === 'degraded'
|
|
2020
|
+
|| queryVectorIndexAccelerationSyncFailureCount > 0) {
|
|
2021
|
+
checks.push({
|
|
2022
|
+
checkId: 'query_vector_acceleration_index_sync_health',
|
|
2023
|
+
status: 'warn',
|
|
2024
|
+
message: 'External vector acceleration index sync is degraded and needs stabilization before release-grade ANN rollout.',
|
|
2025
|
+
observed: queryVectorAccelerationIndexSyncObserved,
|
|
2026
|
+
expected: 'indexSyncStatus=ready with syncFailureCount=0',
|
|
2027
|
+
});
|
|
2028
|
+
}
|
|
2029
|
+
else if (queryVectorIndexAccelerationIndexSyncStatus !== 'ready'
|
|
2030
|
+
|| queryVectorIndexAccelerationSyncRequestCount <= 0
|
|
2031
|
+
|| queryVectorIndexAccelerationSyncSuccessCount <= 0
|
|
2032
|
+
|| queryVectorIndexAccelerationSyncedAtomCount <= 0
|
|
2033
|
+
|| !queryVectorIndexAccelerationSyncedIndexSignature) {
|
|
2034
|
+
checks.push({
|
|
2035
|
+
checkId: 'query_vector_acceleration_index_sync_health',
|
|
2036
|
+
status: 'warn',
|
|
2037
|
+
message: 'External vector acceleration sync telemetry is incomplete for the latest runtime cycle.',
|
|
2038
|
+
observed: queryVectorAccelerationIndexSyncObserved,
|
|
2039
|
+
expected: 'indexSyncStatus=ready, syncRequestCount>0, syncSuccessCount>0, syncedIndexSignature!=empty, syncedAtomCount>0',
|
|
2040
|
+
});
|
|
2041
|
+
}
|
|
2042
|
+
else {
|
|
2043
|
+
checks.push({
|
|
2044
|
+
checkId: 'query_vector_acceleration_index_sync_health',
|
|
2045
|
+
status: 'pass',
|
|
2046
|
+
message: 'External vector acceleration index sync telemetry is healthy.',
|
|
2047
|
+
observed: queryVectorAccelerationIndexSyncObserved,
|
|
2048
|
+
expected: 'indexSyncStatus=ready with populated synced index telemetry',
|
|
2049
|
+
});
|
|
2050
|
+
}
|
|
2051
|
+
const queryVectorAccelerationHasCorrelationFields = (queryVectorIndexAccelerationLastRequestId.length > 0
|
|
2052
|
+
|| queryVectorIndexAccelerationLastErrorCode.length > 0
|
|
2053
|
+
|| queryVectorIndexAccelerationLastRetryAfterMs > 0);
|
|
2054
|
+
const queryVectorAccelerationTraceabilityObserved = (`adapterId=${queryVectorIndexAccelerationAdapterId || 'unknown'},`
|
|
2055
|
+
+ ` externalConnector=${queryVectorAccelerationExternalConnector},`
|
|
2056
|
+
+ ` requestCount=${queryVectorIndexAccelerationRequestCount},`
|
|
2057
|
+
+ ` healthStatus=${queryVectorIndexAccelerationHealthStatus},`
|
|
2058
|
+
+ ` circuitState=${queryVectorIndexAccelerationCircuitState},`
|
|
2059
|
+
+ ` lastRequestId=${queryVectorIndexAccelerationLastRequestId || '<none>'},`
|
|
2060
|
+
+ ` lastErrorCode=${queryVectorIndexAccelerationLastErrorCode || '<none>'},`
|
|
2061
|
+
+ ` lastRetryAfterMs=${queryVectorIndexAccelerationLastRetryAfterMs}`);
|
|
2062
|
+
if (!queryVectorIndexEnabled || queryVectorIndexStatus !== 'ready') {
|
|
2063
|
+
checks.push({
|
|
2064
|
+
checkId: 'query_vector_acceleration_traceability',
|
|
2065
|
+
status: 'warn',
|
|
2066
|
+
message: 'Vector acceleration traceability cannot be validated before vector index reaches ready state.',
|
|
2067
|
+
observed: `vectorIndexEnabled=${queryVectorIndexEnabled}, vectorIndexStatus=${queryVectorIndexStatus}`,
|
|
2068
|
+
expected: 'vectorIndexEnabled=true and vectorIndexStatus=ready',
|
|
2069
|
+
});
|
|
2070
|
+
}
|
|
2071
|
+
else if (!queryVectorIndexAccelerationEnabled) {
|
|
2072
|
+
checks.push({
|
|
2073
|
+
checkId: 'query_vector_acceleration_traceability',
|
|
2074
|
+
status: 'warn',
|
|
2075
|
+
message: 'Vector acceleration traceability check is running in fallback mode because acceleration is disabled.',
|
|
2076
|
+
observed: `accelerationEnabled=${queryVectorIndexAccelerationEnabled}, adapterId=${queryVectorIndexAccelerationAdapterId || 'unknown'}`,
|
|
2077
|
+
expected: 'accelerationEnabled=true for connector traceability validation',
|
|
2078
|
+
});
|
|
2079
|
+
}
|
|
2080
|
+
else if (!queryVectorAccelerationExternalConnector) {
|
|
2081
|
+
checks.push({
|
|
2082
|
+
checkId: 'query_vector_acceleration_traceability',
|
|
2083
|
+
status: 'pass',
|
|
2084
|
+
message: 'Vector acceleration is using local adapter path; external connector correlation fields are not required.',
|
|
2085
|
+
observed: queryVectorAccelerationTraceabilityObserved,
|
|
2086
|
+
expected: 'external connector adapters should emit request correlation fields',
|
|
2087
|
+
});
|
|
2088
|
+
}
|
|
2089
|
+
else if (!queryVectorAccelerationHasCorrelationFields
|
|
2090
|
+
&& (queryVectorIndexAccelerationHealthStatus === 'unavailable'
|
|
2091
|
+
|| queryVectorIndexAccelerationCircuitState === 'open')) {
|
|
2092
|
+
checks.push({
|
|
2093
|
+
checkId: 'query_vector_acceleration_traceability',
|
|
2094
|
+
status: 'fail',
|
|
2095
|
+
message: 'External vector acceleration connector is unstable but correlation fields are missing.',
|
|
2096
|
+
observed: queryVectorAccelerationTraceabilityObserved,
|
|
2097
|
+
expected: 'lastRequestId|lastErrorCode|lastRetryAfterMs should be present during unstable connector states',
|
|
2098
|
+
});
|
|
2099
|
+
}
|
|
2100
|
+
else if (!queryVectorAccelerationHasCorrelationFields
|
|
2101
|
+
&& (queryVectorIndexAccelerationHealthStatus === 'degraded'
|
|
2102
|
+
|| queryVectorIndexAccelerationConsecutiveFailures > 0
|
|
2103
|
+
|| queryVectorIndexAccelerationShortCircuitCount > 0
|
|
2104
|
+
|| queryVectorIndexAccelerationRequestCount >= thresholds.minQuerySampleSize)) {
|
|
2105
|
+
checks.push({
|
|
2106
|
+
checkId: 'query_vector_acceleration_traceability',
|
|
2107
|
+
status: 'warn',
|
|
2108
|
+
message: 'External vector acceleration connector shows instability but correlation fields are still sparse.',
|
|
2109
|
+
observed: queryVectorAccelerationTraceabilityObserved,
|
|
2110
|
+
expected: 'lastRequestId|lastErrorCode|lastRetryAfterMs should be captured for external connector troubleshooting',
|
|
2111
|
+
});
|
|
2112
|
+
}
|
|
2113
|
+
else if (!queryVectorAccelerationHasCorrelationFields) {
|
|
2114
|
+
checks.push({
|
|
2115
|
+
checkId: 'query_vector_acceleration_traceability',
|
|
2116
|
+
status: 'warn',
|
|
2117
|
+
message: 'External vector acceleration connector has not emitted correlation fields yet.',
|
|
2118
|
+
observed: queryVectorAccelerationTraceabilityObserved,
|
|
2119
|
+
expected: 'lastRequestId|lastErrorCode|lastRetryAfterMs should appear after representative connector traffic',
|
|
2120
|
+
});
|
|
2121
|
+
}
|
|
2122
|
+
else {
|
|
2123
|
+
checks.push({
|
|
2124
|
+
checkId: 'query_vector_acceleration_traceability',
|
|
2125
|
+
status: 'pass',
|
|
2126
|
+
message: 'External vector acceleration connector correlation fields are available for incident drilldown.',
|
|
2127
|
+
observed: queryVectorAccelerationTraceabilityObserved,
|
|
2128
|
+
expected: 'lastRequestId|lastErrorCode|lastRetryAfterMs available',
|
|
2129
|
+
});
|
|
2130
|
+
}
|
|
2131
|
+
const queryVectorAccelerationShortCircuitFailBudgetExceeded = (queryVectorIndexAccelerationShortCircuitCount
|
|
2132
|
+
>= thresholds.queryVectorAccelerationShortCircuitFailCount
|
|
2133
|
+
|| queryVectorIndexAccelerationShortCircuitRatioPct
|
|
2134
|
+
>= thresholds.queryVectorAccelerationShortCircuitFailRatioPct);
|
|
2135
|
+
const queryVectorAccelerationShortCircuitWarnBudgetExceeded = (queryVectorIndexAccelerationShortCircuitCount
|
|
2136
|
+
>= thresholds.queryVectorAccelerationShortCircuitWarnCount
|
|
2137
|
+
|| queryVectorIndexAccelerationShortCircuitRatioPct
|
|
2138
|
+
>= thresholds.queryVectorAccelerationShortCircuitWarnRatioPct);
|
|
2139
|
+
const queryVectorAccelerationConsecutiveFailuresFailBudgetExceeded = (queryVectorIndexAccelerationConsecutiveFailures
|
|
2140
|
+
>= thresholds.queryVectorAccelerationConsecutiveFailuresFailCount);
|
|
2141
|
+
const queryVectorAccelerationConsecutiveFailuresWarnBudgetExceeded = (queryVectorIndexAccelerationConsecutiveFailures
|
|
2142
|
+
>= thresholds.queryVectorAccelerationConsecutiveFailuresWarnCount);
|
|
2143
|
+
const queryVectorAccelerationHalfOpenSuccessFailBudgetExceeded = (queryVectorIndexAccelerationHalfOpenProbeCount > 0
|
|
2144
|
+
&& queryVectorIndexAccelerationHalfOpenSuccessRatePct
|
|
2145
|
+
< thresholds.queryVectorAccelerationHalfOpenSuccessFailRatioPct);
|
|
2146
|
+
const queryVectorAccelerationHalfOpenSuccessWarnBudgetExceeded = (queryVectorIndexAccelerationHalfOpenProbeCount > 0
|
|
2147
|
+
&& queryVectorIndexAccelerationHalfOpenSuccessRatePct
|
|
2148
|
+
< thresholds.queryVectorAccelerationHalfOpenSuccessWarnRatioPct);
|
|
2149
|
+
const shouldEvaluateQueryVectorAccelerationCircuitBudget = (queryVectorIndexEnabled
|
|
2150
|
+
&& queryVectorIndexStatus === 'ready'
|
|
2151
|
+
&& queryVectorIndexAccelerationEnabled);
|
|
2152
|
+
const queryVectorAccelerationCircuitFailBudgetExceeded = (shouldEvaluateQueryVectorAccelerationCircuitBudget
|
|
2153
|
+
&& (queryVectorIndexAccelerationCircuitState === 'open'
|
|
2154
|
+
|| queryVectorAccelerationShortCircuitFailBudgetExceeded
|
|
2155
|
+
|| queryVectorAccelerationConsecutiveFailuresFailBudgetExceeded
|
|
2156
|
+
|| queryVectorAccelerationHalfOpenSuccessFailBudgetExceeded));
|
|
2157
|
+
const queryVectorAccelerationCircuitWarnBudgetExceeded = (shouldEvaluateQueryVectorAccelerationCircuitBudget
|
|
2158
|
+
&& (queryVectorAccelerationCircuitFailBudgetExceeded
|
|
2159
|
+
|| queryVectorIndexAccelerationCircuitState === 'half_open'
|
|
2160
|
+
|| queryVectorAccelerationShortCircuitWarnBudgetExceeded
|
|
2161
|
+
|| queryVectorAccelerationConsecutiveFailuresWarnBudgetExceeded
|
|
2162
|
+
|| queryVectorAccelerationHalfOpenSuccessWarnBudgetExceeded));
|
|
2163
|
+
queryVectorIndexAccelerationCircuitWarnBudgetExceeded = queryVectorAccelerationCircuitWarnBudgetExceeded;
|
|
2164
|
+
queryVectorIndexAccelerationCircuitFailBudgetExceeded = queryVectorAccelerationCircuitFailBudgetExceeded;
|
|
2165
|
+
queryVectorIndexAccelerationCircuitBudgetStatus = queryVectorAccelerationCircuitFailBudgetExceeded
|
|
2166
|
+
? 'fail'
|
|
2167
|
+
: (queryVectorAccelerationCircuitWarnBudgetExceeded ? 'warn' : 'ok');
|
|
2168
|
+
if (!queryVectorIndexEnabled || queryVectorIndexStatus !== 'ready') {
|
|
2169
|
+
checks.push({
|
|
2170
|
+
checkId: 'query_vector_acceleration_circuit_state',
|
|
2171
|
+
status: 'warn',
|
|
2172
|
+
message: 'Vector acceleration circuit state cannot be validated before vector index reaches ready state.',
|
|
2173
|
+
observed: `vectorIndexEnabled=${queryVectorIndexEnabled}, vectorIndexStatus=${queryVectorIndexStatus}`,
|
|
2174
|
+
expected: 'vectorIndexEnabled=true and vectorIndexStatus=ready',
|
|
2175
|
+
});
|
|
2176
|
+
}
|
|
2177
|
+
else if (!queryVectorIndexAccelerationEnabled) {
|
|
2178
|
+
checks.push({
|
|
2179
|
+
checkId: 'query_vector_acceleration_circuit_state',
|
|
2180
|
+
status: 'warn',
|
|
2181
|
+
message: 'Vector acceleration circuit governance is running in fallback mode because acceleration is disabled.',
|
|
2182
|
+
observed: `accelerationEnabled=${queryVectorIndexAccelerationEnabled}, circuitState=${queryVectorIndexAccelerationCircuitState}`,
|
|
2183
|
+
expected: 'accelerationEnabled=true and circuitState=closed|unknown',
|
|
2184
|
+
});
|
|
2185
|
+
}
|
|
2186
|
+
else if (queryVectorIndexAccelerationCircuitState === 'open') {
|
|
2187
|
+
checks.push({
|
|
2188
|
+
checkId: 'query_vector_acceleration_circuit_state',
|
|
2189
|
+
status: 'fail',
|
|
2190
|
+
message: 'Vector acceleration circuit is open; connector calls are currently short-circuited.',
|
|
2191
|
+
observed: (`circuitState=${queryVectorIndexAccelerationCircuitState},`
|
|
2192
|
+
+ ` shortCircuitCount=${queryVectorIndexAccelerationShortCircuitCount},`
|
|
2193
|
+
+ ` shortCircuitRatio=${queryVectorIndexAccelerationShortCircuitRatioPct}%,`
|
|
2194
|
+
+ ` consecutiveFailures=${queryVectorIndexAccelerationConsecutiveFailures}`),
|
|
2195
|
+
expected: 'circuitState=closed|unknown',
|
|
2196
|
+
});
|
|
2197
|
+
}
|
|
2198
|
+
else if (queryVectorIndexAccelerationCircuitState === 'half_open'
|
|
2199
|
+
&& queryVectorAccelerationCircuitFailBudgetExceeded) {
|
|
2200
|
+
checks.push({
|
|
2201
|
+
checkId: 'query_vector_acceleration_circuit_state',
|
|
2202
|
+
status: 'fail',
|
|
2203
|
+
message: 'Vector acceleration circuit is half-open but recovery/instability metrics exceed fail budgets.',
|
|
2204
|
+
observed: (`circuitState=${queryVectorIndexAccelerationCircuitState},`
|
|
2205
|
+
+ ` shortCircuitCount=${queryVectorIndexAccelerationShortCircuitCount},`
|
|
2206
|
+
+ ` shortCircuitRatio=${queryVectorIndexAccelerationShortCircuitRatioPct}%,`
|
|
2207
|
+
+ ` consecutiveFailures=${queryVectorIndexAccelerationConsecutiveFailures},`
|
|
2208
|
+
+ ` halfOpenProbeCount=${queryVectorIndexAccelerationHalfOpenProbeCount},`
|
|
2209
|
+
+ ` halfOpenSuccessRate=${queryVectorIndexAccelerationHalfOpenSuccessRatePct}%`),
|
|
2210
|
+
expected: (`shortCircuitCount<${thresholds.queryVectorAccelerationShortCircuitFailCount},`
|
|
2211
|
+
+ ` shortCircuitRatio<${thresholds.queryVectorAccelerationShortCircuitFailRatioPct}%,`
|
|
2212
|
+
+ ` consecutiveFailures<${thresholds.queryVectorAccelerationConsecutiveFailuresFailCount},`
|
|
2213
|
+
+ ` halfOpenSuccessRate>=${thresholds.queryVectorAccelerationHalfOpenSuccessFailRatioPct}% (if probes>0)`),
|
|
2214
|
+
});
|
|
2215
|
+
}
|
|
2216
|
+
else if (queryVectorIndexAccelerationCircuitState === 'half_open') {
|
|
2217
|
+
checks.push({
|
|
2218
|
+
checkId: 'query_vector_acceleration_circuit_state',
|
|
2219
|
+
status: 'warn',
|
|
2220
|
+
message: 'Vector acceleration circuit is half-open and recovery probes are in progress.',
|
|
2221
|
+
observed: (`circuitState=${queryVectorIndexAccelerationCircuitState},`
|
|
2222
|
+
+ ` shortCircuitRatio=${queryVectorIndexAccelerationShortCircuitRatioPct}%,`
|
|
2223
|
+
+ ` halfOpenProbeCount=${queryVectorIndexAccelerationHalfOpenProbeCount},`
|
|
2224
|
+
+ ` halfOpenSuccessRate=${queryVectorIndexAccelerationHalfOpenSuccessRatePct}%`),
|
|
2225
|
+
expected: (`circuitState=closed|unknown and halfOpenSuccessRate>=${thresholds.queryVectorAccelerationHalfOpenSuccessWarnRatioPct}%`
|
|
2226
|
+
+ ' (if probes>0)'),
|
|
2227
|
+
});
|
|
2228
|
+
}
|
|
2229
|
+
else if (queryVectorAccelerationCircuitFailBudgetExceeded) {
|
|
2230
|
+
checks.push({
|
|
2231
|
+
checkId: 'query_vector_acceleration_circuit_state',
|
|
2232
|
+
status: 'fail',
|
|
2233
|
+
message: 'Vector acceleration circuit is closed but instability metrics exceed fail budgets.',
|
|
2234
|
+
observed: (`circuitState=${queryVectorIndexAccelerationCircuitState},`
|
|
2235
|
+
+ ` shortCircuitCount=${queryVectorIndexAccelerationShortCircuitCount},`
|
|
2236
|
+
+ ` shortCircuitRatio=${queryVectorIndexAccelerationShortCircuitRatioPct}%,`
|
|
2237
|
+
+ ` consecutiveFailures=${queryVectorIndexAccelerationConsecutiveFailures},`
|
|
2238
|
+
+ ` halfOpenProbeCount=${queryVectorIndexAccelerationHalfOpenProbeCount},`
|
|
2239
|
+
+ ` halfOpenSuccessRate=${queryVectorIndexAccelerationHalfOpenSuccessRatePct}%`),
|
|
2240
|
+
expected: (`shortCircuitCount<${thresholds.queryVectorAccelerationShortCircuitFailCount},`
|
|
2241
|
+
+ ` shortCircuitRatio<${thresholds.queryVectorAccelerationShortCircuitFailRatioPct}%,`
|
|
2242
|
+
+ ` consecutiveFailures<${thresholds.queryVectorAccelerationConsecutiveFailuresFailCount},`
|
|
2243
|
+
+ ` halfOpenSuccessRate>=${thresholds.queryVectorAccelerationHalfOpenSuccessFailRatioPct}% (if probes>0)`),
|
|
2244
|
+
});
|
|
2245
|
+
}
|
|
2246
|
+
else if (queryVectorAccelerationCircuitWarnBudgetExceeded) {
|
|
2247
|
+
checks.push({
|
|
2248
|
+
checkId: 'query_vector_acceleration_circuit_state',
|
|
2249
|
+
status: 'warn',
|
|
2250
|
+
message: 'Vector acceleration circuit is closed but instability metrics exceed warn budgets.',
|
|
2251
|
+
observed: (`circuitState=${queryVectorIndexAccelerationCircuitState},`
|
|
2252
|
+
+ ` shortCircuitCount=${queryVectorIndexAccelerationShortCircuitCount},`
|
|
2253
|
+
+ ` shortCircuitRatio=${queryVectorIndexAccelerationShortCircuitRatioPct}%,`
|
|
2254
|
+
+ ` consecutiveFailures=${queryVectorIndexAccelerationConsecutiveFailures},`
|
|
2255
|
+
+ ` halfOpenProbeCount=${queryVectorIndexAccelerationHalfOpenProbeCount},`
|
|
2256
|
+
+ ` halfOpenSuccessRate=${queryVectorIndexAccelerationHalfOpenSuccessRatePct}%`),
|
|
2257
|
+
expected: (`shortCircuitCount<${thresholds.queryVectorAccelerationShortCircuitWarnCount},`
|
|
2258
|
+
+ ` shortCircuitRatio<${thresholds.queryVectorAccelerationShortCircuitWarnRatioPct}%,`
|
|
2259
|
+
+ ` consecutiveFailures<${thresholds.queryVectorAccelerationConsecutiveFailuresWarnCount},`
|
|
2260
|
+
+ ` halfOpenSuccessRate>=${thresholds.queryVectorAccelerationHalfOpenSuccessWarnRatioPct}% (if probes>0)`),
|
|
2261
|
+
});
|
|
2262
|
+
}
|
|
2263
|
+
else {
|
|
2264
|
+
checks.push({
|
|
2265
|
+
checkId: 'query_vector_acceleration_circuit_state',
|
|
2266
|
+
status: 'pass',
|
|
2267
|
+
message: 'Vector acceleration circuit is closed and stable.',
|
|
2268
|
+
observed: (`circuitState=${queryVectorIndexAccelerationCircuitState},`
|
|
2269
|
+
+ ` requestCount=${queryVectorIndexAccelerationRequestCount},`
|
|
2270
|
+
+ ` retryCount=${queryVectorIndexAccelerationRetryCount},`
|
|
2271
|
+
+ ` shortCircuitRatio=${queryVectorIndexAccelerationShortCircuitRatioPct}%`),
|
|
2272
|
+
expected: 'circuitState=closed|unknown',
|
|
2273
|
+
});
|
|
2274
|
+
}
|
|
2275
|
+
const queryVectorAccelerationCalibrationObserved = (`externalConnector=${queryVectorAccelerationExternalConnector},`
|
|
2276
|
+
+ ` healthStatus=${queryVectorIndexAccelerationHealthStatus},`
|
|
2277
|
+
+ ` circuitState=${queryVectorIndexAccelerationCircuitState},`
|
|
2278
|
+
+ ` circuitBudgetStatus=${queryVectorIndexAccelerationCircuitBudgetStatus},`
|
|
2279
|
+
+ ` indexSyncStatus=${queryVectorIndexAccelerationIndexSyncStatus},`
|
|
2280
|
+
+ ` syncSuccessCount=${queryVectorIndexAccelerationSyncSuccessCount},`
|
|
2281
|
+
+ ` syncedIndexSignature=${queryVectorIndexAccelerationSyncedIndexSignature || '<none>'},`
|
|
2282
|
+
+ ` syncedAtomCount=${queryVectorIndexAccelerationSyncedAtomCount},`
|
|
2283
|
+
+ ` sampleReady=${queryVectorAccelerationPrefilterSampleReady},`
|
|
2284
|
+
+ ` selectionActive=${queryVectorAccelerationPrefilterSelectionActive},`
|
|
2285
|
+
+ ` stableConnector=${queryVectorAccelerationStableConnector},`
|
|
2286
|
+
+ ` canEvaluateCandidateRatio=${queryVectorAccelerationCanEvaluatePrefilterReduction},`
|
|
2287
|
+
+ ` prefilterBudgetStatus=${queryVectorAccelerationPrefilterReductionSevere ? 'fail' : (queryVectorAccelerationPrefilterReductionWeak ? 'warn' : 'ok')},`
|
|
2288
|
+
+ ` hasCorrelationFields=${queryVectorAccelerationHasCorrelationFields}`);
|
|
2289
|
+
const queryVectorAccelerationIndexSyncReadyForCalibration = (!queryVectorAccelerationExternalConnector
|
|
2290
|
+
|| (queryVectorIndexAccelerationIndexSyncStatus === 'ready'
|
|
2291
|
+
&& queryVectorIndexAccelerationSyncRequestCount > 0
|
|
2292
|
+
&& queryVectorIndexAccelerationSyncSuccessCount > 0
|
|
2293
|
+
&& queryVectorIndexAccelerationSyncedAtomCount > 0
|
|
2294
|
+
&& queryVectorIndexAccelerationSyncedIndexSignature.length > 0));
|
|
2295
|
+
const queryVectorAccelerationTraceabilityReadyForCalibration = (!queryVectorAccelerationExternalConnector
|
|
2296
|
+
|| queryVectorAccelerationHasCorrelationFields);
|
|
2297
|
+
const queryVectorAccelerationPrefilterBudgetStatus = (queryVectorAccelerationPrefilterSampleReady
|
|
2298
|
+
&& queryVectorAccelerationPrefilterSelectionActive
|
|
2299
|
+
&& queryVectorAccelerationCanEvaluatePrefilterReduction)
|
|
2300
|
+
? (queryVectorAccelerationPrefilterReductionSevere
|
|
2301
|
+
? 'fail'
|
|
2302
|
+
: (queryVectorAccelerationPrefilterReductionWeak ? 'warn' : 'ok'))
|
|
2303
|
+
: 'ok';
|
|
2304
|
+
if (!queryVectorIndexEnabled || queryVectorIndexStatus !== 'ready') {
|
|
2305
|
+
checks.push({
|
|
2306
|
+
checkId: 'query_vector_acceleration_calibration_readiness',
|
|
2307
|
+
status: 'warn',
|
|
2308
|
+
message: 'ANN calibration readiness cannot be evaluated before the vector index reaches ready state.',
|
|
2309
|
+
observed: `vectorIndexEnabled=${queryVectorIndexEnabled}, vectorIndexStatus=${queryVectorIndexStatus}`,
|
|
2310
|
+
expected: 'vectorIndexEnabled=true and vectorIndexStatus=ready',
|
|
2311
|
+
});
|
|
2312
|
+
}
|
|
2313
|
+
else if (!queryVectorIndexAccelerationEnabled) {
|
|
2314
|
+
checks.push({
|
|
2315
|
+
checkId: 'query_vector_acceleration_calibration_readiness',
|
|
2316
|
+
status: 'warn',
|
|
2317
|
+
message: 'ANN calibration readiness is blocked because vector acceleration is disabled.',
|
|
2318
|
+
observed: `accelerationEnabled=${queryVectorIndexAccelerationEnabled}, mode=${queryVectorIndexAccelerationMode}`,
|
|
2319
|
+
expected: 'accelerationEnabled=true and mode=ann_prefilter',
|
|
2320
|
+
});
|
|
2321
|
+
}
|
|
2322
|
+
else if (queryVectorIndexAccelerationMode !== 'ann_prefilter') {
|
|
2323
|
+
checks.push({
|
|
2324
|
+
checkId: 'query_vector_acceleration_calibration_readiness',
|
|
2325
|
+
status: 'warn',
|
|
2326
|
+
message: 'ANN calibration readiness is pending because ann_prefilter mode is not active.',
|
|
2327
|
+
observed: queryVectorAccelerationCalibrationObserved,
|
|
2328
|
+
expected: 'mode=ann_prefilter with representative telemetry',
|
|
2329
|
+
});
|
|
2330
|
+
}
|
|
2331
|
+
else if (queryVectorIndexAccelerationHealthStatus === 'unavailable'
|
|
2332
|
+
|| queryVectorIndexAccelerationCircuitBudgetStatus === 'fail'
|
|
2333
|
+
|| queryVectorAccelerationPrefilterBudgetStatus === 'fail') {
|
|
2334
|
+
checks.push({
|
|
2335
|
+
checkId: 'query_vector_acceleration_calibration_readiness',
|
|
2336
|
+
status: 'fail',
|
|
2337
|
+
message: 'ANN calibration readiness is blocked by unstable connector or fail-budget telemetry.',
|
|
2338
|
+
observed: queryVectorAccelerationCalibrationObserved,
|
|
2339
|
+
expected: 'healthStatus=ready|unknown, circuitBudgetStatus=ok|warn, prefilterBudgetStatus=ok|warn',
|
|
2340
|
+
});
|
|
2341
|
+
}
|
|
2342
|
+
else if (!queryVectorAccelerationIndexSyncReadyForCalibration
|
|
2343
|
+
|| !queryVectorAccelerationPrefilterSampleReady
|
|
2344
|
+
|| !queryVectorAccelerationPrefilterSelectionActive
|
|
2345
|
+
|| !queryVectorAccelerationStableConnector
|
|
2346
|
+
|| !queryVectorAccelerationCanEvaluatePrefilterReduction
|
|
2347
|
+
|| !queryVectorAccelerationTraceabilityReadyForCalibration
|
|
2348
|
+
|| queryVectorIndexAccelerationCircuitBudgetStatus === 'warn'
|
|
2349
|
+
|| queryVectorAccelerationPrefilterBudgetStatus === 'warn') {
|
|
2350
|
+
checks.push({
|
|
2351
|
+
checkId: 'query_vector_acceleration_calibration_readiness',
|
|
2352
|
+
status: 'warn',
|
|
2353
|
+
message: 'ANN calibration readiness is not closed yet; collect stable representative telemetry before threshold tuning.',
|
|
2354
|
+
observed: queryVectorAccelerationCalibrationObserved,
|
|
2355
|
+
expected: 'sync telemetry ready, sampleReady=true, selectionActive=true, stableConnector=true, canEvaluateCandidateRatio=true, hasCorrelationFields=true when external, and budgets not exceeded',
|
|
2356
|
+
});
|
|
2357
|
+
}
|
|
2358
|
+
else {
|
|
2359
|
+
checks.push({
|
|
2360
|
+
checkId: 'query_vector_acceleration_calibration_readiness',
|
|
2361
|
+
status: 'pass',
|
|
2362
|
+
message: 'ANN calibration readiness prerequisites are satisfied for the current runtime window.',
|
|
2363
|
+
observed: queryVectorAccelerationCalibrationObserved,
|
|
2364
|
+
expected: 'representative ANN telemetry available with stable connector and evaluable budget signals',
|
|
2365
|
+
});
|
|
2366
|
+
}
|
|
2367
|
+
}
|
|
2368
|
+
else {
|
|
2369
|
+
checks.push({
|
|
2370
|
+
checkId: 'query_vector_index_status',
|
|
2371
|
+
status: 'pass',
|
|
2372
|
+
message: 'Local vector index status check skipped because local_vector backend is not active.',
|
|
2373
|
+
observed: `configuredQueryBackend=${params.configuredQueryBackend}`,
|
|
2374
|
+
expected: 'configuredQueryBackend=local_vector',
|
|
2375
|
+
});
|
|
2376
|
+
checks.push({
|
|
2377
|
+
checkId: 'query_vector_index_persistence',
|
|
2378
|
+
status: 'pass',
|
|
2379
|
+
message: 'Local vector index persistence check skipped because local_vector backend is not active.',
|
|
2380
|
+
observed: `configuredQueryBackend=${params.configuredQueryBackend}`,
|
|
2381
|
+
expected: 'configuredQueryBackend=local_vector',
|
|
2382
|
+
});
|
|
2383
|
+
checks.push({
|
|
2384
|
+
checkId: 'query_vector_acceleration_mode',
|
|
2385
|
+
status: 'pass',
|
|
2386
|
+
message: 'Local vector acceleration mode check skipped because local_vector backend is not active.',
|
|
2387
|
+
observed: `configuredQueryBackend=${params.configuredQueryBackend}`,
|
|
2388
|
+
expected: 'configuredQueryBackend=local_vector',
|
|
2389
|
+
});
|
|
2390
|
+
checks.push({
|
|
2391
|
+
checkId: 'query_vector_acceleration_representation_consistency',
|
|
2392
|
+
status: 'pass',
|
|
2393
|
+
message: 'Local vector acceleration representation-consistency check skipped because local_vector backend is not active.',
|
|
2394
|
+
observed: `configuredQueryBackend=${params.configuredQueryBackend}`,
|
|
2395
|
+
expected: 'configuredQueryBackend=local_vector',
|
|
2396
|
+
});
|
|
2397
|
+
checks.push({
|
|
2398
|
+
checkId: 'query_vector_acceleration_prefilter_effectiveness',
|
|
2399
|
+
status: 'pass',
|
|
2400
|
+
message: 'Local vector acceleration prefilter-effectiveness check skipped because local_vector backend is not active.',
|
|
2401
|
+
observed: `configuredQueryBackend=${params.configuredQueryBackend}`,
|
|
2402
|
+
expected: 'configuredQueryBackend=local_vector',
|
|
2403
|
+
});
|
|
2404
|
+
checks.push({
|
|
2405
|
+
checkId: 'query_vector_acceleration_calibration_readiness',
|
|
2406
|
+
status: 'pass',
|
|
2407
|
+
message: 'Local vector acceleration calibration-readiness check skipped because local_vector backend is not active.',
|
|
2408
|
+
observed: `configuredQueryBackend=${params.configuredQueryBackend}`,
|
|
2409
|
+
expected: 'configuredQueryBackend=local_vector',
|
|
2410
|
+
});
|
|
2411
|
+
checks.push({
|
|
2412
|
+
checkId: 'query_vector_acceleration_health',
|
|
2413
|
+
status: 'pass',
|
|
2414
|
+
message: 'Local vector acceleration health check skipped because local_vector backend is not active.',
|
|
2415
|
+
observed: `configuredQueryBackend=${params.configuredQueryBackend}`,
|
|
2416
|
+
expected: 'configuredQueryBackend=local_vector',
|
|
2417
|
+
});
|
|
2418
|
+
checks.push({
|
|
2419
|
+
checkId: 'query_vector_acceleration_index_sync_health',
|
|
2420
|
+
status: 'pass',
|
|
2421
|
+
message: 'Local vector acceleration index sync check skipped because local_vector backend is not active.',
|
|
2422
|
+
observed: `configuredQueryBackend=${params.configuredQueryBackend}`,
|
|
2423
|
+
expected: 'configuredQueryBackend=local_vector',
|
|
2424
|
+
});
|
|
2425
|
+
checks.push({
|
|
2426
|
+
checkId: 'query_vector_acceleration_traceability',
|
|
2427
|
+
status: 'pass',
|
|
2428
|
+
message: 'Local vector acceleration traceability check skipped because local_vector backend is not active.',
|
|
2429
|
+
observed: `configuredQueryBackend=${params.configuredQueryBackend}`,
|
|
2430
|
+
expected: 'configuredQueryBackend=local_vector',
|
|
2431
|
+
});
|
|
2432
|
+
checks.push({
|
|
2433
|
+
checkId: 'query_vector_acceleration_circuit_state',
|
|
2434
|
+
status: 'pass',
|
|
2435
|
+
message: 'Local vector acceleration circuit-state check skipped because local_vector backend is not active.',
|
|
2436
|
+
observed: `configuredQueryBackend=${params.configuredQueryBackend}`,
|
|
2437
|
+
expected: 'configuredQueryBackend=local_vector',
|
|
2438
|
+
});
|
|
2439
|
+
}
|
|
2440
|
+
if (params.configuredQueryBackend === 'keyword_only') {
|
|
2441
|
+
checks.push({
|
|
2442
|
+
checkId: 'query_graph_retrieval_capability',
|
|
2443
|
+
status: 'warn',
|
|
2444
|
+
message: 'Keyword-only query backend limits graph/semantic retrieval depth.',
|
|
2445
|
+
observed: `configuredQueryBackend=${params.configuredQueryBackend}`,
|
|
2446
|
+
expected: 'configuredQueryBackend=local_hybrid|local_vector',
|
|
2447
|
+
});
|
|
2448
|
+
}
|
|
2449
|
+
else {
|
|
2450
|
+
checks.push({
|
|
2451
|
+
checkId: 'query_graph_retrieval_capability',
|
|
2452
|
+
status: 'pass',
|
|
2453
|
+
message: 'Graph-aware query backend is enabled.',
|
|
2454
|
+
observed: `configuredQueryBackend=${params.configuredQueryBackend}`,
|
|
2455
|
+
});
|
|
2456
|
+
}
|
|
2457
|
+
if (safeQueryCount < thresholds.minQuerySampleSize) {
|
|
2458
|
+
checks.push({
|
|
2459
|
+
checkId: 'query_fallback_ratio',
|
|
2460
|
+
status: 'warn',
|
|
2461
|
+
message: 'Insufficient query sample size for stable fallback ratio assessment.',
|
|
2462
|
+
observed: `queryCount=${safeQueryCount}, fallbackCount=${safeFallbackCount}, fallbackRatio=${queryFallbackRatioPct}%`,
|
|
2463
|
+
expected: `queryCount>=${thresholds.minQuerySampleSize}`,
|
|
2464
|
+
});
|
|
2465
|
+
}
|
|
2466
|
+
else if (queryFallbackRatioPct > thresholds.queryFallbackFailRatioPct) {
|
|
2467
|
+
checks.push({
|
|
2468
|
+
checkId: 'query_fallback_ratio',
|
|
2469
|
+
status: 'fail',
|
|
2470
|
+
message: 'Query backend fallback ratio exceeds hard reliability ceiling.',
|
|
2471
|
+
observed: `fallbackRatio=${queryFallbackRatioPct}%`,
|
|
2472
|
+
expected: `fallbackRatio<=${thresholds.queryFallbackFailRatioPct}%`,
|
|
2473
|
+
});
|
|
2474
|
+
}
|
|
2475
|
+
else if (queryFallbackRatioPct > thresholds.queryFallbackWarnRatioPct) {
|
|
2476
|
+
checks.push({
|
|
2477
|
+
checkId: 'query_fallback_ratio',
|
|
2478
|
+
status: 'warn',
|
|
2479
|
+
message: 'Query backend fallback ratio exceeds preferred budget.',
|
|
2480
|
+
observed: `fallbackRatio=${queryFallbackRatioPct}%`,
|
|
2481
|
+
expected: `fallbackRatio<=${thresholds.queryFallbackWarnRatioPct}%`,
|
|
2482
|
+
});
|
|
2483
|
+
}
|
|
2484
|
+
else {
|
|
2485
|
+
checks.push({
|
|
2486
|
+
checkId: 'query_fallback_ratio',
|
|
2487
|
+
status: 'pass',
|
|
2488
|
+
message: 'Query backend fallback ratio is within budget.',
|
|
2489
|
+
observed: `fallbackRatio=${queryFallbackRatioPct}%`,
|
|
2490
|
+
expected: `fallbackRatio<=${thresholds.queryFallbackWarnRatioPct}%`,
|
|
2491
|
+
});
|
|
2492
|
+
}
|
|
2493
|
+
if (queryExplainabilitySampleCount <= 0) {
|
|
2494
|
+
checks.push({
|
|
2495
|
+
checkId: 'query_evidence_coverage_ratio',
|
|
2496
|
+
status: 'pass',
|
|
2497
|
+
message: 'Explainability evidence telemetry is unavailable; quality gate deferred until samples are collected.',
|
|
2498
|
+
observed: `sampleCount=${queryExplainabilitySampleCount}, evidenceCoverage=${queryEvidenceCoverageRatioPct}%`,
|
|
2499
|
+
expected: `sampleCount>=${thresholds.minQuerySampleSize}`,
|
|
2500
|
+
});
|
|
2501
|
+
checks.push({
|
|
2502
|
+
checkId: 'query_temporal_validity_ratio',
|
|
2503
|
+
status: 'pass',
|
|
2504
|
+
message: 'Temporal-validity telemetry is unavailable; quality gate deferred until samples are collected.',
|
|
2505
|
+
observed: `sampleCount=${queryExplainabilitySampleCount}, temporalValidity=${queryTemporalValidityPassRatioPct}%`,
|
|
2506
|
+
expected: `sampleCount>=${thresholds.minQuerySampleSize}`,
|
|
2507
|
+
});
|
|
2508
|
+
}
|
|
2509
|
+
else if (queryExplainabilitySampleCount < thresholds.minQuerySampleSize) {
|
|
2510
|
+
checks.push({
|
|
2511
|
+
checkId: 'query_evidence_coverage_ratio',
|
|
2512
|
+
status: 'warn',
|
|
2513
|
+
message: 'Insufficient query sample size for stable explainability coverage assessment.',
|
|
2514
|
+
observed: `sampleCount=${queryExplainabilitySampleCount}, evidenceCoverage=${queryEvidenceCoverageRatioPct}%`,
|
|
2515
|
+
expected: `sampleCount>=${thresholds.minQuerySampleSize}`,
|
|
2516
|
+
});
|
|
2517
|
+
checks.push({
|
|
2518
|
+
checkId: 'query_temporal_validity_ratio',
|
|
2519
|
+
status: 'warn',
|
|
2520
|
+
message: 'Insufficient query sample size for stable temporal validity assessment.',
|
|
2521
|
+
observed: `sampleCount=${queryExplainabilitySampleCount}, temporalValidity=${queryTemporalValidityPassRatioPct}%`,
|
|
2522
|
+
expected: `sampleCount>=${thresholds.minQuerySampleSize}`,
|
|
2523
|
+
});
|
|
2524
|
+
}
|
|
2525
|
+
else {
|
|
2526
|
+
if (queryEvidenceCoverageRatioPct < thresholds.queryEvidenceCoverageFailRatioPct) {
|
|
2527
|
+
checks.push({
|
|
2528
|
+
checkId: 'query_evidence_coverage_ratio',
|
|
2529
|
+
status: 'fail',
|
|
2530
|
+
message: 'Query evidence coverage ratio is below hard explainability floor.',
|
|
2531
|
+
observed: `evidenceCoverage=${queryEvidenceCoverageRatioPct}%`,
|
|
2532
|
+
expected: `evidenceCoverage>=${thresholds.queryEvidenceCoverageFailRatioPct}%`,
|
|
2533
|
+
});
|
|
2534
|
+
}
|
|
2535
|
+
else if (queryEvidenceCoverageRatioPct < thresholds.queryEvidenceCoverageWarnRatioPct) {
|
|
2536
|
+
checks.push({
|
|
2537
|
+
checkId: 'query_evidence_coverage_ratio',
|
|
2538
|
+
status: 'warn',
|
|
2539
|
+
message: 'Query evidence coverage ratio is below preferred explainability target.',
|
|
2540
|
+
observed: `evidenceCoverage=${queryEvidenceCoverageRatioPct}%`,
|
|
2541
|
+
expected: `evidenceCoverage>=${thresholds.queryEvidenceCoverageWarnRatioPct}%`,
|
|
2542
|
+
});
|
|
2543
|
+
}
|
|
2544
|
+
else {
|
|
2545
|
+
checks.push({
|
|
2546
|
+
checkId: 'query_evidence_coverage_ratio',
|
|
2547
|
+
status: 'pass',
|
|
2548
|
+
message: 'Query evidence coverage ratio is within explainability target.',
|
|
2549
|
+
observed: `evidenceCoverage=${queryEvidenceCoverageRatioPct}%`,
|
|
2550
|
+
expected: `evidenceCoverage>=${thresholds.queryEvidenceCoverageWarnRatioPct}%`,
|
|
2551
|
+
});
|
|
2552
|
+
}
|
|
2553
|
+
if (queryTemporalValidityPassRatioPct < thresholds.queryTemporalValidityFailRatioPct) {
|
|
2554
|
+
checks.push({
|
|
2555
|
+
checkId: 'query_temporal_validity_ratio',
|
|
2556
|
+
status: 'fail',
|
|
2557
|
+
message: 'Query temporal validity pass ratio is below hard floor.',
|
|
2558
|
+
observed: `temporalValidity=${queryTemporalValidityPassRatioPct}%`,
|
|
2559
|
+
expected: `temporalValidity>=${thresholds.queryTemporalValidityFailRatioPct}%`,
|
|
2560
|
+
});
|
|
2561
|
+
}
|
|
2562
|
+
else if (queryTemporalValidityPassRatioPct < thresholds.queryTemporalValidityWarnRatioPct) {
|
|
2563
|
+
checks.push({
|
|
2564
|
+
checkId: 'query_temporal_validity_ratio',
|
|
2565
|
+
status: 'warn',
|
|
2566
|
+
message: 'Query temporal validity pass ratio is below preferred target.',
|
|
2567
|
+
observed: `temporalValidity=${queryTemporalValidityPassRatioPct}%`,
|
|
2568
|
+
expected: `temporalValidity>=${thresholds.queryTemporalValidityWarnRatioPct}%`,
|
|
2569
|
+
});
|
|
2570
|
+
}
|
|
2571
|
+
else {
|
|
2572
|
+
checks.push({
|
|
2573
|
+
checkId: 'query_temporal_validity_ratio',
|
|
2574
|
+
status: 'pass',
|
|
2575
|
+
message: 'Query temporal validity pass ratio is within target.',
|
|
2576
|
+
observed: `temporalValidity=${queryTemporalValidityPassRatioPct}%`,
|
|
2577
|
+
expected: `temporalValidity>=${thresholds.queryTemporalValidityWarnRatioPct}%`,
|
|
2578
|
+
});
|
|
2579
|
+
}
|
|
2580
|
+
}
|
|
2581
|
+
if (queryBackendComparisonSampleCount <= 0) {
|
|
2582
|
+
checks.push({
|
|
2583
|
+
checkId: 'query_backend_explainability_gap',
|
|
2584
|
+
status: 'pass',
|
|
2585
|
+
message: 'Backend comparison explainability telemetry is unavailable; gap gate deferred until samples are collected.',
|
|
2586
|
+
observed: `sampleCount=${queryBackendComparisonSampleCount}, maxGap=${queryBackendComparisonMaxExplainabilityGapRatioPct}%`,
|
|
2587
|
+
expected: `sampleCount>=${thresholds.minQuerySampleSize}`,
|
|
2588
|
+
});
|
|
2589
|
+
}
|
|
2590
|
+
else if (queryBackendComparisonSampleCount < thresholds.minQuerySampleSize) {
|
|
2591
|
+
checks.push({
|
|
2592
|
+
checkId: 'query_backend_explainability_gap',
|
|
2593
|
+
status: 'warn',
|
|
2594
|
+
message: 'Backend comparison sample size is insufficient for stable explainability-gap assessment.',
|
|
2595
|
+
observed: [
|
|
2596
|
+
`sampleCount=${queryBackendComparisonSampleCount}`,
|
|
2597
|
+
`evidenceGap=${queryBackendComparisonEvidenceGapRatioPct}%`,
|
|
2598
|
+
`relationGap=${queryBackendComparisonRelationGapRatioPct}%`,
|
|
2599
|
+
`temporalGap=${queryBackendComparisonTemporalGapRatioPct}%`,
|
|
2600
|
+
`maxGap=${queryBackendComparisonMaxExplainabilityGapRatioPct}%`,
|
|
2601
|
+
].join(', '),
|
|
2602
|
+
expected: `sampleCount>=${thresholds.minQuerySampleSize}`,
|
|
2603
|
+
});
|
|
2604
|
+
}
|
|
2605
|
+
else if (queryBackendComparisonMaxExplainabilityGapRatioPct > thresholds.queryBackendExplainabilityGapFailRatioPct) {
|
|
2606
|
+
checks.push({
|
|
2607
|
+
checkId: 'query_backend_explainability_gap',
|
|
2608
|
+
status: 'fail',
|
|
2609
|
+
message: 'Cross-backend explainability gap exceeds hard consistency ceiling.',
|
|
2610
|
+
observed: [
|
|
2611
|
+
`sampleCount=${queryBackendComparisonSampleCount}`,
|
|
2612
|
+
`evidenceGap=${queryBackendComparisonEvidenceGapRatioPct}%`,
|
|
2613
|
+
`relationGap=${queryBackendComparisonRelationGapRatioPct}%`,
|
|
2614
|
+
`temporalGap=${queryBackendComparisonTemporalGapRatioPct}%`,
|
|
2615
|
+
`maxGap=${queryBackendComparisonMaxExplainabilityGapRatioPct}%`,
|
|
2616
|
+
].join(', '),
|
|
2617
|
+
expected: `maxGap<=${thresholds.queryBackendExplainabilityGapFailRatioPct}%`,
|
|
2618
|
+
});
|
|
2619
|
+
}
|
|
2620
|
+
else if (queryBackendComparisonMaxExplainabilityGapRatioPct > thresholds.queryBackendExplainabilityGapWarnRatioPct) {
|
|
2621
|
+
checks.push({
|
|
2622
|
+
checkId: 'query_backend_explainability_gap',
|
|
2623
|
+
status: 'warn',
|
|
2624
|
+
message: 'Cross-backend explainability gap exceeds preferred consistency budget.',
|
|
2625
|
+
observed: [
|
|
2626
|
+
`sampleCount=${queryBackendComparisonSampleCount}`,
|
|
2627
|
+
`evidenceGap=${queryBackendComparisonEvidenceGapRatioPct}%`,
|
|
2628
|
+
`relationGap=${queryBackendComparisonRelationGapRatioPct}%`,
|
|
2629
|
+
`temporalGap=${queryBackendComparisonTemporalGapRatioPct}%`,
|
|
2630
|
+
`maxGap=${queryBackendComparisonMaxExplainabilityGapRatioPct}%`,
|
|
2631
|
+
].join(', '),
|
|
2632
|
+
expected: `maxGap<=${thresholds.queryBackendExplainabilityGapWarnRatioPct}%`,
|
|
2633
|
+
});
|
|
2634
|
+
}
|
|
2635
|
+
else {
|
|
2636
|
+
checks.push({
|
|
2637
|
+
checkId: 'query_backend_explainability_gap',
|
|
2638
|
+
status: 'pass',
|
|
2639
|
+
message: 'Cross-backend explainability gap is within consistency budget.',
|
|
2640
|
+
observed: [
|
|
2641
|
+
`sampleCount=${queryBackendComparisonSampleCount}`,
|
|
2642
|
+
`evidenceGap=${queryBackendComparisonEvidenceGapRatioPct}%`,
|
|
2643
|
+
`relationGap=${queryBackendComparisonRelationGapRatioPct}%`,
|
|
2644
|
+
`temporalGap=${queryBackendComparisonTemporalGapRatioPct}%`,
|
|
2645
|
+
`maxGap=${queryBackendComparisonMaxExplainabilityGapRatioPct}%`,
|
|
2646
|
+
].join(', '),
|
|
2647
|
+
expected: `maxGap<=${thresholds.queryBackendExplainabilityGapWarnRatioPct}%`,
|
|
2648
|
+
});
|
|
2649
|
+
}
|
|
2650
|
+
if (!hasQueryBackendComparisonTrendConfig) {
|
|
2651
|
+
checks.push({
|
|
2652
|
+
checkId: 'query_backend_trend_config',
|
|
2653
|
+
status: 'pass',
|
|
2654
|
+
message: 'Backend comparison trend config is unavailable; config sanity gate deferred.',
|
|
2655
|
+
observed: 'config=missing',
|
|
2656
|
+
expected: 'limit/windowSize/minSamples configured',
|
|
2657
|
+
});
|
|
2658
|
+
}
|
|
2659
|
+
else if (queryBackendComparisonTrendLimit < queryBackendComparisonTrendRequiredRecords) {
|
|
2660
|
+
checks.push({
|
|
2661
|
+
checkId: 'query_backend_trend_config',
|
|
2662
|
+
status: 'fail',
|
|
2663
|
+
message: 'Backend comparison trend config is internally inconsistent for dual-window evaluation.',
|
|
2664
|
+
observed: [
|
|
2665
|
+
`limit=${queryBackendComparisonTrendLimit}`,
|
|
2666
|
+
`windowSize=${queryBackendComparisonTrendWindowSize}`,
|
|
2667
|
+
`minSamples=${queryBackendComparisonTrendMinSamples}`,
|
|
2668
|
+
`requiredRecords=${queryBackendComparisonTrendRequiredRecords}`,
|
|
2669
|
+
].join(', '),
|
|
2670
|
+
expected: `limit>=${queryBackendComparisonTrendRequiredRecords}`,
|
|
2671
|
+
});
|
|
2672
|
+
}
|
|
2673
|
+
else if (queryBackendComparisonSampleCount > 0
|
|
2674
|
+
&& queryBackendComparisonSampleCount < queryBackendComparisonTrendRequiredRecords) {
|
|
2675
|
+
checks.push({
|
|
2676
|
+
checkId: 'query_backend_trend_config',
|
|
2677
|
+
status: 'warn',
|
|
2678
|
+
message: 'Backend comparison trend config needs more history samples to produce stable windows.',
|
|
2679
|
+
observed: [
|
|
2680
|
+
`sampleCount=${queryBackendComparisonSampleCount}`,
|
|
2681
|
+
`limit=${queryBackendComparisonTrendLimit}`,
|
|
2682
|
+
`windowSize=${queryBackendComparisonTrendWindowSize}`,
|
|
2683
|
+
`minSamples=${queryBackendComparisonTrendMinSamples}`,
|
|
2684
|
+
`requiredRecords=${queryBackendComparisonTrendRequiredRecords}`,
|
|
2685
|
+
].join(', '),
|
|
2686
|
+
expected: `sampleCount>=${queryBackendComparisonTrendRequiredRecords}`,
|
|
2687
|
+
});
|
|
2688
|
+
}
|
|
2689
|
+
else {
|
|
2690
|
+
checks.push({
|
|
2691
|
+
checkId: 'query_backend_trend_config',
|
|
2692
|
+
status: 'pass',
|
|
2693
|
+
message: 'Backend comparison trend config is compatible with current sample budget.',
|
|
2694
|
+
observed: [
|
|
2695
|
+
`sampleCount=${queryBackendComparisonSampleCount}`,
|
|
2696
|
+
`limit=${queryBackendComparisonTrendLimit}`,
|
|
2697
|
+
`windowSize=${queryBackendComparisonTrendWindowSize}`,
|
|
2698
|
+
`minSamples=${queryBackendComparisonTrendMinSamples}`,
|
|
2699
|
+
`requiredRecords=${queryBackendComparisonTrendRequiredRecords}`,
|
|
2700
|
+
].join(', '),
|
|
2701
|
+
expected: `limit>=${queryBackendComparisonTrendRequiredRecords}`,
|
|
2702
|
+
});
|
|
2703
|
+
}
|
|
2704
|
+
if (queryBackendComparisonTrendStatus === 'regressing') {
|
|
2705
|
+
if (queryBackendComparisonTrendConfidenceRatioPct
|
|
2706
|
+
>= thresholds.queryBackendTrendFailConfidenceRatioPct) {
|
|
2707
|
+
checks.push({
|
|
2708
|
+
checkId: 'query_backend_comparison_trend',
|
|
2709
|
+
status: 'fail',
|
|
2710
|
+
message: 'Backend comparison trend is regressing with high confidence.',
|
|
2711
|
+
observed: queryBackendComparisonTrendObserved,
|
|
2712
|
+
expected: [
|
|
2713
|
+
`status in {stable, improving}`,
|
|
2714
|
+
`or regressingConfidence<${thresholds.queryBackendTrendFailConfidenceRatioPct}%`,
|
|
2715
|
+
].join(', '),
|
|
2716
|
+
});
|
|
2717
|
+
}
|
|
2718
|
+
else {
|
|
2719
|
+
checks.push({
|
|
2720
|
+
checkId: 'query_backend_comparison_trend',
|
|
2721
|
+
status: 'warn',
|
|
2722
|
+
message: 'Backend comparison trend is regressing and requires monitoring.',
|
|
2723
|
+
observed: queryBackendComparisonTrendObserved,
|
|
2724
|
+
expected: [
|
|
2725
|
+
`status in {stable, improving}`,
|
|
2726
|
+
`or regressingConfidence<${thresholds.queryBackendTrendWarnConfidenceRatioPct}%`,
|
|
2727
|
+
].join(', '),
|
|
2728
|
+
});
|
|
2729
|
+
}
|
|
2730
|
+
}
|
|
2731
|
+
else if (queryBackendComparisonTrendStatus === 'insufficient_data') {
|
|
2732
|
+
checks.push({
|
|
2733
|
+
checkId: 'query_backend_comparison_trend',
|
|
2734
|
+
status: 'warn',
|
|
2735
|
+
message: 'Backend comparison trend has insufficient data.',
|
|
2736
|
+
observed: queryBackendComparisonTrendObserved,
|
|
2737
|
+
expected: 'status in {stable, improving, regressing}',
|
|
2738
|
+
});
|
|
2739
|
+
}
|
|
2740
|
+
else if (queryBackendComparisonTrendStatus === 'stable'
|
|
2741
|
+
|| queryBackendComparisonTrendStatus === 'improving') {
|
|
2742
|
+
checks.push({
|
|
2743
|
+
checkId: 'query_backend_comparison_trend',
|
|
2744
|
+
status: 'pass',
|
|
2745
|
+
message: queryBackendComparisonTrendStatus === 'improving'
|
|
2746
|
+
? 'Backend comparison trend is improving.'
|
|
2747
|
+
: 'Backend comparison trend is stable.',
|
|
2748
|
+
observed: queryBackendComparisonTrendObserved,
|
|
2749
|
+
expected: 'status in {stable, improving}',
|
|
2750
|
+
});
|
|
2751
|
+
}
|
|
2752
|
+
else {
|
|
2753
|
+
if (!hasQueryBackendComparisonTrendSignal) {
|
|
2754
|
+
checks.push({
|
|
2755
|
+
checkId: 'query_backend_comparison_trend',
|
|
2756
|
+
status: 'pass',
|
|
2757
|
+
message: 'Backend comparison trend telemetry is unavailable; trend gate deferred until samples are collected.',
|
|
2758
|
+
observed: queryBackendComparisonTrendObserved || 'status=unknown',
|
|
2759
|
+
expected: 'trend telemetry available',
|
|
2760
|
+
});
|
|
2761
|
+
}
|
|
2762
|
+
else {
|
|
2763
|
+
checks.push({
|
|
2764
|
+
checkId: 'query_backend_comparison_trend',
|
|
2765
|
+
status: 'warn',
|
|
2766
|
+
message: 'Backend comparison trend signal is unavailable.',
|
|
2767
|
+
observed: queryBackendComparisonTrendObserved || 'status=unknown',
|
|
2768
|
+
expected: 'status in {stable, improving, regressing, insufficient_data}',
|
|
2769
|
+
});
|
|
2770
|
+
}
|
|
2771
|
+
}
|
|
2772
|
+
if (hasApiRequestErrorTelemetry) {
|
|
2773
|
+
if (apiTraceWindowRequests <= 0) {
|
|
2774
|
+
checks.push({
|
|
2775
|
+
checkId: 'api_invalid_request_ratio',
|
|
2776
|
+
status: 'warn',
|
|
2777
|
+
message: 'API request trace has no traffic sample yet for invalid-request governance.',
|
|
2778
|
+
observed: apiInvalidRequestObserved,
|
|
2779
|
+
expected: 'requests>=1',
|
|
2780
|
+
});
|
|
2781
|
+
}
|
|
2782
|
+
else if (apiTraceWindowErrors <= 0) {
|
|
2783
|
+
checks.push({
|
|
2784
|
+
checkId: 'api_invalid_request_ratio',
|
|
2785
|
+
status: 'pass',
|
|
2786
|
+
message: 'No API validation errors observed in the current trace window.',
|
|
2787
|
+
observed: apiInvalidRequestObserved,
|
|
2788
|
+
expected: `invalid/error<=${thresholds.apiInvalidRequestWarnRatioPct}%`,
|
|
2789
|
+
});
|
|
2790
|
+
}
|
|
2791
|
+
else if (apiTraceWindowErrors < thresholds.apiInvalidRequestMinErrorSample) {
|
|
2792
|
+
checks.push({
|
|
2793
|
+
checkId: 'api_invalid_request_ratio',
|
|
2794
|
+
status: 'warn',
|
|
2795
|
+
message: 'Insufficient API error sample size for stable invalid-request ratio assessment.',
|
|
2796
|
+
observed: apiInvalidRequestObserved,
|
|
2797
|
+
expected: `errors>=${thresholds.apiInvalidRequestMinErrorSample}`,
|
|
2798
|
+
});
|
|
2799
|
+
}
|
|
2800
|
+
else if (apiTraceWindowInvalidRequestRatioPct > thresholds.apiInvalidRequestFailRatioPct) {
|
|
2801
|
+
checks.push({
|
|
2802
|
+
checkId: 'api_invalid_request_ratio',
|
|
2803
|
+
status: 'fail',
|
|
2804
|
+
message: 'Invalid-request error ratio exceeds hard API robustness ceiling.',
|
|
2805
|
+
observed: apiInvalidRequestObserved,
|
|
2806
|
+
expected: `invalid/error<=${thresholds.apiInvalidRequestFailRatioPct}%`,
|
|
2807
|
+
});
|
|
2808
|
+
}
|
|
2809
|
+
else if (apiTraceWindowInvalidRequestRatioPct > thresholds.apiInvalidRequestWarnRatioPct) {
|
|
2810
|
+
checks.push({
|
|
2811
|
+
checkId: 'api_invalid_request_ratio',
|
|
2812
|
+
status: 'warn',
|
|
2813
|
+
message: 'Invalid-request error ratio exceeds preferred API robustness budget.',
|
|
2814
|
+
observed: apiInvalidRequestObserved,
|
|
2815
|
+
expected: `invalid/error<=${thresholds.apiInvalidRequestWarnRatioPct}%`,
|
|
2816
|
+
});
|
|
2817
|
+
}
|
|
2818
|
+
else {
|
|
2819
|
+
checks.push({
|
|
2820
|
+
checkId: 'api_invalid_request_ratio',
|
|
2821
|
+
status: 'pass',
|
|
2822
|
+
message: 'Invalid-request error ratio is within API robustness budget.',
|
|
2823
|
+
observed: apiInvalidRequestObserved,
|
|
2824
|
+
expected: `invalid/error<=${thresholds.apiInvalidRequestWarnRatioPct}%`,
|
|
2825
|
+
});
|
|
2826
|
+
}
|
|
2827
|
+
if (apiTraceWindowInvalidRequests <= 0) {
|
|
2828
|
+
checks.push({
|
|
2829
|
+
checkId: 'api_invalid_request_hotspots',
|
|
2830
|
+
status: 'pass',
|
|
2831
|
+
message: 'No invalid-request hotspot endpoint detected in the current trace window.',
|
|
2832
|
+
observed: apiInvalidRequestObserved,
|
|
2833
|
+
expected: `topPathInvalidCount<=${thresholds.apiInvalidRequestHotspotWarnCount}`,
|
|
2834
|
+
});
|
|
2835
|
+
}
|
|
2836
|
+
else if (apiTraceWindowInvalidRequestTopPaths.length === 0) {
|
|
2837
|
+
checks.push({
|
|
2838
|
+
checkId: 'api_invalid_request_hotspots',
|
|
2839
|
+
status: 'warn',
|
|
2840
|
+
message: 'Invalid-request errors detected but hotspot endpoint aggregation is unavailable.',
|
|
2841
|
+
observed: apiInvalidRequestObserved,
|
|
2842
|
+
expected: 'invalidTopPaths available',
|
|
2843
|
+
});
|
|
2844
|
+
}
|
|
2845
|
+
else if (apiInvalidRequestHotspotPeakCount >= thresholds.apiInvalidRequestHotspotFailCount) {
|
|
2846
|
+
checks.push({
|
|
2847
|
+
checkId: 'api_invalid_request_hotspots',
|
|
2848
|
+
status: 'fail',
|
|
2849
|
+
message: 'Invalid-request errors are concentrated on a hotspot endpoint.',
|
|
2850
|
+
observed: apiInvalidRequestObserved,
|
|
2851
|
+
expected: `topPathInvalidCount<${thresholds.apiInvalidRequestHotspotFailCount}`,
|
|
2852
|
+
});
|
|
2853
|
+
}
|
|
2854
|
+
else if (apiInvalidRequestHotspotPeakCount >= thresholds.apiInvalidRequestHotspotWarnCount) {
|
|
2855
|
+
checks.push({
|
|
2856
|
+
checkId: 'api_invalid_request_hotspots',
|
|
2857
|
+
status: 'warn',
|
|
2858
|
+
message: 'Invalid-request hotspot endpoint is above preferred concentration budget.',
|
|
2859
|
+
observed: apiInvalidRequestObserved,
|
|
2860
|
+
expected: `topPathInvalidCount<${thresholds.apiInvalidRequestHotspotWarnCount}`,
|
|
2861
|
+
});
|
|
2862
|
+
}
|
|
2863
|
+
else {
|
|
2864
|
+
checks.push({
|
|
2865
|
+
checkId: 'api_invalid_request_hotspots',
|
|
2866
|
+
status: 'pass',
|
|
2867
|
+
message: 'Invalid-request hotspot concentration is within budget.',
|
|
2868
|
+
observed: apiInvalidRequestObserved,
|
|
2869
|
+
expected: `topPathInvalidCount<${thresholds.apiInvalidRequestHotspotWarnCount}`,
|
|
2870
|
+
});
|
|
2871
|
+
}
|
|
2872
|
+
if (apiTraceWindowRequests <= 0) {
|
|
2873
|
+
checks.push({
|
|
2874
|
+
checkId: 'api_server_error_ratio',
|
|
2875
|
+
status: 'warn',
|
|
2876
|
+
message: 'API request trace has no traffic sample yet for server-error governance.',
|
|
2877
|
+
observed: apiServerErrorObserved,
|
|
2878
|
+
expected: 'requests>=1',
|
|
2879
|
+
});
|
|
2880
|
+
}
|
|
2881
|
+
else if (apiTraceWindowRequests < thresholds.apiServerErrorMinRequestSample) {
|
|
2882
|
+
checks.push({
|
|
2883
|
+
checkId: 'api_server_error_ratio',
|
|
2884
|
+
status: 'warn',
|
|
2885
|
+
message: 'Insufficient API traffic sample size for stable server-error ratio assessment.',
|
|
2886
|
+
observed: apiServerErrorObserved,
|
|
2887
|
+
expected: `requests>=${thresholds.apiServerErrorMinRequestSample}`,
|
|
2888
|
+
});
|
|
2889
|
+
}
|
|
2890
|
+
else if (apiTraceWindowServerErrorRatioPct > thresholds.apiServerErrorFailRatioPct) {
|
|
2891
|
+
checks.push({
|
|
2892
|
+
checkId: 'api_server_error_ratio',
|
|
2893
|
+
status: 'fail',
|
|
2894
|
+
message: 'Server-error ratio exceeds hard API reliability ceiling.',
|
|
2895
|
+
observed: apiServerErrorObserved,
|
|
2896
|
+
expected: `server/total<=${thresholds.apiServerErrorFailRatioPct}%`,
|
|
2897
|
+
});
|
|
2898
|
+
}
|
|
2899
|
+
else if (apiTraceWindowServerErrorRatioPct > thresholds.apiServerErrorWarnRatioPct) {
|
|
2900
|
+
checks.push({
|
|
2901
|
+
checkId: 'api_server_error_ratio',
|
|
2902
|
+
status: 'warn',
|
|
2903
|
+
message: 'Server-error ratio exceeds preferred API reliability budget.',
|
|
2904
|
+
observed: apiServerErrorObserved,
|
|
2905
|
+
expected: `server/total<=${thresholds.apiServerErrorWarnRatioPct}%`,
|
|
2906
|
+
});
|
|
2907
|
+
}
|
|
2908
|
+
else {
|
|
2909
|
+
checks.push({
|
|
2910
|
+
checkId: 'api_server_error_ratio',
|
|
2911
|
+
status: 'pass',
|
|
2912
|
+
message: 'Server-error ratio is within API reliability budget.',
|
|
2913
|
+
observed: apiServerErrorObserved,
|
|
2914
|
+
expected: `server/total<=${thresholds.apiServerErrorWarnRatioPct}%`,
|
|
2915
|
+
});
|
|
2916
|
+
}
|
|
2917
|
+
if (apiTraceWindowServerErrors <= 0) {
|
|
2918
|
+
checks.push({
|
|
2919
|
+
checkId: 'api_server_error_hotspots',
|
|
2920
|
+
status: 'pass',
|
|
2921
|
+
message: 'No server-error hotspot endpoint detected in the current trace window.',
|
|
2922
|
+
observed: apiServerErrorObserved,
|
|
2923
|
+
expected: `topPathServerErrorCount<${thresholds.apiServerErrorHotspotWarnCount}`,
|
|
2924
|
+
});
|
|
2925
|
+
}
|
|
2926
|
+
else if (apiTraceWindowServerErrorTopPaths.length === 0) {
|
|
2927
|
+
checks.push({
|
|
2928
|
+
checkId: 'api_server_error_hotspots',
|
|
2929
|
+
status: 'warn',
|
|
2930
|
+
message: 'Server errors detected but hotspot endpoint aggregation is unavailable.',
|
|
2931
|
+
observed: apiServerErrorObserved,
|
|
2932
|
+
expected: 'serverTopPaths available',
|
|
2933
|
+
});
|
|
2934
|
+
}
|
|
2935
|
+
else if (apiServerErrorHotspotPeakCount >= thresholds.apiServerErrorHotspotFailCount) {
|
|
2936
|
+
checks.push({
|
|
2937
|
+
checkId: 'api_server_error_hotspots',
|
|
2938
|
+
status: 'fail',
|
|
2939
|
+
message: 'Server errors are concentrated on a hotspot endpoint.',
|
|
2940
|
+
observed: apiServerErrorObserved,
|
|
2941
|
+
expected: `topPathServerErrorCount<${thresholds.apiServerErrorHotspotFailCount}`,
|
|
2942
|
+
});
|
|
2943
|
+
}
|
|
2944
|
+
else if (apiServerErrorHotspotPeakCount >= thresholds.apiServerErrorHotspotWarnCount) {
|
|
2945
|
+
checks.push({
|
|
2946
|
+
checkId: 'api_server_error_hotspots',
|
|
2947
|
+
status: 'warn',
|
|
2948
|
+
message: 'Server-error hotspot endpoint is above preferred concentration budget.',
|
|
2949
|
+
observed: apiServerErrorObserved,
|
|
2950
|
+
expected: `topPathServerErrorCount<${thresholds.apiServerErrorHotspotWarnCount}`,
|
|
2951
|
+
});
|
|
2952
|
+
}
|
|
2953
|
+
else {
|
|
2954
|
+
checks.push({
|
|
2955
|
+
checkId: 'api_server_error_hotspots',
|
|
2956
|
+
status: 'pass',
|
|
2957
|
+
message: 'Server-error hotspot concentration is within budget.',
|
|
2958
|
+
observed: apiServerErrorObserved,
|
|
2959
|
+
expected: `topPathServerErrorCount<${thresholds.apiServerErrorHotspotWarnCount}`,
|
|
2960
|
+
});
|
|
2961
|
+
}
|
|
2962
|
+
if (apiTraceWindowRequests <= 0) {
|
|
2963
|
+
checks.push({
|
|
2964
|
+
checkId: 'api_transient_error_ratio',
|
|
2965
|
+
status: 'warn',
|
|
2966
|
+
message: 'API request trace has no traffic sample yet for transient-error governance.',
|
|
2967
|
+
observed: apiTransientErrorObserved,
|
|
2968
|
+
expected: 'requests>=1',
|
|
2969
|
+
});
|
|
2970
|
+
}
|
|
2971
|
+
else if (apiTraceWindowRequests < thresholds.apiTransientErrorMinRequestSample) {
|
|
2972
|
+
checks.push({
|
|
2973
|
+
checkId: 'api_transient_error_ratio',
|
|
2974
|
+
status: 'warn',
|
|
2975
|
+
message: 'Insufficient API traffic sample size for stable transient-error ratio assessment.',
|
|
2976
|
+
observed: apiTransientErrorObserved,
|
|
2977
|
+
expected: `requests>=${thresholds.apiTransientErrorMinRequestSample}`,
|
|
2978
|
+
});
|
|
2979
|
+
}
|
|
2980
|
+
else if (apiTraceWindowTransientErrorRatioPct > thresholds.apiTransientErrorFailRatioPct) {
|
|
2981
|
+
checks.push({
|
|
2982
|
+
checkId: 'api_transient_error_ratio',
|
|
2983
|
+
status: 'fail',
|
|
2984
|
+
message: 'Transient-error ratio exceeds hard API stability ceiling.',
|
|
2985
|
+
observed: apiTransientErrorObserved,
|
|
2986
|
+
expected: `transient/total<=${thresholds.apiTransientErrorFailRatioPct}%`,
|
|
2987
|
+
});
|
|
2988
|
+
}
|
|
2989
|
+
else if (apiTraceWindowTransientErrorRatioPct > thresholds.apiTransientErrorWarnRatioPct) {
|
|
2990
|
+
checks.push({
|
|
2991
|
+
checkId: 'api_transient_error_ratio',
|
|
2992
|
+
status: 'warn',
|
|
2993
|
+
message: 'Transient-error ratio exceeds preferred API stability budget.',
|
|
2994
|
+
observed: apiTransientErrorObserved,
|
|
2995
|
+
expected: `transient/total<=${thresholds.apiTransientErrorWarnRatioPct}%`,
|
|
2996
|
+
});
|
|
2997
|
+
}
|
|
2998
|
+
else {
|
|
2999
|
+
checks.push({
|
|
3000
|
+
checkId: 'api_transient_error_ratio',
|
|
3001
|
+
status: 'pass',
|
|
3002
|
+
message: 'Transient-error ratio is within API stability budget.',
|
|
3003
|
+
observed: apiTransientErrorObserved,
|
|
3004
|
+
expected: `transient/total<=${thresholds.apiTransientErrorWarnRatioPct}%`,
|
|
3005
|
+
});
|
|
3006
|
+
}
|
|
3007
|
+
if (apiTraceWindowTransientErrors <= 0) {
|
|
3008
|
+
checks.push({
|
|
3009
|
+
checkId: 'api_transient_error_hotspots',
|
|
3010
|
+
status: 'pass',
|
|
3011
|
+
message: 'No transient-error hotspot endpoint detected in the current trace window.',
|
|
3012
|
+
observed: apiTransientErrorObserved,
|
|
3013
|
+
expected: `topPathTransientErrorCount<${thresholds.apiTransientErrorHotspotWarnCount}`,
|
|
3014
|
+
});
|
|
3015
|
+
}
|
|
3016
|
+
else if (apiTraceWindowTransientErrorTopPaths.length === 0) {
|
|
3017
|
+
checks.push({
|
|
3018
|
+
checkId: 'api_transient_error_hotspots',
|
|
3019
|
+
status: 'warn',
|
|
3020
|
+
message: 'Transient errors detected but hotspot endpoint aggregation is unavailable.',
|
|
3021
|
+
observed: apiTransientErrorObserved,
|
|
3022
|
+
expected: 'transientTopPaths available',
|
|
3023
|
+
});
|
|
3024
|
+
}
|
|
3025
|
+
else if (apiTransientErrorHotspotPeakCount >= thresholds.apiTransientErrorHotspotFailCount) {
|
|
3026
|
+
checks.push({
|
|
3027
|
+
checkId: 'api_transient_error_hotspots',
|
|
3028
|
+
status: 'fail',
|
|
3029
|
+
message: 'Transient errors are concentrated on a hotspot endpoint.',
|
|
3030
|
+
observed: apiTransientErrorObserved,
|
|
3031
|
+
expected: `topPathTransientErrorCount<${thresholds.apiTransientErrorHotspotFailCount}`,
|
|
3032
|
+
});
|
|
3033
|
+
}
|
|
3034
|
+
else if (apiTransientErrorHotspotPeakCount >= thresholds.apiTransientErrorHotspotWarnCount) {
|
|
3035
|
+
checks.push({
|
|
3036
|
+
checkId: 'api_transient_error_hotspots',
|
|
3037
|
+
status: 'warn',
|
|
3038
|
+
message: 'Transient-error hotspot endpoint is above preferred concentration budget.',
|
|
3039
|
+
observed: apiTransientErrorObserved,
|
|
3040
|
+
expected: `topPathTransientErrorCount<${thresholds.apiTransientErrorHotspotWarnCount}`,
|
|
3041
|
+
});
|
|
3042
|
+
}
|
|
3043
|
+
else {
|
|
3044
|
+
checks.push({
|
|
3045
|
+
checkId: 'api_transient_error_hotspots',
|
|
3046
|
+
status: 'pass',
|
|
3047
|
+
message: 'Transient-error hotspot concentration is within budget.',
|
|
3048
|
+
observed: apiTransientErrorObserved,
|
|
3049
|
+
expected: `topPathTransientErrorCount<${thresholds.apiTransientErrorHotspotWarnCount}`,
|
|
3050
|
+
});
|
|
3051
|
+
}
|
|
3052
|
+
if (apiTraceWindowRequests <= 0) {
|
|
3053
|
+
checks.push({
|
|
3054
|
+
checkId: 'api_latency_p95',
|
|
3055
|
+
status: 'warn',
|
|
3056
|
+
message: 'API request trace has no traffic sample yet for latency governance.',
|
|
3057
|
+
observed: apiLatencyObserved,
|
|
3058
|
+
expected: 'requests>=1',
|
|
3059
|
+
});
|
|
3060
|
+
}
|
|
3061
|
+
else if (apiTraceWindowRequests < thresholds.apiLatencyMinRequestSample) {
|
|
3062
|
+
checks.push({
|
|
3063
|
+
checkId: 'api_latency_p95',
|
|
3064
|
+
status: 'warn',
|
|
3065
|
+
message: 'Insufficient API traffic sample size for stable p95 latency assessment.',
|
|
3066
|
+
observed: apiLatencyObserved,
|
|
3067
|
+
expected: `requests>=${thresholds.apiLatencyMinRequestSample}`,
|
|
3068
|
+
});
|
|
3069
|
+
}
|
|
3070
|
+
else if (apiTraceP95DurationMs > thresholds.apiLatencyP95FailMs) {
|
|
3071
|
+
checks.push({
|
|
3072
|
+
checkId: 'api_latency_p95',
|
|
3073
|
+
status: 'fail',
|
|
3074
|
+
message: 'API p95 latency exceeds hard responsiveness ceiling.',
|
|
3075
|
+
observed: apiLatencyObserved,
|
|
3076
|
+
expected: `p95<=${thresholds.apiLatencyP95FailMs}ms`,
|
|
3077
|
+
});
|
|
3078
|
+
}
|
|
3079
|
+
else if (apiTraceP95DurationMs > thresholds.apiLatencyP95WarnMs) {
|
|
3080
|
+
checks.push({
|
|
3081
|
+
checkId: 'api_latency_p95',
|
|
3082
|
+
status: 'warn',
|
|
3083
|
+
message: 'API p95 latency exceeds preferred responsiveness budget.',
|
|
3084
|
+
observed: apiLatencyObserved,
|
|
3085
|
+
expected: `p95<=${thresholds.apiLatencyP95WarnMs}ms`,
|
|
3086
|
+
});
|
|
3087
|
+
}
|
|
3088
|
+
else {
|
|
3089
|
+
checks.push({
|
|
3090
|
+
checkId: 'api_latency_p95',
|
|
3091
|
+
status: 'pass',
|
|
3092
|
+
message: 'API p95 latency is within responsiveness budget.',
|
|
3093
|
+
observed: apiLatencyObserved,
|
|
3094
|
+
expected: `p95<=${thresholds.apiLatencyP95WarnMs}ms`,
|
|
3095
|
+
});
|
|
3096
|
+
}
|
|
3097
|
+
if (apiTraceSlowTopPaths.length <= 0) {
|
|
3098
|
+
checks.push({
|
|
3099
|
+
checkId: 'api_latency_hotspots',
|
|
3100
|
+
status: 'pass',
|
|
3101
|
+
message: 'No latency hotspot endpoint detected in the current trace window.',
|
|
3102
|
+
observed: apiLatencyObserved,
|
|
3103
|
+
expected: `topPathP95<${thresholds.apiLatencyHotspotWarnMs}ms`,
|
|
3104
|
+
});
|
|
3105
|
+
}
|
|
3106
|
+
else if (apiLatencyHotspotPeakP95Ms > thresholds.apiLatencyHotspotFailMs) {
|
|
3107
|
+
checks.push({
|
|
3108
|
+
checkId: 'api_latency_hotspots',
|
|
3109
|
+
status: 'fail',
|
|
3110
|
+
message: 'API latency is concentrated on a severe hotspot endpoint.',
|
|
3111
|
+
observed: apiLatencyObserved,
|
|
3112
|
+
expected: `topPathP95<=${thresholds.apiLatencyHotspotFailMs}ms`,
|
|
3113
|
+
});
|
|
3114
|
+
}
|
|
3115
|
+
else if (apiLatencyHotspotPeakP95Ms > thresholds.apiLatencyHotspotWarnMs) {
|
|
3116
|
+
checks.push({
|
|
3117
|
+
checkId: 'api_latency_hotspots',
|
|
3118
|
+
status: 'warn',
|
|
3119
|
+
message: 'API latency hotspot endpoint is above preferred budget.',
|
|
3120
|
+
observed: apiLatencyObserved,
|
|
3121
|
+
expected: `topPathP95<=${thresholds.apiLatencyHotspotWarnMs}ms`,
|
|
3122
|
+
});
|
|
3123
|
+
}
|
|
3124
|
+
else {
|
|
3125
|
+
checks.push({
|
|
3126
|
+
checkId: 'api_latency_hotspots',
|
|
3127
|
+
status: 'pass',
|
|
3128
|
+
message: 'API latency hotspot profile is within budget.',
|
|
3129
|
+
observed: apiLatencyObserved,
|
|
3130
|
+
expected: `topPathP95<=${thresholds.apiLatencyHotspotWarnMs}ms`,
|
|
3131
|
+
});
|
|
3132
|
+
}
|
|
3133
|
+
}
|
|
3134
|
+
if (trendStatus === 'regressing') {
|
|
3135
|
+
checks.push({
|
|
3136
|
+
checkId: 'quality_trend_direction',
|
|
3137
|
+
status: 'fail',
|
|
3138
|
+
message: 'Learning quality trend is regressing and requires intervention.',
|
|
3139
|
+
observed: trendObserved,
|
|
3140
|
+
expected: 'status in {stable, improving}',
|
|
3141
|
+
});
|
|
3142
|
+
}
|
|
3143
|
+
else if (trendStatus === 'insufficient_data') {
|
|
3144
|
+
checks.push({
|
|
3145
|
+
checkId: 'quality_trend_direction',
|
|
3146
|
+
status: 'warn',
|
|
3147
|
+
message: 'Learning quality trend has insufficient data for stable governance decisions.',
|
|
3148
|
+
observed: trendObserved,
|
|
3149
|
+
expected: 'status in {stable, improving}',
|
|
3150
|
+
});
|
|
3151
|
+
}
|
|
3152
|
+
else if (trendStatus === 'improving' || trendStatus === 'stable') {
|
|
3153
|
+
checks.push({
|
|
3154
|
+
checkId: 'quality_trend_direction',
|
|
3155
|
+
status: 'pass',
|
|
3156
|
+
message: trendStatus === 'improving'
|
|
3157
|
+
? 'Learning quality trend is improving.'
|
|
3158
|
+
: 'Learning quality trend is stable.',
|
|
3159
|
+
observed: trendObserved,
|
|
3160
|
+
expected: 'status in {stable, improving}',
|
|
3161
|
+
});
|
|
3162
|
+
}
|
|
3163
|
+
else {
|
|
3164
|
+
checks.push({
|
|
3165
|
+
checkId: 'quality_trend_direction',
|
|
3166
|
+
status: 'warn',
|
|
3167
|
+
message: 'Learning quality trend signal is unavailable.',
|
|
3168
|
+
observed: trendObserved || 'status=unknown',
|
|
3169
|
+
expected: 'status in {stable, improving, regressing, insufficient_data}',
|
|
3170
|
+
});
|
|
3171
|
+
}
|
|
3172
|
+
if (sessionPlanQualityRecords <= 0) {
|
|
3173
|
+
checks.push({
|
|
3174
|
+
checkId: 'session_plan_quality_gate',
|
|
3175
|
+
status: 'warn',
|
|
3176
|
+
message: 'Session plan quality gate history is unavailable.',
|
|
3177
|
+
observed: 'records=0',
|
|
3178
|
+
expected: 'records>=1',
|
|
3179
|
+
});
|
|
3180
|
+
}
|
|
3181
|
+
else if (sessionPlanQualityFailureStreak >= thresholds.sessionPlanQualityFailFailureStreak) {
|
|
3182
|
+
checks.push({
|
|
3183
|
+
checkId: 'session_plan_quality_gate',
|
|
3184
|
+
status: 'fail',
|
|
3185
|
+
message: 'Session plan quality gate is repeatedly failing.',
|
|
3186
|
+
observed: [
|
|
3187
|
+
`records=${sessionPlanQualityRecords}`,
|
|
3188
|
+
`passRate=${sessionPlanQualityPassRatePct}%`,
|
|
3189
|
+
`failureStreak=${sessionPlanQualityFailureStreak}`,
|
|
3190
|
+
sessionPlanQualityCommonFailedGates
|
|
3191
|
+
? `failedGates=${sessionPlanQualityCommonFailedGates}`
|
|
3192
|
+
: '',
|
|
3193
|
+
].filter(Boolean).join(', '),
|
|
3194
|
+
expected: `failureStreak<${thresholds.sessionPlanQualityFailFailureStreak}`,
|
|
3195
|
+
});
|
|
3196
|
+
}
|
|
3197
|
+
else if (sessionPlanQualityFailureStreak >= thresholds.sessionPlanQualityWarnFailureStreak) {
|
|
3198
|
+
checks.push({
|
|
3199
|
+
checkId: 'session_plan_quality_gate',
|
|
3200
|
+
status: 'warn',
|
|
3201
|
+
message: 'Session plan quality gate recently failed and requires monitoring.',
|
|
3202
|
+
observed: [
|
|
3203
|
+
`records=${sessionPlanQualityRecords}`,
|
|
3204
|
+
`passRate=${sessionPlanQualityPassRatePct}%`,
|
|
3205
|
+
`failureStreak=${sessionPlanQualityFailureStreak}`,
|
|
3206
|
+
sessionPlanQualityCommonFailedGates
|
|
3207
|
+
? `failedGates=${sessionPlanQualityCommonFailedGates}`
|
|
3208
|
+
: '',
|
|
3209
|
+
].filter(Boolean).join(', '),
|
|
3210
|
+
expected: `failureStreak<${thresholds.sessionPlanQualityWarnFailureStreak}`,
|
|
3211
|
+
});
|
|
3212
|
+
}
|
|
3213
|
+
else {
|
|
3214
|
+
checks.push({
|
|
3215
|
+
checkId: 'session_plan_quality_gate',
|
|
3216
|
+
status: 'pass',
|
|
3217
|
+
message: 'Session plan quality gate is stable.',
|
|
3218
|
+
observed: [
|
|
3219
|
+
`records=${sessionPlanQualityRecords}`,
|
|
3220
|
+
`passRate=${sessionPlanQualityPassRatePct}%`,
|
|
3221
|
+
`failureStreak=${sessionPlanQualityFailureStreak}`,
|
|
3222
|
+
].join(', '),
|
|
3223
|
+
expected: `failureStreak<${thresholds.sessionPlanQualityWarnFailureStreak}`,
|
|
3224
|
+
});
|
|
3225
|
+
}
|
|
3226
|
+
if (sessionPlanTrendStatus === 'regressing') {
|
|
3227
|
+
checks.push({
|
|
3228
|
+
checkId: 'session_plan_quality_trend',
|
|
3229
|
+
status: 'fail',
|
|
3230
|
+
message: 'Session plan quality trend is regressing.',
|
|
3231
|
+
observed: sessionPlanTrendObserved,
|
|
3232
|
+
expected: 'status in {stable, improving}',
|
|
3233
|
+
});
|
|
3234
|
+
}
|
|
3235
|
+
else if (sessionPlanTrendStatus === 'insufficient_data') {
|
|
3236
|
+
checks.push({
|
|
3237
|
+
checkId: 'session_plan_quality_trend',
|
|
3238
|
+
status: 'warn',
|
|
3239
|
+
message: 'Session plan quality trend has insufficient data.',
|
|
3240
|
+
observed: sessionPlanTrendObserved,
|
|
3241
|
+
expected: 'status in {stable, improving}',
|
|
3242
|
+
});
|
|
3243
|
+
}
|
|
3244
|
+
else if (sessionPlanTrendStatus === 'improving' || sessionPlanTrendStatus === 'stable') {
|
|
3245
|
+
checks.push({
|
|
3246
|
+
checkId: 'session_plan_quality_trend',
|
|
3247
|
+
status: 'pass',
|
|
3248
|
+
message: sessionPlanTrendStatus === 'improving'
|
|
3249
|
+
? 'Session plan quality trend is improving.'
|
|
3250
|
+
: 'Session plan quality trend is stable.',
|
|
3251
|
+
observed: sessionPlanTrendObserved,
|
|
3252
|
+
expected: 'status in {stable, improving}',
|
|
3253
|
+
});
|
|
3254
|
+
}
|
|
3255
|
+
else {
|
|
3256
|
+
checks.push({
|
|
3257
|
+
checkId: 'session_plan_quality_trend',
|
|
3258
|
+
status: 'warn',
|
|
3259
|
+
message: 'Session plan quality trend signal is unavailable.',
|
|
3260
|
+
observed: sessionPlanTrendObserved || 'status=unknown',
|
|
3261
|
+
expected: 'status in {stable, improving, regressing, insufficient_data}',
|
|
3262
|
+
});
|
|
3263
|
+
}
|
|
3264
|
+
if (sessionStrategyTotalRecords <= 0) {
|
|
3265
|
+
checks.push({
|
|
3266
|
+
checkId: 'orchestration_path_strategy_alignment',
|
|
3267
|
+
status: 'pass',
|
|
3268
|
+
message: 'No session strategy executions recorded yet; alignment check is waiting for telemetry.',
|
|
3269
|
+
observed: sessionStrategyObserved,
|
|
3270
|
+
expected: 'strategyRecords>=1 after first strategy-enabled execution',
|
|
3271
|
+
});
|
|
3272
|
+
}
|
|
3273
|
+
else if (sessionStrategyStrategyRecords <= 0) {
|
|
3274
|
+
checks.push({
|
|
3275
|
+
checkId: 'orchestration_path_strategy_alignment',
|
|
3276
|
+
status: 'warn',
|
|
3277
|
+
message: 'Path strategy outcome telemetry is unavailable for orchestration alignment checks.',
|
|
3278
|
+
observed: sessionStrategyObserved,
|
|
3279
|
+
expected: 'strategyRecords>=1',
|
|
3280
|
+
});
|
|
3281
|
+
}
|
|
3282
|
+
else if (sessionStrategySelectionSourceTrendCount <= 0) {
|
|
3283
|
+
checks.push({
|
|
3284
|
+
checkId: 'orchestration_path_strategy_alignment',
|
|
3285
|
+
status: 'warn',
|
|
3286
|
+
message: 'No trend-driven path strategy selections recorded yet.',
|
|
3287
|
+
observed: sessionStrategyObserved,
|
|
3288
|
+
expected: 'strategy_trend selections>=1',
|
|
3289
|
+
});
|
|
3290
|
+
}
|
|
3291
|
+
else if (sessionStrategySelectionSourceTrendCount < 3) {
|
|
3292
|
+
checks.push({
|
|
3293
|
+
checkId: 'orchestration_path_strategy_alignment',
|
|
3294
|
+
status: 'warn',
|
|
3295
|
+
message: 'Trend-driven path strategy sample size is too small for stable alignment decisions.',
|
|
3296
|
+
observed: sessionStrategyObserved,
|
|
3297
|
+
expected: 'strategy_trend selections>=3',
|
|
3298
|
+
});
|
|
3299
|
+
}
|
|
3300
|
+
else if (sessionStrategyTrendAutoAverageMasteryDeltaPct < -2
|
|
3301
|
+
|| sessionStrategyTrendAutoNegativeRatioPct >= 60) {
|
|
3302
|
+
checks.push({
|
|
3303
|
+
checkId: 'orchestration_path_strategy_alignment',
|
|
3304
|
+
status: 'fail',
|
|
3305
|
+
message: 'Trend-driven path strategy selections are regressing mastery outcomes.',
|
|
3306
|
+
observed: sessionStrategyObserved,
|
|
3307
|
+
expected: 'trendAutoAvgDelta>=0% and trendAutoNegative<40%',
|
|
3308
|
+
});
|
|
3309
|
+
}
|
|
3310
|
+
else if (sessionStrategyTrendAutoAverageMasteryDeltaPct < 0
|
|
3311
|
+
|| sessionStrategyTrendAutoNegativeRatioPct >= 40) {
|
|
3312
|
+
checks.push({
|
|
3313
|
+
checkId: 'orchestration_path_strategy_alignment',
|
|
3314
|
+
status: 'warn',
|
|
3315
|
+
message: 'Trend-driven path strategy selections show weak alignment and require monitoring.',
|
|
3316
|
+
observed: sessionStrategyObserved,
|
|
3317
|
+
expected: 'trendAutoAvgDelta>=0% and trendAutoNegative<40%',
|
|
3318
|
+
});
|
|
3319
|
+
}
|
|
3320
|
+
else {
|
|
3321
|
+
checks.push({
|
|
3322
|
+
checkId: 'orchestration_path_strategy_alignment',
|
|
3323
|
+
status: 'pass',
|
|
3324
|
+
message: 'Trend-driven path strategy selections are aligned with mastery improvements.',
|
|
3325
|
+
observed: sessionStrategyObserved,
|
|
3326
|
+
expected: 'trendAutoAvgDelta>=0% and trendAutoNegative<40%',
|
|
3327
|
+
});
|
|
3328
|
+
}
|
|
3329
|
+
if (memoryPolicyStatus === 'risk') {
|
|
3330
|
+
checks.push({
|
|
3331
|
+
checkId: 'memory_policy_health',
|
|
3332
|
+
status: 'fail',
|
|
3333
|
+
message: 'Memory policy health is at risk and requires immediate cleanup/retraining.',
|
|
3334
|
+
observed: memoryPolicyObserved,
|
|
3335
|
+
expected: 'status in {healthy, watch}',
|
|
3336
|
+
});
|
|
3337
|
+
}
|
|
3338
|
+
else if (memoryPolicyStatus === 'watch') {
|
|
3339
|
+
checks.push({
|
|
3340
|
+
checkId: 'memory_policy_health',
|
|
3341
|
+
status: 'warn',
|
|
3342
|
+
message: 'Memory policy health is degraded and should be monitored.',
|
|
3343
|
+
observed: memoryPolicyObserved,
|
|
3344
|
+
expected: 'status=healthy',
|
|
3345
|
+
});
|
|
3346
|
+
}
|
|
3347
|
+
else if (memoryPolicyStatus === 'healthy') {
|
|
3348
|
+
checks.push({
|
|
3349
|
+
checkId: 'memory_policy_health',
|
|
3350
|
+
status: 'pass',
|
|
3351
|
+
message: 'Memory policy health is stable.',
|
|
3352
|
+
observed: memoryPolicyObserved,
|
|
3353
|
+
expected: 'status=healthy',
|
|
3354
|
+
});
|
|
3355
|
+
}
|
|
3356
|
+
else if (memoryPolicyStatus === 'insufficient_data') {
|
|
3357
|
+
checks.push({
|
|
3358
|
+
checkId: 'memory_policy_health',
|
|
3359
|
+
status: 'warn',
|
|
3360
|
+
message: 'Memory policy diagnostics have insufficient data.',
|
|
3361
|
+
observed: memoryPolicyObserved,
|
|
3362
|
+
expected: 'status in {healthy, watch, risk}',
|
|
3363
|
+
});
|
|
3364
|
+
}
|
|
3365
|
+
else {
|
|
3366
|
+
checks.push({
|
|
3367
|
+
checkId: 'memory_policy_health',
|
|
3368
|
+
status: 'warn',
|
|
3369
|
+
message: 'Memory policy diagnostics are unavailable.',
|
|
3370
|
+
observed: memoryPolicyObserved || 'status=unknown',
|
|
3371
|
+
expected: 'status in {healthy, watch, risk, insufficient_data}',
|
|
3372
|
+
});
|
|
3373
|
+
}
|
|
3374
|
+
if (memoryPolicyTrendStatus === 'regressing') {
|
|
3375
|
+
checks.push({
|
|
3376
|
+
checkId: 'memory_policy_trend',
|
|
3377
|
+
status: 'fail',
|
|
3378
|
+
message: 'Memory policy trend is regressing.',
|
|
3379
|
+
observed: memoryPolicyTrendObserved,
|
|
3380
|
+
expected: 'status in {stable, improving}',
|
|
3381
|
+
});
|
|
3382
|
+
}
|
|
3383
|
+
else if (memoryPolicyTrendStatus === 'insufficient_data') {
|
|
3384
|
+
checks.push({
|
|
3385
|
+
checkId: 'memory_policy_trend',
|
|
3386
|
+
status: 'warn',
|
|
3387
|
+
message: 'Memory policy trend has insufficient data.',
|
|
3388
|
+
observed: memoryPolicyTrendObserved,
|
|
3389
|
+
expected: 'status in {stable, improving}',
|
|
3390
|
+
});
|
|
3391
|
+
}
|
|
3392
|
+
else if (memoryPolicyTrendStatus === 'improving' || memoryPolicyTrendStatus === 'stable') {
|
|
3393
|
+
checks.push({
|
|
3394
|
+
checkId: 'memory_policy_trend',
|
|
3395
|
+
status: 'pass',
|
|
3396
|
+
message: memoryPolicyTrendStatus === 'improving'
|
|
3397
|
+
? 'Memory policy trend is improving.'
|
|
3398
|
+
: 'Memory policy trend is stable.',
|
|
3399
|
+
observed: memoryPolicyTrendObserved,
|
|
3400
|
+
expected: 'status in {stable, improving}',
|
|
3401
|
+
});
|
|
3402
|
+
}
|
|
3403
|
+
else {
|
|
3404
|
+
checks.push({
|
|
3405
|
+
checkId: 'memory_policy_trend',
|
|
3406
|
+
status: 'warn',
|
|
3407
|
+
message: 'Memory policy trend signal is unavailable.',
|
|
3408
|
+
observed: memoryPolicyTrendObserved || 'status=unknown',
|
|
3409
|
+
expected: 'status in {stable, improving, regressing, insufficient_data}',
|
|
3410
|
+
});
|
|
3411
|
+
}
|
|
3412
|
+
if (params.knowledgeStalenessDiagnostics) {
|
|
3413
|
+
if (knowledgeStalenessEvaluatedDocuments <= 0) {
|
|
3414
|
+
checks.push({
|
|
3415
|
+
checkId: 'knowledge_staleness_data',
|
|
3416
|
+
status: 'warn',
|
|
3417
|
+
message: 'Knowledge staleness diagnostics are enabled but no documents were evaluated.',
|
|
3418
|
+
observed: knowledgeStalenessObserved,
|
|
3419
|
+
expected: 'evaluated>=1',
|
|
3420
|
+
});
|
|
3421
|
+
}
|
|
3422
|
+
else {
|
|
3423
|
+
checks.push({
|
|
3424
|
+
checkId: 'knowledge_staleness_data',
|
|
3425
|
+
status: 'pass',
|
|
3426
|
+
message: 'Knowledge staleness diagnostics are available.',
|
|
3427
|
+
observed: knowledgeStalenessObserved,
|
|
3428
|
+
expected: 'evaluated>=1',
|
|
3429
|
+
});
|
|
3430
|
+
}
|
|
3431
|
+
const severeStaleThreshold = Math.max(3, Math.ceil(knowledgeStalenessEvaluatedDocuments * 0.4));
|
|
3432
|
+
if (knowledgeStalenessReadErrorDocuments > 0) {
|
|
3433
|
+
checks.push({
|
|
3434
|
+
checkId: 'knowledge_staleness_health',
|
|
3435
|
+
status: 'fail',
|
|
3436
|
+
message: 'Knowledge staleness health is blocked by source read errors.',
|
|
3437
|
+
observed: knowledgeStalenessObserved,
|
|
3438
|
+
expected: 'readError=0 and stale=0',
|
|
3439
|
+
});
|
|
3440
|
+
}
|
|
3441
|
+
else if (knowledgeStalenessStaleDocuments >= severeStaleThreshold) {
|
|
3442
|
+
checks.push({
|
|
3443
|
+
checkId: 'knowledge_staleness_health',
|
|
3444
|
+
status: 'fail',
|
|
3445
|
+
message: 'Knowledge staleness ratio exceeds hard governance threshold.',
|
|
3446
|
+
observed: knowledgeStalenessObserved,
|
|
3447
|
+
expected: `stale<${severeStaleThreshold}`,
|
|
3448
|
+
});
|
|
3449
|
+
}
|
|
3450
|
+
else if (knowledgeStalenessStaleDocuments > 0
|
|
3451
|
+
|| knowledgeStalenessHashMismatchDocuments > 0
|
|
3452
|
+
|| knowledgeStalenessMissingSourceDocuments > 0) {
|
|
3453
|
+
checks.push({
|
|
3454
|
+
checkId: 'knowledge_staleness_health',
|
|
3455
|
+
status: 'warn',
|
|
3456
|
+
message: 'Knowledge staleness requires incremental rebuild monitoring.',
|
|
3457
|
+
observed: knowledgeStalenessObserved,
|
|
3458
|
+
expected: 'stale=0',
|
|
3459
|
+
});
|
|
3460
|
+
}
|
|
3461
|
+
else {
|
|
3462
|
+
checks.push({
|
|
3463
|
+
checkId: 'knowledge_staleness_health',
|
|
3464
|
+
status: 'pass',
|
|
3465
|
+
message: 'Knowledge staleness health is stable.',
|
|
3466
|
+
observed: knowledgeStalenessObserved,
|
|
3467
|
+
expected: 'stale=0',
|
|
3468
|
+
});
|
|
3469
|
+
}
|
|
3470
|
+
}
|
|
3471
|
+
const hasSessionMemorySignal = Boolean(params.sessionActionTelemetry);
|
|
3472
|
+
if (hasSessionMemorySignal) {
|
|
3473
|
+
const sessionMemoryObserved = [
|
|
3474
|
+
`executions=${sessionActionExecutionCount}`,
|
|
3475
|
+
`persisted=${sessionMemoryPersistedCount}`,
|
|
3476
|
+
`promotedActions=${sessionMemoryPromotionAppliedCount}`,
|
|
3477
|
+
`promotedEntries=${sessionMemoryPromotionCount}`,
|
|
3478
|
+
`promotionCoverage=${sessionMemoryPromotionCoveragePct}%`,
|
|
3479
|
+
].join(', ');
|
|
3480
|
+
if (sessionActionExecutionCount <= 0) {
|
|
3481
|
+
checks.push({
|
|
3482
|
+
checkId: 'session_memory_promotion_coverage',
|
|
3483
|
+
status: 'warn',
|
|
3484
|
+
message: 'Session action telemetry is available but no executions have been recorded yet.',
|
|
3485
|
+
observed: sessionMemoryObserved,
|
|
3486
|
+
expected: 'executions>=1',
|
|
3487
|
+
});
|
|
3488
|
+
}
|
|
3489
|
+
else if (sessionMemoryPersistedCount <= 0) {
|
|
3490
|
+
checks.push({
|
|
3491
|
+
checkId: 'session_memory_promotion_coverage',
|
|
3492
|
+
status: 'warn',
|
|
3493
|
+
message: 'Session actions executed without memory persistence; promotion coverage cannot be assessed.',
|
|
3494
|
+
observed: sessionMemoryObserved,
|
|
3495
|
+
expected: 'persisted>=1',
|
|
3496
|
+
});
|
|
3497
|
+
}
|
|
3498
|
+
else if (sessionMemoryPromotionAppliedCount <= 0) {
|
|
3499
|
+
checks.push({
|
|
3500
|
+
checkId: 'session_memory_promotion_coverage',
|
|
3501
|
+
status: 'warn',
|
|
3502
|
+
message: 'Memory persistence is active but no promotion events were observed.',
|
|
3503
|
+
observed: sessionMemoryObserved,
|
|
3504
|
+
expected: 'promotedActions>=1',
|
|
3505
|
+
});
|
|
3506
|
+
}
|
|
3507
|
+
else if (sessionMemoryPromotionCoveragePct < 10) {
|
|
3508
|
+
checks.push({
|
|
3509
|
+
checkId: 'session_memory_promotion_coverage',
|
|
3510
|
+
status: 'warn',
|
|
3511
|
+
message: 'Memory promotion coverage is low relative to persisted session actions.',
|
|
3512
|
+
observed: sessionMemoryObserved,
|
|
3513
|
+
expected: 'promotionCoverage>=10%',
|
|
3514
|
+
});
|
|
3515
|
+
}
|
|
3516
|
+
else {
|
|
3517
|
+
checks.push({
|
|
3518
|
+
checkId: 'session_memory_promotion_coverage',
|
|
3519
|
+
status: 'pass',
|
|
3520
|
+
message: 'Session memory promotion coverage is within target range.',
|
|
3521
|
+
observed: sessionMemoryObserved,
|
|
3522
|
+
expected: 'promotionCoverage>=10%',
|
|
3523
|
+
});
|
|
3524
|
+
}
|
|
3525
|
+
}
|
|
3526
|
+
if (String(params.queryDiagnostics.lastError || '').trim()) {
|
|
3527
|
+
checks.push({
|
|
3528
|
+
checkId: 'query_backend_last_error',
|
|
3529
|
+
status: 'warn',
|
|
3530
|
+
message: 'Recent query backend error detected.',
|
|
3531
|
+
observed: String(params.queryDiagnostics.lastError || '').slice(0, 160),
|
|
3532
|
+
});
|
|
3533
|
+
}
|
|
3534
|
+
else {
|
|
3535
|
+
checks.push({
|
|
3536
|
+
checkId: 'query_backend_last_error',
|
|
3537
|
+
status: 'pass',
|
|
3538
|
+
message: 'No recent query backend error detected.',
|
|
3539
|
+
observed: 'none',
|
|
3540
|
+
});
|
|
3541
|
+
}
|
|
3542
|
+
const hasTutorGovernanceSignal = Boolean(params.tutorAdapterTelemetry
|
|
3543
|
+
|| params.tutorRoutingConfig
|
|
3544
|
+
|| params.tutorTraceDiagnostics
|
|
3545
|
+
|| params.tutorProviderTrendDiagnostics
|
|
3546
|
+
|| params.tutorProviderTrendHistory);
|
|
3547
|
+
if (hasTutorGovernanceSignal) {
|
|
3548
|
+
if (tutorAdaptersTotal <= 0) {
|
|
3549
|
+
checks.push({
|
|
3550
|
+
checkId: 'tutor_adapter_inventory',
|
|
3551
|
+
status: 'warn',
|
|
3552
|
+
message: 'No tutor adapters are registered for runtime orchestration.',
|
|
3553
|
+
observed: 'totalAdapters=0',
|
|
3554
|
+
expected: 'totalAdapters>=1',
|
|
3555
|
+
});
|
|
3556
|
+
}
|
|
3557
|
+
else if (tutorAdaptersActive <= 0 && tutorRequests > 0) {
|
|
3558
|
+
checks.push({
|
|
3559
|
+
checkId: 'tutor_adapter_inventory',
|
|
3560
|
+
status: 'warn',
|
|
3561
|
+
message: 'Tutor adapters exist but no adapter is currently active.',
|
|
3562
|
+
observed: `totalAdapters=${tutorAdaptersTotal}, activeAdapters=${tutorAdaptersActive}, requests=${tutorRequests}`,
|
|
3563
|
+
expected: 'activeAdapters>=1',
|
|
3564
|
+
});
|
|
3565
|
+
}
|
|
3566
|
+
else {
|
|
3567
|
+
checks.push({
|
|
3568
|
+
checkId: 'tutor_adapter_inventory',
|
|
3569
|
+
status: 'pass',
|
|
3570
|
+
message: 'Tutor adapter inventory is available for routing.',
|
|
3571
|
+
observed: `totalAdapters=${tutorAdaptersTotal}, activeAdapters=${tutorAdaptersActive}, requests=${tutorRequests}`,
|
|
3572
|
+
expected: 'totalAdapters>=1',
|
|
3573
|
+
});
|
|
3574
|
+
}
|
|
3575
|
+
if (tutorRoutingAdapterTimeoutMs < 300 || tutorRoutingAdapterTimeoutMs > 60000) {
|
|
3576
|
+
checks.push({
|
|
3577
|
+
checkId: 'tutor_adapter_timeout_budget',
|
|
3578
|
+
status: 'fail',
|
|
3579
|
+
message: 'Tutor adapter timeout is outside hard governance budget.',
|
|
3580
|
+
observed: `adapterTimeoutMs=${tutorRoutingAdapterTimeoutMs}`,
|
|
3581
|
+
expected: '300<=adapterTimeoutMs<=60000',
|
|
3582
|
+
});
|
|
3583
|
+
}
|
|
3584
|
+
else if (tutorRoutingAdapterTimeoutMs < 1000 || tutorRoutingAdapterTimeoutMs > 30000) {
|
|
3585
|
+
checks.push({
|
|
3586
|
+
checkId: 'tutor_adapter_timeout_budget',
|
|
3587
|
+
status: 'warn',
|
|
3588
|
+
message: 'Tutor adapter timeout is outside the recommended operating window.',
|
|
3589
|
+
observed: `adapterTimeoutMs=${tutorRoutingAdapterTimeoutMs}`,
|
|
3590
|
+
expected: '1000<=adapterTimeoutMs<=30000',
|
|
3591
|
+
});
|
|
3592
|
+
}
|
|
3593
|
+
else {
|
|
3594
|
+
checks.push({
|
|
3595
|
+
checkId: 'tutor_adapter_timeout_budget',
|
|
3596
|
+
status: 'pass',
|
|
3597
|
+
message: 'Tutor adapter timeout is within recommended governance window.',
|
|
3598
|
+
observed: `adapterTimeoutMs=${tutorRoutingAdapterTimeoutMs}`,
|
|
3599
|
+
expected: '1000<=adapterTimeoutMs<=30000',
|
|
3600
|
+
});
|
|
3601
|
+
}
|
|
3602
|
+
const dominantFallbackProviderObserved = [
|
|
3603
|
+
`providers=${tutorProviderCount}`,
|
|
3604
|
+
`fallbackTraces=${tutorFallbackTraceCount}`,
|
|
3605
|
+
`dominantProvider=${tutorDominantProviderName || 'none'}`,
|
|
3606
|
+
`dominantShare=${tutorDominantProviderSharePct}%`,
|
|
3607
|
+
`dominantFallbackProvider=${tutorDominantFallbackProviderName || 'none'}`,
|
|
3608
|
+
`dominantFallbackShare=${tutorDominantFallbackProviderSharePct}%`,
|
|
3609
|
+
].join(', ');
|
|
3610
|
+
if (!params.tutorTraceDiagnostics) {
|
|
3611
|
+
checks.push({
|
|
3612
|
+
checkId: 'tutor_provider_concentration',
|
|
3613
|
+
status: 'pass',
|
|
3614
|
+
message: 'Tutor provider concentration diagnostics are not enabled for this runtime snapshot.',
|
|
3615
|
+
observed: dominantFallbackProviderObserved,
|
|
3616
|
+
expected: 'enable tutor trace diagnostics for concentration checks',
|
|
3617
|
+
});
|
|
3618
|
+
}
|
|
3619
|
+
else if (tutorRequests <= 0) {
|
|
3620
|
+
checks.push({
|
|
3621
|
+
checkId: 'tutor_provider_concentration',
|
|
3622
|
+
status: 'warn',
|
|
3623
|
+
message: 'No tutor requests recorded yet; provider concentration cannot be evaluated.',
|
|
3624
|
+
observed: dominantFallbackProviderObserved,
|
|
3625
|
+
expected: 'requests>=1 with provider breakdown',
|
|
3626
|
+
});
|
|
3627
|
+
}
|
|
3628
|
+
else if (tutorProviderCount <= 0) {
|
|
3629
|
+
checks.push({
|
|
3630
|
+
checkId: 'tutor_provider_concentration',
|
|
3631
|
+
status: 'warn',
|
|
3632
|
+
message: 'Tutor trace diagnostics are missing provider breakdown data.',
|
|
3633
|
+
observed: dominantFallbackProviderObserved,
|
|
3634
|
+
expected: 'providerBreakdown>=1',
|
|
3635
|
+
});
|
|
3636
|
+
}
|
|
3637
|
+
else if (tutorFallbackTraceCount >= 5
|
|
3638
|
+
&& tutorDominantFallbackProviderSharePct >= 95) {
|
|
3639
|
+
checks.push({
|
|
3640
|
+
checkId: 'tutor_provider_concentration',
|
|
3641
|
+
status: 'fail',
|
|
3642
|
+
message: 'Tutor fallback traffic is concentrated on a single provider and risks a single point of failure.',
|
|
3643
|
+
observed: dominantFallbackProviderObserved,
|
|
3644
|
+
expected: 'dominantFallbackProviderShare<95% when fallbackTraces>=5',
|
|
3645
|
+
});
|
|
3646
|
+
}
|
|
3647
|
+
else if (tutorFallbackTraceCount >= 3
|
|
3648
|
+
&& tutorDominantFallbackProviderSharePct >= 80) {
|
|
3649
|
+
checks.push({
|
|
3650
|
+
checkId: 'tutor_provider_concentration',
|
|
3651
|
+
status: 'warn',
|
|
3652
|
+
message: 'Tutor fallback traffic is skewed toward one provider.',
|
|
3653
|
+
observed: dominantFallbackProviderObserved,
|
|
3654
|
+
expected: 'dominantFallbackProviderShare<80% when fallbackTraces>=3',
|
|
3655
|
+
});
|
|
3656
|
+
}
|
|
3657
|
+
else if (tutorProviderCount <= 1
|
|
3658
|
+
&& tutorRequests >= tutorRoutingMinSamples) {
|
|
3659
|
+
checks.push({
|
|
3660
|
+
checkId: 'tutor_provider_concentration',
|
|
3661
|
+
status: 'warn',
|
|
3662
|
+
message: 'Tutor routing currently depends on a single provider footprint.',
|
|
3663
|
+
observed: dominantFallbackProviderObserved,
|
|
3664
|
+
expected: 'providerBreakdown>=2 when requests are stable',
|
|
3665
|
+
});
|
|
3666
|
+
}
|
|
3667
|
+
else {
|
|
3668
|
+
checks.push({
|
|
3669
|
+
checkId: 'tutor_provider_concentration',
|
|
3670
|
+
status: 'pass',
|
|
3671
|
+
message: 'Tutor provider concentration is within acceptable resilience range.',
|
|
3672
|
+
observed: dominantFallbackProviderObserved,
|
|
3673
|
+
expected: 'dominantFallbackProviderShare<80% when fallbackTraces>=3',
|
|
3674
|
+
});
|
|
3675
|
+
}
|
|
3676
|
+
const tutorProviderTrendObserved = [
|
|
3677
|
+
`providers=${tutorProviderTrendProviders.length}`,
|
|
3678
|
+
`regressing=${tutorProviderTrendRegressingCount}`,
|
|
3679
|
+
`improving=${tutorProviderTrendImprovingCount}`,
|
|
3680
|
+
`insufficient=${tutorProviderTrendInsufficientDataCount}`,
|
|
3681
|
+
`historyRecords=${tutorProviderTrendHistoryRecords}`,
|
|
3682
|
+
`historyRegressing=${tutorProviderTrendHistoryRegressingRecords}`,
|
|
3683
|
+
`historyStable=${tutorProviderTrendHistoryStableRecords}`,
|
|
3684
|
+
`historyImproving=${tutorProviderTrendHistoryImprovingRecords}`,
|
|
3685
|
+
`historyInsufficient=${tutorProviderTrendHistoryInsufficientDataRecords}`,
|
|
3686
|
+
`topRegressing=${tutorProviderTrendTopRegressingProviderName || 'none'}`,
|
|
3687
|
+
`topScore=${tutorProviderTrendTopRegressingScore}`,
|
|
3688
|
+
`topConfidence=${tutorProviderTrendTopRegressingConfidence}`,
|
|
3689
|
+
`focus=${tutorProviderTrendRecommendedFocusProviderName || 'none'}`,
|
|
3690
|
+
].join(', ');
|
|
3691
|
+
if (!params.tutorProviderTrendDiagnostics) {
|
|
3692
|
+
checks.push({
|
|
3693
|
+
checkId: 'tutor_provider_trend_regression',
|
|
3694
|
+
status: 'pass',
|
|
3695
|
+
message: 'Tutor provider trend diagnostics are not enabled for this runtime snapshot.',
|
|
3696
|
+
observed: tutorProviderTrendObserved,
|
|
3697
|
+
expected: 'enable tutor provider trend diagnostics for regression checks',
|
|
3698
|
+
});
|
|
3699
|
+
}
|
|
3700
|
+
else if (tutorProviderTrendProviders.length <= 0) {
|
|
3701
|
+
checks.push({
|
|
3702
|
+
checkId: 'tutor_provider_trend_regression',
|
|
3703
|
+
status: 'warn',
|
|
3704
|
+
message: 'Tutor provider trend diagnostics returned no provider trend records.',
|
|
3705
|
+
observed: tutorProviderTrendObserved,
|
|
3706
|
+
expected: 'providers>=1',
|
|
3707
|
+
});
|
|
3708
|
+
}
|
|
3709
|
+
else if (tutorProviderTrendRegressingCount >= 2
|
|
3710
|
+
|| (tutorProviderTrendRegressingCount >= 1
|
|
3711
|
+
&& tutorProviderTrendTopRegressingConfidence >= 60
|
|
3712
|
+
&& tutorProviderTrendTopRegressingScore >= 25)
|
|
3713
|
+
|| (tutorProviderTrendHistoryRecords >= 5
|
|
3714
|
+
&& tutorProviderTrendHistoryRegressingRecords >= 3)) {
|
|
3715
|
+
checks.push({
|
|
3716
|
+
checkId: 'tutor_provider_trend_regression',
|
|
3717
|
+
status: 'fail',
|
|
3718
|
+
message: 'Tutor provider trend diagnostics indicate sustained provider regression.',
|
|
3719
|
+
observed: tutorProviderTrendObserved,
|
|
3720
|
+
expected: 'regressingProviders=0 or low-confidence transient drift only',
|
|
3721
|
+
});
|
|
3722
|
+
}
|
|
3723
|
+
else if (tutorProviderTrendRegressingCount >= 1) {
|
|
3724
|
+
checks.push({
|
|
3725
|
+
checkId: 'tutor_provider_trend_regression',
|
|
3726
|
+
status: 'warn',
|
|
3727
|
+
message: 'Tutor provider trend diagnostics detected at least one regressing provider.',
|
|
3728
|
+
observed: tutorProviderTrendObserved,
|
|
3729
|
+
expected: 'regressingProviders=0',
|
|
3730
|
+
});
|
|
3731
|
+
}
|
|
3732
|
+
else if (tutorProviderTrendHistoryRecords >= 3
|
|
3733
|
+
&& tutorProviderTrendHistoryRegressingRecords >= 1) {
|
|
3734
|
+
checks.push({
|
|
3735
|
+
checkId: 'tutor_provider_trend_regression',
|
|
3736
|
+
status: 'warn',
|
|
3737
|
+
message: 'Tutor provider trend history indicates intermittent provider regression.',
|
|
3738
|
+
observed: tutorProviderTrendObserved,
|
|
3739
|
+
expected: 'historyRegressingRecords=0 across recent windows',
|
|
3740
|
+
});
|
|
3741
|
+
}
|
|
3742
|
+
else if (tutorProviderTrendInsufficientDataCount >= tutorProviderTrendProviders.length
|
|
3743
|
+
&& tutorRequests >= tutorRoutingMinSamples) {
|
|
3744
|
+
checks.push({
|
|
3745
|
+
checkId: 'tutor_provider_trend_regression',
|
|
3746
|
+
status: 'warn',
|
|
3747
|
+
message: 'Tutor trend diagnostics remain sample-starved despite sufficient request volume.',
|
|
3748
|
+
observed: tutorProviderTrendObserved,
|
|
3749
|
+
expected: 'providers with sufficient trend windows >=1',
|
|
3750
|
+
});
|
|
3751
|
+
}
|
|
3752
|
+
else {
|
|
3753
|
+
checks.push({
|
|
3754
|
+
checkId: 'tutor_provider_trend_regression',
|
|
3755
|
+
status: 'pass',
|
|
3756
|
+
message: 'Tutor provider trend diagnostics show no active regression.',
|
|
3757
|
+
observed: tutorProviderTrendObserved,
|
|
3758
|
+
expected: 'regressingProviders=0',
|
|
3759
|
+
});
|
|
3760
|
+
}
|
|
3761
|
+
if (!tutorRoutingEnabled) {
|
|
3762
|
+
checks.push({
|
|
3763
|
+
checkId: 'tutor_routing_health_budget',
|
|
3764
|
+
status: 'warn',
|
|
3765
|
+
message: 'Adaptive tutor routing is disabled; telemetry health budgets are not enforced.',
|
|
3766
|
+
observed: [
|
|
3767
|
+
`enabled=false`,
|
|
3768
|
+
`failedRatio=${tutorFailedRatioPct}%`,
|
|
3769
|
+
`downgradedRatio=${tutorDowngradedRatioPct}%`,
|
|
3770
|
+
`fallbackRatio=${tutorProviderFallbackRatioPct}%`,
|
|
3771
|
+
`avgProviderAttempts=${tutorAverageProviderAttemptCount}`,
|
|
3772
|
+
`avgConfidence=${tutorAverageConfidence}`,
|
|
3773
|
+
].join(', '),
|
|
3774
|
+
expected: 'enabled=true',
|
|
3775
|
+
});
|
|
3776
|
+
}
|
|
3777
|
+
else if (tutorRequests < tutorRoutingMinSamples) {
|
|
3778
|
+
checks.push({
|
|
3779
|
+
checkId: 'tutor_routing_health_budget',
|
|
3780
|
+
status: 'warn',
|
|
3781
|
+
message: 'Tutor routing telemetry sample size is insufficient for stable governance.',
|
|
3782
|
+
observed: [
|
|
3783
|
+
`requests=${tutorRequests}`,
|
|
3784
|
+
`minSamples=${tutorRoutingMinSamples}`,
|
|
3785
|
+
`failedRatio=${tutorFailedRatioPct}%`,
|
|
3786
|
+
`downgradedRatio=${tutorDowngradedRatioPct}%`,
|
|
3787
|
+
`fallbackRatio=${tutorProviderFallbackRatioPct}%`,
|
|
3788
|
+
`avgProviderAttempts=${tutorAverageProviderAttemptCount}`,
|
|
3789
|
+
`avgConfidence=${tutorAverageConfidence}`,
|
|
3790
|
+
].join(', '),
|
|
3791
|
+
expected: `requests>=${tutorRoutingMinSamples}`,
|
|
3792
|
+
});
|
|
3793
|
+
}
|
|
3794
|
+
else {
|
|
3795
|
+
const fallbackWarnRatioPct = Number(clamp(Math.max(60, tutorRoutingMaxDowngradedRatioPct * 1.5), 0, 100).toFixed(4));
|
|
3796
|
+
const fallbackFailRatioPct = Number(clamp(Math.max(85, tutorRoutingMaxDowngradedRatioPct * 2), 0, 100).toFixed(4));
|
|
3797
|
+
const attemptWarnCount = 2;
|
|
3798
|
+
const attemptFailCount = 4;
|
|
3799
|
+
const severeConfidenceFloor = Number(clamp(tutorRoutingMinAverageConfidence - 0.2, 0, 1).toFixed(4));
|
|
3800
|
+
if (tutorFailedRatioPct > tutorRoutingMaxFailedRatioPct
|
|
3801
|
+
|| tutorDowngradedRatioPct > tutorRoutingMaxDowngradedRatioPct
|
|
3802
|
+
|| tutorProviderFallbackRatioPct > fallbackFailRatioPct
|
|
3803
|
+
|| tutorAverageProviderAttemptCount >= attemptFailCount
|
|
3804
|
+
|| tutorAverageConfidence < severeConfidenceFloor) {
|
|
3805
|
+
checks.push({
|
|
3806
|
+
checkId: 'tutor_routing_health_budget',
|
|
3807
|
+
status: 'fail',
|
|
3808
|
+
message: 'Tutor routing telemetry violates hard health budgets.',
|
|
3809
|
+
observed: [
|
|
3810
|
+
`failedRatio=${tutorFailedRatioPct}%`,
|
|
3811
|
+
`downgradedRatio=${tutorDowngradedRatioPct}%`,
|
|
3812
|
+
`fallbackRatio=${tutorProviderFallbackRatioPct}%`,
|
|
3813
|
+
`avgProviderAttempts=${tutorAverageProviderAttemptCount}`,
|
|
3814
|
+
`avgConfidence=${tutorAverageConfidence}`,
|
|
3815
|
+
].join(', '),
|
|
3816
|
+
expected: [
|
|
3817
|
+
`failedRatio<=${tutorRoutingMaxFailedRatioPct}%`,
|
|
3818
|
+
`downgradedRatio<=${tutorRoutingMaxDowngradedRatioPct}%`,
|
|
3819
|
+
`fallbackRatio<=${fallbackFailRatioPct}%`,
|
|
3820
|
+
`avgProviderAttempts<${attemptFailCount}`,
|
|
3821
|
+
`avgConfidence>=${severeConfidenceFloor}`,
|
|
3822
|
+
].join(', '),
|
|
3823
|
+
});
|
|
3824
|
+
}
|
|
3825
|
+
else if (tutorFailedRatioPct > tutorRoutingMaxFailedRatioPct * 0.7
|
|
3826
|
+
|| tutorDowngradedRatioPct > tutorRoutingMaxDowngradedRatioPct * 0.75
|
|
3827
|
+
|| tutorProviderFallbackRatioPct > fallbackWarnRatioPct
|
|
3828
|
+
|| tutorAverageProviderAttemptCount >= attemptWarnCount
|
|
3829
|
+
|| tutorAverageConfidence < tutorRoutingMinAverageConfidence) {
|
|
3830
|
+
checks.push({
|
|
3831
|
+
checkId: 'tutor_routing_health_budget',
|
|
3832
|
+
status: 'warn',
|
|
3833
|
+
message: 'Tutor routing telemetry is close to configured health limits.',
|
|
3834
|
+
observed: [
|
|
3835
|
+
`failedRatio=${tutorFailedRatioPct}%`,
|
|
3836
|
+
`downgradedRatio=${tutorDowngradedRatioPct}%`,
|
|
3837
|
+
`fallbackRatio=${tutorProviderFallbackRatioPct}%`,
|
|
3838
|
+
`avgProviderAttempts=${tutorAverageProviderAttemptCount}`,
|
|
3839
|
+
`avgConfidence=${tutorAverageConfidence}`,
|
|
3840
|
+
].join(', '),
|
|
3841
|
+
expected: [
|
|
3842
|
+
`failedRatio<=${tutorRoutingMaxFailedRatioPct}%`,
|
|
3843
|
+
`downgradedRatio<=${tutorRoutingMaxDowngradedRatioPct}%`,
|
|
3844
|
+
`fallbackRatio<=${fallbackWarnRatioPct}%`,
|
|
3845
|
+
`avgProviderAttempts<${attemptWarnCount}`,
|
|
3846
|
+
`avgConfidence>=${tutorRoutingMinAverageConfidence}`,
|
|
3847
|
+
].join(', '),
|
|
3848
|
+
});
|
|
3849
|
+
}
|
|
3850
|
+
else {
|
|
3851
|
+
checks.push({
|
|
3852
|
+
checkId: 'tutor_routing_health_budget',
|
|
3853
|
+
status: 'pass',
|
|
3854
|
+
message: 'Tutor routing telemetry is within configured health budgets.',
|
|
3855
|
+
observed: [
|
|
3856
|
+
`failedRatio=${tutorFailedRatioPct}%`,
|
|
3857
|
+
`downgradedRatio=${tutorDowngradedRatioPct}%`,
|
|
3858
|
+
`fallbackRatio=${tutorProviderFallbackRatioPct}%`,
|
|
3859
|
+
`avgProviderAttempts=${tutorAverageProviderAttemptCount}`,
|
|
3860
|
+
`avgConfidence=${tutorAverageConfidence}`,
|
|
3861
|
+
].join(', '),
|
|
3862
|
+
expected: [
|
|
3863
|
+
`failedRatio<=${tutorRoutingMaxFailedRatioPct}%`,
|
|
3864
|
+
`downgradedRatio<=${tutorRoutingMaxDowngradedRatioPct}%`,
|
|
3865
|
+
`fallbackRatio<=${fallbackWarnRatioPct}%`,
|
|
3866
|
+
`avgProviderAttempts<${attemptWarnCount}`,
|
|
3867
|
+
`avgConfidence>=${tutorRoutingMinAverageConfidence}`,
|
|
3868
|
+
].join(', '),
|
|
3869
|
+
});
|
|
3870
|
+
}
|
|
3871
|
+
}
|
|
3872
|
+
const routingObserved = [
|
|
3873
|
+
`requests=${tutorRequests}`,
|
|
3874
|
+
`strategy=${tutorLastRoutingStrategy}`,
|
|
3875
|
+
`dynamicMode=${tutorRoutingDynamicPreferredMode}`,
|
|
3876
|
+
`dynamicSuggested=${tutorRoutingDynamicModeSuggestionActive}`,
|
|
3877
|
+
`fallbackRatio=${tutorProviderFallbackRatioPct}%`,
|
|
3878
|
+
`avgProviderAttempts=${tutorAverageProviderAttemptCount}`,
|
|
3879
|
+
`score=${tutorLastRoutingScore}`,
|
|
3880
|
+
tutorLastRoutingReason ? `reason=${tutorLastRoutingReason}` : '',
|
|
3881
|
+
tutorRoutingDynamicModeReason ? `dynamicReason=${tutorRoutingDynamicModeReason}` : '',
|
|
3882
|
+
].filter(Boolean).join(', ');
|
|
3883
|
+
if (tutorRequests <= 0) {
|
|
3884
|
+
checks.push({
|
|
3885
|
+
checkId: 'tutor_routing_traceability',
|
|
3886
|
+
status: 'warn',
|
|
3887
|
+
message: 'No tutor routing decisions recorded yet.',
|
|
3888
|
+
observed: routingObserved,
|
|
3889
|
+
expected: 'requests>=1 with strategy trace',
|
|
3890
|
+
});
|
|
3891
|
+
}
|
|
3892
|
+
else if (tutorLastRoutingStrategy === 'unknown') {
|
|
3893
|
+
checks.push({
|
|
3894
|
+
checkId: 'tutor_routing_traceability',
|
|
3895
|
+
status: 'warn',
|
|
3896
|
+
message: 'Tutor routing strategy trace is missing from telemetry.',
|
|
3897
|
+
observed: routingObserved,
|
|
3898
|
+
expected: 'strategy in {explicit_adapter_id, explicit_provider_mode, adaptive_health_routing, fallback_default}',
|
|
3899
|
+
});
|
|
3900
|
+
}
|
|
3901
|
+
else if (tutorRoutingEnabled
|
|
3902
|
+
&& tutorRequests >= tutorRoutingMinSamples
|
|
3903
|
+
&& tutorLastRoutingStrategy === 'fallback_default') {
|
|
3904
|
+
checks.push({
|
|
3905
|
+
checkId: 'tutor_routing_traceability',
|
|
3906
|
+
status: 'warn',
|
|
3907
|
+
message: 'Tutor routing frequently falls back to default adapter under active routing.',
|
|
3908
|
+
observed: routingObserved,
|
|
3909
|
+
expected: 'strategy!=fallback_default when adaptive routing is healthy',
|
|
3910
|
+
});
|
|
3911
|
+
}
|
|
3912
|
+
else {
|
|
3913
|
+
checks.push({
|
|
3914
|
+
checkId: 'tutor_routing_traceability',
|
|
3915
|
+
status: 'pass',
|
|
3916
|
+
message: 'Tutor routing decisions are traceable.',
|
|
3917
|
+
observed: routingObserved,
|
|
3918
|
+
expected: 'strategy trace available',
|
|
3919
|
+
});
|
|
3920
|
+
}
|
|
3921
|
+
const fallbackWarnRatioPct = Number(clamp(Math.max(60, tutorRoutingMaxDowngradedRatioPct * 1.5), 0, 100).toFixed(4));
|
|
3922
|
+
const fallbackFailRatioPct = Number(clamp(Math.max(85, tutorRoutingMaxDowngradedRatioPct * 2), 0, 100).toFixed(4));
|
|
3923
|
+
const severeConfidenceFloor = Number(clamp(tutorRoutingMinAverageConfidence - 0.2, 0, 1).toFixed(4));
|
|
3924
|
+
const dynamicAlignmentExpected = (tutorRoutingDynamicModeSuggestionActive
|
|
3925
|
+
? `preferredMode=auto or preferredMode=${tutorRoutingDynamicPreferredMode}`
|
|
3926
|
+
: 'dynamicSuggested=false or preferredMode matches suggestion');
|
|
3927
|
+
const dynamicAlignmentObserved = [
|
|
3928
|
+
`preferredMode=${tutorRoutingPreferredMode}`,
|
|
3929
|
+
`dynamicMode=${tutorRoutingDynamicPreferredMode}`,
|
|
3930
|
+
`dynamicSuggested=${tutorRoutingDynamicModeSuggestionActive}`,
|
|
3931
|
+
`strategy=${tutorLastRoutingStrategy}`,
|
|
3932
|
+
`requests=${tutorRequests}/${tutorRoutingMinSamples}`,
|
|
3933
|
+
`failedRatio=${tutorFailedRatioPct}%`,
|
|
3934
|
+
`downgradedRatio=${tutorDowngradedRatioPct}%`,
|
|
3935
|
+
`fallbackRatio=${tutorProviderFallbackRatioPct}%`,
|
|
3936
|
+
`avgAttempts=${tutorAverageProviderAttemptCount}`,
|
|
3937
|
+
`avgConfidence=${tutorAverageConfidence}`,
|
|
3938
|
+
tutorRoutingDynamicModeReason ? `dynamicReason=${tutorRoutingDynamicModeReason}` : '',
|
|
3939
|
+
].filter(Boolean).join(', ');
|
|
3940
|
+
const hasDynamicModeConflict = (tutorRoutingDynamicModeSuggestionActive
|
|
3941
|
+
&& tutorRoutingPreferredMode !== 'auto'
|
|
3942
|
+
&& tutorRoutingPreferredMode !== tutorRoutingDynamicPreferredMode);
|
|
3943
|
+
if (!tutorRoutingEnabled) {
|
|
3944
|
+
checks.push({
|
|
3945
|
+
checkId: 'tutor_routing_dynamic_mode_alignment',
|
|
3946
|
+
status: 'warn',
|
|
3947
|
+
message: 'Adaptive tutor routing is disabled; dynamic mode alignment cannot be enforced.',
|
|
3948
|
+
observed: dynamicAlignmentObserved,
|
|
3949
|
+
expected: dynamicAlignmentExpected,
|
|
3950
|
+
});
|
|
3951
|
+
}
|
|
3952
|
+
else if (!hasDynamicModeConflict) {
|
|
3953
|
+
checks.push({
|
|
3954
|
+
checkId: 'tutor_routing_dynamic_mode_alignment',
|
|
3955
|
+
status: 'pass',
|
|
3956
|
+
message: 'Tutor routing preferred mode is aligned with dynamic mode guidance.',
|
|
3957
|
+
observed: dynamicAlignmentObserved,
|
|
3958
|
+
expected: dynamicAlignmentExpected,
|
|
3959
|
+
});
|
|
3960
|
+
}
|
|
3961
|
+
else if (tutorRequests < tutorRoutingMinSamples
|
|
3962
|
+
|| tutorLastRoutingStrategy !== 'adaptive_health_routing') {
|
|
3963
|
+
checks.push({
|
|
3964
|
+
checkId: 'tutor_routing_dynamic_mode_alignment',
|
|
3965
|
+
status: 'warn',
|
|
3966
|
+
message: 'Dynamic mode guidance conflicts with pinned preferred mode, but routing evidence is still limited.',
|
|
3967
|
+
observed: dynamicAlignmentObserved,
|
|
3968
|
+
expected: dynamicAlignmentExpected,
|
|
3969
|
+
});
|
|
3970
|
+
}
|
|
3971
|
+
else if (tutorFailedRatioPct > tutorRoutingMaxFailedRatioPct
|
|
3972
|
+
|| tutorDowngradedRatioPct > tutorRoutingMaxDowngradedRatioPct
|
|
3973
|
+
|| tutorProviderFallbackRatioPct > fallbackFailRatioPct
|
|
3974
|
+
|| tutorAverageProviderAttemptCount >= 4
|
|
3975
|
+
|| tutorAverageConfidence < severeConfidenceFloor) {
|
|
3976
|
+
checks.push({
|
|
3977
|
+
checkId: 'tutor_routing_dynamic_mode_alignment',
|
|
3978
|
+
status: 'fail',
|
|
3979
|
+
message: 'Pinned preferred mode conflicts with dynamic routing guidance under severe degradation.',
|
|
3980
|
+
observed: dynamicAlignmentObserved,
|
|
3981
|
+
expected: dynamicAlignmentExpected,
|
|
3982
|
+
});
|
|
3983
|
+
}
|
|
3984
|
+
else if (tutorFailedRatioPct > tutorRoutingMaxFailedRatioPct * 0.7
|
|
3985
|
+
|| tutorDowngradedRatioPct > tutorRoutingMaxDowngradedRatioPct * 0.75
|
|
3986
|
+
|| tutorProviderFallbackRatioPct > fallbackWarnRatioPct
|
|
3987
|
+
|| tutorAverageProviderAttemptCount >= 2
|
|
3988
|
+
|| tutorAverageConfidence < tutorRoutingMinAverageConfidence) {
|
|
3989
|
+
checks.push({
|
|
3990
|
+
checkId: 'tutor_routing_dynamic_mode_alignment',
|
|
3991
|
+
status: 'warn',
|
|
3992
|
+
message: 'Pinned preferred mode conflicts with dynamic routing guidance and health budgets are tightening.',
|
|
3993
|
+
observed: dynamicAlignmentObserved,
|
|
3994
|
+
expected: dynamicAlignmentExpected,
|
|
3995
|
+
});
|
|
3996
|
+
}
|
|
3997
|
+
else {
|
|
3998
|
+
checks.push({
|
|
3999
|
+
checkId: 'tutor_routing_dynamic_mode_alignment',
|
|
4000
|
+
status: 'warn',
|
|
4001
|
+
message: 'Pinned preferred mode conflicts with current dynamic routing recommendation.',
|
|
4002
|
+
observed: dynamicAlignmentObserved,
|
|
4003
|
+
expected: dynamicAlignmentExpected,
|
|
4004
|
+
});
|
|
4005
|
+
}
|
|
4006
|
+
}
|
|
4007
|
+
const checkDebugTraceHints = {
|
|
4008
|
+
query_fallback_ratio: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4009
|
+
pathPrefix: '/api/knowledge/query',
|
|
4010
|
+
statusAtLeast: 400,
|
|
4011
|
+
}),
|
|
4012
|
+
query_evidence_coverage_ratio: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4013
|
+
pathPrefix: '/api/knowledge/query',
|
|
4014
|
+
statusAtLeast: 400,
|
|
4015
|
+
}),
|
|
4016
|
+
query_temporal_validity_ratio: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4017
|
+
pathPrefix: '/api/knowledge/query',
|
|
4018
|
+
statusAtLeast: 400,
|
|
4019
|
+
}),
|
|
4020
|
+
query_backend_explainability_gap: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4021
|
+
pathPrefix: '/api/knowledge/query/compare-backends',
|
|
4022
|
+
statusAtLeast: 400,
|
|
4023
|
+
}),
|
|
4024
|
+
query_backend_comparison_trend: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4025
|
+
pathPrefix: '/api/knowledge/query/compare-backends/trend',
|
|
4026
|
+
statusAtLeast: 400,
|
|
4027
|
+
}),
|
|
4028
|
+
query_backend_trend_config: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4029
|
+
pathPrefix: '/api/knowledge/query/compare-backends/trend',
|
|
4030
|
+
statusAtLeast: 400,
|
|
4031
|
+
}),
|
|
4032
|
+
query_backend_last_error: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4033
|
+
pathPrefix: '/api/knowledge/query',
|
|
4034
|
+
statusAtLeast: 400,
|
|
4035
|
+
}),
|
|
4036
|
+
query_backend_runtime_health: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4037
|
+
pathPrefix: '/api/knowledge/query-backend-diagnostics',
|
|
4038
|
+
statusAtLeast: 400,
|
|
4039
|
+
}),
|
|
4040
|
+
store_graphdb_connector_health: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4041
|
+
pathPrefix: '/api/knowledge/store-diagnostics',
|
|
4042
|
+
statusAtLeast: 400,
|
|
4043
|
+
}),
|
|
4044
|
+
store_graphdb_connector_budget: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4045
|
+
pathPrefix: '/api/knowledge/store-diagnostics',
|
|
4046
|
+
statusAtLeast: 400,
|
|
4047
|
+
}),
|
|
4048
|
+
query_vector_index_status: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4049
|
+
pathPrefix: '/api/knowledge/query-backend-diagnostics',
|
|
4050
|
+
statusAtLeast: 400,
|
|
4051
|
+
}),
|
|
4052
|
+
query_vector_index_persistence: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4053
|
+
pathPrefix: '/api/knowledge/query-backend-diagnostics',
|
|
4054
|
+
statusAtLeast: 400,
|
|
4055
|
+
}),
|
|
4056
|
+
query_vector_acceleration_mode: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4057
|
+
pathPrefix: '/api/knowledge/query-backend-diagnostics',
|
|
4058
|
+
statusAtLeast: 400,
|
|
4059
|
+
}),
|
|
4060
|
+
query_vector_acceleration_representation_consistency: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4061
|
+
pathPrefix: '/api/knowledge/query-backend-diagnostics',
|
|
4062
|
+
statusAtLeast: 400,
|
|
4063
|
+
}),
|
|
4064
|
+
query_vector_acceleration_prefilter_effectiveness: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4065
|
+
pathPrefix: '/api/knowledge/query-backend-diagnostics',
|
|
4066
|
+
statusAtLeast: 400,
|
|
4067
|
+
}),
|
|
4068
|
+
query_vector_acceleration_calibration_readiness: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4069
|
+
pathPrefix: '/api/knowledge/query-backend-diagnostics',
|
|
4070
|
+
statusAtLeast: 400,
|
|
4071
|
+
}),
|
|
4072
|
+
query_vector_acceleration_health: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4073
|
+
pathPrefix: '/api/knowledge/query-backend-diagnostics',
|
|
4074
|
+
statusAtLeast: 400,
|
|
4075
|
+
}),
|
|
4076
|
+
query_vector_acceleration_index_sync_health: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4077
|
+
pathPrefix: '/api/knowledge/query-backend-diagnostics',
|
|
4078
|
+
statusAtLeast: 400,
|
|
4079
|
+
}),
|
|
4080
|
+
query_vector_acceleration_traceability: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4081
|
+
pathPrefix: '/api/knowledge/query-backend-diagnostics',
|
|
4082
|
+
statusAtLeast: 400,
|
|
4083
|
+
}),
|
|
4084
|
+
query_vector_acceleration_circuit_state: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4085
|
+
pathPrefix: '/api/knowledge/query-backend-diagnostics',
|
|
4086
|
+
statusAtLeast: 400,
|
|
4087
|
+
}),
|
|
4088
|
+
api_invalid_request_ratio: apiInvalidRequestRatioDebugTraceHint,
|
|
4089
|
+
api_invalid_request_hotspots: apiInvalidRequestHotspotsDebugTraceHint,
|
|
4090
|
+
api_server_error_ratio: apiServerErrorRatioDebugTraceHint,
|
|
4091
|
+
api_server_error_hotspots: apiServerErrorHotspotsDebugTraceHint,
|
|
4092
|
+
api_transient_error_ratio: apiTransientErrorRatioDebugTraceHint,
|
|
4093
|
+
api_transient_error_hotspots: apiTransientErrorHotspotsDebugTraceHint,
|
|
4094
|
+
api_latency_p95: apiLatencyRatioDebugTraceHint,
|
|
4095
|
+
api_latency_hotspots: apiLatencyHotspotsDebugTraceHint,
|
|
4096
|
+
quality_trend_direction: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4097
|
+
pathPrefix: '/api/knowledge/quality',
|
|
4098
|
+
statusAtLeast: 400,
|
|
4099
|
+
}),
|
|
4100
|
+
session_plan_quality_gate: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4101
|
+
pathPrefix: '/api/knowledge/session/plan',
|
|
4102
|
+
statusAtLeast: 400,
|
|
4103
|
+
}),
|
|
4104
|
+
session_plan_quality_trend: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4105
|
+
pathPrefix: '/api/knowledge/session/plan',
|
|
4106
|
+
statusAtLeast: 400,
|
|
4107
|
+
}),
|
|
4108
|
+
orchestration_path_strategy_alignment: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4109
|
+
pathPrefix: '/api/knowledge/session/history',
|
|
4110
|
+
statusAtLeast: 400,
|
|
4111
|
+
}),
|
|
4112
|
+
memory_policy_health: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4113
|
+
pathPrefix: '/api/knowledge/memory',
|
|
4114
|
+
statusAtLeast: 400,
|
|
4115
|
+
}),
|
|
4116
|
+
memory_policy_trend: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4117
|
+
pathPrefix: '/api/knowledge/memory',
|
|
4118
|
+
statusAtLeast: 400,
|
|
4119
|
+
}),
|
|
4120
|
+
knowledge_staleness_data: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4121
|
+
pathPrefix: '/api/knowledge/staleness',
|
|
4122
|
+
statusAtLeast: 400,
|
|
4123
|
+
}),
|
|
4124
|
+
knowledge_staleness_health: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4125
|
+
pathPrefix: '/api/knowledge/staleness',
|
|
4126
|
+
statusAtLeast: 400,
|
|
4127
|
+
}),
|
|
4128
|
+
session_memory_promotion_coverage: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4129
|
+
pathPrefix: '/api/knowledge/session',
|
|
4130
|
+
statusAtLeast: 400,
|
|
4131
|
+
}),
|
|
4132
|
+
tutor_adapter_inventory: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4133
|
+
pathPrefix: '/api/knowledge/tutor',
|
|
4134
|
+
statusAtLeast: 400,
|
|
4135
|
+
}),
|
|
4136
|
+
tutor_provider_concentration: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4137
|
+
pathPrefix: '/api/knowledge/tutor/trace-diagnostics',
|
|
4138
|
+
statusAtLeast: 400,
|
|
4139
|
+
}),
|
|
4140
|
+
tutor_provider_trend_regression: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4141
|
+
pathPrefix: '/api/knowledge/tutor/trace-diagnostics/providers',
|
|
4142
|
+
statusAtLeast: 400,
|
|
4143
|
+
}),
|
|
4144
|
+
tutor_routing_health_budget: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4145
|
+
pathPrefix: '/api/knowledge/tutor',
|
|
4146
|
+
statusAtLeast: 400,
|
|
4147
|
+
}),
|
|
4148
|
+
tutor_routing_traceability: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4149
|
+
pathPrefix: '/api/knowledge/tutor',
|
|
4150
|
+
statusAtLeast: 400,
|
|
4151
|
+
}),
|
|
4152
|
+
tutor_routing_dynamic_mode_alignment: normalizeRuntimeCapabilityDebugTraceHint({
|
|
4153
|
+
pathPrefix: '/api/knowledge/tutor/trace-diagnostics/providers/history',
|
|
4154
|
+
statusAtLeast: 400,
|
|
4155
|
+
}),
|
|
4156
|
+
};
|
|
4157
|
+
const recommendedActionContext = {
|
|
4158
|
+
thresholds,
|
|
4159
|
+
apiScopePathPrefix: defaultApiTraceHintPathPrefix,
|
|
4160
|
+
apiScopeMethod: defaultApiTraceHintMethod,
|
|
4161
|
+
apiInvalidRequestTopRoute,
|
|
4162
|
+
apiServerErrorTopRoute,
|
|
4163
|
+
apiTransientErrorTopRoute,
|
|
4164
|
+
apiLatencyTopRoute,
|
|
4165
|
+
apiTraceP95DurationMs,
|
|
4166
|
+
apiLatencyHotspotPeakP95Ms,
|
|
4167
|
+
qualityTrendStatus: trendStatus,
|
|
4168
|
+
qualityTrendConfidence: trendConfidence,
|
|
4169
|
+
sessionPlanQualityFailureStreak,
|
|
4170
|
+
sessionPlanQualityPassRatePct,
|
|
4171
|
+
sessionPlanTrendStatus,
|
|
4172
|
+
memoryPolicyStatus,
|
|
4173
|
+
memoryPolicyHealthScore,
|
|
4174
|
+
memoryPolicyTrendStatus,
|
|
4175
|
+
knowledgeStalenessEvaluatedDocuments,
|
|
4176
|
+
knowledgeStalenessStaleDocuments,
|
|
4177
|
+
knowledgeStalenessHashMismatchDocuments,
|
|
4178
|
+
knowledgeStalenessMissingSourceDocuments,
|
|
4179
|
+
knowledgeStalenessReadErrorDocuments,
|
|
4180
|
+
knowledgeStalenessFreshnessRatioPct,
|
|
4181
|
+
sessionMemoryPromotionCoveragePct,
|
|
4182
|
+
sessionStrategyTotalRecords,
|
|
4183
|
+
sessionStrategyStrategyRecords,
|
|
4184
|
+
sessionStrategyTrendAutoSelectionSharePct,
|
|
4185
|
+
sessionStrategyTrendAutoAverageMasteryDeltaPct,
|
|
4186
|
+
sessionStrategyTrendAutoNegativeRatioPct,
|
|
4187
|
+
sessionStrategyModeFallbackSelectionSharePct,
|
|
4188
|
+
sessionStrategySelectionSourceExplicitCount,
|
|
4189
|
+
sessionStrategySelectionSourceTrendCount,
|
|
4190
|
+
sessionStrategySelectionSourceFallbackCount,
|
|
4191
|
+
sessionStrategySelectionSourceUnknownCount,
|
|
4192
|
+
sessionStrategyTopAverageStrategy,
|
|
4193
|
+
sessionStrategyTopAverageMasteryDeltaPct,
|
|
4194
|
+
sessionStrategyTopAverageNegativeRatioPct,
|
|
4195
|
+
tutorAdaptersTotal,
|
|
4196
|
+
tutorAdaptersActive,
|
|
4197
|
+
tutorRequests,
|
|
4198
|
+
tutorProviderFallbackResponses,
|
|
4199
|
+
tutorProviderFallbackRatioPct,
|
|
4200
|
+
tutorAverageProviderAttemptCount,
|
|
4201
|
+
tutorProviderCount,
|
|
4202
|
+
tutorDominantProviderName,
|
|
4203
|
+
tutorDominantProviderSharePct,
|
|
4204
|
+
tutorDominantFallbackProviderName,
|
|
4205
|
+
tutorDominantFallbackProviderSharePct,
|
|
4206
|
+
tutorProviderTrendRegressingCount,
|
|
4207
|
+
tutorProviderTrendImprovingCount,
|
|
4208
|
+
tutorProviderTrendInsufficientDataCount,
|
|
4209
|
+
tutorProviderTrendTopRegressingProvider: tutorProviderTrendTopRegressingProviderName,
|
|
4210
|
+
tutorProviderTrendTopRegressingScore,
|
|
4211
|
+
tutorProviderTrendTopRegressingConfidence,
|
|
4212
|
+
tutorProviderTrendRecommendedFocusProviderName,
|
|
4213
|
+
tutorProviderTrendHistoryRecords,
|
|
4214
|
+
tutorProviderTrendHistoryRegressingRecords,
|
|
4215
|
+
tutorProviderTrendHistoryStableRecords,
|
|
4216
|
+
tutorProviderTrendHistoryImprovingRecords,
|
|
4217
|
+
tutorProviderTrendHistoryInsufficientDataRecords,
|
|
4218
|
+
tutorFailedRatioPct,
|
|
4219
|
+
tutorDowngradedRatioPct,
|
|
4220
|
+
tutorAverageConfidence,
|
|
4221
|
+
tutorRoutingEnabled,
|
|
4222
|
+
tutorRoutingPreferredMode,
|
|
4223
|
+
tutorRoutingMinSamples,
|
|
4224
|
+
tutorRoutingMaxFailedRatioPct,
|
|
4225
|
+
tutorRoutingMaxDowngradedRatioPct,
|
|
4226
|
+
tutorRoutingMinAverageConfidence,
|
|
4227
|
+
tutorRoutingAdapterTimeoutMs,
|
|
4228
|
+
tutorLastRoutingStrategy,
|
|
4229
|
+
tutorRoutingDynamicPreferredMode,
|
|
4230
|
+
tutorRoutingDynamicModeReason,
|
|
4231
|
+
tutorRoutingDynamicModeSuggestionActive,
|
|
4232
|
+
queryBackendRuntimeReady,
|
|
4233
|
+
queryBackendRuntimeId,
|
|
4234
|
+
queryVectorIndexEnabled,
|
|
4235
|
+
queryVectorIndexStatus,
|
|
4236
|
+
queryVectorIndexPersisted,
|
|
4237
|
+
queryVectorIndexLoadedFromDisk,
|
|
4238
|
+
queryVectorIndexAtomCount,
|
|
4239
|
+
queryVectorIndexLocation,
|
|
4240
|
+
queryVectorIndexAccelerationEnabled,
|
|
4241
|
+
queryVectorIndexAccelerationMode,
|
|
4242
|
+
queryVectorIndexAccelerationLastSelectionMode,
|
|
4243
|
+
queryVectorIndexAccelerationLastCandidateCount,
|
|
4244
|
+
queryVectorIndexAccelerationAdapterId,
|
|
4245
|
+
queryVectorIndexAccelerationAdapterError,
|
|
4246
|
+
queryVectorIndexAccelerationHealthStatus,
|
|
4247
|
+
queryVectorIndexAccelerationHealthMessage,
|
|
4248
|
+
queryVectorIndexAccelerationIndexSyncStatus,
|
|
4249
|
+
queryVectorIndexAccelerationIndexSyncMessage,
|
|
4250
|
+
queryVectorIndexAccelerationLastSyncAt,
|
|
4251
|
+
queryVectorIndexAccelerationSyncRequestCount,
|
|
4252
|
+
queryVectorIndexAccelerationSyncSuccessCount,
|
|
4253
|
+
queryVectorIndexAccelerationSyncFailureCount,
|
|
4254
|
+
queryVectorIndexAccelerationSyncedIndexSignature,
|
|
4255
|
+
queryVectorIndexAccelerationSyncedAtomCount,
|
|
4256
|
+
queryVectorIndexAccelerationRepresentationVersion,
|
|
4257
|
+
queryVectorIndexAccelerationEmbeddingModelId,
|
|
4258
|
+
queryVectorIndexAccelerationEmbeddingDimension,
|
|
4259
|
+
queryVectorIndexAccelerationIndexSignature,
|
|
4260
|
+
queryVectorIndexAccelerationRepresentationStatus,
|
|
4261
|
+
queryVectorIndexAccelerationRepresentationStatusReason,
|
|
4262
|
+
queryVectorIndexAccelerationRepresentationStrictMode,
|
|
4263
|
+
queryVectorIndexAccelerationCircuitState,
|
|
4264
|
+
queryVectorIndexAccelerationConsecutiveFailures,
|
|
4265
|
+
queryVectorIndexAccelerationRequestCount,
|
|
4266
|
+
queryVectorIndexAccelerationRetryCount,
|
|
4267
|
+
queryVectorIndexAccelerationShortCircuitCount,
|
|
4268
|
+
queryVectorIndexAccelerationSuccessCount,
|
|
4269
|
+
queryVectorIndexAccelerationFailureCount,
|
|
4270
|
+
queryVectorIndexAccelerationHalfOpenProbeCount,
|
|
4271
|
+
queryVectorIndexAccelerationHalfOpenSuccessRatePct,
|
|
4272
|
+
};
|
|
4273
|
+
const checksWithDebugTraceHints = checks.map((check) => {
|
|
4274
|
+
const hint = checkDebugTraceHints[check.checkId];
|
|
4275
|
+
const normalizedDebugHint = hint || check.debugTraceHint;
|
|
4276
|
+
const priorityScore = computeRuntimeCapabilityPriorityScore(check);
|
|
4277
|
+
const recommendedActions = buildRuntimeCapabilityRecommendedActions({
|
|
4278
|
+
...check,
|
|
4279
|
+
debugTraceHint: normalizedDebugHint,
|
|
4280
|
+
}, recommendedActionContext);
|
|
4281
|
+
return {
|
|
4282
|
+
...check,
|
|
4283
|
+
debugTraceHint: normalizedDebugHint,
|
|
4284
|
+
priorityScore,
|
|
4285
|
+
recommendedActions,
|
|
4286
|
+
};
|
|
4287
|
+
});
|
|
4288
|
+
const sortedChecks = sortRuntimeCapabilityChecksByPriority(checksWithDebugTraceHints);
|
|
4289
|
+
const failCount = sortedChecks.filter((check) => check.status === 'fail').length;
|
|
4290
|
+
const warnCount = sortedChecks.filter((check) => check.status === 'warn').length;
|
|
4291
|
+
const passCount = sortedChecks.filter((check) => check.status === 'pass').length;
|
|
4292
|
+
const topRiskCheck = sortedChecks.find((check) => check.status === 'fail' || check.status === 'warn') || null;
|
|
4293
|
+
const topRiskCheckId = String(topRiskCheck?.checkId || '').trim().toLowerCase();
|
|
4294
|
+
const topRiskStatus = topRiskCheck
|
|
4295
|
+
? topRiskCheck.status
|
|
4296
|
+
: 'none';
|
|
4297
|
+
const topRiskPriorityScore = Math.max(0, Math.floor(Number(topRiskCheck?.priorityScore || 0)));
|
|
4298
|
+
const topRiskRecommendedActions = Array.isArray(topRiskCheck?.recommendedActions)
|
|
4299
|
+
? topRiskCheck?.recommendedActions
|
|
4300
|
+
.map((item) => String(item || '').replace(/\s+/g, ' ').trim().slice(0, 220))
|
|
4301
|
+
.filter(Boolean)
|
|
4302
|
+
.slice(0, 5)
|
|
4303
|
+
: [];
|
|
4304
|
+
const overallStatus = failCount > 0
|
|
4305
|
+
? 'blocked'
|
|
4306
|
+
: (warnCount > 0 ? 'degraded' : 'ready');
|
|
4307
|
+
return {
|
|
4308
|
+
generatedAt: params.generatedAt,
|
|
4309
|
+
overallStatus,
|
|
4310
|
+
thresholds,
|
|
4311
|
+
checks: sortedChecks,
|
|
4312
|
+
summary: {
|
|
4313
|
+
passCount,
|
|
4314
|
+
warnCount,
|
|
4315
|
+
failCount,
|
|
4316
|
+
},
|
|
4317
|
+
signals: {
|
|
4318
|
+
configuredStoreBackend: params.configuredStoreBackend,
|
|
4319
|
+
configuredQueryBackend: params.configuredQueryBackend,
|
|
4320
|
+
storeType: params.store.storeType,
|
|
4321
|
+
storeUsingFallback: params.store.usingFallback === true,
|
|
4322
|
+
graphDbConnectorHealthStatus,
|
|
4323
|
+
graphDbConnectorHealthMessage,
|
|
4324
|
+
graphDbConnectorCircuitState,
|
|
4325
|
+
graphDbConnectorRequestCount,
|
|
4326
|
+
graphDbConnectorRetryCount,
|
|
4327
|
+
graphDbConnectorShortCircuitCount,
|
|
4328
|
+
graphDbConnectorSuccessCount,
|
|
4329
|
+
graphDbConnectorFailureCount,
|
|
4330
|
+
graphDbConnectorFailureRatioPct,
|
|
4331
|
+
graphDbConnectorConsecutiveFailures,
|
|
4332
|
+
graphDbConnectorShortCircuitRatioPct,
|
|
4333
|
+
graphDbConnectorWarnBudgetExceeded,
|
|
4334
|
+
graphDbConnectorFailBudgetExceeded,
|
|
4335
|
+
graphDbConnectorBudgetStatus,
|
|
4336
|
+
graphDbConnectorLastRequestId,
|
|
4337
|
+
graphDbConnectorLastErrorCode,
|
|
4338
|
+
graphDbConnectorLastStatusCode,
|
|
4339
|
+
graphDbConnectorLastRetryAfterMs,
|
|
4340
|
+
queryBackendId: params.queryDiagnostics.backendId,
|
|
4341
|
+
queryBackendRuntimeReady,
|
|
4342
|
+
queryBackendRuntimeId,
|
|
4343
|
+
queryVectorIndexEnabled,
|
|
4344
|
+
queryVectorIndexStatus,
|
|
4345
|
+
queryVectorIndexPersisted,
|
|
4346
|
+
queryVectorIndexLoadedFromDisk,
|
|
4347
|
+
queryVectorIndexAtomCount,
|
|
4348
|
+
queryVectorIndexLocation,
|
|
4349
|
+
queryVectorIndexAccelerationEnabled,
|
|
4350
|
+
queryVectorIndexAccelerationMode,
|
|
4351
|
+
queryVectorIndexAccelerationLastSelectionMode,
|
|
4352
|
+
queryVectorIndexAccelerationLastCandidateCount,
|
|
4353
|
+
queryVectorIndexAccelerationAdapterId,
|
|
4354
|
+
queryVectorIndexAccelerationAdapterError,
|
|
4355
|
+
queryVectorIndexAccelerationHealthStatus,
|
|
4356
|
+
queryVectorIndexAccelerationHealthMessage,
|
|
4357
|
+
queryVectorIndexAccelerationIndexSyncStatus,
|
|
4358
|
+
queryVectorIndexAccelerationIndexSyncMessage,
|
|
4359
|
+
queryVectorIndexAccelerationLastSyncAt,
|
|
4360
|
+
queryVectorIndexAccelerationSyncRequestCount,
|
|
4361
|
+
queryVectorIndexAccelerationSyncSuccessCount,
|
|
4362
|
+
queryVectorIndexAccelerationSyncFailureCount,
|
|
4363
|
+
queryVectorIndexAccelerationSyncedIndexSignature,
|
|
4364
|
+
queryVectorIndexAccelerationSyncedAtomCount,
|
|
4365
|
+
queryVectorIndexAccelerationRepresentationVersion,
|
|
4366
|
+
queryVectorIndexAccelerationEmbeddingModelId,
|
|
4367
|
+
queryVectorIndexAccelerationEmbeddingDimension,
|
|
4368
|
+
queryVectorIndexAccelerationIndexSignature,
|
|
4369
|
+
queryVectorIndexAccelerationRepresentationStatus,
|
|
4370
|
+
queryVectorIndexAccelerationRepresentationStatusReason,
|
|
4371
|
+
queryVectorIndexAccelerationRepresentationStrictMode,
|
|
4372
|
+
queryVectorIndexAccelerationLastRequestId,
|
|
4373
|
+
queryVectorIndexAccelerationLastErrorCode,
|
|
4374
|
+
queryVectorIndexAccelerationLastRetryAfterMs,
|
|
4375
|
+
queryVectorIndexAccelerationCircuitState,
|
|
4376
|
+
queryVectorIndexAccelerationConsecutiveFailures,
|
|
4377
|
+
queryVectorIndexAccelerationRequestCount,
|
|
4378
|
+
queryVectorIndexAccelerationRetryCount,
|
|
4379
|
+
queryVectorIndexAccelerationShortCircuitCount,
|
|
4380
|
+
queryVectorIndexAccelerationShortCircuitRatioPct,
|
|
4381
|
+
queryVectorIndexAccelerationSuccessCount,
|
|
4382
|
+
queryVectorIndexAccelerationFailureCount,
|
|
4383
|
+
queryVectorIndexAccelerationHalfOpenProbeCount,
|
|
4384
|
+
queryVectorIndexAccelerationHalfOpenSuccessRatePct,
|
|
4385
|
+
queryVectorIndexAccelerationCircuitWarnBudgetExceeded,
|
|
4386
|
+
queryVectorIndexAccelerationCircuitFailBudgetExceeded,
|
|
4387
|
+
queryVectorIndexAccelerationCircuitBudgetStatus,
|
|
4388
|
+
queryFallbackCount: safeFallbackCount,
|
|
4389
|
+
queryCount: safeQueryCount,
|
|
4390
|
+
queryFallbackRatioPct,
|
|
4391
|
+
queryExplainabilitySampleCount,
|
|
4392
|
+
queryEvidenceCoverageRatioPct,
|
|
4393
|
+
queryRelationPathCoverageRatioPct,
|
|
4394
|
+
queryTemporalValidityPassRatioPct,
|
|
4395
|
+
queryAverageEvidenceSpanCount,
|
|
4396
|
+
queryAverageRelationPathLength,
|
|
4397
|
+
queryBackendComparisonSampleCount,
|
|
4398
|
+
queryBackendComparisonEvidenceGapRatioPct,
|
|
4399
|
+
queryBackendComparisonRelationGapRatioPct,
|
|
4400
|
+
queryBackendComparisonTemporalGapRatioPct,
|
|
4401
|
+
queryBackendComparisonMaxExplainabilityGapRatioPct,
|
|
4402
|
+
queryBackendComparisonTrendStatus,
|
|
4403
|
+
queryBackendComparisonTrendScore,
|
|
4404
|
+
queryBackendComparisonTrendConfidence,
|
|
4405
|
+
queryBackendComparisonTrendLimit,
|
|
4406
|
+
queryBackendComparisonTrendWindowSize,
|
|
4407
|
+
queryBackendComparisonTrendMinSamples,
|
|
4408
|
+
queryBackendComparisonTrendRequiredRecords,
|
|
4409
|
+
apiTraceWindowRequests,
|
|
4410
|
+
apiTraceWindowErrors,
|
|
4411
|
+
apiTraceWindowInvalidRequests,
|
|
4412
|
+
apiTraceWindowInvalidRequestRatioPct,
|
|
4413
|
+
apiTraceWindowInvalidRequestToTotalRatioPct,
|
|
4414
|
+
apiTraceWindowServerErrors,
|
|
4415
|
+
apiTraceWindowServerErrorRatioPct,
|
|
4416
|
+
apiTraceWindowTransientErrors,
|
|
4417
|
+
apiTraceWindowTransientErrorRatioPct,
|
|
4418
|
+
apiTraceAverageDurationMs,
|
|
4419
|
+
apiTraceP95DurationMs,
|
|
4420
|
+
apiTraceScopePathPrefix,
|
|
4421
|
+
apiTraceScopeMethod,
|
|
4422
|
+
apiTraceWindowInvalidRequestTopPaths,
|
|
4423
|
+
apiTraceWindowServerErrorTopPaths,
|
|
4424
|
+
apiTraceWindowTransientErrorTopPaths,
|
|
4425
|
+
apiTraceSlowTopPaths,
|
|
4426
|
+
qualityTrendStatus: trendStatus,
|
|
4427
|
+
qualityTrendScore: trendScore,
|
|
4428
|
+
qualityTrendConfidence: trendConfidence,
|
|
4429
|
+
sessionPlanQualityRecords,
|
|
4430
|
+
sessionPlanQualityPassRatePct,
|
|
4431
|
+
sessionPlanQualityFailureStreak,
|
|
4432
|
+
sessionPlanQualityTrendStatus: sessionPlanTrendStatus,
|
|
4433
|
+
sessionPlanQualityTrendScore: sessionPlanTrendScore,
|
|
4434
|
+
sessionPlanQualityTrendConfidence: sessionPlanTrendConfidence,
|
|
4435
|
+
memoryPolicyStatus,
|
|
4436
|
+
memoryPolicyHealthScore,
|
|
4437
|
+
memoryPolicyTotalEntries,
|
|
4438
|
+
memoryPolicyExpiredEntries,
|
|
4439
|
+
memoryPolicyStaleEntries,
|
|
4440
|
+
memoryPolicyLowConfidenceEntries,
|
|
4441
|
+
memoryPolicyTrendStatus,
|
|
4442
|
+
memoryPolicyTrendScore,
|
|
4443
|
+
memoryPolicyTrendConfidence,
|
|
4444
|
+
knowledgeStalenessStaleDocuments,
|
|
4445
|
+
knowledgeStalenessFreshnessRatioPct,
|
|
4446
|
+
knowledgeStalenessHashMismatchDocuments,
|
|
4447
|
+
knowledgeStalenessMissingSourceDocuments,
|
|
4448
|
+
knowledgeStalenessReadErrorDocuments,
|
|
4449
|
+
sessionActionExecutionCount,
|
|
4450
|
+
sessionMemoryPersistedCount,
|
|
4451
|
+
sessionMemoryPromotionAppliedCount,
|
|
4452
|
+
sessionMemoryPromotionCount,
|
|
4453
|
+
sessionMemoryPromotionCoveragePct,
|
|
4454
|
+
sessionStrategyTotalRecords,
|
|
4455
|
+
sessionStrategyStrategyRecords,
|
|
4456
|
+
sessionStrategyTrendAutoSelectionSharePct,
|
|
4457
|
+
sessionStrategyTrendAutoAverageMasteryDeltaPct,
|
|
4458
|
+
sessionStrategyTrendAutoNegativeRatioPct,
|
|
4459
|
+
sessionStrategyModeFallbackSelectionSharePct,
|
|
4460
|
+
sessionStrategySelectionSourceExplicitCount,
|
|
4461
|
+
sessionStrategySelectionSourceTrendCount,
|
|
4462
|
+
sessionStrategySelectionSourceFallbackCount,
|
|
4463
|
+
sessionStrategySelectionSourceUnknownCount,
|
|
4464
|
+
sessionStrategyTopAverageStrategy,
|
|
4465
|
+
sessionStrategyTopAverageMasteryDeltaPct,
|
|
4466
|
+
sessionStrategyTopAverageNegativeRatioPct,
|
|
4467
|
+
tutorAdaptersTotal,
|
|
4468
|
+
tutorAdaptersActive,
|
|
4469
|
+
tutorRequests,
|
|
4470
|
+
tutorAcceptedResponses,
|
|
4471
|
+
tutorDowngradedResponses,
|
|
4472
|
+
tutorFailedResponses,
|
|
4473
|
+
tutorProviderFallbackResponses,
|
|
4474
|
+
tutorProviderFallbackRatioPct,
|
|
4475
|
+
tutorAverageProviderAttemptCount,
|
|
4476
|
+
tutorProviderCount,
|
|
4477
|
+
tutorDominantProviderName,
|
|
4478
|
+
tutorDominantProviderSharePct,
|
|
4479
|
+
tutorDominantFallbackProviderName,
|
|
4480
|
+
tutorDominantFallbackProviderSharePct,
|
|
4481
|
+
tutorProviderTrendRegressingCount,
|
|
4482
|
+
tutorProviderTrendImprovingCount,
|
|
4483
|
+
tutorProviderTrendInsufficientDataCount,
|
|
4484
|
+
tutorProviderTrendTopRegressingProvider: tutorProviderTrendTopRegressingProviderName,
|
|
4485
|
+
tutorProviderTrendTopRegressingScore,
|
|
4486
|
+
tutorProviderTrendTopRegressingConfidence,
|
|
4487
|
+
tutorProviderTrendRecommendedFocusProviderName,
|
|
4488
|
+
tutorProviderTrendHistoryRecords,
|
|
4489
|
+
tutorProviderTrendHistoryRegressingRecords,
|
|
4490
|
+
tutorProviderTrendHistoryStableRecords,
|
|
4491
|
+
tutorProviderTrendHistoryImprovingRecords,
|
|
4492
|
+
tutorProviderTrendHistoryInsufficientDataRecords,
|
|
4493
|
+
tutorFailedRatioPct,
|
|
4494
|
+
tutorDowngradedRatioPct,
|
|
4495
|
+
tutorAverageConfidence,
|
|
4496
|
+
tutorRoutingEnabled,
|
|
4497
|
+
tutorRoutingPreferredMode,
|
|
4498
|
+
tutorRoutingAdapterTimeoutMs,
|
|
4499
|
+
tutorLastRoutingStrategy,
|
|
4500
|
+
tutorLastRoutingReason,
|
|
4501
|
+
tutorLastRoutingScore,
|
|
4502
|
+
tutorRoutingDynamicPreferredMode,
|
|
4503
|
+
tutorRoutingDynamicModeReason,
|
|
4504
|
+
tutorRoutingDynamicModeSuggestionActive,
|
|
4505
|
+
topRiskCheckId: topRiskCheckId || '',
|
|
4506
|
+
topRiskStatus,
|
|
4507
|
+
topRiskPriorityScore,
|
|
4508
|
+
topRiskRecommendedActions,
|
|
4509
|
+
},
|
|
4510
|
+
};
|
|
4511
|
+
}
|
|
4512
|
+
function normalizeRuntimeCapabilityCheckIdToken(value) {
|
|
4513
|
+
return String(value || '')
|
|
4514
|
+
.trim()
|
|
4515
|
+
.toLowerCase()
|
|
4516
|
+
.replace(/[^a-z0-9_:-]+/g, '')
|
|
4517
|
+
.slice(0, 128);
|
|
4518
|
+
}
|
|
4519
|
+
function toRuntimeCapabilityRunbookCheck(check) {
|
|
4520
|
+
if (!check) {
|
|
4521
|
+
return null;
|
|
4522
|
+
}
|
|
4523
|
+
return {
|
|
4524
|
+
checkId: normalizeRuntimeCapabilityCheckIdToken(check.checkId),
|
|
4525
|
+
status: check.status,
|
|
4526
|
+
message: String(check.message || '').trim().slice(0, 280),
|
|
4527
|
+
observed: String(check.observed || '').trim().slice(0, 420),
|
|
4528
|
+
expected: String(check.expected || '').trim().slice(0, 220) || undefined,
|
|
4529
|
+
debugTraceHint: normalizeRuntimeCapabilityDebugTraceHint(check.debugTraceHint),
|
|
4530
|
+
priorityScore: Math.max(0, Math.floor(Number(check.priorityScore || 0))),
|
|
4531
|
+
recommendedActions: normalizeRuntimeCapabilityRecommendedActions(Array.isArray(check.recommendedActions) ? check.recommendedActions : []) || [],
|
|
4532
|
+
};
|
|
4533
|
+
}
|
|
4534
|
+
function resolveRuntimeCapabilityRunbookVerificationTargets(selectedCheck) {
|
|
4535
|
+
if (!selectedCheck) {
|
|
4536
|
+
return [
|
|
4537
|
+
'Re-fetch /api/knowledge/runtime-capability-matrix and confirm overallStatus remains ready.',
|
|
4538
|
+
];
|
|
4539
|
+
}
|
|
4540
|
+
const targets = [];
|
|
4541
|
+
if (selectedCheck.checkId === 'tutor_provider_trend_regression') {
|
|
4542
|
+
targets.push('Query /api/knowledge/tutor/trace-diagnostics/providers?source=llm-adapter and verify regressingProviders decreases.');
|
|
4543
|
+
targets.push('Query /api/knowledge/tutor/trace-diagnostics/providers/history?source=llm-adapter and compare the latest two windows for focus provider drift.');
|
|
4544
|
+
targets.push('Re-run tutor actions for the focus provider mode and confirm routing reason shows reduced trend penalty.');
|
|
4545
|
+
}
|
|
4546
|
+
if (selectedCheck.checkId === 'tutor_routing_dynamic_mode_alignment') {
|
|
4547
|
+
targets.push('Query /api/knowledge/session/orchestration/config and align preferredMode with dynamic recommendation (auto/local/cloud).');
|
|
4548
|
+
targets.push('Query /api/knowledge/tutor/trace-diagnostics/providers/history?source=llm-adapter and verify mode conflict trend is reduced in the latest window.');
|
|
4549
|
+
targets.push('Run tutor actions without explicit mode hints and confirm lastRouting dynamicPreferredMode conflict clears.');
|
|
4550
|
+
}
|
|
4551
|
+
if (selectedCheck.checkId === 'orchestration_path_strategy_alignment') {
|
|
4552
|
+
targets.push('Query /api/knowledge/session/history?userId=<userId>&pathStrategySelectionSource=strategy_trend&sinceMinutes=10080 and verify trend-driven strategy selections recover to non-negative mastery delta.');
|
|
4553
|
+
targets.push('Query /api/knowledge/quality/trend and confirm strategyBreakdown trend agrees with session history outcome telemetry.');
|
|
4554
|
+
targets.push('If trend-driven outcomes remain negative, tighten strategy auto-path confidence settings in /api/knowledge/session/orchestration/config.');
|
|
4555
|
+
}
|
|
4556
|
+
if (selectedCheck.checkId === 'query_backend_runtime_health') {
|
|
4557
|
+
targets.push('Query /api/knowledge/query-backend-diagnostics and verify diagnostics.runtime.ready is true with matching backend ids.');
|
|
4558
|
+
targets.push('If runtime readiness fails, switch query backend to local_hybrid via /api/knowledge/query-backend-config and confirm retrieval continuity.');
|
|
4559
|
+
}
|
|
4560
|
+
if (selectedCheck.checkId === 'store_graphdb_connector_health') {
|
|
4561
|
+
targets.push('Query /api/knowledge/store-diagnostics and verify store.connector.healthStatus/healthMessage/circuitState are stable.');
|
|
4562
|
+
targets.push('Run /api/knowledge/store/reload and confirm connector telemetry keeps circuitState=closed with controlled retry/shortCircuit counters.');
|
|
4563
|
+
targets.push('If connector remains degraded/unavailable, validate endpoint reachability and timeout/retry budgets before restoring strict rollout.');
|
|
4564
|
+
}
|
|
4565
|
+
if (selectedCheck.checkId === 'store_graphdb_connector_budget') {
|
|
4566
|
+
targets.push('Query /api/knowledge/store-diagnostics and validate connector request/failure/shortCircuit/consecutiveFailures counters against configured runtime thresholds.');
|
|
4567
|
+
targets.push('Run /api/knowledge/store/reload and replay representative graphdb-backed traffic, then verify failureRatio/shortCircuitRatio return below warn budgets.');
|
|
4568
|
+
targets.push('If budgets stay above thresholds, tune NOTE_CONNECTION_RUNTIME_STORE_GRAPHDB_CONNECTOR_* and NOTE_CONNECTION_KNOWLEDGE_GRAPHDB_HTTP_* circuit/retry env settings before widening strict rollout.');
|
|
4569
|
+
}
|
|
4570
|
+
if (selectedCheck.checkId === 'query_vector_index_status'
|
|
4571
|
+
|| selectedCheck.checkId === 'query_vector_index_persistence'
|
|
4572
|
+
|| selectedCheck.checkId === 'query_vector_acceleration_mode'
|
|
4573
|
+
|| selectedCheck.checkId === 'query_vector_acceleration_representation_consistency'
|
|
4574
|
+
|| selectedCheck.checkId === 'query_vector_acceleration_prefilter_effectiveness'
|
|
4575
|
+
|| selectedCheck.checkId === 'query_vector_acceleration_calibration_readiness'
|
|
4576
|
+
|| selectedCheck.checkId === 'query_vector_acceleration_health'
|
|
4577
|
+
|| selectedCheck.checkId === 'query_vector_acceleration_index_sync_health'
|
|
4578
|
+
|| selectedCheck.checkId === 'query_vector_acceleration_traceability'
|
|
4579
|
+
|| selectedCheck.checkId === 'query_vector_acceleration_circuit_state') {
|
|
4580
|
+
targets.push('Query /api/knowledge/query-backend-diagnostics and confirm diagnostics.runtime.vectorIndex.status=ready with persisted=true when local_vector is active.');
|
|
4581
|
+
targets.push('Issue /api/knowledge/query requests using local_vector and verify vectorIndex atomCount/signature are populated and stable.');
|
|
4582
|
+
targets.push('Verify diagnostics.runtime.vectorIndex.acceleration indicates ann_prefilter mode (or document intentional full_scan fallback).');
|
|
4583
|
+
if (selectedCheck.checkId === 'query_vector_acceleration_prefilter_effectiveness') {
|
|
4584
|
+
targets.push('Verify diagnostics.runtime.vectorIndex.acceleration.lastSelectionMode is token_prefilter|token_signature_prefilter with non-zero lastCandidateCount under representative ANN traffic.');
|
|
4585
|
+
}
|
|
4586
|
+
if (selectedCheck.checkId === 'query_vector_acceleration_calibration_readiness') {
|
|
4587
|
+
targets.push('Verify calibration prerequisites are all present in the same runtime window: sync telemetry ready, sampleReady=true, selectionActive=true, stableConnector=true, canEvaluateCandidateRatio=true, and correlation fields present for external connectors.');
|
|
4588
|
+
targets.push('Delay threshold tuning until diagnostics show circuitBudgetStatus=ok and prefilter budget status is clear for representative ANN traffic.');
|
|
4589
|
+
}
|
|
4590
|
+
if (selectedCheck.checkId === 'query_vector_acceleration_representation_consistency') {
|
|
4591
|
+
targets.push('Verify diagnostics.runtime.vectorIndex.acceleration representationStatus is aligned and representationVersion/embeddingModelId/embeddingDimension/indexSignature are populated for the active adapter.');
|
|
4592
|
+
targets.push('If representationStrictMode=true, verify mismatch incidents are blocked and surfaced with vector_acceleration_representation_mismatch diagnostics evidence.');
|
|
4593
|
+
}
|
|
4594
|
+
if (selectedCheck.checkId === 'query_vector_acceleration_index_sync_health'
|
|
4595
|
+
|| selectedCheck.checkId === 'query_vector_acceleration_health'
|
|
4596
|
+
|| selectedCheck.checkId === 'query_vector_acceleration_traceability'
|
|
4597
|
+
|| selectedCheck.checkId === 'query_vector_acceleration_circuit_state'
|
|
4598
|
+
|| selectedCheck.checkId === 'query_vector_acceleration_mode'
|
|
4599
|
+
|| selectedCheck.checkId === 'query_vector_index_status'
|
|
4600
|
+
|| selectedCheck.checkId === 'query_vector_index_persistence') {
|
|
4601
|
+
targets.push('For external connectors, verify diagnostics.runtime.vectorIndex.acceleration.indexSyncStatus/indexSyncMessage/lastSyncAt plus syncRequestCount/syncSuccessCount/syncFailureCount and syncedIndexSignature/syncedAtomCount after ingest-driven refreshes.');
|
|
4602
|
+
}
|
|
4603
|
+
if (selectedCheck.checkId === 'query_vector_acceleration_health'
|
|
4604
|
+
|| selectedCheck.checkId === 'query_vector_acceleration_traceability'
|
|
4605
|
+
|| selectedCheck.checkId === 'query_vector_acceleration_circuit_state'
|
|
4606
|
+
|| selectedCheck.checkId === 'query_vector_acceleration_mode'
|
|
4607
|
+
|| selectedCheck.checkId === 'query_vector_index_status'
|
|
4608
|
+
|| selectedCheck.checkId === 'query_vector_index_persistence') {
|
|
4609
|
+
targets.push('Verify diagnostics.runtime.vectorIndex.acceleration.healthStatus is ready|unknown (or capture degraded/unavailable reason with connector remediation evidence).');
|
|
4610
|
+
}
|
|
4611
|
+
if (selectedCheck.checkId === 'query_vector_acceleration_traceability'
|
|
4612
|
+
|| selectedCheck.checkId === 'query_vector_acceleration_circuit_state') {
|
|
4613
|
+
targets.push('Verify diagnostics.runtime.vectorIndex.acceleration.circuitState is closed|unknown and shortCircuitCount remains controlled; for external connectors also confirm lastRequestId/lastErrorCode/lastRetryAfterMs are populated after representative query traffic.');
|
|
4614
|
+
}
|
|
4615
|
+
}
|
|
4616
|
+
if (selectedCheck.expected) {
|
|
4617
|
+
targets.push(`Satisfy check expectation: ${selectedCheck.expected}`);
|
|
4618
|
+
}
|
|
4619
|
+
if (selectedCheck.debugTraceHint) {
|
|
4620
|
+
const pathPrefix = String(selectedCheck.debugTraceHint.pathPrefix || '').trim() || '/api/knowledge';
|
|
4621
|
+
const statusAtLeast = Math.max(0, Math.floor(Number(selectedCheck.debugTraceHint.statusAtLeast || 0)));
|
|
4622
|
+
const method = normalizeRuntimeCapabilityHttpMethod(selectedCheck.debugTraceHint.method || '');
|
|
4623
|
+
const errorCode = String(selectedCheck.debugTraceHint.errorCode || '').trim();
|
|
4624
|
+
targets.push(`Collect API evidence via /api/runtime-request-trace filter(pathPrefix=${pathPrefix},statusAtLeast=${statusAtLeast},method=${method || '<all>'},errorCode=${errorCode || '<none>'}).`);
|
|
4625
|
+
}
|
|
4626
|
+
if (selectedCheck.recommendedActions.length > 0) {
|
|
4627
|
+
targets.push(`Execute recommended actions and re-evaluate check ${selectedCheck.checkId}.`);
|
|
4628
|
+
}
|
|
4629
|
+
targets.push(`Re-fetch /api/knowledge/runtime-capability-matrix and verify ${selectedCheck.checkId} status moves to pass.`);
|
|
4630
|
+
return normalizeRuntimeCapabilityRecommendedActions(targets) || [];
|
|
4631
|
+
}
|
|
4632
|
+
function buildRuntimeCapabilityRunbook(matrix, requestedCheckIdRaw) {
|
|
4633
|
+
const requestedCheckId = normalizeRuntimeCapabilityCheckIdToken(requestedCheckIdRaw);
|
|
4634
|
+
const checks = Array.isArray(matrix?.checks)
|
|
4635
|
+
? matrix.checks
|
|
4636
|
+
: [];
|
|
4637
|
+
const checksById = new Map(checks.map((check) => [normalizeRuntimeCapabilityCheckIdToken(check?.checkId), check]));
|
|
4638
|
+
const topRiskSignalCheckId = normalizeRuntimeCapabilityCheckIdToken(matrix?.signals?.topRiskCheckId || '');
|
|
4639
|
+
const topRiskFromSignal = topRiskSignalCheckId
|
|
4640
|
+
? checksById.get(topRiskSignalCheckId)
|
|
4641
|
+
: null;
|
|
4642
|
+
const topRiskFromChecks = checks.find((check) => check.status === 'fail' || check.status === 'warn') || null;
|
|
4643
|
+
const topRiskCheck = topRiskFromSignal || topRiskFromChecks || null;
|
|
4644
|
+
const requestedCheck = requestedCheckId
|
|
4645
|
+
? (checksById.get(requestedCheckId) || null)
|
|
4646
|
+
: null;
|
|
4647
|
+
const selectedCheck = requestedCheck || topRiskCheck || null;
|
|
4648
|
+
const selectionSource = requestedCheck
|
|
4649
|
+
? 'requested'
|
|
4650
|
+
: (requestedCheckId
|
|
4651
|
+
? (selectedCheck ? 'top_risk_fallback' : 'none')
|
|
4652
|
+
: (selectedCheck ? 'top_risk' : 'none'));
|
|
4653
|
+
const selectedRunbookCheck = toRuntimeCapabilityRunbookCheck(selectedCheck);
|
|
4654
|
+
const topRiskRunbookCheck = toRuntimeCapabilityRunbookCheck(topRiskCheck);
|
|
4655
|
+
const traceFilter = normalizeRuntimeCapabilityDebugTraceHint(selectedRunbookCheck?.debugTraceHint) || {
|
|
4656
|
+
pathPrefix: '/api/knowledge',
|
|
4657
|
+
statusAtLeast: selectedRunbookCheck ? 400 : 0,
|
|
4658
|
+
method: '',
|
|
4659
|
+
errorCode: '',
|
|
4660
|
+
};
|
|
4661
|
+
const verificationTargets = resolveRuntimeCapabilityRunbookVerificationTargets(selectedRunbookCheck);
|
|
4662
|
+
return {
|
|
4663
|
+
generatedAt: String(matrix?.generatedAt || new Date().toISOString()),
|
|
4664
|
+
overallStatus: matrix?.overallStatus || 'degraded',
|
|
4665
|
+
summary: {
|
|
4666
|
+
passCount: Math.max(0, Math.floor(Number(matrix?.summary?.passCount || 0))),
|
|
4667
|
+
warnCount: Math.max(0, Math.floor(Number(matrix?.summary?.warnCount || 0))),
|
|
4668
|
+
failCount: Math.max(0, Math.floor(Number(matrix?.summary?.failCount || 0))),
|
|
4669
|
+
},
|
|
4670
|
+
requestedCheckId,
|
|
4671
|
+
selectionSource,
|
|
4672
|
+
selectedCheck: selectedRunbookCheck,
|
|
4673
|
+
topRiskCheck: topRiskRunbookCheck,
|
|
4674
|
+
traceFilter,
|
|
4675
|
+
verificationTargets,
|
|
4676
|
+
};
|
|
4677
|
+
}
|