claude-smart 0.2.49 → 0.2.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/README.md +10 -43
- package/bin/claude-smart.js +105 -0
- package/package.json +3 -2
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/.codex-plugin/plugin.json +1 -1
- package/plugin/.coverage +0 -0
- package/plugin/README.md +4 -3
- package/plugin/dashboard/app/api/reflexio/[...path]/route.ts +4 -2
- package/plugin/dashboard/app/dashboard/page.tsx +6 -1
- package/plugin/dashboard/app/preferences/[id]/page.tsx +18 -6
- package/plugin/dashboard/app/preferences/page.tsx +32 -35
- package/plugin/dashboard/app/sessions/[sessionId]/page.tsx +16 -1
- package/plugin/dashboard/app/sessions/page.tsx +2 -0
- package/plugin/dashboard/app/skills/page.tsx +65 -50
- package/plugin/dashboard/app/skills/project/[id]/page.tsx +17 -8
- package/plugin/dashboard/app/skills/shared/[id]/page.tsx +1 -6
- package/plugin/dashboard/components/common/host-badge.tsx +118 -0
- package/plugin/dashboard/components/common/learning-application-badge.tsx +34 -0
- package/plugin/dashboard/components/common/learnings-badge.tsx +1 -1
- package/plugin/dashboard/components/common/page-header.tsx +3 -3
- package/plugin/dashboard/lib/config-file.ts +5 -1
- package/plugin/dashboard/lib/host-attribution.ts +62 -0
- package/plugin/dashboard/lib/session-reader.ts +40 -2
- package/plugin/dashboard/lib/types.ts +7 -1
- package/plugin/pyproject.toml +1 -1
- package/plugin/scripts/_lib.sh +197 -1
- package/plugin/scripts/backend-python-runner.py +46 -0
- package/plugin/scripts/backend-service.sh +757 -119
- package/plugin/scripts/codex-hook.js +63 -225
- package/plugin/scripts/dashboard-open.sh +6 -4
- package/plugin/scripts/dashboard-service.sh +117 -136
- package/plugin/scripts/hook_entry.sh +3 -0
- package/plugin/scripts/smart-install.sh +15 -1
- package/plugin/src/claude_smart/cli.py +14 -0
- package/plugin/src/claude_smart/context_inject.py +3 -0
- package/plugin/src/claude_smart/env_config.py +4 -1
- package/plugin/src/claude_smart/events/post_tool.py +2 -1
- package/plugin/src/claude_smart/events/session_end.py +2 -1
- package/plugin/src/claude_smart/events/stop.py +3 -0
- package/plugin/src/claude_smart/events/user_prompt.py +2 -1
- package/plugin/src/claude_smart/internal_call.py +5 -2
- package/plugin/src/claude_smart/optimizer_assistant.py +59 -13
- package/plugin/src/claude_smart/publish.py +59 -7
- package/plugin/src/claude_smart/reflexio_adapter.py +137 -14
- package/plugin/src/claude_smart/runtime.py +15 -6
- package/plugin/src/claude_smart/state.py +211 -52
- package/plugin/uv.lock +1 -1
- package/plugin/vendor/reflexio/.env.example +13 -0
- package/plugin/vendor/reflexio/reflexio/README.md +7 -3
- package/plugin/vendor/reflexio/reflexio/__init__.py +12 -0
- package/plugin/vendor/reflexio/reflexio/client/client.py +126 -3
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/state.py +10 -3
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_state.py +28 -0
- package/plugin/vendor/reflexio/reflexio/lib/_search.py +41 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/entities.py +177 -25
- package/plugin/vendor/reflexio/reflexio/models/api_schema/eval_overview_schema.py +7 -1
- package/plugin/vendor/reflexio/reflexio/models/api_schema/internal_schema.py +2 -1
- package/plugin/vendor/reflexio/reflexio/models/api_schema/retriever_schema.py +63 -4
- package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/converters.py +1 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/entities.py +8 -1
- package/plugin/vendor/reflexio/reflexio/models/config_schema.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/README.md +22 -4
- package/plugin/vendor/reflexio/reflexio/server/__init__.py +18 -8
- package/plugin/vendor/reflexio/reflexio/server/__main__.py +6 -0
- package/plugin/vendor/reflexio/reflexio/server/api.py +66 -3
- package/plugin/vendor/reflexio/reflexio/server/billing_meter.py +263 -3
- package/plugin/vendor/reflexio/reflexio/server/callback_executor.py +164 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_embedding.py +81 -81
- package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_subprocess.py +28 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_text_generation.py +62 -5
- package/plugin/vendor/reflexio/reflexio/server/llm/embedding_service.py +19 -52
- package/plugin/vendor/reflexio/reflexio/server/llm/model_defaults.py +28 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/claude_code_provider.py +9 -1
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/embedder_warmup.py +329 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/embedding_service_provider.py +85 -10
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/local_embedding_provider.py +20 -5
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/nomic_embedding_provider.py +77 -9
- package/plugin/vendor/reflexio/reflexio/server/org_fanout.py +184 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/document_expansion/v1.0.0.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/document_expansion/v1.1.0.prompt.md +32 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.3.3.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.4.0.prompt.md +63 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/query_reformulation/v1.0.0.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/query_reformulation/v2.0.0.prompt.md +30 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/retrieved_learning_impact/v1.0.0.prompt.md +51 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/retrieved_learning_relevance/v1.0.0.prompt.md +39 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/shadow_comparison/v1.0.0.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/shadow_comparison/v1.1.0.prompt.md +43 -0
- package/plugin/vendor/reflexio/reflexio/server/routes/config.py +3 -3
- package/plugin/vendor/reflexio/reflexio/server/routes/evaluation.py +122 -28
- package/plugin/vendor/reflexio/reflexio/server/routes/system.py +22 -3
- package/plugin/vendor/reflexio/reflexio/server/scheduling.py +64 -3
- package/plugin/vendor/reflexio/reflexio/server/services/README.md +6 -4
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/README.md +3 -2
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/_eval_health.py +41 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/components/retrieved_learning_evaluator.py +554 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/regen_jobs.py +35 -1
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/runner.py +253 -101
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/scheduler.py +5 -7
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/service.py +27 -0
- package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_extraction_lifecycle.py +11 -5
- package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_should_run.py +4 -4
- package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_usage_billing.py +6 -2
- package/plugin/vendor/reflexio/reflexio/server/services/base_generation_service.py +255 -74
- package/plugin/vendor/reflexio/reflexio/server/services/deduplication_utils.py +72 -0
- package/plugin/vendor/reflexio/reflexio/server/services/deferred_learning_plan.py +270 -0
- package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/worker.py +85 -16
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/components/hero_state.py +2 -11
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/components/rule_attribution.py +6 -2
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/service.py +17 -13
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/agent_run_records.py +18 -3
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/outcome.py +12 -1
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/prior_answer_search.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/resumable_agent.py +2 -1
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_worker.py +66 -0
- package/plugin/vendor/reflexio/reflexio/server/services/generation_service.py +510 -63
- package/plugin/vendor/reflexio/reflexio/server/services/governance/service.py +2 -0
- package/plugin/vendor/reflexio/reflexio/server/services/lineage/gc_scheduler.py +179 -99
- package/plugin/vendor/reflexio/reflexio/server/services/lineage/vector_backfill_sweep.py +139 -0
- package/plugin/vendor/reflexio/reflexio/server/services/operation_state_utils.py +66 -27
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/aggregation_trigger.py +177 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/aggregator.py +68 -2
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/consolidator.py +360 -49
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/extractor.py +27 -30
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/playbook_edit_apply.py +8 -1
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/service.py +137 -69
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/optimizer.py +20 -1
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/scheduler.py +13 -6
- package/plugin/vendor/reflexio/reflexio/server/services/pre_retrieval/_query_reformulator.py +40 -25
- package/plugin/vendor/reflexio/reflexio/server/services/profile/components/consolidator.py +12 -39
- package/plugin/vendor/reflexio/reflexio/server/services/profile/components/extractor.py +30 -35
- package/plugin/vendor/reflexio/reflexio/server/services/profile/service.py +122 -44
- package/plugin/vendor/reflexio/reflexio/server/services/reflection/service.py +457 -107
- package/plugin/vendor/reflexio/reflexio/server/services/retrieval/session_dedup.py +127 -0
- package/plugin/vendor/reflexio/reflexio/server/services/retrieval/temporal.py +104 -0
- package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/dispatcher.py +139 -0
- package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/judge.py +16 -6
- package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/worker.py +137 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/governance_validation.py +12 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/lifecycle_filters.py +54 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/retention.py +41 -3
- package/plugin/vendor/reflexio/reflexio/server/services/storage/retention_mixin.py +40 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/__init__.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_base.py +50 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_extras.py +11 -4
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_governance.py +28 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py +36 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_shadow_verdicts.py +4 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/base/_deletion.py +16 -5
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_erase_execution.py +104 -42
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_subject_barrier.py +7 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_agent.py +43 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_eval_results.py +430 -5
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_user.py +45 -7
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_interaction_store.py +146 -6
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_profile_store.py +42 -7
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/__init__.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_extras.py +9 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_learning_jobs.py +50 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_operations.py +7 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_retrieval_log.py +3 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_shadow_verdicts.py +4 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/agent_run/_models.py +12 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/evaluation_state_keys.py +78 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_agent.py +38 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_eval_results.py +171 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_user.py +52 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/_interaction_store.py +59 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/_profile_store.py +52 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/retrieved_learning_state.py +226 -0
- package/plugin/vendor/reflexio/reflexio/server/services/tagging/tagging_scheduler.py +5 -6
- package/plugin/vendor/reflexio/reflexio/server/services/unified_search_service.py +209 -29
- package/plugin/vendor/reflexio/reflexio/server/usage_metrics.py +3 -0
- package/plugin/vendor/reflexio/reflexio/test_support/llm_mock.py +61 -0
- package/plugin/vendor/reflexio/reflexio/test_support/llm_model_registry.py +33 -0
- package/plugin/vendor/reflexio/reflexio/server/services/search/__init__.py +0 -0
|
@@ -29,6 +29,8 @@ from reflexio.models.api_schema.eval_overview_schema import (
|
|
|
29
29
|
from reflexio.models.api_schema.retriever_schema import (
|
|
30
30
|
GetAgentSuccessEvaluationResultsRequest,
|
|
31
31
|
GetEvaluationResultsViewResponse,
|
|
32
|
+
GetRetrievedLearningEvaluationResultsRequest,
|
|
33
|
+
GetRetrievedLearningEvaluationResultsResponse,
|
|
32
34
|
)
|
|
33
35
|
from reflexio.models.api_schema.ui.converters import (
|
|
34
36
|
to_evaluation_result_view,
|
|
@@ -48,6 +50,12 @@ from reflexio.server.services.agent_success_evaluation.regen_jobs import (
|
|
|
48
50
|
from reflexio.server.services.agent_success_evaluation.runner import (
|
|
49
51
|
run_group_evaluation,
|
|
50
52
|
)
|
|
53
|
+
from reflexio.server.services.storage.storage_base.evaluation_state_keys import (
|
|
54
|
+
build_grade_on_demand_cache_key,
|
|
55
|
+
)
|
|
56
|
+
from reflexio.server.services.storage.storage_base.retrieved_learning_state import (
|
|
57
|
+
session_fingerprint,
|
|
58
|
+
)
|
|
51
59
|
|
|
52
60
|
logger = logging.getLogger(__name__)
|
|
53
61
|
router = APIRouter()
|
|
@@ -90,28 +98,35 @@ def _grade_on_demand_cache_key(
|
|
|
90
98
|
# injective: distinct component tuples can never collapse to the same key even
|
|
91
99
|
# when a component itself contains the ``::`` delimiter. Keeps the prefix intact
|
|
92
100
|
# for prefix-based filtering and the key human-readable for inspection.
|
|
93
|
-
|
|
94
|
-
|
|
101
|
+
# Delegates to the shared builder so governance erasure scrubs the exact
|
|
102
|
+
# same keys this route writes.
|
|
103
|
+
return build_grade_on_demand_cache_key(
|
|
104
|
+
org_id, session_id, agent_version, evaluation_name
|
|
95
105
|
)
|
|
96
|
-
return f"{_GRADE_ON_DEMAND_CACHE_KEY_PREFIX}::{parts}"
|
|
97
106
|
|
|
98
107
|
|
|
99
108
|
def _read_grade_on_demand_cache(
|
|
100
|
-
storage: Any, cache_key: str, *, now: int
|
|
101
|
-
) -> int | None:
|
|
102
|
-
"""Return the cached ``result_id`` if
|
|
109
|
+
storage: Any, cache_key: str, *, now: int, user_id: str, session_id: str
|
|
110
|
+
) -> tuple[int | None, str | None] | None:
|
|
111
|
+
"""Return the cached ``(result_id, retrieved_learning_status)`` if valid.
|
|
103
112
|
|
|
104
|
-
Returns None on
|
|
105
|
-
|
|
106
|
-
|
|
113
|
+
Returns None on: no entry, malformed entry, entry older than the 24h TTL,
|
|
114
|
+
or — for entries carrying retrieved-learning fields — a session whose
|
|
115
|
+
retrieved-learning state no longer matches the cached fingerprint (a later
|
|
116
|
+
publish/delete changed the session, so cached completion must not be
|
|
117
|
+
claimed). Legacy entries without retrieved fields stay valid and report
|
|
118
|
+
``retrieved_learning_status=None``.
|
|
107
119
|
|
|
108
120
|
Args:
|
|
109
121
|
storage: The request's storage backend.
|
|
110
122
|
cache_key (str): Key produced by ``_grade_on_demand_cache_key``.
|
|
111
123
|
now (int): Current Unix-seconds wall-clock timestamp.
|
|
124
|
+
user_id (str): Session owner, for retrieved-state revalidation.
|
|
125
|
+
session_id (str): The cached session, for retrieved-state revalidation.
|
|
112
126
|
|
|
113
127
|
Returns:
|
|
114
|
-
|
|
128
|
+
tuple | None: ``(result_id, retrieved_learning_status)`` when fresh,
|
|
129
|
+
else None.
|
|
115
130
|
"""
|
|
116
131
|
cached_state = storage.get_operation_state(cache_key)
|
|
117
132
|
if not cached_state:
|
|
@@ -125,7 +140,27 @@ def _read_grade_on_demand_cache(
|
|
|
125
140
|
if (now - last_graded_at) >= _GRADE_ON_DEMAND_CACHE_TTL_SECONDS:
|
|
126
141
|
return None
|
|
127
142
|
cached_result_id = state.get("result_id")
|
|
128
|
-
|
|
143
|
+
if not isinstance(cached_result_id, int):
|
|
144
|
+
return None
|
|
145
|
+
cached_fingerprint = state.get("retrieved_learning_fingerprint")
|
|
146
|
+
cached_retrieved_status = state.get("retrieved_learning_status")
|
|
147
|
+
if isinstance(cached_fingerprint, str) and cached_fingerprint:
|
|
148
|
+
# Recompute the CURRENT session fingerprint — a later publish/delete
|
|
149
|
+
# changes it without touching any state row — and require the
|
|
150
|
+
# persisted terminal state to still match it and the cached status.
|
|
151
|
+
current_fingerprint = session_fingerprint(
|
|
152
|
+
storage.load_bounded_retrieved_learning_snapshot(user_id, session_id)
|
|
153
|
+
)
|
|
154
|
+
if current_fingerprint != cached_fingerprint:
|
|
155
|
+
return None
|
|
156
|
+
current = storage.get_matching_retrieved_learning_terminal_state(
|
|
157
|
+
user_id, session_id, current_fingerprint
|
|
158
|
+
)
|
|
159
|
+
if current is None or current.get("status") != cached_retrieved_status:
|
|
160
|
+
return None
|
|
161
|
+
return cached_result_id, (
|
|
162
|
+
cached_retrieved_status if isinstance(cached_retrieved_status, str) else None
|
|
163
|
+
)
|
|
129
164
|
|
|
130
165
|
|
|
131
166
|
def _resolve_session_user_id(storage: Any, session_id: str) -> str | None:
|
|
@@ -157,6 +192,7 @@ def _find_fresh_result_id(
|
|
|
157
192
|
agent_version: str,
|
|
158
193
|
evaluation_name: str,
|
|
159
194
|
previous_result_ids: set[int],
|
|
195
|
+
accept_existing: bool,
|
|
160
196
|
) -> int | None:
|
|
161
197
|
"""Locate the result_id written by the most-recent grade for this session.
|
|
162
198
|
|
|
@@ -170,6 +206,8 @@ def _find_fresh_result_id(
|
|
|
170
206
|
agent_version (str): The version dimension.
|
|
171
207
|
evaluation_name (str): Evaluator/result namespace to isolate readback.
|
|
172
208
|
previous_result_ids (set[int]): Matching rows observed before grading.
|
|
209
|
+
accept_existing (bool): Whether a completed grade may have updated an
|
|
210
|
+
existing row in place, as the Supabase backend does.
|
|
173
211
|
|
|
174
212
|
Returns:
|
|
175
213
|
int | None: result_id of the latest matching row, or None if the
|
|
@@ -182,9 +220,11 @@ def _find_fresh_result_id(
|
|
|
182
220
|
agent_version=agent_version,
|
|
183
221
|
)
|
|
184
222
|
fresh_result_ids = [rid for rid in result_ids if rid not in previous_result_ids]
|
|
185
|
-
if
|
|
186
|
-
return
|
|
187
|
-
|
|
223
|
+
if fresh_result_ids:
|
|
224
|
+
return max(fresh_result_ids)
|
|
225
|
+
if accept_existing and result_ids:
|
|
226
|
+
return max(result_ids)
|
|
227
|
+
return None
|
|
188
228
|
|
|
189
229
|
|
|
190
230
|
@router.post(
|
|
@@ -217,6 +257,28 @@ def get_agent_success_evaluation_results(
|
|
|
217
257
|
)
|
|
218
258
|
|
|
219
259
|
|
|
260
|
+
@router.post(
|
|
261
|
+
"/api/get_retrieved_learning_evaluation_results",
|
|
262
|
+
response_model=GetRetrievedLearningEvaluationResultsResponse,
|
|
263
|
+
)
|
|
264
|
+
def get_retrieved_learning_evaluation_results(
|
|
265
|
+
request: GetRetrievedLearningEvaluationResultsRequest,
|
|
266
|
+
org_id: str = Depends(default_get_org_id),
|
|
267
|
+
) -> GetRetrievedLearningEvaluationResultsResponse:
|
|
268
|
+
"""Get per-learning retrieved-learning evaluation verdicts.
|
|
269
|
+
|
|
270
|
+
Args:
|
|
271
|
+
request (GetRetrievedLearningEvaluationResultsRequest): Optional
|
|
272
|
+
user/session filters + limit.
|
|
273
|
+
org_id (str): Organization ID.
|
|
274
|
+
|
|
275
|
+
Returns:
|
|
276
|
+
GetRetrievedLearningEvaluationResultsResponse: Matching verdicts.
|
|
277
|
+
"""
|
|
278
|
+
reflexio = reflexio_cache.get_reflexio(org_id=org_id)
|
|
279
|
+
return reflexio.get_retrieved_learning_evaluation_results(request)
|
|
280
|
+
|
|
281
|
+
|
|
220
282
|
@router.post(
|
|
221
283
|
"/api/get_evaluation_overview",
|
|
222
284
|
response_model=GetEvaluationOverviewResponse,
|
|
@@ -417,15 +479,8 @@ def grade_on_demand(
|
|
|
417
479
|
)
|
|
418
480
|
now = int(datetime.now(UTC).timestamp())
|
|
419
481
|
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
return GradeOnDemandResponse(
|
|
423
|
-
session_id=payload.session_id,
|
|
424
|
-
result_id=cached_result_id,
|
|
425
|
-
cached=True,
|
|
426
|
-
skipped_reason=None,
|
|
427
|
-
)
|
|
428
|
-
|
|
482
|
+
# User resolution happens before the cache read: revalidating a cached
|
|
483
|
+
# retrieved-learning status needs the session owner's id.
|
|
429
484
|
user_id = _resolve_session_user_id(storage, payload.session_id)
|
|
430
485
|
if user_id is None:
|
|
431
486
|
return GradeOnDemandResponse(
|
|
@@ -435,6 +490,19 @@ def grade_on_demand(
|
|
|
435
490
|
skipped_reason="NO_REQUESTS",
|
|
436
491
|
)
|
|
437
492
|
|
|
493
|
+
cached = _read_grade_on_demand_cache(
|
|
494
|
+
storage, cache_key, now=now, user_id=user_id, session_id=payload.session_id
|
|
495
|
+
)
|
|
496
|
+
if cached is not None:
|
|
497
|
+
cached_result_id, cached_retrieved_status = cached
|
|
498
|
+
return GradeOnDemandResponse(
|
|
499
|
+
session_id=payload.session_id,
|
|
500
|
+
result_id=cached_result_id,
|
|
501
|
+
cached=True,
|
|
502
|
+
skipped_reason=None,
|
|
503
|
+
retrieved_learning_status=cached_retrieved_status,
|
|
504
|
+
)
|
|
505
|
+
|
|
438
506
|
previous_result_ids = set(
|
|
439
507
|
storage.get_agent_success_evaluation_result_ids(
|
|
440
508
|
user_id=user_id,
|
|
@@ -455,7 +523,7 @@ def grade_on_demand(
|
|
|
455
523
|
# The cache key namespaces are distinct so the two markers do not
|
|
456
524
|
# interfere; the explicit force_regenerate=True here is what makes
|
|
457
525
|
# an on-demand grade always do real work on a cache miss.
|
|
458
|
-
run_group_evaluation(
|
|
526
|
+
outcome = run_group_evaluation(
|
|
459
527
|
org_id=org_id,
|
|
460
528
|
user_id=user_id,
|
|
461
529
|
session_id=payload.session_id,
|
|
@@ -473,18 +541,44 @@ def grade_on_demand(
|
|
|
473
541
|
agent_version=payload.agent_version,
|
|
474
542
|
evaluation_name=evaluation_name,
|
|
475
543
|
previous_result_ids=previous_result_ids,
|
|
544
|
+
accept_existing=outcome.agent_success_status == "complete",
|
|
476
545
|
)
|
|
477
546
|
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
)
|
|
547
|
+
# Cache only settled outcomes. A terminal retrieved status is cached with
|
|
548
|
+
# its fingerprint ONLY after confirming the persisted state still
|
|
549
|
+
# linearizes at that fingerprint (a concurrent publish would have changed
|
|
550
|
+
# it); the cache read revalidates again, so a lost race here can never
|
|
551
|
+
# serve stale completion. Degraded/failed/pending/stale/superseded
|
|
552
|
+
# retrieved outcomes are not cached at all — the user's next explicit
|
|
553
|
+
# click must retry rather than short-circuit.
|
|
554
|
+
retrieved_status = outcome.retrieved_learning_status
|
|
555
|
+
if retrieved_status in ("complete", "not_applicable") and (
|
|
556
|
+
outcome.retrieved_learning_fingerprint
|
|
557
|
+
):
|
|
558
|
+
confirmed = storage.get_matching_retrieved_learning_terminal_state(
|
|
559
|
+
user_id, payload.session_id, outcome.retrieved_learning_fingerprint
|
|
560
|
+
)
|
|
561
|
+
cache_entry: dict[str, Any] = {"last_graded_at": now, "result_id": result_id}
|
|
562
|
+
if confirmed is not None and confirmed.get("status") == retrieved_status:
|
|
563
|
+
cache_entry["retrieved_learning_status"] = retrieved_status
|
|
564
|
+
cache_entry["retrieved_learning_fingerprint"] = (
|
|
565
|
+
outcome.retrieved_learning_fingerprint
|
|
566
|
+
)
|
|
567
|
+
storage.upsert_operation_state(cache_key, cache_entry)
|
|
568
|
+
elif retrieved_status == "skipped":
|
|
569
|
+
# Retrieved evaluation not applicable to this deployment/session
|
|
570
|
+
# shape; keep the legacy agent-success-only cache behavior.
|
|
571
|
+
storage.upsert_operation_state(
|
|
572
|
+
cache_key,
|
|
573
|
+
{"last_graded_at": now, "result_id": result_id},
|
|
574
|
+
)
|
|
482
575
|
|
|
483
576
|
return GradeOnDemandResponse(
|
|
484
577
|
session_id=payload.session_id,
|
|
485
578
|
result_id=result_id,
|
|
486
579
|
cached=False,
|
|
487
580
|
skipped_reason=None,
|
|
581
|
+
retrieved_learning_status=retrieved_status,
|
|
488
582
|
)
|
|
489
583
|
|
|
490
584
|
|
|
@@ -13,6 +13,7 @@ from fastapi import (
|
|
|
13
13
|
Request,
|
|
14
14
|
status,
|
|
15
15
|
)
|
|
16
|
+
from fastapi.responses import JSONResponse
|
|
16
17
|
|
|
17
18
|
from reflexio.models.api_schema.retriever_schema import (
|
|
18
19
|
GetDashboardStatsRequest,
|
|
@@ -55,9 +56,27 @@ def root() -> dict[str, str]:
|
|
|
55
56
|
}
|
|
56
57
|
|
|
57
58
|
|
|
58
|
-
@router.get("/health")
|
|
59
|
-
async def health_check() -> dict[str, str]:
|
|
60
|
-
"""Health check endpoint for ECS/container orchestration.
|
|
59
|
+
@router.get("/health", response_model=None)
|
|
60
|
+
async def health_check() -> dict[str, str] | JSONResponse:
|
|
61
|
+
"""Health check endpoint for ECS/container orchestration.
|
|
62
|
+
|
|
63
|
+
When the in-process-embedder warm-before-ready gate is active (a future
|
|
64
|
+
``REFLEXIO_EMBEDDING_PROVIDER=inprocess`` + ``local/*`` deployment) and the
|
|
65
|
+
embedder has not finished loading, return HTTP 503 so the load balancer
|
|
66
|
+
holds traffic until the model is warm. In every other configuration — the
|
|
67
|
+
current daemon-mode prod state included — the gate is inactive and this
|
|
68
|
+
returns exactly the historical 200 body.
|
|
69
|
+
"""
|
|
70
|
+
from reflexio.server.llm.providers.embedder_warmup import (
|
|
71
|
+
inprocess_local_gate_active,
|
|
72
|
+
is_embedder_ready,
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
if inprocess_local_gate_active() and not is_embedder_ready():
|
|
76
|
+
return JSONResponse(
|
|
77
|
+
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
|
78
|
+
content={"status": "starting"},
|
|
79
|
+
)
|
|
61
80
|
return {"status": "healthy"}
|
|
62
81
|
|
|
63
82
|
|
|
@@ -19,6 +19,9 @@ set of hooks so no observable behaviour changes:
|
|
|
19
19
|
``start`` is a logged no-op when a feature gate is off).
|
|
20
20
|
- :meth:`_on_started` / :meth:`_on_stopped` emit each scheduler's own start/stop
|
|
21
21
|
log line (through its own module logger, so output is byte-identical).
|
|
22
|
+
- ``leader_gate`` (optional): a fleet-coordination gate consulted before each
|
|
23
|
+
tick; ``None`` (the default and the OSS-local case) preserves today's
|
|
24
|
+
always-tick behavior byte-for-byte.
|
|
22
25
|
|
|
23
26
|
A scheduler whose loop is materially different (e.g. a bounded-attempt retrier
|
|
24
27
|
that waits *before* each attempt and exits on first success) may override
|
|
@@ -31,7 +34,31 @@ nothing from ``reflexio_ext``.
|
|
|
31
34
|
|
|
32
35
|
from __future__ import annotations
|
|
33
36
|
|
|
37
|
+
import logging
|
|
34
38
|
import threading
|
|
39
|
+
from typing import Protocol
|
|
40
|
+
|
|
41
|
+
logger = logging.getLogger(__name__)
|
|
42
|
+
|
|
43
|
+
# How long a non-leader waits before re-checking the gate. Fixed (no interval
|
|
44
|
+
# memoization): a follower that becomes leader starts ticking within <=60s.
|
|
45
|
+
_FOLLOWER_POLL_SECONDS: float = 60.0
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class LeaderGate(Protocol):
|
|
49
|
+
"""Fleet-coordination gate consulted before each tick.
|
|
50
|
+
|
|
51
|
+
Implementations must NEVER raise from ``should_run`` — error handling
|
|
52
|
+
(including fail-open) lives inside the implementation (spec §4.1). The
|
|
53
|
+
scheduler base nonetheless defends against a contract violation: if
|
|
54
|
+
``should_run`` raises anyway, the base logs the error and fails open
|
|
55
|
+
(runs the tick) rather than trusting the contract blindly and letting the
|
|
56
|
+
daemon thread die silently.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
def should_run(self) -> bool:
|
|
60
|
+
"""Return whether this instance should run the next tick."""
|
|
61
|
+
...
|
|
35
62
|
|
|
36
63
|
|
|
37
64
|
class ThreadedScheduler:
|
|
@@ -44,12 +71,17 @@ class ThreadedScheduler:
|
|
|
44
71
|
|
|
45
72
|
Args:
|
|
46
73
|
thread_name (str): OS thread name for the daemon (aids debugging / logs).
|
|
74
|
+
leader_gate (LeaderGate | None): Optional fleet-coordination gate
|
|
75
|
+
consulted before each tick. Defaults to None (always tick).
|
|
47
76
|
"""
|
|
48
77
|
|
|
49
|
-
def __init__(
|
|
78
|
+
def __init__(
|
|
79
|
+
self, *, thread_name: str, leader_gate: LeaderGate | None = None
|
|
80
|
+
) -> None:
|
|
50
81
|
self._thread_name = thread_name
|
|
51
82
|
self._stop_event = threading.Event()
|
|
52
83
|
self._thread: threading.Thread | None = None
|
|
84
|
+
self._leader_gate = leader_gate
|
|
53
85
|
|
|
54
86
|
def start(self) -> None:
|
|
55
87
|
"""Start the daemon thread, unless it is gated off or already running.
|
|
@@ -125,8 +157,37 @@ class ThreadedScheduler:
|
|
|
125
157
|
"""
|
|
126
158
|
raise NotImplementedError
|
|
127
159
|
|
|
160
|
+
def _elected_interval(self) -> float:
|
|
161
|
+
"""Run one gated iteration; return the seconds to wait before the next.
|
|
162
|
+
|
|
163
|
+
``None`` gate -> tick (today's behavior). Gate ``True`` -> tick.
|
|
164
|
+
Gate ``False`` -> skip and wait the fixed follower poll.
|
|
165
|
+
|
|
166
|
+
The gate contract says ``should_run`` never raises (see
|
|
167
|
+
:class:`LeaderGate`), but this has zero defense of its own: an escaped
|
|
168
|
+
exception here would kill the daemon thread permanently and silently.
|
|
169
|
+
A raise is therefore caught defensively and treated as fail-open (tick
|
|
170
|
+
runs; duplicate work is safe, silence is not) rather than propagated.
|
|
171
|
+
|
|
172
|
+
Returns:
|
|
173
|
+
float: Seconds to wait before the next loop iteration.
|
|
174
|
+
"""
|
|
175
|
+
if self._leader_gate is None:
|
|
176
|
+
return self._run_once()
|
|
177
|
+
try:
|
|
178
|
+
should_run = self._leader_gate.should_run()
|
|
179
|
+
except Exception:
|
|
180
|
+
logger.exception(
|
|
181
|
+
"event=%s_leader_gate_error — failing open", self._thread_name
|
|
182
|
+
)
|
|
183
|
+
return self._run_once()
|
|
184
|
+
if should_run:
|
|
185
|
+
return self._run_once()
|
|
186
|
+
logger.debug("event=%s_skip_not_leader", self._thread_name)
|
|
187
|
+
return _FOLLOWER_POLL_SECONDS
|
|
188
|
+
|
|
128
189
|
def _run_loop(self) -> None:
|
|
129
|
-
"""Drive
|
|
190
|
+
"""Drive gated iterations until stopped, waiting each returned interval."""
|
|
130
191
|
while not self._stop_event.is_set():
|
|
131
|
-
interval = self.
|
|
192
|
+
interval = self._elected_interval()
|
|
132
193
|
self._stop_event.wait(interval)
|
|
@@ -22,7 +22,7 @@ strings before deleting old import paths in the same PR.
|
|
|
22
22
|
|
|
23
23
|
| File | Purpose |
|
|
24
24
|
|------|---------|
|
|
25
|
-
| `generation_service.py` | `GenerationService` — saves interactions, runs profile + playbook generation
|
|
25
|
+
| `generation_service.py` | `GenerationService` — saves interactions, runs/defer-runs profile + playbook generation, schedules deferred evaluation when `session_id` is present, exposes `run_deferred_learning()` for the non-durable publish worker, and the durable `compute_deferred_learning()` → `persist_deferred_learning()` → `emit_deferred_learning_side_effects()` split (compute outside any scope; only persist + fence inside `commit_scope()`). |
|
|
26
26
|
| `publish_learning_worker.py` | Deferred post-persist publish learning worker — queues async publish learning after durable interaction writes and requeues under publish limiter pressure. |
|
|
27
27
|
| `base_generation_service.py` + `base_generation/` | `BaseGenerationService` stable import surface plus mixins for batch progress, config filtering, extraction lifecycle, should-run prechecks, status transitions, and usage billing. Per-extractor timeout `EXTRACTOR_TIMEOUT_SECONDS = 300`. |
|
|
28
28
|
| `operation_state_utils.py` | `OperationStateManager` — all `_operation_state` access (progress, concurrency locks, extractor/aggregator bookmarks, cluster fingerprints, cancellation). |
|
|
@@ -38,10 +38,11 @@ strings before deleting old import paths in the same PR.
|
|
|
38
38
|
| `agent_success_evaluation/` | `AgentSuccessEvaluationService` | `service.py` (session-level service), `runner.py` (`run_group_evaluation`), `scheduler.py` (`GroupEvaluationScheduler`, 10-min defer), `regen_jobs.py`, `components/evaluator.py` |
|
|
39
39
|
| `reflection/` | `ReflectionService` | `service.py`, `components/extractor.py` — post-horizon reflection; runs **before** extraction so extractors read post-reflection state |
|
|
40
40
|
|
|
41
|
-
## Async Extraction
|
|
41
|
+
## Durable and Async Extraction
|
|
42
42
|
|
|
43
43
|
| Directory | Purpose |
|
|
44
44
|
|-----------|---------|
|
|
45
|
+
| `durable_learning/` | Durable `learning_jobs` scheduler + worker. Gated by `REFLEXIO_DURABLE_LEARNING_QUEUE`; discovers orgs with pending work, claims jobs with leases, then per job runs `compute_deferred_learning()` **outside** any writer transaction, applies `persist_deferred_learning()` + fenced `complete_learning_job()` inside a short `storage.commit_scope()`, and fires `emit_deferred_learning_side_effects()` post-commit — fenced completion guarantees exactly-once side effects. |
|
|
45
46
|
| `extraction/` | Shared async extraction runtime: `resumable_agent.py`, `resume_scheduler.py`, `resume_worker.py`, `pending_tool_call_dispatch.py` (`ask_human`), `prior_answer_search.py`, `agent_run_records.py`, and `outcome.py`. Long-horizon / tool-mediated extraction continues outside the request path. See [README](extraction/README.md). |
|
|
46
47
|
|
|
47
48
|
## Evaluation, Search & Integrations
|
|
@@ -57,13 +58,13 @@ strings before deleting old import paths in the same PR.
|
|
|
57
58
|
| `pre_retrieval/` | `QueryReformulator` (`_query_reformulator.py`) + `DocumentExpander` (`_document_expander.py`) - query rewrite and doc expansion for recall. Compact by design; see [README](pre_retrieval/README.md). |
|
|
58
59
|
| `tagging/` | `TaggingService` (`service.py`) + deferred `tagging_scheduler.py` - post-generation profile/playbook tagging. Compact by design; see [README](tagging/README.md). |
|
|
59
60
|
| `unified_search_service.py` | `run_unified_search()` — two-phase parallel search across profiles / agent playbooks / user playbooks. |
|
|
60
|
-
| `retrieval/` | `relevance_floor.py` — result relevance thresholding. |
|
|
61
|
+
| `retrieval/` | `relevance_floor.py` — result relevance thresholding. `temporal.py` — temporal post-processing driven by reformulation signals: query time windows → per-arm SQL filters, near-duplicate freshness collapse for current-value questions, timestamp ordering for latest-value questions. (Superseded/expired rows are already excluded by storage search SQL.) |
|
|
61
62
|
|
|
62
63
|
## Persistence & Config
|
|
63
64
|
|
|
64
65
|
| Path | Purpose |
|
|
65
66
|
|------|---------|
|
|
66
|
-
| `storage/` | `storage_base/` and `sqlite_storage/` keep legacy domain facades while focused subpackages own `profiles/`, `playbook/`, `agent_run/`, `governance/`, and SQLite `base/` helpers; plus `governance_validation.py` and `retention*.py`. Access via `request_context.storage` only. |
|
|
67
|
+
| `storage/` | `storage_base/` and `sqlite_storage/` keep legacy domain facades while focused subpackages own `profiles/`, `playbook/`, `agent_run/`, `governance/`, durable `learning_jobs`, and SQLite `base/` helpers; plus `governance_validation.py` and `retention*.py`. Access via `request_context.storage` only. |
|
|
67
68
|
| `configurator/` | `DefaultConfigurator` — loads YAML config and creates the storage backend. |
|
|
68
69
|
|
|
69
70
|
## Key Rules
|
|
@@ -72,5 +73,6 @@ strings before deleting old import paths in the same PR.
|
|
|
72
73
|
- **NEVER import storage implementations directly** — use `request_context.storage` (`BaseStorage`).
|
|
73
74
|
- **ALWAYS use `LiteLLMClient`** for completions/embeddings and `request_context.prompt_manager.render_prompt(...)` for prompts — no hardcoded prompts, no direct OpenAI/Claude clients.
|
|
74
75
|
- **All `_operation_state` writes go through `OperationStateManager`** — don't touch the table directly (it backs locks, bookmarks, progress, and cancellation).
|
|
76
|
+
- **All durable queue claims go through `LearningJobStoreABC`** — do not update `learning_jobs` directly; completion is fenced by `claim_token` and must roll back the surrounding `commit_scope` when superseded.
|
|
75
77
|
- **`tool_can_use` lives at root `Config`** — shared by playbook extraction and success evaluation, not per-service.
|
|
76
78
|
- **Preserve governance subject refs/barriers** — route validation through `services/governance/` and `storage/governance_validation.py`; do not bypass retention or subject-write checks in storage implementations.
|
|
@@ -5,9 +5,10 @@ Session-level agent success evaluation module.
|
|
|
5
5
|
## Module Shape
|
|
6
6
|
|
|
7
7
|
- `service.py`: `AgentSuccessEvaluationService`, the request-path service that runs configured evaluators and saves result rows.
|
|
8
|
-
- `runner.py`: `run_group_evaluation(...)`, the background/manual workflow entry point that loads a session, runs the service,
|
|
8
|
+
- `runner.py`: `run_group_evaluation(...)`, the background/manual workflow entry point that loads a session, runs the service, and marks operation state. After agent-success work it also runs the retrieved-learning evaluation (independent completion; generation + session-fingerprint fenced) and returns a `GroupEvaluationOutcome` carrying both statuses.
|
|
9
9
|
- `scheduler.py`: `GroupEvaluationScheduler`, the deferred inactivity scheduler.
|
|
10
10
|
- `components/evaluator.py`: `AgentSuccessEvaluator`, the LLM evaluator component.
|
|
11
|
+
- `components/retrieved_learning_evaluator.py`: `RetrievedLearningEvaluator`, per-learning relevance/impact judges over `Interaction.retrieved_learnings`; results go to the `retrieved_learning_evaluation` table.
|
|
11
12
|
- `regen_jobs.py`: regeneration job planning and execution; remains root-level because API/admin regenerate flows import it directly.
|
|
12
13
|
- `_eval_health.py`: producer/scheduler health counters.
|
|
13
14
|
- `agent_success_evaluation_constants.py`: prompt/model output constants.
|
|
@@ -15,7 +16,7 @@ Session-level agent success evaluation module.
|
|
|
15
16
|
|
|
16
17
|
## Prompt IDs
|
|
17
18
|
|
|
18
|
-
- Owns `agent_success_evaluation`.
|
|
19
|
+
- Owns `agent_success_evaluation`, `retrieved_learning_relevance`, and `retrieved_learning_impact`.
|
|
19
20
|
- Keeps historical/configured prompt ID `agent_success_evaluation_with_comparison` stable where prompt mapping tests require it.
|
|
20
21
|
|
|
21
22
|
Do not reintroduce the deleted service/evaluator/runner/scheduler legacy
|
package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/_eval_health.py
CHANGED
|
@@ -51,6 +51,34 @@ class EvalHealth:
|
|
|
51
51
|
self._skip_counts: Counter[SkipReason] = Counter()
|
|
52
52
|
self._failures: deque[float] = deque()
|
|
53
53
|
self._last_tick_monotonic: float | None = None
|
|
54
|
+
self._retrieved_outcome_counts: Counter[str] = Counter()
|
|
55
|
+
self._retrieved_invalid_ref_count = 0
|
|
56
|
+
self._retrieved_failed_relevance_chunks = 0
|
|
57
|
+
self._retrieved_failed_impact_chunks = 0
|
|
58
|
+
|
|
59
|
+
def record_retrieved_outcome(
|
|
60
|
+
self, status: str, *, diagnostics: dict[str, Any] | None = None
|
|
61
|
+
) -> None:
|
|
62
|
+
"""Record one retrieved-learning evaluation outcome.
|
|
63
|
+
|
|
64
|
+
Args:
|
|
65
|
+
status (str): Final invocation status (complete/degraded/failed/
|
|
66
|
+
not_applicable/...).
|
|
67
|
+
diagnostics (dict, optional): Evaluator diagnostics; the
|
|
68
|
+
invalid-ref and failed-chunk counters are accumulated from it.
|
|
69
|
+
"""
|
|
70
|
+
with self._lock:
|
|
71
|
+
self._retrieved_outcome_counts[status] += 1
|
|
72
|
+
if diagnostics:
|
|
73
|
+
self._retrieved_invalid_ref_count += int(
|
|
74
|
+
diagnostics.get("invalid_ref_count") or 0
|
|
75
|
+
)
|
|
76
|
+
self._retrieved_failed_relevance_chunks += int(
|
|
77
|
+
diagnostics.get("failed_relevance_chunks") or 0
|
|
78
|
+
)
|
|
79
|
+
self._retrieved_failed_impact_chunks += int(
|
|
80
|
+
diagnostics.get("failed_impact_chunks") or 0
|
|
81
|
+
)
|
|
54
82
|
|
|
55
83
|
def record_skip(self, reason: SkipReason) -> None:
|
|
56
84
|
"""Bump the counter for `reason` by one.
|
|
@@ -99,6 +127,12 @@ class EvalHealth:
|
|
|
99
127
|
"skip_counts": {r.value: self._skip_counts[r] for r in SkipReason},
|
|
100
128
|
"producer_failures_24h": len(self._failures),
|
|
101
129
|
"last_tick_monotonic": self._last_tick_monotonic,
|
|
130
|
+
"retrieved_learning": {
|
|
131
|
+
"outcome_counts": dict(self._retrieved_outcome_counts),
|
|
132
|
+
"invalid_ref_count": self._retrieved_invalid_ref_count,
|
|
133
|
+
"failed_relevance_chunks": self._retrieved_failed_relevance_chunks,
|
|
134
|
+
"failed_impact_chunks": self._retrieved_failed_impact_chunks,
|
|
135
|
+
},
|
|
102
136
|
}
|
|
103
137
|
|
|
104
138
|
def _trim_locked(self, now_ts: float) -> None:
|
|
@@ -116,6 +150,13 @@ def record_skip(reason: SkipReason) -> None:
|
|
|
116
150
|
_HEALTH.record_skip(reason)
|
|
117
151
|
|
|
118
152
|
|
|
153
|
+
def record_retrieved_outcome(
|
|
154
|
+
status: str, *, diagnostics: dict[str, Any] | None = None
|
|
155
|
+
) -> None:
|
|
156
|
+
"""Module-level proxy to the singleton."""
|
|
157
|
+
_HEALTH.record_retrieved_outcome(status, diagnostics=diagnostics)
|
|
158
|
+
|
|
159
|
+
|
|
119
160
|
def record_producer_failure(at_ts: float | None = None) -> None:
|
|
120
161
|
"""Module-level proxy to the singleton."""
|
|
121
162
|
_HEALTH.record_producer_failure(at_ts=at_ts)
|