claude-smart 0.2.48 → 0.2.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/README.md +11 -40
- package/bin/claude-smart.js +393 -55
- package/package.json +3 -2
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/.codex-plugin/plugin.json +1 -1
- package/plugin/.coverage +0 -0
- package/plugin/README.md +4 -3
- package/plugin/dashboard/app/api/reflexio/[...path]/route.ts +4 -2
- package/plugin/dashboard/app/dashboard/page.tsx +6 -1
- package/plugin/dashboard/app/preferences/[id]/page.tsx +18 -6
- package/plugin/dashboard/app/preferences/page.tsx +32 -35
- package/plugin/dashboard/app/sessions/[sessionId]/page.tsx +16 -1
- package/plugin/dashboard/app/sessions/page.tsx +2 -0
- package/plugin/dashboard/app/skills/page.tsx +65 -50
- package/plugin/dashboard/app/skills/project/[id]/page.tsx +17 -8
- package/plugin/dashboard/app/skills/shared/[id]/page.tsx +1 -6
- package/plugin/dashboard/components/common/host-badge.tsx +118 -0
- package/plugin/dashboard/components/common/learning-application-badge.tsx +34 -0
- package/plugin/dashboard/components/common/learnings-badge.tsx +1 -1
- package/plugin/dashboard/components/common/page-header.tsx +3 -3
- package/plugin/dashboard/lib/config-file.ts +5 -1
- package/plugin/dashboard/lib/host-attribution.ts +62 -0
- package/plugin/dashboard/lib/session-reader.ts +40 -2
- package/plugin/dashboard/lib/types.ts +7 -1
- package/plugin/opencode/dist/server.mjs +60 -2
- package/plugin/opencode/server.mts +64 -2
- package/plugin/pyproject.toml +2 -2
- package/plugin/scripts/_lib.sh +255 -7
- package/plugin/scripts/backend-python-runner.py +46 -0
- package/plugin/scripts/backend-service.sh +766 -123
- package/plugin/scripts/codex-hook.js +79 -229
- package/plugin/scripts/dashboard-open.sh +6 -4
- package/plugin/scripts/dashboard-service.sh +118 -137
- package/plugin/scripts/ensure-plugin-root.sh +106 -8
- package/plugin/scripts/hook_entry.sh +3 -0
- package/plugin/scripts/opencode-claude-compat.js +8 -1
- package/plugin/scripts/smart-install.sh +116 -21
- package/plugin/src/README.md +1 -1
- package/plugin/src/claude_smart/cli.py +93 -29
- package/plugin/src/claude_smart/context_inject.py +3 -0
- package/plugin/src/claude_smart/env_config.py +56 -11
- package/plugin/src/claude_smart/events/post_tool.py +2 -1
- package/plugin/src/claude_smart/events/session_end.py +2 -1
- package/plugin/src/claude_smart/events/stop.py +3 -0
- package/plugin/src/claude_smart/events/user_prompt.py +2 -1
- package/plugin/src/claude_smart/internal_call.py +5 -2
- package/plugin/src/claude_smart/optimizer_assistant.py +59 -13
- package/plugin/src/claude_smart/publish.py +59 -7
- package/plugin/src/claude_smart/reflexio_adapter.py +137 -14
- package/plugin/src/claude_smart/runtime.py +15 -6
- package/plugin/src/claude_smart/state.py +211 -52
- package/plugin/uv.lock +5 -5
- package/plugin/vendor/reflexio/.env.example +13 -0
- package/plugin/vendor/reflexio/reflexio/README.md +7 -3
- package/plugin/vendor/reflexio/reflexio/__init__.py +12 -0
- package/plugin/vendor/reflexio/reflexio/cli/README.md +3 -0
- package/plugin/vendor/reflexio/reflexio/client/client.py +166 -9
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/state.py +10 -3
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_state.py +28 -0
- package/plugin/vendor/reflexio/reflexio/lib/_search.py +41 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/entities.py +195 -25
- package/plugin/vendor/reflexio/reflexio/models/api_schema/eval_overview_schema.py +7 -1
- package/plugin/vendor/reflexio/reflexio/models/api_schema/internal_schema.py +2 -1
- package/plugin/vendor/reflexio/reflexio/models/api_schema/retriever_schema.py +63 -4
- package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/converters.py +1 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/entities.py +8 -1
- package/plugin/vendor/reflexio/reflexio/models/config_schema.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/README.md +22 -4
- package/plugin/vendor/reflexio/reflexio/server/__init__.py +18 -8
- package/plugin/vendor/reflexio/reflexio/server/__main__.py +6 -0
- package/plugin/vendor/reflexio/reflexio/server/api.py +79 -5
- package/plugin/vendor/reflexio/reflexio/server/billing_meter.py +263 -3
- package/plugin/vendor/reflexio/reflexio/server/callback_executor.py +164 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_embedding.py +81 -81
- package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_subprocess.py +28 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_text_generation.py +63 -6
- package/plugin/vendor/reflexio/reflexio/server/llm/embedding_service.py +19 -52
- package/plugin/vendor/reflexio/reflexio/server/llm/litellm_client.py +1 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/model_defaults.py +28 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/claude_code_provider.py +9 -1
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/embedder_warmup.py +329 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/embedding_service_provider.py +85 -10
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/local_embedding_provider.py +199 -2
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/nomic_embedding_provider.py +77 -9
- package/plugin/vendor/reflexio/reflexio/server/org_fanout.py +184 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/document_expansion/v1.0.0.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/document_expansion/v1.1.0.prompt.md +32 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.3.3.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.4.0.prompt.md +63 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/query_reformulation/v1.0.0.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/query_reformulation/v2.0.0.prompt.md +30 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/retrieved_learning_impact/v1.0.0.prompt.md +51 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/retrieved_learning_relevance/v1.0.0.prompt.md +39 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/shadow_comparison/v1.0.0.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/shadow_comparison/v1.1.0.prompt.md +43 -0
- package/plugin/vendor/reflexio/reflexio/server/routes/config.py +3 -3
- package/plugin/vendor/reflexio/reflexio/server/routes/evaluation.py +122 -28
- package/plugin/vendor/reflexio/reflexio/server/routes/interactions.py +63 -2
- package/plugin/vendor/reflexio/reflexio/server/routes/system.py +22 -3
- package/plugin/vendor/reflexio/reflexio/server/scheduling.py +64 -3
- package/plugin/vendor/reflexio/reflexio/server/services/README.md +6 -4
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/README.md +3 -2
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/_eval_health.py +41 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/components/retrieved_learning_evaluator.py +554 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/regen_jobs.py +35 -1
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/runner.py +253 -101
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/scheduler.py +5 -7
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/service.py +27 -0
- package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_extraction_lifecycle.py +11 -5
- package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_should_run.py +4 -4
- package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_usage_billing.py +6 -2
- package/plugin/vendor/reflexio/reflexio/server/services/base_generation_service.py +255 -74
- package/plugin/vendor/reflexio/reflexio/server/services/deduplication_utils.py +72 -0
- package/plugin/vendor/reflexio/reflexio/server/services/deferred_learning_plan.py +270 -0
- package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/__init__.py +13 -0
- package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/scheduler.py +142 -0
- package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/worker.py +262 -0
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/components/hero_state.py +2 -11
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/components/rule_attribution.py +6 -2
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/service.py +17 -13
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/agent_run_records.py +18 -3
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/outcome.py +12 -1
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/prior_answer_search.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/resumable_agent.py +2 -1
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_scheduler.py +1 -3
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_worker.py +66 -0
- package/plugin/vendor/reflexio/reflexio/server/services/generation_service.py +640 -80
- package/plugin/vendor/reflexio/reflexio/server/services/governance/service.py +2 -0
- package/plugin/vendor/reflexio/reflexio/server/services/lineage/gc_scheduler.py +179 -99
- package/plugin/vendor/reflexio/reflexio/server/services/lineage/vector_backfill_sweep.py +139 -0
- package/plugin/vendor/reflexio/reflexio/server/services/operation_state_utils.py +66 -27
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/aggregation_trigger.py +177 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/aggregator.py +68 -2
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/consolidator.py +360 -49
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/extractor.py +27 -30
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/playbook_edit_apply.py +8 -1
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/service.py +137 -69
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/optimizer.py +20 -1
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/scheduler.py +13 -6
- package/plugin/vendor/reflexio/reflexio/server/services/pre_retrieval/_query_reformulator.py +40 -25
- package/plugin/vendor/reflexio/reflexio/server/services/profile/components/consolidator.py +12 -39
- package/plugin/vendor/reflexio/reflexio/server/services/profile/components/extractor.py +30 -35
- package/plugin/vendor/reflexio/reflexio/server/services/profile/service.py +166 -66
- package/plugin/vendor/reflexio/reflexio/server/services/reflection/service.py +457 -107
- package/plugin/vendor/reflexio/reflexio/server/services/retrieval/session_dedup.py +127 -0
- package/plugin/vendor/reflexio/reflexio/server/services/retrieval/temporal.py +104 -0
- package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/dispatcher.py +139 -0
- package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/judge.py +16 -6
- package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/worker.py +137 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/governance_validation.py +12 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/lifecycle_filters.py +54 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/retention.py +41 -3
- package/plugin/vendor/reflexio/reflexio/server/services/storage/retention_mixin.py +40 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/__init__.py +2 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_base.py +190 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_extras.py +11 -4
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_governance.py +28 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py +414 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_lineage.py +11 -5
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_requests.py +8 -3
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_shadow_verdicts.py +4 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/base/_deletion.py +16 -5
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/base/_fts_vec.py +86 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_erase_execution.py +104 -42
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_subject_barrier.py +7 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_agent.py +59 -7
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_eval_results.py +430 -5
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_source_linkage.py +8 -3
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_user.py +57 -12
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_interaction_store.py +197 -12
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_profile_store.py +70 -11
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/__init__.py +17 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_commit_scope.py +9 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_extras.py +9 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_learning_jobs.py +269 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_operations.py +7 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_retrieval_log.py +3 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_shadow_verdicts.py +4 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/agent_run/_models.py +12 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/evaluation_state_keys.py +78 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_agent.py +38 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_eval_results.py +171 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_user.py +52 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/_interaction_store.py +82 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/_profile_store.py +74 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/retrieved_learning_state.py +226 -0
- package/plugin/vendor/reflexio/reflexio/server/services/tagging/tagging_scheduler.py +5 -6
- package/plugin/vendor/reflexio/reflexio/server/services/unified_search_service.py +209 -29
- package/plugin/vendor/reflexio/reflexio/server/usage_metrics.py +3 -0
- package/plugin/vendor/reflexio/reflexio/test_support/llm_mock.py +61 -0
- package/plugin/vendor/reflexio/reflexio/test_support/llm_model_registry.py +33 -0
- package/plugin/vendor/reflexio/reflexio/server/services/search/__init__.py +0 -0
package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/components/hero_state.py
CHANGED
|
@@ -11,7 +11,6 @@ from enum import StrEnum
|
|
|
11
11
|
|
|
12
12
|
_FULL_MIN_DAYS = 14
|
|
13
13
|
_FULL_MIN_SHADOW_N = 500
|
|
14
|
-
_SHADOW_OFF_MIN_DAYS = 7
|
|
15
14
|
|
|
16
15
|
|
|
17
16
|
class HeroState(StrEnum):
|
|
@@ -50,7 +49,7 @@ def compute_hero_state(
|
|
|
50
49
|
migration pair in supabase/data/supabase/migrations/. The
|
|
51
50
|
per-turn shadow grade in F1 lives on a different surface.)
|
|
52
51
|
total_results (int): Total AgentSuccessEvaluationResult rows in the
|
|
53
|
-
trend window
|
|
52
|
+
trend window, used to distinguish EMPTY from states with data.
|
|
54
53
|
|
|
55
54
|
Returns:
|
|
56
55
|
HeroState: The single applicable state. Empty wins over everything;
|
|
@@ -59,15 +58,7 @@ def compute_hero_state(
|
|
|
59
58
|
if total_results == 0:
|
|
60
59
|
return HeroState.EMPTY
|
|
61
60
|
if not shadow_enabled:
|
|
62
|
-
|
|
63
|
-
days_since_first_eval is not None
|
|
64
|
-
and days_since_first_eval >= _SHADOW_OFF_MIN_DAYS
|
|
65
|
-
):
|
|
66
|
-
return HeroState.SHADOW_OFF
|
|
67
|
-
# <7 days since first eval AND shadow off → still onboarding;
|
|
68
|
-
# render as EMPTY so the frontend shows onboarding rather than a
|
|
69
|
-
# partly-formed trend.
|
|
70
|
-
return HeroState.EMPTY
|
|
61
|
+
return HeroState.SHADOW_OFF
|
|
71
62
|
if days_since_first_eval is None or days_since_first_eval < _FULL_MIN_DAYS:
|
|
72
63
|
return HeroState.EARLY
|
|
73
64
|
if n_shadow_in_window < _FULL_MIN_SHADOW_N:
|
|
@@ -10,7 +10,11 @@ from __future__ import annotations
|
|
|
10
10
|
from collections.abc import Mapping
|
|
11
11
|
from dataclasses import dataclass
|
|
12
12
|
|
|
13
|
-
|
|
13
|
+
from reflexio.models.api_schema.domain import CitationKind
|
|
14
|
+
|
|
15
|
+
CitationKey = tuple[
|
|
16
|
+
CitationKind, str
|
|
17
|
+
] # (kind, real_id) — matches PlaybookApplicationStat
|
|
14
18
|
SessionIdentity = tuple[str, str] # (user_id, session_id)
|
|
15
19
|
|
|
16
20
|
|
|
@@ -19,7 +23,7 @@ class RuleAttribution:
|
|
|
19
23
|
"""One row of the "rules that moved the needle" panel."""
|
|
20
24
|
|
|
21
25
|
rule_id: str
|
|
22
|
-
kind:
|
|
26
|
+
kind: CitationKind
|
|
23
27
|
title: str
|
|
24
28
|
successes_with: int
|
|
25
29
|
failures_with: int
|
|
@@ -47,6 +47,7 @@ from reflexio.server.services.evaluation_overview.components.hero_state import (
|
|
|
47
47
|
compute_hero_state,
|
|
48
48
|
)
|
|
49
49
|
from reflexio.server.services.evaluation_overview.components.rule_attribution import (
|
|
50
|
+
CitationKey,
|
|
50
51
|
compute_net_sessions,
|
|
51
52
|
)
|
|
52
53
|
from reflexio.server.services.evaluation_overview.components.shadow_aggregation import (
|
|
@@ -80,14 +81,14 @@ class EvaluationOverviewService:
|
|
|
80
81
|
Time windows used here:
|
|
81
82
|
- ``[request.from_ts, request.to_ts]`` is the *trend* window. The hero
|
|
82
83
|
chart, rule attribution, and the rule-attribution session set are
|
|
83
|
-
all computed over it. The frontend
|
|
84
|
-
|
|
84
|
+
all computed over it. The frontend defaults to a 7-day range but
|
|
85
|
+
the user can widen it via the date pickers.
|
|
85
86
|
- The *tile baseline* and *distribution baseline* are tighter:
|
|
86
87
|
``last_7d`` (≤ to_ts) vs ``prior_7d`` (the 7d before that).
|
|
87
|
-
Tying these to ``request.from_ts`` is wrong —
|
|
88
|
-
|
|
89
|
-
so every tile would display "no baseline"
|
|
90
|
-
data the org has.
|
|
88
|
+
Tying these to ``request.from_ts`` is wrong — for any window wider
|
|
89
|
+
than 7 days ``prior`` would land before the requested window and
|
|
90
|
+
always be empty, so every tile would display "no baseline"
|
|
91
|
+
regardless of how much data the org has.
|
|
91
92
|
"""
|
|
92
93
|
# Tile + distribution baselines are always last-7d-vs-prior-7d,
|
|
93
94
|
# anchored to ``request.to_ts`` (which is "now" from the frontend's
|
|
@@ -206,6 +207,9 @@ class EvaluationOverviewService:
|
|
|
206
207
|
results: list[AgentSuccessEvaluationResult],
|
|
207
208
|
earliest_eval_ts: int | None,
|
|
208
209
|
) -> HeroBlock:
|
|
210
|
+
# days_since only feeds the EARLY-vs-FULL gate, which is reachable only
|
|
211
|
+
# when shadow_enabled is True. Shadow is hardcoded off below, so this is
|
|
212
|
+
# currently inert — retained for the (pending) shadow re-enable.
|
|
209
213
|
if earliest_eval_ts is None:
|
|
210
214
|
days_since = None
|
|
211
215
|
else:
|
|
@@ -264,8 +268,8 @@ class EvaluationOverviewService:
|
|
|
264
268
|
def _build_attribution(
|
|
265
269
|
self,
|
|
266
270
|
results: list[AgentSuccessEvaluationResult],
|
|
267
|
-
citations_by_session: dict[ResultKey, list[
|
|
268
|
-
rule_titles: dict[
|
|
271
|
+
citations_by_session: dict[ResultKey, list[CitationKey]],
|
|
272
|
+
rule_titles: dict[CitationKey, str],
|
|
269
273
|
) -> list[RuleAttributionRow]:
|
|
270
274
|
is_success_by_session = {
|
|
271
275
|
(r.user_id, r.session_id): r.is_success for r in results
|
|
@@ -291,7 +295,7 @@ class EvaluationOverviewService:
|
|
|
291
295
|
|
|
292
296
|
def _load_citations(
|
|
293
297
|
self, result_keys: list[ResultKey]
|
|
294
|
-
) -> tuple[dict[ResultKey, list[
|
|
298
|
+
) -> tuple[dict[ResultKey, list[CitationKey]], dict[CitationKey, str]]:
|
|
295
299
|
"""Pull `Interaction.citations` keyed by user/session, with title lookup.
|
|
296
300
|
|
|
297
301
|
Falls back to empty data when the underlying storage method returns
|
|
@@ -300,8 +304,8 @@ class EvaluationOverviewService:
|
|
|
300
304
|
"""
|
|
301
305
|
wanted = set(result_keys)
|
|
302
306
|
session_ids = sorted({session_id for _, session_id in result_keys})
|
|
303
|
-
citations_by_session: dict[ResultKey, list[
|
|
304
|
-
rule_titles: dict[
|
|
307
|
+
citations_by_session: dict[ResultKey, list[CitationKey]] = defaultdict(list)
|
|
308
|
+
rule_titles: dict[CitationKey, str] = {}
|
|
305
309
|
for citation in self.storage.get_citations_by_session_ids(session_ids): # type: ignore[attr-defined]
|
|
306
310
|
result_key = (citation.user_id, citation.session_id)
|
|
307
311
|
if result_key not in wanted:
|
|
@@ -369,8 +373,8 @@ class EvaluationOverviewService:
|
|
|
369
373
|
previous: list[AgentSuccessEvaluationResult],
|
|
370
374
|
session_sources: dict[ResultKey, str],
|
|
371
375
|
bucket: BucketLiteral,
|
|
372
|
-
citations_by_session: dict[ResultKey, list[
|
|
373
|
-
rule_titles: dict[
|
|
376
|
+
citations_by_session: dict[ResultKey, list[CitationKey]],
|
|
377
|
+
rule_titles: dict[CitationKey, str],
|
|
374
378
|
current_scores: list[ImportedScore],
|
|
375
379
|
prior_scores: list[ImportedScore],
|
|
376
380
|
) -> SourceSetComparison:
|
|
@@ -47,14 +47,25 @@ def build_extractor_agent_run_record(
|
|
|
47
47
|
org_id: str,
|
|
48
48
|
extractor_kind: str,
|
|
49
49
|
user_id: str | None,
|
|
50
|
-
request_id: str,
|
|
51
50
|
agent_version: str | None,
|
|
52
51
|
source: str | None,
|
|
53
52
|
request_interaction_data_models: list[RequestInteractionDataModel],
|
|
54
53
|
extractor_config: BaseModel,
|
|
55
54
|
service_config: Any,
|
|
56
55
|
agent_context: str,
|
|
56
|
+
generation_request_id: str | None = None,
|
|
57
|
+
request_id: str | None = None,
|
|
57
58
|
) -> AgentRunRecord:
|
|
59
|
+
if generation_request_id is not None:
|
|
60
|
+
if request_id is not None and request_id != generation_request_id:
|
|
61
|
+
raise TypeError(
|
|
62
|
+
"generation_request_id and request_id must match when both are provided"
|
|
63
|
+
)
|
|
64
|
+
elif request_id is not None:
|
|
65
|
+
generation_request_id = request_id
|
|
66
|
+
else:
|
|
67
|
+
raise TypeError("generation_request_id is required")
|
|
68
|
+
|
|
58
69
|
source_interaction_ids = extract_source_interaction_ids(
|
|
59
70
|
request_interaction_data_models
|
|
60
71
|
)
|
|
@@ -66,7 +77,9 @@ def build_extractor_agent_run_record(
|
|
|
66
77
|
org_id=org_id,
|
|
67
78
|
extractor_kind=extractor_kind,
|
|
68
79
|
user_id=user_id,
|
|
69
|
-
request_id
|
|
80
|
+
# The durable storage boundary still uses the legacy ``request_id``
|
|
81
|
+
# field name for run provenance.
|
|
82
|
+
request_id=generation_request_id,
|
|
70
83
|
agent_version=agent_version,
|
|
71
84
|
source=source,
|
|
72
85
|
source_interaction_ids=source_interaction_ids,
|
|
@@ -80,7 +93,9 @@ def build_extractor_agent_run_record(
|
|
|
80
93
|
),
|
|
81
94
|
status=AgentRunStatus.RUNNING,
|
|
82
95
|
generation_request_snapshot={
|
|
83
|
-
|
|
96
|
+
# Snapshot payloads also keep the legacy ``request_id`` key so
|
|
97
|
+
# existing storage readers do not need a migration.
|
|
98
|
+
"request_id": generation_request_id,
|
|
84
99
|
"source": source,
|
|
85
100
|
"source_interaction_ids": source_interaction_ids,
|
|
86
101
|
"session_count": len(request_interaction_data_models),
|
|
@@ -7,6 +7,9 @@ from typing import TYPE_CHECKING, Literal
|
|
|
7
7
|
|
|
8
8
|
if TYPE_CHECKING:
|
|
9
9
|
from reflexio.server.llm.token_accounting import RunTokenTotals
|
|
10
|
+
from reflexio.server.services.deferred_learning_plan import (
|
|
11
|
+
ExtractorBookmarkAdvance,
|
|
12
|
+
)
|
|
10
13
|
|
|
11
14
|
|
|
12
15
|
@dataclass(frozen=True)
|
|
@@ -17,6 +20,9 @@ class ExtractionOutcome[T]:
|
|
|
17
20
|
items: list[T] = field(default_factory=list)
|
|
18
21
|
run_id: str | None = None
|
|
19
22
|
token_totals: RunTokenTotals | None = None
|
|
23
|
+
# The stride-bookmark advance the extractor no longer applies itself (F1);
|
|
24
|
+
# applied downstream in persist (durable) or ``.run()``'s persist half.
|
|
25
|
+
bookmark_advance: ExtractorBookmarkAdvance | None = None
|
|
20
26
|
|
|
21
27
|
@classmethod
|
|
22
28
|
def completed(
|
|
@@ -25,9 +31,14 @@ class ExtractionOutcome[T]:
|
|
|
25
31
|
*,
|
|
26
32
|
run_id: str | None = None,
|
|
27
33
|
token_totals: RunTokenTotals | None = None,
|
|
34
|
+
bookmark_advance: ExtractorBookmarkAdvance | None = None,
|
|
28
35
|
) -> ExtractionOutcome[T]:
|
|
29
36
|
return cls(
|
|
30
|
-
status="completed",
|
|
37
|
+
status="completed",
|
|
38
|
+
items=items,
|
|
39
|
+
run_id=run_id,
|
|
40
|
+
token_totals=token_totals,
|
|
41
|
+
bookmark_advance=bookmark_advance,
|
|
31
42
|
)
|
|
32
43
|
|
|
33
44
|
@classmethod
|
|
@@ -151,7 +151,7 @@ def format_prior_knowledge_context(matches: list[PriorAnswerMatch]) -> str | Non
|
|
|
151
151
|
for match in resolved:
|
|
152
152
|
lines.extend(
|
|
153
153
|
[
|
|
154
|
-
f"-
|
|
154
|
+
f"- pending_tool_call_id: {match.pending_tool_call_id}",
|
|
155
155
|
f" question: {match.question_text}",
|
|
156
156
|
f" answer: {_format_result(match.result)}",
|
|
157
157
|
f" resolved_at: {_format_datetime(match.resolved_at)}",
|
|
@@ -161,6 +161,7 @@ def run_resumable_extraction_agent(
|
|
|
161
161
|
log_label: str,
|
|
162
162
|
) -> AgentRunResult:
|
|
163
163
|
"""Run and finalize a config-gated classic extraction agent pass."""
|
|
164
|
+
generation_request_id = request_id
|
|
164
165
|
pending_config = _pending_tool_call_config(request_context)
|
|
165
166
|
storage = request_context.storage
|
|
166
167
|
if storage is None:
|
|
@@ -172,7 +173,7 @@ def run_resumable_extraction_agent(
|
|
|
172
173
|
org_id=request_context.org_id,
|
|
173
174
|
extractor_kind=extractor_kind,
|
|
174
175
|
user_id=user_id,
|
|
175
|
-
|
|
176
|
+
generation_request_id=generation_request_id,
|
|
176
177
|
agent_version=agent_version,
|
|
177
178
|
source=source,
|
|
178
179
|
request_interaction_data_models=request_interaction_data_models,
|
|
@@ -107,9 +107,7 @@ class ExtractionResumeScheduler(ThreadedScheduler):
|
|
|
107
107
|
try:
|
|
108
108
|
bootstrap_ctx = self.request_context_factory(self.bootstrap_org_id)
|
|
109
109
|
config = bootstrap_ctx.configurator.get_config()
|
|
110
|
-
poll_interval =
|
|
111
|
-
config.pending_tool_call_config.resume_poll_interval_seconds
|
|
112
|
-
)
|
|
110
|
+
poll_interval = config.pending_tool_call_config.resume_poll_interval_seconds
|
|
113
111
|
self._expire_pending_tool_calls(bootstrap_ctx)
|
|
114
112
|
for org_id in self._discover_org_ids(bootstrap_ctx):
|
|
115
113
|
if self._stop_event.is_set():
|
|
@@ -764,6 +764,7 @@ class ExtractionResumeWorker:
|
|
|
764
764
|
force_extraction=True,
|
|
765
765
|
)
|
|
766
766
|
service._finalize_extracted_items(items)
|
|
767
|
+
self._record_finalized_learnings(run, items, entity_type="profile")
|
|
767
768
|
return
|
|
768
769
|
if run.binding.extractor_kind == "playbook":
|
|
769
770
|
service = PlaybookGenerationService(
|
|
@@ -779,11 +780,76 @@ class ExtractionResumeWorker:
|
|
|
779
780
|
force_extraction=True,
|
|
780
781
|
)
|
|
781
782
|
service._finalize_extracted_items(items)
|
|
783
|
+
self._record_finalized_learnings(run, items, entity_type="user_playbook")
|
|
782
784
|
return
|
|
783
785
|
raise ResumeWorkerError(
|
|
784
786
|
f"Unsupported extractor kind {run.binding.extractor_kind!r}"
|
|
785
787
|
)
|
|
786
788
|
|
|
789
|
+
def _record_finalized_learnings(
|
|
790
|
+
self, run: AgentRunRecord, items: list[Any], *, entity_type: str
|
|
791
|
+
) -> None:
|
|
792
|
+
"""Emit ``learnings_generated`` for a finalized resumable-extraction batch.
|
|
793
|
+
|
|
794
|
+
Prefers one event per learning id (``entity_id``/``profile_id`` for
|
|
795
|
+
profiles, ``user_playbook_id`` for playbooks) when every item in
|
|
796
|
+
``items`` carries a durable id — the common case, since these ids are
|
|
797
|
+
assigned by the extractor (profile) or by ``save_user_playbooks``
|
|
798
|
+
in-place during ``_finalize_extracted_items`` (playbook), which has
|
|
799
|
+
already run by the time this is called. Falls back to the
|
|
800
|
+
count-based aggregate event when any item lacks one (e.g. dropped by
|
|
801
|
+
within-batch/consolidator dedup before persist, leaving a default
|
|
802
|
+
``user_playbook_id=0``) — this avoids both fabricating an id for a row
|
|
803
|
+
that never persisted and colliding on the shared default-id key.
|
|
804
|
+
Totals are preserved either way: ``len(items)`` learnings are counted
|
|
805
|
+
whether via ``len(learning_ids)`` per-record events or one aggregate
|
|
806
|
+
``count=len(items)`` event.
|
|
807
|
+
"""
|
|
808
|
+
from reflexio.server.billing_meter import (
|
|
809
|
+
emit_learnings_generated,
|
|
810
|
+
emit_learnings_generated_records,
|
|
811
|
+
)
|
|
812
|
+
|
|
813
|
+
if not items:
|
|
814
|
+
return
|
|
815
|
+
|
|
816
|
+
id_attr = "profile_id" if entity_type == "profile" else "user_playbook_id"
|
|
817
|
+
learning_ids = [
|
|
818
|
+
str(getattr(item, id_attr))
|
|
819
|
+
for item in items
|
|
820
|
+
if getattr(item, id_attr, None)
|
|
821
|
+
]
|
|
822
|
+
metadata = {
|
|
823
|
+
"run_id": run.id,
|
|
824
|
+
"extractor_kind": run.binding.extractor_kind,
|
|
825
|
+
}
|
|
826
|
+
if len(learning_ids) == len(items):
|
|
827
|
+
emit_learnings_generated_records(
|
|
828
|
+
org_id=self.request_context.org_id,
|
|
829
|
+
configurator=self.request_context.configurator,
|
|
830
|
+
learning_ids=learning_ids,
|
|
831
|
+
source="resumable_extraction",
|
|
832
|
+
pipeline=run.binding.extractor_kind,
|
|
833
|
+
user_id=run.binding.user_id,
|
|
834
|
+
request_id=run.binding.request_id,
|
|
835
|
+
agent_version=run.binding.agent_version,
|
|
836
|
+
entity_type=entity_type,
|
|
837
|
+
metadata=metadata,
|
|
838
|
+
)
|
|
839
|
+
return
|
|
840
|
+
emit_learnings_generated(
|
|
841
|
+
org_id=self.request_context.org_id,
|
|
842
|
+
configurator=self.request_context.configurator,
|
|
843
|
+
count=len(items),
|
|
844
|
+
source="resumable_extraction",
|
|
845
|
+
pipeline=run.binding.extractor_kind,
|
|
846
|
+
user_id=run.binding.user_id,
|
|
847
|
+
request_id=run.binding.request_id,
|
|
848
|
+
agent_version=run.binding.agent_version,
|
|
849
|
+
entity_type=entity_type,
|
|
850
|
+
metadata=metadata,
|
|
851
|
+
)
|
|
852
|
+
|
|
787
853
|
def _schedule_finalized_tagging(self, run: AgentRunRecord) -> None:
|
|
788
854
|
user_id = run.binding.user_id
|
|
789
855
|
if not user_id:
|