claude-smart 0.2.49 → 0.2.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/README.md +10 -43
- package/bin/claude-smart.js +105 -0
- package/package.json +3 -2
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/.codex-plugin/plugin.json +1 -1
- package/plugin/.coverage +0 -0
- package/plugin/README.md +4 -3
- package/plugin/dashboard/app/api/reflexio/[...path]/route.ts +4 -2
- package/plugin/dashboard/app/dashboard/page.tsx +6 -1
- package/plugin/dashboard/app/preferences/[id]/page.tsx +18 -6
- package/plugin/dashboard/app/preferences/page.tsx +32 -35
- package/plugin/dashboard/app/sessions/[sessionId]/page.tsx +16 -1
- package/plugin/dashboard/app/sessions/page.tsx +2 -0
- package/plugin/dashboard/app/skills/page.tsx +65 -50
- package/plugin/dashboard/app/skills/project/[id]/page.tsx +17 -8
- package/plugin/dashboard/app/skills/shared/[id]/page.tsx +1 -6
- package/plugin/dashboard/components/common/host-badge.tsx +118 -0
- package/plugin/dashboard/components/common/learning-application-badge.tsx +34 -0
- package/plugin/dashboard/components/common/learnings-badge.tsx +1 -1
- package/plugin/dashboard/components/common/page-header.tsx +3 -3
- package/plugin/dashboard/lib/config-file.ts +5 -1
- package/plugin/dashboard/lib/host-attribution.ts +62 -0
- package/plugin/dashboard/lib/session-reader.ts +40 -2
- package/plugin/dashboard/lib/types.ts +7 -1
- package/plugin/pyproject.toml +1 -1
- package/plugin/scripts/_lib.sh +197 -1
- package/plugin/scripts/backend-python-runner.py +46 -0
- package/plugin/scripts/backend-service.sh +757 -119
- package/plugin/scripts/codex-hook.js +63 -225
- package/plugin/scripts/dashboard-open.sh +6 -4
- package/plugin/scripts/dashboard-service.sh +117 -136
- package/plugin/scripts/hook_entry.sh +3 -0
- package/plugin/scripts/smart-install.sh +15 -1
- package/plugin/src/claude_smart/cli.py +14 -0
- package/plugin/src/claude_smart/context_inject.py +3 -0
- package/plugin/src/claude_smart/env_config.py +4 -1
- package/plugin/src/claude_smart/events/post_tool.py +2 -1
- package/plugin/src/claude_smart/events/session_end.py +2 -1
- package/plugin/src/claude_smart/events/stop.py +3 -0
- package/plugin/src/claude_smart/events/user_prompt.py +2 -1
- package/plugin/src/claude_smart/internal_call.py +5 -2
- package/plugin/src/claude_smart/optimizer_assistant.py +59 -13
- package/plugin/src/claude_smart/publish.py +59 -7
- package/plugin/src/claude_smart/reflexio_adapter.py +137 -14
- package/plugin/src/claude_smart/runtime.py +15 -6
- package/plugin/src/claude_smart/state.py +211 -52
- package/plugin/uv.lock +1 -1
- package/plugin/vendor/reflexio/.env.example +13 -0
- package/plugin/vendor/reflexio/reflexio/README.md +7 -3
- package/plugin/vendor/reflexio/reflexio/__init__.py +12 -0
- package/plugin/vendor/reflexio/reflexio/client/client.py +126 -3
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/state.py +10 -3
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_state.py +28 -0
- package/plugin/vendor/reflexio/reflexio/lib/_search.py +41 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/entities.py +177 -25
- package/plugin/vendor/reflexio/reflexio/models/api_schema/eval_overview_schema.py +7 -1
- package/plugin/vendor/reflexio/reflexio/models/api_schema/internal_schema.py +2 -1
- package/plugin/vendor/reflexio/reflexio/models/api_schema/retriever_schema.py +63 -4
- package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/converters.py +1 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/entities.py +8 -1
- package/plugin/vendor/reflexio/reflexio/models/config_schema.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/README.md +22 -4
- package/plugin/vendor/reflexio/reflexio/server/__init__.py +18 -8
- package/plugin/vendor/reflexio/reflexio/server/__main__.py +6 -0
- package/plugin/vendor/reflexio/reflexio/server/api.py +66 -3
- package/plugin/vendor/reflexio/reflexio/server/billing_meter.py +263 -3
- package/plugin/vendor/reflexio/reflexio/server/callback_executor.py +164 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_embedding.py +81 -81
- package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_subprocess.py +28 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_text_generation.py +62 -5
- package/plugin/vendor/reflexio/reflexio/server/llm/embedding_service.py +19 -52
- package/plugin/vendor/reflexio/reflexio/server/llm/model_defaults.py +28 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/claude_code_provider.py +9 -1
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/embedder_warmup.py +329 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/embedding_service_provider.py +85 -10
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/local_embedding_provider.py +20 -5
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/nomic_embedding_provider.py +77 -9
- package/plugin/vendor/reflexio/reflexio/server/org_fanout.py +184 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/document_expansion/v1.0.0.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/document_expansion/v1.1.0.prompt.md +32 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.3.3.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.4.0.prompt.md +63 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/query_reformulation/v1.0.0.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/query_reformulation/v2.0.0.prompt.md +30 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/retrieved_learning_impact/v1.0.0.prompt.md +51 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/retrieved_learning_relevance/v1.0.0.prompt.md +39 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/shadow_comparison/v1.0.0.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/shadow_comparison/v1.1.0.prompt.md +43 -0
- package/plugin/vendor/reflexio/reflexio/server/routes/config.py +3 -3
- package/plugin/vendor/reflexio/reflexio/server/routes/evaluation.py +122 -28
- package/plugin/vendor/reflexio/reflexio/server/routes/system.py +22 -3
- package/plugin/vendor/reflexio/reflexio/server/scheduling.py +64 -3
- package/plugin/vendor/reflexio/reflexio/server/services/README.md +6 -4
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/README.md +3 -2
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/_eval_health.py +41 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/components/retrieved_learning_evaluator.py +554 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/regen_jobs.py +35 -1
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/runner.py +253 -101
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/scheduler.py +5 -7
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/service.py +27 -0
- package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_extraction_lifecycle.py +11 -5
- package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_should_run.py +4 -4
- package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_usage_billing.py +6 -2
- package/plugin/vendor/reflexio/reflexio/server/services/base_generation_service.py +255 -74
- package/plugin/vendor/reflexio/reflexio/server/services/deduplication_utils.py +72 -0
- package/plugin/vendor/reflexio/reflexio/server/services/deferred_learning_plan.py +270 -0
- package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/worker.py +85 -16
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/components/hero_state.py +2 -11
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/components/rule_attribution.py +6 -2
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/service.py +17 -13
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/agent_run_records.py +18 -3
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/outcome.py +12 -1
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/prior_answer_search.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/resumable_agent.py +2 -1
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_worker.py +66 -0
- package/plugin/vendor/reflexio/reflexio/server/services/generation_service.py +510 -63
- package/plugin/vendor/reflexio/reflexio/server/services/governance/service.py +2 -0
- package/plugin/vendor/reflexio/reflexio/server/services/lineage/gc_scheduler.py +179 -99
- package/plugin/vendor/reflexio/reflexio/server/services/lineage/vector_backfill_sweep.py +139 -0
- package/plugin/vendor/reflexio/reflexio/server/services/operation_state_utils.py +66 -27
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/aggregation_trigger.py +177 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/aggregator.py +68 -2
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/consolidator.py +360 -49
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/extractor.py +27 -30
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/playbook_edit_apply.py +8 -1
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/service.py +137 -69
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/optimizer.py +20 -1
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/scheduler.py +13 -6
- package/plugin/vendor/reflexio/reflexio/server/services/pre_retrieval/_query_reformulator.py +40 -25
- package/plugin/vendor/reflexio/reflexio/server/services/profile/components/consolidator.py +12 -39
- package/plugin/vendor/reflexio/reflexio/server/services/profile/components/extractor.py +30 -35
- package/plugin/vendor/reflexio/reflexio/server/services/profile/service.py +122 -44
- package/plugin/vendor/reflexio/reflexio/server/services/reflection/service.py +457 -107
- package/plugin/vendor/reflexio/reflexio/server/services/retrieval/session_dedup.py +127 -0
- package/plugin/vendor/reflexio/reflexio/server/services/retrieval/temporal.py +104 -0
- package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/dispatcher.py +139 -0
- package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/judge.py +16 -6
- package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/worker.py +137 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/governance_validation.py +12 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/lifecycle_filters.py +54 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/retention.py +41 -3
- package/plugin/vendor/reflexio/reflexio/server/services/storage/retention_mixin.py +40 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/__init__.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_base.py +50 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_extras.py +11 -4
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_governance.py +28 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py +36 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_shadow_verdicts.py +4 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/base/_deletion.py +16 -5
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_erase_execution.py +104 -42
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_subject_barrier.py +7 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_agent.py +43 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_eval_results.py +430 -5
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_user.py +45 -7
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_interaction_store.py +146 -6
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_profile_store.py +42 -7
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/__init__.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_extras.py +9 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_learning_jobs.py +50 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_operations.py +7 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_retrieval_log.py +3 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_shadow_verdicts.py +4 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/agent_run/_models.py +12 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/evaluation_state_keys.py +78 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_agent.py +38 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_eval_results.py +171 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_user.py +52 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/_interaction_store.py +59 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/_profile_store.py +52 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/retrieved_learning_state.py +226 -0
- package/plugin/vendor/reflexio/reflexio/server/services/tagging/tagging_scheduler.py +5 -6
- package/plugin/vendor/reflexio/reflexio/server/services/unified_search_service.py +209 -29
- package/plugin/vendor/reflexio/reflexio/server/usage_metrics.py +3 -0
- package/plugin/vendor/reflexio/reflexio/test_support/llm_mock.py +61 -0
- package/plugin/vendor/reflexio/reflexio/test_support/llm_model_registry.py +33 -0
- package/plugin/vendor/reflexio/reflexio/server/services/search/__init__.py +0 -0
|
@@ -17,6 +17,7 @@ Description: FastAPI backend server that processes user interactions to generate
|
|
|
17
17
|
- [Profile Generation](#profile-generation)
|
|
18
18
|
- [Playbook Extraction](#playbook-extraction)
|
|
19
19
|
- [Agent Success Evaluation](#agent-success-evaluation)
|
|
20
|
+
- [Durable Learning Queue](#durable-learning-queue)
|
|
20
21
|
- [Reflection and Async Extraction](#reflection-and-async-extraction)
|
|
21
22
|
- [Shadow Comparison and Evaluation Overview](#shadow-comparison-and-evaluation-overview)
|
|
22
23
|
- [Playbook Optimizer and Braintrust](#playbook-optimizer-and-braintrust)
|
|
@@ -38,6 +39,7 @@ Description: FastAPI backend server that processes user interactions to generate
|
|
|
38
39
|
- **Endpoint Helpers**: `api_endpoints/` - Shared handlers/helpers plus `RequestContext` used by route modules
|
|
39
40
|
- **Extension Registry**: `extensions.py` - Capability and service registry for optional OSS/enterprise integrations
|
|
40
41
|
- **Core Service**: `services/generation_service.py` - Main orchestrator
|
|
42
|
+
- **Durable Learning**: `services/durable_learning/` - background queue worker for deferred post-publish extraction
|
|
41
43
|
|
|
42
44
|
## Cache
|
|
43
45
|
|
|
@@ -86,7 +88,7 @@ Description: FastAPI backend server that processes user interactions to generate
|
|
|
86
88
|
- **Health/version**: `GET /`, `GET /health`, `GET /healthz`, `GET /healthz/eval`, `GET /meta/version`
|
|
87
89
|
- **Identity/config**: `GET /api/whoami`, `GET /api/my_config`, `GET /api/get_config`, `POST /api/set_config`, `POST /api/update_config`
|
|
88
90
|
- **Publish/direct writes**: `POST /api/publish_interaction`, `POST /api/add_user_profile`, `POST /api/add_user_playbook`, `POST /api/add_agent_playbook`
|
|
89
|
-
- **Retrieval**: `POST /api/get_requests`, `POST /api/get_interactions`, `GET /api/get_all_interactions`, `POST /api/get_profiles`, `GET /api/get_all_profiles`, `POST /api/get_user_playbooks`, `POST /api/get_agent_playbooks`, `POST /api/get_agent_success_evaluation_results`
|
|
91
|
+
- **Retrieval**: `POST /api/get_requests`, `POST /api/get_interactions`, `GET /api/get_all_interactions`, `GET /api/learning_status`, `POST /api/get_profiles`, `GET /api/get_all_profiles`, `POST /api/get_user_playbooks`, `POST /api/get_agent_playbooks`, `POST /api/get_agent_success_evaluation_results`, `POST /api/get_retrieved_learning_evaluation_results`
|
|
90
92
|
- **Search/stats**: `POST /api/search`, `POST /api/search_profiles`, `POST /api/rerank_user_profiles`, `POST /api/search_interactions`, `POST /api/search_user_playbooks`, `POST /api/search_agent_playbooks`, `GET /api/storage_stats`, `GET /api/get_profile_statistics`, `POST /api/get_dashboard_stats`, `POST /api/get_playbook_application_stats`
|
|
91
93
|
- **Profile lifecycle**: `POST /api/rerun_profile_generation`, `POST /api/manual_profile_generation`, `POST /api/upgrade_all_profiles`, `POST /api/downgrade_all_profiles`, `GET /api/profile_change_log`, `PUT /api/update_user_profile`, `DELETE /api/delete_profile`, `DELETE /api/delete_profiles_by_ids`, `DELETE /api/delete_all_profiles`
|
|
92
94
|
- **Playbook lifecycle**: `POST /api/rerun_playbook_generation`, `POST /api/manual_playbook_generation`, `POST /api/run_playbook_aggregation`, `GET /api/playbook_aggregation_change_logs`, `POST /api/upgrade_all_user_playbooks`, `POST /api/downgrade_all_user_playbooks`, `PUT /api/update_agent_playbook_status`, `PUT /api/update_agent_playbook`, `PUT /api/update_user_playbook`, `DELETE /api/delete_agent_playbook`, `DELETE /api/delete_user_playbook`, `DELETE /api/delete_agent_playbooks_by_ids`, `DELETE /api/delete_user_playbooks_by_ids`, `DELETE /api/delete_all_playbooks`, `DELETE /api/delete_all_user_playbooks`, `DELETE /api/delete_all_agent_playbooks`
|
|
@@ -210,12 +212,14 @@ python -m reflexio.server.scripts.manage_invitation_codes list --show-used
|
|
|
210
212
|
- **Profile memory**: `profile/` extracts, deduplicates, and applies user profile updates.
|
|
211
213
|
- **Playbook memory**: `playbook/` extracts user playbooks, consolidates them against existing rows, aggregates them into agent playbooks, and tracks aggregation change logs.
|
|
212
214
|
- **Evaluation**: `agent_success_evaluation/service.py`, `agent_success_evaluation/runner.py`, `agent_success_evaluation/scheduler.py`, `agent_success_evaluation/components/evaluator.py`, `shadow_comparison/`, and `evaluation_overview/` handle session grading, per-turn shadow verdicts, regeneration jobs, and dashboard-facing rollups.
|
|
215
|
+
- **Durable learning queue**: `durable_learning/scheduler.py` and `durable_learning/worker.py` drain `learning_jobs` after deferred publishes and report coverage through `GET /api/learning_status`.
|
|
213
216
|
- **Async clarification**: `extraction/` and `reflection/` manage resumable agent runs, pending tool calls, prior-answer search, and long-horizon reflection updates.
|
|
214
217
|
- **Search preparation**: `pre_retrieval/` and `unified_search_service.py` handle query reformulation, document expansion, embeddings, and cross-entity search orchestration.
|
|
215
218
|
- **Optimization/integrations**: `playbook_optimizer/` and `braintrust/` run candidate playbook optimization, rollout support, and Braintrust export/sync.
|
|
216
219
|
- **Lineage**: `lineage/` resolves active records across superseded chains and schedules tombstone garbage collection for profile/playbook storage.
|
|
217
220
|
- **Governance**: `governance/` defines subject-reference contracts and retention/barrier policy helpers used by storage and lineage paths.
|
|
218
221
|
- **Persistence/config**: `storage/`, `configurator/`, and `operation_state_utils.py` provide storage abstractions, config loading, locks, bookmarks, progress, and cancellation.
|
|
222
|
+
- **Usage metering**: `billing_meter.py` converts learning/search signals into optional `usage_events` without importing enterprise types.
|
|
219
223
|
|
|
220
224
|
### Orchestrator
|
|
221
225
|
|
|
@@ -415,7 +419,9 @@ Key files:
|
|
|
415
419
|
- `agent_success_evaluation_constants.py`: Output schema (`AgentSuccessEvaluationOutput`)
|
|
416
420
|
- `agent_success_evaluation_utils.py`: Message construction utilities
|
|
417
421
|
- `scheduler.py`: `GroupEvaluationScheduler` singleton - min-heap priority queue with daemon thread, defers evaluation until 10 min after last request in session
|
|
418
|
-
- `runner.py`: `run_group_evaluation()` - fetches all requests/interactions for a session, builds `RequestInteractionDataModel` list, runs `service.py`
|
|
422
|
+
- `runner.py`: `run_group_evaluation()` - fetches all requests/interactions for a session, builds `RequestInteractionDataModel` list, runs `service.py`, then the retrieved-learning phase; returns `GroupEvaluationOutcome` (per-family statuses)
|
|
423
|
+
- `components/retrieved_learning_evaluator.py`: `RetrievedLearningEvaluator` - per-learning relevance/impact judges over `Interaction.retrieved_learnings`; results replace the session's `retrieved_learning_evaluation` snapshot atomically (generation + session-fingerprint fenced, see `services/storage/storage_base/retrieved_learning_state.py`)
|
|
424
|
+
- `services/storage/storage_base/evaluation_state_keys.py`: single source of truth for the three evaluation `_operation_state` key formats (agent-success marker, grade-on-demand cache, retrieved-learning state) shared by producers and governance erasure
|
|
419
425
|
|
|
420
426
|
**Flow**: Interactions → `agent_success_evaluation/scheduler.py` → `agent_success_evaluation/runner.py` → `agent_success_evaluation/service.py` → `agent_success_evaluation/components/evaluator.py` → `AgentSuccessEvaluationResult` → Storage
|
|
421
427
|
|
|
@@ -423,7 +429,18 @@ Key files:
|
|
|
423
429
|
|
|
424
430
|
**Tool Context**: Reads `tool_can_use` from root `Config` level (shared with playbook extraction).
|
|
425
431
|
|
|
426
|
-
**Shadow Comparison**: Session-level shadow comparison was retracted in F1 because multi-turn shadow content suffers from trajectory contamination (turn 2+ user messages react to the regular response, not the shadow). The `regular_vs_shadow` field on `AgentSuccessEvaluationResult` is preserved as a nullable historical column but is always `None` on newly produced rows. Per-turn shadow comparison lives in a dedicated `services/shadow_comparison/` judge that writes
|
|
432
|
+
**Shadow Comparison**: Session-level shadow comparison was retracted in F1 because multi-turn shadow content suffers from trajectory contamination (turn 2+ user messages react to the regular response, not the shadow). The `regular_vs_shadow` field on `AgentSuccessEvaluationResult` is preserved as a nullable historical column but is always `None` on newly produced rows. Per-turn shadow comparison is scheduled from the publish path whenever an assistant interaction carries `shadow_content`; it lives in a dedicated `services/shadow_comparison/` judge that writes verdicts to a separate table, independent of session-level evaluation sampling.
|
|
433
|
+
|
|
434
|
+
### Durable Learning Queue
|
|
435
|
+
|
|
436
|
+
**Directory**: `services/durable_learning/`
|
|
437
|
+
|
|
438
|
+
Key files:
|
|
439
|
+
- `scheduler.py`: `DurableLearningScheduler` plus `maybe_start_durable_learning()`; starts only when `REFLEXIO_DURABLE_LEARNING_QUEUE` is truthy and polls orgs with actionable queue rows.
|
|
440
|
+
- `worker.py`: `DurableLearningWorker`; claims leased jobs, reloads the persisted request, then splits each job into `compute_deferred_learning()` (LLM extraction + dedup + embeddings, **no** writer transaction held) → `persist_deferred_learning()` + fenced `complete_learning_job()` inside one short `storage.commit_scope()` → `emit_deferred_learning_side_effects()` post-commit (billing / telemetry / tagging / lock release).
|
|
441
|
+
- `services/storage/storage_base/_learning_jobs.py`: `LearningJobStoreABC`, queue status types, coverage-based request status, and the direct-storage contract implemented by each backend.
|
|
442
|
+
|
|
443
|
+
**Pattern**: `POST /api/publish_interaction` returns immediately when `wait_for_response=false`; callers use the returned `request_id` with `GET /api/learning_status`. Queue workers run the LLM compute **outside** any writer transaction; only the persist half + the fenced `complete_learning_job()` run inside `storage.commit_scope()`, and must raise/rollback if `complete_learning_job()` returns 0 because another worker stole the lease.
|
|
427
444
|
|
|
428
445
|
### Reflection and Async Extraction
|
|
429
446
|
|
|
@@ -444,7 +461,8 @@ Key files:
|
|
|
444
461
|
**Directories**: `services/shadow_comparison/`, `services/evaluation_overview/`
|
|
445
462
|
|
|
446
463
|
Key files:
|
|
447
|
-
- `shadow_comparison/judge.py`: Per-turn regular-vs-shadow judge
|
|
464
|
+
- `shadow_comparison/judge.py`: Per-turn regular-vs-shadow judge
|
|
465
|
+
- `shadow_comparison/dispatcher.py` and `shadow_comparison/worker.py`: Publish-time dispatch and bounded background execution for shadow verdict writes
|
|
448
466
|
- `shadow_comparison/outcome.py`: Verdict outcome model helpers
|
|
449
467
|
- `evaluation_overview/service.py`: Aggregates evaluation-page metrics
|
|
450
468
|
- `evaluation_overview/components/hero_state.py`, `evaluation_overview/components/distribution.py`, `evaluation_overview/components/rule_attribution.py`, `evaluation_overview/components/shadow_aggregation.py`: Focused aggregation helpers
|
|
@@ -230,13 +230,16 @@ if DEBUG_LOG_TO_CONSOLE:
|
|
|
230
230
|
logging.ERROR
|
|
231
231
|
)
|
|
232
232
|
else:
|
|
233
|
-
# Production (DEBUG_LOG_TO_CONSOLE off):
|
|
234
|
-
#
|
|
235
|
-
#
|
|
236
|
-
#
|
|
237
|
-
#
|
|
238
|
-
#
|
|
239
|
-
|
|
233
|
+
# Production (DEBUG_LOG_TO_CONSOLE off): default the root logger to WARNING.
|
|
234
|
+
# App-wide INFO on a busy server drove ~3x log volume and grew memory to the
|
|
235
|
+
# container ceiling over a few hours (prod incident 2026-07-04) — so INFO is
|
|
236
|
+
# OPT-IN per subsystem, not global. A stdout StreamHandler is still attached at
|
|
237
|
+
# INFO so that any logger explicitly raised to INFO reaches the container log
|
|
238
|
+
# driver (e.g. CloudWatch awslogs); everything else stays WARNING+ and cheap.
|
|
239
|
+
#
|
|
240
|
+
# REFLEXIO_INFO_LOGGERS: comma-separated logger-name prefixes to surface at INFO
|
|
241
|
+
# (e.g. the billing money-path). Empty/unset => WARNING-only (the safe default).
|
|
242
|
+
root_logger.setLevel(logging.WARNING)
|
|
240
243
|
if not any(isinstance(h, logging.StreamHandler) for h in root_logger.handlers):
|
|
241
244
|
prod_console_handler = logging.StreamHandler(sys.stdout)
|
|
242
245
|
prod_console_handler.setLevel(logging.INFO)
|
|
@@ -245,7 +248,14 @@ else:
|
|
|
245
248
|
)
|
|
246
249
|
root_logger.addHandler(prod_console_handler)
|
|
247
250
|
|
|
248
|
-
|
|
251
|
+
for _info_logger in (
|
|
252
|
+
name.strip()
|
|
253
|
+
for name in os.getenv("REFLEXIO_INFO_LOGGERS", "").split(",")
|
|
254
|
+
if name.strip()
|
|
255
|
+
):
|
|
256
|
+
logging.getLogger(_info_logger).setLevel(logging.INFO)
|
|
257
|
+
|
|
258
|
+
# Keep noisy loggers quiet regardless of the above (mirror the debug branch).
|
|
249
259
|
for _noisy in ("litellm", "LiteLLM", "httpx", "httpcore", "openai", "urllib3"):
|
|
250
260
|
logging.getLogger(_noisy).setLevel(logging.WARNING)
|
|
251
261
|
logging.getLogger("reflexio.server.site_var.site_var_manager").setLevel(
|
|
@@ -15,6 +15,7 @@ Flags mirror the subset of ``uvicorn`` CLI options that
|
|
|
15
15
|
from __future__ import annotations
|
|
16
16
|
|
|
17
17
|
import argparse
|
|
18
|
+
import os
|
|
18
19
|
import sys
|
|
19
20
|
|
|
20
21
|
import uvicorn
|
|
@@ -100,6 +101,11 @@ def main(argv: list[str] | None = None) -> None:
|
|
|
100
101
|
)
|
|
101
102
|
raise SystemExit(2)
|
|
102
103
|
|
|
104
|
+
# Record the configured worker count so per-worker startup guards can read
|
|
105
|
+
# it (uvicorn exposes no worker-count env inside a worker process). See
|
|
106
|
+
# ``embedder_warmup._detected_worker_count``.
|
|
107
|
+
os.environ["REFLEXIO_SERVER_WORKERS"] = str(1 if args.reload else args.workers)
|
|
108
|
+
|
|
103
109
|
if args.reload:
|
|
104
110
|
uvicorn.run(
|
|
105
111
|
args.app,
|
|
@@ -16,7 +16,7 @@ iterates (e.g. the QPS billable-endpoint scan over ``core_router.routes``).
|
|
|
16
16
|
|
|
17
17
|
import inspect
|
|
18
18
|
import logging
|
|
19
|
-
from collections.abc import Callable
|
|
19
|
+
from collections.abc import Callable, Iterable
|
|
20
20
|
from typing import TYPE_CHECKING
|
|
21
21
|
|
|
22
22
|
if TYPE_CHECKING:
|
|
@@ -77,6 +77,34 @@ __all__ = [
|
|
|
77
77
|
"limiter",
|
|
78
78
|
]
|
|
79
79
|
|
|
80
|
+
|
|
81
|
+
def _log_multi_worker_daemons() -> None:
|
|
82
|
+
"""Warn once at startup when multiple worker processes will run daemons.
|
|
83
|
+
|
|
84
|
+
``--workers N`` (N>1) runs the full daemon set in every worker process
|
|
85
|
+
(duplicate ticking). Safe by the concurrent-tick invariant — every daemon
|
|
86
|
+
tick is concurrent-safe — but worth one visible line (design D3).
|
|
87
|
+
|
|
88
|
+
Reuses ``embedder_warmup._detected_worker_count`` (checks
|
|
89
|
+
``REFLEXIO_SERVER_WORKERS`` then falls back to ``WEB_CONCURRENCY``)
|
|
90
|
+
instead of re-implementing a narrower env read here. When the count is
|
|
91
|
+
undetectable (``None``), this logs nothing — it cannot verify the count,
|
|
92
|
+
so it must not assume a single worker.
|
|
93
|
+
"""
|
|
94
|
+
from reflexio.server.llm.providers.embedder_warmup import (
|
|
95
|
+
_detected_worker_count,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
workers = _detected_worker_count()
|
|
99
|
+
if workers is not None and workers > 1:
|
|
100
|
+
logger.warning(
|
|
101
|
+
"event=multi_worker_daemons workers=%d — background daemons tick in "
|
|
102
|
+
"every worker process (duplicate ticking; safe: ticks are "
|
|
103
|
+
"concurrent-safe by design)",
|
|
104
|
+
workers,
|
|
105
|
+
)
|
|
106
|
+
|
|
107
|
+
|
|
80
108
|
# ``core_router`` stays an aggregator: it ``include_router``s every domain
|
|
81
109
|
# sub-router so ``core_router.routes`` still enumerates all data-plane handlers
|
|
82
110
|
# (enterprise QPS enforcement iterates this list). The include order is
|
|
@@ -269,6 +297,7 @@ def create_app(
|
|
|
269
297
|
capabilities: "CapabilityRegistry | None" = None,
|
|
270
298
|
app_context_factory: "Callable[[], AppContext] | None" = None,
|
|
271
299
|
profile: "DeploymentProfile | None" = None,
|
|
300
|
+
durable_org_ids_provider: Callable[[], Iterable[str]] | None = None,
|
|
272
301
|
) -> FastAPI:
|
|
273
302
|
"""Factory to create a FastAPI app.
|
|
274
303
|
|
|
@@ -310,6 +339,16 @@ def create_app(
|
|
|
310
339
|
provided it is the single source for router-group mounting, auth, and
|
|
311
340
|
data-plane lifespan gating (the legacy knobs still populate the derived
|
|
312
341
|
profile, so callers may pass either — they express the same thing).
|
|
342
|
+
durable_org_ids_provider: Optional zero-arg callable returning the org_ids
|
|
343
|
+
with actionable durable-learning work, threaded straight through to
|
|
344
|
+
``maybe_start_durable_learning(org_ids_provider=...)``. When None
|
|
345
|
+
(default) the single-ref bootstrap default is used, so behaviour is
|
|
346
|
+
byte-identical to before this parameter existed. It is a plain
|
|
347
|
+
injectable seam so a deployment (e.g. enterprise cross-ref fan-out) can
|
|
348
|
+
supply its own discovery WITHOUT this factory importing deployment
|
|
349
|
+
logic. It is only consulted when ``REFLEXIO_DURABLE_LEARNING_QUEUE`` is
|
|
350
|
+
on — when the flag is off ``maybe_start_durable_learning`` returns None
|
|
351
|
+
without ever calling the provider.
|
|
313
352
|
|
|
314
353
|
Returns:
|
|
315
354
|
Configured FastAPI application.
|
|
@@ -354,6 +393,9 @@ def create_app(
|
|
|
354
393
|
from reflexio.server.services.lineage.gc_scheduler import (
|
|
355
394
|
maybe_start_lineage_gc,
|
|
356
395
|
)
|
|
396
|
+
from reflexio.server.services.lineage.vector_backfill_sweep import (
|
|
397
|
+
install_missing_vector_backfill_sweep,
|
|
398
|
+
)
|
|
357
399
|
|
|
358
400
|
@asynccontextmanager
|
|
359
401
|
async def lifespan(app: FastAPI) -> AsyncIterator[None]: # noqa: ARG001
|
|
@@ -361,7 +403,21 @@ def create_app(
|
|
|
361
403
|
gc_scheduler = None
|
|
362
404
|
durable_learning_scheduler = None
|
|
363
405
|
started_caps: list = []
|
|
406
|
+
# D8 config guards + D5 warm-before-ready. Both are dormant unless the
|
|
407
|
+
# deployment has flipped to the in-process local embedder
|
|
408
|
+
# (REFLEXIO_EMBEDDING_PROVIDER=inprocess + local/* default); pre-flip
|
|
409
|
+
# this is a no-op and /health stays byte-for-byte unchanged. Run before
|
|
410
|
+
# the data-plane block so a gated deployment warms regardless of the
|
|
411
|
+
# mount profile (otherwise /health could 503 forever).
|
|
412
|
+
from reflexio.server.llm.providers.embedder_warmup import (
|
|
413
|
+
maybe_start_embedder_warmup,
|
|
414
|
+
run_startup_config_guards,
|
|
415
|
+
)
|
|
416
|
+
|
|
417
|
+
run_startup_config_guards()
|
|
418
|
+
maybe_start_embedder_warmup()
|
|
364
419
|
if mounts_data_plane:
|
|
420
|
+
_log_multi_worker_daemons()
|
|
365
421
|
log_publish_hardware_capacity()
|
|
366
422
|
validate_llm_availability()
|
|
367
423
|
from reflexio.server.llm.rerank import prewarm as _prewarm_cross_encoder
|
|
@@ -376,17 +432,24 @@ def create_app(
|
|
|
376
432
|
lambda org_id: RequestContext(org_id=org_id),
|
|
377
433
|
bootstrap_org_id=bootstrap_org_id,
|
|
378
434
|
)
|
|
435
|
+
# Register the missing-vector backfill sweep (opt-in via
|
|
436
|
+
# REFLEXIO_MISSING_VECTOR_BACKFILL_ENABLED) BEFORE starting the GC
|
|
437
|
+
# scheduler, so its per-org hook is visible when maybe_start_lineage_gc
|
|
438
|
+
# evaluates its start conditions. No-op when the flag is off.
|
|
439
|
+
install_missing_vector_backfill_sweep()
|
|
379
440
|
gc_scheduler = maybe_start_lineage_gc(
|
|
380
441
|
lambda org_id: RequestContext(org_id=org_id),
|
|
381
442
|
bootstrap_org_id=bootstrap_org_id,
|
|
382
443
|
)
|
|
383
444
|
# Durable learning drains the learning_jobs queue per org. Gated on
|
|
384
445
|
# REFLEXIO_DURABLE_LEARNING_QUEUE; the default provider discovers
|
|
385
|
-
# orgs-with-work via the bootstrap storage (single-ref).
|
|
386
|
-
#
|
|
446
|
+
# orgs-with-work via the bootstrap storage (single-ref). A deployment
|
|
447
|
+
# may inject its own discovery via ``durable_org_ids_provider`` (e.g.
|
|
448
|
+
# enterprise cross-ref fan-out); when None the single-ref default runs.
|
|
387
449
|
durable_learning_scheduler = maybe_start_durable_learning(
|
|
388
450
|
lambda org_id: RequestContext(org_id=org_id),
|
|
389
451
|
bootstrap_org_id=bootstrap_org_id,
|
|
452
|
+
org_ids_provider=durable_org_ids_provider,
|
|
390
453
|
)
|
|
391
454
|
try:
|
|
392
455
|
if capabilities is not None:
|
|
@@ -8,8 +8,15 @@ wrapper over the ``record_usage_event`` hook (which only enqueues). No DB I/O.
|
|
|
8
8
|
|
|
9
9
|
from __future__ import annotations
|
|
10
10
|
|
|
11
|
+
import logging
|
|
12
|
+
import uuid
|
|
13
|
+
from collections.abc import Mapping
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
11
16
|
from reflexio.server.usage_metrics import record_usage_event
|
|
12
17
|
|
|
18
|
+
logger = logging.getLogger(__name__)
|
|
19
|
+
|
|
13
20
|
_INTERNAL = (
|
|
14
21
|
"internal" # == BillingCallerType.INTERNAL.value (kept literal; OSS stays clean)
|
|
15
22
|
)
|
|
@@ -29,7 +36,10 @@ def record_extraction_tokens(
|
|
|
29
36
|
) -> None:
|
|
30
37
|
"""Emit the Learning cost facet — call only when extraction fired.
|
|
31
38
|
|
|
32
|
-
No-op when ``billing_input_tokens <= 0``.
|
|
39
|
+
No-op when ``billing_input_tokens <= 0``. Each call mints a fresh
|
|
40
|
+
``event_key=f"tok:{uuid4()}"`` so two token emits under the same
|
|
41
|
+
``request_id`` (e.g. profile + playbook extraction in one request) never
|
|
42
|
+
collapse into one billed event downstream.
|
|
33
43
|
|
|
34
44
|
Args:
|
|
35
45
|
org_id: Organisation identifier.
|
|
@@ -51,6 +61,7 @@ def record_extraction_tokens(
|
|
|
51
61
|
pipeline=pipeline,
|
|
52
62
|
request_id=request_id,
|
|
53
63
|
session_id=session_id,
|
|
64
|
+
event_key=f"tok:{uuid.uuid4()}",
|
|
54
65
|
count_value=billing_input_tokens,
|
|
55
66
|
prompt_tokens=prompt_tokens,
|
|
56
67
|
completion_tokens=completion_tokens,
|
|
@@ -68,12 +79,26 @@ def record_learnings_generated(
|
|
|
68
79
|
platform_llm: bool | None,
|
|
69
80
|
platform_storage: bool | None,
|
|
70
81
|
pipeline: str | None = None,
|
|
82
|
+
user_id: str | None = None,
|
|
71
83
|
request_id: str | None = None,
|
|
72
84
|
session_id: str | None = None,
|
|
85
|
+
source: str | None = None,
|
|
86
|
+
agent_version: str | None = None,
|
|
87
|
+
playbook_name: str | None = None,
|
|
88
|
+
entity_type: str | None = None,
|
|
89
|
+
metadata: Mapping[str, Any] | None = None,
|
|
73
90
|
) -> None:
|
|
74
91
|
"""Emit the Learning value facet — number of profiles/playbooks generated.
|
|
75
92
|
|
|
76
|
-
|
|
93
|
+
Documented FALLBACK for callers that genuinely lack a per-record id list
|
|
94
|
+
(e.g. dedup/consolidation can reduce the persisted count below the raw
|
|
95
|
+
extracted count, so there is no safe 1:1 id per unit of ``count``). Prefer
|
|
96
|
+
:func:`record_learnings_generated_records` whenever the caller has the
|
|
97
|
+
durable learning ids in scope. No-op when ``count <= 0``.
|
|
98
|
+
|
|
99
|
+
Emits a single event carrying a synthesized ``event_key=f"learn-batch:{uuid4()}"``
|
|
100
|
+
(distinct per call) so this aggregate event still has a dedup key, even
|
|
101
|
+
though it is not entity-backed.
|
|
77
102
|
|
|
78
103
|
Args:
|
|
79
104
|
org_id: Organisation identifier.
|
|
@@ -81,8 +106,14 @@ def record_learnings_generated(
|
|
|
81
106
|
platform_llm: True iff the platform supplies the LLM for this org.
|
|
82
107
|
platform_storage: True iff the platform supplies storage; None defers to rollup.
|
|
83
108
|
pipeline: Optional pipeline tag (e.g. ``"playbook"``).
|
|
109
|
+
user_id: Optional user ID tied to the generated learning.
|
|
84
110
|
request_id: Optional request correlation ID.
|
|
85
111
|
session_id: Optional session ID.
|
|
112
|
+
source: Optional metering source/path label.
|
|
113
|
+
agent_version: Optional agent version tied to the generated learning.
|
|
114
|
+
playbook_name: Optional playbook name for playbook learnings.
|
|
115
|
+
entity_type: Optional entity type (e.g. ``"profile"``).
|
|
116
|
+
metadata: Optional path-specific usage metadata.
|
|
86
117
|
"""
|
|
87
118
|
if count <= 0:
|
|
88
119
|
return
|
|
@@ -91,15 +122,237 @@ def record_learnings_generated(
|
|
|
91
122
|
event_name="learnings_generated",
|
|
92
123
|
event_category="learning",
|
|
93
124
|
pipeline=pipeline,
|
|
125
|
+
user_id=user_id,
|
|
94
126
|
request_id=request_id,
|
|
95
127
|
session_id=session_id,
|
|
128
|
+
source=source,
|
|
129
|
+
agent_version=agent_version,
|
|
130
|
+
playbook_name=playbook_name,
|
|
131
|
+
entity_type=entity_type,
|
|
132
|
+
event_key=f"learn-batch:{uuid.uuid4()}",
|
|
96
133
|
count_value=count,
|
|
97
134
|
platform_llm=platform_llm,
|
|
98
135
|
platform_storage=platform_storage,
|
|
99
136
|
caller_type=_INTERNAL,
|
|
137
|
+
metadata=metadata,
|
|
100
138
|
)
|
|
101
139
|
|
|
102
140
|
|
|
141
|
+
def record_learnings_generated_records(
|
|
142
|
+
*,
|
|
143
|
+
org_id: str,
|
|
144
|
+
learning_ids: list[str],
|
|
145
|
+
platform_llm: bool | None,
|
|
146
|
+
platform_storage: bool | None,
|
|
147
|
+
pipeline: str | None = None,
|
|
148
|
+
user_id: str | None = None,
|
|
149
|
+
request_id: str | None = None,
|
|
150
|
+
session_id: str | None = None,
|
|
151
|
+
source: str | None = None,
|
|
152
|
+
agent_version: str | None = None,
|
|
153
|
+
playbook_name: str | None = None,
|
|
154
|
+
entity_type: str | None = None,
|
|
155
|
+
metadata: Mapping[str, Any] | None = None,
|
|
156
|
+
) -> None:
|
|
157
|
+
"""Emit the Learning value facet — one event per generated learning record.
|
|
158
|
+
|
|
159
|
+
Entity-backed alternative to :func:`record_learnings_generated`: emits one
|
|
160
|
+
``learnings_generated`` event per id in ``learning_ids`` (``count_value=1``,
|
|
161
|
+
``event_key=f"learn:{entity_type}:{id}"``, ``entity_id=id``) instead of a
|
|
162
|
+
single aggregate event, so downstream dedup can key on the learning id.
|
|
163
|
+
The ``entity_type`` segment is required for collision-freedom: entity-backed
|
|
164
|
+
callers draw ids from separate autoincrement primary keys in separate
|
|
165
|
+
tables (e.g. ``user_playbook_id`` and ``agent_playbook_id`` both start at
|
|
166
|
+
1), so the same integer id can legitimately occur in two tables — without
|
|
167
|
+
the entity-type segment those would mint the same ``event_key`` and
|
|
168
|
+
collapse into one event downstream. When ``entity_type`` is falsy, a
|
|
169
|
+
stable ``"_"`` placeholder is used (``learn:_:{id}``) rather than emitting
|
|
170
|
+
``entity_type=None`` literally into the key. The summed ``count_value``
|
|
171
|
+
across the emitted events equals ``len(learning_ids)`` — unchanged from
|
|
172
|
+
the total a caller would have passed as ``count`` to
|
|
173
|
+
:func:`record_learnings_generated`.
|
|
174
|
+
|
|
175
|
+
Callers must pass real, durable ids — never fabricate one to pad the
|
|
176
|
+
list. No-op when ``learning_ids`` is empty.
|
|
177
|
+
|
|
178
|
+
Args:
|
|
179
|
+
org_id: Organisation identifier.
|
|
180
|
+
learning_ids: Ids of the learnings durably generated in this run
|
|
181
|
+
(e.g. ``profile_id`` / ``user_playbook_id`` / ``agent_playbook_id``).
|
|
182
|
+
platform_llm: True iff the platform supplies the LLM for this org.
|
|
183
|
+
platform_storage: True iff the platform supplies storage; None defers to rollup.
|
|
184
|
+
pipeline: Optional pipeline tag (e.g. ``"playbook"``).
|
|
185
|
+
user_id: Optional user ID tied to the generated learning.
|
|
186
|
+
request_id: Optional request correlation ID.
|
|
187
|
+
session_id: Optional session ID.
|
|
188
|
+
source: Optional metering source/path label.
|
|
189
|
+
agent_version: Optional agent version tied to the generated learning.
|
|
190
|
+
playbook_name: Optional playbook name for playbook learnings.
|
|
191
|
+
entity_type: Optional entity type (e.g. ``"profile"``).
|
|
192
|
+
metadata: Optional path-specific usage metadata (shared across events).
|
|
193
|
+
"""
|
|
194
|
+
key_entity_type = entity_type or "_"
|
|
195
|
+
for learning_id in learning_ids:
|
|
196
|
+
record_usage_event(
|
|
197
|
+
org_id=org_id,
|
|
198
|
+
event_name="learnings_generated",
|
|
199
|
+
event_category="learning",
|
|
200
|
+
pipeline=pipeline,
|
|
201
|
+
user_id=user_id,
|
|
202
|
+
request_id=request_id,
|
|
203
|
+
session_id=session_id,
|
|
204
|
+
source=source,
|
|
205
|
+
agent_version=agent_version,
|
|
206
|
+
playbook_name=playbook_name,
|
|
207
|
+
entity_type=entity_type,
|
|
208
|
+
entity_id=learning_id,
|
|
209
|
+
event_key=f"learn:{key_entity_type}:{learning_id}",
|
|
210
|
+
count_value=1,
|
|
211
|
+
platform_llm=platform_llm,
|
|
212
|
+
platform_storage=platform_storage,
|
|
213
|
+
caller_type=_INTERNAL,
|
|
214
|
+
metadata=metadata,
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def emit_learnings_generated(
|
|
219
|
+
*,
|
|
220
|
+
org_id: str,
|
|
221
|
+
configurator: Any,
|
|
222
|
+
count: int,
|
|
223
|
+
source: str,
|
|
224
|
+
pipeline: str | None = None,
|
|
225
|
+
user_id: str | None = None,
|
|
226
|
+
request_id: str | None = None,
|
|
227
|
+
agent_version: str | None = None,
|
|
228
|
+
playbook_name: str | None = None,
|
|
229
|
+
entity_type: str | None = None,
|
|
230
|
+
metadata: Mapping[str, Any] | None = None,
|
|
231
|
+
) -> None:
|
|
232
|
+
"""Resolve ``platform_llm`` from config and emit the Learning value facet.
|
|
233
|
+
|
|
234
|
+
Convenience wrapper for the non-extraction learning-mutation paths (reflection,
|
|
235
|
+
resumable-extraction finalization, aggregation, offline-tuner auto-apply). It
|
|
236
|
+
owns the ``configurator.get_config()`` + ``platform_llm_from_config`` lookup so
|
|
237
|
+
each call site stays a thin one-liner, and — critically — is **guarded**: the
|
|
238
|
+
product path must never fail because metering failed, so config resolution and
|
|
239
|
+
emission are wrapped and any exception is logged and swallowed (mirroring the
|
|
240
|
+
extraction path's ``_record_billing_learning_events``). No-op when
|
|
241
|
+
``count <= 0``.
|
|
242
|
+
|
|
243
|
+
Args:
|
|
244
|
+
org_id: Organisation identifier.
|
|
245
|
+
configurator: Object exposing ``get_config()`` for platform-LLM resolution.
|
|
246
|
+
count: Number of learnings durably produced by this path.
|
|
247
|
+
source: Metering source/path label (e.g. ``"reflection"``).
|
|
248
|
+
pipeline: Optional pipeline tag (e.g. ``"playbook"``).
|
|
249
|
+
user_id: Optional user ID tied to the generated learning.
|
|
250
|
+
request_id: Optional request correlation ID.
|
|
251
|
+
agent_version: Optional agent version tied to the generated learning.
|
|
252
|
+
playbook_name: Optional playbook name for playbook learnings.
|
|
253
|
+
entity_type: Optional entity type (e.g. ``"profile"``).
|
|
254
|
+
metadata: Optional path-specific usage metadata.
|
|
255
|
+
"""
|
|
256
|
+
if count <= 0:
|
|
257
|
+
return
|
|
258
|
+
try:
|
|
259
|
+
from reflexio.server.billing_signals import platform_llm_from_config
|
|
260
|
+
|
|
261
|
+
config = configurator.get_config()
|
|
262
|
+
record_learnings_generated(
|
|
263
|
+
org_id=org_id,
|
|
264
|
+
count=count,
|
|
265
|
+
platform_llm=platform_llm_from_config(config),
|
|
266
|
+
platform_storage=None,
|
|
267
|
+
pipeline=pipeline,
|
|
268
|
+
user_id=user_id,
|
|
269
|
+
request_id=request_id,
|
|
270
|
+
source=source,
|
|
271
|
+
agent_version=agent_version,
|
|
272
|
+
playbook_name=playbook_name,
|
|
273
|
+
entity_type=entity_type,
|
|
274
|
+
metadata=metadata,
|
|
275
|
+
)
|
|
276
|
+
except Exception:
|
|
277
|
+
logger.warning(
|
|
278
|
+
"emit_learnings_generated failed for source=%s org=%s; "
|
|
279
|
+
"learnings_generated event not emitted",
|
|
280
|
+
source,
|
|
281
|
+
org_id,
|
|
282
|
+
exc_info=True,
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def emit_learnings_generated_records(
|
|
287
|
+
*,
|
|
288
|
+
org_id: str,
|
|
289
|
+
configurator: Any,
|
|
290
|
+
learning_ids: list[str],
|
|
291
|
+
source: str,
|
|
292
|
+
pipeline: str | None = None,
|
|
293
|
+
user_id: str | None = None,
|
|
294
|
+
request_id: str | None = None,
|
|
295
|
+
agent_version: str | None = None,
|
|
296
|
+
playbook_name: str | None = None,
|
|
297
|
+
entity_type: str | None = None,
|
|
298
|
+
metadata: Mapping[str, Any] | None = None,
|
|
299
|
+
) -> None:
|
|
300
|
+
"""Resolve ``platform_llm`` from config and emit one event per learning id.
|
|
301
|
+
|
|
302
|
+
Entity-backed counterpart to :func:`emit_learnings_generated`, currently
|
|
303
|
+
adopted by two of the non-extraction learning-mutation paths —
|
|
304
|
+
resumable-extraction finalization and aggregation — the callers with
|
|
305
|
+
durable per-record ids in scope. Extraction, reflection, and offline-tuner
|
|
306
|
+
auto-apply do not have a safe 1:1 id per unit of count (see
|
|
307
|
+
:func:`record_learnings_generated_records`) and use the count-based
|
|
308
|
+
:func:`emit_learnings_generated` fallback instead. Same guard semantics:
|
|
309
|
+
config resolution and emission are wrapped and any exception is logged
|
|
310
|
+
and swallowed — the product path must never fail because metering
|
|
311
|
+
failed. No-op when ``learning_ids`` is empty.
|
|
312
|
+
|
|
313
|
+
Args:
|
|
314
|
+
org_id: Organisation identifier.
|
|
315
|
+
configurator: Object exposing ``get_config()`` for platform-LLM resolution.
|
|
316
|
+
learning_ids: Ids of the learnings durably produced by this path.
|
|
317
|
+
source: Metering source/path label (e.g. ``"reflection"``).
|
|
318
|
+
pipeline: Optional pipeline tag (e.g. ``"playbook"``).
|
|
319
|
+
user_id: Optional user ID tied to the generated learning.
|
|
320
|
+
request_id: Optional request correlation ID.
|
|
321
|
+
agent_version: Optional agent version tied to the generated learning.
|
|
322
|
+
playbook_name: Optional playbook name for playbook learnings.
|
|
323
|
+
entity_type: Optional entity type (e.g. ``"profile"``).
|
|
324
|
+
metadata: Optional path-specific usage metadata (shared across events).
|
|
325
|
+
"""
|
|
326
|
+
if not learning_ids:
|
|
327
|
+
return
|
|
328
|
+
try:
|
|
329
|
+
from reflexio.server.billing_signals import platform_llm_from_config
|
|
330
|
+
|
|
331
|
+
config = configurator.get_config()
|
|
332
|
+
record_learnings_generated_records(
|
|
333
|
+
org_id=org_id,
|
|
334
|
+
learning_ids=learning_ids,
|
|
335
|
+
platform_llm=platform_llm_from_config(config),
|
|
336
|
+
platform_storage=None,
|
|
337
|
+
pipeline=pipeline,
|
|
338
|
+
user_id=user_id,
|
|
339
|
+
request_id=request_id,
|
|
340
|
+
source=source,
|
|
341
|
+
agent_version=agent_version,
|
|
342
|
+
playbook_name=playbook_name,
|
|
343
|
+
entity_type=entity_type,
|
|
344
|
+
metadata=metadata,
|
|
345
|
+
)
|
|
346
|
+
except Exception:
|
|
347
|
+
logger.warning(
|
|
348
|
+
"emit_learnings_generated_records failed for source=%s org=%s; "
|
|
349
|
+
"learnings_generated events not emitted",
|
|
350
|
+
source,
|
|
351
|
+
org_id,
|
|
352
|
+
exc_info=True,
|
|
353
|
+
)
|
|
354
|
+
|
|
355
|
+
|
|
103
356
|
def record_applied_learnings(
|
|
104
357
|
*,
|
|
105
358
|
org_id: str,
|
|
@@ -114,6 +367,8 @@ def record_applied_learnings(
|
|
|
114
367
|
"""Emit the Application line — surfaced top-K learnings.
|
|
115
368
|
|
|
116
369
|
No-op unless ``caller_type == "production_agent"`` AND ``surfaced_count > 0``.
|
|
370
|
+
Each call mints a fresh ``event_key=f"applied:{uuid4()}"`` — a distinct
|
|
371
|
+
key per search-response moment, never collapsed by ``request_id``.
|
|
117
372
|
|
|
118
373
|
Args:
|
|
119
374
|
org_id: Organisation identifier.
|
|
@@ -134,6 +389,7 @@ def record_applied_learnings(
|
|
|
134
389
|
pipeline=pipeline,
|
|
135
390
|
request_id=request_id,
|
|
136
391
|
session_id=session_id,
|
|
392
|
+
event_key=f"applied:{uuid.uuid4()}",
|
|
137
393
|
count_value=surfaced_count,
|
|
138
394
|
platform_llm=platform_llm,
|
|
139
395
|
platform_storage=platform_storage,
|
|
@@ -152,7 +408,10 @@ def record_search_request(
|
|
|
152
408
|
|
|
153
409
|
No-op unless ``caller_type == "production_agent"``. Unlike
|
|
154
410
|
:func:`record_applied_learnings`, empty search responses still count because
|
|
155
|
-
this measures requests made, not learnings surfaced.
|
|
411
|
+
this measures requests made, not learnings surfaced. Each call mints a
|
|
412
|
+
fresh ``event_key=f"search:{uuid4()}"`` — a distinct key per request, so
|
|
413
|
+
two searches under the same ``request_id`` are never collapsed into one
|
|
414
|
+
billed event downstream.
|
|
156
415
|
|
|
157
416
|
Args:
|
|
158
417
|
org_id: Organisation identifier.
|
|
@@ -168,6 +427,7 @@ def record_search_request(
|
|
|
168
427
|
event_category="application",
|
|
169
428
|
request_id=request_id,
|
|
170
429
|
session_id=session_id,
|
|
430
|
+
event_key=f"search:{uuid.uuid4()}",
|
|
171
431
|
count_value=1,
|
|
172
432
|
caller_type=caller_type,
|
|
173
433
|
)
|