claude-smart 0.2.49 → 0.2.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (177) hide show
  1. package/.claude-plugin/marketplace.json +2 -2
  2. package/README.md +10 -43
  3. package/bin/claude-smart.js +105 -0
  4. package/package.json +3 -2
  5. package/plugin/.claude-plugin/plugin.json +1 -1
  6. package/plugin/.codex-plugin/plugin.json +1 -1
  7. package/plugin/.coverage +0 -0
  8. package/plugin/README.md +4 -3
  9. package/plugin/dashboard/app/api/reflexio/[...path]/route.ts +4 -2
  10. package/plugin/dashboard/app/dashboard/page.tsx +6 -1
  11. package/plugin/dashboard/app/preferences/[id]/page.tsx +18 -6
  12. package/plugin/dashboard/app/preferences/page.tsx +32 -35
  13. package/plugin/dashboard/app/sessions/[sessionId]/page.tsx +16 -1
  14. package/plugin/dashboard/app/sessions/page.tsx +2 -0
  15. package/plugin/dashboard/app/skills/page.tsx +65 -50
  16. package/plugin/dashboard/app/skills/project/[id]/page.tsx +17 -8
  17. package/plugin/dashboard/app/skills/shared/[id]/page.tsx +1 -6
  18. package/plugin/dashboard/components/common/host-badge.tsx +118 -0
  19. package/plugin/dashboard/components/common/learning-application-badge.tsx +34 -0
  20. package/plugin/dashboard/components/common/learnings-badge.tsx +1 -1
  21. package/plugin/dashboard/components/common/page-header.tsx +3 -3
  22. package/plugin/dashboard/lib/config-file.ts +5 -1
  23. package/plugin/dashboard/lib/host-attribution.ts +62 -0
  24. package/plugin/dashboard/lib/session-reader.ts +40 -2
  25. package/plugin/dashboard/lib/types.ts +7 -1
  26. package/plugin/pyproject.toml +1 -1
  27. package/plugin/scripts/_lib.sh +197 -1
  28. package/plugin/scripts/backend-python-runner.py +46 -0
  29. package/plugin/scripts/backend-service.sh +757 -119
  30. package/plugin/scripts/codex-hook.js +63 -225
  31. package/plugin/scripts/dashboard-open.sh +6 -4
  32. package/plugin/scripts/dashboard-service.sh +117 -136
  33. package/plugin/scripts/hook_entry.sh +3 -0
  34. package/plugin/scripts/smart-install.sh +15 -1
  35. package/plugin/src/claude_smart/cli.py +14 -0
  36. package/plugin/src/claude_smart/context_inject.py +3 -0
  37. package/plugin/src/claude_smart/env_config.py +4 -1
  38. package/plugin/src/claude_smart/events/post_tool.py +2 -1
  39. package/plugin/src/claude_smart/events/session_end.py +2 -1
  40. package/plugin/src/claude_smart/events/stop.py +3 -0
  41. package/plugin/src/claude_smart/events/user_prompt.py +2 -1
  42. package/plugin/src/claude_smart/internal_call.py +5 -2
  43. package/plugin/src/claude_smart/optimizer_assistant.py +59 -13
  44. package/plugin/src/claude_smart/publish.py +59 -7
  45. package/plugin/src/claude_smart/reflexio_adapter.py +137 -14
  46. package/plugin/src/claude_smart/runtime.py +15 -6
  47. package/plugin/src/claude_smart/state.py +211 -52
  48. package/plugin/uv.lock +1 -1
  49. package/plugin/vendor/reflexio/.env.example +13 -0
  50. package/plugin/vendor/reflexio/reflexio/README.md +7 -3
  51. package/plugin/vendor/reflexio/reflexio/__init__.py +12 -0
  52. package/plugin/vendor/reflexio/reflexio/client/client.py +126 -3
  53. package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/state.py +10 -3
  54. package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_state.py +28 -0
  55. package/plugin/vendor/reflexio/reflexio/lib/_search.py +41 -0
  56. package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/entities.py +177 -25
  57. package/plugin/vendor/reflexio/reflexio/models/api_schema/eval_overview_schema.py +7 -1
  58. package/plugin/vendor/reflexio/reflexio/models/api_schema/internal_schema.py +2 -1
  59. package/plugin/vendor/reflexio/reflexio/models/api_schema/retriever_schema.py +63 -4
  60. package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/converters.py +1 -0
  61. package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/entities.py +8 -1
  62. package/plugin/vendor/reflexio/reflexio/models/config_schema.py +1 -1
  63. package/plugin/vendor/reflexio/reflexio/server/README.md +22 -4
  64. package/plugin/vendor/reflexio/reflexio/server/__init__.py +18 -8
  65. package/plugin/vendor/reflexio/reflexio/server/__main__.py +6 -0
  66. package/plugin/vendor/reflexio/reflexio/server/api.py +66 -3
  67. package/plugin/vendor/reflexio/reflexio/server/billing_meter.py +263 -3
  68. package/plugin/vendor/reflexio/reflexio/server/callback_executor.py +164 -0
  69. package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_embedding.py +81 -81
  70. package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_subprocess.py +28 -0
  71. package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_text_generation.py +62 -5
  72. package/plugin/vendor/reflexio/reflexio/server/llm/embedding_service.py +19 -52
  73. package/plugin/vendor/reflexio/reflexio/server/llm/model_defaults.py +28 -0
  74. package/plugin/vendor/reflexio/reflexio/server/llm/providers/claude_code_provider.py +9 -1
  75. package/plugin/vendor/reflexio/reflexio/server/llm/providers/embedder_warmup.py +329 -0
  76. package/plugin/vendor/reflexio/reflexio/server/llm/providers/embedding_service_provider.py +85 -10
  77. package/plugin/vendor/reflexio/reflexio/server/llm/providers/local_embedding_provider.py +20 -5
  78. package/plugin/vendor/reflexio/reflexio/server/llm/providers/nomic_embedding_provider.py +77 -9
  79. package/plugin/vendor/reflexio/reflexio/server/org_fanout.py +184 -0
  80. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/document_expansion/v1.0.0.prompt.md +1 -1
  81. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/document_expansion/v1.1.0.prompt.md +32 -0
  82. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.3.3.prompt.md +1 -1
  83. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.4.0.prompt.md +63 -0
  84. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/query_reformulation/v1.0.0.prompt.md +1 -1
  85. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/query_reformulation/v2.0.0.prompt.md +30 -0
  86. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/retrieved_learning_impact/v1.0.0.prompt.md +51 -0
  87. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/retrieved_learning_relevance/v1.0.0.prompt.md +39 -0
  88. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/shadow_comparison/v1.0.0.prompt.md +1 -1
  89. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/shadow_comparison/v1.1.0.prompt.md +43 -0
  90. package/plugin/vendor/reflexio/reflexio/server/routes/config.py +3 -3
  91. package/plugin/vendor/reflexio/reflexio/server/routes/evaluation.py +122 -28
  92. package/plugin/vendor/reflexio/reflexio/server/routes/system.py +22 -3
  93. package/plugin/vendor/reflexio/reflexio/server/scheduling.py +64 -3
  94. package/plugin/vendor/reflexio/reflexio/server/services/README.md +6 -4
  95. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/README.md +3 -2
  96. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/_eval_health.py +41 -0
  97. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/components/retrieved_learning_evaluator.py +554 -0
  98. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/regen_jobs.py +35 -1
  99. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/runner.py +253 -101
  100. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/scheduler.py +5 -7
  101. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/service.py +27 -0
  102. package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_extraction_lifecycle.py +11 -5
  103. package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_should_run.py +4 -4
  104. package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_usage_billing.py +6 -2
  105. package/plugin/vendor/reflexio/reflexio/server/services/base_generation_service.py +255 -74
  106. package/plugin/vendor/reflexio/reflexio/server/services/deduplication_utils.py +72 -0
  107. package/plugin/vendor/reflexio/reflexio/server/services/deferred_learning_plan.py +270 -0
  108. package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/worker.py +85 -16
  109. package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/components/hero_state.py +2 -11
  110. package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/components/rule_attribution.py +6 -2
  111. package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/service.py +17 -13
  112. package/plugin/vendor/reflexio/reflexio/server/services/extraction/agent_run_records.py +18 -3
  113. package/plugin/vendor/reflexio/reflexio/server/services/extraction/outcome.py +12 -1
  114. package/plugin/vendor/reflexio/reflexio/server/services/extraction/prior_answer_search.py +1 -1
  115. package/plugin/vendor/reflexio/reflexio/server/services/extraction/resumable_agent.py +2 -1
  116. package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_worker.py +66 -0
  117. package/plugin/vendor/reflexio/reflexio/server/services/generation_service.py +510 -63
  118. package/plugin/vendor/reflexio/reflexio/server/services/governance/service.py +2 -0
  119. package/plugin/vendor/reflexio/reflexio/server/services/lineage/gc_scheduler.py +179 -99
  120. package/plugin/vendor/reflexio/reflexio/server/services/lineage/vector_backfill_sweep.py +139 -0
  121. package/plugin/vendor/reflexio/reflexio/server/services/operation_state_utils.py +66 -27
  122. package/plugin/vendor/reflexio/reflexio/server/services/playbook/aggregation_trigger.py +177 -0
  123. package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/aggregator.py +68 -2
  124. package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/consolidator.py +360 -49
  125. package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/extractor.py +27 -30
  126. package/plugin/vendor/reflexio/reflexio/server/services/playbook/playbook_edit_apply.py +8 -1
  127. package/plugin/vendor/reflexio/reflexio/server/services/playbook/service.py +137 -69
  128. package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/optimizer.py +20 -1
  129. package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/scheduler.py +13 -6
  130. package/plugin/vendor/reflexio/reflexio/server/services/pre_retrieval/_query_reformulator.py +40 -25
  131. package/plugin/vendor/reflexio/reflexio/server/services/profile/components/consolidator.py +12 -39
  132. package/plugin/vendor/reflexio/reflexio/server/services/profile/components/extractor.py +30 -35
  133. package/plugin/vendor/reflexio/reflexio/server/services/profile/service.py +122 -44
  134. package/plugin/vendor/reflexio/reflexio/server/services/reflection/service.py +457 -107
  135. package/plugin/vendor/reflexio/reflexio/server/services/retrieval/session_dedup.py +127 -0
  136. package/plugin/vendor/reflexio/reflexio/server/services/retrieval/temporal.py +104 -0
  137. package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/dispatcher.py +139 -0
  138. package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/judge.py +16 -6
  139. package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/worker.py +137 -0
  140. package/plugin/vendor/reflexio/reflexio/server/services/storage/governance_validation.py +12 -0
  141. package/plugin/vendor/reflexio/reflexio/server/services/storage/lifecycle_filters.py +54 -0
  142. package/plugin/vendor/reflexio/reflexio/server/services/storage/retention.py +41 -3
  143. package/plugin/vendor/reflexio/reflexio/server/services/storage/retention_mixin.py +40 -2
  144. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/__init__.py +1 -1
  145. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_base.py +50 -0
  146. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_extras.py +11 -4
  147. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_governance.py +28 -1
  148. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py +36 -1
  149. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_shadow_verdicts.py +4 -0
  150. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/base/_deletion.py +16 -5
  151. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_erase_execution.py +104 -42
  152. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_subject_barrier.py +7 -1
  153. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_agent.py +43 -0
  154. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_eval_results.py +430 -5
  155. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_user.py +45 -7
  156. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_interaction_store.py +146 -6
  157. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_profile_store.py +42 -7
  158. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/__init__.py +1 -1
  159. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_extras.py +9 -2
  160. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_learning_jobs.py +50 -0
  161. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_operations.py +7 -0
  162. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_retrieval_log.py +3 -1
  163. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_shadow_verdicts.py +4 -0
  164. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/agent_run/_models.py +12 -1
  165. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/evaluation_state_keys.py +78 -0
  166. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_agent.py +38 -0
  167. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_eval_results.py +171 -0
  168. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_user.py +52 -1
  169. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/_interaction_store.py +59 -0
  170. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/_profile_store.py +52 -2
  171. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/retrieved_learning_state.py +226 -0
  172. package/plugin/vendor/reflexio/reflexio/server/services/tagging/tagging_scheduler.py +5 -6
  173. package/plugin/vendor/reflexio/reflexio/server/services/unified_search_service.py +209 -29
  174. package/plugin/vendor/reflexio/reflexio/server/usage_metrics.py +3 -0
  175. package/plugin/vendor/reflexio/reflexio/test_support/llm_mock.py +61 -0
  176. package/plugin/vendor/reflexio/reflexio/test_support/llm_model_registry.py +33 -0
  177. package/plugin/vendor/reflexio/reflexio/server/services/search/__init__.py +0 -0
@@ -8,6 +8,7 @@ from dataclasses import dataclass
8
8
 
9
9
  DEFAULT_ROW_RETENTION_LIMIT = 250_000
10
10
  ROW_RETENTION_DELETE_FRACTION = 0.20
11
+ TOMBSTONE_STATUSES = ("archived", "merged", "superseded", "expired")
11
12
 
12
13
 
13
14
  @dataclass(frozen=True, slots=True)
@@ -18,17 +19,32 @@ class RetentionTarget:
18
19
  table_name: str
19
20
  order_column: str
20
21
  id_columns: tuple[str, ...]
22
+ priority_statuses: tuple[str, ...] = ()
21
23
 
22
24
 
23
25
  RETENTION_TARGETS: tuple[RetentionTarget, ...] = (
24
- RetentionTarget("profiles", "profiles", "created_at", ("profile_id",)),
26
+ RetentionTarget(
27
+ "profiles",
28
+ "profiles",
29
+ "created_at",
30
+ ("profile_id",),
31
+ priority_statuses=TOMBSTONE_STATUSES,
32
+ ),
25
33
  RetentionTarget("interactions", "interactions", "created_at", ("interaction_id",)),
26
34
  RetentionTarget("requests", "requests", "created_at", ("request_id",)),
27
35
  RetentionTarget(
28
- "user_playbooks", "user_playbooks", "created_at", ("user_playbook_id",)
36
+ "user_playbooks",
37
+ "user_playbooks",
38
+ "created_at",
39
+ ("user_playbook_id",),
40
+ priority_statuses=TOMBSTONE_STATUSES,
29
41
  ),
30
42
  RetentionTarget(
31
- "agent_playbooks", "agent_playbooks", "created_at", ("agent_playbook_id",)
43
+ "agent_playbooks",
44
+ "agent_playbooks",
45
+ "created_at",
46
+ ("agent_playbook_id",),
47
+ priority_statuses=TOMBSTONE_STATUSES,
32
48
  ),
33
49
  RetentionTarget(
34
50
  "agent_success_evaluation_result",
@@ -36,6 +52,25 @@ RETENTION_TARGETS: tuple[RetentionTarget, ...] = (
36
52
  "created_at",
37
53
  ("result_id",),
38
54
  ),
55
+ # Grouped session target: keyed on (user_id, session_id) — not result_id —
56
+ # so retention always removes whole session snapshots, never a partial
57
+ # per-learning subset. Rows within a session share created_at (earliest
58
+ # request timestamp), so ordering keeps groups adjacent. The session's
59
+ # retrieved-eval _operation_state row is intentionally left in place: it
60
+ # is content-free (digest + counters) and self-heals on the next
61
+ # publish/forced evaluation.
62
+ RetentionTarget(
63
+ "retrieved_learning_evaluation",
64
+ "retrieved_learning_evaluation",
65
+ "created_at",
66
+ ("user_id", "session_id"),
67
+ ),
68
+ RetentionTarget(
69
+ "offline_tuner_reward_label",
70
+ "offline_tuner_reward_label",
71
+ "label_created_at",
72
+ ("reward_label_id",),
73
+ ),
39
74
  RetentionTarget("share_links", "share_links", "created_at", ("id",)),
40
75
  RetentionTarget(
41
76
  "agent_playbook_source_user_playbooks",
@@ -117,6 +152,9 @@ RETENTION_CASCADES: dict[str, tuple[CascadeRef, ...]] = {
117
152
  "playbook_retrieval_logs": (
118
153
  CascadeRef("playbook_retrieval_log_items", "retrieval_log_id"),
119
154
  ),
155
+ "offline_tuner_reward_label": (
156
+ CascadeRef("offline_tuner_reward_label_target", "reward_label_id"),
157
+ ),
120
158
  }
121
159
 
122
160
 
@@ -98,12 +98,40 @@ class RetentionMixin(ABC):
98
98
  target = get_retention_target(target_name)
99
99
  if not self._retention_table_exists(target.table_name):
100
100
  return 0
101
- keys = self._retention_select_oldest_keys(target, count)
101
+ keys = self._retention_select_keys(target, count)
102
102
  if not keys:
103
103
  return 0
104
104
  self._retention_perform_delete(target, keys)
105
105
  return len(keys)
106
106
 
107
+ def _retention_select_keys(
108
+ self, target: RetentionTarget, count: int
109
+ ) -> list[tuple[Any, ...]]:
110
+ """Select tombstones first, then oldest rows when a target opts in.
111
+
112
+ The tombstone pass can never under-delete: the fallback select returns
113
+ ``min(count, table_rows)`` keys and ``seen`` only ever holds tombstones
114
+ already counted, so the union still reaches ``count`` whenever the
115
+ table holds that many rows.
116
+ """
117
+ if not target.priority_statuses:
118
+ return self._retention_select_oldest_keys(target, count)
119
+
120
+ keys = self._retention_select_oldest_keys(
121
+ target, count, statuses=target.priority_statuses
122
+ )
123
+ if len(keys) >= count:
124
+ return keys
125
+
126
+ seen = set(keys)
127
+ for key in self._retention_select_oldest_keys(target, count):
128
+ if key not in seen:
129
+ keys.append(key)
130
+ seen.add(key)
131
+ if len(keys) == count:
132
+ break
133
+ return keys
134
+
107
135
  def _retention_perform_delete(
108
136
  self, target: RetentionTarget, keys: list[tuple[Any, ...]]
109
137
  ) -> None:
@@ -130,13 +158,23 @@ class RetentionMixin(ABC):
130
158
 
131
159
  @abstractmethod
132
160
  def _retention_select_oldest_keys(
133
- self, target: RetentionTarget, count: int
161
+ self,
162
+ target: RetentionTarget,
163
+ count: int,
164
+ statuses: tuple[str, ...] | None = None,
134
165
  ) -> list[tuple[Any, ...]]:
135
166
  """Return up to ``count`` oldest key tuples for ``target``.
136
167
 
137
168
  Each key tuple contains values aligned with ``target.id_columns``.
138
169
  Ordering is by ``target.order_column`` ascending with the id
139
170
  columns as a stable tiebreaker.
171
+
172
+ Args:
173
+ target (RetentionTarget): Target whose keys are being selected.
174
+ count (int): Maximum number of key tuples to return.
175
+ statuses (tuple[str, ...] | None): When not None, restrict the
176
+ select to rows whose ``status`` is one of these values. An
177
+ empty tuple matches nothing (never every row).
140
178
  """
141
179
  raise NotImplementedError
142
180
 
@@ -1,4 +1,3 @@
1
- from ._learning_jobs import SQLiteLearningJobStoreMixin
2
1
  from ._base import (
3
2
  SQLiteStorageBase,
4
3
  _cosine_similarity,
@@ -10,6 +9,7 @@ from ._base import (
10
9
  )
11
10
  from ._extras import ExtrasMixin
12
11
  from ._governance import SQLiteGovernanceMixin
12
+ from ._learning_jobs import SQLiteLearningJobStoreMixin
13
13
  from ._lineage import SQLiteLineageMixin
14
14
  from ._operations import OperationMixin
15
15
  from ._requests import RequestMixin
@@ -30,6 +30,7 @@ from reflexio.models.api_schema.service_schemas import (
30
30
  ProfileTimeToLive,
31
31
  RegularVsShadow,
32
32
  Request,
33
+ RetrievedLearning,
33
34
  Status,
34
35
  ToolUsed,
35
36
  UserActionType,
@@ -456,6 +457,12 @@ def _row_to_interaction(row: sqlite3.Row) -> Interaction:
456
457
  if citations_raw and isinstance(citations_raw, list)
457
458
  else []
458
459
  )
460
+ retrieved_learnings_raw = _json_loads(d.get("retrieved_learnings"))
461
+ retrieved_learnings = (
462
+ [RetrievedLearning(**c) for c in retrieved_learnings_raw if isinstance(c, dict)]
463
+ if retrieved_learnings_raw and isinstance(retrieved_learnings_raw, list)
464
+ else []
465
+ )
459
466
  return Interaction(
460
467
  interaction_id=d["interaction_id"],
461
468
  user_id=d["user_id"],
@@ -471,6 +478,7 @@ def _row_to_interaction(row: sqlite3.Row) -> Interaction:
471
478
  expert_content=d.get("expert_content") or "",
472
479
  tools_used=tools_used,
473
480
  citations=citations,
481
+ retrieved_learnings=retrieved_learnings,
474
482
  )
475
483
 
476
484
 
@@ -662,6 +670,16 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
662
670
  api_key_config=self.api_key_config,
663
671
  )
664
672
  self.embedding_dimensions = EMBEDDING_DIMENSIONS
673
+ # Text-generation model for storage-time document expansion. The
674
+ # shared self.llm_client is pinned to the EMBEDDING model, which is
675
+ # not a chat model — expansion calls must override the model or they
676
+ # fail (e.g. local/minilm-l6-v2 cannot serve completions).
677
+ self._expansion_model_name = resolve_model_name(
678
+ ModelRole.GENERATION,
679
+ site_var_value=site_var.get("default_generation_model_name"),
680
+ config_override=llm_config.generation_model_name if llm_config else None,
681
+ api_key_config=self.api_key_config,
682
+ )
665
683
 
666
684
  litellm_config = LiteLLMConfig(
667
685
  model=self.embedding_model_name,
@@ -855,6 +873,14 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
855
873
  )
856
874
  self.conn.commit()
857
875
 
876
+ if "retrieved_learnings" not in columns:
877
+ logger.info("Adding retrieved_learnings column to interactions table.")
878
+ with self._lock:
879
+ self.conn.execute(
880
+ "ALTER TABLE interactions ADD COLUMN retrieved_learnings TEXT"
881
+ )
882
+ self.conn.commit()
883
+
858
884
  def _migrate_feedback_schema(self) -> None:
859
885
  """Drop old-schema feedback/playbook tables so _DDL can recreate them.
860
886
 
@@ -1630,6 +1656,7 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
1630
1656
  expander = DocumentExpander(
1631
1657
  llm_client=self.llm_client,
1632
1658
  prompt_manager=PromptManager(),
1659
+ model_name=self._expansion_model_name,
1633
1660
  )
1634
1661
  result = expander.expand(content)
1635
1662
  return result.expanded_text or None
@@ -1848,6 +1875,7 @@ CREATE TABLE IF NOT EXISTS interactions (
1848
1875
  expert_content TEXT NOT NULL DEFAULT '',
1849
1876
  tools_used TEXT,
1850
1877
  citations TEXT,
1878
+ retrieved_learnings TEXT,
1851
1879
  embedding TEXT
1852
1880
  );
1853
1881
  CREATE INDEX IF NOT EXISTS idx_interactions_user_id ON interactions(user_id);
@@ -1949,6 +1977,28 @@ CREATE INDEX IF NOT EXISTS idx_eval_agent_version_created_at_desc
1949
1977
  CREATE INDEX IF NOT EXISTS idx_eval_identity_created_at_desc
1950
1978
  ON agent_success_evaluation_result(user_id, session_id, evaluation_name, agent_version, created_at DESC);
1951
1979
 
1980
+ CREATE TABLE IF NOT EXISTS retrieved_learning_evaluation (
1981
+ result_id INTEGER PRIMARY KEY AUTOINCREMENT,
1982
+ user_id TEXT NOT NULL,
1983
+ session_id TEXT NOT NULL,
1984
+ agent_version TEXT NOT NULL DEFAULT '',
1985
+ kind TEXT NOT NULL,
1986
+ learning_id TEXT NOT NULL,
1987
+ is_relevant INTEGER,
1988
+ relevance_reason TEXT NOT NULL DEFAULT '',
1989
+ impact TEXT,
1990
+ impact_reason TEXT NOT NULL DEFAULT '',
1991
+ created_at INTEGER NOT NULL,
1992
+ governance_subject_ref TEXT,
1993
+ UNIQUE (user_id, session_id, kind, learning_id)
1994
+ );
1995
+ CREATE INDEX IF NOT EXISTS idx_rle_created_at_result_id
1996
+ ON retrieved_learning_evaluation(created_at DESC, result_id DESC);
1997
+ CREATE INDEX IF NOT EXISTS idx_rle_session_id
1998
+ ON retrieved_learning_evaluation(session_id);
1999
+ CREATE INDEX IF NOT EXISTS idx_rle_subject_ref
2000
+ ON retrieved_learning_evaluation(governance_subject_ref);
2001
+
1952
2002
  CREATE TABLE IF NOT EXISTS agent_playbook_source_user_playbooks (
1953
2003
  agent_playbook_id INTEGER NOT NULL,
1954
2004
  user_playbook_id INTEGER NOT NULL,
@@ -25,7 +25,14 @@ from ._base import (
25
25
  _row_to_interaction,
26
26
  )
27
27
 
28
- type _CitationKind = Literal["playbook", "profile"]
28
+ type _CitationKind = Literal["playbook", "profile", "user_playbook", "agent_playbook"]
29
+
30
+ _SUPPORTED_CITATION_KINDS: tuple[_CitationKind, ...] = (
31
+ "playbook",
32
+ "profile",
33
+ "user_playbook",
34
+ "agent_playbook",
35
+ )
29
36
 
30
37
 
31
38
  class ExtrasMixin:
@@ -291,7 +298,7 @@ class ExtrasMixin:
291
298
  continue
292
299
  kind = c.get("kind")
293
300
  real_id = c.get("real_id")
294
- if kind not in ("playbook", "profile") or not real_id:
301
+ if kind not in _SUPPORTED_CITATION_KINDS or not real_id:
295
302
  continue
296
303
  key: tuple[_CitationKind, str] = (
297
304
  cast(_CitationKind, kind),
@@ -453,12 +460,12 @@ class ExtrasMixin:
453
460
  continue
454
461
  kind = citation.get("kind")
455
462
  real_id = citation.get("real_id")
456
- if kind and real_id:
463
+ if kind in _SUPPORTED_CITATION_KINDS and real_id:
457
464
  out.append(
458
465
  SessionCitation(
459
466
  user_id=row["user_id"],
460
467
  session_id=row["session_id"],
461
- kind=str(kind),
468
+ kind=cast(_CitationKind, kind),
462
469
  real_id=str(real_id),
463
470
  title=str(citation.get("title") or ""),
464
471
  )
@@ -392,11 +392,27 @@ class SQLiteGovernanceMixin:
392
392
  WHERE user_id = ?""",
393
393
  (user_id,),
394
394
  ).fetchone()
395
+ rle_result_row = self.conn.execute(
396
+ """SELECT COUNT(*) AS cnt
397
+ FROM retrieved_learning_evaluation
398
+ WHERE user_id = ?""",
399
+ (user_id,),
400
+ ).fetchone()
401
+ session_count_row = self.conn.execute(
402
+ "SELECT COUNT(DISTINCT session_id) AS cnt FROM requests WHERE user_id = ?",
403
+ (user_id,),
404
+ ).fetchone()
395
405
  profile_rows = self.conn.execute(
396
406
  "SELECT profile_id FROM profiles WHERE user_id = ?",
397
407
  (user_id,),
398
408
  ).fetchall()
399
- if request_row is None or interaction_row is None or eval_result_row is None:
409
+ if (
410
+ request_row is None
411
+ or interaction_row is None
412
+ or eval_result_row is None
413
+ or rle_result_row is None
414
+ or session_count_row is None
415
+ ):
400
416
  raise ValueError("Missing governance count rows")
401
417
  profile_ids = [str(row["profile_id"]) for row in profile_rows]
402
418
  purge_profile_ids, delete_profile_ids = self._deps()._partition_purge_vs_delete(
@@ -420,6 +436,17 @@ class SQLiteGovernanceMixin:
420
436
  "profile_purge": len(purge_profile_ids),
421
437
  "user_playbook": len(delete_playbook_ids),
422
438
  "agent_success_evaluation_result": int(eval_result_row["cnt"]),
439
+ # Offline tuner tables are enterprise-only (tenant stream); the
440
+ # OSS SQLite backend has no such tables, so the planned and
441
+ # deleted counts are structurally zero. The delete-target matrix
442
+ # validator still requires the target rows to exist.
443
+ "offline_tuner_reward_label": 0,
444
+ "offline_tuner_reward_label_target_by_target_owner": 0,
445
+ "retrieved_learning_evaluation_result": int(rle_result_row["cnt"]),
446
+ # Planned as an upper bound: up to 3 evaluation state namespaces
447
+ # per session (retrieved-eval state, agent-success marker,
448
+ # grade-cache rows). The delete phase reports the exact count.
449
+ "evaluation_operation_state": 3 * int(session_count_row["cnt"]),
423
450
  "user_playbook_purge": len(purge_playbook_ids),
424
451
  }
425
452
 
@@ -262,11 +262,18 @@ class SQLiteLearningJobStoreMixin(LearningJobStoreABC):
262
262
  job_id: str,
263
263
  claim_token: str,
264
264
  dead: bool,
265
+ refund_attempt: bool = False,
265
266
  ) -> None:
266
267
  """Fenced fail/dead transition — sets status, clears token for retry.
267
268
 
268
269
  Does NOT increment attempts: claim_learning_jobs already incremented on
269
270
  delivery. attempts tracks delivery count; fail only transitions status.
271
+
272
+ ``refund_attempt=True`` (the same-user-contention requeue, F4) decrements
273
+ ``attempts`` by one (``MAX(attempts - 1, 0)``) so a contention cycle
274
+ (claim +1, contention-release -1) nets to zero and ``attempts`` stays
275
+ bounded regardless of how many times the job loses the per-user race.
276
+ Defaults to ``False`` (the ``dead`` retry path is unchanged).
270
277
  """
271
278
  new_status = "dead" if dead else "failed"
272
279
  # Clear claim_token and claim_expires_at only for 'failed' so it's reclaimable.
@@ -282,10 +289,11 @@ class SQLiteLearningJobStoreMixin(LearningJobStoreABC):
282
289
  status = ?,
283
290
  claim_token = CASE WHEN ? THEN claim_token ELSE NULL END,
284
291
  claim_expires_at = CASE WHEN ? THEN claim_expires_at ELSE NULL END,
292
+ attempts = CASE WHEN ? THEN MAX(attempts - 1, 0) ELSE attempts END,
285
293
  updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
286
294
  WHERE job_id = ? AND claim_token = ? AND status = 'claimed'
287
295
  """,
288
- (new_status, dead, dead, job_id, claim_token),
296
+ (new_status, dead, dead, refund_attempt, job_id, claim_token),
289
297
  )
290
298
  if own_txn:
291
299
  self.conn.commit()
@@ -316,6 +324,33 @@ class SQLiteLearningJobStoreMixin(LearningJobStoreABC):
316
324
  )
317
325
  return [str(row["org_id"]) for row in rows]
318
326
 
327
+ @SQLiteStorageBase.handle_exceptions
328
+ def get_oldest_pending_learning_job_age_seconds(self) -> float | None:
329
+ """Age in seconds of the oldest pending job, or None if none pending.
330
+
331
+ Computes the age entirely in SQLite using ``strftime('%s','now')`` to
332
+ avoid app/DB clock skew. The table is accessed without a schema prefix
333
+ (SQLite has no schema routing — the local DB file is the ref).
334
+ """
335
+ row = self._fetchall(
336
+ """
337
+ SELECT (strftime('%s', 'now') - strftime('%s', MIN(created_at))) AS age_seconds
338
+ FROM learning_jobs
339
+ WHERE status = 'pending'
340
+ """,
341
+ (),
342
+ )
343
+ return None if row[0]["age_seconds"] is None else float(row[0]["age_seconds"])
344
+
345
+ @SQLiteStorageBase.handle_exceptions
346
+ def count_learning_jobs_by_status(self, status: str) -> int:
347
+ """Count of learning jobs with the given status on this storage ref."""
348
+ row = self._fetchall(
349
+ "SELECT COUNT(*) AS cnt FROM learning_jobs WHERE status = ?",
350
+ (status,),
351
+ )
352
+ return int(row[0]["cnt"])
353
+
319
354
  @SQLiteStorageBase.handle_exceptions
320
355
  def get_learning_status_for_request(
321
356
  self,
@@ -74,6 +74,10 @@ class ShadowVerdictsMixin:
74
74
  _fetchone: Any
75
75
  _fetchall: Any
76
76
 
77
+ def supports_shadow_comparison_verdicts(self) -> bool:
78
+ """Return whether this backend can persist shadow comparison verdicts."""
79
+ return True
80
+
77
81
  @SQLiteStorageBase.handle_exceptions
78
82
  def save_shadow_comparison_verdict(
79
83
  self, verdict: ShadowComparisonVerdict
@@ -46,14 +46,25 @@ class SQLiteDeletionMixin:
46
46
 
47
47
  @SQLiteStorageBase.handle_exceptions
48
48
  def _retention_select_oldest_keys(
49
- self, target: RetentionTarget, count: int
49
+ self,
50
+ target: RetentionTarget,
51
+ count: int,
52
+ statuses: tuple[str, ...] | None = None,
50
53
  ) -> list[tuple[Any, ...]]:
54
+ if statuses is not None and not statuses:
55
+ return []
51
56
  id_sql = ", ".join(target.id_columns)
52
- tiebreak_sql = id_sql
57
+ where_sql = ""
58
+ params: list[Any] = []
59
+ if statuses:
60
+ placeholders = ", ".join("?" for _ in statuses)
61
+ where_sql = f"WHERE status IN ({placeholders}) "
62
+ params.extend(statuses)
63
+ params.append(count)
53
64
  rows = self._fetchall(
54
- f"SELECT {id_sql} FROM {target.table_name} " # noqa: S608
55
- f"ORDER BY {target.order_column} ASC, {tiebreak_sql} ASC LIMIT ?",
56
- (count,),
65
+ f"SELECT {id_sql} FROM {target.table_name} {where_sql}" # noqa: S608
66
+ f"ORDER BY {target.order_column} ASC, {id_sql} ASC LIMIT ?",
67
+ tuple(params),
57
68
  )
58
69
  return [tuple(row[col] for col in target.id_columns) for row in rows]
59
70
 
@@ -30,7 +30,6 @@ resolve through the composed MRO.
30
30
 
31
31
  from __future__ import annotations
32
32
 
33
- import json
34
33
  import sqlite3
35
34
  import threading
36
35
  from collections.abc import Callable
@@ -50,6 +49,14 @@ from reflexio.server.services.storage.governance_validation import (
50
49
  _successful_erase_identity,
51
50
  _validate_governance_purge_id,
52
51
  )
52
+ from reflexio.server.services.storage.storage_base.evaluation_state_keys import (
53
+ GRADE_ON_DEMAND_CACHE_PREFIX,
54
+ build_agent_success_marker_key,
55
+ build_grade_on_demand_session_prefix,
56
+ )
57
+ from reflexio.server.services.storage.storage_base.retrieved_learning_state import (
58
+ build_retrieved_learning_state_key,
59
+ )
53
60
 
54
61
  from .._governance import _row_to_audit_event, _row_to_purge_operation
55
62
 
@@ -157,13 +164,6 @@ class GovernanceEraseExecutionMixin:
157
164
  raise ValueError(
158
165
  "Current user playbooks no longer match prepared purge snapshot"
159
166
  )
160
- request_ids = [
161
- str(row["request_id"])
162
- for row in self.conn.execute(
163
- "SELECT request_id FROM requests WHERE user_id = ?",
164
- (user_id,),
165
- ).fetchall()
166
- ]
167
167
  profile_rows = self.conn.execute(
168
168
  "SELECT rowid, profile_id FROM profiles WHERE user_id = ?",
169
169
  (user_id,),
@@ -183,40 +183,12 @@ class GovernanceEraseExecutionMixin:
183
183
  )
184
184
  purge_upb_ids = [int(entity_id) for entity_id in purge_upb_str_ids]
185
185
  delete_upb_ids = [int(entity_id) for entity_id in delete_upb_str_ids]
186
- erased_entity_ids = [
187
- *request_ids,
188
- *[str(interaction_id) for interaction_id in interaction_ids],
189
- *all_profile_ids,
190
- *[str(user_playbook_id) for user_playbook_id in raw_upb_ids],
191
- ]
192
- if erased_entity_ids:
193
- erased_entity_id_set = set(erased_entity_ids)
194
- lineage_source_event_ids: list[int] = []
195
- for row in self.conn.execute(
196
- "SELECT event_id, source_ids FROM lineage_event WHERE org_id = ?",
197
- (self.org_id,),
198
- ).fetchall():
199
- try:
200
- source_ids = json.loads(str(row["source_ids"] or "[]"))
201
- except json.JSONDecodeError:
202
- source_ids = []
203
- if any(
204
- str(source_id) in erased_entity_id_set for source_id in source_ids
205
- ):
206
- lineage_source_event_ids.append(int(row["event_id"]))
207
- deps._delete_in_chunks(
208
- "lineage_event", "event_id", lineage_source_event_ids
209
- )
210
- placeholders = ",".join("?" for _ in erased_entity_ids)
211
- self.conn.execute(
212
- f"""DELETE FROM lineage_event
213
- WHERE org_id = ?
214
- AND (
215
- request_id IN ({placeholders})
216
- OR entity_id IN ({placeholders})
217
- )""", # noqa: S608
218
- [self.org_id, *erased_entity_ids, *erased_entity_ids],
219
- )
186
+ # SEC-016: retain the content-free lineage_event skeleton on erase to
187
+ # match Supabase (which never enumerates lineage_event for erasure).
188
+ # lineage_event has no PII/content column, no foreign keys, and no
189
+ # FTS/vec shadow tables, so leaving these rows in place is safe and
190
+ # preserves the audit/lineage skeleton. Content-bearing entities are
191
+ # still deleted/purged below.
220
192
  delete_profile_rowids = [
221
193
  profile_rowid_by_id[profile_id]
222
194
  for profile_id in delete_profile_ids
@@ -240,6 +212,23 @@ class GovernanceEraseExecutionMixin:
240
212
  WHERE user_id = ?""",
241
213
  (user_id,),
242
214
  )
215
+ rle_results_cur = self.conn.execute(
216
+ """DELETE FROM retrieved_learning_evaluation
217
+ WHERE user_id = ?""",
218
+ (user_id,),
219
+ )
220
+ # Snapshot the user's session ids BEFORE requests are deleted — the
221
+ # evaluation _operation_state namespaces are keyed by session.
222
+ session_ids = [
223
+ str(row["session_id"])
224
+ for row in self.conn.execute(
225
+ "SELECT DISTINCT session_id FROM requests WHERE user_id = ?",
226
+ (user_id,),
227
+ ).fetchall()
228
+ ]
229
+ eval_operation_states = self._delete_evaluation_operation_states_locked(
230
+ user_id, session_ids
231
+ )
243
232
  requests_cur = self.conn.execute(
244
233
  "DELETE FROM requests WHERE user_id = ?",
245
234
  (user_id,),
@@ -280,10 +269,75 @@ class GovernanceEraseExecutionMixin:
280
269
  "profiles": len(delete_profile_ids),
281
270
  "requests": requests_cur.rowcount,
282
271
  "agent_success_evaluation_results": eval_results_cur.rowcount,
272
+ # Enterprise-only tables absent from the OSS SQLite backend;
273
+ # reported as zero so the delete target matrix stays complete.
274
+ "offline_tuner_reward_labels": 0,
275
+ "offline_tuner_reward_label_targets_by_target_owner": 0,
276
+ "retrieved_learning_evaluation_results": rle_results_cur.rowcount,
277
+ "evaluation_operation_states": eval_operation_states,
283
278
  "purged_profiles": purged_profiles,
284
279
  "purged_user_playbooks": purged_user_playbooks,
285
280
  }
286
281
 
282
+ def _delete_evaluation_operation_states_locked(
283
+ self, user_id: str, session_ids: list[str]
284
+ ) -> int:
285
+ """Scrub the three evaluation ``_operation_state`` namespaces.
286
+
287
+ Deletes, for every session the user owns: the retrieved-learning
288
+ generation/completion state, the ``agent_success_group_eval`` marker
289
+ (pre-existing RTBF gap), and the ``grade_on_demand`` cache rows
290
+ (pre-existing RTBF gap). Exact keys are used for the first two; the
291
+ grade cache is matched by the injective per-session key prefix so any
292
+ agent_version / evaluation_name variant is covered.
293
+
294
+ Returns:
295
+ int: Number of state rows deleted.
296
+ """
297
+ if not session_ids:
298
+ return 0
299
+ deleted = 0
300
+ exact_keys = [
301
+ build_retrieved_learning_state_key(user_id, session_id)
302
+ for session_id in session_ids
303
+ ] + [
304
+ build_agent_success_marker_key(self.org_id, user_id, session_id)
305
+ for session_id in session_ids
306
+ ]
307
+ chunk_size = 500
308
+ for start in range(0, len(exact_keys), chunk_size):
309
+ chunk = exact_keys[start : start + chunk_size]
310
+ placeholders = ",".join("?" for _ in chunk)
311
+ cur = self.conn.execute(
312
+ f"DELETE FROM _operation_state WHERE service_name IN ({placeholders})", # noqa: S608
313
+ chunk,
314
+ )
315
+ deleted += cur.rowcount
316
+ # Grade-cache keys embed agent_version/evaluation_name, so enumerate
317
+ # the namespace and match each session's injective prefix in Python
318
+ # (LIKE-escaping free-form session ids is not safe).
319
+ prefixes = tuple(
320
+ build_grade_on_demand_session_prefix(self.org_id, session_id)
321
+ for session_id in session_ids
322
+ )
323
+ grade_keys = [
324
+ str(row["service_name"])
325
+ for row in self.conn.execute(
326
+ "SELECT service_name FROM _operation_state WHERE service_name LIKE ?",
327
+ (f"{GRADE_ON_DEMAND_CACHE_PREFIX}::%",),
328
+ ).fetchall()
329
+ if str(row["service_name"]).startswith(prefixes)
330
+ ]
331
+ for start in range(0, len(grade_keys), chunk_size):
332
+ chunk = grade_keys[start : start + chunk_size]
333
+ placeholders = ",".join("?" for _ in chunk)
334
+ cur = self.conn.execute(
335
+ f"DELETE FROM _operation_state WHERE service_name IN ({placeholders})", # noqa: S608
336
+ chunk,
337
+ )
338
+ deleted += cur.rowcount
339
+ return deleted
340
+
287
341
  def apply_governance_user_data_delete(
288
342
  self, purge_id: str, user_id: str
289
343
  ) -> dict[str, int]:
@@ -294,6 +348,14 @@ class GovernanceEraseExecutionMixin:
294
348
  "profiles": "profile",
295
349
  "requests": "request",
296
350
  "agent_success_evaluation_results": "agent_success_evaluation_result",
351
+ "offline_tuner_reward_labels": "offline_tuner_reward_label",
352
+ "offline_tuner_reward_label_targets_by_target_owner": (
353
+ "offline_tuner_reward_label_target_by_target_owner"
354
+ ),
355
+ "retrieved_learning_evaluation_results": (
356
+ "retrieved_learning_evaluation_result"
357
+ ),
358
+ "evaluation_operation_states": "evaluation_operation_state",
297
359
  "purged_profiles": "profile_purge",
298
360
  "purged_user_playbooks": "user_playbook_purge",
299
361
  }