claude-smart 0.2.48 → 0.2.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/.claude-plugin/marketplace.json +2 -2
  2. package/README.md +11 -40
  3. package/bin/claude-smart.js +393 -55
  4. package/package.json +3 -2
  5. package/plugin/.claude-plugin/plugin.json +1 -1
  6. package/plugin/.codex-plugin/plugin.json +1 -1
  7. package/plugin/.coverage +0 -0
  8. package/plugin/README.md +4 -3
  9. package/plugin/dashboard/app/api/reflexio/[...path]/route.ts +4 -2
  10. package/plugin/dashboard/app/dashboard/page.tsx +6 -1
  11. package/plugin/dashboard/app/preferences/[id]/page.tsx +18 -6
  12. package/plugin/dashboard/app/preferences/page.tsx +32 -35
  13. package/plugin/dashboard/app/sessions/[sessionId]/page.tsx +16 -1
  14. package/plugin/dashboard/app/sessions/page.tsx +2 -0
  15. package/plugin/dashboard/app/skills/page.tsx +65 -50
  16. package/plugin/dashboard/app/skills/project/[id]/page.tsx +17 -8
  17. package/plugin/dashboard/app/skills/shared/[id]/page.tsx +1 -6
  18. package/plugin/dashboard/components/common/host-badge.tsx +118 -0
  19. package/plugin/dashboard/components/common/learning-application-badge.tsx +34 -0
  20. package/plugin/dashboard/components/common/learnings-badge.tsx +1 -1
  21. package/plugin/dashboard/components/common/page-header.tsx +3 -3
  22. package/plugin/dashboard/lib/config-file.ts +5 -1
  23. package/plugin/dashboard/lib/host-attribution.ts +62 -0
  24. package/plugin/dashboard/lib/session-reader.ts +40 -2
  25. package/plugin/dashboard/lib/types.ts +7 -1
  26. package/plugin/opencode/dist/server.mjs +60 -2
  27. package/plugin/opencode/server.mts +64 -2
  28. package/plugin/pyproject.toml +2 -2
  29. package/plugin/scripts/_lib.sh +255 -7
  30. package/plugin/scripts/backend-python-runner.py +46 -0
  31. package/plugin/scripts/backend-service.sh +766 -123
  32. package/plugin/scripts/codex-hook.js +79 -229
  33. package/plugin/scripts/dashboard-open.sh +6 -4
  34. package/plugin/scripts/dashboard-service.sh +118 -137
  35. package/plugin/scripts/ensure-plugin-root.sh +106 -8
  36. package/plugin/scripts/hook_entry.sh +3 -0
  37. package/plugin/scripts/opencode-claude-compat.js +8 -1
  38. package/plugin/scripts/smart-install.sh +116 -21
  39. package/plugin/src/README.md +1 -1
  40. package/plugin/src/claude_smart/cli.py +93 -29
  41. package/plugin/src/claude_smart/context_inject.py +3 -0
  42. package/plugin/src/claude_smart/env_config.py +56 -11
  43. package/plugin/src/claude_smart/events/post_tool.py +2 -1
  44. package/plugin/src/claude_smart/events/session_end.py +2 -1
  45. package/plugin/src/claude_smart/events/stop.py +3 -0
  46. package/plugin/src/claude_smart/events/user_prompt.py +2 -1
  47. package/plugin/src/claude_smart/internal_call.py +5 -2
  48. package/plugin/src/claude_smart/optimizer_assistant.py +59 -13
  49. package/plugin/src/claude_smart/publish.py +59 -7
  50. package/plugin/src/claude_smart/reflexio_adapter.py +137 -14
  51. package/plugin/src/claude_smart/runtime.py +15 -6
  52. package/plugin/src/claude_smart/state.py +211 -52
  53. package/plugin/uv.lock +5 -5
  54. package/plugin/vendor/reflexio/.env.example +13 -0
  55. package/plugin/vendor/reflexio/reflexio/README.md +7 -3
  56. package/plugin/vendor/reflexio/reflexio/__init__.py +12 -0
  57. package/plugin/vendor/reflexio/reflexio/cli/README.md +3 -0
  58. package/plugin/vendor/reflexio/reflexio/client/client.py +166 -9
  59. package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/state.py +10 -3
  60. package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_state.py +28 -0
  61. package/plugin/vendor/reflexio/reflexio/lib/_search.py +41 -0
  62. package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/entities.py +195 -25
  63. package/plugin/vendor/reflexio/reflexio/models/api_schema/eval_overview_schema.py +7 -1
  64. package/plugin/vendor/reflexio/reflexio/models/api_schema/internal_schema.py +2 -1
  65. package/plugin/vendor/reflexio/reflexio/models/api_schema/retriever_schema.py +63 -4
  66. package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/converters.py +1 -0
  67. package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/entities.py +8 -1
  68. package/plugin/vendor/reflexio/reflexio/models/config_schema.py +1 -1
  69. package/plugin/vendor/reflexio/reflexio/server/README.md +22 -4
  70. package/plugin/vendor/reflexio/reflexio/server/__init__.py +18 -8
  71. package/plugin/vendor/reflexio/reflexio/server/__main__.py +6 -0
  72. package/plugin/vendor/reflexio/reflexio/server/api.py +79 -5
  73. package/plugin/vendor/reflexio/reflexio/server/billing_meter.py +263 -3
  74. package/plugin/vendor/reflexio/reflexio/server/callback_executor.py +164 -0
  75. package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_embedding.py +81 -81
  76. package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_subprocess.py +28 -0
  77. package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_text_generation.py +63 -6
  78. package/plugin/vendor/reflexio/reflexio/server/llm/embedding_service.py +19 -52
  79. package/plugin/vendor/reflexio/reflexio/server/llm/litellm_client.py +1 -0
  80. package/plugin/vendor/reflexio/reflexio/server/llm/model_defaults.py +28 -0
  81. package/plugin/vendor/reflexio/reflexio/server/llm/providers/claude_code_provider.py +9 -1
  82. package/plugin/vendor/reflexio/reflexio/server/llm/providers/embedder_warmup.py +329 -0
  83. package/plugin/vendor/reflexio/reflexio/server/llm/providers/embedding_service_provider.py +85 -10
  84. package/plugin/vendor/reflexio/reflexio/server/llm/providers/local_embedding_provider.py +199 -2
  85. package/plugin/vendor/reflexio/reflexio/server/llm/providers/nomic_embedding_provider.py +77 -9
  86. package/plugin/vendor/reflexio/reflexio/server/org_fanout.py +184 -0
  87. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/document_expansion/v1.0.0.prompt.md +1 -1
  88. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/document_expansion/v1.1.0.prompt.md +32 -0
  89. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.3.3.prompt.md +1 -1
  90. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.4.0.prompt.md +63 -0
  91. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/query_reformulation/v1.0.0.prompt.md +1 -1
  92. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/query_reformulation/v2.0.0.prompt.md +30 -0
  93. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/retrieved_learning_impact/v1.0.0.prompt.md +51 -0
  94. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/retrieved_learning_relevance/v1.0.0.prompt.md +39 -0
  95. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/shadow_comparison/v1.0.0.prompt.md +1 -1
  96. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/shadow_comparison/v1.1.0.prompt.md +43 -0
  97. package/plugin/vendor/reflexio/reflexio/server/routes/config.py +3 -3
  98. package/plugin/vendor/reflexio/reflexio/server/routes/evaluation.py +122 -28
  99. package/plugin/vendor/reflexio/reflexio/server/routes/interactions.py +63 -2
  100. package/plugin/vendor/reflexio/reflexio/server/routes/system.py +22 -3
  101. package/plugin/vendor/reflexio/reflexio/server/scheduling.py +64 -3
  102. package/plugin/vendor/reflexio/reflexio/server/services/README.md +6 -4
  103. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/README.md +3 -2
  104. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/_eval_health.py +41 -0
  105. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/components/retrieved_learning_evaluator.py +554 -0
  106. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/regen_jobs.py +35 -1
  107. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/runner.py +253 -101
  108. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/scheduler.py +5 -7
  109. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/service.py +27 -0
  110. package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_extraction_lifecycle.py +11 -5
  111. package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_should_run.py +4 -4
  112. package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_usage_billing.py +6 -2
  113. package/plugin/vendor/reflexio/reflexio/server/services/base_generation_service.py +255 -74
  114. package/plugin/vendor/reflexio/reflexio/server/services/deduplication_utils.py +72 -0
  115. package/plugin/vendor/reflexio/reflexio/server/services/deferred_learning_plan.py +270 -0
  116. package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/__init__.py +13 -0
  117. package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/scheduler.py +142 -0
  118. package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/worker.py +262 -0
  119. package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/components/hero_state.py +2 -11
  120. package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/components/rule_attribution.py +6 -2
  121. package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/service.py +17 -13
  122. package/plugin/vendor/reflexio/reflexio/server/services/extraction/agent_run_records.py +18 -3
  123. package/plugin/vendor/reflexio/reflexio/server/services/extraction/outcome.py +12 -1
  124. package/plugin/vendor/reflexio/reflexio/server/services/extraction/prior_answer_search.py +1 -1
  125. package/plugin/vendor/reflexio/reflexio/server/services/extraction/resumable_agent.py +2 -1
  126. package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_scheduler.py +1 -3
  127. package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_worker.py +66 -0
  128. package/plugin/vendor/reflexio/reflexio/server/services/generation_service.py +640 -80
  129. package/plugin/vendor/reflexio/reflexio/server/services/governance/service.py +2 -0
  130. package/plugin/vendor/reflexio/reflexio/server/services/lineage/gc_scheduler.py +179 -99
  131. package/plugin/vendor/reflexio/reflexio/server/services/lineage/vector_backfill_sweep.py +139 -0
  132. package/plugin/vendor/reflexio/reflexio/server/services/operation_state_utils.py +66 -27
  133. package/plugin/vendor/reflexio/reflexio/server/services/playbook/aggregation_trigger.py +177 -0
  134. package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/aggregator.py +68 -2
  135. package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/consolidator.py +360 -49
  136. package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/extractor.py +27 -30
  137. package/plugin/vendor/reflexio/reflexio/server/services/playbook/playbook_edit_apply.py +8 -1
  138. package/plugin/vendor/reflexio/reflexio/server/services/playbook/service.py +137 -69
  139. package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/optimizer.py +20 -1
  140. package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/scheduler.py +13 -6
  141. package/plugin/vendor/reflexio/reflexio/server/services/pre_retrieval/_query_reformulator.py +40 -25
  142. package/plugin/vendor/reflexio/reflexio/server/services/profile/components/consolidator.py +12 -39
  143. package/plugin/vendor/reflexio/reflexio/server/services/profile/components/extractor.py +30 -35
  144. package/plugin/vendor/reflexio/reflexio/server/services/profile/service.py +166 -66
  145. package/plugin/vendor/reflexio/reflexio/server/services/reflection/service.py +457 -107
  146. package/plugin/vendor/reflexio/reflexio/server/services/retrieval/session_dedup.py +127 -0
  147. package/plugin/vendor/reflexio/reflexio/server/services/retrieval/temporal.py +104 -0
  148. package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/dispatcher.py +139 -0
  149. package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/judge.py +16 -6
  150. package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/worker.py +137 -0
  151. package/plugin/vendor/reflexio/reflexio/server/services/storage/governance_validation.py +12 -0
  152. package/plugin/vendor/reflexio/reflexio/server/services/storage/lifecycle_filters.py +54 -0
  153. package/plugin/vendor/reflexio/reflexio/server/services/storage/retention.py +41 -3
  154. package/plugin/vendor/reflexio/reflexio/server/services/storage/retention_mixin.py +40 -2
  155. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/__init__.py +2 -0
  156. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_base.py +190 -2
  157. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_extras.py +11 -4
  158. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_governance.py +28 -1
  159. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py +414 -0
  160. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_lineage.py +11 -5
  161. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_requests.py +8 -3
  162. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_shadow_verdicts.py +4 -0
  163. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/base/_deletion.py +16 -5
  164. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/base/_fts_vec.py +86 -2
  165. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_erase_execution.py +104 -42
  166. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_subject_barrier.py +7 -1
  167. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_agent.py +59 -7
  168. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_eval_results.py +430 -5
  169. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_source_linkage.py +8 -3
  170. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_user.py +57 -12
  171. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_interaction_store.py +197 -12
  172. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_profile_store.py +70 -11
  173. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/__init__.py +17 -0
  174. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_commit_scope.py +9 -0
  175. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_extras.py +9 -2
  176. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_learning_jobs.py +269 -0
  177. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_operations.py +7 -0
  178. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_retrieval_log.py +3 -1
  179. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_shadow_verdicts.py +4 -0
  180. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/agent_run/_models.py +12 -1
  181. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/evaluation_state_keys.py +78 -0
  182. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_agent.py +38 -0
  183. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_eval_results.py +171 -0
  184. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_user.py +52 -1
  185. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/_interaction_store.py +82 -1
  186. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/_profile_store.py +74 -2
  187. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/retrieved_learning_state.py +226 -0
  188. package/plugin/vendor/reflexio/reflexio/server/services/tagging/tagging_scheduler.py +5 -6
  189. package/plugin/vendor/reflexio/reflexio/server/services/unified_search_service.py +209 -29
  190. package/plugin/vendor/reflexio/reflexio/server/usage_metrics.py +3 -0
  191. package/plugin/vendor/reflexio/reflexio/test_support/llm_mock.py +61 -0
  192. package/plugin/vendor/reflexio/reflexio/test_support/llm_model_registry.py +33 -0
  193. package/plugin/vendor/reflexio/reflexio/server/services/search/__init__.py +0 -0
@@ -0,0 +1,414 @@
1
+ """SQLite implementation of the durable learning-job queue (Task 3)."""
2
+
3
+ import sqlite3
4
+ import time
5
+ import uuid
6
+ from typing import Any
7
+
8
+ from reflexio.server.services.storage.storage_base._learning_jobs import (
9
+ _ABSENCE_DONE_AFTER_SECONDS,
10
+ LearningJob,
11
+ LearningJobStoreABC,
12
+ LearningStatus,
13
+ )
14
+
15
+ from ._base import SQLiteStorageBase, _epoch_to_iso, _iso_to_epoch
16
+
17
+
18
+ def _row_to_learning_job(row: sqlite3.Row) -> LearningJob:
19
+ """Convert a sqlite3.Row from learning_jobs to a LearningJob dataclass."""
20
+ d = dict(row)
21
+ ct = d.get("covers_through")
22
+ return LearningJob(
23
+ job_id=d["job_id"],
24
+ org_id=d["org_id"],
25
+ user_id=d["user_id"],
26
+ job_type=d["job_type"],
27
+ latest_request_id=d.get("latest_request_id"),
28
+ status=d["status"],
29
+ attempts=d["attempts"],
30
+ claim_token=d.get("claim_token"),
31
+ covers_through=float(_iso_to_epoch(ct)) if ct else None,
32
+ force_extraction=bool(d.get("force_extraction", 0)),
33
+ skip_aggregation=bool(d.get("skip_aggregation", 0)),
34
+ max_attempts=int(d.get("max_attempts", 3)),
35
+ )
36
+
37
+
38
+ class SQLiteLearningJobStoreMixin(LearningJobStoreABC):
39
+ """SQLite implementation of the learning-job queue.
40
+
41
+ Relies on instance attributes provided by SQLiteStorageBase via MRO:
42
+ ``conn``, ``_lock``, ``_own_transaction``, ``org_id``.
43
+ """
44
+
45
+ # Type annotations for attributes/methods supplied by SQLiteStorageBase via MRO
46
+ _lock: Any
47
+ conn: sqlite3.Connection
48
+ org_id: str
49
+ _own_transaction: Any
50
+ _fetchall: Any
51
+
52
+ @SQLiteStorageBase.handle_exceptions
53
+ def enqueue_learning_job(
54
+ self,
55
+ *,
56
+ org_id: str,
57
+ user_id: str,
58
+ request_id: str,
59
+ covers_through: float,
60
+ job_type: str = "learning",
61
+ force_extraction: bool = False,
62
+ skip_aggregation: bool = False,
63
+ ) -> str:
64
+ """Coalescing upsert — safe to call inside a commit_scope."""
65
+ job_id = str(uuid.uuid4())
66
+ # int() truncates sub-second precision — intentional (second-precision epochs).
67
+ iso_covers = _epoch_to_iso(int(covers_through))
68
+ fe_int = int(force_extraction)
69
+ sa_int = int(skip_aggregation)
70
+ with self._lock:
71
+ own_txn = self._own_transaction()
72
+ try:
73
+ if own_txn:
74
+ self.conn.execute("BEGIN IMMEDIATE")
75
+ row = self.conn.execute(
76
+ """
77
+ INSERT INTO learning_jobs
78
+ (job_id, org_id, user_id, job_type, latest_request_id,
79
+ covers_through, status, force_extraction, skip_aggregation,
80
+ created_at, updated_at)
81
+ VALUES
82
+ (?, ?, ?, ?, ?,
83
+ ?, 'pending', ?, ?,
84
+ strftime('%Y-%m-%dT%H:%M:%fZ','now'),
85
+ strftime('%Y-%m-%dT%H:%M:%fZ','now'))
86
+ ON CONFLICT (org_id, user_id, job_type) WHERE status = 'pending'
87
+ DO UPDATE SET
88
+ latest_request_id = excluded.latest_request_id,
89
+ covers_through = CASE
90
+ WHEN learning_jobs.covers_through > excluded.covers_through
91
+ THEN learning_jobs.covers_through
92
+ ELSE excluded.covers_through
93
+ END,
94
+ force_extraction = excluded.force_extraction,
95
+ skip_aggregation = excluded.skip_aggregation,
96
+ updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
97
+ RETURNING job_id
98
+ """,
99
+ (
100
+ job_id,
101
+ org_id,
102
+ user_id,
103
+ job_type,
104
+ request_id,
105
+ iso_covers,
106
+ fe_int,
107
+ sa_int,
108
+ ),
109
+ ).fetchone()
110
+ if own_txn:
111
+ self.conn.commit()
112
+ except Exception:
113
+ if own_txn:
114
+ self.conn.rollback()
115
+ raise
116
+ if row is None:
117
+ raise RuntimeError("enqueue_learning_job RETURNING job_id returned no row")
118
+ return str(row["job_id"])
119
+
120
+ @SQLiteStorageBase.handle_exceptions
121
+ def claim_learning_jobs(
122
+ self,
123
+ *,
124
+ claimed_by: str,
125
+ limit: int,
126
+ lease_seconds: int,
127
+ ) -> list[LearningJob]:
128
+ """BEGIN IMMEDIATE + SELECT + UPDATE to atomically claim jobs."""
129
+ with self._lock:
130
+ own_txn = self._own_transaction()
131
+ try:
132
+ if own_txn:
133
+ self.conn.execute("BEGIN IMMEDIATE")
134
+
135
+ # Find candidate job_ids using DB's now() to avoid clock skew.
136
+ # Include 'failed' so a failed-but-not-dead job is naturally
137
+ # reclaimable without a manual status reset.
138
+ candidate_rows = self.conn.execute(
139
+ """
140
+ SELECT job_id FROM learning_jobs
141
+ WHERE org_id = ?
142
+ AND (
143
+ status = 'pending'
144
+ OR status = 'failed'
145
+ OR (status = 'claimed'
146
+ AND claim_expires_at < strftime('%Y-%m-%dT%H:%M:%fZ','now'))
147
+ )
148
+ ORDER BY created_at
149
+ LIMIT ?
150
+ """,
151
+ (self.org_id, limit),
152
+ ).fetchall()
153
+
154
+ claimed: list[LearningJob] = []
155
+ for cand in candidate_rows:
156
+ job_id = cand["job_id"]
157
+ claim_token = str(uuid.uuid4())
158
+ updated = self.conn.execute(
159
+ """
160
+ UPDATE learning_jobs SET
161
+ status = 'claimed',
162
+ claimed_by = ?,
163
+ claim_token = ?,
164
+ claim_expires_at = strftime(
165
+ '%Y-%m-%dT%H:%M:%fZ', 'now',
166
+ ? || ' seconds'
167
+ ),
168
+ updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now'),
169
+ attempts = attempts + 1
170
+ WHERE job_id = ?
171
+ RETURNING *
172
+ """,
173
+ (claimed_by, claim_token, str(lease_seconds), job_id),
174
+ ).fetchone()
175
+ if updated is not None:
176
+ claimed.append(_row_to_learning_job(updated))
177
+
178
+ if own_txn:
179
+ self.conn.commit()
180
+ except Exception:
181
+ if own_txn:
182
+ self.conn.rollback()
183
+ raise
184
+
185
+ return claimed
186
+
187
+ @SQLiteStorageBase.handle_exceptions
188
+ def heartbeat_learning_job(
189
+ self,
190
+ *,
191
+ job_id: str,
192
+ claim_token: str,
193
+ lease_seconds: int,
194
+ ) -> bool:
195
+ """Extend the lease; return True if the token is still live."""
196
+ with self._lock:
197
+ own_txn = self._own_transaction()
198
+ try:
199
+ if own_txn:
200
+ self.conn.execute("BEGIN IMMEDIATE")
201
+ cur = self.conn.execute(
202
+ """
203
+ UPDATE learning_jobs SET
204
+ claim_expires_at = strftime(
205
+ '%Y-%m-%dT%H:%M:%fZ', 'now',
206
+ ? || ' seconds'
207
+ ),
208
+ updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
209
+ WHERE job_id = ? AND claim_token = ? AND status = 'claimed'
210
+ """,
211
+ (str(lease_seconds), job_id, claim_token),
212
+ )
213
+ updated = cur.rowcount == 1
214
+ if own_txn:
215
+ self.conn.commit()
216
+ except Exception:
217
+ if own_txn:
218
+ self.conn.rollback()
219
+ raise
220
+
221
+ return updated
222
+
223
+ @SQLiteStorageBase.handle_exceptions
224
+ def complete_learning_job(
225
+ self,
226
+ *,
227
+ job_id: str,
228
+ claim_token: str,
229
+ ) -> int:
230
+ """Fenced completion — returns rowcount (0=superseded, 1=success).
231
+
232
+ Safe to call inside a commit_scope — no own BEGIN/COMMIT issued.
233
+ """
234
+ with self._lock:
235
+ own_txn = self._own_transaction()
236
+ try:
237
+ if own_txn:
238
+ self.conn.execute("BEGIN IMMEDIATE")
239
+ cur = self.conn.execute(
240
+ """
241
+ UPDATE learning_jobs SET
242
+ status = 'done',
243
+ updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
244
+ WHERE job_id = ? AND claim_token = ? AND status = 'claimed'
245
+ """,
246
+ (job_id, claim_token),
247
+ )
248
+ rowcount = cur.rowcount
249
+ if own_txn:
250
+ self.conn.commit()
251
+ except Exception:
252
+ if own_txn:
253
+ self.conn.rollback()
254
+ raise
255
+
256
+ return rowcount
257
+
258
+ @SQLiteStorageBase.handle_exceptions
259
+ def fail_learning_job(
260
+ self,
261
+ *,
262
+ job_id: str,
263
+ claim_token: str,
264
+ dead: bool,
265
+ refund_attempt: bool = False,
266
+ ) -> None:
267
+ """Fenced fail/dead transition — sets status, clears token for retry.
268
+
269
+ Does NOT increment attempts: claim_learning_jobs already incremented on
270
+ delivery. attempts tracks delivery count; fail only transitions status.
271
+
272
+ ``refund_attempt=True`` (the same-user-contention requeue, F4) decrements
273
+ ``attempts`` by one (``MAX(attempts - 1, 0)``) so a contention cycle
274
+ (claim +1, contention-release -1) nets to zero and ``attempts`` stays
275
+ bounded regardless of how many times the job loses the per-user race.
276
+ Defaults to ``False`` (the ``dead`` retry path is unchanged).
277
+ """
278
+ new_status = "dead" if dead else "failed"
279
+ # Clear claim_token and claim_expires_at only for 'failed' so it's reclaimable.
280
+ # For 'dead', we keep claim_token set for auditability (won't be reclaimed anyway).
281
+ with self._lock:
282
+ own_txn = self._own_transaction()
283
+ try:
284
+ if own_txn:
285
+ self.conn.execute("BEGIN IMMEDIATE")
286
+ self.conn.execute(
287
+ """
288
+ UPDATE learning_jobs SET
289
+ status = ?,
290
+ claim_token = CASE WHEN ? THEN claim_token ELSE NULL END,
291
+ claim_expires_at = CASE WHEN ? THEN claim_expires_at ELSE NULL END,
292
+ attempts = CASE WHEN ? THEN MAX(attempts - 1, 0) ELSE attempts END,
293
+ updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
294
+ WHERE job_id = ? AND claim_token = ? AND status = 'claimed'
295
+ """,
296
+ (new_status, dead, dead, refund_attempt, job_id, claim_token),
297
+ )
298
+ if own_txn:
299
+ self.conn.commit()
300
+ except Exception:
301
+ if own_txn:
302
+ self.conn.rollback()
303
+ raise
304
+
305
+ @SQLiteStorageBase.handle_exceptions
306
+ def list_org_ids_with_pending_learning_jobs(self) -> list[str]:
307
+ """Distinct org_ids with actionable jobs (cross-org, not org-scoped).
308
+
309
+ Uses the DB's now() for the expired-lease comparison to avoid clock skew,
310
+ mirroring ``claim_learning_jobs``. Index-aided by ``learning_jobs_poll``
311
+ (partial index narrows the scan to non-terminal rows; org_id still requires
312
+ a heap fetch).
313
+ """
314
+ rows = self._fetchall(
315
+ """
316
+ SELECT DISTINCT org_id FROM learning_jobs
317
+ WHERE status = 'pending'
318
+ OR status = 'failed'
319
+ OR (status = 'claimed'
320
+ AND claim_expires_at < strftime('%Y-%m-%dT%H:%M:%fZ','now'))
321
+ ORDER BY org_id ASC
322
+ """,
323
+ (),
324
+ )
325
+ return [str(row["org_id"]) for row in rows]
326
+
327
+ @SQLiteStorageBase.handle_exceptions
328
+ def get_oldest_pending_learning_job_age_seconds(self) -> float | None:
329
+ """Age in seconds of the oldest pending job, or None if none pending.
330
+
331
+ Computes the age entirely in SQLite using ``strftime('%s','now')`` to
332
+ avoid app/DB clock skew. The table is accessed without a schema prefix
333
+ (SQLite has no schema routing — the local DB file is the ref).
334
+ """
335
+ row = self._fetchall(
336
+ """
337
+ SELECT (strftime('%s', 'now') - strftime('%s', MIN(created_at))) AS age_seconds
338
+ FROM learning_jobs
339
+ WHERE status = 'pending'
340
+ """,
341
+ (),
342
+ )
343
+ return None if row[0]["age_seconds"] is None else float(row[0]["age_seconds"])
344
+
345
+ @SQLiteStorageBase.handle_exceptions
346
+ def count_learning_jobs_by_status(self, status: str) -> int:
347
+ """Count of learning jobs with the given status on this storage ref."""
348
+ row = self._fetchall(
349
+ "SELECT COUNT(*) AS cnt FROM learning_jobs WHERE status = ?",
350
+ (status,),
351
+ )
352
+ return int(row[0]["cnt"])
353
+
354
+ @SQLiteStorageBase.handle_exceptions
355
+ def get_learning_status_for_request(
356
+ self,
357
+ *,
358
+ user_id: str,
359
+ request_created_at: float,
360
+ ) -> LearningStatus:
361
+ """Coverage-based status lookup (§3.6 rule).
362
+
363
+ Converts request_created_at epoch to ISO for lexicographic comparison
364
+ with stored covers_through ISO strings (same format, both UTC).
365
+ """
366
+ req_iso = _epoch_to_iso(int(request_created_at))
367
+ rows = self._fetchall(
368
+ "SELECT status, covers_through FROM learning_jobs "
369
+ "WHERE org_id = ? AND user_id = ?",
370
+ (self.org_id, user_id),
371
+ )
372
+
373
+ has_pending = False
374
+ has_claimed_covering = False
375
+ has_dead_covering = False
376
+ has_failed = False
377
+
378
+ for row in rows:
379
+ status = row["status"]
380
+ ct: str | None = row["covers_through"]
381
+ covers = ct is not None and ct >= req_iso
382
+
383
+ if covers and status == "done":
384
+ return "done"
385
+ if covers and status == "claimed":
386
+ has_claimed_covering = True
387
+ if covers and status == "dead":
388
+ has_dead_covering = True
389
+ if status == "failed":
390
+ # 'failed' is reclaimable (attempts < max_attempts); treat as pending.
391
+ # Accumulate as a flag so a covering done row (encountered later in the
392
+ # iteration) is not shadowed by a failed row yielded earlier.
393
+ has_failed = True
394
+ if status == "pending":
395
+ has_pending = True
396
+ if status == "claimed":
397
+ # Deliberately treats any claimed job as covering, regardless of its
398
+ # covers_through value — it will extend the window once it completes.
399
+ has_claimed_covering = True
400
+
401
+ if has_claimed_covering:
402
+ return "processing"
403
+ if has_pending:
404
+ return "pending"
405
+ if has_failed:
406
+ return "pending"
407
+ if has_dead_covering:
408
+ return "failed"
409
+ # Absence semantics: terminal rows (done/dead) are GC'd after 24-72 h.
410
+ # Only treat absence as "done" once the request is old enough that a done
411
+ # row would have been reaped; a recent request with no rows is still pending.
412
+ if time.time() - request_created_at >= _ABSENCE_DONE_AFTER_SECONDS:
413
+ return "done"
414
+ return "pending"
@@ -127,6 +127,7 @@ class SQLiteLineageMixin:
127
127
  conn: sqlite3.Connection
128
128
  _lock: threading.RLock
129
129
  org_id: str
130
+ _own_transaction: Any
130
131
 
131
132
  def append_lineage_event(self, event: LineageEvent) -> int:
132
133
  """Append an event; idempotent on (org_id, entity_type, entity_id, op, request_id).
@@ -173,10 +174,12 @@ class SQLiteLineageMixin:
173
174
  ),
174
175
  ).fetchone()
175
176
  eid = row[0] if row else None
176
- self.conn.commit()
177
+ if self._own_transaction():
178
+ self.conn.commit()
177
179
  return int(eid) if eid is not None else 0
178
180
  last = cur.lastrowid
179
- self.conn.commit()
181
+ if self._own_transaction():
182
+ self.conn.commit()
180
183
  return int(last) if last is not None else 0
181
184
 
182
185
  def get_lineage_events(
@@ -301,7 +304,8 @@ class SQLiteLineageMixin:
301
304
  request_id=context.request_id,
302
305
  reason=context.reason,
303
306
  )
304
- self.conn.commit()
307
+ if self._own_transaction():
308
+ self.conn.commit()
305
309
 
306
310
  def supersede_record(
307
311
  self,
@@ -343,7 +347,8 @@ class SQLiteLineageMixin:
343
347
  (Status.SUPERSEDED.value, successor_id, _epoch_now(), incumbent_id),
344
348
  )
345
349
  if cur.rowcount == 0:
346
- self.conn.commit()
350
+ if self._own_transaction():
351
+ self.conn.commit()
347
352
  return False
348
353
  _append_event_stmt(
349
354
  self.conn,
@@ -357,7 +362,8 @@ class SQLiteLineageMixin:
357
362
  request_id=context.request_id,
358
363
  reason=context.reason,
359
364
  )
360
- self.conn.commit()
365
+ if self._own_transaction():
366
+ self.conn.commit()
361
367
  return True
362
368
 
363
369
  def purge_content(self, *, entity_type: EntityType, entity_id: str) -> bool:
@@ -32,6 +32,7 @@ class RequestMixin:
32
32
  _fetchall: Any
33
33
  _subject_ref_for_user_id: Any
34
34
  _assert_subject_writable_locked: Any
35
+ _own_transaction: Any
35
36
 
36
37
  # ------------------------------------------------------------------
37
38
  # Request methods
@@ -42,8 +43,10 @@ class RequestMixin:
42
43
  created_at_iso = _epoch_to_iso(request.created_at)
43
44
  subject_ref = self._subject_ref_for_user_id(request.user_id)
44
45
  with self._lock:
46
+ own_txn = self._own_transaction()
45
47
  try:
46
- self.conn.execute("BEGIN IMMEDIATE")
48
+ if own_txn:
49
+ self.conn.execute("BEGIN IMMEDIATE")
47
50
  self._assert_subject_writable_locked(subject_ref)
48
51
  self.conn.execute(
49
52
  """INSERT OR REPLACE INTO requests
@@ -61,9 +64,11 @@ class RequestMixin:
61
64
  subject_ref,
62
65
  ),
63
66
  )
64
- self.conn.commit()
67
+ if own_txn:
68
+ self.conn.commit()
65
69
  except Exception:
66
- self.conn.rollback()
70
+ if own_txn:
71
+ self.conn.rollback()
67
72
  raise
68
73
 
69
74
  @SQLiteStorageBase.handle_exceptions
@@ -74,6 +74,10 @@ class ShadowVerdictsMixin:
74
74
  _fetchone: Any
75
75
  _fetchall: Any
76
76
 
77
+ def supports_shadow_comparison_verdicts(self) -> bool:
78
+ """Return whether this backend can persist shadow comparison verdicts."""
79
+ return True
80
+
77
81
  @SQLiteStorageBase.handle_exceptions
78
82
  def save_shadow_comparison_verdict(
79
83
  self, verdict: ShadowComparisonVerdict
@@ -46,14 +46,25 @@ class SQLiteDeletionMixin:
46
46
 
47
47
  @SQLiteStorageBase.handle_exceptions
48
48
  def _retention_select_oldest_keys(
49
- self, target: RetentionTarget, count: int
49
+ self,
50
+ target: RetentionTarget,
51
+ count: int,
52
+ statuses: tuple[str, ...] | None = None,
50
53
  ) -> list[tuple[Any, ...]]:
54
+ if statuses is not None and not statuses:
55
+ return []
51
56
  id_sql = ", ".join(target.id_columns)
52
- tiebreak_sql = id_sql
57
+ where_sql = ""
58
+ params: list[Any] = []
59
+ if statuses:
60
+ placeholders = ", ".join("?" for _ in statuses)
61
+ where_sql = f"WHERE status IN ({placeholders}) "
62
+ params.extend(statuses)
63
+ params.append(count)
53
64
  rows = self._fetchall(
54
- f"SELECT {id_sql} FROM {target.table_name} " # noqa: S608
55
- f"ORDER BY {target.order_column} ASC, {tiebreak_sql} ASC LIMIT ?",
56
- (count,),
65
+ f"SELECT {id_sql} FROM {target.table_name} {where_sql}" # noqa: S608
66
+ f"ORDER BY {target.order_column} ASC, {id_sql} ASC LIMIT ?",
67
+ tuple(params),
57
68
  )
58
69
  return [tuple(row[col] for col in target.id_columns) for row in rows]
59
70
 
@@ -6,6 +6,11 @@ index sidecar update is durable. They are sqlite-only — there is no
6
6
  supabase/postgres counterpart. Composed into ``SQLiteStorage`` ahead of
7
7
  ``SQLiteStorageBase``; the boot-time ``_migrate_vec_tables`` (residual in
8
8
  ``_base.py``) resolves ``self._vec_upsert`` through the composed MRO.
9
+
10
+ When a ``commit_scope`` is active (``_scope_depth > 0``), the public helpers
11
+ buffer the operation into ``_deferred_index_ops`` for post-commit flushing;
12
+ the ``_..._now`` variants always execute immediately and are called by
13
+ ``_flush_index_op`` after the outer commit.
9
14
  """
10
15
 
11
16
  from __future__ import annotations
@@ -23,10 +28,50 @@ class SQLiteFtsVecMixin:
23
28
  conn: sqlite3.Connection
24
29
  _fetchall: Any
25
30
  _has_sqlite_vec: bool
31
+ _scope_depth: int
32
+ _deferred_index_ops: list[tuple[str, Any]]
33
+
34
+ # ------------------------------------------------------------------
35
+ # Dispatcher — called by commit_scope after the outer commit
36
+ # ------------------------------------------------------------------
37
+
38
+ def _flush_index_op(self, kind: str, args: Any) -> None:
39
+ """Dispatch a buffered index op after the outer commit."""
40
+ if kind == "fts_upsert":
41
+ table, rowid, text_fields = args
42
+ self._fts_upsert_now(table, rowid, **text_fields)
43
+ elif kind == "fts_delete":
44
+ table, rowid = args
45
+ self._fts_delete_now(table, rowid)
46
+ elif kind == "fts_upsert_profile":
47
+ profile_id, content = args
48
+ self._fts_upsert_profile_now(profile_id, content)
49
+ elif kind == "fts_delete_profile":
50
+ (profile_id,) = args
51
+ self._fts_delete_profile_now(profile_id)
52
+ elif kind == "vec_upsert":
53
+ table, rowid, embedding = args
54
+ self._vec_upsert_now(table, rowid, embedding)
55
+ elif kind == "vec_delete":
56
+ table, rowid = args
57
+ self._vec_delete_now(table, rowid)
58
+ else:
59
+ raise ValueError(f"unknown index op kind: {kind!r}")
60
+
61
+ # ------------------------------------------------------------------
62
+ # FTS helpers — public (defer when in scope) + _now (immediate)
63
+ # ------------------------------------------------------------------
26
64
 
27
- # FTS helpers
28
65
  def _fts_upsert(self, table: str, rowid: int, **text_fields: str | None) -> None:
29
66
  """Insert or update an FTS row. Deletes old entry first to avoid duplicates."""
67
+ if self._scope_depth > 0:
68
+ self._deferred_index_ops.append(("fts_upsert", (table, rowid, text_fields)))
69
+ return
70
+ self._fts_upsert_now(table, rowid, **text_fields)
71
+
72
+ def _fts_upsert_now(
73
+ self, table: str, rowid: int, **text_fields: str | None
74
+ ) -> None:
30
75
  with self._lock:
31
76
  self.conn.execute(f"DELETE FROM {table} WHERE rowid = ?", (rowid,)) # noqa: S608
32
77
  cols = list(text_fields.keys())
@@ -40,12 +85,26 @@ class SQLiteFtsVecMixin:
40
85
  self.conn.commit()
41
86
 
42
87
  def _fts_delete(self, table: str, rowid: int) -> None:
88
+ if self._scope_depth > 0:
89
+ self._deferred_index_ops.append(("fts_delete", (table, rowid)))
90
+ return
91
+ self._fts_delete_now(table, rowid)
92
+
93
+ def _fts_delete_now(self, table: str, rowid: int) -> None:
43
94
  with self._lock:
44
95
  self.conn.execute(f"DELETE FROM {table} WHERE rowid = ?", (rowid,)) # noqa: S608
45
96
  self.conn.commit()
46
97
 
47
98
  def _fts_upsert_profile(self, profile_id: str, content: str) -> None:
48
99
  """FTS for profiles uses profile_id TEXT as key column."""
100
+ if self._scope_depth > 0:
101
+ self._deferred_index_ops.append(
102
+ ("fts_upsert_profile", (profile_id, content))
103
+ )
104
+ return
105
+ self._fts_upsert_profile_now(profile_id, content)
106
+
107
+ def _fts_upsert_profile_now(self, profile_id: str, content: str) -> None:
49
108
  with self._lock:
50
109
  self.conn.execute(
51
110
  "DELETE FROM profiles_fts WHERE profile_id = ?", (profile_id,)
@@ -57,15 +116,32 @@ class SQLiteFtsVecMixin:
57
116
  self.conn.commit()
58
117
 
59
118
  def _fts_delete_profile(self, profile_id: str) -> None:
119
+ if self._scope_depth > 0:
120
+ self._deferred_index_ops.append(("fts_delete_profile", (profile_id,)))
121
+ return
122
+ self._fts_delete_profile_now(profile_id)
123
+
124
+ def _fts_delete_profile_now(self, profile_id: str) -> None:
60
125
  with self._lock:
61
126
  self.conn.execute(
62
127
  "DELETE FROM profiles_fts WHERE profile_id = ?", (profile_id,)
63
128
  )
64
129
  self.conn.commit()
65
130
 
66
- # Vec helpers (sqlite-vec)
131
+ # ------------------------------------------------------------------
132
+ # Vec helpers (sqlite-vec) — public (defer when in scope) + _now (immediate)
133
+ # ------------------------------------------------------------------
134
+
67
135
  def _vec_upsert(self, table: str, rowid: int, embedding: list[float]) -> None:
68
136
  """Insert or update a vec table row. No-op when sqlite-vec is unavailable."""
137
+ if not self._has_sqlite_vec:
138
+ return
139
+ if self._scope_depth > 0:
140
+ self._deferred_index_ops.append(("vec_upsert", (table, rowid, embedding)))
141
+ return
142
+ self._vec_upsert_now(table, rowid, embedding)
143
+
144
+ def _vec_upsert_now(self, table: str, rowid: int, embedding: list[float]) -> None:
69
145
  if not self._has_sqlite_vec:
70
146
  return
71
147
  with self._lock:
@@ -78,6 +154,14 @@ class SQLiteFtsVecMixin:
78
154
 
79
155
  def _vec_delete(self, table: str, rowid: int) -> None:
80
156
  """Delete a vec table row. No-op when sqlite-vec is unavailable."""
157
+ if not self._has_sqlite_vec:
158
+ return
159
+ if self._scope_depth > 0:
160
+ self._deferred_index_ops.append(("vec_delete", (table, rowid)))
161
+ return
162
+ self._vec_delete_now(table, rowid)
163
+
164
+ def _vec_delete_now(self, table: str, rowid: int) -> None:
81
165
  if not self._has_sqlite_vec:
82
166
  return
83
167
  with self._lock: