claude-smart 0.2.49 → 0.2.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +2 -2
- package/README.md +10 -43
- package/bin/claude-smart.js +105 -0
- package/package.json +3 -2
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/.codex-plugin/plugin.json +1 -1
- package/plugin/.coverage +0 -0
- package/plugin/README.md +4 -3
- package/plugin/dashboard/app/api/reflexio/[...path]/route.ts +4 -2
- package/plugin/dashboard/app/dashboard/page.tsx +6 -1
- package/plugin/dashboard/app/preferences/[id]/page.tsx +18 -6
- package/plugin/dashboard/app/preferences/page.tsx +32 -35
- package/plugin/dashboard/app/sessions/[sessionId]/page.tsx +16 -1
- package/plugin/dashboard/app/sessions/page.tsx +2 -0
- package/plugin/dashboard/app/skills/page.tsx +65 -50
- package/plugin/dashboard/app/skills/project/[id]/page.tsx +17 -8
- package/plugin/dashboard/app/skills/shared/[id]/page.tsx +1 -6
- package/plugin/dashboard/components/common/host-badge.tsx +118 -0
- package/plugin/dashboard/components/common/learning-application-badge.tsx +34 -0
- package/plugin/dashboard/components/common/learnings-badge.tsx +1 -1
- package/plugin/dashboard/components/common/page-header.tsx +3 -3
- package/plugin/dashboard/lib/config-file.ts +5 -1
- package/plugin/dashboard/lib/host-attribution.ts +62 -0
- package/plugin/dashboard/lib/session-reader.ts +40 -2
- package/plugin/dashboard/lib/types.ts +7 -1
- package/plugin/pyproject.toml +1 -1
- package/plugin/scripts/_lib.sh +197 -1
- package/plugin/scripts/backend-python-runner.py +46 -0
- package/plugin/scripts/backend-service.sh +757 -119
- package/plugin/scripts/codex-hook.js +63 -225
- package/plugin/scripts/dashboard-open.sh +6 -4
- package/plugin/scripts/dashboard-service.sh +117 -136
- package/plugin/scripts/hook_entry.sh +3 -0
- package/plugin/scripts/smart-install.sh +15 -1
- package/plugin/src/claude_smart/cli.py +14 -0
- package/plugin/src/claude_smart/context_inject.py +3 -0
- package/plugin/src/claude_smart/env_config.py +4 -1
- package/plugin/src/claude_smart/events/post_tool.py +2 -1
- package/plugin/src/claude_smart/events/session_end.py +2 -1
- package/plugin/src/claude_smart/events/stop.py +3 -0
- package/plugin/src/claude_smart/events/user_prompt.py +2 -1
- package/plugin/src/claude_smart/internal_call.py +5 -2
- package/plugin/src/claude_smart/optimizer_assistant.py +59 -13
- package/plugin/src/claude_smart/publish.py +59 -7
- package/plugin/src/claude_smart/reflexio_adapter.py +137 -14
- package/plugin/src/claude_smart/runtime.py +15 -6
- package/plugin/src/claude_smart/state.py +211 -52
- package/plugin/uv.lock +1 -1
- package/plugin/vendor/reflexio/.env.example +13 -0
- package/plugin/vendor/reflexio/reflexio/README.md +7 -3
- package/plugin/vendor/reflexio/reflexio/__init__.py +12 -0
- package/plugin/vendor/reflexio/reflexio/client/client.py +126 -3
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/state.py +10 -3
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_state.py +28 -0
- package/plugin/vendor/reflexio/reflexio/lib/_search.py +41 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/entities.py +177 -25
- package/plugin/vendor/reflexio/reflexio/models/api_schema/eval_overview_schema.py +7 -1
- package/plugin/vendor/reflexio/reflexio/models/api_schema/internal_schema.py +2 -1
- package/plugin/vendor/reflexio/reflexio/models/api_schema/retriever_schema.py +63 -4
- package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/converters.py +1 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/entities.py +8 -1
- package/plugin/vendor/reflexio/reflexio/models/config_schema.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/README.md +22 -4
- package/plugin/vendor/reflexio/reflexio/server/__init__.py +18 -8
- package/plugin/vendor/reflexio/reflexio/server/__main__.py +6 -0
- package/plugin/vendor/reflexio/reflexio/server/api.py +66 -3
- package/plugin/vendor/reflexio/reflexio/server/billing_meter.py +263 -3
- package/plugin/vendor/reflexio/reflexio/server/callback_executor.py +164 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_embedding.py +81 -81
- package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_subprocess.py +28 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_text_generation.py +62 -5
- package/plugin/vendor/reflexio/reflexio/server/llm/embedding_service.py +19 -52
- package/plugin/vendor/reflexio/reflexio/server/llm/model_defaults.py +28 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/claude_code_provider.py +9 -1
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/embedder_warmup.py +329 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/embedding_service_provider.py +85 -10
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/local_embedding_provider.py +20 -5
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/nomic_embedding_provider.py +77 -9
- package/plugin/vendor/reflexio/reflexio/server/org_fanout.py +184 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/document_expansion/v1.0.0.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/document_expansion/v1.1.0.prompt.md +32 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.3.3.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.4.0.prompt.md +63 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/query_reformulation/v1.0.0.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/query_reformulation/v2.0.0.prompt.md +30 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/retrieved_learning_impact/v1.0.0.prompt.md +51 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/retrieved_learning_relevance/v1.0.0.prompt.md +39 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/shadow_comparison/v1.0.0.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/shadow_comparison/v1.1.0.prompt.md +43 -0
- package/plugin/vendor/reflexio/reflexio/server/routes/config.py +3 -3
- package/plugin/vendor/reflexio/reflexio/server/routes/evaluation.py +122 -28
- package/plugin/vendor/reflexio/reflexio/server/routes/system.py +22 -3
- package/plugin/vendor/reflexio/reflexio/server/scheduling.py +64 -3
- package/plugin/vendor/reflexio/reflexio/server/services/README.md +6 -4
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/README.md +3 -2
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/_eval_health.py +41 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/components/retrieved_learning_evaluator.py +554 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/regen_jobs.py +35 -1
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/runner.py +253 -101
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/scheduler.py +5 -7
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/service.py +27 -0
- package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_extraction_lifecycle.py +11 -5
- package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_should_run.py +4 -4
- package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_usage_billing.py +6 -2
- package/plugin/vendor/reflexio/reflexio/server/services/base_generation_service.py +255 -74
- package/plugin/vendor/reflexio/reflexio/server/services/deduplication_utils.py +72 -0
- package/plugin/vendor/reflexio/reflexio/server/services/deferred_learning_plan.py +270 -0
- package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/worker.py +85 -16
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/components/hero_state.py +2 -11
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/components/rule_attribution.py +6 -2
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/service.py +17 -13
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/agent_run_records.py +18 -3
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/outcome.py +12 -1
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/prior_answer_search.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/resumable_agent.py +2 -1
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_worker.py +66 -0
- package/plugin/vendor/reflexio/reflexio/server/services/generation_service.py +510 -63
- package/plugin/vendor/reflexio/reflexio/server/services/governance/service.py +2 -0
- package/plugin/vendor/reflexio/reflexio/server/services/lineage/gc_scheduler.py +179 -99
- package/plugin/vendor/reflexio/reflexio/server/services/lineage/vector_backfill_sweep.py +139 -0
- package/plugin/vendor/reflexio/reflexio/server/services/operation_state_utils.py +66 -27
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/aggregation_trigger.py +177 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/aggregator.py +68 -2
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/consolidator.py +360 -49
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/extractor.py +27 -30
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/playbook_edit_apply.py +8 -1
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/service.py +137 -69
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/optimizer.py +20 -1
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/scheduler.py +13 -6
- package/plugin/vendor/reflexio/reflexio/server/services/pre_retrieval/_query_reformulator.py +40 -25
- package/plugin/vendor/reflexio/reflexio/server/services/profile/components/consolidator.py +12 -39
- package/plugin/vendor/reflexio/reflexio/server/services/profile/components/extractor.py +30 -35
- package/plugin/vendor/reflexio/reflexio/server/services/profile/service.py +122 -44
- package/plugin/vendor/reflexio/reflexio/server/services/reflection/service.py +457 -107
- package/plugin/vendor/reflexio/reflexio/server/services/retrieval/session_dedup.py +127 -0
- package/plugin/vendor/reflexio/reflexio/server/services/retrieval/temporal.py +104 -0
- package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/dispatcher.py +139 -0
- package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/judge.py +16 -6
- package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/worker.py +137 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/governance_validation.py +12 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/lifecycle_filters.py +54 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/retention.py +41 -3
- package/plugin/vendor/reflexio/reflexio/server/services/storage/retention_mixin.py +40 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/__init__.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_base.py +50 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_extras.py +11 -4
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_governance.py +28 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py +36 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_shadow_verdicts.py +4 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/base/_deletion.py +16 -5
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_erase_execution.py +104 -42
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_subject_barrier.py +7 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_agent.py +43 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_eval_results.py +430 -5
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_user.py +45 -7
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_interaction_store.py +146 -6
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_profile_store.py +42 -7
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/__init__.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_extras.py +9 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_learning_jobs.py +50 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_operations.py +7 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_retrieval_log.py +3 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_shadow_verdicts.py +4 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/agent_run/_models.py +12 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/evaluation_state_keys.py +78 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_agent.py +38 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_eval_results.py +171 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_user.py +52 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/_interaction_store.py +59 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/_profile_store.py +52 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/retrieved_learning_state.py +226 -0
- package/plugin/vendor/reflexio/reflexio/server/services/tagging/tagging_scheduler.py +5 -6
- package/plugin/vendor/reflexio/reflexio/server/services/unified_search_service.py +209 -29
- package/plugin/vendor/reflexio/reflexio/server/usage_metrics.py +3 -0
- package/plugin/vendor/reflexio/reflexio/test_support/llm_mock.py +61 -0
- package/plugin/vendor/reflexio/reflexio/test_support/llm_model_registry.py +33 -0
- package/plugin/vendor/reflexio/reflexio/server/services/search/__init__.py +0 -0
|
@@ -8,6 +8,7 @@ from dataclasses import dataclass
|
|
|
8
8
|
|
|
9
9
|
DEFAULT_ROW_RETENTION_LIMIT = 250_000
|
|
10
10
|
ROW_RETENTION_DELETE_FRACTION = 0.20
|
|
11
|
+
TOMBSTONE_STATUSES = ("archived", "merged", "superseded", "expired")
|
|
11
12
|
|
|
12
13
|
|
|
13
14
|
@dataclass(frozen=True, slots=True)
|
|
@@ -18,17 +19,32 @@ class RetentionTarget:
|
|
|
18
19
|
table_name: str
|
|
19
20
|
order_column: str
|
|
20
21
|
id_columns: tuple[str, ...]
|
|
22
|
+
priority_statuses: tuple[str, ...] = ()
|
|
21
23
|
|
|
22
24
|
|
|
23
25
|
RETENTION_TARGETS: tuple[RetentionTarget, ...] = (
|
|
24
|
-
RetentionTarget(
|
|
26
|
+
RetentionTarget(
|
|
27
|
+
"profiles",
|
|
28
|
+
"profiles",
|
|
29
|
+
"created_at",
|
|
30
|
+
("profile_id",),
|
|
31
|
+
priority_statuses=TOMBSTONE_STATUSES,
|
|
32
|
+
),
|
|
25
33
|
RetentionTarget("interactions", "interactions", "created_at", ("interaction_id",)),
|
|
26
34
|
RetentionTarget("requests", "requests", "created_at", ("request_id",)),
|
|
27
35
|
RetentionTarget(
|
|
28
|
-
"user_playbooks",
|
|
36
|
+
"user_playbooks",
|
|
37
|
+
"user_playbooks",
|
|
38
|
+
"created_at",
|
|
39
|
+
("user_playbook_id",),
|
|
40
|
+
priority_statuses=TOMBSTONE_STATUSES,
|
|
29
41
|
),
|
|
30
42
|
RetentionTarget(
|
|
31
|
-
"agent_playbooks",
|
|
43
|
+
"agent_playbooks",
|
|
44
|
+
"agent_playbooks",
|
|
45
|
+
"created_at",
|
|
46
|
+
("agent_playbook_id",),
|
|
47
|
+
priority_statuses=TOMBSTONE_STATUSES,
|
|
32
48
|
),
|
|
33
49
|
RetentionTarget(
|
|
34
50
|
"agent_success_evaluation_result",
|
|
@@ -36,6 +52,25 @@ RETENTION_TARGETS: tuple[RetentionTarget, ...] = (
|
|
|
36
52
|
"created_at",
|
|
37
53
|
("result_id",),
|
|
38
54
|
),
|
|
55
|
+
# Grouped session target: keyed on (user_id, session_id) — not result_id —
|
|
56
|
+
# so retention always removes whole session snapshots, never a partial
|
|
57
|
+
# per-learning subset. Rows within a session share created_at (earliest
|
|
58
|
+
# request timestamp), so ordering keeps groups adjacent. The session's
|
|
59
|
+
# retrieved-eval _operation_state row is intentionally left in place: it
|
|
60
|
+
# is content-free (digest + counters) and self-heals on the next
|
|
61
|
+
# publish/forced evaluation.
|
|
62
|
+
RetentionTarget(
|
|
63
|
+
"retrieved_learning_evaluation",
|
|
64
|
+
"retrieved_learning_evaluation",
|
|
65
|
+
"created_at",
|
|
66
|
+
("user_id", "session_id"),
|
|
67
|
+
),
|
|
68
|
+
RetentionTarget(
|
|
69
|
+
"offline_tuner_reward_label",
|
|
70
|
+
"offline_tuner_reward_label",
|
|
71
|
+
"label_created_at",
|
|
72
|
+
("reward_label_id",),
|
|
73
|
+
),
|
|
39
74
|
RetentionTarget("share_links", "share_links", "created_at", ("id",)),
|
|
40
75
|
RetentionTarget(
|
|
41
76
|
"agent_playbook_source_user_playbooks",
|
|
@@ -117,6 +152,9 @@ RETENTION_CASCADES: dict[str, tuple[CascadeRef, ...]] = {
|
|
|
117
152
|
"playbook_retrieval_logs": (
|
|
118
153
|
CascadeRef("playbook_retrieval_log_items", "retrieval_log_id"),
|
|
119
154
|
),
|
|
155
|
+
"offline_tuner_reward_label": (
|
|
156
|
+
CascadeRef("offline_tuner_reward_label_target", "reward_label_id"),
|
|
157
|
+
),
|
|
120
158
|
}
|
|
121
159
|
|
|
122
160
|
|
|
@@ -98,12 +98,40 @@ class RetentionMixin(ABC):
|
|
|
98
98
|
target = get_retention_target(target_name)
|
|
99
99
|
if not self._retention_table_exists(target.table_name):
|
|
100
100
|
return 0
|
|
101
|
-
keys = self.
|
|
101
|
+
keys = self._retention_select_keys(target, count)
|
|
102
102
|
if not keys:
|
|
103
103
|
return 0
|
|
104
104
|
self._retention_perform_delete(target, keys)
|
|
105
105
|
return len(keys)
|
|
106
106
|
|
|
107
|
+
def _retention_select_keys(
|
|
108
|
+
self, target: RetentionTarget, count: int
|
|
109
|
+
) -> list[tuple[Any, ...]]:
|
|
110
|
+
"""Select tombstones first, then oldest rows when a target opts in.
|
|
111
|
+
|
|
112
|
+
The tombstone pass can never under-delete: the fallback select returns
|
|
113
|
+
``min(count, table_rows)`` keys and ``seen`` only ever holds tombstones
|
|
114
|
+
already counted, so the union still reaches ``count`` whenever the
|
|
115
|
+
table holds that many rows.
|
|
116
|
+
"""
|
|
117
|
+
if not target.priority_statuses:
|
|
118
|
+
return self._retention_select_oldest_keys(target, count)
|
|
119
|
+
|
|
120
|
+
keys = self._retention_select_oldest_keys(
|
|
121
|
+
target, count, statuses=target.priority_statuses
|
|
122
|
+
)
|
|
123
|
+
if len(keys) >= count:
|
|
124
|
+
return keys
|
|
125
|
+
|
|
126
|
+
seen = set(keys)
|
|
127
|
+
for key in self._retention_select_oldest_keys(target, count):
|
|
128
|
+
if key not in seen:
|
|
129
|
+
keys.append(key)
|
|
130
|
+
seen.add(key)
|
|
131
|
+
if len(keys) == count:
|
|
132
|
+
break
|
|
133
|
+
return keys
|
|
134
|
+
|
|
107
135
|
def _retention_perform_delete(
|
|
108
136
|
self, target: RetentionTarget, keys: list[tuple[Any, ...]]
|
|
109
137
|
) -> None:
|
|
@@ -130,13 +158,23 @@ class RetentionMixin(ABC):
|
|
|
130
158
|
|
|
131
159
|
@abstractmethod
|
|
132
160
|
def _retention_select_oldest_keys(
|
|
133
|
-
self,
|
|
161
|
+
self,
|
|
162
|
+
target: RetentionTarget,
|
|
163
|
+
count: int,
|
|
164
|
+
statuses: tuple[str, ...] | None = None,
|
|
134
165
|
) -> list[tuple[Any, ...]]:
|
|
135
166
|
"""Return up to ``count`` oldest key tuples for ``target``.
|
|
136
167
|
|
|
137
168
|
Each key tuple contains values aligned with ``target.id_columns``.
|
|
138
169
|
Ordering is by ``target.order_column`` ascending with the id
|
|
139
170
|
columns as a stable tiebreaker.
|
|
171
|
+
|
|
172
|
+
Args:
|
|
173
|
+
target (RetentionTarget): Target whose keys are being selected.
|
|
174
|
+
count (int): Maximum number of key tuples to return.
|
|
175
|
+
statuses (tuple[str, ...] | None): When not None, restrict the
|
|
176
|
+
select to rows whose ``status`` is one of these values. An
|
|
177
|
+
empty tuple matches nothing (never every row).
|
|
140
178
|
"""
|
|
141
179
|
raise NotImplementedError
|
|
142
180
|
|
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
from ._learning_jobs import SQLiteLearningJobStoreMixin
|
|
2
1
|
from ._base import (
|
|
3
2
|
SQLiteStorageBase,
|
|
4
3
|
_cosine_similarity,
|
|
@@ -10,6 +9,7 @@ from ._base import (
|
|
|
10
9
|
)
|
|
11
10
|
from ._extras import ExtrasMixin
|
|
12
11
|
from ._governance import SQLiteGovernanceMixin
|
|
12
|
+
from ._learning_jobs import SQLiteLearningJobStoreMixin
|
|
13
13
|
from ._lineage import SQLiteLineageMixin
|
|
14
14
|
from ._operations import OperationMixin
|
|
15
15
|
from ._requests import RequestMixin
|
|
@@ -30,6 +30,7 @@ from reflexio.models.api_schema.service_schemas import (
|
|
|
30
30
|
ProfileTimeToLive,
|
|
31
31
|
RegularVsShadow,
|
|
32
32
|
Request,
|
|
33
|
+
RetrievedLearning,
|
|
33
34
|
Status,
|
|
34
35
|
ToolUsed,
|
|
35
36
|
UserActionType,
|
|
@@ -456,6 +457,12 @@ def _row_to_interaction(row: sqlite3.Row) -> Interaction:
|
|
|
456
457
|
if citations_raw and isinstance(citations_raw, list)
|
|
457
458
|
else []
|
|
458
459
|
)
|
|
460
|
+
retrieved_learnings_raw = _json_loads(d.get("retrieved_learnings"))
|
|
461
|
+
retrieved_learnings = (
|
|
462
|
+
[RetrievedLearning(**c) for c in retrieved_learnings_raw if isinstance(c, dict)]
|
|
463
|
+
if retrieved_learnings_raw and isinstance(retrieved_learnings_raw, list)
|
|
464
|
+
else []
|
|
465
|
+
)
|
|
459
466
|
return Interaction(
|
|
460
467
|
interaction_id=d["interaction_id"],
|
|
461
468
|
user_id=d["user_id"],
|
|
@@ -471,6 +478,7 @@ def _row_to_interaction(row: sqlite3.Row) -> Interaction:
|
|
|
471
478
|
expert_content=d.get("expert_content") or "",
|
|
472
479
|
tools_used=tools_used,
|
|
473
480
|
citations=citations,
|
|
481
|
+
retrieved_learnings=retrieved_learnings,
|
|
474
482
|
)
|
|
475
483
|
|
|
476
484
|
|
|
@@ -662,6 +670,16 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
|
|
|
662
670
|
api_key_config=self.api_key_config,
|
|
663
671
|
)
|
|
664
672
|
self.embedding_dimensions = EMBEDDING_DIMENSIONS
|
|
673
|
+
# Text-generation model for storage-time document expansion. The
|
|
674
|
+
# shared self.llm_client is pinned to the EMBEDDING model, which is
|
|
675
|
+
# not a chat model — expansion calls must override the model or they
|
|
676
|
+
# fail (e.g. local/minilm-l6-v2 cannot serve completions).
|
|
677
|
+
self._expansion_model_name = resolve_model_name(
|
|
678
|
+
ModelRole.GENERATION,
|
|
679
|
+
site_var_value=site_var.get("default_generation_model_name"),
|
|
680
|
+
config_override=llm_config.generation_model_name if llm_config else None,
|
|
681
|
+
api_key_config=self.api_key_config,
|
|
682
|
+
)
|
|
665
683
|
|
|
666
684
|
litellm_config = LiteLLMConfig(
|
|
667
685
|
model=self.embedding_model_name,
|
|
@@ -855,6 +873,14 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
|
|
|
855
873
|
)
|
|
856
874
|
self.conn.commit()
|
|
857
875
|
|
|
876
|
+
if "retrieved_learnings" not in columns:
|
|
877
|
+
logger.info("Adding retrieved_learnings column to interactions table.")
|
|
878
|
+
with self._lock:
|
|
879
|
+
self.conn.execute(
|
|
880
|
+
"ALTER TABLE interactions ADD COLUMN retrieved_learnings TEXT"
|
|
881
|
+
)
|
|
882
|
+
self.conn.commit()
|
|
883
|
+
|
|
858
884
|
def _migrate_feedback_schema(self) -> None:
|
|
859
885
|
"""Drop old-schema feedback/playbook tables so _DDL can recreate them.
|
|
860
886
|
|
|
@@ -1630,6 +1656,7 @@ class SQLiteStorageBase(RetentionMixin, BaseStorage):
|
|
|
1630
1656
|
expander = DocumentExpander(
|
|
1631
1657
|
llm_client=self.llm_client,
|
|
1632
1658
|
prompt_manager=PromptManager(),
|
|
1659
|
+
model_name=self._expansion_model_name,
|
|
1633
1660
|
)
|
|
1634
1661
|
result = expander.expand(content)
|
|
1635
1662
|
return result.expanded_text or None
|
|
@@ -1848,6 +1875,7 @@ CREATE TABLE IF NOT EXISTS interactions (
|
|
|
1848
1875
|
expert_content TEXT NOT NULL DEFAULT '',
|
|
1849
1876
|
tools_used TEXT,
|
|
1850
1877
|
citations TEXT,
|
|
1878
|
+
retrieved_learnings TEXT,
|
|
1851
1879
|
embedding TEXT
|
|
1852
1880
|
);
|
|
1853
1881
|
CREATE INDEX IF NOT EXISTS idx_interactions_user_id ON interactions(user_id);
|
|
@@ -1949,6 +1977,28 @@ CREATE INDEX IF NOT EXISTS idx_eval_agent_version_created_at_desc
|
|
|
1949
1977
|
CREATE INDEX IF NOT EXISTS idx_eval_identity_created_at_desc
|
|
1950
1978
|
ON agent_success_evaluation_result(user_id, session_id, evaluation_name, agent_version, created_at DESC);
|
|
1951
1979
|
|
|
1980
|
+
CREATE TABLE IF NOT EXISTS retrieved_learning_evaluation (
|
|
1981
|
+
result_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
1982
|
+
user_id TEXT NOT NULL,
|
|
1983
|
+
session_id TEXT NOT NULL,
|
|
1984
|
+
agent_version TEXT NOT NULL DEFAULT '',
|
|
1985
|
+
kind TEXT NOT NULL,
|
|
1986
|
+
learning_id TEXT NOT NULL,
|
|
1987
|
+
is_relevant INTEGER,
|
|
1988
|
+
relevance_reason TEXT NOT NULL DEFAULT '',
|
|
1989
|
+
impact TEXT,
|
|
1990
|
+
impact_reason TEXT NOT NULL DEFAULT '',
|
|
1991
|
+
created_at INTEGER NOT NULL,
|
|
1992
|
+
governance_subject_ref TEXT,
|
|
1993
|
+
UNIQUE (user_id, session_id, kind, learning_id)
|
|
1994
|
+
);
|
|
1995
|
+
CREATE INDEX IF NOT EXISTS idx_rle_created_at_result_id
|
|
1996
|
+
ON retrieved_learning_evaluation(created_at DESC, result_id DESC);
|
|
1997
|
+
CREATE INDEX IF NOT EXISTS idx_rle_session_id
|
|
1998
|
+
ON retrieved_learning_evaluation(session_id);
|
|
1999
|
+
CREATE INDEX IF NOT EXISTS idx_rle_subject_ref
|
|
2000
|
+
ON retrieved_learning_evaluation(governance_subject_ref);
|
|
2001
|
+
|
|
1952
2002
|
CREATE TABLE IF NOT EXISTS agent_playbook_source_user_playbooks (
|
|
1953
2003
|
agent_playbook_id INTEGER NOT NULL,
|
|
1954
2004
|
user_playbook_id INTEGER NOT NULL,
|
|
@@ -25,7 +25,14 @@ from ._base import (
|
|
|
25
25
|
_row_to_interaction,
|
|
26
26
|
)
|
|
27
27
|
|
|
28
|
-
type _CitationKind = Literal["playbook", "profile"]
|
|
28
|
+
type _CitationKind = Literal["playbook", "profile", "user_playbook", "agent_playbook"]
|
|
29
|
+
|
|
30
|
+
_SUPPORTED_CITATION_KINDS: tuple[_CitationKind, ...] = (
|
|
31
|
+
"playbook",
|
|
32
|
+
"profile",
|
|
33
|
+
"user_playbook",
|
|
34
|
+
"agent_playbook",
|
|
35
|
+
)
|
|
29
36
|
|
|
30
37
|
|
|
31
38
|
class ExtrasMixin:
|
|
@@ -291,7 +298,7 @@ class ExtrasMixin:
|
|
|
291
298
|
continue
|
|
292
299
|
kind = c.get("kind")
|
|
293
300
|
real_id = c.get("real_id")
|
|
294
|
-
if kind not in
|
|
301
|
+
if kind not in _SUPPORTED_CITATION_KINDS or not real_id:
|
|
295
302
|
continue
|
|
296
303
|
key: tuple[_CitationKind, str] = (
|
|
297
304
|
cast(_CitationKind, kind),
|
|
@@ -453,12 +460,12 @@ class ExtrasMixin:
|
|
|
453
460
|
continue
|
|
454
461
|
kind = citation.get("kind")
|
|
455
462
|
real_id = citation.get("real_id")
|
|
456
|
-
if kind and real_id:
|
|
463
|
+
if kind in _SUPPORTED_CITATION_KINDS and real_id:
|
|
457
464
|
out.append(
|
|
458
465
|
SessionCitation(
|
|
459
466
|
user_id=row["user_id"],
|
|
460
467
|
session_id=row["session_id"],
|
|
461
|
-
kind=
|
|
468
|
+
kind=cast(_CitationKind, kind),
|
|
462
469
|
real_id=str(real_id),
|
|
463
470
|
title=str(citation.get("title") or ""),
|
|
464
471
|
)
|
package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_governance.py
CHANGED
|
@@ -392,11 +392,27 @@ class SQLiteGovernanceMixin:
|
|
|
392
392
|
WHERE user_id = ?""",
|
|
393
393
|
(user_id,),
|
|
394
394
|
).fetchone()
|
|
395
|
+
rle_result_row = self.conn.execute(
|
|
396
|
+
"""SELECT COUNT(*) AS cnt
|
|
397
|
+
FROM retrieved_learning_evaluation
|
|
398
|
+
WHERE user_id = ?""",
|
|
399
|
+
(user_id,),
|
|
400
|
+
).fetchone()
|
|
401
|
+
session_count_row = self.conn.execute(
|
|
402
|
+
"SELECT COUNT(DISTINCT session_id) AS cnt FROM requests WHERE user_id = ?",
|
|
403
|
+
(user_id,),
|
|
404
|
+
).fetchone()
|
|
395
405
|
profile_rows = self.conn.execute(
|
|
396
406
|
"SELECT profile_id FROM profiles WHERE user_id = ?",
|
|
397
407
|
(user_id,),
|
|
398
408
|
).fetchall()
|
|
399
|
-
if
|
|
409
|
+
if (
|
|
410
|
+
request_row is None
|
|
411
|
+
or interaction_row is None
|
|
412
|
+
or eval_result_row is None
|
|
413
|
+
or rle_result_row is None
|
|
414
|
+
or session_count_row is None
|
|
415
|
+
):
|
|
400
416
|
raise ValueError("Missing governance count rows")
|
|
401
417
|
profile_ids = [str(row["profile_id"]) for row in profile_rows]
|
|
402
418
|
purge_profile_ids, delete_profile_ids = self._deps()._partition_purge_vs_delete(
|
|
@@ -420,6 +436,17 @@ class SQLiteGovernanceMixin:
|
|
|
420
436
|
"profile_purge": len(purge_profile_ids),
|
|
421
437
|
"user_playbook": len(delete_playbook_ids),
|
|
422
438
|
"agent_success_evaluation_result": int(eval_result_row["cnt"]),
|
|
439
|
+
# Offline tuner tables are enterprise-only (tenant stream); the
|
|
440
|
+
# OSS SQLite backend has no such tables, so the planned and
|
|
441
|
+
# deleted counts are structurally zero. The delete-target matrix
|
|
442
|
+
# validator still requires the target rows to exist.
|
|
443
|
+
"offline_tuner_reward_label": 0,
|
|
444
|
+
"offline_tuner_reward_label_target_by_target_owner": 0,
|
|
445
|
+
"retrieved_learning_evaluation_result": int(rle_result_row["cnt"]),
|
|
446
|
+
# Planned as an upper bound: up to 3 evaluation state namespaces
|
|
447
|
+
# per session (retrieved-eval state, agent-success marker,
|
|
448
|
+
# grade-cache rows). The delete phase reports the exact count.
|
|
449
|
+
"evaluation_operation_state": 3 * int(session_count_row["cnt"]),
|
|
423
450
|
"user_playbook_purge": len(purge_playbook_ids),
|
|
424
451
|
}
|
|
425
452
|
|
package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py
CHANGED
|
@@ -262,11 +262,18 @@ class SQLiteLearningJobStoreMixin(LearningJobStoreABC):
|
|
|
262
262
|
job_id: str,
|
|
263
263
|
claim_token: str,
|
|
264
264
|
dead: bool,
|
|
265
|
+
refund_attempt: bool = False,
|
|
265
266
|
) -> None:
|
|
266
267
|
"""Fenced fail/dead transition — sets status, clears token for retry.
|
|
267
268
|
|
|
268
269
|
Does NOT increment attempts: claim_learning_jobs already incremented on
|
|
269
270
|
delivery. attempts tracks delivery count; fail only transitions status.
|
|
271
|
+
|
|
272
|
+
``refund_attempt=True`` (the same-user-contention requeue, F4) decrements
|
|
273
|
+
``attempts`` by one (``MAX(attempts - 1, 0)``) so a contention cycle
|
|
274
|
+
(claim +1, contention-release -1) nets to zero and ``attempts`` stays
|
|
275
|
+
bounded regardless of how many times the job loses the per-user race.
|
|
276
|
+
Defaults to ``False`` (the ``dead`` retry path is unchanged).
|
|
270
277
|
"""
|
|
271
278
|
new_status = "dead" if dead else "failed"
|
|
272
279
|
# Clear claim_token and claim_expires_at only for 'failed' so it's reclaimable.
|
|
@@ -282,10 +289,11 @@ class SQLiteLearningJobStoreMixin(LearningJobStoreABC):
|
|
|
282
289
|
status = ?,
|
|
283
290
|
claim_token = CASE WHEN ? THEN claim_token ELSE NULL END,
|
|
284
291
|
claim_expires_at = CASE WHEN ? THEN claim_expires_at ELSE NULL END,
|
|
292
|
+
attempts = CASE WHEN ? THEN MAX(attempts - 1, 0) ELSE attempts END,
|
|
285
293
|
updated_at = strftime('%Y-%m-%dT%H:%M:%fZ','now')
|
|
286
294
|
WHERE job_id = ? AND claim_token = ? AND status = 'claimed'
|
|
287
295
|
""",
|
|
288
|
-
(new_status, dead, dead, job_id, claim_token),
|
|
296
|
+
(new_status, dead, dead, refund_attempt, job_id, claim_token),
|
|
289
297
|
)
|
|
290
298
|
if own_txn:
|
|
291
299
|
self.conn.commit()
|
|
@@ -316,6 +324,33 @@ class SQLiteLearningJobStoreMixin(LearningJobStoreABC):
|
|
|
316
324
|
)
|
|
317
325
|
return [str(row["org_id"]) for row in rows]
|
|
318
326
|
|
|
327
|
+
@SQLiteStorageBase.handle_exceptions
|
|
328
|
+
def get_oldest_pending_learning_job_age_seconds(self) -> float | None:
|
|
329
|
+
"""Age in seconds of the oldest pending job, or None if none pending.
|
|
330
|
+
|
|
331
|
+
Computes the age entirely in SQLite using ``strftime('%s','now')`` to
|
|
332
|
+
avoid app/DB clock skew. The table is accessed without a schema prefix
|
|
333
|
+
(SQLite has no schema routing — the local DB file is the ref).
|
|
334
|
+
"""
|
|
335
|
+
row = self._fetchall(
|
|
336
|
+
"""
|
|
337
|
+
SELECT (strftime('%s', 'now') - strftime('%s', MIN(created_at))) AS age_seconds
|
|
338
|
+
FROM learning_jobs
|
|
339
|
+
WHERE status = 'pending'
|
|
340
|
+
""",
|
|
341
|
+
(),
|
|
342
|
+
)
|
|
343
|
+
return None if row[0]["age_seconds"] is None else float(row[0]["age_seconds"])
|
|
344
|
+
|
|
345
|
+
@SQLiteStorageBase.handle_exceptions
|
|
346
|
+
def count_learning_jobs_by_status(self, status: str) -> int:
|
|
347
|
+
"""Count of learning jobs with the given status on this storage ref."""
|
|
348
|
+
row = self._fetchall(
|
|
349
|
+
"SELECT COUNT(*) AS cnt FROM learning_jobs WHERE status = ?",
|
|
350
|
+
(status,),
|
|
351
|
+
)
|
|
352
|
+
return int(row[0]["cnt"])
|
|
353
|
+
|
|
319
354
|
@SQLiteStorageBase.handle_exceptions
|
|
320
355
|
def get_learning_status_for_request(
|
|
321
356
|
self,
|
package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_shadow_verdicts.py
CHANGED
|
@@ -74,6 +74,10 @@ class ShadowVerdictsMixin:
|
|
|
74
74
|
_fetchone: Any
|
|
75
75
|
_fetchall: Any
|
|
76
76
|
|
|
77
|
+
def supports_shadow_comparison_verdicts(self) -> bool:
|
|
78
|
+
"""Return whether this backend can persist shadow comparison verdicts."""
|
|
79
|
+
return True
|
|
80
|
+
|
|
77
81
|
@SQLiteStorageBase.handle_exceptions
|
|
78
82
|
def save_shadow_comparison_verdict(
|
|
79
83
|
self, verdict: ShadowComparisonVerdict
|
package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/base/_deletion.py
CHANGED
|
@@ -46,14 +46,25 @@ class SQLiteDeletionMixin:
|
|
|
46
46
|
|
|
47
47
|
@SQLiteStorageBase.handle_exceptions
|
|
48
48
|
def _retention_select_oldest_keys(
|
|
49
|
-
self,
|
|
49
|
+
self,
|
|
50
|
+
target: RetentionTarget,
|
|
51
|
+
count: int,
|
|
52
|
+
statuses: tuple[str, ...] | None = None,
|
|
50
53
|
) -> list[tuple[Any, ...]]:
|
|
54
|
+
if statuses is not None and not statuses:
|
|
55
|
+
return []
|
|
51
56
|
id_sql = ", ".join(target.id_columns)
|
|
52
|
-
|
|
57
|
+
where_sql = ""
|
|
58
|
+
params: list[Any] = []
|
|
59
|
+
if statuses:
|
|
60
|
+
placeholders = ", ".join("?" for _ in statuses)
|
|
61
|
+
where_sql = f"WHERE status IN ({placeholders}) "
|
|
62
|
+
params.extend(statuses)
|
|
63
|
+
params.append(count)
|
|
53
64
|
rows = self._fetchall(
|
|
54
|
-
f"SELECT {id_sql} FROM {target.table_name} " # noqa: S608
|
|
55
|
-
f"ORDER BY {target.order_column} ASC, {
|
|
56
|
-
(
|
|
65
|
+
f"SELECT {id_sql} FROM {target.table_name} {where_sql}" # noqa: S608
|
|
66
|
+
f"ORDER BY {target.order_column} ASC, {id_sql} ASC LIMIT ?",
|
|
67
|
+
tuple(params),
|
|
57
68
|
)
|
|
58
69
|
return [tuple(row[col] for col in target.id_columns) for row in rows]
|
|
59
70
|
|
|
@@ -30,7 +30,6 @@ resolve through the composed MRO.
|
|
|
30
30
|
|
|
31
31
|
from __future__ import annotations
|
|
32
32
|
|
|
33
|
-
import json
|
|
34
33
|
import sqlite3
|
|
35
34
|
import threading
|
|
36
35
|
from collections.abc import Callable
|
|
@@ -50,6 +49,14 @@ from reflexio.server.services.storage.governance_validation import (
|
|
|
50
49
|
_successful_erase_identity,
|
|
51
50
|
_validate_governance_purge_id,
|
|
52
51
|
)
|
|
52
|
+
from reflexio.server.services.storage.storage_base.evaluation_state_keys import (
|
|
53
|
+
GRADE_ON_DEMAND_CACHE_PREFIX,
|
|
54
|
+
build_agent_success_marker_key,
|
|
55
|
+
build_grade_on_demand_session_prefix,
|
|
56
|
+
)
|
|
57
|
+
from reflexio.server.services.storage.storage_base.retrieved_learning_state import (
|
|
58
|
+
build_retrieved_learning_state_key,
|
|
59
|
+
)
|
|
53
60
|
|
|
54
61
|
from .._governance import _row_to_audit_event, _row_to_purge_operation
|
|
55
62
|
|
|
@@ -157,13 +164,6 @@ class GovernanceEraseExecutionMixin:
|
|
|
157
164
|
raise ValueError(
|
|
158
165
|
"Current user playbooks no longer match prepared purge snapshot"
|
|
159
166
|
)
|
|
160
|
-
request_ids = [
|
|
161
|
-
str(row["request_id"])
|
|
162
|
-
for row in self.conn.execute(
|
|
163
|
-
"SELECT request_id FROM requests WHERE user_id = ?",
|
|
164
|
-
(user_id,),
|
|
165
|
-
).fetchall()
|
|
166
|
-
]
|
|
167
167
|
profile_rows = self.conn.execute(
|
|
168
168
|
"SELECT rowid, profile_id FROM profiles WHERE user_id = ?",
|
|
169
169
|
(user_id,),
|
|
@@ -183,40 +183,12 @@ class GovernanceEraseExecutionMixin:
|
|
|
183
183
|
)
|
|
184
184
|
purge_upb_ids = [int(entity_id) for entity_id in purge_upb_str_ids]
|
|
185
185
|
delete_upb_ids = [int(entity_id) for entity_id in delete_upb_str_ids]
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
if erased_entity_ids:
|
|
193
|
-
erased_entity_id_set = set(erased_entity_ids)
|
|
194
|
-
lineage_source_event_ids: list[int] = []
|
|
195
|
-
for row in self.conn.execute(
|
|
196
|
-
"SELECT event_id, source_ids FROM lineage_event WHERE org_id = ?",
|
|
197
|
-
(self.org_id,),
|
|
198
|
-
).fetchall():
|
|
199
|
-
try:
|
|
200
|
-
source_ids = json.loads(str(row["source_ids"] or "[]"))
|
|
201
|
-
except json.JSONDecodeError:
|
|
202
|
-
source_ids = []
|
|
203
|
-
if any(
|
|
204
|
-
str(source_id) in erased_entity_id_set for source_id in source_ids
|
|
205
|
-
):
|
|
206
|
-
lineage_source_event_ids.append(int(row["event_id"]))
|
|
207
|
-
deps._delete_in_chunks(
|
|
208
|
-
"lineage_event", "event_id", lineage_source_event_ids
|
|
209
|
-
)
|
|
210
|
-
placeholders = ",".join("?" for _ in erased_entity_ids)
|
|
211
|
-
self.conn.execute(
|
|
212
|
-
f"""DELETE FROM lineage_event
|
|
213
|
-
WHERE org_id = ?
|
|
214
|
-
AND (
|
|
215
|
-
request_id IN ({placeholders})
|
|
216
|
-
OR entity_id IN ({placeholders})
|
|
217
|
-
)""", # noqa: S608
|
|
218
|
-
[self.org_id, *erased_entity_ids, *erased_entity_ids],
|
|
219
|
-
)
|
|
186
|
+
# SEC-016: retain the content-free lineage_event skeleton on erase to
|
|
187
|
+
# match Supabase (which never enumerates lineage_event for erasure).
|
|
188
|
+
# lineage_event has no PII/content column, no foreign keys, and no
|
|
189
|
+
# FTS/vec shadow tables, so leaving these rows in place is safe and
|
|
190
|
+
# preserves the audit/lineage skeleton. Content-bearing entities are
|
|
191
|
+
# still deleted/purged below.
|
|
220
192
|
delete_profile_rowids = [
|
|
221
193
|
profile_rowid_by_id[profile_id]
|
|
222
194
|
for profile_id in delete_profile_ids
|
|
@@ -240,6 +212,23 @@ class GovernanceEraseExecutionMixin:
|
|
|
240
212
|
WHERE user_id = ?""",
|
|
241
213
|
(user_id,),
|
|
242
214
|
)
|
|
215
|
+
rle_results_cur = self.conn.execute(
|
|
216
|
+
"""DELETE FROM retrieved_learning_evaluation
|
|
217
|
+
WHERE user_id = ?""",
|
|
218
|
+
(user_id,),
|
|
219
|
+
)
|
|
220
|
+
# Snapshot the user's session ids BEFORE requests are deleted — the
|
|
221
|
+
# evaluation _operation_state namespaces are keyed by session.
|
|
222
|
+
session_ids = [
|
|
223
|
+
str(row["session_id"])
|
|
224
|
+
for row in self.conn.execute(
|
|
225
|
+
"SELECT DISTINCT session_id FROM requests WHERE user_id = ?",
|
|
226
|
+
(user_id,),
|
|
227
|
+
).fetchall()
|
|
228
|
+
]
|
|
229
|
+
eval_operation_states = self._delete_evaluation_operation_states_locked(
|
|
230
|
+
user_id, session_ids
|
|
231
|
+
)
|
|
243
232
|
requests_cur = self.conn.execute(
|
|
244
233
|
"DELETE FROM requests WHERE user_id = ?",
|
|
245
234
|
(user_id,),
|
|
@@ -280,10 +269,75 @@ class GovernanceEraseExecutionMixin:
|
|
|
280
269
|
"profiles": len(delete_profile_ids),
|
|
281
270
|
"requests": requests_cur.rowcount,
|
|
282
271
|
"agent_success_evaluation_results": eval_results_cur.rowcount,
|
|
272
|
+
# Enterprise-only tables absent from the OSS SQLite backend;
|
|
273
|
+
# reported as zero so the delete target matrix stays complete.
|
|
274
|
+
"offline_tuner_reward_labels": 0,
|
|
275
|
+
"offline_tuner_reward_label_targets_by_target_owner": 0,
|
|
276
|
+
"retrieved_learning_evaluation_results": rle_results_cur.rowcount,
|
|
277
|
+
"evaluation_operation_states": eval_operation_states,
|
|
283
278
|
"purged_profiles": purged_profiles,
|
|
284
279
|
"purged_user_playbooks": purged_user_playbooks,
|
|
285
280
|
}
|
|
286
281
|
|
|
282
|
+
def _delete_evaluation_operation_states_locked(
|
|
283
|
+
self, user_id: str, session_ids: list[str]
|
|
284
|
+
) -> int:
|
|
285
|
+
"""Scrub the three evaluation ``_operation_state`` namespaces.
|
|
286
|
+
|
|
287
|
+
Deletes, for every session the user owns: the retrieved-learning
|
|
288
|
+
generation/completion state, the ``agent_success_group_eval`` marker
|
|
289
|
+
(pre-existing RTBF gap), and the ``grade_on_demand`` cache rows
|
|
290
|
+
(pre-existing RTBF gap). Exact keys are used for the first two; the
|
|
291
|
+
grade cache is matched by the injective per-session key prefix so any
|
|
292
|
+
agent_version / evaluation_name variant is covered.
|
|
293
|
+
|
|
294
|
+
Returns:
|
|
295
|
+
int: Number of state rows deleted.
|
|
296
|
+
"""
|
|
297
|
+
if not session_ids:
|
|
298
|
+
return 0
|
|
299
|
+
deleted = 0
|
|
300
|
+
exact_keys = [
|
|
301
|
+
build_retrieved_learning_state_key(user_id, session_id)
|
|
302
|
+
for session_id in session_ids
|
|
303
|
+
] + [
|
|
304
|
+
build_agent_success_marker_key(self.org_id, user_id, session_id)
|
|
305
|
+
for session_id in session_ids
|
|
306
|
+
]
|
|
307
|
+
chunk_size = 500
|
|
308
|
+
for start in range(0, len(exact_keys), chunk_size):
|
|
309
|
+
chunk = exact_keys[start : start + chunk_size]
|
|
310
|
+
placeholders = ",".join("?" for _ in chunk)
|
|
311
|
+
cur = self.conn.execute(
|
|
312
|
+
f"DELETE FROM _operation_state WHERE service_name IN ({placeholders})", # noqa: S608
|
|
313
|
+
chunk,
|
|
314
|
+
)
|
|
315
|
+
deleted += cur.rowcount
|
|
316
|
+
# Grade-cache keys embed agent_version/evaluation_name, so enumerate
|
|
317
|
+
# the namespace and match each session's injective prefix in Python
|
|
318
|
+
# (LIKE-escaping free-form session ids is not safe).
|
|
319
|
+
prefixes = tuple(
|
|
320
|
+
build_grade_on_demand_session_prefix(self.org_id, session_id)
|
|
321
|
+
for session_id in session_ids
|
|
322
|
+
)
|
|
323
|
+
grade_keys = [
|
|
324
|
+
str(row["service_name"])
|
|
325
|
+
for row in self.conn.execute(
|
|
326
|
+
"SELECT service_name FROM _operation_state WHERE service_name LIKE ?",
|
|
327
|
+
(f"{GRADE_ON_DEMAND_CACHE_PREFIX}::%",),
|
|
328
|
+
).fetchall()
|
|
329
|
+
if str(row["service_name"]).startswith(prefixes)
|
|
330
|
+
]
|
|
331
|
+
for start in range(0, len(grade_keys), chunk_size):
|
|
332
|
+
chunk = grade_keys[start : start + chunk_size]
|
|
333
|
+
placeholders = ",".join("?" for _ in chunk)
|
|
334
|
+
cur = self.conn.execute(
|
|
335
|
+
f"DELETE FROM _operation_state WHERE service_name IN ({placeholders})", # noqa: S608
|
|
336
|
+
chunk,
|
|
337
|
+
)
|
|
338
|
+
deleted += cur.rowcount
|
|
339
|
+
return deleted
|
|
340
|
+
|
|
287
341
|
def apply_governance_user_data_delete(
|
|
288
342
|
self, purge_id: str, user_id: str
|
|
289
343
|
) -> dict[str, int]:
|
|
@@ -294,6 +348,14 @@ class GovernanceEraseExecutionMixin:
|
|
|
294
348
|
"profiles": "profile",
|
|
295
349
|
"requests": "request",
|
|
296
350
|
"agent_success_evaluation_results": "agent_success_evaluation_result",
|
|
351
|
+
"offline_tuner_reward_labels": "offline_tuner_reward_label",
|
|
352
|
+
"offline_tuner_reward_label_targets_by_target_owner": (
|
|
353
|
+
"offline_tuner_reward_label_target_by_target_owner"
|
|
354
|
+
),
|
|
355
|
+
"retrieved_learning_evaluation_results": (
|
|
356
|
+
"retrieved_learning_evaluation_result"
|
|
357
|
+
),
|
|
358
|
+
"evaluation_operation_states": "evaluation_operation_state",
|
|
297
359
|
"purged_profiles": "profile_purge",
|
|
298
360
|
"purged_user_playbooks": "user_playbook_purge",
|
|
299
361
|
}
|