claude-smart 0.2.46 → 0.2.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +19 -11
- package/bin/claude-smart.js +290 -68
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/.codex-plugin/plugin.json +1 -1
- package/plugin/README.md +11 -10
- package/plugin/dashboard/app/layout.tsx +20 -0
- package/plugin/opencode/dist/server.mjs +76 -2
- package/plugin/opencode/server.mts +79 -2
- package/plugin/pyproject.toml +6 -2
- package/plugin/scripts/smart-install.sh +7 -1
- package/plugin/src/claude_smart/cli.py +210 -22
- package/plugin/src/claude_smart/context_format.py +9 -9
- package/plugin/src/claude_smart/cs_cite.py +66 -19
- package/plugin/uv.lock +5 -5
- package/plugin/vendor/reflexio/README.md +3 -3
- package/plugin/vendor/reflexio/pyproject.toml +1 -1
- package/plugin/vendor/reflexio/reflexio/README.md +3 -1
- package/plugin/vendor/reflexio/reflexio/cli/bootstrap_config.py +1 -1
- package/plugin/vendor/reflexio/reflexio/cli/commands/setup_cmd.py +2 -2
- package/plugin/vendor/reflexio/reflexio/cli/utils.py +44 -1
- package/plugin/vendor/reflexio/reflexio/client/client.py +97 -0
- package/plugin/vendor/reflexio/reflexio/lib/_agent_playbook.py +8 -0
- package/plugin/vendor/reflexio/reflexio/lib/_base.py +15 -0
- package/plugin/vendor/reflexio/reflexio/lib/_generation.py +9 -8
- package/plugin/vendor/reflexio/reflexio/lib/_profiles.py +27 -16
- package/plugin/vendor/reflexio/reflexio/lib/_search.py +23 -5
- package/plugin/vendor/reflexio/reflexio/lib/_user_playbook.py +9 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/__init__.py +1 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/governance.py +117 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/retriever_schema.py +45 -3
- package/plugin/vendor/reflexio/reflexio/models/config_schema.py +16 -0
- package/plugin/vendor/reflexio/reflexio/server/README.md +14 -2
- package/plugin/vendor/reflexio/reflexio/server/api.py +176 -29
- package/plugin/vendor/reflexio/reflexio/server/api_endpoints/request_context.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/{_auth.py → auth.py} +2 -0
- package/plugin/vendor/reflexio/reflexio/server/extensions.py +213 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/model_defaults.py +4 -4
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/claude_code_provider.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/llm/rerank/cross_encoder_reranker.py +12 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_context/v4.4.0.prompt.md +14 -2
- package/plugin/vendor/reflexio/reflexio/server/services/README.md +3 -1
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_worker.py +8 -0
- package/plugin/vendor/reflexio/reflexio/server/services/governance/config.py +52 -0
- package/plugin/vendor/reflexio/reflexio/server/services/governance/service.py +378 -0
- package/plugin/vendor/reflexio/reflexio/server/services/governance/subject_refs.py +34 -0
- package/plugin/vendor/reflexio/reflexio/server/services/lineage/gc_scheduler.py +45 -19
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/README.md +9 -1
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/aggregation_prompt_processing.py +100 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/aggregator.py +129 -111
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/service.py +9 -9
- package/plugin/vendor/reflexio/reflexio/server/services/profile/components/extractor.py +3 -2
- package/plugin/vendor/reflexio/reflexio/server/services/retrieval/recency.py +211 -0
- package/plugin/vendor/reflexio/reflexio/server/services/retrieval/relevance_floor.py +29 -13
- package/plugin/vendor/reflexio/reflexio/server/services/storage/error.py +4 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/governance_validation.py +681 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/__init__.py +14 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_base.py +2 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_extras.py +49 -19
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_governance.py +1965 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_lineage.py +5 -3
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_playbook.py +1 -2133
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_profiles.py +262 -107
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_requests.py +73 -33
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/__init__.py +13 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_agent.py +952 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_eval_results.py +189 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_optimization.py +247 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_source_linkage.py +145 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_user.py +838 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/__init__.py +19 -3
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_extras.py +4 -4
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_governance.py +148 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_playbook.py +0 -909
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_profiles.py +13 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_requests.py +2 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/__init__.py +13 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_agent.py +365 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_eval_results.py +124 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_optimization.py +85 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_source_linkage.py +47 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_user.py +333 -0
- package/plugin/vendor/reflexio/reflexio/server/services/unified_search_service.py +153 -12
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/user_detail_stripping.py +0 -84
|
@@ -52,6 +52,7 @@ strings before deleting old import paths in the same PR.
|
|
|
52
52
|
| `playbook_optimizer/` | Scenario-based playbook optimization: mature flat package with `optimizer.py`, `scheduler.py`, `rollout.py`, `judge.py`, `models.py`, `scenario_resolver.py`, `gepa_adapter.py`, and `assistant_webhook.py`. See [README](playbook_optimizer/README.md). |
|
|
53
53
|
| `braintrust/` | Braintrust export/sync: `service.py`, `client.py`, `_cron.py`, `_encryption.py`. |
|
|
54
54
|
| `lineage/` | Current-record resolution and tombstone GC: `resolve.py`, `gc_scheduler.py`. |
|
|
55
|
+
| `governance/` | Subject-reference and retention/barrier policy contracts: `config.py`, `service.py`, `subject_refs.py`. Storage and lineage code use these helpers to preserve governance constraints during writes, supersedes, and cleanup. |
|
|
55
56
|
| `pre_retrieval/` | `QueryReformulator` (`_query_reformulator.py`) + `DocumentExpander` (`_document_expander.py`) - query rewrite and doc expansion for recall. Compact by design; see [README](pre_retrieval/README.md). |
|
|
56
57
|
| `tagging/` | `TaggingService` (`service.py`) + deferred `tagging_scheduler.py` - post-generation profile/playbook tagging. Compact by design; see [README](tagging/README.md). |
|
|
57
58
|
| `unified_search_service.py` | `run_unified_search()` — two-phase parallel search across profiles / agent playbooks / user playbooks. |
|
|
@@ -61,7 +62,7 @@ strings before deleting old import paths in the same PR.
|
|
|
61
62
|
|
|
62
63
|
| Path | Purpose |
|
|
63
64
|
|------|---------|
|
|
64
|
-
| `storage/` | `storage_base/` (`BaseStorage` split by domain, including `_lineage.py`) + `sqlite_storage/` (
|
|
65
|
+
| `storage/` | `storage_base/` (`BaseStorage` split by domain, including `_governance.py` and `_lineage.py`) + `sqlite_storage/` (governance validation, retention barriers, lineage/tombstones) + `governance_validation.py` + `retention*.py`. Access via `request_context.storage` only. |
|
|
65
66
|
| `configurator/` | `DefaultConfigurator` — loads YAML config and creates the storage backend. |
|
|
66
67
|
|
|
67
68
|
## Key Rules
|
|
@@ -71,3 +72,4 @@ strings before deleting old import paths in the same PR.
|
|
|
71
72
|
- **ALWAYS use `LiteLLMClient`** for completions/embeddings and `request_context.prompt_manager.render_prompt(...)` for prompts — no hardcoded prompts, no direct OpenAI/Claude clients.
|
|
72
73
|
- **All `_operation_state` writes go through `OperationStateManager`** — don't touch the table directly (it backs locks, bookmarks, progress, and cancellation).
|
|
73
74
|
- **`tool_can_use` lives at root `Config`** — shared by playbook extraction and success evaluation, not per-service.
|
|
75
|
+
- **Preserve governance subject refs/barriers** — route validation through `services/governance/` and `storage/governance_validation.py`; do not bypass retention or subject-write checks in storage implementations.
|
|
@@ -484,11 +484,18 @@ class ExtractionResumeWorker:
|
|
|
484
484
|
raw_profiles = [
|
|
485
485
|
profile.model_dump() for profile in result.output.profiles or []
|
|
486
486
|
]
|
|
487
|
+
source_interaction_ids = [
|
|
488
|
+
interaction.interaction_id
|
|
489
|
+
for request_model in request_interaction_data_models
|
|
490
|
+
for interaction in request_model.interactions
|
|
491
|
+
if interaction.interaction_id
|
|
492
|
+
]
|
|
487
493
|
return (
|
|
488
494
|
extractor._convert_raw_to_user_profiles(
|
|
489
495
|
raw_profiles=raw_profiles,
|
|
490
496
|
user_id=run.binding.user_id,
|
|
491
497
|
request_id=run.binding.request_id,
|
|
498
|
+
source_interaction_ids=source_interaction_ids,
|
|
492
499
|
),
|
|
493
500
|
result.pending_tool_call_ids,
|
|
494
501
|
)
|
|
@@ -698,6 +705,7 @@ class ExtractionResumeWorker:
|
|
|
698
705
|
raw_profiles=raw_profiles,
|
|
699
706
|
user_id=run.binding.user_id,
|
|
700
707
|
request_id=run.binding.request_id,
|
|
708
|
+
source_interaction_ids=list(run.binding.source_interaction_ids),
|
|
701
709
|
),
|
|
702
710
|
run.pending_tool_call_ids,
|
|
703
711
|
)
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from reflexio.server.env_utils import env_str, env_truthy
|
|
4
|
+
from reflexio.server.services.governance.subject_refs import (
|
|
5
|
+
actor_ref,
|
|
6
|
+
request_ref,
|
|
7
|
+
subject_ref,
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
LOCAL_GOVERNANCE_REF_SECRET = "reflexio-local-governance-ref-secret" # noqa: S105
|
|
11
|
+
_PRODUCTION_DEPLOYMENT_MODES = {"platform", "self_host"}
|
|
12
|
+
_PRODUCTION_ENVS = {"production", "staging"}
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _is_local_dev_or_test() -> bool:
|
|
16
|
+
deployment_mode = env_str("DEPLOYMENT_MODE").lower()
|
|
17
|
+
reflexio_env = env_str("REFLEXIO_ENV", "development").lower()
|
|
18
|
+
if env_truthy(env_str("REFLEXIO_TEST_MODE")):
|
|
19
|
+
return True
|
|
20
|
+
return (
|
|
21
|
+
deployment_mode not in _PRODUCTION_DEPLOYMENT_MODES
|
|
22
|
+
and reflexio_env not in _PRODUCTION_ENVS
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def get_governance_ref_secret() -> str:
|
|
27
|
+
secret = env_str("REFLEXIO_GOVERNANCE_REF_SECRET")
|
|
28
|
+
if secret:
|
|
29
|
+
return secret
|
|
30
|
+
if _is_local_dev_or_test():
|
|
31
|
+
return LOCAL_GOVERNANCE_REF_SECRET
|
|
32
|
+
raise RuntimeError("REFLEXIO_GOVERNANCE_REF_SECRET is required")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def governance_subject_ref(org_id: str, user_id: str, secret: str) -> str:
|
|
36
|
+
return subject_ref(f"{org_id}:subject:{user_id}", secret)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def governance_request_ref(org_id: str, request_id: str, secret: str) -> str:
|
|
40
|
+
return request_ref(f"{org_id}:request:{request_id}", secret)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def governance_actor_ref(
|
|
44
|
+
org_id: str,
|
|
45
|
+
credential_kind: str,
|
|
46
|
+
principal_or_fingerprint: str,
|
|
47
|
+
secret: str,
|
|
48
|
+
) -> str:
|
|
49
|
+
return actor_ref(
|
|
50
|
+
f"{org_id}:actor:{credential_kind}:{principal_or_fingerprint}",
|
|
51
|
+
secret,
|
|
52
|
+
)
|
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from contextlib import suppress
|
|
4
|
+
from typing import Any, Literal, TypedDict
|
|
5
|
+
|
|
6
|
+
from reflexio.models.api_schema.domain.governance import (
|
|
7
|
+
AuditEvent,
|
|
8
|
+
PurgeOperationTarget,
|
|
9
|
+
SubjectWriteBarrier,
|
|
10
|
+
UserEraseResult,
|
|
11
|
+
UserExportResult,
|
|
12
|
+
)
|
|
13
|
+
from reflexio.server.services.governance.config import (
|
|
14
|
+
get_governance_ref_secret,
|
|
15
|
+
governance_request_ref,
|
|
16
|
+
governance_subject_ref,
|
|
17
|
+
)
|
|
18
|
+
from reflexio.server.services.governance.subject_refs import stable_id
|
|
19
|
+
|
|
20
|
+
_DELETE_TARGET_NAME_TO_RESULT_KEY = {
|
|
21
|
+
"interaction": "interactions",
|
|
22
|
+
"user_playbook": "user_playbooks",
|
|
23
|
+
"profile": "profiles",
|
|
24
|
+
"request": "requests",
|
|
25
|
+
"agent_success_evaluation_result": "agent_success_evaluation_results",
|
|
26
|
+
"profile_purge": "purged_profiles",
|
|
27
|
+
"user_playbook_purge": "purged_user_playbooks",
|
|
28
|
+
}
|
|
29
|
+
_REQUIRED_DELETE_TARGET_NAMES = tuple(_DELETE_TARGET_NAME_TO_RESULT_KEY)
|
|
30
|
+
_USER_PLAYBOOK_PAGE_SIZE = 1000
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class GovernanceActorContext(TypedDict):
|
|
34
|
+
actor_type: Literal["api_token", "jwt", "system"]
|
|
35
|
+
actor_ref: str | None
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class GovernanceService:
|
|
39
|
+
def __init__(self, *, storage: Any, org_id: str, ref_secret: str) -> None:
|
|
40
|
+
self.storage = storage
|
|
41
|
+
self.org_id = org_id
|
|
42
|
+
self.ref_secret = ref_secret
|
|
43
|
+
|
|
44
|
+
def export_user(
|
|
45
|
+
self,
|
|
46
|
+
*,
|
|
47
|
+
user_id: str,
|
|
48
|
+
request_id: str,
|
|
49
|
+
actor_context: GovernanceActorContext | None = None,
|
|
50
|
+
) -> UserExportResult:
|
|
51
|
+
self._assert_storage_ref_secret_matches()
|
|
52
|
+
subref = governance_subject_ref(self.org_id, user_id, self.ref_secret)
|
|
53
|
+
reqref = governance_request_ref(self.org_id, request_id, self.ref_secret)
|
|
54
|
+
export_id = stable_id("export", f"{self.org_id}:export:{subref}:{reqref}")
|
|
55
|
+
actor_type = actor_context["actor_type"] if actor_context else "system"
|
|
56
|
+
actor_ref = actor_context["actor_ref"] if actor_context else None
|
|
57
|
+
requests, sessions = self._load_user_requests_and_sessions(user_id)
|
|
58
|
+
bundle: dict[str, Any] = {
|
|
59
|
+
"profiles": [
|
|
60
|
+
profile.model_dump()
|
|
61
|
+
for profile in self.storage.get_user_profile(user_id)
|
|
62
|
+
],
|
|
63
|
+
"interactions": [
|
|
64
|
+
interaction.model_dump()
|
|
65
|
+
for interaction in self.storage.get_user_interaction(user_id)
|
|
66
|
+
],
|
|
67
|
+
"requests": [request.model_dump() for request in requests],
|
|
68
|
+
"sessions": sessions,
|
|
69
|
+
"user_playbooks": [
|
|
70
|
+
playbook.model_dump() for playbook in self._iter_user_playbooks(user_id)
|
|
71
|
+
],
|
|
72
|
+
}
|
|
73
|
+
self.storage.append_audit_event(
|
|
74
|
+
AuditEvent(
|
|
75
|
+
org_id=self.org_id,
|
|
76
|
+
actor_type=actor_type,
|
|
77
|
+
actor_ref=actor_ref,
|
|
78
|
+
operation="EXPORT",
|
|
79
|
+
entity_type="request",
|
|
80
|
+
subject_ref=subref,
|
|
81
|
+
request_ref=reqref,
|
|
82
|
+
idempotency_key=export_id,
|
|
83
|
+
detail={"count": sum(len(items) for items in bundle.values())},
|
|
84
|
+
)
|
|
85
|
+
)
|
|
86
|
+
return UserExportResult(subject_ref=subref, export_id=export_id, bundle=bundle)
|
|
87
|
+
|
|
88
|
+
def erase_user(
|
|
89
|
+
self,
|
|
90
|
+
*,
|
|
91
|
+
user_id: str,
|
|
92
|
+
request_id: str,
|
|
93
|
+
actor_context: GovernanceActorContext | None = None,
|
|
94
|
+
) -> UserEraseResult:
|
|
95
|
+
self._assert_storage_ref_secret_matches()
|
|
96
|
+
subref = governance_subject_ref(self.org_id, user_id, self.ref_secret)
|
|
97
|
+
reqref = governance_request_ref(self.org_id, request_id, self.ref_secret)
|
|
98
|
+
actor_type = actor_context["actor_type"] if actor_context else "system"
|
|
99
|
+
actor_ref = actor_context["actor_ref"] if actor_context else None
|
|
100
|
+
idempotency_key = stable_id(
|
|
101
|
+
"idem",
|
|
102
|
+
f"{self.org_id}:user_erasure:{subref}:{reqref}",
|
|
103
|
+
)
|
|
104
|
+
purge_id = stable_id("purge", idempotency_key)
|
|
105
|
+
purge = self.storage.begin_purge_operation(
|
|
106
|
+
purge_id=purge_id,
|
|
107
|
+
idempotency_key=idempotency_key,
|
|
108
|
+
operation_type="user_erasure",
|
|
109
|
+
scope_type="user",
|
|
110
|
+
subject_ref=subref,
|
|
111
|
+
request_ref=reqref,
|
|
112
|
+
)
|
|
113
|
+
if purge.status == "complete":
|
|
114
|
+
barrier = self._completed_barrier_for_retry(
|
|
115
|
+
subject_ref=subref, purge_id=purge_id
|
|
116
|
+
)
|
|
117
|
+
if barrier.status != "erased":
|
|
118
|
+
raise ValueError(
|
|
119
|
+
"Completed purge retry requires an erased subject barrier"
|
|
120
|
+
)
|
|
121
|
+
return UserEraseResult(
|
|
122
|
+
subject_ref=subref,
|
|
123
|
+
purge_id=purge_id,
|
|
124
|
+
status="complete",
|
|
125
|
+
deleted_counts=self._deleted_counts_from_targets(purge_id),
|
|
126
|
+
rebuilt_agent_playbook_ids=(
|
|
127
|
+
self._rebuilt_agent_playbook_ids_from_targets(purge_id)
|
|
128
|
+
),
|
|
129
|
+
)
|
|
130
|
+
try:
|
|
131
|
+
self.storage.begin_subject_erasure_barrier(subref, purge_id)
|
|
132
|
+
if not self.storage.purge_targets_prepared(purge_id):
|
|
133
|
+
self.storage.prepare_governance_erase_targets(
|
|
134
|
+
purge_id,
|
|
135
|
+
user_id,
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
if not self._delete_targets_complete(purge_id):
|
|
139
|
+
self.storage.apply_governance_user_data_delete(purge_id, user_id)
|
|
140
|
+
deleted_counts = self._deleted_counts_from_targets(purge_id)
|
|
141
|
+
|
|
142
|
+
rebuilt_agent_playbook_ids: list[int] = []
|
|
143
|
+
completed = self.storage.complete_subject_erasure_barrier_after_empty_check(
|
|
144
|
+
purge_id,
|
|
145
|
+
AuditEvent(
|
|
146
|
+
org_id=self.org_id,
|
|
147
|
+
actor_type=actor_type,
|
|
148
|
+
actor_ref=actor_ref,
|
|
149
|
+
operation="ERASE",
|
|
150
|
+
entity_type="request",
|
|
151
|
+
subject_ref=subref,
|
|
152
|
+
request_ref=reqref,
|
|
153
|
+
idempotency_key=purge_id,
|
|
154
|
+
detail={
|
|
155
|
+
"deleted_counts": deleted_counts,
|
|
156
|
+
"rebuilt_agent_playbook_ids": rebuilt_agent_playbook_ids,
|
|
157
|
+
},
|
|
158
|
+
),
|
|
159
|
+
)
|
|
160
|
+
except Exception as exc:
|
|
161
|
+
with suppress(Exception):
|
|
162
|
+
self.storage.fail_subject_erasure_barrier(
|
|
163
|
+
subref,
|
|
164
|
+
purge_id,
|
|
165
|
+
error_code="governance_erase_failed",
|
|
166
|
+
error_detail=type(exc).__name__,
|
|
167
|
+
)
|
|
168
|
+
with suppress(Exception):
|
|
169
|
+
self.storage.fail_purge_operation(
|
|
170
|
+
purge_id,
|
|
171
|
+
error_code="governance_erase_failed",
|
|
172
|
+
error_detail=type(exc).__name__,
|
|
173
|
+
)
|
|
174
|
+
raise
|
|
175
|
+
return UserEraseResult(
|
|
176
|
+
subject_ref=subref,
|
|
177
|
+
purge_id=purge_id,
|
|
178
|
+
status=completed.status,
|
|
179
|
+
deleted_counts=deleted_counts,
|
|
180
|
+
rebuilt_agent_playbook_ids=rebuilt_agent_playbook_ids,
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
def _assert_storage_ref_secret_matches(self) -> None:
|
|
184
|
+
storage_secret = get_governance_ref_secret()
|
|
185
|
+
if storage_secret != self.ref_secret:
|
|
186
|
+
raise RuntimeError(
|
|
187
|
+
"GovernanceService ref_secret must match REFLEXIO_GOVERNANCE_REF_SECRET "
|
|
188
|
+
"for governance operations"
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
def _completed_barrier_for_retry(
|
|
192
|
+
self, *, subject_ref: str, purge_id: str
|
|
193
|
+
) -> SubjectWriteBarrier:
|
|
194
|
+
barrier = self.storage.get_subject_write_barrier(subject_ref)
|
|
195
|
+
if barrier is None or barrier.purge_id != purge_id:
|
|
196
|
+
raise ValueError(
|
|
197
|
+
"Completed purge retry requires the matching subject barrier"
|
|
198
|
+
)
|
|
199
|
+
return barrier
|
|
200
|
+
|
|
201
|
+
def _load_user_requests_and_sessions(
|
|
202
|
+
self, user_id: str
|
|
203
|
+
) -> tuple[list[Any], list[dict[str, Any]]]:
|
|
204
|
+
requests: list[Any] = []
|
|
205
|
+
sessions_by_id: dict[str, list[str]] = {}
|
|
206
|
+
offset = 0
|
|
207
|
+
page_size = 1000
|
|
208
|
+
|
|
209
|
+
while True:
|
|
210
|
+
grouped_sessions = self.storage.get_sessions(
|
|
211
|
+
user_id=user_id,
|
|
212
|
+
top_k=page_size,
|
|
213
|
+
offset=offset,
|
|
214
|
+
)
|
|
215
|
+
returned_rows = 0
|
|
216
|
+
for session_id, rows in grouped_sessions.items():
|
|
217
|
+
returned_rows += len(rows)
|
|
218
|
+
request_ids = sessions_by_id.setdefault(session_id, [])
|
|
219
|
+
for row in rows:
|
|
220
|
+
if row.request is None:
|
|
221
|
+
continue
|
|
222
|
+
requests.append(row.request)
|
|
223
|
+
request_ids.append(row.request.request_id)
|
|
224
|
+
if returned_rows < page_size:
|
|
225
|
+
break
|
|
226
|
+
offset += page_size
|
|
227
|
+
|
|
228
|
+
sessions = [
|
|
229
|
+
{"session_id": session_id, "request_ids": request_ids}
|
|
230
|
+
for session_id, request_ids in sessions_by_id.items()
|
|
231
|
+
]
|
|
232
|
+
return requests, sessions
|
|
233
|
+
|
|
234
|
+
def _iter_user_playbooks(self, user_id: str) -> list[Any]:
|
|
235
|
+
playbooks: list[Any] = []
|
|
236
|
+
offset = 0
|
|
237
|
+
while True:
|
|
238
|
+
page = self.storage.get_user_playbooks(
|
|
239
|
+
user_id=user_id,
|
|
240
|
+
limit=_USER_PLAYBOOK_PAGE_SIZE,
|
|
241
|
+
offset=offset,
|
|
242
|
+
)
|
|
243
|
+
playbooks.extend(page)
|
|
244
|
+
if len(page) < _USER_PLAYBOOK_PAGE_SIZE:
|
|
245
|
+
break
|
|
246
|
+
offset += _USER_PLAYBOOK_PAGE_SIZE
|
|
247
|
+
return playbooks
|
|
248
|
+
|
|
249
|
+
def _delete_targets_complete(self, purge_id: str) -> bool:
|
|
250
|
+
delete_targets = {
|
|
251
|
+
target.target_name: target
|
|
252
|
+
for target in self.storage.list_purge_targets(purge_id, phase="delete")
|
|
253
|
+
}
|
|
254
|
+
return all(
|
|
255
|
+
delete_targets.get(target_name) is not None
|
|
256
|
+
and delete_targets[target_name].status == "complete"
|
|
257
|
+
for target_name in _REQUIRED_DELETE_TARGET_NAMES
|
|
258
|
+
)
|
|
259
|
+
|
|
260
|
+
def _deleted_counts_from_targets(self, purge_id: str) -> dict[str, int]:
|
|
261
|
+
counts: dict[str, int] = {}
|
|
262
|
+
for target in self.storage.list_purge_targets(purge_id, phase="delete"):
|
|
263
|
+
result_key = _DELETE_TARGET_NAME_TO_RESULT_KEY.get(target.target_name)
|
|
264
|
+
if result_key is None:
|
|
265
|
+
continue
|
|
266
|
+
counts[result_key] = int(target.deleted_count)
|
|
267
|
+
return counts
|
|
268
|
+
|
|
269
|
+
def _rebuilt_agent_playbook_ids_from_targets(self, purge_id: str) -> list[int]:
|
|
270
|
+
return [
|
|
271
|
+
int(target.target_ref)
|
|
272
|
+
for target in self.storage.list_purge_targets(
|
|
273
|
+
purge_id,
|
|
274
|
+
phase="rebuild_without_erased_sources",
|
|
275
|
+
)
|
|
276
|
+
if (
|
|
277
|
+
target.target_name == "agent_playbook"
|
|
278
|
+
and target.target_ref
|
|
279
|
+
and target.status == "complete"
|
|
280
|
+
)
|
|
281
|
+
]
|
|
282
|
+
|
|
283
|
+
def _rebuild_agent_playbooks(self, purge_id: str) -> list[int]:
|
|
284
|
+
rebuilt_ids: list[int] = []
|
|
285
|
+
for target in self.storage.list_purge_targets(
|
|
286
|
+
purge_id,
|
|
287
|
+
phase="rebuild_without_erased_sources",
|
|
288
|
+
):
|
|
289
|
+
if target.target_name != "agent_playbook" or not target.target_ref:
|
|
290
|
+
continue
|
|
291
|
+
agent_playbook_id = int(target.target_ref)
|
|
292
|
+
if target.status == "complete":
|
|
293
|
+
rebuilt_ids.append(agent_playbook_id)
|
|
294
|
+
continue
|
|
295
|
+
remaining_source_windows = self._remaining_source_windows(target)
|
|
296
|
+
rebuild_fields = self._build_rebuilt_agent_playbook_fields(
|
|
297
|
+
remaining_source_windows
|
|
298
|
+
)
|
|
299
|
+
self.storage.apply_governance_agent_playbook_rebuild(
|
|
300
|
+
purge_id=purge_id,
|
|
301
|
+
agent_playbook_id=agent_playbook_id,
|
|
302
|
+
remaining_source_windows=remaining_source_windows,
|
|
303
|
+
content=rebuild_fields["content"],
|
|
304
|
+
trigger=rebuild_fields["trigger"],
|
|
305
|
+
rationale=rebuild_fields["rationale"],
|
|
306
|
+
blocking_issue=rebuild_fields["blocking_issue"],
|
|
307
|
+
expanded_terms=rebuild_fields["expanded_terms"],
|
|
308
|
+
tags=rebuild_fields["tags"],
|
|
309
|
+
)
|
|
310
|
+
rebuilt_ids.append(agent_playbook_id)
|
|
311
|
+
return rebuilt_ids
|
|
312
|
+
|
|
313
|
+
def _remaining_source_windows(
|
|
314
|
+
self,
|
|
315
|
+
target: PurgeOperationTarget,
|
|
316
|
+
) -> list[dict[str, object]]:
|
|
317
|
+
detail = target.detail or {}
|
|
318
|
+
remaining = detail.get("remaining_source_windows", [])
|
|
319
|
+
if not isinstance(remaining, list):
|
|
320
|
+
raise ValueError("remaining_source_windows must be a list")
|
|
321
|
+
return remaining
|
|
322
|
+
|
|
323
|
+
def _build_rebuilt_agent_playbook_fields(
|
|
324
|
+
self,
|
|
325
|
+
remaining_source_windows: list[dict[str, object]],
|
|
326
|
+
) -> dict[str, Any]:
|
|
327
|
+
user_playbook_ids: list[int] = []
|
|
328
|
+
for window in remaining_source_windows:
|
|
329
|
+
raw_user_playbook_id = window.get("user_playbook_id")
|
|
330
|
+
if isinstance(raw_user_playbook_id, int):
|
|
331
|
+
user_playbook_ids.append(raw_user_playbook_id)
|
|
332
|
+
playbooks_by_id = {
|
|
333
|
+
playbook.user_playbook_id: playbook
|
|
334
|
+
for playbook in self.storage.get_user_playbooks_by_ids_any_user(
|
|
335
|
+
user_playbook_ids
|
|
336
|
+
)
|
|
337
|
+
if playbook.user_playbook_id
|
|
338
|
+
}
|
|
339
|
+
remaining_playbooks = [
|
|
340
|
+
playbooks_by_id[user_playbook_id]
|
|
341
|
+
for user_playbook_id in user_playbook_ids
|
|
342
|
+
if user_playbook_id in playbooks_by_id
|
|
343
|
+
]
|
|
344
|
+
return {
|
|
345
|
+
"content": self._join_non_empty_strings(
|
|
346
|
+
playbook.content for playbook in remaining_playbooks
|
|
347
|
+
),
|
|
348
|
+
"trigger": self._join_non_empty_strings(
|
|
349
|
+
playbook.trigger for playbook in remaining_playbooks
|
|
350
|
+
),
|
|
351
|
+
"rationale": self._join_non_empty_strings(
|
|
352
|
+
playbook.rationale for playbook in remaining_playbooks
|
|
353
|
+
),
|
|
354
|
+
"blocking_issue": next(
|
|
355
|
+
(
|
|
356
|
+
playbook.blocking_issue.model_dump()
|
|
357
|
+
for playbook in remaining_playbooks
|
|
358
|
+
if playbook.blocking_issue is not None
|
|
359
|
+
),
|
|
360
|
+
None,
|
|
361
|
+
),
|
|
362
|
+
"expanded_terms": self._join_non_empty_strings(
|
|
363
|
+
playbook.expanded_terms for playbook in remaining_playbooks
|
|
364
|
+
),
|
|
365
|
+
"tags": self._merge_tags(remaining_playbooks),
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
def _join_non_empty_strings(self, values: Any) -> str | None:
|
|
369
|
+
joined = "\n".join(value for value in values if value)
|
|
370
|
+
return joined or None
|
|
371
|
+
|
|
372
|
+
def _merge_tags(self, playbooks: list[Any]) -> list[str] | None:
|
|
373
|
+
merged_tags: list[str] = []
|
|
374
|
+
for playbook in playbooks:
|
|
375
|
+
for tag in playbook.tags or []:
|
|
376
|
+
if tag not in merged_tags:
|
|
377
|
+
merged_tags.append(tag)
|
|
378
|
+
return merged_tags or None
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import hashlib
|
|
4
|
+
import hmac
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def _hmac_ref(prefix: str, raw: str, secret: str) -> str:
|
|
8
|
+
if not raw:
|
|
9
|
+
raise ValueError("raw value must be non-empty")
|
|
10
|
+
if not secret:
|
|
11
|
+
raise ValueError("secret must be non-empty")
|
|
12
|
+
digest = hmac.new(secret.encode(), raw.encode(), hashlib.sha256).hexdigest()[:32]
|
|
13
|
+
return f"{prefix}_v1_{digest}"
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def subject_ref(raw: str, secret: str) -> str:
|
|
17
|
+
return _hmac_ref("subref", raw, secret)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def actor_ref(raw: str, secret: str) -> str:
|
|
21
|
+
return _hmac_ref("actref", raw, secret)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def request_ref(raw: str, secret: str) -> str:
|
|
25
|
+
return _hmac_ref("reqref", raw, secret)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def stable_id(prefix: str, material: str) -> str:
|
|
29
|
+
if not prefix:
|
|
30
|
+
raise ValueError("prefix must be non-empty")
|
|
31
|
+
if not material:
|
|
32
|
+
raise ValueError("material must be non-empty")
|
|
33
|
+
digest = hashlib.sha256(material.encode()).hexdigest()[:32]
|
|
34
|
+
return f"{prefix}_{digest}"
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
"""Process-local scheduler for lineage tombstone garbage collection.
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
3
|
+
Startup is gated on ``lineage_gc.enabled`` (default on). Each tick evaluates
|
|
4
|
+
every org independently and hard-deletes expired tombstones per that org's
|
|
5
|
+
``lineage_gc`` config. One org's failure never stalls the loop; errors are
|
|
6
|
+
captured as Sentry anomalies and the loop continues to the next org.
|
|
7
|
+
|
|
8
|
+
Governance-retention reclamation is a premium concern handled by the enterprise
|
|
9
|
+
GovernanceRetentionCapability (reflexio_ext), not here.
|
|
8
10
|
"""
|
|
9
11
|
|
|
10
12
|
from __future__ import annotations
|
|
@@ -30,7 +32,7 @@ _ENTITY_TYPES = ("user_playbook", "agent_playbook", "profile")
|
|
|
30
32
|
|
|
31
33
|
|
|
32
34
|
class LineageGCScheduler:
|
|
33
|
-
"""Polling daemon that
|
|
35
|
+
"""Polling daemon that hard-deletes expired tombstones per org."""
|
|
34
36
|
|
|
35
37
|
def __init__(
|
|
36
38
|
self,
|
|
@@ -96,33 +98,32 @@ class LineageGCScheduler:
|
|
|
96
98
|
break
|
|
97
99
|
try:
|
|
98
100
|
ctx = self.request_context_factory(org_id)
|
|
101
|
+
if ctx.storage is None:
|
|
102
|
+
continue
|
|
99
103
|
cfg = ctx.configurator.get_config()
|
|
100
104
|
if not cfg.lineage_gc.enabled:
|
|
101
105
|
continue
|
|
102
|
-
if ctx.storage is None:
|
|
103
|
-
continue
|
|
104
106
|
older_than_epoch = (
|
|
105
107
|
int(time.time())
|
|
106
108
|
- cfg.lineage_gc.tombstone_grace_window_days * 86400
|
|
107
109
|
)
|
|
108
|
-
|
|
110
|
+
tombstone_deleted = 0
|
|
109
111
|
for entity_type in _ENTITY_TYPES:
|
|
110
|
-
|
|
112
|
+
tombstone_deleted += ctx.storage.gc_expired_tombstones(
|
|
111
113
|
entity_type=entity_type,
|
|
112
114
|
older_than_epoch=older_than_epoch,
|
|
113
115
|
)
|
|
114
|
-
|
|
115
|
-
if total_deleted:
|
|
116
|
+
if tombstone_deleted:
|
|
116
117
|
logger.info(
|
|
117
|
-
"event=lineage_gc_tick org_id=%s
|
|
118
|
+
"event=lineage_gc_tick org_id=%s tombstone_deleted=%d",
|
|
118
119
|
org_id,
|
|
119
|
-
|
|
120
|
+
tombstone_deleted,
|
|
120
121
|
)
|
|
121
|
-
if
|
|
122
|
+
if tombstone_deleted > _HIGH_VOLUME_THRESHOLD:
|
|
122
123
|
capture_anomaly(
|
|
123
124
|
"lineage.gc.high_volume",
|
|
124
125
|
org_id=org_id,
|
|
125
|
-
count=
|
|
126
|
+
count=tombstone_deleted,
|
|
126
127
|
)
|
|
127
128
|
except Exception:
|
|
128
129
|
capture_anomaly("lineage.gc.run_failed", org_id=org_id)
|
|
@@ -147,10 +148,11 @@ def maybe_start_lineage_gc(
|
|
|
147
148
|
*,
|
|
148
149
|
bootstrap_org_id: str,
|
|
149
150
|
) -> LineageGCScheduler | None:
|
|
150
|
-
"""Start the
|
|
151
|
+
"""Start the scheduler only when bootstrap config enables tombstone GC.
|
|
151
152
|
|
|
152
|
-
|
|
153
|
-
|
|
153
|
+
Startup requires bootstrap-org config to enable tombstone GC via
|
|
154
|
+
``lineage_gc.enabled``. Tombstone-GC enablement criteria (must
|
|
155
|
+
ALL hold before enabling for a production org):
|
|
154
156
|
|
|
155
157
|
1. **Mechanism**: GC ages tombstones by ``retired_at`` (the INTEGER epoch
|
|
156
158
|
written at every tombstone write-path). Rows with ``retired_at = NULL``
|
|
@@ -175,6 +177,30 @@ def maybe_start_lineage_gc(
|
|
|
175
177
|
try:
|
|
176
178
|
ctx = request_context_factory(bootstrap_org_id)
|
|
177
179
|
cfg = ctx.configurator.get_config()
|
|
180
|
+
|
|
181
|
+
# Dead-knob warning: audit-event retention is an ENTERPRISE-only feature
|
|
182
|
+
# (handled by reflexio_ext GovernanceRetentionCapability). In an OSS-only
|
|
183
|
+
# deployment the knob is accepted but does nothing — warn loudly. Detected
|
|
184
|
+
# via the configurator class: enterprise swaps in EnterpriseConfigurator at
|
|
185
|
+
# construction, so the OSS DefaultConfigurator means "no enterprise here".
|
|
186
|
+
from reflexio.server.services.configurator.configurator import ( # noqa: PLC0415
|
|
187
|
+
DefaultConfigurator,
|
|
188
|
+
get_configurator_class,
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
gr = getattr(cfg, "governance_retention", None)
|
|
192
|
+
if (
|
|
193
|
+
gr is not None
|
|
194
|
+
and getattr(gr, "audit_events_retention_enabled", False)
|
|
195
|
+
and get_configurator_class() is DefaultConfigurator
|
|
196
|
+
):
|
|
197
|
+
logger.warning(
|
|
198
|
+
"event=governance_retention_knob_ignored "
|
|
199
|
+
"audit_events_retention_enabled=True but this is an OSS-only "
|
|
200
|
+
"deployment — audit-event retention is an enterprise-only feature "
|
|
201
|
+
"and will NOT run here."
|
|
202
|
+
)
|
|
203
|
+
|
|
178
204
|
if not cfg.lineage_gc.enabled:
|
|
179
205
|
return None
|
|
180
206
|
except Exception as exc:
|
|
@@ -16,7 +16,7 @@ Description: Playbook extraction, aggregation, and consolidation pipeline
|
|
|
16
16
|
|------|---------|
|
|
17
17
|
| `playbook_service_constants.py` | Prompt IDs for all playbook operations |
|
|
18
18
|
| `playbook_service_utils.py` | Request dataclasses, Pydantic output schemas, message construction utilities |
|
|
19
|
-
| `
|
|
19
|
+
| `aggregation_prompt_processing.py` | Optional aggregation-boundary interfaces and helpers for prompt preprocessing, contextual prompt guidance, and output post-processing |
|
|
20
20
|
|
|
21
21
|
## Architecture
|
|
22
22
|
|
|
@@ -49,6 +49,14 @@ Triggered manually via `/api/run_playbook_aggregation`. Clusters user playbooks
|
|
|
49
49
|
**Key Methods**:
|
|
50
50
|
- `get_clusters(user_playbooks, config)` - HDBSCAN/Agglomerative clustering on embeddings
|
|
51
51
|
- `aggregate()` - Full aggregation pipeline with LLM-based consolidation
|
|
52
|
+
|
|
53
|
+
**Optional prompt processing**: deployments can register an
|
|
54
|
+
`AggregationPromptProcessor` via the `AGGREGATION_PROMPT_PROCESSOR` ServiceKey
|
|
55
|
+
(`register_service`). The aggregator applies the processor only at the
|
|
56
|
+
aggregation prompt boundary, carries an opaque per-cluster processing context,
|
|
57
|
+
injects extra prompt guidance only when preprocessing changed prompt input, and
|
|
58
|
+
post-processes generated outputs before storage or model-response logging.
|
|
59
|
+
|
|
52
60
|
**Change Log**: The legacy `playbook_aggregation_change_logs` table is retired (Track B, 2026-06-24) — the aggregator no longer writes it. The change-log view is reconstructed on demand from `lineage_event` via `reconstruct_playbook_aggregation_change_log` (`lib/_agent_playbook.py`): each run emits `op=aggregate` events (the "added" side) and `status_change→superseded` events from the supersede calls (the "removed" side), grouped by the run's `request_id`. Per-row `updated` pairing is not reconstructed (`updated_agent_playbooks=[]`, a tolerated parity delta).
|
|
53
61
|
|
|
54
62
|
**Clustering**: Embeds user playbooks -> HDBSCAN clustering -> falls back to Agglomerative if too few clusters
|