claude-smart 0.2.45 → 0.2.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +3 -3
- package/README.md +47 -10
- package/bin/claude-smart.js +535 -23
- package/package.json +22 -3
- package/plugin/.claude-plugin/plugin.json +4 -2
- package/plugin/.codex-plugin/plugin.json +1 -1
- package/plugin/README.md +27 -6
- package/plugin/commands/clear-all.md +1 -0
- package/plugin/commands/dashboard.md +1 -0
- package/plugin/commands/learn.md +1 -0
- package/plugin/commands/restart.md +1 -0
- package/plugin/commands/show.md +1 -0
- package/plugin/dashboard/app/configure/env/page.tsx +40 -14
- package/plugin/dashboard/app/configure/server/page.tsx +51 -1
- package/plugin/dashboard/app/layout.tsx +20 -0
- package/plugin/dashboard/app/preferences/[id]/page.tsx +8 -8
- package/plugin/dashboard/app/skills/project/[id]/page.tsx +7 -3
- package/plugin/dashboard/app/skills/shared/[id]/page.tsx +10 -3
- package/plugin/dashboard/lib/claude-settings-file.ts +20 -10
- package/plugin/dashboard/lib/reflexio-client.ts +16 -0
- package/plugin/dashboard/lib/status.ts +10 -3
- package/plugin/dashboard/lib/types.ts +18 -6
- package/plugin/hooks/codex-hooks.json +0 -1
- package/plugin/opencode/assistant-buffer.ts +108 -0
- package/plugin/opencode/dist/assistant-buffer.js +96 -0
- package/plugin/opencode/dist/internal.js +12 -0
- package/plugin/opencode/dist/payload.js +85 -0
- package/plugin/opencode/dist/server.mjs +232 -0
- package/plugin/opencode/internal.ts +18 -0
- package/plugin/opencode/package.json +3 -0
- package/plugin/opencode/payload.ts +90 -0
- package/plugin/opencode/server.mts +251 -0
- package/plugin/opencode/tsconfig.json +13 -0
- package/plugin/pyproject.toml +25 -4
- package/plugin/scripts/_lib.sh +5 -1
- package/plugin/scripts/backend-service.sh +22 -2
- package/plugin/scripts/ensure-plugin-root.sh +4 -4
- package/plugin/scripts/hook_entry.sh +1 -1
- package/plugin/scripts/opencode-claude-compat +4 -0
- package/plugin/scripts/opencode-claude-compat.cmd +3 -0
- package/plugin/scripts/opencode-claude-compat.js +225 -0
- package/plugin/scripts/smart-install.sh +17 -9
- package/plugin/src/README.md +1 -1
- package/plugin/src/claude_smart/cli.py +498 -12
- package/plugin/src/claude_smart/context_format.py +9 -9
- package/plugin/src/claude_smart/cs_cite.py +66 -19
- package/plugin/src/claude_smart/env_config.py +12 -4
- package/plugin/src/claude_smart/events/session_start.py +26 -7
- package/plugin/src/claude_smart/events/stop.py +1 -1
- package/plugin/src/claude_smart/ids.py +1 -1
- package/plugin/src/claude_smart/reflexio_adapter.py +5 -5
- package/plugin/src/claude_smart/runtime.py +7 -1
- package/plugin/uv.lock +5 -5
- package/plugin/vendor/reflexio/.env.example +54 -30
- package/plugin/vendor/reflexio/README.md +17 -11
- package/plugin/vendor/reflexio/pyproject.toml +13 -1
- package/plugin/vendor/reflexio/reflexio/README.md +4 -1
- package/plugin/vendor/reflexio/reflexio/cli/README.md +9 -7
- package/plugin/vendor/reflexio/reflexio/cli/__main__.py +10 -1
- package/plugin/vendor/reflexio/reflexio/cli/bootstrap_config.py +22 -5
- package/plugin/vendor/reflexio/reflexio/cli/commands/interactions.py +34 -4
- package/plugin/vendor/reflexio/reflexio/cli/commands/profiles.py +12 -7
- package/plugin/vendor/reflexio/reflexio/cli/commands/services.py +13 -6
- package/plugin/vendor/reflexio/reflexio/cli/commands/setup_cmd.py +212 -52
- package/plugin/vendor/reflexio/reflexio/cli/commands/shortcuts.py +15 -2
- package/plugin/vendor/reflexio/reflexio/cli/env_loader.py +292 -22
- package/plugin/vendor/reflexio/reflexio/cli/log_format.py +29 -4
- package/plugin/vendor/reflexio/reflexio/cli/run_services.py +17 -8
- package/plugin/vendor/reflexio/reflexio/cli/stop_services.py +13 -10
- package/plugin/vendor/reflexio/reflexio/cli/utils.py +44 -1
- package/plugin/vendor/reflexio/reflexio/client/client.py +154 -13
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/README.md +135 -257
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/npm/openclaw-smart/bin/openclaw-smart.js +13 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/npm/openclaw-smart/package.json +15 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/README.md +38 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/index.ts +151 -110
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/openclaw.plugin.json +20 -10
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/package.json +28 -6
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/pyproject.toml +42 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/scripts/_lib.sh +371 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/scripts/backend-log-runner.sh +33 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/scripts/backend-service.sh +271 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/scripts/cli.sh +68 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/scripts/dashboard-open.sh +15 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/scripts/ensure-plugin-root.sh +84 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/scripts/hook_entry.sh +106 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/scripts/npm-cli.js +219 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/scripts/smart-install.sh +269 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/skills/clear-all/SKILL.md +8 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/skills/dashboard/SKILL.md +8 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/skills/learn/SKILL.md +10 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/skills/reflexio/SKILL.md +15 -44
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/skills/restart/SKILL.md +6 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/skills/show/SKILL.md +8 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/cli.py +634 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/context_format.py +224 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/context_inject.py +79 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/events/__init__.py +0 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/events/after_tool_call.py +160 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/events/agent_end.py +187 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/events/before_prompt_build.py +69 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/events/before_tool_call.py +30 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/events/session_end.py +36 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/events/session_start.py +130 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/hook.py +131 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/ids.py +94 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/internal_call.py +75 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/oc_cite.py +196 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/optimizer_assistant.py +272 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/publish.py +96 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/query_compose.py +66 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/reflexio_adapter.py +336 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/runtime.py +47 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/stall_banner.py +61 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/state.py +323 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/__init__.py +0 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/integration/__init__.py +0 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/integration/test_e2e_session_loop.py +190 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/integration/test_publish_to_local_reflexio_integration.py +112 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/integration/test_recursion_guard_integration.py +86 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/integration/test_search_inject_integration.py +144 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_cli.py +184 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_events_after_tool_call.py +142 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_events_agent_end.py +233 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_events_before_prompt_build.py +116 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_events_before_tool_call.py +35 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_events_session_end.py +47 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_events_session_start.py +109 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_hook_dispatch.py +117 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_ids.py +41 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_internal_call.py +50 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_oc_cite.py +88 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_publish.py +48 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_query_compose.py +48 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_reflexio_adapter.py +188 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_runtime.py +41 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_state.py +235 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests-ts/__mocks__/plugin-entry-stub.ts +6 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests-ts/test_npm_cli.test.ts +56 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests-ts/test_shim_dispatch.test.ts +171 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tsconfig.build.json +18 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/{tsconfig.json → plugin/tsconfig.json} +2 -2
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/uv.lock +3835 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/vitest.config.ts +18 -0
- package/plugin/vendor/reflexio/reflexio/lib/_agent_playbook.py +246 -10
- package/plugin/vendor/reflexio/reflexio/lib/_base.py +15 -0
- package/plugin/vendor/reflexio/reflexio/lib/_config.py +11 -4
- package/plugin/vendor/reflexio/reflexio/lib/_generation.py +17 -5
- package/plugin/vendor/reflexio/reflexio/lib/_lineage_parity_readers.py +187 -0
- package/plugin/vendor/reflexio/reflexio/lib/_profiles.py +224 -21
- package/plugin/vendor/reflexio/reflexio/lib/_reflection.py +1 -1
- package/plugin/vendor/reflexio/reflexio/lib/_search.py +52 -25
- package/plugin/vendor/reflexio/reflexio/lib/_user_playbook.py +11 -1
- package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/__init__.py +1 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/entities.py +127 -38
- package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/enums.py +4 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/governance.py +117 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/eval_overview_schema.py +64 -41
- package/plugin/vendor/reflexio/reflexio/models/api_schema/internal_schema.py +19 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/pending_tool_call_schema.py +12 -5
- package/plugin/vendor/reflexio/reflexio/models/api_schema/retriever_schema.py +97 -3
- package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/converters.py +5 -1
- package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/entities.py +5 -1
- package/plugin/vendor/reflexio/reflexio/models/config_schema.py +104 -7
- package/plugin/vendor/reflexio/reflexio/models/structured_output.py +148 -0
- package/plugin/vendor/reflexio/reflexio/server/OVERVIEW.md +3 -3
- package/plugin/vendor/reflexio/reflexio/server/README.md +58 -33
- package/plugin/vendor/reflexio/reflexio/server/__init__.py +29 -5
- package/plugin/vendor/reflexio/reflexio/server/__main__.py +2 -2
- package/plugin/vendor/reflexio/reflexio/server/api.py +899 -103
- package/plugin/vendor/reflexio/reflexio/server/api_endpoints/precondition_checks.py +7 -0
- package/plugin/vendor/reflexio/reflexio/server/api_endpoints/request_context.py +4 -2
- package/plugin/vendor/reflexio/reflexio/server/auth.py +90 -0
- package/plugin/vendor/reflexio/reflexio/server/billing_meter.py +173 -0
- package/plugin/vendor/reflexio/reflexio/server/billing_signals.py +64 -0
- package/plugin/vendor/reflexio/reflexio/server/cache/reflexio_cache.py +46 -16
- package/plugin/vendor/reflexio/reflexio/server/env_utils.py +65 -0
- package/plugin/vendor/reflexio/reflexio/server/extensions.py +213 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/embedding_service.py +19 -4
- package/plugin/vendor/reflexio/reflexio/server/llm/litellm_client.py +264 -77
- package/plugin/vendor/reflexio/reflexio/server/llm/llm_utils.py +69 -7
- package/plugin/vendor/reflexio/reflexio/server/llm/model_defaults.py +4 -4
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/claude_code_provider.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/embedding_service_provider.py +75 -16
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/openclaw_provider.py +280 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/rerank/cross_encoder_reranker.py +12 -1
- package/plugin/vendor/reflexio/reflexio/server/llm/token_accounting.py +48 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/tools.py +107 -17
- package/plugin/vendor/reflexio/reflexio/server/operation_limiter.py +360 -4
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/memory_reflection/v1.6.0.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/memory_reflection/v1.7.0.prompt.md +85 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_aggregation/v2.2.0.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_aggregation/v2.3.0.prompt.md +253 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.3.2.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.3.3.prompt.md +74 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_context/v4.2.3.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_context/v4.3.0.prompt.md +266 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_context/v4.4.0.prompt.md +278 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_context_expert/v3.3.0.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_context_expert/v3.4.0.prompt.md +137 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_main/v1.2.0.prompt.md +1 -1
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_main/v1.3.0.prompt.md +34 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/profile_update_instruction_start/v1.0.0.prompt.md +5 -12
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/profile_update_instruction_start/v1.1.0.prompt.md +5 -12
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/profile_update_instruction_start/v1.2.0.prompt.md +155 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/tagging/v1.0.0.prompt.md +23 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_manager.py +89 -32
- package/plugin/vendor/reflexio/reflexio/server/services/README.md +28 -11
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/README.md +22 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/__init__.py +1 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/agent_success_evaluation_constants.py +4 -2
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/agent_success_evaluation_utils.py +1 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/components/__init__.py +7 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/{agent_success_evaluator.py → components/evaluator.py} +3 -16
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/regen_jobs.py +28 -72
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/{group_evaluation_runner.py → runner.py} +12 -10
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/{delayed_group_evaluator.py → scheduler.py} +10 -2
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/{agent_success_evaluation_service.py → service.py} +36 -1
- package/plugin/vendor/reflexio/reflexio/server/services/base_generation_service.py +160 -0
- package/plugin/vendor/reflexio/reflexio/server/services/configurator/base_configurator.py +5 -0
- package/plugin/vendor/reflexio/reflexio/server/services/deduplication_utils.py +1 -1
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/README.md +9 -0
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/components/__init__.py +0 -0
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/{rule_attribution.py → components/rule_attribution.py} +10 -9
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/eval_sampler.py +15 -24
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/service.py +238 -164
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/README.md +31 -0
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/outcome.py +13 -3
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/pending_tool_call_dispatch.py +3 -2
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/resumable_agent.py +32 -118
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_worker.py +47 -16
- package/plugin/vendor/reflexio/reflexio/server/services/generation_service.py +105 -23
- package/plugin/vendor/reflexio/reflexio/server/services/governance/config.py +52 -0
- package/plugin/vendor/reflexio/reflexio/server/services/governance/service.py +378 -0
- package/plugin/vendor/reflexio/reflexio/server/services/governance/subject_refs.py +34 -0
- package/plugin/vendor/reflexio/reflexio/server/services/lineage/__init__.py +0 -0
- package/plugin/vendor/reflexio/reflexio/server/services/lineage/gc_scheduler.py +219 -0
- package/plugin/vendor/reflexio/reflexio/server/services/lineage/resolve.py +98 -0
- package/plugin/vendor/reflexio/reflexio/server/services/operation_state_utils.py +26 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/README.md +18 -11
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/aggregation_prompt_processing.py +100 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/__init__.py +5 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/{playbook_aggregator.py → components/aggregator.py} +334 -178
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/{playbook_consolidator.py → components/consolidator.py} +168 -62
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/{playbook_extractor.py → components/extractor.py} +5 -1
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/playbook_edit_apply.py +74 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/playbook_service_utils.py +33 -3
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/{playbook_generation_service.py → service.py} +121 -19
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/README.md +10 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/models.py +2 -1
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/optimizer.py +178 -30
- package/plugin/vendor/reflexio/reflexio/server/services/pre_retrieval/README.md +9 -0
- package/plugin/vendor/reflexio/reflexio/server/services/profile/components/__init__.py +17 -0
- package/plugin/vendor/reflexio/reflexio/server/services/profile/{profile_deduplicator.py → components/consolidator.py} +7 -6
- package/plugin/vendor/reflexio/reflexio/server/services/profile/{profile_extractor.py → components/extractor.py} +31 -22
- package/plugin/vendor/reflexio/reflexio/server/services/profile/profile_generation_service_utils.py +6 -13
- package/plugin/vendor/reflexio/reflexio/server/services/profile/{profile_generation_service.py → service.py} +32 -50
- package/plugin/vendor/reflexio/reflexio/server/services/reflection/__init__.py +4 -8
- package/plugin/vendor/reflexio/reflexio/server/services/reflection/components/__init__.py +7 -0
- package/plugin/vendor/reflexio/reflexio/server/services/reflection/reflection_service_utils.py +12 -1
- package/plugin/vendor/reflexio/reflexio/server/services/reflection/{reflection_service.py → service.py} +36 -16
- package/plugin/vendor/reflexio/reflexio/server/services/retrieval/recency.py +211 -0
- package/plugin/vendor/reflexio/reflexio/server/services/retrieval/relevance_floor.py +99 -6
- package/plugin/vendor/reflexio/reflexio/server/services/service_utils.py +36 -1
- package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/README.md +8 -0
- package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/outcome.py +1 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/error.py +31 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/governance_validation.py +681 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/retention.py +9 -9
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/__init__.py +16 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_agent_run.py +4 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_base.py +644 -197
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_extras.py +106 -119
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_governance.py +1965 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_lineage.py +630 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_operations.py +38 -6
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_playbook.py +31 -1369
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_profiles.py +756 -210
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_requests.py +148 -34
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_shadow_verdicts.py +23 -9
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_stall_state.py +5 -2
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/__init__.py +13 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_agent.py +952 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_eval_results.py +189 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_optimization.py +247 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_source_linkage.py +145 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_user.py +838 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/__init__.py +150 -23
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_agent_run.py +2 -8
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_base.py +5 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_extras.py +40 -57
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_governance.py +148 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_lineage.py +215 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_operations.py +20 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_playbook.py +4 -677
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_profiles.py +134 -1
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_requests.py +62 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_retrieval_log.py +51 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_shadow_verdicts.py +17 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/__init__.py +13 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_agent.py +365 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_eval_results.py +124 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_optimization.py +85 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_source_linkage.py +47 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_user.py +333 -0
- package/plugin/vendor/reflexio/reflexio/server/services/tagging/README.md +8 -0
- package/plugin/vendor/reflexio/reflexio/server/services/tagging/__init__.py +1 -0
- package/plugin/vendor/reflexio/reflexio/server/services/tagging/service.py +200 -0
- package/plugin/vendor/reflexio/reflexio/server/services/tagging/tagging_scheduler.py +149 -0
- package/plugin/vendor/reflexio/reflexio/server/services/unified_search_service.py +372 -34
- package/plugin/vendor/reflexio/reflexio/server/site_var/README.md +2 -1
- package/plugin/vendor/reflexio/reflexio/server/site_var/feature_flags.py +120 -1
- package/plugin/vendor/reflexio/reflexio/server/site_var/site_var_sources/feature_flags.json +4 -0
- package/plugin/vendor/reflexio/reflexio/server/site_var/site_var_sources/search_settings.json +5 -0
- package/plugin/vendor/reflexio/reflexio/server/tracing.py +30 -0
- package/plugin/vendor/reflexio/reflexio/server/usage_metrics.py +18 -0
- package/plugin/vendor/reflexio/reflexio/test_support/llm_mock.py +61 -26
- package/plugin/vendor/reflexio/reflexio/test_support/llm_model_registry.py +41 -4
- package/scripts/setup-claude-smart.sh +8 -3
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/TESTING.md +0 -517
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/hook/handler.js +0 -473
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/package-lock.json +0 -2156
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/package.json +0 -18
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/hook/handler.ts +0 -241
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/hook/setup.ts +0 -140
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/lib/publish.ts +0 -113
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/lib/search.ts +0 -52
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/lib/server.ts +0 -103
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/lib/sqlite-buffer.ts +0 -156
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/lib/user-id.ts +0 -134
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/rules/reflexio.md +0 -24
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/publish_clawhub.sh +0 -278
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/references/HOOK.md +0 -164
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/scripts/install.sh +0 -36
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/scripts/uninstall.sh +0 -35
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/tests/publish.test.ts +0 -27
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/tests/search.test.ts +0 -31
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/tests/server.test.ts +0 -42
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/tests/setup.test.ts +0 -49
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/tests/sqlite-buffer.test.ts +0 -91
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/tests/user-id.test.ts +0 -50
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/vitest.config.ts +0 -13
- package/plugin/vendor/reflexio/reflexio/server/_auth.py +0 -25
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/group_aggregation.py +0 -192
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/invariants.py +0 -303
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/plan.py +0 -138
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/tools.py +0 -1125
- /package/plugin/vendor/reflexio/reflexio/integrations/{__init__.py → openclaw/plugin/src/openclaw_smart/__init__.py} +0 -0
- /package/plugin/vendor/reflexio/reflexio/integrations/openclaw/{types → plugin/types}/openclaw.d.ts +0 -0
- /package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/{distribution.py → components/distribution.py} +0 -0
- /package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/{hero_state.py → components/hero_state.py} +0 -0
- /package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/{shadow_aggregation.py → components/shadow_aggregation.py} +0 -0
- /package/plugin/vendor/reflexio/reflexio/server/services/reflection/{reflection_extractor.py → components/extractor.py} +0 -0
|
@@ -0,0 +1,1965 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import sqlite3
|
|
5
|
+
import threading
|
|
6
|
+
from typing import Any, Literal, Protocol, cast
|
|
7
|
+
|
|
8
|
+
from reflexio.models.api_schema.domain import AgentPlaybook, AgentPlaybookSourceWindow
|
|
9
|
+
from reflexio.models.api_schema.domain.enums import Status
|
|
10
|
+
from reflexio.models.api_schema.domain.governance import (
|
|
11
|
+
AuditEvent,
|
|
12
|
+
PurgeOperation,
|
|
13
|
+
PurgeOperationTarget,
|
|
14
|
+
SubjectBarrierStatus,
|
|
15
|
+
SubjectWriteBarrier,
|
|
16
|
+
)
|
|
17
|
+
from reflexio.models.config_schema import GovernanceRetentionConfig
|
|
18
|
+
from reflexio.server.services.governance.config import (
|
|
19
|
+
get_governance_ref_secret,
|
|
20
|
+
governance_subject_ref,
|
|
21
|
+
)
|
|
22
|
+
from reflexio.server.services.storage.error import SubjectWriteBarrierError
|
|
23
|
+
from reflexio.server.services.storage.governance_validation import (
|
|
24
|
+
_ALLOWED_PURGE_OPERATION_TYPES,
|
|
25
|
+
_ALLOWED_PURGE_SCOPE_TYPES,
|
|
26
|
+
_ALLOWED_PURGE_TARGET_DETAIL_KEYS,
|
|
27
|
+
_ALLOWED_PURGE_TARGET_NAMES,
|
|
28
|
+
_ALLOWED_PURGE_TARGET_PHASES,
|
|
29
|
+
_ALLOWED_PURGE_TARGET_STATUSES,
|
|
30
|
+
_CANONICAL_DELETE_TARGET_NAMES,
|
|
31
|
+
_PREPARE_PHASE,
|
|
32
|
+
_SNAPSHOT_TARGET_NAME,
|
|
33
|
+
_canonicalize_audit_event_for_persistence,
|
|
34
|
+
_canonicalize_governance_windows,
|
|
35
|
+
_epoch_now,
|
|
36
|
+
_is_successful_erase_event,
|
|
37
|
+
_parse_governance_window_list,
|
|
38
|
+
_successful_erase_identity,
|
|
39
|
+
_validate_governance_deleted_count,
|
|
40
|
+
_validate_governance_detail,
|
|
41
|
+
_validate_governance_enum,
|
|
42
|
+
_validate_governance_error_code,
|
|
43
|
+
_validate_governance_error_detail,
|
|
44
|
+
_validate_governance_idempotency_key,
|
|
45
|
+
_validate_governance_int_list,
|
|
46
|
+
_validate_governance_prefixed_ref,
|
|
47
|
+
_validate_governance_purge_id,
|
|
48
|
+
_validate_governance_target_ref,
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
_LEGACY_AUDIT_REQUEST_REF = "reqref_v1_legacy_unknown"
|
|
52
|
+
|
|
53
|
+
_PURGE_OPERATION_TARGETS_TABLE_DDL = """
|
|
54
|
+
CREATE TABLE IF NOT EXISTS purge_operation_targets (
|
|
55
|
+
org_id TEXT NOT NULL,
|
|
56
|
+
purge_id TEXT NOT NULL,
|
|
57
|
+
target_name TEXT NOT NULL,
|
|
58
|
+
target_ref TEXT NOT NULL DEFAULT '',
|
|
59
|
+
phase TEXT NOT NULL,
|
|
60
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
61
|
+
detail TEXT,
|
|
62
|
+
deleted_count INTEGER NOT NULL DEFAULT 0,
|
|
63
|
+
error_detail TEXT,
|
|
64
|
+
started_at INTEGER,
|
|
65
|
+
completed_at INTEGER,
|
|
66
|
+
PRIMARY KEY (org_id, purge_id, target_name, target_ref, phase),
|
|
67
|
+
FOREIGN KEY (org_id, purge_id) REFERENCES purge_operations(org_id, purge_id) ON DELETE CASCADE
|
|
68
|
+
);
|
|
69
|
+
"""
|
|
70
|
+
|
|
71
|
+
GOVERNANCE_DDL = f"""
|
|
72
|
+
CREATE TABLE IF NOT EXISTS audit_events (
|
|
73
|
+
event_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
74
|
+
org_id TEXT NOT NULL,
|
|
75
|
+
actor_type TEXT NOT NULL DEFAULT 'system',
|
|
76
|
+
actor_ref TEXT,
|
|
77
|
+
operation TEXT NOT NULL,
|
|
78
|
+
entity_type TEXT NOT NULL,
|
|
79
|
+
entity_id TEXT,
|
|
80
|
+
subject_ref TEXT,
|
|
81
|
+
request_ref TEXT NOT NULL,
|
|
82
|
+
idempotency_key TEXT,
|
|
83
|
+
status TEXT NOT NULL DEFAULT 'ok',
|
|
84
|
+
detail TEXT,
|
|
85
|
+
created_at INTEGER NOT NULL
|
|
86
|
+
);
|
|
87
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_audit_events_org_idem
|
|
88
|
+
ON audit_events(org_id, idempotency_key)
|
|
89
|
+
WHERE idempotency_key IS NOT NULL;
|
|
90
|
+
CREATE INDEX IF NOT EXISTS idx_audit_events_subject_created
|
|
91
|
+
ON audit_events(org_id, subject_ref, created_at, event_id);
|
|
92
|
+
CREATE INDEX IF NOT EXISTS idx_audit_events_org_created
|
|
93
|
+
ON audit_events(org_id, created_at, event_id);
|
|
94
|
+
|
|
95
|
+
CREATE TABLE IF NOT EXISTS purge_operations (
|
|
96
|
+
org_id TEXT NOT NULL,
|
|
97
|
+
purge_id TEXT NOT NULL,
|
|
98
|
+
operation_type TEXT NOT NULL,
|
|
99
|
+
scope_type TEXT NOT NULL,
|
|
100
|
+
subject_ref TEXT,
|
|
101
|
+
request_ref TEXT NOT NULL,
|
|
102
|
+
idempotency_key TEXT NOT NULL,
|
|
103
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
104
|
+
error_code TEXT,
|
|
105
|
+
error_detail TEXT,
|
|
106
|
+
created_at INTEGER NOT NULL,
|
|
107
|
+
updated_at INTEGER NOT NULL,
|
|
108
|
+
completed_at INTEGER,
|
|
109
|
+
PRIMARY KEY (org_id, purge_id)
|
|
110
|
+
);
|
|
111
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_purge_operations_org_idem
|
|
112
|
+
ON purge_operations(org_id, idempotency_key);
|
|
113
|
+
|
|
114
|
+
{_PURGE_OPERATION_TARGETS_TABLE_DDL}
|
|
115
|
+
CREATE INDEX IF NOT EXISTS idx_purge_targets_purge_phase
|
|
116
|
+
ON purge_operation_targets(org_id, purge_id, phase, status);
|
|
117
|
+
|
|
118
|
+
CREATE TABLE IF NOT EXISTS subject_write_barriers (
|
|
119
|
+
org_id TEXT NOT NULL,
|
|
120
|
+
subject_ref TEXT NOT NULL,
|
|
121
|
+
purge_id TEXT NOT NULL,
|
|
122
|
+
status TEXT NOT NULL,
|
|
123
|
+
error_code TEXT,
|
|
124
|
+
error_detail TEXT,
|
|
125
|
+
created_at INTEGER NOT NULL,
|
|
126
|
+
updated_at INTEGER NOT NULL,
|
|
127
|
+
PRIMARY KEY (org_id, subject_ref),
|
|
128
|
+
CHECK (status IN ('erasing', 'erased', 'failed'))
|
|
129
|
+
);
|
|
130
|
+
CREATE INDEX IF NOT EXISTS idx_subject_write_barriers_org_purge
|
|
131
|
+
ON subject_write_barriers(org_id, purge_id);
|
|
132
|
+
"""
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def init_governance_tables(conn: sqlite3.Connection) -> None:
|
|
136
|
+
_upgrade_legacy_purge_operation_targets_table(conn)
|
|
137
|
+
conn.executescript(GOVERNANCE_DDL)
|
|
138
|
+
_enforce_audit_request_ref_not_null(conn)
|
|
139
|
+
_ensure_governance_subject_ref_columns(conn)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _ensure_governance_subject_ref_columns(conn: sqlite3.Connection) -> None:
|
|
143
|
+
for table in (
|
|
144
|
+
"requests",
|
|
145
|
+
"interactions",
|
|
146
|
+
"profiles",
|
|
147
|
+
"user_playbooks",
|
|
148
|
+
"agent_success_evaluation_result",
|
|
149
|
+
):
|
|
150
|
+
columns = [row[1] for row in conn.execute(f"PRAGMA table_info({table})")]
|
|
151
|
+
if not columns:
|
|
152
|
+
continue
|
|
153
|
+
if "governance_subject_ref" in columns:
|
|
154
|
+
pass
|
|
155
|
+
else:
|
|
156
|
+
conn.execute(f"ALTER TABLE {table} ADD COLUMN governance_subject_ref TEXT")
|
|
157
|
+
conn.execute(
|
|
158
|
+
f"CREATE INDEX IF NOT EXISTS idx_{table}_governance_subject_ref "
|
|
159
|
+
f"ON {table}(governance_subject_ref)"
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _json_dumps(obj: Any) -> str | None:
|
|
164
|
+
if obj is None:
|
|
165
|
+
return None
|
|
166
|
+
return json.dumps(obj, default=str)
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def _json_loads(text: str | None) -> Any:
|
|
170
|
+
if not text:
|
|
171
|
+
return None
|
|
172
|
+
return json.loads(text)
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _build_agent_playbook_source_window_rows(
|
|
176
|
+
agent_playbook_id: int, windows: list[AgentPlaybookSourceWindow]
|
|
177
|
+
) -> list[tuple[int, int, str]]:
|
|
178
|
+
by_id: dict[int, list[int]] = {}
|
|
179
|
+
for window in windows:
|
|
180
|
+
ids = by_id.setdefault(window.user_playbook_id, [])
|
|
181
|
+
seen = set(ids)
|
|
182
|
+
for source_id in window.source_interaction_ids:
|
|
183
|
+
if source_id not in seen:
|
|
184
|
+
ids.append(source_id)
|
|
185
|
+
seen.add(source_id)
|
|
186
|
+
return [
|
|
187
|
+
(
|
|
188
|
+
agent_playbook_id,
|
|
189
|
+
user_playbook_id,
|
|
190
|
+
_json_dumps(source_interaction_ids) or "[]",
|
|
191
|
+
)
|
|
192
|
+
for user_playbook_id, source_interaction_ids in by_id.items()
|
|
193
|
+
]
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _upgrade_legacy_purge_operation_targets_table(conn: sqlite3.Connection) -> None:
|
|
197
|
+
target_columns = [
|
|
198
|
+
row[1] for row in conn.execute("PRAGMA table_info(purge_operation_targets)")
|
|
199
|
+
]
|
|
200
|
+
if not target_columns or "org_id" in target_columns:
|
|
201
|
+
return
|
|
202
|
+
conn.execute(
|
|
203
|
+
"ALTER TABLE purge_operation_targets RENAME TO purge_operation_targets_legacy"
|
|
204
|
+
)
|
|
205
|
+
conn.executescript(_PURGE_OPERATION_TARGETS_TABLE_DDL)
|
|
206
|
+
conn.execute(
|
|
207
|
+
"""INSERT INTO purge_operation_targets (
|
|
208
|
+
org_id, purge_id, target_name, target_ref, phase, status, detail,
|
|
209
|
+
deleted_count, error_detail, started_at, completed_at
|
|
210
|
+
)
|
|
211
|
+
SELECT uniquely_mapped_purges.org_id, legacy.purge_id, legacy.target_name,
|
|
212
|
+
legacy.target_ref, legacy.phase, legacy.status, legacy.detail,
|
|
213
|
+
legacy.deleted_count, legacy.error_detail, legacy.started_at,
|
|
214
|
+
legacy.completed_at
|
|
215
|
+
FROM purge_operation_targets_legacy AS legacy
|
|
216
|
+
JOIN (
|
|
217
|
+
SELECT MIN(org_id) AS org_id, purge_id
|
|
218
|
+
FROM purge_operations
|
|
219
|
+
GROUP BY purge_id
|
|
220
|
+
HAVING COUNT(*) = 1
|
|
221
|
+
) AS uniquely_mapped_purges
|
|
222
|
+
ON uniquely_mapped_purges.purge_id = legacy.purge_id"""
|
|
223
|
+
)
|
|
224
|
+
conn.execute("DROP TABLE purge_operation_targets_legacy")
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _enforce_audit_request_ref_not_null(conn: sqlite3.Connection) -> None:
|
|
228
|
+
audit_columns = [row[1] for row in conn.execute("PRAGMA table_info(audit_events)")]
|
|
229
|
+
if not audit_columns:
|
|
230
|
+
return
|
|
231
|
+
conn.execute(
|
|
232
|
+
"UPDATE audit_events SET request_ref = ? WHERE request_ref IS NULL",
|
|
233
|
+
(_LEGACY_AUDIT_REQUEST_REF,),
|
|
234
|
+
)
|
|
235
|
+
conn.execute(
|
|
236
|
+
"""
|
|
237
|
+
CREATE TRIGGER IF NOT EXISTS audit_events_request_ref_not_null
|
|
238
|
+
BEFORE INSERT ON audit_events
|
|
239
|
+
WHEN NEW.request_ref IS NULL
|
|
240
|
+
BEGIN
|
|
241
|
+
SELECT RAISE(ABORT, 'audit_events.request_ref is required');
|
|
242
|
+
END
|
|
243
|
+
"""
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _row_to_audit_event(row: sqlite3.Row) -> AuditEvent:
|
|
248
|
+
return AuditEvent(
|
|
249
|
+
org_id=row["org_id"],
|
|
250
|
+
actor_type=row["actor_type"],
|
|
251
|
+
actor_ref=row["actor_ref"],
|
|
252
|
+
operation=row["operation"],
|
|
253
|
+
entity_type=row["entity_type"],
|
|
254
|
+
entity_id=row["entity_id"],
|
|
255
|
+
subject_ref=row["subject_ref"],
|
|
256
|
+
request_ref=row["request_ref"],
|
|
257
|
+
idempotency_key=row["idempotency_key"],
|
|
258
|
+
status=row["status"],
|
|
259
|
+
detail=_json_loads(row["detail"]),
|
|
260
|
+
created_at=row["created_at"],
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _row_to_purge_operation(row: sqlite3.Row) -> PurgeOperation:
|
|
265
|
+
return PurgeOperation(
|
|
266
|
+
purge_id=row["purge_id"],
|
|
267
|
+
org_id=row["org_id"],
|
|
268
|
+
operation_type=row["operation_type"],
|
|
269
|
+
scope_type=row["scope_type"],
|
|
270
|
+
subject_ref=row["subject_ref"],
|
|
271
|
+
request_ref=row["request_ref"],
|
|
272
|
+
idempotency_key=row["idempotency_key"],
|
|
273
|
+
status=row["status"],
|
|
274
|
+
error_code=row["error_code"],
|
|
275
|
+
error_detail=row["error_detail"],
|
|
276
|
+
created_at=row["created_at"],
|
|
277
|
+
updated_at=row["updated_at"],
|
|
278
|
+
completed_at=row["completed_at"],
|
|
279
|
+
)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _row_to_purge_target(row: sqlite3.Row) -> PurgeOperationTarget:
|
|
283
|
+
return PurgeOperationTarget(
|
|
284
|
+
purge_id=row["purge_id"],
|
|
285
|
+
target_name=row["target_name"],
|
|
286
|
+
target_ref=row["target_ref"],
|
|
287
|
+
phase=row["phase"],
|
|
288
|
+
status=row["status"],
|
|
289
|
+
detail=_json_loads(row["detail"]),
|
|
290
|
+
deleted_count=row["deleted_count"],
|
|
291
|
+
error_detail=row["error_detail"],
|
|
292
|
+
started_at=row["started_at"],
|
|
293
|
+
completed_at=row["completed_at"],
|
|
294
|
+
)
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _row_to_subject_write_barrier(row: sqlite3.Row) -> SubjectWriteBarrier:
|
|
298
|
+
return SubjectWriteBarrier(
|
|
299
|
+
org_id=str(row["org_id"]),
|
|
300
|
+
subject_ref=str(row["subject_ref"]),
|
|
301
|
+
purge_id=str(row["purge_id"]),
|
|
302
|
+
status=cast(SubjectBarrierStatus, str(row["status"])),
|
|
303
|
+
error_code=row["error_code"],
|
|
304
|
+
error_detail=row["error_detail"],
|
|
305
|
+
created_at=int(row["created_at"]),
|
|
306
|
+
updated_at=int(row["updated_at"]),
|
|
307
|
+
)
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
class _SQLiteGovernanceDeps(Protocol):
|
|
311
|
+
conn: sqlite3.Connection
|
|
312
|
+
_lock: threading.RLock
|
|
313
|
+
org_id: str
|
|
314
|
+
_has_sqlite_vec: bool
|
|
315
|
+
|
|
316
|
+
def _fetchall(
|
|
317
|
+
self, sql: str, params: list[Any] | tuple[Any, ...]
|
|
318
|
+
) -> list[sqlite3.Row]: ...
|
|
319
|
+
|
|
320
|
+
def _fetchone(
|
|
321
|
+
self, sql: str, params: list[Any] | tuple[Any, ...]
|
|
322
|
+
) -> sqlite3.Row | None: ...
|
|
323
|
+
|
|
324
|
+
def _partition_purge_vs_delete(
|
|
325
|
+
self, entity_type: Literal["profile", "user_playbook"], ids: list[str]
|
|
326
|
+
) -> tuple[list[str], list[str]]: ...
|
|
327
|
+
|
|
328
|
+
def _delete_in_chunks(
|
|
329
|
+
self, table_name: str, column_name: str, values: list[Any]
|
|
330
|
+
) -> None: ...
|
|
331
|
+
|
|
332
|
+
def _delete_source_windows_for_user_playbook_ids(
|
|
333
|
+
self, user_playbook_ids: list[int]
|
|
334
|
+
) -> None: ...
|
|
335
|
+
|
|
336
|
+
def _get_embedding(self, text: str) -> list[float]: ...
|
|
337
|
+
|
|
338
|
+
def set_source_windows_for_agent_playbook(
|
|
339
|
+
self, agent_playbook_id: int, windows: list[AgentPlaybookSourceWindow]
|
|
340
|
+
) -> None: ...
|
|
341
|
+
|
|
342
|
+
def get_source_windows_for_agent_playbook(
|
|
343
|
+
self, agent_playbook_id: int
|
|
344
|
+
) -> list[AgentPlaybookSourceWindow]: ...
|
|
345
|
+
|
|
346
|
+
def get_agent_playbook_by_id(
|
|
347
|
+
self,
|
|
348
|
+
agent_playbook_id: int,
|
|
349
|
+
*,
|
|
350
|
+
include_tombstones: bool = False,
|
|
351
|
+
) -> AgentPlaybook | None: ...
|
|
352
|
+
|
|
353
|
+
def _index_agent_playbook_fts_vec(self, ap: AgentPlaybook) -> None: ...
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
class SQLiteGovernanceMixin:
|
|
357
|
+
"""SQLite governance storage primitives."""
|
|
358
|
+
|
|
359
|
+
conn: sqlite3.Connection
|
|
360
|
+
_lock: threading.RLock
|
|
361
|
+
org_id: str
|
|
362
|
+
|
|
363
|
+
def _deps(self) -> _SQLiteGovernanceDeps:
|
|
364
|
+
return cast(_SQLiteGovernanceDeps, self)
|
|
365
|
+
|
|
366
|
+
def _barrier_from_purge(
|
|
367
|
+
self,
|
|
368
|
+
purge_operation: PurgeOperation,
|
|
369
|
+
*,
|
|
370
|
+
subject_ref: str,
|
|
371
|
+
) -> SubjectWriteBarrier:
|
|
372
|
+
if purge_operation.subject_ref != subject_ref:
|
|
373
|
+
raise ValueError(
|
|
374
|
+
"Purge operation subject_ref must match the barrier subject_ref"
|
|
375
|
+
)
|
|
376
|
+
status_by_purge_status: dict[str, SubjectBarrierStatus] = {
|
|
377
|
+
"pending": "erasing",
|
|
378
|
+
"running": "erasing",
|
|
379
|
+
"complete": "erased",
|
|
380
|
+
"failed": "failed",
|
|
381
|
+
}
|
|
382
|
+
return SubjectWriteBarrier(
|
|
383
|
+
org_id=purge_operation.org_id,
|
|
384
|
+
subject_ref=subject_ref,
|
|
385
|
+
purge_id=purge_operation.purge_id,
|
|
386
|
+
status=status_by_purge_status[purge_operation.status],
|
|
387
|
+
error_code=purge_operation.error_code,
|
|
388
|
+
error_detail=purge_operation.error_detail,
|
|
389
|
+
created_at=purge_operation.created_at,
|
|
390
|
+
updated_at=purge_operation.updated_at,
|
|
391
|
+
)
|
|
392
|
+
|
|
393
|
+
def _active_subject_barrier_locked(self, subject_ref: str) -> sqlite3.Row | None:
|
|
394
|
+
return self.conn.execute(
|
|
395
|
+
"""SELECT * FROM subject_write_barriers
|
|
396
|
+
WHERE org_id = ? AND subject_ref = ? AND status IN ('erasing', 'erased')""",
|
|
397
|
+
(self.org_id, subject_ref),
|
|
398
|
+
).fetchone()
|
|
399
|
+
|
|
400
|
+
def _assert_subject_writable_locked(self, subject_ref: str) -> None:
|
|
401
|
+
row = self._active_subject_barrier_locked(subject_ref)
|
|
402
|
+
if row is not None:
|
|
403
|
+
raise SubjectWriteBarrierError(
|
|
404
|
+
f"subject {subject_ref} is blocked by erasure barrier {row['purge_id']}"
|
|
405
|
+
)
|
|
406
|
+
|
|
407
|
+
def _subject_ref_for_user_id(self, user_id: str) -> str:
|
|
408
|
+
return governance_subject_ref(
|
|
409
|
+
self.org_id,
|
|
410
|
+
user_id,
|
|
411
|
+
get_governance_ref_secret(),
|
|
412
|
+
)
|
|
413
|
+
|
|
414
|
+
def _legacy_request_ids_for_subject_locked(self, subject_ref: str) -> set[str]:
|
|
415
|
+
request_ids: set[str] = set()
|
|
416
|
+
for row in self.conn.execute(
|
|
417
|
+
"""SELECT request_id, user_id
|
|
418
|
+
FROM requests
|
|
419
|
+
WHERE governance_subject_ref IS NULL"""
|
|
420
|
+
):
|
|
421
|
+
user_id = str(row["user_id"])
|
|
422
|
+
if self._subject_ref_for_user_id(user_id) != subject_ref:
|
|
423
|
+
continue
|
|
424
|
+
request_ids.add(str(row["request_id"]))
|
|
425
|
+
return request_ids
|
|
426
|
+
|
|
427
|
+
def _legacy_user_id_rows_remain_locked(
|
|
428
|
+
self,
|
|
429
|
+
*,
|
|
430
|
+
table: str,
|
|
431
|
+
subject_ref: str,
|
|
432
|
+
request_ids: set[str] | None = None,
|
|
433
|
+
request_id_column: str | None = None,
|
|
434
|
+
) -> bool:
|
|
435
|
+
sql = f"SELECT user_id{', ' + request_id_column if request_id_column else ''} FROM {table} WHERE governance_subject_ref IS NULL" # noqa: S608
|
|
436
|
+
for row in self.conn.execute(sql):
|
|
437
|
+
user_id = str(row["user_id"])
|
|
438
|
+
if self._subject_ref_for_user_id(user_id) == subject_ref:
|
|
439
|
+
return True
|
|
440
|
+
if (
|
|
441
|
+
request_ids
|
|
442
|
+
and request_id_column is not None
|
|
443
|
+
and str(row[request_id_column]) in request_ids
|
|
444
|
+
):
|
|
445
|
+
return True
|
|
446
|
+
return False
|
|
447
|
+
|
|
448
|
+
def _same_subject_rows_remain_locked(self, subject_ref: str) -> bool:
|
|
449
|
+
legacy_request_ids = self._legacy_request_ids_for_subject_locked(subject_ref)
|
|
450
|
+
for table in (
|
|
451
|
+
"requests",
|
|
452
|
+
"interactions",
|
|
453
|
+
"profiles",
|
|
454
|
+
"user_playbooks",
|
|
455
|
+
"agent_success_evaluation_result",
|
|
456
|
+
):
|
|
457
|
+
row = self.conn.execute(
|
|
458
|
+
f"""SELECT 1 FROM {table}
|
|
459
|
+
WHERE governance_subject_ref = ?
|
|
460
|
+
LIMIT 1""",
|
|
461
|
+
(subject_ref,),
|
|
462
|
+
).fetchone()
|
|
463
|
+
if row is not None:
|
|
464
|
+
return True
|
|
465
|
+
if legacy_request_ids:
|
|
466
|
+
return True
|
|
467
|
+
if self._legacy_user_id_rows_remain_locked(
|
|
468
|
+
table="interactions",
|
|
469
|
+
subject_ref=subject_ref,
|
|
470
|
+
request_ids=legacy_request_ids,
|
|
471
|
+
request_id_column="request_id",
|
|
472
|
+
):
|
|
473
|
+
return True
|
|
474
|
+
if self._legacy_user_id_rows_remain_locked(
|
|
475
|
+
table="profiles",
|
|
476
|
+
subject_ref=subject_ref,
|
|
477
|
+
request_ids=legacy_request_ids,
|
|
478
|
+
request_id_column="generated_from_request_id",
|
|
479
|
+
):
|
|
480
|
+
return True
|
|
481
|
+
if self._legacy_user_id_rows_remain_locked(
|
|
482
|
+
table="user_playbooks",
|
|
483
|
+
subject_ref=subject_ref,
|
|
484
|
+
request_ids=legacy_request_ids,
|
|
485
|
+
request_id_column="request_id",
|
|
486
|
+
):
|
|
487
|
+
return True
|
|
488
|
+
return self._legacy_user_id_rows_remain_locked(
|
|
489
|
+
table="agent_success_evaluation_result",
|
|
490
|
+
subject_ref=subject_ref,
|
|
491
|
+
)
|
|
492
|
+
|
|
493
|
+
def _replace_agent_playbook_source_windows_locked(
|
|
494
|
+
self, agent_playbook_id: int, windows: list[AgentPlaybookSourceWindow]
|
|
495
|
+
) -> None:
|
|
496
|
+
self.conn.execute(
|
|
497
|
+
"DELETE FROM agent_playbook_source_user_playbooks WHERE agent_playbook_id = ?",
|
|
498
|
+
(agent_playbook_id,),
|
|
499
|
+
)
|
|
500
|
+
source_window_rows = _build_agent_playbook_source_window_rows(
|
|
501
|
+
agent_playbook_id, windows
|
|
502
|
+
)
|
|
503
|
+
if source_window_rows:
|
|
504
|
+
self.conn.executemany(
|
|
505
|
+
"""INSERT OR IGNORE INTO agent_playbook_source_user_playbooks
|
|
506
|
+
(agent_playbook_id, user_playbook_id, source_interaction_ids)
|
|
507
|
+
VALUES (?, ?, ?)""",
|
|
508
|
+
source_window_rows,
|
|
509
|
+
)
|
|
510
|
+
|
|
511
|
+
def _delete_agent_playbook_search_rows_locked(self, agent_playbook_id: int) -> None:
|
|
512
|
+
self.conn.execute(
|
|
513
|
+
"DELETE FROM agent_playbooks_fts WHERE rowid = ?",
|
|
514
|
+
(agent_playbook_id,),
|
|
515
|
+
)
|
|
516
|
+
if self._deps()._has_sqlite_vec:
|
|
517
|
+
self.conn.execute(
|
|
518
|
+
"DELETE FROM agent_playbooks_vec WHERE rowid = ?",
|
|
519
|
+
(agent_playbook_id,),
|
|
520
|
+
)
|
|
521
|
+
|
|
522
|
+
def _upsert_agent_playbook_search_rows_locked(
|
|
523
|
+
self,
|
|
524
|
+
*,
|
|
525
|
+
agent_playbook_id: int,
|
|
526
|
+
trigger: str | None,
|
|
527
|
+
content: str,
|
|
528
|
+
expanded_terms: str | None,
|
|
529
|
+
embedding: list[float],
|
|
530
|
+
) -> None:
|
|
531
|
+
self._delete_agent_playbook_search_rows_locked(agent_playbook_id)
|
|
532
|
+
fts_parts = [trigger or "", content]
|
|
533
|
+
if expanded_terms:
|
|
534
|
+
fts_parts.append(expanded_terms)
|
|
535
|
+
self.conn.execute(
|
|
536
|
+
"INSERT INTO agent_playbooks_fts(rowid, search_text) VALUES (?, ?)",
|
|
537
|
+
(
|
|
538
|
+
agent_playbook_id,
|
|
539
|
+
" ".join(part for part in fts_parts if part) or "",
|
|
540
|
+
),
|
|
541
|
+
)
|
|
542
|
+
if self._deps()._has_sqlite_vec and embedding:
|
|
543
|
+
self.conn.execute(
|
|
544
|
+
"INSERT INTO agent_playbooks_vec(rowid, embedding) VALUES (?, ?)",
|
|
545
|
+
(agent_playbook_id, json.dumps(embedding)),
|
|
546
|
+
)
|
|
547
|
+
|
|
548
|
+
def _validate_prepared_delete_target_matrix_locked(self, purge_id: str) -> None:
|
|
549
|
+
snapshot = self.conn.execute(
|
|
550
|
+
"""SELECT 1 FROM purge_operation_targets
|
|
551
|
+
WHERE org_id = ? AND purge_id = ? AND target_name = ? AND target_ref = 'all'
|
|
552
|
+
AND phase = ? AND status = 'complete'""",
|
|
553
|
+
(self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE),
|
|
554
|
+
).fetchone()
|
|
555
|
+
if snapshot is None:
|
|
556
|
+
raise ValueError("Cannot delete user data without target snapshot marker")
|
|
557
|
+
delete_rows = self.conn.execute(
|
|
558
|
+
"""SELECT target_name, status FROM purge_operation_targets
|
|
559
|
+
WHERE org_id = ? AND purge_id = ? AND phase = 'delete'
|
|
560
|
+
AND target_ref = 'all'""",
|
|
561
|
+
(self.org_id, purge_id),
|
|
562
|
+
).fetchall()
|
|
563
|
+
delete_statuses = {
|
|
564
|
+
str(row["target_name"]): str(row["status"]) for row in delete_rows
|
|
565
|
+
}
|
|
566
|
+
missing_delete_targets = [
|
|
567
|
+
target_name
|
|
568
|
+
for target_name in _CANONICAL_DELETE_TARGET_NAMES
|
|
569
|
+
if delete_statuses.get(target_name) not in {"pending", "complete"}
|
|
570
|
+
]
|
|
571
|
+
if missing_delete_targets:
|
|
572
|
+
raise ValueError(
|
|
573
|
+
"Cannot delete user data without complete delete target matrix: "
|
|
574
|
+
+ ", ".join(missing_delete_targets)
|
|
575
|
+
)
|
|
576
|
+
|
|
577
|
+
def _validate_hide_for_rebuild_targets_locked(self, purge_id: str) -> None:
|
|
578
|
+
rebuild_rows = self.conn.execute(
|
|
579
|
+
"""SELECT DISTINCT target_ref
|
|
580
|
+
FROM purge_operation_targets
|
|
581
|
+
WHERE org_id = ? AND purge_id = ? AND target_name = 'agent_playbook'
|
|
582
|
+
AND phase = 'rebuild_without_erased_sources' AND target_ref != ''
|
|
583
|
+
ORDER BY target_ref ASC""",
|
|
584
|
+
(self.org_id, purge_id),
|
|
585
|
+
).fetchall()
|
|
586
|
+
if not rebuild_rows:
|
|
587
|
+
return
|
|
588
|
+
hidden_refs = {
|
|
589
|
+
str(row["target_ref"])
|
|
590
|
+
for row in self.conn.execute(
|
|
591
|
+
"""SELECT target_ref
|
|
592
|
+
FROM purge_operation_targets
|
|
593
|
+
WHERE org_id = ? AND purge_id = ? AND target_name = 'agent_playbook'
|
|
594
|
+
AND phase = 'hide_for_rebuild' AND status = 'complete'""",
|
|
595
|
+
(self.org_id, purge_id),
|
|
596
|
+
).fetchall()
|
|
597
|
+
}
|
|
598
|
+
missing_hidden_refs = [
|
|
599
|
+
str(row["target_ref"])
|
|
600
|
+
for row in rebuild_rows
|
|
601
|
+
if str(row["target_ref"]) not in hidden_refs
|
|
602
|
+
]
|
|
603
|
+
if missing_hidden_refs:
|
|
604
|
+
raise ValueError(
|
|
605
|
+
"Cannot delete user data before hide_for_rebuild completes for "
|
|
606
|
+
f"planned agent_playbooks: {', '.join(missing_hidden_refs)}"
|
|
607
|
+
)
|
|
608
|
+
|
|
609
|
+
def _planned_governance_delete_counts(
|
|
610
|
+
self, user_id: str, owned_user_playbook_ids: set[int]
|
|
611
|
+
) -> dict[str, int]:
|
|
612
|
+
request_row = self.conn.execute(
|
|
613
|
+
"SELECT COUNT(*) AS cnt FROM requests WHERE user_id = ?",
|
|
614
|
+
(user_id,),
|
|
615
|
+
).fetchone()
|
|
616
|
+
interaction_row = self.conn.execute(
|
|
617
|
+
"SELECT COUNT(*) AS cnt FROM interactions WHERE user_id = ?",
|
|
618
|
+
(user_id,),
|
|
619
|
+
).fetchone()
|
|
620
|
+
eval_result_row = self.conn.execute(
|
|
621
|
+
"""SELECT COUNT(*) AS cnt
|
|
622
|
+
FROM agent_success_evaluation_result
|
|
623
|
+
WHERE user_id = ?""",
|
|
624
|
+
(user_id,),
|
|
625
|
+
).fetchone()
|
|
626
|
+
profile_rows = self.conn.execute(
|
|
627
|
+
"SELECT profile_id FROM profiles WHERE user_id = ?",
|
|
628
|
+
(user_id,),
|
|
629
|
+
).fetchall()
|
|
630
|
+
if request_row is None or interaction_row is None or eval_result_row is None:
|
|
631
|
+
raise ValueError("Missing governance count rows")
|
|
632
|
+
profile_ids = [str(row["profile_id"]) for row in profile_rows]
|
|
633
|
+
purge_profile_ids, delete_profile_ids = self._deps()._partition_purge_vs_delete(
|
|
634
|
+
"profile",
|
|
635
|
+
profile_ids,
|
|
636
|
+
)
|
|
637
|
+
playbook_ids = [
|
|
638
|
+
str(user_playbook_id)
|
|
639
|
+
for user_playbook_id in sorted(owned_user_playbook_ids)
|
|
640
|
+
]
|
|
641
|
+
purge_playbook_ids, delete_playbook_ids = (
|
|
642
|
+
self._deps()._partition_purge_vs_delete(
|
|
643
|
+
"user_playbook",
|
|
644
|
+
playbook_ids,
|
|
645
|
+
)
|
|
646
|
+
)
|
|
647
|
+
return {
|
|
648
|
+
"request": int(request_row["cnt"]),
|
|
649
|
+
"interaction": int(interaction_row["cnt"]),
|
|
650
|
+
"profile": len(delete_profile_ids),
|
|
651
|
+
"profile_purge": len(purge_profile_ids),
|
|
652
|
+
"user_playbook": len(delete_playbook_ids),
|
|
653
|
+
"agent_success_evaluation_result": int(eval_result_row["cnt"]),
|
|
654
|
+
"user_playbook_purge": len(purge_playbook_ids),
|
|
655
|
+
}
|
|
656
|
+
|
|
657
|
+
def _owned_user_playbook_ids_locked(self, user_id: str) -> set[int]:
|
|
658
|
+
return {
|
|
659
|
+
int(row["user_playbook_id"])
|
|
660
|
+
for row in self.conn.execute(
|
|
661
|
+
"SELECT user_playbook_id FROM user_playbooks WHERE user_id = ?",
|
|
662
|
+
(user_id,),
|
|
663
|
+
).fetchall()
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
def _prepared_owned_user_playbook_ids_locked(self, purge_id: str) -> set[int]:
|
|
667
|
+
row = self.conn.execute(
|
|
668
|
+
"""SELECT detail FROM purge_operation_targets
|
|
669
|
+
WHERE org_id = ? AND purge_id = ? AND target_name = ?
|
|
670
|
+
AND target_ref = 'all' AND phase = ? AND status = 'complete'""",
|
|
671
|
+
(self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE),
|
|
672
|
+
).fetchone()
|
|
673
|
+
if row is None:
|
|
674
|
+
raise ValueError("Prepared target snapshot is missing")
|
|
675
|
+
detail = _json_loads(row["detail"])
|
|
676
|
+
if not isinstance(detail, dict):
|
|
677
|
+
raise ValueError("Prepared target snapshot detail is missing")
|
|
678
|
+
return set(
|
|
679
|
+
_validate_governance_int_list(
|
|
680
|
+
"owned_user_playbook_ids",
|
|
681
|
+
detail.get("owned_user_playbook_ids"),
|
|
682
|
+
)
|
|
683
|
+
)
|
|
684
|
+
|
|
685
|
+
def _append_audit_event_with_cursor(
|
|
686
|
+
self, cur: sqlite3.Connection | sqlite3.Cursor, event: AuditEvent
|
|
687
|
+
) -> bool:
|
|
688
|
+
inserted = cur.execute(
|
|
689
|
+
"""INSERT OR IGNORE INTO audit_events (
|
|
690
|
+
org_id, actor_type, actor_ref, operation, entity_type, entity_id,
|
|
691
|
+
subject_ref, request_ref, idempotency_key, status, detail, created_at
|
|
692
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
|
|
693
|
+
(
|
|
694
|
+
event.org_id,
|
|
695
|
+
event.actor_type,
|
|
696
|
+
event.actor_ref,
|
|
697
|
+
event.operation,
|
|
698
|
+
event.entity_type,
|
|
699
|
+
event.entity_id,
|
|
700
|
+
event.subject_ref,
|
|
701
|
+
event.request_ref,
|
|
702
|
+
event.idempotency_key,
|
|
703
|
+
event.status,
|
|
704
|
+
_json_dumps(event.detail),
|
|
705
|
+
event.created_at,
|
|
706
|
+
),
|
|
707
|
+
)
|
|
708
|
+
return inserted.rowcount > 0
|
|
709
|
+
|
|
710
|
+
def _record_purge_target_locked(
|
|
711
|
+
self,
|
|
712
|
+
*,
|
|
713
|
+
purge_id: str,
|
|
714
|
+
target_name: str,
|
|
715
|
+
target_ref: str,
|
|
716
|
+
phase: str,
|
|
717
|
+
status: Literal["pending", "running", "failed", "complete"],
|
|
718
|
+
detail: dict[str, object] | None,
|
|
719
|
+
deleted_count: int,
|
|
720
|
+
error_detail: str | None,
|
|
721
|
+
) -> None:
|
|
722
|
+
purge_id = _validate_governance_purge_id("purge_id", purge_id)
|
|
723
|
+
_validate_governance_enum(
|
|
724
|
+
"target_name",
|
|
725
|
+
target_name,
|
|
726
|
+
allowed=_ALLOWED_PURGE_TARGET_NAMES,
|
|
727
|
+
)
|
|
728
|
+
_validate_governance_enum(
|
|
729
|
+
"phase",
|
|
730
|
+
phase,
|
|
731
|
+
allowed=_ALLOWED_PURGE_TARGET_PHASES,
|
|
732
|
+
)
|
|
733
|
+
_validate_governance_enum(
|
|
734
|
+
"status",
|
|
735
|
+
status,
|
|
736
|
+
allowed=_ALLOWED_PURGE_TARGET_STATUSES,
|
|
737
|
+
)
|
|
738
|
+
detail = _validate_governance_detail(
|
|
739
|
+
"detail",
|
|
740
|
+
detail,
|
|
741
|
+
allowed_keys=_ALLOWED_PURGE_TARGET_DETAIL_KEYS,
|
|
742
|
+
)
|
|
743
|
+
error_detail = _validate_governance_error_detail(error_detail)
|
|
744
|
+
target_ref = _validate_governance_target_ref(
|
|
745
|
+
target_name=target_name,
|
|
746
|
+
phase=phase,
|
|
747
|
+
target_ref=target_ref,
|
|
748
|
+
)
|
|
749
|
+
deleted_count = _validate_governance_deleted_count(deleted_count)
|
|
750
|
+
now = _epoch_now()
|
|
751
|
+
existing = self.conn.execute(
|
|
752
|
+
"""SELECT started_at, completed_at
|
|
753
|
+
FROM purge_operation_targets
|
|
754
|
+
WHERE org_id = ? AND purge_id = ? AND target_name = ? AND target_ref = ? AND phase = ?""",
|
|
755
|
+
(self.org_id, purge_id, target_name, target_ref, phase),
|
|
756
|
+
).fetchone()
|
|
757
|
+
started_at = existing["started_at"] if existing else None
|
|
758
|
+
completed_at = existing["completed_at"] if existing else None
|
|
759
|
+
if started_at is None and status in {"running", "failed", "complete"}:
|
|
760
|
+
started_at = now
|
|
761
|
+
if status in {"failed", "complete"}:
|
|
762
|
+
completed_at = now
|
|
763
|
+
self.conn.execute(
|
|
764
|
+
"""INSERT INTO purge_operation_targets (
|
|
765
|
+
org_id, purge_id, target_name, target_ref, phase, status, detail,
|
|
766
|
+
deleted_count, error_detail, started_at, completed_at
|
|
767
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
768
|
+
ON CONFLICT(org_id, purge_id, target_name, target_ref, phase) DO UPDATE SET
|
|
769
|
+
status = excluded.status,
|
|
770
|
+
detail = COALESCE(excluded.detail, purge_operation_targets.detail),
|
|
771
|
+
deleted_count = excluded.deleted_count,
|
|
772
|
+
error_detail = excluded.error_detail,
|
|
773
|
+
started_at = COALESCE(purge_operation_targets.started_at, excluded.started_at),
|
|
774
|
+
completed_at = excluded.completed_at""",
|
|
775
|
+
(
|
|
776
|
+
self.org_id,
|
|
777
|
+
purge_id,
|
|
778
|
+
target_name,
|
|
779
|
+
target_ref,
|
|
780
|
+
phase,
|
|
781
|
+
status,
|
|
782
|
+
_json_dumps(detail),
|
|
783
|
+
deleted_count,
|
|
784
|
+
error_detail,
|
|
785
|
+
started_at,
|
|
786
|
+
completed_at,
|
|
787
|
+
),
|
|
788
|
+
)
|
|
789
|
+
self.conn.execute(
|
|
790
|
+
"""UPDATE purge_operations
|
|
791
|
+
SET status = CASE
|
|
792
|
+
WHEN status IN ('complete', 'failed') THEN status
|
|
793
|
+
WHEN ? IN ('running', 'complete') THEN 'running'
|
|
794
|
+
ELSE status
|
|
795
|
+
END,
|
|
796
|
+
updated_at = ?
|
|
797
|
+
WHERE purge_id = ? AND org_id = ?""",
|
|
798
|
+
(status, now, purge_id, self.org_id),
|
|
799
|
+
)
|
|
800
|
+
|
|
801
|
+
def append_audit_event(self, event: AuditEvent) -> bool:
|
|
802
|
+
if _is_successful_erase_event(event):
|
|
803
|
+
raise ValueError(
|
|
804
|
+
"Successful ERASE audit rows may only be written by "
|
|
805
|
+
"complete_purge_operation_with_audit()"
|
|
806
|
+
)
|
|
807
|
+
if event.org_id != self.org_id:
|
|
808
|
+
raise ValueError("Audit event org_id must match storage org_id")
|
|
809
|
+
event = _canonicalize_audit_event_for_persistence(event)
|
|
810
|
+
with self._lock:
|
|
811
|
+
inserted = self._append_audit_event_with_cursor(self.conn, event)
|
|
812
|
+
self.conn.commit()
|
|
813
|
+
return inserted
|
|
814
|
+
|
|
815
|
+
def list_audit_events(
|
|
816
|
+
self, subject_ref: str | None = None, *, org_id: str | None = None
|
|
817
|
+
) -> list[AuditEvent]:
|
|
818
|
+
deps = self._deps()
|
|
819
|
+
if org_id is not None and org_id != self.org_id:
|
|
820
|
+
raise ValueError("Audit event org_id must match storage org_id")
|
|
821
|
+
sql = "SELECT * FROM audit_events WHERE org_id = ?"
|
|
822
|
+
params: list[Any] = [self.org_id]
|
|
823
|
+
if subject_ref is not None:
|
|
824
|
+
sql += " AND subject_ref = ?"
|
|
825
|
+
params.append(subject_ref)
|
|
826
|
+
sql += " ORDER BY created_at ASC, event_id ASC"
|
|
827
|
+
rows = deps._fetchall(sql, params)
|
|
828
|
+
return [_row_to_audit_event(row) for row in rows]
|
|
829
|
+
|
|
830
|
+
def _purge_governance_entity_content_locked(
|
|
831
|
+
self,
|
|
832
|
+
*,
|
|
833
|
+
entity_type: Literal["profile", "user_playbook"],
|
|
834
|
+
entity_id: str,
|
|
835
|
+
rowid: int,
|
|
836
|
+
) -> bool:
|
|
837
|
+
from ._lineage import _PURGE_SQL, _append_event_stmt
|
|
838
|
+
|
|
839
|
+
sql = _PURGE_SQL[entity_type]
|
|
840
|
+
cur = self.conn.execute(sql, (entity_id,))
|
|
841
|
+
if cur.rowcount <= 0:
|
|
842
|
+
return False
|
|
843
|
+
_append_event_stmt(
|
|
844
|
+
self.conn,
|
|
845
|
+
org_id=self.org_id,
|
|
846
|
+
entity_type=entity_type,
|
|
847
|
+
entity_id=entity_id,
|
|
848
|
+
op="purge",
|
|
849
|
+
prov="wasPurged",
|
|
850
|
+
source_ids=[],
|
|
851
|
+
actor="erasure",
|
|
852
|
+
request_id=f"purge_{entity_id}",
|
|
853
|
+
reason="content_purge",
|
|
854
|
+
)
|
|
855
|
+
if entity_type == "profile":
|
|
856
|
+
self.conn.execute(
|
|
857
|
+
"DELETE FROM profiles_fts WHERE profile_id = ?",
|
|
858
|
+
(entity_id,),
|
|
859
|
+
)
|
|
860
|
+
if self._deps()._has_sqlite_vec:
|
|
861
|
+
self.conn.execute(
|
|
862
|
+
"DELETE FROM profiles_vec WHERE rowid = ?",
|
|
863
|
+
(rowid,),
|
|
864
|
+
)
|
|
865
|
+
else:
|
|
866
|
+
self.conn.execute(
|
|
867
|
+
"DELETE FROM user_playbooks_fts WHERE rowid = ?",
|
|
868
|
+
(rowid,),
|
|
869
|
+
)
|
|
870
|
+
if self._deps()._has_sqlite_vec:
|
|
871
|
+
self.conn.execute(
|
|
872
|
+
"DELETE FROM user_playbooks_vec WHERE rowid = ?",
|
|
873
|
+
(rowid,),
|
|
874
|
+
)
|
|
875
|
+
return True
|
|
876
|
+
|
|
877
|
+
def _clear_user_data_for_governance_locked(
|
|
878
|
+
self,
|
|
879
|
+
user_id: str,
|
|
880
|
+
*,
|
|
881
|
+
expected_user_playbook_ids: set[int] | None = None,
|
|
882
|
+
) -> dict[str, int]:
|
|
883
|
+
deps = self._deps()
|
|
884
|
+
interaction_ids = [
|
|
885
|
+
int(row["interaction_id"])
|
|
886
|
+
for row in self.conn.execute(
|
|
887
|
+
"SELECT interaction_id FROM interactions WHERE user_id = ?",
|
|
888
|
+
(user_id,),
|
|
889
|
+
).fetchall()
|
|
890
|
+
]
|
|
891
|
+
raw_upb_ids = [
|
|
892
|
+
int(row["user_playbook_id"])
|
|
893
|
+
for row in self.conn.execute(
|
|
894
|
+
"SELECT user_playbook_id FROM user_playbooks WHERE user_id = ?",
|
|
895
|
+
(user_id,),
|
|
896
|
+
).fetchall()
|
|
897
|
+
]
|
|
898
|
+
if (
|
|
899
|
+
expected_user_playbook_ids is not None
|
|
900
|
+
and set(raw_upb_ids) != expected_user_playbook_ids
|
|
901
|
+
):
|
|
902
|
+
raise ValueError(
|
|
903
|
+
"Current user playbooks no longer match prepared purge snapshot"
|
|
904
|
+
)
|
|
905
|
+
request_ids = [
|
|
906
|
+
str(row["request_id"])
|
|
907
|
+
for row in self.conn.execute(
|
|
908
|
+
"SELECT request_id FROM requests WHERE user_id = ?",
|
|
909
|
+
(user_id,),
|
|
910
|
+
).fetchall()
|
|
911
|
+
]
|
|
912
|
+
profile_rows = self.conn.execute(
|
|
913
|
+
"SELECT rowid, profile_id FROM profiles WHERE user_id = ?",
|
|
914
|
+
(user_id,),
|
|
915
|
+
).fetchall()
|
|
916
|
+
profile_rowid_by_id = {
|
|
917
|
+
str(row["profile_id"]): int(row["rowid"]) for row in profile_rows
|
|
918
|
+
}
|
|
919
|
+
all_profile_ids = list(profile_rowid_by_id)
|
|
920
|
+
|
|
921
|
+
purge_profile_ids, delete_profile_ids = deps._partition_purge_vs_delete(
|
|
922
|
+
"profile",
|
|
923
|
+
all_profile_ids,
|
|
924
|
+
)
|
|
925
|
+
purge_upb_str_ids, delete_upb_str_ids = deps._partition_purge_vs_delete(
|
|
926
|
+
"user_playbook",
|
|
927
|
+
[str(user_playbook_id) for user_playbook_id in raw_upb_ids],
|
|
928
|
+
)
|
|
929
|
+
purge_upb_ids = [int(entity_id) for entity_id in purge_upb_str_ids]
|
|
930
|
+
delete_upb_ids = [int(entity_id) for entity_id in delete_upb_str_ids]
|
|
931
|
+
erased_entity_ids = [
|
|
932
|
+
*request_ids,
|
|
933
|
+
*[str(interaction_id) for interaction_id in interaction_ids],
|
|
934
|
+
*all_profile_ids,
|
|
935
|
+
*[str(user_playbook_id) for user_playbook_id in raw_upb_ids],
|
|
936
|
+
]
|
|
937
|
+
if erased_entity_ids:
|
|
938
|
+
erased_entity_id_set = set(erased_entity_ids)
|
|
939
|
+
lineage_source_event_ids: list[int] = []
|
|
940
|
+
for row in self.conn.execute(
|
|
941
|
+
"SELECT event_id, source_ids FROM lineage_event WHERE org_id = ?",
|
|
942
|
+
(self.org_id,),
|
|
943
|
+
).fetchall():
|
|
944
|
+
try:
|
|
945
|
+
source_ids = json.loads(str(row["source_ids"] or "[]"))
|
|
946
|
+
except json.JSONDecodeError:
|
|
947
|
+
source_ids = []
|
|
948
|
+
if any(
|
|
949
|
+
str(source_id) in erased_entity_id_set for source_id in source_ids
|
|
950
|
+
):
|
|
951
|
+
lineage_source_event_ids.append(int(row["event_id"]))
|
|
952
|
+
deps._delete_in_chunks(
|
|
953
|
+
"lineage_event", "event_id", lineage_source_event_ids
|
|
954
|
+
)
|
|
955
|
+
placeholders = ",".join("?" for _ in erased_entity_ids)
|
|
956
|
+
self.conn.execute(
|
|
957
|
+
f"""DELETE FROM lineage_event
|
|
958
|
+
WHERE org_id = ?
|
|
959
|
+
AND (
|
|
960
|
+
request_id IN ({placeholders})
|
|
961
|
+
OR entity_id IN ({placeholders})
|
|
962
|
+
)""", # noqa: S608
|
|
963
|
+
[self.org_id, *erased_entity_ids, *erased_entity_ids],
|
|
964
|
+
)
|
|
965
|
+
delete_profile_rowids = [
|
|
966
|
+
profile_rowid_by_id[profile_id]
|
|
967
|
+
for profile_id in delete_profile_ids
|
|
968
|
+
if profile_id in profile_rowid_by_id
|
|
969
|
+
]
|
|
970
|
+
|
|
971
|
+
deps._delete_in_chunks("interactions_fts", "rowid", interaction_ids)
|
|
972
|
+
deps._delete_in_chunks("user_playbooks_fts", "rowid", delete_upb_ids)
|
|
973
|
+
deps._delete_in_chunks("profiles_fts", "profile_id", delete_profile_ids)
|
|
974
|
+
if deps._has_sqlite_vec:
|
|
975
|
+
deps._delete_in_chunks("interactions_vec", "rowid", interaction_ids)
|
|
976
|
+
deps._delete_in_chunks("user_playbooks_vec", "rowid", delete_upb_ids)
|
|
977
|
+
deps._delete_in_chunks("profiles_vec", "rowid", delete_profile_rowids)
|
|
978
|
+
|
|
979
|
+
interactions_cur = self.conn.execute(
|
|
980
|
+
"DELETE FROM interactions WHERE user_id = ?",
|
|
981
|
+
(user_id,),
|
|
982
|
+
)
|
|
983
|
+
eval_results_cur = self.conn.execute(
|
|
984
|
+
"""DELETE FROM agent_success_evaluation_result
|
|
985
|
+
WHERE user_id = ?""",
|
|
986
|
+
(user_id,),
|
|
987
|
+
)
|
|
988
|
+
requests_cur = self.conn.execute(
|
|
989
|
+
"DELETE FROM requests WHERE user_id = ?",
|
|
990
|
+
(user_id,),
|
|
991
|
+
)
|
|
992
|
+
if raw_upb_ids:
|
|
993
|
+
deps._delete_source_windows_for_user_playbook_ids(raw_upb_ids)
|
|
994
|
+
if delete_upb_ids:
|
|
995
|
+
deps._delete_in_chunks("user_playbooks", "user_playbook_id", delete_upb_ids)
|
|
996
|
+
if delete_profile_ids:
|
|
997
|
+
deps._delete_in_chunks("profiles", "profile_id", delete_profile_ids)
|
|
998
|
+
|
|
999
|
+
purged_profiles = 0
|
|
1000
|
+
for profile_id in purge_profile_ids:
|
|
1001
|
+
rowid = profile_rowid_by_id.get(profile_id)
|
|
1002
|
+
if rowid is None:
|
|
1003
|
+
continue
|
|
1004
|
+
purged_profiles += int(
|
|
1005
|
+
self._purge_governance_entity_content_locked(
|
|
1006
|
+
entity_type="profile",
|
|
1007
|
+
entity_id=profile_id,
|
|
1008
|
+
rowid=rowid,
|
|
1009
|
+
)
|
|
1010
|
+
)
|
|
1011
|
+
|
|
1012
|
+
purged_user_playbooks = 0
|
|
1013
|
+
for user_playbook_id in purge_upb_ids:
|
|
1014
|
+
purged_user_playbooks += int(
|
|
1015
|
+
self._purge_governance_entity_content_locked(
|
|
1016
|
+
entity_type="user_playbook",
|
|
1017
|
+
entity_id=str(user_playbook_id),
|
|
1018
|
+
rowid=user_playbook_id,
|
|
1019
|
+
)
|
|
1020
|
+
)
|
|
1021
|
+
|
|
1022
|
+
return {
|
|
1023
|
+
"interactions": interactions_cur.rowcount,
|
|
1024
|
+
"user_playbooks": len(delete_upb_ids),
|
|
1025
|
+
"profiles": len(delete_profile_ids),
|
|
1026
|
+
"requests": requests_cur.rowcount,
|
|
1027
|
+
"agent_success_evaluation_results": eval_results_cur.rowcount,
|
|
1028
|
+
"purged_profiles": purged_profiles,
|
|
1029
|
+
"purged_user_playbooks": purged_user_playbooks,
|
|
1030
|
+
}
|
|
1031
|
+
|
|
1032
|
+
def begin_purge_operation(
|
|
1033
|
+
self,
|
|
1034
|
+
purge_id: str,
|
|
1035
|
+
idempotency_key: str,
|
|
1036
|
+
operation_type: Literal["user_erasure", "org_purge"],
|
|
1037
|
+
scope_type: Literal["user", "org"],
|
|
1038
|
+
subject_ref: str | None,
|
|
1039
|
+
request_ref: str,
|
|
1040
|
+
) -> PurgeOperation:
|
|
1041
|
+
_validate_governance_enum(
|
|
1042
|
+
"operation_type",
|
|
1043
|
+
operation_type,
|
|
1044
|
+
allowed=_ALLOWED_PURGE_OPERATION_TYPES,
|
|
1045
|
+
)
|
|
1046
|
+
_validate_governance_enum(
|
|
1047
|
+
"scope_type",
|
|
1048
|
+
scope_type,
|
|
1049
|
+
allowed=_ALLOWED_PURGE_SCOPE_TYPES,
|
|
1050
|
+
)
|
|
1051
|
+
_validate_governance_prefixed_ref(
|
|
1052
|
+
"subject_ref", subject_ref, prefix="subref_v1_"
|
|
1053
|
+
)
|
|
1054
|
+
_validate_governance_prefixed_ref(
|
|
1055
|
+
"request_ref", request_ref, prefix="reqref_v1_"
|
|
1056
|
+
)
|
|
1057
|
+
validated_purge_id = _validate_governance_purge_id("purge_id", purge_id)
|
|
1058
|
+
validated_idempotency_key = cast(
|
|
1059
|
+
str,
|
|
1060
|
+
_validate_governance_idempotency_key("idempotency_key", idempotency_key),
|
|
1061
|
+
)
|
|
1062
|
+
now = _epoch_now()
|
|
1063
|
+
with self._lock:
|
|
1064
|
+
existing = self.conn.execute(
|
|
1065
|
+
"""SELECT * FROM purge_operations
|
|
1066
|
+
WHERE org_id = ? AND idempotency_key = ?""",
|
|
1067
|
+
(self.org_id, validated_idempotency_key),
|
|
1068
|
+
).fetchone()
|
|
1069
|
+
if existing is not None:
|
|
1070
|
+
existing_operation = _row_to_purge_operation(existing)
|
|
1071
|
+
expected_identity = {
|
|
1072
|
+
"purge_id": validated_purge_id,
|
|
1073
|
+
"operation_type": operation_type,
|
|
1074
|
+
"scope_type": scope_type,
|
|
1075
|
+
"subject_ref": subject_ref,
|
|
1076
|
+
"request_ref": request_ref,
|
|
1077
|
+
}
|
|
1078
|
+
for field_name, expected_value in expected_identity.items():
|
|
1079
|
+
if getattr(existing_operation, field_name) != expected_value:
|
|
1080
|
+
raise ValueError(
|
|
1081
|
+
"Existing purge operation for idempotency_key has "
|
|
1082
|
+
f"mismatched {field_name}"
|
|
1083
|
+
)
|
|
1084
|
+
return _row_to_purge_operation(existing)
|
|
1085
|
+
self.conn.execute(
|
|
1086
|
+
"""INSERT INTO purge_operations (
|
|
1087
|
+
purge_id, org_id, operation_type, scope_type, subject_ref,
|
|
1088
|
+
request_ref, idempotency_key, status, created_at, updated_at
|
|
1089
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, 'pending', ?, ?)""",
|
|
1090
|
+
(
|
|
1091
|
+
validated_purge_id,
|
|
1092
|
+
self.org_id,
|
|
1093
|
+
operation_type,
|
|
1094
|
+
scope_type,
|
|
1095
|
+
subject_ref,
|
|
1096
|
+
request_ref,
|
|
1097
|
+
validated_idempotency_key,
|
|
1098
|
+
now,
|
|
1099
|
+
now,
|
|
1100
|
+
),
|
|
1101
|
+
)
|
|
1102
|
+
self.conn.commit()
|
|
1103
|
+
return self.get_purge_operation(validated_purge_id)
|
|
1104
|
+
|
|
1105
|
+
def record_purge_target(
|
|
1106
|
+
self,
|
|
1107
|
+
purge_id: str,
|
|
1108
|
+
target_name: str,
|
|
1109
|
+
phase: str,
|
|
1110
|
+
status: Literal["pending", "running", "failed", "complete"],
|
|
1111
|
+
target_ref: str = "",
|
|
1112
|
+
detail: dict[str, object] | None = None,
|
|
1113
|
+
deleted_count: int = 0,
|
|
1114
|
+
error_detail: str | None = None,
|
|
1115
|
+
) -> None:
|
|
1116
|
+
purge_id = _validate_governance_purge_id("purge_id", purge_id)
|
|
1117
|
+
_validate_governance_enum(
|
|
1118
|
+
"target_name",
|
|
1119
|
+
target_name,
|
|
1120
|
+
allowed=_ALLOWED_PURGE_TARGET_NAMES,
|
|
1121
|
+
)
|
|
1122
|
+
_validate_governance_enum(
|
|
1123
|
+
"phase",
|
|
1124
|
+
phase,
|
|
1125
|
+
allowed=_ALLOWED_PURGE_TARGET_PHASES,
|
|
1126
|
+
)
|
|
1127
|
+
_validate_governance_enum(
|
|
1128
|
+
"status",
|
|
1129
|
+
status,
|
|
1130
|
+
allowed=_ALLOWED_PURGE_TARGET_STATUSES,
|
|
1131
|
+
)
|
|
1132
|
+
with self._lock:
|
|
1133
|
+
self._record_purge_target_locked(
|
|
1134
|
+
purge_id=purge_id,
|
|
1135
|
+
target_name=target_name,
|
|
1136
|
+
target_ref=target_ref,
|
|
1137
|
+
phase=phase,
|
|
1138
|
+
status=status,
|
|
1139
|
+
detail=detail,
|
|
1140
|
+
deleted_count=deleted_count,
|
|
1141
|
+
error_detail=error_detail,
|
|
1142
|
+
)
|
|
1143
|
+
self.conn.commit()
|
|
1144
|
+
|
|
1145
|
+
def list_purge_targets(
|
|
1146
|
+
self, purge_id: str, phase: str | None = None
|
|
1147
|
+
) -> list[PurgeOperationTarget]:
|
|
1148
|
+
purge_id = _validate_governance_purge_id("purge_id", purge_id)
|
|
1149
|
+
deps = self._deps()
|
|
1150
|
+
sql = "SELECT * FROM purge_operation_targets WHERE org_id = ? AND purge_id = ?"
|
|
1151
|
+
params: list[Any] = [self.org_id, purge_id]
|
|
1152
|
+
if phase is not None:
|
|
1153
|
+
sql += " AND phase = ?"
|
|
1154
|
+
params.append(phase)
|
|
1155
|
+
sql += " ORDER BY phase ASC, target_name ASC, target_ref ASC"
|
|
1156
|
+
rows = deps._fetchall(sql, params)
|
|
1157
|
+
return [_row_to_purge_target(row) for row in rows]
|
|
1158
|
+
|
|
1159
|
+
def purge_targets_prepared(self, purge_id: str) -> bool:
|
|
1160
|
+
purge_id = _validate_governance_purge_id("purge_id", purge_id)
|
|
1161
|
+
row = self._deps()._fetchone(
|
|
1162
|
+
"""SELECT 1 FROM purge_operation_targets
|
|
1163
|
+
WHERE org_id = ? AND purge_id = ? AND target_name = ? AND target_ref = 'all'
|
|
1164
|
+
AND phase = ? AND status = 'complete'""",
|
|
1165
|
+
(self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE),
|
|
1166
|
+
)
|
|
1167
|
+
return row is not None
|
|
1168
|
+
|
|
1169
|
+
def prepare_governance_erase_targets(
|
|
1170
|
+
self,
|
|
1171
|
+
purge_id: str,
|
|
1172
|
+
user_id: str,
|
|
1173
|
+
owned_user_playbook_ids: set[int] | None = None,
|
|
1174
|
+
) -> None:
|
|
1175
|
+
purge_id = _validate_governance_purge_id("purge_id", purge_id)
|
|
1176
|
+
with self._lock:
|
|
1177
|
+
if self.purge_targets_prepared(purge_id):
|
|
1178
|
+
return
|
|
1179
|
+
try:
|
|
1180
|
+
self.conn.execute("BEGIN IMMEDIATE")
|
|
1181
|
+
owned_user_playbook_ids = (
|
|
1182
|
+
set(owned_user_playbook_ids)
|
|
1183
|
+
if owned_user_playbook_ids is not None
|
|
1184
|
+
else self._owned_user_playbook_ids_locked(user_id)
|
|
1185
|
+
)
|
|
1186
|
+
targets = self._planned_governance_delete_counts(
|
|
1187
|
+
user_id,
|
|
1188
|
+
owned_user_playbook_ids,
|
|
1189
|
+
)
|
|
1190
|
+
for target_name, count in targets.items():
|
|
1191
|
+
self._record_purge_target_locked(
|
|
1192
|
+
purge_id=purge_id,
|
|
1193
|
+
target_name=target_name,
|
|
1194
|
+
target_ref="all",
|
|
1195
|
+
phase="delete",
|
|
1196
|
+
status="pending",
|
|
1197
|
+
detail={"count": count},
|
|
1198
|
+
deleted_count=0,
|
|
1199
|
+
error_detail=None,
|
|
1200
|
+
)
|
|
1201
|
+
self._record_purge_target_locked(
|
|
1202
|
+
purge_id=purge_id,
|
|
1203
|
+
target_name=_SNAPSHOT_TARGET_NAME,
|
|
1204
|
+
target_ref="all",
|
|
1205
|
+
phase=_PREPARE_PHASE,
|
|
1206
|
+
status="complete",
|
|
1207
|
+
detail={
|
|
1208
|
+
"owned_user_playbook_ids": sorted(owned_user_playbook_ids),
|
|
1209
|
+
},
|
|
1210
|
+
deleted_count=0,
|
|
1211
|
+
error_detail=None,
|
|
1212
|
+
)
|
|
1213
|
+
self.conn.commit()
|
|
1214
|
+
except Exception:
|
|
1215
|
+
self.conn.rollback()
|
|
1216
|
+
raise
|
|
1217
|
+
|
|
1218
|
+
def hide_governance_agent_playbooks_for_rebuild(self, purge_id: str) -> list[int]:
|
|
1219
|
+
purge_id = _validate_governance_purge_id("purge_id", purge_id)
|
|
1220
|
+
with self._lock:
|
|
1221
|
+
try:
|
|
1222
|
+
self.conn.execute("BEGIN IMMEDIATE")
|
|
1223
|
+
target_rows = self.conn.execute(
|
|
1224
|
+
"""SELECT target_ref
|
|
1225
|
+
FROM purge_operation_targets
|
|
1226
|
+
WHERE org_id = ? AND purge_id = ?
|
|
1227
|
+
AND target_name = 'agent_playbook'
|
|
1228
|
+
AND phase = 'rebuild_without_erased_sources'
|
|
1229
|
+
AND target_ref != ''
|
|
1230
|
+
AND status != 'complete'
|
|
1231
|
+
ORDER BY CAST(target_ref AS INTEGER) ASC""",
|
|
1232
|
+
(self.org_id, purge_id),
|
|
1233
|
+
).fetchall()
|
|
1234
|
+
agent_playbook_ids = [int(row["target_ref"]) for row in target_rows]
|
|
1235
|
+
if not agent_playbook_ids:
|
|
1236
|
+
self.conn.commit()
|
|
1237
|
+
return []
|
|
1238
|
+
placeholders = ",".join("?" for _ in agent_playbook_ids)
|
|
1239
|
+
self.conn.execute(
|
|
1240
|
+
f"""UPDATE agent_playbooks
|
|
1241
|
+
SET status = ?
|
|
1242
|
+
WHERE agent_playbook_id IN ({placeholders})""",
|
|
1243
|
+
[Status.ARCHIVE_IN_PROGRESS.value, *agent_playbook_ids],
|
|
1244
|
+
)
|
|
1245
|
+
for agent_playbook_id in agent_playbook_ids:
|
|
1246
|
+
self._record_purge_target_locked(
|
|
1247
|
+
purge_id=purge_id,
|
|
1248
|
+
target_name="agent_playbook",
|
|
1249
|
+
target_ref=str(agent_playbook_id),
|
|
1250
|
+
phase="hide_for_rebuild",
|
|
1251
|
+
status="complete",
|
|
1252
|
+
detail=None,
|
|
1253
|
+
deleted_count=0,
|
|
1254
|
+
error_detail=None,
|
|
1255
|
+
)
|
|
1256
|
+
self._record_purge_target_locked(
|
|
1257
|
+
purge_id=purge_id,
|
|
1258
|
+
target_name="agent_playbook",
|
|
1259
|
+
target_ref=str(agent_playbook_id),
|
|
1260
|
+
phase="rebuild_without_erased_sources",
|
|
1261
|
+
status="running",
|
|
1262
|
+
detail=None,
|
|
1263
|
+
deleted_count=0,
|
|
1264
|
+
error_detail=None,
|
|
1265
|
+
)
|
|
1266
|
+
self.conn.commit()
|
|
1267
|
+
except Exception:
|
|
1268
|
+
self.conn.rollback()
|
|
1269
|
+
raise
|
|
1270
|
+
return agent_playbook_ids
|
|
1271
|
+
|
|
1272
|
+
def apply_governance_user_data_delete(
|
|
1273
|
+
self, purge_id: str, user_id: str
|
|
1274
|
+
) -> dict[str, int]:
|
|
1275
|
+
purge_id = _validate_governance_purge_id("purge_id", purge_id)
|
|
1276
|
+
name_map = {
|
|
1277
|
+
"interactions": "interaction",
|
|
1278
|
+
"user_playbooks": "user_playbook",
|
|
1279
|
+
"profiles": "profile",
|
|
1280
|
+
"requests": "request",
|
|
1281
|
+
"agent_success_evaluation_results": "agent_success_evaluation_result",
|
|
1282
|
+
"purged_profiles": "profile_purge",
|
|
1283
|
+
"purged_user_playbooks": "user_playbook_purge",
|
|
1284
|
+
}
|
|
1285
|
+
with self._lock:
|
|
1286
|
+
try:
|
|
1287
|
+
self.conn.execute("BEGIN")
|
|
1288
|
+
self._validate_prepared_delete_target_matrix_locked(purge_id)
|
|
1289
|
+
self._validate_hide_for_rebuild_targets_locked(purge_id)
|
|
1290
|
+
expected_user_playbook_ids = (
|
|
1291
|
+
self._prepared_owned_user_playbook_ids_locked(purge_id)
|
|
1292
|
+
)
|
|
1293
|
+
counts = self._clear_user_data_for_governance_locked(
|
|
1294
|
+
user_id,
|
|
1295
|
+
expected_user_playbook_ids=expected_user_playbook_ids,
|
|
1296
|
+
)
|
|
1297
|
+
for key, value in counts.items():
|
|
1298
|
+
self._record_purge_target_locked(
|
|
1299
|
+
purge_id=purge_id,
|
|
1300
|
+
target_name=name_map.get(key, key),
|
|
1301
|
+
target_ref="all",
|
|
1302
|
+
phase="delete",
|
|
1303
|
+
status="complete",
|
|
1304
|
+
detail={"count": int(value)},
|
|
1305
|
+
deleted_count=int(value),
|
|
1306
|
+
error_detail=None,
|
|
1307
|
+
)
|
|
1308
|
+
self.conn.commit()
|
|
1309
|
+
except Exception:
|
|
1310
|
+
self.conn.rollback()
|
|
1311
|
+
raise
|
|
1312
|
+
return counts
|
|
1313
|
+
|
|
1314
|
+
def apply_governance_agent_playbook_rebuild(
|
|
1315
|
+
self,
|
|
1316
|
+
purge_id: str,
|
|
1317
|
+
agent_playbook_id: int,
|
|
1318
|
+
remaining_source_windows: list[dict[str, object]],
|
|
1319
|
+
content: str | None,
|
|
1320
|
+
trigger: str | None,
|
|
1321
|
+
rationale: str | None,
|
|
1322
|
+
blocking_issue: dict[str, object] | None,
|
|
1323
|
+
expanded_terms: str | None,
|
|
1324
|
+
tags: list[str] | None,
|
|
1325
|
+
) -> None:
|
|
1326
|
+
purge_id = _validate_governance_purge_id("purge_id", purge_id)
|
|
1327
|
+
windows = _parse_governance_window_list(
|
|
1328
|
+
"remaining_source_windows", remaining_source_windows
|
|
1329
|
+
)
|
|
1330
|
+
canonical_remaining_windows = [window.model_dump() for window in windows]
|
|
1331
|
+
content_value = content or ""
|
|
1332
|
+
trigger_value = trigger or None
|
|
1333
|
+
embedding_text = trigger_value or content_value
|
|
1334
|
+
embedding = (
|
|
1335
|
+
self._deps()._get_embedding(embedding_text) if embedding_text else []
|
|
1336
|
+
)
|
|
1337
|
+
with self._lock:
|
|
1338
|
+
try:
|
|
1339
|
+
self.conn.execute("BEGIN")
|
|
1340
|
+
rebuild_target_row = self.conn.execute(
|
|
1341
|
+
"""SELECT status, detail
|
|
1342
|
+
FROM purge_operation_targets
|
|
1343
|
+
WHERE org_id = ? AND purge_id = ? AND target_name = 'agent_playbook'
|
|
1344
|
+
AND target_ref = ? AND phase = 'rebuild_without_erased_sources'""",
|
|
1345
|
+
(self.org_id, purge_id, str(agent_playbook_id)),
|
|
1346
|
+
).fetchone()
|
|
1347
|
+
if rebuild_target_row is None:
|
|
1348
|
+
raise ValueError("planned rebuild target does not exist")
|
|
1349
|
+
if rebuild_target_row["status"] == "complete":
|
|
1350
|
+
raise ValueError("planned rebuild target is already complete")
|
|
1351
|
+
rebuild_detail = _json_loads(rebuild_target_row["detail"])
|
|
1352
|
+
if not isinstance(rebuild_detail, dict) or not {
|
|
1353
|
+
"original_source_windows",
|
|
1354
|
+
"previous_lifecycle_status",
|
|
1355
|
+
"remaining_source_windows",
|
|
1356
|
+
}.issubset(rebuild_detail):
|
|
1357
|
+
raise ValueError(
|
|
1358
|
+
"planned rebuild target is missing source window detail"
|
|
1359
|
+
)
|
|
1360
|
+
planned_remaining_windows = _canonicalize_governance_windows(
|
|
1361
|
+
"planned remaining_source_windows",
|
|
1362
|
+
cast(
|
|
1363
|
+
list[dict[str, object]],
|
|
1364
|
+
rebuild_detail["remaining_source_windows"],
|
|
1365
|
+
),
|
|
1366
|
+
)
|
|
1367
|
+
if planned_remaining_windows != canonical_remaining_windows:
|
|
1368
|
+
raise ValueError(
|
|
1369
|
+
"remaining_source_windows must match the planned rebuild target"
|
|
1370
|
+
)
|
|
1371
|
+
previous_lifecycle_status = cast(
|
|
1372
|
+
str | None, rebuild_detail["previous_lifecycle_status"]
|
|
1373
|
+
)
|
|
1374
|
+
hide_target_row = self.conn.execute(
|
|
1375
|
+
"""SELECT status
|
|
1376
|
+
FROM purge_operation_targets
|
|
1377
|
+
WHERE org_id = ? AND purge_id = ? AND target_name = 'agent_playbook'
|
|
1378
|
+
AND target_ref = ? AND phase = 'hide_for_rebuild'""",
|
|
1379
|
+
(self.org_id, purge_id, str(agent_playbook_id)),
|
|
1380
|
+
).fetchone()
|
|
1381
|
+
if hide_target_row is None or hide_target_row["status"] != "complete":
|
|
1382
|
+
raise ValueError("hide_for_rebuild target must be complete")
|
|
1383
|
+
if windows:
|
|
1384
|
+
cur = self.conn.execute(
|
|
1385
|
+
"""UPDATE agent_playbooks
|
|
1386
|
+
SET content = ?, trigger = ?, rationale = ?, blocking_issue = ?,
|
|
1387
|
+
embedding = ?, expanded_terms = ?, tags = ?, status = ?
|
|
1388
|
+
WHERE agent_playbook_id = ?""",
|
|
1389
|
+
(
|
|
1390
|
+
content_value,
|
|
1391
|
+
trigger_value,
|
|
1392
|
+
rationale,
|
|
1393
|
+
json.dumps(blocking_issue)
|
|
1394
|
+
if blocking_issue is not None
|
|
1395
|
+
else None,
|
|
1396
|
+
_json_dumps(embedding),
|
|
1397
|
+
expanded_terms,
|
|
1398
|
+
_json_dumps(tags),
|
|
1399
|
+
previous_lifecycle_status,
|
|
1400
|
+
agent_playbook_id,
|
|
1401
|
+
),
|
|
1402
|
+
)
|
|
1403
|
+
if cur.rowcount == 0:
|
|
1404
|
+
raise ValueError(
|
|
1405
|
+
f"Agent playbook with ID {agent_playbook_id} not found"
|
|
1406
|
+
)
|
|
1407
|
+
self._replace_agent_playbook_source_windows_locked(
|
|
1408
|
+
agent_playbook_id, windows
|
|
1409
|
+
)
|
|
1410
|
+
self._upsert_agent_playbook_search_rows_locked(
|
|
1411
|
+
agent_playbook_id=agent_playbook_id,
|
|
1412
|
+
trigger=trigger_value,
|
|
1413
|
+
content=content_value,
|
|
1414
|
+
expanded_terms=expanded_terms,
|
|
1415
|
+
embedding=embedding,
|
|
1416
|
+
)
|
|
1417
|
+
else:
|
|
1418
|
+
from ._playbook import _emit_hard_delete_playbook
|
|
1419
|
+
|
|
1420
|
+
self._delete_agent_playbook_search_rows_locked(agent_playbook_id)
|
|
1421
|
+
self.conn.execute(
|
|
1422
|
+
"DELETE FROM agent_playbook_source_user_playbooks WHERE agent_playbook_id = ?",
|
|
1423
|
+
(agent_playbook_id,),
|
|
1424
|
+
)
|
|
1425
|
+
cur = self.conn.execute(
|
|
1426
|
+
"DELETE FROM agent_playbooks WHERE agent_playbook_id = ?",
|
|
1427
|
+
(agent_playbook_id,),
|
|
1428
|
+
)
|
|
1429
|
+
if cur.rowcount == 0:
|
|
1430
|
+
raise ValueError(
|
|
1431
|
+
f"Agent playbook with ID {agent_playbook_id} not found"
|
|
1432
|
+
)
|
|
1433
|
+
_emit_hard_delete_playbook(
|
|
1434
|
+
self.conn,
|
|
1435
|
+
org_id=self.org_id,
|
|
1436
|
+
entity_type="agent_playbook",
|
|
1437
|
+
entity_id=str(agent_playbook_id),
|
|
1438
|
+
request_id=purge_id,
|
|
1439
|
+
)
|
|
1440
|
+
self._record_purge_target_locked(
|
|
1441
|
+
purge_id=purge_id,
|
|
1442
|
+
target_name="agent_playbook",
|
|
1443
|
+
target_ref=str(agent_playbook_id),
|
|
1444
|
+
phase="rebuild_without_erased_sources",
|
|
1445
|
+
status="complete",
|
|
1446
|
+
detail=None,
|
|
1447
|
+
deleted_count=0,
|
|
1448
|
+
error_detail=None,
|
|
1449
|
+
)
|
|
1450
|
+
self.conn.commit()
|
|
1451
|
+
except Exception:
|
|
1452
|
+
self.conn.rollback()
|
|
1453
|
+
raise
|
|
1454
|
+
|
|
1455
|
+
def complete_purge_operation_with_audit(
|
|
1456
|
+
self, purge_id: str, audit_event: AuditEvent
|
|
1457
|
+
) -> PurgeOperation:
|
|
1458
|
+
purge_id = _validate_governance_purge_id("purge_id", purge_id)
|
|
1459
|
+
if audit_event.org_id != self.org_id:
|
|
1460
|
+
raise ValueError("Audit event org_id must match storage org_id")
|
|
1461
|
+
if audit_event.idempotency_key != purge_id:
|
|
1462
|
+
raise ValueError("Audit event idempotency key must match purge_id")
|
|
1463
|
+
if not _is_successful_erase_event(audit_event, purge_id=purge_id):
|
|
1464
|
+
raise ValueError(
|
|
1465
|
+
"Completion requires a successful ERASE audit event for this purge"
|
|
1466
|
+
)
|
|
1467
|
+
audit_event = _canonicalize_audit_event_for_persistence(audit_event)
|
|
1468
|
+
now = _epoch_now()
|
|
1469
|
+
with self._lock:
|
|
1470
|
+
try:
|
|
1471
|
+
self.conn.execute("BEGIN IMMEDIATE")
|
|
1472
|
+
row = self.conn.execute(
|
|
1473
|
+
"SELECT * FROM purge_operations WHERE purge_id = ? AND org_id = ?",
|
|
1474
|
+
(purge_id, self.org_id),
|
|
1475
|
+
).fetchone()
|
|
1476
|
+
if row is None:
|
|
1477
|
+
raise ValueError(f"Purge operation {purge_id!r} not found")
|
|
1478
|
+
purge_operation = _row_to_purge_operation(row)
|
|
1479
|
+
if purge_operation.subject_ref != audit_event.subject_ref:
|
|
1480
|
+
raise ValueError(
|
|
1481
|
+
"Audit event subject_ref must match purge operation subject_ref"
|
|
1482
|
+
)
|
|
1483
|
+
if purge_operation.request_ref != audit_event.request_ref:
|
|
1484
|
+
raise ValueError(
|
|
1485
|
+
"Audit event request_ref must match purge operation request_ref"
|
|
1486
|
+
)
|
|
1487
|
+
barrier_row = self.conn.execute(
|
|
1488
|
+
"""SELECT status FROM subject_write_barriers
|
|
1489
|
+
WHERE org_id = ? AND subject_ref = ? AND purge_id = ?""",
|
|
1490
|
+
(self.org_id, audit_event.subject_ref, purge_id),
|
|
1491
|
+
).fetchone()
|
|
1492
|
+
if barrier_row is None or barrier_row["status"] != "erasing":
|
|
1493
|
+
raise ValueError("subject erasure barrier is missing")
|
|
1494
|
+
snapshot = self.conn.execute(
|
|
1495
|
+
"""SELECT 1 FROM purge_operation_targets
|
|
1496
|
+
WHERE org_id = ? AND purge_id = ? AND target_name = ? AND target_ref = 'all'
|
|
1497
|
+
AND phase = ? AND status = 'complete'""",
|
|
1498
|
+
(self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE),
|
|
1499
|
+
).fetchone()
|
|
1500
|
+
if snapshot is None:
|
|
1501
|
+
raise ValueError(
|
|
1502
|
+
"Cannot complete purge without target snapshot marker"
|
|
1503
|
+
)
|
|
1504
|
+
delete_rows = self.conn.execute(
|
|
1505
|
+
"""SELECT target_name, status FROM purge_operation_targets
|
|
1506
|
+
WHERE org_id = ? AND purge_id = ? AND phase = 'delete'
|
|
1507
|
+
AND target_ref = 'all'""",
|
|
1508
|
+
(self.org_id, purge_id),
|
|
1509
|
+
).fetchall()
|
|
1510
|
+
delete_statuses = {
|
|
1511
|
+
str(row["target_name"]): str(row["status"]) for row in delete_rows
|
|
1512
|
+
}
|
|
1513
|
+
missing_delete_targets = [
|
|
1514
|
+
target_name
|
|
1515
|
+
for target_name in _CANONICAL_DELETE_TARGET_NAMES
|
|
1516
|
+
if delete_statuses.get(target_name) != "complete"
|
|
1517
|
+
]
|
|
1518
|
+
if missing_delete_targets:
|
|
1519
|
+
raise ValueError(
|
|
1520
|
+
"Cannot complete purge without complete delete target matrix: "
|
|
1521
|
+
+ ", ".join(missing_delete_targets)
|
|
1522
|
+
)
|
|
1523
|
+
incomplete = self.conn.execute(
|
|
1524
|
+
"""SELECT 1 FROM purge_operation_targets
|
|
1525
|
+
WHERE org_id = ? AND purge_id = ? AND status != 'complete'
|
|
1526
|
+
LIMIT 1""",
|
|
1527
|
+
(self.org_id, purge_id),
|
|
1528
|
+
).fetchone()
|
|
1529
|
+
if incomplete is not None:
|
|
1530
|
+
raise ValueError("Cannot complete purge with incomplete targets")
|
|
1531
|
+
existing_audit_row = self.conn.execute(
|
|
1532
|
+
"""SELECT * FROM audit_events
|
|
1533
|
+
WHERE org_id = ? AND idempotency_key = ?""",
|
|
1534
|
+
(self.org_id, purge_id),
|
|
1535
|
+
).fetchone()
|
|
1536
|
+
if existing_audit_row is not None:
|
|
1537
|
+
existing_event = _row_to_audit_event(existing_audit_row)
|
|
1538
|
+
if not _is_successful_erase_event(
|
|
1539
|
+
existing_event, purge_id=purge_id
|
|
1540
|
+
):
|
|
1541
|
+
raise ValueError(
|
|
1542
|
+
"Existing audit row for purge_id must be the matching "
|
|
1543
|
+
"successful ERASE row"
|
|
1544
|
+
)
|
|
1545
|
+
if _successful_erase_identity(
|
|
1546
|
+
existing_event
|
|
1547
|
+
) != _successful_erase_identity(audit_event):
|
|
1548
|
+
raise ValueError(
|
|
1549
|
+
"Existing audit row for purge_id must be the matching "
|
|
1550
|
+
"successful ERASE row"
|
|
1551
|
+
)
|
|
1552
|
+
else:
|
|
1553
|
+
self._append_audit_event_with_cursor(self.conn, audit_event)
|
|
1554
|
+
existing_audit_row = self.conn.execute(
|
|
1555
|
+
"""SELECT * FROM audit_events
|
|
1556
|
+
WHERE org_id = ? AND idempotency_key = ?""",
|
|
1557
|
+
(self.org_id, purge_id),
|
|
1558
|
+
).fetchone()
|
|
1559
|
+
if existing_audit_row is None:
|
|
1560
|
+
raise ValueError(
|
|
1561
|
+
"Completion requires exactly one successful ERASE audit row "
|
|
1562
|
+
"for the purge_id"
|
|
1563
|
+
)
|
|
1564
|
+
existing_event = _row_to_audit_event(existing_audit_row)
|
|
1565
|
+
if not _is_successful_erase_event(existing_event, purge_id=purge_id):
|
|
1566
|
+
raise ValueError(
|
|
1567
|
+
"Completion requires exactly one matching successful ERASE "
|
|
1568
|
+
"audit row for the purge_id"
|
|
1569
|
+
)
|
|
1570
|
+
if _successful_erase_identity(
|
|
1571
|
+
existing_event
|
|
1572
|
+
) != _successful_erase_identity(audit_event):
|
|
1573
|
+
raise ValueError(
|
|
1574
|
+
"Completion requires exactly one matching successful ERASE "
|
|
1575
|
+
"audit row for the purge_id"
|
|
1576
|
+
)
|
|
1577
|
+
self.conn.execute(
|
|
1578
|
+
"""UPDATE purge_operations
|
|
1579
|
+
SET status = 'complete',
|
|
1580
|
+
error_code = NULL,
|
|
1581
|
+
error_detail = NULL,
|
|
1582
|
+
updated_at = ?,
|
|
1583
|
+
completed_at = ?
|
|
1584
|
+
WHERE purge_id = ? AND org_id = ?""",
|
|
1585
|
+
(now, now, purge_id, self.org_id),
|
|
1586
|
+
)
|
|
1587
|
+
self.conn.commit()
|
|
1588
|
+
except Exception:
|
|
1589
|
+
self.conn.rollback()
|
|
1590
|
+
raise
|
|
1591
|
+
return self.get_purge_operation(purge_id)
|
|
1592
|
+
|
|
1593
|
+
def begin_subject_erasure_barrier(
|
|
1594
|
+
self, subject_ref: str, purge_id: str
|
|
1595
|
+
) -> SubjectWriteBarrier:
|
|
1596
|
+
_validate_governance_prefixed_ref(
|
|
1597
|
+
"subject_ref", subject_ref, prefix="subref_v1_"
|
|
1598
|
+
)
|
|
1599
|
+
validated_purge_id = _validate_governance_purge_id("purge_id", purge_id)
|
|
1600
|
+
now = _epoch_now()
|
|
1601
|
+
with self._lock:
|
|
1602
|
+
try:
|
|
1603
|
+
self.conn.execute("BEGIN IMMEDIATE")
|
|
1604
|
+
purge_row = self.conn.execute(
|
|
1605
|
+
"""SELECT * FROM purge_operations
|
|
1606
|
+
WHERE purge_id = ? AND org_id = ?""",
|
|
1607
|
+
(validated_purge_id, self.org_id),
|
|
1608
|
+
).fetchone()
|
|
1609
|
+
if purge_row is None:
|
|
1610
|
+
raise ValueError(
|
|
1611
|
+
f"Purge operation {validated_purge_id!r} not found"
|
|
1612
|
+
)
|
|
1613
|
+
purge_operation = _row_to_purge_operation(purge_row)
|
|
1614
|
+
if purge_operation.subject_ref != subject_ref:
|
|
1615
|
+
raise ValueError(
|
|
1616
|
+
"Purge operation subject_ref must match the barrier subject_ref"
|
|
1617
|
+
)
|
|
1618
|
+
existing_barrier = self.conn.execute(
|
|
1619
|
+
"""SELECT * FROM subject_write_barriers
|
|
1620
|
+
WHERE org_id = ? AND subject_ref = ?""",
|
|
1621
|
+
(self.org_id, subject_ref),
|
|
1622
|
+
).fetchone()
|
|
1623
|
+
if (
|
|
1624
|
+
existing_barrier is not None
|
|
1625
|
+
and str(existing_barrier["purge_id"]) != validated_purge_id
|
|
1626
|
+
):
|
|
1627
|
+
raise ValueError(
|
|
1628
|
+
"Existing barrier purge_id must match the requested purge_id"
|
|
1629
|
+
)
|
|
1630
|
+
if (
|
|
1631
|
+
existing_barrier is not None
|
|
1632
|
+
and str(existing_barrier["status"]) == "erased"
|
|
1633
|
+
):
|
|
1634
|
+
row = existing_barrier
|
|
1635
|
+
self.conn.commit()
|
|
1636
|
+
return _row_to_subject_write_barrier(row)
|
|
1637
|
+
self.conn.execute(
|
|
1638
|
+
"""INSERT INTO subject_write_barriers
|
|
1639
|
+
(org_id, subject_ref, purge_id, status, created_at, updated_at)
|
|
1640
|
+
VALUES (?, ?, ?, 'erasing', ?, ?)
|
|
1641
|
+
ON CONFLICT(org_id, subject_ref) DO UPDATE SET
|
|
1642
|
+
purge_id = excluded.purge_id,
|
|
1643
|
+
status = 'erasing',
|
|
1644
|
+
error_code = NULL,
|
|
1645
|
+
error_detail = NULL,
|
|
1646
|
+
updated_at = excluded.updated_at""",
|
|
1647
|
+
(self.org_id, subject_ref, validated_purge_id, now, now),
|
|
1648
|
+
)
|
|
1649
|
+
row = self.conn.execute(
|
|
1650
|
+
"""SELECT * FROM subject_write_barriers
|
|
1651
|
+
WHERE org_id = ? AND subject_ref = ?""",
|
|
1652
|
+
(self.org_id, subject_ref),
|
|
1653
|
+
).fetchone()
|
|
1654
|
+
self.conn.commit()
|
|
1655
|
+
except Exception:
|
|
1656
|
+
self.conn.rollback()
|
|
1657
|
+
raise
|
|
1658
|
+
if row is None:
|
|
1659
|
+
raise ValueError("subject erasure barrier insert failed")
|
|
1660
|
+
return _row_to_subject_write_barrier(row)
|
|
1661
|
+
|
|
1662
|
+
def assert_subject_writable(self, subject_ref: str) -> None:
|
|
1663
|
+
_validate_governance_prefixed_ref(
|
|
1664
|
+
"subject_ref", subject_ref, prefix="subref_v1_"
|
|
1665
|
+
)
|
|
1666
|
+
with self._lock:
|
|
1667
|
+
try:
|
|
1668
|
+
self.conn.execute("BEGIN IMMEDIATE")
|
|
1669
|
+
self._assert_subject_writable_locked(subject_ref)
|
|
1670
|
+
self.conn.commit()
|
|
1671
|
+
except Exception:
|
|
1672
|
+
self.conn.rollback()
|
|
1673
|
+
raise
|
|
1674
|
+
|
|
1675
|
+
def complete_subject_erasure_barrier_after_empty_check(
|
|
1676
|
+
self, purge_id: str, audit_event: AuditEvent
|
|
1677
|
+
) -> PurgeOperation:
|
|
1678
|
+
purge_id = _validate_governance_purge_id("purge_id", purge_id)
|
|
1679
|
+
if audit_event.org_id != self.org_id:
|
|
1680
|
+
raise ValueError("Audit event org_id must match storage org_id")
|
|
1681
|
+
if audit_event.idempotency_key != purge_id:
|
|
1682
|
+
raise ValueError("Audit event idempotency key must match purge_id")
|
|
1683
|
+
if not _is_successful_erase_event(audit_event, purge_id=purge_id):
|
|
1684
|
+
raise ValueError(
|
|
1685
|
+
"Completion requires a successful ERASE audit event for this purge"
|
|
1686
|
+
)
|
|
1687
|
+
audit_event = _canonicalize_audit_event_for_persistence(audit_event)
|
|
1688
|
+
now = _epoch_now()
|
|
1689
|
+
with self._lock:
|
|
1690
|
+
try:
|
|
1691
|
+
self.conn.execute("BEGIN IMMEDIATE")
|
|
1692
|
+
row = self.conn.execute(
|
|
1693
|
+
"SELECT * FROM purge_operations WHERE purge_id = ? AND org_id = ?",
|
|
1694
|
+
(purge_id, self.org_id),
|
|
1695
|
+
).fetchone()
|
|
1696
|
+
if row is None:
|
|
1697
|
+
raise ValueError(f"Purge operation {purge_id!r} not found")
|
|
1698
|
+
purge_operation = _row_to_purge_operation(row)
|
|
1699
|
+
if purge_operation.subject_ref != audit_event.subject_ref:
|
|
1700
|
+
raise ValueError(
|
|
1701
|
+
"Audit event subject_ref must match purge operation subject_ref"
|
|
1702
|
+
)
|
|
1703
|
+
if purge_operation.request_ref != audit_event.request_ref:
|
|
1704
|
+
raise ValueError(
|
|
1705
|
+
"Audit event request_ref must match purge operation request_ref"
|
|
1706
|
+
)
|
|
1707
|
+
snapshot = self.conn.execute(
|
|
1708
|
+
"""SELECT 1 FROM purge_operation_targets
|
|
1709
|
+
WHERE org_id = ? AND purge_id = ? AND target_name = ? AND target_ref = 'all'
|
|
1710
|
+
AND phase = ? AND status = 'complete'""",
|
|
1711
|
+
(self.org_id, purge_id, _SNAPSHOT_TARGET_NAME, _PREPARE_PHASE),
|
|
1712
|
+
).fetchone()
|
|
1713
|
+
if snapshot is None:
|
|
1714
|
+
raise ValueError(
|
|
1715
|
+
"Cannot complete purge without target snapshot marker"
|
|
1716
|
+
)
|
|
1717
|
+
if self._same_subject_rows_remain_locked(audit_event.subject_ref or ""):
|
|
1718
|
+
raise ValueError("same-subject rows remain")
|
|
1719
|
+
delete_rows = self.conn.execute(
|
|
1720
|
+
"""SELECT target_name, status FROM purge_operation_targets
|
|
1721
|
+
WHERE org_id = ? AND purge_id = ? AND phase = 'delete'
|
|
1722
|
+
AND target_ref = 'all'""",
|
|
1723
|
+
(self.org_id, purge_id),
|
|
1724
|
+
).fetchall()
|
|
1725
|
+
delete_statuses = {
|
|
1726
|
+
str(target_row["target_name"]): str(target_row["status"])
|
|
1727
|
+
for target_row in delete_rows
|
|
1728
|
+
}
|
|
1729
|
+
missing_delete_targets = [
|
|
1730
|
+
target_name
|
|
1731
|
+
for target_name in _CANONICAL_DELETE_TARGET_NAMES
|
|
1732
|
+
if delete_statuses.get(target_name) != "complete"
|
|
1733
|
+
]
|
|
1734
|
+
if missing_delete_targets:
|
|
1735
|
+
raise ValueError(
|
|
1736
|
+
"Cannot complete purge without complete delete target matrix: "
|
|
1737
|
+
+ ", ".join(missing_delete_targets)
|
|
1738
|
+
)
|
|
1739
|
+
incomplete = self.conn.execute(
|
|
1740
|
+
"""SELECT 1 FROM purge_operation_targets
|
|
1741
|
+
WHERE org_id = ? AND purge_id = ? AND status != 'complete'
|
|
1742
|
+
LIMIT 1""",
|
|
1743
|
+
(self.org_id, purge_id),
|
|
1744
|
+
).fetchone()
|
|
1745
|
+
if incomplete is not None:
|
|
1746
|
+
raise ValueError("Cannot complete purge with incomplete targets")
|
|
1747
|
+
existing_audit_row = self.conn.execute(
|
|
1748
|
+
"""SELECT * FROM audit_events
|
|
1749
|
+
WHERE org_id = ? AND idempotency_key = ?""",
|
|
1750
|
+
(self.org_id, purge_id),
|
|
1751
|
+
).fetchone()
|
|
1752
|
+
if existing_audit_row is not None:
|
|
1753
|
+
existing_event = _row_to_audit_event(existing_audit_row)
|
|
1754
|
+
if not _is_successful_erase_event(
|
|
1755
|
+
existing_event, purge_id=purge_id
|
|
1756
|
+
):
|
|
1757
|
+
raise ValueError(
|
|
1758
|
+
"Existing audit row for purge_id must be the matching "
|
|
1759
|
+
"successful ERASE row"
|
|
1760
|
+
)
|
|
1761
|
+
if _successful_erase_identity(
|
|
1762
|
+
existing_event
|
|
1763
|
+
) != _successful_erase_identity(audit_event):
|
|
1764
|
+
raise ValueError(
|
|
1765
|
+
"Existing audit row for purge_id must be the matching "
|
|
1766
|
+
"successful ERASE row"
|
|
1767
|
+
)
|
|
1768
|
+
else:
|
|
1769
|
+
self._append_audit_event_with_cursor(self.conn, audit_event)
|
|
1770
|
+
existing_audit_row = self.conn.execute(
|
|
1771
|
+
"""SELECT * FROM audit_events
|
|
1772
|
+
WHERE org_id = ? AND idempotency_key = ?""",
|
|
1773
|
+
(self.org_id, purge_id),
|
|
1774
|
+
).fetchone()
|
|
1775
|
+
if existing_audit_row is None:
|
|
1776
|
+
raise ValueError(
|
|
1777
|
+
"Completion requires exactly one successful ERASE audit row "
|
|
1778
|
+
"for the purge_id"
|
|
1779
|
+
)
|
|
1780
|
+
existing_event = _row_to_audit_event(existing_audit_row)
|
|
1781
|
+
if not _is_successful_erase_event(existing_event, purge_id=purge_id):
|
|
1782
|
+
raise ValueError(
|
|
1783
|
+
"Completion requires exactly one matching successful ERASE "
|
|
1784
|
+
"audit row for the purge_id"
|
|
1785
|
+
)
|
|
1786
|
+
if _successful_erase_identity(
|
|
1787
|
+
existing_event
|
|
1788
|
+
) != _successful_erase_identity(audit_event):
|
|
1789
|
+
raise ValueError(
|
|
1790
|
+
"Completion requires exactly one matching successful ERASE "
|
|
1791
|
+
"audit row for the purge_id"
|
|
1792
|
+
)
|
|
1793
|
+
barrier_update = self.conn.execute(
|
|
1794
|
+
"""UPDATE subject_write_barriers
|
|
1795
|
+
SET status = 'erased', error_code = NULL, error_detail = NULL, updated_at = ?
|
|
1796
|
+
WHERE org_id = ? AND subject_ref = ? AND purge_id = ? AND status = 'erasing'""",
|
|
1797
|
+
(now, self.org_id, audit_event.subject_ref, purge_id),
|
|
1798
|
+
)
|
|
1799
|
+
if barrier_update.rowcount != 1:
|
|
1800
|
+
raise ValueError("subject erasure barrier is missing")
|
|
1801
|
+
self.conn.execute(
|
|
1802
|
+
"""UPDATE purge_operations
|
|
1803
|
+
SET status = 'complete',
|
|
1804
|
+
error_code = NULL,
|
|
1805
|
+
error_detail = NULL,
|
|
1806
|
+
updated_at = ?,
|
|
1807
|
+
completed_at = ?
|
|
1808
|
+
WHERE purge_id = ? AND org_id = ?""",
|
|
1809
|
+
(now, now, purge_id, self.org_id),
|
|
1810
|
+
)
|
|
1811
|
+
self.conn.commit()
|
|
1812
|
+
except Exception:
|
|
1813
|
+
self.conn.rollback()
|
|
1814
|
+
raise
|
|
1815
|
+
return self.get_purge_operation(purge_id)
|
|
1816
|
+
|
|
1817
|
+
def fail_subject_erasure_barrier(
|
|
1818
|
+
self,
|
|
1819
|
+
subject_ref: str,
|
|
1820
|
+
purge_id: str,
|
|
1821
|
+
error_code: str,
|
|
1822
|
+
error_detail: str,
|
|
1823
|
+
) -> SubjectWriteBarrier:
|
|
1824
|
+
_validate_governance_prefixed_ref(
|
|
1825
|
+
"subject_ref", subject_ref, prefix="subref_v1_"
|
|
1826
|
+
)
|
|
1827
|
+
validated_purge_id = _validate_governance_purge_id("purge_id", purge_id)
|
|
1828
|
+
validated_error_code = _validate_governance_error_code(error_code)
|
|
1829
|
+
validated_error_detail = _validate_governance_error_detail(error_detail)
|
|
1830
|
+
now = _epoch_now()
|
|
1831
|
+
with self._lock:
|
|
1832
|
+
try:
|
|
1833
|
+
self.conn.execute("BEGIN IMMEDIATE")
|
|
1834
|
+
update_cursor = self.conn.execute(
|
|
1835
|
+
"""UPDATE subject_write_barriers
|
|
1836
|
+
SET status = 'failed',
|
|
1837
|
+
error_code = ?,
|
|
1838
|
+
error_detail = ?,
|
|
1839
|
+
updated_at = ?
|
|
1840
|
+
WHERE org_id = ? AND subject_ref = ? AND purge_id = ?
|
|
1841
|
+
AND status = 'erasing'""",
|
|
1842
|
+
(
|
|
1843
|
+
validated_error_code,
|
|
1844
|
+
validated_error_detail,
|
|
1845
|
+
now,
|
|
1846
|
+
self.org_id,
|
|
1847
|
+
subject_ref,
|
|
1848
|
+
validated_purge_id,
|
|
1849
|
+
),
|
|
1850
|
+
)
|
|
1851
|
+
if update_cursor.rowcount != 1:
|
|
1852
|
+
raise ValueError(
|
|
1853
|
+
"subject erasure barrier failure requires a matching barrier"
|
|
1854
|
+
)
|
|
1855
|
+
purge_row = self.conn.execute(
|
|
1856
|
+
"SELECT 1 FROM purge_operations WHERE purge_id = ? AND org_id = ?",
|
|
1857
|
+
(validated_purge_id, self.org_id),
|
|
1858
|
+
).fetchone()
|
|
1859
|
+
if purge_row is not None:
|
|
1860
|
+
self.conn.execute(
|
|
1861
|
+
"""UPDATE purge_operations
|
|
1862
|
+
SET status = 'failed', error_code = ?, error_detail = ?,
|
|
1863
|
+
updated_at = ?, completed_at = ?
|
|
1864
|
+
WHERE purge_id = ? AND org_id = ?""",
|
|
1865
|
+
(
|
|
1866
|
+
validated_error_code,
|
|
1867
|
+
validated_error_detail,
|
|
1868
|
+
now,
|
|
1869
|
+
now,
|
|
1870
|
+
validated_purge_id,
|
|
1871
|
+
self.org_id,
|
|
1872
|
+
),
|
|
1873
|
+
)
|
|
1874
|
+
row = self.conn.execute(
|
|
1875
|
+
"""SELECT * FROM subject_write_barriers
|
|
1876
|
+
WHERE org_id = ? AND subject_ref = ? AND purge_id = ?""",
|
|
1877
|
+
(self.org_id, subject_ref, validated_purge_id),
|
|
1878
|
+
).fetchone()
|
|
1879
|
+
self.conn.commit()
|
|
1880
|
+
except Exception:
|
|
1881
|
+
self.conn.rollback()
|
|
1882
|
+
raise
|
|
1883
|
+
if row is None:
|
|
1884
|
+
raise ValueError("subject erasure barrier update failed")
|
|
1885
|
+
return _row_to_subject_write_barrier(row)
|
|
1886
|
+
|
|
1887
|
+
def get_subject_write_barrier(self, subject_ref: str) -> SubjectWriteBarrier | None:
|
|
1888
|
+
_validate_governance_prefixed_ref(
|
|
1889
|
+
"subject_ref", subject_ref, prefix="subref_v1_"
|
|
1890
|
+
)
|
|
1891
|
+
row = self.conn.execute(
|
|
1892
|
+
"""SELECT * FROM subject_write_barriers
|
|
1893
|
+
WHERE org_id = ? AND subject_ref = ?""",
|
|
1894
|
+
(self.org_id, subject_ref),
|
|
1895
|
+
).fetchone()
|
|
1896
|
+
if row is None:
|
|
1897
|
+
return None
|
|
1898
|
+
return _row_to_subject_write_barrier(row)
|
|
1899
|
+
|
|
1900
|
+
def fail_purge_operation(
|
|
1901
|
+
self, purge_id: str, error_code: str, error_detail: str
|
|
1902
|
+
) -> PurgeOperation:
|
|
1903
|
+
purge_id = _validate_governance_purge_id("purge_id", purge_id)
|
|
1904
|
+
validated_error_code = _validate_governance_error_code(error_code)
|
|
1905
|
+
validated_error_detail = _validate_governance_error_detail(error_detail)
|
|
1906
|
+
now = _epoch_now()
|
|
1907
|
+
with self._lock:
|
|
1908
|
+
cur = self.conn.execute(
|
|
1909
|
+
"""UPDATE purge_operations
|
|
1910
|
+
SET status = 'failed', error_code = ?, error_detail = ?,
|
|
1911
|
+
updated_at = ?, completed_at = ?
|
|
1912
|
+
WHERE purge_id = ? AND org_id = ? AND status != 'complete'""",
|
|
1913
|
+
(
|
|
1914
|
+
validated_error_code,
|
|
1915
|
+
validated_error_detail,
|
|
1916
|
+
now,
|
|
1917
|
+
now,
|
|
1918
|
+
purge_id,
|
|
1919
|
+
self.org_id,
|
|
1920
|
+
),
|
|
1921
|
+
)
|
|
1922
|
+
if cur.rowcount == 0:
|
|
1923
|
+
existing = self.conn.execute(
|
|
1924
|
+
"SELECT status FROM purge_operations WHERE purge_id = ? AND org_id = ?",
|
|
1925
|
+
(purge_id, self.org_id),
|
|
1926
|
+
).fetchone()
|
|
1927
|
+
if existing is not None and str(existing["status"]) == "complete":
|
|
1928
|
+
raise ValueError("Purge operation is already complete")
|
|
1929
|
+
raise ValueError(f"Purge operation {purge_id!r} not found")
|
|
1930
|
+
self.conn.commit()
|
|
1931
|
+
return self.get_purge_operation(purge_id)
|
|
1932
|
+
|
|
1933
|
+
def get_purge_operation(self, purge_id: str) -> PurgeOperation:
|
|
1934
|
+
purge_id = _validate_governance_purge_id("purge_id", purge_id)
|
|
1935
|
+
row = self._deps()._fetchone(
|
|
1936
|
+
"SELECT * FROM purge_operations WHERE purge_id = ? AND org_id = ?",
|
|
1937
|
+
(purge_id, self.org_id),
|
|
1938
|
+
)
|
|
1939
|
+
if row is None:
|
|
1940
|
+
raise ValueError(f"Purge operation {purge_id!r} not found")
|
|
1941
|
+
return _row_to_purge_operation(row)
|
|
1942
|
+
|
|
1943
|
+
def gc_governance_retention(self, *, config: GovernanceRetentionConfig) -> int:
|
|
1944
|
+
if not config.audit_events_retention_enabled:
|
|
1945
|
+
return 0
|
|
1946
|
+
cutoff_epoch = _epoch_now() - config.audit_events_retention_days * 24 * 60 * 60
|
|
1947
|
+
with self._lock:
|
|
1948
|
+
cur = self.conn.execute(
|
|
1949
|
+
"""DELETE FROM audit_events
|
|
1950
|
+
WHERE event_id IN (
|
|
1951
|
+
SELECT event_id
|
|
1952
|
+
FROM audit_events
|
|
1953
|
+
WHERE org_id = ? AND created_at < ?
|
|
1954
|
+
ORDER BY created_at ASC, event_id ASC
|
|
1955
|
+
LIMIT ?
|
|
1956
|
+
)""",
|
|
1957
|
+
(
|
|
1958
|
+
self.org_id,
|
|
1959
|
+
cutoff_epoch,
|
|
1960
|
+
config.audit_events_delete_batch_limit,
|
|
1961
|
+
),
|
|
1962
|
+
)
|
|
1963
|
+
deleted = int(cur.rowcount or 0)
|
|
1964
|
+
self.conn.commit()
|
|
1965
|
+
return deleted
|