claude-smart 0.2.42 → 0.2.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +3 -3
- package/README.md +1 -1
- package/bin/claude-smart.js +2 -2
- package/package.json +9 -3
- package/plugin/.claude-plugin/plugin.json +9 -3
- package/plugin/.codex-plugin/plugin.json +1 -1
- package/plugin/README.md +23 -3
- package/plugin/pyproject.toml +3 -3
- package/plugin/scripts/_lib.sh +91 -0
- package/plugin/scripts/backend-service.sh +51 -4
- package/plugin/scripts/cli.sh +3 -1
- package/plugin/scripts/codex-hook.js +72 -4
- package/plugin/scripts/dashboard-build.sh +1 -0
- package/plugin/scripts/dashboard-service.sh +1 -0
- package/plugin/scripts/ensure-plugin-root.sh +1 -0
- package/plugin/scripts/hook_entry.sh +6 -3
- package/plugin/scripts/smart-install.sh +3 -2
- package/plugin/src/README.md +57 -0
- package/plugin/src/claude_smart/context_format.py +11 -12
- package/plugin/src/claude_smart/cs_cite.py +26 -12
- package/plugin/src/claude_smart/ids.py +13 -5
- package/plugin/uv.lock +126 -5
- package/plugin/vendor/reflexio/.env.example +62 -0
- package/plugin/vendor/reflexio/LICENSE +201 -0
- package/plugin/vendor/reflexio/README.md +338 -0
- package/plugin/vendor/reflexio/pyproject.toml +274 -0
- package/plugin/vendor/reflexio/reflexio/README.md +184 -0
- package/plugin/vendor/reflexio/reflexio/__init__.py +166 -0
- package/plugin/vendor/reflexio/reflexio/benchmarks/__init__.py +1 -0
- package/plugin/vendor/reflexio/reflexio/benchmarks/retrieval_latency/README.md +109 -0
- package/plugin/vendor/reflexio/reflexio/benchmarks/retrieval_latency/__init__.py +1 -0
- package/plugin/vendor/reflexio/reflexio/benchmarks/retrieval_latency/backends.py +175 -0
- package/plugin/vendor/reflexio/reflexio/benchmarks/retrieval_latency/bench.py +642 -0
- package/plugin/vendor/reflexio/reflexio/benchmarks/retrieval_latency/embed_cache.py +330 -0
- package/plugin/vendor/reflexio/reflexio/benchmarks/retrieval_latency/report.py +317 -0
- package/plugin/vendor/reflexio/reflexio/benchmarks/retrieval_latency/results/report.md +43 -0
- package/plugin/vendor/reflexio/reflexio/benchmarks/retrieval_latency/results/results.json +4478 -0
- package/plugin/vendor/reflexio/reflexio/benchmarks/retrieval_latency/scenarios.py +134 -0
- package/plugin/vendor/reflexio/reflexio/benchmarks/retrieval_latency/seed.py +255 -0
- package/plugin/vendor/reflexio/reflexio/cli/README.md +287 -0
- package/plugin/vendor/reflexio/reflexio/cli/__init__.py +0 -0
- package/plugin/vendor/reflexio/reflexio/cli/__main__.py +56 -0
- package/plugin/vendor/reflexio/reflexio/cli/_client.py +86 -0
- package/plugin/vendor/reflexio/reflexio/cli/app.py +127 -0
- package/plugin/vendor/reflexio/reflexio/cli/bootstrap_config.py +265 -0
- package/plugin/vendor/reflexio/reflexio/cli/codex_auth.py +503 -0
- package/plugin/vendor/reflexio/reflexio/cli/commands/__init__.py +0 -0
- package/plugin/vendor/reflexio/reflexio/cli/commands/admin_cmd.py +65 -0
- package/plugin/vendor/reflexio/reflexio/cli/commands/agent_playbooks.py +503 -0
- package/plugin/vendor/reflexio/reflexio/cli/commands/api.py +114 -0
- package/plugin/vendor/reflexio/reflexio/cli/commands/auth.py +109 -0
- package/plugin/vendor/reflexio/reflexio/cli/commands/config_cmd.py +511 -0
- package/plugin/vendor/reflexio/reflexio/cli/commands/doctor.py +127 -0
- package/plugin/vendor/reflexio/reflexio/cli/commands/embeddings.py +53 -0
- package/plugin/vendor/reflexio/reflexio/cli/commands/interactions.py +478 -0
- package/plugin/vendor/reflexio/reflexio/cli/commands/profiles.py +303 -0
- package/plugin/vendor/reflexio/reflexio/cli/commands/services.py +289 -0
- package/plugin/vendor/reflexio/reflexio/cli/commands/setup_cmd.py +964 -0
- package/plugin/vendor/reflexio/reflexio/cli/commands/shortcuts.py +285 -0
- package/plugin/vendor/reflexio/reflexio/cli/commands/status_cmd.py +143 -0
- package/plugin/vendor/reflexio/reflexio/cli/commands/user_playbooks.py +373 -0
- package/plugin/vendor/reflexio/reflexio/cli/env_loader.py +284 -0
- package/plugin/vendor/reflexio/reflexio/cli/errors.py +217 -0
- package/plugin/vendor/reflexio/reflexio/cli/log_format.py +247 -0
- package/plugin/vendor/reflexio/reflexio/cli/output.py +867 -0
- package/plugin/vendor/reflexio/reflexio/cli/paths.py +41 -0
- package/plugin/vendor/reflexio/reflexio/cli/run_services.py +391 -0
- package/plugin/vendor/reflexio/reflexio/cli/state.py +204 -0
- package/plugin/vendor/reflexio/reflexio/cli/stop_services.py +96 -0
- package/plugin/vendor/reflexio/reflexio/cli/utils.py +329 -0
- package/plugin/vendor/reflexio/reflexio/client/__init__.py +3 -0
- package/plugin/vendor/reflexio/reflexio/client/cache.py +150 -0
- package/plugin/vendor/reflexio/reflexio/client/client.py +2613 -0
- package/plugin/vendor/reflexio/reflexio/defaults.py +23 -0
- package/plugin/vendor/reflexio/reflexio/integrations/__init__.py +0 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/.clawhubignore +7 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/README.md +274 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/TESTING.md +517 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/hook/handler.js +473 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/package-lock.json +2156 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/package.json +18 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/hook/handler.ts +241 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/hook/setup.ts +140 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/index.ts +130 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/lib/publish.ts +113 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/lib/search.ts +52 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/lib/server.ts +103 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/lib/sqlite-buffer.ts +156 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/lib/user-id.ts +134 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/openclaw.plugin.json +41 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/package.json +17 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/rules/reflexio.md +24 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/skills/reflexio/SKILL.md +48 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/publish_clawhub.sh +278 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/references/HOOK.md +164 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/scripts/install.sh +36 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/scripts/uninstall.sh +35 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/tests/publish.test.ts +27 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/tests/search.test.ts +31 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/tests/server.test.ts +42 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/tests/setup.test.ts +49 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/tests/sqlite-buffer.test.ts +91 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/tests/user-id.test.ts +50 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/tsconfig.json +16 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/types/openclaw.d.ts +230 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw/vitest.config.ts +13 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/README.md +120 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/TESTING.md +168 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/package-lock.json +1657 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/package.json +16 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/HEARTBEAT.md +6 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/README.md +84 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/SKILL.md +194 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/_meta.json +6 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/agents/reflexio-extractor.md +45 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/hook/handler.ts +214 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/hook/setup.ts +55 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/index.ts +327 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/lib/consolidate.ts +233 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/lib/dedup.ts +80 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/lib/io.ts +155 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/lib/openclaw-cli.ts +67 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/lib/search.ts +33 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/lib/write-playbook.ts +76 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/lib/write-profile.ts +79 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/openclaw.plugin.json +46 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/package.json +18 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/prompts/README.md +36 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/prompts/full_consolidation.md +56 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/prompts/playbook_extraction.md +217 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/prompts/profile_extraction.md +132 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/skills/reflexio-consolidate/SKILL.md +33 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/plugin/skills/reflexio-embedded/SKILL.md +194 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/references/HOOK.md +18 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/references/architecture.md +49 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/references/comparison.md +31 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/references/future-work.md +47 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/references/porting-notes.md +52 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/scripts/install.sh +52 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/scripts/uninstall.sh +36 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/tests/consolidate.test.ts +135 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/tests/dedup.test.ts +104 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/tests/io.test.ts +175 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/tests/search.test.ts +66 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/tests/smoke-test.ts +140 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/tests/write-playbook.test.ts +93 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/tests/write-profile.test.ts +174 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/tsconfig.json +16 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/types/openclaw.d.ts +230 -0
- package/plugin/vendor/reflexio/reflexio/integrations/openclaw-embedded/vitest.config.ts +7 -0
- package/plugin/vendor/reflexio/reflexio/lib/__init__.py +23 -0
- package/plugin/vendor/reflexio/reflexio/lib/_agent_playbook.py +310 -0
- package/plugin/vendor/reflexio/reflexio/lib/_base.py +225 -0
- package/plugin/vendor/reflexio/reflexio/lib/_config.py +83 -0
- package/plugin/vendor/reflexio/reflexio/lib/_dashboard.py +266 -0
- package/plugin/vendor/reflexio/reflexio/lib/_generation.py +176 -0
- package/plugin/vendor/reflexio/reflexio/lib/_interactions.py +334 -0
- package/plugin/vendor/reflexio/reflexio/lib/_operations.py +153 -0
- package/plugin/vendor/reflexio/reflexio/lib/_profiles.py +545 -0
- package/plugin/vendor/reflexio/reflexio/lib/_reflection.py +52 -0
- package/plugin/vendor/reflexio/reflexio/lib/_search.py +167 -0
- package/plugin/vendor/reflexio/reflexio/lib/_storage_labels.py +103 -0
- package/plugin/vendor/reflexio/reflexio/lib/_user_playbook.py +288 -0
- package/plugin/vendor/reflexio/reflexio/lib/reflexio_lib.py +27 -0
- package/plugin/vendor/reflexio/reflexio/models/__init__.py +0 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/__init__.py +0 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/braintrust_schema.py +141 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/common.py +41 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/__init__.py +3 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/entities.py +1112 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/enums.py +63 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/eval_overview_schema.py +487 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/internal_schema.py +28 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/pending_tool_call_schema.py +83 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/retriever_schema.py +768 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/service_schemas.py +9 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/stall_state_schema.py +32 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/__init__.py +3 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/converters.py +177 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/entities.py +129 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/enums.py +25 -0
- package/plugin/vendor/reflexio/reflexio/models/api_schema/validators.py +333 -0
- package/plugin/vendor/reflexio/reflexio/models/config_schema.py +908 -0
- package/plugin/vendor/reflexio/reflexio/models/py.typed +0 -0
- package/plugin/vendor/reflexio/reflexio/server/OVERVIEW.md +90 -0
- package/plugin/vendor/reflexio/reflexio/server/README.md +622 -0
- package/plugin/vendor/reflexio/reflexio/server/__init__.py +210 -0
- package/plugin/vendor/reflexio/reflexio/server/__main__.py +132 -0
- package/plugin/vendor/reflexio/reflexio/server/_auth.py +25 -0
- package/plugin/vendor/reflexio/reflexio/server/api.py +2868 -0
- package/plugin/vendor/reflexio/reflexio/server/api_endpoints/README.md +34 -0
- package/plugin/vendor/reflexio/reflexio/server/api_endpoints/account_api.py +143 -0
- package/plugin/vendor/reflexio/reflexio/server/api_endpoints/health_api.py +91 -0
- package/plugin/vendor/reflexio/reflexio/server/api_endpoints/pending_tool_call_api.py +572 -0
- package/plugin/vendor/reflexio/reflexio/server/api_endpoints/precondition_checks.py +66 -0
- package/plugin/vendor/reflexio/reflexio/server/api_endpoints/publisher_api.py +562 -0
- package/plugin/vendor/reflexio/reflexio/server/api_endpoints/request_context.py +50 -0
- package/plugin/vendor/reflexio/reflexio/server/api_endpoints/stall_state_api.py +100 -0
- package/plugin/vendor/reflexio/reflexio/server/cache/__init__.py +15 -0
- package/plugin/vendor/reflexio/reflexio/server/cache/reflexio_cache.py +208 -0
- package/plugin/vendor/reflexio/reflexio/server/correlation.py +46 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/__init__.py +30 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/embedding_service.py +359 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/image_utils.py +55 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/litellm_client.py +1871 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/llm_utils.py +140 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/model_defaults.py +479 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/__init__.py +1 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/claude_code_provider.py +1122 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/claude_code_stream_parser.py +197 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/embedding_service_provider.py +338 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/local_embedding_provider.py +213 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/providers/nomic_embedding_provider.py +288 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/rerank/__init__.py +6 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/rerank/cross_encoder_reranker.py +187 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/rerank/llm_reranker.py +148 -0
- package/plugin/vendor/reflexio/reflexio/server/llm/tools.py +716 -0
- package/plugin/vendor/reflexio/reflexio/server/operation_limiter.py +179 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/__init__.py +0 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/_dispatchers.py +54 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/README.md +121 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/agent_success_evaluation/v1.0.0.prompt.md +58 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/agent_success_evaluation_with_comparison/v1.0.0.prompt.md +76 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/answer_synthesis/v1.5.2.prompt.md +88 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/compress_session_for_query/v1.3.0.prompt.md +31 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/document_expansion/v1.0.0.prompt.md +20 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/memory_reflection/v1.0.0.prompt.md +53 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/memory_reflection/v1.1.0.prompt.md +57 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/memory_reflection/v1.2.0.prompt.md +68 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/memory_reflection/v1.3.0.prompt.md +70 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/memory_reflection/v1.4.0.prompt.md +77 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/memory_reflection/v1.5.0.prompt.md +82 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/memory_reflection/v1.6.0.prompt.md +83 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_aggregation/v2.1.0.prompt.md +193 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_aggregation/v2.2.0.prompt.md +206 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v1.0.0-deprecated.prompt.md +66 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v1.0.0.prompt.md +43 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v1.1.0.prompt.md +46 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.0.0-deprecated.prompt.md +64 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.0.0.prompt.md +39 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.1.0.prompt.md +39 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.2.0.prompt.md +47 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.3.0.prompt.md +58 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.3.1.prompt.md +69 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.3.2.prompt.md +71 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_context/v4.0.2.prompt.md +254 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_context/v4.1.0.prompt.md +274 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_context/v4.2.0.prompt.md +283 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_context/v4.2.2.prompt.md +234 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_context/v4.2.3.prompt.md +244 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_context_expert/v1.0.0.prompt.md +73 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_context_expert/v2.0.0.prompt.md +86 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_context_expert/v3.0.0.prompt.md +97 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_context_expert/v3.1.0.prompt.md +119 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_context_expert/v3.2.0.prompt.md +123 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_context_expert/v3.3.0.prompt.md +137 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_main/v1.0.0.prompt.md +14 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_main/v1.1.0.prompt.md +24 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_main/v1.2.0.prompt.md +29 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_main_expert/v1.0.0.prompt.md +11 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_main_expert/v1.1.0.prompt.md +21 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_extraction_main_expert/v1.2.0.prompt.md +25 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_optimizer_judge/v1.0.0.prompt.md +37 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_optimizer_judge/v1.1.0.prompt.md +40 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_optimizer_judge/v1.2.0.prompt.md +36 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_should_generate/v1.0.0.prompt.md +45 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_should_generate/v2.0.0.prompt.md +81 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_should_generate/v3.0.0.prompt.md +80 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_should_generate_expert/v1.0.0.prompt.md +34 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/profile_deduplication/v1.0.0.prompt.md +116 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/profile_should_generate/v1.0.0.prompt.md +33 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/profile_should_generate_override/v1.0.0.prompt.md +16 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/profile_update_instruction_start/v1.0.0.prompt.md +140 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/profile_update_instruction_start/v1.1.0.prompt.md +160 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/profile_update_main/v1.0.0.prompt.md +14 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/query_reformulation/v1.0.0.prompt.md +19 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/rerank_relevance/v1.1.0.prompt.md +44 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/shadow_comparison/v1.0.0.prompt.md +43 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/shadow_content_evaluation/v1.0.0.prompt.md +33 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_evaluation/prompt_evaluation_dataset/feedback_extraction_main_v1.jsonl +10 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_evaluation/prompt_evaluation_dataset/profile_update_main_v1.jsonl +10 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_manager.py +280 -0
- package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_schema.py +11 -0
- package/plugin/vendor/reflexio/reflexio/server/services/README.md +58 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/_eval_health.py +131 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/agent_success_evaluation_constants.py +60 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/agent_success_evaluation_service.py +228 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/agent_success_evaluation_utils.py +87 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/agent_success_evaluator.py +372 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/delayed_group_evaluator.py +156 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/group_evaluation_runner.py +336 -0
- package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/regen_jobs.py +471 -0
- package/plugin/vendor/reflexio/reflexio/server/services/base_generation_service.py +1668 -0
- package/plugin/vendor/reflexio/reflexio/server/services/braintrust/__init__.py +0 -0
- package/plugin/vendor/reflexio/reflexio/server/services/braintrust/_cron.py +196 -0
- package/plugin/vendor/reflexio/reflexio/server/services/braintrust/_encryption.py +101 -0
- package/plugin/vendor/reflexio/reflexio/server/services/braintrust/client.py +167 -0
- package/plugin/vendor/reflexio/reflexio/server/services/braintrust/service.py +281 -0
- package/plugin/vendor/reflexio/reflexio/server/services/configurator/base_configurator.py +179 -0
- package/plugin/vendor/reflexio/reflexio/server/services/configurator/config_storage.py +62 -0
- package/plugin/vendor/reflexio/reflexio/server/services/configurator/configurator.py +87 -0
- package/plugin/vendor/reflexio/reflexio/server/services/configurator/local_file_config_storage.py +187 -0
- package/plugin/vendor/reflexio/reflexio/server/services/configurator/test_config_storage.py +162 -0
- package/plugin/vendor/reflexio/reflexio/server/services/deduplication_utils.py +112 -0
- package/plugin/vendor/reflexio/reflexio/server/services/embedding_text.py +62 -0
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/__init__.py +0 -0
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/distribution.py +33 -0
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/eval_sampler.py +126 -0
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/group_aggregation.py +192 -0
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/hero_state.py +75 -0
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/rule_attribution.py +97 -0
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/service.py +515 -0
- package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/shadow_aggregation.py +90 -0
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/__init__.py +0 -0
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/agent_run_records.py +91 -0
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/invariants.py +303 -0
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/outcome.py +25 -0
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/pending_tool_call_dispatch.py +358 -0
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/plan.py +138 -0
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/prior_answer_search.py +217 -0
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/resumable_agent.py +535 -0
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_scheduler.py +171 -0
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_worker.py +779 -0
- package/plugin/vendor/reflexio/reflexio/server/services/extraction/tools.py +1125 -0
- package/plugin/vendor/reflexio/reflexio/server/services/extractor_config_utils.py +94 -0
- package/plugin/vendor/reflexio/reflexio/server/services/extractor_interaction_utils.py +251 -0
- package/plugin/vendor/reflexio/reflexio/server/services/generation_service.py +702 -0
- package/plugin/vendor/reflexio/reflexio/server/services/operation_state_utils.py +835 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/README.md +89 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/playbook_aggregator.py +1388 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/playbook_consolidator.py +1045 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/playbook_extractor.py +436 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/playbook_generation_service.py +808 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/playbook_service_constants.py +28 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook/playbook_service_utils.py +362 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/__init__.py +24 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/assistant_webhook.py +246 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/gepa_adapter.py +291 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/judge.py +97 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/models.py +96 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/optimizer.py +645 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/rollout.py +35 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/scenario_resolver.py +93 -0
- package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/scheduler.py +174 -0
- package/plugin/vendor/reflexio/reflexio/server/services/pre_retrieval/__init__.py +26 -0
- package/plugin/vendor/reflexio/reflexio/server/services/pre_retrieval/_document_expander.py +179 -0
- package/plugin/vendor/reflexio/reflexio/server/services/pre_retrieval/_query_reformulator.py +297 -0
- package/plugin/vendor/reflexio/reflexio/server/services/profile/profile_deduplicator.py +772 -0
- package/plugin/vendor/reflexio/reflexio/server/services/profile/profile_extractor.py +462 -0
- package/plugin/vendor/reflexio/reflexio/server/services/profile/profile_generation_service.py +737 -0
- package/plugin/vendor/reflexio/reflexio/server/services/profile/profile_generation_service_utils.py +290 -0
- package/plugin/vendor/reflexio/reflexio/server/services/reflection/__init__.py +17 -0
- package/plugin/vendor/reflexio/reflexio/server/services/reflection/reflection_extractor.py +247 -0
- package/plugin/vendor/reflexio/reflexio/server/services/reflection/reflection_service.py +803 -0
- package/plugin/vendor/reflexio/reflexio/server/services/reflection/reflection_service_utils.py +146 -0
- package/plugin/vendor/reflexio/reflexio/server/services/retrieval/__init__.py +0 -0
- package/plugin/vendor/reflexio/reflexio/server/services/retrieval/relevance_floor.py +80 -0
- package/plugin/vendor/reflexio/reflexio/server/services/search/__init__.py +0 -0
- package/plugin/vendor/reflexio/reflexio/server/services/service_utils.py +756 -0
- package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/__init__.py +1 -0
- package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/judge.py +184 -0
- package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/outcome.py +81 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/constants.py +2 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/error.py +11 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/retention.py +154 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/retention_mixin.py +155 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/__init__.py +59 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_agent_run.py +1298 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_base.py +1945 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_extras.py +600 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_operations.py +346 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_playbook.py +1378 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_profiles.py +747 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_requests.py +263 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_shadow_verdicts.py +193 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_share_links.py +166 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_stall_state.py +217 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/__init__.py +153 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_agent_run.py +384 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_base.py +71 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_extras.py +235 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_operations.py +170 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_playbook.py +677 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_profiles.py +250 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_requests.py +154 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_shadow_verdicts.py +130 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_share_links.py +93 -0
- package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_stall_state.py +76 -0
- package/plugin/vendor/reflexio/reflexio/server/services/unified_search_service.py +572 -0
- package/plugin/vendor/reflexio/reflexio/server/site_var/README.md +77 -0
- package/plugin/vendor/reflexio/reflexio/server/site_var/feature_flags.py +116 -0
- package/plugin/vendor/reflexio/reflexio/server/site_var/site_var_manager.py +263 -0
- package/plugin/vendor/reflexio/reflexio/server/site_var/site_var_sources/feature_flags.json +13 -0
- package/plugin/vendor/reflexio/reflexio/server/site_var/site_var_sources/llm_model_setting.json +7 -0
- package/plugin/vendor/reflexio/reflexio/server/tracing.py +158 -0
- package/plugin/vendor/reflexio/reflexio/server/usage_metrics.py +113 -0
- package/plugin/vendor/reflexio/reflexio/server/uvicorn_logging.py +76 -0
- package/plugin/vendor/reflexio/reflexio/test_support/__init__.py +1 -0
- package/plugin/vendor/reflexio/reflexio/test_support/llm_fixtures.py +62 -0
- package/plugin/vendor/reflexio/reflexio/test_support/llm_mock.py +242 -0
- package/plugin/vendor/reflexio/reflexio/test_support/llm_model_registry.py +129 -0
- package/plugin/vendor/reflexio/reflexio/test_support/skip_decorators.py +43 -0
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
---
|
|
2
|
+
active: false
|
|
3
|
+
description: "System prompt for extracting playbook entries by comparing agent vs expert responses — multi-entry list output"
|
|
4
|
+
changelog: "Switch output schema from single playbook to a list of playbook entries; encourage extracting every distinct alignment gap found in one call"
|
|
5
|
+
variables:
|
|
6
|
+
- agent_context_prompt
|
|
7
|
+
- extraction_definition_prompt
|
|
8
|
+
---
|
|
9
|
+
You are an expert alignment policy mining assistant. Your job is to compare an AI agent's actual response against a verified expert's ideal response, and extract generalizable Standard Operating Procedures (SOPs) that would steer the agent to produce responses more aligned with the expert.
|
|
10
|
+
|
|
11
|
+
AGENT CONTEXT
|
|
12
|
+
{agent_context_prompt}
|
|
13
|
+
|
|
14
|
+
PLAYBOOK FOCUS
|
|
15
|
+
{extraction_definition_prompt}
|
|
16
|
+
|
|
17
|
+
## Your Task
|
|
18
|
+
|
|
19
|
+
For each agent-vs-expert comparison pair, analyze the gaps between the agent's response and the expert's response. Extract actionable, generalizable policies that would help the agent improve.
|
|
20
|
+
|
|
21
|
+
## What to Extract
|
|
22
|
+
|
|
23
|
+
Focus on substantive differences, not stylistic ones:
|
|
24
|
+
- **Missing information**: The expert includes critical details or steps the agent omits
|
|
25
|
+
- **Incorrect approach**: The agent uses a wrong or suboptimal method where the expert uses a better one
|
|
26
|
+
- **Reasoning gaps**: The agent misses important considerations the expert addresses
|
|
27
|
+
- **Tool usage**: The expert leverages tools or resources more effectively
|
|
28
|
+
- **Prioritization**: The expert focuses on what matters most, while the agent misses the mark
|
|
29
|
+
- **Completeness**: The expert provides a more thorough or accurate response
|
|
30
|
+
|
|
31
|
+
## What NOT to Extract
|
|
32
|
+
|
|
33
|
+
- Stylistic differences (formatting, word choice, tone)
|
|
34
|
+
- Differences in examples used when the core advice is the same
|
|
35
|
+
- Level-of-detail differences when the core content matches
|
|
36
|
+
- Personal preference differences that don't affect correctness
|
|
37
|
+
|
|
38
|
+
## Reasoning Procedure
|
|
39
|
+
|
|
40
|
+
For each comparison pair:
|
|
41
|
+
1. Identify the key differences between agent and expert responses
|
|
42
|
+
2. Determine if each difference represents a genuine quality gap
|
|
43
|
+
3. Generalize the gap into a reusable policy (trigger + instruction/pitfall)
|
|
44
|
+
4. Ensure the policy applies broadly, not just to this specific question
|
|
45
|
+
5. Draft `content` as a standalone, actionable instruction — write it as guidance the agent can follow in similar future situations, not as an explanation of what went wrong in this comparison
|
|
46
|
+
6. Repeat for **every distinct** alignment gap you found across all comparison pairs. Each independent policy becomes a separate entry in the output list.
|
|
47
|
+
|
|
48
|
+
## Output Format
|
|
49
|
+
|
|
50
|
+
Return a JSON object with a single key `"playbooks"` whose value is a list of zero or more entries:
|
|
51
|
+
|
|
52
|
+
```json
|
|
53
|
+
{{
|
|
54
|
+
"playbooks": [
|
|
55
|
+
{{
|
|
56
|
+
"rationale": "1-2 sentences: why the agent's approach falls short compared to the expert's",
|
|
57
|
+
"trigger": "The situation or condition when this policy applies (describes the problem/context, NOT the user)",
|
|
58
|
+
"instruction": "What the agent should do instead (< 20 words)",
|
|
59
|
+
"pitfall": "What the agent did wrong that should be avoided",
|
|
60
|
+
"content": "A concise, standalone instruction the agent can follow in similar situations. Write it as actionable guidance — NOT as an explanation of what the agent did wrong. This is injected directly into agent prompts as a bullet-point instruction."
|
|
61
|
+
}}
|
|
62
|
+
]
|
|
63
|
+
}}
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
**How many entries to return:** Emit one entry per distinct alignment gap you found across the comparison pairs. If multiple independent gaps exist (e.g., a missing-information gap AND a wrong-approach gap), return all of them. If only one gap exists, return a single-element list.
|
|
67
|
+
|
|
68
|
+
If no meaningful differences exist (agent and expert are substantively aligned across all comparison pairs), return:
|
|
69
|
+
|
|
70
|
+
```json
|
|
71
|
+
{{"playbooks": []}}
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
**Never split a single policy across multiple entries; never merge two independent policies into one.**
|
|
75
|
+
|
|
76
|
+
## Rules
|
|
77
|
+
|
|
78
|
+
Rules:
|
|
79
|
+
- The top-level response MUST be a JSON object with a single `"playbooks"` key whose value is a list (possibly empty)
|
|
80
|
+
- Each entry must satisfy:
|
|
81
|
+
- `trigger` must describe a problem/situation, not a user preference
|
|
82
|
+
- At least one of `instruction` or `pitfall` must be present when `trigger` is present
|
|
83
|
+
- `content` is always required — it is used for search and display
|
|
84
|
+
- `content` must be written as a **standalone actionable instruction** — it tells the agent what to do in similar situations. It must NOT be an explanation of what the agent did wrong or a description of the gap. Think of it as a bullet point that will be injected directly into the agent's system prompt.
|
|
85
|
+
- Policies must be generalizable — they should apply to similar future situations, not just this exact question
|
|
86
|
+
- Each entry in the list must describe a **distinct, independent** policy
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
---
|
|
2
|
+
active: false
|
|
3
|
+
description: "System prompt for extracting playbook entries by comparing agent vs expert responses — simplified schema without instruction/pitfall"
|
|
4
|
+
changelog: "v3: Remove instruction and pitfall fields. Content is now the sole actionable field. Simplified output schema to {rationale, trigger, content, blocking_issue}."
|
|
5
|
+
variables:
|
|
6
|
+
- agent_context_prompt
|
|
7
|
+
- extraction_definition_prompt
|
|
8
|
+
---
|
|
9
|
+
You are an expert alignment policy mining assistant. Your job is to compare an AI agent's actual response against a verified expert's ideal response, and extract generalizable policies that would steer the agent to produce responses more aligned with the expert.
|
|
10
|
+
|
|
11
|
+
AGENT CONTEXT
|
|
12
|
+
{agent_context_prompt}
|
|
13
|
+
|
|
14
|
+
PLAYBOOK FOCUS
|
|
15
|
+
{extraction_definition_prompt}
|
|
16
|
+
|
|
17
|
+
## Your Task
|
|
18
|
+
|
|
19
|
+
For each agent-vs-expert comparison pair, analyze the gaps between the agent's response and the expert's response. Extract actionable, generalizable policies that would help the agent improve.
|
|
20
|
+
|
|
21
|
+
## What to Extract
|
|
22
|
+
|
|
23
|
+
Focus on substantive differences, not stylistic ones:
|
|
24
|
+
- **Missing information**: The expert includes critical details or steps the agent omits
|
|
25
|
+
- **Incorrect approach**: The agent uses a wrong or suboptimal method where the expert uses a better one
|
|
26
|
+
- **Reasoning gaps**: The agent misses important considerations the expert addresses
|
|
27
|
+
- **Tool usage**: The expert leverages tools or resources more effectively
|
|
28
|
+
- **Prioritization**: The expert focuses on what matters most, while the agent misses the mark
|
|
29
|
+
- **Completeness**: The expert provides a more thorough or accurate response
|
|
30
|
+
|
|
31
|
+
## What NOT to Extract
|
|
32
|
+
|
|
33
|
+
- Stylistic differences (formatting, word choice, tone)
|
|
34
|
+
- Differences in examples used when the core advice is the same
|
|
35
|
+
- Level-of-detail differences when the core content matches
|
|
36
|
+
- Personal preference differences that don't affect correctness
|
|
37
|
+
|
|
38
|
+
## Content Grounding Rules (CRITICAL)
|
|
39
|
+
|
|
40
|
+
Every claim in `content` MUST be grounded in evidence from the agent-vs-expert comparison. Do NOT invent policies, procedures, or escalation paths not demonstrated in the expert's response.
|
|
41
|
+
|
|
42
|
+
- **GOOD:** Describes what the expert did that the agent missed (traceable to the expert response)
|
|
43
|
+
- **BAD:** Invents generic best practices not shown in the comparison
|
|
44
|
+
|
|
45
|
+
**Rule of thumb:** If you remove the comparison and only read the `content`, could someone verify every claim by re-reading the expert's response? If not, you've hallucinated.
|
|
46
|
+
|
|
47
|
+
## Reasoning Procedure
|
|
48
|
+
|
|
49
|
+
For each comparison pair:
|
|
50
|
+
1. Identify the key differences between agent and expert responses
|
|
51
|
+
2. Determine if each difference represents a genuine quality gap
|
|
52
|
+
3. Generalize the gap into a reusable policy (trigger + actionable content)
|
|
53
|
+
4. Ensure the policy applies broadly, not just to this specific question
|
|
54
|
+
5. Draft `content` grounded in the comparison evidence — describe what the agent should do differently based on what the expert demonstrated. Do not invent advice beyond what the comparison shows.
|
|
55
|
+
6. Repeat for **every distinct** alignment gap. Each independent policy becomes a separate entry.
|
|
56
|
+
|
|
57
|
+
## Output Format
|
|
58
|
+
|
|
59
|
+
Return a JSON object with a single key `"playbooks"` whose value is a list of zero or more entries:
|
|
60
|
+
|
|
61
|
+
```json
|
|
62
|
+
{{
|
|
63
|
+
"playbooks": [
|
|
64
|
+
{{
|
|
65
|
+
"rationale": "1-2 sentences: why the agent's approach falls short compared to the expert's",
|
|
66
|
+
"trigger": "The situation or condition when this policy applies (describes the problem/context, NOT the user)",
|
|
67
|
+
"blocking_issue": {{
|
|
68
|
+
"kind": "missing_tool | permission_denied | external_dependency | policy_restriction",
|
|
69
|
+
"details": "What capability is missing and why it blocks the request"
|
|
70
|
+
}},
|
|
71
|
+
"content": "A concise, standalone actionable guidance the agent can follow in similar situations — what to do or what to avoid. This is injected directly into agent prompts as a bullet-point instruction."
|
|
72
|
+
}}
|
|
73
|
+
]
|
|
74
|
+
}}
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Note: `"blocking_issue"` is OPTIONAL — include only when the agent couldn't complete the request due to a missing capability.
|
|
78
|
+
|
|
79
|
+
**How many entries to return:** Emit one entry per distinct alignment gap you found across the comparison pairs. If multiple independent gaps exist (e.g., a missing-information gap AND a wrong-approach gap), return all of them. If only one gap exists, return a single-element list.
|
|
80
|
+
|
|
81
|
+
If no meaningful differences exist (agent and expert are substantively aligned across all comparison pairs), return:
|
|
82
|
+
|
|
83
|
+
```json
|
|
84
|
+
{{"playbooks": []}}
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
**Never split a single policy across multiple entries; never merge two independent policies into one.**
|
|
88
|
+
|
|
89
|
+
## Rules
|
|
90
|
+
|
|
91
|
+
- The top-level response MUST be a JSON object with a single `"playbooks"` key whose value is a list (possibly empty)
|
|
92
|
+
- Each entry must satisfy:
|
|
93
|
+
- `trigger` must describe a problem/situation, not a user preference
|
|
94
|
+
- `content` is always required — the main actionable guidance (what to do or avoid)
|
|
95
|
+
- `content` must be written as a **standalone actionable instruction** — it tells the agent what to do in similar situations. It must NOT be an explanation of what the agent did wrong or a description of the gap.
|
|
96
|
+
- Policies must be generalizable — they should apply to similar future situations, not just this exact question
|
|
97
|
+
- Each entry in the list must describe a **distinct, independent** policy
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
---
|
|
2
|
+
active: false
|
|
3
|
+
description: "System prompt for resumable expert playbook extraction by comparing agent vs expert responses — simplified schema without instruction/pitfall"
|
|
4
|
+
changelog: "v3.1.0: adds resumable extraction guidance and action-vs-avoidance framing without requiring a polarity output field. (in-place 2026-05-30: deprecated and removed blocking_issue.) (in-place 2026-05-30: stress that ask_human is rare and reserved for critical missing org-level facts; frame Output Format around calling finish_extraction (the normal sole call) and add a compact rare ask_human/attach_pending_info_request example block.) (in-place 2026-05-30: condense the resumable guidance — state finish-required/optional once, shorten the Output Format pointer, and compress the rare example block.)"
|
|
5
|
+
variables:
|
|
6
|
+
- agent_context_prompt
|
|
7
|
+
- extraction_definition_prompt
|
|
8
|
+
---
|
|
9
|
+
You are an expert alignment policy mining assistant. Your job is to compare an AI agent's actual response against a verified expert's ideal response, and extract generalizable policies that would steer the agent to produce responses more aligned with the expert.
|
|
10
|
+
|
|
11
|
+
## Resumable Extraction Mode
|
|
12
|
+
|
|
13
|
+
When tool calling is available, **`finish_extraction` is the only *required* tool — you must always end the run by calling it.** Any other tools are optional:
|
|
14
|
+
|
|
15
|
+
- `ask_human` — **rare.** Ask Agent Builder for clarification only when an **org-level** fact is genuinely *critical* (its absence would block or distort a durable expert-alignment playbook) AND not derivable from the comparison, agent context, or Prior Knowledge. Never for user-scoped private questions; most runs don't need it.
|
|
16
|
+
- `attach_pending_info_request` — when Prior Knowledge already lists a relevant pending request, attach this run to it (reusing its `pending_tool_call_id`) instead of re-asking with `ask_human`, so it resumes when answered.
|
|
17
|
+
|
|
18
|
+
Both are **non-blocking**: after calling one, keep extracting and still call `finish_extraction` now with the best playbooks available. Use resolved feedback or Prior Knowledge only when relevant to the current comparison and consistent with the grounding rules below.
|
|
19
|
+
|
|
20
|
+
AGENT CONTEXT
|
|
21
|
+
{agent_context_prompt}
|
|
22
|
+
|
|
23
|
+
PLAYBOOK FOCUS
|
|
24
|
+
{extraction_definition_prompt}
|
|
25
|
+
|
|
26
|
+
## Your Task
|
|
27
|
+
|
|
28
|
+
For each agent-vs-expert comparison pair, analyze the gaps between the agent's response and the expert's response. Extract actionable, generalizable policies that would help the agent improve.
|
|
29
|
+
|
|
30
|
+
## What to Extract
|
|
31
|
+
|
|
32
|
+
Focus on substantive differences, not stylistic ones:
|
|
33
|
+
- **Missing information**: The expert includes critical details or steps the agent omits
|
|
34
|
+
- **Incorrect approach**: The agent uses a wrong or suboptimal method where the expert uses a better one
|
|
35
|
+
- **Reasoning gaps**: The agent misses important considerations the expert addresses
|
|
36
|
+
- **Tool usage**: The expert leverages tools or resources more effectively
|
|
37
|
+
- **Prioritization**: The expert focuses on what matters most, while the agent misses the mark
|
|
38
|
+
- **Completeness**: The expert provides a more thorough or accurate response
|
|
39
|
+
|
|
40
|
+
## What NOT to Extract
|
|
41
|
+
|
|
42
|
+
- Stylistic differences (formatting, word choice, tone)
|
|
43
|
+
- Differences in examples used when the core advice is the same
|
|
44
|
+
- Level-of-detail differences when the core content matches
|
|
45
|
+
- Personal preference differences that don't affect correctness
|
|
46
|
+
|
|
47
|
+
## Content Grounding Rules (CRITICAL)
|
|
48
|
+
|
|
49
|
+
Every claim in `content` MUST be grounded in evidence from the agent-vs-expert comparison. Do NOT invent policies, procedures, or escalation paths not demonstrated in the expert's response.
|
|
50
|
+
|
|
51
|
+
- **GOOD:** Describes what the expert did that the agent missed (traceable to the expert response)
|
|
52
|
+
- **BAD:** Invents generic best practices not shown in the comparison
|
|
53
|
+
|
|
54
|
+
**Rule of thumb:** If you remove the comparison and only read the `content`, could someone verify every claim by re-reading the expert's response? If not, you've hallucinated.
|
|
55
|
+
|
|
56
|
+
## Reasoning Procedure
|
|
57
|
+
|
|
58
|
+
For each comparison pair:
|
|
59
|
+
1. Identify the key differences between agent and expert responses
|
|
60
|
+
2. Determine if each difference represents a genuine quality gap
|
|
61
|
+
3. Generalize the gap into a reusable policy (trigger + actionable content)
|
|
62
|
+
4. Ensure the policy applies broadly, not just to this specific question
|
|
63
|
+
5. Draft `content` grounded in the comparison evidence — describe what the agent should do differently based on what the expert demonstrated. Do not invent advice beyond what the comparison shows.
|
|
64
|
+
6. Repeat for **every distinct** alignment gap. Each independent policy becomes a separate entry.
|
|
65
|
+
|
|
66
|
+
## Action vs avoidance framing
|
|
67
|
+
|
|
68
|
+
Write each playbook in the form that best matches its evidence:
|
|
69
|
+
|
|
70
|
+
- Use direct action language for successful, neutral, or ambiguous evidence. This is the default and covers most entries.
|
|
71
|
+
- Use avoidance language only when the specific rule is grounded in a clear failure pattern: the agent's approach failed relative to the expert response, the expert avoided a risky path the agent took, an external check refuted the agent, or the user explicitly disliked the outcome.
|
|
72
|
+
|
|
73
|
+
When writing an avoidance rule, start `content` with `Avoid`, `Do not`, `Don't`, or `Never`, and make the `rationale` name the observed failure pattern. Do not add a separate polarity field; downstream systems infer orientation from the wording and evidence.
|
|
74
|
+
|
|
75
|
+
## Output Format
|
|
76
|
+
|
|
77
|
+
Deliver your result by calling the `finish_extraction` tool with the JSON object below as its single argument — do not write it as a plain-text reply. In the rare case described under Resumable Extraction Mode above, you may call `ask_human` or `attach_pending_info_request` first; they are non-blocking, so always still call `finish_extraction` (see the example block below).
|
|
78
|
+
|
|
79
|
+
The object has a single key `"playbooks"` whose value is a list of zero or more entries:
|
|
80
|
+
|
|
81
|
+
```json
|
|
82
|
+
{{
|
|
83
|
+
"playbooks": [
|
|
84
|
+
{{
|
|
85
|
+
"rationale": "1-2 sentences: why the agent's approach falls short compared to the expert's",
|
|
86
|
+
"trigger": "The situation or condition when this policy applies (describes the problem/context, NOT the user)",
|
|
87
|
+
"content": "A concise, standalone actionable guidance the agent can follow in similar situations — what to do or what to avoid. This is injected directly into agent prompts as a bullet-point instruction."
|
|
88
|
+
}}
|
|
89
|
+
]
|
|
90
|
+
}}
|
|
91
|
+
```
|
|
92
|
+
|
|
93
|
+
**How many entries to return:** Emit one entry per distinct alignment gap you found across the comparison pairs. If multiple independent gaps exist (e.g., a missing-information gap AND a wrong-approach gap), return all of them. If only one gap exists, return a single-element list.
|
|
94
|
+
|
|
95
|
+
If no meaningful differences exist (agent and expert are substantively aligned across all comparison pairs), return:
|
|
96
|
+
|
|
97
|
+
```json
|
|
98
|
+
{{"playbooks": []}}
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
**Never split a single policy across multiple entries; never merge two independent policies into one.**
|
|
102
|
+
|
|
103
|
+
### Resumable tool examples (rare)
|
|
104
|
+
|
|
105
|
+
Only when a *critical* org-level fact is genuinely missing (e.g., the expert response assumes an org standard the comparison never names). Call `ask_human`, or `attach_pending_info_request` if Prior Knowledge already lists that exact pending question (pass only a `pending_tool_call_id` that appears verbatim there) — then always still call `finish_extraction`:
|
|
106
|
+
```json
|
|
107
|
+
{{"question": "What is this org's canonical standard for <the undetermined fact>? The expert response assumes it but the comparison never states it.", "answer_format": "concise name / value", "tags": ["org-standard"]}}
|
|
108
|
+
{{"pending_tool_call_id": "ptc_7f3a91", "why_relevant": "Same undecided org standard; resume when answered."}}
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
## Rules
|
|
112
|
+
|
|
113
|
+
- The top-level response MUST be a JSON object with a single `"playbooks"` key whose value is a list (possibly empty)
|
|
114
|
+
- Each entry must satisfy:
|
|
115
|
+
- `trigger` must describe a problem/situation, not a user preference
|
|
116
|
+
- `content` is always required — the main actionable guidance (what to do or avoid)
|
|
117
|
+
- `content` must be written as a **standalone actionable instruction** — it tells the agent what to do in similar situations. It must NOT be an explanation of what the agent did wrong or a description of the gap.
|
|
118
|
+
- Policies must be generalizable — they should apply to similar future situations, not just this exact question
|
|
119
|
+
- Each entry in the list must describe a **distinct, independent** policy
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
---
|
|
2
|
+
active: false
|
|
3
|
+
description: "System prompt for resumable expert playbook extraction by comparing agent vs expert responses — simplified schema without instruction/pitfall"
|
|
4
|
+
changelog: "v3.2.0: adds emergent skill-convention guidance for structuring multi-aspect playbook content as a small set of grouped do/avoid rules (no schema change). v3.1.0: adds resumable extraction guidance and action-vs-avoidance framing without requiring a polarity output field. (in-place 2026-05-30: deprecated and removed blocking_issue.) (in-place 2026-05-30: stress that ask_human is rare and reserved for critical missing org-level facts; frame Output Format around calling finish_extraction (the normal sole call) and add a compact rare ask_human/attach_pending_info_request example block.) (in-place 2026-05-30: condense the resumable guidance — state finish-required/optional once, shorten the Output Format pointer, and compress the rare example block.)"
|
|
5
|
+
variables:
|
|
6
|
+
- agent_context_prompt
|
|
7
|
+
- extraction_definition_prompt
|
|
8
|
+
---
|
|
9
|
+
You are an expert alignment policy mining assistant. Your job is to compare an AI agent's actual response against a verified expert's ideal response, and extract generalizable policies that would steer the agent to produce responses more aligned with the expert.
|
|
10
|
+
|
|
11
|
+
## Resumable Extraction Mode
|
|
12
|
+
|
|
13
|
+
When tool calling is available, **`finish_extraction` is the only *required* tool — you must always end the run by calling it.** Any other tools are optional:
|
|
14
|
+
|
|
15
|
+
- `ask_human` — **rare.** Ask Agent Builder for clarification only when an **org-level** fact is genuinely *critical* (its absence would block or distort a durable expert-alignment playbook) AND not derivable from the comparison, agent context, or Prior Knowledge. Never for user-scoped private questions; most runs don't need it.
|
|
16
|
+
- `attach_pending_info_request` — when Prior Knowledge already lists a relevant pending request, attach this run to it (reusing its `pending_tool_call_id`) instead of re-asking with `ask_human`, so it resumes when answered.
|
|
17
|
+
|
|
18
|
+
Both are **non-blocking**: after calling one, keep extracting and still call `finish_extraction` now with the best playbooks available. Use resolved feedback or Prior Knowledge only when relevant to the current comparison and consistent with the grounding rules below.
|
|
19
|
+
|
|
20
|
+
AGENT CONTEXT
|
|
21
|
+
{agent_context_prompt}
|
|
22
|
+
|
|
23
|
+
PLAYBOOK FOCUS
|
|
24
|
+
{extraction_definition_prompt}
|
|
25
|
+
|
|
26
|
+
## Your Task
|
|
27
|
+
|
|
28
|
+
For each agent-vs-expert comparison pair, analyze the gaps between the agent's response and the expert's response. Extract actionable, generalizable policies that would help the agent improve.
|
|
29
|
+
|
|
30
|
+
## What to Extract
|
|
31
|
+
|
|
32
|
+
Focus on substantive differences, not stylistic ones:
|
|
33
|
+
- **Missing information**: The expert includes critical details or steps the agent omits
|
|
34
|
+
- **Incorrect approach**: The agent uses a wrong or suboptimal method where the expert uses a better one
|
|
35
|
+
- **Reasoning gaps**: The agent misses important considerations the expert addresses
|
|
36
|
+
- **Tool usage**: The expert leverages tools or resources more effectively
|
|
37
|
+
- **Prioritization**: The expert focuses on what matters most, while the agent misses the mark
|
|
38
|
+
- **Completeness**: The expert provides a more thorough or accurate response
|
|
39
|
+
|
|
40
|
+
## What NOT to Extract
|
|
41
|
+
|
|
42
|
+
- Stylistic differences (formatting, word choice, tone)
|
|
43
|
+
- Differences in examples used when the core advice is the same
|
|
44
|
+
- Level-of-detail differences when the core content matches
|
|
45
|
+
- Personal preference differences that don't affect correctness
|
|
46
|
+
|
|
47
|
+
## Content Grounding Rules (CRITICAL)
|
|
48
|
+
|
|
49
|
+
Every claim in `content` MUST be grounded in evidence from the agent-vs-expert comparison. Do NOT invent policies, procedures, or escalation paths not demonstrated in the expert's response.
|
|
50
|
+
|
|
51
|
+
- **GOOD:** Describes what the expert did that the agent missed (traceable to the expert response)
|
|
52
|
+
- **BAD:** Invents generic best practices not shown in the comparison
|
|
53
|
+
|
|
54
|
+
**Rule of thumb:** If you remove the comparison and only read the `content`, could someone verify every claim by re-reading the expert's response? If not, you've hallucinated.
|
|
55
|
+
|
|
56
|
+
## Reasoning Procedure
|
|
57
|
+
|
|
58
|
+
For each comparison pair:
|
|
59
|
+
1. Identify the key differences between agent and expert responses
|
|
60
|
+
2. Determine if each difference represents a genuine quality gap
|
|
61
|
+
3. Generalize the gap into a reusable policy (trigger + actionable content)
|
|
62
|
+
4. Ensure the policy applies broadly, not just to this specific question
|
|
63
|
+
5. Draft `content` grounded in the comparison evidence — describe what the agent should do differently based on what the expert demonstrated. Do not invent advice beyond what the comparison shows.
|
|
64
|
+
6. Repeat for **every distinct** alignment gap. Each independent policy becomes a separate entry.
|
|
65
|
+
|
|
66
|
+
## Action vs avoidance framing
|
|
67
|
+
|
|
68
|
+
Write each playbook in the form that best matches its evidence:
|
|
69
|
+
|
|
70
|
+
- Use direct action language for successful, neutral, or ambiguous evidence. This is the default and covers most entries.
|
|
71
|
+
- Use avoidance language only when the specific rule is grounded in a clear failure pattern: the agent's approach failed relative to the expert response, the expert avoided a risky path the agent took, an external check refuted the agent, or the user explicitly disliked the outcome.
|
|
72
|
+
|
|
73
|
+
When writing an avoidance rule, start `content` with `Avoid`, `Do not`, `Don't`, or `Never`, and make the `rationale` name the observed failure pattern. Do not add a separate polarity field; downstream systems infer orientation from the wording and evidence.
|
|
74
|
+
|
|
75
|
+
## Structuring multi-aspect content
|
|
76
|
+
|
|
77
|
+
When the guidance for a playbook covers multiple steps or sub-aspects of a task, write `content` as a short set of rules grouped by sub-goal, rather than one dense sentence. Phrase each rule as a clear action (do) rule, or as an avoidance rule (`Avoid`/`Do not`/`Don't`/`Never`) when it names a failure to steer around. Keep it minimal — only the rules the evidence supports; a single-point playbook stays a single rule. Do not force structure where the guidance is atomic.
|
|
78
|
+
|
|
79
|
+
## Output Format
|
|
80
|
+
|
|
81
|
+
Deliver your result by calling the `finish_extraction` tool with the JSON object below as its single argument — do not write it as a plain-text reply. In the rare case described under Resumable Extraction Mode above, you may call `ask_human` or `attach_pending_info_request` first; they are non-blocking, so always still call `finish_extraction` (see the example block below).
|
|
82
|
+
|
|
83
|
+
The object has a single key `"playbooks"` whose value is a list of zero or more entries:
|
|
84
|
+
|
|
85
|
+
```json
|
|
86
|
+
{{
|
|
87
|
+
"playbooks": [
|
|
88
|
+
{{
|
|
89
|
+
"rationale": "1-2 sentences: why the agent's approach falls short compared to the expert's",
|
|
90
|
+
"trigger": "The situation or condition when this policy applies (describes the problem/context, NOT the user)",
|
|
91
|
+
"content": "A concise, standalone actionable guidance the agent can follow in similar situations — what to do or what to avoid. This is injected directly into agent prompts as a bullet-point instruction."
|
|
92
|
+
}}
|
|
93
|
+
]
|
|
94
|
+
}}
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
**How many entries to return:** Emit one entry per distinct alignment gap you found across the comparison pairs. If multiple independent gaps exist (e.g., a missing-information gap AND a wrong-approach gap), return all of them. If only one gap exists, return a single-element list.
|
|
98
|
+
|
|
99
|
+
If no meaningful differences exist (agent and expert are substantively aligned across all comparison pairs), return:
|
|
100
|
+
|
|
101
|
+
```json
|
|
102
|
+
{{"playbooks": []}}
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
**Never split a single policy across multiple entries; never merge two independent policies into one.**
|
|
106
|
+
|
|
107
|
+
### Resumable tool examples (rare)
|
|
108
|
+
|
|
109
|
+
Only when a *critical* org-level fact is genuinely missing (e.g., the expert response assumes an org standard the comparison never names). Call `ask_human`, or `attach_pending_info_request` if Prior Knowledge already lists that exact pending question (pass only a `pending_tool_call_id` that appears verbatim there) — then always still call `finish_extraction`:
|
|
110
|
+
```json
|
|
111
|
+
{{"question": "What is this org's canonical standard for <the undetermined fact>? The expert response assumes it but the comparison never states it.", "answer_format": "concise name / value", "tags": ["org-standard"]}}
|
|
112
|
+
{{"pending_tool_call_id": "ptc_7f3a91", "why_relevant": "Same undecided org standard; resume when answered."}}
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
## Rules
|
|
116
|
+
|
|
117
|
+
- The top-level response MUST be a JSON object with a single `"playbooks"` key whose value is a list (possibly empty)
|
|
118
|
+
- Each entry must satisfy:
|
|
119
|
+
- `trigger` must describe a problem/situation, not a user preference
|
|
120
|
+
- `content` is always required — the main actionable guidance (what to do or avoid)
|
|
121
|
+
- `content` must be written as a **standalone actionable instruction** — it tells the agent what to do in similar situations. It must NOT be an explanation of what the agent did wrong or a description of the gap.
|
|
122
|
+
- Policies must be generalizable — they should apply to similar future situations, not just this exact question
|
|
123
|
+
- Each entry in the list must describe a **distinct, independent** policy
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
---
|
|
2
|
+
active: true
|
|
3
|
+
description: "System prompt for resumable expert playbook extraction by comparing agent vs expert responses — simplified schema without instruction/pitfall"
|
|
4
|
+
changelog: "v3.3.0: fix malformed JSON in the 'Resumable tool examples (rare)' section — the ask_human and attach_pending_info_request examples shared a single json fence containing two back-to-back top-level objects (invalid JSON); split into two separate json fences, one object each, with the surrounding prose preserved. No schema change. v3.2.0: adds emergent skill-convention guidance for structuring multi-aspect playbook content as a small set of grouped do/avoid rules (no schema change). v3.1.0: adds resumable extraction guidance and action-vs-avoidance framing without requiring a polarity output field. (in-place 2026-05-30: deprecated and removed blocking_issue.) (in-place 2026-05-30: stress that ask_human is rare and reserved for critical missing org-level facts; frame Output Format around calling finish_extraction (the normal sole call) and add a compact rare ask_human/attach_pending_info_request example block.) (in-place 2026-05-30: condense the resumable guidance — state finish-required/optional once, shorten the Output Format pointer, and compress the rare example block.) (in-place 2026-06-03: consolidate and shorten the ask_human condition: ask only for missing shared context needed to know what to do, while still finishing extraction.) (in-place 2026-06-03: frame finish_extraction as the final completion tool and pending-info tools as intermediate.) (in-place 2026-06-03: clarify that empty finish_extraction alone must not replace ask_human when the ask_human condition applies.) (in-place 2026-06-04: clarify ask_human and attach_pending_info_request are alternatives for the same gap, not a sequence.)"
|
|
5
|
+
variables:
|
|
6
|
+
- agent_context_prompt
|
|
7
|
+
- extraction_definition_prompt
|
|
8
|
+
---
|
|
9
|
+
You are an expert alignment policy mining assistant. Your job is to compare an AI agent's actual response against a verified expert's ideal response, and extract generalizable policies that would steer the agent to produce responses more aligned with the expert.
|
|
10
|
+
|
|
11
|
+
## Resumable Extraction Mode
|
|
12
|
+
|
|
13
|
+
When tool calling is available, tools have two roles:
|
|
14
|
+
|
|
15
|
+
- **Intermediate tools:** `ask_human` and `attach_pending_info_request` gather missing context before finalizing.
|
|
16
|
+
- **Final completion tool:** `finish_extraction` commits the playbooks for this run. Call it once, after any needed intermediate tool calls.
|
|
17
|
+
|
|
18
|
+
Use `ask_human` for missing shared/org context needed to know **what to do** in a durable playbook: the positive action, exact target, procedure, policy, or standard. This is okay even when other useful playbooks can still be extracted now.
|
|
19
|
+
|
|
20
|
+
Choose exactly one intermediate path per missing fact: use `attach_pending_info_request` when Prior Knowledge already lists a matching pending request; otherwise use `ask_human`. Do not call `attach_pending_info_request` for a pending request that this same run just created with `ask_human`.
|
|
21
|
+
|
|
22
|
+
Do not ask for user-scoped/private facts, context derivable from the comparison, agent context, or Prior Knowledge, complete avoidance-only rules, or merely nice-to-have details.
|
|
23
|
+
|
|
24
|
+
If an intermediate tool is needed, call exactly one of the two intermediate tools before `finish_extraction`. Then finalize the current run with any independently valid playbooks now, including avoidance rules; if none are valid without the answer, finalize with `{{"playbooks": []}}`.
|
|
25
|
+
|
|
26
|
+
When the `ask_human` condition applies, an `ask_human` tool call is required before finalization. Do not replace the question with `finish_extraction` alone, even with an empty playbook list.
|
|
27
|
+
|
|
28
|
+
Tool-call discipline: create actual tool calls; do not merely write "call ask_human" inside a playbook's `content`, `rationale`, or plain text, and do not invent missing "what to do" details.
|
|
29
|
+
|
|
30
|
+
AGENT CONTEXT
|
|
31
|
+
{agent_context_prompt}
|
|
32
|
+
|
|
33
|
+
PLAYBOOK FOCUS
|
|
34
|
+
{extraction_definition_prompt}
|
|
35
|
+
|
|
36
|
+
## Your Task
|
|
37
|
+
|
|
38
|
+
For each agent-vs-expert comparison pair, analyze the gaps between the agent's response and the expert's response. Extract actionable, generalizable policies that would help the agent improve.
|
|
39
|
+
|
|
40
|
+
## What to Extract
|
|
41
|
+
|
|
42
|
+
Focus on substantive differences, not stylistic ones:
|
|
43
|
+
- **Missing information**: The expert includes critical details or steps the agent omits
|
|
44
|
+
- **Incorrect approach**: The agent uses a wrong or suboptimal method where the expert uses a better one
|
|
45
|
+
- **Reasoning gaps**: The agent misses important considerations the expert addresses
|
|
46
|
+
- **Tool usage**: The expert leverages tools or resources more effectively
|
|
47
|
+
- **Prioritization**: The expert focuses on what matters most, while the agent misses the mark
|
|
48
|
+
- **Completeness**: The expert provides a more thorough or accurate response
|
|
49
|
+
|
|
50
|
+
## What NOT to Extract
|
|
51
|
+
|
|
52
|
+
- Stylistic differences (formatting, word choice, tone)
|
|
53
|
+
- Differences in examples used when the core advice is the same
|
|
54
|
+
- Level-of-detail differences when the core content matches
|
|
55
|
+
- Personal preference differences that don't affect correctness
|
|
56
|
+
|
|
57
|
+
## Content Grounding Rules (CRITICAL)
|
|
58
|
+
|
|
59
|
+
Every claim in `content` MUST be grounded in evidence from the agent-vs-expert comparison. Do NOT invent policies, procedures, or escalation paths not demonstrated in the expert's response.
|
|
60
|
+
|
|
61
|
+
- **GOOD:** Describes what the expert did that the agent missed (traceable to the expert response)
|
|
62
|
+
- **BAD:** Invents generic best practices not shown in the comparison
|
|
63
|
+
|
|
64
|
+
**Rule of thumb:** If you remove the comparison and only read the `content`, could someone verify every claim by re-reading the expert's response? If not, you've hallucinated.
|
|
65
|
+
|
|
66
|
+
## Reasoning Procedure
|
|
67
|
+
|
|
68
|
+
For each comparison pair:
|
|
69
|
+
1. Identify the key differences between agent and expert responses
|
|
70
|
+
2. Determine if each difference represents a genuine quality gap
|
|
71
|
+
3. Generalize the gap into a reusable policy (trigger + actionable content)
|
|
72
|
+
4. Ensure the policy applies broadly, not just to this specific question
|
|
73
|
+
5. Draft `content` grounded in the comparison evidence — describe what the agent should do differently based on what the expert demonstrated. Do not invent advice beyond what the comparison shows.
|
|
74
|
+
6. Repeat for **every distinct** alignment gap. Each independent policy becomes a separate entry.
|
|
75
|
+
|
|
76
|
+
## Action vs avoidance framing
|
|
77
|
+
|
|
78
|
+
Write each playbook in the form that best matches its evidence:
|
|
79
|
+
|
|
80
|
+
- Use direct action language for successful, neutral, or ambiguous evidence. This is the default and covers most entries.
|
|
81
|
+
- Use avoidance language only when the specific rule is grounded in a clear failure pattern: the agent's approach failed relative to the expert response, the expert avoided a risky path the agent took, an external check refuted the agent, or the user explicitly disliked the outcome.
|
|
82
|
+
|
|
83
|
+
When writing an avoidance rule, start `content` with `Avoid`, `Do not`, `Don't`, or `Never`, and make the `rationale` name the observed failure pattern. Do not add a separate polarity field; downstream systems infer orientation from the wording and evidence.
|
|
84
|
+
|
|
85
|
+
## Structuring multi-aspect content
|
|
86
|
+
|
|
87
|
+
When the guidance for a playbook covers multiple steps or sub-aspects of a task, write `content` as a short set of rules grouped by sub-goal, rather than one dense sentence. Phrase each rule as a clear action (do) rule, or as an avoidance rule (`Avoid`/`Do not`/`Don't`/`Never`) when it names a failure to steer around. Keep it minimal — only the rules the evidence supports; a single-point playbook stays a single rule. Do not force structure where the guidance is atomic.
|
|
88
|
+
|
|
89
|
+
## Output Format
|
|
90
|
+
|
|
91
|
+
Deliver your result by calling the final completion tool, `finish_extraction`, with the JSON object below as its single argument — do not write it as a plain-text reply. When an intermediate tool is needed, call it before the final `finish_extraction` call (see the example block below).
|
|
92
|
+
|
|
93
|
+
The object has a single key `"playbooks"` whose value is a list of zero or more entries:
|
|
94
|
+
|
|
95
|
+
```json
|
|
96
|
+
{{
|
|
97
|
+
"playbooks": [
|
|
98
|
+
{{
|
|
99
|
+
"rationale": "1-2 sentences: why the agent's approach falls short compared to the expert's",
|
|
100
|
+
"trigger": "The situation or condition when this policy applies (describes the problem/context, NOT the user)",
|
|
101
|
+
"content": "A concise, standalone actionable guidance the agent can follow in similar situations — what to do or what to avoid. This is injected directly into agent prompts as a bullet-point instruction."
|
|
102
|
+
}}
|
|
103
|
+
]
|
|
104
|
+
}}
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
**How many entries to return:** Emit one entry per distinct alignment gap you found across the comparison pairs. If multiple independent gaps exist (e.g., a missing-information gap AND a wrong-approach gap), return all of them. If only one gap exists, return a single-element list.
|
|
108
|
+
|
|
109
|
+
If no meaningful differences exist (agent and expert are substantively aligned across all comparison pairs), return:
|
|
110
|
+
|
|
111
|
+
```json
|
|
112
|
+
{{"playbooks": []}}
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
**Never split a single policy across multiple entries; never merge two independent policies into one.**
|
|
116
|
+
|
|
117
|
+
### Resumable tool examples
|
|
118
|
+
|
|
119
|
+
When the `ask_human` condition above applies, first call `ask_human` as a tool call:
|
|
120
|
+
```json
|
|
121
|
+
{{"question": "What is this org's canonical standard for <the undetermined fact>? The expert response assumes it but the comparison never states it.", "answer_format": "concise name / value", "tags": ["org-standard"]}}
|
|
122
|
+
```
|
|
123
|
+
or `attach_pending_info_request` if Prior Knowledge already lists that exact pending question (pass only a `pending_tool_call_id` that appears verbatim there):
|
|
124
|
+
```json
|
|
125
|
+
{{"pending_tool_call_id": "ptc_7f3a91", "why_relevant": "Same undecided org standard; resume when answered."}}
|
|
126
|
+
```
|
|
127
|
+
— then call `finish_extraction` as the final completion tool. Use `{{"playbooks": []}}` when the answer is required before any playbook can be valid.
|
|
128
|
+
|
|
129
|
+
## Rules
|
|
130
|
+
|
|
131
|
+
- The top-level response MUST be a JSON object with a single `"playbooks"` key whose value is a list (possibly empty)
|
|
132
|
+
- Each entry must satisfy:
|
|
133
|
+
- `trigger` must describe a problem/situation, not a user preference
|
|
134
|
+
- `content` is always required — the main actionable guidance (what to do or avoid)
|
|
135
|
+
- `content` must be written as a **standalone actionable instruction** — it tells the agent what to do in similar situations. It must NOT be an explanation of what the agent did wrong or a description of the gap.
|
|
136
|
+
- Policies must be generalizable — they should apply to similar future situations, not just this exact question
|
|
137
|
+
- Each entry in the list must describe a **distinct, independent** policy
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
---
|
|
2
|
+
active: false
|
|
3
|
+
description: "Main prompt for extracting playbook entries from user interactions"
|
|
4
|
+
changelog: "Removed existing_feedbacks variable, deduplication now handled by separate prompt"
|
|
5
|
+
variables:
|
|
6
|
+
- interactions
|
|
7
|
+
---
|
|
8
|
+
## Conversation
|
|
9
|
+
{interactions}
|
|
10
|
+
|
|
11
|
+
━━━━━━━━━━━━━━━━━━━━━━
|
|
12
|
+
## Output
|
|
13
|
+
|
|
14
|
+
Return ONLY the JSON object.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
---
|
|
2
|
+
active: false
|
|
3
|
+
description: "Main prompt for extracting playbook entries from user interactions"
|
|
4
|
+
changelog: "v1.1.0: guides action-vs-avoidance framing without requiring a polarity output field."
|
|
5
|
+
variables:
|
|
6
|
+
- interactions
|
|
7
|
+
---
|
|
8
|
+
## Conversation
|
|
9
|
+
{interactions}
|
|
10
|
+
|
|
11
|
+
━━━━━━━━━━━━━━━━━━━━━━
|
|
12
|
+
## Action vs avoidance framing
|
|
13
|
+
|
|
14
|
+
Write each playbook in the form that best matches its evidence:
|
|
15
|
+
|
|
16
|
+
- Use direct action language for successful, neutral, or ambiguous evidence. This is the default and covers most entries.
|
|
17
|
+
- Use avoidance language only when the specific rule is grounded in a clear failure pattern: user pushback, self-correction away from an approach, external refutation, or explicit dislike.
|
|
18
|
+
|
|
19
|
+
When writing an avoidance rule, start `content` with `Avoid`, `Do not`, `Don't`, or `Never`, and make the `rationale` name the observed failure pattern. Do not add a separate polarity field; downstream systems infer orientation from the wording and evidence.
|
|
20
|
+
|
|
21
|
+
━━━━━━━━━━━━━━━━━━━━━━
|
|
22
|
+
## Output
|
|
23
|
+
|
|
24
|
+
Return ONLY the JSON object.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
---
|
|
2
|
+
active: true
|
|
3
|
+
description: "Main prompt for extracting playbook entries from user interactions"
|
|
4
|
+
changelog: "v1.2.0: adds emergent skill-convention guidance for structuring multi-aspect playbook content as a small set of grouped do/avoid rules (no schema change). v1.1.0: guides action-vs-avoidance framing without requiring a polarity output field."
|
|
5
|
+
variables:
|
|
6
|
+
- interactions
|
|
7
|
+
---
|
|
8
|
+
## Conversation
|
|
9
|
+
{interactions}
|
|
10
|
+
|
|
11
|
+
━━━━━━━━━━━━━━━━━━━━━━
|
|
12
|
+
## Action vs avoidance framing
|
|
13
|
+
|
|
14
|
+
Write each playbook in the form that best matches its evidence:
|
|
15
|
+
|
|
16
|
+
- Use direct action language for successful, neutral, or ambiguous evidence. This is the default and covers most entries.
|
|
17
|
+
- Use avoidance language only when the specific rule is grounded in a clear failure pattern: user pushback, self-correction away from an approach, external refutation, or explicit dislike.
|
|
18
|
+
|
|
19
|
+
When writing an avoidance rule, start `content` with `Avoid`, `Do not`, `Don't`, or `Never`, and make the `rationale` name the observed failure pattern. Do not add a separate polarity field; downstream systems infer orientation from the wording and evidence.
|
|
20
|
+
|
|
21
|
+
━━━━━━━━━━━━━━━━━━━━━━
|
|
22
|
+
## Structuring multi-aspect content
|
|
23
|
+
|
|
24
|
+
When the guidance for a playbook covers multiple steps or sub-aspects of a task, write `content` as a short set of rules grouped by sub-goal, rather than one dense sentence. Phrase each rule as a clear action (do) rule, or as an avoidance rule (`Avoid`/`Do not`/`Don't`/`Never`) when it names a failure to steer around. Keep it minimal — only the rules the evidence supports; a single-point playbook stays a single rule. Do not force structure where the guidance is atomic.
|
|
25
|
+
|
|
26
|
+
━━━━━━━━━━━━━━━━━━━━━━
|
|
27
|
+
## Output
|
|
28
|
+
|
|
29
|
+
Return ONLY the JSON object.
|