claude-smart 0.2.48 → 0.2.50

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/.claude-plugin/marketplace.json +2 -2
  2. package/README.md +11 -40
  3. package/bin/claude-smart.js +393 -55
  4. package/package.json +3 -2
  5. package/plugin/.claude-plugin/plugin.json +1 -1
  6. package/plugin/.codex-plugin/plugin.json +1 -1
  7. package/plugin/.coverage +0 -0
  8. package/plugin/README.md +4 -3
  9. package/plugin/dashboard/app/api/reflexio/[...path]/route.ts +4 -2
  10. package/plugin/dashboard/app/dashboard/page.tsx +6 -1
  11. package/plugin/dashboard/app/preferences/[id]/page.tsx +18 -6
  12. package/plugin/dashboard/app/preferences/page.tsx +32 -35
  13. package/plugin/dashboard/app/sessions/[sessionId]/page.tsx +16 -1
  14. package/plugin/dashboard/app/sessions/page.tsx +2 -0
  15. package/plugin/dashboard/app/skills/page.tsx +65 -50
  16. package/plugin/dashboard/app/skills/project/[id]/page.tsx +17 -8
  17. package/plugin/dashboard/app/skills/shared/[id]/page.tsx +1 -6
  18. package/plugin/dashboard/components/common/host-badge.tsx +118 -0
  19. package/plugin/dashboard/components/common/learning-application-badge.tsx +34 -0
  20. package/plugin/dashboard/components/common/learnings-badge.tsx +1 -1
  21. package/plugin/dashboard/components/common/page-header.tsx +3 -3
  22. package/plugin/dashboard/lib/config-file.ts +5 -1
  23. package/plugin/dashboard/lib/host-attribution.ts +62 -0
  24. package/plugin/dashboard/lib/session-reader.ts +40 -2
  25. package/plugin/dashboard/lib/types.ts +7 -1
  26. package/plugin/opencode/dist/server.mjs +60 -2
  27. package/plugin/opencode/server.mts +64 -2
  28. package/plugin/pyproject.toml +2 -2
  29. package/plugin/scripts/_lib.sh +255 -7
  30. package/plugin/scripts/backend-python-runner.py +46 -0
  31. package/plugin/scripts/backend-service.sh +766 -123
  32. package/plugin/scripts/codex-hook.js +79 -229
  33. package/plugin/scripts/dashboard-open.sh +6 -4
  34. package/plugin/scripts/dashboard-service.sh +118 -137
  35. package/plugin/scripts/ensure-plugin-root.sh +106 -8
  36. package/plugin/scripts/hook_entry.sh +3 -0
  37. package/plugin/scripts/opencode-claude-compat.js +8 -1
  38. package/plugin/scripts/smart-install.sh +116 -21
  39. package/plugin/src/README.md +1 -1
  40. package/plugin/src/claude_smart/cli.py +93 -29
  41. package/plugin/src/claude_smart/context_inject.py +3 -0
  42. package/plugin/src/claude_smart/env_config.py +56 -11
  43. package/plugin/src/claude_smart/events/post_tool.py +2 -1
  44. package/plugin/src/claude_smart/events/session_end.py +2 -1
  45. package/plugin/src/claude_smart/events/stop.py +3 -0
  46. package/plugin/src/claude_smart/events/user_prompt.py +2 -1
  47. package/plugin/src/claude_smart/internal_call.py +5 -2
  48. package/plugin/src/claude_smart/optimizer_assistant.py +59 -13
  49. package/plugin/src/claude_smart/publish.py +59 -7
  50. package/plugin/src/claude_smart/reflexio_adapter.py +137 -14
  51. package/plugin/src/claude_smart/runtime.py +15 -6
  52. package/plugin/src/claude_smart/state.py +211 -52
  53. package/plugin/uv.lock +5 -5
  54. package/plugin/vendor/reflexio/.env.example +13 -0
  55. package/plugin/vendor/reflexio/reflexio/README.md +7 -3
  56. package/plugin/vendor/reflexio/reflexio/__init__.py +12 -0
  57. package/plugin/vendor/reflexio/reflexio/cli/README.md +3 -0
  58. package/plugin/vendor/reflexio/reflexio/client/client.py +166 -9
  59. package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/src/openclaw_smart/state.py +10 -3
  60. package/plugin/vendor/reflexio/reflexio/integrations/openclaw/plugin/tests/test_state.py +28 -0
  61. package/plugin/vendor/reflexio/reflexio/lib/_search.py +41 -0
  62. package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/entities.py +195 -25
  63. package/plugin/vendor/reflexio/reflexio/models/api_schema/eval_overview_schema.py +7 -1
  64. package/plugin/vendor/reflexio/reflexio/models/api_schema/internal_schema.py +2 -1
  65. package/plugin/vendor/reflexio/reflexio/models/api_schema/retriever_schema.py +63 -4
  66. package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/converters.py +1 -0
  67. package/plugin/vendor/reflexio/reflexio/models/api_schema/ui/entities.py +8 -1
  68. package/plugin/vendor/reflexio/reflexio/models/config_schema.py +1 -1
  69. package/plugin/vendor/reflexio/reflexio/server/README.md +22 -4
  70. package/plugin/vendor/reflexio/reflexio/server/__init__.py +18 -8
  71. package/plugin/vendor/reflexio/reflexio/server/__main__.py +6 -0
  72. package/plugin/vendor/reflexio/reflexio/server/api.py +79 -5
  73. package/plugin/vendor/reflexio/reflexio/server/billing_meter.py +263 -3
  74. package/plugin/vendor/reflexio/reflexio/server/callback_executor.py +164 -0
  75. package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_embedding.py +81 -81
  76. package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_subprocess.py +28 -0
  77. package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_text_generation.py +63 -6
  78. package/plugin/vendor/reflexio/reflexio/server/llm/embedding_service.py +19 -52
  79. package/plugin/vendor/reflexio/reflexio/server/llm/litellm_client.py +1 -0
  80. package/plugin/vendor/reflexio/reflexio/server/llm/model_defaults.py +28 -0
  81. package/plugin/vendor/reflexio/reflexio/server/llm/providers/claude_code_provider.py +9 -1
  82. package/plugin/vendor/reflexio/reflexio/server/llm/providers/embedder_warmup.py +329 -0
  83. package/plugin/vendor/reflexio/reflexio/server/llm/providers/embedding_service_provider.py +85 -10
  84. package/plugin/vendor/reflexio/reflexio/server/llm/providers/local_embedding_provider.py +199 -2
  85. package/plugin/vendor/reflexio/reflexio/server/llm/providers/nomic_embedding_provider.py +77 -9
  86. package/plugin/vendor/reflexio/reflexio/server/org_fanout.py +184 -0
  87. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/document_expansion/v1.0.0.prompt.md +1 -1
  88. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/document_expansion/v1.1.0.prompt.md +32 -0
  89. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.3.3.prompt.md +1 -1
  90. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/playbook_consolidation/v2.4.0.prompt.md +63 -0
  91. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/query_reformulation/v1.0.0.prompt.md +1 -1
  92. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/query_reformulation/v2.0.0.prompt.md +30 -0
  93. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/retrieved_learning_impact/v1.0.0.prompt.md +51 -0
  94. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/retrieved_learning_relevance/v1.0.0.prompt.md +39 -0
  95. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/shadow_comparison/v1.0.0.prompt.md +1 -1
  96. package/plugin/vendor/reflexio/reflexio/server/prompt/prompt_bank/shadow_comparison/v1.1.0.prompt.md +43 -0
  97. package/plugin/vendor/reflexio/reflexio/server/routes/config.py +3 -3
  98. package/plugin/vendor/reflexio/reflexio/server/routes/evaluation.py +122 -28
  99. package/plugin/vendor/reflexio/reflexio/server/routes/interactions.py +63 -2
  100. package/plugin/vendor/reflexio/reflexio/server/routes/system.py +22 -3
  101. package/plugin/vendor/reflexio/reflexio/server/scheduling.py +64 -3
  102. package/plugin/vendor/reflexio/reflexio/server/services/README.md +6 -4
  103. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/README.md +3 -2
  104. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/_eval_health.py +41 -0
  105. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/components/retrieved_learning_evaluator.py +554 -0
  106. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/regen_jobs.py +35 -1
  107. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/runner.py +253 -101
  108. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/scheduler.py +5 -7
  109. package/plugin/vendor/reflexio/reflexio/server/services/agent_success_evaluation/service.py +27 -0
  110. package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_extraction_lifecycle.py +11 -5
  111. package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_should_run.py +4 -4
  112. package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_usage_billing.py +6 -2
  113. package/plugin/vendor/reflexio/reflexio/server/services/base_generation_service.py +255 -74
  114. package/plugin/vendor/reflexio/reflexio/server/services/deduplication_utils.py +72 -0
  115. package/plugin/vendor/reflexio/reflexio/server/services/deferred_learning_plan.py +270 -0
  116. package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/__init__.py +13 -0
  117. package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/scheduler.py +142 -0
  118. package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/worker.py +262 -0
  119. package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/components/hero_state.py +2 -11
  120. package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/components/rule_attribution.py +6 -2
  121. package/plugin/vendor/reflexio/reflexio/server/services/evaluation_overview/service.py +17 -13
  122. package/plugin/vendor/reflexio/reflexio/server/services/extraction/agent_run_records.py +18 -3
  123. package/plugin/vendor/reflexio/reflexio/server/services/extraction/outcome.py +12 -1
  124. package/plugin/vendor/reflexio/reflexio/server/services/extraction/prior_answer_search.py +1 -1
  125. package/plugin/vendor/reflexio/reflexio/server/services/extraction/resumable_agent.py +2 -1
  126. package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_scheduler.py +1 -3
  127. package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_worker.py +66 -0
  128. package/plugin/vendor/reflexio/reflexio/server/services/generation_service.py +640 -80
  129. package/plugin/vendor/reflexio/reflexio/server/services/governance/service.py +2 -0
  130. package/plugin/vendor/reflexio/reflexio/server/services/lineage/gc_scheduler.py +179 -99
  131. package/plugin/vendor/reflexio/reflexio/server/services/lineage/vector_backfill_sweep.py +139 -0
  132. package/plugin/vendor/reflexio/reflexio/server/services/operation_state_utils.py +66 -27
  133. package/plugin/vendor/reflexio/reflexio/server/services/playbook/aggregation_trigger.py +177 -0
  134. package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/aggregator.py +68 -2
  135. package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/consolidator.py +360 -49
  136. package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/extractor.py +27 -30
  137. package/plugin/vendor/reflexio/reflexio/server/services/playbook/playbook_edit_apply.py +8 -1
  138. package/plugin/vendor/reflexio/reflexio/server/services/playbook/service.py +137 -69
  139. package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/optimizer.py +20 -1
  140. package/plugin/vendor/reflexio/reflexio/server/services/playbook_optimizer/scheduler.py +13 -6
  141. package/plugin/vendor/reflexio/reflexio/server/services/pre_retrieval/_query_reformulator.py +40 -25
  142. package/plugin/vendor/reflexio/reflexio/server/services/profile/components/consolidator.py +12 -39
  143. package/plugin/vendor/reflexio/reflexio/server/services/profile/components/extractor.py +30 -35
  144. package/plugin/vendor/reflexio/reflexio/server/services/profile/service.py +166 -66
  145. package/plugin/vendor/reflexio/reflexio/server/services/reflection/service.py +457 -107
  146. package/plugin/vendor/reflexio/reflexio/server/services/retrieval/session_dedup.py +127 -0
  147. package/plugin/vendor/reflexio/reflexio/server/services/retrieval/temporal.py +104 -0
  148. package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/dispatcher.py +139 -0
  149. package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/judge.py +16 -6
  150. package/plugin/vendor/reflexio/reflexio/server/services/shadow_comparison/worker.py +137 -0
  151. package/plugin/vendor/reflexio/reflexio/server/services/storage/governance_validation.py +12 -0
  152. package/plugin/vendor/reflexio/reflexio/server/services/storage/lifecycle_filters.py +54 -0
  153. package/plugin/vendor/reflexio/reflexio/server/services/storage/retention.py +41 -3
  154. package/plugin/vendor/reflexio/reflexio/server/services/storage/retention_mixin.py +40 -2
  155. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/__init__.py +2 -0
  156. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_base.py +190 -2
  157. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_extras.py +11 -4
  158. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_governance.py +28 -1
  159. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py +414 -0
  160. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_lineage.py +11 -5
  161. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_requests.py +8 -3
  162. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_shadow_verdicts.py +4 -0
  163. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/base/_deletion.py +16 -5
  164. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/base/_fts_vec.py +86 -2
  165. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_erase_execution.py +104 -42
  166. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_subject_barrier.py +7 -1
  167. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_agent.py +59 -7
  168. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_eval_results.py +430 -5
  169. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_source_linkage.py +8 -3
  170. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_user.py +57 -12
  171. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_interaction_store.py +197 -12
  172. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_profile_store.py +70 -11
  173. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/__init__.py +17 -0
  174. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_commit_scope.py +9 -0
  175. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_extras.py +9 -2
  176. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_learning_jobs.py +269 -0
  177. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_operations.py +7 -0
  178. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_retrieval_log.py +3 -1
  179. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_shadow_verdicts.py +4 -0
  180. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/agent_run/_models.py +12 -1
  181. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/evaluation_state_keys.py +78 -0
  182. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_agent.py +38 -0
  183. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_eval_results.py +171 -0
  184. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/playbook/_user.py +52 -1
  185. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/_interaction_store.py +82 -1
  186. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/_profile_store.py +74 -2
  187. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/retrieved_learning_state.py +226 -0
  188. package/plugin/vendor/reflexio/reflexio/server/services/tagging/tagging_scheduler.py +5 -6
  189. package/plugin/vendor/reflexio/reflexio/server/services/unified_search_service.py +209 -29
  190. package/plugin/vendor/reflexio/reflexio/server/usage_metrics.py +3 -0
  191. package/plugin/vendor/reflexio/reflexio/test_support/llm_mock.py +61 -0
  192. package/plugin/vendor/reflexio/reflexio/test_support/llm_model_registry.py +33 -0
  193. package/plugin/vendor/reflexio/reflexio/server/services/search/__init__.py +0 -0
@@ -18,9 +18,23 @@ from __future__ import annotations
18
18
  import importlib.util
19
19
  import logging
20
20
  import os
21
+ import shutil
21
22
  import threading
23
+ from collections.abc import Iterator
24
+ from contextlib import contextmanager
25
+ from pathlib import Path
22
26
  from typing import Any
23
27
 
28
+ try:
29
+ import fcntl
30
+ except ImportError: # pragma: no cover - Windows only
31
+ fcntl = None # type: ignore[assignment]
32
+
33
+ try:
34
+ import msvcrt
35
+ except ImportError: # pragma: no cover - POSIX only
36
+ msvcrt = None # type: ignore[assignment]
37
+
24
38
  _LOGGER = logging.getLogger(__name__)
25
39
 
26
40
  _ENV_ENABLE = "CLAUDE_SMART_USE_LOCAL_EMBEDDING"
@@ -36,6 +50,17 @@ _TARGET_DIM = 512
36
50
  # ValueError that ONNXMiniLM_L6_V2 throws on over-length input.
37
51
  _MAX_CHARS = 800
38
52
 
53
+ # Chroma keeps this file list private inside ONNXMiniLM_L6_V2's download helper.
54
+ # Keep this in sync with chromadb's all-MiniLM-L6-v2 extracted archive layout.
55
+ _MINILM_EXPECTED_FILES = (
56
+ "config.json",
57
+ "model.onnx",
58
+ "special_tokens_map.json",
59
+ "tokenizer_config.json",
60
+ "tokenizer.json",
61
+ "vocab.txt",
62
+ )
63
+
39
64
 
40
65
  class LocalEmbedderError(RuntimeError):
41
66
  """Raised when the local embedder is called without chromadb installed."""
@@ -49,7 +74,17 @@ class LocalEmbedder:
49
74
 
50
75
  def __init__(self) -> None:
51
76
  self._ef: Any | None = None
77
+ # Guards lazy construction / cache-recovery swaps of ``self._ef``.
52
78
  self._ef_lock = threading.Lock()
79
+ # Serializes the actual encode call. The shared ONNX embedding function
80
+ # is NOT thread-safe — concurrent ``ef(...)`` calls interleave and
81
+ # corrupt each other's padding/attention buffers (observed in prod as
82
+ # tensor-shape mismatches). ``threading.Lock`` is non-reentrant, so this
83
+ # is a SEPARATE lock from ``_ef_lock`` (the recovery path re-acquires
84
+ # ``_ef_lock`` while we hold this one): fixed order is ``_encode_lock``
85
+ # outer, ``_ef_lock`` inner, which cannot deadlock. NOTE: this lock is
86
+ # not FIFO-fair — do not turn it into a fairness queue without measuring.
87
+ self._encode_lock = threading.Lock()
53
88
 
54
89
  @classmethod
55
90
  def get(cls) -> LocalEmbedder:
@@ -96,11 +131,61 @@ class LocalEmbedder:
96
131
  ``_TARGET_DIM`` (512) floats with the last 128 positions
97
132
  zero-padded.
98
133
  """
99
- ef = self._load()
100
134
  safe_inputs = [(text or "")[:_MAX_CHARS] for text in texts]
101
- raw = ef(safe_inputs)
135
+ # Serialize the encode: the shared ONNX embedder is not thread-safe.
136
+ # ``_load()`` and ``_retry_embed_after_cache_clear()`` both acquire
137
+ # ``_ef_lock`` internally; running them under ``_encode_lock`` fixes the
138
+ # lock order (``_encode_lock`` outer) so recovery can never deadlock.
139
+ with self._encode_lock:
140
+ ef = self._load()
141
+ try:
142
+ raw = ef(safe_inputs)
143
+ except Exception as exc: # noqa: BLE001 - Chroma raises varied cache errors.
144
+ raw = self._retry_embed_after_cache_clear(ef, exc, safe_inputs)
102
145
  return [_pad(vec) for vec in raw]
103
146
 
147
+ def _retry_embed_after_cache_clear(
148
+ self, failed_ef: Any, exc: Exception, safe_inputs: list[str]
149
+ ) -> Any:
150
+ with self._ef_lock:
151
+ if self._ef is not None and self._ef is not failed_ef:
152
+ return self._ef(safe_inputs)
153
+
154
+ embedding_cls = type(failed_ef)
155
+ recovery = _recoverable_minilm_cache(embedding_cls, exc)
156
+ if recovery is None:
157
+ raise exc
158
+ cache_path, cache_was_complete = recovery
159
+
160
+ with _exclusive_file_lock(_minilm_cache_lock_path(cache_path)):
161
+ if _should_clear_minilm_cache(
162
+ embedding_cls, cache_path, exc, cache_was_complete
163
+ ):
164
+ shutil.rmtree(cache_path, ignore_errors=True)
165
+ recovery_action = "clearing"
166
+ _LOGGER.warning(
167
+ "Failed to use local MiniLM embedder; cleared cache %s "
168
+ "and retrying once",
169
+ cache_path,
170
+ exc_info=True,
171
+ )
172
+ else:
173
+ recovery_action = "waiting for another process to refresh"
174
+ _LOGGER.info(
175
+ "MiniLM cache %s was refreshed by another process; retrying",
176
+ cache_path,
177
+ )
178
+ try:
179
+ fresh_ef = embedding_cls()
180
+ self._ef = fresh_ef
181
+ return fresh_ef(safe_inputs)
182
+ except Exception as retry_exc:
183
+ raise LocalEmbedderError(
184
+ "Local MiniLM cache recovery failed after "
185
+ f"{recovery_action} {cache_path}. Delete this cache "
186
+ "directory, restart Reflexio, and retry local embedding."
187
+ ) from retry_exc
188
+
104
189
 
105
190
  def _pad(vec: Any) -> list[float]:
106
191
  """Zero-pad a 384-dim vector to ``_TARGET_DIM`` as a plain list[float]."""
@@ -113,6 +198,118 @@ def _pad(vec: Any) -> list[float]:
113
198
  return floats + [0.0] * (_TARGET_DIM - len(floats))
114
199
 
115
200
 
201
+ def _recoverable_minilm_cache(
202
+ embedding_cls: Any, exc: Exception
203
+ ) -> tuple[Path, bool] | None:
204
+ cache_path = _minilm_cache_path(embedding_cls)
205
+ if cache_path is None:
206
+ return None
207
+ cache_was_complete = _minilm_cache_complete(embedding_cls, cache_path)
208
+ if cache_was_complete and not _complete_minilm_cache_error_identifies_cache(
209
+ embedding_cls, cache_path, exc
210
+ ):
211
+ return None
212
+
213
+ return cache_path, cache_was_complete
214
+
215
+
216
+ def _minilm_cache_path(embedding_cls: Any) -> Path | None:
217
+ download_path = getattr(embedding_cls, "DOWNLOAD_PATH", None)
218
+ if not download_path:
219
+ return None
220
+
221
+ model_name = getattr(embedding_cls, "MODEL_NAME", None)
222
+ cache_path = Path(str(download_path)).expanduser()
223
+ if not model_name or cache_path.name != model_name:
224
+ return None
225
+
226
+ return cache_path
227
+
228
+
229
+ def _minilm_cache_lock_path(cache_path: Path) -> Path:
230
+ return cache_path.parent / f".{cache_path.name}.lock"
231
+
232
+
233
+ @contextmanager
234
+ def _exclusive_file_lock(lock_path: Path) -> Iterator[None]:
235
+ lock_path.parent.mkdir(parents=True, exist_ok=True)
236
+ with lock_path.open("a+b") as lock_file:
237
+ lock_file.seek(0)
238
+ if lock_file.read(1) == b"":
239
+ lock_file.write(b"\0")
240
+ lock_file.flush()
241
+ lock_file.seek(0)
242
+
243
+ if fcntl is not None:
244
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
245
+ try:
246
+ yield
247
+ finally:
248
+ fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN)
249
+ elif msvcrt is not None:
250
+ msvcrt.locking( # type: ignore[reportAttributeAccessIssue]
251
+ lock_file.fileno(),
252
+ msvcrt.LK_LOCK, # type: ignore[reportAttributeAccessIssue]
253
+ 1,
254
+ )
255
+ try:
256
+ yield
257
+ finally:
258
+ lock_file.seek(0)
259
+ msvcrt.locking( # type: ignore[reportAttributeAccessIssue]
260
+ lock_file.fileno(),
261
+ msvcrt.LK_UNLCK, # type: ignore[reportAttributeAccessIssue]
262
+ 1,
263
+ )
264
+ else:
265
+ yield
266
+
267
+
268
+ def _should_clear_minilm_cache(
269
+ embedding_cls: Any, cache_path: Path, exc: Exception, cache_was_complete: bool
270
+ ) -> bool:
271
+ cache_is_complete = _minilm_cache_complete(embedding_cls, cache_path)
272
+ if not cache_is_complete:
273
+ return True
274
+
275
+ if not cache_was_complete:
276
+ return False
277
+
278
+ return _complete_minilm_cache_error_identifies_cache(embedding_cls, cache_path, exc)
279
+
280
+
281
+ def _complete_minilm_cache_error_identifies_cache(
282
+ embedding_cls: Any, cache_path: Path, exc: Exception
283
+ ) -> bool:
284
+ chain = _exception_chain(exc)
285
+ archive_name = str(getattr(embedding_cls, "ARCHIVE_FILENAME", ""))
286
+ messages = "\n".join(str(chain_exc) for chain_exc in chain)
287
+ return (
288
+ str(cache_path) in messages
289
+ or bool(archive_name and archive_name in messages)
290
+ or "does not match expected SHA256" in messages
291
+ )
292
+
293
+
294
+ def _minilm_cache_complete(embedding_cls: Any, cache_path: Path) -> bool:
295
+ extracted_folder = str(getattr(embedding_cls, "EXTRACTED_FOLDER_NAME", "onnx"))
296
+ return all(
297
+ (cache_path / extracted_folder / file_name).exists()
298
+ for file_name in _MINILM_EXPECTED_FILES
299
+ )
300
+
301
+
302
+ def _exception_chain(exc: Exception) -> list[BaseException]:
303
+ chain: list[BaseException] = []
304
+ seen: set[int] = set()
305
+ current: BaseException | None = exc
306
+ while current is not None and id(current) not in seen:
307
+ seen.add(id(current))
308
+ chain.append(current)
309
+ current = current.__cause__ or current.__context__
310
+ return chain
311
+
312
+
116
313
  _REGISTERED = False
117
314
 
118
315
 
@@ -63,12 +63,64 @@ _MAX_CHARS = 32_000
63
63
  _DEFAULT_ENCODE_BATCH_SIZE = 4
64
64
  _ENV_BATCH_SIZE = "REFLEXIO_EMBED_BATCH_SIZE"
65
65
 
66
+ # D7 (Phase 2 prep): optionally pin torch intra-op thread count so the embedder
67
+ # does not oversubscribe CPU under bounded concurrency. Unset (the default) =
68
+ # leave torch untouched, i.e. zero behaviour change. Applies at Nomic model load
69
+ # (both the in-process fallback and the shared daemon run NomicEmbedder). Note the
70
+ # chromadb/minilm LocalEmbedder path uses onnxruntime, not torch, so this knob does
71
+ # not affect it.
72
+ _ENV_TORCH_THREADS = "REFLEXIO_EMBED_TORCH_THREADS"
73
+ _torch_threads_pinned = False
74
+ _torch_threads_lock = threading.Lock()
75
+
66
76
 
67
77
  def _encode_batch_size() -> int:
68
78
  """Resolve the encode mini-batch size from env, defaulting to 4."""
69
79
  return positive_int_env(_ENV_BATCH_SIZE, _DEFAULT_ENCODE_BATCH_SIZE, _LOGGER)
70
80
 
71
81
 
82
+ def _maybe_pin_torch_threads() -> None:
83
+ """Pin torch intra-op threads to ``REFLEXIO_EMBED_TORCH_THREADS`` if set.
84
+
85
+ Dormant by default: when the env var is unset (or blank) this is a no-op and
86
+ ``torch.set_num_threads`` is never called, so torch keeps its default
87
+ autodetected thread count. When set to a positive int it is applied exactly
88
+ once per process (guarded against re-calling). A non-positive or non-integer
89
+ value is ignored with a warning.
90
+ """
91
+ global _torch_threads_pinned
92
+ if _torch_threads_pinned:
93
+ return
94
+ raw = os.environ.get(_ENV_TORCH_THREADS)
95
+ if not raw:
96
+ return
97
+ try:
98
+ n = int(raw)
99
+ except ValueError:
100
+ _LOGGER.warning(
101
+ "%s must be a positive integer; got %r — leaving torch threads at "
102
+ "their default.",
103
+ _ENV_TORCH_THREADS,
104
+ raw,
105
+ )
106
+ return
107
+ if n < 1:
108
+ _LOGGER.warning(
109
+ "%s must be >= 1; got %d — leaving torch threads at their default.",
110
+ _ENV_TORCH_THREADS,
111
+ n,
112
+ )
113
+ return
114
+ with _torch_threads_lock:
115
+ if _torch_threads_pinned:
116
+ return
117
+ import torch
118
+
119
+ torch.set_num_threads(n)
120
+ _torch_threads_pinned = True
121
+ _LOGGER.info("Pinned torch intra-op threads to %d (%s)", n, _ENV_TORCH_THREADS)
122
+
123
+
72
124
  class NomicEmbedderError(RuntimeError):
73
125
  """Raised when the Nomic embedder is requested but its deps are missing."""
74
126
 
@@ -118,6 +170,7 @@ class NomicEmbedder:
118
170
  "sentence-transformers is required for the Nomic local "
119
171
  "embedder. Install with `uv add sentence-transformers`."
120
172
  ) from exc
173
+ _maybe_pin_torch_threads()
121
174
  _LOGGER.info(
122
175
  "Loading Nomic embedding model %s — first call may download "
123
176
  "~550 MB to %s",
@@ -157,15 +210,30 @@ class NomicEmbedder:
157
210
  """
158
211
  model = self._load()
159
212
  safe = [(t or "")[:_MAX_CHARS] for t in texts]
160
- # show_progress_bar=False so server logs stay clean during ingest
161
- # batches. convert_to_numpy=True returns a numpy ndarray; we slice
162
- # and renormalise per-row before converting to plain Python lists.
163
- raw = model.encode(
164
- safe,
165
- batch_size=_encode_batch_size(),
166
- show_progress_bar=False,
167
- convert_to_numpy=True,
168
- )
213
+ # The sentence-transformers model is NOT thread-safe: nomic-bert's
214
+ # rotary-embedding code mutates instance attrs (``_cos_cached`` /
215
+ # ``_sin_cached``) on every forward pass, so two concurrent
216
+ # ``encode()`` calls on this shared singleton corrupt each other's
217
+ # buffers (observed in prod as "size of tensor a (N) must match tensor
218
+ # b (M)" and "'NoneType' object is not subscriptable"). Serialize the
219
+ # encode so every caller — daemon, in-process fallback, prewarm,
220
+ # regeneration — is safe by construction. ``_load()`` acquires and
221
+ # releases ``_model_lock`` and returns before we re-acquire it here, so
222
+ # there is no nesting/deadlock. NOTE: ``threading.Lock`` is not FIFO-fair
223
+ # — do not "improve" this into a fairness queue without measuring; the
224
+ # micro-batch coalescing in ``embedding_service`` is where throughput
225
+ # comes from, not parallel encodes on one model.
226
+ with self._model_lock:
227
+ # show_progress_bar=False so server logs stay clean during ingest
228
+ # batches. convert_to_numpy=True returns a numpy ndarray; we slice
229
+ # and renormalise per-row (below, outside the lock) before
230
+ # converting to plain Python lists.
231
+ raw = model.encode(
232
+ safe,
233
+ batch_size=_encode_batch_size(),
234
+ show_progress_bar=False,
235
+ convert_to_numpy=True,
236
+ )
169
237
  return [_truncate_and_renormalise(vec.tolist()) for vec in raw]
170
238
 
171
239
 
@@ -0,0 +1,184 @@
1
+ """Bounded parallel org iteration for org-walking daemon ticks.
2
+
3
+ Replaces serial ``for org_id in org_ids`` loops in daemon ticks with a
4
+ bounded per-tick thread pool (spec section 6.1). The pool is created per call
5
+ and abandoned (not joined) at the end so a stuck org cannot eat a slot across
6
+ ticks; stragglers finish in the background. Python cannot kill threads —
7
+ "timeout" means stop waiting + log, and the caller escalates repeats.
8
+
9
+ Each org's actual start/end time is tracked, so timeout classification is
10
+ honest about queue-wait vs. run-time: an org that never got a worker before
11
+ its wait elapsed is STARVED (not counted as a timeout, not retried); an org
12
+ that has run for less than the budget gets one more wait for the remaining
13
+ budget; only an org that has genuinely run for >= the budget is reported as
14
+ timed out. Each future is waited on for at most ~2x the budget.
15
+
16
+ Abandonment is per-tick, not process-wide: ``ThreadPoolExecutor`` worker
17
+ threads are non-daemon, so CPython still joins ALL of them at interpreter
18
+ exit — a forever-hung ``fn`` can block graceful process shutdown until the
19
+ orchestrator's kill grace expires; ``shutdown(wait=False)`` here only frees
20
+ the NEXT tick's capacity, not exit-time joining.
21
+
22
+ Error isolation is the caller's contract: ``fn`` must catch its own per-org
23
+ exceptions (LineageGC's sweep body already does); this helper only
24
+ backstop-logs unexpected escapes.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ import logging
30
+ import threading
31
+ import time
32
+ from collections.abc import Callable, Iterable
33
+ from concurrent.futures import Future, ThreadPoolExecutor
34
+ from concurrent.futures import TimeoutError as FuturesTimeoutError
35
+
36
+ logger = logging.getLogger(__name__)
37
+
38
+
39
+ def iterate_orgs_bounded(
40
+ org_ids: Iterable[str],
41
+ fn: Callable[[str], None],
42
+ *,
43
+ max_workers: int,
44
+ per_org_timeout_seconds: float,
45
+ stop_event: threading.Event | None = None,
46
+ ) -> list[str]:
47
+ """Run ``fn(org_id)`` for each org with bounded concurrency and a per-org timeout.
48
+
49
+ Args:
50
+ org_ids (Iterable[str]): Orgs to process this tick.
51
+ fn (Callable[[str], None]): Per-org work body; must handle its own errors.
52
+ max_workers (int): Pool width; ``<= 1`` runs serially with no pool.
53
+ per_org_timeout_seconds (float): Run-time budget per org (not
54
+ queue-wait time). An org that never got a worker before its wait
55
+ elapsed is logged as ``event=org_sweep_starved`` and excluded
56
+ from the result (not timed out, not retried). An org that has
57
+ actually run for less than the budget is given one more wait for
58
+ the remaining budget; only once it has genuinely run for at
59
+ least the budget is it logged as ``event=org_sweep_timeout`` and
60
+ moved on (the straggler thread keeps running in the background).
61
+ stop_event (threading.Event | None): When set, stop submitting new
62
+ orgs; in-flight orgs finish (matches serial-loop shutdown semantics).
63
+
64
+ Returns:
65
+ list[str]: Org ids that genuinely timed out this call (ran for at
66
+ least the budget without finishing); starved orgs are excluded.
67
+ Callers track repeats across ticks and escalate.
68
+ """
69
+ timed_out: list[str] = []
70
+ if max_workers <= 1:
71
+ for org_id in org_ids:
72
+ if stop_event is not None and stop_event.is_set():
73
+ break
74
+ fn(org_id)
75
+ return timed_out
76
+
77
+ starts: dict[str, float] = {}
78
+ ends: dict[str, float] = {}
79
+ record_lock = threading.Lock()
80
+
81
+ def _tracked(org_id: str) -> None:
82
+ with record_lock:
83
+ starts[org_id] = time.monotonic()
84
+ try:
85
+ fn(org_id)
86
+ finally:
87
+ with record_lock:
88
+ ends[org_id] = time.monotonic()
89
+
90
+ executor = ThreadPoolExecutor(
91
+ max_workers=max_workers, thread_name_prefix="org-fanout"
92
+ )
93
+ futures: list[tuple[str, Future[None]]] = []
94
+ try:
95
+ for org_id in org_ids:
96
+ if stop_event is not None and stop_event.is_set():
97
+ break
98
+ futures.append((org_id, executor.submit(_tracked, org_id)))
99
+ for org_id, future in futures:
100
+ if _wait_for_org(
101
+ org_id,
102
+ future,
103
+ per_org_timeout_seconds=per_org_timeout_seconds,
104
+ starts=starts,
105
+ ends=ends,
106
+ lock=record_lock,
107
+ ):
108
+ timed_out.append(org_id)
109
+ finally:
110
+ # Per-tick pool, abandoned not joined (spec 6.1): stragglers finish in
111
+ # the background; the NEXT tick gets a fresh pool and full width.
112
+ executor.shutdown(wait=False)
113
+ return timed_out
114
+
115
+
116
+ def _wait_for_org(
117
+ org_id: str,
118
+ future: Future[None],
119
+ *,
120
+ per_org_timeout_seconds: float,
121
+ starts: dict[str, float],
122
+ ends: dict[str, float],
123
+ lock: threading.Lock,
124
+ ) -> bool:
125
+ """Wait for one org's future; return True iff it genuinely timed out.
126
+
127
+ Classifies a ``FuturesTimeoutError`` honestly instead of treating
128
+ queue-wait as run time: an org with no recorded start never got a
129
+ worker (STARVED — logged, not counted as a timeout, no further wait); an
130
+ org that has run for less than the budget gets one more wait for the
131
+ remaining budget. Total wait per org is bounded to ~2x the budget.
132
+
133
+ Args:
134
+ org_id (str): The org this future belongs to.
135
+ future (Future[None]): The submitted future for ``org_id``.
136
+ per_org_timeout_seconds (float): The run-time budget for this org.
137
+ starts (dict[str, float]): Monotonic start times, keyed by org id.
138
+ ends (dict[str, float]): Monotonic end times, keyed by org id.
139
+ lock (threading.Lock): Guards ``starts``/``ends``.
140
+
141
+ Returns:
142
+ bool: True if the org genuinely timed out (ran >= the budget without
143
+ finishing); False if it completed, was starved, or raised an
144
+ unexpected error.
145
+ """
146
+ try:
147
+ future.result(timeout=per_org_timeout_seconds)
148
+ return False
149
+ except FuturesTimeoutError:
150
+ pass
151
+ except Exception:
152
+ # fn owns error isolation; anything escaping is a bug — log, never
153
+ # let one org's escape skip the remaining result waits.
154
+ logger.exception("event=org_fanout_unexpected_error org_id=%s", org_id)
155
+ return False
156
+
157
+ with lock:
158
+ start = starts.get(org_id)
159
+ finished = org_id in ends
160
+ if finished:
161
+ # Completed between the timeout firing and this check — not a timeout.
162
+ return False
163
+ if start is None:
164
+ logger.warning("event=org_sweep_starved org_id=%s", org_id)
165
+ return False
166
+
167
+ elapsed_running = time.monotonic() - start
168
+ remaining = per_org_timeout_seconds - elapsed_running
169
+ if remaining > 0:
170
+ try:
171
+ future.result(timeout=remaining)
172
+ return False
173
+ except FuturesTimeoutError:
174
+ pass
175
+ except Exception:
176
+ logger.exception("event=org_fanout_unexpected_error org_id=%s", org_id)
177
+ return False
178
+
179
+ logger.warning(
180
+ "event=org_sweep_timeout org_id=%s timeout_seconds=%s",
181
+ org_id,
182
+ per_org_timeout_seconds,
183
+ )
184
+ return True
@@ -1,5 +1,5 @@
1
1
  ---
2
- active: true
2
+ active: false
3
3
  description: "Extracts key terms and generates synonyms for document FTS expansion"
4
4
  variables:
5
5
  - content
@@ -0,0 +1,32 @@
1
+ ---
2
+ active: true
3
+ description: "Extracts key terms, synonyms, and fact-key phrasings for document FTS expansion"
4
+ changelog: "v1.1.0: also emit fact-key phrasings — (attribute: value) paraphrases and category names for brands/tools/proper nouns — so paraphrased queries hit via FTS"
5
+ variables:
6
+ - content
7
+ ---
8
+ Given this document content, produce expansion terms that make the document findable by paraphrased searches. Output valid JSON only, nothing else — a single object mapping each key concept to 2-4 related search phrasings.
9
+
10
+ Include BOTH kinds of expansion:
11
+ 1. Synonyms / closely related terms for each key concept.
12
+ 2. Fact-key phrasings: when the content states a fact (subject-attribute-value), add the attribute/category words a person would search with. In particular, map brands, products, tools, and proper nouns to their CATEGORY ("Coke" → "soda", "soft drink"; "Alembic" → "database migration tool"; "Thrive Market" → "grocery service").
13
+
14
+ Example:
15
+ Input: "User always orders Coke at restaurants"
16
+ Output:
17
+ {{
18
+ "Coke": ["soda", "soft drink", "beverage"],
19
+ "orders at restaurants": ["drink preference", "dining habit"]
20
+ }}
21
+
22
+ Example:
23
+ Input: "User reports backup failure with S3 timeout on large datasets"
24
+ Output:
25
+ {{
26
+ "backup": ["sync", "replication", "data backup"],
27
+ "failure": ["error", "crash", "issue"],
28
+ "S3": ["cloud storage", "AWS", "object storage"],
29
+ "timeout": ["latency", "delay", "slow response"]
30
+ }}
31
+
32
+ Content: {content}
@@ -1,5 +1,5 @@
1
1
  ---
2
- active: true
2
+ active: false
3
3
  description: "Reconcile newly-extracted playbooks against existing storage. Decides per-candidate whether each one unifies with one-or-more existing rows (growing a coherent multi-rule skill, mixed do/avoid rules allowed), is rejected as redundant against an existing row, should differentiate (refine both triggers), or is independent."
4
4
  changelog: "v2.3.3: adds antecedent-not-reaction trigger guidance for unified and differentiated triggers. (in-place 2026-06-15: adds concise anti-overmerge guard for concrete operational surfaces; restates the numeric id fields (`superseded_by_existing_id`, `existing_id`, `archive_existing_ids`) as EXISTING list-position integers — e.g. emit `0` for `[EXISTING-0]` — superseding the prior 'bare DB id, labels never valid' framing.) v2.3.2: tightens schema guidance for integer id fields and compact unify content; forbids display-label strings in numeric fields. v2.3.1: replaces schema example-style output prompting with compact format guidance for the four decision shapes. v2.3.0: `unify` now COMPOSES — it may grow a broader multi-rule skill from coherent related fragments (related sub-aspects of one task), not only same-trigger duplicates. A skill MAY hold mixed-polarity rules (do-rules and avoid-rules for different sub-aspects). Replaced the mechanical single-orientation/same-polarity contract with an LLM-judged no-self-contradiction guard: do NOT unify if combining the rules would make the skill contradict itself on the same situation (same trigger/condition with opposite advice) — route those to `differentiate` or `reject_new`. Re-synthesis (fewest/most-general rules, preserve avoid-detail) and the over-budget `differentiate` preference are unchanged. v2.2.0: `unify` re-synthesizes leaner, more general content (MDL / simplify-in-place) instead of concatenating — fewest, most general rules that still cover all inputs; drop redundant/subsumed wording. Asymmetric fidelity: compress/generalize success guidance, but preserve every distinct avoidance/failure detail verbatim — never soften a named pitfall into a vague platitude. Prefer `differentiate` over `unify` when merging would force an over-long, low-cohesion rule. v2.1.0: `unify` no longer emits a `polarity` field — the unified row's orientation is derived from its wording (recommendation vs avoidance) by the apply-path polarity validator. Same-trigger opposite-orientation rules still must not unify; express orientation through the rule wording. v2.0.0: collapsed 5-kind union → 4-kind. `unify` subsumes `duplicate`+`prefer_new`; `reject_new` replaces `prefer_existing`. Output schema is structurally incompatible with v1.x."
5
5
  variables:
@@ -0,0 +1,63 @@
1
+ ---
2
+ active: true
3
+ description: "Reconcile newly-extracted playbooks against existing storage. Decides per-candidate whether each one unifies with one-or-more existing rows (growing a coherent multi-rule skill, mixed do/avoid rules allowed), is rejected as redundant against an existing row, should differentiate (refine both triggers), or is independent."
4
+ changelog: "v2.4.0: makes the NEW-id partition contract explicit. Decisions must cover every NEW id exactly once; multi-NEW unify/reject_new/independent decisions may list multiple NEW labels when one decision consumes multiple rows, and archive_existing_ids is clarified as EXISTING-only. Body also tightened ~45% (deduplicated the partition contract and no-self-contradiction statements; compressed the kind definitions, unify-writing rules, and output guidance) with no behavioral rule removed. v2.3.3: adds antecedent-not-reaction trigger guidance for unified and differentiated triggers. (in-place 2026-06-15: adds concise anti-overmerge guard for concrete operational surfaces; restates the numeric id fields (`superseded_by_existing_id`, `existing_id`, `archive_existing_ids`) as EXISTING list-position integers — e.g. emit `0` for `[EXISTING-0]` — superseding the prior 'bare DB id, labels never valid' framing.) v2.3.2: tightens schema guidance for integer id fields and compact unify content; forbids display-label strings in numeric fields. v2.3.1: replaces schema example-style output prompting with compact format guidance for the four decision shapes. v2.3.0: `unify` now COMPOSES — it may grow a broader multi-rule skill from coherent related fragments (related sub-aspects of one task), not only same-trigger duplicates. A skill MAY hold mixed-polarity rules (do-rules and avoid-rules for different sub-aspects). Replaced the mechanical single-orientation/same-polarity contract with an LLM-judged no-self-contradiction guard: do NOT unify if combining the rules would make the skill contradict itself on the same situation (same trigger/condition with opposite advice) — route those to `differentiate` or `reject_new`. Re-synthesis (fewest/most-general rules, preserve avoid-detail) and the over-budget `differentiate` preference are unchanged. v2.2.0: `unify` re-synthesizes leaner, more general content (MDL / simplify-in-place) instead of concatenating — fewest, most general rules that still cover all inputs; drop redundant/subsumed wording. Asymmetric fidelity: compress/generalize success guidance, but preserve every distinct avoidance/failure detail verbatim — never soften a named pitfall into a vague platitude. Prefer `differentiate` over `unify` when merging would force an over-long, low-cohesion rule. v2.1.0: `unify` no longer emits a `polarity` field — the unified row's orientation is derived from its wording (recommendation vs avoidance) by the apply-path polarity validator. Same-trigger opposite-orientation rules still must not unify; express orientation through the rule wording. v2.0.0: collapsed 5-kind union → 4-kind. `unify` subsumes `duplicate`+`prefer_new`; `reject_new` replaces `prefer_existing`. Output schema is structurally incompatible with v1.x."
5
+ variables:
6
+ - new_playbook_count
7
+ - new_playbooks
8
+ - existing_playbooks
9
+ ---
10
+ You are reconciling newly-extracted playbooks against the related existing playbook rows already in storage. Decide each new candidate's relationship to the existing rows.
11
+
12
+ Each rendered row carries `Content`, `Trigger`, `Rationale`, `Name`, `Source`, and `Last Modified`. Read a rule's orientation (do-this vs avoid-this) from its `Content`/`Rationale` wording; differing orientation alone is not a reason to avoid merging. Compare rules primarily by `Trigger` plus the actual situation each rule addresses.
13
+
14
+ Write every final `trigger`, `refined_new_trigger`, and `refined_existing_trigger` as the antecedent situation where a future agent can act before the mistake recurs, not as the user's later repair request — e.g. "When editing one AWS deployment doc for an env-var/secret/port change", not "When syncing two AWS deployment docs."
15
+
16
+ [New playbooks (count: {new_playbook_count})]
17
+ {new_playbooks}
18
+
19
+ [Existing related playbooks]
20
+ {existing_playbooks}
21
+
22
+ # Decision kinds
23
+
24
+ Your decisions must cover every NEW id exactly once: each rendered NEW label appears in exactly one decision, and one decision may consume several NEW rows when that reflects the final outcome.
25
+
26
+ - **unify** — the NEW belongs in the same skill as one or more EXISTING rows: either dedup/supersede (same rule, or stronger/broader/more specific evidence) or compose (a related sub-aspect of the same task grows a broader multi-rule skill instead of forcing `reject_new`/`independent`). Provide the final `content`, `trigger`, and `rationale`. `archive_existing_ids` only refers to rows from the EXISTING list; it never consumes NEW rows (use an empty list when none are absorbed). If the unified rule uses facts or details from more than one NEW row, `new_id` MUST list all of those labels, e.g. `"new_id": ["NEW-0", "NEW-1"]`. A skill MAY hold mixed do/avoid rules for **different** sub-aspects of the one task; only unify fragments that genuinely share the same task scope.
27
+
28
+ - **reject_new** — an EXISTING row already covers NEW or makes it redundant. Name the winner's list position via `superseded_by_existing_id` (for `[EXISTING-0]`, emit `0`).
29
+
30
+ - **differentiate** — both rules are valid in distinct contexts (typically same trigger, opposite advice, differing contexts). Set `refined_new_trigger` and `refined_existing_trigger` strictly narrower than the originals AND mutually exclusive. Handles exactly one NEW row.
31
+
32
+ - **independent** — different topic or task from every existing row; insert NEW as-is, no archive.
33
+
34
+ `new_id` may be a list for `unify`, `reject_new`, and `independent` when one decision intentionally consumes several NEW rows. Example multi-NEW unify (the final rule uses details from both NEW rows, so both labels are consumed by the one decision):
35
+
36
+ ```json
37
+ {{"decisions": [{{"kind": "unify", "new_id": ["NEW-0", "NEW-1"], "archive_existing_ids": [], "content": "<merged rule>", "trigger": "<antecedent situation>", "rationale": "<why both rows merged>"}}]}}
38
+ ```
39
+
40
+ # How to write a unified skill (re-synthesis, not concatenation)
41
+
42
+ - Produce the fewest, most general rules that still cover all the inputs; drop redundant or subsumed wording. Keep each rule a clean, self-contained do-rule or avoid-rule for its own sub-aspect — do not blur distinct rules into one.
43
+ - Asymmetric fidelity: compress and generalize the success guidance, but preserve every distinct avoidance/failure detail in its specific form — never collapse a named pitfall, concrete error, or specific anti-pattern into a vague platitude.
44
+ - Preserve concrete operational surfaces (target groups, SGs, health checks, task defs); do not merge fanout rules into generic sweep/audit rules unless those surfaces remain explicit.
45
+ - Keep the final `content` compact enough to store directly. Prefer `differentiate` (or `independent`) over `unify` when merging would force an over-long, low-cohesion skill or lose distinct failure detail — two focused rules beat one bloated skill.
46
+
47
+ # Hard constraints
48
+
49
+ - No self-contradiction: a skill must never hold opposite advice for the SAME situation (same trigger/condition). Mixed do/avoid rules for different sub-aspects are fine and expected. A NEW + EXISTING pair with same-situation opposite advice MUST route to `differentiate` or `reject_new` — never `unify`, never `independent`. When such a stalemate is balanced, default to `reject_new`; storage stability wins ties.
50
+ - `differentiate.refined_new_trigger` and `refined_existing_trigger` MUST be non-empty and strictly narrower than the originals.
51
+ - Numeric id fields MUST contain bare EXISTING list-position integers (for `[EXISTING-3]`, emit `3`), never bracketed labels or label strings.
52
+ - Every NEW label MUST appear exactly once across the full `decisions` array. If one decision consumes several NEW rows, put those labels in that decision's `new_id` list and do not emit separate decisions for them.
53
+
54
+ # Output format
55
+
56
+ Respond ONLY with a valid JSON object matching `PlaybookConsolidationOutput`: `{{"decisions": [<decision objects that together cover every NEW id exactly once>]}}`. Every decision includes `kind` and `new_id` (a NEW label string, or a list of labels for multi-NEW `unify`/`reject_new`/`independent`); an optional `reason` may explain any decision.
57
+
58
+ - `unify`: `archive_existing_ids` (list of EXISTING position integers) plus final compact `content`, `trigger`, and `rationale`.
59
+ - `reject_new`: `superseded_by_existing_id` (EXISTING position integer).
60
+ - `differentiate`: `existing_id` (EXISTING position integer) plus non-empty `refined_new_trigger` and `refined_existing_trigger`.
61
+ - `independent`: only `kind` and `new_id`.
62
+
63
+ Do not emit markdown, prose, comments, chain-of-thought, or top-level keys other than `decisions`. Do not include fields from another decision kind.
@@ -1,5 +1,5 @@
1
1
  ---
2
- active: true
2
+ active: false
3
3
  description: "Reformulates a user search query into clean, standalone natural language"
4
4
  variables:
5
5
  - query