claude-smart 0.2.47 → 0.2.49

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (139) hide show
  1. package/.claude-plugin/marketplace.json +1 -1
  2. package/README.md +6 -2
  3. package/bin/claude-smart.js +289 -56
  4. package/package.json +1 -1
  5. package/plugin/.claude-plugin/plugin.json +1 -1
  6. package/plugin/.codex-plugin/plugin.json +1 -1
  7. package/plugin/dashboard/app/sessions/[sessionId]/page.tsx +36 -2
  8. package/plugin/dashboard/package-lock.json +61 -390
  9. package/plugin/dashboard/package.json +2 -2
  10. package/plugin/opencode/dist/server.mjs +60 -2
  11. package/plugin/opencode/server.mts +64 -2
  12. package/plugin/pyproject.toml +2 -2
  13. package/plugin/scripts/_lib.sh +58 -6
  14. package/plugin/scripts/backend-service.sh +15 -10
  15. package/plugin/scripts/codex-hook.js +16 -4
  16. package/plugin/scripts/dashboard-service.sh +1 -1
  17. package/plugin/scripts/ensure-plugin-root.sh +106 -8
  18. package/plugin/scripts/opencode-claude-compat.js +8 -1
  19. package/plugin/scripts/smart-install.sh +101 -20
  20. package/plugin/src/README.md +1 -1
  21. package/plugin/src/claude_smart/cli.py +79 -29
  22. package/plugin/src/claude_smart/env_config.py +53 -11
  23. package/plugin/uv.lock +5 -5
  24. package/plugin/vendor/reflexio/.env.example +7 -0
  25. package/plugin/vendor/reflexio/pyproject.toml +2 -1
  26. package/plugin/vendor/reflexio/reflexio/README.md +8 -5
  27. package/plugin/vendor/reflexio/reflexio/cli/README.md +3 -0
  28. package/plugin/vendor/reflexio/reflexio/client/client.py +40 -6
  29. package/plugin/vendor/reflexio/reflexio/lib/_config.py +23 -18
  30. package/plugin/vendor/reflexio/reflexio/lib/_interactions.py +16 -1
  31. package/plugin/vendor/reflexio/reflexio/lib/_search.py +15 -11
  32. package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/entities.py +18 -0
  33. package/plugin/vendor/reflexio/reflexio/models/api_schema/domain/enums.py +1 -0
  34. package/plugin/vendor/reflexio/reflexio/models/config_schema.py +24 -3
  35. package/plugin/vendor/reflexio/reflexio/server/README.md +25 -8
  36. package/plugin/vendor/reflexio/reflexio/server/__init__.py +21 -2
  37. package/plugin/vendor/reflexio/reflexio/server/api.py +148 -3277
  38. package/plugin/vendor/reflexio/reflexio/server/api_endpoints/README.md +4 -3
  39. package/plugin/vendor/reflexio/reflexio/server/api_endpoints/publisher_api.py +16 -1
  40. package/plugin/vendor/reflexio/reflexio/server/cache/reflexio_cache.py +62 -36
  41. package/plugin/vendor/reflexio/reflexio/server/deployment_profile.py +69 -0
  42. package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_embedding.py +424 -0
  43. package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_json_extraction.py +249 -0
  44. package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_structured_output.py +195 -0
  45. package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_subprocess.py +152 -0
  46. package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_text_generation.py +980 -0
  47. package/plugin/vendor/reflexio/reflexio/server/llm/_litellm_types.py +110 -0
  48. package/plugin/vendor/reflexio/reflexio/server/llm/litellm_client.py +72 -1817
  49. package/plugin/vendor/reflexio/reflexio/server/llm/providers/claude_code_provider.py +56 -4
  50. package/plugin/vendor/reflexio/reflexio/server/llm/providers/local_embedding_provider.py +183 -1
  51. package/plugin/vendor/reflexio/reflexio/server/middleware.py +244 -0
  52. package/plugin/vendor/reflexio/reflexio/server/operation_limiter.py +9 -1
  53. package/plugin/vendor/reflexio/reflexio/server/rate_limit.py +79 -0
  54. package/plugin/vendor/reflexio/reflexio/server/routes/__init__.py +6 -0
  55. package/plugin/vendor/reflexio/reflexio/server/routes/_common.py +25 -0
  56. package/plugin/vendor/reflexio/reflexio/server/routes/_metering.py +98 -0
  57. package/plugin/vendor/reflexio/reflexio/server/routes/braintrust.py +129 -0
  58. package/plugin/vendor/reflexio/reflexio/server/routes/config.py +210 -0
  59. package/plugin/vendor/reflexio/reflexio/server/routes/evaluation.py +549 -0
  60. package/plugin/vendor/reflexio/reflexio/server/routes/interactions.py +320 -0
  61. package/plugin/vendor/reflexio/reflexio/server/routes/playbooks.py +578 -0
  62. package/plugin/vendor/reflexio/reflexio/server/routes/profiles.py +423 -0
  63. package/plugin/vendor/reflexio/reflexio/server/routes/provenance.py +345 -0
  64. package/plugin/vendor/reflexio/reflexio/server/routes/search.py +349 -0
  65. package/plugin/vendor/reflexio/reflexio/server/routes/system.py +261 -0
  66. package/plugin/vendor/reflexio/reflexio/server/scheduling.py +132 -0
  67. package/plugin/vendor/reflexio/reflexio/server/services/README.md +3 -2
  68. package/plugin/vendor/reflexio/reflexio/server/services/base_generation/__init__.py +38 -0
  69. package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_batch_progress.py +298 -0
  70. package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_config_filter.py +152 -0
  71. package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_extraction_lifecycle.py +243 -0
  72. package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_should_run.py +299 -0
  73. package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_status_change.py +273 -0
  74. package/plugin/vendor/reflexio/reflexio/server/services/base_generation/_usage_billing.py +258 -0
  75. package/plugin/vendor/reflexio/reflexio/server/services/base_generation_service.py +17 -1183
  76. package/plugin/vendor/reflexio/reflexio/server/services/deduplication_utils.py +8 -1
  77. package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/__init__.py +13 -0
  78. package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/scheduler.py +142 -0
  79. package/plugin/vendor/reflexio/reflexio/server/services/durable_learning/worker.py +193 -0
  80. package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_scheduler.py +24 -41
  81. package/plugin/vendor/reflexio/reflexio/server/services/generation_service.py +335 -113
  82. package/plugin/vendor/reflexio/reflexio/server/services/lineage/gc_scheduler.py +362 -81
  83. package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/aggregator.py +68 -490
  84. package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/aggregator_clustering.py +184 -0
  85. package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/aggregator_postprocessing.py +130 -0
  86. package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/aggregator_prompt_formatting.py +212 -0
  87. package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/consolidator.py +258 -69
  88. package/plugin/vendor/reflexio/reflexio/server/services/profile/service.py +44 -22
  89. package/plugin/vendor/reflexio/reflexio/server/services/publish_learning_worker.py +288 -0
  90. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/__init__.py +31 -4
  91. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_agent_run.py +10 -1167
  92. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_base.py +196 -353
  93. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_governance.py +0 -1513
  94. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_learning_jobs.py +379 -0
  95. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_lineage.py +17 -6
  96. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_profiles.py +7 -1281
  97. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_requests.py +8 -3
  98. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/_share_links.py +30 -0
  99. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/agent_run/__init__.py +9 -0
  100. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/agent_run/_agent_run_store.py +506 -0
  101. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/agent_run/_pending_tool_call_store.py +704 -0
  102. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/agent_run/_run_tool_dependency_store.py +123 -0
  103. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/base/__init__.py +6 -0
  104. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/base/_deletion.py +263 -0
  105. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/base/_fts_vec.py +216 -0
  106. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/__init__.py +13 -0
  107. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_audit.py +122 -0
  108. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_erase_execution.py +465 -0
  109. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_purge.py +387 -0
  110. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_rebuild_hide.py +332 -0
  111. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/governance/_subject_barrier.py +511 -0
  112. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_agent.py +22 -10
  113. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_source_linkage.py +8 -3
  114. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/playbook/_user.py +25 -12
  115. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/__init__.py +9 -0
  116. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_interaction_store.py +308 -0
  117. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_profile_store.py +920 -0
  118. package/plugin/vendor/reflexio/reflexio/server/services/storage/sqlite_storage/profiles/_search.py +270 -0
  119. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/__init__.py +50 -8
  120. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_agent_run.py +64 -370
  121. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_commit_scope.py +9 -0
  122. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_learning_jobs.py +219 -0
  123. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_share_links.py +20 -0
  124. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/agent_run/__init__.py +9 -0
  125. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/agent_run/_agent_run_store.py +86 -0
  126. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/agent_run/_models.py +195 -0
  127. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/agent_run/_pending_tool_call_store.py +148 -0
  128. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/agent_run/_run_tool_dependency_store.py +38 -0
  129. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/governance/__init__.py +13 -0
  130. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/governance/_audit.py +29 -0
  131. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/governance/_erase_execution.py +30 -0
  132. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/governance/_purge.py +74 -0
  133. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/governance/_rebuild_hide.py +32 -0
  134. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/governance/_subject_barrier.py +49 -0
  135. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/__init__.py +9 -0
  136. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/_interaction_store.py +95 -0
  137. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/{_profiles.py → profiles/_profile_store.py} +60 -80
  138. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/profiles/_search.py +32 -0
  139. package/plugin/vendor/reflexio/reflexio/server/services/storage/storage_base/_governance.py +0 -148
@@ -1,6 +1,5 @@
1
1
  from __future__ import annotations
2
2
 
3
- import hashlib
4
3
  import logging
5
4
  import os
6
5
  import time
@@ -28,6 +27,16 @@ from reflexio.server.services.playbook.aggregation_prompt_processing import (
28
27
  AggregationPromptProcessingContext,
29
28
  AggregationPromptProcessor,
30
29
  )
30
+ from reflexio.server.services.playbook.components import (
31
+ aggregator_clustering,
32
+ aggregator_prompt_formatting,
33
+ )
34
+ from reflexio.server.services.playbook.components.aggregator_clustering import (
35
+ CLUSTERING_ALGORITHM_THRESHOLD,
36
+ )
37
+ from reflexio.server.services.playbook.components.aggregator_postprocessing import (
38
+ AggregationPostProcessing,
39
+ )
31
40
  from reflexio.server.services.playbook.playbook_service_constants import (
32
41
  PlaybookServiceConstants,
33
42
  )
@@ -43,11 +52,6 @@ from reflexio.server.usage_metrics import record_usage_event
43
52
 
44
53
  logger = logging.getLogger(__name__)
45
54
 
46
- # Threshold for switching between clustering algorithms
47
- # Below this, use Agglomerative (works better with small datasets)
48
- # Above this, use HDBSCAN (scales better, handles noise)
49
- CLUSTERING_ALGORITHM_THRESHOLD = 50
50
-
51
55
 
52
56
  class PlaybookAggregator:
53
57
  def __init__(
@@ -63,17 +67,15 @@ class PlaybookAggregator:
63
67
  self.request_context = request_context
64
68
  self.agent_version = agent_version
65
69
  self.aggregation_prompt_processor = aggregation_prompt_processor
70
+ # Cohesive pre/post-processing component (the enterprise redaction
71
+ # Protocol seam). Constructed from the SAME injected instance stored
72
+ # above — do NOT re-resolve the AGGREGATION_PROMPT_PROCESSOR ServiceKey.
73
+ self._postproc = AggregationPostProcessing(aggregation_prompt_processor)
66
74
 
67
75
  # ===============================
68
76
  # private methods - operation state
69
77
  # ===============================
70
78
 
71
- @staticmethod
72
- def _format_prompt_extra_instructions(instructions: str | None) -> str:
73
- if not instructions or not instructions.strip():
74
- return ""
75
- return f"{instructions.strip()}\n"
76
-
77
79
  def _create_state_manager(self) -> OperationStateManager:
78
80
  """
79
81
  Create an OperationStateManager for the playbook aggregator.
@@ -180,393 +182,74 @@ class PlaybookAggregator:
180
182
  last_processed_id=max_id,
181
183
  )
182
184
 
183
- def _format_cluster_input(self, cluster_playbooks: list[UserPlaybook]) -> str:
184
- """
185
- Format a cluster of playbooks for the aggregation prompt using per-item format.
186
-
187
- Each playbook is shown as a self-contained unit with content as the
188
- primary content, followed by optional structured fields as supplementary metadata.
189
-
190
- Args:
191
- cluster_playbooks: List of raw playbooks in this cluster
192
-
193
- Returns:
194
- str: Formatted input for the aggregation prompt
195
- """
196
- blocks = []
197
- for idx, fb in enumerate(cluster_playbooks, 1):
198
- lines = [f"[{idx}]"]
199
- if fb.content:
200
- lines.append(f'Content: "{fb.content}"')
201
- if fb.trigger:
202
- lines.append(f'Trigger: "{fb.trigger}"')
203
- if fb.rationale:
204
- lines.append(f'Rationale: "{fb.rationale}"')
205
- blocks.append("\n".join(lines))
206
- return "\n\n".join(blocks) if blocks else "(No playbook items)"
207
-
208
- def _preprocess_prompt_field(
209
- self,
210
- text: str | None,
211
- shared_state: dict[str, object],
212
- processing_context: AggregationPromptProcessingContext | None = None,
213
- ) -> str | None:
214
- if text is None or self.aggregation_prompt_processor is None:
215
- return text
216
- result = self.aggregation_prompt_processor.preprocess_prompt_text(
217
- text,
218
- shared_state=shared_state,
219
- context=processing_context,
220
- )
221
- if processing_context is not None and (result.changed or result.text != text):
222
- processing_context.changed = True
223
- return result.text
185
+ # ===============================
186
+ # private methods - aggregation pre/post-processing
187
+ # ===============================
188
+ # Bodies live on the AggregationPostProcessing component (self._postproc).
189
+ # These thin delegators are kept for OSS test call-sites that invoke them by
190
+ # name (test_playbook_aggregator.py). Internal callers use self._postproc.
224
191
 
225
- def _preprocess_user_playbook_for_prompt(
192
+ def _postprocess_aggregation_output(
226
193
  self,
227
- playbook: UserPlaybook,
228
- shared_state: dict[str, object],
194
+ value: object,
229
195
  processing_context: AggregationPromptProcessingContext | None = None,
230
- ) -> UserPlaybook:
231
- if self.aggregation_prompt_processor is None:
232
- return playbook
233
- return playbook.model_copy(
234
- update={
235
- "content": self._preprocess_prompt_field(
236
- playbook.content, shared_state, processing_context
237
- )
238
- or "",
239
- "trigger": self._preprocess_prompt_field(
240
- playbook.trigger, shared_state, processing_context
241
- ),
242
- "rationale": self._preprocess_prompt_field(
243
- playbook.rationale, shared_state, processing_context
244
- ),
245
- }
246
- )
196
+ ) -> tuple[object, int]:
197
+ return self._postproc._postprocess_aggregation_output(value, processing_context)
247
198
 
248
199
  def _aggregation_prompt_extra_instructions_for_context(
249
200
  self,
250
201
  processing_context: AggregationPromptProcessingContext | None,
251
202
  ) -> str:
252
- if (
253
- processing_context is None
254
- or not processing_context.changed
255
- or self.aggregation_prompt_processor is None
256
- ):
257
- return ""
258
-
259
- context_instructions = self.aggregation_prompt_processor.prompt_instructions(
203
+ return self._postproc._aggregation_prompt_extra_instructions_for_context(
260
204
  processing_context
261
205
  )
262
- if not isinstance(context_instructions, str):
263
- context_instructions = None
264
- return self._format_prompt_extra_instructions(context_instructions)
265
-
266
- def _postprocess_aggregation_output(
267
- self,
268
- value: object,
269
- processing_context: AggregationPromptProcessingContext | None = None,
270
- ) -> tuple[object, int]:
271
- if self.aggregation_prompt_processor is None:
272
- return value, 0
273
- result = self.aggregation_prompt_processor.postprocess_aggregation_output(
274
- value,
275
- context=processing_context,
276
- )
277
- return result.value, result.artifacts_removed
278
-
279
- def _postprocess_aggregation_response(
280
- self,
281
- response: PlaybookAggregationOutput,
282
- processing_context: AggregationPromptProcessingContext | None = None,
283
- ) -> tuple[PlaybookAggregationOutput, int]:
284
- processed, artifact_count = self._postprocess_aggregation_output(
285
- response,
286
- processing_context,
287
- )
288
- if not isinstance(processed, PlaybookAggregationOutput):
289
- return response, 0
290
- return processed, artifact_count
291
206
 
292
207
  def _record_postprocessing_artifacts(self, artifact_count: int) -> None:
293
- if artifact_count <= 0:
294
- return
295
- logger.warning(
296
- "Post-processed %d residual artifacts in aggregated playbook output",
297
- artifact_count,
298
- )
208
+ self._postproc._record_postprocessing_artifacts(artifact_count)
299
209
 
300
210
  @staticmethod
301
211
  def _get_direction_key(fb: UserPlaybook) -> str:
302
- """
303
- Extract a similarity key from a user playbook for grouping.
304
-
305
- Returns the raw content used for token-overlap comparison. Grouping is
306
- purely content-similarity based: under Option B a skill may legitimately
307
- hold mixed-orientation rules (do-rules and avoid-rules for different
308
- sub-aspects of one task), so whole-content polarity is NOT derived or
309
- gated here. Preserving distinct do/avoid rules when similar items are
310
- merged is the aggregation prompt's responsibility.
311
-
312
- Args:
313
- fb: A user playbook item
314
-
315
- Returns:
316
- str: Content used as the similarity key for grouping
317
- """
318
- return fb.content or ""
212
+ return aggregator_prompt_formatting.get_direction_key(fb)
319
213
 
320
214
  @staticmethod
321
215
  def _token_overlap(str1: str, str2: str, threshold: float = 0.6) -> bool:
322
- """
323
- Check if two strings have significant token overlap using asymmetric containment.
324
-
325
- Computes the ratio of shared tokens to the smaller set, so a short string
326
- contained in a longer one still counts as a match.
327
-
328
- Args:
329
- str1: First string
330
- str2: Second string
331
- threshold: Minimum overlap ratio
332
-
333
- Returns:
334
- bool: True if overlap ratio >= threshold
335
- """
336
- tokens1 = set(str1.lower().split())
337
- tokens2 = set(str2.lower().split())
338
- if not tokens1 or not tokens2:
339
- return False
340
- intersection = len(tokens1 & tokens2)
341
- overlap_ratio = max(intersection / len(tokens1), intersection / len(tokens2))
342
- return overlap_ratio >= threshold
216
+ return aggregator_prompt_formatting.token_overlap(str1, str2, threshold)
343
217
 
344
218
  @staticmethod
345
219
  def _group_playbooks_by_direction(
346
220
  cluster_playbooks: list[UserPlaybook],
347
221
  threshold: float = 0.6,
348
222
  ) -> list[list[UserPlaybook]]:
349
- """
350
- Group playbooks by similarity of their content.
351
-
352
- Uses greedy single-linkage: each playbook is assigned to the first
353
- existing group that has any member with sufficient token overlap.
354
- Groups are returned sorted by size descending (largest first).
355
-
356
- Grouping is purely content-similarity based and does NOT gate on a
357
- derived whole-content polarity. Under Option B a skill may hold
358
- mixed-orientation rules (do-rules and avoid-rules for different
359
- sub-aspects), and whole-content polarity is undefined for such a skill.
360
- Keeping a do-rule and an avoid-rule as distinct rules when similar items
361
- are merged into one skill is the aggregation prompt's responsibility,
362
- not a mechanical split here.
363
-
364
- Args:
365
- cluster_playbooks: List of raw playbooks to group
366
- threshold: Token overlap threshold for grouping
367
-
368
- Returns:
369
- list[list[UserPlaybook]]: Groups sorted by size descending
370
- """
371
- groups: list[list[UserPlaybook]] = []
372
-
373
- for fb in cluster_playbooks:
374
- key = PlaybookAggregator._get_direction_key(fb)
375
- matched = False
376
- for group in groups:
377
- if any(
378
- PlaybookAggregator._token_overlap(
379
- key,
380
- PlaybookAggregator._get_direction_key(group_fb),
381
- threshold,
382
- )
383
- for group_fb in group
384
- ):
385
- group.append(fb)
386
- matched = True
387
- break
388
- if not matched:
389
- groups.append([fb])
390
-
391
- # Sort by group size descending (largest first)
392
- groups.sort(key=len, reverse=True)
393
- return groups
223
+ return aggregator_prompt_formatting.group_playbooks_by_direction(
224
+ cluster_playbooks, threshold
225
+ )
394
226
 
395
227
  def _format_structured_cluster_input(
396
228
  self,
397
229
  cluster_playbooks: list[UserPlaybook],
398
230
  direction_overlap_threshold: float = 0.6,
399
231
  ) -> str:
400
- """
401
- Format a cluster of playbooks for structured aggregation prompt.
402
-
403
- When the cluster forms a single similarity group, uses the flat-list
404
- format. When distinct similarity groups are detected (multiple groups),
405
- uses a grouped format so the LLM can see which items are similar and
406
- preserve distinct rules (e.g. a do-rule and an avoid-rule) as separate
407
- rules in the merged skill rather than collapsing them.
408
-
409
- Args:
410
- cluster_playbooks: List of raw playbooks in this cluster
411
- direction_overlap_threshold: Token overlap threshold for grouping by direction
412
-
413
- Returns:
414
- str: Formatted input for the aggregation prompt
415
- """
416
- groups = self._group_playbooks_by_direction(
417
- cluster_playbooks, threshold=direction_overlap_threshold
232
+ return aggregator_prompt_formatting.format_structured_cluster_input(
233
+ cluster_playbooks,
234
+ direction_overlap_threshold=direction_overlap_threshold,
418
235
  )
419
236
 
420
- if len(groups) <= 1:
421
- return self._format_flat(cluster_playbooks)
422
- return self._format_grouped(groups)
423
-
424
- def _format_flat(self, cluster_playbooks: list[UserPlaybook]) -> str:
425
- """
426
- Format playbooks as flat bullet lists (original format, used when no conflict).
427
-
428
- Args:
429
- cluster_playbooks: List of raw playbooks in this cluster
430
-
431
- Returns:
432
- str: Formatted input with separate field lists
433
- """
434
- triggers = []
435
- rationales = []
436
-
437
- for fb in cluster_playbooks:
438
- if fb.trigger:
439
- triggers.append(fb.trigger)
440
- if fb.rationale:
441
- rationales.append(fb.rationale)
442
-
443
- lines: list[str] = []
444
-
445
- if triggers:
446
- lines.append("TRIGGER conditions (to be consolidated):")
447
- lines.extend(f"- {trigger}" for trigger in triggers)
448
- else:
449
- lines.append("TRIGGER conditions: (none specified)")
450
-
451
- if rationales:
452
- lines.append("RATIONALE summaries:")
453
- lines.extend(f"- {r}" for r in rationales)
454
-
455
- self._append_freeform_observations(lines, cluster_playbooks)
456
-
457
- return "\n".join(lines)
458
-
459
- def _format_grouped(
460
- self,
461
- groups: list[list[UserPlaybook]],
462
- ) -> str:
463
- """
464
- Format playbooks in grouped layout (used when conflicting directions are detected).
465
-
466
- Args:
467
- groups: AgentPlaybook groups sorted by size descending
468
-
469
- Returns:
470
- str: Formatted input with group headers and per-playbook fields
471
- """
472
- lines: list[str] = [
473
- "The following playbook items are grouped by similarity. "
474
- "Groups are ordered by size (largest first).",
475
- "",
476
- ]
477
-
478
- for idx, group in enumerate(groups, start=1):
479
- count_label = "playbook" if len(group) == 1 else "playbooks"
480
- lines.append(f"Group {idx} ({len(group)} {count_label}):")
481
- for fb in group:
482
- parts: list[str] = []
483
- if fb.trigger:
484
- parts.append(f'Trigger: "{fb.trigger}"')
485
- if fb.rationale:
486
- parts.append(f'Rationale: "{fb.rationale}"')
487
- if not parts and fb.content:
488
- parts.append(f'AgentPlaybook: "{fb.content}"')
489
- if parts:
490
- lines.append(f" - {parts[0]}")
491
- lines.extend(f" {p}" for p in parts[1:])
492
- lines.append("")
493
-
494
- return "\n".join(lines)
495
-
496
- @staticmethod
497
- def _append_freeform_observations(
498
- lines: list[str], cluster_playbooks: list[UserPlaybook]
499
- ) -> None:
500
- """Append freeform observations from cluster playbooks to output lines."""
501
- freeform_observations = [
502
- fb.content for fb in cluster_playbooks if not fb.trigger and fb.content
503
- ]
504
- if freeform_observations:
505
- lines.append("Freeform observations (from freeform cluster members):")
506
- lines.extend(f"- {obs}" for obs in freeform_observations)
507
-
508
237
  # ===============================
509
238
  # private methods - cluster change detection
510
239
  # ===============================
511
240
 
512
241
  @staticmethod
513
242
  def _compute_cluster_fingerprint(cluster_playbooks: list[UserPlaybook]) -> str:
514
- """
515
- Compute a fingerprint for a cluster based on its user_playbook_ids.
516
- The fingerprint is deterministic and order-independent.
517
-
518
- Args:
519
- cluster_playbooks: List of raw playbooks in this cluster
520
-
521
- Returns:
522
- str: SHA-256 hash (truncated to 16 hex chars) of sorted user_playbook_ids
523
- """
524
- sorted_ids = sorted(fb.user_playbook_id for fb in cluster_playbooks)
525
- id_str = ",".join(str(id) for id in sorted_ids)
526
- return hashlib.sha256(id_str.encode()).hexdigest()[:16]
243
+ return aggregator_clustering.compute_cluster_fingerprint(cluster_playbooks)
527
244
 
528
245
  def _determine_cluster_changes(
529
246
  self,
530
247
  clusters: dict[int, list[UserPlaybook]],
531
248
  prev_fingerprints: dict,
532
249
  ) -> tuple[dict[int, list[UserPlaybook]], list[int]]:
533
- """
534
- Compare current cluster fingerprints against stored fingerprints to determine changes.
535
-
536
- Args:
537
- clusters: Current clusters (cluster_id -> list of UserPlaybook)
538
- prev_fingerprints: Previous fingerprint state
539
- (fingerprint_hash -> {"agent_playbook_id": int, "user_playbook_ids": list})
540
-
541
- Returns:
542
- tuple of:
543
- - changed_clusters: Only clusters needing new LLM calls
544
- - playbook_ids_to_archive: Old playbook_ids from changed/disappeared clusters
545
- """
546
- # Compute fingerprints for current clusters
547
- current_fingerprints = {}
548
- for cluster_id, cluster_playbooks in clusters.items():
549
- fp = self._compute_cluster_fingerprint(cluster_playbooks)
550
- current_fingerprints[cluster_id] = fp
551
-
552
- current_fp_set = set(current_fingerprints.values())
553
- prev_fp_set = set(prev_fingerprints.keys())
554
-
555
- # Changed clusters: fingerprints that are new (not in previous state)
556
- changed_clusters = {}
557
- for cluster_id, fp in current_fingerprints.items():
558
- if fp not in prev_fp_set:
559
- changed_clusters[cluster_id] = clusters[cluster_id]
560
-
561
- # Playbook IDs to archive: from fingerprints that disappeared or changed
562
- playbook_ids_to_archive = []
563
- for fp, fp_data in prev_fingerprints.items():
564
- if fp not in current_fp_set:
565
- playbook_id = fp_data.get("agent_playbook_id")
566
- if playbook_id is not None:
567
- playbook_ids_to_archive.append(playbook_id)
568
-
569
- return changed_clusters, playbook_ids_to_archive
250
+ return aggregator_clustering.determine_cluster_changes(
251
+ clusters, prev_fingerprints
252
+ )
570
253
 
571
254
  # ===============================
572
255
  # public methods
@@ -1108,7 +791,9 @@ class PlaybookAggregator:
1108
791
  # Mock mode: cluster by trigger
1109
792
  if os.getenv("MOCK_LLM_RESPONSE", "").lower() == "true":
1110
793
  logger.info("Mock mode: clustering by trigger")
1111
- return self._cluster_by_trigger_mock(user_playbooks, min_cluster_size)
794
+ return aggregator_clustering.cluster_by_trigger_mock(
795
+ user_playbooks, min_cluster_size
796
+ )
1112
797
 
1113
798
  # Extract embeddings from user playbooks
1114
799
  import numpy as np
@@ -1163,139 +848,26 @@ class PlaybookAggregator:
1163
848
 
1164
849
  return clusters
1165
850
 
1166
- def _cluster_by_trigger_mock(
1167
- self, user_playbooks: list[UserPlaybook], min_cluster_size: int
1168
- ) -> dict[int, list[UserPlaybook]]:
1169
- """
1170
- Simple mock clustering by exact trigger match.
1171
-
1172
- Args:
1173
- user_playbooks: List of user playbooks with trigger field
1174
- min_cluster_size: Minimum number of playbooks per cluster
1175
-
1176
- Returns:
1177
- dict[int, list[UserPlaybook]]: Clusters grouped by trigger
1178
- """
1179
- # Group by trigger
1180
- condition_groups: dict[str, list[UserPlaybook]] = {}
1181
- for fb in user_playbooks:
1182
- condition = fb.trigger or ""
1183
- if condition not in condition_groups:
1184
- condition_groups[condition] = []
1185
- condition_groups[condition].append(fb)
1186
-
1187
- # Convert to cluster format, filtering by min_cluster_size
1188
- clusters: dict[int, list[UserPlaybook]] = {}
1189
- cluster_id = 0
1190
- for playbooks_group in condition_groups.values():
1191
- if len(playbooks_group) >= min_cluster_size:
1192
- clusters[cluster_id] = playbooks_group
1193
- cluster_id += 1
1194
-
1195
- logger.info(
1196
- "Mock mode: created %d trigger clusters from %d playbooks",
1197
- len(clusters),
1198
- len(user_playbooks),
1199
- )
1200
- return clusters
1201
-
1202
851
  def _cluster_with_agglomerative(
1203
852
  self,
1204
853
  distance_matrix: np.ndarray,
1205
- min_cluster_size: int, # noqa: ARG002
854
+ min_cluster_size: int,
1206
855
  distance_threshold: float,
1207
856
  ) -> np.ndarray:
1208
- """
1209
- Cluster using Agglomerative Clustering - best for small datasets.
1210
-
1211
- Args:
1212
- distance_matrix: Precomputed cosine distance matrix
1213
- min_cluster_size: Minimum cluster size (used for logging only,
1214
- filtering happens in get_clusters)
1215
- distance_threshold: Maximum cosine distance to merge clusters (1 - similarity_threshold)
1216
-
1217
- Returns:
1218
- np.ndarray: Cluster labels for each point
1219
- """
1220
- from sklearn.cluster import AgglomerativeClustering
1221
-
1222
- logger.info(
1223
- "Using Agglomerative Clustering for %d playbooks (< %d threshold), distance_threshold=%.2f",
1224
- len(distance_matrix),
1225
- CLUSTERING_ALGORITHM_THRESHOLD,
1226
- distance_threshold,
1227
- )
1228
-
1229
- clusterer = AgglomerativeClustering(
1230
- n_clusters=None, # type: ignore[reportArgumentType]
1231
- distance_threshold=distance_threshold,
1232
- metric="precomputed",
1233
- linkage="average",
857
+ return aggregator_clustering.cluster_with_agglomerative(
858
+ distance_matrix, min_cluster_size, distance_threshold
1234
859
  )
1235
860
 
1236
- return clusterer.fit_predict(distance_matrix)
1237
-
1238
861
  def _cluster_with_hdbscan(
1239
862
  self,
1240
863
  distance_matrix: np.ndarray,
1241
864
  min_cluster_size: int,
1242
865
  distance_threshold: float,
1243
866
  ) -> np.ndarray:
1244
- """
1245
- Cluster using HDBSCAN - best for large datasets with potential noise.
1246
-
1247
- Args:
1248
- distance_matrix: Precomputed cosine distance matrix
1249
- min_cluster_size: Minimum number of points to form a cluster
1250
- distance_threshold: Maximum cosine distance for cluster merging (1 - similarity_threshold)
1251
-
1252
- Returns:
1253
- np.ndarray: Cluster labels for each point (-1 indicates noise)
1254
- """
1255
- import hdbscan
1256
-
1257
- logger.info(
1258
- "Using HDBSCAN for %d playbooks (>= %d threshold), distance_threshold=%.2f",
1259
- len(distance_matrix),
1260
- CLUSTERING_ALGORITHM_THRESHOLD,
1261
- distance_threshold,
1262
- )
1263
-
1264
- clusterer = hdbscan.HDBSCAN(
1265
- min_cluster_size=min_cluster_size,
1266
- min_samples=1,
1267
- metric="precomputed",
1268
- cluster_selection_epsilon=distance_threshold,
867
+ return aggregator_clustering.cluster_with_hdbscan(
868
+ distance_matrix, min_cluster_size, distance_threshold
1269
869
  )
1270
870
 
1271
- return clusterer.fit_predict(distance_matrix)
1272
-
1273
- def _generate_playbooks_from_clusters(
1274
- self,
1275
- clusters: dict[int, list[UserPlaybook]],
1276
- existing_approved_playbooks: list[AgentPlaybook],
1277
- direction_overlap_threshold: float = 0.6,
1278
- ) -> list[AgentPlaybook]:
1279
- """
1280
- Generate playbooks from clusters, considering existing approved playbooks.
1281
-
1282
- Args:
1283
- clusters: Dictionary mapping cluster IDs to lists of raw playbooks
1284
- existing_approved_playbooks: List of existing approved playbooks to avoid duplication
1285
- direction_overlap_threshold: Token overlap threshold for grouping by direction
1286
-
1287
- Returns:
1288
- list[AgentPlaybook]: List of newly generated playbooks (excludes duplicates)
1289
- """
1290
- return [
1291
- playbook
1292
- for playbook, _ in self._generate_playbooks_with_source_clusters(
1293
- clusters,
1294
- existing_approved_playbooks,
1295
- direction_overlap_threshold=direction_overlap_threshold,
1296
- )
1297
- ]
1298
-
1299
871
  def _generate_playbooks_with_source_clusters(
1300
872
  self,
1301
873
  clusters: dict[int, list[UserPlaybook]],
@@ -1321,7 +893,7 @@ class PlaybookAggregator:
1321
893
  prompt_cluster_playbooks = cluster_playbooks
1322
894
  else:
1323
895
  prompt_cluster_playbooks = [
1324
- self._preprocess_user_playbook_for_prompt(
896
+ self._postproc._preprocess_user_playbook_for_prompt(
1325
897
  playbook, shared_state, processing_context
1326
898
  )
1327
899
  for playbook in cluster_playbooks
@@ -1418,11 +990,11 @@ class PlaybookAggregator:
1418
990
  trigger=trigger,
1419
991
  )
1420
992
  )
1421
- response, artifact_count = self._postprocess_aggregation_response(
993
+ response, artifact_count = self._postproc._postprocess_aggregation_response(
1422
994
  response,
1423
995
  processing_context,
1424
996
  )
1425
- self._record_postprocessing_artifacts(artifact_count)
997
+ self._postproc._record_postprocessing_artifacts(artifact_count)
1426
998
  playbook = self._process_aggregation_response(response, cluster_playbooks)
1427
999
  if playbook is None:
1428
1000
  return None
@@ -1442,7 +1014,7 @@ class PlaybookAggregator:
1442
1014
  {
1443
1015
  "user_playbooks": raw_playbooks_str,
1444
1016
  "existing_approved_playbooks": existing_approved_playbooks_str,
1445
- "aggregation_prompt_extra_instructions": self._aggregation_prompt_extra_instructions_for_context(
1017
+ "aggregation_prompt_extra_instructions": self._postproc._aggregation_prompt_extra_instructions_for_context(
1446
1018
  processing_context
1447
1019
  ),
1448
1020
  },
@@ -1458,17 +1030,21 @@ class PlaybookAggregator:
1458
1030
  parse_structured_output=True,
1459
1031
  )
1460
1032
  if isinstance(response, PlaybookAggregationOutput):
1461
- response, artifact_count = self._postprocess_aggregation_response(
1462
- response,
1463
- processing_context,
1033
+ response, artifact_count = (
1034
+ self._postproc._postprocess_aggregation_response(
1035
+ response,
1036
+ processing_context,
1037
+ )
1464
1038
  )
1465
- self._record_postprocessing_artifacts(artifact_count)
1039
+ self._postproc._record_postprocessing_artifacts(artifact_count)
1466
1040
  else:
1467
- response, artifact_count = self._postprocess_aggregation_output(
1468
- response,
1469
- processing_context,
1041
+ response, artifact_count = (
1042
+ self._postproc._postprocess_aggregation_output(
1043
+ response,
1044
+ processing_context,
1045
+ )
1470
1046
  )
1471
- self._record_postprocessing_artifacts(artifact_count)
1047
+ self._postproc._record_postprocessing_artifacts(artifact_count)
1472
1048
  log_model_response(logger, "Aggregation structured response", response)
1473
1049
 
1474
1050
  if not isinstance(response, PlaybookAggregationOutput):
@@ -1480,11 +1056,13 @@ class PlaybookAggregator:
1480
1056
 
1481
1057
  return self._process_aggregation_response(response, cluster_playbooks)
1482
1058
  except Exception as exc:
1483
- processed_error, artifact_count = self._postprocess_aggregation_output(
1484
- str(exc),
1485
- processing_context,
1059
+ processed_error, artifact_count = (
1060
+ self._postproc._postprocess_aggregation_output(
1061
+ str(exc),
1062
+ processing_context,
1063
+ )
1486
1064
  )
1487
- self._record_postprocessing_artifacts(artifact_count)
1065
+ self._postproc._record_postprocessing_artifacts(artifact_count)
1488
1066
  logger.error(
1489
1067
  "AgentPlaybook aggregation failed due to %s, returning None.",
1490
1068
  processed_error,