claude-smart 0.2.47 → 0.2.48

package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/aggregator_clustering.py

@@ -0,0 +1,184 @@
+from __future__ import annotations
+
+import hashlib
+import logging
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import numpy as np
+
+from reflexio.models.api_schema.service_schemas import UserPlaybook
+
+logger = logging.getLogger(__name__)
+
+# Threshold for switching between clustering algorithms
+# Below this, use Agglomerative (works better with small datasets)
+# Above this, use HDBSCAN (scales better, handles noise)
+CLUSTERING_ALGORITHM_THRESHOLD = 50
+
+
+def compute_cluster_fingerprint(cluster_playbooks: list[UserPlaybook]) -> str:
+    """
+    Compute a fingerprint for a cluster based on its user_playbook_ids.
+    The fingerprint is deterministic and order-independent.
+
+    Args:
+        cluster_playbooks: List of raw playbooks in this cluster
+
+    Returns:
+        str: SHA-256 hash (truncated to 16 hex chars) of sorted user_playbook_ids
+    """
+    sorted_ids = sorted(fb.user_playbook_id for fb in cluster_playbooks)
+    id_str = ",".join(str(id) for id in sorted_ids)
+    return hashlib.sha256(id_str.encode()).hexdigest()[:16]
+
+
+def determine_cluster_changes(
+    clusters: dict[int, list[UserPlaybook]],
+    prev_fingerprints: dict,
+) -> tuple[dict[int, list[UserPlaybook]], list[int]]:
+    """
+    Compare current cluster fingerprints against stored fingerprints to determine changes.
+
+    Args:
+        clusters: Current clusters (cluster_id -> list of UserPlaybook)
+        prev_fingerprints: Previous fingerprint state
+            (fingerprint_hash -> {"agent_playbook_id": int, "user_playbook_ids": list})
+
+    Returns:
+        tuple of:
+            - changed_clusters: Only clusters needing new LLM calls
+            - playbook_ids_to_archive: Old playbook_ids from changed/disappeared clusters
+    """
+    # Compute fingerprints for current clusters
+    current_fingerprints = {}
+    for cluster_id, cluster_playbooks in clusters.items():
+        fp = compute_cluster_fingerprint(cluster_playbooks)
+        current_fingerprints[cluster_id] = fp
+
+    current_fp_set = set(current_fingerprints.values())
+    prev_fp_set = set(prev_fingerprints.keys())
+
+    # Changed clusters: fingerprints that are new (not in previous state)
+    changed_clusters = {}
+    for cluster_id, fp in current_fingerprints.items():
+        if fp not in prev_fp_set:
+            changed_clusters[cluster_id] = clusters[cluster_id]
+
+    # Playbook IDs to archive: from fingerprints that disappeared or changed
+    playbook_ids_to_archive = []
+    for fp, fp_data in prev_fingerprints.items():
+        if fp not in current_fp_set:
+            playbook_id = fp_data.get("agent_playbook_id")
+            if playbook_id is not None:
+                playbook_ids_to_archive.append(playbook_id)
+
+    return changed_clusters, playbook_ids_to_archive
+
+
+def cluster_by_trigger_mock(
+    user_playbooks: list[UserPlaybook], min_cluster_size: int
+) -> dict[int, list[UserPlaybook]]:
+    """
+    Simple mock clustering by exact trigger match.
+
+    Args:
+        user_playbooks: List of user playbooks with trigger field
+        min_cluster_size: Minimum number of playbooks per cluster
+
+    Returns:
+        dict[int, list[UserPlaybook]]: Clusters grouped by trigger
+    """
+    # Group by trigger
+    condition_groups: dict[str, list[UserPlaybook]] = {}
+    for fb in user_playbooks:
+        condition = fb.trigger or ""
+        if condition not in condition_groups:
+            condition_groups[condition] = []
+        condition_groups[condition].append(fb)
+
+    # Convert to cluster format, filtering by min_cluster_size
+    clusters: dict[int, list[UserPlaybook]] = {}
+    cluster_id = 0
+    for playbooks_group in condition_groups.values():
+        if len(playbooks_group) >= min_cluster_size:
+            clusters[cluster_id] = playbooks_group
+            cluster_id += 1
+
+    logger.info(
+        "Mock mode: created %d trigger clusters from %d playbooks",
+        len(clusters),
+        len(user_playbooks),
+    )
+    return clusters
+
+
+def cluster_with_agglomerative(
+    distance_matrix: np.ndarray,
+    min_cluster_size: int,  # noqa: ARG001
+    distance_threshold: float,
+) -> np.ndarray:
+    """
+    Cluster using Agglomerative Clustering - best for small datasets.
+
+    Args:
+        distance_matrix: Precomputed cosine distance matrix
+        min_cluster_size: Minimum cluster size (used for logging only,
+                          filtering happens in get_clusters)
+        distance_threshold: Maximum cosine distance to merge clusters (1 - similarity_threshold)
+
+    Returns:
+        np.ndarray: Cluster labels for each point
+    """
+    from sklearn.cluster import AgglomerativeClustering
+
+    logger.info(
+        "Using Agglomerative Clustering for %d playbooks (< %d threshold), distance_threshold=%.2f",
+        len(distance_matrix),
+        CLUSTERING_ALGORITHM_THRESHOLD,
+        distance_threshold,
+    )
+
+    clusterer = AgglomerativeClustering(
+        n_clusters=None,  # type: ignore[reportArgumentType]
+        distance_threshold=distance_threshold,
+        metric="precomputed",
+        linkage="average",
+    )
+
+    return clusterer.fit_predict(distance_matrix)
+
+
+def cluster_with_hdbscan(
+    distance_matrix: np.ndarray,
+    min_cluster_size: int,
+    distance_threshold: float,
+) -> np.ndarray:
+    """
+    Cluster using HDBSCAN - best for large datasets with potential noise.
+
+    Args:
+        distance_matrix: Precomputed cosine distance matrix
+        min_cluster_size: Minimum number of points to form a cluster
+        distance_threshold: Maximum cosine distance for cluster merging (1 - similarity_threshold)
+
+    Returns:
+        np.ndarray: Cluster labels for each point (-1 indicates noise)
+    """
+    import hdbscan
+
+    logger.info(
+        "Using HDBSCAN for %d playbooks (>= %d threshold), distance_threshold=%.2f",
+        len(distance_matrix),
+        CLUSTERING_ALGORITHM_THRESHOLD,
+        distance_threshold,
+    )
+
+    clusterer = hdbscan.HDBSCAN(
+        min_cluster_size=min_cluster_size,
+        min_samples=1,
+        metric="precomputed",
+        cluster_selection_epsilon=distance_threshold,
+    )
+
+    return clusterer.fit_predict(distance_matrix)

package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/aggregator_postprocessing.py

@@ -0,0 +1,130 @@
+from __future__ import annotations
+
+import logging
+
+from reflexio.models.api_schema.service_schemas import UserPlaybook
+from reflexio.server.services.playbook.aggregation_prompt_processing import (
+    AggregationPromptProcessingContext,
+    AggregationPromptProcessor,
+)
+from reflexio.server.services.playbook.playbook_service_utils import (
+    PlaybookAggregationOutput,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class AggregationPostProcessing:
+    """Pre/post-processing for playbook aggregation prompts and LLM output.
+
+    Groups the aggregation prompt-processor seam (the enterprise redaction
+    Protocol integration point). Holds the single injected collaborator,
+    ``aggregation_prompt_processor``, by shared reference from the owning
+    ``PlaybookAggregator`` — the SAME object resolved from the
+    ``AGGREGATION_PROMPT_PROCESSOR`` ServiceKey — so the enterprise
+    ``EnterpriseAggregationPromptProcessor`` injection is preserved unchanged.
+    """
+
+    def __init__(
+        self,
+        aggregation_prompt_processor: AggregationPromptProcessor | None,
+    ) -> None:
+        self.aggregation_prompt_processor = aggregation_prompt_processor
+
+    @staticmethod
+    def _format_prompt_extra_instructions(instructions: str | None) -> str:
+        if not instructions or not instructions.strip():
+            return ""
+        return f"{instructions.strip()}\n"
+
+    def _preprocess_prompt_field(
+        self,
+        text: str | None,
+        shared_state: dict[str, object],
+        processing_context: AggregationPromptProcessingContext | None = None,
+    ) -> str | None:
+        if text is None or self.aggregation_prompt_processor is None:
+            return text
+        result = self.aggregation_prompt_processor.preprocess_prompt_text(
+            text,
+            shared_state=shared_state,
+            context=processing_context,
+        )
+        if processing_context is not None and (result.changed or result.text != text):
+            processing_context.changed = True
+        return result.text
+
+    def _preprocess_user_playbook_for_prompt(
+        self,
+        playbook: UserPlaybook,
+        shared_state: dict[str, object],
+        processing_context: AggregationPromptProcessingContext | None = None,
+    ) -> UserPlaybook:
+        if self.aggregation_prompt_processor is None:
+            return playbook
+        return playbook.model_copy(
+            update={
+                "content": self._preprocess_prompt_field(
+                    playbook.content, shared_state, processing_context
+                )
+                or "",
+                "trigger": self._preprocess_prompt_field(
+                    playbook.trigger, shared_state, processing_context
+                ),
+                "rationale": self._preprocess_prompt_field(
+                    playbook.rationale, shared_state, processing_context
+                ),
+            }
+        )
+
+    def _aggregation_prompt_extra_instructions_for_context(
+        self,
+        processing_context: AggregationPromptProcessingContext | None,
+    ) -> str:
+        if (
+            processing_context is None
+            or not processing_context.changed
+            or self.aggregation_prompt_processor is None
+        ):
+            return ""
+
+        context_instructions = self.aggregation_prompt_processor.prompt_instructions(
+            processing_context
+        )
+        if not isinstance(context_instructions, str):
+            context_instructions = None
+        return self._format_prompt_extra_instructions(context_instructions)
+
+    def _postprocess_aggregation_output(
+        self,
+        value: object,
+        processing_context: AggregationPromptProcessingContext | None = None,
+    ) -> tuple[object, int]:
+        if self.aggregation_prompt_processor is None:
+            return value, 0
+        result = self.aggregation_prompt_processor.postprocess_aggregation_output(
+            value,
+            context=processing_context,
+        )
+        return result.value, result.artifacts_removed
+
+    def _postprocess_aggregation_response(
+        self,
+        response: PlaybookAggregationOutput,
+        processing_context: AggregationPromptProcessingContext | None = None,
+    ) -> tuple[PlaybookAggregationOutput, int]:
+        processed, artifact_count = self._postprocess_aggregation_output(
+            response,
+            processing_context,
+        )
+        if not isinstance(processed, PlaybookAggregationOutput):
+            return response, 0
+        return processed, artifact_count
+
+    def _record_postprocessing_artifacts(self, artifact_count: int) -> None:
+        if artifact_count <= 0:
+            return
+        logger.warning(
+            "Post-processed %d residual artifacts in aggregated playbook output",
+            artifact_count,
+        )

package/plugin/vendor/reflexio/reflexio/server/services/playbook/components/aggregator_prompt_formatting.py

@@ -0,0 +1,212 @@
+from __future__ import annotations
+
+from reflexio.models.api_schema.service_schemas import UserPlaybook
+
+
+def get_direction_key(fb: UserPlaybook) -> str:
+    """
+    Extract a similarity key from a user playbook for grouping.
+
+    Returns the raw content used for token-overlap comparison. Grouping is
+    purely content-similarity based: under Option B a skill may legitimately
+    hold mixed-orientation rules (do-rules and avoid-rules for different
+    sub-aspects of one task), so whole-content polarity is NOT derived or
+    gated here. Preserving distinct do/avoid rules when similar items are
+    merged is the aggregation prompt's responsibility.
+
+    Args:
+        fb: A user playbook item
+
+    Returns:
+        str: Content used as the similarity key for grouping
+    """
+    return fb.content or ""
+
+
+def token_overlap(str1: str, str2: str, threshold: float = 0.6) -> bool:
+    """
+    Check if two strings have significant token overlap using asymmetric containment.
+
+    Computes the ratio of shared tokens to the smaller set, so a short string
+    contained in a longer one still counts as a match.
+
+    Args:
+        str1: First string
+        str2: Second string
+        threshold: Minimum overlap ratio
+
+    Returns:
+        bool: True if overlap ratio >= threshold
+    """
+    tokens1 = set(str1.lower().split())
+    tokens2 = set(str2.lower().split())
+    if not tokens1 or not tokens2:
+        return False
+    intersection = len(tokens1 & tokens2)
+    overlap_ratio = max(intersection / len(tokens1), intersection / len(tokens2))
+    return overlap_ratio >= threshold
+
+
+def group_playbooks_by_direction(
+    cluster_playbooks: list[UserPlaybook],
+    threshold: float = 0.6,
+) -> list[list[UserPlaybook]]:
+    """
+    Group playbooks by similarity of their content.
+
+    Uses greedy single-linkage: each playbook is assigned to the first
+    existing group that has any member with sufficient token overlap.
+    Groups are returned sorted by size descending (largest first).
+
+    Grouping is purely content-similarity based and does NOT gate on a
+    derived whole-content polarity. Under Option B a skill may hold
+    mixed-orientation rules (do-rules and avoid-rules for different
+    sub-aspects), and whole-content polarity is undefined for such a skill.
+    Keeping a do-rule and an avoid-rule as distinct rules when similar items
+    are merged into one skill is the aggregation prompt's responsibility,
+    not a mechanical split here.
+
+    Args:
+        cluster_playbooks: List of raw playbooks to group
+        threshold: Token overlap threshold for grouping
+
+    Returns:
+        list[list[UserPlaybook]]: Groups sorted by size descending
+    """
+    groups: list[list[UserPlaybook]] = []
+
+    for fb in cluster_playbooks:
+        key = get_direction_key(fb)
+        matched = False
+        for group in groups:
+            if any(
+                token_overlap(
+                    key,
+                    get_direction_key(group_fb),
+                    threshold,
+                )
+                for group_fb in group
+            ):
+                group.append(fb)
+                matched = True
+                break
+        if not matched:
+            groups.append([fb])
+
+    # Sort by group size descending (largest first)
+    groups.sort(key=len, reverse=True)
+    return groups
+
+
+def format_structured_cluster_input(
+    cluster_playbooks: list[UserPlaybook],
+    direction_overlap_threshold: float = 0.6,
+) -> str:
+    """
+    Format a cluster of playbooks for structured aggregation prompt.
+
+    When the cluster forms a single similarity group, uses the flat-list
+    format. When distinct similarity groups are detected (multiple groups),
+    uses a grouped format so the LLM can see which items are similar and
+    preserve distinct rules (e.g. a do-rule and an avoid-rule) as separate
+    rules in the merged skill rather than collapsing them.
+
+    Args:
+        cluster_playbooks: List of raw playbooks in this cluster
+        direction_overlap_threshold: Token overlap threshold for grouping by direction
+
+    Returns:
+        str: Formatted input for the aggregation prompt
+    """
+    groups = group_playbooks_by_direction(
+        cluster_playbooks, threshold=direction_overlap_threshold
+    )
+
+    if len(groups) <= 1:
+        return format_flat(cluster_playbooks)
+    return format_grouped(groups)
+
+
+def format_flat(cluster_playbooks: list[UserPlaybook]) -> str:
+    """
+    Format playbooks as flat bullet lists (original format, used when no conflict).
+
+    Args:
+        cluster_playbooks: List of raw playbooks in this cluster
+
+    Returns:
+        str: Formatted input with separate field lists
+    """
+    triggers = []
+    rationales = []
+
+    for fb in cluster_playbooks:
+        if fb.trigger:
+            triggers.append(fb.trigger)
+        if fb.rationale:
+            rationales.append(fb.rationale)
+
+    lines: list[str] = []
+
+    if triggers:
+        lines.append("TRIGGER conditions (to be consolidated):")
+        lines.extend(f"- {trigger}" for trigger in triggers)
+    else:
+        lines.append("TRIGGER conditions: (none specified)")
+
+    if rationales:
+        lines.append("RATIONALE summaries:")
+        lines.extend(f"- {r}" for r in rationales)
+
+    append_freeform_observations(lines, cluster_playbooks)
+
+    return "\n".join(lines)
+
+
+def format_grouped(
+    groups: list[list[UserPlaybook]],
+) -> str:
+    """
+    Format playbooks in grouped layout (used when conflicting directions are detected).
+
+    Args:
+        groups: AgentPlaybook groups sorted by size descending
+
+    Returns:
+        str: Formatted input with group headers and per-playbook fields
+    """
+    lines: list[str] = [
+        "The following playbook items are grouped by similarity. "
+        "Groups are ordered by size (largest first).",
+        "",
+    ]
+
+    for idx, group in enumerate(groups, start=1):
+        count_label = "playbook" if len(group) == 1 else "playbooks"
+        lines.append(f"Group {idx} ({len(group)} {count_label}):")
+        for fb in group:
+            parts: list[str] = []
+            if fb.trigger:
+                parts.append(f'Trigger: "{fb.trigger}"')
+            if fb.rationale:
+                parts.append(f'Rationale: "{fb.rationale}"')
+            if not parts and fb.content:
+                parts.append(f'AgentPlaybook: "{fb.content}"')
+            if parts:
+                lines.append(f"  - {parts[0]}")
+                lines.extend(f"    {p}" for p in parts[1:])
+        lines.append("")
+
+    return "\n".join(lines)
+
+
+def append_freeform_observations(
+    lines: list[str], cluster_playbooks: list[UserPlaybook]
+) -> None:
+    """Append freeform observations from cluster playbooks to output lines."""
+    freeform_observations = [
+        fb.content for fb in cluster_playbooks if not fb.trigger and fb.content
+    ]
+    if freeform_observations:
+        lines.append("Freeform observations (from freeform cluster members):")
+        lines.extend(f"- {obs}" for obs in freeform_observations)