claude-smart 0.2.49 → 0.2.50

package/plugin/vendor/reflexio/reflexio/server/services/base_generation_service.py

@@ -5,6 +5,7 @@ Base class for generation services
 import logging
 import re
 import time
+import unicodedata
 import uuid
 from abc import ABC, abstractmethod
 from dataclasses import dataclass
@@ -25,6 +26,10 @@ from reflexio.server.services.base_generation import (
 from reflexio.server.services.base_generation import (
     StatusChangeOperation as StatusChangeOperation,  # re-export for back-compat
 )
+from reflexio.server.services.deferred_learning_plan import (
+    ExtractorBookmarkAdvance,
+    GenerationComputePlan,
+)
 from reflexio.server.services.extractor_config_utils import (
     get_extractor_name,
 )
@@ -38,10 +43,11 @@ class ExtractorExecutionError(RuntimeError):
 logger = logging.getLogger(__name__)
 
 
-# Cheap-signal thresholds for the pre-LLM should_run filter. Tuned for
+# Cheap-signal threshold for the pre-LLM should_run filter. Tuned for
 # coding-assistant traffic where most turns are either slash commands
 # or tool scaffolding, and where the LLM should_run gate costs 5–7s
-# even when it ultimately votes False.
+# even when it ultimately votes False. Non-Latin letters and numbers
+# count twice because many scripts carry more meaning per character.
 _MIN_USER_CONTENT_LEN = 30
 # Heuristic match for reflexio's own extractor system prompts that
 # sometimes leak into the corpus via the claude-code LLM provider's
@@ -76,17 +82,47 @@ def _is_pure_slash_command(content: str) -> bool:
 def _iter_user_contents(
     session_data_models: list[RequestInteractionDataModel],
 ) -> list[str]:
-    """Collect the ``content`` of every User-role interaction, order-preserving."""
+    """Collect non-empty user-authored content, preserving interaction order.
+
+    Role matching is case-insensitive because API and CLI publishers use
+    different casing. These contents feed user-text-only heuristics; they do
+    not determine stride or window eligibility.
+    """
     out: list[str] = []
     for model in session_data_models:
         out.extend(
             interaction.content
             for interaction in model.interactions
-            if interaction.role == "User" and interaction.content
+            if interaction.role.casefold() == "user" and interaction.content
         )
     return out
 
 
+def _weighted_content_length(content: str) -> int:
+    """Count original non-Latin letters and numbers twice.
+
+    Normalize each code point only for script classification so compatibility
+    forms such as full-width Latin stay unweighted without changing the
+    original content length.
+    """
+    stripped = content.strip()
+
+    def is_non_latin_letter_or_number(char: str) -> bool:
+        normalized_chars = [
+            normalized_char
+            for normalized_char in unicodedata.normalize("NFKC", char)
+            if unicodedata.category(normalized_char)[0] in {"L", "N"}
+        ]
+        return bool(normalized_chars) and all(
+            not normalized_char.isascii()
+            and "LATIN" not in unicodedata.name(normalized_char, "")
+            for normalized_char in normalized_chars
+        )
+
+    non_latin_bonus = sum(1 for char in stripped if is_non_latin_letter_or_number(char))
+    return len(stripped) + non_latin_bonus
+
+
 def _cheap_should_run_reject(
     session_data_models: list[RequestInteractionDataModel],
 ) -> str | None:
@@ -97,9 +133,14 @@ def _cheap_should_run_reject(
     LLM call. Returns None when we cannot decide cheaply and the LLM
     should run.
 
-    Rejection rules:
-        - No user message at least ``_MIN_USER_CONTENT_LEN`` chars long
-          (purely short commands / confirmations).
+    When the scoped batch has no non-empty user-authored content, these
+    user-text-only heuristics are inapplicable and the batch falls through to
+    the LLM gate. Stride and window eligibility are evaluated independently.
+
+    Rejection rules when user-authored content is present:
+        - No user message reaches ``_MIN_USER_CONTENT_LEN`` weighted
+          characters (purely short commands / confirmations). Non-Latin
+          letters and numbers count twice.
         - Every user message is a bare slash-command dispatch with no
           substantive trailing text (e.g. ``/commit``, ``/review``,
           ``/claude-smart:tag``). Slash commands that carry user text
@@ -116,14 +157,18 @@ def _cheap_should_run_reject(
     """
     user_contents = _iter_user_contents(session_data_models)
     if not user_contents:
-        return "no_user_turns"
+        return None
 
     for content in user_contents:
         lowered = content.lstrip().lower()
         if any(lowered.startswith(p) for p in _EXTRACTOR_PROMPT_PREFIXES):
             return "extractor_prompt_echo"
 
-    if not any(len(c.strip()) >= _MIN_USER_CONTENT_LEN for c in user_contents):
+    if not any(
+        len(content.strip()) >= _MIN_USER_CONTENT_LEN
+        or _weighted_content_length(content) >= _MIN_USER_CONTENT_LEN
+        for content in user_contents
+    ):
         return "all_user_turns_too_short"
 
     if all(_is_pure_slash_command(c) for c in user_contents):
@@ -159,8 +204,9 @@ class BaseGenerationService(
     ABC,
     Generic[TExtractorConfig, TExtractor, TGenerationServiceConfig, TRequest],  # noqa: UP046
 ):
-    # Only profile/playbook GENERATION services bill the ② Learning line;
-    # reflection/consolidation/aggregation/evaluation are bundled, not metered.
+    # Only profile/playbook GENERATION services emit extraction-run billing here.
+    # Non-extraction learning mutation paths emit their value facet at their own
+    # durable-success point.
     # Default is False so any future subclass is safe by default (opt-IN).
     EMITS_LEARNING_BILLING: bool = False
     """
@@ -208,6 +254,9 @@ class BaseGenerationService(
         }
         self._last_extraction_run_ids: list[str] = []
         self._last_token_totals: RunTokenTotals | None = None
+        # Stride-bookmark advance deferred off the extractor (F1); captured in
+        # ``_execute_extractor`` and applied in the persist half of the run.
+        self._last_bookmark_advance: ExtractorBookmarkAdvance | None = None
         # Window fetched by the should-run gate (_collect_scoped_interactions_for_precheck),
         # stashed so the billing path (_extraction_input_text) can reuse it instead of
         # re-querying storage. None when the gate did not run (bypass paths).
@@ -428,7 +477,7 @@ class BaseGenerationService(
 
         # Get scope ID and request ID for in-progress tracking
         scope_id = self._get_lock_scope_id(request)
-        my_request_id = getattr(request, "request_id", None) or str(uuid.uuid4())
+        lock_request_id = getattr(request, "request_id", None) or str(uuid.uuid4())
 
         state_manager = self._create_state_manager()
 
@@ -439,7 +488,7 @@ class BaseGenerationService(
         # never get extracted.
         my_payload = self._serialize_request_for_queue(request)
         if not state_manager.acquire_lock(
-            my_request_id, scope_id=scope_id, payload=my_payload
+            lock_request_id, scope_id=scope_id, payload=my_payload
         ):
             return  # Another operation is running, we've enqueued ourselves
 
@@ -464,21 +513,21 @@ class BaseGenerationService(
                 # request's ID + payload so the rerun runs against THAT
                 # publish's data, not the original holder's.
                 next_entry = state_manager.release_lock_pop_queue(
-                    my_request_id, scope_id=scope_id
+                    lock_request_id, scope_id=scope_id
                 )
 
                 if next_entry is None:
                     break  # Queue empty — we're done
 
-                next_request_id = next_entry["request_id"]
+                next_lock_request_id = next_entry["request_id"]
                 next_payload = next_entry.get("payload")
 
                 logger.info(
-                    "Draining queued %s request: prev_request_id=%s, next_request_id=%s, "
+                    "Draining queued %s request: prev_lock_request_id=%s, next_lock_request_id=%s, "
                     "payload_present=%s",
                     self._get_service_name(),
-                    my_request_id,
-                    next_request_id,
+                    lock_request_id,
+                    next_lock_request_id,
                     next_payload is not None,
                 )
 
@@ -492,7 +541,7 @@ class BaseGenerationService(
                 else:
                     current_request = request
 
-                my_request_id = next_request_id
+                lock_request_id = next_lock_request_id
 
         except Exception:
             # Clear lock on error to prevent deadlock
@@ -500,69 +549,29 @@ class BaseGenerationService(
             raise
 
     def _run_generation(self, request: TRequest) -> None:
-        """
-        Run the actual generation logic.
+        """Run one generation synchronously: compute -> persist -> emit.
+
+        Thin wrapper preserving the pre-split behavior for the synchronous
+        ``.run()`` / manual / rerun callers: compute (extractor + dedup +
+        embeddings), persist (row writes + the extractor stride-bookmark
+        advance) and post-commit side-effects (telemetry + billing) run together
+        with **no** external ``commit_scope``. Only the durable worker (Task 8/9)
+        drives ``compute_generation`` / ``persist_generation`` /
+        ``emit_generation_side_effects`` separately across its fenced scope.
 
-        Orchestrates validation, config loading, extractor execution, and result
-        processing by delegating to _prepare_generation_run and _execute_extractor.
+        The outer ``except`` records ``generation_failed`` and re-raises
+        ``ExtractorExecutionError`` exactly as before.
 
         Args:
             request: The request object containing parameters
         """
-        if not request:
-            logger.error("Received None request for %s", self._get_service_name())
-            return
-
         generation_start = time.perf_counter()
         try:
-            prepared = self._prepare_generation_run(request)
-            if prepared is None:
+            plan = self.compute_generation(request)
+            if plan is None:
                 return
-
-            self._record_generation_event(
-                event_name="generation_started",
-                outcome="started",
-                count_value=1,
-                metadata={
-                    "identifier": prepared.identifier,
-                    "extractor_name": prepared.extractor_name,
-                },
-            )
-            self._last_extraction_run_ids = []
-            self._last_token_totals = None
-            result = self._execute_extractor(
-                prepared.extractor_config, prepared.identifier
-            )
-            generated_count = self._count_generated_results(result)
-
-            try:
-                if result:
-                    self._process_results([result])
-                self._finalize_extraction_runs()
-            except Exception as exc:
-                self._mark_extraction_runs_finalization_failed(exc)
-                raise
-
-            self._record_generation_event(
-                event_name="generation_succeeded",
-                outcome="success",
-                count_value=generated_count,
-                duration_ms=int((time.perf_counter() - generation_start) * 1000),
-                metadata={
-                    "identifier": prepared.identifier,
-                    "extractor_name": prepared.extractor_name,
-                    "extractor_failed": bool(
-                        self._last_extractor_run_stats.get("failed")
-                    ),
-                    "extractor_timed_out": bool(
-                        self._last_extractor_run_stats.get("timed_out")
-                    ),
-                },
-            )
-            self._record_billing_learning_events(
-                prepared=prepared, generated_count=generated_count
-            )
-
+            self.persist_generation(plan)
+            self.emit_generation_side_effects(plan)
         except Exception as e:
             self._record_generation_event(
                 event_name="generation_failed",
@@ -579,6 +588,178 @@ class BaseGenerationService(
             if isinstance(e, ExtractorExecutionError):
                 raise
 
+    def compute_generation(self, request: TRequest) -> GenerationComputePlan | None:
+        """Compute half of one generation run — NO learning DB write, NO fence.
+
+        Runs the prepare gate, the extractor (thread-pool LLM tool-loop), dedup
+        + embedding resolution (``_resolve_write_plan``), and drives the
+        ``agent_run`` rows to their terminal state (``_finalize_extraction_runs``
+        — agent_run only, §4.3). Snapshots the billing inputs onto the returned
+        plan so ``emit_generation_side_effects`` never depends on the reused
+        instance's mutable ``_last_*``.
+
+        Returns a resolved ``GenerationComputePlan`` for persist/emit, or
+        ``None`` when the prepare gate closes (nothing to run). Raises
+        ``ExtractorExecutionError`` on extractor failure — the caller
+        (``_run_generation`` / the durable worker) records ``generation_failed``.
+        """
+        if not request:
+            logger.error("Received None request for %s", self._get_service_name())
+            return None
+
+        generation_start = time.perf_counter()
+        prepared = self._prepare_generation_run(request)
+        if prepared is None:
+            return None
+
+        self._record_generation_event(
+            event_name="generation_started",
+            outcome="started",
+            count_value=1,
+            metadata={
+                "identifier": prepared.identifier,
+                "extractor_name": prepared.extractor_name,
+            },
+        )
+        self._last_extraction_run_ids = []
+        self._last_token_totals = None
+        self._last_bookmark_advance = None
+        result = self._execute_extractor(prepared.extractor_config, prepared.identifier)
+        generated_count = self._count_generated_results(result)
+
+        try:
+            write_plan = self._resolve_write_plan([result]) if result else None
+            self._finalize_extraction_runs()
+        except Exception as exc:
+            self._mark_extraction_runs_finalization_failed(exc)
+            raise
+
+        return GenerationComputePlan(
+            prepared=prepared,
+            generated_count=generated_count,
+            write_plan=write_plan,
+            bookmark_advance=self._last_bookmark_advance,
+            generation_start=generation_start,
+            extraction_run_ids=list(self._last_extraction_run_ids),
+            token_totals=self._last_token_totals,
+        )
+
+    def persist_generation(self, plan: GenerationComputePlan) -> None:
+        """Persist half — apply the write-plan + the extractor bookmark advance.
+
+        This is the ONLY part that runs inside the durable worker's fenced
+        ``commit_scope`` (and inline on the synchronous ``.run()`` path). It
+        issues only the resolved write-plan's row writes (embeddings already
+        precomputed in compute for Tasks 6-7) and the extractor stride-bookmark
+        advance — **no** events, **no** billing, **no** LLM / embedding compute.
+
+        The bookmark advance is applied HERE so that BOTH the synchronous
+        ``.run()`` path and the durable persist fence advance the bookmark
+        atomically with the row writes it corresponds to (F1) — and it is
+        applied in exactly one place, so it is never double-applied.
+        """
+        if plan.write_plan is not None:
+            self._persist_write_plan(plan.write_plan)
+        self._apply_bookmark_advance(plan.bookmark_advance)
+
+    def emit_generation_side_effects(self, plan: GenerationComputePlan) -> None:
+        """Post-commit side-effects — telemetry + ② Learning billing.
+
+        Runs only for a fence-winning job (the durable worker calls it after the
+        scope commits; ``.run()`` calls it inline). Reads the plan's compute-time
+        snapshot (``generated_count`` / ``prepared`` / ``generation_start``) so a
+        fence-lost job never emits.
+
+        Billing purity note (round-2 finding): ``_record_billing_learning_events``
+        also reads ``self._last_token_totals`` / ``self._last_precheck_sessions``,
+        and the ``generation_succeeded`` metadata reads ``self._last_extractor_run_stats``.
+        These are NOT re-plumbed to the plan — the money helper lives in a
+        separate mixin behind a documented patch seam (``_usage_billing.py`` SINK-3)
+        and re-plumbing it would touch the money path for zero behavior change.
+        It is safe under the **single-use-instance invariant**: every generation
+        service instance runs exactly one compute→persist→emit for one job with
+        no interleaving (``.run()`` is synchronous; the durable orchestration in
+        Task 8 holds one ``(instance, plan)`` pair per profile/playbook service
+        and never re-runs compute on that instance before emit), so ``self._last_*``
+        is exactly as compute left it when emit runs. The plan still snapshots the
+        billing inputs so a future durable reuse can re-plumb locally.
+        """
+        self._record_generation_event(
+            event_name="generation_succeeded",
+            outcome="success",
+            count_value=plan.generated_count,
+            duration_ms=int((time.perf_counter() - plan.generation_start) * 1000),
+            metadata={
+                "identifier": plan.prepared.identifier,
+                "extractor_name": plan.prepared.extractor_name,
+                "extractor_failed": bool(self._last_extractor_run_stats.get("failed")),
+                "extractor_timed_out": bool(
+                    self._last_extractor_run_stats.get("timed_out")
+                ),
+            },
+        )
+        self._record_billing_learning_events(
+            prepared=plan.prepared, generated_count=plan.generated_count
+        )
+
+    @abstractmethod
+    def _resolve_write_plan(self, results: list) -> Any | None:
+        """Compute-half of item finalization — resolve a write-plan, NO DB write.
+
+        Runs the dedup + source/status assignment + embedding precompute and
+        returns a resolved write-plan (or ``None`` when there is nothing to
+        write). Issues NO learning DB write — the write is the persist half's
+        job (``_persist_write_plan``), so this stays inside the compute purity
+        contract (the compute-write-tripwire contract test).
+
+        The durable-split ``ProfileGenerationService`` /
+        ``PlaybookGenerationService`` return a real ``ProfileWritePlan`` /
+        ``PlaybookWritePlan`` (Tasks 6-7). ``AgentSuccessEvaluationService`` was
+        never split into a durable persist path — it keeps a concrete override
+        that writes in compute via its permanent ``_finalize_extracted_items``
+        wrapper (its results never flow through the durable worker fence).
+        """
+
+    @abstractmethod
+    def _persist_write_plan(self, plan: Any) -> None:
+        """Persist-half of item finalization — apply the resolved write-plan.
+
+        Issues only the fence-critical row writes for a plan produced by
+        ``_resolve_write_plan`` (embeddings already precomputed in compute), with
+        NO LLM / embedding / dedup. This is the only item-finalization work that
+        runs inside the durable worker's fenced ``commit_scope``.
+
+        ``AgentSuccessEvaluationService`` (never split) implements this as a
+        no-op — its ``_resolve_write_plan`` performs the write in compute and
+        returns ``None`` so ``persist_generation`` never invokes this.
+        """
+
+    def _apply_bookmark_advance(self, advance: ExtractorBookmarkAdvance | None) -> None:
+        """Apply the deferred extractor stride-bookmark advance (F1).
+
+        The extractor emits its bookmark advance on the ``ExtractionOutcome``
+        rather than self-advancing; ``_execute_extractor`` captures it and
+        ``compute_generation`` snapshots it onto the ``GenerationComputePlan``.
+        This applies it via ``update_extractor_bookmark`` using the same
+        OperationStateManager service name the extractor used, so the bookmark
+        key is byte-identical. No-op when the extractor produced no output
+        (``advance`` is ``None``) or the service has no stride bookmark.
+
+        Args:
+            advance: The bookmark advance from ``plan.bookmark_advance``.
+        """
+        if advance is None or self.storage is None:
+            return
+        service_name = self._get_extractor_state_service_name()
+        if service_name is None:
+            return
+        manager = OperationStateManager(self.storage, self.org_id, service_name)
+        manager.update_extractor_bookmark(
+            extractor_name=advance.extractor_name,
+            processed_interactions=advance.processed_interactions,
+            user_id=advance.user_id,
+        )
+
     def _prepare_generation_run(
         self, request: TRequest
     ) -> PreparedGenerationRun[TExtractorConfig] | None:

package/plugin/vendor/reflexio/reflexio/server/services/deduplication_utils.py

@@ -8,14 +8,86 @@ ProfileConsolidator and PlaybookConsolidator.
 import logging
 from abc import ABC
 from datetime import UTC, datetime
+from typing import Any, cast
 
 from reflexio.server.api_endpoints.request_context import RequestContext
 from reflexio.server.llm.litellm_client import LiteLLMClient
 from reflexio.server.llm.model_defaults import ModelRole, resolve_model_name
+from reflexio.server.services.embedding_text import embedding_input
 from reflexio.server.site_var.site_var_manager import SiteVarManager
 
 logger = logging.getLogger(__name__)
 
+
+def resolve_dedup_query_embeddings(
+    storage: Any,
+    client: LiteLLMClient,
+    query_texts: list[str],
+    *,
+    entity_label: str,
+) -> list[list[float] | None]:
+    """Embed dedup-search queries with the model that indexed the store.
+
+    Prefers the storage's own embedding path (correct model + "query" prefix);
+    falls back to ``client`` pinned to the storage's model/dimensions. Letting
+    the client resolve its default embedding model would be wrong here: it
+    picks the OSS default, which the enterprise embedding daemon rejects
+    (409 model conflict) and which would not match the indexed vectors anyway.
+
+    Args:
+        storage: Storage backend (duck-typed: ``_get_embedding`` preferred,
+            else ``embedding_model_name`` / ``embedding_dimensions``).
+        client: Shared LLM client, used only in the fallback path.
+        query_texts: Query strings to embed.
+        entity_label: Log prefix identifying the caller, e.g. "Profile".
+
+    Returns:
+        One embedding (or None) per query text, in input order. On any
+        failure, all entries are None so the caller degrades to text-only
+        search. A backend that signals embedding-service unavailability with
+        an empty vector (e.g. the Supabase query path) is normalized to None
+        so search falls back to its own embedding/FTS path instead of sending
+        an empty vector to the database.
+    """
+    try:
+        get_storage_embedding = getattr(storage, "_get_embedding", None)
+        if callable(get_storage_embedding):
+            logger.info(
+                "%s dedup query embeddings: source=storage model=%s",
+                entity_label,
+                getattr(storage, "embedding_model_name", "unknown"),
+            )
+            embeddings = [
+                cast(
+                    "list[float] | None",
+                    get_storage_embedding(query_text, purpose="query"),
+                )
+                for query_text in query_texts
+            ]
+        else:
+            embedding_model_name = storage.embedding_model_name
+            embedding_dimensions = storage.embedding_dimensions
+            logger.info(
+                "%s dedup query embeddings: source=llm_client model=%s",
+                entity_label,
+                embedding_model_name,
+            )
+            embeddings = list(
+                client.get_embeddings(
+                    [
+                        embedding_input(query_text, purpose="query")
+                        for query_text in query_texts
+                    ],
+                    model=embedding_model_name,
+                    dimensions=embedding_dimensions,
+                )
+            )
+    except Exception as e:
+        logger.warning("Failed to generate embeddings for dedup search: %s", e)
+        return [None] * len(query_texts)
+    return [emb or None for emb in embeddings]
+
+
 # Format used for "Last Modified" timestamps shown to deduplication LLMs.
 # Includes hours and minutes so same-day contradictions (morning vs evening)
 # can be distinguished.