claude-smart 0.2.44 → 0.2.46

package/plugin/vendor/reflexio/reflexio/server/services/extraction/resumable_agent.py

@@ -40,15 +40,8 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
-FINISH_EXTRACTION_TOOL_NAME = "finish_extraction"
 PROFILE_EXTRACTOR_KIND = "profile"
 
-# Max in-process retries when the model returns a plain-text turn with no tool
-# call (finished_reason="no_tool_call"). tool_choice="required" is not always
-# honored by every provider/model, so a single bad turn would otherwise drop the
-# extraction entirely. Retries re-issue the same prompt from scratch.
-_NO_TOOL_CALL_MAX_RETRIES = 2
-
 
 def _record_agent_usage_event(
     *,
@@ -85,17 +78,6 @@ class AgentRunResult:
     finished_reason: str
 
 
-@dataclass(slots=True)
-class _FinishExtractionContext:
-    output: BaseModel | None = None
-
-
-@dataclass(slots=True)
-class _ExtractionAgentToolContext:
-    finish_context: _FinishExtractionContext
-    extra_tool_context: Any | None = None
-
-
 def _format_resolved_tool_result(record: PendingToolCallRecord) -> str:
     resolved_at = record.resolved_at.isoformat() if record.resolved_at else "unknown"
     return (
@@ -106,7 +88,7 @@ def _format_resolved_tool_result(record: PendingToolCallRecord) -> str:
         f"Result: {record.result or {}}\n\n"
         "Use this Agent Builder feedback only if it is relevant to the "
         "current extraction window. If it adds or corrects durable profile or "
-        "playbook information, include that in finish_extraction."
+        "playbook information, include that in your structured extraction result."
     )
 
 
@@ -128,22 +110,6 @@ def append_resolved_tool_result_context(
     ]
 
 
-def _finish_handler(args: BaseModel, ctx: Any) -> dict[str, Any]:
-    ctx = getattr(ctx, "finish_context", ctx)
-    if not isinstance(ctx, _FinishExtractionContext):
-        raise TypeError(f"Expected _FinishExtractionContext, got {type(ctx).__name__}")
-    ctx.output = args
-    return {"status": "completed"}
-
-
-def create_finish_extraction_tool(output_schema: type[BaseModel]) -> Tool:
-    return Tool(
-        name=FINISH_EXTRACTION_TOOL_NAME,
-        args_model=output_schema,
-        handler=_finish_handler,
-    )
-
-
 def _pending_tool_call_config(request_context: RequestContext) -> Any | None:
     root_config = request_context.configurator.get_config()
     return (
@@ -361,90 +327,38 @@ class ResumableExtractionAgent:
         if run.max_steps_remaining is not None:
             max_steps = min(max_steps, max(0, run.max_steps_remaining))
 
-        # When only finish_extraction is registered (no human/prior-knowledge
-        # tools), force that tool so the degenerate one-call loop is strictly
-        # equivalent to the old single-shot response_format extraction —
-        # the model cannot return a no-tool turn and yield empty output.
+        # The extractor delivers its result as a native structured response
+        # (response_format=output_schema): a turn with no tool call IS the
+        # committed output, parsed into output_schema. ask_human /
+        # attach_pending_info_request stay real tools the model may call
+        # mid-run, so tool_choice="auto" — forcing a tool would suppress the
+        # structured finish and is what produced the no_tool_call failures.
         #
-        # When the async-info tools are also registered we cannot force a single
-        # tool (the model must be free to pick ask_human vs finish_extraction),
-        # but we still require *some* tool call via "required". This prevents a
-        # weak tool-caller (e.g. MiniMax) from emitting the answer as plain text
-        # with zero tool_calls, which the loop would otherwise treat as a no-op
-        # finish and drop the output.
-        tool_choice: str | dict[str, Any] = (
-            {
-                "type": "function",
-                "function": {"name": FINISH_EXTRACTION_TOOL_NAME},
-            }
-            if not extra_tools
-            else "required"
+        # No retry loop: a plain (no-tool) turn is now the SUCCESS terminus
+        # (finished_reason="structured_output"), not a dropped output, and the
+        # client already retries once on a malformed structured parse. The
+        # async-info tool handlers read ctx via getattr(ctx, "extra_tool_context",
+        # ctx), so the bare context object can be passed directly.
+        registry = ToolRegistry(list(extra_tools or []))
+        result = run_tool_loop(
+            client=self.client,
+            messages=messages,
+            registry=registry,
+            model_role=self.model_role,
+            max_steps=max_steps,
+            ctx=extra_tool_context,
+            response_format=output_schema,
+            tool_choice="auto",
+            log_label=log_label,
         )
 
-        # Retry only the no_tool_call termination: the model emitted plain text
-        # with zero tool calls, so nothing was committed and re-issuing the same
-        # prompt is safe. Each attempt uses a fresh finish_ctx/registry so no
-        # partial state leaks across attempts. We do NOT retry once the model has
-        # already registered async tool calls (e.g. attach_pending_info_request)
-        # — restarting from scratch would discard them.
-        for attempt in range(_NO_TOOL_CALL_MAX_RETRIES + 1):
-            finish_ctx = _FinishExtractionContext()
-            ctx: Any = (
-                _ExtractionAgentToolContext(
-                    finish_context=finish_ctx,
-                    extra_tool_context=extra_tool_context,
-                )
-                if extra_tool_context is not None
-                else finish_ctx
-            )
-            registry = ToolRegistry(
-                [*(extra_tools or []), create_finish_extraction_tool(output_schema)]
-            )
-
-            result = run_tool_loop(
-                client=self.client,
-                messages=messages,
-                registry=registry,
-                model_role=self.model_role,
-                max_steps=max_steps,
-                ctx=ctx,
-                finish_tool_name=FINISH_EXTRACTION_TOOL_NAME,
-                tool_choice=tool_choice,
-                log_label=log_label,
-            )
-
-            should_retry = (
-                result.finished_reason == "no_tool_call"
-                and not result.pending_tool_call_ids
-                and attempt < _NO_TOOL_CALL_MAX_RETRIES
-            )
-            if not should_retry:
-                break
-            logger.warning(
-                "event=extraction_agent_retry org_id=%s user_id=%s "
-                "extractor_kind=%s run_id=%s request_id=%s "
-                "finished_reason=%s attempt=%d max_retries=%d",
-                run.binding.org_id,
-                run.binding.user_id,
-                run.binding.extractor_kind,
-                run.id,
-                run.binding.request_id,
-                result.finished_reason,
-                attempt + 1,
-                _NO_TOOL_CALL_MAX_RETRIES,
-            )
-            _record_agent_usage_event(
-                run=run,
-                event_name="extraction_agent_retry",
-                error_kind=result.finished_reason,
-                metadata={"attempt": attempt + 1},
-            )
-
-        committed_output = (
-            finish_ctx.output.model_dump() if finish_ctx.output is not None else None
-        )
+        output = result.structured_output
+        committed_output = output.model_dump() if output is not None else None
         active_statuses = (AgentRunStatus.RUNNING, AgentRunStatus.RESUMING)
-        if result.finished_reason == "finish_tool" and committed_output is not None:
+        if (
+            result.finished_reason == "structured_output"
+            and committed_output is not None
+        ):
             stored_run = self.storage.update_agent_run_status(
                 run.id,
                 AgentRunStatus.AGENT_COMPLETED,
@@ -515,19 +429,19 @@ class ResumableExtractionAgent:
                 run.id,
                 run.binding.request_id,
                 result.finished_reason,
-                finish_ctx.output is not None,
+                output is not None,
             )
             _record_agent_usage_event(
                 run=run,
                 event_name="extraction_agent_failed",
                 outcome="failed",
                 error_kind=result.finished_reason,
-                metadata={"has_output": finish_ctx.output is not None},
+                metadata={"has_output": output is not None},
             )
 
         return AgentRunResult(
             run_id=run.id,
-            output=finish_ctx.output,
+            output=output,
             pending_tool_call_ids=result.pending_tool_call_ids,
             messages=result.messages,
             trace=result.trace,

package/plugin/vendor/reflexio/reflexio/server/services/extraction/resume_worker.py

@@ -28,29 +28,29 @@ from reflexio.server.services.extraction.resumable_agent import (
     ResumableExtractionAgent,
     create_pending_info_tools_for_extractor_kind,
 )
-from reflexio.server.services.playbook.playbook_extractor import PlaybookExtractor
-from reflexio.server.services.playbook.playbook_generation_service import (
-    PlaybookGenerationService,
-    PlaybookGenerationServiceConfig,
-)
+from reflexio.server.services.playbook.components.extractor import PlaybookExtractor
 from reflexio.server.services.playbook.playbook_service_utils import (
     StructuredPlaybookList,
     construct_expert_playbook_extraction_messages,
     construct_playbook_extraction_messages_from_sessions,
     has_expert_content,
 )
-from reflexio.server.services.profile.profile_extractor import (
+from reflexio.server.services.playbook.service import (
+    PlaybookGenerationService,
+    PlaybookGenerationServiceConfig,
+)
+from reflexio.server.services.profile.components.extractor import (
     MAX_EXISTING_PROFILES_FOR_CONTEXT,
     ProfileExtractor,
 )
-from reflexio.server.services.profile.profile_generation_service import (
-    ProfileGenerationService,
-    ProfileGenerationServiceConfig,
-)
 from reflexio.server.services.profile.profile_generation_service_utils import (
     StructuredProfilesOutput,
     construct_profile_extraction_messages_from_sessions,
 )
+from reflexio.server.services.profile.service import (
+    ProfileGenerationService,
+    ProfileGenerationServiceConfig,
+)
 from reflexio.server.services.service_utils import (
     extract_interactions_from_request_interaction_data_models,
 )
@@ -61,6 +61,7 @@ from reflexio.server.services.storage.storage_base import (
     PendingToolCallRecord,
     PendingToolCallStatus,
 )
+from reflexio.server.services.tagging.tagging_scheduler import schedule_tagging
 from reflexio.server.site_var.site_var_manager import SiteVarManager
 from reflexio.server.tracing import sentry_tags
 
@@ -133,7 +134,7 @@ def _request_interaction_models_from_ids(
             )
         models.append(
             RequestInteractionDataModel(
-                session_id=request.session_id or request.request_id,
+                session_id=request.session_id,
                 request=request,
                 interactions=ordered,
             )
@@ -272,6 +273,7 @@ class ExtractionResumeWorker:
         try:
             self.storage.update_agent_run_status(run.id, AgentRunStatus.FINALIZING)
             self._finalize_items(run, items)
+            self._schedule_finalized_tagging(run)
             self.storage.consume_run_tool_dependencies(run.id)
             finalized_status = (
                 AgentRunStatus.FINALIZED_PENDING_TOOL
@@ -315,6 +317,7 @@ class ExtractionResumeWorker:
         try:
             items, pending_tool_call_ids = self._items_from_committed_output(run)
             self._finalize_items(run, items)
+            self._schedule_finalized_tagging(run)
             self.storage.consume_run_tool_dependencies(run.id)
             finalized_status = (
                 AgentRunStatus.FINALIZED_PENDING_TOOL
@@ -460,11 +463,6 @@ class ExtractionResumeWorker:
                 else ""
             ),
             extraction_definition_prompt=extractor_config.extraction_definition_prompt.strip(),
-            metadata_definition_prompt=(
-                extractor_config.metadata_definition_prompt.strip()
-                if extractor_config.metadata_definition_prompt
-                else None
-            ),
             existing_profiles=context_profiles,
         )
         result = self._resume_agent(
@@ -777,3 +775,28 @@ class ExtractionResumeWorker:
         raise ResumeWorkerError(
             f"Unsupported extractor kind {run.binding.extractor_kind!r}"
         )
+
+    def _schedule_finalized_tagging(self, run: AgentRunRecord) -> None:
+        user_id = run.binding.user_id
+        if not user_id:
+            return
+        if run.binding.extractor_kind not in ("profile", "playbook"):
+            return
+
+        # Defer tagging off this worker's drain loop. The pass is idempotent
+        # (skips already-tagged entities), so running both profile and playbook
+        # tagging is safe regardless of this run's extractor kind.
+        try:
+            schedule_tagging(
+                org_id=self.request_context.org_id,
+                user_id=user_id,
+                agent_version=run.binding.agent_version or "",
+                request_context=self.request_context,
+                llm_client=self.client,
+            )
+        except Exception:
+            logger.exception(
+                "Failed to schedule tagging for finalized %s run %s",
+                run.binding.extractor_kind,
+                run.id,
+            )

package/plugin/vendor/reflexio/reflexio/server/services/generation_service.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import contextvars
+import hashlib
 import logging
 import os
 import threading
@@ -21,33 +22,34 @@ from reflexio.models.api_schema.service_schemas import (
 from reflexio.models.config_schema import Config
 from reflexio.server.api_endpoints.request_context import RequestContext
 from reflexio.server.llm.litellm_client import LiteLLMClient
-from reflexio.server.services.agent_success_evaluation.delayed_group_evaluator import (
-    GroupEvaluationScheduler,
-)
-from reflexio.server.services.agent_success_evaluation.group_evaluation_runner import (
+from reflexio.server.services.agent_success_evaluation.runner import (
     run_group_evaluation,
 )
-from reflexio.server.services.operation_state_utils import OperationStateManager
-from reflexio.server.services.playbook.playbook_generation_service import (
-    PlaybookGenerationService,
+from reflexio.server.services.agent_success_evaluation.scheduler import (
+    GroupEvaluationScheduler,
 )
+from reflexio.server.services.operation_state_utils import OperationStateManager
 from reflexio.server.services.playbook.playbook_service_utils import (
     PlaybookGenerationRequest,
 )
-from reflexio.server.services.profile.profile_generation_service import (
-    ProfileGenerationService,
+from reflexio.server.services.playbook.service import (
+    PlaybookGenerationService,
 )
 from reflexio.server.services.profile.profile_generation_service_utils import (
     ProfileGenerationRequest,
 )
-from reflexio.server.services.reflection.reflection_service import ReflectionService
+from reflexio.server.services.profile.service import (
+    ProfileGenerationService,
+)
 from reflexio.server.services.reflection.reflection_service_utils import (
     ReflectionServiceRequest,
 )
+from reflexio.server.services.reflection.service import ReflectionService
 from reflexio.server.services.storage.retention import (
     delete_count_for_retention,
     get_row_retention_limits,
 )
+from reflexio.server.services.tagging.tagging_scheduler import schedule_tagging
 from reflexio.server.tracing import sentry_tags
 from reflexio.server.usage_metrics import record_usage_event
 
@@ -79,6 +81,15 @@ _retention_cleanup_last_run: dict[tuple[str, str], float] = {}
 _retention_cleanup_lock = threading.Lock()
 
 
+def _stable_group_sampling_fraction(
+    org_id: str, user_id: str, session_id: str
+) -> float:
+    """Return a deterministic [0, 1) sample value for one session."""
+    key = f"{org_id}\0{user_id}\0{session_id}".encode()
+    digest = hashlib.sha256(key).digest()
+    return int.from_bytes(digest[:8], "big") / 2**64
+
+
 @dataclass
 class GenerationServiceResult:
     """Result of a GenerationService.run call.
@@ -204,7 +215,7 @@ class GenerationService:
                 org_id=self.org_id,
                 user_id=user_id,
                 request_id=request_id,
-                session_id=publish_user_interaction_request.session_id or None,
+                session_id=publish_user_interaction_request.session_id,
                 source=publish_user_interaction_request.source,
                 agent_version=agent_version,
                 event_name="publish_request_received",
@@ -213,16 +224,15 @@ class GenerationService:
                 count_value=len(new_interactions),
             )
 
-            # Store Request — propagate customer-stamped metadata so the
-            # eval pipeline (e.g. F2 sticky-group aggregator) can read it
-            # back from the first request of each session.
+            # Store Request before adding interactions so downstream workers can
+            # resolve the session's source and evaluation-only status.
             new_request = Request(
                 request_id=request_id,
                 user_id=user_id,
                 source=publish_user_interaction_request.source,
                 agent_version=agent_version,
-                session_id=publish_user_interaction_request.session_id or None,
-                metadata=publish_user_interaction_request.metadata,
+                session_id=publish_user_interaction_request.session_id,
+                evaluation_only=publish_user_interaction_request.evaluation_only,
             )
             self.storage.add_request(new_request)  # type: ignore[reportOptionalMemberAccess]
 
@@ -234,6 +244,33 @@ class GenerationService:
             # Extract source (empty string treated as None)
             source = publish_user_interaction_request.source or None
 
+            if publish_user_interaction_request.evaluation_only:
+                self._schedule_group_evaluation_if_needed(
+                    new_request=new_request,
+                    user_id=user_id,
+                    agent_version=agent_version,
+                    source=source,
+                )
+                record_usage_event(
+                    org_id=self.org_id,
+                    user_id=user_id,
+                    request_id=request_id,
+                    session_id=new_request.session_id,
+                    source=source,
+                    agent_version=agent_version,
+                    backend="evaluation_only",
+                    event_name="publish_request_succeeded",
+                    event_category="publish",
+                    outcome="success",
+                    count_value=len(new_interactions),
+                    duration_ms=int((time.perf_counter() - publish_start) * 1000),
+                    metadata={
+                        "evaluation_only": True,
+                        "warning_count": len(result.warnings),
+                    },
+                )
+                return result
+
             if (
                 not publish_user_interaction_request.override_learning_stall
                 and (stall_warning := self._active_learning_stall_warning()) is not None
@@ -262,7 +299,10 @@ class GenerationService:
             # profile/playbook context. Wrapped in a broad except so a
             # reflection bug never breaks the publish.
             self._maybe_run_reflection(
-                user_id=user_id, agent_version=agent_version, source=source
+                user_id=user_id,
+                request_id=request_id,
+                agent_version=agent_version,
+                source=source,
             )
 
             # Create generation services and requests
@@ -343,7 +383,24 @@ class GenerationService:
             finally:
                 executor.shutdown(wait=False, cancel_futures=True)
 
-            # Schedule delayed group evaluation if session_id is present
+            # Tagging runs an LLM call per newly generated entity, so defer it off
+            # the publish path. The scheduler debounces bursts and is idempotent
+            # (already-tagged entities are skipped).
+            try:
+                schedule_tagging(
+                    org_id=self.org_id,
+                    user_id=user_id,
+                    agent_version=agent_version,
+                    request_context=self.request_context,
+                    llm_client=self.client,
+                )
+            except Exception:
+                logger.exception(
+                    "Failed to schedule tagging for publish request %s",
+                    request_id,
+                )
+
+            # Schedule delayed group evaluation for the required session.
             self._schedule_group_evaluation_if_needed(
                 new_request=new_request,
                 user_id=user_id,
@@ -373,7 +430,7 @@ class GenerationService:
                 org_id=self.org_id,
                 user_id=user_id,
                 request_id=result.request_id,
-                session_id=publish_user_interaction_request.session_id or None,
+                session_id=publish_user_interaction_request.session_id,
                 source=publish_user_interaction_request.source,
                 agent_version=agent_version,
                 event_name="publish_request_failed",
@@ -408,7 +465,7 @@ class GenerationService:
         agent_version: str,
         source: str | None,
     ) -> None:
-        """Enqueue agent-success evaluation for this session, if session_id is set.
+        """Enqueue agent-success evaluation for this session.
 
         Must be called once per publish — from BOTH the classic and the agentic
         extraction code paths — so that ``AgentSuccessEvaluationResult`` records
@@ -417,13 +474,20 @@ class GenerationService:
 
         Args:
             new_request (Request): The just-stored request whose session is being
-                published into. ``new_request.session_id`` gates scheduling.
+                published into.
             user_id (str): The user owning the session.
             agent_version (str): Agent version string carried into the evaluator.
             source (str | None): Optional source label.
         """
         session_id = new_request.session_id
-        if not session_id:
+        if not self._should_sample_group_evaluation(
+            user_id=user_id, session_id=session_id
+        ):
+            logger.info(
+                "Skipping group evaluation scheduling for unsampled session=%s user=%s",
+                session_id,
+                user_id,
+            )
             return
 
         scheduler = GroupEvaluationScheduler.get_instance()
@@ -464,8 +528,25 @@ class GenerationService:
             ),
         )
 
+    def _should_sample_group_evaluation(self, *, user_id: str, session_id: str) -> bool:
+        config = self.configurator.get_config()
+        agent_success_config = getattr(config, "agent_success_config", None)
+        if agent_success_config is None:
+            return False
+
+        sampling_rate = agent_success_config.sampling_rate
+        if sampling_rate >= 1.0:
+            return True
+        if sampling_rate <= 0.0:
+            return False
+
+        return (
+            _stable_group_sampling_fraction(self.org_id, user_id, session_id)
+            < sampling_rate
+        )
+
     def _maybe_run_reflection(
-        self, *, user_id: str, agent_version: str, source: str | None
+        self, *, user_id: str, request_id: str, agent_version: str, source: str | None
     ) -> None:
         """Best-effort reflection pass before extraction.
 
@@ -480,6 +561,7 @@ class GenerationService:
             service.run(
                 ReflectionServiceRequest(
                     user_id=user_id,
+                    request_id=request_id,
                     agent_version=agent_version,
                     source=source,
                 )