@clawpump/claw-agent 0.1.5 → 0.1.7

package/agent/hermes_cli/nous_subscription.py

@@ -39,6 +39,9 @@ MANAGED_FEATURE_COVERAGE_CATEGORY: Dict[str, str] = {
     "image_gen": "fal",
     "video_gen": "fal-video",
     "tts": "openai-audio",
+    # STT shares the TTS coverage category: both ride the managed
+    # "openai-audio" gateway endpoint (speech + transcriptions).
+    "stt": "openai-audio",
     "browser": "browser-use",
     "modal": "modal",
 }
@@ -85,6 +88,10 @@ class NousSubscriptionFeatures:
     def tts(self) -> NousFeatureState:
         return self.features["tts"]
 
+    @property
+    def stt(self) -> NousFeatureState:
+        return self.features["stt"]
+
     @property
     def browser(self) -> NousFeatureState:
         return self.features["browser"]
@@ -98,7 +105,7 @@ class NousSubscriptionFeatures:
         return self.features["modal"]
 
     def items(self) -> Iterable[NousFeatureState]:
-        ordered = ("web", "image_gen", "video_gen", "tts", "browser", "modal")
+        ordered = ("web", "image_gen", "video_gen", "tts", "stt", "browser", "modal")
         for key in ordered:
             yield self.features[key]
 
@@ -159,6 +166,33 @@ def _has_agent_browser() -> bool:
     return bool(agent_browser_bin or local_bin.exists())
 
 
+def _local_browser_runnable() -> bool:
+    """Return True when the *local* browser backend would actually start.
+
+    The ``agent-browser`` CLI being present is necessary but not sufficient for
+    local mode: agent-browser also needs a Chromium build on disk (without one
+    it hangs on first use until the command timeout fires), unless the
+    Lightpanda engine is selected — text-only navigation needs no Chromium.
+
+    This mirrors the local-mode tail of
+    :func:`tools.browser_tool.check_browser_requirements`, so the setup/status
+    surfaces advertise local browser readiness only when the runtime would
+    actually run it. Cloud providers (Browserbase, Browser Use, Firecrawl) host
+    their own Chromium and therefore gate on :func:`_has_agent_browser` alone.
+    """
+    if not _has_agent_browser():
+        return False
+    try:
+        from tools.browser_tool import _chromium_installed, _using_lightpanda_engine
+    except Exception:
+        # If the runtime probe can't be imported, fall back to binary presence
+        # (prior behaviour) rather than crashing the setup/status surface.
+        return True
+    if _using_lightpanda_engine():
+        return True
+    return _chromium_installed()
+
+
 def _browser_label(current_provider: str) -> str:
     mapping = {
         "browserbase": "Browserbase",
@@ -182,19 +216,57 @@ def _tts_label(current_provider: str) -> str:
     return mapping.get(current_provider or "edge", current_provider or "Edge TTS")
 
 
+def _stt_label(current_provider: str) -> str:
+    mapping = {
+        "openai": "OpenAI Whisper",
+        "groq": "Groq Whisper",
+        "mistral": "Mistral Voxtral Transcribe",
+        "local": "Local faster-whisper",
+    }
+    return mapping.get(current_provider or "local", current_provider or "Local faster-whisper")
+
+
+def _local_stt_backend_available() -> bool:
+    """Whether a local STT backend could serve transcription right now.
+
+    True when faster-whisper is importable or a custom local STT command
+    is configured. Used both for feature detection and to stop
+    ``apply_nous_managed_defaults`` from flipping a working local setup
+    to the managed gateway.
+    """
+    if get_env_value("HERMES_LOCAL_STT_COMMAND"):
+        return True
+    try:
+        from tools.transcription_tools import _HAS_FASTER_WHISPER
+
+        return bool(_HAS_FASTER_WHISPER)
+    except Exception:
+        return False
+
+
 def _resolve_browser_feature_state(
     *,
     browser_tool_enabled: bool,
     browser_provider: str,
     browser_provider_explicit: bool,
     browser_local_available: bool,
+    browser_local_runnable: bool,
     direct_camofox: bool,
     direct_browserbase: bool,
     direct_browser_use: bool,
     direct_firecrawl: bool,
     managed_browser_available: bool,
 ) -> tuple[str, bool, bool, bool]:
-    """Resolve browser availability using the same precedence as runtime."""
+    """Resolve browser availability using the same precedence as runtime.
+
+    ``browser_local_available`` means "the agent-browser CLI is present" — the
+    only local requirement for cloud providers, which host their own Chromium.
+    ``browser_local_runnable`` additionally requires a usable local Chromium
+    build (or the Lightpanda engine), mirroring the local-mode tail of
+    :func:`tools.browser_tool.check_browser_requirements`. Local mode must gate
+    on the latter, or setup/status advertise a browser that fails on first use
+    when Chromium is missing.
+    """
     if direct_camofox:
         return "camofox", True, bool(browser_tool_enabled), False
 
@@ -223,7 +295,7 @@ def _resolve_browser_feature_state(
             return current_provider, False, False, False
 
         current_provider = "local"
-        available = bool(browser_local_available)
+        available = bool(browser_local_runnable)
         active = bool(browser_tool_enabled and available)
         return current_provider, available, active, False
 
@@ -243,7 +315,7 @@ def _resolve_browser_feature_state(
         active = bool(browser_tool_enabled and available)
         return "browserbase", available, active, False
 
-    available = bool(browser_local_available)
+    available = bool(browser_local_runnable)
     active = bool(browser_tool_enabled and available)
     return "local", available, active, False
 
@@ -290,6 +362,7 @@ def get_nous_subscription_features(
 
     web_cfg = config.get("web") if isinstance(config.get("web"), dict) else {}
     tts_cfg = config.get("tts") if isinstance(config.get("tts"), dict) else {}
+    stt_cfg = config.get("stt") if isinstance(config.get("stt"), dict) else {}
     browser_cfg = config.get("browser") if isinstance(config.get("browser"), dict) else {}
     terminal_cfg = config.get("terminal") if isinstance(config.get("terminal"), dict) else {}
 
@@ -299,6 +372,11 @@ def get_nous_subscription_features(
     web_search_backend = str(web_cfg.get("search_backend") or "").strip().lower()
     web_extract_backend = str(web_cfg.get("extract_backend") or "").strip().lower()
     tts_provider = str(tts_cfg.get("provider") or "edge").strip().lower()
+    # STT default is "local" (faster-whisper) per DEFAULT_CONFIG, which
+    # requires `pip install faster-whisper`. For Nous subscribers we'd
+    # rather route through the managed OpenAI audio gateway — see
+    # apply_nous_managed_defaults below.
+    stt_provider = str(stt_cfg.get("provider") or "local").strip().lower()
     browser_provider_explicit = "cloud_provider" in browser_cfg
     browser_provider = normalize_browser_cloud_provider(
         browser_cfg.get("cloud_provider") if browser_provider_explicit else None
@@ -315,6 +393,7 @@ def get_nous_subscription_features(
     # prevent gateway routing.
     web_use_gateway = _uses_gateway(web_cfg)
     tts_use_gateway = _uses_gateway(tts_cfg)
+    stt_use_gateway = _uses_gateway(stt_cfg)
     browser_use_gateway = _uses_gateway(browser_cfg)
     image_gen_cfg = config.get("image_gen") if isinstance(config.get("image_gen"), dict) else {}
     image_use_gateway = _uses_gateway(image_gen_cfg)
@@ -335,6 +414,22 @@ def get_nous_subscription_features(
     direct_browser_use = bool(get_env_value("BROWSER_USE_API_KEY"))
     direct_modal = has_direct_modal_credentials()
 
+    # STT direct providers. OpenAI Whisper reuses the same audio key as
+    # OpenAI TTS — resolve_openai_audio_api_key() reads VOICE_TOOLS_OPENAI_KEY
+    # and falls back to OPENAI_API_KEY. The local provider's "direct"
+    # signal is whether faster-whisper is importable; we lazy-import so
+    # this module stays cheap on the happy path.
+    direct_openai_stt = bool(resolve_openai_audio_api_key())
+    direct_groq_stt = bool(get_env_value("GROQ_API_KEY"))
+    direct_mistral_stt = bool(get_env_value("MISTRAL_API_KEY"))
+    try:
+        from tools.transcription_tools import _HAS_FASTER_WHISPER
+        local_stt_available = bool(_HAS_FASTER_WHISPER) or bool(
+            get_env_value("HERMES_LOCAL_STT_COMMAND")
+        )
+    except Exception:
+        local_stt_available = bool(get_env_value("HERMES_LOCAL_STT_COMMAND"))
+
     # When use_gateway is set, suppress direct credentials for managed detection
     if web_use_gateway:
         direct_firecrawl = False
@@ -348,6 +443,11 @@ def get_nous_subscription_features(
     if tts_use_gateway:
         direct_openai_tts = False
         direct_elevenlabs = False
+    if stt_use_gateway:
+        direct_openai_stt = False
+        direct_groq_stt = False
+        direct_mistral_stt = False
+        local_stt_available = False
     if browser_use_gateway:
         direct_browser_use = False
         direct_browserbase = False
@@ -379,6 +479,10 @@ def get_nous_subscription_features(
         and is_managed_tool_gateway_ready("openai-audio")
         and _entitled_for("openai-audio")
     )
+    # STT and TTS share the same managed gateway endpoint ("openai-audio")
+    # because the OpenAI audio API covers both /audio/speech (TTS) and
+    # /audio/transcriptions (STT). One probe (and one entitlement), used by both.
+    managed_stt_available = managed_tts_available
     managed_browser_available = (
         managed_tools_flag
         and nous_auth_present
@@ -444,7 +548,26 @@ def get_nous_subscription_features(
     )
     tts_active = bool(tts_tool_enabled and tts_available)
 
+    # STT availability per provider. Unlike TTS, STT isn't a model-callable
+    # tool — the gateway voice middleware calls it on every inbound voice
+    # message — so toolset_enabled is N/A and we treat stt as always
+    # "enabled" if a usable provider is configured.
+    stt_current_provider = stt_provider or "local"
+    stt_managed = (
+        stt_current_provider == "openai"
+        and managed_stt_available
+        and not direct_openai_stt
+    )
+    stt_available = bool(
+        (stt_current_provider == "local" and local_stt_available)
+        or (stt_current_provider == "openai" and (managed_stt_available or direct_openai_stt))
+        or (stt_current_provider == "groq" and direct_groq_stt)
+        or (stt_current_provider == "mistral" and direct_mistral_stt)
+    )
+    stt_active = stt_available
+
     browser_local_available = _has_agent_browser()
+    browser_local_runnable = _local_browser_runnable()
     (
         browser_current_provider,
         browser_available,
@@ -455,6 +578,7 @@ def get_nous_subscription_features(
         browser_provider=browser_provider,
         browser_provider_explicit=browser_provider_explicit,
         browser_local_available=browser_local_available,
+        browser_local_runnable=browser_local_runnable,
         direct_camofox=direct_camofox,
         direct_browserbase=direct_browserbase,
         direct_browser_use=direct_browser_use,
@@ -498,6 +622,13 @@ def get_nous_subscription_features(
     if isinstance(raw_tts_cfg, dict) and "provider" in raw_tts_cfg:
         tts_explicit_configured = tts_provider not in {"", "edge"}
 
+    # STT considers any non-default provider explicit. "local" is the
+    # DEFAULT_CONFIG seed, so seeing it doesn't mean the user picked it.
+    stt_explicit_configured = False
+    raw_stt_cfg = config.get("stt")
+    if isinstance(raw_stt_cfg, dict) and "provider" in raw_stt_cfg:
+        stt_explicit_configured = stt_provider not in {"", "local"}
+
     features = {
         "web": NousFeatureState(
             key="web",
@@ -547,6 +678,21 @@ def get_nous_subscription_features(
             current_provider=_tts_label(tts_current_provider),
             explicit_configured=tts_explicit_configured,
         ),
+        "stt": NousFeatureState(
+            key="stt",
+            label="Speech-to-text",
+            included_by_default=True,
+            available=stt_available,
+            active=stt_active,
+            managed_by_nous=stt_managed,
+            direct_override=stt_active and not stt_managed,
+            # STT isn't toolset-gated (gateway middleware calls it
+            # unconditionally on inbound voice), so report True so the
+            # status display doesn't flag it as "tool disabled".
+            toolset_enabled=True,
+            current_provider=_stt_label(stt_current_provider),
+            explicit_configured=stt_explicit_configured,
+        ),
         "browser": NousFeatureState(
             key="browser",
             label="Browser automation",
@@ -614,6 +760,11 @@ def apply_nous_managed_defaults(
         tts_cfg = {}
         config["tts"] = tts_cfg
 
+    stt_cfg = config.get("stt")
+    if not isinstance(stt_cfg, dict):
+        stt_cfg = {}
+        config["stt"] = stt_cfg
+
     browser_cfg = config.get("browser")
     if not isinstance(browser_cfg, dict):
         browser_cfg = {}
@@ -635,6 +786,30 @@ def apply_nous_managed_defaults(
         tts_cfg["provider"] = "openai"
         changed.add("tts")
 
+    # STT: same pattern as TTS. The DEFAULT_CONFIG seed is "local"
+    # (requires `pip install faster-whisper`); for Nous subscribers we
+    # flip it to "openai" so the managed audio gateway handles transcription
+    # via the same auth as TTS. Skipped when the user has explicitly
+    # configured STT, has direct credentials for a non-managed provider,
+    # has a working local backend (faster-whisper installed or a custom
+    # local command — strong intent signal that "local" was a choice, not
+    # just the DEFAULT_CONFIG seed), or isn't entitled to the managed
+    # "openai-audio" category (flipping would point at a gateway that
+    # refuses them, silently breaking voice transcription).
+    if (
+        not features.stt.explicit_configured
+        and not _local_stt_backend_available()
+        and not (
+            resolve_openai_audio_api_key()
+            or get_env_value("GROQ_API_KEY")
+            or get_env_value("MISTRAL_API_KEY")
+        )
+        and features.account_info is not None
+        and features.account_info.tool_gateway_entitled_for("openai-audio")
+    ):
+        stt_cfg["provider"] = "openai"
+        changed.add("stt")
+
     if "browser" in selected_toolsets and not features.browser.explicit_configured and not (
         get_env_value("BROWSER_USE_API_KEY")
         or get_env_value("BROWSERBASE_API_KEY")
@@ -677,6 +852,7 @@ _GATEWAY_TOOL_LABELS = {
     "image_gen": "Image generation (FAL)",
     "video_gen": "Video generation (FAL)",
     "tts": "Text-to-speech (OpenAI TTS)",
+    "stt": "Speech-to-text (OpenAI Whisper)",
     "browser": "Browser automation (Browser Use)",
 }
 
@@ -698,6 +874,15 @@ def _get_gateway_direct_credentials() -> Dict[str, bool]:
             resolve_openai_audio_api_key()
             or get_env_value("ELEVENLABS_API_KEY")
         ),
+        # STT direct credentials. OpenAI Whisper shares the audio key
+        # with TTS via resolve_openai_audio_api_key() — counting it here
+        # too is intentional: if the user has an OpenAI audio key they
+        # don't need the gateway for either.
+        "stt": bool(
+            resolve_openai_audio_api_key()
+            or get_env_value("GROQ_API_KEY")
+            or get_env_value("MISTRAL_API_KEY")
+        ),
         "browser": bool(
             get_env_value("BROWSER_USE_API_KEY")
             or (get_env_value("BROWSERBASE_API_KEY") and get_env_value("BROWSERBASE_PROJECT_ID"))
@@ -710,10 +895,11 @@ _GATEWAY_DIRECT_LABELS = {
     "image_gen": "FAL key",
     "video_gen": "FAL key",
     "tts": "OpenAI/ElevenLabs key",
+    "stt": "OpenAI/Groq/Mistral key",
     "browser": "Browser Use/Browserbase key",
 }
 
-_ALL_GATEWAY_KEYS = ("web", "image_gen", "video_gen", "tts", "browser")
+_ALL_GATEWAY_KEYS = ("web", "image_gen", "video_gen", "tts", "stt", "browser")
 
 
 def get_gateway_eligible_tools(
@@ -759,6 +945,7 @@ def get_gateway_eligible_tools(
         "image_gen": _uses_gateway(config.get("image_gen")),
         "video_gen": _uses_gateway(config.get("video_gen")),
         "tts": _uses_gateway(config.get("tts")),
+        "stt": _uses_gateway(config.get("stt")),
         "browser": _uses_gateway(config.get("browser")),
     }
 
@@ -805,6 +992,11 @@ def apply_gateway_defaults(
         tts_cfg = {}
         config["tts"] = tts_cfg
 
+    stt_cfg = config.get("stt")
+    if not isinstance(stt_cfg, dict):
+        stt_cfg = {}
+        config["stt"] = stt_cfg
+
     browser_cfg = config.get("browser")
     if not isinstance(browser_cfg, dict):
         browser_cfg = {}
@@ -820,6 +1012,11 @@ def apply_gateway_defaults(
         tts_cfg["use_gateway"] = True
         changed.add("tts")
 
+    if "stt" in tool_keys:
+        stt_cfg["provider"] = "openai"
+        stt_cfg["use_gateway"] = True
+        changed.add("stt")
+
     if "browser" in tool_keys:
         browser_cfg["cloud_provider"] = "browser-use"
         browser_cfg["use_gateway"] = True

package/agent/hermes_cli/platforms.py

@@ -24,6 +24,7 @@ PLATFORMS: OrderedDict[str, PlatformInfo] = OrderedDict([
     ("discord",        PlatformInfo(label="💬 Discord",         default_toolset="hermes-discord")),
     ("slack",          PlatformInfo(label="💼 Slack",           default_toolset="hermes-slack")),
     ("whatsapp",       PlatformInfo(label="📱 WhatsApp",        default_toolset="hermes-whatsapp")),
+    ("whatsapp_cloud", PlatformInfo(label="📱 WhatsApp Business (Cloud)", default_toolset="hermes-whatsapp")),
     ("signal",         PlatformInfo(label="📡 Signal",          default_toolset="hermes-signal")),
     ("bluebubbles",    PlatformInfo(label="💙 BlueBubbles",     default_toolset="hermes-bluebubbles")),
     ("email",          PlatformInfo(label="📧 Email",           default_toolset="hermes-email")),