@openhands/extensions 0.10.0 → 0.12.0

package/tests/test_deprecations.py

@@ -33,6 +33,28 @@ def _load_checker():
     return module
 
 
+def _sample_package(checker, tmp_path: Path, *, version: str):
+    """A throwaway package holding one deprecation scheduled for removal in 0.12.0.
+
+    The deadline tests use this instead of the shipped source so they assert the
+    checker's behaviour rather than whichever deprecations happen to be pending.
+    """
+    pyproject = tmp_path / "pyproject.toml"
+    pyproject.write_text(f'[project]\nname = "sample"\nversion = "{version}"\n')
+
+    source_root = tmp_path / "sample"
+    source_root.mkdir()
+    (source_root / "legacy.py").write_text(
+        '@deprecated(deprecated_in="0.10.0", removed_in="0.12.0")\n'
+        "def legacy_helper() -> None:\n"
+        "    pass\n"
+    )
+
+    return checker.PackageConfig(
+        name="sample", pyproject=pyproject, source_roots=(source_root,)
+    )
+
+
 def test_deprecated_decorator_warns_and_preserves_calls() -> None:
     @deprecated(
         deprecated_in="0.10.0",
@@ -107,9 +129,13 @@ warn_cleanup(\"temporary workaround\", cleanup_by=date(2030, 1, 1))
 
 def test_release_please_deadline_passes_before_removal_version(
     monkeypatch: pytest.MonkeyPatch,
+    tmp_path: Path,
 ) -> None:
     checker = _load_checker()
-    monkeypatch.setattr(checker, "_load_current_version", lambda _: "0.10.0")
+    monkeypatch.setattr(checker, "REPO_ROOT", tmp_path)
+    monkeypatch.setattr(
+        checker, "PACKAGES", (_sample_package(checker, tmp_path, version="0.11.0"),)
+    )
 
     assert checker.main() == 0
 
@@ -117,14 +143,23 @@ def test_release_please_deadline_passes_before_removal_version(
 def test_release_please_deadline_fails_at_removal_version(
     monkeypatch: pytest.MonkeyPatch,
     capsys: pytest.CaptureFixture[str],
+    tmp_path: Path,
 ) -> None:
     checker = _load_checker()
-    monkeypatch.setattr(checker, "_load_current_version", lambda _: "0.12.0")
+    monkeypatch.setattr(checker, "REPO_ROOT", tmp_path)
+    monkeypatch.setattr(
+        checker, "PACKAGES", (_sample_package(checker, tmp_path, version="0.12.0"),)
+    )
 
     assert checker.main() == 1
     output = capsys.readouterr().out
-    assert "list_integration_catalog" in output
-    assert "get_integration_catalog_entry" in output
+    assert "legacy_helper" in output
+    assert "removed in:    0.12.0" in output
+
+
+def test_shipped_source_has_no_overdue_deprecations() -> None:
+    """The gate the release PR runs: nothing shipped is past its deadline."""
+    assert _load_checker().main() == 0
 
 
 def test_date_based_deadlines_use_the_sdk_semantics() -> None:

package/tests/test_integration_catalog_in_sync.py

@@ -39,6 +39,19 @@ def _catalog_entries() -> list[dict]:
     )
 
 
+def _python_entries(
+    mcp: bool | None = None,
+    oauth: bool | None = None,
+) -> list[dict]:
+    """Catalog entries as seen through the supported Python model API."""
+    return [
+        model.model_dump(exclude_none=True)
+        for model in openhands_extensions.list_integration_catalog_models(
+            mcp=mcp, oauth=oauth
+        )
+    ]
+
+
 def _supports_mcp(entry: dict) -> bool:
     return any(option.get("provider") == "mcp" for option in entry["connectionOptions"])
 
@@ -84,8 +97,8 @@ def test_python_snapshot_is_built_from_catalog_directory() -> None:
     }
 
 
-def test_python_list_integration_catalog_returns_raw_entries() -> None:
-    entries = openhands_extensions.list_integration_catalog()
+def test_python_catalog_models_expose_no_derived_fields() -> None:
+    entries = _python_entries()
     assert entries == _catalog_entries()
     for entry in entries:
         assert "supportsMcp" not in entry
@@ -99,8 +112,14 @@ def test_python_list_integration_catalog_returns_raw_entries() -> None:
         assert "runtimeAvailability" not in entry
 
 
+def test_raw_dict_accessors_were_removed_in_0_12_0() -> None:
+    # Deprecated in 0.10.0, removed in 0.12.0. The typed model API replaces them.
+    assert not hasattr(openhands_extensions, "list_integration_catalog")
+    assert not hasattr(openhands_extensions, "get_integration_catalog_entry")
+
+
 def test_python_list_integration_catalog_models_returns_typed_entries() -> None:
-    entries = openhands_extensions.list_integration_catalog()
+    entries = _catalog_entries()
     models = openhands_extensions.list_integration_catalog_models()
 
     assert [model.model_dump(exclude_none=True) for model in models] == entries
@@ -189,7 +208,7 @@ def test_catalog_models_require_connectability_and_install_metadata() -> None:
 
 
 def test_logo_metadata_is_serializable_and_language_agnostic() -> None:
-    entries = openhands_extensions.list_integration_catalog()
+    entries = _python_entries()
     with_logo = [entry for entry in entries if entry.get("logoUrl")]
     assert with_logo
     assert any(entry["id"].startswith("cloudflare-") for entry in with_logo)
@@ -199,16 +218,14 @@ def test_logo_metadata_is_serializable_and_language_agnostic() -> None:
         assert "react" not in entry["logoUrl"].lower()
 
 
-def test_get_integration_catalog_entry_round_trip() -> None:
-    sample_id = openhands_extensions.list_integration_catalog()[0]["id"]
-    sample = openhands_extensions.get_integration_catalog_entry(sample_id)
+def test_get_integration_catalog_entry_model_round_trip() -> None:
+    sample_id = _python_entries()[0]["id"]
+    sample = openhands_extensions.get_integration_catalog_entry_model(sample_id)
     assert sample is not None
-    assert sample == next(
-        entry
-        for entry in openhands_extensions.list_integration_catalog()
-        if entry["id"] == sample_id
+    assert sample.model_dump(exclude_none=True) == next(
+        entry for entry in _python_entries() if entry["id"] == sample_id
     )
-    assert openhands_extensions.get_integration_catalog_entry("nope") is None
+    assert openhands_extensions.get_integration_catalog_entry_model("nope") is None
 
 
 def _js_call(expr: str) -> str:
@@ -227,16 +244,16 @@ def test_js_reads_the_same_catalog_as_python() -> None:
         "import { listIntegrationCatalog } from './integrations/index.js';\n"
         "process.stdout.write(JSON.stringify(listIntegrationCatalog()));"
     )
-    assert json.loads(integrations) == openhands_extensions.list_integration_catalog()
+    assert json.loads(integrations) == _python_entries()
 
-    sample_id = openhands_extensions.list_integration_catalog()[0]["id"]
+    sample_id = _python_entries()[0]["id"]
     sample = _js_call(
         "import { getIntegrationCatalogEntry } from './integrations/index.js';\n"
         f"process.stdout.write(JSON.stringify(getIntegrationCatalogEntry({json.dumps(sample_id)})));"
     )
-    assert json.loads(sample) == openhands_extensions.get_integration_catalog_entry(
-        sample_id
-    )
+    entry_model = openhands_extensions.get_integration_catalog_entry_model(sample_id)
+    assert entry_model is not None
+    assert json.loads(sample) == entry_model.model_dump(exclude_none=True)
 
 
 def _js_filter(mcp, oauth) -> list[str]:
@@ -249,74 +266,51 @@ def _js_filter(mcp, oauth) -> list[str]:
 
 
 def test_filter_mcp_only() -> None:
-    py_ids = {e["id"] for e in openhands_extensions.list_integration_catalog(mcp=True)}
+    py_ids = {e["id"] for e in _python_entries(mcp=True)}
     js_ids = set(_js_filter("true", "undefined"))
-    all_mcp = {
-        e["id"]
-        for e in openhands_extensions.list_integration_catalog()
-        if _supports_mcp(e)
-    }
+    all_mcp = {e["id"] for e in _python_entries() if _supports_mcp(e)}
     assert py_ids == js_ids == all_mcp
     assert "filesystem" in py_ids
 
 
 def test_filter_oauth_only() -> None:
-    py_ids = {
-        e["id"] for e in openhands_extensions.list_integration_catalog(oauth=True)
-    }
+    py_ids = {e["id"] for e in _python_entries(oauth=True)}
     js_ids = set(_js_filter("undefined", "true"))
-    all_oauth = {
-        e["id"]
-        for e in openhands_extensions.list_integration_catalog()
-        if _supports_oauth(e)
-    }
+    all_oauth = {e["id"] for e in _python_entries() if _supports_oauth(e)}
     assert py_ids == js_ids == all_oauth
     assert py_ids
 
 
 def test_filter_oauth_not_mcp() -> None:
-    py_ids = {
-        e["id"]
-        for e in openhands_extensions.list_integration_catalog(oauth=True, mcp=False)
-    }
+    py_ids = {e["id"] for e in _python_entries(oauth=True, mcp=False)}
     js_ids = set(_js_filter("false", "true"))
     expected = {
         e["id"]
-        for e in openhands_extensions.list_integration_catalog()
+        for e in _python_entries()
         if _supports_oauth(e) and not _supports_mcp(e)
     }
     assert py_ids == js_ids == expected
 
 
 def test_filter_none_returns_all() -> None:
-    assert len(openhands_extensions.list_integration_catalog()) == len(
-        openhands_extensions.list_integration_catalog(mcp=None, oauth=None)
-    )
+    assert len(_python_entries()) == len(_python_entries(mcp=None, oauth=None))
 
 
 def test_accessors_return_independent_copies() -> None:
-    catalog_a = openhands_extensions.list_integration_catalog()
-    catalog_b = openhands_extensions.list_integration_catalog()
-    assert catalog_a == catalog_b
-    assert catalog_a is not catalog_b
-    catalog_a[0]["__mutated"] = True
-    assert "__mutated" not in openhands_extensions.list_integration_catalog()[0]
-
-    sample_id = openhands_extensions.list_integration_catalog()[0]["id"]
-    sample = openhands_extensions.get_integration_catalog_entry(sample_id)
-    assert sample is not None
-    sample["__mutated"] = True
-    assert "__mutated" not in openhands_extensions.get_integration_catalog_entry(
-        sample_id
-    )
+    models_a = openhands_extensions.list_integration_catalog_models()
+    models_b = openhands_extensions.list_integration_catalog_models()
+    assert models_a == models_b
+    assert models_a[0] is not models_b[0]
 
+    # Mutating the exported snapshot must not leak into the cached catalog.
+    # The models forbid extra fields, so a leak surfaces as a validation error.
     snapshot = openhands_extensions.INTEGRATION_CATALOG_SNAPSHOT
     snapshot["integrations"][0]["__mutated"] = True
-    assert "__mutated" not in openhands_extensions.list_integration_catalog()[0]
+    assert "__mutated" not in _python_entries()[0]
 
 
 def test_js_accessors_return_independent_copies() -> None:
-    sample_id = openhands_extensions.list_integration_catalog()[0]["id"]
+    sample_id = _python_entries()[0]["id"]
     expr = f"""
         import {{
           INTEGRATION_CATALOG,
@@ -342,11 +336,7 @@ def test_js_accessors_return_independent_copies() -> None:
 
 
 def test_oauth_config_lives_on_connection_options() -> None:
-    oauth_entries = [
-        entry
-        for entry in openhands_extensions.list_integration_catalog()
-        if _supports_oauth(entry)
-    ]
+    oauth_entries = [entry for entry in _python_entries() if _supports_oauth(entry)]
     assert oauth_entries
     for entry in oauth_entries:
         assert "registrationDefaults" not in entry

package/tests/test_live_integration_smoke.py

@@ -0,0 +1,366 @@
+"""Live smoke tests for the daily-workflow integrations (OSS-5193).
+
+These tests exercise the published catalog entries for GitHub, Linear, and
+Slack against the real services, through the same client stack
+(``openhands.sdk.mcp.create_mcp_tools``) that backs the agent-server's
+``POST /api/mcp/test`` probe behind Agent Canvas's Test connection.
+
+Every test is skipped unless real credentials are present in the environment,
+so credential-less CI runs them as skips with zero network access. To run
+live:
+
+    GITHUB_PERSONAL_ACCESS_TOKEN=... \
+    LINEAR_API_KEY=... \
+    SLACK_BOT_TOKEN=... SLACK_TEAM_ID=... \
+    uv run --group test pytest tests/test_live_integration_smoke.py -rA -s
+
+Preconditions:
+- The GitHub token needs the scopes stated in the catalog entry (classic:
+  ``repo`` and ``read:user``).
+- The Linear API key needs at least read access.
+- The Slack bot must be a member of at least one channel with message history
+  (``/invite @your-bot``), and its token needs the scopes stated in the
+  catalog entry.
+
+Evidence lines are printed as ``[OSS-5193] ...`` and contain only tool names,
+counts, and provider error codes — never credentials or message content.
+"""
+
+import json
+import os
+from contextlib import contextmanager
+
+import mcp.types
+import pytest
+from openhands.sdk.mcp import MCPError, MCPServer, MCPTimeoutError, create_mcp_tools
+
+from openhands_extensions import get_integration_catalog_entry_model
+
+
+GITHUB_TOKEN_ENV = "GITHUB_PERSONAL_ACCESS_TOKEN"
+LINEAR_TOKEN_ENV = "LINEAR_API_KEY"
+SLACK_ENV_KEYS = ("SLACK_TEAM_ID", "SLACK_BOT_TOKEN")
+
+HTTP_TIMEOUT = 30.0
+STDIO_TIMEOUT = 90.0  # a cold `npx -y` run downloads the Slack server first
+
+# Auth failure codes the Agent Canvas interpreter reclassifies as credential
+# failures (agent-canvas src/utils/mcp-credential-validation.ts).
+SLACK_AUTH_FAILURES = {
+    "invalid_auth",
+    "not_authed",
+    "account_inactive",
+    "token_revoked",
+    "token_expired",
+}
+
+requires_github = pytest.mark.skipif(
+    not os.environ.get(GITHUB_TOKEN_ENV),
+    reason=f"set {GITHUB_TOKEN_ENV} to run the live GitHub MCP smoke test",
+)
+requires_linear = pytest.mark.skipif(
+    not os.environ.get(LINEAR_TOKEN_ENV),
+    reason=f"set {LINEAR_TOKEN_ENV} to run the live Linear MCP smoke test",
+)
+requires_slack = pytest.mark.skipif(
+    not all(os.environ.get(key) for key in SLACK_ENV_KEYS),
+    reason=f"set {' and '.join(SLACK_ENV_KEYS)} to run the live Slack MCP smoke test",
+)
+
+
+def _evidence(line):
+    print(f"[OSS-5193] {line}")
+
+
+def _installable_option(entry_id):
+    """The option a local Agent Canvas install uses — the GUI filters out
+    provider-OAuth options (isLocallyInstallableMcpOption in agent-canvas)."""
+    entry = get_integration_catalog_entry_model(entry_id)
+    assert entry is not None, f"catalog entry {entry_id!r} missing"
+    for option in entry.connectionOptions:
+        if option.auth.strategy != "oauth2":
+            return option
+    raise AssertionError(f"{entry_id}: no locally installable connection option")
+
+
+def _remote_server(option, token):
+    return MCPServer.model_validate(
+        {
+            "url": option.transport.url,
+            "transport": "http",
+            "auth": {"strategy": option.auth.strategy, "value": token},
+        }
+    )
+
+
+def _stdio_server(option, env):
+    return MCPServer.model_validate(
+        {
+            "transport": "stdio",
+            "command": option.transport.command,
+            "args": list(option.transport.args),
+            "env": env,
+        }
+    )
+
+
+@contextmanager
+def _connect(server_id, server, timeout):
+    """Yield ``(tool_names, call)`` through the same code path as the
+    agent-server probe (mcp_router._probe_mcp_server): connect, list tools,
+    invoke read-only tools, join TextContent blocks."""
+    with create_mcp_tools({server_id: server}, timeout=timeout) as client:
+        tool_names = [tool.name for tool in client.tools]
+
+        def call(name, arguments):
+            result = client.call_async_from_sync(
+                client.call_tool_mcp,
+                name=name,
+                arguments=arguments,
+                timeout=timeout,
+            )
+            text = "\n".join(
+                block.text
+                for block in result.content
+                if isinstance(block, mcp.types.TextContent)
+            )
+            return bool(result.isError), text
+
+        yield tool_names, call
+
+
+def _first_advertised(tool_names, candidates):
+    for name, arguments in candidates:
+        if name in tool_names:
+            return name, arguments
+    pytest.fail(
+        "none of the candidate tools "
+        f"{sorted({name for name, _ in candidates})} are advertised; "
+        f"server offers: {sorted(tool_names)}"
+    )
+
+
+@requires_github
+def test_github_identity_and_pr_read():
+    option = _installable_option("github")
+    # The env var doubles as the catalog's secret name; fail loudly on drift.
+    assert option.auth.credentialSecretName == GITHUB_TOKEN_ENV
+    server = _remote_server(option, os.environ[GITHUB_TOKEN_ENV])
+
+    with _connect("github", server, HTTP_TIMEOUT) as (tool_names, call):
+        assert "get_me" in tool_names, (
+            "Agent Canvas's credential probe calls get_me; "
+            f"server offers: {sorted(tool_names)}"
+        )
+        is_error, text = call("get_me", {})
+        assert not is_error, f"get_me failed: {text}"
+        assert "login" in text
+        _evidence(f"github: {len(tool_names)} tools; get_me ok (login present)")
+
+        # Representative PR lookup against a merged, permanently public PR in
+        # this very repo (OpenHands/extensions#408).
+        name, arguments = _first_advertised(
+            tool_names,
+            [
+                (
+                    "pull_request_read",
+                    {
+                        "method": "get",
+                        "owner": "OpenHands",
+                        "repo": "extensions",
+                        "pullNumber": 408,
+                    },
+                ),
+                (
+                    "get_pull_request",
+                    {"owner": "OpenHands", "repo": "extensions", "pullNumber": 408},
+                ),
+                (
+                    "search_pull_requests",
+                    {"query": "repo:OpenHands/extensions is:pr 408"},
+                ),
+                ("list_pull_requests", {"owner": "OpenHands", "repo": "extensions"}),
+            ],
+        )
+        is_error, text = call(name, arguments)
+        if is_error and "pullNumber" in arguments:
+            # Tolerate snake_case argument naming on the hosted server.
+            arguments = {
+                ("pull_number" if key == "pullNumber" else key): value
+                for key, value in arguments.items()
+            }
+            is_error, text = call(name, arguments)
+        assert not is_error, f"{name} failed: {text}"
+        assert text.strip()
+        if "408" in json.dumps(arguments):
+            assert "408" in text, f"{name} response does not reference PR 408"
+        _evidence(f"github: PR lookup ok via {name}")
+
+
+@requires_github
+def test_github_invalid_token_yields_structured_failure():
+    option = _installable_option("github")
+    bogus = "github_pat_invalid_oss5193"
+    server = _remote_server(option, bogus)
+
+    try:
+        with _connect("github-invalid", server, HTTP_TIMEOUT) as (tool_names, call):
+            # Some deployments accept the connection and fail on invocation.
+            if "get_me" in tool_names:
+                is_error, text = call("get_me", {})
+            else:
+                is_error, text = True, "get_me not advertised"
+            assert is_error, "expected an auth failure with an invalid token"
+            assert bogus not in text, "failure text must not echo the credential"
+            _evidence("github: invalid token -> tool-level error (is_error=true)")
+    except MCPTimeoutError:
+        pytest.fail("invalid GitHub token should fail fast, not time out")
+    except MCPError as exc:
+        detail = str(exc.__cause__ or exc.__context__ or exc)
+        assert bogus not in detail, "failure text must not echo the credential"
+        _evidence(
+            "github: invalid token -> connection-stage failure "
+            "(Canvas error_kind=connection)"
+        )
+
+
+@requires_linear
+def test_linear_teams_and_issue_read():
+    option = _installable_option("linear")
+    server = _remote_server(option, os.environ[LINEAR_TOKEN_ENV])
+
+    with _connect("linear", server, HTTP_TIMEOUT) as (tool_names, call):
+        assert "list_teams" in tool_names, (
+            "Agent Canvas's credential probe calls list_teams; "
+            f"server offers: {sorted(tool_names)}"
+        )
+        is_error, text = call("list_teams", {})
+        assert not is_error, f"list_teams failed: {text}"
+        _evidence(f"linear: {len(tool_names)} tools; list_teams ok")
+
+        name, arguments = _first_advertised(
+            tool_names,
+            [("list_issues", {"limit": 1}), ("list_my_issues", {"limit": 1})],
+        )
+        is_error, text = call(name, arguments)
+        assert not is_error, f"{name} failed: {text}"
+        assert text.strip()
+        _evidence(f"linear: issue read ok via {name}")
+
+
+@requires_linear
+def test_linear_invalid_token_yields_structured_failure():
+    option = _installable_option("linear")
+    bogus = "lin_api_invalid_oss5193"
+    server = _remote_server(option, bogus)
+
+    try:
+        with _connect("linear-invalid", server, HTTP_TIMEOUT) as (tool_names, call):
+            if "list_teams" in tool_names:
+                is_error, text = call("list_teams", {})
+            else:
+                is_error, text = True, "list_teams not advertised"
+            assert is_error, "expected an auth failure with an invalid token"
+            assert bogus not in text, "failure text must not echo the credential"
+            _evidence("linear: invalid token -> tool-level error (is_error=true)")
+    except MCPTimeoutError:
+        pytest.fail("invalid Linear token should fail fast, not time out")
+    except MCPError as exc:
+        detail = str(exc.__cause__ or exc.__context__ or exc)
+        assert bogus not in detail, "failure text must not echo the credential"
+        _evidence(
+            "linear: invalid token -> connection-stage failure "
+            "(Canvas error_kind=connection)"
+        )
+
+
+@requires_slack
+def test_slack_channel_and_thread_read():
+    option = _installable_option("slack")
+    env_keys = [field.key for field in option.transport.envFields or []]
+    assert set(env_keys) == set(SLACK_ENV_KEYS)
+    server = _stdio_server(option, {key: os.environ[key] for key in env_keys})
+
+    with _connect("slack", server, STDIO_TIMEOUT) as (tool_names, call):
+        for required in (
+            "slack_list_channels",
+            "slack_get_channel_history",
+            "slack_get_thread_replies",
+        ):
+            assert required in tool_names, (
+                f"{required} not advertised; server offers: {sorted(tool_names)}"
+            )
+
+        is_error, text = call("slack_list_channels", {"limit": 5})
+        assert not is_error, f"slack_list_channels failed: {text}"
+        payload = json.loads(text)
+        assert payload.get("ok") is True, (
+            f"slack_list_channels not ok: {payload.get('error')}"
+        )
+        channels = payload.get("channels") or []
+        assert channels, "no channels visible to the bot"
+        _evidence(
+            f"slack: {len(tool_names)} tools; list_channels ok "
+            f"({len(channels)} channel(s))"
+        )
+
+        for channel in channels:
+            is_error, text = call(
+                "slack_get_channel_history",
+                {"channel_id": channel["id"], "limit": 10},
+            )
+            if is_error:
+                continue
+            history = json.loads(text)
+            if history.get("ok") is not True:
+                continue  # e.g. not_in_channel — try the next channel
+            for message in history.get("messages") or []:
+                ts = message.get("thread_ts") or message.get("ts")
+                if not ts:
+                    continue
+                is_error, text = call(
+                    "slack_get_thread_replies",
+                    {"channel_id": channel["id"], "thread_ts": ts},
+                )
+                assert not is_error, f"slack_get_thread_replies failed: {text}"
+                replies = json.loads(text)
+                assert replies.get("ok") is True, (
+                    f"slack_get_thread_replies not ok: {replies.get('error')}"
+                )
+                _evidence(
+                    "slack: channel history + thread read ok "
+                    f"({len(replies.get('messages') or [])} message(s) in thread)"
+                )
+                return
+        pytest.fail(
+            "no channel with readable message history: invite the bot to a "
+            "public channel that has messages (/invite @your-bot) and rerun"
+        )
+
+
+@requires_slack
+def test_slack_invalid_token_yields_in_band_error():
+    option = _installable_option("slack")
+    bogus = "xoxb-invalid-oss5193"
+    server = _stdio_server(
+        option,
+        {"SLACK_TEAM_ID": os.environ["SLACK_TEAM_ID"], "SLACK_BOT_TOKEN": bogus},
+    )
+
+    with _connect("slack-invalid", server, STDIO_TIMEOUT) as (tool_names, call):
+        # The stdio server starts and lists tools with any token — the exact
+        # reason Agent Canvas interprets the in-band payload of a read-only
+        # tool call instead of trusting tools/list.
+        assert "slack_list_channels" in tool_names
+        is_error, text = call("slack_list_channels", {"limit": 1})
+        assert bogus not in text, "failure text must not echo the credential"
+        payload = json.loads(text)
+        assert payload.get("ok") is False
+        assert payload.get("error") in SLACK_AUTH_FAILURES, (
+            f"unexpected error code {payload.get('error')!r}; Canvas reclassifies "
+            f"only {sorted(SLACK_AUTH_FAILURES)} as credential failures"
+        )
+        _evidence(
+            f"slack: invalid token -> in-band ok=false error={payload.get('error')} "
+            f"(is_error={is_error})"
+        )

package/automations/catalog/github-pr-reviewer.json

@@ -1,13 +0,0 @@
-{
-  "id": "github-pr-reviewer",
-  "name": "GitHub Code Review Agent",
-  "category": "Code review",
-  "description": "Watch for a configurable label on GitHub pull requests, inspect full PR and repository context, and post an AI review comment once per label event.",
-  "requiredIntegrationIds": [
-    "github"
-  ],
-  "popularityRank": 100,
-  "estimatedSetupMinutes": 4,
-  "prompt": "/pr-reviewer:setup",
-  "exampleImplementation": "Trigger: cron polling for open GitHub PRs with a configured label such as openhands-review\nRequired secret: GITHUB_PERSONAL_ACCESS_TOKEN\n\n1. Read repository, trigger label, review tone, and polling schedule from setup.\n2. List open PRs and find the latest matching GitHub labeled issue event for each labeled PR.\n3. Deduplicate on the label event ID so every label application queues exactly one review.\n4. Start an OpenHands conversation that clones the repo, checks out the exact PR head SHA, and inspects PR discussion, review comments, changed files, diff, and surrounding code.\n5. Post an acknowledgement with the conversation link, then post the final AI review comment only if the PR is still open and the head SHA has not changed."
-}

package/automations/catalog/github-repo-monitor.json

@@ -1,11 +0,0 @@
-{
-  "id": "github-repo-monitor",
-  "name": "GitHub repository monitor",
-  "category": "Developer tools",
-  "description": "Watch a repository for @OpenHands mentions in issues and PR comments, start a conversation, and post the agent's reply back to GitHub.",
-  "requiredIntegrationIds": ["github"],
-  "popularityRank": 98,
-  "estimatedSetupMinutes": 5,
-  "prompt": "/github-monitor:poll",
-  "exampleImplementation": "Trigger: cron, every minute (configurable)\nRequired secret: GITHUB_PERSONAL_ACCESS_TOKEN\n\n1. Poll GitHub for new issue and PR comments since the last run.\n2. Match comments containing the trigger phrase (case-insensitive, default: @OpenHands).\n3. Post an acknowledgment comment with a link to the new OpenHands conversation.\n4. Forward follow-up replies in the same thread to the running conversation.\n5. Post the agent's final response back to GitHub when the conversation completes."
-}

package/automations/catalog/incident-retrospective-drafter.json

@@ -1,15 +0,0 @@
-{
-  "id": "incident-retrospective-drafter",
-  "name": "Incident retrospective drafter",
-  "category": "Reliability",
-  "description": "Collect incident chatter and issue updates, then draft a timeline and follow-up checklist.",
-  "requiredIntegrationIds": [
-    "slack",
-    "linear",
-    "notion"
-  ],
-  "popularityRank": 78,
-  "estimatedSetupMinutes": 8,
-  "prompt": "/incident-retro:setup",
-  "exampleImplementation": "Trigger: manual, cron, or incident label added\nRequired MCPs: Slack, Linear, Notion\n\n1. Gather incident messages, timestamps, and linked issue references.\n2. Pull Linear follow-up tickets, owners, and status.\n3. TODO: define the incident identifier format and approved Notion template.\n4. Build a timeline, impact summary, root-cause hypotheses, and action items.\n5. Publish a draft retrospective and notify the incident owner."
-}