@salesforce/afv-skills 1.13.0 → 1.15.0

package/skills/investigating-agentforce-architecture/scripts/tests/test_probe_channels.py

@@ -0,0 +1,359 @@
+"""Tests for channel probe TTL (7d), mandatory-field gate,
+runtime invalidate + re-probe.
+
+Every test mocks `sf_cli.run_sf` — no real org calls. Cache dir is
+redirected into a tempdir via monkey-patching `config.PROBE_CACHE_ROOT`
+to keep tests hermetic.
+"""
+from __future__ import annotations
+
+import json
+import os
+import stat
+import sys
+import tempfile
+import time
+import unittest
+import unittest.mock as mock
+from pathlib import Path
+
+from . import _bootstrap  # noqa: F401
+
+import config  # type: ignore
+import probe_channels  # type: ignore
+
+
+def _fake_describe_ok(fields: list[str]) -> dict:
+    """Shape the sf_cli.run_sf output: success envelope wrapping a fields list."""
+    return {
+        "status": 0,
+        "result": {
+            "fields": [{"name": f, "queryable": True} for f in fields],
+        },
+    }
+
+
+# A "universal" success payload: every mandatory field for every sObject
+# present. Individual tests override for specific sObjects.
+def _all_fields_ok_run_sf(recipe: str, **params: str) -> dict:
+    sobject = params["SOBJECT"]
+    mandatory = probe_channels.MANDATORY_FIELDS.get(sobject, set())
+    # Add a couple of bonus fields so queryable_fields isn't identical to
+    # mandatory_missing-inverse by construction.
+    extra = {"CreatedDate", "LastModifiedDate"}
+    return _fake_describe_ok(sorted(mandatory | extra))
+
+
+class ProbeChannelsCacheTests(unittest.TestCase):
+    """Cache hit / miss / TTL semantics."""
+
+    def setUp(self) -> None:
+        self._tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self._tmp.cleanup)
+        self._orig_root = config.PROBE_CACHE_ROOT
+        config.PROBE_CACHE_ROOT = Path(self._tmp.name) / "_channel_probe"
+
+    def tearDown(self) -> None:
+        config.PROBE_CACHE_ROOT = self._orig_root
+
+    def test_cache_hit_within_ttl_no_network(self):
+        """A fresh cache file + force_refresh=False → no run_sf calls."""
+        cache_dir = config.build_probe_cache_dir("00D000000000ABC", "v60.0")
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        cache_file = cache_dir / "channels.json"
+        payload = {
+            "_schema": "channels/1",
+            "_built_at_utc": time.time(),  # right now → fresh
+            "status": "OK",
+            "channels": {"BotDefinition": {"queryable_fields": ["Id"],
+                                           "mandatory_missing": [],
+                                           "describe_error": None}},
+        }
+        cache_file.write_text(json.dumps(payload))
+
+        with mock.patch.object(probe_channels, "run_sf") as m:
+            out = probe_channels.probe_channels(
+                "myorg", "00D000000000ABC", "v60.0", force_refresh=False,
+            )
+        self.assertEqual(out["status"], "OK")
+        m.assert_not_called()
+
+    def test_cache_stale_triggers_reprobe(self):
+        """Cached file older than TTL → re-run describes."""
+        cache_dir = config.build_probe_cache_dir("00D000000000ABC", "v60.0")
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        cache_file = cache_dir / "channels.json"
+        stale = {
+            "_schema": "channels/1",
+            "_built_at_utc": time.time() - (8 * 86400),  # 8 days old
+            "status": "OK",
+            "channels": {},
+        }
+        cache_file.write_text(json.dumps(stale))
+
+        with mock.patch.object(probe_channels, "run_sf",
+                               side_effect=_all_fields_ok_run_sf) as m:
+            out = probe_channels.probe_channels(
+                "myorg", "00D000000000ABC", "v60.0", force_refresh=False,
+            )
+        self.assertEqual(out["status"], "OK")
+        # One call per sObject.
+        self.assertEqual(m.call_count, len(probe_channels.ALL_SOBJECTS))
+
+    def test_force_refresh_always_reprobes(self):
+        """force_refresh=True bypasses a fresh cache."""
+        cache_dir = config.build_probe_cache_dir("00D000000000ABC", "v60.0")
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        cache_file = cache_dir / "channels.json"
+        fresh = {
+            "_schema": "channels/1",
+            "_built_at_utc": time.time(),
+            "status": "OK",
+            "channels": {},
+        }
+        cache_file.write_text(json.dumps(fresh))
+
+        with mock.patch.object(probe_channels, "run_sf",
+                               side_effect=_all_fields_ok_run_sf) as m:
+            probe_channels.probe_channels(
+                "myorg", "00D000000000ABC", "v60.0", force_refresh=True,
+            )
+        self.assertEqual(m.call_count, len(probe_channels.ALL_SOBJECTS))
+
+
+class InvalidateAndReprobeTests(unittest.TestCase):
+    """runtime escape valve when SOQL returns INVALID_FIELD."""
+
+    def setUp(self) -> None:
+        self._tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self._tmp.cleanup)
+        self._orig_root = config.PROBE_CACHE_ROOT
+        config.PROBE_CACHE_ROOT = Path(self._tmp.name) / "_channel_probe"
+
+    def tearDown(self) -> None:
+        config.PROBE_CACHE_ROOT = self._orig_root
+
+    def test_invalidate_deletes_cache_and_reruns(self):
+        cache_dir = config.build_probe_cache_dir("00D000000000ABC", "v60.0")
+        cache_dir.mkdir(parents=True, exist_ok=True)
+        cache_file = cache_dir / "channels.json"
+        cache_file.write_text(json.dumps({
+            "_schema": "channels/1",
+            "_built_at_utc": time.time(),
+            "status": "OK",
+            "channels": {},
+        }))
+
+        with mock.patch.object(probe_channels, "run_sf",
+                               side_effect=_all_fields_ok_run_sf) as m:
+            out = probe_channels.invalidate_and_reprobe(
+                "myorg", "00D000000000ABC", "v60.0",
+            )
+
+        self.assertEqual(out["status"], "OK")
+        self.assertTrue(cache_file.is_file(), "fresh cache must be rewritten")
+        self.assertEqual(m.call_count, len(probe_channels.ALL_SOBJECTS))
+
+    def test_invalidate_tolerates_missing_cache(self):
+        """Cold-path invalidate (no file yet) must not crash."""
+        with mock.patch.object(probe_channels, "run_sf",
+                               side_effect=_all_fields_ok_run_sf):
+            out = probe_channels.invalidate_and_reprobe(
+                "myorg", "00D000000000ABC", "v60.0",
+            )
+        self.assertEqual(out["status"], "OK")
+
+
+class MandatoryFieldGateTests(unittest.TestCase):
+    """Missing mandatory field → status flips to PROBE_FAILED."""
+
+    def setUp(self) -> None:
+        self._tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self._tmp.cleanup)
+        self._orig_root = config.PROBE_CACHE_ROOT
+        config.PROBE_CACHE_ROOT = Path(self._tmp.name) / "_channel_probe"
+
+    def tearDown(self) -> None:
+        config.PROBE_CACHE_ROOT = self._orig_root
+
+    def test_missing_invocation_target_flags_probe_failed(self):
+        """SF release drops `InvocationTarget` on
+        GenAiFunctionDefinition → probe must surface PROBE_FAILED.
+        """
+
+        def mock_run_sf(recipe: str, **params: str) -> dict:
+            sobject = params["SOBJECT"]
+            mandatory = set(probe_channels.MANDATORY_FIELDS.get(sobject, set()))
+            if sobject == "GenAiFunctionDefinition":
+                # Simulate the schema drift: drop InvocationTarget.
+                mandatory.discard("InvocationTarget")
+            return _fake_describe_ok(sorted(mandatory))
+
+        with mock.patch.object(probe_channels, "run_sf", side_effect=mock_run_sf):
+            out = probe_channels.probe_channels(
+                "myorg", "00D000000000ABC", "v60.0", force_refresh=True,
+            )
+
+        self.assertEqual(out["status"], "PROBE_FAILED")
+        entry = out["channels"]["GenAiFunctionDefinition"]
+        self.assertIn("InvocationTarget", entry["mandatory_missing"])
+        # Other sObjects unaffected.
+        self.assertEqual(
+            out["channels"]["BotDefinition"]["mandatory_missing"], [],
+        )
+
+
+class CacheFilePermissionsTests(unittest.TestCase):
+    """cached channels.json must be 0o600 (owner RW only)."""
+
+    def setUp(self) -> None:
+        self._tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self._tmp.cleanup)
+        self._orig_root = config.PROBE_CACHE_ROOT
+        config.PROBE_CACHE_ROOT = Path(self._tmp.name) / "_channel_probe"
+
+    def tearDown(self) -> None:
+        config.PROBE_CACHE_ROOT = self._orig_root
+
+    def test_cache_file_is_mode_0o600(self):
+        with mock.patch.object(probe_channels, "run_sf",
+                               side_effect=_all_fields_ok_run_sf):
+            probe_channels.probe_channels(
+                "myorg", "00D000000000ABC", "v60.0", force_refresh=True,
+            )
+        cache_file = (
+            config.build_probe_cache_dir("00D000000000ABC", "v60.0")
+            / "channels.json"
+        )
+        self.assertTrue(cache_file.is_file())
+        mode = stat.S_IMODE(cache_file.stat().st_mode)
+        self.assertEqual(
+            mode, 0o600,
+            f"cache file mode must be 0o600, got 0o{mode:03o} ",
+        )
+
+
+class CacheFreshnessBoundsTests(unittest.TestCase):
+    """`_is_cache_fresh` must reject future timestamps as stale.
+
+    A `_built_at_utc` in the future is a symptom of clock skew (NTP
+    drift after suspend, container clock jump) or cache tampering —
+    either way, we treat it as untrustworthy and force a reprobe.
+    """
+
+    def setUp(self) -> None:
+        self._tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self._tmp.cleanup)
+
+    def test_future_timestamp_is_not_fresh(self):
+        # Year 3000 → unambiguously future.
+        future_epoch = 32503680000.0
+        cache_file = Path(self._tmp.name) / "channels.json"
+        cache_file.write_text(json.dumps({
+            "_schema": "channels/1",
+            "_built_at_utc": future_epoch,
+            "status": "OK",
+            "channels": {},
+        }))
+        self.assertFalse(
+            probe_channels._is_cache_fresh(cache_file, ttl_days=7),
+            "future _built_at_utc must be rejected as stale",
+        )
+
+    def test_present_timestamp_is_fresh(self):
+        cache_file = Path(self._tmp.name) / "channels.json"
+        cache_file.write_text(json.dumps({
+            "_schema": "channels/1",
+            "_built_at_utc": time.time(),
+            "status": "OK",
+            "channels": {},
+        }))
+        self.assertTrue(probe_channels._is_cache_fresh(cache_file, ttl_days=7))
+
+
+class WriteCacheConcurrencyTests(unittest.TestCase):
+    """concurrent _write_cache calls must not collide on tmp names.
+
+    The prior design used `path.with_suffix(path.suffix + ".tmp")` — a
+    deterministic name shared by every concurrent writer. This test
+    fires N threads that all write the same target path; after the dust
+    settles there must be exactly one final file, no leftover `.tmp`
+    files, and the contents must be valid JSON from ONE of the writers
+    (any of them is acceptable — what matters is integrity).
+    """
+
+    def setUp(self) -> None:
+        self._tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self._tmp.cleanup)
+
+    def test_concurrent_writes_no_collision_no_corruption(self):
+        import threading
+
+        target_dir = Path(self._tmp.name) / "probe"
+        target_path = target_dir / "channels.json"
+
+        N = 8
+        errors: list[BaseException] = []
+        barrier = threading.Barrier(N)
+
+        def worker(i: int) -> None:
+            payload = {
+                "_schema": "channels/1",
+                "_built_at_utc": float(i + 1),
+                "status": "OK",
+                "channels": {"writer": {"queryable_fields": [f"w{i}"]}},
+            }
+            barrier.wait()  # maximise contention
+            try:
+                probe_channels._write_cache(target_path, payload)
+            except BaseException as e:  # noqa: BLE001 — thread boundary
+                errors.append(e)
+
+        threads = [threading.Thread(target=worker, args=(i,)) for i in range(N)]
+        for t in threads:
+            t.start()
+        for t in threads:
+            t.join(timeout=5.0)
+
+        self.assertEqual(errors, [], f"worker errors: {errors}")
+        self.assertTrue(target_path.is_file())
+
+        # No stray tmp files left behind.
+        leftover = list(target_dir.glob(".channels.*.tmp"))
+        self.assertEqual(
+            leftover, [],
+            f"concurrent writes left tmp leftovers: {leftover}",
+        )
+
+        # Final file is parseable JSON (not a half-written corruption).
+        parsed = json.loads(target_path.read_text())
+        # One of the N payloads won the race; its _built_at_utc is in [1, N].
+        self.assertIn(parsed["_built_at_utc"], [float(i + 1) for i in range(N)])
+
+    def test_parent_dir_permissions_are_0o700(self):
+        """parent dir is normalized to 0o700 after write.
+
+        Even if the directory pre-existed with looser permissions, a
+        successful _write_cache leaves it at 0o700.
+        """
+        target_path = Path(self._tmp.name) / "probe" / "channels.json"
+        target_path.parent.mkdir(parents=True, exist_ok=True)
+        # Deliberately widen perms to prove _write_cache narrows them.
+        os.chmod(target_path.parent, 0o755)
+
+        probe_channels._write_cache(target_path, {
+            "_schema": "channels/1",
+            "_built_at_utc": time.time(),
+            "status": "OK",
+            "channels": {},
+        })
+
+        mode = stat.S_IMODE(target_path.parent.stat().st_mode)
+        self.assertEqual(
+            mode, 0o700,
+            f"parent dir mode must be 0o700, got 0o{mode:03o}",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

package/skills/investigating-agentforce-architecture/scripts/tests/test_probe_cli_recipes.py

@@ -0,0 +1,185 @@
+"""structural validation of the describe CLI recipes.
+
+Before these recipes shipped, `scripts/probe_channels.py::_describe_sobject`
+called `run_sf("describe_sobject", ...)` / `run_sf("describe_tooling_sobject",
+...)` against YAMLs that didn't exist on disk. Every unit test mocked
+`run_sf`, so the suite stayed green — but the first real invocation would
+have raised `SfCliError: recipe not found`. This module closes that gap
+by loading each YAML via `sf_cli._load_recipe` and asserting the schema
+`sf_cli.run_sf` expects.
+
+Deliberately does NOT invoke `sf sobject describe` — that needs a live
+org. Structural validation only; the wire-level contract is covered by
+the existing probe_channels integration path once the skill ships.
+"""
+from __future__ import annotations
+
+import unittest
+import unittest.mock as mock
+from pathlib import Path
+
+from . import _bootstrap  # noqa: F401
+
+import sf_cli  # type: ignore
+
+# SKILL_ROOT is now file-relative (config.py uses
+# Path(__file__).resolve().parent.parent), so config.CLI_DIR auto-resolves
+# to the repo's assets/cli/ under test. _REPO_CLI_DIR is kept for tests
+# that compare paths or pass them as explicit args.
+_REPO_CLI_DIR = (
+    Path(__file__).resolve().parent.parent.parent / "assets" / "cli"
+)
+
+
+class DescribeRecipeStructureTests(unittest.TestCase):
+    """Both describe recipes must parse + conform to the run_sf schema."""
+
+    def setUp(self) -> None:
+        # patch the module-level CLI_DIR binding inside sf_cli so
+        # _load_recipe reads from the repo's assets/cli/. `sf_cli`
+        # imported `CLI_DIR` by name at module load, so we must patch
+        # the *local* reference in sf_cli — patching `config.CLI_DIR`
+        # alone would leave the stale import behind.
+        self._patch = mock.patch.object(sf_cli, "CLI_DIR", _REPO_CLI_DIR)
+        self._patch.start()
+        self.addCleanup(self._patch.stop)
+
+    def _load(self, name: str) -> dict:
+        # _load_recipe is the same function run_sf uses at runtime, so if
+        # it parses here it will parse in production. yaml.safe_load + the
+        # mapping-shape guard both run inside _load_recipe .
+        return sf_cli._load_recipe(name)
+
+    # ---- parsing + mapping shape -----------------------------------------
+
+    def test_describe_sobject_parses(self):
+        recipe = self._load("describe_sobject")
+        self.assertIsInstance(recipe, dict)
+        self.assertEqual(recipe.get("name"), "describe_sobject")
+
+    def test_describe_tooling_sobject_parses(self):
+        recipe = self._load("describe_tooling_sobject")
+        self.assertIsInstance(recipe, dict)
+        self.assertEqual(recipe.get("name"), "describe_tooling_sobject")
+
+    # ---- required_params: a list, covers what probe_channels passes -------
+
+    def test_required_params_are_list_of_strings(self):
+        for name in ("describe_sobject", "describe_tooling_sobject"):
+            with self.subTest(recipe=name):
+                recipe = self._load(name)
+                required = recipe.get("required_params")
+                self.assertIsInstance(required, list)
+                self.assertTrue(
+                    all(isinstance(p, str) for p in required),
+                    f"{name}: required_params must be list[str]",
+                )
+                # probe_channels._describe_sobject passes exactly these two.
+                # If the recipe drops either, run_sf will SfCliError before
+                # the subprocess call — so assert both are present.
+                self.assertIn("ORG_ALIAS", required)
+                self.assertIn("SOBJECT", required)
+
+    # ---- argv: a list of strings with both placeholders referenced --------
+
+    def test_argv_is_list_of_strings(self):
+        for name in ("describe_sobject", "describe_tooling_sobject"):
+            with self.subTest(recipe=name):
+                recipe = self._load(name)
+                argv = recipe.get("argv")
+                self.assertIsInstance(argv, list)
+                self.assertGreater(len(argv), 0)
+                self.assertTrue(
+                    all(isinstance(e, str) for e in argv),
+                    f"{name}: argv must be list[str]",
+                )
+
+    def test_argv_references_both_required_placeholders(self):
+        """Every key in required_params must appear as `{{KEY}}` somewhere
+        in argv — otherwise the substitution is a no-op and the caller's
+        input never reaches the CLI. Defensive sanity check, not a
+        functional run.
+        """
+        for name in ("describe_sobject", "describe_tooling_sobject"):
+            with self.subTest(recipe=name):
+                recipe = self._load(name)
+                argv_joined = " ".join(recipe["argv"])
+                for key in recipe.get("required_params") or []:
+                    self.assertIn(
+                        f"{{{{{key}}}}}", argv_joined,
+                        f"{name}: required param {key} not referenced in argv",
+                    )
+
+    # ---- success_check contract -----------------------------------------
+
+    def test_success_check_is_stdout_json_status_zero(self):
+        """run_sf's success-detection code path (in sf_cli.run_sf) assumes
+        `stdout_json_status_zero`. Any other value silently changes the
+        pass/fail contract, so pin it.
+        """
+        for name in ("describe_sobject", "describe_tooling_sobject"):
+            with self.subTest(recipe=name):
+                recipe = self._load(name)
+                self.assertEqual(
+                    recipe.get("success_check"), "stdout_json_status_zero",
+                )
+
+    # ---- timeout is a positive int ---------------------------------------
+
+    def test_timeout_is_positive_int(self):
+        for name in ("describe_sobject", "describe_tooling_sobject"):
+            with self.subTest(recipe=name):
+                recipe = self._load(name)
+                t = recipe.get("timeout_seconds")
+                self.assertIsInstance(t, int)
+                self.assertGreater(t, 0)
+
+    # ---- auth_required_stderr_patterns -----------------------------------
+
+    def test_auth_patterns_cover_the_known_failures(self):
+        """sf CLI surfaces `NoOrgAuthenticationError` for a never-logged-in
+        alias and `AuthInfoError` for a stale/revoked session. Both must
+        classify as AuthRequired, not a generic SfCliError, so the skill
+        can prompt the user to re-login instead of aborting.
+        """
+        for name in ("describe_sobject", "describe_tooling_sobject"):
+            with self.subTest(recipe=name):
+                recipe = self._load(name)
+                patterns = recipe.get("auth_required_stderr_patterns") or []
+                self.assertIn("NoOrgAuthenticationError", patterns)
+                self.assertIn("AuthInfoError", patterns)
+
+    # ---- tooling recipe has --use-tooling-api; data-API recipe does NOT ---
+
+    def test_tooling_recipe_uses_tooling_api_flag(self):
+        recipe = self._load("describe_tooling_sobject")
+        self.assertIn("--use-tooling-api", recipe["argv"])
+
+    def test_data_api_recipe_does_not_use_tooling_api_flag(self):
+        """Separating the two recipes rather than toggling via a param
+        was a deliberate design call (call sites stay explicit). Verify
+        the separation hasn't regressed into a single merged recipe.
+        """
+        recipe = self._load("describe_sobject")
+        self.assertNotIn("--use-tooling-api", recipe["argv"])
+
+    # ---- run_sf missing-param surfacing runs end-to-end through recipe ----
+
+    def test_run_sf_raises_cleanly_when_required_param_absent(self):
+        """This exercises the full load_recipe → required_params check
+        path inside run_sf, with a REAL (on-disk) recipe rather than a
+        mock. If the YAML parses but required_params is malformed, this
+        would surface as something other than SfCliError with a clear
+        'missing required params' message.
+        """
+        with self.assertRaises(sf_cli.SfCliError) as ctx:
+            # Deliberately omit SOBJECT — should raise at the required-
+            # params check, well before subprocess.run. Pass ORG_ALIAS so
+            # we isolate the failure to the missing param.
+            sf_cli.run_sf("describe_sobject", ORG_ALIAS="whatever")
+        self.assertIn("missing required params", str(ctx.exception))
+        self.assertIn("SOBJECT", str(ctx.exception))
+
+
+if __name__ == "__main__":
+    unittest.main()