failproofai 1.0.1 → 1.0.2-beta.1

package/.next/standalone/fp-cloud-cli/tests/test_fp_home_contract.py

@@ -0,0 +1,198 @@
+"""The CLI writes into a home another component owns, so both halves must agree.
+
+`~/.failproofai/` is a governed layout. `src/hooks/fp-home.ts` is its register —
+"nothing outside this file may join a path onto the failproofai home" — and
+`resettablePaths()` is a *filter over* `HOME_CLASSES`, so what protects the
+CLI's credential from a reset is its entry in that table, not the fact that the
+file happens to exist.
+
+Until this file existed, nothing checked that the two sides agreed. `config.py`
+said so itself, in a comment above `FPCLI_SUBDIR`: *"Mirrors ``fpcliDir`` in
+``src/hooks/fp-home.ts`` — change one, change the other; nothing checks."*
+Verified by experiment: renaming `fpcliDir` to `fp-cloud-cli` in the TypeScript and
+leaving Python alone left 53 TS tests and 59 Python tests all passing, with the
+register describing a directory nothing writes and the real credential sitting
+at a path the register had never heard of.
+
+That is the same shape as the SDK's `tests/test_spool_contract.py` next door,
+which reads the Rust and the TypeScript that define its spool root. This is the
+CLI's version of it.
+
+The assertions run against source text rather than a running Node, because
+requiring a toolchain to test a Python package would mean this skips in every
+environment that matters and guards nothing — the failure mode the SDK's own
+contract test had for its whole life before it moved into this repo.
+"""
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+
+import pytest
+
+from fp_cli import config as cfg
+
+# tests/ -> fp-cloud-cli/ -> repo root.
+#
+# Guarded: `parents[2]` raises IndexError on a shallower tree, and a shallower
+# tree is exactly the packaged-sdist case `_read_source` below handles. Raising
+# here would be a COLLECTION error, which aborts the whole suite instead of
+# skipping the one file that needs the repository.
+_HERE = Path(__file__).resolve()
+REPO_ROOT = _HERE.parents[2] if len(_HERE.parents) > 2 else _HERE.parent
+FP_HOME_TS = REPO_ROOT / "src" / "hooks" / "fp-home.ts"
+
+#: Set by CI. Turns "the source I read is missing" from a skip into a failure.
+REQUIRE = os.environ.get("FP_CLI_REQUIRE_CONTRACT", "").strip().lower() in {
+    "1",
+    "true",
+    "yes",
+    "on",
+}
+
+
+def _read_source() -> str:
+    """`fp-home.ts`, or skip/fail depending on ``REQUIRE``.
+
+    Missing means one of two things: this is an installed sdist (fine — there is
+    no repo to read), or the file moved (not fine). ``REQUIRE`` distinguishes
+    them, because from in here they look identical.
+    """
+    if FP_HOME_TS.is_file():
+        return FP_HOME_TS.read_text(encoding="utf-8")
+    message = (
+        f"{FP_HOME_TS} is missing. In a packaged sdist that is expected. In the "
+        "repository it means the layout register moved, and this contract is now "
+        "unguarded — re-point this test at the new location rather than deleting it."
+    )
+    if REQUIRE:
+        pytest.fail(message)
+    pytest.skip(message)
+
+
+def _declared(pattern: str, source: str, what: str) -> str:
+    """The single capture of `pattern`, failing loudly if it matched 0 or 2+.
+
+    A regex over source that quietly matches nothing is worse than no test: it
+    passes forever while checking a file that has been rewritten around it.
+    """
+    matches = re.findall(pattern, source)
+    assert len(matches) == 1, (
+        f"expected exactly one declaration of {what} in fp-home.ts, found "
+        f"{len(matches)}. The register was restructured; re-anchor this test "
+        f"rather than loosening the pattern."
+    )
+    return matches[0]
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# The paths themselves
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+def test_the_subdirectory_name_agrees():
+    """`fpcliDir` in TypeScript vs `FPCLI_SUBDIR` here."""
+    source = _read_source()
+    declared = _declared(
+        r'export const fpcliDir = \(home\?: string\) => atHome\(home, "([^"]+)"\)',
+        source,
+        "fpcliDir",
+    )
+    assert declared == cfg.FPCLI_SUBDIR, (
+        f"fp-home.ts registers the CLI directory as {declared!r} and this package "
+        f"writes {cfg.FPCLI_SUBDIR!r}. The credential would sit at a path the "
+        f"layout register has never heard of, so nothing protects it from a reset "
+        f"the moment somebody classifies its parent."
+    )
+
+
+def test_the_credential_filename_agrees():
+    source = _read_source()
+    declared = _declared(
+        r'export const fpcliAuthFile = \(home\?: string\) => resolve\(fpcliDir\(home\), "([^"]+)"\)',
+        source,
+        "fpcliAuthFile",
+    )
+    assert declared == cfg.config_path().name
+
+
+def test_the_home_directory_name_agrees():
+    """Both sides hardcode `.failproofai`; neither imports it from the other."""
+    source = _read_source()
+    declared = _declared(
+        r'return process\.env\.FAILPROOFAI_HOME \|\| resolve\(homedir\(\), "([^"]+)"\)',
+        source,
+        "the failproofai home",
+    )
+    home = cfg.base_dir().parent.name
+    assert declared == home, f"fp-home.ts says {declared!r}, this package writes {home!r}"
+
+
+def test_the_shared_home_override_variable_agrees():
+    """`FAILPROOFAI_HOME` relocates the whole layout; the CLI must follow it."""
+    source = _read_source()
+    assert "process.env.FAILPROOFAI_HOME" in source
+    assert "FAILPROOFAI_HOME" in Path(cfg.__file__).read_text(encoding="utf-8")
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# The classification — what actually keeps a reset off the credential
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+def test_the_credential_is_registered_as_user_typed():
+    """This entry, not the file's existence, is what survives `resettablePaths()`.
+
+    `resettablePaths()` filters `HOME_CLASSES`, so an unregistered path survives
+    only by accident — and only until someone lists its parent. `user-typed` is
+    the class that says "a person typed this; nothing regenerates it", which is
+    exactly a login.
+    """
+    source = _read_source()
+    assert re.search(r"\{\s*path:\s*fpcliAuthFile,\s*class:\s*\"user-typed\"\s*\}", source), (
+        "fpcliAuthFile is no longer classified `user-typed` in HOME_CLASSES. A "
+        "reset walks that table; the CLI's session is not regenerable and must "
+        "never be on the delete list."
+    )
+
+
+def test_the_directory_itself_is_deliberately_unclassified():
+    """`auditDir`'s rule: classify the children, never the parent.
+
+    A `user-typed` parent would protect a cache added later; a `derived` parent
+    would delete the session beside it.
+    """
+    source = _read_source()
+    assert not re.search(r"\{\s*path:\s*fpcliDir,\s*class:", source), (
+        "fpcliDir is now classified as a whole. If the directory has grown a "
+        "second file, classify that file — one class cannot be right for a "
+        "credential and a cache at once."
+    )
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# The anchors, so none of the above can pass vacuously
+# ─────────────────────────────────────────────────────────────────────────────
+
+
+def test_the_register_still_contains_what_these_patterns_anchor_on():
+    """Every regex above reads source text, and source text gets rewritten.
+
+    If `fp-home.ts` is restructured so the patterns stop matching, the tests
+    above fail loudly via `_declared`. This one covers the rest of the file:
+    the names must still be exported and still be mentioned in the class table.
+    """
+    source = _read_source()
+    for anchor in (
+        "export const fpcliDir",
+        "export const fpcliAuthFile",
+        "export const HOME_CLASSES",
+        "fpcliAuthFile, class:",
+    ):
+        assert anchor in source, f"fp-home.ts no longer contains {anchor!r}"
+
+
+def test_this_file_can_be_imported_outside_the_repository():
+    """A collection error here would abort the whole CLI suite, not just this file."""
+    assert REPO_ROOT.is_absolute()

package/.next/standalone/fp-cloud-cli/tests/test_hardening.py

@@ -0,0 +1,186 @@
+"""Regression tests for the cli-bug-fix hardening pass.
+
+Covers the crash/robustness fixes: pagination cursor type-mix, guarded file reads,
+the non-JSON read-body trap, 429 messaging, explicit ISO date validation, non-finite
+score bounds, and the empty-token (no silent saved-session fallback) guard.
+"""
+
+from __future__ import annotations
+
+from fp_cli import _click_compat as click  # the Click Typer is running
+import httpx
+import pytest
+import respx
+
+from fp_cli import client, config, dates
+from fp_cli._context import validate_score_filters
+from fp_cli.app import app
+from fp_cli.client import ClientContext
+from fp_cli.commands._write import read_text_arg
+from fp_cli.errors import ApiError
+from fp_cli.models import Page
+
+BASE = "http://dash.test"
+
+
+def _ctx() -> ClientContext:
+    return ClientContext(base_url=BASE, token="t", org="globex")
+
+
+# --- paginate: int/str cursor mix must not crash (was a TypeError) ----------
+
+
+def test_paginate_mixed_cursor_types_no_crash():
+    # start_cursor is a str (from --cursor); pages return int next_cursor.
+    pages = [Page(items=[1, 2], next_cursor=100), Page(items=[3], next_cursor=None)]
+    seq = iter(pages)
+
+    def fetch(cursor, limit):
+        return next(seq)
+
+    got = list(client.paginate(fetch, limit=10, page_size=2, start_cursor="9999999999999999"))
+    assert got == [1, 2, 3]
+
+
+def test_paginate_stops_on_repeated_cursor():
+    # A server that returns the same cursor forever must not loop forever.
+    def fetch(cursor, limit):
+        return Page(items=["x"], next_cursor=42)
+
+    got = list(client.paginate(fetch, limit=1000, page_size=1, start_cursor=None))
+    assert 0 < len(got) < 1000  # bounded by the seen-cursor guard
+
+
+# --- read_text_arg: missing file -> usage error, not a traceback -----------
+
+
+def test_read_text_arg_missing_file_is_usage_error():
+    with pytest.raises(click.BadParameter):
+        read_text_arg("/no/such/file.json")
+
+
+# --- _get_json: non-JSON 2xx body -> ApiError, not raw JSONDecodeError ------
+
+
+@respx.mock
+def test_get_json_non_json_body_raises_clean_apierror():
+    respx.get(f"{BASE}/api/events").mock(
+        return_value=httpx.Response(200, text="<html>not json</html>")
+    )
+    with pytest.raises(ApiError):
+        client._get_json(_ctx(), "/api/events")
+
+
+@respx.mock
+def test_429_includes_retry_after():
+    respx.get(f"{BASE}/api/events").mock(
+        return_value=httpx.Response(429, headers={"retry-after": "5"}, json={})
+    )
+    with pytest.raises(ApiError) as ei:
+        client._get_json(_ctx(), "/api/events")
+    assert "Retry after 5" in str(ei.value)
+
+
+# --- dates: explicit --from/--to validation --------------------------------
+
+
+def test_resolve_range_rejects_garbage_date():
+    with pytest.raises(ValueError):
+        dates.resolve_range(ts_from="not-a-date")
+
+
+def test_resolve_range_rejects_date_only():
+    with pytest.raises(ValueError):
+        dates.resolve_range(ts_from="2026-06-01")
+
+
+def test_resolve_range_accepts_full_iso():
+    frm, _ = dates.resolve_range(ts_from="2026-05-01T00:00:00Z")
+    assert frm == "2026-05-01T00:00:00Z"
+
+
+# --- validate_score_filters: reject non-finite bounds ----------------------
+
+
+@pytest.mark.parametrize("value", ["helpfulness:nan..", "x:0.0..inf", "y:-inf..1"])
+def test_score_filter_rejects_non_finite(value):
+    with pytest.raises(click.BadParameter):
+        validate_score_filters([value])
+
+
+def test_score_filter_accepts_valid_ranges():
+    validate_score_filters(["helpfulness:0.5..0.8", "x:..0.3", "y:0.9.."])  # no raise
+
+
+@pytest.mark.parametrize("value", ["helpfulness:..", "x:..", "metric:.."])
+def test_score_filter_rejects_both_bounds_empty(value):
+    # `KEY:..` (no min, no max) is meaningless — the server silently drops it and
+    # returns the UNFILTERED set, so it must be a clean client-side usage error.
+    with pytest.raises(click.BadParameter):
+        validate_score_filters([value])
+
+
+# --- empty --token must NOT silently use the saved session -----------------
+
+
+def test_empty_token_does_not_fall_back_to_saved_session(home, runner):
+    config.save_config(config.CliConfig(base_url=BASE, session_token="saved-tok"))
+    # `--token ""` (e.g. an unset CI var) is an explicit "no auth", not a fallback.
+    result = runner.invoke(app, ["--base-url", BASE, "--token", "", "events"])
+    assert result.exit_code == 4, result.output  # not logged in (was: used saved-tok)
+
+
+def test_negative_timeout_is_usage_error(home, runner):
+    result = runner.invoke(app, ["--base-url", BASE, "--timeout", "-1", "events"])
+    assert result.exit_code == 2, result.output
+
+
+def test_a_saved_insecure_does_not_follow_you_to_another_dashboard(logged_in, runner, monkeypatch):
+    """`insecure` was stored per MACHINE and applied to every base URL.
+
+    One `fp --base-url https://dash.internal --insecure login` against a
+    self-signed dev box — the use its own help text names — left certificate
+    verification off for the next `fp --base-url https://app.befailproof.ai
+    login`, which is the request that carries the OTP and receives the session
+    token, and for `fp keys create`, which carries a freshly minted API key in
+    the body. `clear_token` deliberately preserves `insecure` across a logout, so
+    it outlived the session it was set for.
+    """
+    cfg = config.load_config()
+    cfg.insecure = True
+    cfg.base_url = "https://dash.internal"
+    config.save_config(cfg)
+
+    seen = {}
+
+    class _Recorder(httpx.Client):
+        def __init__(self, *a, **kw):
+            seen["verify"] = kw.get("verify")
+            super().__init__(*a, **kw)
+
+    monkeypatch.setattr(httpx, "Client", _Recorder)
+
+    with respx.mock(assert_all_called=False) as m:
+        m.get(url__regex=r".*").mock(return_value=httpx.Response(200, json={"keys": []}))
+        runner.invoke(app, ["--base-url", "https://app.befailproof.ai", "keys", "list"])
+
+    assert seen.get("verify") is not False, (
+        "a preference granted for dash.internal was applied to a different dashboard"
+    )
+
+
+def test_the_tls_warning_survives_quiet(logged_in, runner, monkeypatch):
+    """`--quiet` shares one gate with cosmetic status chrome; this is not that.
+
+    `fp --quiet …` on a machine where someone once ran `login --insecure` ran
+    every request with verification off and printed nothing anywhere.
+    """
+    cfg = config.load_config()
+    cfg.insecure = True
+    config.save_config(cfg)
+
+    with respx.mock(assert_all_called=False) as m:
+        m.get(url__regex=r".*").mock(return_value=httpx.Response(200, json={"keys": []}))
+        result = runner.invoke(app, ["--quiet", "keys", "list"])
+
+    assert "TLS verification disabled" in result.stderr, result.stderr

package/.next/standalone/fp-cloud-cli/tests/test_help_table_coverage.py

@@ -0,0 +1,127 @@
+"""The top-level help screen is a HAND-MAINTAINED table, not Click's command tree.
+
+`output.render_top_level_help` renders `_TOP_LEVEL_GROUPS`, a literal list. A command
+registered on the Typer app but missing from that list works perfectly and is invisible
+in `fp help` forever — there is no error, no warning, and no other test that looks.
+
+These tests close that gap in both directions, and additionally assert the help text
+carries the current command name, which is the thing a rename silently leaves stale
+(Click derives its own `Usage:` line from argv[0], so the auto-generated half updates
+itself and the hand-written half does not).
+"""
+
+from __future__ import annotations
+
+import re
+
+from typer.main import get_command
+
+from fp_cli import output
+from fp_cli.app import app
+
+
+def _registered_commands() -> set[str]:
+    """Every command name Click actually knows about."""
+    cmd = get_command(app)
+    return set(cmd.commands)  # type: ignore[attr-defined]
+
+
+def _help_table_commands() -> set[str]:
+    """Every command name the hand-maintained help table advertises."""
+    names = set()
+    for _group, entries in output._TOP_LEVEL_GROUPS:
+        for entry in entries:
+            names.add(entry[0])
+    return names
+
+
+def test_every_registered_command_appears_in_the_help_table():
+    missing = _registered_commands() - _help_table_commands()
+    assert not missing, (
+        f"these commands are registered but absent from output._TOP_LEVEL_GROUPS, so "
+        f"`fp help` will never mention them: {sorted(missing)}"
+    )
+
+
+def test_the_help_table_never_advertises_a_command_that_does_not_exist():
+    extra = _help_table_commands() - _registered_commands()
+    assert not extra, (
+        f"output._TOP_LEVEL_GROUPS advertises commands that are not registered — "
+        f"`fp <name>` would be a usage error: {sorted(extra)}"
+    )
+
+
+def test_the_help_table_is_not_empty():
+    """Guards the assertions above from passing vacuously if the structure changes."""
+    assert len(_help_table_commands()) >= 15
+
+
+def test_help_chrome_carries_no_retired_product_name():
+    """A rename updates Click's generated usage for free and the prose not at all."""
+    rendered = "\n".join(
+        str(entry) for _group, entries in output._TOP_LEVEL_GROUPS for entry in entries
+    )
+    examples = "\n".join(f"{cmd} {why}" for cmd, why in output._TOP_LEVEL_EXAMPLES)
+    haystack = f"{rendered}\n{examples}".lower()
+    assert "agenteye" not in haystack, "retired product name still present in help chrome"
+
+
+def test_the_help_examples_use_the_current_command_name():
+    cmds = [cmd for cmd, _why in output._TOP_LEVEL_EXAMPLES]
+    assert cmds, "no examples to check"
+    assert all(c.startswith("fp ") for c in cmds), (
+        f"help examples must invoke `fp`: {[c for c in cmds if not c.startswith('fp ')]}"
+    )
+
+
+def test_no_module_advertises_the_retired_env_var_namespace():
+    """`FP_*` is this CLI's namespace. `AGENTEYE_*` names that remain must be explicit
+    references to OTHER components (the collector's ingest key, the dashboard's admin
+    key, the SDK/collector spool dir) and never something this CLI reads."""
+    import pathlib
+
+    pkg = pathlib.Path(output.__file__).resolve().parent
+    allowed = {"AGENTEYE_KEY", "AGENTEYE_API_KEY", "AGENTEYE_HOME"}
+    found = set()
+    for mod in pkg.rglob("*.py"):
+        for m in re.finditer(r"AGENTEYE" + r"_[A-Z_]+", mod.read_text()):
+            found.add(m.group(0))
+    assert found <= allowed, (
+        f"these retired env vars are still referenced in the package: {sorted(found - allowed)}"
+    )
+
+
+def test_every_subcommand_the_hint_column_advertises_actually_exists():
+    """The THIRD element of each row was never checked against anything.
+
+    `_help_table_commands()` reads `entry[0]` only, so the ten standalone leaves
+    and the thirteen group names were guarded in both directions and every
+    advertised SUBCOMMAND was not. `issues` listed a bare `comment`, which is not
+    a command — the group exposes `comment-add`, `comment-list` and
+    `comment-delete` — so `fp help` told users to run something that exits 2 with
+    "No such command 'comment'".
+
+    A `*` suffix is a deliberate wildcard (`comment-*`, `context-*`); a leading
+    `-` is a flag, not a subcommand. Everything else must be real.
+    """
+    cmd = get_command(app)
+    bad = []
+    for _group, entries in output._TOP_LEVEL_GROUPS:
+        for entry in entries:
+            name, hint = entry[0], (entry[2] if len(entry) > 2 else "")
+            group = cmd.commands.get(name)  # type: ignore[attr-defined]
+            subcommands = set(getattr(group, "commands", {}) or {})
+            if not subcommands:
+                continue  # a standalone leaf; its hint is prose, not a command list
+            for token in hint.split():
+                if token.startswith("-"):
+                    continue
+                if token.endswith("*"):
+                    prefix = token[:-1]
+                    if not any(sub.startswith(prefix) for sub in subcommands):
+                        bad.append(f"{name}: '{token}' matches no subcommand")
+                    continue
+                if token not in subcommands:
+                    bad.append(f"{name}: '{token}' is not a subcommand of `fp {name}`")
+
+    assert not bad, "the help screen advertises commands that do not exist:\n  " + "\n  ".join(bad)