failproofai 1.0.12-beta.2 → 1.0.13-beta.1

package/.next/standalone/fp-cloud-cli/tests/test_cloud_observability.py

@@ -0,0 +1,116 @@
+import json
+
+import pytest
+import respx
+
+from fp_cli.app import app
+
+BASE = "http://dash.test"
+DASHBOARD = {"name": "Quality", "description": "Keep", "definition": {"filters": {"env": "prod"}}}
+TILE = {"id": "t1", "query_id": "q1", "title": "Before", "chart_type": "table",
+        "chart_config": {}, "param_bindings": {}, "pos_x": 0, "pos_y": 0, "pos_w": 6, "pos_h": 4}
+RESULT = {"columns": [{"name": "agent", "type": "String"}, {"name": "n", "type": "Int64"}],
+          "rows": [["agent-001", 42]], "truncated": False, "elapsed_ms": 1}
+CASES = [
+    (["dashboards", "list"], "GET", "/dashboards", {"dashboards": []}),
+    (["dashboards", "show", "d1"], "GET", "/dashboards/d1", {"dashboard": DASHBOARD, "tiles": []}),
+    (["dashboards", "create", "--name", "Quality"], "POST", "/dashboards", {"id": "d1"}),
+    (["dashboards", "update", "d1", "--name", "New"], "PUT", "/dashboards/d1", {"id": "d1"}),
+    (["dashboards", "delete", "d1"], "DELETE", "/dashboards/d1", {"deleted": True}),
+    (["dashboards", "tiles", "d1", "list"], "GET", "/dashboards/d1/tiles", {"tiles": [TILE]}),
+    (["dashboards", "tiles", "d1", "add", "--query", "q1", "--chart", "bar"], "POST", "/dashboards/d1/tiles", {"id": "t1"}),
+    (["dashboards", "tiles", "d1", "update", "t1", "--title", "After"], "PUT", "/dashboards/d1/tiles/t1", {"id": "t1"}),
+    (["dashboards", "tiles", "d1", "remove", "t1"], "DELETE", "/dashboards/d1/tiles/t1", {"deleted": True}),
+    (["dashboards", "tiles", "d1", "layout", "--file", "{layout}"], "PUT", "/dashboards/d1/tiles/layout", {"updated": 1}),
+    (["sessions", "show", "s1"], "GET", "/events/summary", {"events": [], "next_cursor": None}),
+    (["sessions", "export", "s1"], "GET", "/sessions/s1/export", {"session_id": "s1", "events": []}),
+    (["sessions", "history", "s1"], "GET", "/sessions/s1/evaluation-history", []),
+    (["events", "latency"], "GET", "/events/latency_aggregate", {"p95": 42}),
+    (["sentiment", "summary"], "GET", "/events/sentiment", {"positive": 1}),
+    (["sentiment", "messages"], "GET", "/events/sentiment/messages", []),
+    (["sentiment", "sessions"], "GET", "/events/sentiment/sessions", []),
+    (["sentiment", "session", "s1"], "GET", "/events/sentiment/sessions/s1", {}),
+    (["query", "export", "q1", "--format", "json"], "POST", "/queries/run", RESULT),
+    (["query", "export", "--sql", "SELECT 1", "--format", "csv"], "POST", "/queries/run", RESULT),
+]
+
+
+@pytest.fixture
+def files(tmp_path):
+    path = tmp_path / "layout.json"
+    path.write_text(json.dumps({"tiles": [{k: v for k, v in TILE.items() if k in ("id", "pos_x", "pos_y", "pos_w", "pos_h")}]}))
+    return {"layout": str(path)}
+
+
+@pytest.mark.parametrize("key_mode", [False, True])
+@pytest.mark.parametrize("argv,method,path,response", CASES, ids=lambda v: " ".join(v) if isinstance(v, list) else str(v))
+def test_observability_commands(logged_in, runner, files, key_mode, argv, method, path, response):
+    prefix = "/v1" if key_mode else "/api"
+    with respx.mock(assert_all_called=False) as mock:
+        mock.get(f"{BASE}{prefix}/dashboards/d1").respond(200, json={"dashboard": DASHBOARD, "tiles": []})
+        mock.get(f"{BASE}{prefix}/dashboards/d1/tiles").respond(200, json={"tiles": [TILE]})
+        mock.get(f"{BASE}{prefix}/sessions").respond(200, json={"sessions": [{"session_id": "s1"}]})
+        route = mock.request(method, f"{BASE}{prefix}{path}").respond(200, json=response)
+        result = runner.invoke(app, ["--json", *(["--api-key", "key"] if key_mode else []), *[v.format(**files) for v in argv]])
+    assert result.exit_code == 0, result.output
+    assert route.called
+    json.loads(result.stdout)
+    if argv[:2] == ["dashboards", "create"]:
+        assert json.loads(route.calls[0].request.content) == {"name": "Quality", "description": "", "definition": {}}
+    if argv[:2] == ["dashboards", "update"]:
+        assert json.loads(route.calls[0].request.content) == {**DASHBOARD, "name": "New"}
+    if argv[:4] == ["dashboards", "tiles", "d1", "update"]:
+        assert json.loads(route.calls[0].request.content) == {k: ("After" if k == "title" else v) for k, v in TILE.items() if k != "id"}
+
+
+@pytest.mark.parametrize("argv,method,path,response", CASES)
+@pytest.mark.parametrize("key_mode", [False, True])
+def test_observability_404(logged_in, runner, files, key_mode, argv, method, path, response):
+    with respx.mock as mock:
+        mock.route().respond(404, json={"error": "unknown route"})
+        result = runner.invoke(app, ["--json", *(["--api-key", "key"] if key_mode else []), *[v.format(**files) for v in argv]])
+    assert result.exit_code == 6, result.output
+    assert "not available on this FailproofAI Cloud yet" in result.stdout
+
+
+def test_query_compose(logged_in, runner):
+    stream = 'data: {"type":"done","sql":"SELECT 1"}\n\n'
+    with respx.mock as mock:
+        draft = mock.post(f"{BASE}/api/agent/compose-sql").respond(200, text=stream, headers={"content-type": "text/event-stream"})
+        run = mock.post(f"{BASE}/api/queries/run").respond(200, json=RESULT)
+        result = runner.invoke(app, ["--json", "query", "compose", "Count events", "--run"])
+    assert result.exit_code == 0, result.output
+    assert json.loads(draft.calls[0].request.content) == {"intent": "Count events"}
+    assert json.loads(run.calls[0].request.content) == {"sql": "SELECT 1", "params": []}
+
+
+def test_query_compose_key_denial(logged_in, runner):
+    with respx.mock as mock:
+        result = runner.invoke(app, ["--json", "--api-key", "key", "query", "compose", "Count events"])
+        assert not mock.calls
+    assert result.exit_code == 2
+    assert "fp login" in result.stdout
+
+
+def test_exports_write_files(logged_in, runner, tmp_path):
+    path = tmp_path / "data.csv"
+    with respx.mock as mock:
+        mock.post(f"{BASE}/api/queries/run").respond(200, json=RESULT)
+        result = runner.invoke(app, ["--json", "query", "export", "--sql", "SELECT 1", "--out", str(path)])
+    assert result.exit_code == 0
+    assert path.read_text().splitlines() == ["agent,n", "agent-001,42"]
+
+
+@pytest.mark.parametrize("argv", [
+    ["dashboards", "create", "--name", ""],
+    ["dashboards", "update", "d1"],
+    ["dashboards", "tiles", "d1", "add", "--query", "q1", "--chart", "invalid"],
+    ["sentiment", "messages", "--limit", "0"],
+    ["query", "export", "--sql", "SELECT 1", "--format", "xml"],
+    ["query", "export", "q1", "--sql", "SELECT 1"],
+])
+def test_validation(logged_in, runner, argv):
+    with respx.mock as mock:
+        result = runner.invoke(app, ["--json", *argv])
+        assert not mock.calls
+    assert result.exit_code == 2, result.output

package/.next/standalone/fp-cloud-cli/tests/test_cloud_transport.py

@@ -0,0 +1,141 @@
+import pytest
+import respx
+
+from fp_cli import client
+from fp_cli.errors import NotFoundError
+
+
+@pytest.mark.parametrize("family", [
+    "evaluator-definitions", "evaluator-instances", "evaluator-schema-profile",
+    "evaluation-jobs", "dashboards", "otel", "offers", "team-trial",
+])
+def test_added_v1_families(family):
+    assert client._v1_path(f"/api/{family}") == f"/v1/{family}"
+
+
+@pytest.mark.parametrize("method", ["GET", "POST", "PATCH", "DELETE"])
+@pytest.mark.parametrize("key_mode", [False, True])
+def test_cloud_request_degrades(method, key_mode):
+    ctx = client.ClientContext(
+        "http://dash.test", token=None if key_mode else "tok",
+        api_key="key" if key_mode else None,
+        auth_mode=client.AuthMode.API_KEY if key_mode else client.AuthMode.SESSION,
+    )
+    prefix = "v1" if key_mode else "api"
+    with respx.mock:
+        route = respx.request(method, f"http://dash.test/{prefix}/dashboards").respond(404, json={})
+        with pytest.raises(NotFoundError, match="not available on this FailproofAI Cloud yet") as exc:
+            client.cloud_request(ctx, method, "/api/dashboards")
+    assert exc.value.exit_code == 6
+    headers = route.calls[0].request.headers
+    assert ("authorization" in headers) is key_mode
+    assert ("cookie" in headers) is not key_mode
+
+
+def test_new_capability_html_404_degrades():
+    ctx = client.ClientContext("http://dash.test", api_key="key", auth_mode=client.AuthMode.API_KEY)
+    with respx.mock as mock:
+        mock.get("http://dash.test/v1/dashboards").respond(404, text="<html>old Cloud</html>", headers={"content-type": "text/html"})
+        with pytest.raises(NotFoundError, match="not available on this FailproofAI Cloud yet") as exc:
+            client.cloud_request(ctx, "GET", "/api/dashboards")
+    assert exc.value.exit_code == 6
+
+
+@pytest.mark.parametrize("key_mode", [False, True])
+@pytest.mark.parametrize("method,path,body,params", [
+    ("PATCH", "/api/evaluator-definitions/stale-id", {"enabled": False}, None),
+    ("GET", "/api/sessions/missing/export", None, None),
+    ("POST", "/api/sessions/missing/re-evaluate", None, None),
+    ("GET", "/api/dashboards/missing/tiles", None, None),
+    ("DELETE", "/api/dashboards/board/tiles/stale-tile", None, None),
+    ("GET", "/api/events/sentiment/sessions/missing", None, None),
+    ("GET", "/api/audits/contracts/stale-id", None, None),
+    ("POST", "/api/queries/run", {"query_id": "missing"}, None),
+    ("POST", "/api/agent/answer", {"callId": "missing", "decision": "reject"}, None),
+    ("DELETE", "/api/settings/model-context-windows", None, {"model": "missing"}),
+])
+def test_record_404_is_not_route_unavailable(key_mode, method, path, body, params):
+    ctx = client.ClientContext(
+        "http://dash.test", token=None if key_mode else "tok",
+        api_key="key" if key_mode else None,
+        auth_mode=client.AuthMode.API_KEY if key_mode else client.AuthMode.SESSION,
+    )
+    if key_mode and path.startswith("/api/agent/"):
+        pytest.skip("the agent has no key route; tested via session-only command guards")
+    wire_path = client._v1_path(path) if key_mode else path
+    with respx.mock as mock:
+        mock.request(method, f"http://dash.test{wire_path}").respond(404, json={})
+        with pytest.raises(NotFoundError) as exc:
+            client.cloud_request(ctx, method, path, body=body, params=params)
+    assert exc.value.exit_code == 6
+    assert "Record not found" in exc.value.message
+    assert "not available on this FailproofAI Cloud yet" not in exc.value.message
+
+
+@pytest.mark.parametrize("response,route_unknown", [
+    ({"code": "route_not_found"}, True),
+    ({"error": {"code": "route_not_found"}}, True),
+    ({"error": "unknown endpoint"}, True),
+    ({"message": "Cannot PATCH /evaluator-definitions/stale-id"}, True),
+    ({"error": "route does not exist"}, True),
+    ({"error": "old Cloud route absent"}, True),
+    ({"error": "evaluator definition not found"}, False),
+    ({"error": "session not found"}, False),
+    ({"error": "query not found"}, False),
+    ({"error": "not available in demo"}, False),
+])
+def test_404_response_overrides_request_shape(response, route_unknown):
+    ctx = client.ClientContext("http://dash.test", token="tok")
+    with respx.mock as mock:
+        mock.patch("http://dash.test/api/evaluator-definitions/stale-id").respond(404, json=response)
+        with pytest.raises(NotFoundError) as exc:
+            client.cloud_request(ctx, "PATCH", "/api/evaluator-definitions/stale-id", body={"enabled": False})
+    assert exc.value.exit_code == 6
+    if route_unknown:
+        assert exc.value.message == "not available on this FailproofAI Cloud yet"
+    else:
+        assert exc.value.message == response.get("error", response.get("message"))
+
+
+@pytest.mark.parametrize("body", [{"code": "record_not_found"}, {"error": "resource_not_found"}])
+def test_record_error_code_on_collection_is_not_unavailable(body):
+    ctx = client.ClientContext("http://dash.test", token="tok")
+    with respx.mock as mock:
+        mock.post("http://dash.test/api/queries/run").respond(404, json=body)
+        with pytest.raises(NotFoundError, match="Record not found") as exc:
+            client.cloud_request(ctx, "POST", "/api/queries/run", body={"sql": "SELECT 1"})
+    assert "not available" not in exc.value.message
+
+
+def test_collection_with_missing_saved_query_preserves_server_error():
+    ctx = client.ClientContext("http://dash.test", token="tok")
+    with respx.mock as mock:
+        mock.post("http://dash.test/api/queries/run").respond(404, json={"error": "saved query not found"})
+        with pytest.raises(NotFoundError, match="saved query not found"):
+            client.cloud_request(ctx, "POST", "/api/queries/run", body={"query_id": "stale"})
+
+
+def test_stream_404_distinguishes_route_and_record():
+    ctx = client.ClientContext("http://dash.test", token="tok")
+    for error, expected in [
+        ({"code": "unknown_route"}, "not available on this FailproofAI Cloud yet"),
+        ({"error": "conversation not found"}, "conversation not found"),
+    ]:
+        with respx.mock as mock:
+            mock.post("http://dash.test/api/agent/chat").respond(404, json=error)
+            with pytest.raises(NotFoundError, match=expected):
+                list(client.cloud_stream(ctx, "/api/agent/chat", {"conversationId": "missing"}))
+
+
+@pytest.mark.parametrize("path", [
+    "/api/agent/conversations/stale",
+    "/api/offers/stale",
+    "/api/integrations/slack/stale",
+])
+def test_named_endpoint_families_do_not_hide_record_404s(path):
+    ctx = client.ClientContext("http://dash.test", token="tok")
+    with respx.mock as mock:
+        mock.get(f"http://dash.test{path}").respond(404, json={})
+        with pytest.raises(NotFoundError, match="Record not found") as exc:
+            client.cloud_request(ctx, "GET", path)
+    assert "not available" not in exc.value.message

package/.next/standalone/fp-cloud-cli/tests/test_commands.py

@@ -522,7 +522,7 @@ def test_sessions_table(logged_in, runner):
     )
     result = runner.invoke(app, ["sessions"])
     assert result.exit_code == 0, result.output
-    assert "sessions" in result.stdout  # the boxed panel title
+    assert "SESSIONS" in result.stdout
     assert "done" in result.stdout  # status flattened up from latest_evaluation
 
 
@@ -679,7 +679,7 @@ def test_help_command(home, runner):
     result = runner.invoke(app, ["help"])
     assert result.exit_code == 0
     out = result.stdout
-    assert "Commands" in out  # the grouped top-level panel
+    assert "COMMANDS" in out
     # the four purpose groups, in order
     for g in ("ESSENTIALS", "OBSERVE", "MANAGE", "TOOLS"):
         assert g in out
@@ -744,7 +744,7 @@ def test_fields_unknown_is_usage_error(logged_in, runner):
 def test_dash_h_alias_shows_help(home, runner):
     result = runner.invoke(app, ["-h"])
     assert result.exit_code == 0
-    assert "Commands" in result.stdout and "sessions" in result.stdout
+    assert "COMMANDS" in result.stdout and "sessions" in result.stdout
     assert "ESSENTIALS" in result.stdout  # the grouped top-level help, via the rich_format_help override
 
 

package/.next/standalone/fp-cloud-cli/tests/test_dashboards_agent.py

@@ -171,7 +171,7 @@ def test_agent_health_renders_configured_card(logged_in, runner):
     result = runner.invoke(app, ["agent", "health"])
     assert result.exit_code == 0, result.output
     out = result.stdout
-    assert "assistant" in out and "configured" in out and "claude-x" in out
+    assert "ASSISTANT" in out and "configured" in out and "claude-x" in out
 
 
 @respx.mock
@@ -192,7 +192,7 @@ def test_agent_models_renders_default_marker(logged_in, runner):
     result = runner.invoke(app, ["agent", "models"])
     assert result.exit_code == 0, result.output
     out = result.stdout
-    assert "models" in out and "m-default" in out and "m-fast" in out and "default" in out
+    assert "MODELS" in out and "m-default" in out and "m-fast" in out and "default" in out
 
 
 @respx.mock
@@ -213,7 +213,7 @@ def test_agent_chats_renders_table(logged_in, runner):
     )
     result = runner.invoke(app, ["agent", "chats"])
     assert result.exit_code == 0, result.output
-    assert "chats" in result.stdout and "perf review" in result.stdout
+    assert "CHATS" in result.stdout and "perf review" in result.stdout
     assert "chat-id" in result.stdout and "c1" in result.stdout  # full chat-id column by default
     assert "open one with" not in (result.stderr or "")           # footer removed