failproofai 1.0.12-beta.2 → 1.0.12

package/.next/standalone/fp-cloud-cli/fp_cli/commands/evaluators_cmds.py

@@ -0,0 +1,380 @@
+"""Author, test and deploy managed evaluators, independently of result readers."""
+
+from __future__ import annotations
+
+import json
+import re
+from datetime import datetime, timedelta, timezone
+from typing import List, Optional
+
+import typer
+
+from .. import client as api, cloud_output, dates, evaluator_spec as spec, output
+from .._context import GLOBALS_EPILOG, require_auth
+from ..errors import ApiError, NotFoundError
+from . import _cloud, _write
+
+PATH = "/api/evaluator-definitions"
+
+
+def definitions(cctx) -> list:
+    return _cloud.rows(api.cloud_request(cctx, "GET", PATH), "definitions")
+
+
+def select(items: list, handle: str) -> dict:
+    key, _, version = handle.partition("@")
+    matches = [r for r in items if r.get("eval_key") == key and (not version or str(r.get("eval_version")) == version)]
+    if not matches:
+        raise NotFoundError(f"no evaluator {handle}; run `fp evaluators list --all`")
+    if not version:
+        enabled = [r for r in matches if r.get("enabled", True)]
+        matches = enabled or matches
+    if len(matches) != 1:
+        spec.fail(f"{handle} matches multiple versions; use KEY@VERSION")
+    return matches[0]
+
+
+def list_defs(ctx: typer.Context, key: Optional[str] = typer.Option(None, "--key"),
+              all_versions: bool = typer.Option(False, "--all")) -> None:
+    """List definitions (enabled only unless --all). Needs evaluations:read."""
+    items = definitions(require_auth(ctx.obj))
+    items = [r for r in items if (not key or r.get("eval_key") == key) and (all_versions or r.get("enabled", True))]
+    _cloud.emit(ctx.obj, items, "Evaluators")
+
+
+def show(ctx: typer.Context, handle: str = typer.Argument(..., metavar="KEY[@VERSION]")) -> None:
+    """Show one definition, including its full source."""
+    _cloud.emit(ctx.obj, select(definitions(require_auth(ctx.obj)), handle), "Evaluator")
+
+
+def _create(state, cctx, body, sessions, disabled=False, force=False) -> dict:
+    if sessions:
+        if spec.source_kind(body) == "jev":
+            output.warn("Jev dry-runs spend model budget.")
+        preview = api.cloud_request(cctx, "POST", f"{PATH}/dry-run", body=spec.dry_run_body(body, sessions))
+        if not state.json:
+            cloud_output.dry_run(preview)
+        if dry_run_failed(preview) and force:
+            output.warn("Dry-run failed; --force authorizes creating the evaluator anyway.")
+        else:
+            check_dry_run(preview)
+    _write.confirm(state, f"Create evaluator {body['eval_key']}@{body['eval_version']}", assume_yes=False)
+    row = api.cloud_request(cctx, "POST", PATH, body=body)
+    if not isinstance(row, dict) or not row.get("id"):
+        raise ApiError("Cloud created no identifiable evaluator; check evaluators list --all before retrying")
+    if disabled:
+        api.cloud_request(cctx, "PATCH", f"{PATH}/{row['id']}", body={"enabled": False})
+        row = {**row, "enabled": False}
+    _write.record_action("evaluator_created", resource="evaluator", mode=spec.source_kind(body), success=True)
+    return row
+
+
+def create(
+    ctx: typer.Context,
+    key: Optional[str] = typer.Option(None, "--key"),
+    name: Optional[str] = typer.Option(None, "--name"),
+    version: Optional[str] = typer.Option(None, "--version"),
+    label: Optional[List[str]] = typer.Option(None, "--label"),
+    condition: Optional[str] = typer.Option(None, "--condition", help="Python expression or @file."),
+    code: Optional[str] = typer.Option(None, "--code", help="Restricted Python expression file, or - for stdin."),
+    result_kind: Optional[str] = typer.Option(None, "--result-kind"),
+    timeout: Optional[float] = typer.Option(None, "--timeout"),
+    judge: bool = typer.Option(False, "--judge"),
+    criteria: Optional[str] = typer.Option(None, "--criteria"),
+    step: Optional[List[str]] = typer.Option(None, "--step"),
+    threshold: Optional[float] = typer.Option(None, "--threshold"),
+    jev: Optional[str] = typer.Option(None, "--jev", help="noul or score."),
+    instructions: Optional[str] = typer.Option(None, "--instructions"),
+    true: Optional[str] = typer.Option(None, "--true"),
+    false: Optional[str] = typer.Option(None, "--false"),
+    level: Optional[List[str]] = typer.Option(None, "--level", help="3–5 distinct levels, worst to best."),
+    file: Optional[str] = typer.Option(None, "--file"),
+    dry_run_on: Optional[List[str]] = typer.Option(None, "--dry-run-on"),
+    disabled: bool = typer.Option(False, "--disabled"),
+    force: bool = typer.Option(False, "--force", help="Create despite failed --dry-run-on results (review the failures first)."),
+) -> None:
+    """Create a code, LLM judge or Jev evaluator (enabled by default).
+
+    Needs evaluations:trigger; session dry-run also needs events:read.
+
+    Code: `fp --json evaluators create --key concise --name Concise --code concise.py --result-kind score --dry-run-on run-001`
+
+    Judge: `fp --json evaluators create --key relevance --name Relevance --judge --criteria "Grade relevance to the question" --step "Assess answer specificity" --threshold 0.7`
+
+    Jev: `fp --json evaluators create --key resolved --name Resolved --jev noul --instructions "Assess whether the request was resolved" --true "Fully resolved" --false "Not resolved" --threshold 0.8`
+
+    Spec: `fp --json evaluators create --file evaluator.json`
+    """
+    body = spec.build(
+        key=key, name=name, version=version, label=label, condition=condition,
+        code=code, result_kind=result_kind, timeout=timeout, judge=judge, criteria=criteria,
+        step=step, threshold_value=threshold, jev=jev, instructions=instructions,
+        true=true, false=false, level=level, file=file,
+    )
+    # Validate before any request, including the judge refusal / session bound.
+    if dry_run_on:
+        spec.dry_run_body(body, dry_run_on)
+    row = _create(ctx.obj, require_auth(ctx.obj), body, dry_run_on, disabled, force)
+    if ctx.obj.json:
+        output.emit_json(row)
+    else:
+        cloud_output.evaluator({**body, **row}, "Evaluator created")
+
+
+def deploy(
+    ctx: typer.Context,
+    key: str = typer.Argument(..., metavar="EVAL_KEY"),
+    name: Optional[str] = typer.Option(None, "--name"),
+    version: Optional[str] = typer.Option(None, "--version"),
+    label: Optional[List[str]] = typer.Option(None, "--label"),
+    condition: Optional[str] = typer.Option(None, "--condition"),
+    code: Optional[str] = typer.Option(None, "--code"),
+    result_kind: Optional[str] = typer.Option(None, "--result-kind"),
+    timeout: Optional[float] = typer.Option(None, "--timeout"),
+    judge: bool = typer.Option(False, "--judge"),
+    criteria: Optional[str] = typer.Option(None, "--criteria"),
+    step: Optional[List[str]] = typer.Option(None, "--step"),
+    threshold: Optional[float] = typer.Option(None, "--threshold"),
+    jev: Optional[str] = typer.Option(None, "--jev"),
+    instructions: Optional[str] = typer.Option(None, "--instructions"),
+    true: Optional[str] = typer.Option(None, "--true"),
+    false: Optional[str] = typer.Option(None, "--false"),
+    level: Optional[List[str]] = typer.Option(None, "--level"),
+    file: Optional[str] = typer.Option(None, "--file"),
+    dry_run_on: Optional[List[str]] = typer.Option(None, "--dry-run-on"),
+    backfill: Optional[str] = typer.Option(None, "--backfill", help="Score stored sessions too, e.g. 7d (up to 90d). Judge/Jev backfills spend model budget."),
+    force: bool = typer.Option(False, "--force", help="Deploy despite failed --dry-run-on results (review the failures first)."),
+) -> None:
+    """Publish a NEW immutable version, then disable enabled predecessors.
+
+    Numeric versions auto-increment; for other version names pass --version.
+    A failed predecessor PATCH leaves the new version enabled and reports its id.
+    """
+    body = spec.build(
+        key=None if file else key, name=name, version=None if file else version, label=label,
+        condition=condition, code=code, result_kind=result_kind, timeout=timeout,
+        judge=judge, criteria=criteria, step=step, threshold_value=threshold, jev=jev,
+        instructions=instructions, true=true, false=false, level=level, file=file, complete=False,
+    )
+    if file and body.get("eval_key") is not None and body["eval_key"] != key:
+        spec.fail("The spec eval_key must match the deploy argument")
+    body["eval_key"] = key
+    if version is not None:
+        body["eval_version"] = version
+    if dry_run_on:
+        spec.dry_run_body(body, dry_run_on)
+    backfill_body = None
+    if backfill:
+        if not re.fullmatch(r"[1-9][0-9]*d", backfill) or int(backfill[:-1]) > 90:
+            spec.fail("--backfill must be a duration in days from 1d to 90d")
+        now = datetime.now(timezone.utc)
+        backfill_body = {
+            "from": dates._iso(now - timedelta(days=int(backfill[:-1]))),
+            "to": dates._iso(now),
+            "eval_key": key,
+            "dry_run": False,
+        }
+        if spec.source_kind(body) != "code":
+            output.warn("Backfilling a judge/Jev evaluator spends the org's model budget.")
+    cctx = require_auth(ctx.obj)
+    existing = [r for r in definitions(cctx) if r.get("eval_key") == key]
+    if "eval_version" not in body:
+        versions = [str(r.get("eval_version", "")) for r in existing]
+        if any(not v.isdigit() for v in versions):
+            spec.fail("Non-numeric versions exist; specify a new --version")
+        body["eval_version"] = str(max([int(v) for v in versions] or [0]) + 1)
+    if any(str(r.get("eval_version")) == body["eval_version"] for r in existing):
+        spec.fail(f"Version {key}@{body['eval_version']} already exists; deploy needs a NEW version")
+    body.setdefault("display_name", existing[-1].get("display_name", key) if existing else key)
+    if existing and not file:
+        # A new version keeps the previous version's scope and labels unless a flag
+        # changes them; silently dropping the condition would widen the evaluator to
+        # every session. A --file spec states every field itself.
+        latest = max(existing, key=lambda r: str(r.get("created_at") or ""))
+        if condition is None:
+            body["condition_source"] = latest.get("condition_source")
+        if not label:
+            body["labels"] = list(latest.get("labels") or [])
+    spec.validate(body)
+    created = _create(ctx.obj, cctx, body, dry_run_on, force=force)
+    disabled_ids = []
+    try:
+        for old in existing:
+            if old.get("enabled", True):
+                api.cloud_request(cctx, "PATCH", f"{PATH}/{old['id']}", body={"enabled": False})
+                disabled_ids.append(old["id"])
+        if backfill_body:
+            api.cloud_request(cctx, "POST", "/api/evaluations/backfill", body=backfill_body)
+    except Exception as exc:
+        output.warn(f"Published {key}@{body['eval_version']} (id {created['id']}); disabled {len(disabled_ids)} predecessors. Follow-up failed; inspect evaluators list --all before retrying.")
+        raise exc
+    _write.record_action("evaluator_deployed", resource="evaluator", success=True)
+    if ctx.obj.json:
+        output.emit_json({"created": created, "disabled": disabled_ids})
+    else:
+        cloud_output.deployed({**body, **created}, [r for r in existing if r["id"] in disabled_ids])
+
+
+def draft(
+    ctx: typer.Context,
+    intent: str = typer.Argument(...),
+    kind: str = typer.Option("code", "--kind"),
+    out: str = typer.Option("evaluator.json", "--out"),
+) -> None:
+    """AI-compose an API spec file. Session only; review it before deployment."""
+    _cloud.session_only(ctx.obj, "evaluators draft")
+    if kind not in ("code", "judge", "jev"):
+        spec.fail("--kind must be code, judge or jev")
+    data = api.cloud_request(require_auth(ctx.obj), "POST", "/api/agent/compose-eval",
+        body={"intent": f"Author a {kind} evaluator. {intent}", "resultKind": "score"})
+    if not isinstance(data, dict):
+        spec.fail("Cloud draft was not an evaluator JSON spec")
+    body = spec.from_draft(data, kind)
+    _cloud.write_file(out, json.dumps(body, indent=2, ensure_ascii=False) + "\n")
+    _cloud.emit(ctx.obj, {"out": out, "spec": body}, "Evaluator draft")
+
+
+def dry_run(
+    ctx: typer.Context,
+    code: Optional[str] = typer.Option(None, "--code"),
+    result_kind: Optional[str] = typer.Option(None, "--result-kind"),
+    timeout: Optional[float] = typer.Option(None, "--timeout"),
+    condition: Optional[str] = typer.Option(None, "--condition"),
+    judge: bool = typer.Option(False, "--judge"),
+    criteria: Optional[str] = typer.Option(None, "--criteria"),
+    step: Optional[List[str]] = typer.Option(None, "--step"),
+    threshold: Optional[float] = typer.Option(None, "--threshold"),
+    jev: Optional[str] = typer.Option(None, "--jev"),
+    instructions: Optional[str] = typer.Option(None, "--instructions"),
+    true: Optional[str] = typer.Option(None, "--true"),
+    false: Optional[str] = typer.Option(None, "--false"),
+    level: Optional[List[str]] = typer.Option(None, "--level"),
+    file: Optional[str] = typer.Option(None, "--file"),
+    session: Optional[List[str]] = typer.Option(None, "--session"),
+) -> None:
+    """Compile only, or run on ≤10 sessions. Cloud refuses judge dry-runs.
+
+    Jev dry-runs execute model calls and spend budget, including before deployment.
+    """
+    body = spec.build(
+        code=code, result_kind=result_kind, timeout=timeout, condition=condition,
+        judge=judge, criteria=criteria, step=step, threshold_value=threshold, jev=jev,
+        instructions=instructions, true=true, false=false, level=level, file=file, complete=False,
+    )
+    body = spec.dry_run_body(body, session)
+    if spec.source_kind(body) == "jev":
+        output.warn("Jev dry-runs spend model budget.")
+    result = api.cloud_request(require_auth(ctx.obj), "POST", f"{PATH}/dry-run", body=body)
+    _cloud.emit(ctx.obj, result, "Evaluator dry-run")
+    if dry_run_failed(result):
+        # Preserve the full dry-run result in --json, but do not signal success.
+        raise typer.Exit(code=1)
+
+
+def dry_run_failed(result):
+    return isinstance(result, dict) and (
+        result.get("ok") is False
+        or any(r.get("ok") is False for r in result.get("runs", []) if isinstance(r, dict))
+        or result.get("summary", {}).get("failed", 0) > 0
+    )
+
+
+def check_dry_run(result):
+    if dry_run_failed(result):
+        failures = [r for r in result.get("runs", []) if r.get("ok") is False]
+        detail = "; ".join(str(r.get("error_message") or r.get("error_kind") or "run failed") for r in failures)
+        raise ApiError(f"Dry-run failed; evaluator was not created. {detail}".strip())
+
+
+def toggle(ctx, handle, enabled):
+    cctx = require_auth(ctx.obj)
+    row = select(definitions(cctx), handle)
+    _write.confirm(ctx.obj, f"{'Enable' if enabled else 'Disable'} {handle}", assume_yes=False)
+    result = api.cloud_request(cctx, "PATCH", f"{PATH}/{row['id']}", body={"enabled": enabled})
+    _write.record_action("evaluator_toggled", resource="evaluator", success=True)
+    if ctx.obj.json:
+        output.emit_json(result or {**row, "enabled": enabled})
+    else:
+        cloud_output.evaluator({**row, **(result or {}), "enabled": enabled})
+
+
+def enable(ctx: typer.Context, handle: str = typer.Argument(...)) -> None:
+    """Enable KEY[@VERSION]."""
+    toggle(ctx, handle, True)
+
+
+def disable(ctx: typer.Context, handle: str = typer.Argument(...)) -> None:
+    """Disable KEY[@VERSION]."""
+    toggle(ctx, handle, False)
+
+
+def label_def(ctx: typer.Context, handle: str = typer.Argument(...),
+              add: Optional[List[str]] = typer.Option(None, "--add"),
+              remove: Optional[List[str]] = typer.Option(None, "--remove")) -> None:
+    """Add/remove labels on KEY[@VERSION]."""
+    if not add and not remove:
+        spec.fail("Pass --add or --remove")
+    if set(add or []) & set(remove or []):
+        spec.fail("A label cannot be both added and removed")
+    cctx = require_auth(ctx.obj)
+    row = select(definitions(cctx), handle)
+    labels = list(dict.fromkeys([v for v in row.get("labels", []) if v not in (remove or [])] + (add or [])))
+    spec.validate_labels(labels)
+    _write.confirm(ctx.obj, f"Label {handle}", assume_yes=False)
+    data = api.cloud_request(cctx, "PATCH", f"{PATH}/{row['id']}", body={"labels": labels})
+    if ctx.obj.json:
+        output.emit_json(data or {**row, "labels": labels})
+    else:
+        cloud_output.evaluator({**row, **(data or {}), "labels": labels}, "Evaluator labels")
+
+
+def instances(ctx: typer.Context) -> None:
+    """List managed evaluator instances."""
+    _cloud.emit(ctx.obj, api.cloud_request(require_auth(ctx.obj), "GET", "/api/evaluator-instances"), "Instances")
+
+
+def schema(ctx: typer.Context) -> None:
+    """Schema profile. Needs evaluations:trigger and events:read."""
+    _cloud.emit(ctx.obj, api.cloud_request(require_auth(ctx.obj), "GET", "/api/evaluator-schema-profile"), "Schema profile")
+
+
+def jobs(ctx: typer.Context) -> None:
+    """List evaluation jobs."""
+    _cloud.emit(ctx.obj, api.cloud_request(require_auth(ctx.obj), "GET", "/api/evaluation-jobs"), "Evaluation jobs")
+
+
+def backfill(ctx: typer.Context, frm: str = typer.Option(..., "--from"),
+             to: str = typer.Option(..., "--to"), key: Optional[str] = typer.Option(None, "--key"),
+             dry_run: bool = typer.Option(False, "--dry-run")) -> None:
+    """Score stored sessions an evaluator has not scored yet (at most 90 days).
+
+    Covers code, LLM judge and Jev evaluators (every enabled one, or only --key).
+    Only missing (session, evaluator) pairs are queued; --dry-run counts them.
+    Needs evaluations:trigger.
+    """
+    try:
+        frm = dates._validate_iso(frm, "--from")
+        to = dates._validate_iso(to, "--to")
+        start = datetime.fromisoformat(frm.replace("Z", "+00:00"))
+        end = datetime.fromisoformat(to.replace("Z", "+00:00"))
+    except ValueError as exc:
+        spec.fail(str(exc))
+    if end <= start or end - start > timedelta(days=90):
+        spec.fail("Backfill needs an increasing range of at most 90 days")
+    if not dry_run:
+        output.warn("Judge and Jev evaluators in the backfill spend the org's model budget; --dry-run counts the work first.")
+    _write.confirm(ctx.obj, "Backfill evaluations", assume_yes=dry_run)
+    _cloud.emit(ctx.obj, api.cloud_request(require_auth(ctx.obj), "POST", "/api/evaluations/backfill",
+        body={"from": dates._iso(start), "to": dates._iso(end), "eval_key": key, "dry_run": dry_run}), "Evaluation backfill")
+
+
+def register(app: typer.Typer) -> None:
+    group = typer.Typer(no_args_is_help=True, rich_markup_mode="markdown", help="Author, test and deploy code, LLM-judge and Jev evaluators.", context_settings={"help_option_names": ["-h", "--help"]})
+    for name, fn in (
+        ("list", list_defs), ("show", show), ("create", create), ("deploy", deploy),
+        ("draft", draft), ("dry-run", dry_run), ("enable", enable), ("disable", disable),
+        ("label", label_def), ("instances", instances), ("schema", schema), ("jobs", jobs),
+        ("backfill", backfill),
+    ):
+        group.command(name, epilog=GLOBALS_EPILOG)(fn)
+    app.add_typer(group, name="evaluators")

package/.next/standalone/fp-cloud-cli/fp_cli/commands/events_cmds.py

@@ -70,6 +70,8 @@ def events(
     * `fp --json events --session-id run-001 --all` — a run's full timeline: event types + summaries (light, fast)
     * `fp --json events --full --session-id run-001 --all | jq '.events[].payload'` — that run's raw payloads (full feed, bounded to one run)
     """
+    if ctx.invoked_subcommand:
+        return
     state: AppState = ctx.obj
     cctx = require_auth(state)
     frm, to = resolve_dates(since, ts_from, ts_to)
@@ -155,5 +157,16 @@ def events(
         output.recheck_filters_hint(active_filters)
 
 
+def latency(ctx: typer.Context, since: Optional[str] = typer.Option(None, "--since")) -> None:
+    """Aggregate event latency."""
+    from . import _cloud
+    frm, to = resolve_dates(since, None, None)
+    _cloud.emit(ctx.obj, api.cloud_request(require_auth(ctx.obj), "GET", "/api/events/latency_aggregate",
+        params={"ts_from": frm, "ts_to": to}), "Event latency")
+
+
 def register(app: typer.Typer) -> None:
-    app.command("events", epilog=GLOBALS_EPILOG)(events)
+    group = typer.Typer(invoke_without_command=True, rich_markup_mode="markdown", context_settings={"help_option_names": ["-h", "--help"]})
+    group.callback(epilog=GLOBALS_EPILOG)(events)
+    group.command("latency", epilog=GLOBALS_EPILOG)(latency)
+    app.add_typer(group, name="events")

package/.next/standalone/fp-cloud-cli/fp_cli/commands/incidents_cmds.py

@@ -14,7 +14,8 @@ import typer
 
 from .. import client as api
 from .. import output
-from .._context import GLOBALS_EPILOG, AppState, require_auth, validate_limit
+from .. import _click_compat as click
+from .._context import GLOBALS_EPILOG, AppState, require_auth, resolve_dates, validate_limit
 from ..errors import ApiError, ForbiddenError, NotFoundError
 from . import _write
 
@@ -68,12 +69,40 @@ def _fail(state: AppState, exc: Exception, *, incident_id: str = "") -> None:
     raise exc
 
 
+def _issue_filters(
+    agent_id, env, machine, source, audit_id, assignee, archived, search, since,
+) -> dict:
+    if archived is not None and archived not in ("false", "true", "only", "all"):
+        raise click.UsageError("--archived must be false, only or all")
+    if source is not None:
+        bad = [s for s in source.split(",") if s.strip() and s.strip() not in ("alert", "audit", "manual")]
+        if bad:
+            raise click.UsageError("--source must be alert, audit or manual (CSV)")
+    opened_from = None
+    if since is not None:
+        opened_from, _ = resolve_dates(since, None, None)
+    return {
+        "agent": agent_id, "environment": env, "machine": machine, "source": source,
+        "audit_id": audit_id, "assignee": assignee, "archived": archived, "q": search,
+        "opened_from": opened_from,
+    }
+
+
 def incidents_list(
     ctx: typer.Context,
     state_filter: Optional[str] = typer.Option(None, "--state", help="Filter by state(s): firing, acknowledged, resolved, closed (CSV)."),
     alert_id: Optional[str] = typer.Option(None, "--alert-id", help="Only incidents for this alert."),
     limit: int = typer.Option(50, "--limit", "-n", help="Max incidents to return."),
     show_id: bool = typer.Option(False, "--show-id", help="Show the full incident id instead of the short form (always full in --json)."),
+    agent_id: Optional[str] = typer.Option(None, "--agent-id", help="Only issues touching these agent ids (CSV)."),
+    env: Optional[str] = typer.Option(None, "--env", help="Only these environments (CSV)."),
+    machine: Optional[str] = typer.Option(None, "--machine", help="Only these machine ids (CSV)."),
+    source: Optional[str] = typer.Option(None, "--source", help="alert, audit and/or manual (CSV)."),
+    audit_id: Optional[str] = typer.Option(None, "--audit-id", help="Only issues from these audits (CSV of ids)."),
+    assignee: Optional[str] = typer.Option(None, "--assignee", help="Emails, `me` or `unassigned` (CSV)."),
+    archived: Optional[str] = typer.Option(None, "--archived", help="false (default), only or all."),
+    search: Optional[str] = typer.Option(None, "--search", help="Case-insensitive text in the title or summary."),
+    since: Optional[str] = typer.Option(None, "--since", help="Opened within: 15m 1h 6h 24h 7d."),
 ) -> None:
     """List incidents in a boxed table (newest-opened first).
 
@@ -94,7 +123,8 @@ def incidents_list(
     _validate_states(state_filter)
     validate_limit(limit)
     cctx = require_auth(state)
-    incidents = api.list_incidents(cctx, state=state_filter, alert_id=alert_id, limit=limit)
+    extra = _issue_filters(agent_id, env, machine, source, audit_id, assignee, archived, search, since)
+    incidents = api.list_incidents(cctx, state=state_filter, alert_id=alert_id, limit=limit, filters=extra)
     if state.json:
         output.emit_json({"issues": incidents})
         return
@@ -105,6 +135,16 @@ def incidents_list(
 def incidents_count(
     ctx: typer.Context,
     state_filter: Optional[str] = typer.Option(None, "--state", help="Filter by state(s) (CSV)."),
+    alert_id: Optional[str] = typer.Option(None, "--alert-id", help="Only issues from these alerts (CSV of ids)."),
+    agent_id: Optional[str] = typer.Option(None, "--agent-id", help="Only issues touching these agent ids (CSV)."),
+    env: Optional[str] = typer.Option(None, "--env", help="Only these environments (CSV)."),
+    machine: Optional[str] = typer.Option(None, "--machine", help="Only these machine ids (CSV)."),
+    source: Optional[str] = typer.Option(None, "--source", help="alert, audit and/or manual (CSV)."),
+    audit_id: Optional[str] = typer.Option(None, "--audit-id", help="Only issues from these audits (CSV of ids)."),
+    assignee: Optional[str] = typer.Option(None, "--assignee", help="Emails, `me` or `unassigned` (CSV)."),
+    archived: Optional[str] = typer.Option(None, "--archived", help="false (default), only or all."),
+    search: Optional[str] = typer.Option(None, "--search", help="Case-insensitive text in the title or summary."),
+    since: Optional[str] = typer.Option(None, "--since", help="Opened within: 15m 1h 6h 24h 7d."),
 ) -> None:
     """Count incidents (optionally by state) as a compact stat card.
 
@@ -118,7 +158,10 @@ def incidents_count(
     state: AppState = ctx.obj
     _validate_states(state_filter)
     cctx = require_auth(state)
-    count = api.count_incidents(cctx, state=state_filter)
+    extra = _issue_filters(agent_id, env, machine, source, audit_id, assignee, archived, search, since)
+    if alert_id:
+        extra["alert_id"] = alert_id
+    count = api.count_incidents(cctx, state=state_filter, filters=extra)
     if state.json:
         output.emit_json({"count": count})
     else:
@@ -261,10 +304,11 @@ def incidents_close(
 
     For an issue that came from an audit, this also marks the finding `dismissed`. It does
     NOT write the finding's org-wide suppression: closing one issue never hides that pattern
-    in your other audits. Use `fp audits finding-status <id> --action dismiss` for that.
+    in your other audits. Use `fp audits dismiss <finding-id>` for that.
 
-    Needs `issues:close`. Exits 9 (conflict) if the issue already ended — an issue ends once,
-    and closing must not overwrite the record that someone believed they had fixed it.
+    Needs `issues:close`. Exits 1 (the server answers 409 conflict) if the issue already
+    ended — an issue ends once, and closing must not overwrite the record that someone
+    believed they had fixed it.
 
     With `--json`: `{"closed": true, "id": "<id>"}` (or `{cancelled: true}` on a declined
     prompt).
@@ -666,6 +710,37 @@ def incidents_open(
     output.render_incident_opened(summary=summary, severity=sev, state=st, title=hero)
 
 
+def incidents_facets(
+    ctx: typer.Context,
+    state_filter: Optional[str] = typer.Option(None, "--state", help="Filter by state(s) (CSV)."),
+    alert_id: Optional[str] = typer.Option(None, "--alert-id", help="Only issues from these alerts (CSV of ids)."),
+    agent_id: Optional[str] = typer.Option(None, "--agent-id", help="Only issues touching these agent ids (CSV)."),
+    env: Optional[str] = typer.Option(None, "--env", help="Only these environments (CSV)."),
+    machine: Optional[str] = typer.Option(None, "--machine", help="Only these machine ids (CSV)."),
+    source: Optional[str] = typer.Option(None, "--source", help="alert, audit and/or manual (CSV)."),
+    audit_id: Optional[str] = typer.Option(None, "--audit-id", help="Only issues from these audits (CSV of ids)."),
+    assignee: Optional[str] = typer.Option(None, "--assignee", help="Emails, `me` or `unassigned` (CSV)."),
+    archived: Optional[str] = typer.Option(None, "--archived", help="false (default), only or all."),
+    search: Optional[str] = typer.Option(None, "--search", help="Case-insensitive text in the title or summary."),
+    since: Optional[str] = typer.Option(None, "--since", help="Opened within: 15m 1h 6h 24h 7d."),
+) -> None:
+    """The values the issue filters can take right now (agents, environments, machines,
+    sources, assignees), with counts — the dashboard inbox's filter bar. Needs `issues:read`.
+
+    Example:
+
+    * `fp issues facets --state firing,acknowledged`
+    """
+    from . import _cloud
+    state: AppState = ctx.obj
+    _validate_states(state_filter)
+    extra = _issue_filters(agent_id, env, machine, source, audit_id, assignee, archived, search, since)
+    params = {"state": state_filter, "alert_id": alert_id, **extra}
+    data = api.cloud_request(require_auth(state), "GET", "/api/issues/facets",
+                             params={k: v for k, v in params.items() if v is not None})
+    _cloud.emit(state, data, "Issue facets")
+
+
 def register(app: typer.Typer) -> None:
     inc = typer.Typer(
         no_args_is_help=True,
@@ -675,6 +750,7 @@ def register(app: typer.Typer) -> None:
     )
     inc.command("list", epilog=GLOBALS_EPILOG)(incidents_list)
     inc.command("count", epilog=GLOBALS_EPILOG)(incidents_count)
+    inc.command("facets", epilog=GLOBALS_EPILOG)(incidents_facets)
     inc.command("show", epilog=GLOBALS_EPILOG)(incidents_show)
     inc.command("ack", epilog=GLOBALS_EPILOG)(incidents_ack)
     inc.command("assign", epilog=GLOBALS_EPILOG)(incidents_assign)

package/.next/standalone/fp-cloud-cli/fp_cli/commands/list_cmds.py

@@ -24,6 +24,9 @@ _LIST_KINDS = {
     "hooks": ("hook_names", "events:read", "Hook names seen across events.", "seen across events"),
     "tools": ("tool_names", "events:read", "Tool names seen across events.", "seen across events"),
     "error_types": ("error_types", "events:read", "Error types seen across events.", "seen across events"),
+    "trigger_events": ("trigger_events", "events:read", "Hook trigger events seen across events.", "seen across events"),
+    "eval_envs": ("eval_environments", "evaluations:read", "Environments that have evaluation results.", "with evaluation results"),
+    "metric_keys": ("metric_keys", "evaluations:read", "Evaluation metric keys (metrics, not scores).", "evaluation metric keys"),
 }
 
 def _make_list_cmd(name: str, facet_key: str, perm: str, summary: str, description: str):
@@ -56,7 +59,7 @@ def register(app: typer.Typer) -> None:
         context_settings={"help_option_names": ["-h", "--help"]},
         help="List the distinct values behind the dashboard's filter dropdowns. Subcommands: "
         "**envs**, **agents**, **event_types**, **score_filters**, **models**, **hooks**, "
-        "**tools**, **error_types**.",
+        "**tools**, **error_types**, **trigger_events**, **eval_envs**, **metric_keys**.",
     )
     for name, (facet_key, perm, summary, description) in _LIST_KINDS.items():
         list_app.command(name, epilog=GLOBALS_EPILOG)(_make_list_cmd(name, facet_key, perm, summary, description))

package/.next/standalone/fp-cloud-cli/fp_cli/commands/orgs_cmds.py

@@ -316,4 +316,21 @@ def register(app: typer.Typer) -> None:
     orgs_app.command("switch", epilog=GLOBALS_EPILOG)(org_switch)
     orgs_app.command("current", epilog=GLOBALS_EPILOG)(org_current)
     orgs_app.command("perms", epilog=GLOBALS_EPILOG)(org_perms)
+    orgs_app.command("create", epilog=GLOBALS_EPILOG)(org_create)
     app.add_typer(orgs_app, name="orgs")
+
+
+def org_create(ctx: typer.Context, name: str = typer.Option(..., "--name"),
+               slug: Optional[str] = typer.Option(None, "--slug")) -> None:
+    """Create an organization (session only; does not switch the saved org)."""
+    from . import _cloud
+    from .account_cmds import request
+    from .. import _click_compat as click
+    _cloud.session_only(ctx.obj, "orgs create")
+    if not name.strip():
+        raise click.UsageError("--name cannot be empty")
+    import re
+    slug = slug or re.sub(r"[^a-z0-9]+", "-", name.lower()).strip("-")
+    if not re.fullmatch(r"[a-z0-9]+(?:-[a-z0-9]+)*", slug):
+        raise click.UsageError("--slug must be lowercase letters/digits separated by hyphens")
+    request(ctx, "POST", "/api/orgs", {"name": name, "slug": slug})