failproofai 1.0.12-beta.1 → 1.0.12

package/.next/standalone/app/actions/update-jev-config.ts

@@ -10,7 +10,7 @@
  * `writeJsonAtomically` the CLI's `jev setup` calls, with the same
  * `{ mode: 0o600, dirMode: 0o700 }`, after the same `validateJevConfig` the
  * loader itself runs. Nothing here re-states a rule that lives in
- * `jev-config.ts`: not the URL scheme, not "plain http only in observe mode",
+ * `jev-config.ts`: not the URL scheme, not "plain http to loopback only",
  * not "cloudflare needs an account id". A second copy of those rules is how the
  * dashboard ends up writing a file the hooks then refuse — the exact failure
  * mode this module exists to avoid.
@@ -101,6 +101,7 @@ import { chmodSync, existsSync, statSync, unlinkSync } from "node:fs";
 import { dirname } from "node:path";
 import { headers } from "next/headers";
 import { writeJsonAtomically } from "@/lib/atomic-write";
+import { CLOUD_MANAGED_JEV_MODE, cloudManagesJevMode } from "@/src/hooks/fp-config";
 import {
   hostnameFromHostHeader,
   isLoopbackHostname,
@@ -475,8 +476,8 @@ export async function saveJevConfigAction(input: JevConfigInput): Promise<JevWri
  * endpoint and the provider are exactly what they were. It is written the way
  * every other write here is (`writeJsonAtomically`, 0600, owner-only
  * directory), after the loader's own validator has checked the result with
- * stand-ins in the key slots, so `enforce` is refused where the loader would
- * refuse it (plain http to localhost) rather than written and then ignored.
+ * stand-ins in the key slots, so a mode the loader would refuse is refused
+ * here rather than written and then ignored.
  *
  * Refused for a file the loader calls too open: someone else may have written
  * the endpoint it names, and re-saving it at 0600 would start trusting that
@@ -489,6 +490,7 @@ export async function setJevModeAction(requested: string): Promise<JevWriteResul
   if (mode === null) {
     return { ok: false, problem: 'mode must be "off", "observe" or "enforce".' };
   }
+  if (cloudManagesJevMode()) return { ok: false, problem: CLOUD_MANAGED_JEV_MODE };
 
   const existingFile = readJevConfigFileForUpdate();
   if (existingFile === null) {

package/.next/standalone/app/settings/jev-panel.tsx

@@ -45,7 +45,7 @@
  * and a save leaves whatever is stored alone.
  *
  * Client-side validation here is a convenience only. Every rule — the URL
- * scheme, plain http being refused outside observe mode, cloudflare's account id
+ * scheme (plain http to loopback only), cloudflare's account id
  * — is enforced server-side by the same `validateJevConfig` the loader runs.
  *
  * ## FailproofAI Cloud

package/.next/standalone/fp-cloud-cli/CHANGELOG.md

@@ -4,30 +4,30 @@
 
 ### Added
 
+- Hosted evaluator authoring for code, LLM judge and Jev: create/deploy, AI draft specs, dry-run, labels, toggles, instances, schema, jobs and backfill.
+- Dashboards and tiles; session overview/export/history/re-evaluate; latency and sentiment; query composition/export; alert recipients/issues and agent contracts.
+- Policy author/backtest/candidacy, assistant artifact tables, fail-closed interactive write approvals, and agent-curated home briefs.
+- Organization creation, permission-set roles, member deletion/defaults, usage windows, offers/trial, model context windows, OTEL and Slack controls.
+- `fp policies backtest` covers every policy kind: a published `jev`/`both` policy, or `--semantic checks.json` (with or without a source file), runs the dashboard's every-kind backtest — a free estimate by default, `--run [--sample N]` to spend Jev allowance. New `--kind --semantic --hours --agent-id --env --run --sample`; a regex file or id still uses the module backtest.
+- `fp policies author --kind regex|jev|both` asks the assistant for Jev checks or a hybrid policy.
+- `fp issues list|count` take the inbox filters (`--agent-id --env --machine --source --audit-id --assignee --archived --search --since`, and `count --alert-id`); new `fp issues facets` lists the values they can take.
+- `fp sessions import FILE...` imports Claude Code transcripts, Codex rollouts or event JSONL (`--format --env --dry-run`), as the dashboard's import dialog does.
+- `fp list trigger_events`, `fp list eval_envs` and `fp list metric_keys`.
 - `fp fleet deploy --target POLICY=INTEGRATION[/agt_ID]` scopes a machine's assignment; repeat the flag to include multiple alternatives. `--all-agents POLICY` explicitly clears its scope. Deploy plans, show, history, rollback and JSON readback retain assignment targets.
 - `fp policies publish <id> --kind regex|jev|both [--source f.mjs] [--semantic f.json]`: a
   FailproofAI Cloud policy can carry Jev checks — `jev` (declarations only, no JavaScript) or
   `both` (JavaScript reviewable by exactly its own checks; the server derives the authority).
   The kind follows from what is given when `--kind` is omitted, and a kind that contradicts its
   inputs is exit 2 before anything is sent. The positional source still works.
-- `fp fleet deploy … --jev-mode off|observe|enforce|local` sets the machine's Jev mode from
-  FailproofAI Cloud (overriding the machine's own; `local` hands it back). Omitted, the request
-  carries no `jevMode` and the mode is unchanged; `--jev-mode` alone is a valid deploy. An
-  `observe` effect on a `jev` policy is refused (exit 2) — Jev is observed through the mode.
-- `fp fleet jev-mode <machine…|--all> <mode>` changes only the Jev mode, through its own
-  route (`PUT /api/enforcement/deployments/{id}/jev-mode`), never the policy set — and
-  `fp fleet deploy --jev-mode` with no change to the set now uses it too. A deploy is a full
-  replace of the set the CLI read, which leaves out disabled policies' assignments, so a
-  mode change sent as a deploy deleted them and reverted concurrent edits.
-- `fp fleet rollback` says which Jev mode it restores (`jev mode <now> → <then>`, and
-  `jevModeBefore` in `--json`); `fp fleet history` shows each generation's Jev mode and marks a
-  mode-only generation as a change.
-- The `--jev-mode` and `fleet jev-mode` help says what `enforce` does: Jev's checks block calls
-  as well as clearing what they review, machine-wide (installed packs' checks included); an
-  `observe` effect on a `both` policy withholds its Jev checks.
-- The `fleet deploy`, `fleet jev-mode` and `policies publish` help says Jev checks run on
-  FailproofAI Cloud and nothing is installed on the machine, and `fleet show`/`fleet jev-mode`
-  name the reasons a machine with a mode cannot ask (`jev_unconfigured`, `transcripts_disabled`)
+- No Jev mode is set by hand: FailproofAI Cloud derives a machine's Jev mode from what is
+  deployed to it (on while a `jev`/`both` policy is, off otherwise), so `fp fleet deploy` has
+  no `--jev-mode` and there is no `fp fleet jev-mode`. An `observe` effect applies to `jev`
+  and `both` policies like any other: their checks are recorded and decide nothing.
+- `fp fleet rollback` says when the Jev mode differs (`jev mode <now> → <then>`, and
+  `jevModeBefore` in `--json`); `fp fleet history` shows each generation's Jev mode.
+- The `fleet deploy` and `policies publish` help says Jev checks run on
+  FailproofAI Cloud and nothing is installed on the machine, and `fleet show`
+  names the reasons a machine with a mode cannot ask (`jev_unconfigured`, `transcripts_disabled`)
   and the `jev_budget` report.
 - `fp policies list/show` print each version's kind and Jev check names (`show` also the
   declarations); `fp fleet list/show` print the machine's Jev mode and the policy errors it
@@ -50,6 +50,9 @@
 
 ### Changed
 
+- New Cloud API families work in key mode; unsupported session-only commands deny before HTTP and unknown-route 404s exit 6 with an availability message.
+- Shared screens use the mint/pink wordmark and grey labels, without boxes/rules or shortened ids; existing JSON contracts are unchanged.
+- The agent skill, command reference, README and Cloud CLI docs include end-to-end evaluator workflows. No version bump or release is performed.
 - `fp policies publish`'s refusal of Jev fields in the JavaScript now says where Jev checks go:
   the policy's own declarations, `--kind both --semantic checks.json`.
 - `fp guardrails summary` shows how many machines run Jev in each mode, reading the summary's
@@ -65,6 +68,13 @@
 
 ### Fixed
 
+- Remove the unused colour-keyed permission rank, document full-id rendering, emit UTC `Z` timestamps for deploy backfills, and report approval-answer failures without a traceback while restoring the terminal (#885).
+- Reject natural-language binary judge steps, distinguish unknown routes from missing records on 404, normalize standalone backfill ranges to UTC `Z`, expose all subcommands in help, and render artifact headers before titles with consistent session-only wording (#885).
+- New Cloud commands use deliberate human summaries/tables and actionable empty states at 80/140 columns without raw response blobs. Evaluator previews display before writes and require explicit `--force` after failures; plan/claim deny key mode, deploy specs can omit the positional key, and older settings-route 404s degrade clearly. Existing JSON payloads stay unchanged (#885).
+- `fp evaluators deploy` keeps the previous version's condition and labels unless `--condition`/`--label` change them, so a new version no longer silently widens to every session (#885).
+- Evaluation backfill covers code, LLM judge and Jev evaluators, as FailproofAI Cloud now does: `fp evaluators deploy --backfill` no longer refuses judge/Jev, and both commands warn that judge/Jev backfills spend model budget instead of saying they are skipped.
+- `fp usage windows` says it needs an API key (the dashboard has no session route for it) instead of reporting the route as unavailable.
+- `fp issues close --help` no longer claims exit 9 for an issue that already ended: it is exit 1, as it always was, and it names `fp audits dismiss` rather than a command that does not exist.
 - `fp fleet history` keeps `gen`, `when` and `jev` visible at 80 columns (rollback needs `gen`): the policy list is shortened, then dropped, and `change` wraps first.
 - `fp fleet show` and `fp fleet history` name an exact-profile target by the profile label the machine reports (`claude/.claude-work`, with the `agt_…` id kept in `show`), falling back to the bare id.
 

package/.next/standalone/fp-cloud-cli/README.md

@@ -144,19 +144,17 @@ earlier version is), `jev` (Jev checks only, from a JSON file of declarations
 each exactly a failproofai pack manifest's semantic entry), or `both`
 (JavaScript whose verdict its own Jev checks may clear, deployed as one).
 Jev checks run on FailproofAI Cloud; nothing is installed on the machine.
-`--jev-mode` sets a machine's Jev mode from FailproofAI Cloud, overriding the
-machine's own; omit it and the mode is left alone. Under `observe` or `enforce`
-the machine sends each checked tool call to FailproofAI Cloud, which asks the
-checks deployed to it. The mode is the machine's, so it covers installed packs'
-checks too, and in `enforce` Jev's checks **block** calls as well as clearing
-the regex verdicts they review. `fp fleet jev-mode` changes only the mode, and
-never rewrites a machine's policy set.
+While a `jev` or `both` policy is deployed to a machine, FailproofAI Cloud
+turns Jev on there and the machine sends each checked tool call to it; with
+none, Jev is off. There is no Jev mode to set: trial a policy with `:observe`
+(its checks are recorded, never applied), then deploy it as `enforce`, where
+Jev's checks **block** calls as well as clearing the regex verdicts they
+review.
 
 ```bash
 fp policies publish prod-db-intent --kind jev --semantic ./checks.json
 fp policies publish no-prod-db --kind both --source ./rule.mjs --semantic ./checks.json
-fp fleet deploy ci-runner-01 --add prod-db-intent --jev-mode observe
-fp fleet jev-mode --all enforce   # the mode alone, every machine with a deployment
+fp fleet deploy ci-runner-01 --add prod-db-intent:observe
 fp fleet show ci-runner-01    # its Jev mode, and any policy errors it reported
 ```
 
@@ -213,6 +211,126 @@ These commands are **session-only** (`fp login`). They are absent from the
 versioned API an API key authenticates against, so `--api-key` exits 2 with the
 reason rather than failing at the request.
 
+## Author an evaluator
+
+`fp evaluators` authors hosted definitions; `fp evals` continues to read results.
+Creation enables a definition by default. Check `fp evaluators instances` first:
+Cloud needs an enabled managed instance. Writes need `evaluations:trigger`;
+session dry-runs and `evaluators schema` also need `events:read`.
+
+Write `error_free.py` as a restricted Python expression (no imports or functions):
+
+```python
+EvalResult(score=Score(1.0 if session.count("error") == 0 else 0.0), reasoning="Error-free session")
+```
+
+```bash
+fp --json evaluators create --key error_free --name "Error free" --code error_free.py \
+  --result-kind score --dry-run-on run-001
+fp --json evaluators dry-run --code error_free.py --session run-001
+fp --json evaluators deploy error_free --code error_free.py --version 2 --dry-run-on run-001
+fp --json evaluators list --key error_free --all
+```
+
+LLM judge (Cloud selects the model; judge dry-runs are refused):
+
+```bash
+fp --json evaluators create --key relevance --name Relevance --judge \
+  --criteria "Grade how directly the answer addresses the request" --threshold 0.7
+fp --json evaluators deploy relevance --version 2 --judge \
+  --criteria "Grade relevance and completeness" --threshold 0.8
+fp --json sessions re-evaluate run-001
+fp --json sessions history run-001
+```
+
+Jev binary or graded rubric:
+
+```bash
+fp --json evaluators create --key resolved --name Resolved --jev noul \
+  --instructions "Was the request resolved?" --true "Resolved" --false "Unresolved"
+fp --json evaluators dry-run --jev noul --instructions "Was the request resolved?" \
+  --true "Resolved" --false "Unresolved" --session run-001
+fp --json evaluators deploy resolved --version 2 --jev score \
+  --instructions "Grade resolution quality" \
+  --level "Unresolved" --level "Partly resolved" --level "Fully resolved"
+```
+
+Jev dry-runs spend model budget. Rubrics need 3–5 distinct levels, worst first.
+Thresholds are 0–1; judge/Jev always return a score. `deploy` creates the new
+immutable version before disabling older enabled versions. If disabling fails,
+the CLI names the published id; inspect `list --all` and recover explicitly.
+`--backfill 7d` works for every kind (judge/Jev spend model budget); `evaluators backfill --from T --to T` accepts ≤90 days and `--dry-run` counts the work first.
+
+Human output shows each `--dry-run-on` session's outcome before creation or
+deployment. Any failed run stops the write; after reviewing the failure, an
+explicit `--force` overrides this guard. Existing `--json` response shapes stay
+unchanged.
+
+AI drafting is session-only and writes an API spec that create/deploy accept:
+
+```bash
+fp login
+fp --json evaluators draft "Measure error-free sessions with code" --kind code --out evaluator.json
+# Review the spec, especially eval_key and evaluator_source, before enabling it.
+fp --json evaluators create --file evaluator.json --dry-run-on run-001
+fp --json evaluators dry-run --file evaluator.json
+```
+
+Spec fields: `eval_key`, `display_name`, `eval_version`, `result_kind`, `labels`,
+`condition_source`, `evaluator_source`, `timeout_seconds`. Judge/Jev sources are
+JSON envelope **strings**, not nested objects. `--file` is exclusive with discrete
+spec flags; deploy may override a file's version. Files written by the CLI are
+created exclusively, so existing files are never silently replaced.
+See [the command reference](skill/references/commands.md#author-an-evaluator).
+
+## Dashboard parity
+
+```text
+fp dashboards list|show|create|update|delete
+fp dashboards tiles <id> list|add|update|remove|layout
+fp sessions show|export|history|re-evaluate <id>
+fp events latency
+fp sentiment summary|messages|sessions|session <id>
+fp query compose "question" [--run]                  # session
+fp query export [<id>] [--sql SQL] --format csv|json [--out FILE]
+fp alerts recipients
+fp alerts issues <id> [--create --title ... --summary ...]
+fp audits contracts list|show|create|update
+fp policies author "intent"|backtest <file-or-id>|candidacy --issue <id>  # session
+fp agent brief [--range 24h|7d]                      # session
+fp orgs create --name N [--slug S]                   # session
+fp roles list|show|create|update|delete
+fp users delete <id>
+fp users defaults
+fp usage windows
+fp plan                                           # session
+fp plan claim <offer>                             # session
+fp settings context-windows list|set <model> <tokens>|delete <model>
+fp otel status|enable|disable
+fp slack status|disconnect                          # session
+```
+
+`fp agent ask` runs artifact SQL through the permission-gated query runner and
+renders tables/stats. It prompts before approving a write on a terminal.
+Scripts and `--json` **decline** unless you explicitly pass `--approve`.
+Normal create/update commands keep their existing non-interactive confirmation
+behavior; assistant approvals are a separate safety boundary.
+
+All new commands accept the global `--json` before the command. Session-only
+commands fail before HTTP in API-key mode, with exit 2 and a `fp login` hint.
+Unknown-route 404s exit 6 with “not available on this FailproofAI Cloud yet”.
+Missing-record 404s also exit 6 but report that the record was not found.
+`plan` and `plan claim` require a signed-in person and deny API-key mode before
+HTTP with the standard `fp login` message.
+Human screens use the mint/pink wordmark, grey labels, open tables and full ids;
+existing JSON shapes are unchanged.
+
+Human commands show summaries, tables and wrapped text rather than response
+objects. Use global `--json` for raw API results. For `sessions export` or
+`query export --format json`, use `--out FILE` to save the JSON; without either
+option, the terminal shows a readable session or query view. CSV exports remain
+plain CSV for piping.
+
 ## Configuration
 
 | Setting | Flag | Env var | Default |

package/.next/standalone/fp-cloud-cli/fp_cli/app.py

@@ -17,12 +17,15 @@ from . import orgs as orgsmod
 from ._context import AppState
 from ._version import __version__
 from .commands import (
+    account_cmds,
     agent_cmds,
     alerts_cmds,
     audits_cmds,
     auth_cmds,
+    dashboards_cmds,
     errors_cmds,
     evals_cmds,
+    evaluators_cmds,
     events_cmds,
     fleet_cmds,
     guardrails_cmds,
@@ -33,6 +36,7 @@ from .commands import (
     policies_cmds,
     queries_cmds,
     sessions_cmds,
+    sentiment_cmds,
     settings_cmds,
     usage_cmds,
     users_cmds,
@@ -128,6 +132,12 @@ app = typer.Typer(
 # function Typer calls for this; falls back silently if its internals change.
 try:  # pragma: no cover - exercised via real command output, not unit assertions
     import typer.rich_utils as _rich_utils
+    # Keep generated subcommand help on the same shared screen primitives.
+    _rich_utils.Panel = output.Panel
+    _rich_utils.Table = output.Table
+    _rich_utils.STYLE_OPTION = output.theme.PINK
+    _rich_utils.STYLE_COMMANDS_TABLE_FIRST_COLUMN = output.theme.PINK
+    _rich_utils.STYLE_OPTION_ENVVAR = output.theme.LABEL
 
     import re as _re
 
@@ -327,7 +337,7 @@ def main(
             "must be a positive number of seconds.", param_hint="--timeout"
         )
     cfg = cfgmod.load_config()
-    output.configure(no_color=no_color, quiet=quiet, json=json_output)
+    output.configure(no_color=no_color, quiet=quiet, json=json_output, context=ctx.invoked_subcommand or "Cloud")
     # Resolve the credential BEFORE telemetry starts: the distinct-id depends on it.
     # `get_parameter_source` is what separates a flag from its env var — the values
     # alone cannot, and the precedence rules differ between the two (see resolve_auth).
@@ -452,6 +462,9 @@ orgs_cmds.register(app)
 events_cmds.register(app)
 sessions_cmds.register(app)
 evals_cmds.register(app)
+evaluators_cmds.register(app)
+dashboards_cmds.register(app)
+sentiment_cmds.register(app)
 errors_cmds.register(app)
 usage_cmds.register(app)
 list_cmds.register(app)
@@ -466,6 +479,7 @@ agent_cmds.register(app)
 policies_cmds.register(app)
 fleet_cmds.register(app)
 guardrails_cmds.register(app)
+account_cmds.register(app)
 
 
 def _elapsed_ms(start: float) -> int: