failproofai 1.0.6-beta.1 → 1.0.7-beta.0

package/.next/standalone/fp-cloud-cli/tests/test_alerting.py

@@ -642,3 +642,187 @@ def test_issues_show_malformed_id_is_not_found(logged_in, runner):
     result = runner.invoke(app, ["--json", "issues", "show", "not-a-uuid"])
     assert result.exit_code == 6
     assert "no issue not-a-uuid" in json.loads(result.stdout)["error"]
+
+# --- close / archive / clear ------------------------------------------------
+#
+# The issue lifecycle grew a second terminal state (`closed`) and an orthogonal
+# archived flag. The CLI restates the server's contract by hand — the state
+# allowlist, the paths, the response keys — with nothing checking the two agree,
+# so these pin the CLI half.
+
+
+@respx.mock
+def test_issues_close_posts_to_close_not_resolve(logged_in, runner):
+    # The whole point of `close` is that it is NOT `resolve`: a resolved issue
+    # reopens when its audit finding recurs and a closed one does not, so a
+    # close that quietly hit /resolve would silently give the opposite
+    # behaviour with an identical success line.
+    resolve = respx.post(f"{BASE}/api/issues/i1/resolve").mock(return_value=httpx.Response(200, json={}))
+    close = respx.post(f"{BASE}/api/issues/i1/close").mock(
+        return_value=httpx.Response(200, json={"id": "i1", "state": "closed", "closed_at": "t"}))
+    result = runner.invoke(app, ["--json", "issues", "close", "i1", "--yes"])
+    assert result.exit_code == 0, result.output
+    assert json.loads(result.stdout) == {"closed": True, "id": "i1"}
+    assert close.called
+    assert not resolve.called
+
+
+@respx.mock
+def test_issues_close_human_line(logged_in, runner):
+    respx.post(f"{BASE}/api/issues/i1/close").mock(return_value=httpx.Response(200, json={}))
+    result = runner.invoke(app, ["issues", "close", "i1", "--yes"])
+    assert result.exit_code == 0, result.output
+    assert "closed issue" in result.stderr
+
+
+@respx.mock
+def test_issues_close_declined_emits_the_json_cancel_envelope(logged_in, runner, monkeypatch):
+    monkeypatch.setattr(_write, "should_prompt", lambda *a, **k: True)
+    monkeypatch.setattr(output, "confirm_incident_close", lambda *a, **k: False)
+    respx.get(f"{BASE}/api/issues/i1").mock(
+        return_value=httpx.Response(200, json={"id": "i1", "alert_name": "p95"}))
+    close = respx.post(f"{BASE}/api/issues/i1/close").mock(return_value=httpx.Response(200, json={}))
+    result = runner.invoke(app, ["--json", "issues", "close", "i1"])
+    assert result.exit_code == 0, result.output
+    assert json.loads(result.stdout) == {"cancelled": True}
+    assert not close.called
+
+
+@respx.mock
+def test_issues_archive_and_unarchive_hit_their_own_verbs(logged_in, runner):
+    arch = respx.post(f"{BASE}/api/issues/i1/archive").mock(return_value=httpx.Response(200, json={}))
+    unarch = respx.post(f"{BASE}/api/issues/i1/unarchive").mock(return_value=httpx.Response(200, json={}))
+
+    result = runner.invoke(app, ["--json", "issues", "archive", "i1"])
+    assert result.exit_code == 0, result.output
+    assert json.loads(result.stdout) == {"archived": True, "id": "i1"}
+    assert arch.called and not unarch.called
+
+    result = runner.invoke(app, ["--json", "issues", "unarchive", "i1"])
+    assert result.exit_code == 0, result.output
+    assert json.loads(result.stdout) == {"archived": False, "id": "i1"}
+    assert unarch.called
+
+
+@respx.mock
+def test_issues_archive_does_not_prompt(logged_in, runner, monkeypatch):
+    # Archiving destroys nothing and `unarchive` undoes it, so it must not ask —
+    # a prompt here would be the only blocking one in a cleanup loop.
+    monkeypatch.setattr(_write, "should_prompt", lambda *a, **k: True)
+    route = respx.post(f"{BASE}/api/issues/i1/archive").mock(return_value=httpx.Response(200, json={}))
+    result = runner.invoke(app, ["--json", "issues", "archive", "i1"])
+    assert result.exit_code == 0, result.output
+    assert route.called
+
+
+def test_issues_clear_requires_exactly_one_scope(logged_in, runner):
+    # The three scopes have very different blast radii, so there is deliberately
+    # no default and no way to pass two.
+    none_given = runner.invoke(app, ["--json", "issues", "clear"])
+    assert none_given.exit_code == 2, none_given.output
+
+    two_given = runner.invoke(app, ["--json", "issues", "clear", "--all-audits", "--everything"])
+    assert two_given.exit_code == 2, two_given.output
+
+
+@respx.mock
+def test_issues_clear_dry_run_writes_nothing(logged_in, runner):
+    calls = []
+
+    def record(request):
+        calls.append(json.loads(request.content))
+        return httpx.Response(200, json={"issues": 12, "findings": 9, "dry_run": True, "scope": "all_audits"})
+
+    respx.post(f"{BASE}/api/issues/bulk-clear").mock(side_effect=record)
+    result = runner.invoke(app, ["--json", "issues", "clear", "--all-audits", "--dry-run"])
+    assert result.exit_code == 0, result.output
+    assert json.loads(result.stdout)["issues"] == 12
+    # Exactly one request, and it carried dry_run — a second (writing) call here
+    # would mean --dry-run cleared the board.
+    assert len(calls) == 1
+    assert calls[0] == {"scope": "all_audits", "dry_run": True}
+
+
+@respx.mock
+def test_issues_clear_previews_then_writes(logged_in, runner):
+    calls = []
+
+    def record(request):
+        body = json.loads(request.content)
+        calls.append(body)
+        return httpx.Response(200, json={
+            "issues": 3, "findings": 2, "dry_run": body["dry_run"], "scope": "audit"})
+
+    respx.post(f"{BASE}/api/issues/bulk-clear").mock(side_effect=record)
+    result = runner.invoke(app, ["--json", "issues", "clear", "--audit", "a1", "--yes"])
+    assert result.exit_code == 0, result.output
+    # Preview first (so the confirm can name a number), then the real write.
+    assert [c["dry_run"] for c in calls] == [True, False]
+    assert all(c["audit_id"] == "a1" and c["scope"] == "audit" for c in calls)
+    assert json.loads(result.stdout)["issues"] == 3
+
+
+@respx.mock
+def test_issues_clear_declined_emits_the_json_cancel_envelope(logged_in, runner, monkeypatch):
+    monkeypatch.setattr(_write, "should_prompt", lambda *a, **k: True)
+    monkeypatch.setattr(output, "confirm_issues_clear", lambda *a, **k: False)
+    calls = []
+
+    def record(request):
+        calls.append(json.loads(request.content))
+        return httpx.Response(200, json={"issues": 5, "findings": 4, "dry_run": True, "scope": "everything"})
+
+    respx.post(f"{BASE}/api/issues/bulk-clear").mock(side_effect=record)
+    result = runner.invoke(app, ["--json", "issues", "clear", "--everything"])
+    assert result.exit_code == 0, result.output
+    assert json.loads(result.stdout) == {"cancelled": True}
+    # Only the preview ran. A declined confirm that had already written would be
+    # the worst possible version of this bug.
+    assert [c["dry_run"] for c in calls] == [True]
+
+
+@respx.mock
+def test_issues_clear_on_an_empty_scope_does_not_prompt_or_write(logged_in, runner, monkeypatch):
+    monkeypatch.setattr(_write, "should_prompt", lambda *a, **k: True)
+
+    def boom(*a, **k):  # pragma: no cover - asserts it is never reached
+        raise AssertionError("prompted to confirm clearing zero issues")
+
+    monkeypatch.setattr(output, "confirm_issues_clear", boom)
+    calls = []
+
+    def record(request):
+        calls.append(json.loads(request.content))
+        return httpx.Response(200, json={"issues": 0, "findings": 0, "dry_run": True, "scope": "all_audits"})
+
+    respx.post(f"{BASE}/api/issues/bulk-clear").mock(side_effect=record)
+    result = runner.invoke(app, ["--json", "issues", "clear", "--all-audits"])
+    assert result.exit_code == 0, result.output
+    assert json.loads(result.stdout)["issues"] == 0
+    assert [c["dry_run"] for c in calls] == [True]
+
+
+def test_issues_list_accepts_closed_as_a_state(logged_in, runner):
+    # `--state` is validated client-side (exit 2) against a hand-copied list. A
+    # state missing from it is rejected here for something the server accepts.
+    from fp_cli.commands import incidents_cmds
+
+    assert "closed" in incidents_cmds._STATES
+    bad = runner.invoke(app, ["--json", "issues", "list", "--state", "nope"])
+    assert bad.exit_code == 2, bad.output
+
+
+@respx.mock
+def test_issues_show_surfaces_closed_at_and_archived_at(logged_in, runner):
+    # `Incident.from_dict` is an ALLOWLIST — a key it does not name is dropped
+    # with no error, which renders as a permanently blank column.
+    respx.get(f"{BASE}/api/issues/i1").mock(return_value=httpx.Response(200, json={
+        "id": "i1", "state": "closed", "closed_at": "2026-09-22T10:00:00Z",
+        "archived_at": "2026-09-22T11:00:00Z", "alert_severity": "warning",
+    }))
+    from fp_cli import client as api
+
+    inc = api.get_incident(api.ClientContext(base_url=BASE, token="s"), "i1")
+    assert inc.closed_at == "2026-09-22T10:00:00Z"
+    assert inc.archived_at == "2026-09-22T11:00:00Z"
+    assert inc.state == "closed"

package/.next/standalone/fp-cloud-cli/uv.lock

@@ -372,7 +372,7 @@ wheels = [
 
 [[package]]
 name = "posthog"
-version = "7.53.0"
+version = "7.58.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "backoff" },
@@ -380,9 +380,9 @@ dependencies = [
     { name = "requests" },
     { name = "typing-extensions" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/f2/c4/17a2a764a63e061f768a56d50c245e3012da536b5a15ed2521616b8869b7/posthog-7.53.0.tar.gz", hash = "sha256:38223a8769b9db7ab906d32f5b87c4cde534cc91b30f1dcfa4443577819e068c", size = 565313, upload-time = "2026-09-11T17:51:16.199Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/ef/d9/6e351a6c739fec6e78c1be494f4d39512271167d3eb5905ccafa783c3234/posthog-7.58.0.tar.gz", hash = "sha256:43dcdeea3af4dbdece3322c5e37864cbdad6d0e507c20182bb33a960900bb077", size = 668207, upload-time = "2026-09-18T15:00:06.96Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/03/9d/4af58450b986666c810dc1fbae04e41cdec62ab31ae1e0e51bb5a4b75f71/posthog-7.53.0-py3-none-any.whl", hash = "sha256:ca9176334cf0c7799fe5518f7b9dc02d3f2f709a859da4332f14b3586b9d4554", size = 662783, upload-time = "2026-09-11T17:51:14.275Z" },
+    { url = "https://files.pythonhosted.org/packages/24/60/997629e929e96641d297f7bf67543026eeaf5f8a3de66363f1cbe366c0ee/posthog-7.58.0-py3-none-any.whl", hash = "sha256:2ac581ce52c63acb3a551c9066652c3d29e1de1ea06e70655ebe8b866e80f334", size = 781569, upload-time = "2026-09-18T15:00:04.587Z" },
 ]
 
 [[package]]

package/.next/standalone/package.json

@@ -1,6 +1,6 @@
 {
   "name": "failproofai",
-  "version": "1.0.6-beta.1",
+  "version": "1.0.7-beta.0",
   "description": "Observability and enforcement for AI agent harnesses. 39 built-in policies hooked into 12 of them — Claude Code, Codex, Cursor, Hermes, OpenClaw and more — blocking the tool call before it runs. Local dashboard included, no account needed.",
   "bin": {
     "failproofai": "./dist/cli.mjs",
@@ -81,14 +81,14 @@
     "access": "public"
   },
   "devDependencies": {
-    "@anthropic-ai/sdk": "^0.125.0",
+    "@anthropic-ai/sdk": "^0.127.0",
     "@mdx-js/mdx": "^3.1.1",
     "@tailwindcss/postcss": "^4.3.1",
     "@tanstack/react-virtual": "^3.14.10",
     "@testing-library/jest-dom": "^7.0.0",
     "@testing-library/react": "^16.3.2",
     "@testing-library/user-event": "^14.6.5",
-    "@types/node": "26.5.1",
+    "@types/node": "26.6.1",
     "@types/react": "19.3.0",
     "@types/react-dom": "^19.2.3",
     "@vitejs/plugin-react": "^6.1.0",
@@ -122,15 +122,15 @@
     "browserslist": "4.28.8"
   },
   "optionalDependencies": {
-    "@failproofai/failproofaid-linux-x64": "1.0.6-beta.1",
-    "@failproofai/failproofaid-linux-arm64": "1.0.6-beta.1",
-    "@failproofai/failproofaid-darwin-x64": "1.0.6-beta.1",
-    "@failproofai/failproofaid-darwin-arm64": "1.0.6-beta.1"
+    "@failproofai/failproofaid-linux-x64": "1.0.7-beta.0",
+    "@failproofai/failproofaid-linux-arm64": "1.0.7-beta.0",
+    "@failproofai/failproofaid-darwin-x64": "1.0.7-beta.0",
+    "@failproofai/failproofaid-darwin-arm64": "1.0.7-beta.0"
   },
   "failproofaidBinaries": {
-    "linux-x64": "b6f877cd407c3322ff8ad2f98a096b6ea15b0bd1d63de6276c689368345c1cd2",
-    "linux-arm64": "0da90a816b9dc707efe9b82176923cbf0f00026179c816e34eaad362ce2c0fa4",
-    "darwin-x64": "bcbec488353fa04a1953a87cad15054e4796117cac4fd60b17fb1c87d62b7b1e",
-    "darwin-arm64": "6830989a91a27642cdcc63fcfad529c69c0deb3f3abcf5ad40fd3516289b451d"
+    "linux-x64": "3ac234868fe9149179c32981da9ae1209371e98df7ffb09c6a4947664a3d6857",
+    "linux-arm64": "3603faac5741c321a63ba5a963e0cb4bb38c84663a63b53f764faae020116f90",
+    "darwin-x64": "c8a5b32a1e97b5a7f76ce4e42c78f0d566d108be65fdca3c6cd59c405f39b946",
+    "darwin-arm64": "255ac741c3ee3069591b1eb8bbf397baa0722ef240b83a8540b73f3f1b35722c"
   }
 }

package/.next/standalone/sdk/python/CHANGELOG.md

@@ -9,6 +9,13 @@ in `failproofai_sdk/_version.py` becomes that release's GitHub Release body. A
 release whose section is missing or empty is refused before anything is built —
 see `scripts/changelog-section.py`.
 
+## 0.0.1b5 — 2026-09-22
+
+Open for the next release. `0.0.1b4` published on 2026-09-22 and the `bump` job
+moved the version here automatically; nothing has landed against `0.0.1b5` yet.
+Add entries as changes merge — this section becomes the GitHub Release body when
+it ships.
+
 ## 0.0.1b4 — 2026-09-12
 
 Open for the next release. `0.0.1b3` published on 2026-09-12 and the `bump` job

package/.next/standalone/sdk/python/failproofai_sdk/_version.py

@@ -1 +1 @@
-__version__ = "0.0.1b4"
+__version__ = "0.0.1b5"

package/.next/standalone/sdk/python/failproofai_sdk/evaluator/__init__.py

@@ -10,6 +10,7 @@ from failproofai_sdk.evaluator.authoring import (
     EvalDefinition,
     EvalResult,
     Evaluator,
+    ManagedCompiler,
     Metric,
     Score,
 )
@@ -67,6 +68,7 @@ __all__ = [
     "EvalSelection",
     "ExecutionMode",
     "Evaluator",
+    "ManagedCompiler",
     "EvaluatorAPIError",
     "EvaluatorClient",
     "EvaluatorKind",

package/.next/standalone/sdk/python/failproofai_sdk/evaluator/authoring.py

@@ -36,6 +36,17 @@ ConditionFunction = Callable[
     [SessionTranscript], "bool | ConditionResult | Awaitable[bool | ConditionResult]"
 ]
 CancellationFunction = Callable[[SessionTranscript], "Any | Awaitable[Any]"]
+# Compiles server-authored (managed) source into something callable with a
+# session. Signature mirrors `source.compile_evaluator`, which is the default.
+#
+# A host that serves managed definitions whose source is NOT a restricted Python
+# expression — FailproofAI's own managed worker, whose source may instead be a
+# declarative judge document — installs its own compiler here. The SDK stays
+# agnostic: it never inspects the source, it just hands it to whoever compiles.
+# Returning an `async def` is supported and is the right shape for a compiler
+# whose work is IO-bound (see `WorkerRuntime._invoke`, which awaits an awaitable
+# result on the event loop rather than parking a thread on it).
+ManagedCompiler = Callable[..., EvalFunction]
 
 
 def _bounded(value: str, *, field_name: str, maximum: int) -> str:
@@ -284,11 +295,23 @@ class EvalDefinition:
 class Evaluator:
     """A process-local collection of explicitly versioned evaluations."""
 
-    def __init__(self, *, name: str, version: str) -> None:
+    def __init__(
+        self,
+        *,
+        name: str,
+        version: str,
+        managed_compiler: ManagedCompiler | None = None,
+    ) -> None:
         self.name = _bounded(name, field_name="name", maximum=MAX_DISPLAY_NAME_BYTES)
         self.version = _bounded(
             version, field_name="version", maximum=MAX_VERSION_BYTES
         )
+        # `None` keeps the default: managed source is compiled and sandboxed by
+        # `source.compile_evaluator`. Customer workers never set this — their
+        # definitions are LOCAL and never carry source at all.
+        if managed_compiler is not None and not callable(managed_compiler):
+            raise TypeError("managed_compiler must be callable")
+        self.managed_compiler = managed_compiler
         self._definitions: dict[str, EvalDefinition] = {}
 
     def eval(

package/.next/standalone/sdk/python/failproofai_sdk/evaluator/runtime.py

@@ -88,7 +88,12 @@ def _utc_now() -> str:
     )
 
 
-def _deferred_managed_eval(source: str, timeout_seconds: int | None, eval_key: str):
+def _deferred_managed_eval(
+    source: str,
+    timeout_seconds: int | None,
+    eval_key: str,
+    compiler: Any = None,
+):
     """Compile server-authored source lazily, at invocation time.
 
     Compilation can reject unsafe or malformed source (``UnsafeEvaluatorSource``).
@@ -98,10 +103,19 @@ def _deferred_managed_eval(source: str, timeout_seconds: int | None, eval_key: s
     dead-letters cleanly as one failed run instead of raising out of assignment
     setup, crashing the task, and forcing the whole assignment to be reclaimed
     and retried until its attempt budget is exhausted.
+
+    ``compiler`` defaults to :func:`compile_evaluator` — the restricted-expression
+    sandbox. A host serving managed definitions in another shape installs its own
+    via ``Evaluator(managed_compiler=...)``; the SDK never inspects the source.
+    The thunk stays SYNCHRONOUS either way, so a sandboxed compile still runs in
+    the runtime's sized executor exactly as before. A compiler that returns an
+    ``async def`` simply makes this thunk return a coroutine, which ``_invoke``
+    awaits on the event loop — the right place for IO-bound work.
     """
+    compile_source = compiler if compiler is not None else compile_evaluator
 
     def evaluate(session: Any) -> Any:
-        return compile_evaluator(
+        return compile_source(
             source, timeout_seconds=timeout_seconds, eval_key=eval_key
         )(session)
 
@@ -551,6 +565,7 @@ class WorkerRuntime:
                         run.evaluator_source,
                         run.timeout_seconds or descriptor.timeout_seconds,
                         descriptor.eval_key,
+                        self.evaluator.managed_compiler,
                     ),
                     condition=None,
                     on_cancel=None,
@@ -597,9 +612,15 @@ class WorkerRuntime:
         started = time.monotonic()
         # A synchronous evaluator runs in the executor thread; if it overruns the
         # wall-clock timeout below, the thread cannot be cancelled and is orphaned.
-        sync_function = not inspect.iscoroutinefunction(definition.function)
+        #
+        # Decided AFTER the fact, not from the function object: a deferred managed
+        # thunk is a sync callable that may return a coroutine, which `_invoke`
+        # awaits on the loop. That case is cancellable and leaves no orphan, so
+        # counting it as one would send an operator hunting a hung thread that
+        # does not exist.
+        awaited_on_loop: list[bool] = []
         try:
-            invocation = self._invoke(definition.function, session)
+            invocation = self._invoke(definition.function, session, awaited_on_loop)
             # Always bound the evaluation. A definition with no declared
             # timeout_seconds falls back to DEFAULT_EVAL_TIMEOUT_SECONDS rather
             # than awaiting unbounded — an unbounded local eval that hangs would
@@ -630,7 +651,7 @@ class WorkerRuntime:
             # the forked managed sandbox was killed by its CPU/memory/time budget —
             # the real, thread-uncancellable case. Both are a timed-out run.
             await self._cancel_hook(definition, session)
-            if isinstance(timeout_error, asyncio.TimeoutError) and sync_function:
+            if isinstance(timeout_error, asyncio.TimeoutError) and not awaited_on_loop:
                 # The awaiter gave up while a SYNCHRONOUS evaluator was still
                 # running in the executor. CPython cannot interrupt that thread,
                 # so it is now orphaned — it runs to completion (or forever)
@@ -824,12 +845,26 @@ class WorkerRuntime:
             remaining = min(remaining, float(timeout_seconds))
         return remaining
 
-    async def _invoke(self, function, session):
+    async def _invoke(self, function, session, awaited: list[bool] | None = None):
+        """Run an evaluation or condition, on the loop or in the executor.
+
+        ``awaited`` is an out-parameter: append-only, and appended to exactly when
+        the work ends up being awaited on the EVENT LOOP rather than occupying an
+        executor thread. The timeout handler needs that distinction — a coroutine
+        is cancelled cleanly, a running thread is not — and it cannot be known
+        from the function object alone, because a deferred managed thunk is a
+        plain sync callable that may still RETURN a coroutine.
+        """
         if inspect.iscoroutinefunction(function):
+            if awaited is not None:
+                awaited.append(True)
             return await function(session)
         loop = asyncio.get_running_loop()
         result = await loop.run_in_executor(self._eval_executor, function, session)
         if inspect.isawaitable(result):
+            # The thread has already returned; only the coroutine is outstanding.
+            if awaited is not None:
+                awaited.append(True)
             return await result
         return result
 

package/.next/standalone/sdk/python/tests/test_evaluator_runtime.py

@@ -1101,3 +1101,91 @@ def test_condition_phase_deadline_and_budget_are_lease_bounded():
     assert runtime._condition_budget(deadline, None) == pytest.approx(95, abs=2)
     assert runtime._condition_budget(deadline, 10) == pytest.approx(10, abs=0.05)
     assert runtime._condition_budget(time.monotonic(), None) < 0
+
+
+def test_managed_compiler_hook_receives_source_verbatim_and_may_be_async():
+    # The SDK must never inspect managed source. FailproofAI's hosted worker
+    # serves definitions whose source is a declarative judge document rather than
+    # a restricted Python expression; it installs its own compiler, and the SDK's
+    # only job is to hand the string over unchanged. The compiler returns an
+    # `async def` here because judge work is IO-bound — `_invoke` must await that
+    # on the event loop rather than parking an executor thread on it.
+    source = '{"kind": "judge", "criteria": "was it helpful", "threshold": 0.5}'
+    seen: dict[str, object] = {}
+
+    def compiler(raw, *, timeout_seconds, eval_key):
+        seen["source"] = raw
+        seen["timeout_seconds"] = timeout_seconds
+        seen["eval_key"] = eval_key
+
+        async def judge(session):
+            seen["session_id"] = session.session_id
+            return EvalResult(score=Score(0.9, passed=True), summary="judged")
+
+        return judge
+
+    class HostedClient(FakeClient):
+        def __init__(self):
+            super().__init__()
+            self.assignment = replace(
+                self.assignment,
+                definitions_url=f"/v1/evaluator/assignments/{self.assignment.assignment_id}/definitions",
+            )
+
+        def definitions(self, assignment, *, worker_id):
+            return DefinitionsResponse(
+                assignment_id=assignment.assignment_id,
+                catalog_revision="sha256:hosted",
+                definitions=(
+                    AssignmentDefinition(
+                        eval_key="answer_quality",
+                        display_name="Answer quality",
+                        eval_version="1",
+                        result_kind=ResultKind.SCORE,
+                        execution_mode=ExecutionMode.PYTHON,
+                        source_checksum=source_checksum(None, source),
+                    ),
+                ),
+            )
+
+        def plan(self, assignment_id, request):
+            self.plans.append(request)
+            return PlanResponse(
+                assignment_id=assignment_id,
+                assignment_status="planned",
+                runs=(
+                    PlannedRun(
+                        "run-judge",
+                        "answer_quality",
+                        "1",
+                        execution_mode=ExecutionMode.PYTHON,
+                        evaluator_source=source,
+                        source_checksum=source_checksum(None, source),
+                        timeout_seconds=30,
+                    ),
+                ),
+            )
+
+    client = HostedClient()
+    evaluator = Evaluator(name="managed", version="1", managed_compiler=compiler)
+    asyncio.run(_runtime(evaluator, client).process_assignment(client.assignment))
+
+    # Verbatim: not parsed, not normalised, not validated by the SDK.
+    assert seen["source"] == source
+    assert seen["eval_key"] == "answer_quality"
+    assert seen["timeout_seconds"] == 30
+    assert seen["session_id"] == client.assignment.session_id
+
+    assert len(client.submissions) == 1
+    run_id, result = client.submissions[0]
+    assert run_id == "run-judge"
+    assert result.status.value == "succeeded"
+    assert result.results[0].numeric_value == 0.9
+
+
+def test_managed_compiler_defaults_to_the_sandbox_and_rejects_non_callables():
+    # No compiler installed => the restricted-expression sandbox, unchanged. This
+    # is what every customer worker and every existing deployment gets.
+    assert Evaluator(name="default", version="1").managed_compiler is None
+    with pytest.raises(TypeError):
+        Evaluator(name="bad", version="1", managed_compiler="not-callable")

package/.next/standalone/sdk/python/uv.lock

@@ -751,7 +751,7 @@ wheels = [
 
 [[package]]
 name = "crewai"
-version = "1.15.21"
+version = "1.15.22"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "aiofiles" },
@@ -786,14 +786,14 @@ dependencies = [
     { name = "tomli" },
     { name = "tomli-w" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/ab/d4/58a44b3450f95c0cb2654bccc01bc5cd65bff768b57189f11cecf641d18d/crewai-1.15.21.tar.gz", hash = "sha256:87eb5c7ad772db5b21cfd4c219800c1886a6a5076d02617db16824f944264345", size = 7975540, upload-time = "2026-09-09T22:54:56.424Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/38/3e/e79f59badc4194f5d80e9ae5b07d35d96206d705270a2b2a2b5508b0a54f/crewai-1.15.22.tar.gz", hash = "sha256:0b1c654f35e07dfe41720337c56542b334644a32c5e4498a6e38e5e7da7dd82e", size = 8057957, upload-time = "2026-09-16T22:10:43.476Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/c6/2b/38d8b378ec1f55d893f2d52dfb6349b282f464f40535ff53b047916f21d4/crewai-1.15.21-py3-none-any.whl", hash = "sha256:08ef2605260b0ffa7c54218fbcab8d3c4dc8d6307bd3b43513985529f6f666ab", size = 1144433, upload-time = "2026-09-09T22:54:54.299Z" },
+    { url = "https://files.pythonhosted.org/packages/f0/92/b19a70b6109ce2c2bd24dc63f8b4022a7097816ef236a1d9de1733738db8/crewai-1.15.22-py3-none-any.whl", hash = "sha256:7675c2bb1166933c60d67596936cd8b4ce63d77361dd3aeb925b210180298e5a", size = 1189089, upload-time = "2026-09-16T22:10:41.471Z" },
 ]
 
 [[package]]
 name = "crewai-cli"
-version = "1.15.21"
+version = "1.15.22"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "appdirs" },
@@ -813,14 +813,14 @@ dependencies = [
     { name = "tomli-w" },
     { name = "uv" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/c7/22/4747b13eeedcc073d6b5c5af1738491d321b26846ab9d979def67fbecbff/crewai_cli-1.15.21.tar.gz", hash = "sha256:9b949c50d20daef8b51a0c54e7726a0570b55233eef8ffd47febfe329137b3da", size = 235951, upload-time = "2026-09-09T22:54:59.153Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/3f/ed/f143440caca0fd6c5cb985710f79f0a6ebffbbb5468ccf842d55b5500ce6/crewai_cli-1.15.22.tar.gz", hash = "sha256:b7e00daedfbee9102ce4fcaffa10224b27c9c02e2b976c98a25acf214730acae", size = 244293, upload-time = "2026-09-16T22:10:46.178Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/ad/b4/3352d0f1bb431bc2d8f4aaabd045a30bd374a50c682e15c80fc1290b8d52/crewai_cli-1.15.21-py3-none-any.whl", hash = "sha256:f65b4d0f55816a1a2d3b86c9c087527ddb1c85d868a4d2a75931edc04482cbd9", size = 198948, upload-time = "2026-09-09T22:54:57.921Z" },
+    { url = "https://files.pythonhosted.org/packages/d6/af/5e06129c84ba60a7f48ee181d6b0c3f270f00e2dcebaf48556325cc1a59b/crewai_cli-1.15.22-py3-none-any.whl", hash = "sha256:b44abcc69b90f30c4d3b93657542879f5e2ac83506757a38a1e2e404a9d84300", size = 202381, upload-time = "2026-09-16T22:10:44.901Z" },
 ]
 
 [[package]]
 name = "crewai-core"
-version = "1.15.21"
+version = "1.15.22"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "appdirs" },
@@ -836,9 +836,9 @@ dependencies = [
     { name = "rich" },
     { name = "tomli" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/b3/4c/28f0bcf8b77bcb5adc782b6403001fb23e83ec26e6a6c7a5aee97b480450/crewai_core-1.15.21.tar.gz", hash = "sha256:8ca73bd33e0fe1de1a25193bd3c468e9e23aee55f6b4001c4f43e65a3fdbde81", size = 39213, upload-time = "2026-09-09T22:55:01.395Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/95/27/1ed033b746506039e21291eb64fd4624e9de0b0489dfd5128e7efb936e92/crewai_core-1.15.22.tar.gz", hash = "sha256:6b85c609536bdca62e93eca59933f3609f79b8e207e8f6d3c59a8365a5ed5d78", size = 43090, upload-time = "2026-09-16T22:10:48.274Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/a1/36/9a415b0420ff9dff74d9413fb48a438754ebaa102fc73b85f7923d5819a1/crewai_core-1.15.21-py3-none-any.whl", hash = "sha256:26143369fb3278c41ea843f2bab29437c78d962738b98e8f846aac3225c65bf5", size = 42480, upload-time = "2026-09-09T22:55:00.218Z" },
+    { url = "https://files.pythonhosted.org/packages/02/45/f2eb1866dc98d75617ebe942e9575b6fa1f2f2b7c6352ea22cb2faf75931/crewai_core-1.15.22-py3-none-any.whl", hash = "sha256:8b97a45a35e663f17af65ed2a318bf428d5806ce20a54b0fce726ebe53eedb75", size = 44375, upload-time = "2026-09-16T22:10:47.06Z" },
 ]
 
 [[package]]
@@ -3906,7 +3906,7 @@ wheels = [
 
 [[package]]
 name = "pydantic-ai-slim"
-version = "2.42.0"
+version = "2.45.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "anyio" },
@@ -3919,9 +3919,9 @@ dependencies = [
     { name = "pydantic-graph" },
     { name = "typing-inspection" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/47/ec/59161f7aa44251775afec888615b84905634252b7dac6f0acf2f430ffadf/pydantic_ai_slim-2.42.0.tar.gz", hash = "sha256:045c788b1636e0ed79542f2b7562d2688a3e3701744ed7379a928fceec446336", size = 1453904, upload-time = "2026-09-09T03:48:51.06Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/70/ad/d39e46c192e04df2e620871592054185aed4e212558d90bf89436aa7628d/pydantic_ai_slim-2.45.0.tar.gz", hash = "sha256:34cf5216ebe05cb3d343129bb24a1bd011c66e89dd620f00afa982bd2484e3bf", size = 1562650, upload-time = "2026-09-18T04:50:15.315Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/80/d5/5f9489b6d8583366c32c76671f4d5c3a91cf7e0e0a7073269d9fee756f28/pydantic_ai_slim-2.42.0-py3-none-any.whl", hash = "sha256:a042b73551470c40a2ab7cd532e26c20f592b5731ae544ff3eecbcdf6ceebcc4", size = 1714452, upload-time = "2026-09-09T03:48:43.81Z" },
+    { url = "https://files.pythonhosted.org/packages/de/5b/1397c8c65eaeae781ec489db648ab24125268b749d1f256ba7c13b07891e/pydantic_ai_slim-2.45.0-py3-none-any.whl", hash = "sha256:0914f4e307c64401b676bbba2d613da69f74dbeaab4e2e8516d0d9b53f4ab483", size = 1857668, upload-time = "2026-09-18T04:50:07.395Z" },
 ]
 
 [[package]]
@@ -4044,7 +4044,7 @@ wheels = [
 
 [[package]]
 name = "pydantic-graph"
-version = "2.42.0"
+version = "2.45.0"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "anyio" },
@@ -4052,9 +4052,9 @@ dependencies = [
     { name = "pydantic" },
     { name = "typing-inspection" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/a3/c2/f14d82d10c7f4e4e4d20c880593745e05686b176ac397fe52a5ec40959a9/pydantic_graph-2.42.0.tar.gz", hash = "sha256:eb6c416dee9e3415195194c44ca3abeeae8853bff095d38ac798c7c3395f815d", size = 45189, upload-time = "2026-09-09T03:48:53.398Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/56/ee/d4f4c73eb91241af7a06e4c36ef58c18d3622dc03d8343825464ec06cb2e/pydantic_graph-2.45.0.tar.gz", hash = "sha256:c07e8cee55f25d79ea2ea55de67fc67d0353ec2807e3a1328f42995edf022c23", size = 45192, upload-time = "2026-09-18T04:50:17.706Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/7a/c7/eafabe22e04e4e8dfb8e01725e051feed75d3cda937054a727cffc525497/pydantic_graph-2.42.0-py3-none-any.whl", hash = "sha256:25c47047ac1bbad2fdc6dace23958c6ec700a6366abc18d67dccd41411185df7", size = 52700, upload-time = "2026-09-09T03:48:46.931Z" },
+    { url = "https://files.pythonhosted.org/packages/27/56/d040149787ae9ade628f05c1e7f30657437731b0e2bb3a2cdb6cb979b7af/pydantic_graph-2.45.0-py3-none-any.whl", hash = "sha256:60fef7eab40a103510f0baf6d17cdd11418672cb64a7524089fbaf845f45aa1a", size = 52701, upload-time = "2026-09-18T04:50:10.72Z" },
 ]
 
 [[package]]