failproofai 1.0.3 → 1.0.4-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.next/standalone/.next/BUILD_ID +1 -1
- package/.next/standalone/.next/build-manifest.json +5 -5
- package/.next/standalone/.next/prerender-manifest.json +5 -5
- package/.next/standalone/.next/required-server-files.json +1 -1
- package/.next/standalone/.next/server/app/_global-error/page/build-manifest.json +2 -2
- package/.next/standalone/.next/server/app/_global-error/page/server-reference-manifest.json +1 -1
- package/.next/standalone/.next/server/app/_global-error/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/_global-error/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/_global-error.html +1 -1
- package/.next/standalone/.next/server/app/_global-error.rsc +7 -7
- package/.next/standalone/.next/server/app/_global-error.segments/__PAGE__.segment.rsc +6 -6
- package/.next/standalone/.next/server/app/_global-error.segments/_full.segment.rsc +7 -7
- package/.next/standalone/.next/server/app/_global-error.segments/_tree.segment.rsc +1 -1
- package/.next/standalone/.next/server/app/_not-found/page/build-manifest.json +2 -2
- package/.next/standalone/.next/server/app/_not-found/page/server-reference-manifest.json +1 -1
- package/.next/standalone/.next/server/app/_not-found/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/_not-found/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/_not-found.html +1 -1
- package/.next/standalone/.next/server/app/_not-found.rsc +15 -15
- package/.next/standalone/.next/server/app/_not-found.segments/_full.segment.rsc +15 -15
- package/.next/standalone/.next/server/app/_not-found.segments/_not-found/__PAGE__.segment.rsc +14 -14
- package/.next/standalone/.next/server/app/_not-found.segments/_tree.segment.rsc +2 -2
- package/.next/standalone/.next/server/app/api/audit/invite/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/audit/invite/route_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/api/audit/run/route.js +4 -4
- package/.next/standalone/.next/server/app/api/audit/run/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/audit/run/route_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/api/audit/status/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/audit/status/route_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/api/auth/login-request/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/auth/login-request/route_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/api/auth/login-verify/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/auth/login-verify/route_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/api/auth/logout/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/auth/logout/route_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/api/auth/status/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/auth/status/route_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/api/download/[project]/[session]/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/download/[project]/[session]/route_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/audit/page/build-manifest.json +2 -2
- package/.next/standalone/.next/server/app/audit/page/server-reference-manifest.json +37 -4
- package/.next/standalone/.next/server/app/audit/page.js +2 -2
- package/.next/standalone/.next/server/app/audit/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/audit/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/index.html +1 -1
- package/.next/standalone/.next/server/app/index.rsc +15 -15
- package/.next/standalone/.next/server/app/index.segments/__PAGE__.segment.rsc +14 -14
- package/.next/standalone/.next/server/app/index.segments/_full.segment.rsc +15 -15
- package/.next/standalone/.next/server/app/index.segments/_tree.segment.rsc +2 -2
- package/.next/standalone/.next/server/app/page/build-manifest.json +2 -2
- package/.next/standalone/.next/server/app/page/server-reference-manifest.json +1 -1
- package/.next/standalone/.next/server/app/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/policies/page/build-manifest.json +2 -2
- package/.next/standalone/.next/server/app/policies/page/server-reference-manifest.json +14 -14
- package/.next/standalone/.next/server/app/policies/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/policies/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/project/[name]/page/build-manifest.json +2 -2
- package/.next/standalone/.next/server/app/project/[name]/page/server-reference-manifest.json +1 -1
- package/.next/standalone/.next/server/app/project/[name]/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/project/[name]/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page/build-manifest.json +2 -2
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page/react-loadable-manifest.json +2 -2
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page/server-reference-manifest.json +2 -2
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/projects/page/build-manifest.json +2 -2
- package/.next/standalone/.next/server/app/projects/page/server-reference-manifest.json +1 -1
- package/.next/standalone/.next/server/app/projects/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/projects/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/settings/page/build-manifest.json +2 -2
- package/.next/standalone/.next/server/app/settings/page/server-reference-manifest.json +4 -4
- package/.next/standalone/.next/server/app/settings/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/settings/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/chunks/{[externals]__0-9ti7z._.js → [externals]__1lh7m5d._.js} +1 -1
- package/.next/standalone/.next/server/chunks/{[externals]__0u3wrvq._.js → [externals]__1rqkg_y._.js} +1 -1
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0o07qi9._.js +1 -1
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__1adacul._.js +1 -1
- package/.next/standalone/.next/server/chunks/{_1-akg6w._.js → _08w6xzm._.js} +2 -2
- package/.next/standalone/.next/server/chunks/_0tovk6q._.js +1 -1
- package/.next/standalone/.next/server/chunks/_0trp3yc._.js +1 -1
- package/.next/standalone/.next/server/chunks/_1ek68ln._.js +4 -4
- package/.next/standalone/.next/server/chunks/lib_telemetry_ts_0dctyyw._.js +1 -1
- package/.next/standalone/.next/server/chunks/node_modules_posthog-node_dist_entrypoints_index_node_mjs_01r25oi._.js +2 -2
- package/.next/standalone/.next/server/chunks/node_modules_posthog-node_dist_entrypoints_index_node_mjs_09z9-p7._.js +2 -2
- package/.next/standalone/.next/server/chunks/package_json_[json]_cjs_1nxcc4v._.js +1 -1
- package/.next/standalone/.next/server/chunks/src_hooks_fp-home_ts_09kv0bn._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__013jr2b._.js +2 -2
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__01wy8d-._.js +2 -2
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__02npjtd._.js +2 -2
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0da85px._.js +2 -2
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0ftmoxc._.js +2 -2
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0oa1lav._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0p-5p8u._.js +2 -2
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0rgu2r3._.js +3 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0s740oi._.js +2 -2
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__1p2otjt._.js +2 -2
- package/.next/standalone/.next/server/chunks/ssr/_0-oij9d._.js +23 -0
- package/.next/standalone/.next/server/chunks/ssr/_08x1r5t._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/_0l2hi_d._.js +3 -0
- package/.next/standalone/.next/server/chunks/ssr/_0oq1dh6._.js +3 -0
- package/.next/standalone/.next/server/chunks/ssr/_1es2j7i._.js +47 -13
- package/.next/standalone/.next/server/chunks/ssr/_1u8-lu2._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/_next-internal_server_app_policies_page_actions_1sp2-yo.js +2 -2
- package/.next/standalone/.next/server/chunks/ssr/app_audit__components_audit-dashboard_tsx_0p9ud47._.js +1 -69
- package/.next/standalone/.next/server/chunks/ssr/app_audit__components_rerun-button_tsx_0blihgw._.js +3 -0
- package/.next/standalone/.next/server/chunks/ssr/app_global-error_tsx_1kp6l3x._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/app_policies_hooks-client_tsx_19dqvpc._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/app_settings_settings-client_tsx_20lq-mq._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/node_modules_13n822a._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/node_modules_posthog-node_dist_entrypoints_index_node_mjs_11bnuzn._.js +2 -2
- package/.next/standalone/.next/server/chunks/ssr/src_hooks_builtin-policies_ts_09j2ndl._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/src_hooks_fp-config_ts_04t589g._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/src_hooks_fp-home_ts_0je3xkv._.js +1 -1
- package/.next/standalone/.next/server/middleware-build-manifest.js +5 -5
- package/.next/standalone/.next/server/pages/404.html +1 -1
- package/.next/standalone/.next/server/pages/500.html +1 -1
- package/.next/standalone/.next/server/server-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/server-reference-manifest.json +54 -21
- package/.next/standalone/.next/static/chunks/010bv1w6j171t.js +1 -0
- package/.next/standalone/.next/static/chunks/04r6ch8uf_n8m.js +1 -0
- package/.next/standalone/.next/static/chunks/{29-iqnp9v_axt.css → 0o-hh5_turzlz.css} +1 -1
- package/.next/standalone/.next/static/chunks/{40nr6ehj4p3cw.css → 0qmsunv04x4na.css} +1 -1
- package/.next/standalone/.next/static/chunks/{0ollg-nnu3t3e.js → 0wz8yftk18ts2.js} +1 -1
- package/.next/standalone/.next/static/chunks/0zebh1n9jkfbt.js +1 -0
- package/.next/standalone/.next/static/chunks/1eyl7fzkmnkf_.js +1 -0
- package/.next/standalone/.next/static/chunks/1pb1oztsbwcss.js +1 -0
- package/.next/standalone/.next/static/chunks/{1-_zkmnm-jwei.js → 2aquitk72k2op.js} +1 -1
- package/.next/standalone/.next/static/chunks/2bi_1y0a_smt7.js +6 -0
- package/.next/standalone/.next/static/chunks/{42yfkhekx440s.js → 2ej3b8gk5ittu.js} +1 -1
- package/.next/standalone/.next/static/chunks/{354-n7p7labpn.js → 2zafgs90r_leq.js} +1 -1
- package/.next/standalone/.next/static/chunks/{2xm5mj1d21b4n.js → 32spub4wqjem-.js} +1 -1
- package/.next/standalone/.next/static/chunks/{1a6u7l3giejch.js → 3m4upvybtrexd.js} +1 -1
- package/.next/standalone/.next/static/chunks/{2k9f4tyv04809.css → 3uyhmm01g64k4.css} +1 -0
- package/.next/standalone/.next/static/chunks/{turbopack-2x-o099q64i8h.js → turbopack-3s3c-u_u_l0u9.js} +1 -1
- package/.next/standalone/.opencode/plugins/failproofai.mjs +51 -27
- package/.next/standalone/app/actions/get-leaks.ts +100 -0
- package/.next/standalone/app/audit/_components/audit-dashboard.tsx +112 -33
- package/.next/standalone/app/audit/_components/audit-poster.tsx +29 -10
- package/.next/standalone/app/audit/_components/come-back-better-section.tsx +6 -1
- package/.next/standalone/app/audit/_components/empty-state.tsx +4 -1
- package/.next/standalone/app/audit/_components/how-to-improve-section.tsx +19 -5
- package/.next/standalone/app/audit/_components/leak-section.tsx +179 -0
- package/.next/standalone/app/audit/_components/share-templates.ts +110 -44
- package/.next/standalone/app/audit/audit-styles.css +114 -1
- package/.next/standalone/fp-cloud-cli/CHANGELOG.md +26 -3
- package/.next/standalone/fp-cloud-cli/fp_cli/_click_compat.py +31 -8
- package/.next/standalone/fp-cloud-cli/fp_cli/permissions.py +1 -0
- package/.next/standalone/fp-cloud-cli/pyproject.toml +9 -2
- package/.next/standalone/fp-cloud-cli/tests/test_click_compat.py +54 -0
- package/.next/standalone/fp-cloud-cli/uv.lock +9 -12
- package/.next/standalone/lib/auth/api-server-client.ts +28 -0
- package/.next/standalone/lib/claude-sessions.ts +101 -27
- package/.next/standalone/node_modules/@next/env/package.json +1 -1
- package/.next/standalone/node_modules/next/dist/build/swc/index.js +1 -1
- package/.next/standalone/node_modules/next/dist/compiled/next-server/app-page-turbo-experimental.runtime.prod.js +1 -1
- package/.next/standalone/node_modules/next/dist/compiled/next-server/app-page-turbo.runtime.prod.js +1 -1
- package/.next/standalone/node_modules/next/dist/compiled/next-server/app-route-turbo.runtime.prod.js +1 -1
- package/.next/standalone/node_modules/next/dist/compiled/next-server/pages-turbo.runtime.prod.js +5 -5
- package/.next/standalone/node_modules/next/dist/experimental/testmode/fetch.js +10 -2
- package/.next/standalone/node_modules/next/dist/lib/patch-incorrect-lockfile.js +3 -3
- package/.next/standalone/node_modules/next/dist/lib/typescript/runTypeScriptCli.js +2 -2
- package/.next/standalone/node_modules/next/dist/lib/verify-typescript-setup.js +3 -1
- package/.next/standalone/node_modules/next/dist/server/config.js +1 -1
- package/.next/standalone/node_modules/next/dist/server/dev/hot-reloader-turbopack.js +2 -2
- package/.next/standalone/node_modules/next/dist/server/dev/hot-reloader-webpack.js +1 -1
- package/.next/standalone/node_modules/next/dist/server/lib/app-info-log.js +1 -1
- package/.next/standalone/node_modules/next/dist/server/lib/incremental-cache/file-system-cache.js +17 -4
- package/.next/standalone/node_modules/next/dist/server/lib/start-server.js +1 -1
- package/.next/standalone/node_modules/next/dist/shared/lib/router/utils/escape-path-delimiters.js +1 -1
- package/.next/standalone/node_modules/next/dist/telemetry/anonymous-meta.js +1 -1
- package/.next/standalone/node_modules/next/dist/telemetry/events/swc-load-failure.js +1 -1
- package/.next/standalone/node_modules/next/dist/telemetry/events/version.js +2 -2
- package/.next/standalone/node_modules/next/package.json +11 -11
- package/.next/standalone/package.json +14 -13
- package/.next/standalone/sdk/python/CHANGELOG.md +155 -0
- package/.next/standalone/sdk/python/README.md +8 -0
- package/.next/standalone/sdk/python/examples/evaluator_worker.py +121 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_version.py +1 -1
- package/.next/standalone/sdk/python/failproofai_sdk/evaluator/__init__.py +101 -0
- package/.next/standalone/sdk/python/failproofai_sdk/evaluator/__main__.py +49 -0
- package/.next/standalone/sdk/python/failproofai_sdk/evaluator/_sandbox_runner.py +63 -0
- package/.next/standalone/sdk/python/failproofai_sdk/evaluator/authoring.py +404 -0
- package/.next/standalone/sdk/python/failproofai_sdk/evaluator/client.py +299 -0
- package/.next/standalone/sdk/python/failproofai_sdk/evaluator/protocol.py +754 -0
- package/.next/standalone/sdk/python/failproofai_sdk/evaluator/runtime.py +936 -0
- package/.next/standalone/sdk/python/failproofai_sdk/evaluator/source.py +684 -0
- package/.next/standalone/sdk/python/tests/fixtures/evaluator_v2/README.md +28 -0
- package/.next/standalone/sdk/python/tests/fixtures/evaluator_v2/contract.json +252 -0
- package/.next/standalone/sdk/python/tests/test_evaluator_authoring.py +124 -0
- package/.next/standalone/sdk/python/tests/test_evaluator_client.py +253 -0
- package/.next/standalone/sdk/python/tests/test_evaluator_example.py +35 -0
- package/.next/standalone/sdk/python/tests/test_evaluator_http_e2e.py +636 -0
- package/.next/standalone/sdk/python/tests/test_evaluator_main.py +47 -0
- package/.next/standalone/sdk/python/tests/test_evaluator_protocol.py +246 -0
- package/.next/standalone/sdk/python/tests/test_evaluator_review_fixes.py +164 -0
- package/.next/standalone/sdk/python/tests/test_evaluator_runtime.py +1103 -0
- package/.next/standalone/sdk/python/tests/test_evaluator_source.py +430 -0
- package/.next/standalone/sdk/python/tests/test_zero_dependencies.py +17 -0
- package/.next/standalone/sdk/python/uv.lock +29 -29
- package/.next/standalone/server.js +1 -1
- package/README.md +13 -58
- package/dist/cli.mjs +5368 -8756
- package/dist/worker.mjs +539 -312
- package/lib/auth/api-server-client.ts +28 -0
- package/lib/claude-sessions.ts +101 -27
- package/package.json +14 -13
- package/pi-extension/index.ts +42 -4
- package/scripts/changelog-open.py +115 -0
- package/src/audit/cli.ts +134 -0
- package/src/audit/desktop-notify.ts +420 -0
- package/src/audit/harm-report.ts +101 -0
- package/src/audit/index.ts +141 -13
- package/src/audit/leak-fingerprint.ts +200 -0
- package/src/audit/leak-notice.ts +161 -0
- package/src/audit/leak-record.ts +232 -0
- package/src/audit/leak-scan.ts +292 -0
- package/src/audit/leak-store.ts +217 -0
- package/src/audit/macos-notifier.ts +310 -0
- package/src/audit/redact-example.ts +266 -14
- package/src/audit/report-harm.ts +12 -1
- package/src/audit/report.ts +6 -3
- package/src/audit/schedule-cli.ts +37 -0
- package/src/audit/scoring.ts +49 -0
- package/src/audit/types.ts +41 -0
- package/src/hooks/builtin-policies.ts +67 -2
- package/src/hooks/configure-wizard.ts +20 -0
- package/src/hooks/fp-config.ts +71 -11
- package/src/hooks/fp-home.ts +27 -0
- package/src/hooks/handler.ts +59 -1
- package/src/hooks/integrations.ts +47 -13
- package/src/hooks/notice.ts +155 -0
- package/src/hooks/uninstall-cli.ts +15 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__1b5jzxr._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/_0j6gd_k._.js +0 -23
- package/.next/standalone/.next/server/chunks/ssr/_0v8eebd._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/_1ylae7o._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/_1zopuov._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/node_modules_html-to-image_es_index_1ao30b1.js +0 -3
- package/.next/standalone/.next/static/chunks/03slu0c8_71y7.js +0 -1
- package/.next/standalone/.next/static/chunks/0nm11ysn0153h.js +0 -1
- package/.next/standalone/.next/static/chunks/0u64yey446k4f.css +0 -1
- package/.next/standalone/.next/static/chunks/1r9676mkvihto.js +0 -1
- package/.next/standalone/.next/static/chunks/30pct5s734jua.js +0 -6
- package/.next/standalone/.next/static/chunks/3pibxnbhjza7n.js +0 -69
- package/.next/standalone/.next/static/chunks/3qfoeqhy5l10x.js +0 -1
- package/.next/standalone/.next/static/chunks/3zkg2s2vzxc3d.js +0 -1
- package/.next/standalone/.next/static/chunks/41b5ptaocvi_l.js +0 -1
- /package/.next/standalone/.next/static/{JBYgkXkA1PS48m9P_hKNw → aKNv4Kmv98Xpwns0Xa-C-}/_buildManifest.js +0 -0
- /package/.next/standalone/.next/static/{JBYgkXkA1PS48m9P_hKNw → aKNv4Kmv98Xpwns0Xa-C-}/_clientMiddlewareManifest.js +0 -0
- /package/.next/standalone/.next/static/{JBYgkXkA1PS48m9P_hKNw → aKNv4Kmv98Xpwns0Xa-C-}/_ssgManifest.js +0 -0
|
@@ -0,0 +1,430 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from failproofai_sdk.evaluator import EvalResult, Score
|
|
6
|
+
from failproofai_sdk.evaluator.protocol import SessionTranscript, TranscriptEvent
|
|
7
|
+
from failproofai_sdk.evaluator.source import (
|
|
8
|
+
MAX_AST_NODES,
|
|
9
|
+
MAX_EVALUATOR_SOURCE_BYTES,
|
|
10
|
+
MAX_POW_EXPONENT,
|
|
11
|
+
MAX_SANDBOX_TIMEOUT_SECONDS,
|
|
12
|
+
EvaluationSandboxUnavailable,
|
|
13
|
+
EvaluationTimeout,
|
|
14
|
+
UnsafeEvaluatorSource,
|
|
15
|
+
_clamp_budget,
|
|
16
|
+
compile_condition,
|
|
17
|
+
compile_evaluator,
|
|
18
|
+
source_checksum,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def Session(event_count: int = 3) -> SessionTranscript:
|
|
23
|
+
"""A real, serializable transcript — managed evals now run in a subprocess and
|
|
24
|
+
the transcript crosses the boundary via `to_wire`, so a dummy object won't do.
|
|
25
|
+
Each event carries a dict payload (so `events[0].payload.get` is reachable)."""
|
|
26
|
+
events = tuple(
|
|
27
|
+
TranscriptEvent(
|
|
28
|
+
id=f"e{i}",
|
|
29
|
+
ts="2026-08-28T12:00:00.000000Z",
|
|
30
|
+
event_type="tool_use",
|
|
31
|
+
payload={"k": "v", "tool_name": "search"},
|
|
32
|
+
)
|
|
33
|
+
for i in range(event_count)
|
|
34
|
+
)
|
|
35
|
+
return SessionTranscript(
|
|
36
|
+
assignment_id="a",
|
|
37
|
+
session_id="s",
|
|
38
|
+
session_revision_id="r",
|
|
39
|
+
agent_id="agent",
|
|
40
|
+
environment="test",
|
|
41
|
+
started_at="2026-08-28T12:00:00.000000Z",
|
|
42
|
+
ended_at="2026-08-28T12:00:01.000000Z",
|
|
43
|
+
event_count=event_count,
|
|
44
|
+
events=events,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def test_restricted_expressions_can_evaluate_conditions_and_results():
|
|
49
|
+
assert compile_condition("session.event_count > 0")(Session()) is True
|
|
50
|
+
result = compile_evaluator("EvalResult(score=Score(0.75, passed=True))")(
|
|
51
|
+
Session()
|
|
52
|
+
)
|
|
53
|
+
assert isinstance(result, EvalResult)
|
|
54
|
+
assert result.score == Score(0.75, passed=True)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@pytest.mark.parametrize(
|
|
58
|
+
"source",
|
|
59
|
+
[
|
|
60
|
+
"__import__('os').system('id')",
|
|
61
|
+
"session.__class__",
|
|
62
|
+
"(lambda: 1)()",
|
|
63
|
+
"[x for x in ().__class__.__base__.__subclasses__()]",
|
|
64
|
+
],
|
|
65
|
+
)
|
|
66
|
+
def test_restricted_expressions_reject_escape_primitives(source):
|
|
67
|
+
with pytest.raises(UnsafeEvaluatorSource):
|
|
68
|
+
compile_evaluator(source)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@pytest.mark.parametrize(
|
|
72
|
+
"source",
|
|
73
|
+
[
|
|
74
|
+
# `str.format` / `str.format_map` traverse a format string's field spec
|
|
75
|
+
# at the C level, reaching attributes the AST dunder guard never sees.
|
|
76
|
+
# These reached real `__builtins__` before the denylist landed.
|
|
77
|
+
'"{0.__class__.__init__.__globals__[__builtins__][__import__]}".format(session)',
|
|
78
|
+
'"{0.__class__}".format(session)',
|
|
79
|
+
'str.format("{0.__class__}", session)',
|
|
80
|
+
'"{a.__class__}".format_map({"a": session})',
|
|
81
|
+
# A reasoning string is where a leak would surface — block it there too.
|
|
82
|
+
'EvalResult(score=Score(0.5), reasoning="{0.__class__}".format(session))',
|
|
83
|
+
],
|
|
84
|
+
)
|
|
85
|
+
def test_restricted_expressions_reject_format_string_traversal(source):
|
|
86
|
+
with pytest.raises(UnsafeEvaluatorSource):
|
|
87
|
+
compile_evaluator(source)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@pytest.mark.parametrize(
|
|
91
|
+
"source",
|
|
92
|
+
[
|
|
93
|
+
# Generator/frame/code introspection reaches the eval globals and, via
|
|
94
|
+
# dict.update on them, could poison a shared namespace. None of these
|
|
95
|
+
# attribute names start with "_", so only the default-deny allowlist
|
|
96
|
+
# stops them.
|
|
97
|
+
"EvalResult(score=Score(0.5), reasoning=str((x for x in [1]).gi_frame.f_globals))",
|
|
98
|
+
"EvalResult(score=Score(0.5), reasoning=str((x for x in [1]).gi_code))",
|
|
99
|
+
'EvalResult(score=Score(0.5), reasoning=str((x for x in [1]).gi_frame.f_globals.update({"P": 1})))',
|
|
100
|
+
# `mro` is a public method on the type metaclass; it reaches `object`.
|
|
101
|
+
"EvalResult(score=Score(0.5), reasoning=str(str.mro()[-1]))",
|
|
102
|
+
"EvalResult(score=Score(0.5), reasoning=str(int.mro()))",
|
|
103
|
+
# A live function's identity would leak a heap pointer (ASLR defeat).
|
|
104
|
+
"EvalResult(score=Score(0.5), reasoning=str(EvalResult.result_items))",
|
|
105
|
+
],
|
|
106
|
+
)
|
|
107
|
+
def test_restricted_expressions_reject_introspection_attributes(source):
|
|
108
|
+
with pytest.raises(UnsafeEvaluatorSource, match="attribute"):
|
|
109
|
+
compile_evaluator(source)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def test_no_reachable_construct_leaks_a_heap_pointer_repr():
|
|
113
|
+
# An object's default repr (`<... object at 0x...>`) leaks a live host heap
|
|
114
|
+
# address (ASLR/memory-layout disclosure) if coerced into a result field.
|
|
115
|
+
# Generator expressions are rejected at compile; `enumerate` is not bound, so
|
|
116
|
+
# it raises NameError at run time and becomes a bounded failed run instead of
|
|
117
|
+
# a disclosure. Either way, a pointer must never reach a result string.
|
|
118
|
+
import re
|
|
119
|
+
|
|
120
|
+
from failproofai_sdk.evaluator.source import _SAFE_GLOBALS
|
|
121
|
+
|
|
122
|
+
with pytest.raises(UnsafeEvaluatorSource, match="GeneratorExp"):
|
|
123
|
+
compile_evaluator("EvalResult(score=Score(1.0), reasoning=str((x for x in [1])))")
|
|
124
|
+
|
|
125
|
+
evaluate = compile_evaluator(
|
|
126
|
+
"EvalResult(score=Score(1.0), reasoning=str(enumerate([1])))"
|
|
127
|
+
)
|
|
128
|
+
with pytest.raises(NameError):
|
|
129
|
+
evaluate(Session())
|
|
130
|
+
|
|
131
|
+
# No value bound into the evaluation namespace reprs to a heap pointer.
|
|
132
|
+
pointer = re.compile(r"0x[0-9a-fA-F]+")
|
|
133
|
+
leaky = {
|
|
134
|
+
name: repr(value)
|
|
135
|
+
for name, value in _SAFE_GLOBALS.items()
|
|
136
|
+
if name != "__builtins__" and pointer.search(repr(value))
|
|
137
|
+
}
|
|
138
|
+
assert leaky == {}
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@pytest.mark.parametrize(
|
|
142
|
+
"inner",
|
|
143
|
+
[
|
|
144
|
+
# A bound method's repr is `<... at 0xADDR>` — a live heap pointer. These
|
|
145
|
+
# methods stay allowlisted (real evals CALL them), but referencing one as a
|
|
146
|
+
# bare VALUE only serves to stringify that repr, so it is now rejected at
|
|
147
|
+
# COMPILE — the disclosure is closed at its source, not at the output.
|
|
148
|
+
"session.events[0].payload.get",
|
|
149
|
+
"''.join",
|
|
150
|
+
"'x'.encode",
|
|
151
|
+
"'a,b'.split",
|
|
152
|
+
],
|
|
153
|
+
)
|
|
154
|
+
def test_bare_bound_method_reference_is_rejected_at_compile(inner):
|
|
155
|
+
for field in (
|
|
156
|
+
f'EvalResult(score=Score(1.0), reasoning=str({inner}))',
|
|
157
|
+
f'EvalResult(score=Score(1.0), summary=str({inner}))',
|
|
158
|
+
f'EvalResult(score=Score(1.0, display_value=str({inner})))',
|
|
159
|
+
f'EvalResult(score=Score(1.0), labels=(str({inner}),))',
|
|
160
|
+
):
|
|
161
|
+
with pytest.raises(UnsafeEvaluatorSource, match="only to call it"):
|
|
162
|
+
compile_evaluator(field)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def test_heap_pointer_output_guard_bypasses_are_closed_at_compile():
|
|
166
|
+
# An adversarial-review finding: the output-boundary regex was anchored on `<`,
|
|
167
|
+
# so a managed source could keep the address while reshaping the wrapper text —
|
|
168
|
+
# str(...).replace("<",""), an f-string, or %-formatting all coerce a bound
|
|
169
|
+
# method at a point the old scan missed. Each needs a BARE bound-method
|
|
170
|
+
# reference, which the compile-time call-site rule now rejects outright.
|
|
171
|
+
bypasses = [
|
|
172
|
+
# str(...).replace("<","") strips the old regex's `<` anchor.
|
|
173
|
+
'EvalResult(score=Score(1.0), reasoning=str(dict().get).replace("<", ""))',
|
|
174
|
+
# f-strings coerce at the C level, past the `str` global.
|
|
175
|
+
'EvalResult(score=Score(1.0), reasoning=f"{session.events[0].payload.get}")',
|
|
176
|
+
# %-formatting coerces at the C level too.
|
|
177
|
+
'EvalResult(score=Score(1.0), reasoning="%s" % session.events[0].payload.get)',
|
|
178
|
+
]
|
|
179
|
+
for src in bypasses:
|
|
180
|
+
with pytest.raises(UnsafeEvaluatorSource, match="only to call it"):
|
|
181
|
+
compile_evaluator(src)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def test_called_methods_and_data_attributes_still_stringify():
|
|
185
|
+
# The call-site rule blocks only BARE method references. Calling methods and
|
|
186
|
+
# reading data attributes (both pointer-free) must still work — including the
|
|
187
|
+
# f-string and %-formatting paths — so legitimate evaluations are unaffected.
|
|
188
|
+
reasoning_call = compile_evaluator(
|
|
189
|
+
'EvalResult(score=Score(1.0), '
|
|
190
|
+
'reasoning=str(session.events[0].payload.get("tool_name")))'
|
|
191
|
+
)(Session())
|
|
192
|
+
assert reasoning_call.reasoning == "search"
|
|
193
|
+
|
|
194
|
+
fstring = compile_evaluator(
|
|
195
|
+
'EvalResult(score=Score(1.0), reasoning=f"n={session.event_count}")'
|
|
196
|
+
)(Session())
|
|
197
|
+
assert fstring.reasoning == "n=3"
|
|
198
|
+
|
|
199
|
+
percent = compile_evaluator(
|
|
200
|
+
'EvalResult(score=Score(1.0), reasoning="pct=%d" % (session.event_count * 10))'
|
|
201
|
+
)(Session())
|
|
202
|
+
assert percent.reasoning == "pct=30"
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def test_each_evaluation_gets_isolated_globals_so_it_cannot_poison_the_next():
|
|
206
|
+
# Even setting aside the allowlist, one evaluation must not be able to leave
|
|
207
|
+
# state behind for the next. Compiling and running twice must not share a
|
|
208
|
+
# mutable namespace.
|
|
209
|
+
from failproofai_sdk.evaluator.source import _fresh_globals
|
|
210
|
+
|
|
211
|
+
first = _fresh_globals()
|
|
212
|
+
second = _fresh_globals()
|
|
213
|
+
assert first is not second
|
|
214
|
+
assert first["__builtins__"] is not second["__builtins__"]
|
|
215
|
+
first["__poison__"] = "leaked"
|
|
216
|
+
assert "__poison__" not in second
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def test_format_denylist_does_not_block_legitimate_string_methods():
|
|
220
|
+
# The fix is a targeted denylist of `format`/`format_map`, not a ban on all
|
|
221
|
+
# string methods — ordinary evaluations must still compile and run.
|
|
222
|
+
evaluate = compile_evaluator(
|
|
223
|
+
'EvalResult(score=Score(0.9, passed=True), '
|
|
224
|
+
'reasoning="tools=" + str(session.event_count).upper())'
|
|
225
|
+
)
|
|
226
|
+
result = evaluate(Session())
|
|
227
|
+
assert result.reasoning == "tools=3"
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def test_restricted_expressions_reject_statements_and_oversized_source():
|
|
231
|
+
with pytest.raises(UnsafeEvaluatorSource, match="one expression"):
|
|
232
|
+
compile_evaluator("import os")
|
|
233
|
+
with pytest.raises(UnsafeEvaluatorSource, match="exceeds"):
|
|
234
|
+
compile_evaluator("x" * (MAX_EVALUATOR_SOURCE_BYTES + 1))
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def test_result_and_condition_types_are_checked_at_runtime():
|
|
238
|
+
with pytest.raises(TypeError, match="EvalResult"):
|
|
239
|
+
compile_evaluator("True")(Session())
|
|
240
|
+
with pytest.raises(TypeError, match="bool or ConditionResult"):
|
|
241
|
+
compile_condition("1")(Session())
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def test_source_checksum_covers_condition_and_evaluator_together():
|
|
245
|
+
base = source_checksum(None, "EvalResult()")
|
|
246
|
+
assert base == source_checksum(None, "EvalResult()")
|
|
247
|
+
assert base != source_checksum("True", "EvalResult()")
|
|
248
|
+
assert base != source_checksum(None, "EvalResult(summary='changed')")
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
# --- SEC-001: managed source cannot exhaust the worker (killable-fork sandbox) ---
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def test_compute_bomb_is_killed_within_its_budget():
|
|
255
|
+
# `sum(range(10**9))` would burn CPU for ~20s in-process, uncancellable — a
|
|
256
|
+
# CPU-bound loop (not a big allocation) so the wall-clock/CPU budget is what
|
|
257
|
+
# stops it, deterministically, rather than the memory ceiling. The forked
|
|
258
|
+
# sandbox kills it at its budget.
|
|
259
|
+
import time as _time
|
|
260
|
+
|
|
261
|
+
evaluate = compile_evaluator(
|
|
262
|
+
"EvalResult(score=Score(1.0), reasoning=str(sum(range(10**9))))",
|
|
263
|
+
timeout_seconds=1,
|
|
264
|
+
)
|
|
265
|
+
started = _time.monotonic()
|
|
266
|
+
with pytest.raises(EvaluationTimeout):
|
|
267
|
+
evaluate(Session())
|
|
268
|
+
assert _time.monotonic() - started < 5 # bounded by the ~1s budget, not ~20s
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def test_condition_compute_bomb_is_also_bounded():
|
|
272
|
+
# `10**9`, matching the evaluator bomb above, NOT `10**8`.
|
|
273
|
+
#
|
|
274
|
+
# The property under test is "a CPU bomb in a condition is stopped by the
|
|
275
|
+
# sandbox budget", and the bomb has to be big enough that it cannot finish
|
|
276
|
+
# inside that budget on ANY machine the suite runs on. At 10**8 it was only
|
|
277
|
+
# ~1.35 CPU-seconds against a 1-second budget — a 1.35x margin — so on a fast
|
|
278
|
+
# runner the sum simply completed and nothing timed out. It failed exactly
|
|
279
|
+
# that way on CI under Python 3.14, which is faster here than 3.13 (1.35s vs
|
|
280
|
+
# 1.44s measured), while passing locally: a machine-speed coin flip, not a
|
|
281
|
+
# real signal about the sandbox.
|
|
282
|
+
#
|
|
283
|
+
# 10**9 restores the ~13x margin the evaluator twin already had. It costs no
|
|
284
|
+
# extra wall-clock: the sandbox kills the child at its budget either way, so
|
|
285
|
+
# a bigger bomb only widens the gap between "killed" and "could have
|
|
286
|
+
# finished". Do not shrink it back.
|
|
287
|
+
condition = compile_condition("sum(range(10**9)) > 0", timeout_seconds=1)
|
|
288
|
+
with pytest.raises(EvaluationTimeout):
|
|
289
|
+
condition(Session())
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def test_literal_pow_exponent_bomb_is_rejected_at_compile():
|
|
293
|
+
with pytest.raises(UnsafeEvaluatorSource, match="exponent"):
|
|
294
|
+
compile_evaluator(f"EvalResult(score=Score(10 ** {MAX_POW_EXPONENT + 1}))")
|
|
295
|
+
# A small constant exponent stays allowed.
|
|
296
|
+
result = compile_evaluator(
|
|
297
|
+
"EvalResult(score=Score(1.0), reasoning=str(2 ** 3))"
|
|
298
|
+
)(Session())
|
|
299
|
+
assert result.reasoning == "8"
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def test_oversized_expression_is_rejected_at_compile():
|
|
303
|
+
huge = "[" + ",".join("1" for _ in range(MAX_AST_NODES)) + "]"
|
|
304
|
+
with pytest.raises(UnsafeEvaluatorSource, match="too large"):
|
|
305
|
+
compile_evaluator(f"EvalResult(score=Score(len({huge}) / len({huge})))")
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def test_normal_managed_eval_survives_the_fork_boundary():
|
|
309
|
+
# A session-dependent result must round-trip out of the forked child intact.
|
|
310
|
+
result = compile_evaluator(
|
|
311
|
+
"EvalResult(score=Score(1.0 if session.event_count > 0 else 0.0), "
|
|
312
|
+
"reasoning=str(session.event_count))"
|
|
313
|
+
)(Session())
|
|
314
|
+
assert isinstance(result, EvalResult)
|
|
315
|
+
assert result.score.value == 1.0
|
|
316
|
+
assert result.reasoning == "3"
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def test_sandbox_fails_closed_without_a_serializable_transcript():
|
|
320
|
+
# The transcript crosses into the subprocess via `to_wire`. A session that
|
|
321
|
+
# can't be serialized cannot be sandboxed, so refuse rather than run unbounded.
|
|
322
|
+
class NotATranscript:
|
|
323
|
+
event_count = 1
|
|
324
|
+
|
|
325
|
+
with pytest.raises(EvaluationSandboxUnavailable):
|
|
326
|
+
compile_evaluator("EvalResult(score=Score(1.0))")(NotATranscript())
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def test_server_timeout_cannot_exceed_the_hard_ceiling():
|
|
330
|
+
# SEC-001: a large server-provided timeout must not remove the execution bound.
|
|
331
|
+
assert _clamp_budget(10**9) == float(MAX_SANDBOX_TIMEOUT_SECONDS)
|
|
332
|
+
assert _clamp_budget(0) == 30.0
|
|
333
|
+
assert _clamp_budget(None) == 30.0
|
|
334
|
+
assert _clamp_budget(5) == 5.0
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def test_oversized_result_is_rejected_before_it_crosses_back():
|
|
338
|
+
# A result with far more than the 25-item limit must be rejected INSIDE the
|
|
339
|
+
# sandbox (via result_items), so a huge result can never be serialized and
|
|
340
|
+
# shipped back to OOM the worker (SEC-001).
|
|
341
|
+
src = (
|
|
342
|
+
"EvalResult(score=Score(1.0), "
|
|
343
|
+
"metrics={'m' + str(i): float(i) for i in range(200)})"
|
|
344
|
+
)
|
|
345
|
+
with pytest.raises(ValueError, match="at most"):
|
|
346
|
+
compile_evaluator(src, eval_key="q")(Session())
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def test_sandbox_slot_wait_counts_against_the_timeout(monkeypatch):
|
|
350
|
+
# SEC-001: acquiring a concurrency slot must count against the wall-clock budget.
|
|
351
|
+
# `asyncio.wait_for` only cancels the awaiter, so a run that blocked UNBOUNDED on
|
|
352
|
+
# a busy slot would still launch a sandbox after its caller was reported timed
|
|
353
|
+
# out — 28 threads could queue behind 4 long sandboxes and starve the worker.
|
|
354
|
+
# With one slot and three 1s compute bombs, all three must resolve within ~one
|
|
355
|
+
# budget (the holder is killed at ~1s; the two queued behind it exhaust their
|
|
356
|
+
# budget waiting and time out WITHOUT ever spawning a child), not three serialized
|
|
357
|
+
# budgets (~3s).
|
|
358
|
+
import threading
|
|
359
|
+
import time as _time
|
|
360
|
+
|
|
361
|
+
from failproofai_sdk.evaluator import source as _source
|
|
362
|
+
|
|
363
|
+
monkeypatch.setattr(_source, "_SANDBOX_SLOTS", threading.Semaphore(1))
|
|
364
|
+
bomb = compile_evaluator(
|
|
365
|
+
"EvalResult(score=Score(1.0), reasoning=str(sum(range(10**9))))",
|
|
366
|
+
timeout_seconds=1,
|
|
367
|
+
)
|
|
368
|
+
session = Session()
|
|
369
|
+
errors: list[str] = []
|
|
370
|
+
|
|
371
|
+
def run():
|
|
372
|
+
try:
|
|
373
|
+
bomb(session)
|
|
374
|
+
except Exception as error: # noqa: BLE001
|
|
375
|
+
errors.append(type(error).__name__)
|
|
376
|
+
|
|
377
|
+
threads = [threading.Thread(target=run) for _ in range(3)]
|
|
378
|
+
started = _time.monotonic()
|
|
379
|
+
for thread in threads:
|
|
380
|
+
thread.start()
|
|
381
|
+
for thread in threads:
|
|
382
|
+
thread.join(timeout=15)
|
|
383
|
+
elapsed = _time.monotonic() - started
|
|
384
|
+
|
|
385
|
+
assert errors == ["EvaluationTimeout"] * 3, errors
|
|
386
|
+
# Bounded by ~one budget, NOT three serialized ones — proving the queued runs
|
|
387
|
+
# timed out on slot acquisition instead of each waiting then running in turn.
|
|
388
|
+
assert elapsed < 2.5, f"queued sandboxes were not bounded by the timeout: {elapsed:.2f}s"
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def test_allocation_bomb_is_bounded_by_the_per_sandbox_memory_limit():
|
|
392
|
+
# A ~1.6 GiB allocation exceeds the per-sandbox RLIMIT_AS and is killed, so it
|
|
393
|
+
# cannot exhaust the worker even wrapped in an otherwise-valid result. With the
|
|
394
|
+
# concurrent-sandbox cap this also bounds the aggregate across concurrent runs.
|
|
395
|
+
src = "EvalResult(score=Score(1.0 if len([0] * 200000000) >= 0 else 0.0))"
|
|
396
|
+
with pytest.raises((EvaluationTimeout, MemoryError)):
|
|
397
|
+
compile_evaluator(src, timeout_seconds=5)(Session())
|
|
398
|
+
|
|
399
|
+
|
|
400
|
+
def test_sandbox_fails_closed_when_kernel_resource_limits_are_unavailable(monkeypatch):
|
|
401
|
+
# SEC-001: on a platform without the stdlib ``resource`` module (e.g. Windows),
|
|
402
|
+
# the sandbox child cannot install RLIMIT_CPU / RLIMIT_AS on itself, so managed
|
|
403
|
+
# source must be refused BEFORE any child is spawned rather than run unbounded.
|
|
404
|
+
import failproofai_sdk.evaluator.source as source
|
|
405
|
+
|
|
406
|
+
monkeypatch.setattr(source, "_resource", None)
|
|
407
|
+
|
|
408
|
+
def _no_spawn(*args, **kwargs):
|
|
409
|
+
raise AssertionError("a sandbox child must not be started when limits are unavailable")
|
|
410
|
+
|
|
411
|
+
monkeypatch.setattr(source.subprocess, "Popen", _no_spawn)
|
|
412
|
+
|
|
413
|
+
with pytest.raises(EvaluationSandboxUnavailable):
|
|
414
|
+
source.compile_evaluator("EvalResult(score=Score(1.0))")(Session())
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def test_comprehension_body_can_read_session():
|
|
418
|
+
# A list/set/dict comprehension resolves a free name like `session` from
|
|
419
|
+
# GLOBALS. When `session` was only in eval locals, such a source raised
|
|
420
|
+
# NameError on CPython 3.10 (a supported version). Both sandbox paths must
|
|
421
|
+
# now evaluate a session-dependent comprehension (COR-001).
|
|
422
|
+
assert (
|
|
423
|
+
compile_condition("len([session.event_count for i in range(1)]) > 0")(Session())
|
|
424
|
+
is True
|
|
425
|
+
)
|
|
426
|
+
result = compile_evaluator(
|
|
427
|
+
"EvalResult(score=Score(1.0), "
|
|
428
|
+
"reasoning=str([session.event_count for i in range(1)]))"
|
|
429
|
+
)(Session(event_count=3))
|
|
430
|
+
assert result.reasoning == "[3]"
|
|
@@ -300,6 +300,23 @@ def test_importing_the_package_loads_no_framework():
|
|
|
300
300
|
)
|
|
301
301
|
|
|
302
302
|
|
|
303
|
+
def test_importing_the_package_does_not_load_the_evaluator_runtime():
|
|
304
|
+
"""Telemetry-only users do not pay for the separate worker surface."""
|
|
305
|
+
import json
|
|
306
|
+
import subprocess
|
|
307
|
+
|
|
308
|
+
probe = (
|
|
309
|
+
"import json, sys; import failproofai_sdk; "
|
|
310
|
+
"print(json.dumps(sorted(m for m in sys.modules "
|
|
311
|
+
"if m.startswith('failproofai_sdk.evaluator'))))"
|
|
312
|
+
)
|
|
313
|
+
result = subprocess.run(
|
|
314
|
+
[sys.executable, "-c", probe], capture_output=True, text=True, cwd=str(ROOT), timeout=60
|
|
315
|
+
)
|
|
316
|
+
assert result.returncode == 0, result.stderr
|
|
317
|
+
assert json.loads(result.stdout.strip()) == []
|
|
318
|
+
|
|
319
|
+
|
|
303
320
|
def test_the_adapter_registry_holds_strings_not_modules():
|
|
304
321
|
"""`_REGISTRY` maps a name to a dotted path; importing it here would defeat it."""
|
|
305
322
|
from failproofai_sdk.integrations import _REGISTRY
|
|
@@ -2,14 +2,14 @@ version = 1
|
|
|
2
2
|
revision = 3
|
|
3
3
|
requires-python = ">=3.10"
|
|
4
4
|
resolution-markers = [
|
|
5
|
-
"python_full_version == '3.13.*'",
|
|
6
5
|
"python_full_version >= '3.14' and sys_platform == 'win32'",
|
|
7
6
|
"python_full_version >= '3.14' and sys_platform != 'win32'",
|
|
7
|
+
"python_full_version == '3.13.*'",
|
|
8
8
|
"python_full_version == '3.12.*' and sys_platform == 'win32'",
|
|
9
|
-
"python_full_version == '3.11.*' and sys_platform == 'win32'",
|
|
10
|
-
"python_full_version < '3.11' and sys_platform == 'win32'",
|
|
11
9
|
"python_full_version == '3.12.*' and sys_platform != 'win32'",
|
|
10
|
+
"python_full_version == '3.11.*' and sys_platform == 'win32'",
|
|
12
11
|
"python_full_version == '3.11.*' and sys_platform != 'win32'",
|
|
12
|
+
"python_full_version < '3.11' and sys_platform == 'win32'",
|
|
13
13
|
"python_full_version < '3.11' and sys_platform != 'win32'",
|
|
14
14
|
]
|
|
15
15
|
|
|
@@ -751,7 +751,7 @@ wheels = [
|
|
|
751
751
|
|
|
752
752
|
[[package]]
|
|
753
753
|
name = "crewai"
|
|
754
|
-
version = "1.15.
|
|
754
|
+
version = "1.15.20"
|
|
755
755
|
source = { registry = "https://pypi.org/simple" }
|
|
756
756
|
dependencies = [
|
|
757
757
|
{ name = "aiofiles" },
|
|
@@ -786,14 +786,14 @@ dependencies = [
|
|
|
786
786
|
{ name = "tomli" },
|
|
787
787
|
{ name = "tomli-w" },
|
|
788
788
|
]
|
|
789
|
-
sdist = { url = "https://files.pythonhosted.org/packages/
|
|
789
|
+
sdist = { url = "https://files.pythonhosted.org/packages/c7/07/3a6d94fbb3bd465c653208a28d3e59d539d00cb5bef55626188a46ed7457/crewai-1.15.20.tar.gz", hash = "sha256:c4a4d02c27eca7502479ce8cf3fcdd1658fcc9948aa86962ad19e581530032b4", size = 7965515, upload-time = "2026-09-04T12:42:59.848Z" }
|
|
790
790
|
wheels = [
|
|
791
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
791
|
+
{ url = "https://files.pythonhosted.org/packages/b0/cd/fa00e87bf061e8bb02546cd00f58c4d85694b0fe027afae5eb1f578c610c/crewai-1.15.20-py3-none-any.whl", hash = "sha256:8bf272a687ad3320b681f23a183a6ac445f2fa7fc78b52943b835acc2eb158d6", size = 1141815, upload-time = "2026-09-04T12:42:58.166Z" },
|
|
792
792
|
]
|
|
793
793
|
|
|
794
794
|
[[package]]
|
|
795
795
|
name = "crewai-cli"
|
|
796
|
-
version = "1.15.
|
|
796
|
+
version = "1.15.20"
|
|
797
797
|
source = { registry = "https://pypi.org/simple" }
|
|
798
798
|
dependencies = [
|
|
799
799
|
{ name = "appdirs" },
|
|
@@ -813,14 +813,14 @@ dependencies = [
|
|
|
813
813
|
{ name = "tomli-w" },
|
|
814
814
|
{ name = "uv" },
|
|
815
815
|
]
|
|
816
|
-
sdist = { url = "https://files.pythonhosted.org/packages/
|
|
816
|
+
sdist = { url = "https://files.pythonhosted.org/packages/2e/30/8521f26795a71179b684c5b2d8135e78f8b4fcabe79acae93e0b3d9a42e2/crewai_cli-1.15.20.tar.gz", hash = "sha256:21f1735ee7e6818f658f46d59844d6f04700af5398c632a3e0fd42c8e6180c93", size = 233643, upload-time = "2026-09-04T12:43:02.502Z" }
|
|
817
817
|
wheels = [
|
|
818
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
818
|
+
{ url = "https://files.pythonhosted.org/packages/b7/eb/a04ae71e6226c10ae921911a8b8216cef6ae5d1db98c5b36f941058728c3/crewai_cli-1.15.20-py3-none-any.whl", hash = "sha256:2d692143cb4a46ef97f927fc1a2f6035790d2ee2fe4d1bb50fad9f884576ce6a", size = 198183, upload-time = "2026-09-04T12:43:01.266Z" },
|
|
819
819
|
]
|
|
820
820
|
|
|
821
821
|
[[package]]
|
|
822
822
|
name = "crewai-core"
|
|
823
|
-
version = "1.15.
|
|
823
|
+
version = "1.15.20"
|
|
824
824
|
source = { registry = "https://pypi.org/simple" }
|
|
825
825
|
dependencies = [
|
|
826
826
|
{ name = "appdirs" },
|
|
@@ -836,9 +836,9 @@ dependencies = [
|
|
|
836
836
|
{ name = "rich" },
|
|
837
837
|
{ name = "tomli" },
|
|
838
838
|
]
|
|
839
|
-
sdist = { url = "https://files.pythonhosted.org/packages/
|
|
839
|
+
sdist = { url = "https://files.pythonhosted.org/packages/c8/7f/75b9f75f90f6639edb68e1591bc6fd54ef1381cbdd4f10f41f62ef12b985/crewai_core-1.15.20.tar.gz", hash = "sha256:5a3ad81107172f24b6ee14a24aff87f93e3b56da7220a0e5584992d9b9476939", size = 39207, upload-time = "2026-09-04T12:43:04.352Z" }
|
|
840
840
|
wheels = [
|
|
841
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
841
|
+
{ url = "https://files.pythonhosted.org/packages/b8/6a/f5e1f1114a6900275e240feffae6580ddccc6f5e4716372b3ec97511e3ab/crewai_core-1.15.20-py3-none-any.whl", hash = "sha256:09a96fbd08e2f849e4a521ecfdab88b62fcb3b94292f4e7039c0bc3b9de67d40", size = 42481, upload-time = "2026-09-04T12:43:03.421Z" },
|
|
842
842
|
]
|
|
843
843
|
|
|
844
844
|
[[package]]
|
|
@@ -1200,15 +1200,15 @@ wheels = [
|
|
|
1200
1200
|
|
|
1201
1201
|
[[package]]
|
|
1202
1202
|
name = "genai-prices"
|
|
1203
|
-
version = "0.1.
|
|
1203
|
+
version = "0.1.6"
|
|
1204
1204
|
source = { registry = "https://pypi.org/simple" }
|
|
1205
1205
|
dependencies = [
|
|
1206
1206
|
{ name = "httpx2" },
|
|
1207
1207
|
{ name = "pydantic" },
|
|
1208
1208
|
]
|
|
1209
|
-
sdist = { url = "https://files.pythonhosted.org/packages/
|
|
1209
|
+
sdist = { url = "https://files.pythonhosted.org/packages/1a/16/e5a507d42c0eb629b48ebe6c278f2d8c3f929bb6b28f18108bdd66d8ae12/genai_prices-0.1.6.tar.gz", hash = "sha256:802c1e4cc3ed5e70a09083b83af441a58d91f62e12768f7f1b6b26c98a33fcac", size = 111810, upload-time = "2026-09-02T14:53:54.895Z" }
|
|
1210
1210
|
wheels = [
|
|
1211
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
1211
|
+
{ url = "https://files.pythonhosted.org/packages/48/a1/43fa2a4c5557cd977e83eecec265b0b77b726b0c7b9f2f180b46c6fdb458/genai_prices-0.1.6-py3-none-any.whl", hash = "sha256:35ac8043dbcf2958488129413bfecba7304fe12a68ad4a78c5b0d15281e82814", size = 118834, upload-time = "2026-09-02T14:53:53.758Z" },
|
|
1212
1212
|
]
|
|
1213
1213
|
|
|
1214
1214
|
[[package]]
|
|
@@ -2003,7 +2003,7 @@ wheels = [
|
|
|
2003
2003
|
|
|
2004
2004
|
[[package]]
|
|
2005
2005
|
name = "langchain-core"
|
|
2006
|
-
version = "1.6.
|
|
2006
|
+
version = "1.6.1"
|
|
2007
2007
|
source = { registry = "https://pypi.org/simple" }
|
|
2008
2008
|
dependencies = [
|
|
2009
2009
|
{ name = "httpx" },
|
|
@@ -2017,9 +2017,9 @@ dependencies = [
|
|
|
2017
2017
|
{ name = "typing-extensions" },
|
|
2018
2018
|
{ name = "uuid-utils" },
|
|
2019
2019
|
]
|
|
2020
|
-
sdist = { url = "https://files.pythonhosted.org/packages/
|
|
2020
|
+
sdist = { url = "https://files.pythonhosted.org/packages/90/12/aff76ca89c219ebe6f9dd3c5dbc4e3b1cf5450e9fc7037dccad23d45cd7a/langchain_core-1.6.1.tar.gz", hash = "sha256:1b156cb395aac4f009a8a1b38a574c7d948fe2d5f74c96e0d8a5017b4149e04f", size = 1003359, upload-time = "2026-08-27T19:31:14.956Z" }
|
|
2021
2021
|
wheels = [
|
|
2022
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
2022
|
+
{ url = "https://files.pythonhosted.org/packages/8e/25/f50dd65673c819aa33d3c34df58c115dbb6ec627d19f93e6e401dd0fc8d7/langchain_core-1.6.1-py3-none-any.whl", hash = "sha256:954a84132a5cb0435d27b910e336347b6744ecc18fbeef1e2de7029a0959841a", size = 571478, upload-time = "2026-08-27T19:31:13.34Z" },
|
|
2023
2023
|
]
|
|
2024
2024
|
|
|
2025
2025
|
[[package]]
|
|
@@ -2674,12 +2674,12 @@ name = "networkx"
|
|
|
2674
2674
|
version = "3.6.1"
|
|
2675
2675
|
source = { registry = "https://pypi.org/simple" }
|
|
2676
2676
|
resolution-markers = [
|
|
2677
|
-
"python_full_version == '3.13.*'",
|
|
2678
2677
|
"python_full_version >= '3.14' and sys_platform == 'win32'",
|
|
2679
2678
|
"python_full_version >= '3.14' and sys_platform != 'win32'",
|
|
2679
|
+
"python_full_version == '3.13.*'",
|
|
2680
2680
|
"python_full_version == '3.12.*' and sys_platform == 'win32'",
|
|
2681
|
-
"python_full_version == '3.11.*' and sys_platform == 'win32'",
|
|
2682
2681
|
"python_full_version == '3.12.*' and sys_platform != 'win32'",
|
|
2682
|
+
"python_full_version == '3.11.*' and sys_platform == 'win32'",
|
|
2683
2683
|
"python_full_version == '3.11.*' and sys_platform != 'win32'",
|
|
2684
2684
|
]
|
|
2685
2685
|
sdist = { url = "https://files.pythonhosted.org/packages/6a/51/63fe664f3908c97be9d2e4f1158eb633317598cfa6e1fc14af5383f17512/networkx-3.6.1.tar.gz", hash = "sha256:26b7c357accc0c8cde558ad486283728b65b6a95d85ee1cd66bafab4c8168509", size = 2517025, upload-time = "2025-12-08T17:02:39.908Z" }
|
|
@@ -2857,9 +2857,9 @@ name = "numpy"
|
|
|
2857
2857
|
version = "2.5.2"
|
|
2858
2858
|
source = { registry = "https://pypi.org/simple" }
|
|
2859
2859
|
resolution-markers = [
|
|
2860
|
-
"python_full_version == '3.13.*'",
|
|
2861
2860
|
"python_full_version >= '3.14' and sys_platform == 'win32'",
|
|
2862
2861
|
"python_full_version >= '3.14' and sys_platform != 'win32'",
|
|
2862
|
+
"python_full_version == '3.13.*'",
|
|
2863
2863
|
"python_full_version == '3.12.*' and sys_platform == 'win32'",
|
|
2864
2864
|
"python_full_version == '3.12.*' and sys_platform != 'win32'",
|
|
2865
2865
|
]
|
|
@@ -3906,7 +3906,7 @@ wheels = [
|
|
|
3906
3906
|
|
|
3907
3907
|
[[package]]
|
|
3908
3908
|
name = "pydantic-ai-slim"
|
|
3909
|
-
version = "2.
|
|
3909
|
+
version = "2.39.0"
|
|
3910
3910
|
source = { registry = "https://pypi.org/simple" }
|
|
3911
3911
|
dependencies = [
|
|
3912
3912
|
{ name = "anyio" },
|
|
@@ -3919,9 +3919,9 @@ dependencies = [
|
|
|
3919
3919
|
{ name = "pydantic-graph" },
|
|
3920
3920
|
{ name = "typing-inspection" },
|
|
3921
3921
|
]
|
|
3922
|
-
sdist = { url = "https://files.pythonhosted.org/packages/
|
|
3922
|
+
sdist = { url = "https://files.pythonhosted.org/packages/e1/f9/17eb24ebd9c8dbc9812f94d15534e440df3fb9979bb351895cd9b83eb8cc/pydantic_ai_slim-2.39.0.tar.gz", hash = "sha256:7c4c6cba3f1658e456fe1686288667aaf4fda71bda6a95740304fef540549bc2", size = 1375061, upload-time = "2026-09-04T04:30:22.756Z" }
|
|
3923
3923
|
wheels = [
|
|
3924
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
3924
|
+
{ url = "https://files.pythonhosted.org/packages/7a/62/fe1c2bbab6af6ea238c447e68f0113e2c555625c828f57575e8918214a81/pydantic_ai_slim-2.39.0-py3-none-any.whl", hash = "sha256:c8e7cd0b027a383fdf03ef6341737099dbe0f0b5b890a98a1461c4652e79e1b2", size = 1618873, upload-time = "2026-09-04T04:30:14.699Z" },
|
|
3925
3925
|
]
|
|
3926
3926
|
|
|
3927
3927
|
[[package]]
|
|
@@ -4044,7 +4044,7 @@ wheels = [
|
|
|
4044
4044
|
|
|
4045
4045
|
[[package]]
|
|
4046
4046
|
name = "pydantic-graph"
|
|
4047
|
-
version = "2.
|
|
4047
|
+
version = "2.39.0"
|
|
4048
4048
|
source = { registry = "https://pypi.org/simple" }
|
|
4049
4049
|
dependencies = [
|
|
4050
4050
|
{ name = "anyio" },
|
|
@@ -4052,9 +4052,9 @@ dependencies = [
|
|
|
4052
4052
|
{ name = "pydantic" },
|
|
4053
4053
|
{ name = "typing-inspection" },
|
|
4054
4054
|
]
|
|
4055
|
-
sdist = { url = "https://files.pythonhosted.org/packages/
|
|
4055
|
+
sdist = { url = "https://files.pythonhosted.org/packages/1a/11/64fe78978331bc4716af4be6f1be14c6d0f59176e311f42e91da41c0b8bd/pydantic_graph-2.39.0.tar.gz", hash = "sha256:38afcb88c6dce861896fe7185735419eb435b61ff26a756260087df928b5b22e", size = 45191, upload-time = "2026-09-04T04:30:24.94Z" }
|
|
4056
4056
|
wheels = [
|
|
4057
|
-
{ url = "https://files.pythonhosted.org/packages/
|
|
4057
|
+
{ url = "https://files.pythonhosted.org/packages/f8/79/d0944b6d15ecc7930539cca29b17a5af219a62f0d97f257370617b02f02e/pydantic_graph-2.39.0-py3-none-any.whl", hash = "sha256:d754f1f0f73e5dc45e2f1f9019270d6409e4bd35e141a6f14af41d38c06e766f", size = 52701, upload-time = "2026-09-04T04:30:18.145Z" },
|
|
4058
4058
|
]
|
|
4059
4059
|
|
|
4060
4060
|
[[package]]
|
|
@@ -4627,12 +4627,12 @@ name = "rpds-py"
|
|
|
4627
4627
|
version = "2026.6.3"
|
|
4628
4628
|
source = { registry = "https://pypi.org/simple" }
|
|
4629
4629
|
resolution-markers = [
|
|
4630
|
-
"python_full_version == '3.13.*'",
|
|
4631
4630
|
"python_full_version >= '3.14' and sys_platform == 'win32'",
|
|
4632
4631
|
"python_full_version >= '3.14' and sys_platform != 'win32'",
|
|
4632
|
+
"python_full_version == '3.13.*'",
|
|
4633
4633
|
"python_full_version == '3.12.*' and sys_platform == 'win32'",
|
|
4634
|
-
"python_full_version == '3.11.*' and sys_platform == 'win32'",
|
|
4635
4634
|
"python_full_version == '3.12.*' and sys_platform != 'win32'",
|
|
4635
|
+
"python_full_version == '3.11.*' and sys_platform == 'win32'",
|
|
4636
4636
|
"python_full_version == '3.11.*' and sys_platform != 'win32'",
|
|
4637
4637
|
]
|
|
4638
4638
|
sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" }
|