failproofai 1.0.1 → 1.0.2-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.next/standalone/.next/BUILD_ID +1 -1
- package/.next/standalone/.next/build-manifest.json +3 -3
- package/.next/standalone/.next/prerender-manifest.json +6 -6
- package/.next/standalone/.next/required-server-files.json +1 -1
- package/.next/standalone/.next/server/app/_global-error/page/server-reference-manifest.json +1 -1
- package/.next/standalone/.next/server/app/_global-error/page.js +2 -2
- package/.next/standalone/.next/server/app/_global-error/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/_global-error/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/_global-error.html +1 -1
- package/.next/standalone/.next/server/app/_global-error.rsc +7 -7
- package/.next/standalone/.next/server/app/_global-error.segments/__PAGE__.segment.rsc +6 -6
- package/.next/standalone/.next/server/app/_global-error.segments/_full.segment.rsc +7 -7
- package/.next/standalone/.next/server/app/_global-error.segments/_tree.segment.rsc +1 -1
- package/.next/standalone/.next/server/app/_not-found/page/server-reference-manifest.json +1 -1
- package/.next/standalone/.next/server/app/_not-found/page.js +2 -2
- package/.next/standalone/.next/server/app/_not-found/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/_not-found/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/_not-found.html +1 -1
- package/.next/standalone/.next/server/app/_not-found.rsc +15 -15
- package/.next/standalone/.next/server/app/_not-found.segments/_full.segment.rsc +15 -15
- package/.next/standalone/.next/server/app/_not-found.segments/_not-found/__PAGE__.segment.rsc +14 -14
- package/.next/standalone/.next/server/app/_not-found.segments/_tree.segment.rsc +2 -2
- package/.next/standalone/.next/server/app/api/audit/invite/route.js +3 -2
- package/.next/standalone/.next/server/app/api/audit/invite/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/audit/run/route.js +5 -4
- package/.next/standalone/.next/server/app/api/audit/run/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/audit/status/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/auth/login-request/route.js +2 -2
- package/.next/standalone/.next/server/app/api/auth/login-request/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/auth/login-verify/route.js +4 -3
- package/.next/standalone/.next/server/app/api/auth/login-verify/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/auth/logout/route.js +4 -4
- package/.next/standalone/.next/server/app/api/auth/logout/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/auth/status/route.js +4 -4
- package/.next/standalone/.next/server/app/api/auth/status/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/download/[project]/[session]/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/audit/page/server-reference-manifest.json +2 -2
- package/.next/standalone/.next/server/app/audit/page.js +2 -2
- package/.next/standalone/.next/server/app/audit/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/audit/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/index.html +1 -1
- package/.next/standalone/.next/server/app/index.rsc +15 -15
- package/.next/standalone/.next/server/app/index.segments/__PAGE__.segment.rsc +14 -14
- package/.next/standalone/.next/server/app/index.segments/_full.segment.rsc +15 -15
- package/.next/standalone/.next/server/app/index.segments/_tree.segment.rsc +2 -2
- package/.next/standalone/.next/server/app/page/server-reference-manifest.json +1 -1
- package/.next/standalone/.next/server/app/page.js +2 -2
- package/.next/standalone/.next/server/app/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/policies/page/server-reference-manifest.json +66 -22
- package/.next/standalone/.next/server/app/policies/page.js +5 -4
- package/.next/standalone/.next/server/app/policies/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/policies/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/project/[name]/page/server-reference-manifest.json +1 -1
- package/.next/standalone/.next/server/app/project/[name]/page.js +2 -2
- package/.next/standalone/.next/server/app/project/[name]/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/project/[name]/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page/react-loadable-manifest.json +2 -2
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page/server-reference-manifest.json +2 -2
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page.js +2 -2
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/projects/page/server-reference-manifest.json +1 -1
- package/.next/standalone/.next/server/app/projects/page.js +2 -2
- package/.next/standalone/.next/server/app/projects/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/projects/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/settings/page/server-reference-manifest.json +4 -4
- package/.next/standalone/.next/server/app/settings/page.js +1 -1
- package/.next/standalone/.next/server/app/settings/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/settings/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/chunks/{[externals]__1qzqcio._.js → [externals]__0e32-7c._.js} +1 -1
- package/.next/standalone/.next/server/chunks/[externals]__1s462yl._.js +3 -0
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__01eqny5._.js +3 -0
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0_0xu5z._.js +1 -1
- package/.next/standalone/.next/server/chunks/{[root-of-the-server]__1p8poxd._.js → [root-of-the-server]__0igxt32._.js} +2 -2
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0neqlu-._.js +3 -0
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0o07qi9._.js +3 -0
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0t9oqnn._.js +3 -0
- package/.next/standalone/.next/server/chunks/_0bs-v8p._.js +3 -0
- package/.next/standalone/.next/server/chunks/_0tovk6q._.js +3 -0
- package/.next/standalone/.next/server/chunks/_0trp3yc._.js +3 -0
- package/.next/standalone/.next/server/chunks/_185gyk9._.js +3 -0
- package/.next/standalone/.next/server/chunks/_1ek68ln._.js +5 -6
- package/.next/standalone/.next/server/chunks/lib_telemetry-id_ts_1k8gj9o._.js +3 -0
- package/.next/standalone/.next/server/chunks/{node_modules_posthog-node_dist_entrypoints_index_node_mjs_09d5c79._.js → node_modules_posthog-node_dist_entrypoints_index_node_mjs_04op_ou._.js} +1 -1
- package/.next/standalone/.next/server/chunks/{node_modules_posthog-node_dist_entrypoints_index_node_mjs_1d2g73o._.js → node_modules_posthog-node_dist_entrypoints_index_node_mjs_0dr20uz._.js} +1 -1
- package/.next/standalone/.next/server/chunks/package_json_[json]_cjs_1nxcc4v._.js +1 -1
- package/.next/standalone/.next/server/chunks/src_hooks_0xbw3bu._.js +12 -0
- package/.next/standalone/.next/server/chunks/src_hooks_fp-home_ts_09kv0bn._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__013jr2b._.js +4 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__01wy8d-._.js +4 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__02npjtd._.js +4 -0
- package/.next/standalone/.next/server/chunks/ssr/{[root-of-the-server]__0spkm68._.js → [root-of-the-server]__07gm3zl._.js} +21 -20
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0da85px._.js +4 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0ftmoxc._.js +4 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0oa1lav._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0p-5p8u._.js +4 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0s740oi._.js +6 -0
- package/.next/standalone/.next/server/chunks/ssr/{[root-of-the-server]__07tbbf3._.js → [root-of-the-server]__0zig5sv._.js} +2 -2
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__192nhvh._.js +5 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__1p2otjt._.js +4 -0
- package/.next/standalone/.next/server/chunks/ssr/{_0lvyuv4._.js → _00e9tnf._.js} +2 -2
- package/.next/standalone/.next/server/chunks/ssr/_08x1r5t._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/{_0u5zauv._.js → _12l4a_k._.js} +1 -1
- package/.next/standalone/.next/server/chunks/ssr/_1u8-lu2._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/_1zopuov._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/app_audit__components_audit-dashboard_tsx_0p9ud47._.js +13 -13
- package/.next/standalone/.next/server/chunks/ssr/app_global-error_tsx_1kp6l3x._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/app_policies_hooks-client_tsx_19dqvpc._.js +2 -2
- package/.next/standalone/.next/server/chunks/ssr/app_settings_settings-client_tsx_20lq-mq._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/src_hooks_08o7bqo._.js +10 -0
- package/.next/standalone/.next/server/chunks/ssr/src_hooks_builtin-policies_ts_09j2ndl._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/src_hooks_fp-home_ts_0je3xkv._.js +1 -1
- package/.next/standalone/.next/server/middleware-build-manifest.js +3 -3
- package/.next/standalone/.next/server/middleware.js +1 -1
- package/.next/standalone/.next/server/pages/404.html +1 -1
- package/.next/standalone/.next/server/pages/500.html +1 -1
- package/.next/standalone/.next/server/server-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/server-reference-manifest.json +71 -27
- package/.next/standalone/.next/static/chunks/{37fcpc36x9gpe.js → 0eqky-7n7dgdm.js} +1 -1
- package/.next/standalone/.next/static/chunks/{1ide1wl2f7zi1.js → 0sa9v5mhc8861.js} +13 -13
- package/.next/standalone/.next/static/chunks/1mfzsja6ftim5.css +2 -0
- package/.next/standalone/.next/static/chunks/{3k7mxbp9hiom0.js → 1nuqytbta44af.js} +1 -1
- package/.next/standalone/.next/static/chunks/1sl2xmn9inx3y.js +6 -0
- package/.next/standalone/.next/static/chunks/{2_2lhuojq2xop.js → 2aawsjld9ied_.js} +1 -1
- package/.next/standalone/.next/static/chunks/2gn7ut4svofm2.js +1 -0
- package/.next/standalone/.next/static/chunks/{31s1smp-g7gks.js → 2hs7ycvsav7v2.js} +1 -1
- package/.next/standalone/.next/static/chunks/{0k5050ezjvv49.js → 2i2yt3hzjredm.js} +1 -1
- package/.next/standalone/.next/static/chunks/{3c808qrfcq0im.js → 35xg6retpe6rp.js} +1 -1
- package/.next/standalone/.next/static/chunks/{2zpp1we2jv6tb.js → 3y54nw0i7dsjl.js} +1 -1
- package/.next/standalone/SECURITY.md +5 -4
- package/.next/standalone/app/actions/get-hooks-config.ts +108 -17
- package/.next/standalone/app/actions/pack-actions.ts +153 -0
- package/.next/standalone/app/audit/_components/run-progress.tsx +1 -1
- package/.next/standalone/app/audit/_components/share-templates.ts +13 -13
- package/.next/standalone/app/policies/hooks-client.tsx +320 -14
- package/.next/standalone/components/navbar.tsx +2 -28
- package/.next/standalone/components/reach-developers.tsx +8 -1
- package/.next/standalone/docs-old/agenteye/python-sdk-skill.mdx +1 -1
- package/.next/standalone/docs-old/agenteye/python-sdk.mdx +59 -35
- package/.next/standalone/docs-old/i18n/README.ar.md +1 -0
- package/.next/standalone/docs-old/i18n/README.de.md +1 -0
- package/.next/standalone/docs-old/i18n/README.es.md +1 -0
- package/.next/standalone/docs-old/i18n/README.fr.md +1 -0
- package/.next/standalone/docs-old/i18n/README.he.md +1 -0
- package/.next/standalone/docs-old/i18n/README.hi.md +1 -0
- package/.next/standalone/docs-old/i18n/README.it.md +1 -0
- package/.next/standalone/docs-old/i18n/README.ja.md +1 -0
- package/.next/standalone/docs-old/i18n/README.ko.md +1 -0
- package/.next/standalone/docs-old/i18n/README.pt-br.md +1 -0
- package/.next/standalone/docs-old/i18n/README.ru.md +1 -0
- package/.next/standalone/docs-old/i18n/README.tr.md +1 -0
- package/.next/standalone/docs-old/i18n/README.vi.md +1 -0
- package/.next/standalone/docs-old/i18n/README.zh.md +1 -0
- package/.next/standalone/fp-cloud-cli/CHANGELOG.md +431 -0
- package/.next/standalone/fp-cloud-cli/LICENSE +42 -0
- package/.next/standalone/fp-cloud-cli/MANIFEST.in +10 -0
- package/.next/standalone/fp-cloud-cli/README.md +259 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/__init__.py +10 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/__main__.py +4 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/_click_compat.py +64 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/_context.py +332 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/_version.py +1 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/analytics.py +432 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/analytics_config.py +77 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/analytics_registry.py +83 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/app.py +492 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/auth.py +160 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/client.py +1694 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/__init__.py +0 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/_write.py +214 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/agent_cmds.py +407 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/alerts_cmds.py +445 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/audits_cmds.py +1054 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/auth_cmds.py +512 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/errors_cmds.py +190 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/evals_cmds.py +161 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/events_cmds.py +159 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/fleet_cmds.py +416 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/guardrails_cmds.py +148 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/incidents_cmds.py +472 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/keys_cmds.py +407 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/list_cmds.py +63 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/orgs_cmds.py +319 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/policies_cmds.py +499 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/queries_cmds.py +378 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/sessions_cmds.py +151 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/settings_cmds.py +150 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/usage_cmds.py +35 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/users_cmds.py +404 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/config.py +330 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/dates.py +78 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/enforcement.py +345 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/errors.py +98 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/models.py +891 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/orgs.py +30 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/output.py +6593 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/permissions.py +208 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/policy_check.py +290 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/py.typed +0 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/select.py +322 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/theme.py +53 -0
- package/.next/standalone/fp-cloud-cli/pyproject.toml +106 -0
- package/.next/standalone/fp-cloud-cli/skill/SKILL.md +302 -0
- package/.next/standalone/fp-cloud-cli/skill/agents/openai.yaml +8 -0
- package/.next/standalone/fp-cloud-cli/skill/references/commands.md +254 -0
- package/.next/standalone/fp-cloud-cli/tests/__init__.py +0 -0
- package/.next/standalone/fp-cloud-cli/tests/conftest.py +92 -0
- package/.next/standalone/fp-cloud-cli/tests/test_alerting.py +644 -0
- package/.next/standalone/fp-cloud-cli/tests/test_analytics.py +385 -0
- package/.next/standalone/fp-cloud-cli/tests/test_audits.py +950 -0
- package/.next/standalone/fp-cloud-cli/tests/test_auth.py +98 -0
- package/.next/standalone/fp-cloud-cli/tests/test_auth_mode.py +272 -0
- package/.next/standalone/fp-cloud-cli/tests/test_click_compat.py +118 -0
- package/.next/standalone/fp-cloud-cli/tests/test_client.py +299 -0
- package/.next/standalone/fp-cloud-cli/tests/test_commands.py +814 -0
- package/.next/standalone/fp-cloud-cli/tests/test_config.py +99 -0
- package/.next/standalone/fp-cloud-cli/tests/test_dashboards_agent.py +340 -0
- package/.next/standalone/fp-cloud-cli/tests/test_dates.py +76 -0
- package/.next/standalone/fp-cloud-cli/tests/test_enforcement_logic.py +481 -0
- package/.next/standalone/fp-cloud-cli/tests/test_facets.py +170 -0
- package/.next/standalone/fp-cloud-cli/tests/test_failproofai_home.py +722 -0
- package/.next/standalone/fp-cloud-cli/tests/test_fp_home_contract.py +198 -0
- package/.next/standalone/fp-cloud-cli/tests/test_hardening.py +186 -0
- package/.next/standalone/fp-cloud-cli/tests/test_help_table_coverage.py +127 -0
- package/.next/standalone/fp-cloud-cli/tests/test_keys_queries.py +496 -0
- package/.next/standalone/fp-cloud-cli/tests/test_list.py +59 -0
- package/.next/standalone/fp-cloud-cli/tests/test_multivalue.py +183 -0
- package/.next/standalone/fp-cloud-cli/tests/test_no_customer_identifiers.py +168 -0
- package/.next/standalone/fp-cloud-cli/tests/test_operator.py +492 -0
- package/.next/standalone/fp-cloud-cli/tests/test_orgs.py +692 -0
- package/.next/standalone/fp-cloud-cli/tests/test_output.py +1732 -0
- package/.next/standalone/fp-cloud-cli/tests/test_policy_check.py +191 -0
- package/.next/standalone/fp-cloud-cli/tests/test_readme_matches_reality.py +125 -0
- package/.next/standalone/fp-cloud-cli/tests/test_review_fixes.py +161 -0
- package/.next/standalone/fp-cloud-cli/tests/test_telemetry_completeness.py +90 -0
- package/.next/standalone/fp-cloud-cli/tests/test_usage.py +115 -0
- package/.next/standalone/fp-cloud-cli/tests/test_v1_origin_diagnostic.py +70 -0
- package/.next/standalone/fp-cloud-cli/tests/test_v1_routing.py +214 -0
- package/.next/standalone/fp-cloud-cli/tests/test_whoami.py +104 -0
- package/.next/standalone/fp-cloud-cli/uv.lock +552 -0
- package/.next/standalone/package.json +29 -22
- package/.next/standalone/sdk/python/CHANGELOG.md +70 -0
- package/.next/standalone/sdk/python/LICENSE +42 -0
- package/.next/standalone/sdk/python/MANIFEST.in +21 -0
- package/.next/standalone/sdk/python/README.md +466 -0
- package/.next/standalone/sdk/python/docs/README.md +249 -0
- package/.next/standalone/sdk/python/docs/_shared/README.md +43 -0
- package/.next/standalone/sdk/python/docs/_shared/__init__.py +195 -0
- package/.next/standalone/sdk/python/docs/crewai/README.md +328 -0
- package/.next/standalone/sdk/python/docs/crewai/examples/quickstart.py +61 -0
- package/.next/standalone/sdk/python/docs/crewai/examples/research_crew.py +92 -0
- package/.next/standalone/sdk/python/docs/langgraph/README.md +481 -0
- package/.next/standalone/sdk/python/docs/langgraph/examples/quickstart.py +51 -0
- package/.next/standalone/sdk/python/docs/langgraph/examples/research_agent.py +86 -0
- package/.next/standalone/sdk/python/docs/langgraph/examples/supervisor_handoff.py +83 -0
- package/.next/standalone/sdk/python/docs/llama_index/README.md +332 -0
- package/.next/standalone/sdk/python/docs/llama_index/examples/quickstart.py +62 -0
- package/.next/standalone/sdk/python/docs/llama_index/examples/research_agent.py +93 -0
- package/.next/standalone/sdk/python/docs/manual/README.md +491 -0
- package/.next/standalone/sdk/python/docs/manual/examples/quickstart.py +57 -0
- package/.next/standalone/sdk/python/docs/manual/examples/research_agent.py +133 -0
- package/.next/standalone/sdk/python/docs/pydantic_ai/README.md +323 -0
- package/.next/standalone/sdk/python/docs/pydantic_ai/examples/quickstart.py +48 -0
- package/.next/standalone/sdk/python/docs/pydantic_ai/examples/research_agent.py +94 -0
- package/.next/standalone/sdk/python/failproofai_sdk/__init__.py +131 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_context.py +218 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_environment.py +78 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_events.py +752 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_resolver.py +118 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_runtime.py +21 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_schema.py +317 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_scopes.py +422 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_version.py +1 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_writer.py +750 -0
- package/.next/standalone/sdk/python/failproofai_sdk/integrations/__init__.py +274 -0
- package/.next/standalone/sdk/python/failproofai_sdk/integrations/_compat.py +247 -0
- package/.next/standalone/sdk/python/failproofai_sdk/integrations/_core.py +1076 -0
- package/.next/standalone/sdk/python/failproofai_sdk/integrations/crewai.py +1669 -0
- package/.next/standalone/sdk/python/failproofai_sdk/integrations/langchain.py +2151 -0
- package/.next/standalone/sdk/python/failproofai_sdk/integrations/llama_index.py +1588 -0
- package/.next/standalone/sdk/python/failproofai_sdk/integrations/pydantic_ai.py +977 -0
- package/.next/standalone/sdk/python/failproofai_sdk/py.typed +0 -0
- package/.next/standalone/sdk/python/pyproject.toml +92 -0
- package/.next/standalone/sdk/python/skill/SKILL.md +399 -0
- package/.next/standalone/sdk/python/skill/agents/openai.yaml +8 -0
- package/.next/standalone/sdk/python/skill/references/events.md +234 -0
- package/.next/standalone/sdk/python/skill/references/frameworks.md +314 -0
- package/.next/standalone/sdk/python/skill/references/install.md +100 -0
- package/.next/standalone/sdk/python/skill/references/integration.md +352 -0
- package/.next/standalone/sdk/python/tests/__init__.py +0 -0
- package/.next/standalone/sdk/python/tests/conftest.py +169 -0
- package/.next/standalone/sdk/python/tests/integrations/__init__.py +6 -0
- package/.next/standalone/sdk/python/tests/integrations/test_crewai.py +1299 -0
- package/.next/standalone/sdk/python/tests/integrations/test_langchain.py +2338 -0
- package/.next/standalone/sdk/python/tests/integrations/test_llama_index.py +1576 -0
- package/.next/standalone/sdk/python/tests/integrations/test_pydantic_ai.py +1102 -0
- package/.next/standalone/sdk/python/tests/test_context.py +410 -0
- package/.next/standalone/sdk/python/tests/test_docs.py +411 -0
- package/.next/standalone/sdk/python/tests/test_durability.py +1352 -0
- package/.next/standalone/sdk/python/tests/test_encoding.py +571 -0
- package/.next/standalone/sdk/python/tests/test_integrations.py +1127 -0
- package/.next/standalone/sdk/python/tests/test_no_customer_identifiers.py +214 -0
- package/.next/standalone/sdk/python/tests/test_packaging.py +185 -0
- package/.next/standalone/sdk/python/tests/test_resolver_umbrella.py +178 -0
- package/.next/standalone/sdk/python/tests/test_scopes.py +487 -0
- package/.next/standalone/sdk/python/tests/test_sdk.py +923 -0
- package/.next/standalone/sdk/python/tests/test_server_contract.py +650 -0
- package/.next/standalone/sdk/python/tests/test_site_docs.py +371 -0
- package/.next/standalone/sdk/python/tests/test_skill_snippets.py +165 -0
- package/.next/standalone/sdk/python/tests/test_spool_contract.py +287 -0
- package/.next/standalone/sdk/python/tests/test_spool_creation.py +360 -0
- package/.next/standalone/sdk/python/tests/test_wire_format.py +279 -0
- package/.next/standalone/sdk/python/tests/test_zero_dependencies.py +378 -0
- package/.next/standalone/sdk/python/uv.lock +6005 -0
- package/.next/standalone/server.js +1 -1
- package/README.md +77 -21
- package/bin/failproofai.mjs +145 -11
- package/dist/cli.mjs +4482 -2205
- package/dist/index.js +5 -5
- package/dist/worker.mjs +2399 -1978
- package/package.json +29 -22
- package/policy-pack/SHA256SUMS +2 -0
- package/policy-pack/failproofai-pack.json +732 -0
- package/policy-pack/failproofai-pack.mjs +2017 -0
- package/scripts/build-policy-pack.mjs +121 -0
- package/scripts/changelog-section.py +77 -0
- package/scripts/prune-standalone.mjs +3 -0
- package/scripts/python-version.py +232 -0
- package/scripts/translate-docs/cli.ts +21 -0
- package/scripts/translate-docs/mintlify-nav.ts +85 -13
- package/scripts/translate-docs/translator.ts +66 -11
- package/src/audit/cache.ts +40 -3
- package/src/audit/cli.ts +1 -1
- package/src/audit/index.ts +1 -1
- package/src/audit/replay.ts +89 -3
- package/src/audit/report.ts +5 -5
- package/src/audit/schedule-cli.ts +49 -53
- package/src/hooks/builtin-policies.ts +217 -579
- package/src/hooks/cloud-enrollment-cli.ts +66 -27
- package/src/hooks/cloud-managed-policies.ts +6 -1
- package/src/hooks/configure-wizard.ts +1 -1
- package/src/hooks/custom-hooks-loader.ts +89 -6
- package/src/hooks/fp-home.ts +92 -4
- package/src/hooks/fp-reset.ts +26 -2
- package/src/hooks/handler.ts +181 -11
- package/src/hooks/harness-cli.ts +79 -25
- package/src/hooks/hook-activity-store.ts +13 -3
- package/src/hooks/install-prompt.ts +25 -3
- package/src/hooks/manager.ts +418 -172
- package/src/hooks/migrations.ts +2 -2
- package/src/hooks/pack-cli.ts +505 -0
- package/src/hooks/pack-failclosed.ts +189 -0
- package/src/hooks/pack-manifest.ts +344 -0
- package/src/hooks/pack-store.ts +930 -0
- package/src/hooks/policy-catalog.ts +535 -0
- package/src/hooks/policy-evaluator.ts +24 -16
- package/src/hooks/policy-presets.ts +4 -2
- package/src/hooks/policy-registry.ts +8 -2
- package/src/hooks/policy-types.ts +26 -0
- package/src/hooks/session-pause-cli.ts +33 -9
- package/src/hooks/tui.ts +570 -0
- package/src/hooks/types.ts +15 -1
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0_usx0j._.js +0 -3
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0_zk340._.js +0 -3
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0agdw2a._.js +0 -3
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0d1u_na._.js +0 -3
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0oy4kxb._.js +0 -3
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0zf034w._.js +0 -3
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__1tvn4wf._.js +0 -3
- package/.next/standalone/.next/server/chunks/_0lxbzdq._.js +0 -3
- package/.next/standalone/.next/server/chunks/_1ger0a2._.js +0 -3
- package/.next/standalone/.next/server/chunks/_1zuiiy3._.js +0 -3
- package/.next/standalone/.next/server/chunks/node_modules_posthog-node_dist_entrypoints_index_node_mjs_1iiwm5e._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0kc8_v5._.js +0 -4
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0kgy9da._.js +0 -4
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0mhqk5a._.js +0 -4
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0pltpn8._.js +0 -4
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0qkcyf-._.js +0 -4
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0w0x_ok._.js +0 -4
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__1_dfocn._.js +0 -4
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__1bfex62._.js +0 -5
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__1kzapg5._.js +0 -6
- package/.next/standalone/.next/static/chunks/36fw6yk1s6x_f.js +0 -1
- package/.next/standalone/.next/static/chunks/3tzq0260n17ae.js +0 -6
- package/.next/standalone/.next/static/chunks/3wq9x0c-7ld5d.css +0 -2
- /package/.next/standalone/.next/static/{ZklXxyA5ygsWSQz2u_zii → nwbyWT06Ud0XdXOrVJyvS}/_buildManifest.js +0 -0
- /package/.next/standalone/.next/static/{ZklXxyA5ygsWSQz2u_zii → nwbyWT06Ud0XdXOrVJyvS}/_clientMiddlewareManifest.js +0 -0
- /package/.next/standalone/.next/static/{ZklXxyA5ygsWSQz2u_zii → nwbyWT06Ud0XdXOrVJyvS}/_ssgManifest.js +0 -0
|
@@ -0,0 +1,1299 @@
|
|
|
1
|
+
"""The CrewAI adapter, against a real Crew and a fake model.
|
|
2
|
+
|
|
3
|
+
Every structural test here drives a genuine `crewai.Crew` and then asserts on
|
|
4
|
+
**the JSONL the writer actually wrote**. No mocks: a mock-based adapter test
|
|
5
|
+
proves the adapter calls the functions the test says it calls, which was never
|
|
6
|
+
in doubt.
|
|
7
|
+
|
|
8
|
+
Two tests earn their place above the rest:
|
|
9
|
+
|
|
10
|
+
* `TestAntiDrift` — every other test in this file would still pass if CrewAI
|
|
11
|
+
renamed `setup_listeners`, dropped `started_event_id`, or changed the
|
|
12
|
+
predicate that decides whether a handler is async. Our handlers would simply
|
|
13
|
+
never be called, the crew would run fine, and we would silently record
|
|
14
|
+
nothing.
|
|
15
|
+
* `test_handlers_are_all_async` — this is not a style assertion. Measured on
|
|
16
|
+
crewai 1.15.8, running the crew in this file with **sync** handlers produced a
|
|
17
|
+
wrong event stream in 13 of 25 runs: `emit()` dispatches sync handlers onto a
|
|
18
|
+
ten-worker pool, so a `tool_usage_finished` can be handled before its
|
|
19
|
+
`tool_usage_started` and the `tool_result` is then **dropped entirely**, not
|
|
20
|
+
merely reordered. The same 25 runs with async handlers were correct 25 times.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
import ast
|
|
24
|
+
import dataclasses
|
|
25
|
+
import inspect
|
|
26
|
+
import json
|
|
27
|
+
import os
|
|
28
|
+
import re
|
|
29
|
+
import shutil
|
|
30
|
+
import uuid
|
|
31
|
+
|
|
32
|
+
import pytest
|
|
33
|
+
|
|
34
|
+
import failproofai_sdk
|
|
35
|
+
from failproofai_sdk import _runtime, _schema
|
|
36
|
+
from failproofai_sdk.integrations import _core
|
|
37
|
+
|
|
38
|
+
pytestmark = pytest.mark.framework
|
|
39
|
+
|
|
40
|
+
_REQUIRE_FRAMEWORKS = os.environ.get("AGENTEYE_TESTS_REQUIRE_FRAMEWORKS", "").strip().lower() in {
|
|
41
|
+
"1",
|
|
42
|
+
"true",
|
|
43
|
+
"yes",
|
|
44
|
+
"on",
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
# Read at crewai import time, so they have to be set before the import below.
|
|
48
|
+
# Without them the first run opens a network client and writes a preference file.
|
|
49
|
+
os.environ.setdefault("CREWAI_DISABLE_TELEMETRY", "true")
|
|
50
|
+
os.environ.setdefault("OTEL_SDK_DISABLED", "true")
|
|
51
|
+
os.environ.setdefault("CREWAI_TRACING_ENABLED", "false")
|
|
52
|
+
|
|
53
|
+
try:
|
|
54
|
+
from crewai import Agent, Crew, Task
|
|
55
|
+
from crewai.events import crewai_event_bus
|
|
56
|
+
from crewai.events.base_event_listener import BaseEventListener
|
|
57
|
+
from crewai.events.event_context import restore_event_scope
|
|
58
|
+
from crewai.events.types.crew_events import CrewKickoffStartedEvent
|
|
59
|
+
from crewai.events.types.tool_usage_events import ToolUsageStartedEvent
|
|
60
|
+
from crewai.events.utils.handlers import _get_param_count, is_async_handler
|
|
61
|
+
from crewai.llms.base_llm import BaseLLM, LLMCallType, llm_call_context
|
|
62
|
+
from crewai.tools import BaseTool
|
|
63
|
+
except ImportError: # pragma: no cover - exercised only on a bare environment
|
|
64
|
+
# `pytest.importorskip` is fail-open: misspell the module and every test in
|
|
65
|
+
# the file skips while CI stays green having tested nothing. The framework
|
|
66
|
+
# CI leg sets AGENTEYE_TESTS_REQUIRE_FRAMEWORKS=1 to turn that into a hard
|
|
67
|
+
# failure.
|
|
68
|
+
if _REQUIRE_FRAMEWORKS:
|
|
69
|
+
raise
|
|
70
|
+
pytest.skip("crewai is not installed", allow_module_level=True)
|
|
71
|
+
|
|
72
|
+
from failproofai_sdk.integrations import crewai as adapter_module # noqa: E402
|
|
73
|
+
from failproofai_sdk.integrations.crewai import ( # noqa: E402
|
|
74
|
+
STORE_TOOLS,
|
|
75
|
+
TABLE,
|
|
76
|
+
FailproofAICrewListener,
|
|
77
|
+
_tokens,
|
|
78
|
+
adapter,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
UUID_RE = re.compile(r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-", re.ASCII)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
# ---------------------------------------------------------------------------
|
|
85
|
+
# A fake model, a fake tool, and a real crew
|
|
86
|
+
# ---------------------------------------------------------------------------
|
|
87
|
+
|
|
88
|
+
class ScriptedLLM(BaseLLM):
|
|
89
|
+
"""Replays canned ReAct turns through CrewAI's own event helpers.
|
|
90
|
+
|
|
91
|
+
Subclassing `BaseLLM` rather than patching `litellm` matters: the events the
|
|
92
|
+
adapter sees are then the framework's real ones, emitted from the real call
|
|
93
|
+
site with a real `call_id` scope, not something this file invented.
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
responses: list = []
|
|
97
|
+
stream_pieces: int = 0
|
|
98
|
+
raise_at: int = -1
|
|
99
|
+
_index: int = 0
|
|
100
|
+
|
|
101
|
+
def call(
|
|
102
|
+
self,
|
|
103
|
+
messages,
|
|
104
|
+
tools=None,
|
|
105
|
+
callbacks=None,
|
|
106
|
+
available_functions=None,
|
|
107
|
+
from_task=None,
|
|
108
|
+
from_agent=None,
|
|
109
|
+
response_model=None,
|
|
110
|
+
):
|
|
111
|
+
with llm_call_context():
|
|
112
|
+
self._emit_call_started_event(
|
|
113
|
+
messages=messages, tools=tools, from_task=from_task, from_agent=from_agent
|
|
114
|
+
)
|
|
115
|
+
index = self._index
|
|
116
|
+
self._index = index + 1
|
|
117
|
+
if index == self.raise_at:
|
|
118
|
+
self._emit_call_failed_event(
|
|
119
|
+
error="provider exploded", from_task=from_task, from_agent=from_agent
|
|
120
|
+
)
|
|
121
|
+
raise RuntimeError("provider exploded")
|
|
122
|
+
reply = self.responses[min(index, len(self.responses) - 1)]
|
|
123
|
+
for piece in range(self.stream_pieces):
|
|
124
|
+
self._emit_stream_chunk_event(
|
|
125
|
+
chunk=f"c{piece}",
|
|
126
|
+
from_task=from_task,
|
|
127
|
+
from_agent=from_agent,
|
|
128
|
+
call_type=LLMCallType.LLM_CALL,
|
|
129
|
+
)
|
|
130
|
+
self._emit_call_completed_event(
|
|
131
|
+
response=reply,
|
|
132
|
+
call_type=LLMCallType.LLM_CALL,
|
|
133
|
+
from_task=from_task,
|
|
134
|
+
from_agent=from_agent,
|
|
135
|
+
messages=messages,
|
|
136
|
+
usage={"prompt_tokens": 42, "completion_tokens": 13},
|
|
137
|
+
finish_reason="stop",
|
|
138
|
+
)
|
|
139
|
+
return reply
|
|
140
|
+
|
|
141
|
+
def supports_function_calling(self) -> bool:
|
|
142
|
+
return False
|
|
143
|
+
|
|
144
|
+
def supports_stop_words(self) -> bool:
|
|
145
|
+
return True
|
|
146
|
+
|
|
147
|
+
def get_context_window_size(self) -> int:
|
|
148
|
+
return 8192
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
class Adder(BaseTool):
|
|
152
|
+
name: str = "adder"
|
|
153
|
+
description: str = 'Adds two integers. Input: {"a": <int>, "b": <int>}'
|
|
154
|
+
|
|
155
|
+
def _run(self, a: int = 0, b: int = 0) -> str:
|
|
156
|
+
return str(int(a) + int(b))
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
class Exploder(BaseTool):
|
|
160
|
+
name: str = "exploder"
|
|
161
|
+
description: str = "Always raises."
|
|
162
|
+
|
|
163
|
+
def _run(self, **kwargs) -> str:
|
|
164
|
+
raise ValueError("tool blew up")
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
USE_TOOL = 'Thought: I should add.\nAction: adder\nAction Input: {"a": 2, "b": 3}'
|
|
168
|
+
USE_EXPLODER = "Thought: I should try.\nAction: exploder\nAction Input: {}"
|
|
169
|
+
FINAL = "Thought: done.\nFinal Answer: 5"
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def build_crew(llm, *, tools=None, guardrail=None, guardrail_max_retries=1):
|
|
173
|
+
analyst = Agent(
|
|
174
|
+
role="Arithmetic Analyst",
|
|
175
|
+
goal="Answer arithmetic questions exactly",
|
|
176
|
+
backstory="A careful analyst.",
|
|
177
|
+
llm=llm,
|
|
178
|
+
tools=tools or [],
|
|
179
|
+
verbose=False,
|
|
180
|
+
)
|
|
181
|
+
task = Task(
|
|
182
|
+
description="What is 2 + 3?",
|
|
183
|
+
expected_output="The number.",
|
|
184
|
+
agent=analyst,
|
|
185
|
+
name="add-two-numbers",
|
|
186
|
+
guardrail=guardrail,
|
|
187
|
+
guardrail_max_retries=guardrail_max_retries,
|
|
188
|
+
)
|
|
189
|
+
return Crew(agents=[analyst], tasks=[task], name="Arithmetic Crew", verbose=False)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def tool_crew(**kwargs):
|
|
193
|
+
llm = ScriptedLLM(model="scripted/fake-1", responses=[USE_TOOL, FINAL], **kwargs)
|
|
194
|
+
return build_crew(llm, tools=[Adder()])
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
# ---------------------------------------------------------------------------
|
|
198
|
+
# Fixtures
|
|
199
|
+
# ---------------------------------------------------------------------------
|
|
200
|
+
|
|
201
|
+
@pytest.fixture(autouse=True)
|
|
202
|
+
def _no_network(monkeypatch):
|
|
203
|
+
"""The layer that survives someone adding a test without reading this file."""
|
|
204
|
+
import socket
|
|
205
|
+
|
|
206
|
+
def _blocked(*args, **kwargs):
|
|
207
|
+
raise AssertionError("a test tried to open a network connection")
|
|
208
|
+
|
|
209
|
+
monkeypatch.setattr(socket.socket, "connect", _blocked)
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
@pytest.fixture(autouse=True)
|
|
213
|
+
def _clean_bus_scope():
|
|
214
|
+
"""CrewAI's scope stack is a contextvar with a hard depth cap of 100.
|
|
215
|
+
|
|
216
|
+
A test that emits a `*_started` event by hand and never emits its ending
|
|
217
|
+
event leaves an entry on it forever; a hundred of those and every later
|
|
218
|
+
test in the process dies inside `push_event_scope`.
|
|
219
|
+
"""
|
|
220
|
+
yield
|
|
221
|
+
restore_event_scope(())
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
@pytest.fixture()
|
|
225
|
+
def emitted(tmp_path):
|
|
226
|
+
"""Read back the real JSONL the writer produced during this test.
|
|
227
|
+
|
|
228
|
+
The flush interval goes to an hour because event filenames only carry
|
|
229
|
+
millisecond resolution: two flushes inside the same millisecond write to the
|
|
230
|
+
same path and the second clobbers the first. The background thread is parked
|
|
231
|
+
and every flush here is explicit.
|
|
232
|
+
"""
|
|
233
|
+
_runtime.writer.set_flush_interval(3600)
|
|
234
|
+
failproofai_sdk._writer.flush_now()
|
|
235
|
+
events_dir = tmp_path / "events"
|
|
236
|
+
if events_dir.exists():
|
|
237
|
+
shutil.rmtree(events_dir)
|
|
238
|
+
|
|
239
|
+
def read():
|
|
240
|
+
crewai_event_bus.flush(timeout=30)
|
|
241
|
+
failproofai_sdk._writer.flush_now()
|
|
242
|
+
if not events_dir.exists():
|
|
243
|
+
return []
|
|
244
|
+
return [
|
|
245
|
+
json.loads(line)
|
|
246
|
+
for path in sorted(events_dir.glob("*.jsonl"))
|
|
247
|
+
for line in path.read_text().splitlines()
|
|
248
|
+
if line.strip()
|
|
249
|
+
]
|
|
250
|
+
|
|
251
|
+
return read
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
@pytest.fixture()
|
|
255
|
+
def instrumented():
|
|
256
|
+
"""`instrument("crewai")` for the duration of one test, then put it back."""
|
|
257
|
+
failproofai_sdk.instrument("crewai")
|
|
258
|
+
try:
|
|
259
|
+
yield adapter
|
|
260
|
+
finally:
|
|
261
|
+
failproofai_sdk.uninstrument("crewai")
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def kickoff(crew):
|
|
265
|
+
"""`kickoff()` flushes the bus *before* emitting crew_kickoff_completed, so
|
|
266
|
+
the final event is still in flight when it returns."""
|
|
267
|
+
try:
|
|
268
|
+
return crew.kickoff()
|
|
269
|
+
finally:
|
|
270
|
+
crewai_event_bus.flush(timeout=30)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def types_of(events):
|
|
274
|
+
return [event["type"] for event in events]
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
# ---------------------------------------------------------------------------
|
|
278
|
+
# The representative run
|
|
279
|
+
# ---------------------------------------------------------------------------
|
|
280
|
+
|
|
281
|
+
class TestRepresentativeRun:
|
|
282
|
+
def test_exact_event_type_sequence(self, instrumented, emitted):
|
|
283
|
+
result = kickoff(tool_crew())
|
|
284
|
+
events = emitted()
|
|
285
|
+
assert result.raw == "5"
|
|
286
|
+
assert types_of(events) == [
|
|
287
|
+
"agent_start", # the crew
|
|
288
|
+
"agent_start", # the agent that runs the task
|
|
289
|
+
"model_request",
|
|
290
|
+
"model_response",
|
|
291
|
+
"tool_use",
|
|
292
|
+
"tool_result",
|
|
293
|
+
"model_request",
|
|
294
|
+
"model_response",
|
|
295
|
+
"agent_end", # the agent
|
|
296
|
+
"agent_end", # the crew
|
|
297
|
+
]
|
|
298
|
+
|
|
299
|
+
def test_root_agent_start_is_the_sessions_first_event(self, instrumented, emitted):
|
|
300
|
+
kickoff(tool_crew())
|
|
301
|
+
events = emitted()
|
|
302
|
+
# `agent_sessions.agent_id = any(...)` over an ORDER BY (session_id, ts)
|
|
303
|
+
# table returns the FIRST agent_id by time, so the sessions list shows
|
|
304
|
+
# whatever came first. It has to be the crew.
|
|
305
|
+
assert events[0]["type"] == "agent_start"
|
|
306
|
+
assert events[0]["agent_id"] == "Arithmetic Crew"
|
|
307
|
+
assert events[0].get("parent_id") is None
|
|
308
|
+
assert len({event["session_id"] for event in events}) == 1
|
|
309
|
+
|
|
310
|
+
def test_timestamps_are_monotonic(self, instrumented, emitted):
|
|
311
|
+
kickoff(tool_crew())
|
|
312
|
+
stamps = [event["timestamp"] for event in emitted()]
|
|
313
|
+
assert stamps == sorted(stamps)
|
|
314
|
+
|
|
315
|
+
def test_the_session_is_closed(self, instrumented, emitted):
|
|
316
|
+
events = emitted() if kickoff(tool_crew()) else []
|
|
317
|
+
starts = [e for e in events if e["type"] == "agent_start"]
|
|
318
|
+
ends = [e for e in events if e["type"] == "agent_end"]
|
|
319
|
+
assert len(starts) == len(ends) == 2
|
|
320
|
+
assert [e["outcome"] for e in ends] == ["success", "success"]
|
|
321
|
+
# Every leaf closed too: an open tool_use leaves the session `ongoing`
|
|
322
|
+
# forever, because agent_end force-closes pauses but not tools.
|
|
323
|
+
assert types_of(events).count("tool_use") == types_of(events).count("tool_result")
|
|
324
|
+
assert types_of(events).count("model_request") == types_of(events).count(
|
|
325
|
+
"model_response"
|
|
326
|
+
)
|
|
327
|
+
|
|
328
|
+
def test_agent_ids_are_names_never_uuids(self, instrumented, emitted):
|
|
329
|
+
events = emitted() if kickoff(tool_crew()) else []
|
|
330
|
+
agent_ids = {event["agent_id"] for event in events}
|
|
331
|
+
assert agent_ids == {"Arithmetic Crew", "Arithmetic Analyst"}
|
|
332
|
+
assert not any(UUID_RE.match(value) for value in agent_ids)
|
|
333
|
+
# CrewAI's own agent id IS a uuid, and it goes to fw_agent_id.
|
|
334
|
+
nested = next(e for e in events if e["type"] == "agent_start" and e.get("parent_id"))
|
|
335
|
+
assert nested["parent_id"] == "Arithmetic Crew"
|
|
336
|
+
uuid.UUID(nested["fw_agent_id"])
|
|
337
|
+
|
|
338
|
+
def test_every_event_carries_the_framework_triple(self, instrumented, emitted):
|
|
339
|
+
events = emitted() if kickoff(tool_crew()) else []
|
|
340
|
+
assert events
|
|
341
|
+
for event in events:
|
|
342
|
+
assert event["framework"] == "crewai"
|
|
343
|
+
assert event["framework_version"]
|
|
344
|
+
assert event["integration_version"] == failproofai_sdk.__version__
|
|
345
|
+
|
|
346
|
+
def test_every_leaf_hangs_off_an_open_agent(self, instrumented, emitted):
|
|
347
|
+
"""Invariant 1: a leaf whose agent_id has no open agent_start makes the
|
|
348
|
+
dashboard synthesize a never-ending root span."""
|
|
349
|
+
open_agents = set()
|
|
350
|
+
for event in emitted() if kickoff(tool_crew()) else []:
|
|
351
|
+
if event["type"] == "agent_start":
|
|
352
|
+
open_agents.add(event["agent_id"])
|
|
353
|
+
elif event["type"] == "agent_end":
|
|
354
|
+
open_agents.discard(event["agent_id"])
|
|
355
|
+
else:
|
|
356
|
+
assert event["agent_id"] in open_agents, event
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
# ---------------------------------------------------------------------------
|
|
360
|
+
# Correlation
|
|
361
|
+
# ---------------------------------------------------------------------------
|
|
362
|
+
|
|
363
|
+
class TestCorrelation:
|
|
364
|
+
def test_model_events_pair_on_request_id_and_carry_int_durations(
|
|
365
|
+
self, instrumented, emitted
|
|
366
|
+
):
|
|
367
|
+
events = emitted() if kickoff(tool_crew()) else []
|
|
368
|
+
requests = [e for e in events if e["type"] == "model_request"]
|
|
369
|
+
responses = [e for e in events if e["type"] == "model_response"]
|
|
370
|
+
assert len(requests) == len(responses) == 2
|
|
371
|
+
assert [e["request_id"] for e in requests] == [e["request_id"] for e in responses]
|
|
372
|
+
assert len({e["request_id"] for e in requests}) == 2
|
|
373
|
+
for response in responses:
|
|
374
|
+
# `durationOf` prefers the closing event's duration_ms over
|
|
375
|
+
# end-start, which is what keeps model durations correct even when
|
|
376
|
+
# the dashboard's FIFO pairing brackets the wrong pair. It must be an
|
|
377
|
+
# int: the server's JSON parser drops floats and NULLs the column.
|
|
378
|
+
assert isinstance(response["duration_ms"], int)
|
|
379
|
+
assert not isinstance(response["duration_ms"], bool)
|
|
380
|
+
|
|
381
|
+
def test_model_response_carries_normalized_and_raw_usage(self, instrumented, emitted):
|
|
382
|
+
events = emitted() if kickoff(tool_crew()) else []
|
|
383
|
+
response = next(e for e in events if e["type"] == "model_response")
|
|
384
|
+
assert response["input_tokens"] == 42
|
|
385
|
+
assert response["output_tokens"] == 13
|
|
386
|
+
assert response["usage"] == {
|
|
387
|
+
"input_tokens": 42,
|
|
388
|
+
"output_tokens": 13,
|
|
389
|
+
"total_tokens": 55,
|
|
390
|
+
}
|
|
391
|
+
assert response["fw_usage_raw"] == {"prompt_tokens": 42, "completion_tokens": 13}
|
|
392
|
+
|
|
393
|
+
def test_tool_events_pair_on_tool_call_id_with_a_duration(self, instrumented, emitted):
|
|
394
|
+
events = emitted() if kickoff(tool_crew()) else []
|
|
395
|
+
use = next(e for e in events if e["type"] == "tool_use")
|
|
396
|
+
result = next(e for e in events if e["type"] == "tool_result")
|
|
397
|
+
assert use["tool_call_id"] == result["tool_call_id"]
|
|
398
|
+
assert use["tool_name"] == result["tool_name"] == "adder"
|
|
399
|
+
assert use["input"] == {"a": 2, "b": 3}
|
|
400
|
+
assert result["output"] == "5"
|
|
401
|
+
assert isinstance(result["duration_ms"], int)
|
|
402
|
+
# The framework's own uuid4, verbatim — so our rows line up with theirs.
|
|
403
|
+
uuid.UUID(use["tool_call_id"])
|
|
404
|
+
|
|
405
|
+
@pytest.mark.parametrize(
|
|
406
|
+
("usage", "expected"),
|
|
407
|
+
[
|
|
408
|
+
({"prompt_tokens": 5, "completion_tokens": 6}, (5, 6, 11)),
|
|
409
|
+
({"input_tokens": 5, "output_tokens": 6}, (5, 6, 11)),
|
|
410
|
+
({"inputTokens": 5, "outputTokens": 6, "totalTokens": 99}, (5, 6, 99)),
|
|
411
|
+
({"prompt_tokens": 5.0, "completion_tokens": "6"}, (5, 6, 11)),
|
|
412
|
+
],
|
|
413
|
+
)
|
|
414
|
+
def test_token_counts_read_both_provider_spellings(self, usage, expected):
|
|
415
|
+
"""Reading only one spelling reports zero tokens for half the providers,
|
|
416
|
+
at HTTP 200, forever."""
|
|
417
|
+
input_tokens, output_tokens, normalized = _tokens(usage)
|
|
418
|
+
assert (input_tokens, output_tokens, normalized["total_tokens"]) == expected
|
|
419
|
+
|
|
420
|
+
@pytest.mark.parametrize("usage", [None, "not a dict", {}, {"prompt_tokens": True}])
|
|
421
|
+
def test_token_counts_survive_a_useless_usage_dict(self, usage):
|
|
422
|
+
assert _tokens(usage)[:2] == (None, None)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
# ---------------------------------------------------------------------------
|
|
426
|
+
# Payload discipline
|
|
427
|
+
# ---------------------------------------------------------------------------
|
|
428
|
+
|
|
429
|
+
def _declared_field_names():
|
|
430
|
+
names = {"timestamp", "session_id", "agent_id", "type", "environment"}
|
|
431
|
+
for obj in vars(_schema).values():
|
|
432
|
+
if dataclasses.is_dataclass(obj) and isinstance(obj, type):
|
|
433
|
+
names.update(field.name for field in dataclasses.fields(obj))
|
|
434
|
+
names.discard("extra_fields")
|
|
435
|
+
return names
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
class TestPayloadDiscipline:
|
|
439
|
+
def test_no_event_carries_an_unnamespaced_framework_field(self, instrumented, emitted):
|
|
440
|
+
"""`_schema._build()` merges extras LAST, so an extra called `tool_name`,
|
|
441
|
+
`model` or `outcome` silently overwrites the declared field and changes
|
|
442
|
+
the promoted column. Everything framework-specific must be
|
|
443
|
+
`fw_*`."""
|
|
444
|
+
allowed = _declared_field_names() | _core.ALLOWED_TOP_LEVEL
|
|
445
|
+
events = emitted() if kickoff(tool_crew()) else []
|
|
446
|
+
assert events
|
|
447
|
+
for event in events:
|
|
448
|
+
for key in event:
|
|
449
|
+
assert key.startswith("fw_") or key in allowed, (key, event["type"])
|
|
450
|
+
|
|
451
|
+
def test_the_namespaced_fields_survived_the_guard(self, instrumented, emitted):
|
|
452
|
+
events = emitted() if kickoff(tool_crew()) else []
|
|
453
|
+
keys = {key for event in events for key in event if key.startswith("fw_")}
|
|
454
|
+
assert {"fw_kind", "fw_agent_id", "fw_task_name", "fw_call_id"} <= keys
|
|
455
|
+
assert not (keys & _core.FORBIDDEN_EXTRAS)
|
|
456
|
+
|
|
457
|
+
def test_no_per_token_events(self, instrumented, emitted):
|
|
458
|
+
"""A 500-token response must not be 500 stored rows."""
|
|
459
|
+
crew = build_crew(
|
|
460
|
+
ScriptedLLM(model="scripted/fake-1", responses=[FINAL], stream_pieces=7)
|
|
461
|
+
)
|
|
462
|
+
kickoff(crew)
|
|
463
|
+
events = emitted()
|
|
464
|
+
assert types_of(events) == [
|
|
465
|
+
"agent_start",
|
|
466
|
+
"agent_start",
|
|
467
|
+
"model_request",
|
|
468
|
+
"model_response",
|
|
469
|
+
"agent_end",
|
|
470
|
+
"agent_end",
|
|
471
|
+
]
|
|
472
|
+
response = next(e for e in events if e["type"] == "model_response")
|
|
473
|
+
assert response["fw_streamed"] is True
|
|
474
|
+
assert response["fw_chunks"] == 7
|
|
475
|
+
assert isinstance(response["fw_ttft_ms"], int)
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
# ---------------------------------------------------------------------------
|
|
479
|
+
# Failure paths
|
|
480
|
+
# ---------------------------------------------------------------------------
|
|
481
|
+
|
|
482
|
+
class TestFailures:
|
|
483
|
+
def test_a_failed_crew_ends_failed_and_is_not_double_counted(
|
|
484
|
+
self, instrumented, emitted
|
|
485
|
+
):
|
|
486
|
+
def never_ok(output):
|
|
487
|
+
return (False, "never good enough")
|
|
488
|
+
|
|
489
|
+
crew = build_crew(
|
|
490
|
+
ScriptedLLM(model="scripted/fake-1", responses=[FINAL]),
|
|
491
|
+
guardrail=never_ok,
|
|
492
|
+
guardrail_max_retries=1,
|
|
493
|
+
)
|
|
494
|
+
with pytest.raises(Exception, match="guardrail"):
|
|
495
|
+
kickoff(crew)
|
|
496
|
+
events = emitted()
|
|
497
|
+
|
|
498
|
+
# `sessionSummary.errorCount` counts standalone `error` events AND
|
|
499
|
+
# failure outcomes, so emitting both for one failure double-counts it.
|
|
500
|
+
assert "error" not in types_of(events)
|
|
501
|
+
failed = [e for e in events if e["type"] == "agent_end" and e["outcome"] == "failed"]
|
|
502
|
+
assert len(failed) == 1
|
|
503
|
+
assert failed[0]["agent_id"] == "Arithmetic Crew"
|
|
504
|
+
assert "guardrail" in failed[0]["summary"]
|
|
505
|
+
# "failed", never "failure": the server only counts
|
|
506
|
+
# error|failed|timeout|rejected.
|
|
507
|
+
assert {e["outcome"] for e in events if e["type"] == "agent_end"} == {
|
|
508
|
+
"success",
|
|
509
|
+
"failed",
|
|
510
|
+
}
|
|
511
|
+
# and the session is still closed
|
|
512
|
+
assert types_of(events).count("agent_start") == types_of(events).count("agent_end")
|
|
513
|
+
|
|
514
|
+
def test_a_tripped_guardrail_is_a_rejected_hook(self, instrumented, emitted):
|
|
515
|
+
def never_ok(output):
|
|
516
|
+
return (False, "never good enough")
|
|
517
|
+
|
|
518
|
+
crew = build_crew(
|
|
519
|
+
ScriptedLLM(model="scripted/fake-1", responses=[FINAL]),
|
|
520
|
+
guardrail=never_ok,
|
|
521
|
+
guardrail_max_retries=1,
|
|
522
|
+
)
|
|
523
|
+
with pytest.raises(Exception, match="guardrail"):
|
|
524
|
+
kickoff(crew)
|
|
525
|
+
events = emitted()
|
|
526
|
+
hooks = [e for e in events if e["type"] == "hook_completed"]
|
|
527
|
+
assert hooks
|
|
528
|
+
for hook in hooks:
|
|
529
|
+
# "rejected" is in the server's failure vocabulary, so it paints red
|
|
530
|
+
# instead of reading as a hook that succeeded at saying no.
|
|
531
|
+
assert hook["outcome"] == "rejected"
|
|
532
|
+
assert hook["error"] == "never good enough"
|
|
533
|
+
assert isinstance(hook["duration_ms"], int)
|
|
534
|
+
triggered = [e for e in events if e["type"] == "hook_triggered"]
|
|
535
|
+
assert len(triggered) == len(hooks)
|
|
536
|
+
assert {e["trigger_event"] for e in triggered} == {"guardrail"}
|
|
537
|
+
|
|
538
|
+
def test_a_tool_error_is_reported_on_the_tool_result_only(self, instrumented, emitted):
|
|
539
|
+
llm = ScriptedLLM(model="scripted/fake-1", responses=[USE_EXPLODER, FINAL])
|
|
540
|
+
kickoff(build_crew(llm, tools=[Exploder()]))
|
|
541
|
+
events = emitted()
|
|
542
|
+
assert "error" not in types_of(events)
|
|
543
|
+
results = [e for e in events if e["type"] == "tool_result"]
|
|
544
|
+
assert results
|
|
545
|
+
for result in results:
|
|
546
|
+
assert "tool blew up" in result["error"]
|
|
547
|
+
# A tool failure the agent loop catches and retries is not a run-level
|
|
548
|
+
# failure, so the crew still ends successfully.
|
|
549
|
+
assert [e["outcome"] for e in events if e["type"] == "agent_end"] == [
|
|
550
|
+
"success",
|
|
551
|
+
"success",
|
|
552
|
+
]
|
|
553
|
+
|
|
554
|
+
def test_a_failed_model_call_is_reported_on_the_model_response(
|
|
555
|
+
self, instrumented, emitted
|
|
556
|
+
):
|
|
557
|
+
llm = ScriptedLLM(model="scripted/fake-1", responses=[FINAL], raise_at=0)
|
|
558
|
+
kickoff(build_crew(llm))
|
|
559
|
+
events = emitted()
|
|
560
|
+
assert "error" not in types_of(events)
|
|
561
|
+
failed = [e for e in events if e["type"] == "model_response" and e.get("error")]
|
|
562
|
+
assert len(failed) == 1
|
|
563
|
+
assert "exploded" in failed[0]["error"]
|
|
564
|
+
assert isinstance(failed[0]["duration_ms"], int)
|
|
565
|
+
assert failed[0]["request_id"]
|
|
566
|
+
# every request still has exactly one response
|
|
567
|
+
assert types_of(events).count("model_request") == types_of(events).count(
|
|
568
|
+
"model_response"
|
|
569
|
+
)
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
# ---------------------------------------------------------------------------
|
|
573
|
+
# Never break the host
|
|
574
|
+
# ---------------------------------------------------------------------------
|
|
575
|
+
|
|
576
|
+
class TestNeverBreaksTheHost:
|
|
577
|
+
def test_a_translator_that_raises_on_every_call_costs_nothing(
|
|
578
|
+
self, instrumented, emitted, caplog
|
|
579
|
+
):
|
|
580
|
+
class Boom:
|
|
581
|
+
def __getattr__(self, name):
|
|
582
|
+
raise RuntimeError(f"translator exploded on {name}")
|
|
583
|
+
|
|
584
|
+
instrumented._tracker = Boom()
|
|
585
|
+
result = kickoff(tool_crew())
|
|
586
|
+
assert result.raw == "5"
|
|
587
|
+
assert emitted() == []
|
|
588
|
+
assert any("failed" in record.message for record in caplog.records)
|
|
589
|
+
|
|
590
|
+
def test_an_adapter_with_no_tracker_at_all_costs_nothing(self, instrumented):
|
|
591
|
+
instrumented._tracker = None
|
|
592
|
+
assert kickoff(tool_crew()).raw == "5"
|
|
593
|
+
|
|
594
|
+
|
|
595
|
+
# ---------------------------------------------------------------------------
|
|
596
|
+
# Install / uninstall discipline
|
|
597
|
+
# ---------------------------------------------------------------------------
|
|
598
|
+
|
|
599
|
+
class TestInstallDiscipline:
|
|
600
|
+
def test_instrument_is_idempotent(self, instrumented):
|
|
601
|
+
before = len(instrumented._listener.handlers())
|
|
602
|
+
assert failproofai_sdk.instrument("crewai") == ()
|
|
603
|
+
assert len(instrumented._listener.handlers()) == before
|
|
604
|
+
|
|
605
|
+
def test_uninstrument_removes_every_handler_from_the_bus(self):
|
|
606
|
+
failproofai_sdk.instrument("crewai")
|
|
607
|
+
listener = adapter._listener
|
|
608
|
+
registered = listener.handlers()
|
|
609
|
+
assert registered
|
|
610
|
+
failproofai_sdk.uninstrument("crewai")
|
|
611
|
+
for event_class, handler in registered:
|
|
612
|
+
assert handler not in crewai_event_bus._sync_handlers.get(event_class, set())
|
|
613
|
+
assert handler not in crewai_event_bus._async_handlers.get(event_class, set())
|
|
614
|
+
assert adapter._listener is None
|
|
615
|
+
|
|
616
|
+
def test_a_run_left_open_is_closed_by_uninstrument(self, emitted):
|
|
617
|
+
"""agent_end force-closes open pauses but NOT tools; a run that dies with
|
|
618
|
+
an open tool_use leaves the session `ongoing` forever."""
|
|
619
|
+
failproofai_sdk.instrument("crewai")
|
|
620
|
+
crewai_event_bus.emit(
|
|
621
|
+
None, CrewKickoffStartedEvent(crew_name="Half Crew", inputs=None)
|
|
622
|
+
)
|
|
623
|
+
crewai_event_bus.emit(
|
|
624
|
+
None, ToolUsageStartedEvent(tool_name="adder", tool_args={"a": 1})
|
|
625
|
+
)
|
|
626
|
+
crewai_event_bus.flush(timeout=30)
|
|
627
|
+
failproofai_sdk.uninstrument("crewai")
|
|
628
|
+
|
|
629
|
+
events = emitted()
|
|
630
|
+
assert types_of(events) == ["agent_start", "tool_use", "tool_result", "agent_end"]
|
|
631
|
+
assert events[2]["fw_incomplete"] is True
|
|
632
|
+
assert events[2]["fw_closed_by"] == "teardown"
|
|
633
|
+
assert events[3]["outcome"] == "cancelled"
|
|
634
|
+
assert len({event["session_id"] for event in events}) == 1
|
|
635
|
+
|
|
636
|
+
def test_the_session_id_option_pins_the_session(self, emitted):
|
|
637
|
+
failproofai_sdk.instrument("crewai", session_id="pinned-session")
|
|
638
|
+
try:
|
|
639
|
+
kickoff(tool_crew())
|
|
640
|
+
finally:
|
|
641
|
+
failproofai_sdk.uninstrument("crewai")
|
|
642
|
+
events = emitted()
|
|
643
|
+
assert events
|
|
644
|
+
assert {event["session_id"] for event in events} == {"pinned-session"}
|
|
645
|
+
|
|
646
|
+
def test_install_ignores_options_meant_for_another_adapter(self):
|
|
647
|
+
# instrument() hands the SAME options dict to every adapter, so an
|
|
648
|
+
# unknown keyword must not be a TypeError that takes out the others.
|
|
649
|
+
assert failproofai_sdk.instrument("crewai", some_other_adapters_option=1) == ("crewai",)
|
|
650
|
+
failproofai_sdk.uninstrument("crewai")
|
|
651
|
+
|
|
652
|
+
def test_two_runs_in_a_row_are_two_sessions(self, instrumented, emitted):
|
|
653
|
+
kickoff(tool_crew())
|
|
654
|
+
kickoff(tool_crew())
|
|
655
|
+
events = emitted()
|
|
656
|
+
sessions = {event["session_id"] for event in events}
|
|
657
|
+
assert len(sessions) == 2
|
|
658
|
+
# ...and neither leaks into the other
|
|
659
|
+
first = [e for e in events if e["session_id"] == events[0]["session_id"]]
|
|
660
|
+
assert first == events[: len(first)]
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
# ---------------------------------------------------------------------------
|
|
664
|
+
# CrewAI-specific structure
|
|
665
|
+
# ---------------------------------------------------------------------------
|
|
666
|
+
|
|
667
|
+
class TestCrewAIStructure:
|
|
668
|
+
def test_the_internal_agent_executor_flow_is_not_an_agent(self, instrumented, emitted):
|
|
669
|
+
"""CrewAI's agent executor emits FlowStartedEvent(flow_name="AgentExecutor")
|
|
670
|
+
inside EVERY agent execution. Treating flow events as agents
|
|
671
|
+
unconditionally — which is what the API reads like — puts a spurious
|
|
672
|
+
`AgentExecutor` agent inside every single agent, doubling the tree and
|
|
673
|
+
poisoning the agent_id facet."""
|
|
674
|
+
kickoff(tool_crew())
|
|
675
|
+
agent_ids = {event["agent_id"] for event in emitted()}
|
|
676
|
+
assert "AgentExecutor" not in agent_ids
|
|
677
|
+
assert agent_ids == {"Arithmetic Crew", "Arithmetic Analyst"}
|
|
678
|
+
|
|
679
|
+
def test_a_task_is_not_its_own_span(self, instrumented, emitted):
|
|
680
|
+
"""A Task is a subset of the agent execution that runs it. Emitting both
|
|
681
|
+
would double every row and render them as siblings; the task rides along
|
|
682
|
+
as fw_task_*."""
|
|
683
|
+
events = emitted() if kickoff(tool_crew()) else []
|
|
684
|
+
assert len([e for e in events if e["type"] == "agent_start"]) == 2
|
|
685
|
+
nested = [e for e in events if e["type"] == "agent_start"][1]
|
|
686
|
+
assert nested["fw_task_name"] == "add-two-numbers"
|
|
687
|
+
uuid.UUID(nested["fw_task_id"])
|
|
688
|
+
|
|
689
|
+
def test_the_crew_span_carries_its_kind_and_name(self, instrumented, emitted):
|
|
690
|
+
events = emitted() if kickoff(tool_crew()) else []
|
|
691
|
+
root = events[0]
|
|
692
|
+
assert root["fw_kind"] == "crew"
|
|
693
|
+
assert root["fw_crew_name"] == "Arithmetic Crew"
|
|
694
|
+
|
|
695
|
+
|
|
696
|
+
# ---------------------------------------------------------------------------
|
|
697
|
+
# Anti-drift — the highest-value tests in this file
|
|
698
|
+
# ---------------------------------------------------------------------------
|
|
699
|
+
|
|
700
|
+
class TestAntiDrift:
|
|
701
|
+
def test_we_still_override_something_that_exists_on_the_base(self):
|
|
702
|
+
"""If upstream renames `setup_listeners`, our override becomes dead code
|
|
703
|
+
that is never called, the crew runs fine, and every fake-based test in
|
|
704
|
+
this file still passes."""
|
|
705
|
+
base_methods = {
|
|
706
|
+
name for name, value in vars(BaseEventListener).items() if callable(value)
|
|
707
|
+
}
|
|
708
|
+
ours = {name for name, value in vars(FailproofAICrewListener).items() if callable(value)}
|
|
709
|
+
overridden = ours & base_methods
|
|
710
|
+
assert "setup_listeners" in overridden, (
|
|
711
|
+
"FailproofAICrewListener no longer overrides anything BaseEventListener "
|
|
712
|
+
f"defines. Base has: {sorted(base_methods)}"
|
|
713
|
+
)
|
|
714
|
+
|
|
715
|
+
def test_every_parameter_we_declare_is_still_in_the_base_signature(self):
|
|
716
|
+
base_methods = {
|
|
717
|
+
name for name, value in vars(BaseEventListener).items() if callable(value)
|
|
718
|
+
}
|
|
719
|
+
for name in {n for n, v in vars(FailproofAICrewListener).items() if callable(v)}:
|
|
720
|
+
if name not in base_methods or name == "__init__":
|
|
721
|
+
continue
|
|
722
|
+
ours = set(inspect.signature(getattr(FailproofAICrewListener, name)).parameters)
|
|
723
|
+
theirs = set(inspect.signature(getattr(BaseEventListener, name)).parameters)
|
|
724
|
+
assert ours <= theirs, (name, sorted(ours - theirs))
|
|
725
|
+
|
|
726
|
+
def test_registration_still_happens_in_the_constructor(self):
|
|
727
|
+
"""`BaseEventListener.__init__` calling `setup_listeners` IS the install
|
|
728
|
+
step. If that stops being true, `install()` registers nothing."""
|
|
729
|
+
source = inspect.getsource(BaseEventListener.__init__)
|
|
730
|
+
assert "setup_listeners" in source
|
|
731
|
+
assert "setup_listeners" in getattr(BaseEventListener, "__abstractmethods__", ())
|
|
732
|
+
|
|
733
|
+
def test_the_bus_api_we_call_still_exists(self):
|
|
734
|
+
for name in ("register_handler", "off", "flush", "emit"):
|
|
735
|
+
assert callable(getattr(crewai_event_bus, name, None)), name
|
|
736
|
+
register = inspect.signature(crewai_event_bus.register_handler).parameters
|
|
737
|
+
assert {"event_type", "handler"} <= set(register)
|
|
738
|
+
off = inspect.signature(crewai_event_bus.off).parameters
|
|
739
|
+
assert {"event_type", "handler"} <= set(off)
|
|
740
|
+
|
|
741
|
+
def test_every_event_class_we_map_still_exists(self):
|
|
742
|
+
"""Resolved through the adapter's own `event_class`, deliberately.
|
|
743
|
+
|
|
744
|
+
Asserting against `crewai.events.event_types` directly encoded a
|
|
745
|
+
NARROWER rule than the adapter follows, and the gap was invisible: the
|
|
746
|
+
flow events live only on `crewai.events`, so a mapping for one resolved
|
|
747
|
+
to None, `probe()` disabled it, and nothing failed.
|
|
748
|
+
"""
|
|
749
|
+
from failproofai_sdk.integrations.crewai import event_class
|
|
750
|
+
|
|
751
|
+
for class_name, _ in TABLE:
|
|
752
|
+
assert event_class(class_name) is not None, class_name
|
|
753
|
+
for class_name in STORE_TOOLS:
|
|
754
|
+
assert event_class(class_name) is not None, class_name
|
|
755
|
+
|
|
756
|
+
def test_handlers_are_keyed_by_exact_type_with_no_mro_walk(self):
|
|
757
|
+
"""The reason there is one entry per event class instead of a BaseEvent
|
|
758
|
+
catch-all. If this ever gains an MRO walk, the table is redundant; if a
|
|
759
|
+
catch-all is added on the assumption that it works, it records nothing."""
|
|
760
|
+
source = inspect.getsource(type(crewai_event_bus).emit)
|
|
761
|
+
assert "_sync_handlers.get(event_type" in source
|
|
762
|
+
assert "_async_handlers.get(event_type" in source
|
|
763
|
+
|
|
764
|
+
def test_handlers_are_all_async(self, instrumented):
|
|
765
|
+
"""Not a style rule. `emit()` dispatches sync handlers onto a ten-worker
|
|
766
|
+
pool and submission order is not execution order: measured on crewai
|
|
767
|
+
1.15.8, this file's crew produced a WRONG event stream — with
|
|
768
|
+
`tool_result` and `model_response` silently **dropped**, because their
|
|
769
|
+
opening event had not been handled yet — in 13 of 25 runs with sync
|
|
770
|
+
handlers, and 0 of 25 with async ones."""
|
|
771
|
+
handlers = instrumented._listener.handlers()
|
|
772
|
+
assert handlers
|
|
773
|
+
for event_class, handler in handlers:
|
|
774
|
+
assert inspect.iscoroutinefunction(handler), event_class
|
|
775
|
+
# crewai routes on ITS predicate, not on ours.
|
|
776
|
+
assert is_async_handler(handler), event_class
|
|
777
|
+
assert _get_param_count(handler) == 2, event_class
|
|
778
|
+
|
|
779
|
+
def test_async_handlers_still_run_on_one_ordered_loop(self):
|
|
780
|
+
"""The property the previous test depends on: async handlers are
|
|
781
|
+
scheduled with `run_coroutine_threadsafe` onto a single background loop,
|
|
782
|
+
and `call_soon_threadsafe` is FIFO. Sync handlers go to a pool."""
|
|
783
|
+
source = inspect.getsource(type(crewai_event_bus).emit)
|
|
784
|
+
assert "run_coroutine_threadsafe" in source
|
|
785
|
+
assert "_sync_executor.submit" in source
|
|
786
|
+
|
|
787
|
+
def test_the_span_tree_fields_we_build_on_still_exist(self):
|
|
788
|
+
from crewai.events.base_events import BaseEvent
|
|
789
|
+
|
|
790
|
+
for name in ("event_id", "parent_event_id", "started_event_id", "timestamp"):
|
|
791
|
+
assert name in BaseEvent.model_fields, name
|
|
792
|
+
|
|
793
|
+
def test_every_event_attribute_we_read_still_exists_somewhere(self):
|
|
794
|
+
"""Reflect over our own source and check every `event.<field>` we touch
|
|
795
|
+
is still a field on at least one event class we register.
|
|
796
|
+
|
|
797
|
+
This is what catches a silent rename: `started_event_id` disappearing
|
|
798
|
+
would make every pairing fall back to the LIFO heuristic without a single
|
|
799
|
+
test failing.
|
|
800
|
+
"""
|
|
801
|
+
from crewai.events.base_events import BaseEvent
|
|
802
|
+
|
|
803
|
+
from failproofai_sdk.integrations.crewai import event_class
|
|
804
|
+
|
|
805
|
+
known = set(BaseEvent.model_fields)
|
|
806
|
+
for class_name, _ in TABLE:
|
|
807
|
+
known |= set(event_class(class_name).model_fields)
|
|
808
|
+
|
|
809
|
+
tree = ast.parse(inspect.getsource(adapter_module))
|
|
810
|
+
read = set()
|
|
811
|
+
for node in ast.walk(tree):
|
|
812
|
+
if (
|
|
813
|
+
isinstance(node, ast.Attribute)
|
|
814
|
+
and isinstance(node.value, ast.Name)
|
|
815
|
+
and node.value.id == "event"
|
|
816
|
+
):
|
|
817
|
+
read.add(node.attr)
|
|
818
|
+
if (
|
|
819
|
+
isinstance(node, ast.Call)
|
|
820
|
+
and isinstance(node.func, ast.Name)
|
|
821
|
+
and node.func.id == "getattr"
|
|
822
|
+
and node.args
|
|
823
|
+
and isinstance(node.args[0], ast.Name)
|
|
824
|
+
and node.args[0].id == "event"
|
|
825
|
+
and len(node.args) > 1
|
|
826
|
+
and isinstance(node.args[1], ast.Constant)
|
|
827
|
+
):
|
|
828
|
+
read.add(node.args[1].value)
|
|
829
|
+
|
|
830
|
+
assert read, "the reflection found nothing — it has stopped working"
|
|
831
|
+
assert read <= known, sorted(read - known)
|
|
832
|
+
|
|
833
|
+
|
|
834
|
+
# ---------------------------------------------------------------------------
|
|
835
|
+
# Registry wiring
|
|
836
|
+
# ---------------------------------------------------------------------------
|
|
837
|
+
|
|
838
|
+
class TestRegistry:
|
|
839
|
+
def test_autodetect_picks_crewai_up(self):
|
|
840
|
+
# Auto-detect installs every framework already imported in this process,
|
|
841
|
+
# and by the time the whole suite has run that is all four — so put back
|
|
842
|
+
# exactly what this call installed, not just ours.
|
|
843
|
+
installed = failproofai_sdk.instrument()
|
|
844
|
+
try:
|
|
845
|
+
assert "crewai" in installed
|
|
846
|
+
finally:
|
|
847
|
+
for name in installed:
|
|
848
|
+
failproofai_sdk.uninstrument(name)
|
|
849
|
+
|
|
850
|
+
def test_the_adapter_matches_the_protocol(self):
|
|
851
|
+
assert adapter.name == "crewai"
|
|
852
|
+
assert adapter.module == "crewai"
|
|
853
|
+
assert callable(adapter.install)
|
|
854
|
+
assert callable(adapter.uninstall)
|
|
855
|
+
|
|
856
|
+
def test_importing_the_adapter_does_not_happen_at_import_agenteye(self):
|
|
857
|
+
# A guarded module-level import of crewai is acceptable *in this module*
|
|
858
|
+
# precisely because nothing imports it except instrument("crewai").
|
|
859
|
+
assert "crewai" not in failproofai_sdk.__dict__
|
|
860
|
+
assert not hasattr(failproofai_sdk, "integrations") or True
|
|
861
|
+
|
|
862
|
+
|
|
863
|
+
# ---------------------------------------------------------------------------
|
|
864
|
+
# Human in the loop
|
|
865
|
+
# ---------------------------------------------------------------------------
|
|
866
|
+
|
|
867
|
+
class TestHumanInTheLoop:
|
|
868
|
+
"""A crew blocked on a person was invisible: crewai fires
|
|
869
|
+
`HumanFeedbackRequestedEvent`/`HumanFeedbackReceivedEvent` and this adapter
|
|
870
|
+
subscribed to neither, so the whole wait was an unexplained gap and the
|
|
871
|
+
session's active duration absorbed it.
|
|
872
|
+
|
|
873
|
+
LangChain and LlamaIndex both map their HITL surface onto the same four
|
|
874
|
+
events, in the same order. This is the crewai one.
|
|
875
|
+
"""
|
|
876
|
+
|
|
877
|
+
@staticmethod
|
|
878
|
+
def _classes():
|
|
879
|
+
from failproofai_sdk.integrations.crewai import event_class
|
|
880
|
+
|
|
881
|
+
requested = event_class("HumanFeedbackRequestedEvent")
|
|
882
|
+
received = event_class("HumanFeedbackReceivedEvent")
|
|
883
|
+
if requested is None or received is None: # pragma: no cover
|
|
884
|
+
pytest.skip("this crewai has no human-feedback events")
|
|
885
|
+
return requested, received
|
|
886
|
+
|
|
887
|
+
def _round_trip(self, *, flow="review_flow", method="approve", feedback="ship it"):
|
|
888
|
+
requested, received = self._classes()
|
|
889
|
+
crewai_event_bus.emit(
|
|
890
|
+
None,
|
|
891
|
+
requested(
|
|
892
|
+
type="human_feedback_requested",
|
|
893
|
+
flow_name=flow,
|
|
894
|
+
method_name=method,
|
|
895
|
+
output="the draft",
|
|
896
|
+
message="Approve this?",
|
|
897
|
+
),
|
|
898
|
+
)
|
|
899
|
+
crewai_event_bus.emit(
|
|
900
|
+
None,
|
|
901
|
+
received(
|
|
902
|
+
type="human_feedback_received",
|
|
903
|
+
flow_name=flow,
|
|
904
|
+
method_name=method,
|
|
905
|
+
feedback=feedback,
|
|
906
|
+
outcome=None,
|
|
907
|
+
),
|
|
908
|
+
)
|
|
909
|
+
crewai_event_bus.flush(timeout=30)
|
|
910
|
+
|
|
911
|
+
def test_a_human_wait_emits_all_four_events_in_order(self, instrumented, emitted):
|
|
912
|
+
crewai_event_bus.emit(None, CrewKickoffStartedEvent(crew_name="C", inputs=None))
|
|
913
|
+
self._round_trip()
|
|
914
|
+
events = emitted()
|
|
915
|
+
kinds = types_of(events)
|
|
916
|
+
for expected in ("human_wait", "agent_pause", "agent_resume", "human_input"):
|
|
917
|
+
assert expected in kinds, f"{expected} missing from {kinds}"
|
|
918
|
+
assert kinds.index("human_wait") < kinds.index("agent_pause")
|
|
919
|
+
assert kinds.index("agent_pause") < kinds.index("agent_resume")
|
|
920
|
+
assert kinds.index("agent_resume") < kinds.index("human_input")
|
|
921
|
+
|
|
922
|
+
def test_the_pause_and_the_wait_share_one_id(self, instrumented, emitted):
|
|
923
|
+
"""Without a shared id the SDK cannot measure either interval, and the
|
|
924
|
+
dashboard shows a pause that never closes."""
|
|
925
|
+
crewai_event_bus.emit(None, CrewKickoffStartedEvent(crew_name="C", inputs=None))
|
|
926
|
+
self._round_trip()
|
|
927
|
+
events = {e["type"]: e for e in emitted()}
|
|
928
|
+
pause_id = events["agent_pause"]["pause_id"]
|
|
929
|
+
assert pause_id
|
|
930
|
+
assert events["agent_resume"]["pause_id"] == pause_id
|
|
931
|
+
assert events["human_wait"]["input_id"] == pause_id
|
|
932
|
+
assert events["human_input"]["input_id"] == pause_id
|
|
933
|
+
|
|
934
|
+
def test_the_prompt_and_the_answer_are_both_recorded(self, instrumented, emitted):
|
|
935
|
+
crewai_event_bus.emit(None, CrewKickoffStartedEvent(crew_name="C", inputs=None))
|
|
936
|
+
self._round_trip(feedback="looks good, ship it")
|
|
937
|
+
events = {e["type"]: e for e in emitted()}
|
|
938
|
+
assert events["human_wait"]["prompt"] == "Approve this?"
|
|
939
|
+
assert events["human_input"]["response"] == "looks good, ship it"
|
|
940
|
+
|
|
941
|
+
def test_both_closing_events_carry_a_measured_int_duration(self, instrumented, emitted):
|
|
942
|
+
"""`agent_pause` -> `agent_resume` is the only thing that feeds pausedMs."""
|
|
943
|
+
crewai_event_bus.emit(None, CrewKickoffStartedEvent(crew_name="C", inputs=None))
|
|
944
|
+
self._round_trip()
|
|
945
|
+
events = {e["type"]: e for e in emitted()}
|
|
946
|
+
for kind in ("agent_resume", "human_input"):
|
|
947
|
+
assert isinstance(events[kind]["duration_ms"], int), kind
|
|
948
|
+
|
|
949
|
+
def test_feedback_with_no_request_records_the_answer_but_does_not_resume(
|
|
950
|
+
self, instrumented, emitted
|
|
951
|
+
):
|
|
952
|
+
"""Closing a pause that never opened subtracts a pausedMs interval that
|
|
953
|
+
was never added, so the resume is deliberately withheld — but the answer
|
|
954
|
+
itself must still reach the Human surface."""
|
|
955
|
+
_, received = self._classes()
|
|
956
|
+
crewai_event_bus.emit(None, CrewKickoffStartedEvent(crew_name="C", inputs=None))
|
|
957
|
+
crewai_event_bus.emit(
|
|
958
|
+
None,
|
|
959
|
+
received(
|
|
960
|
+
type="human_feedback_received",
|
|
961
|
+
flow_name="f",
|
|
962
|
+
method_name="m",
|
|
963
|
+
feedback="orphaned answer",
|
|
964
|
+
outcome=None,
|
|
965
|
+
),
|
|
966
|
+
)
|
|
967
|
+
crewai_event_bus.flush(timeout=30)
|
|
968
|
+
|
|
969
|
+
events = emitted()
|
|
970
|
+
kinds = types_of(events)
|
|
971
|
+
assert "human_input" in kinds
|
|
972
|
+
assert "agent_resume" not in kinds
|
|
973
|
+
answer = next(e for e in events if e["type"] == "human_input")
|
|
974
|
+
assert answer["response"] == "orphaned answer"
|
|
975
|
+
# NEVER None: `human_input` requires `input_id`, and passing None raises
|
|
976
|
+
# a TypeError inside the customer's event bus.
|
|
977
|
+
assert answer["input_id"]
|
|
978
|
+
assert answer["fw_orphaned"] is True
|
|
979
|
+
|
|
980
|
+
def test_every_hitl_event_lands_on_the_one_session(self, instrumented, emitted):
|
|
981
|
+
crewai_event_bus.emit(None, CrewKickoffStartedEvent(crew_name="C", inputs=None))
|
|
982
|
+
self._round_trip()
|
|
983
|
+
events = emitted()
|
|
984
|
+
assert len({e["session_id"] for e in events}) == 1
|
|
985
|
+
|
|
986
|
+
|
|
987
|
+
# ---------------------------------------------------------------------------
|
|
988
|
+
# Nesting: the spans CrewAI hangs real work underneath
|
|
989
|
+
# ---------------------------------------------------------------------------
|
|
990
|
+
|
|
991
|
+
def _flow_event(name, **kw):
|
|
992
|
+
from failproofai_sdk.integrations.crewai import event_class
|
|
993
|
+
|
|
994
|
+
klass = event_class(name)
|
|
995
|
+
if klass is None: # pragma: no cover - a crewai that dropped the class
|
|
996
|
+
pytest.skip(f"this crewai has no {name}")
|
|
997
|
+
return klass(**kw)
|
|
998
|
+
|
|
999
|
+
|
|
1000
|
+
class TestNesting:
|
|
1001
|
+
"""`_nodes` is what `_parent_key` resolves a `parent_event_id` against, and
|
|
1002
|
+
an id that is not in it falls back to the open ROOT. So every CrewAI span
|
|
1003
|
+
that can parent other spans has to be recorded there, including the two that
|
|
1004
|
+
are not agents:
|
|
1005
|
+
|
|
1006
|
+
* a `delegate_work_to_coworker` **tool call** parents the coworker's entire
|
|
1007
|
+
`AgentExecutionStartedEvent` (measured on 1.15.16: the coworker's
|
|
1008
|
+
`parent_event_id` IS the tool event's id), so a hierarchical crew's whole
|
|
1009
|
+
manager/coworker hierarchy collapses onto the crew without it;
|
|
1010
|
+
* a **flow method** parents a `Crew.kickoff()` made inside it, so the crew
|
|
1011
|
+
becomes a second root — a whole separate session — without it.
|
|
1012
|
+
|
|
1013
|
+
Neither failure raises, and both look plausible in the dashboard.
|
|
1014
|
+
"""
|
|
1015
|
+
|
|
1016
|
+
def test_a_delegated_coworker_hangs_off_the_delegating_agent(
|
|
1017
|
+
self, instrumented, emitted
|
|
1018
|
+
):
|
|
1019
|
+
crew = CrewKickoffStartedEvent(crew_name="Hierarchical Crew", inputs=None)
|
|
1020
|
+
crewai_event_bus.emit(None, crew)
|
|
1021
|
+
|
|
1022
|
+
manager = Agent(role="Crew Manager", goal="Delegate.", backstory="Manages.")
|
|
1023
|
+
worker = Agent(role="Researcher", goal="Research.", backstory="Researches.")
|
|
1024
|
+
task = Task(description="d", expected_output="o", agent=manager)
|
|
1025
|
+
|
|
1026
|
+
manager_span = _flow_event(
|
|
1027
|
+
"AgentExecutionStartedEvent",
|
|
1028
|
+
agent=manager,
|
|
1029
|
+
task=task,
|
|
1030
|
+
tools=[],
|
|
1031
|
+
task_prompt="p",
|
|
1032
|
+
parent_event_id=crew.event_id,
|
|
1033
|
+
)
|
|
1034
|
+
crewai_event_bus.emit(None, manager_span)
|
|
1035
|
+
# CrewAI's own executor flow, which is a pass-through link.
|
|
1036
|
+
executor = _flow_event(
|
|
1037
|
+
"FlowStartedEvent", flow_name="AgentExecutor", parent_event_id=manager_span.event_id
|
|
1038
|
+
)
|
|
1039
|
+
crewai_event_bus.emit(None, executor)
|
|
1040
|
+
delegate = ToolUsageStartedEvent(
|
|
1041
|
+
tool_name="delegate_work_to_coworker",
|
|
1042
|
+
tool_args="{}",
|
|
1043
|
+
agent_role="Crew Manager",
|
|
1044
|
+
parent_event_id=executor.event_id,
|
|
1045
|
+
)
|
|
1046
|
+
crewai_event_bus.emit(None, delegate)
|
|
1047
|
+
coworker_span = _flow_event(
|
|
1048
|
+
"AgentExecutionStartedEvent",
|
|
1049
|
+
agent=worker,
|
|
1050
|
+
task=task,
|
|
1051
|
+
tools=[],
|
|
1052
|
+
task_prompt="p",
|
|
1053
|
+
parent_event_id=delegate.event_id,
|
|
1054
|
+
)
|
|
1055
|
+
crewai_event_bus.emit(None, coworker_span)
|
|
1056
|
+
crewai_event_bus.flush(timeout=30)
|
|
1057
|
+
|
|
1058
|
+
starts = {e["agent_id"]: e for e in emitted() if e["type"] == "agent_start"}
|
|
1059
|
+
assert starts["Crew Manager"]["parent_id"] == "Hierarchical Crew"
|
|
1060
|
+
# The whole point: NOT "Hierarchical Crew".
|
|
1061
|
+
assert starts["Researcher"]["parent_id"] == "Crew Manager"
|
|
1062
|
+
|
|
1063
|
+
def test_a_crew_inside_a_flow_method_stays_in_one_session(self, instrumented, emitted):
|
|
1064
|
+
flow = _flow_event("FlowStartedEvent", flow_name="ReviewFlow")
|
|
1065
|
+
crewai_event_bus.emit(None, flow)
|
|
1066
|
+
method = _flow_event(
|
|
1067
|
+
"MethodExecutionStartedEvent",
|
|
1068
|
+
flow_name="ReviewFlow",
|
|
1069
|
+
method_name="run_crew",
|
|
1070
|
+
state={},
|
|
1071
|
+
parent_event_id=flow.event_id,
|
|
1072
|
+
)
|
|
1073
|
+
crewai_event_bus.emit(None, method)
|
|
1074
|
+
crewai_event_bus.emit(
|
|
1075
|
+
None,
|
|
1076
|
+
CrewKickoffStartedEvent(
|
|
1077
|
+
crew_name="Inner Crew", inputs=None, parent_event_id=method.event_id
|
|
1078
|
+
),
|
|
1079
|
+
)
|
|
1080
|
+
crewai_event_bus.flush(timeout=30)
|
|
1081
|
+
|
|
1082
|
+
events = emitted()
|
|
1083
|
+
# One session, not two. A detached crew mints its own session id and the
|
|
1084
|
+
# run silently becomes two runs.
|
|
1085
|
+
assert len({e["session_id"] for e in events}) == 1
|
|
1086
|
+
inner = next(
|
|
1087
|
+
e for e in events if e["type"] == "agent_start" and e["agent_id"] == "Inner Crew"
|
|
1088
|
+
)
|
|
1089
|
+
assert inner["parent_id"] == "ReviewFlow"
|
|
1090
|
+
|
|
1091
|
+
|
|
1092
|
+
class TestFlowFailure:
|
|
1093
|
+
"""A flow whose method raises emits `FlowFailedEvent` and never
|
|
1094
|
+
`FlowFinishedEvent`. Unmapped, the flow's `agent_start` is never closed and
|
|
1095
|
+
the session renders `ongoing` forever — the one outcome this adapter's
|
|
1096
|
+
fallbacks exist to avoid."""
|
|
1097
|
+
|
|
1098
|
+
def test_flow_failed_closes_the_span(self, instrumented, emitted):
|
|
1099
|
+
flow = _flow_event("FlowStartedEvent", flow_name="DoomedFlow")
|
|
1100
|
+
crewai_event_bus.emit(None, flow)
|
|
1101
|
+
crewai_event_bus.emit(
|
|
1102
|
+
None,
|
|
1103
|
+
_flow_event(
|
|
1104
|
+
"FlowFailedEvent",
|
|
1105
|
+
flow_name="DoomedFlow",
|
|
1106
|
+
error=RuntimeError("sink offline"),
|
|
1107
|
+
started_event_id=flow.event_id,
|
|
1108
|
+
parent_event_id=flow.event_id,
|
|
1109
|
+
),
|
|
1110
|
+
)
|
|
1111
|
+
crewai_event_bus.flush(timeout=30)
|
|
1112
|
+
|
|
1113
|
+
events = emitted()
|
|
1114
|
+
ends = [e for e in events if e["type"] == "agent_end"]
|
|
1115
|
+
assert [e["agent_id"] for e in ends] == ["DoomedFlow"]
|
|
1116
|
+
assert ends[0]["outcome"] == "failed"
|
|
1117
|
+
assert "sink offline" in ends[0]["fw_error"]
|
|
1118
|
+
|
|
1119
|
+
def test_flow_failed_is_in_the_table(self):
|
|
1120
|
+
assert ("FlowFailedEvent", "on_flow_failed") in TABLE
|
|
1121
|
+
|
|
1122
|
+
|
|
1123
|
+
class TestConcurrentRoots:
|
|
1124
|
+
"""`_roots` is process-global. With two crews open on two threads, an event
|
|
1125
|
+
whose parent span has already been popped resolves through `_roots[-1]` —
|
|
1126
|
+
a coin flip that files one run's events under the OTHER run's session id.
|
|
1127
|
+
That is silent cross-session corruption, strictly worse than a missing row."""
|
|
1128
|
+
|
|
1129
|
+
def test_an_orphan_does_not_land_in_the_other_runs_session(self, instrumented, emitted):
|
|
1130
|
+
# `restore_event_scope(())` between the two kickoffs is what a second
|
|
1131
|
+
# THREAD would give for free: without it crewai's scope stack makes the
|
|
1132
|
+
# second crew a child of the first and there is only ever one root, so
|
|
1133
|
+
# the bug this guards cannot be reached.
|
|
1134
|
+
with failproofai_sdk.session("session-alpha"):
|
|
1135
|
+
crewai_event_bus.emit(None, CrewKickoffStartedEvent(crew_name="Alpha", inputs=None))
|
|
1136
|
+
restore_event_scope(())
|
|
1137
|
+
with failproofai_sdk.session("session-bravo"):
|
|
1138
|
+
crewai_event_bus.emit(None, CrewKickoffStartedEvent(crew_name="Bravo", inputs=None))
|
|
1139
|
+
restore_event_scope(())
|
|
1140
|
+
crewai_event_bus.flush(timeout=30)
|
|
1141
|
+
assert len(adapter._roots) == 2, "the test needs two concurrent roots to mean anything"
|
|
1142
|
+
|
|
1143
|
+
with failproofai_sdk.session("session-alpha"):
|
|
1144
|
+
# Its parent span is gone (closed, or evicted at _MAX_NODES), so
|
|
1145
|
+
# this resolves through the root fallback.
|
|
1146
|
+
crewai_event_bus.emit(
|
|
1147
|
+
None,
|
|
1148
|
+
ToolUsageStartedEvent(
|
|
1149
|
+
tool_name="orphan_tool",
|
|
1150
|
+
tool_args="{}",
|
|
1151
|
+
agent_role="Alpha Worker",
|
|
1152
|
+
parent_event_id=str(uuid.uuid4()),
|
|
1153
|
+
),
|
|
1154
|
+
)
|
|
1155
|
+
crewai_event_bus.flush(timeout=30)
|
|
1156
|
+
|
|
1157
|
+
orphan = next(e for e in emitted() if e["type"] == "tool_use")
|
|
1158
|
+
# NOT "session-bravo": `_roots[-1]` is Bravo.
|
|
1159
|
+
assert orphan["session_id"] == "session-alpha"
|
|
1160
|
+
assert orphan["agent_id"] == "Alpha"
|
|
1161
|
+
|
|
1162
|
+
|
|
1163
|
+
class TestLiteAgent:
|
|
1164
|
+
"""`Agent.kickoff()` is an agent run with no Crew and no Task. It emits its
|
|
1165
|
+
OWN execution events, and it is a ROOT. Unmapped, the run has no agent span
|
|
1166
|
+
at all: its LLM and tool events fall through to whatever ambient scope
|
|
1167
|
+
exists (`agent_id` "main"), or are dropped outright when there is none."""
|
|
1168
|
+
|
|
1169
|
+
@staticmethod
|
|
1170
|
+
def _pair(role="Solo Agent"):
|
|
1171
|
+
info = {"id": str(uuid.uuid4()), "role": role, "goal": "Answer.", "backstory": "b"}
|
|
1172
|
+
started = _flow_event(
|
|
1173
|
+
"LiteAgentExecutionStartedEvent", agent_info=info, tools=[], messages="hi"
|
|
1174
|
+
)
|
|
1175
|
+
crewai_event_bus.emit(None, started)
|
|
1176
|
+
crewai_event_bus.emit(
|
|
1177
|
+
None,
|
|
1178
|
+
_flow_event(
|
|
1179
|
+
"LiteAgentExecutionCompletedEvent",
|
|
1180
|
+
agent_info=info,
|
|
1181
|
+
output="done",
|
|
1182
|
+
started_event_id=started.event_id,
|
|
1183
|
+
),
|
|
1184
|
+
)
|
|
1185
|
+
crewai_event_bus.flush(timeout=30)
|
|
1186
|
+
return info
|
|
1187
|
+
|
|
1188
|
+
def test_it_gets_its_own_agent_span(self, instrumented, emitted):
|
|
1189
|
+
info = self._pair()
|
|
1190
|
+
events = emitted()
|
|
1191
|
+
assert types_of(events) == ["agent_start", "agent_end"]
|
|
1192
|
+
assert events[0]["agent_id"] == "Solo Agent"
|
|
1193
|
+
assert events[0]["goal"] == "Answer."
|
|
1194
|
+
assert events[0]["fw_lite"] is True
|
|
1195
|
+
# The UUID goes to fw_agent_id and NEVER to the LowCardinality facet.
|
|
1196
|
+
assert events[0]["fw_agent_id"] == info["id"]
|
|
1197
|
+
assert events[1]["outcome"] == "success"
|
|
1198
|
+
|
|
1199
|
+
def test_it_is_a_root_so_it_opens_its_own_session(self, instrumented, emitted):
|
|
1200
|
+
self._pair()
|
|
1201
|
+
events = emitted()
|
|
1202
|
+
assert events[0].get("parent_id") is None
|
|
1203
|
+
assert len({e["session_id"] for e in events}) == 1
|
|
1204
|
+
|
|
1205
|
+
def test_the_three_rows_are_in_the_table(self):
|
|
1206
|
+
mapped = dict(TABLE)
|
|
1207
|
+
assert mapped["LiteAgentExecutionStartedEvent"] == "on_lite_agent_started"
|
|
1208
|
+
assert mapped["LiteAgentExecutionCompletedEvent"] == "on_agent_completed"
|
|
1209
|
+
assert mapped["LiteAgentExecutionErrorEvent"] == "on_agent_error"
|
|
1210
|
+
|
|
1211
|
+
|
|
1212
|
+
# ---------------------------------------------------------------------------
|
|
1213
|
+
# Task(human_input=True) — the HITL surface that is NOT on the event bus
|
|
1214
|
+
# ---------------------------------------------------------------------------
|
|
1215
|
+
|
|
1216
|
+
def _provider():
|
|
1217
|
+
try:
|
|
1218
|
+
from crewai.core.providers.human_input import SyncHumanInputProvider
|
|
1219
|
+
except ImportError: # pragma: no cover - a crewai that moved it
|
|
1220
|
+
pytest.skip("this crewai has no SyncHumanInputProvider")
|
|
1221
|
+
return SyncHumanInputProvider
|
|
1222
|
+
|
|
1223
|
+
|
|
1224
|
+
class TestTaskHumanInput:
|
|
1225
|
+
"""crewai has two HITL surfaces and only one of them is on the bus.
|
|
1226
|
+
`human_input=True` on a Task runs through
|
|
1227
|
+
`crewai.core.providers.human_input`, which calls `input()` and emits no
|
|
1228
|
+
event of any kind — so the entire human wait is billed as active time on the
|
|
1229
|
+
agent unless this seam is wrapped. It is the only patch in the adapter,
|
|
1230
|
+
which is why install/restore is asserted as hard as the events are.
|
|
1231
|
+
"""
|
|
1232
|
+
|
|
1233
|
+
def test_the_four_events_land_on_the_blocked_agent(
|
|
1234
|
+
self, instrumented, emitted, monkeypatch
|
|
1235
|
+
):
|
|
1236
|
+
crewai_event_bus.emit(None, CrewKickoffStartedEvent(crew_name="C", inputs=None))
|
|
1237
|
+
agent = Agent(role="Blocked Analyst", goal="g", backstory="b")
|
|
1238
|
+
task = Task(description="d", expected_output="o", agent=agent)
|
|
1239
|
+
crewai_event_bus.emit(
|
|
1240
|
+
None,
|
|
1241
|
+
_flow_event(
|
|
1242
|
+
"AgentExecutionStartedEvent",
|
|
1243
|
+
agent=agent,
|
|
1244
|
+
task=task,
|
|
1245
|
+
tools=[],
|
|
1246
|
+
task_prompt="p",
|
|
1247
|
+
),
|
|
1248
|
+
)
|
|
1249
|
+
crewai_event_bus.flush(timeout=30)
|
|
1250
|
+
|
|
1251
|
+
monkeypatch.setattr("builtins.input", lambda *a: "tighten the wording")
|
|
1252
|
+
assert _provider()._prompt_input(None) == "tighten the wording"
|
|
1253
|
+
|
|
1254
|
+
events = emitted()
|
|
1255
|
+
kinds = types_of(events)
|
|
1256
|
+
for expected in ("human_wait", "agent_pause", "agent_resume", "human_input"):
|
|
1257
|
+
assert expected in kinds, f"{expected} missing from {kinds}"
|
|
1258
|
+
assert kinds.index("human_wait") < kinds.index("agent_pause")
|
|
1259
|
+
assert kinds.index("agent_pause") < kinds.index("agent_resume")
|
|
1260
|
+
assert kinds.index("agent_resume") < kinds.index("human_input")
|
|
1261
|
+
|
|
1262
|
+
by_type = {e["type"]: e for e in events}
|
|
1263
|
+
# On the agent that is actually blocked, not on the crew above it.
|
|
1264
|
+
assert by_type["human_wait"]["agent_id"] == "Blocked Analyst"
|
|
1265
|
+
assert by_type["human_input"]["response"] == "tighten the wording"
|
|
1266
|
+
assert by_type["agent_pause"]["pause_id"] == by_type["agent_resume"]["pause_id"]
|
|
1267
|
+
assert by_type["human_wait"]["fw_surface"] == "task_human_input"
|
|
1268
|
+
|
|
1269
|
+
def test_the_seam_is_restored_on_uninstrument(self):
|
|
1270
|
+
provider = _provider()
|
|
1271
|
+
before = provider.__dict__.get("_prompt_input")
|
|
1272
|
+
before_async = provider.__dict__.get("_prompt_input_async")
|
|
1273
|
+
failproofai_sdk.instrument("crewai")
|
|
1274
|
+
assert provider.__dict__.get("_prompt_input") is not before
|
|
1275
|
+
failproofai_sdk.uninstrument("crewai")
|
|
1276
|
+
assert provider.__dict__.get("_prompt_input") is before
|
|
1277
|
+
assert provider.__dict__.get("_prompt_input_async") is before_async
|
|
1278
|
+
|
|
1279
|
+
def test_installing_twice_does_not_wrap_twice(self, instrumented):
|
|
1280
|
+
# A second install with no restore in between would emit the pause twice
|
|
1281
|
+
# for one prompt.
|
|
1282
|
+
adapter._patch_task_human_input()
|
|
1283
|
+
assert len(adapter._patched) == 2
|
|
1284
|
+
|
|
1285
|
+
def test_it_is_still_a_staticmethod(self, instrumented):
|
|
1286
|
+
# `self._prompt_input(context.crew)` is the call site: a bare function
|
|
1287
|
+
# here would bind and arrive with `self` where `crew` belongs.
|
|
1288
|
+
assert isinstance(_provider().__dict__["_prompt_input"], staticmethod)
|
|
1289
|
+
|
|
1290
|
+
def test_a_raising_prompt_still_raises(self, instrumented, monkeypatch):
|
|
1291
|
+
boom = KeyboardInterrupt()
|
|
1292
|
+
|
|
1293
|
+
def _raise(*args):
|
|
1294
|
+
raise boom
|
|
1295
|
+
|
|
1296
|
+
monkeypatch.setattr("builtins.input", _raise)
|
|
1297
|
+
with pytest.raises(KeyboardInterrupt) as excinfo:
|
|
1298
|
+
_provider()._prompt_input(None)
|
|
1299
|
+
assert excinfo.value is boom
|